Add support for adding release info into adafruit/circuitpython-org
This also changes the build script to python with better output.crypto-aes
Scott Shawcroft
4 years ago
parent
ab94344bae
commit
9d07e95351
@ -0,0 +1 @@
|
||||
build-*/
|
||||
@ -0,0 +1 @@
|
||||
Subproject commit 393c2756714b5cccf028a82c23b873d36b2d9e8b
|
||||
@ -0,0 +1,70 @@
|
||||
import os
|
||||
from sh import rm
|
||||
from sh import make
|
||||
import sys
|
||||
import subprocess
|
||||
import shutil
|
||||
import build_board_info as build_info
|
||||
import time
|
||||
|
||||
for port in build_info.SUPPORTED_PORTS:
|
||||
rm("-rf", "../ports/{}/build*".format(port))
|
||||
|
||||
ROSIE_SETUPS = ["rosie-ci"]
|
||||
rosie_ok = {}
|
||||
for rosie in ROSIE_SETUPS:
|
||||
rosie_ok[rosie] = True
|
||||
|
||||
PARALLEL = "-j 5"
|
||||
travis = False
|
||||
if "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true":
|
||||
PARALLEL="-j 2"
|
||||
travis = True
|
||||
|
||||
all_boards = build_info.get_board_mapping()
|
||||
build_boards = list(all_boards.keys())
|
||||
if "TRAVIS_BOARDS" in os.environ:
|
||||
build_boards = os.environ["TRAVIS_BOARDS"].split()
|
||||
|
||||
sha, version = build_info.get_version_info()
|
||||
|
||||
languages = build_info.get_languages()
|
||||
exit_status = 0
|
||||
for board in build_boards:
|
||||
bin_directory = "../bin/{}/".format(board)
|
||||
os.makedirs(bin_directory, exist_ok=True)
|
||||
board_info = all_boards[board]
|
||||
|
||||
for language in languages:
|
||||
start_time = time.monotonic()
|
||||
make_result = subprocess.run("make -C ../ports/" + board_info["port"] + " TRANSLATION=" + language + " BOARD=" + board, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
build_duration = time.monotonic() - start_time
|
||||
success = "\033[32msucceeded\033[0m"
|
||||
if make_result.returncode != 0:
|
||||
exit_status = make_result.returncode
|
||||
success = "\033[31mfailed\033[0m"
|
||||
|
||||
for extension in board_info["extensions"]:
|
||||
temp_filename = "../ports/{port}/build-{board}/firmware.{extension}".format(port=board_info["port"], board=board, extension=extension)
|
||||
final_filename = "adafruit-circuitpython-{board}-{language}-{version}.{extension}".format(board=board, language=language, version=version, extension=extension)
|
||||
final_filename = os.path.join(bin_directory, final_filename)
|
||||
shutil.copyfile(temp_filename, final_filename)
|
||||
|
||||
if travis:
|
||||
print('travis_fold:start:adafruit-bins-{}-{}\\r'.format(language, board))
|
||||
print("Build {} for {} took {:.2f}s and {}".format(board, language, build_duration, success))
|
||||
print(len(make_result.stdout))
|
||||
print(make_result.stdout.decode("utf-8"))
|
||||
# Only upload to Rosie if its a pull request.
|
||||
if travis:
|
||||
for rosie in ROSIE_SETUPS:
|
||||
if not rosie_ok[rosie]:
|
||||
break
|
||||
print("Uploading to https://{rosie}.ngrok.io/upload/{sha}".format(rosie=rosie, sha=sha))
|
||||
#curl -F "file=@$final_filename" https://$rosie.ngrok.io/upload/$sha
|
||||
if travis:
|
||||
print('travis_fold:end:adafruit-bins-{}-{}\\r'.format(language, board))
|
||||
|
||||
print()
|
||||
|
||||
sys.exit(exit_status)
|
||||
@ -1,116 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
rm -rf ports/atmel-samd/build*
|
||||
rm -rf ports/esp8266/build*
|
||||
rm -rf ports/nrf/build*
|
||||
|
||||
# Alphabetical.
|
||||
HW_BOARDS="\
|
||||
arduino_mkr1300 \
|
||||
arduino_zero \
|
||||
catwan_usbstick \
|
||||
circuitplayground_express \
|
||||
circuitplayground_express_crickit \
|
||||
feather_huzzah \
|
||||
feather_m0_adalogger \
|
||||
feather_m0_basic \
|
||||
feather_m0_express \
|
||||
feather_m0_express_crickit \
|
||||
feather_m0_rfm69 \
|
||||
feather_m0_rfm9x \
|
||||
feather_m4_express \
|
||||
feather_nrf52832 \
|
||||
feather_nrf52840_express \
|
||||
feather_radiofruit_zigbee \
|
||||
gemma_m0 \
|
||||
grandcentral_m4_express \
|
||||
hallowing_m0_express \
|
||||
itsybitsy_m0_express \
|
||||
itsybitsy_m4_express \
|
||||
makerdiary_nrf52840_mdk \
|
||||
metro_m0_express \
|
||||
metro_m4_express \
|
||||
meowmeow \
|
||||
pca10056 \
|
||||
pca10059 \
|
||||
pirkey_m0 \
|
||||
trellis_m4_express \
|
||||
trinket_m0 \
|
||||
"
|
||||
ROSIE_SETUPS="rosie-ci"
|
||||
|
||||
PARALLEL="-j 5"
|
||||
if [[ "$TRAVIS" == "true" ]]; then
|
||||
PARALLEL="-j 2"
|
||||
fi
|
||||
|
||||
if [[ -z "$TRAVIS_BOARD" ]]; then
|
||||
boards=$HW_BOARDS
|
||||
else
|
||||
boards=$TRAVIS_BOARD
|
||||
fi
|
||||
|
||||
version=`git describe --tags --exact-match`
|
||||
if [[ $? -ne 0 ]]; then
|
||||
version=`date +%Y%m%d`-`git rev-parse --short HEAD`
|
||||
fi
|
||||
|
||||
|
||||
if [[ "$TRAVIS" == "true" ]]; then
|
||||
sha=$TRAVIS_COMMIT
|
||||
if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then
|
||||
version=`date +%Y%m%d`-`echo $TRAVIS_PULL_REQUEST_SHA | cut -c1-7`
|
||||
sha=$TRAVIS_PULL_REQUEST_SHA
|
||||
fi
|
||||
fi
|
||||
|
||||
for board in $boards; do
|
||||
mkdir -p bin/$board/
|
||||
extension=uf2
|
||||
for language_file in $(ls locale/*.po); do
|
||||
language=$(basename -s .po $language_file)
|
||||
echo "Building $board for $language"
|
||||
# There is a bug in the Huzzah Makefile that causes it to fail occasionally with -j > 1.
|
||||
if [[ $board == "feather_huzzah" ]]; then
|
||||
make -C ports/esp8266 TRANSLATION=$language BOARD=$board
|
||||
(( exit_status = exit_status || $? ))
|
||||
temp_filename=ports/esp8266/build/firmware-combined.bin
|
||||
extension=bin
|
||||
elif [[ $board == "feather_nrf52832" ]]; then
|
||||
make $PARALLEL -C ports/nrf TRANSLATION=$language BOARD=$board
|
||||
(( exit_status = exit_status || $? ))
|
||||
temp_filename=ports/nrf/build-$board-s132/firmware.bin
|
||||
extension=bin
|
||||
elif [[ $board == "feather_nrf52840_express" || $board == "pca10056" || $board == "pca10059" ]]; then
|
||||
make $PARALLEL -C ports/nrf TRANSLATION=$language BOARD=$board SD=s140
|
||||
(( exit_status = exit_status || $? ))
|
||||
temp_filename=ports/nrf/build-$board-s140/firmware.uf2
|
||||
extension=uf2
|
||||
elif [[ $board == "makerdiary_nrf52840_mdk" ]]; then
|
||||
make $PARALLEL -C ports/nrf TRANSLATION=$language BOARD=$board SD=s140
|
||||
(( exit_status = exit_status || $? ))
|
||||
temp_filename=ports/nrf/build-$board-s140/firmware.hex
|
||||
extension=hex
|
||||
else
|
||||
time make $PARALLEL -C ports/atmel-samd TRANSLATION=$language BOARD=$board
|
||||
(( exit_status = exit_status || $? ))
|
||||
cp ports/atmel-samd/build-$board/firmware.bin bin/$board/adafruit-circuitpython-$board-$language-$version.bin
|
||||
(( exit_status = exit_status || $? ))
|
||||
temp_filename=ports/atmel-samd/build-$board/firmware.uf2
|
||||
extension=uf2
|
||||
fi
|
||||
final_filename=bin/$board/adafruit-circuitpython-$board-$language-$version.$extension
|
||||
cp $temp_filename $final_filename
|
||||
(( exit_status = exit_status || $? ))
|
||||
# Only upload to Rosie if its a pull request.
|
||||
if [[ "$TRAVIS" == "true" ]]; then
|
||||
for rosie in $ROSIE_SETUPS; do
|
||||
echo "Uploading to https://$rosie.ngrok.io/upload/$sha"
|
||||
curl -F "file=@$final_filename" https://$rosie.ngrok.io/upload/$sha
|
||||
done
|
||||
fi
|
||||
echo
|
||||
done
|
||||
done
|
||||
|
||||
exit $exit_status
|
||||
@ -0,0 +1,243 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import sh
|
||||
import base64
|
||||
from datetime import date
|
||||
from sh.contrib import git
|
||||
|
||||
sys.path.append("adabot")
|
||||
import adabot.github_requests as github
|
||||
|
||||
SUPPORTED_PORTS = ["nrf", "esp8266", "atmel-samd"]
|
||||
|
||||
BIN = ('bin',)
|
||||
UF2 = ('uf2',)
|
||||
BIN_UF2 = ('bin', 'uf2')
|
||||
HEX = ('hex',)
|
||||
|
||||
# Default extensions
|
||||
extension_by_port = {
|
||||
"nrf": BIN,
|
||||
"esp8266": BIN,
|
||||
"atmel-samd": UF2,
|
||||
}
|
||||
|
||||
# Per board overrides
|
||||
extension_by_board = {
|
||||
"feather_nrf52832": BIN,
|
||||
"arduino_zero": BIN,
|
||||
"feather_m0_adalogger": BIN_UF2,
|
||||
"feather_m0_basic": BIN_UF2,
|
||||
"makerdiary_nrf52840_mdk": HEX
|
||||
}
|
||||
|
||||
def get_languages():
|
||||
languages = []
|
||||
for f in os.scandir("../locale"):
|
||||
if f.name.endswith(".po"):
|
||||
languages.append(f.name[:-3])
|
||||
return languages
|
||||
|
||||
def get_board_mapping():
|
||||
boards = {}
|
||||
for port in SUPPORTED_PORTS:
|
||||
board_path = os.path.join("../ports", port, "boards")
|
||||
for board_path in os.scandir(board_path):
|
||||
if board_path.is_dir():
|
||||
board_files = os.listdir(board_path.path)
|
||||
board_id = board_path.name
|
||||
extensions = extension_by_port[port]
|
||||
extensions = extension_by_board.get(board_path.name, extensions)
|
||||
boards[board_id] = {"port": port, "extensions": extensions, "download_count": 0}
|
||||
return boards
|
||||
|
||||
def get_version_info():
|
||||
version = None
|
||||
sha = git("rev-parse", "--short", "HEAD").stdout.decode("utf-8")
|
||||
try:
|
||||
version = git("describe", "--tags", "--exact-match").stdout.decode("utf-8")
|
||||
except sh.ErrorReturnCode_128:
|
||||
# No exact match
|
||||
pass
|
||||
|
||||
if "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true":
|
||||
sha = os.environ["TRAVIS_COMMIT"]
|
||||
if os.environ["TRAVIS_PULL_REQUEST"] != "false":
|
||||
sha = os.environ["TRAVIS_PULL_REQUEST_SHA"]
|
||||
|
||||
if not version:
|
||||
version="{}-{}".format(date.today().strftime("%Y%m%d"), sha[:7])
|
||||
|
||||
return sha, version
|
||||
|
||||
def get_current_info():
|
||||
response = github.get("/repos/adafruit/circuitpython-org/git/refs/heads/master")
|
||||
if not response.ok:
|
||||
print(response.text)
|
||||
raise RuntimeError("cannot get master sha")
|
||||
commit_sha = response.json()["object"]["sha"]
|
||||
|
||||
response = github.get("/repos/adafruit/circuitpython-org/contents/_data/files.json?ref=" + commit_sha)
|
||||
if not response.ok:
|
||||
print(response.text)
|
||||
raise RuntimeError("cannot get previous files.json")
|
||||
|
||||
response = response.json()
|
||||
|
||||
git_info = commit_sha, response["sha"]
|
||||
return git_info, json.loads(base64.b64decode(response["content"]))
|
||||
|
||||
def create_pr(changes, updated, git_info):
|
||||
commit_sha, original_blob_sha = git_info
|
||||
branch_name = "new_release_" + changes["new_release"]
|
||||
|
||||
updated = json.dumps(updated, sort_keys=True, indent=4).encode("utf-8") + b"\n"
|
||||
print(updated.decode("utf-8"))
|
||||
pr_title = "Automated website update for release {}".format(changes["new_release"])
|
||||
boards = ""
|
||||
if changes["new_boards"]:
|
||||
boards = "New boards:\n* " + "\n* ".join(changes["new_boards"])
|
||||
languages = ""
|
||||
if changes["new_languages"]:
|
||||
languages = "New languages:\n* " + "\n* ".join(changes["new_languages"])
|
||||
message = "Automated website update for release {} by AdaBot.\n\n{}\n\n{}\n".format(
|
||||
changes["new_release"],
|
||||
boards,
|
||||
languages
|
||||
)
|
||||
|
||||
create_branch = {
|
||||
"ref": "refs/heads/" + branch_name,
|
||||
"sha": commit_sha
|
||||
}
|
||||
response = github.post("/repos/adafruit-adabot/circuitpython-org/git/refs", json=create_branch)
|
||||
if not response.ok and response.json()["message"] != "Reference already exists":
|
||||
print("unable to create branch")
|
||||
print(response.text)
|
||||
return
|
||||
|
||||
update_file = {
|
||||
"message": message,
|
||||
"content": base64.b64encode(updated).decode("utf-8"),
|
||||
"sha": original_blob_sha,
|
||||
"branch": branch_name
|
||||
}
|
||||
|
||||
response = github.put("/repos/adafruit-adabot/circuitpython-org/contents/_data/files.json", json=update_file)
|
||||
if not response.ok:
|
||||
print("unable to post new file")
|
||||
print(response.text)
|
||||
return
|
||||
pr_info = {
|
||||
"title": pr_title,
|
||||
"head": "adafruit-adabot:" + branch_name,
|
||||
"base": "master",
|
||||
"body": message,
|
||||
"maintainer_can_modify": True
|
||||
}
|
||||
response = github.post("/repos/adafruit/circuitpython-org/pulls", json=pr_info)
|
||||
if not response.ok:
|
||||
print("unable to create pr")
|
||||
print(response.text)
|
||||
return
|
||||
print(changes)
|
||||
print(pr_info)
|
||||
|
||||
def update_downloads(boards, release):
|
||||
response = github.get("/repos/adafruit/circuitpython/releases/tags/{}".format(release))
|
||||
if not response.ok:
|
||||
print(response.text)
|
||||
raise RuntimeError("cannot get previous release info")
|
||||
|
||||
assets = response.json()["assets"]
|
||||
for asset in assets:
|
||||
boards[asset["name"].split("-")[2]]["download_count"] += asset["download_count"]
|
||||
|
||||
|
||||
def print_active_user():
|
||||
response = github.get("/user")
|
||||
if response.ok:
|
||||
print("Logged in as {}".format(response.json()["login"]))
|
||||
else:
|
||||
print("Not logged in")
|
||||
|
||||
def generate_download_info():
|
||||
boards = {}
|
||||
errors = []
|
||||
|
||||
new_tag = os.environ.get("TRAVIS_TAG", "4.0.0-alpha.3")
|
||||
|
||||
changes = {
|
||||
"new_release": new_tag,
|
||||
"new_boards": [],
|
||||
"new_languages": []
|
||||
}
|
||||
|
||||
print_active_user()
|
||||
|
||||
sha, this_version = get_version_info()
|
||||
|
||||
git_info, current_info = get_current_info()
|
||||
|
||||
languages = get_languages()
|
||||
|
||||
new_stable = "-" not in new_tag
|
||||
|
||||
previous_releases = set()
|
||||
previous_languages = set()
|
||||
|
||||
# Delete the release we are replacing
|
||||
for board in current_info:
|
||||
info = current_info[board]
|
||||
for version in info["versions"]:
|
||||
previous_releases.add(version["version"])
|
||||
previous_languages.update(version["files"].keys())
|
||||
if version["stable"] == new_stable:
|
||||
info["versions"].remove(version)
|
||||
|
||||
board_mapping = get_board_mapping()
|
||||
|
||||
print(previous_releases)
|
||||
for release in previous_releases:
|
||||
update_downloads(board_mapping, release)
|
||||
|
||||
for port in SUPPORTED_PORTS:
|
||||
board_path = os.path.join("../ports", port, "boards")
|
||||
for board_path in os.scandir(board_path):
|
||||
if board_path.is_dir():
|
||||
board_files = os.listdir(board_path)
|
||||
board_id = board_path.name
|
||||
board_info = board_mapping[board_id]
|
||||
|
||||
if board_id not in current_info:
|
||||
changes["new_boards"].append(board_id)
|
||||
current_info[board_id] = {"downloads": 0,
|
||||
"versions": []}
|
||||
|
||||
new_version = {
|
||||
"stable": new_stable,
|
||||
"version": new_tag,
|
||||
"files": {}
|
||||
}
|
||||
for language in languages:
|
||||
files = []
|
||||
new_version["files"][language] = files
|
||||
for extension in board_info["extensions"]:
|
||||
files.append("https://github.com/adafruit/circuitpython/releases/download/{tag}/adafruit-circuitpython-{board}-{language}-{tag}.{extension}".format(tag=new_tag, board=board_id, language=language, extension=extension))
|
||||
current_info[board_id]["downloads"] = board_info["download_count"]
|
||||
current_info[board_id]["versions"].append(new_version)
|
||||
|
||||
changes["new_languages"] = set(languages) - previous_languages
|
||||
|
||||
if changes["new_languages"]:
|
||||
create_pr(changes, current_info, git_info)
|
||||
|
||||
if __name__ == "__main__":
|
||||
if "TRAVIS_TAG" in os.environ:
|
||||
generate_download_info()
|
||||
else:
|
||||
print("skipping website update because this isn't a tag")
|
||||
Loading…
Reference in New Issue