diff --git a/.github/workflows/_get_app_metadata.yml b/.github/workflows/_get_app_metadata.yml index c37a42a..5f2083a 100644 --- a/.github/workflows/_get_app_metadata.yml +++ b/.github/workflows/_get_app_metadata.yml @@ -77,7 +77,7 @@ jobs: then >&2 echo "/!\ No $APP_MANIFEST manifest detected!" >&2 echo "This file is mandatory, please add it on your repository" - >&2 echo "Documentation here: https://github.com/LedgerHQ/ledgered/blob/master/doc/utils/manifest.md" + >&2 echo "Documentation here: https://github.com/LedgerHQ/ledgered/blob/master/doc/manifest.md" exit 1 fi diff --git a/.github/workflows/reusable_build_several.yml b/.github/workflows/reusable_build_several.yml new file mode 100644 index 0000000..e6d27f4 --- /dev/null +++ b/.github/workflows/reusable_build_several.yml @@ -0,0 +1,49 @@ +name: Building several apps, depending on given inputs + +on: + workflow_call: + inputs: + sdk: + description: | + The type of SDK the apps use. Could be `c`, `rust` or `all` (defaults to `all`) + required: false + default: 'all' + type: string + sdk_reference: + description: | + Commit of the SDK to checkout. + If not specified, use the current app-builder SDK versions + required: false + default: 'None' + type: string + devices: + description: | + The list of device(s) the CI will run on. If an app does not implement a device, it will be ignored. + Defaults to the all the devices. + required: false + default: 'all' + type: string + +jobs: + + call_get_apps_metadata: + name: Rertieve applications metadata + uses: ./.github/workflows/reusable_get_apps_metadata.yml + with: + sdk: ${{ inputs.sdk }} + devices: ${{ inputs.devices }} + + build_applications: + name: Build all selected applications using the reusable workflow + needs: call_get_apps_metadata + strategy: + fail-fast: false + matrix: + app: ${{ fromJSON(needs.call_get_apps_metadata.outputs.apps_metadata) }} + uses: ./.github/workflows/reusable_build.yml + with: + app_repository: ${{ matrix.app.repository }} + run_for_devices: ${{ matrix.app.devices }} + upload_app_binaries_artifact: ${{ matrix.app.name }} # else will colide with other matrix' jobs + app_branch_name: ${{ matrix.app.default_branch }} # else takes the current repo default branch + builder: ledger-app-builder # needed for Rust applications diff --git a/.github/workflows/reusable_get_apps_metadata.yml b/.github/workflows/reusable_get_apps_metadata.yml new file mode 100644 index 0000000..4ae0b22 --- /dev/null +++ b/.github/workflows/reusable_get_apps_metadata.yml @@ -0,0 +1,43 @@ +name: Get apps metadatas + +on: + workflow_call: + inputs: + sdk: + description: | + The type of SDK the apps use. Could be `c`, `rust` or `all` (defaults to `all`) + required: false + default: 'all' + type: string + devices: + description: | + The list of device(s) the CI will run on. If an app does not implement a device, it will be ignored. + Defaults to the all the devices. + + As GitHub does not support (yet?) array as an input, this field must be filled as a string, with escaped + quotation marks. For instance: "[\"nanos+\", \"nanox\", \"stax\"]" + required: false + default: 'all' + type: string + outputs: + apps_metadata: + description: | + The metadata of all apps matching the given filters. The format will be: + [ { "default_branch": "main", "devices": "[\"nanos\", \"nanos+\", \"nanox\", \"flex\", \"stax\"]", "name": "app-boilerplate, "repository": "LedgerHQ/app-boilerplate }, ... ] + value: ${{ jobs.define_matrix.outputs.apps_config }} + +jobs: + + define_matrix: + name: Generate applications list for each device according to manifests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install dependencies + run: | + pip install ledgered + - name: Define the list of application to build by device + id: process_devices + run: | + devices=$(python ./script/parse_all_apps.py -s ${{ inputs.sdk }} -d ${{ inputs.devices }} -t ${{ secrets.GITHUB_TOKEN }} -l 10) + echo "apps_config=$devices" >> $GITHUB_OUTPUT diff --git a/scripts/parse_all_apps.py b/scripts/parse_all_apps.py new file mode 100644 index 0000000..3ce1929 --- /dev/null +++ b/scripts/parse_all_apps.py @@ -0,0 +1,100 @@ +import json +import logging +from argparse import ArgumentParser +from collections import defaultdict +from dataclasses import asdict, dataclass +from ledgered.github import AppRepository, Condition, GitHubLedgerHQ, GitHubApps, \ + NoManifestException + +LOGGER_FORMAT = "[%(asctime)s][%(levelname)s] %(name)s - %(message)s" +logging.root.handlers.clear() +handler = logging.StreamHandler() +handler.setFormatter(logging.Formatter(LOGGER_FORMAT)) +logging.root.addHandler(handler) +logging.root.setLevel(logging.INFO) + +devices = ["nanos", "nanos+", "nanox", "flex", "stax"] + + +@dataclass +class AppInfo: + default_branch: str + devices: list[str] + name: str + repository: str + + def __init__(self, app: AppRepository, filtered_devices: set[str]): + self.default_branch = app.default_branch + compatible_devices = app.manifest.app.devices + self.devices = '["' + '", "'.join(compatible_devices.intersection(filtered_devices)) + '"]' + self.name = app.name + self.repository = f"LedgerHQ/{app.name}" + + +def arg_parse(): + parser = ArgumentParser() + parser.add_argument("-d", "--devices", nargs="+", required=True, choices=["nanos", "nanosp", "nanos+", "nanox", "flex", "stax", "all"], + help="Devices to filter on. Accepts several successive values (separated with space). " + "Valid values are 'nanos', 'nanosp', 'nanos+', 'nanox', 'stax', 'flex', 'all'.") + parser.add_argument("-l", "--limit", required=False, default=0, type=int, + help="Limit the number of application to parse (testing purpose for instance)") + parser.add_argument("-s", "--sdk", required=False, default="all", type=str, choices=["C", "c", "Rust", "rust", "all"], + help="SDK to filter on. Only apps using the SDK are selected. Defaults to 'all'") + parser.add_argument("-t", "--github_token", required=False, default="", type=str, + help="A GitHub token to avoid GH API limitation") + + args = parser.parse_args() + + selected_devices = list() + for d in args.devices: + if d.lower() == "nanosp": + d = "nanos+" + if d.lower() in devices: + selected_devices.append(d) + continue + if d.lower() == "all": + selected_devices = devices + break + logging.warning(f"Unknown device target '{d}'. Ignoring.") + args.devices = selected_devices + + return args + + +def main(): + args = arg_parse() + + selected_devices = set(args.devices) + + logging.info("Fetching application repositories from GitHub") + if args.github_token: + gh = GitHubLedgerHQ(args.github_token) + else: + gh = GitHubLedgerHQ() + apps = gh.apps.filter(archived=Condition.WITHOUT, private=Condition.WITHOUT) + + selected_sdk: list[str] + if args.sdk.lower() == "all": + selected_sdk = ["c", "rust"] + else: + selected_sdk = [args.sdk.lower()] + + selected_apps = list() + for index, app in enumerate(apps): + logging.info(f"Managing app '{app.name}'") + try: + if not app.manifest.app.sdk in selected_sdk: + logging.debug(f"Wrong SDK, ignoring this app") + continue + selected_apps.append(asdict(AppInfo(app, selected_devices))) + except NoManifestException: + logging.warning(f"Application '{app.name}' has no manifest! Ignoring.") + + if args.limit != 0 and index > args.limit: + break + + print(json.dumps(selected_apps)) + + +if __name__ == "__main__": + main()