diff --git a/.github/changelogs.py b/.github/changelogs.py new file mode 100644 index 00000000000..8d72d3b3d3d --- /dev/null +++ b/.github/changelogs.py @@ -0,0 +1,454 @@ +from itertools import product +import subprocess +import json +import time +from typing import Any +import re +from collections import defaultdict + +REGISTRY = "docker://ghcr.io/ublue-os/" + +IMAGE_MATRIX_LATEST = { + "experience": ["base", "dx"], + "de": ["kde", "gnome"], + "image_flavor": ["main", "nvidia", "hwe", "hwe-nvidia"], +} +IMAGE_MATRIX = { + "experience": ["base", "dx"], + "de": ["kde", "gnome"], + "image_flavor": ["main", "nvidia"], +} + +RETRIES = 3 +RETRY_WAIT = 5 +FEDORA_PATTERN = re.compile(r"\.fc\d\d") +START_PATTERN = lambda target: re.compile(rf"{target}-[0-9]+") + +PATTERN_ADD = "\n| ✨ | {name} | | {version} |" +PATTERN_CHANGE = "\n| 🔄 | {name} | {prev} | {new} |" +PATTERN_REMOVE = "\n| ❌ | {name} | {version} | |" +PATTERN_PKGREL_CHANGED = "{prev} ➡️ {new}" +PATTERN_PKGREL = "{version}" +COMMON_PAT = "### All Images\n| | Name | Previous | New |\n| --- | --- | --- | --- |{changes}\n\n" +OTHER_NAMES = { + "base": "### Base Images\n| | Name | Previous | New |\n| --- | --- | --- | --- |{changes}\n\n", + "dx": "### [Dev Experience Images](https://docs.projectbluefin.io/bluefin-dx)\n| | Name | Previous | New |\n| --- | --- | --- | --- |{changes}\n\n", + "kde": "### [Aurora Images](https://getaurora.dev/)\n| | Name | Previous | New |\n| --- | --- | --- | --- |{changes}\n\n", + "gnome": "### [Bluefin Images](https://projectbluefin.io/)\n| | Name | Previous | New |\n| --- | --- | --- | --- |{changes}\n\n", + "nvidia": "### Nvidia Images\n| | Name | Previous | New |\n| --- | --- | --- | --- |{changes}\n\n", + "hwe": "### HWE Images\n| | Name | Previous | New |\n| --- | --- | --- | --- |{changes}\n\n", +} + +COMMITS_FORMAT = "### Commits\n| Hash | Subject |\n| --- | --- |{commits}\n\n" +COMMIT_FORMAT = "\n| **[{short}](https://github.com/ublue-os/bluefin/commit/{hash})** | {subject} |" + +CHANGELOG_TITLE = "{tag}: {pretty}" +CHANGELOG_FORMAT = """\ +{handwritten} + +From previous `{target}` version `{prev}` there have been the following changes. **One package per new version shown.** + +### Major packages +| Name | Version | +| --- | --- | +| **Kernel** | {pkgrel:kernel} | +| **Gnome** | {pkgrel:gnome-control-center-filesystem} | +| **KDE** | {pkgrel:plasma-desktop} | +| **Mesa** | {pkgrel:mesa-filesystem} | +| **Podman** | {pkgrel:podman} | + +### Major DX packages +| Name | Version | +| --- | --- | +| **Incus** | {pkgrel:incus} | +| **Docker** | {pkgrel:docker-ce} | +| **Devpod** | {pkgrel:devpod} | + +{changes} + +### How to rebase +For current users, type the following to rebase to this version: +## For this branch (if latest): +# Bluefin +```bash +sudo bootc switch ghcr.io/ublue-os/bluefin:{target} --enforce-container-sigpolicy +``` +# Aurora +```bash +sudo bootc switch ghcr.io/ublue-os/aurora:{target} --enforce-container-sigpolicy +``` +## For this specific image: +# Bluefin +```bash +sudo bootc switch ghcr.io/ublue-os/bluefin:{curr} --enforce-container-sigpolicy +``` +# Aurora +```bash +sudo bootc switch ghcr.io/ublue-os/aurora:{curr} --enforce-container-sigpolicy +``` + +### Documentation +Be sure to read the [documentation](https://docs.projectbluefin.io/) for more information +on how to use your cloud native system. +""" +HANDWRITTEN_PLACEHOLDER = """\ +This is an automatically generated changelog for release `{curr}`.""" + +BLACKLIST_VERSIONS = [ + "kernel", + "gnome-control-center-filesystem", + "plasma-desktop", + "mesa-filesystem", + "podman", + "docker-ce", + "incus", + "devpod" +] + + +def get_images(target: str): + if target == "latest": + matrix = IMAGE_MATRIX_LATEST + else: + matrix = IMAGE_MATRIX + + for experience, de, image_flavor in product(*matrix.values()): + img = "" + if de == "gnome": + img += "bluefin" + elif de == "kde": + img += "aurora" + + if experience == "dx": + img += "-dx" + + if image_flavor != "main": + img += "-" + img += image_flavor + + yield img, experience, de, image_flavor + + +def get_manifests(target: str): + out = {} + imgs = list(get_images(target)) + for j, (img, _, _, _) in enumerate(imgs): + output = None + print(f"Getting {img}:{target} manifest ({j+1}/{len(imgs)}).") + for i in range(RETRIES): + try: + output = subprocess.run( + ["skopeo", "inspect", REGISTRY + img + ":" + target], + check=True, + stdout=subprocess.PIPE, + ).stdout + break + except subprocess.CalledProcessError: + print( + f"Failed to get {img}:{target}, retrying in {RETRY_WAIT} seconds ({i+1}/{RETRIES})" + ) + time.sleep(RETRY_WAIT) + if output is None: + print(f"Failed to get {img}:{target}, skipping") + continue + out[img] = json.loads(output) + return out + + +def get_tags(target: str, manifests: dict[str, Any]): + tags = set() + + for manifest in manifests.values(): + for tag in manifest["RepoTags"]: + # Tags ending with .0 should not exist + if tag.endswith(".0"): + continue + if re.match(START_PATTERN(target), tag): + tags.add(tag) + + tags = list(sorted(tags)) + assert len(tags) > 2, "No current and previous tags found" + return tags[-2], tags[-1] + + +def get_packages(manifests: dict[str, Any]): + packages = {} + for img, manifest in manifests.items(): + try: + packages[img] = json.loads(manifest["Labels"]["dev.hhd.rechunk.info"])[ + "packages" + ] + except Exception as e: + print(f"Failed to get packages for {img}:\n{e}") + return packages + + +def get_package_groups(target: str, prev: dict[str, Any], manifests: dict[str, Any]): + common = set() + others = {k: set() for k in OTHER_NAMES.keys()} + + npkg = get_packages(manifests) + ppkg = get_packages(prev) + + keys = set(npkg.keys()) | set(ppkg.keys()) + pkg = defaultdict(set) + for k in keys: + pkg[k] = set(npkg.get(k, {})) | set(ppkg.get(k, {})) + + # Find common packages + first = True + for img, experience, de, image_flavor in get_images(target): + if img not in pkg: + continue + + if first: + for p in pkg[img]: + common.add(p) + else: + for c in common.copy(): + if c not in pkg[img]: + common.remove(c) + + first = False + + # Find other packages + for t, other in others.items(): + print(t) + first = True + for img, experience, de, image_flavor in get_images(target): + if img not in pkg: + continue + + if t == "hwe" and "hwe" not in image_flavor: + continue + if t == "nvidia" and "nvidia" not in image_flavor: + continue + if t == "kde" and de != "kde": + continue + if t == "gnome" and de != "gnome": + continue + if t == "base" and experience != "base": + continue + if t == "dx" and experience != "dx": + continue + + if first: + for p in pkg[img]: + if p not in common: + other.add(p) + else: + for c in other.copy(): + if c not in pkg[img]: + other.remove(c) + + first = False + + return sorted(common), {k: sorted(v) for k, v in others.items()} + + +def get_versions(manifests: dict[str, Any]): + versions = {} + pkgs = get_packages(manifests) + for img_pkgs in pkgs.values(): + for pkg, v in img_pkgs.items(): + versions[pkg] = re.sub(FEDORA_PATTERN, "", v) + return versions + + +def calculate_changes(pkgs: list[str], prev: dict[str, str], curr: dict[str, str]): + added = [] + changed = [] + removed = [] + + blacklist_ver = set([curr.get(v, None) for v in BLACKLIST_VERSIONS]) + + for pkg in pkgs: + # Clearup changelog by removing mentioned packages + if pkg in BLACKLIST_VERSIONS: + continue + if pkg in curr and curr.get(pkg, None) in blacklist_ver: + continue + if pkg in prev and prev.get(pkg, None) in blacklist_ver: + continue + + if pkg not in prev: + added.append(pkg) + elif pkg not in curr: + removed.append(pkg) + elif prev[pkg] != curr[pkg]: + changed.append(pkg) + + blacklist_ver.add(curr.get(pkg, None)) + blacklist_ver.add(prev.get(pkg, None)) + + out = "" + for pkg in added: + out += PATTERN_ADD.format(name=pkg, version=curr[pkg]) + for pkg in changed: + out += PATTERN_CHANGE.format(name=pkg, prev=prev[pkg], new=curr[pkg]) + for pkg in removed: + out += PATTERN_REMOVE.format(name=pkg, version=prev[pkg]) + return out + + +def get_commits(prev_manifests, manifests, workdir: str): + try: + start = next(iter(prev_manifests.values()))["Labels"][ + "org.opencontainers.image.revision" + ] + finish = next(iter(manifests.values()))["Labels"][ + "org.opencontainers.image.revision" + ] + + commits = subprocess.run( + [ + "git", + "-C", + workdir, + "log", + "--pretty=format:%H %h %s", + f"{start}..{finish}", + ], + check=True, + stdout=subprocess.PIPE, + ).stdout.decode("utf-8") + + out = "" + for commit in commits.split("\n"): + if not commit: + continue + hash, short, subject = commit.split(" ", 2) + + if subject.lower().startswith("merge"): + continue + + out += ( + COMMIT_FORMAT.replace("{short}", short) + .replace("{subject}", subject) + .replace("{hash}", hash) + ) + + if out: + return COMMITS_FORMAT.format(commits=out) + return "" + except Exception as e: + print(f"Failed to get commits:\n{e}") + return "" + + +def generate_changelog( + handwritten: str | None, + target: str, + pretty: str | None, + workdir: str, + prev_manifests, + manifests, +): + common, others = get_package_groups(target, prev_manifests, manifests) + versions = get_versions(manifests) + prev_versions = get_versions(prev_manifests) + + prev, curr = get_tags(target, manifests) + + if not pretty: + # Generate pretty version since we dont have it + try: + finish: str = next(iter(manifests.values()))["Labels"][ + "org.opencontainers.image.revision" + ] + except Exception as e: + print(f"Failed to get finish hash:\n{e}") + finish = "" + + # Remove .0 from curr + curr_pretty = re.sub(r"\.\d{1,2}$", "", curr) + # Remove target- from curr + curr_pretty = re.sub(rf"^[a-z]+-", "", curr_pretty) + pretty = target.capitalize() + " (F" + curr_pretty + if finish and target != "stable": + pretty += ", #" + finish[:7] + pretty += ")" + + title = CHANGELOG_TITLE.format_map(defaultdict(str, tag=curr, pretty=pretty)) + + changelog = CHANGELOG_FORMAT + + changelog = ( + changelog.replace("{handwritten}", handwritten if handwritten else HANDWRITTEN_PLACEHOLDER) + .replace("{target}", target) + .replace("{prev}", prev) + .replace("{curr}", curr) + ) + + for pkg, v in versions.items(): + if pkg not in prev_versions or prev_versions[pkg] == v: + changelog = changelog.replace( + "{pkgrel:" + pkg + "}", PATTERN_PKGREL.format(version=v) + ) + else: + changelog = changelog.replace( + "{pkgrel:" + pkg + "}", + PATTERN_PKGREL_CHANGED.format(prev=prev_versions[pkg], new=v), + ) + + changes = "" + changes += get_commits(prev_manifests, manifests, workdir) + common = calculate_changes(common, prev_versions, versions) + if common: + changes += COMMON_PAT.format(changes=common) + for k, v in others.items(): + chg = calculate_changes(v, prev_versions, versions) + if chg: + changes += OTHER_NAMES[k].format(changes=chg) + + changelog = changelog.replace("{changes}", changes) + + return title, changelog + + +def main(): + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("target", help="Target tag") + parser.add_argument("output", help="Output environment file") + parser.add_argument("changelog", help="Output changelog file") + parser.add_argument("--pretty", help="Subject for the changelog") + parser.add_argument("--workdir", help="Git directory for commits") + parser.add_argument("--handwritten", help="Handwritten changelog") + args = parser.parse_args() + + # Remove refs/tags, refs/heads, refs/remotes e.g. + # Tags cannot include / anyway. + target = args.target.split('/')[-1] + + if target == "main": + target = "stable" + + manifests = get_manifests(target) + prev, curr = get_tags(target, manifests) + print(f"Previous tag: {prev}") + print(f" Current tag: {curr}") + + prev_manifests = get_manifests(prev) + title, changelog = generate_changelog( + args.handwritten, + target, + args.pretty, + args.workdir, + prev_manifests, + manifests, + ) + + print(f"Changelog:\n# {title}\n{changelog}") + print(f"\nOutput:\nTITLE=\"{title}\"\nTAG={curr}") + + with open(args.changelog, "w") as f: + f.write(changelog) + + with open(args.output, "w") as f: + f.write(f'TITLE="{title}"\nTAG={curr}\n') + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/reusable-build.yml b/.github/workflows/reusable-build.yml index 049461e64dd..135743d78fc 100644 --- a/.github/workflows/reusable-build.yml +++ b/.github/workflows/reusable-build.yml @@ -176,7 +176,7 @@ jobs: elif [[ "${{ matrix.fedora_version }}" =~ "stable" && "${{ github.event_name }}" =~ workflow_dispatch ]]; then BUILD_TAGS+=("stable" "stable-${TIMESTAMP}") elif [[ ! "${{ matrix.fedora_version}}" =~ "stable" ]]; then - BUILD_TAGS=("${{ env.FEDORA_VERSION }}" "${{ env.FEDORA_VERSION }}-${TIMESTAMP}") + BUILD_TAGS+=("${{ env.FEDORA_VERSION }}" "${{ env.FEDORA_VERSION }}-${TIMESTAMP}") fi # Prepend testing if built on testing branch @@ -460,14 +460,9 @@ jobs: fi done - # build_iso: - # name: iso + # generate_release: + # name: Generate Release # needs: [check] - # if: (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && (github.ref_name == 'main' || github.ref_name == 'testing') && inputs.fedora_version != 'beta' && inputs.fedora_version != 'latest' - # # Eventually would be nice for building images in PRs - # #if: ${{ endsWith(github.event.pull_request.title, '[ISO]') }} - # uses: ./.github/workflows/reusable-build-iso.yml + # if: github.event_name != 'pull_request' # secrets: inherit - # with: - # brand_name: ${{ inputs.brand_name }} - # fedora_version: ${{ inputs.fedora_version }} + # uses: ./.github/workflows/generate_release.yml