diff --git a/bootstrap-deps.json b/bootstrap-deps.json new file mode 100644 index 00000000000..ad0843fecad --- /dev/null +++ b/bootstrap-deps.json @@ -0,0 +1,275 @@ +[ + { + "package": "array", + "version": "0.5.4.0", + "source": "pre-existing", + "revision": null, + "sha256": null + }, + { + "package": "base", + "version": "4.13.0.0", + "source": "pre-existing", + "revision": null, + "sha256": null + }, + { + "package": "binary", + "version": "0.8.7.0", + "source": "pre-existing", + "revision": null, + "sha256": null + }, + { + "package": "bytestring", + "version": "0.10.10.0", + "source": "pre-existing", + "revision": null, + "sha256": null + }, + { + "package": "containers", + "version": "0.6.2.1", + "source": "pre-existing", + "revision": null, + "sha256": null + }, + { + "package": "deepseq", + "version": "1.4.4.0", + "source": "pre-existing", + "revision": null, + "sha256": null + }, + { + "package": "directory", + "version": "1.3.4.0", + "source": "pre-existing", + "revision": null, + "sha256": null + }, + { + "package": "filepath", + "version": "1.4.2.1", + "source": "pre-existing", + "revision": null, + "sha256": null + }, + { + "package": "mtl", + "version": "2.2.2", + "source": "pre-existing", + "revision": null, + "sha256": null + }, + { + "package": "parsec", + "version": "3.1.14.0", + "source": "pre-existing", + "revision": null, + "sha256": null + }, + { + "package": "pretty", + "version": "1.1.3.6", + "source": "pre-existing", + "revision": null, + "sha256": null + }, + { + "package": "process", + "version": "1.6.7.0", + "source": "pre-existing", + "revision": null, + "sha256": null + }, + { + "package": "text", + "version": "1.2.4.0", + "source": "pre-existing", + "revision": null, + "sha256": null + }, + { + "package": "time", + "version": "1.9.3", + "source": "pre-existing", + "revision": null, + "sha256": null + }, + { + "package": "transformers", + "version": "0.5.6.2", + "source": "pre-existing", + "revision": null, + "sha256": null + }, + { + "package": "unix", + "version": "2.7.2.2", + "source": "pre-existing", + "revision": null, + "sha256": null + }, + { + "package": "Cabal", + "version": "3.3.0.0", + "source": "local", + "revision": null, + "sha256": null + }, + { + "package": "network", + "version": "3.1.1.1", + "source": "hackage", + "revision": null, + "sha256": "d7ef590173fff2ab522fbc167f3fafb867e4ecfca279eb3ef0d137b51f142c9a" + }, + { + "package": "template-haskell", + "version": "2.15.0.0", + "source": "pre-existing", + "revision": null, + "sha256": null + }, + { + "package": "network-uri", + "version": "2.6.3.0", + "source": "hackage", + "revision": null, + "sha256": "a01c1389f15d2cc2e847914737f706133bb11f0c5f8ee89711a36a25b7afa723" + }, + { + "package": "HTTP", + "version": "4000.3.14", + "source": "hackage", + "revision": null, + "sha256": "a602d7f30e917164c6a634f8cb1f5df4849048858db01380a0875e16e5aa687b" + }, + { + "package": "ghc-prim", + "version": "0.5.3", + "source": "pre-existing", + "revision": null, + "sha256": null + }, + { + "package": "integer-gmp", + "version": "1.0.2.0", + "source": "pre-existing", + "revision": null, + "sha256": null + }, + { + "package": "hashable", + "version": "1.3.0.0", + "source": "hackage", + "revision": null, + "sha256": "822e5413fbccca6ae884d3aba4066422c8b5d58d23d18b9ecb5c03273bb19ab4" + }, + { + "package": "stm", + "version": "2.5.0.0", + "source": "pre-existing", + "revision": null, + "sha256": null + }, + { + "package": "async", + "version": "2.2.2", + "source": "hackage", + "revision": null, + "sha256": "4b4ab1ac82c45144d82c6daf6cb6ba45eab9957dad44787fa5e869e23d73bbff" + }, + { + "package": "base16-bytestring", + "version": "0.1.1.6", + "source": "hackage", + "revision": null, + "sha256": "5afe65a152c5418f5f4e3579a5e0d5ca19c279dc9bf31c1a371ccbe84705c449" + }, + { + "package": "cryptohash-sha256", + "version": "0.11.101.0", + "source": "hackage", + "revision": 4, + "sha256": "52756435dbea248e344fbcbcc5df5307f60dfacf337dfd11ae30f1c7a4da05dd" + }, + { + "package": "echo", + "version": "0.1.3", + "source": "hackage", + "revision": 1, + "sha256": "704f07310f8272d170f8ab7fb2a2c13f15d8501ef8310801e36964c8eff485ef" + }, + { + "package": "random", + "version": "1.1", + "source": "hackage", + "revision": null, + "sha256": "b718a41057e25a3a71df693ab0fe2263d492e759679b3c2fea6ea33b171d3a5a" + }, + { + "package": "edit-distance", + "version": "0.2.2.1", + "source": "hackage", + "revision": null, + "sha256": "3e8885ee2f56ad4da940f043ae8f981ee2fe336b5e8e4ba3f7436cff4f526c4a" + }, + { + "package": "base64-bytestring", + "version": "1.0.0.3", + "source": "hackage", + "revision": null, + "sha256": "ef159d60ec14c0a3f3e26bab5c9fd7634d5e1b983c6a64f0b0c3261efe008fc7" + }, + { + "package": "ed25519", + "version": "0.0.5.0", + "source": "hackage", + "revision": 2, + "sha256": "d8a5958ebfa9309790efade64275dc5c441b568645c45ceed1b0c6ff36d6156d" + }, + { + "package": "lukko", + "version": "0.1.1.2", + "source": "hackage", + "revision": null, + "sha256": "8a79d113dc0ccef16c24d83379cc457485943027e777529c46362fecc06607d2" + }, + { + "package": "tar", + "version": "0.5.1.1", + "source": "hackage", + "revision": null, + "sha256": "b384449f62b2b0aa3e6d2cb1004b8060b01f21ec93e7b63e7af6d8fad8a9f1de" + }, + { + "package": "zlib", + "version": "0.6.2.1", + "source": "hackage", + "revision": null, + "sha256": "f0f810ff173560b60392db448455c0513b3239f48e43cb494b3733aa559621d0" + }, + { + "package": "hackage-security", + "version": "0.6.0.1", + "source": "hackage", + "revision": null, + "sha256": "9162b473af5a21c1ff32a50b972b9acf51f4c901604a22cf08a2dccac2f82f17" + }, + { + "package": "resolv", + "version": "0.1.2.0", + "source": "hackage", + "revision": null, + "sha256": "81a2bafad484db123cf8d17a02d98bb388a127fd0f822fa022589468a0e64671" + }, + { + "package": "cabal-install", + "version": "3.3.0.0", + "source": "local", + "revision": null, + "sha256": null + } +] \ No newline at end of file diff --git a/cabal-install/bootstrap.py b/cabal-install/bootstrap.py new file mode 100644 index 00000000000..5ce25dbd53c --- /dev/null +++ b/cabal-install/bootstrap.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +print(""" +DO NOT use this script if you have another recent cabal-install available. +This script is intended only for bootstrapping cabal-install on new +architectures. +""") + +import hashlib +import json +from pathlib import Path +import shutil +import subprocess +from enum import Enum +from typing import Set, Optional, Dict, List, Tuple, \ + NewType, BinaryIO, NamedTuple + +ghc_path = Path('ghc') +ghc_pkg_path = Path('ghc-pkg') + +PACKAGES = Path('packages') +PKG_DB = PACKAGES / 'packages.conf' +BOOTSTRAP_DEPS_JSON = Path('bootstrap-deps.json') + +PackageName = NewType('PackageName', str) +Version = NewType('Version', str) + +class PackageSource(Enum): + HACKAGE = 'hackage' + PREEXISTING = 'pre-existing' + LOCAL = 'local' + +BootstrapDep = NamedTuple('BootstrapDep', [ + ('package', PackageName), + ('version', Version), + ('source', PackageSource), + # only valid when source == HACKAGE + ('revision', Optional[int]), + ('sha256', Optional[bytes]), +]) + +class BadTarball(Exception): + def __init__(self, path: str, expected_sha256: bytes, found_sha256: bytes): + self.path = path + self.expected_sha256 = expected_sha256 + self.found_sha256 = found_sha256 + + def __str__(self): + return '\n'.join([ + f'Bad tarball hash: {self.path}', + f' expected: {self.expected_sha256}', + f' found: {self.found_sha256}', + ]) + +def package_url(package: PackageName, version: Version) -> str: + return f'https://hackage.haskell.org/package/{package}-{version}/{package}-{version}.tar.gz' + +def package_cabal_url(package: PackageName, version: Version, revision: int) -> str: + return f'https://hackage.haskell.org/package/{package}-{version}/revision/{revision}.cabal' + +def fetch_package(package: PackageName, + version: Version, + revision: Optional[int], + sha256: bytes + ) -> Path: + import urllib.request + + # Download source distribution + out = PACKAGES / (f'{package}-{version}.tar.gz') + if not out.exists(): + print(f'Fetching {package}-{version}...') + out.parent.mkdir(parents=True, exist_ok=True) + url = package_url(package, version) + with urllib.request.urlopen(url) as resp: + shutil.copyfileobj(resp, out.open('wb')) + + # Download revised cabal file + cabal_file = PACKAGES / f'{package}.cabal' + if revision is not None and not cabal_file.exists(): + url = package_cabal_url(package, version, revision) + with urllib.request.urlopen(url) as resp: + shutil.copyfileobj(resp, cabal_file.open('wb')) + + h = hash_file(hashlib.sha256(), out.open('rb')) + if sha256 != h: + raise BadTarball(out, sha256, h) + + return out + +def read_bootstrap_deps() -> List[BootstrapDep]: + deps = json.load(open('bootstrap-deps.json')) + def from_json(o: object) -> BootstrapDep: + o['source'] = PackageSource(o['source']) + return BootstrapDep(**o) + + return [from_json(dep) for dep in deps] + +def install_dep(dep: BootstrapDep) -> None: + if dep.source == PackageSource.PREEXISTING: + # We expect it to be in the compiler's bootstrap package set + subprocess.run([str(ghc_pkg_path), 'describe', f'{dep.package}-{dep.version}'], + check=True, stdout=subprocess.DEVNULL) + print(f'Using {dep.package}-{dep.version} from GHC...') + return + + elif dep.source == PackageSource.HACKAGE: + tarball = fetch_package(dep.package, dep.version, dep.revision, dep.sha256) + subprocess.run(['tar', 'xf', tarball.resolve()], + cwd=PACKAGES, check=True) + sdist_dir = PACKAGES / f'{dep.package}-{dep.version}' + + # Update cabal file with revision + if dep.revision is not None: + shutil.copyfile(PACKAGES / f'{dep.package}.cabal', + sdist_dir / f'{dep.package}.cabal') + + elif dep.source == PackageSource.LOCAL: + if dep.package == 'Cabal': + sdist_dir = Path('Cabal').resolve() + elif dep.package == 'cabal-install': + sdist_dir = Path('cabal-install').resolve() + else: + raise 'hi' + + install_sdist(sdist_dir) + +def install_sdist(sdist_dir: Path): + prefix = (PACKAGES / 'tmp').resolve() + configure_args = [ + f'--package-db={PKG_DB.resolve()}', + f'--prefix={prefix}', + f'--with-compiler={ghc_path}', + f'--with-hc-pkg={ghc_pkg_path}', + ] + + def check_call(args: List[str]) -> None: + subprocess.run(args, cwd=sdist_dir, check=True) + + check_call([str(ghc_path), '--make', 'Setup']) + check_call(['./Setup', 'configure'] + configure_args) + check_call(['./Setup', 'build']) + check_call(['./Setup', 'install']) + +def hash_file(h, f: BinaryIO) -> bytes: + while True: + d = f.read(1024) + if len(d) == 0: + return h.hexdigest() + + h.update(d) + + +# Cabal plan.json representation +UnitId = NewType('UnitId', str) +PlanUnit = NewType('PlanUnit', object) + +def read_plan(project_dir: Path) -> Dict[UnitId, PlanUnit]: + path = project_dir / 'dist-newstyle' / 'cache' / 'plan.json' + plan = json.load(path.open('rb')) + return { + UnitId(c['id']): PlanUnit(c) + for c in plan['install-plan'] + } + +def extract_plan() -> None: + units = read_plan(Path('.')) + target_unit = [ + unit + for unit in units.values() + if unit['pkg-name'] == 'cabal-install' + if unit['component-name'] == 'exe:cabal' + ][0] + + def unit_to_bootstrap_dep(unit: PlanUnit) -> BootstrapDep: + if 'pkg-src' in unit and unit['pkg-src']['type'] == 'local': + source = PackageSource.LOCAL + elif unit['type'] == 'configured': + source = PackageSource.HACKAGE + elif unit['type'] == 'pre-existing': + source = PackageSource.PREEXISTING + + return BootstrapDep(package = unit['pkg-name'], + version = unit['pkg-version'], + source = source, + revision = None, + sha256 = unit.get('pkg-src-sha256')) + + def unit_ids_deps(unit_ids: List[UnitId]) -> List[BootstrapDep]: + deps = [] + for unit_id in unit_ids: + unit = units[unit_id] + deps += unit_deps(unit) + deps.append(unit_to_bootstrap_dep(unit)) + + return deps + + def unit_deps(unit: PlanUnit) -> List[BootstrapDep]: + if unit['type'] == 'pre-existing': + return [] + + deps = [] + if 'components' in unit: + for comp_name, comp in unit['components'].items(): + deps += unit_ids_deps(comp['depends']) + if 'depends' in unit: + deps += unit_ids_deps(unit['depends']) + + return deps + + deps = remove_duplicates(unit_deps(target_unit) + [unit_to_bootstrap_dep(target_unit)]) + return deps + +def write_bootstrap_deps(deps: List[BootstrapDep]): + def to_json(dep: BootstrapDep) -> object: + return { + 'package': dep.package, + 'version': dep.version, + 'source': dep.source.value, + 'revision': dep.revision, + 'sha256': dep.sha256 + } + + json.dump([to_json(dep) for dep in deps], + BOOTSTRAP_DEPS_JSON.open('w'), + indent=2) + +def remove_duplicates(xs: list) -> list: + # it's easier to build lists and remove duplicates later than + # to implement an order-preserving set. + out = [] + for x in xs: + if x not in out: + out.append(x) + + return out + +def bootstrap() -> None: + if not PKG_DB.exists(): + print(f'Creating package database {PKG_DB}') + PKG_DB.parent.mkdir(parents=True, exist_ok=True) + subprocess.run([ghc_pkg_path, 'init', PKG_DB]) + + deps = read_bootstrap_deps() + for dep in deps: + install_dep(dep) + +def main() -> None: + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--extract-plan', action='store_true', + help='generate bootstrap-deps.json from plan.json') + args = parser.parse_args() + + if args.extract_plan: + deps = extract_plan() + write_bootstrap_deps(deps) + print(f'dependencies written to {BOOTSTRAP_DEPS_JSON}') + else: + bootstrap() + cabal_path = PACKAGES + print(f''' + Bootstrapping finished! + + The resulting cabal-install executable can be found in + in {cabal_path}. You now should use this to build a full + cabal-install distribution. + ''') + +if __name__ == '__main__': + main()