diff --git a/README.md b/README.md index 586ffa2..6657773 100644 --- a/README.md +++ b/README.md @@ -2,32 +2,4 @@ Installing conda packages into a browser -## Using - -This package has 2 methods: -- `installCondaPackage(prefix, url, Module.FS, untarjs, verbose)` - downloading one conda package and saving it into a browser. It returns shared libs if a package has them. - -- `bootstrapFromEmpackPackedEnvironment( packagesJsonUrl, verbose, skipLoadingSharedLibs,Module, pkgRootUrl)` - downloading empack_env_meta.json and installing all conda packages from this file. - -The example of using: - -```ts -import { - bootstrapFromEmpackPackedEnvironment, - IPackagesInfo -} from '@emscripten-forge/mambajs'; - - const packagesJsonUrl = `http://localhost:8888/empack_env_meta.json`; - const pkgRootUrl = 'kernel/kernel_packages'; - cosnt verbose = true; - - let packageData: IPackagesInfo = {}; - packageData = await bootstrapFromEmpackPackedEnvironment( - packagesJsonUrl, - verbose, - false, - Module, - pkgRootUrl -); - -``` +This is still in progress, APIs are subject to change quickly diff --git a/src/dynload/dynload.js b/src/dynload/dynload.js index cae3ca1..8b42a23 100644 --- a/src/dynload/dynload.js +++ b/src/dynload/dynload.js @@ -43,8 +43,7 @@ function isInSharedLibraryPath(prefix, libPath){ export async function loadDynlibsFromPackage( prefix, - pkg_file_name, - pkg_is_shared_library, + pkgName, dynlibPaths, Module ) { @@ -56,41 +55,26 @@ export async function loadDynlibsFromPackage( else{ var sitepackages = `${prefix}/lib/python3.11/site-packages` } - const auditWheelLibDir = `${sitepackages}/${ - pkg_file_name.split("-")[0] - }.libs`; + const auditWheelLibDir = `${sitepackages}/${pkgName}.libs`; // This prevents from reading large libraries multiple times. const readFileMemoized = memoize(Module.FS.readFile); - const forceGlobal = !!pkg_is_shared_library; - - let dynlibs = []; - - if (forceGlobal) { - dynlibs = Object.keys(dynlibPaths).map((path) =>{ - return { - path: path, - global: true, - }; - }); - } else { - const globalLibs = calculateGlobalLibs( + const globalLibs = calculateGlobalLibs( dynlibPaths, readFileMemoized, Module - ); + ); - dynlibs = Object.keys(dynlibPaths).map((path) =>{ + dynlibs = dynlibPaths.map((path) =>{ const global = globalLibs.has(Module.PATH.basename(path)); return { - path: path, - global: global || !! pkg_is_shared_library || isInSharedLibraryPath(prefix, path) || path.startsWith(auditWheelLibDir), + path: path, + global: global || isInSharedLibraryPath(prefix, path) || path.startsWith(auditWheelLibDir), }; - }); - } + }); dynlibs.sort((lib1, lib2) => Number(lib2.global) - Number(lib1.global)); for (const { path, global } of dynlibs) { @@ -172,7 +156,7 @@ function calculateGlobalLibs( const globalLibs = new Set(); - Object.keys(libs).map((lib) => { + libs.map((lib) => { const binary = readFile(lib); const needed = Module.getDylinkMetadata(binary).neededDynlibs; needed.forEach((lib) => { @@ -193,9 +177,9 @@ async function loadDynlib(prefix, lib, global, searchDirs, readFileFunc, Module) if (searchDirs === undefined) { searchDirs = []; } - + const releaseDynlibLock = await acquireDynlibLock(); - + try { const fs = createDynlibFS(prefix, lib, searchDirs, readFileFunc, Module); diff --git a/src/helper.ts b/src/helper.ts index 2959457..451731f 100644 --- a/src/helper.ts +++ b/src/helper.ts @@ -9,31 +9,51 @@ export interface IEmpackEnvMetaPkg { url: string; } -export async function fetchJson(url: string): Promise { - let response = await fetch(url); - if (!response.ok) { - throw new Error(`HTTP error! status: ${response.status}`); - } - let json = await response.json(); - return json; +export interface IEmpackEnvMeta { + prefix: string; + packages: IEmpackEnvMetaPkg[]; } +/** + * Shared libraries. list of .so files + */ +export type TSharedLibs = string[]; + +/** + * Shared libraries. A map package name -> list of .so files + */ +export type TSharedLibsMap = { [pkgName: string]: TSharedLibs }; + export function getParentDirectory(filePath: string): string { return filePath.substring(0, filePath.lastIndexOf('/')); } -export function getSharedLibs(files: FilesData, prefix: string): FilesData { - let sharedLibs: FilesData = {}; +export function getSharedLibs(files: FilesData, prefix: string): TSharedLibs { + let sharedLibs: TSharedLibs = []; Object.keys(files).map(file => { - if (file.endsWith('.so') || file.includes('.so.')) { - sharedLibs[`${prefix}/${file}`] = files[file]; + if ( + (file.endsWith('.so') || file.includes('.so.')) && + checkWasmMagicNumber(files[file]) + ) { + sharedLibs.push(`${prefix}/${file}`); } }); return sharedLibs; } +export function checkWasmMagicNumber(uint8Array: Uint8Array): boolean { + const WASM_MAGIC_NUMBER = [0x00, 0x61, 0x73, 0x6d]; + + return ( + uint8Array[0] === WASM_MAGIC_NUMBER[0] && + uint8Array[1] === WASM_MAGIC_NUMBER[1] && + uint8Array[2] === WASM_MAGIC_NUMBER[2] && + uint8Array[3] === WASM_MAGIC_NUMBER[3] + ); +} + export function isCondaMeta(files: FilesData): boolean { let isCondaMetaFile = false; Object.keys(files).forEach(filename => { @@ -45,22 +65,6 @@ export function isCondaMeta(files: FilesData): boolean { return isCondaMetaFile; } -export function getPythonVersion( - packages: IEmpackEnvMetaPkg[] -): number[] | undefined { - let pythonPackage: IEmpackEnvMetaPkg | undefined = undefined; - for (let i = 0; i < packages.length; i++) { - if (packages[i].name == 'python') { - pythonPackage = packages[i]; - break; - } - } - - if (pythonPackage) { - return pythonPackage.version.split('.').map(x => parseInt(x)); - } -} - export function saveFiles(FS: any, files: FilesData, prefix: string): void { try { Object.keys(files).forEach(filename => { @@ -76,33 +80,13 @@ export function saveFiles(FS: any, files: FilesData, prefix: string): void { } } -export async function bootstrapPythonPackage( - pythonPackage: IEmpackEnvMetaPkg, - pythonVersion: number[], - verbose: boolean, - untarjs: IUnpackJSAPI, - Module: any, - pkgRootUrl: string, - prefix: string -): Promise { - let url = pythonPackage.url - ? pythonPackage.url - : `${pkgRootUrl}/${pythonPackage.filename}`; - if (verbose) { - console.log(`Installing a python package from ${url}`); - } - await installCondaPackage(prefix, url, Module.FS, untarjs, verbose); - await Module.init_phase_1(prefix, pythonVersion, verbose); -} - export async function installCondaPackage( prefix: string, url: string, FS: any, untarjs: IUnpackJSAPI, verbose: boolean -): Promise { - let sharedLibs: FilesData = {}; +): Promise { if (!url) { throw new Error(`There is no file in ${url}`); } @@ -154,10 +138,12 @@ export async function installCondaPackage( if (prefix === '/') { newPrefix = ''; } + if (Object.keys(installedFiles).length !== 0) { - sharedLibs = getSharedLibs(installedFiles, newPrefix); + return getSharedLibs(installedFiles, newPrefix); } - return sharedLibs; + + return []; } export function saveCondaMetaFile( diff --git a/src/index.ts b/src/index.ts index 53f6caa..ed879ab 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,91 +1,178 @@ -import { FilesData, initUntarJS } from '@emscripten-forge/untarjs'; +import { initUntarJS, IUnpackJSAPI } from '@emscripten-forge/untarjs'; import { - fetchJson, - getPythonVersion, + IEmpackEnvMeta, IEmpackEnvMetaPkg, - installCondaPackage + installCondaPackage, + TSharedLibsMap } from './helper'; import { loadDynlibsFromPackage } from './dynload/dynload'; -export const bootstrapFromEmpackPackedEnvironment = async ( - packagesJsonUrl: string, - verbose: boolean = true, - skipLoadingSharedLibs: boolean = false, - Module: any, - pkgRootUrl: string, - bootstrapPython = false -): Promise => { - if (verbose) { - console.log('fetching packages.json from', packagesJsonUrl); +export * from './helper'; + +export function getPythonVersion( + packages: IEmpackEnvMetaPkg[] +): number[] | undefined { + let pythonPackage: IEmpackEnvMetaPkg | undefined = undefined; + for (let i = 0; i < packages.length; i++) { + if (packages[i].name == 'python') { + pythonPackage = packages[i]; + break; + } + } + + if (pythonPackage) { + return pythonPackage.version.split('.').map(x => parseInt(x)); } +} + +export interface IBootstrapEmpackPackedEnvironmentOptions { + /** + * The empack lock file + */ + empackEnvMeta: IEmpackEnvMeta; + + /** + * The URL (CDN or similar) from which to download packages + */ + pkgRootUrl: string; + + /** + * The Emscripten Module + */ + Module: any; + + /** + * Whether to build in verbose mode, default to silent + */ + verbose?: boolean; + + /** + * The untarjs API. If not provided, one will be initialized. + */ + untarjs?: IUnpackJSAPI; +} - let empackEnvMeta = await fetchJson(packagesJsonUrl); - let allPackages: IEmpackEnvMetaPkg[] = empackEnvMeta.packages; - let prefix = empackEnvMeta.prefix; +/** + * Bootstrap a filesystem from an empack lock file. And return the installed shared libs. + * + * @param options + * @returns The installed shared libraries as a TSharedLibs + */ +export const bootstrapEmpackPackedEnvironment = async ( + options: IBootstrapEmpackPackedEnvironmentOptions +): Promise => { + const { empackEnvMeta, pkgRootUrl, Module, verbose } = options; - const untarjsReady = initUntarJS(); - const untarjs = await untarjsReady; + let untarjs: IUnpackJSAPI; + if (options.untarjs) { + untarjs = options.untarjs; + } else { + const untarjsReady = initUntarJS(); + untarjs = await untarjsReady; + } + + const sharedLibsMap: TSharedLibsMap = {}; - if (allPackages?.length) { - let sharedLibs = await Promise.all( - allPackages.map(pkg => { + if (empackEnvMeta.packages.length) { + await Promise.all( + empackEnvMeta.packages.map(async pkg => { const packageUrl = pkg?.url ?? `${pkgRootUrl}/${pkg.filename}`; if (verbose) { console.log(`Install ${pkg.filename} taken from ${packageUrl}`); } - return installCondaPackage( - prefix, + + sharedLibsMap[pkg.name] = await installCondaPackage( + empackEnvMeta.prefix, packageUrl, Module.FS, untarjs, - verbose + !!verbose ); }) ); await waitRunDependencies(Module); - if (!skipLoadingSharedLibs) { - await loadShareLibs(allPackages, sharedLibs, prefix, Module); - } } - if (bootstrapPython) { - // Assuming these are defined by pyjs - const pythonVersion = getPythonVersion(allPackages); - await Module.init_phase_1(prefix, pythonVersion, verbose); - Module.init_phase_2(prefix, pythonVersion, verbose); - } + return sharedLibsMap; }; -const loadShareLibs = ( - packages: IEmpackEnvMetaPkg[], - sharedLibs: FilesData[], - prefix: string, - Module: any -): Promise => { - return Promise.all( - packages.map(async (pkg, i) => { - let packageShareLibs = sharedLibs[i]; - if (Object.keys(packageShareLibs).length) { - let verifiedWasmSharedLibs: FilesData = {}; - Object.keys(packageShareLibs).map(path => { - const isValidWasm = checkWasmMagicNumber(packageShareLibs[path]); - if (isValidWasm) { - verifiedWasmSharedLibs[path] = packageShareLibs[path]; - } - }); - if (Object.keys(verifiedWasmSharedLibs).length) { - return await loadDynlibsFromPackage( - prefix, - pkg.name, - false, - verifiedWasmSharedLibs, - Module - ); - } - } - }) +export interface IBootstrapPythonOptions { + /** + * The Python version as a list e.g. [3, 11] + */ + pythonVersion: number[]; + + /** + * The environment prefix + */ + prefix: string; + + /** + * The Emscripten Module + */ + Module: any; + + /** + * Whether to build in verbose mode, default to silent + */ + verbose?: boolean; +} + +/** + * Bootstrap Python runtime + * + * @param options + */ +export async function bootstrapPython(options: IBootstrapPythonOptions) { + // Assuming these are defined by pyjs + await options.Module.init_phase_1( + options.prefix, + options.pythonVersion, + options.verbose ); -}; + options.Module.init_phase_2( + options.prefix, + options.pythonVersion, + options.verbose + ); +} + +export interface ILoadSharedLibsOptions { + /** + * Shared libs to load + */ + sharedLibs: TSharedLibsMap; + + /** + * The environment prefix + */ + prefix: string; + + /** + * The Emscripten Module + */ + Module: any; +} + +export async function loadShareLibs( + options: ILoadSharedLibsOptions +): Promise { + const { sharedLibs, prefix, Module } = options; + + const sharedLibsLoad: Promise[] = []; + + for (const pkgName of Object.keys(sharedLibs)) { + const packageShareLibs = sharedLibs[pkgName]; + + if (packageShareLibs) { + sharedLibsLoad.push( + loadDynlibsFromPackage(prefix, pkgName, packageShareLibs, Module) + ); + } + } + + return Promise.all(sharedLibsLoad); +} const waitRunDependencies = (Module: any): Promise => { const promise = new Promise(r => { @@ -99,19 +186,3 @@ const waitRunDependencies = (Module: any): Promise => { Module.removeRunDependency('dummy'); return promise; }; - -const checkWasmMagicNumber = (uint8Array: Uint8Array): boolean => { - const WASM_MAGIC_NUMBER = [0x00, 0x61, 0x73, 0x6d]; - - return ( - uint8Array[0] === WASM_MAGIC_NUMBER[0] && - uint8Array[1] === WASM_MAGIC_NUMBER[1] && - uint8Array[2] === WASM_MAGIC_NUMBER[2] && - uint8Array[3] === WASM_MAGIC_NUMBER[3] - ); -}; - -export default { - installCondaPackage, - bootstrapFromEmpackPackedEnvironment -};