Split bootstrap of the filesystem / Python interpreter / loading shar…

…ed logic (#21) * Split mambajs * Iterate * Iterate * Linter * Iterate * Finalize * Export more * Cleanup README for now
emscripten-forge · Jan 17, 2025 · 25458d3 · 25458d3
1 parent f1dd148
commit 25458d3
Show file tree

Hide file tree

Showing 4 changed files with 198 additions and 185 deletions.
diff --git a/README.md b/README.md
@@ -2,32 +2,4 @@
 
 Installing conda packages into a browser
 
-## Using
-
-This package has 2 methods:
-- `installCondaPackage(prefix, url, Module.FS, untarjs, verbose)` - downloading one conda package and saving it into a browser. It returns shared libs if a package has them.
-
-- `bootstrapFromEmpackPackedEnvironment( packagesJsonUrl, verbose, skipLoadingSharedLibs,Module, pkgRootUrl)` - downloading empack_env_meta.json and installing all conda packages from this file.
-
-The example of using:
-
-```ts
-import {
-  bootstrapFromEmpackPackedEnvironment,
-  IPackagesInfo
-} from '@emscripten-forge/mambajs';
-
- const packagesJsonUrl = `http://localhost:8888/empack_env_meta.json`;
- const pkgRootUrl = 'kernel/kernel_packages';
- cosnt verbose = true;
-
- let packageData: IPackagesInfo = {};
- packageData = await bootstrapFromEmpackPackedEnvironment(
-    packagesJsonUrl,
-    verbose,
-    false,
-    Module,
-    pkgRootUrl
-);
-
-```
+This is still in progress, APIs are subject to change quickly
diff --git a/src/dynload/dynload.js b/src/dynload/dynload.js
@@ -43,8 +43,7 @@ function isInSharedLibraryPath(prefix, libPath){
 
 export async function loadDynlibsFromPackage(
     prefix,
-    pkg_file_name,
-    pkg_is_shared_library,
+    pkgName,
     dynlibPaths,
     Module
   ) {
@@ -56,41 +55,26 @@ export async function loadDynlibsFromPackage(
     else{
         var sitepackages = `${prefix}/lib/python3.11/site-packages`
     }
-    const auditWheelLibDir = `${sitepackages}/${
-        pkg_file_name.split("-")[0]
-    }.libs`;
+    const auditWheelLibDir = `${sitepackages}/${pkgName}.libs`;
 
     // This prevents from reading large libraries multiple times.
     const readFileMemoized = memoize(Module.FS.readFile);
-    const forceGlobal = !!pkg_is_shared_library;
-
-
 
     let dynlibs = [];
 
-
-    if (forceGlobal) {
-      dynlibs = Object.keys(dynlibPaths).map((path) =>{
-        return {
-          path: path,
-          global: true,
-        };
-      });
-    } else {
-      const globalLibs = calculateGlobalLibs(
+    const globalLibs = calculateGlobalLibs(
         dynlibPaths,
         readFileMemoized,
         Module
-      );
+    );
 
-      dynlibs = Object.keys(dynlibPaths).map((path) =>{
+    dynlibs = dynlibPaths.map((path) =>{
         const global = globalLibs.has(Module.PATH.basename(path));
         return {
-          path: path,
-          global: global || !! pkg_is_shared_library || isInSharedLibraryPath(prefix, path) || path.startsWith(auditWheelLibDir),
+            path: path,
+            global: global || isInSharedLibraryPath(prefix, path) || path.startsWith(auditWheelLibDir),
         };
-      });
-    }
+    });
 
     dynlibs.sort((lib1, lib2) => Number(lib2.global) - Number(lib1.global));
     for (const { path, global } of dynlibs) {
@@ -172,7 +156,7 @@ function calculateGlobalLibs(
 
     const globalLibs = new Set();
 
-    Object.keys(libs).map((lib) => {
+    libs.map((lib) => {
         const binary = readFile(lib);
         const needed = Module.getDylinkMetadata(binary).neededDynlibs;
         needed.forEach((lib) => {
@@ -193,9 +177,9 @@ async function loadDynlib(prefix, lib, global, searchDirs, readFileFunc, Module)
     if (searchDirs === undefined) {
         searchDirs = [];
     }
- 
+
     const releaseDynlibLock = await acquireDynlibLock();
-    
+
     try {
         const fs = createDynlibFS(prefix, lib, searchDirs, readFileFunc, Module);
 

diff --git a/src/helper.ts b/src/helper.ts
@@ -9,31 +9,51 @@ export interface IEmpackEnvMetaPkg {
   url: string;
 }
 
-export async function fetchJson(url: string): Promise<any> {
-  let response = await fetch(url);
-  if (!response.ok) {
-    throw new Error(`HTTP error! status: ${response.status}`);
-  }
-  let json = await response.json();
-  return json;
+export interface IEmpackEnvMeta {
+  prefix: string;
+  packages: IEmpackEnvMetaPkg[];
 }
 
+/**
+ * Shared libraries. list of .so files
+ */
+export type TSharedLibs = string[];
+
+/**
+ * Shared libraries. A map package name -> list of .so files
+ */
+export type TSharedLibsMap = { [pkgName: string]: TSharedLibs };
+
 export function getParentDirectory(filePath: string): string {
   return filePath.substring(0, filePath.lastIndexOf('/'));
 }
 
-export function getSharedLibs(files: FilesData, prefix: string): FilesData {
-  let sharedLibs: FilesData = {};
+export function getSharedLibs(files: FilesData, prefix: string): TSharedLibs {
+  let sharedLibs: TSharedLibs = [];
 
   Object.keys(files).map(file => {
-    if (file.endsWith('.so') || file.includes('.so.')) {
-      sharedLibs[`${prefix}/${file}`] = files[file];
+    if (
+      (file.endsWith('.so') || file.includes('.so.')) &&
+      checkWasmMagicNumber(files[file])
+    ) {
+      sharedLibs.push(`${prefix}/${file}`);
     }
   });
 
   return sharedLibs;
 }
 
+export function checkWasmMagicNumber(uint8Array: Uint8Array): boolean {
+  const WASM_MAGIC_NUMBER = [0x00, 0x61, 0x73, 0x6d];
+
+  return (
+    uint8Array[0] === WASM_MAGIC_NUMBER[0] &&
+    uint8Array[1] === WASM_MAGIC_NUMBER[1] &&
+    uint8Array[2] === WASM_MAGIC_NUMBER[2] &&
+    uint8Array[3] === WASM_MAGIC_NUMBER[3]
+  );
+}
+
 export function isCondaMeta(files: FilesData): boolean {
   let isCondaMetaFile = false;
   Object.keys(files).forEach(filename => {
@@ -45,22 +65,6 @@ export function isCondaMeta(files: FilesData): boolean {
   return isCondaMetaFile;
 }
 
-export function getPythonVersion(
-  packages: IEmpackEnvMetaPkg[]
-): number[] | undefined {
-  let pythonPackage: IEmpackEnvMetaPkg | undefined = undefined;
-  for (let i = 0; i < packages.length; i++) {
-    if (packages[i].name == 'python') {
-      pythonPackage = packages[i];
-      break;
-    }
-  }
-
-  if (pythonPackage) {
-    return pythonPackage.version.split('.').map(x => parseInt(x));
-  }
-}
-
 export function saveFiles(FS: any, files: FilesData, prefix: string): void {
   try {
     Object.keys(files).forEach(filename => {
@@ -76,33 +80,13 @@ export function saveFiles(FS: any, files: FilesData, prefix: string): void {
   }
 }
 
-export async function bootstrapPythonPackage(
-  pythonPackage: IEmpackEnvMetaPkg,
-  pythonVersion: number[],
-  verbose: boolean,
-  untarjs: IUnpackJSAPI,
-  Module: any,
-  pkgRootUrl: string,
-  prefix: string
-): Promise<void> {
-  let url = pythonPackage.url
-    ? pythonPackage.url
-    : `${pkgRootUrl}/${pythonPackage.filename}`;
-  if (verbose) {
-    console.log(`Installing a python package from ${url}`);
-  }
-  await installCondaPackage(prefix, url, Module.FS, untarjs, verbose);
-  await Module.init_phase_1(prefix, pythonVersion, verbose);
-}
-
 export async function installCondaPackage(
   prefix: string,
   url: string,
   FS: any,
   untarjs: IUnpackJSAPI,
   verbose: boolean
-): Promise<FilesData> {
-  let sharedLibs: FilesData = {};
+): Promise<TSharedLibs> {
   if (!url) {
     throw new Error(`There is no file in ${url}`);
   }
@@ -154,10 +138,12 @@ export async function installCondaPackage(
   if (prefix === '/') {
     newPrefix = '';
   }
+
   if (Object.keys(installedFiles).length !== 0) {
-    sharedLibs = getSharedLibs(installedFiles, newPrefix);
+    return getSharedLibs(installedFiles, newPrefix);
   }
-  return sharedLibs;
+
+  return [];
 }
 
 export function saveCondaMetaFile(