Scaffold from online emulator

docs/static/schemas/pilet-v0.json

@@ -36,6 +36,10 @@
           "path": {
             "type": "string",
             "description": "Defines the path to the emulator package of this app shell. If not given, the emulator package is resolved by the name of the given app shell key."
+          },
+          "url": {
+            "type": "string",
+            "description": "Defines the URL to the emulator website of this app shell. If provided, the emulator package is downloaded and updated automatically from the source."
           }
         },
         "required": []

src/tooling/piral-cli/src/apps/add-piral-instance-pilet.ts

@@ -74,13 +74,14 @@ export async function addPiralInstancePilet(baseDir = process.cwd(), options: Ad
       const piletJsonDir = dirname(piletJsonPath);
       const root = await findPiletRoot(piletJsonDir);
       const oldContent = await readJson(piletJsonDir, piletJson);
-      const [appName] = await installPiralInstance(app, fullBase, root, npmClient);
+      const [appName, _, appDetails] = await installPiralInstance(app, fullBase, root, npmClient);
 
       const newContent = {
         ...oldContent,
         piralInstances: {
           ...oldContent.piralInstances,
           [appName]: {
+            ...appDetails,
             selected,
           },
         },

src/tooling/piral-cli/src/apps/new-pilet.ts

@@ -184,7 +184,7 @@ always-auth=true`,
       ),
     );
 
-    const [packageName, packageVersion] = await installPiralInstance(
+    const [packageName, packageVersion, packageDetails] = await installPiralInstance(
       source || `empty-piral@${cliVersion}`,
       fullBase,
       root,
@@ -209,6 +209,7 @@ always-auth=true`,
     await patchPiletPackage(root, packageName, packageVersion, piralInfo, isEmulator, {
       language,
       bundler: bundlerName,
+      details: packageDetails,
     });
 
     const chosenTemplate = template || preSelectedTemplate || 'default';

src/tooling/piral-cli/src/common/emulator.ts

@@ -1,3 +1,4 @@
+import { resolve as resolveUrl } from 'url';
 import { join, resolve, relative, basename } from 'path';
 import { findDependencyVersion, copyScaffoldingFiles, isValidDependency, flattenExternals } from './package';
 import { createPiralStubIndexIfNotExists } from './template';
@@ -7,9 +8,46 @@ import { createNpmPackage } from './npm';
 import { createPiralDeclaration } from './declaration';
 import { ForceOverwrite } from './enums';
 import { createTarball } from './archive';
-import { createDirectory, removeDirectory, matchFiles, removeAny, getFileNames } from './io';
+import { createDirectory, removeDirectory, matchFiles, removeAny, getFileNames, writeBinary } from './io';
 import { updateExistingJson, readJson, writeJson, createFileIfNotExists } from './io';
-import { LogLevels, SharedDependency, PiletsInfo, TemplateFileLocation } from '../types';
+import { axios } from '../external';
+import { LogLevels, SharedDependency, PiletsInfo, TemplateFileLocation, Importmap } from '../types';
+
+export interface EmulatorWebsiteManifestFiles {
+  typings: string;
+  main: string;
+  app: string;
+  assets: Array<string>;
+}
+
+export interface EmulatorWebsiteManifest {
+  name: string;
+  description: string;
+  version: string;
+  timestamp: string;
+  scaffolding: {
+    pilets: PiletsInfo;
+    cli: string;
+  };
+  files: EmulatorWebsiteManifestFiles;
+  importmap: Importmap;
+  dependencies: {
+    optional: Record<string, string>;
+    included: Record<string, string>;
+  };
+}
+
+async function downloadEmulatorFiles(manifestUrl: string, target: string, files: EmulatorWebsiteManifestFiles) {
+  const requiredFiles = [files.typings, files.main, files.app];
+
+  await Promise.all(
+    requiredFiles.map(async (file) => {
+      const res = await axios.default.get(resolveUrl(manifestUrl, file), { responseType: 'arraybuffer' });
+      const data: Buffer = res.data;
+      await writeBinary(target, file, data);
+    }),
+  );
+}
 
 export async function createEmulatorSources(
   sourceDir: string,
@@ -102,7 +140,7 @@ export async function createEmulatorSources(
     },
     main: `./${join(appDir, 'index.js')}`,
     typings: `./${join(appDir, 'index.d.ts')}`,
-    app: `./${join(appDir, '/index.html')}`,
+    app: `./${join(appDir, 'index.html')}`,
     peerDependencies: {},
     optionalDependencies,
     devDependencies: {
@@ -211,43 +249,84 @@ export async function createEmulatorWebsite(
   }, {} as Record<string, string>);
 
   const allFiles = await matchFiles(targetDir, '*');
+  const data: EmulatorWebsiteManifest = {
+    name: piralPkg.name,
+    description: piralPkg.description,
+    version: piralPkg.version,
+    timestamp: new Date().toISOString(),
+    scaffolding: {
+      pilets,
+      cli: cliVersion,
+    },
+    files: {
+      typings: 'index.d.ts',
+      main: basename(targetFile),
+      app: 'index.html',
+      assets: allFiles.map((file) => relative(targetDir, file)),
+    },
+    importmap: {
+      imports: importmapEntries,
+    },
+    dependencies: {
+      optional: optionalDependencies,
+      included: {
+        ...allDeps,
+        ...externalDependencies,
+      },
+    },
+  };
+
+  await writeJson(targetDir, 'emulator.json', data, true);
+
+  // generate the associated index.d.ts
+  await createPiralDeclaration(sourceDir, piralPkg.app ?? `./src/index.html`, targetDir, ForceOverwrite.yes, logLevel);
+
+  return targetDir;
+}
+
+export async function scaffoldFromEmulatorWebsite(rootDir: string, manifestUrl: string) {
+  const response = await axios.default.get(manifestUrl);
+  const emulatorJson: EmulatorWebsiteManifest = response.data;
+
+  const targetDir = resolve(rootDir, 'node_modules', emulatorJson.name);
+  const appDirName = 'app';
+  const mainFile = 'index.js';
+  const appDir = resolve(targetDir, appDirName);
+  await createDirectory(appDir);
 
   await writeJson(
     targetDir,
-    'emulator.json',
+    packageJson,
     {
-      name: piralPkg.name,
-      description: piralPkg.description,
-      version: piralPkg.version,
-      timestamp: new Date().toISOString(),
-      scaffolding: {
-        pilets,
-        cli: cliVersion,
-      },
-      files: {
-        typings: 'index.d.ts',
-        main: basename(targetFile),
-        app: 'index.html',
-        assets: allFiles.map(file => relative(targetDir, file)),
-      },
-      importmap: {
-        imports: importmapEntries,
-      },
-      dependencies: {
-        optional: optionalDependencies,
-        included: {
-          ...allDeps,
-          ...externalDependencies,
-        },
+      name: emulatorJson.name,
+      description: emulatorJson.description,
+      version: emulatorJson.version,
+      importmap: emulatorJson.importmap,
+      pilets: emulatorJson.scaffolding.pilets,
+      piralCLI: {
+        version: emulatorJson.scaffolding.cli,
+        timestamp: emulatorJson.timestamp,
+        source: manifestUrl,
+        generated: true,
       },
+      main: `./${join(appDirName, mainFile)}`,
+      typings: `./${join(appDirName, emulatorJson.files.typings)}`,
+      app: `./${join(appDirName, emulatorJson.files.app)}`,
+      peerDependencies: {},
+      optionalDependencies: emulatorJson.dependencies.optional,
+      devDependencies: emulatorJson.dependencies.included,
     },
     true,
   );
 
-  // generate the associated index.d.ts
-  await createPiralDeclaration(sourceDir, piralPkg.app ?? `./src/index.html`, targetDir, ForceOverwrite.yes, logLevel);
+  // actually including this one hints that the app shell should have been included - which is forbidden
+  await createPiralStubIndexIfNotExists(appDir, mainFile, ForceOverwrite.yes, {
+    name: emulatorJson.name,
+    outFile: emulatorJson.files.main,
+  });
 
-  return targetDir;
+  await downloadEmulatorFiles(manifestUrl, appDir, emulatorJson.files);
+  return emulatorJson;
 }
 
 export async function packageEmulator(rootDir: string) {

src/tooling/piral-cli/src/common/npm.ts

@@ -195,7 +195,7 @@ export function publishNpmPackage(
   interactive = false,
 ): Promise<string> {
   const { publishPackage, loginUser } = clients.npm;
-  return publishPackage(target, file, ...flags).catch(err => {
+  return publishPackage(target, file, ...flags).catch((err) => {
     if (!interactive) {
       throw err;
     }
@@ -266,20 +266,31 @@ export function makeNpmAlias(name: string, version: string) {
 }
 
 export function isGitPackage(fullName: string) {
-  log('generalDebug_0003', 'Checking if its a Git package ...');
+  log('generalDebug_0003', 'Checking if its a git package ...');
 
   if (fullName) {
     const gitted = fullName.startsWith(gitPrefix);
 
     if (gitted || /^(https?|ssh):\/\/.*\.git$/.test(fullName)) {
-      log('generalDebug_0003', 'Found a Git package by name.');
+      log('generalDebug_0003', 'Found a git package by name.');
       return true;
     }
   }
 
   return false;
 }
 
+export function isRemotePackage(fullName: string) {
+  log('generalDebug_0003', 'Checking if its a remote package ...');
+
+  if (fullName && /^https?:\/\/.*/.test(fullName)) {
+    log('generalDebug_0003', 'Found a remote package by name.');
+    return true;
+  }
+
+  return false;
+}
+
 export function makeGitUrl(fullName: string) {
   const gitted = fullName.startsWith(gitPrefix);
   return gitted ? fullName : `${gitPrefix}${fullName}`;
@@ -309,6 +320,8 @@ export async function dissectPackageName(
   if (isGitPackage(fullName)) {
     const gitUrl = makeGitUrl(fullName);
     return [gitUrl, 'latest', false, 'git'];
+  } else if (isRemotePackage(fullName)) {
+    return [fullName, 'latest', false, 'remote'];
   } else if (isLocalPackage(baseDir, fullName)) {
     const fullPath = resolveAbsPath(baseDir, fullName);
     const exists = await checkExists(fullPath);

src/tooling/piral-cli/src/common/package.ts

@@ -38,6 +38,7 @@ import {
   PiletDefinition,
   AppDefinition,
   PiralInstancePackageData,
+  PiralInstanceDetails,
 } from '../types';
 
 export interface PiralInstanceData {
@@ -653,7 +654,7 @@ export async function patchPiletPackage(
   version: string,
   piralInfo: PiralPackageData,
   fromEmulator: boolean,
-  newInfo?: { language: SourceLanguage; bundler: string },
+  newInfo?: { language: SourceLanguage; bundler: string; details: PiralInstanceDetails },
 ) {
   log('generalDebug_0003', `Patching the package.json in "${root}" ...`);
   const pkg = await getPiletPackage(root, name, version, piralInfo, fromEmulator, newInfo);
@@ -664,7 +665,7 @@ export async function patchPiletPackage(
   await updateExistingJson(root, piletJson, {
     $schema: piletJsonSchemaUrl,
     piralInstances: {
-      [name]: {},
+      [name]: newInfo?.details || {},
     },
   });
   log('generalDebug_0003', `Succesfully patched the pilet.json.`);

src/tooling/piral-cli/src/common/shell.ts

@@ -1,21 +1,29 @@
 import { progress } from './log';
-import {
-  combinePackageRef,
-  dissectPackageName,
-  getPackageName,
-  getPackageVersion,
-  installNpmPackage,
-  isLinkedPackage,
-} from './npm';
-import { NpmClientType } from '../types';
+import { combinePackageRef, getPackageName, getPackageVersion } from './npm';
+import { dissectPackageName, installNpmPackage, isLinkedPackage } from './npm';
+import { NpmClientType, PiralInstanceDetails } from '../types';
+import { scaffoldFromEmulatorWebsite } from './emulator';
 
 export async function installPiralInstance(
   usedSource: string,
   baseDir: string,
   rootDir: string,
   npmClient: NpmClientType,
-): Promise<[name: string, version: string]> {
+): Promise<[name: string, version: string, details: PiralInstanceDetails]> {
   const [sourceName, sourceVersion, hadVersion, type] = await dissectPackageName(baseDir, usedSource);
+
+  if (type === 'remote') {
+    const emulatorJson = await scaffoldFromEmulatorWebsite(rootDir, sourceName);
+
+    return [
+      emulatorJson.name,
+      emulatorJson.version,
+      {
+        url: sourceName,
+      },
+    ];
+  }
+
   const isLocal = isLinkedPackage(sourceName, type, hadVersion, rootDir);
 
   if (!isLocal) {
@@ -30,5 +38,5 @@ export async function installPiralInstance(
   const packageName = await getPackageName(rootDir, sourceName, type);
   const packageVersion = getPackageVersion(hadVersion, sourceName, sourceVersion, type, rootDir);
 
-  return [packageName, packageVersion];
+  return [packageName, packageVersion, {}];
 }

src/tooling/piral-cli/src/types/internal.ts

@@ -1,19 +1,20 @@
 import type { LogLevels } from './common';
 import type { ImportmapVersions, PiletSchemaVersion } from './public';
 
+export interface PiralInstanceDetails {
+  selected?: boolean;
+  port?: number;
+  path?: string;
+  url?: string;
+}
+
 /**
  * Shape of the pilet.json
  */
 export interface PiletDefinition {
   schemaVersion?: PiletSchemaVersion;
   importmapVersions?: ImportmapVersions;
-  piralInstances?: Record<
-    string,
-    {
-      selected?: boolean;
-      port?: number;
-    }
-  >;
+  piralInstances?: Record<string, PiralInstanceDetails>;
 }
 
 export interface PackageFiles {

src/tooling/piral-cli/src/types/public.ts

@@ -236,7 +236,7 @@ export type PiralBuildType = 'all' | 'release' | 'emulator' | 'emulator-sources'
 
 export type PiletBuildType = 'default' | 'standalone' | 'manifest';
 
-export type PackageType = 'registry' | 'file' | 'git';
+export type PackageType = 'registry' | 'file' | 'git' | 'remote';
 
 export type NpmClientType = 'npm' | 'yarn' | 'pnp' | 'pnpm' | 'lerna' | 'rush' | 'bun';