Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge d7d0c3d into 22e779a
Browse files Browse the repository at this point in the history
imsnif authored Dec 5, 2019
2 parents 22e779a + d7d0c3d commit 2e2c078
Showing 11 changed files with 621 additions and 147 deletions.
51 changes: 34 additions & 17 deletions components/core/capsule/capsule.ts
Original file line number Diff line number Diff line change
@@ -1,17 +1,20 @@
import State from "./state";
import State from './state';
import Container from './container';
// @ts-ignore
import { Volume } from "memfs/lib/volume";
import Console from "./console";
import librarian from 'librarian';
import { Volume } from 'memfs/lib/volume';
import Console from './console';
// @ts-ignore
import { Union } from 'unionfs';
import { ContainerFS } from './container';
import { Exec } from "./container";
import { Exec } from './container';

import loader from '../../../src/cli/loader'; // TODO: better (have the capsule accept the loader as an arg?)

export class ContainerFactoryOptions {
image: string = '';
config: object = {};
};
}

export default class Capsule {
constructor(
@@ -41,6 +44,8 @@ export default class Capsule {
*/
static image = 'ubuntu';

componentName?: string;

/**
* default capsule config.
*/
@@ -63,15 +68,29 @@ export default class Capsule {
this.container.on(event, fn);
}

updateFs(fs: {[path: string]: string}, fn: Function): void {
Object.keys(fs).forEach((path) => {
updateFs(fs: { [path: string]: string }, fn: Function): void {
Object.keys(fs).forEach(path => {
// @ts-ignorex
this.fs.writeFile(path, fs[path], () => {
if (Object.keys(fs).length === 1) fn();
});
});
}

async execNode(executable: string, args: any) {
// TODO: better
const loaderPrefix = this.componentName ? `isolating ${this.componentName}` : '';
const log = message => (this.componentName ? loader.setText(`${loaderPrefix}: ${message}`) : {});
const { patchFileSystem } = librarian.api();
const onScriptRun = () =>
this.componentName ? loader.setText(`running build for ${this.componentName} in an isolated environment`) : {}; // TODO: do this from the compiler/tester so we can customize the message
await patchFileSystem(executable, { args, cwd: this.container.path, log, onScriptRun });
}

setComponentName(componentName: string) {
this.componentName = componentName;
}

outputFile(file: string, data: any, options: Object): Promise<any> {
return this.container.outputFile(file, data, options);
}
@@ -89,7 +108,7 @@ export default class Capsule {
}

resume() {
return this.container.resume()
return this.container.resume();
}

stop() {
@@ -117,21 +136,19 @@ export default class Capsule {

private static buildFs(memFs: Volume, containerFs: ContainerFS): Volume {
const fs = new Union();
fs
.use(memFs)
.use(containerFs);
fs.use(memFs).use(containerFs);

return fs;
}

static async create<T extends Capsule>(
containerFactory: (options: ContainerFactoryOptions) => Promise<Container>,
volume: Volume = new Volume(),
initialState: State = new State(),
console: Console = new Console()
): Promise<T> {
containerFactory: (options: ContainerFactoryOptions) => Promise<Container>,
volume: Volume = new Volume(),
initialState: State = new State(),
console: Console = new Console()
): Promise<T> {
const container = await containerFactory({ image: this.image, config: this.config });
const fs = await ContainerFS.fromJSON(container, {});
return (new this(container, this.buildFs(volume, fs), console, initialState) as T);
return new this(container, this.buildFs(volume, fs), console, initialState) as T;
}
}
25 changes: 13 additions & 12 deletions components/core/capsule/container/container.ts
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@ export default interface Container {
* container id
*/
id: string;
path: string;

/**
* execute a command on the container
@@ -20,13 +21,13 @@ export default interface Container {
/**
* put a file or a directory to the container.
*/
put(files: {[path: string]: string}, options: { overwrite?: boolean, path: string }): Promise<void>;
put(files: { [path: string]: string }, options: { overwrite?: boolean; path: string }): Promise<void>;

on(event: string, fn: (data: any) => void): void;

on(event: string, fn: (data: any) => void): void;

/**
* start a container.
*/
*/
start(): Promise<void>;

/**
@@ -69,37 +70,37 @@ export type ContainerStatus = {
/**
* array of open container ports
*/
ports: number[],
ports: number[];

/**
* container host
*/
host: string,
host: string;
};

export type CommitOptions = {
/**
* repository name for the created image
*/
repo: string,
repo: string;

/**
* tag name for the create image
*/
tag: string,
tag: string;

/**
* commit message
*/
comment: string,
comment: string;

/**
* author of the image.
*/
author: string,
author: string;

/**
* whether to pause the container before committing
*/
pause: boolean,
pause: boolean;
};
2 changes: 1 addition & 1 deletion e2e/commands/isolate.e2e.1.ts
Original file line number Diff line number Diff line change
@@ -40,7 +40,7 @@ describe('run bit isolate', function() {
describe('with the same parameters as pack is using', () => {
let isolatePath;
before(() => {
isolatePath = helper.command.isolateComponent('bar/foo', '-olw');
isolatePath = helper.command.isolateComponent('bar/foo', '-owls');
});
it('should be able to generate the links correctly and require the dependencies', () => {
const appJsFixture = `const barFoo = require('./');
2 changes: 1 addition & 1 deletion e2e/functionalities/custom-module-resolutions.e2e.2.ts
Original file line number Diff line number Diff line change
@@ -91,7 +91,7 @@ describe('custom module resolutions', function() {
describe('importing the component using isolated environment', () => {
let isolatePath;
before(() => {
isolatePath = helper.command.isolateComponent('bar/foo', '-olw');
isolatePath = helper.command.isolateComponent('bar/foo', '-olws');
});
it('should be able to generate the links correctly and require the dependencies', () => {
const appJsFixture = `const barFoo = require('./');
585 changes: 501 additions & 84 deletions package-lock.json

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
@@ -112,6 +112,7 @@
"is-glob": "^4.0.0",
"isbinaryfile": "^4.0.2",
"jfs": "^0.2.6",
"librarian": "github:teambit/librarian#release",
"lodash.assignwith": "^4.2.0",
"lodash.groupby": "^4.6.0",
"lodash.merge": "^4.6.2",
2 changes: 2 additions & 0 deletions src/app.ts
Original file line number Diff line number Diff line change
@@ -4,6 +4,8 @@ import buildRegistrar from './cli/command-registrar-builder';
import loadExtensions from './extensions/extensions-loader';
import HooksManager from './hooks';

process.env.MEMFS_DONT_WARN = 'true'; // suppress fs experimental warnings from memfs

// removing this, default to longStackTraces also when env is `development`, which impacts the
// performance dramatically. (see http://bluebirdjs.com/docs/api/promise.longstacktraces.html)
Promise.config({
11 changes: 4 additions & 7 deletions src/cli/commands/public-cmds/isolate-cmd.ts
Original file line number Diff line number Diff line change
@@ -11,7 +11,7 @@ export default class Isolate extends Command {
['d', 'directory [directory] ', 'path to store isolated component'],
['w', 'write-bit-dependencies [boolean] ', 'write bit components dependencies to package.json file'],
['l', 'npm-links [boolean]', 'point dependencies link files to npm package'],
['i', 'install-packages [boolean]', 'install npm package dependencies'],
['s', 'skip-npm-install [boolean]', 'do not install npm package dependencies'],
['', 'install-peer-dependencies [boolean]', 'install peer npm package dependencies'],
['', 'dist', 'write dist files (when exist) to the configured directory'],
['', 'conf', 'write the configuration file (bit.json)'],
@@ -40,7 +40,7 @@ export default class Isolate extends Command {
directory?: string;
writeBitDependencies?: boolean;
npmLinks?: boolean;
installPackages?: boolean;
skipNpmInstall?: boolean;
installPeerDependencies?: boolean;
dist?: boolean;
conf?: boolean;
@@ -53,9 +53,6 @@ export default class Isolate extends Command {
useCapsule?: boolean;
}
): Promise<any> {
// console.log('im here');
// console.log(opts);
// return '';
const concreteOpts: WorkspaceIsolateOptions = {
writeToPath: opts.directory,
override: opts.override === true,
@@ -65,8 +62,8 @@ export default class Isolate extends Command {
createNpmLinkFiles: opts.npmLinks === true,
saveDependenciesAsComponents: opts.saveDependenciesAsComponents !== false,
writeDists: opts.dist === true,
installNpmPackages: !!opts.installPackages, // convert to boolean
installPeerDependencies: !!opts.installPackages, // convert to boolean
installNpmPackages: !opts.skipNpmInstall,
installPeerDependencies: !opts.skipNpmInstall,
verbose: opts.verbose === true,
excludeRegistryPrefix: !!opts.excludeRegistryPrefix,
silentPackageManagerResult: false,
10 changes: 8 additions & 2 deletions src/consumer/component-ops/build-component.ts
Original file line number Diff line number Diff line change
@@ -299,16 +299,22 @@ async function _runBuild({
let shouldBuildUponDependenciesChanges;
const isolateFunc = async ({
targetDir,
shouldBuildDependencies
shouldBuildDependencies,
installNpmPackages,
keepExistingCapsule
}: {
targetDir?: string;
shouldBuildDependencies?: boolean;
installNpmPackages?: boolean;
keepExistingCapsule?: boolean;
}): Promise<{ capsule: Capsule; componentWithDependencies: ComponentWithDependencies }> => {
shouldBuildUponDependenciesChanges = shouldBuildDependencies;
const isolator = await Isolator.getInstance('fs', scope, consumer, targetDir);
const componentWithDependencies = await isolator.isolate(component.id, {
shouldBuildDependencies,
writeDists: false
writeDists: false,
installNpmPackages,
keepExistingCapsule
});
return new ExtensionIsolateResult(isolator, componentWithDependencies);
};
19 changes: 15 additions & 4 deletions src/consumer/component/sources/data-to-persist.ts
Original file line number Diff line number Diff line change
@@ -71,14 +71,25 @@ export default class DataToPersist {
await this._persistFilesToFS();
await this._persistSymlinksToFS();
}
async persistAllToCapsule(capsule: Capsule) {
async persistAllToCapsule(capsule: Capsule, opts = { keepExistingCapsule: false }) {
this._log();
this._validateRelative();
await Promise.all(this.remove.map(pathToRemove => capsule.removePath(pathToRemove.path)));
await Promise.all(this.files.map(file => this._writeFileToCapsule(capsule, file)));
if (!opts.keepExistingCapsule) {
await Promise.all(this.remove.map(pathToRemove => capsule.removePath(pathToRemove.path)));
}
await Promise.all(
this.files.map(file =>
this._writeFileToCapsule(capsule, file, { overwriteExistingFile: !!opts.keepExistingCapsule })
)
);
await Promise.all(this.symlinks.map(symlink => this.atomicSymlink(capsule, symlink)));
}
async _writeFileToCapsule(capsule: Capsule, file: AbstractVinyl) {
async _writeFileToCapsule(capsule: Capsule, file: AbstractVinyl, opts = { overwriteExistingFile: false }) {
// overwriteExistingFile: if a file with the same name exists in the capsule, overwrite it
if (opts.overwriteExistingFile) {
await capsule.removePath(file.path);
return capsule.outputFile(file.path, file.contents, {});
}
if (file.override === false) {
// @todo, capsule hack. use capsule.fs once you get it as a component.
// @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
60 changes: 41 additions & 19 deletions src/environment/isolator.ts
Original file line number Diff line number Diff line change
@@ -2,6 +2,7 @@ import R from 'ramda';
import * as path from 'path';
import semver from 'semver';
import pMapSeries from 'p-map-series';
import { runModule } from 'librarian';
import Capsule from '../../components/core/capsule';
import createCapsule from './capsule-factory';
import Consumer from '../consumer/consumer';
@@ -22,6 +23,7 @@ import BitMap from '../consumer/bit-map';
import { getManipulateDirForComponentWithDependencies } from '../consumer/component-ops/manipulate-dir';
import GeneralError from '../error/general-error';
import { PathOsBased } from '../utils/path';
import loader from '../cli/loader';

export interface IsolateOptions {
writeToPath?: PathOsBased; // Path to write the component to
@@ -34,6 +36,7 @@ export interface IsolateOptions {
writeDists?: boolean; // Write dist files
shouldBuildDependencies?: boolean; // Build all depedencies before the isolation (used by tools like ts compiler)
installNpmPackages?: boolean; // Install the package dependencies
keepExistingCapsule?: boolean; // Do not delete the capsule after using it (useful for incremental builds)
installPeerDependencies?: boolean; // Install the peer package dependencies
verbose?: boolean; // Print more logs
excludeRegistryPrefix?: boolean; // exclude the registry prefix from the component's name in the package.json
@@ -55,19 +58,30 @@ export default class Isolator {
_npmVersionHasValidated = false;
// @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
componentRootDir: string;
constructor(capsule: Capsule, scope: Scope, consumer?: Consumer) {
dir?: string;
constructor(capsule: Capsule, scope: Scope, consumer?: Consumer, dir?: string) {
this.capsule = capsule;
this.scope = scope;
this.consumer = consumer;
this.dir = dir;
}

static async getInstance(containerType = 'fs', scope: Scope, consumer?: Consumer, dir?: string): Promise<Isolator> {
logger.debug(`Isolator.getInstance, creating a capsule with an ${containerType} container, dir ${dir || 'N/A'}`);
const capsule = await createCapsule(containerType, dir);
return new Isolator(capsule, scope, consumer);
return new Isolator(capsule, scope, consumer, dir);
}

async isolate(componentId: BitId, opts: IsolateOptions): Promise<ComponentWithDependencies> {
const loaderPrefix = `isolating component - ${componentId.name}`;
loader.setText(loaderPrefix);
const log = message => loader.setText(`${loaderPrefix}: ${message}`);
// @ts-ignore TODO: this should be part of the capsule interface
this.capsule.execNode = async (executable, args) => {
const onScriptRun = () => loader.setText(`building component - ${componentId.name}`);
// TODO: do this from the compiler/tester so that the isolator doesn't need to know if it's a builder/tester/*...
await runModule(executable, { args, cwd: this.dir, log, onScriptRun });
};
const componentWithDependencies: ComponentWithDependencies = await this._loadComponent(componentId);
if (opts.shouldBuildDependencies) {
topologicalSortComponentDependencies(componentWithDependencies);
@@ -83,6 +97,8 @@ export default class Isolator {
});
}
const writeToPath = opts.writeToPath;
// default should be true
const installNpmPackages = typeof opts.installNpmPackages === 'undefined' ? true : opts.installNpmPackages;
const concreteOpts: ManyComponentsWriterParams = {
componentsWithDependencies: [componentWithDependencies],
writeToPath,
@@ -93,7 +109,7 @@ export default class Isolator {
createNpmLinkFiles: opts.createNpmLinkFiles,
saveDependenciesAsComponents: opts.saveDependenciesAsComponents !== false,
writeDists: opts.writeDists,
installNpmPackages: !!opts.installNpmPackages, // convert to boolean
installNpmPackages,
installPeerDependencies: !!opts.installPeerDependencies, // convert to boolean
addToRootPackageJson: false,
verbose: opts.verbose,
@@ -103,34 +119,40 @@ export default class Isolator {
};
this.componentWithDependencies = componentWithDependencies;
this.manyComponentsWriter = new ManyComponentsWriter(concreteOpts);
await this.writeComponentsAndDependencies();
await this.installComponentPackages();
await this.writeLinks();
await this.writeComponentsAndDependencies({ keepExistingCapsule: !!opts.keepExistingCapsule });
await this.installComponentPackages({
installNpmPackages,
keepExistingCapsule: !!opts.keepExistingCapsule
});
await this.writeLinks({ keepExistingCapsule: !!opts.keepExistingCapsule });
this.capsuleBitMap = this.manyComponentsWriter.bitMap;
return componentWithDependencies;
}

async writeComponentsAndDependencies() {
async writeComponentsAndDependencies(opts = { keepExistingCapsule: false }) {
logger.debug('ManyComponentsWriter, writeAllToIsolatedCapsule');
this._manipulateDir();
await this.manyComponentsWriter._populateComponentsFilesToWrite();
await this.manyComponentsWriter._populateComponentsDependenciesToWrite();
await this._persistComponentsDataToCapsule();
await this._persistComponentsDataToCapsule({ keepExistingCapsule: !!opts.keepExistingCapsule });
}

async installComponentPackages() {
async installComponentPackages(opts = { installNpmPackages: true, keepExistingCapsule: false }) {
// @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
this.capsulePackageJson = this.componentWithDependencies.component.packageJsonFile;
// @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
this.componentRootDir = this.componentWithDependencies.component.writtenPath;
await this._addComponentsToRoot();
await this._addComponentsToRoot({ keepExistingCapsule: !!opts.keepExistingCapsule });
logger.debug('ManyComponentsWriter, install packages on capsule');
await this._installWithPeerOption();
if (opts.installNpmPackages) {
await this._installWithPeerOption();
}
}

async writeLinks() {
async writeLinks(opts = { keepExistingCapsule: false }) {
const links = await this.manyComponentsWriter._getAllLinks();
await links.persistAllToCapsule(this.capsule);
// links is a DataToPersist instance
await links.persistAllToCapsule(this.capsule, { keepExistingCapsule: !!opts.keepExistingCapsule });
}

/**
@@ -175,14 +197,14 @@ export default class Isolator {
return loadFlattenedDependenciesForCapsule(consumer, component);
}

async _persistComponentsDataToCapsule() {
async _persistComponentsDataToCapsule(opts = { keepExistingCapsule: false }) {
const dataToPersist = new DataToPersist();
const allComponents = [this.componentWithDependencies.component, ...this.componentWithDependencies.allDependencies];
allComponents.forEach(component => dataToPersist.merge(component.dataToPersist));
await dataToPersist.persistAllToCapsule(this.capsule);
await dataToPersist.persistAllToCapsule(this.capsule, { keepExistingCapsule: !!opts.keepExistingCapsule });
}

async _addComponentsToRoot(): Promise<void> {
async _addComponentsToRoot(opts = { keepExistingCapsule: false }): Promise<void> {
// @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
// @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
const capsulePath = this.capsule.container.getPath();
@@ -203,13 +225,13 @@ export default class Isolator {
}, {});
if (R.isEmpty(componentsToAdd)) return;
this.capsulePackageJson.addDependencies(componentsToAdd);
await this._writeCapsulePackageJson();
await this._writeCapsulePackageJson({ keepExistingCapsule: !!opts.keepExistingCapsule });
}

async _writeCapsulePackageJson() {
async _writeCapsulePackageJson(opts = { keepExistingCapsule: false }) {
const dataToPersist = new DataToPersist();
dataToPersist.addFile(this.capsulePackageJson.toVinylFile());
dataToPersist.persistAllToCapsule(this.capsule);
return dataToPersist.persistAllToCapsule(this.capsule, { keepExistingCapsule: !!opts.keepExistingCapsule });
}

async _getNpmVersion() {

0 comments on commit 2e2c078

Please sign in to comment.