Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge d7d0c3d into 22e779a
Browse files Browse the repository at this point in the history
imsnif authored Dec 5, 2019

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
2 parents 22e779a + d7d0c3d commit 2e2c078
Showing 11 changed files with 621 additions and 147 deletions.
51 changes: 34 additions & 17 deletions components/core/capsule/capsule.ts
Original file line number Diff line number Diff line change
@@ -1,17 +1,20 @@
import State from "./state";
import State from './state';
import Container from './container';
// @ts-ignore
import { Volume } from "memfs/lib/volume";
import Console from "./console";
import librarian from 'librarian';
import { Volume } from 'memfs/lib/volume';
import Console from './console';
// @ts-ignore
import { Union } from 'unionfs';
import { ContainerFS } from './container';
import { Exec } from "./container";
import { Exec } from './container';

import loader from '../../../src/cli/loader'; // TODO: better (have the capsule accept the loader as an arg?)

export class ContainerFactoryOptions {
image: string = '';
config: object = {};
};
}

export default class Capsule {
constructor(
@@ -41,6 +44,8 @@ export default class Capsule {
*/
static image = 'ubuntu';

componentName?: string;

/**
* default capsule config.
*/
@@ -63,15 +68,29 @@ export default class Capsule {
this.container.on(event, fn);
}

updateFs(fs: {[path: string]: string}, fn: Function): void {
Object.keys(fs).forEach((path) => {
updateFs(fs: { [path: string]: string }, fn: Function): void {
Object.keys(fs).forEach(path => {
// @ts-ignorex
this.fs.writeFile(path, fs[path], () => {
if (Object.keys(fs).length === 1) fn();
});
});
}

async execNode(executable: string, args: any) {
// TODO: better
const loaderPrefix = this.componentName ? `isolating ${this.componentName}` : '';
const log = message => (this.componentName ? loader.setText(`${loaderPrefix}: ${message}`) : {});
const { patchFileSystem } = librarian.api();
const onScriptRun = () =>
this.componentName ? loader.setText(`running build for ${this.componentName} in an isolated environment`) : {}; // TODO: do this from the compiler/tester so we can customize the message
await patchFileSystem(executable, { args, cwd: this.container.path, log, onScriptRun });
}

setComponentName(componentName: string) {
this.componentName = componentName;
}

outputFile(file: string, data: any, options: Object): Promise<any> {
return this.container.outputFile(file, data, options);
}
@@ -89,7 +108,7 @@ export default class Capsule {
}

resume() {
return this.container.resume()
return this.container.resume();
}

stop() {
@@ -117,21 +136,19 @@ export default class Capsule {

private static buildFs(memFs: Volume, containerFs: ContainerFS): Volume {
const fs = new Union();
fs
.use(memFs)
.use(containerFs);
fs.use(memFs).use(containerFs);

return fs;
}

static async create<T extends Capsule>(
containerFactory: (options: ContainerFactoryOptions) => Promise<Container>,
volume: Volume = new Volume(),
initialState: State = new State(),
console: Console = new Console()
): Promise<T> {
containerFactory: (options: ContainerFactoryOptions) => Promise<Container>,
volume: Volume = new Volume(),
initialState: State = new State(),
console: Console = new Console()
): Promise<T> {
const container = await containerFactory({ image: this.image, config: this.config });
const fs = await ContainerFS.fromJSON(container, {});
return (new this(container, this.buildFs(volume, fs), console, initialState) as T);
return new this(container, this.buildFs(volume, fs), console, initialState) as T;
}
}
25 changes: 13 additions & 12 deletions components/core/capsule/container/container.ts
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@ export default interface Container {
* container id
*/
id: string;
path: string;

/**
* execute a command on the container
@@ -20,13 +21,13 @@ export default interface Container {
/**
* put a file or a directory to the container.
*/
put(files: {[path: string]: string}, options: { overwrite?: boolean, path: string }): Promise<void>;
put(files: { [path: string]: string }, options: { overwrite?: boolean; path: string }): Promise<void>;

on(event: string, fn: (data: any) => void): void;

on(event: string, fn: (data: any) => void): void;

/**
* start a container.
*/
*/
start(): Promise<void>;

/**
@@ -69,37 +70,37 @@ export type ContainerStatus = {
/**
* array of open container ports
*/
ports: number[],
ports: number[];

/**
* container host
*/
host: string,
host: string;
};

export type CommitOptions = {
/**
* repository name for the created image
*/
repo: string,
repo: string;

/**
* tag name for the create image
*/
tag: string,
tag: string;

/**
* commit message
*/
comment: string,
comment: string;

/**
* author of the image.
*/
author: string,
author: string;

/**
* whether to pause the container before committing
*/
pause: boolean,
pause: boolean;
};
2 changes: 1 addition & 1 deletion e2e/commands/isolate.e2e.1.ts
Original file line number Diff line number Diff line change
@@ -40,7 +40,7 @@ describe('run bit isolate', function() {
describe('with the same parameters as pack is using', () => {
let isolatePath;
before(() => {
isolatePath = helper.command.isolateComponent('bar/foo', '-olw');
isolatePath = helper.command.isolateComponent('bar/foo', '-owls');
});
it('should be able to generate the links correctly and require the dependencies', () => {
const appJsFixture = `const barFoo = require('./');
2 changes: 1 addition & 1 deletion e2e/functionalities/custom-module-resolutions.e2e.2.ts
Original file line number Diff line number Diff line change
@@ -91,7 +91,7 @@ describe('custom module resolutions', function() {
describe('importing the component using isolated environment', () => {
let isolatePath;
before(() => {
isolatePath = helper.command.isolateComponent('bar/foo', '-olw');
isolatePath = helper.command.isolateComponent('bar/foo', '-olws');
});
it('should be able to generate the links correctly and require the dependencies', () => {
const appJsFixture = `const barFoo = require('./');
585 changes: 501 additions & 84 deletions package-lock.json

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
@@ -112,6 +112,7 @@
"is-glob": "^4.0.0",
"isbinaryfile": "^4.0.2",
"jfs": "^0.2.6",
"librarian": "github:teambit/librarian#release",
"lodash.assignwith": "^4.2.0",
"lodash.groupby": "^4.6.0",
"lodash.merge": "^4.6.2",
2 changes: 2 additions & 0 deletions src/app.ts
Original file line number Diff line number Diff line change
@@ -4,6 +4,8 @@ import buildRegistrar from './cli/command-registrar-builder';
import loadExtensions from './extensions/extensions-loader';
import HooksManager from './hooks';

process.env.MEMFS_DONT_WARN = 'true'; // suppress fs experimental warnings from memfs

// removing this, default to longStackTraces also when env is `development`, which impacts the
// performance dramatically. (see http://bluebirdjs.com/docs/api/promise.longstacktraces.html)
Promise.config({
11 changes: 4 additions & 7 deletions src/cli/commands/public-cmds/isolate-cmd.ts
Original file line number Diff line number Diff line change
@@ -11,7 +11,7 @@ export default class Isolate extends Command {
['d', 'directory [directory] ', 'path to store isolated component'],
['w', 'write-bit-dependencies [boolean] ', 'write bit components dependencies to package.json file'],
['l', 'npm-links [boolean]', 'point dependencies link files to npm package'],
['i', 'install-packages [boolean]', 'install npm package dependencies'],
['s', 'skip-npm-install [boolean]', 'do not install npm package dependencies'],
['', 'install-peer-dependencies [boolean]', 'install peer npm package dependencies'],
['', 'dist', 'write dist files (when exist) to the configured directory'],
['', 'conf', 'write the configuration file (bit.json)'],
@@ -40,7 +40,7 @@ export default class Isolate extends Command {
directory?: string;
writeBitDependencies?: boolean;
npmLinks?: boolean;
installPackages?: boolean;
skipNpmInstall?: boolean;
installPeerDependencies?: boolean;
dist?: boolean;
conf?: boolean;
@@ -53,9 +53,6 @@ export default class Isolate extends Command {
useCapsule?: boolean;
}
): Promise<any> {
// console.log('im here');
// console.log(opts);
// return '';
const concreteOpts: WorkspaceIsolateOptions = {
writeToPath: opts.directory,
override: opts.override === true,
@@ -65,8 +62,8 @@ export default class Isolate extends Command {
createNpmLinkFiles: opts.npmLinks === true,
saveDependenciesAsComponents: opts.saveDependenciesAsComponents !== false,
writeDists: opts.dist === true,
installNpmPackages: !!opts.installPackages, // convert to boolean
installPeerDependencies: !!opts.installPackages, // convert to boolean
installNpmPackages: !opts.skipNpmInstall,
installPeerDependencies: !opts.skipNpmInstall,
verbose: opts.verbose === true,
excludeRegistryPrefix: !!opts.excludeRegistryPrefix,
silentPackageManagerResult: false,
10 changes: 8 additions & 2 deletions src/consumer/component-ops/build-component.ts
Original file line number Diff line number Diff line change
@@ -299,16 +299,22 @@ async function _runBuild({
let shouldBuildUponDependenciesChanges;
const isolateFunc = async ({
targetDir,
shouldBuildDependencies
shouldBuildDependencies,
installNpmPackages,
keepExistingCapsule
}: {
targetDir?: string;
shouldBuildDependencies?: boolean;
installNpmPackages?: boolean;
keepExistingCapsule?: boolean;
}): Promise<{ capsule: Capsule; componentWithDependencies: ComponentWithDependencies }> => {
shouldBuildUponDependenciesChanges = shouldBuildDependencies;
const isolator = await Isolator.getInstance('fs', scope, consumer, targetDir);
const componentWithDependencies = await isolator.isolate(component.id, {
shouldBuildDependencies,
writeDists: false
writeDists: false,
installNpmPackages,
keepExistingCapsule
});
return new ExtensionIsolateResult(isolator, componentWithDependencies);
};
19 changes: 15 additions & 4 deletions src/consumer/component/sources/data-to-persist.ts
Original file line number Diff line number Diff line change
@@ -71,14 +71,25 @@ export default class DataToPersist {
await this._persistFilesToFS();
await this._persistSymlinksToFS();
}
async persistAllToCapsule(capsule: Capsule) {
async persistAllToCapsule(capsule: Capsule, opts = { keepExistingCapsule: false }) {
this._log();
this._validateRelative();
await Promise.all(this.remove.map(pathToRemove => capsule.removePath(pathToRemove.path)));
await Promise.all(this.files.map(file => this._writeFileToCapsule(capsule, file)));
if (!opts.keepExistingCapsule) {
await Promise.all(this.remove.map(pathToRemove => capsule.removePath(pathToRemove.path)));
}
await Promise.all(
this.files.map(file =>
this._writeFileToCapsule(capsule, file, { overwriteExistingFile: !!opts.keepExistingCapsule })
)
);
await Promise.all(this.symlinks.map(symlink => this.atomicSymlink(capsule, symlink)));
}
async _writeFileToCapsule(capsule: Capsule, file: AbstractVinyl) {
async _writeFileToCapsule(capsule: Capsule, file: AbstractVinyl, opts = { overwriteExistingFile: false }) {
// overwriteExistingFile: if a file with the same name exists in the capsule, overwrite it
if (opts.overwriteExistingFile) {
await capsule.removePath(file.path);
return capsule.outputFile(file.path, file.contents, {});
}
if (file.override === false) {
// @todo, capsule hack. use capsule.fs once you get it as a component.
// @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
60 changes: 41 additions & 19 deletions src/environment/isolator.ts
Original file line number Diff line number Diff line change
@@ -2,6 +2,7 @@ import R from 'ramda';
import * as path from 'path';
import semver from 'semver';
import pMapSeries from 'p-map-series';
import { runModule } from 'librarian';
import Capsule from '../../components/core/capsule';
import createCapsule from './capsule-factory';
import Consumer from '../consumer/consumer';
@@ -22,6 +23,7 @@ import BitMap from '../consumer/bit-map';
import { getManipulateDirForComponentWithDependencies } from '../consumer/component-ops/manipulate-dir';
import GeneralError from '../error/general-error';
import { PathOsBased } from '../utils/path';
import loader from '../cli/loader';

export interface IsolateOptions {
writeToPath?: PathOsBased; // Path to write the component to
@@ -34,6 +36,7 @@ export interface IsolateOptions {
writeDists?: boolean; // Write dist files
shouldBuildDependencies?: boolean; // Build all depedencies before the isolation (used by tools like ts compiler)
installNpmPackages?: boolean; // Install the package dependencies
keepExistingCapsule?: boolean; // Do not delete the capsule after using it (useful for incremental builds)
installPeerDependencies?: boolean; // Install the peer package dependencies
verbose?: boolean; // Print more logs
excludeRegistryPrefix?: boolean; // exclude the registry prefix from the component's name in the package.json
@@ -55,19 +58,30 @@ export default class Isolator {
_npmVersionHasValidated = false;
// @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
componentRootDir: string;
constructor(capsule: Capsule, scope: Scope, consumer?: Consumer) {
dir?: string;
constructor(capsule: Capsule, scope: Scope, consumer?: Consumer, dir?: string) {
this.capsule = capsule;
this.scope = scope;
this.consumer = consumer;
this.dir = dir;
}

static async getInstance(containerType = 'fs', scope: Scope, consumer?: Consumer, dir?: string): Promise<Isolator> {
logger.debug(`Isolator.getInstance, creating a capsule with an ${containerType} container, dir ${dir || 'N/A'}`);
const capsule = await createCapsule(containerType, dir);
return new Isolator(capsule, scope, consumer);
return new Isolator(capsule, scope, consumer, dir);
}

async isolate(componentId: BitId, opts: IsolateOptions): Promise<ComponentWithDependencies> {
const loaderPrefix = `isolating component - ${componentId.name}`;
loader.setText(loaderPrefix);
const log = message => loader.setText(`${loaderPrefix}: ${message}`);
// @ts-ignore TODO: this should be part of the capsule interface
this.capsule.execNode = async (executable, args) => {
const onScriptRun = () => loader.setText(`building component - ${componentId.name}`);
// TODO: do this from the compiler/tester so that the isolator doesn't need to know if it's a builder/tester/*...
await runModule(executable, { args, cwd: this.dir, log, onScriptRun });
};
const componentWithDependencies: ComponentWithDependencies = await this._loadComponent(componentId);
if (opts.shouldBuildDependencies) {
topologicalSortComponentDependencies(componentWithDependencies);
@@ -83,6 +97,8 @@ export default class Isolator {
});
}
const writeToPath = opts.writeToPath;
// default should be true
const installNpmPackages = typeof opts.installNpmPackages === 'undefined' ? true : opts.installNpmPackages;
const concreteOpts: ManyComponentsWriterParams = {
componentsWithDependencies: [componentWithDependencies],
writeToPath,
@@ -93,7 +109,7 @@ export default class Isolator {
createNpmLinkFiles: opts.createNpmLinkFiles,
saveDependenciesAsComponents: opts.saveDependenciesAsComponents !== false,
writeDists: opts.writeDists,
installNpmPackages: !!opts.installNpmPackages, // convert to boolean
installNpmPackages,
installPeerDependencies: !!opts.installPeerDependencies, // convert to boolean
addToRootPackageJson: false,
verbose: opts.verbose,
@@ -103,34 +119,40 @@ export default class Isolator {
};
this.componentWithDependencies = componentWithDependencies;
this.manyComponentsWriter = new ManyComponentsWriter(concreteOpts);
await this.writeComponentsAndDependencies();
await this.installComponentPackages();
await this.writeLinks();
await this.writeComponentsAndDependencies({ keepExistingCapsule: !!opts.keepExistingCapsule });
await this.installComponentPackages({
installNpmPackages,
keepExistingCapsule: !!opts.keepExistingCapsule
});
await this.writeLinks({ keepExistingCapsule: !!opts.keepExistingCapsule });
this.capsuleBitMap = this.manyComponentsWriter.bitMap;
return componentWithDependencies;
}

async writeComponentsAndDependencies() {
async writeComponentsAndDependencies(opts = { keepExistingCapsule: false }) {
logger.debug('ManyComponentsWriter, writeAllToIsolatedCapsule');
this._manipulateDir();
await this.manyComponentsWriter._populateComponentsFilesToWrite();
await this.manyComponentsWriter._populateComponentsDependenciesToWrite();
await this._persistComponentsDataToCapsule();
await this._persistComponentsDataToCapsule({ keepExistingCapsule: !!opts.keepExistingCapsule });
}

async installComponentPackages() {
async installComponentPackages(opts = { installNpmPackages: true, keepExistingCapsule: false }) {
// @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
this.capsulePackageJson = this.componentWithDependencies.component.packageJsonFile;
// @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
this.componentRootDir = this.componentWithDependencies.component.writtenPath;
await this._addComponentsToRoot();
await this._addComponentsToRoot({ keepExistingCapsule: !!opts.keepExistingCapsule });
logger.debug('ManyComponentsWriter, install packages on capsule');
await this._installWithPeerOption();
if (opts.installNpmPackages) {
await this._installWithPeerOption();
}
}

async writeLinks() {
async writeLinks(opts = { keepExistingCapsule: false }) {
const links = await this.manyComponentsWriter._getAllLinks();
await links.persistAllToCapsule(this.capsule);
// links is a DataToPersist instance
await links.persistAllToCapsule(this.capsule, { keepExistingCapsule: !!opts.keepExistingCapsule });
}

/**
@@ -175,14 +197,14 @@ export default class Isolator {
return loadFlattenedDependenciesForCapsule(consumer, component);
}

async _persistComponentsDataToCapsule() {
async _persistComponentsDataToCapsule(opts = { keepExistingCapsule: false }) {
const dataToPersist = new DataToPersist();
const allComponents = [this.componentWithDependencies.component, ...this.componentWithDependencies.allDependencies];
allComponents.forEach(component => dataToPersist.merge(component.dataToPersist));
await dataToPersist.persistAllToCapsule(this.capsule);
await dataToPersist.persistAllToCapsule(this.capsule, { keepExistingCapsule: !!opts.keepExistingCapsule });
}

async _addComponentsToRoot(): Promise<void> {
async _addComponentsToRoot(opts = { keepExistingCapsule: false }): Promise<void> {
// @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
// @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
const capsulePath = this.capsule.container.getPath();
@@ -203,13 +225,13 @@ export default class Isolator {
}, {});
if (R.isEmpty(componentsToAdd)) return;
this.capsulePackageJson.addDependencies(componentsToAdd);
await this._writeCapsulePackageJson();
await this._writeCapsulePackageJson({ keepExistingCapsule: !!opts.keepExistingCapsule });
}

async _writeCapsulePackageJson() {
async _writeCapsulePackageJson(opts = { keepExistingCapsule: false }) {
const dataToPersist = new DataToPersist();
dataToPersist.addFile(this.capsulePackageJson.toVinylFile());
dataToPersist.persistAllToCapsule(this.capsule);
return dataToPersist.persistAllToCapsule(this.capsule, { keepExistingCapsule: !!opts.keepExistingCapsule });
}

async _getNpmVersion() {

0 comments on commit 2e2c078

Please sign in to comment.