From d773077fc9a4cfa1e533d3d352f9aef2340e7ad5 Mon Sep 17 00:00:00 2001 From: develar Date: Fri, 29 Jul 2016 19:10:53 +0200 Subject: [PATCH] feat: Make sure compiled app has same hash on different computers Closes #604 --- src/asarUtil.ts | 173 ++++++++++++++++++++++----------- typings/read-package-json.d.ts | 5 - 2 files changed, 118 insertions(+), 60 deletions(-) delete mode 100644 typings/read-package-json.d.ts diff --git a/src/asarUtil.ts b/src/asarUtil.ts index f88666af67d..4ae2460218a 100644 --- a/src/asarUtil.ts +++ b/src/asarUtil.ts @@ -1,6 +1,6 @@ import { AsarFileInfo, listPackage, statFile, AsarOptions } from "asar-electron-builder" -import { statOrNull } from "./util/util" -import { lstat, readdir, readFile, Stats, createWriteStream, ensureDir, createReadStream } from "fs-extra-p" +import { statOrNull, debug } from "./util/util" +import { lstat, readdir, readFile, Stats, createWriteStream, ensureDir, createReadStream, readJson } from "fs-extra-p" import { Promise as BluebirdPromise } from "bluebird" import * as path from "path" import pathSorter = require("path-sort") @@ -14,7 +14,8 @@ const Filesystem = require("asar-electron-builder/lib/filesystem") //noinspection JSUnusedLocalSymbols const __awaiter = require("./util/awaiter") -const concurrency = {concurrency: 50} +const MAX_FILE_REQUESTS = 32 +const concurrency = {concurrency: MAX_FILE_REQUESTS} const NODE_MODULES_PATTERN = path.sep + "node_modules" + path.sep function walk(dirPath: string, consumer: (file: string, stat: Stats) => void, filter: (file: string) => boolean, addRootToResult?: boolean): BluebirdPromise> { @@ -122,72 +123,100 @@ async function order(src: string, filenames: Array, options: any) { return filenamesSorted } -async function createPackageFromFiles(src: string, dest: string, files: Array, metadata: Map, options: any) { - // search auto unpacked dir - const autoUnpackDirs = new Set() - - const createDirPromises: Array> = [ensureDir(path.dirname(dest))] - const unpackedDest = `${dest}.unpacked` - - if (options.smartUnpack !== false) { - for (let file of files) { - const index = file.lastIndexOf(NODE_MODULES_PATTERN) - if (index < 0) { - continue - } +async function detectUnpackedDirs(src: string, files: Array, metadata: Map, autoUnpackDirs: Set, createDirPromises: Array>, unpackedDest: string, packageFileToData: Map>) { + const packageJsonStringLength = "package.json".length + const readPackageJsonPromises: Array> = [] + for (let file of files) { + const index = file.lastIndexOf(NODE_MODULES_PATTERN) + if (index < 0) { + continue + } - const nextSlashIndex = file.indexOf(path.sep, index + NODE_MODULES_PATTERN.length + 1) - if (nextSlashIndex < 0) { - continue - } + const nextSlashIndex = file.indexOf(path.sep, index + NODE_MODULES_PATTERN.length + 1) + if (nextSlashIndex < 0) { + continue + } - if (!metadata.get(file)!.isFile()) { - continue - } + if (!metadata.get(file)!.isFile()) { + continue + } - const nodeModuleDir = file.substring(0, nextSlashIndex) - if (autoUnpackDirs.has(nodeModuleDir)) { - const fileParent = path.dirname(file) - if (fileParent != nodeModuleDir && !autoUnpackDirs.has(fileParent)) { - autoUnpackDirs.add(fileParent) - createDirPromises.push(ensureDir(path.join(unpackedDest, path.relative(src, fileParent)))) - } - continue - } + const nodeModuleDir = file.substring(0, nextSlashIndex) - const ext = path.extname(file) - let shouldUnpack = false - if (ext === ".dll" || ext === ".exe") { - shouldUnpack = true - } - else if (ext === "") { - shouldUnpack = await isBinaryFile(file) - } + if (file.length == (nodeModuleDir.length + 1 + packageJsonStringLength) && file.endsWith("package.json")) { + const promise = readJson(file) - if (!shouldUnpack) { - continue + if (readPackageJsonPromises.length > MAX_FILE_REQUESTS) { + await BluebirdPromise.all(readPackageJsonPromises) + readPackageJsonPromises.length = 0 } + readPackageJsonPromises.push(promise) + packageFileToData.set(file, promise) + } - log(`${path.relative(src, nodeModuleDir)} is not packed into asar archive - contains executable code`) - autoUnpackDirs.add(nodeModuleDir) + if (autoUnpackDirs.has(nodeModuleDir)) { const fileParent = path.dirname(file) - if (fileParent != nodeModuleDir) { + if (fileParent != nodeModuleDir && !autoUnpackDirs.has(fileParent)) { autoUnpackDirs.add(fileParent) - // create parent dir to be able to copy file later without directory existence check + + if (createDirPromises.length > MAX_FILE_REQUESTS) { + await BluebirdPromise.all(createDirPromises) + createDirPromises.length = 0 + } createDirPromises.push(ensureDir(path.join(unpackedDest, path.relative(src, fileParent)))) } + continue } - } - const unpackDir = options.unpackDir == null ? null : new Minimatch(options.unpackDir) - const unpack = options.unpack == null ? null : new Minimatch(options.unpack, { - matchBase: true - }) + const ext = path.extname(file) + let shouldUnpack = false + if (ext === ".dll" || ext === ".exe") { + shouldUnpack = true + } + else if (ext === "") { + shouldUnpack = await isBinaryFile(file) + } + + if (!shouldUnpack) { + continue + } + log(`${path.relative(src, nodeModuleDir)} is not packed into asar archive - contains executable code`) + autoUnpackDirs.add(nodeModuleDir) + const fileParent = path.dirname(file) + if (fileParent != nodeModuleDir) { + autoUnpackDirs.add(fileParent) + // create parent dir to be able to copy file later without directory existence check + createDirPromises.push(ensureDir(path.join(unpackedDest, path.relative(src, fileParent)))) + } + } + + if (readPackageJsonPromises.length > 0) { + await BluebirdPromise.all(readPackageJsonPromises) + } if (createDirPromises.length > 0) { await BluebirdPromise.all(createDirPromises) createDirPromises.length = 0 } +} + +async function createPackageFromFiles(src: string, dest: string, files: Array, metadata: Map, options: any) { + // search auto unpacked dir + const autoUnpackDirs = new Set() + + const createDirPromises: Array> = [ensureDir(path.dirname(dest))] + const unpackedDest = `${dest}.unpacked` + const changedFiles = new Map() + + const packageFileToData = new Map>() + if (options.smartUnpack !== false) { + await detectUnpackedDirs(src, files, metadata, autoUnpackDirs, createDirPromises, unpackedDest, packageFileToData) + } + + const unpackDir = options.unpackDir == null ? null : new Minimatch(options.unpackDir) + const unpack = options.unpack == null ? null : new Minimatch(options.unpack, { + matchBase: true + }) const toPack: Array = [] const filesystem = new Filesystem(src) @@ -217,7 +246,7 @@ async function createPackageFromFiles(src: string, dest: string, files: Array 50) { + if (copyPromises.length > MAX_FILE_REQUESTS) { await BluebirdPromise.all(copyPromises) copyPromises.length = 0 } @@ -226,6 +255,11 @@ async function createPackageFromFiles(src: string, dest: string, files: Array): Promise { +function cleanupPackageJson(file: string, stat: Stats, data: any, changedFiles: Map) { + try { + let writeFile = false + for (let prop of Object.getOwnPropertyNames(data)) { + if (prop[0] === "_" || prop === "dist" || prop === "gitHead" || prop === "keywords") { + delete data[prop] + writeFile = true + } + } + + if (writeFile) { + const value = JSON.stringify(data, null, 2) + changedFiles.set(file, value) + stat.size = Buffer.byteLength(value) + } + } + catch (e) { + debug(e) + } +} + +function writeAsarFile(filesystem: any, dest: string, toPack: Array, changedFiles: Map): Promise { const headerPickle = pickle.createEmpty() headerPickle.writeString(JSON.stringify(filesystem.header)) const headerBuf = headerPickle.toBuffer() @@ -283,7 +338,15 @@ function writeAsarFile(filesystem: any, dest: string, toPack: Array): Pr return } - const readStream = createReadStream(list[index]) + const file = list[index] + + const data = changedFiles.get(file) + if (data != null) { + writeStream.write(data, () => w(list, index + 1)) + return + } + + const readStream = createReadStream(file) readStream.on("error", reject) readStream.once("end", () => w(list, index + 1)) readStream.pipe(writeStream, { diff --git a/typings/read-package-json.d.ts b/typings/read-package-json.d.ts deleted file mode 100644 index a8cee7c282d..00000000000 --- a/typings/read-package-json.d.ts +++ /dev/null @@ -1,5 +0,0 @@ -declare module "read-package-json" { - function readJson(file: string, callback: (error: Error, data: any) => void): void - - export = readJson -} \ No newline at end of file