From bbf062f9da6eb044aee06644b86b793345e509a5 Mon Sep 17 00:00:00 2001 From: Olivier Tassinari Date: Tue, 17 Jul 2018 18:09:37 +0200 Subject: [PATCH] [icons] New iteration --- .babelrc | 13 +- docs/src/modules/components/Notifications.js | 9 +- modules/waterfall/Batcher.js | 47 +++++++ modules/waterfall/Queue.js | 59 ++++++++ modules/waterfall/README.md | 3 + modules/waterfall/forEach.js | 8 ++ modules/waterfall/metric.js | 126 ++++++++++++++++++ modules/waterfall/retry.js | 34 +++++ modules/waterfall/sleep.js | 7 + modules/waterfall/waitUntil.js | 17 +++ packages/material-ui-icons/builder.js | 13 +- .../filters/rename/material-design-icons.js | 1 + packages/material-ui-icons/package.json | 4 +- .../material-ui-icons/scripts/download.js | 32 +++-- .../expected/Accessibility.js | 2 +- yarn.lock | 2 +- 16 files changed, 348 insertions(+), 29 deletions(-) create mode 100644 modules/waterfall/Batcher.js create mode 100644 modules/waterfall/Queue.js create mode 100644 modules/waterfall/README.md create mode 100644 modules/waterfall/forEach.js create mode 100644 modules/waterfall/metric.js create mode 100644 modules/waterfall/retry.js create mode 100644 modules/waterfall/sleep.js create mode 100644 modules/waterfall/waitUntil.js diff --git a/.babelrc b/.babelrc index 1889fd4c7f5dfa..9fe7b15219d541 100644 --- a/.babelrc +++ b/.babelrc @@ -26,7 +26,16 @@ ] }, "development": { - "sourceMaps": "both" + "plugins": [ + [ + "module-resolver", + { + "alias": { + "modules": "./modules", + } + } + ] + ] }, "docs-development": { "plugins": [ @@ -40,6 +49,7 @@ "@material-ui/icons": "./packages/material-ui-icons/src", "@material-ui/lab": "./packages/material-ui-lab/src", "docs": "./docs", + "modules": "./modules", "pages": "./pages" } } @@ -58,6 +68,7 @@ "@material-ui/icons": "./packages/material-ui-icons/src", "@material-ui/lab": "./packages/material-ui-lab/src", "docs": "./docs", + "modules": "./modules", "pages": "./pages" } } diff --git a/docs/src/modules/components/Notifications.js b/docs/src/modules/components/Notifications.js index 01eb9ffedde123..22bf276ca1ce48 100644 --- a/docs/src/modules/components/Notifications.js +++ b/docs/src/modules/components/Notifications.js @@ -4,6 +4,7 @@ import 'isomorphic-fetch'; import React from 'react'; import Button from '@material-ui/core/Button'; import Snackbar from '@material-ui/core/Snackbar'; +import sleep from 'modules/waterfall/sleep'; function getLastSeenNotification() { const seen = document.cookie.replace( @@ -13,18 +14,12 @@ function getLastSeenNotification() { return seen === '' ? 0 : parseInt(seen, 10); } -function pause(timeout) { - return new Promise(accept => { - setTimeout(accept, timeout); - }); -} - let messages = null; async function getMessages() { try { if (!messages) { - await pause(1e3); // Soften the pressure on the main thread. + await sleep(1e3); // Soften the pressure on the main thread. const result = await fetch( 'https://raw.githubusercontent.com/mui-org/material-ui/master/docs/notifications.json', ); diff --git a/modules/waterfall/Batcher.js b/modules/waterfall/Batcher.js new file mode 100644 index 00000000000000..d3fd0ee52028c9 --- /dev/null +++ b/modules/waterfall/Batcher.js @@ -0,0 +1,47 @@ +// Inspired by http://caolan.github.io/async/docs.html#cargo +// The main difference is that we have a timeout. +class Batcher { + pendingEntries = []; + + timeout = null; + + context = {}; + + constructor(worker, options = {}) { + // max waiting time before flushing the pending entries (process them) + this.maxWait = options.maxWait || 1000; + // max number of entries in the queue before flushing them (process them) + this.maxItems = options.maxItems || 100; + this.worker = worker; + } + + // public method + push(entries, contextItem) { + this.context = contextItem; + this.pendingEntries = this.pendingEntries.concat(entries); + + if (this.pendingEntries.length >= this.maxItems) { + return this.sendItems(); + } + + clearTimeout(this.timeout); + this.timeout = setTimeout(() => { + this.sendItems(); + }, this.maxWait); + + return null; + } + + sendItems() { + const pendingEntries = this.pendingEntries.splice(0); // Transfer the item to the job. + clearTimeout(this.timeout); + return this.worker(pendingEntries, this.context); + } + + clear() { + clearTimeout(this.timeout); + this.pendingEntries = []; + } +} + +export default Batcher; diff --git a/modules/waterfall/Queue.js b/modules/waterfall/Queue.js new file mode 100644 index 00000000000000..23b858b7626a60 --- /dev/null +++ b/modules/waterfall/Queue.js @@ -0,0 +1,59 @@ +import waitUntil from './waitUntil'; + +class Queue { + pendingEntries = []; + + inFlight = 0; + + err = null; + + constructor(worker, options = {}) { + this.worker = worker; + this.concurrency = options.concurrency || 1; + } + + push = entries => { + this.pendingEntries = this.pendingEntries.concat(entries); + this.process(); + }; + + process = () => { + const scheduled = this.pendingEntries.splice(0, this.concurrency - this.inFlight); + this.inFlight += scheduled.length; + scheduled.forEach(async task => { + try { + await this.worker(task); + } catch (err) { + this.err = err; + } finally { + this.inFlight -= 1; + } + + if (this.pendingEntries.length > 0) { + this.process(); + } + }); + }; + + wait = (options = {}) => { + return waitUntil( + () => { + if (this.err) { + this.pendingEntries = []; + throw this.err; + } + + return { + predicate: options.empty + ? this.inFlight === 0 && this.pendingEntries.length === 0 + : this.concurrency > this.pendingEntries.length, + }; + }, + { + delay: 50, + }, + ); + }; +} + +export default Queue; diff --git a/modules/waterfall/README.md b/modules/waterfall/README.md new file mode 100644 index 00000000000000..374e3c31bd9707 --- /dev/null +++ b/modules/waterfall/README.md @@ -0,0 +1,3 @@ +# Waterfall + +A set of utility functions for handling async/await at scale. diff --git a/modules/waterfall/forEach.js b/modules/waterfall/forEach.js new file mode 100644 index 00000000000000..ad11eed70dcd27 --- /dev/null +++ b/modules/waterfall/forEach.js @@ -0,0 +1,8 @@ +async function forEach(array, iteratee) { + for (let i = 0; i < array.length; i += 1) { + // eslint-disable-next-line no-await-in-loop + await iteratee(array[i], i); + } +} + +export default forEach; diff --git a/modules/waterfall/metric.js b/modules/waterfall/metric.js new file mode 100644 index 00000000000000..c01482242b98e9 --- /dev/null +++ b/modules/waterfall/metric.js @@ -0,0 +1,126 @@ +// The API is inspired by console.time +// The implementation is isomorphic. +import warning from 'warning'; + +const times = new Map(); + +const implementations = { + mark: { + start: name => { + times.set(name, performance.now()); + performance.mark(`metric_${name}_start`); + }, + end: name => { + const endMark = `metric_${name}_end`; + performance.mark(endMark); + const startMark = `metric_${name}_start`; + performance.measure(name, startMark, endMark); + const duration = performance.getEntriesByName(name)[0].duration; + return duration; + }, + }, + now: { + start: name => { + times.set(name, performance.now()); + }, + end: name => { + const time = times.get(name); + const duration = performance.now() - time; + return duration; + }, + }, + hrtime: { + start: name => { + // https://nodejs.org/api/process.html#process_process_hrtime_time + times.set(name, process.hrtime()); + }, + end: name => { + const time = times.get(name); + const durations = process.hrtime(time); + const duration = durations[0] / 1e3 + durations[1] / 1e6; + return duration; + }, + }, +}; + +let getImplementationCache; + +function getImplementation() { + if (getImplementationCache) { + return getImplementationCache; + } + + if (typeof performance !== 'undefined' && performance.mark) { + getImplementationCache = implementations.mark; + } else if (typeof performance !== 'undefined' && performance.now) { + getImplementationCache = implementations.now; + } else if (process.hrtime) { + getImplementationCache = implementations.hrtime; + } else { + throw new Error('No performance API available'); + } + + return getImplementationCache; +} + +class Metric { + /** + * Call to begin a measurement. + */ + static start(name) { + warning(!times.get(name), 'Recording already started'); + getImplementation().start(name); + } + + /** + * Returns the duration of the timing metric. The unit is milliseconds. + * @type {number} + */ + static end(name) { + if (!times.get(name)) { + throw new Error(`No such name '${name}' for metric`); + } + + const duration = getImplementation().end(name); + times.delete(name); + return duration; + } + + name = ''; + + /** + * @param {string} name A name for the metric. + */ + constructor(name) { + if (!name) { + throw new Error('Please provide a metric name'); + } + + this.name = name; + } + + /** + * Call to begin a measurement. + */ + start(name) { + if (name) { + throw new Error('The name argument is not supported'); + } + + Metric.start(this.name); + } + + /** + * Returns the duration of the timing metric. The unit is milliseconds. + * @type {number} + */ + end(name) { + if (name) { + throw new Error('The name argument is not supported'); + } + + return Metric.end(this.name); + } +} + +export default Metric; diff --git a/modules/waterfall/retry.js b/modules/waterfall/retry.js new file mode 100644 index 00000000000000..5bda24bf9ecaad --- /dev/null +++ b/modules/waterfall/retry.js @@ -0,0 +1,34 @@ +// Inspired by https://github.com/zeit/async-retry +// Without the retry dependency (1 kB gzipped +) +async function retry(tryFunction, options = {}) { + const { retries = 3 } = options; + + let tries = 0; + let output = null; + let exitErr = null; + + const bail = err => { + exitErr = err; + }; + + while (tries < retries) { + tries += 1; + try { + // eslint-disable-next-line no-await-in-loop + output = await tryFunction({ tries, bail }); + break; + } catch (err) { + if (tries >= retries) { + throw err; + } + } + } + + if (exitErr) { + throw exitErr; + } + + return output; +} + +export default retry; diff --git a/modules/waterfall/sleep.js b/modules/waterfall/sleep.js new file mode 100644 index 00000000000000..8c8a12bb7d5a68 --- /dev/null +++ b/modules/waterfall/sleep.js @@ -0,0 +1,7 @@ +function sleep(delay = 0) { + return new Promise(resolve => { + setTimeout(resolve, delay); + }); +} + +export default sleep; diff --git a/modules/waterfall/waitUntil.js b/modules/waterfall/waitUntil.js new file mode 100644 index 00000000000000..42bc466d19c5c2 --- /dev/null +++ b/modules/waterfall/waitUntil.js @@ -0,0 +1,17 @@ +import sleep from './sleep'; + +export default async function waitUntil(test, options = {}) { + const { delay = 5e3, tries = -1 } = options; + const { predicate, result } = await test(); + + if (predicate) { + return result; + } + + if (tries - 1 === 0) { + throw new Error('tries limit reached'); + } + + await sleep(delay); + return waitUntil(test, { ...options, tries: tries > 0 ? tries - 1 : tries }); +} diff --git a/packages/material-ui-icons/builder.js b/packages/material-ui-icons/builder.js index d74b8fbb16ede5..a19d50ca9e996a 100755 --- a/packages/material-ui-icons/builder.js +++ b/packages/material-ui-icons/builder.js @@ -5,6 +5,7 @@ import yargs from 'yargs'; import path from 'path'; import rimraf from 'rimraf'; import Mustache from 'mustache'; +import util from 'util'; import glob from 'glob'; import mkdirp from 'mkdirp'; import SVGO from 'svgo'; @@ -12,6 +13,8 @@ import SVGO from 'svgo'; const RENAME_FILTER_DEFAULT = './filters/rename/default'; const RENAME_FILTER_MUI = './filters/rename/material-design-icons'; +const globAsync = util.promisify(glob); + const svgo = new SVGO({ plugins: [ { cleanupAttrs: true }, @@ -134,9 +137,9 @@ async function processFile(svgPath, destPath, options) { * make index.js, it exports all of SVGIcon classes. * @param {object} options */ -function processIndex(options) { - const index = glob - .sync(path.join(options.outputDir, '*.js')) +async function processIndex(options) { + const files = await globAsync(path.join(options.outputDir, '*.js')); + const index = files .map(file => { const typename = path.basename(file).replace('.js', ''); return `export { default as ${typename} } from './${typename}';\n`; @@ -179,7 +182,7 @@ async function main(options) { if (!fs.existsSync(options.outputDir)) { fs.mkdirSync(options.outputDir); } - const files = glob.sync(path.join(options.svgDir, options.glob)); + const files = await globAsync(path.join(options.svgDir, options.glob)); await asyncForEach(files, async svgPath => { const svgPathObj = path.parse(svgPath); @@ -192,7 +195,7 @@ async function main(options) { await processFile(svgPath, destPath, options); }); - processIndex(options); + await processIndex(options); if (options.disableLog) { // bring back stdout diff --git a/packages/material-ui-icons/filters/rename/material-design-icons.js b/packages/material-ui-icons/filters/rename/material-design-icons.js index 5a90699d9e24b3..c2ed9048cddf1b 100644 --- a/packages/material-ui-icons/filters/rename/material-design-icons.js +++ b/packages/material-ui-icons/filters/rename/material-design-icons.js @@ -4,6 +4,7 @@ function myDestRewriter(svgPathObj: Object) { let fileName = svgPathObj.base; fileName = fileName + .slice(3) .replace('_24px.svg', '.js') .replace(/(^.)|(_)(.)/g, (match, p1, p2, p3) => (p1 || p3).toUpperCase()); diff --git a/packages/material-ui-icons/package.json b/packages/material-ui-icons/package.json index 931474081d41cd..f7afa2e951e2d9 100644 --- a/packages/material-ui-icons/package.json +++ b/packages/material-ui-icons/package.json @@ -25,7 +25,7 @@ "test": "yarn test:unit", "test:unit": "cd ../../ && ./node_modules/.bin/cross-env NODE_ENV=test ./node_modules/.bin/mocha packages/material-ui-icons/{,**/}*.test.js", "test:watch": "yarn test:unit --watch", - "download": "../../node_modules/.bin/babel-node ./scripts/download.js", + "download": "cd ../../ && ./node_modules/.bin/babel-node packages/material-ui-icons/scripts/download.js", "src:icons": "../../node_modules/.bin/babel-node ./builder.js --output-dir ./src --svg-dir ./material-design-icons --glob '/**/*_24px.svg' --renameFilter ./filters/rename/material-design-icons.js", "prebuild": "../../node_modules/.bin/rimraf material-design-icons && ../../node_modules/.bin/rimraf build", "build:es2015": "../../node_modules/.bin/cross-env NODE_ENV=production ../../node_modules/.bin/babel ./src --out-dir ./build", @@ -43,7 +43,7 @@ "react-dom": "^16.3.0" }, "dependencies": { - "recompose": "^0.26.0 || ^0.27.0" + "recompose": "^0.27.0" }, "devDependencies": { "fs-extra": "^6.0.1", diff --git a/packages/material-ui-icons/scripts/download.js b/packages/material-ui-icons/scripts/download.js index 17d47b6ad9d6c6..87fee85953705f 100644 --- a/packages/material-ui-icons/scripts/download.js +++ b/packages/material-ui-icons/scripts/download.js @@ -1,7 +1,10 @@ /* eslint-disable no-console */ import fse from 'fs-extra'; +import path from 'path'; import yargs from 'yargs'; +import Queue from 'modules/waterfall/Queue'; +import retry from 'modules/waterfall/retry'; import 'isomorphic-fetch'; const themeMap = { @@ -38,15 +41,8 @@ const sizes = { }, }; -async function asyncForEach(array, callback, startAfter = 0) { - for (let index = startAfter; index < array.length; index += 1) { - // eslint-disable-next-line no-await-in-loop - await callback(array[index], index); - } -} - -function downloadIcon(icon, index) { - console.log(`downloadIcon ${index}: ${icon.id}`); +function downloadIcon(icon) { + console.log(`downloadIcon ${icon.index}: ${icon.id}`); return Promise.all( Object.keys(themeMap).map(async theme => { @@ -56,7 +52,7 @@ function downloadIcon(icon, index) { ); const SVG = await response.text(); await fse.writeFile( - `./material-design-icons/${icon.id}${themeMap[theme]}_${size}px.svg`, + path.join(__dirname, `../material-design-icons/ic_${icon.id}${themeMap[theme]}_${size}px.svg`), SVG, ); }), @@ -72,11 +68,23 @@ async function run() { await fse.ensureDir('material-design-icons'); const response = await fetch('https://material.io/tools/icons/static/data.json'); const data = await response.json(); - const icons = data.categories.reduce((acc, item) => { + let icons = data.categories.reduce((acc, item) => { return acc.concat(item.icons); }, []); + icons = icons.map((icon, index) => ({ index, ...icon })); + icons = icons.splice(argv.startAfter || 0); console.log(`${icons.length} icons to download`); - await asyncForEach(icons, downloadIcon, argv.startAfter); + + const queue = new Queue( + async icon => { + await retry(async () => { + await downloadIcon(icon); + }); + }, + { concurrency: 10 }, + ); + queue.push(icons); + await queue.wait({ empty: true }); } catch (err) { console.log('err', err); throw err; diff --git a/packages/material-ui-icons/test/fixtures/material-design-icons/expected/Accessibility.js b/packages/material-ui-icons/test/fixtures/material-design-icons/expected/Accessibility.js index 5925ec5dcf8e75..6bdc1b1539b0d0 100644 --- a/packages/material-ui-icons/test/fixtures/material-design-icons/expected/Accessibility.js +++ b/packages/material-ui-icons/test/fixtures/material-design-icons/expected/Accessibility.js @@ -2,5 +2,5 @@ import React from 'react'; import createSvgIcon from './utils/createSvgIcon'; export default createSvgIcon( - + , 'Accessibility'); diff --git a/yarn.lock b/yarn.lock index 0573b1eff4bd71..2245bee2cf6fc2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8639,7 +8639,7 @@ recast@^0.15.0: private "~0.1.5" source-map "~0.6.1" -"recompose@^0.26.0 || ^0.27.0", recompose@^0.27.0: +recompose@^0.27.0: version "0.27.1" resolved "https://registry.yarnpkg.com/recompose/-/recompose-0.27.1.tgz#1a49e931f183634516633bbb4f4edbfd3f38a7ba" dependencies: