Use Config file instead of Environment Variables (#2227)

* Use cosmiconfig to read modular configuration, while allowing Env variables to override it 

* Add configuration tests

* Change the default CDN to esm.sh

* Update documentation to reflect new configuration approach

* Use INTERNAL_ env variables to read configuration in JS files that can't call the config function

* Refactor config() logic 
Co-authored-by: Steve King <[email protected]>

.changeset/cyan-flowers-rest.md

@@ -0,0 +1,6 @@
+---
+'modular-scripts': major
+---
+
+Changed default CDN from Skypack to esm.sh as skypack is no longer actively
+maintained. Add support for configuring modular through a configuration file.

__fixtures__/test-config/.modular.js

@@ -0,0 +1,3 @@
+module.exports = {
+  useModularEsbuild: true,
+};

docs/configuration.md

@@ -4,8 +4,101 @@ nav_order: 10
 
 # Configuration
 
-Modular has minimal configuration because of its philosophy. However there is a
-set of minimum configuration required.
+Because of its philosophy, Modular has a restricted set of configurable
+behaviours. Additionally it requires some minimal configuration within the
+`package.json`s in the repository, all handled by Modular itself.
+
+## Configuration File
+
+We allow a number of Modular behaviours to be configured via a dedicated Modular
+config file, `.modular.js`, located at the root of the repository.
+
+We support the following file names/formats:
+
+- `modular` property within your `package.json`
+- `.modular.js`
+- `.modularrc`
+- `.modularrc.json`
+- `.modularrc.yaml`
+- `.modularrc.yml`
+- `.modularrc.js`
+- `.modularrc.cjs`
+- `modular.config.js`
+- `modular.config.cjs`
+
+Example `.modular.js` file contents with all configurable attributes and their
+default values:
+
+```js
+module.exports = {
+  useModularEsbuild: false,
+  externalCdnTemplate: 'https://esm.sh/[name]@[version]',
+  externalBlockList: [],
+  externalAllowList: ['**'],
+  publicUrl: '',
+  generateSourceMap: true,
+};
+```
+
+### **useModularEsbuild**
+
+**Type**: `boolean`
+
+**Default**: `false` - Uses [Webpack](https://webpack.js.org/)
+
+Use esbuild instead of default Webpack. Only affects Views and ESM Views.
+
+### **externalCdnTemplate**
+
+**Type**: `string`
+
+**Default**: `https://esm.sh/[name]@[version]` - [esm.sh](https://esm.sh/)
+
+Template to resolve the URL used to fetch packages from a CDN. Defaults to
+esm.sh. Only applies to ESM Views.
+
+### **externalBlockList**
+
+**Type**: `string[]`
+
+**Default**: `[]` - No packages
+
+Packages that should be bundled and not fetched from a CDN. We recommend
+allowing all packages to be handled by the CDN, except for particular cases
+where they would not work correctly. See
+[known-limitations](./esm-views/known-limitations.md). Defaults to none. Only
+applies to ESM Views.
+
+### **externalAllowList**
+
+**Type**: `string[]`
+
+**Default**: `[**]` - All packages
+
+Packages that should be fetched from a CDN. We recommend allowing all packages
+to be handled by the CDN, except for particular cases where they would not work
+correctly. See [known-limitations](./esm-views/known-limitations.md). Defaults
+to all packages. Only applies to ESM Views.
+
+### **publicUrl**
+
+**Type**: `string`
+
+**Default**: `''` - No Public URL
+
+Same as Create React App PUBLIC_URL. Instead of assuming the application is
+hosted in the web server's root or subpath specified by homepage in
+package.json, assets will be referenced to the URL provided.
+
+### **generateSourceMap**
+
+**Type**: `boolean`
+
+**Default**: `true`
+
+Should build process generate a source map - can be disabled for performance
+reasons. Source maps are resource heavy and can cause out of memory issue for
+large source files.
 
 ## `package.json#modular`
 

docs/esm-views/customize-bundle-strategy.md

@@ -7,15 +7,17 @@ title: Customize bundling strategy
 # Customize bundling / rewriting strategy
 
 By default, all external dependencies are rewritten to a CDN URL and none is
-bundled. This logic can be controlled using two environment variables:
-
-1. `EXTERNAL_ALLOW_LIST` is a comma-separated string that specifies which
-   dependencies are allowed to be rewritten to the CDN; if not specified, its
-   default value is `**` ( -> all dependencies are rewritten)
-2. `EXTERNAL_BLOCK_LIST` is a comma-separated string that specifies which
-   dependencies are **not** allowed to be rewritten to the CDN; if not specified
-   its default value is empty ( -> no dependency excluded, i.e. all dependencies
-   are rewritten)
+bundled. This logic can be controlled using two environment variables or by
+using a [modular configuration file](../configuration.md).:
+
+1. [`EXTERNAL_ALLOW_LIST`](../configuration.md#externalallowlist) is a
+   comma-separated string that specifies which dependencies are allowed to be
+   rewritten to the CDN; if not specified, its default value is `**` ( -> all
+   dependencies are rewritten)
+2. [`EXTERNAL_BLOCK_LIST`](../configuration.md#externalblocklist) is a
+   comma-separated string that specifies which dependencies are **not** allowed
+   to be rewritten to the CDN; if not specified its default value is empty ( ->
+   no dependency excluded, i.e. all dependencies are rewritten)
 
 The allow / block lists are parsed and processed according to this logic:
 

docs/esm-views/esm-cdn.md

@@ -27,7 +27,8 @@ evaluated only once, it plays well with stateful libraries.
 # Customise the ESM CDN
 
 You can specify a CDN template to rewrite dependencies using the environment
-variable `EXTERNAL_CDN_TEMPLATE`.
+variable [`EXTERNAL_CDN_TEMPLATE`](../configuration.md#externalcdntemplate), or
+by using a [modular configuration file](../configuration.md).
 
 For example:
 

docs/releases/4.0.x.md

@@ -9,6 +9,7 @@ title: 4.0.x
 
 - Node 18 Support
 - Updated Jest to [^29.3.1](https://github.com/facebook/jest/releases)
+- Support for a dedicated [Modular configuration file](../configuration.md)
 
 ## Breaking Changes
 
@@ -66,6 +67,13 @@ manually complete the tasks previously covered by these commands.
 - Dropped support for minor versions of Node 14.17 and Node 16 version 16.9 and
   below
   - Now support Node ^14.18.0, >=16.10.0, and >=18.0.0
+- Dropped `USE_MODULAR_WEBPACK` environment variable, as Webpack is used by
+  default. Use `USE_MODULAR_ESBUILD` env variable or `useModularEsbuild` in a
+  [modular configuration file](../configuration.md) to use esbuild
+- Changed default Content Delivery Network for ESM Views to esm.sh instead of
+  Skypack, as it is no longer actively maintained. The CDN can still be
+  configured through the `EXTERNAL_CDN_TEMPLATE` environment variable or through
+  a [modular configuration file](../configuration.md).
 
 # Merged Changes
 

packages/modular-scripts/package.json

@@ -36,6 +36,7 @@
     "@rollup/plugin-node-resolve": "13.3.0",
     "@svgr/core": "6.2.1",
     "@svgr/webpack": "6.2.1",
+    "@types/lodash": "^4.14.191",
     "@types/micromatch": "4.0.2",
     "@types/npmcli__arborist": "^5.6.0",
     "@types/yarnpkg__lockfile": "^1.1.5",
@@ -49,6 +50,7 @@
     "chalk": "4.1.2",
     "change-case": "4.1.2",
     "commander": "9.4.0",
+    "cosmiconfig": "^8.0.0",
     "cross-spawn": "7.0.3",
     "css-loader": "6.7.1",
     "css-minimizer-webpack-plugin": "3.4.1",

packages/modular-scripts/react-scripts/config/paths.js

@@ -46,7 +46,9 @@ const resolveModular = (relativePath) =>
 const publicUrlOrPath = getPublicUrlOrPath(
   process.env.NODE_ENV === 'development',
   require(resolveApp('package.json')).homepage,
-  process.env.PUBLIC_URL,
+  process.env.INTERNAL_PUBLIC_URL === ''
+    ? undefined
+    : process.env.INTERNAL_PUBLIC_URL,
 );
 
 const buildPath = path.join(modularRoot, 'dist', modularPackageName);

packages/modular-scripts/react-scripts/config/webpack.config.js

@@ -37,7 +37,7 @@ const useReactCreateRoot = getEnvironmentVariable(
 const styleImports = getEnvironmentVariable('MODULAR_STYLE_IMPORT_MAPS', []);
 
 // Source maps are resource heavy and can cause out of memory issue for large source files.
-const shouldUseSourceMap = process.env.GENERATE_SOURCEMAP !== 'false';
+const shouldUseSourceMap = process.env.INTERNAL_GENERATE_SOURCEMAP !== 'false';
 
 const imageInlineSizeLimit = parseInt(
   process.env.IMAGE_INLINE_SIZE_LIMIT || '10000',

packages/modular-scripts/react-scripts/scripts/start.js

@@ -24,13 +24,11 @@ const isCI = require('is-ci');
 const paths = require('../config/paths');
 const configFactory = require('../config/webpack.config');
 const createDevServerConfig = require('../config/webpackDevServer.config');
-
 const isInteractive = process.stdout.isTTY;
 
 // Tools like Cloud9 rely on this.
 const DEFAULT_PORT = parseInt(process.env.PORT, 10) || 3000;
 const HOST = process.env.HOST || '0.0.0.0';
-
 if (process.env.HOST) {
   log(
     chalk.cyan(

packages/modular-scripts/src/__tests__/__snapshots__/esmView.test.ts.snap

@@ -1,7 +1,7 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
 exports[`modular-scripts WHEN building a esm-view THEN matches the entrypoint snapshot 1`] = `
-"import * as t from "https://cdn.skypack.dev/[email protected]";
+"import * as t from "https://esm.sh/[email protected]";
 function e() {
   return t.createElement(
     "div",

packages/modular-scripts/src/__tests__/utils/config.test.ts

@@ -0,0 +1,51 @@
+import execa from 'execa';
+import { copyFileSync } from 'fs';
+import path from 'path';
+import { createModularTestContext } from '../../test/utils';
+import getModularRoot from '../../utils/getModularRoot';
+
+const modularRoot = getModularRoot();
+const configFixtures = path.join(modularRoot, '__fixtures__', 'test-config');
+
+/**
+ * Run modular with provided arguments in specified directory
+ */
+function modular(
+  args: string,
+  cwd: string,
+  opts: Record<string, unknown> = {},
+) {
+  return execa('yarnpkg', ['modular', ...args.split(' ')], {
+    cwd,
+    cleanup: true,
+    ...opts,
+  });
+}
+
+// Temporary test context paths set by createTempModularRepoWithTemplate()
+let tempModularRepo: string;
+
+describe('A simple modular repo with a .modular.js config file', () => {
+  beforeEach(async () => {
+    tempModularRepo = createModularTestContext();
+    await modular('add test-app --unstable-type app', tempModularRepo);
+    copyFileSync(
+      path.join(configFixtures, '.modular.js'),
+      path.join(tempModularRepo, '.modular.js'),
+    );
+  });
+  it('builds using esbuild as specified in config file', async () => {
+    const result = await modular(`build test-app --verbose`, tempModularRepo);
+    expect(result.stdout).toContain('Building with esbuild');
+    expect(result.exitCode).toBe(0);
+  });
+  it('builds using webpack if the environment variable is provided as it overrides the config', async () => {
+    const result = await modular(`build test-app --verbose`, tempModularRepo, {
+      env: {
+        USE_MODULAR_ESBUILD: 'false',
+      },
+    });
+    expect(result.stdout).toContain('Building with Webpack');
+    expect(result.exitCode).toBe(0);
+  });
+});

packages/modular-scripts/src/build/buildPackage/makeBundle.ts

@@ -20,6 +20,7 @@ import getRelativeLocation from '../../utils/getRelativeLocation';
 import createEsbuildBrowserslistTarget from '../../utils/createEsbuildBrowserslistTarget';
 
 import type { ModularPackageJson } from '@modular-scripts/modular-types';
+import { getConfig } from '../../utils/config';
 
 const outputDirectory = 'dist';
 const extensions = ['.ts', '.tsx', '.js', '.jsx'];
@@ -107,7 +108,7 @@ export async function makeBundle(
 
   const outputOptions: rollup.OutputOptions = {
     freeze: false,
-    sourcemap: true, // TODO: read this off env
+    sourcemap: getConfig('generateSourceMap'),
     sourcemapPathTransform(relativeSourcePath: string, sourceMapPath: string) {
       // make source map input files relative to the `${packagePath}/dist-${format}` within
       // the package directory

packages/modular-scripts/src/build/index.ts

@@ -37,24 +37,13 @@ import {
 } from './esbuildFileSizeReporter';
 import { getDependencyInfo } from '../utils/getDependencyInfo';
 import { isReactNewApi } from '../utils/isReactNewApi';
+import { getConfig } from '../utils/config';
 
 async function buildStandalone(
   target: string,
   type: Extract<ModularType, 'app' | 'esm-view'>,
 ) {
-  // True if there's no preference set - or the preference is for webpack.
-  const useWebpack =
-    !process.env.USE_MODULAR_WEBPACK ||
-    process.env.USE_MODULAR_WEBPACK === 'true';
-
-  // True if the preferene IS set and the preference is esbuid.
-  const useEsbuild =
-    process.env.USE_MODULAR_ESBUILD &&
-    process.env.USE_MODULAR_ESBUILD === 'true';
-
-  // If you want to use webpack then we'll always use webpack. But if you've indicated
-  // you want esbuild - then we'll switch you to the new fancy world.
-  const isEsbuild = !useWebpack || useEsbuild;
+  const isEsbuild = getConfig('useModularEsbuild');
 
   // Setup Paths
   const modularRoot = getModularRoot();
@@ -132,6 +121,7 @@ async function buildStandalone(
   let cssEntryPoint: string | undefined;
 
   if (isEsbuild) {
+    logger.debug('Building with esbuild');
     const { default: buildEsbuildApp } = await import(
       '../esbuild-scripts/build'
     );
@@ -140,6 +130,7 @@ async function buildStandalone(
     cssEntryPoint = getEntryPoint(paths, result, '.css');
     assets = createEsbuildAssets(paths, result);
   } else {
+    logger.debug('Building with Webpack');
     // create-react-app doesn't support plain module outputs yet,
     // so --preserve-modules has no effect here
 
@@ -160,6 +151,8 @@ async function buildStandalone(
         MODULAR_IS_APP: JSON.stringify(isApp),
         MODULAR_IMPORT_MAP: JSON.stringify(Object.fromEntries(importMap || [])),
         MODULAR_USE_REACT_CREATE_ROOT: JSON.stringify(useReactCreateRoot),
+        INTERNAL_PUBLIC_URL: getConfig('publicUrl'),
+        INTERNAL_GENERATE_SOURCEMAP: String(getConfig('generateSourceMap')),
       },
     });
 

packages/modular-scripts/src/esbuild-scripts/config/createEsbuildConfig.ts

@@ -9,6 +9,7 @@ import * as logger from '../../utils/logger';
 import moduleScopePlugin from '../plugins/moduleScopePlugin';
 import svgrPlugin from '../plugins/svgr';
 import workerFactoryPlugin from '../plugins/workerFactoryPlugin';
+import { getConfig } from '../../utils/config';
 
 export default function createEsbuildConfig(
   paths: Paths,
@@ -46,7 +47,7 @@ export default function createEsbuildConfig(
     resolveExtensions: paths.moduleFileExtensions.map(
       (extension) => `.${extension}`,
     ),
-    sourcemap: true,
+    sourcemap: getConfig('generateSourceMap'),
     loader: {
       // loaders for images which are supported as files
       '.avif': 'file',