feat: export registry (#3842)

* feat: registry first iteration

Signed-off-by: Sami Mazouz <sychocouldy@gmail.com>

* feat: improve webpack auto export loader

Signed-off-by: Sami Mazouz <sychocouldy@gmail.com>

* chore: remove `compat` API

Signed-off-by: Sami Mazouz <sychocouldy@gmail.com>

* chore: cleanup

Signed-off-by: Sami Mazouz <sychocouldy@gmail.com>

---------

Signed-off-by: Sami Mazouz <sychocouldy@gmail.com>

js-packages/webpack-config/README.md

@@ -31,27 +31,3 @@ Add another build script to your `package.json` like the one below:
 You'll need to configure a `tsconfig.json` file to ensure your IDE sets up Typescript support correctly.
 
 For details about this, see the [`flarum/flarum-tsconfig` repository](https://github.com/flarum/flarum-tsconfig)
-
-## Options
-
-### `useExtensions`
-
-`Array<string>`, defaults to `[]`.
-
-An array of extensions whose modules should be made available. This is a shortcut to add [`externals`](https://webpack.js.org/configuration/externals/) configuration for extension modules. Imported extension modules will not be bundled, but will instead refer to the extension's exports included in the Flarum runtime (ie. `flarum.extensions["vendor/package"]`).
-
-For example, to access the Tags extension module within your extension:
-
-**forum.js**
-
-```js
-import { Tag } from '@flarum/tags/forum';
-```
-
-**webpack.config.js**
-
-```js
-module.exports = config({
-  useExtensions: ['flarum/tags'],
-});
-```

js-packages/webpack-config/package.json

@@ -1,8 +1,9 @@
 {
   "name": "flarum-webpack-config",
+  "type": "module",
   "version": "2.0.2",
   "description": "Webpack config for Flarum JS and TS transpilation.",
-  "main": "index.js",
+  "main": "src/index.cjs",
   "author": "Flarum Team",
   "license": "MIT",
   "prettier": "@flarum/prettier-config",
@@ -21,13 +22,18 @@
     "@babel/preset-typescript": "^7.18.6",
     "@babel/runtime": "^7.20.1",
     "babel-loader": "^9.1.0",
+    "loader-utils": "^1.4.0",
+    "schema-utils": "^3.0.0",
     "typescript": "^4.9.3",
     "webpack": "^5.76.0",
     "webpack-bundle-analyzer": "^4.7.0"
   },
   "devDependencies": {
-    "prettier": "^2.8.0",
-    "@flarum/prettier-config": "^1.0.0"
+    "@flarum/prettier-config": "^1.0.0",
+    "babel-jest": "^29.5.0",
+    "jest": "^29.5.0",
+    "memfs": "^3.5.3",
+    "prettier": "^2.8.0"
   },
   "scripts": {
     "dev": "echo 'skipping..'",
@@ -39,6 +45,10 @@
     "build-typings": "echo 'skipping..'",
     "post-build-typings": "echo 'skipping..'",
     "check-typings": "echo 'skipping..'",
-    "check-typings-coverage": "echo 'skipping..'"
+    "check-typings-coverage": "echo 'skipping..'",
+    "test": "jest"
+  },
+  "jest": {
+    "testEnvironment": "node"
   }
 }

js-packages/webpack-config/src/autoExportLoader.cjs

@@ -0,0 +1,185 @@
+/**
+ * Auto Export Loader
+ *
+ * This loader will automatically pick up all core and extension exports and add them to the registry.
+ */
+
+const path = require('path');
+const fs = require('fs');
+const { validate } = require('schema-utils');
+const { getOptions, interpolateName } = require('loader-utils');
+
+const optionsSchema = {
+  type: 'object',
+  properties: {
+    extension: {
+      type: 'string',
+    },
+  },
+};
+
+let namespace;
+
+function addAutoExports(source, pathToModule, moduleName) {
+  let addition = '';
+
+  const defaultExportMatches = [...source.matchAll(/export\s+?default\s(?:abstract\s)?(?:(?:function|abstract|class)\s)?([A-Za-z_]*)/gm)];
+  const defaultExport = defaultExportMatches.length ? defaultExportMatches[0][1] : null;
+
+  // In case of an index.js file that exports multiple modules
+  // we need to add the directory as a module.
+  // For an example checkout the `common/extenders/index.js` file.
+  if (moduleName === 'index') {
+    const id = pathToModule.substring(0, pathToModule.length - 1);
+
+    // Add code at the end of the file to add the file to registry
+    addition += `\nflarum.reg.add('${namespace}', '${id}', ${defaultExport})`;
+  }
+
+  // In a normal case, we do one of two things:
+  else {
+    // 1. If there is a default export, we add the module to the registry with the default export.
+    //    Example: `export default class Foo {}` will be added to the registry as `Foo`,
+    //              and can be imported using `import Foo from 'flarum/../Foo'`.
+    if (defaultExport) {
+      // Add code at the end of the file to add the file to registry
+      addition += `\nflarum.reg.add('${namespace}', '${pathToModule}${moduleName}', ${defaultExport})`;
+    }
+
+    // 2. If there is no default export, then there are named exports,
+    //    so we add the module to the registry with the map of named exports.
+    //    Example: `export class Foo {}` will be added to the registry as `{ Foo: 'Foo' }`,
+    //              and can be imported using `import { Foo } from 'flarum/../Foo'`,
+    //              (checkout the `common/utils/string.ts` file for an example).
+    else {
+      // Another two case scenarios is when using `export { A, B } from 'x'`.
+      // 2.1. If there is a default export, we add the module to the registry with the default export and ignore the named exports.
+      //      Example: `export { nanoid as default, x } from 'nanoid'` will be added to the registry as `nanoid`,
+      //                and can be imported using `import nanoid from 'flarum/../nanoid'`. x will be ignored.
+      const objectExportWithDefaultMatches = [...source.matchAll(/export\s+?{.*as\s+?default.*}\s+?from\s+?['"](.*)['"]/gm)];
+
+      if (objectExportWithDefaultMatches.length) {
+        let objectDefaultExport = null;
+
+        source = source.replace(/export\s+?{\s?([A-z_0-9]*)\s?as\s+?default.*}\s+?from\s+?['"](.*)['"]/gm, (match, defaultExport, path) => {
+          objectDefaultExport = defaultExport;
+
+          return `import { ${defaultExport} } from '${path}';\nexport default ${defaultExport}`;
+        });
+
+        addition += `\nflarum.reg.add('${namespace}', '${pathToModule}${moduleName}', ${objectDefaultExport})`;
+      }
+      // 2.2. If there is no default export, check for direct exports from other modules.
+      //      We add the module to the registry with the map of named exports.
+      //      Example: `export { A, B } from 'nanoid'` will be added to the registry as `{ A, B }`,
+      //                and can be imported using `import { A, B } from 'flarum/../nanoid'`.
+      else {
+        const exportCurlyPattern = /export\s+?{(.*)}\s+?from\s+?['"](.*)['"]/gm;
+        const namedExportMatches = [...source.matchAll(exportCurlyPattern)];
+
+        if (namedExportMatches.length) {
+          source = source.replaceAll(exportCurlyPattern, (match, names, path) => {
+            return names
+              .split(',')
+              .map((name) => `import { ${name} } from '${path}';\nexport { ${name} }`)
+              .join('\n');
+          });
+
+          // Addition to the registry is taken care of in step 2.3
+        }
+      }
+
+      // 2.3. Finally, we check for all named exports
+      //      these can be `export function|class|.. Name ..`
+      //      or `export { ... };
+      {
+        const matches = [...source.matchAll(/export\s+?(?:\* as|function|{\s*([A-z0-9, ]+)+\s?}|const|abstract\s?|class)+?\s?([A-Za-z_]*)?/gm)];
+
+        if (matches.length) {
+          const map = matches.reduce((map, match) => {
+            const names = match[1] ? match[1].split(',') : (match[2] ? [match[2]] : null);
+
+            if (!names) {
+              return map;
+            }
+
+            for (let name of names) {
+              name = name.trim();
+
+              if (name === 'interface' || name === '') {
+                continue;
+              }
+
+              map += `${name}: ${name},`;
+            }
+
+            return map;
+          }, '');
+
+          // Add code at the end of the file to add the file to registry
+          if (map) addition += `\nflarum.reg.add('${namespace}', '${pathToModule}${moduleName}', { ${map} })`;
+        }
+      }
+    }
+  }
+
+  return source + addition;
+}
+
+// Custom loader logic
+module.exports = function autoExportLoader(source) {
+  const options = getOptions(this) || {};
+
+  validate(optionsSchema, options, {
+    name: 'Flarum Webpack Loader',
+    composerPath: 'Path to the extension composer.json file',
+  });
+
+  // Ensure that composer.json is watched for changes
+  // so that the loader is run again when composer.json
+  // is updated.
+  const composerJsonPath = path.resolve(options.composerPath || '../composer.json');
+  this.addDependency(composerJsonPath);
+
+  // Get the namespace of the module to be exported
+  // the namespace is essentially just the usual extension ID.
+  if (!namespace) {
+    const composerJson = JSON.parse(fs.readFileSync(composerJsonPath, 'utf8'));
+
+    // Get the value of the 'name' property
+    namespace =
+      composerJson.name === 'flarum/core' ? 'core' : composerJson.name.replace('/flarum-ext-', '-').replace('/flarum-', '').replace('/', '-');
+  }
+
+  // Get the type of the module to be exported
+  const location = interpolateName(this, '[folder]/', {
+    context: this.rootContext || this.context,
+  });
+
+  // Get the name of module to be exported
+  const moduleName = interpolateName(this, '[name]', {
+    context: this.rootContext || this.context,
+  });
+
+  // Don't export low level files
+  if ((/(admin|forum)\/$/.test(location) && moduleName !== 'app') || /(compat|ExportRegistry|registry)$/.test(moduleName)) {
+    return source;
+  }
+
+  // Don't export index.js of common
+  if (moduleName === 'index' && location === 'common/') {
+    return source;
+  }
+
+  // Don't export extend.js of extensions
+  if (namespace !== 'core' && /extend$/.test(moduleName)) {
+    return source;
+  }
+
+  // Get the path of the module to be exported
+  // relative to the src directory.
+  // Example: src/forum/components/UserCard.js => forum/components
+  const pathToModule = this.resourcePath.replace(path.resolve(this.rootContext, 'src') + '/', '').replace(/[A-Za-z_]+\.[A-Za-z_]+$/, '');
+
+  return addAutoExports(source, pathToModule, moduleName);
+};

js-packages/webpack-config/src/index.cjs

@@ -55,7 +55,7 @@ if (useBundleAnalyzer) {
   plugins.push(new (require('webpack-bundle-analyzer').BundleAnalyzerPlugin)());
 }
 
-module.exports = function (options = {}) {
+module.exports = function () {
   return {
     // Set up entry points for each of the forum + admin apps, but only
     // if they exist.
@@ -69,12 +69,15 @@ module.exports = function (options = {}) {
 
     module: {
       rules: [
+        {
+          include: /src/, // Only apply this loader to files in the src directory
+          loader: path.resolve(__dirname, './autoExportLoader.cjs'),
+        },
         {
           // Matches .js, .jsx, .ts, .tsx
-          // See: https://regexr.com/5snjd
           test: /\.[jt]sx?$/,
           loader: require.resolve('babel-loader'),
-          options: require('./babel.config'),
+          options: require('../babel.config.cjs'),
           resolve: {
             fullySpecified: false,
           },
@@ -91,33 +94,24 @@ module.exports = function (options = {}) {
 
     externals: [
       {
-        '@flarum/core/forum': 'flarum.core',
-        '@flarum/core/admin': 'flarum.core',
         jquery: 'jQuery',
       },
 
-      (function () {
-        const externals = {};
-
-        if (options.useExtensions) {
-          for (const extension of options.useExtensions) {
-            externals['@' + extension] =
-              externals['@' + extension + '/forum'] =
-              externals['@' + extension + '/admin'] =
-                "flarum.extensions['" + extension + "']";
-          }
-        }
-
-        return externals;
-      })(),
-
-      // Support importing old-style core modules.
       function ({ request }, callback) {
+        let namespace;
+        let id;
         let matches;
         if ((matches = /^flarum\/(.+)$/.exec(request))) {
-          return callback(null, "root flarum.core.compat['" + matches[1] + "']");
+          namespace = 'core';
+          id = matches[1];
+        } else if ((matches = /^ext:([^\/]+)\/(?:flarum-(?:ext-)?)?([^\/]+)(?:\/(.+))?$/.exec(request))) {
+          namespace = `${matches[1]}-${matches[2]}`;
+          id = matches[3];
+        } else {
+          return callback();
         }
-        callback();
+
+        return callback(null, `root flarum.reg.get('${namespace}', '${id}')`);
       },
     ],
 

js-packages/webpack-config/tests/autoExportLoader.test.js

@@ -0,0 +1,54 @@
+/**
+ * @jest-environment node
+ */
+import compiler from './compiler.js';
+import 'regenerator-runtime/runtime';
+
+const compile = async (path, useFinalOutput = false) => {
+  const stats = await compiler(path);
+  return useFinalOutput
+    ? stats.finalOutput
+    : stats.toJson({
+        source: true,
+      }).modules[0].source;
+};
+
+test('A directory with index.js that exports multiple modules adds the directory as a module', async () => {
+  let output = await compile('src/common/bars/index.js', true);
+  expect(output).toContain("flarum.reg.add('flarum-framework', 'common/bars', bars)");
+
+  output = await compile('src/common/bars/Acme.js');
+  expect(output).toContain("flarum.reg.add('flarum-framework', 'common/bars/Acme', Acme)");
+
+  output = await compile('src/common/bars/Foo.js');
+  expect(output).toContain("flarum.reg.add('flarum-framework', 'common/bars/Foo', Foo)");
+});
+
+test('Simple default exports are added', async () => {
+  const output = await compile('src/common/Test.js');
+  expect(output).toContain("flarum.reg.add('flarum-framework', 'common/Test', Test)");
+});
+
+test('Named exports are added', async () => {
+  const output = await compile('src/common/foos/namedExports.js');
+  expect(output).toContain(
+    "flarum.reg.add('flarum-framework', 'common/foos/namedExports', { baz: baz,foo: foo,Bar: Bar,sasha: sasha,flarum: flarum,david: david, })"
+  );
+});
+
+test('Export as default from another module is added', async () => {
+  const output = await compile('src/common/foos/exportDefaultFrom.js', true);
+  expect(output).toContain("flarum.reg.add('flarum-framework', 'common/foos/exportDefaultFrom', potato");
+});
+
+test('Export from other modules is added', async () => {
+  const output = await compile('src/common/foos/exportFrom.js', true);
+  expect(output).toContain("flarum.reg.add('flarum-framework', 'common/foos/exportFrom', { potato: potato,franz: franz, }");
+});
+
+test('Export from with other named exports works', async () => {
+  const output = await compile('src/common/foos/exportFromWithNamedExports.js', true);
+  expect(output).toContain(
+    "flarum.reg.add('flarum-framework', 'common/foos/exportFromWithNamedExports', { potato: potato,franz: franz,baz: baz,foo: foo,Bar: Bar,sasha: sasha,forum: forum,david: david, }"
+  );
+});

js-packages/webpack-config/tests/compiler.js

@@ -0,0 +1,48 @@
+import path from 'path';
+import webpack from 'webpack';
+import { createFsFromVolume, Volume } from 'memfs';
+import * as fs from 'fs';
+
+export default (fixture, options = {}) => {
+  const compiler = webpack({
+    context: __dirname,
+    entry: `./${fixture}`,
+    output: {
+      path: path.resolve(__dirname),
+      filename: 'bundle.js',
+    },
+    module: {
+      rules: [
+        {
+          test: /\.js$/,
+          use: {
+            loader: path.resolve(__dirname, '../src/autoExportLoader.cjs'),
+            options: {
+              ...options,
+              composerPath: '../../composer.json',
+            },
+          },
+        },
+      ],
+    },
+    optimization: {
+      minimize: false,
+      minimizer: [],
+    },
+  });
+
+  compiler.outputFileSystem = createFsFromVolume(new Volume());
+  compiler.outputFileSystem.join = path.join.bind(path);
+
+  return new Promise((resolve, reject) => {
+    compiler.run((err, stats) => {
+      if (err) reject(err);
+      if (stats.hasErrors()) reject(stats.toJson().errors);
+
+      const outputFilepath = path.join(compiler.options.output.path, compiler.options.output.filename);
+      stats.finalOutput = compiler.outputFileSystem.readFileSync(outputFilepath, 'utf-8');
+
+      resolve(stats);
+    });
+  });
+};

js-packages/webpack-config/tests/src/common/Test.js

@@ -0,0 +1 @@
+export default class Test {}

js-packages/webpack-config/tests/src/common/bars/Acme.js

@@ -0,0 +1 @@
+export default class Acme {}

js-packages/webpack-config/tests/src/common/bars/Foo.js

@@ -0,0 +1 @@
+export default class Foo {}

js-packages/webpack-config/tests/src/common/bars/index.js

@@ -0,0 +1,9 @@
+import Acme from './Acme.js';
+import Foo from './Foo.js';
+
+const bars = {
+  Acme,
+  Foo,
+};
+
+export default bars;

js-packages/webpack-config/tests/src/common/foos/exportDefaultFrom.js

@@ -0,0 +1 @@
+export { potato as default } from '../support/potato.js';

js-packages/webpack-config/tests/src/common/foos/exportFrom.js

@@ -0,0 +1 @@
+export { potato, franz } from '../support/potato.js';

js-packages/webpack-config/tests/src/common/foos/exportFromWithNamedExports.js

@@ -0,0 +1,16 @@
+export { potato, franz } from '../support/potato.js';
+
+export function baz() {}
+
+export function foo() {}
+
+export class Bar {}
+
+const sasha = 'camel';
+
+export { sasha };
+
+const forum = 'Flarum';
+const david = 'david';
+
+export { forum, david };

js-packages/webpack-config/tests/src/common/foos/namedExports.js

@@ -0,0 +1,14 @@
+export function baz() {}
+
+export function foo() {}
+
+export class Bar {}
+
+const sasha = 'camel';
+
+export { sasha };
+
+const flarum = 'Flarum';
+const david = 'david';
+
+export { flarum, david };

js-packages/webpack-config/tests/src/common/support/potato.js

@@ -0,0 +1,4 @@
+const potato = 'potato';
+const franz = 'franz';
+
+export { potato, franz };