feat: Code Splitting (#3860)

* feat: configure webpack to allow splitting chunks * feat: `JsDirectoryCompiler` and expose js assets URL * chore: support es2020 dynamic importing * feat: control which URL to fetch chunks from * feat: allow showing async modals & split 'LogInModal' * feat: split `SignUpModal` * feat: allow rendering async pages & split `UserSecurityPage` * fix: module might not be listed in chunk * feat: lazy load user pages * feat: track the chunk containing each module * chore: lightly warn * chore: split `Composer` * feat: add common frontend (for split common chunks) * fix: jsDoc typing imports should be ignored * feat: split `PostStream` `ForgotPasswordModal` and `EditUserModal` * fix: multiple inline async imports not picked up * chore: new `common` frontend assets only needs a jsdir compiler * feat: add revision hash to chunk import URL * fix: nothing to split for `admin` frontend yet * chore: cleanup registry API * chore: throw an error in debug mode if attempting to import a non-loaded module * feat: defer `extend` & `override` until after module registration * fix: plugin not picking up on all module sources * fix: must override default chunk loader function from webpack plugin * feat: split tags `TagDiscussionModal` and `TagSelectionModal` * fix: wrong export name * feat: import chunked modules from external packages * feat: extensions compatibility * feat: Router frontend extender async component * chore: clean JS output path (removes stale chunks) * fix: common chunks also need flushing * chore: flush backend stale chunks * Apply fixes from StyleCI * feat: loading alert when async page component is loading * chore: `yarn format` * chore: typings * chore: remove exception * Apply fixes from StyleCI * chore(infra): bundlewatch * chore(infra): bundlewatch split chunks * feat: split text editor * chore: tag typings * chore: bundlewatch * fix: windows paths * fix: wrong planned ext import format
2025-08-06 08:27:42 +02:00 · 2023-08-02 17:57:57 +01:00
parent 2ffbc44b4e
commit 229a7affa5
87 changed files with 1217 additions and 304 deletions
--- a/js-packages/webpack-config/src/OverrideChunkLoaderFunction.cjs
+++ b/js-packages/webpack-config/src/OverrideChunkLoaderFunction.cjs
@@ -0,0 +1,21 @@
+/**
+ * This plugin overrides the webpack chunk loader function `__webpack_require__.l` which is a webpack constant
+ * with `flarum.reg.loadChunk`, which resides in the flarum app.
+ */
+
+class OverrideChunkLoaderFunction {
+  apply(compiler) {
+    // We don't want to literally override its source.
+    // We want to override the function that is called by webpack.
+    // By adding a new line to reassing the function to our own function.
+    // The function is called by webpack so we can't just override it.
+    compiler.hooks.compilation.tap('OverrideChunkLoaderFunction', (compilation) => {
+      compilation.mainTemplate.hooks.requireEnsure.tap('OverrideChunkLoaderFunction', (source) => {
+        return source + '\nconst originalLoadChunk = __webpack_require__.l;\n__webpack_require__.l = flarum.reg.loadChunk.bind(flarum.reg, originalLoadChunk);';
+      });
+    });
+  }
+}
+
+
+module.exports = OverrideChunkLoaderFunction;
--- a/js-packages/webpack-config/src/RegisterAsyncChunksPlugin.cjs
+++ b/js-packages/webpack-config/src/RegisterAsyncChunksPlugin.cjs
@@ -0,0 +1,110 @@
+const path = require("path");
+const extensionId = require("./extensionId.cjs");
+const {Compilation} = require("webpack");
+
+class RegisterAsyncChunksPlugin {
+  apply(compiler) {
+    compiler.hooks.thisCompilation.tap("RegisterAsyncChunksPlugin", (compilation) => {
+      let alreadyOptimized = false;
+      compilation.hooks.unseal.tap("RegisterAsyncChunksPlugin", () => {
+        alreadyOptimized = false;
+      });
+
+      compilation.hooks.processAssets.tap(
+        {
+          name: "RegisterAsyncChunksPlugin",
+          stage: Compilation.PROCESS_ASSETS_STAGE_ADDITIONAL,
+        },
+        () => {
+          if (alreadyOptimized) return;
+          alreadyOptimized = true;
+
+          const chunks = Array.from(compilation.chunks);
+          const chunkModuleMemory = [];
+
+          const modulesToCheck = [];
+
+          for (const chunk of chunks) {
+            for (const module of compilation.chunkGraph.getChunkModulesIterable(chunk)) {
+              // A normal module.
+              if (module?.resource && module.resource.split(path.sep).includes('src') && module._source?._value.includes("webpackChunkName: ")) {
+                modulesToCheck.push(module);
+              }
+
+              // A ConcatenatedModule.
+              if (module?.modules) {
+                module.modules.forEach((module) => {
+                  if (module.resource && module.resource.split(path.sep).includes('src') && module._source?._value.includes("webpackChunkName: ")) {
+                    modulesToCheck.push(module);
+                  }
+                });
+              }
+            }
+          }
+
+          for (const module of modulesToCheck) {
+            // If the module source has an async webpack chunk, add the chunk id to flarum.reg
+            // at the end of the module source.
+
+            const reg = [];
+
+            // Each line that has a webpackChunkName comment.
+            [...module._source._value.matchAll(/.*\/\* webpackChunkName: .* \*\/.*/gm)].forEach(([match]) => {
+              [...match.matchAll(/(.*?) webpackChunkName: '([^']*)'.*? \*\/ '([^']+)'.*?/gm)]
+                .forEach(([match, _, urlPath, importPath]) => {
+                  // Import path is relative to module.resource, so we need to resolve it
+                  const importPathResolved = path.resolve(path.dirname(module.resource), importPath);
+                  const thisComposerJson = require(path.resolve(process.cwd(), '../composer.json'));
+                  const namespace = extensionId(thisComposerJson.name);
+
+                  const chunkModules = (c) => Array.from(compilation.chunkGraph.getChunkModulesIterable(c));
+
+                  const relevantChunk = chunks.find(
+                    (chunk) => chunkModules(chunk)?.find(
+                      (module) => module.resource?.split('.')[0] === importPathResolved || module.rootModule?.resource?.split('.')[0] === importPathResolved
+                    )
+                  );
+
+                  if (! relevantChunk) {
+                    console.error(`Could not find chunk for ${importPathResolved}`);
+                    return match;
+                  }
+
+                  let concatenatedModule = chunkModules(relevantChunk)[0];
+                  const moduleId = compilation.chunkGraph.getModuleId(concatenatedModule);
+                  const registrableModulesUrlPaths = new Map();
+                  registrableModulesUrlPaths.set(urlPath, [relevantChunk.id, moduleId, namespace, urlPath]);
+
+                  if (concatenatedModule?.rootModule) {
+                    // This is a chunk with many modules, we need to register all of them.
+                    concatenatedModule.modules?.forEach((module) => {
+                      // The path right after the src/ directory, without the extension.
+                      const regPathSep = `\\${path.sep}`;
+                      const urlPath = module.resource.replace(`/.*${regPathSep}src(.*)${regPathSep}\..*/`, '$1');
+
+                      if (! registrableModulesUrlPaths.has(urlPath)) {
+                        registrableModulesUrlPaths.set(urlPath, [relevantChunk.id, moduleId, namespace, urlPath]);
+                      }
+                    });
+                  }
+
+                  registrableModulesUrlPaths.forEach(([chunkId, moduleId, namespace, urlPath]) => {
+                    if (! chunkModuleMemory.includes(urlPath)) {
+                      reg.push(`flarum.reg.addChunkModule('${chunkId}', '${moduleId}', '${namespace}', '${urlPath}');`);
+                      chunkModuleMemory.push(urlPath);
+                    }
+                  });
+
+                  return match;
+                });
+            });
+
+            module._source._value += reg.join('\n');
+          }
+        }
+      );
+    });
+  }
+}
+
+module.exports = RegisterAsyncChunksPlugin;
--- a/js-packages/webpack-config/src/autoChunkNameLoader.cjs
+++ b/js-packages/webpack-config/src/autoChunkNameLoader.cjs
@@ -0,0 +1,83 @@
+const path = require("path");
+const {getOptions} = require("loader-utils");
+const {validate} = require("schema-utils");
+const fs = require("fs");
+
+const optionsSchema = {
+  type: 'object',
+  properties: {
+    extension: {
+      type: 'string',
+    },
+  },
+};
+
+let namespace;
+
+module.exports = function autoChunkNameLoader(source) {
+  const options = getOptions(this) || {};
+
+  validate(optionsSchema, options, {
+    name: 'Flarum Webpack Loader',
+    composerPath: 'Path to the extension composer.json file',
+  });
+
+  // Ensure that composer.json is watched for changes
+  // so that the loader is run again when composer.json
+  // is updated.
+  const composerJsonPath = path.resolve(options.composerPath || '../composer.json');
+  this.addDependency(composerJsonPath);
+
+  // Get the namespace of the module to be exported
+  // the namespace is essentially just the usual extension ID.
+  if (!namespace) {
+    const composerJson = JSON.parse(fs.readFileSync(composerJsonPath, 'utf8'));
+
+    // Get the value of the 'name' property
+    namespace =
+      composerJson.name === 'flarum/core' ? 'core' : composerJson.name.replace('/flarum-ext-', '-').replace('/flarum-', '').replace('/', '-');
+  }
+
+  // Get the absolute path to this module
+  const pathToThisModule = this.resourcePath;
+
+  // Find all lines that have an async import.
+  source = source.replaceAll(/^.*import\(['"].*['"]\).*$/gm, (match) => {
+    // Skip if this is inside a jsDoc comment.
+    if (/^\s*\*\s*@.*/.test(match)) {
+      return match;
+    }
+
+    // In this line.
+    // Replace all `import('path/to/module')` with `import(/* webpackChunkName: "relative/path/to/module/from/src" */ 'relative/path/to/module')`.
+    // Or, if attempting to import an external (from core or an extension) replace with a call to the right method that will compute the URL.
+    return match.replaceAll(/(.*?)import\(['"]([^'"]*)['"]\)/gm, (match, pre, relativePathToImport) => {
+      const externalImport = relativePathToImport.match(/^(flarum\/|ext:)/);
+
+      if (externalImport) {
+        return `${pre}flarum.reg.asyncModuleImport('${relativePathToImport}')`;
+      } else {
+        // Compute the absolute path from src to the module being imported
+        // based on the path of the file being imported from.
+        const absolutePathToImport = path.resolve(path.dirname(pathToThisModule), relativePathToImport);
+        let chunkPath = relativePathToImport;
+
+        if (absolutePathToImport.includes('src')) {
+          chunkPath = absolutePathToImport.split('src/')[1];
+        }
+
+        const webpackCommentOptions = {
+          webpackChunkName: chunkPath,
+          webpackMode: 'lazy-once',
+        };
+
+        const comment = Object.entries(webpackCommentOptions).map(([key, value]) => `${key}: '${value}'`).join(', ');
+
+        // Return the new import statement
+        return `${pre}import(/* ${comment} */ '${relativePathToImport}')`;
+      }
+    });
+  });
+
+  return source;
+};
--- a/js-packages/webpack-config/src/autoExportLoader.cjs
+++ b/js-packages/webpack-config/src/autoExportLoader.cjs
@@ -8,6 +8,7 @@ const path = require('path');
 const fs = require('fs');
 const { validate } = require('schema-utils');
 const { getOptions, interpolateName } = require('loader-utils');
+const extensionId = require('./extensionId.cjs');

 const optionsSchema = {
  type: 'object',
@@ -33,7 +34,7 @@ function addAutoExports(source, pathToModule, moduleName) {
    const id = pathToModule.substring(0, pathToModule.length - 1);

    // Add code at the end of the file to add the file to registry
-    addition += `\nflarum.reg.add('${namespace}', '${id}', ${defaultExport})`;
+    addition += `\nflarum.reg.add('${namespace}', '${id}', ${defaultExport});`;
  }

  // In a normal case, we do one of two things:
@@ -43,7 +44,7 @@ function addAutoExports(source, pathToModule, moduleName) {
    //              and can be imported using `import Foo from 'flarum/../Foo'`.
    if (defaultExport) {
      // Add code at the end of the file to add the file to registry
-      addition += `\nflarum.reg.add('${namespace}', '${pathToModule}${moduleName}', ${defaultExport})`;
+      addition += `\nflarum.reg.add('${namespace}', '${pathToModule}${moduleName}', ${defaultExport});`;
    }

    // 2. If there is no default export, then there are named exports,
@@ -67,7 +68,7 @@ function addAutoExports(source, pathToModule, moduleName) {
          return `import { ${defaultExport} } from '${path}';\nexport default ${defaultExport}`;
        });

-        addition += `\nflarum.reg.add('${namespace}', '${pathToModule}${moduleName}', ${objectDefaultExport})`;
+        addition += `\nflarum.reg.add('${namespace}', '${pathToModule}${moduleName}', ${objectDefaultExport});`;
      }
      // 2.2. If there is no default export, check for direct exports from other modules.
      //      We add the module to the registry with the map of named exports.
@@ -117,7 +118,7 @@ function addAutoExports(source, pathToModule, moduleName) {
          }, '');

          // Add code at the end of the file to add the file to registry
-          if (map) addition += `\nflarum.reg.add('${namespace}', '${pathToModule}${moduleName}', { ${map} })`;
+          if (map) addition += `\nflarum.reg.add('${namespace}', '${pathToModule}${moduleName}', { ${map} });`;
        }
      }
    }
@@ -147,8 +148,7 @@ module.exports = function autoExportLoader(source) {
    const composerJson = JSON.parse(fs.readFileSync(composerJsonPath, 'utf8'));

    // Get the value of the 'name' property
-    namespace =
-      composerJson.name === 'flarum/core' ? 'core' : composerJson.name.replace('/flarum-ext-', '-').replace('/flarum-', '').replace('/', '-');
+    namespace = extensionId(composerJson.name);
  }

  // Get the type of the module to be exported
--- a/js-packages/webpack-config/src/extensionId.cjs
+++ b/js-packages/webpack-config/src/extensionId.cjs
@@ -0,0 +1,3 @@
+module.exports = (name) => name === 'flarum/core'
+  ? 'core'
+  : name.replace('/flarum-ext-', '-').replace('/flarum-', '').replace('/', '-')
--- a/js-packages/webpack-config/src/index.cjs
+++ b/js-packages/webpack-config/src/index.cjs
@@ -1,6 +1,8 @@
 const fs = require('fs');
 const path = require('path');
 const { NormalModuleReplacementPlugin } = require('webpack');
+const RegisterAsyncChunksPlugin = require("./RegisterAsyncChunksPlugin.cjs");
+const OverrideChunkLoaderFunction = require("./OverrideChunkLoaderFunction.cjs");

 const entryPointNames = ['forum', 'admin'];
 const entryPointExts = ['js', 'ts'];
@@ -55,6 +57,14 @@ if (useBundleAnalyzer) {
  plugins.push(new (require('webpack-bundle-analyzer').BundleAnalyzerPlugin)());
 }

+/**
+ * This plugin allows us to register each async chunk with flarum.reg.addChunk.
+ * This works hand-in-hand with the autoChunkNameLoader, which adds a comment
+ * inside each async import with the chunk name and other webpack config.
+ */
+plugins.push(new RegisterAsyncChunksPlugin());
+plugins.push(new OverrideChunkLoaderFunction());
+
 module.exports = function () {
  return {
    // Set up entry points for each of the forum + admin apps, but only
@@ -70,9 +80,13 @@ module.exports = function () {
    module: {
      rules: [
        {
-          include: /src/, // Only apply this loader to files in the src directory
+          include: /src/,
          loader: path.resolve(__dirname, './autoExportLoader.cjs'),
        },
+        {
+          include: /src/,
+          loader: path.resolve(__dirname, './autoChunkNameLoader.cjs'),
+        },
        {
          // Matches .js, .jsx, .ts, .tsx
          test: /\.[jt]sx?$/,
@@ -85,11 +99,22 @@ module.exports = function () {
      ],
    },

+    optimization: {
+      splitChunks: {
+        chunks: 'async',
+        cacheGroups: {
+          // Avoid node_modules being split into separate chunks
+          defaultVendors: false,
+        }
+      }
+    },
+
    output: {
      path: path.resolve(process.cwd(), 'dist'),
      library: 'module.exports',
      libraryTarget: 'assign',
      devtoolNamespace: require(path.resolve(process.cwd(), 'package.json')).name,
+      clean: true,
    },

    externals: [