Browse Source

Consolidate and simplify core webpack build processes.

pull/4524/head
Richard Tibbles 3 years ago
parent
commit
84f9c1f557
No known key found for this signature in database GPG Key ID: C131EA1E79A12C5C
  1. 1
      build_tools/build_plugins.txt
  2. 4
      kolibri/core/assets/src/core-app/constructor.js
  3. 3
      kolibri/core/package.json
  4. 2
      kolibri/core/webpack.config.js
  5. 5
      packages/kolibri-tools/.eslintrc.js
  6. 28
      packages/kolibri-tools/lib/apiSpecExportTools.js
  7. 45
      packages/kolibri-tools/lib/bundleStats.js
  8. 185
      packages/kolibri-tools/lib/cli.js
  9. 49
      packages/kolibri-tools/lib/i18n.js
  10. 149
      packages/kolibri-tools/lib/parse_bundle_plugin.js
  11. 56
      packages/kolibri-tools/lib/production.js
  12. 149
      packages/kolibri-tools/lib/read_bundle_plugins.js
  13. 303
      packages/kolibri-tools/lib/webpack.config.base.js
  14. 28
      packages/kolibri-tools/lib/webpack.config.dev.js
  15. 17
      packages/kolibri-tools/lib/webpack.config.js
  16. 25
      packages/kolibri-tools/lib/webpack.config.prod.js
  17. 20
      packages/kolibri-tools/lib/webpack.config.trs.js
  18. 146
      packages/kolibri-tools/lib/webpackdevserver.js
  19. 11
      packages/kolibri-tools/lib/webpackdevserverconfig.js
  20. 70
      packages/kolibri-tools/test/test_webpack.config.base.spec.js
  21. 6
      packages/publish.js

1
build_tools/build_plugins.txt

@ -1,3 +1,2 @@
kolibri.core
kolibri.plugins.*
kolibri_exercise_perseus_plugin

4
kolibri/core/assets/src/core-app/constructor.js

@ -44,10 +44,6 @@ export default class CoreApp {
// Assign API spec
Object.assign(this, apiSpec);
// Assign any overridden core API elements here
// Use the default object if it has been specified using an ES6 default export.
merge(this, __coreAPISpec.default || __coreAPISpec);
const mediator = new Mediator();
Vue.prototype.Kolibri = this;

3
kolibri/core/package.json

@ -32,5 +32,8 @@
"vue-popperjs":"^1.5.4",
"vue-router": "^2.1.1",
"vuex": "^3.0.1"
},
"devDependencies": {
"kolibri-tools": "0.12.0-dev.1"
}
}

2
kolibri/core/webpack.config.js

@ -1,5 +1,5 @@
// This is only used during build time, so OK to reference files outside of Kolibri src.
const { kolibriName } = require('kolibri-tools/kolibriName');
const { kolibriName } = require('kolibri-tools/lib/kolibriName');
module.exports = {
output: {

5
packages/kolibri-tools/.eslintrc.js

@ -32,7 +32,6 @@ module.exports = {
},
globals: {
__version: true,
__coreAPISpec: true,
__filename: true,
__copyrightYear: true,
},
@ -46,9 +45,7 @@ module.exports = {
plugins: ['import', 'vue', 'kolibri'],
settings: {
'import/resolver': {
[path.resolve(
path.join(path.dirname(__filename), './lib/alias_import_resolver.js')
)]: {
[path.resolve(path.join(path.dirname(__filename), './lib/alias_import_resolver.js'))]: {
extensions: ['.js', '.vue'],
},
},

28
packages/kolibri-tools/lib/apiSpecExportTools.js

@ -23,7 +23,7 @@ const specFilePath = path.resolve(
);
function specModule(filePath) {
var rootPath = path.dirname(filePath);
const rootPath = path.dirname(filePath);
function newPath(p1) {
if (p1.startsWith('.')) {
return path.join(rootPath, p1);
@ -35,11 +35,11 @@ function specModule(filePath) {
// Read the spec file and do a regex replace to change all instances of 'require('...')'
// to just be the string of the require path.
// Our strict linting rules should ensure that this regex suffices.
var apiSpecFile = fs.readFileSync(filePath, { encoding: 'utf-8' });
const apiSpecFile = fs.readFileSync(filePath, { encoding: 'utf-8' });
var apiSpecTree = espree.parse(apiSpecFile, { sourceType: 'module', ecmaVersion: 2018 });
const apiSpecTree = espree.parse(apiSpecFile, { sourceType: 'module', ecmaVersion: 2018 });
var pathLookup = {};
const pathLookup = {};
apiSpecTree.body.forEach(function(dec) {
if (dec.type === espree.Syntax.ImportDeclaration) {
@ -47,15 +47,16 @@ function specModule(filePath) {
}
});
var properties = apiSpecTree.body.find(dec => dec.type === espree.Syntax.ExportDefaultDeclaration)
.declaration.properties;
const properties = apiSpecTree.body.find(
dec => dec.type === espree.Syntax.ExportDefaultDeclaration
).declaration.properties;
function recurseProperties(props) {
props.forEach(prop => {
if (prop.value.type === espree.Syntax.ObjectExpression) {
recurseProperties(prop.value.properties);
} else if (prop.value.type === espree.Syntax.Identifier) {
var path = pathLookup[prop.key.name];
const path = pathLookup[prop.key.name];
(prop.value = {
type: 'Literal',
value: path,
@ -69,7 +70,7 @@ function specModule(filePath) {
recurseProperties(properties);
// Manually construct an AST that will contain the apiSpec object we need
var objectTree = {
const objectTree = {
type: 'Program',
body: [
{
@ -380,7 +381,7 @@ function coreExternals() {
/*
* Function for creating a hash of externals for modules that are exposed on the core kolibri object.
*/
var externalsObj = {
const externalsObj = {
kolibri: kolibriName,
};
function recurseObjectKeysAndExternalize(obj, pathArray) {
@ -400,11 +401,11 @@ function coreExternals() {
return externalsObj;
}
function coreAliases(localAPISpec) {
function coreAliases() {
/*
* Function for creating a hash of aliases for modules that are exposed on the core kolibri object.
*/
var aliasesObj = Object.assign({}, baseAliases);
const aliasesObj = Object.assign({}, baseAliases);
function recurseObjectKeysAndAlias(obj, pathArray) {
if (typeof obj === 'object') {
Object.keys(obj).forEach(function(key) {
@ -426,11 +427,6 @@ function coreAliases(localAPISpec) {
}
}
recurseObjectKeysAndAlias(apiSpec, ['kolibri']);
if (localAPISpec) {
// If there is a local API spec being injected, just overwrite previous aliases.
var localSpec = specModule(localAPISpec);
recurseObjectKeysAndAlias(localSpec, ['kolibri']);
}
return aliasesObj;
}

45
packages/kolibri-tools/lib/bundleStats.js

@ -0,0 +1,45 @@
const viewer = require('webpack-bundle-analyzer/lib/viewer');
const webpack = require('webpack');
// Import the production configuration so that we remain consistent
const webpackConfig = require('./production').webpackConfig;
const logger = require('./logging');
const buildLogging = logger.getLogger('Kolibri Build Stats');
const basePort = 8889;
function buildWebpack(data, index, startCallback, doneCallback) {
const bundle = webpackConfig(data);
const compiler = webpack(bundle, (err, stats) => {
if (stats.hasErrors()) {
buildLogging.error(`There was a build error for ${bundle.name}`);
process.exit(1);
} else {
const port = basePort + index;
viewer.startServer(stats.toJson(), {
openBrowser: false,
port,
});
}
});
compiler.hooks.compile.tap('Process', startCallback);
compiler.hooks.done.tap('Process', doneCallback);
return compiler;
}
if (require.main === module) {
const data = JSON.parse(process.env.data);
const index = JSON.parse(process.env.index);
buildWebpack(
data,
index,
() => {
process.send('compile');
},
() => {
process.send('done');
}
);
}
module.exports = buildWebpack;

185
packages/kolibri-tools/lib/cli.js

@ -5,9 +5,6 @@ const program = require('commander');
const version = require('../package.json').version;
const logger = require('./logging');
const readWebpackJson = require('./read_webpack_json');
const webpackConfigProd = require('./webpack.config.prod');
const webpackConfigDev = require('./webpack.config.dev');
const webpackConfigI18N = require('./webpack.config.trs');
// ensure the correct version of node is being used
// (specified in package.json)
@ -21,6 +18,40 @@ function list(val) {
program.version(version).description('Tools for Kolibri frontend plugins');
function statsCompletionCallback(bundleData) {
const express = require('express');
const http = require('http');
const host = '127.0.0.1';
const rootPort = 8888;
if (bundleData.length > 1) {
const app = express();
let response = `<html>
<body>
<h1>Kolibri Stats Links</h1>
<ul>`;
bundleData.forEach((bundle, i) => {
response += `<li><a href="http://${host}:${rootPort + i + 1}">${bundle.name}</a></li>`;
});
response += '</ul></body></html>';
app.use('/', (req, res) => {
res.send(response);
});
const server = http.createServer(app);
server.listen(rootPort, host, () => {
const url = `http://${host}:${server.address().port}`;
logger.info(
`Webpack Bundle Analyzer Reports are available at ${url}\n` + `Use ${'Ctrl+C'} to close it`
);
});
} else {
const url = `http://${host}:${rootPort + 1}`;
logger.info(
`Webpack Bundle Analyzer Report is available at ${url}\n` + `Use ${'Ctrl+C'} to close it`
);
}
}
// Build
program
.command('build')
@ -42,8 +73,8 @@ program
list,
[]
)
.option('-m, --multi', 'Run using multiple cores to improve build speed', false)
.action(function(mode, options) {
const webpack = require('webpack');
const { fork } = require('child_process');
const buildLogging = logger.getLogger('Kolibri Build');
const modes = {
@ -78,6 +109,7 @@ program
program.help();
process.exit(1);
}
const multi = options.multi || process.env.KOLIBRI_BUILD_MULTI;
const bundleData = readWebpackJson({
pluginFile: options.file,
plugins: options.plugins,
@ -87,103 +119,80 @@ program
cliLogging.log('No valid bundle data was returned from the plugins specified');
process.exit(1);
}
const webpackConfig = {
[modes.PROD]: webpackConfigProd,
[modes.DEV]: webpackConfigDev,
[modes.I18N]: webpackConfigI18N,
[modes.STATS]: webpackConfigProd,
const buildModule = {
[modes.PROD]: 'production.js',
[modes.DEV]: 'webpackdevserver.js',
[modes.I18N]: 'i18n.js',
[modes.STATS]: 'bundleStats.js',
}[mode];
if (mode === modes.DEV) {
const modulePath = path.resolve(__dirname, buildModule);
function spawnWebpackProcesses({ completionCallback = null, persistent = true } = {}) {
const numberOfBundles = bundleData.length;
let currentlyCompiling = 0;
let currentlyCompiling = numberOfBundles;
// The way we are binding this callback to the webpack compilation hooks
// it seems to miss this on first compilation, so we will only use this for
// watched builds where rebuilds are possible.
function startCallback() {
currentlyCompiling += 1;
}
function doneCallback() {
currentlyCompiling -= 1;
if (currentlyCompiling === 0) {
buildLogging.info('All builds complete!');
if (completionCallback) {
completionCallback(bundleData);
}
}
}
const children = [];
for (let index = 0; index < numberOfBundles; index++) {
const data = JSON.stringify(bundleData[index]);
const forked = fork(path.resolve(__dirname, './webpackdevserver.js'), {
env: {
data,
index,
},
});
children.push(forked);
forked.on('exit', (code, signal) => {
children.forEach(process => {
process.kill(signal);
if (multi) {
const data = JSON.stringify(bundleData[index]);
const childProcess = fork(modulePath, {
env: {
data,
index,
},
stdio: 'inherit',
});
process.exit(code);
});
forked.on('message', msg => {
if (msg === 'compile') {
currentlyCompiling += 1;
} else if (msg === 'done') {
currentlyCompiling -= 1;
}
if (currentlyCompiling === 0) {
buildLogging.info('All builds complete!');
children.push(childProcess);
if (persistent) {
childProcess.on('exit', (code, signal) => {
children.forEach(child => {
child.kill(signal);
});
process.exit(code);
});
}
});
childProcess.on('message', msg => {
if (msg === 'compile') {
startCallback();
} else if (msg === 'done') {
doneCallback();
}
});
} else {
const buildFunction = require(modulePath);
buildFunction(bundleData[index], index, startCallback, doneCallback);
}
}
} else if (mode === modes.CLEAN) {
}
if (mode === modes.CLEAN) {
const clean = require('./clean');
clean(bundleData);
} else if (mode === modes.STATS) {
const viewer = require('webpack-bundle-analyzer/lib/viewer');
const config = webpackConfig(bundleData);
const express = require('express');
const http = require('http');
webpack(config, (err, stats) => {
if (stats.hasErrors()) {
buildLogging.error('There was a build error');
process.exit(1);
} else {
let port = 8888;
const host = '127.0.0.1';
const rootPort = port;
const servers = {};
Promise.all(
stats.stats.map(stat => {
port += 1;
viewer.startServer(stat.toJson(), {
openBrowser: false,
port,
});
servers[stat.compilation.options.name] = port;
})
).then(() => {
const app = express();
let response = `<html>
<body>
<h1>Kolibri Stats Links</h1>
<ul>`;
Object.keys(servers).forEach(key => {
response += `<li><a href="http://${host}:${servers[key]}">${key}</a></li>`;
});
response += '</ul></body></html>';
app.use('/', (req, res) => {
res.send(response);
});
const server = http.createServer(app);
server.listen(rootPort, host, () => {
const url = `http://${host}:${server.address().port}`;
logger.info(
`Webpack Bundle Analyzer Reports are available at ${url}\n` +
`Use ${'Ctrl+C'} to close it`
);
});
});
}
spawnWebpackProcesses({
completionCallback: statsCompletionCallback,
});
} else if (mode === modes.DEV) {
spawnWebpackProcesses();
} else {
const config = webpackConfig(bundleData);
webpack(config, (err, stats) => {
if (stats.hasErrors()) {
buildLogging.error('There was a build error');
buildLogging.log(stats.toString('errors-only'));
process.exit(1);
}
process.exit(0);
// Don't persist for production builds or message extraction
spawnWebpackProcesses({
persistent: false,
});
}
});

49
packages/kolibri-tools/lib/i18n.js

@ -0,0 +1,49 @@
/*
* This defines the translation settings for our webpack build.
* Anything defined here is only applied during frontend message extraction.
*/
const os = require('os');
const webpack = require('webpack');
const logger = require('./logging');
const webpackBaseConfig = require('./webpack.config.base');
function webpackConfig(pluginData) {
const pluginBundle = webpackBaseConfig(pluginData);
pluginBundle.mode = 'development';
pluginBundle.output.path = os.tmpdir();
return pluginBundle;
}
const buildLogging = logger.getLogger('Kolibri Frontend Message Extraction');
function buildWebpack(data, index, startCallback, doneCallback) {
const bundle = webpackConfig(data);
const compiler = webpack(bundle, (err, stats) => {
if (stats.hasErrors()) {
buildLogging.error(`There was a build error for ${bundle.name}`);
buildLogging.log(stats.toString('errors-only'));
process.exit(1);
}
});
compiler.hooks.compile.tap('Process', startCallback);
compiler.hooks.done.tap('Process', doneCallback);
return compiler;
}
if (require.main === module) {
const data = JSON.parse(process.env.data);
const index = JSON.parse(process.env.index);
buildWebpack(
data,
index,
() => {
process.send('compile');
},
() => {
process.send('done');
}
);
}
module.exports = buildWebpack;

149
packages/kolibri-tools/lib/parse_bundle_plugin.js

@ -1,149 +0,0 @@
'use strict';
/**
* Bundle plugin parser module.
* @module parseBundlePlugin
* This file defines a function for parsing frontend plugin specific information in order to
* add plugin specific configuration options to the base webpack config defined in the
* webpack.config.base.js file. Any configuration that does not require specific information
* about the plugin being built (like the entry file, the path for the plugin, etc)
* should be added in that file and not in here.
*/
var path = require('path');
var BundleTracker = require('webpack-bundle-tracker');
var webpack = require('webpack');
var _ = require('lodash');
var merge = require('webpack-merge');
var MiniCssExtractPlugin = require('mini-css-extract-plugin');
var WebpackRTLPlugin = require('webpack-rtl-plugin');
var { VueLoaderPlugin } = require('vue-loader');
const WebpackMessages = require('webpack-messages');
var extract$trs = require('./extract_$trs');
var logging = require('./logging');
var base_config = require('./webpack.config.base');
/**
* Turn an object containing the vital information for a frontend plugin and return a bundle
* configuration for webpack.
* @param {Object} data - An object that contains the data for configuring the bundle.
* @param {string} data.src_file - The Javascript source file that initializes the plugin.
* @param {string} data.name - The name that the plugin is referred to by.
* @param {string} data.static_dir - Directory path to the module in which the plugin is defined.
* @param {string} data.stats_file - The name of the webpack bundle stats file that the plugin data
* @returns {Object} bundle - An object defining the webpack config.
*/
var parseBundlePlugin = function(data) {
if (
typeof data.src_file === 'undefined' ||
typeof data.name === 'undefined' ||
typeof data.static_dir === 'undefined' ||
typeof data.stats_file === 'undefined' ||
typeof data.locale_data_folder === 'undefined' ||
typeof data.plugin_path === 'undefined' ||
typeof data.version === 'undefined'
) {
logging.error(data.name + ' plugin is misconfigured, missing parameter(s)');
return;
}
// Start from a base configuration file that defines common features of the webpack configuration
// for all Kolibri plugins (including the core app).
var base_bundle = _.cloneDeep(base_config);
var local_config;
try {
local_config = require(path.resolve(path.join(data.plugin_path, 'webpack.config.js')));
} catch (e) {
local_config = {};
}
if (local_config.coreAPISpec) {
// Resolve this path now so that it can be unproblematically resolved later.
local_config.coreAPISpec = path.resolve(path.join(data.plugin_path, local_config.coreAPISpec));
}
// Calculate the output path here
var outputPath;
if (process.env.DEV_SERVER) {
var devServerConfig = require('./webpackdevserverconfig');
// Set output path to local dir, as no files will be written - all built files are cached in
// memory.
outputPath = devServerConfig.basePath
? path.resolve(path.join('./', devServerConfig.basePath))
: path.resolve('./');
} else {
outputPath = path.resolve(path.join(data.static_dir, data.name));
}
var bundle = {
// Set the main entry for this module, set the name based on the data.name and the path to the
// entry file from the data.src_file
entry: {
[data.name]: path.join(data.plugin_path, data.src_file),
},
name: data.name,
output: {
path: outputPath,
filename: '[name]-' + data.version + '.js',
// Need to define this in order for chunks to be named
// Without this chunks from different bundles will likely have colliding names
chunkFilename: '[name]-' + data.version + '.js',
},
resolve: {
modules: [
// Add local resolution paths
path.join(data.plugin_path, 'node_modules'),
path.join(process.cwd(), 'node_modules'),
],
},
resolveLoader: {
// Add local resolution paths for loaders
modules: [
path.join(data.plugin_path, 'node_modules'),
path.join(process.cwd(), 'node_modules'),
],
},
plugins: [
new VueLoaderPlugin(),
new MiniCssExtractPlugin({
filename: '[name]' + data.version + '.css',
chunkFilename: '[name]' + data.version + '[id].css',
}),
new WebpackRTLPlugin({
minify: {
zindex: false,
// prevent renaming keyframes
reduceIdents: false,
},
}),
// BundleTracker creates stats about our built files which we can then pass to Django to
// allow our template tags to load the correct frontend files.
new BundleTracker({
path: path.dirname(data.stats_file),
filename: path.basename(data.stats_file),
}),
// Plugins know their own name, by having a variable that we define here, based on the name
// they are given in kolibri_plugins.py inside their relevant module.
// We also pass in the events hashes here as well, as they are defined in the Python
// specification of the KolibriModule.
new webpack.DefinePlugin({
__kolibriModuleName: JSON.stringify(data.name),
__version: JSON.stringify(data.version),
}),
new extract$trs(data.locale_data_folder, data.name),
// Add custom messages per bundle.
new WebpackMessages({
name: data.name,
logger: str => logging.info(str),
}),
],
};
bundle = merge.smart(bundle, base_bundle, local_config);
return bundle;
};
module.exports = parseBundlePlugin;

56
packages/kolibri-tools/lib/production.js

@ -0,0 +1,56 @@
/*
* This defines the production settings for our webpack build.
* Anything defined here is only applied during production building.
*/
const webpack = require('webpack');
const webpackBaseConfig = require('./webpack.config.base');
const logger = require('./logging');
function webpackConfig(pluginData) {
const pluginBundle = webpackBaseConfig(pluginData);
pluginBundle.mode = 'production';
pluginBundle.stats = 'normal';
pluginBundle.plugins = pluginBundle.plugins.concat([
new webpack.LoaderOptionsPlugin({
minimize: true,
debug: false,
}),
]);
return pluginBundle;
}
const buildLogging = logger.getLogger('Kolibri Production Build');
function buildWebpack(data, index, startCallback, doneCallback) {
const bundle = webpackConfig(data);
const compiler = webpack(bundle, (err, stats) => {
if (stats.hasErrors()) {
buildLogging.error(`There was a build error for ${bundle.name}`);
buildLogging.log(stats.toString('errors-only'));
process.exit(1);
}
});
compiler.hooks.compile.tap('Process', startCallback);
compiler.hooks.done.tap('Process', doneCallback);
return compiler;
}
if (require.main === module) {
const data = JSON.parse(process.env.data);
const index = JSON.parse(process.env.index);
buildWebpack(
data,
index,
() => {
process.send('compile');
},
() => {
process.send('done');
}
);
}
module.exports = buildWebpack;
module.exports.webpackConfig = webpackConfig;

149
packages/kolibri-tools/lib/read_bundle_plugins.js

@ -1,149 +0,0 @@
'use strict';
/**
* Bundle plugin Python config reader module.
* @module readBundlePlugins
*/
const _ = require('lodash');
const webpack = require('webpack');
const logging = require('./logging');
const parseBundlePlugin = require('./parse_bundle_plugin');
const coreExternals = require('./apiSpecExportTools').coreExternals;
const coreAliases = require('./apiSpecExportTools').coreAliases;
const { kolibriName } = require('./kolibriName');
function setNodePaths(nodePaths) {
/*
* This is a filthy hack. Do as I say, not as I do.
* Taken from: https://gist.github.com/branneman/8048520#6-the-hack
* This forces the NODE_PATH environment variable to include the main
* kolibri node_modules folder, so that even plugins being built outside
* of the kolibri folder will have access to all installed loaders, etc.
* Doing it here, rather than at command invocation, allows us to do this
* in a cross platform way, and also to avoid having to prepend it to all
* our commands that end up invoking webpack.
*/
nodePaths.forEach(nodePath => {
var delimiter = process.platform === 'win32' ? ';' : ':';
process.env.NODE_PATH = process.env.NODE_PATH + delimiter + nodePath;
});
require('module').Module._initPaths();
}
/**
* Extract the information regarding front end plugin configuration using a
* Python script to import information about the relevant plugins and then run methods
* against them to create the config data.
* @returns {Array} bundles - An array containing webpack config objects.
*/
var readBundlePlugins = function(bundles) {
bundles = bundles.map(parseBundlePlugin).filter(function(bundle) {
return bundle;
});
if (bundles.length > 0) {
for (var k = 0; k < bundles.length; k++) {
for (var j = 0; j < bundles.length; j++) {
// We want to prevent the same bundle being built twice, so enforce that here by checking
// no duplicates.
if (k !== j) {
// Only one key per object here, so just get the first key
if (Object.keys(bundles[k].entry)[0] === Object.keys(bundles[j].entry)[0]) {
logging.error('Duplicate keys: ' + Object.keys(bundles[k].entry)[0]);
}
}
}
}
}
// A bundle can specify a modification to the coreAPI.
var coreAPISpec = (
_.find(bundles, function(bundle) {
return bundle.coreAPISpec;
}) || {}
).coreAPISpec;
// Check that there is only one bundle modifying the coreAPI spec.
if (
_.filter(bundles, function(bundle) {
return bundle.coreAPISpec;
}).length > 1
) {
logging.warn('You have more than one coreAPISpec modification specified.');
}
// All references to the core API spec will be referenced as subproperties of the global
// Kolibri object in the browser, so any references to anything we bundle in the core API
// will be replaced by a property reference to the global object, e.g. for a Vue import:
// `import Vue from 'vue';` webpack will replace it with a reference to Vue bundled into the
// core Kolibri app.
var core_externals = coreExternals();
bundles.forEach(function(bundle) {
Object.assign(bundle.resolve.alias, coreAliases(coreAPISpec));
// Only the default bundle is built for library output to a global variable
if (bundle.output.library !== kolibriName) {
// If this is not the core bundle, then we need to add the external library mappings.
bundle.externals = core_externals;
} else {
bundle.externals = { kolibri: bundle.output.library };
if (coreAPISpec) {
bundle.plugins.push(
new webpack.ProvidePlugin({
__coreAPISpec: coreAPISpec,
})
);
bundle.plugins.push(
new webpack.DefinePlugin({
__copyrightYear: new Date().getFullYear(),
})
);
} else {
bundle.plugins.push(
new webpack.DefinePlugin({
__coreAPISpec: '{}',
__copyrightYear: new Date().getFullYear(),
})
);
}
}
});
var nodePaths = [];
// We add some custom configuration options to the bundles that webpack 2 dislikes, clean them
// up here.
bundles.forEach(function(bundle) {
delete bundle.coreAPISpec;
if (bundle.nodePaths) {
if (!Array.isArray(bundle.nodePaths)) {
nodePaths.push(bundle.nodePaths);
} else {
nodePaths = nodePaths.concat(bundle.nodePaths);
}
}
delete bundle.nodePaths;
});
// Allow individual plugins to set extra node paths - this is potentially dangerous,
// because different node modules might have the same module in them, and we can't
// predict which one will get resolved first. Caveat emptor.
setNodePaths(nodePaths);
// Sort bundles to give a consistent return order in case this is being read by different
// processes.
return bundles.sort((a, b) => {
if (a.name > b.name) {
return 1;
} else if (a.name < b.name) {
return -1;
}
return 0;
});
};
module.exports = readBundlePlugins;

303
packages/kolibri-tools/lib/webpack.config.base.js

@ -3,28 +3,32 @@
* build and testing environments. If you need to add anything to the general
* webpack config, like adding loaders for different asset types, different
* preLoaders or Plugins - they should be done here. If you are looking to add
* dev specific features, please do so in webpack.config.dev.js - if you wish
* to add test specific features, these can be done in the karma.conf.js.
*
* Note:
* This file is not called directly by webpack.
* It copied once for each plugin by parse_bundle_plugin.js
* and used as a template, with additional plugin-specific
* modifications made on top. Any entries that require plugin specific
* information are added in parse_bundle_plugin.js - such as access to
* plugin name, plugin file paths, and version information.
* dev specific features, please do so in webpackdevserver.js - if you wish
* to add test specific features.
*/
var path = require('path');
var MiniCssExtractPlugin = require('mini-css-extract-plugin');
var UglifyJsPlugin = require('uglifyjs-webpack-plugin');
var OptimizeCSSAssetsPlugin = require('optimize-css-assets-webpack-plugin');
const path = require('path');
const fs = require('fs');
const BundleTracker = require('webpack-bundle-tracker');
const webpack = require('webpack');
const merge = require('webpack-merge');
const MiniCssExtractPlugin = require('mini-css-extract-plugin');
const WebpackRTLPlugin = require('webpack-rtl-plugin');
const OptimizeCSSAssetsPlugin = require('optimize-css-assets-webpack-plugin');
const UglifyJsPlugin = require('uglifyjs-webpack-plugin');
const { VueLoaderPlugin } = require('vue-loader');
const WebpackMessages = require('webpack-messages');
const extract$trs = require('./extract_$trs');
const logging = require('./logging');
const coreExternals = require('./apiSpecExportTools').coreExternals();
const coreAliases = require('./apiSpecExportTools').coreAliases();
const { kolibriName } = require('./kolibriName');
var production = process.env.NODE_ENV === 'production';
const production = process.env.NODE_ENV === 'production';
var base_dir = path.join(__dirname, '..');
const base_dir = path.join(__dirname, '..');
var postCSSLoader = {
const postCSSLoader = {
loader: 'postcss-loader',
options: {
config: { path: path.resolve(__dirname, '../postcss.config.js') },
@ -32,13 +36,13 @@ var postCSSLoader = {
},
};
var cssLoader = {
const cssLoader = {
loader: 'css-loader',
options: { minimize: production, sourceMap: !production },
};
// for scss blocks
var sassLoaders = [
const sassLoaders = [
MiniCssExtractPlugin.loader,
cssLoader,
postCSSLoader,
@ -49,88 +53,195 @@ var sassLoaders = [
},
];
// primary webpack config
module.exports = {
module: {
rules: [
// Preprocessing rules
{
test: /\.(html|vue)$/,
enforce: 'pre',
// handles <mat-svg/>, <ion-svg/>, <iconic-svg/>, and <file-svg/> svg inlining
loader: 'svg-icon-inline-loader',
exclude: /node_modules/,
},
// Transpilation and code loading rules
{
test: /\.vue$/,
loader: 'vue-loader',
options: {
compilerOptions: {
preserveWhitespace: false,
/**
* Turn an object containing the vital information for a frontend plugin and return a bundle
* configuration for webpack.
* @param {Object} data - An object that contains the data for configuring the bundle.
* @param {string} data.src_file - The Javascript source file that initializes the plugin.
* @param {string} data.name - The name that the plugin is referred to by.
* @param {string} data.static_dir - Directory path to the module in which the plugin is defined.
* @param {string} data.stats_file - The name of the webpack bundle stats file that the plugin data
* @returns {Object} bundle - An object defining the webpack config.
*/
module.exports = data => {
if (
typeof data.src_file === 'undefined' ||
typeof data.name === 'undefined' ||
typeof data.static_dir === 'undefined' ||
typeof data.stats_file === 'undefined' ||
typeof data.locale_data_folder === 'undefined' ||
typeof data.plugin_path === 'undefined' ||
typeof data.version === 'undefined'
) {
logging.error(data.name + ' plugin is misconfigured, missing parameter(s)');
return;
}
let local_config = {};
try {
const localConfigPath = path.resolve(path.join(data.plugin_path, 'webpack.config.js'));
if (fs.existsSync(localConfigPath)) {
local_config = require(localConfigPath);
}
} catch (e) {
logging.error('Local webpack config import failed with error ' + e);
local_config = {};
}
let externals;
if (!local_config.output || local_config.output.library !== kolibriName) {
// If this is not the core bundle, then we need to add the external library mappings.
externals = coreExternals;
} else {
externals = { kolibri: kolibriName };
}
let bundle = {
// Set the main entry for this module, set the name based on the data.name and the path to the
// entry file from the data.src_file
entry: {
[data.name]: path.join(data.plugin_path, data.src_file),
},
externals,
name: data.name,
module: {
rules: [
// Preprocessing rules
{
test: /\.(html|vue)$/,
enforce: 'pre',
// handles <mat-svg/>, <ion-svg/>, <iconic-svg/>, and <file-svg/> svg inlining
loader: 'svg-icon-inline-loader',
exclude: /node_modules/,
},
// Transpilation and code loading rules
{
test: /\.vue$/,
loader: 'vue-loader',
options: {
compilerOptions: {
preserveWhitespace: false,
},
},
},
},
{
test: /\.js$/,
loader: 'buble-loader',
options: {
objectAssign: 'Object.assign',
{
test: /\.js$/,
loader: 'buble-loader',
options: {
objectAssign: 'Object.assign',
},
},
},
{
test: /\.css$/,
use: [MiniCssExtractPlugin.loader, cssLoader, postCSSLoader],
},
{
test: /\.s[a|c]ss$/,
use: sassLoaders,
},
{
test: /\.(png|jpe?g|gif|svg)$/,
use: {
loader: 'url-loader',
options: { limit: 10000, name: '[name].[ext]?[hash]' },
{
test: /\.css$/,
use: [MiniCssExtractPlugin.loader, cssLoader, postCSSLoader],
},
},
// Use url loader to load font files.
{
test: /\.(eot|woff|ttf|woff2)$/,
use: {
loader: 'url-loader',
options: { name: '[name].[ext]?[hash]' },
{
test: /\.s[a|c]ss$/,
use: sassLoaders,
},
},
],
},
optimization: {
minimizer: [
new UglifyJsPlugin({
cache: true,
parallel: true,
sourceMap: false,
{
test: /\.(png|jpe?g|gif|svg)$/,
use: {
loader: 'url-loader',
options: { limit: 10000, name: '[name].[ext]?[hash]' },
},
},
// Use url loader to load font files.
{
test: /\.(eot|woff|ttf|woff2)$/,
use: {
loader: 'url-loader',
options: { name: '[name].[ext]?[hash]' },
},
},
],
},
node: {
__filename: true,
},
optimization: {
minimizer: [
new UglifyJsPlugin({
cache: true,
parallel: true,
sourceMap: false,
}),
new OptimizeCSSAssetsPlugin({}),
],
},
output: {
path: path.resolve(path.join(data.static_dir, data.name)),
filename: '[name]-' + data.version + '.js',
// Need to define this in order for chunks to be named
// Without this chunks from different bundles will likely have colliding names
chunkFilename: '[name]-' + data.version + '.js',
},
resolve: {
extensions: ['.js', '.vue', '.scss'],
alias: coreAliases,
modules: [
// Add local resolution paths
path.join(data.plugin_path, 'node_modules'),
path.join(process.cwd(), 'node_modules'),
// Add resolution paths for modules to allow any plugin to
// access kolibri-tools/node_modules modules during bundling.
base_dir,
path.join(base_dir, 'node_modules'),
],
},
resolveLoader: {
modules: [
// Add local resolution paths for loaders
path.join(data.plugin_path, 'node_modules'),
path.join(process.cwd(), 'node_modules'),
// Add resolution paths for loaders to allow any plugin to
// access kolibri-tools/node_modules loaders during bundling.
base_dir,
path.join(base_dir, 'node_modules'),
],
},
plugins: [
new VueLoaderPlugin(),
new MiniCssExtractPlugin({
filename: '[name]' + data.version + '.css',
chunkFilename: '[name]' + data.version + '[id].css',
}),
new WebpackRTLPlugin({
minify: {
zindex: false,
// prevent renaming keyframes
reduceIdents: false,
},
}),
// BundleTracker creates stats about our built files which we can then pass to Django to
// allow our template tags to load the correct frontend files.
new BundleTracker({
path: path.dirname(data.stats_file),
filename: path.basename(data.stats_file),
}),
// Plugins know their own name, by having a variable that we define here, based on the name
// they are given in kolibri_plugins.py inside their relevant module.
// Also define the current plugin version (for kolibri plugins bundled with kolibri, this is
// the kolibri version).
// Also add the copyright year for auto updated copyright footers.
new webpack.DefinePlugin({
__kolibriModuleName: JSON.stringify(data.name),
__version: JSON.stringify(data.version),
__copyrightYear: new Date().getFullYear(),
}),
new extract$trs(data.locale_data_folder, data.name),
// Add custom messages per bundle.
new WebpackMessages({
name: data.name,
logger: str => logging.info(str),
}),
new OptimizeCSSAssetsPlugin({}),
],
},
plugins: [],
resolve: {
extensions: ['.js', '.vue', '.scss'],
alias: {},
modules: [
// Add resolution paths for modules to allow any plugin to
// access kolibri-tools/node_modules modules during bundling.
base_dir,
path.join(base_dir, 'node_modules'),
],
},
resolveLoader: {
// Add resolution paths for loaders to allow any plugin to
// access kolibri-tools/node_modules loaders during bundling.
modules: [base_dir, path.join(base_dir, 'node_modules')],
},
node: {
__filename: true,
},
stats: 'minimal',
stats: 'minimal',
};
bundle = merge.smart(bundle, local_config);
return bundle;
};

28
packages/kolibri-tools/lib/webpack.config.dev.js

@ -1,28 +0,0 @@
/*
* This takes all bundles defined in our production webpack configuration and adds inline source
* maps to all of them for easier debugging.
* Any dev specific modifications to the build should be specified in here, where each bundles[i]
* object is a webpack configuration object that needs to be edited/manipulated to add features to.
*/
const webpack = require('webpack');
const readBundlePlugins = require('./read_bundle_plugins');
function bundles(pluginsData) {
const pluginBundles = readBundlePlugins(pluginsData);
for (var i = 0; i < pluginBundles.length; i++) {
pluginBundles[i].devtool = '#cheap-module-source-map';
pluginBundles[i].mode = 'development';
pluginBundles[i].plugins = pluginBundles[i].plugins.concat([
new webpack.DefinePlugin({
'process.env': {
NODE_ENV: '"debug"',
},
}),
]);
}
return pluginBundles;
}
module.exports = bundles;

17
packages/kolibri-tools/lib/webpack.config.js

@ -1,17 +0,0 @@
/*
* This file acts as the entry point for webpack building of frontend assets. From here, all
* Kolibri Plugin folders will be scanned for kolibri_plugins.py and the relevant KolibriModule
* metadata extracted for webpack configuration.
* The 'bundles' emitted as the module.exports are the Webpack configuration objects that are then
* parsed by Webpack to bundle the assets. See recurseBundlePlugins and associated functions for
* details of how we make these configurations and Webpack documentation for details on what the
* configuration bundles do.
*/
// ensure the correct version of node is being used
// (specified in package.json)
require('engine-strict').check();
var readBundlePlugins = require('./read_bundle_plugins');
module.exports = readBundlePlugins();

25
packages/kolibri-tools/lib/webpack.config.prod.js

@ -1,25 +0,0 @@
/*
* This defines the production settings for our webpack build.
* Anything defined here is only applied during production building.
*/
const webpack = require('webpack');
const readBundlePlugins = require('./read_bundle_plugins');
function bundles(pluginsData) {
const pluginBundles = readBundlePlugins(pluginsData);
for (var i = 0; i < pluginBundles.length; i++) {
pluginBundles[i].mode = 'production';
pluginBundles[i].stats = 'normal';
pluginBundles[i].plugins = pluginBundles[i].plugins.concat([
new webpack.LoaderOptionsPlugin({
minimize: true,
debug: false,
}),
]);
}
return pluginBundles;
}
module.exports = bundles;

20
packages/kolibri-tools/lib/webpack.config.trs.js

@ -1,20 +0,0 @@
/*
* This defines the production settings for our webpack build.
* Anything defined here is only applied during production building.
*/
const os = require('os');
const webpack = require('webpack');
const readBundlePlugins = require('./read_bundle_plugins');
function bundles(pluginsData) {
const pluginBundles = readBundlePlugins(pluginsData);
for (var i = 0; i < pluginBundles.length; i++) {
pluginBundles[i].mode = 'development';
pluginBundles[i].output.path = os.tmpdir();
}
return pluginBundles;
}
module.exports = bundles;

146
packages/kolibri-tools/lib/webpackdevserver.js

@ -1,53 +1,101 @@
/*
* This takes all bundles defined in our production webpack configuration and adds inline source
* maps to all of them for easier debugging.
* Any dev specific modifications to the build should be specified in here, where each bundle
* in the webpackConfig function is a webpack configuration object that needs to be
* edited/manipulated to add features to.
*/
process.env.DEV_SERVER = true;
var WebpackDevServer = require('webpack-dev-server');
var webpack = require('webpack');
var openInEditor = require('launch-editor-middleware');
var devServerConfig = require('./webpackdevserverconfig');
var logging = require('./logging');
var bundleFn = require('./webpack.config.dev');
const data = JSON.parse(process.env.data);
const index = JSON.parse(process.env.index);
const bundle = bundleFn([data])[0];
const port = devServerConfig.port + index;
const address = devServerConfig.address;
const basePath = devServerConfig.basePath;
const publicPath = 'http://' + address + ':' + port + '/' + (basePath ? basePath + '/' : '');
bundle.output.publicPath = publicPath;
const compiler = webpack(bundle);
compiler.hooks.compile.tap('Process', () => {
process.send('compile');
});
compiler.hooks.done.tap('Process', () => {
process.send('done');
});
var server = new WebpackDevServer(compiler, {
// webpack-dev-server options
// contentBase: "http://localhost:3000/",
// Can also be an array, or: contentBase: "http://localhost/",
// Set this as true if you want to access dev server from arbitrary url.
// This is handy if you are using a html5 router.
historyApiFallback: false,
// Set this if you want to enable gzip compression for assets
compress: true,
// webpack-dev-middleware options
watchOptions: {
aggregateTimeout: 300,
poll: 1000,
},
// It's a required option.
publicPath,
stats: 'minimal',
headers: {
'Access-Control-Allow-Origin': '*',
const path = require('path');
const WebpackDevServer = require('webpack-dev-server');
const webpack = require('webpack');
const openInEditor = require('launch-editor-middleware');
const webpackBaseConfig = require('./webpack.config.base');
const devServerConfig = {
address: 'localhost',
port: 3000,
host: '0.0.0.0',
basePath: 'js-dist',
get publicPath() {
return (
'http://' + this.address + ':' + this.port + '/' + (this.basePath ? this.basePath + '/' : '')
);
},
});
server.use('/__open-in-editor', openInEditor());
};
function webpackConfig(pluginData) {
const pluginBundle = webpackBaseConfig(pluginData);
pluginBundle.devtool = '#cheap-module-source-map';
pluginBundle.mode = 'development';
pluginBundle.plugins = pluginBundle.plugins.concat([
new webpack.DefinePlugin({
'process.env': {
NODE_ENV: '"debug"',
},
}),
]);
pluginBundle.output.path = path.resolve(path.join('./', devServerConfig.basePath));
return pluginBundle;
}
function buildWebpack(data, index, startCallback, doneCallback) {
const bundle = webpackConfig(data);
const port = devServerConfig.port + index;
const address = devServerConfig.address;
const basePath = devServerConfig.basePath;
const publicPath = 'http://' + address + ':' + port + '/' + (basePath ? basePath + '/' : '');
bundle.output.publicPath = publicPath;
const compiler = webpack(bundle);
const server = new WebpackDevServer(compiler, {
// webpack-dev-server options
// contentBase: "http://localhost:3000/",
// Can also be an array, or: contentBase: "http://localhost/",
// Set this as true if you want to access dev server from arbitrary url.
// This is handy if you are using a html5 router.
historyApiFallback: false,
// Set this if you want to enable gzip compression for assets
compress: true,
// webpack-dev-middleware options
watchOptions: {
aggregateTimeout: 300,
poll: 1000,
},
// It's a required option.
publicPath,
stats: 'minimal',
headers: {
'Access-Control-Allow-Origin': '*',
},
});
compiler.hooks.compile.tap('Process', startCallback);
compiler.hooks.done.tap('Process', doneCallback);
server.use('/__open-in-editor', openInEditor());
server.listen(port, devServerConfig.host, function() {});
return compiler;
}
if (require.main === module) {
const data = JSON.parse(process.env.data);
const index = JSON.parse(process.env.index);
buildWebpack(
data,
index,
() => {
process.send('compile');
},
() => {
process.send('done');
}
);
}
server.listen(port, devServerConfig.host, function() {});
module.exports = buildWebpack;

11
packages/kolibri-tools/lib/webpackdevserverconfig.js

@ -1,11 +0,0 @@
module.exports = {
address: 'localhost',
port: 3000,
host: '0.0.0.0',
basePath: 'js-dist',
get publicPath() {
return (
'http://' + this.address + ':' + this.port + '/' + (this.basePath ? this.basePath + '/' : '')
);
},
};

70
packages/kolibri-tools/test/test_bundle_parse.spec.js → packages/kolibri-tools/test/test_webpack.config.base.spec.js

@ -1,8 +1,6 @@
const path = require('path');
const _ = require('lodash');
const parseBundlePlugin = require('../lib/parse_bundle_plugin');
const readBundlePlugins = require('../lib/read_bundle_plugins');
const webpackConfigBase = require('../lib/webpack.config.base');
jest.mock('../lib/apiSpecExportTools', () => ({
coreAliases: () => ({}),
@ -24,72 +22,61 @@ const baseData = {
plugin_path: 'kolibri/plugin',
};
const baseData1 = {
name: 'kolibri.plugin.test.test_plugin1',
src_file: 'src/file1.js',
stats_file: 'output1.json',
static_url_root: 'static1',
static_dir: 'kolibri/plugin/test1',
locale_data_folder: 'kolibri/locale/test1',
version: 'test',
plugin_path: 'kolibri/plugin1',
};
describe('parseBundlePlugin', function() {
describe('webpackConfigBase', function() {
let data;
beforeEach(function() {
data = _.clone(baseData);
});
describe('input is valid, bundles output', function() {
it('should have one entry', function() {
expect(typeof parseBundlePlugin(data, '/')).not.toEqual('undefined');
expect(Object.keys(webpackConfigBase(data).entry)).toHaveLength(1);
});
it('should set the entry name to data.name', function() {
expect(Object.keys(parseBundlePlugin(data).entry)[0]).toEqual(data.name);
expect(Object.keys(webpackConfigBase(data).entry)[0]).toEqual(data.name);
});
it('should set the entry path to the path to the source file', function() {
expect(parseBundlePlugin(data).entry[data.name]).toEqual(
expect(webpackConfigBase(data).entry[data.name]).toEqual(