From f35c81eeedb536a3150863f891f14527ce5c1b12 Mon Sep 17 00:00:00 2001 From: Ivan Malagon Date: Tue, 21 Mar 2017 18:13:25 +0100 Subject: [PATCH] Webpack running fast --- Gruntfile.js | 54 ++++----- .../affectedFiles/fetchRequires.config.js | 7 ++ lib/build/affectedFiles/fetchRequires.js | 25 +++- lib/build/affectedSpecs.js | 14 ++- lib/build/tasks/jasmine.js | 7 +- lib/build/tasks/watch.js | 2 +- lib/build/tasks/webpack/webpack.config.js | 69 +++++++++++ lib/build/tasks/webpack/webpack.js | 112 ++++++++++++++++++ package.json | 6 +- 9 files changed, 257 insertions(+), 39 deletions(-) create mode 100644 lib/build/affectedFiles/fetchRequires.config.js create mode 100644 lib/build/tasks/webpack/webpack.config.js create mode 100644 lib/build/tasks/webpack/webpack.js diff --git a/Gruntfile.js b/Gruntfile.js index 56802abf2a..bf2484f9c1 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -3,8 +3,8 @@ var timer = require("grunt-timer"); var colors = require('colors'); var jasmineCfg = require('./lib/build/tasks/jasmine.js'); var duplicatedDependencies = require('./lib/build/tasks/shrinkwrap-duplicated-dependencies.js'); -var retrieveAffectedSpecs = require('./lib/build/affectedSpecs'); - +var webpackTask = require('./lib/build/tasks/webpack/webpack.js'); + var REQUIRED_NPM_VERSION = /2.14.[0-9]+/; var REQUIRED_NODE_VERSION = /0.10.[0-9]+/; var SHRINKWRAP_MODULES_TO_VALIDATE = [ @@ -21,9 +21,9 @@ var SHRINKWRAP_MODULES_TO_VALIDATE = [ 'turbo-carto' ]; - /** - * CartoDB UI assets generation - */ +/** + * CartoDB UI assets generation + */ module.exports = function(grunt) { @@ -46,8 +46,8 @@ var SHRINKWRAP_MODULES_TO_VALIDATE = [ done && done(err ? new Error(err): null); }); } - checkVersion('npm -v', REQUIRED_NPM_VERSION, 'npm', done); - checkVersion('node -v', REQUIRED_NODE_VERSION, 'node', done); + //checkVersion('npm -v', REQUIRED_NPM_VERSION, 'npm', done); + //checkVersion('node -v', REQUIRED_NODE_VERSION, 'node', done); } preFlight(function (err) { @@ -348,38 +348,36 @@ var SHRINKWRAP_MODULES_TO_VALIDATE = [ 'copy:js_test_spec_client_cartodb3' ]); + // Affected specs section - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + grunt.registerTask('affected', 'Generate only affected specs', function () { - var done = this.async(); + webpackTask.affected.call(this, grunt); + }); - retrieveAffectedSpecs(grunt) - .then(function (affectedSpecs) { - console.log(colors.yellow(affectedSpecs.length + ' specs found.')); - - var newSrc = ['lib/build/source-map-support.js'] - .concat(affectedSpecs); - // .concat([ - // '!lib/assets/test/spec/cartodb3/deep-insights-integrations.spec.js', - // 'lib/assets/test/spec/node_modules/**/*.spec.js', - // 'lib/assets/test/spec/cartodb3/deep-insights-integrations.spec.js' - //]); - - grunt.config.set('browserify.affected_specs.src', newSrc); - done(); - }) - .catch(function (reason) { - throw new Error(reason); - }); + grunt.registerTask('bootstrap_webpack_builder_specs', 'Create the webpack compiler', function () { + webpackTask.bootstrap.call(this, 'builder_specs'); }); - grunt.registerTask('affected_dev', 'Build only specs affected by changes in current branch', [ + grunt.registerTask('webpack:builder_specs', 'Webpack compilation task for builder specs', function () { + webpackTask.compile.call(this, 'builder_specs'); + }); + + /** + * `grunt affected_specs` compile Builder specs using only affected ones by the current branch. + * `grunt affected_specs --specs=all` compile all Builder specs. + */ + grunt.registerTask('affected_specs', 'Build only specs affected by changes in current branch', [ 'copy_builder', 'affected', - 'browserify:affected_specs', + 'bootstrap_webpack_builder_specs', + 'webpack:builder_specs', 'jasmine:affected:build', 'connect', 'watch:js_affected' ]); + // / Affected specs section - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + /** * Delegate task to commandline. * @param {String} name - If taskname starts with npm it's run a npm script (i.e. `npm run foobar` diff --git a/lib/build/affectedFiles/fetchRequires.config.js b/lib/build/affectedFiles/fetchRequires.config.js new file mode 100644 index 0000000000..848211765a --- /dev/null +++ b/lib/build/affectedFiles/fetchRequires.config.js @@ -0,0 +1,7 @@ +var cfg = { + requirePattern: "require\\('(\\..+)'\\)", + defaultExt: '.js', + validExtensions: ['.tpl', '.json', '.js', '.mustache'] +} + +module.exports = cfg; diff --git a/lib/build/affectedFiles/fetchRequires.js b/lib/build/affectedFiles/fetchRequires.js index c550ca983a..6999793b41 100644 --- a/lib/build/affectedFiles/fetchRequires.js +++ b/lib/build/affectedFiles/fetchRequires.js @@ -1,17 +1,36 @@ var fs = require('fs-extra'); var path = require('path'); var Promise = require('bluebird'); +var config = require('./fetchRequires.config.js'); + +function checkConfig () { + if (!config.requirePattern) { + throw new Error("requirePattern needed at fetchRequires.config.js - Use it to define the regular expression to use for finding require invocations. Example: require\\('(\\..+)'\\)"); + } + if (!config.defaultExt) { + throw new Error("defaultExt needed at fetchRequires.config.js - Use it to define the default extension to use for modules. Example: '.js'"); + } + if (!config.validExtensions) { + throw new Error("validExtensions needed at fetchRequires.config.js - Use it to laod modules as is. That's it, to define valid extensions that don't need appending the default extension. Example: ['.tpl', '.json']"); + } +} var fetchRequires = function (absoluteFilePath) { - var requirePattern = "require\\('(\\..+)'\\)"; + checkConfig(); + var requirePattern = config.requirePattern; var requireGlobalRegex = new RegExp(requirePattern, 'g'); var requireRegex = new RegExp(requirePattern); - var defaultExt = '.js'; - var validExtensions = ['.tpl', '.json', '.js', '.mustache']; + var defaultExt = config.defaultExt; + var validExtensions = config.validExtensions; return new Promise(function (resolve, reject) { fs.readFile(absoluteFilePath, 'utf8', function (err, data) { if (err) { + if (err.code && err.code === 'ENOENT') { + var error = new Error("ENOENT: no such file '" + err.path + "'. Perhaps we're adding the default extension to a valid file. Check `validExtensions` in fetchRequires.config.js and add a new one if needed."); + reject(error); + return; + } reject(err); return; } diff --git a/lib/build/affectedSpecs.js b/lib/build/affectedSpecs.js index c288660373..dc63c73125 100644 --- a/lib/build/affectedSpecs.js +++ b/lib/build/affectedSpecs.js @@ -1,6 +1,7 @@ var Promise = require('bluebird'); var colors = require('colors'); var StringDecoder = require('string_decoder').StringDecoder; +var separator = require('path').sep; function getChunkParser (openingTag, closingTag) { var output = ''; @@ -37,8 +38,17 @@ function retrieveAffectedSpecs (grunt, npmScript) { var child = grunt.util.spawn({ cmd: 'npm', args: ['run', npmScript] - }, function doneFunction () { - resolve(affectedSpecs); + }, function doneFunction (error, result, code) { + if (error) { + reject(error); + return; + } + + var startLib = /^lib/; + var relativeSpecs = affectedSpecs.map(function (spec) { + return spec.replace(startLib, '.' + separator + 'lib'); + }); + resolve(relativeSpecs); }); child.stdout.on('data', function (chunk) { diff --git a/lib/build/tasks/jasmine.js b/lib/build/tasks/jasmine.js index d89cb336e2..ff54aa8083 100644 --- a/lib/build/tasks/jasmine.js +++ b/lib/build/tasks/jasmine.js @@ -40,9 +40,12 @@ module.exports = { display: 'short', helpers: js_files._spec_helpers3, specs: [ - '.grunt/affected-specs.js' + '.grunt/vendor.affected-specs.js', + '.grunt/main.affected-specs.js' ], - vendor: ['node_modules/jasmine-ajax/lib/mock-ajax.js'] + vendor: [ + 'node_modules/jasmine-ajax/lib/mock-ajax.js', + 'node_modules/underscore/underscore-min.js'] } } }; diff --git a/lib/build/tasks/watch.js b/lib/build/tasks/watch.js index e27ba46cbe..b1821c9b82 100644 --- a/lib/build/tasks/watch.js +++ b/lib/build/tasks/watch.js @@ -130,7 +130,7 @@ exports.task = function () { tasks: [ 'copy_builder', 'affected', - 'browserify:affected_specs', + 'webpack:builder_specs', 'jasmine:affected:build' ], options: { diff --git a/lib/build/tasks/webpack/webpack.config.js b/lib/build/tasks/webpack/webpack.config.js new file mode 100644 index 0000000000..10bb424aa9 --- /dev/null +++ b/lib/build/tasks/webpack/webpack.config.js @@ -0,0 +1,69 @@ +var path = require('path'); +var webpack = require('webpack'); + +module.exports = { + task: function () { + var cfg = {}; + + cfg.builder_specs = { + entry: { + main: [ + // To be filled by grunt + ] + }, + output: { + path: path.resolve(path.resolve('.'), '.grunt'), + filename: '[name].affected-specs.js' + }, + module: { + rules: [ + { + test: /\.js$/, + loader: 'shim-loader', + query: { + shim: { + 'wax.cartodb.js': { + exports: 'wax' + }, + 'html-css-sanitizer': { + exports: 'html' + } + } + } + }, + { + test: /\.tpl$/, + use: 'tpl-loader' + }, + {test: /\.mustache$/, use: 'mustache-loader'} + ], + exprContextRegExp: /$^/, + exprContextCritical: false + }, + plugins: [ + new webpack.optimize.CommonsChunkPlugin({ + name: 'vendor', + minChunks: function (module) { + return module.context && module.context.indexOf('node_modules') !== -1; + } + }), + new webpack.ProvidePlugin({ + $: 'jquery', + jQuery: 'jquery', + ['window.jQuery']: 'jquery' + }), + new webpack.SourceMapDevToolPlugin({ + filename: '[file].map', + exclude: /vendor/ + }) + ], + target: 'web', + node: { + fs: 'empty' + }, + stats: 'normal' + }; + + return cfg; + } +}; diff --git a/lib/build/tasks/webpack/webpack.js b/lib/build/tasks/webpack/webpack.js new file mode 100644 index 0000000000..787e97b28f --- /dev/null +++ b/lib/build/tasks/webpack/webpack.js @@ -0,0 +1,112 @@ +var webpack = require('webpack'); +var _ = require('underscore'); +var colors = require('colors'); +var pretty = require('prettysize'); +var glob = require('glob'); +var retrieveAffectedSpecs = require('../../affectedSpecs'); +var cfg = require('./webpack.config.js').task(); + +var compiler = {}; +var affectedSpecs = []; +var cache = {}; +var paths = { + builder_specs: './lib/assets/core/test/spec/cartodb3/**/*.spec.js' +}; + +/** + * affected - To be used as part of a 'registerTask' Grunt definition + */ +var affected = function (grunt) { + var done = this.async(); + + affectedSpecs = [ + './lib/build/source-map-support.js', + './lib/assets/core/javascripts/cartodb3/components/form-components/index.js' + ]; + + if (grunt.option('specs') === 'all') { + var allSpecs = glob.sync(paths.builder_specs); + console.log(colors.yellow('All specs. ' + allSpecs.length + ' specs found.')); + affectedSpecs = affectedSpecs.concat(allSpecs); + done(); + } else { + retrieveAffectedSpecs(grunt) + .then(function (specsList) { + console.log(colors.yellow(specsList.length + ' specs found.')); + affectedSpecs = affectedSpecs.concat(specsList); + done(); + }) + .catch(function (reason) { + throw new Error(reason); + }); + } +} + +var bootstrap = function (config) { + if (!config) { + throw new Error('Please provide subconfiguration key for webpack.'); + } + + if (!cfg[config]) { + throw new Error(config + ' section needed in webpack.config.js'); + } + + cfg[config].entry = function () { + return affectedSpecs; + } + + compiler[config] = webpack(cfg[config]); + cache[config] = {}; + compiler[config].apply(new webpack.CachePlugin(cache[config])); +}; + +function logAssets(assets) { + _.each(assets, function (asset) { + var trace = asset.name; + trace += ' ' + pretty(asset.size); + console.log(colors.yellow(trace)); + }); +} + +/** + * compile - To be used as part of a 'registerTask' Grunt definition + */ +var compile = function (config) { + if (!config) { + throw new Error('Please provide subconfiguration key for webpack.'); + } + + var done = this.async(); + + compiler[config].run(function (err, stats) { + if (err) { + console.error(err.stack || err); + if (err.details) { + console.error(err.details); + } + done(); + return; + } + + var info = stats.toJson(); + + if (stats.hasErrors()) { + console.error(colors.red(info.errors)); + } + + if (stats.hasWarnings()) { + console.warn(colors.yellow(info.warnings)); + } + + console.log(colors.yellow('Time: ' + info.time)); + logAssets(info.assets); + + done(); + }); +}; + +module.exports = { + affected: affected, + bootstrap: bootstrap, + compile: compile +}; diff --git a/package.json b/package.json index b322c6b2b6..f8179b2166 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "exports-loader": "0.6.4", "fs-extra": "2.0.0", "git-rev": "0.2.1", + "glob": "7.1.1", "grunt": "0.4.5", "grunt-available-tasks": "0.5.4", "grunt-aws": "0.3.0", @@ -72,7 +73,6 @@ "grunt-mustache": "0.1.7", "grunt-postcss": "0.5.5", "grunt-timer": "0.3.3", - "grunt-webpack": "2.0.1", "hogan.js": "^3.0.2", "jasmine": "2.5.2", "jasmine-ajax": "git://github.com/nobuti/jasmine-ajax.git#master", @@ -81,6 +81,7 @@ "minimist": "1.2.0", "mustache-loader": "^0.4.1", "open": "0.0.5", + "prettysize": "0.0.3", "recursive-readdir": "2.1.1", "semistandard": "7.0.4", "shim-loader": "0.1.0", @@ -175,7 +176,6 @@ "preupdate-internal-deps": "echo 'DEPRECATED TASK, use `npm run-script update-internal-deps` instead'", "update-internal-deps": "rm -rf node_modules && npm install --no-shrinkwrap && npm shrinkwrap", "branch-files": "node lib/build/branchFiles/branchFiles.js", - "affected_specs": "node lib/build/branchFiles/branchFiles.js | xargs node lib/build/affectedFiles/affectedFiles.js", - "webpack": "webpack" + "affected_specs": "node lib/build/branchFiles/branchFiles.js | xargs node lib/build/affectedFiles/affectedFiles.js" } }