From e94a2d8e961c7ed97d3af2b9ef7c5d0fda7e288c Mon Sep 17 00:00:00 2001 From: Guy Bedford Date: Fri, 18 Jan 2019 23:02:18 +0200 Subject: [PATCH 1/2] Asset relocator refactoring (#234) * Refactor binding analysis * check some vm overemissions --- scripts/build.js | 8 + src/loaders/relocate-loader.js | 588 +++++++----------------- src/utils/binary-locators.js | 83 ++++ src/utils/wrappers.js | 156 +++++++ test/unit/asset-fs-inline-tpl/output.js | 16 +- test/unit/main-equal/output.js | 4 +- 6 files changed, 422 insertions(+), 433 deletions(-) create mode 100644 src/utils/binary-locators.js create mode 100644 src/utils/wrappers.js diff --git a/scripts/build.js b/scripts/build.js index 7649f666..780bb8b7 100644 --- a/scripts/build.js +++ b/scripts/build.js @@ -70,6 +70,14 @@ async function main() { Object.keys(sourcemapAssets).length || Object.keys(typescriptAssets).some(asset => !asset.startsWith('lib/')) ) { + console.log(Object.keys(cliAssets)); + console.log(Object.keys(indexAssets).filter(asset => !asset.startsWith('locales/'))); + console.log(Object.keys(nodeLoaderAssets)); + console.log(Object.keys(relocateLoaderAssets)); + console.log(Object.keys(shebangLoaderAssets)); + console.log(Object.keys(tsLoaderAssets)); + console.log(Object.keys(sourcemapAssets)); + console.log(Object.keys(typescriptAssets).filter(asset => !asset.startsWith('lib/'))); console.error("New assets are being emitted by the core build"); } diff --git a/src/loaders/relocate-loader.js b/src/loaders/relocate-loader.js index 6ef5105b..72505620 100644 --- a/src/loaders/relocate-loader.js +++ b/src/loaders/relocate-loader.js @@ -1,5 +1,5 @@ const path = require('path'); -const fs = require('graceful-fs'); +const { readFile, stat, statSync, existsSync } = require('graceful-fs'); const { walk } = require('estree-walker'); const MagicString = require('magic-string'); const { attachScopes } = require('rollup-pluginutils'); @@ -10,93 +10,11 @@ const getUniqueAssetName = require('../utils/dedupe-names'); const sharedlibEmit = require('../utils/sharedlib-emit'); const glob = require('glob'); const getPackageBase = require('../utils/get-package-base'); +const { pregyp, nbind } = require('../utils/binary-locators'); +const handleWrappers = require('../utils/wrappers'); -// binary support for inlining logic from - node-pre-gyp/lib/pre-binding.js -function isPregypId (id) { - return id === 'node-pre-gyp' || - id === 'node-pre-gyp/lib/pre-binding' || - id === 'node-pre-gyp/lib/pre-binding.js'; -} -const versioning = require('node-pre-gyp/lib/util/versioning.js'); -const napi = require('node-pre-gyp/lib/util/napi.js'); -const pregyp = { - find (package_json_path, opts) { - const package_json = JSON.parse(fs.readFileSync(package_json_path).toString()); - versioning.validate_config(package_json, opts); - var napi_build_version; - if (napi.get_napi_build_versions (package_json, opts)) { - napi_build_version = napi.get_best_napi_build_version(package_json, opts); - } - opts = opts || {}; - if (!opts.module_root) opts.module_root = path.dirname(package_json_path); - var meta = versioning.evaluate(package_json,opts,napi_build_version); - return meta.module; - } -}; - -function getNbind () { - // Adapted from nbind.js - function makeModulePathList(root, name) { - return ([ - [root, name], - [root, 'build', name], - [root, 'build', 'Debug', name], - [root, 'build', 'Release', name], - [root, 'out', 'Debug', name], - [root, 'Debug', name], - [root, 'out', 'Release', name], - [root, 'Release', name], - [root, 'build', 'default', name], - [ - root, - process.env['NODE_BINDINGS_COMPILED_DIR'] || 'compiled', - process.versions.node, - process.platform, - process.arch, - name - ] - ]); - } - function findCompiledModule(basePath, specList) { - var resolvedList = []; - var ext = path.extname(basePath); - for (var _i = 0, specList_1 = specList; _i < specList_1.length; _i++) { - var spec = specList_1[_i]; - if (ext == spec.ext) { - try { - spec.path = eval('require.resolve(basePath)'); - return spec; - } - catch (err) { - resolvedList.push(basePath); - } - } - } - for (var _a = 0, specList_2 = specList; _a < specList_2.length; _a++) { - var spec = specList_2[_a]; - for (var _b = 0, _c = makeModulePathList(basePath, spec.name); _b < _c.length; _b++) { - var pathParts = _c[_b]; - var resolvedPath = path.resolve.apply(path, pathParts); - try { - spec.path = eval('require.resolve(resolvedPath)'); - } - catch (err) { - resolvedList.push(resolvedPath); - continue; - } - return spec; - } - } - return null; - } - function find(basePath = process.cwd()) { - return findCompiledModule(basePath, [ - { ext: '.node', name: 'nbind.node', type: 'node' }, - { ext: '.js', name: 'nbind.js', type: 'emcc' } - ]); - } - return { init: find, find: find }; -} +const staticPath = Object.assign({ default: path }, path); +const staticFs = { default: { existsSync }, existsSync }; function isExpressionReference(node, parent) { if (parent.type === 'MemberExpression') return parent.computed || node === parent.object; @@ -116,161 +34,6 @@ function isExpressionReference(node, parent) { return true; } -// Wrapper detections for require extraction handles: -// -// When.js-style AMD wrapper: -// (function (define) { 'use strict' define(function (require) { ... }) }) -// (typeof define === 'function' && define.amd ? define : function (factory) { module.exports = factory(require); }) -// -> -// (function (define) { 'use strict' define(function () { ... }) }) -// (typeof define === 'function' && define.amd ? define : function (factory) { module.exports = factory(require); }) -// -// Browserify-style wrapper -// (function(f){if(typeof exports==="object"&&typeof module!=="undefined"){module.exports=f()}else if(typeof define==="function"&&define.amd){define([],f)}else{var g;if(typeof window!=="undefined"){g=window}else if(typeof global!=="undefined"){g=global}else if(typeof self!=="undefined"){g=self}else{g=this}g.bugsnag = f()}})(function(){var define,module,exports;return (function(){function r(e,n,t){function o(i,f){if(!n[i]){if(!e[i]){var c="function"==typeof require&&require;if(!f&&c)return c(i,!0);if(u)return u(i,!0);var a=new Error("Cannot find module '"+i+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[i]={exports:{}};e[i][0].call(p.exports,function(r){var n=e[i][1][r];return o(n||r)},p,p.exports,r,e,n,t)}return n[i].exports}for(var u="function"==typeof require&&require,i=0;i -// (function(f){if(typeof exports==="object"&&typeof module!=="undefined"){module.exports=f()}else if(typeof define==="function"&&define.amd){define([],f)}else{var g;if(typeof window!=="undefined"){g=window}else if(typeof global!=="undefined"){g=global}else if(typeof self!=="undefined"){g=self}else{g=this}g.bugsnag = f()}})(function(){var define,module,exports;return (function(){function r(e,n,t){function o(i,f){if(!n[i]){if(!e[i]){var c="function"==typeof require&&require;if(!f&&c)return c(i,!0);if(u)return u(i,!0);var a=new Error("Cannot find module '"+i+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[i]={exports:{}};e[i][0].call(p.exports,function(r){var n=e[i][1][r];return o(n||r)},p,p.exports,r,e,n,t)}return n[i].exports}for(var u="function"==typeof require&&require,i=0;i decl.init === null && decl.id.type === 'Identifier') && - arg.body.body[1].type === 'ReturnStatement' && - arg.body.body[1].argument.type === 'CallExpression' && - arg.body.body[1].argument.callee.type === 'CallExpression' && - arg.body.body[1].argument.arguments.length && - arg.body.body[1].argument.arguments.every(arg => arg.type === 'Literal' && typeof arg.value === 'number') && - arg.body.body[1].argument.callee.callee.type === 'CallExpression' && - arg.body.body[1].argument.callee.callee.callee.type === 'FunctionExpression' && - arg.body.body[1].argument.callee.callee.arguments.length === 0 && - // (dont go deeper into browserify loader internals than this) - arg.body.body[1].argument.callee.arguments.length === 3 && - arg.body.body[1].argument.callee.arguments[0].type === 'ObjectExpression' && - arg.body.body[1].argument.callee.arguments[1].type === 'ObjectExpression' && - arg.body.body[1].argument.callee.arguments[2].type === 'ArrayExpression') { - const modules = arg.body.body[1].argument.callee.arguments[0].properties; - - // verify modules is the expected data structure - // in the process, extract external requires - const externals = {}; - if (modules.every(m => { - if (m.type !== 'Property' || - m.computed !== false || - m.key.type !== 'Literal' || - typeof m.key.value !== 'number' || - m.value.type !== 'ArrayExpression' || - m.value.elements.length !== 2 || - m.value.elements[0].type !== 'FunctionExpression' || - m.value.elements[1].type !== 'ObjectExpression') - return false; - - // detect externals from undefined moduleMap values - const moduleMap = m.value.elements[1].properties; - for (const prop of moduleMap) { - if (prop.type !== 'Property' || - (prop.value.type !== 'Identifier' && prop.value.type !== 'Literal') || - prop.key.type !== 'Literal' || - typeof prop.key.value !== 'string' || - prop.computed) - return false; - if (prop.value.type === 'Identifier' && prop.value.name === 'undefined') - externals[prop.key.value] = true; - } - return true; - })) { - // if we have externals, inline them into the browserify cache for webpack to pick up - const externalIds = Object.keys(externals); - if (externalIds.length) { - const cache = arg.body.body[1].argument.callee.arguments[1]; - const renderedExternals = externalIds.map(ext => `"${ext}": { exports: require("${ext}") }`).join(',\n '); - magicString.appendRight(cache.end - 1, renderedExternals); - transformed = true; - } - } - } - } - return { ast, scope, transformed }; -} - const relocateRegEx = /_\_dirname|_\_filename|require\.main|node-pre-gyp|bindings|define|require\(\s*[^'"]/; module.exports = function (code) { @@ -282,6 +45,32 @@ module.exports = function (code) { if (id.endsWith('.json') || !code.match(relocateRegEx)) return this.callback(null, code); + // calculate the base-level package folder to load bindings from + const pkgBase = getPackageBase(id); + + const staticModules = Object.assign(Object.create(null), { + path: staticPath, + fs: staticFs, + 'node-pre-gyp': pregyp, + 'node-pre-gyp/lib/pre-binding': pregyp, + 'node-pre-gyp/lib/pre-binding.js': pregyp, + 'nbind': nbind + }); + + let staticBindingsInstance = false; + function createBindings () { + return (opts = {}) => { + if (typeof opts === 'string') + opts = { bindings: opts }; + if (!opts.path) { + opts.path = true; + staticBindingsInstance = true; + } + opts.module_root = pkgBase; + return bindings(opts); + }; + } + const emitAsset = (assetPath) => { // JS assets to support require(assetPath) and not fs-based handling // NB package.json is ambiguous here... @@ -299,14 +88,14 @@ module.exports = function (code) { const name = assetState.assets[assetPath] || (assetState.assets[assetPath] = getUniqueAssetName(outName, assetPath, assetState.assetNames)); - // console.log('Emitting ' + assetPath + ' for module ' + id); + console.log('Emitting ' + assetPath + ' for module ' + id); assetEmissionPromises = assetEmissionPromises.then(async () => { const [source, permissions] = await Promise.all([ new Promise((resolve, reject) => - fs.readFile(assetPath, (err, source) => err ? reject(err) : resolve(source)) + readFile(assetPath, (err, source) => err ? reject(err) : resolve(source)) ), await new Promise((resolve, reject) => - fs.stat(assetPath, (err, stats) => err ? reject(err) : resolve(stats.mode)) + stat(assetPath, (err, stats) => err ? reject(err) : resolve(stats.mode)) ) ]); assetState.assetPermissions[name] = permissions; @@ -329,10 +118,10 @@ module.exports = function (code) { return; const [source, permissions] = await Promise.all([ new Promise((resolve, reject) => - fs.readFile(file, (err, source) => err ? reject(err) : resolve(source)) + readFile(file, (err, source) => err ? reject(err) : resolve(source)) ), await new Promise((resolve, reject) => - fs.stat(file, (err, stats) => err ? reject(err) : resolve(stats.mode)) + stat(file, (err, stats) => err ? reject(err) : resolve(stats.mode)) ) ]); assetState.assetPermissions[name + file.substr(assetDirPath.length)] = permissions; @@ -360,54 +149,63 @@ module.exports = function (code) { let scope = attachScopes(ast, 'scope'); - let pathId, pathImportIds = {}; - let fsId; - let pregypId, bindingsId, nbindId; + const knownBindings = Object.assign(Object.create(null), { + __dirname: { + shadowDepth: 0, + value: path.resolve(id, '..') + }, + __filename: { + shadowDepth: 0, + value: id + } + }); + + if (!isESM) + knownBindings.require = { + shadowDepth: 0 + }; - const shadowDepths = Object.create(null); - shadowDepths.__filename = 0; - shadowDepths.__dirname = 0; - if (!isESM) { - shadowDepths.require = 0; + function setKnownBinding (name, value) { + // require is somewhat special in that we shadow it but don't + // statically analyze it ("known unknown" of sorts) + if (name === 'require') return; + knownBindings[name] = { + shadowDepth: 0, + value: value + }; } - else { + function getKnownBinding (name) { + const binding = knownBindings[name]; + if (binding) { + if (binding.shadowDepth === 0) { + return binding.value; + } + } + } + + let nbindId, pregypId, bindingsId; + + if (isESM) { for (const decl of ast.body) { - // Detects: - // import * as path from 'path'; - // import path from 'path'; - // import { join } from 'path'; if (decl.type === 'ImportDeclaration') { const source = decl.source.value; - if (source === 'path') { + const staticModule = staticModules[source]; + if (staticModule) { for (const impt of decl.specifiers) { - if (impt.type === 'ImportNamespaceSpecifier' || impt.type === 'ImportDefaultSpecifier') { - pathId = impt.local.name; - shadowDepths[pathId] = 0; - } - else if (impt.type === 'ImportSpecifier') { - pathImportIds[impt.local.name] = impt.imported.name; - shadowDepths[impt.local.name] = 0; - } - } - } - // import binary from 'node-pre-gyp'; - // import * as binary from 'node-pre-gyp'; - // import { find } from 'node-pre-gyp' not yet implemented - else if (isPregypId(source)) { - for (const impt of decl.specifiers) { - if (impt.type === 'ImportNamespaceSpecifier' || impt.type === 'ImportDefaultSpecifier') { - pregypId = impt.local.name; - shadowDepths[pregypId] = 0; - } - } - } - // import bindings from 'bindings'; - else if (source === 'bindings') { - for (const impt of decl.specifiers) { - if (impt.type === 'ImportDefaultSpecifier') { - bindingsId = impt.local.name; - shadowDepths[bindingsId] = 0; - } + let bindingId; + if (impt.type === 'ImportNamespaceSpecifier') + setKnownBinding(bindingId = impt.local.name, staticModule); + else if (impt.type === 'ImportDefaultSpecifier' && 'default' in staticModule) + setKnownBinding(bindingId = impt.local.name, staticModule.default); + else if (impt.type === 'ImportSpecifier' && impt.imported.name in staticModule) + setKnownBinding(bindingId = impt.local.name, staticModule[impt.imported.name]); + + if (source === 'bindings') + bindingsId = bindingId; + else if (source === 'node-pre-gyp' || source === 'node-pre-gyp/lib/pre-binding' || source === 'node-pre-gyp/lib/pre-binding.js') + pregypId = bindingId; + else if (source === 'nbind') + nbindId = bindingId; } } } @@ -416,62 +214,18 @@ module.exports = function (code) { let transformed = false; - let staticBindingsInstance = false; - // calculate the base-level package folder to load bindings from - const pkgBase = getPackageBase(id); - function createBindings () { - return (opts = {}) => { - if (typeof opts === 'string') - opts = { bindings: opts }; - if (!opts.path) { - opts.path = true; - staticBindingsInstance = true; - } - opts.module_root = pkgBase; - return bindings(opts); - }; - } - function computeStaticValue (expr, bindingsReq) { + function computeStaticValue (expr) { staticBindingsInstance = false; // function expression analysis disabled due to static-eval locals bug if (expr.type === 'FunctionExpression') return; - const vars = {}; - if (shadowDepths.__dirname === 0) - vars.__dirname = path.resolve(id, '..'); - if (shadowDepths.__filename === 0) - vars.__filename = id; - if (pathId) { - if (shadowDepths[pathId] === 0) - vars[pathId] = path; - } - if (fsId) { - if (shadowDepths[fsId] === 0) - vars[fsId] = { - existsSync: fs.existsSync - }; - } - if (pregypId) { - if (shadowDepths[pregypId] === 0) - vars[pregypId] = pregyp; - } - if (bindingsId) { - if (shadowDepths[bindingsId] === 0) - vars[bindingsId] = createBindings(); - } - if (nbindId) { - if (shadowDepths[nbindId] === 0) - vars[nbindId] = getNbind(); - } - for (const pathFn of Object.keys(pathImportIds)) { - if (shadowDepths[pathFn] === 0) - vars[pathFn] = path[pathImportIds[pathFn]]; - } - if (bindingsReq && shadowDepths.require === 0) - vars.require = function (reqId) { - if (reqId === 'bindings') - return createBindings(); - }; + + const vars = Object.create(null); + Object.keys(knownBindings).forEach(name => { + const { shadowDepth, value } = knownBindings[name]; + if (shadowDepth === 0 && value !== undefined) + vars[name] = value; + }); // evaluate returns undefined for non-statically-analyzable return evaluate(expr, vars); @@ -487,24 +241,21 @@ module.exports = function (code) { node.type === 'CallExpression' && node.callee.type === 'Identifier' && node.callee.name === 'require' && - shadowDepths.require === 0 && + knownBindings.require.shadowDepth === 0 && node.arguments.length === 1 && node.arguments[0].type === 'Literal'; } // detect require(...) - function isRequire (node, requireResolve) { + function isRequire (node) { return node && node.type === 'CallExpression' && (node.callee.type === 'Identifier' && node.callee.name === 'require' && - shadowDepths.require === 0 || + knownBindings.require.shadowDepth === 0 || node.callee.type === 'MemberExpression' && node.callee.object.type === 'Identifier' && - node.callee.object.name === 'require' && - (!requireResolve || - node.callee.property.type === 'Identifier' && - node.callee.property.name === 'resolve')); + node.callee.object.name === 'require'); } function isAnalyzableRequire (expression) { @@ -521,8 +272,8 @@ module.exports = function (code) { if (node.scope) { scope = node.scope; for (const id in node.scope.declarations) { - if (id in shadowDepths) - shadowDepths[id]++; + if (id in knownBindings) + knownBindings[id].shadowDepth++; } } @@ -533,9 +284,10 @@ module.exports = function (code) { // __dirname, __filename, binary only currently as well as require('bindings')(...) // Can add require.resolve, import.meta.url, even path-like environment variables if (node.type === 'Identifier' && isExpressionReference(node, parent)) { - if (!shadowDepths[node.name]) { - if (node.name === '__dirname' || node.name === '__filename' || - node.name === pregypId || node.name === bindingsId) { + if (node.name === '__dirname' || node.name === '__filename' || + node.name === pregypId || node.name === bindingsId) { + const binding = getKnownBinding(node.name); + if (binding) { staticChildValue = computeStaticValue(node, false); // if it computes, then we start backtracking if (staticChildValue) { @@ -550,7 +302,7 @@ module.exports = function (code) { else if (node.type === 'CallExpression' && !isESM && isStaticRequire(node.callee) && node.callee.arguments[0].value === 'bindings') { - staticChildValue = computeStaticValue(node, true); + staticChildValue = createBindings()(computeStaticValue(node.arguments[0], true)); if (staticChildValue) { staticChildNode = node; staticChildValueBindingsInstance = staticBindingsInstance; @@ -584,7 +336,7 @@ module.exports = function (code) { else if (!isESM && node.type === 'MemberExpression' && node.object.type === 'Identifier' && node.object.name === 'require' && - !shadowDepths.require && + knownBindings.require.shadowDepth === 0 && node.property.type === 'Identifier' && node.property.name === 'main' && !node.computed) { @@ -598,85 +350,65 @@ module.exports = function (code) { transformed = true; } else if (!isESM && node.type === 'Property' && node.value.type === 'Identifier' && - node.value.name === 'require' && !shadowDepths.require) { + node.value.name === 'require' && knownBindings.require.shadowDepth === 0) { magicString.overwrite(node.value.start, node.value.end, '__non_webpack_require__'); transformed = true; } - - // for now we only support top-level variable declarations - // so "var { join } = require('path')" will only detect in the top scope. - // Intermediate scope handling for these requires is straightforward, but - // would need nested shadow depth handling of the pathIds. - else if (parent === ast && node.type === 'VariableDeclaration') { + else if (node.type === 'VariableDeclaration') { for (const decl of node.declarations) { - // var path = require('path') - if (decl.id.type === 'Identifier' && - !isESM && isStaticRequire(decl.init)) { - if (decl.init.arguments[0].value === 'path') { - pathId = decl.id.name; - shadowDepths[pathId] = 0; - return this.skip(); - } - // var fs = require('fs') - else if (decl.init.arguments[0].value === 'fs') { - fsId = decl.id.name; - shadowDepths[fsId] = 0; - return this.skip(); - } - // var binary = require('node-pre-gyp') - else if (isPregypId(decl.init.arguments[0].value)) { - pregypId = decl.id.name; - shadowDepths[pregypId] = 0; - return this.skip(); - } - // var bindings = require('bindings') - else if (decl.init.arguments[0].value === 'bindings') { - bindingsId = decl.id.name; - shadowDepths[bindingsId] = 0; - return this.skip(); - } - // var nbind = require('nbind') - else if (decl.init.arguments[0].value === 'nbind') { - nbindId = decl.id.name; - shadowDepths[nbindId] = 0; + let binding; + if (!isESM && isStaticRequire(decl.init)) { + const source = decl.init.arguments[0].value; + const staticModule = staticModules[source]; + if (staticModule) { + // var known = require('known'); + if (decl.id.type === 'Identifier') { + setKnownBinding(decl.id.name, staticModule.default); + if (source === 'bindings') + bindingsId = decl.id.name; + else if (source === 'node-pre-gyp' || source === 'node-pre-gyp/lib/pre-binding' || source === 'node-pre-gyp/lib/pre-binding.js') + pregypId = decl.id.name; + else if (source === 'nbind') + nbindId = decl.id.name; + } + // var { known } = require('known); + else if (decl.id.type === 'ObjectPattern') { + for (const prop of decl.id.properties) { + if (prop.type !== 'Property' || + prop.key.type !== 'Identifier' || + prop.value.type !== 'Identifier' || + !(prop.key.name in staticModule)) + continue; + setKnownBinding(prop.value.name, staticModule[prop.key.name]); + } + } } } - // var { join } = path | require('path'); - else if (decl.id.type === 'ObjectPattern' && decl.init && - (decl.init.type === 'Identifier' && decl.init.name === pathId && shadowDepths[pathId] === 0) || - !isESM && isStaticRequire(decl.init) && decl.init.arguments[0].value === 'path') { - for (const prop of decl.id.properties) { - if (prop.type !== 'Property' || - prop.key.type !== 'Identifier' || - prop.value.type !== 'Identifier') - continue; - pathImportIds[prop.value.name] = prop.key.name; - shadowDepths[prop.key.name] = 0; - return this.skip(); - } + // var { knownProp } = known; + else if (decl.id.type === 'ObjectPattern' && + decl.init && decl.init.type === 'Identifier' && + (binding = getKnownBinding(decl.init.name)) !== undefined && + prop.key.name in binding) { + setKnownBinding(prop.value.name, binding[prop.key.name]); } - // var join = path.join + // var known = known.knownProp; else if (decl.id.type === 'Identifier' && - decl.init && - decl.init.type === 'MemberExpression' && - decl.init.object.type === 'Identifier' && - decl.init.object.name === pathId && - shadowDepths[decl.init.object.name] === 0 && - decl.init.computed === false && - decl.init.property.type === 'Identifier') { - pathImportIds[decl.init.property.name] = decl.id.name; - shadowDepths[decl.id.name] = 0; - return this.skip(); + decl.init && + decl.init.type === 'MemberExpression' && + decl.init.computed === false && + decl.init.object.type === 'Identifier' && + decl.init.property.type === 'Identifier' && + (binding = getKnownBinding(decl.init.object.name)) !== undefined && + decl.init.property.name in binding) { + setKnownBinding(decl.id.name, binding[decl.init.property.name]); } } } else if (node.type === 'AssignmentExpression') { // path = require('path') - if (isStaticRequire(node.right) && node.right.arguments[0].value === 'path' && + if (isStaticRequire(node.right) && node.right.arguments[0].value in staticModules && node.left.type === 'Identifier' && scope.declarations[node.left.name]) { - pathId = node.left.name; - shadowDepths[pathId] = 0; - return this.skip(); + setKnownBinding(node.left.name, staticModules[node.right.arguments[0].value]); } } }, @@ -684,8 +416,11 @@ module.exports = function (code) { if (node.scope) { scope = scope.parent; for (const id in node.scope.declarations) { - if (id in shadowDepths) { - shadowDepths[id]--; + if (id in knownBindings) { + if (knownBindings[id].shadowDepth > 0) + knownBindings[id].shadowDepth--; + else + delete knownBindings[id]; } } } @@ -700,12 +435,19 @@ module.exports = function (code) { staticChildValueBindingsInstance = staticBindingsInstance; return; } + // Filter out emitting assets for a __filename call on its own + if (staticChildNode.type === 'Identifier' && staticChildNode.name === '__filename' || + staticChildNode.type === 'ReturnStatement' && staticChildNode.argument.type === 'Identifier' && + staticChildNode.argument.name === '__filename') { + staticChildNode = staticChilValue = undefined; + return; + } // no static value -> see if we should emit the asset if it exists // Currently we only handle files. In theory whole directories could also be emitted if necessary. let stats; if (typeof staticChildValue === 'string') { try { - stats = fs.statSync(staticChildValue); + stats = statSync(staticChildValue); } catch (e) {} } diff --git a/src/utils/binary-locators.js b/src/utils/binary-locators.js new file mode 100644 index 00000000..4ce1b834 --- /dev/null +++ b/src/utils/binary-locators.js @@ -0,0 +1,83 @@ +const path = require("path"); +const fs = require("fs"); + +// pregyp +const versioning = require("node-pre-gyp/lib/util/versioning.js"); +const napi = require("node-pre-gyp/lib/util/napi.js"); +const pregypFind = (package_json_path, opts) => { + const package_json = JSON.parse(fs.readFileSync(package_json_path).toString()); + versioning.validate_config(package_json, opts); + var napi_build_version; + if (napi.get_napi_build_versions (package_json, opts)) { + napi_build_version = napi.get_best_napi_build_version(package_json, opts); + } + opts = opts || {}; + if (!opts.module_root) opts.module_root = path.dirname(package_json_path); + var meta = versioning.evaluate(package_json,opts,napi_build_version); + return meta.module; +}; +exports.pregyp = { default: { find: pregypFind }, find: pregypFind }; + +// nbind +// Adapted from nbind.js +function makeModulePathList(root, name) { + return ([ + [root, name], + [root, "build", name], + [root, "build", "Debug", name], + [root, "build", "Release", name], + [root, "out", "Debug", name], + [root, "Debug", name], + [root, "out", "Release", name], + [root, "Release", name], + [root, "build", "default", name], + [ + root, + process.env["NODE_BINDINGS_COMPILED_DIR"] || "compiled", + process.versions.node, + process.platform, + process.arch, + name + ] + ]); +} +function findCompiledModule(basePath, specList) { + var resolvedList = []; + var ext = path.extname(basePath); + for (var _i = 0, specList_1 = specList; _i < specList_1.length; _i++) { + var spec = specList_1[_i]; + if (ext == spec.ext) { + try { + spec.path = eval("require.resolve(basePath)"); + return spec; + } + catch (err) { + resolvedList.push(basePath); + } + } + } + for (var _a = 0, specList_2 = specList; _a < specList_2.length; _a++) { + var spec = specList_2[_a]; + for (var _b = 0, _c = makeModulePathList(basePath, spec.name); _b < _c.length; _b++) { + var pathParts = _c[_b]; + var resolvedPath = path.resolve.apply(path, pathParts); + try { + spec.path = eval("require.resolve(resolvedPath)"); + } + catch (err) { + resolvedList.push(resolvedPath); + continue; + } + return spec; + } + } + return null; +} +function find(basePath = process.cwd()) { + return findCompiledModule(basePath, [ + { ext: ".node", name: "nbind.node", type: "node" }, + { ext: ".js", name: "nbind.js", type: "emcc" } + ]); +} +exports.nbind = { default: { init: find, find }, init: find, find }; + diff --git a/src/utils/wrappers.js b/src/utils/wrappers.js new file mode 100644 index 00000000..d8704171 --- /dev/null +++ b/src/utils/wrappers.js @@ -0,0 +1,156 @@ +// Wrapper detections for require extraction handles: +// +// When.js-style AMD wrapper: +// (function (define) { 'use strict' define(function (require) { ... }) }) +// (typeof define === 'function' && define.amd ? define : function (factory) { module.exports = factory(require); }) +// -> +// (function (define) { 'use strict' define(function () { ... }) }) +// (typeof define === 'function' && define.amd ? define : function (factory) { module.exports = factory(require); }) +// +// Browserify-style wrapper +// (function(f){if(typeof exports==="object"&&typeof module!=="undefined"){module.exports=f()}else if(typeof define==="function"&&define.amd){define([],f)}else{var g;if(typeof window!=="undefined"){g=window}else if(typeof global!=="undefined"){g=global}else if(typeof self!=="undefined"){g=self}else{g=this}g.bugsnag = f()}})(function(){var define,module,exports;return (function(){function r(e,n,t){function o(i,f){if(!n[i]){if(!e[i]){var c="function"==typeof require&&require;if(!f&&c)return c(i,!0);if(u)return u(i,!0);var a=new Error("Cannot find module '"+i+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[i]={exports:{}};e[i][0].call(p.exports,function(r){var n=e[i][1][r];return o(n||r)},p,p.exports,r,e,n,t)}return n[i].exports}for(var u="function"==typeof require&&require,i=0;i +// (function(f){if(typeof exports==="object"&&typeof module!=="undefined"){module.exports=f()}else if(typeof define==="function"&&define.amd){define([],f)}else{var g;if(typeof window!=="undefined"){g=window}else if(typeof global!=="undefined"){g=global}else if(typeof self!=="undefined"){g=self}else{g=this}g.bugsnag = f()}})(function(){var define,module,exports;return (function(){function r(e,n,t){function o(i,f){if(!n[i]){if(!e[i]){var c="function"==typeof require&&require;if(!f&&c)return c(i,!0);if(u)return u(i,!0);var a=new Error("Cannot find module '"+i+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[i]={exports:{}};e[i][0].call(p.exports,function(r){var n=e[i][1][r];return o(n||r)},p,p.exports,r,e,n,t)}return n[i].exports}for(var u="function"==typeof require&&require,i=0;i decl.init === null && decl.id.type === 'Identifier') && + arg.body.body[1].type === 'ReturnStatement' && + arg.body.body[1].argument.type === 'CallExpression' && + arg.body.body[1].argument.callee.type === 'CallExpression' && + arg.body.body[1].argument.arguments.length && + arg.body.body[1].argument.arguments.every(arg => arg.type === 'Literal' && typeof arg.value === 'number') && + arg.body.body[1].argument.callee.callee.type === 'CallExpression' && + arg.body.body[1].argument.callee.callee.callee.type === 'FunctionExpression' && + arg.body.body[1].argument.callee.callee.arguments.length === 0 && + // (dont go deeper into browserify loader internals than this) + arg.body.body[1].argument.callee.arguments.length === 3 && + arg.body.body[1].argument.callee.arguments[0].type === 'ObjectExpression' && + arg.body.body[1].argument.callee.arguments[1].type === 'ObjectExpression' && + arg.body.body[1].argument.callee.arguments[2].type === 'ArrayExpression') { + const modules = arg.body.body[1].argument.callee.arguments[0].properties; + + // verify modules is the expected data structure + // in the process, extract external requires + const externals = {}; + if (modules.every(m => { + if (m.type !== 'Property' || + m.computed !== false || + m.key.type !== 'Literal' || + typeof m.key.value !== 'number' || + m.value.type !== 'ArrayExpression' || + m.value.elements.length !== 2 || + m.value.elements[0].type !== 'FunctionExpression' || + m.value.elements[1].type !== 'ObjectExpression') + return false; + + // detect externals from undefined moduleMap values + const moduleMap = m.value.elements[1].properties; + for (const prop of moduleMap) { + if (prop.type !== 'Property' || + (prop.value.type !== 'Identifier' && prop.value.type !== 'Literal') || + prop.key.type !== 'Literal' || + typeof prop.key.value !== 'string' || + prop.computed) + return false; + if (prop.value.type === 'Identifier' && prop.value.name === 'undefined') + externals[prop.key.value] = true; + } + return true; + })) { + // if we have externals, inline them into the browserify cache for webpack to pick up + const externalIds = Object.keys(externals); + if (externalIds.length) { + const cache = arg.body.body[1].argument.callee.arguments[1]; + const renderedExternals = externalIds.map(ext => `"${ext}": { exports: require("${ext}") }`).join(',\n '); + magicString.appendRight(cache.end - 1, renderedExternals); + transformed = true; + } + } + } + } + return { ast, scope, transformed }; +} + +module.exports = handleWrappers; diff --git a/test/unit/asset-fs-inline-tpl/output.js b/test/unit/asset-fs-inline-tpl/output.js index 4ee992e0..cca7eabb 100644 --- a/test/unit/asset-fs-inline-tpl/output.js +++ b/test/unit/asset-fs-inline-tpl/output.js @@ -32,23 +32,23 @@ module.exports = /******/ /******/ /******/ // Load entry module and return exports -/******/ return __webpack_require__(717); +/******/ return __webpack_require__(241); /******/ }) /************************************************************************/ /******/ ({ -/***/ 66: -/***/ (function(module) { +/***/ 241: +/***/ (function(__unusedmodule, __unusedexports, __webpack_require__) { -module.exports = require("fs"); +const fs = __webpack_require__(66); +console.log(fs.readFileSync(__dirname + '/asset.txt')); /***/ }), -/***/ 717: -/***/ (function(__unusedmodule, __unusedexports, __webpack_require__) { +/***/ 66: +/***/ (function(module) { -const fs = __webpack_require__(66); -console.log(fs.readFileSync(__dirname + '/asset.txt')); +module.exports = require("fs"); /***/ }) diff --git a/test/unit/main-equal/output.js b/test/unit/main-equal/output.js index c680db06..edb55863 100644 --- a/test/unit/main-equal/output.js +++ b/test/unit/main-equal/output.js @@ -36,12 +36,12 @@ module.exports = /******/ runtime(__webpack_require__); /******/ /******/ // Load entry module and return exports -/******/ return __webpack_require__(__webpack_require__.s = 336); +/******/ return __webpack_require__(__webpack_require__.s = 599); /******/ }) /************************************************************************/ /******/ ({ -/***/ 336: +/***/ 599: /***/ (function(module, __unusedexports, __webpack_require__) { /* module decorator */ module = __webpack_require__.nmd(module); From 1c0ae20293da6181b32a1504da4141daaabcf60a Mon Sep 17 00:00:00 2001 From: Guy Bedford Date: Fri, 18 Jan 2019 23:45:43 +0200 Subject: [PATCH 2/2] v8cache experimental flag (#112) * v8cache experimental flag * v8cache fixes for different filename, console handling * v8cache self build * fixup global context handling * update wrapper approach * fixup rebase --- scripts/build.js | 72 ++++++++++++------- src/cli.js | 7 +- src/index.js | 27 +++++-- src/loaders/relocate-loader.js | 2 +- .../output.js | 12 ++-- test/unit/tsconfig-paths/output.js | 12 ++-- 6 files changed, 84 insertions(+), 48 deletions(-) diff --git a/scripts/build.js b/scripts/build.js index 780bb8b7..105c8197 100644 --- a/scripts/build.js +++ b/scripts/build.js @@ -10,10 +10,13 @@ async function main() { const { code: cli, assets: cliAssets } = await ncc( __dirname + "/../src/cli", { + filename: "cli.js", externals: ["./index.js"], - minify: true + minify: true, + v8cache: true } ); + const { code: index, assets: indexAssets } = await ncc( __dirname + "/../src/index", { @@ -22,36 +25,44 @@ async function main() { // bundle, webpack (and therefore ncc) cannot currently bundle // chokidar, which is quite convenient externals: ["chokidar"], - minify: true + filename: "index.js", + minify: true, + v8cache: true } ); const { code: nodeLoader, assets: nodeLoaderAssets } = await ncc( __dirname + "/../src/loaders/node-loader", - { minify: true } + { + filename: "node-loader.js", + minify: true, + v8cache: true + } ); const { code: relocateLoader, assets: relocateLoaderAssets } = await ncc( __dirname + "/../src/loaders/relocate-loader", - { minify: true } + { filename: "relocate-loader.js", minify: true, v8cache: true } ); const { code: shebangLoader, assets: shebangLoaderAssets } = await ncc( __dirname + "/../src/loaders/shebang-loader", - { minify: true } + { filename: "shebang-loader.js", minify: true, v8cache: true } ); const { code: tsLoader, assets: tsLoaderAssets } = await ncc( __dirname + "/../src/loaders/ts-loader", { externals: ["typescript"], - minify: true + filename: "ts-loader.js", + minify: true, + v8cache: true } ); const { code: sourcemapSupport, assets: sourcemapAssets } = await ncc( require.resolve("source-map-support/register"), - { minfiy: true } + { filename: "source-register.js", minfiy: true, v8cache: true } ); const { code: typescript, assets: typescriptAssets } = await ncc( @@ -60,27 +71,38 @@ async function main() { ); // detect unexpected asset emissions from core build - if ( - Object.keys(cliAssets).length || - Object.keys(indexAssets).some(asset => !asset.startsWith('locales/')) || - Object.keys(nodeLoaderAssets).length || - Object.keys(relocateLoaderAssets).length || - Object.keys(shebangLoaderAssets).length || - Object.keys(tsLoaderAssets).length || - Object.keys(sourcemapAssets).length || - Object.keys(typescriptAssets).some(asset => !asset.startsWith('lib/')) - ) { - console.log(Object.keys(cliAssets)); - console.log(Object.keys(indexAssets).filter(asset => !asset.startsWith('locales/'))); - console.log(Object.keys(nodeLoaderAssets)); - console.log(Object.keys(relocateLoaderAssets)); - console.log(Object.keys(shebangLoaderAssets)); - console.log(Object.keys(tsLoaderAssets)); - console.log(Object.keys(sourcemapAssets)); - console.log(Object.keys(typescriptAssets).filter(asset => !asset.startsWith('lib/'))); + const unknownAssets = [ + ...Object.keys(cliAssets), + ...Object.keys(indexAssets).filter(asset => !asset.startsWith('locales/')), + ...Object.keys(nodeLoaderAssets), + ...Object.keys(relocateLoaderAssets), + ...Object.keys(shebangLoaderAssets), + ...Object.keys(tsLoaderAssets).filter(asset => !asset.startsWith('lib/')), + ...Object.keys(sourcemapAssets), + ...Object.keys(typescriptAssets).filter(asset => !asset.startsWith('lib/')) + ].filter(asset => !asset.endsWith('.js.cache') && !asset.endsWith('.cache.js')); + + if (unknownAssets.length) { console.error("New assets are being emitted by the core build"); + console.log(unknownAssets); } + writeFileSync(__dirname + "/../dist/ncc/cli.js.cache", cliAssets["cli.js.cache"]); + writeFileSync(__dirname + "/../dist/ncc/index.js.cache", indexAssets["index.js.cache"]); + writeFileSync(__dirname + "/../dist/ncc/sourcemap-register.js.cache", sourcemapAssets["sourcemap-register.js.cache"]); + writeFileSync(__dirname + "/../dist/ncc/loaders/node-loader.js.cache", nodeLoaderAssets["node-loader.js.cache"]); + writeFileSync(__dirname + "/../dist/ncc/loaders/relocate-loader.js.cache", relocateLoaderAssets["relocate-loader.js.cache"]); + writeFileSync(__dirname + "/../dist/ncc/loaders/shebang-loader.js.cache", shebangLoaderAssets["shebang-loader.js.cache"]); + writeFileSync(__dirname + "/../dist/ncc/loaders/ts-loader.js.cache", tsLoaderAssets["ts-loader.js.cache"]); + + writeFileSync(__dirname + "/../dist/ncc/cli.js.cache.js", cliAssets["cli.js.cache.js"]); + writeFileSync(__dirname + "/../dist/ncc/index.js.cache.js", indexAssets["index.js.cache.js"]); + writeFileSync(__dirname + "/../dist/ncc/sourcemap-register.js.cache.js", sourcemapAssets["sourcemap-register.js.cache.js"]); + writeFileSync(__dirname + "/../dist/ncc/loaders/node-loader.js.cache.js", nodeLoaderAssets["node-loader.js.cache.js"]); + writeFileSync(__dirname + "/../dist/ncc/loaders/relocate-loader.js.cache.js", relocateLoaderAssets["relocate-loader.js.cache.js"]); + writeFileSync(__dirname + "/../dist/ncc/loaders/shebang-loader.js.cache.js", shebangLoaderAssets["shebang-loader.js.cache.js"]); + writeFileSync(__dirname + "/../dist/ncc/loaders/ts-loader.js.cache.js", tsLoaderAssets["ts-loader.js.cache.js"]); + writeFileSync(__dirname + "/../dist/ncc/cli.js", cli); writeFileSync(__dirname + "/../dist/ncc/index.js", index); writeFileSync(__dirname + "/../dist/ncc/typescript/index.js", ` diff --git a/src/cli.js b/src/cli.js index 577c8c1a..d276c315 100755 --- a/src/cli.js +++ b/src/cli.js @@ -23,6 +23,7 @@ Options: -e, --external [mod] Skip bundling 'mod'. Can be used many times -q, --quiet Disable build summaries / non-error outputs -w, --watch Start a watched build + --v8-cache Emit a build using the v8 compile cache `; let args; @@ -42,7 +43,8 @@ try { "--quiet": Boolean, "-q": "--quiet", "--watch": Boolean, - "-w": "--watch" + "-w": "--watch", + "--v8-cache": Boolean }); } catch (e) { if (e.message.indexOf("Unknown or unexpected option") === -1) throw e; @@ -173,7 +175,8 @@ switch (args._[0]) { externals: args["--external"], sourceMap: args["--source-map"] || run, cache: args["--no-cache"] ? false : undefined, - watch: args["--watch"] + watch: args["--watch"], + v8cache: args["--v8-cache"] } ); diff --git a/src/index.js b/src/index.js index 48efe937..d224f3b7 100644 --- a/src/index.js +++ b/src/index.js @@ -37,17 +37,24 @@ module.exports = ( filename = "index.js", minify = false, sourceMap = false, - watch = false + watch = false, + v8cache = false } = {} ) => { const resolvedEntry = resolve.sync(entry); const shebangMatch = fs.readFileSync(resolvedEntry).toString().match(shebangRegEx); const mfs = new MemoryFS(); + const assetNames = Object.create(null); + assetNames[filename] = true; + if (sourceMap) + assetNames[filename + '.map'] = true; + if (v8cache) + assetNames[filename + '.cache'] = assetNames[filename + '.cache.js'] = true; const resolvePlugins = []; let tsconfigMatchPath; const assetState = { assets: Object.create(null), - assetNames: Object.create(null), + assetNames, assetPermissions: undefined }; assetState.assetNames[filename] = true; @@ -272,8 +279,8 @@ module.exports = ( getFlatFiles(mfs.data, assets, assetState.assetPermissions); delete assets[filename]; delete assets[filename + ".map"]; - let code = mfs.readFileSync("/" + filename, "utf8"); - let map = sourceMap ? mfs.readFileSync("/" + filename + ".map", "utf8") : null; + let code = mfs.readFileSync(`/${filename}`, "utf8"); + let map = sourceMap ? mfs.readFileSync(`/${filename}.map`, "utf8") : null; if (minify) { const result = terser.minify(code, { @@ -294,6 +301,18 @@ module.exports = ( ({ code, map } = { code: result.code, map: result.map }); } + if (v8cache) { + const { Script } = require('vm'); + assets[filename + '.cache'] = new Script(code).createCachedData(); + assets[filename + '.cache.js'] = code; + if (map) + assets[filename + '.map'] = map; + code = `const { readFileSync } = require('fs'), { Script } = require('vm'), { wrap } = require('module');\n` + + `const source = readFileSync(__dirname + '/${filename}.cache.js').toString(), cachedData = readFileSync(__dirname + '/${filename}.cache');\n` + + `(new Script(wrap(source), { cachedData }).runInThisContext())(exports, require, module, __filename, __dirname);\n`; + if (map) map = {}; + } + if (shebangMatch) { code = shebangMatch[0] + code; // add a line offset to the sourcemap diff --git a/src/loaders/relocate-loader.js b/src/loaders/relocate-loader.js index 72505620..cf0914f0 100644 --- a/src/loaders/relocate-loader.js +++ b/src/loaders/relocate-loader.js @@ -88,7 +88,7 @@ module.exports = function (code) { const name = assetState.assets[assetPath] || (assetState.assets[assetPath] = getUniqueAssetName(outName, assetPath, assetState.assetNames)); - console.log('Emitting ' + assetPath + ' for module ' + id); + // console.log('Emitting ' + assetPath + ' for module ' + id); assetEmissionPromises = assetEmissionPromises.then(async () => { const [source, permissions] = await Promise.all([ new Promise((resolve, reject) => diff --git a/test/unit/tsconfig-paths-conflicting-external/output.js b/test/unit/tsconfig-paths-conflicting-external/output.js index a45e673d..1132f92d 100644 --- a/test/unit/tsconfig-paths-conflicting-external/output.js +++ b/test/unit/tsconfig-paths-conflicting-external/output.js @@ -43,20 +43,16 @@ module.exports = "use strict"; exports.__esModule = true; -var _module_1 = __webpack_require__(563); +var _module_1 = __webpack_require__(816); console.log(_module_1["default"]); /***/ }), -/***/ 563: -/***/ (function(__unusedmodule, exports) { - -"use strict"; - -exports.__esModule = true; -exports["default"] = {}; +/***/ 816: +/***/ (function(module) { +module.exports = require("@module"); /***/ }) diff --git a/test/unit/tsconfig-paths/output.js b/test/unit/tsconfig-paths/output.js index f3aa579a..805f6078 100644 --- a/test/unit/tsconfig-paths/output.js +++ b/test/unit/tsconfig-paths/output.js @@ -43,20 +43,16 @@ module.exports = "use strict"; exports.__esModule = true; -var _module_1 = __webpack_require__(637); +var _module_1 = __webpack_require__(816); console.log(_module_1["default"]); /***/ }), -/***/ 637: -/***/ (function(__unusedmodule, exports) { - -"use strict"; - -exports.__esModule = true; -exports["default"] = {}; +/***/ 816: +/***/ (function(module) { +module.exports = require("@module"); /***/ })