'use strict'
const { Walker: IgnoreWalker } = require('ignore-walk')
const { lstatSync: lstat, readFileSync: readFile } = require('fs')
const { basename, dirname, extname, join, relative, resolve, sep } = require('path')
// symbols used to represent synthetic rule sets
const defaultRules = Symbol('npm-packlist.rules.default')
const strictRules = Symbol('npm-packlist.rules.strict')
// There may be others, but :?|<> are handled by node-tar
const nameIsBadForWindows = file => /\*/.test(file)
// these are the default rules that are applied to everything except for non-link bundled deps
const defaults = [
'.npmignore',
'.gitignore',
'**/.git',
'**/.svn',
'**/.hg',
'**/CVS',
'**/.git/**',
'**/.svn/**',
'**/.hg/**',
'**/CVS/**',
'/.lock-wscript',
'/.wafpickle-*',
'/build/config.gypi',
'npm-debug.log',
'**/.npmrc',
'.*.swp',
'.DS_Store',
'**/.DS_Store/**',
'._*',
'**/._*/**',
'*.orig',
'/archived-packages/**',
]
const strictDefaults = [
// these are forcibly included at all levels
'!/readme{,.*[^~$]}',
'!/copying{,.*[^~$]}',
'!/license{,.*[^~$]}',
'!/licence{,.*[^~$]}',
// these are forcibly excluded
'/.git',
]
const normalizePath = (path) => path.split('\\').join('/')
const readOutOfTreeIgnoreFiles = (root, rel, result = []) => {
for (const file of ['.npmignore', '.gitignore']) {
try {
const ignoreContent = readFile(join(root, file), { encoding: 'utf8' })
result.push(ignoreContent)
// break the loop immediately after reading, this allows us to prioritize
// the .npmignore and discard the .gitignore if one is present
break
} catch (err) {
// we ignore ENOENT errors completely because we don't care if the file doesn't exist
// but we throw everything else because failing to read a file that does exist is
// something that the user likely wants to know about
// istanbul ignore next -- we do not need to test a thrown error
if (err.code !== 'ENOENT') {
throw err
}
}
}
if (!rel) {
return result
}
const firstRel = rel.split(sep, 1)[0]
const newRoot = join(root, firstRel)
const newRel = relative(newRoot, join(root, rel))
return readOutOfTreeIgnoreFiles(newRoot, newRel, result)
}
class PackWalker extends IgnoreWalker {
constructor (tree, opts) {
const options = {
...opts,
includeEmpty: false,
follow: false,
// we path.resolve() here because ignore-walk doesn't do it and we want full paths
path: resolve(opts?.path || tree.path).replace(/\\/g, '/'),
ignoreFiles: opts?.ignoreFiles || [
defaultRules,
'package.json',
'.npmignore',
'.gitignore',
strictRules,
],
}
super(options)
this.isPackage = options.isPackage
this.seen = options.seen || new Set()
this.tree = tree
this.requiredFiles = options.requiredFiles || []
const additionalDefaults = []
if (options.prefix && options.workspaces) {
const path = normalizePath(options.path)
const prefix = normalizePath(options.prefix)
const workspaces = options.workspaces.map((ws) => normalizePath(ws))
// istanbul ignore else - this does nothing unless we need it to
if (path !== prefix && workspaces.includes(path)) {
// if path and prefix are not the same directory, and workspaces has path in it
// then we know path is a workspace directory. in order to not drop ignore rules
// from directories between the workspaces root (prefix) and the workspace itself
// (path) we need to find and read those now
const relpath = relative(options.prefix, dirname(options.path))
additionalDefaults.push(...readOutOfTreeIgnoreFiles(options.prefix, relpath))
} else if (path === prefix) {
// on the other hand, if the path and prefix are the same, then we ignore workspaces
// so that we don't pack a workspace as part of the root project. append them as
// normalized relative paths from the root
additionalDefaults.push(...workspaces.map((w) => normalizePath(relative(options.path, w))))
}
}
// go ahead and inject the default rules now
this.injectRules(defaultRules, [...defaults, ...additionalDefaults])
if (!this.isPackage) {
// if this instance is not a package, then place some strict default rules, and append
// known required files for this directory
this.injectRules(strictRules, [
...strictDefaults,
...this.requiredFiles.map((file) => `!${file}`),
])
}
}
// overridden method: we intercept the reading of the package.json file here so that we can
// process it into both the package.json file rules as well as the strictRules synthetic rule set
addIgnoreFile (file, callback) {
// if we're adding anything other than package.json, then let ignore-walk handle it
if (file !== 'package.json' || !this.isPackage) {
return super.addIgnoreFile(file, callback)
}
return this.processPackage(callback)
}
// overridden method: if we're done, but we're a package, then we also need to evaluate bundles
// before we actually emit our done event
emit (ev, data) {
if (ev !== 'done' || !this.isPackage) {
return super.emit(ev, data)
}
// we intentionally delay the done event while keeping the function sync here
// eslint-disable-next-line promise/catch-or-return, promise/always-return
this.gatherBundles().then(() => {
super.emit('done', this.result)
})
return true
}
// overridden method: before actually filtering, we make sure that we've removed the rules for
// files that should no longer take effect due to our order of precedence
filterEntries () {
if (this.ignoreRules['package.json']) {
// package.json means no .npmignore or .gitignore
this.ignoreRules['.npmignore'] = null
this.ignoreRules['.gitignore'] = null
} else if (this.ignoreRules['.npmignore']) {
// .npmignore means no .gitignore
this.ignoreRules['.gitignore'] = null
}
return super.filterEntries()
}
// overridden method: we never want to include anything that isn't a file or directory
onstat (opts, callback) {
if (!opts.st.isFile() && !opts.st.isDirectory()) {
return callback()
}
return super.onstat(opts, callback)
}
// overridden method: we want to refuse to pack files that are invalid, node-tar protects us from
// a lot of them but not all
stat (opts, callback) {
if (nameIsBadForWindows(opts.entry)) {
return callback()
}
return super.stat(opts, callback)
}
// overridden method: this is called to create options for a child walker when we step
// in to a normal child directory (this will never be a bundle). the default method here
// copies the root's `ignoreFiles` value, but we don't want to respect package.json for
// subdirectories, so we override it with a list that intentionally omits package.json
walkerOpt (entry, opts) {
let ignoreFiles = null
// however, if we have a tree, and we have workspaces, and the directory we're about
// to step into is a workspace, then we _do_ want to respect its package.json
if (this.tree.workspaces) {
const workspaceDirs = [...this.tree.workspaces.values()]
.map((dir) => dir.replace(/\\/g, '/'))
const entryPath = join(this.path, entry).replace(/\\/g, '/')
if (workspaceDirs.includes(entryPath)) {
ignoreFiles = [
defaultRules,
'package.json',
'.npmignore',
'.gitignore',
strictRules,
]
}
} else {
ignoreFiles = [
defaultRules,
'.npmignore',
'.gitignore',
strictRules,
]
}
return {
...super.walkerOpt(entry, opts),
ignoreFiles,
// we map over our own requiredFiles and pass ones that are within this entry
requiredFiles: this.requiredFiles
.map((file) => {
if (relative(file, entry) === '..') {
return relative(entry, file).replace(/\\/g, '/')
}
return false
})
.filter(Boolean),
}
}
// overridden method: we want child walkers to be instances of this class, not ignore-walk
walker (entry, opts, callback) {
new PackWalker(this.tree, this.walkerOpt(entry, opts)).on('done', callback).start()
}
// overridden method: we use a custom sort method to help compressibility
sort (a, b) {
// optimize for compressibility
// extname, then basename, then locale alphabetically
// https://twitter.com/isntitvacant/status/1131094910923231232
const exta = extname(a).toLowerCase()
const extb = extname(b).toLowerCase()
const basea = basename(a).toLowerCase()
const baseb = basename(b).toLowerCase()
return exta.localeCompare(extb, 'en') ||
basea.localeCompare(baseb, 'en') ||
a.localeCompare(b, 'en')
}
// convenience method: this joins the given rules with newlines, appends a trailing newline,
// and calls the internal onReadIgnoreFile method
injectRules (filename, rules, callback = () => {}) {
this.onReadIgnoreFile(filename, `${rules.join('\n')}\n`, callback)
}
// custom method: this is called by addIgnoreFile when we find a package.json, it uses the
// arborist tree to pull both default rules and strict rules for the package
processPackage (callback) {
const {
bin,
browser,
files,
main,
} = this.tree.package
// rules in these arrays are inverted since they are patterns we want to _not_ ignore
const ignores = []
const strict = [
...strictDefaults,
'!/package.json',
'/.git',
'/node_modules',
'/package-lock.json',
'/yarn.lock',
'/pnpm-lock.yaml',
]
// if we have a files array in our package, we need to pull rules from it
if (files) {
for (let file of files) {
// invert the rule because these are things we want to include
if (file.startsWith('/')) {
file = file.slice(1)
} else if (file.startsWith('./')) {
file = file.slice(2)
} else if (file.endsWith('/*')) {
file = file.slice(0, -2)
}
const inverse = `!${file}`
try {
// if an entry in the files array is a specific file, then we need to include it as a
// strict requirement for this package. if it's a directory or a pattern, it's a default
// pattern instead. this is ugly, but we have to stat to find out if it's a file
const stat = lstat(join(this.path, file.replace(/^!+/, '')).replace(/\\/g, '/'))
// if we have a file and we know that, it's strictly required
if (stat.isFile()) {
strict.unshift(inverse)
this.requiredFiles.push(file)
} else if (stat.isDirectory()) {
// otherwise, it's a default ignore, and since we got here we know it's not a pattern
// so we include the directory contents
ignores.push(inverse)
ignores.push(`${inverse}/**`)
}
// if the thing exists, but is neither a file or a directory, we don't want it at all
} catch (err) {
// if lstat throws, then we assume we're looking at a pattern and treat it as a default
ignores.push(inverse)
}
}
// we prepend a '*' to exclude everything, followed by our inverted file rules
// which now mean to include those
this.injectRules('package.json', ['*', ...ignores])
}
// browser is required
if (browser) {
strict.push(`!/${browser}`)
}
// main is required
if (main) {
strict.push(`!/${main}`)
}
// each bin is required
if (bin) {
for (const key in bin) {
strict.push(`!/${bin[key]}`)
}
}
// and now we add all of the strict rules to our synthetic file
this.injectRules(strictRules, strict, callback)
}
// custom method: after we've finished gathering the files for the root package, we call this
// before emitting the 'done' event in order to gather all of the files for bundled deps
async gatherBundles () {
if (this.seen.has(this.tree)) {
return
}
// add this node to our seen tracker
this.seen.add(this.tree)
// if we're the project root, then we look at our bundleDependencies, otherwise we got here
// because we're a bundled dependency of the root, which means we need to include all prod
// and optional dependencies in the bundle
let toBundle
if (this.tree.isProjectRoot) {
const { bundleDependencies } = this.tree.package
toBundle = bundleDependencies || []
} else {
const { dependencies, optionalDependencies } = this.tree.package
toBundle = Object.keys(dependencies || {}).concat(Object.keys(optionalDependencies || {}))
}
for (const dep of toBundle) {
const edge = this.tree.edgesOut.get(dep)
// no edgeOut = missing node, so skip it. we can't pack it if it's not here
// we also refuse to pack peer dependencies and dev dependencies
if (!edge || edge.peer || edge.dev) {
continue
}
// get a reference to the node we're bundling
const node = this.tree.edgesOut.get(dep).to
// if there's no node, this is most likely an optional dependency that hasn't been
// installed. just skip it.
if (!node) {
continue
}
// we use node.path for the path because we want the location the node was linked to,
// not where it actually lives on disk
const path = node.path
// but link nodes don't have edgesOut, so we need to pass in the target of the node
// in order to make sure we correctly traverse its dependencies
const tree = node.target
// and start building options to be passed to the walker for this package
const walkerOpts = {
path,
isPackage: true,
ignoreFiles: [],
seen: this.seen, // pass through seen so we can prevent infinite circular loops
}
// if our node is a link, we apply defaultRules. we don't do this for regular bundled
// deps because their .npmignore and .gitignore files are excluded by default and may
// override defaults
if (node.isLink) {
walkerOpts.ignoreFiles.push(defaultRules)
}
// _all_ nodes will follow package.json rules from their package root
walkerOpts.ignoreFiles.push('package.json')
// only link nodes will obey .npmignore or .gitignore
if (node.isLink) {
walkerOpts.ignoreFiles.push('.npmignore')
walkerOpts.ignoreFiles.push('.gitignore')
}
// _all_ nodes follow strict rules
walkerOpts.ignoreFiles.push(strictRules)
// create a walker for this dependency and gather its results
const walker = new PackWalker(tree, walkerOpts)
const bundled = await new Promise((pResolve, pReject) => {
walker.on('error', pReject)
walker.on('done', pResolve)
walker.start()
})
// now we make sure we have our paths correct from the root, and accumulate everything into
// our own result set to deduplicate
const relativeFrom = relative(this.root, walker.path)
for (const file of bundled) {
this.result.add(join(relativeFrom, file).replace(/\\/g, '/'))
}
}
}
}
const walk = (tree, options, callback) => {
if (typeof options === 'function') {
callback = options
options = {}
}
const p = new Promise((pResolve, pReject) => {
new PackWalker(tree, { ...options, isPackage: true })
.on('done', pResolve).on('error', pReject).start()
})
return callback ? p.then(res => callback(null, res), callback) : p
}
module.exports = walk
walk.Walker = PackWalker
|