Skip to content

Commit

Permalink
fs: allow exclude option in globs to accept glob patterns
Browse files Browse the repository at this point in the history 
Signed-off-by: Daeyeon Jeong <daeyeon.dev@gmail.com>
PR-URL: #56489
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Chemi Atlow <chemi@atlow.co.il>
Reviewed-By: Moshe Atlow <moshe@atlow.co.il>
Reviewed-By: Jason Zhang <xzha4350@gmail.com>
  • Loading branch information
@daeyeon
daeyeon authored Jan 8, 2025
1 parent 5747828 commit dc5d0f9
Show file tree
Hide file tree
Showing 3 changed files with 227 additions and 36 deletions.
18 changes: 15 additions & 3 deletions doc/api/fs.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1074,6 +1074,9 @@ behavior is similar to `cp dir1/ dir2/`.
<!-- YAML
added: v22.0.0
changes:
- version: REPLACEME
pr-url: https://github.com/nodejs/node/pull/56489
description: Add support for `exclude` option to accept glob patterns.
- version: v22.2.0
pr-url: https://github.com/nodejs/node/pull/52837
description: Add support for `withFileTypes` as an option.
Expand All @@ -1084,7 +1087,8 @@ changes:
* `pattern` {string|string\[]}
* `options` {Object}
* `cwd` {string} current working directory. **Default:** `process.cwd()`
* `exclude` {Function} Function to filter out files/directories. Return
* `exclude` {Function|string\[]} Function to filter out files/directories or a
list of glob patterns to be excluded. If a function is provided, return
`true` to exclude the item, `false` to include it. **Default:** `undefined`.
* `withFileTypes` {boolean} `true` if the glob should return paths as Dirents,
`false` otherwise. **Default:** `false`.
Expand Down Expand Up @@ -3120,6 +3124,9 @@ descriptor. See [`fs.utimes()`][].
<!-- YAML
added: v22.0.0
changes:
- version: REPLACEME
pr-url: https://github.com/nodejs/node/pull/56489
description: Add support for `exclude` option to accept glob patterns.
- version: v22.2.0
pr-url: https://github.com/nodejs/node/pull/52837
description: Add support for `withFileTypes` as an option.
Expand All @@ -3131,7 +3138,8 @@ changes:
* `options` {Object}
* `cwd` {string} current working directory. **Default:** `process.cwd()`
* `exclude` {Function} Function to filter out files/directories. Return
* `exclude` {Function|string\[]} Function to filter out files/directories or a
list of glob patterns to be excluded. If a function is provided, return
`true` to exclude the item, `false` to include it. **Default:** `undefined`.
* `withFileTypes` {boolean} `true` if the glob should return paths as Dirents,
`false` otherwise. **Default:** `false`.
Expand Down Expand Up @@ -5656,6 +5664,9 @@ Synchronous version of [`fs.futimes()`][]. Returns `undefined`.
<!-- YAML
added: v22.0.0
changes:
- version: REPLACEME
pr-url: https://github.com/nodejs/node/pull/56489
description: Add support for `exclude` option to accept glob patterns.
- version: v22.2.0
pr-url: https://github.com/nodejs/node/pull/52837
description: Add support for `withFileTypes` as an option.
Expand All @@ -5666,7 +5677,8 @@ changes:
* `pattern` {string|string\[]}
* `options` {Object}
* `cwd` {string} current working directory. **Default:** `process.cwd()`
* `exclude` {Function} Function to filter out files/directories. Return
* `exclude` {Function|string\[]} Function to filter out files/directories or a
list of glob patterns to be excluded. If a function is provided, return
`true` to exclude the item, `false` to include it. **Default:** `undefined`.
* `withFileTypes` {boolean} `true` if the glob should return paths as Dirents,
`false` otherwise. **Default:** `false`.
Expand Down
151 changes: 118 additions & 33 deletions lib/internal/fs/glob.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

const {
ArrayFrom,
ArrayIsArray,
ArrayPrototypeAt,
ArrayPrototypeFlatMap,
ArrayPrototypeMap,
Expand All @@ -24,12 +25,18 @@ const {
isMacOS,
} = require('internal/util');
const {
validateFunction,
validateObject,
validateString,
validateStringArray,
} = require('internal/validators');
const { DirentFromStats } = require('internal/fs/utils');
const {
codes: {
ERR_INVALID_ARG_TYPE,
},
hideStackFrames,
} = require('internal/errors');
const assert = require('internal/assert');

let minimatch;
function lazyMinimatch() {
Expand Down Expand Up @@ -63,6 +70,45 @@ function getDirentSync(path) {
return new DirentFromStats(basename(path), stat, dirname(path));
}

/**
* @callback validateStringArrayOrFunction
* @param {*} value
* @param {string} name
*/
const validateStringArrayOrFunction = hideStackFrames((value, name) => {
if (ArrayIsArray(value)) {
for (let i = 0; i < value.length; ++i) {
if (typeof value[i] !== 'string') {
throw new ERR_INVALID_ARG_TYPE(`${name}[${i}]`, 'string', value[i]);
}
}
return;
}
if (typeof value !== 'function') {
throw new ERR_INVALID_ARG_TYPE(name, ['string[]', 'function'], value);
}
});

/**
* @param {string} pattern
* @param {options} options
* @returns {Minimatch}
*/
function createMatcher(pattern, options = kEmptyObject) {
const opts = {
__proto__: null,
nocase: isWindows || isMacOS,
windowsPathsNoEscape: true,
nonegate: true,
nocomment: true,
optimizationLevel: 2,
platform: process.platform,
nocaseMagicOnly: true,
...options,
};
return new (lazyMinimatch().Minimatch)(pattern, opts);
}

class Cache {
#cache = new SafeMap();
#statsCache = new SafeMap();
Expand Down Expand Up @@ -188,24 +234,56 @@ class Pattern {
}
}

class ResultSet extends SafeSet {
#root = '.';
#isExcluded = () => false;
constructor(i) { super(i); } // eslint-disable-line no-useless-constructor

setup(root, isExcludedFn) {
this.#root = root;
this.#isExcluded = isExcludedFn;
}

add(value) {
if (this.#isExcluded(resolve(this.#root, value))) {
return false;
}
super.add(value);
return true;
}
}

class Glob {
#root;
#exclude;
#cache = new Cache();
#results = new SafeSet();
#results = new ResultSet();
#queue = [];
#subpatterns = new SafeMap();
#patterns;
#withFileTypes;
#isExcluded = () => false;
constructor(pattern, options = kEmptyObject) {
validateObject(options, 'options');
const { exclude, cwd, withFileTypes } = options;
if (exclude != null) {
validateFunction(exclude, 'options.exclude');
}
this.#root = cwd ?? '.';
this.#exclude = exclude;
this.#withFileTypes = !!withFileTypes;
if (exclude != null) {
validateStringArrayOrFunction(exclude, 'options.exclude');
if (ArrayIsArray(exclude)) {
assert(typeof this.#root === 'string');
// Convert the path part of exclude patterns to absolute paths for
// consistent comparison before instantiating matchers.
const matchers = exclude
.map((pattern) => resolve(this.#root, pattern))
.map((pattern) => createMatcher(pattern));
this.#isExcluded = (value) =>
matchers.some((matcher) => matcher.match(value));
this.#results.setup(this.#root, this.#isExcluded);
} else {
this.#exclude = exclude;
}
}
let patterns;
if (typeof pattern === 'object') {
validateStringArray(pattern, 'patterns');
Expand All @@ -214,17 +292,7 @@ class Glob {
validateString(pattern, 'patterns');
patterns = [pattern];
}
this.matchers = ArrayPrototypeMap(patterns, (pattern) => new (lazyMinimatch().Minimatch)(pattern, {
__proto__: null,
nocase: isWindows || isMacOS,
windowsPathsNoEscape: true,
nonegate: true,
nocomment: true,
optimizationLevel: 2,
platform: process.platform,
nocaseMagicOnly: true,
}));

this.matchers = ArrayPrototypeMap(patterns, (pattern) => createMatcher(pattern));
this.#patterns = ArrayPrototypeFlatMap(this.matchers, (matcher) => ArrayPrototypeMap(matcher.set,
(pattern, i) => new Pattern(
pattern,
Expand Down Expand Up @@ -255,6 +323,9 @@ class Glob {
);
}
#addSubpattern(path, pattern) {
if (this.#isExcluded(path)) {
return;
}
if (!this.#subpatterns.has(path)) {
this.#subpatterns.set(path, [pattern]);
} else {
Expand All @@ -273,6 +344,9 @@ class Glob {
const isLast = pattern.isLast(isDirectory);
const isFirst = pattern.isFirst();

if (this.#isExcluded(fullpath)) {
return;
}
if (isFirst && isWindows && typeof pattern.at(0) === 'string' && StringPrototypeEndsWith(pattern.at(0), ':')) {
// Absolute path, go to root
this.#addSubpattern(`${pattern.at(0)}\\`, pattern.child(new SafeSet().add(1)));
Expand Down Expand Up @@ -461,6 +535,9 @@ class Glob {
const isLast = pattern.isLast(isDirectory);
const isFirst = pattern.isFirst();

if (this.#isExcluded(fullpath)) {
return;
}
if (isFirst && isWindows && typeof pattern.at(0) === 'string' && StringPrototypeEndsWith(pattern.at(0), ':')) {
// Absolute path, go to root
this.#addSubpattern(`${pattern.at(0)}\\`, pattern.child(new SafeSet().add(1)));
Expand Down Expand Up @@ -489,8 +566,9 @@ class Glob {
if (stat && (p || isDirectory)) {
const result = join(path, p);
if (!this.#results.has(result)) {
this.#results.add(result);
yield this.#withFileTypes ? stat : result;
if (this.#results.add(result)) {
yield this.#withFileTypes ? stat : result;
}
}
}
if (pattern.indexes.size === 1 && pattern.indexes.has(last)) {
Expand All @@ -501,8 +579,9 @@ class Glob {
// If pattern ends with **, add to results
// if path is ".", add it only if pattern starts with "." or pattern is exactly "**"
if (!this.#results.has(path)) {
this.#results.add(path);
yield this.#withFileTypes ? stat : path;
if (this.#results.add(path)) {
yield this.#withFileTypes ? stat : path;
}
}
}

Expand Down Expand Up @@ -551,8 +630,9 @@ class Glob {
} else if (!fromSymlink && index === last) {
// If ** is last, add to results
if (!this.#results.has(entryPath)) {
this.#results.add(entryPath);
yield this.#withFileTypes ? entry : entryPath;
if (this.#results.add(entryPath)) {
yield this.#withFileTypes ? entry : entryPath;
}
}
}

Expand All @@ -562,8 +642,9 @@ class Glob {
if (nextMatches && nextIndex === last && !isLast) {
// If next pattern is the last one, add to results
if (!this.#results.has(entryPath)) {
this.#results.add(entryPath);
yield this.#withFileTypes ? entry : entryPath;
if (this.#results.add(entryPath)) {
yield this.#withFileTypes ? entry : entryPath;
}
}
} else if (nextMatches && entry.isDirectory()) {
// Pattern matched, meaning two patterns forward
Expand Down Expand Up @@ -598,15 +679,17 @@ class Glob {
if (!this.#cache.seen(path, pattern, nextIndex)) {
this.#cache.add(path, pattern.child(new SafeSet().add(nextIndex)));
if (!this.#results.has(path)) {
this.#results.add(path);
yield this.#withFileTypes ? this.#cache.statSync(fullpath) : path;
if (this.#results.add(path)) {
yield this.#withFileTypes ? this.#cache.statSync(fullpath) : path;
}
}
}
if (!this.#cache.seen(path, pattern, nextIndex) || !this.#cache.seen(parent, pattern, nextIndex)) {
this.#cache.add(parent, pattern.child(new SafeSet().add(nextIndex)));
if (!this.#results.has(parent)) {
this.#results.add(parent);
yield this.#withFileTypes ? this.#cache.statSync(join(this.#root, parent)) : parent;
if (this.#results.add(parent)) {
yield this.#withFileTypes ? this.#cache.statSync(join(this.#root, parent)) : parent;
}
}
}
}
Expand All @@ -621,8 +704,9 @@ class Glob {
// If current pattern is ".", proceed to test next pattern
if (nextIndex === last) {
if (!this.#results.has(entryPath)) {
this.#results.add(entryPath);
yield this.#withFileTypes ? entry : entryPath;
if (this.#results.add(entryPath)) {
yield this.#withFileTypes ? entry : entryPath;
}
}
} else {
subPatterns.add(nextIndex + 1);
Expand All @@ -634,8 +718,9 @@ class Glob {
// add next pattern to potential patterns, or to results if it's the last pattern
if (index === last) {
if (!this.#results.has(entryPath)) {
this.#results.add(entryPath);
yield this.#withFileTypes ? entry : entryPath;
if (this.#results.add(entryPath)) {
yield this.#withFileTypes ? entry : entryPath;
}
}
} else if (entry.isDirectory()) {
subPatterns.add(nextIndex);
Expand Down
Loading

0 comments on commit dc5d0f9

Please sign in to comment.