Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fs: allow exclude option in globs to accept glob patterns #56489

Merged
merged 2 commits into from
Jan 8, 2025
Merged

fs: allow exclude option in globs to accept glob patterns #56489

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b1f9020
fs: allow `exclude` option in globs to accept glob patterns
daeyeon Jan 5, 2025
4cf780d
fixup! fs: allow `exclude` option in globs to accept glob patterns
daeyeon Jan 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 15 additions & 3 deletions doc/api/fs.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1074,6 +1074,9 @@ behavior is similar to `cp dir1/ dir2/`.
<!-- YAML
added: v22.0.0
changes:
- version: REPLACEME
pr-url: https://github.com/nodejs/node/pull/56489
description: Add support for `exclude` option to accept glob patterns.
- version: v22.2.0
pr-url: https://github.com/nodejs/node/pull/52837
description: Add support for `withFileTypes` as an option.
Expand All @@ -1084,7 +1087,8 @@ changes:
* `pattern` {string|string\[]}
* `options` {Object}
* `cwd` {string} current working directory. **Default:** `process.cwd()`
* `exclude` {Function} Function to filter out files/directories. Return
* `exclude` {Function|string\[]} Function to filter out files/directories or a
list of glob patterns to be excluded. If a function is provided, return
`true` to exclude the item, `false` to include it. **Default:** `undefined`.
* `withFileTypes` {boolean} `true` if the glob should return paths as Dirents,
`false` otherwise. **Default:** `false`.
Expand Down Expand Up @@ -3120,6 +3124,9 @@ descriptor. See [`fs.utimes()`][].
<!-- YAML
added: v22.0.0
changes:
- version: REPLACEME
pr-url: https://github.com/nodejs/node/pull/56489
description: Add support for `exclude` option to accept glob patterns.
- version: v22.2.0
pr-url: https://github.com/nodejs/node/pull/52837
description: Add support for `withFileTypes` as an option.
Expand All @@ -3131,7 +3138,8 @@ changes:

* `options` {Object}
* `cwd` {string} current working directory. **Default:** `process.cwd()`
* `exclude` {Function} Function to filter out files/directories. Return
* `exclude` {Function|string\[]} Function to filter out files/directories or a
list of glob patterns to be excluded. If a function is provided, return
`true` to exclude the item, `false` to include it. **Default:** `undefined`.
* `withFileTypes` {boolean} `true` if the glob should return paths as Dirents,
`false` otherwise. **Default:** `false`.
Expand Down Expand Up @@ -5656,6 +5664,9 @@ Synchronous version of [`fs.futimes()`][]. Returns `undefined`.
<!-- YAML
added: v22.0.0
changes:
- version: REPLACEME
pr-url: https://github.com/nodejs/node/pull/56489
description: Add support for `exclude` option to accept glob patterns.
- version: v22.2.0
pr-url: https://github.com/nodejs/node/pull/52837
description: Add support for `withFileTypes` as an option.
Expand All @@ -5666,7 +5677,8 @@ changes:
* `pattern` {string|string\[]}
* `options` {Object}
* `cwd` {string} current working directory. **Default:** `process.cwd()`
* `exclude` {Function} Function to filter out files/directories. Return
* `exclude` {Function|string\[]} Function to filter out files/directories or a
list of glob patterns to be excluded. If a function is provided, return
`true` to exclude the item, `false` to include it. **Default:** `undefined`.
* `withFileTypes` {boolean} `true` if the glob should return paths as Dirents,
`false` otherwise. **Default:** `false`.
Expand Down
151 changes: 118 additions & 33 deletions lib/internal/fs/glob.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

const {
ArrayFrom,
ArrayIsArray,
ArrayPrototypeAt,
ArrayPrototypeFlatMap,
ArrayPrototypeMap,
Expand All @@ -24,12 +25,18 @@ const {
isMacOS,
} = require('internal/util');
const {
validateFunction,
validateObject,
validateString,
validateStringArray,
} = require('internal/validators');
const { DirentFromStats } = require('internal/fs/utils');
const {
codes: {
ERR_INVALID_ARG_TYPE,
},
hideStackFrames,
} = require('internal/errors');
const assert = require('internal/assert');

let minimatch;
function lazyMinimatch() {
Expand Down Expand Up @@ -63,6 +70,45 @@ function getDirentSync(path) {
return new DirentFromStats(basename(path), stat, dirname(path));
}

/**
* @callback validateStringArrayOrFunction
* @param {*} value
* @param {string} name
*/
const validateStringArrayOrFunction = hideStackFrames((value, name) => {
if (ArrayIsArray(value)) {
for (let i = 0; i < value.length; ++i) {
if (typeof value[i] !== 'string') {
throw new ERR_INVALID_ARG_TYPE(`${name}[${i}]`, 'string', value[i]);
}
}
return;
}
if (typeof value !== 'function') {
throw new ERR_INVALID_ARG_TYPE(name, ['string[]', 'function'], value);
}
});

/**
* @param {string} pattern
* @param {options} options
* @returns {Minimatch}
*/
function createMatcher(pattern, options = kEmptyObject) {
const opts = {
__proto__: null,
nocase: isWindows || isMacOS,
windowsPathsNoEscape: true,
nonegate: true,
nocomment: true,
optimizationLevel: 2,
platform: process.platform,
nocaseMagicOnly: true,
...options,
};
return new (lazyMinimatch().Minimatch)(pattern, opts);
}

class Cache {
#cache = new SafeMap();
#statsCache = new SafeMap();
Expand Down Expand Up @@ -188,24 +234,56 @@ class Pattern {
}
}

class ResultSet extends SafeSet {
#root = '.';
#isExcluded = () => false;
constructor(i) { super(i); } // eslint-disable-line no-useless-constructor

setup(root, isExcludedFn) {
this.#root = root;
this.#isExcluded = isExcludedFn;
}

add(value) {
if (this.#isExcluded(resolve(this.#root, value))) {
return false;
}
super.add(value);
return true;
}
}

class Glob {
#root;
#exclude;
#cache = new Cache();
#results = new SafeSet();
#results = new ResultSet();
#queue = [];
#subpatterns = new SafeMap();
#patterns;
#withFileTypes;
#isExcluded = () => false;
constructor(pattern, options = kEmptyObject) {
validateObject(options, 'options');
const { exclude, cwd, withFileTypes } = options;
if (exclude != null) {
validateFunction(exclude, 'options.exclude');
}
this.#root = cwd ?? '.';
this.#exclude = exclude;
this.#withFileTypes = !!withFileTypes;
if (exclude != null) {
validateStringArrayOrFunction(exclude, 'options.exclude');
if (ArrayIsArray(exclude)) {
assert(typeof this.#root === 'string');
// Convert the path part of exclude patterns to absolute paths for
// consistent comparison before instantiating matchers.
const matchers = exclude
.map((pattern) => resolve(this.#root, pattern))
.map((pattern) => createMatcher(pattern));
this.#isExcluded = (value) =>
matchers.some((matcher) => matcher.match(value));
this.#results.setup(this.#root, this.#isExcluded);
} else {
this.#exclude = exclude;
}
}
let patterns;
if (typeof pattern === 'object') {
validateStringArray(pattern, 'patterns');
Expand All @@ -214,17 +292,7 @@ class Glob {
validateString(pattern, 'patterns');
patterns = [pattern];
}
this.matchers = ArrayPrototypeMap(patterns, (pattern) => new (lazyMinimatch().Minimatch)(pattern, {
__proto__: null,
nocase: isWindows || isMacOS,
windowsPathsNoEscape: true,
nonegate: true,
nocomment: true,
optimizationLevel: 2,
platform: process.platform,
nocaseMagicOnly: true,
}));

this.matchers = ArrayPrototypeMap(patterns, (pattern) => createMatcher(pattern));
this.#patterns = ArrayPrototypeFlatMap(this.matchers, (matcher) => ArrayPrototypeMap(matcher.set,
(pattern, i) => new Pattern(
pattern,
Expand Down Expand Up @@ -255,6 +323,9 @@ class Glob {
);
}
#addSubpattern(path, pattern) {
if (this.#isExcluded(path)) {
return;
}
if (!this.#subpatterns.has(path)) {
this.#subpatterns.set(path, [pattern]);
} else {
Expand All @@ -273,6 +344,9 @@ class Glob {
const isLast = pattern.isLast(isDirectory);
const isFirst = pattern.isFirst();

if (this.#isExcluded(fullpath)) {
return;
}
if (isFirst && isWindows && typeof pattern.at(0) === 'string' && StringPrototypeEndsWith(pattern.at(0), ':')) {
// Absolute path, go to root
this.#addSubpattern(`${pattern.at(0)}\\`, pattern.child(new SafeSet().add(1)));
Expand Down Expand Up @@ -461,6 +535,9 @@ class Glob {
const isLast = pattern.isLast(isDirectory);
const isFirst = pattern.isFirst();

if (this.#isExcluded(fullpath)) {
return;
}
if (isFirst && isWindows && typeof pattern.at(0) === 'string' && StringPrototypeEndsWith(pattern.at(0), ':')) {
// Absolute path, go to root
this.#addSubpattern(`${pattern.at(0)}\\`, pattern.child(new SafeSet().add(1)));
Expand Down Expand Up @@ -489,8 +566,9 @@ class Glob {
if (stat && (p || isDirectory)) {
const result = join(path, p);
if (!this.#results.has(result)) {
this.#results.add(result);
yield this.#withFileTypes ? stat : result;
if (this.#results.add(result)) {
yield this.#withFileTypes ? stat : result;
}
}
}
if (pattern.indexes.size === 1 && pattern.indexes.has(last)) {
Expand All @@ -501,8 +579,9 @@ class Glob {
// If pattern ends with **, add to results
// if path is ".", add it only if pattern starts with "." or pattern is exactly "**"
if (!this.#results.has(path)) {
this.#results.add(path);
yield this.#withFileTypes ? stat : path;
if (this.#results.add(path)) {
yield this.#withFileTypes ? stat : path;
}
}
}

Expand Down Expand Up @@ -551,8 +630,9 @@ class Glob {
} else if (!fromSymlink && index === last) {
// If ** is last, add to results
if (!this.#results.has(entryPath)) {
this.#results.add(entryPath);
yield this.#withFileTypes ? entry : entryPath;
if (this.#results.add(entryPath)) {
yield this.#withFileTypes ? entry : entryPath;
}
}
}

Expand All @@ -562,8 +642,9 @@ class Glob {
if (nextMatches && nextIndex === last && !isLast) {
// If next pattern is the last one, add to results
if (!this.#results.has(entryPath)) {
this.#results.add(entryPath);
yield this.#withFileTypes ? entry : entryPath;
if (this.#results.add(entryPath)) {
yield this.#withFileTypes ? entry : entryPath;
}
}
} else if (nextMatches && entry.isDirectory()) {
// Pattern matched, meaning two patterns forward
Expand Down Expand Up @@ -598,15 +679,17 @@ class Glob {
if (!this.#cache.seen(path, pattern, nextIndex)) {
this.#cache.add(path, pattern.child(new SafeSet().add(nextIndex)));
if (!this.#results.has(path)) {
this.#results.add(path);
yield this.#withFileTypes ? this.#cache.statSync(fullpath) : path;
if (this.#results.add(path)) {
yield this.#withFileTypes ? this.#cache.statSync(fullpath) : path;
}
}
}
if (!this.#cache.seen(path, pattern, nextIndex) || !this.#cache.seen(parent, pattern, nextIndex)) {
this.#cache.add(parent, pattern.child(new SafeSet().add(nextIndex)));
if (!this.#results.has(parent)) {
this.#results.add(parent);
yield this.#withFileTypes ? this.#cache.statSync(join(this.#root, parent)) : parent;
if (this.#results.add(parent)) {
yield this.#withFileTypes ? this.#cache.statSync(join(this.#root, parent)) : parent;
}
}
}
}
Expand All @@ -621,8 +704,9 @@ class Glob {
// If current pattern is ".", proceed to test next pattern
if (nextIndex === last) {
if (!this.#results.has(entryPath)) {
this.#results.add(entryPath);
yield this.#withFileTypes ? entry : entryPath;
if (this.#results.add(entryPath)) {
yield this.#withFileTypes ? entry : entryPath;
}
}
} else {
subPatterns.add(nextIndex + 1);
Expand All @@ -634,8 +718,9 @@ class Glob {
// add next pattern to potential patterns, or to results if it's the last pattern
if (index === last) {
if (!this.#results.has(entryPath)) {
this.#results.add(entryPath);
yield this.#withFileTypes ? entry : entryPath;
if (this.#results.add(entryPath)) {
yield this.#withFileTypes ? entry : entryPath;
}
}
} else if (entry.isDirectory()) {
subPatterns.add(nextIndex);
Expand Down
Loading
Loading