Skip to content

Commit

Permalink
Attempt TreeSitter (json)regex parsing
Browse files Browse the repository at this point in the history
  • Loading branch information
@RedCMD
RedCMD committed Jan 13, 2024
1 parent 276d95c commit b36cc0a
Show file tree
Hide file tree
Showing 14 changed files with 10,058 additions and 698 deletions.
60 changes: 43 additions & 17 deletions out/DiagnosticCollection.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,24 +29,50 @@ function Diagnostics(document, Diagnostics) {
if (!vscode.languages.match(extension_1.DocumentSelector, document)) {
return;
}
// vscode.window.showInformationMessage(JSON.stringify("diagnostics"))
const tree = (0, TreeSitter_1.getTree)(document);
if (tree == null) {
return;
}
const diagnostics = [];
const language = tree.getLanguage();
const query = language.query(`(` +
` (ERROR) @ERROR` +
`)`);
const queryCaptures = query.captures(tree.rootNode);
for (const queryCapture of queryCaptures) {
const node = queryCapture.node;
const text = node.text;
const range = (0, TreeSitter_1.toRange)(node);
const diagnostic = new vscode.Diagnostic(range, `JSON error: \`${text}\``, vscode.DiagnosticSeverity.Warning);
diagnostics.push(diagnostic);
// vscode.window.showInformationMessage(JSON.stringify(text));
const JSON = false;
const Regex = true;
if (JSON) {
// vscode.window.showInformationMessage(JSON.stringify("diagnostics"))
const tree = (0, TreeSitter_1.getTree)(document);
if (tree == null) {
return;
}
// const language = tree.getLanguage()
// const query = language.query(
// `(` +
// ` (ERROR) @ERROR` +
// `)`
// );
// const queryCaptures = query.captures(tree.rootNode);
const queryCaptures = (0, TreeSitter_1.queryNode)(tree.rootNode, `(ERROR) @ERROR`);
for (const queryCapture of queryCaptures) {
const node = queryCapture.node;
const text = node.text;
const range = (0, TreeSitter_1.toRange)(node);
const diagnostic = new vscode.Diagnostic(range, `JSON error: \`${text}\``, vscode.DiagnosticSeverity.Warning);
diagnostics.push(diagnostic);
// vscode.window.showInformationMessage(JSON.stringify(text));
}
}
if (Regex) {
// vscode.window.showInformationMessage(JSON.stringify("diagnostics"))
const trees = (0, TreeSitter_1.getTrees)(document).regexTrees;
if (trees == null) {
return;
}
for (const id in trees) {
const tree = trees[id];
const queryCaptures = (0, TreeSitter_1.queryNode)(tree.rootNode, `(ERROR) @ERROR`);
for (const queryCapture of queryCaptures) {
const node = queryCapture.node;
const text = node.text;
const range = (0, TreeSitter_1.toRange)(node);
const diagnostic = new vscode.Diagnostic(range, `Regex error: \`${text}\``, vscode.DiagnosticSeverity.Warning);
diagnostics.push(diagnostic);
// vscode.window.showInformationMessage(JSON.stringify(text));
}
}
}
// vscode.window.showInformationMessage(JSON.stringify(diagnostics))
Diagnostics.set(document.uri, diagnostics);
Expand Down
23 changes: 18 additions & 5 deletions out/DocumentSymbolProvider.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,23 +79,30 @@ const SymbolKind = {
};
exports.DocumentSymbolProvider = {
async provideDocumentSymbols(document) {
const tree = (0, TreeSitter_1.getTree)(document);
// const tree = getTree(document)
const symbols = [];
if (true) {
this.getAllChildren(tree.rootNode, symbols);
const tree = (0, TreeSitter_1.getTree)(document);
this.getAllChildren(tree.rootNode, symbols, document);
}
else {
else if (false) {
let i = 0;
for (const symbol in SymbolKind) {
const documentSymbol = new vscode.DocumentSymbol(symbol, SymbolKind[symbol].toString(), SymbolKind[symbol], new vscode.Range(i, 0, i, 1), new vscode.Range(i, 0, i, 1));
symbols.push(documentSymbol);
i++;
}
}
else if (true) {
const regexTrees = (0, TreeSitter_1.getTrees)(document).regexTrees;
for (const regexTree in regexTrees) {
this.getAllChildren(regexTrees[regexTree].rootNode, symbols, document);
}
}
// vscode.window.showInformationMessage(JSON.stringify(symbols))
return symbols;
},
async getAllChildren(node, symbols) {
async getAllChildren(node, symbols, document) {
let symbolsChildren = [];
let documentSymbol;
if (false) {
Expand All @@ -119,7 +126,7 @@ exports.DocumentSymbolProvider = {
}
else {
for (let index = 0; index < node.namedChildCount; index++)
this.getAllChildren(node.namedChild(index), symbolsChildren);
this.getAllChildren(node.namedChild(index), symbolsChildren, document);
let name = '';
switch (node.type) {
case 'pattern':
Expand All @@ -138,6 +145,12 @@ exports.DocumentSymbolProvider = {
case 'value':
name = node.text;
break;
case 'regex':
node = (0, TreeSitter_1.getRegexNode)(document, node);
for (const regexChildNode of node.namedChildren) {
this.getAllChildren(regexChildNode, symbolsChildren, document);
}
break;
}
if (name == '')
name = node.type;
Expand Down
132 changes: 96 additions & 36 deletions out/TreeSitter.js
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,31 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.initTreeSitter = exports.toPoint = exports.toRange = exports.queryForPosition = exports.queryNode = exports.getTree = void 0;
exports.initTreeSitter = exports.toPoint = exports.toRange = exports.queryForPosition = exports.queryNode = exports.getRegexNode = exports.getTree = exports.getTrees = void 0;
const vscode = require("vscode");
const Parser = require("web-tree-sitter");
const extension_1 = require("./extension");
const trees = {};
function getTrees(document) {
const uriString = document.uri.toString();
return trees[uriString];
}
exports.getTrees = getTrees;
/**
* @deprecated use {@link getTrees()} instead
*/
function getTree(document) {
const uriString = document.uri.toString();
const tree = trees[uriString]?.jsonTree;
return tree;
}
exports.getTree = getTree;
function getRegexNode(document, node) {
const uriString = document.uri.toString();
const regexTrees = trees[uriString]?.regexTrees;
const regexTree = regexTrees[node.id];
return regexTree.rootNode;
}
exports.getRegexNode = getRegexNode;
function queryNode(node, queryString, startPoint, endPoint) {
const language = node.tree.getLanguage();
const query = language.query(queryString);
Expand Down Expand Up @@ -66,15 +81,6 @@ async function initTreeSitter(context) {
const regexWasm = context.asAbsolutePath('out/tree-sitter-regextm.wasm');
const regexLanguage = await Parser.Language.load(regexWasm);
regexParser.setLanguage(regexLanguage);
// const regexTree = regexParser.parse('code');
// const activeDocument = vscode.window.activeTextEditor?.document;
// if (activeDocument) {
// if (vscode.languages.match(DocumentSelector, activeDocument)) {
// const uriString = activeDocument.uri.toString();
// const tree = parserJSON.parse(activeDocument.getText());
// trees[uriString] = tree;
// }
// }
vscode.window.visibleTextEditors.forEach(editor => {
// vscode.window.showInformationMessage(JSON.stringify("visible"));
parseTextDocument(editor.document, jsonParser, regexParser);
Expand Down Expand Up @@ -102,25 +108,52 @@ function parseTextDocument(document, jsonParser, regexParser) {
// vscode.window.showInformationMessage(JSON.stringify(document.uri));
const uriString = document.uri.toString();
if (uriString in trees) {
// return;
vscode.window.showInformationMessage(JSON.stringify("JSON TextMate: Why are we here?"));
return;
}
const tree = jsonParser.parse(document.getText());
trees[uriString] = { jsonTree: tree };
const languageJSON = tree.getLanguage();
const query = languageJSON.query(`(` +
` (regex) @regex` +
`)`);
const queryCaptures = query.captures(tree.rootNode);
trees[uriString].regexCaptures = queryCaptures;
const regexTrees = [];
const text = document.getText();
const jsonTree = jsonParser.parse(text);
// const languageJSON = jsonParser.getLanguage();
// const query = languageJSON.query(`(regex) @regex`);
// const queryCaptures = query.captures(jsonTree.rootNode);
const queryCaptures = queryNode(jsonTree.rootNode, `(regex) @regex`);
const regexTrees = {};
for (const queryCapture of queryCaptures) {
const text = queryCapture.node.text;
const regexTree = regexParser.parse(text);
regexTrees.push(regexTree);
const node = queryCapture.node;
const range = {
startPosition: node.startPosition,
endPosition: node.endPosition,
startIndex: node.startIndex,
endIndex: node.endIndex
};
const ranges = [];
ranges.push(range);
const options = { includedRanges: ranges };
const regexTree = regexParser.parse(text, null, options);
regexTrees[node.id] = regexTree;
}
trees[uriString].regexTrees = regexTrees;
// trees[uriString] = { tree: tree, regexCaptures: queryCaptures };
// vscode.window.showInformationMessage(JSON.stringify(trees[uriString].regexTrees));
trees[uriString] = {
jsonTree: jsonTree,
regexTrees: regexTrees
};
// let index = 0;
// const regexIds = {};
// const ranges: Parser.Range[] = [];
// for (const queryCapture of queryCaptures) {
// const node = queryCapture.node;
// const range: Parser.Range = {
// startPosition: node.startPosition,
// endPosition: node.endPosition,
// startIndex: node.startIndex,
// endIndex: node.endIndex
// };
// ranges.push(range);
// const id = node.id;
// regexIds[id] = index++;
// }
// const Options: Parser.Options = { includedRanges: ranges };
// const regexTree = regexParser.parse(text, jsonTree, Options);
// if(node.hasChanges()) {}
}
function reparseTextDocument(edits, JSONParser, regexParser) {
const document = edits.document;
Expand All @@ -131,20 +164,26 @@ function reparseTextDocument(edits, JSONParser, regexParser) {
if (!(uriString in trees)) {
return;
}
const oldTree = trees[uriString].jsonTree;
const jsonTreeOld = trees[uriString].jsonTree;
const text = document.getText();
// const trees = getTrees(document);
// if (!trees) {
// return;
// }
// const oldTree = trees.jsonTree;
for (const edit of edits.contentChanges) {
const startIndex = edit.rangeOffset;
const oldEndIndex = edit.rangeOffset + edit.rangeLength;
const newEndIndex = edit.rangeOffset + edit.text.length;
const startPos = edits.document.positionAt(startIndex);
const oldEndPos = edits.document.positionAt(oldEndIndex);
const newEndPos = edits.document.positionAt(newEndIndex);
const startPosition = { row: startPos.line, column: startPos.character };
const oldEndPosition = { row: oldEndPos.line, column: oldEndPos.character };
const newEndPosition = { row: newEndPos.line, column: newEndPos.character };
// const startPosition = asPoint(startPos);
// const oldEndPosition = asPoint(oldEndPos);
// const newEndPosition = asPoint(newEndPos);
// const startPosition: Parser.Point = { row: startPos.line, column: startPos.character };
// const oldEndPosition: Parser.Point = { row: oldEndPos.line, column: oldEndPos.character };
// const newEndPosition: Parser.Point = { row: newEndPos.line, column: newEndPos.character };
const startPosition = toPoint(startPos);
const oldEndPosition = toPoint(oldEndPos);
const newEndPosition = toPoint(newEndPos);
const delta = {
startIndex,
oldEndIndex,
Expand All @@ -153,10 +192,27 @@ function reparseTextDocument(edits, JSONParser, regexParser) {
oldEndPosition,
newEndPosition,
};
oldTree.edit(delta);
jsonTreeOld.edit(delta);
}
const jsonTree = JSONParser.parse(text, jsonTreeOld);
// trees[uriString].jsonTree = jsonTree;
// Todo: only reparse modified regex nodes. tree.getChangedRanges();
const queryCaptures = queryNode(jsonTree.rootNode, `(regex) @regex`);
const regexTrees = {};
for (const queryCapture of queryCaptures) {
const node = queryCapture.node;
const range = {
startPosition: node.startPosition,
endPosition: node.endPosition,
startIndex: node.startIndex,
endIndex: node.endIndex
};
const ranges = [];
ranges.push(range);
const options = { includedRanges: ranges };
const regexTree = regexParser.parse(text, jsonTreeOld, options);
regexTrees[node.id] = regexTree;
}
const tree = JSONParser.parse(document.getText(), oldTree);
trees[uriString].jsonTree = tree;
// vscode.window.showInformationMessage(JSON.stringify(tree));
// const changedRanges = tree.getChangedRanges(oldTree);
// vscode.window.showInformationMessage(JSON.stringify(tree.rootNode.firstNamedChild));
Expand Down Expand Up @@ -186,5 +242,9 @@ function reparseTextDocument(edits, JSONParser, regexParser) {
// // }
// // }
// }
trees[uriString] = {
jsonTree: jsonTree,
regexTrees: regexTrees
};
// vscode.window.showInformationMessage(JSON.stringify(trees[uriString].regexTrees));
}
Binary file modified out/tree-sitter-regextm.wasm
View file
Binary file not shown.
Loading

0 comments on commit b36cc0a

Please sign in to comment.