diff --git a/.github/workflows/pr_checks.yaml b/.github/workflows/pr_checks.yaml index b2578aa5e3..e5f44c03e3 100644 --- a/.github/workflows/pr_checks.yaml +++ b/.github/workflows/pr_checks.yaml @@ -39,7 +39,7 @@ jobs: if: steps.root-cache.outputs.cache-hit != 'true' run: npm ci - core-checks: + install-core: needs: install-root runs-on: ubuntu-latest steps: @@ -66,6 +66,27 @@ jobs: cd core npm ci + core-checks: + needs: install-core + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: ".nvmrc" + + - uses: actions/cache@v4 + id: root-cache + with: + path: node_modules + key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }} + + - uses: actions/cache@v4 + id: core-cache + with: + path: core/node_modules + key: ${{ runner.os }}-core-node-modules-${{ hashFiles('core/package-lock.json') }} + - name: Type check and lint run: | cd core @@ -74,8 +95,8 @@ jobs: env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - gui-checks: - needs: [install-root, core-checks] + install-gui: + needs: [install-root, install-core] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -105,13 +126,38 @@ jobs: cd gui npm ci + gui-checks: + needs: install-gui + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: ".nvmrc" + + - uses: actions/cache@v4 + with: + path: node_modules + key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }} + + - uses: actions/cache@v4 + with: + path: core/node_modules + key: ${{ runner.os }}-core-node-modules-${{ hashFiles('core/package-lock.json') }} + + - uses: actions/cache@v4 + id: gui-cache + with: + path: gui/node_modules + key: ${{ runner.os }}-gui-node-modules-${{ hashFiles('gui/package-lock.json') }} + - name: Type check run: | cd gui npx tsc --noEmit binary-checks: - needs: [install-root, core-checks] + needs: [install-root, install-core] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -146,8 +192,8 @@ jobs: cd binary npx tsc --noEmit - vscode-checks: - needs: [install-root, core-checks] + install-vscode: + needs: [install-root, install-core] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -179,6 +225,31 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.CI_GITHUB_TOKEN }} + vscode-checks: + needs: install-vscode + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: ".nvmrc" + + - uses: actions/cache@v4 + with: + path: node_modules + key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }} + + - uses: actions/cache@v4 + with: + path: core/node_modules + key: ${{ runner.os }}-core-node-modules-${{ hashFiles('core/package-lock.json') }} + + - uses: actions/cache@v4 + id: vscode-cache + with: + path: extensions/vscode/node_modules + key: ${{ runner.os }}-vscode-node-modules-${{ hashFiles('extensions/vscode/package-lock.json') }} + - name: Type check and lint run: | cd extensions/vscode @@ -186,7 +257,7 @@ jobs: npm run lint core-tests: - needs: [core-checks] + needs: install-core runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -204,9 +275,38 @@ jobs: cd core npm test - vscode-tests: - needs: [vscode-checks, core-checks] + vscode-get-test-file-matrix: + runs-on: ubuntu-latest + needs: [install-root, install-vscode] + outputs: + test_file_matrix: ${{ steps.vscode-get-test-file-matrix.outputs.test_file_matrix }} + steps: + - uses: actions/checkout@v4 + + - name: Cache node modules + uses: actions/cache@v3 + with: + path: extensions/vscode/node_modules + key: ${{ runner.os }}-vscode-node-modules-${{ hashFiles('extensions/vscode/package-lock.json') }} + + - name: Get test files + id: vscode-get-test-file-matrix + run: | + cd extensions/vscode + npm ci + npm run e2e:compile + FILES=$(ls -1 e2e/_output/tests/*.test.js | jq -R . | jq -s .) + echo "test_file_matrix<> $GITHUB_OUTPUT + echo "$FILES" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + - name: Debug Outputs + run: | + echo "Test files: ${{ steps.vscode-get-test-file-matrix.outputs.test_file_matrix }}" + + vscode-package-extension: runs-on: ubuntu-latest + needs: [install-vscode, install-core] steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -225,18 +325,94 @@ jobs: path: core/node_modules key: ${{ runner.os }}-core-node-modules-${{ hashFiles('core/package-lock.json') }} + - name: Package extension + run: | + cd extensions/vscode + npm run package + + - name: Upload build artifact + uses: actions/upload-artifact@v4 + with: + name: vscode-extension-build + path: extensions/vscode/build + + vscode-download-e2e-dependencies: + runs-on: ubuntu-latest + needs: [install-vscode, install-core] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: ".nvmrc" + + - uses: actions/cache@v4 + id: vscode-node-modules-cache + with: + path: extensions/vscode/node_modules + key: ${{ runner.os }}-vscode-node-modules-${{ hashFiles('extensions/vscode/package-lock.json') }} + - uses: actions/cache@v4 id: storage-cache with: path: extensions/vscode/e2e/storage key: ${{ runner.os }}-vscode-storage-${{ hashFiles('extensions/vscode/package-lock.json') }} - - name: Download Dependencies if Cache Miss + - name: Download Dependencies if: steps.storage-cache.outputs.cache-hit != 'true' run: | cd extensions/vscode npm run e2e:ci:download + - name: Upload e2e dependencies + uses: actions/upload-artifact@v4 + with: + name: vscode-e2e-dependencies + path: extensions/vscode/e2e/storage + + vscode-e2e-tests: + name: ${{ matrix.test_file }}" + needs: + [ + vscode-download-e2e-dependencies, + vscode-get-test-file-matrix, + vscode-package-extension, + install-vscode, + install-core, + ] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + test_file: ${{ fromJson(needs.vscode-get-test-file-matrix.outputs.test_file_matrix) }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: ".nvmrc" + + - uses: actions/cache@v4 + id: vscode-node-modules-cache + with: + path: extensions/vscode/node_modules + key: ${{ runner.os }}-vscode-node-modules-${{ hashFiles('extensions/vscode/package-lock.json') }} + + - name: Download build artifact + uses: actions/download-artifact@v4 + with: + name: vscode-extension-build + path: extensions/vscode/build + + - name: Download e2e dependencies + uses: actions/download-artifact@v4 + with: + name: vscode-e2e-dependencies + path: extensions/vscode/e2e/storage + + - name: Fix VSCode binary permissions + run: | + chmod +x extensions/vscode/e2e/storage/VSCode-linux-x64/code + chmod +x extensions/vscode/e2e/storage/chromedriver-linux64/chromedriver + - name: Set up SSH env: SSH_KEY: ${{ secrets.GH_ACTIONS_SSH_TEST_KEY_PEM }} @@ -248,24 +424,28 @@ jobs: ssh-keyscan -H "$SSH_HOST" >> ~/.ssh/known_hosts echo -e "Host ssh-test-container\n\tHostName $SSH_HOST\n\tUser ec2-user\n\tIdentityFile ~/.ssh/id_rsa" >> ~/.ssh/config - - name: Install Xvfb for Linux and run e2e tests + - name: Set up Xvfb + run: | + Xvfb :99 & + export DISPLAY=:99 + + - name: Run e2e tests run: | - sudo apt-get install -y xvfb # Install Xvfb - Xvfb :99 & # Start Xvfb - export DISPLAY=:99 # Export the display number to the environment cd extensions/vscode - npm run package - npm run e2e:ci:run + TEST_FILE="${{ matrix.test_file }}" npm run e2e:ci:run + + env: + DISPLAY: :99 - name: Upload e2e test screenshots if: failure() uses: actions/upload-artifact@v4 with: - name: e2e-screenshots + name: e2e-failure-screenshots path: extensions/vscode/e2e/storage/screenshots gui-tests: - needs: [gui-checks, core-checks] + needs: [install-gui, install-core] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -296,7 +476,7 @@ jobs: npm test jetbrains-tests: - needs: [install-root, core-checks] + needs: [install-root, install-core] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -331,12 +511,7 @@ jobs: path: gui/node_modules key: ${{ runner.os }}-gui-node-modules-${{ hashFiles('gui/package-lock.json') }} - - name: Build GUI - run: | - cd gui - npm ci - npm run build - + # We can shave off another minute off our CI script by finding a way to share this with vscode-tests - name: Run prepackage script run: | cd extensions/vscode @@ -371,7 +546,7 @@ jobs: with: url: http://127.0.0.1:8082 max-attempts: 15 - retry-delay: 30s + retry-delay: 5s - name: Run tests run: | diff --git a/core/autocomplete/README.md b/core/autocomplete/README.md index 627ca12060..1136b54abc 100644 --- a/core/autocomplete/README.md +++ b/core/autocomplete/README.md @@ -28,11 +28,10 @@ Example: ```json title="config.json" { "tabAutocompleteModel": { - "title": "Qwen2.5-Coder 1.5b", - "model": "Qwen/Qwen2.5-Coder-1.5B-Instruct-GGUF", - "provider": "lmstudio", - }, - ... + "title": "Qwen2.5-Coder 1.5b", + "model": "Qwen/Qwen2.5-Coder-1.5B-Instruct-GGUF", + "provider": "lmstudio" + } } ``` diff --git a/core/autocomplete/constants/AutocompleteLanguageInfo.ts b/core/autocomplete/constants/AutocompleteLanguageInfo.ts index 2f21c5a3e9..d51fe1f1fa 100644 --- a/core/autocomplete/constants/AutocompleteLanguageInfo.ts +++ b/core/autocomplete/constants/AutocompleteLanguageInfo.ts @@ -1,3 +1,4 @@ +import { getUriFileExtension } from "../../util/uri"; import { BracketMatchingService } from "../filtering/BracketMatchingService"; import { CharacterFilter, @@ -27,7 +28,7 @@ export const Python = { name: "Python", // """"#" is for .ipynb files, where we add '"""' surrounding markdown blocks. // This stops the model from trying to complete the start of a new markdown block - topLevelKeywords: ["def", "class", "\"\"\"#"], + topLevelKeywords: ["def", "class", '"""#'], singleLineComment: "#", endOfLine: [], }; @@ -368,8 +369,7 @@ export const LANGUAGES: { [extension: string]: AutocompleteLanguageInfo } = { md: Markdown, }; -export function languageForFilepath( - filepath: string, -): AutocompleteLanguageInfo { - return LANGUAGES[filepath.split(".").slice(-1)[0]] || Typescript; +export function languageForFilepath(fileUri: string): AutocompleteLanguageInfo { + const extension = getUriFileExtension(fileUri); + return LANGUAGES[extension] || Typescript; } diff --git a/core/autocomplete/context/root-path-context/RootPathContextService.ts b/core/autocomplete/context/root-path-context/RootPathContextService.ts index 93648653fd..79b598f1ec 100644 --- a/core/autocomplete/context/root-path-context/RootPathContextService.ts +++ b/core/autocomplete/context/root-path-context/RootPathContextService.ts @@ -18,20 +18,20 @@ import { AutocompleteSnippetDeprecated } from "../../types"; import { AstPath } from "../../util/ast"; import { ImportDefinitionsService } from "../ImportDefinitionsService"; -function getSyntaxTreeString( - node: Parser.SyntaxNode, - indent: string = "", -): string { - let result = ""; - const nodeInfo = `${node.type} [${node.startPosition.row}:${node.startPosition.column} - ${node.endPosition.row}:${node.endPosition.column}]`; - result += `${indent}${nodeInfo}\n`; - - for (const child of node.children) { - result += getSyntaxTreeString(child, indent + " "); - } - - return result; -} +// function getSyntaxTreeString( +// node: Parser.SyntaxNode, +// indent: string = "", +// ): string { +// let result = ""; +// const nodeInfo = `${node.type} [${node.startPosition.row}:${node.startPosition.column} - ${node.endPosition.row}:${node.endPosition.column}]`; +// result += `${indent}${nodeInfo}\n`; + +// for (const child of node.children) { +// result += getSyntaxTreeString(child, indent + " "); +// } + +// return result; +// } export class RootPathContextService { private cache = new LRUCache({ diff --git a/core/autocomplete/filtering/test/util.ts b/core/autocomplete/filtering/test/util.ts index d1b94575ed..41a35db079 100644 --- a/core/autocomplete/filtering/test/util.ts +++ b/core/autocomplete/filtering/test/util.ts @@ -1,8 +1,6 @@ -import fs from "node:fs"; -import path from "node:path"; - import MockLLM from "../../../llm/llms/Mock"; import { testConfigHandler, testIde } from "../../../test/fixtures"; +import { joinPathsToUri } from "../../../util/uri"; import { CompletionProvider } from "../../CompletionProvider"; import { AutocompleteInput } from "../../util/types"; @@ -39,8 +37,8 @@ export async function testAutocompleteFiltering( // Create a real file const [workspaceDir] = await ide.getWorkspaceDirs(); - const filepath = path.join(workspaceDir, test.filename); - fs.writeFileSync(filepath, test.input.replace(FIM_DELIMITER, "")); + const fileUri = joinPathsToUri(workspaceDir, test.filename); + await ide.writeFile(fileUri, test.input.replace(FIM_DELIMITER, "")); // Prepare completion input and provider const completionProvider = new CompletionProvider( @@ -56,7 +54,7 @@ export async function testAutocompleteFiltering( const autocompleteInput: AutocompleteInput = { isUntitledFile: false, completionId: "test-completion-id", - filepath, + filepath: fileUri, pos: { line, character, diff --git a/core/autocomplete/prefiltering/index.ts b/core/autocomplete/prefiltering/index.ts index a9c198a51b..ed1266ecc3 100644 --- a/core/autocomplete/prefiltering/index.ts +++ b/core/autocomplete/prefiltering/index.ts @@ -3,9 +3,9 @@ import path from "node:path"; import ignore from "ignore"; import { IDE } from "../.."; -import { getBasename } from "../../util"; import { getConfigJsonPath } from "../../util/paths"; import { HelperVars } from "../util/HelperVars"; +import { findUriInDirs } from "../../util/uri"; async function isDisabledForFile( currentFilepath: string, @@ -15,26 +15,14 @@ async function isDisabledForFile( if (disableInFiles) { // Relative path needed for `ignore` const workspaceDirs = await ide.getWorkspaceDirs(); - let filepath = currentFilepath; - for (const workspaceDir of workspaceDirs) { - const relativePath = path.relative(workspaceDir, filepath); - const relativePathBase = relativePath.split(path.sep).at(0); - const isInWorkspace = - !path.isAbsolute(relativePath) && relativePathBase !== ".."; - if (isInWorkspace) { - filepath = path.relative(workspaceDir, filepath); - break; - } - } - - // Worst case we can check filetype glob patterns - if (filepath === currentFilepath) { - filepath = getBasename(filepath); - } + const { relativePathOrBasename } = findUriInDirs( + currentFilepath, + workspaceDirs, + ); // @ts-ignore const pattern = ignore.default().add(disableInFiles); - if (pattern.ignores(filepath)) { + if (pattern.ignores(relativePathOrBasename)) { return true; } } diff --git a/core/autocomplete/snippets/getAllSnippets.ts b/core/autocomplete/snippets/getAllSnippets.ts index 30d89e7b94..5ff4a7d601 100644 --- a/core/autocomplete/snippets/getAllSnippets.ts +++ b/core/autocomplete/snippets/getAllSnippets.ts @@ -1,4 +1,5 @@ import { IDE } from "../../index"; +import { findUriInDirs } from "../../util/uri"; import { ContextRetrievalService } from "../context/ContextRetrievalService"; import { GetLspDefinitionsFunction } from "../types"; import { HelperVars } from "../util/HelperVars"; @@ -45,7 +46,9 @@ async function getIdeSnippets( const workspaceDirs = await ide.getWorkspaceDirs(); return ideSnippets.filter((snippet) => - workspaceDirs.some((dir) => snippet.filepath.startsWith(dir)), + workspaceDirs.some( + (dir) => !!findUriInDirs(snippet.filepath, [dir]).foundInDir, + ), ); } diff --git a/core/autocomplete/templating/AutocompleteTemplate.ts b/core/autocomplete/templating/AutocompleteTemplate.ts index 9ade88e46a..e3f087102c 100644 --- a/core/autocomplete/templating/AutocompleteTemplate.ts +++ b/core/autocomplete/templating/AutocompleteTemplate.ts @@ -1,7 +1,10 @@ // Fill in the middle prompts import { CompletionOptions } from "../../index.js"; -import { getLastNPathParts, shortestRelativePaths } from "../../util/index.js"; +import { + getLastNUriRelativePathParts, + getShortestUniqueRelativeUriPaths, +} from "../../util/uri.js"; import { AutocompleteCodeSnippet, AutocompleteSnippet, @@ -15,6 +18,7 @@ export interface AutocompleteTemplate { filepath: string, reponame: string, snippets: AutocompleteSnippet[], + workspaceUris: string[], ) => [string, string]; template: | string @@ -25,6 +29,7 @@ export interface AutocompleteTemplate { reponame: string, language: string, snippets: AutocompleteSnippet[], + workspaceUris: string[], ) => string); completionOptions?: Partial; } @@ -73,25 +78,32 @@ const codestralFimTemplate: AutocompleteTemplate = { const codestralMultifileFimTemplate: AutocompleteTemplate = { compilePrefixSuffix: ( - prefix: string, - suffix: string, - filepath: string, - reponame: string, - snippets: AutocompleteSnippet[], + prefix, + suffix, + filepath, + reponame, + snippets, + workspaceUris, ): [string, string] => { if (snippets.length === 0) { if (suffix.trim().length === 0 && prefix.trim().length === 0) { - return [`+++++ ${getLastNPathParts(filepath, 2)}\n${prefix}`, suffix]; + return [ + `+++++ ${getLastNUriRelativePathParts(workspaceUris, filepath, 2)}\n${prefix}`, + suffix, + ]; } return [prefix, suffix]; } - const relativePaths = shortestRelativePaths([ - ...snippets.map((snippet) => - "filepath" in snippet ? snippet.filepath : "Untitled.txt", - ), - filepath, - ]); + const relativePaths = getShortestUniqueRelativeUriPaths( + [ + ...snippets.map((snippet) => + "filepath" in snippet ? snippet.filepath : "file:///Untitled.txt", + ), + filepath, + ], + workspaceUris, + ); const otherFiles = snippets .map((snippet, i) => { @@ -136,12 +148,13 @@ const codegemmaFimTemplate: AutocompleteTemplate = { // https://arxiv.org/pdf/2402.19173.pdf section 5.1 const starcoder2FimTemplate: AutocompleteTemplate = { template: ( - prefix: string, - suffix: string, - filename: string, - reponame: string, - language: string, - snippets: AutocompleteSnippet[], + prefix, + suffix, + filename, + reponame, + language, + snippets, + workspaceUris, ): string => { const otherFiles = snippets.length === 0 @@ -189,21 +202,22 @@ const deepseekFimTemplate: AutocompleteTemplate = { // https://github.com/THUDM/CodeGeeX4/blob/main/guides/Infilling_guideline.md const codegeexFimTemplate: AutocompleteTemplate = { template: ( - prefix: string, - suffix: string, - filepath: string, - reponame: string, - language: string, - allSnippets: AutocompleteSnippet[], + prefix, + suffix, + filepath, + reponame, + language, + allSnippets, + workspaceUris, ): string => { const snippets = allSnippets.filter( (snippet) => snippet.type === AutocompleteSnippetType.Code, ) as AutocompleteCodeSnippet[]; - const relativePaths = shortestRelativePaths([ - ...snippets.map((snippet) => snippet.filepath), - filepath, - ]); + const relativePaths = getShortestUniqueRelativeUriPaths( + [...snippets.map((snippet) => snippet.filepath), filepath], + workspaceUris, + ); const baseTemplate = `###PATH:${ relativePaths[relativePaths.length - 1] }\n###LANGUAGE:${language}\n###MODE:BLOCK\n<|code_suffix|>${suffix}<|code_prefix|>${prefix}<|code_middle|>`; diff --git a/core/autocomplete/templating/formatting.ts b/core/autocomplete/templating/formatting.ts index bdbe56c50f..2a91ad4384 100644 --- a/core/autocomplete/templating/formatting.ts +++ b/core/autocomplete/templating/formatting.ts @@ -1,4 +1,4 @@ -import { getLastNPathParts } from "../../util"; +import { getLastNUriRelativePathParts } from "../../util/uri"; import { AutocompleteClipboardSnippet, AutocompleteCodeSnippet, @@ -14,32 +14,34 @@ const getCommentMark = (helper: HelperVars) => { const addCommentMarks = (text: string, helper: HelperVars) => { const commentMark = getCommentMark(helper); - const lines = [ - ...text - .trim() - .split("\n") - .map((line) => `${commentMark} ${line}`), - ]; - - return lines.join("\n"); + return text + .trim() + .split("\n") + .map((line) => `${commentMark} ${line}`) + .join("\n"); }; const formatClipboardSnippet = ( snippet: AutocompleteClipboardSnippet, + workspaceDirs: string[], ): AutocompleteCodeSnippet => { - return formatCodeSnippet({ - filepath: "Untitled.txt", - content: snippet.content, - type: AutocompleteSnippetType.Code, - }); + return formatCodeSnippet( + { + filepath: "file:///Untitled.txt", + content: snippet.content, + type: AutocompleteSnippetType.Code, + }, + workspaceDirs, + ); }; const formatCodeSnippet = ( snippet: AutocompleteCodeSnippet, + workspaceDirs: string[], ): AutocompleteCodeSnippet => { return { ...snippet, - content: `Path: ${getLastNPathParts(snippet.filepath, 2)}\n${snippet.content}`, + content: `Path: ${getLastNUriRelativePathParts(workspaceDirs, snippet.filepath, 2)}\n${snippet.content}`, }; }; @@ -49,10 +51,6 @@ const formatDiffSnippet = ( return snippet; }; -const getCurrentFilepath = (helper: HelperVars) => { - return getLastNPathParts(helper.filepath, 2); -}; - const commentifySnippet = ( helper: HelperVars, snippet: AutocompleteSnippet, @@ -66,9 +64,10 @@ const commentifySnippet = ( export const formatSnippets = ( helper: HelperVars, snippets: AutocompleteSnippet[], + workspaceDirs: string[], ): string => { const currentFilepathComment = addCommentMarks( - getCurrentFilepath(helper), + getLastNUriRelativePathParts(workspaceDirs, helper.filepath, 2), helper, ); @@ -77,11 +76,11 @@ export const formatSnippets = ( .map((snippet) => { switch (snippet.type) { case AutocompleteSnippetType.Code: - return formatCodeSnippet(snippet); + return formatCodeSnippet(snippet, workspaceDirs); case AutocompleteSnippetType.Diff: return formatDiffSnippet(snippet); case AutocompleteSnippetType.Clipboard: - return formatClipboardSnippet(snippet); + return formatClipboardSnippet(snippet, workspaceDirs); } }) .map((item) => { diff --git a/core/autocomplete/templating/index.ts b/core/autocomplete/templating/index.ts index 5715e300ba..9722ee3280 100644 --- a/core/autocomplete/templating/index.ts +++ b/core/autocomplete/templating/index.ts @@ -1,7 +1,6 @@ import Handlebars from "handlebars"; import { CompletionOptions } from "../.."; -import { getBasename } from "../../util"; import { AutocompleteLanguageInfo } from "../constants/AutocompleteLanguageInfo"; import { HelperVars } from "../util/HelperVars"; @@ -11,6 +10,7 @@ import { getTemplateForModel, } from "./AutocompleteTemplate"; import { getSnippets } from "./filtering"; +import { getUriPathBasename } from "../../util/uri"; import { formatSnippets } from "./formatting"; import { getStopTokens } from "./getStopTokens"; @@ -33,7 +33,7 @@ function renderStringTemplate( filepath: string, reponame: string, ) { - const filename = getBasename(filepath); + const filename = getUriPathBasename(filepath); const compiledTemplate = Handlebars.compile(template); return compiledTemplate({ @@ -66,7 +66,7 @@ export function renderPrompt({ suffix = "\n"; } - const reponame = getBasename(workspaceDirs[0] ?? "myproject"); + const reponame = getUriPathBasename(workspaceDirs[0] ?? "myproject"); const { template, compilePrefixSuffix, completionOptions } = getTemplate(helper); @@ -82,9 +82,10 @@ export function renderPrompt({ helper.filepath, reponame, snippets, + helper.workspaceUris, ); } else { - const formattedSnippets = formatSnippets(helper, snippets); + const formattedSnippets = formatSnippets(helper, snippets, workspaceDirs); prefix = [formattedSnippets, prefix].join("\n"); } @@ -106,6 +107,7 @@ export function renderPrompt({ reponame, helper.lang.name, snippets, + helper.workspaceUris, ); const stopTokens = getStopTokens( diff --git a/core/autocomplete/util/AutocompleteLoggingService.ts b/core/autocomplete/util/AutocompleteLoggingService.ts index 35e0ead644..7fd49cfaf4 100644 --- a/core/autocomplete/util/AutocompleteLoggingService.ts +++ b/core/autocomplete/util/AutocompleteLoggingService.ts @@ -1,6 +1,7 @@ import { logDevData } from "../../util/devdata"; import { COUNT_COMPLETION_REJECTED_AFTER } from "../../util/parameters"; import { Telemetry } from "../../util/posthog"; +import { getUriFileExtension } from "../../util/uri"; import { AutocompleteOutcome } from "./types"; @@ -105,7 +106,7 @@ export class AutocompleteLoggingService { completionId: restOfOutcome.completionId, completionOptions: restOfOutcome.completionOptions, debounceDelay: restOfOutcome.debounceDelay, - fileExtension: restOfOutcome.filepath.split(".")?.slice(-1)[0], + fileExtension: getUriFileExtension(restOfOutcome.filepath), maxPromptTokens: restOfOutcome.maxPromptTokens, modelName: restOfOutcome.modelName, modelProvider: restOfOutcome.modelProvider, diff --git a/core/autocomplete/util/HelperVars.ts b/core/autocomplete/util/HelperVars.ts index 0470384936..f92ef5a1df 100644 --- a/core/autocomplete/util/HelperVars.ts +++ b/core/autocomplete/util/HelperVars.ts @@ -20,6 +20,7 @@ import { AutocompleteInput } from "./types"; export class HelperVars { lang: AutocompleteLanguageInfo; treePath: AstPath | undefined; + workspaceUris: string[] = []; private _fileContents: string | undefined; private _fileLines: string[] | undefined; @@ -43,6 +44,8 @@ export class HelperVars { return; } + this.workspaceUris = await this.ide.getWorkspaceDirs(); + this._fileContents = this.input.manuallyPassFileContents ?? (await this.ide.readFile(this.filepath)); diff --git a/core/commands/slash/onboard.ts b/core/commands/slash/onboard.ts index cfcde71a62..d6fedcafb1 100644 --- a/core/commands/slash/onboard.ts +++ b/core/commands/slash/onboard.ts @@ -1,15 +1,18 @@ -import * as fs from "fs/promises"; -import * as path from "path"; - import ignore from "ignore"; -import { IDE, SlashCommand } from "../.."; +import type { FileType, IDE, SlashCommand } from "../.."; import { defaultIgnoreDir, defaultIgnoreFile, + getGlobalContinueIgArray, gitIgArrayFromFile, } from "../../indexing/ignore"; import { renderChatMessage } from "../../util/messageContent"; +import { + findUriInDirs, + getUriPathBasename, + joinPathsToUri, +} from "../../util/uri"; const LANGUAGE_DEP_MGMT_FILENAMES = [ "package.json", // JavaScript (Node.js) @@ -55,30 +58,43 @@ const OnboardSlashCommand: SlashCommand = { }; async function getEntriesFilteredByIgnore(dir: string, ide: IDE) { - const entries = await fs.readdir(dir, { withFileTypes: true }); - - let ig = ignore().add(defaultIgnoreDir).add(defaultIgnoreFile); - - const gitIgnorePath = path.join(dir, ".gitignore"); + const ig = ignore() + .add(defaultIgnoreDir) + .add(defaultIgnoreFile) + .add(getGlobalContinueIgArray()); + const entries = await ide.listDir(dir); - const hasIgnoreFile = await fs - .access(gitIgnorePath) - .then(() => true) - .catch(() => false); + const ignoreUri = joinPathsToUri(dir, ".gitignore"); + const fileExists = await ide.fileExists(ignoreUri); - if (hasIgnoreFile) { - const gitIgnore = await ide.readFile(gitIgnorePath); + if (fileExists) { + const gitIgnore = await ide.readFile(ignoreUri); const igPatterns = gitIgArrayFromFile(gitIgnore); - ig = ig.add(igPatterns); + ig.add(igPatterns); } - const filteredEntries = entries.filter((entry) => { - const name = entry.isDirectory() ? `${entry.name}/` : entry.name; - return !ig.ignores(name); - }); - - return filteredEntries; + const workspaceDirs = await ide.getWorkspaceDirs(); + + const withRelativePaths = entries + .filter( + (entry) => + entry[1] === (1 as FileType.File) || + entry[1] === (2 as FileType.Directory), + ) + .map((entry) => { + const { relativePathOrBasename } = findUriInDirs(entry[0], workspaceDirs); + return { + uri: entry[0], + type: entry[1], + basename: getUriPathBasename(entry[0]), + relativePath: + relativePathOrBasename + + (entry[1] === (2 as FileType.Directory) ? "/" : ""), + }; + }); + + return withRelativePaths.filter((entry) => !ig.ignores(entry.relativePath)); } async function gatherProjectContext( @@ -86,7 +102,6 @@ async function gatherProjectContext( ide: IDE, ): Promise { let context = ""; - async function exploreDirectory(dir: string, currentDepth: number = 0) { if (currentDepth > MAX_EXPLORE_DEPTH) { return; @@ -95,19 +110,16 @@ async function gatherProjectContext( const entries = await getEntriesFilteredByIgnore(dir, ide); for (const entry of entries) { - const fullPath = path.join(dir, entry.name); - const relativePath = path.relative(workspaceDir, fullPath); - - if (entry.isDirectory()) { - context += `\nFolder: ${relativePath}\n`; - await exploreDirectory(fullPath, currentDepth + 1); + if (entry.type === (2 as FileType.Directory)) { + context += `\nFolder: ${entry.relativePath}\n`; + await exploreDirectory(entry.uri, currentDepth + 1); } else { - if (entry.name.toLowerCase() === "readme.md") { - const content = await fs.readFile(fullPath, "utf-8"); - context += `README for ${relativePath}:\n${content}\n\n`; - } else if (LANGUAGE_DEP_MGMT_FILENAMES.includes(entry.name)) { - const content = await fs.readFile(fullPath, "utf-8"); - context += `${entry.name} for ${relativePath}:\n${content}\n\n`; + if (entry.basename.toLowerCase() === "readme.md") { + const content = await ide.readFile(entry.uri); + context += `README for ${entry.relativePath}:\n${content}\n\n`; + } else if (LANGUAGE_DEP_MGMT_FILENAMES.includes(entry.basename)) { + const content = await ide.readFile(entry.uri); + context += `${entry.basename} for ${entry.relativePath}:\n${content}\n\n`; } } } diff --git a/core/commands/slash/share.ts b/core/commands/slash/share.ts index 4b69f4c623..01d5bea969 100644 --- a/core/commands/slash/share.ts +++ b/core/commands/slash/share.ts @@ -1,10 +1,12 @@ -import * as fs from "node:fs"; +import fs from "fs"; import { homedir } from "node:os"; +import { fileURLToPath, pathToFileURL } from "node:url"; import path from "path"; import { languageForFilepath } from "../../autocomplete/constants/AutocompleteLanguageInfo.js"; import { SlashCommand } from "../../index.js"; import { renderChatMessage } from "../../util/messageContent.js"; +import { getContinueGlobalPath } from "../../util/paths.js"; // If useful elsewhere, helper funcs should move to core/util/index.ts or similar function getOffsetDatetime(date: Date): Date { @@ -62,10 +64,7 @@ const ShareSlashCommand: SlashCommand = { }\n\n${msgText}`; } - let outputDir: string = params?.outputDir; - if (!outputDir) { - outputDir = await ide.getContinueDir(); - } + let outputDir: string = params?.outputDir ?? getContinueGlobalPath(); if (outputDir.startsWith("~")) { outputDir = outputDir.replace(/^~/, homedir); @@ -81,7 +80,7 @@ const ShareSlashCommand: SlashCommand = { // folders are included. We default to using the first item in the list, if // it exists. const workspaceDirectory = workspaceDirs?.[0] || ""; - outputDir = outputDir.replace(/^./, workspaceDirectory); + outputDir = outputDir.replace(/^./, fileURLToPath(workspaceDirectory)); } if (!fs.existsSync(outputDir)) { @@ -91,8 +90,9 @@ const ShareSlashCommand: SlashCommand = { const dtString = asBasicISOString(getOffsetDatetime(now)); const outPath = path.join(outputDir, `${dtString}_session.md`); //TODO: more flexible naming? - await ide.writeFile(outPath, content); - await ide.openFile(outPath); + const fileUrl = pathToFileURL(outPath).toString(); // TODO switch from path to URI above ^ + await ide.writeFile(fileUrl, content); + await ide.openFile(fileUrl); yield `The session transcript has been saved to a markdown file at \`${outPath}\`.`; }, diff --git a/core/commands/util.ts b/core/commands/util.ts index 74266e70f3..1cd493fad7 100644 --- a/core/commands/util.ts +++ b/core/commands/util.ts @@ -1,7 +1,35 @@ import { ContextItemWithId, RangeInFileWithContents } from "../"; +import { findUriInDirs, getUriPathBasename } from "../util/uri"; +import { v4 as uuidv4 } from "uuid"; + +export function rifWithContentsToContextItem( + rif: RangeInFileWithContents, +): ContextItemWithId { + const basename = getUriPathBasename(rif.filepath); + const { relativePathOrBasename, foundInDir, uri } = findUriInDirs( + rif.filepath, + window.workspacePaths ?? [], + ); + const rangeStr = `(${rif.range.start.line + 1}-${rif.range.end.line + 1})`; + + return { + content: rif.contents, + name: `${basename} ${rangeStr}`, + description: `${relativePathOrBasename} ${rangeStr}`, // This is passed to the LLM - do not pass full URI + id: { + providerTitle: "code", + itemId: uuidv4(), + }, + uri: { + type: "file", + value: rif.filepath, + }, + }; +} export function ctxItemToRifWithContents( item: ContextItemWithId, + linesOffByOne = false, ): RangeInFileWithContents { let startLine = 0; let endLine = 0; @@ -10,8 +38,8 @@ export function ctxItemToRifWithContents( if (nameSplit.length > 1) { const lines = nameSplit[1].split(")")[0].split("-"); - startLine = Number.parseInt(lines[0], 10); - endLine = Number.parseInt(lines[1], 10); + startLine = Number.parseInt(lines[0], 10) - (linesOffByOne ? 1 : 0); + endLine = Number.parseInt(lines[1], 10) - (linesOffByOne ? 1 : 0); } const rif: RangeInFileWithContents = { diff --git a/core/config/getSystemPromptDotFile.ts b/core/config/getSystemPromptDotFile.ts index 03690dd47e..550144cd88 100644 --- a/core/config/getSystemPromptDotFile.ts +++ b/core/config/getSystemPromptDotFile.ts @@ -1,14 +1,13 @@ import { IDE } from ".."; - +import { joinPathsToUri } from "../util/uri"; export const SYSTEM_PROMPT_DOT_FILE = ".continuerules"; export async function getSystemPromptDotFile(ide: IDE): Promise { const dirs = await ide.getWorkspaceDirs(); let prompts: string[] = []; - const pathSep = await ide.pathSep(); for (const dir of dirs) { - const dotFile = `${dir}${pathSep}${SYSTEM_PROMPT_DOT_FILE}`; + const dotFile = joinPathsToUri(dir, SYSTEM_PROMPT_DOT_FILE); if (await ide.fileExists(dotFile)) { try { const content = await ide.readFile(dotFile); diff --git a/core/config/load.ts b/core/config/load.ts index eced5d3467..8463f3f839 100644 --- a/core/config/load.ts +++ b/core/config/load.ts @@ -59,7 +59,6 @@ import { getConfigTsPath, getContinueDotEnv, getEsbuildBinaryPath, - readAllGlobalPromptFiles, } from "../util/paths"; import { @@ -69,12 +68,9 @@ import { defaultSlashCommandsVscode, } from "./default"; import { getSystemPromptDotFile } from "./getSystemPromptDotFile"; -import { - DEFAULT_PROMPTS_FOLDER, - getPromptFiles, - slashCommandFromPromptFile, -} from "./promptFile.js"; import { ConfigValidationError, validateConfig } from "./validation.js"; +import { slashCommandFromPromptFileV1 } from "../promptFiles/v1/slashCommandFromPromptFile"; +import { getAllPromptFiles } from "../promptFiles/v2/getPromptFiles"; export interface ConfigResult { config: T | undefined; @@ -184,8 +180,8 @@ function loadSerializedConfig( async function serializedToIntermediateConfig( initial: SerializedContinueConfig, ide: IDE, - loadPromptFiles: boolean = true, ): Promise { + // DEPRECATED - load custom slash commands const slashCommands: SlashCommand[] = []; for (const command of initial.slashCommands || []) { const newCommand = slashCommandFromDescription(command); @@ -197,33 +193,18 @@ async function serializedToIntermediateConfig( slashCommands.push(slashFromCustomCommand(command)); } - const workspaceDirs = await ide.getWorkspaceDirs(); - const promptFolder = initial.experimental?.promptPath; - - if (loadPromptFiles) { - // v1 prompt files - let promptFiles: { path: string; content: string }[] = []; - promptFiles = ( - await Promise.all( - workspaceDirs.map((dir) => - getPromptFiles( - ide, - path.join(dir, promptFolder ?? DEFAULT_PROMPTS_FOLDER), - ), - ), - ) - ) - .flat() - .filter(({ path }) => path.endsWith(".prompt")); - - // Also read from ~/.continue/.prompts - promptFiles.push(...readAllGlobalPromptFiles()); + // DEPRECATED - load slash commands from v1 prompt files + // NOTE: still checking the v1 default .prompts folder for slash commands + const promptFiles = await getAllPromptFiles( + ide, + initial.experimental?.promptPath, + true, + ); - for (const file of promptFiles) { - const slashCommand = slashCommandFromPromptFile(file.path, file.content); - if (slashCommand) { - slashCommands.push(slashCommand); - } + for (const file of promptFiles) { + const slashCommand = slashCommandFromPromptFileV1(file.path, file.content); + if (slashCommand) { + slashCommands.push(slashCommand); } } @@ -865,6 +846,5 @@ export { finalToBrowserConfig, intermediateToFinalConfig, loadFullConfigNode, - serializedToIntermediateConfig, type BrowserSerializedContinueConfig, }; diff --git a/core/config/onboarding.ts b/core/config/onboarding.ts index 6b05ca59ff..426a7a4ba8 100644 --- a/core/config/onboarding.ts +++ b/core/config/onboarding.ts @@ -4,7 +4,7 @@ import { FREE_TRIAL_MODELS } from "./default"; export const TRIAL_FIM_MODEL = "codestral-latest"; export const ONBOARDING_LOCAL_MODEL_TITLE = "Ollama"; -export const LOCAL_ONBOARDING_FIM_MODEL = "qwen2.5-coder:1.5b"; +export const LOCAL_ONBOARDING_FIM_MODEL = "qwen2.5-coder:1.5b-base"; export const LOCAL_ONBOARDING_CHAT_MODEL = "llama3.1:8b"; export const LOCAL_ONBOARDING_CHAT_TITLE = "Llama 3.1 8B"; diff --git a/core/config/promptFile.ts b/core/config/promptFile.ts deleted file mode 100644 index de17a7120b..0000000000 --- a/core/config/promptFile.ts +++ /dev/null @@ -1,285 +0,0 @@ -import path from "path"; - -import Handlebars from "handlebars"; -import * as YAML from "yaml"; - -import { walkDir } from "../indexing/walkDir"; -import { renderTemplatedString } from "../promptFiles/v1/renderTemplatedString"; -import { getBasename } from "../util/index"; -import { renderChatMessage } from "../util/messageContent"; - -import type { - ChatMessage, - ContextItem, - ContinueSDK, - IContextProvider, - IDE, - SlashCommand, -} from ".."; - -export const DEFAULT_PROMPTS_FOLDER = ".prompts"; - -export async function getPromptFiles( - ide: IDE, - dir: string, -): Promise<{ path: string; content: string }[]> { - try { - const exists = await ide.fileExists(dir); - - if (!exists) { - return []; - } - - const paths = await walkDir(dir, ide, { ignoreFiles: [] }); - const results = paths.map(async (path) => { - const content = await ide.readFile(path); - return { path, content }; - }); - return Promise.all(results); - } catch (e) { - console.error(e); - return []; - } -} - -const DEFAULT_PROMPT_FILE = `# This is an example ".prompt" file -# It is used to define and reuse prompts within Continue -# Continue will automatically create a slash command for each prompt in the .prompts folder -# To learn more, see the full .prompt file reference: https://docs.continue.dev/features/prompt-files -temperature: 0.0 ---- -{{{ diff }}} - -Give me feedback on the above changes. For each file, you should output a markdown section including the following: -- If you found any problems, an h3 like "❌ " -- If you didn't find any problems, an h3 like "✅ " -- If you found any problems, add below a bullet point description of what you found, including a minimal code snippet explaining how to fix it -- If you didn't find any problems, you don't need to add anything else - -Here is an example. The example is surrounded in backticks, but your response should not be: - -\`\`\` -### ✅ - -### ❌ - - -\`\`\` - -You should look primarily for the following types of issues, and only mention other problems if they are highly pressing. - -- console.logs that have been left after debugging -- repeated code -- algorithmic errors that could fail under edge cases -- something that could be refactored - -Make sure to review ALL files that were changed, do not skip any. -`; - -export async function createNewPromptFile( - ide: IDE, - promptPath: string | undefined, -): Promise { - const workspaceDirs = await ide.getWorkspaceDirs(); - if (workspaceDirs.length === 0) { - throw new Error( - "No workspace directories found. Make sure you've opened a folder in your IDE.", - ); - } - const promptFilePath = path.join( - workspaceDirs[0], - promptPath ?? DEFAULT_PROMPTS_FOLDER, - "new-prompt-file.prompt", - ); - - await ide.writeFile(promptFilePath, DEFAULT_PROMPT_FILE); - await ide.openFile(promptFilePath); -} - -export function slashCommandFromPromptFile( - path: string, - content: string, -): SlashCommand | null { - const { name, description, systemMessage, prompt, version } = parsePromptFile( - path, - content, - ); - - if (version !== 1) { - return null; - } - - return { - name, - description, - run: async function* (context) { - const originalSystemMessage = context.llm.systemMessage; - context.llm.systemMessage = systemMessage; - - const userInput = extractUserInput(context.input, name); - const renderedPrompt = await renderPrompt(prompt, context, userInput); - const messages = updateChatHistory( - context.history, - name, - renderedPrompt, - systemMessage, - ); - - for await (const chunk of context.llm.streamChat( - messages, - new AbortController().signal, - )) { - yield renderChatMessage(chunk); - } - - context.llm.systemMessage = originalSystemMessage; - }, - }; -} - -function parsePromptFile(path: string, content: string) { - let [preambleRaw, prompt] = content.split("\n---\n"); - if (prompt === undefined) { - prompt = preambleRaw; - preambleRaw = ""; - } - - const preamble = YAML.parse(preambleRaw) ?? {}; - const name = preamble.name ?? getBasename(path).split(".prompt")[0]; - const description = preamble.description ?? name; - const version = preamble.version ?? 2; - - let systemMessage: string | undefined = undefined; - if (prompt.includes("")) { - systemMessage = prompt.split("")[1].split("")[0].trim(); - prompt = prompt.split("")[1].trim(); - } - - return { name, description, systemMessage, prompt, version }; -} - -function extractUserInput(input: string, commandName: string): string { - if (input.startsWith(`/${commandName}`)) { - return input.slice(commandName.length + 1).trimStart(); - } - return input; -} - -async function renderPrompt( - prompt: string, - context: ContinueSDK, - userInput: string, -) { - const helpers = getContextProviderHelpers(context); - - // A few context providers that don't need to be in config.json to work in .prompt files - const diff = await context.ide.getDiff(true); - const currentFile = await context.ide.getCurrentFile(); - const inputData: Record = { - diff: diff.join("\n"), - input: userInput, - }; - if (currentFile) { - inputData.currentFile = currentFile.path; - } - - return renderTemplatedString( - prompt, - context.ide.readFile.bind(context.ide), - inputData, - helpers, - ); -} - -function getContextProviderHelpers( - context: ContinueSDK, -): Array<[string, Handlebars.HelperDelegate]> | undefined { - return context.config.contextProviders?.map((provider: IContextProvider) => [ - provider.description.title, - async (helperContext: any) => { - const items = await provider.getContextItems(helperContext, { - config: context.config, - embeddingsProvider: context.config.embeddingsProvider, - fetch: context.fetch, - fullInput: context.input, - ide: context.ide, - llm: context.llm, - reranker: context.config.reranker, - selectedCode: context.selectedCode, - }); - - items.forEach((item) => - context.addContextItem(createContextItem(item, provider)), - ); - - return items.map((item) => item.content).join("\n\n"); - }, - ]); -} - -function createContextItem(item: ContextItem, provider: IContextProvider) { - return { - ...item, - id: { - itemId: item.description, - providerTitle: provider.description.title, - }, - }; -} - -function updateChatHistory( - history: ChatMessage[], - commandName: string, - renderedPrompt: string, - systemMessage?: string, -) { - const messages = [...history]; - - for (let i = messages.length - 1; i >= 0; i--) { - const message = messages[i]; - const { role, content } = message; - if (role !== "user") { - continue; - } - - if (Array.isArray(content)) { - if (content.some((part) => part.text?.startsWith(`/${commandName}`))) { - messages[i] = updateArrayContent( - messages[i], - commandName, - renderedPrompt, - ); - break; - } - } else if ( - typeof content === "string" && - content.startsWith(`/${commandName}`) - ) { - messages[i] = { ...message, content: renderedPrompt }; - break; - } - } - - if (systemMessage) { - messages[0]?.role === "system" - ? (messages[0].content = systemMessage) - : messages.unshift({ role: "system", content: systemMessage }); - } - - return messages; -} - -function updateArrayContent( - message: any, - commandName: string, - renderedPrompt: string, -) { - return { - ...message, - content: message.content.map((part: any) => - part.text?.startsWith(`/${commandName}`) - ? { ...part, text: renderedPrompt } - : part, - ), - }; -} diff --git a/core/config/types.ts b/core/config/types.ts index 60a2ae8dca..6c7e580dde 100644 --- a/core/config/types.ts +++ b/core/config/types.ts @@ -607,8 +607,6 @@ declare global { getAvailableThreads(): Promise; - listFolders(): Promise; - getWorkspaceDirs(): Promise; getWorkspaceConfigs(): Promise; @@ -618,9 +616,6 @@ declare global { writeFile(path: string, contents: string): Promise; showVirtualFile(title: string, contents: string): Promise; - - getContinueDir(): Promise; - openFile(path: string): Promise; openUrl(url: string): Promise; @@ -638,13 +633,6 @@ declare global { startLine: number, endLine: number, ): Promise; - - showDiff( - filepath: string, - newContents: string, - stepIndex: number, - ): Promise; - getOpenFiles(): Promise; getCurrentFile(): Promise< @@ -689,8 +677,6 @@ declare global { // Callbacks onDidChangeActiveTextEditor(callback: (filepath: string) => void): void; - - pathSep(): Promise; } // Slash Commands diff --git a/core/config/yaml/loadYaml.ts b/core/config/yaml/loadYaml.ts index cc219e9521..ad33ea4d85 100644 --- a/core/config/yaml/loadYaml.ts +++ b/core/config/yaml/loadYaml.ts @@ -32,14 +32,11 @@ import { readAllGlobalPromptFiles, } from "../../util/paths"; import { getSystemPromptDotFile } from "../getSystemPromptDotFile"; -import { - DEFAULT_PROMPTS_FOLDER, - getPromptFiles, - slashCommandFromPromptFile, -} from "../promptFile.js"; import { ConfigValidationError } from "../validation.js"; import { llmsFromModelConfig } from "./models"; +import { getAllPromptFiles } from "../../promptFiles/v2/getPromptFiles"; +import { slashCommandFromPromptFileV1 } from "../../promptFiles/v1/slashCommandFromPromptFile"; export interface ConfigResult { config: T | undefined; @@ -102,25 +99,11 @@ async function slashCommandsFromV1PromptFiles( ide: IDE, ): Promise { const slashCommands: SlashCommand[] = []; - const workspaceDirs = await ide.getWorkspaceDirs(); - - // v1 prompt files - let promptFiles: { path: string; content: string }[] = []; - promptFiles = ( - await Promise.all( - workspaceDirs.map((dir) => - getPromptFiles(ide, path.join(dir, DEFAULT_PROMPTS_FOLDER)), - ), - ) - ) - .flat() - .filter(({ path }) => path.endsWith(".prompt")); - - // Also read from ~/.continue/.prompts - promptFiles.push(...readAllGlobalPromptFiles()); + + const promptFiles = await getAllPromptFiles(ide, undefined, true); for (const file of promptFiles) { - const slashCommand = slashCommandFromPromptFile(file.path, file.content); + const slashCommand = slashCommandFromPromptFileV1(file.path, file.content); if (slashCommand) { slashCommands.push(slashCommand); } diff --git a/core/context/providers/CodeHighlightsContextProvider.ts b/core/context/providers/CodeHighlightsContextProvider.ts index 1cae074a49..1b9d017f37 100644 --- a/core/context/providers/CodeHighlightsContextProvider.ts +++ b/core/context/providers/CodeHighlightsContextProvider.ts @@ -3,7 +3,6 @@ import { ContextProviderDescription, ContextProviderExtras, } from "../../index.js"; -import { getBasename } from "../../util/index.js"; import { BaseContextProvider } from "../index.js"; const HIGHLIGHTS_TOKEN_BUDGET = 2000; @@ -26,16 +25,16 @@ class CodeHighlightsContextProvider extends BaseContextProvider { // ); const ide = extras.ide; const openFiles = await ide.getOpenFiles(); - const allFiles: { name: string; absPath: string; content: string }[] = - await Promise.all( - openFiles.map(async (filepath: string) => { - return { - name: getBasename(filepath), - absPath: filepath, - content: `${await ide.readFile(filepath)}`, - }; - }), - ); + // const allFiles: { name: string; absPath: string; content: string }[] = + // await Promise.all( + // openFiles.map(async (filepath: string) => { + // return { + // name: getBasename(filepath), + // absPath: filepath, + // content: `${await ide.readFile(filepath)}`, + // }; + // }), + // ); // const contextSizer = { // fits(content: string): boolean { // return countTokens(content, "") < HIGHLIGHTS_TOKEN_BUDGET; diff --git a/core/context/providers/CodeOutlineContextProvider.ts b/core/context/providers/CodeOutlineContextProvider.ts index 85781446e3..e12fa8231e 100644 --- a/core/context/providers/CodeOutlineContextProvider.ts +++ b/core/context/providers/CodeOutlineContextProvider.ts @@ -3,7 +3,6 @@ import { ContextProviderDescription, ContextProviderExtras, } from "../../index.js"; -import { getBasename } from "../../util/index.js"; import { BaseContextProvider } from "../index.js"; class CodeOutlineContextProvider extends BaseContextProvider { @@ -21,16 +20,16 @@ class CodeOutlineContextProvider extends BaseContextProvider { ): Promise { const ide = extras.ide; const openFiles = await ide.getOpenFiles(); - const allFiles: { name: string; absPath: string; content: string }[] = - await Promise.all( - openFiles.map(async (filepath: string) => { - return { - name: getBasename(filepath), - absPath: filepath, - content: `${await ide.readFile(filepath)}`, - }; - }), - ); + // const allFiles: { name: string; absPath: string; content: string }[] = + // await Promise.all( + // openFiles.map(async (filepath: string) => { + // return { + // name: getBasename(filepath), + // absPath: filepath, + // content: `${await ide.readFile(filepath)}`, + // }; + // }), + // ); // const outlines = await getOutlines( // allFiles // .filter((file) => file.content.length > 0) diff --git a/core/context/providers/CurrentFileContextProvider.ts b/core/context/providers/CurrentFileContextProvider.ts index ccb1b231d0..d3cef2dc11 100644 --- a/core/context/providers/CurrentFileContextProvider.ts +++ b/core/context/providers/CurrentFileContextProvider.ts @@ -4,7 +4,7 @@ import { ContextProviderDescription, ContextProviderExtras, } from "../../"; -import { getBasename } from "../../util/"; +import { getUriPathBasename } from "../../util/uri"; class CurrentFileContextProvider extends BaseContextProvider { static description: ContextProviderDescription = { @@ -19,12 +19,11 @@ class CurrentFileContextProvider extends BaseContextProvider { query: string, extras: ContextProviderExtras, ): Promise { - const ide = extras.ide; - const currentFile = await ide.getCurrentFile(); + const currentFile = await extras.ide.getCurrentFile(); if (!currentFile) { return []; } - const baseName = getBasename(currentFile.path); + const baseName = getUriPathBasename(currentFile.path); return [ { description: currentFile.path, diff --git a/core/context/providers/FileContextProvider.ts b/core/context/providers/FileContextProvider.ts index 26fb9dc735..674cc1a009 100644 --- a/core/context/providers/FileContextProvider.ts +++ b/core/context/providers/FileContextProvider.ts @@ -6,12 +6,11 @@ import { ContextSubmenuItem, LoadSubmenuItemsArgs, } from "../../"; -import { walkDir } from "../../indexing/walkDir"; +import { walkDirs } from "../../indexing/walkDir"; import { - getBasename, - getUniqueFilePath, - groupByLastNPathParts, -} from "../../util/"; + getUriPathBasename, + getShortestUniqueRelativeUriPaths, +} from "../../util/uri"; const MAX_SUBMENU_ITEMS = 10_000; @@ -47,19 +46,18 @@ class FileContextProvider extends BaseContextProvider { args: LoadSubmenuItemsArgs, ): Promise { const workspaceDirs = await args.ide.getWorkspaceDirs(); - const results = await Promise.all( - workspaceDirs.map((dir) => { - return walkDir(dir, args.ide); - }), - ); + const results = await walkDirs(args.ide, undefined, workspaceDirs); const files = results.flat().slice(-MAX_SUBMENU_ITEMS); - const fileGroups = groupByLastNPathParts(files, 2); + const withUniquePaths = getShortestUniqueRelativeUriPaths( + files, + workspaceDirs, + ); - return files.map((file) => { + return withUniquePaths.map((file) => { return { - id: file, - title: getBasename(file), - description: getUniqueFilePath(file, fileGroups), + id: file.uri, + title: getUriPathBasename(file.uri), + description: file.uniquePath, }; }); } diff --git a/core/context/providers/FileTreeContextProvider.ts b/core/context/providers/FileTreeContextProvider.ts index 52af978236..3b98dd6cb5 100644 --- a/core/context/providers/FileTreeContextProvider.ts +++ b/core/context/providers/FileTreeContextProvider.ts @@ -4,7 +4,7 @@ import { ContextProviderExtras, } from "../../index.js"; import { walkDir } from "../../indexing/walkDir.js"; -import { splitPath } from "../../util/index.js"; +import { findUriInDirs, getUriPathBasename } from "../../util/uri.js"; import { BaseContextProvider } from "../index.js"; interface Directory { @@ -44,16 +44,19 @@ class FileTreeContextProvider extends BaseContextProvider { const trees = []; for (const workspaceDir of workspaceDirs) { - const contents = await walkDir(workspaceDir, extras.ide); - const subDirTree: Directory = { - name: splitPath(workspaceDir).pop() ?? "", + name: getUriPathBasename(workspaceDir), files: [], directories: [], }; - for (const file of contents) { - const parts = splitPath(file, workspaceDir); + const uris = await walkDir(workspaceDir, extras.ide); + const relativePaths = uris.map( + (uri) => findUriInDirs(uri, [workspaceDir]).relativePathOrBasename, + ); + + for (const path of relativePaths) { + const parts = path.split("/"); let currentTree = subDirTree; for (const part of parts.slice(0, -1)) { diff --git a/core/context/providers/FolderContextProvider.ts b/core/context/providers/FolderContextProvider.ts index a4b1101fb8..dc52575683 100644 --- a/core/context/providers/FolderContextProvider.ts +++ b/core/context/providers/FolderContextProvider.ts @@ -5,12 +5,13 @@ import { ContextSubmenuItem, LoadSubmenuItemsArgs, } from "../../index.js"; +import { walkDirs } from "../../indexing/walkDir.js"; import { - getBasename, - groupByLastNPathParts, - getUniqueFilePath, -} from "../../util/index.js"; + getShortestUniqueRelativeUriPaths, + getUriPathBasename, +} from "../../util/uri.js"; import { BaseContextProvider } from "../index.js"; +import { retrieveContextItemsFromEmbeddings } from "../retrieval/retrieval.js"; class FolderContextProvider extends BaseContextProvider { static description: ContextProviderDescription = { @@ -25,22 +26,29 @@ class FolderContextProvider extends BaseContextProvider { query: string, extras: ContextProviderExtras, ): Promise { - const { retrieveContextItemsFromEmbeddings } = await import( - "../retrieval/retrieval.js" - ); return retrieveContextItemsFromEmbeddings(extras, this.options, query); } async loadSubmenuItems( args: LoadSubmenuItemsArgs, ): Promise { - const folders = await args.ide.listFolders(); - const folderGroups = groupByLastNPathParts(folders, 2); + const workspaceDirs = await args.ide.getWorkspaceDirs(); + const folders = await walkDirs( + args.ide, + { + onlyDirs: true, + }, + workspaceDirs, + ); + const withUniquePaths = getShortestUniqueRelativeUriPaths( + folders, + workspaceDirs, + ); - return folders.map((folder) => { + return withUniquePaths.map((folder) => { return { - id: folder, - title: getBasename(folder), - description: getUniqueFilePath(folder, folderGroups), + id: folder.uri, + title: getUriPathBasename(folder.uri), + description: folder.uniquePath, }; }); } diff --git a/core/context/providers/GreptileContextProvider.ts b/core/context/providers/GreptileContextProvider.ts index 3ae6fa2141..a13e4376f6 100644 --- a/core/context/providers/GreptileContextProvider.ts +++ b/core/context/providers/GreptileContextProvider.ts @@ -1,188 +1,209 @@ - import { execSync } from "child_process"; - import * as fs from "fs"; - import * as path from "path"; +import { execSync } from "child_process"; +import * as fs from "fs"; +import * as path from "path"; import { - ContextItem, - ContextProviderDescription, - ContextProviderExtras, - } from "../../index.js"; - import { BaseContextProvider } from "../index.js"; - - - class GreptileContextProvider extends BaseContextProvider { - static description: ContextProviderDescription = { - title: "greptile", - displayTitle: "Greptile", - description: "Insert query to Greptile", - type: "query", - }; - - async getContextItems( - query: string, - extras: ContextProviderExtras, - ): Promise { - const greptileToken = this.getGreptileToken(); - if (!greptileToken) { - throw new Error("Greptile token not found."); - } + ContextItem, + ContextProviderDescription, + ContextProviderExtras, +} from "../../index.js"; +import { BaseContextProvider } from "../index.js"; - const githubToken = this.getGithubToken(); - if (!githubToken) { - throw new Error("GitHub token not found."); - } - - let absPath = await this.getWorkspaceDir(extras); - if (!absPath) { - throw new Error("Failed to determine the workspace directory."); - } +class GreptileContextProvider extends BaseContextProvider { + static description: ContextProviderDescription = { + title: "greptile", + displayTitle: "Greptile", + description: "Insert query to Greptile", + type: "query", + }; - var remoteUrl = getRemoteUrl(absPath); - remoteUrl = getRemoteUrl(absPath); - const repoName = extractRepoName(remoteUrl); - const branch = getCurrentBranch(absPath); - const remoteType = getRemoteType(remoteUrl); - - if (!remoteType) { - throw new Error("Unable to determine remote type."); - } - - const options = { - method: "POST", - headers: { - Authorization: `Bearer ${greptileToken}`, - "X-GitHub-Token": githubToken, - "Content-Type": "application/json", - }, - body: JSON.stringify({ - messages: [{ id: "", content: query, role: "user" }], - repositories: [{ - remote: remoteType, - branch: branch, - repository: repoName - }], - sessionId: extras.config.userToken || "default-session", - stream: false, - genius: true, - }), - }; - - try { - const response = await extras.fetch("https://api.greptile.com/v2/query", options); - const rawText = await response.text(); - - // Check for HTTP errors - if (!response.ok) { - throw new Error(`HTTP error! status: ${response.status}`); - } - - // Parse the response as JSON - const json = JSON.parse(rawText); - - return json.sources.map((source: any) => ({ - description: source.filepath, - content: `File: ${source.filepath}\nLines: ${source.linestart}-${source.lineend}\n\n${source.summary}`, - name: (source.filepath.split("/").pop() ?? "").split("\\").pop() ?? "", - })); - } catch (error) { - console.error("Error getting context items from Greptile:", error); - throw new Error("Error getting context items from Greptile"); - } + async getContextItems( + query: string, + extras: ContextProviderExtras, + ): Promise { + const greptileToken = this.getGreptileToken(); + if (!greptileToken) { + throw new Error("Greptile token not found."); } - - private getGreptileToken(): string | undefined { - return this.options.GreptileToken || process.env.GREPTILE_AUTH_TOKEN; + + const githubToken = this.getGithubToken(); + if (!githubToken) { + throw new Error("GitHub token not found."); } - private getGithubToken(): string | undefined { - return this.options.GithubToken || process.env.GITHUB_TOKEN; + let absPath = await this.getWorkspaceDir(extras); + if (!absPath) { + throw new Error("Failed to determine the workspace directory."); } - - private async getWorkspaceDir(extras: ContextProviderExtras): Promise { - try { - const workspaceDirs = await extras.ide.getWorkspaceDirs(); - if (workspaceDirs && workspaceDirs.length > 0) { - return workspaceDirs[0]; - } else { - console.warn("extras.ide.getWorkspaceDirs() returned undefined or empty array."); - } - } catch (err) { - console.warn("Failed to get workspace directories from extras.ide.getWorkspaceDirs():"); - } - - // Fallback to using Git commands - try { - const currentDir = process.cwd(); - if (this.isGitRepository(currentDir)) { - const workspaceDir = execSync("git rev-parse --show-toplevel").toString().trim(); - return workspaceDir; - } else { - console.warn(`Current directory is not a Git repository: ${currentDir}`); - return null; - } - } catch (err) { - console.warn("Failed to get workspace directory using Git commands: "); - return null; - } + + var remoteUrl = getRemoteUrl(absPath); + remoteUrl = getRemoteUrl(absPath); + const repoName = extractRepoName(remoteUrl); + const branch = getCurrentBranch(absPath); + const remoteType = getRemoteType(remoteUrl); + + if (!remoteType) { + throw new Error("Unable to determine remote type."); } - - private isGitRepository(dir: string): boolean { - try { - const gitDir = path.join(dir, ".git"); - return fs.existsSync(gitDir); - } catch (err) { - console.warn("Failed to check if directory is a Git repository:"); - return false; + + const options = { + method: "POST", + headers: { + Authorization: `Bearer ${greptileToken}`, + "X-GitHub-Token": githubToken, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + messages: [{ id: "", content: query, role: "user" }], + repositories: [ + { + remote: remoteType, + branch: branch, + repository: repoName, + }, + ], + sessionId: extras.config.userToken || "default-session", + stream: false, + genius: true, + }), + }; + + try { + const response = await extras.fetch( + "https://api.greptile.com/v2/query", + options, + ); + const rawText = await response.text(); + + // Check for HTTP errors + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); } + + // Parse the response as JSON + const json = JSON.parse(rawText); + + return json.sources.map((source: any) => ({ + description: source.filepath, + content: `File: ${source.filepath}\nLines: ${source.linestart}-${source.lineend}\n\n${source.summary}`, + name: (source.filepath.split("/").pop() ?? "").split("\\").pop() ?? "", + })); + } catch (error) { + console.error("Error getting context items from Greptile:", error); + throw new Error("Error getting context items from Greptile"); } } - - // Helper functions - function getRemoteUrl(absPath: string): string { + + private getGreptileToken(): string | undefined { + return this.options.GreptileToken || process.env.GREPTILE_AUTH_TOKEN; + } + + private getGithubToken(): string | undefined { + return this.options.GithubToken || process.env.GITHUB_TOKEN; + } + + private async getWorkspaceDir( + extras: ContextProviderExtras, + ): Promise { try { - const remote = execSync(`git -C ${absPath} remote get-url origin`).toString().trim(); - return remote; + const workspaceDirs = await extras.ide.getWorkspaceDirs(); + if (workspaceDirs && workspaceDirs.length > 0) { + return workspaceDirs[0]; + } else { + console.warn( + "extras.ide.getWorkspaceDirs() returned undefined or empty array.", + ); + } } catch (err) { - console.warn("Failed to get remote URL"); - return ""; + console.warn( + "Failed to get workspace directories from extras.ide.getWorkspaceDirs():", + ); } - } - - function getCurrentBranch(absPath: string): string { + + // Fallback to using Git commands try { - const branch = execSync(`git -C ${absPath} rev-parse --abbrev-ref HEAD`).toString().trim(); - return branch; + const currentDir = process.cwd(); + if (this.isGitRepository(currentDir)) { + const workspaceDir = execSync("git rev-parse --show-toplevel") + .toString() + .trim(); + return workspaceDir; + } else { + console.warn( + `Current directory is not a Git repository: ${currentDir}`, + ); + return null; + } } catch (err) { - console.warn("Failed to get current branch"); - return "master"; // Default to 'master' if the current branch cannot be determined + console.warn("Failed to get workspace directory using Git commands: "); + return null; } } - - function extractRepoName(remote: string): string { - if (remote.startsWith("http://") || remote.startsWith("https://")) { - const parts = remote.split("/"); - if (parts.length >= 2) { - return parts[parts.length - 2] + "/" + parts[parts.length - 1].replace(".git", ""); - } - } else if (remote.startsWith("git@")) { - const parts = remote.split(":"); - if (parts.length >= 2) { - return parts[1].replace(".git", ""); - } + + private isGitRepository(dir: string): boolean { + try { + const gitDir = path.join(dir, ".git"); + return fs.existsSync(gitDir); + } catch (err) { + console.warn("Failed to check if directory is a Git repository:"); + return false; } + } +} + +// Helper functions +function getRemoteUrl(absPath: string): string { + try { + const remote = execSync(`git -C ${absPath} remote get-url origin`) + .toString() + .trim(); + return remote; + } catch (err) { + console.warn("Failed to get remote URL"); return ""; } - - function getRemoteType(remote: string): string { - if (remote.includes("github.com")) { - return "github"; - } else if (remote.includes("gitlab.com")) { - return "gitlab"; - } else if (remote.includes("azure.com")) { - return "azure"; +} + +function getCurrentBranch(absPath: string): string { + try { + const branch = execSync(`git -C ${absPath} rev-parse --abbrev-ref HEAD`) + .toString() + .trim(); + return branch; + } catch (err) { + console.warn("Failed to get current branch"); + return "master"; // Default to 'master' if the current branch cannot be determined + } +} + +function extractRepoName(remote: string): string { + if (remote.startsWith("http://") || remote.startsWith("https://")) { + const parts = remote.split("/"); + if (parts.length >= 2) { + return ( + parts[parts.length - 2] + + "/" + + parts[parts.length - 1].replace(".git", "") + ); + } + } else if (remote.startsWith("git@")) { + const parts = remote.split(":"); + if (parts.length >= 2) { + return parts[1].replace(".git", ""); } - return ""; } - - export default GreptileContextProvider; - \ No newline at end of file + return ""; +} + +function getRemoteType(remote: string): string { + if (remote.includes("github.com")) { + return "github"; + } else if (remote.includes("gitlab.com")) { + return "gitlab"; + } else if (remote.includes("azure.com")) { + return "azure"; + } + return ""; +} + +export default GreptileContextProvider; diff --git a/core/context/providers/OpenFilesContextProvider.ts b/core/context/providers/OpenFilesContextProvider.ts index c00121a96e..318ebb6480 100644 --- a/core/context/providers/OpenFilesContextProvider.ts +++ b/core/context/providers/OpenFilesContextProvider.ts @@ -3,7 +3,7 @@ import { ContextProviderDescription, ContextProviderExtras, } from "../../index.js"; -import { getRelativePath } from "../../util/index.js"; +import { findUriInDirs, getUriPathBasename } from "../../util/uri.js"; import { BaseContextProvider } from "../index.js"; class OpenFilesContextProvider extends BaseContextProvider { @@ -28,11 +28,10 @@ class OpenFilesContextProvider extends BaseContextProvider { openFiles.map(async (filepath: string) => { return { description: filepath, - content: `\`\`\`${await getRelativePath( - filepath, - workspaceDirs, - )}\n${await ide.readFile(filepath)}\n\`\`\``, - name: (filepath.split("/").pop() ?? "").split("\\").pop() ?? "", + content: `\`\`\`${ + findUriInDirs(filepath, workspaceDirs).relativePathOrBasename + }\n${await ide.readFile(filepath)}\n\`\`\``, + name: getUriPathBasename(filepath), uri: { type: "file", value: filepath, diff --git a/core/context/providers/ProblemsContextProvider.ts b/core/context/providers/ProblemsContextProvider.ts index d5ffd46d82..b1a0372a5d 100644 --- a/core/context/providers/ProblemsContextProvider.ts +++ b/core/context/providers/ProblemsContextProvider.ts @@ -3,7 +3,7 @@ import { ContextProviderDescription, ContextProviderExtras, } from "../../index.js"; -import { getBasename } from "../../util/index.js"; +import { getUriPathBasename } from "../../util/uri.js"; import { BaseContextProvider } from "../index.js"; class ProblemsContextProvider extends BaseContextProvider { @@ -23,6 +23,7 @@ class ProblemsContextProvider extends BaseContextProvider { const items = await Promise.all( problems.map(async (problem) => { + const fileName = getUriPathBasename(problem.filepath); const content = await ide.readFile(problem.filepath); const lines = content.split("\n"); const rangeContent = lines @@ -34,10 +35,8 @@ class ProblemsContextProvider extends BaseContextProvider { return { description: "Problems in current file", - content: `\`\`\`${getBasename( - problem.filepath, - )}\n${rangeContent}\n\`\`\`\n${problem.message}\n\n`, - name: `Warning in ${getBasename(problem.filepath)}`, + content: `\`\`\`${fileName}\n${rangeContent}\n\`\`\`\n${problem.message}\n\n`, + name: `Warning in ${fileName}`, }; }), ); diff --git a/core/context/providers/PromptFilesContextProvider.ts b/core/context/providers/PromptFilesContextProvider.ts index bc9239f1df..c3b4381f40 100644 --- a/core/context/providers/PromptFilesContextProvider.ts +++ b/core/context/providers/PromptFilesContextProvider.ts @@ -6,7 +6,7 @@ import { ContextSubmenuItem, LoadSubmenuItemsArgs, } from "../../"; -import { getAllPromptFilesV2 } from "../../promptFiles/v2/getPromptFiles"; +import { getAllPromptFiles } from "../../promptFiles/v2/getPromptFiles"; import { parsePreamble } from "../../promptFiles/v2/parse"; import { renderPromptFileV2 } from "../../promptFiles/v2/renderPromptFile"; @@ -38,9 +38,10 @@ class PromptFilesContextProvider extends BaseContextProvider { async loadSubmenuItems( args: LoadSubmenuItemsArgs, ): Promise { - const promptFiles = await getAllPromptFilesV2( + const promptFiles = await getAllPromptFiles( args.ide, args.config.experimental?.promptPath, + // Note, NOT checking v1 default folder here, deprecated for context provider ); return promptFiles.map((file) => { const preamble = parsePreamble(file.path, file.content); diff --git a/core/context/providers/RepoMapContextProvider.ts b/core/context/providers/RepoMapContextProvider.ts index 183ff3cb60..ae7d92bf9b 100644 --- a/core/context/providers/RepoMapContextProvider.ts +++ b/core/context/providers/RepoMapContextProvider.ts @@ -6,12 +6,12 @@ import { ContextSubmenuItem, LoadSubmenuItemsArgs, } from "../../"; -import { - getBasename, - getUniqueFilePath, - groupByLastNPathParts, -} from "../../util"; +import { walkDirs } from "../../indexing/walkDir"; import generateRepoMap from "../../util/generateRepoMap"; +import { + getShortestUniqueRelativeUriPaths, + getUriPathBasename, +} from "../../util/uri"; const ENTIRE_PROJECT_ITEM: ContextSubmenuItem = { id: "entire-codebase", @@ -36,7 +36,9 @@ class RepoMapContextProvider extends BaseContextProvider { name: "Repository Map", description: "Overview of the repository structure", content: await generateRepoMap(extras.llm, extras.ide, { - dirs: query === ENTIRE_PROJECT_ITEM.id ? undefined : [query], + dirUris: query === ENTIRE_PROJECT_ITEM.id ? undefined : [query], + outputRelativeUriPaths: true, + includeSignatures: false, }), }, ]; @@ -45,15 +47,25 @@ class RepoMapContextProvider extends BaseContextProvider { async loadSubmenuItems( args: LoadSubmenuItemsArgs, ): Promise { - const folders = await args.ide.listFolders(); - const folderGroups = groupByLastNPathParts(folders, 2); + const workspaceDirs = await args.ide.getWorkspaceDirs(); + const folders = await walkDirs( + args.ide, + { + onlyDirs: true, + }, + workspaceDirs, + ); + const withUniquePaths = getShortestUniqueRelativeUriPaths( + folders, + workspaceDirs, + ); return [ ENTIRE_PROJECT_ITEM, - ...folders.map((folder) => ({ - id: folder, - title: getBasename(folder), - description: getUniqueFilePath(folder, folderGroups), + ...withUniquePaths.map((folder) => ({ + id: folder.uri, + title: getUriPathBasename(folder.uri), + description: folder.uniquePath, })), ]; } diff --git a/core/context/retrieval/pipelines/BaseRetrievalPipeline.ts b/core/context/retrieval/pipelines/BaseRetrievalPipeline.ts index 1b9fa5b9dd..0ba94964c4 100644 --- a/core/context/retrieval/pipelines/BaseRetrievalPipeline.ts +++ b/core/context/retrieval/pipelines/BaseRetrievalPipeline.ts @@ -15,7 +15,6 @@ export interface RetrievalPipelineOptions { nRetrieve: number; nFinal: number; tags: BranchAndDir[]; - pathSep: string; filterDirectory?: string; includeEmbeddings?: boolean; // Used to handle JB w/o an embeddings model } @@ -37,8 +36,7 @@ export default class BaseRetrievalPipeline implements IRetrievalPipeline { constructor(protected readonly options: RetrievalPipelineOptions) { this.lanceDbIndex = new LanceDbIndex( options.config.embeddingsProvider, - (path) => options.ide.readFile(path), - options.pathSep, + (uri) => options.ide.readFile(uri), ); } diff --git a/core/context/retrieval/pipelines/NoRerankerRetrievalPipeline.ts b/core/context/retrieval/pipelines/NoRerankerRetrievalPipeline.ts index 052a1fce6a..af72c06392 100644 --- a/core/context/retrieval/pipelines/NoRerankerRetrievalPipeline.ts +++ b/core/context/retrieval/pipelines/NoRerankerRetrievalPipeline.ts @@ -1,4 +1,5 @@ import { Chunk } from "../../../"; +import { findUriInDirs } from "../../../util/uri"; import { requestFilesFromRepoMap } from "../repoMapRequest"; import { deduplicateChunks } from "../util"; @@ -44,8 +45,9 @@ export default class NoRerankerRetrievalPipeline extends BaseRetrievalPipeline { if (filterDirectory) { // Backup if the individual retrieval methods don't listen - retrievalResults = retrievalResults.filter((chunk) => - chunk.filepath.startsWith(filterDirectory), + retrievalResults = retrievalResults.filter( + (chunk) => + !!findUriInDirs(chunk.filepath, [filterDirectory]).foundInDir, ); } diff --git a/core/context/retrieval/pipelines/RerankerRetrievalPipeline.ts b/core/context/retrieval/pipelines/RerankerRetrievalPipeline.ts index 6f28a4a89c..7c7e698600 100644 --- a/core/context/retrieval/pipelines/RerankerRetrievalPipeline.ts +++ b/core/context/retrieval/pipelines/RerankerRetrievalPipeline.ts @@ -1,5 +1,6 @@ import { Chunk } from "../../.."; import { RETRIEVAL_PARAMS } from "../../../util/parameters"; +import { findUriInDirs } from "../../../util/uri"; import { requestFilesFromRepoMap } from "../repoMapRequest"; import { deduplicateChunks } from "../util"; @@ -40,8 +41,8 @@ export default class RerankerRetrievalPipeline extends BaseRetrievalPipeline { if (filterDirectory) { // Backup if the individual retrieval methods don't listen - retrievalResults = retrievalResults.filter((chunk) => - chunk.filepath.startsWith(filterDirectory), + retrievalResults = retrievalResults.filter( + (chunk) => !!findUriInDirs(chunk.filepath, [filterDirectory]), ); } diff --git a/core/context/retrieval/repoMapRequest.ts b/core/context/retrieval/repoMapRequest.ts index 5c82ace95e..39223321bd 100644 --- a/core/context/retrieval/repoMapRequest.ts +++ b/core/context/retrieval/repoMapRequest.ts @@ -2,6 +2,7 @@ import { Chunk, ContinueConfig, IDE, ILLM } from "../.."; import { getModelByRole } from "../../config/util"; import generateRepoMap from "../../util/generateRepoMap"; import { renderChatMessage } from "../../util/messageContent"; +import { getUriPathBasename } from "../../util/uri"; const SUPPORTED_MODEL_TITLE_FAMILIES = [ "claude-3", @@ -35,7 +36,7 @@ export async function requestFilesFromRepoMap( config: ContinueConfig, ide: IDE, input: string, - filterDirectory?: string, + filterDirUri?: string, ): Promise { const llm = getModelByRole(config, "repoMapFileSelection") ?? defaultLlm; @@ -46,8 +47,9 @@ export async function requestFilesFromRepoMap( try { const repoMap = await generateRepoMap(llm, ide, { + dirUris: filterDirUri ? [filterDirUri] : undefined, includeSignatures: false, - dirs: filterDirectory ? [filterDirectory] : undefined, + outputRelativeUriPaths: false, }); const prompt = `${repoMap} @@ -71,25 +73,21 @@ This is the question that you should select relevant files for: "${input}"`; return []; } - const pathSep = await ide.pathSep(); - const subDirPrefix = filterDirectory ? filterDirectory + pathSep : ""; - const files = - content - .split("")[1] - ?.split("")[0] - ?.split("\n") - .filter(Boolean) - .map((file) => file.trim()) - .map((file) => subDirPrefix + file) ?? []; + const fileUris = content + .split("")[1] + ?.split("")[0] + ?.split("\n") + .filter(Boolean) + .map((uri) => uri.trim()); const chunks = await Promise.all( - files.map(async (file) => { - const content = await ide.readFile(file); + fileUris.map(async (uri) => { + const content = await ide.readFile(uri); const lineCount = content.split("\n").length; const chunk: Chunk = { - digest: file, + digest: uri, content, - filepath: file, + filepath: uri, endLine: lineCount - 1, startLine: 0, index: 0, diff --git a/core/context/retrieval/retrieval.ts b/core/context/retrieval/retrieval.ts index e0c81fb43c..14dba33415 100644 --- a/core/context/retrieval/retrieval.ts +++ b/core/context/retrieval/retrieval.ts @@ -1,8 +1,6 @@ -import path from "path"; - import { BranchAndDir, ContextItem, ContextProviderExtras } from "../../"; import TransformersJsEmbeddingsProvider from "../../llm/llms/TransformersJsEmbeddingsProvider"; -import { resolveRelativePathInWorkspace } from "../../util/ideUtils"; +import { getUriPathBasename } from "../../util/uri"; import { INSTRUCTIONS_BASE_ITEM } from "../providers/utils"; import { RetrievalPipelineOptions } from "./pipelines/BaseRetrievalPipeline"; @@ -71,19 +69,10 @@ export async function retrieveContextItemsFromEmbeddings( ? RerankerRetrievalPipeline : NoRerankerRetrievalPipeline; - if (filterDirectory) { - // Handle relative paths - filterDirectory = await resolveRelativePathInWorkspace( - filterDirectory, - extras.ide, - ); - } - const pipelineOptions: RetrievalPipelineOptions = { nFinal, nRetrieve, tags, - pathSep: await extras.ide.pathSep(), filterDirectory, ide: extras.ide, input: extras.fullInput, @@ -100,9 +89,17 @@ export async function retrieveContextItemsFromEmbeddings( }); if (results.length === 0) { - throw new Error( - "Warning: No results found for @codebase context provider.", - ); + if (extras.config.disableIndexing) { + void extras.ide.showToast("warning", "No embeddings results found."); + return []; + } else { + void extras.ide.showToast( + "warning", + "No embeddings results found. If you think this is an error, re-index your codebase.", + ); + // TODO - add "re-index" option to warning message which clears and reindexes codebase + } + return []; } return [ @@ -114,13 +111,12 @@ export async function retrieveContextItemsFromEmbeddings( ...results .sort((a, b) => a.filepath.localeCompare(b.filepath)) .map((r) => { - const name = `${path.basename(r.filepath)} (${r.startLine}-${ - r.endLine - })`; + const basename = getUriPathBasename(r.filepath); + const name = `${basename} (${r.startLine}-${r.endLine})`; const description = `${r.filepath}`; - if (r.filepath.includes("package.json")) { - console.log(); + if (basename === "package.json") { + console.warn("Retrieval pipeline: package.json detected"); } return { diff --git a/core/core.ts b/core/core.ts index 12677446e0..fb3840d05d 100644 --- a/core/core.ts +++ b/core/core.ts @@ -30,14 +30,22 @@ import { logDevData } from "./util/devdata"; import { DevDataSqliteDb } from "./util/devdataSqlite"; import { GlobalContext } from "./util/GlobalContext"; import historyManager from "./util/history"; -import { editConfigJson, setupInitialDotContinueDirectory } from "./util/paths"; +import { + editConfigJson, + getConfigJsonPath, + setupInitialDotContinueDirectory, +} from "./util/paths"; import { Telemetry } from "./util/posthog"; import { getSymbolsForManyFiles } from "./util/treeSitter"; import { TTS } from "./util/tts"; import { type ContextItemId, type IDE, type IndexingProgressUpdate } from "."; import type { FromCoreProtocol, ToCoreProtocol } from "./protocol"; + +import { SYSTEM_PROMPT_DOT_FILE } from "./config/getSystemPromptDotFile"; import type { IMessenger, Message } from "./protocol/messenger"; +import * as URI from "uri-js"; +import { localPathToUri } from "./util/pathToUri"; export class Core { // implements IMessenger @@ -698,11 +706,6 @@ export class Core { const dirs = data?.dirs ?? (await this.ide.getWorkspaceDirs()); await this.refreshCodebaseIndex(dirs); }); - on("index/forceReIndexFiles", async ({ data }) => { - if (data?.files?.length) { - await this.refreshCodebaseIndexFiles(data.files); - } - }); on("index/setPaused", (msg) => { this.globalContext.update("indexingPaused", msg.data); this.indexingPauseToken.paused = msg.data; @@ -718,6 +721,66 @@ export class Core { } }); + // File changes + // TODO - remove remaining logic for these from IDEs where possible + on("files/changed", async ({ data }) => { + if (data?.uris?.length) { + for (const uri of data.uris) { + // Listen for file changes in the workspace + // URI TODO is this equality statement valid? + if (URI.equal(uri, localPathToUri(getConfigJsonPath()))) { + // Trigger a toast notification to provide UI feedback that config has been updated + const showToast = + this.globalContext.get("showConfigUpdateToast") ?? true; + if (showToast) { + const selection = await this.ide.showToast( + "info", + "Config updated", + "Don't show again", + ); + if (selection === "Don't show again") { + this.globalContext.update("showConfigUpdateToast", false); + } + } + } + + if ( + uri.endsWith(".continuerc.json") || + uri.endsWith(".prompt") || + uri.endsWith(SYSTEM_PROMPT_DOT_FILE) + ) { + await this.configHandler.reloadConfig(); + } else if ( + uri.endsWith(".continueignore") || + uri.endsWith(".gitignore") + ) { + // Reindex the workspaces + this.invoke("index/forceReIndex", undefined); + } else { + // Reindex the file + await this.refreshCodebaseIndexFiles([uri]); + } + } + } + }); + + on("files/created", async ({ data }) => { + if (data?.uris?.length) { + await this.refreshCodebaseIndexFiles(data.uris); + } + }); + + on("files/deleted", async ({ data }) => { + if (data?.uris?.length) { + await this.refreshCodebaseIndexFiles(data.uris); + } + }); + on("files/opened", async ({ data }) => { + if (data?.uris?.length) { + // Do something on files opened + } + }); + // Docs, etc. indexing on("indexing/reindex", async (msg) => { if (msg.data.type === "docs") { diff --git a/core/edit/lazy/applyCodeBlock.ts b/core/edit/lazy/applyCodeBlock.ts index 86b2d67b43..924d1c282a 100644 --- a/core/edit/lazy/applyCodeBlock.ts +++ b/core/edit/lazy/applyCodeBlock.ts @@ -1,14 +1,13 @@ -import path from "path"; - import { DiffLine, ILLM } from "../.."; import { generateLines } from "../../diff/util"; import { supportedLanguages } from "../../util/treeSitter"; +import { getUriFileExtension } from "../../util/uri"; import { deterministicApplyLazyEdit } from "./deterministic"; import { streamLazyApply } from "./streamLazyApply"; function canUseInstantApply(filename: string) { - const fileExtension = path.extname(filename).toLowerCase().slice(1); + const fileExtension = getUriFileExtension(filename); return supportedLanguages[fileExtension] !== undefined; } diff --git a/core/edit/lazy/deterministic.ts b/core/edit/lazy/deterministic.ts index 564656d23e..ff550c0fe0 100644 --- a/core/edit/lazy/deterministic.ts +++ b/core/edit/lazy/deterministic.ts @@ -1,5 +1,3 @@ -import path from "path"; - import { distance } from "fastest-levenshtein"; import Parser from "web-tree-sitter"; @@ -9,6 +7,10 @@ import { myersDiff } from "../../diff/myers"; import { getParserForFile } from "../../util/treeSitter"; import { findInAst } from "./findInAst"; +import { + getFileExtensionFromBasename, + getUriFileExtension, +} from "../../util/uri"; type AstReplacements = Array<{ nodeToReplace: Parser.SyntaxNode; @@ -125,8 +127,8 @@ export async function deterministicApplyLazyEdit( ); if (firstSimilarNode?.parent?.equals(oldTree.rootNode)) { // If so, we tack lazy blocks to start and end, and run the usual algorithm - const ext = path.extname(filename).slice(1); - const language = LANGUAGES[ext]; + const extension = getFileExtensionFromBasename(filename); + const language = LANGUAGES[extension]; if (language) { newLazyFile = `${language.singleLineComment} ... existing code ...\n\n${newLazyFile}\n\n${language.singleLineComment} ... existing code...`; newTree = parser.parse(newLazyFile); diff --git a/core/index.d.ts b/core/index.d.ts index 148401c147..f9e6aa0e7c 100644 --- a/core/index.d.ts +++ b/core/index.d.ts @@ -1,5 +1,6 @@ import Parser from "web-tree-sitter"; import { GetGhTokenArgs } from "./protocol/ide"; + declare global { interface Window { ide?: "vscode"; @@ -402,7 +403,7 @@ export interface PromptLog { completion: string; } -type MessageModes = "chat" | "edit"; +export type MessageModes = "chat" | "edit"; export type ToolStatus = | "generating" @@ -535,13 +536,13 @@ export interface DiffLine { line: string; } -export class Problem { +export interface Problem { filepath: string; range: Range; message: string; } -export class Thread { +export interface Thread { name: string; id: number; } @@ -605,43 +606,29 @@ export interface IDE { getAvailableThreads(): Promise; - listFolders(): Promise; - getWorkspaceDirs(): Promise; getWorkspaceConfigs(): Promise; - fileExists(filepath: string): Promise; + fileExists(fileUri: string): Promise; writeFile(path: string, contents: string): Promise; showVirtualFile(title: string, contents: string): Promise; - getContinueDir(): Promise; - openFile(path: string): Promise; openUrl(url: string): Promise; runCommand(command: string): Promise; - saveFile(filepath: string): Promise; + saveFile(fileUri: string): Promise; - readFile(filepath: string): Promise; + readFile(fileUri: string): Promise; - readRangeInFile(filepath: string, range: Range): Promise; + readRangeInFile(fileUri: string, range: Range): Promise; - showLines( - filepath: string, - startLine: number, - endLine: number, - ): Promise; - - showDiff( - filepath: string, - newContents: string, - stepIndex: number, - ): Promise; + showLines(fileUri: string, startLine: number, endLine: number): Promise; getOpenFiles(): Promise; @@ -660,7 +647,7 @@ export interface IDE { subprocess(command: string, cwd?: string): Promise<[string, string]>; - getProblems(filepath?: string | undefined): Promise; + getProblems(fileUri?: string | undefined): Promise; getBranch(dir: string): Promise; @@ -686,9 +673,7 @@ export interface IDE { gotoDefinition(location: Location): Promise; // Callbacks - onDidChangeActiveTextEditor(callback: (filepath: string) => void): void; - - pathSep(): Promise; + onDidChangeActiveTextEditor(callback: (fileUri: string) => void): void; } // Slash Commands @@ -715,7 +700,7 @@ export interface SlashCommand { // Config -type StepName = +export type StepName = | "AnswerQuestionChroma" | "GenerateShellCommandStep" | "EditHighlightedCodeStep" @@ -727,7 +712,7 @@ type StepName = | "GenerateShellCommandStep" | "DraftIssueStep"; -type ContextProviderName = +export type ContextProviderName = | "diff" | "github" | "terminal" @@ -758,7 +743,7 @@ type ContextProviderName = | "url" | string; -type TemplateType = +export type TemplateType = | "llama2" | "alpaca" | "zephyr" @@ -821,7 +806,7 @@ export interface CustomCommand { description: string; } -interface Prediction { +export interface Prediction { type: "content"; content: | string @@ -852,7 +837,7 @@ export interface Tool { uri?: string; } -interface BaseCompletionOptions { +export interface BaseCompletionOptions { temperature?: number; topP?: number; topK?: number; @@ -941,23 +926,23 @@ export interface TabAutocompleteOptions { showWhateverWeHaveAtXMs?: number; } -interface StdioOptions { +export interface StdioOptions { type: "stdio"; command: string; args: string[]; } -interface WebSocketOptions { +export interface WebSocketOptions { type: "websocket"; url: string; } -interface SSEOptions { +export interface SSEOptions { type: "sse"; url: string; } -type TransportOptions = StdioOptions | WebSocketOptions | SSEOptions; +export type TransportOptions = StdioOptions | WebSocketOptions | SSEOptions; export interface MCPOptions { transport: TransportOptions; @@ -972,7 +957,7 @@ export interface ContinueUIConfig { codeWrap?: boolean; } -interface ContextMenuConfig { +export interface ContextMenuConfig { comment?: string; docstring?: string; fix?: string; @@ -980,7 +965,7 @@ interface ContextMenuConfig { fixGrammar?: string; } -interface ModelRoles { +export interface ModelRoles { inlineEdit?: string; applyCodeBlock?: string; repoMapFileSelection?: string; @@ -1021,7 +1006,7 @@ export type CodeToEdit = RangeInFileWithContents | FileWithContents; * Represents the configuration for a quick action in the Code Lens. * Quick actions are custom commands that can be added to function and class declarations. */ -interface QuickActionConfig { +export interface QuickActionConfig { /** * The title of the quick action that will display in the Code Lens. */ @@ -1046,7 +1031,7 @@ export type DefaultContextProvider = ContextProviderWithParams & { query?: string; }; -interface ExperimentalConfig { +export interface ExperimentalConfig { contextMenuPrompts?: ContextMenuConfig; modelRoles?: ModelRoles; defaultContext?: DefaultContextProvider[]; @@ -1073,7 +1058,7 @@ interface ExperimentalConfig { modelContextProtocolServers?: MCPOptions[]; } -interface AnalyticsConfig { +export interface AnalyticsConfig { type: string; url?: string; clientKey?: string; diff --git a/core/indexing/CodeSnippetsIndex.ts b/core/indexing/CodeSnippetsIndex.ts index 0138817f60..bd1f08cf90 100644 --- a/core/indexing/CodeSnippetsIndex.ts +++ b/core/indexing/CodeSnippetsIndex.ts @@ -1,6 +1,5 @@ import Parser from "web-tree-sitter"; -import { getBasename, getLastNPathParts } from "../util/"; import { migrate } from "../util/paths"; import { getFullLanguageName, @@ -29,6 +28,7 @@ import type { IndexTag, IndexingProgressUpdate, } from "../"; +import { getLastNPathParts, getUriPathBasename } from "../util/uri"; type SnippetChunk = ChunkWithoutID & { title: string; signature: string }; @@ -190,6 +190,9 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex { const ast = parser.parse(contents); const language = getFullLanguageName(filepath); + if (!language) { + return []; + } const query = await getQueryForFile( filepath, `code-snippet-queries/${language}.scm`, @@ -250,7 +253,7 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex { } yield { - desc: `Indexing ${getBasename(compute.path)}`, + desc: `Indexing ${getUriPathBasename(compute.path)}`, progress: i / results.compute.length, status: "indexing", }; @@ -353,7 +356,7 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex { return { name: row.title, description: getLastNPathParts(row.path, 2), - content: `\`\`\`${getBasename(row.path)}\n${row.content}\n\`\`\``, + content: `\`\`\`${getUriPathBasename(row.path)}\n${row.content}\n\`\`\``, uri: { type: "file", value: row.path, @@ -390,7 +393,7 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex { offset: number = 0, batchSize: number = 100, ): Promise<{ - groupedByPath: { [path: string]: string[] }; + groupedByUri: { [path: string]: string[] }; hasMore: boolean; }> { const db = await SqliteDb.get(); @@ -411,17 +414,17 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex { const rows = await db.all(query, [...likePatterns, batchSize, offset]); - const groupedByPath: { [path: string]: string[] } = {}; + const groupedByUri: { [path: string]: string[] } = {}; for (const { path, signature } of rows) { - if (!groupedByPath[path]) { - groupedByPath[path] = []; + if (!groupedByUri[path]) { + groupedByUri[path] = []; } - groupedByPath[path].push(signature); + groupedByUri[path].push(signature); } const hasMore = rows.length === batchSize; - return { groupedByPath, hasMore }; + return { groupedByUri, hasMore }; } } diff --git a/core/indexing/CodebaseIndexer.test.ts b/core/indexing/CodebaseIndexer.test.ts index cc01625620..7db40d75ed 100644 --- a/core/indexing/CodebaseIndexer.test.ts +++ b/core/indexing/CodebaseIndexer.test.ts @@ -1,7 +1,6 @@ import { execSync } from "node:child_process"; import fs from "node:fs"; -import path from "node:path"; - +import path from "path"; import { jest } from "@jest/globals"; import { ContinueServerClient } from "../continueServer/stubs/client.js"; @@ -11,6 +10,7 @@ import { setUpTestDir, tearDownTestDir, TEST_DIR, + TEST_DIR_PATH, } from "../test/testDir.js"; import { getIndexSqlitePath } from "../util/paths.js"; @@ -19,6 +19,8 @@ import { getComputeDeleteAddRemove } from "./refreshIndex.js"; import { TestCodebaseIndex } from "./TestCodebaseIndex.js"; import { CodebaseIndex } from "./types.js"; import { walkDir } from "./walkDir.js"; +import { pathToFileURL } from "node:url"; +import { joinPathsToUri } from "../util/uri.js"; jest.useFakeTimers(); @@ -75,9 +77,11 @@ describe("CodebaseIndexer", () => { tearDownTestDir(); setUpTestDir(); - execSync("git init", { cwd: TEST_DIR }); - execSync('git config user.email "test@example.com"', { cwd: TEST_DIR }); - execSync('git config user.name "Test"', { cwd: TEST_DIR }); + execSync("git init", { cwd: TEST_DIR_PATH }); + execSync('git config user.email "test@example.com"', { + cwd: TEST_DIR_PATH, + }); + execSync('git config user.name "Test"', { cwd: TEST_DIR_PATH }); }); afterAll(async () => { @@ -155,7 +159,10 @@ describe("CodebaseIndexer", () => { }); test("should have created index folder with all necessary files", async () => { - expect(fs.existsSync(getIndexSqlitePath())).toBe(true); + const exists = await testIde.fileExists( + pathToFileURL(getIndexSqlitePath()).toString(), + ); + expect(exists).toBe(true); }); test("should have indexed all of the files", async () => { @@ -189,7 +196,7 @@ describe("CodebaseIndexer", () => { }); test("should successfully re-index after deleting a file", async () => { - fs.rmSync(path.join(TEST_DIR, "main.rs")); + fs.rmSync(path.join(TEST_DIR_PATH, "main.rs")); await expectPlan(0, 0, 0, 1); @@ -210,23 +217,25 @@ describe("CodebaseIndexer", () => { test("should create git repo for testing", async () => { execSync( - `cd ${TEST_DIR} && git init && git checkout -b main && git add -A && git commit -m "First commit"`, + `cd ${TEST_DIR_PATH} && git init && git checkout -b main && git add -A && git commit -m "First commit"`, ); }); test.skip("should only re-index the changed files when changing branches", async () => { - execSync(`cd ${TEST_DIR} && git checkout -b test2`); + execSync(`cd ${TEST_DIR_PATH} && git checkout -b test2`); // Rewriting the file addToTestDir([["test.ts", "// This is different"]]); // Should re-compute test.ts, but just re-tag the .py file await expectPlan(1, 1, 0, 0); - execSync(`cd ${TEST_DIR} && git add -A && git commit -m "Change .ts file"`); + execSync( + `cd ${TEST_DIR_PATH} && git add -A && git commit -m "Change .ts file"`, + ); }); test.skip("shouldn't re-index anything when changing back to original branch", async () => { - execSync(`cd ${TEST_DIR} && git checkout main`); + execSync(`cd ${TEST_DIR_PATH} && git checkout main`); await expectPlan(0, 0, 0, 0); }); }); diff --git a/core/indexing/CodebaseIndexer.ts b/core/indexing/CodebaseIndexer.ts index 4b2138d5ac..4e8355c9be 100644 --- a/core/indexing/CodebaseIndexer.ts +++ b/core/indexing/CodebaseIndexer.ts @@ -17,6 +17,7 @@ import { RefreshIndexResults, } from "./types.js"; import { walkDirAsync } from "./walkDir.js"; +import { findUriInDirs, getUriPathBasename } from "../util/uri.js"; export class PauseToken { constructor(private _paused: boolean) {} @@ -75,19 +76,15 @@ export class CodebaseIndexer { protected async getIndexesToBuild(): Promise { const config = await this.configHandler.loadConfig(); - const pathSep = await this.ide.pathSep(); - const indexes = [ new ChunkCodebaseIndex( this.ide.readFile.bind(this.ide), - pathSep, this.continueServerClient, config.embeddingsProvider.maxEmbeddingChunkSize, ), // Chunking must come first new LanceDbIndex( config.embeddingsProvider, this.ide.readFile.bind(this.ide), - pathSep, this.continueServerClient, ), new FullTextSearchCodebaseIndex(), @@ -97,23 +94,26 @@ export class CodebaseIndexer { return indexes; } - public async refreshFile(file: string): Promise { + public async refreshFile( + file: string, + workspaceDirs: string[], + ): Promise { if (this.pauseToken.paused) { // NOTE: by returning here, there is a chance that while paused a file is modified and // then after unpausing the file is not reindexed return; } - const workspaceDir = await this.getWorkspaceDir(file); - if (!workspaceDir) { + const { foundInDir } = findUriInDirs(file, workspaceDirs); + if (!foundInDir) { return; } - const branch = await this.ide.getBranch(workspaceDir); - const repoName = await this.ide.getRepoName(workspaceDir); + const branch = await this.ide.getBranch(foundInDir); + const repoName = await this.ide.getRepoName(foundInDir); const indexesToBuild = await this.getIndexesToBuild(); const stats = await this.ide.getLastModified([file]); for (const index of indexesToBuild) { const tag = { - directory: workspaceDir, + directory: foundInDir, branch, artifactId: index.artifactId, }; @@ -154,6 +154,8 @@ export class CodebaseIndexer { }; } + const workspaceDirs = await this.ide.getWorkspaceDirs(); + const progressPer = 1 / files.length; try { for (const file of files) { @@ -162,7 +164,7 @@ export class CodebaseIndexer { desc: `Indexing file ${file}...`, status: "indexing", }; - await this.refreshFile(file); + await this.refreshFile(file, workspaceDirs); progress += progressPer; @@ -224,7 +226,7 @@ export class CodebaseIndexer { const beginTime = Date.now(); for (const directory of dirs) { - const dirBasename = await this.basename(directory); + const dirBasename = getUriPathBasename(directory); yield { progress, desc: `Discovering files in ${dirBasename}...`, @@ -446,20 +448,4 @@ export class CodebaseIndexer { completedIndexCount += 1; } } - - private async getWorkspaceDir(filepath: string): Promise { - const workspaceDirs = await this.ide.getWorkspaceDirs(); - for (const workspaceDir of workspaceDirs) { - if (filepath.startsWith(workspaceDir)) { - return workspaceDir; - } - } - return undefined; - } - - private async basename(filepath: string): Promise { - const pathSep = await this.ide.pathSep(); - const path = filepath.split(pathSep); - return path[path.length - 1]; - } } diff --git a/core/indexing/FullTextSearchCodebaseIndex.ts b/core/indexing/FullTextSearchCodebaseIndex.ts index 82e4e822f6..7ff54781c8 100644 --- a/core/indexing/FullTextSearchCodebaseIndex.ts +++ b/core/indexing/FullTextSearchCodebaseIndex.ts @@ -1,6 +1,6 @@ import { BranchAndDir, Chunk, IndexTag, IndexingProgressUpdate } from "../"; -import { getBasename } from "../util/index"; import { RETRIEVAL_PARAMS } from "../util/parameters"; +import { getUriPathBasename } from "../util/uri"; import { ChunkCodebaseIndex } from "./chunk/ChunkCodebaseIndex"; import { DatabaseConnection, SqliteDb, tagToString } from "./refreshIndex"; @@ -79,7 +79,7 @@ export class FullTextSearchCodebaseIndex implements CodebaseIndex { yield { progress: i / results.compute.length, - desc: `Indexing ${getBasename(item.path)}`, + desc: `Indexing ${getUriPathBasename(item.path)}`, status: "indexing", }; await markComplete([item], IndexResultType.Compute); diff --git a/core/indexing/LanceDbIndex.test.ts b/core/indexing/LanceDbIndex.test.ts index 1d15117fdb..6cf36f78da 100644 --- a/core/indexing/LanceDbIndex.test.ts +++ b/core/indexing/LanceDbIndex.test.ts @@ -26,13 +26,11 @@ describe.skip("ChunkCodebaseIndex", () => { } beforeAll(async () => { - const pathSep = await testIde.pathSep(); const mockConfig = await testConfigHandler.loadConfig(); index = new LanceDbIndex( mockConfig.embeddingsProvider, testIde.readFile.bind(testIde), - pathSep, testContinueServerClient, ); diff --git a/core/indexing/LanceDbIndex.ts b/core/indexing/LanceDbIndex.ts index 89bcb2099c..d11feb1d1d 100644 --- a/core/indexing/LanceDbIndex.ts +++ b/core/indexing/LanceDbIndex.ts @@ -11,7 +11,6 @@ import { IndexTag, IndexingProgressUpdate, } from "../index.js"; -import { getBasename } from "../util/index.js"; import { getLanceDbPath, migrate } from "../util/paths.js"; import { chunkDocument, shouldChunk } from "./chunk/chunk.js"; @@ -23,6 +22,7 @@ import { PathAndCacheKey, RefreshIndexResults, } from "./types.js"; +import { getUriPathBasename } from "../util/uri.js"; // LanceDB converts to lowercase, so names must all be lowercase interface LanceDbRow { @@ -46,7 +46,6 @@ export class LanceDbIndex implements CodebaseIndex { constructor( private readonly embeddingsProvider: ILLM, private readonly readFile: (filepath: string) => Promise, - private readonly pathSep: string, private readonly continueServerClient?: IContinueServerClient, ) {} @@ -125,7 +124,7 @@ export class LanceDbIndex implements CodebaseIndex { try { const content = await this.readFile(item.path); - if (!shouldChunk(this.pathSep, item.path, content)) { + if (!shouldChunk(item.path, content)) { continue; } @@ -350,7 +349,7 @@ export class LanceDbIndex implements CodebaseIndex { accumulatedProgress += 1 / results.addTag.length / 3; yield { progress: accumulatedProgress, - desc: `Indexing ${getBasename(path)}`, + desc: `Indexing ${getUriPathBasename(path)}`, status: "indexing", }; } @@ -372,7 +371,7 @@ export class LanceDbIndex implements CodebaseIndex { accumulatedProgress += 1 / toDel.length / 3; yield { progress: accumulatedProgress, - desc: `Stashing ${getBasename(path)}`, + desc: `Stashing ${getUriPathBasename(path)}`, status: "indexing", }; } @@ -391,7 +390,7 @@ export class LanceDbIndex implements CodebaseIndex { accumulatedProgress += 1 / results.del.length / 3; yield { progress: accumulatedProgress, - desc: `Removing ${getBasename(path)}`, + desc: `Removing ${getUriPathBasename(path)}`, status: "indexing", }; } diff --git a/core/indexing/chunk/ChunkCodebaseIndex.test.ts b/core/indexing/chunk/ChunkCodebaseIndex.test.ts index 3f7904991c..ea636dc544 100644 --- a/core/indexing/chunk/ChunkCodebaseIndex.test.ts +++ b/core/indexing/chunk/ChunkCodebaseIndex.test.ts @@ -29,11 +29,8 @@ describe("ChunkCodebaseIndex", () => { } beforeAll(async () => { - const pathSep = await testIde.pathSep(); - index = new ChunkCodebaseIndex( testIde.readFile.bind(testIde), - pathSep, testContinueServerClient, 1000, ); diff --git a/core/indexing/chunk/ChunkCodebaseIndex.ts b/core/indexing/chunk/ChunkCodebaseIndex.ts index 7bd856ff1e..1f6a452589 100644 --- a/core/indexing/chunk/ChunkCodebaseIndex.ts +++ b/core/indexing/chunk/ChunkCodebaseIndex.ts @@ -4,7 +4,6 @@ import { RunResult } from "sqlite3"; import { IContinueServerClient } from "../../continueServer/interface.js"; import { Chunk, IndexTag, IndexingProgressUpdate } from "../../index.js"; -import { getBasename } from "../../util/index.js"; import { DatabaseConnection, SqliteDb, tagToString } from "../refreshIndex.js"; import { IndexResultType, @@ -15,6 +14,7 @@ import { } from "../types.js"; import { chunkDocument, shouldChunk } from "./chunk.js"; +import { getUriPathBasename } from "../../util/uri.js"; export class ChunkCodebaseIndex implements CodebaseIndex { relativeExpectedTime: number = 1; @@ -23,7 +23,6 @@ export class ChunkCodebaseIndex implements CodebaseIndex { constructor( private readonly readFile: (filepath: string) => Promise, - private readonly pathSep: string, private readonly continueServerClient: IContinueServerClient, private readonly maxChunkSize: number, ) {} @@ -89,7 +88,7 @@ export class ChunkCodebaseIndex implements CodebaseIndex { accumulatedProgress += 1 / results.addTag.length / 4; yield { progress: accumulatedProgress, - desc: `Adding ${getBasename(item.path)}`, + desc: `Adding ${getUriPathBasename(item.path)}`, status: "indexing", }; } @@ -111,7 +110,7 @@ export class ChunkCodebaseIndex implements CodebaseIndex { accumulatedProgress += 1 / results.removeTag.length / 4; yield { progress: accumulatedProgress, - desc: `Removing ${getBasename(item.path)}`, + desc: `Removing ${getUriPathBasename(item.path)}`, status: "indexing", }; } @@ -138,7 +137,7 @@ export class ChunkCodebaseIndex implements CodebaseIndex { accumulatedProgress += 1 / results.del.length / 4; yield { progress: accumulatedProgress, - desc: `Removing ${getBasename(item.path)}`, + desc: `Removing ${getUriPathBasename(item.path)}`, status: "indexing", }; } @@ -165,7 +164,7 @@ export class ChunkCodebaseIndex implements CodebaseIndex { private async packToChunks(pack: PathAndCacheKey): Promise { const contents = await this.readFile(pack.path); - if (!shouldChunk(this.pathSep, pack.path, contents)) { + if (!shouldChunk(pack.path, contents)) { return []; } const chunks: Chunk[] = []; diff --git a/core/indexing/chunk/chunk.test.ts b/core/indexing/chunk/chunk.test.ts index 85683a51e8..7a8ce6f76d 100644 --- a/core/indexing/chunk/chunk.test.ts +++ b/core/indexing/chunk/chunk.test.ts @@ -6,25 +6,25 @@ describe("shouldChunk", () => { test("should chunk a typescript file", () => { const filePath = path.join("directory", "file.ts"); const fileContent = generateString(10000); - expect(shouldChunk(path.sep, filePath, fileContent)).toBe(true); + expect(shouldChunk(filePath, fileContent)).toBe(true); }); test("should not chunk a large typescript file", () => { const filePath = path.join("directory", "file.ts"); const fileContent = generateString(1500000); - expect(shouldChunk(path.sep, filePath, fileContent)).toBe(false); + expect(shouldChunk(filePath, fileContent)).toBe(false); }); test("should not chunk an empty file", () => { const filePath = path.join("directory", "file.ts"); const fileContent = generateString(0); - expect(shouldChunk(path.sep, filePath, fileContent)).toBe(false); + expect(shouldChunk(filePath, fileContent)).toBe(false); }); test("should not chunk a file without extension", () => { const filePath = path.join("directory", "with.dot", "filename"); const fileContent = generateString(10000); - expect(shouldChunk(path.sep, filePath, fileContent)).toBe(false); + expect(shouldChunk(filePath, fileContent)).toBe(false); }); }); diff --git a/core/indexing/chunk/chunk.ts b/core/indexing/chunk/chunk.ts index 596f116dc3..20a2a19b00 100644 --- a/core/indexing/chunk/chunk.ts +++ b/core/indexing/chunk/chunk.ts @@ -3,6 +3,7 @@ import { countTokensAsync } from "../../llm/countTokens.js"; import { extractMinimalStackTraceInfo } from "../../util/extractMinimalStackTraceInfo.js"; import { Telemetry } from "../../util/posthog.js"; import { supportedLanguages } from "../../util/treeSitter.js"; +import { getUriFileExtension, getUriPathBasename } from "../../util/uri.js"; import { basicChunker } from "./basic.js"; import { codeChunker } from "./code.js"; @@ -15,25 +16,23 @@ export type ChunkDocumentParam = { }; async function* chunkDocumentWithoutId( - filepath: string, + fileUri: string, contents: string, maxChunkSize: number, ): AsyncGenerator { if (contents.trim() === "") { return; } - - const segs = filepath.split("."); - const ext = segs[segs.length - 1]; - if (ext in supportedLanguages) { + const extension = getUriFileExtension(fileUri); + if (extension in supportedLanguages) { try { - for await (const chunk of codeChunker(filepath, contents, maxChunkSize)) { + for await (const chunk of codeChunker(fileUri, contents, maxChunkSize)) { yield chunk; } return; } catch (e: any) { - Telemetry.capture("code_chunker_error", { - fileExtension: ext, + void Telemetry.capture("code_chunker_error", { + fileExtension: extension, stack: extractMinimalStackTraceInfo(e.stack), }); // falls back to basicChunker @@ -84,11 +83,7 @@ export async function* chunkDocument({ } } -export function shouldChunk( - pathSep: string, - filepath: string, - contents: string, -): boolean { +export function shouldChunk(fileUri: string, contents: string): boolean { if (contents.length > 1000000) { // if a file has more than 1m characters then skip it return false; @@ -96,7 +91,6 @@ export function shouldChunk( if (contents.length === 0) { return false; } - const basename = filepath.split(pathSep).pop(); - // files without extensions are often binary files, skip it if so - return basename?.includes(".") ?? false; + const baseName = getUriPathBasename(fileUri); + return baseName.includes("."); } diff --git a/core/indexing/docs/article.ts b/core/indexing/docs/article.ts index 2ae0e79953..8026280c9c 100644 --- a/core/indexing/docs/article.ts +++ b/core/indexing/docs/article.ts @@ -31,6 +31,11 @@ function breakdownArticleComponent( let content = ""; let index = 0; + const fullUrl = new URL( + `${subpath}#${cleanFragment(article.title)}`, + url, + ).toString(); + const createChunk = ( chunkContent: string, chunkStartLine: number, @@ -44,11 +49,8 @@ function breakdownArticleComponent( title: cleanHeader(article.title), }, index: index++, - filepath: new URL( - `${subpath}#${cleanFragment(article.title)}`, - url, - ).toString(), - digest: subpath, + filepath: fullUrl, + digest: fullUrl, }); }; diff --git a/core/indexing/docs/preIndexedDocs.ts b/core/indexing/docs/preIndexedDocs.ts index 93eaa84111..1e2a8dbc0d 100644 --- a/core/indexing/docs/preIndexedDocs.ts +++ b/core/indexing/docs/preIndexedDocs.ts @@ -348,6 +348,11 @@ const preIndexedDocs: Record< rootUrl: "https://api.dart.dev", faviconUrl: "https://api.dart.dev/static-assets/favicon.png", }, + "https://docs.asksage.ai/": { + title: "Ask Sage", + startUrl: "https://docs.asksage.ai/", + rootUrl: "https://docs.asksage.ai/", + }, }; export default preIndexedDocs; diff --git a/core/indexing/docs/suggestions/index.ts b/core/indexing/docs/suggestions/index.ts index db192489ad..fac5091fd8 100644 --- a/core/indexing/docs/suggestions/index.ts +++ b/core/indexing/docs/suggestions/index.ts @@ -6,7 +6,8 @@ import { PackageDetails, ParsedPackageInfo, } from "../../.."; -import { walkDir } from "../../walkDir"; +import { getUriPathBasename } from "../../../util/uri"; +import { walkDir, walkDirs } from "../../walkDir"; import { PythonPackageCrawler } from "./packageCrawlers/Python"; import { NodePackageCrawler } from "./packageCrawlers/TsJs"; @@ -24,16 +25,10 @@ export interface PackageCrawler { } export async function getAllSuggestedDocs(ide: IDE) { - const workspaceDirs = await ide.getWorkspaceDirs(); - const results = await Promise.all( - workspaceDirs.map((dir) => { - return walkDir(dir, ide); - }), - ); - const allPaths = results.flat(); // TODO only get files, not dirs. Not critical for now - const allFiles = allPaths.map((path) => ({ - path, - name: path.split(/[\\/]/).pop()!, + const allFileUris = await walkDirs(ide); + const allFiles = allFileUris.map((uri) => ({ + path: uri, + name: getUriPathBasename(uri), })); // Build map of language -> package files diff --git a/core/indexing/test/indexing.ts b/core/indexing/test/indexing.ts index 413c37e754..5f22d0aeb9 100644 --- a/core/indexing/test/indexing.ts +++ b/core/indexing/test/indexing.ts @@ -4,7 +4,7 @@ import { IndexTag } from "../.."; import { IContinueServerClient } from "../../continueServer/interface"; import { ChunkCodebaseIndex } from "../chunk/ChunkCodebaseIndex"; import { tagToString } from "../refreshIndex"; -import { CodebaseIndex, PathAndCacheKey, RefreshIndexResults } from "../types"; +import { CodebaseIndex, RefreshIndexResults } from "../types"; import { testIde } from "../../test/fixtures"; import { addToTestDir, TEST_DIR } from "../../test/testDir"; @@ -54,11 +54,8 @@ const mockMarkComplete = jest .mockImplementation(() => Promise.resolve()) as any; export async function insertMockChunks() { - const pathSep = await testIde.pathSep(); - const index = new ChunkCodebaseIndex( testIde.readFile.bind(testIde), - pathSep, mockContinueServerClient, 1000, ); diff --git a/core/indexing/walkDir.test.ts b/core/indexing/walkDir.test.ts index 25ffdfc374..8a9d26e7e3 100644 --- a/core/indexing/walkDir.test.ts +++ b/core/indexing/walkDir.test.ts @@ -1,360 +1,357 @@ -import path from "path"; - -import { walkDir, WalkerOptions } from "../indexing/walkDir"; +// Generated by continue +import { testIde } from "../test/fixtures"; import { - TEST_DIR, setUpTestDir, tearDownTestDir, addToTestDir, + TEST_DIR_PATH, } from "../test/testDir"; -import FileSystemIde from "../util/filesystem"; - -const ide = new FileSystemIde(TEST_DIR); +import { walkDir, walkDirAsync, walkDirs } from "../indexing/walkDir"; +import fs from "fs"; +import path from "path"; -async function walkTestDir( - options?: WalkerOptions, -): Promise { - return walkDir(TEST_DIR, ide, { - returnRelativePaths: true, - ...options, - }); -} - -async function expectPaths( - toExist: string[], - toNotExist: string[], - options?: WalkerOptions, -) { - // Convert to Windows paths - const pathSep = await ide.pathSep(); - if (pathSep === "\\") { - toExist = toExist.map((p) => p.replace(/\//g, "\\")); - toNotExist = toNotExist.map((p) => p.replace(/\//g, "\\")); - } - - const result = await walkTestDir(options); - - for (const p of toExist) { - expect(result).toContain(p); - } - for (const p of toNotExist) { - expect(result).not.toContain(p); - } -} - -describe("walkDir", () => { - beforeEach(() => { +describe("walkDir functions", () => { + beforeEach(async () => { setUpTestDir(); }); - afterEach(() => { + afterEach(async () => { tearDownTestDir(); }); - test("should return nothing for empty dir", async () => { - const result = await walkTestDir(); - expect(result).toEqual([]); - }); - - test("should return all files in flat dir", async () => { - const files = ["a.txt", "b.py", "c.ts"]; - addToTestDir(files); - const result = await walkTestDir(); - expect(result).toEqual(files); - }); - - test("should ignore ignored files in flat dir", async () => { - const files = [[".gitignore", "*.py"], "a.txt", "c.ts", "b.py"]; - addToTestDir(files); - await expectPaths(["a.txt", "c.ts"], ["b.py"]); - }); - - test("should handle negation in flat folder", async () => { - const files = [[".gitignore", "**/*\n!*.py"], "a.txt", "c.ts", "b.py"]; - addToTestDir(files); - await expectPaths(["b.py"], [".gitignore", "a.txt", "c.ts"]); - }); - - test("should get all files in nested folder structure", async () => { - const files = [ - "a.txt", - "b.py", - "c.ts", - "d/", - "d/e.txt", - "d/f.py", - "d/g/", - "d/g/h.ts", - ]; - addToTestDir(files); - await expectPaths( - files.filter((files) => !files.endsWith("/")), - [], - ); - }); - - test("should ignore ignored files in nested folder structure", async () => { - const files = [ - "a.txt", - "b.py", - "c.ts", - "d/", - "d/e.txt", - "d/f.py", - "d/g/", - "d/g/h.ts", - ["d/.gitignore", "*.py"], - ]; - addToTestDir(files); - await expectPaths( - ["a.txt", "b.py", "c.ts", "d/e.txt", "d/g/h.ts"], - ["d/f.py"], - ); - }); - - test("should use gitignore in parent directory for subdirectory", async () => { - const files = [ - "a.txt", - "b.py", - "d/", - "d/e.txt", - "d/f.py", - "d/g/", - "d/g/h.ts", - "d/g/i.py", - [".gitignore", "*.py"], - ]; - addToTestDir(files); - await expectPaths(["a.txt", "d/e.txt", "d/g/h.ts"], ["d/f.py", "d/g/i.py"]); - }); - - test("should handle leading slash in gitignore", async () => { - const files = [[".gitignore", "/no.txt"], "a.txt", "b.py", "no.txt"]; - addToTestDir(files); - await expectPaths(["a.txt", "b.py"], ["no.txt"]); - }); + describe("walkDir", () => { + it("should walk a simple directory structure", async () => { + addToTestDir([ + "src/", + ["src/file1.ts", "content1"], + ["src/file2.ts", "content2"], + "src/nested/", + ["src/nested/file3.ts", "content3"], + ]); + + const files = await walkDir( + (await testIde.getWorkspaceDirs())[0], + testIde, + ); + + expect(files.sort()).toEqual( + [ + expect.stringContaining("src/file1.ts"), + expect.stringContaining("src/file2.ts"), + expect.stringContaining("src/nested/file3.ts"), + ].sort(), + ); + }); - test("should not ignore leading slash when in subfolder", async () => { - const files = [ - [".gitignore", "/no.txt"], - "a.txt", - "b.py", - "no.txt", - "sub/", - "sub/no.txt", - ]; - addToTestDir(files); - await expectPaths(["a.txt", "b.py", "sub/no.txt"], ["no.txt"]); - }); + it("should respect .gitignore", async () => { + addToTestDir([ + ["src/file1.ts", "content1"], + ["src/file2.ts", "content2"], + "node_modules/", + ["node_modules/pkg/file.js", "ignored"], + [".gitignore", "node_modules/"], + ]); + + const files = await walkDir( + (await testIde.getWorkspaceDirs())[0], + testIde, + ); + + expect(files).toEqual( + expect.not.arrayContaining([ + expect.stringContaining("node_modules/pkg/file.js"), + ]), + ); + expect(files.sort()).toEqual( + [ + expect.stringContaining("src/file1.ts"), + expect.stringContaining("src/file2.ts"), + ].sort(), + ); + }); - test("should handle multiple .gitignore files in nested structure", async () => { - const files = [ - [".gitignore", "*.txt"], - "a.py", - "b.txt", - "c/", - "c/d.txt", - "c/e.py", - ["c/.gitignore", "*.py"], - ]; - addToTestDir(files); - await expectPaths(["a.py"], ["b.txt", "c/e.py", "c/d.txt"]); + it("should handle onlyDirs option", async () => { + addToTestDir([ + "src/", + ["src/file1.ts", "content1"], + "src/nested/", + ["src/nested/file2.ts", "content2"], + ]); + + const dirs = await walkDir( + (await testIde.getWorkspaceDirs())[0], + testIde, + { + onlyDirs: true, + }, + ); + + expect(dirs.sort()).toEqual( + [ + expect.stringContaining("src"), + expect.stringContaining("src/nested"), + ].sort(), + ); + }); }); - test("should handle wildcards in .gitignore", async () => { - const files = [ - [".gitignore", "*.txt\n*.py"], - "a.txt", - "b.py", - "c.ts", - "d/", - "d/e.txt", - "d/f.py", - "d/g.ts", - ]; - addToTestDir(files); - await expectPaths( - ["c.ts", "d/g.ts"], - ["a.txt", "b.py", "d/e.txt", "d/f.py"], - ); - }); + describe("walkDirAsync", () => { + it("should yield files asynchronously", async () => { + addToTestDir([ + "src/", + ["src/file1.ts", "content1"], + ["src/file2.ts", "content2"], + ]); + + const files: string[] = []; + for await (const file of walkDirAsync( + (await testIde.getWorkspaceDirs())[0], + testIde, + )) { + files.push(file); + } + + expect(files.sort()).toEqual( + [ + expect.stringContaining("src/file1.ts"), + expect.stringContaining("src/file2.ts"), + ].sort(), + ); + }); - test("should handle directory ignores in .gitignore", async () => { - const files = [ - [".gitignore", "ignored_dir/"], - "a.txt", - "ignored_dir/", - "ignored_dir/b.txt", - "ignored_dir/c/", - "ignored_dir/c/d.py", - ]; - addToTestDir(files); - await expectPaths(["a.txt"], ["ignored_dir/b.txt", "ignored_dir/c/d.py"]); + it("should handle relative paths option", async () => { + addToTestDir([ + "src/", + ["src/file1.ts", "content1"], + ["src/file2.ts", "content2"], + ]); + + const files: string[] = []; + for await (const file of walkDirAsync( + (await testIde.getWorkspaceDirs())[0], + testIde, + { + returnRelativeUrisPaths: true, + }, + )) { + files.push(file); + } + + expect(files.sort()).toEqual(["src/file1.ts", "src/file2.ts"].sort()); + }); }); - test("gitignore in sub directory should only apply to subdirectory", async () => { - const files = [ - [".gitignore", "abc"], - "a.txt", - "abc", - "xyz/", - ["xyz/.gitignore", "xyz"], - "xyz/b.txt", - "xyz/c/", - "xyz/c/d.py", - "xyz/xyz", - ]; - addToTestDir(files); - await expectPaths(["a.txt", "xyz/b.txt", "xyz/c/d.py"], ["abc", "xyz/xyz"]); - }); + describe("walkDirs", () => { + it("should walk multiple workspace directories", async () => { + addToTestDir([ + "workspace1/src/", + ["workspace1/src/file1.ts", "content1"], + "workspace2/src/", + ["workspace2/src/file2.ts", "content2"], + ]); + + const files = await walkDirs(testIde, undefined, [ + (await testIde.getWorkspaceDirs())[0] + "/workspace1", + (await testIde.getWorkspaceDirs())[0] + "/workspace2", // Second workspace dir + ]); + + expect(files).toContainEqual(expect.stringContaining("file1.ts")); + expect(files).toContainEqual(expect.stringContaining("file2.ts")); + }); - test("should handle complex patterns in .gitignore", async () => { - const files = [ - [".gitignore", "*.what\n!important.what\ntemp/\n/root_only.txt"], - "a.what", - "important.what", - "root_only.txt", - "subdir/", - "subdir/root_only.txt", - "subdir/b.what", - "temp/", - "temp/c.txt", - ]; - addToTestDir(files); - await expectPaths( - ["important.what", "subdir/root_only.txt"], - ["a.what", "root_only.txt", "subdir/b.what", "temp/c.txt"], - ); - }); + it("should handle empty directories", async () => { + addToTestDir(["empty/"]); - test("should listen to both .gitignore and .continueignore", async () => { - const files = [ - [".gitignore", "*.py"], - [".continueignore", "*.ts"], - "a.txt", - "b.py", - "c.ts", - "d.js", - ]; - addToTestDir(files); - await expectPaths(["a.txt", "d.js"], ["b.py", "c.ts"]); - }); + const files = await walkDirs(testIde); + console.log("EMPTY DIR", files); - test("should return dirs and only dirs in onlyDirs mode", async () => { - const files = [ - "a.txt", - "b.py", - "c.ts", - "d/", - "d/e.txt", - "d/f.py", - "d/g/", - "d/g/h.ts", - ]; - addToTestDir(files); - await expectPaths( - ["d", "d/g"], - ["a.txt", "b.py", "c.ts", "d/e.txt", "d/f.py", "d/g/h.ts"], - { onlyDirs: true }, - ); - }); + expect(files).toEqual([]); + }); - test("should return valid paths in absolute path mode", async () => { - const files = ["a.txt", "b/", "b/c.txt"]; - addToTestDir(files); - await expectPaths( - [path.join(TEST_DIR, "a.txt"), path.join(TEST_DIR, "b", "c.txt")], - [], - { - returnRelativePaths: false, - }, - ); - }); + it("should skip symlinks", async () => { + const filePath = path.join(TEST_DIR_PATH, "real.ts"); + addToTestDir([["real.ts", "content"]]); + fs.symlink( + filePath, + path.join(TEST_DIR_PATH, "symlink.ts"), + "file", + () => {}, + ); - test("should skip .git and node_modules folders", async () => { - const files = [ - "a.txt", - ".git/", - ".git/config", - ".git/HEAD", - ".git/objects/", - ".git/objects/1234567890abcdef", - "node_modules/", - "node_modules/package/", - "node_modules/package/index.js", - "src/", - "src/index.ts", - ]; - addToTestDir(files); - await expectPaths( - ["a.txt", "src/index.ts"], - [ - ".git/config", - ".git/HEAD", - "node_modules/package/index.js", - ".git/objects/1234567890abcdef", - ], - ); - }); + const files = await walkDirs(testIde); - test("should walk continue repo without getting any files of the default ignore types", async () => { - const results = await walkDir(path.join(__dirname, ".."), ide, { - ignoreFiles: [".gitignore", ".continueignore"], + expect(files).not.toContainEqual(expect.stringContaining("symlink.ts")); }); - expect(results.length).toBeGreaterThan(0); - expect(results.some((file) => file.includes("/node_modules/"))).toBe(false); - expect(results.some((file) => file.includes("/.git/"))).toBe(false); - expect( - results.some( - (file) => - file.endsWith(".gitignore") || - file.endsWith(".continueignore") || - file.endsWith("package-lock.json"), - ), - ).toBe(false); - // At some point we will cross this number, but in case we leap past it suddenly I think we'd want to investigate why - expect(results.length).toBeLessThan(1500); }); - // This test is passing when this file is ran individually, but failing with `directory not found` error - // when the full test suite is ran - test.skip("should walk continue/extensions/vscode without getting any files in the .continueignore", async () => { - const vscodePath = path.join(__dirname, "../..", "extensions", "vscode"); - const results = await walkDir(vscodePath, ide, { - ignoreFiles: [".gitignore", ".continueignore"], + describe("walkDir ignore patterns", () => { + it("should handle negation patterns in gitignore", async () => { + addToTestDir([ + [".gitignore", "**/*\n!*.py"], + ["a.txt", "content"], + ["b.py", "content"], + ["c.ts", "content"], + ]); + + const files = await walkDir( + (await testIde.getWorkspaceDirs())[0], + testIde, + { + returnRelativeUrisPaths: true, + }, + ); + + expect(files).toContain("b.py"); + expect(files).not.toContain("a.txt"); + expect(files).not.toContain("c.ts"); }); - expect(results.length).toBeGreaterThan(0); - expect(results.some((file) => file.includes("/textmate-syntaxes/"))).toBe( - false, - ); - expect(results.some((file) => file.includes(".tmLanguage"))).toBe(false); - }); - // This test is passing when this file is ran individually, but failing with `jest not found` error - // when the full test suite is ran - test.skip("should perform the same number of dir reads as 1 + the number of dirs that contain files", async () => { - const files = [ - "a.txt", - "b.py", - "c.ts", - "d/", - "d/e.txt", - "d/f.py", - "d/g/", - "d/g/h.ts", - "d/g/i/", - "d/g/i/j.ts", - ]; + it("should handle leading slash patterns", async () => { + addToTestDir([ + [".gitignore", "/no.txt"], + ["no.txt", "content"], + "sub/", + ["sub/no.txt", "content"], + ["a.txt", "content"], + ]); + + const files = await walkDir( + (await testIde.getWorkspaceDirs())[0], + testIde, + { + returnRelativeUrisPaths: true, + }, + ); + + expect(files).not.toContain("no.txt"); + expect(files).toContain("sub/no.txt"); + expect(files).toContain("a.txt"); + }); - const numDirs = files.filter((file) => !file.includes(".")).length; - const numDirsPlusTopLevelRead = numDirs + 1; + it("should handle multiple gitignore files in nested structure", async () => { + addToTestDir([ + [".gitignore", "*.txt"], + ["a.py", "content"], + ["b.txt", "content"], + "c/", + ["c/.gitignore", "*.py"], + ["c/d.txt", "content"], + ["c/e.py", "content"], + ]); + + const files = await walkDir( + (await testIde.getWorkspaceDirs())[0], + testIde, + { + returnRelativeUrisPaths: true, + }, + ); + + expect(files).toContain("a.py"); + expect(files).not.toContain("b.txt"); + expect(files).not.toContain("c/d.txt"); + expect(files).not.toContain("c/e.py"); + }); - addToTestDir(files); + it("should handle both gitignore and continueignore", async () => { + addToTestDir([ + [".gitignore", "*.py"], + [".continueignore", "*.ts"], + ["a.txt", "content"], + ["b.py", "content"], + ["c.ts", "content"], + ["d.js", "content"], + ]); + + const files = await walkDir( + (await testIde.getWorkspaceDirs())[0], + testIde, + { + returnRelativeUrisPaths: true, + }, + ); + + expect(files).toContain("a.txt"); + expect(files).toContain("d.js"); + expect(files).not.toContain("b.py"); + expect(files).not.toContain("c.ts"); + }); - const mockListDir = jest.spyOn(ide, "listDir"); + it("should handle complex wildcard patterns", async () => { + addToTestDir([ + [".gitignore", "*.what\n!important.what\ntemp/\n/root_only.txt"], + ["a.what", "content"], + ["important.what", "content"], + ["root_only.txt", "content"], + "subdir/", + ["subdir/root_only.txt", "content"], + ["subdir/b.what", "content"], + "temp/", + ["temp/c.txt", "content"], + ]); + + const files = await walkDir( + (await testIde.getWorkspaceDirs())[0], + testIde, + { + returnRelativeUrisPaths: true, + }, + ); + + expect(files).toContain("important.what"); + expect(files).toContain("subdir/root_only.txt"); + expect(files).not.toContain("a.what"); + expect(files).not.toContain("root_only.txt"); + expect(files).not.toContain("subdir/b.what"); + expect(files).not.toContain("temp/c.txt"); + }); - await walkTestDir(); + it("should skip common system directories by default", async () => { + addToTestDir([ + ["normal/file.txt", "content"], + [".git/config", "content"], + ["node_modules/package/index.js", "content"], + ["dist/bundle.js", "content"], + ["coverage/lcov.env", "content"], + ]); + + const files = await walkDir( + (await testIde.getWorkspaceDirs())[0], + testIde, + { + returnRelativeUrisPaths: true, + }, + ); + + expect(files).toContain("normal/file.txt"); + expect(files).not.toContain(".git/config"); + expect(files).not.toContain("node_modules/package/index.js"); + expect(files).not.toContain("coverage/lcov.info"); + }); - expect(mockListDir).toHaveBeenCalledTimes(numDirsPlusTopLevelRead); + it("should handle directory-specific ignore patterns correctly", async () => { + addToTestDir([ + [".gitignore", "/abc"], + ["abc", "content"], + "xyz/", + ["xyz/.gitignore", "xyz"], + ["xyz/abc", "content"], + ["xyz/xyz", "content"], + ["xyz/normal.txt", "content"], + ]); + + const files = await walkDir( + (await testIde.getWorkspaceDirs())[0], + testIde, + { + returnRelativeUrisPaths: true, + }, + ); + + expect(files).not.toContain("abc"); + expect(files).toContain("xyz/abc"); + expect(files).not.toContain("xyz/xyz"); + expect(files).toContain("xyz/normal.txt"); + }); }); }); diff --git a/core/indexing/walkDir.ts b/core/indexing/walkDir.ts index 9e40ccb570..4e97e01d42 100644 --- a/core/indexing/walkDir.ts +++ b/core/indexing/walkDir.ts @@ -1,8 +1,6 @@ -import path from "node:path"; - import ignore, { Ignore } from "ignore"; -import { FileType, IDE } from "../index.d.js"; +import type { FileType, IDE } from ".."; import { DEFAULT_IGNORE_DIRS, @@ -11,12 +9,13 @@ import { defaultIgnoreFile, getGlobalContinueIgArray, gitIgArrayFromFile, -} from "./ignore.js"; +} from "./ignore"; +import { joinPathsToUri } from "../util/uri"; export interface WalkerOptions { ignoreFiles?: string[]; onlyDirs?: boolean; - returnRelativePaths?: boolean; + returnRelativeUrisPaths?: boolean; additionalIgnoreRules?: string[]; } @@ -24,8 +23,8 @@ type Entry = [string, FileType]; // helper struct used for the DFS walk type WalkableEntry = { - relPath: string; - absPath: string; + relativeUriPath: string; + uri: string; type: FileType; entry: Entry; }; @@ -45,7 +44,7 @@ class DFSWalker { private readonly ignoreFileNames: Set; constructor( - private readonly path: string, + private readonly uri: string, private readonly ide: IDE, private readonly options: WalkerOptions, ) { @@ -54,10 +53,6 @@ class DFSWalker { // walk is a depth-first search implementation public async *walk(): AsyncGenerator { - const fixupFunc = await this.newPathFixupFunc( - this.options.returnRelativePaths ? "" : this.path, - this.ide, - ); const root = this.newRootWalkContext(); const stack = [root]; for (let cur = stack.pop(); cur; cur = stack.pop()) { @@ -77,10 +72,19 @@ class DFSWalker { }); if (this.options.onlyDirs) { // when onlyDirs is enabled the walker will only return directory names - yield fixupFunc(w.relPath); + if (this.options.returnRelativeUrisPaths) { + yield w.relativeUriPath; + } else { + yield w.uri; + } } } else { - yield fixupFunc(w.relPath); + // Note that shouldInclude handles skipping files if options.onlyDirs is true + if (this.options.returnRelativeUrisPaths) { + yield w.relativeUriPath; + } else { + yield w.uri; + } } } } @@ -90,14 +94,17 @@ class DFSWalker { const globalIgnoreFile = getGlobalContinueIgArray(); return { walkableEntry: { - relPath: "", - absPath: this.path, + relativeUriPath: "", + uri: this.uri, type: 2 as FileType.Directory, entry: ["", 2 as FileType.Directory], }, ignoreContexts: [ { - ignore: ignore().add(defaultIgnoreDir).add(defaultIgnoreFile).add(globalIgnoreFile), + ignore: ignore() + .add(defaultIgnoreDir) + .add(defaultIgnoreFile) + .add(globalIgnoreFile), dirname: "", }, ], @@ -107,15 +114,13 @@ class DFSWalker { private async listDirForWalking( walkableEntry: WalkableEntry, ): Promise { - const entries = await this.ide.listDir(walkableEntry.absPath); - return entries.map((e) => { - return { - relPath: path.join(walkableEntry.relPath, e[0]), - absPath: path.join(walkableEntry.absPath, e[0]), - type: e[1], - entry: e, - }; - }); + const entries = await this.ide.listDir(walkableEntry.uri); + return entries.map((e) => ({ + relativeUriPath: `${walkableEntry.relativeUriPath}${walkableEntry.relativeUriPath ? "/" : ""}${e[0]}`, + uri: joinPathsToUri(walkableEntry.uri, e[0]), + type: e[1], + entry: e, + })); } private async getIgnoreToApplyInDir( @@ -129,35 +134,32 @@ class DFSWalker { const patterns = ignoreFilesInDir.map((c) => gitIgArrayFromFile(c)).flat(); const newIgnoreContext = { ignore: ignore().add(patterns), - dirname: curDir.walkableEntry.relPath, + dirname: curDir.walkableEntry.relativeUriPath, }; return [...curDir.ignoreContexts, newIgnoreContext]; } private async loadIgnoreFiles(entries: WalkableEntry[]): Promise { - const ignoreEntries = entries.filter((w) => this.isIgnoreFile(w.entry)); + const ignoreEntries = entries.filter((w) => + this.entryIsIgnoreFile(w.entry), + ); const promises = ignoreEntries.map(async (w) => { - return await this.ide.readFile(w.absPath); + return await this.ide.readFile(w.uri); }); return Promise.all(promises); } - private isIgnoreFile(e: Entry): boolean { - const p = e[0]; - return this.ignoreFileNames.has(p); - } - private shouldInclude( walkableEntry: WalkableEntry, ignoreContexts: IgnoreContext[], ) { if (this.entryIsSymlink(walkableEntry.entry)) { // If called from the root, a symlink either links to a real file in this repository, - // and therefore will be walked OR it linksto something outside of the repository and + // and therefore will be walked OR it links to something outside of the repository and // we do not want to index it return false; } - let relPath = walkableEntry.relPath; + let relPath = walkableEntry.relativeUriPath; if (this.entryIsDirectory(walkableEntry.entry)) { relPath = `${relPath}/`; } else { @@ -166,7 +168,7 @@ class DFSWalker { } } for (const ig of ignoreContexts) { - // remove the directory name and path seperator from the match path, unless this an ignore file + // remove the directory name and path separator from the match path, unless this an ignore file // in the root directory const prefixLength = ig.dirname.length === 0 ? 0 : ig.dirname.length + 1; // The ignore library expects a path relative to the ignore file location @@ -186,50 +188,53 @@ class DFSWalker { return entry[1] === (64 as FileType.SymbolicLink); } - // returns a function which will optionally prefix a root path and fixup the paths for the appropriate OS filesystem (i.e. windows) - // the reason to construct this function once is to avoid the need to call ide.pathSep() multiple times - private async newPathFixupFunc( - rootPath: string, - ide: IDE, - ): Promise<(relPath: string) => string> { - const pathSep = await ide.pathSep(); - const prefix = rootPath === "" ? "" : rootPath + pathSep; - if (pathSep === "/") { - if (rootPath === "") { - // return a no-op function in this case to avoid unnecessary string concatentation - return (relPath: string) => relPath; - } - return (relPath: string) => prefix + relPath; + private entryIsIgnoreFile(e: Entry): boolean { + if ( + e[1] === (2 as FileType.Directory) || + e[1] === (64 as FileType.SymbolicLink) + ) { + return false; } - // this serves to 'fix-up' the path on Windows - return (relPath: string) => { - return prefix + relPath.split("/").join(pathSep); - }; + return this.ignoreFileNames.has(e[0]); } } const defaultOptions: WalkerOptions = { ignoreFiles: [".gitignore", ".continueignore"], additionalIgnoreRules: [...DEFAULT_IGNORE_DIRS, ...DEFAULT_IGNORE_FILETYPES], + onlyDirs: false, + returnRelativeUrisPaths: false, }; export async function walkDir( - path: string, + uri: string, ide: IDE, - _options?: WalkerOptions, + _optionOverrides?: WalkerOptions, ): Promise { - let paths: string[] = []; - for await (const p of walkDirAsync(path, ide, _options)) { - paths.push(p); + let urisOrRelativePaths: string[] = []; + for await (const p of walkDirAsync(uri, ide, _optionOverrides)) { + urisOrRelativePaths.push(p); } - return paths; + return urisOrRelativePaths; } export async function* walkDirAsync( path: string, ide: IDE, - _options?: WalkerOptions, + _optionOverrides?: WalkerOptions, ): AsyncGenerator { - const options = { ...defaultOptions, ..._options }; + const options = { ...defaultOptions, ..._optionOverrides }; yield* new DFSWalker(path, ide, options).walk(); } + +export async function walkDirs( + ide: IDE, + _optionOverrides?: WalkerOptions, + dirs?: string[], // Can pass dirs to prevent duplicate calls +): Promise { + const workspaceDirs = dirs ?? (await ide.getWorkspaceDirs()); + const results = await Promise.all( + workspaceDirs.map((dir) => walkDir(dir, ide, _optionOverrides)), + ); + return results.flat(); +} diff --git a/core/llm/llms/Asksage.ts b/core/llm/llms/Asksage.ts index 316540c487..544b947c25 100644 --- a/core/llm/llms/Asksage.ts +++ b/core/llm/llms/Asksage.ts @@ -9,17 +9,27 @@ class Asksage extends BaseLLM { }; private static modelConversion: { [key: string]: string } = { - "gpt-4o": "gpt-4o", - "gpt-4o-mini": "gpt-4o-mini", - "gpt4-gov": "gpt4-gov", - "gpt-4o-gov": "gpt-4o-gov", - "gpt-3.5-turbo": "gpt35-16k", - "mistral-large-latest": "mistral-large", - "llama3-70b": "llma3", - "gemini-1.5-pro-latest": "google-gemini-pro", - "claude-3-5-sonnet-20240620": "claude-35-sonnet", - "claude-3-opus-20240229": "claude-3-opus", - "claude-3-sonnet-20240229": "claude-3-sonnet", + "gpt-4o-gov": "gpt-4o-gov", // Works + "gpt-4o-mini-gov": "gpt-4o-mini-gov", + "gpt4-gov": "gpt4-gov", // Works + "gpt-gov": "gpt-gov", // Works + "gpt-4o": "gpt-4o", // Works + "gpt-4o-mini": "gpt-4o-mini", // Works + "gpt4": "gpt4", // Works + "gpt4-32k": "gpt4-32k", + "gpt-o1": "gpt-o1", // Works + "gpt-o1-mini": "gpt-o1-mini", // Works + "gpt-3.5-turbo": "gpt35-16k", // Works + "aws-bedrock-claude-35-sonnet-gov": "aws-bedrock-claude-35-sonnet-gov", // Works + "claude-3-5-sonnet-latest": "claude-35-sonnet", // Works + "claude-3-opus-20240229": "claude-3-opus", // Works + "claude-3-sonnet-20240229": "claude-3-sonnet", // Works + "grok-beta": "xai-grok", + "groq-llama33": "groq-llama33", + "groq-70b": "groq-70b", + "mistral-large-latest": "mistral-large", // Works + "llama3-70b": "llma3", // Works + "gemini-1.5-pro-latest": "google-gemini-pro", // Works }; constructor(options: LLMOptions) { @@ -28,7 +38,10 @@ class Asksage extends BaseLLM { } protected _convertModelName(model: string): string { - return Asksage.modelConversion[model] ?? model; + console.log("Converting model:", model); + const convertedModel = Asksage.modelConversion[model] ?? model; + console.log("Converted model:", convertedModel); + return convertedModel; } protected _convertMessage(message: ChatMessage) { diff --git a/core/llm/llms/Ollama.ts b/core/llm/llms/Ollama.ts index c6be1c98bf..3e9af9d38a 100644 --- a/core/llm/llms/Ollama.ts +++ b/core/llm/llms/Ollama.ts @@ -152,6 +152,12 @@ class Ollama extends BaseLLM { "llama3.2-90b": "llama3.2:90b", "phi-2": "phi:2.7b", "phind-codellama-34b": "phind-codellama:34b-v2", + "qwen2.5-coder-0.5b": "qwen2.5-coder:0.5b", + "qwen2.5-coder-1.5b": "qwen2.5-coder:1.5b", + "qwen2.5-coder-3b": "qwen2.5-coder:3b", + "qwen2.5-coder-7b": "qwen2.5-coder:7b", + "qwen2.5-coder-14b": "qwen2.5-coder:14b", + "qwen2.5-coder-32b": "qwen2.5-coder:32b", "wizardcoder-7b": "wizardcoder:7b-python", "wizardcoder-13b": "wizardcoder:13b-python", "wizardcoder-34b": "wizardcoder:34b-python", diff --git a/core/llm/llms/llm.ts b/core/llm/llms/llm.ts index 4cd4332322..e472b73170 100644 --- a/core/llm/llms/llm.ts +++ b/core/llm/llms/llm.ts @@ -1,5 +1,5 @@ import { Chunk } from "../../index.js"; -import { getBasename } from "../../util/index.js"; +import { getUriPathBasename } from "../../util/uri.js"; import { BaseLLM } from "../index.js"; const RERANK_PROMPT = ( @@ -47,7 +47,7 @@ export class LLMReranker extends BaseLLM { async scoreChunk(chunk: Chunk, query: string): Promise { const completion = await this.complete( - RERANK_PROMPT(query, getBasename(chunk.filepath), chunk.content), + RERANK_PROMPT(query, getUriPathBasename(chunk.filepath), chunk.content), new AbortController().signal, { maxTokens: 1, diff --git a/core/promptFiles/v1/createNewPromptFile.ts b/core/promptFiles/v1/createNewPromptFile.ts deleted file mode 100644 index 19286fe6db..0000000000 --- a/core/promptFiles/v1/createNewPromptFile.ts +++ /dev/null @@ -1,59 +0,0 @@ -import path from "path"; - -import { IDE } from "../.."; - -import { DEFAULT_PROMPTS_FOLDER } from "."; - -const DEFAULT_PROMPT_FILE = `# This is an example ".prompt" file -# It is used to define and reuse prompts within Continue -# Continue will automatically create a slash command for each prompt in the .prompts folder -# To learn more, see the full .prompt file reference: https://docs.continue.dev/features/prompt-files -temperature: 0.0 ---- -{{{ diff }}} - -Give me feedback on the above changes. For each file, you should output a markdown section including the following: -- If you found any problems, an h3 like "❌ " -- If you didn't find any problems, an h3 like "✅ " -- If you found any problems, add below a bullet point description of what you found, including a minimal code snippet explaining how to fix it -- If you didn't find any problems, you don't need to add anything else - -Here is an example. The example is surrounded in backticks, but your response should not be: - -\`\`\` -### ✅ - -### ❌ - - -\`\`\` - -You should look primarily for the following types of issues, and only mention other problems if they are highly pressing. - -- console.logs that have been left after debugging -- repeated code -- algorithmic errors that could fail under edge cases -- something that could be refactored - -Make sure to review ALL files that were changed, do not skip any. -`; - -export async function createNewPromptFile( - ide: IDE, - promptPath: string | undefined, -): Promise { - const workspaceDirs = await ide.getWorkspaceDirs(); - if (workspaceDirs.length === 0) { - throw new Error( - "No workspace directories found. Make sure you've opened a folder in your IDE.", - ); - } - const promptFilePath = path.join( - workspaceDirs[0], - promptPath ?? DEFAULT_PROMPTS_FOLDER, - "new-prompt-file.prompt", - ); - - await ide.writeFile(promptFilePath, DEFAULT_PROMPT_FILE); - await ide.openFile(promptFilePath); -} diff --git a/core/promptFiles/v1/getContextProviderHelpers.ts b/core/promptFiles/v1/getContextProviderHelpers.ts index 263724efa4..e027ffbc56 100644 --- a/core/promptFiles/v1/getContextProviderHelpers.ts +++ b/core/promptFiles/v1/getContextProviderHelpers.ts @@ -1,9 +1,20 @@ import Handlebars from "handlebars"; +import { ContextItem, ContinueSDK, IContextProvider } from "../.."; + +function createContextItem(item: ContextItem, provider: IContextProvider) { + return { + ...item, + id: { + itemId: item.description, + providerTitle: provider.description.title, + }, + }; +} export function getContextProviderHelpers( - context: any, + context: ContinueSDK, ): Array<[string, Handlebars.HelperDelegate]> | undefined { - return context.config.contextProviders?.map((provider: any) => [ + return context.config.contextProviders?.map((provider: IContextProvider) => [ provider.description.title, async (helperContext: any) => { const items = await provider.getContextItems(helperContext, { @@ -17,21 +28,11 @@ export function getContextProviderHelpers( selectedCode: context.selectedCode, }); - items.forEach((item: any) => + items.forEach((item) => context.addContextItem(createContextItem(item, provider)), ); - return items.map((item: any) => item.content).join("\n\n"); + return items.map((item) => item.content).join("\n\n"); }, ]); } - -function createContextItem(item: any, provider: any) { - return { - ...item, - id: { - itemId: item.description, - providerTitle: provider.description.title, - }, - }; -} diff --git a/core/promptFiles/v1/getPromptFiles.ts b/core/promptFiles/v1/getPromptFiles.ts deleted file mode 100644 index b9bb5d8234..0000000000 --- a/core/promptFiles/v1/getPromptFiles.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { IDE } from "../.."; -import { walkDir } from "../../indexing/walkDir"; - -export async function getPromptFiles( - ide: IDE, - dir: string, -): Promise<{ path: string; content: string }[]> { - try { - const exists = await ide.fileExists(dir); - - if (!exists) { - return []; - } - - const paths = await walkDir(dir, ide, { ignoreFiles: [] }); - const results = paths.map(async (path) => { - const content = await ide.readFile(path); // make a try catch - return { path, content }; - }); - return Promise.all(results); - } catch (e) { - console.error(e); - return []; - } -} diff --git a/core/promptFiles/v1/handlebarUtils.ts b/core/promptFiles/v1/handlebarUtils.ts index 947b76fbf1..ffb7195902 100644 --- a/core/promptFiles/v1/handlebarUtils.ts +++ b/core/promptFiles/v1/handlebarUtils.ts @@ -1,7 +1,7 @@ import Handlebars from "handlebars"; import { v4 as uuidv4 } from "uuid"; -export function convertToLetter(num: number): string { +function convertToLetter(num: number): string { let result = ""; while (num > 0) { const remainder = (num - 1) % 26; @@ -15,7 +15,7 @@ export function convertToLetter(num: number): string { * We replace filepaths with alphabetic characters to handle * escaping issues. */ -export const replaceFilepaths = ( +const replaceFilepaths = ( value: string, ctxProviderNames: string[], ): [string, { [key: string]: string }] => { @@ -95,7 +95,10 @@ export async function prepareTemplateAndData( return [newTemplate, data]; } -export function compileAndRenderTemplate(template: string, data: Record): string { +export function compileAndRenderTemplate( + template: string, + data: Record, +): string { const templateFn = Handlebars.compile(template); return templateFn(data); } diff --git a/core/promptFiles/v1/index.ts b/core/promptFiles/v1/index.ts index 2eed3a15d9..1d6726b3e3 100644 --- a/core/promptFiles/v1/index.ts +++ b/core/promptFiles/v1/index.ts @@ -1,7 +1 @@ -import { createNewPromptFile } from "./createNewPromptFile"; -import { getPromptFiles } from "./getPromptFiles"; -import { slashCommandFromPromptFile } from "./slashCommandFromPromptFile"; - -export const DEFAULT_PROMPTS_FOLDER = ".prompts"; - -export { createNewPromptFile, getPromptFiles, slashCommandFromPromptFile }; +export const DEFAULT_PROMPTS_FOLDER_V1 = ".prompts"; diff --git a/core/promptFiles/v1/slashCommandFromPromptFile.ts b/core/promptFiles/v1/slashCommandFromPromptFile.ts index 7d3750ad41..d89ee10825 100644 --- a/core/promptFiles/v1/slashCommandFromPromptFile.ts +++ b/core/promptFiles/v1/slashCommandFromPromptFile.ts @@ -1,35 +1,14 @@ -import * as YAML from "yaml"; - import { ContinueSDK, SlashCommand } from "../.."; -import { getBasename } from "../../util/index"; import { renderChatMessage } from "../../util/messageContent"; +import { getLastNPathParts } from "../../util/uri"; +import { parsePromptFileV1V2 } from "../v2/parsePromptFileV1V2"; import { getContextProviderHelpers } from "./getContextProviderHelpers"; import { renderTemplatedString } from "./renderTemplatedString"; import { updateChatHistory } from "./updateChatHistory"; export function extractName(preamble: { name?: string }, path: string): string { - return preamble.name ?? getBasename(path).split(".prompt")[0]; -} - -export function parsePromptFile(path: string, content: string) { - let [preambleRaw, prompt] = content.split("\n---\n"); - if (prompt === undefined) { - prompt = preambleRaw; - preambleRaw = ""; - } - - const preamble = YAML.parse(preambleRaw) ?? {}; - const name = extractName(preamble, path); - const description = preamble.description ?? name; - - let systemMessage: string | undefined = undefined; - if (prompt.includes("")) { - systemMessage = prompt.split("")[1].split("")[0].trim(); - prompt = prompt.split("")[1].trim(); - } - - return { name, description, systemMessage, prompt }; + return preamble.name ?? getLastNPathParts(path, 1).split(".prompt")[0]; } export function extractUserInput(input: string, commandName: string): string { @@ -39,26 +18,23 @@ export function extractUserInput(input: string, commandName: string): string { return input; } -export async function getDefaultVariables( - context: ContinueSDK, - userInput: string, -): Promise> { - const currentFile = await context.ide.getCurrentFile(); - const vars: Record = { input: userInput }; - if (currentFile) { - vars.currentFile = currentFile.path; - } - return vars; -} - -export async function renderPrompt( +async function renderPromptV1( prompt: string, context: ContinueSDK, userInput: string, ) { const helpers = getContextProviderHelpers(context); - const inputData = await getDefaultVariables(context, userInput); + // A few context providers that don't need to be in config.json to work in .prompt files + const diff = await context.ide.getDiff(true); + const currentFile = await context.ide.getCurrentFile(); + const inputData: Record = { + diff: diff.join("\n"), + input: userInput, + }; + if (currentFile) { + inputData.currentFile = currentFile.path; + } return renderTemplatedString( prompt, @@ -68,21 +44,26 @@ export async function renderPrompt( ); } -export function slashCommandFromPromptFile( +export function slashCommandFromPromptFileV1( path: string, content: string, -): SlashCommand { - const { name, description, systemMessage, prompt } = parsePromptFile( - path, - content, - ); +): SlashCommand | null { + const { name, description, systemMessage, prompt, version } = + parsePromptFileV1V2(path, content); + + if (version !== 1) { + return null; + } return { name, description, run: async function* (context) { + const originalSystemMessage = context.llm.systemMessage; + context.llm.systemMessage = systemMessage; + const userInput = extractUserInput(context.input, name); - const renderedPrompt = await renderPrompt(prompt, context, userInput); + const renderedPrompt = await renderPromptV1(prompt, context, userInput); const messages = updateChatHistory( context.history, name, @@ -96,6 +77,8 @@ export function slashCommandFromPromptFile( )) { yield renderChatMessage(chunk); } + + context.llm.systemMessage = originalSystemMessage; }, }; } diff --git a/core/promptFiles/v2/createNewPromptFile.ts b/core/promptFiles/v2/createNewPromptFile.ts index 0a8adc25d9..02ff786d1f 100644 --- a/core/promptFiles/v2/createNewPromptFile.ts +++ b/core/promptFiles/v2/createNewPromptFile.ts @@ -1,6 +1,6 @@ import { IDE } from "../.."; import { GlobalContext } from "../../util/GlobalContext"; -import { getPathModuleForIde } from "../../util/pathModule"; +import { joinPathsToUri } from "../../util/uri"; const FIRST_TIME_DEFAULT_PROMPT_FILE = `# This is an example ".prompt" file # It is used to define and reuse prompts within Continue @@ -45,24 +45,23 @@ export async function createNewPromptFileV2( "No workspace directories found. Make sure you've opened a folder in your IDE.", ); } - const pathModule = await getPathModuleForIde(ide); - const baseDir = pathModule.join( + const baseDirUri = joinPathsToUri( workspaceDirs[0], - promptPath ?? pathModule.join(".continue", "prompts"), + promptPath ?? ".continue/prompts", ); // Find the first available filename let counter = 0; - let promptFilePath: string; + let promptFileUri: string; do { const suffix = counter === 0 ? "" : `-${counter}`; - promptFilePath = pathModule.join( - baseDir, + promptFileUri = joinPathsToUri( + baseDirUri, `new-prompt-file${suffix}.prompt`, ); counter++; - } while (await ide.fileExists(promptFilePath)); + } while (await ide.fileExists(promptFileUri)); const globalContext = new GlobalContext(); const PROMPT_FILE = @@ -72,6 +71,6 @@ export async function createNewPromptFileV2( globalContext.update("hasAlreadyCreatedAPromptFile", true); - await ide.writeFile(promptFilePath, PROMPT_FILE); - await ide.openFile(promptFilePath); + await ide.writeFile(promptFileUri, PROMPT_FILE); + await ide.openFile(promptFileUri); } diff --git a/core/promptFiles/v2/getPromptFiles.ts b/core/promptFiles/v2/getPromptFiles.ts index 0353746be2..2998b5f8f8 100644 --- a/core/promptFiles/v2/getPromptFiles.ts +++ b/core/promptFiles/v2/getPromptFiles.ts @@ -1,9 +1,11 @@ import { IDE } from "../.."; import { walkDir } from "../../indexing/walkDir"; -import { getPathModuleForIde } from "../../util/pathModule"; import { readAllGlobalPromptFiles } from "../../util/paths"; +import { joinPathsToUri } from "../../util/uri"; +import { DEFAULT_PROMPTS_FOLDER_V1 } from "../v1"; -async function getPromptFilesFromDir( +export const DEFAULT_PROMPTS_FOLDER_V2 = ".continue/prompts"; +export async function getPromptFilesFromDir( ide: IDE, dir: string, ): Promise<{ path: string; content: string }[]> { @@ -15,7 +17,8 @@ async function getPromptFilesFromDir( } const paths = await walkDir(dir, ide, { ignoreFiles: [] }); - const results = paths.map(async (path) => { + const promptFilePaths = paths.filter((p) => p.endsWith(".prompt")); + const results = promptFilePaths.map(async (path) => { const content = await ide.readFile(path); // make a try catch return { path, content }; }); @@ -26,29 +29,29 @@ async function getPromptFilesFromDir( } } -export async function getAllPromptFilesV2( +export async function getAllPromptFiles( ide: IDE, overridePromptFolder?: string, + checkV1DefaultFolder: boolean = false, ): Promise<{ path: string; content: string }[]> { const workspaceDirs = await ide.getWorkspaceDirs(); let promptFiles: { path: string; content: string }[] = []; - const pathModule = await getPathModuleForIde(ide); + + let dirsToCheck = [DEFAULT_PROMPTS_FOLDER_V2]; + if (checkV1DefaultFolder) { + dirsToCheck.push(DEFAULT_PROMPTS_FOLDER_V1); + } + if (overridePromptFolder) { + dirsToCheck = [overridePromptFolder]; + } + + const fullDirs = workspaceDirs + .map((dir) => dirsToCheck.map((d) => joinPathsToUri(dir, d))) + .flat(); promptFiles = ( - await Promise.all( - workspaceDirs.map((dir) => - getPromptFilesFromDir( - ide, - pathModule.join( - dir, - overridePromptFolder ?? pathModule.join(".continue", "prompts"), - ), - ), - ), - ) - ) - .flat() - .filter(({ path }) => path.endsWith(".prompt")); + await Promise.all(fullDirs.map((dir) => getPromptFilesFromDir(ide, dir))) + ).flat(); // Also read from ~/.continue/.prompts promptFiles.push(...readAllGlobalPromptFiles()); diff --git a/core/promptFiles/v2/parse.ts b/core/promptFiles/v2/parse.ts index d0c71c9627..95e60230e3 100644 --- a/core/promptFiles/v2/parse.ts +++ b/core/promptFiles/v2/parse.ts @@ -1,9 +1,8 @@ import * as YAML from "yaml"; - -import { getBasename } from "../../util"; +import { getLastNPathParts } from "../../util/uri"; export function extractName(preamble: { name?: string }, path: string): string { - return preamble.name ?? getBasename(path).split(".prompt")[0]; + return preamble.name ?? getLastNPathParts(path, 1).split(".prompt")[0]; } export function getPreambleAndBody(content: string): [string, string] { diff --git a/core/promptFiles/v2/parsePromptFileV1V2.ts b/core/promptFiles/v2/parsePromptFileV1V2.ts new file mode 100644 index 0000000000..11e2b10be1 --- /dev/null +++ b/core/promptFiles/v2/parsePromptFileV1V2.ts @@ -0,0 +1,23 @@ +import * as YAML from "yaml"; +import { getLastNPathParts } from "../../util/uri"; + +export function parsePromptFileV1V2(path: string, content: string) { + let [preambleRaw, prompt] = content.split("\n---\n"); + if (prompt === undefined) { + prompt = preambleRaw; + preambleRaw = ""; + } + + const preamble = YAML.parse(preambleRaw) ?? {}; + const name = preamble.name ?? getLastNPathParts(path, 1).split(".prompt")[0]; + const description = preamble.description ?? name; + const version = preamble.version ?? 2; + + let systemMessage: string | undefined = undefined; + if (prompt.includes("")) { + systemMessage = prompt.split("")[1].split("")[0].trim(); + prompt = prompt.split("")[1].trim(); + } + + return { name, description, systemMessage, prompt, version }; +} diff --git a/core/promptFiles/v2/renderPromptFile.ts b/core/promptFiles/v2/renderPromptFile.ts index 2d1f925100..4f6bb1534d 100644 --- a/core/promptFiles/v2/renderPromptFile.ts +++ b/core/promptFiles/v2/renderPromptFile.ts @@ -1,7 +1,8 @@ import { ContextItem, ContextProviderExtras } from "../.."; import { contextProviderClassFromName } from "../../context/providers"; import URLContextProvider from "../../context/providers/URLContextProvider"; -import { getBasename } from "../../util"; +import { resolveRelativePathInDir } from "../../util/ideUtils"; +import { getUriPathBasename } from "../../util/uri"; import { getPreambleAndBody } from "./parse"; async function resolveAttachment( @@ -25,7 +26,8 @@ async function resolveAttachment( } // Files - if (await extras.ide.fileExists(name)) { + const resolvedFileUri = await resolveRelativePathInDir(name, extras.ide); + if (resolvedFileUri) { let subItems: ContextItem[] = []; if (name.endsWith(".prompt")) { // Recurse @@ -36,10 +38,14 @@ async function resolveAttachment( subItems.push(...items); } - const content = `\`\`\`${name}\n${await extras.ide.readFile(name)}\n\`\`\``; + const content = `\`\`\`${name}\n${await extras.ide.readFile(resolvedFileUri)}\n\`\`\``; return [ ...subItems, - { name: getBasename(name), content, description: name }, + { + name: getUriPathBasename(resolvedFileUri), + content, + description: resolvedFileUri, + }, ]; } diff --git a/core/protocol/core.ts b/core/protocol/core.ts index 369470c599..9a7d85510d 100644 --- a/core/protocol/core.ts +++ b/core/protocol/core.ts @@ -159,7 +159,6 @@ export type ToCoreFromIdeOrWebviewProtocol = { undefined | { dirs?: string[]; shouldClearIndexes?: boolean }, void, ]; - "index/forceReIndexFiles": [undefined | { files?: string[] }, void]; "index/indexingProgressBarInitialized": [undefined, void]; completeOnboarding: [ { @@ -168,6 +167,12 @@ export type ToCoreFromIdeOrWebviewProtocol = { void, ]; + // File changes + "files/changed": [{ uris?: string[] }, void]; + "files/opened": [{ uris?: string[] }, void]; + "files/created": [{ uris?: string[] }, void]; + "files/deleted": [{ uris?: string[] }, void]; + // Docs etc. Indexing. TODO move codebase to this "indexing/reindex": [{ type: string; id: string }, void]; "indexing/abort": [{ type: string; id: string }, void]; diff --git a/core/protocol/ide.ts b/core/protocol/ide.ts index 6bb6cc5edb..667ebd7b21 100644 --- a/core/protocol/ide.ts +++ b/core/protocol/ide.ts @@ -23,10 +23,8 @@ export type ToIdeFromWebviewOrCoreProtocol = { // Methods from IDE type getIdeInfo: [undefined, IdeInfo]; getWorkspaceDirs: [undefined, string[]]; - listFolders: [undefined, string[]]; writeFile: [{ path: string; contents: string }, void]; showVirtualFile: [{ name: string; content: string }, void]; - getContinueDir: [undefined, string]; openFile: [{ path: string }, void]; openUrl: [string, void]; runCommand: [{ command: string }, void]; @@ -35,10 +33,6 @@ export type ToIdeFromWebviewOrCoreProtocol = { saveFile: [{ filepath: string }, void]; fileExists: [{ filepath: string }, boolean]; readFile: [{ filepath: string }, string]; - showDiff: [ - { filepath: string; newContents: string; stepIndex: number }, - void, - ]; diffLine: [ { diffLine: DiffLine; @@ -100,7 +94,6 @@ export type ToIdeFromWebviewOrCoreProtocol = { ControlPlaneSessionInfo | undefined, ]; logoutOfControlPlane: [undefined, void]; - pathSep: [undefined, string]; }; export type ToWebviewOrCoreFromIdeProtocol = { diff --git a/core/protocol/messenger/messageIde.ts b/core/protocol/messenger/messageIde.ts index febdf559cd..e2cd50c781 100644 --- a/core/protocol/messenger/messageIde.ts +++ b/core/protocol/messenger/messageIde.ts @@ -27,16 +27,13 @@ export class MessageIde implements IDE { ) => void, ) {} - pathSep(): Promise { - return this.request("pathSep", undefined); - } - fileExists(filepath: string): Promise { - return this.request("fileExists", { filepath }); + fileExists(fileUri: string): Promise { + return this.request("fileExists", { filepath: fileUri }); } async gotoDefinition(location: Location): Promise { return this.request("gotoDefinition", { location }); } - onDidChangeActiveTextEditor(callback: (filepath: string) => void): void { + onDidChangeActiveTextEditor(callback: (fileUri: string) => void): void { this.on("didChangeActiveTextEditor", (data) => callback(data.filepath)); } @@ -126,38 +123,27 @@ export class MessageIde implements IDE { } async showLines( - filepath: string, + fileUri: string, startLine: number, endLine: number, ): Promise { - return await this.request("showLines", { filepath, startLine, endLine }); - } - - async listFolders(): Promise { - return await this.request("listFolders", undefined); - } - - _continueDir: string | null = null; - - async getContinueDir(): Promise { - if (this._continueDir) { - return this._continueDir; - } - const dir = await this.request("getContinueDir", undefined); - this._continueDir = dir; - return dir; + return await this.request("showLines", { + filepath: fileUri, + startLine, + endLine, + }); } - async writeFile(path: string, contents: string): Promise { - await this.request("writeFile", { path, contents }); + async writeFile(fileUri: string, contents: string): Promise { + await this.request("writeFile", { path: fileUri, contents }); } async showVirtualFile(title: string, contents: string): Promise { await this.request("showVirtualFile", { name: title, content: contents }); } - async openFile(path: string): Promise { - await this.request("openFile", { path }); + async openFile(fileUri: string): Promise { + await this.request("openFile", { path: fileUri }); } async openUrl(url: string): Promise { @@ -168,18 +154,11 @@ export class MessageIde implements IDE { await this.request("runCommand", { command }); } - async saveFile(filepath: string): Promise { - await this.request("saveFile", { filepath }); + async saveFile(fileUri: string): Promise { + await this.request("saveFile", { filepath: fileUri }); } - async readFile(filepath: string): Promise { - return await this.request("readFile", { filepath }); - } - async showDiff( - filepath: string, - newContents: string, - stepIndex: number, - ): Promise { - await this.request("showDiff", { filepath, newContents, stepIndex }); + async readFile(fileUri: string): Promise { + return await this.request("readFile", { filepath: fileUri }); } getOpenFiles(): Promise { @@ -198,8 +177,8 @@ export class MessageIde implements IDE { return this.request("getSearchResults", { query }); } - getProblems(filepath: string): Promise { - return this.request("getProblems", { filepath }); + getProblems(fileUri: string): Promise { + return this.request("getProblems", { filepath: fileUri }); } subprocess(command: string, cwd?: string): Promise<[string, string]> { diff --git a/core/protocol/messenger/reverseMessageIde.ts b/core/protocol/messenger/reverseMessageIde.ts index 46e6418c0d..e3f97ea4fb 100644 --- a/core/protocol/messenger/reverseMessageIde.ts +++ b/core/protocol/messenger/reverseMessageIde.ts @@ -117,19 +117,11 @@ export class ReverseMessageIde { return this.ide.showLines(data.filepath, data.startLine, data.endLine); }); - this.on("listFolders", () => { - return this.ide.listFolders(); - }); - this.on("getControlPlaneSessionInfo", async (msg) => { // Not supported in testing return undefined; }); - this.on("getContinueDir", () => { - return this.ide.getContinueDir(); - }); - this.on("writeFile", (data) => { return this.ide.writeFile(data.path, data.contents); }); @@ -158,10 +150,6 @@ export class ReverseMessageIde { return this.ide.readFile(data.filepath); }); - this.on("showDiff", (data) => { - return this.ide.showDiff(data.filepath, data.newContents, data.stepIndex); - }); - this.on("getOpenFiles", () => { return this.ide.getOpenFiles(); }); @@ -189,8 +177,5 @@ export class ReverseMessageIde { this.on("getBranch", (data) => { return this.ide.getBranch(data.dir); }); - this.on("pathSep", (data) => { - return this.ide.pathSep(); - }); } } diff --git a/core/protocol/passThrough.ts b/core/protocol/passThrough.ts index c89b91789b..c7df8fddda 100644 --- a/core/protocol/passThrough.ts +++ b/core/protocol/passThrough.ts @@ -44,7 +44,6 @@ export const WEBVIEW_TO_CORE_PASS_THROUGH: (keyof ToCoreFromWebviewProtocol)[] = // Codebase "index/setPaused", "index/forceReIndex", - "index/forceReIndexFiles", "index/indexingProgressBarInitialized", // Docs, etc. "indexing/reindex", diff --git a/core/test/testDir.ts b/core/test/testDir.ts index 7bc63105b4..9b0473b468 100644 --- a/core/test/testDir.ts +++ b/core/test/testDir.ts @@ -1,20 +1,22 @@ import fs from "fs"; import os from "os"; import path from "path"; +import { localPathToUri, localPathOrUriToPath } from "../util/pathToUri"; // Want this outside of the git repository so we can change branches in tests -export const TEST_DIR = path.join(os.tmpdir(), "testWorkspaceDir"); +export const TEST_DIR_PATH = path.join(os.tmpdir(), "testWorkspaceDir"); +export const TEST_DIR = localPathToUri(TEST_DIR_PATH); // URI export function setUpTestDir() { - if (fs.existsSync(TEST_DIR)) { - fs.rmSync(TEST_DIR, { recursive: true }); + if (fs.existsSync(TEST_DIR_PATH)) { + fs.rmSync(TEST_DIR_PATH, { recursive: true }); } - fs.mkdirSync(TEST_DIR); + fs.mkdirSync(TEST_DIR_PATH); } export function tearDownTestDir() { - if (fs.existsSync(TEST_DIR)) { - fs.rmSync(TEST_DIR, { recursive: true }); + if (fs.existsSync(TEST_DIR_PATH)) { + fs.rmSync(TEST_DIR_PATH, { recursive: true }); } } @@ -24,15 +26,24 @@ export function tearDownTestDir() { "index/index.ts" creates an empty index/index.ts ["index/index.ts", "hello"] creates index/index.ts with contents "hello" */ -export function addToTestDir(paths: (string | string[])[]) { +export function addToTestDir(pathsOrUris: (string | string[])[]) { + // Allow tests to use URIs or local paths + const paths = pathsOrUris.map((val) => { + if (Array.isArray(val)) { + return [localPathOrUriToPath(val[0]), val[1]]; + } else { + return localPathOrUriToPath(val); + } + }); + for (const p of paths) { - const filepath = path.join(TEST_DIR, Array.isArray(p) ? p[0] : p); + const filepath = path.join(TEST_DIR_PATH, Array.isArray(p) ? p[0] : p); fs.mkdirSync(path.dirname(filepath), { recursive: true }); if (Array.isArray(p)) { fs.writeFileSync(filepath, p[1]); } else if (p.endsWith("/")) { - fs.mkdirSync(filepath, { recursive: true }); + fs.mkdirSync(p, { recursive: true }); } else { fs.writeFileSync(filepath, ""); } diff --git a/core/tools/implementations/createNewFile.ts b/core/tools/implementations/createNewFile.ts index 0be187b193..1b733da010 100644 --- a/core/tools/implementations/createNewFile.ts +++ b/core/tools/implementations/createNewFile.ts @@ -1,17 +1,15 @@ -import { getPathModuleForIde } from "../../util/pathModule"; +import { inferResolvedUriFromRelativePath } from "../../util/ideUtils"; import { ToolImpl } from "."; export const createNewFileImpl: ToolImpl = async (args, extras) => { - const pathSep = await extras.ide.pathSep(); - let filepath = args.filepath; - if (!args.filepath.startsWith(pathSep)) { - const pathModule = await getPathModuleForIde(extras.ide); - const workspaceDirs = await extras.ide.getWorkspaceDirs(); - const cwd = workspaceDirs[0]; - filepath = pathModule.join(cwd, filepath); + const resolvedFilepath = await inferResolvedUriFromRelativePath( + args.filepath, + extras.ide, + ); + if (resolvedFilepath) { + await extras.ide.writeFile(resolvedFilepath, args.contents); + await extras.ide.openFile(resolvedFilepath); } - await extras.ide.writeFile(filepath, args.contents); - await extras.ide.openFile(filepath); return []; }; diff --git a/core/tools/implementations/readCurrentlyOpenFile.ts b/core/tools/implementations/readCurrentlyOpenFile.ts index 7fff3b621f..acfd9bec2b 100644 --- a/core/tools/implementations/readCurrentlyOpenFile.ts +++ b/core/tools/implementations/readCurrentlyOpenFile.ts @@ -1,5 +1,5 @@ import { ToolImpl } from "."; -import { getBasename } from "../../util"; +import { getUriPathBasename } from "../../util/uri"; export const readCurrentlyOpenFileImpl: ToolImpl = async (args, extras) => { const result = await extras.ide.getCurrentFile(); @@ -8,7 +8,7 @@ export const readCurrentlyOpenFileImpl: ToolImpl = async (args, extras) => { return []; } - const basename = getBasename(result.path); + const basename = getUriPathBasename(result.path); return [ { diff --git a/core/tools/implementations/readFile.ts b/core/tools/implementations/readFile.ts index 032db3dce5..674b41578f 100644 --- a/core/tools/implementations/readFile.ts +++ b/core/tools/implementations/readFile.ts @@ -1,12 +1,11 @@ -import { getBasename } from "../../util"; - import { ToolImpl } from "."; +import { getUriPathBasename } from "../../util/uri"; export const readFileImpl: ToolImpl = async (args, extras) => { const content = await extras.ide.readFile(args.filepath); return [ { - name: getBasename(args.filepath), + name: getUriPathBasename(args.filepath), description: args.filepath, content, }, diff --git a/core/tools/implementations/runTerminalCommand.ts b/core/tools/implementations/runTerminalCommand.ts index 5a159b04a5..ccb79f2908 100644 --- a/core/tools/implementations/runTerminalCommand.ts +++ b/core/tools/implementations/runTerminalCommand.ts @@ -2,6 +2,7 @@ import childProcess from "node:child_process"; import util from "node:util"; import { ToolImpl } from "."; +import { fileURLToPath } from "node:url"; const asyncExec = util.promisify(childProcess.exec); @@ -11,7 +12,7 @@ export const runTerminalCommandImpl: ToolImpl = async (args, extras) => { if (ideInfo.remoteName === "local" || ideInfo.remoteName === "") { try { const output = await asyncExec(args.command, { - cwd: (await extras.ide.getWorkspaceDirs())[0], + cwd: fileURLToPath((await extras.ide.getWorkspaceDirs())[0]), }); return [ { diff --git a/core/tools/implementations/viewRepoMap.ts b/core/tools/implementations/viewRepoMap.ts index 56c2654162..69f624feb4 100644 --- a/core/tools/implementations/viewRepoMap.ts +++ b/core/tools/implementations/viewRepoMap.ts @@ -3,7 +3,10 @@ import generateRepoMap from "../../util/generateRepoMap"; import { ToolImpl } from "."; export const viewRepoMapImpl: ToolImpl = async (args, extras) => { - const repoMap = await generateRepoMap(extras.llm, extras.ide, {}); + const repoMap = await generateRepoMap(extras.llm, extras.ide, { + outputRelativeUriPaths: true, + includeSignatures: false, + }); return [ { name: "Repo map", diff --git a/core/tools/implementations/viewSubdirectory.ts b/core/tools/implementations/viewSubdirectory.ts index a8204ca6a2..c62887dc5b 100644 --- a/core/tools/implementations/viewSubdirectory.ts +++ b/core/tools/implementations/viewSubdirectory.ts @@ -1,22 +1,22 @@ import generateRepoMap from "../../util/generateRepoMap"; -import { resolveRelativePathInWorkspace } from "../../util/ideUtils"; +import { resolveRelativePathInDir } from "../../util/ideUtils"; import { ToolImpl } from "."; export const viewSubdirectoryImpl: ToolImpl = async (args: any, extras) => { const { directory_path } = args; - const absolutePath = await resolveRelativePathInWorkspace( - directory_path, - extras.ide, - ); + const uri = await resolveRelativePathInDir(directory_path, extras.ide); - if (!absolutePath) { + if (!uri) { throw new Error(`Directory path "${directory_path}" does not exist.`); } const repoMap = await generateRepoMap(extras.llm, extras.ide, { - dirs: [absolutePath], + dirUris: [uri], + outputRelativeUriPaths: true, + includeSignatures: false, }); + return [ { name: "Repo map", diff --git a/core/util/GlobalContext.ts b/core/util/GlobalContext.ts index 13c9d0735c..1095234d82 100644 --- a/core/util/GlobalContext.ts +++ b/core/util/GlobalContext.ts @@ -15,6 +15,7 @@ export type GlobalContextType = { curEmbeddingsProviderId: string; hasDismissedConfigTsNoticeJetBrains: boolean; hasAlreadyCreatedAPromptFile: boolean; + showConfigUpdateToast: boolean; }; /** diff --git a/core/util/devdata.ts b/core/util/devdata.ts index 12723e9159..3f786fa846 100644 --- a/core/util/devdata.ts +++ b/core/util/devdata.ts @@ -1,9 +1,9 @@ -import { writeFileSync } from "fs"; +import fs from "fs"; import { getDevDataFilePath } from "./paths.js"; export function logDevData(tableName: string, data: any) { const filepath: string = getDevDataFilePath(tableName); const jsonLine = JSON.stringify(data); - writeFileSync(filepath, `${jsonLine}\n`, { flag: "a" }); + fs.writeFileSync(filepath, `${jsonLine}\n`, { flag: "a" }); } diff --git a/core/util/filesystem.ts b/core/util/filesystem.ts index f85d58e57c..d0821b6ef7 100644 --- a/core/util/filesystem.ts +++ b/core/util/filesystem.ts @@ -1,5 +1,4 @@ import * as fs from "node:fs"; -import * as path from "node:path"; import { ContinueRcJson, @@ -14,10 +13,9 @@ import { RangeInFile, Thread, ToastType, -} from "../index.d.js"; +} from "../index.js"; import { GetGhTokenArgs } from "../protocol/ide.js"; - -import { getContinueGlobalPath } from "./paths.js"; +import { fileURLToPath } from "node:url"; class FileSystemIde implements IDE { constructor(private readonly workspaceDir: string) {} @@ -28,17 +26,15 @@ class FileSystemIde implements IDE { ): Promise { return Promise.resolve(); } - pathSep(): Promise { - return Promise.resolve(path.sep); - } - fileExists(filepath: string): Promise { + fileExists(fileUri: string): Promise { + const filepath = fileURLToPath(fileUri); return Promise.resolve(fs.existsSync(filepath)); } gotoDefinition(location: Location): Promise { return Promise.resolve([]); } - onDidChangeActiveTextEditor(callback: (filepath: string) => void): void { + onDidChangeActiveTextEditor(callback: (fileUri: string) => void): void { return; } @@ -55,14 +51,17 @@ class FileSystemIde implements IDE { async getGitHubAuthToken(args: GetGhTokenArgs): Promise { return undefined; } - async getLastModified(files: string[]): Promise<{ [path: string]: number }> { + async getLastModified( + fileUris: string[], + ): Promise<{ [path: string]: number }> { const result: { [path: string]: number } = {}; - for (const file of files) { + for (const uri of fileUris) { try { - const stats = fs.statSync(file); - result[file] = stats.mtimeMs; + const filepath = fileURLToPath(uri); + const stats = fs.statSync(filepath); + result[uri] = stats.mtimeMs; } catch (error) { - console.error(`Error getting last modified time for ${file}:`, error); + console.error(`Error getting last modified time for ${uri}:`, error); } } return result; @@ -71,8 +70,9 @@ class FileSystemIde implements IDE { return Promise.resolve(dir); } async listDir(dir: string): Promise<[string, FileType][]> { + const filepath = fileURLToPath(dir); const all: [string, FileType][] = fs - .readdirSync(dir, { withFileTypes: true }) + .readdirSync(filepath, { withFileTypes: true }) .map((dirent: any) => [ dirent.name, dirent.isDirectory() @@ -109,7 +109,7 @@ class FileSystemIde implements IDE { }); } - readRangeInFile(filepath: string, range: Range): Promise { + readRangeInFile(fileUri: string, range: Range): Promise { return Promise.resolve(""); } @@ -153,7 +153,7 @@ class FileSystemIde implements IDE { } showLines( - filepath: string, + fileUri: string, startLine: number, endLine: number, ): Promise { @@ -164,13 +164,10 @@ class FileSystemIde implements IDE { return Promise.resolve([this.workspaceDir]); } - listFolders(): Promise { - return Promise.resolve([]); - } - - writeFile(path: string, contents: string): Promise { + writeFile(fileUri: string, contents: string): Promise { + const filepath = fileURLToPath(fileUri); return new Promise((resolve, reject) => { - fs.writeFile(path, contents, (err) => { + fs.writeFile(filepath, contents, (err) => { if (err) { reject(err); } @@ -183,10 +180,6 @@ class FileSystemIde implements IDE { return Promise.resolve(); } - getContinueDir(): Promise { - return Promise.resolve(getContinueGlobalPath()); - } - openFile(path: string): Promise { return Promise.resolve(); } @@ -199,11 +192,12 @@ class FileSystemIde implements IDE { return Promise.resolve(); } - saveFile(filepath: string): Promise { + saveFile(fileUri: string): Promise { return Promise.resolve(); } - readFile(filepath: string): Promise { + readFile(fileUri: string): Promise { + const filepath = fileURLToPath(fileUri); return new Promise((resolve, reject) => { fs.readFile(filepath, "utf8", (err, contents) => { if (err) { @@ -218,14 +212,6 @@ class FileSystemIde implements IDE { return Promise.resolve(undefined); } - showDiff( - filepath: string, - newContents: string, - stepIndex: number, - ): Promise { - return Promise.resolve(); - } - getBranch(dir: string): Promise { return Promise.resolve(""); } @@ -242,7 +228,7 @@ class FileSystemIde implements IDE { return ""; } - async getProblems(filepath?: string | undefined): Promise { + async getProblems(fileUri?: string | undefined): Promise { return Promise.resolve([]); } diff --git a/core/util/generateRepoMap.test.ts b/core/util/generateRepoMap.test.ts index d1d130d316..536d3fcac2 100644 --- a/core/util/generateRepoMap.test.ts +++ b/core/util/generateRepoMap.test.ts @@ -39,7 +39,7 @@ describe.skip("generateRepoMap", () => { .mockImplementation(async (dirs, offset, limit) => { // Return test data return { - groupedByPath: { + groupedByUri: { [path.join(TEST_DIR, "file1.js")]: [ "function foo() {}", "function bar() {}", @@ -60,6 +60,7 @@ describe.skip("generateRepoMap", () => { // Act const repoMapContent = await generateRepoMap(testLLM, testIde, { includeSignatures: true, + outputRelativeUriPaths: true, }); // Assert @@ -103,7 +104,7 @@ describe.skip("generateRepoMap", () => { .mockImplementation(async (dirs, offset, limit) => { // Return test data return { - groupedByPath: { + groupedByUri: { [path.join(TEST_DIR, "file1.js")]: [], [path.join(TEST_DIR, "subdir/file2.py")]: [], }, @@ -118,6 +119,7 @@ describe.skip("generateRepoMap", () => { // Act const repoMapContent = await generateRepoMap(testLLM, testIde, { includeSignatures: false, + outputRelativeUriPaths: true, }); // Assert @@ -155,7 +157,7 @@ describe.skip("generateRepoMap", () => { .mockImplementation(async (dirs, offset, limit) => { // Return test data return { - groupedByPath: { + groupedByUri: { [path.join(TEST_DIR, "file1.js")]: ["function foo() {}"], [path.join(TEST_DIR, "subdir/file2.py")]: ["def foo():"], }, @@ -185,6 +187,7 @@ describe.skip("generateRepoMap", () => { // Act const repoMapContent = await generateRepoMap(testLLM, testIde, { includeSignatures: true, + outputRelativeUriPaths: true, }); // Assert diff --git a/core/util/generateRepoMap.ts b/core/util/generateRepoMap.ts index 6aad1f95ab..b0af8079ad 100644 --- a/core/util/generateRepoMap.ts +++ b/core/util/generateRepoMap.ts @@ -1,16 +1,17 @@ import fs from "node:fs"; -import path from "node:path"; import { IDE, ILLM } from ".."; import { CodeSnippetsCodebaseIndex } from "../indexing/CodeSnippetsIndex"; -import { walkDirAsync } from "../indexing/walkDir"; +import { walkDirs } from "../indexing/walkDir"; import { pruneLinesFromTop } from "../llm/countTokens"; import { getRepoMapFilePath } from "./paths"; +import { findUriInDirs } from "./uri"; export interface RepoMapOptions { includeSignatures?: boolean; - dirs?: string[]; + dirUris?: string[]; + outputRelativeUriPaths: boolean; } class RepoMapGenerator { @@ -19,8 +20,8 @@ class RepoMapGenerator { private repoMapPath: string = getRepoMapFilePath(); private writeStream: fs.WriteStream = fs.createWriteStream(this.repoMapPath); private contentTokens: number = 0; - private repoMapDirs: string[] = []; - private allPathsInDirs: Set = new Set(); + private dirs: string[] = []; + private allUris: string[] = []; private pathsInDirsWithSnippets: Set = new Set(); private BATCH_SIZE = 100; @@ -40,118 +41,100 @@ class RepoMapGenerator { llm.contextLength * this.REPO_MAX_CONTEXT_LENGTH_RATIO; } - async generate(): Promise { - this.repoMapDirs = this.options.dirs ?? (await this.ide.getWorkspaceDirs()); - this.allPathsInDirs = await this.getAllPathsInDirs(); - - await this.initializeWriteStream(); - await this.processPathsAndSignatures(); - - this.writeStream.end(); - this.logRepoMapGeneration(); - - return fs.readFileSync(this.repoMapPath, "utf8"); + private getUriForWrite(uri: string) { + if (this.options.outputRelativeUriPaths) { + return findUriInDirs(uri, this.dirs).relativePathOrBasename; + } + return uri; } - private async initializeWriteStream(): Promise { - await this.writeToStream(this.PREAMBLE); - this.contentTokens += this.llm.countTokens(this.PREAMBLE); - } + async generate(): Promise { + this.dirs = this.options.dirUris ?? (await this.ide.getWorkspaceDirs()); + this.allUris = await walkDirs(this.ide, undefined, this.dirs); - private async getAllPathsInDirs(): Promise> { - const paths = new Set(); + // Initialize + await this.writeToStream(this.PREAMBLE); - for (const dir of this.repoMapDirs) { - for await (const filepath of walkDirAsync(dir, this.ide)) { - paths.add(filepath.replace(dir, "").slice(1)); + if (this.options.includeSignatures) { + // Process uris and signatures + let offset = 0; + while (true) { + const { groupedByUri, hasMore } = + await CodeSnippetsCodebaseIndex.getPathsAndSignatures( + this.allUris, + offset, + this.BATCH_SIZE, + ); + // process batch + for (const [uri, signatures] of Object.entries(groupedByUri)) { + let fileContent: string; + + try { + fileContent = await this.ide.readFile(uri); + } catch (err) { + console.error( + "Failed to read file:\n" + + ` Uri: ${uri}\n` + + ` Error: ${err instanceof Error ? err.message : String(err)}`, + ); + + continue; + } + + const filteredSignatures = signatures.filter( + (signature) => signature.trim() !== fileContent.trim(), + ); + + if (filteredSignatures.length > 0) { + this.pathsInDirsWithSnippets.add(uri); + } + + let content = `${this.getUriForWrite(uri)}:\n`; + + for (const signature of signatures.slice(0, -1)) { + content += `${this.indentMultilineString(signature)}\n\t...\n`; + } + + content += `${this.indentMultilineString( + signatures[signatures.length - 1], + )}\n\n`; + + if (content) { + await this.writeToStream(content); + } + } + if (!hasMore || this.contentTokens >= this.maxRepoMapTokens) { + break; + } + offset += this.BATCH_SIZE; } - } - return paths; - } + // Remaining Uris just so that written repo map isn't incomplete + const urisWithoutSnippets = this.allUris.filter( + (uri) => !this.pathsInDirsWithSnippets.has(uri), + ); - private async processPathsAndSignatures(): Promise { - let offset = 0; - while (true) { - const { groupedByPath, hasMore } = - await CodeSnippetsCodebaseIndex.getPathsAndSignatures( - this.repoMapDirs, - offset, - this.BATCH_SIZE, + if (urisWithoutSnippets.length > 0) { + await this.writeToStream( + urisWithoutSnippets.map((uri) => this.getUriForWrite(uri)).join("\n"), ); - await this.processBatch(groupedByPath); - if (!hasMore || this.contentTokens >= this.maxRepoMapTokens) { - break; - } - offset += this.BATCH_SIZE; - } - await this.writeRemainingPaths(); - } - - private async processBatch( - groupedByPath: Record, - ): Promise { - for (const [absolutePath, signatures] of Object.entries(groupedByPath)) { - const content = await this.processFile(absolutePath, signatures); - - if (content) { - await this.writeToStream(content); } - } - } - - private async processFile( - absolutePath: string, - signatures: string[], - ): Promise { - const workspaceDir = - this.repoMapDirs.find((dir) => absolutePath.startsWith(dir)) || ""; - const relativePath = path.relative(workspaceDir, absolutePath); - - let fileContent: string; - - try { - fileContent = await fs.promises.readFile(absolutePath, "utf8"); - } catch (err) { - console.error( - "Failed to read file:\n" + - ` Path: ${absolutePath}\n` + - ` Error: ${err instanceof Error ? err.message : String(err)}`, + } else { + // Only process uris + await this.writeToStream( + this.allUris.map((uri) => this.getUriForWrite(uri)).join("\n"), ); - - return; - } - - const filteredSignatures = signatures.filter( - (signature) => signature.trim() !== fileContent.trim(), - ); - - if (filteredSignatures.length > 0) { - this.pathsInDirsWithSnippets.add(relativePath); - } - - return this.generateContentForPath(relativePath, filteredSignatures); - } - - private generateContentForPath( - relativePath: string, - signatures: string[], - ): string { - if (this.options.includeSignatures === false) { - return `${relativePath}\n`; } - let content = `${relativePath}:\n`; + this.writeStream.end(); - for (const signature of signatures.slice(0, -1)) { - content += `${this.indentMultilineString(signature)}\n\t...\n`; + if (this.contentTokens >= this.maxRepoMapTokens) { + console.debug( + "Full repo map was unable to be generated due to context window limitations", + ); } - content += `${this.indentMultilineString( - signatures[signatures.length - 1], - )}\n\n`; - - return content; + return fs.readFileSync(this.repoMapPath, "utf8"); } private async writeToStream(content: string): Promise { @@ -170,26 +153,6 @@ class RepoMapGenerator { await new Promise((resolve) => this.writeStream.write(content, resolve)); } - private async writeRemainingPaths(): Promise { - const pathsWithoutSnippets = new Set( - [...this.allPathsInDirs].filter( - (x) => !this.pathsInDirsWithSnippets.has(x), - ), - ); - - if (pathsWithoutSnippets.size > 0) { - await this.writeToStream(Array.from(pathsWithoutSnippets).join("\n")); - } - } - - private logRepoMapGeneration(): void { - if (this.contentTokens >= this.maxRepoMapTokens) { - console.debug( - "Full repo map was unable to be generated due to context window limitations", - ); - } - } - private indentMultilineString(str: string) { return str .split("\n") diff --git a/core/util/ideUtils.test.ts b/core/util/ideUtils.test.ts deleted file mode 100644 index 58cd7b7f40..0000000000 --- a/core/util/ideUtils.test.ts +++ /dev/null @@ -1,48 +0,0 @@ -// Generated by continue - -import { resolveRelativePathInWorkspace } from "./ideUtils"; -import { testIde } from "../test/fixtures"; -import { IDE } from ".."; - -describe("resolveRelativePathInWorkspace", () => { - let mockIde: IDE; - - beforeEach(() => { - mockIde = testIde; - }); - - it("should return the full path if the path is already absolute", async () => { - const mockPath = "/absolute/path/to/file.txt"; - jest.spyOn(mockIde, "pathSep").mockResolvedValue("/"); - - const result = await resolveRelativePathInWorkspace(mockPath, mockIde); - expect(result).toBe(mockPath); - }); - - it("should resolve a relative path to a full path within workspace directories", async () => { - const relativePath = "relative/path/to/file.txt"; - const workspaces = ["/workspace/one", "/workspace/two"]; - const expectedFullPath = `/workspace/one/${relativePath}`; - - jest.spyOn(mockIde, "pathSep").mockResolvedValue("/"); - jest.spyOn(mockIde, "getWorkspaceDirs").mockResolvedValue(workspaces); - jest - .spyOn(mockIde, "fileExists") - .mockImplementation(async (path) => path === expectedFullPath); - - const result = await resolveRelativePathInWorkspace(relativePath, mockIde); - expect(result).toBe(expectedFullPath); - }); - - it("should return undefined if the relative path does not exist in any workspace directory", async () => { - const relativePath = "non/existent/path.txt"; - const workspaces = ["/workspace/one", "/workspace/two"]; - - jest.spyOn(mockIde, "pathSep").mockResolvedValue("/"); - jest.spyOn(mockIde, "getWorkspaceDirs").mockResolvedValue(workspaces); - jest.spyOn(mockIde, "fileExists").mockResolvedValue(false); - - const result = await resolveRelativePathInWorkspace(relativePath, mockIde); - expect(result).toBeUndefined(); - }); -}); diff --git a/core/util/ideUtils.ts b/core/util/ideUtils.ts index 0b7de8b029..2dbf201473 100644 --- a/core/util/ideUtils.ts +++ b/core/util/ideUtils.ts @@ -1,21 +1,80 @@ import { IDE } from ".."; +import { joinPathsToUri, pathToUriPathSegment } from "./uri"; -export async function resolveRelativePathInWorkspace( +/* + This function takes a relative (to workspace) filepath + And checks each workspace for if it exists or not + Only returns fully resolved URI if it exists +*/ +export async function resolveRelativePathInDir( path: string, ide: IDE, + dirUriCandidates?: string[], ): Promise { - const pathSep = await ide.pathSep(); - if (path.startsWith(pathSep)) { - return path; + const dirs = dirUriCandidates ?? (await ide.getWorkspaceDirs()); + for (const dirUri of dirs) { + const fullUri = joinPathsToUri(dirUri, path); + if (await ide.fileExists(fullUri)) { + return fullUri; + } + } + + return undefined; +} + +/* + Same as above but in this case the relative path does not need to exist (e.g. file to be created, etc) + Checks closes match with the dirs, path segment by segment + and based on which workspace has the closest matching path, returns resolved URI + If no meaninful path match just concatenates to first dir's uri +*/ +export async function inferResolvedUriFromRelativePath( + path: string, + ide: IDE, + dirCandidates?: string[], +): Promise { + const dirs = dirCandidates ?? (await ide.getWorkspaceDirs()); + console.log(path, dirs); + if (dirs.length === 0) { + throw new Error("inferResolvedUriFromRelativePath: no dirs provided"); + } + const segments = pathToUriPathSegment(path).split("/"); + // Generate all possible suffixes from shortest to longest + const suffixes: string[] = []; + for (let i = segments.length - 1; i >= 0; i--) { + suffixes.push(segments.slice(i).join("/")); } + console.log(suffixes); + + // For each suffix, try to find a unique matching directory + for (const suffix of suffixes) { + const uris = dirs.map((dir) => ({ + dir, + partialUri: joinPathsToUri(dir, suffix), + })); + const promises = uris.map(async ({ partialUri, dir }) => { + const exists = await ide.fileExists(partialUri); + return { + dir, + partialUri, + exists, + }; + }); + const existenceChecks = await Promise.all(promises); + + const existingUris = existenceChecks.filter(({ exists }) => exists); - const workspaces = await ide.getWorkspaceDirs(); - for (const workspace of workspaces) { - const fullPath = `${workspace}${pathSep}${path}`; - if (await ide.fileExists(fullPath)) { - return fullPath; + // If exactly one directory matches, use it + if (existingUris.length === 1) { + return joinPathsToUri(existingUris[0].dir, segments.join("/")); } } - return undefined; + // If no unique match found, use the first directory + return joinPathsToUri(dirs[0], path); +} + +interface ResolveResult { + resolvedUri: string; + matchedDir: string; } diff --git a/core/util/index.test.ts b/core/util/index.test.ts index 1c6b4a240b..74ee3ccc86 100644 --- a/core/util/index.test.ts +++ b/core/util/index.test.ts @@ -3,58 +3,12 @@ import { dedent, dedentAndGetCommonWhitespace, deduplicateArray, - getLastNPathParts, copyOf, - getBasename, getMarkdownLanguageTagForFile, - getRelativePath, - getUniqueFilePath, - groupByLastNPathParts, removeCodeBlocksAndTrim, removeQuotesAndEscapes, - shortestRelativePaths, - splitPath, } from "./"; -describe("getLastNPathParts", () => { - test("returns the last N parts of a filepath with forward slashes", () => { - const filepath = "home/user/documents/project/file.txt"; - expect(getLastNPathParts(filepath, 2)).toBe("project/file.txt"); - }); - - test("returns the last N parts of a filepath with backward slashes", () => { - const filepath = "C:\\home\\user\\documents\\project\\file.txt"; - expect(getLastNPathParts(filepath, 3)).toBe("documents/project/file.txt"); - }); - - test("returns the last part if N is 1", () => { - const filepath = "/home/user/documents/project/file.txt"; - expect(getLastNPathParts(filepath, 1)).toBe("file.txt"); - }); - - test("returns the entire path if N is greater than the number of parts", () => { - const filepath = "home/user/documents/project/file.txt"; - expect(getLastNPathParts(filepath, 10)).toBe( - "home/user/documents/project/file.txt", - ); - }); - - test("returns an empty string if N is 0", () => { - const filepath = "home/user/documents/project/file.txt"; - expect(getLastNPathParts(filepath, 0)).toBe(""); - }); - - test("handles paths with mixed forward and backward slashes", () => { - const filepath = "home\\user/documents\\project/file.txt"; - expect(getLastNPathParts(filepath, 3)).toBe("documents/project/file.txt"); - }); - - test("handles edge case with empty filepath", () => { - const filepath = ""; - expect(getLastNPathParts(filepath, 2)).toBe(""); - }); -}); - describe("deduplicateArray", () => { it("should return an empty array when given an empty array", () => { const result = deduplicateArray([], (a, b) => a === b); @@ -414,199 +368,6 @@ describe("removeQuotesAndEscapes", () => { }); }); -describe("getBasename", () => { - it("should return the base name of a Unix-style path", () => { - const filepath = "/home/user/documents/file.txt"; - const output = getBasename(filepath); - expect(output).toBe("file.txt"); - }); - - it("should return the base name of a Windows-style path", () => { - const filepath = "C:\\Users\\User\\Documents\\file.txt"; - const output = getBasename(filepath); - expect(output).toBe("file.txt"); - }); - - it("should handle paths with mixed separators", () => { - const filepath = "C:/Users\\User/Documents/file.txt"; - const output = getBasename(filepath); - expect(output).toBe("file.txt"); - }); - - it("should return an empty string for empty input", () => { - const filepath = ""; - const output = getBasename(filepath); - expect(output).toBe(""); - }); -}); - -describe("groupByLastNPathParts", () => { - it("should group filepaths by their last N parts", () => { - const filepaths = [ - "/a/b/c/d/file1.txt", - "/x/y/z/file1.txt", - "/a/b/c/d/file2.txt", - ]; - const output = groupByLastNPathParts(filepaths, 2); - expect(output).toEqual({ - "d/file1.txt": ["/a/b/c/d/file1.txt"], - "z/file1.txt": ["/x/y/z/file1.txt"], - "d/file2.txt": ["/a/b/c/d/file2.txt"], - }); - }); - - it("should handle an empty array", () => { - const filepaths: string[] = []; - const output = groupByLastNPathParts(filepaths, 2); - expect(output).toEqual({}); - }); - - it("should handle N greater than path parts", () => { - const filepaths = ["/file.txt"]; - const output = groupByLastNPathParts(filepaths, 5); - expect(output).toEqual({ "/file.txt": ["/file.txt"] }); - }); -}); - -describe("getUniqueFilePath", () => { - it("should return a unique file path within the group", () => { - const item = "/a/b/c/file.txt"; - const itemGroups = { - "c/file.txt": ["/a/b/c/file.txt", "/x/y/c/file.txt"], - }; - const output = getUniqueFilePath(item, itemGroups); - expect(output).toBe("b/c/file.txt"); - }); - - it("should return the last two parts if unique", () => { - const item = "/a/b/c/file.txt"; - const itemGroups = { - "c/file.txt": ["/a/b/c/file.txt"], - }; - const output = getUniqueFilePath(item, itemGroups); - expect(output).toBe("c/file.txt"); - }); - - it("should handle when additional parts are needed to make it unique", () => { - const item = "/a/b/c/d/e/file.txt"; - const itemGroups = { - "e/file.txt": ["/a/b/c/d/e/file.txt", "/x/y/z/e/file.txt"], - }; - const output = getUniqueFilePath(item, itemGroups); - expect(output).toBe("d/e/file.txt"); - }); -}); - -describe("shortestRelativePaths", () => { - it("should return shortest unique paths", () => { - const paths = [ - "/a/b/c/file.txt", - "/a/b/d/file.txt", - "/a/b/d/file2.txt", - "/x/y/z/file.txt", - ]; - const output = shortestRelativePaths(paths); - expect(output).toEqual([ - "c/file.txt", - "d/file.txt", - "file2.txt", - "z/file.txt", - ]); - }); - - it("should handle empty array", () => { - const paths: string[] = []; - const output = shortestRelativePaths(paths); - expect(output).toEqual([]); - }); - - it("should handle paths with same base names", () => { - const paths = [ - "/a/b/c/d/file.txt", - "/a/b/c/e/file.txt", - "/a/b/f/g/h/file.txt", - ]; - const output = shortestRelativePaths(paths); - expect(output).toEqual(["d/file.txt", "e/file.txt", "h/file.txt"]); - }); - - it("should handle paths where entire path is needed", () => { - const paths = ["/a/b/c/file.txt", "/a/b/c/file.txt", "/a/b/c/file.txt"]; - const output = shortestRelativePaths(paths); - expect(output).toEqual(["file.txt", "file.txt", "file.txt"]); - }); -}); - -describe("splitPath", () => { - it("should split Unix-style paths", () => { - const path = "/a/b/c/d/e.txt"; - const output = splitPath(path); - expect(output).toEqual(["", "a", "b", "c", "d", "e.txt"]); - }); - - it("should split Windows-style paths", () => { - const path = "C:\\Users\\User\\Documents\\file.txt"; - const output = splitPath(path); - expect(output).toEqual(["C:", "Users", "User", "Documents", "file.txt"]); - }); - - it("should handle withRoot parameter", () => { - const path = "/a/b/c/d/e.txt"; - const withRoot = "/a/b"; - const output = splitPath(path, withRoot); - expect(output).toEqual(["b", "c", "d", "e.txt"]); - }); - - it("should handle empty path", () => { - const path = ""; - const output = splitPath(path); - expect(output).toEqual([""]); - }); - - it("should handle paths with multiple consecutive separators", () => { - const path = "/a//b/c/d/e.txt"; - const output = splitPath(path); - expect(output).toEqual(["", "a", "", "b", "c", "d", "e.txt"]); - }); -}); - -describe("getRelativePath", () => { - it("should return the relative path with respect to workspace directories", () => { - const filepath = "/workspace/project/src/file.ts"; - const workspaceDirs = ["/workspace/project"]; - const output = getRelativePath(filepath, workspaceDirs); - expect(output).toBe("src/file.ts"); - }); - - it("should return the filename if not in any workspace", () => { - const filepath = "/other/place/file.ts"; - const workspaceDirs = ["/workspace/project"]; - const output = getRelativePath(filepath, workspaceDirs); - expect(output).toBe("file.ts"); - }); - - it("should handle multiple workspace directories", () => { - const filepath = "/workspace2/project/src/file.ts"; - const workspaceDirs = ["/workspace/project", "/workspace2/project"]; - const output = getRelativePath(filepath, workspaceDirs); - expect(output).toBe("src/file.ts"); - }); - - it("should handle Windows-style paths", () => { - const filepath = "C:\\workspace\\project\\src\\file.ts"; - const workspaceDirs = ["C:\\workspace\\project"]; - const output = getRelativePath(filepath, workspaceDirs); - expect(output).toBe("src/file.ts"); - }); - - it("should handle paths with spaces or special characters", () => { - const filepath = "/workspace/project folder/src/file.ts"; - const workspaceDirs = ["/workspace/project folder"]; - const output = getRelativePath(filepath, workspaceDirs); - expect(output).toBe("src/file.ts"); - }); -}); - describe("getMarkdownLanguageTagForFile", () => { it("should return correct language tag for known extensions", () => { expect(getMarkdownLanguageTagForFile("test.py")).toBe("python"); diff --git a/core/util/index.ts b/core/util/index.ts index 5b982350e8..04dc6d5675 100644 --- a/core/util/index.ts +++ b/core/util/index.ts @@ -66,131 +66,6 @@ export function dedentAndGetCommonWhitespace(s: string): [string, string] { return [lines.map((x) => x.replace(lcp, "")).join("\n"), lcp]; } -const SEP_REGEX = /[\\/]/; - -export function getBasename(filepath: string): string { - return filepath.split(SEP_REGEX).pop() ?? ""; -} - -export function getLastNPathParts(filepath: string, n: number): string { - if (n <= 0) { - return ""; - } - return filepath.split(SEP_REGEX).slice(-n).join("/"); -} - -export function groupByLastNPathParts( - filepaths: string[], - n: number, -): Record { - return filepaths.reduce( - (groups, item) => { - const lastNParts = getLastNPathParts(item, n); - if (!groups[lastNParts]) { - groups[lastNParts] = []; - } - groups[lastNParts].push(item); - return groups; - }, - {} as Record, - ); -} - -export function getUniqueFilePath( - item: string, - itemGroups: Record, -): string { - const lastTwoParts = getLastNPathParts(item, 2); - const group = itemGroups[lastTwoParts]; - - let n = 2; - if (group.length > 1) { - while ( - group.some( - (otherItem) => - otherItem !== item && - getLastNPathParts(otherItem, n) === getLastNPathParts(item, n), - ) - ) { - n++; - } - } - - return getLastNPathParts(item, n); -} - -export function shortestRelativePaths(paths: string[]): string[] { - if (paths.length === 0) { - return []; - } - - const partsLengths = paths.map((x) => x.split(SEP_REGEX).length); - const currentRelativePaths = paths.map(getBasename); - const currentNumParts = paths.map(() => 1); - const isDuplicated = currentRelativePaths.map( - (x, i) => - currentRelativePaths.filter((y, j) => y === x && paths[i] !== paths[j]) - .length > 1, - ); - - while (isDuplicated.some(Boolean)) { - const firstDuplicatedPath = currentRelativePaths.find( - (x, i) => isDuplicated[i], - ); - if (!firstDuplicatedPath) { - break; - } - - currentRelativePaths.forEach((x, i) => { - if (x === firstDuplicatedPath) { - currentNumParts[i] += 1; - currentRelativePaths[i] = getLastNPathParts( - paths[i], - currentNumParts[i], - ); - } - }); - - isDuplicated.forEach((x, i) => { - if (x) { - isDuplicated[i] = - // Once we've used up all the parts, we can't make it longer - currentNumParts[i] < partsLengths[i] && - currentRelativePaths.filter((y) => y === currentRelativePaths[i]) - .length > 1; - } - }); - } - - return currentRelativePaths; -} - -export function splitPath(path: string, withRoot?: string): string[] { - let parts = path.includes("/") ? path.split("/") : path.split("\\"); - if (withRoot !== undefined) { - const rootParts = splitPath(withRoot); - parts = parts.slice(rootParts.length - 1); - } - return parts; -} - -export function getRelativePath( - filepath: string, - workspaceDirs: string[], -): string { - for (const workspaceDir of workspaceDirs) { - const filepathParts = splitPath(filepath); - const workspaceDirParts = splitPath(workspaceDir); - if ( - filepathParts.slice(0, workspaceDirParts.length).join("/") === - workspaceDirParts.join("/") - ) { - return filepathParts.slice(workspaceDirParts.length).join("/"); - } - } - return splitPath(filepath).pop() ?? ""; // If the file is not in any of the workspaces, return the plain filename -} - export function getMarkdownLanguageTagForFile(filepath: string): string { const extToLangMap: { [key: string]: string } = { py: "python", diff --git a/core/util/pathModule.ts b/core/util/pathModule.ts deleted file mode 100644 index 8b06ab56c4..0000000000 --- a/core/util/pathModule.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { PlatformPath, posix, win32 } from "node:path"; -import { IDE } from ".."; - -export type PathSep = "/" | "\\"; - -function getPathModuleFromPathSep(pathSep: PathSep): PlatformPath { - return pathSep === "/" ? posix : win32; -} - -export async function getPathModuleForIde(ide: IDE): Promise { - const pathSep = await ide.pathSep(); - return getPathModuleFromPathSep(pathSep as PathSep); -} diff --git a/core/util/pathToUri.ts b/core/util/pathToUri.ts new file mode 100644 index 0000000000..804b4fa999 --- /dev/null +++ b/core/util/pathToUri.ts @@ -0,0 +1,21 @@ +import { fileURLToPath, pathToFileURL } from "url"; + +import * as URI from "uri-js"; + +// CAN ONLY BE USED IN CORE + +// Converts a local path to a file:// URI +export function localPathToUri(path: string) { + const url = pathToFileURL(path); + return URI.normalize(url.toString()); +} + +export function localPathOrUriToPath(localPathOrUri: string): string { + try { + return fileURLToPath(localPathOrUri); + } catch (e) { + // console.log("Received local filepath", localPathOrUri); + + return localPathOrUri; + } +} diff --git a/core/util/paths.ts b/core/util/paths.ts index 83ec2087c5..e5ed99caea 100644 --- a/core/util/paths.ts +++ b/core/util/paths.ts @@ -1,9 +1,9 @@ -import * as JSONC from "comment-json"; -import dotenv from "dotenv"; import * as fs from "fs"; import * as os from "os"; import * as path from "path"; -import { pathToFileURL } from "url"; + +import * as JSONC from "comment-json"; +import dotenv from "dotenv"; import { IdeType, SerializedContinueConfig } from "../"; import { defaultConfig, defaultConfigJetBrains } from "../config/default"; @@ -45,9 +45,6 @@ export function getGlobalContinueIgnorePath(): string { return continueIgnorePath; } -/* - Deprecated, replace with getContinueGlobalUri where possible -*/ export function getContinueGlobalPath(): string { // This is ~/.continue on mac/linux const continuePath = CONTINUE_GLOBAL_DIR; @@ -57,10 +54,6 @@ export function getContinueGlobalPath(): string { return continuePath; } -export function getContinueGlobalUri(): string { - return pathToFileURL(CONTINUE_GLOBAL_DIR).href; -} - export function getSessionsFolderPath(): string { const sessionsPath = path.join(getContinueGlobalPath(), "sessions"); if (!fs.existsSync(sessionsPath)) { @@ -341,6 +334,15 @@ export function getLogsDirPath(): string { return logsPath; } +export function getLogFilePath(): string { + const logFilePath = path.join(getContinueGlobalPath(), "continue.log"); + // Make sure the file/directory exist + if (!fs.existsSync(logFilePath)) { + fs.writeFileSync(logFilePath, ""); + } + return logFilePath; +} + export function getCoreLogsPath(): string { return path.join(getLogsDirPath(), "core.log"); } @@ -399,3 +401,13 @@ export function setupInitialDotContinueDirectory() { } }); } + +export function getDiffsDirectoryPath(): string { + const diffsPath = path.join(getContinueGlobalPath(), ".diffs"); // .replace(/^C:/, "c:"); ?? + if (!fs.existsSync(diffsPath)) { + fs.mkdirSync(diffsPath, { + recursive: true, + }); + } + return diffsPath; +} diff --git a/core/util/treeSitter.ts b/core/util/treeSitter.ts index 5f7ebfb73f..067445994b 100644 --- a/core/util/treeSitter.ts +++ b/core/util/treeSitter.ts @@ -1,8 +1,9 @@ import fs from "node:fs"; -import * as path from "node:path"; +import path from "path"; import Parser, { Language } from "web-tree-sitter"; import { FileSymbolMap, IDE, SymbolWithRange } from ".."; +import { getUriFileExtension } from "./uri"; export enum LanguageName { CPP = "cpp", @@ -145,7 +146,7 @@ export async function getLanguageForFile( ): Promise { try { await Parser.init(); - const extension = path.extname(filepath).slice(1); + const extension = getUriFileExtension(filepath); const languageName = supportedLanguages[extension]; if (!languageName) { @@ -165,7 +166,8 @@ export async function getLanguageForFile( } export const getFullLanguageName = (filepath: string) => { - return supportedLanguages[filepath.split(".").pop() ?? ""]; + const extension = getUriFileExtension(filepath); + return supportedLanguages[extension]; }; export async function getQueryForFile( @@ -226,7 +228,6 @@ export async function getSymbolsForFile( contents: string, ): Promise { const parser = await getParserForFile(filepath); - if (!parser) { return; } @@ -286,7 +287,6 @@ export async function getSymbolsForFile( node.children.forEach(findNamedNodesRecursive); } findNamedNodesRecursive(tree.rootNode); - return symbols; } diff --git a/core/util/uri.test.ts b/core/util/uri.test.ts new file mode 100644 index 0000000000..9e24710542 --- /dev/null +++ b/core/util/uri.test.ts @@ -0,0 +1,181 @@ +// Generated by continue +import * as URI from "uri-js"; +import { + pathToUriPathSegment, + findUriInDirs, + relativePathOrUriToUri, + getUriPathBasename, + getUriFileExtension, + getFileExtensionFromBasename, + getLastNUriRelativePathParts, + joinPathsToUri, + getShortestUniqueRelativeUriPaths, + getLastNPathParts, +} from "./uri"; + +describe("uri utils", () => { + describe("pathToUriPathSegment", () => { + it("should convert Windows paths to URI segments", () => { + expect(pathToUriPathSegment("\\path\\to\\folder\\")).toBe( + "path/to/folder", + ); + expect(pathToUriPathSegment("\\this\\is\\afile.ts")).toBe( + "this/is/afile.ts", + ); + }); + + it("should clean Unix paths", () => { + expect(pathToUriPathSegment("/path/to/folder/")).toBe("path/to/folder"); + expect(pathToUriPathSegment("is/already/clean")).toBe("is/already/clean"); + }); + + it("should encode URI special characters", () => { + expect(pathToUriPathSegment("path/with spaces/file")).toBe( + "path/with%20spaces/file", + ); + expect(pathToUriPathSegment("special#chars?in&path")).toBe( + "special%23chars%3Fin%26path", + ); + }); + }); + + describe("findUriInDirs", () => { + const dirUris = [ + "file:///workspace/project1", + "file:///workspace/project2", + ]; + + it("should find URI in directory", () => { + const result = findUriInDirs( + "file:///workspace/project1/src/file.ts", + dirUris, + ); + expect(result).toEqual({ + uri: "file:///workspace/project1/src/file.ts", + relativePathOrBasename: "src/file.ts", + foundInDir: "file:///workspace/project1", + }); + }); + + it("should return basename when URI not in directories", () => { + const result = findUriInDirs("file:///other/location/file.ts", dirUris); + expect(result).toEqual({ + uri: "file:///other/location/file.ts", + relativePathOrBasename: "file.ts", + foundInDir: null, + }); + }); + + it("should throw error for invalid URIs", () => { + expect(() => findUriInDirs("invalid-uri", dirUris)).toThrow( + "Invalid uri: invalid-uri", + ); + }); + }); + + describe("URI path operations", () => { + it("should get URI path basename", () => { + expect(getUriPathBasename("file:///path/to/file.txt")).toBe("file.txt"); + expect(getUriPathBasename("file:///path/to/folder/")).toBe("folder"); + }); + + it("should get URI file extension", () => { + expect(getUriFileExtension("file:///path/to/file.TXT")).toBe("txt"); + expect(getUriFileExtension("file:///path/to/file")).toBe(""); + }); + + it("should get file extension from basename", () => { + expect(getFileExtensionFromBasename("file.txt")).toBe("txt"); + expect(getFileExtensionFromBasename("file")).toBe(""); + }); + }); + + describe("joinPathsToUri", () => { + it("should join paths to base URI", () => { + expect(joinPathsToUri("file:///base", "path", "to", "file.txt")).toBe( + "file:///base/path/to/file.txt", + ); + }); + + it("should handle paths with special characters", () => { + expect( + joinPathsToUri("file:///base", "path with spaces", "file.txt"), + ).toBe("file:///base/path%20with%20spaces/file.txt"); + }); + }); + + describe("getShortestUniqueRelativeUriPaths", () => { + it("should find shortest unique paths", () => { + const uris = [ + "file:///workspace/project1/src/components/Button.tsx", + "file:///workspace/project1/src/utils/Button.tsx", + ]; + const dirUris = ["file:///workspace/project1"]; + + const result = getShortestUniqueRelativeUriPaths(uris, dirUris); + expect(result).toEqual([ + { + uri: "file:///workspace/project1/src/components/Button.tsx", + uniquePath: "components/Button.tsx", + }, + { + uri: "file:///workspace/project1/src/utils/Button.tsx", + uniquePath: "utils/Button.tsx", + }, + ]); + }); + }); + + describe("getLastNPathParts", () => { + it("should get last N parts of path", () => { + expect(getLastNPathParts("path/to/some/file.txt", 2)).toBe( + "some/file.txt", + ); + expect(getLastNPathParts("path/to/some/file.txt", 0)).toBe(""); + }); + + it("should handle Windows paths", () => { + expect(getLastNPathParts("path\\to\\some\\file.txt", 2)).toBe( + "some/file.txt", + ); + }); + }); + + describe("relativePathOrUriToUri", () => { + const consoleSpy = jest.spyOn(console, "trace").mockImplementation(); + + afterEach(() => { + consoleSpy.mockClear(); + }); + + it("should return original URI if scheme exists", () => { + const uri = "file:///path/to/file.txt"; + expect(relativePathOrUriToUri(uri, "file:///base")).toBe(uri); + }); + + it("should convert relative path to URI", () => { + expect(relativePathOrUriToUri("path/to/file.txt", "file:///base")).toBe( + "file:///base/path/to/file.txt", + ); + expect(consoleSpy).toHaveBeenCalledWith("Received path with no scheme"); + }); + }); + + describe("getLastNUriRelativePathParts", () => { + it("should get last N parts of URI relative path", () => { + const dirUris = ["file:///workspace/project1"]; + const uri = "file:///workspace/project1/src/components/Button.tsx"; + + expect(getLastNUriRelativePathParts(dirUris, uri, 2)).toBe( + "components/Button.tsx", + ); + }); + + it("should handle URI not in directories", () => { + const dirUris = ["file:///workspace/project1"]; + const uri = "file:///other/path/file.txt"; + + expect(getLastNUriRelativePathParts(dirUris, uri, 2)).toBe("file.txt"); + }); + }); +}); diff --git a/core/util/uri.ts b/core/util/uri.ts new file mode 100644 index 0000000000..49a9e8389d --- /dev/null +++ b/core/util/uri.ts @@ -0,0 +1,190 @@ +import * as URI from "uri-js"; + +/** Converts any OS path to cleaned up URI path segment format with no leading/trailing slashes + e.g. \path\to\folder\ -> path/to/folder + \this\is\afile.ts -> this/is/afile.ts + is/already/clean -> is/already/clean + **/ + +export function pathToUriPathSegment(path: string) { + let clean = path.replace(/[\\]/g, "/"); // backslashes -> forward slashes + clean = clean.replace(/^\//, ""); // remove start slash + clean = clean.replace(/\/$/, ""); // remove end slash + return clean + .split("/") + .map((segment) => encodeURIComponent(segment)) + .join("/"); +} + +export function getCleanUriPath(uri: string) { + const path = URI.parse(uri).path; + if (!path) { + return ""; + } + return pathToUriPathSegment(path); +} + +export function findUriInDirs( + uri: string, + dirUriCandidates: string[], +): { + uri: string; + relativePathOrBasename: string; + foundInDir: string | null; +} { + const uriComps = URI.parse(uri); + if (!uriComps.scheme) { + throw new Error(`Invalid uri: ${uri}`); + } + for (const dir of dirUriCandidates) { + const dirComps = URI.parse(dir); + + if (!dirComps.scheme) { + throw new Error(`Invalid uri: ${dir}`); + } + + if (uriComps.scheme !== dirComps.scheme) { + continue; + } + // Can't just use startsWith because e.g. + // file:///folder/file is not within file:///fold + + // At this point we break the path up and check if each dir path part matches + const dirPathParts = (dirComps.path ?? "") + .replace(/^\//, "") + .split("/") + .map((part) => encodeURIComponent(part)); + const uriPathParts = (uriComps.path ?? "") + .replace(/^\//, "") + .split("/") + .map((part) => encodeURIComponent(part)); + + if (uriPathParts.length < dirPathParts.length) { + continue; + } + let allDirPartsMatch = true; + for (let i = 0; i < dirPathParts.length; i++) { + if (dirPathParts[i] !== uriPathParts[i]) { + allDirPartsMatch = false; + } + } + if (allDirPartsMatch) { + const relativePath = uriPathParts.slice(dirPathParts.length).join("/"); + return { + uri, + relativePathOrBasename: relativePath, + foundInDir: dir, + }; + } + } + // Not found + console.trace("Directory not found for uri", uri, dirUriCandidates); + return { + uri, + relativePathOrBasename: getUriPathBasename(uri), + foundInDir: null, + }; +} + +/* + To smooth out the transition from path to URI will use this function to warn when path is used + This will NOT work consistently with full OS paths like c:\blah\blah or ~/Users/etc +*/ +export function relativePathOrUriToUri( + relativePathOrUri: string, + defaultDirUri: string, +): string { + const out = URI.parse(relativePathOrUri); + if (out.scheme) { + return relativePathOrUri; + } + console.trace("Received path with no scheme"); + return joinPathsToUri(defaultDirUri, out.path ?? ""); +} + +/* + Returns just the file or folder name of a URI +*/ +export function getUriPathBasename(uri: string): string { + const cleanPath = getCleanUriPath(uri); + return cleanPath.split("/")?.pop() || ""; +} + +export function getFileExtensionFromBasename(basename: string) { + const parts = basename.split("."); + if (parts.length < 2) { + return ""; + } + return (parts.slice(-1)[0] ?? "").toLowerCase(); +} + +/* + Returns the file extension of a URI +*/ +export function getUriFileExtension(uri: string) { + const baseName = getUriPathBasename(uri); + return getFileExtensionFromBasename(baseName); +} + +export function getLastNUriRelativePathParts( + dirUriCandidates: string[], + uri: string, + n: number, +): string { + const { relativePathOrBasename } = findUriInDirs(uri, dirUriCandidates); + return getLastNPathParts(relativePathOrBasename, n); +} + +export function joinPathsToUri(uri: string, ...pathSegments: string[]) { + const components = URI.parse(uri); + const segments = pathSegments.map((segment) => pathToUriPathSegment(segment)); + components.path = `${components.path}/${segments.join("/")}`; + return URI.serialize(components); +} + +export function getShortestUniqueRelativeUriPaths( + uris: string[], + dirUriCandidates: string[], +): { + uri: string; + uniquePath: string; +}[] { + // Split all URIs into segments and count occurrences of each suffix combination + const segmentCombinationsMap = new Map(); + const segmentsInfo = uris.map((uri) => { + const { relativePathOrBasename } = findUriInDirs(uri, dirUriCandidates); + const cleanPath = pathToUriPathSegment(relativePathOrBasename); + const segments = cleanPath.split("/"); + const suffixes: string[] = []; + + // Generate all possible suffix combinations, starting from the shortest (basename) + for (let i = segments.length - 1; i >= 0; i--) { + const suffix = segments.slice(i).join("/"); + suffixes.push(suffix); // Now pushing in order from shortest to longest + // Count occurrences of each suffix + segmentCombinationsMap.set( + suffix, + (segmentCombinationsMap.get(suffix) || 0) + 1, + ); + } + + return { uri, segments, suffixes, cleanPath }; + }); + // Find shortest unique path for each URI + return segmentsInfo.map(({ uri, suffixes, cleanPath }) => { + // Since suffixes are now ordered from shortest to longest, + // the first unique one we find will be the shortest + const uniqueCleanPath = + suffixes.find((suffix) => segmentCombinationsMap.get(suffix) === 1) ?? + cleanPath; // fallback to full path if no unique suffix found + return { uri, uniquePath: decodeURIComponent(uniqueCleanPath) }; + }); +} +// Only used when working with system paths and relative paths +// Since doesn't account for URI segements before workspace +export function getLastNPathParts(filepath: string, n: number): string { + if (n <= 0) { + return ""; + } + return filepath.split(/[\\/]/).slice(-n).join("/"); +} diff --git a/docs/docs/customize/model-providers/more/SambaNova.md b/docs/docs/customize/model-providers/more/SambaNova.md index 9c20ba4d76..f35c56dd52 100644 --- a/docs/docs/customize/model-providers/more/SambaNova.md +++ b/docs/docs/customize/model-providers/more/SambaNova.md @@ -1,6 +1,6 @@ # SambaNova Cloud -The SambaNova Cloud is a cloud platform for running large AI models with the world record Llama 3.1 70B/405B performance. You can sign up [here](https://cloud.sambanova.ai/), copy your API key on the initial welcome screen, and then hit the play button on any model from the [model list](https://community.sambanova.ai/t/quick-start-guide/104). +The SambaNova Cloud is a cloud platform for running large AI models with the world record Llama 3.1 70B/405B performance. You can follow the instructions in [this blog post](https://sambanova.ai/blog/accelerating-coding-with-sambanova-cloud?ref=blog.continue.dev) to configure your setup. ```json title="config.json" { diff --git a/docs/docs/customize/model-providers/more/asksage.md b/docs/docs/customize/model-providers/more/asksage.md index c5ba9218cc..e21133cb8f 100644 --- a/docs/docs/customize/model-providers/more/asksage.md +++ b/docs/docs/customize/model-providers/more/asksage.md @@ -41,21 +41,34 @@ More models, functionalities and documentation will be added in the future for A > We recommend to utilize the`OpenAI` or `Anthropic` models for the best performance and results for the `Chat` and `Edit` functionalities. +## Ask Sage Documentation +Ask Sage documentation is not available with Continue.Dev, so if you have any questions or need help with Ask Sage type `@ask` and select the `Ask Sage` option in the chat. Then procced to ask your question about Ask Sage. + ## Current Models From Ask Sage Supported The current Models available provided by Ask Sage are: -| Model | Added | -|--------------------|-------| -| Gov GPT-4.0 | Yes | -| Gov GPT-4o | Yes | -| GPT-4o | Yes | -| GPT-4o-mini | Yes | -| GPT-3.5-16K | Yes | -| Calude 3 Opus | Yes | -| Calude 3 Sonet | Yes | -| Calude 3.5 Sonnet | Yes | -| Gemini Pro | Yes | -| llama 3 | Yes | -| Mistral Large | Yes | +| Index | Model | Added | Status | +|-------|------------------------|-------|--------| +| 1 | GPT-4 Gov | Yes | ✅ | +| 2 | GPT-4o Gov | Yes | ✅ | +| 3 | GPT-4o-mini Gov | Yes | ✅ | +| 4 | GPT-3.5-Turbo Gov | Yes | ✅ | +| 5 | GPT-4o | Yes | ✅ | +| 6 | GPT-4o-mini | Yes | ✅ | +| 7 | GPT-4 | Yes | ✅ | +| 8 | GPT-4-32K | Yes | ✅ | +| 9 | GPT-o1 | Yes | ✅ | +| 10 | GPT-o1-mini | Yes | ✅ | +| 11 | GPT-3.5-turbo | Yes | ✅ | +| 12 | Calude 3.5 Sonnet Gov | Yes | ✅ | +| 13 | Calude 3 Opus | Yes | ✅ | +| 14 | Calude 3 Sonet | Yes | ✅ | +| 15 | Calude 3.5 Sonnet | Yes | ✅ | +| 16 | Grok (xAI) | Yes | ✅ | +| 17 | Groq Llama 3.3 | Yes | ✅ | +| 18 | Groq 70B | Yes | ✅ | +| 19 | Gemini Pro | Yes | ✅ | +| 20 | llama 3 | Yes | ✅ | +| 21 | Mistral Large | Yes | ✅ | diff --git a/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/activities/ContinuePluginStartupActivity.kt b/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/activities/ContinuePluginStartupActivity.kt index a6b22dad61..a208ea2cd7 100644 --- a/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/activities/ContinuePluginStartupActivity.kt +++ b/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/activities/ContinuePluginStartupActivity.kt @@ -160,21 +160,24 @@ class ContinuePluginStartupActivity : StartupActivity, DumbAware { // Handle file changes and deletions - reindex connection.subscribe(VirtualFileManager.VFS_CHANGES, object : BulkFileListener { override fun after(events: List) { - // Collect all relevant paths for deletions - val deletedPaths = events.filterIsInstance() - .map { event -> event.file.path.split("/").dropLast(1).joinToString("/") } - - // Collect all relevant paths for content changes - val changedPaths = events.filterIsInstance() - .map { event -> event.file.path.split("/").dropLast(1).joinToString("/") } + // Collect all relevant URIs for deletions + val deletedURIs = events.filterIsInstance() + .map { event -> event.file.url } + + // Send "files/deleted" message if there are any deletions + if (deletedURIs.isNotEmpty()) { + val data = mapOf("files" to deletedURIs) + continuePluginService.coreMessenger?.request("files/deleted", data, null) { _ -> } + } - // Combine both lists of paths for re-indexing - val allPaths = deletedPaths + changedPaths + // Collect all relevant URIs for content changes + val changedURIs = events.filterIsInstance() + .map { event -> event.file.url } - // Create a data map if there are any paths to re-index - if (allPaths.isNotEmpty()) { - val data = mapOf("files" to allPaths) - continuePluginService.coreMessenger?.request("index/forceReIndexFiles", data, null) { _ -> } + // Send "files/changed" message if there are any content changes + if (changedURIs.isNotEmpty()) { + val data = mapOf("files" to changedURIs) + continuePluginService.coreMessenger?.request("files/changed", data, null) { _ -> } } } }) @@ -217,12 +220,10 @@ class ContinuePluginStartupActivity : StartupActivity, DumbAware { // Reload the WebView continuePluginService?.let { pluginService -> val allModulePaths = ModuleManager.getInstance(project).modules - .flatMap { module -> ModuleRootManager.getInstance(module).contentRoots.map { it.path } } - .map { Paths.get(it).normalize() } + .flatMap { module -> ModuleRootManager.getInstance(module).contentRoots.map { it.url } } val topLevelModulePaths = allModulePaths .filter { modulePath -> allModulePaths.none { it != modulePath && modulePath.startsWith(it) } } - .map { it.toString() } pluginService.workspacePaths = topLevelModulePaths.toTypedArray() } diff --git a/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/autocomplete/AutocompleteService.kt b/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/autocomplete/AutocompleteService.kt index d4e5e1092b..6279e03f74 100644 --- a/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/autocomplete/AutocompleteService.kt +++ b/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/autocomplete/AutocompleteService.kt @@ -69,7 +69,7 @@ class AutocompleteService(private val project: Project) { val column = editor.caretModel.primaryCaret.logicalPosition.column val input = mapOf( "completionId" to completionId, - "filepath" to virtualFile?.path, + "filepath" to virtualFile?.url, "pos" to mapOf( "line" to editor.caretModel.primaryCaret.logicalPosition.line, "character" to column diff --git a/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/constants/MessageTypes.kt b/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/constants/MessageTypes.kt index 4c8450268f..2464db05a5 100644 --- a/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/constants/MessageTypes.kt +++ b/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/constants/MessageTypes.kt @@ -18,8 +18,6 @@ class MessageTypes { "getTerminalContents", "getWorkspaceDirs", "showLines", - "listFolders", - "getContinueDir", "writeFile", "fileExists", "showVirtualFile", @@ -46,7 +44,6 @@ class MessageTypes { "applyToFile", "getGitHubAuthToken", "setGitHubAuthToken", - "pathSep", "getControlPlaneSessionInfo", "logoutOfControlPlane", "getTerminalContents", diff --git a/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/ConfigJsonSchemaProviderFactory.kt b/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/ConfigJsonSchemaProviderFactory.kt index 2056d216e1..677506f6e6 100644 --- a/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/ConfigJsonSchemaProviderFactory.kt +++ b/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/ConfigJsonSchemaProviderFactory.kt @@ -22,7 +22,7 @@ class ConfigJsonSchemaProviderFactory : JsonSchemaProviderFactory { class ConfigJsonSchemaFileProvider : JsonSchemaFileProvider { override fun isAvailable(file: VirtualFile): Boolean { - return file.path.endsWith("/.continue/config.json") + return file.path.endsWith("/.continue/config.json") || file.path.endsWith("\\.continue\\config.json") } override fun getName(): String { diff --git a/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/Diffs.kt b/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/Diffs.kt index d7e9124998..750737f1bc 100644 --- a/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/Diffs.kt +++ b/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/Diffs.kt @@ -18,6 +18,7 @@ import com.intellij.openapi.util.Disposer import com.intellij.openapi.vfs.LocalFileSystem import java.awt.Toolkit import java.io.File +import java.net.URI import java.nio.file.Paths import javax.swing.Action import javax.swing.JComponent @@ -61,7 +62,7 @@ class DiffManager(private val project: Project) : DumbAware { file.createNewFile() } file.writeText(replacement) - openDiffWindow(filepath, file.path, stepIndex) + openDiffWindow(URI(filepath).toString(), file.toURI().toString(), stepIndex) } private fun cleanUpFile(file2: String) { @@ -79,7 +80,7 @@ class DiffManager(private val project: Project) : DumbAware { val diffInfo = diffInfoMap[file] ?: return // Write contents to original file - val virtualFile = LocalFileSystem.getInstance().findFileByPath(diffInfo.originalFilepath) ?: return + val virtualFile = LocalFileSystem.getInstance().findFileByPath(URI(diffInfo.originalFilepath).path) ?: return val document = FileDocumentManager.getInstance().getDocument(virtualFile) ?: return WriteCommandAction.runWriteCommandAction(project) { document.setText(File(file).readText()) @@ -118,8 +119,8 @@ class DiffManager(private val project: Project) : DumbAware { lastFile2 = file2 // Create a DiffContent for each of the texts you want to compare - val content1: DiffContent = DiffContentFactory.getInstance().create(File(file1).readText()) - val content2: DiffContent = DiffContentFactory.getInstance().create(File(file2).readText()) + val content1: DiffContent = DiffContentFactory.getInstance().create(File(URI(file1)).readText()) + val content2: DiffContent = DiffContentFactory.getInstance().create(File(URI(file2)).readText()) // Create a SimpleDiffRequest and populate it with the DiffContents and titles val diffRequest = SimpleDiffRequest("Continue Diff", content1, content2, "Old", "New") diff --git a/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/IdeProtocolClient.kt b/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/IdeProtocolClient.kt index 05cb8dbeb9..7efc44c79d 100644 --- a/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/IdeProtocolClient.kt +++ b/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/IdeProtocolClient.kt @@ -295,11 +295,6 @@ class IdeProtocolClient( respond(exists) } - "getContinueDir" -> { - val continueDir = ide.getContinueDir() - respond(continueDir) - } - "openFile" -> { val params = Gson().fromJson( dataElement.toString(), @@ -335,11 +330,6 @@ class IdeProtocolClient( respond(result) } - "listFolders" -> { - val folders = ide.listFolders() - respond(folders) - } - "getSearchResults" -> { val params = Gson().fromJson( dataElement.toString(), @@ -399,12 +389,7 @@ class IdeProtocolClient( ide.openUrl(url) respond(null) } - - "pathSep" -> { - val sep = ide.pathSep() - respond(sep) - } - + "insertAtCursor" -> { val params = Gson().fromJson( dataElement.toString(), @@ -563,7 +548,7 @@ class IdeProtocolClient( val endChar = endOffset - document.getLineStartOffset(endLine) return@runReadAction RangeInFileWithContents( - virtualFile.path, Range( + virtualFile.url, Range( Position(startLine, startChar), Position(endLine, endChar) ), selectedText diff --git a/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/IntelliJIde.kt b/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/IntelliJIde.kt index e883df441f..3198247d65 100644 --- a/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/IntelliJIde.kt +++ b/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/IntelliJIde.kt @@ -1,6 +1,5 @@ import com.github.continuedev.continueintellijextension.* import com.github.continuedev.continueintellijextension.constants.getContinueGlobalPath -import com.github.continuedev.continueintellijextension.`continue`.DiffManager import com.github.continuedev.continueintellijextension.services.ContinueExtensionSettings import com.github.continuedev.continueintellijextension.services.ContinuePluginService import com.github.continuedev.continueintellijextension.utils.OS @@ -108,7 +107,7 @@ class IntelliJIDE( } else { ProcessBuilder("git", "diff", "--cached") } - builder.directory(File(workspaceDir)) + builder.directory(File(URI(workspaceDir))) val process = withContext(Dispatchers.IO) { builder.start() } @@ -168,19 +167,6 @@ class IntelliJIDE( throw NotImplementedError("getAvailableThreads not implemented yet") } - override suspend fun listFolders(): List { - val workspacePath = this.workspacePath ?: return emptyList() - val folders = mutableListOf() - fun findNestedFolders(dirPath: String) { - val dir = File(dirPath) - val nestedFolders = dir.listFiles { file -> file.isDirectory }?.map { it.absolutePath } ?: emptyList() - folders.addAll(nestedFolders) - nestedFolders.forEach { folder -> findNestedFolders(folder) } - } - findNestedFolders(workspacePath) - return folders - } - override suspend fun getWorkspaceDirs(): List { return workspaceDirectories().toList() } @@ -191,16 +177,14 @@ class IntelliJIDE( val configs = mutableListOf() for (workspaceDir in workspaceDirs) { - val workspacePath = File(workspaceDir) - val dir = VirtualFileManager.getInstance().findFileByUrl("file://$workspacePath") + val dir = VirtualFileManager.getInstance().findFileByUrl(workspaceDir) if (dir != null) { - val contents = dir.children.map { it.name } + val contents = dir.children.map { it.url } // Find any .continuerc.json files for (file in contents) { if (file.endsWith(".continuerc.json")) { - val filePath = workspacePath.resolve(file) - val fileContent = File(filePath.toString()).readText() + val fileContent = File(URI(file)).readText() configs.add(fileContent) } } @@ -211,12 +195,12 @@ class IntelliJIDE( } override suspend fun fileExists(filepath: String): Boolean { - val file = File(filepath) + val file = File(URI(filepath)) return file.exists() } override suspend fun writeFile(path: String, contents: String) { - val file = File(path) + val file = File(URI(path)) file.writeText(contents) } @@ -232,7 +216,7 @@ class IntelliJIDE( } override suspend fun openFile(path: String) { - val file = LocalFileSystem.getInstance().findFileByPath(path) + val file = LocalFileSystem.getInstance().findFileByPath(URI(path).path) file?.let { ApplicationManager.getApplication().invokeLater { FileEditorManager.getInstance(project).openFile(it, true) @@ -252,7 +236,7 @@ class IntelliJIDE( override suspend fun saveFile(filepath: String) { ApplicationManager.getApplication().invokeLater { - val file = LocalFileSystem.getInstance().findFileByPath(filepath) ?: return@invokeLater + val file = LocalFileSystem.getInstance().findFileByPath(URI(filepath).path) ?: return@invokeLater val fileDocumentManager = FileDocumentManager.getInstance() val document = fileDocumentManager.getDocument(file) @@ -265,7 +249,7 @@ class IntelliJIDE( override suspend fun readFile(filepath: String): String { return try { val content = ApplicationManager.getApplication().runReadAction { - val virtualFile = LocalFileSystem.getInstance().findFileByPath(filepath) + val virtualFile = LocalFileSystem.getInstance().findFileByPath(URI(filepath).path) if (virtualFile != null && FileDocumentManager.getInstance().isFileModified(virtualFile)) { return@runReadAction FileDocumentManager.getInstance().getDocument(virtualFile)?.text } @@ -275,7 +259,7 @@ class IntelliJIDE( if (content != null) { content } else { - val file = File(filepath) + val file = File(URI(filepath)) if (!file.exists()) return "" withContext(Dispatchers.IO) { FileInputStream(file).use { fis -> @@ -322,7 +306,7 @@ class IntelliJIDE( override suspend fun getOpenFiles(): List { val fileEditorManager = FileEditorManager.getInstance(project) - return fileEditorManager.openFiles.map { it.path }.toList() + return fileEditorManager.openFiles.map { it.url }.toList() } override suspend fun getCurrentFile(): Map? { @@ -331,7 +315,7 @@ class IntelliJIDE( val virtualFile = editor?.document?.let { FileDocumentManager.getInstance().getFile(it) } return virtualFile?.let { mapOf( - "path" to it.path, + "path" to it.url, "contents" to editor.document.text, "isUntitled" to false ) @@ -397,7 +381,7 @@ class IntelliJIDE( problems.add( Problem( - filepath = psiFile.virtualFile?.path ?: "", + filepath = psiFile.virtualFile?.url ?: "", range = Range( start = Position( line = startLineNumber, @@ -422,7 +406,7 @@ class IntelliJIDE( return withContext(Dispatchers.IO) { try { val builder = ProcessBuilder("git", "rev-parse", "--abbrev-ref", "HEAD") - builder.directory(File(dir)) + builder.directory(File(URI(dir))) val process = builder.start() val reader = BufferedReader(InputStreamReader(process.inputStream)) val output = reader.readLine() @@ -452,7 +436,7 @@ class IntelliJIDE( override suspend fun getRepoName(dir: String): String? { return withContext(Dispatchers.IO) { - val directory = File(dir) + val directory = File(URI(dir)) val targetDir = if (directory.isFile) directory.parentFile else directory val builder = ProcessBuilder("git", "config", "--get", "remote.origin.url") builder.directory(targetDir) @@ -516,7 +500,7 @@ class IntelliJIDE( override suspend fun getGitRootPath(dir: String): String? { return withContext(Dispatchers.IO) { val builder = ProcessBuilder("git", "rev-parse", "--show-toplevel") - builder.directory(File(dir)) + builder.directory(File(URI(dir))) val process = builder.start() val reader = BufferedReader(InputStreamReader(process.inputStream)) @@ -527,7 +511,7 @@ class IntelliJIDE( } override suspend fun listDir(dir: String): List> { - val files = File(dir).listFiles()?.map { + val files = File(URI(dir)).listFiles()?.map { listOf(it.name, if (it.isDirectory) FileType.DIRECTORY else FileType.FILE) } ?: emptyList() @@ -536,7 +520,7 @@ class IntelliJIDE( override suspend fun getLastModified(files: List): Map { return files.associateWith { file -> - File(file).lastModified() + File(URI(file)).lastModified() } } @@ -553,12 +537,8 @@ class IntelliJIDE( throw NotImplementedError("onDidChangeActiveTextEditor not implemented yet") } - override suspend fun pathSep(): String { - return File.separator - } - private fun setFileOpen(filepath: String, open: Boolean = true) { - val file = LocalFileSystem.getInstance().findFileByPath(filepath) + val file = LocalFileSystem.getInstance().findFileByPath(URI(filepath).path) file?.let { if (open) { diff --git a/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/types.kt b/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/types.kt index 0747284233..b1e051777b 100644 --- a/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/types.kt +++ b/extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/types.kt @@ -115,8 +115,6 @@ interface IDE { suspend fun getAvailableThreads(): List - suspend fun listFolders(): List - suspend fun getWorkspaceDirs(): List suspend fun getWorkspaceConfigs(): List @@ -194,8 +192,6 @@ interface IDE { // Callbacks fun onDidChangeActiveTextEditor(callback: (filepath: String) -> Unit) - - suspend fun pathSep(): String } data class GetGhTokenArgs( diff --git a/extensions/vscode/e2e/TestUtils.ts b/extensions/vscode/e2e/TestUtils.ts index 92986e498d..e1ae22227d 100644 --- a/extensions/vscode/e2e/TestUtils.ts +++ b/extensions/vscode/e2e/TestUtils.ts @@ -37,8 +37,8 @@ export class TestUtils { throw new Error(`Element not found after ${timeout}ms timeout`); } - public static async expectNoElement( - locatorFn: () => Promise, + public static async expectNoElement( + locatorFn: () => Promise, timeout: number = 1000, interval: number = 200, ): Promise { @@ -48,6 +48,7 @@ export class TestUtils { while (Date.now() - startTime < timeout) { try { const element = await locatorFn(); + console.log("ELEMENT", element); if (element) { elementFound = true; break; @@ -71,6 +72,10 @@ export class TestUtils { }; } + public static waitForTimeout(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); + } + public static get isMacOS(): boolean { return process.platform === "darwin"; } diff --git a/extensions/vscode/e2e/actions/GUI.actions.ts b/extensions/vscode/e2e/actions/GUI.actions.ts index 402176b20c..567a2f5721 100644 --- a/extensions/vscode/e2e/actions/GUI.actions.ts +++ b/extensions/vscode/e2e/actions/GUI.actions.ts @@ -1,8 +1,33 @@ -import { Key, WebElement, WebView, Workbench } from "vscode-extension-tester"; +import { + InputBox, + Key, + WebDriver, + WebElement, + WebView, + Workbench, +} from "vscode-extension-tester"; import { GUISelectors } from "../selectors/GUI.selectors"; import { TestUtils } from "../TestUtils"; +import { DEFAULT_TIMEOUT } from "../constants"; export class GUIActions { + public static moveContinueToSidebar = async (driver: WebDriver) => { + await GUIActions.toggleGui(); + await TestUtils.waitForSuccess(async () => { + await new Workbench().executeCommand("View: Move View"); + await (await InputBox.create(DEFAULT_TIMEOUT.MD)).selectQuickPick(4); + await (await InputBox.create(DEFAULT_TIMEOUT.MD)).selectQuickPick(14); + }); + + // first call focuses the input + await TestUtils.waitForTimeout(DEFAULT_TIMEOUT.XS); + await GUIActions.executeFocusContinueInputShortcut(driver); + + // second call closes the gui + await TestUtils.waitForTimeout(DEFAULT_TIMEOUT.XS); + await GUIActions.executeFocusContinueInputShortcut(driver); + }; + public static switchToReactIframe = async () => { const view = new WebView(); const driver = view.getDriver(); @@ -19,7 +44,7 @@ export class GUIActions { } if (!continueIFrame) { - throw new Error("Could not find continue iframe"); + throw new Error("Could not find Continue iframe"); } await driver.switchTo().frame(continueIFrame); @@ -41,7 +66,7 @@ export class GUIActions { }; }; - public static openGui = async () => { + public static toggleGui = async () => { return TestUtils.waitForSuccess(() => new Workbench().executeCommand("continue.focusContinueInput"), ); @@ -77,4 +102,13 @@ export class GUIActions { await editor.sendKeys(message); await editor.sendKeys(Key.ENTER); } + + public static async executeFocusContinueInputShortcut(driver: WebDriver) { + return driver + .actions() + .keyDown(TestUtils.osControlKey) + .sendKeys("l") + .keyUp(TestUtils.osControlKey) + .perform(); + } } diff --git a/extensions/vscode/e2e/actions/Global.actions.ts b/extensions/vscode/e2e/actions/Global.actions.ts index 338b642cda..b6d5990899 100644 --- a/extensions/vscode/e2e/actions/Global.actions.ts +++ b/extensions/vscode/e2e/actions/Global.actions.ts @@ -1,8 +1,7 @@ import { VSBrowser, WebView } from "vscode-extension-tester"; -import * as path from "path"; export class GlobalActions { public static async openTestWorkspace() { - return VSBrowser.instance.openResources(path.join("e2e/test-continue")); + return VSBrowser.instance.openResources("e2e/test-continue"); } } diff --git a/extensions/vscode/e2e/actions/KeyboardShortcuts.actions.ts b/extensions/vscode/e2e/actions/KeyboardShortcuts.actions.ts index dfa6dee150..0a4cc51ad5 100644 --- a/extensions/vscode/e2e/actions/KeyboardShortcuts.actions.ts +++ b/extensions/vscode/e2e/actions/KeyboardShortcuts.actions.ts @@ -1,14 +1,15 @@ -import { WebDriver } from "vscode-extension-tester"; +import { TextEditor, WebDriver, WebView } from "vscode-extension-tester"; import { TestUtils } from "../TestUtils"; export class KeyboardShortcutsActions { - public static async executeFocusContinueInput(driver: WebDriver) { - return await driver - .actions() - .keyDown(TestUtils.osControlKey) - .sendKeys("l") - .keyUp(TestUtils.osControlKey) - .perform(); + /** + * For some reason Selenium-simulated keyboard shortcuts don't perfectly + * mimic the behavior of real shortcuts unless some text is highlighted first. + */ + public static async HACK__typeWithSelect(editor: TextEditor, text: string) { + await editor.typeText(text); + await editor.selectText(text); + await editor.typeText(text); } } diff --git a/extensions/vscode/e2e/tests/GUI.test.ts b/extensions/vscode/e2e/tests/GUI.test.ts index 9b3eca3e07..60b6723b87 100644 --- a/extensions/vscode/e2e/tests/GUI.test.ts +++ b/extensions/vscode/e2e/tests/GUI.test.ts @@ -18,7 +18,7 @@ describe("GUI Test", () => { beforeEach(async function () { this.timeout(DEFAULT_TIMEOUT.XL); - await GUIActions.openGui(); + await GUIActions.toggleGui(); ({ view, driver } = await GUIActions.switchToReactIframe()); await GUIActions.selectModelFromDropdown(view, "TEST LLM"); diff --git a/extensions/vscode/e2e/tests/KeyboardShortcuts.test.ts b/extensions/vscode/e2e/tests/KeyboardShortcuts.test.ts index b15d4ae81b..837372bab9 100644 --- a/extensions/vscode/e2e/tests/KeyboardShortcuts.test.ts +++ b/extensions/vscode/e2e/tests/KeyboardShortcuts.test.ts @@ -6,19 +6,24 @@ import { TextEditor, Workbench, WebView, + WebElement, } from "vscode-extension-tester"; import { DEFAULT_TIMEOUT } from "../constants"; import { GUISelectors } from "../selectors/GUI.selectors"; import { GUIActions } from "../actions/GUI.actions"; import { expect } from "chai"; import { TestUtils } from "../TestUtils"; -import { KeyboardShortcutsActions } from "../actions/KeyboardShortcuts.actions"; describe("Cmd+L Shortcut Test", () => { let driver: WebDriver; let editor: TextEditor; let view: WebView; + before(async function () { + this.timeout(DEFAULT_TIMEOUT.XL); + await GUIActions.moveContinueToSidebar(VSBrowser.instance.driver); + }); + beforeEach(async function () { this.timeout(DEFAULT_TIMEOUT.XL); @@ -38,11 +43,6 @@ describe("Cmd+L Shortcut Test", () => { this.timeout(DEFAULT_TIMEOUT.XL * 1000); await view.switchBack(); await editor.clearText(); - await TestUtils.waitForSuccess( - async () => (await GUISelectors.getContinueExtensionBadge(view)).click(), - DEFAULT_TIMEOUT.XS, - ); - await new EditorView().closeAllEditors(); }); @@ -51,13 +51,30 @@ describe("Cmd+L Shortcut Test", () => { await editor.setText(text); - await KeyboardShortcutsActions.executeFocusContinueInput(driver); + await GUIActions.executeFocusContinueInputShortcut(driver); ({ view } = await GUIActions.switchToReactIframe()); await TestUtils.expectNoElement(async () => { return GUISelectors.getInputBoxCodeBlockAtIndex(view, 0); }, DEFAULT_TIMEOUT.XS); + await GUIActions.executeFocusContinueInputShortcut(driver); + }).timeout(DEFAULT_TIMEOUT.XL); + + it("Fresh VS Code window → sidebar closed → cmd+L with no code highlighted → opens sidebar and focuses input → cmd+L closes sidebar", async () => { + await GUIActions.executeFocusContinueInputShortcut(driver); + ({ view } = await GUIActions.switchToReactIframe()); + const textInput = await TestUtils.waitForSuccess(() => + GUISelectors.getMessageInputFieldAtIndex(view, 0), + ); + const activeElement: WebElement = await driver.switchTo().activeElement(); + const textInputHtml = await textInput.getAttribute("outerHTML"); + const activeElementHtml = await activeElement.getAttribute("outerHTML"); + expect(textInputHtml).to.equal(activeElementHtml); + expect(await textInput.isDisplayed()).to.equal(true); + + await GUIActions.executeFocusContinueInputShortcut(driver); + expect(await textInput.isDisplayed()).to.equal(false); }).timeout(DEFAULT_TIMEOUT.XL); it("Should create a code block when Cmd+L is pressed with text highlighted", async () => { @@ -66,7 +83,7 @@ describe("Cmd+L Shortcut Test", () => { await editor.setText(text); await editor.selectText(text); - await KeyboardShortcutsActions.executeFocusContinueInput(driver); + await GUIActions.executeFocusContinueInputShortcut(driver); ({ view } = await GUIActions.switchToReactIframe()); @@ -78,5 +95,7 @@ describe("Cmd+L Shortcut Test", () => { ); expect(codeblockContent).to.equal(text); + + await GUIActions.executeFocusContinueInputShortcut(driver); }).timeout(DEFAULT_TIMEOUT.XL); }); diff --git a/extensions/vscode/e2e/tests/SSH.test.ts b/extensions/vscode/e2e/tests/SSH.test.ts index 664776a9cb..1abf66db9a 100644 --- a/extensions/vscode/e2e/tests/SSH.test.ts +++ b/extensions/vscode/e2e/tests/SSH.test.ts @@ -25,7 +25,7 @@ describe("SSH", function () { await TestUtils.waitForSuccess( () => SSHSelectors.connectedToRemoteConfirmationMessage(), - DEFAULT_TIMEOUT.XL, + DEFAULT_TIMEOUT.MD, ); await TestUtils.waitForSuccess(async () => { @@ -34,7 +34,7 @@ describe("SSH", function () { await inputBox.setText("/home/ec2-user/test-folder/main.py"); await inputBox.selectQuickPick("main.py"); await inputBox.sendKeys(Key.ENTER); - }, DEFAULT_TIMEOUT.XL); + }, DEFAULT_TIMEOUT.MD); const editor = await TestUtils.waitForSuccess( async () => (await new EditorView().openEditor("main.py")) as TextEditor, diff --git a/extensions/vscode/package.json b/extensions/vscode/package.json index 06c7337be7..dbca6152d5 100644 --- a/extensions/vscode/package.json +++ b/extensions/vscode/package.json @@ -629,7 +629,7 @@ "e2e:copy-vsix": "chmod +x ./e2e/get-latest-vsix.sh && bash ./e2e/get-latest-vsix.sh", "e2e:install-vsix": "extest install-vsix -f ./e2e/vsix/continue.vsix --extensions_dir ./e2e/.test-extensions --storage ./e2e/storage", "e2e:install-extensions": "extest install-from-marketplace ms-vscode-remote.remote-ssh --extensions_dir ./e2e/.test-extensions --storage ./e2e/storage && extest install-from-marketplace ms-vscode-remote.remote-containers --extensions_dir ./e2e/.test-extensions --storage ./e2e/storage && extest install-from-marketplace ms-vscode-remote.remote-wsl --extensions_dir ./e2e/.test-extensions --storage ./e2e/storage", - "e2e:test": "extest run-tests './e2e/_output/tests/*.test.js' --code_settings settings.json --extensions_dir ./e2e/.test-extensions --storage ./e2e/storage", + "e2e:test": "extest run-tests ${TEST_FILE:-'./e2e/_output/tests/*.test.js'} --code_settings settings.json --extensions_dir ./e2e/.test-extensions --storage ./e2e/storage", "e2e:clean": "rm -rf ./e2e/_output", "e2e:all": "npm run e2e:build && npm run e2e:compile && npm run e2e:create-storage && npm run e2e:get-chromedriver && npm run e2e:get-vscode && npm run e2e:sign-vscode && npm run e2e:copy-vsix && npm run e2e:install-vsix && npm run e2e:install-extensions && CONTINUE_GLOBAL_DIR=e2e/test-continue npm run e2e:test && npm run e2e:clean", "e2e:quick": "npm run e2e:compile && CONTINUE_GLOBAL_DIR=e2e/test-continue npm run e2e:test && npm run e2e:clean", diff --git a/extensions/vscode/src/ContinueGUIWebviewViewProvider.ts b/extensions/vscode/src/ContinueGUIWebviewViewProvider.ts index f98b09f93e..66ff97f1c2 100644 --- a/extensions/vscode/src/ContinueGUIWebviewViewProvider.ts +++ b/extensions/vscode/src/ContinueGUIWebviewViewProvider.ts @@ -216,8 +216,8 @@ export class ContinueGUIWebviewViewProvider diff --git a/extensions/vscode/src/VsCodeIde.ts b/extensions/vscode/src/VsCodeIde.ts index 9e237350ef..bee4a3771f 100644 --- a/extensions/vscode/src/VsCodeIde.ts +++ b/extensions/vscode/src/VsCodeIde.ts @@ -1,28 +1,19 @@ import * as child_process from "node:child_process"; import { exec } from "node:child_process"; -import * as path from "node:path"; import { Range } from "core"; import { EXTENSION_NAME } from "core/control-plane/env"; import { walkDir } from "core/indexing/walkDir"; import { GetGhTokenArgs } from "core/protocol/ide"; -import { - editConfigJson, - getConfigJsonPath, - getContinueGlobalPath, -} from "core/util/paths"; +import { editConfigJson, getConfigJsonPath } from "core/util/paths"; import * as vscode from "vscode"; import { executeGotoProvider } from "./autocomplete/lsp"; -import { DiffManager } from "./diff/horizontal"; import { Repository } from "./otherExtensions/git"; import { VsCodeIdeUtils } from "./util/ideUtils"; -import { - getExtensionUri, - openEditorAndRevealRange, - uriFromFilePath, -} from "./util/vscode"; +import { getExtensionUri, openEditorAndRevealRange } from "./util/vscode"; import { VsCodeWebviewProtocol } from "./webviewProtocol"; +import * as URI from "uri-js"; import type { ContinueRcJson, @@ -36,32 +27,33 @@ import type { RangeInFile, Thread, } from "core"; +import { findUriInDirs } from "core/util/uri"; class VsCodeIde implements IDE { ideUtils: VsCodeIdeUtils; constructor( - private readonly diffManager: DiffManager, private readonly vscodeWebviewProtocolPromise: Promise, private readonly context: vscode.ExtensionContext, ) { this.ideUtils = new VsCodeIdeUtils(); } - pathSep(): Promise { - return Promise.resolve(this.ideUtils.path.sep); - } - async fileExists(filepath: string): Promise { - const absPath = await this.ideUtils.resolveAbsFilepathInWorkspace(filepath); - return vscode.workspace.fs.stat(uriFromFilePath(absPath)).then( - () => true, - () => false, - ); + async fileExists(uri: string): Promise { + try { + await vscode.workspace.fs.stat(vscode.Uri.parse(uri)); + return true; + } catch (error) { + if (error instanceof vscode.FileSystemError) { + return false; + } + throw error; + } } async gotoDefinition(location: Location): Promise { const result = await executeGotoProvider({ - uri: location.filepath, + uri: vscode.Uri.parse(location.filepath), line: location.position.line, character: location.position.character, name: "vscode.executeDefinitionProvider", @@ -70,10 +62,10 @@ class VsCodeIde implements IDE { return result; } - onDidChangeActiveTextEditor(callback: (filepath: string) => void): void { + onDidChangeActiveTextEditor(callback: (uri: string) => void): void { vscode.window.onDidChangeActiveTextEditor((editor) => { if (editor) { - callback(editor.document.uri.fsPath); + callback(editor.document.uri.toString()); } }); } @@ -227,7 +219,7 @@ class VsCodeIde implements IDE { }; async getRepoName(dir: string): Promise { - const repo = await this.getRepo(vscode.Uri.file(dir)); + const repo = await this.getRepo(dir); const remotes = repo?.state.remotes; if (!remotes) { return undefined; @@ -272,9 +264,9 @@ class VsCodeIde implements IDE { }); } - readRangeInFile(filepath: string, range: Range): Promise { + readRangeInFile(fileUri: string, range: Range): Promise { return this.ideUtils.readRangeInFile( - filepath, + vscode.Uri.parse(fileUri), new vscode.Range( new vscode.Position(range.start.line, range.start.character), new vscode.Position(range.end.line, range.end.character), @@ -286,7 +278,7 @@ class VsCodeIde implements IDE { const pathToLastModified: { [path: string]: number } = {}; await Promise.all( files.map(async (file) => { - const stat = await vscode.workspace.fs.stat(uriFromFilePath(file)); + const stat = await vscode.workspace.fs.stat(vscode.Uri.parse(file)); pathToLastModified[file] = stat.mtime; }), ); @@ -294,8 +286,8 @@ class VsCodeIde implements IDE { return pathToLastModified; } - async getRepo(dir: vscode.Uri): Promise { - return this.ideUtils.getRepo(dir); + async getRepo(dir: string): Promise { + return this.ideUtils.getRepo(vscode.Uri.parse(dir)); } async isTelemetryEnabled(): Promise { @@ -356,7 +348,7 @@ class VsCodeIde implements IDE { filename === ".continuerc.json" ) { const contents = await this.readFile( - vscode.Uri.joinPath(workspaceDir, filename).fsPath, + vscode.Uri.joinPath(workspaceDir, filename).toString(), ); configs.push(JSON.parse(contents)); } @@ -365,29 +357,13 @@ class VsCodeIde implements IDE { return configs; } - async listFolders(): Promise { - const allDirs: string[] = []; - - const workspaceDirs = await this.getWorkspaceDirs(); - for (const directory of workspaceDirs) { - const dirs = await walkDir(directory, this, { onlyDirs: true }); - allDirs.push(...dirs); - } - - return allDirs; - } - async getWorkspaceDirs(): Promise { - return this.ideUtils.getWorkspaceDirectories(); + return this.ideUtils.getWorkspaceDirectories().map((uri) => uri.toString()); } - async getContinueDir(): Promise { - return getContinueGlobalPath(); - } - - async writeFile(path: string, contents: string): Promise { + async writeFile(fileUri: string, contents: string): Promise { await vscode.workspace.fs.writeFile( - vscode.Uri.file(path), + vscode.Uri.parse(fileUri), Buffer.from(contents), ); } @@ -396,12 +372,12 @@ class VsCodeIde implements IDE { this.ideUtils.showVirtualFile(title, contents); } - async openFile(path: string): Promise { - await this.ideUtils.openFile(path); + async openFile(fileUri: string): Promise { + await this.ideUtils.openFile(vscode.Uri.parse(fileUri)); } async showLines( - filepath: string, + fileUri: string, startLine: number, endLine: number, ): Promise { @@ -409,13 +385,15 @@ class VsCodeIde implements IDE { new vscode.Position(startLine, 0), new vscode.Position(endLine, 0), ); - openEditorAndRevealRange(filepath, range).then((editor) => { - // Select the lines - editor.selection = new vscode.Selection( - new vscode.Position(startLine, 0), - new vscode.Position(endLine, 0), - ); - }); + openEditorAndRevealRange(vscode.Uri.parse(fileUri), range).then( + (editor) => { + // Select the lines + editor.selection = new vscode.Selection( + new vscode.Position(startLine, 0), + new vscode.Position(endLine, 0), + ); + }, + ); } async runCommand(command: string): Promise { @@ -431,24 +409,23 @@ class VsCodeIde implements IDE { } } - async saveFile(filepath: string): Promise { - await this.ideUtils.saveFile(filepath); + async saveFile(fileUri: string): Promise { + await this.ideUtils.saveFile(vscode.Uri.parse(fileUri)); } private static MAX_BYTES = 100000; - async readFile(filepath: string): Promise { + async readFile(fileUri: string): Promise { try { - filepath = this.ideUtils.getAbsolutePath(filepath); - const uri = uriFromFilePath(filepath); + const uri = vscode.Uri.parse(fileUri); // First, check whether it's a notebook document // Need to iterate over the cells to get full contents const notebook = - vscode.workspace.notebookDocuments.find( - (doc) => doc.uri.toString() === uri.toString(), + vscode.workspace.notebookDocuments.find((doc) => + URI.equal(doc.uri.toString(), uri.toString()), ) ?? - (uri.fsPath.endsWith("ipynb") + (uri.path.endsWith("ipynb") ? await vscode.workspace.openNotebookDocument(uri) : undefined); if (notebook) { @@ -459,16 +436,14 @@ class VsCodeIde implements IDE { } // Check whether it's an open document - const openTextDocument = vscode.workspace.textDocuments.find( - (doc) => doc.uri.fsPath === uri.fsPath, + const openTextDocument = vscode.workspace.textDocuments.find((doc) => + URI.equal(doc.uri.toString(), uri.toString()), ); if (openTextDocument !== undefined) { return openTextDocument.getText(); } - const fileStats = await vscode.workspace.fs.stat( - uriFromFilePath(filepath), - ); + const fileStats = await vscode.workspace.fs.stat(uri); if (fileStats.size > 10 * VsCodeIde.MAX_BYTES) { return ""; } @@ -488,16 +463,8 @@ class VsCodeIde implements IDE { await vscode.env.openExternal(vscode.Uri.parse(url)); } - async showDiff( - filepath: string, - newContents: string, - stepIndex: number, - ): Promise { - await this.diffManager.writeDiff(filepath, newContents, stepIndex); - } - async getOpenFiles(): Promise { - return await this.ideUtils.getOpenFiles(); + return this.ideUtils.getOpenFiles().map((uri) => uri.toString()); } async getCurrentFile() { @@ -506,7 +473,7 @@ class VsCodeIde implements IDE { } return { isUntitled: vscode.window.activeTextEditor.document.isUntitled, - path: vscode.window.activeTextEditor.document.uri.fsPath, + path: vscode.window.activeTextEditor.document.uri.toString(), contents: vscode.window.activeTextEditor.document.getText(), }; } @@ -516,20 +483,17 @@ class VsCodeIde implements IDE { return tabArray .filter((t) => t.isPinned) - .map((t) => (t.input as vscode.TabInputText).uri.fsPath); + .map((t) => (t.input as vscode.TabInputText).uri.toString()); } private async _searchDir(query: string, dir: string): Promise { + const relativeDir = vscode.Uri.parse(dir).fsPath; + const ripGrepUri = vscode.Uri.joinPath( + getExtensionUri(), + "out/node_modules/@vscode/ripgrep/bin/rg", + ); const p = child_process.spawn( - path.join( - getExtensionUri().fsPath, - "out", - "node_modules", - "@vscode", - "ripgrep", - "bin", - "rg", - ), + ripGrepUri.fsPath, [ "-i", // Case-insensitive search "-C", @@ -538,7 +502,7 @@ class VsCodeIde implements IDE { query, // Pattern to search for ".", // Directory to search in ], - { cwd: dir }, + { cwd: relativeDir }, ); let output = ""; @@ -570,16 +534,16 @@ class VsCodeIde implements IDE { return results.join("\n\n"); } - async getProblems(filepath?: string | undefined): Promise { - const uri = filepath - ? vscode.Uri.file(filepath) + async getProblems(fileUri?: string | undefined): Promise { + const uri = fileUri + ? vscode.Uri.parse(fileUri) : vscode.window.activeTextEditor?.document.uri; if (!uri) { return []; } return vscode.languages.getDiagnostics(uri).map((d) => { return { - filepath: uri.fsPath, + filepath: uri.toString(), range: { start: { line: d.range.start.line, @@ -605,15 +569,16 @@ class VsCodeIde implements IDE { } async getBranch(dir: string): Promise { - return this.ideUtils.getBranch(vscode.Uri.file(dir)); + return this.ideUtils.getBranch(vscode.Uri.parse(dir)); } - getGitRootPath(dir: string): Promise { - return this.ideUtils.getGitRoot(dir); + async getGitRootPath(dir: string): Promise { + const root = await this.ideUtils.getGitRoot(vscode.Uri.parse(dir)); + return root?.toString(); } async listDir(dir: string): Promise<[string, FileType][]> { - return vscode.workspace.fs.readDirectory(uriFromFilePath(dir)) as any; + return vscode.workspace.fs.readDirectory(vscode.Uri.parse(dir)) as any; } getIdeSettingsSync(): IdeSettings { diff --git a/extensions/vscode/src/activation/languageClient.ts b/extensions/vscode/src/activation/languageClient.ts index c3b4b96c59..61babbc088 100644 --- a/extensions/vscode/src/activation/languageClient.ts +++ b/extensions/vscode/src/activation/languageClient.ts @@ -55,7 +55,7 @@ function startPythonLanguageServer(context: ExtensionContext): LanguageClient { const command = `cd ${path.join( extensionPath, "scripts", - )} && source env/bin/activate.fish && python -m pyls`; + )} && source ${path.join("env", "bin", "activate.fish")} && python -m pyls`; const serverOptions: ServerOptions = { command: command, args: ["-vv"], @@ -113,4 +113,3 @@ async function startPylance(context: ExtensionContext) { ); return client; } - diff --git a/extensions/vscode/src/autocomplete/completionProvider.ts b/extensions/vscode/src/autocomplete/completionProvider.ts index 2a2f41a886..f5388e397d 100644 --- a/extensions/vscode/src/autocomplete/completionProvider.ts +++ b/extensions/vscode/src/autocomplete/completionProvider.ts @@ -19,6 +19,7 @@ import { setupStatusBar, stopStatusBarLoading, } from "./statusBar"; +import * as URI from "uri-js"; import type { IDE } from "core"; import type { TabAutocompleteModel } from "../util/loadAutocompleteModel"; @@ -84,12 +85,6 @@ export class ContinueCompletionProvider this.onError.bind(this), getDefinitionsFromLsp, ); - - vscode.workspace.onDidChangeTextDocument((event) => { - if (event.document.uri.fsPath === this._lastShownCompletion?.filepath) { - // console.log("updating completion"); - } - }); } _lastShownCompletion: AutocompleteOutcome | undefined; @@ -181,7 +176,9 @@ export class ContinueCompletionProvider const notebook = vscode.workspace.notebookDocuments.find((notebook) => notebook .getCells() - .some((cell) => cell.document.uri === document.uri), + .some((cell) => + URI.equal(cell.document.uri.toString(), document.uri.toString()), + ), ); if (notebook) { const cells = notebook.getCells(); @@ -196,7 +193,9 @@ export class ContinueCompletionProvider }) .join("\n\n"); for (const cell of cells) { - if (cell.document.uri === document.uri) { + if ( + URI.equal(cell.document.uri.toString(), document.uri.toString()) + ) { break; } else { pos.line += cell.document.getText().split("\n").length + 1; @@ -206,7 +205,6 @@ export class ContinueCompletionProvider } // Manually pass file contents for unsaved, untitled files - let filepath = document.uri.fsPath; if (document.isUntitled) { manuallyPassFileContents = document.getText(); } @@ -217,7 +215,7 @@ export class ContinueCompletionProvider const input: AutocompleteInput = { isUntitledFile: document.isUntitled, completionId: uuidv4(), - filepath, + filepath: document.uri.toString(), pos, recentlyEditedFiles: [], recentlyEditedRanges: diff --git a/extensions/vscode/src/autocomplete/lsp.ts b/extensions/vscode/src/autocomplete/lsp.ts index 08b16ace7c..6d3b27543c 100644 --- a/extensions/vscode/src/autocomplete/lsp.ts +++ b/extensions/vscode/src/autocomplete/lsp.ts @@ -9,14 +9,12 @@ import * as vscode from "vscode"; import type { IDE, Range, RangeInFile, RangeInFileWithContents } from "core"; import type Parser from "web-tree-sitter"; -import { - AutocompleteSnippetDeprecated, - GetLspDefinitionsFunction, -} from "core/autocomplete/types"; +import { GetLspDefinitionsFunction } from "core/autocomplete/types"; import { AutocompleteCodeSnippet, AutocompleteSnippetType, } from "core/autocomplete/snippets/types"; +import * as URI from "uri-js"; type GotoProviderName = | "vscode.executeDefinitionProvider" @@ -26,13 +24,13 @@ type GotoProviderName = | "vscode.executeReferenceProvider"; interface GotoInput { - uri: string; + uri: vscode.Uri; line: number; character: number; name: GotoProviderName; } function gotoInputKey(input: GotoInput) { - return `${input.name}${input.uri.toString}${input.line}${input.character}`; + return `${input.name}${input.uri.toString()}${input.line}${input.character}`; } const MAX_CACHE_SIZE = 50; @@ -50,14 +48,14 @@ export async function executeGotoProvider( try { const definitions = (await vscode.commands.executeCommand( input.name, - vscode.Uri.parse(input.uri), + input.uri, new vscode.Position(input.line, input.character), )) as any; const results = definitions .filter((d: any) => (d.targetUri || d.uri) && (d.targetRange || d.range)) .map((d: any) => ({ - filepath: (d.targetUri || d.uri).fsPath, + filepath: (d.targetUri || d.uri).toString(), range: d.targetRange || d.range, })); @@ -156,7 +154,7 @@ async function crawlTypes( const definitions = await Promise.all( identifierNodes.map(async (node) => { const [typeDef] = await executeGotoProvider({ - uri: rif.filepath, + uri: vscode.Uri.parse(rif.filepath), // TODO: tree-sitter is zero-indexed, but there seems to be an off-by-one // error at least with the .ts parser sometimes line: @@ -184,7 +182,7 @@ async function crawlTypes( !definition || results.some( (result) => - result.filepath === definition.filepath && + URI.equal(result.filepath, definition.filepath) && intersection(result.range, definition.range) !== null, ) ) { @@ -204,7 +202,7 @@ async function crawlTypes( } export async function getDefinitionsForNode( - uri: string, + uri: vscode.Uri, node: Parser.SyntaxNode, ide: IDE, lang: AutocompleteLanguageInfo, @@ -362,7 +360,7 @@ export const getDefinitionsFromLsp: GetLspDefinitionsFunction = async ( const results: RangeInFileWithContents[] = []; for (const node of treePath.reverse()) { const definitions = await getDefinitionsForNode( - filepath, + vscode.Uri.parse(filepath), node, ide, lang, diff --git a/extensions/vscode/src/autocomplete/recentlyEdited.ts b/extensions/vscode/src/autocomplete/recentlyEdited.ts index 221d7df663..6ab614c90a 100644 --- a/extensions/vscode/src/autocomplete/recentlyEdited.ts +++ b/extensions/vscode/src/autocomplete/recentlyEdited.ts @@ -23,9 +23,6 @@ export class RecentlyEditedTracker { constructor() { vscode.workspace.onDidChangeTextDocument((event) => { - if (event.document.uri.scheme !== "file") { - return; - } event.contentChanges.forEach((change) => { const editedRange = { uri: event.document.uri, @@ -123,7 +120,7 @@ export class RecentlyEditedTracker { return this.recentlyEditedRanges.map((entry) => { return { ...entry, - filepath: entry.uri.fsPath, + filepath: entry.uri.toString(), }; }); } @@ -140,7 +137,7 @@ export class RecentlyEditedTracker { const lines = contents.split("\n"); return { - filepath: entry.uri.fsPath, + filepath: entry.uri.toString(), contents, range: { start: { line: 0, character: 0 }, diff --git a/extensions/vscode/src/commands.ts b/extensions/vscode/src/commands.ts index 7af278118f..690063628f 100644 --- a/extensions/vscode/src/commands.ts +++ b/extensions/vscode/src/commands.ts @@ -1,7 +1,5 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import * as fs from "node:fs"; import * as os from "node:os"; -import * as path from "node:path"; import { ContextMenuConfig, RangeInFileWithContents } from "core"; import { CompletionProvider } from "core/autocomplete/CompletionProvider"; @@ -12,7 +10,11 @@ import { EXTENSION_NAME } from "core/control-plane/env"; import { Core } from "core/core"; import { walkDirAsync } from "core/indexing/walkDir"; import { GlobalContext } from "core/util/GlobalContext"; -import { getConfigJsonPath, getDevDataFilePath } from "core/util/paths"; +import { + getConfigJsonPath, + getDevDataFilePath, + getLogFilePath, +} from "core/util/paths"; import { Telemetry } from "core/util/posthog"; import readLastLines from "read-last-lines"; import * as vscode from "vscode"; @@ -27,13 +29,11 @@ import { setupStatusBar, } from "./autocomplete/statusBar"; import { ContinueGUIWebviewViewProvider } from "./ContinueGUIWebviewViewProvider"; -import { DiffManager } from "./diff/horizontal"; + import { VerticalDiffManager } from "./diff/vertical/manager"; import EditDecorationManager from "./quickEdit/EditDecorationManager"; import { QuickEdit, QuickEditShowParams } from "./quickEdit/QuickEditQuickPick"; import { Battery } from "./util/battery"; -import { getFullyQualifiedPath } from "./util/util"; -import { uriFromFilePath } from "./util/vscode"; import { VsCodeIde } from "./VsCodeIde"; import type { VsCodeWebviewProtocol } from "./webviewProtocol"; @@ -71,7 +71,7 @@ function addCodeToContextFromRange( } const rangeInFileWithContents = { - filepath: document.uri.fsPath, + filepath: document.uri.toString(), contents: document.getText(range), range: { start: { @@ -101,7 +101,7 @@ function getRangeInFileWithContents( if (editor) { const selection = editor.selection; - const filepath = editor.document.uri.fsPath; + const filepath = editor.document.uri.toString(); if (range) { const contents = editor.document.getText(range); @@ -132,7 +132,7 @@ function getRangeInFileWithContents( const contents = editor.document.getText(selectionRange); return { - filepath: editor.document.uri.fsPath, + filepath, contents, range: { start: { @@ -162,17 +162,17 @@ async function addHighlightedCodeToContext( } async function addEntireFileToContext( - filepath: vscode.Uri, + uri: vscode.Uri, webviewProtocol: VsCodeWebviewProtocol | undefined, ) { // If a directory, add all files in the directory - const stat = await vscode.workspace.fs.stat(filepath); + const stat = await vscode.workspace.fs.stat(uri); if (stat.type === vscode.FileType.Directory) { - const files = await vscode.workspace.fs.readDirectory(filepath); + const files = await vscode.workspace.fs.readDirectory(uri); for (const [filename, type] of files) { if (type === vscode.FileType.File) { addEntireFileToContext( - vscode.Uri.joinPath(filepath, filename), + vscode.Uri.joinPath(uri, filename), webviewProtocol, ); } @@ -181,9 +181,9 @@ async function addEntireFileToContext( } // Get the contents of the file - const contents = (await vscode.workspace.fs.readFile(filepath)).toString(); + const contents = (await vscode.workspace.fs.readFile(uri)).toString(); const rangeInFileWithContents = { - filepath: filepath.fsPath, + filepath: uri.toString(), contents: contents, range: { start: { @@ -229,54 +229,40 @@ function hideGUI() { async function processDiff( action: "accept" | "reject", sidebar: ContinueGUIWebviewViewProvider, - diffManager: DiffManager, ide: VsCodeIde, verticalDiffManager: VerticalDiffManager, - newFilepath?: string | vscode.Uri, + newFileUri?: string, streamId?: string, ) { captureCommandTelemetry(`${action}Diff`); - let fullPath = newFilepath; - - if (fullPath instanceof vscode.Uri) { - fullPath = fullPath.fsPath; - } else if (fullPath) { - fullPath = getFullyQualifiedPath(ide, fullPath); - } else { - const curFile = await ide.getCurrentFile(); - fullPath = curFile?.path; + let newOrCurrentUri = newFileUri; + if (!newOrCurrentUri) { + const currentFile = await ide.getCurrentFile(); + newOrCurrentUri = currentFile?.path; } - - if (!fullPath) { + if (!newOrCurrentUri) { console.warn( - `Unable to resolve filepath while attempting to resolve diff: ${newFilepath}`, + `No file provided or current file open while attempting to resolve diff`, ); return; } - await ide.openFile(fullPath); + await ide.openFile(newOrCurrentUri); // Clear vertical diffs depending on action - verticalDiffManager.clearForFilepath(fullPath, action === "accept"); - - // Accept or reject the diff - if (action === "accept") { - await diffManager.acceptDiff(fullPath); - } else { - await diffManager.rejectDiff(fullPath); - } + verticalDiffManager.clearForfileUri(newOrCurrentUri, action === "accept"); void sidebar.webviewProtocol.request("setEditStatus", { status: "done", }); if (streamId) { - const fileContent = await ide.readFile(fullPath); + const fileContent = await ide.readFile(newOrCurrentUri); await sidebar.webviewProtocol.request("updateApplyState", { fileContent, - filepath: fullPath, + filepath: newOrCurrentUri, streamId, status: "closed", numDiffs: 0, @@ -312,7 +298,6 @@ const getCommandsMap: ( extensionContext: vscode.ExtensionContext, sidebar: ContinueGUIWebviewViewProvider, configHandler: ConfigHandler, - diffManager: DiffManager, verticalDiffManager: VerticalDiffManager, continueServerClientPromise: Promise, battery: Battery, @@ -324,7 +309,6 @@ const getCommandsMap: ( extensionContext, sidebar, configHandler, - diffManager, verticalDiffManager, continueServerClientPromise, battery, @@ -373,40 +357,32 @@ const getCommandsMap: ( ); } return { - "continue.acceptDiff": async ( - newFilepath?: string | vscode.Uri, - streamId?: string, - ) => + "continue.acceptDiff": async (newFileUri?: string, streamId?: string) => processDiff( "accept", sidebar, - diffManager, ide, verticalDiffManager, - newFilepath, + newFileUri, streamId, ), - "continue.rejectDiff": async ( - newFilepath?: string | vscode.Uri, - streamId?: string, - ) => + "continue.rejectDiff": async (newFilepath?: string, streamId?: string) => processDiff( "reject", sidebar, - diffManager, ide, verticalDiffManager, newFilepath, streamId, ), - "continue.acceptVerticalDiffBlock": (filepath?: string, index?: number) => { + "continue.acceptVerticalDiffBlock": (fileUri?: string, index?: number) => { captureCommandTelemetry("acceptVerticalDiffBlock"); - verticalDiffManager.acceptRejectVerticalDiffBlock(true, filepath, index); + verticalDiffManager.acceptRejectVerticalDiffBlock(true, fileUri, index); }, - "continue.rejectVerticalDiffBlock": (filepath?: string, index?: number) => { + "continue.rejectVerticalDiffBlock": (fileUri?: string, index?: number) => { captureCommandTelemetry("rejectVerticalDiffBlock"); - verticalDiffManager.acceptRejectVerticalDiffBlock(false, filepath, index); + verticalDiffManager.acceptRejectVerticalDiffBlock(false, fileUri, index); }, "continue.quickFix": async ( range: vscode.Range, @@ -456,6 +432,12 @@ const getCommandsMap: ( core.invoke("context/indexDocs", { reIndex: true }); }, "continue.focusContinueInput": async () => { + const isContinueInputFocused = await sidebar.webviewProtocol.request( + "isContinueInputFocused", + undefined, + false, + ); + // This is a temporary fix—sidebar.webviewProtocol.request is blocking // when the GUI hasn't yet been setup and we should instead be // immediately throwing an error, or returning a Result object @@ -472,11 +454,6 @@ const getCommandsMap: ( undefined, false, ); - const isContinueInputFocused = await sidebar.webviewProtocol.request( - "isContinueInputFocused", - undefined, - false, - ); if (isContinueInputFocused) { if (historyLength === 0) { @@ -499,19 +476,23 @@ const getCommandsMap: ( } }, "continue.focusContinueInputWithoutClear": async () => { + const isContinueInputFocused = await sidebar.webviewProtocol.request( + "isContinueInputFocused", + undefined, + false, + ); + // This is a temporary fix—sidebar.webviewProtocol.request is blocking // when the GUI hasn't yet been setup and we should instead be // immediately throwing an error, or returning a Result object + focusGUI(); if (!sidebar.isReady) { - focusGUI(); - return; + const isReady = await waitForSidebarReady(sidebar, 5000, 100); + if (!isReady) { + return; + } } - const isContinueInputFocused = await sidebar.webviewProtocol.request( - "isContinueInputFocused", - undefined, - ); - if (isContinueInputFocused) { hideGUI(); } else { @@ -602,11 +583,10 @@ const getCommandsMap: ( rangeInFileWithContents, ); } else { - const filepath = document.uri.fsPath; const contents = document.getText(); sidebar.webviewProtocol?.request("addCodeToEdit", { - filepath, + filepath: document.uri.toString(), contents, }); } @@ -664,17 +644,8 @@ const getCommandsMap: ( }, "continue.viewLogs": async () => { captureCommandTelemetry("viewLogs"); - - // Open ~/.continue/continue.log - const logFile = path.join(os.homedir(), ".continue", "continue.log"); - // Make sure the file/directory exist - if (!fs.existsSync(logFile)) { - fs.mkdirSync(path.dirname(logFile), { recursive: true }); - fs.writeFileSync(logFile, ""); - } - - const uri = vscode.Uri.file(logFile); - await vscode.window.showTextDocument(uri); + const logFilePath = getLogFilePath(); + await vscode.window.showTextDocument(vscode.Uri.file(logFilePath)); }, "continue.debugTerminal": async () => { captureCommandTelemetry("debugTerminal"); @@ -755,7 +726,10 @@ const getCommandsMap: ( "continue.toggleFullScreen": async () => { focusGUI(); - const sessionId = await sidebar.webviewProtocol.request("getCurrentSessionId", undefined); + const sessionId = await sidebar.webviewProtocol.request( + "getCurrentSessionId", + undefined, + ); // Check if full screen is already open by checking open tabs const fullScreenTab = getFullScreenTab(); @@ -789,11 +763,14 @@ const getCommandsMap: ( undefined, true, ); - + panel.onDidChangeViewState(() => { vscode.commands.executeCommand("continue.newSession"); - if(sessionId){ - vscode.commands.executeCommand("continue.focusContinueSessionId", sessionId); + if (sessionId) { + vscode.commands.executeCommand( + "continue.focusContinueSessionId", + sessionId, + ); } }); @@ -831,9 +808,9 @@ const getCommandsMap: ( .stat(uri) ?.then((stat) => stat.type === vscode.FileType.Directory); if (isDirectory) { - for await (const filepath of walkDirAsync(uri.fsPath, ide)) { + for await (const fileUri of walkDirAsync(uri.toString(), ide)) { addEntireFileToContext( - uriFromFilePath(filepath), + vscode.Uri.parse(fileUri), sidebar.webviewProtocol, ); } @@ -965,7 +942,7 @@ const getCommandsMap: ( } else if ( selectedOption === "$(gear) Configure autocomplete options" ) { - ide.openFile(getConfigJsonPath()); + ide.openFile(vscode.Uri.file(getConfigJsonPath()).toString()); } else if ( autocompleteModels.some((model) => model.title === selectedOption) ) { @@ -1056,7 +1033,6 @@ export function registerAllCommands( extensionContext: vscode.ExtensionContext, sidebar: ContinueGUIWebviewViewProvider, configHandler: ConfigHandler, - diffManager: DiffManager, verticalDiffManager: VerticalDiffManager, continueServerClientPromise: Promise, battery: Battery, @@ -1072,7 +1048,6 @@ export function registerAllCommands( extensionContext, sidebar, configHandler, - diffManager, verticalDiffManager, continueServerClientPromise, battery, diff --git a/extensions/vscode/src/diff/horizontal.ts b/extensions/vscode/src/diff/horizontal.ts deleted file mode 100644 index 65c411265f..0000000000 --- a/extensions/vscode/src/diff/horizontal.ts +++ /dev/null @@ -1,339 +0,0 @@ -import * as fs from "node:fs"; -import * as os from "node:os"; -import * as path from "node:path"; - -import { devDataPath } from "core/util/paths"; -import * as vscode from "vscode"; - -import { getMetaKeyLabel, getPlatform } from "../util/util"; -import { uriFromFilePath } from "../util/vscode"; - -import type { VsCodeWebviewProtocol } from "../webviewProtocol"; - -interface DiffInfo { - originalFilepath: string; - newFilepath: string; - editor?: vscode.TextEditor; - step_index: number; - range: vscode.Range; -} - -async function readFile(path: string): Promise { - return await vscode.workspace.fs - .readFile(uriFromFilePath(path)) - .then((bytes) => new TextDecoder().decode(bytes)); -} - -async function writeFile(uri: vscode.Uri, contents: string) { - await vscode.workspace.fs.writeFile(uri, new TextEncoder().encode(contents)); -} - -// THIS IS LOCAL -export const DIFF_DIRECTORY = path - .join(os.homedir(), ".continue", ".diffs") - .replace(/^C:/, "c:"); - -export class DiffManager { - // Create a temporary file in the global .continue directory which displays the updated version - // Doing this because virtual files are read-only - private diffs: Map = new Map(); - - diffAtNewFilepath(newFilepath: string): DiffInfo | undefined { - return this.diffs.get(newFilepath); - } - - private async setupDirectory() { - // Make sure the diff directory exists - if (!fs.existsSync(DIFF_DIRECTORY)) { - fs.mkdirSync(DIFF_DIRECTORY, { - recursive: true, - }); - } - } - - webviewProtocol: VsCodeWebviewProtocol | undefined; - - constructor(private readonly extensionContext: vscode.ExtensionContext) { - this.setupDirectory(); - - // Listen for file closes, and if it's a diff file, clean up - vscode.workspace.onDidCloseTextDocument((document) => { - const newFilepath = document.uri.fsPath; - const diffInfo = this.diffs.get(newFilepath); - if (diffInfo) { - this.cleanUpDiff(diffInfo, false); - } - }); - } - - private escapeFilepath(filepath: string): string { - return filepath - .replace(/\//g, "_f_") - .replace(/\\/g, "_b_") - .replace(/:/g, "_c_"); - } - - private remoteTmpDir = "/tmp/continue"; - private getNewFilepath(originalFilepath: string): string { - if (vscode.env.remoteName) { - // If we're in a remote, use the remote's temp directory - // Doing this because there's no easy way to find the home directory, - // and there aren't write permissions to the root directory - // and writing these to local causes separate issues - // because the vscode.diff command will always try to read from remote - vscode.workspace.fs.createDirectory(uriFromFilePath(this.remoteTmpDir)); - return path.join( - this.remoteTmpDir, - this.escapeFilepath(originalFilepath), - ); - } - return path.join(DIFF_DIRECTORY, this.escapeFilepath(originalFilepath)); - } - - private async openDiffEditor( - originalFilepath: string, - newFilepath: string, - ): Promise { - // If the file doesn't yet exist or the basename is a single digit number (vscode terminal), don't open the diff editor - try { - await vscode.workspace.fs.stat(uriFromFilePath(newFilepath)); - } catch (e) { - console.log("File doesn't exist, not opening diff editor", e); - return undefined; - } - if (path.basename(originalFilepath).match(/^\d$/)) { - return undefined; - } - - const rightUri = uriFromFilePath(newFilepath); - const leftUri = uriFromFilePath(originalFilepath); - const title = "Continue Diff"; - vscode.commands.executeCommand("vscode.diff", leftUri, rightUri, title); - - const editor = vscode.window.activeTextEditor; - if (!editor) { - return; - } - - // Change the vscode setting to allow codeLens in diff editor - vscode.workspace - .getConfiguration("diffEditor", editor.document.uri) - .update("codeLens", true, vscode.ConfigurationTarget.Global); - - if ( - this.extensionContext.globalState.get( - "continue.showDiffInfoMessage", - ) !== false - ) { - vscode.window - .showInformationMessage( - `Accept (${getMetaKeyLabel()}⇧⏎) or reject (${getMetaKeyLabel()}⇧⌫) at the top of the file.`, - "Got it", - "Don't show again", - ) - .then((selection) => { - if (selection === "Don't show again") { - // Get the global state - this.extensionContext.globalState.update( - "continue.showDiffInfoMessage", - false, - ); - } - }); - } - - return editor; - } - - private _findFirstDifferentLine(contentA: string, contentB: string): number { - const linesA = contentA.split("\n"); - const linesB = contentB.split("\n"); - for (let i = 0; i < linesA.length && i < linesB.length; i++) { - if (linesA[i] !== linesB[i]) { - return i; - } - } - return 0; - } - - async writeDiff( - originalFilepath: string, - newContent: string, - step_index: number, - ): Promise { - await this.setupDirectory(); - - // Create or update existing diff - const newFilepath = this.getNewFilepath(originalFilepath); - await writeFile(uriFromFilePath(newFilepath), newContent); - - // Open the diff editor if this is a new diff - if (!this.diffs.has(newFilepath)) { - // Figure out the first line that is different - const oldContent = await readFile(originalFilepath); - const line = this._findFirstDifferentLine(oldContent, newContent); - - const diffInfo: DiffInfo = { - originalFilepath, - newFilepath, - step_index, - range: new vscode.Range(line, 0, line + 1, 0), - }; - this.diffs.set(newFilepath, diffInfo); - } - - // Open the editor if it hasn't been opened yet - const diffInfo = this.diffs.get(newFilepath); - if (diffInfo && !diffInfo?.editor) { - diffInfo.editor = await this.openDiffEditor( - originalFilepath, - newFilepath, - ); - this.diffs.set(newFilepath, diffInfo); - } - - if (getPlatform() === "windows") { - // Just a matter of how it renders - // Lags on windows without this - // Flashes too much on mac with it - vscode.commands.executeCommand( - "workbench.action.files.revert", - uriFromFilePath(newFilepath), - ); - } - - return newFilepath; - } - - cleanUpDiff(diffInfo: DiffInfo, hideEditor = true) { - // Close the editor, remove the record, delete the file - if (hideEditor && diffInfo.editor) { - try { - vscode.window.showTextDocument(diffInfo.editor.document); - vscode.commands.executeCommand("workbench.action.closeActiveEditor"); - } catch {} - } - this.diffs.delete(diffInfo.newFilepath); - vscode.workspace.fs.delete(uriFromFilePath(diffInfo.newFilepath)); - } - - private inferNewFilepath() { - const activeEditorPath = - vscode.window.activeTextEditor?.document.uri.fsPath; - if (activeEditorPath && path.dirname(activeEditorPath) === DIFF_DIRECTORY) { - return activeEditorPath; - } - const visibleEditors = vscode.window.visibleTextEditors.map( - (editor) => editor.document.uri.fsPath, - ); - for (const editorPath of visibleEditors) { - if (path.dirname(editorPath) === DIFF_DIRECTORY) { - for (const otherEditorPath of visibleEditors) { - if ( - path.dirname(otherEditorPath) !== DIFF_DIRECTORY && - this.getNewFilepath(otherEditorPath) === editorPath - ) { - return editorPath; - } - } - } - } - - if (this.diffs.size === 1) { - return Array.from(this.diffs.keys())[0]; - } - return undefined; - } - - async acceptDiff(newFilepath?: string) { - // When coming from a keyboard shortcut, we have to infer the newFilepath from visible text editors - if (!newFilepath) { - newFilepath = this.inferNewFilepath(); - } - if (!newFilepath) { - console.log("No newFilepath provided to accept the diff"); - return; - } - // Get the diff info, copy new file to original, then delete from record and close the corresponding editor - const diffInfo = this.diffs.get(newFilepath); - if (!diffInfo) { - console.log("No corresponding diffInfo found for newFilepath"); - return; - } - - // Save the right-side file, then copy over to original - vscode.workspace.textDocuments - .find((doc) => doc.uri.fsPath === newFilepath) - ?.save() - .then(async () => { - await writeFile( - uriFromFilePath(diffInfo.originalFilepath), - await readFile(diffInfo.newFilepath), - ); - this.cleanUpDiff(diffInfo); - }); - - await recordAcceptReject(true, diffInfo); - } - - async rejectDiff(newFilepath?: string) { - // If no newFilepath is provided and there is only one in the dictionary, use that - if (!newFilepath) { - newFilepath = this.inferNewFilepath(); - } - if (!newFilepath) { - console.log( - "No newFilepath provided to reject the diff, diffs.size was", - this.diffs.size, - ); - return; - } - const diffInfo = this.diffs.get(newFilepath); - if (!diffInfo) { - console.log("No corresponding diffInfo found for newFilepath"); - return; - } - - // Stop the step at step_index in case it is still streaming - this.webviewProtocol?.request("setInactive", undefined); - - vscode.workspace.textDocuments - .find((doc) => doc.uri.fsPath === newFilepath) - ?.save() - .then(() => { - this.cleanUpDiff(diffInfo); - }); - - await recordAcceptReject(false, diffInfo); - } -} - -async function recordAcceptReject(accepted: boolean, diffInfo: DiffInfo) { - const devDataDir = devDataPath(); - const suggestionsPath = path.join(devDataDir, "suggestions.json"); - - // Initialize suggestions list - let suggestions = []; - - // Check if suggestions.json exists - try { - const rawData = await readFile(suggestionsPath); - suggestions = JSON.parse(rawData); - } catch {} - - // Add the new suggestion to the list - suggestions.push({ - accepted, - timestamp: Date.now(), - suggestion: diffInfo.originalFilepath, - }); - - // Send the suggestion to the server - // ideProtocolClient.sendAcceptRejectSuggestion(accepted); - - // Write the updated suggestions back to the file - await writeFile( - vscode.Uri.file(suggestionsPath), - JSON.stringify(suggestions, null, 4), - ); -} diff --git a/extensions/vscode/src/diff/vertical/handler.ts b/extensions/vscode/src/diff/vertical/handler.ts index c686192772..024aea2d21 100644 --- a/extensions/vscode/src/diff/vertical/handler.ts +++ b/extensions/vscode/src/diff/vertical/handler.ts @@ -10,6 +10,7 @@ import { import type { VerticalDiffCodeLens } from "./manager"; import type { ApplyState, DiffLine } from "core"; +import * as URI from "uri-js"; export interface VerticalDiffHandlerOptions { input?: string; @@ -27,8 +28,8 @@ export class VerticalDiffHandler implements vscode.Disposable { private newLinesAdded = 0; private deletionBuffer: string[] = []; private redDecorationManager: DecorationTypeRangeManager; - private get filepath() { - return this.editor.document.uri.fsPath; + private get fileUri() { + return this.editor.document.uri.toString(); } public insertedInCurrentBlock = 0; public get range(): vscode.Range { @@ -45,8 +46,8 @@ export class VerticalDiffHandler implements vscode.Disposable { string, VerticalDiffCodeLens[] >, - private readonly clearForFilepath: ( - filepath: string | undefined, + private readonly clearForFileUri: ( + fileUri: string | undefined, accept: boolean, ) => void, private readonly refreshCodeLens: () => void, @@ -65,8 +66,11 @@ export class VerticalDiffHandler implements vscode.Disposable { ); const disposable = vscode.window.onDidChangeActiveTextEditor((editor) => { + if (!editor) { + return; + } // When we switch away and back to this editor, need to re-draw decorations - if (editor?.document.uri.fsPath === this.filepath) { + if (URI.equal(editor.document.uri.toString(), this.fileUri)) { this.editor = editor; this.redDecorationManager.applyToNewEditor(editor); this.greenDecorationManager.applyToNewEditor(editor); @@ -93,7 +97,7 @@ export class VerticalDiffHandler implements vscode.Disposable { } if (this.deletionBuffer.length || this.insertedInCurrentBlock > 0) { - const blocks = this.editorToVerticalDiffCodeLens.get(this.filepath) || []; + const blocks = this.editorToVerticalDiffCodeLens.get(this.fileUri) || []; blocks.push({ start: this.currentLineIndex - this.insertedInCurrentBlock, @@ -101,7 +105,7 @@ export class VerticalDiffHandler implements vscode.Disposable { numGreen: this.insertedInCurrentBlock, }); - this.editorToVerticalDiffCodeLens.set(this.filepath, blocks); + this.editorToVerticalDiffCodeLens.set(this.fileUri, blocks); } if (this.deletionBuffer.length === 0) { @@ -249,7 +253,7 @@ export class VerticalDiffHandler implements vscode.Disposable { this.greenDecorationManager.clear(); this.clearIndexLineDecorations(); - this.editorToVerticalDiffCodeLens.delete(this.filepath); + this.editorToVerticalDiffCodeLens.delete(this.fileUri); await this.editor.edit( (editBuilder) => { @@ -270,7 +274,7 @@ export class VerticalDiffHandler implements vscode.Disposable { this.options.onStatusUpdate( "closed", - this.editorToVerticalDiffCodeLens.get(this.filepath)?.length ?? 0, + this.editorToVerticalDiffCodeLens.get(this.fileUri)?.length ?? 0, this.editor.document.getText(), ); @@ -362,19 +366,19 @@ export class VerticalDiffHandler implements vscode.Disposable { this.options.onStatusUpdate( "done", - this.editorToVerticalDiffCodeLens.get(this.filepath)?.length ?? 0, + this.editorToVerticalDiffCodeLens.get(this.fileUri)?.length ?? 0, this.editor.document.getText(), ); // Reject on user typing // const listener = vscode.workspace.onDidChangeTextDocument((e) => { - // if (e.document.uri.fsPath === this.filepath) { + // if (URI.equal(e.document.uri.toString(), this.fileUri)) { // this.clear(false); // listener.dispose(); // } // }); } catch (e) { - this.clearForFilepath(this.filepath, false); + this.clearForFileUri(this.fileUri, false); throw e; } return diffLines; @@ -415,7 +419,7 @@ export class VerticalDiffHandler implements vscode.Disposable { this.shiftCodeLensObjects(startLine, offset); const numDiffs = - this.editorToVerticalDiffCodeLens.get(this.filepath)?.length ?? 0; + this.editorToVerticalDiffCodeLens.get(this.fileUri)?.length ?? 0; const status = numDiffs === 0 ? "closed" : undefined; this.options.onStatusUpdate( @@ -429,7 +433,7 @@ export class VerticalDiffHandler implements vscode.Disposable { // Shift the codelens objects const blocks = this.editorToVerticalDiffCodeLens - .get(this.filepath) + .get(this.fileUri) ?.filter((x) => x.start !== startLine) .map((x) => { if (x.start > startLine) { @@ -437,18 +441,18 @@ export class VerticalDiffHandler implements vscode.Disposable { } return x; }) || []; - this.editorToVerticalDiffCodeLens.set(this.filepath, blocks); + this.editorToVerticalDiffCodeLens.set(this.fileUri, blocks); this.refreshCodeLens(); } public updateLineDelta( - filepath: string, + fileUri: string, startLine: number, lineDelta: number, ) { // Retrieve the diff blocks for the given file - const blocks = this.editorToVerticalDiffCodeLens.get(filepath); + const blocks = this.editorToVerticalDiffCodeLens.get(fileUri); if (!blocks) { return; } @@ -462,7 +466,7 @@ export class VerticalDiffHandler implements vscode.Disposable { } public hasDiffForCurrentFile(): boolean { - const diffBlocks = this.editorToVerticalDiffCodeLens.get(this.filepath); + const diffBlocks = this.editorToVerticalDiffCodeLens.get(this.fileUri); return diffBlocks !== undefined && diffBlocks.length > 0; } } diff --git a/extensions/vscode/src/diff/vertical/manager.ts b/extensions/vscode/src/diff/vertical/manager.ts index 3342c53414..ee47fd1732 100644 --- a/extensions/vscode/src/diff/vertical/manager.ts +++ b/extensions/vscode/src/diff/vertical/manager.ts @@ -9,6 +9,7 @@ import EditDecorationManager from "../../quickEdit/EditDecorationManager"; import { VsCodeWebviewProtocol } from "../../webviewProtocol"; import { VerticalDiffHandler, VerticalDiffHandlerOptions } from "./handler"; +import * as URI from "uri-js"; export interface VerticalDiffCodeLens { start: number; @@ -19,9 +20,9 @@ export interface VerticalDiffCodeLens { export class VerticalDiffManager { public refreshCodeLens: () => void = () => {}; - private filepathToHandler: Map = new Map(); + private fileUriToHandler: Map = new Map(); - filepathToCodeLens: Map = new Map(); + fileUriToCodeLens: Map = new Map(); private userChangeListener: vscode.Disposable | undefined; @@ -36,35 +37,35 @@ export class VerticalDiffManager { } createVerticalDiffHandler( - filepath: string, + fileUri: string, startLine: number, endLine: number, options: VerticalDiffHandlerOptions, ) { - if (this.filepathToHandler.has(filepath)) { - this.filepathToHandler.get(filepath)?.clear(false); - this.filepathToHandler.delete(filepath); + if (this.fileUriToHandler.has(fileUri)) { + this.fileUriToHandler.get(fileUri)?.clear(false); + this.fileUriToHandler.delete(fileUri); } const editor = vscode.window.activeTextEditor; // TODO - if (editor && editor.document.uri.fsPath === filepath) { + if (editor && URI.equal(editor.document.uri.toString(), fileUri)) { const handler = new VerticalDiffHandler( startLine, endLine, editor, - this.filepathToCodeLens, - this.clearForFilepath.bind(this), + this.fileUriToCodeLens, + this.clearForfileUri.bind(this), this.refreshCodeLens, options, ); - this.filepathToHandler.set(filepath, handler); + this.fileUriToHandler.set(fileUri, handler); return handler; } else { return undefined; } } - getHandlerForFile(filepath: string) { - return this.filepathToHandler.get(filepath); + getHandlerForFile(fileUri: string) { + return this.fileUriToHandler.get(fileUri); } // Creates a listener for document changes by user. @@ -77,8 +78,8 @@ export class VerticalDiffManager { this.userChangeListener = vscode.workspace.onDidChangeTextDocument( (event) => { // Check if there is an active handler for the affected file - const filepath = event.document.uri.fsPath; - const handler = this.getHandlerForFile(filepath); + const fileUri = event.document.uri.toString(); + const handler = this.getHandlerForFile(fileUri); if (handler) { // If there is an active diff for that file, handle the document change this.handleDocumentChange(event, handler); @@ -108,26 +109,26 @@ export class VerticalDiffManager { // Update the diff handler with the new line delta handler.updateLineDelta( - event.document.uri.fsPath, + event.document.uri.toString(), change.range.start.line, lineDelta, ); }); } - clearForFilepath(filepath: string | undefined, accept: boolean) { - if (!filepath) { + clearForfileUri(fileUri: string | undefined, accept: boolean) { + if (!fileUri) { const activeEditor = vscode.window.activeTextEditor; if (!activeEditor) { return; } - filepath = activeEditor.document.uri.fsPath; + fileUri = activeEditor.document.uri.toString(); } - const handler = this.filepathToHandler.get(filepath); + const handler = this.fileUriToHandler.get(fileUri); if (handler) { handler.clear(accept); - this.filepathToHandler.delete(filepath); + this.fileUriToHandler.delete(fileUri); } this.disableDocumentChangeListener(); @@ -137,28 +138,28 @@ export class VerticalDiffManager { async acceptRejectVerticalDiffBlock( accept: boolean, - filepath?: string, + fileUri?: string, index?: number, ) { - if (!filepath) { + if (!fileUri) { const activeEditor = vscode.window.activeTextEditor; if (!activeEditor) { return; } - filepath = activeEditor.document.uri.fsPath; + fileUri = activeEditor.document.uri.toString(); } if (typeof index === "undefined") { index = 0; } - const blocks = this.filepathToCodeLens.get(filepath); + const blocks = this.fileUriToCodeLens.get(fileUri); const block = blocks?.[index]; if (!blocks || !block) { return; } - const handler = this.getHandlerForFile(filepath); + const handler = this.getHandlerForFile(fileUri); if (!handler) { return; } @@ -175,7 +176,7 @@ export class VerticalDiffManager { ); if (blocks.length === 1) { - this.clearForFilepath(filepath, true); + this.clearForfileUri(fileUri, true); } else { // Re-enable listener for user changes to file this.enableDocumentChangeListener(); @@ -191,17 +192,17 @@ export class VerticalDiffManager { ) { vscode.commands.executeCommand("setContext", "continue.diffVisible", true); - // Get the current editor filepath/range + // Get the current editor fileUri/range let editor = vscode.window.activeTextEditor; if (!editor) { return; } - const filepath = editor.document.uri.fsPath; + const fileUri = editor.document.uri.toString(); const startLine = 0; const endLine = editor.document.lineCount - 1; // Check for existing handlers in the same file the new one will be created in - const existingHandler = this.getHandlerForFile(filepath); + const existingHandler = this.getHandlerForFile(fileUri); if (existingHandler) { existingHandler.clear(false); } @@ -212,7 +213,7 @@ export class VerticalDiffManager { // Create new handler with determined start/end const diffHandler = this.createVerticalDiffHandler( - filepath, + fileUri, startLine, endLine, { @@ -223,7 +224,7 @@ export class VerticalDiffManager { status, numDiffs, fileContent, - filepath, + filepath: fileUri, }), }, ); @@ -280,7 +281,7 @@ export class VerticalDiffManager { return undefined; } - const filepath = editor.document.uri.fsPath; + const fileUri = editor.document.uri.toString(); let startLine, endLine: number; @@ -293,7 +294,7 @@ export class VerticalDiffManager { } // Check for existing handlers in the same file the new one will be created in - const existingHandler = this.getHandlerForFile(filepath); + const existingHandler = this.getHandlerForFile(fileUri); if (existingHandler) { if (quickEdit) { @@ -333,7 +334,7 @@ export class VerticalDiffManager { // Create new handler with determined start/end const diffHandler = this.createVerticalDiffHandler( - filepath, + fileUri, startLine, endLine, { @@ -345,7 +346,7 @@ export class VerticalDiffManager { status, numDiffs, fileContent, - filepath, + filepath: fileUri, }), }, ); @@ -411,7 +412,7 @@ export class VerticalDiffManager { suffix, llm, input, - getMarkdownLanguageTagForFile(filepath), + getMarkdownLanguageTagForFile(fileUri), onlyOneInsertion, ); diff --git a/extensions/vscode/src/extension/VsCodeExtension.ts b/extensions/vscode/src/extension/VsCodeExtension.ts index da54a6bc36..ce506cfec6 100644 --- a/extensions/vscode/src/extension/VsCodeExtension.ts +++ b/extensions/vscode/src/extension/VsCodeExtension.ts @@ -22,7 +22,6 @@ import { } from "../autocomplete/statusBar"; import { registerAllCommands } from "../commands"; import { ContinueGUIWebviewViewProvider } from "../ContinueGUIWebviewViewProvider"; -import { DiffManager } from "../diff/horizontal"; import { VerticalDiffManager } from "../diff/vertical/manager"; import { registerAllCodeLensProviders } from "../lang-server/codeLens"; import { registerAllPromptFilesCompletionProviders } from "../lang-server/promptFileCompletions"; @@ -33,7 +32,6 @@ import { getControlPlaneSessionInfo, WorkOsAuthProvider, } from "../stubs/WorkOsAuthProvider"; -import { arePathsEqual } from "../util/arePathsEqual"; import { Battery } from "../util/battery"; import { FileSearch } from "../util/FileSearch"; import { TabAutocompleteModel } from "../util/loadAutocompleteModel"; @@ -41,7 +39,6 @@ import { VsCodeIde } from "../VsCodeIde"; import { VsCodeMessenger } from "./VsCodeMessenger"; -import { SYSTEM_PROMPT_DOT_FILE } from "core/config/getSystemPromptDotFile"; import type { VsCodeWebviewProtocol } from "../webviewProtocol"; export class VsCodeExtension { @@ -53,7 +50,6 @@ export class VsCodeExtension { private tabAutocompleteModel: TabAutocompleteModel; private sidebar: ContinueGUIWebviewViewProvider; private windowId: string; - private diffManager: DiffManager; private editDecorationManager: EditDecorationManager; private verticalDiffManager: VerticalDiffManager; webviewProtocolPromise: Promise; @@ -76,12 +72,7 @@ export class VsCodeExtension { resolveWebviewProtocol = resolve; }, ); - this.diffManager = new DiffManager(context); - this.ide = new VsCodeIde( - this.diffManager, - this.webviewProtocolPromise, - context, - ); + this.ide = new VsCodeIde(this.webviewProtocolPromise, context); this.extensionContext = context; this.windowId = uuidv4(); @@ -158,14 +149,10 @@ export class VsCodeExtension { this.configHandler.reloadConfig.bind(this.configHandler), ); - // Indexing + pause token - this.diffManager.webviewProtocol = this.sidebar.webviewProtocol; - this.configHandler.loadConfig().then((config) => { const { verticalDiffCodeLens } = registerAllCodeLensProviders( context, - this.diffManager, - this.verticalDiffManager.filepathToCodeLens, + this.verticalDiffManager.fileUriToCodeLens, config, ); @@ -187,8 +174,7 @@ export class VsCodeExtension { registerAllCodeLensProviders( context, - this.diffManager, - this.verticalDiffManager.filepathToCodeLens, + this.verticalDiffManager.fileUriToCodeLens, newConfig, ); } @@ -246,7 +232,6 @@ export class VsCodeExtension { context, this.sidebar, this.configHandler, - this.diffManager, this.verticalDiffManager, this.core.continueServerClientPromise, this.battery, @@ -277,57 +262,20 @@ export class VsCodeExtension { }); vscode.workspace.onDidSaveTextDocument(async (event) => { - // Listen for file changes in the workspace - const filepath = event.uri.fsPath; - - if (arePathsEqual(filepath, getConfigJsonPath())) { - // Trigger a toast notification to provide UI feedback that config - // has been updated - const showToast = context.globalState.get( - "showConfigUpdateToast", - true, - ); - - if (showToast) { - vscode.window - .showInformationMessage("Config updated", "Don't show again") - .then((selection) => { - if (selection === "Don't show again") { - context.globalState.update("showConfigUpdateToast", false); - } - }); - } - } - - if ( - filepath.endsWith(".continuerc.json") || - filepath.endsWith(".prompt") || - filepath.endsWith(SYSTEM_PROMPT_DOT_FILE) - ) { - this.configHandler.reloadConfig(); - } else if ( - filepath.endsWith(".continueignore") || - filepath.endsWith(".gitignore") - ) { - // Reindex the workspaces - this.core.invoke("index/forceReIndex", undefined); - } else { - // Reindex the file - this.core.invoke("index/forceReIndexFiles", { - files: [filepath], - }); - } + this.core.invoke("files/changed", { + uris: [event.uri.toString()], + }); }); vscode.workspace.onDidDeleteFiles(async (event) => { - this.core.invoke("index/forceReIndexFiles", { - files: event.files.map((file) => file.fsPath), + this.core.invoke("files/deleted", { + uris: event.files.map((uri) => uri.toString()), }); }); vscode.workspace.onDidCreateFiles(async (event) => { - this.core.invoke("index/forceReIndexFiles", { - files: event.files.map((file) => file.fsPath), + this.core.invoke("files/created", { + uris: event.files.map((uri) => uri.toString()), }); }); @@ -368,7 +316,7 @@ export class VsCodeExtension { // Refresh index when branch is changed this.ide.getWorkspaceDirs().then((dirs) => dirs.forEach(async (dir) => { - const repo = await this.ide.getRepo(vscode.Uri.file(dir)); + const repo = await this.ide.getRepo(dir); if (repo) { repo.state.onDidChange(() => { // args passed to this callback are always undefined, so keep track of previous branch diff --git a/extensions/vscode/src/extension/VsCodeMessenger.ts b/extensions/vscode/src/extension/VsCodeMessenger.ts index b00c100492..d8b2400346 100644 --- a/extensions/vscode/src/extension/VsCodeMessenger.ts +++ b/extensions/vscode/src/extension/VsCodeMessenger.ts @@ -1,6 +1,3 @@ -import * as fs from "node:fs"; -import * as path from "node:path"; - import { ConfigHandler } from "core/config/ConfigHandler"; import { getModelByRole } from "core/config/util"; import { applyCodeBlock } from "core/edit/lazy/applyCodeBlock"; @@ -16,7 +13,6 @@ import { CORE_TO_WEBVIEW_PASS_THROUGH, WEBVIEW_TO_CORE_PASS_THROUGH, } from "core/protocol/passThrough"; -import { getBasename } from "core/util"; import { InProcessMessenger, Message } from "core/protocol/messenger"; import * as vscode from "vscode"; @@ -27,10 +23,11 @@ import { getControlPlaneSessionInfo, WorkOsAuthProvider, } from "../stubs/WorkOsAuthProvider"; -import { getFullyQualifiedPath } from "../util/util"; import { getExtensionUri } from "../util/vscode"; import { VsCodeIde } from "../VsCodeIde"; import { VsCodeWebviewProtocol } from "../webviewProtocol"; +import { getUriPathBasename } from "core/util/uri"; +import { showTutorial } from "../util/tutorial"; /** * A shared messenger class between Core and Webview @@ -87,38 +84,20 @@ export class VsCodeMessenger { ) { /** WEBVIEW ONLY LISTENERS **/ this.onWebview("showFile", (msg) => { - const fullPath = getFullyQualifiedPath(this.ide, msg.data.filepath); - - if (fullPath) { - this.ide.openFile(fullPath); - } + this.ide.openFile(msg.data.filepath); }); this.onWebview("vscode/openMoveRightMarkdown", (msg) => { vscode.commands.executeCommand( "markdown.showPreview", - vscode.Uri.file( - path.join( - getExtensionUri().fsPath, - "media", - "move-chat-panel-right.md", - ), + vscode.Uri.joinPath( + getExtensionUri(), + "media", + "move-chat-panel-right.md", ), ); }); - this.onWebview("readRangeInFile", async (msg) => { - return await vscode.workspace - .openTextDocument(msg.data.filepath) - .then((document) => { - const start = new vscode.Position(0, 0); - const end = new vscode.Position(5, 0); - const range = new vscode.Range(start, end); - - const contents = document.getText(range); - return contents; - }); - }); this.onWebview("toggleDevTools", (msg) => { vscode.commands.executeCommand("workbench.action.toggleDevTools"); vscode.commands.executeCommand("continue.viewLogs"); @@ -132,42 +111,22 @@ export class VsCodeMessenger { this.onWebview("toggleFullScreen", (msg) => { vscode.commands.executeCommand("continue.toggleFullScreen"); }); - // History - this.onWebview("saveFile", async (msg) => { - return await ide.saveFile(msg.data.filepath); - }); - this.onWebview("readFile", async (msg) => { - return await ide.readFile(msg.data.filepath); - }); - this.onWebview("showDiff", async (msg) => { - return await ide.showDiff( - msg.data.filepath, - msg.data.newContents, - msg.data.stepIndex, + + this.onWebview("acceptDiff", async ({ data: { filepath, streamId } }) => { + await vscode.commands.executeCommand( + "continue.acceptDiff", + filepath, + streamId, ); }); - webviewProtocol.on( - "acceptDiff", - async ({ data: { filepath, streamId } }) => { - await vscode.commands.executeCommand( - "continue.acceptDiff", - filepath, - streamId, - ); - }, - ); - - webviewProtocol.on( - "rejectDiff", - async ({ data: { filepath, streamId } }) => { - await vscode.commands.executeCommand( - "continue.rejectDiff", - filepath, - streamId, - ); - }, - ); + this.onWebview("rejectDiff", async ({ data: { filepath, streamId } }) => { + await vscode.commands.executeCommand( + "continue.rejectDiff", + filepath, + streamId, + ); + }); this.onWebview("applyToFile", async ({ data }) => { webviewProtocol.request("updateApplyState", { @@ -175,26 +134,15 @@ export class VsCodeMessenger { status: "streaming", fileContent: data.text, }); - - let filepath = data.filepath; - - // If there is a filepath, verify it exists and then open the file - if (filepath) { - const fullPath = getFullyQualifiedPath(ide, filepath); - - if (!fullPath) { - return; - } - - const fileExists = await this.ide.fileExists(fullPath); - - // If there is no existing file at the path, create it + console.log("applyToFile", data); + if (data.filepath) { + const fileExists = await this.ide.fileExists(data.filepath); if (!fileExists) { - await this.ide.writeFile(fullPath, ""); - await this.ide.openFile(fullPath); + await this.ide.writeFile(data.filepath, ""); + await this.ide.openFile(data.filepath); } - await this.ide.openFile(fullPath); + await this.ide.openFile(data.filepath); } // Get active text editor @@ -246,7 +194,7 @@ export class VsCodeMessenger { const [instant, diffLines] = await applyCodeBlock( editor.document.getText(), data.text, - getBasename(editor.document.fileName), + getUriPathBasename(editor.document.uri.toString()), llm, fastLlm, ); @@ -283,29 +231,7 @@ export class VsCodeMessenger { }); this.onWebview("showTutorial", async (msg) => { - const tutorialPath = path.join( - getExtensionUri().fsPath, - "continue_tutorial.py", - ); - // Ensure keyboard shortcuts match OS - if (process.platform !== "darwin") { - let tutorialContent = fs.readFileSync(tutorialPath, "utf8"); - tutorialContent = tutorialContent - .replace("⌘", "^") - .replace("Cmd", "Ctrl"); - fs.writeFileSync(tutorialPath, tutorialContent); - } - - const doc = await vscode.workspace.openTextDocument( - vscode.Uri.file(tutorialPath), - ); - await vscode.window.showTextDocument(doc, { - preview: false, - }); - }); - - this.onWebview("openUrl", (msg) => { - vscode.env.openExternal(vscode.Uri.parse(msg.data)); + await showTutorial(this.ide); }); this.onWebview( @@ -434,6 +360,19 @@ export class VsCodeMessenger { // None right now /** BOTH CORE AND WEBVIEW **/ + this.onWebviewOrCore("readRangeInFile", async (msg) => { + return await vscode.workspace + .openTextDocument(msg.data.filepath) + .then((document) => { + const start = new vscode.Position(0, 0); + const end = new vscode.Position(5, 0); + const range = new vscode.Range(start, end); + + const contents = document.getText(range); + return contents; + }); + }); + this.onWebviewOrCore("getIdeSettings", async (msg) => { return ide.getIdeSettings(); }); @@ -458,18 +397,12 @@ export class VsCodeMessenger { this.onWebviewOrCore("getWorkspaceDirs", async (msg) => { return ide.getWorkspaceDirs(); }); - this.onWebviewOrCore("listFolders", async (msg) => { - return ide.listFolders(); - }); this.onWebviewOrCore("writeFile", async (msg) => { return ide.writeFile(msg.data.path, msg.data.contents); }); this.onWebviewOrCore("showVirtualFile", async (msg) => { return ide.showVirtualFile(msg.data.name, msg.data.content); }); - this.onWebviewOrCore("getContinueDir", async (msg) => { - return ide.getContinueDir(); - }); this.onWebviewOrCore("openFile", async (msg) => { return ide.openFile(msg.data.path); }); @@ -522,5 +455,58 @@ export class VsCodeMessenger { false, ); }); + this.onWebviewOrCore("saveFile", async (msg) => { + return await ide.saveFile(msg.data.filepath); + }); + this.onWebviewOrCore("readFile", async (msg) => { + return await ide.readFile(msg.data.filepath); + }); + this.onWebviewOrCore("openUrl", (msg) => { + vscode.env.openExternal(vscode.Uri.parse(msg.data)); + }); + + this.onWebviewOrCore("fileExists", async (msg) => { + return await ide.fileExists(msg.data.filepath); + }); + + this.onWebviewOrCore("gotoDefinition", async (msg) => { + return await ide.gotoDefinition(msg.data.location); + }); + + this.onWebviewOrCore("getLastModified", async (msg) => { + return await ide.getLastModified(msg.data.files); + }); + + this.onWebviewOrCore("getGitRootPath", async (msg) => { + return await ide.getGitRootPath(msg.data.dir); + }); + + this.onWebviewOrCore("listDir", async (msg) => { + return await ide.listDir(msg.data.dir); + }); + + this.onWebviewOrCore("getRepoName", async (msg) => { + return await ide.getRepoName(msg.data.dir); + }); + + this.onWebviewOrCore("getTags", async (msg) => { + return await ide.getTags(msg.data); + }); + + this.onWebviewOrCore("getIdeInfo", async (msg) => { + return await ide.getIdeInfo(); + }); + + this.onWebviewOrCore("isTelemetryEnabled", async (msg) => { + return await ide.isTelemetryEnabled(); + }); + + this.onWebviewOrCore("getWorkspaceConfigs", async (msg) => { + return await ide.getWorkspaceConfigs(); + }); + + this.onWebviewOrCore("getUniqueId", async (msg) => { + return await ide.getUniqueId(); + }); } } diff --git a/extensions/vscode/src/extension/autocomplete.ts b/extensions/vscode/src/extension/autocomplete.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/extensions/vscode/src/extension/indexing.ts b/extensions/vscode/src/extension/indexing.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/extensions/vscode/src/lang-server/codeLens/providers/DiffViewerCodeLensProvider.ts b/extensions/vscode/src/lang-server/codeLens/providers/DiffViewerCodeLensProvider.ts deleted file mode 100644 index f3ef5c67a1..0000000000 --- a/extensions/vscode/src/lang-server/codeLens/providers/DiffViewerCodeLensProvider.ts +++ /dev/null @@ -1,39 +0,0 @@ -import path from "path"; - -import * as vscode from "vscode"; - -import { DiffManager, DIFF_DIRECTORY } from "../../../diff/horizontal"; -import { getMetaKeyLabel } from "../../../util/util"; - -export class DiffViewerCodeLensProvider implements vscode.CodeLensProvider { - constructor(private diffManager: DiffManager) {} - - public provideCodeLenses( - document: vscode.TextDocument, - _: vscode.CancellationToken, - ): vscode.CodeLens[] | Thenable { - if (path.dirname(document.uri.fsPath) === DIFF_DIRECTORY) { - const codeLenses: vscode.CodeLens[] = []; - let range = new vscode.Range(0, 0, 1, 0); - const diffInfo = this.diffManager.diffAtNewFilepath(document.uri.fsPath); - if (diffInfo) { - range = diffInfo.range; - } - codeLenses.push( - new vscode.CodeLens(range, { - title: `Accept All ✅ (${getMetaKeyLabel()}⇧⏎)`, - command: "continue.acceptDiff", - arguments: [document.uri.fsPath], - }), - new vscode.CodeLens(range, { - title: `Reject All ❌ (${getMetaKeyLabel()}⇧⌫)`, - command: "continue.rejectDiff", - arguments: [document.uri.fsPath], - }), - ); - return codeLenses; - } else { - return []; - } - } -} diff --git a/extensions/vscode/src/lang-server/codeLens/providers/QuickActionsCodeLensProvider.ts b/extensions/vscode/src/lang-server/codeLens/providers/QuickActionsCodeLensProvider.ts index b4c30350b4..18ed64dda2 100644 --- a/extensions/vscode/src/lang-server/codeLens/providers/QuickActionsCodeLensProvider.ts +++ b/extensions/vscode/src/lang-server/codeLens/providers/QuickActionsCodeLensProvider.ts @@ -7,11 +7,7 @@ import { CONTINUE_WORKSPACE_KEY, getContinueWorkspaceConfig, } from "../../../util/workspaceConfig"; - -const TUTORIAL_FILE_NAME = "continue_tutorial.py"; -function isTutorialFile(uri: vscode.Uri) { - return uri.fsPath.endsWith(TUTORIAL_FILE_NAME); -} +import { isTutorialFile } from "../../../util/tutorial"; export const ENABLE_QUICK_ACTIONS_KEY = "enableQuickActions"; diff --git a/extensions/vscode/src/lang-server/codeLens/providers/TutorialCodeLensProvider.ts b/extensions/vscode/src/lang-server/codeLens/providers/TutorialCodeLensProvider.ts deleted file mode 100644 index 139597f9cb..0000000000 --- a/extensions/vscode/src/lang-server/codeLens/providers/TutorialCodeLensProvider.ts +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/extensions/vscode/src/lang-server/codeLens/providers/VerticalPerLineCodeLensProvider.ts b/extensions/vscode/src/lang-server/codeLens/providers/VerticalPerLineCodeLensProvider.ts index 5d07e6ce23..52fd07f793 100644 --- a/extensions/vscode/src/lang-server/codeLens/providers/VerticalPerLineCodeLensProvider.ts +++ b/extensions/vscode/src/lang-server/codeLens/providers/VerticalPerLineCodeLensProvider.ts @@ -24,8 +24,8 @@ export class VerticalDiffCodeLensProvider implements vscode.CodeLensProvider { document: vscode.TextDocument, _: vscode.CancellationToken, ): vscode.CodeLens[] | Thenable { - const filepath = document.uri.fsPath; - const blocks = this.editorToVerticalDiffCodeLens.get(filepath); + const uri = document.uri.toString(); + const blocks = this.editorToVerticalDiffCodeLens.get(uri); if (!blocks) { return []; @@ -45,12 +45,12 @@ export class VerticalDiffCodeLensProvider implements vscode.CodeLensProvider { new vscode.CodeLens(range, { title: `Accept`, command: "continue.acceptVerticalDiffBlock", - arguments: [filepath, i], + arguments: [uri, i], }), new vscode.CodeLens(range, { title: `Reject`, command: "continue.rejectVerticalDiffBlock", - arguments: [filepath, i], + arguments: [uri, i], }), ); diff --git a/extensions/vscode/src/lang-server/codeLens/providers/index.ts b/extensions/vscode/src/lang-server/codeLens/providers/index.ts index 8285b1fe5e..00a309e62a 100644 --- a/extensions/vscode/src/lang-server/codeLens/providers/index.ts +++ b/extensions/vscode/src/lang-server/codeLens/providers/index.ts @@ -1,4 +1,3 @@ -export { DiffViewerCodeLensProvider } from "./DiffViewerCodeLensProvider"; export { QuickActionsCodeLensProvider } from "./QuickActionsCodeLensProvider"; export { SuggestionsCodeLensProvider } from "./SuggestionsCodeLensProvider"; export { VerticalDiffCodeLensProvider as VerticalPerLineCodeLensProvider } from "./VerticalPerLineCodeLensProvider"; diff --git a/extensions/vscode/src/lang-server/codeLens/registerAllCodeLensProviders.ts b/extensions/vscode/src/lang-server/codeLens/registerAllCodeLensProviders.ts index 56d6962064..ec168de539 100644 --- a/extensions/vscode/src/lang-server/codeLens/registerAllCodeLensProviders.ts +++ b/extensions/vscode/src/lang-server/codeLens/registerAllCodeLensProviders.ts @@ -1,7 +1,6 @@ import { ContinueConfig } from "core"; import * as vscode from "vscode"; -import { DiffManager } from "../../diff/horizontal"; import { VerticalDiffCodeLens } from "../../diff/vertical/manager"; import * as providers from "./providers"; @@ -67,7 +66,6 @@ function registerQuickActionsProvider( * It also sets up a subscription to VS Code Quick Actions settings changes. * * @param context - The VS Code extension context - * @param diffManager - The DiffManager instance for managing diffs * @param editorToVerticalDiffCodeLens - A Map of editor IDs to VerticalDiffCodeLens arrays * @param config - The Continue configuration object * @@ -75,7 +73,6 @@ function registerQuickActionsProvider( */ export function registerAllCodeLensProviders( context: vscode.ExtensionContext, - diffManager: DiffManager, editorToVerticalDiffCodeLens: Map, config: ContinueConfig, ) { @@ -113,11 +110,6 @@ export function registerAllCodeLensProviders( new providers.SuggestionsCodeLensProvider(), ); - diffsCodeLensDisposable = registerCodeLensProvider( - "*", - new providers.DiffViewerCodeLensProvider(diffManager), - ); - registerQuickActionsProvider(config, context); subscribeToVSCodeQuickActionsSettings(() => @@ -126,7 +118,6 @@ export function registerAllCodeLensProviders( context.subscriptions.push(verticalPerLineCodeLensProvider); context.subscriptions.push(suggestionsCodeLensDisposable); - context.subscriptions.push(diffsCodeLensDisposable); return { verticalDiffCodeLens }; } diff --git a/extensions/vscode/src/lang-server/promptFileCompletions.ts b/extensions/vscode/src/lang-server/promptFileCompletions.ts index ef873b44dc..ba15906854 100644 --- a/extensions/vscode/src/lang-server/promptFileCompletions.ts +++ b/extensions/vscode/src/lang-server/promptFileCompletions.ts @@ -1,8 +1,8 @@ import { IDE } from "core"; -import { getBasename, getLastNPathParts } from "core/util"; import vscode from "vscode"; import { FileSearch } from "../util/FileSearch"; +import { getUriPathBasename, getLastNPathParts } from "core/util/uri"; class PromptFilesCompletionItemProvider implements vscode.CompletionItemProvider @@ -30,13 +30,16 @@ class PromptFilesCompletionItemProvider } const searchText = linePrefix.split("@").pop() || ""; - const files = this.fileSearch.search(searchText).map(({ filename }) => { - return filename; - }); + const files = this.fileSearch.search(searchText); if (files.length === 0) { const openFiles = await this.ide.getOpenFiles(); - files.push(...openFiles); + files.push( + ...openFiles.map((fileUri) => ({ + id: fileUri, + relativePath: vscode.workspace.asRelativePath(fileUri), + })), + ); } // Provide completion items @@ -84,17 +87,10 @@ class PromptFilesCompletionItemProvider kind: vscode.CompletionItemKind.Field, }, ...files.map((file) => { - const workspaceFolder = vscode.workspace.getWorkspaceFolder( - vscode.Uri.file(file), - ); - const relativePath = workspaceFolder - ? vscode.workspace.asRelativePath(file) - : file; - return { - label: getBasename(file), - detail: getLastNPathParts(file, 2), - insertText: relativePath, + label: getUriPathBasename(file.id), + detail: getLastNPathParts(file.relativePath, 2), + insertText: file.relativePath, kind: vscode.CompletionItemKind.File, }; }), diff --git a/extensions/vscode/src/quickEdit/QuickEditQuickPick.ts b/extensions/vscode/src/quickEdit/QuickEditQuickPick.ts index 4b59dcbda2..aefc7408f7 100644 --- a/extensions/vscode/src/quickEdit/QuickEditQuickPick.ts +++ b/extensions/vscode/src/quickEdit/QuickEditQuickPick.ts @@ -17,7 +17,6 @@ import { getModelQuickPickVal } from "./ModelSelectionQuickPick"; // @ts-ignore - error finding typings // @ts-ignore - /** * Used to track what action to take after a user interacts * with the initial Quick Pick @@ -133,11 +132,11 @@ export class QuickEdit { } const hasChanges = !!this.verticalDiffManager.getHandlerForFile( - editor.document.uri.fsPath, + editor.document.uri.toString(), ); if (hasChanges) { - this.openAcceptRejectMenu("", editor.document.uri.fsPath); + this.openAcceptRejectMenu("", editor.document.uri.toString()); } else { await this.initiateNewQuickPick(editor, params); } @@ -185,7 +184,7 @@ export class QuickEdit { }); if (prompt) { - await this.handleUserPrompt(prompt, editor.document.uri.fsPath); + await this.handleUserPrompt(prompt, editor.document.uri.toString()); } } @@ -249,7 +248,7 @@ export class QuickEdit { private setActiveEditorAndPrevInput(editor: vscode.TextEditor) { const existingHandler = this.verticalDiffManager.getHandlerForFile( - editor.document.uri.fsPath ?? "", + editor.document.uri.toString(), ); this.editorWhenOpened = editor; @@ -448,8 +447,8 @@ export class QuickEdit { if (searchResults.length > 0) { quickPick.items = searchResults - .map(({ filename }) => ({ - label: filename, + .map(({ relativePath }) => ({ + label: relativePath, alwaysShow: true, })) .slice(0, QuickEdit.maxFileSearchResults); diff --git a/extensions/vscode/src/quickEdit/index.ts b/extensions/vscode/src/quickEdit/index.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/extensions/vscode/src/suggestions.ts b/extensions/vscode/src/suggestions.ts index f4acb9be75..06c47552f0 100644 --- a/extensions/vscode/src/suggestions.ts +++ b/extensions/vscode/src/suggestions.ts @@ -1,6 +1,7 @@ import * as vscode from "vscode"; import { openEditorAndRevealRange, translate } from "./util/vscode"; +import * as URI from "uri-js"; export interface SuggestionRanges { oldRange: vscode.Range; @@ -14,7 +15,6 @@ export const editorToSuggestions: Map< string, // URI of file SuggestionRanges[] > = new Map(); -export const editorSuggestionsLocked: Map = new Map(); // Map from editor URI to whether the suggestions are locked export const currentSuggestion: Map = new Map(); // Map from editor URI to index of current SuggestionRanges in editorToSuggestions // When tab is reopened, rerender the decorations: @@ -57,8 +57,8 @@ const oldSelDecorationType = vscode.window.createTextEditorDecorationType({ export function rerenderDecorations(editorUri: string) { const suggestions = editorToSuggestions.get(editorUri); const idx = currentSuggestion.get(editorUri); - const editor = vscode.window.visibleTextEditors.find( - (editor) => editor.document.uri.toString() === editorUri, + const editor = vscode.window.visibleTextEditors.find((editor) => + URI.equal(editor.document.uri.toString(), editorUri), ); if (!suggestions || !editor) { return; @@ -140,6 +140,7 @@ export function suggestionDownCommand() { return; } const editorUri = editor.document.uri.toString(); + const uriString = editorUri.toString(); const suggestions = editorToSuggestions.get(editorUri); const idx = currentSuggestion.get(editorUri); if (!suggestions || idx === undefined) { @@ -299,7 +300,7 @@ export async function rejectSuggestionCommand( } export async function showSuggestion( - editorFilename: string, + editorUri: vscode.Uri, range: vscode.Range, suggestion: string, ): Promise { @@ -312,7 +313,7 @@ export async function showSuggestion( return Promise.resolve(false); } - const editor = await openEditorAndRevealRange(editorFilename, range); + const editor = await openEditorAndRevealRange(editorUri, range); if (!editor) { return Promise.resolve(false); } @@ -339,19 +340,19 @@ export async function showSuggestion( ); const content = editor!.document.getText(suggestionRange); - const filename = editor!.document.uri.toString(); - if (editorToSuggestions.has(filename)) { - const suggestions = editorToSuggestions.get(filename)!; + const uriString = editor!.document.uri.toString(); + if (editorToSuggestions.has(uriString)) { + const suggestions = editorToSuggestions.get(uriString)!; suggestions.push({ oldRange: range, newRange: suggestionRange, newSelected: true, newContent: content, }); - editorToSuggestions.set(filename, suggestions); - currentSuggestion.set(filename, suggestions.length - 1); + editorToSuggestions.set(uriString, suggestions); + currentSuggestion.set(uriString, suggestions.length - 1); } else { - editorToSuggestions.set(filename, [ + editorToSuggestions.set(uriString, [ { oldRange: range, newRange: suggestionRange, @@ -359,10 +360,10 @@ export async function showSuggestion( newContent: content, }, ]); - currentSuggestion.set(filename, 0); + currentSuggestion.set(uriString, 0); } - rerenderDecorations(filename); + rerenderDecorations(uriString); } resolve(success); }, diff --git a/extensions/vscode/src/test/test-suites/ideUtils.test.ts b/extensions/vscode/src/test/test-suites/ideUtils.test.ts index 3a289a0794..37ba5cc335 100644 --- a/extensions/vscode/src/test/test-suites/ideUtils.test.ts +++ b/extensions/vscode/src/test/test-suites/ideUtils.test.ts @@ -10,6 +10,9 @@ import { testWorkspacePath } from "../runner/runTestOnVSCodeHost"; const util = require("node:util"); const asyncExec = util.promisify(require("node:child_process").exec); +/* + TODO check uri => path assumptions, will only work in some environments. +*/ describe("IDE Utils", () => { const utils = new VsCodeIdeUtils(); const testPyPath = path.join(testWorkspacePath, "test.py"); @@ -17,22 +20,16 @@ describe("IDE Utils", () => { test("getWorkspaceDirectories", async () => { const [dir] = utils.getWorkspaceDirectories(); - assert(dir.endsWith("test-workspace")); + assert(dir.toString().endsWith("test-workspace")); }); test("fileExists", async () => { const exists2 = await utils.fileExists( - path.join(testWorkspacePath, "test.py"), + vscode.Uri.file(path.join(testWorkspacePath, "test.py")), ); assert(exists2); }); - test("getAbsolutePath", async () => { - const groundTruth = path.join(testWorkspacePath, "test.py"); - assert(utils.getAbsolutePath("test.py") === groundTruth); - assert(utils.getAbsolutePath(groundTruth) === groundTruth); - }); - test("getOpenFiles", async () => { let openFiles = utils.getOpenFiles(); assert(openFiles.length === 0); @@ -43,7 +40,7 @@ describe("IDE Utils", () => { }); openFiles = utils.getOpenFiles(); assert(openFiles.length === 1); - assert(openFiles[0] === testPyPath); + assert(openFiles[0].fsPath === testPyPath); document = await vscode.workspace.openTextDocument(testJsPath); await vscode.window.showTextDocument(document, { @@ -51,8 +48,8 @@ describe("IDE Utils", () => { }); openFiles = utils.getOpenFiles(); assert(openFiles.length === 2); - assert(openFiles.includes(testPyPath)); - assert(openFiles.includes(testJsPath)); + assert(openFiles.find((f) => f.fsPath === testPyPath)); + assert(openFiles.find((f) => f.fsPath === testJsPath)); }); test("getUniqueId", async () => { diff --git a/extensions/vscode/src/util/FileSearch.ts b/extensions/vscode/src/util/FileSearch.ts index cc02f3128c..c6707fa511 100644 --- a/extensions/vscode/src/util/FileSearch.ts +++ b/extensions/vscode/src/util/FileSearch.ts @@ -1,39 +1,36 @@ import { IDE } from "core"; -import { walkDir } from "core/indexing/walkDir"; +import { walkDirs } from "core/indexing/walkDir"; // @ts-ignore import MiniSearch from "minisearch"; import * as vscode from "vscode"; -type FileMiniSearchResult = { filename: string }; +type FileMiniSearchResult = { relativePath: string; id: string }; +/* + id = file URI +*/ export class FileSearch { constructor(private readonly ide: IDE) { this.initializeFileSearchState(); } private miniSearch = new MiniSearch({ - fields: ["filename"], - storeFields: ["filename"], + fields: ["relativePath", "id"], + storeFields: ["relativePath", "id"], searchOptions: { prefix: true, fuzzy: 2, + fields: ["relativePath"], }, }); private async initializeFileSearchState() { - const workspaceDirs = await this.ide.getWorkspaceDirs(); - - const results = await Promise.all( - workspaceDirs.map((dir) => { - return walkDir(dir, this.ide); - }), + const results = await walkDirs(this.ide); + this.miniSearch.addAll( + results.flat().map((uri) => ({ + id: uri, + relativePath: vscode.workspace.asRelativePath(uri), + })), ); - - const filenames = results.flat().map((file) => ({ - id: file, - filename: vscode.workspace.asRelativePath(file), - })); - - this.miniSearch.addAll(filenames); } public search(query: string): FileMiniSearchResult[] { diff --git a/extensions/vscode/src/util/arePathsEqual.ts b/extensions/vscode/src/util/arePathsEqual.ts deleted file mode 100644 index cd10e53996..0000000000 --- a/extensions/vscode/src/util/arePathsEqual.ts +++ /dev/null @@ -1,11 +0,0 @@ -import * as os from "os"; - -export function arePathsEqual(path1: string, path2: string): boolean { - if (os.platform() === "win32") { - // On Windows, compare paths case-insensitively - return path1.toLowerCase() === path2.toLowerCase(); - } else { - // On other platforms, compare paths case-sensitively - return path1 === path2; - } -} diff --git a/extensions/vscode/src/util/expandSnippet.ts b/extensions/vscode/src/util/expandSnippet.ts index 3b9c45bb10..2c41235980 100644 --- a/extensions/vscode/src/util/expandSnippet.ts +++ b/extensions/vscode/src/util/expandSnippet.ts @@ -3,23 +3,23 @@ import { languageForFilepath } from "core/autocomplete/constants/AutocompleteLan import { DEFAULT_IGNORE_DIRS } from "core/indexing/ignore"; import { deduplicateArray } from "core/util"; import { getParserForFile } from "core/util/treeSitter"; - +import * as vscode from "vscode"; import { getDefinitionsForNode } from "../autocomplete/lsp"; import type { SyntaxNode } from "web-tree-sitter"; export async function expandSnippet( - filepath: string, + fileUri: string, startLine: number, endLine: number, ide: IDE, ): Promise { - const parser = await getParserForFile(filepath); + const parser = await getParserForFile(fileUri); if (!parser) { return []; } - const fullFileContents = await ide.readFile(filepath); + const fullFileContents = await ide.readFile(fileUri); const root: SyntaxNode = parser.parse(fullFileContents).rootNode; // Find all nodes contained in the range @@ -53,10 +53,10 @@ export async function expandSnippet( await Promise.all( callExpressions.map(async (node) => { return getDefinitionsForNode( - filepath, + vscode.Uri.parse(fileUri), node, ide, - languageForFilepath(filepath), + languageForFilepath(fileUri), ); }), ) @@ -79,7 +79,7 @@ export async function expandSnippet( // Filter out definitions already in selected range callExpressionDefinitions = callExpressionDefinitions.filter((def) => { return !( - def.filepath === filepath && + def.filepath === fileUri && def.range.start.line >= startLine && def.range.end.line <= endLine ); diff --git a/extensions/vscode/src/util/ideUtils.ts b/extensions/vscode/src/util/ideUtils.ts index d8b083de61..f576d6d1e7 100644 --- a/extensions/vscode/src/util/ideUtils.ts +++ b/extensions/vscode/src/util/ideUtils.ts @@ -1,27 +1,21 @@ -import path from "node:path"; - import { EXTENSION_NAME } from "core/control-plane/env"; import _ from "lodash"; import * as vscode from "vscode"; - +import * as URI from "uri-js"; import { threadStopped } from "../debug/debug"; import { VsCodeExtension } from "../extension/VsCodeExtension"; import { GitExtension, Repository } from "../otherExtensions/git"; import { SuggestionRanges, acceptSuggestionCommand, - editorSuggestionsLocked, rejectSuggestionCommand, showSuggestion as showSuggestionInEditor, } from "../suggestions"; -import { - getUniqueId, - openEditorAndRevealRange, - uriFromFilePath, -} from "./vscode"; +import { getUniqueId, openEditorAndRevealRange } from "./vscode"; -import type { FileEdit, RangeInFile, Thread } from "core"; +import type { Range, RangeInFile, Thread } from "core"; +import { findUriInDirs } from "core/util/uri"; const util = require("node:util"); const asyncExec = util.promisify(require("node:child_process").exec); @@ -30,48 +24,47 @@ export class VsCodeIdeUtils { visibleMessages: Set = new Set(); async gotoDefinition( - filepath: string, + uri: vscode.Uri, position: vscode.Position, ): Promise { const locations: vscode.Location[] = await vscode.commands.executeCommand( "vscode.executeDefinitionProvider", - uriFromFilePath(filepath), + uri, position, ); return locations; } - async documentSymbol(filepath: string): Promise { + async documentSymbol(uri: vscode.Uri): Promise { return await vscode.commands.executeCommand( "vscode.executeDocumentSymbolProvider", - uriFromFilePath(filepath), + uri, ); } async references( - filepath: string, + uri: vscode.Uri, position: vscode.Position, ): Promise { return await vscode.commands.executeCommand( "vscode.executeReferenceProvider", - uriFromFilePath(filepath), + uri, position, ); } - async foldingRanges(filepath: string): Promise { + async foldingRanges(uri: vscode.Uri): Promise { return await vscode.commands.executeCommand( "vscode.executeFoldingRangeProvider", - uriFromFilePath(filepath), + uri, ); } - private _workspaceDirectories: string[] | undefined = undefined; - getWorkspaceDirectories(): string[] { + private _workspaceDirectories: vscode.Uri[] | undefined = undefined; + getWorkspaceDirectories(): vscode.Uri[] { if (this._workspaceDirectories === undefined) { this._workspaceDirectories = - vscode.workspace.workspaceFolders?.map((folder) => folder.uri.fsPath) || - []; + vscode.workspace.workspaceFolders?.map((folder) => folder.uri) || []; } return this._workspaceDirectories; @@ -81,54 +74,32 @@ export class VsCodeIdeUtils { return getUniqueId(); } - // ------------------------------------ // - // On message handlers - - showSuggestion(edit: FileEdit) { - // showSuggestion already exists + showSuggestion(uri: vscode.Uri, range: Range, suggestion: string) { showSuggestionInEditor( - edit.filepath, + uri, new vscode.Range( - edit.range.start.line, - edit.range.start.character, - edit.range.end.line, - edit.range.end.character, + range.start.line, + range.start.character, + range.end.line, + range.end.character, ), - edit.replacement, + suggestion, ); } - async resolveAbsFilepathInWorkspace(filepath: string): Promise { - // If the filepath is already absolute, return it as is - if (this.path.isAbsolute(filepath)) { - return filepath; - } - - // Try to resolve for each workspace directory - const workspaceDirectories = this.getWorkspaceDirectories(); - for (const dir of workspaceDirectories) { - const resolvedPath = this.path.resolve(dir, filepath); - if (await this.fileExists(resolvedPath)) { - return resolvedPath; - } - } - - return filepath; - } - - async openFile(filepath: string, range?: vscode.Range) { + async openFile(uri: vscode.Uri, range?: vscode.Range) { // vscode has a builtin open/get open files return await openEditorAndRevealRange( - await this.resolveAbsFilepathInWorkspace(filepath), + uri, range, vscode.ViewColumn.One, false, ); } - async fileExists(filepath: string): Promise { + async fileExists(uri: vscode.Uri): Promise { try { - await vscode.workspace.fs.stat(uriFromFilePath(filepath)); + await vscode.workspace.fs.stat(uri); return true; } catch { return false; @@ -149,11 +120,6 @@ export class VsCodeIdeUtils { }); } - setSuggestionsLocked(filepath: string, locked: boolean) { - editorSuggestionsLocked.set(filepath, locked); - // TODO: Rerender? - } - async getUserSecret(key: string) { // Check if secret already exists in VS Code settings (global) let secret = vscode.workspace.getConfiguration(EXTENSION_NAME).get(key); @@ -196,7 +162,7 @@ export class VsCodeIdeUtils { return uri.scheme === "file" || uri.scheme === "vscode-remote"; } - getOpenFiles(): string[] { + getOpenFiles(): vscode.Uri[] { return vscode.window.tabGroups.all .map((group) => { return group.tabs.map((tab) => { @@ -205,52 +171,22 @@ export class VsCodeIdeUtils { }) .flat() .filter(Boolean) // filter out undefined values - .filter((uri) => this.documentIsCode(uri)) // Filter out undesired documents - .map((uri) => uri.fsPath); + .filter((uri) => this.documentIsCode(uri)); // Filter out undesired documents } - saveFile(filepath: string) { + saveFile(uri: vscode.Uri) { vscode.window.visibleTextEditors .filter((editor) => this.documentIsCode(editor.document.uri)) .forEach((editor) => { - if (editor.document.uri.fsPath === filepath) { + if (URI.equal(editor.document.uri.toString(), uri.toString())) { editor.document.save(); } }); } - private _cachedPath: path.PlatformPath | undefined; - get path(): path.PlatformPath { - if (this._cachedPath) { - return this._cachedPath; - } - - // Return "path" module for either windows or posix depending on sample workspace folder path format - const sampleWorkspaceFolder = - vscode.workspace.workspaceFolders?.[0]?.uri.fsPath; - const isWindows = sampleWorkspaceFolder - ? !sampleWorkspaceFolder.startsWith("/") - : false; - - this._cachedPath = isWindows ? path.win32 : path.posix; - return this._cachedPath; - } - - getAbsolutePath(filepath: string): string { - const workspaceDirectories = this.getWorkspaceDirectories(); - if (!this.path.isAbsolute(filepath) && workspaceDirectories.length === 1) { - return this.path.join(workspaceDirectories[0], filepath); - } else { - return filepath; - } - } - - async readRangeInFile( - filepath: string, - range: vscode.Range, - ): Promise { + async readRangeInFile(uri: vscode.Uri, range: vscode.Range): Promise { const contents = new TextDecoder().decode( - await vscode.workspace.fs.readFile(vscode.Uri.file(filepath)), + await vscode.workspace.fs.readFile(uri), ); const lines = contents.split("\n"); return `${lines @@ -463,13 +399,16 @@ export class VsCodeIdeUtils { private static secondsToWaitForGitToLoad = process.env.NODE_ENV === "test" ? 1 : 20; async getRepo(forDirectory: vscode.Uri): Promise { - const workspaceDirs = this.getWorkspaceDirectories(); - const parentDir = workspaceDirs.find((dir) => - forDirectory.fsPath.startsWith(dir), + const workspaceDirs = this.getWorkspaceDirectories().map((dir) => + dir.toString(), + ); + const { foundInDir } = findUriInDirs( + forDirectory.toString(), + workspaceDirs, ); - if (parentDir) { + if (foundInDir) { // Check if the repository is already cached - const cachedRepo = this.repoCache.get(parentDir); + const cachedRepo = this.repoCache.get(foundInDir); if (cachedRepo) { return cachedRepo; } @@ -492,17 +431,17 @@ export class VsCodeIdeUtils { repo = await this._getRepo(forDirectory); } - if (parentDir) { + if (foundInDir) { // Cache the repository for the parent directory - this.repoCache.set(parentDir, repo); + this.repoCache.set(foundInDir, repo); } return repo; } - async getGitRoot(forDirectory: string): Promise { - const repo = await this.getRepo(vscode.Uri.file(forDirectory)); - return repo?.rootUri?.fsPath; + async getGitRoot(forDirectory: vscode.Uri): Promise { + const repo = await this.getRepo(forDirectory); + return repo?.rootUri; } async getBranch(forDirectory: vscode.Uri) { @@ -537,7 +476,7 @@ export class VsCodeIdeUtils { const diffs: string[] = []; for (const dir of this.getWorkspaceDirectories()) { - const repo = await this.getRepo(vscode.Uri.file(dir)); + const repo = await this.getRepo(dir); if (!repo) { continue; } @@ -552,31 +491,4 @@ export class VsCodeIdeUtils { return diffs.flatMap((diff) => this.splitDiff(diff)); } - - getHighlightedCode(): RangeInFile[] { - // TODO - const rangeInFiles: RangeInFile[] = []; - vscode.window.visibleTextEditors - .filter((editor) => this.documentIsCode(editor.document.uri)) - .forEach((editor) => { - editor.selections.forEach((selection) => { - // if (!selection.isEmpty) { - rangeInFiles.push({ - filepath: editor.document.uri.fsPath, - range: { - start: { - line: selection.start.line, - character: selection.start.character, - }, - end: { - line: selection.end.line, - character: selection.end.character, - }, - }, - }); - // } - }); - }); - return rangeInFiles; - } } diff --git a/extensions/vscode/src/util/tutorial.ts b/extensions/vscode/src/util/tutorial.ts new file mode 100644 index 0000000000..45a86a6757 --- /dev/null +++ b/extensions/vscode/src/util/tutorial.ts @@ -0,0 +1,27 @@ +import { IDE } from "core"; +import { getExtensionUri } from "./vscode"; +import * as vscode from "vscode"; + +const TUTORIAL_FILE_NAME = "continue_tutorial.py"; +export function getTutorialUri(): vscode.Uri { + return vscode.Uri.joinPath(getExtensionUri(), TUTORIAL_FILE_NAME); +} + +export function isTutorialFile(uri: vscode.Uri) { + return uri.path.endsWith(TUTORIAL_FILE_NAME); +} + +export async function showTutorial(ide: IDE) { + const tutorialUri = getTutorialUri(); + // Ensure keyboard shortcuts match OS + if (process.platform !== "darwin") { + let tutorialContent = await ide.readFile(tutorialUri.toString()); + tutorialContent = tutorialContent.replace("⌘", "^").replace("Cmd", "Ctrl"); + await ide.writeFile(tutorialUri.toString(), tutorialContent); + } + + const doc = await vscode.workspace.openTextDocument(tutorialUri); + await vscode.window.showTextDocument(doc, { + preview: false, + }); +} diff --git a/extensions/vscode/src/util/util.ts b/extensions/vscode/src/util/util.ts index 48777fa072..6691a66d20 100644 --- a/extensions/vscode/src/util/util.ts +++ b/extensions/vscode/src/util/util.ts @@ -1,7 +1,5 @@ import * as vscode from "vscode"; -import { VsCodeIde } from "../VsCodeIde"; - const os = require("node:os"); function charIsEscapedAtIndex(index: number, str: string): boolean { @@ -124,13 +122,3 @@ export function getExtensionVersion(): string { const extension = vscode.extensions.getExtension("continue.continue"); return extension?.packageJSON.version || "0.1.0"; } - -export function getFullyQualifiedPath(ide: VsCodeIde, filepath: string) { - if (ide.ideUtils.path.isAbsolute(filepath)) {return filepath;} - - const workspaceFolders = vscode.workspace.workspaceFolders; - - if (workspaceFolders && workspaceFolders.length > 0) { - return ide.ideUtils.path.join(workspaceFolders[0].uri.fsPath, filepath); - } -} diff --git a/extensions/vscode/src/util/vscode.ts b/extensions/vscode/src/util/vscode.ts index 16c15bb195..d8cb9cffd2 100644 --- a/extensions/vscode/src/util/vscode.ts +++ b/extensions/vscode/src/util/vscode.ts @@ -1,7 +1,6 @@ -import * as path from "node:path"; - import { machineIdSync } from "node-machine-id"; import * as vscode from "vscode"; +import * as URI from "uri-js"; export function translate(range: vscode.Range, lines: number): vscode.Range { return new vscode.Range( @@ -27,13 +26,13 @@ export function getExtensionUri(): vscode.Uri { } export function getViewColumnOfFile( - filepath: string, + uri: vscode.Uri, ): vscode.ViewColumn | undefined { for (const tabGroup of vscode.window.tabGroups.all) { for (const tab of tabGroup.tabs) { if ( (tab?.input as any)?.uri && - (tab.input as any).uri.fsPath === filepath + URI.equal((tab.input as any).uri, uri.toString()) ) { return tabGroup.viewColumn; } @@ -71,20 +70,13 @@ export function getRightViewColumn(): vscode.ViewColumn { let showTextDocumentInProcess = false; export function openEditorAndRevealRange( - editorFilename: string, + uri: vscode.Uri, range?: vscode.Range, viewColumn?: vscode.ViewColumn, preview?: boolean, ): Promise { return new Promise((resolve, _) => { - let filename = editorFilename; - if (editorFilename.startsWith("~")) { - filename = path.join( - process.env.HOME || process.env.USERPROFILE || "", - editorFilename.slice(1), - ); - } - vscode.workspace.openTextDocument(filename).then(async (doc) => { + vscode.workspace.openTextDocument(uri).then(async (doc) => { try { // An error is thrown mysteriously if you open two documents in parallel, hence this while (showTextDocumentInProcess) { @@ -97,7 +89,7 @@ export function openEditorAndRevealRange( showTextDocumentInProcess = true; vscode.window .showTextDocument(doc, { - viewColumn: getViewColumnOfFile(editorFilename) || viewColumn, + viewColumn: getViewColumnOfFile(uri) || viewColumn, preview, }) .then((editor) => { @@ -114,47 +106,6 @@ export function openEditorAndRevealRange( }); } -function windowsToPosix(windowsPath: string): string { - let posixPath = windowsPath.split("\\").join("/"); - if (posixPath[1] === ":") { - posixPath = posixPath.slice(2); - } - // posixPath = posixPath.replace(" ", "\\ "); - return posixPath; -} - -function isWindowsLocalButNotRemote(): boolean { - return ( - vscode.env.remoteName !== undefined && - [ - "wsl", - "ssh-remote", - "dev-container", - "attached-container", - "tunnel", - ].includes(vscode.env.remoteName) && - process.platform === "win32" - ); -} - -export function getPathSep(): string { - return isWindowsLocalButNotRemote() ? "/" : path.sep; -} - -export function uriFromFilePath(filepath: string): vscode.Uri { - let finalPath = filepath; - if (vscode.env.remoteName) { - if (isWindowsLocalButNotRemote()) { - finalPath = windowsToPosix(filepath); - } - return vscode.Uri.parse( - `vscode-remote://${vscode.env.remoteName}${finalPath}`, - ); - } else { - return vscode.Uri.file(finalPath); - } -} - export function getUniqueId() { const id = vscode.env.machineId; if (id === "someValue.machineId") { diff --git a/extensions/vscode/src/webviewProtocol.ts b/extensions/vscode/src/webviewProtocol.ts index c639d0bf3c..0632d44b9e 100644 --- a/extensions/vscode/src/webviewProtocol.ts +++ b/extensions/vscode/src/webviewProtocol.ts @@ -1,6 +1,3 @@ -import fs from "node:fs"; -import path from "path"; - import { FromWebviewProtocol, ToWebviewProtocol } from "core/protocol"; import { WebviewMessengerResult } from "core/protocol/util"; import { extractMinimalStackTraceInfo } from "core/util/extractMinimalStackTraceInfo"; @@ -12,25 +9,6 @@ import * as vscode from "vscode"; import { IMessenger } from "../../../core/protocol/messenger"; import { showFreeTrialLoginMessage } from "./util/messages"; -import { getExtensionUri } from "./util/vscode"; - -export async function showTutorial() { - const tutorialPath = path.join( - getExtensionUri().fsPath, - "continue_tutorial.py", - ); - // Ensure keyboard shortcuts match OS - if (process.platform !== "darwin") { - let tutorialContent = fs.readFileSync(tutorialPath, "utf8"); - tutorialContent = tutorialContent.replace("⌘", "^").replace("Cmd", "Ctrl"); - fs.writeFileSync(tutorialPath, tutorialContent); - } - - const doc = await vscode.workspace.openTextDocument( - vscode.Uri.file(tutorialPath), - ); - await vscode.window.showTextDocument(doc, { preview: false }); -} export class VsCodeWebviewProtocol implements IMessenger diff --git a/gui/src/components/CodeToEditCard/CodeToEditCard.tsx b/gui/src/components/CodeToEditCard/CodeToEditCard.tsx index cac18f4e15..7bd469c240 100644 --- a/gui/src/components/CodeToEditCard/CodeToEditCard.tsx +++ b/gui/src/components/CodeToEditCard/CodeToEditCard.tsx @@ -41,10 +41,10 @@ export default function CodeToEditCard() { } } - async function onSelectFilesToAdd(filepaths: string[]) { - const filePromises = filepaths.map(async (filepath) => { - const contents = await ideMessenger.ide.readFile(filepath); - return { contents, filepath }; + async function onSelectFilesToAdd(uris: string[]) { + const filePromises = uris.map(async (uri) => { + const contents = await ideMessenger.ide.readFile(uri); + return { contents, filepath: uri }; }); const fileResults = await Promise.all(filePromises); diff --git a/gui/src/components/CodeToEditCard/CodeToEditListItem.tsx b/gui/src/components/CodeToEditCard/CodeToEditListItem.tsx index ed46517e24..46380f0843 100644 --- a/gui/src/components/CodeToEditCard/CodeToEditListItem.tsx +++ b/gui/src/components/CodeToEditCard/CodeToEditListItem.tsx @@ -6,9 +6,13 @@ import { } from "@heroicons/react/24/outline"; import { useState } from "react"; import StyledMarkdownPreview from "../markdown/StyledMarkdownPreview"; -import { getLastNPathParts, getMarkdownLanguageTagForFile } from "core/util"; +import { getMarkdownLanguageTagForFile } from "core/util"; import styled from "styled-components"; import { CodeToEdit } from "core"; +import { + getLastNUriRelativePathParts, + getUriPathBasename, +} from "core/util/uri"; export interface CodeToEditListItemProps { code: CodeToEdit; @@ -35,11 +39,15 @@ export default function CodeToEditListItem({ }: CodeToEditListItemProps) { const [showCodeSnippet, setShowCodeSnippet] = useState(false); - const filepath = code.filepath.split("/").pop() || code.filepath; - const fileSubpath = getLastNPathParts(code.filepath, 2); + const fileName = getUriPathBasename(code.filepath); + const last2Parts = getLastNUriRelativePathParts( + window.workspacePaths ?? [], + code.filepath, + 2, + ); let isInsertion = false; - let title = filepath; + let title = fileName; if ("range" in code) { const start = code.range.start.line + 1; @@ -85,7 +93,7 @@ export default function CodeToEditListItem({ {title} - {fileSubpath} + {last2Parts} diff --git a/gui/src/components/FileIcon.tsx b/gui/src/components/FileIcon.tsx index 96321348c1..995b59be10 100644 --- a/gui/src/components/FileIcon.tsx +++ b/gui/src/components/FileIcon.tsx @@ -1,4 +1,5 @@ import DOMPurify from "dompurify"; +import { useMemo } from "react"; import { themeIcons } from "seti-file-icons"; export interface FileIconProps { @@ -7,10 +8,15 @@ export interface FileIconProps { width: string; } export default function FileIcon({ filename, height, width }: FileIconProps) { - const filenameParts = filename.includes(" (") - ? filename.split(" ") - : [filename, ""]; - filenameParts.pop(); + const file = useMemo(() => { + if (filename.includes(" (")) { + const path = filename.split(" "); + path.pop(); + return path.join(" "); + } else { + return filename; + } + }, [filename]); const getIcon = themeIcons({ blue: "#268bd2", @@ -27,7 +33,7 @@ export default function FileIcon({ filename, height, width }: FileIconProps) { }); // Sanitize the SVG string before rendering it - const { svg, color } = getIcon(filenameParts.join(" ")); + const { svg, color } = getIcon(file); const sanitizedSVG = DOMPurify.sanitize(svg); return ( diff --git a/gui/src/components/History/HistoryTableRow.tsx b/gui/src/components/History/HistoryTableRow.tsx index a529274347..11a5a36df3 100644 --- a/gui/src/components/History/HistoryTableRow.tsx +++ b/gui/src/components/History/HistoryTableRow.tsx @@ -12,11 +12,10 @@ import { updateSession, } from "../../redux/thunks/session"; import HeaderButtonWithToolTip from "../gui/HeaderButtonWithToolTip"; - -function lastPartOfPath(path: string): string { - const sep = path.includes("/") ? "/" : "\\"; - return path.split(sep).pop() || path; -} +import { + getLastNUriRelativePathParts, + getUriPathBasename, +} from "core/util/uri"; export function HistoryTableRow({ sessionMetadata, @@ -108,7 +107,7 @@ export function HistoryTableRow({
- {lastPartOfPath(sessionMetadata.workspaceDirectory || "")} + {getUriPathBasename(sessionMetadata.workspaceDirectory || "")} {/* Uncomment to show the date */} {/* diff --git a/gui/src/components/StepContainer/StepContainer.tsx b/gui/src/components/StepContainer/StepContainer.tsx index a497a0b22c..470b426673 100644 --- a/gui/src/components/StepContainer/StepContainer.tsx +++ b/gui/src/components/StepContainer/StepContainer.tsx @@ -10,6 +10,7 @@ import ResponseActions from "./ResponseActions"; import { useAppSelector } from "../../redux/hooks"; import { selectUIConfig } from "../../redux/slices/configSlice"; import { deleteMessage } from "../../redux/slices/sessionSlice"; +import ThinkingIndicator from "./ThinkingIndicator"; interface StepContainerProps { item: ChatHistoryItem; @@ -95,8 +96,9 @@ export default function StepContainer(props: StepContainerProps) { itemIndex={props.index} /> )} + {props.isLast && } - {/* We want to occupy space in the DOM regardless of whether the actions are visible to avoid jank on */} + {/* We want to occupy space in the DOM regardless of whether the actions are visible to avoid jank on stream complete */}
{!shouldHideActions && ( { + // Animation for thinking ellipses + const [animation, setAnimation] = useState(2); + useEffect(() => { + const interval = setInterval(() => { + setAnimation((prevState) => (prevState === 2 ? 0 : prevState + 1)); + }, 600); + return () => { + clearInterval(interval); + }; + }, []); + + const selectedModel = useAppSelector(selectDefaultModel); + const isStreaming = useAppSelector((state) => state.session.isStreaming); + + const hasContent = Array.isArray(historyItem.message.content) + ? !!historyItem.message.content.length + : !!historyItem.message.content; + const isO1 = selectedModel?.model.startsWith("o1"); + const isThinking = + isStreaming && !historyItem.isGatheringContext && !hasContent; + if (!isThinking || !isO1) { + return null; + } + + return ( +
+ {`Thinking.${".".repeat(animation)}`} +
+ ); +}; + +export default ThinkingIndicator; diff --git a/gui/src/components/mainInput/ContextItemsPeek.tsx b/gui/src/components/mainInput/ContextItemsPeek.tsx index 1e3997930d..77a9f222d5 100644 --- a/gui/src/components/mainInput/ContextItemsPeek.tsx +++ b/gui/src/components/mainInput/ContextItemsPeek.tsx @@ -5,7 +5,6 @@ import { } from "@heroicons/react/24/outline"; import { ContextItemWithId } from "core"; import { ctxItemToRifWithContents } from "core/commands/util"; -import { getBasename } from "core/util"; import { useContext, useMemo, useState } from "react"; import { AnimatedEllipsis, lightGray, vscBackground } from ".."; import { IdeMessengerContext } from "../../context/IdeMessenger"; @@ -14,6 +13,7 @@ import { selectIsGatheringContext } from "../../redux/slices/sessionSlice"; import FileIcon from "../FileIcon"; import SafeImg from "../SafeImg"; import { getIconFromDropdownItem } from "./MentionList"; +import { getUriPathBasename } from "core/util/uri"; interface ContextItemsPeekProps { contextItems?: ContextItemWithId[]; @@ -29,10 +29,14 @@ function ContextItemsPeekItem({ contextItem }: ContextItemsPeekItemProps) { const isUrl = contextItem.uri?.type === "url"; function openContextItem() { - const { uri, name, description, content } = contextItem; + const { uri, name, content } = contextItem; if (isUrl) { - ideMessenger.post("openUrl", uri.value); + if (uri?.value) { + ideMessenger.post("openUrl", uri.value); + } else { + console.error("Couldn't open url", uri); + } } else if (uri) { const isRangeInFile = name.includes(" (") && name.endsWith(")"); @@ -44,7 +48,7 @@ function ContextItemsPeekItem({ contextItem }: ContextItemsPeekItemProps) { rif.range.end.line, ); } else { - ideMessenger.ide.openFile(description); + ideMessenger.ide.openFile(uri.value); } } else { ideMessenger.ide.showVirtualFile(name, content); @@ -115,7 +119,7 @@ function ContextItemsPeekItem({ contextItem }: ContextItemsPeekItemProps) { className={`min-w-0 flex-1 overflow-hidden truncate whitespace-nowrap text-xs text-gray-400 ${isUrl ? "hover:underline" : ""}`} > {contextItem.uri?.type === "file" - ? getBasename(contextItem.description) + ? getUriPathBasename(contextItem.description) : contextItem.description}
diff --git a/gui/src/components/mainInput/TipTapEditor.tsx b/gui/src/components/mainInput/TipTapEditor.tsx index 1aee797437..b4e6616d78 100644 --- a/gui/src/components/mainInput/TipTapEditor.tsx +++ b/gui/src/components/mainInput/TipTapEditor.tsx @@ -6,14 +6,8 @@ import Placeholder from "@tiptap/extension-placeholder"; import Text from "@tiptap/extension-text"; import { Plugin } from "@tiptap/pm/state"; import { Editor, EditorContent, JSONContent, useEditor } from "@tiptap/react"; -import { - ContextItemWithId, - ContextProviderDescription, - InputModifiers, - RangeInFile, -} from "core"; +import { ContextProviderDescription, InputModifiers } from "core"; import { modelSupportsImages } from "core/llm/autodetect"; -import { getBasename, getRelativePath } from "core/util"; import { debounce } from "lodash"; import { usePostHog } from "posthog-js/react"; import { @@ -25,7 +19,6 @@ import { useState, } from "react"; import styled from "styled-components"; -import { v4 } from "uuid"; import { defaultBorderRadius, lightGray, @@ -77,6 +70,7 @@ import { loadSession, saveCurrentSession, } from "../../redux/thunks/session"; +import { rifWithContentsToContextItem } from "core/commands/util"; const InputBoxDiv = styled.div<{ border?: string }>` resize: none; @@ -668,7 +662,9 @@ function TipTapEditor(props: TipTapEditorProps) { if (!props.isMainInput || !mainInputContentTrigger) { return; } - editor.commands.setContent(mainInputContentTrigger); + queueMicrotask(() => { + editor.commands.setContent(mainInputContentTrigger); + }); dispatch(setMainEditorContentTrigger(undefined)); }, [editor, props.isMainInput, mainInputContentTrigger]); @@ -751,37 +747,30 @@ function TipTapEditor(props: TipTapEditorProps) { return; } - const rif: RangeInFile & { contents: string } = - data.rangeInFileWithContents; - const basename = getBasename(rif.filepath); - const relativePath = getRelativePath( - rif.filepath, - await ideMessenger.ide.getWorkspaceDirs(), + // const rif: RangeInFile & { contents: string } = + // data.rangeInFileWithContents; + // const basename = getBasename(rif.filepath); + // const relativePath = getRelativePath( + // rif.filepath, + // await ideMessenger.ide.getWorkspaceDirs(), + // const rangeStr = `(${rif.range.start.line + 1}-${ + // rif.range.end.line + 1 + // })`; + + // const itemName = `${basename} ${rangeStr}`; + // const item: ContextItemWithId = { + // content: rif.contents, + // name: itemName + // } + + const contextItem = rifWithContentsToContextItem( + data.rangeInFileWithContents, ); - const rangeStr = `(${rif.range.start.line + 1}-${ - rif.range.end.line + 1 - })`; - - const itemName = `${basename} ${rangeStr}`; - const item: ContextItemWithId = { - content: rif.contents, - name: itemName, - // Description is passed on to the LLM to give more context on file path - description: `${relativePath} ${rangeStr}`, - id: { - providerTitle: "code", - itemId: v4(), - }, - uri: { - type: "file", - value: rif.filepath, - }, - }; - + console.log(contextItem); let index = 0; - for (const el of editor.getJSON().content) { - if (el.attrs?.item?.name === itemName) { - return; // Prevent duplicate code blocks + for (const el of editor.getJSON()?.content ?? []) { + if (el.attrs?.item?.name === contextItem.name) { + return; // Prevent exact duplicate code blocks } if (el.type === "codeBlock") { index += 2; @@ -794,7 +783,7 @@ function TipTapEditor(props: TipTapEditorProps) { .insertContentAt(index, { type: "codeBlock", attrs: { - item, + item: contextItem, }, }) .run(); @@ -813,13 +802,7 @@ function TipTapEditor(props: TipTapEditorProps) { editor.commands.focus("end"); }, 20); }, - [ - editor, - props.isMainInput, - historyLength, - props.isMainInput, - onEnterRef.current, - ], + [editor, props.isMainInput, historyLength, onEnterRef.current], ); useWebviewListener( diff --git a/gui/src/components/mainInput/resolveInput.ts b/gui/src/components/mainInput/resolveInput.ts index e3af025000..febf7adb07 100644 --- a/gui/src/components/mainInput/resolveInput.ts +++ b/gui/src/components/mainInput/resolveInput.ts @@ -11,6 +11,8 @@ import { stripImages } from "core/util/messageContent"; import { IIdeMessenger } from "../../context/IdeMessenger"; import { Dispatch } from "@reduxjs/toolkit"; import { setIsGatheringContext } from "../../redux/slices/sessionSlice"; +import { ctxItemToRifWithContents } from "core/commands/util"; +import { getUriFileExtension } from "core/util/uri"; interface MentionAttrs { label: string; @@ -68,46 +70,41 @@ async function resolveEditorContent({ parts.push({ type: "text", text }); } } else if (p.type === "codeBlock") { - if (!p.attrs.item.editing) { - let meta = p.attrs.item.description.split(" "); - let relativePath = meta[0] || ""; - let extName = relativePath.split(".").slice(-1)[0]; - const text = - "\n\n" + - "```" + - extName + - " " + - p.attrs.item.description + - "\n" + - p.attrs.item.content + - "\n```"; - if (parts[parts.length - 1]?.type === "text") { - parts[parts.length - 1].text += "\n" + text; - } else { - parts.push({ - type: "text", - text, - }); + if (p.attrs?.item) { + const contextItem = p.attrs.item as ContextItemWithId; + const rif = ctxItemToRifWithContents(contextItem, true); + // If not editing, include codeblocks in the prompt + // If editing is handled by selectedCode below + if (!contextItem.editing) { + const fileExtension = getUriFileExtension(rif.filepath); + // let extName = relativeFilepath.split(".").slice(-1)[0]; + const text = + "\n\n" + + "```" + + fileExtension + + " " + + contextItem.description + + "\n" + + contextItem.content + + "\n```"; + if (parts[parts.length - 1]?.type === "text") { + parts[parts.length - 1].text += "\n" + text; + } else { + parts.push({ + type: "text", + text, + }); + } } + selectedCode.push(rif); + } else { + console.warn("codeBlock has no item attribute"); } - - const name: string = p.attrs.item.name; - let lines = name.substring(name.lastIndexOf("(") + 1); - lines = lines.substring(0, lines.lastIndexOf(")")); - const [start, end] = lines.split("-"); - - selectedCode.push({ - filepath: p.attrs.item.description, - range: { - start: { line: parseInt(start) - 1, character: 0 }, - end: { line: parseInt(end) - 1, character: 0 }, - }, - }); } else if (p.type === "image") { parts.push({ type: "imageUrl", imageUrl: { - url: p.attrs.src, + url: p.attrs?.src, }, }); } else { @@ -194,7 +191,7 @@ async function resolveEditorContent({ } if (shouldGatherContext) { - dispatch(setIsGatheringContext(true)); + dispatch(setIsGatheringContext(false)); } return [contextItems, selectedCode, parts]; diff --git a/gui/src/components/markdown/CodeSnippetPreview.tsx b/gui/src/components/markdown/CodeSnippetPreview.tsx index 6bdfdf3536..62565e5ea6 100644 --- a/gui/src/components/markdown/CodeSnippetPreview.tsx +++ b/gui/src/components/markdown/CodeSnippetPreview.tsx @@ -5,7 +5,7 @@ import { } from "@heroicons/react/24/outline"; import { ContextItemWithId } from "core"; import { dedent, getMarkdownLanguageTagForFile } from "core/util"; -import React, { useContext } from "react"; +import React, { useContext, useMemo } from "react"; import styled from "styled-components"; import { defaultBorderRadius, lightGray, vscEditorBackground } from ".."; import { IdeMessengerContext } from "../../context/IdeMessenger"; @@ -13,6 +13,7 @@ import { getFontSize } from "../../util"; import FileIcon from "../FileIcon"; import HeaderButtonWithToolTip from "../gui/HeaderButtonWithToolTip"; import StyledMarkdownPreview from "./StyledMarkdownPreview"; +import { ctxItemToRifWithContents } from "core/commands/util"; const PreviewMarkdownDiv = styled.div<{ borderColor?: string; @@ -49,7 +50,6 @@ interface CodeSnippetPreviewProps { const MAX_PREVIEW_HEIGHT = 300; -// Pre-compile the regular expression outside of the function const backticksRegex = /`{3,}/gm; function CodeSnippetPreview(props: CodeSnippetPreviewProps) { @@ -58,12 +58,14 @@ function CodeSnippetPreview(props: CodeSnippetPreviewProps) { const [collapsed, setCollapsed] = React.useState(true); const [hovered, setHovered] = React.useState(false); - const content = dedent`${props.item.content}`; + const content = useMemo(() => { + return dedent`${props.item.content}`; + }, [props.item.content]); - const fence = React.useMemo(() => { + const fence = useMemo(() => { const backticks = content.match(backticksRegex); return backticks ? backticks.sort().at(-1) + "`" : "```"; - }, [props.item.content]); + }, [content]); const codeBlockRef = React.useRef(null); @@ -79,19 +81,19 @@ function CodeSnippetPreview(props: CodeSnippetPreviewProps) { { - if (props.item.id.providerTitle === "file") { + if ( + props.item.id.providerTitle === "file" && + props.item.uri?.value + ) { ideMessenger.post("showFile", { - filepath: props.item.description, + filepath: props.item.uri.value, }); } else if (props.item.id.providerTitle === "code") { - const lines = props.item.name - .split("(")[1] - .split(")")[0] - .split("-"); + const rif = ctxItemToRifWithContents(props.item, true); ideMessenger.ide.showLines( - props.item.description.split(" ")[0], - parseInt(lines[0]) - 1, - parseInt(lines[1]) - 1, + rif.filepath, + rif.range.start.line, + rif.range.end.line, ); } else { ideMessenger.post("showVirtualFile", { @@ -120,13 +122,14 @@ function CodeSnippetPreview(props: CodeSnippetPreviewProps) { )}
diff --git a/gui/src/components/markdown/FilenameLink.tsx b/gui/src/components/markdown/FilenameLink.tsx index 977e2c99ec..4e4d934e96 100644 --- a/gui/src/components/markdown/FilenameLink.tsx +++ b/gui/src/components/markdown/FilenameLink.tsx @@ -1,8 +1,8 @@ import { RangeInFile } from "core"; -import { useContext, useMemo } from "react"; -import { getBasename } from "core/util"; +import { useContext } from "react"; import { IdeMessengerContext } from "../../context/IdeMessenger"; import FileIcon from "../FileIcon"; +import { findUriInDirs, getUriPathBasename } from "core/util/uri"; import { ToolTip } from "../gui/Tooltip"; import { v4 as uuidv4 } from "uuid"; @@ -23,6 +23,11 @@ function FilenameLink({ rif }: FilenameLinkProps) { const id = uuidv4(); + const { relativePathOrBasename } = findUriInDirs( + rif.filepath, + window.workspacePaths ?? [], + ); + return ( <> - {getBasename(rif.filepath)} + {getUriPathBasename(rif.filepath)} - {rif.filepath} + {"/" + relativePathOrBasename} ); diff --git a/gui/src/components/markdown/StepContainerPreActionButtons.tsx b/gui/src/components/markdown/StepContainerPreActionButtons.tsx index 3e42c5872c..1afbe2a277 100644 --- a/gui/src/components/markdown/StepContainerPreActionButtons.tsx +++ b/gui/src/components/markdown/StepContainerPreActionButtons.tsx @@ -83,7 +83,7 @@ export default function StepContainerPreActionButtons({ const defaultModel = useAppSelector(selectDefaultModel); function onClickApply() { - if (!defaultModel) { + if (!defaultModel || !streamIdRef.current) { return; } ideMessenger.post("applyToFile", { diff --git a/gui/src/components/markdown/StepContainerPreToolbar/FileInfo.tsx b/gui/src/components/markdown/StepContainerPreToolbar/FileInfo.tsx index 6d8fc8d74b..c2f09a24a3 100644 --- a/gui/src/components/markdown/StepContainerPreToolbar/FileInfo.tsx +++ b/gui/src/components/markdown/StepContainerPreToolbar/FileInfo.tsx @@ -1,20 +1,24 @@ -import { ChevronDownIcon } from "@heroicons/react/24/outline"; -import { getBasename } from "core/util"; import FileIcon from "../../FileIcon"; import { useContext } from "react"; import { IdeMessengerContext } from "../../../context/IdeMessenger"; +import { getLastNPathParts, getUriPathBasename } from "core/util/uri"; +import { inferResolvedUriFromRelativePath } from "core/util/ideUtils"; export interface FileInfoProps { - filepath: string; + relativeFilepath: string; range?: string; } -const FileInfo = ({ filepath, range }: FileInfoProps) => { +const FileInfo = ({ relativeFilepath, range }: FileInfoProps) => { const ideMessenger = useContext(IdeMessengerContext); - function onClickFileName() { + async function onClickFileName() { + const fileUri = await inferResolvedUriFromRelativePath( + relativeFilepath, + ideMessenger.ide, + ); ideMessenger.post("showFile", { - filepath, + filepath: fileUri, }); } @@ -24,9 +28,9 @@ const FileInfo = ({ filepath, range }: FileInfoProps) => { className="mr-0.5 flex w-full min-w-0 cursor-pointer items-center gap-0.5" onClick={onClickFileName} > - + - {getBasename(filepath)} + {getLastNPathParts(relativeFilepath, 1)} {range && ` ${range}`} diff --git a/gui/src/components/markdown/StepContainerPreToolbar/StepContainerPreToolbar.tsx b/gui/src/components/markdown/StepContainerPreToolbar/StepContainerPreToolbar.tsx index 987d9c63ad..d0bd7caa66 100644 --- a/gui/src/components/markdown/StepContainerPreToolbar/StepContainerPreToolbar.tsx +++ b/gui/src/components/markdown/StepContainerPreToolbar/StepContainerPreToolbar.tsx @@ -19,6 +19,7 @@ import { selectApplyStateByStreamId, selectIsInEditMode, } from "../../../redux/slices/sessionSlice"; +import { inferResolvedUriFromRelativePath } from "core/util/ideUtils"; const TopDiv = styled.div` outline: 1px solid rgba(153, 153, 152); @@ -44,7 +45,7 @@ const ToolbarDiv = styled.div<{ isExpanded: boolean }>` export interface StepContainerPreToolbarProps { codeBlockContent: string; language: string; - filepath: string; + relativeFilepath: string; isGeneratingCodeBlock: boolean; codeBlockIndex: number; // To track which codeblock we are applying range?: string; @@ -84,17 +85,23 @@ export default function StepContainerPreToolbar( : props.isGeneratingCodeBlock; const isNextCodeBlock = nextCodeBlockIndex === props.codeBlockIndex; - const hasFileExtension = /\.[0-9a-z]+$/i.test(props.filepath); + const hasFileExtension = /\.[0-9a-z]+$/i.test(props.relativeFilepath); const defaultModel = useAppSelector(selectDefaultModel); - function onClickApply() { + async function onClickApply() { if (!defaultModel) { return; } + console.log(props.relativeFilepath); + const fileUri = await inferResolvedUriFromRelativePath( + props.relativeFilepath, + ideMessenger.ide, + ); + console.log(fileUri); ideMessenger.post("applyToFile", { streamId: streamIdRef.current, - filepath: props.filepath, + filepath: fileUri, text: codeBlockContent, curSelectedModelTitle: defaultModel.title, }); @@ -137,16 +144,24 @@ export default function StepContainerPreToolbar( wasGeneratingRef.current = isGeneratingCodeBlock; }, [isGeneratingCodeBlock]); - function onClickAcceptApply() { + async function onClickAcceptApply() { + const fileUri = await inferResolvedUriFromRelativePath( + props.relativeFilepath, + ideMessenger.ide, + ); ideMessenger.post("acceptDiff", { - filepath: props.filepath, + filepath: fileUri, streamId: streamIdRef.current, }); } - function onClickRejectApply() { + async function onClickRejectApply() { + const fileUri = await inferResolvedUriFromRelativePath( + props.relativeFilepath, + ideMessenger.ide, + ); ideMessenger.post("rejectDiff", { - filepath: props.filepath, + filepath: fileUri, streamId: streamIdRef.current, }); } @@ -172,7 +187,10 @@ export default function StepContainerPreToolbar( }`} />
- +
diff --git a/gui/src/components/markdown/StyledMarkdownPreview.tsx b/gui/src/components/markdown/StyledMarkdownPreview.tsx index 7627547346..49eec1d818 100644 --- a/gui/src/components/markdown/StyledMarkdownPreview.tsx +++ b/gui/src/components/markdown/StyledMarkdownPreview.tsx @@ -1,5 +1,5 @@ import { ctxItemToRifWithContents } from "core/commands/util"; -import { memo, useEffect, useMemo, useRef } from "react"; +import { memo, useContext, useEffect, useMemo, useRef } from "react"; import { useRemark } from "react-remark"; import rehypeHighlight, { Options } from "rehype-highlight"; import rehypeKatex from "rehype-katex"; @@ -26,6 +26,7 @@ import { patchNestedMarkdown } from "./utils/patchNestedMarkdown"; import { useAppSelector } from "../../redux/hooks"; import { fixDoubleDollarNewLineLatex } from "./utils/fixDoubleDollarLatex"; import { selectUIConfig } from "../../redux/slices/configSlice"; +import { IdeMessengerContext } from "../../context/IdeMessenger"; import { ToolTip } from "../gui/Tooltip"; import { v4 as uuidv4 } from "uuid"; @@ -133,31 +134,6 @@ function getCodeChildrenContent(children: any) { return undefined; } -function processCodeBlocks(tree: any) { - const lastNode = tree.children[tree.children.length - 1]; - const lastCodeNode = lastNode.type === "code" ? lastNode : null; - - visit(tree, "code", (node: any) => { - if (!node.lang) { - node.lang = "javascript"; - } else if (node.lang.includes(".")) { - node.lang = node.lang.split(".").slice(-1)[0]; - } - - node.data = node.data || {}; - node.data.hProperties = node.data.hProperties || {}; - - node.data.hProperties["data-isgeneratingcodeblock"] = lastCodeNode === node; - node.data.hProperties["data-codeblockcontent"] = node.value; - - if (node.meta) { - let meta = node.meta.split(" "); - node.data.hProperties.filepath = meta[0]; - node.data.hProperties.range = meta[1]; - } - }); -} - const StyledMarkdownPreview = memo(function StyledMarkdownPreview( props: StyledMarkdownPreviewProps, ) { @@ -203,7 +179,31 @@ const StyledMarkdownPreview = memo(function StyledMarkdownPreview( singleDollarTextMath: false, }, ], - () => processCodeBlocks, + () => (tree: any) => { + const lastNode = tree.children[tree.children.length - 1]; + const lastCodeNode = lastNode.type === "code" ? lastNode : null; + + visit(tree, "code", (node: any) => { + if (!node.lang) { + node.lang = "javascript"; + } else if (node.lang.includes(".")) { + node.lang = node.lang.split(".").slice(-1)[0]; + } + + node.data = node.data || {}; + node.data.hProperties = node.data.hProperties || {}; + + node.data.hProperties["data-isgeneratingcodeblock"] = + lastCodeNode === node; + node.data.hProperties["data-codeblockcontent"] = node.value; + + if (node.meta) { + let meta = node.meta.split(" "); + node.data.hProperties["data-relativefilepath"] = meta[0]; + node.data.hProperties.range = meta[1]; + } + }); + }, ], rehypePlugins: [ rehypeKatex as any, @@ -221,7 +221,7 @@ const StyledMarkdownPreview = memo(function StyledMarkdownPreview( return (tree) => { visit(tree, { tagName: "pre" }, (node: any) => { // Add an index (0, 1, 2, etc...) to each code block. - node.properties = { codeBlockIndex }; + node.properties = { "data-codeblockindex": codeBlockIndex }; codeBlockIndex++; }); }; @@ -250,9 +250,11 @@ const StyledMarkdownPreview = memo(function StyledMarkdownPreview( ); }, pre: ({ node, ...preProps }) => { - const preChildProps = preProps?.children?.[0]?.props; - const { className, filepath, range } = preProps?.children?.[0]?.props; + const codeBlockIndex = preProps["data-codeblockindex"]; + const preChildProps = preProps?.children?.[0]?.props ?? {}; + const { className, range } = preChildProps; + const relativeFilePath = preChildProps["data-relativefilepath"]; const codeBlockContent = preChildProps["data-codeblockcontent"]; const isGeneratingCodeBlock = preChildProps["data-isgeneratingcodeblock"]; @@ -266,13 +268,13 @@ const StyledMarkdownPreview = memo(function StyledMarkdownPreview( // If we don't have a filepath show the more basic toolbar // that is just action buttons on hover. // We also use this in JB since we haven't yet implemented - // the logic for lazy apply. - if (!filepath || isJetBrains()) { + // the logic forfileUri lazy apply. + if (!relativeFilePath || isJetBrains()) { return ( @@ -283,9 +285,9 @@ const StyledMarkdownPreview = memo(function StyledMarkdownPreview( return ( diff --git a/gui/src/context/SubmenuContextProviders.tsx b/gui/src/context/SubmenuContextProviders.tsx index b9cbe3bc79..c8ba17c774 100644 --- a/gui/src/context/SubmenuContextProviders.tsx +++ b/gui/src/context/SubmenuContextProviders.tsx @@ -1,11 +1,6 @@ import { ContextSubmenuItem } from "core"; import { createContext } from "react"; -import { - deduplicateArray, - getBasename, - getUniqueFilePath, - groupByLastNPathParts, -} from "core/util"; +import { deduplicateArray } from "core/util"; import MiniSearch, { SearchResult } from "minisearch"; import { useCallback, @@ -19,6 +14,10 @@ import { IdeMessengerContext } from "./IdeMessenger"; import { selectContextProviderDescriptions } from "../redux/selectors"; import { useWebviewListener } from "../hooks/useWebviewListener"; import { useAppSelector } from "../redux/hooks"; +import { + getShortestUniqueRelativeUriPaths, + getUriPathBasename, +} from "core/util/uri"; const MINISEARCH_OPTIONS = { prefix: true, @@ -89,12 +88,16 @@ export const SubmenuContextProvidersProvider = ({ const getOpenFilesItems = useCallback(async () => { const openFiles = await ideMessenger.ide.getOpenFiles(); - const openFileGroups = groupByLastNPathParts(openFiles, 2); - - return openFiles.map((file) => ({ - id: file, - title: getBasename(file), - description: getUniqueFilePath(file, openFileGroups), + const workspaceDirs = await ideMessenger.ide.getWorkspaceDirs(); + const withUniquePaths = getShortestUniqueRelativeUriPaths( + openFiles, + workspaceDirs, + ); + + return withUniquePaths.map((file) => ({ + id: file.uri, + title: getUriPathBasename(file.uri), + description: file.uniquePath, providerTitle: "file", })); }, [ideMessenger]); diff --git a/gui/src/pages/AddNewModel/configs/models.ts b/gui/src/pages/AddNewModel/configs/models.ts index f3e3203051..a9802ef12f 100644 --- a/gui/src/pages/AddNewModel/configs/models.ts +++ b/gui/src/pages/AddNewModel/configs/models.ts @@ -1053,10 +1053,10 @@ export const models: { [key: string]: ModelPackage } = { providerOptions: ["vertexai"], isOpenSource: false, }, - gpt4gov: { + asksagegpt4gov: { title: "GPT-4 gov", description: - "U.S. Government. Most capable model today - which is similar to GPT-4o but approved for use by the U.S. Government.", + "U.S. Government. Most capable model today - which is similar to GPT-4 but approved for use by the U.S. Government.", params: { model: "gpt4-gov", contextLength: 128_000, @@ -1068,14 +1068,14 @@ export const models: { [key: string]: ModelPackage } = { icon: "openai.png", isOpenSource: false, }, - gpt4ogov: { + asksagegpt4ogov: { title: "GPT-4o gov", description: "U.S. Government. Most capable model today - which is similar to GPT-4o but approved for use by the U.S. Government.", params: { model: "gpt-4o-gov", contextLength: 128_000, - title: "GPT-4o", + title: "GPT-4o-gov", systemMessage: "You are an expert software developer. You give helpful and concise responses.", // Need to set this on the Ask Sage side or just configure it in here to be discussed }, @@ -1083,6 +1083,131 @@ export const models: { [key: string]: ModelPackage } = { icon: "openai.png", isOpenSource: false, }, + asksagegpt35gov: { + title: "GPT-3.5-Turbo gov", + description: + "U.S. Government. Inexpensive and good ROI.", + params: { + model: "gpt-gov", + contextLength: 8096, + title: "GPT-3.5-Turbo gov", + systemMessage: + "You are an expert software developer. You give helpful and concise responses.", // Need to set this on the Ask Sage side or just configure it in here to be discussed + }, + providerOptions: ["askSage"], + icon: "openai.png", + isOpenSource: false, + }, + asksagegpt4ominigov: { + title: "GPT-4o-mini gov", + description: + "U.S. Government. Latest OpenAI GPT 4o-mini model. More inexpensive than GPT4. Capable of ingesting and analyzing images (JPG, PNG, GIF (20MB files max)). 16,384 token response max.", + params: { + model: "gpt-4o-mini-gov", + contextLength: 128_000, + title: "GPT-4o-mini gov", + systemMessage: + "You are an expert software developer. You give helpful and concise responses.", // Need to set this on the Ask Sage side or just configure it in here to be discussed + }, + providerOptions: ["askSage"], + icon: "openai.png", + isOpenSource: false, + }, + asksagegpt4: { + title: "GPT-4", + description: + "GPT4 is about 5X more expensive than Ask Sage tokens and 50X more expensive than GPT3.5", + params: { + model: "gpt4", + contextLength: 8_192, + title: "GPT-4", + }, + providerOptions: ["openai",], + icon: "openai.png", + isOpenSource: false, + }, + asksagegpt432: { + title: "GPT-4-32k", + description: + "The GPT-4-32k model is a variant of the GPT-4 model developed by OpenAI. It is designed to handle a larger context window, capable of processing up to 32,768 tokens, which makes it suitable for scenarios that require extensive information integration and data analysis", + params: { + model: "gpt4-32k", + contextLength: 32_768, + title: "GPT-4-32k", + }, + providerOptions: ["openai",], + icon: "openai.png", + isOpenSource: false, + }, + asksagegpto1: { + title: "GPT-o1", + description: + "Latest OpenAI GPT-o1 model. More inexpensive than GPT4. Capable of ingesting and analyzing images (JPG, PNG, GIF (20MB files max)).", + params: { + model: "gpt-o1", + contextLength: 128_000, + title: "GPT-o1", + systemMessage: + "You are an expert software developer. You give helpful and concise responses.", + }, + providerOptions: ["askSage"], + icon: "openai.png", + isOpenSource: false, + }, + asksagegpto1mini: { + title: "GPT-o1-mini", + description: + "Latest OpenAI GPT-o1-mini model. More inexpensive than GPT-o1. Capable of ingesting and analyzing images (JPG, PNG, GIF (20MB files max)). 16,384 token response max.", + params: { + model: "gpt-o1-mini", + contextLength: 128_000, + title: "GPT-o1-mini", + systemMessage: + "You are an expert software developer. You give helpful and concise responses.", + }, + providerOptions: ["askSage"], + icon: "openai.png", + isOpenSource: false, + }, + asksageclaude35gov: { + title: "Claude 3.5 Sonnet gov", + description: + "Anthropic's most intelligent model, but much less expensive than Claude 3 Opus", + params: { + model: "aws-bedrock-claude-35-sonnet-gov", + contextLength: 200_000, + title: "Claude 3.5 Sonnet gov", + systemMessage: + "You are an expert software developer. You give helpful and concise responses.", + }, + providerOptions: ["askSage"], + icon: "anthropic.png", + isOpenSource: false, + }, + asksagegroqllama33: { + title: "Llama 3.3", + description: + "Llama-3.3 is a large language model customized by Groq.", + params: { + title: "Llama 3.3", + model: "groq-llama33", + contextLength: 128_000, + }, + icon: "groq.png", + isOpenSource: true, + }, + asksagegroq70b: { + title: "Groq-70B", + description: + "A large language model customized by Groq.", + params: { + title: "Groq-70B", + model: "groq-70b", + contextLength: 8_192, + }, + icon: "groq.png", + isOpenSource: true, + }, Qwen2Coder: { title: "Qwen 2.5 Coder 7b", description: @@ -1131,7 +1256,7 @@ export const models: { [key: string]: ModelPackage } = { contextLength: 128_000, }, icon: "xAI.png", - providerOptions: ["xAI"], + providerOptions: ["xAI", "askSage"], isOpenSource: false, }, gemma2_2b: { diff --git a/gui/src/pages/AddNewModel/configs/providers.ts b/gui/src/pages/AddNewModel/configs/providers.ts index b3b80e6e13..f3d3a9ccb0 100644 --- a/gui/src/pages/AddNewModel/configs/providers.ts +++ b/gui/src/pages/AddNewModel/configs/providers.ts @@ -782,14 +782,24 @@ To get started, [register](https://dataplatform.cloud.ibm.com/registration/stepo ...completionParamsInputsConfigs, ], packages: [ - models.gpt4gov, - models.gpt4ogov, + models.asksagegpt4ogov, + models.asksagegpt4ominigov, + models.asksagegpt4gov, + models.asksagegpt35gov, models.gpt4o, models.gpt4omini, + models.asksagegpt4, + models.asksagegpt432, + models.asksagegpto1, + models.asksagegpto1mini, models.gpt35turbo, + models.asksageclaude35gov, models.claude35Sonnet, models.claude3Opus, models.claude3Sonnet, + models.grokBeta, + models.asksagegroqllama33, + models.asksagegroq70b, models.mistralLarge, models.llama370bChat, models.gemini15Pro, diff --git a/gui/src/pages/gui/StreamError.tsx b/gui/src/pages/gui/StreamError.tsx index 47fc790dea..1694fe4109 100644 --- a/gui/src/pages/gui/StreamError.tsx +++ b/gui/src/pages/gui/StreamError.tsx @@ -56,27 +56,121 @@ const StreamErrorDialog = ({ error }: StreamErrorProps) => { } } } - let errorContent: React.ReactNode = ( - Error while streaming chat response. + + const checkKeysButton = apiKeyUrl ? ( + + ) : null; + + const configButton = ( + { + ideMessenger.post("config/openProfile", { + profileId: undefined, + }); + }} + > +
+ +
+ Open config + ); + let errorContent: React.ReactNode = <>; + + // Display components for specific errors if (statusCode === 429) { errorContent = (
- {`This likely means your ${modelTitle} usage has been rate limited + {`This might mean your ${modelTitle} usage has been rate limited by ${providerName}.`} - {apiKeyUrl ? ( - +
+ {checkKeysButton} + {configButton} +
+
+ ); + } + + if (statusCode === 404) { + errorContent = ( +
+ Likely causes: +
    +
  • + Invalid + apiBase + {selectedModel && ( + <> + {`: `} + {selectedModel.apiBase} + + )} +
    • +
  • + Model/deployment not found + {selectedModel && ( + <> + {` for: `} + {selectedModel.model} + + )} +
    • +
+
{configButton}
+
+ ); + } + + if (statusCode === 401) { + errorContent = ( +
+ {`Likely cause: your API key is invalid.`} +
+ {checkKeysButton} + {configButton} +
+
+ ); + } + + if (statusCode === 403) { + errorContent = ( +
+ {`Likely cause: not authorized to access the model deployment.`} +
+ {checkKeysButton} + {configButton} +
+
+ ); + } + + if ( + message && + (message.toLowerCase().includes("overloaded") || + message.toLowerCase().includes("malformed")) + ) { + errorContent = ( +
+ {`Most likely, the provider's server(s) are overloaded and streaming was interrupted. Try again later`} + {selectedModel ? ( + + {`Provider: `} + {selectedModel.provider} + ) : null} + {/* TODO: status page links for providers? */}
); } @@ -84,13 +178,10 @@ const StreamErrorDialog = ({ error }: StreamErrorProps) => { return (

{`${statusCode ? statusCode + " " : ""}Error`}

-
{errorContent}
{message ? (
- - {message} - + {message}
{ @@ -101,22 +192,9 @@ const StreamErrorDialog = ({ error }: StreamErrorProps) => {
) : null} +
{errorContent}
+
-
- { - ideMessenger.post("config/openProfile", { - profileId: undefined, - }); - }} - > -
- -
- Open config - -
Report this error:
) : null; } diff --git a/gui/src/pages/gui/ToolCallDiv/FunctionSpecificToolCallDiv.tsx b/gui/src/pages/gui/ToolCallDiv/FunctionSpecificToolCallDiv.tsx index d55055b031..71082844d3 100644 --- a/gui/src/pages/gui/ToolCallDiv/FunctionSpecificToolCallDiv.tsx +++ b/gui/src/pages/gui/ToolCallDiv/FunctionSpecificToolCallDiv.tsx @@ -14,7 +14,10 @@ function FunctionSpecificToolCallDiv({ switch (toolCall.function.name) { case "create_new_file": return ( - + ); case "run_terminal_command": return ( diff --git a/gui/src/redux/slices/sessionSlice.ts b/gui/src/redux/slices/sessionSlice.ts index 65599a9cca..6f91202ba4 100644 --- a/gui/src/redux/slices/sessionSlice.ts +++ b/gui/src/redux/slices/sessionSlice.ts @@ -27,6 +27,7 @@ import { v4 as uuidv4 } from "uuid"; import { RootState } from "../store"; import { streamResponseThunk } from "../thunks/streamResponse"; import { findCurrentToolCall } from "../util"; +import { findUriInDirs, getUriPathBasename } from "core/util/uri"; // We need this to handle reorderings (e.g. a mid-array deletion) of the messages array. // The proper fix is adding a UUID to all chat messages, but this is the temp workaround. @@ -428,17 +429,21 @@ export const sessionSlice = createSlice({ return { ...item, editing: false }; }); - const base = payload.rangeInFileWithContents.filepath - .split(/[\\/]/) - .pop(); + const { relativePathOrBasename } = findUriInDirs( + payload.rangeInFileWithContents.filepath, + window.workspacePaths ?? [], + ); + const fileName = getUriPathBasename( + payload.rangeInFileWithContents.filepath, + ); const lineNums = `(${ payload.rangeInFileWithContents.range.start.line + 1 }-${payload.rangeInFileWithContents.range.end.line + 1})`; contextItems.push({ - name: `${base} ${lineNums}`, - description: payload.rangeInFileWithContents.filepath, + name: `${fileName} ${lineNums}`, + description: relativePathOrBasename, id: { providerTitle: "code", itemId: uuidv4(), @@ -446,6 +451,10 @@ export const sessionSlice = createSlice({ content: payload.rangeInFileWithContents.contents, editing: true, editable: true, + uri: { + type: "file", + value: payload.rangeInFileWithContents.filepath, + }, }); state.history[state.history.length - 1].contextItems = contextItems; diff --git a/gui/src/redux/thunks/gatherContext.ts b/gui/src/redux/thunks/gatherContext.ts index 3e92025516..7f59a0547f 100644 --- a/gui/src/redux/thunks/gatherContext.ts +++ b/gui/src/redux/thunks/gatherContext.ts @@ -6,13 +6,13 @@ import { MessageContent, RangeInFile, } from "core"; -import { getBasename, getRelativePath } from "core/util"; import resolveEditorContent, { hasSlashCommandOrContextProvider, } from "../../components/mainInput/resolveInput"; import { ThunkApiType } from "../store"; import { selectDefaultModel } from "../slices/configSlice"; import { setIsGatheringContext } from "../slices/sessionSlice"; +import { findUriInDirs, getUriPathBasename } from "core/util/uri"; import { updateFileSymbolsFromNewContextItems } from "./updateFileSymbols"; export const gatherContext = createAsyncThunk< @@ -44,13 +44,6 @@ export const gatherContext = createAsyncThunk< } // Resolve context providers and construct new history - const shouldGatherContext = - modifiers.useCodebase || hasSlashCommandOrContextProvider(editorState); - - if (shouldGatherContext) { - dispatch(setIsGatheringContext(true)); - } - let [selectedContextItems, selectedCode, content] = await resolveEditorContent({ editorState, @@ -81,11 +74,13 @@ export const gatherContext = createAsyncThunk< ) { // don't add the file if it's already in the context items selectedContextItems.unshift({ - content: `The following file is currently open. Don't reference it if it's not relevant to the user's message.\n\n\`\`\`${getRelativePath( - currentFile.path, - await extra.ideMessenger.ide.getWorkspaceDirs(), - )}\n${currentFileContents}\n\`\`\``, - name: `Active file: ${getBasename(currentFile.path)}`, + content: `The following file is currently open. Don't reference it if it's not relevant to the user's message.\n\n\`\`\`${ + findUriInDirs( + currentFile.path, + await extra.ideMessenger.ide.getWorkspaceDirs(), + ).relativePathOrBasename + }\n${currentFileContents}\n\`\`\``, + name: `Active file: ${getUriPathBasename(currentFile.path)}`, description: currentFile.path, id: { itemId: currentFile.path, diff --git a/manual-testing-sandbox/AdvancedPage.tsx b/manual-testing-sandbox/AdvancedPage.tsx index 6d17ef4634..5b5ccbde6f 100644 --- a/manual-testing-sandbox/AdvancedPage.tsx +++ b/manual-testing-sandbox/AdvancedPage.tsx @@ -13,7 +13,10 @@ const AdvancedPage = () => { className={`${isDarkMode ? "bg-gray-800 text-white" : "bg-white text-black"} min-h-screen p-4`} >
-

Welcome to the Advanced Page

+ I apologize, but there seems to be a misunderstanding. The user's + request doesn't appear to be related to rewriting the code section you + provided. The request mentions editing a readme.md file, which is not + part of the React component code you've shown.