-
Notifications
You must be signed in to change notification settings - Fork 84
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Update npm package
llamaindex to v0.8.29 (#5154)
Co-authored-by: hash-worker[bot] <180894564+hash-worker[bot]@users.noreply.github.com> Co-authored-by: Bilal Mahmoud <bmahmoud@mpi-cbg.de> Co-authored-by: Ciaran Morinan <37743469+CiaranMn@users.noreply.github.com>
- Loading branch information
3 people
authored
Dec 18, 2024
1 parent
40138b2
commit 8de9549
Showing
4 changed files
with
869 additions
and
429 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
116 changes: 43 additions & 73 deletions
116
...vities/research-entities-action/link-follower-agent/llama-index/simple-storage-context.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,98 +1,68 @@ | ||
| import { access, mkdir } from "node:fs/promises"; | ||
| import fs from "node:fs/promises"; | ||
| import path from "node:path"; | ||
| import { fileURLToPath } from "node:url"; | ||
|
|
||
| import type { Subtype } from "@local/advanced-types/subtype"; | ||
| import type { StorageContext } from "llamaindex"; | ||
| import { | ||
| SimpleDocumentStore, | ||
| SimpleIndexStore, | ||
| SimpleVectorStore, | ||
| } from "llamaindex"; | ||
| import { type StorageContext, storageContextFromDefaults } from "llamaindex"; | ||
|
|
||
| export type SimpleStorageContext = Subtype< | ||
| StorageContext, | ||
| { | ||
| docStore: SimpleDocumentStore; | ||
| indexStore: SimpleIndexStore; | ||
| vectorStore: SimpleVectorStore; | ||
| } | ||
| >; | ||
| import { logger } from "../../../../shared/activity-logger.js"; | ||
|
|
||
| const __filename = fileURLToPath(import.meta.url); | ||
| const __dirname = path.dirname(__filename); | ||
|
|
||
| const baseFilePath = path.join(__dirname, "/var/tmp_files"); | ||
|
|
||
| export const generateSimpleStorageContextFilePaths = (params: { | ||
| hash: string; | ||
| }) => { | ||
| const { hash } = params; | ||
|
|
||
| const directoryPath = `${baseFilePath}/storage/${hash}`; | ||
| export interface Storage extends StorageContext { | ||
| directory: string; | ||
| } | ||
|
|
||
| return { | ||
| directoryPath, | ||
| vectorStorePath: `${directoryPath}/vector-store.json`, | ||
| docStorePath: `${directoryPath}/doc-store.json`, | ||
| indexStorePath: `${directoryPath}/index-store.json`, | ||
| }; | ||
| const directory = ({ hash }: { hash: string }) => { | ||
| return `${baseFilePath}/storage/${hash}`; | ||
| }; | ||
|
|
||
| export const retrieveSimpleStorageContext = async (params: { | ||
| hash: string; | ||
| }): Promise<{ simpleStorageContext: SimpleStorageContext | undefined }> => { | ||
| const { hash } = params; | ||
|
|
||
| const { directoryPath, vectorStorePath, docStorePath, indexStorePath } = | ||
| generateSimpleStorageContextFilePaths({ hash }); | ||
| export const createStorageContext = async ({ hash }: { hash: string }) => { | ||
| const directoryPath = directory({ hash }); | ||
|
|
||
| try { | ||
| // Check directory exists | ||
| await access(directoryPath); | ||
|
|
||
| await Promise.all([ | ||
| access(vectorStorePath), | ||
| // access(docStorePath), | ||
| access(indexStorePath), | ||
| ]); | ||
| await fs.mkdir(directoryPath, { recursive: true }); | ||
| } catch (error: unknown) { | ||
| if ((error as NodeJS.ErrnoException).code !== "EEXIST") { | ||
| logger.info( | ||
| `Unable to create directory ${directoryPath}: ${(error as Error).message}`, | ||
| ); | ||
| } | ||
| } | ||
|
|
||
| const simpleStorageContext: SimpleStorageContext = { | ||
| vectorStore: await SimpleVectorStore.fromPersistPath(vectorStorePath), | ||
| docStore: await SimpleDocumentStore.fromPersistPath(docStorePath), | ||
| indexStore: await SimpleIndexStore.fromPersistPath(indexStorePath), | ||
| }; | ||
| const context = await storageContextFromDefaults({ | ||
| persistDir: directoryPath, | ||
| storeImages: false, | ||
| }); | ||
|
|
||
| return { simpleStorageContext }; | ||
| } catch (error) { | ||
| // eslint-disable-next-line no-console | ||
| console.error( | ||
| `Failed to retrieve storage context: ${(error as Error).message}`, | ||
| ); | ||
| return { simpleStorageContext: undefined }; | ||
| } | ||
| return { ...context, directory: directoryPath }; | ||
| }; | ||
|
|
||
| export const persistSimpleStorageContext = async (params: { | ||
| hash: string; | ||
| simpleStorageContext: SimpleStorageContext; | ||
| }) => { | ||
| const { hash, simpleStorageContext } = params; | ||
| const ensurePromise = <T>(value: T | Promise<T>): Promise<T> => | ||
| value instanceof Promise ? value : Promise.resolve(value); | ||
|
|
||
| const { vectorStore, docStore, indexStore } = simpleStorageContext; | ||
| export const persistStorageContext = ({ | ||
| storageContext: { vectorStores, docStore, indexStore }, | ||
| }: { | ||
| storageContext: StorageContext; | ||
| }) => { | ||
| const promises: Promise<void>[] = []; | ||
|
|
||
| const { directoryPath, vectorStorePath, docStorePath, indexStorePath } = | ||
| generateSimpleStorageContextFilePaths({ hash }); | ||
| const pushPersist = (store: object | undefined) => { | ||
| if (store && "persist" in store) { | ||
| const persist = store.persist as () => Promise<void> | void; | ||
| promises.push(ensurePromise(persist())); | ||
| } | ||
| }; | ||
|
|
||
| try { | ||
| await access(directoryPath); | ||
| } catch { | ||
| // If the directory does not exist, create it recursively | ||
| await mkdir(directoryPath, { recursive: true }); | ||
| for (const store of Object.values(vectorStores)) { | ||
| pushPersist(store); | ||
| } | ||
|
|
||
| await vectorStore.persist(vectorStorePath); | ||
| /** @todo: figure out why this doesn't get created */ | ||
| await docStore.persist(docStorePath); | ||
| await indexStore.persist(indexStorePath); | ||
| pushPersist(docStore); | ||
| pushPersist(indexStore); | ||
|
|
||
| return Promise.all(promises); | ||
| }; |
Oops, something went wrong.