diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cf01633b..6b6b6f48 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v3 with: - version: 9.0.2 + version: 9.0.3 - name: Use Node.js ${{ matrix.node-version }} uses: actions/setup-node@v4 with: diff --git a/package.json b/package.json index 35b4c806..238eb3b3 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,7 @@ "turbo": "1.10.16", "typescript": "^4.8.4" }, - "packageManager": "pnpm@9.0.2", + "packageManager": "pnpm@9.0.3", "pnpm": { "onlyBuiltDependencies": [ "faiss-node", diff --git a/packages/components/nodes/vectorstores/Supabase/Supabase.ts b/packages/components/nodes/vectorstores/Supabase/Supabase.ts index fd24f373..b5472ef4 100644 --- a/packages/components/nodes/vectorstores/Supabase/Supabase.ts +++ b/packages/components/nodes/vectorstores/Supabase/Supabase.ts @@ -130,7 +130,7 @@ class Supabase_VectorStores implements INode { try { if (recordManager) { - const vectorStore = await SupabaseVectorStore.fromExistingIndex(embeddings, { + const vectorStore = await SupabaseUpsertVectorStore.fromExistingIndex(embeddings, { client, tableName: tableName, queryName: queryName @@ -148,7 +148,7 @@ class Supabase_VectorStores implements INode { }) return res } else { - await SupabaseVectorStore.fromDocuments(finalDocs, embeddings, { + await SupabaseUpsertVectorStore.fromDocuments(finalDocs, embeddings, { client, tableName: tableName, queryName: queryName @@ -190,4 +190,33 @@ class Supabase_VectorStores implements INode { } } +class SupabaseUpsertVectorStore extends SupabaseVectorStore { + async addVectors(vectors: number[][], documents: Document[]): Promise { + if (vectors.length === 0) { + return [] + } + const rows = vectors.map((embedding, idx) => ({ + content: documents[idx].pageContent, + embedding, + metadata: documents[idx].metadata + })) + + let returnedIds: string[] = [] + for (let i = 0; i < rows.length; i += this.upsertBatchSize) { + const chunk = rows.slice(i, i + this.upsertBatchSize).map((row, index) => { + return { id: index, ...row } + }) + + const res = await this.client.from(this.tableName).upsert(chunk).select() + if (res.error) { + throw new Error(`Error inserting: ${res.error.message} ${res.status} ${res.statusText}`) + } + if (res.data) { + returnedIds = returnedIds.concat(res.data.map((row) => row.id)) + } + } + return returnedIds + } +} + module.exports = { nodeClass: Supabase_VectorStores } diff --git a/packages/components/nodes/vectorstores/Supabase/Supabase_Upsert.ts b/packages/components/nodes/vectorstores/Supabase/Supabase_Upsert.ts index 002c9d79..219019ea 100644 --- a/packages/components/nodes/vectorstores/Supabase/Supabase_Upsert.ts +++ b/packages/components/nodes/vectorstores/Supabase/Supabase_Upsert.ts @@ -108,7 +108,7 @@ class SupabaseUpsert_VectorStores implements INode { finalDocs.push(new Document(flattenDocs[i])) } - const vectorStore = await SupabaseVectorStore.fromDocuments(finalDocs, embeddings, { + const vectorStore = await SupabaseUpsertVectorStore.fromDocuments(finalDocs, embeddings, { client, tableName: tableName, queryName: queryName @@ -125,4 +125,33 @@ class SupabaseUpsert_VectorStores implements INode { } } +class SupabaseUpsertVectorStore extends SupabaseVectorStore { + async addVectors(vectors: number[][], documents: Document[]): Promise { + if (vectors.length === 0) { + return [] + } + const rows = vectors.map((embedding, idx) => ({ + content: documents[idx].pageContent, + embedding, + metadata: documents[idx].metadata + })) + + let returnedIds: string[] = [] + for (let i = 0; i < rows.length; i += this.upsertBatchSize) { + const chunk = rows.slice(i, i + this.upsertBatchSize).map((row, index) => { + return { id: index, ...row } + }) + + const res = await this.client.from(this.tableName).upsert(chunk).select() + if (res.error) { + throw new Error(`Error inserting: ${res.error.message} ${res.status} ${res.statusText}`) + } + if (res.data) { + returnedIds = returnedIds.concat(res.data.map((row) => row.id)) + } + } + return returnedIds + } +} + module.exports = { nodeClass: SupabaseUpsert_VectorStores }