farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 17:01:00 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add batchSize parameter to Qdrant_VectorStores and QdrantUpsert_Vecto… (#2215)

Browse Source 
* Add batchSize parameter to Qdrant_VectorStores and QdrantUpsert_VectorStores

* Add optional parameter 'optional' to batchSize in Qdrant_VectorStores and QdrantUpsert_VectorStores also try calling normal mode if it fail failback to batch mode

* Update default values for batchSize in Qdrant_VectorStores and QdrantUpsert_VectorStores

* Update PNPM version to v9.0.2 in Dockerfile

* Update Qdrant.ts

* Update Dockerfile

* Update Qdrant_Upsert.ts

* Update Qdrant.ts

* lint fix Qdrant_Upsert.ts

* increment qdrant node version

* increment qdrant upsert node version

---------

Co-authored-by: Henry Heng <henryheng@flowiseai.com>
This commit is contained in:
RABBAH Mahmoud
2024-04-29 22:20:32 +01:00
committed by GitHub
parent 5775947586
commit c4eb75ddde
2 changed files with 66 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/vectorstores/Qdrant/Qdrant.ts
+35 -6
View File
@@ -32,7 +32,7 @@ class Qdrant_VectorStores implements INode {
constructor() {
this.label = 'Qdrant'
this.name = 'qdrant'
this.version = 2.0
this.version = 3.0
this.type = 'Qdrant'
this.icon = 'qdrant.png'
this.category = 'Vector Stores'
@@ -86,6 +86,15 @@ class Qdrant_VectorStores implements INode {
default: 1536,
additionalParams: true
},
{
label: 'Upsert Batch Size',
name: 'batchSize',
type: 'number',
step: 1,
description: 'Upsert in batches of size N',
additionalParams: true,
optional: true
},
{
label: 'Similarity',
name: 'qdrantSimilarity',
@@ -159,6 +168,7 @@ class Qdrant_VectorStores implements INode {
const qdrantSimilarity = nodeData.inputs?.qdrantSimilarity
const qdrantVectorDimension = nodeData.inputs?.qdrantVectorDimension
const recordManager = nodeData.inputs?.recordManager
const _batchSize = nodeData.inputs?.batchSize
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const qdrantApiKey = getCredentialParam('qdrantApiKey', credentialData, nodeData)
@@ -218,10 +228,21 @@ class Qdrant_VectorStores implements INode {
}))
try {
await client.upsert(collectionName, {
wait: true,
points
})
if (_batchSize) {
const batchSize = parseInt(_batchSize, 10)
for (let i = 0; i < points.length; i += batchSize) {
const batchPoints = points.slice(i, i + batchSize)
await client.upsert(collectionName, {
wait: true,
points: batchPoints
})
}
} else {
await client.upsert(collectionName, {
wait: true,
points
})
}
} catch (e: any) {
const error = new Error(`${e?.status ?? 'Undefined error code'} ${e?.message}: ${e?.data?.status?.error}`)
throw error
@@ -257,7 +278,15 @@ class Qdrant_VectorStores implements INode {
return res
} else {
await QdrantVectorStore.fromDocuments(finalDocs, embeddings, dbConfig)
if (_batchSize) {
const batchSize = parseInt(_batchSize, 10)
for (let i = 0; i < finalDocs.length; i += batchSize) {
const batch = finalDocs.slice(i, i + batchSize)
await QdrantVectorStore.fromDocuments(batch, embeddings, dbConfig)
}
} else {
await QdrantVectorStore.fromDocuments(finalDocs, embeddings, dbConfig)
}
return { numAdded: finalDocs.length, addedDocs: finalDocs }
}
} catch (e) {