farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-22 11:01:22 +03:00
Code Issues Packages Projects Releases Wiki Activity

Meilisearch - Added metadata as filterable attributes and polling on enqueued tasks (#3042)

Browse Source 
* added polling for enqueued tasks,metadata filterable

* delete index feature with polling on task

* added search filter interface

* Update pnpm-lock.yaml

---------

Co-authored-by: Henry Heng <henryheng@flowiseai.com>
This commit is contained in:
Mohamed Yasser Oaf
2024-08-23 18:27:07 +03:00
committed by GitHub
parent 72a03dc6e8
commit 0f58d31493
2 changed files with 116 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/vectorstores/Meilisearch/Meilisearch.ts
+109 -10
View File
@@ -55,7 +55,7 @@ class MeilisearchRetriever_node implements INode {
label: 'Host',
name: 'host',
type: 'string',
description: 'This is the URL for the desired Meilisearch instance'
description: "This is the URL for the desired Meilisearch instance, the URL must not end with a '/'"
},
{
label: 'Index Uid',
@@ -63,11 +63,17 @@ class MeilisearchRetriever_node implements INode {
type: 'string',
description: 'UID for the index to answer from'
},
{
label: 'Delete Index if exists',
name: 'deleteIndex',
type: 'boolean',
optional: true
},
{
label: 'Top K',
name: 'K',
type: 'number',
description: 'number of top searches to return as context',
description: 'number of top searches to return as context, default is 4',
additionalParams: true,
optional: true
},
@@ -75,7 +81,15 @@ class MeilisearchRetriever_node implements INode {
label: 'Semantic Ratio',
name: 'semanticRatio',
type: 'number',
description: 'percentage of sematic reasoning in meilisearch hybrid search',
description: 'percentage of sematic reasoning in meilisearch hybrid search, default is 0.75',
additionalParams: true,
optional: true
},
{
label: 'Search Filter',
name: 'searchFilter',
type: 'string',
description: 'search filter to apply on searchable attributes',
additionalParams: true,
optional: true
}
@@ -104,6 +118,7 @@ class MeilisearchRetriever_node implements INode {
const docs = nodeData.inputs?.document as Document[]
const host = nodeData.inputs?.host as string
const indexUid = nodeData.inputs?.indexUid as string
const deleteIndex = nodeData.inputs?.deleteIndex as boolean
const embeddings = nodeData.inputs?.embeddings as Embeddings
let embeddingDimension: number = 384
const client = new Meilisearch({
@@ -132,17 +147,52 @@ class MeilisearchRetriever_node implements INode {
finalDocs.push(documentForIndexing)
}
}
let taskUid_created: number = 0
if (deleteIndex) {
try {
const deleteResponse = await client.deleteIndex(indexUid)
taskUid_created = deleteResponse.taskUid
let deleteTaskStatus = await client.getTask(taskUid_created)
while (deleteTaskStatus.status !== 'succeeded') {
deleteTaskStatus = await client.getTask(taskUid_created)
if (deleteTaskStatus.error !== null || deleteTaskStatus.status === 'failed') {
throw new Error('Error during index deletion task: ' + deleteTaskStatus.error)
}
}
} catch (error) {
console.error(error)
console.warn('Error occured when deleting your index, if it did not exist, we will create one for you... ')
}
}
let index: any
try {
index = await client.getIndex(indexUid)
} catch (error) {
console.error('Error fetching index:', error)
await client.createIndex(indexUid, { primaryKey: 'objectID' })
} finally {
index = await client.getIndex(indexUid)
console.warn('Index not found, creating a new index...')
try {
const createResponse = await client.createIndex(indexUid, { primaryKey: 'objectID' })
taskUid_created = createResponse.taskUid
let createTaskStatus = await client.getTask(taskUid_created)
while (createTaskStatus.status !== 'succeeded') {
createTaskStatus = await client.getTask(taskUid_created)
if (createTaskStatus.error !== null || createTaskStatus.status === 'failed') {
throw new Error('Error during index creation task: ' + createTaskStatus.error)
}
}
index = await client.getIndex(indexUid)
} catch (taskError) {
console.error('Error during index creation process:', taskError)
}
}
try {
await index.updateFilterableAttributes(['metadata'])
await index.updateSettings({
embedders: {
ollama: {
@@ -151,23 +201,72 @@ class MeilisearchRetriever_node implements INode {
}
}
})
await index.addDocuments(finalDocs)
const addResponse = await index.addDocuments(finalDocs)
taskUid_created = addResponse.taskUid
let AddTaskStatus = await client.getTask(taskUid_created)
while (AddTaskStatus.status !== 'succeeded') {
AddTaskStatus = await client.getTask(taskUid_created)
if (AddTaskStatus.error !== null || AddTaskStatus.status === 'failed') {
throw new Error('Error during documents adding task: ' + AddTaskStatus.error)
}
}
index = await client.getIndex(indexUid)
} catch (error) {
console.error('Error occurred while adding documents:', error)
}
return
return { numAdded: finalDocs.length, addedDocs: finalDocs }
}
}
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const meilisearchSearchApiKey = getCredentialParam('meilisearchSearchApiKey', credentialData, nodeData)
const meilisearchAdminApiKey = getCredentialParam('meilisearchAdminApiKey', credentialData, nodeData)
const host = nodeData.inputs?.host as string
const indexUid = nodeData.inputs?.indexUid as string
const K = nodeData.inputs?.K as string
const semanticRatio = nodeData.inputs?.semanticRatio as string
const embeddings = nodeData.inputs?.embeddings as Embeddings
const searchFilter = nodeData.inputs?.searchFilter as string
const hybridsearchretriever = new MeilisearchRetriever(host, meilisearchSearchApiKey, indexUid, K, semanticRatio, embeddings)
const experimentalEndpoint = host + '/experimental-features/'
const token = meilisearchAdminApiKey
const experimentalOptions = {
method: 'PATCH',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${token}`
},
body: JSON.stringify({
vectorStore: true
})
}
try {
const response = await fetch(experimentalEndpoint, experimentalOptions)
if (!response.ok) {
throw new Error(`Failed to enable vectorStore: ${response.statusText}`)
}
const data = await response.json()
const vectorStoreEnabled = data.vectorStore
if (vectorStoreEnabled !== true) {
throw new Error('Failed to enable vectorStore, vectorStrore property returned is not true')
}
} catch (error) {
console.error('Error enabling vectorStore feature:', error)
}
const hybridsearchretriever = new MeilisearchRetriever(
host,
meilisearchSearchApiKey,
indexUid,
K,
semanticRatio,
embeddings,
searchFilter
)
return hybridsearchretriever
}
}
packages/components/nodes/vectorstores/Meilisearch/core.ts
+7 -2
View File
@@ -13,6 +13,7 @@ export class MeilisearchRetriever extends BaseRetriever {
private K: string
private semanticRatio: string
private embeddings: Embeddings
private searchFilter: string
constructor(
host: string,
meilisearchSearchApiKey: any,
@@ -20,6 +21,7 @@ export class MeilisearchRetriever extends BaseRetriever {
K: string,
semanticRatio: string,
embeddings: Embeddings,
searchFilter: string,
fields?: CustomRetrieverInput
) {
super(fields)
@@ -27,9 +29,10 @@ export class MeilisearchRetriever extends BaseRetriever {
this.host = host
this.indexUid = indexUid
this.embeddings = embeddings
this.searchFilter = searchFilter
if (semanticRatio == '') {
this.semanticRatio = '0.5'
this.semanticRatio = '0.75'
} else {
let semanticRatio_Float = parseFloat(semanticRatio)
if (semanticRatio_Float > 1.0) {
@@ -59,6 +62,7 @@ export class MeilisearchRetriever extends BaseRetriever {
const questionEmbedding = await this.embeddings.embedQuery(query)
// Perform the search
const searchResults = await index.search(query, {
filter: this.searchFilter,
vector: questionEmbedding,
limit: parseInt(this.K), // Optional: Limit the number of results
attributesToRetrieve: ['*'], // Optional: Specify which fields to retrieve
@@ -80,7 +84,8 @@ export class MeilisearchRetriever extends BaseRetriever {
new Document({
pageContent: hit.pageContent,
metadata: {
objectID: hit.objectID
objectID: hit.objectID,
...hit.metadata
}
})
)