mirror of
https://github.com/farcasclaudiu/Flowise.git
synced 2026-06-28 19:00:59 +03:00
Added meilisearch retriever component (#2824)
* added meilisearch retriever and credentials.ts * added semantic ratio * removed a TODO implementatio * meilisearch component implemented with searching and upsert functionality (#3) meilisearch retriever component created , searching for an existing index and upserting a new or existing index has been implemented , component utilizes langchain and meilisearch vector search Reviewed-on: https://git.beyond.cc/ntg/flowise/pulls/3 Reviewed-by: mohamed1999akram <mohamed1999akram@gmail.com> * added CI/CD for ntg branch, added proper dockerfile for flowise-ntg (#4) Reviewed-on: https://git.beyond.cc/ntg/flowise/pulls/4 Reviewed-by: mohammad <mohammad@noreply.git.beyond.cc> * modified os version , removed linting errors , removed cypress github actions (#5) added --no-lock-file flag to pass CICD , made the runner run on debian and not ubuntu , removed code that caused warnings to pass linting Reviewed-on: https://git.beyond.cc/ntg/flowise/pulls/5 Reviewed-by: omaryassery <omarryassser@gmail.com> * removed unnecessary QEMU install action (#6) Reviewed-on: https://git.beyond.cc/ntg/flowise/pulls/6 Reviewed-by: omaryassery <omarryassser@gmail.com> * removed cypress installation and linting from dockerfile (#7) Reviewed-on: https://git.beyond.cc/ntg/flowise/pulls/7 Reviewed-by: isameh <isameh@ntgclarity.com> * dockerfile-ntg-modification (#9) dockerfile-ntg modified to copy all working directory before calling pnpm install Reviewed-on: https://git.beyond.cc/ntg/flowise/pulls/9 Reviewed-by: isameh <isameh@ntgclarity.com> * resolved comments, reverted CI/CD * add test docker build yml back * moved meilisearch to vector store folder * Update Meilisearch.ts --------- Co-authored-by: Henry <hzj94@hotmail.com> Co-authored-by: Henry Heng <henryheng@flowiseai.com>
This commit is contained in:
Mohamed Yasser Oaf
committed by
GitHub
GitHub
parent
0a36aa7ef4
commit
d5153c3840
Binary file not shown.
|
After Width: | Height: | Size: 8.6 KiB |
@@ -0,0 +1,174 @@
|
||||
import { getCredentialData, getCredentialParam } from '../../../src'
|
||||
import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { Meilisearch } from 'meilisearch'
|
||||
import { MeilisearchRetriever } from './core'
|
||||
import { flatten } from 'lodash'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
|
||||
class MeilisearchRetriever_node implements INode {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
badge: string
|
||||
outputs: INodeOutputsValue[]
|
||||
author?: string
|
||||
|
||||
constructor() {
|
||||
this.label = 'Meilisearch'
|
||||
this.name = 'meilisearch'
|
||||
this.version = 1.0
|
||||
this.type = 'Meilisearch'
|
||||
this.icon = 'Meilisearch.png'
|
||||
this.category = 'Vector Stores'
|
||||
this.badge = 'NEW'
|
||||
this.description = `Upsert embedded data and perform similarity search upon query using Meilisearch hybrid search functionality`
|
||||
this.baseClasses = ['BaseRetriever']
|
||||
this.credential = {
|
||||
label: 'Connect Credential',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
credentialNames: ['meilisearchApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
type: 'Document',
|
||||
list: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Host',
|
||||
name: 'host',
|
||||
type: 'string',
|
||||
description: 'This is the URL for the desired Meilisearch instance'
|
||||
},
|
||||
{
|
||||
label: 'Index Uid',
|
||||
name: 'indexUid',
|
||||
type: 'string',
|
||||
description: 'UID for the index to answer from'
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'K',
|
||||
type: 'number',
|
||||
description: 'number of top searches to return as context',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Semantic Ratio',
|
||||
name: 'semanticRatio',
|
||||
type: 'number',
|
||||
description: 'percentage of sematic reasoning in meilisearch hybrid search',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Meilisearch Retriever',
|
||||
name: 'MeilisearchRetriever',
|
||||
description: 'retrieve answers',
|
||||
baseClasses: this.baseClasses
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Meilisearch Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
}
|
||||
]
|
||||
}
|
||||
//@ts-ignore
|
||||
vectorStoreMethods = {
|
||||
async upsert(nodeData: INodeData, options: ICommonObject): Promise<any> {
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const meilisearchAdminApiKey = getCredentialParam('meilisearchAdminApiKey', credentialData, nodeData)
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
const host = nodeData.inputs?.host as string
|
||||
const indexUid = nodeData.inputs?.indexUid as string
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
let embeddingDimension: number = 384
|
||||
const client = new Meilisearch({
|
||||
host: host,
|
||||
apiKey: meilisearchAdminApiKey
|
||||
})
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
if (flattenDocs[i] && flattenDocs[i].pageContent) {
|
||||
const uniqueId = uuidv4()
|
||||
const { pageContent, metadata } = flattenDocs[i]
|
||||
const docEmbedding = await embeddings.embedQuery(pageContent)
|
||||
embeddingDimension = docEmbedding.length
|
||||
const documentForIndexing = {
|
||||
pageContent,
|
||||
metadata,
|
||||
objectID: uniqueId,
|
||||
_vectors: {
|
||||
ollama: {
|
||||
embeddings: docEmbedding,
|
||||
regenerate: false
|
||||
}
|
||||
}
|
||||
}
|
||||
finalDocs.push(documentForIndexing)
|
||||
}
|
||||
}
|
||||
let index: any
|
||||
try {
|
||||
index = await client.getIndex(indexUid)
|
||||
} catch (error) {
|
||||
console.error('Error fetching index:', error)
|
||||
await client.createIndex(indexUid, { primaryKey: 'objectID' })
|
||||
} finally {
|
||||
index = await client.getIndex(indexUid)
|
||||
}
|
||||
|
||||
try {
|
||||
await index.updateSettings({
|
||||
embedders: {
|
||||
ollama: {
|
||||
source: 'userProvided',
|
||||
dimensions: embeddingDimension
|
||||
}
|
||||
}
|
||||
})
|
||||
await index.addDocuments(finalDocs)
|
||||
} catch (error) {
|
||||
console.error('Error occurred while adding documents:', error)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const meilisearchSearchApiKey = getCredentialParam('meilisearchSearchApiKey', credentialData, nodeData)
|
||||
const host = nodeData.inputs?.host as string
|
||||
const indexUid = nodeData.inputs?.indexUid as string
|
||||
const K = nodeData.inputs?.K as string
|
||||
const semanticRatio = nodeData.inputs?.semanticRatio as string
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
|
||||
const hybridsearchretriever = new MeilisearchRetriever(host, meilisearchSearchApiKey, indexUid, K, semanticRatio, embeddings)
|
||||
return hybridsearchretriever
|
||||
}
|
||||
}
|
||||
module.exports = { nodeClass: MeilisearchRetriever_node }
|
||||
@@ -0,0 +1,92 @@
|
||||
import { BaseRetriever, type BaseRetrieverInput } from '@langchain/core/retrievers'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { Meilisearch } from 'meilisearch'
|
||||
import { Embeddings } from '@langchain/core/embeddings'
|
||||
|
||||
export interface CustomRetrieverInput extends BaseRetrieverInput {}
|
||||
|
||||
export class MeilisearchRetriever extends BaseRetriever {
|
||||
lc_namespace = ['langchain', 'retrievers']
|
||||
private readonly meilisearchSearchApiKey: any
|
||||
private readonly host: any
|
||||
private indexUid: string
|
||||
private K: string
|
||||
private semanticRatio: string
|
||||
private embeddings: Embeddings
|
||||
constructor(
|
||||
host: string,
|
||||
meilisearchSearchApiKey: any,
|
||||
indexUid: string,
|
||||
K: string,
|
||||
semanticRatio: string,
|
||||
embeddings: Embeddings,
|
||||
fields?: CustomRetrieverInput
|
||||
) {
|
||||
super(fields)
|
||||
this.meilisearchSearchApiKey = meilisearchSearchApiKey
|
||||
this.host = host
|
||||
this.indexUid = indexUid
|
||||
this.embeddings = embeddings
|
||||
|
||||
if (semanticRatio == '') {
|
||||
this.semanticRatio = '0.5'
|
||||
} else {
|
||||
let semanticRatio_Float = parseFloat(semanticRatio)
|
||||
if (semanticRatio_Float > 1.0) {
|
||||
this.semanticRatio = '1.0'
|
||||
} else if (semanticRatio_Float < 0.0) {
|
||||
this.semanticRatio = '0.0'
|
||||
} else {
|
||||
this.semanticRatio = semanticRatio
|
||||
}
|
||||
}
|
||||
|
||||
if (K == '') {
|
||||
K = '4'
|
||||
}
|
||||
this.K = K
|
||||
}
|
||||
|
||||
async _getRelevantDocuments(query: string): Promise<Document[]> {
|
||||
// Pass `runManager?.getChild()` when invoking internal runnables to enable tracing
|
||||
// const additionalDocs = await someOtherRunnable.invoke(params, runManager?.getChild())
|
||||
const client = new Meilisearch({
|
||||
host: this.host,
|
||||
apiKey: this.meilisearchSearchApiKey
|
||||
})
|
||||
|
||||
const index = await client.index(this.indexUid)
|
||||
const questionEmbedding = await this.embeddings.embedQuery(query)
|
||||
// Perform the search
|
||||
const searchResults = await index.search(query, {
|
||||
vector: questionEmbedding,
|
||||
limit: parseInt(this.K), // Optional: Limit the number of results
|
||||
attributesToRetrieve: ['*'], // Optional: Specify which fields to retrieve
|
||||
hybrid: {
|
||||
semanticRatio: parseFloat(this.semanticRatio),
|
||||
embedder: 'ollama'
|
||||
}
|
||||
})
|
||||
const hits = searchResults.hits
|
||||
let documents: Document[] = [
|
||||
new Document({
|
||||
pageContent: 'mock page',
|
||||
metadata: {}
|
||||
})
|
||||
]
|
||||
try {
|
||||
documents = hits.map(
|
||||
(hit: any) =>
|
||||
new Document({
|
||||
pageContent: hit.pageContent,
|
||||
metadata: {
|
||||
objectID: hit.objectID
|
||||
}
|
||||
})
|
||||
)
|
||||
} catch (e) {
|
||||
console.error('Error occurred while adding documents:', e)
|
||||
}
|
||||
return documents
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user