farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 15:00:57 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into feature/output-parsers

Browse Source 
# Conflicts:
#	packages/ui/src/views/chatmessage/ChatMessage.js
This commit is contained in:
Henry
2023-11-03 13:10:06 +00:00
parent 74b7033c41 130f24c330
commit b44c203807
82 changed files with 3193 additions and 444 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/credentials/RedisCacheUrlApi.credential.ts
+25
View File
@@ -0,0 +1,25 @@
import { INodeParams, INodeCredential } from '../src/Interface'
class RedisCacheUrlApi implements INodeCredential {
label: string
name: string
version: number
description: string
inputs: INodeParams[]
constructor() {
this.label = 'Redis Cache URL'
this.name = 'redisCacheUrlApi'
this.version = 1.0
this.inputs = [
{
label: 'Redis URL',
name: 'redisUrl',
type: 'string',
default: '127.0.0.1'
}
]
}
}
module.exports = { credClass: RedisCacheUrlApi }
packages/components/credentials/SearchApi.credential.ts
+26
View File
@@ -0,0 +1,26 @@
import { INodeParams, INodeCredential } from '../src/Interface'
class SearchApi implements INodeCredential {
label: string
name: string
version: number
description: string
inputs: INodeParams[]
constructor() {
this.label = 'Search API'
this.name = 'searchApi'
this.version = 1.0
this.description =
'Sign in to <a target="_blank" href="https://www.searchapi.io/">SearchApi</a> to obtain a free API key from the dashboard.'
this.inputs = [
{
label: 'SearchApi API Key',
name: 'searchApiKey',
type: 'password'
}
]
}
}
module.exports = { credClass: SearchApi }
packages/components/nodes/agents/CSVAgent/CSVAgent.ts
+1 -1
View File
@@ -131,7 +131,7 @@ json.dumps(my_dict)`
const code = `import pandas as pd\n${pythonCode}`
finalResult = await pyodide.runPythonAsync(code)
} catch (error) {
throw new Error(`Sorry, I'm unable to find answer for question: "${input}" using follwoing code: "${pythonCode}"`)
throw new Error(`Sorry, I'm unable to find answer for question: "${input}" using following code: "${pythonCode}"`)
}
}
packages/components/nodes/cache/RedisCache/RedisCache.ts
Vendored
+18 -11
View File
@@ -30,7 +30,7 @@ class RedisCache implements INode {
name: 'credential',
type: 'credential',
optional: true,
credentialNames: ['redisCacheApi']
credentialNames: ['redisCacheApi', 'redisCacheUrlApi']
}
this.inputs = [
{
@@ -48,17 +48,24 @@ class RedisCache implements INode {
const ttl = nodeData.inputs?.ttl as string
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const username = getCredentialParam('redisCacheUser', credentialData, nodeData)
const password = getCredentialParam('redisCachePwd', credentialData, nodeData)
const portStr = getCredentialParam('redisCachePort', credentialData, nodeData)
const host = getCredentialParam('redisCacheHost', credentialData, nodeData)
const redisUrl = getCredentialParam('redisUrl', credentialData, nodeData)
const client = new Redis({
port: portStr ? parseInt(portStr) : 6379,
host,
username,
password
})
let client: Redis
if (!redisUrl || redisUrl === '') {
const username = getCredentialParam('redisCacheUser', credentialData, nodeData)
const password = getCredentialParam('redisCachePwd', credentialData, nodeData)
const portStr = getCredentialParam('redisCachePort', credentialData, nodeData)
const host = getCredentialParam('redisCacheHost', credentialData, nodeData)
client = new Redis({
port: portStr ? parseInt(portStr) : 6379,
host,
username,
password
})
} else {
client = new Redis(redisUrl)
}
const redisClient = new LangchainRedisCache(client)
packages/components/nodes/cache/RedisCache/RedisEmbeddingsCache.ts
Vendored
+19 -11
View File
@@ -30,7 +30,7 @@ class RedisEmbeddingsCache implements INode {
name: 'credential',
type: 'credential',
optional: true,
credentialNames: ['redisCacheApi']
credentialNames: ['redisCacheApi', 'redisCacheUrlApi']
}
this.inputs = [
{
@@ -63,17 +63,25 @@ class RedisEmbeddingsCache implements INode {
const underlyingEmbeddings = nodeData.inputs?.embeddings as Embeddings
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const username = getCredentialParam('redisCacheUser', credentialData, nodeData)
const password = getCredentialParam('redisCachePwd', credentialData, nodeData)
const portStr = getCredentialParam('redisCachePort', credentialData, nodeData)
const host = getCredentialParam('redisCacheHost', credentialData, nodeData)
const redisUrl = getCredentialParam('redisUrl', credentialData, nodeData)
let client: Redis
if (!redisUrl || redisUrl === '') {
const username = getCredentialParam('redisCacheUser', credentialData, nodeData)
const password = getCredentialParam('redisCachePwd', credentialData, nodeData)
const portStr = getCredentialParam('redisCachePort', credentialData, nodeData)
const host = getCredentialParam('redisCacheHost', credentialData, nodeData)
client = new Redis({
port: portStr ? parseInt(portStr) : 6379,
host,
username,
password
})
} else {
client = new Redis(redisUrl)
}
const client = new Redis({
port: portStr ? parseInt(portStr) : 6379,
host,
username,
password
})
ttl ??= '3600'
let ttlNumber = parseInt(ttl, 10)
const redisStore = new RedisByteStore({
packages/components/nodes/chains/ConversationChain/ConversationChain.ts
+3 -1
View File
@@ -71,7 +71,9 @@ class ConversationChain_Chains implements INode {
const flattenDocs = docs && docs.length ? flatten(docs) : []
const finalDocs = []
for (let i = 0; i < flattenDocs.length; i += 1) {
finalDocs.push(new Document(flattenDocs[i]))
if (flattenDocs[i] && flattenDocs[i].pageContent) {
finalDocs.push(new Document(flattenDocs[i]))
}
}
let finalText = ''
packages/components/nodes/documentloaders/SearchApi/SearchAPI.ts
+109
View File
@@ -0,0 +1,109 @@
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { TextSplitter } from 'langchain/text_splitter'
import { SearchApiLoader } from 'langchain/document_loaders/web/searchapi'
import { getCredentialData, getCredentialParam } from '../../../src'
// Provides access to multiple search engines using the SearchApi.
// For available parameters & engines, refer to: https://www.searchapi.io/docs/google
class SearchAPI_DocumentLoaders implements INode {
label: string
name: string
version: number
description: string
type: string
icon: string
category: string
baseClasses: string[]
credential: INodeParams
inputs: INodeParams[]
constructor() {
this.label = 'SearchApi For Web Search'
this.name = 'searchApi'
this.version = 1.0
this.type = 'Document'
this.icon = 'searchapi.svg'
this.category = 'Document Loaders'
this.description = 'Load data from real-time search results'
this.baseClasses = [this.type]
this.credential = {
label: 'Connect Credential',
name: 'credential',
type: 'credential',
optional: false,
credentialNames: ['searchApi']
}
this.inputs = [
{
label: 'Query',
name: 'query',
type: 'string',
optional: true
},
{
label: 'Custom Parameters',
name: 'customParameters',
type: 'json',
optional: true,
additionalParams: true
},
{
label: 'Text Splitter',
name: 'textSplitter',
type: 'TextSplitter',
optional: true
},
{
label: 'Metadata',
name: 'metadata',
type: 'json',
optional: true,
additionalParams: true
}
]
}
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
const query = nodeData.inputs?.query as string
const customParameters = nodeData.inputs?.customParameters
const metadata = nodeData.inputs?.metadata
// Fetch the API credentials for this node
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const searchApiKey = getCredentialParam('searchApiKey', credentialData, nodeData)
// Check and parse custom parameters (should be JSON or object)
const parsedParameters = typeof customParameters === 'object' ? customParameters : JSON.parse(customParameters || '{}')
// Prepare the configuration for the SearchApiLoader
const loaderConfig = {
q: query,
apiKey: searchApiKey,
...parsedParameters
}
// Initialize the loader with the given configuration
const loader = new SearchApiLoader(loaderConfig)
// Fetch documents, split if a text splitter is provided
const docs = textSplitter ? await loader.loadAndSplit() : await loader.load()
if (metadata) {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
return docs.map((doc) => {
return {
...doc,
metadata: {
...doc.metadata,
...parsedMetadata
}
}
})
}
return docs
}
}
module.exports = { nodeClass: SearchAPI_DocumentLoaders }
packages/components/nodes/documentloaders/SearchApi/searchapi.svg
+1
View File
@@ -0,0 +1 @@
<svg id="SvgjsSvg1001" width="75.27131652832031" height="63.92396926879883" xmlns="http://www.w3.org/2000/svg" version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:svgjs="http://svgjs.com/svgjs" viewBox="0 0 75.27131652832031 63.92396926879883"><defs id="SvgjsDefs1002"></defs><rect id="SvgjsRect1008" width="75.27131652832031" height="63.92396926879883" fill="transparent"></rect><g id="SvgjsG1009" transform="matrix(1,0,0,1,-39.50003433227539,-50.53549575805664)"><title>0479_octopus_verti</title><path id="color_1" d="M97.24234,109.8245a2.57759,2.57759,0,0,1-2.57778,2.57778,9.80672,9.80672,0,0,1-9.79558-9.79557V91.58366a2.57779,2.57779,0,0,1,5.15557,0v11.02305a4.64509,4.64509,0,0,0,4.64,4.64A2.57759,2.57759,0,0,1,97.24234,109.8245ZM112.19348,93.223a2.57759,2.57759,0,0,0-2.57778,2.57779,4.64,4.64,0,1,1-9.28,0V73.73554a23.2,23.2,0,1,0-46.40009,0V95.80076a4.64,4.64,0,1,1-9.28,0,2.57779,2.57779,0,1,0-5.15557,0,9.79558,9.79558,0,0,0,19.59115,0V73.73554a18.04449,18.04449,0,0,1,36.089,0V95.80076a9.79558,9.79558,0,0,0,19.59115,0A2.57759,2.57759,0,0,0,112.19348,93.223ZM77.13563,91.78a2.57759,2.57759,0,0,0-2.57778,2.57778v17.52893a2.57779,2.57779,0,0,0,5.15557,0V94.3578A2.57759,2.57759,0,0,0,77.13563,91.78ZM66.8245,89.00588a2.57759,2.57759,0,0,0-2.57778,2.57778v11.02305a4.64509,4.64509,0,0,1-4.64,4.64,2.57778,2.57778,0,1,0,0,5.15556,9.80671,9.80671,0,0,0,9.79557-9.79557V91.58366A2.57759,2.57759,0,0,0,66.8245,89.00588ZM69.918,70.12639a3.6089,3.6089,0,1,0,3.6089,3.6089A3.60891,3.60891,0,0,0,69.918,70.12639Zm18.04448,3.6089a3.6089,3.6089,0,1,0-3.60889,3.60889A3.60891,3.60891,0,0,0,87.96251,73.73529Z" fill="#3730a3"></path></g></svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.6 KiB

packages/components/nodes/documentloaders/Text/Text.ts
+3 -2
View File
@@ -19,7 +19,7 @@ class Text_DocumentLoaders implements INode {
constructor() {
this.label = 'Text File'
this.name = 'textFile'
this.version = 2.0
this.version = 3.0
this.type = 'Document'
this.icon = 'textFile.svg'
this.category = 'Document Loaders'
@@ -30,7 +30,8 @@ class Text_DocumentLoaders implements INode {
label: 'Txt File',
name: 'txtFile',
type: 'file',
fileType: '.txt'
fileType:
'.txt, .html, .aspx, .asp, .cpp, .c, .cs, .css, .go, .h, .java, .js, .less, .ts, .php, .proto, .python, .py, .rst, .ruby, .rb, .rs, .scala, .sc, .scss, .sol, .sql, .swift, .markdown, .md, .tex, .ltx, .vb, .xml'
},
{
label: 'Text Splitter',
packages/components/nodes/documentloaders/VectorStoreToDocument/VectorStoreToDocument.ts
+11 -2
View File
@@ -17,7 +17,7 @@ class VectorStoreToDocument_DocumentLoaders implements INode {
constructor() {
this.label = 'VectorStore To Document'
this.name = 'vectorStoreToDocument'
this.version = 1.0
this.version = 2.0
this.type = 'Document'
this.icon = 'vectorretriever.svg'
this.category = 'Document Loaders'
@@ -29,6 +29,14 @@ class VectorStoreToDocument_DocumentLoaders implements INode {
name: 'vectorStore',
type: 'VectorStore'
},
{
label: 'Query',
name: 'query',
type: 'string',
description: 'Query to retrieve documents from vector database. If not specified, user question will be used',
optional: true,
acceptVariable: true
},
{
label: 'Minimum Score (%)',
name: 'minScore',
@@ -56,11 +64,12 @@ class VectorStoreToDocument_DocumentLoaders implements INode {
async init(nodeData: INodeData, input: string): Promise<any> {
const vectorStore = nodeData.inputs?.vectorStore as VectorStore
const minScore = nodeData.inputs?.minScore as number
const query = nodeData.inputs?.query as string
const output = nodeData.outputs?.output as string
const topK = (vectorStore as any)?.k ?? 4
const docs = await vectorStore.similaritySearchWithScore(input, topK)
const docs = await vectorStore.similaritySearchWithScore(query ?? input, topK)
// eslint-disable-next-line no-console
console.log('\x1b[94m\x1b[1m\n*****VectorStore Documents*****\n\x1b[0m\x1b[0m')
// eslint-disable-next-line no-console
packages/components/nodes/embeddings/AWSBedrockEmbedding/AWSBedrockEmbedding.ts
+4 -1
View File
@@ -79,7 +79,10 @@ class AWSBedrockEmbedding_Embeddings implements INode {
label: 'Model Name',
name: 'model',
type: 'options',
options: [{ label: 'amazon.titan-embed-text-v1', name: 'amazon.titan-embed-text-v1' }],
options: [
{ label: 'amazon.titan-embed-text-v1', name: 'amazon.titan-embed-text-v1' },
{ label: 'amazon.titan-embed-g1-text-02', name: 'amazon.titan-embed-g1-text-02' }
],
default: 'amazon.titan-embed-text-v1'
}
]
packages/components/nodes/embeddings/OpenAIEmbeddingCustom/OpenAIEmbeddingCustom.ts
+94
View File
@@ -0,0 +1,94 @@
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { OpenAIEmbeddings, OpenAIEmbeddingsParams } from 'langchain/embeddings/openai'
class OpenAIEmbeddingCustom_Embeddings implements INode {
label: string
name: string
version: number
type: string
icon: string
category: string
description: string
baseClasses: string[]
credential: INodeParams
inputs: INodeParams[]
constructor() {
this.label = 'OpenAI Embeddings Custom'
this.name = 'openAIEmbeddingsCustom'
this.version = 1.0
this.type = 'OpenAIEmbeddingsCustom'
this.icon = 'openai.png'
this.category = 'Embeddings'
this.description = 'OpenAI API to generate embeddings for a given text'
this.baseClasses = [this.type, ...getBaseClasses(OpenAIEmbeddings)]
this.credential = {
label: 'Connect Credential',
name: 'credential',
type: 'credential',
credentialNames: ['openAIApi']
}
this.inputs = [
{
label: 'Strip New Lines',
name: 'stripNewLines',
type: 'boolean',
optional: true,
additionalParams: true
},
{
label: 'Batch Size',
name: 'batchSize',
type: 'number',
optional: true,
additionalParams: true
},
{
label: 'Timeout',
name: 'timeout',
type: 'number',
optional: true,
additionalParams: true
},
{
label: 'BasePath',
name: 'basepath',
type: 'string',
optional: true,
additionalParams: true
},
{
label: 'Model Name',
name: 'modelName',
type: 'string',
optional: true
}
]
}
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
const stripNewLines = nodeData.inputs?.stripNewLines as boolean
const batchSize = nodeData.inputs?.batchSize as string
const timeout = nodeData.inputs?.timeout as string
const basePath = nodeData.inputs?.basepath as string
const modelName = nodeData.inputs?.modelName as string
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData)
const obj: Partial<OpenAIEmbeddingsParams> & { openAIApiKey?: string } = {
openAIApiKey
}
if (stripNewLines) obj.stripNewLines = stripNewLines
if (batchSize) obj.batchSize = parseInt(batchSize, 10)
if (timeout) obj.timeout = parseInt(timeout, 10)
if (modelName) obj.modelName = modelName
const model = new OpenAIEmbeddings(obj, { basePath })
return model
}
}
module.exports = { nodeClass: OpenAIEmbeddingCustom_Embeddings }
packages/components/nodes/embeddings/OpenAIEmbeddingCustom/openai.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 3.9 KiB

packages/components/nodes/memory/BufferWindowMemory/BufferWindowMemory.ts
+2 -2
View File
@@ -20,7 +20,7 @@ class BufferWindowMemory_Memory implements INode {
this.type = 'BufferWindowMemory'
this.icon = 'memory.svg'
this.category = 'Memory'
this.description = 'Uses a window of size k to surface the last k back-and-forths to use as memory'
this.description = 'Uses a window of size k to surface the last k back-and-forth to use as memory'
this.baseClasses = [this.type, ...getBaseClasses(BufferWindowMemory)]
this.inputs = [
{
@@ -40,7 +40,7 @@ class BufferWindowMemory_Memory implements INode {
name: 'k',
type: 'number',
default: '4',
description: 'Window of size k to surface the last k back-and-forths to use as memory.'
description: 'Window of size k to surface the last k back-and-forth to use as memory.'
}
]
}
packages/components/nodes/memory/RedisBackedChatMemory/RedisBackedChatMemory.ts
+56 -19
View File
@@ -1,9 +1,9 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface'
import { getBaseClasses } from '../../../src/utils'
import { ICommonObject } from '../../../src'
import { INode, INodeData, INodeParams, ICommonObject } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { BufferMemory, BufferMemoryInput } from 'langchain/memory'
import { RedisChatMessageHistory, RedisChatMessageHistoryInput } from 'langchain/stores/message/redis'
import { createClient } from 'redis'
import { RedisChatMessageHistory, RedisChatMessageHistoryInput } from 'langchain/stores/message/ioredis'
import { mapStoredMessageToChatMessage, BaseMessage } from 'langchain/schema'
import { Redis } from 'ioredis'
class RedisBackedChatMemory_Memory implements INode {
label: string
@@ -15,23 +15,25 @@ class RedisBackedChatMemory_Memory implements INode {
category: string
baseClasses: string[]
inputs: INodeParams[]
credential: INodeParams
constructor() {
this.label = 'Redis-Backed Chat Memory'
this.name = 'RedisBackedChatMemory'
this.version = 1.0
this.version = 2.0
this.type = 'RedisBackedChatMemory'
this.icon = 'redis.svg'
this.category = 'Memory'
this.description = 'Summarizes the conversation and stores the memory in Redis server'
this.baseClasses = [this.type, ...getBaseClasses(BufferMemory)]
this.credential = {
label: 'Connect Credential',
name: 'credential',
type: 'credential',
optional: true,
credentialNames: ['redisCacheApi', 'redisCacheUrlApi']
}
this.inputs = [
{
label: 'Base URL',
name: 'baseURL',
type: 'string',
default: 'redis://localhost:6379'
},
{
label: 'Session Id',
name: 'sessionId',
@@ -60,11 +62,11 @@ class RedisBackedChatMemory_Memory implements INode {
}
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
return initalizeRedis(nodeData, options)
return await initalizeRedis(nodeData, options)
}
async clearSessionMemory(nodeData: INodeData, options: ICommonObject): Promise<void> {
const redis = initalizeRedis(nodeData, options)
const redis = await initalizeRedis(nodeData, options)
const sessionId = nodeData.inputs?.sessionId as string
const chatId = options?.chatId as string
options.logger.info(`Clearing Redis memory session ${sessionId ? sessionId : chatId}`)
@@ -73,8 +75,7 @@ class RedisBackedChatMemory_Memory implements INode {
}
}
const initalizeRedis = (nodeData: INodeData, options: ICommonObject): BufferMemory => {
const baseURL = nodeData.inputs?.baseURL as string
const initalizeRedis = async (nodeData: INodeData, options: ICommonObject): Promise<BufferMemory> => {
const sessionId = nodeData.inputs?.sessionId as string
const sessionTTL = nodeData.inputs?.sessionTTL as number
const memoryKey = nodeData.inputs?.memoryKey as string
@@ -83,10 +84,29 @@ const initalizeRedis = (nodeData: INodeData, options: ICommonObject): BufferMemo
let isSessionIdUsingChatMessageId = false
if (!sessionId && chatId) isSessionIdUsingChatMessageId = true
const redisClient = createClient({ url: baseURL })
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const redisUrl = getCredentialParam('redisUrl', credentialData, nodeData)
let client: Redis
if (!redisUrl || redisUrl === '') {
const username = getCredentialParam('redisCacheUser', credentialData, nodeData)
const password = getCredentialParam('redisCachePwd', credentialData, nodeData)
const portStr = getCredentialParam('redisCachePort', credentialData, nodeData)
const host = getCredentialParam('redisCacheHost', credentialData, nodeData)
client = new Redis({
port: portStr ? parseInt(portStr) : 6379,
host,
username,
password
})
} else {
client = new Redis(redisUrl)
}
let obj: RedisChatMessageHistoryInput = {
sessionId: sessionId ? sessionId : chatId,
client: redisClient
client
}
if (sessionTTL) {
@@ -98,10 +118,27 @@ const initalizeRedis = (nodeData: INodeData, options: ICommonObject): BufferMemo
const redisChatMessageHistory = new RedisChatMessageHistory(obj)
redisChatMessageHistory.getMessages = async (): Promise<BaseMessage[]> => {
const rawStoredMessages = await client.lrange((redisChatMessageHistory as any).sessionId, 0, -1)
const orderedMessages = rawStoredMessages.reverse().map((message) => JSON.parse(message))
return orderedMessages.map(mapStoredMessageToChatMessage)
}
redisChatMessageHistory.addMessage = async (message: BaseMessage): Promise<void> => {
const messageToAdd = [message].map((msg) => msg.toDict())
await client.lpush((redisChatMessageHistory as any).sessionId, JSON.stringify(messageToAdd[0]))
if (sessionTTL) {
await client.expire((redisChatMessageHistory as any).sessionId, sessionTTL)
}
}
redisChatMessageHistory.clear = async (): Promise<void> => {
await client.del((redisChatMessageHistory as any).sessionId)
}
const memory = new BufferMemoryExtended({
memoryKey,
chatHistory: redisChatMessageHistory,
returnMessages: true,
isSessionIdUsingChatMessageId
})
return memory
packages/components/nodes/memory/ZepMemory/ZepMemory.ts
+1 -1
View File
@@ -60,7 +60,7 @@ class ZepMemory_Memory implements INode {
name: 'k',
type: 'number',
default: '10',
description: 'Window of size k to surface the last k back-and-forths to use as memory.'
description: 'Window of size k to surface the last k back-and-forth to use as memory.'
},
{
label: 'Auto Summary Template',
packages/components/nodes/prompts/ChatPromptTemplate/ChatPromptTemplate.ts
+6 -2
View File
@@ -51,7 +51,7 @@ class ChatPromptTemplate_Prompts implements INode {
async init(nodeData: INodeData): Promise<any> {
const systemMessagePrompt = nodeData.inputs?.systemMessagePrompt as string
const humanMessagePrompt = nodeData.inputs?.humanMessagePrompt as string
const promptValuesStr = nodeData.inputs?.promptValues as string
const promptValuesStr = nodeData.inputs?.promptValues
const prompt = ChatPromptTemplate.fromMessages([
SystemMessagePromptTemplate.fromTemplate(systemMessagePrompt),
@@ -60,7 +60,11 @@ class ChatPromptTemplate_Prompts implements INode {
let promptValues: ICommonObject = {}
if (promptValuesStr) {
promptValues = JSON.parse(promptValuesStr)
try {
promptValues = typeof promptValuesStr === 'object' ? promptValuesStr : JSON.parse(promptValuesStr)
} catch (exception) {
throw new Error("Invalid JSON in the ChatPromptTemplate's promptValues: " + exception)
}
}
// @ts-ignore
prompt.promptValues = promptValues
packages/components/nodes/prompts/FewShotPromptTemplate/FewShotPromptTemplate.ts
+11 -3
View File
@@ -55,7 +55,7 @@ class FewShotPromptTemplate_Prompts implements INode {
placeholder: `Word: {input}\nAntonym:`
},
{
label: 'Example Seperator',
label: 'Example Separator',
name: 'exampleSeparator',
type: 'string',
placeholder: `\n\n`
@@ -80,7 +80,7 @@ class FewShotPromptTemplate_Prompts implements INode {
}
async init(nodeData: INodeData): Promise<any> {
const examplesStr = nodeData.inputs?.examples as string
const examplesStr = nodeData.inputs?.examples
const prefix = nodeData.inputs?.prefix as string
const suffix = nodeData.inputs?.suffix as string
const exampleSeparator = nodeData.inputs?.exampleSeparator as string
@@ -88,7 +88,15 @@ class FewShotPromptTemplate_Prompts implements INode {
const examplePrompt = nodeData.inputs?.examplePrompt as PromptTemplate
const inputVariables = getInputVariables(suffix)
const examples: Example[] = JSON.parse(examplesStr)
let examples: Example[] = []
if (examplesStr) {
try {
examples = typeof examplesStr === 'object' ? examplesStr : JSON.parse(examplesStr)
} catch (exception) {
throw new Error("Invalid JSON in the FewShotPromptTemplate's examples: " + exception)
}
}
try {
const obj: FewShotPromptTemplateInput = {
packages/components/nodes/prompts/PromptTemplate/PromptTemplate.ts
+6 -2
View File
@@ -43,11 +43,15 @@ class PromptTemplate_Prompts implements INode {
async init(nodeData: INodeData): Promise<any> {
const template = nodeData.inputs?.template as string
const promptValuesStr = nodeData.inputs?.promptValues as string
const promptValuesStr = nodeData.inputs?.promptValues
let promptValues: ICommonObject = {}
if (promptValuesStr) {
promptValues = JSON.parse(promptValuesStr)
try {
promptValues = typeof promptValuesStr === 'object' ? promptValuesStr : JSON.parse(promptValuesStr)
} catch (exception) {
throw new Error("Invalid JSON in the PromptTemplate's promptValues: " + exception)
}
}
const inputVariables = getInputVariables(template)
packages/components/nodes/textsplitters/CharacterTextSplitter/CharacterTextSplitter.ts
+1 -1
View File
@@ -41,7 +41,7 @@ class CharacterTextSplitter_TextSplitters implements INode {
name: 'separator',
type: 'string',
placeholder: `" "`,
description: 'Seperator to determine when to split the text, will override the default separator',
description: 'Separator to determine when to split the text, will override the default separator',
optional: true
}
]
packages/components/nodes/textsplitters/RecursiveCharacterTextSplitter/RecursiveCharacterTextSplitter.ts
+4 -3
View File
@@ -41,8 +41,9 @@ class RecursiveCharacterTextSplitter_TextSplitters implements INode {
name: 'separators',
type: 'string',
rows: 4,
description: 'Array of custom seperators to determine when to split the text, will override the default separators',
description: 'Array of custom separators to determine when to split the text, will override the default separators',
placeholder: `["|", "##", ">", "-"]`,
additionalParams: true,
optional: true
}
]
@@ -51,7 +52,7 @@ class RecursiveCharacterTextSplitter_TextSplitters implements INode {
async init(nodeData: INodeData): Promise<any> {
const chunkSize = nodeData.inputs?.chunkSize as string
const chunkOverlap = nodeData.inputs?.chunkOverlap as string
const separators = nodeData.inputs?.separators as string
const separators = nodeData.inputs?.separators
const obj = {} as RecursiveCharacterTextSplitterParams
@@ -59,7 +60,7 @@ class RecursiveCharacterTextSplitter_TextSplitters implements INode {
if (chunkOverlap) obj.chunkOverlap = parseInt(chunkOverlap, 10)
if (separators) {
try {
obj.separators = JSON.parse(separators)
obj.separators = typeof separators === 'object' ? separators : JSON.parse(separators)
} catch (e) {
throw new Error(e)
}
packages/components/nodes/tools/SearchApi/SearchAPI.ts
+42
View File
@@ -0,0 +1,42 @@
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { SearchApi } from 'langchain/tools'
class SearchAPI_Tools implements INode {
label: string
name: string
version: number
description: string
type: string
icon: string
category: string
baseClasses: string[]
credential: INodeParams
inputs: INodeParams[]
constructor() {
this.label = 'SearchApi'
this.name = 'searchAPI'
this.version = 1.0
this.type = 'SearchAPI'
this.icon = 'searchapi.svg'
this.category = 'Tools'
this.description = 'Real-time API for accessing Google Search data'
this.inputs = []
this.credential = {
label: 'Connect Credential',
name: 'credential',
type: 'credential',
credentialNames: ['searchApi']
}
this.baseClasses = [this.type, ...getBaseClasses(SearchApi)]
}
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const searchApiKey = getCredentialParam('searchApiKey', credentialData, nodeData)
return new SearchApi(searchApiKey)
}
}
module.exports = { nodeClass: SearchAPI_Tools }
packages/components/nodes/tools/SearchApi/searchapi.svg
+1
View File
@@ -0,0 +1 @@
<svg id="SvgjsSvg1001" width="75.27131652832031" height="63.92396926879883" xmlns="http://www.w3.org/2000/svg" version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:svgjs="http://svgjs.com/svgjs" viewBox="0 0 75.27131652832031 63.92396926879883"><defs id="SvgjsDefs1002"></defs><rect id="SvgjsRect1008" width="75.27131652832031" height="63.92396926879883" fill="transparent"></rect><g id="SvgjsG1009" transform="matrix(1,0,0,1,-39.50003433227539,-50.53549575805664)"><title>0479_octopus_verti</title><path id="color_1" d="M97.24234,109.8245a2.57759,2.57759,0,0,1-2.57778,2.57778,9.80672,9.80672,0,0,1-9.79558-9.79557V91.58366a2.57779,2.57779,0,0,1,5.15557,0v11.02305a4.64509,4.64509,0,0,0,4.64,4.64A2.57759,2.57759,0,0,1,97.24234,109.8245ZM112.19348,93.223a2.57759,2.57759,0,0,0-2.57778,2.57779,4.64,4.64,0,1,1-9.28,0V73.73554a23.2,23.2,0,1,0-46.40009,0V95.80076a4.64,4.64,0,1,1-9.28,0,2.57779,2.57779,0,1,0-5.15557,0,9.79558,9.79558,0,0,0,19.59115,0V73.73554a18.04449,18.04449,0,0,1,36.089,0V95.80076a9.79558,9.79558,0,0,0,19.59115,0A2.57759,2.57759,0,0,0,112.19348,93.223ZM77.13563,91.78a2.57759,2.57759,0,0,0-2.57778,2.57778v17.52893a2.57779,2.57779,0,0,0,5.15557,0V94.3578A2.57759,2.57759,0,0,0,77.13563,91.78ZM66.8245,89.00588a2.57759,2.57759,0,0,0-2.57778,2.57778v11.02305a4.64509,4.64509,0,0,1-4.64,4.64,2.57778,2.57778,0,1,0,0,5.15556,9.80671,9.80671,0,0,0,9.79557-9.79557V91.58366A2.57759,2.57759,0,0,0,66.8245,89.00588ZM69.918,70.12639a3.6089,3.6089,0,1,0,3.6089,3.6089A3.60891,3.60891,0,0,0,69.918,70.12639Zm18.04448,3.6089a3.6089,3.6089,0,1,0-3.60889,3.60889A3.60891,3.60891,0,0,0,87.96251,73.73529Z" fill="#3730a3"></path></g></svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.6 KiB

packages/components/nodes/vectorstores/Chroma/Chroma_Upsert.ts
+3 -1
View File
@@ -98,7 +98,9 @@ class ChromaUpsert_VectorStores implements INode {
const flattenDocs = docs && docs.length ? flatten(docs) : []
const finalDocs = []
for (let i = 0; i < flattenDocs.length; i += 1) {
finalDocs.push(new Document(flattenDocs[i]))
if (flattenDocs[i] && flattenDocs[i].pageContent) {
finalDocs.push(new Document(flattenDocs[i]))
}
}
const obj: {
packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Upsert.ts
+3 -1
View File
@@ -38,7 +38,9 @@ class ElasicsearchUpsert_VectorStores extends ElasticSearchBase implements INode
const flattenDocs = docs && docs.length ? flatten(docs) : []
const finalDocs = []
for (let i = 0; i < flattenDocs.length; i += 1) {
finalDocs.push(new Document(flattenDocs[i]))
if (flattenDocs[i] && flattenDocs[i].pageContent) {
finalDocs.push(new Document(flattenDocs[i]))
}
}
// The following code is a workaround for a bug (Langchain Issue #1589) in the underlying library.
packages/components/nodes/vectorstores/Faiss_Upsert/Faiss_Upsert.ts
+3 -1
View File
@@ -80,7 +80,9 @@ class FaissUpsert_VectorStores implements INode {
const flattenDocs = docs && docs.length ? flatten(docs) : []
const finalDocs = []
for (let i = 0; i < flattenDocs.length; i += 1) {
finalDocs.push(new Document(flattenDocs[i]))
if (flattenDocs[i] && flattenDocs[i].pageContent) {
finalDocs.push(new Document(flattenDocs[i]))
}
}
const vectorStore = await FaissStore.fromDocuments(finalDocs, embeddings)
packages/components/nodes/vectorstores/InMemory/InMemoryVectorStore.ts
+3 -1
View File
@@ -71,7 +71,9 @@ class InMemoryVectorStore_VectorStores implements INode {
const flattenDocs = docs && docs.length ? flatten(docs) : []
const finalDocs = []
for (let i = 0; i < flattenDocs.length; i += 1) {
finalDocs.push(new Document(flattenDocs[i]))
if (flattenDocs[i] && flattenDocs[i].pageContent) {
finalDocs.push(new Document(flattenDocs[i]))
}
}
const vectorStore = await MemoryVectorStore.fromDocuments(finalDocs, embeddings)
packages/components/nodes/vectorstores/Milvus/Milvus_Upsert.ts
+3 -1
View File
@@ -110,7 +110,9 @@ class Milvus_Upsert_VectorStores implements INode {
const flattenDocs = docs && docs.length ? flatten(docs) : []
const finalDocs = []
for (let i = 0; i < flattenDocs.length; i += 1) {
finalDocs.push(new Document(flattenDocs[i]))
if (flattenDocs[i] && flattenDocs[i].pageContent) {
finalDocs.push(new Document(flattenDocs[i]))
}
}
const vectorStore = await MilvusUpsert.fromDocuments(finalDocs, embeddings, milVusArgs)
packages/components/nodes/vectorstores/OpenSearch_Upsert/OpenSearch_Upsert.ts
+3 -1
View File
@@ -86,7 +86,9 @@ class OpenSearchUpsert_VectorStores implements INode {
const flattenDocs = docs && docs.length ? flatten(docs) : []
const finalDocs = []
for (let i = 0; i < flattenDocs.length; i += 1) {
finalDocs.push(new Document(flattenDocs[i]))
if (flattenDocs[i] && flattenDocs[i].pageContent) {
finalDocs.push(new Document(flattenDocs[i]))
}
}
const client = new Client({
packages/components/nodes/vectorstores/Pinecone/Pinecone_Upsert.ts
+3 -1
View File
@@ -106,7 +106,9 @@ class PineconeUpsert_VectorStores implements INode {
const flattenDocs = docs && docs.length ? flatten(docs) : []
const finalDocs = []
for (let i = 0; i < flattenDocs.length; i += 1) {
finalDocs.push(new Document(flattenDocs[i]))
if (flattenDocs[i] && flattenDocs[i].pageContent) {
finalDocs.push(new Document(flattenDocs[i]))
}
}
const obj: PineconeLibArgs = {
packages/components/nodes/vectorstores/Postgres_Existing/Postgres_Exisiting.ts
+18
View File
@@ -65,6 +65,13 @@ class Postgres_Existing_VectorStores implements INode {
additionalParams: true,
optional: true
},
{
label: 'Additional Configuration',
name: 'additionalConfig',
type: 'json',
additionalParams: true,
optional: true
},
{
label: 'Top K',
name: 'topK',
@@ -96,11 +103,22 @@ class Postgres_Existing_VectorStores implements INode {
const _tableName = nodeData.inputs?.tableName as string
const tableName = _tableName ? _tableName : 'documents'
const embeddings = nodeData.inputs?.embeddings as Embeddings
const additionalConfig = nodeData.inputs?.additionalConfig as string
const output = nodeData.outputs?.output as string
const topK = nodeData.inputs?.topK as string
const k = topK ? parseFloat(topK) : 4
let additionalConfiguration = {}
if (additionalConfig) {
try {
additionalConfiguration = typeof additionalConfig === 'object' ? additionalConfig : JSON.parse(additionalConfig)
} catch (exception) {
throw new Error('Invalid JSON in the Additional Configuration: ' + exception)
}
}
const postgresConnectionOptions = {
...additionalConfiguration,
type: 'postgres',
host: nodeData.inputs?.host as string,
port: nodeData.inputs?.port as number,
packages/components/nodes/vectorstores/Postgres_Upsert/Postgres_Upsert.ts
+21 -1
View File
@@ -72,6 +72,13 @@ class PostgresUpsert_VectorStores implements INode {
additionalParams: true,
optional: true
},
{
label: 'Additional Configuration',
name: 'additionalConfig',
type: 'json',
additionalParams: true,
optional: true
},
{
label: 'Top K',
name: 'topK',
@@ -104,11 +111,22 @@ class PostgresUpsert_VectorStores implements INode {
const tableName = _tableName ? _tableName : 'documents'
const docs = nodeData.inputs?.document as Document[]
const embeddings = nodeData.inputs?.embeddings as Embeddings
const additionalConfig = nodeData.inputs?.additionalConfig as string
const output = nodeData.outputs?.output as string
const topK = nodeData.inputs?.topK as string
const k = topK ? parseFloat(topK) : 4
let additionalConfiguration = {}
if (additionalConfig) {
try {
additionalConfiguration = typeof additionalConfig === 'object' ? additionalConfig : JSON.parse(additionalConfig)
} catch (exception) {
throw new Error('Invalid JSON in the Additional Configuration: ' + exception)
}
}
const postgresConnectionOptions = {
...additionalConfiguration,
type: 'postgres',
host: nodeData.inputs?.host as string,
port: nodeData.inputs?.port as number,
@@ -125,7 +143,9 @@ class PostgresUpsert_VectorStores implements INode {
const flattenDocs = docs && docs.length ? flatten(docs) : []
const finalDocs = []
for (let i = 0; i < flattenDocs.length; i += 1) {
finalDocs.push(new Document(flattenDocs[i]))
if (flattenDocs[i] && flattenDocs[i].pageContent) {
finalDocs.push(new Document(flattenDocs[i]))
}
}
const vectorStore = await TypeORMVectorStore.fromDocuments(finalDocs, embeddings, args)
packages/components/nodes/vectorstores/Qdrant_Existing/Qdrant_Existing.ts
+43 -3
View File
@@ -23,7 +23,7 @@ class Qdrant_Existing_VectorStores implements INode {
constructor() {
this.label = 'Qdrant Load Existing Index'
this.name = 'qdrantExistingIndex'
this.version = 1.0
this.version = 2.0
this.type = 'Qdrant'
this.icon = 'qdrant.png'
this.category = 'Vector Stores'
@@ -55,8 +55,39 @@ class Qdrant_Existing_VectorStores implements INode {
type: 'string'
},
{
label: 'Qdrant Collection Cofiguration',
label: 'Vector Dimension',
name: 'qdrantVectorDimension',
type: 'number',
default: 1536,
additionalParams: true
},
{
label: 'Similarity',
name: 'qdrantSimilarity',
description: 'Similarity measure used in Qdrant.',
type: 'options',
default: 'Cosine',
options: [
{
label: 'Cosine',
name: 'Cosine'
},
{
label: 'Euclid',
name: 'Euclid'
},
{
label: 'Dot',
name: 'Dot'
}
],
additionalParams: true
},
{
label: 'Additional Collection Cofiguration',
name: 'qdrantCollectionConfiguration',
description:
'Refer to <a target="_blank" href="https://qdrant.tech/documentation/concepts/collections">collection docs</a> for more reference',
type: 'json',
optional: true,
additionalParams: true
@@ -98,6 +129,8 @@ class Qdrant_Existing_VectorStores implements INode {
const collectionName = nodeData.inputs?.qdrantCollection as string
let qdrantCollectionConfiguration = nodeData.inputs?.qdrantCollectionConfiguration
const embeddings = nodeData.inputs?.embeddings as Embeddings
const qdrantSimilarity = nodeData.inputs?.qdrantSimilarity
const qdrantVectorDimension = nodeData.inputs?.qdrantVectorDimension
const output = nodeData.outputs?.output as string
const topK = nodeData.inputs?.topK as string
let queryFilter = nodeData.inputs?.queryFilter
@@ -126,7 +159,14 @@ class Qdrant_Existing_VectorStores implements INode {
typeof qdrantCollectionConfiguration === 'object'
? qdrantCollectionConfiguration
: JSON.parse(qdrantCollectionConfiguration)
dbConfig.collectionConfig = qdrantCollectionConfiguration
dbConfig.collectionConfig = {
...qdrantCollectionConfiguration,
vectors: {
...qdrantCollectionConfiguration.vectors,
size: qdrantVectorDimension ? parseInt(qdrantVectorDimension, 10) : 1536,
distance: qdrantSimilarity ?? 'Cosine'
}
}
}
if (queryFilter) {
packages/components/nodes/vectorstores/Qdrant_Upsert/Qdrant_Upsert.ts
+43 -3
View File
@@ -25,7 +25,7 @@ class QdrantUpsert_VectorStores implements INode {
constructor() {
this.label = 'Qdrant Upsert Document'
this.name = 'qdrantUpsert'
this.version = 1.0
this.version = 2.0
this.type = 'Qdrant'
this.icon = 'qdrant.png'
this.category = 'Vector Stores'
@@ -62,6 +62,35 @@ class QdrantUpsert_VectorStores implements INode {
name: 'qdrantCollection',
type: 'string'
},
{
label: 'Vector Dimension',
name: 'qdrantVectorDimension',
type: 'number',
default: 1536,
additionalParams: true
},
{
label: 'Similarity',
name: 'qdrantSimilarity',
description: 'Similarity measure used in Qdrant.',
type: 'options',
default: 'Cosine',
options: [
{
label: 'Cosine',
name: 'Cosine'
},
{
label: 'Euclid',
name: 'Euclid'
},
{
label: 'Dot',
name: 'Dot'
}
],
additionalParams: true
},
{
label: 'Top K',
name: 'topK',
@@ -99,6 +128,9 @@ class QdrantUpsert_VectorStores implements INode {
const collectionName = nodeData.inputs?.qdrantCollection as string
const docs = nodeData.inputs?.document as Document[]
const embeddings = nodeData.inputs?.embeddings as Embeddings
const qdrantSimilarity = nodeData.inputs?.qdrantSimilarity
const qdrantVectorDimension = nodeData.inputs?.qdrantVectorDimension
const output = nodeData.outputs?.output as string
const topK = nodeData.inputs?.topK as string
const k = topK ? parseFloat(topK) : 4
@@ -115,13 +147,21 @@ class QdrantUpsert_VectorStores implements INode {
const flattenDocs = docs && docs.length ? flatten(docs) : []
const finalDocs = []
for (let i = 0; i < flattenDocs.length; i += 1) {
finalDocs.push(new Document(flattenDocs[i]))
if (flattenDocs[i] && flattenDocs[i].pageContent) {
finalDocs.push(new Document(flattenDocs[i]))
}
}
const dbConfig: QdrantLibArgs = {
client,
url: qdrantServerUrl,
collectionName
collectionName,
collectionConfig: {
vectors: {
size: qdrantVectorDimension ? parseInt(qdrantVectorDimension, 10) : 1536,
distance: qdrantSimilarity ?? 'Cosine'
}
}
}
const retrieverConfig: RetrieverConfig = {
packages/components/nodes/vectorstores/Redis/RedisSearchBase.ts
+215
View File
@@ -0,0 +1,215 @@
import {
getBaseClasses,
getCredentialData,
getCredentialParam,
ICommonObject,
INodeData,
INodeOutputsValue,
INodeParams
} from '../../../src'
import { Embeddings } from 'langchain/embeddings/base'
import { VectorStore } from 'langchain/vectorstores/base'
import { Document } from 'langchain/document'
import { createClient, SearchOptions } from 'redis'
import { RedisVectorStore } from 'langchain/vectorstores/redis'
import { escapeSpecialChars, unEscapeSpecialChars } from './utils'
export abstract class RedisSearchBase {
label: string
name: string
version: number
description: string
type: string
icon: string
category: string
baseClasses: string[]
inputs: INodeParams[]
credential: INodeParams
outputs: INodeOutputsValue[]
redisClient: ReturnType<typeof createClient>
protected constructor() {
this.type = 'Redis'
this.icon = 'redis.svg'
this.category = 'Vector Stores'
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
this.credential = {
label: 'Connect Credential',
name: 'credential',
type: 'credential',
credentialNames: ['redisCacheUrlApi', 'redisCacheApi']
}
this.inputs = [
{
label: 'Embeddings',
name: 'embeddings',
type: 'Embeddings'
},
{
label: 'Index Name',
name: 'indexName',
placeholder: '<VECTOR_INDEX_NAME>',
type: 'string'
},
{
label: 'Replace Index?',
name: 'replaceIndex',
description: 'Selecting this option will delete the existing index and recreate a new one',
default: false,
type: 'boolean'
},
{
label: 'Content Field',
name: 'contentKey',
description: 'Name of the field (column) that contains the actual content',
type: 'string',
default: 'content',
additionalParams: true,
optional: true
},
{
label: 'Metadata Field',
name: 'metadataKey',
description: 'Name of the field (column) that contains the metadata of the document',
type: 'string',
default: 'metadata',
additionalParams: true,
optional: true
},
{
label: 'Vector Field',
name: 'vectorKey',
description: 'Name of the field (column) that contains the vector',
type: 'string',
default: 'content_vector',
additionalParams: true,
optional: true
},
{
label: 'Top K',
name: 'topK',
description: 'Number of top results to fetch. Default to 4',
placeholder: '4',
type: 'number',
additionalParams: true,
optional: true
}
]
this.outputs = [
{
label: 'Redis Retriever',
name: 'retriever',
baseClasses: this.baseClasses
},
{
label: 'Redis Vector Store',
name: 'vectorStore',
baseClasses: [this.type, ...getBaseClasses(RedisVectorStore)]
}
]
}
abstract constructVectorStore(
embeddings: Embeddings,
indexName: string,
replaceIndex: boolean,
docs: Document<Record<string, any>>[] | undefined
): Promise<VectorStore>
async init(nodeData: INodeData, _: string, options: ICommonObject, docs: Document<Record<string, any>>[] | undefined): Promise<any> {
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const indexName = nodeData.inputs?.indexName as string
let contentKey = nodeData.inputs?.contentKey as string
let metadataKey = nodeData.inputs?.metadataKey as string
let vectorKey = nodeData.inputs?.vectorKey as string
const embeddings = nodeData.inputs?.embeddings as Embeddings
const topK = nodeData.inputs?.topK as string
const replaceIndex = nodeData.inputs?.replaceIndex as boolean
const k = topK ? parseFloat(topK) : 4
const output = nodeData.outputs?.output as string
let redisUrl = getCredentialParam('redisUrl', credentialData, nodeData)
if (!redisUrl || redisUrl === '') {
const username = getCredentialParam('redisCacheUser', credentialData, nodeData)
const password = getCredentialParam('redisCachePwd', credentialData, nodeData)
const portStr = getCredentialParam('redisCachePort', credentialData, nodeData)
const host = getCredentialParam('redisCacheHost', credentialData, nodeData)
redisUrl = 'redis://' + username + ':' + password + '@' + host + ':' + portStr
}
this.redisClient = createClient({ url: redisUrl })
await this.redisClient.connect()
const vectorStore = await this.constructVectorStore(embeddings, indexName, replaceIndex, docs)
if (!contentKey || contentKey === '') contentKey = 'content'
if (!metadataKey || metadataKey === '') metadataKey = 'metadata'
if (!vectorKey || vectorKey === '') vectorKey = 'content_vector'
const buildQuery = (query: number[], k: number, filter?: string[]): [string, SearchOptions] => {
const vectorScoreField = 'vector_score'
let hybridFields = '*'
// if a filter is set, modify the hybrid query
if (filter && filter.length) {
// `filter` is a list of strings, then it's applied using the OR operator in the metadata key
hybridFields = `@${metadataKey}:(${filter.map(escapeSpecialChars).join('|')})`
}
const baseQuery = `${hybridFields} => [KNN ${k} @${vectorKey} $vector AS ${vectorScoreField}]`
const returnFields = [metadataKey, contentKey, vectorScoreField]
const options: SearchOptions = {
PARAMS: {
vector: Buffer.from(new Float32Array(query).buffer)
},
RETURN: returnFields,
SORTBY: vectorScoreField,
DIALECT: 2,
LIMIT: {
from: 0,
size: k
}
}
return [baseQuery, options]
}
vectorStore.similaritySearchVectorWithScore = async (
query: number[],
k: number,
filter?: string[]
): Promise<[Document, number][]> => {
const results = await this.redisClient.ft.search(indexName, ...buildQuery(query, k, filter))
const result: [Document, number][] = []
if (results.total) {
for (const res of results.documents) {
if (res.value) {
const document = res.value
if (document.vector_score) {
const metadataString = unEscapeSpecialChars(document[metadataKey] as string)
result.push([
new Document({
pageContent: document[contentKey] as string,
metadata: JSON.parse(metadataString)
}),
Number(document.vector_score)
])
}
}
}
}
return result
}
if (output === 'retriever') {
return vectorStore.asRetriever(k)
} else if (output === 'vectorStore') {
;(vectorStore as any).k = k
return vectorStore
}
return vectorStore
}
}
packages/components/nodes/vectorstores/Redis/Redis_Existing.ts
+42
View File
@@ -0,0 +1,42 @@
import { ICommonObject, INode, INodeData } from '../../../src/Interface'
import { Embeddings } from 'langchain/embeddings/base'
import { VectorStore } from 'langchain/vectorstores/base'
import { RedisVectorStore, RedisVectorStoreConfig } from 'langchain/vectorstores/redis'
import { Document } from 'langchain/document'
import { RedisSearchBase } from './RedisSearchBase'
class RedisExisting_VectorStores extends RedisSearchBase implements INode {
constructor() {
super()
this.label = 'Redis Load Existing Index'
this.name = 'RedisIndex'
this.version = 1.0
this.description = 'Load existing index from Redis (i.e: Document has been upserted)'
// Remove deleteIndex from inputs as it is not applicable while fetching data from Redis
let input = this.inputs.find((i) => i.name === 'deleteIndex')
if (input) this.inputs.splice(this.inputs.indexOf(input), 1)
}
async constructVectorStore(
embeddings: Embeddings,
indexName: string,
// eslint-disable-next-line unused-imports/no-unused-vars
replaceIndex: boolean,
_: Document<Record<string, any>>[]
): Promise<VectorStore> {
const storeConfig: RedisVectorStoreConfig = {
redisClient: this.redisClient,
indexName: indexName
}
return new RedisVectorStore(embeddings, storeConfig)
}
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
return super.init(nodeData, _, options, undefined)
}
}
module.exports = { nodeClass: RedisExisting_VectorStores }
packages/components/nodes/vectorstores/Redis/Redis_Upsert.ts
+63
View File
@@ -0,0 +1,63 @@
import { ICommonObject, INode, INodeData } from '../../../src/Interface'
import { Embeddings } from 'langchain/embeddings/base'
import { Document } from 'langchain/document'
import { flatten } from 'lodash'
import { RedisSearchBase } from './RedisSearchBase'
import { VectorStore } from 'langchain/vectorstores/base'
import { RedisVectorStore, RedisVectorStoreConfig } from 'langchain/vectorstores/redis'
import { escapeAllStrings } from './utils'
class RedisUpsert_VectorStores extends RedisSearchBase implements INode {
constructor() {
super()
this.label = 'Redis Upsert Document'
this.name = 'RedisUpsert'
this.version = 1.0
this.description = 'Upsert documents to Redis'
this.inputs.unshift({
label: 'Document',
name: 'document',
type: 'Document',
list: true
})
}
async constructVectorStore(
embeddings: Embeddings,
indexName: string,
replaceIndex: boolean,
docs: Document<Record<string, any>>[]
): Promise<VectorStore> {
const storeConfig: RedisVectorStoreConfig = {
redisClient: this.redisClient,
indexName: indexName
}
if (replaceIndex) {
let response = await this.redisClient.ft.dropIndex(indexName)
if (process.env.DEBUG === 'true') {
// eslint-disable-next-line no-console
console.log(`Redis Vector Store :: Dropping index [${indexName}], Received Response [${response}]`)
}
}
return await RedisVectorStore.fromDocuments(docs, embeddings, storeConfig)
}
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
const docs = nodeData.inputs?.document as Document[]
const flattenDocs = docs && docs.length ? flatten(docs) : []
const finalDocs = []
for (let i = 0; i < flattenDocs.length; i += 1) {
if (flattenDocs[i] && flattenDocs[i].pageContent) {
const document = new Document(flattenDocs[i])
escapeAllStrings(document.metadata)
finalDocs.push(document)
}
}
return super.init(nodeData, _, options, flattenDocs)
}
}
module.exports = { nodeClass: RedisUpsert_VectorStores }
packages/components/nodes/vectorstores/Redis/redis.svg
+1
View File
@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 128 128" id="redis"><path fill="#A41E11" d="M121.8 93.1c-6.7 3.5-41.4 17.7-48.8 21.6-7.4 3.9-11.5 3.8-17.3 1s-42.7-17.6-49.4-20.8c-3.3-1.6-5-2.9-5-4.2v-12.7s48-10.5 55.8-13.2c7.8-2.8 10.4-2.9 17-.5s46.1 9.5 52.6 11.9v12.5c0 1.3-1.5 2.7-4.9 4.4z"></path><path fill="#D82C20" d="M121.8 80.5c-6.7 3.5-41.4 17.7-48.8 21.6-7.4 3.9-11.5 3.8-17.3 1-5.8-2.8-42.7-17.7-49.4-20.9-6.6-3.2-6.8-5.4-.3-7.9 6.5-2.6 43.2-17 51-19.7 7.8-2.8 10.4-2.9 17-.5s41.1 16.1 47.6 18.5c6.7 2.4 6.9 4.4.2 7.9z"></path><path fill="#A41E11" d="M121.8 72.5c-6.7 3.5-41.4 17.7-48.8 21.6-7.4 3.8-11.5 3.8-17.3 1-5.8-2.8-42.7-17.7-49.4-20.9-3.3-1.6-5-2.9-5-4.2v-12.7s48-10.5 55.8-13.2c7.8-2.8 10.4-2.9 17-.5s46.1 9.5 52.6 11.9v12.5c0 1.3-1.5 2.7-4.9 4.5z"></path><path fill="#D82C20" d="M121.8 59.8c-6.7 3.5-41.4 17.7-48.8 21.6-7.4 3.8-11.5 3.8-17.3 1-5.8-2.8-42.7-17.7-49.4-20.9s-6.8-5.4-.3-7.9c6.5-2.6 43.2-17 51-19.7 7.8-2.8 10.4-2.9 17-.5s41.1 16.1 47.6 18.5c6.7 2.4 6.9 4.4.2 7.9z"></path><path fill="#A41E11" d="M121.8 51c-6.7 3.5-41.4 17.7-48.8 21.6-7.4 3.8-11.5 3.8-17.3 1-5.8-2.7-42.7-17.6-49.4-20.8-3.3-1.6-5.1-2.9-5.1-4.2v-12.7s48-10.5 55.8-13.2c7.8-2.8 10.4-2.9 17-.5s46.1 9.5 52.6 11.9v12.5c.1 1.3-1.4 2.6-4.8 4.4z"></path><path fill="#D82C20" d="M121.8 38.3c-6.7 3.5-41.4 17.7-48.8 21.6-7.4 3.8-11.5 3.8-17.3 1s-42.7-17.6-49.4-20.8-6.8-5.4-.3-7.9c6.5-2.6 43.2-17 51-19.7 7.8-2.8 10.4-2.9 17-.5s41.1 16.1 47.6 18.5c6.7 2.4 6.9 4.4.2 7.8z"></path><path fill="#fff" d="M80.4 26.1l-10.8 1.2-2.5 5.8-3.9-6.5-12.5-1.1 9.3-3.4-2.8-5.2 8.8 3.4 8.2-2.7-2.2 5.4zM66.5 54.5l-20.3-8.4 29.1-4.4z"></path><ellipse cx="38.4" cy="35.4" fill="#fff" rx="15.5" ry="6"></ellipse><path fill="#7A0C00" d="M93.3 27.7l17.2 6.8-17.2 6.8z"></path><path fill="#AD2115" d="M74.3 35.3l19-7.6v13.6l-1.9.8z"></path></svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.8 KiB

packages/components/nodes/vectorstores/Redis/utils.ts
+25
View File
@@ -0,0 +1,25 @@
/*
* Escapes all '-' characters.
* Redis Search considers '-' as a negative operator, hence we need
* to escape it
*/
export const escapeSpecialChars = (str: string) => {
return str.replaceAll('-', '\\-')
}
export const escapeAllStrings = (obj: object) => {
Object.keys(obj).forEach((key: string) => {
// @ts-ignore
let item = obj[key]
if (typeof item === 'object') {
escapeAllStrings(item)
} else if (typeof item === 'string') {
// @ts-ignore
obj[key] = escapeSpecialChars(item)
}
})
}
export const unEscapeSpecialChars = (str: string) => {
return str.replaceAll('\\-', '-')
}
packages/components/nodes/vectorstores/Singlestore_Upsert/Singlestore_Upsert.ts
+3 -1
View File
@@ -140,7 +140,9 @@ class SingleStoreUpsert_VectorStores implements INode {
const flattenDocs = docs && docs.length ? flatten(docs) : []
const finalDocs = []
for (let i = 0; i < flattenDocs.length; i += 1) {
finalDocs.push(new Document(flattenDocs[i]))
if (flattenDocs[i] && flattenDocs[i].pageContent) {
finalDocs.push(new Document(flattenDocs[i]))
}
}
let vectorStore: SingleStoreVectorStore
packages/components/nodes/vectorstores/Vectara/Vectara_Upsert.ts
+3 -1
View File
@@ -132,7 +132,9 @@ class VectaraUpsert_VectorStores implements INode {
const flattenDocs = docs && docs.length ? flatten(docs) : []
const finalDocs = []
for (let i = 0; i < flattenDocs.length; i += 1) {
finalDocs.push(new Document(flattenDocs[i]))
if (flattenDocs[i] && flattenDocs[i].pageContent) {
finalDocs.push(new Document(flattenDocs[i]))
}
}
const vectorStore = await VectaraStore.fromDocuments(finalDocs, embeddings, vectaraArgs)
packages/components/nodes/vectorstores/Weaviate_Upsert/Weaviate_Upsert.ts
+3 -1
View File
@@ -143,7 +143,9 @@ class WeaviateUpsert_VectorStores implements INode {
const flattenDocs = docs && docs.length ? flatten(docs) : []
const finalDocs = []
for (let i = 0; i < flattenDocs.length; i += 1) {
finalDocs.push(new Document(flattenDocs[i]))
if (flattenDocs[i] && flattenDocs[i].pageContent) {
finalDocs.push(new Document(flattenDocs[i]))
}
}
const obj: WeaviateLibArgs = {
packages/components/nodes/vectorstores/Zep/Zep_Upsert.ts
+3 -1
View File
@@ -106,7 +106,9 @@ class Zep_Upsert_VectorStores implements INode {
const flattenDocs = docs && docs.length ? flatten(docs) : []
const finalDocs = []
for (let i = 0; i < flattenDocs.length; i += 1) {
finalDocs.push(new Document(flattenDocs[i]))
if (flattenDocs[i] && flattenDocs[i].pageContent) {
finalDocs.push(new Document(flattenDocs[i]))
}
}
const zepConfig: IZepConfig = {
packages/components/src/utils.ts
+3 -2
View File
@@ -133,6 +133,7 @@ export const getNodeModulesPackagePath = (packageName: string): string => {
* @returns {boolean}
*/
export const getInputVariables = (paramValue: string): string[] => {
if (typeof paramValue !== 'string') return []
let returnVal = paramValue
const variableStack = []
const inputVariables = []
@@ -302,7 +303,7 @@ async function crawl(baseURL: string, currentURL: string, pages: string[], limit
}
/**
* Prep URL before passing into recursive carwl function
* Prep URL before passing into recursive crawl function
* @param {string} stringURL
* @param {number} limit
* @returns {Promise<string[]>}
@@ -446,7 +447,7 @@ export const getCredentialData = async (selectedCredentialId: string, options: I
if (!credential) return {}
// Decrpyt credentialData
// Decrypt credentialData
const decryptedCredentialData = await decryptCredentialData(credential.encryptedData)
return decryptedCredentialData