farcasclaudiu/Flowise
1
0
Fork 0
mirror of https://github.com/farcasclaudiu/Flowise.git synced 2026-06-28 21:00:58 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'FlowiseAI:main' into main

Browse Source
This commit is contained in:
Darien Kindlund
2024-01-28 20:34:00 -08:00
committed by GitHub
parent 8ca8e0ede0 e72d9e4094
commit 6f7b7408e1
61 changed files with 1725 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/components/nodes/cache/RedisCache/RedisCache.ts
Vendored
+40 -3
View File
@@ -1,9 +1,46 @@
import { getBaseClasses, getCredentialData, getCredentialParam, ICommonObject, INode, INodeData, INodeParams } from '../../../src'
import { RedisCache as LangchainRedisCache } from 'langchain/cache/ioredis'
import { Redis } from 'ioredis'
import { Redis, RedisOptions } from 'ioredis'
import { isEqual } from 'lodash'
import { Generation, ChatGeneration, StoredGeneration, mapStoredMessageToChatMessage } from 'langchain/schema'
import hash from 'object-hash'
let redisClientSingleton: Redis
let redisClientOption: RedisOptions
let redisClientUrl: string
const getRedisClientbyOption = (option: RedisOptions) => {
if (!redisClientSingleton) {
// if client doesn't exists
redisClientSingleton = new Redis(option)
redisClientOption = option
return redisClientSingleton
} else if (redisClientSingleton && !isEqual(option, redisClientOption)) {
// if client exists but option changed
redisClientSingleton.quit()
redisClientSingleton = new Redis(option)
redisClientOption = option
return redisClientSingleton
}
return redisClientSingleton
}
const getRedisClientbyUrl = (url: string) => {
if (!redisClientSingleton) {
// if client doesn't exists
redisClientSingleton = new Redis(url)
redisClientUrl = url
return redisClientSingleton
} else if (redisClientSingleton && url !== redisClientUrl) {
// if client exists but option changed
redisClientSingleton.quit()
redisClientSingleton = new Redis(url)
redisClientUrl = url
return redisClientSingleton
}
return redisClientSingleton
}
class RedisCache implements INode {
label: string
name: string
@@ -60,7 +97,7 @@ class RedisCache implements INode {
const tlsOptions = sslEnabled === true ? { tls: { rejectUnauthorized: false } } : {}
client = new Redis({
client = getRedisClientbyOption({
port: portStr ? parseInt(portStr) : 6379,
host,
username,
@@ -68,7 +105,7 @@ class RedisCache implements INode {
...tlsOptions
})
} else {
client = new Redis(redisUrl)
client = getRedisClientbyUrl(redisUrl)
}
const redisClient = new LangchainRedisCache(client)
packages/components/nodes/cache/RedisCache/RedisEmbeddingsCache.ts
Vendored
+40 -3
View File
@@ -1,9 +1,46 @@
import { getBaseClasses, getCredentialData, getCredentialParam, ICommonObject, INode, INodeData, INodeParams } from '../../../src'
import { Redis } from 'ioredis'
import { Redis, RedisOptions } from 'ioredis'
import { isEqual } from 'lodash'
import { CacheBackedEmbeddings } from 'langchain/embeddings/cache_backed'
import { RedisByteStore } from 'langchain/storage/ioredis'
import { Embeddings } from 'langchain/embeddings/base'
let redisClientSingleton: Redis
let redisClientOption: RedisOptions
let redisClientUrl: string
const getRedisClientbyOption = (option: RedisOptions) => {
if (!redisClientSingleton) {
// if client doesn't exists
redisClientSingleton = new Redis(option)
redisClientOption = option
return redisClientSingleton
} else if (redisClientSingleton && !isEqual(option, redisClientOption)) {
// if client exists but option changed
redisClientSingleton.quit()
redisClientSingleton = new Redis(option)
redisClientOption = option
return redisClientSingleton
}
return redisClientSingleton
}
const getRedisClientbyUrl = (url: string) => {
if (!redisClientSingleton) {
// if client doesn't exists
redisClientSingleton = new Redis(url)
redisClientUrl = url
return redisClientSingleton
} else if (redisClientSingleton && url !== redisClientUrl) {
// if client exists but option changed
redisClientSingleton.quit()
redisClientSingleton = new Redis(url)
redisClientUrl = url
return redisClientSingleton
}
return redisClientSingleton
}
class RedisEmbeddingsCache implements INode {
label: string
name: string
@@ -75,7 +112,7 @@ class RedisEmbeddingsCache implements INode {
const tlsOptions = sslEnabled === true ? { tls: { rejectUnauthorized: false } } : {}
client = new Redis({
client = getRedisClientbyOption({
port: portStr ? parseInt(portStr) : 6379,
host,
username,
@@ -83,7 +120,7 @@ class RedisEmbeddingsCache implements INode {
...tlsOptions
})
} else {
client = new Redis(redisUrl)
client = getRedisClientbyUrl(redisUrl)
}
ttl ??= '3600'
packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts
+9 -1
View File
@@ -19,7 +19,7 @@ class ChatOpenAI_ChatModels implements INode {
constructor() {
this.label = 'ChatOpenAI'
this.name = 'chatOpenAI'
this.version = 2.0
this.version = 3.0
this.type = 'ChatOpenAI'
this.icon = 'openai.svg'
this.category = 'Chat Models'
@@ -47,6 +47,14 @@ class ChatOpenAI_ChatModels implements INode {
label: 'gpt-4',
name: 'gpt-4'
},
{
label: 'gpt-4-turbo-preview',
name: 'gpt-4-turbo-preview'
},
{
label: 'gpt-4-0125-preview',
name: 'gpt-4-0125-preview'
},
{
label: 'gpt-4-1106-preview',
name: 'gpt-4-1106-preview'
packages/components/nodes/documentloaders/Cheerio/Cheerio.ts
+22 -10
View File
@@ -1,4 +1,4 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { TextSplitter } from 'langchain/text_splitter'
import { CheerioWebBaseLoader, WebBaseLoaderParams } from 'langchain/document_loaders/web/cheerio'
import { test } from 'linkifyjs'
@@ -63,6 +63,7 @@ class Cheerio_DocumentLoaders implements INode {
name: 'limit',
type: 'number',
optional: true,
default: '10',
additionalParams: true,
description:
'Only used when "Get Relative Links Method" is selected. Set 0 to retrieve all relative links, default limit is 10.',
@@ -86,11 +87,12 @@ class Cheerio_DocumentLoaders implements INode {
]
}
async init(nodeData: INodeData): Promise<any> {
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
const metadata = nodeData.inputs?.metadata
const relativeLinksMethod = nodeData.inputs?.relativeLinksMethod as string
let limit = nodeData.inputs?.limit as string
const selectedLinks = nodeData.inputs?.selectedLinks as string[]
let limit = parseInt(nodeData.inputs?.limit as string)
let url = nodeData.inputs?.url as string
url = url.trim()
@@ -117,23 +119,33 @@ class Cheerio_DocumentLoaders implements INode {
}
return docs
} catch (err) {
if (process.env.DEBUG === 'true') console.error(`error in CheerioWebBaseLoader: ${err.message}, on page: ${url}`)
if (process.env.DEBUG === 'true') options.logger.error(`error in CheerioWebBaseLoader: ${err.message}, on page: ${url}`)
}
}
let docs = []
if (relativeLinksMethod) {
if (process.env.DEBUG === 'true') console.info(`Start ${relativeLinksMethod}`)
if (!limit) limit = '10'
else if (parseInt(limit) < 0) throw new Error('Limit cannot be less than 0')
if (process.env.DEBUG === 'true') options.logger.info(`Start ${relativeLinksMethod}`)
if (!limit) limit = 10
else if (limit < 0) throw new Error('Limit cannot be less than 0')
const pages: string[] =
relativeLinksMethod === 'webCrawl' ? await webCrawl(url, parseInt(limit)) : await xmlScrape(url, parseInt(limit))
if (process.env.DEBUG === 'true') console.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
selectedLinks && selectedLinks.length > 0
? selectedLinks.slice(0, limit)
: relativeLinksMethod === 'webCrawl'
? await webCrawl(url, limit)
: await xmlScrape(url, limit)
if (process.env.DEBUG === 'true') options.logger.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
if (!pages || pages.length === 0) throw new Error('No relative links found')
for (const page of pages) {
docs.push(...(await cheerioLoader(page)))
}
if (process.env.DEBUG === 'true') console.info(`Finish ${relativeLinksMethod}`)
if (process.env.DEBUG === 'true') options.logger.info(`Finish ${relativeLinksMethod}`)
} else if (selectedLinks && selectedLinks.length > 0) {
if (process.env.DEBUG === 'true')
options.logger.info(`pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`)
for (const page of selectedLinks) {
docs.push(...(await cheerioLoader(page)))
}
} else {
docs = await cheerioLoader(url)
}
packages/components/nodes/documentloaders/Playwright/Playwright.ts
+22 -10
View File
@@ -1,4 +1,4 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { TextSplitter } from 'langchain/text_splitter'
import { Browser, Page, PlaywrightWebBaseLoader, PlaywrightWebBaseLoaderOptions } from 'langchain/document_loaders/web/playwright'
import { test } from 'linkifyjs'
@@ -61,6 +61,7 @@ class Playwright_DocumentLoaders implements INode {
name: 'limit',
type: 'number',
optional: true,
default: '10',
additionalParams: true,
description:
'Only used when "Get Relative Links Method" is selected. Set 0 to retrieve all relative links, default limit is 10.',
@@ -114,11 +115,12 @@ class Playwright_DocumentLoaders implements INode {
]
}
async init(nodeData: INodeData): Promise<any> {
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
const metadata = nodeData.inputs?.metadata
const relativeLinksMethod = nodeData.inputs?.relativeLinksMethod as string
let limit = nodeData.inputs?.limit as string
const selectedLinks = nodeData.inputs?.selectedLinks as string[]
let limit = parseInt(nodeData.inputs?.limit as string)
let waitUntilGoToOption = nodeData.inputs?.waitUntilGoToOption as 'load' | 'domcontentloaded' | 'networkidle' | 'commit' | undefined
let waitForSelector = nodeData.inputs?.waitForSelector as string
@@ -158,23 +160,33 @@ class Playwright_DocumentLoaders implements INode {
}
return docs
} catch (err) {
if (process.env.DEBUG === 'true') console.error(`error in PlaywrightWebBaseLoader: ${err.message}, on page: ${url}`)
if (process.env.DEBUG === 'true') options.logger.error(`error in PlaywrightWebBaseLoader: ${err.message}, on page: ${url}`)
}
}
let docs = []
if (relativeLinksMethod) {
if (process.env.DEBUG === 'true') console.info(`Start ${relativeLinksMethod}`)
if (!limit) limit = '10'
else if (parseInt(limit) < 0) throw new Error('Limit cannot be less than 0')
if (process.env.DEBUG === 'true') options.logger.info(`Start ${relativeLinksMethod}`)
if (!limit) limit = 10
else if (limit < 0) throw new Error('Limit cannot be less than 0')
const pages: string[] =
relativeLinksMethod === 'webCrawl' ? await webCrawl(url, parseInt(limit)) : await xmlScrape(url, parseInt(limit))
if (process.env.DEBUG === 'true') console.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
selectedLinks && selectedLinks.length > 0
? selectedLinks.slice(0, limit)
: relativeLinksMethod === 'webCrawl'
? await webCrawl(url, limit)
: await xmlScrape(url, limit)
if (process.env.DEBUG === 'true') options.logger.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
if (!pages || pages.length === 0) throw new Error('No relative links found')
for (const page of pages) {
docs.push(...(await playwrightLoader(page)))
}
if (process.env.DEBUG === 'true') console.info(`Finish ${relativeLinksMethod}`)
if (process.env.DEBUG === 'true') options.logger.info(`Finish ${relativeLinksMethod}`)
} else if (selectedLinks && selectedLinks.length > 0) {
if (process.env.DEBUG === 'true')
options.logger.info(`pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`)
for (const page of selectedLinks) {
docs.push(...(await playwrightLoader(page)))
}
} else {
docs = await playwrightLoader(url)
}
packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts
+22 -10
View File
@@ -1,4 +1,4 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface'
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { TextSplitter } from 'langchain/text_splitter'
import { Browser, Page, PuppeteerWebBaseLoader, PuppeteerWebBaseLoaderOptions } from 'langchain/document_loaders/web/puppeteer'
import { test } from 'linkifyjs'
@@ -62,6 +62,7 @@ class Puppeteer_DocumentLoaders implements INode {
name: 'limit',
type: 'number',
optional: true,
default: '10',
additionalParams: true,
description:
'Only used when "Get Relative Links Method" is selected. Set 0 to retrieve all relative links, default limit is 10.',
@@ -115,11 +116,12 @@ class Puppeteer_DocumentLoaders implements INode {
]
}
async init(nodeData: INodeData): Promise<any> {
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
const metadata = nodeData.inputs?.metadata
const relativeLinksMethod = nodeData.inputs?.relativeLinksMethod as string
let limit = nodeData.inputs?.limit as string
const selectedLinks = nodeData.inputs?.selectedLinks as string[]
let limit = parseInt(nodeData.inputs?.limit as string)
let waitUntilGoToOption = nodeData.inputs?.waitUntilGoToOption as PuppeteerLifeCycleEvent
let waitForSelector = nodeData.inputs?.waitForSelector as string
@@ -159,23 +161,33 @@ class Puppeteer_DocumentLoaders implements INode {
}
return docs
} catch (err) {
if (process.env.DEBUG === 'true') console.error(`error in PuppeteerWebBaseLoader: ${err.message}, on page: ${url}`)
if (process.env.DEBUG === 'true') options.logger.error(`error in PuppeteerWebBaseLoader: ${err.message}, on page: ${url}`)
}
}
let docs = []
if (relativeLinksMethod) {
if (process.env.DEBUG === 'true') console.info(`Start ${relativeLinksMethod}`)
if (!limit) limit = '10'
else if (parseInt(limit) < 0) throw new Error('Limit cannot be less than 0')
if (process.env.DEBUG === 'true') options.logger.info(`Start ${relativeLinksMethod}`)
if (!limit) limit = 10
else if (limit < 0) throw new Error('Limit cannot be less than 0')
const pages: string[] =
relativeLinksMethod === 'webCrawl' ? await webCrawl(url, parseInt(limit)) : await xmlScrape(url, parseInt(limit))
if (process.env.DEBUG === 'true') console.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
selectedLinks && selectedLinks.length > 0
? selectedLinks.slice(0, limit)
: relativeLinksMethod === 'webCrawl'
? await webCrawl(url, limit)
: await xmlScrape(url, limit)
if (process.env.DEBUG === 'true') options.logger.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
if (!pages || pages.length === 0) throw new Error('No relative links found')
for (const page of pages) {
docs.push(...(await puppeteerLoader(page)))
}
if (process.env.DEBUG === 'true') console.info(`Finish ${relativeLinksMethod}`)
if (process.env.DEBUG === 'true') options.logger.info(`Finish ${relativeLinksMethod}`)
} else if (selectedLinks && selectedLinks.length > 0) {
if (process.env.DEBUG === 'true')
options.logger.info(`pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`)
for (const page of selectedLinks) {
docs.push(...(await puppeteerLoader(page)))
}
} else {
docs = await puppeteerLoader(url)
}
packages/components/nodes/embeddings/OpenAIEmbedding/OpenAIEmbedding.ts
+25 -2
View File
@@ -17,7 +17,7 @@ class OpenAIEmbedding_Embeddings implements INode {
constructor() {
this.label = 'OpenAI Embeddings'
this.name = 'openAIEmbeddings'
this.version = 1.0
this.version = 2.0
this.type = 'OpenAIEmbeddings'
this.icon = 'openai.svg'
this.category = 'Embeddings'
@@ -30,6 +30,27 @@ class OpenAIEmbedding_Embeddings implements INode {
credentialNames: ['openAIApi']
}
this.inputs = [
{
label: 'Model Name',
name: 'modelName',
type: 'options',
options: [
{
label: 'text-embedding-3-large',
name: 'text-embedding-3-large'
},
{
label: 'text-embedding-3-small',
name: 'text-embedding-3-small'
},
{
label: 'text-embedding-ada-002',
name: 'text-embedding-ada-002'
}
],
default: 'text-embedding-ada-002',
optional: true
},
{
label: 'Strip New Lines',
name: 'stripNewLines',
@@ -66,12 +87,14 @@ class OpenAIEmbedding_Embeddings implements INode {
const batchSize = nodeData.inputs?.batchSize as string
const timeout = nodeData.inputs?.timeout as string
const basePath = nodeData.inputs?.basepath as string
const modelName = nodeData.inputs?.modelName as string
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData)
const obj: Partial<OpenAIEmbeddingsParams> & { openAIApiKey?: string } = {
openAIApiKey
openAIApiKey,
modelName
}
if (stripNewLines) obj.stripNewLines = stripNewLines
packages/components/nodes/memory/MongoDBMemory/MongoDBMemory.ts
+19 -3
View File
@@ -5,6 +5,24 @@ import { mapStoredMessageToChatMessage, AIMessage, HumanMessage, BaseMessage } f
import { convertBaseMessagetoIMessage, getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { FlowiseMemory, ICommonObject, IMessage, INode, INodeData, INodeParams, MemoryMethods, MessageType } from '../../../src/Interface'
let mongoClientSingleton: MongoClient
let mongoUrl: string
const getMongoClient = async (newMongoUrl: string) => {
if (!mongoClientSingleton) {
// if client doesn't exists
mongoClientSingleton = new MongoClient(newMongoUrl)
mongoUrl = newMongoUrl
return mongoClientSingleton
} else if (mongoClientSingleton && newMongoUrl !== mongoUrl) {
// if client exists but url changed
mongoClientSingleton.close()
mongoClientSingleton = new MongoClient(newMongoUrl)
mongoUrl = newMongoUrl
return mongoClientSingleton
}
return mongoClientSingleton
}
class MongoDB_Memory implements INode {
label: string
name: string
@@ -79,9 +97,7 @@ const initializeMongoDB = async (nodeData: INodeData, options: ICommonObject): P
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const mongoDBConnectUrl = getCredentialParam('mongoDBConnectUrl', credentialData, nodeData)
const client = new MongoClient(mongoDBConnectUrl)
await client.connect()
const client = await getMongoClient(mongoDBConnectUrl)
const collection = client.db(databaseName).collection(collectionName)
const mongoDBChatMessageHistory = new MongoDBChatMessageHistory({
packages/components/nodes/memory/RedisBackedChatMemory/RedisBackedChatMemory.ts
+40 -21
View File
@@ -1,10 +1,47 @@
import { Redis } from 'ioredis'
import { Redis, RedisOptions } from 'ioredis'
import { isEqual } from 'lodash'
import { BufferMemory, BufferMemoryInput } from 'langchain/memory'
import { RedisChatMessageHistory, RedisChatMessageHistoryInput } from 'langchain/stores/message/ioredis'
import { mapStoredMessageToChatMessage, BaseMessage, AIMessage, HumanMessage } from 'langchain/schema'
import { INode, INodeData, INodeParams, ICommonObject, MessageType, IMessage, MemoryMethods, FlowiseMemory } from '../../../src/Interface'
import { convertBaseMessagetoIMessage, getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
let redisClientSingleton: Redis
let redisClientOption: RedisOptions
let redisClientUrl: string
const getRedisClientbyOption = (option: RedisOptions) => {
if (!redisClientSingleton) {
// if client doesn't exists
redisClientSingleton = new Redis(option)
redisClientOption = option
return redisClientSingleton
} else if (redisClientSingleton && !isEqual(option, redisClientOption)) {
// if client exists but option changed
redisClientSingleton.quit()
redisClientSingleton = new Redis(option)
redisClientOption = option
return redisClientSingleton
}
return redisClientSingleton
}
const getRedisClientbyUrl = (url: string) => {
if (!redisClientSingleton) {
// if client doesn't exists
redisClientSingleton = new Redis(url)
redisClientUrl = url
return redisClientSingleton
} else if (redisClientSingleton && url !== redisClientUrl) {
// if client exists but option changed
redisClientSingleton.quit()
redisClientSingleton = new Redis(url)
redisClientUrl = url
return redisClientSingleton
}
return redisClientSingleton
}
class RedisBackedChatMemory_Memory implements INode {
label: string
name: string
@@ -95,7 +132,7 @@ const initalizeRedis = async (nodeData: INodeData, options: ICommonObject): Prom
const tlsOptions = sslEnabled === true ? { tls: { rejectUnauthorized: false } } : {}
client = new Redis({
client = getRedisClientbyOption({
port: portStr ? parseInt(portStr) : 6379,
host,
username,
@@ -103,7 +140,7 @@ const initalizeRedis = async (nodeData: INodeData, options: ICommonObject): Prom
...tlsOptions
})
} else {
client = new Redis(redisUrl)
client = getRedisClientbyUrl(redisUrl)
}
let obj: RedisChatMessageHistoryInput = {
@@ -120,24 +157,6 @@ const initalizeRedis = async (nodeData: INodeData, options: ICommonObject): Prom
const redisChatMessageHistory = new RedisChatMessageHistory(obj)
/*redisChatMessageHistory.getMessages = async (): Promise<BaseMessage[]> => {
const rawStoredMessages = await client.lrange((redisChatMessageHistory as any).sessionId, windowSize ? -windowSize : 0, -1)
const orderedMessages = rawStoredMessages.reverse().map((message) => JSON.parse(message))
return orderedMessages.map(mapStoredMessageToChatMessage)
}
redisChatMessageHistory.addMessage = async (message: BaseMessage): Promise<void> => {
const messageToAdd = [message].map((msg) => msg.toDict())
await client.lpush((redisChatMessageHistory as any).sessionId, JSON.stringify(messageToAdd[0]))
if (sessionTTL) {
await client.expire((redisChatMessageHistory as any).sessionId, sessionTTL)
}
}
redisChatMessageHistory.clear = async (): Promise<void> => {
await client.del((redisChatMessageHistory as any).sessionId)
}*/
const memory = new BufferMemoryExtended({
memoryKey: memoryKey ?? 'chat_history',
chatHistory: redisChatMessageHistory,
packages/components/nodes/utilities/CustomFunction/CustomFunction.ts
+14 -1
View File
@@ -52,11 +52,19 @@ class CustomFunction_Utilities implements INode {
label: 'Output',
name: 'output',
baseClasses: ['string', 'number', 'boolean', 'json', 'array']
},
{
label: 'Ending Node',
name: 'EndingNode',
baseClasses: [this.type]
}
]
}
async init(nodeData: INodeData, input: string, options: ICommonObject): Promise<any> {
const isEndingNode = nodeData?.outputs?.output === 'EndingNode'
if (isEndingNode && !options.isRun) return // prevent running both init and run twice
const javascriptFunction = nodeData.inputs?.javascriptFunction as string
const functionInputVariablesRaw = nodeData.inputs?.functionInputVariables
const appDataSource = options.appDataSource as DataSource
@@ -123,7 +131,8 @@ class CustomFunction_Utilities implements INode {
const vm = new NodeVM(nodeVMOptions)
try {
const response = await vm.run(`module.exports = async function() {${javascriptFunction}}()`, __dirname)
if (typeof response === 'string') {
if (typeof response === 'string' && !isEndingNode) {
return handleEscapeCharacters(response, false)
}
return response
@@ -131,6 +140,10 @@ class CustomFunction_Utilities implements INode {
throw new Error(e)
}
}
async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string> {
return await this.init(nodeData, input, { ...options, isRun: true })
}
}
module.exports = { nodeClass: CustomFunction_Utilities }
packages/components/nodes/vectorstores/Pinecone/Pinecone.ts
-2
View File
@@ -105,7 +105,6 @@ class Pinecone_VectorStores implements INode {
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const pineconeApiKey = getCredentialParam('pineconeApiKey', credentialData, nodeData)
const pineconeEnv = getCredentialParam('pineconeEnv', credentialData, nodeData)
const client = new Pinecone({
apiKey: pineconeApiKey
@@ -144,7 +143,6 @@ class Pinecone_VectorStores implements INode {
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const pineconeApiKey = getCredentialParam('pineconeApiKey', credentialData, nodeData)
const pineconeEnv = getCredentialParam('pineconeEnv', credentialData, nodeData)
const client = new Pinecone({
apiKey: pineconeApiKey
packages/components/nodes/vectorstores/Qdrant/Qdrant.ts
+1 -1
View File
@@ -194,7 +194,7 @@ class Qdrant_VectorStores implements INode {
const qdrantVectorDimension = nodeData.inputs?.qdrantVectorDimension
const output = nodeData.outputs?.output as string
const topK = nodeData.inputs?.topK as string
let queryFilter = nodeData.inputs?.queryFilter
let queryFilter = nodeData.inputs?.qdrantFilter
const k = topK ? parseFloat(topK) : 4
packages/components/nodes/vectorstores/Qdrant/Qdrant_Existing.ts
+1 -1
View File
@@ -135,7 +135,7 @@ class Qdrant_Existing_VectorStores implements INode {
const qdrantVectorDimension = nodeData.inputs?.qdrantVectorDimension
const output = nodeData.outputs?.output as string
const topK = nodeData.inputs?.topK as string
let queryFilter = nodeData.inputs?.queryFilter
let queryFilter = nodeData.inputs?.qdrantFilter
const k = topK ? parseFloat(topK) : 4
packages/components/nodes/vectorstores/Redis/Redis.ts
+25 -6
View File
@@ -1,5 +1,5 @@
import { flatten } from 'lodash'
import { createClient, SearchOptions } from 'redis'
import { flatten, isEqual } from 'lodash'
import { createClient, SearchOptions, RedisClientOptions } from 'redis'
import { Embeddings } from 'langchain/embeddings/base'
import { RedisVectorStore, RedisVectorStoreConfig } from 'langchain/vectorstores/redis'
import { Document } from 'langchain/document'
@@ -7,6 +7,27 @@ import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { escapeAllStrings, escapeSpecialChars, unEscapeSpecialChars } from './utils'
let redisClientSingleton: ReturnType<typeof createClient>
let redisClientOption: RedisClientOptions
const getRedisClient = async (option: RedisClientOptions) => {
if (!redisClientSingleton) {
// if client doesn't exists
redisClientSingleton = createClient(option)
await redisClientSingleton.connect()
redisClientOption = option
return redisClientSingleton
} else if (redisClientSingleton && !isEqual(option, redisClientOption)) {
// if client exists but option changed
redisClientSingleton.quit()
redisClientSingleton = createClient(option)
await redisClientSingleton.connect()
redisClientOption = option
return redisClientSingleton
}
return redisClientSingleton
}
class Redis_VectorStores implements INode {
label: string
name: string
@@ -149,8 +170,7 @@ class Redis_VectorStores implements INode {
}
try {
const redisClient = createClient({ url: redisUrl })
await redisClient.connect()
const redisClient = await getRedisClient({ url: redisUrl })
const storeConfig: RedisVectorStoreConfig = {
redisClient: redisClient,
@@ -210,8 +230,7 @@ class Redis_VectorStores implements INode {
redisUrl = 'redis://' + username + ':' + password + '@' + host + ':' + portStr
}
const redisClient = createClient({ url: redisUrl })
await redisClient.connect()
const redisClient = await getRedisClient({ url: redisUrl })
const storeConfig: RedisVectorStoreConfig = {
redisClient: redisClient,
packages/components/nodes/vectorstores/Redis/RedisSearchBase.ts
+24 -4
View File
@@ -7,13 +7,34 @@ import {
INodeOutputsValue,
INodeParams
} from '../../../src'
import { Embeddings } from 'langchain/embeddings/base'
import { VectorStore } from 'langchain/vectorstores/base'
import { Document } from 'langchain/document'
import { createClient, SearchOptions } from 'redis'
import { createClient, SearchOptions, RedisClientOptions } from 'redis'
import { RedisVectorStore } from 'langchain/vectorstores/redis'
import { escapeSpecialChars, unEscapeSpecialChars } from './utils'
import { isEqual } from 'lodash'
let redisClientSingleton: ReturnType<typeof createClient>
let redisClientOption: RedisClientOptions
const getRedisClient = async (option: RedisClientOptions) => {
if (!redisClientSingleton) {
// if client doesn't exists
redisClientSingleton = createClient(option)
await redisClientSingleton.connect()
redisClientOption = option
return redisClientSingleton
} else if (redisClientSingleton && !isEqual(option, redisClientOption)) {
// if client exists but option changed
redisClientSingleton.quit()
redisClientSingleton = createClient(option)
await redisClientSingleton.connect()
redisClientOption = option
return redisClientSingleton
}
return redisClientSingleton
}
export abstract class RedisSearchBase {
label: string
@@ -141,8 +162,7 @@ export abstract class RedisSearchBase {
redisUrl = 'redis://' + username + ':' + password + '@' + host + ':' + portStr
}
this.redisClient = createClient({ url: redisUrl })
await this.redisClient.connect()
this.redisClient = await getRedisClient({ url: redisUrl })
const vectorStore = await this.constructVectorStore(embeddings, indexName, replaceIndex, docs)
if (!contentKey || contentKey === '') contentKey = 'content'
packages/components/nodes/vectorstores/Redis/Redis_Existing.ts
-1
View File
@@ -3,7 +3,6 @@ import { Embeddings } from 'langchain/embeddings/base'
import { VectorStore } from 'langchain/vectorstores/base'
import { RedisVectorStore, RedisVectorStoreConfig } from 'langchain/vectorstores/redis'
import { Document } from 'langchain/document'
import { RedisSearchBase } from './RedisSearchBase'
class RedisExisting_VectorStores extends RedisSearchBase implements INode {
packages/components/nodes/vectorstores/Redis/Redis_Upsert.ts
-1
View File
@@ -1,7 +1,6 @@
import { ICommonObject, INode, INodeData } from '../../../src/Interface'
import { Embeddings } from 'langchain/embeddings/base'
import { Document } from 'langchain/document'
import { flatten } from 'lodash'
import { RedisSearchBase } from './RedisSearchBase'
import { VectorStore } from 'langchain/vectorstores/base'