Add more nodes for agents, loaders

This commit is contained in:
Henry
2023-04-10 13:56:44 +01:00
parent 05c86ff9c5
commit 58e06718d1
57 changed files with 1584 additions and 89 deletions
@@ -0,0 +1,64 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface'
class ConversationalAgent_Agents implements INode {
label: string
name: string
description: string
type: string
icon: string
category: string
baseClasses: string[]
inputs: INodeParams[]
constructor() {
this.label = 'Conversational Agent'
this.name = 'conversationalAgent'
this.type = 'AgentExecutor'
this.category = 'Agents'
this.icon = 'agent.svg'
this.description = 'Conversational agent for a chat model. It will utilize chat specific prompts'
this.inputs = [
{
label: 'Allowed Tools',
name: 'tools',
type: 'Tool',
list: true
},
{
label: 'Chat Model',
name: 'model',
type: 'BaseChatModel'
},
{
label: 'Memory',
name: 'memory',
type: 'BaseChatMemory'
}
]
}
async getBaseClasses(): Promise<string[]> {
return ['AgentExecutor']
}
async init(nodeData: INodeData): Promise<any> {
const { initializeAgentExecutor } = await import('langchain/agents')
const model = nodeData.inputs?.model
const tools = nodeData.inputs?.tools
const memory = nodeData.inputs?.memory
const executor = await initializeAgentExecutor(tools, model, 'chat-conversational-react-description', true)
executor.memory = memory
return executor
}
async run(nodeData: INodeData, input: string): Promise<string> {
const executor = nodeData.instance
const result = await executor.call({ input })
return result?.output
}
}
module.exports = { nodeClass: ConversationalAgent_Agents }
@@ -0,0 +1,9 @@
<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-robot" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round">
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
<path d="M7 7h10a2 2 0 0 1 2 2v1l1 1v3l-1 1v3a2 2 0 0 1 -2 2h-10a2 2 0 0 1 -2 -2v-3l-1 -1v-3l1 -1v-1a2 2 0 0 1 2 -2z"></path>
<path d="M10 16h4"></path>
<circle cx="8.5" cy="11.5" r=".5" fill="currentColor"></circle>
<circle cx="15.5" cy="11.5" r=".5" fill="currentColor"></circle>
<path d="M9 7l-1 -4"></path>
<path d="M15 7l1 -4"></path>
</svg>

After

Width:  |  Height:  |  Size: 650 B

@@ -0,0 +1,58 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface'
class MRLKAgentChat_Agents implements INode {
label: string
name: string
description: string
type: string
icon: string
category: string
baseClasses: string[]
inputs: INodeParams[]
constructor() {
this.label = 'MRLK Agent for Chat Models'
this.name = 'mrlkAgentChat'
this.type = 'AgentExecutor'
this.category = 'Agents'
this.icon = 'agent.svg'
this.description = 'Agent that uses the ReAct Framework to decide what action to take, optimized to be used with Chat Models'
this.inputs = [
{
label: 'Allowed Tools',
name: 'tools',
type: 'Tool',
list: true
},
{
label: 'Chat Model',
name: 'model',
type: 'BaseChatModel'
}
]
}
async getBaseClasses(): Promise<string[]> {
return ['AgentExecutor']
}
async init(nodeData: INodeData): Promise<any> {
const { initializeAgentExecutor } = await import('langchain/agents')
const model = nodeData.inputs?.model
const tools = nodeData.inputs?.tools
const executor = await initializeAgentExecutor(tools, model, 'chat-zero-shot-react-description', true)
return executor
}
async run(nodeData: INodeData, input: string): Promise<string> {
const executor = nodeData.instance
const result = await executor.call({ input })
return result?.output
}
}
module.exports = { nodeClass: MRLKAgentChat_Agents }
@@ -0,0 +1,9 @@
<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-robot" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round">
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
<path d="M7 7h10a2 2 0 0 1 2 2v1l1 1v3l-1 1v3a2 2 0 0 1 -2 2h-10a2 2 0 0 1 -2 -2v-3l-1 -1v-3l1 -1v-1a2 2 0 0 1 2 -2z"></path>
<path d="M10 16h4"></path>
<circle cx="8.5" cy="11.5" r=".5" fill="currentColor"></circle>
<circle cx="15.5" cy="11.5" r=".5" fill="currentColor"></circle>
<path d="M9 7l-1 -4"></path>
<path d="M15 7l1 -4"></path>
</svg>

After

Width:  |  Height:  |  Size: 650 B

@@ -1,6 +1,6 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface'
class MRLKAgentLLM implements INode {
class MRLKAgentLLM_Agents implements INode {
label: string
name: string
description: string
@@ -55,4 +55,4 @@ class MRLKAgentLLM implements INode {
}
}
module.exports = { nodeClass: MRLKAgentLLM }
module.exports = { nodeClass: MRLKAgentLLM_Agents }
@@ -0,0 +1,74 @@
import { ICommonObject, IMessage, INode, INodeData, INodeParams } from '../../../src/Interface'
import { getBaseClasses } from '../../../src/utils'
class ConversationalRetrievalQAChain_Chains implements INode {
label: string
name: string
type: string
icon: string
category: string
baseClasses: string[]
description: string
inputs: INodeParams[]
constructor() {
this.label = 'Conversational Retrieval QA Chain'
this.name = 'conversationalRetrievalQAChain'
this.type = 'ConversationalRetrievalQAChain'
this.icon = 'chain.svg'
this.category = 'Chains'
this.description = 'Document QA - built on RetrievalQAChain to provide a chat history component'
this.inputs = [
{
label: 'LLM',
name: 'llm',
type: 'BaseLanguageModel'
},
{
label: 'Vector Store Retriever',
name: 'vectorStoreRetriever',
type: 'BaseRetriever'
}
]
}
async getBaseClasses(): Promise<string[]> {
const { ConversationalRetrievalQAChain } = await import('langchain/chains')
return getBaseClasses(ConversationalRetrievalQAChain)
}
async init(nodeData: INodeData): Promise<any> {
const { ConversationalRetrievalQAChain } = await import('langchain/chains')
const llm = nodeData.inputs?.llm
const vectorStoreRetriever = nodeData.inputs?.vectorStoreRetriever
const chain = ConversationalRetrievalQAChain.fromLLM(llm, vectorStoreRetriever)
return chain
}
async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string> {
const chain = nodeData.instance
let chatHistory = ''
if (options && options.chatHistory) {
const histories: IMessage[] = options.chatHistory
chatHistory = histories
.map((item) => {
return item.message
})
.join('')
}
const obj = {
question: input,
chat_history: chatHistory ? chatHistory : []
}
const res = await chain.call(obj)
return res?.text
}
}
module.exports = { nodeClass: ConversationalRetrievalQAChain_Chains }
@@ -0,0 +1,6 @@
<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-dna" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round">
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
<path d="M14.828 14.828a4 4 0 1 0 -5.656 -5.656a4 4 0 0 0 5.656 5.656z"></path>
<path d="M9.172 20.485a4 4 0 1 0 -5.657 -5.657"></path>
<path d="M14.828 3.515a4 4 0 0 0 5.657 5.657"></path>
</svg>

After

Width:  |  Height:  |  Size: 489 B

@@ -28,6 +28,17 @@ class LLMChain_Chains implements INode {
label: 'Prompt',
name: 'prompt',
type: 'BasePromptTemplate'
},
{
label: 'Format Prompt Values',
name: 'promptValues',
type: 'string',
rows: 5,
placeholder: `{
"input_language": "English",
"output_language": "French"
}`,
optional: true
}
]
}
@@ -48,13 +59,39 @@ class LLMChain_Chains implements INode {
}
async run(nodeData: INodeData, input: string): Promise<string> {
const prompt = nodeData.instance.prompt.inputVariables // ["product"]
if (prompt.length > 1) throw new Error('Prompt can only contains 1 literal string {}. Multiples are found')
const inputVariables = nodeData.instance.prompt.inputVariables // ["product"]
const chain = nodeData.instance
const res = await chain.run(input)
return res
if (inputVariables.length === 1) {
const res = await chain.run(input)
return res
} else if (inputVariables.length > 1) {
const promptValuesStr = nodeData.inputs?.promptValues as string
if (!promptValuesStr) throw new Error('Please provide Prompt Values')
const promptValues = JSON.parse(promptValuesStr.replace(/\s/g, ''))
let seen = []
for (const variable of inputVariables) {
seen.push(variable)
if (promptValues[variable]) {
seen.pop()
}
}
if (seen.length === 1) {
const options = {
...promptValues,
[seen.pop()]: input
}
const res = await chain.call(options)
return res?.text
} else throw new Error('Please provide Prompt Values')
} else {
const res = await chain.run(input)
return res
}
}
}
@@ -0,0 +1,57 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface'
class RetrievalQAChain_Chains implements INode {
label: string
name: string
type: string
icon: string
category: string
baseClasses: string[]
description: string
inputs: INodeParams[]
constructor() {
this.label = 'RetrievalQA Chain'
this.name = 'retrievalQAChain'
this.type = 'RetrievalQAChain'
this.icon = 'chain.svg'
this.category = 'Chains'
this.description = 'QA chain to answer a question based on the retrieved documents'
this.inputs = [
{
label: 'LLM',
name: 'llm',
type: 'BaseLanguageModel'
},
{
label: 'Vector Store Retriever',
name: 'vectorStoreRetriever',
type: 'BaseRetriever'
}
]
}
async getBaseClasses(): Promise<string[]> {
return ['BaseChain']
}
async init(nodeData: INodeData): Promise<any> {
const { RetrievalQAChain } = await import('langchain/chains')
const llm = nodeData.inputs?.llm
const vectorStoreRetriever = nodeData.inputs?.vectorStoreRetriever
const chain = RetrievalQAChain.fromLLM(llm, vectorStoreRetriever)
return chain
}
async run(nodeData: INodeData, input: string): Promise<string> {
const chain = nodeData.instance
const obj = {
query: input
}
const res = await chain.call(obj)
return res?.text
}
}
module.exports = { nodeClass: RetrievalQAChain_Chains }
@@ -0,0 +1,6 @@
<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-dna" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round">
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
<path d="M14.828 14.828a4 4 0 1 0 -5.656 -5.656a4 4 0 0 0 5.656 5.656z"></path>
<path d="M9.172 20.485a4 4 0 1 0 -5.657 -5.657"></path>
<path d="M14.828 3.515a4 4 0 0 0 5.657 5.657"></path>
</svg>

After

Width:  |  Height:  |  Size: 489 B

@@ -0,0 +1,75 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface'
import { getBaseClasses } from '../../../src/utils'
class ChatOpenAI_ChatModels implements INode {
label: string
name: string
type: string
icon: string
category: string
description: string
baseClasses: string[]
inputs: INodeParams[]
constructor() {
this.label = 'ChatOpenAI'
this.name = 'chatOpenAI'
this.type = 'ChatOpenAI'
this.icon = 'openai.png'
this.category = 'Chat Models'
this.description = 'Wrapper around OpenAI large language models that use the Chat endpoint'
this.inputs = [
{
label: 'OpenAI Api Key',
name: 'openAIApiKey',
type: 'password'
},
{
label: 'Model Name',
name: 'modelName',
type: 'options',
options: [
{
label: 'gpt-3.5-turbo',
name: 'gpt-3.5-turbo'
},
{
label: 'gpt-3.5-turbo-0301',
name: 'gpt-3.5-turbo-0301'
}
],
default: 'gpt-3.5-turbo',
optional: true
},
{
label: 'Temperature',
name: 'temperature',
type: 'number',
default: 0.9,
optional: true
}
]
}
async getBaseClasses(): Promise<string[]> {
const { ChatOpenAI } = await import('langchain/chat_models')
return getBaseClasses(ChatOpenAI)
}
async init(nodeData: INodeData): Promise<any> {
const { ChatOpenAI } = await import('langchain/chat_models')
const temperature = nodeData.inputs?.temperature as string
const modelName = nodeData.inputs?.modelName as string
const openAIApiKey = nodeData.inputs?.openAIApiKey as string
const model = new ChatOpenAI({
temperature: parseInt(temperature, 10),
modelName,
openAIApiKey
})
return model
}
}
module.exports = { nodeClass: ChatOpenAI_ChatModels }
@@ -0,0 +1,81 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface'
class Github_DocumentLoaders implements INode {
label: string
name: string
description: string
type: string
icon: string
category: string
baseClasses: string[]
inputs: INodeParams[]
constructor() {
this.label = 'Github'
this.name = 'github'
this.type = 'Github'
this.icon = 'github.png'
this.category = 'Document Loaders'
this.description = `Load data from a GitHub repository`
this.inputs = [
{
label: 'Repo Link',
name: 'repoLink',
type: 'string',
placeholder: 'https://github.com/FlowiseAI/Flowise'
},
{
label: 'Branch',
name: 'branch',
type: 'string',
default: 'main'
},
{
label: 'Access Token',
name: 'accessToken',
type: 'password',
placeholder: '<GITHUB_ACCESS_TOKEN>',
optional: true
},
{
label: 'Text Splitter',
name: 'textSplitter',
type: 'TextSplitter',
optional: true
}
]
}
async getBaseClasses(): Promise<string[]> {
return ['Document']
}
async init(nodeData: INodeData): Promise<any> {
const { GithubRepoLoader } = await import('langchain/document_loaders')
const repoLink = nodeData.inputs?.repoLink as string
const branch = nodeData.inputs?.branch as string
const accessToken = nodeData.inputs?.accessToken as string
const textSplitter = nodeData.inputs?.textSplitter
const options = {
branch,
recursive: false,
unknown: 'warn'
} as any
if (accessToken) options.accessToken = accessToken
const loader = new GithubRepoLoader(repoLink, options)
if (textSplitter) {
const docs = await loader.loadAndSplit(textSplitter)
return docs
} else {
const docs = await loader.load()
return docs
}
}
}
module.exports = { nodeClass: Github_DocumentLoaders }
Binary file not shown.

After

Width:  |  Height:  |  Size: 16 KiB

@@ -0,0 +1,90 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface'
class Pdf_DocumentLoaders implements INode {
label: string
name: string
description: string
type: string
icon: string
category: string
baseClasses: string[]
inputs: INodeParams[]
constructor() {
this.label = 'Pdf File'
this.name = 'pdfFile'
this.type = 'PDF'
this.icon = 'pdf.svg'
this.category = 'Document Loaders'
this.description = `Load data from PDF files`
this.inputs = [
{
label: 'Pdf File',
name: 'pdfFile',
type: 'file',
fileType: '.pdf'
},
{
label: 'Text Splitter',
name: 'textSplitter',
type: 'TextSplitter',
optional: true
},
{
label: 'Usage',
name: 'usage',
type: 'options',
options: [
{
label: 'One document per page',
name: 'perPage'
},
{
label: 'One document per file',
name: 'perFile'
}
],
default: 'perPage'
}
]
}
async getBaseClasses(): Promise<string[]> {
return ['Document']
}
async init(nodeData: INodeData): Promise<any> {
const { PDFLoader } = await import('langchain/document_loaders')
const textSplitter = nodeData.inputs?.textSplitter
const pdfFileBase64 = nodeData.inputs?.pdfFile as string
const usage = nodeData.inputs?.usage as string
const splitDataURI = pdfFileBase64.split(',')
splitDataURI.pop()
const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
const blob = new Blob([bf])
if (usage === 'perFile') {
const loader = new PDFLoader(blob, { splitPages: false })
if (textSplitter) {
const docs = await loader.loadAndSplit(textSplitter)
return docs
} else {
const docs = await loader.load()
return docs
}
} else {
const loader = new PDFLoader(blob)
if (textSplitter) {
const docs = await loader.loadAndSplit(textSplitter)
return docs
} else {
const docs = await loader.load()
return docs
}
}
}
}
module.exports = { nodeClass: Pdf_DocumentLoaders }
@@ -0,0 +1,7 @@
<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-pdf" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round">
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
<path d="M10 8v8h2a2 2 0 0 0 2 -2v-4a2 2 0 0 0 -2 -2h-2z"></path>
<path d="M3 12h2a2 2 0 1 0 0 -4h-2v8"></path>
<path d="M17 12h3"></path>
<path d="M21 8h-4v8"></path>
</svg>

After

Width:  |  Height:  |  Size: 470 B

@@ -0,0 +1,61 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface'
class Text_DocumentLoaders implements INode {
label: string
name: string
description: string
type: string
icon: string
category: string
baseClasses: string[]
inputs: INodeParams[]
constructor() {
this.label = 'Text File'
this.name = 'textFile'
this.type = 'Text'
this.icon = 'textFile.svg'
this.category = 'Document Loaders'
this.description = `Load data from text files`
this.inputs = [
{
label: 'Txt File',
name: 'txtFile',
type: 'file',
fileType: '.txt'
},
{
label: 'Text Splitter',
name: 'textSplitter',
type: 'TextSplitter',
optional: true
}
]
}
async getBaseClasses(): Promise<string[]> {
return ['Document']
}
async init(nodeData: INodeData): Promise<any> {
const { TextLoader } = await import('langchain/document_loaders')
const textSplitter = nodeData.inputs?.textSplitter
const txtFileBase64 = nodeData.inputs?.txtFile as string
const splitDataURI = txtFileBase64.split(',')
splitDataURI.pop()
const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
const blob = new Blob([bf])
const loader = new TextLoader(blob)
if (textSplitter) {
const docs = await loader.loadAndSplit(textSplitter)
return docs
} else {
const docs = await loader.load()
return docs
}
}
}
module.exports = { nodeClass: Text_DocumentLoaders }
@@ -0,0 +1,7 @@
<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-clipboard-text" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round">
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
<path d="M9 5h-2a2 2 0 0 0 -2 2v12a2 2 0 0 0 2 2h10a2 2 0 0 0 2 -2v-12a2 2 0 0 0 -2 -2h-2"></path>
<path d="M9 3m0 2a2 2 0 0 1 2 -2h2a2 2 0 0 1 2 2v0a2 2 0 0 1 -2 2h-2a2 2 0 0 1 -2 -2z"></path>
<path d="M9 12h6"></path>
<path d="M9 16h6"></path>
</svg>

After

Width:  |  Height:  |  Size: 559 B

@@ -0,0 +1,44 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface'
import { getBaseClasses } from '../../../src/utils'
class OpenAIEmbedding_Embeddings implements INode {
label: string
name: string
type: string
icon: string
category: string
description: string
baseClasses: string[]
inputs: INodeParams[]
constructor() {
this.label = 'OpenAI Embeddings'
this.name = 'openAIEmbeddings'
this.type = 'OpenAIEmbeddings'
this.icon = 'openai.png'
this.category = 'Embeddings'
this.description = 'OpenAI API to generate embeddings for a given text'
this.inputs = [
{
label: 'OpenAI Api Key',
name: 'openAIApiKey',
type: 'password'
}
]
}
async getBaseClasses(): Promise<string[]> {
const { OpenAIEmbeddings } = await import('langchain/embeddings')
return getBaseClasses(OpenAIEmbeddings)
}
async init(nodeData: INodeData): Promise<any> {
const { OpenAIEmbeddings } = await import('langchain/embeddings')
const openAIApiKey = nodeData.inputs?.openAIApiKey as string
const model = new OpenAIEmbeddings({ openAIApiKey })
return model
}
}
module.exports = { nodeClass: OpenAIEmbedding_Embeddings }
Binary file not shown.

After

Width:  |  Height:  |  Size: 3.9 KiB

@@ -0,0 +1,64 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface'
import { getBaseClasses } from '../../../src/utils'
class HuggingFaceInference_LLMs implements INode {
label: string
name: string
type: string
icon: string
category: string
description: string
baseClasses: string[]
inputs: INodeParams[]
constructor() {
this.label = 'HuggingFace Inference'
this.name = 'huggingFaceInference_LLMs'
this.type = 'HuggingFaceInference'
this.icon = 'huggingface.png'
this.category = 'LLMs'
this.description = 'Wrapper around OpenAI large language models'
this.inputs = [
{
label: 'Model',
name: 'model',
type: 'options',
options: [
{
label: 'gpt2',
name: 'gpt2'
}
],
default: 'gpt2',
optional: true
},
{
label: 'Temperature',
name: 'temperature',
type: 'number',
default: 0.7,
optional: true
}
]
}
async getBaseClasses(): Promise<string[]> {
const { HuggingFaceInference } = await import('langchain/llms')
return getBaseClasses(HuggingFaceInference)
}
async init(nodeData: INodeData): Promise<any> {
const { HuggingFaceInference } = await import('langchain/llms')
const temperature = nodeData.inputs?.temperature as string
const model = nodeData.inputs?.model as string
const huggingFace = new HuggingFaceInference({
temperature: parseInt(temperature, 10),
model
})
return huggingFace
}
}
module.exports = { nodeClass: HuggingFaceInference_LLMs }
Binary file not shown.

After

Width:  |  Height:  |  Size: 118 KiB

@@ -0,0 +1,54 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface'
import { getBaseClasses } from '../../../src/utils'
class BufferMemory_Memory implements INode {
label: string
name: string
description: string
type: string
icon: string
category: string
baseClasses: string[]
inputs: INodeParams[]
constructor() {
this.label = 'Buffer Memory'
this.name = 'bufferMemory'
this.type = 'BufferMemory'
this.icon = 'memory.svg'
this.category = 'Memory'
this.description = 'Perform calculations on response'
this.inputs = [
{
label: 'Memory Key',
name: 'memoryKey',
type: 'string',
default: 'chat_history'
},
{
label: 'Input Key',
name: 'inputKey',
type: 'string',
default: 'input'
}
]
}
async getBaseClasses(): Promise<string[]> {
const { BufferMemory } = await import('langchain/memory')
return getBaseClasses(BufferMemory)
}
async init(nodeData: INodeData): Promise<any> {
const { BufferMemory } = await import('langchain/memory')
const memoryKey = nodeData.inputs?.memoryKey as string
const inputKey = nodeData.inputs?.inputKey as string
return new BufferMemory({
returnMessages: true,
memoryKey,
inputKey
})
}
}
module.exports = { nodeClass: BufferMemory_Memory }
@@ -0,0 +1,8 @@
<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-book" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round">
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
<path d="M3 19a9 9 0 0 1 9 0a9 9 0 0 1 9 0"></path>
<path d="M3 6a9 9 0 0 1 9 0a9 9 0 0 1 9 0"></path>
<path d="M3 6l0 13"></path>
<path d="M12 6l0 13"></path>
<path d="M21 6l0 13"></path>
</svg>

After

Width:  |  Height:  |  Size: 495 B

@@ -0,0 +1,57 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface'
import { getBaseClasses } from '../../../src/utils'
class ChatPromptTemplate_Prompts implements INode {
label: string
name: string
description: string
type: string
icon: string
category: string
baseClasses: string[]
inputs: INodeParams[]
constructor() {
this.label = 'Chat Prompt Template'
this.name = 'chatPromptTemplate'
this.type = 'ChatPromptTemplate'
this.icon = 'prompt.svg'
this.category = 'Prompts'
this.description = 'Schema to represent a chat prompt'
this.inputs = [
{
label: 'System Message',
name: 'systemMessagePrompt',
type: 'string',
rows: 3,
placeholder: `You are a helpful assistant that translates {input_language} to {output_language}.`
},
{
label: 'Human Message',
name: 'humanMessagePrompt',
type: 'string',
rows: 3,
placeholder: `{text}`
}
]
}
async getBaseClasses(): Promise<string[]> {
const { ChatPromptTemplate } = await import('langchain/prompts')
return getBaseClasses(ChatPromptTemplate)
}
async init(nodeData: INodeData): Promise<any> {
const { ChatPromptTemplate, SystemMessagePromptTemplate, HumanMessagePromptTemplate } = await import('langchain/prompts')
const systemMessagePrompt = nodeData.inputs?.systemMessagePrompt as string
const humanMessagePrompt = nodeData.inputs?.humanMessagePrompt as string
const prompt = ChatPromptTemplate.fromPromptMessages([
SystemMessagePromptTemplate.fromTemplate(systemMessagePrompt),
HumanMessagePromptTemplate.fromTemplate(humanMessagePrompt)
])
return prompt
}
}
module.exports = { nodeClass: ChatPromptTemplate_Prompts }
@@ -0,0 +1,6 @@
<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-terminal-2" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round">
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
<path d="M8 9l3 3l-3 3"></path>
<path d="M13 15l3 0"></path>
<path d="M3 4m0 2a2 2 0 0 1 2 -2h14a2 2 0 0 1 2 2v12a2 2 0 0 1 -2 2h-14a2 2 0 0 1 -2 -2z"></path>
</svg>

After

Width:  |  Height:  |  Size: 465 B

@@ -0,0 +1,112 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface'
import { getBaseClasses, getInputVariables } from '../../../src/utils'
class FewShotPromptTemplate_Prompts implements INode {
label: string
name: string
description: string
type: string
icon: string
category: string
baseClasses: string[]
inputs: INodeParams[]
constructor() {
this.label = 'Few Shot Prompt Template'
this.name = 'fewShotPromptTemplate'
this.type = 'FewShotPromptTemplate'
this.icon = 'prompt.svg'
this.category = 'Prompts'
this.description = 'Prompt template you can build with examples'
this.inputs = [
{
label: 'Examples',
name: 'examples',
type: 'string',
rows: 5,
placeholder: `[
{ "word": "happy", "antonym": "sad" },
{ "word": "tall", "antonym": "short" },
]`
},
{
label: 'Example Prompt',
name: 'examplePrompt',
type: 'BasePromptTemplate'
},
{
label: 'Prefix',
name: 'prefix',
type: 'string',
rows: 3,
placeholder: `Give the antonym of every input`
},
{
label: 'Suffix',
name: 'suffix',
type: 'string',
rows: 3,
placeholder: `Word: {input}\nAntonym:`
},
{
label: 'Example Seperator',
name: 'exampleSeparator',
type: 'string',
placeholder: `\n\n`
},
{
label: 'Template Format',
name: 'templateFormat',
type: 'options',
options: [
{
label: 'f-string',
name: 'f-string'
},
{
label: 'jinja-2',
name: 'jinja-2'
}
],
default: `f-string`
}
]
}
async getBaseClasses(): Promise<string[]> {
const { FewShotPromptTemplate } = await import('langchain/prompts')
return getBaseClasses(FewShotPromptTemplate)
}
async init(nodeData: INodeData): Promise<any> {
const { FewShotPromptTemplate } = await import('langchain/prompts')
const examplesStr = nodeData.inputs?.examples as string
const prefix = nodeData.inputs?.prefix as string
const suffix = nodeData.inputs?.suffix as string
const exampleSeparator = nodeData.inputs?.exampleSeparator as string
const templateFormat = nodeData.inputs?.templateFormat
const examplePrompt = nodeData.inputs?.examplePrompt
const inputVariables = getInputVariables(suffix)
const examples = JSON.parse(examplesStr.replace(/\s/g, ''))
try {
const prompt = new FewShotPromptTemplate({
examples,
examplePrompt,
prefix,
suffix,
inputVariables,
exampleSeparator,
templateFormat
})
return prompt
} catch (e) {
throw new Error(e)
}
}
}
module.exports = { nodeClass: FewShotPromptTemplate_Prompts }
@@ -0,0 +1,6 @@
<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-terminal-2" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round">
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
<path d="M8 9l3 3l-3 3"></path>
<path d="M13 15l3 0"></path>
<path d="M3 4m0 2a2 2 0 0 1 2 -2h14a2 2 0 0 1 2 2v12a2 2 0 0 1 -2 2h-14a2 2 0 0 1 -2 -2z"></path>
</svg>

After

Width:  |  Height:  |  Size: 465 B

@@ -17,15 +17,14 @@ class PromptTemplate_Prompts implements INode {
this.type = 'PromptTemplate'
this.icon = 'prompt.svg'
this.category = 'Prompts'
this.description = 'Schema to represent a basic prompt for an LLM. Template can only contains 1 literal string {}'
this.description = 'Schema to represent a basic prompt for an LLM'
this.inputs = [
{
label: 'Template',
name: 'template',
type: 'string',
rows: 5,
default: 'What is a good name for a company that makes {product}?',
placeholder: 'What is a good name for a company that makes {product}?'
placeholder: `What is a good name for a company that makes {product}?`
}
]
}
@@ -42,10 +41,11 @@ class PromptTemplate_Prompts implements INode {
const inputVariables = getInputVariables(template)
try {
const prompt = new PromptTemplate({
const options = {
template,
inputVariables: inputVariables
})
inputVariables
}
const prompt = new PromptTemplate(options)
return prompt
} catch (e) {
throw new Error(e)
@@ -0,0 +1,59 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface'
import { getBaseClasses } from '../../../src/utils'
class RecursiveCharacterTextSplitter_TextSplitters implements INode {
label: string
name: string
description: string
type: string
icon: string
category: string
baseClasses: string[]
inputs: INodeParams[]
constructor() {
this.label = 'Recursive Character Text Splitter'
this.name = 'recursiveCharacterTextSplitter'
this.type = 'RecursiveCharacterTextSplitter'
this.icon = 'textsplitter.svg'
this.category = 'Text Splitters'
this.description = `Split documents recursively by different characters - starting with "\n\n", then "\n", then " "`
this.inputs = [
{
label: 'Chunk Size',
name: 'chunkSize',
type: 'number',
default: 1000,
optional: true
},
{
label: 'Chunk Overlap',
name: 'chunkOverlap',
type: 'number',
optional: true
}
]
}
async getBaseClasses(): Promise<string[]> {
const { RecursiveCharacterTextSplitter } = await import('langchain/text_splitter')
return getBaseClasses(RecursiveCharacterTextSplitter)
}
async init(nodeData: INodeData): Promise<any> {
const { RecursiveCharacterTextSplitter } = await import('langchain/text_splitter')
const chunkSize = nodeData.inputs?.chunkSize as string
const chunkOverlap = nodeData.inputs?.chunkOverlap as string
const obj = {} as any
if (chunkSize) obj.chunkSize = parseInt(chunkSize, 10)
if (chunkOverlap) obj.chunkOverlap = parseInt(chunkOverlap, 10)
const splitter = new RecursiveCharacterTextSplitter(obj)
return splitter
}
}
module.exports = { nodeClass: RecursiveCharacterTextSplitter_TextSplitters }
@@ -0,0 +1,7 @@
<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-abc" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round">
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
<path d="M3 16v-6a2 2 0 1 1 4 0v6"></path>
<path d="M3 13h4"></path>
<path d="M10 8v6a2 2 0 1 0 4 0v-1a2 2 0 1 0 -4 0v1"></path>
<path d="M20.732 12a2 2 0 0 0 -3.732 1v1a2 2 0 0 0 3.726 1.01"></path>
</svg>

After

Width:  |  Height:  |  Size: 502 B

@@ -0,0 +1,52 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface'
class Chroma_Existing_VectorStores implements INode {
label: string
name: string
description: string
type: string
icon: string
category: string
baseClasses: string[]
inputs: INodeParams[]
constructor() {
this.label = 'Chroma Load Existing Index'
this.name = 'chromaExistingIndex'
this.type = 'Chroma'
this.icon = 'chroma.svg'
this.category = 'Vector Stores'
this.description = 'Load existing index from Chroma (i.e: Document has been upserted)'
this.inputs = [
{
label: 'Embeddings',
name: 'embeddings',
type: 'Embeddings'
},
{
label: 'Collection Name',
name: 'collectionName',
type: 'string'
}
]
}
async getBaseClasses(): Promise<string[]> {
return ['BaseRetriever']
}
async init(nodeData: INodeData): Promise<any> {
const { Chroma } = await import('langchain/vectorstores')
const collectionName = nodeData.inputs?.collectionName as string
const embeddings = nodeData.inputs?.embeddings
const vectorStore = await Chroma.fromExistingCollection(embeddings, {
collectionName
})
const retriever = vectorStore.asRetriever()
return retriever
}
}
module.exports = { nodeClass: Chroma_Existing_VectorStores }
@@ -0,0 +1,7 @@
<svg width="209" height="135" viewBox="0 0 209 135" fill="none" xmlns="http://www.w3.org/2000/svg">
<ellipse cx="136.019" cy="67.2304" rx="66.6667" ry="64" fill="#FFDE2D"/>
<ellipse cx="69.352" cy="67.2304" rx="66.6667" ry="64" fill="#327EFF"/>
<path d="M2.68528 67.2304C2.68527 31.8842 32.5329 3.23047 69.3519 3.23047L69.3519 67.2304L2.68528 67.2304Z" fill="#327EFF"/>
<path d="M136.019 67.2305C136.019 102.577 106.171 131.23 69.3519 131.23L69.3519 67.2305L136.019 67.2305Z" fill="#FF6446"/>
<path d="M69.352 67.2304C69.352 31.8842 99.1997 3.23047 136.019 3.23047L136.019 67.2304L69.352 67.2304Z" fill="#FF6446"/>
</svg>

After

Width:  |  Height:  |  Size: 622 B

@@ -0,0 +1,65 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface'
class ChromaUpsert_VectorStores implements INode {
label: string
name: string
description: string
type: string
icon: string
category: string
baseClasses: string[]
inputs: INodeParams[]
constructor() {
this.label = 'Chroma Upsert Document'
this.name = 'chromaUpsert'
this.type = 'Chroma'
this.icon = 'chroma.svg'
this.category = 'Vector Stores'
this.description = 'Upsert documents to Chroma'
this.inputs = [
{
label: 'Document',
name: 'document',
type: 'Document'
},
{
label: 'Embeddings',
name: 'embeddings',
type: 'Embeddings'
},
{
label: 'Collection Name',
name: 'collectionName',
type: 'string'
}
]
}
async getBaseClasses(): Promise<string[]> {
return ['BaseRetriever']
}
async init(nodeData: INodeData): Promise<any> {
const { Chroma } = await import('langchain/vectorstores')
const { Document } = await import('langchain/document')
const collectionName = nodeData.inputs?.collectionName as string
const docs = nodeData.inputs?.document
const embeddings = nodeData.inputs?.embeddings
const finalDocs = []
for (let i = 0; i < docs.length; i += 1) {
finalDocs.push(new Document(docs[i]))
}
const result = await Chroma.fromDocuments(finalDocs, embeddings, {
collectionName
})
const retriever = result.asRetriever()
return retriever
}
}
module.exports = { nodeClass: ChromaUpsert_VectorStores }
@@ -0,0 +1,7 @@
<svg width="209" height="135" viewBox="0 0 209 135" fill="none" xmlns="http://www.w3.org/2000/svg">
<ellipse cx="136.019" cy="67.2304" rx="66.6667" ry="64" fill="#FFDE2D"/>
<ellipse cx="69.352" cy="67.2304" rx="66.6667" ry="64" fill="#327EFF"/>
<path d="M2.68528 67.2304C2.68527 31.8842 32.5329 3.23047 69.3519 3.23047L69.3519 67.2304L2.68528 67.2304Z" fill="#327EFF"/>
<path d="M136.019 67.2305C136.019 102.577 106.171 131.23 69.3519 131.23L69.3519 67.2305L136.019 67.2305Z" fill="#FF6446"/>
<path d="M69.352 67.2304C69.352 31.8842 99.1997 3.23047 136.019 3.23047L136.019 67.2304L69.352 67.2304Z" fill="#FF6446"/>
</svg>

After

Width:  |  Height:  |  Size: 622 B

@@ -0,0 +1,73 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface'
import { PineconeClient } from '@pinecone-database/pinecone'
class Pinecone_Existing_VectorStores implements INode {
label: string
name: string
description: string
type: string
icon: string
category: string
baseClasses: string[]
inputs: INodeParams[]
constructor() {
this.label = 'Pinecone Load Existing Index'
this.name = 'pineconeExistingIndex'
this.type = 'Pinecone'
this.icon = 'pinecone.png'
this.category = 'Vector Stores'
this.description = 'Load existing index from Pinecone (i.e: Document has been upserted)'
this.inputs = [
{
label: 'Embeddings',
name: 'embeddings',
type: 'Embeddings'
},
{
label: 'Pinecone Api Key',
name: 'pineconeApiKey',
type: 'password'
},
{
label: 'Pinecone Environment',
name: 'pineconeEnv',
type: 'string'
},
{
label: 'Pinecone Index',
name: 'pineconeIndex',
type: 'string'
}
]
}
async getBaseClasses(): Promise<string[]> {
return ['BaseRetriever']
}
async init(nodeData: INodeData): Promise<any> {
const { PineconeStore } = await import('langchain/vectorstores')
const pineconeApiKey = nodeData.inputs?.pineconeApiKey as string
const pineconeEnv = nodeData.inputs?.pineconeEnv as string
const index = nodeData.inputs?.pineconeIndex as string
const embeddings = nodeData.inputs?.embeddings
const client = new PineconeClient()
await client.init({
apiKey: pineconeApiKey,
environment: pineconeEnv
})
const pineconeIndex = client.Index(index)
const vectorStore = await PineconeStore.fromExistingIndex(embeddings, {
pineconeIndex
})
const retriever = vectorStore.asRetriever()
return retriever
}
}
module.exports = { nodeClass: Pinecone_Existing_VectorStores }
Binary file not shown.

After

Width:  |  Height:  |  Size: 2.4 KiB

@@ -0,0 +1,86 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface'
import { PineconeClient } from '@pinecone-database/pinecone'
class PineconeUpsert_VectorStores implements INode {
label: string
name: string
description: string
type: string
icon: string
category: string
baseClasses: string[]
inputs: INodeParams[]
constructor() {
this.label = 'Pinecone Upsert Document'
this.name = 'pineconeUpsert'
this.type = 'Pinecone'
this.icon = 'pinecone.png'
this.category = 'Vector Stores'
this.description = 'Upsert documents to Pinecone'
this.inputs = [
{
label: 'Document',
name: 'document',
type: 'Document'
},
{
label: 'Embeddings',
name: 'embeddings',
type: 'Embeddings'
},
{
label: 'Pinecone Api Key',
name: 'pineconeApiKey',
type: 'password'
},
{
label: 'Pinecone Environment',
name: 'pineconeEnv',
type: 'string'
},
{
label: 'Pinecone Index',
name: 'pineconeIndex',
type: 'string'
}
]
}
async getBaseClasses(): Promise<string[]> {
return ['BaseRetriever']
}
async init(nodeData: INodeData): Promise<any> {
const { PineconeStore } = await import('langchain/vectorstores')
const { Document } = await import('langchain/document')
const pineconeApiKey = nodeData.inputs?.pineconeApiKey as string
const pineconeEnv = nodeData.inputs?.pineconeEnv as string
const index = nodeData.inputs?.pineconeIndex as string
const docs = nodeData.inputs?.document
const embeddings = nodeData.inputs?.embeddings
const client = new PineconeClient()
await client.init({
apiKey: pineconeApiKey,
environment: pineconeEnv
})
const pineconeIndex = client.Index(index)
const finalDocs = []
for (let i = 0; i < docs.length; i += 1) {
finalDocs.push(new Document(docs[i]))
}
const result = await PineconeStore.fromDocuments(finalDocs, embeddings, {
pineconeIndex
})
const retriever = result.asRetriever()
return retriever
}
}
module.exports = { nodeClass: PineconeUpsert_VectorStores }
Binary file not shown.

After

Width:  |  Height:  |  Size: 2.4 KiB

+5
View File
@@ -16,13 +16,18 @@
},
"license": "SEE LICENSE IN LICENSE.md",
"dependencies": {
"@dqbd/tiktoken": "^1.0.4",
"@huggingface/inference": "^1.6.3",
"@pinecone-database/pinecone": "^0.0.12",
"axios": "^0.27.2",
"chromadb": "^1.3.1",
"dotenv": "^16.0.0",
"express": "^4.17.3",
"form-data": "^4.0.0",
"langchain": "^0.0.44",
"moment": "^2.29.3",
"node-fetch": "2",
"pdfjs-dist": "^3.5.141",
"ws": "^8.9.0"
},
"devDependencies": {
+9 -1
View File
@@ -17,6 +17,8 @@ export type NodeParamsType =
export type CommonType = string | number | boolean | undefined | null
export type MessageType = 'apiMessage' | 'userMessage'
/**
* Others
*/
@@ -49,6 +51,7 @@ export interface INodeParams {
rows?: number
list?: boolean
placeholder?: string
fileType?: string
}
export interface INodeExecutionData {
@@ -74,10 +77,15 @@ export interface INode extends INodeProperties {
inputs?: INodeParams[]
getBaseClasses?(): Promise<string[]>
getInstance?(nodeData: INodeData): Promise<string>
run?(nodeData: INodeData, input: string): Promise<string>
run?(nodeData: INodeData, input: string, options?: ICommonObject): Promise<string>
}
export interface INodeData extends INodeProperties {
inputs?: ICommonObject
instance?: any
}
export interface IMessage {
message: string
type: MessageType
}
+7
View File
@@ -4,6 +4,13 @@ import * as path from 'path'
export const numberOrExpressionRegex = '^(\\d+\\.?\\d*|{{.*}})$' //return true if string consists only numbers OR expression {{}}
export const notEmptyRegex = '(.|\\s)*\\S(.|\\s)*' //return true if string is not empty or blank
/**
* Get base classes of components
*
* @export
* @param {any} targetClass
* @returns {string[]}
*/
export const getBaseClasses = (targetClass: any) => {
const baseClasses: string[] = []