Feature/Indexing (#1802)

* indexing

* fix for multiple files upsert

* fix default Postgres port

* fix SQLite node description

* add MySQLRecordManager node

* fix MySQL unique index

* add upsert history

* update jsx ui

* lint-fix

* update dialog details

* update llamaindex pinecone

---------

Co-authored-by: chungyau97 <chungyau97@gmail.com>
This commit is contained in:
Henry Heng
2024-04-02 23:47:19 +01:00
committed by GitHub
parent 957694a912
commit e422ce287b
67 changed files with 3006 additions and 246 deletions
@@ -2,7 +2,7 @@ import path from 'path'
import { flatten } from 'lodash'
import { storageContextFromDefaults, serviceContextFromDefaults, VectorStoreIndex, Document } from 'llamaindex'
import { Document as LCDocument } from 'langchain/document'
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface'
import { getUserHome } from '../../../src'
class SimpleStoreUpsert_LlamaIndex_VectorStores implements INode {
@@ -79,7 +79,7 @@ class SimpleStoreUpsert_LlamaIndex_VectorStores implements INode {
//@ts-ignore
vectorStoreMethods = {
async upsert(nodeData: INodeData): Promise<void> {
async upsert(nodeData: INodeData): Promise<Partial<IndexingResult>> {
const basePath = nodeData.inputs?.basePath as string
const docs = nodeData.inputs?.document as LCDocument[]
const embeddings = nodeData.inputs?.embeddings
@@ -105,6 +105,7 @@ class SimpleStoreUpsert_LlamaIndex_VectorStores implements INode {
try {
await VectorStoreIndex.fromDocuments(llamadocs, { serviceContext, storageContext })
return { numAdded: finalDocs.length, addedDocs: finalDocs }
} catch (e) {
throw new Error(e)
}