From 4fd73b86ed9ca198a560b821bb4cabf1ac1bdb8f Mon Sep 17 00:00:00 2001 From: atilgner Date: Tue, 13 Jun 2023 13:13:49 -0700 Subject: [PATCH 1/3] feat: added puppeteer loader node --- .../documentloaders/Puppeteer/Puppeteer.ts | 122 ++++++++++++++++++ .../documentloaders/Puppeteer/puppeteer.svg | 14 ++ packages/components/package.json | 5 +- 3 files changed, 139 insertions(+), 2 deletions(-) create mode 100644 packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts create mode 100644 packages/components/nodes/documentloaders/Puppeteer/puppeteer.svg diff --git a/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts b/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts new file mode 100644 index 00000000..1331c736 --- /dev/null +++ b/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts @@ -0,0 +1,122 @@ +import { INode, INodeData, INodeParams } from '../../../src/Interface' +import { TextSplitter } from 'langchain/text_splitter' +import { PuppeteerWebBaseLoader } from 'langchain/document_loaders/web/puppeteer' +import { test } from 'linkifyjs' +import { getAvailableURLs } from '../../../src' + +class Puppeteer_DocumentLoaders implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + + constructor() { + this.label = 'Puppeteer Web Scraper' + this.name = 'puppeteerWebScraper' + this.type = 'Document' + this.icon = 'puppeteer.svg' + this.category = 'Document Loaders' + this.description = `Load data from webpages` + this.baseClasses = [this.type] + this.inputs = [ + { + label: 'URL', + name: 'url', + type: 'string' + }, + { + label: 'Text Splitter', + name: 'textSplitter', + type: 'TextSplitter', + optional: true + }, + { + label: 'Web Scrape for Relative Links', + name: 'webScrape', + type: 'boolean', + optional: true, + additionalParams: true + }, + { + label: 'Web Scrape Links Limit', + name: 'limit', + type: 'number', + default: 10, + optional: true, + additionalParams: true + }, + { + label: 'Metadata', + name: 'metadata', + type: 'json', + optional: true, + additionalParams: true + } + ] + } + + async init(nodeData: INodeData): Promise { + const textSplitter = nodeData.inputs?.textSplitter as TextSplitter + const metadata = nodeData.inputs?.metadata + const webScrape = nodeData.inputs?.webScrape as boolean + let limit = nodeData.inputs?.limit as string + + let url = nodeData.inputs?.url as string + url = url.trim() + if (!test(url)) { + throw new Error('Invalid URL') + } + + const puppeteerLoader = async (url: string): Promise => { + let docs = [] + const loader = new PuppeteerWebBaseLoader(url) + if (textSplitter) { + docs = await loader.loadAndSplit(textSplitter) + } else { + docs = await loader.load() + } + return docs + } + + let availableUrls: string[] + let docs = [] + if (webScrape) { + if (!limit) limit = '10' + availableUrls = await getAvailableURLs(url, parseInt(limit)) + for (let i = 0; i < availableUrls.length; i++) { + try { + docs.push(...(await puppeteerLoader(availableUrls[i]))) + } catch (error) { + console.error('Error loading url with puppeteer. URL: ', availableUrls[i], 'Error: ', error) + continue + } + } + } else { + docs = await puppeteerLoader(url) + } + + if (metadata) { + const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) + let finaldocs = [] + for (const doc of docs) { + const newdoc = { + ...doc, + metadata: { + ...doc.metadata, + ...parsedMetadata + } + } + finaldocs.push(newdoc) + } + return finaldocs + } + + return docs + } +} + +module.exports = { nodeClass: Puppeteer_DocumentLoaders } diff --git a/packages/components/nodes/documentloaders/Puppeteer/puppeteer.svg b/packages/components/nodes/documentloaders/Puppeteer/puppeteer.svg new file mode 100644 index 00000000..8477fc52 --- /dev/null +++ b/packages/components/nodes/documentloaders/Puppeteer/puppeteer.svg @@ -0,0 +1,14 @@ + + + + + + + + \ No newline at end of file diff --git a/packages/components/package.json b/packages/components/package.json index a778ea8f..3bcac0d9 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -32,15 +32,16 @@ "faiss-node": "^0.2.1", "form-data": "^4.0.0", "graphql": "^16.6.0", + "html-to-text": "^9.0.5", "langchain": "^0.0.91", "linkifyjs": "^4.1.1", "mammoth": "^1.5.1", "moment": "^2.29.3", "node-fetch": "^2.6.11", "pdf-parse": "^1.1.1", + "puppeteer": "^20.7.1", "weaviate-ts-client": "^1.1.0", - "ws": "^8.9.0", - "html-to-text": "^9.0.5" + "ws": "^8.9.0" }, "devDependencies": { "@types/gulp": "4.0.9", From 2e43b233a5adfd81fb8a11e7f57f8eca63a9af9c Mon Sep 17 00:00:00 2001 From: Henry Date: Wed, 14 Jun 2023 19:48:44 +0100 Subject: [PATCH 2/3] add 16k model --- .../nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts | 28 ++++--- packages/server/marketplaces/API Agent.json | 84 +++++++++++-------- packages/server/marketplaces/AutoGPT.json | 28 ++++--- packages/server/marketplaces/BabyAGI.json | 28 ++++--- .../server/marketplaces/ChatGPTPlugin.json | 28 ++++--- .../marketplaces/Conversational Agent.json | 28 ++++--- .../Conversational Retrieval QA Chain.json | 28 ++++--- .../server/marketplaces/Github Repo QnA.json | 28 ++++--- .../marketplaces/Multi Prompt Chain.json | 28 ++++--- .../Multi Retrieval QA Chain.json | 28 ++++--- .../server/marketplaces/OpenAI Agent.json | 28 ++++--- .../Simple Conversation Chain.json | 28 ++++--- packages/server/marketplaces/Translator.json | 28 ++++--- packages/server/marketplaces/WebBrowser.json | 56 +++++++------ 14 files changed, 272 insertions(+), 204 deletions(-) diff --git a/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts b/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts index 7ddb17af..26f54db8 100644 --- a/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts +++ b/packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts @@ -35,29 +35,33 @@ class ChatOpenAI_ChatModels implements INode { label: 'gpt-4', name: 'gpt-4' }, - { - label: 'gpt-4-0314', - name: 'gpt-4-0314' - }, - { - label: 'gpt-4-32k-0314', - name: 'gpt-4-32k-0314' - }, { label: 'gpt-4-0613', name: 'gpt-4-0613' }, + { + label: 'gpt-4-32k', + name: 'gpt-4-32k' + }, + { + label: 'gpt-4-32k-0613', + name: 'gpt-4-32k-0613' + }, { label: 'gpt-3.5-turbo', name: 'gpt-3.5-turbo' }, - { - label: 'gpt-3.5-turbo-0301', - name: 'gpt-3.5-turbo-0301' - }, { label: 'gpt-3.5-turbo-0613', name: 'gpt-3.5-turbo-0613' + }, + { + label: 'gpt-3.5-turbo-16k', + name: 'gpt-3.5-turbo-16k' + }, + { + label: 'gpt-3.5-turbo-16k-0613', + name: 'gpt-3.5-turbo-16k-0613' } ], default: 'gpt-3.5-turbo', diff --git a/packages/server/marketplaces/API Agent.json b/packages/server/marketplaces/API Agent.json index ce4b7fb0..8ca79926 100644 --- a/packages/server/marketplaces/API Agent.json +++ b/packages/server/marketplaces/API Agent.json @@ -278,29 +278,33 @@ "label": "gpt-4", "name": "gpt-4" }, - { - "label": "gpt-4-0314", - "name": "gpt-4-0314" - }, - { - "label": "gpt-4-32k-0314", - "name": "gpt-4-32k-0314" - }, { "label": "gpt-4-0613", "name": "gpt-4-0613" }, + { + "label": "gpt-4-32k", + "name": "gpt-4-32k" + }, + { + "label": "gpt-4-32k-0613", + "name": "gpt-4-32k-0613" + }, { "label": "gpt-3.5-turbo", "name": "gpt-3.5-turbo" }, - { - "label": "gpt-3.5-turbo-0301", - "name": "gpt-3.5-turbo-0301" - }, { "label": "gpt-3.5-turbo-0613", "name": "gpt-3.5-turbo-0613" + }, + { + "label": "gpt-3.5-turbo-16k", + "name": "gpt-3.5-turbo-16k" + }, + { + "label": "gpt-3.5-turbo-16k-0613", + "name": "gpt-3.5-turbo-16k-0613" } ], "default": "gpt-3.5-turbo", @@ -481,29 +485,33 @@ "label": "gpt-4", "name": "gpt-4" }, - { - "label": "gpt-4-0314", - "name": "gpt-4-0314" - }, - { - "label": "gpt-4-32k-0314", - "name": "gpt-4-32k-0314" - }, { "label": "gpt-4-0613", "name": "gpt-4-0613" }, + { + "label": "gpt-4-32k", + "name": "gpt-4-32k" + }, + { + "label": "gpt-4-32k-0613", + "name": "gpt-4-32k-0613" + }, { "label": "gpt-3.5-turbo", "name": "gpt-3.5-turbo" }, - { - "label": "gpt-3.5-turbo-0301", - "name": "gpt-3.5-turbo-0301" - }, { "label": "gpt-3.5-turbo-0613", "name": "gpt-3.5-turbo-0613" + }, + { + "label": "gpt-3.5-turbo-16k", + "name": "gpt-3.5-turbo-16k" + }, + { + "label": "gpt-3.5-turbo-16k-0613", + "name": "gpt-3.5-turbo-16k-0613" } ], "default": "gpt-3.5-turbo", @@ -628,29 +636,33 @@ "label": "gpt-4", "name": "gpt-4" }, - { - "label": "gpt-4-0314", - "name": "gpt-4-0314" - }, - { - "label": "gpt-4-32k-0314", - "name": "gpt-4-32k-0314" - }, { "label": "gpt-4-0613", "name": "gpt-4-0613" }, + { + "label": "gpt-4-32k", + "name": "gpt-4-32k" + }, + { + "label": "gpt-4-32k-0613", + "name": "gpt-4-32k-0613" + }, { "label": "gpt-3.5-turbo", "name": "gpt-3.5-turbo" }, - { - "label": "gpt-3.5-turbo-0301", - "name": "gpt-3.5-turbo-0301" - }, { "label": "gpt-3.5-turbo-0613", "name": "gpt-3.5-turbo-0613" + }, + { + "label": "gpt-3.5-turbo-16k", + "name": "gpt-3.5-turbo-16k" + }, + { + "label": "gpt-3.5-turbo-16k-0613", + "name": "gpt-3.5-turbo-16k-0613" } ], "default": "gpt-3.5-turbo", diff --git a/packages/server/marketplaces/AutoGPT.json b/packages/server/marketplaces/AutoGPT.json index 4bb68722..47926272 100644 --- a/packages/server/marketplaces/AutoGPT.json +++ b/packages/server/marketplaces/AutoGPT.json @@ -124,29 +124,33 @@ "label": "gpt-4", "name": "gpt-4" }, - { - "label": "gpt-4-0314", - "name": "gpt-4-0314" - }, - { - "label": "gpt-4-32k-0314", - "name": "gpt-4-32k-0314" - }, { "label": "gpt-4-0613", "name": "gpt-4-0613" }, + { + "label": "gpt-4-32k", + "name": "gpt-4-32k" + }, + { + "label": "gpt-4-32k-0613", + "name": "gpt-4-32k-0613" + }, { "label": "gpt-3.5-turbo", "name": "gpt-3.5-turbo" }, - { - "label": "gpt-3.5-turbo-0301", - "name": "gpt-3.5-turbo-0301" - }, { "label": "gpt-3.5-turbo-0613", "name": "gpt-3.5-turbo-0613" + }, + { + "label": "gpt-3.5-turbo-16k", + "name": "gpt-3.5-turbo-16k" + }, + { + "label": "gpt-3.5-turbo-16k-0613", + "name": "gpt-3.5-turbo-16k-0613" } ], "default": "gpt-3.5-turbo", diff --git a/packages/server/marketplaces/BabyAGI.json b/packages/server/marketplaces/BabyAGI.json index aefb9902..572d73f1 100644 --- a/packages/server/marketplaces/BabyAGI.json +++ b/packages/server/marketplaces/BabyAGI.json @@ -227,29 +227,33 @@ "label": "gpt-4", "name": "gpt-4" }, - { - "label": "gpt-4-0314", - "name": "gpt-4-0314" - }, - { - "label": "gpt-4-32k-0314", - "name": "gpt-4-32k-0314" - }, { "label": "gpt-4-0613", "name": "gpt-4-0613" }, + { + "label": "gpt-4-32k", + "name": "gpt-4-32k" + }, + { + "label": "gpt-4-32k-0613", + "name": "gpt-4-32k-0613" + }, { "label": "gpt-3.5-turbo", "name": "gpt-3.5-turbo" }, - { - "label": "gpt-3.5-turbo-0301", - "name": "gpt-3.5-turbo-0301" - }, { "label": "gpt-3.5-turbo-0613", "name": "gpt-3.5-turbo-0613" + }, + { + "label": "gpt-3.5-turbo-16k", + "name": "gpt-3.5-turbo-16k" + }, + { + "label": "gpt-3.5-turbo-16k-0613", + "name": "gpt-3.5-turbo-16k-0613" } ], "default": "gpt-3.5-turbo", diff --git a/packages/server/marketplaces/ChatGPTPlugin.json b/packages/server/marketplaces/ChatGPTPlugin.json index 4d5d1869..76964a09 100644 --- a/packages/server/marketplaces/ChatGPTPlugin.json +++ b/packages/server/marketplaces/ChatGPTPlugin.json @@ -287,29 +287,33 @@ "label": "gpt-4", "name": "gpt-4" }, - { - "label": "gpt-4-0314", - "name": "gpt-4-0314" - }, - { - "label": "gpt-4-32k-0314", - "name": "gpt-4-32k-0314" - }, { "label": "gpt-4-0613", "name": "gpt-4-0613" }, + { + "label": "gpt-4-32k", + "name": "gpt-4-32k" + }, + { + "label": "gpt-4-32k-0613", + "name": "gpt-4-32k-0613" + }, { "label": "gpt-3.5-turbo", "name": "gpt-3.5-turbo" }, - { - "label": "gpt-3.5-turbo-0301", - "name": "gpt-3.5-turbo-0301" - }, { "label": "gpt-3.5-turbo-0613", "name": "gpt-3.5-turbo-0613" + }, + { + "label": "gpt-3.5-turbo-16k", + "name": "gpt-3.5-turbo-16k" + }, + { + "label": "gpt-3.5-turbo-16k-0613", + "name": "gpt-3.5-turbo-16k-0613" } ], "default": "gpt-3.5-turbo", diff --git a/packages/server/marketplaces/Conversational Agent.json b/packages/server/marketplaces/Conversational Agent.json index d6d61e86..b0295fd6 100644 --- a/packages/server/marketplaces/Conversational Agent.json +++ b/packages/server/marketplaces/Conversational Agent.json @@ -34,29 +34,33 @@ "label": "gpt-4", "name": "gpt-4" }, - { - "label": "gpt-4-0314", - "name": "gpt-4-0314" - }, - { - "label": "gpt-4-32k-0314", - "name": "gpt-4-32k-0314" - }, { "label": "gpt-4-0613", "name": "gpt-4-0613" }, + { + "label": "gpt-4-32k", + "name": "gpt-4-32k" + }, + { + "label": "gpt-4-32k-0613", + "name": "gpt-4-32k-0613" + }, { "label": "gpt-3.5-turbo", "name": "gpt-3.5-turbo" }, - { - "label": "gpt-3.5-turbo-0301", - "name": "gpt-3.5-turbo-0301" - }, { "label": "gpt-3.5-turbo-0613", "name": "gpt-3.5-turbo-0613" + }, + { + "label": "gpt-3.5-turbo-16k", + "name": "gpt-3.5-turbo-16k" + }, + { + "label": "gpt-3.5-turbo-16k-0613", + "name": "gpt-3.5-turbo-16k-0613" } ], "default": "gpt-3.5-turbo", diff --git a/packages/server/marketplaces/Conversational Retrieval QA Chain.json b/packages/server/marketplaces/Conversational Retrieval QA Chain.json index 4d8b2881..ed190cdc 100644 --- a/packages/server/marketplaces/Conversational Retrieval QA Chain.json +++ b/packages/server/marketplaces/Conversational Retrieval QA Chain.json @@ -348,29 +348,33 @@ "label": "gpt-4", "name": "gpt-4" }, - { - "label": "gpt-4-0314", - "name": "gpt-4-0314" - }, - { - "label": "gpt-4-32k-0314", - "name": "gpt-4-32k-0314" - }, { "label": "gpt-4-0613", "name": "gpt-4-0613" }, + { + "label": "gpt-4-32k", + "name": "gpt-4-32k" + }, + { + "label": "gpt-4-32k-0613", + "name": "gpt-4-32k-0613" + }, { "label": "gpt-3.5-turbo", "name": "gpt-3.5-turbo" }, - { - "label": "gpt-3.5-turbo-0301", - "name": "gpt-3.5-turbo-0301" - }, { "label": "gpt-3.5-turbo-0613", "name": "gpt-3.5-turbo-0613" + }, + { + "label": "gpt-3.5-turbo-16k", + "name": "gpt-3.5-turbo-16k" + }, + { + "label": "gpt-3.5-turbo-16k-0613", + "name": "gpt-3.5-turbo-16k-0613" } ], "default": "gpt-3.5-turbo", diff --git a/packages/server/marketplaces/Github Repo QnA.json b/packages/server/marketplaces/Github Repo QnA.json index 852c9741..92867957 100644 --- a/packages/server/marketplaces/Github Repo QnA.json +++ b/packages/server/marketplaces/Github Repo QnA.json @@ -365,29 +365,33 @@ "label": "gpt-4", "name": "gpt-4" }, - { - "label": "gpt-4-0314", - "name": "gpt-4-0314" - }, - { - "label": "gpt-4-32k-0314", - "name": "gpt-4-32k-0314" - }, { "label": "gpt-4-0613", "name": "gpt-4-0613" }, + { + "label": "gpt-4-32k", + "name": "gpt-4-32k" + }, + { + "label": "gpt-4-32k-0613", + "name": "gpt-4-32k-0613" + }, { "label": "gpt-3.5-turbo", "name": "gpt-3.5-turbo" }, - { - "label": "gpt-3.5-turbo-0301", - "name": "gpt-3.5-turbo-0301" - }, { "label": "gpt-3.5-turbo-0613", "name": "gpt-3.5-turbo-0613" + }, + { + "label": "gpt-3.5-turbo-16k", + "name": "gpt-3.5-turbo-16k" + }, + { + "label": "gpt-3.5-turbo-16k-0613", + "name": "gpt-3.5-turbo-16k-0613" } ], "default": "gpt-3.5-turbo", diff --git a/packages/server/marketplaces/Multi Prompt Chain.json b/packages/server/marketplaces/Multi Prompt Chain.json index d4aa4550..339476e7 100644 --- a/packages/server/marketplaces/Multi Prompt Chain.json +++ b/packages/server/marketplaces/Multi Prompt Chain.json @@ -294,29 +294,33 @@ "label": "gpt-4", "name": "gpt-4" }, - { - "label": "gpt-4-0314", - "name": "gpt-4-0314" - }, - { - "label": "gpt-4-32k-0314", - "name": "gpt-4-32k-0314" - }, { "label": "gpt-4-0613", "name": "gpt-4-0613" }, + { + "label": "gpt-4-32k", + "name": "gpt-4-32k" + }, + { + "label": "gpt-4-32k-0613", + "name": "gpt-4-32k-0613" + }, { "label": "gpt-3.5-turbo", "name": "gpt-3.5-turbo" }, - { - "label": "gpt-3.5-turbo-0301", - "name": "gpt-3.5-turbo-0301" - }, { "label": "gpt-3.5-turbo-0613", "name": "gpt-3.5-turbo-0613" + }, + { + "label": "gpt-3.5-turbo-16k", + "name": "gpt-3.5-turbo-16k" + }, + { + "label": "gpt-3.5-turbo-16k-0613", + "name": "gpt-3.5-turbo-16k-0613" } ], "default": "gpt-3.5-turbo", diff --git a/packages/server/marketplaces/Multi Retrieval QA Chain.json b/packages/server/marketplaces/Multi Retrieval QA Chain.json index f3cd1fcc..04df2fee 100644 --- a/packages/server/marketplaces/Multi Retrieval QA Chain.json +++ b/packages/server/marketplaces/Multi Retrieval QA Chain.json @@ -298,29 +298,33 @@ "label": "gpt-4", "name": "gpt-4" }, - { - "label": "gpt-4-0314", - "name": "gpt-4-0314" - }, - { - "label": "gpt-4-32k-0314", - "name": "gpt-4-32k-0314" - }, { "label": "gpt-4-0613", "name": "gpt-4-0613" }, + { + "label": "gpt-4-32k", + "name": "gpt-4-32k" + }, + { + "label": "gpt-4-32k-0613", + "name": "gpt-4-32k-0613" + }, { "label": "gpt-3.5-turbo", "name": "gpt-3.5-turbo" }, - { - "label": "gpt-3.5-turbo-0301", - "name": "gpt-3.5-turbo-0301" - }, { "label": "gpt-3.5-turbo-0613", "name": "gpt-3.5-turbo-0613" + }, + { + "label": "gpt-3.5-turbo-16k", + "name": "gpt-3.5-turbo-16k" + }, + { + "label": "gpt-3.5-turbo-16k-0613", + "name": "gpt-3.5-turbo-16k-0613" } ], "default": "gpt-3.5-turbo", diff --git a/packages/server/marketplaces/OpenAI Agent.json b/packages/server/marketplaces/OpenAI Agent.json index 77192eb1..7e685546 100644 --- a/packages/server/marketplaces/OpenAI Agent.json +++ b/packages/server/marketplaces/OpenAI Agent.json @@ -34,29 +34,33 @@ "label": "gpt-4", "name": "gpt-4" }, - { - "label": "gpt-4-0314", - "name": "gpt-4-0314" - }, - { - "label": "gpt-4-32k-0314", - "name": "gpt-4-32k-0314" - }, { "label": "gpt-4-0613", "name": "gpt-4-0613" }, + { + "label": "gpt-4-32k", + "name": "gpt-4-32k" + }, + { + "label": "gpt-4-32k-0613", + "name": "gpt-4-32k-0613" + }, { "label": "gpt-3.5-turbo", "name": "gpt-3.5-turbo" }, - { - "label": "gpt-3.5-turbo-0301", - "name": "gpt-3.5-turbo-0301" - }, { "label": "gpt-3.5-turbo-0613", "name": "gpt-3.5-turbo-0613" + }, + { + "label": "gpt-3.5-turbo-16k", + "name": "gpt-3.5-turbo-16k" + }, + { + "label": "gpt-3.5-turbo-16k-0613", + "name": "gpt-3.5-turbo-16k-0613" } ], "default": "gpt-3.5-turbo", diff --git a/packages/server/marketplaces/Simple Conversation Chain.json b/packages/server/marketplaces/Simple Conversation Chain.json index 93cf06d7..f7e654db 100644 --- a/packages/server/marketplaces/Simple Conversation Chain.json +++ b/packages/server/marketplaces/Simple Conversation Chain.json @@ -34,29 +34,33 @@ "label": "gpt-4", "name": "gpt-4" }, - { - "label": "gpt-4-0314", - "name": "gpt-4-0314" - }, - { - "label": "gpt-4-32k-0314", - "name": "gpt-4-32k-0314" - }, { "label": "gpt-4-0613", "name": "gpt-4-0613" }, + { + "label": "gpt-4-32k", + "name": "gpt-4-32k" + }, + { + "label": "gpt-4-32k-0613", + "name": "gpt-4-32k-0613" + }, { "label": "gpt-3.5-turbo", "name": "gpt-3.5-turbo" }, - { - "label": "gpt-3.5-turbo-0301", - "name": "gpt-3.5-turbo-0301" - }, { "label": "gpt-3.5-turbo-0613", "name": "gpt-3.5-turbo-0613" + }, + { + "label": "gpt-3.5-turbo-16k", + "name": "gpt-3.5-turbo-16k" + }, + { + "label": "gpt-3.5-turbo-16k-0613", + "name": "gpt-3.5-turbo-16k-0613" } ], "default": "gpt-3.5-turbo", diff --git a/packages/server/marketplaces/Translator.json b/packages/server/marketplaces/Translator.json index 5a57a8f5..fda400e2 100644 --- a/packages/server/marketplaces/Translator.json +++ b/packages/server/marketplaces/Translator.json @@ -104,29 +104,33 @@ "label": "gpt-4", "name": "gpt-4" }, - { - "label": "gpt-4-0314", - "name": "gpt-4-0314" - }, - { - "label": "gpt-4-32k-0314", - "name": "gpt-4-32k-0314" - }, { "label": "gpt-4-0613", "name": "gpt-4-0613" }, + { + "label": "gpt-4-32k", + "name": "gpt-4-32k" + }, + { + "label": "gpt-4-32k-0613", + "name": "gpt-4-32k-0613" + }, { "label": "gpt-3.5-turbo", "name": "gpt-3.5-turbo" }, - { - "label": "gpt-3.5-turbo-0301", - "name": "gpt-3.5-turbo-0301" - }, { "label": "gpt-3.5-turbo-0613", "name": "gpt-3.5-turbo-0613" + }, + { + "label": "gpt-3.5-turbo-16k", + "name": "gpt-3.5-turbo-16k" + }, + { + "label": "gpt-3.5-turbo-16k-0613", + "name": "gpt-3.5-turbo-16k-0613" } ], "default": "gpt-3.5-turbo", diff --git a/packages/server/marketplaces/WebBrowser.json b/packages/server/marketplaces/WebBrowser.json index d533c75f..f87fe07e 100644 --- a/packages/server/marketplaces/WebBrowser.json +++ b/packages/server/marketplaces/WebBrowser.json @@ -34,29 +34,33 @@ "label": "gpt-4", "name": "gpt-4" }, - { - "label": "gpt-4-0314", - "name": "gpt-4-0314" - }, - { - "label": "gpt-4-32k-0314", - "name": "gpt-4-32k-0314" - }, { "label": "gpt-4-0613", "name": "gpt-4-0613" }, + { + "label": "gpt-4-32k", + "name": "gpt-4-32k" + }, + { + "label": "gpt-4-32k-0613", + "name": "gpt-4-32k-0613" + }, { "label": "gpt-3.5-turbo", "name": "gpt-3.5-turbo" }, - { - "label": "gpt-3.5-turbo-0301", - "name": "gpt-3.5-turbo-0301" - }, { "label": "gpt-3.5-turbo-0613", "name": "gpt-3.5-turbo-0613" + }, + { + "label": "gpt-3.5-turbo-16k", + "name": "gpt-3.5-turbo-16k" + }, + { + "label": "gpt-3.5-turbo-16k-0613", + "name": "gpt-3.5-turbo-16k-0613" } ], "default": "gpt-3.5-turbo", @@ -318,29 +322,33 @@ "label": "gpt-4", "name": "gpt-4" }, - { - "label": "gpt-4-0314", - "name": "gpt-4-0314" - }, - { - "label": "gpt-4-32k-0314", - "name": "gpt-4-32k-0314" - }, { "label": "gpt-4-0613", "name": "gpt-4-0613" }, + { + "label": "gpt-4-32k", + "name": "gpt-4-32k" + }, + { + "label": "gpt-4-32k-0613", + "name": "gpt-4-32k-0613" + }, { "label": "gpt-3.5-turbo", "name": "gpt-3.5-turbo" }, - { - "label": "gpt-3.5-turbo-0301", - "name": "gpt-3.5-turbo-0301" - }, { "label": "gpt-3.5-turbo-0613", "name": "gpt-3.5-turbo-0613" + }, + { + "label": "gpt-3.5-turbo-16k", + "name": "gpt-3.5-turbo-16k" + }, + { + "label": "gpt-3.5-turbo-16k-0613", + "name": "gpt-3.5-turbo-16k-0613" } ], "default": "gpt-3.5-turbo", From 919d04930616c12065621f5acb01c7f4c33bb359 Mon Sep 17 00:00:00 2001 From: atilgner Date: Wed, 14 Jun 2023 12:09:31 -0700 Subject: [PATCH 3/3] fix: accidentally deleted html to text --- packages/components/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/components/package.json b/packages/components/package.json index 0aa161f8..b0a58161 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -40,7 +40,8 @@ "pdf-parse": "^1.1.1", "puppeteer": "^20.7.1", "weaviate-ts-client": "^1.1.0", - "ws": "^8.9.0" + "ws": "^8.9.0", + "html-to-text": "^9.0.5" }, "devDependencies": { "@types/gulp": "4.0.9",