diff --git a/packages/components/nodes/textsplitters/CharacterTextSplitter/CharacterTextSplitter.ts b/packages/components/nodes/textsplitters/CharacterTextSplitter/CharacterTextSplitter.ts index f9427d10..ed972ad6 100644 --- a/packages/components/nodes/textsplitters/CharacterTextSplitter/CharacterTextSplitter.ts +++ b/packages/components/nodes/textsplitters/CharacterTextSplitter/CharacterTextSplitter.ts @@ -23,12 +23,6 @@ class CharacterTextSplitter_TextSplitters implements INode { this.description = `splits only on one type of character (defaults to "\\n\\n").` this.baseClasses = [this.type, ...getBaseClasses(CharacterTextSplitter)] this.inputs = [ - { - label: 'Separator', - name: 'separator', - type: 'string', - optional: true - }, { label: 'Chunk Size', name: 'chunkSize', @@ -41,6 +35,14 @@ class CharacterTextSplitter_TextSplitters implements INode { name: 'chunkOverlap', type: 'number', optional: true + }, + { + label: 'Custom Separator', + name: 'separator', + type: 'string', + placeholder: `" "`, + description: 'Seperator to determine when to split the text, will override the default separator', + optional: true } ] } diff --git a/packages/components/nodes/textsplitters/RecursiveCharacterTextSplitter/RecursiveCharacterTextSplitter.ts b/packages/components/nodes/textsplitters/RecursiveCharacterTextSplitter/RecursiveCharacterTextSplitter.ts index dcca70ba..ec9095b6 100644 --- a/packages/components/nodes/textsplitters/RecursiveCharacterTextSplitter/RecursiveCharacterTextSplitter.ts +++ b/packages/components/nodes/textsplitters/RecursiveCharacterTextSplitter/RecursiveCharacterTextSplitter.ts @@ -16,7 +16,7 @@ class RecursiveCharacterTextSplitter_TextSplitters implements INode { constructor() { this.label = 'Recursive Character Text Splitter' this.name = 'recursiveCharacterTextSplitter' - this.version = 1.0 + this.version = 2.0 this.type = 'RecursiveCharacterTextSplitter' this.icon = 'textsplitter.svg' this.category = 'Text Splitters' @@ -35,6 +35,15 @@ class RecursiveCharacterTextSplitter_TextSplitters implements INode { name: 'chunkOverlap', type: 'number', optional: true + }, + { + label: 'Custom Separators', + name: 'separators', + type: 'string', + rows: 4, + description: 'Array of custom seperators to determine when to split the text, will override the default separators', + placeholder: `["|", "##", ">", "-"]`, + optional: true } ] } @@ -42,11 +51,19 @@ class RecursiveCharacterTextSplitter_TextSplitters implements INode { async init(nodeData: INodeData): Promise { const chunkSize = nodeData.inputs?.chunkSize as string const chunkOverlap = nodeData.inputs?.chunkOverlap as string + const separators = nodeData.inputs?.separators as string const obj = {} as RecursiveCharacterTextSplitterParams if (chunkSize) obj.chunkSize = parseInt(chunkSize, 10) if (chunkOverlap) obj.chunkOverlap = parseInt(chunkOverlap, 10) + if (separators) { + try { + obj.separators = JSON.parse(separators) + } catch (e) { + throw new Error(e) + } + } const splitter = new RecursiveCharacterTextSplitter(obj)