add token text splitter

This commit is contained in:
chungyau97
2023-05-06 11:46:50 +07:00
parent aa0795c4ed
commit 61a0cee551
3 changed files with 94 additions and 1 deletions
+1 -1
View File
@@ -16,7 +16,7 @@
},
"license": "SEE LICENSE IN LICENSE.md",
"dependencies": {
"@dqbd/tiktoken": "^1.0.4",
"@dqbd/tiktoken": "^1.0.7",
"@huggingface/inference": "^1.6.3",
"@pinecone-database/pinecone": "^0.0.12",
"@supabase/supabase-js": "^2.21.0",