allow chunk_overlap with zero value (#457) #none

Co-authored-by: Tadashi <tadashi@cinnamon.is>
This commit is contained in:
Pedro Lima
2024-11-05 07:12:52 +00:00
committed by GitHub
parent d127fec9f7
commit 908452cc18

View File

@@ -742,7 +742,7 @@ class IndexDocumentPipeline(BaseFileIndexIndexing):
loader=reader, loader=reader,
splitter=TokenSplitter( splitter=TokenSplitter(
chunk_size=chunk_size or 1024, chunk_size=chunk_size or 1024,
chunk_overlap=chunk_overlap or 256, chunk_overlap=chunk_overlap if chunk_overlap is not None else 256,
separator="\n\n", separator="\n\n",
backup_separators=["\n", ".", "\u200B"], backup_separators=["\n", ".", "\u200B"],
), ),