Bases: ProcessorBase
SimpleTxtProcessor is a class that implements the ProcessorBase interface.
It is used to process the files with the Simple Txt parser.
Source code in core/quivr_core/processor/implementations/simple_txt_processor.py
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60 | class SimpleTxtProcessor(ProcessorBase):
"""
SimpleTxtProcessor is a class that implements the ProcessorBase interface.
It is used to process the files with the Simple Txt parser.
"""
supported_extensions = [FileExtension.txt]
def __init__(
self, splitter_config: SplitterConfig = SplitterConfig(), **kwargs
) -> None:
super().__init__(**kwargs)
self.splitter_config = splitter_config
@property
def processor_metadata(self) -> dict[str, Any]:
return {
"processor_cls": "SimpleTxtProcessor",
"splitter": self.splitter_config.model_dump(),
}
async def process_file_inner(self, file: QuivrFile) -> list[Document]:
async with aiofiles.open(file.path, mode="r") as f:
content = await f.read()
doc = Document(page_content=content)
docs = recursive_character_splitter(
doc, self.splitter_config.chunk_size, self.splitter_config.chunk_overlap
)
return docs
|