Spaces:
Running on Zero
Running on Zero
Update document_loader.py
Browse files- document_loader.py +3 -3
document_loader.py
CHANGED
|
@@ -22,7 +22,7 @@ from config import DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE
|
|
| 22 |
|
| 23 |
@dataclass
|
| 24 |
class TextUnit:
|
| 25 |
-
"""
|
| 26 |
|
| 27 |
text: str
|
| 28 |
source_name: str
|
|
@@ -35,7 +35,7 @@ class TextUnit:
|
|
| 35 |
|
| 36 |
@dataclass
|
| 37 |
class Chunk:
|
| 38 |
-
"""
|
| 39 |
|
| 40 |
chunk_id: str
|
| 41 |
text: str
|
|
@@ -48,7 +48,7 @@ class Chunk:
|
|
| 48 |
|
| 49 |
@property
|
| 50 |
def citation(self) -> str:
|
| 51 |
-
"""
|
| 52 |
bits = [self.source_name]
|
| 53 |
|
| 54 |
if self.section:
|
|
|
|
| 22 |
|
| 23 |
@dataclass
|
| 24 |
class TextUnit:
|
| 25 |
+
""" extracted text and its source metadata before chunking"""
|
| 26 |
|
| 27 |
text: str
|
| 28 |
source_name: str
|
|
|
|
| 35 |
|
| 36 |
@dataclass
|
| 37 |
class Chunk:
|
| 38 |
+
"""searchable text chunk with source and citation metadata"""
|
| 39 |
|
| 40 |
chunk_id: str
|
| 41 |
text: str
|
|
|
|
| 48 |
|
| 49 |
@property
|
| 50 |
def citation(self) -> str:
|
| 51 |
+
"""readable citation from available source metadata"""
|
| 52 |
bits = [self.source_name]
|
| 53 |
|
| 54 |
if self.section:
|