Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
|
@@ -14,11 +14,10 @@ model_name = "roberta-base"
|
|
| 14 |
tokenizer = RobertaTokenizer.from_pretrained(model_name, map_location=torch.device('cpu'))
|
| 15 |
|
| 16 |
def text_to_sentences(text):
|
| 17 |
-
re.
|
| 18 |
-
return re.split(r'[.!?]', text)
|
| 19 |
|
| 20 |
# function to concatenate sentences into chunks of size 900 or less
|
| 21 |
-
def
|
| 22 |
sentences = text_to_sentences(text)
|
| 23 |
chunks = []
|
| 24 |
current_chunk = ""
|
|
@@ -47,7 +46,7 @@ def predict(query, device="cpu"):
|
|
| 47 |
return real
|
| 48 |
|
| 49 |
def findRealProb(text):
|
| 50 |
-
chunksOfText = (
|
| 51 |
results = []
|
| 52 |
for chunk in chunksOfText:
|
| 53 |
output = predict(chunk)
|
|
|
|
| 14 |
tokenizer = RobertaTokenizer.from_pretrained(model_name, map_location=torch.device('cpu'))
|
| 15 |
|
| 16 |
def text_to_sentences(text):
|
| 17 |
+
return re.split(r'(?<=[^A-Z].[.?]) +(?=[A-Z])', text)
|
|
|
|
| 18 |
|
| 19 |
# function to concatenate sentences into chunks of size 900 or less
|
| 20 |
+
def chunks_of_900(text, chunk_size=900):
|
| 21 |
sentences = text_to_sentences(text)
|
| 22 |
chunks = []
|
| 23 |
current_chunk = ""
|
|
|
|
| 46 |
return real
|
| 47 |
|
| 48 |
def findRealProb(text):
|
| 49 |
+
chunksOfText = (chunks_of_900(text))
|
| 50 |
results = []
|
| 51 |
for chunk in chunksOfText:
|
| 52 |
output = predict(chunk)
|