Update app.py

app.py

@@ -19,10 +19,28 @@ def read_in_text(url):
   with open(url, 'r') as file:
     article = file.read()
     return article
+    
+def clean_text(url):
+  text = read_in_text(url)
+  text = text.encode("ascii", errors="ignore").decode(
+          "ascii"
+    )  # remove non-ascii, Chinese characters
+    
+  text = re.sub(r"\n", " ", text)
+  text = re.sub(r"\n\n", " ", text)
+  text = re.sub(r"\t", " ", text)
+  text = text.strip(" ")
+  text = re.sub(
+        " +", " ", text
+    ).strip()  # get rid of multiple spaces and replace with a single
+
+  text = re.sub(r'Date\s\d{1,2}\/\d{1,2}\/\d{4}', '', text) #remove date
+  text = re.sub(r'\d{1,2}:\d{2}\s[A-Z]+\s[A-Z]+', '', text) #remove time
+  return text
  
 #Defining a function to get the category of the news article   
 def get_category(file):
-  text = read_in_text(file.name)
+  text = clean_text(file.name)
 
   input_tensor = tokenizer.encode(text, return_tensors='pt', truncation=True)
   logits = model(input_tensor).logits