modify app.py

app.py

@@ -68,7 +68,7 @@ class MGTHuman(datasets.GeneratorBasedBuilder):
         else:
             return text
 
-    def get_text_by_index(self, filepath, index):
+    def get_text_by_index(self, filepath, index, cut_tokens=False, max_tokens=2048):
         count = 0
         with open(filepath, 'r') as f:
             data = json.load(f)
@@ -76,7 +76,9 @@ class MGTHuman(datasets.GeneratorBasedBuilder):
             if not row["text"].strip():
                 continue
             if count == index:
-                text = self.truncate_text(row["text"], max_tokens=2048)
+                text = row["text"]
+                if cut_tokens:
+                    text = self.truncate_text(text, max_tokens)
                 return text
             count += 1
         return "Index 超出范围，请输入有效的数字。"
@@ -124,9 +126,12 @@ if uploaded_folder:
         
         # 输入序号查看文本
         index_to_view = st.number_input("输入要查看的文本序号", min_value=0, max_value=total_entries - 1, step=1)
+
+        # 添加复选框以选择是否切割文本
+        cut_tokens = st.checkbox("是否对文本进行token切割", value=False)
         
         if st.button("显示文本"):
-            text = mgt_human.get_text_by_index(file_to_display, index=index_to_view)
+            text = mgt_human.get_text_by_index(file_to_display, index=index_to_view, cut_tokens=cut_tokens)
             st.write("对应的文本内容为：", text)
     else:
         st.write("未找到任何 JSON 文件，请检查 ZIP 文件结构。")