Create app.py

app.py

@@ -0,0 +1,36 @@
+import gradio as gr
+import json
+import PyPDF2
+import markdown
+
+def convert_to_jsonl(file):
+    content = ""
+    if file.name.endswith('.txt'):
+        content = file.read().decode('utf-8')
+    elif file.name.endswith('.pdf'):
+        pdf_reader = PyPDF2.PdfReader(file)
+        for page in pdf_reader.pages:
+            content += page.extract_text()
+    elif file.name.endswith('.md'):
+        content = file.read().decode('utf-8')
+        content = markdown.markdown(content)
+    
+    lines = content.split('\n')
+    jsonl_output = ""
+    for i, line in enumerate(lines):
+        if line.strip():
+            json_obj = {"id": i, "text": line.strip()}
+            jsonl_output += json.dumps(json_obj) + "\n"
+    
+    return jsonl_output
+
+iface = gr.Interface(
+    fn=convert_to_jsonl,
+    inputs=gr.File(label="Upload .txt, .pdf, or .md file"),
+    outputs=gr.File(label="Download JSONL file"),
+    title="Raw Text to Indexed JSONL Converter",
+    description="Upload a .txt, .pdf, or .md file to convert it to an indexed JSONL file for LLM training."
+)
+
+if __name__ == "__main__":
+    iface.launch()