| from __future__ import annotations |
| from typing import TYPE_CHECKING, Any, Callable, Dict, List, Tuple, Type |
| import logging |
| import json |
| import os |
| import datetime |
| import hashlib |
| import csv |
| import requests |
| import re |
| import html |
| import markdown2 |
| import torch |
| import sys |
| import gc |
| from pygments.lexers import guess_lexer, ClassNotFound |
|
|
| import gradio as gr |
| from pypinyin import lazy_pinyin |
| import tiktoken |
| import mdtex2html |
| from markdown import markdown |
| from pygments import highlight |
| from pygments.lexers import guess_lexer,get_lexer_by_name |
| from pygments.formatters import HtmlFormatter |
| from beschreibungen import * |
|
|
| logging.basicConfig( |
| level=logging.INFO, |
| format="%(asctime)s [%(levelname)s] [%(filename)s:%(lineno)d] %(message)s", |
| ) |
|
|
|
|
| def markdown_to_html_with_syntax_highlight(md_str): |
| def replacer(match): |
| lang = match.group(1) or "text" |
| code = match.group(2) |
| lang = lang.strip() |
| |
| if lang=="text": |
| lexer = guess_lexer(code) |
| lang = lexer.name |
| |
| try: |
| lexer = get_lexer_by_name(lang, stripall=True) |
| except ValueError: |
| lexer = get_lexer_by_name("python", stripall=True) |
| formatter = HtmlFormatter() |
| |
| highlighted_code = highlight(code, lexer, formatter) |
|
|
| return f'<pre><code class="{lang}">{highlighted_code}</code></pre>' |
|
|
| code_block_pattern = r"```(\w+)?\n([\s\S]+?)\n```" |
| md_str = re.sub(code_block_pattern, replacer, md_str, flags=re.MULTILINE) |
|
|
| html_str = markdown(md_str) |
| return html_str |
|
|
|
|
| def normalize_markdown(md_text: str) -> str: |
| lines = md_text.split("\n") |
| normalized_lines = [] |
| inside_list = False |
|
|
| for i, line in enumerate(lines): |
| if re.match(r"^(\d+\.|-|\*|\+)\s", line.strip()): |
| if not inside_list and i > 0 and lines[i - 1].strip() != "": |
| normalized_lines.append("") |
| inside_list = True |
| normalized_lines.append(line) |
| elif inside_list and line.strip() == "": |
| if i < len(lines) - 1 and not re.match( |
| r"^(\d+\.|-|\*|\+)\s", lines[i + 1].strip() |
| ): |
| normalized_lines.append(line) |
| continue |
| else: |
| inside_list = False |
| normalized_lines.append(line) |
|
|
| return "\n".join(normalized_lines) |
|
|
|
|
| def convert_mdtext(md_text): |
| code_block_pattern = re.compile(r"```(.*?)(?:```|$)", re.DOTALL) |
| inline_code_pattern = re.compile(r"`(.*?)`", re.DOTALL) |
| code_blocks = code_block_pattern.findall(md_text) |
| non_code_parts = code_block_pattern.split(md_text)[::2] |
|
|
| result = [] |
| for non_code, code in zip(non_code_parts, code_blocks + [""]): |
| if non_code.strip(): |
| non_code = normalize_markdown(non_code) |
| if inline_code_pattern.search(non_code): |
| result.append(markdown(non_code, extensions=["tables"])) |
| else: |
| result.append(mdtex2html.convert(non_code, extensions=["tables"])) |
| if code.strip(): |
| code = f"\n```{code}\n\n```" |
| code = markdown_to_html_with_syntax_highlight(code) |
| result.append(code) |
| result = "".join(result) |
| result += ALREADY_CONVERTED_MARK |
| return result |
|
|
| def convert_asis(userinput): |
| return f"<p style=\"white-space:pre-wrap;\">{html.escape(userinput)}</p>"+ALREADY_CONVERTED_MARK |
|
|
| def detect_converted_mark(userinput): |
| if userinput.endswith(ALREADY_CONVERTED_MARK): |
| return True |
| else: |
| return False |
|
|
|
|
|
|
| def detect_language(code): |
| if code.startswith("\n"): |
| first_line = "" |
| else: |
| first_line = code.strip().split("\n", 1)[0] |
| language = first_line.lower() if first_line else "" |
| code_without_language = code[len(first_line) :].lstrip() if first_line else code |
| return language, code_without_language |
|
|
| def convert_to_markdown(text): |
| text = text.replace("$","$") |
| def replace_leading_tabs_and_spaces(line): |
| new_line = [] |
| |
| for char in line: |
| if char == "\t": |
| new_line.append("	") |
| elif char == " ": |
| new_line.append(" ") |
| else: |
| break |
| return "".join(new_line) + line[len(new_line):] |
|
|
| markdown_text = "" |
| lines = text.split("\n") |
| in_code_block = False |
|
|
| for line in lines: |
| if in_code_block is False and line.startswith("```"): |
| in_code_block = True |
| markdown_text += f"{line}\n" |
| elif in_code_block is True and line.startswith("```"): |
| in_code_block = False |
| markdown_text += f"{line}\n" |
| elif in_code_block: |
| markdown_text += f"{line}\n" |
| else: |
| line = replace_leading_tabs_and_spaces(line) |
| line = re.sub(r"^(#)", r"\\\1", line) |
| markdown_text += f"{line} \n" |
|
|
| return markdown_text |
|
|
| def add_language_tag(text): |
| def detect_language(code_block): |
| try: |
| lexer = guess_lexer(code_block) |
| return lexer.name.lower() |
| except ClassNotFound: |
| return "" |
|
|
| code_block_pattern = re.compile(r"(```)(\w*\n[^`]+```)", re.MULTILINE) |
|
|
| def replacement(match): |
| code_block = match.group(2) |
| if match.group(2).startswith("\n"): |
| language = detect_language(code_block) |
| if language: |
| return f"```{language}{code_block}```" |
| else: |
| return f"```\n{code_block}```" |
| else: |
| return match.group(1) + code_block + "```" |
|
|
| text2 = code_block_pattern.sub(replacement, text) |
| return text2 |
|
|
| def delete_last_conversation(chatbot, history): |
| if len(chatbot) > 0: |
| chatbot.pop() |
|
|
| if len(history) > 0: |
| history.pop() |
| |
| return ( |
| chatbot, |
| history, |
| "Delete Done", |
| ) |
|
|
| def reset_state(): |
| return [], [], "Reset Done" |
|
|
| def reset_textbox(): |
| return gr.update(value=""),"" |
|
|
| def cancel_outputing(): |
| return "Stop Done" |
|
|
|
|
| class State: |
| interrupted = False |
|
|
| def interrupt(self): |
| self.interrupted = True |
|
|
| def recover(self): |
| self.interrupted = False |
| shared_state = State() |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| def is_stop_word_or_prefix(s: str, stop_words: list) -> bool: |
| for stop_word in stop_words: |
| if s.endswith(stop_word): |
| return True |
| for i in range(1, len(stop_word)): |
| if s.endswith(stop_word[:i]): |
| return True |
| return False |
|
|
|
|
|
|
