fix: isolate uploaded Markdown from application templates
#13
by GuillaumeSalouHF HF Staff - opened
- 1_📝_form.py +8 -17
- middleMan.py +12 -3
- tests/test_markdown_rendering.py +70 -0
1_📝_form.py
CHANGED
|
@@ -8,7 +8,6 @@ import requests
|
|
| 8 |
from huggingface_hub import hf_hub_download, upload_file
|
| 9 |
import pandas as pd
|
| 10 |
from huggingface_hub import create_repo
|
| 11 |
-
import os
|
| 12 |
from middleMan import parse_into_jinja_markdown as pj
|
| 13 |
|
| 14 |
|
|
@@ -89,14 +88,6 @@ def validate(self, repo_type="model"):
|
|
| 89 |
raise exc
|
| 90 |
|
| 91 |
|
| 92 |
-
## Save uploaded [markdown] file to directory to be used by jinja parser function
|
| 93 |
-
def save_uploadedfile(uploadedfile):
|
| 94 |
-
with open(os.path.join("temp_uploaded_filed_Dir",uploadedfile.name),"wb") as f:
|
| 95 |
-
f.write(uploadedfile.getbuffer())
|
| 96 |
-
st.success("Saved File:{} to temp_uploaded_filed_Dir".format(uploadedfile.name))
|
| 97 |
-
return uploadedfile.name
|
| 98 |
-
|
| 99 |
-
|
| 100 |
def main_page():
|
| 101 |
|
| 102 |
|
|
@@ -166,6 +157,7 @@ def main_page():
|
|
| 166 |
|
| 167 |
"check_box": bool,
|
| 168 |
"markdown_upload":" ",
|
|
|
|
| 169 |
"legal_view":bool,
|
| 170 |
"researcher_view":bool,
|
| 171 |
"beginner_technical_view":bool,
|
|
@@ -229,12 +221,14 @@ def main_page():
|
|
| 229 |
|
| 230 |
# Read a single file
|
| 231 |
uploaded_file = st.file_uploader("Choose a file", type = ['md'], help = 'Please choose a markdown (.md) file type to upload')
|
|
|
|
| 232 |
if uploaded_file is not None:
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
|
|
|
| 238 |
|
| 239 |
elif st.session_state.task =='fill-mask' or 'translation' or 'token-classification' or ' sentence-similarity' or 'summarization' or 'question-answering' or 'text2text-generation' or 'text-classification' or 'text-generation' or 'conversational':
|
| 240 |
#st.session_state.markdown_upload = open(
|
|
@@ -249,9 +243,6 @@ def main_page():
|
|
| 249 |
#########################################
|
| 250 |
### Uploading model card to HUB
|
| 251 |
#########################################
|
| 252 |
-
out_markdown =open( st.session_state.markdown_upload, "r+"
|
| 253 |
-
).read()
|
| 254 |
-
print_out_final = f"{out_markdown}"
|
| 255 |
st.markdown("## Export Loaded Model Card to Hub")
|
| 256 |
with st.form("Upload to 🤗 Hub"):
|
| 257 |
st.markdown("Use a token with write access from [here](https://hf.co/settings/tokens)")
|
|
|
|
| 8 |
from huggingface_hub import hf_hub_download, upload_file
|
| 9 |
import pandas as pd
|
| 10 |
from huggingface_hub import create_repo
|
|
|
|
| 11 |
from middleMan import parse_into_jinja_markdown as pj
|
| 12 |
|
| 13 |
|
|
|
|
| 88 |
raise exc
|
| 89 |
|
| 90 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 91 |
def main_page():
|
| 92 |
|
| 93 |
|
|
|
|
| 157 |
|
| 158 |
"check_box": bool,
|
| 159 |
"markdown_upload":" ",
|
| 160 |
+
"uploaded_markdown": None,
|
| 161 |
"legal_view":bool,
|
| 162 |
"researcher_view":bool,
|
| 163 |
"beginner_technical_view":bool,
|
|
|
|
| 221 |
|
| 222 |
# Read a single file
|
| 223 |
uploaded_file = st.file_uploader("Choose a file", type = ['md'], help = 'Please choose a markdown (.md) file type to upload')
|
| 224 |
+
st.session_state.uploaded_markdown = None
|
| 225 |
if uploaded_file is not None:
|
| 226 |
+
try:
|
| 227 |
+
st.session_state.uploaded_markdown = uploaded_file.getvalue().decode("utf-8-sig")
|
| 228 |
+
except UnicodeDecodeError:
|
| 229 |
+
st.error("Please upload a UTF-8 Markdown file.")
|
| 230 |
+
st.stop()
|
| 231 |
+
st.success("Loaded Markdown for this session.")
|
| 232 |
|
| 233 |
elif st.session_state.task =='fill-mask' or 'translation' or 'token-classification' or ' sentence-similarity' or 'summarization' or 'question-answering' or 'text2text-generation' or 'text-classification' or 'text-generation' or 'conversational':
|
| 234 |
#st.session_state.markdown_upload = open(
|
|
|
|
| 243 |
#########################################
|
| 244 |
### Uploading model card to HUB
|
| 245 |
#########################################
|
|
|
|
|
|
|
|
|
|
| 246 |
st.markdown("## Export Loaded Model Card to Hub")
|
| 247 |
with st.form("Upload to 🤗 Hub"):
|
| 248 |
st.markdown("Use a token with write access from [here](https://hf.co/settings/tokens)")
|
middleMan.py
CHANGED
|
@@ -6,6 +6,7 @@ from markdownTagExtract import tag_checker,listToString,to_markdown
|
|
| 6 |
#from specific_extraction import extract_it
|
| 7 |
from modelcards import CardData, ModelCard
|
| 8 |
from jinja2 import Environment, FileSystemLoader
|
|
|
|
| 9 |
|
| 10 |
|
| 11 |
def is_float(value):
|
|
@@ -17,8 +18,16 @@ def is_float(value):
|
|
| 17 |
|
| 18 |
## Handles parsing jinja variable templates
|
| 19 |
def parse_into_jinja_markdown():
|
| 20 |
-
|
| 21 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
# to add:
|
| 23 |
# - parent model
|
| 24 |
# to fix:
|
|
@@ -133,4 +142,4 @@ def extract_section(current_template, start_tag, end_tag):
|
|
| 133 |
|
| 134 |
def main():
|
| 135 |
#card.save('current_card.md')
|
| 136 |
-
return
|
|
|
|
| 6 |
#from specific_extraction import extract_it
|
| 7 |
from modelcards import CardData, ModelCard
|
| 8 |
from jinja2 import Environment, FileSystemLoader
|
| 9 |
+
from pathlib import Path
|
| 10 |
|
| 11 |
|
| 12 |
def is_float(value):
|
|
|
|
| 18 |
|
| 19 |
## Handles parsing jinja variable templates
|
| 20 |
def parse_into_jinja_markdown():
|
| 21 |
+
# Imported cards are session-local data, never executable templates.
|
| 22 |
+
uploaded_markdown = st.session_state.get("uploaded_markdown")
|
| 23 |
+
if uploaded_markdown is not None:
|
| 24 |
+
return uploaded_markdown
|
| 25 |
+
|
| 26 |
+
template_name = st.session_state.markdown_upload
|
| 27 |
+
if template_name not in {"language_model_template1.md", "current_card.md"}:
|
| 28 |
+
raise ValueError("Only built-in model card templates can be rendered.")
|
| 29 |
+
env = Environment(loader=FileSystemLoader(Path(__file__).resolve().parent), autoescape=True)
|
| 30 |
+
temp = env.get_template(template_name)
|
| 31 |
# to add:
|
| 32 |
# - parent model
|
| 33 |
# to fix:
|
|
|
|
| 142 |
|
| 143 |
def main():
|
| 144 |
#card.save('current_card.md')
|
| 145 |
+
return
|
tests/test_markdown_rendering.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Run with: python -m unittest discover -s tests -v."""
|
| 2 |
+
import os
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
import tempfile
|
| 5 |
+
from types import SimpleNamespace
|
| 6 |
+
import unittest
|
| 7 |
+
from unittest.mock import patch
|
| 8 |
+
|
| 9 |
+
import middleMan
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class FormState(dict):
|
| 13 |
+
"""Blank form fixture; named fields are supplied by each test."""
|
| 14 |
+
|
| 15 |
+
def __missing__(self, key):
|
| 16 |
+
return ""
|
| 17 |
+
|
| 18 |
+
def __getattr__(self, key):
|
| 19 |
+
return self[key]
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class MarkdownRenderingTests(unittest.TestCase):
|
| 23 |
+
def render(self, **values):
|
| 24 |
+
state = FormState(markdown_upload="language_model_template1.md", model_name="Example model")
|
| 25 |
+
state.update(values)
|
| 26 |
+
with patch.object(middleMan, "st", SimpleNamespace(session_state=state)):
|
| 27 |
+
return middleMan.parse_into_jinja_markdown()
|
| 28 |
+
|
| 29 |
+
def test_imported_markdown_preserves_literal_template_syntax(self):
|
| 30 |
+
markdown = "# My card\n\n{{ 7 * 7 }}\n{% include 'README.md' %}\n"
|
| 31 |
+
self.assertEqual(self.render(uploaded_markdown=markdown), markdown)
|
| 32 |
+
|
| 33 |
+
def test_empty_import_is_not_replaced_with_a_default_template(self):
|
| 34 |
+
self.assertEqual(self.render(uploaded_markdown=""), "")
|
| 35 |
+
|
| 36 |
+
def test_imported_markdown_is_not_shared_with_another_session(self):
|
| 37 |
+
marker = "SESSION_A_ONLY"
|
| 38 |
+
self.assertEqual(self.render(uploaded_markdown=marker), marker)
|
| 39 |
+
self.assertNotIn(marker, self.render())
|
| 40 |
+
|
| 41 |
+
def test_only_built_in_templates_can_be_loaded(self):
|
| 42 |
+
for filename in ["README.md", "../language_model_template1.md", "/tmp/card.md"]:
|
| 43 |
+
with self.subTest(filename=filename), self.assertRaises(ValueError):
|
| 44 |
+
self.render(markdown_upload=filename)
|
| 45 |
+
|
| 46 |
+
def test_built_in_templates_still_render_form_fields(self):
|
| 47 |
+
for filename in ["language_model_template1.md", "current_card.md"]:
|
| 48 |
+
with self.subTest(filename=filename):
|
| 49 |
+
content = self.render(markdown_upload=filename)
|
| 50 |
+
self.assertIn("Example model", content)
|
| 51 |
+
self.assertNotIn("{{ model_id }}", content)
|
| 52 |
+
|
| 53 |
+
def test_form_values_are_data_even_for_built_in_templates(self):
|
| 54 |
+
self.assertIn("{{ 7 * 7 }}", self.render(model_name="{{ 7 * 7 }}"))
|
| 55 |
+
|
| 56 |
+
def test_working_directory_cannot_shadow_built_in_templates(self):
|
| 57 |
+
original_directory = Path.cwd()
|
| 58 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 59 |
+
Path(directory, "language_model_template1.md").write_text("SHADOW_TEMPLATE")
|
| 60 |
+
try:
|
| 61 |
+
os.chdir(directory)
|
| 62 |
+
content = self.render()
|
| 63 |
+
finally:
|
| 64 |
+
os.chdir(original_directory)
|
| 65 |
+
self.assertNotIn("SHADOW_TEMPLATE", content)
|
| 66 |
+
self.assertIn("Example model", content)
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
if __name__ == "__main__":
|
| 70 |
+
unittest.main()
|