Deploy (see actual commits on https://github.com/mlcommons/croissant).

app.py

@@ -7,7 +7,7 @@ from core.constants import OAUTH_STATE
 from core.constants import REDIRECT_URI
 from core.query_params import get_project_timestamp
 from core.state import CurrentProject
-from core.state import get_cached_user
+from core.state import get_user
 from core.state import User
 from utils import init_state
 from views.splash import render_splash
@@ -19,7 +19,7 @@ col1.header("Croissant Editor")
 
 init_state()
 
-user = get_cached_user()
+user = get_user()
 
 if OAUTH_CLIENT_ID and not user:
     query_params = st.experimental_get_query_params()
@@ -31,8 +31,7 @@ if OAUTH_CLIENT_ID and not user:
         try:
             st.session_state[User] = User.connect(code)
             # Clear the cache to force retrieving the new user.
-            get_cached_user.clear()
-            get_cached_user()
+            get_user()
         except:
             raise
         finally:
@@ -56,7 +55,6 @@ def _back_to_menu():
 def _logout():
     """Logs the user out."""
     st.cache_data.clear()
-    get_cached_user.clear()
     st.session_state[User] = None
     _back_to_menu()
 

core/constants.py

@@ -35,8 +35,3 @@ METADATA = "Metadata"
 RESOURCES = "Resources"
 RECORD_SETS = "Record Sets"
 TABS = [OVERVIEW, METADATA, RESOURCES, RECORD_SETS]
-
-NAMES_INFO = (
-    "Names are used as identifiers. They are unique and cannot contain special"
-    " characters. The interface will replace any special characters."
-)

core/names.py

@@ -1,13 +1,8 @@
 """Module to handle naming of RecordSets and distribution."""
 
-import re
-
-NAME_PATTERN_REGEX = "[^a-zA-Z0-9\\-_\\.]"
-
 
 def find_unique_name(names: set[str], name: str):
     """Find a unique UID."""
-    name = re.sub(NAME_PATTERN_REGEX, "_", name)
     while name in names:
         name = f"{name}_0"
     return name

core/names_test.py

@@ -5,7 +5,6 @@ from .names import find_unique_name
 
 def test_find_unique_name():
     names = set(["first", "second", "first_0"])
-    assert find_unique_name(names, "are there spaces") == "are_there_spaces"
     assert find_unique_name(names, "first") == "first_0_0"
     assert find_unique_name(names, "second") == "second_0"
     assert find_unique_name(names, "third") == "third"

core/past_projects.py

@@ -8,12 +8,12 @@ from core.constants import PAST_PROJECTS_PATH
 from core.query_params import set_project
 from core.state import CurrentProject
 from core.state import FileObject
-from core.state import get_cached_user
+from core.state import get_user
 from core.state import Metadata
 
 
 def load_past_projects_paths() -> list[epath.Path]:
-    user = get_cached_user()
+    user = get_user()
     past_projects_path = PAST_PROJECTS_PATH(user)
     past_projects_path.mkdir(parents=True, exist_ok=True)
     return sorted(list(past_projects_path.iterdir()), reverse=True)

core/state.py

@@ -83,9 +83,8 @@ class User:
         )
 
 
-@st.cache_data(ttl=datetime.timedelta(hours=1))
-def get_cached_user():
-    """Caches user in session_state."""
+def get_user():
+    """Get user from session_state."""
     return st.session_state.get(User)
 
 
@@ -102,7 +101,7 @@ class CurrentProject:
 
     @classmethod
     def from_timestamp(cls, timestamp: str) -> CurrentProject | None:
-        user = get_cached_user()
+        user = get_user()
         if user is None and OAUTH_CLIENT_ID:
             return None
         else:

events/metadata.py

@@ -2,7 +2,6 @@ import enum
 
 import streamlit as st
 
-from core.names import find_unique_name
 from core.state import Metadata
 
 # List from:
@@ -98,7 +97,7 @@ class MetadataEvent(enum.Enum):
 
 def handle_metadata_change(event: MetadataEvent, metadata: Metadata, key: str):
     if event == MetadataEvent.NAME:
-        metadata.name = find_unique_name(set(), st.session_state[key])
+        metadata.name = st.session_state[key]
     elif event == MetadataEvent.DESCRIPTION:
         metadata.description = st.session_state[key]
     elif event == MetadataEvent.LICENSE:

views/files.py

@@ -3,7 +3,6 @@ import streamlit as st
 from components.safe_button import button_with_confirmation
 from components.tree import render_tree
 from core.constants import DF_HEIGHT
-from core.constants import NAMES_INFO
 from core.constants import OAUTH_CLIENT_ID
 from core.files import code_to_index
 from core.files import file_from_form
@@ -203,11 +202,6 @@ def _render_resource(prefix: int, file: Resource, is_file_object: bool):
         default=file.contained_in,
         options=parent_options,
         key=key,
-        help=(
-            "FileObjects and FileSets can be nested. Specifying `Parents` allows to"
-            " nest a FileObject/FileSet within another FileObject/FileSet. An example"
-            " of this is when images (FileSet) are nested within an archive (FileSet)."
-        ),
         on_change=handle_resource_change,
         args=(ResourceEvent.CONTAINED_IN, file, key),
     )
@@ -216,7 +210,6 @@ def _render_resource(prefix: int, file: Resource, is_file_object: bool):
         needed_field("Name"),
         value=file.name,
         key=key,
-        help=f"The name of the resource. {NAMES_INFO}",
         on_change=handle_resource_change,
         args=(ResourceEvent.NAME, file, key),
     )
@@ -232,10 +225,9 @@ def _render_resource(prefix: int, file: Resource, is_file_object: bool):
     if is_file_object:
         key = f"{prefix}_content_url"
         st.text_input(
-            needed_field("Content URL or local path"),
+            needed_field("Content URL"),
             value=file.content_url,
             key=key,
-            help="The URL or local file path pointing to the original FileObject.",
             on_change=handle_resource_change,
             args=(ResourceEvent.CONTENT_URL, file, key),
         )
@@ -252,7 +244,6 @@ def _render_resource(prefix: int, file: Resource, is_file_object: bool):
             "Content size",
             value=file.content_size,
             key=key,
-            help="The size of the original FileObject in bytes.",
             on_change=handle_resource_change,
             args=(ResourceEvent.CONTENT_SIZE, file, key),
         )
@@ -271,10 +262,6 @@ def _render_resource(prefix: int, file: Resource, is_file_object: bool):
         index=code_to_index(file.encoding_format),
         options=FILE_TYPES.keys(),
         key=key,
-        help=(
-            "MIME type corresponding to"
-            " ([sc:encodingFormat](https://schema.org/encodingFormat))."
-        ),
         on_change=handle_resource_change,
         args=(ResourceEvent.ENCODING_FORMAT, file, key),
     )

views/overview.py

@@ -3,7 +3,6 @@ from typing import Any
 
 import streamlit as st
 
-from core.constants import NAMES_INFO
 from core.state import Metadata
 import mlcroissant as mlc
 from utils import needed_field
@@ -52,7 +51,6 @@ def render_overview():
             label=needed_field("Name"),
             key=key,
             value=metadata.name,
-            help=f"The name of the dataset. {NAMES_INFO}",
             placeholder="Dataset",
             on_change=handle_metadata_change,
             args=(MetadataEvent.NAME, metadata, key),
@@ -84,14 +82,7 @@ def render_overview():
             * 100
             / (3 * metadata_weight)
         )
-        col_a.metric(
-            "Completion",
-            f"{completion}%",
-            help=(
-                "Approximation of the total completion based on the number of fields"
-                " that are filled."
-            ),
-        )
+        col_a.metric("Completion", f"{completion}%")
         col_b.metric("Number of metadata fields", fields)
         col_c.metric("Number of resources", len(metadata.distribution))
         col_d.metric("Number of RecordSets", len(metadata.record_sets))

views/record_sets.py

@@ -10,7 +10,6 @@ from rdflib import term
 import streamlit as st
 
 from components.safe_button import button_with_confirmation
-from core.constants import NAMES_INFO
 from core.data_types import MLC_DATA_TYPES
 from core.data_types import mlc_to_str_data_type
 from core.data_types import STR_DATA_TYPES
@@ -241,7 +240,6 @@ def _render_left_panel():
                 needed_field("Name"),
                 placeholder="Name without special character.",
                 key=key,
-                help=f"The name of the RecordSet. {NAMES_INFO}",
                 value=record_set.name,
                 on_change=handle_record_set_change,
                 args=(RecordSetEvent.NAME, record_set, key),
@@ -259,13 +257,6 @@ def _render_left_panel():
             st.checkbox(
                 "The RecordSet is an enumeration",
                 key=key,
-                help=(
-                    "Enumerations indicate that the RecordSet takes its values from a"
-                    " finite set. Similar to `ClassLabel` in"
-                    " [TFDS](https://www.tensorflow.org/datasets/api_docs/python/tfds/features/ClassLabel)"
-                    " or [Hugging"
-                    " Face](https://huggingface.co/docs/datasets/v2.15.0/en/package_reference/main_classes#datasets.ClassLabel)."
-                ),
                 value=record_set.is_enumeration,
                 on_change=handle_record_set_change,
                 args=(RecordSetEvent.IS_ENUMERATION, record_set, key),
@@ -274,10 +265,6 @@ def _render_left_panel():
             st.checkbox(
                 "The RecordSet has in-line data",
                 key=key,
-                help=(
-                    "In-line data allows to embed data directly within the JSON-LD"
-                    " without referencing another data source."
-                ),
                 value=bool(record_set.data),
                 on_change=handle_record_set_change,
                 args=(RecordSetEvent.HAS_DATA, record_set, key),
@@ -337,14 +324,8 @@ def _render_left_panel():
             )
             data_editor_key = _data_editor_key(record_set_key, record_set)
             st.markdown(
-                needed_field("Fields"),
-                help=(
-                    "Add/delete fields by directly editing the table. **Warning**: the"
-                    " table contains information about the fields--not the data"
-                    " directly. If you wish to embed data, tick the `The RecordSet is"
-                    " an enumeration` box. To edit fields details, click the"
-                    " button `Edit fields details` below."
-                ),
+                f"{needed_field('Fields')} (add/delete fields by directly editing the"
+                " table)"
             )
             st.data_editor(
                 fields,
@@ -456,7 +437,6 @@ def _render_right_panel():
                     needed_field("Name"),
                     placeholder="Name without special character.",
                     key=key,
-                    help=f"The name of the field. {NAMES_INFO}",
                     value=field.name,
                     on_change=handle_field_change,
                     args=(FieldEvent.NAME, field, key),
@@ -470,29 +450,32 @@ def _render_right_panel():
                     value=field.description,
                     args=(FieldEvent.DESCRIPTION, field, key),
                 )
-                data_type_index = None
                 if field.data_types:
                     data_type = field.data_types[0]
                     if isinstance(data_type, str):
                         data_type = term.URIRef(data_type)
                     if data_type in MLC_DATA_TYPES:
                         data_type_index = MLC_DATA_TYPES.index(data_type)
+                    else:
+                        data_type_index = None
+                else:
+                    data_type_index = None
                 key = f"{prefix}-datatypes"
                 col3.selectbox(
                     needed_field("Data type"),
                     index=data_type_index,
                     options=STR_DATA_TYPES,
                     key=key,
-                    help=(
-                        "The type of the data. `Text` corresponds to"
-                        " https://schema.org/Text, etc."
-                    ),
                     on_change=handle_field_change,
                     args=(FieldEvent.DATA_TYPE, field, key),
                 )
                 possible_sources = _get_possible_sources(metadata)
-                render_source(record_set, field, possible_sources)
-                render_references(record_set, field, possible_sources)
+                render_source(
+                    record_set_key, record_set, field, field_key, possible_sources
+                )
+                render_references(
+                    record_set_key, record_set, field, field_key, possible_sources
+                )
 
                 st.divider()
 

views/source.py

@@ -12,15 +12,6 @@ from events.fields import TransformType
 import mlcroissant as mlc
 from utils import needed_field
 
-_JSON_PATH_DOCUMENTATION = (
-    "The JSON path if the data source is a JSON (see"
-    " [documentation](https://www.ietf.org/archive/id/draft-goessner-dispatch-jsonpath-00.html))."
-)
-_EXTRACT_DOCUMENTATION = (
-    "The extraction method to get the value of the field (column in a CSV, etc)."
-)
-_COLUMN_NAME_DOCUMENTATION = "The name of the column if the data source is a CSV."
-
 
 class SourceType:
     """The type of the source (distribution or field)."""
@@ -114,8 +105,10 @@ def _handle_remove_reference(field):
 
 
 def render_source(
+    record_set_key: int,
     record_set: RecordSet,
     field: Field,
+    field_key: int,
     possible_sources: list[str],
 ):
     """Renders the form for the source."""
@@ -130,13 +123,10 @@ def render_source(
         index = None
     key = f"{prefix}-source"
     col1.selectbox(
-        needed_field("Data source"),
+        needed_field("Source"),
         index=index,
         options=options,
         key=key,
-        help=(
-            "Data sources can be other resources (FileObject, FileSet) or other fields."
-        ),
         on_change=handle_field_change,
         args=(FieldEvent.SOURCE, field, key),
     )
@@ -145,7 +135,6 @@ def render_source(
             needed_field("Extract"),
             index=_get_extract_index(source),
             key=f"{prefix}-extract",
-            help=_EXTRACT_DOCUMENTATION,
             options=EXTRACT_TYPES,
             on_change=handle_field_change,
             args=(FieldEvent.SOURCE_EXTRACT, field, key),
@@ -156,7 +145,6 @@ def render_source(
                 needed_field("Column name"),
                 value=source.extract.column,
                 key=key,
-                help=_COLUMN_NAME_DOCUMENTATION,
                 on_change=handle_field_change,
                 args=(FieldEvent.SOURCE_EXTRACT_COLUMN, field, key),
             )
@@ -166,7 +154,6 @@ def render_source(
                 needed_field("JSON path"),
                 value=source.extract.json_path,
                 key=key,
-                help=_JSON_PATH_DOCUMENTATION,
                 on_change=handle_field_change,
                 args=(FieldEvent.SOURCE_EXTRACT_JSON_PATH, field, key),
             )
@@ -183,23 +170,18 @@ def render_source(
                 key=key,
                 options=TRANSFORM_TYPES,
                 on_change=handle_field_change,
-                help="One or more transformations to apply after extracting the field.",
                 args=(FieldEvent.TRANSFORM, field, key),
                 kwargs={"number": number},
             )
             if selected == TransformType.FORMAT:
                 key = f"{prefix}-{number}-transform-format"
                 col3.text_input(
-                    needed_field("Format a date"),
+                    needed_field("Format"),
                     value=transform.format,
                     key=key,
                     on_change=handle_field_change,
-                    help=(
-                        "For dates, use [`Python format"
-                        " codes`](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes)."
-                    ),
                     args=(selected, field, key),
-                    kwargs={"number": number},
+                    kwargs={"number": number, "type": "format"},
                 )
             elif selected == TransformType.JSON_PATH:
                 key = f"{prefix}-{number}-jsonpath"
@@ -208,9 +190,8 @@ def render_source(
                     value=transform.json_path,
                     key=key,
                     on_change=handle_field_change,
-                    help=_JSON_PATH_DOCUMENTATION,
                     args=(selected, field, key),
-                    kwargs={"number": number},
+                    kwargs={"number": number, "type": "format"},
                 )
             elif selected == TransformType.REGEX:
                 key = f"{prefix}-{number}-regex"
@@ -219,14 +200,8 @@ def render_source(
                     value=transform.regex,
                     key=key,
                     on_change=handle_field_change,
-                    help=(
-                        "A regular expression following [`re` Python"
-                        " convention](https://docs.python.org/3/library/re.html#regular-expression-syntax)"
-                        " with one capturing group. The result of the operation will be"
-                        " the last captured group."
-                    ),
                     args=(selected, field, key),
-                    kwargs={"number": number},
+                    kwargs={"number": number, "type": "format"},
                 )
             elif selected == TransformType.REPLACE:
                 key = f"{prefix}-{number}-replace"
@@ -235,13 +210,8 @@ def render_source(
                     value=transform.replace,
                     key=key,
                     on_change=handle_field_change,
-                    help=(
-                        "A replace pattern separated by a `/`, i.e."
-                        " `string_to_replace/string_to_substitute` in order to replace"
-                        " `string_to_replace` by `string_to_substitute`."
-                    ),
                     args=(selected, field, key),
-                    kwargs={"number": number},
+                    kwargs={"number": number, "type": "format"},
                 )
             elif selected == TransformType.SEPARATOR:
                 key = f"{prefix}-{number}-separator"
@@ -250,9 +220,8 @@ def render_source(
                     value=transform.separator,
                     key=key,
                     on_change=handle_field_change,
-                    help="A separator to split strings on, e.g. `|` to split `a|b|c`.",
                     args=(selected, field, key),
-                    kwargs={"number": number},
+                    kwargs={"number": number, "type": "format"},
                 )
 
             def _handle_remove_transform(field, number):
@@ -261,7 +230,6 @@ def render_source(
             col4.button(
                 "✖️",
                 key=f"{prefix}-{number}-remove-transform",
-                help="Remove the transformation.",
                 on_click=_handle_remove_transform,
                 args=(field, number),
             )
@@ -275,15 +243,16 @@ def render_source(
     col1.button(
         "Add transform on data",
         key=f"{prefix}-close-fields",
-        help="Add a transformation.",
         on_click=_handle_add_transform,
         args=(field,),
     )
 
 
 def render_references(
+    record_set_key: int,
     record_set: RecordSet,
     field: Field,
+    field_key: int,
     possible_sources: list[str],
 ):
     """Renders the form for references."""
@@ -317,7 +286,6 @@ def render_references(
                 index=_get_extract_index(references),
                 key=key,
                 options=EXTRACT_TYPES,
-                help=_EXTRACT_DOCUMENTATION,
                 on_change=handle_field_change,
                 args=(FieldEvent.REFERENCE_EXTRACT, field, key),
             )
@@ -327,7 +295,6 @@ def render_references(
                     needed_field("Column name"),
                     value=references.extract.column,
                     key=key,
-                    help=_COLUMN_NAME_DOCUMENTATION,
                     on_change=handle_field_change,
                     args=(FieldEvent.REFERENCE_EXTRACT_COLUMN, field, key),
                 )
@@ -337,14 +304,12 @@ def render_references(
                     needed_field("JSON path"),
                     value=references.extract.json_path,
                     key=key,
-                    help=_JSON_PATH_DOCUMENTATION,
                     on_change=handle_field_change,
                     args=(FieldEvent.REFERENCE_EXTRACT_JSON_PATH, field, key),
                 )
         col4.button(
             "✖️",
             key=f"{key}-remove-reference",
-            help="Remove the join.",
             on_click=_handle_remove_reference,
             args=(field,),
         )

views/splash.py

@@ -13,8 +13,6 @@ import mlcroissant as mlc
 from views.load import render_load
 from views.previous_files import render_previous_files
 
-_HUGGING_FACE_URL = "https://huggingface.co/datasets/"
-
 _DATASETS = {
     "Titanic": ["data/embarkation_ports.csv", "data/genders.csv"],
     "FLORES-200": [],
@@ -36,7 +34,9 @@ def render_splash():
         )
     col1, col2 = st.columns([1, 1], gap="large")
     with col1:
-        with st.expander("**Create from scratch!**", expanded=True):
+        with st.expander("**Load an existing Croissant JSON-LD file**", expanded=True):
+            render_load()
+        with st.expander("**Create from scratch**", expanded=True):
 
             def create_new_croissant():
                 st.session_state[Metadata] = Metadata()
@@ -81,37 +81,6 @@ def render_splash():
                 type="primary",
                 args=(dataset,),
             )
-        with st.expander("**Load a dataset from Hugging Face!**", expanded=True):
-            url = st.text_input(
-                label="Hugging Face URL",
-                placeholder="https://huggingface.co/datasets/mnist",
-            )
-            if url.startswith(_HUGGING_FACE_URL):
-                url = url.replace(_HUGGING_FACE_URL, "")
-
-                def download_huggingface_json(name: str):
-                    api_url = f"https://datasets-server.huggingface.co/croissant?dataset={name}"
-                    json = requests.get(api_url, headers=None).json()
-                    try:
-                        metadata = mlc.Metadata.from_json(mlc.Issues(), json, None)
-                        st.session_state[Metadata] = Metadata.from_canonical(metadata)
-                        save_current_project()
-                    except Exception:
-                        st.error(f"Malformed JSON: {json}")
-
-                st.button(
-                    f'Download "{url}"',
-                    on_click=download_huggingface_json,
-                    type="primary",
-                    args=(url,),
-                )
-            elif url:
-                st.error(
-                    f"Unknown URL {url}. Hugging Face URLS should look like"
-                    f" {_HUGGING_FACE_URL}somedataset."
-                )
-        with st.expander("**Load an existing Croissant JSON-LD file**", expanded=True):
-            render_load()
     with col2:
         with st.expander("**Past projects**", expanded=True):
             render_previous_files()