Deploy (see actual commits on https://github.com/mlcommons/croissant).

app.py

@@ -20,7 +20,6 @@ col1.header("Croissant Editor")
 init_state()
 
 user = get_cached_user()
-print("USER", user)
 
 if OAUTH_CLIENT_ID and not user:
     query_params = st.experimental_get_query_params()

core/constants.py

@@ -33,5 +33,5 @@ DF_HEIGHT = 150
 OVERVIEW = "Overview"
 METADATA = "Metadata"
 RESOURCES = "Resources"
-RECORD_SETS = "RecordSets"
+RECORD_SETS = "Record Sets"
 TABS = [OVERVIEW, METADATA, RESOURCES, RECORD_SETS]

core/state.py

@@ -168,7 +168,7 @@ class RecordSet:
     """Record Set analogue for editor"""
 
     name: str = ""
-    data: Any = None
+    data: list[Any] | None = None
     description: str | None = None
     is_enumeration: bool | None = None
     key: str | list[str] | None = None
@@ -208,9 +208,14 @@ class Metadata:
         """Renames a RecordSet by changing all the references to this RecordSet."""
         for i, record_set in enumerate(self.record_sets):
             for j, field in enumerate(record_set.fields):
+                possible_uid = f"{old_name}/"
                 # Update source
                 source = field.source
-                if source and source.uid and source.uid.startswith(old_name):
+                if (
+                    source
+                    and source.uid
+                    and (source.uid.startswith(possible_uid) or source.uid == old_name)
+                ):
                     new_uid = source.uid.replace(old_name, new_name, 1)
                     self.record_sets[i].fields[j].source.uid = new_uid
                 # Update references
@@ -218,7 +223,10 @@ class Metadata:
                 if (
                     references
                     and references.uid
-                    and references.uid.startswith(old_name)
+                    and (
+                        references.uid.startswith(possible_uid)
+                        or references.uid == old_name
+                    )
                 ):
                     new_uid = references.uid.replace(old_name, new_name, 1)
                     self.record_sets[i].fields[j].references.uid = new_uid

deploy_to_hf.sh

@@ -3,12 +3,15 @@ echo "Deleting $HF_REPO..."
 rm -rf ${HF_REPO}
 git clone git@hf.co:spaces/marcenacp/croissant-editor ${HF_REPO}
 echo "Copying files from $PWD to $HF_REPO..."
-rsync -aP --exclude="README.md" --exclude="*node_modules*" --exclude="*__pycache__*" . ${HF_REPO}
+rsync -aP --exclude="README.md" --exclude="*node_modules*" --exclude="cypress/*" --exclude="*__pycache__*" . ${HF_REPO}
 cd ${HF_REPO}
-echo "Now push with: 'cd $HF_REPO && git add && git commit && git push'."
+git add .
+git commit -m "Deploy (see actual commits on https://github.com/mlcommons/croissant)."
+echo "Now push with: 'cd $HF_REPO && git push'."
 echo "Warning: if it fails, you may need to follow https://huggingface.co/docs/hub/security-git-ssh#generating-a-new-ssh-keypair"
 echo "On Hugging Face Spaces, you might have to set the following environment variables:"
 echo "- REDIRECT_URI"
 echo "- OAUTH_STATE"
 echo "- OAUTH_CLIENT_ID"
 echo "- OAUTH_CLIENT_SECRET"
+echo "Visit: https://huggingface.co/spaces/marcenacp/croissant-editor"

events/record_sets.py

@@ -13,6 +13,8 @@ class RecordSetEvent(enum.Enum):
     NAME = "NAME"
     DESCRIPTION = "DESCRIPTION"
     IS_ENUMERATION = "IS_ENUMERATION"
+    HAS_DATA = "HAS_DATA"
+    CHANGE_DATA = "CHANGE_DATA"
 
 
 def handle_record_set_change(event: RecordSetEvent, record_set: RecordSet, key: str):
@@ -28,4 +30,16 @@ def handle_record_set_change(event: RecordSetEvent, record_set: RecordSet, key:
         record_set.description = value
     elif event == RecordSetEvent.IS_ENUMERATION:
         record_set.is_enumeration = value
+    elif event == RecordSetEvent.HAS_DATA:
+        if value:
+            record_set.data = []
+        else:
+            record_set.data = None
+    elif event == RecordSetEvent.CHANGE_DATA:
+        for index, new_value in value["edited_rows"].items():
+            record_set.data[index] = {**record_set.data[index], **new_value}
+        for row in value["added_rows"]:
+            record_set.data.append(row)
+        for row in value["deleted_rows"]:
+            del record_set.data[row]
     expand_record_set(record_set=record_set)

views/foo.py.py

@@ -0,0 +1,36 @@
+import multiprocessing
+import time
+from typing import TypedDict
+
+
+class _Result(TypedDict):
+    bar: int
+
+
+def bar(result):
+    while True:
+        time.sleep(1)
+        result["bar"] += 1
+        print(result["bar"])
+        if result["bar"] > 5:
+            return
+
+
+def foo():
+    """Generates the data and waits at most _TIMEOUT_SECONDS."""
+    with multiprocessing.Manager() as manager:
+        result: _Result = manager.dict(bar=0)
+        process = multiprocessing.Process(target=bar, args=(result,))
+        process.start()
+        if not process.is_alive():
+            return result
+        time.sleep(3)
+        if process.is_alive():
+            process.kill()
+            result["exception"] = TimeoutError(
+                "The generation took too long and was killed."
+            )
+        return _Result(**result)
+
+
+print("FINAL RESULT", foo().get("bar"))

views/overview.py

@@ -1,3 +1,4 @@
+import dataclasses
 from typing import Any
 
 import streamlit as st
@@ -8,12 +9,22 @@ from utils import needed_field
 from views.metadata import handle_metadata_change
 from views.metadata import MetadataEvent
 
+_NON_RELEVANT_METADATA = ["name", "distribution", "record_sets", "rdf"]
 
-def _plural(array: list[Any]):
-    if array:
-        return "s"
-    else:
-        return ""
+_INFO_TEXT = """Croissant files are composed of three layers:
+
+- **Metadata** about the dataset covering Responsible AI, licensing and attributes of
+                [sc\:Dataset](https://schema.org/Dataset).
+- **Resources**: The contents of a dataset as the underlying files
+                ([`FileObject`](https://github.com/mlcommons/croissant/blob/main/docs/croissant-spec.md#fileobject))
+                and/or sets of files ([`FileSet`](https://github.com/mlcommons/croissant/blob/main/docs/croissant-spec.md#fileset)).
+- **RecordSets**: the sets of structured records obtained from one or more resources
+                (typically a file or set of files) and the structure of these records,
+                expressed as a set of fields (e.g., the columns of a table).
+
+The next three tabs will guide you through filling those layers. The errors if any will
+be displayed on this page. Once you are ready, you can download the dataset by clicking
+the export button in the upper right corner."""
 
 
 def render_overview():
@@ -21,7 +32,7 @@ def render_overview():
     col1, col2 = st.columns([1, 1], gap="medium")
     with col1:
         key = "metadata-name"
-        st.text_input(
+        name = st.text_input(
             label=needed_field("Name"),
             key=key,
             value=metadata.name,
@@ -29,8 +40,10 @@ def render_overview():
             on_change=handle_metadata_change,
             args=(MetadataEvent.NAME, metadata, key),
         )
+        if not name:
+            st.stop()
         key = "metadata-url"
-        st.text_input(
+        url = st.text_input(
             label=needed_field("URL"),
             key=key,
             value=metadata.url,
@@ -38,6 +51,8 @@ def render_overview():
             on_change=handle_metadata_change,
             args=(MetadataEvent.URL, metadata, key),
         )
+        if not url:
+            st.stop()
         key = "metadata-description"
         st.text_area(
             label="Description",
@@ -47,29 +62,35 @@ def render_overview():
             on_change=handle_metadata_change,
             args=(MetadataEvent.DESCRIPTION, metadata, key),
         )
-
-        st.subheader(
-            f"{len(metadata.distribution)} File" + _plural(metadata.distribution)
-        )
-        st.subheader(
-            f"{len(metadata.record_sets)} Record Set" + _plural(metadata.distribution)
-        )
+        st.divider()
+        left, middle, right = st.columns([1, 1, 1])
+        fields = [
+            field
+            for field, value in dataclasses.asdict(metadata).items()
+            if value and field not in _NON_RELEVANT_METADATA
+        ]
+        left.metric("Number of metadata", len(fields))
+        middle.metric("Number of resources", len(metadata.distribution))
+        right.metric("Number of RecordSets", len(metadata.record_sets))
     with col2:
         user_started_editing = metadata.record_sets or metadata.distribution
         if user_started_editing:
-            st.subheader("Croissant File Validation")
+            warning = ""
             try:
                 issues = metadata.to_canonical().issues
                 if issues.errors:
-                    st.markdown("##### Errors:")
+                    warning += "**Errors**\n"
                     for error in issues.errors:
-                        st.write(error)
+                        warning += f"{error}\n"
                 if issues.warnings:
-                    st.markdown("##### Warnings:")
+                    warning += "**Warnings**\n"
                     for warning in issues.warnings:
-                        st.write(warning)
-                if not issues.errors and not issues.warnings:
-                    st.write("No validation issues detected!")
+                        warning += f"{warning}\n"
             except mlc.ValidationError as exception:
-                st.markdown("##### Errors:")
-                st.write(str(exception))
+                warning += "**Errors**\n"
+                warning += f"{str(exception)}\n"
+            if warning:
+                st.warning(warning, icon="⚠️")
+            else:
+                st.success("No validation issues detected!", icon="✅")
+        st.info(_INFO_TEXT, icon="💡")

views/record_sets.py

@@ -1,4 +1,7 @@
-from typing import Any
+import multiprocessing
+import textwrap
+import time
+from typing import TypedDict
 
 import numpy as np
 import pandas as pd
@@ -28,6 +31,65 @@ DATA_TYPES = [
     mlc.DataType.URL,
 ]
 
+_NUM_RECORDS = 3
+_TIMEOUT_SECONDS = 1
+
+
+class _Result(TypedDict):
+    df: pd.DataFrame | None
+    exception: Exception | None
+
+
+@st.cache_data(show_spinner="Generating the dataset...")
+def _generate_data_with_timeout(record_set: RecordSet) -> _Result:
+    """Generates the data and waits at most _TIMEOUT_SECONDS."""
+    with multiprocessing.Manager() as manager:
+        result: _Result = manager.dict(df=None, exception=None)
+        args = (record_set, result)
+        process = multiprocessing.Process(target=_generate_data, args=args)
+        process.start()
+        if not process.is_alive():
+            return _Result(**result)
+        time.sleep(_TIMEOUT_SECONDS)
+        if process.is_alive():
+            process.kill()
+            result["exception"] = TimeoutError(
+                "The generation took too long and was killed. Please, use the CLI as"
+                " described in"
+                " https://github.com/mlcommons/croissant/tree/main/python/mlcroissant#verifyload-a-croissant-dataset."
+            )
+        return _Result(**result)
+
+
+def _generate_data(record_set: RecordSet, result: _Result) -> pd.DataFrame | None:
+    """Generates the first _NUM_RECORDS records."""
+    try:
+        metadata: Metadata = st.session_state[Metadata]
+        if not metadata:
+            raise ValueError(
+                "The dataset is still incomplete. Please, go to the overview to see"
+                " errors."
+            )
+        croissant = metadata.to_canonical()
+        if croissant:
+            dataset = mlc.Dataset.from_metadata(croissant)
+            records = iter(dataset.records(record_set=record_set.name))
+            df = []
+            for i, record in enumerate(iter(records)):
+                if i >= _NUM_RECORDS:
+                    break
+                # Decode bytes as str:
+                for key, value in record.items():
+                    if isinstance(value, bytes):
+                        try:
+                            record[key] = value.decode("utf-8")
+                        except:
+                            pass
+                df.append(record)
+            result["df"] = pd.DataFrame(df)
+    except Exception as exception:
+        result["exception"] = exception
+
 
 def _handle_close_fields():
     st.session_state[SelectedRecordSet] = None
@@ -116,23 +178,22 @@ def _handle_fields_change(record_set_key: int, record_set: RecordSet):
             name=added_row.get(FieldDataFrame.NAME),
             description=added_row.get(FieldDataFrame.DESCRIPTION),
             data_types=[added_row.get(FieldDataFrame.DATA_TYPE)],
-            source=mlc.Source(
-                uid="foo",
-                node_type="distribution",
-                extract=mlc.Extract(column=""),
-            ),
+            source=mlc.Source(),
             references=mlc.Source(),
         )
         st.session_state[Metadata].add_field(record_set_key, field)
     for field_key in result["deleted_rows"]:
         st.session_state[Metadata].remove_field(record_set_key, field_key)
+    # Reset the in-line data if it exists.
+    if record_set.data:
+        record_set.data = []
 
 
 class FieldDataFrame:
     """Names of the columns in the pd.DataFrame for `fields`."""
 
-    NAME = "Name"
-    DESCRIPTION = "Description"
+    NAME = "Field name"
+    DESCRIPTION = "Field description"
     DATA_TYPE = "Data type"
     SOURCE_UID = "Source"
     SOURCE_EXTRACT = "Source extract"
@@ -144,17 +205,14 @@ class FieldDataFrame:
 def render_record_sets():
     col1, col2 = st.columns([1, 1])
     with col1:
-        _render_left_panel()
+        with st.spinner("Generating the dataset..."):
+            _render_left_panel()
     with col2:
         _render_right_panel()
 
 
 def _render_left_panel():
     """Left panel: visualization of all RecordSets as expandable forms."""
-    distribution = st.session_state[Metadata].distribution
-    if not distribution:
-        st.markdown("Please add resources first.")
-        return
     record_sets = st.session_state[Metadata].record_sets
     record_set: RecordSet
     for record_set_key, record_set in enumerate(record_sets):
@@ -188,12 +246,20 @@ def _render_left_panel():
                 on_change=handle_record_set_change,
                 args=(RecordSetEvent.IS_ENUMERATION, record_set, key),
             )
+            key = f"{prefix}-has-data"
+            st.checkbox(
+                "Whether the RecordSet has in-line data",
+                key=key,
+                value=bool(record_set.data),
+                on_change=handle_record_set_change,
+                args=(RecordSetEvent.HAS_DATA, record_set, key),
+            )
 
             joins = _find_joins(record_set.fields)
             has_join = st.checkbox(
-                "Whether the RecordSet contains joins. To add a new join, add a"
-                f" field with a source in `{record_set.name}` and a reference to"
-                " another RecordSet or FileSet/FileObject.",
+                "Whether the RecordSet contains joins. To add a new join, add a field"
+                " with a source in `RecordSet`/`FileSet`/`FileObject` and a reference"
+                " to another `RecordSet`/`FileSet`/`FileObject`.",
                 key=f"{prefix}-has-joins",
                 value=bool(joins),
                 disabled=True,
@@ -248,8 +314,7 @@ def _render_left_panel():
             )
             st.data_editor(
                 fields,
-                # There is a bug with `st.data_editor` when the df is empty.
-                use_container_width=not fields.empty,
+                use_container_width=True,
                 num_rows="dynamic",
                 key=data_editor_key,
                 column_config={
@@ -273,6 +338,26 @@ def _render_left_panel():
                 on_change=_handle_fields_change,
                 args=(record_set_key, record_set),
             )
+            result: _Result = _generate_data_with_timeout(record_set)
+            df, exception = result.get("df"), result.get("exception")
+            if exception is None and df is not None and not df.empty:
+                st.markdown("Previsualize the data:")
+                st.dataframe(df, use_container_width=True)
+            # The generation is not triggered if record_set has in-line `data`.
+            elif not record_set.data:
+                left, right = st.columns([1, 10])
+                if exception:
+                    left.button(
+                        "⚠️",
+                        key=f"idea-{prefix}",
+                        disabled=True,
+                        help=textwrap.dedent(f"""**Error**:
+```
+{exception}
+```
+"""),
+                    )
+                right.markdown("No preview is possible.")
 
             st.button(
                 "Edit fields details",
@@ -297,56 +382,80 @@ def _render_right_panel():
     record_set = selected.record_set
     record_set_key = selected.record_set_key
     with st.expander("**Fields**", expanded=True):
-        for field_key, field in enumerate(record_set.fields):
-            prefix = f"{record_set_key}-{field.name}-{field_key}"
-            col1, col2, col3 = st.columns([1, 1, 1])
-
-            key = f"{prefix}-name"
-            col1.text_input(
-                needed_field("Name"),
-                placeholder="Name without special character.",
-                key=key,
-                value=field.name,
-                on_change=handle_field_change,
-                args=(FieldEvent.NAME, field, key),
+        if isinstance(record_set.data, list):
+            st.markdown(
+                f"{needed_field('Data')}. This RecordSet is marked as having in-line"
+                " data. Please, list the data below:"
             )
-            key = f"{prefix}-description"
-            col2.text_input(
-                "Description",
-                placeholder="Provide a clear description of the RecordSet.",
+            key = f"{record_set_key}-fields-data"
+            columns = [field.name for field in record_set.fields]
+            st.data_editor(
+                pd.DataFrame(record_set.data, columns=columns),
+                use_container_width=True,
+                num_rows="dynamic",
                 key=key,
-                on_change=handle_field_change,
-                value=field.description,
-                args=(FieldEvent.DESCRIPTION, field, key),
+                column_config={
+                    field.name: st.column_config.TextColumn(
+                        field.name,
+                        help=field.description,
+                        required=True,
+                    )
+                    for field in record_set.fields
+                },
+                on_change=handle_record_set_change,
+                args=(RecordSetEvent.CHANGE_DATA, record_set, key),
             )
-            if field.data_types:
-                data_type = field.data_types[0]
-                if isinstance(data_type, str):
-                    data_type = term.URIRef(data_type)
-                if data_type in DATA_TYPES:
-                    data_type_index = DATA_TYPES.index(data_type)
+        else:
+            for field_key, field in enumerate(record_set.fields):
+                prefix = f"{record_set_key}-{field.name}-{field_key}"
+                col1, col2, col3 = st.columns([1, 1, 1])
+
+                key = f"{prefix}-name"
+                col1.text_input(
+                    needed_field("Name"),
+                    placeholder="Name without special character.",
+                    key=key,
+                    value=field.name,
+                    on_change=handle_field_change,
+                    args=(FieldEvent.NAME, field, key),
+                )
+                key = f"{prefix}-description"
+                col2.text_input(
+                    "Description",
+                    placeholder="Provide a clear description of the RecordSet.",
+                    key=key,
+                    on_change=handle_field_change,
+                    value=field.description,
+                    args=(FieldEvent.DESCRIPTION, field, key),
+                )
+                if field.data_types:
+                    data_type = field.data_types[0]
+                    if isinstance(data_type, str):
+                        data_type = term.URIRef(data_type)
+                    if data_type in DATA_TYPES:
+                        data_type_index = DATA_TYPES.index(data_type)
+                    else:
+                        data_type_index = None
                 else:
                     data_type_index = None
-            else:
-                data_type_index = None
-            key = f"{prefix}-datatypes"
-            col3.selectbox(
-                needed_field("Data type"),
-                index=data_type_index,
-                options=DATA_TYPES,
-                key=key,
-                on_change=handle_field_change,
-                args=(FieldEvent.DATA_TYPE, field, key),
-            )
-            possible_sources = _get_possible_sources(metadata)
-            render_source(
-                record_set_key, record_set, field, field_key, possible_sources
-            )
-            render_references(
-                record_set_key, record_set, field, field_key, possible_sources
-            )
-
-            st.divider()
+                key = f"{prefix}-datatypes"
+                col3.selectbox(
+                    needed_field("Data type"),
+                    index=data_type_index,
+                    options=DATA_TYPES,
+                    key=key,
+                    on_change=handle_field_change,
+                    args=(FieldEvent.DATA_TYPE, field, key),
+                )
+                possible_sources = _get_possible_sources(metadata)
+                render_source(
+                    record_set_key, record_set, field, field_key, possible_sources
+                )
+                render_references(
+                    record_set_key, record_set, field, field_key, possible_sources
+                )
+
+                st.divider()
 
         st.button(
             "Close",