{
  "id": "data_tokenization",
  "concept": "Data & Tokenization",
  "fiction": "The knight enters the Great Archive and meets the Runesmiths, learning how to mine raw knowledge and chisel it into mathematical runes.",
  "questions": [
    {
      "id": "dt_q1",
      "shown": {
        "question": "What is the name of the official Hugging Face library used to efficiently download, load, and process vast amounts of data?",
        "concept_brief": "Ask the knight for the name of the specific grimoire dedicated entirely to handling the great tomes of knowledge."
      },
      "check": {
        "strategy": "keyword",
        "spec": {
          "correct_any": [
            "datasets",
            "dataset"
          ]
        }
      },
      "reveal": "The `datasets` library."
    },
    {
      "id": "dt_q2",
      "shown": {
        "question": "Which main function from the `datasets` library is used to fetch a dataset from the Hub or a local file?",
        "concept_brief": "What is the precise incantation used to summon a tome from the archive to the reading desk?"
      },
      "check": {
        "strategy": "keyword",
        "spec": {
          "correct_any": [
            "load_dataset"
          ]
        }
      },
      "reveal": "The `load_dataset` function."
    },
    {
      "id": "dt_q3",
      "shown": {
        "question": "When a dataset is too massive to fit into your computer's memory (RAM), what feature allows you to load and process the data on-the-fly as you iterate through it?",
        "concept_brief": "If the tome is too heavy to lift, what magic allows a knight to read the pages as they flow past like a river?"
      },
      "check": {
        "strategy": "keyword",
        "spec": {
          "correct_any": [
            "streaming",
            "stream"
          ]
        }
      },
      "reveal": "Streaming."
    },
    {
      "id": "dt_q4",
      "shown": {
        "question": "What method is used to efficiently apply a transformation function to every single example in a dataset?",
        "concept_brief": "By what command does a scribe cast a translation spell over every single page of a tome at once?"
      },
      "check": {
        "strategy": "keyword",
        "spec": {
          "correct_any": [
            "map",
            "map()"
          ]
        }
      },
      "reveal": "The `map` method."
    },
    {
      "id": "dt_q5",
      "shown": {
        "question": "Explain in your own words why machine learning models cannot understand raw text directly and require tokenization.",
        "concept_brief": "Ask the knight why spirits of logic cannot hear mortal words. Why must speech be turned into numbers?"
      },
      "check": {
        "strategy": "llm_judge",
        "spec": {
          "rubric": [
            "Mentions that models are mathematical/computational engines",
            "Mentions the need to convert text into numbers or IDs"
          ]
        }
      },
      "reveal": "Models are mathematical entities and require text to be converted into numerical token IDs to process them."
    },
    {
      "id": "dt_q6",
      "shown": {
        "question": "Which class in the `transformers` library is most commonly used to automatically load the correct tokenizer for a given model?",
        "concept_brief": "What specific incantation from the grimoire summons the exact Runesmith matched to a spirit?"
      },
      "check": {
        "strategy": "keyword",
        "spec": {
          "correct_any": [
            "autotokenizer",
            "AutoTokenizer"
          ]
        }
      },
      "reveal": "The `AutoTokenizer` class."
    },
    {
      "id": "dt_q7",
      "shown": {
        "question": "Write one line of Python code that encodes the text \"hello\" into token IDs using a `tokenizer` object.",
        "concept_brief": "The Runesmith's own tool can transform plain speech -- invoke it upon the text."
      },
      "check": {
        "strategy": "ast",
        "spec": {
          "must_call_any": [
            "encode",
            "tokenizer"
          ]
        }
      },
      "reveal": "tokenizer(\"hello\") (or tokenizer.encode(\"hello\"))"
    },
    {
      "id": "dt_q8",
      "shown": {
        "question": "What is the term for the complete, fixed set of unique tokens a specific tokenizer knows how to process?",
        "concept_brief": "What is the name of the great ledger that lists every single rune a specific smith has ever learned?"
      },
      "check": {
        "strategy": "keyword",
        "spec": {
          "correct_any": [
            "vocabulary",
            "vocab"
          ]
        }
      },
      "reveal": "The vocabulary (or vocab)."
    },
    {
      "id": "dt_q9",
      "shown": {
        "question": "When processing multiple texts of different lengths in a batch, what technique is used to add empty tokens so they all match the length of the longest text?",
        "concept_brief": "When casting multiple spells at once, they must all be the same length. How do we fill the empty space in the shorter incantations?"
      },
      "check": {
        "strategy": "keyword",
        "spec": {
          "correct_any": [
            "padding",
            "pad"
          ]
        }
      },
      "reveal": "Padding."
    },
    {
      "id": "dt_q10",
      "shown": {
        "question": "When padding is applied, what secondary array of 1s and 0s is generated to tell the model which tokens are real text and which are just padding?",
        "concept_brief": "What is the name of the filter that tells the spirit to ignore the empty filler runes and focus only on the true magic?"
      },
      "check": {
        "strategy": "keyword",
        "spec": {
          "correct_any": [
            "attention mask",
            "attention_mask"
          ]
        }
      },
      "reveal": "The attention mask."
    }
  ]
}