config.yaml

# ============================================================
# Code Generation Assistant - Project Configuration
# ============================================================
# Central config so every phase reads the same settings.
# Change `use_sample: true` to test the pipeline on synthetic
# data; set false to pull the real CodeSearchNet from HuggingFace.

data:
  # HuggingFace dataset id. The canonical id is "code_search_net".
  # If that fails to load, try the community mirror:
  #   "code-search-net/code_search_net"
  hf_dataset_id: "code_search_net"
  # Languages to include. Start with python only; add "java" etc. later.
  languages:
    - python
  # When true, the pipeline uses a small synthetic sample instead of
  # downloading the real dataset (useful for offline testing / CI).
  use_sample: false
  sample_size: 200          # rows generated when use_sample is true
  max_rows: 5000            # cap on real HF data (0 = no cap); keeps local runs fast

cleaning:
  min_doc_words: 3          # drop docstrings shorter than this (words)
  max_doc_words: 120        # drop overly long docstrings (likely noise)
  min_code_chars: 20        # drop trivially short functions
  max_code_tokens: 512      # drop functions longer than this (token budget)
  drop_exact_duplicates: true
  drop_non_ascii_docs: true # drop docstrings that are mostly non-ASCII
  # Substrings that flag low-quality / autogenerated docstrings.
  doc_blocklist:
    - "todo"
    - "fixme"
    - "auto-generated"
    - "autogenerated"
    - "do not edit"

split:
  train: 0.8
  val: 0.1
  test: 0.1
  seed: 42

models:
  embed_model: "sentence-transformers/all-MiniLM-L6-v2"
  gen_model: "Qwen/Qwen2.5-Coder-0.5B-Instruct"
  top_k: 3

paths:
  data_dir: "data"
  raw_dir: "data/raw"
  processed_dir: "data/processed"
  eda_dir: "data/eda"
  index_dir: "data/index"