Update Alfa-Code pipeline: HF datasets + Hub connect

.env.example

@@ -0,0 +1,4 @@
+# HF_TOKEN=hf_xxx paste from https://huggingface.co/settings/tokens
+HF_TOKEN=
+HF_USERNAME=
+HF_DATASET_REPO=alfa-code-multimodal

.gitignore

@@ -0,0 +1,7 @@
+__pycache__/
+.venv/
+.env
+data/raw/
+data/processed/
+*.parquet
+*.bin

README.md

@@ -1,3 +1,21 @@
----
-license: apache-2.0
----
+# Alfa Code - Multimodal Coding GGUF
+
+## 1. Connect to HuggingFace Hub
+1. Token: https://huggingface.co/settings/tokens (read + write) -> copy `hf_xxx`
+2. `copy .env.example .env` then fill `HF_TOKEN=hf_xxx`
+3. Or one-shot:
+  `python scripts/hf_connect.py --token hf_xxx`
+
+Verify: `python scripts/hf_connect.py` -> should print `Logged in as: <you>`
+
+## 2. Get data (best HF sets)
+Public test (no login): `python scripts/download_data.py --dataset code_small_test --max-rows 500`
+Gated large (needs login):
+  `python scripts/hf_connect.py --token hf_xxx`
+  `python scripts/download_data.py --dataset code_pretrain --max-rows 1000 --streaming`
+
+Output: `data/processed/train.jsonl` with `{modality, prompt, think, answer}`
+
+## 3. Push to Hub
+`python scripts/hf_connect.py --create-repo --repo alfa-code-multimodal`
+`python scripts/hf_connect.py --upload data/processed --repo alfa-code-multimodal`

data/processed/train.jsonl

@@ -0,0 +1,5 @@
+{"modality": "code+reasoning", "prompt": "Create a Java class which sorts the given array of numbers.\n[9, 2, 4, 3, 6, 1]", "think": "", "answer": "class ArraySort { \n  \n    void sort(int arr[]) { \n        int n = arr.length; \n  \n        // One by one move boundary of unsorted subarray \n        for (int i = 0; i < n-1; i++) { \n            \n            // Find the minimum element in unsorted array \n            int min_index = i; \n            for (int j = i+1; j < n; j++) \n                if (arr[j] < arr[min_index]) \n                    min_index = j; \n  \n            // Swap the found minimum element with the first element \n            int temp = arr[min_index]; \n            arr[min_index] = arr[i]; \n            arr[i] = temp; \n        } \n    } \n  \n    // Prints the array \n    void printArray(int arr[]) { \n        int n = arr.length; \n        for (int i=0; i<n; ++i) \n            System.out.print(arr[i] + \" \"); \n        System.out.println(); \n    } \n  \n    // Driver code to test above \n    public static void main(String args[]) \n    { \n        ArraySort ob = new ArraySort(); \n        int arr[] = {9, 2, 4, 3, 6, 1}; \n        ob.sort(arr); \n        System.out.println(\"Sorted array\"); \n        ob.printArray(arr); \n    } \n}"}
+{"modality": "code+reasoning", "prompt": "Create an algorithm in JavaScript to find the closest pair of points from a given set of points.\nlet points = [[2, 0], [4, 4], [-2, 2], [-3, -4], [4, -2]];", "think": "", "answer": "// algorithm to find closest pair of points \n// uses divide and conquer approach \nconst closestPair = (points) => { \n  // base case: two or fewer points\n  if (points.length <= 2) {\n    return points;\n  }\n\n  // divide\n  const middle = Math.floor(points.length / 2);\n  const left = points.slice(0, middle);\n  const right = points.slice(middle);\n  \n  // conquer\n  const leftPoints = closestPair(left);\n  const rightPoints = closestPair(right);\n\n  const closestPair = getClosestPair(leftPoints, rightPoints);\n\n  // combine\n  return closestPair;\n}\n \n// get closest pair between two sets of points\nconst getClosestPair = (leftPoints, rightPoints) => {\n  // calculating the minimum distance\n  let minDist = getDistance(leftPoints[0], rightPoints[0]);\n  let closestPair = [leftPoints[0], rightPoints[0]];\n  \n  for (let i = 0; i < leftPoints.length; i++) {\n    for (let j = 0; j < rightPoints.length; j++) {\n      let dist = getDistance(leftPoints[i], rightPoints[j]);\n      if (dist < minDist) {\n        minDist = dist;\n        closestPair = [leftPoints[i], rightPoints[j]];\n      }\n    }\n  }\n \n  // return the minimum distance and its pair\n  return closestPair;\n}\n \n// calculate the Euclidean distance between two points\nconst getDistance = (point1, point2) => {\n  let xDist = Math.abs(point2[0] - point1[0]);\n  let yDist = Math.abs(point2[1] - point1[1]);\n \n  return Math.sqrt(Math.pow(xDist, 2) + Math.pow(yDist, 2));\n}\n\n// testing our function \nlet points = [[2, 0], [4, 4], [-2, 2], [-3, -4], [4, -2]];\nlet closestPoints = closestPair(points);\nconsole.log(closestPoints);\n//Output: [[-2, 2], [2, 0]]"}
+{"modality": "code+reasoning", "prompt": "Change the following loop so that it prints out the numbers from 10 to 1 in descending order.\nfor i in range(1, 11):\n    print(i)", "think": "", "answer": "for i in range(10, 0, -1):\n    print(i)"}
+{"modality": "code+reasoning", "prompt": "Write an SQL query that returns the 5 most popular books in the library.\nSELECT * FROM libraryBook", "think": "", "answer": "SELECT bookID, bookTitle, COUNT(*) AS Count \nFROM libraryBook \nGROUP BY bookID, bookTitle \nORDER BY Count DESC \nLIMIT 5;"}
+{"modality": "code+reasoning", "prompt": "Train an AI model to predict whether an email is spam or not.\n", "think": "", "answer": "import nltk\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.naive_bayes import MultinomialNB\n\n# loading data\ndata = [('This is an email about about a promotion', 'spam'),\n\t('We want to transfer money to your account', 'spam'),\n\t('This email is about programming', 'not_spam'),\n\t('There is a new version of python', 'not_spam'),\n]\n\n# extracting features\nX, y = [], []\nfor feat, lab in data:\n\tX.append(feat)\n\ty.append(lab)\n\ncv = CountVectorizer()\nX_vect = cv.fit_transform(X)\n\n# training the model\nmodel = MultinomialNB()\nmodel.fit(X_vect, y)\n\n# predicting\nprediction = model.predict(cv.transform([\"This is an email about a discount sale\"]))\nprint(prediction)\n\n# Output\n['spam']"}

requirements.txt

@@ -0,0 +1,6 @@
+huggingface_hub>=1.0
+datasets>=3.0
+transformers>=4.40
+hf_transfer>=0.1
+pyyaml>=6.0
+tqdm>=4.66

scripts/download_data.py

@@ -0,0 +1,53 @@
+"""Download + normalize HF datasets to data/processed/train.jsonl"""
+import argparse, json, os
+from pathlib import Path
+from datasets import load_dataset
+
+MAP = {
+    "code_small_test": ("HuggingFaceH4/CodeAlpaca_20K", None, None),  # public, no login needed
+    "code_pretrain": ("bigcode/the-stack-v2", None, None),  # gated, needs HF_TOKEN
+    "code_instruct": ("bigcode/self-oss-instruct", None, None),
+    "reasoning": ("Open-Orca/OpenOrca", None, None),
+    "image_to_code": ("HuggingFaceM4/websight", None, None),
+}
+
+OUT = Path("data/processed/train.jsonl")
+OUT.parent.mkdir(parents=True, exist_ok=True)
+
+def normalize_code_instruct(row):
+    # self-oss-instruct fields vary; keep raw + unified prompt
+    prompt = row.get("instruction") or row.get("prompt") or row.get("question") or ""
+    answer = row.get("output") or row.get("response") or row.get("answer") or ""
+    return {"modality": "code+reasoning", "prompt": prompt, "think": "", "answer": answer}
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--dataset", default="code_small_test", choices=list(MAP.keys()))
+    ap.add_argument("--max-rows", type=int, default=500)
+    ap.add_argument("--streaming", action="store_true", help="stream for large sets like the-stack-v2")
+    args = ap.parse_args()
+
+    ds_id, subset, text_col = MAP[args.dataset]
+    print(f"Loading {ds_id} ...")
+    ds = load_dataset(ds_id, subset, split="train", streaming=args.streaming)
+    if args.streaming:
+        it = iter(ds)
+        rows = [next(it) for _ in range(args.max_rows)]
+    else:
+        ds = ds.select(range(min(args.max_rows, len(ds))))
+        rows = list(ds)
+
+    with open(OUT, "a", encoding="utf-8") as f:
+        for r in rows:
+            if args.dataset == "code_small_test":
+                # CodeAlpaca_20K: {prompt, completion}
+                obj = {"modality": "code+reasoning", "prompt": r.get("prompt", "")[:4000], "think": "", "answer": r.get("completion", "")[:8000]}
+            elif args.dataset in ("code_instruct", "reasoning"):
+                obj = normalize_code_instruct(r)
+            else:
+                obj = {"modality": "image", "prompt": str(r)[:4000], "think": "", "answer": ""}
+            f.write(json.dumps(obj, ensure_ascii=False) + "\n")
+    print(f"Wrote {len(rows)} rows -> {OUT}")
+
+if __name__ == "__main__":
+    main()

scripts/hf_connect.py

@@ -0,0 +1,49 @@
+"""Connect to HuggingFace Hub: login, whoami, create dataset repo, upload."""
+import argparse, os, sys
+from pathlib import Path
+
+def get_token(args):
+    tok = args.token or os.getenv("HF_TOKEN")
+    if not tok and Path(".env").exists():
+        for line in Path(".env").read_text().splitlines():
+            if line.startswith("HF_TOKEN="):
+                tok = line.split("=", 1)[1].strip()
+    return tok
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--token", default=None, help="hf_xxx token")
+    ap.add_argument("--username", default=None)
+    ap.add_argument("--repo", default="alfa-code-multimodal")
+    ap.add_argument("--create-repo", action="store_true")
+    ap.add_argument("--upload", default=None, help="local path to upload, e.g. data/processed")
+    args = ap.parse_args()
+
+    from huggingface_hub import HfApi, login, whoami
+    tok = get_token(args)
+    if not tok:
+        print("No HF_TOKEN found. Create one at https://huggingface.co/settings/tokens (read+write), then:")
+        print("  python scripts/hf_connect.py --token hf_xxx")
+        print("  # or copy .env.example to .env and fill HF_TOKEN")
+        sys.exit(1)
+
+    login(token=tok)
+    info = whoami(token=tok)
+    print(f"Logged in as: {info['name']}")
+
+    if args.create_repo:
+        api = HfApi()
+        repo_id = f"{info['name']}/{args.repo}" if "/" not in args.repo else args.repo
+        api.create_repo(repo_id=repo_id, repo_type="dataset", exist_ok=True)
+        print(f"Dataset repo ready: https://huggingface.co/datasets/{repo_id}")
+
+    if args.upload:
+        from huggingface_hub import HfApi
+        api = HfApi()
+        username = args.username or info["name"]
+        repo_id = f"{username}/{args.repo}" if "/" not in args.repo else args.repo
+        api.upload_folder(folder_path=args.upload, repo_id=repo_id, repo_type="dataset")
+        print(f"Uploaded {args.upload} -> {repo_id}")
+
+if __name__ == "__main__":
+    main()

scripts/list_datasets.py

@@ -0,0 +1,17 @@
+"""Best HuggingFace datasets for multimodal coding model (code+reasoning+image+video)."""
+DATASETS = {
+    # code text - gated large, needs HF_TOKEN
+    "code_pretrain": "bigcode/the-stack-v2",
+    # public test - no login needed
+    "code_small_test": "HuggingFaceH4/CodeAlpaca_20K",
+    # reasoning / instruction
+    "code_instruct": "bigcode/self-oss-instruct",
+    "reasoning_general": "Open-Orca/OpenOrca",
+    # image -> code (screenshot + html)
+    "image_to_code": "HuggingFaceM4/websight",
+}
+
+print("Recommended HF datasets:")
+for k, v in DATASETS.items():
+    print(f"  {k:18s} -> {v}")
+print("\nRun: python scripts/download_data.py --dataset code_small_test --max-rows 500")