Upload .\tests\test_basic.py with huggingface_hub

.//tests//test_basic.py

@@ -0,0 +1,189 @@
+"""Tests for BwengeAi data collection."""
+
+import json
+import tempfile
+from pathlib import Path
+
+import pytest
+
+
+class TestDataProcessor:
+    """Tests for DataProcessor."""
+
+    def test_clean_text(self):
+        from src.data_collection.data_processor import DataProcessor
+
+        processor = DataProcessor(
+            raw_dir=tempfile.mkdtemp(),
+            processed_dir=tempfile.mkdtemp(),
+        )
+
+        text = "  This is a test   with   extra   spaces  "
+        cleaned = processor.clean_text(text)
+        assert cleaned == "This is a test with extra spaces"
+
+        html_text = "<p>Hello <b>world</b></p>"
+        cleaned = processor.clean_text(html_text)
+        assert "<p>" not in cleaned
+        assert "<b>" not in cleaned
+        assert "Hello world" == cleaned
+
+        short_text = "Hi"
+        cleaned = processor.clean_text(short_text)
+        assert cleaned == ""
+
+        min_text = "ab"
+        cleaned = processor.clean_text(min_text)
+        assert cleaned == ""
+
+        valid_text = "This is valid text"
+        cleaned = processor.clean_text(valid_text)
+        assert cleaned == "This is valid text"
+
+    def test_deduplicate(self):
+        from src.data_collection.data_processor import DataProcessor
+
+        processor = DataProcessor(
+            raw_dir=tempfile.mkdtemp(),
+            processed_dir=tempfile.mkdtemp(),
+        )
+
+        texts = ["Hello world", "Hello world", "Another text", "hello world"]
+        unique = processor.deduplicate(texts)
+
+        assert len(unique) == 2
+        assert "Hello world" in unique
+        assert "Another text" in unique
+
+    def test_create_training_format(self):
+        from src.data_collection.data_processor import DataProcessor
+
+        processor = DataProcessor(
+            raw_dir=tempfile.mkdtemp(),
+            processed_dir=tempfile.mkdtemp(),
+        )
+
+        texts = ["Short", "This is a longer text that should be included"]
+        training_data = processor.create_training_format(texts)
+
+        assert len(training_data) == 1
+        assert training_data[0]["text"] == "This is a longer text that should be included"
+
+    def test_create_instruction_format(self):
+        from src.data_collection.data_processor import DataProcessor
+
+        processor = DataProcessor(
+            raw_dir=tempfile.mkdtemp(),
+            processed_dir=tempfile.mkdtemp(),
+        )
+
+        data = [
+            {"instruction": "What is Kinyarwanda?", "output": "Kinyarwanda is a language spoken in Rwanda."},
+            {"question": "What is the capital of Rwanda?", "answer": "Kigali"},
+        ]
+
+        formatted = processor.create_instruction_format(data)
+
+        assert len(formatted) == 2
+        assert formatted[0]["instruction"] == "What is Kinyarwanda?"
+        assert formatted[0]["output"] == "Kinyarwanda is a language spoken in Rwanda."
+
+    def test_load_jsonl(self):
+        from src.data_collection.data_processor import DataProcessor
+
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=False) as f:
+            f.write('{"text": "hello"}\n')
+            f.write('{"text": "world"}\n')
+            temp_path = f.name
+
+        processor = DataProcessor(
+            raw_dir=tempfile.mkdtemp(),
+            processed_dir=tempfile.mkdtemp(),
+        )
+
+        data = processor.load_jsonl(Path(temp_path))
+
+        assert len(data) == 2
+        assert data[0]["text"] == "hello"
+        assert data[1]["text"] == "world"
+
+        Path(temp_path).unlink()
+
+    def test_is_likely_kinyarwanda(self):
+        from src.data_collection.data_processor import is_likely_kinyarwanda
+
+        assert is_likely_kinyarwanda("U Rwanda ni igihugu cyiza cyane mu Afrika") is True
+        assert is_likely_kinyarwanda("Hello world this is English text") is True
+        assert is_likely_kinyarwanda("") is True
+        assert is_likely_kinyarwanda("Short") is True
+
+    def test_count_tokens(self):
+        from src.data_collection.data_processor import count_tokens
+
+        assert count_tokens("hello world") == 2
+        assert count_tokens("") == 0
+        assert count_tokens("one two three four five") == 5
+
+    def test_create_chat_format(self):
+        from src.data_collection.data_processor import DataProcessor
+
+        processor = DataProcessor(
+            raw_dir=tempfile.mkdtemp(),
+            processed_dir=tempfile.mkdtemp(),
+        )
+
+        data = [
+            {"input": "Muraho", "output": "Amakuru"},
+        ]
+
+        formatted = processor.create_chat_format(data)
+
+        assert len(formatted) == 1
+        assert formatted[0]["messages"][0]["role"] == "user"
+        assert formatted[0]["messages"][0]["content"] == "Muraho"
+
+
+class TestBwengeEvaluator:
+    """Tests for BwengeEvaluator."""
+
+    def test_compute_rouge(self):
+        from src.evaluation.metrics import BwengeEvaluator
+
+        evaluator = BwengeEvaluator({})
+
+        predictions = ["The cat sat on the mat"]
+        references = ["The cat is on the mat"]
+
+        scores = evaluator.compute_rouge(predictions, references)
+
+        assert "rouge1_fmeasure" in scores
+        assert "rouge2_fmeasure" in scores
+        assert "rougeL_fmeasure" in scores
+        assert 0 <= scores["rouge1_fmeasure"] <= 1
+
+    def test_compute_bleu(self):
+        from src.evaluation.metrics import BwengeEvaluator
+
+        evaluator = BwengeEvaluator({})
+
+        predictions = ["The cat sat on the mat"]
+        references = ["The cat sat on the mat"]
+
+        scores = evaluator.compute_bleu(predictions, references)
+
+        assert "bleu_1" in scores
+        assert "bleu_4" in scores
+        assert scores["bleu_1"] > 0.5
+
+    def test_compute_accuracy(self):
+        from src.evaluation.metrics import BwengeEvaluator
+
+        evaluator = BwengeEvaluator({})
+
+        predictions = ["hello", "world", "test"]
+        references = ["hello", "wrong", "test"]
+
+        scores = evaluator.compute_accuracy(predictions, references)
+
+        assert scores["accuracy"] == 2 / 3
+        assert scores["exact_matches"] == 2