apps/accounts/tests/test_ner_layers.py

"""Per-layer unit tests for Module 8 NER layers.

Each layer's heavy model is mocked so CI doesn't pull ~1 GB of BERT weights.
Runtime integration with real models is covered by
test_resume_parser_integration.py (gated on GAPGUIDE_ML_SMOKE=1).
"""
from __future__ import annotations

from unittest.mock import MagicMock, patch

import pytest


# ---------------------------------------------------------------------------
# Nucha layer
# ---------------------------------------------------------------------------
class TestNuchaLayer:
    def test_predict_extracts_hard_and_soft_skills(self):
        """Pipeline returns HSKILL/SSKILL spans → alias dict keyed on span."""
        from apps.accounts.ner import nucha

        fake_pipeline = MagicMock(return_value=[
            {'entity_group': 'HSKILL', 'score': 0.93, 'word': 'Python',
             'start': 0, 'end': 6},
            {'entity_group': 'HSKILL', 'score': 0.88, 'word': 'SQL',
             'start': 10, 'end': 13},
            {'entity_group': 'SSKILL', 'score': 0.71, 'word': 'teamwork',
             'start': 20, 'end': 28},
            # Non-skill tokens must be dropped.
            {'entity_group': 'O', 'score': 0.99, 'word': 'and',
             'start': 14, 'end': 17},
        ])

        with patch.object(nucha, '_pipeline', fake_pipeline):
            out = nucha.layer.predict("Python SQL and teamwork")

        assert out == {
            'Python': pytest.approx(0.93),
            'SQL': pytest.approx(0.88),
            'teamwork': pytest.approx(0.71),
        }

    def test_empty_text_returns_empty(self):
        from apps.accounts.ner import nucha
        with patch.object(nucha, '_pipeline', MagicMock()) as m:
            assert nucha.layer.predict("") == {}
            m.assert_not_called()  # shortcut before the pipeline runs

    def test_duplicate_spans_keep_max_confidence(self):
        from apps.accounts.ner import nucha
        fake = MagicMock(return_value=[
            {'entity_group': 'HSKILL', 'score': 0.5, 'word': 'Python'},
            {'entity_group': 'HSKILL', 'score': 0.9, 'word': 'Python'},
            {'entity_group': 'HSKILL', 'score': 0.7, 'word': 'Python'},
        ])
        with patch.object(nucha, '_pipeline', fake):
            out = nucha.layer.predict("Python Python Python")
        assert out == {'Python': pytest.approx(0.9)}


# ---------------------------------------------------------------------------
# JobBERT layer
# ---------------------------------------------------------------------------
class TestJobBertLayer:
    def test_predict_accepts_bio_labels(self):
        from apps.accounts.ner import jobbert

        fake = MagicMock(return_value=[
            {'entity_group': 'Skill', 'score': 0.87, 'word': 'React'},
            {'entity_group': 'Skill', 'score': 0.81, 'word': 'Docker'},
            # Non-skill label dropped.
            {'entity_group': 'O', 'score': 0.99, 'word': 'the'},
        ])
        with patch.object(jobbert, '_pipeline', fake):
            out = jobbert.layer.predict("React Docker the project")
        assert out == {'React': pytest.approx(0.87), 'Docker': pytest.approx(0.81)}

    def test_label_casing_is_ignored(self):
        from apps.accounts.ner import jobbert
        fake = MagicMock(return_value=[
            {'entity_group': 'SKILL', 'score': 0.9, 'word': 'AWS'},
            {'entity_group': 'skill', 'score': 0.8, 'word': 'GCP'},
        ])
        with patch.object(jobbert, '_pipeline', fake):
            out = jobbert.layer.predict("AWS GCP")
        assert 'AWS' in out and 'GCP' in out

    def test_multitoken_skill_is_merged_via_offsets(self):
        """F-NER-FRAG: with character offsets, a skill's B token + adjacent I
        tokens re-stitch into one term (jjzha's suffix-less BIO labels stop
        aggregation_strategy='simple' from doing this). Two separate B-started
        skills stay distinct even when adjacent."""
        from apps.accounts.ner import jobbert

        text = "machine learning and Python"
        fake = MagicMock(return_value=[
            {'entity_group': 'B', 'score': 0.90, 'word': 'machine',
             'start': 0, 'end': 7},
            {'entity_group': 'I', 'score': 0.80, 'word': 'learning',
             'start': 8, 'end': 16},  # adjacent (gap == 1) → merges
            {'entity_group': 'B', 'score': 0.95, 'word': 'Python',
             'start': 21, 'end': 27},  # new B → stays separate
        ])
        with patch.object(jobbert, '_pipeline', fake):
            out = jobbert.layer.predict(text)
        assert out == {
            'machine learning': pytest.approx(0.80),  # min score of the group
            'Python': pytest.approx(0.95),
        }

    def test_adjacent_begin_spans_stay_separate(self):
        """Two B-tagged skills with only a space between them are NOT merged —
        a 'B' always starts a fresh term."""
        from apps.accounts.ner import jobbert

        fake = MagicMock(return_value=[
            {'entity_group': 'B', 'score': 0.9, 'word': 'Python',
             'start': 0, 'end': 6},
            {'entity_group': 'B', 'score': 0.85, 'word': 'SQL',
             'start': 7, 'end': 10},
        ])
        with patch.object(jobbert, '_pipeline', fake):
            out = jobbert.layer.predict("Python SQL")
        assert out == {'Python': pytest.approx(0.9), 'SQL': pytest.approx(0.85)}

    def test_gap_greater_than_one_keeps_tokens_separate(self):
        """F-NER-FRAG boundary: a B token and a following I token separated by
        more than one char (gap == 2) do NOT merge — pins the upper bound of
        the continuation predicate (`0 <= start - cur_end <= 1`)."""
        from apps.accounts.ner import jobbert

        text = "machine  learning"  # two spaces → 2-char gap between the spans
        fake = MagicMock(return_value=[
            {'entity_group': 'B', 'score': 0.90, 'word': 'machine',
             'start': 0, 'end': 7},
            {'entity_group': 'I', 'score': 0.80, 'word': 'learning',
             'start': 9, 'end': 17},  # gap == 2 → stays separate
        ])
        with patch.object(jobbert, '_pipeline', fake):
            out = jobbert.layer.predict(text)
        assert out == {
            'machine': pytest.approx(0.90),
            'learning': pytest.approx(0.80),
        }

    def test_bare_bio_prefixes_are_kept(self):
        """Regression guard for the label-filter bug in jobbert.py.

        The real jjzha/jobbert_skill_extraction model, under
        aggregation_strategy="simple", emits entity_group as bare BIO
        prefixes ("B" / "I") rather than the entity name. The old filter
        (`"skill" not in label`) dropped every span. Since the model has
        exactly one entity type (SKILL), every non-O span IS a skill.
        """
        from apps.accounts.ner import jobbert

        fake = MagicMock(return_value=[
            {'entity_group': 'B', 'score': 0.91, 'word': 'Python'},
            {'entity_group': 'I', 'score': 0.76, 'word': 'Kubernetes'},
            # Outside-span must still be dropped.
            {'entity_group': 'O', 'score': 0.99, 'word': 'the'},
        ])
        with patch.object(jobbert, '_pipeline', fake):
            out = jobbert.layer.predict("Python and Kubernetes the prod cluster")
        assert out == {
            'Python': pytest.approx(0.91),
            'Kubernetes': pytest.approx(0.76),
        }


# ---------------------------------------------------------------------------
# SBERT layer — DB-backed via SkillEmbedding + pgvector CosineDistance
# ---------------------------------------------------------------------------
class TestSbertLayer:
    """Unit-level: we don't want to hit pgvector here. Stub the ORM path and
    verify the layer's own logic (threshold + aggregation)."""

    def test_below_threshold_is_dropped(self):
        import numpy as np

        from apps.accounts.ner import sbert

        # Stub noun phrase extraction to avoid loading spacy.
        with patch.object(sbert, '_candidate_phrases', return_value=['banana']):
            fake_model = MagicMock()
            # Real SentenceTransformer.encode returns a numpy ndarray; the
            # layer calls .tolist() on it, so mirror the shape here.
            fake_model.encode = MagicMock(return_value=np.array([[0.1] * 384]))
            with patch.object(sbert, '_get_model', return_value=fake_model):
                # pgvector query returns a match whose cosine distance is too
                # high (similarity = 1 - 0.5 = 0.5 < 0.65 threshold).
                fake_emb = MagicMock()
                fake_emb.distance = 0.5
                fake_emb.skill.skill_name = 'Python'
                fake_qs = MagicMock()
                fake_qs.annotate.return_value.order_by.return_value.select_related.return_value.first.return_value = fake_emb

                with patch('apps.skills.models.SkillEmbedding.objects', fake_qs):
                    out = sbert.layer.predict("I like banana smoothies")

        assert out == {}  # below threshold → dropped


# ---------------------------------------------------------------------------
# Lexical floor — already covered by test_resume_parser.py; just confirm the
# layer wrapper returns the expected shape.
# ---------------------------------------------------------------------------
@pytest.mark.django_db
class TestLexicalLayer:
    def test_wraps_existing_candidate_matches(self):
        from apps.accounts.ner import lexical
        from apps.skills.models import Skill

        Skill.objects.create(skill_name='Python', category='Programming',
                             difficulty_level='BEGINNER')
        Skill.objects.create(skill_name='SQL', category='Database',
                             difficulty_level='BEGINNER')

        out = lexical.layer.predict("Strong Python and SQL background.")
        assert set(out) == {'Python', 'SQL'}
        assert all(0 < v <= 1.0 for v in out.values())