Pujan-Dev commited on
Commit
0d1c39e
·
1 Parent(s): 6b04257

fixed :fixed the bias

Browse files
features/nepali_text_classifier/controller.py CHANGED
@@ -1,5 +1,7 @@
1
  import asyncio
 
2
  import logging
 
3
  from io import BytesIO
4
  from fastapi import HTTPException, UploadFile, status, Depends
5
  from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
@@ -38,7 +40,24 @@ def _sentence_bias_strength(overall_confidence: float) -> float:
38
  return min(0.80, 0.40 + 0.40 * (_clamp(overall_confidence, 0.0, 100.0) / 100.0))
39
 
40
 
41
- def _biased_sentence_result(sentence_result: dict, overall_confidence: float, target_label: str = "Human") -> dict:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
42
  raw_label = sentence_result["label"]
43
  raw_confidence = float(sentence_result["confidence"])
44
  raw_ai = _raw_ai_score(raw_label, raw_confidence)
@@ -51,6 +70,10 @@ def _biased_sentence_result(sentence_result: dict, overall_confidence: float, ta
51
  # Force final label toward overall target to ensure overall bias is applied.
52
  biased_label = target_label
53
  biased_confidence = biased_ai if target_label == "AI" else (100.0 - biased_ai)
 
 
 
 
54
 
55
  return {
56
  "biased_label": biased_label,
@@ -134,7 +157,12 @@ async def handle_sentence_level_analysis(text: str, models: str | None = None):
134
  for sentence in sentences:
135
  end_symbol_for_NP_text(sentence)
136
  result = await asyncio.to_thread(classify_text, sentence, selected_models, 2)
137
- biased = _biased_sentence_result(result, overall_confidence, target_label=overall_label)
 
 
 
 
 
138
  results.append({
139
  "text": sentence,
140
  "result": biased["biased_label"],
@@ -168,7 +196,12 @@ async def handle_file_sentence(file:UploadFile, models: str | None = None):
168
  end_symbol_for_NP_text(sentence)
169
 
170
  result = await asyncio.to_thread(classify_text, sentence, selected_models, 2)
171
- biased = _biased_sentence_result(result, overall_confidence, target_label=overall_label)
 
 
 
 
 
172
  results.append({
173
  "text": sentence,
174
  "result": biased["biased_label"],
 
1
  import asyncio
2
+ import hashlib
3
  import logging
4
+ import random
5
  from io import BytesIO
6
  from fastapi import HTTPException, UploadFile, status, Depends
7
  from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
 
40
  return min(0.80, 0.40 + 0.40 * (_clamp(overall_confidence, 0.0, 100.0) / 100.0))
41
 
42
 
43
+ def _deterministic_jitter(seed_text: str, max_jitter: float = 3.0) -> float:
44
+ digest = hashlib.sha256(seed_text.encode("utf-8")).digest()
45
+ seed_value = int.from_bytes(digest[:8], byteorder="big", signed=False)
46
+ rng = random.Random(seed_value)
47
+ return rng.uniform(-max_jitter, max_jitter)
48
+
49
+
50
+ def _add_likelihood_randomness(likelihood: float, seed_text: str, max_jitter: float = 3.0) -> float:
51
+ jitter = _deterministic_jitter(seed_text=seed_text, max_jitter=max_jitter)
52
+ return _clamp(likelihood + jitter, 50.0, 99.95)
53
+
54
+
55
+ def _biased_sentence_result(
56
+ sentence_result: dict,
57
+ overall_confidence: float,
58
+ target_label: str = "Human",
59
+ seed_text: str = "",
60
+ ) -> dict:
61
  raw_label = sentence_result["label"]
62
  raw_confidence = float(sentence_result["confidence"])
63
  raw_ai = _raw_ai_score(raw_label, raw_confidence)
 
70
  # Force final label toward overall target to ensure overall bias is applied.
71
  biased_label = target_label
72
  biased_confidence = biased_ai if target_label == "AI" else (100.0 - biased_ai)
73
+ biased_confidence = _add_likelihood_randomness(
74
+ biased_confidence,
75
+ seed_text=f"{seed_text}|{target_label}|{round(overall_confidence, 2)}",
76
+ )
77
 
78
  return {
79
  "biased_label": biased_label,
 
157
  for sentence in sentences:
158
  end_symbol_for_NP_text(sentence)
159
  result = await asyncio.to_thread(classify_text, sentence, selected_models, 2)
160
+ biased = _biased_sentence_result(
161
+ result,
162
+ overall_confidence,
163
+ target_label=overall_label,
164
+ seed_text=sentence,
165
+ )
166
  results.append({
167
  "text": sentence,
168
  "result": biased["biased_label"],
 
196
  end_symbol_for_NP_text(sentence)
197
 
198
  result = await asyncio.to_thread(classify_text, sentence, selected_models, 2)
199
+ biased = _biased_sentence_result(
200
+ result,
201
+ overall_confidence,
202
+ target_label=overall_label,
203
+ seed_text=sentence,
204
+ )
205
  results.append({
206
  "text": sentence,
207
  "result": biased["biased_label"],