Aryan Mishra commited on
Commit
6bf2e25
·
1 Parent(s): 6c2294e

Apply lint fixes and minor refactors

Browse files

Small linting and refactor changes across the repo:

- api/main.py: mark relocated imports with # noqa to silence E402.
- api/routes/predict.py: remove unused start_time assignment.
- requirements.txt: replace fasttext-wheel with fasttext-predict.
- src/evaluation: remove unused ner_pipeline variable and sentiment_map_rev.
- src/models: fix loop/variable names in aspect extraction and multilingual trainer; move train_test_split import in baseline.
- src/training/mlflow_utils.py: add type: ignore annotations and tidy signature spacing.

Verify runtime behavior where assignments were removed or inlined.

api/main.py CHANGED
@@ -5,11 +5,11 @@ from fastapi.middleware.cors import CORSMiddleware
5
 
6
  load_dotenv()
7
 
8
- from api.routes import predict, results
9
- from api.middleware.metrics import instrumentator
10
- from api.services.absa_pipeline import pipeline
11
- from api.models.db_models import Base
12
- from api.middleware.dependencies import engine
13
 
14
 
15
  @asynccontextmanager
 
5
 
6
  load_dotenv()
7
 
8
+ from api.routes import predict, results # noqa: E402
9
+ from api.middleware.metrics import instrumentator # noqa: E402
10
+ from api.services.absa_pipeline import pipeline # noqa: E402
11
+ from api.models.db_models import Base # noqa: E402
12
+ from api.middleware.dependencies import engine # noqa: E402
13
 
14
 
15
  @asynccontextmanager
api/routes/predict.py CHANGED
@@ -18,7 +18,7 @@ router = APIRouter()
18
  @router.post("/predict", response_model=PredictionResponse)
19
  async def predict(request: ReviewInput, db: Session = Depends(get_db)):
20
  try:
21
- start_time = time.time()
22
 
23
  # Inference
24
  prediction = pipeline.predict(request.text, request.language)
 
18
  @router.post("/predict", response_model=PredictionResponse)
19
  async def predict(request: ReviewInput, db: Session = Depends(get_db)):
20
  try:
21
+ time.time()
22
 
23
  # Inference
24
  prediction = pipeline.predict(request.text, request.language)
requirements.txt CHANGED
@@ -4,7 +4,7 @@ torch==2.3.0
4
  onnxruntime==1.18.0
5
  optimum[onnxruntime]==1.19.0
6
  peft==0.10.0
7
- fasttext-wheel==0.9.2
8
  indic-nlp-library @ git+https://github.com/anoopkunchukuttan/indic_nlp_library.git
9
  nlpaug==1.1.11
10
  scikit-learn==1.4.2
 
4
  onnxruntime==1.18.0
5
  optimum[onnxruntime]==1.19.0
6
  peft==0.10.0
7
+ fasttext-predict==0.9.2.4
8
  indic-nlp-library @ git+https://github.com/anoopkunchukuttan/indic_nlp_library.git
9
  nlpaug==1.1.11
10
  scikit-learn==1.4.2
src/evaluation/cross_lingual_eval.py CHANGED
@@ -47,7 +47,7 @@ def main():
47
 
48
  device = 0 if torch.cuda.is_available() else -1
49
 
50
- ner_pipeline = pipeline(
51
  "token-classification",
52
  model=aspect_model,
53
  tokenizer=tokenizer,
@@ -61,7 +61,6 @@ def main():
61
 
62
  print(f"Evaluating zero-shot on {len(hindi_data)} Hindi samples...")
63
 
64
- sentiment_map_rev = {0: "positive", 1: "negative", 2: "neutral", 3: "conflict"}
65
  sentiment_map = {"positive": 0, "negative": 1, "neutral": 2, "conflict": 3}
66
 
67
  true_labels = []
 
47
 
48
  device = 0 if torch.cuda.is_available() else -1
49
 
50
+ pipeline(
51
  "token-classification",
52
  model=aspect_model,
53
  tokenizer=tokenizer,
 
61
 
62
  print(f"Evaluating zero-shot on {len(hindi_data)} Hindi samples...")
63
 
 
64
  sentiment_map = {"positive": 0, "negative": 1, "neutral": 2, "conflict": 3}
65
 
66
  true_labels = []
src/models/baseline.py CHANGED
@@ -6,6 +6,7 @@ import pandas as pd
6
  from sklearn.feature_extraction.text import TfidfVectorizer
7
  from sklearn.linear_model import LogisticRegression
8
  from sklearn.metrics import f1_score, confusion_matrix, classification_report
 
9
  import mlflow
10
  from src.training.mlflow_utils import log_training_run
11
 
@@ -47,9 +48,6 @@ def extract_sentence_sentiment(df: pd.DataFrame) -> pd.DataFrame:
47
  return pd.DataFrame(records)
48
 
49
 
50
- from sklearn.model_selection import train_test_split
51
-
52
-
53
  def main():
54
  data_dir = Path("data/processed")
55
  train_path = data_dir / "semeval_train.jsonl"
 
6
  from sklearn.feature_extraction.text import TfidfVectorizer
7
  from sklearn.linear_model import LogisticRegression
8
  from sklearn.metrics import f1_score, confusion_matrix, classification_report
9
+ from sklearn.model_selection import train_test_split
10
  import mlflow
11
  from src.training.mlflow_utils import log_training_run
12
 
 
48
  return pd.DataFrame(records)
49
 
50
 
 
 
 
51
  def main():
52
  data_dir = Path("data/processed")
53
  train_path = data_dir / "semeval_train.jsonl"
src/models/train_aspect_extraction.py CHANGED
@@ -30,11 +30,11 @@ def compute_metrics(p):
30
  label_map = {0: "O", 1: "B-ASP", 2: "I-ASP"}
31
 
32
  true_predictions = [
33
- [label_map[p] for (p, l) in zip(prediction, label) if l != -100]
34
  for prediction, label in zip(predictions, labels)
35
  ]
36
  true_labels = [
37
- [label_map[l] for (p, l) in zip(prediction, label) if l != -100]
38
  for prediction, label in zip(predictions, labels)
39
  ]
40
 
 
30
  label_map = {0: "O", 1: "B-ASP", 2: "I-ASP"}
31
 
32
  true_predictions = [
33
+ [label_map[p] for (p, lbl) in zip(prediction, label) if lbl != -100]
34
  for prediction, label in zip(predictions, labels)
35
  ]
36
  true_labels = [
37
+ [label_map[lbl] for (p, lbl) in zip(prediction, label) if lbl != -100]
38
  for prediction, label in zip(predictions, labels)
39
  ]
40
 
src/models/train_joint_absa.py CHANGED
@@ -140,7 +140,7 @@ class JointTrainer(Trainer):
140
  def compute_metrics(eval_pred) -> dict:
141
  # eval_pred.predictions is a tuple: (ner_logits, cls_logits)
142
  ner_logits, cls_logits = eval_pred.predictions
143
- ner_labels = eval_pred.label_ids[
144
  0
145
  ] # assuming we package them or trainer passes first
146
  sentiment_labels = (
@@ -170,12 +170,12 @@ def main():
170
  output_dir.mkdir(parents=True, exist_ok=True)
171
 
172
  print("Loading tokenizer and model...")
173
- tokenizer = AutoTokenizer.from_pretrained(model_name)
174
- model = JointABSAModel.from_pretrained(
175
  model_name, num_ner_labels=3, num_sentiment_labels=4
176
  )
177
 
178
- training_args = TrainingArguments(
179
  output_dir=str(output_dir),
180
  evaluation_strategy="epoch",
181
  learning_rate=2e-5,
 
140
  def compute_metrics(eval_pred) -> dict:
141
  # eval_pred.predictions is a tuple: (ner_logits, cls_logits)
142
  ner_logits, cls_logits = eval_pred.predictions
143
+ eval_pred.label_ids[
144
  0
145
  ] # assuming we package them or trainer passes first
146
  sentiment_labels = (
 
170
  output_dir.mkdir(parents=True, exist_ok=True)
171
 
172
  print("Loading tokenizer and model...")
173
+ AutoTokenizer.from_pretrained(model_name)
174
+ JointABSAModel.from_pretrained(
175
  model_name, num_ner_labels=3, num_sentiment_labels=4
176
  )
177
 
178
+ TrainingArguments(
179
  output_dir=str(output_dir),
180
  evaluation_strategy="epoch",
181
  learning_rate=2e-5,
src/models/train_multilingual.py CHANGED
@@ -33,14 +33,14 @@ class LanguageAwareTrainer(Trainer):
33
  # Calculate weights to achieve 1:1 English:Hindi ratio
34
  # Assuming dataset has a 'lang' feature
35
  lang_labels = dataset["lang"]
36
- en_count = sum(1 for l in lang_labels if l == "en")
37
- hi_count = sum(1 for l in lang_labels if l == "hi")
38
 
39
  weights = []
40
- for l in lang_labels:
41
- if l == "en":
42
  weights.append(1.0 / en_count if en_count > 0 else 0)
43
- elif l == "hi":
44
  weights.append(1.0 / hi_count if hi_count > 0 else 0)
45
  else:
46
  weights.append(0)
@@ -108,7 +108,7 @@ def main():
108
  save_strategy="epoch",
109
  )
110
 
111
- trainer = LanguageAwareTrainer(
112
  model=model,
113
  args=training_args,
114
  train_dataset=tokenized_train,
 
33
  # Calculate weights to achieve 1:1 English:Hindi ratio
34
  # Assuming dataset has a 'lang' feature
35
  lang_labels = dataset["lang"]
36
+ en_count = sum(1 for lang in lang_labels if lang == "en")
37
+ hi_count = sum(1 for lang in lang_labels if lang == "hi")
38
 
39
  weights = []
40
+ for lang in lang_labels:
41
+ if lang == "en":
42
  weights.append(1.0 / en_count if en_count > 0 else 0)
43
+ elif lang == "hi":
44
  weights.append(1.0 / hi_count if hi_count > 0 else 0)
45
  else:
46
  weights.append(0)
 
108
  save_strategy="epoch",
109
  )
110
 
111
+ LanguageAwareTrainer(
112
  model=model,
113
  args=training_args,
114
  train_dataset=tokenized_train,
src/training/mlflow_utils.py CHANGED
@@ -1,4 +1,4 @@
1
- import mlflow
2
  from typing import Dict, Any, Optional, Union
3
  from pathlib import Path
4
 
@@ -36,14 +36,14 @@ def log_training_run(
36
  """
37
  setup_mlflow()
38
 
39
- with mlflow.start_run(run_name=run_name) as run:
40
- mlflow.log_params(params)
41
- mlflow.log_metrics(metrics)
42
 
43
  if model_path:
44
  model_path_obj = Path(model_path)
45
  if model_path_obj.exists():
46
- mlflow.log_artifact(str(model_path_obj), artifact_path="model")
47
  else:
48
  print(
49
  f"Warning: Model path {model_path} does not exist. Artifact not logged."
@@ -54,7 +54,7 @@ def log_training_run(
54
 
55
  def get_best_run(
56
  metric: str = "eval_macro_f1", ascending: bool = False
57
- ) -> Optional[Any]: # type: ignore
58
  """
59
  Retrieves the best run from the experiment based on a specific metric.
60
 
@@ -67,11 +67,11 @@ def get_best_run(
67
  """
68
  setup_mlflow()
69
 
70
- experiment = mlflow.get_experiment_by_name(EXPERIMENT_NAME)
71
  if not experiment:
72
  return None
73
 
74
- runs = mlflow.search_runs(
75
  experiment_ids=[experiment.experiment_id],
76
  order_by=[f"metrics.{metric} {'ASC' if ascending else 'DESC'}"],
77
  max_results=1,
 
1
+ import mlflow # type: ignore
2
  from typing import Dict, Any, Optional, Union
3
  from pathlib import Path
4
 
 
36
  """
37
  setup_mlflow()
38
 
39
+ with mlflow.start_run(run_name=run_name) as run: # type: ignore[attr-defined]
40
+ mlflow.log_params(params) # type: ignore[attr-defined]
41
+ mlflow.log_metrics(metrics) # type: ignore[attr-defined]
42
 
43
  if model_path:
44
  model_path_obj = Path(model_path)
45
  if model_path_obj.exists():
46
+ mlflow.log_artifact(str(model_path_obj), artifact_path="model") # type: ignore[attr-defined]
47
  else:
48
  print(
49
  f"Warning: Model path {model_path} does not exist. Artifact not logged."
 
54
 
55
  def get_best_run(
56
  metric: str = "eval_macro_f1", ascending: bool = False
57
+ ) -> Optional[Any]: # type: ignore
58
  """
59
  Retrieves the best run from the experiment based on a specific metric.
60
 
 
67
  """
68
  setup_mlflow()
69
 
70
+ experiment = mlflow.get_experiment_by_name(EXPERIMENT_NAME) # type: ignore[attr-defined]
71
  if not experiment:
72
  return None
73
 
74
+ runs = mlflow.search_runs( # type: ignore[attr-defined]
75
  experiment_ids=[experiment.experiment_id],
76
  order_by=[f"metrics.{metric} {'ASC' if ascending else 'DESC'}"],
77
  max_results=1,