AzizMiladi commited on
Commit
dc73111
·
1 Parent(s): 38c90bd

fix(ci): make ruff + mypy green on the new src/ layout

Browse files

Ruff (44 errors → 0):
- 22 auto-fixed: import order (I001), Optional[X] → X | None (UP045),
quoted annotations (UP037), 2 unused imports in tests (F401), 2 empty
f-strings (F541)
- 2 fixed by hand: B904 in api/main.py — `raise HTTPException(...)`
inside `except` now uses `... from e` to preserve the cause
- 20 ignored at config level: E701 multiple-statements-on-one-line. The
`if cond: ws.cell(row=r, column=N, value=…)` block in cms.py is an
intentional column-aligned table mapping conditions → spreadsheet
columns; splitting destroys the visual grid. Documented in pyproject.toml.

Mypy (3 errors → 0):
- Two .to(device) calls on from_pretrained() chains in inference.py
add `# type: ignore[arg-type]` (transformers stubs mis-type the return
so .to() looks wrong).
- _encode() return type was missing — annotate as Any (the real type is
transformers.BatchEncoding but transformers.* is ignored in mypy.ini,
and the caller uses .word_ids() which dict doesn't have).
- `from typing import Any` added to inference.py.

Verified:
- ruff check src/ tests/ → All checks passed
- mypy --config-file mypy.ini src/guichetoi/cms.py src/guichetoi/recommendation.py → Success
- pytest tests/test_cms_generator.py → 71/71 pass

pyproject.toml CHANGED
@@ -79,4 +79,7 @@ select = ["E", "F", "I", "B", "UP"]
79
  ignore = [
80
  "E501", # line length — codebase has long French strings
81
  "B008", # function-call-in-default-arg — used intentionally with FastAPI Depends/File
 
 
 
82
  ]
 
79
  ignore = [
80
  "E501", # line length — codebase has long French strings
81
  "B008", # function-call-in-default-arg — used intentionally with FastAPI Depends/File
82
+ "E701", # multiple-statements-on-one-line — cms.py uses column-aligned
83
+ # `if cond: ws.cell(row=r, column=N, value=…)` as a visual table
84
+ # mapping conditions → spreadsheet columns. Splitting destroys the grid.
85
  ]
src/guichetoi/api/main.py CHANGED
@@ -80,7 +80,6 @@ from fastapi.responses import JSONResponse
80
  from guichetoi import cms as cms_gen
81
  from guichetoi import recommendation as reco
82
 
83
-
84
  # ────────────────────────────────────────────────────────────────────────────
85
  # Logging
86
  # ────────────────────────────────────────────────────────────────────────────
@@ -196,12 +195,12 @@ def _materialise_uploads(files: list[UploadFile]) -> tuple[list[Path], tempfile.
196
  try:
197
  with zipfile.ZipFile(io.BytesIO(raw)) as zf:
198
  zf.extractall(zdir)
199
- except zipfile.BadZipFile:
200
  tmp.cleanup()
201
  raise HTTPException(
202
  status_code=status.HTTP_400_BAD_REQUEST,
203
  detail=f"Invalid ZIP archive: {name}",
204
- )
205
  for p in zdir.rglob("*"):
206
  if not p.is_file():
207
  continue
@@ -318,7 +317,7 @@ def cms(verdict: dict) -> Response:
318
  raise HTTPException(
319
  status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
320
  detail=f"CMS template missing: {e}",
321
- )
322
  finally:
323
  out_path.unlink(missing_ok=True)
324
 
 
80
  from guichetoi import cms as cms_gen
81
  from guichetoi import recommendation as reco
82
 
 
83
  # ────────────────────────────────────────────────────────────────────────────
84
  # Logging
85
  # ────────────────────────────────────────────────────────────────────────────
 
195
  try:
196
  with zipfile.ZipFile(io.BytesIO(raw)) as zf:
197
  zf.extractall(zdir)
198
+ except zipfile.BadZipFile as e:
199
  tmp.cleanup()
200
  raise HTTPException(
201
  status_code=status.HTTP_400_BAD_REQUEST,
202
  detail=f"Invalid ZIP archive: {name}",
203
+ ) from e
204
  for p in zdir.rglob("*"):
205
  if not p.is_file():
206
  continue
 
317
  raise HTTPException(
318
  status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
319
  detail=f"CMS template missing: {e}",
320
+ ) from e
321
  finally:
322
  out_path.unlink(missing_ok=True)
323
 
src/guichetoi/cms.py CHANGED
@@ -401,9 +401,9 @@ def fill_cms(
401
  # PresenceDta (22), Commentaire Faisabilite (23-24): blank, manual
402
 
403
  comment_bits = [
404
- f"Pré-rempli automatiquement (GuichetOI-ML)",
405
  f"Projet {proj_type} · Type site {type_site} · Détection {detection_lbl}",
406
- f"À compléter manuellement : coordonnées XY (Géoréso), Identifiant Processus (Mondofi pour OCC)",
407
  ]
408
  ws.cell(row=r, column=25, value=" — ".join(comment_bits))
409
 
 
401
  # PresenceDta (22), Commentaire Faisabilite (23-24): blank, manual
402
 
403
  comment_bits = [
404
+ "Pré-rempli automatiquement (GuichetOI-ML)",
405
  f"Projet {proj_type} · Type site {type_site} · Détection {detection_lbl}",
406
+ "À compléter manuellement : coordonnées XY (Géoréso), Identifiant Processus (Mondofi pour OCC)",
407
  ]
408
  ws.cell(row=r, column=25, value=" — ".join(comment_bits))
409
 
src/guichetoi/inference.py CHANGED
@@ -25,9 +25,9 @@ import logging
25
  import os
26
  import re
27
  import sys
28
- from dataclasses import dataclass, field, asdict
29
  from pathlib import Path
30
- from typing import Optional
31
 
32
  import torch
33
  from PIL import Image
@@ -361,7 +361,7 @@ def _mandat_checkbox_score(marker: str) -> int:
361
  return 0
362
 
363
 
364
- def _detect_mandat_checkbox(ocr_text: str) -> Optional[str]:
365
  """
366
  Decide which checkbox is X-marked next to 'Je dispose d'un mandat de
367
  représentation du Maître d'ouvrage' on the fiche form.
@@ -394,9 +394,9 @@ def _detect_mandat_checkbox(ocr_text: str) -> Optional[str]:
394
 
395
 
396
  def _clean_field_extractions(
397
- raw_fields: dict[str, "FieldExtraction"],
398
  ocr_text: str,
399
- ) -> dict[str, "FieldExtraction"]:
400
  """
401
  Apply per-field validators + regex fallbacks to the model's raw outputs.
402
 
@@ -604,7 +604,7 @@ def run_ocr(image: Image.Image, cfg: Config) -> OCRResult:
604
  return OCRResult(words, boxes, " ".join(words), "pytesseract")
605
 
606
 
607
- def extract_pdf_text(file_path: Path) -> Optional[str]:
608
  """Quick path: pull embedded text from a PDF without OCR. Returns None if no text or fails."""
609
  if file_path.suffix.lower() != ".pdf" or fitz is None:
610
  return None
@@ -631,7 +631,7 @@ class GuichetOIPipeline:
631
  result = pipeline.run(path) # in your /predict endpoint
632
  """
633
 
634
- def __init__(self, cfg: Config = Config(), device: Optional[str] = None):
635
  self.cfg = cfg
636
  self.device = torch.device(
637
  device or ("cuda" if torch.cuda.is_available() else "cpu")
@@ -649,14 +649,16 @@ class GuichetOIPipeline:
649
  cfg.base_processor, apply_ocr=False,
650
  )
651
 
652
- # Models — moved to device, set to eval mode
 
 
653
  self.classifier = LayoutLMv3ForSequenceClassification.from_pretrained(
654
  resolve_model_path(cfg.classifier_dir)
655
- ).to(self.device).eval()
656
 
657
  self.extractor = LayoutLMv3ForTokenClassification.from_pretrained(
658
  resolve_model_path(cfg.extractor_dir)
659
- ).to(self.device).eval()
660
 
661
  log.info(
662
  f"Pipeline ready · {len(self.doc_classes)} document classes · "
@@ -666,7 +668,9 @@ class GuichetOIPipeline:
666
  # ────────────────────────────────────────────────────────────────────
667
  # Inference primitives
668
  # ────────────────────────────────────────────────────────────────────
669
- def _encode(self, image: Image.Image, words: list[str], boxes: list[list[int]]):
 
 
670
  return self.processor(
671
  image, words, boxes=boxes,
672
  max_length=self.cfg.max_seq_length,
@@ -703,7 +707,7 @@ class GuichetOIPipeline:
703
  id2label = self.extractor.config.id2label
704
 
705
  spans: list[dict] = []
706
- cur: Optional[dict] = None
707
  prev_word = None
708
 
709
  for pos, w_idx in enumerate(word_ids):
@@ -845,7 +849,7 @@ def _save_result(result: InferenceResult, image_path: Path, cfg: Config) -> Path
845
  return out_path
846
 
847
 
848
- def _prompt_for_image_path() -> Optional[str]:
849
  """GUI fallback ONLY when running interactively. Skipped on headless servers."""
850
  if not sys.stdin.isatty():
851
  return None
 
25
  import os
26
  import re
27
  import sys
28
+ from dataclasses import asdict, dataclass, field
29
  from pathlib import Path
30
+ from typing import Any
31
 
32
  import torch
33
  from PIL import Image
 
361
  return 0
362
 
363
 
364
+ def _detect_mandat_checkbox(ocr_text: str) -> str | None:
365
  """
366
  Decide which checkbox is X-marked next to 'Je dispose d'un mandat de
367
  représentation du Maître d'ouvrage' on the fiche form.
 
394
 
395
 
396
  def _clean_field_extractions(
397
+ raw_fields: dict[str, FieldExtraction],
398
  ocr_text: str,
399
+ ) -> dict[str, FieldExtraction]:
400
  """
401
  Apply per-field validators + regex fallbacks to the model's raw outputs.
402
 
 
604
  return OCRResult(words, boxes, " ".join(words), "pytesseract")
605
 
606
 
607
+ def extract_pdf_text(file_path: Path) -> str | None:
608
  """Quick path: pull embedded text from a PDF without OCR. Returns None if no text or fails."""
609
  if file_path.suffix.lower() != ".pdf" or fitz is None:
610
  return None
 
631
  result = pipeline.run(path) # in your /predict endpoint
632
  """
633
 
634
+ def __init__(self, cfg: Config = Config(), device: str | None = None):
635
  self.cfg = cfg
636
  self.device = torch.device(
637
  device or ("cuda" if torch.cuda.is_available() else "cpu")
 
649
  cfg.base_processor, apply_ocr=False,
650
  )
651
 
652
+ # Models — moved to device, set to eval mode.
653
+ # (mypy ignores below: transformers stubs mis-type from_pretrained's
654
+ # return so .to(device) appears to take the wrong arg.)
655
  self.classifier = LayoutLMv3ForSequenceClassification.from_pretrained(
656
  resolve_model_path(cfg.classifier_dir)
657
+ ).to(self.device).eval() # type: ignore[arg-type]
658
 
659
  self.extractor = LayoutLMv3ForTokenClassification.from_pretrained(
660
  resolve_model_path(cfg.extractor_dir)
661
+ ).to(self.device).eval() # type: ignore[arg-type]
662
 
663
  log.info(
664
  f"Pipeline ready · {len(self.doc_classes)} document classes · "
 
668
  # ────────────────────────────────────────────────────────────────────
669
  # Inference primitives
670
  # ────────────────────────────────────────────────────────────────────
671
+ def _encode(self, image: Image.Image, words: list[str], boxes: list[list[int]]) -> Any:
672
+ # Returns a transformers BatchEncoding (dict-like, with .word_ids()).
673
+ # Annotated Any because the transformers stubs are ignored in mypy.ini.
674
  return self.processor(
675
  image, words, boxes=boxes,
676
  max_length=self.cfg.max_seq_length,
 
707
  id2label = self.extractor.config.id2label
708
 
709
  spans: list[dict] = []
710
+ cur: dict | None = None
711
  prev_word = None
712
 
713
  for pos, w_idx in enumerate(word_ids):
 
849
  return out_path
850
 
851
 
852
+ def _prompt_for_image_path() -> str | None:
853
  """GUI fallback ONLY when running interactively. Skipped on headless servers."""
854
  if not sys.stdin.isatty():
855
  return None
src/guichetoi/recommendation.py CHANGED
@@ -35,10 +35,9 @@ import json
35
  import logging
36
  import re
37
  import sys
38
- from dataclasses import dataclass, field, asdict
39
- from pathlib import Path
40
  from collections.abc import Sequence
41
- from typing import Optional
 
42
 
43
  from guichetoi.inference import Config, GuichetOIPipeline, InferenceResult
44
 
@@ -134,9 +133,9 @@ class RecommendationEngine:
134
 
135
  def __init__(
136
  self,
137
- pipeline: Optional[GuichetOIPipeline] = None,
138
  rules: RuleConfig = RuleConfig(),
139
- cfg: Optional[Config] = None,
140
  ):
141
  self.rules = rules
142
  self.pipeline = pipeline or GuichetOIPipeline(cfg=cfg or Config())
@@ -214,7 +213,7 @@ class RecommendationEngine:
214
  r"dossier[\s_-]*de[\s_-]*r[ée]coll?ement",
215
  ]
216
 
217
- def _filename_class_hint(self, filename: str) -> Optional[str]:
218
  name = filename.lower()
219
  for pat, cls in self._FILENAME_HINTS:
220
  if re.search(pat, name):
@@ -476,7 +475,7 @@ class RecommendationEngine:
476
 
477
  return reasons
478
 
479
- def _autorisation_matches(self, ref_urb: str, autorisations: list[DocumentSummary]) -> Optional[bool]:
480
  """
481
  Cross-check the fiche's urbanism reference against the autorisation(s).
482
 
@@ -610,13 +609,13 @@ class RecommendationEngine:
610
  # ────────────────────────────────────────────────────────────────────────────
611
  # Small, file-local helpers
612
  # ────────────────────────────────────────────────────────────────────────────
613
- def _value(payload: Optional[dict]) -> str:
614
  if not payload:
615
  return ""
616
  return (payload.get("value") or "").strip()
617
 
618
 
619
- def _to_int(s: str) -> Optional[int]:
620
  if not s:
621
  return None
622
  digits = re.sub(r"[^\d]", "", s)
@@ -659,7 +658,7 @@ def _norm_ref(s: str) -> str:
659
  # ────────────────────────────────────────────────────────────────────────────
660
  # Folder picker (GUI fallback for interactive runs)
661
  # ────────────────────────────────────────────────────────────────────────────
662
- def _prompt_for_folder() -> Optional[str]:
663
  """
664
  Open a Windows-native directory picker. Returns the selected path, or
665
  None if the dialog is cancelled or unavailable (e.g. headless server).
@@ -747,7 +746,7 @@ def main():
747
  args = parser.parse_args()
748
 
749
  # Resolve input source: explicit --files, then --folder, then GUI picker
750
- folder: Optional[Path] = None
751
  files: list[Path] = []
752
 
753
  if args.files:
 
35
  import logging
36
  import re
37
  import sys
 
 
38
  from collections.abc import Sequence
39
+ from dataclasses import asdict, dataclass, field
40
+ from pathlib import Path
41
 
42
  from guichetoi.inference import Config, GuichetOIPipeline, InferenceResult
43
 
 
133
 
134
  def __init__(
135
  self,
136
+ pipeline: GuichetOIPipeline | None = None,
137
  rules: RuleConfig = RuleConfig(),
138
+ cfg: Config | None = None,
139
  ):
140
  self.rules = rules
141
  self.pipeline = pipeline or GuichetOIPipeline(cfg=cfg or Config())
 
213
  r"dossier[\s_-]*de[\s_-]*r[ée]coll?ement",
214
  ]
215
 
216
+ def _filename_class_hint(self, filename: str) -> str | None:
217
  name = filename.lower()
218
  for pat, cls in self._FILENAME_HINTS:
219
  if re.search(pat, name):
 
475
 
476
  return reasons
477
 
478
+ def _autorisation_matches(self, ref_urb: str, autorisations: list[DocumentSummary]) -> bool | None:
479
  """
480
  Cross-check the fiche's urbanism reference against the autorisation(s).
481
 
 
609
  # ────────────────────────────────────────────────────────────────────────────
610
  # Small, file-local helpers
611
  # ────────────────────────────────────────────────────────────────────────────
612
+ def _value(payload: dict | None) -> str:
613
  if not payload:
614
  return ""
615
  return (payload.get("value") or "").strip()
616
 
617
 
618
+ def _to_int(s: str) -> int | None:
619
  if not s:
620
  return None
621
  digits = re.sub(r"[^\d]", "", s)
 
658
  # ────────────────────────────────────────────────────────────────────────────
659
  # Folder picker (GUI fallback for interactive runs)
660
  # ────────────────────────────────────────────────────────────────────────────
661
+ def _prompt_for_folder() -> str | None:
662
  """
663
  Open a Windows-native directory picker. Returns the selected path, or
664
  None if the dialog is cancelled or unavailable (e.g. headless server).
 
746
  args = parser.parse_args()
747
 
748
  # Resolve input source: explicit --files, then --folder, then GUI picker
749
+ folder: Path | None = None
750
  files: list[Path] = []
751
 
752
  if args.files:
tests/test_cms_generator.py CHANGED
@@ -9,7 +9,6 @@ template and verifies the expected cells are written.
9
  """
10
  from __future__ import annotations
11
 
12
- import tempfile
13
  from datetime import datetime, timedelta
14
  from pathlib import Path
15
 
 
9
  """
10
  from __future__ import annotations
11
 
 
12
  from datetime import datetime, timedelta
13
  from pathlib import Path
14
 
tests/test_inference_postprocess.py CHANGED
@@ -8,8 +8,6 @@ These tests don't load the model — we exercise the pure functions directly.
8
  """
9
  from __future__ import annotations
10
 
11
- import re
12
-
13
  import pytest
14
 
15
 
 
8
  """
9
  from __future__ import annotations
10
 
 
 
11
  import pytest
12
 
13