+
+ """
+ return html
+
+# --- TASK 1: XỬ LÝ ẢNH ---
+async def run_image_task(shared_state, image_input):
+ shared_state['img_status'] = 'processing'
+ shared_state['img_msg'] = "Scanning artifacts..."
+ img_result_obj = TextAnalysisResult()
+ try:
+ final_json = {}
+ final_report_md = ""
+ gen = analyze_saved_images(image_input)
+ if inspect.isasyncgen(gen):
+ async for res in gen: final_json, final_report_md = res
+ else:
+ for res in gen: final_json, final_report_md = res
+
+ # Lưu toàn bộ chuỗi report vào artifact
+ img_result_obj.set_other_artifacts(final_report_md)
+
+ # Parse lấy đúng dòng Auth để set status cho object (để dùng cho short-circuit nếu cần)
+ parsed = parse_child_report(final_report_md)
+ img_result_obj.set_authenticity_assessment(parsed["auth"])
+
+ shared_state['img_status'] = 'done'
+ shared_state['img_msg'] = "Done"
+ except Exception as e:
+ shared_state['img_status'] = 'error'
+ shared_state['img_msg'] = "Error"
+ img_result_obj.set_authenticity_assessment("Error")
+ return img_result_obj
+
+# --- TASK 2: XỬ LÝ TEXT ---
+async def run_text_task(shared_state, text_input):
+ shared_state['txt_status'] = 'processing'
+ shared_state['txt_msg'] = "Verifying logic..."
+ txt_result_obj = TextAnalysisResult()
+ try:
+ await asyncio.to_thread(verify_text_logic, text_input, txt_result_obj)
+ shared_state['txt_status'] = 'done'
+ shared_state['txt_msg'] = "Done"
+ except Exception as e:
+ shared_state['txt_status'] = 'error'
+ shared_state['txt_msg'] = str(e)
+ txt_result_obj.set_authenticity_assessment("Error")
+ return txt_result_obj
+
+# --- MAIN ORCHESTRATOR ---
+async def verify_multimodal_logic(image_state, text_input):
+
+ shared_state = {'img_status': 'waiting', 'img_msg': 'Ready...', 'txt_status': 'waiting', 'txt_msg': 'Ready...'}
+ def get_ui(): return create_status_html("Image Analysis", shared_state['img_status'], shared_state['img_msg']), create_status_html("Text Analysis", shared_state['txt_status'], shared_state['txt_msg']), "..."
+ yield get_ui()
+
+ task_img = asyncio.create_task(run_image_task(shared_state, image_state))
+ task_txt = asyncio.create_task(run_text_task(shared_state, text_input))
+
+ img_res, txt_res = None, None
+
+ # Loop short-circuit
+ while not (task_img.done() and task_txt.done()):
+ yield get_ui()
+ if task_img.done() and img_res is None:
+ try:
+ img_res = task_img.result()
+ # Check Fake chỉ dựa trên Assessment (ngắn gọn)
+ if is_verdict_fake(img_res.get_authenticity_assessment()):
+ if not task_txt.done(): task_txt.cancel(); shared_state['txt_msg'] = "Stopped (Image is Fake)"
+ break
+ except: pass
+
+ if task_txt.done() and txt_res is None:
+ try:
+ txt_res = task_txt.result()
+ # Check Fake chỉ dựa trên Assessment
+ if is_verdict_fake(txt_res.get_authenticity_assessment()):
+ if not task_img.done(): task_img.cancel(); shared_state['img_msg'] = "Stopped (Text is Fake)"
+ break
+ except: pass
+ await asyncio.sleep(0.1)
+
+ if img_res is None and task_img.done(): img_res = task_img.result()
+ if txt_res is None and task_txt.done(): txt_res = task_txt.result()
+ if not img_res: img_res = TextAnalysisResult(authenticity_assessment="Skipped")
+ if not txt_res: txt_res = TextAnalysisResult(authenticity_assessment="Skipped")
+
+ # =========================================================================
+ # LOGIC MERGE: CHỈ DỰA VÀO DÒNG ASSESSMENT
+ # =========================================================================
+
+ # 1. Parse Image Report để lấy dòng "Authenticity Assessment" sạch
+ img_data_parsed = parse_child_report(img_res.get_other_artifacts())
+ img_auth_line = img_data_parsed["auth"] # VD: "🧑 REAL PHOTO"
+
+ # 2. Lấy dòng Assessment của Text
+ txt_auth_line = txt_res.get_authenticity_assessment() # VD: "REAL (Authentic)"
+
+ # 3. KIỂM TRA FAKE/REAL (Dựa trên 2 dòng trên)
+ img_is_fake = is_verdict_fake(img_auth_line)
+ txt_is_fake = is_verdict_fake(txt_auth_line)
+
+ # --- FIELD 1: Authenticity Assessment ---
+ if img_is_fake or txt_is_fake:
+ final_auth = "🤖 NOT REAL (Fake, Manipulated, or AI)"
+ color_hex = "#dc2626"
+ else:
+ final_auth = " REAL (Authentic)"
+ color_hex = "#16a34a"
+
+ # --- FIELD 2: Verification Tools ---
+ final_tools = f"Verified by our model using algorithms SearchLLM and ImageForensics."
+
+ # --- FIELD 3: Synthetic Type ---
+ final_synth_list = []
+
+ # Chỉ lấy Synthetic Type từ module Ảnh nếu Ảnh bị kết luận là Fake
+ if img_is_fake:
+ s_type = img_data_parsed["synth"] if img_data_parsed["synth"] != "N/A" else "Manipulated Image"
+ final_synth_list.append(f"**Image:** {s_type}")
+
+ # Chỉ lấy Synthetic Type từ module Text nếu Text bị kết luận là Fake
+ if txt_is_fake:
+ s_type = txt_res.get_synthetic_type()
+ if not s_type or s_type == "N/A": s_type = "Generated Content"
+ final_synth_list.append(f"**Text:** {s_type}")
+
+ final_synth_str = "\n".join(final_synth_list) if final_synth_list else "N/A"
+
+ # --- FIELD 4: Other Artifacts (Logic hiển thị Source/Artifacts) ---
+ final_artifacts_str = ""
+
+ # Case: Cả 2 Fake -> Show cả 2
+ if img_is_fake and txt_is_fake:
+ final_artifacts_str = f"**[Image Evidence]**\n{img_data_parsed['artifacts']}\n\n**[Text Evidence]**\n{txt_res.get_other_artifacts()}"
+
+ # Case: Chỉ Ảnh Fake -> Show ảnh
+ elif img_is_fake:
+ final_artifacts_str = f"{img_data_parsed['artifacts']}"
+
+ # Case: Chỉ Text Fake -> Show text
+ elif txt_is_fake:
+ final_artifacts_str = f"{txt_res.get_other_artifacts()}"
+
+ # Case: Cả 2 đều REAL -> Show source (nếu có)
+ else:
+ final_artifacts_str = "Both image and text are verified as authentic by our multi-modal pipeline."
+
+ # Check source ảnh (Khác N/A, khác rỗng)
+ img_src = img_data_parsed.get('artifacts', '').strip()
+ if img_src and img_src != "N/A" and "No details" not in img_src:
+ final_artifacts_str += f"\n\n**For Image:** {img_src}"
+
+ # Check source text
+ txt_src = txt_res.get_other_artifacts().strip()
+ if txt_src and txt_src != "N/A":
+ final_artifacts_str += f"\n\n**For Text:** {txt_src}"
+
+ # TẠO FINAL MARKDOWN
+ final_report_md = f"""
+### 📋 Final Verification Report
+
+**1. Authenticity Assessment:**
+{final_auth}
+
+**2. Verification Tools & Methods:**
+{final_tools}
+
+**3. Synthetic Type (if applicable):**
+{final_synth_str}
+
+**4. Other Artifacts:**
+{final_artifacts_str}
+ """
+
+ yield get_ui()[0], get_ui()[1], final_report_md
\ No newline at end of file
diff --git a/miragenews/models/__init__.py b/miragenews/models/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..86d11c235e709999d121d7540ac4df7ea6cc0170
--- /dev/null
+++ b/miragenews/models/__init__.py
@@ -0,0 +1,27 @@
+from .mirage_img import *
+from .mirage_txt import *
+
+def get_model(config):
+ """
+ Retrieves the model class specified in the config and initializes it with provided parameters.
+ """
+ model_name = config['model']['name']
+ model_params = config['model'].get('params', {})
+
+ # Mapping model names to their classes
+ model_classes = {
+ "img-linear": ImageLinearModel,
+ "cbm-encoder": ObjectClassCBMEncoder,
+ "cbm-predictor": ObjectClassCBMPredictor,
+ "mirage-img": MiRAGeImg,
+ "txt-linear": TextLinearModel,
+ "tbm-predictor": TBMPredictor,
+ "mirage-txt": MiRAGeTxt
+ # Add other models here as needed
+ }
+
+ if model_name in model_classes:
+ model_class = model_classes[model_name]
+ return model_class(**model_params) # Instantiate model with parameters
+ else:
+ raise ValueError(f"Model {model_name} not recognized. Please check config.")
diff --git a/miragenews/models/mirage_img.py b/miragenews/models/mirage_img.py
new file mode 100644
index 0000000000000000000000000000000000000000..d83ed56cf9a1c91b3d24cc727777b0436a15ee6d
--- /dev/null
+++ b/miragenews/models/mirage_img.py
@@ -0,0 +1,46 @@
+import torch
+import torch.nn as nn
+
+class ImageLinearModel(nn.Module):
+ def __init__(self):
+ super(ImageLinearModel, self).__init__()
+ self.linear = nn.Linear(1408, 1)
+ self.sigmoid = nn.Sigmoid()
+
+ def forward(self, image_features):
+ x = self.linear(image_features)
+ x = self.sigmoid(x)
+ return x
+
+class ObjectClassCBMEncoder(nn.Module):
+ def __init__(self):
+ super(ObjectClassCBMEncoder, self).__init__()
+ self.classifiers = nn.ModuleList([nn.Sequential(
+ nn.Linear(1408, 1),
+ nn.Sigmoid()
+ ) for _ in range(300)])
+
+ def forward(self, image_features, classifier_index):
+ return self.classifiers[classifier_index](image_features)
+
+class ObjectClassCBMPredictor(nn.Module):
+ def __init__(self):
+ super(ObjectClassCBMPredictor, self).__init__()
+ self.linear = nn.Linear(300, 1)
+ self.sigmoid = nn.Sigmoid()
+
+ def forward(self, logits_per_image):
+ x = self.linear(logits_per_image)
+ x = self.sigmoid(x)
+ return x
+
+class MiRAGeImg(nn.Module):
+ def __init__(self):
+ super(MiRAGeImg, self).__init__()
+ self.linear = nn.Linear(301, 1)
+ self.sigmoid = nn.Sigmoid()
+
+ def forward(self, logits_per_image):
+ x = self.linear(logits_per_image)
+ x = self.sigmoid(x)
+ return x
\ No newline at end of file
diff --git a/miragenews/models/mirage_txt.py b/miragenews/models/mirage_txt.py
new file mode 100644
index 0000000000000000000000000000000000000000..8ec707b06edb753a97f252923fca9eda0297f4d1
--- /dev/null
+++ b/miragenews/models/mirage_txt.py
@@ -0,0 +1,46 @@
+import torch
+import torch.nn as nn
+
+class TextLinearModel(nn.Module):
+ def __init__(self):
+ super(TextLinearModel, self).__init__()
+ self.linear = nn.Linear(768, 1)
+ self.sigmoid = nn.Sigmoid()
+
+ def forward(self, text_features):
+ x = self.linear(text_features)
+ x = self.sigmoid(x)
+ return x
+
+# class TBMEncoder(nn.Module):
+# def __init__(self):
+# super(TBMEncoder, self).__init__()
+# self.classifiers = nn.ModuleList([nn.Sequential(
+# nn.Linear(1408, 1),
+# nn.Sigmoid()
+# ) for _ in range(300)])
+
+# def forward(self, image_features, classifier_index):
+# return self.classifiers[classifier_index](image_features)
+
+class TBMPredictor(nn.Module):
+ def __init__(self):
+ super(TBMPredictor, self).__init__()
+ self.linear = nn.Linear(18, 1)
+ self.sigmoid = nn.Sigmoid()
+
+ def forward(self, logits_per_text):
+ x = self.linear(logits_per_text)
+ x = self.sigmoid(x)
+ return x
+
+class MiRAGeTxt(nn.Module):
+ def __init__(self):
+ super(MiRAGeTxt, self).__init__()
+ self.linear = nn.Linear(19, 1)
+ self.sigmoid = nn.Sigmoid()
+
+ def forward(self, logits_per_text):
+ x = self.linear(logits_per_text)
+ x = self.sigmoid(x)
+ return x
\ No newline at end of file
diff --git a/miragenews/my_dataset/text/my_single_text_dir/real/real.txt b/miragenews/my_dataset/text/my_single_text_dir/real/real.txt
new file mode 100644
index 0000000000000000000000000000000000000000..6804be6d154e92fa47e3710e7d290575086450e4
--- /dev/null
+++ b/miragenews/my_dataset/text/my_single_text_dir/real/real.txt
@@ -0,0 +1 @@
+An immigrant viciously attacks medical personnel after being wounded in clashes with residents of Rosarno, in the southern region of Calabria.
diff --git a/miragenews/requirements.txt b/miragenews/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..78dc1d737a92e3986caa42f6b09d3c16a1339e82
--- /dev/null
+++ b/miragenews/requirements.txt
@@ -0,0 +1,19 @@
+beautifulsoup4==4.14.3
+clip==1.0
+datasets==4.3.0
+gradio==6.1.0
+httpx==0.28.1
+nltk==3.9.2
+numpy < 2
+Pillow==12.0.0
+protobuf==6.33.2
+python-dotenv==1.2.1
+PyYAML==6.0.3
+Requests==2.32.5
+scikit_learn==1.8.0
+sentence_transformers==5.1.2
+tqdm==4.67.1
+trafilatura==2.0.0
+transformers==4.57.1
+
+
diff --git a/miragenews/scrappingdog.py b/miragenews/scrappingdog.py
new file mode 100644
index 0000000000000000000000000000000000000000..9cef89dd75dc3eca5f2ab4a5eaabf60657e6320a
--- /dev/null
+++ b/miragenews/scrappingdog.py
@@ -0,0 +1,189 @@
+import requests
+from bs4 import BeautifulSoup
+import google.generativeai as genai
+import re
+
+headers = {
+ "User-Agent": "Mozilla/5.0",
+ "Referer": "https://www.duixs.com/"
+}
+domain = "https://www.duixs.com"
+
+def translate_with_gemini(text, target_lang="vi", api_key="AIzaSyBnOoyhsmBjxLB5eHC99soihh7dC3WvQSk",gemini_version="gemini-2.5-flash"):
+ genai.configure(api_key=api_key)
+
+ prompt = f"""
+ Hãy đóng vai một biên dịch viên chuyên nghiệp về tiểu thuyết Tiên Hiệp/Kiếm Hiệp Trung Quốc.
+ Nhiệm vụ của bạn là dịch văn bản sau từ tiếng Trung sang tiếng Việt.
+
+ Yêu cầu quan trọng:
+ 1. Văn phong: Hùng hồn, cổ trang, giữ được cái "chất" của truyện tu tiên.
+ 2. Thuật ngữ: Giữ nguyên các từ Hán Việt đặc thù của thể loại này (ví dụ: Tu vi, Động phủ, Nguyên anh, Pháp bảo...).
+ 3. Đại từ nhân xưng: Dựa vào ngữ cảnh để chọn đại từ phù hợp (Ta/Ngươi, Huynh/Muội, Lão phu/Tiểu tử, Tại hạ/Các hạ...). Tuyệt đối không dùng "tôi/bạn".
+ 4. Tên riêng và địa danh: Phiên âm Hán Việt chuẩn.
+ 5. Định dạng: Giữ nguyên tuyệt đối cấu trúc xuống dòng và hội thoại của bản gốc.
+ 6. Kiểm tra kỹ lỗi chính tả và ngữ pháp, đảm bảo văn bản cuối cùng không có lỗi.
+ 7. Chỉ trả về văn bản đã được dịch, KHÔNG thêm bất kỳ lời giải thích hay chú thích nào khác.
+
+ Văn bản cần dịch:
+ {text}
+ """
+
+ model = genai.GenerativeModel(gemini_version)
+
+ response = model.generate_content(prompt)
+
+ return response.text.strip()
+
+
+# ---------------------------
+# Lấy nội dung 1 chương
+# ---------------------------
+def get_chapter_content(url):
+ # 1. Tải trang đầu tiên
+ print(f"Đang xử lý: {url}")
+ try:
+ html = requests.get(url, headers=headers).text
+ soup = BeautifulSoup(html, "html.parser")
+
+ # Lấy nội dung trang 1
+ full_text = []
+ book_div = soup.select_one("#booktxt")
+ if book_div:
+ full_text.append(book_div.get_text(separator="\n", strip=True))
+
+
+ title_tag = soup.select_one(".bookname")
+
+ total_pages = 1
+ if title_tag:
+ title_text = title_tag.get_text()
+ match = re.search(r'\((\d+)/(\d+)\)', title_text) or re.search(r'(\d+)/(\d+)', title_text)
+
+ if match:
+ total_pages = int(match.group(2))
+ print(f" -> Phát hiện chương có {total_pages} trang con.")
+
+ # 3. Nếu có nhiều hơn 1 trang -> Tự sinh URL
+ if total_pages > 1:
+ # Xử lý cắt đuôi .html để chèn số
+ if url.endswith(".html"):
+ base_url = url[:-5] # Cắt bỏ .html
+ extension = ".html"
+ else:
+ base_url = url
+ extension = ""
+
+ # Vòng lặp từ trang 2 đến trang cuối
+ for i in range(2, total_pages + 1):
+ # Quy luật: url_goc_2.html, url_goc_3.html
+ sub_url = f"{base_url}_{i}{extension}"
+ print(f" -> Đang cào phần {i}/{total_pages}: {sub_url}")
+
+ try:
+ sub_html = requests.get(sub_url, headers=headers).text
+ sub_soup = BeautifulSoup(sub_html, "html.parser")
+ sub_div = sub_soup.select_one("#booktxt")
+ if sub_div:
+ full_text.append(sub_div.get_text(separator="\n", strip=True))
+ except Exception as e:
+ print(f"Lỗi cào trang {sub_url}: {e}")
+
+ return "\n\n".join(full_text)
+
+ except Exception as e:
+ return f"Lỗi xử lý chương: {e}"
+
+# ---------------------------
+# Lấy thông tin cơ bản của truyện
+# ---------------------------
+def get_book_info(url):
+ html = requests.get(url, headers=headers).text
+ soup = BeautifulSoup(html, "html.parser")
+
+ name = soup.select_one('div#info h1').get_text(strip=True)
+ authors = soup.select_one('div#info h1 + p').get_text(strip=True)
+
+ # Xử lý intro
+ intro_div = soup.select_one('div#intro')
+ bad = intro_div.select_one('#downapi')
+ if bad:
+ bad.decompose()
+ intro = intro_div.get_text(strip=True)
+
+ return name, authors, intro, soup
+
+
+# ---------------------------
+# Lấy danh sách URL index page
+# ---------------------------
+def get_index_pages(first_chapter_href):
+ # "txt_123456/xxxx.html" → lấy phần txt_123456
+ part = first_chapter_href.split('/')[2]
+
+ index_url = f"{domain}/read/{part}.html"
+
+ html = requests.get(index_url, headers=headers).text
+ soup = BeautifulSoup(html, "html.parser")
+
+ select_tag = soup.select_one('#indexselect')
+ options = select_tag.select('option')
+
+ index_pages = [domain + opt['value'] for opt in options]
+ return index_pages
+
+
+# ---------------------------
+# Lấy danh sách tất cả chapter
+# ---------------------------
+def get_all_chapters(index_pages):
+ chapter_list = []
+
+ for page in index_pages:
+ html = requests.get(page, headers=headers).text
+ soup = BeautifulSoup(html, "html.parser")
+
+ links = soup.select('div#list dl a[rel="chapter"]')
+ for link in links:
+ chapter_title = link.get_text(strip=True)
+ chapter_url = domain + link['href']
+ chapter_list.append((chapter_title, chapter_url))
+
+ return chapter_list
+
+
+# ---------------------------
+# MAIN
+# ---------------------------
+def main():
+ URL = "https://www.duixs.com/txt_zsg.html"
+
+ # Lấy thông tin truyện
+ name, authors, intro, soup = get_book_info(URL)
+
+ print("Name:", name)
+ print("Authors:", authors)
+ print("Intro:", intro)
+
+ # Lấy link chapter đầu tiên
+ first_chapter = soup.select_one('div#list dl a[rel="chapter"]')
+ first_chapter_url = domain + first_chapter['href']
+ print("First chapter:", first_chapter_url)
+
+ # Lấy danh sách các index pages
+ index_pages = get_index_pages(first_chapter['href'])
+ print("\nTotal index pages:", len(index_pages))
+
+ # Lấy toàn bộ chapter
+ chapters = get_all_chapters(index_pages)
+ print("Total chapters found:", len(chapters))
+
+
+ title, url = chapters[0]
+ print("\n>>> Chapter:", translate_with_gemini(title))
+ print(translate_with_gemini(get_chapter_content(url), gemini_version="gemini-3-pro-preview"))
+
+
+# Chạy
+if __name__ == "__main__":
+ main()
diff --git a/miragenews/test_single_pair.py b/miragenews/test_single_pair.py
new file mode 100644
index 0000000000000000000000000000000000000000..383b6696c8ae063c95d590dc10c7f2c761b04c15
--- /dev/null
+++ b/miragenews/test_single_pair.py
@@ -0,0 +1,131 @@
+import torch
+from models import get_model
+from data import load_config
+from utils import load_model_checkpoint
+import os
+
+# --- 1. TẢI CONFIGS VÀ KHỞI TẠO MODEL (Làm 1 lần) ---
+print("Loading configs and initializing models ONCE...")
+device = "cuda" if torch.cuda.is_available() else "cpu"
+print(f"Using device: {device}")
+
+try:
+ config_img = load_config("configs/image/mirage.yaml")
+ config_multi = load_config("configs/multimodal/mirage.yaml")
+
+ mirage_img = get_model(config_img).to(device)
+
+ # --- 2. Tải Checkpoint Model (Làm 1 lần) ---
+ print("Loading model checkpoint ONCE...")
+ checkpoint_path_img = config_multi['training']['image_model_path']
+ if not os.path.exists(checkpoint_path_img):
+ print(f"FATAL ERROR: Model checkpoint not found at {checkpoint_path_img}")
+ exit()
+ mirage_img, _ = load_model_checkpoint(mirage_img, checkpoint_path_img)
+ print(f"Loaded image model from {checkpoint_path_img}")
+
+ # --- 3. Bật chế độ Evaluation (Làm 1 lần) ---
+ mirage_img.eval()
+ print("Model set to evaluation mode.")
+
+except Exception as e:
+ print(f"FATAL ERROR during model initialization or checkpoint loading: {e}")
+ exit()
+
+# --- 4. HÀM DỰ ĐOÁN TỪ FILE .PT ---
+def predict_authenticity_from_pt(pt_file_path, model, device):
+ """
+ Tải dữ liệu encoding từ file .pt và dự đoán tính xác thực (real/fake).
+
+ Args:
+ pt_file_path (str): Đường dẫn đến file .pt chứa tensor [1, 301].
+ model (torch.nn.Module): Model ảnh đã được tải checkpoint và đặt ở eval mode.
+ device (str): Thiết bị 'cuda' hoặc 'cpu'.
+
+ Returns:
+ tuple: (probability_fake, prediction_label)
+ - probability_fake (float): Xác suất ảnh là GIẢ (0.0 đến 1.0).
+ - prediction_label (str): Nhãn dự đoán ("real" hoặc "fake").
+ Trả về (None, None) nếu có lỗi.
+ """
+ print(f"\n--- Processing: {pt_file_path} ---")
+
+ # 4.1. Kiểm tra file tồn tại
+ if not os.path.exists(pt_file_path):
+ print(f"ERROR: Input file not found at {pt_file_path}")
+ return None, None
+
+ # 4.2. Tải dữ liệu encoding
+ try:
+ image_encodings = torch.load(pt_file_path).to(device)
+ # Kiểm tra shape (phải là [1, 301])
+ if image_encodings.shape != (1, 301):
+ print(f"ERROR: Expected tensor shape [1, 301], but got {image_encodings.shape} from {pt_file_path}")
+ return None, None
+ print(f"Loaded image data (301-dim) with shape: {image_encodings.shape}")
+
+ except Exception as e:
+ print(f"ERROR loading or checking data from {pt_file_path}: {e}")
+ return None, None
+
+ # 4.3. Chạy Inference (Dự đoán)
+ try:
+ with torch.no_grad():
+ output_logits_img = model(image_encodings) # Model ảnh nhận [1, 301]
+ except Exception as e:
+ print(f"ERROR during model inference for {pt_file_path}: {e}")
+ return None, None
+
+ # 4.4. Xử lý kết quả (Logic: 1 = fake)
+ probs_img = torch.sigmoid(output_logits_img)
+ probability_fake = probs_img.squeeze().item() # Xác suất ảnh là GIẢ
+
+ # Quyết định nhãn dựa trên ngưỡng 0.5 (Đảo ngược logic)
+ # Nếu xác suất "fake" >= 0.5 -> fake
+ # Nếu xác suất "fake" < 0.5 -> real
+ prediction_label = "fake" if probability_fake >= 0.5 else "real"
+
+ print(f"Raw output (logits interpreted as 'fake' logit): {output_logits_img.squeeze().item():.4f}")
+ # Cập nhật chuỗi giải thích
+ print(f"Probability (0=real, 1=fake): {probability_fake:.4f}")
+ print(f"Predicted Label: {prediction_label}")
+
+ # Trả về xác suất fake và nhãn
+ return probability_fake, prediction_label
+
+# --- 5. CÁCH SỬ DỤNG HÀM ---
+if __name__ == "__main__":
+ # --- XỬ LÝ MỘT FILE ---
+ # Thay thế bằng đường dẫn đến file .pt bạn muốn kiểm tra
+ input_pt_path_single = "encodings/predictions/image/merged/my_single_image_dir/real.pt"
+
+ print("\n--- Processing a single file ---")
+ prob_fake_single, label_single = predict_authenticity_from_pt(input_pt_path_single, mirage_img, device)
+
+ if prob_fake_single is not None and label_single is not None:
+ print(f"\nFinal result for {input_pt_path_single}: Probability Fake={prob_fake_single:.4f}, Label='{label_single}'")
+ else:
+ print(f"\nFailed to process {input_pt_path_single}.")
+
+ print("\n" + "="*50 + "\n") # Thêm dòng phân cách
+
+ # --- VÍ DỤ XỬ LÝ NHIỀU FILE ---
+ pt_files_to_check = [
+ "encodings/predictions/image/merged/my_single_image_dir/real.pt", # Thay bằng đường dẫn file thật
+ # "encodings/predictions/image/merged/another_dir/fake_image.pt", # ĐÃ XÓA DÒNG NÀY
+ "path/to/nonexistent.pt" # Ví dụ file không tồn tại
+ ]
+ print("\n--- Processing multiple files ---")
+ results = {}
+ for file_path in pt_files_to_check:
+ prob_fake, label = predict_authenticity_from_pt(file_path, mirage_img, device)
+ results[file_path] = (prob_fake, label) # Lưu kết quả vào dictionary
+
+ print("\n--- Summary ---")
+ for file, (prob_fake, label) in results.items():
+ if prob_fake is not None:
+ print(f"{file}: Prob Fake={prob_fake:.4f}, Label='{label}'")
+ else:
+ print(f"{file}: Processing FAILED")
+
+ print("\nScript finished.")
diff --git a/miragenews/text_module/TextAnalysisResult.py b/miragenews/text_module/TextAnalysisResult.py
new file mode 100644
index 0000000000000000000000000000000000000000..11a12e4027a1e23f0eb0edbdada10d5834844aed
--- /dev/null
+++ b/miragenews/text_module/TextAnalysisResult.py
@@ -0,0 +1,70 @@
+import json
+import os
+
+class TextAnalysisResult:
+ def __init__(self,
+ authenticity_assessment: str = "",
+ verification_tools_methods: str = "",
+ synthetic_type: str = "",
+ other_artifacts: str = ""):
+
+ self._authenticity_assessment = authenticity_assessment
+ self._verification_tools_methods = verification_tools_methods
+ self._synthetic_type = synthetic_type
+ self._other_artifacts = other_artifacts
+
+ def get_authenticity_assessment(self) -> str:
+ return self._authenticity_assessment
+
+ def get_verification_tools_methods(self) -> str:
+ return self._verification_tools_methods
+
+ def get_synthetic_type(self) -> str:
+ return self._synthetic_type
+
+ def get_other_artifacts(self) -> str:
+ return self._other_artifacts
+
+ def set_authenticity_assessment(self, value: str):
+ self._authenticity_assessment = value
+
+ def set_verification_tools_methods(self, value: str):
+ self._verification_tools_methods = value
+
+ def set_synthetic_type(self, value: str):
+ self._synthetic_type = value
+
+ def set_other_artifacts(self, value: str):
+ self._other_artifacts = value
+
+ def save_json(self, file_path: str):
+ """
+ Lưu đối tượng vào file JSON theo kiến trúc yêu cầu.
+ """
+ data = {
+ "Authenticity Assessment": self._authenticity_assessment,
+ "Verification Tools & Methods": self._verification_tools_methods,
+ "Synthetic Type (if applicable)": self._synthetic_type,
+ "Other Artifacts": self._other_artifacts
+ }
+
+ try:
+ # Đảm bảo thư mục tồn tại
+ directory = os.path.dirname(file_path)
+ if directory and not os.path.exists(directory):
+ os.makedirs(directory)
+
+ with open(file_path, 'w', encoding='utf-8') as f:
+ # ensure_ascii=False để hiển thị tiếng Việt hoặc ký tự đặc biệt đúng
+ json.dump(data, f, indent=4, ensure_ascii=False)
+
+ print(f"✅ Đã lưu kết quả thành công tại: {file_path}")
+ return True
+ except Exception as e:
+ print(f"❌ Lỗi khi lưu file JSON: {e}")
+ return False
+
+ def __str__(self):
+ """Hiển thị đối tượng dưới dạng string để debug nhanh"""
+ return (f"Authenticity: {self._authenticity_assessment} | "
+ f"Method: {self._verification_tools_methods}")
\ No newline at end of file
diff --git a/miragenews/text_module/__init__.py b/miragenews/text_module/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/miragenews/text_module/config.py b/miragenews/text_module/config.py
new file mode 100644
index 0000000000000000000000000000000000000000..a3c03938351053622b5af09334c16ff033f96153
--- /dev/null
+++ b/miragenews/text_module/config.py
@@ -0,0 +1,15 @@
+# config.py
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+# API Keys
+GOOGLE_API_KEY = os.getenv("GOOGLE_CSE_CX") # Dùng cho Gemini (theo code cũ của bạn)
+GOOGLE_SAFE_BROWSING_API_KEY = os.getenv("GOOGLE_SAFE_BROWSING_API_KEY")
+GOOGLE_CX = os.getenv("GOOGLE_CSE_CX") # Dùng cho Search
+SCRAPINGDOG_API_KEY = os.getenv("SCRAPINGDOG_API_KEY")
+GEMINI_API_KEY = os.getenv("GEMINI_API_KEY")
+
+# Constants
+USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
\ No newline at end of file
diff --git a/miragenews/text_module/llm_utils.py b/miragenews/text_module/llm_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..9693da7d2c4c4df2378acdcec8edafb3722b0130
--- /dev/null
+++ b/miragenews/text_module/llm_utils.py
@@ -0,0 +1,77 @@
+import google.generativeai as genai
+from .models import text_llm_model
+
+MAX_TOKENS = 4096
+
+def ask_llm_about_text(text_content):
+ """
+ Hàm này dùng model mặc định (ví dụ gemini-pro) load từ models.py để phân tích sâu.
+ """
+ if not text_llm_model:
+ return "ERROR", "Text LLM Model not initialized."
+
+ prompt = f"""
+ You are an expert AI text forensic analyst.
+
+ TASK: Analyze the text below and determine if it is "Human-written" or "AI-generated".
+
+ TEXT:
+ "{text_content}"
+
+ CRITERIA:
+ - AI: Overly structured, repetitive, perfect grammar but lack of depth, neutral tone, hallucinations.
+ - Human: Contextual, emotional, slang, imperfections, complex sentence structures.
+
+ OUTPUT FORMAT:
+ Strictly follow this format:
+ Status: [LIKELY AI GENERATED / LIKELY HUMAN WRITTEN]
+ Reason: [Short explanation, max 2 sentences]
+ """
+
+ try:
+ generation_config = genai.types.GenerationConfig(
+ temperature=0.0,
+ max_output_tokens=MAX_TOKENS
+ )
+
+ response = text_llm_model.generate_content(
+ prompt,
+ generation_config=generation_config
+ )
+ return "SUCCESS", response.text.strip()
+ except Exception as e:
+ return "ERROR", str(e)
+
+def ask_llm_to_rewrite(text_content):
+
+ if not text_llm_model:
+ return None
+
+ full_prompt = f"""
+ You are a helpful assistant.
+ Paraphrasing for the text: ```{text_content}``` Only output the paraphrased text without explanation.
+ Rewritten Version:
+ """
+
+ try:
+ flash_model = genai.GenerativeModel('gemini-2.0-flash')
+
+ config = genai.types.GenerationConfig(
+ temperature=0.0,
+ max_output_tokens=MAX_TOKENS
+ )
+
+ # 4. Gọi hàm generate_content
+ response = flash_model.generate_content(
+ full_prompt,
+ generation_config=config
+ )
+
+ if response and response.text:
+ return response.text.strip()
+
+ return None
+
+ except Exception as e:
+ print(f"❌ Gemini Flash Rewrite Error: {e}")
+ return None
\ No newline at end of file
diff --git a/miragenews/text_module/models.py b/miragenews/text_module/models.py
new file mode 100644
index 0000000000000000000000000000000000000000..a5d37a0ed8d31e8995209204f9388e85ca879c8c
--- /dev/null
+++ b/miragenews/text_module/models.py
@@ -0,0 +1,21 @@
+# text_module/models.py
+import google.generativeai as genai
+from sentence_transformers import SentenceTransformer
+from dotenv import load_dotenv
+load_dotenv()
+from .config import GEMINI_API_KEY
+
+print("⏳ Loading Text Embedding Model...")
+text_model = SentenceTransformer('intfloat/multilingual-e5-large')
+
+# --- CONFIG GEMINI (Detection) ---
+text_llm_model = None
+try:
+ if GEMINI_API_KEY:
+ genai.configure(api_key=GEMINI_API_KEY)
+ text_llm_model = genai.GenerativeModel('gemini-2.5-pro')
+ print("✅ Google Gemini (Text LLM) configured.")
+ else:
+ print("⚠️ Missing GEMINI_API_KEY.")
+except Exception as e:
+ print(f"⚠️ Error configuring Gemini: {e}")
diff --git a/miragenews/text_module/pipeline.py b/miragenews/text_module/pipeline.py
new file mode 100644
index 0000000000000000000000000000000000000000..3449ae8ae5751b317f469532ad9178f1f0c47b61
--- /dev/null
+++ b/miragenews/text_module/pipeline.py
@@ -0,0 +1,305 @@
+import time
+import re
+import torch
+import numpy as np
+
+# --- GIẢ ĐỊNH CÁC MODULE CỦA BẠN ---
+# Bạn cần đảm bảo các file này tồn tại trong project structure của bạn
+try:
+ from .models import text_model # Model SentenceTransformer
+ from .search import search_google_official, search_scrapingdog_serp
+ from .scraper import scrape_text_content
+ from .llm_utils import ask_llm_about_text, ask_llm_to_rewrite
+except ImportError:
+ print("⚠️ Warning: Import failed. Running in standalone mode logic checking.")
+ pass
+
+# --- CONFIGURATION ---
+ALPHA = 0.96 # Ngưỡng xác định là REAL (Tuyệt đối)
+DELTA = 0.00923 # Ngưỡng chênh lệch AI Hypothesis
+MIN_CANDIDATE_SIM = 0.50 # Ngưỡng tối thiểu để chạy kiểm tra AI (Step 2)
+TIMEOUT_SECONDS = 300 # ⏳ GIỚI HẠN THỜI GIAN: 5 phút (300 giây)
+
+def split_sentences(text):
+ """Tách văn bản thành các câu."""
+ if not text: return []
+ # Tách dựa trên dấu câu kết thúc hoặc xuống dòng
+ sentences = re.split(r'(?<=[.!?])\s+|[\n\r]+', text)
+ return [s.strip() for s in sentences if s.strip()]
+
+def get_similarity(text1, text2):
+ """Tính độ tương đồng Cosine giữa 2 đoạn text dùng model embedding."""
+ if not text1 or not text2: return 0.0
+ try:
+ emb1 = text_model.encode([text1])
+ emb2 = text_model.encode([text2])
+ return text_model.similarity(emb1, emb2)[0][0].item()
+ except Exception as e:
+ print(f"Embedding Error: {e}")
+ return 0.0
+
+def run_text_alignment_algorithm(input_text, url_content_list):
+ """
+ Algorithm 2: EXTRACT_CANDIDATE & COMPUTE SIMILARITY
+ Tìm đoạn văn bản khớp nhất trong nội dung web so với input của người dùng.
+ """
+ St = split_sentences(input_text)
+
+ # Xử lý input đầu vào cho url_content
+ if isinstance(url_content_list, str):
+ Su = split_sentences(url_content_list)
+ else:
+ Su = url_content_list
+
+ if not St or not Su: return 0.0, "", ""
+
+ # Phase 1: Tìm điểm bắt đầu (Anchor alignment)
+ best_b = -1
+ max_sim_start = -1.0
+ target_start = St[0]
+
+ for i, sent_u in enumerate(Su):
+ sim = get_similarity(target_start, sent_u)
+ if best_b == -1 or sim > max_sim_start:
+ max_sim_start = sim
+ best_b = i
+
+ # Cắt bỏ phần đầu không liên quan của web content
+ if best_b != -1 and max_sim_start > 0.4:
+ Su = Su[best_b:]
+
+ # Phase 2: Sliding Window Matching
+ P = []
+ total_score = 0.0
+ count_matches = 0
+ match_details = []
+ candidate_parts = []
+ MAX_WINDOW = 3
+
+ # Copy list để không ảnh hưởng dữ liệu gốc
+ curr_St = list(St)
+ curr_Su = list(Su)
+
+ while curr_Su and curr_St:
+ best_p_score = -1.0
+ best_pair = None
+ remove_from_St = 0
+ remove_from_Su = 0
+
+ # Cố định câu đầu của Input, quét window bên Web
+ target_st_1 = curr_St[0]
+ limit_k = min(len(curr_Su), MAX_WINDOW)
+ for k in range(1, limit_k + 1):
+ Ck = " ".join(curr_Su[:k])
+ sim = get_similarity(target_st_1, Ck)
+ if best_pair is None or sim > best_p_score:
+ best_p_score = sim
+ best_pair = (target_st_1, Ck)
+ remove_from_St = 1
+ remove_from_Su = k
+
+ # Cố định câu đầu của Web, quét window bên Input
+ target_su_1 = curr_Su[0]
+ limit_l = min(len(curr_St), MAX_WINDOW)
+ for l in range(1, limit_l + 1):
+ Cl = " ".join(curr_St[:l])
+ sim = get_similarity(Cl, target_su_1)
+ if sim > best_p_score:
+ best_p_score = sim
+ best_pair = (Cl, target_su_1)
+ remove_from_St = l
+ remove_from_Su = 1
+
+ # Cập nhật kết quả matching
+ if best_pair:
+ P.append(best_pair)
+ total_score += best_p_score
+ count_matches += 1
+
+ # Lưu lại đoạn text khớp từ web để làm evidence
+ if remove_from_Su > 1:
+ web_part = " ".join(curr_Su[:remove_from_Su])
+ else:
+ web_part = curr_Su[0]
+ candidate_parts.append(web_part)
+
+ if best_p_score > 0.6:
+ match_details.append(f"• Sim: {best_p_score:.2f} | Web: \"{best_pair[1][:50]}...\"")
+
+ curr_St = curr_St[remove_from_St:]
+ curr_Su = curr_Su[remove_from_Su:]
+ else:
+ # Nếu không khớp tí nào, bỏ qua câu hiện tại
+ curr_St.pop(0)
+ curr_Su.pop(0)
+
+ final_avg_score = (total_score / count_matches) if count_matches > 0 else 0.0
+ candidate_content_str = " ".join(candidate_parts)
+
+ return final_avg_score, "\n".join(match_details), candidate_content_str
+
+def verify_text_logic(user_input, result_object):
+ """
+ Hàm chính để xác thực văn bản.
+ Args:
+ user_input (str): Văn bản cần kiểm tra.
+ result_object: Object chứa kết quả (có các method set_...).
+ """
+
+ # [TIMER START] Bắt đầu tính giờ
+ start_time = time.time()
+ timed_out = False
+
+ # 0. Validate input cơ bản
+ if not user_input or len(user_input.strip()) < 5:
+ result_object.set_authenticity_assessment("⚠️ INSUFFICIENT DATA")
+ result_object.set_verification_tools_methods("Input Validation")
+ result_object.set_synthetic_type("N/A")
+ result_object.set_other_artifacts("Input text is too short or empty.")
+ return
+
+ # Clear GPU cache nếu có
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+
+ # Biến lưu trữ kết quả tốt nhất tìm được từ Search
+ best_candidate = {
+ "url": "",
+ "score_c": 0.0,
+ "content_tc": "",
+ "evidence": ""
+ }
+
+ # Danh sách chiến lược tìm kiếm
+ search_strategies = [
+ ("Google CSE", search_google_official),
+ ("ScrapingDog", search_scrapingdog_serp)
+ ]
+
+ tools_used_list = ["SearchLLM Algorithm", "Sentence Transformer"]
+ found_any_url = False
+
+ # --- BƯỚC 1: TÌM KIẾM & PHÂN TÍCH CHUYÊN SÂU ---
+ print(f"🚀 Starting Text Analysis for: {user_input[:30]}...")
+
+ for strategy_name, search_func in search_strategies:
+ # [CHECK TIMEOUT] Kiểm tra trước khi bắt đầu strategy mới
+ if time.time() - start_time > TIMEOUT_SECONDS:
+ timed_out = True
+ break
+
+ try:
+ print(f" Using {strategy_name}...")
+ urls = search_func(user_input, num_results=5)
+ except Exception as e:
+ print(f" Error in search {strategy_name}: {e}")
+ continue
+
+ if not urls: continue
+ found_any_url = True
+ tools_used_list.append(strategy_name)
+
+ # Duyệt qua từng URL tìm được
+ for i, url in enumerate(urls):
+ if time.time() - start_time > TIMEOUT_SECONDS:
+ print(f"⏳ TIMEOUT reached ({TIMEOUT_SECONDS}s). Aborting Search/Scrape.")
+ timed_out = True
+ break
+
+ # Filter cơ bản
+ if any(blocked in url for blocked in []):
+ continue
+
+ # 1.1 Scrape Content
+ try:
+ url_sentences = scrape_text_content(url)
+ if not url_sentences: continue
+
+ # 1.2 Run Matching Algorithm (Tốn thời gian nhất)
+ sigma_c, evidence_str, tc_content = run_text_alignment_algorithm(user_input, url_sentences)
+
+ # Cập nhật kết quả tốt nhất
+ if sigma_c > best_candidate["score_c"]:
+ best_candidate["score_c"] = sigma_c
+ best_candidate["url"] = url
+ best_candidate["content_tc"] = tc_content
+ best_candidate["evidence"] = evidence_str
+
+ # CASE 1: REAL - Nếu khớp gần như tuyệt đối -> Return ngay
+ if sigma_c >= ALPHA:
+ print(f"✅ Found exact match (Sim: {sigma_c:.4f}) at {url}")
+ result_object.set_authenticity_assessment("✅ REAL (Authentic)")
+ result_object.set_verification_tools_methods(", ".join(tools_used_list))
+ result_object.set_synthetic_type("Human/Existing Source")
+
+ artifacts = (f"An exact match has been found.\n"
+ f"Source: {url}\n"
+ f"Similarity Score: {sigma_c:.4f}")
+ result_object.set_other_artifacts(artifacts)
+ return # KẾT THÚC HÀM
+
+ except Exception as e:
+ print(f"⚠️ Error processing URL {url}: {e}")
+ continue
+
+ # Break vòng lặp chiến lược nếu đã timeout trong vòng lặp URL
+ if timed_out:
+ break
+
+ if not timed_out and best_candidate["score_c"] >= MIN_CANDIDATE_SIM:
+
+ # Check thời gian lần cuối trước khi gọi LLM Rewrite
+ if time.time() - start_time <= TIMEOUT_SECONDS:
+ print("🤖 Running AI Regeneration Hypothesis check...")
+ tools_used_list.append("LLM Regeneration Analysis")
+
+ tr_content = ask_llm_to_rewrite(best_candidate["content_tc"])
+
+ if tr_content:
+ sigma_r = get_similarity(user_input, tr_content)
+ diff = sigma_r - best_candidate["score_c"]
+
+ print(f" Delta Diff: {diff:.4f} (Threshold: {DELTA})")
+
+ # CASE 2: AI DETECTED (Nếu bản rewrite giống input hơn bản gốc trên web)
+ if diff >= DELTA:
+ result_object.set_authenticity_assessment("🤖 NOT REAL (Fake, Manipulated, or AI)")
+ result_object.set_verification_tools_methods(", ".join(tools_used_list))
+ result_object.set_synthetic_type("AI Generated (Derived from Source)")
+
+ artifacts = (f"Detected AI generation pattern by our system.\n"
+ f"Potential Source: {best_candidate['url']}\n"
+ f"Rewrite Similarity Gain: +{diff:.4f}")
+ result_object.set_other_artifacts(artifacts)
+ return # KẾT THÚC HÀM
+ else:
+ timed_out = True # Đánh dấu timeout nếu sát giờ quá
+
+ print("🔻 Switching to Direct LLM Analysis (Fallback)...")
+ tools_used_list.append("Direct LLM Analysis")
+
+ # Gọi LLM trực tiếp để đánh giá
+ status_llm, response_llm = ask_llm_about_text(user_input)
+
+ # CASE 3: FALLBACK RESULT
+ if "AI GENERATED" in str(response_llm).upper():
+ auth_status = "🤖 NOT REAL (Fake, Manipulated, or AI)"
+ syn_type = "AI Generated Text"
+ verdict_color = "#dc2626" # Red
+ else:
+ auth_status = "✅ REAL (Authentic)"
+ syn_type = "Human Written"
+ verdict_color = "#16a34a" # Green
+
+ # Set kết quả cuối cùng
+ result_object.set_authenticity_assessment(auth_status)
+ result_object.set_verification_tools_methods(", ".join(tools_used_list))
+ result_object.set_synthetic_type(syn_type)
+
+ # Artifacts bao gồm thông tin fallback
+ final_artifacts = (f"We identified matching text from the provided URL: {best_candidate['url'] if best_candidate['url'] else 'None'}\n\n"
+ f"-------------------\n"
+ f"LLM Reasoning: {response_llm}")
+
+ result_object.set_other_artifacts(final_artifacts)
+ print("🏁 Text Verification Completed.")
\ No newline at end of file
diff --git a/miragenews/text_module/scraper.py b/miragenews/text_module/scraper.py
new file mode 100644
index 0000000000000000000000000000000000000000..b881fe07327b32d3761bb92c20e31bf8b85b95d7
--- /dev/null
+++ b/miragenews/text_module/scraper.py
@@ -0,0 +1,72 @@
+# text_module/scraper.py
+import requests
+import nltk
+import shutil
+import os
+import zipfile
+from bs4 import BeautifulSoup
+import trafilatura
+
+try:
+ from config import SCRAPINGDOG_API_KEY, USER_AGENT
+except ImportError:
+ from .config import SCRAPINGDOG_API_KEY, USER_AGENT
+
+# --- NLTK SETUP ---
+def setup_nltk():
+ try:
+ nltk.data.find('tokenizers/punkt')
+ nltk.data.find('tokenizers/punkt_tab')
+ except (LookupError, zipfile.BadZipFile, OSError):
+ try:
+ for path in nltk.data.path:
+ punkt_path = os.path.join(path, 'tokenizers', 'punkt')
+ if os.path.exists(punkt_path): shutil.rmtree(punkt_path, ignore_errors=True)
+ except: pass
+ nltk.download('punkt')
+ nltk.download('punkt_tab')
+
+setup_nltk()
+
+def parse_text_html(html_content):
+ if not html_content: return []
+
+ text = trafilatura.extract(html_content, include_comments=False, include_tables=False)
+
+ if not text or len(text) < 50: return []
+
+ text = text.replace('\n', ' ')
+ return nltk.sent_tokenize(text)
+
+# --- HÀM CHÍNH ---
+def scrape_text_content(url):
+ """
+ Cào dữ liệu: Thử Requests thường -> Lỗi -> Thử ScrapingDog API
+ """
+ try:
+ headers = {'User-Agent': USER_AGENT}
+ response = requests.get(url, headers=headers, timeout=5)
+ if response.status_code == 200:
+ content = parse_text_html(response.text)
+ if content:
+ return content
+ except Exception:
+ pass
+
+ if SCRAPINGDOG_API_KEY:
+ try:
+ # dynamic=true để xử lý trang web dùng JS (React/Vue...)
+ sd_url = "https://api.scrapingdog.com/scrape"
+ params = {
+ 'api_key': SCRAPINGDOG_API_KEY,
+ 'url': url,
+ 'dynamic': 'true'
+ }
+ r = requests.get(sd_url, params=params, timeout=20)
+ if r.status_code == 200:
+ return parse_text_html(r.text)
+ except Exception as e:
+ print(f"ScrapingDog Error: {e}")
+
+
+ return []
\ No newline at end of file
diff --git a/miragenews/text_module/search.py b/miragenews/text_module/search.py
new file mode 100644
index 0000000000000000000000000000000000000000..b0919c79f9c9d695124119df7a653c5c041a4731
--- /dev/null
+++ b/miragenews/text_module/search.py
@@ -0,0 +1,54 @@
+# text_module/search.py
+import requests
+from .config import GOOGLE_SAFE_BROWSING_API_KEY, GOOGLE_CX, SCRAPINGDOG_API_KEY
+
+def search_google_official(query, num_results=10):
+ """Vòng 1: Dùng Google Custom Search API (Official)"""
+ if not GOOGLE_SAFE_BROWSING_API_KEY or not GOOGLE_CX:
+ print("⚠️ Thiếu Google API Key/CX")
+ return []
+
+ url = "https://www.googleapis.com/customsearch/v1"
+ params = {
+ 'key': GOOGLE_SAFE_BROWSING_API_KEY,
+ 'cx': GOOGLE_CX,
+ 'q': query,
+ 'num': num_results
+ }
+ try:
+ response = requests.get(url, params=params, timeout=10)
+ if response.status_code == 200:
+ data = response.json()
+ return [item['link'] for item in data.get('items', [])]
+ except Exception as e:
+ print(f"❌ Google Official Search Error: {e}")
+ return []
+
+def search_scrapingdog_serp(query, num_results=10):
+ """Vòng 2: Dùng ScrapingDog SERP API"""
+ if not SCRAPINGDOG_API_KEY:
+ print("⚠️ Thiếu SCRAPINGDOG_API_KEY")
+ return []
+
+ url = "https://api.scrapingdog.com/google_search"
+ params = {
+ 'api_key': SCRAPINGDOG_API_KEY,
+ 'query': query,
+ 'results': num_results,
+ 'country': 'vn'
+ }
+
+ try:
+ print(f"🔎 Calling ScrapingDog SERP for: {query}...")
+ response = requests.get(url, params=params, timeout=20)
+ if response.status_code == 200:
+ data = response.json()
+ links = []
+ if 'organic_data' in data:
+ for item in data['organic_data']:
+ if 'link' in item:
+ links.append(item['link'])
+ return links
+ except Exception as e:
+ print(f"❌ ScrapingDog Search Error: {e}")
+ return []
\ No newline at end of file
diff --git a/miragenews/utils/__init__.py b/miragenews/utils/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..4fdede4fb3e78fb1a30483f19b83bb449f6e3285
--- /dev/null
+++ b/miragenews/utils/__init__.py
@@ -0,0 +1,224 @@
+import torch
+import numpy as np
+import json
+from tqdm import tqdm
+from .metrics import calculate_metrics, find_best_threshold
+from miragenews.models import *
+
+# === SHARED FUNCTIONS ===
+
+def save_model_checkpoint(model, save_path, threshold):
+ """
+ Save the model state and threshold as a checkpoint.
+
+ Args:
+ model (torch.nn.Module): The model to save.
+ save_path (str): Path to save the checkpoint.
+ threshold (float): The best threshold to save.
+ """
+ checkpoint = {
+ "model_state_dict": model.state_dict(),
+ "best_threshold": threshold,
+ }
+ torch.save(checkpoint, save_path)
+
+
+def load_model_checkpoint(model, save_path):
+ """
+ Load the model state and best threshold from a checkpoint.
+
+ Args:
+ model (torch.nn.Module): The model to load the state into.
+ save_path (str): Path to the checkpoint.
+
+ Returns:
+ model (torch.nn.Module): Model loaded with state_dict.
+ best_threshold (float): Best threshold saved in the checkpoint.
+ """
+ checkpoint = torch.load(save_path)
+
+ model.load_state_dict(checkpoint["model_state_dict"])
+ best_threshold = checkpoint["best_threshold"] if "best_threshold" in checkpoint else None
+ return model, best_threshold
+
+
+def evaluate_model(model, data_loader, criterion, device="cuda", threshold=0.5, cbm_encoder=None, concept_num=300):
+ """
+ Evaluate the model on a given dataset, calculating metrics using a specified threshold.
+
+ Args:
+ model (torch.nn.Module): The model to evaluate.
+ data_loader (DataLoader): DataLoader for the dataset.
+ criterion: Loss function.
+ device (str): Device for computation.
+ threshold (float): Threshold for binary classification.
+ cbm_encoder (torch.nn.Module, optional): CBM encoder model, if using CBM Predictor.
+ concept_num (int): Number of concepts in CBM Predictor.
+
+ Returns:
+ avg_loss (float): Average loss over the dataset.
+ metrics (dict): Calculated metrics using the specified threshold.
+ """
+ model.eval()
+ y_true = []
+ y_probs = []
+ total_loss = 0.0
+
+ with torch.no_grad():
+ for inputs, labels in data_loader:
+ inputs, labels = inputs.to(device), labels.to(device)
+
+ # Forward pass logic based on model type
+ if isinstance(model, ObjectClassCBMEncoder):
+ classifier_idx = 0
+ outputs = model(inputs.float(), classifier_idx)
+ # elif isinstance(model, ObjectClassCBMPredictor) and cbm_encoder:
+ # concept_features = [cbm_encoder(inputs.float(), i) for i in range(concept_num)]
+ # pred_scores = torch.cat(concept_features, dim=1)
+ # outputs = model(pred_scores.to(device))
+ else:
+ outputs = model(inputs.float())
+
+ # Calculate loss and store probabilities
+
+ loss = criterion(outputs.squeeze(-1), labels.float())
+ total_loss += loss.item()
+ y_true.extend(labels.cpu().numpy())
+ y_probs.extend(outputs.squeeze(-1).cpu().numpy())
+
+ avg_loss = total_loss / len(data_loader)
+ y_true = np.array(y_true)
+ y_probs = np.array(y_probs)
+ metrics = calculate_metrics(y_true, y_probs, threshold)
+
+ model.train()
+ return avg_loss, metrics, y_true, y_probs # Return y_true and y_probs for threshold finding
+
+def save_metrics(metrics, save_path):
+ """
+ Save metrics as a JSON file.
+
+ Args:
+ metrics (dict): Dictionary of metrics to save.
+ save_path (str): Path to the file where metrics will be saved.
+ """
+ with open(save_path, "a") as f:
+ f.write(json.dumps(metrics) + "\n")
+
+# === TRAINING FUNCTION ===
+
+def train_model(model, train_loader, eval_loader, config, classifier_idx=None, cbm_encoder=None, device="cuda"):
+ """
+ Train the model with early stopping and save the best model checkpoint with the threshold.
+ """
+ criterion = torch.nn.BCELoss()
+ optimizer = torch.optim.Adam(model.parameters(), lr=config['training']['learning_rate'])
+ best_val_loss = float('inf')
+ patience_counter = 0
+ best_threshold = 0.5
+
+ for epoch in tqdm(range(config['training']['epochs']), desc="Training"):
+ model.train()
+ running_loss = 0.0
+
+ for batch_idx, (inputs, labels) in enumerate(train_loader):
+ inputs, labels = inputs.to(device), labels.to(device)
+ optimizer.zero_grad()
+ if isinstance(model, ObjectClassCBMEncoder):
+ outputs = model(inputs.float(), classifier_idx)
+ # elif isinstance(model, ObjectClassCBMPredictor) and cbm_encoder:
+ # concept_features = [cbm_encoder(inputs.float(), i) for i in range(config.get('concept_num', 300))]
+ # pred_scores = torch.cat(concept_features, dim=1)
+ # outputs = model(pred_scores.to(device))
+ else:
+ outputs = model(inputs.float())
+
+ loss = criterion(outputs.squeeze(-1), labels.float())
+ loss.backward()
+ optimizer.step()
+ running_loss += loss.item()
+
+ train_loss = running_loss / len(train_loader)
+ # print(f"Epoch {epoch + 1}/{config['training']['epochs']}, Training Loss: {train_loss:.4f}")
+
+ # Evaluate on validation set
+ eval_loss, val_metrics, y_true, y_probs = evaluate_model(model, eval_loader, criterion, device, cbm_encoder=cbm_encoder)
+ # print(f"Validation Loss: {eval_loss:.4f}, Metrics: {val_metrics}")
+
+ # Find and save the best threshold based on validation
+ best_threshold = find_best_threshold(y_true, y_probs)
+ # print(f"Best Threshold Found: {best_threshold}")
+
+ if eval_loss < best_val_loss - 0.001:
+ best_val_loss = eval_loss
+ patience_counter = 0
+ save_model_checkpoint(model, config['training']['save_path'], best_threshold)
+ else:
+ patience_counter += 1
+
+ if patience_counter > 10:
+ print("Early stopping triggered. Training completed.")
+ break
+
+
+# === TESTING FUNCTION ===
+
+def test_model(model, test_loader, checkpoint_path, device="cuda", model_2=None):
+ """
+ Load the best model and threshold, then evaluate on the test set and save metrics.
+
+ Args:
+ model (torch.nn.Module): The trained model.
+ test_loader (DataLoader): DataLoader for the test set.
+ checkpoint_path (str): Path to the saved model checkpoint.
+ device (str): Device for computation (e.g., 'cuda' or 'cpu').
+
+ Returns:
+ dict: Test metrics.
+ """
+ criterion = torch.nn.BCELoss()
+ # Load model and best threshold
+ model, best_threshold = load_model_checkpoint(model, checkpoint_path)
+ # Evaluate on the test set with the loaded threshold
+ test_loss, test_metrics, _, _ = evaluate_model(model, test_loader, criterion, device, threshold=best_threshold)
+ # print("Test Metrics:", test_metrics)
+
+ return test_metrics
+
+def test_multimodal_model(image_model, text_model, test_loader, threshold=0.5, device='cuda'):
+ image_model.eval()
+ text_model.eval()
+ criterion = torch.nn.BCELoss()
+ y_true = []
+ y_probs = []
+ total_loss = 0.0
+
+ with torch.no_grad():
+ for image_inputs, text_inputs, labels in test_loader:
+ image_inputs, text_inputs, labels = image_inputs.to(device), text_inputs.to(device), labels.to(device)
+
+ # Forward pass logic based on model type
+ if isinstance(image_model, ObjectClassCBMEncoder):
+ classifier_idx = 0
+ image_outputs = image_model(image_inputs.float(), classifier_idx)
+ # elif isinstance(model, ObjectClassCBMPredictor) and cbm_encoder:
+ # concept_features = [cbm_encoder(inputs.float(), i) for i in range(concept_num)]
+ # pred_scores = torch.cat(concept_features, dim=1)
+ # outputs = model(pred_scores.to(device))
+ else:
+ image_outputs = image_model(image_inputs.float())
+
+ text_outputs = text_model(text_inputs.float())
+ # Calculate loss and store probabilities
+ outputs = (image_outputs + text_outputs) / 2
+ loss = criterion(outputs.squeeze(-1), labels.float())
+ total_loss += loss.item()
+ y_true.extend(labels.cpu().numpy())
+ y_probs.extend(outputs.squeeze(-1).cpu().numpy())
+
+ avg_loss = total_loss / len(test_loader)
+ y_true = np.array(y_true)
+ y_probs = np.array(y_probs)
+ metrics = calculate_metrics(y_true, y_probs, threshold)
+
+ return metrics
\ No newline at end of file
diff --git a/miragenews/utils/metrics.py b/miragenews/utils/metrics.py
new file mode 100644
index 0000000000000000000000000000000000000000..3087794e6cebef8778da3e40197689d4af73259e
--- /dev/null
+++ b/miragenews/utils/metrics.py
@@ -0,0 +1,56 @@
+from sklearn.metrics import accuracy_score, f1_score, average_precision_score
+import numpy as np
+
+def calculate_metrics(y_true, y_probs, threshold=0.5):
+ """
+ Calculate evaluation metrics for binary classification using a specified threshold.
+
+ Args:
+ y_true (np.array): True binary labels.
+ y_probs (np.array): Predicted probabilities.
+ threshold (float): Threshold to convert probabilities to binary predictions.
+
+ Returns:
+ dict: Dictionary containing accuracy, F1 score, average precision, real accuracy,
+ fake accuracy, and the threshold used.
+ """
+ # Convert probabilities to binary predictions using the specified threshold
+ y_pred = (y_probs > threshold).astype(int)
+
+ # Calculate metrics
+ metrics = {
+ "accuracy": round(accuracy_score(y_true, y_pred), 4),
+ "real_accuracy": round(accuracy_score(y_true[y_true == 0], y_pred[y_true == 0]), 4) if (y_true == 0).any() else 0.0,
+ "fake_accuracy": round(accuracy_score(y_true[y_true == 1], y_pred[y_true == 1]), 4) if (y_true == 1).any() else 0.0,
+ "f1": round(f1_score(y_true, y_pred), 4),
+ "average_precision": round(average_precision_score(y_true, y_probs), 4),
+ "threshold": round(threshold, 4)
+ }
+
+ # Convert all values to native Python types
+ return {k: float(v) if isinstance(v, (np.float32, np.float64)) else v for k, v in metrics.items()}
+
+
+def find_best_threshold(y_true, y_probs):
+ """
+ Find the best threshold for binary classification to maximize accuracy.
+
+ Args:
+ y_true (np.array): True binary labels.
+ y_probs (np.array): Predicted probabilities.
+
+ Returns:
+ float: The best threshold that maximizes accuracy.
+ """
+ thresholds = np.unique(y_probs)
+ best_accuracy = 0.0
+ best_threshold = 0.5
+
+ for thresh in thresholds:
+ y_pred = (y_probs > thresh).astype(int)
+ accuracy = accuracy_score(y_true, y_pred)
+ if accuracy > best_accuracy:
+ best_accuracy = accuracy
+ best_threshold = thresh
+
+ return round(float(best_threshold), 4)
diff --git a/raft_model/raft-things.pth b/raft_model/raft-things.pth
new file mode 100644
index 0000000000000000000000000000000000000000..1e206ac8a2f660bc7620b0806a9278ddb3fc594d
--- /dev/null
+++ b/raft_model/raft-things.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcfa4125d6418f4de95d84aec20a3c5f4e205101715a79f193243c186ac9a7e1
+size 21108000
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..b590b2b234143d5b95fd7b1994b93b51a1d6c027
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,34 @@
+--extra-index-url https://download.pytorch.org/whl/cu121
+torch==2.3.1
+torchvision==0.18.1
+torchaudio==2.3.1
+numpy<2
+pandas
+scikit-learn
+matplotlib
+scipy
+tqdm
+einops
+imageio
+opencv-python
+Pillow==12.0.0
+transformers==4.57.1
+sentence-transformers==5.1.2
+nltk==3.9.2
+tensorboard
+tensorboardX
+blobfile>=1.0.5
+natsort
+trafilatura==2.0.0
+fastapi==0.116.1
+pydantic==2.11.7
+uvicorn[standard]
+python-multipart
+python-dotenv==1.2.1
+httpx==0.28.1
+Requests==2.32.5
+beautifulsoup4==4.14.3
+git+https://github.com/openai/CLIP.git
+google-cloud-vision
+gradio
+google.generativeai
\ No newline at end of file
diff --git a/uploads/video_result.json b/uploads/video_result.json
new file mode 100644
index 0000000000000000000000000000000000000000..2b698277f6eb54a67495248bd11a7dc69c2083d2
--- /dev/null
+++ b/uploads/video_result.json
@@ -0,0 +1,6 @@
+{
+ "Original_Index_0": {
+ "video_name": "Original_Index_0.png",
+ "error": "division by zero"
+ }
+}
\ No newline at end of file
+ {icon} {label}
+ {message}
+
+
+