Spaces:
Configuration error
Configuration error
File size: 2,314 Bytes
7136657 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 | import os
import logging
from src.scraper.scraper import run_auto_pipeline
from src.preprocessing.preprocess import run_preprocessing_pipeline
# Setup logging
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)
def run_full_pipeline(queries: list, limit_per_query: int = 5, lang: str = "id", country: str = "ID"):
"""
Runs the full pipeline:
1. Search & Scrape raw articles based on queries.
2. Preprocess raw articles (cleaning, tokenization, stemming, stopword removal).
"""
logger.info("================ STARTING FULL PIPELINE ================")
# 1. Run Scraper
raw_path = "data/raw_articles.json"
logger.info("Step 1: Running scraper...")
run_auto_pipeline(queries, limit_per_query=limit_per_query, output_dir="data", lang=lang, country=country)
# 2. Run Preprocessor
preprocessed_path = "data/preprocessed_articles.json"
logger.info("Step 2: Running text preprocessing...")
run_preprocessing_pipeline(input_path=raw_path, output_path=preprocessed_path)
# 3. Run Topic Modeling & Trend Clustering
clustered_path = "data/clustered_articles.json"
logger.info("Step 3: Running topic modeling and trend clustering...")
from src.modeling.topic_model import run_clustering_pipeline
run_clustering_pipeline(input_path=preprocessed_path, output_path=clustered_path)
# 4 & 5. Run AI Summarization & Final Analysis
report_path = "data/final_summary_report.json"
logger.info("Step 4 & 5: Running AI summarization and generating final report...")
from src.summarizer.summary_engine import run_summarization_pipeline
run_summarization_pipeline(input_path=clustered_path, output_path=report_path)
logger.info("================ PIPELINE COMPLETED SUCCESSFULLY ================")
if __name__ == "__main__":
# Standard queries for the project
default_queries = [
"FMCG keberlanjutan Indonesia",
"pemasaran digital ritel Indonesia",
"perubahan perilaku konsumen ritel"
]
# Run the pipeline with limit 2 per query for testing
print("Testing pipeline manager with 2 links per query...")
run_full_pipeline(default_queries, limit_per_query=2, lang="id", country="ID")
|