Improved logging

app/database.py

@@ -4,6 +4,7 @@ from sqlalchemy.sql import func
 from contextlib import contextmanager
 import os
 import datetime
+import logging
 
 
 class Base(DeclarativeBase):
@@ -41,15 +42,15 @@ TURSO_AUTH_TOKEN = os.getenv('TURSO_AUTH_TOKEN')
 
 # create an engine
 if not TURSO_DATABASE_URL or not TURSO_AUTH_TOKEN:
-    print('Using local SQLite database')
-    engine = create_engine('sqlite:///news.db', echo=True)
+    logging.info('Using local SQLite database')
+    engine = create_engine('sqlite:///news.db', echo=False)
 else:
     engine = create_engine(
         f'sqlite+{TURSO_DATABASE_URL}?secure=true',
         connect_args={
             'auth_token': TURSO_AUTH_TOKEN
         },
-        echo=True,
+        echo=False,
     )
 
 # create tables if needed

app/news_fetcher.py

@@ -2,12 +2,13 @@ import feedparser
 from .summarizer import Summarizer
 from .scraper import get_articles
 from .database import Article, Source
+import logging
 
 
 def topic_summary(summarizer: Summarizer, query: str, max_results: int = 5, min_cluster_size: int = 2) -> str:
-    print(f'Beginning search for \'{query}\'...')
+    logging.debug(f'Beginning search for \'{query}\'...')
     articles_and_urls = get_articles(query, max_results=max_results)
-    print(f'Retrieved {len(articles_and_urls)} articles')
+    logging.debug(f'Retrieved {len(articles_and_urls)} articles')
     articles = [item[0] for item in articles_and_urls]
     urls = [item[1] for item in articles_and_urls]
     summary = summarizer.combined_summary(articles, min_cluster_size=min_cluster_size)

app/scraper.py

@@ -1,5 +1,6 @@
 from ddgs import DDGS
 import trafilatura
+import logging
 
 
 def retrieve_article(url: str) -> str:
@@ -7,7 +8,8 @@ def retrieve_article(url: str) -> str:
         page = trafilatura.fetch_url(url)
         return trafilatura.extract(page)
     except Exception as e:
-        print(e.args)
+        logging.debug(e.args)
+        logging.error('Error retrieving article')
         return None
   
 
@@ -17,7 +19,8 @@ def get_articles(query, max_results=5, min_article_length=100) -> list[tuple[str
     try:
         search_results = DDGS().news(query, timelimit='d', max_results=max_results)
     except Exception as e:
-        print(e.args)
+        logging.debug(e.args)
+        logging.error('Error searching for articles')
 
     # get article urls
     urls = set([r['url'] for r in search_results])

app/summarizer.py

@@ -3,6 +3,7 @@ from nltk import tokenize
 from sklearn.cluster import HDBSCAN
 from sklearn.metrics.pairwise import cosine_similarity
 import numpy as np
+import logging
 
 
 class Summarizer:
@@ -71,7 +72,7 @@ class Summarizer:
         # combine key sentences from each cluster
         key_sentences = []
         for i, cluster in enumerate(clusters):
-            print(f'Extracting from cluster {i + 1}...')
+            logging.debug(f'Extracting from cluster {i + 1}...')
             top_sentences = Summarizer.rank_cluster_sentences(cluster)
             content = '\n'.join(top_sentences[:3])
             key_sentences.append(content)
@@ -79,7 +80,7 @@ class Summarizer:
         combined = ' '.join(key_sentences)
 
         # summarize all key sentences
-        print('Creating response...')
+        logging.debug('Creating response...')
         summary = self.summarize(
             combined,
             min_length=60,
@@ -92,7 +93,7 @@ class Summarizer:
         if not articles:
             return None
 
-        print('Tokenizing into sentences...')
+        logging.debug('Tokenizing into sentences...')
         # create a list of all sentences from all articles
         sentences = []
         for article in articles:
@@ -100,19 +101,19 @@ class Summarizer:
 
         # remove duplicate sentences
         sentences = sorted(list(set(sentences)), key=sentences.index)
-        print(f'Found {len(sentences)} unique sentences')
+        logging.debug(f'Found {len(sentences)} unique sentences')
 
         if not sentences:
             return None
 
-        print('Creating sentence embeddings...')
+        logging.debug('Creating sentence embeddings...')
         # create embeddings
         embeddings = self.create_embeddings(sentences)
 
-        print('Grouping sentences into clusters...')
+        logging.debug('Grouping sentences into clusters...')
         # group (embeddings of) sentences by similarity
         clusters = self.cluster_sentences(sentences, embeddings, min_cluster_size=min_cluster_size)
-        print(f'Created {len(clusters)} clusters')
+        logging.debug(f'Created {len(clusters)} clusters')
 
         # summarize all clusters into a single summary
         summary = self.summarize_clusters(clusters)

app/update_news.py

@@ -4,6 +4,7 @@ from .summarizer import Summarizer
 from .news_fetcher import news_summary
 from .database import get_session, clear_articles, add_article, add_sources
 import datetime
+import logging
 
 
 def read_rss_feed_urls(filename : str ='rss_feeds.txt') -> list[str]:
@@ -17,7 +18,7 @@ def read_rss_feed_urls(filename : str ='rss_feeds.txt') -> list[str]:
 
 
 def update_news():
-    print(f'Initiating news update: {datetime.datetime.now()}')
+    logging.info(f'Initiating news update: {datetime.datetime.now()}')
 
     # model to create embeddings from sentences
     embedding_model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')

test.py

@@ -1,4 +0,0 @@
-from database import DatabaseConnection
-db = DatabaseConnection()
-print(db.retrieve_articles())
-db.close()