import feedparser from sqlalchemy.orm import Session from datetime import datetime, timezone from typing import List, Dict, Any, Optional from .. import models, schemas from ..config import settings from ..utils.logger import setup_logger # Initialize logger for this service logger = setup_logger(__name__) class RssFetcher: """ Service responsible for fetching, standardizing, and saving news articles from configured RSS feeds. """ def __init__(self, db: Session): self.db = db # Dictionary of RSS sources (can be expanded later) self.sources: Dict[str, str] = { "BBC_NEWS_WORLD": settings.BBC_NEWS_WORLD_RSS, # Add more sources here later, e.g., "REUTERS": "..." } def fetch_all(self) -> List[models.Article]: """ Fetches all configured RSS feeds and saves new articles to the database. Returns: A list of the new Article objects that were successfully saved. """ all_new_articles = [] logger.info(f"Starting to fetch news from {len(self.sources)} configured RSS sources...") for source_name, url in self.sources.items(): try: new_articles = self._fetch_and_save_feed(source_name, url) all_new_articles.extend(new_articles) except Exception as e: logger.error(f"Error fetching RSS feed from {source_name} ({url}): {e}") if all_new_articles: logger.info(f"Successfully processed and saved {len(all_new_articles)} NEW articles to the database.") else: logger.info("No new articles found or saved in this run.") return all_new_articles def _fetch_and_save_feed(self, source_name: str, url: str) -> List[models.Article]: """ Fetches a single RSS feed, standardizes entries, and saves new ones. """ d = feedparser.parse(url) if d.get('bozo') == 1 and d.get('bozo_exception'): logger.warning(f"Failed to parse feed {source_name}: {d['bozo_exception']}") return [] new_articles = [] # We process entries in reverse order (oldest first) to avoid race conditions # if the fetcher is run very frequently. for entry in d.entries: try: # 1. Standardize the data standardized_data = self._standardize_entry(entry, source_name) # 2. Check for duplicate and save if not self._article_exists(standardized_data.link): article_model = self._save_article(standardized_data) new_articles.append(article_model) except Exception as e: # Log the error but continue with the next entry logger.error(f"Failed to process and save entry from {source_name}: {e}") logger.info(f"Source '{source_name}': {len(d.entries)} entries found, {len(new_articles)} new articles saved.") return new_articles def _standardize_entry(self, entry: Dict[str, Any], source_name: str) -> schemas.ArticleBase: """ Maps a raw feedparser entry to the structured ArticleBase Pydantic schema. """ # --- Date Parsing --- # Get the time structure from the entry (usually 'published_parsed') time_struct = entry.get('published_parsed') or entry.get('updated_parsed') if time_struct: # Convert the time structure to a datetime object published_date = datetime(*time_struct[:6], tzinfo=timezone.utc) else: # Fallback to current time if no publish date is available published_date = datetime.now(timezone.utc) # --- Summary/Content Extraction --- summary = entry.get('summary') or entry.get('content', [{}])[0].get('value', 'No summary available.') # --- Pydantic Validation --- # The Pydantic model ensures data integrity before saving to DB return schemas.ArticleBase( title=entry.get('title', 'Untitled Article'), link=entry.link, summary=summary, published_date=published_date, source_name=source_name ) def _article_exists(self, link: str) -> bool: """ Checks if an article with the given link already exists in the database. """ return self.db.query(models.Article).filter(models.Article.link == link).first() is not None def _save_article(self, article_data: schemas.ArticleBase) -> models.Article: """ Creates and saves a new Article model instance to the database. """ db_article = models.Article(**article_data.model_dump()) self.db.add(db_article) self.db.commit() self.db.refresh(db_article) return db_article # Example of how to use it (for reference, this is not run): # from .database import SessionLocal # if __name__ == "__main__": # db = SessionLocal() # fetcher = RssFetcher(db) # fetcher.fetch_all() # db.close()