locate-the-news / backend / app / services / rss_fetcher.py
rss_fetcher.py
Raw
import feedparser
from sqlalchemy.orm import Session
from datetime import datetime, timezone
from typing import List, Dict, Any, Optional

from .. import models, schemas
from ..config import settings
from ..utils.logger import setup_logger

# Initialize logger for this service
logger = setup_logger(__name__)

class RssFetcher:
    """
    Service responsible for fetching, standardizing, and saving news articles
    from configured RSS feeds.
    """
    def __init__(self, db: Session):
        self.db = db
        # Dictionary of RSS sources (can be expanded later)
        self.sources: Dict[str, str] = {
            "BBC_NEWS_WORLD": settings.BBC_NEWS_WORLD_RSS,
            # Add more sources here later, e.g., "REUTERS": "..."
        }

    def fetch_all(self) -> List[models.Article]:
        """
        Fetches all configured RSS feeds and saves new articles to the database.

        Returns:
            A list of the new Article objects that were successfully saved.
        """
        all_new_articles = []
        logger.info(f"Starting to fetch news from {len(self.sources)} configured RSS sources...")

        for source_name, url in self.sources.items():
            try:
                new_articles = self._fetch_and_save_feed(source_name, url)
                all_new_articles.extend(new_articles)
            except Exception as e:
                logger.error(f"Error fetching RSS feed from {source_name} ({url}): {e}")
        
        if all_new_articles:
            logger.info(f"Successfully processed and saved {len(all_new_articles)} NEW articles to the database.")
        else:
            logger.info("No new articles found or saved in this run.")
            
        return all_new_articles

    def _fetch_and_save_feed(self, source_name: str, url: str) -> List[models.Article]:
        """
        Fetches a single RSS feed, standardizes entries, and saves new ones.
        """
        d = feedparser.parse(url)
        
        if d.get('bozo') == 1 and d.get('bozo_exception'):
            logger.warning(f"Failed to parse feed {source_name}: {d['bozo_exception']}")
            return []

        new_articles = []
        
        # We process entries in reverse order (oldest first) to avoid race conditions
        # if the fetcher is run very frequently.
        for entry in d.entries:
            try:
                # 1. Standardize the data
                standardized_data = self._standardize_entry(entry, source_name)
                
                # 2. Check for duplicate and save
                if not self._article_exists(standardized_data.link):
                    article_model = self._save_article(standardized_data)
                    new_articles.append(article_model)
                    
            except Exception as e:
                # Log the error but continue with the next entry
                logger.error(f"Failed to process and save entry from {source_name}: {e}")
                
        logger.info(f"Source '{source_name}': {len(d.entries)} entries found, {len(new_articles)} new articles saved.")
        return new_articles

    def _standardize_entry(self, entry: Dict[str, Any], source_name: str) -> schemas.ArticleBase:
        """
        Maps a raw feedparser entry to the structured ArticleBase Pydantic schema.
        """
        # --- Date Parsing ---
        # Get the time structure from the entry (usually 'published_parsed')
        time_struct = entry.get('published_parsed') or entry.get('updated_parsed')
        if time_struct:
            # Convert the time structure to a datetime object
            published_date = datetime(*time_struct[:6], tzinfo=timezone.utc)
        else:
            # Fallback to current time if no publish date is available
            published_date = datetime.now(timezone.utc) 

        # --- Summary/Content Extraction ---
        summary = entry.get('summary') or entry.get('content', [{}])[0].get('value', 'No summary available.')
        
        # --- Pydantic Validation ---
        # The Pydantic model ensures data integrity before saving to DB
        return schemas.ArticleBase(
            title=entry.get('title', 'Untitled Article'),
            link=entry.link,
            summary=summary,
            published_date=published_date,
            source_name=source_name
        )

    def _article_exists(self, link: str) -> bool:
        """
        Checks if an article with the given link already exists in the database.
        """
        return self.db.query(models.Article).filter(models.Article.link == link).first() is not None

    def _save_article(self, article_data: schemas.ArticleBase) -> models.Article:
        """
        Creates and saves a new Article model instance to the database.
        """
        db_article = models.Article(**article_data.model_dump())
        self.db.add(db_article)
        self.db.commit()
        self.db.refresh(db_article)
        return db_article
    
# Example of how to use it (for reference, this is not run):
# from .database import SessionLocal
# if __name__ == "__main__":
#     db = SessionLocal()
#     fetcher = RssFetcher(db)
#     fetcher.fetch_all()
#     db.close()