locate-the-news / backend / app / services / rss_feed_fetcher.py
rss_feed_fetcher.py
Raw
"""
RSS Feed Fetcher Service - Fetches news from RSS feeds
"""
import feedparser
from typing import List, Dict, Any
from datetime import datetime, timezone
import logging

from ..config import settings
from ..utils.logger import setup_logger

logger = setup_logger(__name__)


class RSSFeedFetcher:
    """
    Service responsible for fetching news articles from RSS feeds.
    """
    
    def __init__(self):
        """Initialize RSS feed fetcher with configured sources."""
        self.rss_sources = settings.rss_sources_list
        self.priority_keywords = settings.priority_keywords_list
    
    def fetch_all_feeds(self, max_articles_per_feed: int = 20) -> List[Dict[str, Any]]:
        """
        Fetch articles from all configured RSS feeds.
        
        Args:
            max_articles_per_feed: Maximum articles to fetch per feed
            
        Returns:
            List of standardized article dictionaries
        """
        all_articles = []
        
        logger.info(f"Fetching from {len(self.rss_sources)} RSS feeds")
        
        for source in self.rss_sources:
            try:
                articles = self._fetch_feed(
                    source['url'],
                    source['name'],
                    max_articles_per_feed
                )
                all_articles.extend(articles)
                logger.info(f"Fetched {len(articles)} articles from {source['name']}")
            except Exception as e:
                logger.error(f"Error fetching RSS feed {source['name']}: {e}")
                continue
        
        logger.info(f"Total RSS articles fetched: {len(all_articles)}")
        return all_articles
    
    def _fetch_feed(
        self,
        feed_url: str,
        source_name: str,
        max_articles: int
    ) -> List[Dict[str, Any]]:
        """
        Fetch and parse a single RSS feed.
        
        Args:
            feed_url: URL of the RSS feed
            source_name: Name of the news source
            max_articles: Maximum number of articles to fetch
            
        Returns:
            List of standardized article dictionaries
        """
        try:
            feed = feedparser.parse(feed_url)
            
            if feed.bozo:
                logger.warning(f"RSS feed {source_name} has parsing issues")
            
            articles = []
            for entry in feed.entries[:max_articles]:
                article = self._standardize_rss_entry(entry, source_name)
                if article:
                    articles.append(article)
            
            return articles
            
        except Exception as e:
            logger.error(f"Error parsing RSS feed {source_name}: {e}")
            return []
    
    def _standardize_rss_entry(
        self,
        entry: Any,
        source_name: str
    ) -> Dict[str, Any]:
        """
        Convert RSS entry to internal article format (NewsAPI-compatible).
        
        Args:
            entry: RSS feed entry from feedparser
            source_name: Name of the news source
            
        Returns:
            Standardized article dictionary or None if invalid
        """
        # Extract title
        title = entry.get('title', '').strip()
        if not title:
            return None
        
        # Extract description/summary
        summary = entry.get('summary', entry.get('description', '')).strip()
        if not summary:
            return None
        
        # Extract link
        link = entry.get('link', '')
        
        # Extract published date
        published_date = None
        if hasattr(entry, 'published_parsed') and entry.published_parsed:
            try:
                published_date = datetime(*entry.published_parsed[:6], tzinfo=timezone.utc)
            except:
                pass
        
        if not published_date:
            published_date = datetime.now(timezone.utc)
        
        # Detect priority
        priority = self._detect_priority(title, summary)
        
        return {
            'title': title,
            'link': link,
            'summary': summary,
            'published_date': published_date,
            'source_name': source_name,
            'priority': priority,
            'country': None  # Will be determined by NLP
        }
    
    def _detect_priority(self, title: str, summary: str) -> str:
        """
        Detect article priority based on keywords.
        
        Args:
            title: Article title
            summary: Article summary
            
        Returns:
            Priority level: 'critical', 'high', 'medium', or 'low'
        """
        text = f"{title} {summary}".lower()
        
        for keyword in self.priority_keywords:
            if keyword in text:
                return 'critical'
        
        return 'medium'


# Example usage
if __name__ == "__main__":
    fetcher = RSSFeedFetcher()
    articles = fetcher.fetch_all_feeds(max_articles_per_feed=5)
    for article in articles:
        print(f"Title: {article['title']}")
        print(f"Source: {article['source_name']}")
        print("---")