"""
RSS Feed Fetcher Service - Fetches news from RSS feeds
"""
import feedparser
from typing import List, Dict, Any
from datetime import datetime, timezone
import logging
from ..config import settings
from ..utils.logger import setup_logger
logger = setup_logger(__name__)
class RSSFeedFetcher:
"""
Service responsible for fetching news articles from RSS feeds.
"""
def __init__(self):
"""Initialize RSS feed fetcher with configured sources."""
self.rss_sources = settings.rss_sources_list
self.priority_keywords = settings.priority_keywords_list
def fetch_all_feeds(self, max_articles_per_feed: int = 20) -> List[Dict[str, Any]]:
"""
Fetch articles from all configured RSS feeds.
Args:
max_articles_per_feed: Maximum articles to fetch per feed
Returns:
List of standardized article dictionaries
"""
all_articles = []
logger.info(f"Fetching from {len(self.rss_sources)} RSS feeds")
for source in self.rss_sources:
try:
articles = self._fetch_feed(
source['url'],
source['name'],
max_articles_per_feed
)
all_articles.extend(articles)
logger.info(f"Fetched {len(articles)} articles from {source['name']}")
except Exception as e:
logger.error(f"Error fetching RSS feed {source['name']}: {e}")
continue
logger.info(f"Total RSS articles fetched: {len(all_articles)}")
return all_articles
def _fetch_feed(
self,
feed_url: str,
source_name: str,
max_articles: int
) -> List[Dict[str, Any]]:
"""
Fetch and parse a single RSS feed.
Args:
feed_url: URL of the RSS feed
source_name: Name of the news source
max_articles: Maximum number of articles to fetch
Returns:
List of standardized article dictionaries
"""
try:
feed = feedparser.parse(feed_url)
if feed.bozo:
logger.warning(f"RSS feed {source_name} has parsing issues")
articles = []
for entry in feed.entries[:max_articles]:
article = self._standardize_rss_entry(entry, source_name)
if article:
articles.append(article)
return articles
except Exception as e:
logger.error(f"Error parsing RSS feed {source_name}: {e}")
return []
def _standardize_rss_entry(
self,
entry: Any,
source_name: str
) -> Dict[str, Any]:
"""
Convert RSS entry to internal article format (NewsAPI-compatible).
Args:
entry: RSS feed entry from feedparser
source_name: Name of the news source
Returns:
Standardized article dictionary or None if invalid
"""
# Extract title
title = entry.get('title', '').strip()
if not title:
return None
# Extract description/summary
summary = entry.get('summary', entry.get('description', '')).strip()
if not summary:
return None
# Extract link
link = entry.get('link', '')
# Extract published date
published_date = None
if hasattr(entry, 'published_parsed') and entry.published_parsed:
try:
published_date = datetime(*entry.published_parsed[:6], tzinfo=timezone.utc)
except:
pass
if not published_date:
published_date = datetime.now(timezone.utc)
# Detect priority
priority = self._detect_priority(title, summary)
return {
'title': title,
'link': link,
'summary': summary,
'published_date': published_date,
'source_name': source_name,
'priority': priority,
'country': None # Will be determined by NLP
}
def _detect_priority(self, title: str, summary: str) -> str:
"""
Detect article priority based on keywords.
Args:
title: Article title
summary: Article summary
Returns:
Priority level: 'critical', 'high', 'medium', or 'low'
"""
text = f"{title} {summary}".lower()
for keyword in self.priority_keywords:
if keyword in text:
return 'critical'
return 'medium'
# Example usage
if __name__ == "__main__":
fetcher = RSSFeedFetcher()
articles = fetcher.fetch_all_feeds(max_articles_per_feed=5)
for article in articles:
print(f"Title: {article['title']}")
print(f"Source: {article['source_name']}")
print("---")