""" NewsAPI Fetcher Service - Fetches real-time news from NewsAPI.org """ from newsapi import NewsApiClient from typing import List, Dict, Any, Optional from datetime import datetime, timezone import logging from ..config import settings from ..utils.logger import setup_logger logger = setup_logger(__name__) class NewsAPIFetcher: """ Service responsible for fetching news articles from NewsAPI. """ def __init__(self, api_key: Optional[str] = None): """ Initialize NewsAPI client. Args: api_key: NewsAPI key (defaults to settings.NEWSAPI_KEY) """ self.api_key = api_key or settings.NEWSAPI_KEY self.client = NewsApiClient(api_key=self.api_key) self.priority_keywords = settings.priority_keywords_list def fetch_top_headlines( self, countries: Optional[List[str]] = None, category: Optional[str] = None, page_size: int = 100 ) -> List[Dict[str, Any]]: """ Fetch top headlines from NewsAPI. Args: countries: List of country codes (e.g., ['us', 'gb']) category: News category (e.g., 'general', 'technology') page_size: Number of articles to fetch (max 100) Returns: List of standardized article dictionaries """ countries = countries or settings.newsapi_countries_list all_articles = [] logger.info(f"Fetching top headlines for countries: {countries}, category: {category}") for country in countries: try: response = self.client.get_top_headlines( country=country, category=category, page_size=page_size ) if response['status'] == 'ok': articles = response.get('articles', []) logger.info(f"Fetched {len(articles)} articles from {country.upper()}") for article in articles: standardized = self._standardize_article(article, country) if standardized: all_articles.append(standardized) else: logger.warning(f"NewsAPI returned non-ok status for {country}: {response}") except Exception as e: logger.error(f"Error fetching headlines for {country}: {e}") continue logger.info(f"Total articles fetched and standardized: {len(all_articles)}") return all_articles def fetch_everything( self, query: str, language: str = 'en', sort_by: str = 'publishedAt', page_size: int = 100 ) -> List[Dict[str, Any]]: """ Search all articles matching a query. Args: query: Search query language: Language code (e.g., 'en') sort_by: Sort order ('relevancy', 'popularity', 'publishedAt') page_size: Number of articles to fetch (max 100) Returns: List of standardized article dictionaries """ try: logger.info(f"Searching articles with query: '{query}'") response = self.client.get_everything( q=query, language=language, sort_by=sort_by, page_size=page_size ) if response['status'] == 'ok': articles = response.get('articles', []) logger.info(f"Found {len(articles)} articles for query '{query}'") standardized_articles = [] for article in articles: standardized = self._standardize_article(article) if standardized: standardized_articles.append(standardized) return standardized_articles else: logger.warning(f"NewsAPI returned non-ok status: {response}") return [] except Exception as e: logger.error(f"Error searching articles: {e}") return [] def _standardize_article( self, article: Dict[str, Any], country: Optional[str] = None ) -> Optional[Dict[str, Any]]: """ Convert NewsAPI article format to internal schema. Args: article: Raw article from NewsAPI country: Country code if from top headlines Returns: Standardized article dictionary or None if invalid """ # Skip articles without description (usually removed content) if not article.get('description') or article.get('description') == '[Removed]': return None # Skip articles without title if not article.get('title'): return None # Parse published date published_str = article.get('publishedAt') if published_str: try: published_date = datetime.fromisoformat(published_str.replace('Z', '+00:00')) except: published_date = datetime.now(timezone.utc) else: published_date = datetime.now(timezone.utc) # Determine source name source = article.get('source', {}) source_name = source.get('name', 'Unknown Source') # Detect priority based on keywords priority = self._detect_priority(article.get('title', ''), article.get('description', '')) return { 'title': article['title'], 'link': article.get('url', ''), 'summary': article.get('description', ''), 'published_date': published_date, 'source_name': source_name, 'priority': priority, 'country': country } def _detect_priority(self, title: str, description: str) -> str: """ Detect article priority based on keywords. Args: title: Article title description: Article description Returns: Priority level: 'critical', 'high', 'medium', or 'low' """ text = f"{title} {description}".lower() # Check for critical keywords for keyword in self.priority_keywords: if keyword in text: return 'critical' # Default to medium priority return 'medium' # Example usage (for testing) if __name__ == "__main__": fetcher = NewsAPIFetcher() articles = fetcher.fetch_top_headlines(page_size=5) for article in articles: print(f"Title: {article['title']}") print(f"Priority: {article['priority']}") print(f"Source: {article['source_name']}") print("---")