locate-the-news / backend / app / services / news_api_fetcher.py
news_api_fetcher.py
Raw
"""
NewsAPI Fetcher Service - Fetches real-time news from NewsAPI.org
"""
from newsapi import NewsApiClient
from typing import List, Dict, Any, Optional
from datetime import datetime, timezone
import logging

from ..config import settings
from ..utils.logger import setup_logger

logger = setup_logger(__name__)


class NewsAPIFetcher:
    """
    Service responsible for fetching news articles from NewsAPI.
    """
    
    def __init__(self, api_key: Optional[str] = None):
        """
        Initialize NewsAPI client.
        
        Args:
            api_key: NewsAPI key (defaults to settings.NEWSAPI_KEY)
        """
        self.api_key = api_key or settings.NEWSAPI_KEY
        self.client = NewsApiClient(api_key=self.api_key)
        self.priority_keywords = settings.priority_keywords_list
    
    def fetch_top_headlines(
        self,
        countries: Optional[List[str]] = None,
        category: Optional[str] = None,
        page_size: int = 100
    ) -> List[Dict[str, Any]]:
        """
        Fetch top headlines from NewsAPI.
        
        Args:
            countries: List of country codes (e.g., ['us', 'gb'])
            category: News category (e.g., 'general', 'technology')
            page_size: Number of articles to fetch (max 100)
            
        Returns:
            List of standardized article dictionaries
        """
        countries = countries or settings.newsapi_countries_list
        all_articles = []
        
        logger.info(f"Fetching top headlines for countries: {countries}, category: {category}")
        
        for country in countries:
            try:
                response = self.client.get_top_headlines(
                    country=country,
                    category=category,
                    page_size=page_size
                )
                
                if response['status'] == 'ok':
                    articles = response.get('articles', [])
                    logger.info(f"Fetched {len(articles)} articles from {country.upper()}")
                    
                    for article in articles:
                        standardized = self._standardize_article(article, country)
                        if standardized:
                            all_articles.append(standardized)
                else:
                    logger.warning(f"NewsAPI returned non-ok status for {country}: {response}")
                    
            except Exception as e:
                logger.error(f"Error fetching headlines for {country}: {e}")
                continue
        
        logger.info(f"Total articles fetched and standardized: {len(all_articles)}")
        return all_articles
    
    def fetch_everything(
        self,
        query: str,
        language: str = 'en',
        sort_by: str = 'publishedAt',
        page_size: int = 100
    ) -> List[Dict[str, Any]]:
        """
        Search all articles matching a query.
        
        Args:
            query: Search query
            language: Language code (e.g., 'en')
            sort_by: Sort order ('relevancy', 'popularity', 'publishedAt')
            page_size: Number of articles to fetch (max 100)
            
        Returns:
            List of standardized article dictionaries
        """
        try:
            logger.info(f"Searching articles with query: '{query}'")
            
            response = self.client.get_everything(
                q=query,
                language=language,
                sort_by=sort_by,
                page_size=page_size
            )
            
            if response['status'] == 'ok':
                articles = response.get('articles', [])
                logger.info(f"Found {len(articles)} articles for query '{query}'")
                
                standardized_articles = []
                for article in articles:
                    standardized = self._standardize_article(article)
                    if standardized:
                        standardized_articles.append(standardized)
                
                return standardized_articles
            else:
                logger.warning(f"NewsAPI returned non-ok status: {response}")
                return []
                
        except Exception as e:
            logger.error(f"Error searching articles: {e}")
            return []
    
    def _standardize_article(
        self,
        article: Dict[str, Any],
        country: Optional[str] = None
    ) -> Optional[Dict[str, Any]]:
        """
        Convert NewsAPI article format to internal schema.
        
        Args:
            article: Raw article from NewsAPI
            country: Country code if from top headlines
            
        Returns:
            Standardized article dictionary or None if invalid
        """
        # Skip articles without description (usually removed content)
        if not article.get('description') or article.get('description') == '[Removed]':
            return None
        
        # Skip articles without title
        if not article.get('title'):
            return None
        
        # Parse published date
        published_str = article.get('publishedAt')
        if published_str:
            try:
                published_date = datetime.fromisoformat(published_str.replace('Z', '+00:00'))
            except:
                published_date = datetime.now(timezone.utc)
        else:
            published_date = datetime.now(timezone.utc)
        
        # Determine source name
        source = article.get('source', {})
        source_name = source.get('name', 'Unknown Source')
        
        # Detect priority based on keywords
        priority = self._detect_priority(article.get('title', ''), article.get('description', ''))
        
        return {
            'title': article['title'],
            'link': article.get('url', ''),
            'summary': article.get('description', ''),
            'published_date': published_date,
            'source_name': source_name,
            'priority': priority,
            'country': country
        }
    
    def _detect_priority(self, title: str, description: str) -> str:
        """
        Detect article priority based on keywords.
        
        Args:
            title: Article title
            description: Article description
            
        Returns:
            Priority level: 'critical', 'high', 'medium', or 'low'
        """
        text = f"{title} {description}".lower()
        
        # Check for critical keywords
        for keyword in self.priority_keywords:
            if keyword in text:
                return 'critical'
        
        # Default to medium priority
        return 'medium'


# Example usage (for testing)
if __name__ == "__main__":
    fetcher = NewsAPIFetcher()
    articles = fetcher.fetch_top_headlines(page_size=5)
    for article in articles:
        print(f"Title: {article['title']}")
        print(f"Priority: {article['priority']}")
        print(f"Source: {article['source_name']}")
        print("---")