Initial clean commit for Raagsan Space

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,13 @@
+---
+title: Testing
+emoji: ⚡
+colorFrom: pink
+colorTo: indigo
+sdk: gradio
+sdk_version: 5.49.1
+app_file: app.py
+pinned: false
+short_description: Testing
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

auth.py

@@ -0,0 +1,231 @@
+#!/usr/bin/env python3
+"""
+Authentication module for News Dashboard
+Handles user authentication, session management, and security
+"""
+
+import hashlib
+import secrets
+import json
+import os
+from datetime import datetime, timedelta
+from typing import Dict, Optional, Tuple
+import logging
+
+logger = logging.getLogger(__name__)
+
+class AuthManager:
+    """Manages user authentication and sessions"""
+    
+    def __init__(self, users_file: str = "users.json", sessions_file: str = "sessions.json"):
+        self.users_file = users_file
+        self.sessions_file = sessions_file
+        self.users = self._load_users()
+        self.sessions = self._load_sessions()
+        self.session_timeout = timedelta(hours=24)  # 24 hours session timeout
+        
+        # Create default admin user if no users exist
+        if not self.users:
+            self._create_default_admin()
+    
+    def _load_users(self) -> Dict[str, Dict]:
+        """Load users from JSON file"""
+        try:
+            if os.path.exists(self.users_file):
+                with open(self.users_file, 'r') as f:
+                    return json.load(f)
+        except Exception as e:
+            logger.error(f"Error loading users: {e}")
+        return {}
+    
+    def _save_users(self):
+        """Save users to JSON file"""
+        try:
+            with open(self.users_file, 'w') as f:
+                json.dump(self.users, f, indent=2)
+        except Exception as e:
+            logger.error(f"Error saving users: {e}")
+    
+    def _load_sessions(self) -> Dict[str, Dict]:
+        """Load sessions from JSON file"""
+        try:
+            if os.path.exists(self.sessions_file):
+                with open(self.sessions_file, 'r') as f:
+                    return json.load(f)
+        except Exception as e:
+            logger.error(f"Error loading sessions: {e}")
+        return {}
+    
+    def _save_sessions(self):
+        """Save sessions to JSON file"""
+        try:
+            with open(self.sessions_file, 'w') as f:
+                json.dump(self.sessions, f, indent=2)
+        except Exception as e:
+            logger.error(f"Error saving sessions: {e}")
+    
+    def _create_default_admin(self):
+        """Create default admin user"""
+        admin_password = "admin123"  # Default password - should be changed
+        self.add_user("admin", admin_password, is_admin=True)
+        logger.warning("Created default admin user with password 'admin123' - PLEASE CHANGE THIS!")
+    
+    def _hash_password(self, password: str) -> str:
+        """Hash password using SHA-256 with salt"""
+        salt = secrets.token_hex(16)
+        password_hash = hashlib.sha256((password + salt).encode()).hexdigest()
+        return f"{salt}:{password_hash}"
+    
+    def _verify_password(self, password: str, stored_hash: str) -> bool:
+        """Verify password against stored hash"""
+        try:
+            salt, password_hash = stored_hash.split(':')
+            return hashlib.sha256((password + salt).encode()).hexdigest() == password_hash
+        except:
+            return False
+    
+    def add_user(self, username: str, password: str, is_admin: bool = False) -> bool:
+        """Add a new user"""
+        if username in self.users:
+            return False
+        
+        self.users[username] = {
+            'password_hash': self._hash_password(password),
+            'is_admin': is_admin,
+            'created_at': datetime.now().isoformat(),
+            'last_login': None
+        }
+        self._save_users()
+        logger.info(f"Added user: {username}")
+        return True
+    
+    def authenticate_user(self, username: str, password: str) -> Tuple[bool, str]:
+        """Authenticate user and return (success, session_token)"""
+        if username not in self.users:
+            return False, ""
+        
+        user = self.users[username]
+        if not self._verify_password(password, user['password_hash']):
+            return False, ""
+        
+        # Update last login
+        user['last_login'] = datetime.now().isoformat()
+        self._save_users()
+        
+        # Create session
+        session_token = secrets.token_urlsafe(32)
+        self.sessions[session_token] = {
+            'username': username,
+            'created_at': datetime.now().isoformat(),
+            'last_activity': datetime.now().isoformat()
+        }
+        self._save_sessions()
+        
+        logger.info(f"User {username} authenticated successfully")
+        return True, session_token
+    
+    def validate_session(self, session_token: str) -> Tuple[bool, Optional[str]]:
+        """Validate session token and return (valid, username)"""
+        if not session_token or session_token not in self.sessions:
+            return False, None
+        
+        session = self.sessions[session_token]
+        last_activity = datetime.fromisoformat(session['last_activity'])
+        
+        # Check if session has expired
+        if datetime.now() - last_activity > self.session_timeout:
+            self.logout_user(session_token)
+            return False, None
+        
+        # Update last activity
+        session['last_activity'] = datetime.now().isoformat()
+        self._save_sessions()
+        
+        return True, session['username']
+    
+    def logout_user(self, session_token: str) -> bool:
+        """Logout user by removing session"""
+        if session_token in self.sessions:
+            del self.sessions[session_token]
+            self._save_sessions()
+            return True
+        return False
+    
+    def is_admin(self, username: str) -> bool:
+        """Check if user is admin"""
+        return username in self.users and self.users[username].get('is_admin', False)
+    
+    def change_password(self, username: str, old_password: str, new_password: str) -> bool:
+        """Change user password"""
+        if username not in self.users:
+            return False
+        
+        user = self.users[username]
+        if not self._verify_password(old_password, user['password_hash']):
+            return False
+        
+        user['password_hash'] = self._hash_password(new_password)
+        self._save_users()
+        logger.info(f"Password changed for user: {username}")
+        return True
+    
+    def get_user_info(self, username: str) -> Optional[Dict]:
+        """Get user information (without password hash)"""
+        if username not in self.users:
+            return None
+        
+        user = self.users[username].copy()
+        del user['password_hash']  # Remove password hash from response
+        return user
+    
+    def list_users(self) -> Dict[str, Dict]:
+        """List all users (admin only)"""
+        result = {}
+        for username, user in self.users.items():
+            result[username] = {
+                'is_admin': user.get('is_admin', False),
+                'created_at': user.get('created_at'),
+                'last_login': user.get('last_login')
+            }
+        return result
+    
+    def delete_user(self, username: str) -> bool:
+        """Delete user (admin only)"""
+        if username not in self.users:
+            return False
+        
+        # Remove all sessions for this user
+        sessions_to_remove = []
+        for token, session in self.sessions.items():
+            if session['username'] == username:
+                sessions_to_remove.append(token)
+        
+        for token in sessions_to_remove:
+            del self.sessions[token]
+        
+        del self.users[username]
+        self._save_users()
+        self._save_sessions()
+        logger.info(f"Deleted user: {username}")
+        return True
+    
+    def cleanup_expired_sessions(self):
+        """Remove expired sessions"""
+        current_time = datetime.now()
+        expired_sessions = []
+        
+        for token, session in self.sessions.items():
+            last_activity = datetime.fromisoformat(session['last_activity'])
+            if current_time - last_activity > self.session_timeout:
+                expired_sessions.append(token)
+        
+        for token in expired_sessions:
+            del self.sessions[token]
+        
+        if expired_sessions:
+            self._save_sessions()
+            logger.info(f"Cleaned up {len(expired_sessions)} expired sessions")
+
+
+# Global auth manager instance
+auth_manager = AuthManager()

data_preprocessor.py

@@ -0,0 +1,776 @@
+#!/usr/bin/env python3
+"""
+Data Preprocessing Pipeline for News Dashboard
+Handles preprocessing of scraped content for translation, summarization, and other operations
+"""
+
+import re
+import logging
+from typing import List, Dict, Any, Optional
+from datetime import datetime
+import hashlib
+import unicodedata
+from scraper_common import scraping_cancelled
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class DataPreprocessor:
+    """
+    Data preprocessing pipeline for news dashboard content
+    """
+    
+    def __init__(self):
+        self.cleaned_data = []
+        self.processing_stats = {
+            'total_processed': 0,
+            'successful_processing': 0,
+            'failed_processing': 0,
+            'content_issues': 0,
+            'metadata_issues': 0
+        }
+    
+    def preprocess_all_data(self, raw_data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """
+        Main preprocessing function that processes all scraped data
+        
+        Args:
+            raw_data: List of dictionaries containing scraped content
+            
+        Returns:
+            List of preprocessed dictionaries ready for downstream operations
+        """
+        logger.info(f"Starting preprocessing of {len(raw_data)} items")
+        
+        processed_data = []
+        
+        for item in raw_data:
+            # Check for cancellation during preprocessing
+            if scraping_cancelled():
+                logger.warning("⚠️ Preprocessing cancelled by user")
+                return processed_data
+                
+            try:
+                processed_item = self._preprocess_single_item(item)
+                if processed_item:
+                    processed_data.append(processed_item)
+                    self.processing_stats['successful_processing'] += 1
+                else:
+                    self.processing_stats['failed_processing'] += 1
+                    
+            except Exception as e:
+                logger.error(f"Error processing item: {str(e)}")
+                self.processing_stats['failed_processing'] += 1
+                
+            self.processing_stats['total_processed'] += 1
+        
+        logger.info(f"Preprocessing completed. Stats: {self.processing_stats}")
+        return processed_data
+    
+    def _preprocess_single_item(self, item: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        """
+        Preprocess a single data item
+        
+        Args:
+            item: Single dictionary containing scraped content
+            
+        Returns:
+            Preprocessed dictionary or None if processing failed
+        """
+        try:
+            # Debug: Log the raw item structure
+            logger.info(f"🔍 Raw item structure for preprocessing:")
+            logger.info(f"  - Keys: {list(item.keys())}")
+            logger.info(f"  - extracted_text length: {len(item.get('extracted_text', ''))}")
+            logger.info(f"  - content length: {len(item.get('content', ''))}")
+            
+            # Create base processed item
+            processed_content = self._clean_and_structure_content(item)
+            processed_item = {
+                'id': self._generate_unique_id(item),
+                'source_metadata': self._extract_source_metadata(item),
+                'content': processed_content,
+                'metadata': self._enrich_metadata(processed_content),
+                'quality_metrics': self._calculate_quality_metrics(processed_content),
+                'processing_timestamp': datetime.now().isoformat(),
+                'ready_for_operations': True
+            }
+            
+            # Debug: Log the processed item structure
+            logger.debug(f"🔍 Processed item structure for {processed_item.get('id', 'unknown')}:")
+            logger.debug(f"  - Keys: {list(processed_item.keys())}")
+            logger.debug(f"  - Content keys: {list(processed_item.get('content', {}).keys())}")
+            logger.debug(f"  - Metadata keys: {list(processed_item.get('metadata', {}).keys())}")
+            
+            # Validate the processed item
+            if self._validate_processed_item(processed_item):
+                return processed_item
+            else:
+                logger.warning(f"Validation failed for item: {processed_item.get('id', 'unknown')}")
+                return None
+                
+        except Exception as e:
+            logger.error(f"Error preprocessing item: {str(e)}")
+            return None
+    
+    def _generate_unique_id(self, item: Dict[str, Any]) -> str:
+        """
+        Generate a unique identifier for the content item
+        
+        Args:
+            item: Raw data item
+            
+        Returns:
+            Unique identifier string
+        """
+        # Handle both text articles and document data
+        url = item.get('url', '') or item.get('file_path', '')
+        title = item.get('title', '')
+        
+        # Create a hash based on URL/file_path and title for uniqueness
+        content_string = f"{url}{title}"
+        return hashlib.md5(content_string.encode()).hexdigest()[:12]
+    
+    def _extract_source_metadata(self, item: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Extract and structure source metadata
+        
+        Args:
+            item: Raw data item
+            
+        Returns:
+            Dictionary containing source metadata
+        """
+        # Handle both text articles and document data
+        content_text = item.get('content', '') or item.get('extracted_text', '')
+        url = item.get('url', '') or item.get('file_path', '')
+        
+        # Preserve original source if it exists, otherwise identify from URL
+        original_source = item.get('source', '')
+        source_website = self._identify_source_website(url)
+        
+        # Use original source if available, otherwise use source_website
+        # If source_website is 'unknown' and we have a URL, try to get source from URL using utils
+        if not original_source and source_website == 'unknown' and url:
+            try:
+                from utils import get_source_from_url
+                original_source = get_source_from_url(url)
+            except:
+                pass
+        
+        result = {
+            'url': url,
+            'title': item.get('title', ''),
+            'date': item.get('date', ''),
+            'category': item.get('category', ''),
+            'source': original_source or self._map_source_website_to_name(source_website),
+            'source_website': source_website,
+            'content_type': self._identify_content_type(item),
+            'file_type': item.get('file_type', ''),  # Preserve original file_type for CSV detection
+            'language': self._detect_language(content_text),
+            'pdf_path': item.get('pdf_path', '') or item.get('file_path', ''),
+            'original_structure': {
+                'has_pdf': bool(item.get('pdf_path') or item.get('file_path')),
+                'content_length': len(content_text),
+                'title_length': len(item.get('title', ''))
+            }
+        }
+        
+        logger.debug(f"🔍 Extracted source metadata category: '{result.get('category', '')}'")
+        logger.debug(f"🔍 Preserved source: '{result.get('source', '')}'")
+        return result
+    
+    def _clean_and_structure_content(self, item: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Clean and structure the content for downstream processing
+        
+        Args:
+            item: Raw data item
+            
+        Returns:
+            Dictionary containing cleaned and structured content
+        """
+        # Handle both text articles and document data
+        raw_content = item.get('content', '') or item.get('extracted_text', '')
+        
+        # Debug: Log content extraction
+        logger.info(f"🔍 Content extraction debug:")
+        logger.info(f"  - item.get('content', ''): '{item.get('content', '')}'")
+        logger.info(f"  - item.get('extracted_text', ''): '{item.get('extracted_text', '')[:100]}...'")
+        logger.info(f"  - raw_content length: {len(raw_content)}")
+        
+        # Clean the content
+        cleaned_content = self._clean_text(raw_content)
+        logger.info(f"  - cleaned_content length: {len(cleaned_content)}")
+        
+        # Extract structured information
+        structured_content = {
+            'raw_text': raw_content,
+            'cleaned_text': cleaned_content,
+            'text_blocks': self._split_into_blocks(cleaned_content),
+            'sentences': self._split_into_sentences(cleaned_content),
+            'summary_ready': self._prepare_for_summarization(cleaned_content),
+            'translation_ready': self._prepare_for_translation(cleaned_content)
+        }
+        
+        return structured_content
+    
+    def _enrich_metadata(self, processed_content: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Enrich metadata with additional information
+        
+        Args:
+            processed_content: Processed content dictionary
+            
+        Returns:
+            Dictionary containing enriched metadata
+        """
+        # Get the cleaned text from the processed content
+        content = processed_content.get('cleaned_text', '')
+        
+        return {
+            'word_count': len(content.split()),
+            'character_count': len(content),
+            'sentence_count': len(self._split_into_sentences(content)),
+            'paragraph_count': len(self._split_into_blocks(content)),
+            'reading_time_minutes': self._calculate_reading_time(content),
+            'complexity_score': self._calculate_complexity_score(content)
+        }
+    
+    def _calculate_quality_metrics(self, processed_content: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Calculate quality metrics for the content
+        
+        Args:
+            processed_content: Processed content dictionary
+            
+        Returns:
+            Dictionary containing quality metrics
+        """
+        content = processed_content.get('cleaned_text', '')
+        title = processed_content.get('title', '')
+        
+        return {
+            'content_quality': {
+                'completeness_score': self._calculate_completeness_score(content),
+                'coherence_score': self._calculate_coherence_score(content),
+                'relevance_score': self._calculate_relevance_score(content, title),
+                'readability_score': self._calculate_readability_score(content)
+            },
+            'data_quality': {
+                'has_title': bool(title.strip()),
+                'has_content': bool(content.strip()),
+                'has_url': bool(processed_content.get('url', '').strip()),
+                'content_length_adequate': len(content) > 100,
+                'title_length_adequate': 10 < len(title) < 200
+            },
+            'processing_quality': {
+                'successfully_cleaned': bool(self._clean_text(content)),
+                'successfully_structured': bool(self._split_into_blocks(content))
+            }
+        }
+    
+    def _clean_text(self, text: str) -> str:
+        """
+        Clean and normalize text content
+        
+        Args:
+            text: Raw text content
+            
+        Returns:
+            Cleaned text content
+        """
+        if not text:
+            return ""
+        
+        # Remove extra whitespace and normalize
+        text = re.sub(r'\s+', ' ', text)
+        text = text.strip()
+        
+        # Remove special characters but keep punctuation
+        text = re.sub(r'[^\w\s\.\,\!\?\;\:\-\(\)]', '', text)
+        
+        # Normalize unicode
+        text = unicodedata.normalize('NFKD', text)
+        
+        # Remove excessive punctuation
+        text = re.sub(r'[\.]{2,}', '.', text)
+        text = re.sub(r'[!]{2,}', '!', text)
+        text = re.sub(r'[?]{2,}', '?', text)
+        
+        return text
+    
+    def _split_into_blocks(self, text: str) -> List[str]:
+        """
+        Split text into logical blocks (paragraphs)
+        
+        Args:
+            text: Text content
+            
+        Returns:
+            List of text blocks
+        """
+        if not text:
+            return []
+        
+        # Split by double newlines or periods followed by space
+        blocks = re.split(r'\n\s*\n|\.\s+(?=[A-Z])', text)
+        return [block.strip() for block in blocks if block.strip()]
+    
+    def _split_into_sentences(self, text: str) -> List[str]:
+        """
+        Split text into sentences
+        
+        Args:
+            text: Text content
+            
+        Returns:
+            List of sentences
+        """
+        if not text:
+            return []
+        
+        # Simple sentence splitting
+        sentences = re.split(r'[.!?]+', text)
+        return [sentence.strip() for sentence in sentences if sentence.strip()]
+    
+    def _prepare_for_summarization(self, text: str) -> Dict[str, Any]:
+        """
+        Prepare content for summarization
+        
+        Args:
+            text: Text content
+            
+        Returns:
+            Dictionary ready for summarization
+        """
+        blocks = self._split_into_blocks(text)
+        sentences = self._split_into_sentences(text)
+        
+        return {
+            'text': text,
+            'blocks': blocks,
+            'sentences': sentences,
+            'block_count': len(blocks),
+            'sentence_count': len(sentences),
+            'avg_sentence_length': sum(len(s.split()) for s in sentences) / len(sentences) if sentences else 0,
+            'summary_priority': self._calculate_summary_priority(text)
+        }
+    
+    def _prepare_for_translation(self, text: str) -> Dict[str, Any]:
+        """
+        Prepare content for translation
+        
+        Args:
+            text: Text content
+            
+        Returns:
+            Dictionary ready for translation
+        """
+        return {
+            'text': text,
+            'language_detected': self._detect_language(text),
+            'translation_blocks': self._split_into_blocks(text),
+            'character_count': len(text),
+            'word_count': len(text.split()),
+            'translation_priority': self._calculate_translation_priority(text)
+        }
+    
+    def _identify_source_website(self, url: str) -> str:
+        """
+        Identify the source website from URL
+        
+        Args:
+            url: URL string
+            
+        Returns:
+            Website identifier
+        """
+        if 'reliefweb.int' in url:
+            return 'reliefweb'
+        elif 'fscluster.org' in url:
+            return 'fscluster'
+        elif 'mopnd.govsomaliland.org' in url:
+            return 'mopnd'
+        elif 'nbs.gov.so' in url:
+            return 'nbs'
+        elif 'humdata.org' in url:
+            return 'hdx'
+        elif 'logcluster.org' in url:
+            return 'logcluster'
+        elif 'fsnau.org' in url:
+            return 'fsnau'
+        elif 'fews.net' in url:
+            return 'fews'
+        elif 'icpac.net' in url:
+            if 'seasonal-forecast' in url.lower():
+                return 'icpac_seasonal_forecast'
+            else:
+                return 'icpac'
+        elif 'faoswalim.org' in url:
+            return 'faoswalim'
+        else:
+            return 'unknown'
+    
+    def _map_source_website_to_name(self, source_website: str) -> str:
+        """
+        Map source website identifier to proper source name
+        
+        Args:
+            source_website: Website identifier (lowercase)
+            
+        Returns:
+            Proper source name
+        """
+        mapping = {
+            'reliefweb': 'ReliefWeb',
+            'fscluster': 'FS Cluster',
+            'mopnd': 'MOPND Somaliland',
+            'nbs': 'NBS Somalia',
+            'hdx': 'HDX Humanitarian Data Exchange',
+            'logcluster': 'LogCluster',
+            'fsnau': 'FSNau - Food Security and Nutrition Analysis Unit',
+            'fews': 'FEWS NET',
+            'icpac': 'ICPAC',
+            'icpac_seasonal_forecast': 'ICPAC - IGAD Climate Prediction and Applications Centre - Seasonal Forecast',
+            'faoswalim': 'FAO SWALIM'
+        }
+        return mapping.get(source_website, 'Unknown')
+    
+    def _identify_content_type(self, item: Dict[str, Any]) -> str:
+        """
+        Identify the type of content
+        
+        Args:
+            item: Raw data item
+            
+        Returns:
+            Content type identifier
+        """
+        # Handle document data with file_type field
+        if item.get('file_type'):
+            file_type = item.get('file_type', '').lower()
+            if 'pdf' in file_type:
+                return 'pdf_document'
+            elif 'doc' in file_type:
+                return 'word_document'
+            elif 'csv' in file_type:
+                return 'csv_data'
+            else:
+                return f'{file_type}_document'
+        
+        # Handle legacy pdf_path field
+        elif item.get('pdf_path') or item.get('file_path'):
+            return 'pdf_document'
+        
+        # Handle URL-based content type detection
+        url = item.get('url', '') or item.get('file_path', '')
+        if 'article' in url.lower():
+            return 'article'
+        elif 'publication' in url.lower():
+            return 'publication'
+        elif 'journal' in url.lower():
+            return 'journal'
+        elif 'event' in url.lower():
+            return 'event'
+        else:
+            return 'general'
+    
+    def _detect_language(self, text: str) -> str:
+        """
+        Detect language of the text (simplified)
+        
+        Args:
+            text: Text content
+            
+        Returns:
+            Language code
+        """
+        if not text:
+            return 'unknown'
+        
+        # Simple language detection based on common words
+        somali_words = ['somalia', 'somaliland', 'puntland', 'mogadishu', 'hargeisa']
+        english_words = ['the', 'and', 'of', 'in', 'to', 'for', 'with', 'on', 'at']
+        
+        text_lower = text.lower()
+        somali_count = sum(1 for word in somali_words if word in text_lower)
+        english_count = sum(1 for word in english_words if word in text_lower)
+        
+        if somali_count > english_count:
+            return 'so'
+        elif english_count > somali_count:
+            return 'en'
+        else:
+            return 'unknown'
+    
+    def _calculate_reading_time(self, text: str) -> float:
+        """
+        Calculate estimated reading time in minutes
+        
+        Args:
+            text: Text content
+            
+        Returns:
+            Reading time in minutes
+        """
+        word_count = len(text.split())
+        return round(word_count / 200, 1)  # Average reading speed: 200 words per minute
+    
+    def _calculate_complexity_score(self, text: str) -> float:
+        """
+        Calculate text complexity score
+        
+        Args:
+            text: Text content
+            
+        Returns:
+            Complexity score (0-1)
+        """
+        if not text:
+            return 0.0
+        
+        sentences = self._split_into_sentences(text)
+        if not sentences:
+            return 0.0
+        
+        avg_sentence_length = sum(len(s.split()) for s in sentences) / len(sentences)
+        long_words = sum(1 for word in text.split() if len(word) > 6)
+        total_words = len(text.split())
+        
+        complexity = (avg_sentence_length / 20) + (long_words / total_words if total_words > 0 else 0)
+        return min(complexity, 1.0)
+    
+    def _calculate_completeness_score(self, content: str) -> float:
+        """
+        Calculate content completeness score
+        
+        Args:
+            content: Text content
+            
+        Returns:
+            Completeness score (0-1)
+        """
+        if not content:
+            return 0.0
+        
+        score = 0.0
+        
+        # Length check
+        if len(content) > 100:
+            score += 0.3
+        
+        # Sentence count check
+        sentences = self._split_into_sentences(content)
+        if len(sentences) > 3:
+            score += 0.3
+        
+        # Paragraph count check
+        blocks = self._split_into_blocks(content)
+        if len(blocks) > 1:
+            score += 0.2
+        
+        # Basic content check
+        if len(content.split()) > 10:
+            score += 0.2
+        
+        return min(score, 1.0)
+    
+    def _calculate_coherence_score(self, content: str) -> float:
+        """
+        Calculate content coherence score
+        
+        Args:
+            content: Text content
+            
+        Returns:
+            Coherence score (0-1)
+        """
+        if not content:
+            return 0.0
+        
+        # Simple coherence based on sentence structure
+        sentences = self._split_into_sentences(content)
+        if len(sentences) < 2:
+            return 0.5
+        
+        # Check for proper sentence endings
+        proper_endings = sum(1 for s in sentences if s.endswith(('.', '!', '?')))
+        coherence = proper_endings / len(sentences)
+        
+        return min(coherence, 1.0)
+    
+    def _calculate_relevance_score(self, content: str, title: str) -> float:
+        """
+        Calculate content relevance score
+        
+        Args:
+            content: Text content
+            title: Title text
+            
+        Returns:
+            Relevance score (0-1)
+        """
+        if not content or not title:
+            return 0.0
+        
+        # Check if title words appear in content
+        title_words = set(title.lower().split())
+        content_words = set(content.lower().split())
+        
+        overlap = len(title_words.intersection(content_words))
+        relevance = overlap / len(title_words) if title_words else 0.0
+        
+        return min(relevance, 1.0)
+    
+    def _calculate_readability_score(self, content: str) -> float:
+        """
+        Calculate readability score
+        
+        Args:
+            content: Text content
+            
+        Returns:
+            Readability score (0-1)
+        """
+        if not content:
+            return 0.0
+        
+        sentences = self._split_into_sentences(content)
+        words = content.split()
+        
+        if not sentences or not words:
+            return 0.0
+        
+        # Simple readability based on sentence length and word length
+        avg_sentence_length = len(words) / len(sentences)
+        avg_word_length = sum(len(word) for word in words) / len(words)
+        
+        # Normalize to 0-1 scale
+        readability = 1.0 - (avg_sentence_length / 50) - (avg_word_length / 10)
+        
+        return max(0.0, min(readability, 1.0))
+    
+    def _calculate_summary_priority(self, text: str) -> str:
+        """
+        Calculate summary priority
+        
+        Args:
+            text: Text content
+            
+        Returns:
+            Priority level
+        """
+        word_count = len(text.split())
+        
+        if word_count > 1000:
+            return 'high'
+        elif word_count > 500:
+            return 'medium'
+        else:
+            return 'low'
+    
+    def _calculate_translation_priority(self, text: str) -> str:
+        """
+        Calculate translation priority
+        
+        Args:
+            text: Text content
+            
+        Returns:
+            Priority level
+        """
+        # Check for important keywords
+        important_keywords = ['emergency', 'crisis', 'disaster', 'flood', 'drought', 'food', 'security']
+        text_lower = text.lower()
+        
+        if any(keyword in text_lower for keyword in important_keywords):
+            return 'high'
+        elif len(text) > 500:
+            return 'medium'
+        else:
+            return 'low'
+    
+    def _validate_processed_item(self, item: Dict[str, Any]) -> bool:
+        """
+        Validate processed item
+        
+        Args:
+            item: Processed item
+            
+        Returns:
+            True if valid, False otherwise
+        """
+        required_fields = ['id', 'source_metadata', 'content', 'metadata']
+        
+        # Debug: Check which fields are missing
+        missing_fields = []
+        for field in required_fields:
+            if field not in item:
+                missing_fields.append(field)
+        
+        if missing_fields:
+            logger.warning(f"❌ Missing required fields: {missing_fields}")
+            logger.warning(f"📋 Available fields: {list(item.keys())}")
+            return False
+        
+        # Check content quality
+        content = item.get('content', {})
+        cleaned_text = content.get('cleaned_text', '')
+        if not cleaned_text:
+            logger.warning(f"❌ No cleaned_text found in content")
+            logger.warning(f"📋 Content structure: {content}")
+            return False
+        
+        # Check metadata quality
+        metadata = item.get('metadata', {})
+        word_count = metadata.get('word_count', 0)
+        if word_count < 10:
+            logger.warning(f"❌ Word count too low: {word_count} (minimum: 10)")
+            logger.warning(f"📋 Metadata: {metadata}")
+            return False
+        
+        logger.debug(f"✅ Validation passed for item {item.get('id', 'unknown')}")
+        return True
+    
+    def get_processing_stats(self) -> Dict[str, Any]:
+        """
+        Get processing statistics
+        
+        Returns:
+            Dictionary containing processing statistics
+        """
+        return self.processing_stats.copy()
+
+
+def preprocess_scraped_data(raw_data: List[Dict[str, Any]], output_path: Optional[str] = None) -> List[Dict[str, Any]]:
+    """
+    Convenience function to preprocess scraped data
+    
+    Args:
+        raw_data: List of raw scraped data
+        output_path: Optional output file path (deprecated - not used)
+        
+    Returns:
+        List of preprocessed data
+    """
+    preprocessor = DataPreprocessor()
+    processed_data = preprocessor.preprocess_all_data(raw_data)
+    
+    return processed_data
+
+
+if __name__ == "__main__":
+    # Example usage
+    sample_data = [
+        {
+            'title': 'Sample Article',
+            'content': 'This is a sample article about water management in Somalia.',
+            'url': 'https://example.com/article1',
+            'date': '2024-01-01'
+        }
+    ]
+    
+    processed = preprocess_scraped_data(sample_data)
+    print(f"Processed {len(processed)} items")

date_filter.py

@@ -0,0 +1,215 @@
+"""
+Date Filtering Module
+Handles date parsing and filtering for articles and documents
+"""
+
+import logging
+from datetime import datetime
+from typing import Optional
+import re
+from dateutil import parser as date_parser
+
+# Configure logging
+logger = logging.getLogger(__name__)
+
+
+def parse_article_date(date_str: str) -> Optional[datetime]:
+    """
+    Parse article date string into datetime object
+    Handles various date formats commonly found in scraped articles
+    
+    Args:
+        date_str: Date string to parse
+        
+    Returns:
+        datetime object if parsing successful, None otherwise
+    """
+    if not date_str or not date_str.strip():
+        return None
+    
+    date_str = date_str.strip()
+    
+    # Try to clean up common prefixes
+    date_str = re.sub(r'^(Posted on|Published on|Date:|Posted:|Published:)\s*', '', date_str, flags=re.IGNORECASE)
+    date_str = date_str.strip()
+    
+    # Try various parsing strategies
+    try:
+        # Strategy 1: Use dateutil parser (handles most formats)
+        try:
+            parsed_date = date_parser.parse(date_str, fuzzy=True, default=datetime.now().replace(hour=0, minute=0, second=0, microsecond=0))
+            logger.debug(f"✅ Successfully parsed date '{date_str}' to {parsed_date}")
+            return parsed_date
+        except (ValueError, TypeError) as e:
+            logger.debug(f"⚠️ dateutil parser failed for '{date_str}': {str(e)}")
+        
+        # Strategy 2: Try common ISO format patterns
+        iso_patterns = [
+            r'(\d{4}-\d{2}-\d{2})',  # YYYY-MM-DD
+            r'(\d{4}/\d{2}/\d{2})',  # YYYY/MM/DD
+            r'(\d{2}-\d{2}-\d{4})',  # DD-MM-YYYY
+            r'(\d{2}/\d{2}/\d{4})',  # DD/MM/YYYY
+        ]
+        
+        for pattern in iso_patterns:
+            match = re.search(pattern, date_str)
+            if match:
+                date_part = match.group(1)
+                try:
+                    # Try parsing with different separators
+                    if '-' in date_part:
+                        parts = date_part.split('-')
+                    elif '/' in date_part:
+                        parts = date_part.split('/')
+                    else:
+                        continue
+                    
+                    if len(parts[0]) == 4:  # YYYY-MM-DD or YYYY/MM/DD
+                        year, month, day = int(parts[0]), int(parts[1]), int(parts[2])
+                        parsed_date = datetime(year, month, day)
+                        logger.debug(f"✅ Successfully parsed date '{date_str}' to {parsed_date} using ISO pattern")
+                        return parsed_date
+                    elif len(parts[2]) == 4:  # DD-MM-YYYY or DD/MM/YYYY
+                        day, month, year = int(parts[0]), int(parts[1]), int(parts[2])
+                        parsed_date = datetime(year, month, day)
+                        logger.debug(f"✅ Successfully parsed date '{date_str}' to {parsed_date} using DD-MM-YYYY pattern")
+                        return parsed_date
+                except (ValueError, IndexError) as e:
+                    logger.debug(f"⚠️ Failed to parse date part '{date_part}': {str(e)}")
+                    continue
+        
+        logger.warning(f"⚠️ Could not parse date string: '{date_str}'")
+        return None
+        
+    except Exception as e:
+        logger.error(f"❌ Unexpected error parsing date '{date_str}': {str(e)}")
+        return None
+
+
+def standardize_date(date_str: str, default_to_current: bool = False) -> Optional[str]:
+    """
+    Standardize a date string to YYYY-MM-DD format for consistent storage and filtering.
+    
+    This function takes a date string in any format, parses it, and returns it
+    in a standardized YYYY-MM-DD format that can be used with the date filter.
+    
+    Args:
+        date_str: Date string in any format (e.g., "January 15, 2024", "15/01/2024", "Posted on 2024-01-15")
+        default_to_current: If True, return current date when parsing fails. If False, return None.
+        
+    Returns:
+        Standardized date string in YYYY-MM-DD format, or None if parsing fails (unless default_to_current=True)
+        
+    Examples:
+        >>> standardize_date("January 15, 2024")
+        '2024-01-15'
+        >>> standardize_date("Posted on 2024-01-15")
+        '2024-01-15'
+        >>> standardize_date("15/01/2024")
+        '2024-01-15'
+        >>> standardize_date("invalid date")
+        None
+        >>> standardize_date("invalid date", default_to_current=True)
+        '2025-01-07'  # Current date
+    """
+    if not date_str or not date_str.strip():
+        if default_to_current:
+            return datetime.now().strftime("%Y-%m-%d")
+        return None
+    
+    # Parse the date string
+    parsed_date = parse_article_date(date_str)
+    
+    if parsed_date is None:
+        if default_to_current:
+            logger.warning(f"⚠️ Could not parse date '{date_str}', using current date")
+            return datetime.now().strftime("%Y-%m-%d")
+        logger.debug(f"⚠️ Could not standardize date '{date_str}'")
+        return None
+    
+    # Return standardized format
+    standardized = parsed_date.strftime("%Y-%m-%d")
+    logger.debug(f"✅ Standardized date '{date_str}' to '{standardized}'")
+    return standardized
+
+
+def parse_date_input(date_input: str) -> Optional[datetime]:
+    """
+    Parse date input from UI (expected to be in YYYY-MM-DD format)
+    
+    Args:
+        date_input: Date string from UI input (YYYY-MM-DD format)
+        
+    Returns:
+        datetime object if parsing successful, None otherwise
+    """
+    if not date_input or not date_input.strip():
+        return None
+    
+    date_input = date_input.strip()
+    
+    try:
+        # Try parsing as YYYY-MM-DD
+        parsed_date = datetime.strptime(date_input, "%Y-%m-%d")
+        logger.debug(f"✅ Successfully parsed date input '{date_input}' to {parsed_date}")
+        return parsed_date
+    except ValueError:
+        try:
+            # Try using dateutil as fallback
+            parsed_date = date_parser.parse(date_input, fuzzy=False)
+            logger.debug(f"✅ Successfully parsed date input '{date_input}' to {parsed_date} using dateutil")
+            return parsed_date
+        except (ValueError, TypeError) as e:
+            logger.warning(f"⚠️ Could not parse date input '{date_input}': {str(e)}")
+            return None
+
+
+def is_date_in_range(article_date_str: str, start_date: Optional[datetime], end_date: Optional[datetime], include_missing: bool = True) -> bool:
+    """
+    Check if article date falls within the selected date range
+    
+    Args:
+        article_date_str: Article date as string
+        start_date: Start date of range (inclusive), None if no start date
+        end_date: End date of range (inclusive), None if no end date
+        include_missing: If True, include articles with missing/invalid dates. If False, exclude them.
+        
+    Returns:
+        True if article date is in range (or if no date range provided), False otherwise
+    """
+    # If no date range provided, include all articles
+    if start_date is None and end_date is None:
+        return True
+    
+    # Try to parse article date
+    article_date = parse_article_date(article_date_str)
+    
+    # Handle missing/invalid dates
+    if article_date is None:
+        logger.debug(f"⚠️ Could not parse article date '{article_date_str}', include_missing={include_missing}")
+        return include_missing
+    
+    # Check if date is within range
+    in_range = True
+    
+    if start_date is not None:
+        # Normalize to start of day for comparison
+        start_normalized = start_date.replace(hour=0, minute=0, second=0, microsecond=0)
+        article_normalized = article_date.replace(hour=0, minute=0, second=0, microsecond=0)
+        if article_normalized < start_normalized:
+            in_range = False
+            logger.debug(f"📅 Article date {article_normalized} is before start date {start_normalized}")
+    
+    if end_date is not None and in_range:
+        # Normalize to end of day for comparison
+        end_normalized = end_date.replace(hour=23, minute=59, second=59, microsecond=999999)
+        article_normalized = article_date.replace(hour=0, minute=0, second=0, microsecond=0)
+        if article_normalized > end_normalized:
+            in_range = False
+            logger.debug(f"📅 Article date {article_normalized} is after end date {end_normalized}")
+    
+    if in_range:
+        logger.debug(f"✅ Article date {article_date} is within range [{start_date}, {end_date}]")
+    
+    return in_range
+

document_processor.py

@@ -0,0 +1,558 @@
+from scraper_common import scrape_news_async, get_pdf_websites
+from datetime import datetime
+import os
+import requests
+from urllib.parse import urlparse
+
+def create_archive_folders(source: str, date: str = None) -> dict:
+    """
+    Create organized archive folder structure for document downloads
+    Returns a dictionary of document type folders:
+    {
+        'date_folder': date_folder,
+        'pdf_folder': pdf_folder,
+        'doc_folder': doc_folder,
+        'csv_folder': csv_folder
+    }
+    """
+    if date is None:
+        date = datetime.now().strftime("%Y-%m-%d")
+    
+    # Create main archive folder if it doesn't exist
+    archive_folder = "archive"
+    if not os.path.exists(archive_folder):
+        os.makedirs(archive_folder)
+    
+    # Normalize source name to prevent duplicate folders
+    # Handle the FS Cluster / fscluster case specifically
+    if source.lower() in ["fs cluster", "fscluster"]:
+        source = "FS Cluster"  # Use consistent name
+    
+    # Create source-specific folder
+    source_folder = os.path.join(archive_folder, source)
+    if not os.path.exists(source_folder):
+        os.makedirs(source_folder)
+    
+    # Create date-specific folder within source
+    date_folder = os.path.join(source_folder, date)
+    if not os.path.exists(date_folder):
+        os.makedirs(date_folder)
+    
+    # Create document type folders within date folder
+    pdf_folder = os.path.join(date_folder, "pdf")
+    doc_folder = os.path.join(date_folder, "doc")
+    csv_folder = os.path.join(date_folder, "csv")
+    
+    # Create folders if they don't exist
+    for folder in [pdf_folder, doc_folder, csv_folder]:
+        if not os.path.exists(folder):
+            os.makedirs(folder)
+    
+    return {
+        'date_folder': date_folder,
+        'pdf_folder': pdf_folder,
+        'doc_folder': doc_folder,
+        'csv_folder': csv_folder
+    }
+
+def download_document(doc_url: str, folder_paths: dict, filename: str = None) -> tuple:
+    """
+    Download document to specified folder and return local file path and document type
+    Returns a tuple of (local_path, file_type)
+    """
+    try:
+        # Generate filename if not provided
+        if not filename:
+            parsed_url = urlparse(doc_url)
+            filename = os.path.basename(parsed_url.path)
+            if not filename or 'downloadfile' in filename:
+                # Special case for MOPND and other sites with encoded filenames
+                filename = f"document_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+        
+        # Determine file type based on URL and/or Content-Type header
+        file_type = "unknown"
+        
+        # Check if URL has specific patterns that indicate file type
+        if (doc_url.lower().endswith('.pdf') or 
+            'pdf' in doc_url.lower() or
+            # MOPND specific patterns
+            'downloadfile' in doc_url.lower() or 
+            # Common base64 encoded PDF prefixes
+            'MjAyNS' in doc_url):  # Base64 pattern often used by MOPND
+            
+            file_type = "pdf"
+            target_folder = folder_paths['pdf_folder']
+            if not filename.endswith('.pdf'):
+                filename += '.pdf'
+        elif any(ext in doc_url.lower() for ext in ['.doc', '.docx', 'msword', 'officedocument']):
+            file_type = "doc"
+            target_folder = folder_paths['doc_folder']
+            if not any(filename.endswith(ext) for ext in ['.doc', '.docx']):
+                filename += '.docx'
+        elif '.csv' in doc_url.lower() or 'spreadsheet' in doc_url.lower():
+            file_type = "csv"
+            target_folder = folder_paths['csv_folder']
+            if not filename.endswith('.csv'):
+                filename += '.csv'
+        else:
+            # Default to PDF if unknown
+            file_type = "pdf"
+            target_folder = folder_paths['pdf_folder']
+            filename += '.pdf'
+        
+        # Set up headers to mimic a browser (helps with sites that block direct downloads)
+        headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.5",
+            "Connection": "keep-alive",
+            "Referer": doc_url
+        }
+        
+        # Download document
+        response = requests.get(doc_url, headers=headers, timeout=30)
+        response.raise_for_status()
+        
+        # Log response info for debugging
+        print(f"Downloaded document size: {len(response.content)} bytes")
+        print(f"Content-Type header: {response.headers.get('Content-Type', 'None')}")
+        
+        # Check Content-Type header to confirm file type
+        content_type = response.headers.get('Content-Type', '').lower()
+        
+        # More comprehensive content type detection
+        if 'pdf' in content_type:
+            file_type = "pdf"
+            if not filename.endswith('.pdf'):
+                filename = filename.rsplit('.', 1)[0] + '.pdf'
+        elif any(doc_type in content_type for doc_type in ['word', 'msword', 'officedocument', 'doc']):
+            file_type = "doc"
+            if not any(filename.endswith(ext) for ext in ['.doc', '.docx']):
+                filename = filename.rsplit('.', 1)[0] + '.docx'
+        elif any(csv_type in content_type for csv_type in ['csv', 'spreadsheet', 'excel', 'text/plain']):
+            file_type = "csv"
+            if not filename.endswith('.csv'):
+                filename = filename.rsplit('.', 1)[0] + '.csv'
+        elif 'octet-stream' in content_type:
+            # Try to detect file type from content
+            try:
+                # Check first few bytes for PDF signature (%PDF-)
+                if len(response.content) >= 5 and response.content[:5] == b'%PDF-':
+                    print("Detected PDF signature in content")
+                    file_type = "pdf"
+                    if not filename.endswith('.pdf'):
+                        filename = filename.rsplit('.', 1)[0] + '.pdf'
+                # Check for CSV-like content (text with commas)
+                elif len(response.content) > 100:
+                    sample = response.content[:1000].decode('utf-8', errors='ignore')
+                    if sample.count(',') > 5 and sample.count('\n') > 2:
+                        print("Content appears to be CSV based on commas and newlines")
+                        file_type = "csv"
+                        if not filename.endswith('.csv'):
+                            filename = filename.rsplit('.', 1)[0] + '.csv'
+            except Exception as e:
+                print(f"Error analyzing file content: {str(e)}")
+                # Keep existing file_type if content analysis fails
+        
+        print(f"Final determined file type: {file_type}")
+            
+        # Update target folder based on detected content type
+        if file_type == "pdf":
+            target_folder = folder_paths['pdf_folder']
+        elif file_type == "doc":
+            target_folder = folder_paths['doc_folder']
+        elif file_type == "csv":
+            target_folder = folder_paths['csv_folder']
+        
+        # Save to local folder
+        local_path = os.path.join(target_folder, filename)
+        with open(local_path, 'wb') as f:
+            f.write(response.content)
+            
+        print(f"Downloaded {file_type.upper()} file: {filename} ({len(response.content)} bytes)")
+        
+        return local_path, file_type
+        
+    except Exception as e:
+        print(f"Error downloading document {doc_url}: {str(e)}")
+        return None, None
+
+def extract_pdf_text_from_file(file_path: str) -> str:
+    """
+    Extract text from local PDF file using multiple methods for better compatibility
+    """
+    from document_scraper import extract_text_from_pdf_file
+    return extract_text_from_pdf_file(file_path)
+
+def process_direct_document(url: str, source: str = None) -> list:
+    """
+    Process a direct document URL without scraping the website
+    This is useful for direct PDF links when you only want to download and extract text
+    """
+    try:
+        # Determine source if not provided
+        if source is None:
+            if "reliefweb.int" in url:
+                source = "ReliefWeb"
+            elif "fscluster.org" in url:
+                source = "FS Cluster"
+            elif "mopnd.govsomaliland.org" in url:
+                source = "MOPND Somaliland"
+            elif "nbs.gov.so" in url:
+                source = "NBS Somalia"
+            elif "data.humdata.org" in url:
+                source = "HDX Humanitarian Data Exchange"
+            elif "logcluster.org" in url:
+                source = "LogCluster"
+            elif "fsnau.org" in url:
+                source = "FSNau - Food Security and Nutrition Analysis Unit"
+            elif "fews.net" in url:
+                source = "FEWS NET"
+            elif "icpac.net" in url:
+                source = "ICPAC"
+            elif "faoswalim.org" in url:
+                source = "FAO SWALIM"
+            else:
+                source = "Unknown"
+        
+        # Create folder structure
+        folder_paths = create_archive_folders(source)
+        
+        # Detect file type from URL
+        url_lower = url.lower()
+        if url_lower.endswith('.pdf'):
+            file_type = "pdf"
+        elif url_lower.endswith('.doc') or url_lower.endswith('.docx'):
+            file_type = "doc"
+        elif url_lower.endswith('.csv'):
+            file_type = "csv"
+        else:
+            # Try to detect file type from URL patterns
+            if 'pdf' in url_lower or 'document' in url_lower or 'report' in url_lower:
+                file_type = "pdf"
+            elif 'csv' in url_lower or 'data' in url_lower or 'dataset' in url_lower or 'export' in url_lower:
+                file_type = "csv"
+            elif 'doc' in url_lower:
+                file_type = "doc"
+            else:
+                file_type = "pdf"  # Default to PDF
+                
+        print(f"Detected file type from URL: {file_type}")
+        
+        # Generate filename
+        filename = f"document_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+        
+        # Download the file
+        local_path, detected_type = download_document(url, folder_paths, filename)
+        
+        if not local_path:
+            return [{
+                "title": "Download Error",
+                "date": datetime.now().strftime("%Y-%m-%d"),
+                "source": source,
+                "file_path": url,
+                "extracted_text": f"Failed to download document: {url}",
+                "file_type": "Error"
+            }]
+        
+        # Extract content based on file type
+        file_type = detected_type.upper() if detected_type else "UNKNOWN"
+        if file_type == "PDF":
+            extracted_text = extract_pdf_text_from_file(local_path)
+        elif file_type == "DOC":
+            extracted_text = f"Text from DOC file: {os.path.basename(local_path)}"
+        elif file_type == "CSV":
+            extracted_text = f"Data from CSV file: {os.path.basename(local_path)}"
+        else:
+            extracted_text = f"Content from {file_type} file: {os.path.basename(local_path)}"
+        
+        # Try to extract a title from the filename
+        title = os.path.basename(url)
+        
+        return [{
+            "title": title,
+            "date": datetime.now().strftime("%Y-%m-%d"),
+            "source": source,
+            "file_path": local_path,
+            "extracted_text": extracted_text,
+            "file_type": file_type
+        }]
+    
+    except Exception as e:
+        return [{
+            "title": f"Error processing document: {str(e)}",
+            "date": datetime.now().strftime("%Y-%m-%d"),
+            "source": "Error",
+            "file_path": url,
+            "extracted_text": f"Failed to process document URL: {url}",
+            "file_type": "Error"
+        }]
+
+async def process_documents_from_url(url: str, extract_website_content: bool = True) -> list:
+    """
+    Process documents from URL using the unified scraper with local PDF downloads
+    
+    Parameters:
+    - url: The URL to process
+    - extract_website_content: If False, only download and extract PDFs without scraping website content
+    
+    Returns:
+    - A list of document dictionaries
+    """
+    try:
+        # If we don't want to extract website content, check if this is a document URL
+        if not extract_website_content:
+            # Check for obvious document extensions first
+            if (url.lower().endswith('.pdf') or 
+                url.lower().endswith('.doc') or 
+                url.lower().endswith('.docx') or
+                url.lower().endswith('.csv')):
+                print(f"Processing direct document URL with extension: {url}")
+                return process_direct_document(url)
+            
+            # Check for URLs that might be documents without extensions
+            # Common patterns in document URLs
+            doc_indicators = [
+                'download', 'file', 'document', 'attachment', 'pdf', 'doc', 'csv',
+                'report', 'publication', 'data', 'dataset', 'export'
+            ]
+            
+            # Check if any of these indicators are in the URL
+            if any(indicator in url.lower() for indicator in doc_indicators):
+                print(f"URL appears to be a document without extension: {url}")
+                print("Attempting direct document processing...")
+                return process_direct_document(url)
+            
+        # Determine website name for folder organization
+        if "reliefweb.int" in url:
+            website_name = "reliefweb"
+            source = "ReliefWeb"
+        elif "fscluster.org" in url:
+            website_name = "fscluster"
+            source = "FS Cluster"
+        elif "mopnd.govsomaliland.org" in url:
+            website_name = "mopnd"
+            source = "MOPND Somaliland"
+        elif "nbs.gov.so" in url:
+            website_name = "nbs"
+            source = "NBS Somalia"
+        elif "data.humdata.org" in url:
+            website_name = "hdx"
+            source = "HDX Humanitarian Data Exchange"
+        elif "logcluster.org" in url:
+            website_name = "logcluster"
+            source = "LogCluster"
+        elif "fsnau.org" in url:
+            if "fsnau.org/publications" in url:
+                website_name = "fsnau_publications"
+                source = "FSNau Publications"
+            else:
+                website_name = "fsnau"
+                source = "FSNau - Food Security and Nutrition Analysis Unit"
+        elif "fews.net" in url:
+            website_name = "fews"
+            source = "FEWS NET - Famine Early Warning Systems Network"
+        elif "icpac.net" in url:
+            if "seasonal-forecast" in url.lower():
+                website_name = "icpac_seasonal_forecast"
+                source = "ICPAC - IGAD Climate Prediction and Applications Centre - Seasonal Forecast"
+            else:
+                website_name = "icpac"
+                source = "ICPAC - IGAD Climate Prediction and Applications Centre"
+        elif "frrims.faoswalim.org" in url:
+            website_name = "faoswalim_frrims_river_levels"
+            source = "FAO SWALIM FRRIMS River Levels"
+        elif "faoswalim.org" in url:
+            if "water/water-publications" in url or "water-publications" in url:
+                website_name = "faoswalim_water_publications"
+                source = "FAO SWALIM Water Publications"
+            elif "flood-watch-bulletin" in url or "ag-document-type/flood-watch-bulletin" in url:
+                website_name = "faoswalim_flood_watch"
+                source = "FAO SWALIM Flood Watch"
+            elif "faoswalim.org/swalim-events" in url:
+                website_name = "faoswalim_events"
+                source = "FAO SWALIM Events"
+            elif "faoswalim.org/swalim-journals" in url:
+                website_name = "faoswalim_journals"
+                source = "FAO SWALIM Journals"
+            elif "faoswalim.org/swalim-publications" in url:
+                website_name = "faoswalim_publications"
+                source = "FAO SWALIM Publications"
+            elif "faoswalim.org/swalim-articles" in url:
+                website_name = "faoswalim_articles"
+                source = "FAO SWALIM Articles"
+            else:
+                website_name = "faoswalim"
+                source = "FAO SWALIM - Somalia Water and Land Information Management"
+        elif "drought.emergency.copernicus.eu" in url:
+            website_name = "copernicus_drought"
+            source = "Copernicus Drought Observatory"
+        else:
+            website_name = "unknown"
+            source = "Unknown"
+        
+        # Create organized archive folder structure
+        folder_paths = create_archive_folders(source)
+        
+        # Process based on the extract_website_content flag
+        if extract_website_content:
+            # Use the unified scraper to get documents - force document mode
+            print("Scraping website content...")
+            articles = await scrape_news_async(url, website_name, force_mode="document")
+        else:
+            # If we're only interested in PDFs, check if this is a page that likely contains PDFs
+            # Dynamically determine if this is a PDF website
+            pdf_websites = get_pdf_websites()
+            if website_name in pdf_websites:
+                print(f"Directly downloading PDFs from {website_name} page without extracting website content...")
+                
+                # Import directly here to avoid circular import
+                from document_scraper import download_and_save_pdf
+                
+                # For PDF-only mode, we return early with a message
+                return [{
+                    "title": f"PDF-Only Mode for {source}",
+                    "date": datetime.now().strftime("%Y-%m-%d"),
+                    "source": source,
+                    "file_path": url,
+                    "extracted_text": f"PDF-only mode requested. Please use the direct document URL to download specific PDFs.",
+                    "file_type": "Info"
+                }]
+            else:
+                # For other sites, fall back to normal scraping (force document mode since we're in document processor)
+                print("PDF-only mode requested but this site isn't configured for direct PDF downloads.")
+                print("Falling back to normal website scraping...")
+                articles = await scrape_news_async(url, website_name, force_mode="document")
+                
+        # Convert articles to document format with local document downloads
+        documents = []
+        for i, article in enumerate(articles):
+            # Check for different possible path fields (regular path, local_file_path, pdf_path, local_path)
+            doc_path = article.get("pdf_path", "") or article.get("local_path", "")  # PDF path or other document URL
+            local_doc_path = article.get("local_file_path", "") or article.get("local_path", "")  # Try to get explicit local path if available
+            
+            # If local_file_path is not set but pdf_path is, use that
+            if not local_doc_path and doc_path:
+                local_doc_path = doc_path
+            
+            # Debug print
+            print(f"Processing article {i+1}:")
+            print(f"  Original doc_path: {doc_path}")
+            print(f"  Local path: {local_doc_path}")
+            
+            extracted_text = article.get("content", "") or article.get("extracted_text", "No content")
+            file_type = article.get("file_type", "Web Content")
+            
+            # If document URL exists, handle appropriately based on whether it's a local path or URL
+            if doc_path:
+                try:
+                    # Check if this is already a local file path (from the archive)
+                    if doc_path.startswith("archive/") or doc_path.startswith("/") or os.path.exists(doc_path):
+                        print(f"Using already archived file: {doc_path}")
+                        local_doc_path = doc_path
+                        
+                        # Determine file type based on extension
+                        if doc_path.lower().endswith(".pdf"):
+                            file_type = "PDF"
+                            extracted_text = article.get("content", "") or article.get("extracted_text", "No content")  # Already extracted by the scraper
+                        elif doc_path.lower().endswith((".doc", ".docx")):
+                            file_type = "DOC"
+                            # Keep content from scraper or add custom message
+                            if not extracted_text or extracted_text == "No content":
+                                extracted_text = f"Text from DOC file: {os.path.basename(doc_path)}"
+                        elif doc_path.lower().endswith(".csv"):
+                            file_type = "CSV"
+                            # Keep content from scraper or add custom message
+                            if not extracted_text or extracted_text == "No content":
+                                extracted_text = f"Data from CSV file: {os.path.basename(doc_path)}"
+                        else:
+                            file_type = "PDF"  # Default to PDF for archived files
+                    else:
+                        # This is a URL, so download it
+                        filename = f"document_{i+1}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+                        local_doc_path, detected_type = download_document(doc_path, folder_paths, filename)
+                        
+                        if local_doc_path:
+                            # Set file type based on detected type
+                            file_type = detected_type.upper() if detected_type else "PDF"
+                            
+                            # Extract text based on file type
+                            if file_type == "PDF":
+                                extracted_text = extract_pdf_text_from_file(local_doc_path)
+                            elif file_type == "DOC":
+                                # For future implementation: extract text from DOC files
+                                extracted_text = f"Text from DOC file: {os.path.basename(local_doc_path)}"
+                            elif file_type == "CSV":
+                                # For future implementation: extract text/preview from CSV files
+                                extracted_text = f"Data from CSV file: {os.path.basename(local_doc_path)}"
+                            else:
+                                # Generic extraction for unknown types
+                                extracted_text = f"Content from {file_type} file: {os.path.basename(local_doc_path)}"
+                        else:
+                            # Fallback to original content if download failed
+                            file_type = "Web Content"
+                            local_doc_path = doc_path  # Keep original URL
+                except Exception as e:
+                    print(f"Error processing document for article {i+1}: {str(e)}")
+                    file_type = "Web Content"
+                    local_doc_path = doc_path  # Keep original URL
+            else:
+                file_type = "Web Content"
+            
+            # Special handling for CSV files - ensure they're always included
+            if file_type == "CSV":
+                # For CSV files, use the extracted_text from the scraper if available
+                # Otherwise, ensure we have at least a basic description
+                if not extracted_text or extracted_text == "No content":
+                    csv_file_name = os.path.basename(local_doc_path) if local_doc_path else article.get("title", "CSV File")
+                    extracted_text = f"CSV File: {csv_file_name}\nFile Path: {local_doc_path or 'Not available'}\n(CSV file downloaded successfully)"
+                
+                # Ensure file_path is set for CSV files
+                if not local_doc_path:
+                    local_doc_path = article.get("local_path", "") or article.get("pdf_path", "")
+            
+            # Make sure we have a valid file path and type
+            document = {
+                "title": article.get("title", "No title"),
+                "date": article.get("date", datetime.now().strftime("%Y-%m-%d")),
+                "source": source,
+                "file_path": local_doc_path if local_doc_path else article.get("pdf_path", "") or article.get("local_path", ""),  # Ensure file_path is set
+                "extracted_text": extracted_text,
+                "file_type": file_type  # This will now be properly set to PDF, DOC, etc.
+            }
+            
+            # Special handling for CSV files - ensure they're always included even if file_path is missing
+            if file_type == "CSV" and not document["file_path"]:
+                # Try to get the URL as fallback
+                document["file_path"] = article.get("url", "")
+                print(f"⚠️ CSV file path not found, using URL: {document['file_path']}")
+            
+            # Special handling for NBS PDF files
+            if document["file_path"] and not document["file_path"].startswith(("http://", "https://")) and "pdf" in document["file_type"].lower():
+                # Force the document type to be PDF
+                document["file_type"] = "PDF"
+                print(f"Confirmed PDF document with local path: {document['file_path']}")
+            
+            # Special handling for CSV files - always include them
+            if file_type == "CSV":
+                print(f"✅ CSV file will be included: {document['title']} at {document['file_path']}")
+            
+            # Log the document info for debugging
+            print(f"Document {i+1}:")
+            print(f"  Title: {document['title']}")
+            print(f"  File Path: {document['file_path']}")
+            print(f"  File Type: {document['file_type']}")
+            print(f"  Text Length: {len(document['extracted_text'])} chars")
+            documents.append(document)
+        
+        return documents
+        
+    except Exception as e:
+        return [{
+            "title": f"Error processing documents: {str(e)}",
+            "date": datetime.now().strftime("%Y-%m-%d"),
+            "source": "Error",
+            "file_path": "",
+            "extracted_text": f"Failed to process URL: {url}",
+            "file_type": "Error"
+        }]

keyword_filter.py

@@ -0,0 +1,219 @@
+"""
+Keyword Filtering Module
+Handles keyword-based article filtering and categorization
+"""
+
+import json
+import logging
+import os
+from typing import Dict, List, Optional, Any
+
+# Configure logging
+logger = logging.getLogger(__name__)
+
+# Keywords configuration file path
+KEYWORDS_CONFIG_FILE = "keywords_config.json"
+
+def load_keywords_config() -> Dict[str, List[str]]:
+    """
+    Load keywords configuration from JSON file
+    
+    Returns:
+        Dictionary with categories as keys and keyword lists as values
+    """
+    try:
+        if not os.path.exists(KEYWORDS_CONFIG_FILE):
+            logger.warning(f"Keywords config file not found: {KEYWORDS_CONFIG_FILE}")
+            return {}
+        
+        with open(KEYWORDS_CONFIG_FILE, 'r', encoding='utf-8') as f:
+            config = json.load(f)
+        
+        # Extract categories from the config structure
+        categories = config.get('categories', {})
+        logger.info(f"Loaded {len(categories)} keyword categories")
+        return categories
+        
+    except Exception as e:
+        logger.error(f"Error loading keywords config: {str(e)}")
+        return {}
+
+def check_keyword_match(text: str, keywords: List[str]) -> bool:
+    """
+    Check if text contains any keyword (case-insensitive partial match)
+    
+    Args:
+        text: Text to search in
+        keywords: List of keywords to search for
+        
+    Returns:
+        True if any keyword is found, False otherwise
+    """
+    if not text or not keywords:
+        return False
+    
+    text_lower = text.lower()
+    for keyword in keywords:
+        if keyword.lower() in text_lower:
+            return True
+    return False
+
+def get_category_for_text(text: str, custom_keywords: str = "") -> Optional[str]:
+    """
+    Filter articles by keywords and assign category if keyword exists in config
+    
+    Args:
+        text: Text to check
+        custom_keywords: Comma-separated keywords to check
+        
+    Returns:
+        Category name if keyword in config, empty string if keyword matches but not in config,
+        None if no match (filter out)
+    """
+    if not text:
+        return None
+    
+    # If no keywords provided, keep all articles
+    if not custom_keywords or not custom_keywords.strip():
+        logger.debug("No keywords provided - keeping all articles")
+        return ""
+    
+    text_lower = text.lower()
+    
+    # Parse keywords
+    keywords_list = [kw.strip().lower() for kw in custom_keywords.split(",") if kw.strip()]
+    
+    # Load categories from config
+    categories = load_keywords_config()
+    
+    # Check if any keyword is present in the text
+    for keyword in keywords_list:
+        if keyword in text_lower:
+            logger.debug(f"Keyword '{keyword}' found in text")
+            
+            # Check if this keyword exists in any category
+            if categories:
+                for category_name, category_keywords in categories.items():
+                    # Check if the matched keyword is in this category
+                    if keyword in [kw.lower() for kw in category_keywords]:
+                        logger.debug(f"Keyword '{keyword}' found in category '{category_name}' - assigning category")
+                        return category_name
+            
+            # Keyword matched but not in any category - keep article with empty category
+            logger.debug(f"Keyword '{keyword}' not in any category - keeping article with empty category")
+            return ""
+    
+    # No keywords matched - filter out
+    logger.debug("No keywords matched - filtering out article")
+    return None
+
+def validate_keywords_structure(json_data: Any) -> tuple[bool, str]:
+    """
+    Validate JSON structure before saving
+    
+    Args:
+        json_data: JSON data to validate
+        
+    Returns:
+        Tuple of (is_valid, error_message)
+    """
+    try:
+        # Check if it's a dictionary
+        if not isinstance(json_data, dict):
+            return False, "Configuration must be a JSON object"
+        
+        # Check if 'categories' key exists
+        if 'categories' not in json_data:
+            return False, "Configuration must have a 'categories' key"
+        
+        categories = json_data['categories']
+        
+        # Check if categories is a dictionary
+        if not isinstance(categories, dict):
+            return False, "'categories' must be a dictionary"
+        
+        # Check each category
+        for category_name, keywords in categories.items():
+            # Category name must be a string
+            if not isinstance(category_name, str):
+                return False, f"Category name must be a string, got {type(category_name)}"
+            
+            # Keywords must be a list
+            if not isinstance(keywords, list):
+                return False, f"Keywords for category '{category_name}' must be a list, got {type(keywords)}"
+            
+            # Each keyword must be a string
+            for i, keyword in enumerate(keywords):
+                if not isinstance(keyword, str):
+                    return False, f"Keyword {i} in category '{category_name}' must be a string, got {type(keyword)}"
+                
+                # Check for empty keywords
+                if not keyword.strip():
+                    return False, f"Empty keyword found in category '{category_name}' at position {i}"
+        
+        return True, "Configuration is valid"
+        
+    except Exception as e:
+        return False, f"Validation error: {str(e)}"
+
+def save_keywords_config(json_data: Any) -> tuple[bool, str]:
+    """
+    Save validated keywords to file
+    
+    Args:
+        json_data: JSON data to save
+        
+    Returns:
+        Tuple of (success, message)
+    """
+    try:
+        # Validate the structure first
+        is_valid, error_message = validate_keywords_structure(json_data)
+        if not is_valid:
+            return False, f"Invalid configuration: {error_message}"
+        
+        # Save to file
+        with open(KEYWORDS_CONFIG_FILE, 'w', encoding='utf-8') as f:
+            json.dump(json_data, f, indent=2, ensure_ascii=False)
+        
+        logger.info(f"Keywords configuration saved successfully to {KEYWORDS_CONFIG_FILE}")
+        return True, "Keywords configuration saved successfully"
+        
+    except Exception as e:
+        error_msg = f"Error saving keywords config: {str(e)}"
+        logger.error(error_msg)
+        return False, error_msg
+
+def filter_article(article_dict: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+    """
+    Check if article matches any category and add category field
+    
+    Args:
+        article_dict: Article dictionary with title and content
+        
+    Returns:
+        Article dict with category field if match found, None if no match
+    """
+    if not article_dict:
+        return None
+    
+    # Combine title and content for keyword matching
+    title = article_dict.get('title', '')
+    content = article_dict.get('content', '')
+    combined_text = f"{title} {content}".strip()
+    
+    if not combined_text:
+        logger.debug("Article has no text content for keyword matching")
+        return None
+    
+    # Get category for the text
+    category = get_category_for_text(combined_text)
+    
+    if category:
+        # Add category to article dict
+        article_dict['category'] = category
+        logger.debug(f"Article categorized as: {category}")
+        return article_dict
+    else:
+        logger.debug("Article did not match any keyword categories")
+        return None

keywords_config.json

@@ -0,0 +1,184 @@
+{
+    "categories": {
+        "Floods / Rainfall / River": [
+            "flood",
+            "rainfall",
+            "river",
+            "shabelle",
+            "juba",
+            "overflow",
+            "inundation",
+            "hydro",
+            "rain gauge",
+            "flash flood",
+            "water level"
+        ],
+        "Drought / Climate Outlook": [
+            "drought",
+            "dry spell",
+            "arid",
+            "below-average",
+            "rainfall deficit",
+            "forecast",
+            "seasonal outlook",
+            "temperature",
+            "climate"
+        ],
+        "Markets / Prices": [
+            "market",
+            "price",
+            "inflation",
+            "maize",
+            "rice",
+            "sorghum",
+            "goat",
+            "livestock",
+            "supply chain",
+            "trade",
+            "cost",
+            "commodities"
+        ],
+        "Food Security / Livelihoods / IPC": [
+            "IPC",
+            "food security",
+            "livelihood",
+            "hunger",
+            "nutrition",
+            "famine",
+            "malnutrition",
+            "food access"
+        ],
+        "Conflict / Security / Incidents": [
+            "attack",
+            "conflict",
+            "clash",
+            "security",
+            "operation",
+            "fighting",
+            "violence",
+            "al-shabaab",
+            "ATMIS"
+        ],
+        "Displacement / Migration": [
+            "displacement",
+            "IDP",
+            "returnees",
+            "refugees",
+            "migration",
+            "evacuation",
+            "camps",
+            "PRMN",
+            "IOM",
+            "DTM"
+        ],
+        "Health / Epidemics": [
+            "cholera",
+            "malaria",
+            "covid",
+            "outbreak",
+            "disease",
+            "health",
+            "vaccination"
+        ],
+        "Economy / CPI / Statistics": [
+            "CPI",
+            "inflation",
+            "consumer price",
+            "GDP",
+            "NBS",
+            "survey",
+            "statistics",
+            "macroeconomy"
+        ],
+        "Agriculture / Crops / Livestock": [
+            "crop",
+            "harvest",
+            "planting",
+            "livestock",
+            "pasture",
+            "production",
+            "agriculture",
+            "farming"
+        ],
+        "Climate / Environment / NDVI": [
+            "NDVI",
+            "vegetation",
+            "land cover",
+            "land degradation",
+            "biodiversity",
+            "LST",
+            "soil moisture"
+        ],
+        "Humanitarian / Reports / Alerts": [
+            "humanitarian",
+            "alert",
+            "emergency",
+            "situation report",
+            "response",
+            "crisis",
+            "report"
+        ],
+        "Governance / Politics": [
+            "government",
+            "parliament",
+            "politics",
+            "election",
+            "president",
+            "minister",
+            "policy"
+        ],
+        "Community / Local News": [
+            "community",
+            "village",
+            "call-in",
+            "radio",
+            "NGO",
+            "awareness",
+            "training",
+            "people"
+        ],
+        "Press Releases / Official Statements": [
+            "press release",
+            "statement",
+            "announcement",
+            "press briefing"
+        ],
+        "Hazards / Disaster Mapping": [
+            "hazard",
+            "GDACS",
+            "UNOSAT",
+            "rapid mapping",
+            "flood map",
+            "damage",
+            "disaster",
+            "emergency"
+        ],
+        "Earth Observation / Satellite Data": [
+            "Sentinel",
+            "Copernicus",
+            "raster",
+            "imagery",
+            "NASA",
+            "geotiff",
+            "satellite"
+        ],
+        "Logistics / Supply Chain": [
+            "logistics",
+            "transport",
+            "port",
+            "corridor",
+            "warehouse",
+            "delivery",
+            "WFP supply"
+        ],
+        "Education / Social / Gender": [
+            "school",
+            "education",
+            "training",
+            "youth",
+            "women",
+            "empowerment",
+            "gender"
+        ]
+    }
+}

model_processor.py

@@ -0,0 +1,411 @@
+#!/usr/bin/env python3
+"""
+Model-based Processing Pipeline for News Dashboard
+Handles summarization and translation using Hugging Face transformers
+"""
+
+import logging
+import torch
+from typing import List, Dict, Any, Optional
+from transformers import (
+    AutoTokenizer, 
+    AutoModelForSeq2SeqLM,
+    pipeline,
+    BartForConditionalGeneration,
+    BartTokenizer
+)
+import warnings
+warnings.filterwarnings("ignore")
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class ModelProcessor:
+    """
+    Model-based processing for summarization and translation
+    """
+    
+    def __init__(self, device: str = "auto"):
+        """
+        Initialize the model processor
+        
+        Args:
+            device: Device to run models on ("auto", "cpu", "cuda")
+        """
+        self.device = self._get_device(device)
+        self.summarization_model = None
+        self.summarization_tokenizer = None
+        self.translation_model = None
+        self.translation_tokenizer = None
+        self.models_loaded = False
+        
+        logger.info(f"ModelProcessor initialized on device: {self.device}")
+    
+    def _get_device(self, device: str) -> str:
+        """
+        Determine the best device to use
+        
+        Args:
+            device: Requested device
+            
+        Returns:
+            Device string
+        """
+        if device == "auto":
+            if torch.cuda.is_available():
+                return "cuda"
+            elif hasattr(torch.backends, 'mps') and torch.backends.mps.is_available():
+                return "mps"  # Apple Silicon
+            else:
+                return "cpu"
+        return device
+    
+    def load_models(self) -> bool:
+        """
+        Load all required models
+        
+        Returns:
+            True if all models loaded successfully, False otherwise
+        """
+        try:
+            logger.info("Loading summarization model...")
+            self._load_summarization_model()
+            
+            logger.info("Loading translation model...")
+            self._load_translation_model()
+            
+            self.models_loaded = True
+            logger.info("All models loaded successfully!")
+            return True
+            
+        except Exception as e:
+            logger.error(f"Error loading models: {str(e)}")
+            return False
+    
+    def _load_summarization_model(self):
+        """
+        Load the summarization model and tokenizer
+        """
+        try:
+            # Use distilbart for good balance of quality and speed
+            model_name = "sshleifer/distilbart-cnn-12-6"
+            
+            self.summarization_tokenizer = AutoTokenizer.from_pretrained(model_name)
+            self.summarization_model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
+            
+            # Move to device
+            self.summarization_model.to(self.device)
+            self.summarization_model.eval()
+            
+            logger.info(f"Summarization model loaded: {model_name}")
+            
+        except Exception as e:
+            logger.error(f"Error loading summarization model: {str(e)}")
+            raise
+    
+    def _load_translation_model(self):
+        """
+        Load the translation model and tokenizer
+        """
+        try:
+            # Use Helsinki-NLP English-Somali model
+            model_name = "Helsinki-NLP/opus-mt-synthetic-en-so"
+            
+            self.translation_tokenizer = AutoTokenizer.from_pretrained(model_name)
+            self.translation_model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
+            
+            # Move to device
+            self.translation_model.to(self.device)
+            self.translation_model.eval()
+            
+            logger.info(f"Translation model loaded: {model_name}")
+            
+        except Exception as e:
+            logger.error(f"Error loading translation model: {str(e)}")
+            raise
+    
+    def process_content(self, content: str, max_length: int = 150, min_length: int = 30) -> Dict[str, Any]:
+        """
+        Process content through summarization and translation
+        
+        Args:
+            content: Text content to process
+            max_length: Maximum length for summary
+            min_length: Minimum length for summary
+            
+        Returns:
+            Dictionary containing processed results
+        """
+        if not self.models_loaded:
+            logger.error("Models not loaded. Call load_models() first.")
+            return {}
+        
+        if not content or len(content.strip()) < 50:
+            logger.warning("Content too short for processing")
+            return {
+                'summary': '',
+                'summary_somali': '',
+                'translation': '',
+                'bullet_points': [],
+                'bullet_points_somali': [],
+                'processing_success': False,
+                'error': 'Content too short'
+            }
+        
+        try:
+            # Summarize content
+            summary = self._summarize_content(content, max_length, min_length)
+            
+            # Create bullet points from summary
+            bullet_points = self._create_bullet_points(summary)
+            
+            # Translate to Somali
+            summary_somali = self._translate_to_somali(summary)
+            content_somali = self._translate_to_somali(content)
+            bullet_points_somali = [self._translate_to_somali(point) for point in bullet_points]
+            
+            return {
+                'summary': summary,
+                'summary_somali': summary_somali,
+                'translation': content_somali,
+                'bullet_points': bullet_points,
+                'bullet_points_somali': bullet_points_somali,
+                'processing_success': True,
+                'error': None
+            }
+            
+        except Exception as e:
+            logger.error(f"Error processing content: {str(e)}")
+            return {
+                'summary': '',
+                'summary_somali': '',
+                'translation': '',
+                'bullet_points': [],
+                'bullet_points_somali': [],
+                'processing_success': False,
+                'error': str(e)
+            }
+    
+    def _summarize_content(self, content: str, max_length: int, min_length: int) -> str:
+        """
+        Summarize content using the loaded model
+        
+        Args:
+            content: Text to summarize
+            max_length: Maximum summary length
+            min_length: Minimum summary length
+            
+        Returns:
+            Summarized text
+        """
+        try:
+            # Tokenize input
+            inputs = self.summarization_tokenizer(
+                content,
+                max_length=1024,  # Model's max input length
+                truncation=True,
+                return_tensors="pt"
+            ).to(self.device)
+            
+            # Generate summary
+            with torch.no_grad():
+                summary_ids = self.summarization_model.generate(
+                    inputs.input_ids,
+                    max_length=max_length,
+                    min_length=min_length,
+                    length_penalty=2.0,
+                    num_beams=4,
+                    early_stopping=True
+                )
+            
+            # Decode summary
+            summary = self.summarization_tokenizer.decode(
+                summary_ids[0], 
+                skip_special_tokens=True
+            )
+            
+            return summary.strip()
+            
+        except Exception as e:
+            logger.error(f"Error in summarization: {str(e)}")
+            return ""
+    
+    def _translate_to_somali(self, text: str) -> str:
+        """
+        Translate text to Somali using the loaded model
+        
+        Args:
+            text: Text to translate
+            
+        Returns:
+            Translated text
+        """
+        if not text or len(text.strip()) < 5:
+            return ""
+        
+        try:
+            # Tokenize input
+            inputs = self.translation_tokenizer(
+                text,
+                max_length=512,  # Model's max input length
+                truncation=True,
+                return_tensors="pt"
+            ).to(self.device)
+            
+            # Generate translation
+            with torch.no_grad():
+                translated_ids = self.translation_model.generate(
+                    inputs.input_ids,
+                    max_length=512,
+                    num_beams=4,
+                    early_stopping=True
+                )
+            
+            # Decode translation
+            translation = self.translation_tokenizer.decode(
+                translated_ids[0], 
+                skip_special_tokens=True
+            )
+            
+            return translation.strip()
+            
+        except Exception as e:
+            logger.error(f"Error in translation: {str(e)}")
+            return text  # Return original text if translation fails
+    
+    def _create_bullet_points(self, summary: str) -> List[str]:
+        """
+        Convert summary into bullet points
+        
+        Args:
+            summary: Summarized text
+            
+        Returns:
+            List of bullet points
+        """
+        if not summary:
+            return []
+        
+        # Split by sentences and create bullet points
+        sentences = [s.strip() for s in summary.split('.') if s.strip()]
+        
+        # Limit to 5 bullet points max
+        bullet_points = []
+        for i, sentence in enumerate(sentences[:5]):
+            if sentence:
+                # Clean up the sentence
+                sentence = sentence.strip()
+                if not sentence.endswith('.'):
+                    sentence += '.'
+                bullet_points.append(sentence)
+        
+        return bullet_points
+    
+    def process_batch(self, data_list: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """
+        Process a batch of data items
+        
+        Args:
+            data_list: List of data items to process
+            
+        Returns:
+            List of processed data items
+        """
+        if not self.models_loaded:
+            logger.error("Models not loaded. Call load_models() first.")
+            return data_list
+        
+        processed_data = []
+        
+        for i, item in enumerate(data_list):
+            logger.info(f"Processing item {i+1}/{len(data_list)}")
+            
+            # Get content from the item
+            content = item.get('content', {})
+            if isinstance(content, dict):
+                text_content = content.get('cleaned_text', '')
+            else:
+                text_content = str(content)
+            
+            # Process the content
+            model_results = self.process_content(text_content)
+            
+            # Add model results to the item
+            item['model_processing'] = model_results
+            
+            # Update content structure with model outputs
+            if isinstance(content, dict):
+                content['model_summary'] = model_results['summary']
+                content['model_summary_somali'] = model_results['summary_somali']
+                content['model_translation'] = model_results['translation']
+                content['bullet_points'] = model_results['bullet_points']
+                content['bullet_points_somali'] = model_results['bullet_points_somali']
+            
+            processed_data.append(item)
+        
+        logger.info(f"Batch processing completed: {len(processed_data)} items processed")
+        return processed_data
+    
+    def get_model_info(self) -> Dict[str, Any]:
+        """
+        Get information about loaded models
+        
+        Returns:
+            Dictionary with model information
+        """
+        return {
+            'models_loaded': self.models_loaded,
+            'device': self.device,
+            'summarization_model': 'distilbart-cnn-12-6' if self.summarization_model else None,
+            'translation_model': 'Helsinki-NLP/opus-mt-synthetic-en-so' if self.translation_model else None,
+            'cuda_available': torch.cuda.is_available(),
+            'mps_available': hasattr(torch.backends, 'mps') and torch.backends.mps.is_available()
+        }
+
+
+def process_with_models(data_list: List[Dict[str, Any]], device: str = "auto") -> List[Dict[str, Any]]:
+    """
+    Convenience function to process data with models
+    
+    Args:
+        data_list: List of data items to process
+        device: Device to run models on
+        
+    Returns:
+        List of processed data items
+    """
+    processor = ModelProcessor(device=device)
+    
+    if not processor.load_models():
+        logger.error("Failed to load models")
+        return data_list
+    
+    return processor.process_batch(data_list)
+
+
+if __name__ == "__main__":
+    # Example usage
+    sample_data = [
+        {
+            'id': 'test1',
+            'content': {
+                'cleaned_text': 'This is a sample article about water management in Somalia. The article discusses the challenges of water scarcity and the need for sustainable water management practices. It also covers the role of international organizations in supporting water infrastructure development.'
+            },
+            'source_metadata': {
+                'title': 'Water Management in Somalia',
+                'url': 'https://example.com'
+            }
+        }
+    ]
+    
+    # Process with models
+    processed = process_with_models(sample_data)
+    
+    # Print results (without full content)
+    for item in processed:
+        print(f"Original: (text length: {len(item['content']['cleaned_text'])} chars)")
+        print(f"Summary: {item['model_processing']['summary']}")
+        print(f"Bullet Points: {item['model_processing']['bullet_points']}")
+        print(f"Somali Translation: {item['model_processing']['summary_somali']}")
+        print("-" * 50)

postBuild

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -eux
+echo ">>> postBuild starting"
+
+python -m pip install --upgrade pip
+python -m pip install playwright
+python -m playwright install --with-deps chromium
+
+# Keep cache path explicit across sessions
+echo 'export PLAYWRIGHT_BROWSERS_PATH=/root/.cache/ms-playwright' >> "$HOME/.bashrc"
+
+echo ">>> postBuild done"

requirements.txt

@@ -0,0 +1,91 @@
+aiofiles==23.2.1
+annotated-types==0.7.0
+anyio==4.11.0
+beautifulsoup4==4.13.5
+Brotli==1.1.0
+certifi==2025.8.3
+cffi==2.0.0
+charset-normalizer==3.4.3
+click==8.3.0
+contourpy==1.3.2
+cryptography==46.0.1
+cycler==0.12.1
+et_xmlfile==2.0.0
+fastapi==0.117.1
+ffmpy==0.6.1
+filelock==3.19.1
+fonttools==4.60.0
+fsspec==2025.9.0
+gradio==5.47.2
+gradio_client==1.13.3
+greenlet==3.2.4
+groovy==0.1.2
+h11==0.16.0
+hf-xet==1.1.10
+httpcore==1.0.9
+httpx==0.28.1
+huggingface-hub==0.35.1
+idna==3.10
+importlib_resources==6.5.2
+Jinja2==3.1.6
+kiwisolver==1.4.9
+lxml==6.0.2
+markdown-it-py==4.0.0
+MarkupSafe==2.1.5
+matplotlib==3.10.6
+mdurl==0.1.2
+mpmath==1.3.0
+networkx==3.4.2
+numpy==2.2.6
+openpyxl==3.1.5
+orjson==3.11.3
+packaging==25.0
+pandas==2.3.2
+pdf2image==1.17.0
+pdfminer.six==20250506
+pdfplumber==0.11.7
+pillow==10.4.0
+playwright==1.55.0
+pycparser==2.23
+pydantic==2.11.9
+pydantic_core==2.33.2
+pydub==0.25.1
+pyee==13.0.0
+Pygments==2.19.2
+PyMuPDF==1.26.4
+pyparsing==3.2.5
+pypdf==6.1.0
+PyPDF2==3.0.1
+pypdfium2==4.30.0
+pytesseract==0.3.13
+python-dateutil==2.9.0.post0
+python-docx==1.2.0
+python-multipart==0.0.20
+pytz==2025.2
+PyYAML==6.0.3
+regex==2025.9.18
+requests==2.32.5
+rich==14.1.0
+ruff==0.13.2
+safehttpx==0.1.6
+safetensors==0.6.2
+semantic-version==2.10.0
+sentencepiece==0.2.1
+shellingham==1.5.4
+six==1.17.0
+sniffio==1.3.1
+soupsieve==2.8
+starlette==0.48.0
+sympy==1.14.0
+tokenizers==0.22.1
+tomlkit==0.12.0
+torch==2.8.0
+tqdm==4.67.1
+transformers==4.57.0
+typer==0.19.2
+typing-inspection==0.4.1
+typing_extensions==4.15.0
+tzdata==2025.2
+urllib3==2.5.0
+uvicorn==0.37.0
+websockets==15.0.1

runtime.txt

@@ -0,0 +1 @@
+python-3.11

scraper_common.py

@@ -0,0 +1,480 @@
+"""
+Common scraper functions - shared utilities for document and text scraping
+"""
+
+import asyncio
+import logging
+import os
+import json
+import hashlib
+from datetime import datetime
+from typing import List, Dict, Any
+from urllib.parse import urljoin, urlparse
+from playwright.async_api import async_playwright
+
+# --- Minimal Playwright hardening for headless containers (ADDED) ---
+os.environ.setdefault("PLAYWRIGHT_BROWSERS_PATH", "/root/.cache/ms-playwright")
+
+PLAYWRIGHT_LAUNCH_KW = dict(
+    headless=True,  # critical in HF Spaces/containers (no X server)
+    args=[
+        "--no-sandbox",
+        "--disable-setuid-sandbox",
+        "--disable-dev-shm-usage",
+        "--disable-gpu",
+        "--no-zygote",
+        "--single-process",
+        "--disable-extensions",
+        "--disable-background-networking",
+    ],
+)
+# --------------------------------------------------------------------
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+# Global timeout tracking for problematic URLs
+TIMEOUT_URLS = set()
+
+# Global flag for document-only scraping mode (text tab should ignore documents)
+DOCUMENT_ONLY_MODE = False
+
+# Global cancellation flag
+_scraping_cancelled = False
+
+# Global browser instance for cancellation
+current_browser = None
+current_page = None
+
+# Global captcha status for UI updates
+_captcha_status = None
+
+# Global constants for limiting scraping scope
+# Set these to None to disable limits, or to a number to limit
+MAX_PDF_LIMIT = 50    # Global limit to only process/download PDFs across all pages
+MAX_ARTICLE_LIMIT = 50 # Limit to only process 3 articles
+MAX_PAGE_LIMIT = 50   # Limit to only scrape 3 pages
+
+# Global PDF counter to track PDFs across all pages
+global_pdf_count = 0
+
+def reset_global_pdf_count():
+    """Reset the global PDF counter"""
+    global global_pdf_count
+    global_pdf_count = 0
+
+def increment_global_pdf_count():
+    """Increment the global PDF counter and return the new count"""
+    global global_pdf_count
+    global_pdf_count += 1
+    return global_pdf_count
+
+def get_global_pdf_count():
+    """Get the current global PDF count"""
+    return global_pdf_count
+
+def is_pdf_limit_reached():
+    """Check if the global PDF limit has been reached"""
+    if MAX_PDF_LIMIT is None:
+        return False
+    return global_pdf_count >= MAX_PDF_LIMIT
+
+# Archive management
+ARCHIVE_DIR = "archive"
+ARCHIVE_INDEX = os.path.join(ARCHIVE_DIR, "archive_index.json")
+
+# Load website configuration
+def load_website_config():
+    """Load website configuration from JSON file"""
+    try:
+        with open('website_config.json', 'r') as f:
+            config = json.load(f)
+        logger.info("✅ Website configuration loaded successfully")
+        return config
+    except Exception as e:
+        logger.error(f"❌ Error loading website configuration: {str(e)}")
+        return {}
+
+# Load the website configuration
+WEBSITE_CONFIG = load_website_config()
+
+def get_pdf_websites() -> List[str]:
+    """
+    Dynamically get list of PDF websites from website_config.json
+    A website is considered a PDF website if it has 'pdf_links', 'file_links', or 'extract_table_as_csv' in its config
+    """
+    pdf_websites = []
+    for website_type, config in WEBSITE_CONFIG.items():
+        if config and isinstance(config, dict):
+            # Check if config has pdf_links, file_links, or extract_table_as_csv
+            if config.get("pdf_links") or config.get("file_links") or config.get("extract_table_as_csv"):
+                pdf_websites.append(website_type)
+    return pdf_websites
+
+def get_content_websites() -> List[str]:
+    """
+    Dynamically get list of content (text) websites from website_config.json
+    A website is considered a content website if it does NOT have 'pdf_links' or 'file_links'
+    """
+    content_websites = []
+    for website_type, config in WEBSITE_CONFIG.items():
+        if config and isinstance(config, dict):
+            if not config.get("pdf_links") and not config.get("file_links"):
+                content_websites.append(website_type)
+    return content_websites
+
+# Debug: Print configured website types when module loads
+_debug_pdf_websites = get_pdf_websites()
+_debug_content_websites = get_content_websites()
+logger.debug(f"📄 PDF Websites configured ({len(_debug_pdf_websites)}): {sorted(_debug_pdf_websites)}")
+logger.debug(f"📰 Content Websites configured ({len(_debug_content_websites)}): {sorted(_debug_content_websites)}")
+
+def validate_website_config(config: dict) -> tuple[bool, str]:
+    """
+    Validate website configuration structure
+    
+    Args:
+        config: Configuration dictionary to validate
+        
+    Returns:
+        Tuple of (is_valid, error_message)
+    """
+    try:
+        if not isinstance(config, dict):
+            return False, "Configuration must be a dictionary"
+        
+        for website_type, website_config in config.items():
+            if not isinstance(website_type, str):
+                return False, f"Website type must be a string, got {type(website_type)}"
+            
+            # Validate website type name (no spaces, valid identifier)
+            if ' ' in website_type or not website_type:
+                return False, f"Website type '{website_type}' must be a valid identifier (no spaces)"
+            
+            if not isinstance(website_config, dict):
+                return False, f"Configuration for '{website_type}' must be a dictionary"
+            
+            # Check required fields: title and content (at least one must be present)
+            if 'title' not in website_config and 'content' not in website_config:
+                return False, f"Website '{website_type}' must have at least 'title' or 'content' field"
+            
+            # Validate field types
+            string_fields = ['article_links', 'page_links', 'title', 'content', 'date', 
+                           'navigation_selector', 'navigation_url_addition', 'recaptcha_text']
+            for field in string_fields:
+                if field in website_config:
+                    value = website_config[field]
+                    # Allow string, None, or list (for content field)
+                    if value is not None and not isinstance(value, (str, list)):
+                        return False, f"Field '{field}' in '{website_type}' must be string, list, or null"
+            
+            # Validate start_page (must be integer >= 0)
+            if 'start_page' in website_config:
+                start_page = website_config['start_page']
+                if start_page is not None:
+                    try:
+                        start_page_int = int(start_page)
+                        if start_page_int < 0:
+                            return False, f"'start_page' in '{website_type}' must be >= 0"
+                    except (ValueError, TypeError):
+                        return False, f"'start_page' in '{website_type}' must be an integer"
+            
+            # Validate array fields
+            array_fields = ['pdf_links', 'file_links']
+            for field in array_fields:
+                if field in website_config:
+                    value = website_config[field]
+                    if value is not None:
+                        if isinstance(value, str):
+                            # Allow string, will be converted to array
+                            pass
+                        elif not isinstance(value, list):
+                            return False, f"Field '{field}' in '{website_type}' must be a list or null"
+        
+        return True, "Configuration is valid"
+        
+    except Exception as e:
+        return False, f"Validation error: {str(e)}"
+
+def save_website_config(config_data: dict) -> tuple[bool, str]:
+    """
+    Save validated website configuration to file
+    
+    Args:
+        config_data: Configuration dictionary to save
+        
+    Returns:
+        Tuple of (success, message)
+    """
+    global WEBSITE_CONFIG
+    
+    try:
+        # Validate the structure first
+        is_valid, error_message = validate_website_config(config_data)
+        if not is_valid:
+            return False, f"Invalid configuration: {error_message}"
+        
+        # Save to file
+        with open('website_config.json', 'w', encoding='utf-8') as f:
+            json.dump(config_data, f, indent=4, ensure_ascii=False)
+        
+        # Reload the global config
+        WEBSITE_CONFIG = load_website_config()
+        
+        logger.info("✅ Website configuration saved successfully")
+        return True, "Website configuration saved successfully"
+        
+    except Exception as e:
+        error_msg = f"Error saving website config: {str(e)}"
+        logger.error(f"❌ {error_msg}")
+        return False, error_msg
+
+def set_document_only_mode(value: bool):
+    """Set the global document-only mode flag."""
+    global DOCUMENT_ONLY_MODE
+    DOCUMENT_ONLY_MODE = value
+
+def is_document_mode_enabled() -> bool:
+    """Check if document-only mode is enabled."""
+    return DOCUMENT_ONLY_MODE
+
+def set_scraping_cancelled(value: bool):
+    """Set the global cancellation flag"""
+    global _scraping_cancelled
+    _scraping_cancelled = value
+
+def scraping_cancelled() -> bool:
+    """Check if scraping has been cancelled"""
+    return _scraping_cancelled
+
+def get_captcha_status():
+    """Get the current captcha status message"""
+    global _captcha_status
+    return _captcha_status
+
+def set_captcha_status(status: str):
+    """Set the captcha status message"""
+    global _captcha_status
+    _captcha_status = status
+
+def clear_captcha_status():
+    """Clear the captcha status"""
+    global _captcha_status
+    _captcha_status = None
+
+async def force_close_browser():
+    """Force close browser and page instances"""
+    global current_browser, current_page
+    try:
+        if current_page:
+            await current_page.close()
+            current_page = None
+        if current_browser:
+            await current_browser.close()
+            current_browser = None
+    except Exception as e:
+        logger.error(f"Error closing browser: {str(e)}")
+
+def convert_to_absolute_url(href: str, base_url: str) -> str:
+    """
+    Convert relative URL to absolute URL
+    """
+    if href.startswith(('http://', 'https://')):
+        return href
+    return urljoin(base_url, href)
+
+def ensure_archive_directory():
+    """Ensure archive directory exists"""
+    if not os.path.exists(ARCHIVE_DIR):
+        os.makedirs(ARCHIVE_DIR)
+        logger.info(f"📁 Created archive directory: {ARCHIVE_DIR}")
+
+async def scrape_news_async(url: str, website_type: str, custom_keywords: str = "", start_date: str = None, end_date: str = None, force_mode: str = None) -> List[dict]:
+    """
+    Main entry point for scraping - delegates to appropriate scraper
+    
+    Args:
+        url: URL to scrape
+        website_type: Website type identifier
+        custom_keywords: Custom keywords for filtering
+        start_date: Optional start date for filtering
+        end_date: Optional end date for filtering
+        force_mode: Force scraper mode - "text" for text scraper, "document" for document scraper, None for auto-detect
+    """
+    try:
+        logger.info(f"🚀 Starting scraping for {website_type} at {url}")
+        
+        # Determine which scraper to use
+        use_document_scraper = False
+        
+        if force_mode == "text":
+            # Force text scraper
+            use_document_scraper = False
+            logger.info(f"📰 Forcing text scraper mode for {website_type}")
+        elif force_mode == "document":
+            # Force document scraper
+            use_document_scraper = True
+            logger.info(f"📄 Forcing document scraper mode for {website_type}")
+        else:
+            # Auto-detect based on config (backward compatible)
+            pdf_websites = get_pdf_websites()
+            use_document_scraper = website_type in pdf_websites
+            if use_document_scraper:
+                logger.info(f"📄 Auto-detected: Using document scraper for {website_type}")
+            else:
+                logger.info(f"📰 Auto-detected: Using text scraper for {website_type}")
+        
+        # Import the appropriate scraper
+        if use_document_scraper:
+            # Document-focused sites
+            from document_scraper import extract_document_content_unified, download_all_pdfs_from_page
+        else:
+            # Text-focused sites
+            from text_scraper import extract_article_content_unified, get_all_article_links_unified, extract_all_articles_unified
+        
+        # Get website configuration
+        config = WEBSITE_CONFIG.get(website_type)
+        if not config:
+            logger.error(f"❌ No configuration found for website type: {website_type}")
+            return [{
+                "title": "Configuration Error",
+                "content": f"No configuration found for website type: {website_type}",
+                "date": datetime.now().strftime("%Y-%m-%d"),
+                "url": url
+            }]
+        
+        # Initialize browser
+        async with async_playwright() as p:
+            # CHANGED: use hardened, headless launch to avoid X server errors
+            browser = await p.chromium.launch(**PLAYWRIGHT_LAUNCH_KW)
+            page = await browser.new_page()
+            
+            # Block ads, CSS, and images for better performance
+            await page.route("**/*", lambda route: (
+                route.abort() if any(blocked in route.request.url.lower() for blocked in [
+                    # Ad domains
+                    "googleads", "doubleclick", "googlesyndication", "google-analytics",
+                    "facebook.com/tr", "googletagmanager", "amazon-adsystem", "adsystem",
+                    "googletagservices", "ads.yahoo.com", "googletagservices",
+                    # CSS files
+                    ".css", "stylesheet", "font-awesome", "bootstrap.css",
+                    # Images
+                    ".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".ico",
+                    "image/", "img/", "images/", "photos/", "pictures/",
+                    # Fonts
+                    ".woff", ".woff2", ".ttf", ".eot", "fonts/", "font/",
+                    # Videos and media
+                    ".mp4", ".avi", ".mov", ".wmv", ".flv", "video/", "media/",
+                    # Analytics and tracking
+                    "analytics", "tracking", "metrics", "stats", "telemetry"
+                ]) else route.continue_()
+            ))
+            
+            # Store browser instance for cancellation
+            global current_browser, current_page
+            current_browser = browser
+            current_page = page
+            
+            try:
+                # Navigate to the main page with retry logic (5 attempts)
+                max_retries = 5
+                retry_count = 0
+                page_loaded = False
+                
+                while retry_count < max_retries and not page_loaded:
+                    try:
+                        retry_count += 1
+                        logger.info(f"🔄 Loading website (attempt {retry_count}/{max_retries}): {url}")
+                        
+                        # Navigate with different strategies based on attempt
+                        if retry_count == 1:
+                            # First attempt: Use domcontentloaded for faster loading
+                            await page.goto(url, wait_until="domcontentloaded", timeout=30000)
+                        elif retry_count == 2:
+                            # Second attempt: Use basic loading
+                            await page.goto(url, timeout=20000)
+                        elif retry_count == 3:
+                            # Third attempt: Use networkidle
+                            await page.goto(url, wait_until="networkidle", timeout=15000)
+                        else:
+                            # Fourth and fifth attempts: Try with shorter timeouts
+                            await page.goto(url, timeout=10000)
+                        
+                        logger.info(f"✅ Successfully loaded website on attempt {retry_count}")
+                        page_loaded = True
+                        
+                    except Exception as e:
+                        logger.warning(f"⚠️ Attempt {retry_count} failed for {url}: {str(e)}")
+                        
+                        if retry_count >= max_retries:
+                            logger.error(f"❌ Failed to load website after {max_retries} attempts: {url}")
+                            return [{
+                                "title": "WEBSITE_LOAD_ERROR",
+                                "content": f"Website is not working. Please try again later. Failed to access website after {max_retries} attempts: {str(e)}",
+                                "date": datetime.now().strftime("%Y-%m-%d"),
+                                "url": url
+                            }]
+                        
+                        # Wait before retry
+                        await asyncio.sleep(2)
+                
+                if not page_loaded:
+                    return [{
+                        "title": "WEBSITE_LOAD_ERROR",
+                        "content": f"Website is not working. Please try again later. Failed to access website after {max_retries} attempts",
+                        "date": datetime.now().strftime("%Y-%m-%d"),
+                        "url": url
+                    }]
+                
+                # Check for captcha on initial page load
+                if use_document_scraper:
+                    from document_scraper import check_and_wait_for_recaptcha
+                    captcha_result = await check_and_wait_for_recaptcha(page, config)
+                    if captcha_result == "CAPTCHA_TIMEOUT":
+                        logger.error("❌ Captcha detected but not solved within timeout period")
+                        return [{
+                            "title": "CAPTCHA_ERROR",
+                            "content": "Captcha detected. Please try again later. The website requires captcha verification which could not be completed automatically.",
+                            "date": datetime.now().strftime("%Y-%m-%d"),
+                            "url": url
+                        }]
+                
+                # Delegate to appropriate scraper based on determined mode
+                if use_document_scraper:
+                    # Document processing
+                    all_articles = await download_all_pdfs_from_page(page, url, config, website_type, start_date, end_date)
+                else:
+                    # Text processing
+                    all_article_links = await get_all_article_links_unified(page, url, config, website_type)
+                    
+                    if not all_article_links:
+                        return [{
+                            "title": "No articles found",
+                            "content": "No articles were found on the specified page",
+                            "date": datetime.now().strftime("%Y-%m-%d"),
+                            "url": url
+                        }]
+                    
+                    # Extract content from all articles
+                    all_articles = await extract_all_articles_unified(page, all_article_links, config, website_type, custom_keywords, start_date, end_date)
+                
+                return all_articles
+                
+            finally:
+                # Clean up browser
+                await browser.close()
+                current_browser = None
+                current_page = None
+                
+    except Exception as e:
+        logger.error(f"❌ Error in main scraping function: {str(e)}")
+        return [{
+            "title": "Scraping Error",
+            "content": f"Error during scraping: {str(e)}",
+            "date": datetime.now().strftime("%Y-%m-%d"),
+            "url": url
+        }]

sessions.json

@@ -0,0 +1 @@
+{}

text_scraper.py

@@ -0,0 +1,546 @@
+"""
+Text Scraper - Handles article and text content processing
+"""
+
+import asyncio
+import logging
+import re
+from datetime import datetime
+from typing import List, Dict, Any
+import time
+# Import common functions from scraper_common
+from scraper_common import (
+    WEBSITE_CONFIG, MAX_ARTICLE_LIMIT, MAX_PAGE_LIMIT,
+    convert_to_absolute_url, scraping_cancelled
+)
+
+# Import keyword filtering utilities
+from keyword_filter import get_category_for_text
+
+# Import date filtering utilities
+from date_filter import is_date_in_range, standardize_date
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+
+def construct_navigation_url(base_url: str, nav_addition: str) -> str:
+    """
+    Construct navigation URL by properly handling trailing slashes and query parameters
+    """
+    # Remove trailing slash from base URL if it exists
+    if base_url.endswith('/'):
+        base_url = base_url.rstrip('/')
+    
+    # Check if nav_addition starts with / or ?
+    if nav_addition.startswith('/'):
+        # Direct path addition
+        return base_url + nav_addition
+    elif nav_addition.startswith('?'):
+        # Query parameter addition
+        return base_url + nav_addition
+    else:
+        # Default: add as path
+        return base_url + '/' + nav_addition
+
+# Global variables for text processing
+mopnd_article_dates = {}
+
+async def get_article_links_with_dates_from_page(page, config: dict, website_type: str) -> List[str]:
+    """
+    Get article links with dates from a single page (for MOPND)
+    """
+    try:
+        logger.info(f"🔍 Extracting article links with dates from page for {website_type}")
+        
+        # Get article link selector (check both article_links and page_links for PDF sites)
+        article_selector = config.get("article_links") or config.get("page_links")
+        if not article_selector:
+            logger.warning("⚠️ No article_links or page_links selector found in config")
+            return []
+        
+        # Get date selector
+        date_selector = config.get("date")
+        if not date_selector:
+            logger.warning("⚠️ No date selector found in config")
+            return []
+        
+        # Get all article link elements
+        link_elements = await page.query_selector_all(article_selector)
+        logger.info(f"📰 Found {len(link_elements)} article link elements")
+        
+        # Get all date elements
+        date_elements = await page.query_selector_all(date_selector)
+        logger.info(f"📅 Found {len(date_elements)} date elements")
+        
+        # Extract links and dates
+        article_links = []
+        for i, link_element in enumerate(link_elements):
+            try:
+                # Get the href attribute
+                href = await link_element.get_attribute("href")
+                if href:
+                    # Convert to absolute URL
+                    absolute_url = convert_to_absolute_url(href, page.url)
+                    article_links.append(absolute_url)
+                    
+                    # Try to get corresponding date (assuming same order)
+                    if i < len(date_elements):
+                        try:
+                            date_text = await date_elements[i].text_content()
+                            if date_text and date_text.strip():
+                                # Store the date for this article URL
+                                mopnd_article_dates[absolute_url] = date_text.strip()
+                                logger.debug(f"✅ Stored date for {absolute_url}: {date_text.strip()}")
+                        except Exception as e:
+                            logger.debug(f"⚠️ Could not extract date for link {i}: {str(e)}")
+                    
+            except Exception as e:
+                logger.warning(f"❌ Error extracting link {i}: {str(e)}")
+                continue
+        
+        logger.info(f"🔗 Extracted {len(article_links)} article links with dates")
+        return article_links
+        
+    except Exception as e:
+        logger.error(f"❌ Error extracting article links with dates: {str(e)}")
+        return []
+
+async def get_all_article_links_unified(page, url: str, config: dict, website_type: str = None) -> List[str]:
+    """
+    Function to get article links from multiple pages with pagination support
+    Stops when no new (non-repeating) articles are found
+    """
+    try:
+        logger.info(f"🔍 Getting article links from: {url}")
+        logger.info(f"🌐 Website type: {website_type}")
+        
+        # Check if navigation is configured
+        navigation_selector = config.get("navigation_selector")
+        navigation_url_addition = config.get("navigation_url_addition")
+        start_page = config.get("start_page", 1)
+        
+        all_article_links = []
+        seen_links = set()  # Track unique links to detect duplicates
+        current_page = start_page
+        consecutive_empty_pages = 0
+        max_consecutive_empty = 2  # Stop after 2 consecutive pages with no new content
+        
+        # Navigate to the initial page
+        await page.goto(url, wait_until="domcontentloaded", timeout=30000)
+        
+        # Handle pagination if configured
+        if navigation_selector and navigation_url_addition:
+            logger.info(f"🧭 Navigation configured: selector={navigation_selector}, url_addition={navigation_url_addition}")
+            logger.info(f"📄 Starting from page: {start_page}")
+            
+            while True:
+                logger.info(f"📄 Processing page {current_page}")
+                
+                # Check MAX_PAGE_LIMIT if set
+                if MAX_PAGE_LIMIT is not None and current_page > MAX_PAGE_LIMIT:
+                    logger.info(f"🛑 Reached MAX_PAGE_LIMIT ({MAX_PAGE_LIMIT}), stopping pagination")
+                    break
+                
+                # Navigate to current page if not the first page
+                if current_page > start_page:
+                    nav_url_addition = navigation_url_addition.replace("{page_no}", str(current_page))
+                    nav_url = construct_navigation_url(url, nav_url_addition)
+                    logger.info(f"🧭 Navigating to: {nav_url}")
+                    await page.goto(nav_url, wait_until="domcontentloaded", timeout=30000)
+                
+                # Check if navigation element exists for next page
+                nav_element = await page.query_selector(navigation_selector)
+                if current_page == start_page and nav_element:
+                    logger.info("✅ Navigation element found, more pages available")
+                elif current_page > start_page and not nav_element:
+                    logger.info("📄 No more navigation elements found, stopping pagination")
+                    break
+                
+                # Extract links from current page
+                page_links = await extract_links_from_current_page(page, config, website_type)
+                
+                if page_links:
+                    # Check for new (non-duplicate) links
+                    new_links = []
+                    for link in page_links:
+                        if link not in seen_links:
+                            seen_links.add(link)
+                            new_links.append(link)
+                    
+                    if new_links:
+                        all_article_links.extend(new_links)
+                        consecutive_empty_pages = 0  # Reset counter
+                        logger.info(f"📰 Found {len(new_links)} new links on page {current_page} (total: {len(page_links)} links on page)")
+                    else:
+                        consecutive_empty_pages += 1
+                        logger.info(f"📰 No new links found on page {current_page} (all {len(page_links)} links were duplicates)")
+                        
+                        # Stop if we've had too many consecutive pages with no new content
+                        if consecutive_empty_pages >= max_consecutive_empty:
+                            logger.info(f"🛑 Stopping pagination: {consecutive_empty_pages} consecutive pages with no new content")
+                            break
+                else:
+                    consecutive_empty_pages += 1
+                    logger.info(f"📰 No links found on page {current_page}")
+                    
+                    # Stop if we've had too many consecutive pages with no content
+                    if consecutive_empty_pages >= max_consecutive_empty:
+                        logger.info(f"🛑 Stopping pagination: {consecutive_empty_pages} consecutive pages with no content")
+                        break
+                
+                current_page += 1
+                
+        else:
+            # No pagination configured, scrape single page only
+            logger.info("📄 No navigation configured - scraping single page only")
+            page_links = await extract_links_from_current_page(page, config, website_type)
+            all_article_links.extend(page_links)
+        
+        logger.info(f"📊 Total unique article links found across all pages: {len(all_article_links)}")
+        return all_article_links
+        
+    except Exception as e:
+        logger.error(f"❌ Error getting article links: {str(e)}")
+        return []
+
+
+async def extract_links_from_current_page(page, config: dict, website_type: str) -> List[str]:
+    """
+    Extract article links from the current page
+    """
+    try:
+        # For MOPND, use special function to get links with dates
+        if website_type == "mopnd":
+            return await get_article_links_with_dates_from_page(page, config, website_type)
+        else:
+            # Regular article link extraction (check both article_links and page_links for PDF sites)
+            article_selector = config.get("article_links") or config.get("page_links")
+            if not article_selector:
+                logger.warning("⚠️ No article_links or page_links selector found in config")
+                return []
+            
+            # Handle different selector types
+            if isinstance(article_selector, list):
+                # If it's a list, use the first selector
+                article_selector = article_selector[0]
+                logger.info(f"📝 Using first selector from list: {article_selector}")
+            elif not isinstance(article_selector, str):
+                logger.error(f"❌ Invalid selector type: {type(article_selector)}. Expected string or list.")
+                return []
+            
+            # Get all article link elements
+            link_elements = await page.query_selector_all(article_selector)
+            logger.info(f"📰 Found {len(link_elements)} article link elements on current page")
+            
+            # Extract links
+            page_links = []
+            for i, link_element in enumerate(link_elements):
+                try:
+                    # First try to get href directly from the element
+                    href = await link_element.get_attribute("href")
+                    
+                    # If no href found, try to find a parent link element
+                    if not href:
+                        parent_link = await link_element.query_selector("a")
+                        if parent_link:
+                            href = await parent_link.get_attribute("href")
+                    
+                    # If still no href, try to find a parent element with href
+                    if not href:
+                        try:
+                            # Try to find a parent link element
+                            parent_link = await link_element.evaluate("""
+                                (element) => {
+                                    let current = element;
+                                    for (let i = 0; i < 5; i++) {
+                                        if (current.tagName === 'A' && current.href) {
+                                            return current.href;
+                                        }
+                                        current = current.parentElement;
+                                        if (!current) break;
+                                    }
+                                    return null;
+                                }
+                            """)
+                            if parent_link:
+                                href = parent_link
+                        except Exception as e:
+                            logger.debug(f"Could not find parent link: {e}")
+                    
+                    if href:
+                        absolute_url = convert_to_absolute_url(href, page.url)
+                        page_links.append(absolute_url)
+                    else:
+                        logger.warning(f"⚠️ No href found for element {i}")
+                except Exception as e:
+                    logger.warning(f"❌ Error extracting link {i}: {str(e)}")
+                    continue
+            
+            return page_links
+            
+    except Exception as e:
+        logger.error(f"❌ Error extracting links from current page: {str(e)}")
+        return []
+
+
+async def extract_all_articles_unified(page, article_links: List[str], config: dict, website_type: str = None, custom_keywords: str = "", start_date: str = None, end_date: str = None) -> List[dict]:
+    """
+    Unified function to extract content from all articles
+    Limited by MAX_ARTICLE_LIMIT if set
+    """
+    logger.info(f"📚 Starting article extraction for {len(article_links)} articles")
+    logger.debug(f"🔧 Website type: {website_type}, Article limit: {MAX_ARTICLE_LIMIT}")
+    
+    all_articles = []
+    
+    # Apply article limit if set
+    if MAX_ARTICLE_LIMIT is not None:
+        if len(article_links) > MAX_ARTICLE_LIMIT:
+            logger.info(f"📊 Limiting to first {MAX_ARTICLE_LIMIT} articles out of {len(article_links)} total")
+            article_links = article_links[:MAX_ARTICLE_LIMIT]
+    
+    logger.info(f"🎯 Processing {len(article_links)} articles")
+    
+    for i, link in enumerate(article_links):
+        if scraping_cancelled():
+            logger.info("🛑 Scraping cancelled, stopping article extraction")
+            break
+            
+        logger.info(f"📰 Processing article {i+1}/{len(article_links)}: {link}")
+        
+        try:
+            # Add timeout to prevent hanging with retry mechanism
+            import asyncio
+            
+            # Try with shorter timeout first
+            try:
+                article_data = await asyncio.wait_for(
+                    extract_article_content_unified(page, link, config, website_type, custom_keywords, start_date, end_date),
+                    timeout=60  # 1 minute timeout per article
+                )
+                if article_data is not None:  # Only append if content was extracted and matched keywords/date
+                    all_articles.append(article_data)
+                else:
+                    logger.info(f"📄 Skipped article {i+1} (no content, no keyword match, or date out of range): {link}")
+            except asyncio.TimeoutError:
+                logger.warning(f"First attempt timeout for article {i+1}, retrying with shorter timeout...")
+                # Retry with even shorter timeout
+                try:
+                    article_data = await asyncio.wait_for(
+                        extract_article_content_unified(page, link, config, website_type, custom_keywords, start_date, end_date),
+                        timeout=30  # 30 seconds timeout for retry
+                    )
+                    if article_data is not None:  # Only append if content was extracted and matched keywords/date
+                        all_articles.append(article_data)
+                    else:
+                        logger.info(f"📄 Skipped article {i+1} (no content, no keyword match, or date out of range): {link}")
+                except asyncio.TimeoutError:
+                    logger.error(f"Timeout extracting article {i+1} after retry: {link}")
+                    all_articles.append({
+                        "title": f"Timeout extracting article {i+1}",
+                        "content": f"Article extraction timed out after multiple attempts: {link}",
+                        "date": datetime.now().strftime("%Y-%m-%d"),
+                        "url": link
+                    })
+            except Exception as e:
+                logger.error(f"Error extracting article {i+1}: {str(e)}")
+                all_articles.append({
+                    "title": f"Error extracting article {i+1}",
+                    "content": f"Error extracting article: {str(e)}",
+                    "date": datetime.now().strftime("%Y-%m-%d"),
+                    "url": link
+                })
+        except Exception as e:
+            logger.error(f"Unexpected error processing article {i+1}: {str(e)}")
+            all_articles.append({
+                "title": f"Error processing article {i+1}",
+                "content": f"Unexpected error: {str(e)}",
+                "date": datetime.now().strftime("%Y-%m-%d"),
+                "url": link
+            })
+    
+    return all_articles
+
+async def extract_article_content_unified(page, article_url: str, config: dict, website_type: str = None, custom_keywords: str = "", start_date: str = None, end_date: str = None) -> dict:
+    """
+    Unified function to extract content from a single article (text-focused)
+    With 5 retry attempts for loading articles
+    """
+    try:
+        max_retries = 5
+        retry_count = 0
+        
+        while retry_count < max_retries:
+            try:
+                retry_count += 1
+                logger.info(f"🔄 Loading article (attempt {retry_count}/{max_retries}): {article_url}")
+                
+                # Navigate to article with different strategies
+                if retry_count == 1:
+                    # First attempt: Use domcontentloaded for faster loading
+                    await page.goto(article_url, wait_until="domcontentloaded", timeout=30000)
+                elif retry_count == 2:
+                    # Second attempt: Use basic loading with shorter timeout
+                    await page.goto(article_url, timeout=20000)
+                elif retry_count == 3:
+                    # Third attempt: Use networkidle with even shorter timeout
+                    await page.goto(article_url, wait_until="networkidle", timeout=15000)
+                else:
+                    # Fourth and fifth attempts: Try with shorter timeouts
+                    await page.goto(article_url, timeout=10000)
+                
+                logger.info(f"✅ Successfully loaded article on attempt {retry_count}")
+                break  # Success, exit retry loop
+                
+            except Exception as e:
+                logger.warning(f"⚠️ Attempt {retry_count} failed for {article_url}: {str(e)}")
+                
+                if retry_count >= max_retries:
+                    logger.error(f"❌ Failed to load article after {max_retries} attempts: {article_url}")
+                    return {
+                        "title": "Network Error",
+                        "content": f"Failed to access article after {max_retries} attempts: {str(e)}",
+                        "date": datetime.now().strftime("%Y-%m-%d"),
+                        "url": article_url
+                    }
+                
+                # Wait before retry
+                import asyncio
+                await asyncio.sleep(2)  # Wait 2 seconds before retry
+        
+        # Extract title
+        title = ""
+        try:
+            title_element = await page.query_selector(config.get("title"))
+            if title_element:
+                title = await title_element.text_content()
+                if title:
+                    title = title.strip()
+        except Exception as e:
+            logger.warning(f"Error extracting title: {str(e)}")
+            title = ""
+        
+        # Use the passed website_type or try to determine it from config
+        if website_type is None:
+            for site_type, site_config in WEBSITE_CONFIG.items():
+                if site_config == config:
+                    website_type = site_type
+                    break
+            if website_type is None:
+                website_type = "unknown"
+        
+        content = ""
+        
+        # Extract content based on website type
+        if website_type == "hiiraan":
+            # Special handling for hiiraan.com
+            content_selector = config.get("content")
+            try:
+                # Get the content directly from the span
+                content_element = await page.query_selector(content_selector)
+                if content_element:
+                    # Get inner HTML and clean it up
+                    html_content = await content_element.inner_html()
+                    
+                    # Remove script tags and their contents
+                    html_content = re.sub(r'<script.*?</script>', '', html_content, flags=re.DOTALL)
+                    
+                    # Remove ads
+                    html_content = re.sub(r'<div class="inline-ad">.*?</div>', '', html_content, flags=re.DOTALL)
+                    
+                    # Extract text from HTML
+                    content = re.sub(r'<.*?>', ' ', html_content)
+                    content = re.sub(r'\s+', ' ', content).strip()
+            except Exception as e:
+                logger.warning(f"Error extracting hiiraan content: {str(e)}")
+                content = ""
+        else:
+            # Regular content extraction
+            content_selector = config.get("content")
+            content = ""
+            try:
+                content_elements = await page.query_selector_all(content_selector)
+                content_parts = []
+                for element in content_elements:
+                    text = await element.text_content()
+                    if text:
+                        content_parts.append(text.strip())
+                content = "\n\n".join(content_parts)
+            except Exception as e:
+                logger.warning(f"Error extracting content: {str(e)}")
+                content = ""
+        
+        # Extract date using configuration selector
+        date_raw = ""
+        
+        # For MOPND, use the date extracted from the main page
+        if website_type == "mopnd" and article_url in mopnd_article_dates:
+            date_raw = mopnd_article_dates[article_url]
+            logger.debug(f"✅ Using MOPND date from main page: {date_raw}")
+        else:
+            # Regular date extraction for other websites
+            date_selector = config.get("date")
+            
+            if date_selector:
+                try:
+                    date_element = await page.query_selector(date_selector)
+                    if date_element:
+                        date_raw = await date_element.text_content()
+                        if date_raw:
+                            date_raw = date_raw.strip()
+                            logger.debug(f"✅ Extracted raw date: {date_raw}")
+                except Exception as e:
+                    logger.warning(f"Error extracting date with selector {date_selector}: {str(e)}")
+        
+        # Standardize the date to YYYY-MM-DD format
+        date = standardize_date(date_raw, default_to_current=True)
+        if not date:
+            date = datetime.now().strftime("%Y-%m-%d")
+            logger.info(f"No date found with config selector, using current date: {date}")
+        
+        # Check date range filtering
+        from date_filter import parse_date_input
+        start_dt = parse_date_input(start_date) if start_date else None
+        end_dt = parse_date_input(end_date) if end_date else None
+        
+        if start_dt is not None or end_dt is not None:
+            if not is_date_in_range(date, start_dt, end_dt, include_missing=False):
+                logger.info(f"📅 Article date {date} is outside date range [{start_date}, {end_date}] - filtering out")
+                return None
+        
+        # Check for keyword matching and category assignment
+        combined_text = f"{title} {content}".strip()
+        category = get_category_for_text(combined_text, custom_keywords)
+        
+        if category is None:
+            logger.info("📂 Article did not match any keyword categories - filtering out")
+            return None
+        elif category:
+            logger.info(f"📂 Article categorized as: {category}")
+        else:
+            logger.info("📂 Article kept with empty category")
+        
+        result = {
+            "title": title or "No title found",
+            "content": content or "No content found",
+            "date": date,
+            "url": article_url,
+            "category": category
+        }
+        
+        logger.info(f"📊 Article result: title='{result['title'][:50]}...', category='{category}'")
+        return result
+        
+    except Exception as e:
+        logger.error(f"Error extracting content from {article_url}: {str(e)}")
+        return {
+            "title": "Error",
+            "content": f"Error extracting content: {str(e)}",
+            "date": datetime.now().strftime("%Y-%m-%d"),
+            "url": article_url
+        }

unified_pipeline.py

@@ -0,0 +1,651 @@
+#!/usr/bin/env python3
+"""
+Unified Processing Pipeline for News Dashboard
+Handles both text and document processing with a clean, cohesive interface
+"""
+
+import logging
+import pandas as pd
+from typing import List, Dict, Any, Optional, Tuple
+from datetime import datetime
+import os
+
+from scraper_common import scrape_news_async
+from document_processor import process_documents_from_url
+from data_preprocessor import DataPreprocessor
+from scraper_common import scraping_cancelled
+
+def determine_website_type(url: str) -> str:
+    """
+    Determine website type from URL based on domain patterns and URL paths
+    """
+    from urllib.parse import urlparse
+    
+    try:
+        parsed_url = urlparse(url)
+        domain = parsed_url.netloc.lower()
+        url_lower = url.lower()
+        
+        # Check for specific URL paths first (more specific matches)
+        if "frrims.faoswalim.org" in domain:
+            return "faoswalim_frrims_river_levels"
+        elif "faoswalim.org" in domain:
+            if "water/water-publications" in url_lower or "water-publications" in url_lower:
+                return "faoswalim_water_publications"
+            elif "flood-watch-bulletin" in url_lower or "ag-document-type/flood-watch-bulletin" in url_lower:
+                return "faoswalim_flood_watch"
+            elif "swalim-articles" in url_lower:
+                return "faoswalim_articles"
+            elif "swalim-events" in url_lower:
+                return "faoswalim_events"
+            elif "swalim-journals" in url_lower:
+                return "faoswalim_journals"
+            elif "swalim-publications" in url_lower:
+                return "faoswalim_publications"
+            else:
+                return "faoswalim"
+        elif "fsnau.org" in domain:
+            if "publications" in url_lower:
+                return "fsnau_publications"
+            else:
+                return "fsnau"
+        
+        # Check for ICPAC seasonal forecast path
+        if "icpac.net" in domain:
+            if "seasonal-forecast" in url_lower:
+                return "icpac_seasonal_forecast"
+            else:
+                return "icpac"
+        
+        # Map domains to website types
+        domain_mapping = {
+            'reliefweb.int': 'reliefweb',
+            'fscluster.org': 'fscluster',
+            'mopnd.govsomaliland.org': 'mopnd',
+            'nbs.gov.so': 'nbs',
+            'data.humdata.org': 'hdx',
+            'logcluster.org': 'logcluster',
+            'fews.net': 'fews',
+            'hiiraan.com': 'hiiraan',
+            'ocha.un.org': 'ocha',
+            'unocha.org': 'ocha',
+            'sodma.gov.so': 'sodma',
+            'atmis-au.org': 'atmis',
+            'garoweonline.com': 'garowe',
+            'goobjoog.com': 'goobjoog',
+            'radiodalsan.com': 'radiodalsan',
+            'radioergo.org': 'radioergo',
+            'drought.emergency.copernicus.eu': 'copernicus_drought'
+        }
+        
+        # Check for exact domain matches
+        for domain_pattern, website_type in domain_mapping.items():
+            if domain_pattern in domain:
+                return website_type
+        
+        # Default fallback
+        return 'unknown'
+        
+    except Exception as e:
+        logger.warning(f"Error determining website type from URL {url}: {str(e)}")
+        return 'unknown'
+
+# Try to import model processor, handle gracefully if not available
+try:
+    from model_processor import ModelProcessor
+    MODELS_AVAILABLE = True
+except ImportError as e:
+    print(f"Warning: Model processor not available: {e}")
+    print("AI features will be disabled. Install torch and transformers for full functionality.")
+    ModelProcessor = None
+    MODELS_AVAILABLE = False
+
+# Configure detailed logging
+import sys
+
+logging.basicConfig(
+    level=logging.DEBUG,
+    format='%(asctime)s - %(name)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s',
+    handlers=[
+        logging.StreamHandler(sys.stdout)
+    ]
+)
+logger = logging.getLogger(__name__)
+
+class UnifiedPipeline:
+    """
+    Unified pipeline for processing both text and document content
+    """
+    
+    def __init__(self, device: str = "auto"):
+        """
+        Initialize the unified pipeline
+        
+        Args:
+            device: Device to run models on
+        """
+        self.device = device
+        self.data_preprocessor = None
+        self.model_processor = None
+        self.initialized = False
+        
+        # Processing statistics
+        self.stats = {
+            'total_processed': 0,
+            'preprocessing_success': 0,
+            'model_processing_success': 0,
+            'final_success': 0,
+            'errors': []
+        }
+    
+    def initialize(self) -> bool:
+        """
+        Initialize all processors
+        
+        Returns:
+            True if all processors initialized successfully
+        """
+        logger.info("🚀 Starting UnifiedPipeline initialization")
+        
+        if self.initialized:
+            logger.info("✅ Pipeline already initialized, skipping")
+            return True
+        
+        try:
+            # Initialize data preprocessor
+            logger.info("🔧 Initializing data preprocessor...")
+            logger.debug(f"📋 Device configuration: {self.device}")
+            self.data_preprocessor = DataPreprocessor()
+            
+            # Initialize model processor (if available)
+            if MODELS_AVAILABLE and ModelProcessor is not None:
+                logger.info("🤖 Initializing model processor...")
+                logger.debug(f"🔧 Model processor device: {self.device}")
+                self.model_processor = ModelProcessor(device=self.device)
+                if not self.model_processor.load_models():
+                    logger.warning("⚠️ Model processor failed to load, continuing without AI features")
+                    self.model_processor = None
+                else:
+                    logger.info("✅ Model processor loaded successfully")
+            else:
+                logger.warning("⚠️ Model processor not available, continuing without AI features")
+                self.model_processor = None
+            
+            self.initialized = True
+            logger.info("✅ Pipeline initialization completed successfully")
+            logger.debug(f"📊 Initialization stats: {self.stats}")
+            return True
+            
+        except Exception as e:
+            logger.error(f"Error initializing pipeline: {str(e)}")
+            self.stats['errors'].append(f"Initialization error: {str(e)}")
+            return False
+    
+    async def process_text_content(self, url: str, custom_keywords: str = "", start_date: str = None, end_date: str = None) -> Tuple[pd.DataFrame, List[Dict[str, Any]]]:
+        """
+        Process text content from URL through the complete pipeline
+        
+        Args:
+            url: URL to scrape content from
+            custom_keywords: Custom keywords for filtering (comma-separated)
+            
+        Returns:
+            Tuple of (DataFrame, full_content_data)
+        """
+        try:
+            logger.info(f"🚀 Starting text content processing for URL: {url}")
+            logger.debug(f"📋 Processing parameters: URL={url}")
+            
+            # Check for cancellation before starting
+            if scraping_cancelled():
+                logger.warning("⚠️ Processing cancelled before starting")
+                return pd.DataFrame(), []
+            
+            # Step 1: Scrape content
+            logger.info("📡 Step 1: Scraping content...")
+            logger.debug("🔍 Initiating website scraping...")
+            
+            # Determine website type from URL
+            website_type = determine_website_type(url)
+            logger.debug(f"🌐 Detected website type: {website_type}")
+            
+            # Force text scraper mode when called from process_text_content
+            scraped_articles = await scrape_news_async(url, website_type, custom_keywords, start_date, end_date, force_mode="text")
+            
+            # Check for cancellation after scraping
+            if scraping_cancelled():
+                logger.warning("⚠️ Processing cancelled after scraping")
+                return pd.DataFrame(), []
+            
+            if not scraped_articles:
+                logger.warning("⚠️ No articles found to process")
+                return pd.DataFrame(), []
+            
+            # Check for special error indicators
+            if len(scraped_articles) == 1:
+                first_article = scraped_articles[0]
+                if first_article.get("title") == "WEBSITE_LOAD_ERROR":
+                    error_msg = first_article.get("content", "Website is not working. Please try again later.")
+                    logger.error(f"❌ {error_msg}")
+                    raise Exception(error_msg)
+                elif first_article.get("title") == "CAPTCHA_ERROR":
+                    error_msg = first_article.get("content", "Captcha detected. Please try again later.")
+                    logger.error(f"❌ {error_msg}")
+                    raise Exception(error_msg)
+            
+            logger.info(f"✅ Scraped {len(scraped_articles)} articles")
+            logger.debug(f"📊 Article details: {[article.get('title', 'No title') for article in scraped_articles]}")
+            
+            # Step 2: Preprocessing
+            logger.info("Step 2: Preprocessing content...")
+            if not self.initialize():
+                logger.warning("Pipeline initialization failed, using raw data")
+                preprocessed_articles = scraped_articles
+            else:
+                preprocessed_articles = self.data_preprocessor.preprocess_all_data(scraped_articles)
+                self.stats['preprocessing_success'] = len(preprocessed_articles)
+                logger.info(f"Preprocessing completed: {len(preprocessed_articles)} articles processed")
+            
+            # Check for cancellation after preprocessing
+            if scraping_cancelled():
+                logger.warning("⚠️ Processing cancelled after preprocessing")
+                return pd.DataFrame(), []
+            
+            # Step 3: Model processing and DataFrame creation
+            logger.info("Step 3: Processing with AI models and creating DataFrame...")
+            df_data = []
+            full_content_data = []
+            
+            for i, article in enumerate(preprocessed_articles, 1):
+                # Check for cancellation during processing
+                if scraping_cancelled():
+                    logger.warning("⚠️ Processing cancelled during model processing")
+                    return pd.DataFrame(), []
+                
+                # Extract content based on preprocessing status
+                content_info = self._extract_content_info(article, is_preprocessed=self.initialized)
+                
+                # Process with AI models if available
+                summary, summary_somali = self._process_with_models(content_info['content'])
+                
+                # Create DataFrame row
+                row_data = {
+                    '#': str(i),
+                    'title': content_info['title'],
+                    'category': content_info.get('category', ''),
+                    'content': content_info['content'],
+                    'summary': summary,
+                    'summary_somali': summary_somali,
+                    'date': content_info['date'],
+                    'url': content_info['url']
+                }
+                logger.debug(f"DataFrame row data: {row_data}")
+                df_data.append(row_data)
+                
+                # Store full content for modal
+                full_content_data.append({
+                    'title': content_info['title'],
+                    'content': content_info['content'],
+                    'date': content_info['date'],
+                    'url': content_info['url']
+                })
+            
+            df = pd.DataFrame(df_data)
+            self.stats['total_processed'] = len(df_data)
+            self.stats['final_success'] = len(df_data)
+            
+            logger.info(f"Text content processing completed: {len(df_data)} items processed")
+            logger.info(f"DataFrame columns: {list(df.columns)}")
+            logger.info(f"DataFrame shape: {df.shape}")
+            if not df.empty:
+                logger.info(f"Sample DataFrame row: {df.iloc[0].to_dict()}")
+            
+            return df, full_content_data
+            
+        except Exception as e:
+            logger.error(f"Error in text content processing: {str(e)}")
+            self.stats['errors'].append(f"Text processing error: {str(e)}")
+            return pd.DataFrame([{
+                '#': '1',
+                'title': f'Error: {str(e)}',
+                'content': '',
+                'summary': '',
+                'summary_somali': '',
+                'date': '',
+                'url': url
+            }]), []
+    
+    async def process_document_content(self, url: str, start_date: str = None, end_date: str = None) -> pd.DataFrame:
+        """
+        Process document content from URL through the complete pipeline
+        
+        Args:
+            url: URL to process documents from
+            
+        Returns:
+            DataFrame with processed document content
+        """
+        try:
+            logger.info(f"Starting document content processing for URL: {url}")
+            
+            # Check for cancellation before starting
+            if scraping_cancelled():
+                logger.warning("⚠️ Document processing cancelled before starting")
+                return pd.DataFrame()
+            
+            # Step 1: Extract documents
+            logger.info("Step 1: Extracting documents...")
+            documents_data = await process_documents_from_url(url.strip())
+            
+            # Check for cancellation after document extraction
+            if scraping_cancelled():
+                logger.warning("⚠️ Document processing cancelled after extraction")
+                return pd.DataFrame()
+            
+            if not documents_data:
+                logger.warning("No documents found to process")
+                return pd.DataFrame()
+            
+            # Check for special error indicators
+            if len(documents_data) == 1:
+                first_doc = documents_data[0]
+                if first_doc.get("title") == "WEBSITE_LOAD_ERROR":
+                    error_msg = first_doc.get("content", "Website is not working. Please try again later.")
+                    logger.error(f"❌ {error_msg}")
+                    raise Exception(error_msg)
+                elif first_doc.get("title") == "CAPTCHA_ERROR":
+                    error_msg = first_doc.get("content", "Captcha detected. Please try again later.")
+                    logger.error(f"❌ {error_msg}")
+                    raise Exception(error_msg)
+            
+            logger.info(f"Extracted {len(documents_data)} documents")
+            
+            # Step 2: Preprocessing
+            logger.info("Step 2: Preprocessing documents...")
+            if not self.initialize():
+                logger.warning("Pipeline initialization failed, using raw data")
+                preprocessed_docs = documents_data
+            else:
+                preprocessed_docs = self.data_preprocessor.preprocess_all_data(documents_data)
+                self.stats['preprocessing_success'] = len(preprocessed_docs)
+                logger.info(f"Preprocessing completed: {len(preprocessed_docs)} documents processed")
+            
+            # Check for cancellation after preprocessing
+            if scraping_cancelled():
+                logger.warning("⚠️ Document processing cancelled after preprocessing")
+                return pd.DataFrame()
+            
+            # Step 3: Model processing and DataFrame creation
+            logger.info("Step 3: Processing with AI models and creating DataFrame...")
+            df_data = []
+            
+            # Apply date filtering if provided (backup filter)
+            from date_filter import is_date_in_range, parse_date_input
+            start_dt = parse_date_input(start_date) if start_date else None
+            end_dt = parse_date_input(end_date) if end_date else None
+            
+            for doc in preprocessed_docs:
+                # Check for cancellation during processing
+                if scraping_cancelled():
+                    logger.warning("⚠️ Document processing cancelled during model processing")
+                    return pd.DataFrame()
+                
+                # Extract content based on preprocessing status
+                content_info = self._extract_document_info(doc, is_preprocessed=self.initialized)
+                
+                # Apply date filtering (backup filter in case dates weren't filtered at scraper level)
+                if start_dt is not None or end_dt is not None:
+                    doc_date = content_info.get('date', '')
+                    if not is_date_in_range(doc_date, start_dt, end_dt, include_missing=False):
+                        logger.debug(f"📅 Document date {doc_date} is outside date range - filtering out in pipeline")
+                        continue
+                
+                # Skip summary generation for CSV and PNG files
+                file_type = content_info.get('file_type', '').upper()
+                if file_type == 'CSV' or file_type == 'PNG':
+                    summary = ""
+                    summary_somali = ""
+                    logger.debug(f"⏭️ Skipping summary generation for {file_type} file: {content_info.get('title', 'Unknown')}")
+                else:
+                    # Process with AI models if available
+                    summary, summary_somali = self._process_with_models(content_info['extracted_text'])
+                
+                # Create DataFrame row
+                df_data.append({
+                    'title': content_info['title'],
+                    'date': content_info['date'],
+                    'source': content_info['source'],
+                    'file_path': content_info['file_path'],
+                    'extracted_text': content_info['extracted_text'],
+                    'summary': summary,
+                    'summary_somali': summary_somali,
+                    'file_type': content_info['file_type']
+                })
+            
+            df = pd.DataFrame(df_data)
+            self.stats['total_processed'] = len(df_data)
+            self.stats['final_success'] = len(df_data)
+            
+            logger.info(f"Document content processing completed: {len(df_data)} items processed")
+            return df
+            
+        except Exception as e:
+            logger.error(f"Error in document content processing: {str(e)}")
+            self.stats['errors'].append(f"Document processing error: {str(e)}")
+            return pd.DataFrame([{
+                'title': f'Error: {str(e)}',
+                'date': '',
+                'source': '',
+                'file_path': '',
+                'extracted_text': '',
+                'summary': '',
+                'summary_somali': '',
+                'file_type': ''
+            }])
+    
+    def _extract_content_info(self, article: Dict[str, Any], is_preprocessed: bool) -> Dict[str, str]:
+        """
+        Extract content information from article
+        
+        Args:
+            article: Article data
+            is_preprocessed: Whether the article has been preprocessed
+            
+        Returns:
+            Dictionary with content information
+        """
+        if is_preprocessed and isinstance(article, dict) and 'content' in article:
+            # Use preprocessed content
+            content_data = article.get('content', {})
+            if isinstance(content_data, dict):
+                return {
+                    'title': article.get('source_metadata', {}).get('title', ''),
+                    'content': content_data.get('cleaned_text', ''),
+                    'date': article.get('source_metadata', {}).get('date', ''),
+                    'url': article.get('source_metadata', {}).get('url', ''),
+                    'category': article.get('source_metadata', {}).get('category', '')
+                }
+        
+        # Fallback to original structure
+        result = {
+            'title': article.get('title', ''),
+            'content': article.get('content', ''),
+            'date': article.get('date', ''),
+            'url': article.get('url', ''),
+            'category': article.get('category', '')
+        }
+        logger.debug(f"Extracted content info: {result}")
+        return result
+    
+    def _extract_document_info(self, doc: Dict[str, Any], is_preprocessed: bool) -> Dict[str, str]:
+        """
+        Extract document information
+        
+        Args:
+            doc: Document data
+            is_preprocessed: Whether the document has been preprocessed
+            
+        Returns:
+            Dictionary with document information
+        """
+        if is_preprocessed and isinstance(doc, dict) and 'content' in doc:
+            # Use preprocessed content
+            content_data = doc.get('content', {})
+            source_metadata = doc.get('source_metadata', {})
+            if isinstance(content_data, dict):
+                # Use 'source' field from source_metadata if available, otherwise fall back to source_website
+                # If source_website is available but source is not, try to map it
+                source = source_metadata.get('source', '')
+                if not source:
+                    source_website = source_metadata.get('source_website', '')
+                    if source_website and source_website != 'unknown':
+                        # Map source_website to proper name
+                        from data_preprocessor import DataPreprocessor
+                        preprocessor = DataPreprocessor()
+                        source = preprocessor._map_source_website_to_name(source_website)
+                    else:
+                        # Last resort: try to get source from URL
+                        url = source_metadata.get('url', '') or source_metadata.get('pdf_path', '')
+                        if url:
+                            try:
+                                from utils import get_source_from_url
+                                source = get_source_from_url(url)
+                            except:
+                                source = 'Unknown'
+                        else:
+                            source = 'Unknown'
+                
+                return {
+                    'title': source_metadata.get('title', ''),
+                    'extracted_text': content_data.get('cleaned_text', ''),
+                    'date': source_metadata.get('date', ''),
+                    'source': source,
+                    'file_path': source_metadata.get('pdf_path', ''),
+                    'file_type': source_metadata.get('file_type', '') or source_metadata.get('content_type', '')
+                }
+        
+        # Fallback to original structure
+        source = doc.get('source', '')
+        if not source:
+            # Try to get source from URL if available
+            url = doc.get('url', '') or doc.get('file_path', '') or doc.get('pdf_path', '')
+            if url:
+                try:
+                    from utils import get_source_from_url
+                    source = get_source_from_url(url)
+                except:
+                    source = 'Unknown'
+            else:
+                source = 'Unknown'
+        
+        return {
+            'title': doc.get('title', ''),
+            'extracted_text': doc.get('extracted_text', ''),
+            'date': doc.get('date', ''),
+            'source': source,
+            'file_path': doc.get('pdf_path', '') or doc.get('local_path', ''),
+            'file_type': doc.get('file_type', '')
+        }
+    
+    def _process_with_models(self, content: str) -> Tuple[str, str]:
+        """
+        Process content with AI models
+        
+        Args:
+            content: Text content to process
+            
+        Returns:
+            Tuple of (summary, summary_somali)
+        """
+        if not self.model_processor or not content.strip():
+            return "", ""
+        
+        try:
+            model_results = self.model_processor.process_content(content)
+            if model_results.get('processing_success', False):
+                self.stats['model_processing_success'] += 1
+                return model_results.get('summary', ''), model_results.get('summary_somali', '')
+        except Exception as e:
+            logger.error(f"Error in model processing: {str(e)}")
+            self.stats['errors'].append(f"Model processing error: {str(e)}")
+        
+        return "", ""
+    
+    def get_stats(self) -> Dict[str, Any]:
+        """
+        Get processing statistics
+        
+        Returns:
+            Dictionary with processing statistics
+        """
+        return {
+            'pipeline_stats': self.stats.copy(),
+            'preprocessing_stats': self.data_preprocessor.get_processing_stats() if self.data_preprocessor else {},
+            'model_info': self.model_processor.get_model_info() if self.model_processor else {}
+        }
+    
+    def reset_stats(self):
+        """Reset processing statistics"""
+        self.stats = {
+            'total_processed': 0,
+            'preprocessing_success': 0,
+            'model_processing_success': 0,
+            'final_success': 0,
+            'errors': []
+        }
+
+
+# Global pipeline instance
+_pipeline = None
+
+def get_pipeline(device: str = "auto") -> UnifiedPipeline:
+    """
+    Get or create the global pipeline instance
+    
+    Args:
+        device: Device to run models on
+        
+    Returns:
+        UnifiedPipeline instance
+    """
+    global _pipeline
+    if _pipeline is None:
+        _pipeline = UnifiedPipeline(device=device)
+    return _pipeline
+
+def process_text_content(url: str, custom_keywords: str = "", start_date: str = None, end_date: str = None) -> Tuple[pd.DataFrame, List[Dict[str, Any]]]:
+    """
+    Convenience function to process text content
+    
+    Args:
+        url: URL to process
+        custom_keywords: Custom keywords for filtering (comma-separated)
+        
+    Returns:
+        Tuple of (DataFrame, full_content_data)
+    """
+    pipeline = get_pipeline()
+    return pipeline.process_text_content(url, custom_keywords, start_date, end_date)
+
+async def process_document_content(url: str, start_date: str = None, end_date: str = None) -> pd.DataFrame:
+    """
+    Convenience function to process document content
+    
+    Args:
+        url: URL to process
+        
+    Returns:
+        DataFrame with processed content
+    """
+    pipeline = get_pipeline()
+    return await pipeline.process_document_content(url)
+
+def get_processing_stats() -> Dict[str, Any]:
+    """
+    Get processing statistics
+    
+    Returns:
+        Dictionary with processing statistics
+    """
+    pipeline = get_pipeline()
+    return pipeline.get_stats()

users.json

@@ -0,0 +1,8 @@
+{
+  "admin": {
+    "password_hash": "2fb04132a771f442e105b12dfa7d1e5b:32d4fa6a6c4f499a7b3f1aabe66e860aebbe9f36ba7deb6bac7317afb90283ad",
+    "is_admin": true,
+    "created_at": "2025-10-15T14:07:15.699649",
+    "last_login": "2025-11-07T21:29:10.246156"
+  }
+}

utils.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""
+Utility functions for News Dashboard
+"""
+
+def get_source_from_url(url: str) -> str:
+    """
+    Determine source name from URL
+    
+    Args:
+        url: URL string
+        
+    Returns:
+        Source name string
+    """
+    if "unocha.org" in url:
+        return "OCHA"
+    elif "sodma.gov.so" in url:
+        return "SODMA"
+    elif "atmis-au.org" in url:
+        return "ATMIS"
+    elif "garoweonline.com" in url:
+        return "Garowe Online"
+    elif "goobjoog.com" in url:
+        return "Goobjoog"
+    elif "radiodalsan.com" in url:
+        print('Radio dalsan found')
+        return "Radio Dalsan"
+    elif "radioergo.org" in url:
+        return "Radio Ergo"
+    elif "hiiraan.com" in url:
+        return "Hiiraan"
+    elif "reliefweb.int" in url:
+        return "ReliefWeb"
+    elif "fscluster.org" in url:
+        return "FS Cluster"
+    elif "mopnd.govsomaliland.org" in url:
+        return "MOPND Somaliland"
+    elif "nbs.gov.so" in url:
+        return "NBS Somalia"
+    elif "data.humdata.org" in url:
+        return "HDX"
+    elif "logcluster.org" in url:
+        return "LogCluster"
+    elif "fsnau.org" in url:
+        if "fsnau.org/publications" in url:
+            return "FSNau Publications"
+        else:
+            return "FSNau"
+    elif "fews.net" in url:
+        return "FEWS NET"
+    elif "icpac.net" in url:
+        if "seasonal-forecast" in url.lower():
+            return "ICPAC - IGAD Climate Prediction and Applications Centre - Seasonal Forecast"
+        else:
+            return "ICPAC"
+    elif "frrims.faoswalim.org" in url:
+        return "FAO SWALIM FRRIMS River Levels"
+    elif "faoswalim.org" in url:
+        if "water/water-publications" in url or "water-publications" in url:
+            return "FAO SWALIM Water Publications"
+        elif "flood-watch-bulletin" in url or "ag-document-type/flood-watch-bulletin" in url:
+            return "FAO SWALIM Flood Watch"
+        elif "faoswalim.org/swalim-events" in url:
+            return "FAO SWALIM Events"
+        elif "faoswalim.org/swalim-journals" in url:
+            return "FAO SWALIM Journals"
+        elif "faoswalim.org/swalim-publications" in url:
+            return "FAO SWALIM Publications"
+        elif "faoswalim.org/swalim-articles" in url:
+            return "FAO SWALIM Articles"
+        else:
+            return "FAO SWALIM"
+    elif "drought.emergency.copernicus.eu" in url:
+        return "Copernicus Drought Observatory"
+    else:
+        return "Unknown"

website_config.json

@@ -0,0 +1,346 @@
+{
+    "ocha": {
+        "base_url": "https://www.unocha.org",
+        "article_links": ".cd-card__title a",
+        "title": ".cd-page-title",
+        "content": ".cd-layout-main-content",
+        "date": ".node__submitted",
+        "navigation_selector": ".cd-pager__item a",
+        "navigation_url_addition": "?page={page_no}",
+        "start_page": 0
+    },
+    "sodma": {
+        "base_url": "https://sodma.gov.so",
+        "article_links": ".entry-title a",
+        "title": ".et_pb_text_inner h1",
+        "content": ".et_pb_post_content_0_tb_body p",
+        "date": ".et_pb_blurb_1_tb_body span",
+        "navigation_selector": null,
+        "navigation_url_addition": null,
+        "start_page": 0
+    },
+    "atmis": {
+        "base_url": "https://atmis-au.org",
+        "article_links": ".grid-title a",
+        "title": ".entry-title",
+        "content": ".p1",
+        "date": ".post-box-meta-single .published",
+        "navigation_selector": ".penci-pagination",
+        "navigation_url_addition": "/page/{page_no}/",
+        "start_page": 1
+    },
+    "hiiraan": {
+        "base_url": "https://www.hiiraan.com",
+        "article_links": ".bullets a",
+        "title": "#desktopcontrol1_newsdesktop3_lbltitle",
+        "content": "#desktopcontrol1_newsdesktop3_lblcontent",
+        "date": "#desktopcontrol1_newsdesktop3_lblcontent p:first-child",
+        "navigation_selector": ".pages",
+        "navigation_url_addition": "/morenews-{page_no}.aspx",
+        "start_page": 1
+    },
+    "garowe": {
+        "base_url": "https://www.garoweonline.com",
+        "article_links": ".col-md-6 a",
+        "title": "#article-content h3",
+        "content": "#article-content p",
+        "date": ".entry-meta a",
+        "navigation_selector": ".page-link",
+        "navigation_url_addition": "?page={page_no}",
+        "start_page": 1
+    },
+    "goobjoog": {
+        "base_url": "https://goobjoog.com",
+        "article_links": ".hover\\:unde , #main .hover\\:underline",
+        "title": ".lg\\:leading-\\[1\\.15\\]",
+        "content": "p",
+        "date": "time",
+        "navigation_selector": ".nav-links",
+        "navigation_url_addition": "/page/{page_no}/",
+        "start_page": 1
+    },
+    "radiodalsan": {
+        "base_url": "https://radiodalsan.com",
+        "article_links": ".jeg_pl_lg_2 .jeg_post_title",
+        "title": ".entry-header .jeg_post_title",
+        "content": ".content-inner span",
+        "date": ".meta_left .jeg_meta_date a",
+        "navigation_selector": ".no_pageinfo",
+        "navigation_url_addition": "/page/{page_no}/",
+        "start_page": 1
+    },
+    "radioergo": {
+        "base_url": "https://radioergo.org",
+        "article_links": ".jeg_post_title a",
+        "title": ".entry-header .jeg_post_title",
+        "content": ".content-inner p",
+        "date": ".meta_left .jeg_meta_date a",
+        "navigation_selector": ".no_pageinfo",
+        "navigation_url_addition": "/page/{page_no}/",
+        "start_page": 1
+    },
+    "mopnd": {
+        "base_url": "https://mopnd.govsomaliland.org",
+        "page_links": ".post_info a",
+        "title": ".post_info a",
+        "content": null,
+        "date": ".post_info small",
+        "pdf_links": ".text-left",
+        "navigation_selector": "#yw0",
+        "navigation_url_addition": "?page={page_no}",
+        "start_page": 1
+    },
+    "fews": {
+        "base_url": "https://fews.net",
+        "page_links": ".animated-link",
+        "title": "#block-outline-frontend-page-title fews-heading",
+        "content": null,
+        "date": ".metadata-bottom-row .metadata-item:has(fews-icon[name=\"clock\"])",
+        "pdf_links": [
+            "fews-button[button-url*=\"/print\"]"
+        ],
+        "navigation_selector": ".pager__link",
+        "navigation_url_addition": "&page=1",
+        "start_page": 0
+    },
+    "icpac": {
+        "base_url": "https://icpac.net",
+        "page_links": ".read",
+        "title": ".document-detail-title",
+        "content": null,
+        "date": ".document-detail-header-meta-item:nth-child(1)",
+        "pdf_links": [
+            ".is-small:nth-child(1)"
+        ],
+        "navigation_selector": ".pagination a",
+        "navigation_url_addition": "?page={page_no}",
+        "start_page": 1
+    },
+    "icpac_seasonal_forecast": {
+        "base_url": "https://www.icpac.net/seasonal-forecast/",
+        "page_links": ".read",
+        "title": ".section-title",
+        "content": null,
+        "date": null,
+        "file_links": [
+            ".is-small:nth-child(1)"
+        ],
+        "navigation_selector": ".pagination a",
+        "navigation_url_addition": "?page={page_no}",
+        "start_page": 1
+    },
+    "copernicus_drought": {
+        "base_url": "https://drought.emergency.copernicus.eu",
+        "page_links": "#sortable a",
+        "title": "#item-title",
+        "content": null,
+        "date": ".ec-panel:nth-child(5)",
+        "pdf_links": [
+            ".dl2"
+        ],
+        "navigation_selector": null,
+        "navigation_url_addition": null,
+        "start_page": 1
+    },
+    "logcluster": {
+        "base_url": "https://logcluster.org",
+        "page_links": ".field--label-above a",
+        "title": "#block-pagetitle .field--label-hidden",
+        "content": null,
+        "date": ".datetime",
+        "pdf_links": [
+            ".btn-lg"
+        ],
+        "recaptcha_text": "Let's confirm you are human",
+        "navigation_selector": ".page-link",
+        "navigation_url_addition": "?page={page_no}",
+        "start_page": 0
+    },
+    "fscluster": {
+        "base_url": "https://fscluster.org",
+        "page_links": ".teaser-document__link",
+        "title": ".file--application-pdf a",
+        "content": null,
+        "date": ".table-content-teaser tr:nth-child(1) td+ td",
+        "pdf_links": [
+            ".file--application-pdf a"
+        ],
+        "navigation_selector": ".pager__link",
+        "navigation_url_addition": "?page={page_no}",
+        "start_page": 0
+    },
+    "nbs": {
+        "base_url": "https://nbs.gov.so",
+        "page_links": null,
+        "title": ".entry-title a",
+        "content": null,
+        "pdf_links": [
+            ".wp-block-button__link"
+        ],
+        "navigation_selector": ".page-numbers",
+        "navigation_url_addition": "/page/{page_no}/",
+        "start_page": 1
+    },
+    "faoswalim_publications": {
+        "base_url": "https://faoswalim.org",
+        "page_links": "h2 a",
+        "title": "h2",
+        "content": null,
+        "date": ".date-display-single",
+        "pdf_links": [
+            ".file a"
+        ],
+        "navigation_selector": ".pager-item a",
+        "navigation_url_addition": "?page={page_no}",
+        "start_page": 0
+    },
+    "faoswalim_flood_watch": {
+        "base_url": "https://faoswalim.org",
+        "page_links": "h2 a",
+        "title": "h2",
+        "content": null,
+        "date": ".date-display-single",
+        "pdf_links": [
+            "#main-body a"
+        ],
+        "navigation_selector": ".pager-item a",
+        "navigation_url_addition": "?page={page_no}",
+        "start_page": 0
+    },
+    "faoswalim_water_publications": {
+        "base_url": "https://faoswalim.org",
+        "page_links": ".field-content a",
+        "title": "h2",
+        "content": null,
+        "date": ".date-display-single",
+        "pdf_links": [
+            "#main-body a"
+        ],
+        "navigation_selector": null,
+        "navigation_url_addition": null,
+        "start_page": 1
+    },
+    "faoswalim_articles": {
+        "base_url": "https://faoswalim.org",
+        "article_links": ".media-heading a",
+        "page_links": ".media-heading a",
+        "title": "h2",
+        "content": "p",
+        "pdf_links": [
+            "#main-body a"
+        ],
+        "navigation_selector": ".pager-item a",
+        "navigation_url_addition": "?page={page_no}",
+        "start_page": 0
+    },
+    "fsnau": {
+        "base_url": "https://fsnau.org",
+        "page_links": null,
+        "title": "FSNau Document",
+        "content": "File Content",
+        "file_links": [
+            "p:nth-child(5) a , p:nth-child(4)"
+        ],
+        "navigation_selector": null,
+        "navigation_url_addition": null,
+        "start_page": 1
+    },
+    "hdx": {
+        "base_url": "https://data.humdata.org",
+        "page_links": null,
+        "title": "HDX Document",
+        "content": "File Content",
+        "date": ".update-date",
+        "file_links": [
+            ".resource-download-button"
+        ],
+        "navigation_selector": null,
+        "navigation_url_addition": null,
+        "start_page": 1
+    },
+    "faoswalim_frrims_river_levels": {
+        "base_url": "https://frrims.faoswalim.org",
+        "page_links": null,
+        "title": null,
+        "content": "td, th",
+        "date": null,
+        "pdf_links": null,
+        "file_links": null,
+        "navigation_selector": null,
+        "navigation_url_addition": null,
+        "start_page": 1,
+        "extract_table_as_csv": true
+    },
+    "faoswalim": {
+        "not useful from here": true,
+        "base_url": "https://faoswalim.org",
+        "page_links": null,
+        "title": "FAO SWALIM Document",
+        "content": "PDF Content",
+        "pdf_links": [
+            "a[href$='.pdf']"
+        ],
+        "navigation_selector": null,
+        "navigation_url_addition": null,
+        "start_page": 1
+    },
+    "faoswalim_journals": {
+        "base_url": "https://faoswalim.org",
+        "page_links": ".field-content a",
+        "title": "h2",
+        "content": "#main-body .content",
+        "pdf_links": [
+            "a[href$='.pdf']",
+            "a[href*='pdf']",
+            "a[href*='document']",
+            "a[href*='attachment']",
+            "a[href*='download']",
+            "a[href*='journal']",
+            "a[href*='publication']",
+            ".file a"
+        ],
+        "navigation_selector": null,
+        "navigation_url_addition": null,
+        "start_page": 1
+    },
+    "faoswalim_events": {
+        "base_url": "https://faoswalim.org",
+        "page_links": "h2 a",
+        "title": "h2",
+        "date": ".submitted span",
+        "content": "#main-body .content div",
+        "pdf_links": [
+            "a[href$='.pdf']",
+            "a[href*='document']",
+            "a[href*='attachment']"
+        ],
+        "navigation_selector": null,
+        "navigation_url_addition": null,
+        "start_page": 1
+    },
+    "fsnau_publications": {
+        "base_url": "https://fsnau.org",
+        "page_links": null,
+        "title": "FSNau Publication",
+        "content": "PDF Content",
+        "pdf_links": [
+            "a[href$='.pdf']"
+        ],
+        "navigation_selector": null,
+        "navigation_url_addition": null,
+        "start_page": 1
+    },
+    "reliefweb": {
+        "base_url": "https://reliefweb.int",
+        "page_links": ".rw-river-article__title a",
+        "title": ".rw-entity-meta__header-title",
+        "content": ".rw-entity-meta__content",
+        "date": ".rw-article__header--with-meta .rw-entity-meta__tag-value--published time",
+        "pdf_links": [
+            ".rw-file__label"
+        ],
+        "navigation_selector": null,
+        "navigation_url_addition": null,
+        "start_page": 1
+    }
+}