Create scraper.py

scraper.py

@@ -0,0 +1,115 @@
+import requests
+from bs4 import BeautifulSoup
+import pandas as pd
+import time
+import random
+import json
+from typing import Dict, Any
+
+class FinancialScraper:
+    def __init__(self):
+        # Best Practice 1: Multiple User-Agents for rotation
+        self.user_agents = [
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0'
+        ]
+        
+        # Best Practice 2: Use session for better performance
+        self.session = requests.Session()
+        self.update_headers()
+        
+        # Best Practice 3: Request delays
+        self.min_delay = 1.0  # Minimum delay between requests
+        self.max_delay = 2.0  # Maximum delay between requests
+        self.last_request_time = 0
+    
+    def update_headers(self):
+        """Rotate user agent for each session"""
+        headers = {
+            'User-Agent': random.choice(self.user_agents),
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.5',
+            'Accept-Encoding': 'gzip, deflate',
+            'Connection': 'keep-alive',
+        }
+        self.session.headers.update(headers)
+    
+    def _rate_limit(self):
+        """Implement rate limiting between requests"""
+        current_time = time.time()
+        time_since_last = current_time - self.last_request_time
+        
+        if time_since_last < self.min_delay:
+            sleep_time = random.uniform(self.min_delay, self.max_delay)
+            time.sleep(sleep_time)
+        
+        self.last_request_time = time.time()
+    
+    def _make_request(self, url: str) -> requests.Response:
+        """Make a rate-limited request with error handling"""
+        self._rate_limit()
+        
+        try:
+            response = self.session.get(url, timeout=10)
+            response.raise_for_status()
+            return response
+        except requests.RequestException as e:
+            raise Exception(f"Request failed: {str(e)}")
+    
+    def scrape_yahoo_summary(self, symbol: str) -> Dict[str, Any]:
+        """Scrape basic financial data from Yahoo Finance"""
+        try:
+            url = f"https://finance.yahoo.com/quote/{symbol.upper()}"
+            response = self._make_request(url)
+            soup = BeautifulSoup(response.content, 'html.parser')
+            
+            data = {'symbol': symbol.upper()}
+            
+            # Extract current price
+            try:
+                price_elem = soup.find('fin-streamer', {'data-field': 'regularMarketPrice'})
+                if price_elem:
+                    data['Current Price'] = f"${price_elem.text}"
+            except:
+                data['Current Price'] = 'N/A'
+            
+            # Extract key statistics from summary table
+            summary_table = soup.find('div', {'data-test': 'left-summary-table'})
+            if summary_table:
+                rows = summary_table.find_all('tr')
+                for row in rows:
+                    cells = row.find_all('td')
+                    if len(cells) == 2:
+                        key = cells[0].text.strip()
+                        value = cells[1].text.strip()
+                        data[key] = value
+            
+            # Get company name
+            try:
+                company_elem = soup.find('h1', {'data-test': 'quote-header'})
+                if company_elem:
+                    company_name = company_elem.text.split('(')[0].strip()
+                    data['Company Name'] = company_name
+            except:
+                pass
+            
+            return data
+            
+        except Exception as e:
+            return {'symbol': symbol.upper(), 'error': f"Failed to scrape summary: {str(e)}"}
+    
+    def scrape_key_statistics(self, symbol: str) -> Dict[str, Any]:
+        """Scrape financial ratios and key statistics"""
+        try:
+            url = f"https://finance.yahoo.com/quote/{symbol.upper()}/key-statistics"
+            response = self._make_request(url)
+            soup = BeautifulSoup(response.content, 'html.parser')
+            
+            ratios = {'symbol': symbol.upper()}
+            
+            # Find all tables containing statistics
+            tables = soup.find_all('table')
+            
+            # Key metrics we want to extract