Upload 317 files

config.py

@@ -192,3 +192,8 @@ SUCCESS_MESSAGES = {
     "cache_cleared": "Cache cleared successfully.",
     "database_initialized": "Database initialized successfully.",
 }
+
+# Backward-compatible alias so that `from config import config`
+# returns the config module itself.
+import sys as _sys
+config = _sys.modules[__name__]

hf-data-engine/.dockerignore

@@ -4,6 +4,9 @@ __pycache__/
 *$py.class
 *.so
 .Python
+env/
+venv/
+ENV/
 build/
 develop-eggs/
 dist/
@@ -19,15 +22,10 @@ wheels/
 *.egg-info/
 .installed.cfg
 *.egg
-MANIFEST
-pip-log.txt
-pip-delete-this-directory.txt
 
-# Virtual environments
-venv/
-ENV/
-env/
-.venv
+# Environment
+.env
+.env.local
 
 # IDE
 .vscode/
@@ -35,87 +33,19 @@ env/
 *.swp
 *.swo
 *~
-.DS_Store
+
+# Tests
+.pytest_cache/
+.coverage
+htmlcov/
 
 # Git
 .git/
 .gitignore
-.gitattributes
 
 # Documentation
 *.md
 docs/
-README*.md
-CHANGELOG.md
-LICENSE
 
-# Testing
-.pytest_cache/
-.coverage
-htmlcov/
-.tox/
-.hypothesis/
-tests/
-test_*.py
-
-# Logs and databases (will be created in container)
+# Logs
 *.log
-logs/
-data/*.db
-data/*.sqlite
-data/*.db-journal
-
-# Environment files (should be set via docker-compose or HF Secrets)
-.env
-.env.*
-!.env.example
-
-# Docker
-docker-compose*.yml
-!docker-compose.yml
-Dockerfile
-.dockerignore
-
-# CI/CD
-.github/
-.gitlab-ci.yml
-.travis.yml
-azure-pipelines.yml
-
-# Temporary files
-*.tmp
-*.bak
-*.swp
-temp/
-tmp/
-
-# Node modules (if any)
-node_modules/
-package-lock.json
-yarn.lock
-
-# OS files
-Thumbs.db
-.DS_Store
-desktop.ini
-
-# Jupyter notebooks
-.ipynb_checkpoints/
-*.ipynb
-
-# Model cache (models will be downloaded in container)
-models/
-.cache/
-.huggingface/
-
-# Large files that shouldn't be in image
-*.tar
-*.tar.gz
-*.zip
-*.rar
-*.7z
-
-# Screenshots and assets not needed
-screenshots/
-assets/*.png
-assets/*.jpg

hf-data-engine/.env.example

@@ -1,17 +1,47 @@
-# HuggingFace Configuration
-HUGGINGFACE_TOKEN=your_token_here
+# Server Configuration
+HOST=0.0.0.0
+PORT=8000
+ENV=production
+VERSION=1.0.0
+
+# Cache Configuration
+CACHE_TYPE=memory
+CACHE_TTL_PRICES=30
+CACHE_TTL_OHLCV=300
+CACHE_TTL_SENTIMENT=600
+CACHE_TTL_MARKET=300
+
+# Redis (if using Redis cache)
+# REDIS_URL=redis://localhost:6379
+
+# Rate Limiting
+RATE_LIMIT_ENABLED=true
+RATE_LIMIT_PRICES=120
+RATE_LIMIT_OHLCV=60
+RATE_LIMIT_SENTIMENT=30
+RATE_LIMIT_HEALTH=0
+
+# Optional API Keys (for higher rate limits)
+# BINANCE_API_KEY=
+# BINANCE_API_SECRET=
+# COINGECKO_API_KEY=
+# CRYPTOCOMPARE_API_KEY=
+# CRYPTOPANIC_API_KEY=
+# NEWSAPI_KEY=
+
+# Features
 ENABLE_SENTIMENT=true
-SENTIMENT_SOCIAL_MODEL=ElKulako/cryptobert
-SENTIMENT_NEWS_MODEL=kk08/CryptoBERT
-HF_REGISTRY_REFRESH_SEC=21600
-HF_HTTP_TIMEOUT=8.0
-
-# Existing API Keys (if any)
-ETHERSCAN_KEY_1=
-ETHERSCAN_KEY_2=
-BSCSCAN_KEY=
-TRONSCAN_KEY=
-COINMARKETCAP_KEY_1=
-COINMARKETCAP_KEY_2=
-NEWSAPI_KEY=
-CRYPTOCOMPARE_KEY=
+ENABLE_NEWS=false
+
+# Circuit Breaker
+CIRCUIT_BREAKER_THRESHOLD=5
+CIRCUIT_BREAKER_TIMEOUT=60
+
+# Request Timeouts
+REQUEST_TIMEOUT=10
+
+# Supported Symbols (comma-separated)
+SUPPORTED_SYMBOLS=BTC,ETH,SOL,XRP,BNB,ADA,DOT,LINK,LTC,BCH,MATIC,AVAX,XLM,TRX
+
+# Supported Intervals (comma-separated)
+SUPPORTED_INTERVALS=1m,5m,15m,1h,4h,1d,1w

hf-data-engine/.gitignore

@@ -4,6 +4,9 @@ __pycache__/
 *$py.class
 *.so
 .Python
+env/
+venv/
+ENV/
 build/
 develop-eggs/
 dist/
@@ -20,29 +23,25 @@ wheels/
 .installed.cfg
 *.egg
 
-# Virtual environments
-venv/
-ENV/
-env/
+# Environment
+.env
+.env.local
 
 # IDE
 .vscode/
 .idea/
 *.swp
 *.swo
+*~
 
-# Data
-data/*.db
-data/*.db-journal
-data/exports/
-crypto_monitor.db
-crypto_monitor.db-journal
-
-# Environment
-.env
+# Tests
+.pytest_cache/
+.coverage
+htmlcov/
 
 # Logs
 *.log
+logs/
 
 # OS
 .DS_Store

hf-data-engine/Dockerfile

@@ -1,41 +1,20 @@
-# Use Python 3.11 Slim base image
 FROM python:3.11-slim
 
-# Set environment variables
-ENV PYTHONUNBUFFERED=1 \
-    PYTHONDONTWRITEBYTECODE=1 \
-    PIP_NO_CACHE_DIR=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    ENABLE_AUTO_DISCOVERY=false
-
-# Install system dependencies
-RUN apt-get update && apt-get install -y \
-    gcc \
-    g++ \
-    curl \
-    && rm -rf /var/lib/apt/lists/*
-
-# Set working directory
 WORKDIR /app
 
-# Copy dependency files
+# Install dependencies
 COPY requirements.txt .
-
-# Install Python dependencies
 RUN pip install --no-cache-dir -r requirements.txt
 
 # Copy application code
 COPY . .
 
-# Create necessary directories
-RUN mkdir -p logs data data/exports data/backups
-
-# Expose ports (Hugging Face uses PORT env variable, default 7860)
-EXPOSE 7860 8000
+# Expose port
+EXPOSE 8000
 
-# Health check (simplified to avoid requests dependency in healthcheck)
-HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \
-    CMD curl -f http://localhost:${PORT:-8000}/health || exit 1
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
+  CMD python -c "import httpx; httpx.get('http://localhost:8000/api/health', timeout=5)"
 
-# Run server with uvicorn (supports Hugging Face PORT env variable)
-CMD ["sh", "-c", "uvicorn api_server_extended:app --host 0.0.0.0 --port ${PORT:-8000}"]
+# Run the application
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]

hf-data-engine/main.py

@@ -1,30 +1,326 @@
-#!/usr/bin/env python3
-"""
-Local Development Entry Point
-This file is for local development only and is NOT used by Hugging Face Docker runtime.
-For production deployment, use: uvicorn api_server_extended:app
-"""
+"""HuggingFace Cryptocurrency Data Engine - Main Application"""
+from __future__ import annotations
+import time
+import logging
+from contextlib import asynccontextmanager
+from fastapi import FastAPI, HTTPException, Query, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+from slowapi import Limiter, _rate_limit_exceeded_handler
+from slowapi.util import get_remote_address
+from slowapi.errors import RateLimitExceeded
+
+from core.config import settings, get_supported_symbols, get_supported_intervals
+from core.aggregator import get_aggregator
+from core.cache import cache, cache_key, get_or_set
+from core.models import (
+    OHLCVResponse, PricesResponse, SentimentResponse,
+    MarketOverviewResponse, HealthResponse, ErrorResponse, ErrorDetail, CacheInfo
+)
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+
+# Rate limiter
+limiter = Limiter(key_func=get_remote_address)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Lifecycle manager for the application"""
+    logger.info("Starting HuggingFace Crypto Data Engine...")
+    logger.info(f"Version: {settings.VERSION}")
+    logger.info(f"Environment: {settings.ENV}")
+
+    # Initialize aggregator
+    aggregator = get_aggregator()
+
+    yield
+
+    # Cleanup
+    logger.info("Shutting down...")
+    await aggregator.close()
+
+
+# Create FastAPI app
+app = FastAPI(
+    title="HuggingFace Cryptocurrency Data Engine",
+    description="Comprehensive cryptocurrency data aggregator with multi-provider support",
+    version=settings.VERSION,
+    lifespan=lifespan
+)
+
+# Add rate limiter
+app.state.limiter = limiter
+app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
+
+# CORS middleware
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.exception_handler(Exception)
+async def global_exception_handler(request: Request, exc: Exception):
+    """Global exception handler"""
+    logger.error(f"Unhandled exception: {exc}", exc_info=True)
+
+    return JSONResponse(
+        status_code=500,
+        content=ErrorResponse(
+            error=ErrorDetail(
+                code="INTERNAL_ERROR",
+                message=str(exc)
+            ),
+            timestamp=int(time.time() * 1000)
+        ).dict()
+    )
+
+
+@app.get("/")
+async def root():
+    """Root endpoint"""
+    return {
+        "service": "HuggingFace Cryptocurrency Data Engine",
+        "version": settings.VERSION,
+        "status": "online",
+        "endpoints": {
+            "health": "/api/health",
+            "ohlcv": "/api/ohlcv",
+            "prices": "/api/prices",
+            "sentiment": "/api/sentiment",
+            "market": "/api/market/overview",
+            "docs": "/docs"
+        }
+    }
+
+
+@app.get("/api/health", response_model=HealthResponse)
+@limiter.limit(f"{settings.RATE_LIMIT_HEALTH or 999999}/minute")
+async def health_check(request: Request):
+    """Health check endpoint with provider status"""
+    aggregator = get_aggregator()
+
+    # Get provider health
+    providers = await aggregator.get_all_provider_health()
+
+    # Determine overall status
+    online_count = sum(1 for p in providers if p.status == "online")
+    if online_count == 0:
+        overall_status = "unhealthy"
+    elif online_count < len(providers) / 2:
+        overall_status = "degraded"
+    else:
+        overall_status = "healthy"
+
+    # Get cache stats
+    cache_stats = cache.get_stats()
+
+    return HealthResponse(
+        status=overall_status,
+        uptime=aggregator.get_uptime(),
+        version=settings.VERSION,
+        providers=providers,
+        cache=CacheInfo(**cache_stats)
+    )
+
+
+@app.get("/api/ohlcv", response_model=OHLCVResponse)
+@limiter.limit(f"{settings.RATE_LIMIT_OHLCV}/minute")
+async def get_ohlcv(
+    request: Request,
+    symbol: str = Query(..., description="Symbol (e.g., BTC, BTCUSDT, BTC/USDT)"),
+    interval: str = Query("1h", description="Interval (1m, 5m, 15m, 1h, 4h, 1d, 1w)"),
+    limit: int = Query(100, ge=1, le=1000, description="Number of candles (1-1000)")
+):
+    """Get OHLCV candlestick data with multi-provider fallback"""
+
+    # Validate interval
+    if interval not in get_supported_intervals():
+        raise HTTPException(
+            status_code=400,
+            detail=f"Invalid interval. Supported: {', '.join(get_supported_intervals())}"
+        )
+
+    # Normalize symbol
+    normalized_symbol = symbol.upper().replace("/", "").replace("-", "")
+
+    # Generate cache key
+    key = cache_key("ohlcv", symbol=normalized_symbol, interval=interval, limit=limit)
+
+    async def fetch():
+        aggregator = get_aggregator()
+        data, source = await aggregator.fetch_ohlcv(normalized_symbol, interval, limit)
+        return {"data": data, "source": source}
+
+    try:
+        # Get from cache or fetch
+        result = await get_or_set(key, settings.CACHE_TTL_OHLCV, fetch)
+
+        return OHLCVResponse(
+            data=result["data"],
+            symbol=normalized_symbol,
+            interval=interval,
+            count=len(result["data"]),
+            source=result["source"],
+            timestamp=int(time.time() * 1000)
+        )
+
+    except Exception as e:
+        logger.error(f"OHLCV fetch failed: {e}")
+        raise HTTPException(
+            status_code=503,
+            detail=ErrorDetail(
+                code="PROVIDER_ERROR",
+                message=f"All data providers failed: {str(e)}"
+            ).dict()
+        )
+
+
+@app.get("/api/prices", response_model=PricesResponse)
+@limiter.limit(f"{settings.RATE_LIMIT_PRICES}/minute")
+async def get_prices(
+    request: Request,
+    symbols: str = Query(None, description="Comma-separated symbols (e.g., BTC,ETH,SOL)"),
+    convert: str = Query("USDT", description="Convert to currency (USD, USDT)")
+):
+    """Get real-time prices with multi-provider aggregation"""
+
+    # Parse symbols
+    if symbols:
+        symbol_list = [s.strip().upper() for s in symbols.split(",")]
+    else:
+        # Use default symbols
+        symbol_list = get_supported_symbols()
+
+    # Generate cache key
+    key = cache_key("prices", symbols=",".join(sorted(symbol_list)))
+
+    async def fetch():
+        aggregator = get_aggregator()
+        data, source = await aggregator.fetch_prices(symbol_list)
+        return {"data": data, "source": source}
+
+    try:
+        # Get from cache or fetch
+        result = await get_or_set(key, settings.CACHE_TTL_PRICES, fetch)
+
+        return PricesResponse(
+            data=result["data"],
+            timestamp=int(time.time() * 1000),
+            source=result["source"]
+        )
+
+    except Exception as e:
+        logger.error(f"Price fetch failed: {e}")
+        raise HTTPException(
+            status_code=503,
+            detail=ErrorDetail(
+                code="PROVIDER_ERROR",
+                message=f"All price providers failed: {str(e)}"
+            ).dict()
+        )
+
+
+@app.get("/api/sentiment", response_model=SentimentResponse)
+@limiter.limit(f"{settings.RATE_LIMIT_SENTIMENT}/minute")
+async def get_sentiment(request: Request):
+    """Get market sentiment data (Fear & Greed Index)"""
+
+    if not settings.ENABLE_SENTIMENT:
+        raise HTTPException(
+            status_code=503,
+            detail="Sentiment analysis is disabled"
+        )
+
+    # Cache key
+    key = cache_key("sentiment")
+
+    async def fetch():
+        aggregator = get_aggregator()
+        return await aggregator.fetch_sentiment()
+
+    try:
+        # Get from cache or fetch
+        data = await get_or_set(key, settings.CACHE_TTL_SENTIMENT, fetch)
+
+        return SentimentResponse(
+            data=data,
+            timestamp=int(time.time() * 1000)
+        )
+
+    except Exception as e:
+        logger.error(f"Sentiment fetch failed: {e}")
+        raise HTTPException(
+            status_code=503,
+            detail=ErrorDetail(
+                code="PROVIDER_ERROR",
+                message=f"Failed to fetch sentiment: {str(e)}"
+            ).dict()
+        )
+
+
+@app.get("/api/market/overview", response_model=MarketOverviewResponse)
+@limiter.limit(f"{settings.RATE_LIMIT_SENTIMENT}/minute")
+async def get_market_overview(request: Request):
+    """Get market overview with global statistics"""
+
+    # Cache key
+    key = cache_key("market_overview")
+
+    async def fetch():
+        aggregator = get_aggregator()
+        return await aggregator.fetch_market_overview()
+
+    try:
+        # Get from cache or fetch
+        data = await get_or_set(key, settings.CACHE_TTL_MARKET, fetch)
+
+        return MarketOverviewResponse(
+            data=data,
+            timestamp=int(time.time() * 1000)
+        )
+
+    except Exception as e:
+        logger.error(f"Market overview fetch failed: {e}")
+        raise HTTPException(
+            status_code=503,
+            detail=ErrorDetail(
+                code="PROVIDER_ERROR",
+                message=f"Failed to fetch market overview: {str(e)}"
+            ).dict()
+        )
+
+
+@app.post("/api/cache/clear")
+async def clear_cache():
+    """Clear all cached data"""
+    cache.clear()
+    return {"success": True, "message": "Cache cleared"}
+
+
+@app.get("/api/cache/stats")
+async def cache_stats():
+    """Get cache statistics"""
+    return cache.get_stats()
+
 
 if __name__ == "__main__":
     import uvicorn
-    import os
-    
-    # Get port from environment or use default
-    port = int(os.getenv("PORT", "8000"))
-    
-    print(f"""
-    ╔═══════════════════════════════════════════════════════════╗
-    ║   🚀 Crypto Monitor - Local Development Server           ║
-    ║   Port: {port}                                                   ║
-    ║   Docs: http://localhost:{port}/docs                            ║
-    ╚═══════════════════════════════════════════════════════════╝
-    """)
-    
-    # Run with reload for local development
+
     uvicorn.run(
-        "api_server_extended:app",
-        host="0.0.0.0",
-        port=port,
-        reload=True,
+        "main:app",
+        host=settings.HOST,
+        port=settings.PORT,
+        reload=(settings.ENV == "development"),
         log_level="info"
     )

hf-data-engine/requirements.txt

@@ -1,58 +1,18 @@
-# Crypto Data Aggregator - Complete Requirements
-# Production-ready dependencies with exact versions
+# FastAPI and server
+fastapi==0.109.0
+uvicorn[standard]==0.27.0
+pydantic==2.5.3
+pydantic-settings==2.1.0
 
-# ==================== GRADIO INTERFACE ====================
-gradio>=4.44.0
+# HTTP client
+httpx==0.26.0
 
-# ==================== DATA PROCESSING ====================
-pandas>=2.0.0
-numpy>=1.24.0
+# Rate limiting
+slowapi==0.1.9
 
-# ==================== HTTP CLIENTS ====================
-requests>=2.31.0
-aiohttp>=3.8.0
-httpx>=0.26.0
+# Optional: Redis support (uncomment if using Redis)
+# redis==5.0.1
+# aioredis==2.0.1
 
-# ==================== WEB BACKEND ====================
-fastapi>=0.109.0
-uvicorn[standard]>=0.27.0
-slowapi>=0.1.9
-python-multipart>=0.0.6
-websockets>=12.0
-
-# ==================== DATA MODELS & CONFIG ====================
-pydantic>=2.5.3
-pydantic-settings>=2.1.0
-
-# ==================== WEB SCRAPING & RSS ====================
-beautifulsoup4>=4.12.0
-feedparser>=6.0.10
-
-# ==================== AI/ML - HUGGING FACE ====================
-transformers>=4.30.0
-torch>=2.0.0
-sentencepiece>=0.1.99
-tokenizers>=0.13.0
-huggingface-hub>=0.16.0
-
-# ==================== PLOTTING & VISUALIZATION ====================
-plotly>=5.14.0
-kaleido>=0.2.1
-
-# ==================== DATABASE & STORAGE ====================
-sqlalchemy>=2.0.25
-
-# ==================== AUTHENTICATION & SECURITY ====================
-PyJWT>=2.8.0
-
-# ==================== DATE/TIME HELPERS ====================
-python-dateutil>=2.8.2
-
-# ==================== OPTIONAL: ACCELERATED INFERENCE ====================
-# accelerate>=0.20.0  # Uncomment for faster model loading
-# bitsandbytes>=0.39.0  # Uncomment for quantization
-
-# ==================== NOTES ====================
-# No API keys required - all data sources are free
-# SQLite is included in Python standard library
-# All packages are production-tested and stable
+# Utilities
+python-dotenv==1.0.0