ETF_Suite_Portal/ETF_Portal/logging_config.py

#!/usr/bin/env python3
"""
Enhanced Logging Configuration for ETF Portal

Provides centralized logging configuration with component-specific logging,
log rotation, and structured logging formats.
"""

import logging
import logging.handlers
from pathlib import Path
from datetime import datetime
from typing import Optional, Dict, Any
import json
import sys
import os

# Constants
LOGS_DIR = Path("logs")
MAX_LOG_SIZE = 10 * 1024 * 1024  # 10MB
BACKUP_COUNT = 5

# Log formats
DETAILED_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s'
SIMPLE_FORMAT = '%(asctime)s - %(levelname)s - %(message)s'

# Component configurations
COMPONENTS = {
    'api': {
        'name': 'API',
        'log_level': logging.INFO,
        'file_prefix': 'api',
        'categories': ['requests', 'responses', 'errors']
    },
    'cache': {
        'name': 'Cache',
        'log_level': logging.INFO,
        'file_prefix': 'cache',
        'categories': ['hits', 'misses', 'cleanup']
    },
    'portfolio': {
        'name': 'Portfolio',
        'log_level': logging.INFO,
        'file_prefix': 'portfolio',
        'categories': ['calculations', 'allocations', 'optimizations']
    },
    'performance': {
        'name': 'Performance',
        'log_level': logging.INFO,
        'file_prefix': 'performance',
        'categories': ['response_times', 'resource_usage']
    }
}

class ComponentLogger:
    """Manages logging for a specific component."""
    
    def __init__(self, component: str):
        """
        Initialize component logger.
        
        Args:
            component: Component name (must be in COMPONENTS)
        """
        if component not in COMPONENTS:
            raise ValueError(f"Unknown component: {component}")
            
        self.component = component
        self.config = COMPONENTS[component]
        self.logger = logging.getLogger(f"etf_portal.{component}")
        self.logger.setLevel(self.config['log_level'])
        
        # Create component-specific log directory
        self.log_dir = LOGS_DIR / component
        self.log_dir.mkdir(parents=True, exist_ok=True)
        
        # Setup handlers
        self._setup_handlers()
        
        # Log initialization
        self.logger.info(f"Initialized {self.config['name']} logger")
    
    def _setup_handlers(self):
        """Setup logging handlers for the component."""
        # Remove any existing handlers
        self.logger.handlers = []
        
        # Create formatters
        detailed_formatter = logging.Formatter(DETAILED_FORMAT)
        simple_formatter = logging.Formatter(SIMPLE_FORMAT)
        
        # Main log file with rotation
        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
        log_file = self.log_dir / f"{self.config['file_prefix']}_{timestamp}.log"
        file_handler = logging.handlers.RotatingFileHandler(
            log_file,
            maxBytes=MAX_LOG_SIZE,
            backupCount=BACKUP_COUNT
        )
        file_handler.setFormatter(detailed_formatter)
        file_handler.setLevel(logging.DEBUG)
        self.logger.addHandler(file_handler)
        
        # Error log file
        error_file = self.log_dir / f"{self.config['file_prefix']}_error_{timestamp}.log"
        error_handler = logging.handlers.RotatingFileHandler(
            error_file,
            maxBytes=MAX_LOG_SIZE,
            backupCount=BACKUP_COUNT
        )
        error_handler.setFormatter(detailed_formatter)
        error_handler.setLevel(logging.ERROR)
        self.logger.addHandler(error_handler)
        
        # Console handler
        console_handler = logging.StreamHandler(sys.stdout)
        console_handler.setFormatter(simple_formatter)
        console_handler.setLevel(logging.INFO)
        self.logger.addHandler(console_handler)
    
    def log_api_call(self, endpoint: str, method: str, params: Dict = None, 
                    response_time: float = None, status_code: int = None):
        """Log API call details."""
        log_data = {
            'endpoint': endpoint,
            'method': method,
            'params': params,
            'response_time': response_time,
            'status_code': status_code
        }
        self.logger.info(f"API Call: {json.dumps(log_data)}")
    
    def log_cache_operation(self, operation: str, key: str, hit: bool = None, 
                           size: int = None, ttl: int = None):
        """Log cache operation details."""
        log_data = {
            'operation': operation,
            'key': key,
            'hit': hit,
            'size': size,
            'ttl': ttl
        }
        self.logger.info(f"Cache Operation: {json.dumps(log_data)}")
    
    def log_portfolio_calculation(self, calculation_type: str, 
                                input_data: Dict = None, 
                                output_data: Dict = None,
                                duration: float = None):
        """Log portfolio calculation details."""
        log_data = {
            'type': calculation_type,
            'input': input_data,
            'output': output_data,
            'duration': duration
        }
        self.logger.info(f"Portfolio Calculation: {json.dumps(log_data)}")
    
    def log_performance_metric(self, metric_name: str, value: float, 
                             unit: str = None, context: Dict = None):
        """Log performance metric details."""
        log_data = {
            'metric': metric_name,
            'value': value,
            'unit': unit,
            'context': context,
            'timestamp': datetime.now().isoformat()
        }
        self.logger.info(f"Performance Metric: {json.dumps(log_data)}")

def setup_logging():
    """Initialize the logging system."""
    # Create logs directory
    LOGS_DIR.mkdir(parents=True, exist_ok=True)
    
    # Create component loggers
    loggers = {}
    for component in COMPONENTS:
        loggers[component] = ComponentLogger(component)
    
    return loggers

# Create loggers for all components
loggers = setup_logging()

# Export component loggers
api_logger = loggers['api'].logger
cache_logger = loggers['cache'].logger
portfolio_logger = loggers['portfolio'].logger
performance_logger = loggers['performance'].logger

# Add performance metric logging methods to loggers
def add_performance_logging(logger):
    """Add performance metric logging methods to a logger."""
    logger.log_performance_metric = lambda metric_name, value, unit=None, context=None: \
        loggers['performance'].log_performance_metric(metric_name, value, unit, context)

# Add performance logging to all loggers
for logger in [api_logger, cache_logger, portfolio_logger, performance_logger]:
    add_performance_logging(logger)