Add dividend trend service, documentation, and requirements backup

ETF_Portal/services/dividend_trend_service.py

@@ -0,0 +1,260 @@
+from typing import Dict, Tuple, List, Optional
+import pandas as pd
+import numpy as np
+from scipy import stats
+from sklearn.ensemble import RandomForestRegressor
+from statsmodels.tsa.arima.model import ARIMA
+import logging
+
+logger = logging.getLogger(__name__)
+
+class DividendTrendService:
+    """Service for calculating comprehensive dividend trend analysis"""
+    
+    def __init__(self):
+        self.min_history_years = 5
+        self.required_data_points = 60  # 5 years of monthly data
+        
+    def calculate_dividend_trend(self, etf_data: Dict) -> Dict:
+        """
+        Calculate comprehensive dividend trend analysis
+        
+        Args:
+            etf_data: Dictionary containing ETF data including:
+                - dividends: List of historical dividend payments
+                - prices: List of historical prices
+                - metadata: ETF metadata (age, type, etc.)
+                
+        Returns:
+            Dictionary containing comprehensive dividend trend analysis
+        """
+        try:
+            # 1. Basic Data Validation
+            if not self._validate_data(etf_data):
+                return self._create_error_response("Insufficient historical data")
+            
+            # 2. Calculate Theoretical Models
+            gordon_growth = self._calculate_gordon_growth(etf_data)
+            ddm_value = self._calculate_ddm(etf_data)
+            
+            # 3. Calculate Empirical Metrics
+            empirical_metrics = self._calculate_empirical_metrics(etf_data)
+            
+            # 4. Calculate Risk Metrics
+            risk_metrics = self._calculate_risk_metrics(etf_data)
+            
+            # 5. Generate ML Predictions
+            ml_predictions = self._generate_ml_predictions(etf_data)
+            
+            return {
+                'gordon_growth': gordon_growth,
+                'ddm_value': ddm_value,
+                'empirical_metrics': empirical_metrics,
+                'risk_metrics': risk_metrics,
+                'ml_predictions': ml_predictions
+            }
+            
+        except Exception as e:
+            logger.error(f"Error calculating dividend trend: {str(e)}")
+            return self._create_error_response(str(e))
+    
+    def _validate_data(self, etf_data: Dict) -> bool:
+        """Validate that we have sufficient historical data"""
+        if not etf_data.get('dividends') or not etf_data.get('prices'):
+            return False
+            
+        dividends = pd.Series(etf_data['dividends'])
+        if len(dividends) < self.required_data_points:
+            return False
+            
+        return True
+    
+    def _calculate_gordon_growth(self, etf_data: Dict) -> float:
+        """Calculate growth rate using Gordon Growth Model"""
+        try:
+            dividends = pd.Series(etf_data['dividends'])
+            prices = pd.Series(etf_data['prices'])
+            
+            # Calculate required rate of return (r)
+            returns = prices.pct_change().dropna()
+            r = returns.mean() * 12  # Annualized
+            
+            # Calculate current dividend (D)
+            D = dividends.iloc[-1]
+            
+            # Calculate current price (P)
+            P = prices.iloc[-1]
+            
+            # Solve for g: P = D/(r-g)
+            g = r - (D/P)
+            
+            return g
+            
+        except Exception as e:
+            logger.error(f"Error in Gordon Growth calculation: {str(e)}")
+            return 0.0
+    
+    def _calculate_ddm(self, etf_data: Dict) -> float:
+        """Calculate value using Dividend Discount Model"""
+        try:
+            dividends = pd.Series(etf_data['dividends'])
+            
+            # Calculate growth rate
+            growth_rate = self._calculate_growth_rate(dividends)
+            
+            # Calculate discount rate (using CAPM)
+            discount_rate = self._calculate_discount_rate(etf_data)
+            
+            # Calculate terminal value
+            terminal_value = self._calculate_terminal_value(dividends, growth_rate, discount_rate)
+            
+            # Calculate present value of dividends
+            present_value = self._calculate_present_value(dividends, discount_rate)
+            
+            return present_value + terminal_value
+            
+        except Exception as e:
+            logger.error(f"Error in DDM calculation: {str(e)}")
+            return 0.0
+    
+    def _calculate_empirical_metrics(self, etf_data: Dict) -> Dict:
+        """Calculate empirical metrics from historical data"""
+        try:
+            dividends = pd.Series(etf_data['dividends'])
+            
+            # Calculate rolling growth
+            rolling_growth = self._calculate_rolling_growth(dividends)
+            
+            # Calculate volatility
+            volatility = self._calculate_volatility(dividends)
+            
+            # Calculate autocorrelation
+            autocorrelation = self._calculate_autocorrelation(dividends)
+            
+            return {
+                'rolling_growth': rolling_growth,
+                'volatility': volatility,
+                'autocorrelation': autocorrelation
+            }
+            
+        except Exception as e:
+            logger.error(f"Error calculating empirical metrics: {str(e)}")
+            return {}
+    
+    def _calculate_risk_metrics(self, etf_data: Dict) -> Dict:
+        """Calculate risk metrics"""
+        try:
+            dividends = pd.Series(etf_data['dividends'])
+            prices = pd.Series(etf_data['prices'])
+            
+            # Calculate coverage ratio
+            coverage_ratio = self._calculate_coverage_ratio(etf_data)
+            
+            # Calculate payout ratio
+            payout_ratio = self._calculate_payout_ratio(etf_data)
+            
+            # Calculate market correlation
+            market_correlation = self._calculate_market_correlation(etf_data)
+            
+            return {
+                'coverage_ratio': coverage_ratio,
+                'payout_ratio': payout_ratio,
+                'market_correlation': market_correlation
+            }
+            
+        except Exception as e:
+            logger.error(f"Error calculating risk metrics: {str(e)}")
+            return {}
+    
+    def _generate_ml_predictions(self, etf_data: Dict) -> Dict:
+        """Generate machine learning predictions"""
+        try:
+            dividends = pd.Series(etf_data['dividends'])
+            
+            # Generate time series forecast
+            forecast = self._generate_time_series_forecast(dividends)
+            
+            # Calculate confidence interval
+            confidence_interval = self._calculate_confidence_interval(forecast)
+            
+            return {
+                'next_year_growth': forecast,
+                'confidence_interval': confidence_interval
+            }
+            
+        except Exception as e:
+            logger.error(f"Error generating ML predictions: {str(e)}")
+            return {}
+    
+    def _create_error_response(self, error_message: str) -> Dict:
+        """Create a standardized error response"""
+        return {
+            'error': error_message,
+            'gordon_growth': 0.0,
+            'ddm_value': 0.0,
+            'empirical_metrics': {},
+            'risk_metrics': {},
+            'ml_predictions': {}
+        }
+    
+    # Helper methods for specific calculations
+    def _calculate_growth_rate(self, dividends: pd.Series) -> float:
+        """Calculate dividend growth rate"""
+        return dividends.pct_change().mean() * 12  # Annualized
+    
+    def _calculate_discount_rate(self, etf_data: Dict) -> float:
+        """Calculate discount rate using CAPM"""
+        # Implementation needed
+        return 0.1  # Placeholder
+    
+    def _calculate_terminal_value(self, dividends: pd.Series, growth_rate: float, discount_rate: float) -> float:
+        """Calculate terminal value for DDM"""
+        # Implementation needed
+        return 0.0  # Placeholder
+    
+    def _calculate_present_value(self, dividends: pd.Series, discount_rate: float) -> float:
+        """Calculate present value of dividends"""
+        # Implementation needed
+        return 0.0  # Placeholder
+    
+    def _calculate_rolling_growth(self, dividends: pd.Series) -> float:
+        """Calculate rolling 12-month growth rate"""
+        return dividends.pct_change(12).mean()
+    
+    def _calculate_volatility(self, dividends: pd.Series) -> float:
+        """Calculate dividend volatility"""
+        return dividends.pct_change().std() * np.sqrt(12)  # Annualized
+    
+    def _calculate_autocorrelation(self, dividends: pd.Series) -> float:
+        """Calculate autocorrelation of dividend payments"""
+        return dividends.autocorr()
+    
+    def _calculate_coverage_ratio(self, etf_data: Dict) -> float:
+        """Calculate dividend coverage ratio"""
+        # Implementation needed
+        return 0.0  # Placeholder
+    
+    def _calculate_payout_ratio(self, etf_data: Dict) -> float:
+        """Calculate payout ratio"""
+        # Implementation needed
+        return 0.0  # Placeholder
+    
+    def _calculate_market_correlation(self, etf_data: Dict) -> float:
+        """Calculate correlation with market returns"""
+        # Implementation needed
+        return 0.0  # Placeholder
+    
+    def _generate_time_series_forecast(self, dividends: pd.Series) -> float:
+        """Generate time series forecast using ARIMA"""
+        try:
+            model = ARIMA(dividends, order=(1,1,1))
+            model_fit = model.fit()
+            forecast = model_fit.forecast(steps=12)
+            return forecast.mean()
+        except:
+            return 0.0
+    
+    def _calculate_confidence_interval(self, forecast: float) -> Tuple[float, float]:
+        """Calculate confidence interval for forecast"""
+        # Implementation needed
+        return (0.0, 0.0)  # Placeholder 

docs/dividend_trend_analysis.md

@@ -0,0 +1,162 @@
+# Dividend Trend Analysis Framework
+
+## 1. Theoretical Foundations
+
+### 1.1 Gordon Growth Model
+- Basic formula: P = D/(r-g)
+- Where:
+  - P = Price
+  - D = Dividend
+  - r = Required rate of return
+  - g = Growth rate
+- Application: Use to validate dividend sustainability
+
+### 1.2 Dividend Discount Model (DDM)
+- Multi-stage DDM for ETFs with varying growth phases
+- Terminal value calculation using industry averages
+- Sensitivity analysis for different growth scenarios
+
+### 1.3 Modern Portfolio Theory (MPT)
+- Dividend yield as a risk factor
+- Correlation with market returns
+- Beta calculation specific to dividend-paying securities
+
+## 2. Empirical Analysis Framework
+
+### 2.1 Historical Data Analysis
+- Rolling 12-month dividend growth rates
+- Year-over-year comparisons
+- Seasonality analysis
+- Maximum drawdown during dividend cuts
+
+### 2.2 Statistical Measures
+- Mean reversion analysis
+- Volatility clustering
+- Autocorrelation of dividend payments
+- Skewness and kurtosis of dividend distributions
+
+### 2.3 Machine Learning Components
+- Time series forecasting (ARIMA/SARIMA)
+- Random Forest for feature importance
+- Gradient Boosting for non-linear relationships
+- Clustering for similar ETF behavior
+
+## 3. Risk Assessment Framework
+
+### 3.1 Quantitative Risk Metrics
+- Dividend Coverage Ratio
+- Payout Ratio
+- Free Cash Flow to Dividend Ratio
+- Interest Coverage Ratio
+
+### 3.2 Market Risk Factors
+- Interest Rate Sensitivity
+- Credit Spread Impact
+- Market Volatility Correlation
+- Sector-Specific Risks
+
+### 3.3 Structural Risk Analysis
+- ETF Structure (Physical vs Synthetic)
+- Tracking Error
+- Liquidity Risk
+- Counterparty Risk
+
+## 4. Implementation Guidelines
+
+### 4.1 Data Requirements
+- Minimum 5 years of historical data
+- Monthly dividend payments
+- NAV/Price history
+- Trading volume
+- AUM (Assets Under Management)
+
+### 4.2 Calculation Methodology
+```python
+def calculate_dividend_trend(etf_data: Dict) -> Dict:
+    """
+    Calculate comprehensive dividend trend analysis
+    
+    Returns:
+    {
+        'gordon_growth': float,  # Growth rate from Gordon model
+        'ddm_value': float,      # Value from DDM
+        'empirical_metrics': {
+            'rolling_growth': float,
+            'volatility': float,
+            'autocorrelation': float
+        },
+        'risk_metrics': {
+            'coverage_ratio': float,
+            'payout_ratio': float,
+            'market_correlation': float
+        },
+        'ml_predictions': {
+            'next_year_growth': float,
+            'confidence_interval': Tuple[float, float]
+        }
+    }
+    """
+    pass
+```
+
+### 4.3 Validation Framework
+- Backtesting against historical data
+- Cross-validation with similar ETFs
+- Stress testing under market conditions
+- Sensitivity analysis of parameters
+
+## 5. Practical Considerations
+
+### 5.1 ETF-Specific Adjustments
+- New ETFs (< 2 years): Use peer comparison
+- Established ETFs: Focus on historical patterns
+- Sector ETFs: Consider industry cycles
+- Global ETFs: Account for currency effects
+
+### 5.2 Market Conditions
+- Interest rate environment
+- Economic cycle position
+- Sector rotation impact
+- Market sentiment indicators
+
+### 5.3 Reporting Standards
+- Clear confidence intervals
+- Multiple scenario analysis
+- Risk factor decomposition
+- Historical comparison benchmarks
+
+## 6. Continuous Improvement
+
+### 6.1 Performance Monitoring
+- Track prediction accuracy
+- Monitor model drift
+- Update parameters quarterly
+- Validate against new data
+
+### 6.2 Model Updates
+- Incorporate new market data
+- Adjust for structural changes
+- Update peer comparisons
+- Refine risk parameters
+
+## 7. Implementation Roadmap
+
+1. Phase 1: Basic Implementation
+   - Gordon Growth Model
+   - Historical trend analysis
+   - Basic risk metrics
+
+2. Phase 2: Advanced Features
+   - Machine Learning components
+   - Market risk factors
+   - Structural analysis
+
+3. Phase 3: Optimization
+   - Parameter tuning
+   - Performance validation
+   - Reporting improvements
+
+4. Phase 4: Maintenance
+   - Regular updates
+   - Performance monitoring
+   - Model refinement 

requirements.txt.backup

@@ -0,0 +1,20 @@
+openai
+langchain
+langchain_openai
+chromadb
+docx2txt
+pypdf
+streamlit>=1.28.0
+tiktoken
+pdfkit
+pandas>=1.5.3
+numpy>=1.24.3
+matplotlib>=3.7.1
+seaborn>=0.12.2
+fmp-python>=0.1.5
+plotly>=5.14.1
+requests>=2.31.0
+reportlab>=3.6.13
+psutil>=5.9.0
+click>=8.1.0
+yfinance>=0.2.36