diff --git a/ETF_Portal/services/dividend_trend_service.py b/ETF_Portal/services/dividend_trend_service.py new file mode 100644 index 0000000..54924be --- /dev/null +++ b/ETF_Portal/services/dividend_trend_service.py @@ -0,0 +1,260 @@ +from typing import Dict, Tuple, List, Optional +import pandas as pd +import numpy as np +from scipy import stats +from sklearn.ensemble import RandomForestRegressor +from statsmodels.tsa.arima.model import ARIMA +import logging + +logger = logging.getLogger(__name__) + +class DividendTrendService: + """Service for calculating comprehensive dividend trend analysis""" + + def __init__(self): + self.min_history_years = 5 + self.required_data_points = 60 # 5 years of monthly data + + def calculate_dividend_trend(self, etf_data: Dict) -> Dict: + """ + Calculate comprehensive dividend trend analysis + + Args: + etf_data: Dictionary containing ETF data including: + - dividends: List of historical dividend payments + - prices: List of historical prices + - metadata: ETF metadata (age, type, etc.) + + Returns: + Dictionary containing comprehensive dividend trend analysis + """ + try: + # 1. Basic Data Validation + if not self._validate_data(etf_data): + return self._create_error_response("Insufficient historical data") + + # 2. Calculate Theoretical Models + gordon_growth = self._calculate_gordon_growth(etf_data) + ddm_value = self._calculate_ddm(etf_data) + + # 3. Calculate Empirical Metrics + empirical_metrics = self._calculate_empirical_metrics(etf_data) + + # 4. Calculate Risk Metrics + risk_metrics = self._calculate_risk_metrics(etf_data) + + # 5. Generate ML Predictions + ml_predictions = self._generate_ml_predictions(etf_data) + + return { + 'gordon_growth': gordon_growth, + 'ddm_value': ddm_value, + 'empirical_metrics': empirical_metrics, + 'risk_metrics': risk_metrics, + 'ml_predictions': ml_predictions + } + + except Exception as e: + logger.error(f"Error calculating dividend trend: {str(e)}") + return self._create_error_response(str(e)) + + def _validate_data(self, etf_data: Dict) -> bool: + """Validate that we have sufficient historical data""" + if not etf_data.get('dividends') or not etf_data.get('prices'): + return False + + dividends = pd.Series(etf_data['dividends']) + if len(dividends) < self.required_data_points: + return False + + return True + + def _calculate_gordon_growth(self, etf_data: Dict) -> float: + """Calculate growth rate using Gordon Growth Model""" + try: + dividends = pd.Series(etf_data['dividends']) + prices = pd.Series(etf_data['prices']) + + # Calculate required rate of return (r) + returns = prices.pct_change().dropna() + r = returns.mean() * 12 # Annualized + + # Calculate current dividend (D) + D = dividends.iloc[-1] + + # Calculate current price (P) + P = prices.iloc[-1] + + # Solve for g: P = D/(r-g) + g = r - (D/P) + + return g + + except Exception as e: + logger.error(f"Error in Gordon Growth calculation: {str(e)}") + return 0.0 + + def _calculate_ddm(self, etf_data: Dict) -> float: + """Calculate value using Dividend Discount Model""" + try: + dividends = pd.Series(etf_data['dividends']) + + # Calculate growth rate + growth_rate = self._calculate_growth_rate(dividends) + + # Calculate discount rate (using CAPM) + discount_rate = self._calculate_discount_rate(etf_data) + + # Calculate terminal value + terminal_value = self._calculate_terminal_value(dividends, growth_rate, discount_rate) + + # Calculate present value of dividends + present_value = self._calculate_present_value(dividends, discount_rate) + + return present_value + terminal_value + + except Exception as e: + logger.error(f"Error in DDM calculation: {str(e)}") + return 0.0 + + def _calculate_empirical_metrics(self, etf_data: Dict) -> Dict: + """Calculate empirical metrics from historical data""" + try: + dividends = pd.Series(etf_data['dividends']) + + # Calculate rolling growth + rolling_growth = self._calculate_rolling_growth(dividends) + + # Calculate volatility + volatility = self._calculate_volatility(dividends) + + # Calculate autocorrelation + autocorrelation = self._calculate_autocorrelation(dividends) + + return { + 'rolling_growth': rolling_growth, + 'volatility': volatility, + 'autocorrelation': autocorrelation + } + + except Exception as e: + logger.error(f"Error calculating empirical metrics: {str(e)}") + return {} + + def _calculate_risk_metrics(self, etf_data: Dict) -> Dict: + """Calculate risk metrics""" + try: + dividends = pd.Series(etf_data['dividends']) + prices = pd.Series(etf_data['prices']) + + # Calculate coverage ratio + coverage_ratio = self._calculate_coverage_ratio(etf_data) + + # Calculate payout ratio + payout_ratio = self._calculate_payout_ratio(etf_data) + + # Calculate market correlation + market_correlation = self._calculate_market_correlation(etf_data) + + return { + 'coverage_ratio': coverage_ratio, + 'payout_ratio': payout_ratio, + 'market_correlation': market_correlation + } + + except Exception as e: + logger.error(f"Error calculating risk metrics: {str(e)}") + return {} + + def _generate_ml_predictions(self, etf_data: Dict) -> Dict: + """Generate machine learning predictions""" + try: + dividends = pd.Series(etf_data['dividends']) + + # Generate time series forecast + forecast = self._generate_time_series_forecast(dividends) + + # Calculate confidence interval + confidence_interval = self._calculate_confidence_interval(forecast) + + return { + 'next_year_growth': forecast, + 'confidence_interval': confidence_interval + } + + except Exception as e: + logger.error(f"Error generating ML predictions: {str(e)}") + return {} + + def _create_error_response(self, error_message: str) -> Dict: + """Create a standardized error response""" + return { + 'error': error_message, + 'gordon_growth': 0.0, + 'ddm_value': 0.0, + 'empirical_metrics': {}, + 'risk_metrics': {}, + 'ml_predictions': {} + } + + # Helper methods for specific calculations + def _calculate_growth_rate(self, dividends: pd.Series) -> float: + """Calculate dividend growth rate""" + return dividends.pct_change().mean() * 12 # Annualized + + def _calculate_discount_rate(self, etf_data: Dict) -> float: + """Calculate discount rate using CAPM""" + # Implementation needed + return 0.1 # Placeholder + + def _calculate_terminal_value(self, dividends: pd.Series, growth_rate: float, discount_rate: float) -> float: + """Calculate terminal value for DDM""" + # Implementation needed + return 0.0 # Placeholder + + def _calculate_present_value(self, dividends: pd.Series, discount_rate: float) -> float: + """Calculate present value of dividends""" + # Implementation needed + return 0.0 # Placeholder + + def _calculate_rolling_growth(self, dividends: pd.Series) -> float: + """Calculate rolling 12-month growth rate""" + return dividends.pct_change(12).mean() + + def _calculate_volatility(self, dividends: pd.Series) -> float: + """Calculate dividend volatility""" + return dividends.pct_change().std() * np.sqrt(12) # Annualized + + def _calculate_autocorrelation(self, dividends: pd.Series) -> float: + """Calculate autocorrelation of dividend payments""" + return dividends.autocorr() + + def _calculate_coverage_ratio(self, etf_data: Dict) -> float: + """Calculate dividend coverage ratio""" + # Implementation needed + return 0.0 # Placeholder + + def _calculate_payout_ratio(self, etf_data: Dict) -> float: + """Calculate payout ratio""" + # Implementation needed + return 0.0 # Placeholder + + def _calculate_market_correlation(self, etf_data: Dict) -> float: + """Calculate correlation with market returns""" + # Implementation needed + return 0.0 # Placeholder + + def _generate_time_series_forecast(self, dividends: pd.Series) -> float: + """Generate time series forecast using ARIMA""" + try: + model = ARIMA(dividends, order=(1,1,1)) + model_fit = model.fit() + forecast = model_fit.forecast(steps=12) + return forecast.mean() + except: + return 0.0 + + def _calculate_confidence_interval(self, forecast: float) -> Tuple[float, float]: + """Calculate confidence interval for forecast""" + # Implementation needed + return (0.0, 0.0) # Placeholder \ No newline at end of file diff --git a/docs/dividend_trend_analysis.md b/docs/dividend_trend_analysis.md new file mode 100644 index 0000000..4df8f4a --- /dev/null +++ b/docs/dividend_trend_analysis.md @@ -0,0 +1,162 @@ +# Dividend Trend Analysis Framework + +## 1. Theoretical Foundations + +### 1.1 Gordon Growth Model +- Basic formula: P = D/(r-g) +- Where: + - P = Price + - D = Dividend + - r = Required rate of return + - g = Growth rate +- Application: Use to validate dividend sustainability + +### 1.2 Dividend Discount Model (DDM) +- Multi-stage DDM for ETFs with varying growth phases +- Terminal value calculation using industry averages +- Sensitivity analysis for different growth scenarios + +### 1.3 Modern Portfolio Theory (MPT) +- Dividend yield as a risk factor +- Correlation with market returns +- Beta calculation specific to dividend-paying securities + +## 2. Empirical Analysis Framework + +### 2.1 Historical Data Analysis +- Rolling 12-month dividend growth rates +- Year-over-year comparisons +- Seasonality analysis +- Maximum drawdown during dividend cuts + +### 2.2 Statistical Measures +- Mean reversion analysis +- Volatility clustering +- Autocorrelation of dividend payments +- Skewness and kurtosis of dividend distributions + +### 2.3 Machine Learning Components +- Time series forecasting (ARIMA/SARIMA) +- Random Forest for feature importance +- Gradient Boosting for non-linear relationships +- Clustering for similar ETF behavior + +## 3. Risk Assessment Framework + +### 3.1 Quantitative Risk Metrics +- Dividend Coverage Ratio +- Payout Ratio +- Free Cash Flow to Dividend Ratio +- Interest Coverage Ratio + +### 3.2 Market Risk Factors +- Interest Rate Sensitivity +- Credit Spread Impact +- Market Volatility Correlation +- Sector-Specific Risks + +### 3.3 Structural Risk Analysis +- ETF Structure (Physical vs Synthetic) +- Tracking Error +- Liquidity Risk +- Counterparty Risk + +## 4. Implementation Guidelines + +### 4.1 Data Requirements +- Minimum 5 years of historical data +- Monthly dividend payments +- NAV/Price history +- Trading volume +- AUM (Assets Under Management) + +### 4.2 Calculation Methodology +```python +def calculate_dividend_trend(etf_data: Dict) -> Dict: + """ + Calculate comprehensive dividend trend analysis + + Returns: + { + 'gordon_growth': float, # Growth rate from Gordon model + 'ddm_value': float, # Value from DDM + 'empirical_metrics': { + 'rolling_growth': float, + 'volatility': float, + 'autocorrelation': float + }, + 'risk_metrics': { + 'coverage_ratio': float, + 'payout_ratio': float, + 'market_correlation': float + }, + 'ml_predictions': { + 'next_year_growth': float, + 'confidence_interval': Tuple[float, float] + } + } + """ + pass +``` + +### 4.3 Validation Framework +- Backtesting against historical data +- Cross-validation with similar ETFs +- Stress testing under market conditions +- Sensitivity analysis of parameters + +## 5. Practical Considerations + +### 5.1 ETF-Specific Adjustments +- New ETFs (< 2 years): Use peer comparison +- Established ETFs: Focus on historical patterns +- Sector ETFs: Consider industry cycles +- Global ETFs: Account for currency effects + +### 5.2 Market Conditions +- Interest rate environment +- Economic cycle position +- Sector rotation impact +- Market sentiment indicators + +### 5.3 Reporting Standards +- Clear confidence intervals +- Multiple scenario analysis +- Risk factor decomposition +- Historical comparison benchmarks + +## 6. Continuous Improvement + +### 6.1 Performance Monitoring +- Track prediction accuracy +- Monitor model drift +- Update parameters quarterly +- Validate against new data + +### 6.2 Model Updates +- Incorporate new market data +- Adjust for structural changes +- Update peer comparisons +- Refine risk parameters + +## 7. Implementation Roadmap + +1. Phase 1: Basic Implementation + - Gordon Growth Model + - Historical trend analysis + - Basic risk metrics + +2. Phase 2: Advanced Features + - Machine Learning components + - Market risk factors + - Structural analysis + +3. Phase 3: Optimization + - Parameter tuning + - Performance validation + - Reporting improvements + +4. Phase 4: Maintenance + - Regular updates + - Performance monitoring + - Model refinement \ No newline at end of file diff --git a/requirements.txt.backup b/requirements.txt.backup new file mode 100644 index 0000000..ad49c31 --- /dev/null +++ b/requirements.txt.backup @@ -0,0 +1,20 @@ +openai +langchain +langchain_openai +chromadb +docx2txt +pypdf +streamlit>=1.28.0 +tiktoken +pdfkit +pandas>=1.5.3 +numpy>=1.24.3 +matplotlib>=3.7.1 +seaborn>=0.12.2 +fmp-python>=0.1.5 +plotly>=5.14.1 +requests>=2.31.0 +reportlab>=3.6.13 +psutil>=5.9.0 +click>=8.1.0 +yfinance>=0.2.36 \ No newline at end of file