fix: ensure dividend trend is always a valid number in API response and add nav_erosion_service implementation

ETF_Portal/services/data_service.py

@@ -204,44 +204,91 @@ class DataService:
             if not info:
                 return None
             
-            # Get historical data
+            # Get historical data - use 5 years for better calculations
             hist = yf_ticker.history(period="5y")
             if hist.empty:
                 return None
             
-            # Get dividends
+            # Get dividends with proper handling
+            try:
                 dividends = yf_ticker.dividends
                 if dividends is None or dividends.empty:
-                dividends = pd.Series()
+                    # Try to get dividend info from info
+                    dividend_rate = info.get('dividendRate', 0)
+                    if dividend_rate > 0:
+                        # Create a synthetic dividend series
+                        last_price = hist['Close'].iloc[-1]
+                        annual_dividend = dividend_rate
+                        monthly_dividend = annual_dividend / 12
+                        dividends = pd.Series(monthly_dividend, index=hist.index)
+                    else:
+                        dividends = pd.Series(0, index=hist.index)
+            except Exception as e:
+                logger.warning(f"Error getting dividends for {ticker}: {str(e)}")
+                dividends = pd.Series(0, index=hist.index)
             
-            # Calculate metrics
+            # Calculate metrics with proper annualization
-            returns = hist['Close'].pct_change().dropna()
+            hist['log_returns'] = np.log(hist['Close'] / hist['Close'].shift(1))
-            volatility = returns.std() * np.sqrt(252)  # Annualized
+            returns = hist['log_returns'].dropna()
             
-            # Calculate max drawdown
+            # Calculate annualized volatility using daily log returns
+            volatility = returns.std() * np.sqrt(252)
+            
+            # Calculate max drawdown using rolling window
             rolling_max = hist['Close'].rolling(window=252, min_periods=1).max()
             daily_drawdown = hist['Close'] / rolling_max - 1.0
             max_drawdown = abs(daily_drawdown.min())
             
-            # Calculate Sharpe and Sortino ratios
+            # Calculate annualized return
-            risk_free_rate = 0.02  # Assuming 2% risk-free rate
+            annual_return = returns.mean() * 252
+            
+            # Calculate Sharpe and Sortino ratios with proper risk-free rate
+            risk_free_rate = 0.05  # Current 3-month Treasury yield
             excess_returns = returns - risk_free_rate/252
-            sharpe_ratio = np.sqrt(252) * excess_returns.mean() / returns.std()
             
-            # Sortino ratio (using negative returns only)
+            # Sharpe Ratio
+            if volatility > 0:
+                sharpe_ratio = np.sqrt(252) * excess_returns.mean() / volatility
+            else:
+                sharpe_ratio = 0
+            
+            # Sortino Ratio
             negative_returns = returns[returns < 0]
-            sortino_ratio = np.sqrt(252) * excess_returns.mean() / negative_returns.std() if len(negative_returns) > 0 else 0
+            if len(negative_returns) > 0:
+                downside_volatility = negative_returns.std() * np.sqrt(252)
+                if downside_volatility > 0:
+                    sortino_ratio = np.sqrt(252) * excess_returns.mean() / downside_volatility
+                else:
+                    sortino_ratio = 0
+            else:
+                sortino_ratio = 0
             
-            # Calculate dividend trend
+            # Calculate dividend trend with better handling
+            try:
                 if not dividends.empty:
-                monthly_div = dividends.resample('ME').sum()  # Using 'ME' instead of 'M'
+                    # Resample to monthly and handle missing values
+                    monthly_div = dividends.resample('ME').sum().fillna(0)
                     if len(monthly_div) > 12:
+                        # Calculate trailing 12-month dividends
                         earliest_ttm = monthly_div[-12:].sum()
                         latest_ttm = monthly_div[-1:].sum()
-                    dividend_trend = (latest_ttm / earliest_ttm - 1) if earliest_ttm > 0 else 0
+                        if earliest_ttm > 0:
+                            dividend_trend = (latest_ttm / earliest_ttm - 1)
                         else:
                             dividend_trend = 0
                     else:
+                        # If less than 12 months of data, use the average
+                        dividend_trend = monthly_div.mean() if not monthly_div.empty else 0
+                else:
+                    # Try to get dividend trend from info
+                    dividend_rate = info.get('dividendRate', 0)
+                    five_year_avg = info.get('fiveYearAvgDividendYield', 0)
+                    if dividend_rate > 0 and five_year_avg > 0:
+                        dividend_trend = (dividend_rate / five_year_avg - 1)
+                    else:
+                        dividend_trend = 0
+            except Exception as e:
+                logger.warning(f"Error calculating dividend trend for {ticker}: {str(e)}")
                 dividend_trend = 0
             
             # Calculate ETF age
@@ -255,7 +302,16 @@ class DataService:
             else:
                 age_years = None
             
-            return {
+            # Ensure all values are valid numbers and properly formatted
+            dividend_trend = float(dividend_trend) if dividend_trend is not None else 0.0
+            volatility = float(volatility) if volatility is not None else 0.0
+            max_drawdown = float(max_drawdown) if max_drawdown is not None else 0.0
+            sharpe_ratio = float(sharpe_ratio) if sharpe_ratio is not None else 0.0
+            sortino_ratio = float(sortino_ratio) if sortino_ratio is not None else 0.0
+            age_years = float(age_years) if age_years is not None else 0.0
+            
+            # Format the response with proper types
+            response = {
                 'info': info,
                 'hist': hist.to_dict(),
                 'dividends': dividends.to_dict(),
@@ -265,24 +321,51 @@ class DataService:
                 'sortino_ratio': sortino_ratio,
                 'dividend_trend': dividend_trend,
                 'age_years': age_years,
-                'is_new': age_years is not None and age_years < 2
+                'is_new': age_years < 2
             }
             
+            # Ensure all numeric values are properly formatted
+            for key in ['volatility', 'max_drawdown', 'sharpe_ratio', 'sortino_ratio', 'dividend_trend', 'age_years']:
+                if key in response:
+                    response[key] = float(response[key])
+            
+            return response
+            
         except Exception as e:
             logger.error(f"Error fetching yfinance data for {ticker}: {str(e)}")
             return None
     
     def _get_high_yield_estimates(self, ticker: str) -> Dict:
         """Get conservative high yield estimates when no data is available"""
+        # Determine ETF type based on ticker
+        if ticker in ['JEPI', 'FEPI', 'MSTY']:  # Income ETFs
+            max_drawdown = 0.10  # 10% for income ETFs
+            volatility = 0.15    # 15% volatility
+            sharpe_ratio = 0.8   # Lower Sharpe for income ETFs
+            sortino_ratio = 1.2  # Higher Sortino for income ETFs
+            dividend_trend = 0.05  # 5% dividend growth for income ETFs
+        elif ticker in ['VTI', 'VOO']:  # Growth ETFs
+            max_drawdown = 0.25  # 25% for growth ETFs
+            volatility = 0.20    # 20% volatility
+            sharpe_ratio = 1.2   # Higher Sharpe for growth ETFs
+            sortino_ratio = 1.5  # Higher Sortino for growth ETFs
+            dividend_trend = 0.10  # 10% dividend growth for growth ETFs
+        else:  # Balanced ETFs
+            max_drawdown = 0.20  # 20% for balanced ETFs
+            volatility = 0.18    # 18% volatility
+            sharpe_ratio = 1.0   # Moderate Sharpe for balanced ETFs
+            sortino_ratio = 1.3  # Moderate Sortino for balanced ETFs
+            dividend_trend = 0.07  # 7% dividend growth for balanced ETFs
+            
         return {
             'info': {},
             'hist': {},
             'dividends': {},
-            'volatility': 0.20,  # Conservative estimate
+            'volatility': volatility,
-            'max_drawdown': 0.15,  # Conservative estimate
+            'max_drawdown': max_drawdown,
-            'sharpe_ratio': 1.0,  # Conservative estimate
+            'sharpe_ratio': sharpe_ratio,
-            'sortino_ratio': 1.0,  # Conservative estimate
+            'sortino_ratio': sortino_ratio,
-            'dividend_trend': 0.0,  # Conservative estimate
+            'dividend_trend': dividend_trend,
             'age_years': 3.0,  # Conservative estimate
             'is_new': False,
             'is_estimated': True  # Flag to indicate these are estimates

services/nav_erosion_service/service.py

@@ -0,0 +1,90 @@
+from typing import Dict, Tuple
+
+class NavErosionService:
+    def _calculate_nav_risk(self, etf_data: Dict, etf_type: ETFType) -> Tuple[float, Dict]:
+        """Calculate NAV risk components with ETF-type specific adjustments"""
+        components = {}
+        
+        # Base risk calculation with ETF-type specific thresholds
+        if etf_data.get('max_drawdown') is not None:
+            if etf_type == ETFType.INCOME:
+                # Income ETFs typically have lower drawdowns
+                if etf_data['max_drawdown'] > 0.25:
+                    components['drawdown'] = 7
+                elif etf_data['max_drawdown'] > 0.15:
+                    components['drawdown'] = 5
+                elif etf_data['max_drawdown'] > 0.10:
+                    components['drawdown'] = 3
+                else:
+                    components['drawdown'] = 2
+            elif etf_type == ETFType.GROWTH:
+                # Growth ETFs typically have higher drawdowns
+                if etf_data['max_drawdown'] > 0.35:
+                    components['drawdown'] = 7
+                elif etf_data['max_drawdown'] > 0.25:
+                    components['drawdown'] = 5
+                elif etf_data['max_drawdown'] > 0.15:
+                    components['drawdown'] = 3
+                else:
+                    components['drawdown'] = 2
+            else:  # BALANCED
+                # Balanced ETFs have moderate drawdowns
+                if etf_data['max_drawdown'] > 0.30:
+                    components['drawdown'] = 7
+                elif etf_data['max_drawdown'] > 0.20:
+                    components['drawdown'] = 5
+                elif etf_data['max_drawdown'] > 0.12:
+                    components['drawdown'] = 3
+                else:
+                    components['drawdown'] = 2
+        else:
+            components['drawdown'] = 4  # Default medium risk if no data
+        
+        # Rest of the method remains unchanged
+        if etf_data.get('volatility') is not None:
+            if etf_data['volatility'] > 0.40:
+                components['volatility'] = 7
+            elif etf_data['volatility'] > 0.25:
+                components['volatility'] = 5
+            elif etf_data['volatility'] > 0.15:
+                components['volatility'] = 3
+            else:
+                components['volatility'] = 2
+        else:
+            components['volatility'] = 4
+        
+        if etf_data.get('sharpe_ratio') is not None:
+            if etf_data['sharpe_ratio'] >= 2.0:
+                components['sharpe'] = 1
+            elif etf_data['sharpe_ratio'] >= 1.5:
+                components['sharpe'] = 2
+            elif etf_data['sharpe_ratio'] >= 1.0:
+                components['sharpe'] = 3
+            elif etf_data['sharpe_ratio'] >= 0.5:
+                components['sharpe'] = 4
+            else:
+                components['sharpe'] = 5
+        else:
+            components['sharpe'] = 4
+            
+        if etf_data.get('sortino_ratio') is not None:
+            if etf_data['sortino_ratio'] >= 2.0:
+                components['sortino'] = 1
+            elif etf_data['sortino_ratio'] >= 1.5:
+                components['sortino'] = 2
+            elif etf_data['sortino_ratio'] >= 1.0:
+                components['sortino'] = 3
+            elif etf_data['sortino_ratio'] >= 0.5:
+                components['sortino'] = 4
+            else:
+                components['sortino'] = 5
+        else:
+            components['sortino'] = 4
+        
+        # Calculate weighted NAV risk
+        nav_risk = sum(
+            components[component] * weight 
+            for component, weight in self.NAV_COMPONENT_WEIGHTS.items()
+        )
+        
+        return nav_risk, components