376 lines
16 KiB
Python
376 lines
16 KiB
Python
"""
|
|
Data Service for ETF data retrieval
|
|
"""
|
|
|
|
import os
|
|
import json
|
|
import pandas as pd
|
|
import numpy as np
|
|
from datetime import datetime, timedelta
|
|
from typing import Dict, Optional
|
|
import yfinance as yf
|
|
import logging
|
|
from pathlib import Path
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
class DataService:
|
|
"""Service for retrieving ETF data with fallback logic"""
|
|
|
|
def __init__(self):
|
|
# Use existing cache structure
|
|
self.base_dir = Path(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
|
|
self.cache_dir = self.base_dir / 'cache'
|
|
self.yf_cache_dir = self.cache_dir / 'yfinance_cache'
|
|
self.fmp_cache_dir = self.cache_dir / 'FMP_cache'
|
|
self.fmp_profiles_dir = self.fmp_cache_dir / 'etf_profiles'
|
|
self.fmp_historical_dir = self.fmp_cache_dir / 'historical_data'
|
|
self.fmp_holdings_dir = self.fmp_cache_dir / 'etf_holdings'
|
|
self.cache_timeout = timedelta(hours=1)
|
|
|
|
def get_etf_data(self, ticker: str) -> Dict:
|
|
"""Get ETF data using fallback logic:
|
|
1. Try FMP cache
|
|
2. Try FMP API
|
|
3. Try yfinance cache
|
|
4. Try yfinance
|
|
5. Use high yield estimates
|
|
"""
|
|
try:
|
|
# Try FMP cache first
|
|
fmp_cached_data = self._get_from_fmp_cache(ticker)
|
|
if fmp_cached_data:
|
|
logger.info(f"Using FMP cached data for {ticker}")
|
|
return fmp_cached_data
|
|
|
|
# Try FMP API
|
|
fmp_data = self._get_from_fmp(ticker)
|
|
if fmp_data:
|
|
logger.info(f"Using FMP data for {ticker}")
|
|
self._save_to_fmp_cache(ticker, fmp_data)
|
|
return fmp_data
|
|
|
|
# Try yfinance cache
|
|
yf_cached_data = self._get_from_yf_cache(ticker)
|
|
if yf_cached_data:
|
|
logger.info(f"Using yfinance cached data for {ticker}")
|
|
return yf_cached_data
|
|
|
|
# Try yfinance
|
|
yf_data = self._get_from_yfinance(ticker)
|
|
if yf_data:
|
|
logger.info(f"Using yfinance data for {ticker}")
|
|
self._save_to_yf_cache(ticker, yf_data)
|
|
return yf_data
|
|
|
|
# Use high yield estimates
|
|
logger.warning(f"Using high yield estimates for {ticker}")
|
|
return self._get_high_yield_estimates(ticker)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error fetching data for {ticker}: {str(e)}")
|
|
return self._get_high_yield_estimates(ticker)
|
|
|
|
def _get_from_fmp_cache(self, ticker: str) -> Optional[Dict]:
|
|
"""Get data from FMP cache if available and not expired"""
|
|
# Check profile cache
|
|
profile_file = self.fmp_profiles_dir / f"{ticker}.json"
|
|
if not profile_file.exists():
|
|
return None
|
|
|
|
try:
|
|
with open(profile_file, 'r') as f:
|
|
profile_data = json.load(f)
|
|
|
|
# Check if cache is expired
|
|
cache_time = datetime.fromisoformat(profile_data['timestamp'])
|
|
if datetime.now() - cache_time > self.cache_timeout:
|
|
return None
|
|
|
|
# Get historical data
|
|
hist_file = self.fmp_historical_dir / f"{ticker}.json"
|
|
if hist_file.exists():
|
|
with open(hist_file, 'r') as f:
|
|
hist_data = json.load(f)
|
|
else:
|
|
hist_data = {}
|
|
|
|
# Get holdings data
|
|
holdings_file = self.fmp_holdings_dir / f"{ticker}.json"
|
|
if holdings_file.exists():
|
|
with open(holdings_file, 'r') as f:
|
|
holdings_data = json.load(f)
|
|
else:
|
|
holdings_data = {}
|
|
|
|
# Combine all data
|
|
return {
|
|
'info': profile_data['data'],
|
|
'hist': hist_data.get('data', {}),
|
|
'holdings': holdings_data.get('data', {}),
|
|
'volatility': profile_data['data'].get('volatility', 0.0),
|
|
'max_drawdown': profile_data['data'].get('maxDrawdown', 0.0),
|
|
'sharpe_ratio': profile_data['data'].get('sharpeRatio', 0.0),
|
|
'sortino_ratio': profile_data['data'].get('sortinoRatio', 0.0),
|
|
'dividend_trend': profile_data['data'].get('dividendTrend', 0.0),
|
|
'age_years': profile_data['data'].get('ageYears', 0.0),
|
|
'is_new': profile_data['data'].get('ageYears', 0.0) < 2
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Error reading FMP cache for {ticker}: {str(e)}")
|
|
return None
|
|
|
|
def _get_from_yf_cache(self, ticker: str) -> Optional[Dict]:
|
|
"""Get data from yfinance cache if available and not expired"""
|
|
cache_file = self.yf_cache_dir / f"{ticker}_data.json"
|
|
if not cache_file.exists():
|
|
return None
|
|
|
|
try:
|
|
with open(cache_file, 'r') as f:
|
|
data = json.load(f)
|
|
|
|
# Check if cache is expired
|
|
cache_time = datetime.fromisoformat(data['timestamp'])
|
|
if datetime.now() - cache_time > self.cache_timeout:
|
|
return None
|
|
|
|
return data['data']
|
|
except Exception as e:
|
|
logger.warning(f"Error reading yfinance cache for {ticker}: {str(e)}")
|
|
return None
|
|
|
|
def _save_to_fmp_cache(self, ticker: str, data: Dict):
|
|
"""Save data to FMP cache"""
|
|
try:
|
|
# Save profile data
|
|
profile_data = {
|
|
'timestamp': datetime.now().isoformat(),
|
|
'data': data['info']
|
|
}
|
|
profile_file = self.fmp_profiles_dir / f"{ticker}.json"
|
|
with open(profile_file, 'w') as f:
|
|
json.dump(profile_data, f)
|
|
|
|
# Save historical data
|
|
if 'hist' in data:
|
|
hist_data = {
|
|
'timestamp': datetime.now().isoformat(),
|
|
'data': data['hist']
|
|
}
|
|
hist_file = self.fmp_historical_dir / f"{ticker}.json"
|
|
with open(hist_file, 'w') as f:
|
|
json.dump(hist_data, f)
|
|
|
|
# Save holdings data
|
|
if 'holdings' in data:
|
|
holdings_data = {
|
|
'timestamp': datetime.now().isoformat(),
|
|
'data': data['holdings']
|
|
}
|
|
holdings_file = self.fmp_holdings_dir / f"{ticker}.json"
|
|
with open(holdings_file, 'w') as f:
|
|
json.dump(holdings_data, f)
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Error saving FMP cache for {ticker}: {str(e)}")
|
|
|
|
def _save_to_yf_cache(self, ticker: str, data: Dict):
|
|
"""Save data to yfinance cache"""
|
|
try:
|
|
cache_data = {
|
|
'timestamp': datetime.now().isoformat(),
|
|
'data': data
|
|
}
|
|
cache_file = self.yf_cache_dir / f"{ticker}_data.json"
|
|
with open(cache_file, 'w') as f:
|
|
json.dump(cache_data, f)
|
|
except Exception as e:
|
|
logger.warning(f"Error saving yfinance cache for {ticker}: {str(e)}")
|
|
|
|
def _get_from_fmp(self, ticker: str) -> Optional[Dict]:
|
|
"""Get data from FMP API"""
|
|
# TODO: Implement FMP API integration
|
|
return None
|
|
|
|
def _get_from_yfinance(self, ticker: str) -> Optional[Dict]:
|
|
"""Get data from yfinance"""
|
|
try:
|
|
yf_ticker = yf.Ticker(ticker)
|
|
|
|
# Get basic info
|
|
info = yf_ticker.info
|
|
if not info:
|
|
return None
|
|
|
|
# Get historical data - use 5 years for better calculations
|
|
hist = yf_ticker.history(period="5y")
|
|
if hist.empty:
|
|
return None
|
|
|
|
# Get dividends with proper handling
|
|
try:
|
|
dividends = yf_ticker.dividends
|
|
if dividends is None or dividends.empty:
|
|
# Try to get dividend info from info
|
|
dividend_rate = info.get('dividendRate', 0)
|
|
if dividend_rate > 0:
|
|
# Create a synthetic dividend series
|
|
last_price = hist['Close'].iloc[-1]
|
|
annual_dividend = dividend_rate
|
|
monthly_dividend = annual_dividend / 12
|
|
dividends = pd.Series(monthly_dividend, index=hist.index)
|
|
else:
|
|
dividends = pd.Series(0, index=hist.index)
|
|
except Exception as e:
|
|
logger.warning(f"Error getting dividends for {ticker}: {str(e)}")
|
|
dividends = pd.Series(0, index=hist.index)
|
|
|
|
# Calculate metrics with proper annualization
|
|
hist['log_returns'] = np.log(hist['Close'] / hist['Close'].shift(1))
|
|
returns = hist['log_returns'].dropna()
|
|
|
|
# Calculate annualized volatility using daily log returns
|
|
volatility = returns.std() * np.sqrt(252)
|
|
|
|
# Calculate max drawdown using rolling window
|
|
rolling_max = hist['Close'].rolling(window=252, min_periods=1).max()
|
|
daily_drawdown = hist['Close'] / rolling_max - 1.0
|
|
max_drawdown = abs(daily_drawdown.min())
|
|
|
|
# Calculate annualized return
|
|
annual_return = returns.mean() * 252
|
|
|
|
# Calculate Sharpe and Sortino ratios with proper risk-free rate
|
|
risk_free_rate = 0.05 # Current 3-month Treasury yield
|
|
excess_returns = returns - risk_free_rate/252
|
|
|
|
# Sharpe Ratio
|
|
if volatility > 0:
|
|
sharpe_ratio = np.sqrt(252) * excess_returns.mean() / volatility
|
|
else:
|
|
sharpe_ratio = 0
|
|
|
|
# Sortino Ratio
|
|
negative_returns = returns[returns < 0]
|
|
if len(negative_returns) > 0:
|
|
downside_volatility = negative_returns.std() * np.sqrt(252)
|
|
if downside_volatility > 0:
|
|
sortino_ratio = np.sqrt(252) * excess_returns.mean() / downside_volatility
|
|
else:
|
|
sortino_ratio = 0
|
|
else:
|
|
sortino_ratio = 0
|
|
|
|
# Calculate dividend trend with better handling
|
|
try:
|
|
if not dividends.empty:
|
|
# Resample to monthly and handle missing values
|
|
monthly_div = dividends.resample('ME').sum().fillna(0)
|
|
if len(monthly_div) > 12:
|
|
# Calculate trailing 12-month dividends
|
|
earliest_ttm = monthly_div[-12:].sum()
|
|
latest_ttm = monthly_div[-1:].sum()
|
|
if earliest_ttm > 0:
|
|
dividend_trend = float((latest_ttm / earliest_ttm - 1))
|
|
else:
|
|
dividend_trend = 0.0
|
|
else:
|
|
# If less than 12 months of data, use the average
|
|
dividend_trend = float(monthly_div.mean()) if not monthly_div.empty else 0.0
|
|
else:
|
|
# Try to get dividend trend from info
|
|
dividend_rate = float(info.get('dividendRate', 0))
|
|
five_year_avg = float(info.get('fiveYearAvgDividendYield', 0))
|
|
if dividend_rate > 0 and five_year_avg > 0:
|
|
dividend_trend = float((dividend_rate / five_year_avg - 1))
|
|
else:
|
|
dividend_trend = 0.0
|
|
except Exception as e:
|
|
logger.warning(f"Error calculating dividend trend for {ticker}: {str(e)}")
|
|
dividend_trend = 0.0
|
|
|
|
# Ensure dividend_trend is a valid float
|
|
dividend_trend = float(dividend_trend) if dividend_trend is not None else 0.0
|
|
if not isinstance(dividend_trend, (int, float)) or pd.isna(dividend_trend):
|
|
dividend_trend = 0.0
|
|
|
|
# Calculate ETF age
|
|
inception_date = info.get('fundInceptionDate')
|
|
if inception_date:
|
|
try:
|
|
inception_date_dt = pd.to_datetime(inception_date, unit='s', utc=True)
|
|
age_years = (pd.Timestamp.now(tz='UTC') - inception_date_dt).days / 365.25
|
|
except:
|
|
age_years = None
|
|
else:
|
|
age_years = None
|
|
|
|
# Ensure all values are valid numbers and properly formatted
|
|
volatility = float(volatility) if volatility is not None else 0.0
|
|
max_drawdown = float(max_drawdown) if max_drawdown is not None else 0.0
|
|
sharpe_ratio = float(sharpe_ratio) if sharpe_ratio is not None else 0.0
|
|
sortino_ratio = float(sortino_ratio) if sortino_ratio is not None else 0.0
|
|
age_years = float(age_years) if age_years is not None else 0.0
|
|
|
|
# Format the response with proper types
|
|
response = {
|
|
'info': info,
|
|
'hist': hist.to_dict(),
|
|
'dividends': dividends.to_dict(),
|
|
'volatility': volatility,
|
|
'max_drawdown': max_drawdown,
|
|
'sharpe_ratio': sharpe_ratio,
|
|
'sortino_ratio': sortino_ratio,
|
|
'dividend_trend': dividend_trend,
|
|
'age_years': age_years,
|
|
'is_new': age_years < 2
|
|
}
|
|
|
|
# Ensure all numeric values are properly formatted
|
|
for key in ['volatility', 'max_drawdown', 'sharpe_ratio', 'sortino_ratio', 'dividend_trend', 'age_years']:
|
|
if key in response:
|
|
response[key] = float(response[key])
|
|
|
|
return response
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error fetching yfinance data for {ticker}: {str(e)}")
|
|
return None
|
|
|
|
def _get_high_yield_estimates(self, ticker: str) -> Dict:
|
|
"""Get conservative high yield estimates when no data is available"""
|
|
# Determine ETF type based on ticker
|
|
if ticker in ['JEPI', 'FEPI', 'MSTY']: # Income ETFs
|
|
max_drawdown = 0.10 # 10% for income ETFs
|
|
volatility = 0.15 # 15% volatility
|
|
sharpe_ratio = 0.8 # Lower Sharpe for income ETFs
|
|
sortino_ratio = 1.2 # Higher Sortino for income ETFs
|
|
dividend_trend = 0.05 # 5% dividend growth for income ETFs
|
|
elif ticker in ['VTI', 'VOO']: # Growth ETFs
|
|
max_drawdown = 0.25 # 25% for growth ETFs
|
|
volatility = 0.20 # 20% volatility
|
|
sharpe_ratio = 1.2 # Higher Sharpe for growth ETFs
|
|
sortino_ratio = 1.5 # Higher Sortino for growth ETFs
|
|
dividend_trend = 0.10 # 10% dividend growth for growth ETFs
|
|
else: # Balanced ETFs
|
|
max_drawdown = 0.20 # 20% for balanced ETFs
|
|
volatility = 0.18 # 18% volatility
|
|
sharpe_ratio = 1.0 # Moderate Sharpe for balanced ETFs
|
|
sortino_ratio = 1.3 # Moderate Sortino for balanced ETFs
|
|
dividend_trend = 0.07 # 7% dividend growth for balanced ETFs
|
|
|
|
return {
|
|
'info': {},
|
|
'hist': {},
|
|
'dividends': {},
|
|
'volatility': volatility,
|
|
'max_drawdown': max_drawdown,
|
|
'sharpe_ratio': sharpe_ratio,
|
|
'sortino_ratio': sortino_ratio,
|
|
'dividend_trend': dividend_trend,
|
|
'age_years': 3.0, # Conservative estimate
|
|
'is_new': False,
|
|
'is_estimated': True # Flag to indicate these are estimates
|
|
} |