feat: improve data source tracking and FMP API fallback mechanism
This commit is contained in:
parent
1fc54d5ea9
commit
81c6a1db48
@ -52,6 +52,32 @@ else:
|
||||
|
||||
FMP_BASE_URL = "https://financialmodelingprep.com/api/v3"
|
||||
|
||||
def test_fmp_data_fetching():
|
||||
"""Test FMP API data fetching with detailed logging."""
|
||||
try:
|
||||
logger.info("=== Starting FMP API Test ===")
|
||||
logger.info(f"FMP API Key available: {bool(FMP_API_KEY)}")
|
||||
logger.info(f"FMP API enabled: {USE_FMP_API}")
|
||||
|
||||
# Test a few high-yield ETFs
|
||||
test_tickers = ["JEPI", "JEPQ", "QYLD"]
|
||||
|
||||
for ticker in test_tickers:
|
||||
logger.info(f"\nTesting {ticker}:")
|
||||
data = fetch_etf_data_fmp(ticker)
|
||||
if data:
|
||||
logger.info(f"Successfully fetched data for {ticker}")
|
||||
logger.info(f"Data source: {data.get('data_source', 'Not specified')}")
|
||||
logger.info(f"Raw data: {data.get('raw_data', 'No raw data')}")
|
||||
else:
|
||||
logger.error(f"Failed to fetch data for {ticker}")
|
||||
|
||||
logger.info("=== FMP API Test Complete ===")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in FMP API test: {str(e)}")
|
||||
logger.error(traceback.format_exc())
|
||||
|
||||
# High-yield ETFs reference data
|
||||
HIGH_YIELD_ETFS = {
|
||||
"MSTY": {"expected_yield": 125.0, "frequency": "Monthly"},
|
||||
@ -617,46 +643,48 @@ def fetch_etf_data_fmp(ticker: str) -> Optional[Dict[str, Any]]:
|
||||
|
||||
# Get profile data for current price
|
||||
profile_url = f"{FMP_BASE_URL}/profile/{ticker}?apikey={FMP_API_KEY}"
|
||||
logger.info(f"Making FMP API call to {profile_url}")
|
||||
logger.info(f"[FMP API] Making profile request to: {profile_url}")
|
||||
profile_response = session.get(profile_url)
|
||||
st.session_state.api_calls += 1
|
||||
logger.info(f"FMP API call count: {st.session_state.api_calls}")
|
||||
logger.info(f"[FMP API] Profile response status: {profile_response.status_code}")
|
||||
logger.info(f"[FMP API] Profile response content: {profile_response.text[:500]}...") # Log first 500 chars
|
||||
|
||||
if profile_response.status_code != 200:
|
||||
logger.error(f"FMP API error for {ticker}: {profile_response.status_code}")
|
||||
logger.error(f"Response content: {profile_response.text}")
|
||||
logger.error(f"[FMP API] Error for {ticker}: {profile_response.status_code}")
|
||||
logger.error(f"[FMP API] Response content: {profile_response.text}")
|
||||
return None
|
||||
|
||||
profile_data = profile_response.json()
|
||||
logger.info(f"FMP profile response for {ticker}: {profile_data}")
|
||||
logger.info(f"[FMP API] Profile data for {ticker}: {profile_data}")
|
||||
|
||||
if not profile_data or not isinstance(profile_data, list) or len(profile_data) == 0:
|
||||
logger.warning(f"No profile data found for {ticker} in FMP")
|
||||
logger.warning(f"[FMP API] No profile data found for {ticker}")
|
||||
return None
|
||||
|
||||
profile = profile_data[0]
|
||||
current_price = float(profile.get('price', 0))
|
||||
if current_price <= 0:
|
||||
logger.error(f"Invalid price for {ticker}: {current_price}")
|
||||
logger.error(f"[FMP API] Invalid price for {ticker}: {current_price}")
|
||||
return None
|
||||
|
||||
# Get dividend history
|
||||
dividend_url = f"{FMP_BASE_URL}/historical-price-full/stock_dividend/{ticker}?apikey={FMP_API_KEY}"
|
||||
logger.info(f"Making FMP API call to {dividend_url}")
|
||||
logger.info(f"[FMP API] Making dividend request to: {dividend_url}")
|
||||
dividend_response = session.get(dividend_url)
|
||||
st.session_state.api_calls += 1
|
||||
logger.info(f"FMP API call count: {st.session_state.api_calls}")
|
||||
logger.info(f"[FMP API] Dividend response status: {dividend_response.status_code}")
|
||||
logger.info(f"[FMP API] Dividend response content: {dividend_response.text[:500]}...") # Log first 500 chars
|
||||
|
||||
if dividend_response.status_code != 200:
|
||||
logger.error(f"FMP API error for dividend data: {dividend_response.status_code}")
|
||||
logger.error(f"Response content: {dividend_response.text}")
|
||||
logger.error(f"[FMP API] Error for dividend data: {dividend_response.status_code}")
|
||||
logger.error(f"[FMP API] Response content: {dividend_response.text}")
|
||||
return None
|
||||
|
||||
dividend_data = dividend_response.json()
|
||||
logger.info(f"FMP dividend response for {ticker}: {dividend_data}")
|
||||
logger.info(f"[FMP API] Dividend data for {ticker}: {dividend_data}")
|
||||
|
||||
if not dividend_data or "historical" not in dividend_data or not dividend_data["historical"]:
|
||||
logger.warning(f"No dividend history found for {ticker}")
|
||||
logger.warning(f"[FMP API] No dividend history found for {ticker}")
|
||||
return None
|
||||
|
||||
# Calculate TTM dividend
|
||||
@ -669,7 +697,7 @@ def fetch_etf_data_fmp(ticker: str) -> Optional[Dict[str, Any]]:
|
||||
recent_dividends = dividends[dividends["date"] >= one_year_ago]
|
||||
|
||||
if recent_dividends.empty:
|
||||
logger.warning(f"No recent dividends found for {ticker}")
|
||||
logger.warning(f"[FMP API] No recent dividends found for {ticker}")
|
||||
return None
|
||||
|
||||
# Calculate TTM dividend
|
||||
@ -677,16 +705,16 @@ def fetch_etf_data_fmp(ticker: str) -> Optional[Dict[str, Any]]:
|
||||
|
||||
# Calculate yield
|
||||
yield_pct = (ttm_dividend / current_price) * 100
|
||||
logger.info(f"Calculated yield for {ticker}: {yield_pct:.2f}% (TTM dividend: ${ttm_dividend:.2f}, Price: ${current_price:.2f})")
|
||||
logger.info(f"[FMP API] Calculated yield for {ticker}: {yield_pct:.2f}% (TTM dividend: ${ttm_dividend:.2f}, Price: ${current_price:.2f})")
|
||||
|
||||
# For high-yield ETFs, verify the yield is reasonable
|
||||
if ticker in HIGH_YIELD_ETFS:
|
||||
expected_yield = HIGH_YIELD_ETFS[ticker]["expected_yield"]
|
||||
if yield_pct < expected_yield * 0.5: # If yield is less than 50% of expected
|
||||
logger.error(f"Calculated yield {yield_pct:.2f}% for {ticker} is much lower than expected {expected_yield}%")
|
||||
logger.error(f"TTM dividend: ${ttm_dividend:.2f}")
|
||||
logger.error(f"Current price: ${current_price:.2f}")
|
||||
logger.error(f"Recent dividends:\n{recent_dividends}")
|
||||
logger.error(f"[FMP API] Calculated yield {yield_pct:.2f}% for {ticker} is much lower than expected {expected_yield}%")
|
||||
logger.error(f"[FMP API] TTM dividend: ${ttm_dividend:.2f}")
|
||||
logger.error(f"[FMP API] Current price: ${current_price:.2f}")
|
||||
logger.error(f"[FMP API] Recent dividends:\n{recent_dividends}")
|
||||
|
||||
# Determine distribution period
|
||||
if len(recent_dividends) >= 2:
|
||||
@ -708,13 +736,18 @@ def fetch_etf_data_fmp(ticker: str) -> Optional[Dict[str, Any]]:
|
||||
"Price": current_price,
|
||||
"Yield (%)": yield_pct,
|
||||
"Distribution Period": dist_period,
|
||||
"Risk Level": "High" if ticker in HIGH_YIELD_ETFS else "Moderate"
|
||||
"Risk Level": "High" if ticker in HIGH_YIELD_ETFS else "Moderate",
|
||||
"data_source": "FMP API", # Add data source identifier
|
||||
"raw_data": { # Store raw data for debugging
|
||||
"profile": profile,
|
||||
"dividend_history": dividend_data["historical"][:5] # Store first 5 dividend records
|
||||
}
|
||||
}
|
||||
logger.info(f"FMP data for {ticker}: {etf_data}")
|
||||
logger.info(f"[FMP API] Final data for {ticker}: {etf_data}")
|
||||
return etf_data
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching FMP data for {ticker}: {str(e)}")
|
||||
logger.error(f"[FMP API] Error fetching data for {ticker}: {str(e)}")
|
||||
logger.error(traceback.format_exc())
|
||||
return None
|
||||
|
||||
@ -759,7 +792,8 @@ def fetch_etf_data_yfinance(ticker: str) -> Optional[Dict[str, Any]]:
|
||||
"Ticker": ticker,
|
||||
"Price": current_price,
|
||||
"Yield (%)": yield_pct,
|
||||
"Risk Level": "High" # Default for high-yield ETFs
|
||||
"Risk Level": "High", # Default for high-yield ETFs
|
||||
"data_source": "yfinance" # Add data source identifier
|
||||
}
|
||||
logger.info(f"yfinance data for {ticker}: {etf_data}")
|
||||
return etf_data
|
||||
@ -787,6 +821,8 @@ def fetch_etf_data(tickers: List[str]) -> pd.DataFrame:
|
||||
logger.info("=== Starting ETF data fetch ===")
|
||||
logger.info(f"Force refresh enabled: {st.session_state.get('force_refresh_data', False)}")
|
||||
logger.info(f"Cache directory: {cache_dir.absolute()}")
|
||||
logger.info(f"FMP API enabled: {USE_FMP_API}")
|
||||
logger.info(f"FMP API key available: {bool(FMP_API_KEY)}")
|
||||
|
||||
for ticker in tickers:
|
||||
if not ticker: # Skip empty tickers
|
||||
@ -821,9 +857,10 @@ def fetch_etf_data(tickers: List[str]) -> pd.DataFrame:
|
||||
|
||||
# Try FMP first if enabled
|
||||
if USE_FMP_API and FMP_API_KEY:
|
||||
logger.info(f"Making FMP API call for {ticker}")
|
||||
logger.info(f"Attempting to fetch data from FMP API for {ticker}")
|
||||
etf_data = fetch_etf_data_fmp(ticker)
|
||||
if etf_data is not None:
|
||||
logger.info(f"Successfully fetched data from FMP API for {ticker}")
|
||||
# Cache the data
|
||||
try:
|
||||
cache_data = {
|
||||
@ -841,11 +878,14 @@ def fetch_etf_data(tickers: List[str]) -> pd.DataFrame:
|
||||
st.session_state.api_calls += 1
|
||||
logger.info(f"Total API calls: {st.session_state.api_calls}")
|
||||
continue
|
||||
else:
|
||||
logger.warning(f"FMP API fetch failed for {ticker}, falling back to yfinance")
|
||||
|
||||
# If FMP fails, try yfinance
|
||||
logger.info(f"Falling back to yfinance for {ticker}")
|
||||
logger.info(f"Fetching data from yfinance for {ticker}")
|
||||
etf_data = fetch_etf_data_yfinance(ticker)
|
||||
if etf_data is not None:
|
||||
logger.info(f"Successfully fetched data from yfinance for {ticker}")
|
||||
# Cache the data
|
||||
try:
|
||||
cache_data = {
|
||||
@ -870,7 +910,8 @@ def fetch_etf_data(tickers: List[str]) -> pd.DataFrame:
|
||||
"Price": 25.0, # Default price for fallback
|
||||
"Yield (%)": HIGH_YIELD_ETFS[ticker]["expected_yield"],
|
||||
"Distribution Period": HIGH_YIELD_ETFS[ticker]["frequency"],
|
||||
"Risk Level": "High"
|
||||
"Risk Level": "High",
|
||||
"data_source": "HIGH_YIELD_ETFS"
|
||||
}
|
||||
data[ticker] = etf_data
|
||||
else:
|
||||
@ -894,6 +935,11 @@ def fetch_etf_data(tickers: List[str]) -> pd.DataFrame:
|
||||
if (df["Yield (%)"] <= 0).any():
|
||||
st.warning("Some ETFs have zero or negative yields")
|
||||
|
||||
# Log data sources used
|
||||
if "data_source" in df.columns:
|
||||
source_counts = df["data_source"].value_counts()
|
||||
logger.info(f"Data sources used:\n{source_counts}")
|
||||
|
||||
logger.info(f"Final DataFrame:\n{df}")
|
||||
return df
|
||||
|
||||
@ -1062,9 +1108,29 @@ def portfolio_summary(final_alloc: pd.DataFrame) -> None:
|
||||
display_df = final_alloc.copy()
|
||||
display_df["Monthly Income"] = display_df["Income Contributed ($)"] / 12
|
||||
|
||||
# Ensure data_source column exists and rename it for display
|
||||
if "data_source" in display_df.columns:
|
||||
display_df = display_df.rename(columns={"data_source": "Data Source"})
|
||||
else:
|
||||
display_df["Data Source"] = "Unknown"
|
||||
|
||||
# Select and order columns for display
|
||||
display_columns = [
|
||||
"Ticker",
|
||||
"Allocation (%)",
|
||||
"Yield (%)",
|
||||
"Price",
|
||||
"Shares",
|
||||
"Capital Allocated ($)",
|
||||
"Monthly Income",
|
||||
"Income Contributed ($)",
|
||||
"Risk Level",
|
||||
"Data Source"
|
||||
]
|
||||
|
||||
# Format the display
|
||||
st.dataframe(
|
||||
display_df.style.format({
|
||||
display_df[display_columns].style.format({
|
||||
"Allocation (%)": "{:.2f}%",
|
||||
"Yield (%)": "{:.2f}%",
|
||||
"Price": "${:,.2f}",
|
||||
@ -1073,6 +1139,26 @@ def portfolio_summary(final_alloc: pd.DataFrame) -> None:
|
||||
"Monthly Income": "${:,.2f}",
|
||||
"Income Contributed ($)": "${:,.2f}"
|
||||
}),
|
||||
column_config={
|
||||
"Ticker": st.column_config.TextColumn("Ticker", disabled=True),
|
||||
"Allocation (%)": st.column_config.NumberColumn(
|
||||
"Allocation (%)",
|
||||
min_value=0.0,
|
||||
max_value=100.0,
|
||||
step=0.1,
|
||||
format="%.1f",
|
||||
required=True
|
||||
),
|
||||
"Yield (%)": st.column_config.TextColumn("Yield (%)", disabled=True),
|
||||
"Price": st.column_config.TextColumn("Price", disabled=True),
|
||||
"Shares": st.column_config.TextColumn("Shares", disabled=True),
|
||||
"Capital Allocated ($)": st.column_config.TextColumn("Capital Allocated ($)", disabled=True),
|
||||
"Monthly Income": st.column_config.TextColumn("Monthly Income", disabled=True),
|
||||
"Income Contributed ($)": st.column_config.TextColumn("Income Contributed ($)", disabled=True),
|
||||
"Risk Level": st.column_config.TextColumn("Risk Level", disabled=True),
|
||||
"Data Source": st.column_config.TextColumn("Data Source", disabled=True)
|
||||
},
|
||||
hide_index=True,
|
||||
use_container_width=True
|
||||
)
|
||||
|
||||
@ -1878,14 +1964,14 @@ with st.sidebar:
|
||||
# Cache clearing options
|
||||
col1, col2 = st.columns(2)
|
||||
with col1:
|
||||
if st.button("Clear All Cache"):
|
||||
if st.button("Clear All Cache", key="clear_all_cache"):
|
||||
clear_cache()
|
||||
st.success("All cache files cleared!")
|
||||
st.session_state.api_calls = 0
|
||||
|
||||
with col2:
|
||||
ticker_to_clear = st.text_input("Clear cache for ticker:", key="cache_ticker")
|
||||
if st.button("Clear") and ticker_to_clear:
|
||||
if st.button("Clear", key="clear_single_cache") and ticker_to_clear:
|
||||
clear_cache(ticker_to_clear)
|
||||
st.success(f"Cache cleared for {ticker_to_clear.upper()}")
|
||||
|
||||
@ -1897,6 +1983,12 @@ with st.sidebar:
|
||||
parallel_processing = st.checkbox("Enable Parallel Processing", value=True,
|
||||
help="Fetch data for multiple ETFs simultaneously")
|
||||
|
||||
# Add FMP API test button
|
||||
st.sidebar.subheader("FMP API Testing")
|
||||
if st.sidebar.button("Test FMP API", key="test_fmp_api_button"):
|
||||
test_fmp_data_fetching()
|
||||
st.sidebar.success("Test completed. Check logs for details.")
|
||||
|
||||
# Display results and interactive allocation adjustment UI after simulation is run
|
||||
if st.session_state.simulation_run and st.session_state.df_data is not None:
|
||||
df = st.session_state.df_data
|
||||
|
||||
Loading…
Reference in New Issue
Block a user