Pascal/ETF_Suite_Portal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: improve data source tracking and FMP API fallback mechanism

Browse Source
This commit is contained in:
Pascal BIBEHE 2025-05-28 14:41:10 +02:00
parent 1fc54d5ea9
commit 81c6a1db48
1 changed files with 121 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

150
pages/ETF_Portfolio_Builder.py
View File

@ -52,6 +52,32 @@ else:
FMP_BASE_URL = "https://financialmodelingprep.com/api/v3" FMP_BASE_URL = "https://financialmodelingprep.com/api/v3"
def test_fmp_data_fetching():
"""Test FMP API data fetching with detailed logging."""
try:
logger.info("=== Starting FMP API Test ===")
logger.info(f"FMP API Key available: {bool(FMP_API_KEY)}")
logger.info(f"FMP API enabled: {USE_FMP_API}")
# Test a few high-yield ETFs
test_tickers = ["JEPI", "JEPQ", "QYLD"]
for ticker in test_tickers:
logger.info(f"\nTesting {ticker}:")
data = fetch_etf_data_fmp(ticker)
if data:
logger.info(f"Successfully fetched data for {ticker}")
logger.info(f"Data source: {data.get('data_source', 'Not specified')}")
logger.info(f"Raw data: {data.get('raw_data', 'No raw data')}")
else:
logger.error(f"Failed to fetch data for {ticker}")
logger.info("=== FMP API Test Complete ===")
except Exception as e:
logger.error(f"Error in FMP API test: {str(e)}")
logger.error(traceback.format_exc())
# High-yield ETFs reference data # High-yield ETFs reference data
HIGH_YIELD_ETFS = { HIGH_YIELD_ETFS = {
"MSTY": {"expected_yield": 125.0, "frequency": "Monthly"}, "MSTY": {"expected_yield": 125.0, "frequency": "Monthly"},
@ -617,46 +643,48 @@ def fetch_etf_data_fmp(ticker: str) -> Optional[Dict[str, Any]]:
# Get profile data for current price # Get profile data for current price
profile_url = f"{FMP_BASE_URL}/profile/{ticker}?apikey={FMP_API_KEY}" profile_url = f"{FMP_BASE_URL}/profile/{ticker}?apikey={FMP_API_KEY}"
logger.info(f"Making FMP API call to {profile_url}") logger.info(f"[FMP API] Making profile request to: {profile_url}")
profile_response = session.get(profile_url) profile_response = session.get(profile_url)
st.session_state.api_calls += 1 st.session_state.api_calls += 1
logger.info(f"FMP API call count: {st.session_state.api_calls}") logger.info(f"[FMP API] Profile response status: {profile_response.status_code}")
logger.info(f"[FMP API] Profile response content: {profile_response.text[:500]}...") # Log first 500 chars
if profile_response.status_code != 200: if profile_response.status_code != 200:
logger.error(f"FMP API error for {ticker}: {profile_response.status_code}") logger.error(f"[FMP API] Error for {ticker}: {profile_response.status_code}")
logger.error(f"Response content: {profile_response.text}") logger.error(f"[FMP API] Response content: {profile_response.text}")
return None return None
profile_data = profile_response.json() profile_data = profile_response.json()
logger.info(f"FMP profile response for {ticker}: {profile_data}") logger.info(f"[FMP API] Profile data for {ticker}: {profile_data}")
if not profile_data or not isinstance(profile_data, list) or len(profile_data) == 0: if not profile_data or not isinstance(profile_data, list) or len(profile_data) == 0:
logger.warning(f"No profile data found for {ticker} in FMP") logger.warning(f"[FMP API] No profile data found for {ticker}")
return None return None
profile = profile_data[0] profile = profile_data[0]
current_price = float(profile.get('price', 0)) current_price = float(profile.get('price', 0))
if current_price <= 0: if current_price <= 0:
logger.error(f"Invalid price for {ticker}: {current_price}") logger.error(f"[FMP API] Invalid price for {ticker}: {current_price}")
return None return None
# Get dividend history # Get dividend history
dividend_url = f"{FMP_BASE_URL}/historical-price-full/stock_dividend/{ticker}?apikey={FMP_API_KEY}" dividend_url = f"{FMP_BASE_URL}/historical-price-full/stock_dividend/{ticker}?apikey={FMP_API_KEY}"
logger.info(f"Making FMP API call to {dividend_url}") logger.info(f"[FMP API] Making dividend request to: {dividend_url}")
dividend_response = session.get(dividend_url) dividend_response = session.get(dividend_url)
st.session_state.api_calls += 1 st.session_state.api_calls += 1
logger.info(f"FMP API call count: {st.session_state.api_calls}") logger.info(f"[FMP API] Dividend response status: {dividend_response.status_code}")
logger.info(f"[FMP API] Dividend response content: {dividend_response.text[:500]}...") # Log first 500 chars
if dividend_response.status_code != 200: if dividend_response.status_code != 200:
logger.error(f"FMP API error for dividend data: {dividend_response.status_code}") logger.error(f"[FMP API] Error for dividend data: {dividend_response.status_code}")
logger.error(f"Response content: {dividend_response.text}") logger.error(f"[FMP API] Response content: {dividend_response.text}")
return None return None
dividend_data = dividend_response.json() dividend_data = dividend_response.json()
logger.info(f"FMP dividend response for {ticker}: {dividend_data}") logger.info(f"[FMP API] Dividend data for {ticker}: {dividend_data}")
if not dividend_data or "historical" not in dividend_data or not dividend_data["historical"]: if not dividend_data or "historical" not in dividend_data or not dividend_data["historical"]:
logger.warning(f"No dividend history found for {ticker}") logger.warning(f"[FMP API] No dividend history found for {ticker}")
return None return None
# Calculate TTM dividend # Calculate TTM dividend
@ -669,7 +697,7 @@ def fetch_etf_data_fmp(ticker: str) -> Optional[Dict[str, Any]]:
recent_dividends = dividends[dividends["date"] >= one_year_ago] recent_dividends = dividends[dividends["date"] >= one_year_ago]
if recent_dividends.empty: if recent_dividends.empty:
logger.warning(f"No recent dividends found for {ticker}") logger.warning(f"[FMP API] No recent dividends found for {ticker}")
return None return None
# Calculate TTM dividend # Calculate TTM dividend
@ -677,16 +705,16 @@ def fetch_etf_data_fmp(ticker: str) -> Optional[Dict[str, Any]]:
# Calculate yield # Calculate yield
yield_pct = (ttm_dividend / current_price) * 100 yield_pct = (ttm_dividend / current_price) * 100
logger.info(f"Calculated yield for {ticker}: {yield_pct:.2f}% (TTM dividend: ${ttm_dividend:.2f}, Price: ${current_price:.2f})") logger.info(f"[FMP API] Calculated yield for {ticker}: {yield_pct:.2f}% (TTM dividend: ${ttm_dividend:.2f}, Price: ${current_price:.2f})")
# For high-yield ETFs, verify the yield is reasonable # For high-yield ETFs, verify the yield is reasonable
if ticker in HIGH_YIELD_ETFS: if ticker in HIGH_YIELD_ETFS:
expected_yield = HIGH_YIELD_ETFS[ticker]["expected_yield"] expected_yield = HIGH_YIELD_ETFS[ticker]["expected_yield"]
if yield_pct < expected_yield * 0.5: # If yield is less than 50% of expected if yield_pct < expected_yield * 0.5: # If yield is less than 50% of expected
logger.error(f"Calculated yield {yield_pct:.2f}% for {ticker} is much lower than expected {expected_yield}%") logger.error(f"[FMP API] Calculated yield {yield_pct:.2f}% for {ticker} is much lower than expected {expected_yield}%")
logger.error(f"TTM dividend: ${ttm_dividend:.2f}") logger.error(f"[FMP API] TTM dividend: ${ttm_dividend:.2f}")
logger.error(f"Current price: ${current_price:.2f}") logger.error(f"[FMP API] Current price: ${current_price:.2f}")
logger.error(f"Recent dividends:\n{recent_dividends}") logger.error(f"[FMP API] Recent dividends:\n{recent_dividends}")
# Determine distribution period # Determine distribution period
if len(recent_dividends) >= 2: if len(recent_dividends) >= 2:
@ -708,13 +736,18 @@ def fetch_etf_data_fmp(ticker: str) -> Optional[Dict[str, Any]]:
"Price": current_price, "Price": current_price,
"Yield (%)": yield_pct, "Yield (%)": yield_pct,
"Distribution Period": dist_period, "Distribution Period": dist_period,
"Risk Level": "High" if ticker in HIGH_YIELD_ETFS else "Moderate" "Risk Level": "High" if ticker in HIGH_YIELD_ETFS else "Moderate",
"data_source": "FMP API", # Add data source identifier
"raw_data": { # Store raw data for debugging
"profile": profile,
"dividend_history": dividend_data["historical"][:5] # Store first 5 dividend records
} }
logger.info(f"FMP data for {ticker}: {etf_data}") }
logger.info(f"[FMP API] Final data for {ticker}: {etf_data}")
return etf_data return etf_data
except Exception as e: except Exception as e:
logger.error(f"Error fetching FMP data for {ticker}: {str(e)}") logger.error(f"[FMP API] Error fetching data for {ticker}: {str(e)}")
logger.error(traceback.format_exc()) logger.error(traceback.format_exc())
return None return None
@ -759,7 +792,8 @@ def fetch_etf_data_yfinance(ticker: str) -> Optional[Dict[str, Any]]:
"Ticker": ticker, "Ticker": ticker,
"Price": current_price, "Price": current_price,
"Yield (%)": yield_pct, "Yield (%)": yield_pct,
"Risk Level": "High" # Default for high-yield ETFs "Risk Level": "High", # Default for high-yield ETFs
"data_source": "yfinance" # Add data source identifier
} }
logger.info(f"yfinance data for {ticker}: {etf_data}") logger.info(f"yfinance data for {ticker}: {etf_data}")
return etf_data return etf_data
@ -787,6 +821,8 @@ def fetch_etf_data(tickers: List[str]) -> pd.DataFrame:
logger.info("=== Starting ETF data fetch ===") logger.info("=== Starting ETF data fetch ===")
logger.info(f"Force refresh enabled: {st.session_state.get('force_refresh_data', False)}") logger.info(f"Force refresh enabled: {st.session_state.get('force_refresh_data', False)}")
logger.info(f"Cache directory: {cache_dir.absolute()}") logger.info(f"Cache directory: {cache_dir.absolute()}")
logger.info(f"FMP API enabled: {USE_FMP_API}")
logger.info(f"FMP API key available: {bool(FMP_API_KEY)}")
for ticker in tickers: for ticker in tickers:
if not ticker: # Skip empty tickers if not ticker: # Skip empty tickers
@ -821,9 +857,10 @@ def fetch_etf_data(tickers: List[str]) -> pd.DataFrame:
# Try FMP first if enabled # Try FMP first if enabled
if USE_FMP_API and FMP_API_KEY: if USE_FMP_API and FMP_API_KEY:
logger.info(f"Making FMP API call for {ticker}") logger.info(f"Attempting to fetch data from FMP API for {ticker}")
etf_data = fetch_etf_data_fmp(ticker) etf_data = fetch_etf_data_fmp(ticker)
if etf_data is not None: if etf_data is not None:
logger.info(f"Successfully fetched data from FMP API for {ticker}")
# Cache the data # Cache the data
try: try:
cache_data = { cache_data = {
@ -841,11 +878,14 @@ def fetch_etf_data(tickers: List[str]) -> pd.DataFrame:
st.session_state.api_calls += 1 st.session_state.api_calls += 1
logger.info(f"Total API calls: {st.session_state.api_calls}") logger.info(f"Total API calls: {st.session_state.api_calls}")
continue continue
else:
logger.warning(f"FMP API fetch failed for {ticker}, falling back to yfinance")
# If FMP fails, try yfinance # If FMP fails, try yfinance
logger.info(f"Falling back to yfinance for {ticker}") logger.info(f"Fetching data from yfinance for {ticker}")
etf_data = fetch_etf_data_yfinance(ticker) etf_data = fetch_etf_data_yfinance(ticker)
if etf_data is not None: if etf_data is not None:
logger.info(f"Successfully fetched data from yfinance for {ticker}")
# Cache the data # Cache the data
try: try:
cache_data = { cache_data = {
@ -870,7 +910,8 @@ def fetch_etf_data(tickers: List[str]) -> pd.DataFrame:
"Price": 25.0, # Default price for fallback "Price": 25.0, # Default price for fallback
"Yield (%)": HIGH_YIELD_ETFS[ticker]["expected_yield"], "Yield (%)": HIGH_YIELD_ETFS[ticker]["expected_yield"],
"Distribution Period": HIGH_YIELD_ETFS[ticker]["frequency"], "Distribution Period": HIGH_YIELD_ETFS[ticker]["frequency"],
"Risk Level": "High" "Risk Level": "High",
"data_source": "HIGH_YIELD_ETFS"
} }
data[ticker] = etf_data data[ticker] = etf_data
else: else:
@ -894,6 +935,11 @@ def fetch_etf_data(tickers: List[str]) -> pd.DataFrame:
if (df["Yield (%)"] <= 0).any(): if (df["Yield (%)"] <= 0).any():
st.warning("Some ETFs have zero or negative yields") st.warning("Some ETFs have zero or negative yields")
# Log data sources used
if "data_source" in df.columns:
source_counts = df["data_source"].value_counts()
logger.info(f"Data sources used:\n{source_counts}")
logger.info(f"Final DataFrame:\n{df}") logger.info(f"Final DataFrame:\n{df}")
return df return df
@ -1062,9 +1108,29 @@ def portfolio_summary(final_alloc: pd.DataFrame) -> None:
display_df = final_alloc.copy() display_df = final_alloc.copy()
display_df["Monthly Income"] = display_df["Income Contributed ($)"] / 12 display_df["Monthly Income"] = display_df["Income Contributed ($)"] / 12
# Ensure data_source column exists and rename it for display
if "data_source" in display_df.columns:
display_df = display_df.rename(columns={"data_source": "Data Source"})
else:
display_df["Data Source"] = "Unknown"
# Select and order columns for display
display_columns = [
"Ticker",
"Allocation (%)",
"Yield (%)",
"Price",
"Shares",
"Capital Allocated ($)",
"Monthly Income",
"Income Contributed ($)",
"Risk Level",
"Data Source"
]
# Format the display # Format the display
st.dataframe( st.dataframe(
display_df.style.format({ display_df[display_columns].style.format({
"Allocation (%)": "{:.2f}%", "Allocation (%)": "{:.2f}%",
"Yield (%)": "{:.2f}%", "Yield (%)": "{:.2f}%",
"Price": "${:,.2f}", "Price": "${:,.2f}",
@ -1073,6 +1139,26 @@ def portfolio_summary(final_alloc: pd.DataFrame) -> None:
"Monthly Income": "${:,.2f}", "Monthly Income": "${:,.2f}",
"Income Contributed ($)": "${:,.2f}" "Income Contributed ($)": "${:,.2f}"
}), }),
column_config={
"Ticker": st.column_config.TextColumn("Ticker", disabled=True),
"Allocation (%)": st.column_config.NumberColumn(
"Allocation (%)",
min_value=0.0,
max_value=100.0,
step=0.1,
format="%.1f",
required=True
),
"Yield (%)": st.column_config.TextColumn("Yield (%)", disabled=True),
"Price": st.column_config.TextColumn("Price", disabled=True),
"Shares": st.column_config.TextColumn("Shares", disabled=True),
"Capital Allocated ($)": st.column_config.TextColumn("Capital Allocated ($)", disabled=True),
"Monthly Income": st.column_config.TextColumn("Monthly Income", disabled=True),
"Income Contributed ($)": st.column_config.TextColumn("Income Contributed ($)", disabled=True),
"Risk Level": st.column_config.TextColumn("Risk Level", disabled=True),
"Data Source": st.column_config.TextColumn("Data Source", disabled=True)
},
hide_index=True,
use_container_width=True use_container_width=True
) )
@ -1878,14 +1964,14 @@ with st.sidebar:
# Cache clearing options # Cache clearing options
col1, col2 = st.columns(2) col1, col2 = st.columns(2)
with col1: with col1:
if st.button("Clear All Cache"): if st.button("Clear All Cache", key="clear_all_cache"):
clear_cache() clear_cache()
st.success("All cache files cleared!") st.success("All cache files cleared!")
st.session_state.api_calls = 0 st.session_state.api_calls = 0
with col2: with col2:
ticker_to_clear = st.text_input("Clear cache for ticker:", key="cache_ticker") ticker_to_clear = st.text_input("Clear cache for ticker:", key="cache_ticker")
if st.button("Clear") and ticker_to_clear: if st.button("Clear", key="clear_single_cache") and ticker_to_clear:
clear_cache(ticker_to_clear) clear_cache(ticker_to_clear)
st.success(f"Cache cleared for {ticker_to_clear.upper()}") st.success(f"Cache cleared for {ticker_to_clear.upper()}")
@ -1897,6 +1983,12 @@ with st.sidebar:
parallel_processing = st.checkbox("Enable Parallel Processing", value=True, parallel_processing = st.checkbox("Enable Parallel Processing", value=True,
help="Fetch data for multiple ETFs simultaneously") help="Fetch data for multiple ETFs simultaneously")
# Add FMP API test button
st.sidebar.subheader("FMP API Testing")
if st.sidebar.button("Test FMP API", key="test_fmp_api_button"):
test_fmp_data_fetching()
st.sidebar.success("Test completed. Check logs for details.")
# Display results and interactive allocation adjustment UI after simulation is run # Display results and interactive allocation adjustment UI after simulation is run
if st.session_state.simulation_run and st.session_state.df_data is not None: if st.session_state.simulation_run and st.session_state.df_data is not None:
df = st.session_state.df_data df = st.session_state.df_data