feat: improve data source tracking and FMP API fallback mechanism

pages/ETF_Portfolio_Builder.py

@@ -52,6 +52,32 @@ else:
 
 FMP_BASE_URL = "https://financialmodelingprep.com/api/v3"
 
+def test_fmp_data_fetching():
+    """Test FMP API data fetching with detailed logging."""
+    try:
+        logger.info("=== Starting FMP API Test ===")
+        logger.info(f"FMP API Key available: {bool(FMP_API_KEY)}")
+        logger.info(f"FMP API enabled: {USE_FMP_API}")
+        
+        # Test a few high-yield ETFs
+        test_tickers = ["JEPI", "JEPQ", "QYLD"]
+        
+        for ticker in test_tickers:
+            logger.info(f"\nTesting {ticker}:")
+            data = fetch_etf_data_fmp(ticker)
+            if data:
+                logger.info(f"Successfully fetched data for {ticker}")
+                logger.info(f"Data source: {data.get('data_source', 'Not specified')}")
+                logger.info(f"Raw data: {data.get('raw_data', 'No raw data')}")
+            else:
+                logger.error(f"Failed to fetch data for {ticker}")
+        
+        logger.info("=== FMP API Test Complete ===")
+        
+    except Exception as e:
+        logger.error(f"Error in FMP API test: {str(e)}")
+        logger.error(traceback.format_exc())
+
 # High-yield ETFs reference data
 HIGH_YIELD_ETFS = {
     "MSTY": {"expected_yield": 125.0, "frequency": "Monthly"},
@@ -617,46 +643,48 @@ def fetch_etf_data_fmp(ticker: str) -> Optional[Dict[str, Any]]:
         
         # Get profile data for current price
         profile_url = f"{FMP_BASE_URL}/profile/{ticker}?apikey={FMP_API_KEY}"
-        logger.info(f"Making FMP API call to {profile_url}")
+        logger.info(f"[FMP API] Making profile request to: {profile_url}")
         profile_response = session.get(profile_url)
         st.session_state.api_calls += 1
-        logger.info(f"FMP API call count: {st.session_state.api_calls}")
+        logger.info(f"[FMP API] Profile response status: {profile_response.status_code}")
+        logger.info(f"[FMP API] Profile response content: {profile_response.text[:500]}...")  # Log first 500 chars
         
         if profile_response.status_code != 200:
-            logger.error(f"FMP API error for {ticker}: {profile_response.status_code}")
+            logger.error(f"[FMP API] Error for {ticker}: {profile_response.status_code}")
-            logger.error(f"Response content: {profile_response.text}")
+            logger.error(f"[FMP API] Response content: {profile_response.text}")
             return None
         
         profile_data = profile_response.json()
-        logger.info(f"FMP profile response for {ticker}: {profile_data}")
+        logger.info(f"[FMP API] Profile data for {ticker}: {profile_data}")
         
         if not profile_data or not isinstance(profile_data, list) or len(profile_data) == 0:
-            logger.warning(f"No profile data found for {ticker} in FMP")
+            logger.warning(f"[FMP API] No profile data found for {ticker}")
             return None
         
         profile = profile_data[0]
         current_price = float(profile.get('price', 0))
         if current_price <= 0:
-            logger.error(f"Invalid price for {ticker}: {current_price}")
+            logger.error(f"[FMP API] Invalid price for {ticker}: {current_price}")
             return None
         
         # Get dividend history
         dividend_url = f"{FMP_BASE_URL}/historical-price-full/stock_dividend/{ticker}?apikey={FMP_API_KEY}"
-        logger.info(f"Making FMP API call to {dividend_url}")
+        logger.info(f"[FMP API] Making dividend request to: {dividend_url}")
         dividend_response = session.get(dividend_url)
         st.session_state.api_calls += 1
-        logger.info(f"FMP API call count: {st.session_state.api_calls}")
+        logger.info(f"[FMP API] Dividend response status: {dividend_response.status_code}")
+        logger.info(f"[FMP API] Dividend response content: {dividend_response.text[:500]}...")  # Log first 500 chars
         
         if dividend_response.status_code != 200:
-            logger.error(f"FMP API error for dividend data: {dividend_response.status_code}")
+            logger.error(f"[FMP API] Error for dividend data: {dividend_response.status_code}")
-            logger.error(f"Response content: {dividend_response.text}")
+            logger.error(f"[FMP API] Response content: {dividend_response.text}")
             return None
         
         dividend_data = dividend_response.json()
-        logger.info(f"FMP dividend response for {ticker}: {dividend_data}")
+        logger.info(f"[FMP API] Dividend data for {ticker}: {dividend_data}")
         
         if not dividend_data or "historical" not in dividend_data or not dividend_data["historical"]:
-            logger.warning(f"No dividend history found for {ticker}")
+            logger.warning(f"[FMP API] No dividend history found for {ticker}")
             return None
         
         # Calculate TTM dividend
@@ -669,7 +697,7 @@ def fetch_etf_data_fmp(ticker: str) -> Optional[Dict[str, Any]]:
         recent_dividends = dividends[dividends["date"] >= one_year_ago]
         
         if recent_dividends.empty:
-            logger.warning(f"No recent dividends found for {ticker}")
+            logger.warning(f"[FMP API] No recent dividends found for {ticker}")
             return None
         
         # Calculate TTM dividend
@@ -677,16 +705,16 @@ def fetch_etf_data_fmp(ticker: str) -> Optional[Dict[str, Any]]:
         
         # Calculate yield
         yield_pct = (ttm_dividend / current_price) * 100
-        logger.info(f"Calculated yield for {ticker}: {yield_pct:.2f}% (TTM dividend: ${ttm_dividend:.2f}, Price: ${current_price:.2f})")
+        logger.info(f"[FMP API] Calculated yield for {ticker}: {yield_pct:.2f}% (TTM dividend: ${ttm_dividend:.2f}, Price: ${current_price:.2f})")
         
         # For high-yield ETFs, verify the yield is reasonable
         if ticker in HIGH_YIELD_ETFS:
             expected_yield = HIGH_YIELD_ETFS[ticker]["expected_yield"]
             if yield_pct < expected_yield * 0.5:  # If yield is less than 50% of expected
-                logger.error(f"Calculated yield {yield_pct:.2f}% for {ticker} is much lower than expected {expected_yield}%")
+                logger.error(f"[FMP API] Calculated yield {yield_pct:.2f}% for {ticker} is much lower than expected {expected_yield}%")
-                logger.error(f"TTM dividend: ${ttm_dividend:.2f}")
+                logger.error(f"[FMP API] TTM dividend: ${ttm_dividend:.2f}")
-                logger.error(f"Current price: ${current_price:.2f}")
+                logger.error(f"[FMP API] Current price: ${current_price:.2f}")
-                logger.error(f"Recent dividends:\n{recent_dividends}")
+                logger.error(f"[FMP API] Recent dividends:\n{recent_dividends}")
         
         # Determine distribution period
         if len(recent_dividends) >= 2:
@@ -708,13 +736,18 @@ def fetch_etf_data_fmp(ticker: str) -> Optional[Dict[str, Any]]:
             "Price": current_price,
             "Yield (%)": yield_pct,
             "Distribution Period": dist_period,
-            "Risk Level": "High" if ticker in HIGH_YIELD_ETFS else "Moderate"
+            "Risk Level": "High" if ticker in HIGH_YIELD_ETFS else "Moderate",
+            "data_source": "FMP API",  # Add data source identifier
+            "raw_data": {  # Store raw data for debugging
+                "profile": profile,
+                "dividend_history": dividend_data["historical"][:5]  # Store first 5 dividend records
+            }
         }
-        logger.info(f"FMP data for {ticker}: {etf_data}")
+        logger.info(f"[FMP API] Final data for {ticker}: {etf_data}")
         return etf_data
         
     except Exception as e:
-        logger.error(f"Error fetching FMP data for {ticker}: {str(e)}")
+        logger.error(f"[FMP API] Error fetching data for {ticker}: {str(e)}")
         logger.error(traceback.format_exc())
         return None
 
@@ -759,7 +792,8 @@ def fetch_etf_data_yfinance(ticker: str) -> Optional[Dict[str, Any]]:
             "Ticker": ticker,
             "Price": current_price,
             "Yield (%)": yield_pct,
-            "Risk Level": "High"  # Default for high-yield ETFs
+            "Risk Level": "High",  # Default for high-yield ETFs
+            "data_source": "yfinance"  # Add data source identifier
         }
         logger.info(f"yfinance data for {ticker}: {etf_data}")
         return etf_data
@@ -787,6 +821,8 @@ def fetch_etf_data(tickers: List[str]) -> pd.DataFrame:
         logger.info("=== Starting ETF data fetch ===")
         logger.info(f"Force refresh enabled: {st.session_state.get('force_refresh_data', False)}")
         logger.info(f"Cache directory: {cache_dir.absolute()}")
+        logger.info(f"FMP API enabled: {USE_FMP_API}")
+        logger.info(f"FMP API key available: {bool(FMP_API_KEY)}")
         
         for ticker in tickers:
             if not ticker:  # Skip empty tickers
@@ -821,9 +857,10 @@ def fetch_etf_data(tickers: List[str]) -> pd.DataFrame:
             
             # Try FMP first if enabled
             if USE_FMP_API and FMP_API_KEY:
-                logger.info(f"Making FMP API call for {ticker}")
+                logger.info(f"Attempting to fetch data from FMP API for {ticker}")
                 etf_data = fetch_etf_data_fmp(ticker)
                 if etf_data is not None:
+                    logger.info(f"Successfully fetched data from FMP API for {ticker}")
                     # Cache the data
                     try:
                         cache_data = {
@@ -841,11 +878,14 @@ def fetch_etf_data(tickers: List[str]) -> pd.DataFrame:
                     st.session_state.api_calls += 1
                     logger.info(f"Total API calls: {st.session_state.api_calls}")
                     continue
+                else:
+                    logger.warning(f"FMP API fetch failed for {ticker}, falling back to yfinance")
             
             # If FMP fails, try yfinance
-            logger.info(f"Falling back to yfinance for {ticker}")
+            logger.info(f"Fetching data from yfinance for {ticker}")
             etf_data = fetch_etf_data_yfinance(ticker)
             if etf_data is not None:
+                logger.info(f"Successfully fetched data from yfinance for {ticker}")
                 # Cache the data
                 try:
                     cache_data = {
@@ -870,7 +910,8 @@ def fetch_etf_data(tickers: List[str]) -> pd.DataFrame:
                     "Price": 25.0,  # Default price for fallback
                     "Yield (%)": HIGH_YIELD_ETFS[ticker]["expected_yield"],
                     "Distribution Period": HIGH_YIELD_ETFS[ticker]["frequency"],
-                    "Risk Level": "High"
+                    "Risk Level": "High",
+                    "data_source": "HIGH_YIELD_ETFS"
                 }
                 data[ticker] = etf_data
             else:
@@ -894,6 +935,11 @@ def fetch_etf_data(tickers: List[str]) -> pd.DataFrame:
         if (df["Yield (%)"] <= 0).any():
             st.warning("Some ETFs have zero or negative yields")
         
+        # Log data sources used
+        if "data_source" in df.columns:
+            source_counts = df["data_source"].value_counts()
+            logger.info(f"Data sources used:\n{source_counts}")
+        
         logger.info(f"Final DataFrame:\n{df}")
         return df
         
@@ -1062,9 +1108,29 @@ def portfolio_summary(final_alloc: pd.DataFrame) -> None:
         display_df = final_alloc.copy()
         display_df["Monthly Income"] = display_df["Income Contributed ($)"] / 12
         
+        # Ensure data_source column exists and rename it for display
+        if "data_source" in display_df.columns:
+            display_df = display_df.rename(columns={"data_source": "Data Source"})
+        else:
+            display_df["Data Source"] = "Unknown"
+        
+        # Select and order columns for display
+        display_columns = [
+            "Ticker",
+            "Allocation (%)",
+            "Yield (%)",
+            "Price",
+            "Shares",
+            "Capital Allocated ($)",
+            "Monthly Income",
+            "Income Contributed ($)",
+            "Risk Level",
+            "Data Source"
+        ]
+        
         # Format the display
         st.dataframe(
-            display_df.style.format({
+            display_df[display_columns].style.format({
                 "Allocation (%)": "{:.2f}%",
                 "Yield (%)": "{:.2f}%",
                 "Price": "${:,.2f}",
@@ -1073,6 +1139,26 @@ def portfolio_summary(final_alloc: pd.DataFrame) -> None:
                 "Monthly Income": "${:,.2f}",
                 "Income Contributed ($)": "${:,.2f}"
             }),
+            column_config={
+                "Ticker": st.column_config.TextColumn("Ticker", disabled=True),
+                "Allocation (%)": st.column_config.NumberColumn(
+                    "Allocation (%)",
+                    min_value=0.0,
+                    max_value=100.0,
+                    step=0.1,
+                    format="%.1f",
+                    required=True
+                ),
+                "Yield (%)": st.column_config.TextColumn("Yield (%)", disabled=True),
+                "Price": st.column_config.TextColumn("Price", disabled=True),
+                "Shares": st.column_config.TextColumn("Shares", disabled=True),
+                "Capital Allocated ($)": st.column_config.TextColumn("Capital Allocated ($)", disabled=True),
+                "Monthly Income": st.column_config.TextColumn("Monthly Income", disabled=True),
+                "Income Contributed ($)": st.column_config.TextColumn("Income Contributed ($)", disabled=True),
+                "Risk Level": st.column_config.TextColumn("Risk Level", disabled=True),
+                "Data Source": st.column_config.TextColumn("Data Source", disabled=True)
+            },
+            hide_index=True,
             use_container_width=True
         )
         
@@ -1878,14 +1964,14 @@ with st.sidebar:
         # Cache clearing options
         col1, col2 = st.columns(2)
         with col1:
-            if st.button("Clear All Cache"):
+            if st.button("Clear All Cache", key="clear_all_cache"):
                 clear_cache()
                 st.success("All cache files cleared!")
                 st.session_state.api_calls = 0
         
         with col2:
             ticker_to_clear = st.text_input("Clear cache for ticker:", key="cache_ticker")
-            if st.button("Clear") and ticker_to_clear:
+            if st.button("Clear", key="clear_single_cache") and ticker_to_clear:
                 clear_cache(ticker_to_clear)
                 st.success(f"Cache cleared for {ticker_to_clear.upper()}")
         
@@ -1897,6 +1983,12 @@ with st.sidebar:
         parallel_processing = st.checkbox("Enable Parallel Processing", value=True, 
                                          help="Fetch data for multiple ETFs simultaneously")
         
+        # Add FMP API test button
+        st.sidebar.subheader("FMP API Testing")
+        if st.sidebar.button("Test FMP API", key="test_fmp_api_button"):
+            test_fmp_data_fetching()
+            st.sidebar.success("Test completed. Check logs for details.")
+
 # Display results and interactive allocation adjustment UI after simulation is run
 if st.session_state.simulation_run and st.session_state.df_data is not None:
     df = st.session_state.df_data