diff --git a/pages/ETF_Portfolio_Builder.py b/pages/ETF_Portfolio_Builder.py index e37c3d5..110b3c7 100644 --- a/pages/ETF_Portfolio_Builder.py +++ b/pages/ETF_Portfolio_Builder.py @@ -52,6 +52,32 @@ else: FMP_BASE_URL = "https://financialmodelingprep.com/api/v3" +def test_fmp_data_fetching(): + """Test FMP API data fetching with detailed logging.""" + try: + logger.info("=== Starting FMP API Test ===") + logger.info(f"FMP API Key available: {bool(FMP_API_KEY)}") + logger.info(f"FMP API enabled: {USE_FMP_API}") + + # Test a few high-yield ETFs + test_tickers = ["JEPI", "JEPQ", "QYLD"] + + for ticker in test_tickers: + logger.info(f"\nTesting {ticker}:") + data = fetch_etf_data_fmp(ticker) + if data: + logger.info(f"Successfully fetched data for {ticker}") + logger.info(f"Data source: {data.get('data_source', 'Not specified')}") + logger.info(f"Raw data: {data.get('raw_data', 'No raw data')}") + else: + logger.error(f"Failed to fetch data for {ticker}") + + logger.info("=== FMP API Test Complete ===") + + except Exception as e: + logger.error(f"Error in FMP API test: {str(e)}") + logger.error(traceback.format_exc()) + # High-yield ETFs reference data HIGH_YIELD_ETFS = { "MSTY": {"expected_yield": 125.0, "frequency": "Monthly"}, @@ -617,46 +643,48 @@ def fetch_etf_data_fmp(ticker: str) -> Optional[Dict[str, Any]]: # Get profile data for current price profile_url = f"{FMP_BASE_URL}/profile/{ticker}?apikey={FMP_API_KEY}" - logger.info(f"Making FMP API call to {profile_url}") + logger.info(f"[FMP API] Making profile request to: {profile_url}") profile_response = session.get(profile_url) st.session_state.api_calls += 1 - logger.info(f"FMP API call count: {st.session_state.api_calls}") + logger.info(f"[FMP API] Profile response status: {profile_response.status_code}") + logger.info(f"[FMP API] Profile response content: {profile_response.text[:500]}...") # Log first 500 chars if profile_response.status_code != 200: - logger.error(f"FMP API error for {ticker}: {profile_response.status_code}") - logger.error(f"Response content: {profile_response.text}") + logger.error(f"[FMP API] Error for {ticker}: {profile_response.status_code}") + logger.error(f"[FMP API] Response content: {profile_response.text}") return None profile_data = profile_response.json() - logger.info(f"FMP profile response for {ticker}: {profile_data}") + logger.info(f"[FMP API] Profile data for {ticker}: {profile_data}") if not profile_data or not isinstance(profile_data, list) or len(profile_data) == 0: - logger.warning(f"No profile data found for {ticker} in FMP") + logger.warning(f"[FMP API] No profile data found for {ticker}") return None profile = profile_data[0] current_price = float(profile.get('price', 0)) if current_price <= 0: - logger.error(f"Invalid price for {ticker}: {current_price}") + logger.error(f"[FMP API] Invalid price for {ticker}: {current_price}") return None # Get dividend history dividend_url = f"{FMP_BASE_URL}/historical-price-full/stock_dividend/{ticker}?apikey={FMP_API_KEY}" - logger.info(f"Making FMP API call to {dividend_url}") + logger.info(f"[FMP API] Making dividend request to: {dividend_url}") dividend_response = session.get(dividend_url) st.session_state.api_calls += 1 - logger.info(f"FMP API call count: {st.session_state.api_calls}") + logger.info(f"[FMP API] Dividend response status: {dividend_response.status_code}") + logger.info(f"[FMP API] Dividend response content: {dividend_response.text[:500]}...") # Log first 500 chars if dividend_response.status_code != 200: - logger.error(f"FMP API error for dividend data: {dividend_response.status_code}") - logger.error(f"Response content: {dividend_response.text}") + logger.error(f"[FMP API] Error for dividend data: {dividend_response.status_code}") + logger.error(f"[FMP API] Response content: {dividend_response.text}") return None dividend_data = dividend_response.json() - logger.info(f"FMP dividend response for {ticker}: {dividend_data}") + logger.info(f"[FMP API] Dividend data for {ticker}: {dividend_data}") if not dividend_data or "historical" not in dividend_data or not dividend_data["historical"]: - logger.warning(f"No dividend history found for {ticker}") + logger.warning(f"[FMP API] No dividend history found for {ticker}") return None # Calculate TTM dividend @@ -669,7 +697,7 @@ def fetch_etf_data_fmp(ticker: str) -> Optional[Dict[str, Any]]: recent_dividends = dividends[dividends["date"] >= one_year_ago] if recent_dividends.empty: - logger.warning(f"No recent dividends found for {ticker}") + logger.warning(f"[FMP API] No recent dividends found for {ticker}") return None # Calculate TTM dividend @@ -677,16 +705,16 @@ def fetch_etf_data_fmp(ticker: str) -> Optional[Dict[str, Any]]: # Calculate yield yield_pct = (ttm_dividend / current_price) * 100 - logger.info(f"Calculated yield for {ticker}: {yield_pct:.2f}% (TTM dividend: ${ttm_dividend:.2f}, Price: ${current_price:.2f})") + logger.info(f"[FMP API] Calculated yield for {ticker}: {yield_pct:.2f}% (TTM dividend: ${ttm_dividend:.2f}, Price: ${current_price:.2f})") # For high-yield ETFs, verify the yield is reasonable if ticker in HIGH_YIELD_ETFS: expected_yield = HIGH_YIELD_ETFS[ticker]["expected_yield"] if yield_pct < expected_yield * 0.5: # If yield is less than 50% of expected - logger.error(f"Calculated yield {yield_pct:.2f}% for {ticker} is much lower than expected {expected_yield}%") - logger.error(f"TTM dividend: ${ttm_dividend:.2f}") - logger.error(f"Current price: ${current_price:.2f}") - logger.error(f"Recent dividends:\n{recent_dividends}") + logger.error(f"[FMP API] Calculated yield {yield_pct:.2f}% for {ticker} is much lower than expected {expected_yield}%") + logger.error(f"[FMP API] TTM dividend: ${ttm_dividend:.2f}") + logger.error(f"[FMP API] Current price: ${current_price:.2f}") + logger.error(f"[FMP API] Recent dividends:\n{recent_dividends}") # Determine distribution period if len(recent_dividends) >= 2: @@ -708,13 +736,18 @@ def fetch_etf_data_fmp(ticker: str) -> Optional[Dict[str, Any]]: "Price": current_price, "Yield (%)": yield_pct, "Distribution Period": dist_period, - "Risk Level": "High" if ticker in HIGH_YIELD_ETFS else "Moderate" + "Risk Level": "High" if ticker in HIGH_YIELD_ETFS else "Moderate", + "data_source": "FMP API", # Add data source identifier + "raw_data": { # Store raw data for debugging + "profile": profile, + "dividend_history": dividend_data["historical"][:5] # Store first 5 dividend records + } } - logger.info(f"FMP data for {ticker}: {etf_data}") + logger.info(f"[FMP API] Final data for {ticker}: {etf_data}") return etf_data except Exception as e: - logger.error(f"Error fetching FMP data for {ticker}: {str(e)}") + logger.error(f"[FMP API] Error fetching data for {ticker}: {str(e)}") logger.error(traceback.format_exc()) return None @@ -759,7 +792,8 @@ def fetch_etf_data_yfinance(ticker: str) -> Optional[Dict[str, Any]]: "Ticker": ticker, "Price": current_price, "Yield (%)": yield_pct, - "Risk Level": "High" # Default for high-yield ETFs + "Risk Level": "High", # Default for high-yield ETFs + "data_source": "yfinance" # Add data source identifier } logger.info(f"yfinance data for {ticker}: {etf_data}") return etf_data @@ -787,6 +821,8 @@ def fetch_etf_data(tickers: List[str]) -> pd.DataFrame: logger.info("=== Starting ETF data fetch ===") logger.info(f"Force refresh enabled: {st.session_state.get('force_refresh_data', False)}") logger.info(f"Cache directory: {cache_dir.absolute()}") + logger.info(f"FMP API enabled: {USE_FMP_API}") + logger.info(f"FMP API key available: {bool(FMP_API_KEY)}") for ticker in tickers: if not ticker: # Skip empty tickers @@ -821,9 +857,10 @@ def fetch_etf_data(tickers: List[str]) -> pd.DataFrame: # Try FMP first if enabled if USE_FMP_API and FMP_API_KEY: - logger.info(f"Making FMP API call for {ticker}") + logger.info(f"Attempting to fetch data from FMP API for {ticker}") etf_data = fetch_etf_data_fmp(ticker) if etf_data is not None: + logger.info(f"Successfully fetched data from FMP API for {ticker}") # Cache the data try: cache_data = { @@ -841,11 +878,14 @@ def fetch_etf_data(tickers: List[str]) -> pd.DataFrame: st.session_state.api_calls += 1 logger.info(f"Total API calls: {st.session_state.api_calls}") continue + else: + logger.warning(f"FMP API fetch failed for {ticker}, falling back to yfinance") # If FMP fails, try yfinance - logger.info(f"Falling back to yfinance for {ticker}") + logger.info(f"Fetching data from yfinance for {ticker}") etf_data = fetch_etf_data_yfinance(ticker) if etf_data is not None: + logger.info(f"Successfully fetched data from yfinance for {ticker}") # Cache the data try: cache_data = { @@ -870,7 +910,8 @@ def fetch_etf_data(tickers: List[str]) -> pd.DataFrame: "Price": 25.0, # Default price for fallback "Yield (%)": HIGH_YIELD_ETFS[ticker]["expected_yield"], "Distribution Period": HIGH_YIELD_ETFS[ticker]["frequency"], - "Risk Level": "High" + "Risk Level": "High", + "data_source": "HIGH_YIELD_ETFS" } data[ticker] = etf_data else: @@ -894,6 +935,11 @@ def fetch_etf_data(tickers: List[str]) -> pd.DataFrame: if (df["Yield (%)"] <= 0).any(): st.warning("Some ETFs have zero or negative yields") + # Log data sources used + if "data_source" in df.columns: + source_counts = df["data_source"].value_counts() + logger.info(f"Data sources used:\n{source_counts}") + logger.info(f"Final DataFrame:\n{df}") return df @@ -1062,9 +1108,29 @@ def portfolio_summary(final_alloc: pd.DataFrame) -> None: display_df = final_alloc.copy() display_df["Monthly Income"] = display_df["Income Contributed ($)"] / 12 + # Ensure data_source column exists and rename it for display + if "data_source" in display_df.columns: + display_df = display_df.rename(columns={"data_source": "Data Source"}) + else: + display_df["Data Source"] = "Unknown" + + # Select and order columns for display + display_columns = [ + "Ticker", + "Allocation (%)", + "Yield (%)", + "Price", + "Shares", + "Capital Allocated ($)", + "Monthly Income", + "Income Contributed ($)", + "Risk Level", + "Data Source" + ] + # Format the display st.dataframe( - display_df.style.format({ + display_df[display_columns].style.format({ "Allocation (%)": "{:.2f}%", "Yield (%)": "{:.2f}%", "Price": "${:,.2f}", @@ -1073,6 +1139,26 @@ def portfolio_summary(final_alloc: pd.DataFrame) -> None: "Monthly Income": "${:,.2f}", "Income Contributed ($)": "${:,.2f}" }), + column_config={ + "Ticker": st.column_config.TextColumn("Ticker", disabled=True), + "Allocation (%)": st.column_config.NumberColumn( + "Allocation (%)", + min_value=0.0, + max_value=100.0, + step=0.1, + format="%.1f", + required=True + ), + "Yield (%)": st.column_config.TextColumn("Yield (%)", disabled=True), + "Price": st.column_config.TextColumn("Price", disabled=True), + "Shares": st.column_config.TextColumn("Shares", disabled=True), + "Capital Allocated ($)": st.column_config.TextColumn("Capital Allocated ($)", disabled=True), + "Monthly Income": st.column_config.TextColumn("Monthly Income", disabled=True), + "Income Contributed ($)": st.column_config.TextColumn("Income Contributed ($)", disabled=True), + "Risk Level": st.column_config.TextColumn("Risk Level", disabled=True), + "Data Source": st.column_config.TextColumn("Data Source", disabled=True) + }, + hide_index=True, use_container_width=True ) @@ -1878,14 +1964,14 @@ with st.sidebar: # Cache clearing options col1, col2 = st.columns(2) with col1: - if st.button("Clear All Cache"): + if st.button("Clear All Cache", key="clear_all_cache"): clear_cache() st.success("All cache files cleared!") st.session_state.api_calls = 0 with col2: ticker_to_clear = st.text_input("Clear cache for ticker:", key="cache_ticker") - if st.button("Clear") and ticker_to_clear: + if st.button("Clear", key="clear_single_cache") and ticker_to_clear: clear_cache(ticker_to_clear) st.success(f"Cache cleared for {ticker_to_clear.upper()}") @@ -1896,6 +1982,12 @@ with st.sidebar: debug_mode = st.checkbox("Enable Debug Mode", help="Show detailed error logs.") parallel_processing = st.checkbox("Enable Parallel Processing", value=True, help="Fetch data for multiple ETFs simultaneously") + + # Add FMP API test button + st.sidebar.subheader("FMP API Testing") + if st.sidebar.button("Test FMP API", key="test_fmp_api_button"): + test_fmp_data_fetching() + st.sidebar.success("Test completed. Check logs for details.") # Display results and interactive allocation adjustment UI after simulation is run if st.session_state.simulation_run and st.session_state.df_data is not None: