chore: Update project configuration and add setup scripts

- Update Docker and Caddy configuration

- Add VPS setup and secrets management scripts

- Add test suite

- Update documentation

- Clean up cache files

.gitignore

@@ -1,3 +1,11 @@
+# Environment variables
+.env
+#.env.*
+
+# Cache directories
+cache/
+**/cache/
+
 # Python
 __pycache__/
 *.py[cod]
@@ -25,8 +33,8 @@ wheels/
 
 # Virtual Environment
 venv/
-env/
 ENV/
+env/
 
 # IDE
 .idea/
@@ -35,15 +43,10 @@ ENV/
 *.swo
 
 # Logs
-logs/
 *.log
-
+logs/
-# Cache
-.cache/
-__pycache__/
 
 # Local development
-.env
 .env.local
 .env.*.local
 

Caddyfile

@@ -8,7 +8,7 @@ invest.trader-lab.com {
 
     # Main ETF Suite Launcher
     handle / {
-        reverse_proxy etf-launcher:8500 {
+        reverse_proxy etf_portal-etf-launcher-1:8500 {
             header_up Host {host}
             header_up X-Real-IP {remote}
             header_up X-Forwarded-For {remote}
@@ -18,7 +18,7 @@ invest.trader-lab.com {
 
     # Static resources for Streamlit
     handle /_stcore/* {
-        reverse_proxy etf-launcher:8500 {
+        reverse_proxy etf_portal-etf-launcher-1:8500 {
             header_up Host {host}
             header_up X-Real-IP {remote}
             header_up X-Forwarded-For {remote}
@@ -27,7 +27,7 @@ invest.trader-lab.com {
     }
 
     handle /static/* {
-        reverse_proxy etf-launcher:8500 {
+        reverse_proxy etf_portal-etf-launcher-1:8500 {
             header_up Host {host}
             header_up X-Real-IP {remote}
             header_up X-Forwarded-For {remote}

README.md

@@ -23,6 +23,21 @@ source venv/bin/activate  # On Linux/Mac
 pip install -e .
 ```
 
+## Environment Setup
+
+1. Copy the environment template:
+   ```bash
+   cp .env.template .env
+   ```
+
+2. Edit the `.env` file and add your API keys:
+   ```
+   FMP_API_KEY=your_api_key_here
+   CACHE_DURATION_HOURS=24
+   ```
+
+3. Never commit the `.env` file to version control.
+
 ## Usage
 
 The ETF Portal provides a command-line interface for managing the application:

cache/48cf9534181b3f9151f2c27d08ecf5c0.json

@@ -1 +0,0 @@
-{"data": {"__pandas_series__": true, "index": [

cache/4ba6cb61b51b694187cfb678fc62dd84.json

@@ -1 +0,0 @@
-{"data": {"longBusinessSummary": "The fund is an actively managed exchange-traded fund that seeks current income while providing direct and/or indirect exposure to the share price of select U.S. listed securities, subject to a limit on potential investment gains. It uses both traditional and synthetic covered call strategies that are designed to produce higher income levels when the underlying securities experience more volatility. The fund is non-diversified.", "companyOfficers": [], "executiveTeam": [], "maxAge": 86400, "priceHint": 2, "previousClose": 6.1, "open": 6.02, "dayLow": 6.02, "dayHigh": 6.13, "regularMarketPreviousClose": 6.1, "regularMarketOpen": 6.02, "regularMarketDayLow": 6.02, "regularMarketDayHigh": 6.13, "trailingPE": 26.701918, "volume": 2869944, "regularMarketVolume": 2869944, "averageVolume": 1994822, "averageVolume10days": 3513250, "averageDailyVolume10Day": 3513250, "bid": 6.1, "ask": 6.12, "bidSize": 8, "askSize": 30, "yield": 1.6552, "totalAssets": 226379696, "fiftyTwoWeekLow": 5.23, "fiftyTwoWeekHigh": 15.22, "fiftyDayAverage": 6.0538, "twoHundredDayAverage": 8.74755, "navPrice": 6.0906, "currency": "USD", "tradeable": false, "category": "Derivative Income", "ytdReturn": -11.21162, "beta3Year": 0.0, "fundFamily": "YieldMax ETFs", "fundInceptionDate": 1709078400, "legalType": "Exchange Traded Fund", "quoteType": "ETF", "symbol": "ULTY", "language": "en-US", "region": "US", "typeDisp": "ETF", "quoteSourceName": "Delayed Quote", "triggerable": true, "customPriceAlertConfidence": "HIGH", "shortName": "Tidal Trust II YieldMax Ultra O", "longName": "YieldMax Ultra Option Income Strategy ETF", "fiftyTwoWeekLowChange": 0.8699999, "fiftyTwoWeekLowChangePercent": 0.16634797, "fiftyTwoWeekRange": "5.23 - 15.22", "fiftyTwoWeekHighChange": -9.120001, "fiftyTwoWeekHighChangePercent": -0.59921163, "fiftyTwoWeekChangePercent": -57.282913, "dividendYield": 165.52, "trailingThreeMonthReturns": -13.87, "trailingThreeMonthNavReturns": -13.87, "netAssets": 226379696.0, "epsTrailingTwelveMonths": 0.228448, "marketState": "CLOSED", "regularMarketChangePercent": 0.0, "regularMarketPrice": 6.1, "corporateActions": [], "postMarketTime": 1748044768, "regularMarketTime": 1748030400, "exchange": "PCX", "messageBoardId": "finmb_1869805004", "exchangeTimezoneName": "America/New_York", "exchangeTimezoneShortName": "EDT", "gmtOffSetMilliseconds": -14400000, "market": "us_market", "esgPopulated": false, "fiftyDayAverageChange": 0.0461998, "fiftyDayAverageChangePercent": 0.007631537, "twoHundredDayAverageChange": -2.64755, "twoHundredDayAverageChangePercent": -0.3026619, "netExpenseRatio": 1.3, "sourceInterval": 15, "exchangeDataDelayedBy": 0, "cryptoTradeable": false, "hasPrePostMarketData": true, "firstTradeDateMilliseconds": 1709217000000, "postMarketChangePercent": 0.49180672, "postMarketPrice": 6.13, "postMarketChange": 0.03000021, "regularMarketChange": 0.0, "regularMarketDayRange": "6.02 - 6.13", "fullExchangeName": "NYSEArca", "averageDailyVolume3Month": 1994822, "trailingPegRatio": null}, "timestamp": "2025-05-24T13:52:58.466664"}

cache/86bd26d004d7951035b2e678a3fa6ccf.json

@@ -1 +0,0 @@
-{"data": {"__pandas_series__": true, "index": [

cache/86cc9fb558c9ab4da62c8a741e919889.json

@@ -1 +0,0 @@
-{"data": {"__pandas_series__": true, "index": [

cache/a4d5861d1e785dcd29a4155b49156709.json

@@ -1 +0,0 @@
-{"data": {"longBusinessSummary": "The fund is an actively managed exchange-traded fund (\u201cETF\u201d) that seeks current income while maintaining the opportunity for exposure to the share price (i.e., the price returns) of the securities of the companies comprising the Solactive FANG Innovation Index. The fund advisor seeks to employ the fund's investment strategy regardless of whether there are periods of adverse market, economic, or other conditions and will not seek to take temporary defensive positions during such periods. It is non-diversified.", "companyOfficers": [], "executiveTeam": [], "maxAge": 86400, "priceHint": 2, "previousClose": 43.66, "open": 43.12, "dayLow": 43.02, "dayHigh": 43.55, "regularMarketPreviousClose": 43.66, "regularMarketOpen": 43.12, "regularMarketDayLow": 43.02, "regularMarketDayHigh": 43.55, "trailingPE": 36.65259, "volume": 129662, "regularMarketVolume": 129662, "averageVolume": 147330, "averageVolume10days": 111310, "averageDailyVolume10Day": 111310, "bid": 43.11, "ask": 43.35, "bidSize": 2, "askSize": 2, "yield": 0.3039, "totalAssets": 426724160, "fiftyTwoWeekLow": 35.44, "fiftyTwoWeekHigh": 56.44, "fiftyDayAverage": 41.907, "twoHundredDayAverage": 48.12085, "navPrice": 43.62, "currency": "USD", "tradeable": false, "category": "Derivative Income", "ytdReturn": -8.89758, "beta3Year": 0.0, "fundFamily": "REX Advisers, LLC", "fundInceptionDate": 1696982400, "legalType": "Exchange Traded Fund", "quoteType": "ETF", "symbol": "FEPI", "language": "en-US", "region": "US", "typeDisp": "ETF", "quoteSourceName": "Nasdaq Real Time Price", "triggerable": true, "customPriceAlertConfidence": "HIGH", "corporateActions": [], "postMarketTime": 1748040939, "regularMarketTime": 1748030400, "hasPrePostMarketData": true, "firstTradeDateMilliseconds": 1697031000000, "postMarketChangePercent": 0.716766, "postMarketPrice": 43.56, "postMarketChange": 0.310001, "regularMarketChange": -0.40999985, "regularMarketDayRange": "43.02 - 43.55", "fullExchangeName": "NasdaqGM", "averageDailyVolume3Month": 147330, "fiftyTwoWeekLowChange": 7.8100014, "fiftyTwoWeekLowChangePercent": 0.22037251, "fiftyTwoWeekRange": "35.44 - 56.44", "fiftyTwoWeekHighChange": -13.189999, "fiftyTwoWeekHighChangePercent": -0.23369949, "fiftyTwoWeekChangePercent": -20.72947, "dividendYield": 30.39, "trailingThreeMonthReturns": -9.55848, "trailingThreeMonthNavReturns": -9.55848, "netAssets": 426724160.0, "epsTrailingTwelveMonths": 1.1799984, "fiftyDayAverageChange": 1.3429985, "fiftyDayAverageChangePercent": 0.032047115, "twoHundredDayAverageChange": -4.8708496, "twoHundredDayAverageChangePercent": -0.10122119, "netExpenseRatio": 0.65, "sourceInterval": 15, "exchangeDataDelayedBy": 0, "ipoExpectedDate": "2023-10-11", "cryptoTradeable": false, "marketState": "CLOSED", "shortName": "REX FANG & Innovation Equity Pr", "longName": "REX FANG & Innovation Equity Premium Income ETF", "exchange": "NGM", "messageBoardId": "finmb_1843173608", "exchangeTimezoneName": "America/New_York", "exchangeTimezoneShortName": "EDT", "gmtOffSetMilliseconds": -14400000, "market": "us_market", "esgPopulated": false, "regularMarketChangePercent": -0.93907434, "regularMarketPrice": 43.25, "trailingPegRatio": null}, "timestamp": "2025-05-24T13:52:58.472581"}

cache/f039bfbd574b918f6ccf351f22a2c0c8.json

@@ -1 +0,0 @@
-{"data": {"longBusinessSummary": "The fund is an actively managed ETF that seeks current income while maintaining the opportunity for exposure to the share price of the common stock of MicroStrategy Incorporated, subject to a limit on potential investment gains. It will seek to employ its investment strategy as it relates to MSTR regardless of whether there are periods of adverse market, economic, or other conditions and will not seek to take temporary defensive positions during such periods. The fund is non-diversified.", "companyOfficers": [], "executiveTeam": [], "maxAge": 86400, "priceHint": 2, "previousClose": 23.08, "open": 22.68, "dayLow": 21.2901, "dayHigh": 22.7, "regularMarketPreviousClose": 23.08, "regularMarketOpen": 22.68, "regularMarketDayLow": 21.2901, "regularMarketDayHigh": 22.7, "volume": 18359202, "regularMarketVolume": 18359202, "averageVolume": 7904033, "averageVolume10days": 11083420, "averageDailyVolume10Day": 11083420, "bid": 21.5, "ask": 21.63, "bidSize": 12, "askSize": 32, "yield": 1.2471, "totalAssets": 3270944256, "fiftyTwoWeekLow": 17.1, "fiftyTwoWeekHigh": 46.5, "fiftyDayAverage": 22.1824, "twoHundredDayAverage": 26.1411, "navPrice": 23.0514, "currency": "USD", "tradeable": false, "category": "Derivative Income", "ytdReturn": 24.89181, "beta3Year": 0.0, "fundFamily": "YieldMax ETFs", "fundInceptionDate": 1708473600, "legalType": "Exchange Traded Fund", "quoteType": "ETF", "symbol": "MSTY", "language": "en-US", "region": "US", "typeDisp": "ETF", "quoteSourceName": "Delayed Quote", "triggerable": true, "customPriceAlertConfidence": "HIGH", "marketState": "CLOSED", "shortName": "Tidal Trust II YieldMax MSTR Op", "regularMarketChangePercent": -6.7591, "regularMarketPrice": 21.52, "corporateActions": [], "longName": "Yieldmax MSTR Option Income Strategy ETF", "postMarketTime": 1748044797, "regularMarketTime": 1748030400, "regularMarketDayRange": "21.2901 - 22.7", "fullExchangeName": "NYSEArca", "averageDailyVolume3Month": 7904033, "fiftyTwoWeekLowChange": 4.42, "fiftyTwoWeekLowChangePercent": 0.25847954, "fiftyTwoWeekRange": "17.1 - 46.5", "fiftyTwoWeekHighChange": -24.98, "fiftyTwoWeekHighChangePercent": -0.53720427, "fiftyTwoWeekChangePercent": -38.654507, "dividendYield": 124.71, "trailingThreeMonthReturns": 13.21012, "trailingThreeMonthNavReturns": 13.21012, "netAssets": 3270944260.0, "fiftyDayAverageChange": -0.6623993, "fiftyDayAverageChangePercent": -0.02986148, "twoHundredDayAverageChange": -4.6210995, "twoHundredDayAverageChangePercent": -0.17677525, "netExpenseRatio": 0.99, "sourceInterval": 15, "exchangeDataDelayedBy": 0, "cryptoTradeable": false, "hasPrePostMarketData": true, "firstTradeDateMilliseconds": 1708612200000, "postMarketChangePercent": -0.18587786, "postMarketPrice": 21.48, "postMarketChange": -0.040000916, "regularMarketChange": -1.56, "exchange": "PCX", "messageBoardId": "finmb_1850981069", "exchangeTimezoneName": "America/New_York", "exchangeTimezoneShortName": "EDT", "gmtOffSetMilliseconds": -14400000, "market": "us_market", "esgPopulated": false, "trailingPegRatio": null}, "timestamp": "2025-05-24T13:52:58.456018"}

docker-compose.yml

@@ -6,6 +6,8 @@ services:
     command: streamlit run ETF_Suite_Launcher.py --server.port=8500
     volumes:
       - .:/app
+    ports:
+      - "8500:8500"
     networks:
       - etf_network
     environment:

requirements.txt

@@ -17,3 +17,4 @@ requests>=2.31.0
 reportlab>=3.6.13
 psutil>=5.9.0
 click>=8.1.0
+yfinance>=0.2.36 

scripts/setup_secrets.py

@@ -0,0 +1,71 @@
+import os
+import sys
+import logging
+from pathlib import Path
+
+def setup_secrets():
+    """Set up secrets for the ETF Portal application."""
+    logging.basicConfig(level=logging.INFO)
+    logger = logging.getLogger(__name__)
+    
+    try:
+        # Get the FMP API key from user
+        fmp_api_key = input("Enter your FMP API key: ").strip()
+        if not fmp_api_key:
+            logger.error("❌ FMP API key is required")
+            return False
+        
+        # Create .streamlit directory if it doesn't exist
+        streamlit_dir = Path(".streamlit")
+        streamlit_dir.mkdir(exist_ok=True)
+        
+        # Create secrets.toml
+        secrets_path = streamlit_dir / "secrets.toml"
+        with open(secrets_path, "w") as f:
+            f.write(f'# FMP API Configuration\n')
+            f.write(f'FMP_API_KEY = "{fmp_api_key}"\n\n')
+            f.write(f'# Cache Configuration\n')
+            f.write(f'CACHE_DURATION_HOURS = 24\n')
+        
+        # Set proper permissions
+        secrets_path.chmod(0o600)  # Only owner can read/write
+        
+        logger.info(f"✅ Secrets file created at {secrets_path}")
+        logger.info("🔒 File permissions set to 600 (owner read/write only)")
+        
+        # Create cache directories
+        cache_dirs = [
+            Path("cache/FMP_cache"),
+            Path("cache/yfinance_cache")
+        ]
+        
+        for cache_dir in cache_dirs:
+            cache_dir.mkdir(parents=True, exist_ok=True)
+            cache_dir.chmod(0o755)  # Owner can read/write/execute, others can read/execute
+            logger.info(f"✅ Cache directory created: {cache_dir}")
+        
+        return True
+        
+    except Exception as e:
+        logger.error(f"❌ Setup failed: {str(e)}")
+        return False
+
+if __name__ == "__main__":
+    print("🔧 ETF Portal Secrets Setup")
+    print("===========================")
+    print("This script will help you set up the secrets for the ETF Portal application.")
+    print("Make sure you have your FMP API key ready.")
+    print()
+    
+    success = setup_secrets()
+    if success:
+        print("\n✅ Setup completed successfully!")
+        print("\nNext steps:")
+        print("1. Run the test script to verify the configuration:")
+        print("   python -m ETF_Portal.tests.test_api_config")
+        print("\n2. If you're deploying to a server, make sure to:")
+        print("   - Set the secrets in your Streamlit dashboard")
+        print("   - Create the cache directories with proper permissions")
+    else:
+        print("\n❌ Setup failed. Check the logs for details.")
+        sys.exit(1) 

scripts/setup_vps.py

@@ -0,0 +1,102 @@
+import os
+import sys
+import logging
+from pathlib import Path
+import subprocess
+
+def setup_vps_environment():
+    """Set up the environment for the ETF Portal on VPS."""
+    logging.basicConfig(level=logging.INFO)
+    logger = logging.getLogger(__name__)
+    
+    try:
+        # Get the FMP API key from user
+        fmp_api_key = input("Enter your FMP API key: ").strip()
+        if not fmp_api_key:
+            logger.error("❌ FMP API key is required")
+            return False
+        
+        # Add to environment file
+        env_file = Path("/etc/environment")
+        if not env_file.exists():
+            env_file = Path.home() / ".bashrc"
+        
+        # Check if key already exists
+        with open(env_file, 'r') as f:
+            content = f.read()
+        
+        if f"FMP_API_KEY={fmp_api_key}" not in content:
+            with open(env_file, 'a') as f:
+                f.write(f'\n# ETF Portal Configuration\n')
+                f.write(f'export FMP_API_KEY="{fmp_api_key}"\n')
+            logger.info(f"✅ Added FMP_API_KEY to {env_file}")
+        else:
+            logger.info("✅ FMP_API_KEY already exists in environment file")
+        
+        # Create cache directories
+        cache_dirs = [
+            Path("cache/FMP_cache"),
+            Path("cache/yfinance_cache")
+        ]
+        
+        for cache_dir in cache_dirs:
+            cache_dir.mkdir(parents=True, exist_ok=True)
+            cache_dir.chmod(0o755)  # Owner can read/write/execute, others can read/execute
+            logger.info(f"✅ Cache directory created: {cache_dir}")
+        
+        # Set up systemd service (if needed)
+        if input("Do you want to set up a systemd service for the ETF Portal? (y/n): ").lower() == 'y':
+            service_content = f"""[Unit]
+Description=ETF Portal Streamlit App
+After=network.target
+
+[Service]
+User={os.getenv('USER')}
+WorkingDirectory={Path.cwd()}
+Environment="FMP_API_KEY={fmp_api_key}"
+ExecStart=/usr/local/bin/streamlit run ETF_Portal/pages/ETF_Analyzer.py
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
+"""
+            service_path = Path("/etc/systemd/system/etf-portal.service")
+            if not service_path.exists():
+                with open(service_path, 'w') as f:
+                    f.write(service_content)
+                logger.info("✅ Created systemd service file")
+                
+                # Reload systemd and enable service
+                subprocess.run(["sudo", "systemctl", "daemon-reload"])
+                subprocess.run(["sudo", "systemctl", "enable", "etf-portal"])
+                logger.info("✅ Enabled ETF Portal service")
+            else:
+                logger.info("✅ Service file already exists")
+        
+        return True
+        
+    except Exception as e:
+        logger.error(f"❌ Setup failed: {str(e)}")
+        return False
+
+if __name__ == "__main__":
+    print("🔧 ETF Portal VPS Setup")
+    print("======================")
+    print("This script will help you set up the ETF Portal on your VPS.")
+    print("Make sure you have your FMP API key ready.")
+    print()
+    
+    success = setup_vps_environment()
+    if success:
+        print("\n✅ Setup completed successfully!")
+        print("\nNext steps:")
+        print("1. Source your environment file:")
+        print("   source /etc/environment  # or source ~/.bashrc")
+        print("\n2. Run the test script to verify the configuration:")
+        print("   python -m ETF_Portal.tests.test_api_config")
+        print("\n3. If you set up the systemd service:")
+        print("   sudo systemctl start etf-portal")
+        print("   sudo systemctl status etf-portal  # Check status")
+    else:
+        print("\n❌ Setup failed. Check the logs for details.")
+        sys.exit(1) 

setup.py

@@ -2,14 +2,20 @@
 from setuptools import setup, find_packages
 
 setup(
-    name="etf-portal",
+    name="ETF_Portal",
     version="0.1.0",
     packages=find_packages(),
     include_package_data=True,
     install_requires=[
-        "click",
+        "streamlit>=1.28.0",
-        "psutil",
+        "pandas>=1.5.3",
-        "streamlit",
+        "numpy>=1.24.3",
+        "matplotlib>=3.7.1",
+        "seaborn>=0.12.2",
+        "fmp-python>=0.1.5",
+        "plotly>=5.14.1",
+        "requests>=2.31.0",
+        "yfinance>=0.2.36",
     ],
     entry_points={
         "console_scripts": [

tests/__init__.py

@@ -0,0 +1 @@
+"""Test package for ETF Portal.""" 

tests/test_api_config.py

@@ -0,0 +1,65 @@
+import streamlit as st
+import logging
+from api import APIFactory
+import pandas as pd
+
+def test_api_configuration():
+    """Test the API configuration and secrets."""
+    logging.basicConfig(level=logging.INFO)
+    logger = logging.getLogger(__name__)
+    
+    try:
+        # Initialize API factory
+        api_factory = APIFactory()
+        
+        # Test FMP client
+        logger.info("Testing FMP client...")
+        fmp_client = api_factory.get_client('fmp')
+        
+        # Test with a known ETF
+        test_symbol = "SPY"
+        
+        # Test profile data
+        logger.info(f"Getting profile data for {test_symbol}...")
+        profile = fmp_client.get_etf_profile(test_symbol)
+        if isinstance(profile, dict) and not profile.get('error'):
+            logger.info("✅ Profile data retrieved successfully")
+        else:
+            logger.error("❌ Failed to get profile data")
+            logger.error(f"Error: {profile.get('message', 'Unknown error')}")
+        
+        # Test historical data
+        logger.info(f"Getting historical data for {test_symbol}...")
+        historical = fmp_client.get_historical_data(test_symbol, period='1mo')
+        if isinstance(historical, pd.DataFrame) and not historical.empty:
+            logger.info("✅ Historical data retrieved successfully")
+            logger.info(f"Data points: {len(historical)}")
+        else:
+            logger.error("❌ Failed to get historical data")
+        
+        # Test cache
+        logger.info("Testing cache...")
+        cache_stats = api_factory.get_cache_stats()
+        logger.info(f"Cache stats: {cache_stats}")
+        
+        # Test fallback to yfinance
+        logger.info("Testing fallback to yfinance...")
+        yfinance_data = api_factory.get_data(test_symbol, 'etf_profile', provider='yfinance')
+        if isinstance(yfinance_data, dict) and not yfinance_data.get('error'):
+            logger.info("✅ YFinance fallback working")
+        else:
+            logger.error("❌ YFinance fallback failed")
+            logger.error(f"Error: {yfinance_data.get('message', 'Unknown error')}")
+        
+        return True
+        
+    except Exception as e:
+        logger.error(f"❌ Test failed: {str(e)}")
+        return False
+
+if __name__ == "__main__":
+    success = test_api_configuration()
+    if success:
+        print("\n✅ All tests passed!")
+    else:
+        print("\n❌ Some tests failed. Check the logs for details.")