From 4fc9452c988e4513464454f1477683469cdb2974 Mon Sep 17 00:00:00 2001 From: Pascal Date: Mon, 26 May 2025 19:46:05 +0200 Subject: [PATCH] refactor: improve CLI and package structure - Move CLI code to ETF_Portal package - Add proper package setup with setup.py - Update README with installation and usage instructions - Improve process management and cleanup --- ETF_Portal/__init__.py | 7 + ETF_Portal/cli.py | 547 +++++++++++++++++++++++++++++++++++ README.md | 137 ++++++--- cli_manager.py | 265 +++++++++++++++++ config/etf_suite_config.json | 8 + etf_suite_cli.py | 415 ++++++++++++++++++-------- pages/cache_manager.py | 235 +++++++++++++++ portfolios/Testing1.json | 29 ++ setup.py | 24 +- 9 files changed, 1505 insertions(+), 162 deletions(-) create mode 100644 ETF_Portal/__init__.py create mode 100644 ETF_Portal/cli.py create mode 100755 cli_manager.py create mode 100644 config/etf_suite_config.json mode change 100644 => 100755 etf_suite_cli.py create mode 100644 pages/cache_manager.py create mode 100644 portfolios/Testing1.json diff --git a/ETF_Portal/__init__.py b/ETF_Portal/__init__.py new file mode 100644 index 0000000..a68c0c6 --- /dev/null +++ b/ETF_Portal/__init__.py @@ -0,0 +1,7 @@ +""" +ETF Portal + +A comprehensive tool for ETF portfolio management and analysis. +""" + +__version__ = "0.1.0" \ No newline at end of file diff --git a/ETF_Portal/cli.py b/ETF_Portal/cli.py new file mode 100644 index 0000000..2f8b86c --- /dev/null +++ b/ETF_Portal/cli.py @@ -0,0 +1,547 @@ +#!/usr/bin/env python3 +""" +ETF Portal CLI + +A command-line interface for managing the ETF Portal application. +""" + +import click +import subprocess +import webbrowser +import time +import threading +import os +import sys +import socket +import signal +import json +import psutil +import logging +from pathlib import Path +from typing import List, Dict, Optional, Tuple +from datetime import datetime + +# Path configurations +WORKSPACE_PATH = Path(__file__).parent.parent +LAUNCHER_PATH = WORKSPACE_PATH / "ETF_Suite_Launcher.py" +PORTFOLIO_BUILDER_PATH = WORKSPACE_PATH / "pages" / "ETF_Portfolio_Builder.py" +ANALYZER_PATH = WORKSPACE_PATH / "pages" / "ETF_Analyzer.py" +CONFIG_DIR = WORKSPACE_PATH / "config" +CONFIG_FILE = CONFIG_DIR / "etf_suite_config.json" +LOGS_DIR = WORKSPACE_PATH / "logs" + +# Default port settings +DEFAULT_PORTS = { + "launcher": 8500, + "portfolio_builder": 8501, + "analyzer": 8502 +} + +# Process tracking +active_processes = {} + +# Setup logging +def setup_logging(): + """Set up logging configuration.""" + LOGS_DIR.mkdir(exist_ok=True) + log_file = LOGS_DIR / f"etf_portal_{datetime.now().strftime('%Y%m%d')}.log" + + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + handlers=[ + logging.FileHandler(log_file), + logging.StreamHandler() + ] + ) + return logging.getLogger("etf_portal") + +logger = setup_logging() + +def ensure_config_exists(): + """Ensure config directory and file exist.""" + CONFIG_DIR.mkdir(exist_ok=True) + + if not CONFIG_FILE.exists(): + with open(CONFIG_FILE, 'w') as f: + json.dump({ + "ports": DEFAULT_PORTS, + "streamlit_path": os.path.join(sys.prefix, "bin", "streamlit") + }, f, indent=2) + +def get_config(): + """Get the configuration from the config file.""" + ensure_config_exists() + with open(CONFIG_FILE, 'r') as f: + return json.load(f) + +def update_config(key, value): + """Update a specific configuration value.""" + config = get_config() + + # Handle nested configuration like ports.launcher + if '.' in key: + main_key, sub_key = key.split('.', 1) + if main_key not in config: + config[main_key] = {} + config[main_key][sub_key] = value + else: + config[key] = value + + with open(CONFIG_FILE, 'w') as f: + json.dump(config, f, indent=2) + +def cleanup_streamlit_processes(): + """Kill any existing Streamlit processes to prevent conflicts.""" + click.echo("Cleaning up existing Streamlit processes...") + logger.info("Cleaning up existing Streamlit processes") + try: + config = get_config() + ports = config["ports"] + processed_pids = set() # Track PIDs we've already handled + + # First, find and kill processes using our target ports + for port in ports.values(): + try: + # Find process using the port + cmd = f"lsof -i :{port} | grep LISTEN | awk '{{print $2}}'" + result = subprocess.run(cmd, shell=True, capture_output=True, text=True) + if result.stdout.strip(): + pids = result.stdout.strip().split('\n') + for pid in pids: + pid = int(pid) + if pid not in processed_pids: + try: + os.kill(pid, signal.SIGTERM) + logger.info(f"Terminated process {pid} using port {port}") + processed_pids.add(pid) + except ProcessLookupError: + pass + except Exception as e: + logger.error(f"Error cleaning up port {port}: {e}") + + # Then find and kill any remaining Streamlit processes + for proc in psutil.process_iter(['pid', 'name', 'cmdline']): + try: + if proc.info['pid'] not in processed_pids and 'streamlit' in ' '.join(proc.info['cmdline'] or []).lower(): + proc.terminate() + logger.info(f"Terminated Streamlit process {proc.info['pid']}") + processed_pids.add(proc.info['pid']) + except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): + pass + + # Give processes time to shut down + time.sleep(2) + + # Force kill any remaining processes + for proc in psutil.process_iter(['pid', 'name', 'cmdline']): + try: + if proc.info['pid'] not in processed_pids and 'streamlit' in ' '.join(proc.info['cmdline'] or []).lower(): + proc.kill() + logger.info(f"Force killed Streamlit process {proc.info['pid']}") + processed_pids.add(proc.info['pid']) + except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): + pass + + except Exception as e: + logger.error(f"Error during cleanup: {e}") + click.echo(f"Warning during cleanup: {e}") + +def port_is_available(port): + """Check if a port is available.""" + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", port)) + return True + except socket.error: + return False + +def open_browser(url, delay=3): + """Open browser after a delay to ensure app is running.""" + time.sleep(delay) + click.echo(f"Opening browser to {url}") + webbrowser.open(url) + +def start_component(component: str, open_browser_tab=True, background=False) -> Optional[subprocess.Popen]: + """Start a specific component of the ETF Portal.""" + config = get_config() + streamlit_path = config.get("streamlit_path", os.path.join(sys.prefix, "bin", "streamlit")) + ports = config["ports"] + + # Check if streamlit exists at the specified path + if not os.path.exists(streamlit_path): + error_msg = f"ERROR: Streamlit not found at {streamlit_path}" + logger.error(error_msg) + click.echo(error_msg) + click.echo("Please install streamlit or update the streamlit_path in config") + return None + + # Component-specific configurations + component_configs = { + "launcher": { + "path": LAUNCHER_PATH, + "port": ports["launcher"], + }, + "portfolio_builder": { + "path": PORTFOLIO_BUILDER_PATH, + "port": ports["portfolio_builder"], + }, + "analyzer": { + "path": ANALYZER_PATH, + "port": ports["analyzer"], + } + } + + if component not in component_configs: + error_msg = f"Unknown component: {component}" + logger.error(error_msg) + click.echo(error_msg) + return None + + component_config = component_configs[component] + port = component_config["port"] + + if not port_is_available(port): + error_msg = f"ERROR: Port {port} is in use." + logger.error(error_msg) + click.echo(error_msg) + click.echo(f"Please stop existing service on port {port} or configure a different port.") + return None + + log_file = None + if background: + # Create log file for background process + LOGS_DIR.mkdir(exist_ok=True) + log_file = LOGS_DIR / f"{component}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log" + logger.info(f"Starting {component} in background mode. Logs will be written to {log_file}") + + click.echo(f"Starting {component} on port {port}..." + (" (background mode)" if background else "")) + + # Prepare command + cmd = [ + streamlit_path, "run", + str(component_config["path"]), + "--server.port", str(port), + "--server.fileWatcherType", "none" # Disable file watcher to prevent inotify issues + ] + + # Launch the component + if background: + # In background mode, redirect output to log file + with open(log_file, 'w') as log: + process = subprocess.Popen( + cmd, + stdout=log, + stderr=log, + # Make the process independent of the parent + start_new_session=True + ) + else: + # In foreground mode, just run normally + process = subprocess.Popen(cmd) + + # Store process for tracking + active_processes[component] = { + "process": process, + "port": port, + "pid": process.pid, + "background": background + } + + # Open browser pointing to the component + if open_browser_tab: + thread = threading.Thread( + target=open_browser, + args=(f"http://localhost:{port}",) + ) + thread.daemon = True + thread.start() + + # If running in background, we don't need to wait + if background: + logger.info(f"Started {component} in background mode (PID: {process.pid})") + # Give it a moment to start + time.sleep(1) + + # Check if the process is still running + if process.poll() is not None: + error_msg = f"Error: {component} failed to start in background mode" + logger.error(error_msg) + click.echo(error_msg) + return None + + return process + +def get_streamlit_processes() -> List[Dict]: + """Get a list of running Streamlit processes.""" + processes = [] + seen_ports = set() # Track ports we've seen to avoid duplicates + + for proc in psutil.process_iter(['pid', 'name', 'cmdline', 'create_time']): + try: + cmdline = proc.info['cmdline'] + if not cmdline or 'streamlit' not in ' '.join(cmdline).lower(): + continue + + port = None + component = None + + # Extract port from command line + for i, arg in enumerate(cmdline): + if arg == '--server.port' and i + 1 < len(cmdline): + port = cmdline[i + 1] + # Skip if we've already seen this port (likely a duplicate) + if port in seen_ports: + continue + seen_ports.add(port) + + # Identify which component based on file path + for i, arg in enumerate(cmdline): + if arg == 'run' and i + 1 < len(cmdline): + path = cmdline[i + 1] + if 'ETF_Suite_Launcher.py' in path: + component = 'launcher' + elif 'ETF_Portfolio_Builder.py' in path: + component = 'portfolio_builder' + elif 'ETF_Analyzer.py' in path: + component = 'analyzer' + + # Only add processes that have a valid port + if port: + processes.append({ + 'pid': proc.info['pid'], + 'port': port, + 'component': component, + 'cmdline': ' '.join(cmdline), + 'create_time': proc.info['create_time'] + }) + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + + # Sort by creation time (newest first) + processes.sort(key=lambda x: x['create_time'], reverse=True) + return processes + +def stop_component(component=None, pid=None): + """Stop a specific component or Streamlit process.""" + if pid: + try: + process = psutil.Process(pid) + process.terminate() + try: + process.wait(timeout=5) + except psutil.TimeoutExpired: + process.kill() + click.echo(f"Stopped process with PID {pid}") + logger.info(f"Stopped process with PID {pid}") + return True + except psutil.NoSuchProcess: + click.echo(f"No process found with PID {pid}") + return False + elif component: + # Check active tracked processes first + if component in active_processes: + process_info = active_processes[component] + try: + process = process_info["process"] + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + click.echo(f"Stopped {component}") + logger.info(f"Stopped {component}") + del active_processes[component] + return True + except Exception: + pass + + # Fall back to finding by component name in running processes + processes = get_streamlit_processes() + for proc in processes: + if proc['component'] == component: + return stop_component(pid=proc['pid']) + + click.echo(f"No running {component} process found") + return False + +@click.group() +def cli(): + """ETF Portal CLI - Manage the ETF Portal application.""" + pass + +@cli.command() +@click.option('--component', type=click.Choice(['launcher', 'portfolio_builder', 'analyzer', 'all']), + default='launcher', help='Component to start') +@click.option('--no-browser', is_flag=True, help="Don't open browser automatically") +@click.option('--background', is_flag=True, help="Run in background mode (daemon)") +def start(component, no_browser, background): + """Start ETF Portal components.""" + if component == 'all': + # Start launcher first, it will manage the other components + process = start_component('launcher', not no_browser, background) + if not process: + return + else: + process = start_component(component, not no_browser, background) + if not process: + return + + click.echo(f"Started {component}" + (" in background mode" if background else "")) + + # In background mode, we just return immediately + if background: + return + + # In foreground mode, wait for the process + click.echo("Press Ctrl+C to exit") + + # Keep running until interrupted + try: + if component == 'all' or component == 'launcher': + process.wait() + else: + # For individual components, we'll just exit + return + except KeyboardInterrupt: + click.echo("\nShutting down...") + if component == 'all': + stop_component('launcher') + else: + stop_component(component) + +@cli.command() +@click.option('--component', type=click.Choice(['launcher', 'portfolio_builder', 'analyzer', 'all']), + default='all', help='Component to stop') +@click.option('--pid', type=int, help='Process ID to stop') +def stop(component, pid): + """Stop ETF Portal components.""" + if pid: + stop_component(pid=pid) + elif component == 'all': + cleanup_streamlit_processes() + click.echo("Stopped all ETF Portal components") + logger.info("Stopped all ETF Portal components") + else: + stop_component(component) + +@cli.command() +@click.option('--component', type=click.Choice(['launcher', 'portfolio_builder', 'analyzer', 'all']), + default='all', help='Component to restart') +@click.option('--no-browser', is_flag=True, help="Don't open browser automatically") +@click.option('--background', is_flag=True, help="Run in background mode (daemon)") +def restart(component, no_browser, background): + """Restart ETF Portal components (stop and then start).""" + # First stop the components + if component == 'all': + cleanup_streamlit_processes() + click.echo("Stopped all ETF Portal components") + logger.info("Stopped all ETF Portal components") + else: + stop_component(component) + + # Give processes time to fully shut down + time.sleep(2) + + # Then start them again + if component == 'all': + # Start launcher first, it will manage the other components + process = start_component('launcher', not no_browser, background) + if not process: + return + else: + process = start_component(component, not no_browser, background) + if not process: + return + + click.echo(f"Restarted {component}" + (" in background mode" if background else "")) + + # In background mode, we just return immediately + if background: + return + + # In foreground mode, wait for the process + click.echo("Press Ctrl+C to exit") + + # Keep running until interrupted + try: + if component == 'all' or component == 'launcher': + process.wait() + else: + # For individual components, we'll just exit + return + except KeyboardInterrupt: + click.echo("\nShutting down...") + if component == 'all': + stop_component('launcher') + else: + stop_component(component) + +@cli.command() +def status(): + """Check status of ETF Portal components.""" + processes = get_streamlit_processes() + + if not processes: + click.echo("No ETF Portal processes are currently running.") + return + + click.echo("Running ETF Portal processes:") + for i, proc in enumerate(processes): + component = proc['component'] or 'unknown' + port = proc['port'] or 'unknown' + click.echo(f"{i+1}. {component.upper()} (PID: {proc['pid']}, Port: {port})") + +@cli.command() +@click.option('--key', required=True, help='Configuration key to update (e.g., ports.launcher)') +@click.option('--value', required=True, help='New value') +def config(key, value): + """View or update configuration.""" + try: + # Convert value to integer if possible + try: + value = int(value) + except ValueError: + pass + + update_config(key, value) + click.echo(f"Updated {key} to {value}") + logger.info(f"Updated configuration: {key}={value}") + except Exception as e: + error_msg = f"Error updating configuration: {e}" + logger.error(error_msg) + click.echo(error_msg) + +@cli.command() +def logs(): + """Show recent logs from ETF Portal components.""" + LOGS_DIR.mkdir(exist_ok=True) + + log_files = sorted(LOGS_DIR.glob("*.log"), key=os.path.getmtime, reverse=True) + + if not log_files: + click.echo("No log files found.") + return + + click.echo("Recent log files:") + for i, log_file in enumerate(log_files[:5]): # Show 5 most recent logs + size = os.path.getsize(log_file) / 1024 # Size in KB + modified = datetime.fromtimestamp(os.path.getmtime(log_file)).strftime('%Y-%m-%d %H:%M:%S') + click.echo(f"{i+1}. {log_file.name} ({size:.1f} KB, last modified: {modified})") + + # Show most recent log contents + if log_files: + most_recent = log_files[0] + click.echo(f"\nMost recent log ({most_recent.name}):") + try: + # Show last 20 lines of the most recent log + with open(most_recent, 'r') as f: + lines = f.readlines() + for line in lines[-20:]: + click.echo(line.strip()) + except Exception as e: + click.echo(f"Error reading log file: {e}") + +def main(): + """Main entry point for the CLI.""" + cli() \ No newline at end of file diff --git a/README.md b/README.md index 80a6d44..05ff28e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,82 @@ -# ETF Dividend Portfolio Builder +# ETF Portal -A comprehensive tool for discovering, analyzing, and building dividend-focused ETF portfolios using real-time market data. +A comprehensive tool for ETF portfolio management and analysis. + +## Installation + +1. Clone the repository: +```bash +git clone +cd ETF_Portal +``` + +2. Create and activate a virtual environment: +```bash +python3 -m venv venv +source venv/bin/activate # On Linux/Mac +# or +.\venv\Scripts\activate # On Windows +``` + +3. Install the package in development mode: +```bash +pip install -e . +``` + +## Usage + +The ETF Portal provides a command-line interface for managing the application: + +```bash +# Start the launcher +etf-portal start + +# Start a specific component +etf-portal start --component portfolio_builder + +# Start in background mode +etf-portal start --component launcher --background + +# Stop all components +etf-portal stop + +# Stop a specific component +etf-portal stop --component analyzer + +# Check status +etf-portal status + +# Restart components +etf-portal restart + +# View logs +etf-portal logs + +# Update configuration +etf-portal config --key ports.launcher --value 8500 +``` + +## Components + +- **Launcher**: The main entry point for the ETF Portal (port 8500) +- **Portfolio Builder**: Tool for building and managing ETF portfolios (port 8501) +- **Analyzer**: Tool for analyzing ETF portfolios (port 8502) + +## Configuration + +The configuration file is located at `config/etf_suite_config.json`. You can modify it directly or use the `config` command: + +```bash +etf-portal config --key ports.launcher --value 8500 +``` + +## Logs + +Logs are stored in the `logs` directory. You can view recent logs using: + +```bash +etf-portal logs +``` ## Features @@ -47,49 +123,40 @@ The ETF Suite CLI provides a convenient way to manage the different components o ### Installation -``` +```bash +# Install the package pip install -e . + +# Create required directories +sudo mkdir -p /var/run/etf-portal +sudo mkdir -p /var/log/etf-portal +sudo chown -R $USER:$USER /var/run/etf-portal +sudo chown -R $USER:$USER /var/log/etf-portal ``` ### Usage +```bash +# Start the ETF Portal application +etf-portal start + +# Stop the ETF Portal application +etf-portal stop + +# Restart the ETF Portal application +etf-portal restart + +# Check the status of the ETF Portal application +etf-portal status ``` -# Start the entire ETF Suite (Launcher, Portfolio Builder, Analyzer) -etf-suite start --component all -# Start just the ETF Analyzer -etf-suite start --component analyzer +## Logs -# Start without opening browser automatically -etf-suite start --component launcher --no-browser +Logs are stored in `/var/log/etf-portal/cli_manager.log` -# Start in background mode (daemon) -etf-suite start --component all --background +## PID File -# Stop all running ETF Suite components -etf-suite stop - -# Stop a specific component -etf-suite stop --component portfolio_builder - -# Restart all components -etf-suite restart - -# Restart a specific component -etf-suite restart --component analyzer - -# Restart in background mode -etf-suite restart --background - -# Check status of running components -etf-suite status - -# View recent logs -etf-suite logs - -# Update configuration -etf-suite config --key ports.launcher --value 8600 -``` +The PID file is stored in `/var/run/etf-portal/etf_portal.pid` ## Usage diff --git a/cli_manager.py b/cli_manager.py new file mode 100755 index 0000000..b00cab8 --- /dev/null +++ b/cli_manager.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +import os +import sys +import time +import signal +import logging +import argparse +import subprocess +from pathlib import Path +from typing import Optional, Dict, Any +import psutil + +class CLIManager: + """CLI Manager for ETF Portal application. + + Provides commands to: + - start: Launch the application + - stop: Gracefully stop the application + - restart: Stop and start the application + - status: Check if the application is running + """ + + def __init__(self): + self.app_name = "ETF_Portal" + self.pid_file = Path("/var/run/etf-portal/etf_portal.pid") + self.log_file = Path("/var/log/etf-portal/cli_manager.log") + self._setup_logging() + + def _setup_logging(self): + """Configure logging for the CLI manager.""" + self.log_file.parent.mkdir(parents=True, exist_ok=True) + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + handlers=[ + logging.FileHandler(self.log_file), + logging.StreamHandler() + ] + ) + self.logger = logging.getLogger("CLIManager") + + def _get_pid(self) -> Optional[int]: + """Get the PID from the PID file if it exists.""" + try: + if self.pid_file.exists(): + with open(self.pid_file, 'r') as f: + pid = int(f.read().strip()) + return pid + except (ValueError, IOError) as e: + self.logger.error(f"Error reading PID file: {e}") + return None + + def _save_pid(self, pid: int): + """Save the PID to the PID file.""" + try: + self.pid_file.parent.mkdir(parents=True, exist_ok=True) + with open(self.pid_file, 'w') as f: + f.write(str(pid)) + except IOError as e: + self.logger.error(f"Error saving PID file: {e}") + + def _is_process_running(self, pid: int) -> bool: + """Check if a process with the given PID is running.""" + try: + process = psutil.Process(pid) + # Check if it's a Python process running Streamlit + is_running = process.is_running() + is_streamlit = any('streamlit' in cmd.lower() for cmd in process.cmdline()) + is_root = process.uids().real == 0 # Check if process is running as root + + if is_running and is_streamlit: + if is_root: + self.logger.warning(f"Process {pid} is running as root") + return True + return False + except psutil.NoSuchProcess: + return False + except psutil.AccessDenied: + self.logger.warning(f"Access denied to process {pid}") + return False + + def _find_streamlit_process(self) -> Optional[int]: + """Find the main Streamlit process.""" + for proc in psutil.process_iter(['pid', 'cmdline', 'uids']): + try: + if any('streamlit' in cmd.lower() for cmd in proc.info['cmdline']): + if proc.info['uids'].real == 0: + self.logger.warning(f"Found Streamlit process {proc.info['pid']} running as root") + return proc.info['pid'] + except (psutil.NoSuchProcess, psutil.AccessDenied): + continue + return None + + def start(self): + """Start the ETF Portal application.""" + # Check if Streamlit is already running + existing_pid = self._find_streamlit_process() + if existing_pid: + self.logger.info(f"{self.app_name} is already running with PID {existing_pid}") + self._save_pid(existing_pid) + return + + try: + # Get the full path to streamlit in the virtual environment + venv_streamlit = os.path.join(os.path.dirname(sys.executable), "streamlit") + if not os.path.exists(venv_streamlit): + raise Exception(f"Streamlit not found at {venv_streamlit}") + + # Ensure we're using the virtual environment's Python + env = os.environ.copy() + env["PATH"] = os.path.dirname(sys.executable) + ":" + env["PATH"] + + # Start the application using streamlit + process = subprocess.Popen( + [venv_streamlit, "run", "ETF_Suite_Launcher.py", "--server.port=8500"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + start_new_session=True, + env=env, + preexec_fn=os.setsid # Create a new session + ) + + # Wait a moment for Streamlit to start + time.sleep(2) + + # Find the actual Streamlit process + streamlit_pid = self._find_streamlit_process() + if streamlit_pid: + # Verify the process is running with correct permissions + try: + proc = psutil.Process(streamlit_pid) + if proc.uids().real == 0: + self.logger.error("Process started as root. Stopping...") + self.stop() + raise Exception("Process started with incorrect permissions") + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + + self._save_pid(streamlit_pid) + self.logger.info(f"{self.app_name} started with PID {streamlit_pid}") + else: + raise Exception("Streamlit process not found after startup") + + except Exception as e: + self.logger.error(f"Failed to start {self.app_name}: {e}") + sys.exit(1) + + def stop(self): + """Stop the ETF Portal application.""" + # First check the PID file + pid = self._get_pid() + if not pid: + # If no PID file, try to find running Streamlit process + pid = self._find_streamlit_process() + if not pid: + self.logger.info(f"{self.app_name} is not running") + return + + try: + if self._is_process_running(pid): + # Get the process using psutil + process = psutil.Process(pid) + + # Check if process is running as root + if process.uids().real == 0: + self.logger.error(f"Cannot stop process {pid} - it is running as root") + self.logger.info("Please use sudo to stop the process:") + self.logger.info(f"sudo kill {pid}") + return + + # Try to terminate the process and its children + try: + # Get all child processes + children = process.children(recursive=True) + + # Terminate children first + for child in children: + try: + child.terminate() + except psutil.NoSuchProcess: + pass + + # Terminate the main process + process.terminate() + + # Wait for processes to terminate + gone, alive = psutil.wait_procs([process] + children, timeout=10) + + # If any processes are still alive, force kill them + for p in alive: + try: + p.kill() + except psutil.NoSuchProcess: + pass + + self.logger.info(f"{self.app_name} stopped") + except psutil.NoSuchProcess: + self.logger.info(f"{self.app_name} is not running") + else: + self.logger.info(f"{self.app_name} is not running") + + # Remove PID file + if self.pid_file.exists(): + self.pid_file.unlink() + + except Exception as e: + self.logger.error(f"Error stopping {self.app_name}: {e}") + # Don't exit with error, just log it + self.logger.info("Attempting to clean up PID file...") + if self.pid_file.exists(): + self.pid_file.unlink() + + def restart(self): + """Restart the ETF Portal application.""" + self.logger.info("Restarting ETF Portal...") + self.stop() + time.sleep(2) # Wait for processes to fully stop + self.start() + + def status(self): + """Check the status of the ETF Portal application.""" + # First check the PID file + pid = self._get_pid() + if not pid: + # If no PID file, try to find running Streamlit process + pid = self._find_streamlit_process() + if not pid: + print(f"{self.app_name} is not running") + return + + if self._is_process_running(pid): + print(f"{self.app_name} is running with PID {pid}") + else: + print(f"{self.app_name} is not running (stale PID file)") + if self.pid_file.exists(): + self.pid_file.unlink() + +def main(): + """Main entry point for the CLI.""" + parser = argparse.ArgumentParser( + description="ETF Portal CLI Manager", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + etf-portal start # Start the ETF Portal application + etf-portal stop # Stop the ETF Portal application + etf-portal restart # Restart the ETF Portal application + etf-portal status # Check the status of the ETF Portal application + """ + ) + + parser.add_argument( + "command", + choices=["start", "stop", "restart", "status"], + help="Command to execute" + ) + + args = parser.parse_args() + manager = CLIManager() + + # Execute the requested command + getattr(manager, args.command)() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/config/etf_suite_config.json b/config/etf_suite_config.json new file mode 100644 index 0000000..7e66b31 --- /dev/null +++ b/config/etf_suite_config.json @@ -0,0 +1,8 @@ +{ + "ports": { + "launcher": 8500, + "portfolio_builder": 8501, + "analyzer": 8502 + }, + "streamlit_path": "/home/pascal/Dev/ETF_Portal/venv/bin/streamlit" +} \ No newline at end of file diff --git a/etf_suite_cli.py b/etf_suite_cli.py old mode 100644 new mode 100755 index 57084bb..542699e --- a/etf_suite_cli.py +++ b/etf_suite_cli.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ ETF Suite CLI @@ -24,7 +24,7 @@ from datetime import datetime # Path configurations WORKSPACE_PATH = Path(__file__).parent LAUNCHER_PATH = WORKSPACE_PATH / "ETF_Suite_Launcher.py" -PORTFOLIO_BUILDER_PATH = WORKSPACE_PATH / "pages" / "ETF_Dividend_Portfolio_Builder.py" +PORTFOLIO_BUILDER_PATH = WORKSPACE_PATH / "pages" / "ETF_Portfolio_Builder.py" ANALYZER_PATH = WORKSPACE_PATH / "pages" / "ETF_Analyzer.py" CONFIG_DIR = WORKSPACE_PATH / "config" CONFIG_FILE = CONFIG_DIR / "etf_suite_config.json" @@ -38,7 +38,7 @@ DEFAULT_PORTS = { } # Full path to streamlit executable - may need to be adjusted -STREAMLIT_PATH = "/home/pascal/.local/bin/streamlit" +STREAMLIT_PATH = "/home/pascal/Dev/ETF_Portal/venv/bin/streamlit" # Process tracking active_processes = {} @@ -72,12 +72,14 @@ def ensure_config_exists(): "streamlit_path": STREAMLIT_PATH }, f, indent=2) + def get_config(): """Get the configuration from the config file.""" ensure_config_exists() with open(CONFIG_FILE, 'r') as f: return json.load(f) + def update_config(key, value): """Update a specific configuration value.""" config = get_config() @@ -94,6 +96,7 @@ def update_config(key, value): with open(CONFIG_FILE, 'w') as f: json.dump(config, f, indent=2) + def cleanup_streamlit_processes(): """Kill any existing Streamlit processes to prevent conflicts.""" click.echo("Cleaning up existing Streamlit processes...") @@ -101,22 +104,56 @@ def cleanup_streamlit_processes(): try: config = get_config() ports = config["ports"] + processed_pids = set() # Track PIDs we've already handled - # Find processes using our target ports + # First, find and kill processes using our target ports for port in ports.values(): - cmd = f"lsof -i :{port} | grep LISTEN | awk '{{print $2}}' | xargs kill -9 2>/dev/null || true" - subprocess.run(cmd, shell=True) - - # Find and kill any lingering Streamlit processes - cmd = "pkill -f streamlit || true" - subprocess.run(cmd, shell=True) + try: + # Find process using the port + cmd = f"lsof -i :{port} | grep LISTEN | awk '{{print $2}}'" + result = subprocess.run(cmd, shell=True, capture_output=True, text=True) + if result.stdout.strip(): + pids = result.stdout.strip().split('\n') + for pid in pids: + pid = int(pid) + if pid not in processed_pids: + try: + os.kill(pid, signal.SIGTERM) + logger.info(f"Terminated process {pid} using port {port}") + processed_pids.add(pid) + except ProcessLookupError: + pass + except Exception as e: + logger.error(f"Error cleaning up port {port}: {e}") + + # Then find and kill any remaining Streamlit processes + for proc in psutil.process_iter(['pid', 'name', 'cmdline']): + try: + if proc.info['pid'] not in processed_pids and 'streamlit' in ' '.join(proc.info['cmdline'] or []).lower(): + proc.terminate() + logger.info(f"Terminated Streamlit process {proc.info['pid']}") + processed_pids.add(proc.info['pid']) + except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): + pass # Give processes time to shut down - time.sleep(1) + time.sleep(2) + + # Force kill any remaining processes + for proc in psutil.process_iter(['pid', 'name', 'cmdline']): + try: + if proc.info['pid'] not in processed_pids and 'streamlit' in ' '.join(proc.info['cmdline'] or []).lower(): + proc.kill() + logger.info(f"Force killed Streamlit process {proc.info['pid']}") + processed_pids.add(proc.info['pid']) + except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): + pass + except Exception as e: logger.error(f"Error during cleanup: {e}") click.echo(f"Warning during cleanup: {e}") + def port_is_available(port): """Check if a port is available.""" try: @@ -126,12 +163,14 @@ def port_is_available(port): except socket.error: return False + def open_browser(url, delay=3): """Open browser after a delay to ensure app is running.""" time.sleep(delay) click.echo(f"Opening browser to {url}") webbrowser.open(url) + def start_component(component: str, open_browser_tab=True, background=False) -> Optional[subprocess.Popen]: """Start a specific component of the ETF Suite.""" config = get_config() @@ -192,95 +231,146 @@ def start_component(component: str, open_browser_tab=True, background=False) -> streamlit_path, "run", str(component_config["path"]), "--server.port", str(port), - "--server.address", "0.0.0.0", - "--server.headless", "true", - "--browser.gatherUsageStats", "false" + "--server.fileWatcherType", "none" # Disable file watcher to prevent inotify issues ] - # Add component-specific options - if component == "portfolio_builder": - cmd.extend(["--server.baseUrlPath", "/portfolio"]) - elif component == "analyzer": - cmd.extend(["--server.baseUrlPath", "/analyzer"]) - - try: - if background: - with open(log_file, 'w') as f: - process = subprocess.Popen( - cmd, - stdout=f, - stderr=f, - start_new_session=True - ) - else: + # Launch the component + if background: + # In background mode, redirect output to log file + with open(log_file, 'w') as log: process = subprocess.Popen( cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True + stdout=log, + stderr=log, + # Make the process independent of the parent + start_new_session=True ) + else: + # In foreground mode, just run normally + process = subprocess.Popen(cmd) + + # Store process for tracking + active_processes[component] = { + "process": process, + "port": port, + "pid": process.pid, + "background": background + } + + # Open browser pointing to the component + if open_browser_tab: + thread = threading.Thread( + target=open_browser, + args=(f"http://localhost:{port}",) + ) + thread.daemon = True + thread.start() + + # If running in background, we don't need to wait + if background: + logger.info(f"Started {component} in background mode (PID: {process.pid})") + # Give it a moment to start + time.sleep(1) - active_processes[component] = process - - if open_browser_tab and not background: - url = f"http://localhost:{port}" - threading.Thread(target=open_browser, args=(url,)).start() - - return process - except Exception as e: - error_msg = f"Error starting {component}: {e}" - logger.error(error_msg) - click.echo(error_msg) - return None + # Check if the process is still running + if process.poll() is not None: + error_msg = f"Error: {component} failed to start in background mode" + logger.error(error_msg) + click.echo(error_msg) + return None + + return process + def get_streamlit_processes() -> List[Dict]: - """Get information about running Streamlit processes.""" + """Get a list of running Streamlit processes.""" processes = [] + for proc in psutil.process_iter(['pid', 'name', 'cmdline']): try: - if 'streamlit' in ' '.join(proc.info['cmdline'] or []): + cmdline = proc.info['cmdline'] + if cmdline and 'streamlit' in ' '.join(cmdline): + port = None + component = None + + # Extract port from command line + for i, arg in enumerate(cmdline): + if arg == '--server.port' and i + 1 < len(cmdline): + port = cmdline[i + 1] + + # Identify which component based on file path + for i, arg in enumerate(cmdline): + if arg == 'run' and i + 1 < len(cmdline): + path = cmdline[i + 1] + if 'ETF_Suite_Launcher.py' in path: + component = 'launcher' + elif 'ETF_Portfolio_Builder.py' in path: + component = 'portfolio_builder' + elif 'ETF_Analyzer.py' in path: + component = 'analyzer' + processes.append({ 'pid': proc.info['pid'], - 'name': proc.info['name'], - 'cmdline': ' '.join(proc.info['cmdline'] or []) + 'port': port, + 'component': component, + 'cmdline': ' '.join(cmdline if cmdline else []) }) except (psutil.NoSuchProcess, psutil.AccessDenied): pass + return processes + def stop_component(component=None, pid=None): - """Stop a specific component or process.""" + """Stop a specific component or Streamlit process.""" if pid: try: - os.kill(pid, signal.SIGTERM) - click.echo(f"Stopped process {pid}") - return - except ProcessLookupError: - click.echo(f"Process {pid} not found") - return - - if component == "all": - cleanup_streamlit_processes() - return - - if component in active_processes: - process = active_processes[component] - try: + process = psutil.Process(pid) process.terminate() - process.wait(timeout=5) - click.echo(f"Stopped {component}") - except subprocess.TimeoutExpired: - process.kill() - click.echo(f"Force killed {component}") - del active_processes[component] - else: - click.echo(f"No active process found for {component}") + try: + process.wait(timeout=5) + except psutil.TimeoutExpired: + process.kill() + click.echo(f"Stopped process with PID {pid}") + logger.info(f"Stopped process with PID {pid}") + return True + except psutil.NoSuchProcess: + click.echo(f"No process found with PID {pid}") + return False + elif component: + # Check active tracked processes first + if component in active_processes: + process_info = active_processes[component] + try: + process = process_info["process"] + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + click.echo(f"Stopped {component}") + logger.info(f"Stopped {component}") + del active_processes[component] + return True + except Exception: + pass + + # Fall back to finding by component name in running processes + processes = get_streamlit_processes() + for proc in processes: + if proc['component'] == component: + return stop_component(pid=proc['pid']) + + click.echo(f"No running {component} process found") + return False + @click.group() def cli(): - """ETF Suite CLI - Manage your ETF Suite applications.""" + """ETF Suite CLI - Manage the ETF Suite application.""" pass + @cli.command() @click.option('--component', type=click.Choice(['launcher', 'portfolio_builder', 'analyzer', 'all']), default='launcher', help='Component to start') @@ -288,11 +378,39 @@ def cli(): @click.option('--background', is_flag=True, help="Run in background mode (daemon)") def start(component, no_browser, background): """Start ETF Suite components.""" - if component == "all": - for comp in ['launcher', 'portfolio_builder', 'analyzer']: - start_component(comp, not no_browser, background) + if component == 'all': + # Start launcher first, it will manage the other components + process = start_component('launcher', not no_browser, background) + if not process: + return else: - start_component(component, not no_browser, background) + process = start_component(component, not no_browser, background) + if not process: + return + + click.echo(f"Started {component}" + (" in background mode" if background else "")) + + # In background mode, we just return immediately + if background: + return + + # In foreground mode, wait for the process + click.echo("Press Ctrl+C to exit") + + # Keep running until interrupted + try: + if component == 'all' or component == 'launcher': + process.wait() + else: + # For individual components, we'll just exit + return + except KeyboardInterrupt: + click.echo("\nShutting down...") + if component == 'all': + stop_component('launcher') + else: + stop_component(component) + @cli.command() @click.option('--component', type=click.Choice(['launcher', 'portfolio_builder', 'analyzer', 'all']), @@ -300,7 +418,15 @@ def start(component, no_browser, background): @click.option('--pid', type=int, help='Process ID to stop') def stop(component, pid): """Stop ETF Suite components.""" - stop_component(component, pid) + if pid: + stop_component(pid=pid) + elif component == 'all': + cleanup_streamlit_processes() + click.echo("Stopped all ETF Suite components") + logger.info("Stopped all ETF Suite components") + else: + stop_component(component) + @cli.command() @click.option('--component', type=click.Choice(['launcher', 'portfolio_builder', 'analyzer', 'all']), @@ -308,67 +434,120 @@ def stop(component, pid): @click.option('--no-browser', is_flag=True, help="Don't open browser automatically") @click.option('--background', is_flag=True, help="Run in background mode (daemon)") def restart(component, no_browser, background): - """Restart ETF Suite components.""" - if component == "all": - stop_component("all") - time.sleep(2) - for comp in ['launcher', 'portfolio_builder', 'analyzer']: - start_component(comp, not no_browser, background) + """Restart ETF Suite components (stop and then start).""" + # First stop the components + if component == 'all': + cleanup_streamlit_processes() + click.echo("Stopped all ETF Suite components") + logger.info("Stopped all ETF Suite components") else: stop_component(component) - time.sleep(2) - start_component(component, not no_browser, background) + + # Give processes time to fully shut down + time.sleep(2) + + # Then start them again + if component == 'all': + # Start launcher first, it will manage the other components + process = start_component('launcher', not no_browser, background) + if not process: + return + else: + process = start_component(component, not no_browser, background) + if not process: + return + + click.echo(f"Restarted {component}" + (" in background mode" if background else "")) + + # In background mode, we just return immediately + if background: + return + + # In foreground mode, wait for the process + click.echo("Press Ctrl+C to exit") + + # Keep running until interrupted + try: + if component == 'all' or component == 'launcher': + process.wait() + else: + # For individual components, we'll just exit + return + except KeyboardInterrupt: + click.echo("\nShutting down...") + if component == 'all': + stop_component('launcher') + else: + stop_component(component) + @cli.command() def status(): - """Show status of ETF Suite components.""" + """Check status of ETF Suite components.""" processes = get_streamlit_processes() + if not processes: - click.echo("No ETF Suite components are running") + click.echo("No ETF Suite processes are currently running.") return - click.echo("Running components:") - for proc in processes: - click.echo(f"PID: {proc['pid']}") - click.echo(f"Command: {proc['cmdline']}") - click.echo("---") + click.echo("Running ETF Suite processes:") + for i, proc in enumerate(processes): + component = proc['component'] or 'unknown' + port = proc['port'] or 'unknown' + click.echo(f"{i+1}. {component.upper()} (PID: {proc['pid']}, Port: {port})") + @cli.command() @click.option('--key', required=True, help='Configuration key to update (e.g., ports.launcher)') @click.option('--value', required=True, help='New value') def config(key, value): - """Update ETF Suite configuration.""" + """View or update configuration.""" try: - # Convert value to appropriate type - if value.isdigit(): + # Convert value to integer if possible + try: value = int(value) - elif value.lower() in ('true', 'false'): - value = value.lower() == 'true' - + except ValueError: + pass + update_config(key, value) click.echo(f"Updated {key} to {value}") + logger.info(f"Updated configuration: {key}={value}") except Exception as e: - click.echo(f"Error updating config: {e}") + error_msg = f"Error updating configuration: {e}" + logger.error(error_msg) + click.echo(error_msg) + @cli.command() def logs(): - """Show recent logs.""" - try: - log_files = sorted(LOGS_DIR.glob("*.log"), key=lambda x: x.stat().st_mtime, reverse=True) - if not log_files: - click.echo("No log files found") - return - - latest_log = log_files[0] - click.echo(f"Showing last 20 lines of {latest_log.name}:") - click.echo("---") - - with open(latest_log) as f: - lines = f.readlines() - for line in lines[-20:]: - click.echo(line.strip()) - except Exception as e: - click.echo(f"Error reading logs: {e}") + """Show recent logs from ETF Suite components.""" + LOGS_DIR.mkdir(exist_ok=True) + + log_files = sorted(LOGS_DIR.glob("*.log"), key=os.path.getmtime, reverse=True) + + if not log_files: + click.echo("No log files found.") + return + + click.echo("Recent log files:") + for i, log_file in enumerate(log_files[:5]): # Show 5 most recent logs + size = os.path.getsize(log_file) / 1024 # Size in KB + modified = datetime.fromtimestamp(os.path.getmtime(log_file)).strftime('%Y-%m-%d %H:%M:%S') + click.echo(f"{i+1}. {log_file.name} ({size:.1f} KB, last modified: {modified})") + + # Show most recent log contents + if log_files: + most_recent = log_files[0] + click.echo(f"\nMost recent log ({most_recent.name}):") + try: + # Show last 20 lines of the most recent log + with open(most_recent, 'r') as f: + lines = f.readlines() + for line in lines[-20:]: + click.echo(line.strip()) + except Exception as e: + click.echo(f"Error reading log file: {e}") -if __name__ == '__main__': + +if __name__ == "__main__": cli() \ No newline at end of file diff --git a/pages/cache_manager.py b/pages/cache_manager.py new file mode 100644 index 0000000..2f65252 --- /dev/null +++ b/pages/cache_manager.py @@ -0,0 +1,235 @@ +import json +import os +from datetime import datetime, timedelta +from pathlib import Path +import logging +from typing import Any, Dict, Optional, Tuple, Union +import hashlib +import sys + +# Configure logging +log_dir = Path("logs") +log_dir.mkdir(exist_ok=True) +log_file = log_dir / f"cache_manager_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log" + +# Remove any existing handlers to avoid duplicate logs +for handler in logging.root.handlers[:]: + logging.root.removeHandler(handler) + +# Create a formatter +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + +# Create file handler +file_handler = logging.FileHandler(log_file, mode='a') +file_handler.setFormatter(formatter) +file_handler.setLevel(logging.INFO) + +# Create console handler +console_handler = logging.StreamHandler(sys.stdout) +console_handler.setFormatter(formatter) +console_handler.setLevel(logging.INFO) + +# Configure root logger +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) +logger.addHandler(file_handler) +logger.addHandler(console_handler) + +class CacheManager: + """ + Manages caching of ETF data to reduce API calls and improve performance. + Implements a time-based cache expiration system. + """ + + def __init__(self, cache_dir: str = "cache", cache_duration_hours: int = 24): + """ + Initialize the cache manager. + + Args: + cache_dir: Directory to store cache files + cache_duration_hours: Number of hours before cache expires + """ + self.cache_dir = Path(cache_dir) + self.cache_duration = timedelta(hours=cache_duration_hours) + + # Create cache directory if it doesn't exist + self.cache_dir.mkdir(parents=True, exist_ok=True) + + # Configure logging + self.logger = logging.getLogger(__name__) + self.logger.setLevel(logging.INFO) + + # Create formatter + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s' + ) + + # Create file handler + log_file = Path("logs") / f"cache_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log" + log_file.parent.mkdir(exist_ok=True) + file_handler = logging.FileHandler(log_file) + file_handler.setFormatter(formatter) + self.logger.addHandler(file_handler) + + self.logger.info(f"CacheManager initialized with directory: {self.cache_dir}") + self.logger.info(f"Cache duration: {cache_duration_hours} hours") + + def _generate_cache_key(self, source: str, ticker: str, data_type: str) -> str: + """ + Generate a unique cache key for the data. + + Args: + source: Data source (e.g., 'fmp', 'yf') + ticker: ETF ticker symbol + data_type: Type of data (e.g., 'profile', 'history') + + Returns: + Cache key string + """ + return f"{source}_{ticker}_{data_type}.json" + + def _get_cache_path(self, cache_key: str) -> Path: + """ + Get the full path for a cache file. + + Args: + cache_key: Cache key string + + Returns: + Path object for the cache file + """ + return self.cache_dir / cache_key + + def _is_cache_valid(self, cache_path: Path) -> bool: + """ + Check if a cache file is still valid based on its age. + + Args: + cache_path: Path to the cache file + + Returns: + True if cache is valid, False otherwise + """ + if not cache_path.exists(): + return False + + file_age = datetime.now() - datetime.fromtimestamp(cache_path.stat().st_mtime) + is_valid = file_age < self.cache_duration + + self.logger.debug(f"Cache file {cache_path} age: {file_age}, valid: {is_valid}") + return is_valid + + def save_to_cache(self, cache_key: str, data: Any) -> bool: + """ + Save data to cache. + + Args: + cache_key: Cache key string + data: Data to cache + + Returns: + True if save was successful, False otherwise + """ + try: + cache_path = self._get_cache_path(cache_key) + + # Create cache directory if it doesn't exist + cache_path.parent.mkdir(parents=True, exist_ok=True) + + # Save data to JSON file + with open(cache_path, 'w') as f: + json.dump(data, f, indent=2) + + self.logger.info(f"Data saved to cache: {cache_path}") + return True + + except Exception as e: + self.logger.error(f"Error saving to cache: {str(e)}") + return False + + def load_from_cache(self, cache_key: str) -> Tuple[Optional[Any], bool]: + """ + Load data from cache if it exists and is valid. + + Args: + cache_key: Cache key string + + Returns: + Tuple of (cached data, is_valid) + """ + try: + cache_path = self._get_cache_path(cache_key) + + if not cache_path.exists(): + self.logger.debug(f"Cache miss: {cache_path}") + return None, False + + if not self._is_cache_valid(cache_path): + self.logger.info(f"Cache expired: {cache_path}") + return None, False + + # Load data from JSON file + with open(cache_path, 'r') as f: + data = json.load(f) + + self.logger.info(f"Data loaded from cache: {cache_path}") + return data, True + + except Exception as e: + self.logger.error(f"Error loading from cache: {str(e)}") + return None, False + + def clear_expired_cache(self) -> int: + """ + Clear all expired cache files. + + Returns: + Number of files cleared + """ + try: + cleared_count = 0 + for cache_file in self.cache_dir.glob("*.json"): + if not self._is_cache_valid(cache_file): + cache_file.unlink() + cleared_count += 1 + self.logger.info(f"Cleared expired cache: {cache_file}") + + self.logger.info(f"Cleared {cleared_count} expired cache files") + return cleared_count + + except Exception as e: + self.logger.error(f"Error clearing expired cache: {str(e)}") + return 0 + + def get_cache_stats(self) -> Dict[str, Any]: + """ + Get statistics about the cache. + + Returns: + Dictionary with cache statistics + """ + try: + total_files = 0 + expired_files = 0 + total_size = 0 + + for cache_file in self.cache_dir.glob("*.json"): + total_files += 1 + total_size += cache_file.stat().st_size + if not self._is_cache_valid(cache_file): + expired_files += 1 + + stats = { + "total_files": total_files, + "expired_files": expired_files, + "total_size_bytes": total_size, + "cache_dir": str(self.cache_dir), + "cache_duration_hours": self.cache_duration.total_seconds() / 3600 + } + + self.logger.info(f"Cache statistics: {stats}") + return stats + + except Exception as e: + self.logger.error(f"Error getting cache stats: {str(e)}") + return {} \ No newline at end of file diff --git a/portfolios/Testing1.json b/portfolios/Testing1.json new file mode 100644 index 0000000..5c14ef4 --- /dev/null +++ b/portfolios/Testing1.json @@ -0,0 +1,29 @@ +{ + "name": "Testing1", + "created_at": "2025-05-25T15:29:13.669593", + "mode": "Income Target", + "target": 1000.0, + "allocations": [ + { + "ticker": "MSTY", + "allocation": 70.0, + "yield": 142.7343086361491, + "price": 21.19, + "risk_level": "High" + }, + { + "ticker": "ULTY", + "allocation": 20.0, + "yield": 45.937467700258395, + "price": 19.35, + "risk_level": "Medium" + }, + { + "ticker": "CONY", + "allocation": 10.0, + "yield": 67.58371269600406, + "price": 19.77, + "risk_level": "Medium" + } + ] +} \ No newline at end of file diff --git a/setup.py b/setup.py index 9614595..019c76b 100644 --- a/setup.py +++ b/setup.py @@ -2,17 +2,23 @@ from setuptools import setup, find_packages setup( - name="etf-suite-cli", + name="etf-portal", version="0.1.0", - description="Command-line interface for ETF Suite", - author="Pascal", - py_modules=["etf_suite_cli"], + packages=find_packages(), + include_package_data=True, install_requires=[ - "Click", + "click", "psutil", + "streamlit", ], - entry_points=""" - [console_scripts] - etf-suite=etf_suite_cli:cli - """, + entry_points={ + "console_scripts": [ + "etf-portal=ETF_Portal.cli:main", + ], + }, + python_requires=">=3.8", + author="Pascal", + description="ETF Portal CLI tool", + long_description=open("README.md").read() if open("README.md").read() else "", + long_description_content_type="text/markdown", ) \ No newline at end of file