diff --git a/GPUcollecter.pm b/GPUcollecter.pm index 2efb0ba..a8fc1af 100644 --- a/GPUcollecter.pm +++ b/GPUcollecter.pm @@ -8,14 +8,43 @@ use Fcntl qw(:flock); use Time::HiRes qw(time); use Fcntl qw(:flock O_CREAT O_EXCL O_WRONLY); +# Debug configuration - set to 0 to disable all debug output +my $DEBUG_ENABLED = 1; +# Debug function showing line number and call chain +# Usage: debug(__LINE__, "message") +sub debug { + return unless $DEBUG_ENABLED; + + my ($line, $message) = @_; + + # Get function call chain + my @caller1 = caller(1); # who called debug() + my @caller2 = caller(2); # parent of caller + + my $sub1 = $caller1[3] || 'main'; + my $sub2 = $caller2[3]; + + $sub1 =~ s/.*:://; # Remove package prefix + + if (defined $sub2) { + $sub2 =~ s/.*:://; + warn "[$sub2 -> $sub1:$line] $message\n"; + } else { + # No parent caller (called from top level) + warn "[$sub1:$line] $message\n"; + } +} + +my $stats_dir = '/var/run/pve-gpu'; my $state_file = '/var/run/pve-gpu/stats.json'; my $lock_file = '/var/run/pve-gpu/pve-gpu-collector.lock'; +my $startup_lock = $lock_file . ".startup"; my $last_snapshot = {}; my $last_mtime = 0; my $is_collector_parent = 0; # Flag to track if this process started collectors -my $last_get_stats_time = 0; # Track when get_stats was last called -my $COLLECTOR_TIMEOUT = 0; # Stop collectors x seconds after last get_stats call +my $last_get_graphic_stats_time = 0; # Track when get_graphic_stats was last called +my $COLLECTOR_TIMEOUT = 0; # Stop collectors x seconds after last get_graphic_stats call my $intel_gpu_enabled = 1; # Set to 0 to disable Intel GPU support my $amd_gpu_enabled = 0; # Set to 1 to enable AMD GPU support (not yet implemented) @@ -119,12 +148,12 @@ sub get_intel_gpu_devices { path => $path, drm_path => "/dev/dri/$card" }; - warn "DEBUG get_intel_gpu_devices: Found GPU device: $card -> $name ($path)"; + debug(__LINE__, "Found GPU device: $card -> $name ($path)"); } } close $fh; } else { - warn "DEBUG get_intel_gpu_devices: Failed to run intel_gpu_top -L: $!"; + debug(__LINE__, "Failed to run intel_gpu_top -L: $!"); } return @devices; @@ -140,31 +169,31 @@ sub collector_for_intel_device { # Each device writes to its own file my $device_state_file = "/var/run/pve-gpu/stats-$device->{card}.json"; - warn "DEBUG collector_for_intel_device: Collector started for device: $drm_dev, writing to $device_state_file"; + debug(__LINE__, "Collector started for device: $drm_dev, writing to $device_state_file"); # Set up signal handlers for graceful shutdown my $shutdown = 0; $SIG{TERM} = sub { - warn "DEBUG collector_for_intel_device: Collector for $device->{card} received SIGTERM"; + debug(__LINE__, "Collector for $device->{card} received SIGTERM"); $shutdown = 1; kill 'TERM', $intel_gpu_top_pid if defined $intel_gpu_top_pid && $intel_gpu_top_pid > 0; }; $SIG{INT} = sub { - warn "DEBUG collector_for_intel_device: Collector for $device->{card} received SIGINT"; + debug(__LINE__, "Collector for $device->{card} received SIGINT"); $shutdown = 1; kill 'TERM', $intel_gpu_top_pid if defined $intel_gpu_top_pid && $intel_gpu_top_pid > 0; }; # Run intel_gpu_top once and keep reading from it - warn "DEBUG: About to open pipe to intel_gpu_top"; + debug(__LINE__, "About to open pipe to intel_gpu_top"); $intel_gpu_top_pid = open(my $fh, '-|', "intel_gpu_top -d $drm_dev -s 1000 -l 2>&1"); unless (defined $intel_gpu_top_pid && $intel_gpu_top_pid > 0) { - warn "DEBUG collector_for_intel_device: Failed to run intel_gpu_top for $drm_dev: $!"; + debug(__LINE__, "Failed to run intel_gpu_top for $drm_dev: $!"); exit 1; } - warn "DEBUG: Pipe opened successfully, PID=$intel_gpu_top_pid"; + debug(__LINE__, "Pipe opened successfully, PID=$intel_gpu_top_pid"); my $line_count = 0; my $node_name = "node0"; # You may want to generate this based on device index @@ -198,17 +227,17 @@ sub collector_for_intel_device { open my $ofh, '>', $device_state_file or die "Failed to open $device_state_file: $!"; print $ofh JSON->new->pretty->encode($device_data); close $ofh; - warn "DEBUG: Wrote stats to $device_state_file (line #$line_count)"; + debug(__LINE__, "Wrote stats to $device_state_file (line #$line_count)"); }; if ($@) { - warn "DEBUG: Error writing stats: $@"; + debug(__LINE__, "Error writing stats: $@"); } } } } close $fh; - warn "DEBUG collector_for_intel_device: Collector for $device->{card} shutting down"; + debug(__LINE__, "Collector for $device->{card} shutting down"); exit 0; } @@ -219,7 +248,7 @@ sub collector_for_intel_device { sub get_amd_gpu_devices { # TODO: Implement AMD GPU detection # Use rocminfo or similar tools to detect AMD GPUs - warn "DEBUG: AMD GPU support not yet implemented"; + debug(__LINE__, "AMD GPU support not yet implemented"); return (); } @@ -227,14 +256,14 @@ sub parse_amd_gpu_line { my ($line) = @_; # TODO: Implement AMD GPU line parsing # Parse rocm-smi or similar output - warn "DEBUG: AMD GPU line parsing not yet implemented"; + debug(__LINE__, "AMD GPU line parsing not yet implemented"); return undef; } sub collector_for_amd_device { my ($device) = @_; # TODO: Implement AMD GPU collector - warn "DEBUG: AMD GPU collector not yet implemented"; + debug(__LINE__, "AMD GPU collector not yet implemented"); exit 0; } @@ -245,7 +274,7 @@ sub collector_for_amd_device { sub get_nvidia_gpu_devices { # TODO: Implement NVIDIA GPU detection # Use nvidia-smi to detect NVIDIA GPUs - warn "DEBUG: NVIDIA GPU support not yet implemented"; + debug(__LINE__, "NVIDIA GPU support not yet implemented"); return (); } @@ -253,14 +282,14 @@ sub parse_nvidia_gpu_line { my ($line) = @_; # TODO: Implement NVIDIA GPU line parsing # Parse nvidia-smi output - warn "DEBUG: NVIDIA GPU line parsing not yet implemented"; + debug(__LINE__, "NVIDIA GPU line parsing not yet implemented"); return undef; } sub collector_for_nvidia_device { my ($device) = @_; # TODO: Implement NVIDIA GPU collector - warn "DEBUG: NVIDIA GPU collector not yet implemented"; + debug(__LINE__, "NVIDIA GPU collector not yet implemented"); exit 0; } @@ -278,178 +307,144 @@ sub is_process_alive { # Main Collector # ============================================================================ -sub start_collector { - warn "DEBUG start_collector: Checking if collector is already running"; - - # Ensure directory exists - my $run_dir = '/var/run/pve-gpu'; - unless (-d $run_dir) { - warn "DEBUG start_collector: Creating directory $run_dir"; - mkdir($run_dir, 0755) or warn "DEBUG start_collector: Failed to create $run_dir: $!"; - } - - # Try to acquire startup lock FIRST (prevents race conditions) - my $startup_lock = $lock_file . ".startup"; - my $startup_fh; - - warn "DEBUG start_collector: Trying to acquire startup lock: $startup_lock"; - - unless (sysopen($startup_fh, $startup_lock, O_CREAT|O_EXCL|O_WRONLY, 0644)) { - # Startup lock exists - check if it's stale - warn "DEBUG start_collector: Startup lock exists, checking if stale"; - - if (open(my $check_fh, '<', $startup_lock)) { - my $lock_pid = <$check_fh>; - chomp $lock_pid if defined $lock_pid; - close($check_fh); - - if (defined $lock_pid && $lock_pid =~ /^\d+$/) { - warn "DEBUG start_collector: Startup lock held by PID $lock_pid"; +sub start_graphics_collectors { - if (is_process_alive($lock_pid)) { - warn "DEBUG start_collector: Lock holder PID $lock_pid is still alive, waiting"; - } else { - # Lock holder is dead, remove stale lock - warn "DEBUG start_collector: Lock holder PID $lock_pid is dead, removing stale startup lock"; - unlink($startup_lock); - - # Try to acquire lock again - unless (sysopen($startup_fh, $startup_lock, O_CREAT|O_EXCL|O_WRONLY, 0644)) { - warn "DEBUG start_collector: Failed to acquire startup lock on retry: $!"; - return; - } - warn "DEBUG start_collector: Acquired startup lock after removing stale lock"; - } - } else { - # Invalid PID in lock file, remove it - warn "DEBUG start_collector: Invalid PID in startup lock, removing"; - unlink($startup_lock); - - # Try to acquire lock again - unless (sysopen($startup_fh, $startup_lock, O_CREAT|O_EXCL|O_WRONLY, 0644)) { - warn "DEBUG start_collector: Failed to acquire startup lock on retry: $!"; - return; - } - warn "DEBUG start_collector: Acquired startup lock after removing invalid lock"; - } - } else { - warn "DEBUG start_collector: Could not read startup lock file: $!"; - return; - } - } else { - warn "DEBUG start_collector: Acquired startup lock on first try"; + if ($intel_gpu_enabled == 0 && $amd_gpu_enabled == 0 && $nvidia_gpu_enabled == 0) { + debug(__LINE__, "No GPU types enabled, skipping collector startup"); + return; } - - # We have the startup lock - print $startup_fh "$$\n"; - close($startup_fh); - warn "DEBUG start_collector: Wrote PID to startup lock"; - + # NOW check if collectors are already running (while holding startup lock) + my %existing_collectors; if (-f $lock_file) { - warn "DEBUG start_collector: Lock file exists: $lock_file"; + debug(__LINE__, "Lock file exists: $lock_file"); if (open(my $lock_fh, '<', $lock_file)) { - warn "DEBUG start_collector: Opened lock file for reading"; - my @pids; + debug(__LINE__, "Opened lock file for reading"); while (my $line = <$lock_fh>) { chomp $line; - push @pids, $line if $line =~ /^\d+$/; - } - close($lock_fh); - - warn "DEBUG start_collector: Found " . scalar(@pids) . " PIDs in lock file: " . join(", ", @pids); - - # Check if ANY collector is alive - my $collector_running = 0; - foreach my $pid (@pids) { - warn "DEBUG start_collector: Checking PID $pid"; - - if (is_process_alive($pid)) { - warn "DEBUG start_collector: Collector already running with PID $pid"; - $collector_running = 1; - last; + if ($line =~ /^(\d+)\s+(\S+)/) { + $existing_collectors{$2} = $1; + } elsif ($line =~ /^(\d+)$/) { + # Backward compatibility: only PID, no card + $existing_collectors{"unknown"} = $1; } } - - warn "DEBUG start_collector: Finished checking PIDs, collector_running = $collector_running"; - - if ($collector_running) { - warn "DEBUG start_collector: Releasing startup lock and returning - collector already running"; - unlink($startup_lock); - return; - } - - # All PIDs are dead, clean up stale lock - warn "DEBUG start_collector: All PIDs dead, removing stale lock file"; - unlink($lock_file); + close($lock_fh); } else { - warn "DEBUG start_collector: Failed to open lock file: $!"; + debug(__LINE__, "Failed to open lock file: $!"); } } else { - warn "DEBUG start_collector: Lock file does not exist"; + debug(__LINE__, "Lock file does not exist. Clean start"); } - # If we reach here, no collectors are running - start new ones - # Fork collector processes - my @child_pids; + # Generalized device collector management for future AMD/NVIDIA support + my @all_devices; + my @all_types; + my @all_collectors; + # Intel if ($intel_gpu_enabled) { - warn "DEBUG start_collector: Checking for intel_gpu_top"; + debug(__LINE__, "Checking for intel_gpu_top"); unless (-x '/usr/bin/intel_gpu_top') { - warn "DEBUG start_collector: intel_gpu_top not executable"; + debug(__LINE__, "intel_gpu_top not executable"); unlink($startup_lock); return; } - warn "DEBUG start_collector: intel_gpu_top is executable"; - - warn "DEBUG start_collector: Getting Intel GPU devices"; + debug(__LINE__, "intel_gpu_top is executable"); + debug(__LINE__, "Getting Intel GPU devices"); my @intel_devices = get_intel_gpu_devices(); - warn "DEBUG start_collector: Got " . scalar(@intel_devices) . " Intel devices"; - unless (@intel_devices) { - warn "DEBUG start_collector: No Intel GPU devices found"; + debug(__LINE__, "No Intel GPU devices found"); unlink($startup_lock); return; } - warn "DEBUG start_collector: Found " . scalar(@intel_devices) . " Intel GPU device(s)"; - - + debug(__LINE__, "Found " . scalar(@intel_devices) . " Intel GPU device(s)"); foreach my $device (@intel_devices) { - warn "DEBUG start_collector: About to fork collector for Intel $device->{card}"; - my $pid = fork(); - - unless (defined $pid) { - warn "DEBUG start_collector: fork failed: $!"; - unlink($startup_lock); - die "fork failed: $!"; - } - - if ($pid == 0) { - # Child process - warn "DEBUG start_collector: In child process for $device->{card}"; - collector_for_intel_device($device); - # collector_for_intel_device calls exit(0) - } else { - warn "DEBUG start_collector: Forked child PID $pid for $device->{card}"; - push @child_pids, $pid; - } + push @all_devices, $device; + push @all_types, 'intel'; } + } + # AMD (future) + if ($amd_gpu_enabled) { + my @amd_devices = get_amd_gpu_devices(); + debug(__LINE__, "Got " . scalar(@amd_devices) . " AMD devices"); + foreach my $device (@amd_devices) { + push @all_devices, $device; + push @all_types, 'amd'; + } + } + # NVIDIA (future) + if ($nvidia_gpu_enabled) { + my @nvidia_devices = get_nvidia_gpu_devices(); + debug(__LINE__, "Got " . scalar(@nvidia_devices) . " NVIDIA devices"); + foreach my $device (@nvidia_devices) { + push @all_devices, $device; + push @all_types, 'nvidia'; + } + } - warn "DEBUG start_collector: Forked " . scalar(@child_pids) . " children: " . join(", ", @child_pids); - } - - # Write child PIDs to lock file + my @child_pids; + my @child_devices; + my @child_types; + for (my $i = 0; $i < @all_devices; $i++) { + my $device = $all_devices[$i]; + my $type = $all_types[$i]; + my $card = $device->{card}; + my $existing_pid = $existing_collectors{$card}; + if ($existing_pid && is_process_alive($existing_pid)) { + debug(__LINE__, "Collector for $type $card already running with PID $existing_pid"); + push @child_pids, $existing_pid; + push @child_devices, $device; + push @child_types, $type; + next; + } + debug(__LINE__, "About to fork collector for $type $card"); + my $pid = fork(); + unless (defined $pid) { + debug(__LINE__, "fork failed: $!"); + unlink($startup_lock); + die "fork failed: $!"; + } + if ($pid == 0) { + # Child process + debug(__LINE__, "In child process for $type $card"); + if ($type eq 'intel') { + collector_for_intel_device($device); + } elsif ($type eq 'amd') { + collector_for_amd_device($device); + } elsif ($type eq 'nvidia') { + collector_for_nvidia_device($device); + } else { + debug(__LINE__, "Unknown GPU type $type for $card"); + exit(1); + } + # Should not reach here + exit(0); + } else { + debug(__LINE__, "Forked child PID $pid for $type $card"); + push @child_pids, $pid; + push @child_devices, $device; + push @child_types, $type; + } + } + debug(__LINE__, "Active children: " . join(", ", @child_pids)); + + # Write child PIDs and device cards/types to lock file if (open(my $lock_fh, '>', $lock_file)) { - warn "DEBUG start_collector: Opened lock file for writing"; - foreach my $pid (@child_pids) { - print $lock_fh "$pid\n"; + debug(__LINE__, "Opened lock file for writing"); + for (my $i = 0; $i < @child_pids; $i++) { + my $pid = $child_pids[$i]; + my $device = $child_devices[$i]; + my $type = $child_types[$i]; + my $card = $device->{card} // ''; + print $lock_fh "$pid $card $type\n"; } close($lock_fh); - warn "DEBUG start_collector: Wrote " . scalar(@child_pids) . " collector PID(s) to lock file"; + debug(__LINE__, "Wrote " . scalar(@child_pids) . " collector PID(s) to lock file"); } else { - warn "DEBUG start_collector: Failed to open lock file for writing: $!"; + debug(__LINE__, "Failed to open lock file for writing: $!"); foreach my $pid (@child_pids) { - warn "DEBUG start_collector: Killing child $pid due to lock file write failure"; + debug(__LINE__, "Killing child $pid due to lock file write failure"); kill 'TERM', $pid; } unlink($startup_lock); @@ -458,44 +453,36 @@ sub start_collector { # Wait briefly to ensure collector is actually running sleep 0.1; - - # Verify at least one child is still alive + + # Verify at least one child is still alive (by PID only) my $any_alive = 0; - foreach my $pid (@child_pids) { - if (kill(0, $pid)) { + for (my $i = 0; $i < @child_pids; $i++) { + my $pid = $child_pids[$i]; + my $device = $child_devices[$i]; + my $card = $device->{card} // ''; + my $alive = kill(0, $pid); + if ($alive) { $any_alive = 1; - warn "DEBUG start_collector: Verified child PID $pid is alive"; + debug(__LINE__, "Verified child PID $pid for $card is alive"); } else { - warn "DEBUG start_collector: WARNING - Child PID $pid died immediately!"; + debug(__LINE__, "WARNING - Child PID $pid for $card died immediately!"); } } - unless ($any_alive) { - warn "DEBUG start_collector: ERROR - No children alive after fork!"; + debug(__LINE__, "ERROR - No children alive after fork!"); } - - # Start pve_mod_monitor process - pve_mod_monitor(); - - - # Remove startup lock LAST - unlink($startup_lock); - warn "DEBUG start_collector: Released startup lock"; - - warn "DEBUG start_collector: Collectors started successfully, returning"; } -sub get_stats { - warn "DEBUG: get_stats() called"; +sub get_graphic_stats { + debug(__LINE__, "get_graphic_stats called"); - start_collector(); - - my $stats_dir = '/var/run/pve-gpu'; + # Start PVE Mod worker, if not already running + pve_mod_worker(); # Find all device-specific stat files my $dh; unless (opendir($dh, $stats_dir)) { - warn "DEBUG: Failed to open stats directory: $stats_dir: $!"; + debug(__LINE__, "Failed to open stats directory: $stats_dir: $!"); return $last_snapshot; } @@ -503,11 +490,11 @@ sub get_stats { closedir($dh); unless (@stat_files) { - warn "DEBUG: No device stat files found in $stats_dir"; + debug(__LINE__, "No device stat files found in $stats_dir"); return $last_snapshot; } - warn "DEBUG: Found " . scalar(@stat_files) . " device stat file(s): " . join(', ', @stat_files); + debug(__LINE__, "Found " . scalar(@stat_files) . " device stat file(s): " . join(', ', @stat_files)); # Check if any files have been modified my $newest_mtime = 0; @@ -522,11 +509,11 @@ sub get_stats { } if ($newest_mtime == $last_mtime) { - warn "DEBUG: No device files modified, returning cached snapshot"; + debug(__LINE__, "No device files modified, returning cached snapshot"); return $last_snapshot; } - warn "DEBUG: Device files modified ($last_mtime -> $newest_mtime), reading and merging files"; + debug(__LINE__, "Device files modified ($last_mtime -> $newest_mtime), reading and merging files"); # Merge all device files my $merged = { @@ -538,12 +525,12 @@ sub get_stats { foreach my $file (@stat_files) { my $filepath = "$stats_dir/$file"; - warn "DEBUG: Reading device file: $filepath"; + debug(__LINE__, "Reading device file: $filepath"); eval { my $fh; unless (open($fh, '<', $filepath)) { - warn "DEBUG: Failed to open $filepath: $!"; + debug(__LINE__, "Failed to open $filepath: $!"); return; } @@ -551,71 +538,145 @@ sub get_stats { my $json = <$fh>; close($fh); - warn "DEBUG: Read $file, JSON length: " . length($json) . " bytes"; + debug(__LINE__, "Read $file, JSON length: " . length($json) . " bytes"); my $device_data = decode_json($json); # Merge this device's data into the main structure foreach my $node_name (keys %$device_data) { $merged->{Graphics}->{Intel}->{$node_name} = $device_data->{$node_name}; - warn "DEBUG: Merged node '$node_name' from $file"; + debug(__LINE__, "Merged node '$node_name' from $file"); } }; if ($@) { - warn "DEBUG: Failed to read/parse $filepath: $@"; + debug(__LINE__, "Failed to read/parse $filepath: $@"); } } # Update cache $last_snapshot = $merged; $last_mtime = $newest_mtime; - $last_get_stats_time = time(); + $last_get_graphic_stats_time = time(); - warn "DEBUG: Successfully merged " . scalar(keys %{$merged->{Graphics}->{Intel}}) . " device node(s)"; + debug(__LINE__, "Successfully merged " . scalar(keys %{$merged->{Graphics}->{Intel}}) . " device node(s)"); return $last_snapshot; } -sub pve_mod_monitor { - return if $monitor_running; +sub pve_mod_starter { + # Try to acquire startup lock FIRST (prevents race conditions) + my $startup_fh; - $monitor_pid = fork(); + debug(__LINE__, "Trying to acquire startup lock: $startup_lock"); + + unless (sysopen($startup_fh, $startup_lock, O_CREAT|O_EXCL|O_WRONLY, 0644)) { + # Startup lock exists - check if it's stale + debug(__LINE__, "Startup lock exists, checking if stale"); + + if (open(my $check_fh, '<', $startup_lock)) { + my $lock_pid = <$check_fh>; + chomp $lock_pid if defined $lock_pid; + close($check_fh); + + if (defined $lock_pid && $lock_pid =~ /^\d+$/) { + debug(__LINE__, "Startup lock held by PID $lock_pid"); - if (!defined $monitor_pid) { - warn "ERROR: Failed to fork monitor process: $!"; - return; - } - - if ($monitor_pid == 0) { - # Child process - $0 = "pve_mod_monitor"; # Set process name for easier identification - - while (1) { - if(time() - $last_get_stats_time > $COLLECTOR_TIMEOUT) { - warn "DEBUG pve_mod_monitor: No get_stats call in the last $COLLECTOR_TIMEOUT seconds, stopping collectors"; - stop_collectors(); - exit(0); + if (is_process_alive($lock_pid)) { + debug(__LINE__, "Lock holder PID $lock_pid is still alive, waiting"); + } else { + # Lock holder is dead, remove stale lock + debug(__LINE__, "Lock holder PID $lock_pid is dead, removing stale startup lock"); + unlink($startup_lock); + + # Try to acquire lock again + unless (sysopen($startup_fh, $startup_lock, O_CREAT|O_EXCL|O_WRONLY, 0644)) { + debug(__LINE__, "Failed to acquire startup lock on retry: $!"); + return; + } + debug(__LINE__, "Acquired startup lock after removing stale lock"); + } + } else { + # Invalid PID in lock file, remove it + debug(__LINE__, "Invalid PID in startup lock, removing"); + unlink($startup_lock); + + # Try to acquire lock again + unless (sysopen($startup_fh, $startup_lock, O_CREAT|O_EXCL|O_WRONLY, 0644)) { + debug(__LINE__, "Failed to acquire startup lock on retry: $!"); + return; + } + debug(__LINE__, "Acquired startup lock after removing invalid lock"); } - sleep(1); - warn "DEBUG pve_mod_monitor: pve_mod_monitor still running"; + } else { + debug(__LINE__, "Could not read startup lock file: $!"); + return; } - - exit(0); # This shouldn't be reached, but just in case + } else { + debug(__LINE__, "Acquired startup lock on first try"); } - # Parent process - $monitor_running = 1; - warn "DEBUG start_monitor: Monitor process started with PID $monitor_pid"; + # We have the startup lock + print $startup_fh "$$\n"; + close($startup_fh); + debug(__LINE__, "Wrote PID, $$, to startup lock"); +} + +sub pve_mod_worker { + # Give the pid a unique name for easier identification + $0 = "pve_mod_worker"; + + # Ensure directory exists + my $run_dir = '/var/run/pve-gpu'; + unless (-d $run_dir) { + debug(__LINE__, "Creating directory $run_dir"); + mkdir($run_dir, 0755) or debug(__LINE__, "Failed to create $run_dir: $!"); + } + + # Acquire startup lock and start application + pve_mod_starter(); + + # Start graphics collectors + start_graphics_collectors(); + + # Start sensor collector + # TBD + + # Start UPS collector + # TBD + + # Start gui activity monitor process + pve_mod_keep_alive(); + + # Remove startup lock LAST + unlink($startup_lock); + debug(__LINE__, "Released startup lock"); + + debug(__LINE__, "Collectors started successfully, returning"); +} + +sub pve_mod_keep_alive { + debug(__LINE__, "Monitor process started with PID $$"); + while (1) { + if(time() - $last_get_graphic_stats_time > $COLLECTOR_TIMEOUT) { + debug(__LINE__, "No get_graphic_stats call in the last $COLLECTOR_TIMEOUT seconds, stopping collectors"); + stop_collectors(); + exit(0); + } + sleep(1); + debug(__LINE__, "pve_mod_keep_alive still running"); + } } sub cleanup { unless ($is_collector_parent) { - warn "DEBUG cleanup: This process did not start collectors, skipping cleanup"; + debug(__LINE__, "This process did not start collectors, skipping cleanup"); return; } - warn "DEBUG cleanup: Starting cleanup (this should rarely happen)"; + debug(__LINE__, "Starting cleanup (this should rarely happen)"); + # todo add remove of stat files + # DON'T cleanup automatically - collectors should keep running # across worker process lifecycles # Only cleanup if explicitly called @@ -626,7 +687,7 @@ sub cleanup { # Instead, add a manual cleanup function that can be called explicitly sub stop_collectors { - warn "DEBUG stop_collectors: Stopping all collectors"; + debug(__LINE__, "Stopping all collectors"); # Read current PIDs from lock file my @pids; @@ -640,7 +701,7 @@ sub stop_collectors { if (@pids) { # Send SIGTERM to all collectors - warn "DEBUG stop_collectors: Sending SIGTERM to " . scalar(@pids) . " process(es)"; + debug(__LINE__, "Sending SIGTERM to " . scalar(@pids) . " process(es)"); foreach my $pid (@pids) { kill('TERM', $pid) if kill(0, $pid); } @@ -663,7 +724,7 @@ sub stop_collectors { # Force kill any survivors foreach my $pid (@pids) { if (kill(0, $pid)) { - warn "DEBUG stop_collectors: Force killing process $pid"; + debug(__LINE__, "Force killing process $pid"); kill('KILL', $pid); } } @@ -671,7 +732,7 @@ sub stop_collectors { unlink $state_file if -f $state_file; unlink $lock_file if -f $lock_file; - warn "DEBUG stop_collectors: Cleanup complete"; + debug(__LINE__, "Cleanup complete"); } END { cleanup() }