From ea76402433b7cb61efce3acb2c929f6cae0050fa Mon Sep 17 00:00:00 2001 From: Meliox Date: Sat, 3 Jan 2026 21:38:46 +0100 Subject: [PATCH] next snapshot --- GPUcollecter.pm | 499 ++++++++++++++++++++++++++++++------------------ 1 file changed, 308 insertions(+), 191 deletions(-) diff --git a/GPUcollecter.pm b/GPUcollecter.pm index a8fc1af..4dcc12a 100644 --- a/GPUcollecter.pm +++ b/GPUcollecter.pm @@ -8,18 +8,18 @@ use Fcntl qw(:flock); use Time::HiRes qw(time); use Fcntl qw(:flock O_CREAT O_EXCL O_WRONLY); -# Debug configuration - set to 0 to disable all debug output -my $DEBUG_ENABLED = 1; +# debug configuration - set to 0 to disable all _debug output +my $debug_ENABLED = 1; -# Debug function showing line number and call chain -# Usage: debug(__LINE__, "message") -sub debug { - return unless $DEBUG_ENABLED; +# debug function showing line number and call chain +# Usage: _debug(__LINE__, "message") +sub _debug { + return unless $debug_ENABLED; my ($line, $message) = @_; # Get function call chain - my @caller1 = caller(1); # who called debug() + my @caller1 = caller(1); # who called _debug() my @caller2 = caller(2); # parent of caller my $sub1 = $caller1[3] || 'main'; @@ -39,12 +39,13 @@ sub debug { my $stats_dir = '/var/run/pve-gpu'; my $state_file = '/var/run/pve-gpu/stats.json'; my $lock_file = '/var/run/pve-gpu/pve-gpu-collector.lock'; +my $monitor_lock = '/var/run/pve-gpu/pve-gpu-monitor.lock'; my $startup_lock = $lock_file . ".startup"; my $last_snapshot = {}; my $last_mtime = 0; my $is_collector_parent = 0; # Flag to track if this process started collectors my $last_get_graphic_stats_time = 0; # Track when get_graphic_stats was last called -my $COLLECTOR_TIMEOUT = 0; # Stop collectors x seconds after last get_graphic_stats call +my $COLLECTOR_TIMEOUT = 10; # Stop collectors x seconds after last get_graphic_stats call my $intel_gpu_enabled = 1; # Set to 0 to disable Intel GPU support my $amd_gpu_enabled = 0; # Set to 1 to enable AMD GPU support (not yet implemented) @@ -57,7 +58,7 @@ my $monitor_running = 0; # ============================================================================ # Parse Intel GPU line output format -sub parse_intel_gpu_line { +sub _parse_intel_gpu_line { my ($line) = @_; # Expected format (with aligned columns): @@ -128,7 +129,7 @@ sub parse_intel_gpu_line { } # Get list of Intel GPU devices -sub get_intel_gpu_devices { +sub _get_intel_gpu_devices { my @devices = (); return @devices unless -x '/usr/bin/intel_gpu_top'; @@ -148,18 +149,18 @@ sub get_intel_gpu_devices { path => $path, drm_path => "/dev/dri/$card" }; - debug(__LINE__, "Found GPU device: $card -> $name ($path)"); + _debug(__LINE__, "Found GPU device: $card -> $name ($path)"); } } close $fh; } else { - debug(__LINE__, "Failed to run intel_gpu_top -L: $!"); + _debug(__LINE__, "Failed to run intel_gpu_top -L: $!"); } return @devices; } -sub collector_for_intel_device { +sub _collector_for_intel_device { my ($device) = @_; $0 = "pve-mod-gpu-intel-collector: $device->{card}"; @@ -169,31 +170,31 @@ sub collector_for_intel_device { # Each device writes to its own file my $device_state_file = "/var/run/pve-gpu/stats-$device->{card}.json"; - debug(__LINE__, "Collector started for device: $drm_dev, writing to $device_state_file"); + _debug(__LINE__, "Collector started for device: $drm_dev, writing to $device_state_file"); # Set up signal handlers for graceful shutdown my $shutdown = 0; $SIG{TERM} = sub { - debug(__LINE__, "Collector for $device->{card} received SIGTERM"); + _debug(__LINE__, "Collector for $device->{card} received SIGTERM"); $shutdown = 1; kill 'TERM', $intel_gpu_top_pid if defined $intel_gpu_top_pid && $intel_gpu_top_pid > 0; }; $SIG{INT} = sub { - debug(__LINE__, "Collector for $device->{card} received SIGINT"); + _debug(__LINE__, "Collector for $device->{card} received SIGINT"); $shutdown = 1; kill 'TERM', $intel_gpu_top_pid if defined $intel_gpu_top_pid && $intel_gpu_top_pid > 0; }; # Run intel_gpu_top once and keep reading from it - debug(__LINE__, "About to open pipe to intel_gpu_top"); + _debug(__LINE__, "About to open pipe to intel_gpu_top"); $intel_gpu_top_pid = open(my $fh, '-|', "intel_gpu_top -d $drm_dev -s 1000 -l 2>&1"); unless (defined $intel_gpu_top_pid && $intel_gpu_top_pid > 0) { - debug(__LINE__, "Failed to run intel_gpu_top for $drm_dev: $!"); + _debug(__LINE__, "Failed to run intel_gpu_top for $drm_dev: $!"); exit 1; } - debug(__LINE__, "Pipe opened successfully, PID=$intel_gpu_top_pid"); + _debug(__LINE__, "Pipe opened successfully, PID=$intel_gpu_top_pid"); my $line_count = 0; my $node_name = "node0"; # You may want to generate this based on device index @@ -209,7 +210,7 @@ sub collector_for_intel_device { # Check if this is a data line if ($line =~ /^\s*[\d\s\.]+$/) { - my $stats = parse_intel_gpu_line($line); + my $stats = _parse_intel_gpu_line($line); if ($stats) { # Build device-specific structure (just the node, not the full Graphics/Intel hierarchy) @@ -227,17 +228,17 @@ sub collector_for_intel_device { open my $ofh, '>', $device_state_file or die "Failed to open $device_state_file: $!"; print $ofh JSON->new->pretty->encode($device_data); close $ofh; - debug(__LINE__, "Wrote stats to $device_state_file (line #$line_count)"); + _debug(__LINE__, "Wrote stats to $device_state_file (line #$line_count)"); }; if ($@) { - debug(__LINE__, "Error writing stats: $@"); + _debug(__LINE__, "Error writing stats: $@"); } } } } close $fh; - debug(__LINE__, "Collector for $device->{card} shutting down"); + _debug(__LINE__, "Collector for $device->{card} shutting down"); exit 0; } @@ -245,25 +246,25 @@ sub collector_for_intel_device { # AMD GPU Support (Placeholder) # ============================================================================ -sub get_amd_gpu_devices { +sub _get_amd_gpu_devices { # TODO: Implement AMD GPU detection # Use rocminfo or similar tools to detect AMD GPUs - debug(__LINE__, "AMD GPU support not yet implemented"); + _debug(__LINE__, "AMD GPU support not yet implemented"); return (); } -sub parse_amd_gpu_line { +sub _parse_amd_gpu_line { my ($line) = @_; # TODO: Implement AMD GPU line parsing # Parse rocm-smi or similar output - debug(__LINE__, "AMD GPU line parsing not yet implemented"); + _debug(__LINE__, "AMD GPU line parsing not yet implemented"); return undef; } -sub collector_for_amd_device { +sub _collector_for_amd_device { my ($device) = @_; # TODO: Implement AMD GPU collector - debug(__LINE__, "AMD GPU collector not yet implemented"); + _debug(__LINE__, "AMD GPU collector not yet implemented"); exit 0; } @@ -274,7 +275,7 @@ sub collector_for_amd_device { sub get_nvidia_gpu_devices { # TODO: Implement NVIDIA GPU detection # Use nvidia-smi to detect NVIDIA GPUs - debug(__LINE__, "NVIDIA GPU support not yet implemented"); + _debug(__LINE__, "NVIDIA GPU support not yet implemented"); return (); } @@ -282,14 +283,14 @@ sub parse_nvidia_gpu_line { my ($line) = @_; # TODO: Implement NVIDIA GPU line parsing # Parse nvidia-smi output - debug(__LINE__, "NVIDIA GPU line parsing not yet implemented"); + _debug(__LINE__, "NVIDIA GPU line parsing not yet implemented"); return undef; } sub collector_for_nvidia_device { my ($device) = @_; # TODO: Implement NVIDIA GPU collector - debug(__LINE__, "NVIDIA GPU collector not yet implemented"); + _debug(__LINE__, "NVIDIA GPU collector not yet implemented"); exit 0; } @@ -298,28 +299,128 @@ sub collector_for_nvidia_device { # ============================================================================ # Check if a process is alive -sub is_process_alive { +sub _is_process_alive { my ($pid) = @_; return -d "/proc/$pid"; } +# ============================================================================ +# API calls +# ============================================================================ + +sub get_graphic_stats { + _debug(__LINE__, "get_graphic_stats called"); + + # Start PVE Mod worker, if not already running + _pve_mod_worker(); + + # Find all device-specific stat files + my $dh; + unless (opendir($dh, $stats_dir)) { + _debug(__LINE__, "Failed to open stats directory: $stats_dir: $!"); + return $last_snapshot; + } + + my @stat_files = grep { /^stats-card\d+\.json$/ } readdir($dh); + closedir($dh); + + unless (@stat_files) { + _debug(__LINE__, "No device stat files found in $stats_dir"); + return $last_snapshot; + } + + _debug(__LINE__, "Found " . scalar(@stat_files) . " device stat file(s): " . join(', ', @stat_files)); + + # Check if any files have been modified + my $newest_mtime = 0; + my $files_changed = 0; + + foreach my $file (@stat_files) { + my $filepath = "$stats_dir/$file"; + my @stat = stat($filepath); + if (@stat && $stat[9] > $newest_mtime) { + $newest_mtime = $stat[9]; + } + } + + if ($newest_mtime == $last_mtime) { + _debug(__LINE__, "No device files modified, returning cached snapshot"); + return $last_snapshot; + } + + _debug(__LINE__, "Device files modified ($last_mtime -> $newest_mtime), reading and merging files"); + + # Merge all device files + my $merged = { + Graphics => { + Intel => {} + } + }; + + foreach my $file (@stat_files) { + my $filepath = "$stats_dir/$file"; + + _debug(__LINE__, "Reading device file: $filepath"); + + eval { + my $fh; + unless (open($fh, '<', $filepath)) { + _debug(__LINE__, "Failed to open $filepath: $!"); + return; + } + + local $/; + my $json = <$fh>; + close($fh); + + _debug(__LINE__, "Read $file, JSON length: " . length($json) . " bytes"); + + my $device_data = decode_json($json); + + # Merge this device's data into the main structure + foreach my $node_name (keys %$device_data) { + $merged->{Graphics}->{Intel}->{$node_name} = $device_data->{$node_name}; + _debug(__LINE__, "Merged node '$node_name' from $file"); + } + }; + if ($@) { + _debug(__LINE__, "Failed to read/parse $filepath: $@"); + } + } + + # Update cache + $last_snapshot = $merged; + $last_mtime = $newest_mtime; + $last_get_graphic_stats_time = time(); + + _debug(__LINE__, "Successfully merged " . scalar(keys %{$merged->{Graphics}->{Intel}}) . " device node(s)"); + + # Notify monitor of activity + _notify_monitor(); + + return $last_snapshot; +} + # ============================================================================ # Main Collector # ============================================================================ -sub start_graphics_collectors { +sub _start_graphics_collectors { if ($intel_gpu_enabled == 0 && $amd_gpu_enabled == 0 && $nvidia_gpu_enabled == 0) { - debug(__LINE__, "No GPU types enabled, skipping collector startup"); + _debug(__LINE__, "No GPU types enabled, skipping collector startup"); return; } + else { + _debug(__LINE__, "Starting graphics collectors"); + } # NOW check if collectors are already running (while holding startup lock) my %existing_collectors; if (-f $lock_file) { - debug(__LINE__, "Lock file exists: $lock_file"); + _debug(__LINE__, "Lock file exists: $lock_file"); if (open(my $lock_fh, '<', $lock_file)) { - debug(__LINE__, "Opened lock file for reading"); + _debug(__LINE__, "Opened lock file for reading"); while (my $line = <$lock_fh>) { chomp $line; if ($line =~ /^(\d+)\s+(\S+)/) { @@ -331,10 +432,10 @@ sub start_graphics_collectors { } close($lock_fh); } else { - debug(__LINE__, "Failed to open lock file: $!"); + _debug(__LINE__, "Failed to open lock file: $!"); } } else { - debug(__LINE__, "Lock file does not exist. Clean start"); + _debug(__LINE__, "Lock file does not exist. Clean start"); } # Generalized device collector management for future AMD/NVIDIA support @@ -344,21 +445,22 @@ sub start_graphics_collectors { # Intel if ($intel_gpu_enabled) { - debug(__LINE__, "Checking for intel_gpu_top"); + _debug(__LINE__, "Intel GPU support enabled"); + _debug(__LINE__, "Checking for intel_gpu_top"); unless (-x '/usr/bin/intel_gpu_top') { - debug(__LINE__, "intel_gpu_top not executable"); + _debug(__LINE__, "intel_gpu_top not executable"); unlink($startup_lock); return; } - debug(__LINE__, "intel_gpu_top is executable"); - debug(__LINE__, "Getting Intel GPU devices"); - my @intel_devices = get_intel_gpu_devices(); + _debug(__LINE__, "intel_gpu_top is executable"); + _debug(__LINE__, "Getting Intel GPU devices"); + my @intel_devices = _get_intel_gpu_devices(); unless (@intel_devices) { - debug(__LINE__, "No Intel GPU devices found"); + _debug(__LINE__, "No Intel GPU devices found"); unlink($startup_lock); return; } - debug(__LINE__, "Found " . scalar(@intel_devices) . " Intel GPU device(s)"); + _debug(__LINE__, "Found " . scalar(@intel_devices) . " Intel GPU device(s)"); foreach my $device (@intel_devices) { push @all_devices, $device; push @all_types, 'intel'; @@ -366,8 +468,8 @@ sub start_graphics_collectors { } # AMD (future) if ($amd_gpu_enabled) { - my @amd_devices = get_amd_gpu_devices(); - debug(__LINE__, "Got " . scalar(@amd_devices) . " AMD devices"); + my @amd_devices = _get_amd_gpu_devices(); + _debug(__LINE__, "Got " . scalar(@amd_devices) . " AMD devices"); foreach my $device (@amd_devices) { push @all_devices, $device; push @all_types, 'amd'; @@ -376,7 +478,7 @@ sub start_graphics_collectors { # NVIDIA (future) if ($nvidia_gpu_enabled) { my @nvidia_devices = get_nvidia_gpu_devices(); - debug(__LINE__, "Got " . scalar(@nvidia_devices) . " NVIDIA devices"); + _debug(__LINE__, "Got " . scalar(@nvidia_devices) . " NVIDIA devices"); foreach my $device (@nvidia_devices) { push @all_devices, $device; push @all_types, 'nvidia'; @@ -391,47 +493,47 @@ sub start_graphics_collectors { my $type = $all_types[$i]; my $card = $device->{card}; my $existing_pid = $existing_collectors{$card}; - if ($existing_pid && is_process_alive($existing_pid)) { - debug(__LINE__, "Collector for $type $card already running with PID $existing_pid"); + if ($existing_pid && _is_process_alive($existing_pid)) { + _debug(__LINE__, "Collector for $type $card already running with PID $existing_pid"); push @child_pids, $existing_pid; push @child_devices, $device; push @child_types, $type; next; } - debug(__LINE__, "About to fork collector for $type $card"); + _debug(__LINE__, "About to fork collector for $type $card"); my $pid = fork(); unless (defined $pid) { - debug(__LINE__, "fork failed: $!"); + _debug(__LINE__, "fork failed: $!"); unlink($startup_lock); die "fork failed: $!"; } if ($pid == 0) { # Child process - debug(__LINE__, "In child process for $type $card"); + _debug(__LINE__, "In child process for $type $card"); if ($type eq 'intel') { - collector_for_intel_device($device); + _collector_for_intel_device($device); } elsif ($type eq 'amd') { - collector_for_amd_device($device); + _collector_for_amd_device($device); } elsif ($type eq 'nvidia') { collector_for_nvidia_device($device); } else { - debug(__LINE__, "Unknown GPU type $type for $card"); + _debug(__LINE__, "Unknown GPU type $type for $card"); exit(1); } # Should not reach here exit(0); } else { - debug(__LINE__, "Forked child PID $pid for $type $card"); + _debug(__LINE__, "Forked child PID $pid for $type $card"); push @child_pids, $pid; push @child_devices, $device; push @child_types, $type; } } - debug(__LINE__, "Active children: " . join(", ", @child_pids)); + _debug(__LINE__, "Active children: " . join(", ", @child_pids)); # Write child PIDs and device cards/types to lock file if (open(my $lock_fh, '>', $lock_file)) { - debug(__LINE__, "Opened lock file for writing"); + _debug(__LINE__, "Opened lock file for writing"); for (my $i = 0; $i < @child_pids; $i++) { my $pid = $child_pids[$i]; my $device = $child_devices[$i]; @@ -440,11 +542,11 @@ sub start_graphics_collectors { print $lock_fh "$pid $card $type\n"; } close($lock_fh); - debug(__LINE__, "Wrote " . scalar(@child_pids) . " collector PID(s) to lock file"); + _debug(__LINE__, "Wrote " . scalar(@child_pids) . " collector PID(s) to lock file"); } else { - debug(__LINE__, "Failed to open lock file for writing: $!"); + _debug(__LINE__, "Failed to open lock file for writing: $!"); foreach my $pid (@child_pids) { - debug(__LINE__, "Killing child $pid due to lock file write failure"); + _debug(__LINE__, "Killing child $pid due to lock file write failure"); kill 'TERM', $pid; } unlink($startup_lock); @@ -463,115 +565,27 @@ sub start_graphics_collectors { my $alive = kill(0, $pid); if ($alive) { $any_alive = 1; - debug(__LINE__, "Verified child PID $pid for $card is alive"); + _debug(__LINE__, "Verified child PID $pid for $card is alive"); } else { - debug(__LINE__, "WARNING - Child PID $pid for $card died immediately!"); + _debug(__LINE__, "WARNING - Child PID $pid for $card died immediately!"); } } unless ($any_alive) { - debug(__LINE__, "ERROR - No children alive after fork!"); + _debug(__LINE__, "ERROR - No children alive after fork!"); } + + _debug(__LINE__, "Graphics collectors started"); } -sub get_graphic_stats { - debug(__LINE__, "get_graphic_stats called"); - - # Start PVE Mod worker, if not already running - pve_mod_worker(); - - # Find all device-specific stat files - my $dh; - unless (opendir($dh, $stats_dir)) { - debug(__LINE__, "Failed to open stats directory: $stats_dir: $!"); - return $last_snapshot; - } - - my @stat_files = grep { /^stats-card\d+\.json$/ } readdir($dh); - closedir($dh); - - unless (@stat_files) { - debug(__LINE__, "No device stat files found in $stats_dir"); - return $last_snapshot; - } - - debug(__LINE__, "Found " . scalar(@stat_files) . " device stat file(s): " . join(', ', @stat_files)); - - # Check if any files have been modified - my $newest_mtime = 0; - my $files_changed = 0; - - foreach my $file (@stat_files) { - my $filepath = "$stats_dir/$file"; - my @stat = stat($filepath); - if (@stat && $stat[9] > $newest_mtime) { - $newest_mtime = $stat[9]; - } - } - - if ($newest_mtime == $last_mtime) { - debug(__LINE__, "No device files modified, returning cached snapshot"); - return $last_snapshot; - } - - debug(__LINE__, "Device files modified ($last_mtime -> $newest_mtime), reading and merging files"); - - # Merge all device files - my $merged = { - Graphics => { - Intel => {} - } - }; - - foreach my $file (@stat_files) { - my $filepath = "$stats_dir/$file"; - - debug(__LINE__, "Reading device file: $filepath"); - - eval { - my $fh; - unless (open($fh, '<', $filepath)) { - debug(__LINE__, "Failed to open $filepath: $!"); - return; - } - - local $/; - my $json = <$fh>; - close($fh); - - debug(__LINE__, "Read $file, JSON length: " . length($json) . " bytes"); - - my $device_data = decode_json($json); - - # Merge this device's data into the main structure - foreach my $node_name (keys %$device_data) { - $merged->{Graphics}->{Intel}->{$node_name} = $device_data->{$node_name}; - debug(__LINE__, "Merged node '$node_name' from $file"); - } - }; - if ($@) { - debug(__LINE__, "Failed to read/parse $filepath: $@"); - } - } - - # Update cache - $last_snapshot = $merged; - $last_mtime = $newest_mtime; - $last_get_graphic_stats_time = time(); - - debug(__LINE__, "Successfully merged " . scalar(keys %{$merged->{Graphics}->{Intel}}) . " device node(s)"); - - return $last_snapshot; -} - -sub pve_mod_starter { +sub _pve_mod_starter { # Try to acquire startup lock FIRST (prevents race conditions) my $startup_fh; - debug(__LINE__, "Trying to acquire startup lock: $startup_lock"); + _debug(__LINE__, "Trying to acquire startup lock: $startup_lock"); unless (sysopen($startup_fh, $startup_lock, O_CREAT|O_EXCL|O_WRONLY, 0644)) { # Startup lock exists - check if it's stale - debug(__LINE__, "Startup lock exists, checking if stale"); + _debug(__LINE__, "Startup lock exists, checking if stale"); if (open(my $check_fh, '<', $startup_lock)) { my $lock_pid = <$check_fh>; @@ -579,64 +593,64 @@ sub pve_mod_starter { close($check_fh); if (defined $lock_pid && $lock_pid =~ /^\d+$/) { - debug(__LINE__, "Startup lock held by PID $lock_pid"); + _debug(__LINE__, "Startup lock held by PID $lock_pid"); - if (is_process_alive($lock_pid)) { - debug(__LINE__, "Lock holder PID $lock_pid is still alive, waiting"); + if (_is_process_alive($lock_pid)) { + _debug(__LINE__, "Lock holder PID $lock_pid is still alive, waiting"); } else { # Lock holder is dead, remove stale lock - debug(__LINE__, "Lock holder PID $lock_pid is dead, removing stale startup lock"); + _debug(__LINE__, "Lock holder PID $lock_pid is dead, removing stale startup lock"); unlink($startup_lock); # Try to acquire lock again unless (sysopen($startup_fh, $startup_lock, O_CREAT|O_EXCL|O_WRONLY, 0644)) { - debug(__LINE__, "Failed to acquire startup lock on retry: $!"); + _debug(__LINE__, "Failed to acquire startup lock on retry: $!"); return; } - debug(__LINE__, "Acquired startup lock after removing stale lock"); + _debug(__LINE__, "Acquired startup lock after removing stale lock"); } } else { # Invalid PID in lock file, remove it - debug(__LINE__, "Invalid PID in startup lock, removing"); + _debug(__LINE__, "Invalid PID in startup lock, removing"); unlink($startup_lock); # Try to acquire lock again unless (sysopen($startup_fh, $startup_lock, O_CREAT|O_EXCL|O_WRONLY, 0644)) { - debug(__LINE__, "Failed to acquire startup lock on retry: $!"); + _debug(__LINE__, "Failed to acquire startup lock on retry: $!"); return; } - debug(__LINE__, "Acquired startup lock after removing invalid lock"); + _debug(__LINE__, "Acquired startup lock after removing invalid lock"); } } else { - debug(__LINE__, "Could not read startup lock file: $!"); + _debug(__LINE__, "Could not read startup lock file: $!"); return; } } else { - debug(__LINE__, "Acquired startup lock on first try"); + _debug(__LINE__, "Acquired startup lock on first try"); } # We have the startup lock print $startup_fh "$$\n"; close($startup_fh); - debug(__LINE__, "Wrote PID, $$, to startup lock"); + _debug(__LINE__, "Wrote PID, $$, to startup lock"); } -sub pve_mod_worker { +sub _pve_mod_worker { # Give the pid a unique name for easier identification $0 = "pve_mod_worker"; # Ensure directory exists my $run_dir = '/var/run/pve-gpu'; unless (-d $run_dir) { - debug(__LINE__, "Creating directory $run_dir"); - mkdir($run_dir, 0755) or debug(__LINE__, "Failed to create $run_dir: $!"); + _debug(__LINE__, "Creating directory $run_dir"); + mkdir($run_dir, 0755) or _debug(__LINE__, "Failed to create $run_dir: $!"); } # Acquire startup lock and start application - pve_mod_starter(); + _pve_mod_starter(); # Start graphics collectors - start_graphics_collectors(); + _start_graphics_collectors(); # Start sensor collector # TBD @@ -644,36 +658,139 @@ sub pve_mod_worker { # Start UPS collector # TBD + _debug(__LINE__, "All collectors started"); + # Start gui activity monitor process - pve_mod_keep_alive(); + _start_monitor_process(); # Remove startup lock LAST unlink($startup_lock); - debug(__LINE__, "Released startup lock"); + _debug(__LINE__, "Released startup lock"); - debug(__LINE__, "Collectors started successfully, returning"); + _debug(__LINE__, "pve_mod_worker started successfully, returning"); } -sub pve_mod_keep_alive { - debug(__LINE__, "Monitor process started with PID $$"); +sub _start_monitor_process { + _debug(__LINE__, "_start_monitor_process called"); + + # Check if monitor is already running + if (-f $monitor_lock) { + _debug(__LINE__, "Monitor lock file exists, checking if monitor is alive"); + if (open my $fh, '<', $monitor_lock) { + my $pid = <$fh>; + close $fh; + chomp $pid if defined $pid; + + if ($pid && $pid =~ /^\d+$/ && _is_process_alive($pid)) { + _debug(__LINE__, "Monitor process already running with PID $pid"); + return; + } else { + _debug(__LINE__, "Stale monitor lock found (PID: " . ($pid // 'undefined') . "), removing"); + unlink($monitor_lock); + } + } + } + + _debug(__LINE__, "Forking new monitor process"); + my $monitor_pid = fork(); + # give the process a unique name + $0 = "pve-mod-monitor"; + + + unless (defined $monitor_pid) { + _debug(__LINE__, "Failed to fork monitor process: $!"); + return; + } + + if ($monitor_pid == 0) { + # Child process - run the monitor + _debug(__LINE__, "Child process forked, calling _pve_mod_keep_alive"); + _pve_mod_keep_alive(); + exit(0); # Should never reach here + } else { + # Parent process - write PID to lock file + _debug(__LINE__, "Forked monitor process with PID $monitor_pid"); + + if (open my $fh, '>', $monitor_lock) { + print $fh "$monitor_pid\n"; + close $fh; + _debug(__LINE__, "Wrote monitor PID to lock file: $monitor_lock"); + } else { + _debug(__LINE__, "Failed to write monitor lock file: $!"); + kill('TERM', $monitor_pid); + } + } +} + +sub _notify_monitor { + _debug(__LINE__, "_notify_monitor called"); + unless (-f $monitor_lock) { + _debug(__LINE__, "Monitor lock file does not exist"); + return; + } + + _debug(__LINE__, "Monitor lock file exists, reading PID"); + if (open my $fh, '<', $monitor_lock) { + my $pid = <$fh>; + close $fh; + chomp $pid if defined $pid; + if ($pid && $pid =~ /^\d+$/ && _is_process_alive($pid)) { + _debug(__LINE__, "Sending USR1 signal to monitor PID $pid"); + kill('USR1', $pid); + } else { + # Stale lock, remove it + _debug(__LINE__, "Monitor lock is stale (PID: " . ($pid // 'undefined') . "), removing"); + unlink($monitor_lock); + } + } else { + _debug(__LINE__, "Failed to open monitor lock file: $!"); + } +} + +# In monitor process: +sub _pve_mod_keep_alive { + $0 = "pve-mod-gpu-monitor"; + _debug(__LINE__, "Monitor process started with PID $$"); + + my $last_activity = time(); + + # Set up signal handlers + $SIG{USR1} = sub { + $last_activity = time(); + _debug(__LINE__, "Activity ping received"); + }; + $SIG{TERM} = sub { + _debug(__LINE__, "Monitor received SIGTERM, shutting down"); + unlink($monitor_lock); + exit(0); + }; + $SIG{INT} = sub { + _debug(__LINE__, "Monitor received SIGINT, shutting down"); + unlink($monitor_lock); + exit(0); + }; + while (1) { - if(time() - $last_get_graphic_stats_time > $COLLECTOR_TIMEOUT) { - debug(__LINE__, "No get_graphic_stats call in the last $COLLECTOR_TIMEOUT seconds, stopping collectors"); + my $idle_time = time() - $last_activity; + + if ($idle_time > $COLLECTOR_TIMEOUT) { + _debug(__LINE__, "No activity for ${idle_time}s (timeout: ${COLLECTOR_TIMEOUT}s), stopping collectors"); stop_collectors(); + unlink($monitor_lock); exit(0); } - sleep(1); - debug(__LINE__, "pve_mod_keep_alive still running"); + + sleep(10); } } sub cleanup { unless ($is_collector_parent) { - debug(__LINE__, "This process did not start collectors, skipping cleanup"); + _debug(__LINE__, "This process did not start collectors, skipping cleanup"); return; } - debug(__LINE__, "Starting cleanup (this should rarely happen)"); + _debug(__LINE__, "Starting cleanup (this should rarely happen)"); # todo add remove of stat files @@ -687,7 +804,7 @@ sub cleanup { # Instead, add a manual cleanup function that can be called explicitly sub stop_collectors { - debug(__LINE__, "Stopping all collectors"); + _debug(__LINE__, "Stopping all collectors"); # Read current PIDs from lock file my @pids; @@ -701,7 +818,7 @@ sub stop_collectors { if (@pids) { # Send SIGTERM to all collectors - debug(__LINE__, "Sending SIGTERM to " . scalar(@pids) . " process(es)"); + _debug(__LINE__, "Sending SIGTERM to " . scalar(@pids) . " process(es)"); foreach my $pid (@pids) { kill('TERM', $pid) if kill(0, $pid); } @@ -724,7 +841,7 @@ sub stop_collectors { # Force kill any survivors foreach my $pid (@pids) { if (kill(0, $pid)) { - debug(__LINE__, "Force killing process $pid"); + _debug(__LINE__, "Force killing process $pid"); kill('KILL', $pid); } } @@ -732,7 +849,7 @@ sub stop_collectors { unlink $state_file if -f $state_file; unlink $lock_file if -f $lock_file; - debug(__LINE__, "Cleanup complete"); + _debug(__LINE__, "Cleanup complete"); } END { cleanup() }