OpenSource/PVE-mods
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

next snapshot

Browse Source
This commit is contained in:
Meliox 2026-01-03 21:38:46 +01:00
parent 7b78e2a3ae
commit ea76402433
1 changed files with 308 additions and 191 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

499
GPUcollecter.pm
View File

@ -8,18 +8,18 @@ use Fcntl qw(:flock);
use Time::HiRes qw(time); use Time::HiRes qw(time);
use Fcntl qw(:flock O_CREAT O_EXCL O_WRONLY); use Fcntl qw(:flock O_CREAT O_EXCL O_WRONLY);
# Debug configuration - set to 0 to disable all debug output # debug configuration - set to 0 to disable all _debug output
my $DEBUG_ENABLED = 1; my $debug_ENABLED = 1;
# Debug function showing line number and call chain # debug function showing line number and call chain
# Usage: debug(__LINE__, "message") # Usage: _debug(__LINE__, "message")
sub debug { sub _debug {
return unless $DEBUG_ENABLED; return unless $debug_ENABLED;
my ($line, $message) = @_; my ($line, $message) = @_;
# Get function call chain # Get function call chain
my @caller1 = caller(1); # who called debug() my @caller1 = caller(1); # who called _debug()
my @caller2 = caller(2); # parent of caller my @caller2 = caller(2); # parent of caller
my $sub1 = $caller1[3] || 'main'; my $sub1 = $caller1[3] || 'main';
@ -39,12 +39,13 @@ sub debug {
my $stats_dir = '/var/run/pve-gpu'; my $stats_dir = '/var/run/pve-gpu';
my $state_file = '/var/run/pve-gpu/stats.json'; my $state_file = '/var/run/pve-gpu/stats.json';
my $lock_file = '/var/run/pve-gpu/pve-gpu-collector.lock'; my $lock_file = '/var/run/pve-gpu/pve-gpu-collector.lock';
my $monitor_lock = '/var/run/pve-gpu/pve-gpu-monitor.lock';
my $startup_lock = $lock_file . ".startup"; my $startup_lock = $lock_file . ".startup";
my $last_snapshot = {}; my $last_snapshot = {};
my $last_mtime = 0; my $last_mtime = 0;
my $is_collector_parent = 0; # Flag to track if this process started collectors my $is_collector_parent = 0; # Flag to track if this process started collectors
my $last_get_graphic_stats_time = 0; # Track when get_graphic_stats was last called my $last_get_graphic_stats_time = 0; # Track when get_graphic_stats was last called
my $COLLECTOR_TIMEOUT = 0; # Stop collectors x seconds after last get_graphic_stats call my $COLLECTOR_TIMEOUT = 10; # Stop collectors x seconds after last get_graphic_stats call
my $intel_gpu_enabled = 1; # Set to 0 to disable Intel GPU support my $intel_gpu_enabled = 1; # Set to 0 to disable Intel GPU support
my $amd_gpu_enabled = 0; # Set to 1 to enable AMD GPU support (not yet implemented) my $amd_gpu_enabled = 0; # Set to 1 to enable AMD GPU support (not yet implemented)
@ -57,7 +58,7 @@ my $monitor_running = 0;
# ============================================================================ # ============================================================================
# Parse Intel GPU line output format # Parse Intel GPU line output format
sub parse_intel_gpu_line { sub _parse_intel_gpu_line {
my ($line) = @_; my ($line) = @_;
# Expected format (with aligned columns): # Expected format (with aligned columns):
@ -128,7 +129,7 @@ sub parse_intel_gpu_line {
} }
# Get list of Intel GPU devices # Get list of Intel GPU devices
sub get_intel_gpu_devices { sub _get_intel_gpu_devices {
my @devices = (); my @devices = ();
return @devices unless -x '/usr/bin/intel_gpu_top'; return @devices unless -x '/usr/bin/intel_gpu_top';
@ -148,18 +149,18 @@ sub get_intel_gpu_devices {
path => $path, path => $path,
drm_path => "/dev/dri/$card" drm_path => "/dev/dri/$card"
}; };
debug(__LINE__, "Found GPU device: $card -> $name ($path)"); _debug(__LINE__, "Found GPU device: $card -> $name ($path)");
} }
} }
close $fh; close $fh;
} else { } else {
debug(__LINE__, "Failed to run intel_gpu_top -L: $!"); _debug(__LINE__, "Failed to run intel_gpu_top -L: $!");
} }
return @devices; return @devices;
} }
sub collector_for_intel_device { sub _collector_for_intel_device {
my ($device) = @_; my ($device) = @_;
$0 = "pve-mod-gpu-intel-collector: $device->{card}"; $0 = "pve-mod-gpu-intel-collector: $device->{card}";
@ -169,31 +170,31 @@ sub collector_for_intel_device {
# Each device writes to its own file # Each device writes to its own file
my $device_state_file = "/var/run/pve-gpu/stats-$device->{card}.json"; my $device_state_file = "/var/run/pve-gpu/stats-$device->{card}.json";
debug(__LINE__, "Collector started for device: $drm_dev, writing to $device_state_file"); _debug(__LINE__, "Collector started for device: $drm_dev, writing to $device_state_file");
# Set up signal handlers for graceful shutdown # Set up signal handlers for graceful shutdown
my $shutdown = 0; my $shutdown = 0;
$SIG{TERM} = sub { $SIG{TERM} = sub {
debug(__LINE__, "Collector for $device->{card} received SIGTERM"); _debug(__LINE__, "Collector for $device->{card} received SIGTERM");
$shutdown = 1; $shutdown = 1;
kill 'TERM', $intel_gpu_top_pid if defined $intel_gpu_top_pid && $intel_gpu_top_pid > 0; kill 'TERM', $intel_gpu_top_pid if defined $intel_gpu_top_pid && $intel_gpu_top_pid > 0;
}; };
$SIG{INT} = sub { $SIG{INT} = sub {
debug(__LINE__, "Collector for $device->{card} received SIGINT"); _debug(__LINE__, "Collector for $device->{card} received SIGINT");
$shutdown = 1; $shutdown = 1;
kill 'TERM', $intel_gpu_top_pid if defined $intel_gpu_top_pid && $intel_gpu_top_pid > 0; kill 'TERM', $intel_gpu_top_pid if defined $intel_gpu_top_pid && $intel_gpu_top_pid > 0;
}; };
# Run intel_gpu_top once and keep reading from it # Run intel_gpu_top once and keep reading from it
debug(__LINE__, "About to open pipe to intel_gpu_top"); _debug(__LINE__, "About to open pipe to intel_gpu_top");
$intel_gpu_top_pid = open(my $fh, '-|', "intel_gpu_top -d $drm_dev -s 1000 -l 2>&1"); $intel_gpu_top_pid = open(my $fh, '-|', "intel_gpu_top -d $drm_dev -s 1000 -l 2>&1");
unless (defined $intel_gpu_top_pid && $intel_gpu_top_pid > 0) { unless (defined $intel_gpu_top_pid && $intel_gpu_top_pid > 0) {
debug(__LINE__, "Failed to run intel_gpu_top for $drm_dev: $!"); _debug(__LINE__, "Failed to run intel_gpu_top for $drm_dev: $!");
exit 1; exit 1;
} }
debug(__LINE__, "Pipe opened successfully, PID=$intel_gpu_top_pid"); _debug(__LINE__, "Pipe opened successfully, PID=$intel_gpu_top_pid");
my $line_count = 0; my $line_count = 0;
my $node_name = "node0"; # You may want to generate this based on device index my $node_name = "node0"; # You may want to generate this based on device index
@ -209,7 +210,7 @@ sub collector_for_intel_device {
# Check if this is a data line # Check if this is a data line
if ($line =~ /^\s*[\d\s\.]+$/) { if ($line =~ /^\s*[\d\s\.]+$/) {
my $stats = parse_intel_gpu_line($line); my $stats = _parse_intel_gpu_line($line);
if ($stats) { if ($stats) {
# Build device-specific structure (just the node, not the full Graphics/Intel hierarchy) # Build device-specific structure (just the node, not the full Graphics/Intel hierarchy)
@ -227,17 +228,17 @@ sub collector_for_intel_device {
open my $ofh, '>', $device_state_file or die "Failed to open $device_state_file: $!"; open my $ofh, '>', $device_state_file or die "Failed to open $device_state_file: $!";
print $ofh JSON->new->pretty->encode($device_data); print $ofh JSON->new->pretty->encode($device_data);
close $ofh; close $ofh;
debug(__LINE__, "Wrote stats to $device_state_file (line #$line_count)"); _debug(__LINE__, "Wrote stats to $device_state_file (line #$line_count)");
}; };
if ($@) { if ($@) {
debug(__LINE__, "Error writing stats: $@"); _debug(__LINE__, "Error writing stats: $@");
} }
} }
} }
} }
close $fh; close $fh;
debug(__LINE__, "Collector for $device->{card} shutting down"); _debug(__LINE__, "Collector for $device->{card} shutting down");
exit 0; exit 0;
} }
@ -245,25 +246,25 @@ sub collector_for_intel_device {
# AMD GPU Support (Placeholder) # AMD GPU Support (Placeholder)
# ============================================================================ # ============================================================================
sub get_amd_gpu_devices { sub _get_amd_gpu_devices {
# TODO: Implement AMD GPU detection # TODO: Implement AMD GPU detection
# Use rocminfo or similar tools to detect AMD GPUs # Use rocminfo or similar tools to detect AMD GPUs
debug(__LINE__, "AMD GPU support not yet implemented"); _debug(__LINE__, "AMD GPU support not yet implemented");
return (); return ();
} }
sub parse_amd_gpu_line { sub _parse_amd_gpu_line {
my ($line) = @_; my ($line) = @_;
# TODO: Implement AMD GPU line parsing # TODO: Implement AMD GPU line parsing
# Parse rocm-smi or similar output # Parse rocm-smi or similar output
debug(__LINE__, "AMD GPU line parsing not yet implemented"); _debug(__LINE__, "AMD GPU line parsing not yet implemented");
return undef; return undef;
} }
sub collector_for_amd_device { sub _collector_for_amd_device {
my ($device) = @_; my ($device) = @_;
# TODO: Implement AMD GPU collector # TODO: Implement AMD GPU collector
debug(__LINE__, "AMD GPU collector not yet implemented"); _debug(__LINE__, "AMD GPU collector not yet implemented");
exit 0; exit 0;
} }
@ -274,7 +275,7 @@ sub collector_for_amd_device {
sub get_nvidia_gpu_devices { sub get_nvidia_gpu_devices {
# TODO: Implement NVIDIA GPU detection # TODO: Implement NVIDIA GPU detection
# Use nvidia-smi to detect NVIDIA GPUs # Use nvidia-smi to detect NVIDIA GPUs
debug(__LINE__, "NVIDIA GPU support not yet implemented"); _debug(__LINE__, "NVIDIA GPU support not yet implemented");
return (); return ();
} }
@ -282,14 +283,14 @@ sub parse_nvidia_gpu_line {
my ($line) = @_; my ($line) = @_;
# TODO: Implement NVIDIA GPU line parsing # TODO: Implement NVIDIA GPU line parsing
# Parse nvidia-smi output # Parse nvidia-smi output
debug(__LINE__, "NVIDIA GPU line parsing not yet implemented"); _debug(__LINE__, "NVIDIA GPU line parsing not yet implemented");
return undef; return undef;
} }
sub collector_for_nvidia_device { sub collector_for_nvidia_device {
my ($device) = @_; my ($device) = @_;
# TODO: Implement NVIDIA GPU collector # TODO: Implement NVIDIA GPU collector
debug(__LINE__, "NVIDIA GPU collector not yet implemented"); _debug(__LINE__, "NVIDIA GPU collector not yet implemented");
exit 0; exit 0;
} }
@ -298,28 +299,128 @@ sub collector_for_nvidia_device {
# ============================================================================ # ============================================================================
# Check if a process is alive # Check if a process is alive
sub is_process_alive { sub _is_process_alive {
my ($pid) = @_; my ($pid) = @_;
return -d "/proc/$pid"; return -d "/proc/$pid";
} }
# ============================================================================
# API calls
# ============================================================================
sub get_graphic_stats {
_debug(__LINE__, "get_graphic_stats called");
# Start PVE Mod worker, if not already running
_pve_mod_worker();
# Find all device-specific stat files
my $dh;
unless (opendir($dh, $stats_dir)) {
_debug(__LINE__, "Failed to open stats directory: $stats_dir: $!");
return $last_snapshot;
}
my @stat_files = grep { /^stats-card\d+\.json$/ } readdir($dh);
closedir($dh);
unless (@stat_files) {
_debug(__LINE__, "No device stat files found in $stats_dir");
return $last_snapshot;
}
_debug(__LINE__, "Found " . scalar(@stat_files) . " device stat file(s): " . join(', ', @stat_files));
# Check if any files have been modified
my $newest_mtime = 0;
my $files_changed = 0;
foreach my $file (@stat_files) {
my $filepath = "$stats_dir/$file";
my @stat = stat($filepath);
if (@stat && $stat[9] > $newest_mtime) {
$newest_mtime = $stat[9];
}
}
if ($newest_mtime == $last_mtime) {
_debug(__LINE__, "No device files modified, returning cached snapshot");
return $last_snapshot;
}
_debug(__LINE__, "Device files modified ($last_mtime -> $newest_mtime), reading and merging files");
# Merge all device files
my $merged = {
Graphics => {
Intel => {}
}
};
foreach my $file (@stat_files) {
my $filepath = "$stats_dir/$file";
_debug(__LINE__, "Reading device file: $filepath");
eval {
my $fh;
unless (open($fh, '<', $filepath)) {
_debug(__LINE__, "Failed to open $filepath: $!");
return;
}
local $/;
my $json = <$fh>;
close($fh);
_debug(__LINE__, "Read $file, JSON length: " . length($json) . " bytes");
my $device_data = decode_json($json);
# Merge this device's data into the main structure
foreach my $node_name (keys %$device_data) {
$merged->{Graphics}->{Intel}->{$node_name} = $device_data->{$node_name};
_debug(__LINE__, "Merged node '$node_name' from $file");
}
};
if ($@) {
_debug(__LINE__, "Failed to read/parse $filepath: $@");
}
}
# Update cache
$last_snapshot = $merged;
$last_mtime = $newest_mtime;
$last_get_graphic_stats_time = time();
_debug(__LINE__, "Successfully merged " . scalar(keys %{$merged->{Graphics}->{Intel}}) . " device node(s)");
# Notify monitor of activity
_notify_monitor();
return $last_snapshot;
}
# ============================================================================ # ============================================================================
# Main Collector # Main Collector
# ============================================================================ # ============================================================================
sub start_graphics_collectors { sub _start_graphics_collectors {
if ($intel_gpu_enabled == 0 && $amd_gpu_enabled == 0 && $nvidia_gpu_enabled == 0) { if ($intel_gpu_enabled == 0 && $amd_gpu_enabled == 0 && $nvidia_gpu_enabled == 0) {
debug(__LINE__, "No GPU types enabled, skipping collector startup"); _debug(__LINE__, "No GPU types enabled, skipping collector startup");
return; return;
} }
else {
_debug(__LINE__, "Starting graphics collectors");
}
# NOW check if collectors are already running (while holding startup lock) # NOW check if collectors are already running (while holding startup lock)
my %existing_collectors; my %existing_collectors;
if (-f $lock_file) { if (-f $lock_file) {
debug(__LINE__, "Lock file exists: $lock_file"); _debug(__LINE__, "Lock file exists: $lock_file");
if (open(my $lock_fh, '<', $lock_file)) { if (open(my $lock_fh, '<', $lock_file)) {
debug(__LINE__, "Opened lock file for reading"); _debug(__LINE__, "Opened lock file for reading");
while (my $line = <$lock_fh>) { while (my $line = <$lock_fh>) {
chomp $line; chomp $line;
if ($line =~ /^(\d+)\s+(\S+)/) { if ($line =~ /^(\d+)\s+(\S+)/) {
@ -331,10 +432,10 @@ sub start_graphics_collectors {
} }
close($lock_fh); close($lock_fh);
} else { } else {
debug(__LINE__, "Failed to open lock file: $!"); _debug(__LINE__, "Failed to open lock file: $!");
} }
} else { } else {
debug(__LINE__, "Lock file does not exist. Clean start"); _debug(__LINE__, "Lock file does not exist. Clean start");
} }
# Generalized device collector management for future AMD/NVIDIA support # Generalized device collector management for future AMD/NVIDIA support
@ -344,21 +445,22 @@ sub start_graphics_collectors {
# Intel # Intel
if ($intel_gpu_enabled) { if ($intel_gpu_enabled) {
debug(__LINE__, "Checking for intel_gpu_top"); _debug(__LINE__, "Intel GPU support enabled");
_debug(__LINE__, "Checking for intel_gpu_top");
unless (-x '/usr/bin/intel_gpu_top') { unless (-x '/usr/bin/intel_gpu_top') {
debug(__LINE__, "intel_gpu_top not executable"); _debug(__LINE__, "intel_gpu_top not executable");
unlink($startup_lock); unlink($startup_lock);
return; return;
} }
debug(__LINE__, "intel_gpu_top is executable"); _debug(__LINE__, "intel_gpu_top is executable");
debug(__LINE__, "Getting Intel GPU devices"); _debug(__LINE__, "Getting Intel GPU devices");
my @intel_devices = get_intel_gpu_devices(); my @intel_devices = _get_intel_gpu_devices();
unless (@intel_devices) { unless (@intel_devices) {
debug(__LINE__, "No Intel GPU devices found"); _debug(__LINE__, "No Intel GPU devices found");
unlink($startup_lock); unlink($startup_lock);
return; return;
} }
debug(__LINE__, "Found " . scalar(@intel_devices) . " Intel GPU device(s)"); _debug(__LINE__, "Found " . scalar(@intel_devices) . " Intel GPU device(s)");
foreach my $device (@intel_devices) { foreach my $device (@intel_devices) {
push @all_devices, $device; push @all_devices, $device;
push @all_types, 'intel'; push @all_types, 'intel';
@ -366,8 +468,8 @@ sub start_graphics_collectors {
} }
# AMD (future) # AMD (future)
if ($amd_gpu_enabled) { if ($amd_gpu_enabled) {
my @amd_devices = get_amd_gpu_devices(); my @amd_devices = _get_amd_gpu_devices();
debug(__LINE__, "Got " . scalar(@amd_devices) . " AMD devices"); _debug(__LINE__, "Got " . scalar(@amd_devices) . " AMD devices");
foreach my $device (@amd_devices) { foreach my $device (@amd_devices) {
push @all_devices, $device; push @all_devices, $device;
push @all_types, 'amd'; push @all_types, 'amd';
@ -376,7 +478,7 @@ sub start_graphics_collectors {
# NVIDIA (future) # NVIDIA (future)
if ($nvidia_gpu_enabled) { if ($nvidia_gpu_enabled) {
my @nvidia_devices = get_nvidia_gpu_devices(); my @nvidia_devices = get_nvidia_gpu_devices();
debug(__LINE__, "Got " . scalar(@nvidia_devices) . " NVIDIA devices"); _debug(__LINE__, "Got " . scalar(@nvidia_devices) . " NVIDIA devices");
foreach my $device (@nvidia_devices) { foreach my $device (@nvidia_devices) {
push @all_devices, $device; push @all_devices, $device;
push @all_types, 'nvidia'; push @all_types, 'nvidia';
@ -391,47 +493,47 @@ sub start_graphics_collectors {
my $type = $all_types[$i]; my $type = $all_types[$i];
my $card = $device->{card}; my $card = $device->{card};
my $existing_pid = $existing_collectors{$card}; my $existing_pid = $existing_collectors{$card};
if ($existing_pid && is_process_alive($existing_pid)) { if ($existing_pid && _is_process_alive($existing_pid)) {
debug(__LINE__, "Collector for $type $card already running with PID $existing_pid"); _debug(__LINE__, "Collector for $type $card already running with PID $existing_pid");
push @child_pids, $existing_pid; push @child_pids, $existing_pid;
push @child_devices, $device; push @child_devices, $device;
push @child_types, $type; push @child_types, $type;
next; next;
} }
debug(__LINE__, "About to fork collector for $type $card"); _debug(__LINE__, "About to fork collector for $type $card");
my $pid = fork(); my $pid = fork();
unless (defined $pid) { unless (defined $pid) {
debug(__LINE__, "fork failed: $!"); _debug(__LINE__, "fork failed: $!");
unlink($startup_lock); unlink($startup_lock);
die "fork failed: $!"; die "fork failed: $!";
} }
if ($pid == 0) { if ($pid == 0) {
# Child process # Child process
debug(__LINE__, "In child process for $type $card"); _debug(__LINE__, "In child process for $type $card");
if ($type eq 'intel') { if ($type eq 'intel') {
collector_for_intel_device($device); _collector_for_intel_device($device);
} elsif ($type eq 'amd') { } elsif ($type eq 'amd') {
collector_for_amd_device($device); _collector_for_amd_device($device);
} elsif ($type eq 'nvidia') { } elsif ($type eq 'nvidia') {
collector_for_nvidia_device($device); collector_for_nvidia_device($device);
} else { } else {
debug(__LINE__, "Unknown GPU type $type for $card"); _debug(__LINE__, "Unknown GPU type $type for $card");
exit(1); exit(1);
} }
# Should not reach here # Should not reach here
exit(0); exit(0);
} else { } else {
debug(__LINE__, "Forked child PID $pid for $type $card"); _debug(__LINE__, "Forked child PID $pid for $type $card");
push @child_pids, $pid; push @child_pids, $pid;
push @child_devices, $device; push @child_devices, $device;
push @child_types, $type; push @child_types, $type;
} }
} }
debug(__LINE__, "Active children: " . join(", ", @child_pids)); _debug(__LINE__, "Active children: " . join(", ", @child_pids));
# Write child PIDs and device cards/types to lock file # Write child PIDs and device cards/types to lock file
if (open(my $lock_fh, '>', $lock_file)) { if (open(my $lock_fh, '>', $lock_file)) {
debug(__LINE__, "Opened lock file for writing"); _debug(__LINE__, "Opened lock file for writing");
for (my $i = 0; $i < @child_pids; $i++) { for (my $i = 0; $i < @child_pids; $i++) {
my $pid = $child_pids[$i]; my $pid = $child_pids[$i];
my $device = $child_devices[$i]; my $device = $child_devices[$i];
@ -440,11 +542,11 @@ sub start_graphics_collectors {
print $lock_fh "$pid $card $type\n"; print $lock_fh "$pid $card $type\n";
} }
close($lock_fh); close($lock_fh);
debug(__LINE__, "Wrote " . scalar(@child_pids) . " collector PID(s) to lock file"); _debug(__LINE__, "Wrote " . scalar(@child_pids) . " collector PID(s) to lock file");
} else { } else {
debug(__LINE__, "Failed to open lock file for writing: $!"); _debug(__LINE__, "Failed to open lock file for writing: $!");
foreach my $pid (@child_pids) { foreach my $pid (@child_pids) {
debug(__LINE__, "Killing child $pid due to lock file write failure"); _debug(__LINE__, "Killing child $pid due to lock file write failure");
kill 'TERM', $pid; kill 'TERM', $pid;
} }
unlink($startup_lock); unlink($startup_lock);
@ -463,115 +565,27 @@ sub start_graphics_collectors {
my $alive = kill(0, $pid); my $alive = kill(0, $pid);
if ($alive) { if ($alive) {
$any_alive = 1; $any_alive = 1;
debug(__LINE__, "Verified child PID $pid for $card is alive"); _debug(__LINE__, "Verified child PID $pid for $card is alive");
} else { } else {
debug(__LINE__, "WARNING - Child PID $pid for $card died immediately!"); _debug(__LINE__, "WARNING - Child PID $pid for $card died immediately!");
} }
} }
unless ($any_alive) { unless ($any_alive) {
debug(__LINE__, "ERROR - No children alive after fork!"); _debug(__LINE__, "ERROR - No children alive after fork!");
} }
_debug(__LINE__, "Graphics collectors started");
} }
sub get_graphic_stats { sub _pve_mod_starter {
debug(__LINE__, "get_graphic_stats called");
# Start PVE Mod worker, if not already running
pve_mod_worker();
# Find all device-specific stat files
my $dh;
unless (opendir($dh, $stats_dir)) {
debug(__LINE__, "Failed to open stats directory: $stats_dir: $!");
return $last_snapshot;
}
my @stat_files = grep { /^stats-card\d+\.json$/ } readdir($dh);
closedir($dh);
unless (@stat_files) {
debug(__LINE__, "No device stat files found in $stats_dir");
return $last_snapshot;
}
debug(__LINE__, "Found " . scalar(@stat_files) . " device stat file(s): " . join(', ', @stat_files));
# Check if any files have been modified
my $newest_mtime = 0;
my $files_changed = 0;
foreach my $file (@stat_files) {
my $filepath = "$stats_dir/$file";
my @stat = stat($filepath);
if (@stat && $stat[9] > $newest_mtime) {
$newest_mtime = $stat[9];
}
}
if ($newest_mtime == $last_mtime) {
debug(__LINE__, "No device files modified, returning cached snapshot");
return $last_snapshot;
}
debug(__LINE__, "Device files modified ($last_mtime -> $newest_mtime), reading and merging files");
# Merge all device files
my $merged = {
Graphics => {
Intel => {}
}
};
foreach my $file (@stat_files) {
my $filepath = "$stats_dir/$file";
debug(__LINE__, "Reading device file: $filepath");
eval {
my $fh;
unless (open($fh, '<', $filepath)) {
debug(__LINE__, "Failed to open $filepath: $!");
return;
}
local $/;
my $json = <$fh>;
close($fh);
debug(__LINE__, "Read $file, JSON length: " . length($json) . " bytes");
my $device_data = decode_json($json);
# Merge this device's data into the main structure
foreach my $node_name (keys %$device_data) {
$merged->{Graphics}->{Intel}->{$node_name} = $device_data->{$node_name};
debug(__LINE__, "Merged node '$node_name' from $file");
}
};
if ($@) {
debug(__LINE__, "Failed to read/parse $filepath: $@");
}
}
# Update cache
$last_snapshot = $merged;
$last_mtime = $newest_mtime;
$last_get_graphic_stats_time = time();
debug(__LINE__, "Successfully merged " . scalar(keys %{$merged->{Graphics}->{Intel}}) . " device node(s)");
return $last_snapshot;
}
sub pve_mod_starter {
# Try to acquire startup lock FIRST (prevents race conditions) # Try to acquire startup lock FIRST (prevents race conditions)
my $startup_fh; my $startup_fh;
debug(__LINE__, "Trying to acquire startup lock: $startup_lock"); _debug(__LINE__, "Trying to acquire startup lock: $startup_lock");
unless (sysopen($startup_fh, $startup_lock, O_CREAT|O_EXCL|O_WRONLY, 0644)) { unless (sysopen($startup_fh, $startup_lock, O_CREAT|O_EXCL|O_WRONLY, 0644)) {
# Startup lock exists - check if it's stale # Startup lock exists - check if it's stale
debug(__LINE__, "Startup lock exists, checking if stale"); _debug(__LINE__, "Startup lock exists, checking if stale");
if (open(my $check_fh, '<', $startup_lock)) { if (open(my $check_fh, '<', $startup_lock)) {
my $lock_pid = <$check_fh>; my $lock_pid = <$check_fh>;
@ -579,64 +593,64 @@ sub pve_mod_starter {
close($check_fh); close($check_fh);
if (defined $lock_pid && $lock_pid =~ /^\d+$/) { if (defined $lock_pid && $lock_pid =~ /^\d+$/) {
debug(__LINE__, "Startup lock held by PID $lock_pid"); _debug(__LINE__, "Startup lock held by PID $lock_pid");
if (is_process_alive($lock_pid)) { if (_is_process_alive($lock_pid)) {
debug(__LINE__, "Lock holder PID $lock_pid is still alive, waiting"); _debug(__LINE__, "Lock holder PID $lock_pid is still alive, waiting");
} else { } else {
# Lock holder is dead, remove stale lock # Lock holder is dead, remove stale lock
debug(__LINE__, "Lock holder PID $lock_pid is dead, removing stale startup lock"); _debug(__LINE__, "Lock holder PID $lock_pid is dead, removing stale startup lock");
unlink($startup_lock); unlink($startup_lock);
# Try to acquire lock again # Try to acquire lock again
unless (sysopen($startup_fh, $startup_lock, O_CREAT|O_EXCL|O_WRONLY, 0644)) { unless (sysopen($startup_fh, $startup_lock, O_CREAT|O_EXCL|O_WRONLY, 0644)) {
debug(__LINE__, "Failed to acquire startup lock on retry: $!"); _debug(__LINE__, "Failed to acquire startup lock on retry: $!");
return; return;
} }
debug(__LINE__, "Acquired startup lock after removing stale lock"); _debug(__LINE__, "Acquired startup lock after removing stale lock");
} }
} else { } else {
# Invalid PID in lock file, remove it # Invalid PID in lock file, remove it
debug(__LINE__, "Invalid PID in startup lock, removing"); _debug(__LINE__, "Invalid PID in startup lock, removing");
unlink($startup_lock); unlink($startup_lock);
# Try to acquire lock again # Try to acquire lock again
unless (sysopen($startup_fh, $startup_lock, O_CREAT|O_EXCL|O_WRONLY, 0644)) { unless (sysopen($startup_fh, $startup_lock, O_CREAT|O_EXCL|O_WRONLY, 0644)) {
debug(__LINE__, "Failed to acquire startup lock on retry: $!"); _debug(__LINE__, "Failed to acquire startup lock on retry: $!");
return; return;
} }
debug(__LINE__, "Acquired startup lock after removing invalid lock"); _debug(__LINE__, "Acquired startup lock after removing invalid lock");
} }
} else { } else {
debug(__LINE__, "Could not read startup lock file: $!"); _debug(__LINE__, "Could not read startup lock file: $!");
return; return;
} }
} else { } else {
debug(__LINE__, "Acquired startup lock on first try"); _debug(__LINE__, "Acquired startup lock on first try");
} }
# We have the startup lock # We have the startup lock
print $startup_fh "$$\n"; print $startup_fh "$$\n";
close($startup_fh); close($startup_fh);
debug(__LINE__, "Wrote PID, $$, to startup lock"); _debug(__LINE__, "Wrote PID, $$, to startup lock");
} }
sub pve_mod_worker { sub _pve_mod_worker {
# Give the pid a unique name for easier identification # Give the pid a unique name for easier identification
$0 = "pve_mod_worker"; $0 = "pve_mod_worker";
# Ensure directory exists # Ensure directory exists
my $run_dir = '/var/run/pve-gpu'; my $run_dir = '/var/run/pve-gpu';
unless (-d $run_dir) { unless (-d $run_dir) {
debug(__LINE__, "Creating directory $run_dir"); _debug(__LINE__, "Creating directory $run_dir");
mkdir($run_dir, 0755) or debug(__LINE__, "Failed to create $run_dir: $!"); mkdir($run_dir, 0755) or _debug(__LINE__, "Failed to create $run_dir: $!");
} }
# Acquire startup lock and start application # Acquire startup lock and start application
pve_mod_starter(); _pve_mod_starter();
# Start graphics collectors # Start graphics collectors
start_graphics_collectors(); _start_graphics_collectors();
# Start sensor collector # Start sensor collector
# TBD # TBD
@ -644,36 +658,139 @@ sub pve_mod_worker {
# Start UPS collector # Start UPS collector
# TBD # TBD
_debug(__LINE__, "All collectors started");
# Start gui activity monitor process # Start gui activity monitor process
pve_mod_keep_alive(); _start_monitor_process();
# Remove startup lock LAST # Remove startup lock LAST
unlink($startup_lock); unlink($startup_lock);
debug(__LINE__, "Released startup lock"); _debug(__LINE__, "Released startup lock");
debug(__LINE__, "Collectors started successfully, returning"); _debug(__LINE__, "pve_mod_worker started successfully, returning");
} }
sub pve_mod_keep_alive { sub _start_monitor_process {
debug(__LINE__, "Monitor process started with PID $$"); _debug(__LINE__, "_start_monitor_process called");
# Check if monitor is already running
if (-f $monitor_lock) {
_debug(__LINE__, "Monitor lock file exists, checking if monitor is alive");
if (open my $fh, '<', $monitor_lock) {
my $pid = <$fh>;
close $fh;
chomp $pid if defined $pid;
if ($pid && $pid =~ /^\d+$/ && _is_process_alive($pid)) {
_debug(__LINE__, "Monitor process already running with PID $pid");
return;
} else {
_debug(__LINE__, "Stale monitor lock found (PID: " . ($pid // 'undefined') . "), removing");
unlink($monitor_lock);
}
}
}
_debug(__LINE__, "Forking new monitor process");
my $monitor_pid = fork();
# give the process a unique name
$0 = "pve-mod-monitor";
unless (defined $monitor_pid) {
_debug(__LINE__, "Failed to fork monitor process: $!");
return;
}
if ($monitor_pid == 0) {
# Child process - run the monitor
_debug(__LINE__, "Child process forked, calling _pve_mod_keep_alive");
_pve_mod_keep_alive();
exit(0); # Should never reach here
} else {
# Parent process - write PID to lock file
_debug(__LINE__, "Forked monitor process with PID $monitor_pid");
if (open my $fh, '>', $monitor_lock) {
print $fh "$monitor_pid\n";
close $fh;
_debug(__LINE__, "Wrote monitor PID to lock file: $monitor_lock");
} else {
_debug(__LINE__, "Failed to write monitor lock file: $!");
kill('TERM', $monitor_pid);
}
}
}
sub _notify_monitor {
_debug(__LINE__, "_notify_monitor called");
unless (-f $monitor_lock) {
_debug(__LINE__, "Monitor lock file does not exist");
return;
}
_debug(__LINE__, "Monitor lock file exists, reading PID");
if (open my $fh, '<', $monitor_lock) {
my $pid = <$fh>;
close $fh;
chomp $pid if defined $pid;
if ($pid && $pid =~ /^\d+$/ && _is_process_alive($pid)) {
_debug(__LINE__, "Sending USR1 signal to monitor PID $pid");
kill('USR1', $pid);
} else {
# Stale lock, remove it
_debug(__LINE__, "Monitor lock is stale (PID: " . ($pid // 'undefined') . "), removing");
unlink($monitor_lock);
}
} else {
_debug(__LINE__, "Failed to open monitor lock file: $!");
}
}
# In monitor process:
sub _pve_mod_keep_alive {
$0 = "pve-mod-gpu-monitor";
_debug(__LINE__, "Monitor process started with PID $$");
my $last_activity = time();
# Set up signal handlers
$SIG{USR1} = sub {
$last_activity = time();
_debug(__LINE__, "Activity ping received");
};
$SIG{TERM} = sub {
_debug(__LINE__, "Monitor received SIGTERM, shutting down");
unlink($monitor_lock);
exit(0);
};
$SIG{INT} = sub {
_debug(__LINE__, "Monitor received SIGINT, shutting down");
unlink($monitor_lock);
exit(0);
};
while (1) { while (1) {
if(time() - $last_get_graphic_stats_time > $COLLECTOR_TIMEOUT) { my $idle_time = time() - $last_activity;
debug(__LINE__, "No get_graphic_stats call in the last $COLLECTOR_TIMEOUT seconds, stopping collectors");
if ($idle_time > $COLLECTOR_TIMEOUT) {
_debug(__LINE__, "No activity for ${idle_time}s (timeout: ${COLLECTOR_TIMEOUT}s), stopping collectors");
stop_collectors(); stop_collectors();
unlink($monitor_lock);
exit(0); exit(0);
} }
sleep(1);
debug(__LINE__, "pve_mod_keep_alive still running"); sleep(10);
} }
} }
sub cleanup { sub cleanup {
unless ($is_collector_parent) { unless ($is_collector_parent) {
debug(__LINE__, "This process did not start collectors, skipping cleanup"); _debug(__LINE__, "This process did not start collectors, skipping cleanup");
return; return;
} }
debug(__LINE__, "Starting cleanup (this should rarely happen)"); _debug(__LINE__, "Starting cleanup (this should rarely happen)");
# todo add remove of stat files # todo add remove of stat files
@ -687,7 +804,7 @@ sub cleanup {
# Instead, add a manual cleanup function that can be called explicitly # Instead, add a manual cleanup function that can be called explicitly
sub stop_collectors { sub stop_collectors {
debug(__LINE__, "Stopping all collectors"); _debug(__LINE__, "Stopping all collectors");
# Read current PIDs from lock file # Read current PIDs from lock file
my @pids; my @pids;
@ -701,7 +818,7 @@ sub stop_collectors {
if (@pids) { if (@pids) {
# Send SIGTERM to all collectors # Send SIGTERM to all collectors
debug(__LINE__, "Sending SIGTERM to " . scalar(@pids) . " process(es)"); _debug(__LINE__, "Sending SIGTERM to " . scalar(@pids) . " process(es)");
foreach my $pid (@pids) { foreach my $pid (@pids) {
kill('TERM', $pid) if kill(0, $pid); kill('TERM', $pid) if kill(0, $pid);
} }
@ -724,7 +841,7 @@ sub stop_collectors {
# Force kill any survivors # Force kill any survivors
foreach my $pid (@pids) { foreach my $pid (@pids) {
if (kill(0, $pid)) { if (kill(0, $pid)) {
debug(__LINE__, "Force killing process $pid"); _debug(__LINE__, "Force killing process $pid");
kill('KILL', $pid); kill('KILL', $pid);
} }
} }
@ -732,7 +849,7 @@ sub stop_collectors {
unlink $state_file if -f $state_file; unlink $state_file if -f $state_file;
unlink $lock_file if -f $lock_file; unlink $lock_file if -f $lock_file;
debug(__LINE__, "Cleanup complete"); _debug(__LINE__, "Cleanup complete");
} }
END { cleanup() } END { cleanup() }