OpenSource/PVE-mods
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Another udpate

Browse Source
This commit is contained in:
Meliox 2026-01-04 14:31:52 +01:00
parent e698160473
commit d6a6257003
1 changed files with 197 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

358
GPUcollecter.pm
View File

@ -7,9 +7,11 @@ use POSIX qw(WNOHANG);
use Fcntl qw(:flock); use Fcntl qw(:flock);
use Time::HiRes qw(time); use Time::HiRes qw(time);
use Fcntl qw(:flock O_CREAT O_EXCL O_WRONLY); use Fcntl qw(:flock O_CREAT O_EXCL O_WRONLY);
use File::Path qw(remove_tree);
# debug configuration - set to 0 to disable all _debug output # debug configuration - set to 0 to disable all _debug output
my $debug_ENABLED = 1; my $debug_ENABLED = 1;
my $VERSION = '1.0.0';
# debug function showing line number and call chain # debug function showing line number and call chain
# Usage: _debug(__LINE__, "message") # Usage: _debug(__LINE__, "message")
@ -36,10 +38,11 @@ sub _debug {
} }
} }
my $stats_dir = '/var/run/pve-gpu'; my $pve_mod_working_dir = '/run/pveproxy/pve-mod';
my $state_file = '/var/run/pve-gpu/stats.json'; my $stats_dir = $pve_mod_working_dir;
my $lock_file = '/var/run/pve-gpu/pve-gpu-collector.lock'; my $state_file = "$pve_mod_working_dir/stats.json";
my $monitor_lock = '/var/run/pve-gpu/pve-mod-monitor.lock'; my $lock_file = "$pve_mod_working_dir/pve-mod-worker.lock";
my $pve_mod_worker_lock = "$pve_mod_working_dir/pve-mod-pve_mod_worker.lock";
my $startup_lock = $lock_file . ".startup"; my $startup_lock = $lock_file . ".startup";
my $last_snapshot = {}; my $last_snapshot = {};
my $last_mtime = 0; my $last_mtime = 0;
@ -50,8 +53,8 @@ my $COLLECTOR_TIMEOUT = 10; # Stop collectors x seconds after last get_graphic
my $intel_gpu_enabled = 1; # Set to 0 to disable Intel GPU support my $intel_gpu_enabled = 1; # Set to 0 to disable Intel GPU support
my $amd_gpu_enabled = 0; # Set to 1 to enable AMD GPU support (not yet implemented) my $amd_gpu_enabled = 0; # Set to 1 to enable AMD GPU support (not yet implemented)
my $nvidia_gpu_enabled = 0; # Set to 1 to enable NVIDIA GPU support (not yet implemented) my $nvidia_gpu_enabled = 0; # Set to 1 to enable NVIDIA GPU support (not yet implemented)
my $monitor_pid; my $pve_mod_worker_pid;
my $monitor_running = 0; my $pve_mod_worker_running = 0;
# ============================================================================ # ============================================================================
# Intel GPU Support # Intel GPU Support
@ -168,7 +171,7 @@ sub _collector_for_intel_device {
my $intel_gpu_top_pid = undef; my $intel_gpu_top_pid = undef;
# Each device writes to its own file # Each device writes to its own file
my $device_state_file = "/var/run/pve-gpu/stats-$device->{card}.json"; my $device_state_file = "$pve_mod_working_dir/stats-$device->{card}.json";
_debug(__LINE__, "Collector started for device: $drm_dev, writing to $device_state_file"); _debug(__LINE__, "Collector started for device: $drm_dev, writing to $device_state_file");
@ -304,17 +307,105 @@ sub _is_process_alive {
return -d "/proc/$pid"; return -d "/proc/$pid";
} }
sub _read_lock_pid {
my ($lock_path) = @_;
return undef unless open(my $fh, '<', $lock_path);
my $pid = <$fh>;
close($fh);
chomp $pid if defined $pid;
return $pid;
}
sub _acquire_exclusive_lock {
my ($lock_path, $purpose) = @_;
$purpose //= 'lock';
my $fh;
# Try to create lock file exclusively
if (sysopen($fh, $lock_path, O_CREAT|O_EXCL|O_WRONLY, 0644)) {
_debug(__LINE__, "Acquired $purpose on first try");
return $fh;
}
# Lock file creation failed - check if it's stale or held by another process
_debug(__LINE__, ucfirst($purpose) . " exists, checking if stale");
my $lock_pid = _read_lock_pid($lock_path);
if (!defined $lock_pid) {
_debug(__LINE__, "Could not read $purpose file: $!");
return undef;
}
if ($lock_pid eq '' || $lock_pid !~ /^\d+$/) {
_debug(__LINE__, "Invalid PID in $purpose: '" . ($lock_pid // 'undefined') . "', removing");
unlink($lock_path);
} elsif (_is_process_alive($lock_pid)) {
_debug(__LINE__, ucfirst($purpose) . " holder PID $lock_pid is still alive");
return undef;
} else {
_debug(__LINE__, ucfirst($purpose) . " holder PID $lock_pid is dead, removing stale lock");
unlink($lock_path);
}
# Try to acquire lock again after cleanup
unless (sysopen($fh, $lock_path, O_CREAT|O_EXCL|O_WRONLY, 0644)) {
_debug(__LINE__, "Failed to acquire $purpose on retry: $!");
return undef;
}
_debug(__LINE__, "Acquired $purpose after removing stale lock");
return $fh;
}
sub _is_lock_stale {
my ($lock_path) = @_;
return 0 unless open(my $fh, '<', $lock_path);
my $lock_pid = <$fh>;
chomp $lock_pid if defined $lock_pid;
close($fh);
# Invalid or missing PID
return 1 unless defined $lock_pid && $lock_pid =~ /^\d+$/;
# Valid PID but process is dead
return !_is_process_alive($lock_pid);
}
sub _ensure_pve_mod_directory_exists {
unless (-d $pve_mod_working_dir) {
_debug(__LINE__, "Creating directory $pve_mod_working_dir");
unless (mkdir($pve_mod_working_dir, 0755)) {
_debug(__LINE__, "Failed to create $pve_mod_working_dir: $!. PVE Mod cannot start.");
die "Failed to create $pve_mod_working_dir: $!";
}
_debug(__LINE__, "Directory $pve_mod_working_dir created");
} else {
_debug(__LINE__, "Directory $pve_mod_working_dir already exists");
}
}
sub _pve_mod_hello {
_debug(__LINE__, "PVE Mod is being started. Version $VERSION");
}
# ============================================================================ # ============================================================================
# API calls # API calls
# ============================================================================ # ============================================================================
sub get_graphic_stats { sub get_graphic_stats {
# todo name the process without overruling other processes # todo name the process without overruling other processes
_debug(__LINE__, "get_graphic_stats called"); _debug(__LINE__, "get_graphic_stats called");
_pve_mod_hello();
# Start PVE Mod worker, if not already running # Start PVE Mod
_pve_mod_worker(); _pve_mod_starter();
# Find all device-specific stat files # Find all device-specific stat files
my $dh; my $dh;
@ -397,8 +488,8 @@ sub get_graphic_stats {
_debug(__LINE__, "Successfully merged " . scalar(keys %{$merged->{Graphics}->{Intel}}) . " device node(s)"); _debug(__LINE__, "Successfully merged " . scalar(keys %{$merged->{Graphics}->{Intel}}) . " device node(s)");
# Notify monitor of activity # Notify pve_mod_worker of activity
_notify_monitor(); _notify_pve_mod_worker();
return $last_snapshot; return $last_snapshot;
} }
@ -423,6 +514,7 @@ sub _start_graphics_collectors {
_debug(__LINE__, "Lock file exists: $lock_file"); _debug(__LINE__, "Lock file exists: $lock_file");
if (open(my $lock_fh, '<', $lock_file)) { if (open(my $lock_fh, '<', $lock_file)) {
_debug(__LINE__, "Opened lock file for reading"); _debug(__LINE__, "Opened lock file for reading");
flock($lock_fh, LOCK_EX) or _debug(__LINE__, "Failed to lock $lock_file: $!");
while (my $line = <$lock_fh>) { while (my $line = <$lock_fh>) {
chomp $line; chomp $line;
if ($line =~ /^(\d+)\s+(\S+)/) { if ($line =~ /^(\d+)\s+(\S+)/) {
@ -543,6 +635,7 @@ sub _start_graphics_collectors {
my $card = $device->{card} // ''; my $card = $device->{card} // '';
print $lock_fh "$pid $card $type\n"; print $lock_fh "$pid $card $type\n";
} }
flock($lock_fh, LOCK_UN);
close($lock_fh); close($lock_fh);
_debug(__LINE__, "Wrote " . scalar(@child_pids) . " collector PID(s) to lock file"); _debug(__LINE__, "Wrote " . scalar(@child_pids) . " collector PID(s) to lock file");
} else { } else {
@ -579,95 +672,30 @@ sub _start_graphics_collectors {
_debug(__LINE__, "Graphics collectors started"); _debug(__LINE__, "Graphics collectors started");
} }
# ============================================================================
# PVE Mod Worker
# ============================================================================
sub _pve_mod_starter { sub _pve_mod_starter {
# Try to acquire startup lock FIRST (prevents race conditions) # Check if pve_mod_worker is already running - if so, entire system is already up
my $startup_fh; _debug(__LINE__, "Checking if pve_mod_worker is already running");
if (_is_pve_mod_worker_running()) {
_debug(__LINE__, "Trying to acquire startup lock: $startup_lock"); _debug(__LINE__, "pve_mod_worker process already running, system is already started");
return "pve_mod_worker process already running, system is already started";
unless (sysopen($startup_fh, $startup_lock, O_CREAT|O_EXCL|O_WRONLY, 0644)) {
# Startup lock exists - check if it's stale
_debug(__LINE__, "Startup lock exists, checking if stale");
if (open(my $check_fh, '<', $startup_lock)) {
my $lock_pid = <$check_fh>;
chomp $lock_pid if defined $lock_pid;
close($check_fh);
if (defined $lock_pid && $lock_pid =~ /^\d+$/) {
_debug(__LINE__, "Startup lock held by PID $lock_pid");
if (_is_process_alive($lock_pid)) {
_debug(__LINE__, "Lock holder PID $lock_pid is still alive, waiting");
} else {
# Lock holder is dead, remove stale lock
_debug(__LINE__, "Lock holder PID $lock_pid is dead, removing stale startup lock");
unlink($startup_lock);
# Try to acquire lock again
unless (sysopen($startup_fh, $startup_lock, O_CREAT|O_EXCL|O_WRONLY, 0644)) {
_debug(__LINE__, "Failed to acquire startup lock on retry: $!");
return;
}
_debug(__LINE__, "Acquired startup lock after removing stale lock");
}
} else {
# Invalid PID in lock file, remove it
_debug(__LINE__, "Invalid PID in startup lock, removing");
unlink($startup_lock);
# Try to acquire lock again
unless (sysopen($startup_fh, $startup_lock, O_CREAT|O_EXCL|O_WRONLY, 0644)) {
_debug(__LINE__, "Failed to acquire startup lock on retry: $!");
return;
}
_debug(__LINE__, "Acquired startup lock after removing invalid lock");
}
} else {
_debug(__LINE__, "Could not read startup lock file: $!");
return;
}
} else {
_debug(__LINE__, "Acquired startup lock on first try");
} }
_debug(__LINE__, "PVE mod worker is not running. PVE Mod will be started.");
# We have the startup lock
# Ensure directory exists
_ensure_pve_mod_directory_exists();
# Try to acquire startup lock FIRST (prevents race conditions)
_debug(__LINE__, "Trying to acquire startup lock: $startup_lock");
my $startup_fh = _acquire_exclusive_lock($startup_lock, 'startup lock');
return unless $startup_fh;
print $startup_fh "$$\n"; print $startup_fh "$$\n";
close($startup_fh); close($startup_fh);
_debug(__LINE__, "Wrote PID, $$, to startup lock"); _debug(__LINE__, "Wrote PID, $$, to startup lock");
}
sub _pve_mod_worker {
# Ensure directory exists
my $run_dir = '/var/run/pve-gpu';
unless (-d $run_dir) {
_debug(__LINE__, "Creating directory $run_dir");
mkdir($run_dir, 0755) or _debug(__LINE__, "Failed to create $run_dir: $!");
}
# Check if monitor is already running
if (-f $monitor_lock) {
_debug(__LINE__, "Monitor lock file exists, checking if monitor is alive");
if (open my $fh, '<', $monitor_lock) {
my $pid = <$fh>;
close $fh;
chomp $pid if defined $pid;
if ($pid && $pid =~ /^\d+$/ && _is_process_alive($pid)) {
_debug(__LINE__, "Monitor process already running with PID $pid, don't start another pve_mod_worker");
return;
} else {
_debug(__LINE__, "Stale monitor lock found (PID: " . ($pid // 'undefined') . "), removing");
unlink $monitor_lock;
}
}
} else {
_debug(__LINE__, "PVE mod worker is not running. It can be started.");
}
# Acquire startup lock and start application
_pve_mod_starter();
# Start graphics collectors # Start graphics collectors
_start_graphics_collectors(); _start_graphics_collectors();
@ -680,8 +708,8 @@ sub _pve_mod_worker {
_debug(__LINE__, "All collectors started"); _debug(__LINE__, "All collectors started");
# Start gui activity monitor process # Start pve mod workers
_pve_mod_monitor(); _pve_mod_worker();
# Remove startup lock LAST # Remove startup lock LAST
unlink($startup_lock); unlink($startup_lock);
@ -691,96 +719,83 @@ sub _pve_mod_worker {
} }
# ============================================================================ # ============================================================================
# Monitor Process # PVE Mod Worker
# ============================================================================ # ============================================================================
sub _pve_mod_monitor { sub _pve_mod_worker {
_debug(__LINE__, "_pve_mod_monitor called"); _debug(__LINE__, "_pve_mod_worker called");
# Check if monitor is already running # Check if worker is already running
if (-f $monitor_lock) { my $pve_mod_worker_fh = _acquire_exclusive_lock($pve_mod_worker_lock, 'pve_mod_worker lock');
_debug(__LINE__, "Monitor lock file exists, checking if monitor is alive"); return unless $pve_mod_worker_fh;
if (open my $fh, '<', $monitor_lock) { print $pve_mod_worker_fh "$$\n";
my $pid = <$fh>; close($pve_mod_worker_fh);
close $fh;
chomp $pid if defined $pid;
if ($pid && $pid =~ /^\d+$/ && _is_process_alive($pid)) {
_debug(__LINE__, "Monitor process already running with PID $pid");
return;
} else {
_debug(__LINE__, "Stale monitor lock found (PID: " . ($pid // 'undefined') . "), removing");
unlink($monitor_lock);
}
}
}
_debug(__LINE__, "Forking new monitor process"); _debug(__LINE__, "Forking new pve_mod_worker process");
my $monitor_pid = fork(); my $pve_mod_worker_pid = fork();
unless (defined $monitor_pid) { unless (defined $pve_mod_worker_pid) {
_debug(__LINE__, "Failed to fork monitor process: $!"); _debug(__LINE__, "Failed to fork pve_mod_worker process: $!");
return; return;
} }
if ($monitor_pid == 0) { if ($pve_mod_worker_pid == 0) {
# Child process - run the monitor # Child process - run the pve_mod_worker
_debug(__LINE__, "Child process forked, calling _pve_mod_keep_alive"); _debug(__LINE__, "Child process forked, calling _pve_mod_keep_alive");
_pve_mod_keep_alive(); _pve_mod_keep_alive();
exit(0); # Should never reach here exit(0); # Should never reach here
} else { } else {
# Parent process - write PID to lock file # Parent process - write PID to lock file
_debug(__LINE__, "Forked monitor process with PID $monitor_pid"); _debug(__LINE__, "Forked pve_mod_worker process with PID $pve_mod_worker_pid");
if (open my $fh, '>', $monitor_lock) { if (open my $fh, '>', $pve_mod_worker_lock) {
print $fh "$monitor_pid\n"; print $fh "$pve_mod_worker_pid\n";
close $fh; close $fh;
_debug(__LINE__, "Wrote monitor PID to lock file: $monitor_lock"); _debug(__LINE__, "Wrote pve_mod_worker PID to lock file: $pve_mod_worker_lock");
} else { } else {
_debug(__LINE__, "Failed to write monitor lock file: $!"); _debug(__LINE__, "Failed to write pve_mod_worker lock file: $!");
kill('TERM', $monitor_pid); kill('TERM', $pve_mod_worker_pid);
} }
} }
_debug(__LINE__, "Monitor process started successfully"); _debug(__LINE__, "pve_mod_worker process started successfully");
} }
sub _notify_monitor { sub _notify_pve_mod_worker {
_debug(__LINE__, "_notify_monitor called"); _debug(__LINE__, "_notify_pve_mod_worker called");
unless (-f $monitor_lock) { unless (-f $pve_mod_worker_lock) {
_debug(__LINE__, "Monitor lock file does not exist"); _debug(__LINE__, "pve_mod_worker lock file does not exist");
return; return;
} }
_debug(__LINE__, "Monitor lock file exists, reading PID"); _debug(__LINE__, "pve_mod_worker lock file exists, reading PID");
if (open my $fh, '<', $monitor_lock) { if (open my $fh, '<', $pve_mod_worker_lock) {
my $pid = <$fh>; my $pid = <$fh>;
close $fh; close $fh;
chomp $pid if defined $pid; chomp $pid if defined $pid;
if ($pid && $pid =~ /^(\d+)$/) { if (defined $pid && $pid =~ /^(\d+)$/) {
# Untaint by capturing in regex - $1 is now untainted # Untaint by capturing in regex - $1 is now untainted
my $clean_pid = $1; my $clean_pid = $1;
if (_is_process_alive($clean_pid)) { if (_is_process_alive($clean_pid)) {
_debug(__LINE__, "Sending USR1 signal to monitor PID $clean_pid"); _debug(__LINE__, "Sending USR1 signal to pve_mod_worker PID $clean_pid");
my $result = kill('USR1', $clean_pid); my $result = kill('USR1', $clean_pid);
_debug(__LINE__, "Signal result: $result"); _debug(__LINE__, "Signal result: $result");
} else { } else {
_debug(__LINE__, "Monitor process $clean_pid is not alive, removing stale lock"); _debug(__LINE__, "pve_mod_worker process $clean_pid is not alive, removing stale lock");
unlink($monitor_lock); unlink($pve_mod_worker_lock);
} }
} else { } else {
# Stale lock, remove it # Stale lock, remove it
_debug(__LINE__, "Monitor lock is stale (PID: " . ($pid // 'undefined') . "), removing"); _debug(__LINE__, "pve_mod_worker lock is stale (PID: " . ($pid // 'undefined') . "), removing");
unlink($monitor_lock); unlink($pve_mod_worker_lock);
} }
} else { } else {
_debug(__LINE__, "Failed to open monitor lock file: $!"); _debug(__LINE__, "Failed to open pve_mod_worker lock file: $!");
} }
} }
# In monitor process:
sub _pve_mod_keep_alive { sub _pve_mod_keep_alive {
_debug(__LINE__, "Monitor process started with PID $$"); _debug(__LINE__, "pve_mod_worker process started with PID $$");
my $last_activity = time(); my $last_activity = time();
@ -790,39 +805,47 @@ sub _pve_mod_keep_alive {
_debug(__LINE__, "Activity ping received"); _debug(__LINE__, "Activity ping received");
}; };
$SIG{TERM} = sub { $SIG{TERM} = sub {
_debug(__LINE__, "Monitor received SIGTERM, shutting down"); _debug(__LINE__, "pve_mod_worker received SIGTERM, shutting down");
unlink($monitor_lock) if -f $monitor_lock; unlink($pve_mod_worker_lock) if -f $pve_mod_worker_lock;
exit(0); exit(0);
}; };
$SIG{INT} = sub { $SIG{INT} = sub {
_debug(__LINE__, "Monitor received SIGINT, shutting down"); _debug(__LINE__, "pve_mod_worker received SIGINT, shutting down");
unlink($monitor_lock) if -f $monitor_lock; unlink($pve_mod_worker_lock) if -f $pve_mod_worker_lock;
exit(0); exit(0);
}; };
_debug(__LINE__, "Entering monitor loop, timeout=${COLLECTOR_TIMEOUT}s"); _debug(__LINE__, "Entering pve_mod_worker loop, timeout=${COLLECTOR_TIMEOUT}s");
while (1) { while (1) {
_debug(__LINE__, "Monitor loop start: checking activity"); _debug(__LINE__, "pve_mod_worker loop start: checking activity");
my $idle_time = time() - $last_activity; my $idle_time = time() - $last_activity;
_debug(__LINE__, "Monitor loop: idle_time=${idle_time}s, timeout=${COLLECTOR_TIMEOUT}s"); _debug(__LINE__, "pve_mod_worker loop: idle_time=${idle_time}s, timeout=${COLLECTOR_TIMEOUT}s");
if ($idle_time > $COLLECTOR_TIMEOUT) { if ($idle_time > $COLLECTOR_TIMEOUT) {
_debug(__LINE__, "Timeout reached, stopping collectors"); _debug(__LINE__, "Timeout reached, stopping collectors");
_stop_collectors(); _stop_collectors();
_debug(__LINE__, "Collectors stopped, exiting monitor"); _debug(__LINE__, "Collectors stopped, exiting pve_mod_worker");
unlink($monitor_lock) if -f $monitor_lock; unlink($pve_mod_worker_lock) if -f $pve_mod_worker_lock;
exit(0); exit(0);
} }
sleep(1); sleep(1);
} }
# Should never reach here # Should never reach here
_debug(__LINE__, "Monitor loop exited unexpectedly!"); _debug(__LINE__, "pve_mod_worker loop exited unexpectedly!");
} }
sub _is_pve_mod_worker_running {
return -f $pve_mod_worker_lock;
}
# ============================================================================
# Other
# ============================================================================
sub _stop_collectors { sub _stop_collectors {
_debug(__LINE__, "Stopping all collectors"); _debug(__LINE__, "Stopping all collectors");
@ -838,13 +861,16 @@ sub _stop_collectors {
} }
close($lock_fh); close($lock_fh);
} }
_debug(__LINE__, "Cleanup complete_1");
if (@pids) { if (@pids) {
_debug(__LINE__, "Sending SIGTERM to " . scalar(@pids) . " collector process(es)"); _debug(__LINE__, "Sending SIGTERM to " . scalar(@pids) . " collector process(es)");
foreach my $pid (@pids) { foreach my $pid (@pids) {
kill('TERM', $pid) if kill(0, $pid); kill('TERM', $pid) if kill(0, $pid);
} }
_debug(__LINE__, "Cleanup complete_2");
# Wait up to 5 seconds for graceful shutdown # Wait up to 5 seconds for graceful shutdown
my $timeout = 5; my $timeout = 5;
my $start = time(); my $start = time();
@ -859,7 +885,7 @@ sub _stop_collectors {
last unless $any_alive; last unless $any_alive;
select(undef, undef, undef, 0.1); select(undef, undef, undef, 0.1);
} }
_debug(__LINE__, "Cleanup complete_3");
# Force kill any survivors # Force kill any survivors
foreach my $pid (@pids) { foreach my $pid (@pids) {
if (kill(0, $pid)) { if (kill(0, $pid)) {
@ -867,13 +893,23 @@ sub _stop_collectors {
kill('KILL', $pid); kill('KILL', $pid);
} }
} }
_debug(__LINE__, "Cleanup complete_4");
} }
# Clean up files if (-f $state_file) {
unlink $state_file if -f $state_file; unlink $state_file or _debug(__LINE__, "Failed to remove $state_file: $!");
unlink $lock_file if -f $lock_file; }
if (-f $lock_file) {
unlink $lock_file or _debug(__LINE__, "Failed to remove $lock_file: $!");
}
_debug(__LINE__, "Cleanup complete"); # Remove pve mod worker directory and all files if it exists
if (-d $pve_mod_working_dir) {
remove_tree($pve_mod_working_dir, { error => \my $err });
_debug(__LINE__, "Cleanup errors: @$err") if @$err;
}
_debug(__LINE__, "Cleanup complete_5");
} }
END { _stop_collectors() } END { _stop_collectors() }