Switchable debugger, consistent debug messages, refactor startup process
This commit is contained in:
Meliox
parent
092d1555b1
commit
7b78e2a3ae
507
GPUcollecter.pm
507
GPUcollecter.pm
@ -8,14 +8,43 @@ use Fcntl qw(:flock);
|
||||
use Time::HiRes qw(time);
|
||||
use Fcntl qw(:flock O_CREAT O_EXCL O_WRONLY);
|
||||
|
||||
# Debug configuration - set to 0 to disable all debug output
|
||||
my $DEBUG_ENABLED = 1;
|
||||
|
||||
# Debug function showing line number and call chain
|
||||
# Usage: debug(__LINE__, "message")
|
||||
sub debug {
|
||||
return unless $DEBUG_ENABLED;
|
||||
|
||||
my ($line, $message) = @_;
|
||||
|
||||
# Get function call chain
|
||||
my @caller1 = caller(1); # who called debug()
|
||||
my @caller2 = caller(2); # parent of caller
|
||||
|
||||
my $sub1 = $caller1[3] || 'main';
|
||||
my $sub2 = $caller2[3];
|
||||
|
||||
$sub1 =~ s/.*:://; # Remove package prefix
|
||||
|
||||
if (defined $sub2) {
|
||||
$sub2 =~ s/.*:://;
|
||||
warn "[$sub2 -> $sub1:$line] $message\n";
|
||||
} else {
|
||||
# No parent caller (called from top level)
|
||||
warn "[$sub1:$line] $message\n";
|
||||
}
|
||||
}
|
||||
|
||||
my $stats_dir = '/var/run/pve-gpu';
|
||||
my $state_file = '/var/run/pve-gpu/stats.json';
|
||||
my $lock_file = '/var/run/pve-gpu/pve-gpu-collector.lock';
|
||||
my $startup_lock = $lock_file . ".startup";
|
||||
my $last_snapshot = {};
|
||||
my $last_mtime = 0;
|
||||
my $is_collector_parent = 0; # Flag to track if this process started collectors
|
||||
my $last_get_stats_time = 0; # Track when get_stats was last called
|
||||
my $COLLECTOR_TIMEOUT = 0; # Stop collectors x seconds after last get_stats call
|
||||
my $last_get_graphic_stats_time = 0; # Track when get_graphic_stats was last called
|
||||
my $COLLECTOR_TIMEOUT = 0; # Stop collectors x seconds after last get_graphic_stats call
|
||||
|
||||
my $intel_gpu_enabled = 1; # Set to 0 to disable Intel GPU support
|
||||
my $amd_gpu_enabled = 0; # Set to 1 to enable AMD GPU support (not yet implemented)
|
||||
@ -119,12 +148,12 @@ sub get_intel_gpu_devices {
|
||||
path => $path,
|
||||
drm_path => "/dev/dri/$card"
|
||||
};
|
||||
warn "DEBUG get_intel_gpu_devices: Found GPU device: $card -> $name ($path)";
|
||||
debug(__LINE__, "Found GPU device: $card -> $name ($path)");
|
||||
}
|
||||
}
|
||||
close $fh;
|
||||
} else {
|
||||
warn "DEBUG get_intel_gpu_devices: Failed to run intel_gpu_top -L: $!";
|
||||
debug(__LINE__, "Failed to run intel_gpu_top -L: $!");
|
||||
}
|
||||
|
||||
return @devices;
|
||||
@ -140,31 +169,31 @@ sub collector_for_intel_device {
|
||||
# Each device writes to its own file
|
||||
my $device_state_file = "/var/run/pve-gpu/stats-$device->{card}.json";
|
||||
|
||||
warn "DEBUG collector_for_intel_device: Collector started for device: $drm_dev, writing to $device_state_file";
|
||||
debug(__LINE__, "Collector started for device: $drm_dev, writing to $device_state_file");
|
||||
|
||||
# Set up signal handlers for graceful shutdown
|
||||
my $shutdown = 0;
|
||||
$SIG{TERM} = sub {
|
||||
warn "DEBUG collector_for_intel_device: Collector for $device->{card} received SIGTERM";
|
||||
debug(__LINE__, "Collector for $device->{card} received SIGTERM");
|
||||
$shutdown = 1;
|
||||
kill 'TERM', $intel_gpu_top_pid if defined $intel_gpu_top_pid && $intel_gpu_top_pid > 0;
|
||||
};
|
||||
$SIG{INT} = sub {
|
||||
warn "DEBUG collector_for_intel_device: Collector for $device->{card} received SIGINT";
|
||||
debug(__LINE__, "Collector for $device->{card} received SIGINT");
|
||||
$shutdown = 1;
|
||||
kill 'TERM', $intel_gpu_top_pid if defined $intel_gpu_top_pid && $intel_gpu_top_pid > 0;
|
||||
};
|
||||
|
||||
# Run intel_gpu_top once and keep reading from it
|
||||
warn "DEBUG: About to open pipe to intel_gpu_top";
|
||||
debug(__LINE__, "About to open pipe to intel_gpu_top");
|
||||
$intel_gpu_top_pid = open(my $fh, '-|', "intel_gpu_top -d $drm_dev -s 1000 -l 2>&1");
|
||||
|
||||
unless (defined $intel_gpu_top_pid && $intel_gpu_top_pid > 0) {
|
||||
warn "DEBUG collector_for_intel_device: Failed to run intel_gpu_top for $drm_dev: $!";
|
||||
debug(__LINE__, "Failed to run intel_gpu_top for $drm_dev: $!");
|
||||
exit 1;
|
||||
}
|
||||
|
||||
warn "DEBUG: Pipe opened successfully, PID=$intel_gpu_top_pid";
|
||||
debug(__LINE__, "Pipe opened successfully, PID=$intel_gpu_top_pid");
|
||||
|
||||
my $line_count = 0;
|
||||
my $node_name = "node0"; # You may want to generate this based on device index
|
||||
@ -198,17 +227,17 @@ sub collector_for_intel_device {
|
||||
open my $ofh, '>', $device_state_file or die "Failed to open $device_state_file: $!";
|
||||
print $ofh JSON->new->pretty->encode($device_data);
|
||||
close $ofh;
|
||||
warn "DEBUG: Wrote stats to $device_state_file (line #$line_count)";
|
||||
debug(__LINE__, "Wrote stats to $device_state_file (line #$line_count)");
|
||||
};
|
||||
if ($@) {
|
||||
warn "DEBUG: Error writing stats: $@";
|
||||
debug(__LINE__, "Error writing stats: $@");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
close $fh;
|
||||
warn "DEBUG collector_for_intel_device: Collector for $device->{card} shutting down";
|
||||
debug(__LINE__, "Collector for $device->{card} shutting down");
|
||||
exit 0;
|
||||
}
|
||||
|
||||
@ -219,7 +248,7 @@ sub collector_for_intel_device {
|
||||
sub get_amd_gpu_devices {
|
||||
# TODO: Implement AMD GPU detection
|
||||
# Use rocminfo or similar tools to detect AMD GPUs
|
||||
warn "DEBUG: AMD GPU support not yet implemented";
|
||||
debug(__LINE__, "AMD GPU support not yet implemented");
|
||||
return ();
|
||||
}
|
||||
|
||||
@ -227,14 +256,14 @@ sub parse_amd_gpu_line {
|
||||
my ($line) = @_;
|
||||
# TODO: Implement AMD GPU line parsing
|
||||
# Parse rocm-smi or similar output
|
||||
warn "DEBUG: AMD GPU line parsing not yet implemented";
|
||||
debug(__LINE__, "AMD GPU line parsing not yet implemented");
|
||||
return undef;
|
||||
}
|
||||
|
||||
sub collector_for_amd_device {
|
||||
my ($device) = @_;
|
||||
# TODO: Implement AMD GPU collector
|
||||
warn "DEBUG: AMD GPU collector not yet implemented";
|
||||
debug(__LINE__, "AMD GPU collector not yet implemented");
|
||||
exit 0;
|
||||
}
|
||||
|
||||
@ -245,7 +274,7 @@ sub collector_for_amd_device {
|
||||
sub get_nvidia_gpu_devices {
|
||||
# TODO: Implement NVIDIA GPU detection
|
||||
# Use nvidia-smi to detect NVIDIA GPUs
|
||||
warn "DEBUG: NVIDIA GPU support not yet implemented";
|
||||
debug(__LINE__, "NVIDIA GPU support not yet implemented");
|
||||
return ();
|
||||
}
|
||||
|
||||
@ -253,14 +282,14 @@ sub parse_nvidia_gpu_line {
|
||||
my ($line) = @_;
|
||||
# TODO: Implement NVIDIA GPU line parsing
|
||||
# Parse nvidia-smi output
|
||||
warn "DEBUG: NVIDIA GPU line parsing not yet implemented";
|
||||
debug(__LINE__, "NVIDIA GPU line parsing not yet implemented");
|
||||
return undef;
|
||||
}
|
||||
|
||||
sub collector_for_nvidia_device {
|
||||
my ($device) = @_;
|
||||
# TODO: Implement NVIDIA GPU collector
|
||||
warn "DEBUG: NVIDIA GPU collector not yet implemented";
|
||||
debug(__LINE__, "NVIDIA GPU collector not yet implemented");
|
||||
exit 0;
|
||||
}
|
||||
|
||||
@ -278,178 +307,144 @@ sub is_process_alive {
|
||||
# Main Collector
|
||||
# ============================================================================
|
||||
|
||||
sub start_collector {
|
||||
warn "DEBUG start_collector: Checking if collector is already running";
|
||||
|
||||
# Ensure directory exists
|
||||
my $run_dir = '/var/run/pve-gpu';
|
||||
unless (-d $run_dir) {
|
||||
warn "DEBUG start_collector: Creating directory $run_dir";
|
||||
mkdir($run_dir, 0755) or warn "DEBUG start_collector: Failed to create $run_dir: $!";
|
||||
}
|
||||
|
||||
# Try to acquire startup lock FIRST (prevents race conditions)
|
||||
my $startup_lock = $lock_file . ".startup";
|
||||
my $startup_fh;
|
||||
|
||||
warn "DEBUG start_collector: Trying to acquire startup lock: $startup_lock";
|
||||
|
||||
unless (sysopen($startup_fh, $startup_lock, O_CREAT|O_EXCL|O_WRONLY, 0644)) {
|
||||
# Startup lock exists - check if it's stale
|
||||
warn "DEBUG start_collector: Startup lock exists, checking if stale";
|
||||
|
||||
if (open(my $check_fh, '<', $startup_lock)) {
|
||||
my $lock_pid = <$check_fh>;
|
||||
chomp $lock_pid if defined $lock_pid;
|
||||
close($check_fh);
|
||||
|
||||
if (defined $lock_pid && $lock_pid =~ /^\d+$/) {
|
||||
warn "DEBUG start_collector: Startup lock held by PID $lock_pid";
|
||||
sub start_graphics_collectors {
|
||||
|
||||
if (is_process_alive($lock_pid)) {
|
||||
warn "DEBUG start_collector: Lock holder PID $lock_pid is still alive, waiting";
|
||||
} else {
|
||||
# Lock holder is dead, remove stale lock
|
||||
warn "DEBUG start_collector: Lock holder PID $lock_pid is dead, removing stale startup lock";
|
||||
unlink($startup_lock);
|
||||
|
||||
# Try to acquire lock again
|
||||
unless (sysopen($startup_fh, $startup_lock, O_CREAT|O_EXCL|O_WRONLY, 0644)) {
|
||||
warn "DEBUG start_collector: Failed to acquire startup lock on retry: $!";
|
||||
return;
|
||||
}
|
||||
warn "DEBUG start_collector: Acquired startup lock after removing stale lock";
|
||||
}
|
||||
} else {
|
||||
# Invalid PID in lock file, remove it
|
||||
warn "DEBUG start_collector: Invalid PID in startup lock, removing";
|
||||
unlink($startup_lock);
|
||||
|
||||
# Try to acquire lock again
|
||||
unless (sysopen($startup_fh, $startup_lock, O_CREAT|O_EXCL|O_WRONLY, 0644)) {
|
||||
warn "DEBUG start_collector: Failed to acquire startup lock on retry: $!";
|
||||
return;
|
||||
}
|
||||
warn "DEBUG start_collector: Acquired startup lock after removing invalid lock";
|
||||
}
|
||||
} else {
|
||||
warn "DEBUG start_collector: Could not read startup lock file: $!";
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
warn "DEBUG start_collector: Acquired startup lock on first try";
|
||||
if ($intel_gpu_enabled == 0 && $amd_gpu_enabled == 0 && $nvidia_gpu_enabled == 0) {
|
||||
debug(__LINE__, "No GPU types enabled, skipping collector startup");
|
||||
return;
|
||||
}
|
||||
|
||||
# We have the startup lock
|
||||
print $startup_fh "$$\n";
|
||||
close($startup_fh);
|
||||
warn "DEBUG start_collector: Wrote PID to startup lock";
|
||||
|
||||
|
||||
# NOW check if collectors are already running (while holding startup lock)
|
||||
my %existing_collectors;
|
||||
if (-f $lock_file) {
|
||||
warn "DEBUG start_collector: Lock file exists: $lock_file";
|
||||
debug(__LINE__, "Lock file exists: $lock_file");
|
||||
if (open(my $lock_fh, '<', $lock_file)) {
|
||||
warn "DEBUG start_collector: Opened lock file for reading";
|
||||
my @pids;
|
||||
debug(__LINE__, "Opened lock file for reading");
|
||||
while (my $line = <$lock_fh>) {
|
||||
chomp $line;
|
||||
push @pids, $line if $line =~ /^\d+$/;
|
||||
}
|
||||
close($lock_fh);
|
||||
|
||||
warn "DEBUG start_collector: Found " . scalar(@pids) . " PIDs in lock file: " . join(", ", @pids);
|
||||
|
||||
# Check if ANY collector is alive
|
||||
my $collector_running = 0;
|
||||
foreach my $pid (@pids) {
|
||||
warn "DEBUG start_collector: Checking PID $pid";
|
||||
|
||||
if (is_process_alive($pid)) {
|
||||
warn "DEBUG start_collector: Collector already running with PID $pid";
|
||||
$collector_running = 1;
|
||||
last;
|
||||
if ($line =~ /^(\d+)\s+(\S+)/) {
|
||||
$existing_collectors{$2} = $1;
|
||||
} elsif ($line =~ /^(\d+)$/) {
|
||||
# Backward compatibility: only PID, no card
|
||||
$existing_collectors{"unknown"} = $1;
|
||||
}
|
||||
}
|
||||
|
||||
warn "DEBUG start_collector: Finished checking PIDs, collector_running = $collector_running";
|
||||
|
||||
if ($collector_running) {
|
||||
warn "DEBUG start_collector: Releasing startup lock and returning - collector already running";
|
||||
unlink($startup_lock);
|
||||
return;
|
||||
}
|
||||
|
||||
# All PIDs are dead, clean up stale lock
|
||||
warn "DEBUG start_collector: All PIDs dead, removing stale lock file";
|
||||
unlink($lock_file);
|
||||
close($lock_fh);
|
||||
} else {
|
||||
warn "DEBUG start_collector: Failed to open lock file: $!";
|
||||
debug(__LINE__, "Failed to open lock file: $!");
|
||||
}
|
||||
} else {
|
||||
warn "DEBUG start_collector: Lock file does not exist";
|
||||
debug(__LINE__, "Lock file does not exist. Clean start");
|
||||
}
|
||||
|
||||
# If we reach here, no collectors are running - start new ones
|
||||
# Fork collector processes
|
||||
my @child_pids;
|
||||
# Generalized device collector management for future AMD/NVIDIA support
|
||||
my @all_devices;
|
||||
my @all_types;
|
||||
my @all_collectors;
|
||||
|
||||
# Intel
|
||||
if ($intel_gpu_enabled) {
|
||||
warn "DEBUG start_collector: Checking for intel_gpu_top";
|
||||
debug(__LINE__, "Checking for intel_gpu_top");
|
||||
unless (-x '/usr/bin/intel_gpu_top') {
|
||||
warn "DEBUG start_collector: intel_gpu_top not executable";
|
||||
debug(__LINE__, "intel_gpu_top not executable");
|
||||
unlink($startup_lock);
|
||||
return;
|
||||
}
|
||||
warn "DEBUG start_collector: intel_gpu_top is executable";
|
||||
|
||||
warn "DEBUG start_collector: Getting Intel GPU devices";
|
||||
debug(__LINE__, "intel_gpu_top is executable");
|
||||
debug(__LINE__, "Getting Intel GPU devices");
|
||||
my @intel_devices = get_intel_gpu_devices();
|
||||
warn "DEBUG start_collector: Got " . scalar(@intel_devices) . " Intel devices";
|
||||
|
||||
unless (@intel_devices) {
|
||||
warn "DEBUG start_collector: No Intel GPU devices found";
|
||||
debug(__LINE__, "No Intel GPU devices found");
|
||||
unlink($startup_lock);
|
||||
return;
|
||||
}
|
||||
warn "DEBUG start_collector: Found " . scalar(@intel_devices) . " Intel GPU device(s)";
|
||||
|
||||
|
||||
debug(__LINE__, "Found " . scalar(@intel_devices) . " Intel GPU device(s)");
|
||||
foreach my $device (@intel_devices) {
|
||||
warn "DEBUG start_collector: About to fork collector for Intel $device->{card}";
|
||||
my $pid = fork();
|
||||
|
||||
unless (defined $pid) {
|
||||
warn "DEBUG start_collector: fork failed: $!";
|
||||
unlink($startup_lock);
|
||||
die "fork failed: $!";
|
||||
}
|
||||
|
||||
if ($pid == 0) {
|
||||
# Child process
|
||||
warn "DEBUG start_collector: In child process for $device->{card}";
|
||||
collector_for_intel_device($device);
|
||||
# collector_for_intel_device calls exit(0)
|
||||
} else {
|
||||
warn "DEBUG start_collector: Forked child PID $pid for $device->{card}";
|
||||
push @child_pids, $pid;
|
||||
}
|
||||
push @all_devices, $device;
|
||||
push @all_types, 'intel';
|
||||
}
|
||||
}
|
||||
# AMD (future)
|
||||
if ($amd_gpu_enabled) {
|
||||
my @amd_devices = get_amd_gpu_devices();
|
||||
debug(__LINE__, "Got " . scalar(@amd_devices) . " AMD devices");
|
||||
foreach my $device (@amd_devices) {
|
||||
push @all_devices, $device;
|
||||
push @all_types, 'amd';
|
||||
}
|
||||
}
|
||||
# NVIDIA (future)
|
||||
if ($nvidia_gpu_enabled) {
|
||||
my @nvidia_devices = get_nvidia_gpu_devices();
|
||||
debug(__LINE__, "Got " . scalar(@nvidia_devices) . " NVIDIA devices");
|
||||
foreach my $device (@nvidia_devices) {
|
||||
push @all_devices, $device;
|
||||
push @all_types, 'nvidia';
|
||||
}
|
||||
}
|
||||
|
||||
warn "DEBUG start_collector: Forked " . scalar(@child_pids) . " children: " . join(", ", @child_pids);
|
||||
}
|
||||
|
||||
# Write child PIDs to lock file
|
||||
my @child_pids;
|
||||
my @child_devices;
|
||||
my @child_types;
|
||||
for (my $i = 0; $i < @all_devices; $i++) {
|
||||
my $device = $all_devices[$i];
|
||||
my $type = $all_types[$i];
|
||||
my $card = $device->{card};
|
||||
my $existing_pid = $existing_collectors{$card};
|
||||
if ($existing_pid && is_process_alive($existing_pid)) {
|
||||
debug(__LINE__, "Collector for $type $card already running with PID $existing_pid");
|
||||
push @child_pids, $existing_pid;
|
||||
push @child_devices, $device;
|
||||
push @child_types, $type;
|
||||
next;
|
||||
}
|
||||
debug(__LINE__, "About to fork collector for $type $card");
|
||||
my $pid = fork();
|
||||
unless (defined $pid) {
|
||||
debug(__LINE__, "fork failed: $!");
|
||||
unlink($startup_lock);
|
||||
die "fork failed: $!";
|
||||
}
|
||||
if ($pid == 0) {
|
||||
# Child process
|
||||
debug(__LINE__, "In child process for $type $card");
|
||||
if ($type eq 'intel') {
|
||||
collector_for_intel_device($device);
|
||||
} elsif ($type eq 'amd') {
|
||||
collector_for_amd_device($device);
|
||||
} elsif ($type eq 'nvidia') {
|
||||
collector_for_nvidia_device($device);
|
||||
} else {
|
||||
debug(__LINE__, "Unknown GPU type $type for $card");
|
||||
exit(1);
|
||||
}
|
||||
# Should not reach here
|
||||
exit(0);
|
||||
} else {
|
||||
debug(__LINE__, "Forked child PID $pid for $type $card");
|
||||
push @child_pids, $pid;
|
||||
push @child_devices, $device;
|
||||
push @child_types, $type;
|
||||
}
|
||||
}
|
||||
debug(__LINE__, "Active children: " . join(", ", @child_pids));
|
||||
|
||||
# Write child PIDs and device cards/types to lock file
|
||||
if (open(my $lock_fh, '>', $lock_file)) {
|
||||
warn "DEBUG start_collector: Opened lock file for writing";
|
||||
foreach my $pid (@child_pids) {
|
||||
print $lock_fh "$pid\n";
|
||||
debug(__LINE__, "Opened lock file for writing");
|
||||
for (my $i = 0; $i < @child_pids; $i++) {
|
||||
my $pid = $child_pids[$i];
|
||||
my $device = $child_devices[$i];
|
||||
my $type = $child_types[$i];
|
||||
my $card = $device->{card} // '';
|
||||
print $lock_fh "$pid $card $type\n";
|
||||
}
|
||||
close($lock_fh);
|
||||
warn "DEBUG start_collector: Wrote " . scalar(@child_pids) . " collector PID(s) to lock file";
|
||||
debug(__LINE__, "Wrote " . scalar(@child_pids) . " collector PID(s) to lock file");
|
||||
} else {
|
||||
warn "DEBUG start_collector: Failed to open lock file for writing: $!";
|
||||
debug(__LINE__, "Failed to open lock file for writing: $!");
|
||||
foreach my $pid (@child_pids) {
|
||||
warn "DEBUG start_collector: Killing child $pid due to lock file write failure";
|
||||
debug(__LINE__, "Killing child $pid due to lock file write failure");
|
||||
kill 'TERM', $pid;
|
||||
}
|
||||
unlink($startup_lock);
|
||||
@ -458,44 +453,36 @@ sub start_collector {
|
||||
|
||||
# Wait briefly to ensure collector is actually running
|
||||
sleep 0.1;
|
||||
|
||||
# Verify at least one child is still alive
|
||||
|
||||
# Verify at least one child is still alive (by PID only)
|
||||
my $any_alive = 0;
|
||||
foreach my $pid (@child_pids) {
|
||||
if (kill(0, $pid)) {
|
||||
for (my $i = 0; $i < @child_pids; $i++) {
|
||||
my $pid = $child_pids[$i];
|
||||
my $device = $child_devices[$i];
|
||||
my $card = $device->{card} // '';
|
||||
my $alive = kill(0, $pid);
|
||||
if ($alive) {
|
||||
$any_alive = 1;
|
||||
warn "DEBUG start_collector: Verified child PID $pid is alive";
|
||||
debug(__LINE__, "Verified child PID $pid for $card is alive");
|
||||
} else {
|
||||
warn "DEBUG start_collector: WARNING - Child PID $pid died immediately!";
|
||||
debug(__LINE__, "WARNING - Child PID $pid for $card died immediately!");
|
||||
}
|
||||
}
|
||||
|
||||
unless ($any_alive) {
|
||||
warn "DEBUG start_collector: ERROR - No children alive after fork!";
|
||||
debug(__LINE__, "ERROR - No children alive after fork!");
|
||||
}
|
||||
|
||||
# Start pve_mod_monitor process
|
||||
pve_mod_monitor();
|
||||
|
||||
|
||||
# Remove startup lock LAST
|
||||
unlink($startup_lock);
|
||||
warn "DEBUG start_collector: Released startup lock";
|
||||
|
||||
warn "DEBUG start_collector: Collectors started successfully, returning";
|
||||
}
|
||||
|
||||
sub get_stats {
|
||||
warn "DEBUG: get_stats() called";
|
||||
sub get_graphic_stats {
|
||||
debug(__LINE__, "get_graphic_stats called");
|
||||
|
||||
start_collector();
|
||||
|
||||
my $stats_dir = '/var/run/pve-gpu';
|
||||
# Start PVE Mod worker, if not already running
|
||||
pve_mod_worker();
|
||||
|
||||
# Find all device-specific stat files
|
||||
my $dh;
|
||||
unless (opendir($dh, $stats_dir)) {
|
||||
warn "DEBUG: Failed to open stats directory: $stats_dir: $!";
|
||||
debug(__LINE__, "Failed to open stats directory: $stats_dir: $!");
|
||||
return $last_snapshot;
|
||||
}
|
||||
|
||||
@ -503,11 +490,11 @@ sub get_stats {
|
||||
closedir($dh);
|
||||
|
||||
unless (@stat_files) {
|
||||
warn "DEBUG: No device stat files found in $stats_dir";
|
||||
debug(__LINE__, "No device stat files found in $stats_dir");
|
||||
return $last_snapshot;
|
||||
}
|
||||
|
||||
warn "DEBUG: Found " . scalar(@stat_files) . " device stat file(s): " . join(', ', @stat_files);
|
||||
debug(__LINE__, "Found " . scalar(@stat_files) . " device stat file(s): " . join(', ', @stat_files));
|
||||
|
||||
# Check if any files have been modified
|
||||
my $newest_mtime = 0;
|
||||
@ -522,11 +509,11 @@ sub get_stats {
|
||||
}
|
||||
|
||||
if ($newest_mtime == $last_mtime) {
|
||||
warn "DEBUG: No device files modified, returning cached snapshot";
|
||||
debug(__LINE__, "No device files modified, returning cached snapshot");
|
||||
return $last_snapshot;
|
||||
}
|
||||
|
||||
warn "DEBUG: Device files modified ($last_mtime -> $newest_mtime), reading and merging files";
|
||||
debug(__LINE__, "Device files modified ($last_mtime -> $newest_mtime), reading and merging files");
|
||||
|
||||
# Merge all device files
|
||||
my $merged = {
|
||||
@ -538,12 +525,12 @@ sub get_stats {
|
||||
foreach my $file (@stat_files) {
|
||||
my $filepath = "$stats_dir/$file";
|
||||
|
||||
warn "DEBUG: Reading device file: $filepath";
|
||||
debug(__LINE__, "Reading device file: $filepath");
|
||||
|
||||
eval {
|
||||
my $fh;
|
||||
unless (open($fh, '<', $filepath)) {
|
||||
warn "DEBUG: Failed to open $filepath: $!";
|
||||
debug(__LINE__, "Failed to open $filepath: $!");
|
||||
return;
|
||||
}
|
||||
|
||||
@ -551,71 +538,145 @@ sub get_stats {
|
||||
my $json = <$fh>;
|
||||
close($fh);
|
||||
|
||||
warn "DEBUG: Read $file, JSON length: " . length($json) . " bytes";
|
||||
debug(__LINE__, "Read $file, JSON length: " . length($json) . " bytes");
|
||||
|
||||
my $device_data = decode_json($json);
|
||||
|
||||
# Merge this device's data into the main structure
|
||||
foreach my $node_name (keys %$device_data) {
|
||||
$merged->{Graphics}->{Intel}->{$node_name} = $device_data->{$node_name};
|
||||
warn "DEBUG: Merged node '$node_name' from $file";
|
||||
debug(__LINE__, "Merged node '$node_name' from $file");
|
||||
}
|
||||
};
|
||||
if ($@) {
|
||||
warn "DEBUG: Failed to read/parse $filepath: $@";
|
||||
debug(__LINE__, "Failed to read/parse $filepath: $@");
|
||||
}
|
||||
}
|
||||
|
||||
# Update cache
|
||||
$last_snapshot = $merged;
|
||||
$last_mtime = $newest_mtime;
|
||||
$last_get_stats_time = time();
|
||||
$last_get_graphic_stats_time = time();
|
||||
|
||||
warn "DEBUG: Successfully merged " . scalar(keys %{$merged->{Graphics}->{Intel}}) . " device node(s)";
|
||||
debug(__LINE__, "Successfully merged " . scalar(keys %{$merged->{Graphics}->{Intel}}) . " device node(s)");
|
||||
|
||||
return $last_snapshot;
|
||||
}
|
||||
|
||||
sub pve_mod_monitor {
|
||||
return if $monitor_running;
|
||||
sub pve_mod_starter {
|
||||
# Try to acquire startup lock FIRST (prevents race conditions)
|
||||
my $startup_fh;
|
||||
|
||||
$monitor_pid = fork();
|
||||
debug(__LINE__, "Trying to acquire startup lock: $startup_lock");
|
||||
|
||||
unless (sysopen($startup_fh, $startup_lock, O_CREAT|O_EXCL|O_WRONLY, 0644)) {
|
||||
# Startup lock exists - check if it's stale
|
||||
debug(__LINE__, "Startup lock exists, checking if stale");
|
||||
|
||||
if (open(my $check_fh, '<', $startup_lock)) {
|
||||
my $lock_pid = <$check_fh>;
|
||||
chomp $lock_pid if defined $lock_pid;
|
||||
close($check_fh);
|
||||
|
||||
if (defined $lock_pid && $lock_pid =~ /^\d+$/) {
|
||||
debug(__LINE__, "Startup lock held by PID $lock_pid");
|
||||
|
||||
if (!defined $monitor_pid) {
|
||||
warn "ERROR: Failed to fork monitor process: $!";
|
||||
return;
|
||||
}
|
||||
|
||||
if ($monitor_pid == 0) {
|
||||
# Child process
|
||||
$0 = "pve_mod_monitor"; # Set process name for easier identification
|
||||
|
||||
while (1) {
|
||||
if(time() - $last_get_stats_time > $COLLECTOR_TIMEOUT) {
|
||||
warn "DEBUG pve_mod_monitor: No get_stats call in the last $COLLECTOR_TIMEOUT seconds, stopping collectors";
|
||||
stop_collectors();
|
||||
exit(0);
|
||||
if (is_process_alive($lock_pid)) {
|
||||
debug(__LINE__, "Lock holder PID $lock_pid is still alive, waiting");
|
||||
} else {
|
||||
# Lock holder is dead, remove stale lock
|
||||
debug(__LINE__, "Lock holder PID $lock_pid is dead, removing stale startup lock");
|
||||
unlink($startup_lock);
|
||||
|
||||
# Try to acquire lock again
|
||||
unless (sysopen($startup_fh, $startup_lock, O_CREAT|O_EXCL|O_WRONLY, 0644)) {
|
||||
debug(__LINE__, "Failed to acquire startup lock on retry: $!");
|
||||
return;
|
||||
}
|
||||
debug(__LINE__, "Acquired startup lock after removing stale lock");
|
||||
}
|
||||
} else {
|
||||
# Invalid PID in lock file, remove it
|
||||
debug(__LINE__, "Invalid PID in startup lock, removing");
|
||||
unlink($startup_lock);
|
||||
|
||||
# Try to acquire lock again
|
||||
unless (sysopen($startup_fh, $startup_lock, O_CREAT|O_EXCL|O_WRONLY, 0644)) {
|
||||
debug(__LINE__, "Failed to acquire startup lock on retry: $!");
|
||||
return;
|
||||
}
|
||||
debug(__LINE__, "Acquired startup lock after removing invalid lock");
|
||||
}
|
||||
sleep(1);
|
||||
warn "DEBUG pve_mod_monitor: pve_mod_monitor still running";
|
||||
} else {
|
||||
debug(__LINE__, "Could not read startup lock file: $!");
|
||||
return;
|
||||
}
|
||||
|
||||
exit(0); # This shouldn't be reached, but just in case
|
||||
} else {
|
||||
debug(__LINE__, "Acquired startup lock on first try");
|
||||
}
|
||||
|
||||
# Parent process
|
||||
$monitor_running = 1;
|
||||
warn "DEBUG start_monitor: Monitor process started with PID $monitor_pid";
|
||||
# We have the startup lock
|
||||
print $startup_fh "$$\n";
|
||||
close($startup_fh);
|
||||
debug(__LINE__, "Wrote PID, $$, to startup lock");
|
||||
}
|
||||
|
||||
sub pve_mod_worker {
|
||||
# Give the pid a unique name for easier identification
|
||||
$0 = "pve_mod_worker";
|
||||
|
||||
# Ensure directory exists
|
||||
my $run_dir = '/var/run/pve-gpu';
|
||||
unless (-d $run_dir) {
|
||||
debug(__LINE__, "Creating directory $run_dir");
|
||||
mkdir($run_dir, 0755) or debug(__LINE__, "Failed to create $run_dir: $!");
|
||||
}
|
||||
|
||||
# Acquire startup lock and start application
|
||||
pve_mod_starter();
|
||||
|
||||
# Start graphics collectors
|
||||
start_graphics_collectors();
|
||||
|
||||
# Start sensor collector
|
||||
# TBD
|
||||
|
||||
# Start UPS collector
|
||||
# TBD
|
||||
|
||||
# Start gui activity monitor process
|
||||
pve_mod_keep_alive();
|
||||
|
||||
# Remove startup lock LAST
|
||||
unlink($startup_lock);
|
||||
debug(__LINE__, "Released startup lock");
|
||||
|
||||
debug(__LINE__, "Collectors started successfully, returning");
|
||||
}
|
||||
|
||||
sub pve_mod_keep_alive {
|
||||
debug(__LINE__, "Monitor process started with PID $$");
|
||||
while (1) {
|
||||
if(time() - $last_get_graphic_stats_time > $COLLECTOR_TIMEOUT) {
|
||||
debug(__LINE__, "No get_graphic_stats call in the last $COLLECTOR_TIMEOUT seconds, stopping collectors");
|
||||
stop_collectors();
|
||||
exit(0);
|
||||
}
|
||||
sleep(1);
|
||||
debug(__LINE__, "pve_mod_keep_alive still running");
|
||||
}
|
||||
}
|
||||
|
||||
sub cleanup {
|
||||
unless ($is_collector_parent) {
|
||||
warn "DEBUG cleanup: This process did not start collectors, skipping cleanup";
|
||||
debug(__LINE__, "This process did not start collectors, skipping cleanup");
|
||||
return;
|
||||
}
|
||||
|
||||
warn "DEBUG cleanup: Starting cleanup (this should rarely happen)";
|
||||
debug(__LINE__, "Starting cleanup (this should rarely happen)");
|
||||
|
||||
# todo add remove of stat files
|
||||
|
||||
# DON'T cleanup automatically - collectors should keep running
|
||||
# across worker process lifecycles
|
||||
# Only cleanup if explicitly called
|
||||
@ -626,7 +687,7 @@ sub cleanup {
|
||||
|
||||
# Instead, add a manual cleanup function that can be called explicitly
|
||||
sub stop_collectors {
|
||||
warn "DEBUG stop_collectors: Stopping all collectors";
|
||||
debug(__LINE__, "Stopping all collectors");
|
||||
|
||||
# Read current PIDs from lock file
|
||||
my @pids;
|
||||
@ -640,7 +701,7 @@ sub stop_collectors {
|
||||
|
||||
if (@pids) {
|
||||
# Send SIGTERM to all collectors
|
||||
warn "DEBUG stop_collectors: Sending SIGTERM to " . scalar(@pids) . " process(es)";
|
||||
debug(__LINE__, "Sending SIGTERM to " . scalar(@pids) . " process(es)");
|
||||
foreach my $pid (@pids) {
|
||||
kill('TERM', $pid) if kill(0, $pid);
|
||||
}
|
||||
@ -663,7 +724,7 @@ sub stop_collectors {
|
||||
# Force kill any survivors
|
||||
foreach my $pid (@pids) {
|
||||
if (kill(0, $pid)) {
|
||||
warn "DEBUG stop_collectors: Force killing process $pid";
|
||||
debug(__LINE__, "Force killing process $pid");
|
||||
kill('KILL', $pid);
|
||||
}
|
||||
}
|
||||
@ -671,7 +732,7 @@ sub stop_collectors {
|
||||
|
||||
unlink $state_file if -f $state_file;
|
||||
unlink $lock_file if -f $lock_file;
|
||||
warn "DEBUG stop_collectors: Cleanup complete";
|
||||
debug(__LINE__, "Cleanup complete");
|
||||
}
|
||||
|
||||
END { cleanup() }
|
||||
|
||||
Loading…
Reference in New Issue
Block a user