OpenSource/PVE-mods
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor so that **pve_mod_worker alone spawns, tracks, and manages all collector processes**, eliminating PID files and external checks, while API calls only read the worker-managed state.

Browse Source
This commit is contained in:
Meliox 2026-01-10 22:57:02 +01:00
parent 447f352823
commit 5a8ecfaced
1 changed files with 64 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

182
GPUcollecter.pm
View File

@ -44,12 +44,13 @@ my $state_file = "$pve_mod_working_dir/stats.json";
my $sensors_state_file = "$pve_mod_working_dir/sensors.json"; my $sensors_state_file = "$pve_mod_working_dir/sensors.json";
my $ups_state_file = "$pve_mod_working_dir/ups.json"; my $ups_state_file = "$pve_mod_working_dir/ups.json";
my $collectors_pid_file = "$pve_mod_working_dir/collectors.pids"; # List of collector PIDs
my $pve_mod_worker_lock = "$pve_mod_working_dir/pve_mod_worker.lock"; # PVE Mod worker lock my $pve_mod_worker_lock = "$pve_mod_working_dir/pve_mod_worker.lock"; # PVE Mod worker lock
my $startup_lock = "$pve_mod_working_dir/startup.lock"; # Exclusive startup lock my $startup_lock = "$pve_mod_working_dir/startup.lock"; # Exclusive startup lock
my $last_snapshot = {}; my $last_snapshot = {};
my $last_mtime = 0; my $last_mtime = 0;
my $is_collector_parent = 0; # Flag to track if this process started collectors
# Collector registry - only populated in worker process
my %collectors = (); # key: device/card name, value: PID
my $last_get_graphic_stats_time = 0; # Track when get_graphic_stats was last called my $last_get_graphic_stats_time = 0; # Track when get_graphic_stats was last called
my $COLLECTOR_TIMEOUT = 10; # Stop collectors x seconds after last get_graphic_stats call my $COLLECTOR_TIMEOUT = 10; # Stop collectors x seconds after last get_graphic_stats call
my $data_pull_interval = 1; # Interval in seconds between data pulls my $data_pull_interval = 1; # Interval in seconds between data pulls
@ -463,74 +464,19 @@ sub _start_child_collector {
if ($pid == 0) { if ($pid == 0) {
# Child process # Child process
$process_type = 'collector';
_debug(__LINE__, "In child process for $collector_name"); _debug(__LINE__, "In child process for $collector_name");
$0 = "pve-mod-$collector_name"; $0 = "pve-mod-$collector_name";
$collector_sub->($device); $collector_sub->($device);
exit(0); exit(0);
} }
# Parent process # Parent process (worker only)
_debug(__LINE__, "Forked child PID $pid for $collector_name"); _debug(__LINE__, "Forked child PID $pid for $collector_name");
return $pid; return $pid;
} }
sub _write_collector_lock { # Legacy PID file functions removed - worker now manages collectors directly via %collectors hash
my ($lock_path, @collector_entries) = @_;
my $lock_fh;
unless (open $lock_fh, '>', $lock_path) {
_debug(__LINE__, "Failed to open lock file for writing: $!");
return 0;
}
foreach my $entry (@collector_entries) {
my ($pid, $name, $type) = @$entry;
# Quote the name to preserve spaces
print $lock_fh "$pid \"$name\" $type\n";
}
close($lock_fh);
_debug(__LINE__, "Wrote " . scalar(@collector_entries) . " collector entries to lock file");
return 1;
}
sub _read_collector_lock {
my ($lock_path) = @_;
my %collectors;
return %collectors unless -f $lock_path;
if (open my $lock_fh, '<', $lock_path) {
while (my $line = <$lock_fh>) {
chomp $line;
# Match: PID "name with spaces" type
if ($line =~ /^(\d+)\s+"([^"]+)"\s+(\S+)/) {
$collectors{$2} = { pid => $1, type => $3 };
}
}
close($lock_fh);
}
return %collectors;
}
sub _is_collector_running {
my ($collector_name, $lock_path) = @_;
my %collectors = _read_collector_lock($lock_path);
if (exists $collectors{$collector_name}) {
my $pid = $collectors{$collector_name}->{pid};
if (_is_process_alive($pid)) {
_debug(__LINE__, "Collector '$collector_name' already running with PID $pid");
return $pid;
} else {
_debug(__LINE__, "Collector '$collector_name' PID $pid is stale");
}
}
return undef;
}
# ============================================================================ # ============================================================================
# Temperature Sensors # Temperature Sensors
@ -1412,11 +1358,16 @@ sub _start_collector {
_debug(__LINE__, "Starting $collector_type collector: $collector_name"); _debug(__LINE__, "Starting $collector_type collector: $collector_name");
# Check if already running # Check if already running (in worker's hash)
my $existing_pid = _is_collector_running($collector_name, $collectors_pid_file); if (exists $collectors{$collector_name}) {
if ($existing_pid) { my $pid = $collectors{$collector_name};
_debug(__LINE__, "$collector_type collector '$collector_name' already running with PID $existing_pid"); if (kill(0, $pid)) {
return $existing_pid; _debug(__LINE__, "$collector_type collector '$collector_name' already running with PID $pid");
return $pid;
} else {
_debug(__LINE__, "Collector '$collector_name' PID $pid is stale, removing from registry");
delete $collectors{$collector_name};
}
} }
# Start the collector # Start the collector
@ -1427,22 +1378,9 @@ sub _start_collector {
return undef; return undef;
} }
# Read existing entries from lock file # Register in worker's hash
my @collector_entries; $collectors{$collector_name} = $pid;
my %existing = _read_collector_lock($collectors_pid_file); _debug(__LINE__, "Registered $collector_type collector '$collector_name' with PID $pid");
foreach my $name (keys %existing) {
push @collector_entries, [$existing{$name}->{pid}, $name, $existing{$name}->{type}];
}
# Add new collector entry
push @collector_entries, [$pid, $collector_name, $collector_type];
# Write updated lock file
unless (_write_collector_lock($collectors_pid_file, @collector_entries)) {
_debug(__LINE__, "Failed to update lock file, terminating $collector_type collector");
kill 'TERM', $pid;
return undef;
}
# Verify it's alive # Verify it's alive
sleep 0.1; sleep 0.1;
@ -1451,6 +1389,7 @@ sub _start_collector {
return $pid; return $pid;
} else { } else {
_debug(__LINE__, "WARNING - $collector_type collector '$collector_name' (PID $pid) died immediately!"); _debug(__LINE__, "WARNING - $collector_type collector '$collector_name' (PID $pid) died immediately!");
delete $collectors{$collector_name};
return undef; return undef;
} }
} }
@ -1603,18 +1542,7 @@ sub _pve_mod_starter {
$startup_fh->flush(); $startup_fh->flush();
_debug(__LINE__, "Wrote PID, $$, to startup lock"); _debug(__LINE__, "Wrote PID, $$, to startup lock");
# Start sensors collector first # Start pve mod worker (which will start all collectors)
_start_sensors_collector();
# Start graphics collectors
_start_graphics_collectors();
# Start UPS collector
_start_ups_collector();
_debug(__LINE__, "All collectors started");
# Start pve mod worker
_pve_mod_worker(); _pve_mod_worker();
# Remove startup lock LAST # Remove startup lock LAST
@ -1708,17 +1636,44 @@ sub _pve_mod_keep_alive {
$last_activity = time(); $last_activity = time();
_debug(__LINE__, "Activity ping received"); _debug(__LINE__, "Activity ping received");
}; };
# SIGCHLD handler to prevent zombies and clean up collector registry
$SIG{CHLD} = sub {
while ((my $pid = waitpid(-1, WNOHANG)) > 0) {
my $exit_status = $? >> 8;
_debug(__LINE__, "Child process $pid exited with status $exit_status");
# Find and remove from collector registry
foreach my $name (keys %collectors) {
if ($collectors{$name} == $pid) {
_debug(__LINE__, "Collector '$name' (PID $pid) exited, removing from registry");
delete $collectors{$name};
last;
}
}
}
};
$SIG{TERM} = sub { $SIG{TERM} = sub {
_debug(__LINE__, "pve_mod_worker received SIGTERM, shutting down"); _debug(__LINE__, "pve_mod_worker received SIGTERM, shutting down");
_stop_collectors();
unlink($pve_mod_worker_lock) if -f $pve_mod_worker_lock; unlink($pve_mod_worker_lock) if -f $pve_mod_worker_lock;
exit(0); exit(0);
}; };
$SIG{INT} = sub { $SIG{INT} = sub {
_debug(__LINE__, "pve_mod_worker received SIGINT, shutting down"); _debug(__LINE__, "pve_mod_worker received SIGINT, shutting down");
_stop_collectors();
unlink($pve_mod_worker_lock) if -f $pve_mod_worker_lock; unlink($pve_mod_worker_lock) if -f $pve_mod_worker_lock;
exit(0); exit(0);
}; };
# Worker now starts all collectors (moved from _pve_mod_starter)
_debug(__LINE__, "Worker starting all collectors");
_start_sensors_collector();
_start_graphics_collectors();
_start_ups_collector();
_debug(__LINE__, "All collectors started by worker");
_debug(__LINE__, "Entering pve_mod_worker loop, timeout=${COLLECTOR_TIMEOUT}s"); _debug(__LINE__, "Entering pve_mod_worker loop, timeout=${COLLECTOR_TIMEOUT}s");
while (1) { while (1) {
@ -1753,37 +1708,25 @@ sub _is_pve_mod_worker_running {
sub _stop_collectors { sub _stop_collectors {
_debug(__LINE__, "Stopping all collectors"); _debug(__LINE__, "Stopping all collectors");
# Read current PIDs from lock file # Get PIDs from worker's collector registry
my @pids; my @pids = values %collectors;
if (open my $lock_fh, '<', $collectors_pid_file) {
while (my $line = <$lock_fh>) {
chomp $line;
# Extract PID from "PID card type" format
if ($line =~ /^(\d+)/) {
push @pids, $1;
}
}
close($lock_fh);
}
_debug(__LINE__, "Cleanup complete_1");
if (@pids) { if (@pids) {
_debug(__LINE__, "Sending SIGTERM to " . scalar(@pids) . " collector process(es)"); _debug(__LINE__, "Sending SIGTERM to " . scalar(@pids) . " collector process(es)");
foreach my $pid (@pids) { foreach my $pid (@pids) {
kill('TERM', $pid) if kill(0, $pid); if (kill(0, $pid)) {
kill('TERM', $pid);
_debug(__LINE__, "Sent SIGTERM to collector PID $pid");
}
} }
_debug(__LINE__, "Cleanup complete_2"); # Wait up to 2 seconds for graceful shutdown
my $timeout = 2;
# Wait up to 5 seconds for graceful shutdown
my $timeout = 30;
my $start = time(); my $start = time();
while (time() - $start < $timeout) { while (time() - $start < $timeout) {
my $any_alive = 0; my $any_alive = 0;
foreach my $pid (@pids) { foreach my $pid (@pids) {
_debug(__LINE__, "Checking if collector process $pid is still alive");
if (kill(0, $pid)) { if (kill(0, $pid)) {
_debug(__LINE__, "Collector process $pid is still alive");
$any_alive = 1; $any_alive = 1;
last; last;
} }
@ -1791,7 +1734,7 @@ sub _stop_collectors {
last unless $any_alive; last unless $any_alive;
select(undef, undef, undef, 0.1); select(undef, undef, undef, 0.1);
} }
_debug(__LINE__, "Cleanup complete_3");
# Force kill any survivors # Force kill any survivors
foreach my $pid (@pids) { foreach my $pid (@pids) {
if (kill(0, $pid)) { if (kill(0, $pid)) {
@ -1799,15 +1742,16 @@ sub _stop_collectors {
kill('KILL', $pid); kill('KILL', $pid);
} }
} }
_debug(__LINE__, "Cleanup complete_4");
} }
# Clear collector registry
%collectors = ();
_debug(__LINE__, "Cleared collector registry");
# Remove state files
if (-f $state_file) { if (-f $state_file) {
unlink $state_file or _debug(__LINE__, "Failed to remove $state_file: $!"); unlink $state_file or _debug(__LINE__, "Failed to remove $state_file: $!");
} }
if (-f $collectors_pid_file) {
unlink $collectors_pid_file or _debug(__LINE__, "Failed to remove $collectors_pid_file: $!");
}
# Remove pve mod worker directory and all files if it exists # Remove pve mod worker directory and all files if it exists
if (-d $pve_mod_working_dir) { if (-d $pve_mod_working_dir) {
@ -1815,7 +1759,7 @@ sub _stop_collectors {
_debug(__LINE__, "Cleanup errors: @$err") if @$err; _debug(__LINE__, "Cleanup errors: @$err") if @$err;
} }
_debug(__LINE__, "Cleanup complete_5"); _debug(__LINE__, "Cleanup complete");
} }
END { END {