refactor nvidia collection and cleanup
This commit is contained in:
Meliox
parent
e19f9174c9
commit
6fb9e1cf75
205
pve-mod.pm
205
pve-mod.pm
@ -271,8 +271,9 @@ sub _collector_for_intel_device {
|
|||||||
sub _parse_graphic_info {
|
sub _parse_graphic_info {
|
||||||
my ($line) = @_;
|
my ($line) = @_;
|
||||||
|
|
||||||
# Create an intel file with
|
# Create a RRD Database (One-Time Setup)
|
||||||
# Timestamp Device index, name, Render/3D, Blitter, Video, VideoEnhance, power consumption
|
|
||||||
|
# Collect intel GPU data and save it into the database
|
||||||
|
|
||||||
return undef;
|
return undef;
|
||||||
}
|
}
|
||||||
@ -304,7 +305,7 @@ sub _collector_for_amd_device {
|
|||||||
}
|
}
|
||||||
|
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
# NVIDIA GPU Support (Placeholder)
|
# NVIDIA GPU Support
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
|
|
||||||
sub get_nvidia_gpu_devices {
|
sub get_nvidia_gpu_devices {
|
||||||
@ -314,8 +315,6 @@ sub get_nvidia_gpu_devices {
|
|||||||
# index, name
|
# index, name
|
||||||
# 0, NVIDIA GeForce RTX 3080
|
# 0, NVIDIA GeForce RTX 3080
|
||||||
# 1, NVIDIA RTX A4000
|
# 1, NVIDIA RTX A4000
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
if ($nvidia_debug_mode && -f $nvidia_debug_devices) {
|
if ($nvidia_debug_mode && -f $nvidia_debug_devices) {
|
||||||
_debug(__LINE__, "Debug mode: reading NVIDIA GPU devices from $nvidia_debug_devices");
|
_debug(__LINE__, "Debug mode: reading NVIDIA GPU devices from $nvidia_debug_devices");
|
||||||
@ -424,6 +423,105 @@ sub parse_nvidia_gpu_line {
|
|||||||
return $stats;
|
return $stats;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
sub _get_and_write_nvidia_stats {
|
||||||
|
my ($devices) = @_;
|
||||||
|
my @all_stats;
|
||||||
|
|
||||||
|
if ($nvidia_debug_mode && -f $nvidia_debug_output) {
|
||||||
|
# Debug mode: read all GPUs from single file
|
||||||
|
_debug(__LINE__, "Debug mode: reading NVIDIA GPU stats from $nvidia_debug_output");
|
||||||
|
if (open my $fh, '<', $nvidia_debug_output) {
|
||||||
|
my $line_num = 0;
|
||||||
|
while (<$fh>) {
|
||||||
|
chomp;
|
||||||
|
$line_num++;
|
||||||
|
|
||||||
|
# Skip header and empty lines
|
||||||
|
next if $line_num == 1 || /^\s*$/;
|
||||||
|
|
||||||
|
# Parse the stats line
|
||||||
|
my $stats = parse_nvidia_gpu_line($_);
|
||||||
|
push @all_stats, $stats if $stats;
|
||||||
|
}
|
||||||
|
close $fh;
|
||||||
|
} else {
|
||||||
|
_debug(__LINE__, "Failed to open debug file $nvidia_debug_output: $!");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
# Production mode: query all GPUs at once
|
||||||
|
my $query = 'index,name,temperature.gpu,utilization.gpu,utilization.memory,memory.used,memory.total,power.draw,power.limit,fan.speed';
|
||||||
|
my $cmd = "nvidia-smi --query-gpu=$query --format=csv,nounits";
|
||||||
|
|
||||||
|
if (open my $fh, '-|', $cmd) {
|
||||||
|
my $line_num = 0;
|
||||||
|
while (<$fh>) {
|
||||||
|
chomp;
|
||||||
|
$line_num++;
|
||||||
|
|
||||||
|
# Skip header and empty lines
|
||||||
|
next if $line_num == 1 || /^\s*$/;
|
||||||
|
|
||||||
|
# Parse the stats line
|
||||||
|
my $stats = parse_nvidia_gpu_line($_);
|
||||||
|
push @all_stats, $stats if $stats;
|
||||||
|
}
|
||||||
|
close $fh;
|
||||||
|
} else {
|
||||||
|
_debug(__LINE__, "Failed to run nvidia-smi: $!");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Write each GPU's stats to its own file
|
||||||
|
foreach my $stats (@all_stats) {
|
||||||
|
my $device_index = $stats->{index};
|
||||||
|
|
||||||
|
# Untaint device_index for file operations (validate it's a number)
|
||||||
|
unless ($device_index =~ /^(\d+)$/) {
|
||||||
|
_debug(__LINE__, "Invalid device index: $device_index, skipping");
|
||||||
|
next;
|
||||||
|
}
|
||||||
|
$device_index = $1; # Now untainted
|
||||||
|
|
||||||
|
my $node_name = "gpu$device_index";
|
||||||
|
my $device_state_file = "$pve_mod_working_dir/stats-nvidia$device_index.json";
|
||||||
|
|
||||||
|
# Find device name from devices array
|
||||||
|
my $device_name = $stats->{name}; # Fallback to name from stats
|
||||||
|
foreach my $dev (@$devices) {
|
||||||
|
if ($dev->{index} == $device_index) {
|
||||||
|
$device_name = $dev->{name};
|
||||||
|
last;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Build device-specific structure
|
||||||
|
my $device_data = {
|
||||||
|
$node_name => {
|
||||||
|
name => $device_name,
|
||||||
|
index => $device_index,
|
||||||
|
stats => $stats
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
# Write to device-specific file
|
||||||
|
eval {
|
||||||
|
open my $ofh, '>', $device_state_file or die "Failed to open $device_state_file: $!";
|
||||||
|
print $ofh JSON->new->pretty->encode($device_data);
|
||||||
|
close $ofh;
|
||||||
|
_debug(__LINE__, "Wrote NVIDIA GPU $device_index stats to $device_state_file");
|
||||||
|
};
|
||||||
|
if ($@) {
|
||||||
|
_debug(__LINE__, "Error writing NVIDIA stats for GPU $device_index: $@");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
unless (@all_stats) {
|
||||||
|
_debug(__LINE__, "No valid NVIDIA GPU stats collected");
|
||||||
|
}
|
||||||
|
|
||||||
|
return scalar(@all_stats);
|
||||||
|
}
|
||||||
|
|
||||||
sub _collector_for_nvidia_devices {
|
sub _collector_for_nvidia_devices {
|
||||||
my ($devices) = @_;
|
my ($devices) = @_;
|
||||||
$process_type = 'collector';
|
$process_type = 'collector';
|
||||||
@ -449,99 +547,8 @@ sub _collector_for_nvidia_devices {
|
|||||||
# 1, NVIDIA RTX A4000, 58, 45, 32, 4120, 16384, 145.50, 200.00, 55
|
# 1, NVIDIA RTX A4000, 58, 45, 32, 4120, 16384, 145.50, 200.00, 55
|
||||||
|
|
||||||
while (!$shutdown) {
|
while (!$shutdown) {
|
||||||
my @all_stats;
|
# Collect and write NVIDIA GPU stats
|
||||||
|
_get_and_write_nvidia_stats($devices);
|
||||||
if ($nvidia_debug_mode && -f $nvidia_debug_output) {
|
|
||||||
# Debug mode: read all GPUs from single file
|
|
||||||
_debug(__LINE__, "Debug mode: reading NVIDIA GPU stats from $nvidia_debug_output");
|
|
||||||
if (open my $fh, '<', $nvidia_debug_output) {
|
|
||||||
my $line_num = 0;
|
|
||||||
while (<$fh>) {
|
|
||||||
chomp;
|
|
||||||
$line_num++;
|
|
||||||
|
|
||||||
# Skip header and empty lines
|
|
||||||
next if $line_num == 1 || /^\s*$/;
|
|
||||||
|
|
||||||
# Parse the stats line
|
|
||||||
my $stats = parse_nvidia_gpu_line($_);
|
|
||||||
push @all_stats, $stats if $stats;
|
|
||||||
}
|
|
||||||
close $fh;
|
|
||||||
} else {
|
|
||||||
_debug(__LINE__, "Failed to open debug file $nvidia_debug_output: $!");
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
# Production mode: query all GPUs at once
|
|
||||||
my $query = 'index,name,temperature.gpu,utilization.gpu,utilization.memory,memory.used,memory.total,power.draw,power.limit,fan.speed';
|
|
||||||
my $cmd = "nvidia-smi --query-gpu=$query --format=csv,nounits";
|
|
||||||
|
|
||||||
if (open my $fh, '-|', $cmd) {
|
|
||||||
my $line_num = 0;
|
|
||||||
while (<$fh>) {
|
|
||||||
chomp;
|
|
||||||
$line_num++;
|
|
||||||
|
|
||||||
# Skip header and empty lines
|
|
||||||
next if $line_num == 1 || /^\s*$/;
|
|
||||||
|
|
||||||
# Parse the stats line
|
|
||||||
my $stats = parse_nvidia_gpu_line($_);
|
|
||||||
push @all_stats, $stats if $stats;
|
|
||||||
}
|
|
||||||
close $fh;
|
|
||||||
} else {
|
|
||||||
_debug(__LINE__, "Failed to run nvidia-smi: $!");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Write each GPU's stats to its own file
|
|
||||||
foreach my $stats (@all_stats) {
|
|
||||||
my $device_index = $stats->{index};
|
|
||||||
|
|
||||||
# Untaint device_index for file operations (validate it's a number)
|
|
||||||
unless ($device_index =~ /^(\d+)$/) {
|
|
||||||
_debug(__LINE__, "Invalid device index: $device_index, skipping");
|
|
||||||
next;
|
|
||||||
}
|
|
||||||
$device_index = $1; # Now untainted
|
|
||||||
|
|
||||||
my $node_name = "gpu$device_index";
|
|
||||||
my $device_state_file = "$pve_mod_working_dir/stats-nvidia$device_index.json";
|
|
||||||
|
|
||||||
# Find device name from devices array
|
|
||||||
my $device_name = $stats->{name}; # Fallback to name from stats
|
|
||||||
foreach my $dev (@$devices) {
|
|
||||||
if ($dev->{index} == $device_index) {
|
|
||||||
$device_name = $dev->{name};
|
|
||||||
last;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Build device-specific structure
|
|
||||||
my $device_data = {
|
|
||||||
$node_name => {
|
|
||||||
name => $device_name,
|
|
||||||
index => $device_index,
|
|
||||||
stats => $stats
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
# Write to device-specific file
|
|
||||||
eval {
|
|
||||||
open my $ofh, '>', $device_state_file or die "Failed to open $device_state_file: $!";
|
|
||||||
print $ofh JSON->new->pretty->encode($device_data);
|
|
||||||
close $ofh;
|
|
||||||
_debug(__LINE__, "Wrote NVIDIA GPU $device_index stats to $device_state_file");
|
|
||||||
};
|
|
||||||
if ($@) {
|
|
||||||
_debug(__LINE__, "Error writing NVIDIA stats for GPU $device_index: $@");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
unless (@all_stats) {
|
|
||||||
_debug(__LINE__, "No valid NVIDIA GPU stats collected");
|
|
||||||
}
|
|
||||||
|
|
||||||
sleep $data_pull_interval unless $shutdown;
|
sleep $data_pull_interval unless $shutdown;
|
||||||
}
|
}
|
||||||
@ -579,8 +586,6 @@ sub _start_child_collector {
|
|||||||
return $pid;
|
return $pid;
|
||||||
}
|
}
|
||||||
|
|
||||||
# Legacy PID file functions removed - worker now manages collectors directly via %collectors hash
|
|
||||||
|
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
# Temperature Sensors
|
# Temperature Sensors
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user