diff --git a/PveMod_SensorInfo.pm b/PveMod_SensorInfo.pm deleted file mode 100644 index 6ecd528..0000000 --- a/PveMod_SensorInfo.pm +++ /dev/null @@ -1,2121 +0,0 @@ -package PVE::API2::PVEMod_SensorInfo; - -use strict; -use warnings; -use JSON; -use POSIX qw(WNOHANG); -use Time::HiRes qw(time); -use Fcntl qw(:flock O_CREAT O_EXCL O_WRONLY); -use File::Path qw(remove_tree make_path); -use PVE::INotify; -use RRDs; - -# debug configuration - set to 0 to disable all _debug output -my $DEBUG_ENABLED = 1; -my $VERSION = '1.0'; - -# ============================================================================ -# Configuration -# ============================================================================ -my %config = ( - gpu => { - intel_enabled => 1, - amd_enabled => 0, - nvidia_enabled => 0, - }, - debug => { - nvidia_mode => 1, - nvidia_devices_file => '/tmp/nvidia-smi-devices.csv', - nvidia_output_file => '/tmp/nvidia-smi-output.csv', - sensors_mode => 0, - sensors_output_file => '/tmp/sensors-output.json', - }, - intervals => { - data_pull => 1, # seconds between data pulls - collector_timeout => 10, # stop collectors after N seconds of inactivity - }, - ups => { - enabled => 1, - device_name => 'ups@192.168.3.2', - }, - paths => { - working_dir => '/run/pveproxy/pve-mod', - }, -); - -# ============================================================================ -# Derived paths and runtime state -# ============================================================================ - -# Derived paths from configuration -my $pve_mod_working_dir = $config{paths}{working_dir}; -my $stats_dir = $pve_mod_working_dir; -my $state_file = "$pve_mod_working_dir/stats.json"; -my $sensors_state_file = "$pve_mod_working_dir/sensors.json"; -my $ups_state_file = "$pve_mod_working_dir/ups.json"; -my $pve_mod_worker_lock = "$pve_mod_working_dir/pve_mod_worker.lock"; -my $startup_lock = "$pve_mod_working_dir/startup.lock"; -my $RRD_SOCKET = '/var/run/rrdcached.sock'; -my $RRD_BASE = '/var/lib/rrdcached/db/pve-mod-gpu'; - -# Runtime state variables -my $process_type = 'main'; # 'main', 'worker', or 'collector' -my $last_snapshot = {}; -my $last_mtime = 0; -my $last_get_graphic_stats_time = 0; -my $pve_mod_worker_pid; -my $pve_mod_worker_running = 0; - -# Collector registry - only populated in worker process -my %collectors = (); # key: device/card name, value: PID - -# ============================================================================ -# Shared Utility Functions -# ============================================================================ - -# debug function showing line number and call chain -# Usage: _debug(__LINE__, "message") -sub _debug { - return unless $DEBUG_ENABLED; - - my ($line, $message) = @_; - - # Get function call chain - my @caller1 = caller(1); # who called _debug() - my @caller2 = caller(2); # parent of caller - - my $sub1 = $caller1[3] || 'main'; - my $sub2 = $caller2[3]; - - $sub1 =~ s/.*:://; # Remove package prefix - - if (defined $sub2) { - $sub2 =~ s/.*:://; - warn "[$sub2 -> $sub1:$line] $message\n"; - } else { - # No parent caller (called from top level) - warn "[$sub1:$line] $message\n"; - } -} - -sub read_sysfs { - my ($path) = @_; - - return "unknown" unless defined $path && -f $path; - - if (open my $fh, '<', $path) { - my $value = <$fh>; - close $fh; - - if (defined $value) { - chomp $value; - # Remove leading/trailing whitespace - $value =~ s/^\s+|\s+$//g; - return $value ne '' ? $value : "unknown"; - } - } - - return "unknown"; -} - -sub _is_process_alive { - my ($pid) = @_; - return -d "/proc/$pid"; -} - -sub _read_lock_pid { - my ($lock_path) = @_; - - return undef unless open(my $fh, '<', $lock_path); - - my $pid = <$fh>; - close($fh); - chomp $pid if defined $pid; - - return $pid; -} - -sub _acquire_exclusive_lock { - my ($lock_path, $purpose) = @_; - $purpose //= 'lock'; - - my $fh; - - # Try to create lock file exclusively - if (sysopen($fh, $lock_path, O_CREAT|O_EXCL|O_WRONLY, 0644)) { - _debug(__LINE__, "Acquired $purpose on first try"); - return $fh; - } - - # Lock file creation failed - check if it's stale or held by another process - _debug(__LINE__, ucfirst($purpose) . " exists, checking if stale"); - - my $lock_pid = _read_lock_pid($lock_path); - - if (!defined $lock_pid) { - _debug(__LINE__, "Could not read $purpose file: $!"); - return undef; - } - - if ($lock_pid eq '' || $lock_pid !~ /^\d+$/) { - _debug(__LINE__, "Invalid PID in $purpose: '" . ($lock_pid // 'undefined') . "', removing"); - unlink($lock_path); - } elsif (_is_process_alive($lock_pid)) { - _debug(__LINE__, ucfirst($purpose) . " holder PID $lock_pid is still alive"); - return undef; - } else { - _debug(__LINE__, ucfirst($purpose) . " holder PID $lock_pid is dead, removing stale lock"); - unlink($lock_path); - } - - # Try to acquire lock again after cleanup - unless (sysopen($fh, $lock_path, O_CREAT|O_EXCL|O_WRONLY, 0644)) { - _debug(__LINE__, "Failed to acquire $purpose on retry: $!"); - return undef; - } - - _debug(__LINE__, "Acquired $purpose after removing stale lock"); - return $fh; -} - -sub _ensure_pve_mod_directory_exists { - unless (-d $pve_mod_working_dir) { - _debug(__LINE__, "Creating directory $pve_mod_working_dir"); - unless (mkdir($pve_mod_working_dir, 0755)) { - _debug(__LINE__, "Failed to create $pve_mod_working_dir: $!. PVE Mod cannot start."); - die "Failed to create $pve_mod_working_dir: $!"; - } - _debug(__LINE__, "Directory $pve_mod_working_dir created"); - } else { - _debug(__LINE__, "Directory $pve_mod_working_dir already exists"); - } -} - -# Generic function to check if required executable exists -# Returns 1 if executable exists or debug mode is enabled for that type -# Returns 0 if executable doesn't exist and debug mode is not enabled -sub _check_executable { - my ($exec_path, $type, $debug_mode_enabled, $debug_file) = @_; - - # If debug mode is enabled for this type, check if debug file exists instead - if (defined $debug_mode_enabled && $debug_mode_enabled) { - if (defined $debug_file && -f $debug_file) { - _debug(__LINE__, "Debug mode enabled for $type, using debug file: $debug_file"); - return 1; - } elsif (defined $debug_file) { - _debug(__LINE__, "Debug mode enabled for $type but debug file missing: $debug_file"); - return 0; - } else { - _debug(__LINE__, "Debug mode enabled for $type, skipping executable check for $exec_path"); - return 1; - } - } - - # Normal mode: check if executable exists - unless (-x $exec_path) { - _debug(__LINE__, "$type executable not found or not executable: $exec_path"); - return 0; - } - - _debug(__LINE__, "$type executable found: $exec_path"); - return 1; -} - -sub _pve_mod_hello { - _debug(__LINE__, "PVE Mod is being started. Version $VERSION"); -} - -# Setup common signal handlers for collector processes -sub _setup_collector_signals { - my ($name, $shutdown_ref, $extra_cleanup) = @_; - - $SIG{TERM} = sub { - _debug(__LINE__, "Collector $name received SIGTERM"); - $$shutdown_ref = 1; - $extra_cleanup->() if $extra_cleanup; - }; - $SIG{INT} = sub { - _debug(__LINE__, "Collector $name received SIGINT"); - $$shutdown_ref = 1; - $extra_cleanup->() if $extra_cleanup; - }; -} - -# Safe JSON file write with error handling -sub _safe_write_json { - my ($filepath, $data, $pretty) = @_; - $pretty //= 1; - - eval { - open my $fh, '>', $filepath or die "Failed to open $filepath: $!"; - my $json = $pretty ? JSON->new->pretty->encode($data) : encode_json($data); - print $fh $json; - close $fh; - _debug(__LINE__, "Wrote JSON to $filepath"); - }; - if ($@) { - _debug(__LINE__, "Error writing to $filepath: $@"); - return 0; - } - return 1; -} - -# Safe JSON file read with error handling -sub _safe_read_json { - my ($filepath, $as_string) = @_; - - return unless -f $filepath; - - my $result; - eval { - open my $fh, '<', $filepath or die "Failed to open $filepath: $!"; - local $/; - my $json = <$fh>; - close $fh; - - if ($as_string) { - $result = $json; - } else { - $result = decode_json($json); - } - _debug(__LINE__, "Read JSON from $filepath"); - }; - if ($@) { - _debug(__LINE__, "Error reading $filepath: $@"); - return; - } - return $result; -} - -# Parse CSV line with trimming -sub _parse_csv_line { - my ($line, $expected_fields) = @_; - - return unless $line; - $line =~ s/^\s+|\s+$//g; - - my @values = map { s/^\s+|\s+$//gr } split(/,/, $line); - - return unless !$expected_fields || @values >= $expected_fields; - return @values; -} - -# Enhance sensors data with cached lookups (unified for drives and CPUs) -sub _enhance_sensors_with_cache { - my ($sensors_output, $cache_ref, $pattern, $lookup_sub, $field_names) = @_; - - $cache_ref //= {}; - - my $sensors_data = _safe_read_json(\$sensors_output); - return $sensors_output unless $sensors_data; - - # For string input, parse it - unless (ref $sensors_data eq 'HASH') { - eval { $sensors_data = decode_json($sensors_output); }; - return $sensors_output if $@; - } - - my @entries = grep { /$pattern/ } keys %{$sensors_data}; - _debug(__LINE__, "Found " . scalar(@entries) . " entries matching pattern"); - - foreach my $entry (@entries) { - my $metadata; - - # Check cache first - if (exists $cache_ref->{$entry}) { - $metadata = $cache_ref->{$entry}; - _debug(__LINE__, "Using cached info for $entry"); - } else { - # Lookup information - $metadata = $lookup_sub->($entry); - $cache_ref->{$entry} = $metadata if $metadata; - } - - # Add metadata to sensors data - if ($metadata && exists $sensors_data->{$entry}) { - foreach my $key (keys %$metadata) { - $sensors_data->{$entry}->{$key} = $metadata->{$key}; - } - _debug(__LINE__, "Enhanced $entry with metadata"); - } - } - - return JSON->new->pretty->canonical->encode($sensors_data); -} - -# ============================================================================ -# Intel GPU Support -# ============================================================================ - -# Parse Intel GPU line output format -sub _parse_intel_gpu_line { - my ($line) = @_; - - # Expected format (with aligned columns): - # Freq MHz IRQ RC6 Power W RCS BCS VCS VECS - # req act /s % gpu pkg % se wa % se wa % se wa % se wa - # 0 0 0 0 0.00 7.47 0.00 0 0 0.00 0 0 0.00 0 0 0.00 0 0 - - # Remove leading/trailing whitespace - $line =~ s/^\s+|\s+$//g; - - # Split by whitespace and filter empty values - my @values = grep { $_ ne '' } split(/\s+/, $line); - - # Expected: req(0) act(1) irq(2) rc6(3) gpu(4) pkg(5) rcs%(6) rcs_se(7) rcs_wa(8) - # bcs%(9) bcs_se(10) bcs_wa(11) vcs%(12) vcs_se(13) vcs_wa(14) vecs%(15) vecs_se(16) vecs_wa(17) - - return unless @values >= 18; - - my $stats = { - frequency => { - requested => $values[0] + 0.0, - actual => $values[1] + 0.0, - unit => "MHz" - }, - interrupts => { - count => $values[2] + 0.0, - unit => "irq/s" - }, - rc6 => { - value => $values[3] + 0.0, - unit => "%" - }, - power => { - GPU => $values[4] + 0.0, - Package => $values[5] + 0.0, - unit => "W" - }, - engines => { - "Render/3D" => { - busy => $values[6] + 0.0, - sema => $values[7] + 0.0, - wait => $values[8] + 0.0, - unit => "%" - }, - Blitter => { - busy => $values[9] + 0.0, - sema => $values[10] + 0.0, - wait => $values[11] + 0.0, - unit => "%" - }, - Video => { - busy => $values[12] + 0.0, - sema => $values[13] + 0.0, - wait => $values[14] + 0.0, - unit => "%" - }, - VideoEnhance => { - busy => $values[15] + 0.0, - sema => $values[16] + 0.0, - wait => $values[17] + 0.0, - unit => "%" - } - }, - clients => {} - }; - - return $stats; -} - -# Get list of Intel GPU devices -sub _get_intel_gpu_devices { - my @devices = (); - - # Check if intel_gpu_top is available (debug mode doesn't apply to device listing) - return @devices unless _check_executable('/usr/bin/intel_gpu_top', 'Intel GPU'); - - _debug(__LINE__, "Getting Intel GPU devices"); - if (open my $fh, '-|', 'intel_gpu_top -L') { - while (<$fh>) { - chomp; - # Parse: "card0 Intel Alderlake_n (Gen12) pci:vendor=8086,device=46D0,card=0" - # or: "card0 Intel Alderlake_n (Gen12) pci:0000:00:02.0" - if (/^(card\d+)\s+(.+?)\s+(pci:[^\s]+)/) { - my $card = $1; - my $name = $2; - my $path = $3; - push @devices, { - card => $card, - name => $name, - path => $path, - drm_path => "/dev/dri/$card" - }; - _debug(__LINE__, "Found Intel device: $card -> $name ($path)"); - } - } - close $fh; - } else { - _debug(__LINE__, "Failed to run intel_gpu_top -L: $!"); - } - - return @devices; -} - -sub _collector_for_intel_device { - my ($device) = @_; - $process_type = 'collector'; - $0 = "collector-gpu-intel-$device->{card}"; - - my $drm_dev = "drm:/dev/dri/$device->{card}"; - my $intel_gpu_top_pid = undef; - - # Each device writes to its own file - my $device_state_file = "$pve_mod_working_dir/stats-$device->{card}.json"; - - _debug(__LINE__, "Collector started for device: $drm_dev, writing to $device_state_file"); - - # Set up signal handlers for graceful shutdown - my $shutdown = 0; - _setup_collector_signals($device->{card}, \$shutdown, sub { - kill 'TERM', $intel_gpu_top_pid if defined $intel_gpu_top_pid && $intel_gpu_top_pid > 0; - }); - - # Run intel_gpu_top once and keep reading from it - _debug(__LINE__, "About to open pipe to intel_gpu_top"); - my $intel_pull_interval = $config{intervals}{data_pull} * 1000; # in milliseconds - $intel_gpu_top_pid = open(my $fh, '-|', "intel_gpu_top -d $drm_dev -s $intel_pull_interval -l 2>&1"); - - unless (defined $intel_gpu_top_pid && $intel_gpu_top_pid > 0) { - _debug(__LINE__, "Failed to run intel_gpu_top for $drm_dev: $!"); - exit 1; - } - - _debug(__LINE__, "Pipe opened successfully, PID=$intel_gpu_top_pid"); - - my $line_count = 0; - my $node_name = "node0"; # You may want to generate this based on device index - - while (my $line = <$fh>) { - last if $shutdown; - - $line_count++; - chomp $line; - - # Skip header lines - next if $line =~ /MHz|IRQ|RC6|Power|RCS|BCS|VCS|VECS|req\s+act|^\s*$/; - - # Check if this is a data line - if ($line =~ /^\s*[\d\s\.]+$/) { - my $stats = _parse_intel_gpu_line($line); - - if ($stats) { - # Build device-specific structure (just the node, not the full Graphics/Intel hierarchy) - my $device_data = { - $node_name => { - name => $device->{name}, - device_path => $device->{path}, - drm_path => $device->{drm_path}, - stats => $stats - } - }; - - # Write to device-specific file - _safe_write_json($device_state_file, $device_data); - _update_intel_gpu_rrd($device->{card}, $stats); - } - } - } - - close $fh; - _debug(__LINE__, "Collector for $device->{card} shutting down"); - exit 0; -} - -# ============================================================================ -# RRD Support for GPU metrics -# ============================================================================ - -sub _get_nodename { - return PVE::INotify::nodename(); -} - -sub _gpu_rrd_path { - my ($card) = @_; - return "$RRD_BASE/" . _get_nodename() . "/$card"; -} - -sub _ensure_intel_gpu_rrd { - my ($card) = @_; - my $path = _gpu_rrd_path($card); - return if -f $path; - - my $dir = "$RRD_BASE/" . _get_nodename(); - make_path($dir, { mode => 0755 }) unless -d $dir; - - RRDs::create( - $path, - '--step', '1', - 'DS:freq_req:GAUGE:120:0:U', - 'DS:freq_act:GAUGE:120:0:U', - 'DS:rc6:GAUGE:120:0:100', - 'DS:power_gpu:GAUGE:120:0:U', - 'DS:power_pkg:GAUGE:120:0:U', - 'DS:render_busy:GAUGE:120:0:100', - 'DS:blitter_busy:GAUGE:120:0:100', - 'DS:video_busy:GAUGE:120:0:100', - 'DS:videnh_busy:GAUGE:120:0:100', - 'RRA:AVERAGE:0.5:1:1440', - 'RRA:AVERAGE:0.5:60:1440', - 'RRA:AVERAGE:0.5:1800:1344', - 'RRA:AVERAGE:0.5:21600:1464', - 'RRA:AVERAGE:0.5:604800:520', - 'RRA:MAX:0.5:1:1440', - 'RRA:MAX:0.5:60:1440', - 'RRA:MAX:0.5:1800:1344', - 'RRA:MAX:0.5:21600:1464', - 'RRA:MAX:0.5:604800:520', - ); - my $err = RRDs::error(); - _debug(__LINE__, "Created Intel GPU RRD $path: " . ($err // 'OK')); -} - -sub _update_intel_gpu_rrd { - my ($card, $stats) = @_; - _ensure_intel_gpu_rrd($card); - my $path = _gpu_rrd_path($card); - - my $freq_req = $stats->{frequency}{requested} // 'U'; - my $freq_act = $stats->{frequency}{actual} // 'U'; - my $rc6 = $stats->{rc6}{value} // 'U'; - my $power_gpu = $stats->{power}{GPU} // 'U'; - my $power_pkg = $stats->{power}{Package} // 'U'; - my $render_busy = $stats->{engines}{'Render/3D'}{busy} // 'U'; - my $blitter = $stats->{engines}{Blitter}{busy} // 'U'; - my $video = $stats->{engines}{Video}{busy} // 'U'; - my $videnh = $stats->{engines}{VideoEnhance}{busy} // 'U'; - - my @daemon_args = (-S $RRD_SOCKET) ? ('--daemon', "unix:$RRD_SOCKET") : (); - RRDs::update( - $path, - @daemon_args, - "N:$freq_req:$freq_act:$rc6:$power_gpu:$power_pkg:$render_busy:$blitter:$video:$videnh", - ); - my $err = RRDs::error(); - _debug(__LINE__, "RRD update intel $card: $err") if $err; -} - -sub _ensure_nvidia_gpu_rrd { - my ($index) = @_; - my $card = "nvidia$index"; - my $path = _gpu_rrd_path($card); - return if -f $path; - - my $dir = "$RRD_BASE/" . _get_nodename(); - make_path($dir, { mode => 0755 }) unless -d $dir; - - RRDs::create( - $path, - '--step', '1', - 'DS:gpu_util:GAUGE:120:0:100', - 'DS:mem_util:GAUGE:120:0:100', - 'DS:mem_used:GAUGE:120:0:U', - 'DS:mem_total:GAUGE:120:0:U', - 'DS:power_draw:GAUGE:120:0:U', - 'DS:power_limit:GAUGE:120:0:U', - 'DS:temp_gpu:GAUGE:120:0:U', - 'DS:fan_speed:GAUGE:120:0:100', - 'RRA:AVERAGE:0.5:1:1440', - 'RRA:AVERAGE:0.5:60:1440', - 'RRA:AVERAGE:0.5:1800:1344', - 'RRA:AVERAGE:0.5:21600:1464', - 'RRA:AVERAGE:0.5:604800:520', - 'RRA:MAX:0.5:1:1440', - 'RRA:MAX:0.5:60:1440', - 'RRA:MAX:0.5:1800:1344', - 'RRA:MAX:0.5:21600:1464', - 'RRA:MAX:0.5:604800:520', - ); - my $err = RRDs::error(); - _debug(__LINE__, "Created NVIDIA GPU RRD $path: " . ($err // 'OK')); -} - -sub _update_nvidia_gpu_rrd { - my ($index, $stats) = @_; - _ensure_nvidia_gpu_rrd($index); - my $card = "nvidia$index"; - my $path = _gpu_rrd_path($card); - - my $gpu_util = $stats->{utilization}{gpu} // 'U'; - my $mem_util = $stats->{utilization}{memory} // 'U'; - my $mem_used = $stats->{memory}{used} // 'U'; - my $mem_total = $stats->{memory}{total} // 'U'; - my $power_draw = $stats->{power}{draw} // 'U'; - my $power_limit = $stats->{power}{limit} // 'U'; - my $temp_gpu = $stats->{temperature}{gpu} // 'U'; - my $fan_speed = $stats->{fan}{speed} // 'U'; - - my @daemon_args = (-S $RRD_SOCKET) ? ('--daemon', "unix:$RRD_SOCKET") : (); - RRDs::update( - $path, - @daemon_args, - "N:$gpu_util:$mem_util:$mem_used:$mem_total:$power_draw:$power_limit:$temp_gpu:$fan_speed", - ); - my $err = RRDs::error(); - _debug(__LINE__, "RRD update nvidia$index: $err") if $err; -} - -# ============================================================================ -# AMD GPU Support (Placeholder) -# ============================================================================ - -sub _get_amd_gpu_devices { - # TODO: Implement AMD GPU detection - # Use rocminfo or similar tools to detect AMD GPUs - _debug(__LINE__, "AMD GPU support not yet implemented"); - return (); -} - -sub _parse_amd_gpu_line { - my ($line) = @_; - # TODO: Implement AMD GPU line parsing - # Parse rocm-smi or similar output - _debug(__LINE__, "AMD GPU line parsing not yet implemented"); - return undef; -} - -sub _collector_for_amd_device { - my ($device) = @_; - # TODO: Implement AMD GPU collector - _debug(__LINE__, "AMD GPU collector not yet implemented"); - exit 0; -} - -# ============================================================================ -# NVIDIA GPU Support -# ============================================================================ - -sub get_nvidia_gpu_devices { - my @devices = (); - - # Expected format (CSV with header): - # index, name - # 0, NVIDIA GeForce RTX 3080 - # 1, NVIDIA RTX A4000 - - # Check if nvidia-smi is available (or debug mode with debug file) - unless (_check_executable('/usr/bin/nvidia-smi', 'NVIDIA', $config{debug}{nvidia_mode}, $config{debug}{nvidia_devices_file})) { - return @devices; - } - - if ($config{debug}{nvidia_mode} && -f $config{debug}{nvidia_devices_file}) { - _debug(__LINE__, "Debug mode: reading NVIDIA GPU devices from $config{debug}{nvidia_devices_file}"); - if (open my $fh, '<', $config{debug}{nvidia_devices_file}) { - my $line_num = 0; - while (<$fh>) { - chomp; - $line_num++; - - # Skip header line and empty lines - next if $line_num == 1 || /^\s*$/; - - # Parse CSV using shared helper - my @values = _parse_csv_line($_, 2); - if (@values) { - push @devices, { - index => $values[0], - name => $values[1], - }; - _debug(__LINE__, "Found NVIDIA GPU device (debug): $values[1] -> (index: $values[0])"); - } - } - close $fh; - } else { - _debug(__LINE__, "Failed to open debug file $config{debug}{nvidia_devices_file}: $!"); - } - } else { - # Use nvidia-smi to get device list - if (open my $fh, '-|', 'nvidia-smi --query-gpu=index,name --format=csv') { - my $line_num = 0; - while (<$fh>) { - chomp; - $line_num++; - - # Skip header line and empty lines - next if $line_num == 1 || /^\s*$/; - - # Parse CSV using shared helper - my @values = _parse_csv_line($_, 2); - if (@values) { - push @devices, { - index => $values[0], - name => $values[1], - }; - _debug(__LINE__, "Found NVIDIA GPU device: $values[1] -> (index: $values[0])"); - } - } - close $fh; - } - } - - return @devices; -} - -sub parse_nvidia_gpu_line { - my ($line) = @_; - - # Expected format (CSV) for multiple GPUs: - # index, name, temperature.gpu, utilization.gpu, utilization.memory, memory.used, memory.total, power.draw, power.limit, fan.speed - #0, NVIDIA GeForce RTX 3080, 62, 79, 44, 8260, 10240, 268.12, 320.00, 67 - - # Parse CSV using shared helper - my @values = _parse_csv_line($line, 10); - return unless @values; - - my $stats = { - index => $values[0] + 0, - name => $values[1], - temperature => { - gpu => $values[2] + 0.0, - unit => "°C" - }, - utilization => { - gpu => $values[3] + 0.0, - memory => $values[4] + 0.0, - unit => "%" - }, - memory => { - used => $values[5] + 0.0, - total => $values[6] + 0.0, - unit => "MiB" - }, - power => { - draw => $values[7] + 0.0, - limit => $values[8] + 0.0, - unit => "W" - }, - fan => { - speed => $values[9] + 0.0, - unit => "%" - } - }; - - return $stats; -} - -sub _get_and_write_nvidia_stats { - my ($devices) = @_; - my @all_stats; - - if ($config{debug}{nvidia_mode} && -f $config{debug}{nvidia_output_file}) { - # Debug mode: read all GPUs from single file - _debug(__LINE__, "Debug mode: reading NVIDIA GPU stats from $config{debug}{nvidia_output_file}"); - if (open my $fh, '<', $config{debug}{nvidia_output_file}) { - my $line_num = 0; - while (<$fh>) { - chomp; - $line_num++; - - # Skip header and empty lines - next if $line_num == 1 || /^\s*$/; - - # Parse the stats line - my $stats = parse_nvidia_gpu_line($_); - push @all_stats, $stats if $stats; - } - close $fh; - } else { - _debug(__LINE__, "Failed to open debug file $config{debug}{nvidia_output_file}: $!"); - } - } else { - # Production mode: check if nvidia-smi is available before querying - unless (_check_executable('/usr/bin/nvidia-smi', 'NVIDIA')) { - _debug(__LINE__, "nvidia-smi not available, cannot collect stats"); - return 0; - } - - # Query all GPUs at once - my $query = 'index,name,temperature.gpu,utilization.gpu,utilization.memory,memory.used,memory.total,power.draw,power.limit,fan.speed'; - my $cmd = "nvidia-smi --query-gpu=$query --format=csv,nounits"; - - if (open my $fh, '-|', $cmd) { - my $line_num = 0; - while (<$fh>) { - chomp; - $line_num++; - - # Skip header and empty lines - next if $line_num == 1 || /^\s*$/; - - # Parse the stats line - my $stats = parse_nvidia_gpu_line($_); - push @all_stats, $stats if $stats; - } - close $fh; - } - } - - # Write each GPU's stats to its own file - foreach my $stats (@all_stats) { - my $device_index = $stats->{index}; - - # Untaint device_index for file operations (validate it's a number) - unless ($device_index =~ /^(\d+)$/) { - _debug(__LINE__, "Invalid device index: $device_index, skipping"); - next; - } - $device_index = $1; # Now untainted - - my $node_name = "gpu$device_index"; - my $device_state_file = "$pve_mod_working_dir/stats-nvidia$device_index.json"; - - # Find device name from devices array - my $device_name = $stats->{name}; # Fallback to name from stats - foreach my $dev (@$devices) { - if ($dev->{index} == $device_index) { - $device_name = $dev->{name}; - last; - } - } - - # Build device-specific structure - my $device_data = { - $node_name => { - name => $device_name, - index => $device_index, - stats => $stats - } - }; - - # Write to device-specific file - _safe_write_json($device_state_file, $device_data); - _update_nvidia_gpu_rrd($device_index, $stats); - } - - unless (@all_stats) { - _debug(__LINE__, "No valid NVIDIA GPU stats collected"); - } - - return scalar(@all_stats); -} - -sub _collector_for_nvidia_devices { - my ($devices) = @_; - $process_type = 'collector'; - - $0 = "collector-gpu-nvidia-all"; - - _debug(__LINE__, "NVIDIA collector started for " . scalar(@$devices) . " GPU(s)"); - - # Set up signal handlers for graceful shutdown - my $shutdown = 0; - _setup_collector_signals('nvidia-all', \$shutdown); - - # Expected CSV format (with header): - # index, name, temperature.gpu, utilization.gpu, utilization.memory, memory.used, memory.total, power.draw, power.limit, fan.speed - # 0, NVIDIA GeForce RTX 3080, 62, 79, 44, 8260, 10240, 268.12, 320.00, 67 - # 1, NVIDIA RTX A4000, 58, 45, 32, 4120, 16384, 145.50, 200.00, 55 - - while (!$shutdown) { - # Collect and write NVIDIA GPU stats - _get_and_write_nvidia_stats($devices); - - sleep $config{intervals}{data_pull} unless $shutdown; - } - - _debug(__LINE__, "NVIDIA collector shutting down"); - exit 0; -} - -# ============================================================================ -# Temperature Sensors -# ============================================================================ - -sub _collector_for_temperature_sensors { - my ($device) = @_; - $process_type = 'collector'; - $0 = "collector-temperature-sensors"; - - _debug(__LINE__, "Temperature sensor collector started"); - - # Check if lm-sensors is available (or debug mode with debug file) - unless (_check_executable('/usr/bin/sensors', 'lm-sensors', $config{debug}{sensors_mode}, $config{debug}{sensors_output_file})) { - _debug(__LINE__, "sensors not available and not in debug mode, exiting"); - exit(1); - } - - # Cache for drive and CPU names - my %cache_ref; - - # Set up signal handlers for graceful shutdown - my $shutdown = 0; - _setup_collector_signals('temperature-sensors', \$shutdown); - - while (!$shutdown) { - my $sensorsData = _get_temperature_sensors(\%cache_ref); - - # Write to sensors state file (as string, not parsed JSON) - eval { - open my $ofh, '>', $sensors_state_file or die "Failed to open $sensors_state_file: $!"; - print $ofh $sensorsData; - close $ofh; - _debug(__LINE__, "Wrote temperature sensor data to $sensors_state_file"); - }; - if ($@) { - _debug(__LINE__, "Error writing temperature sensor data: $@"); - } - - sleep $config{intervals}{data_pull} unless $shutdown; - } - - _debug(__LINE__, "Temperature sensor collector shutting down"); - exit 0; -} - -sub _get_temperature_sensors { - my ($cache_ref) = @_; - - my $sensorsOutput; - - # Collect sensor data from lm-sensors - if ($config{debug}{sensors_mode} && -f $config{debug}{sensors_output_file}) { - # Debug mode: read from file - _debug(__LINE__, "Debug mode: reading sensors data from $config{debug}{sensors_output_file}"); - if (open my $fh, '<', $config{debug}{sensors_output_file}) { - local $/; - $sensorsOutput = <$fh>; - close $fh; - _debug(__LINE__, "Read sensors data from debug file, length: " . length($sensorsOutput) . " bytes"); - } else { - _debug(__LINE__, "Failed to open debug file $config{debug}{sensors_output_file}: $!"); - $sensorsOutput = '{}'; - } - } else { - # Production mode: call sensors command - $sensorsOutput = `sensors -j 2>/dev/null | python3 -m json.tool`; - _debug(__LINE__, "Raw sensors output collected from command"); - } - - _debug(__LINE__, "Raw sensors output collected"); - - # sanitize output - my $sensorsData = _sanitize_sensors($sensorsOutput); - - _debug(__LINE__, "Sanitized sensors output"); - - # translate drive names (pass cache reference) - $sensorsData = _get_drive_names($sensorsData, $cache_ref); - - _debug(__LINE__, "Translated drive names in sensors output"); - - # translate CPU names (pass cache reference) - $sensorsData = _get_cpu_name($sensorsData, $cache_ref); - - _debug(__LINE__, "Translated CPU names in sensors output"); - - # Good, now add a master node called lm sensors exhanced by PVE MOD - my $sensors_json; - eval { - $sensors_json = decode_json($sensorsData); - }; - if ($@) { - _debug(__LINE__, "Failed to parse final sensors JSON: $@"); - return $sensorsData; # Return original output on parse error - } - my $enhanced_data = { - "PVE MOD lm-sensors Enhanced" => $sensors_json - }; - $sensorsData = JSON->new->pretty->encode($enhanced_data); - - - - return $sensorsData; -} - -sub _sanitize_sensors { - my ($sensorsOutput) = @_; - - # Sanitize JSON output to handle common lm-sensors parsing issues - # Replace ERROR lines with placeholder values - $sensorsOutput =~ s/ERROR:.+\s(\w+):\s(.+)/\"$1\": 0.000,/g; - $sensorsOutput =~ s/ERROR:.+\s(\w+)!/\"$1\": 0.000,/g; - - # Remove trailing commas before closing braces - $sensorsOutput =~ s/,\s*(})/$1/g; - - # Replace NaN values with null for valid JSON - $sensorsOutput =~ s/\bNaN\b/null/g; - - # Fix duplicate SODIMM keys by appending temperature sensor number - # This prevents JSON key overwrites when multiple SODIMM sensors exist - # Example: "SODIMM":{"temp3_input":34.0} becomes "SODIMM3":{"temp3_input":34.0} - $sensorsOutput =~ s/\"SODIMM\":\{\"temp(\d+)_input\"/\"SODIMM$1\":\{\"temp$1_input\"/g; - - return $sensorsOutput; -} - -sub _get_drive_names { - my ($sensorsOutput, $cache_ref) = @_; - - # Use empty hash if no cache reference provided (shouldn't happen) - $cache_ref //= {}; - - my @drive_names; - - # Parse sensors output to extract drive entries - my $sensors_data; - eval { - $sensors_data = decode_json($sensorsOutput); - }; - if ($@) { - _debug(__LINE__, "Failed to parse sensors JSON: $@"); - return $sensorsOutput; # Return original output on parse error - } - - # Extract drive entries from sensors data - my @entries = grep { - /^drivetemp-scsi-/ || /^drivetemp-nvme-/ || /^nvme-pci-/ - } keys %{$sensors_data}; - - _debug(__LINE__, "Found " . scalar(@entries) . " drive entries in sensors output"); - - foreach my $entry (@entries) { - my ($dev_path, $model, $serial) = ("unknown", "unknown", "unknown"); - - # Check cache first - if (exists $cache_ref->{$entry}) { - my $cached = $cache_ref->{$entry}; - $dev_path = $cached->{device_path}; - $model = $cached->{model}; - $serial = $cached->{serial}; - _debug(__LINE__, "Using cached drive info for $entry"); - } else { - # Lookup drive information - - # ----- SCSI/SATA ----- - if ($entry =~ /^drivetemp-scsi-(\d+)-(\d+)/) { - my ($host, $id) = ($1, $2); - my $scsi_path = "/sys/class/scsi_disk/$host:$id:0:0/device/block"; - - if (opendir(my $sdh, $scsi_path)) { - my @devs = grep { /^sd/ } readdir($sdh); - closedir($sdh); - if (@devs) { - $dev_path = "/dev/$devs[0]"; - $model = read_sysfs("/sys/class/block/$devs[0]/device/model"); - $serial = read_sysfs("/sys/class/block/$devs[0]/device/serial"); - } - } - - # ----- Numeric NVMe ----- - } elsif ($entry =~ /^drivetemp-nvme-(\d+)/) { - my $nvme_index = $1; - $dev_path = "/dev/nvme${nvme_index}n1"; - if (-e $dev_path) { - $model = read_sysfs("/sys/class/block/nvme${nvme_index}n1/device/model"); - $serial = read_sysfs("/sys/class/block/nvme${nvme_index}n1/device/serial"); - } - - # ----- PCI-style NVMe ----- - } elsif ($entry =~ /^nvme-pci-(\w+)/) { - my $pci_addr = $1; - - # Convert short PCI address to pattern - # nvme-pci-0600 -> 0000:06:00 - # Format: domain:bus:device (function is usually .0) - my $pci_pattern; - if ($pci_addr =~ /^([0-9a-f]{2})([0-9a-f]{2})$/i) { - # Short format like "0600" -> "06:00" - my ($bus, $dev) = ($1, $2); - $pci_pattern = sprintf("%04x:%02x:%02x", 0, hex($bus), hex($dev)); - _debug(__LINE__, "Converted PCI address $pci_addr to pattern $pci_pattern"); - } else { - # Already in some other format, use as-is - $pci_pattern = $pci_addr; - } - - # Try multiple approaches to find the NVMe device - my $found = 0; - - # Approach 1: Check /sys/class/nvme/ - my $nvme_dir = "/sys/class/nvme"; - _debug(__LINE__, "Searching for NVMe devices in $nvme_dir matching PCI pattern $pci_pattern"); - if (opendir(my $ndh, $nvme_dir)) { - my @nvme_devs = grep { /^nvme\d+$/ && -d "$nvme_dir/$_" } readdir($ndh); - closedir($ndh); - - _debug(__LINE__, "Found NVMe devices: " . join(", ", @nvme_devs)); - - foreach my $nvme_dev (@nvme_devs) { - # Check if this nvme device matches our PCI address - my $device_link = readlink("$nvme_dir/$nvme_dev/device"); - if ($device_link && $device_link =~ /$pci_pattern/) { - _debug(__LINE__, "NVMe device $nvme_dev matches PCI pattern $pci_pattern"); - # Found matching device - $dev_path = "/dev/$nvme_dev" . "n1"; - $model = read_sysfs("$nvme_dir/$nvme_dev/model"); - $serial = read_sysfs("$nvme_dir/$nvme_dev/serial"); - $found = 1; - _debug(__LINE__, "Found NVMe device via /sys/class/nvme: $dev_path (matched $pci_pattern)"); - last; - } - _debug(__LINE__, "NVMe device $nvme_dev did not match PCI pattern $pci_pattern"); - } - } - - # Approach 2: Try direct block device lookup if not found - if (!$found && opendir(my $bdh, "/sys/class/block")) { - my @block_devs = grep { /^nvme\d+n\d+$/ } readdir($bdh); - closedir($bdh); - - foreach my $block_dev (@block_devs) { - my $device_link = readlink("/sys/class/block/$block_dev/device"); - if ($device_link && $device_link =~ /$pci_pattern/) { - $dev_path = "/dev/$block_dev"; - # For block devices, go up to the nvme controller for model/serial - my $nvme_ctrl = $block_dev; - $nvme_ctrl =~ s/n\d+$//; # nvme0n1 -> nvme0 - $model = read_sysfs("/sys/class/nvme/$nvme_ctrl/model"); - $serial = read_sysfs("/sys/class/nvme/$nvme_ctrl/serial"); - $found = 1; - _debug(__LINE__, "Found NVMe device via /sys/class/block: $dev_path (matched $pci_pattern)"); - last; - } - } - } - - unless ($found) { - _debug(__LINE__, "Could not find device for nvme-pci-$pci_addr (pattern: $pci_pattern)"); - } - } else { - next; # unknown device type - } - - # Cache the lookup result - $cache_ref->{$entry} = { - device_path => $dev_path, - model => $model, - serial => $serial - }; - - _debug(__LINE__, "Drive: $entry -> $dev_path (Model: $model, Serial: $serial)"); - } - - # Add to result array - push @drive_names, [$entry, $dev_path, $model, $serial]; - } - - # Now enhance the sensors_data structure directly (not as string manipulation) - foreach my $drive_entry (@drive_names) { - my ($original_name, $dev_path, $model, $serial) = @$drive_entry; - - # Add metadata directly to the data structure - if (exists $sensors_data->{$original_name}) { - $sensors_data->{$original_name}->{device_path} = $dev_path; - $sensors_data->{$original_name}->{model} = $model; - $sensors_data->{$original_name}->{serial} = $serial; - _debug(__LINE__, "Enhanced $original_name with drive info"); - } - } - - # Re-encode as pretty JSON - my $enhanced_json = JSON->new->pretty->canonical->encode($sensors_data); - - return $enhanced_json; -} - -sub _get_cpu_name { - my ($sensorsOutput, $cache_ref) = @_; - - # Use empty hash if no cache reference provided - $cache_ref //= {}; - - # Parse sensors output to extract CPU entries - my $sensors_data; - eval { - $sensors_data = decode_json($sensorsOutput); - }; - if ($@) { - _debug(__LINE__, "Failed to parse sensors JSON: $@"); - return $sensorsOutput; # Return original output on parse error - } - - # Extract CPU entries from sensors data - my @entries = grep { /^coretemp-isa-/ || /^k10temp-pci-/ } keys %{$sensors_data}; - - _debug(__LINE__, "Found " . scalar(@entries) . " CPU entries in sensors output"); - - foreach my $entry (@entries) { - my ($cpu_model, $pkg) = ("unknown", "unknown"); - - # Check cache first - if (exists $cache_ref->{$entry}) { - my $cached = $cache_ref->{$entry}; - $cpu_model = $cached->{model}; - $pkg = $cached->{package}; - _debug(__LINE__, "Using cached CPU info for $entry"); - } else { - # Lookup CPU information - - # ----- Intel coretemp ----- - if ($entry =~ /^coretemp-isa-(\d+)/) { - my $isa_id = $1; - - # Find matching hwmon device - for my $hwmon (glob "/sys/class/hwmon/hwmon*") { - my $name = read_sysfs("$hwmon/name"); - next unless $name eq 'coretemp'; - - my $dev = readlink("$hwmon/device"); - next unless $dev; - - # coretemp.0 → package 0 - if ($dev =~ /\.([0-9]+)$/) { - $pkg = $1; - $cpu_model = _cpu_model_by_package($pkg); - _debug(__LINE__, "Found Intel CPU: $entry -> Package $pkg, Model: $cpu_model"); - last; - } - } - } - - # ----- AMD k10temp ----- - elsif ($entry =~ /^k10temp-pci-(\w+)/) { - my $pci_addr = $1; - - # Convert short PCI address to pattern - # k10temp-pci-00c3 -> 0000:00:18.3 - my $pci_pattern; - if ($pci_addr =~ /^([0-9a-f]{2})([0-9a-f]{2})$/i) { - # Short format like "00c3" -> "00:18" (bus:device) - my ($bus, $dev_func) = ($1, $2); - $pci_pattern = sprintf("%04x:%02x:%02x", 0, hex($bus), hex($dev_func)); - _debug(__LINE__, "Converted PCI address $pci_addr to pattern $pci_pattern"); - } - - # Find matching hwmon device - for my $hwmon (glob "/sys/class/hwmon/hwmon*") { - my $name = read_sysfs("$hwmon/name"); - next unless $name eq 'k10temp'; - - my $dev = readlink("$hwmon/device"); - next unless $dev; - - if ($dev =~ /$pci_pattern/ || $dev =~ /$pci_addr/) { - # For AMD, package/node info might be in different location - # Try to determine from PCI device or use 0 as default - $pkg = 0; - - # Attempt to find package from CPU topology - if (opendir(my $dh, "/sys/devices/system/cpu")) { - my @cpus = grep { /^cpu\d+$/ } readdir($dh); - closedir($dh); - - foreach my $cpu (@cpus) { - my $cpu_pkg = read_sysfs("/sys/devices/system/cpu/$cpu/topology/physical_package_id"); - if ($cpu_pkg ne "unknown" && $cpu_pkg =~ /^\d+$/) { - $pkg = $cpu_pkg; - last; - } - } - } - - $cpu_model = _cpu_model_by_package($pkg); - _debug(__LINE__, "Found AMD CPU: $entry -> Package $pkg, Model: $cpu_model"); - last; - } - } - } - - # Cache the lookup result - $cache_ref->{$entry} = { - model => $cpu_model, - package => $pkg - }; - - _debug(__LINE__, "CPU: $entry -> Package $pkg (Model: $cpu_model)"); - } - - # Add metadata directly to the data structure - if (exists $sensors_data->{$entry}) { - $sensors_data->{$entry}->{cpu_model} = $cpu_model; - $sensors_data->{$entry}->{cpu_package} = $pkg; - _debug(__LINE__, "Enhanced $entry with CPU info"); - } - } - - # Re-encode as pretty JSON - my $enhanced_json = JSON->new->pretty->canonical->encode($sensors_data); - - return $enhanced_json; -} - -# Helper function to get CPU model by package ID -sub _cpu_model_by_package { - my ($pkg) = @_; - - # Try to read from /proc/cpuinfo - if (open my $fh, '<', '/proc/cpuinfo') { - my $current_pkg = -1; - my $model_name = "unknown"; - - while (my $line = <$fh>) { - chomp $line; - - # Extract physical id - if ($line =~ /^physical id\s+:\s+(\d+)/) { - $current_pkg = $1; - } - - # Extract model name - if ($line =~ /^model name\s+:\s+(.+)$/) { - $model_name = $1; - $model_name =~ s/^\s+|\s+$//g; # Trim whitespace - - # If this is the package we're looking for, return it - if ($current_pkg == $pkg) { - close($fh); - return $model_name; - } - } - } - close($fh); - - # If we didn't find the specific package, return the last model found - # (single socket systems won't have physical id) - return $model_name if $model_name ne "unknown"; - } - - return "unknown"; -} - -# ============================================================================ -# UPS Support -# ============================================================================ - -sub _collector_for_ups { - my ($device) = @_; - $process_type = 'collector'; - $0 = "collector-ups-$device->{ups_name}"; - _debug(__LINE__, "UPS collector started"); - - # Set up signal handlers for graceful shutdown - my $shutdown = 0; - _setup_collector_signals("ups-$device->{ups_name}", \$shutdown); - - while (!$shutdown) { - my $upsData = _get_ups_status($device->{ups_name}); - - # Write to ups state file (as string, not parsed JSON) - eval { - open my $ofh, '>', $ups_state_file or die "Failed to open $ups_state_file: $!"; - print $ofh $upsData; - close $ofh; - _debug(__LINE__, "Wrote ups data to $ups_state_file"); - }; - if ($@) { - _debug(__LINE__, "Error writing ups data: $@"); - } - - sleep $config{intervals}{data_pull} unless $shutdown; - } - _debug(__LINE__, "UPS collector shutting down"); - exit 0; -} - -sub _get_ups_status { - my ($ups_name) = @_; - - # upsc upsname[@hostname[:port]] - _debug(__LINE__, "Collecting UPS status for $ups_name"); - - # Execute command and capture output - my $output = `/usr/bin/upsc $ups_name 2>/dev/null`; - - unless (defined $output) { - _debug(__LINE__, "Failed to execute upsc"); - return encode_json({ error => "Failed to execute upsc" }); - } - - # Check if we got any output - unless (defined $output && length($output) > 0) { - _debug(__LINE__, "No output from upsc for $ups_name"); - return encode_json({ error => "No data from UPS $ups_name" }); - } - - # Convert upsc output to nested hash structure - my $ups_data = _parse_upsc_output($output); - - # Check if we got any parsed data - unless (keys %$ups_data) { - _debug(__LINE__, "No data received from upsc for $ups_name"); - return encode_json({ error => "No data from UPS $ups_name" }); - } - - # Wrap in UPS name structure - my $result = { - $ups_name => $ups_data - }; - - # Return as pretty JSON - return JSON->new->pretty->canonical->encode($result); -} - -sub _parse_upsc_output { - my ($output) = @_; - - my $ups_data = {}; - - _debug(__LINE__, "Parsing upsc output"); - - eval { - foreach my $line (split /\n/, $output) { - # Skip empty lines and SSL init message - next if $line =~ /^\s*$/; - next if $line =~ /^Init SSL/; - - # Parse key-value pairs (format: "key: value") - if ($line =~ /^([^:]+):\s*(.*)$/) { - my ($key, $value) = ($1, $2); - - # Trim whitespace - $key =~ s/^\s+|\s+$//g; - $value =~ s/^\s+|\s+$//g; - - # Store as flat key-value pairs (no nesting) - # Convert numeric values to numbers, keep strings as strings - if ($value =~ /^-?\d+\.?\d*$/) { - $ups_data->{$key} = $value + 0; - } else { - $ups_data->{$key} = $value; - } - } - } - }; - if ($@) { - _debug(__LINE__, "Error parsing upsc output: $@"); - } - - _debug(__LINE__, "Completed parsing upsc output"); - - return $ups_data; -} - -# ============================================================================ -# API calls -# ============================================================================ - -sub get_graphic_info { - # todo name the process without overruling other processes - _debug(__LINE__, "get_graphic_stats called"); - - # Start PVE Mod - _pve_mod_starter(); - - # Find all device-specific stat files - my $dh; - unless (opendir($dh, $stats_dir)) { - _debug(__LINE__, "Failed to open stats directory: $stats_dir: $!"); - return $last_snapshot; - } - - my @stat_files = grep { /^stats-(card\d+|nvidia\d+)\.json$/ } readdir($dh); - closedir($dh); - - unless (@stat_files) { - _debug(__LINE__, "No device stat files found in $stats_dir"); - return $last_snapshot; - } - - _debug(__LINE__, "Found " . scalar(@stat_files) . " device stat file(s): " . join(', ', @stat_files)); - - # Check if any files have been modified - my $newest_mtime = 0; - my $files_changed = 0; - - foreach my $file (@stat_files) { - my $filepath = "$stats_dir/$file"; - my @stat = stat($filepath); - if (@stat && $stat[9] > $newest_mtime) { - $newest_mtime = $stat[9]; - } - } - - if ($newest_mtime == $last_mtime) { - _debug(__LINE__, "No device files modified, returning cached snapshot"); - return $last_snapshot; - } - - _debug(__LINE__, "Device files modified ($last_mtime -> $newest_mtime), reading and merging files"); - - # Merge all device files - my $merged = { - Graphics => { - Intel => {}, - NVIDIA => {} - } - }; - - foreach my $file (@stat_files) { - my $filepath = "$stats_dir/$file"; - - _debug(__LINE__, "Reading device file: $filepath"); - - eval { - my $fh; - unless (open($fh, '<', $filepath)) { - _debug(__LINE__, "Failed to open $filepath: $!"); - return; - } - - local $/; - my $json = <$fh>; - close($fh); - - _debug(__LINE__, "Read $file, JSON length: " . length($json) . " bytes"); - - my $device_data = decode_json($json); - - # Determine device type from filename and merge accordingly - my $device_type = ($file =~ /^stats-card/) ? 'Intel' : 'NVIDIA'; - - # Merge this device's data into the main structure - foreach my $node_name (keys %$device_data) { - $merged->{Graphics}->{$device_type}->{$node_name} = $device_data->{$node_name}; - _debug(__LINE__, "Merged $device_type node '$node_name' from $file"); - } - }; - if ($@) { - _debug(__LINE__, "Failed to read/parse $filepath: $@"); - } - } - - # Update cache - $last_snapshot = $merged; - $last_mtime = $newest_mtime; - $last_get_graphic_stats_time = time(); - - my $intel_count = scalar(keys %{$merged->{Graphics}->{Intel}}); - my $nvidia_count = scalar(keys %{$merged->{Graphics}->{NVIDIA}}); - _debug(__LINE__, "Successfully merged $intel_count Intel + $nvidia_count NVIDIA device node(s)"); - - # Notify pve_mod_worker of activity - _notify_pve_mod_worker(); - - return $last_snapshot; -} - -sub get_sensors_info { - _debug(__LINE__, "get_sensors_stats called"); - - # Start PVE Mod - _pve_mod_starter(); - - unless (-f $sensors_state_file) { - _debug(__LINE__, "Sensors state file does not exist: $sensors_state_file"); - return {}; - } - - my $sensors_data; - eval { - open my $fh, '<', $sensors_state_file or die "Failed to open $sensors_state_file: $!"; - local $/; - my $json = <$fh>; - close($fh); - $sensors_data = $json; - _debug(__LINE__, "Read sensors data, JSON length: " . length($json) . " bytes"); - _debug(__LINE__, "Read sensors data from $sensors_state_file"); - }; - if ($@) { - _debug(__LINE__, "Failed to read/parse sensors data: $@"); - return {}; - } - - - # Notify pve_mod_worker of activity - _notify_pve_mod_worker(); - - return $sensors_data; -} - -sub get_ups_info { - _debug(__LINE__, "get_ups_stats called"); - - # Start PVE Mod - _pve_mod_starter(); - - unless (-f $ups_state_file) { - _debug(__LINE__, "UPS state file does not exist: $ups_state_file"); - return {}; - } - - my $ups_data; - eval { - open my $fh, '<', $ups_state_file or die "Failed to open $ups_state_file: $!"; - local $/; - my $json = <$fh>; - close($fh); - $ups_data = $json; - _debug(__LINE__, "Read UPS data, JSON length: " . length($json) . " bytes"); - _debug(__LINE__, "Read UPS data from $ups_state_file"); - }; - if ($@) { - _debug(__LINE__, "Failed to read/parse UPS data: $@"); - return {}; - } - - # Notify pve_mod_worker of activity - _notify_pve_mod_worker(); - - return $ups_data; -} - -sub get_pve_mod_version { - return $VERSION; -} - -# ============================================================================ -# Main Collector -# ============================================================================ - -sub _start_collector { - my ($collector_name, $collector_type, $collector_sub, $device) = @_; - - _debug(__LINE__, "Starting $collector_type collector: $collector_name"); - - # Check if already running (in worker's hash) - if (exists $collectors{$collector_name}) { - my $pid = $collectors{$collector_name}; - if (kill(0, $pid)) { - _debug(__LINE__, "$collector_type collector '$collector_name' already running with PID $pid"); - return $pid; - } else { - _debug(__LINE__, "Collector '$collector_name' PID $pid is stale, removing from registry"); - delete $collectors{$collector_name}; - } - } - - # Start the collector - my $pid = _start_child_collector($collector_name, $collector_sub, $device); - - unless ($pid) { - _debug(__LINE__, "Failed to start $collector_type collector '$collector_name'"); - return undef; - } - - # Register in worker's hash - $collectors{$collector_name} = $pid; - _debug(__LINE__, "Registered $collector_type collector '$collector_name' with PID $pid"); - - # Verify it's alive - sleep 0.1; - if (kill(0, $pid)) { - _debug(__LINE__, "Verified $collector_type collector '$collector_name' (PID $pid) is alive"); - return $pid; - } else { - _debug(__LINE__, "WARNING - $collector_type collector '$collector_name' (PID $pid) died immediately!"); - delete $collectors{$collector_name}; - return undef; - } -} - -sub _start_graphics_collectors { - - if (!$config{gpu}{intel_enabled} && !$config{gpu}{amd_enabled} && !$config{gpu}{nvidia_enabled}) { - _debug(__LINE__, "No GPU types enabled, skipping collector startup"); - return; - } - else { - _debug(__LINE__, "Starting graphics collectors"); - } - - # Generalized device collector management for future AMD/NVIDIA support - my @all_devices; - my @all_types; - my @all_collector_subs; - - # Intel - if ($config{gpu}{intel_enabled}) { - _debug(__LINE__, "Intel GPU support enabled"); - _debug(__LINE__, "Checking for intel_gpu_top"); - - return unless _check_executable('/usr/bin/intel_gpu_top', 'Intel'); - - my @intel_devices = _get_intel_gpu_devices(); - unless (@intel_devices) { - _debug(__LINE__, "No Intel GPU devices found"); - } else { - _debug(__LINE__, "Found " . scalar(@intel_devices) . " Intel GPU device(s)"); - foreach my $device (@intel_devices) { - push @all_devices, $device; - push @all_types, 'intel'; - push @all_collector_subs, \&_collector_for_intel_device; - } - } - } - - # AMD (future) - if ($config{gpu}{amd_enabled}) { - _debug(__LINE__, "AMD GPU support enabled"); - - return unless _check_executable('/usr/bin/rocm-smi', 'AMD'); - - my @amd_devices = _get_amd_gpu_devices(); - _debug(__LINE__, "Got " . scalar(@amd_devices) . " AMD devices"); - foreach my $device (@amd_devices) { - push @all_devices, $device; - push @all_types, 'amd'; - push @all_collector_subs, \&_collector_for_amd_device; - } - } - - _debug(__LINE__, "Finished detecting devices. Total collectors to manage: " . scalar(@all_devices)); - - # Start each graphics collector using unified function (Intel/AMD only - NVIDIA handled separately) - my $started_count = 0; - - # NVIDIA - single collector for all devices - if ($config{gpu}{nvidia_enabled}) { - _debug(__LINE__, "NVIDIA GPU support enabled"); - - my @nvidia_devices = get_nvidia_gpu_devices(); - _debug(__LINE__, "Got " . scalar(@nvidia_devices) . " NVIDIA devices"); - - if (@nvidia_devices) { - # Start single collector for all NVIDIA GPUs - my $pid = _start_collector('nvidia-all', 'nvidia', \&_collector_for_nvidia_devices, \@nvidia_devices); - $started_count++ if $pid; - } - } - for (my $i = 0; $i < @all_devices; $i++) { - my $device = $all_devices[$i]; - my $type = $all_types[$i]; - my $collector_sub = $all_collector_subs[$i]; - my $device_name = $device->{card} // $device->{name} // "device$i"; - - my $pid = _start_collector($device_name, $type, $collector_sub, $device); - $started_count++ if $pid; - } - - _debug(__LINE__, "Started/verified $started_count graphics collector(s) (Intel/AMD)"); -} - -sub _start_sensors_collector { - _debug(__LINE__, "Starting temperature sensor collector"); - - # Check if sensors is available (or debug mode with debug file) - unless (_check_executable('/usr/bin/sensors', 'lm-sensors', $config{debug}{sensors_mode}, $config{debug}{sensors_output_file})) { - _debug(__LINE__, "sensors not available and not in debug mode, skipping"); - return; - } - - # Use unified collector startup - _start_collector('sensors', 'sensors', \&_collector_for_temperature_sensors, { name => 'sensors' }); -} - -sub _start_ups_collector { - - if (!$config{ups}{enabled}) { - _debug(__LINE__, "UPS support not enabled, skipping collector startup"); - return; - } - - _debug(__LINE__, "Starting UPS collector"); - - # Check if upsc is available - unless (_check_executable('/usr/bin/upsc', 'UPS')) { - _debug(__LINE__, "upsc not available, skipping UPS collector startup"); - return; - } - - # Check if UPS is configured - unless ($config{ups}{device_name}) { - _debug(__LINE__, "No UPS configured, skipping collector startup"); - return; - } - - # Use unified collector startup - _start_collector('ups', 'ups', \&_collector_for_ups, { ups_name => $config{ups}{device_name} }); -} - -# ============================================================================ -# PVE Mod Worker -# ============================================================================ - -sub _start_child_collector { - my ($collector_name, $collector_sub, $device) = @_; - - _debug(__LINE__, "Starting child collector: $collector_name"); - - my $pid = fork(); - unless (defined $pid) { - _debug(__LINE__, "fork failed for $collector_name: $!"); - return undef; - } - - if ($pid == 0) { - # Child process - $process_type = 'collector'; - _debug(__LINE__, "In child process for $collector_name"); - $0 = "collector-$collector_name"; - $collector_sub->($device); - exit(0); - } - - # Parent process (worker only) - _debug(__LINE__, "Forked child PID $pid for $collector_name"); - return $pid; -} - -sub _pve_mod_starter { - # Check if pve_mod_worker is already running - if so, entire system is already up - _debug(__LINE__, "Checking if pve_mod_worker is already running"); - if (_is_pve_mod_worker_running()) { - _debug(__LINE__, "pve_mod_worker process already running, system is already started"); - return "pve_mod_worker process already running, system is already started"; - } - _debug(__LINE__, "PVE mod worker is not running. PVE Mod will be started."); - - _pve_mod_hello(); - - # Ensure directory exists - _ensure_pve_mod_directory_exists(); - - # Try to get the lock - _debug(__LINE__, "Trying to acquire startup lock: $startup_lock"); - my $startup_fh = _acquire_exclusive_lock($startup_lock, 'startup lock'); - return unless $startup_fh; - - # SECOND CHECK (after lock) - verify nothing changed while waiting - if (_is_pve_mod_worker_running()) { - _debug(__LINE__, "Worker started by another process while we waited for lock"); - close($startup_fh); - unlink($startup_lock); - return "already running"; - } - - # Now we KNOW we're the only one starting things - print $startup_fh "$$\n"; - $startup_fh->flush(); - _debug(__LINE__, "Wrote PID, $$, to startup lock"); - - # Start pve mod worker (which will start all collectors) - _pve_mod_worker(); - - # Remove startup lock LAST - unlink($startup_lock); - _debug(__LINE__, "Released startup lock"); - - _debug(__LINE__, "pve_mod_worker started successfully, returning"); -} - -sub _pve_mod_worker { - _debug(__LINE__, "_pve_mod_worker called"); - - # Check if worker is already running - my $pve_mod_worker_fh = _acquire_exclusive_lock($pve_mod_worker_lock, 'pve_mod_worker lock'); - return unless $pve_mod_worker_fh; - print $pve_mod_worker_fh "$$\n"; - close($pve_mod_worker_fh); - - _debug(__LINE__, "Forking new pve_mod_worker process"); - my $pve_mod_worker_pid = fork(); - - unless (defined $pve_mod_worker_pid) { - _debug(__LINE__, "Failed to fork pve_mod_worker process: $!"); - return; - } - - if ($pve_mod_worker_pid == 0) { - # Child process - run the pve_mod_worker - $0 = "pve_mod_worker_controller"; - _debug(__LINE__, "Child process forked, calling _pve_mod_keep_alive"); - _pve_mod_keep_alive(); - exit(0); # Should never reach here - } else { - # Parent process - write PID to lock file - _debug(__LINE__, "Forked pve_mod_worker process with PID $pve_mod_worker_pid"); - - if (open my $fh, '>', $pve_mod_worker_lock) { - print $fh "$pve_mod_worker_pid\n"; - close $fh; - _debug(__LINE__, "Wrote pve_mod_worker PID to lock file: $pve_mod_worker_lock"); - } else { - _debug(__LINE__, "Failed to write pve_mod_worker lock file: $!"); - kill('TERM', $pve_mod_worker_pid); - } - } - _debug(__LINE__, "pve_mod_worker process started successfully"); -} - -sub _notify_pve_mod_worker { - _debug(__LINE__, "_notify_pve_mod_worker called"); - unless (-f $pve_mod_worker_lock) { - _debug(__LINE__, "pve_mod_worker lock file does not exist"); - return; - } - - _debug(__LINE__, "pve_mod_worker lock file exists, reading PID"); - if (open my $fh, '<', $pve_mod_worker_lock) { - my $pid = <$fh>; - close $fh; - chomp $pid if defined $pid; - if (defined $pid && $pid =~ /^(\d+)$/) { - # Untaint by capturing in regex - $1 is now untainted - my $clean_pid = $1; - - if (_is_process_alive($clean_pid)) { - _debug(__LINE__, "Sending USR1 signal to pve_mod_worker PID $clean_pid"); - my $result = kill('USR1', $clean_pid); - _debug(__LINE__, "Signal result: $result"); - } else { - _debug(__LINE__, "pve_mod_worker process $clean_pid is not alive, removing stale lock"); - unlink($pve_mod_worker_lock); - } - } else { - # Stale lock, remove it - _debug(__LINE__, "pve_mod_worker lock is stale (PID: " . ($pid // 'undefined') . "), removing"); - unlink($pve_mod_worker_lock); - } - } else { - _debug(__LINE__, "Failed to open pve_mod_worker lock file: $!"); - } -} - -sub _pve_mod_keep_alive { - $process_type = 'worker'; - _debug(__LINE__, "pve_mod_worker process started with PID $$"); - - my $last_activity = time(); - - # Set up signal handlers - $SIG{USR1} = sub { - $last_activity = time(); - _debug(__LINE__, "Activity ping received"); - }; - - # SIGCHLD handler to prevent zombies and clean up collector registry - $SIG{CHLD} = sub { - while ((my $pid = waitpid(-1, WNOHANG)) > 0) { - my $exit_status = $? >> 8; - _debug(__LINE__, "Child process $pid exited with status $exit_status"); - - # Find and remove from collector registry - foreach my $name (keys %collectors) { - if ($collectors{$name} == $pid) { - _debug(__LINE__, "Collector '$name' (PID $pid) exited, removing from registry"); - delete $collectors{$name}; - last; - } - } - } - }; - - $SIG{TERM} = sub { - _debug(__LINE__, "pve_mod_worker received SIGTERM, shutting down"); - _stop_child_collectors(); - unlink($pve_mod_worker_lock) if -f $pve_mod_worker_lock; - exit(0); - }; - $SIG{INT} = sub { - _debug(__LINE__, "pve_mod_worker received SIGINT, shutting down"); - _stop_child_collectors(); - unlink($pve_mod_worker_lock) if -f $pve_mod_worker_lock; - exit(0); - }; - - # Worker now starts all collectors (moved from _pve_mod_starter) - _debug(__LINE__, "Worker starting all collectors"); - _start_sensors_collector(); - _start_graphics_collectors(); - _start_ups_collector(); - _debug(__LINE__, "All collectors started by worker"); - - _debug(__LINE__, "Entering pve_mod_worker loop, timeout=$config{intervals}{collector_timeout}s"); - - while (1) { - _debug(__LINE__, "pve_mod_worker loop start: checking activity"); - - my $idle_time = time() - $last_activity; - - _debug(__LINE__, "pve_mod_worker loop: idle_time=${idle_time}s, timeout=$config{intervals}{collector_timeout}s"); - - if ($idle_time > $config{intervals}{collector_timeout}) { - _debug(__LINE__, "Timeout reached, stopping collectors"); - _stop_child_collectors(); - _debug(__LINE__, "Collectors stopped, exiting pve_mod_worker"); - unlink($pve_mod_worker_lock) if -f $pve_mod_worker_lock; - exit(0); - } - sleep(1); - } - - # Should never reach here - _debug(__LINE__, "pve_mod_worker loop exited unexpectedly!"); -} - -sub _is_pve_mod_worker_running { - return -f $pve_mod_worker_lock; -} - -sub _stop_child_collectors { - _debug(__LINE__, "Stopping all collectors"); - - # Get PIDs from worker's collector registry - my @pids = values %collectors; - - if (@pids) { - _debug(__LINE__, "Sending SIGTERM to " . scalar(@pids) . " collector process(es)"); - foreach my $pid (@pids) { - if (kill(0, $pid)) { - kill('TERM', $pid); - _debug(__LINE__, "Sent SIGTERM to collector PID $pid"); - } - } - - # Wait up to 2 seconds for graceful shutdown - my $timeout = 2; - my $start = time(); - while (time() - $start < $timeout) { - my $any_alive = 0; - foreach my $pid (@pids) { - if (kill(0, $pid)) { - $any_alive = 1; - last; - } - } - last unless $any_alive; - select(undef, undef, undef, 0.1); - } - - # Force kill any survivors - foreach my $pid (@pids) { - if (kill(0, $pid)) { - _debug(__LINE__, "Force killing collector process $pid"); - kill('KILL', $pid); - } - } - } - - # Clear collector registry - %collectors = (); - _debug(__LINE__, "Cleared collector registry"); - - # Remove state files - if (-f $state_file) { - unlink $state_file or _debug(__LINE__, "Failed to remove $state_file: $!"); - } - - # Remove pve mod worker directory and all files if it exists - if (-d $pve_mod_working_dir) { - remove_tree($pve_mod_working_dir, { error => \my $err }); - _debug(__LINE__, "Cleanup errors: @$err") if @$err; - } - - _debug(__LINE__, "Cleanup complete"); -} - -END { - if ($process_type eq 'worker') { - _debug(__LINE__, "PVE Mod Worker END block: cleaning up"); - _stop_child_collectors(); - } elsif ($process_type eq 'collector') { - _debug(__LINE__, "Collector ($0) END block: no cleanup needed"); - # Collectors just exit, no cleanup needed - } else { - _debug(__LINE__, "Main process END block: no cleanup needed"); - # Main pveproxy process doesn't cleanup - } -} - -1; diff --git a/PveSensors/PVEMod_Utils.pm b/PveSensors/PVEMod_Utils.pm new file mode 100644 index 0000000..a16d4e1 --- /dev/null +++ b/PveSensors/PVEMod_Utils.pm @@ -0,0 +1,259 @@ +package PVE::PVEMod::Utils; + +use strict; +use warnings; +use Exporter 'import'; + +use JSON; +use Fcntl qw(O_CREAT O_EXCL O_WRONLY); + +use PVE::PVEMod::Config qw($DEBUG_ENABLED $VERSION $pve_mod_working_dir); + +our @EXPORT_OK = qw( + debug + read_sysfs + is_process_alive + read_lock_pid + acquire_exclusive_lock + ensure_pve_mod_directory_exists + check_executable + startup_message + setup_collector_signals + safe_write_json + safe_read_json + parse_csv_line +); + +# ============================================================================ +# Debug +# ============================================================================ + +# debug function showing line number and call chain +# Usage: debug(__LINE__, "message") +sub debug { + return unless $DEBUG_ENABLED; + + my ($line, $message) = @_; + + my @caller1 = caller(1); # who called debug() + my @caller2 = caller(2); # parent of caller + + my $sub1 = $caller1[3] || 'main'; + my $sub2 = $caller2[3]; + + $sub1 =~ s/.*:://; + + if (defined $sub2) { + $sub2 =~ s/.*:://; + warn "[$sub2 -> $sub1:$line] $message\n"; + } else { + warn "[$sub1:$line] $message\n"; + } +} + +# ============================================================================ +# File / Process helpers +# ============================================================================ + +sub read_sysfs { + my ($path) = @_; + + return "unknown" unless defined $path && -f $path; + + if (open my $fh, '<', $path) { + my $value = <$fh>; + close $fh; + + if (defined $value) { + chomp $value; + $value =~ s/^\s+|\s+$//g; + return $value ne '' ? $value : "unknown"; + } + } + + return "unknown"; +} + +sub is_process_alive { + my ($pid) = @_; + return -d "/proc/$pid"; +} + +sub read_lock_pid { + my ($lock_path) = @_; + + return undef unless open(my $fh, '<', $lock_path); + + my $pid = <$fh>; + close($fh); + chomp $pid if defined $pid; + + return $pid; +} + +sub acquire_exclusive_lock { + my ($lock_path, $purpose) = @_; + $purpose //= 'lock'; + + my $fh; + + if (sysopen($fh, $lock_path, O_CREAT|O_EXCL|O_WRONLY, 0644)) { + debug(__LINE__, "Acquired $purpose on first try"); + return $fh; + } + + debug(__LINE__, ucfirst($purpose) . " exists, checking if stale"); + + my $lock_pid = read_lock_pid($lock_path); + + if (!defined $lock_pid) { + debug(__LINE__, "Could not read $purpose file: $!"); + return undef; + } + + if ($lock_pid eq '' || $lock_pid !~ /^\d+$/) { + debug(__LINE__, "Invalid PID in $purpose: '" . ($lock_pid // 'undefined') . "', removing"); + unlink($lock_path); + } elsif (is_process_alive($lock_pid)) { + debug(__LINE__, ucfirst($purpose) . " holder PID $lock_pid is still alive"); + return undef; + } else { + debug(__LINE__, ucfirst($purpose) . " holder PID $lock_pid is dead, removing stale lock"); + unlink($lock_path); + } + + unless (sysopen($fh, $lock_path, O_CREAT|O_EXCL|O_WRONLY, 0644)) { + debug(__LINE__, "Failed to acquire $purpose on retry: $!"); + return undef; + } + + debug(__LINE__, "Acquired $purpose after removing stale lock"); + return $fh; +} + +sub ensure_pve_mod_directory_exists { + unless (-d $pve_mod_working_dir) { + debug(__LINE__, "Creating directory $pve_mod_working_dir"); + unless (mkdir($pve_mod_working_dir, 0755)) { + debug(__LINE__, "Failed to create $pve_mod_working_dir: $!. PVE Mod cannot start."); + die "Failed to create $pve_mod_working_dir: $!"; + } + debug(__LINE__, "Directory $pve_mod_working_dir created"); + } else { + debug(__LINE__, "Directory $pve_mod_working_dir already exists"); + } +} + +# Returns 1 if executable exists, or debug mode is active with a debug file present. +# Returns 0 otherwise. +sub check_executable { + my ($exec_path, $type, $debug_mode_enabled, $debug_file) = @_; + + if (defined $debug_mode_enabled && $debug_mode_enabled) { + if (defined $debug_file && -f $debug_file) { + debug(__LINE__, "Debug mode enabled for $type, using debug file: $debug_file"); + return 1; + } elsif (defined $debug_file) { + debug(__LINE__, "Debug mode enabled for $type but debug file missing: $debug_file"); + return 0; + } else { + debug(__LINE__, "Debug mode enabled for $type, skipping executable check for $exec_path"); + return 1; + } + } + + unless (-x $exec_path) { + debug(__LINE__, "$type executable not found or not executable: $exec_path"); + return 0; + } + + debug(__LINE__, "$type executable found: $exec_path"); + return 1; +} + +sub startup_message { + debug(__LINE__, "PVE Mod is being started. Version $VERSION"); +} + +# Setup common TERM/INT signal handlers for collector processes. +# $shutdown_ref is a scalar ref that will be set to 1 on signal. +sub setup_collector_signals { + my ($name, $shutdown_ref, $extra_cleanup) = @_; + + $SIG{TERM} = sub { + debug(__LINE__, "Collector $name received SIGTERM"); + $$shutdown_ref = 1; + $extra_cleanup->() if $extra_cleanup; + }; + $SIG{INT} = sub { + debug(__LINE__, "Collector $name received SIGINT"); + $$shutdown_ref = 1; + $extra_cleanup->() if $extra_cleanup; + }; +} + +# ============================================================================ +# JSON helpers +# ============================================================================ + +sub safe_write_json { + my ($filepath, $data, $pretty) = @_; + $pretty //= 1; + + eval { + open my $fh, '>', $filepath or die "Failed to open $filepath: $!"; + my $json = $pretty ? JSON->new->pretty->encode($data) : encode_json($data); + print $fh $json; + close $fh; + debug(__LINE__, "Wrote JSON to $filepath"); + }; + if ($@) { + debug(__LINE__, "Error writing to $filepath: $@"); + return 0; + } + return 1; +} + +sub safe_read_json { + my ($filepath, $as_string) = @_; + + return unless -f $filepath; + + my $result; + eval { + open my $fh, '<', $filepath or die "Failed to open $filepath: $!"; + local $/; + my $json = <$fh>; + close $fh; + + if ($as_string) { + $result = $json; + } else { + $result = decode_json($json); + } + debug(__LINE__, "Read JSON from $filepath"); + }; + if ($@) { + debug(__LINE__, "Error reading $filepath: $@"); + return; + } + return $result; +} + +# ============================================================================ +# CSV helper +# ============================================================================ + +sub parse_csv_line { + my ($line, $expected_fields) = @_; + + return unless $line; + $line =~ s/^\s+|\s+$//g; + + my @values = map { s/^\s+|\s+$//gr } split(/,/, $line); + + return unless !$expected_fields || @values >= $expected_fields; + return @values; +} + +1; diff --git a/PveSensorsV2/PVEMod_Collectors/Amd.pm b/PveSensorsV2/PVEMod_Collectors/Amd.pm new file mode 100644 index 0000000..7cacf04 --- /dev/null +++ b/PveSensorsV2/PVEMod_Collectors/Amd.pm @@ -0,0 +1,33 @@ +package PVE::PVEMod::Collector::Amd; + +use strict; +use warnings; +use Exporter 'import'; + +use PVE::PVEMod::Config qw($process_type); +use PVE::PVEMod::Utils qw(debug); + +our @EXPORT_OK = qw( + get_amd_gpu_devices + collector_for_amd_device +); + +# ============================================================================ +# AMD GPU — placeholders (not yet implemented) +# ============================================================================ + +sub get_amd_gpu_devices { + # TODO: Implement AMD GPU detection using rocminfo or rocm-smi + debug(__LINE__, "AMD GPU support not yet implemented"); + return (); +} + +sub collector_for_amd_device { + my ($device) = @_; + $process_type = 'collector'; + # TODO: Implement AMD GPU collector + debug(__LINE__, "AMD GPU collector not yet implemented"); + exit 0; +} + +1; diff --git a/PveSensorsV2/PVEMod_Collectors/Intel.pm b/PveSensorsV2/PVEMod_Collectors/Intel.pm new file mode 100644 index 0000000..510c323 --- /dev/null +++ b/PveSensorsV2/PVEMod_Collectors/Intel.pm @@ -0,0 +1,181 @@ +package PVE::PVEMod::Collector::Intel; + +use strict; +use warnings; +use Exporter 'import'; + +use PVE::PVEMod::Config qw(%config $process_type $pve_mod_working_dir); +use PVE::PVEMod::Utils qw(debug check_executable setup_collector_signals safe_write_json); +use PVE::PVEMod::Store qw(update_intel_gpu_rrd); + +our @EXPORT_OK = qw( + get_intel_gpu_devices + collector_for_intel_device +); + +# ============================================================================ +# Intel GPU — device discovery +# ============================================================================ + +sub get_intel_gpu_devices { + my @devices = (); + + return @devices unless check_executable('/usr/bin/intel_gpu_top', 'Intel GPU'); + + debug(__LINE__, "Getting Intel GPU devices"); + if (open my $fh, '-|', 'intel_gpu_top -L') { + while (<$fh>) { + chomp; + # Parse: "card0 Intel Alderlake_n (Gen12) pci:vendor=8086,device=46D0,card=0" + # or: "card0 Intel Alderlake_n (Gen12) pci:0000:00:02.0" + if (/^(card\d+)\s+(.+?)\s+(pci:[^\s]+)/) { + my ($card, $name, $path) = ($1, $2, $3); + push @devices, { + card => $card, + name => $name, + path => $path, + drm_path => "/dev/dri/$card", + }; + debug(__LINE__, "Found Intel device: $card -> $name ($path)"); + } + } + close $fh; + } else { + debug(__LINE__, "Failed to run intel_gpu_top -L: $!"); + } + + return @devices; +} + +# ============================================================================ +# Intel GPU — data parsing +# ============================================================================ + +sub _parse_intel_gpu_line { + my ($line) = @_; + + # Expected format (whitespace-aligned columns): + # Freq MHz IRQ RC6 Power W RCS BCS VCS VECS + # req act /s % gpu pkg % se wa % se wa % se wa % se wa + # 0 0 0 0 0.00 7.47 0.00 0 0 0.00 0 0 0.00 0 0 0.00 0 0 + + $line =~ s/^\s+|\s+$//g; + my @values = grep { $_ ne '' } split(/\s+/, $line); + + return unless @values >= 18; + + return { + frequency => { + requested => $values[0] + 0.0, + actual => $values[1] + 0.0, + unit => "MHz", + }, + interrupts => { + count => $values[2] + 0.0, + unit => "irq/s", + }, + rc6 => { + value => $values[3] + 0.0, + unit => "%", + }, + power => { + GPU => $values[4] + 0.0, + Package => $values[5] + 0.0, + unit => "W", + }, + engines => { + 'Render/3D' => { + busy => $values[6] + 0.0, + sema => $values[7] + 0.0, + wait => $values[8] + 0.0, + unit => "%", + }, + Blitter => { + busy => $values[9] + 0.0, + sema => $values[10] + 0.0, + wait => $values[11] + 0.0, + unit => "%", + }, + Video => { + busy => $values[12] + 0.0, + sema => $values[13] + 0.0, + wait => $values[14] + 0.0, + unit => "%", + }, + VideoEnhance => { + busy => $values[15] + 0.0, + sema => $values[16] + 0.0, + wait => $values[17] + 0.0, + unit => "%", + }, + }, + clients => {}, + }; +} + +# ============================================================================ +# Intel GPU — long-running collector +# ============================================================================ + +sub collector_for_intel_device { + my ($device) = @_; + $process_type = 'collector'; + $0 = "collector-gpu-intel-$device->{card}"; + + my $drm_dev = "drm:/dev/dri/$device->{card}"; + my $intel_gpu_top_pid = undef; + my $device_state_file = "$pve_mod_working_dir/stats-$device->{card}.json"; + + debug(__LINE__, "Collector started for device: $drm_dev, writing to $device_state_file"); + + my $shutdown = 0; + setup_collector_signals($device->{card}, \$shutdown, sub { + kill 'TERM', $intel_gpu_top_pid + if defined $intel_gpu_top_pid && $intel_gpu_top_pid > 0; + }); + + debug(__LINE__, "About to open pipe to intel_gpu_top"); + my $intel_pull_interval = $config{intervals}{data_pull} * 1000; # milliseconds + $intel_gpu_top_pid = open(my $fh, '-|', + "intel_gpu_top -d $drm_dev -s $intel_pull_interval -l 2>&1"); + + unless (defined $intel_gpu_top_pid && $intel_gpu_top_pid > 0) { + debug(__LINE__, "Failed to run intel_gpu_top for $drm_dev: $!"); + exit 1; + } + + debug(__LINE__, "Pipe opened successfully, PID=$intel_gpu_top_pid"); + + my $node_name = "node0"; + + while (my $line = <$fh>) { + last if $shutdown; + chomp $line; + + next if $line =~ /MHz|IRQ|RC6|Power|RCS|BCS|VCS|VECS|req\s+act|^\s*$/; + + if ($line =~ /^\s*[\d\s\.]+$/) { + my $stats = _parse_intel_gpu_line($line); + + if ($stats) { + my $device_data = { + $node_name => { + name => $device->{name}, + device_path => $device->{path}, + drm_path => $device->{drm_path}, + stats => $stats, + } + }; + + safe_write_json($device_state_file, $device_data); + update_intel_gpu_rrd($device->{card}, $stats); + } + } + } + + close $fh; + debug(__LINE__, "Collector for $device->{card} shutting down"); + exit 0; +} + +1; diff --git a/PveSensorsV2/PVEMod_Collectors/LmSensors.pm b/PveSensorsV2/PVEMod_Collectors/LmSensors.pm new file mode 100644 index 0000000..95c3447 --- /dev/null +++ b/PveSensorsV2/PVEMod_Collectors/LmSensors.pm @@ -0,0 +1,427 @@ +package PVE::PVEMod::Collector::LmSensors; + +use strict; +use warnings; +use Exporter 'import'; + +use JSON; + +use PVE::PVEMod::Config qw(%config $process_type $sensors_state_file); +use PVE::PVEMod::Utils qw(debug check_executable setup_collector_signals read_sysfs); + +our @EXPORT_OK = qw( + collector_for_temperature_sensors +); + +# ============================================================================ +# Temperature Sensors — long-running collector +# ============================================================================ + +sub collector_for_temperature_sensors { + my ($device) = @_; + $process_type = 'collector'; + $0 = "collector-temperature-sensors"; + + debug(__LINE__, "Temperature sensor collector started"); + + unless (check_executable('/usr/bin/sensors', 'lm-sensors', + $config{debug}{sensors_mode}, + $config{debug}{sensors_output_file})) { + debug(__LINE__, "sensors not available and not in debug mode, exiting"); + exit(1); + } + + my %cache; + my $shutdown = 0; + setup_collector_signals('temperature-sensors', \$shutdown); + + while (!$shutdown) { + my $sensors_data = _get_temperature_sensors(\%cache); + + eval { + open my $ofh, '>', $sensors_state_file + or die "Failed to open $sensors_state_file: $!"; + print $ofh $sensors_data; + close $ofh; + debug(__LINE__, "Wrote temperature sensor data to $sensors_state_file"); + }; + if ($@) { + debug(__LINE__, "Error writing temperature sensor data: $@"); + } + + sleep $config{intervals}{data_pull} unless $shutdown; + } + + debug(__LINE__, "Temperature sensor collector shutting down"); + exit 0; +} + +# ============================================================================ +# Temperature Sensors — pipeline +# ============================================================================ + +sub _get_temperature_sensors { + my ($cache_ref) = @_; + + my $sensors_output; + + if ($config{debug}{sensors_mode} && -f $config{debug}{sensors_output_file}) { + debug(__LINE__, "Debug mode: reading sensors data from $config{debug}{sensors_output_file}"); + if (open my $fh, '<', $config{debug}{sensors_output_file}) { + local $/; + $sensors_output = <$fh>; + close $fh; + debug(__LINE__, "Read sensors data from debug file, length: " + . length($sensors_output) . " bytes"); + } else { + debug(__LINE__, "Failed to open debug file $config{debug}{sensors_output_file}: $!"); + $sensors_output = '{}'; + } + } else { + $sensors_output = `sensors -j 2>/dev/null | python3 -m json.tool`; + debug(__LINE__, "Raw sensors output collected from command"); + } + + debug(__LINE__, "Raw sensors output collected"); + + my $data = _sanitize_sensors($sensors_output); + debug(__LINE__, "Sanitized sensors output"); + + $data = _get_drive_names($data, $cache_ref); + debug(__LINE__, "Translated drive names in sensors output"); + + $data = _get_cpu_name($data, $cache_ref); + debug(__LINE__, "Translated CPU names in sensors output"); + + # Wrap in top-level key + my $sensors_json; + eval { $sensors_json = decode_json($data); }; + if ($@) { + debug(__LINE__, "Failed to parse final sensors JSON: $@"); + return $data; + } + + $data = JSON->new->pretty->encode({ "PVE MOD lm-sensors Enhanced" => $sensors_json }); + + return $data; +} + +# ============================================================================ +# Sanitize raw lm-sensors JSON +# ============================================================================ + +sub _sanitize_sensors { + my ($sensors_output) = @_; + + $sensors_output =~ s/ERROR:.+\s(\w+):\s(.+)/\"$1\": 0.000,/g; + $sensors_output =~ s/ERROR:.+\s(\w+)!/\"$1\": 0.000,/g; + $sensors_output =~ s/,\s*(})/$1/g; + $sensors_output =~ s/\bNaN\b/null/g; + + # Fix duplicate SODIMM keys: "SODIMM":{"temp3_input":34.0} → "SODIMM3":{...} + $sensors_output =~ + s/\"SODIMM\":\{\"temp(\d+)_input\"/\"SODIMM$1\":\{\"temp$1_input\"/g; + + return $sensors_output; +} + +# ============================================================================ +# Enrich sensors data with drive device info +# ============================================================================ + +sub _get_drive_names { + my ($sensors_output, $cache_ref) = @_; + $cache_ref //= {}; + + my $sensors_data; + eval { $sensors_data = decode_json($sensors_output); }; + if ($@) { + debug(__LINE__, "Failed to parse sensors JSON: $@"); + return $sensors_output; + } + + my @entries = grep { + /^drivetemp-scsi-/ || /^drivetemp-nvme-/ || /^nvme-pci-/ + } keys %{$sensors_data}; + + debug(__LINE__, "Found " . scalar(@entries) . " drive entries in sensors output"); + + my @drive_names; + + foreach my $entry (@entries) { + my ($dev_path, $model, $serial) = ("unknown", "unknown", "unknown"); + + if (exists $cache_ref->{$entry}) { + my $cached = $cache_ref->{$entry}; + $dev_path = $cached->{device_path}; + $model = $cached->{model}; + $serial = $cached->{serial}; + debug(__LINE__, "Using cached drive info for $entry"); + } else { + # ----- SCSI/SATA ----- + if ($entry =~ /^drivetemp-scsi-(\d+)-(\d+)/) { + my ($host, $id) = ($1, $2); + my $scsi_path = "/sys/class/scsi_disk/$host:$id:0:0/device/block"; + + if (opendir(my $sdh, $scsi_path)) { + my @devs = grep { /^sd/ } readdir($sdh); + closedir($sdh); + if (@devs) { + $dev_path = "/dev/$devs[0]"; + $model = read_sysfs("/sys/class/block/$devs[0]/device/model"); + $serial = read_sysfs("/sys/class/block/$devs[0]/device/serial"); + } + } + + # ----- Numeric NVMe ----- + } elsif ($entry =~ /^drivetemp-nvme-(\d+)/) { + my $nvme_index = $1; + $dev_path = "/dev/nvme${nvme_index}n1"; + if (-e $dev_path) { + $model = read_sysfs("/sys/class/block/nvme${nvme_index}n1/device/model"); + $serial = read_sysfs("/sys/class/block/nvme${nvme_index}n1/device/serial"); + } + + # ----- PCI-style NVMe ----- + } elsif ($entry =~ /^nvme-pci-(\w+)/) { + my $pci_addr = $1; + + # Convert short PCI address (e.g. "0600") to pattern (e.g. "0000:06:00") + my $pci_pattern; + if ($pci_addr =~ /^([0-9a-f]{2})([0-9a-f]{2})$/i) { + my ($bus, $dev) = ($1, $2); + $pci_pattern = sprintf("%04x:%02x:%02x", 0, hex($bus), hex($dev)); + debug(__LINE__, "Converted PCI address $pci_addr to pattern $pci_pattern"); + } else { + $pci_pattern = $pci_addr; + } + + my $found = 0; + my $nvme_dir = "/sys/class/nvme"; + + debug(__LINE__, + "Searching for NVMe devices in $nvme_dir matching PCI pattern $pci_pattern"); + + if (opendir(my $ndh, $nvme_dir)) { + my @nvme_devs = + grep { /^nvme\d+$/ && -d "$nvme_dir/$_" } readdir($ndh); + closedir($ndh); + + debug(__LINE__, "Found NVMe devices: " . join(", ", @nvme_devs)); + + foreach my $nvme_dev (@nvme_devs) { + my $device_link = readlink("$nvme_dir/$nvme_dev/device"); + if ($device_link && $device_link =~ /$pci_pattern/) { + debug(__LINE__, + "NVMe device $nvme_dev matches PCI pattern $pci_pattern"); + $dev_path = "/dev/${nvme_dev}n1"; + $model = read_sysfs("$nvme_dir/$nvme_dev/model"); + $serial = read_sysfs("$nvme_dir/$nvme_dev/serial"); + $found = 1; + debug(__LINE__, + "Found NVMe device via /sys/class/nvme: $dev_path"); + last; + } + debug(__LINE__, + "NVMe device $nvme_dev did not match PCI pattern $pci_pattern"); + } + } + + # Fallback: scan /sys/class/block + if (!$found && opendir(my $bdh, "/sys/class/block")) { + my @block_devs = grep { /^nvme\d+n\d+$/ } readdir($bdh); + closedir($bdh); + + foreach my $block_dev (@block_devs) { + my $device_link = + readlink("/sys/class/block/$block_dev/device"); + if ($device_link && $device_link =~ /$pci_pattern/) { + $dev_path = "/dev/$block_dev"; + (my $nvme_ctrl = $block_dev) =~ s/n\d+$//; + $model = read_sysfs("/sys/class/nvme/$nvme_ctrl/model"); + $serial = read_sysfs("/sys/class/nvme/$nvme_ctrl/serial"); + $found = 1; + debug(__LINE__, + "Found NVMe device via /sys/class/block: $dev_path"); + last; + } + } + } + + unless ($found) { + debug(__LINE__, + "Could not find device for nvme-pci-$pci_addr (pattern: $pci_pattern)"); + } + } else { + next; + } + + $cache_ref->{$entry} = { + device_path => $dev_path, + model => $model, + serial => $serial, + }; + + debug(__LINE__, "Drive: $entry -> $dev_path (Model: $model, Serial: $serial)"); + } + + push @drive_names, [$entry, $dev_path, $model, $serial]; + } + + foreach my $drive_entry (@drive_names) { + my ($original_name, $dev_path, $model, $serial) = @$drive_entry; + if (exists $sensors_data->{$original_name}) { + $sensors_data->{$original_name}->{device_path} = $dev_path; + $sensors_data->{$original_name}->{model} = $model; + $sensors_data->{$original_name}->{serial} = $serial; + debug(__LINE__, "Enhanced $original_name with drive info"); + } + } + + return JSON->new->pretty->canonical->encode($sensors_data); +} + +# ============================================================================ +# Enrich sensors data with CPU model info +# ============================================================================ + +sub _get_cpu_name { + my ($sensors_output, $cache_ref) = @_; + $cache_ref //= {}; + + my $sensors_data; + eval { $sensors_data = decode_json($sensors_output); }; + if ($@) { + debug(__LINE__, "Failed to parse sensors JSON: $@"); + return $sensors_output; + } + + my @entries = + grep { /^coretemp-isa-/ || /^k10temp-pci-/ } keys %{$sensors_data}; + + debug(__LINE__, "Found " . scalar(@entries) . " CPU entries in sensors output"); + + foreach my $entry (@entries) { + my ($cpu_model, $pkg) = ("unknown", "unknown"); + + if (exists $cache_ref->{$entry}) { + my $cached = $cache_ref->{$entry}; + $cpu_model = $cached->{model}; + $pkg = $cached->{package}; + debug(__LINE__, "Using cached CPU info for $entry"); + } else { + # ----- Intel coretemp ----- + if ($entry =~ /^coretemp-isa-(\d+)/) { + for my $hwmon (glob "/sys/class/hwmon/hwmon*") { + my $name = read_sysfs("$hwmon/name"); + next unless $name eq 'coretemp'; + + my $dev = readlink("$hwmon/device"); + next unless $dev; + + if ($dev =~ /\.([0-9]+)$/) { + $pkg = $1; + $cpu_model = _cpu_model_by_package($pkg); + debug(__LINE__, + "Found Intel CPU: $entry -> Package $pkg, Model: $cpu_model"); + last; + } + } + } + + # ----- AMD k10temp ----- + elsif ($entry =~ /^k10temp-pci-(\w+)/) { + my $pci_addr = $1; + my $pci_pattern = $pci_addr; + + if ($pci_addr =~ /^([0-9a-f]{2})([0-9a-f]{2})$/i) { + my ($bus, $dev_func) = ($1, $2); + $pci_pattern = + sprintf("%04x:%02x:%02x", 0, hex($bus), hex($dev_func)); + debug(__LINE__, + "Converted PCI address $pci_addr to pattern $pci_pattern"); + } + + for my $hwmon (glob "/sys/class/hwmon/hwmon*") { + my $name = read_sysfs("$hwmon/name"); + next unless $name eq 'k10temp'; + + my $dev = readlink("$hwmon/device"); + next unless $dev; + + if ($dev =~ /$pci_pattern/ || $dev =~ /$pci_addr/) { + $pkg = 0; + + if (opendir(my $dh, "/sys/devices/system/cpu")) { + my @cpus = grep { /^cpu\d+$/ } readdir($dh); + closedir($dh); + + foreach my $cpu (@cpus) { + my $cpu_pkg = read_sysfs( + "/sys/devices/system/cpu/$cpu/topology/physical_package_id"); + if ($cpu_pkg ne "unknown" && $cpu_pkg =~ /^\d+$/) { + $pkg = $cpu_pkg; + last; + } + } + } + + $cpu_model = _cpu_model_by_package($pkg); + debug(__LINE__, + "Found AMD CPU: $entry -> Package $pkg, Model: $cpu_model"); + last; + } + } + } + + $cache_ref->{$entry} = { model => $cpu_model, package => $pkg }; + debug(__LINE__, "CPU: $entry -> Package $pkg (Model: $cpu_model)"); + } + + if (exists $sensors_data->{$entry}) { + $sensors_data->{$entry}->{cpu_model} = $cpu_model; + $sensors_data->{$entry}->{cpu_package} = $pkg; + debug(__LINE__, "Enhanced $entry with CPU info"); + } + } + + return JSON->new->pretty->canonical->encode($sensors_data); +} + +# ============================================================================ +# CPU model lookup helper +# ============================================================================ + +sub _cpu_model_by_package { + my ($pkg) = @_; + + if (open my $fh, '<', '/proc/cpuinfo') { + my $current_pkg = -1; + my $model_name = "unknown"; + + while (my $line = <$fh>) { + chomp $line; + + if ($line =~ /^physical id\s+:\s+(\d+)/) { + $current_pkg = $1; + } + + if ($line =~ /^model name\s+:\s+(.+)$/) { + $model_name = $1; + $model_name =~ s/^\s+|\s+$//g; + + if ($current_pkg == $pkg) { + close($fh); + return $model_name; + } + } + } + close($fh); + + return $model_name if $model_name ne "unknown"; + } + + return "unknown"; +} + +1; diff --git a/PveSensorsV2/PVEMod_Collectors/Nvidia.pm b/PveSensorsV2/PVEMod_Collectors/Nvidia.pm new file mode 100644 index 0000000..fb08b88 --- /dev/null +++ b/PveSensorsV2/PVEMod_Collectors/Nvidia.pm @@ -0,0 +1,218 @@ +package PVE::PVEMod::Collector::Nvidia; + +use strict; +use warnings; +use Exporter 'import'; + +use PVE::PVEMod::Config qw(%config $process_type $pve_mod_working_dir); +use PVE::PVEMod::Utils qw(debug check_executable setup_collector_signals safe_write_json parse_csv_line); +use PVE::PVEMod::Store qw(update_nvidia_gpu_rrd); + +our @EXPORT_OK = qw( + get_nvidia_gpu_devices + collector_for_nvidia_devices +); + +# ============================================================================ +# NVIDIA GPU — device discovery +# ============================================================================ + +sub get_nvidia_gpu_devices { + my @devices = (); + + unless (check_executable('/usr/bin/nvidia-smi', 'NVIDIA', + $config{debug}{nvidia_mode}, + $config{debug}{nvidia_devices_file})) { + return @devices; + } + + if ($config{debug}{nvidia_mode} && -f $config{debug}{nvidia_devices_file}) { + debug(__LINE__, "Debug mode: reading NVIDIA GPU devices from $config{debug}{nvidia_devices_file}"); + if (open my $fh, '<', $config{debug}{nvidia_devices_file}) { + my $line_num = 0; + while (<$fh>) { + chomp; + $line_num++; + next if $line_num == 1 || /^\s*$/; + my @values = parse_csv_line($_, 2); + if (@values) { + push @devices, { index => $values[0], name => $values[1] }; + debug(__LINE__, "Found NVIDIA GPU device (debug): $values[1] (index: $values[0])"); + } + } + close $fh; + } else { + debug(__LINE__, "Failed to open debug file $config{debug}{nvidia_devices_file}: $!"); + } + } else { + if (open my $fh, '-|', 'nvidia-smi --query-gpu=index,name --format=csv') { + my $line_num = 0; + while (<$fh>) { + chomp; + $line_num++; + next if $line_num == 1 || /^\s*$/; + my @values = parse_csv_line($_, 2); + if (@values) { + push @devices, { index => $values[0], name => $values[1] }; + debug(__LINE__, "Found NVIDIA GPU device: $values[1] (index: $values[0])"); + } + } + close $fh; + } + } + + return @devices; +} + +# ============================================================================ +# NVIDIA GPU — data parsing +# ============================================================================ + +sub _parse_nvidia_gpu_line { + my ($line) = @_; + + # Expected CSV format: + # index, name, temperature.gpu, utilization.gpu, utilization.memory, + # memory.used, memory.total, power.draw, power.limit, fan.speed + + my @values = parse_csv_line($line, 10); + return unless @values; + + return { + index => $values[0] + 0, + name => $values[1], + temperature => { + gpu => $values[2] + 0.0, + unit => "°C", + }, + utilization => { + gpu => $values[3] + 0.0, + memory => $values[4] + 0.0, + unit => "%", + }, + memory => { + used => $values[5] + 0.0, + total => $values[6] + 0.0, + unit => "MiB", + }, + power => { + draw => $values[7] + 0.0, + limit => $values[8] + 0.0, + unit => "W", + }, + fan => { + speed => $values[9] + 0.0, + unit => "%", + }, + }; +} + +# ============================================================================ +# NVIDIA GPU — stat collection and write +# ============================================================================ + +sub _get_and_write_nvidia_stats { + my ($devices) = @_; + my @all_stats; + + if ($config{debug}{nvidia_mode} && -f $config{debug}{nvidia_output_file}) { + debug(__LINE__, "Debug mode: reading NVIDIA GPU stats from $config{debug}{nvidia_output_file}"); + if (open my $fh, '<', $config{debug}{nvidia_output_file}) { + my $line_num = 0; + while (<$fh>) { + chomp; + $line_num++; + next if $line_num == 1 || /^\s*$/; + my $stats = _parse_nvidia_gpu_line($_); + push @all_stats, $stats if $stats; + } + close $fh; + } else { + debug(__LINE__, "Failed to open debug file $config{debug}{nvidia_output_file}: $!"); + } + } else { + unless (check_executable('/usr/bin/nvidia-smi', 'NVIDIA')) { + debug(__LINE__, "nvidia-smi not available, cannot collect stats"); + return 0; + } + + my $query = 'index,name,temperature.gpu,utilization.gpu,utilization.memory,' + . 'memory.used,memory.total,power.draw,power.limit,fan.speed'; + my $cmd = "nvidia-smi --query-gpu=$query --format=csv,nounits"; + + if (open my $fh, '-|', $cmd) { + my $line_num = 0; + while (<$fh>) { + chomp; + $line_num++; + next if $line_num == 1 || /^\s*$/; + my $stats = _parse_nvidia_gpu_line($_); + push @all_stats, $stats if $stats; + } + close $fh; + } + } + + foreach my $stats (@all_stats) { + my $device_index = $stats->{index}; + + unless ($device_index =~ /^(\d+)$/) { + debug(__LINE__, "Invalid device index: $device_index, skipping"); + next; + } + $device_index = $1; # untainted + + my $node_name = "gpu$device_index"; + my $device_state_file = "$pve_mod_working_dir/stats-nvidia$device_index.json"; + + my $device_name = $stats->{name}; + foreach my $dev (@$devices) { + if ($dev->{index} == $device_index) { + $device_name = $dev->{name}; + last; + } + } + + my $device_data = { + $node_name => { + name => $device_name, + index => $device_index, + stats => $stats, + } + }; + + safe_write_json($device_state_file, $device_data); + update_nvidia_gpu_rrd($device_index, $stats); + } + + unless (@all_stats) { + debug(__LINE__, "No valid NVIDIA GPU stats collected"); + } + + return scalar(@all_stats); +} + +# ============================================================================ +# NVIDIA GPU — long-running collector (all devices in one process) +# ============================================================================ + +sub collector_for_nvidia_devices { + my ($devices) = @_; + $process_type = 'collector'; + $0 = "collector-gpu-nvidia-all"; + + debug(__LINE__, "NVIDIA collector started for " . scalar(@$devices) . " GPU(s)"); + + my $shutdown = 0; + setup_collector_signals('nvidia-all', \$shutdown); + + while (!$shutdown) { + _get_and_write_nvidia_stats($devices); + sleep $config{intervals}{data_pull} unless $shutdown; + } + + debug(__LINE__, "NVIDIA collector shutting down"); + exit 0; +} + +1; diff --git a/PveSensorsV2/PVEMod_Collectors/Ups.pm b/PveSensorsV2/PVEMod_Collectors/Ups.pm new file mode 100644 index 0000000..261713f --- /dev/null +++ b/PveSensorsV2/PVEMod_Collectors/Ups.pm @@ -0,0 +1,115 @@ +package PVE::PVEMod::Collector::Ups; + +use strict; +use warnings; +use Exporter 'import'; + +use JSON; + +use PVE::PVEMod::Config qw($process_type $ups_state_file); +use PVE::PVEMod::Utils qw(debug setup_collector_signals); + +our @EXPORT_OK = qw( + collector_for_ups +); + +# ============================================================================ +# UPS — long-running collector +# ============================================================================ + +sub collector_for_ups { + my ($device) = @_; + $process_type = 'collector'; + $0 = "collector-ups-$device->{ups_name}"; + debug(__LINE__, "UPS collector started"); + + my $shutdown = 0; + setup_collector_signals("ups-$device->{ups_name}", \$shutdown); + + while (!$shutdown) { + my $ups_data = _get_ups_status($device->{ups_name}); + + eval { + open my $ofh, '>', $ups_state_file + or die "Failed to open $ups_state_file: $!"; + print $ofh $ups_data; + close $ofh; + debug(__LINE__, "Wrote UPS data to $ups_state_file"); + }; + if ($@) { + debug(__LINE__, "Error writing UPS data: $@"); + } + + sleep 1 unless $shutdown; # $config{intervals}{data_pull} + } + + debug(__LINE__, "UPS collector shutting down"); + exit 0; +} + +# ============================================================================ +# UPS — status query +# ============================================================================ + +sub _get_ups_status { + my ($ups_name) = @_; + + debug(__LINE__, "Collecting UPS status for $ups_name"); + + my $output = `/usr/bin/upsc $ups_name 2>/dev/null`; + + unless (defined $output && length($output) > 0) { + debug(__LINE__, "No output from upsc for $ups_name"); + return encode_json({ error => "No data from UPS $ups_name" }); + } + + my $ups_data = _parse_upsc_output($output); + + unless (keys %$ups_data) { + debug(__LINE__, "No data received from upsc for $ups_name"); + return encode_json({ error => "No data from UPS $ups_name" }); + } + + return JSON->new->pretty->canonical->encode({ $ups_name => $ups_data }); +} + +# ============================================================================ +# UPS — output parser +# ============================================================================ + +sub _parse_upsc_output { + my ($output) = @_; + + my $ups_data = {}; + + debug(__LINE__, "Parsing upsc output"); + + eval { + foreach my $line (split /\n/, $output) { + next if $line =~ /^\s*$/; + next if $line =~ /^Init SSL/; + + if ($line =~ /^([^:]+):\s*(.*)$/) { + my ($key, $value) = ($1, $2); + $key =~ s/^\s+|\s+$//g; + $value =~ s/^\s+|\s+$//g; + + # Coerce numeric values + if ($value =~ /^-?\d+\.?\d*$/) { + $ups_data->{$key} = $value + 0; + } else { + $ups_data->{$key} = $value; + } + } + } + }; + if ($@) { + debug(__LINE__, "Error parsing upsc output: $@"); + } + + debug(__LINE__, "Completed parsing upsc output"); + + return $ups_data; +} + +1; diff --git a/PveSensorsV2/PVEMod_Config.pm b/PveSensorsV2/PVEMod_Config.pm new file mode 100644 index 0000000..a96fd9d --- /dev/null +++ b/PveSensorsV2/PVEMod_Config.pm @@ -0,0 +1,76 @@ +package PVE::PVEMod::Config; + +use strict; +use warnings; +use Exporter 'import'; + +our @EXPORT_OK = qw( + %config + $DEBUG_ENABLED $VERSION $process_type + $pve_mod_working_dir $stats_dir $state_file + $sensors_state_file $ups_state_file + $pve_mod_worker_lock $startup_lock + $RRD_SOCKET $RRD_BASE +); + +# ============================================================================ +# Debug / Version +# ============================================================================ + +our $DEBUG_ENABLED = 1; +our $VERSION = '1.0'; + +# Runtime process-type tag — set to 'worker' or 'collector' after fork. +# Each forked child gets its own copy of this variable. +our $process_type = 'main'; # 'main', 'worker', or 'collector' + +# ============================================================================ +# Configuration +# ============================================================================ + +our %config = ( + gpu => { + intel_enabled => 1, + amd_enabled => 0, + nvidia_enabled => 0, + }, + debug => { + nvidia_mode => 1, + nvidia_devices_file => '/tmp/nvidia-smi-devices.csv', + nvidia_output_file => '/tmp/nvidia-smi-output.csv', + sensors_mode => 0, + sensors_output_file => '/tmp/sensors-output.json', + }, + intervals => { + data_pull => 1, # seconds between data pulls + collector_timeout => 10, # stop collectors after N seconds of inactivity + }, + ups => { + enabled => 1, + device_name => 'ups@192.168.3.2', + }, + paths => { + working_dir => '/run/pveproxy/pve-mod', + }, +); + +# ============================================================================ +# Derived paths +# ============================================================================ + +our $pve_mod_working_dir = $config{paths}{working_dir}; +our $stats_dir = $pve_mod_working_dir; +our $state_file = "$pve_mod_working_dir/stats.json"; +our $sensors_state_file = "$pve_mod_working_dir/sensors.json"; +our $ups_state_file = "$pve_mod_working_dir/ups.json"; +our $pve_mod_worker_lock = "$pve_mod_working_dir/pve_mod_worker.lock"; +our $startup_lock = "$pve_mod_working_dir/startup.lock"; + +# ============================================================================ +# RRD paths +# ============================================================================ + +our $RRD_SOCKET = '/var/run/rrdcached.sock'; +our $RRD_BASE = '/var/lib/rrdcached/db/pve-mod-gpu'; + +1; diff --git a/PveSensorsV2/PVEMod_ProcessManager.pm b/PveSensorsV2/PVEMod_ProcessManager.pm new file mode 100644 index 0000000..3c02822 --- /dev/null +++ b/PveSensorsV2/PVEMod_ProcessManager.pm @@ -0,0 +1,483 @@ +package PVE::PVEMod::ProcessManager; + +use strict; +use warnings; +use Exporter 'import'; + +use POSIX qw(WNOHANG); +use File::Path qw(remove_tree); + +use PVE::PVEMod::Config qw( + %config $process_type + $pve_mod_working_dir $state_file + $pve_mod_worker_lock $startup_lock +); +use PVE::PVEMod::Utils qw( + debug is_process_alive read_lock_pid + acquire_exclusive_lock ensure_pve_mod_directory_exists + check_executable startup_message +); + +use PVE::PVEMod::Collector::Intel qw(get_intel_gpu_devices collector_for_intel_device); +use PVE::PVEMod::Collector::Nvidia qw(get_nvidia_gpu_devices collector_for_nvidia_devices); +use PVE::PVEMod::Collector::Amd qw(get_amd_gpu_devices collector_for_amd_device); +use PVE::PVEMod::Collector::LmSensors qw(collector_for_temperature_sensors); +use PVE::PVEMod::Collector::Ups qw(collector_for_ups); + +our @EXPORT_OK = qw( + pve_mod_starter + notify_pve_mod_worker +); + +# Collector registry — only populated inside the worker process. +# Each forked child has its own copy; the parent never accesses this after forking. +my %collectors = (); + +# ============================================================================ +# Public API (called from SensorInfo) +# ============================================================================ + +# Ensures the worker is running. Starts it if necessary (double-checked locking). +sub pve_mod_starter { + debug(__LINE__, "Checking if pve_mod_worker is already running"); + if (_is_pve_mod_worker_running()) { + debug(__LINE__, "pve_mod_worker process already running, system is already started"); + return "pve_mod_worker process already running, system is already started"; + } + debug(__LINE__, "PVE mod worker is not running. PVE Mod will be started."); + + startup_message(); + ensure_pve_mod_directory_exists(); + + debug(__LINE__, "Trying to acquire startup lock: $startup_lock"); + my $startup_fh = acquire_exclusive_lock($startup_lock, 'startup lock'); + return unless $startup_fh; + + # Second check after acquiring lock + if (_is_pve_mod_worker_running()) { + debug(__LINE__, "Worker started by another process while we waited for lock"); + close($startup_fh); + unlink($startup_lock); + return "already running"; + } + + print $startup_fh "$$\n"; + $startup_fh->flush(); + debug(__LINE__, "Wrote PID $$ to startup lock"); + + _pve_mod_worker(); + + unlink($startup_lock); + debug(__LINE__, "Released startup lock"); + debug(__LINE__, "pve_mod_worker started successfully, returning"); +} + +# Sends SIGUSR1 to the worker to reset the inactivity timer. +sub notify_pve_mod_worker { + debug(__LINE__, "notify_pve_mod_worker called"); + unless (-f $pve_mod_worker_lock) { + debug(__LINE__, "pve_mod_worker lock file does not exist"); + return; + } + + debug(__LINE__, "pve_mod_worker lock file exists, reading PID"); + if (open my $fh, '<', $pve_mod_worker_lock) { + my $pid = <$fh>; + close $fh; + chomp $pid if defined $pid; + if (defined $pid && $pid =~ /^(\d+)$/) { + my $clean_pid = $1; + + if (is_process_alive($clean_pid)) { + debug(__LINE__, "Sending USR1 signal to pve_mod_worker PID $clean_pid"); + my $result = kill('USR1', $clean_pid); + debug(__LINE__, "Signal result: $result"); + } else { + debug(__LINE__, + "pve_mod_worker process $clean_pid is not alive, removing stale lock"); + unlink($pve_mod_worker_lock); + } + } else { + debug(__LINE__, + "pve_mod_worker lock is stale (PID: " . ($pid // 'undefined') . "), removing"); + unlink($pve_mod_worker_lock); + } + } else { + debug(__LINE__, "Failed to open pve_mod_worker lock file: $!"); + } +} + +# ============================================================================ +# Worker process management +# ============================================================================ + +sub _is_pve_mod_worker_running { + return -f $pve_mod_worker_lock; +} + +# Forks the worker process and records its PID in the lock file. +sub _pve_mod_worker { + debug(__LINE__, "_pve_mod_worker called"); + + my $pve_mod_worker_fh = + acquire_exclusive_lock($pve_mod_worker_lock, 'pve_mod_worker lock'); + return unless $pve_mod_worker_fh; + print $pve_mod_worker_fh "$$\n"; + close($pve_mod_worker_fh); + + debug(__LINE__, "Forking new pve_mod_worker process"); + my $pve_mod_worker_pid = fork(); + + unless (defined $pve_mod_worker_pid) { + debug(__LINE__, "Failed to fork pve_mod_worker process: $!"); + return; + } + + if ($pve_mod_worker_pid == 0) { + # Child + $0 = "pve_mod_worker_controller"; + debug(__LINE__, "Child process forked, calling _pve_mod_keep_alive"); + _pve_mod_keep_alive(); + exit(0); + } + + # Parent — update lock file with real child PID + debug(__LINE__, "Forked pve_mod_worker process with PID $pve_mod_worker_pid"); + if (open my $fh, '>', $pve_mod_worker_lock) { + print $fh "$pve_mod_worker_pid\n"; + close $fh; + debug(__LINE__, "Wrote pve_mod_worker PID to lock file: $pve_mod_worker_lock"); + } else { + debug(__LINE__, "Failed to write pve_mod_worker lock file: $!"); + kill('TERM', $pve_mod_worker_pid); + } + + debug(__LINE__, "pve_mod_worker process started successfully"); +} + +# ============================================================================ +# Worker keep-alive loop +# ============================================================================ + +sub _pve_mod_keep_alive { + $process_type = 'worker'; + debug(__LINE__, "pve_mod_worker process started with PID $$"); + + my $last_activity = time(); + + $SIG{USR1} = sub { + $last_activity = time(); + debug(__LINE__, "Activity ping received"); + }; + + $SIG{CHLD} = sub { + while ((my $pid = waitpid(-1, WNOHANG)) > 0) { + my $exit_status = $? >> 8; + debug(__LINE__, "Child process $pid exited with status $exit_status"); + + foreach my $name (keys %collectors) { + if ($collectors{$name} == $pid) { + debug(__LINE__, + "Collector '$name' (PID $pid) exited, removing from registry"); + delete $collectors{$name}; + last; + } + } + } + }; + + $SIG{TERM} = sub { + debug(__LINE__, "pve_mod_worker received SIGTERM, shutting down"); + _stop_child_collectors(); + unlink($pve_mod_worker_lock) if -f $pve_mod_worker_lock; + exit(0); + }; + $SIG{INT} = sub { + debug(__LINE__, "pve_mod_worker received SIGINT, shutting down"); + _stop_child_collectors(); + unlink($pve_mod_worker_lock) if -f $pve_mod_worker_lock; + exit(0); + }; + + debug(__LINE__, "Worker starting all collectors"); + _start_sensors_collector(); + _start_graphics_collectors(); + _start_ups_collector(); + debug(__LINE__, "All collectors started by worker"); + + debug(__LINE__, + "Entering pve_mod_worker loop, timeout=$config{intervals}{collector_timeout}s"); + + while (1) { + debug(__LINE__, "pve_mod_worker loop start: checking activity"); + + my $idle_time = time() - $last_activity; + debug(__LINE__, + "pve_mod_worker loop: idle_time=${idle_time}s, " + . "timeout=$config{intervals}{collector_timeout}s"); + + if ($idle_time > $config{intervals}{collector_timeout}) { + debug(__LINE__, "Timeout reached, stopping collectors"); + _stop_child_collectors(); + debug(__LINE__, "Collectors stopped, exiting pve_mod_worker"); + unlink($pve_mod_worker_lock) if -f $pve_mod_worker_lock; + exit(0); + } + sleep(1); + } + + debug(__LINE__, "pve_mod_worker loop exited unexpectedly!"); +} + +# ============================================================================ +# Collector startup helpers (called from worker loop) +# ============================================================================ + +sub _start_sensors_collector { + debug(__LINE__, "Starting temperature sensor collector"); + + unless (check_executable('/usr/bin/sensors', 'lm-sensors', + $config{debug}{sensors_mode}, + $config{debug}{sensors_output_file})) { + debug(__LINE__, "sensors not available and not in debug mode, skipping"); + return; + } + + _start_collector('sensors', 'sensors', + \&collector_for_temperature_sensors, + { name => 'sensors' }); +} + +sub _start_ups_collector { + unless ($config{ups}{enabled}) { + debug(__LINE__, "UPS support not enabled, skipping collector startup"); + return; + } + + debug(__LINE__, "Starting UPS collector"); + + unless (check_executable('/usr/bin/upsc', 'UPS')) { + debug(__LINE__, "upsc not available, skipping UPS collector startup"); + return; + } + + unless ($config{ups}{device_name}) { + debug(__LINE__, "No UPS configured, skipping collector startup"); + return; + } + + _start_collector('ups', 'ups', \&collector_for_ups, + { ups_name => $config{ups}{device_name} }); +} + +sub _start_graphics_collectors { + unless ($config{gpu}{intel_enabled} + || $config{gpu}{amd_enabled} + || $config{gpu}{nvidia_enabled}) { + debug(__LINE__, "No GPU types enabled, skipping collector startup"); + return; + } + + debug(__LINE__, "Starting graphics collectors"); + + my (@all_devices, @all_types, @all_collector_subs); + + # Intel + if ($config{gpu}{intel_enabled}) { + debug(__LINE__, "Intel GPU support enabled"); + return unless check_executable('/usr/bin/intel_gpu_top', 'Intel'); + + my @intel_devices = get_intel_gpu_devices(); + if (@intel_devices) { + debug(__LINE__, "Found " . scalar(@intel_devices) . " Intel GPU device(s)"); + for my $device (@intel_devices) { + push @all_devices, $device; + push @all_types, 'intel'; + push @all_collector_subs, \&collector_for_intel_device; + } + } else { + debug(__LINE__, "No Intel GPU devices found"); + } + } + + # AMD (future) + if ($config{gpu}{amd_enabled}) { + debug(__LINE__, "AMD GPU support enabled"); + return unless check_executable('/usr/bin/rocm-smi', 'AMD'); + + my @amd_devices = get_amd_gpu_devices(); + debug(__LINE__, "Got " . scalar(@amd_devices) . " AMD devices"); + for my $device (@amd_devices) { + push @all_devices, $device; + push @all_types, 'amd'; + push @all_collector_subs, \&collector_for_amd_device; + } + } + + debug(__LINE__, + "Finished detecting devices. Total Intel/AMD collectors to manage: " + . scalar(@all_devices)); + + my $started_count = 0; + + # NVIDIA — single collector for all GPUs + if ($config{gpu}{nvidia_enabled}) { + debug(__LINE__, "NVIDIA GPU support enabled"); + my @nvidia_devices = get_nvidia_gpu_devices(); + debug(__LINE__, "Got " . scalar(@nvidia_devices) . " NVIDIA devices"); + + if (@nvidia_devices) { + my $pid = _start_collector('nvidia-all', 'nvidia', + \&collector_for_nvidia_devices, + \@nvidia_devices); + $started_count++ if $pid; + } + } + + for (my $i = 0; $i < @all_devices; $i++) { + my $device = $all_devices[$i]; + my $type = $all_types[$i]; + my $collector_sub = $all_collector_subs[$i]; + my $device_name = $device->{card} // $device->{name} // "device$i"; + + my $pid = _start_collector($device_name, $type, $collector_sub, $device); + $started_count++ if $pid; + } + + debug(__LINE__, + "Started/verified $started_count graphics collector(s) (Intel/AMD)"); +} + +# ============================================================================ +# Generic collector start/stop +# ============================================================================ + +sub _start_collector { + my ($collector_name, $collector_type, $collector_sub, $device) = @_; + + debug(__LINE__, "Starting $collector_type collector: $collector_name"); + + if (exists $collectors{$collector_name}) { + my $pid = $collectors{$collector_name}; + if (kill(0, $pid)) { + debug(__LINE__, + "$collector_type collector '$collector_name' already running with PID $pid"); + return $pid; + } else { + debug(__LINE__, + "Collector '$collector_name' PID $pid is stale, removing from registry"); + delete $collectors{$collector_name}; + } + } + + my $pid = _start_child_collector($collector_name, $collector_sub, $device); + + unless ($pid) { + debug(__LINE__, "Failed to start $collector_type collector '$collector_name'"); + return undef; + } + + $collectors{$collector_name} = $pid; + debug(__LINE__, + "Registered $collector_type collector '$collector_name' with PID $pid"); + + sleep 0.1; + if (kill(0, $pid)) { + debug(__LINE__, + "Verified $collector_type collector '$collector_name' (PID $pid) is alive"); + return $pid; + } else { + debug(__LINE__, + "WARNING - $collector_type collector '$collector_name' (PID $pid) died immediately!"); + delete $collectors{$collector_name}; + return undef; + } +} + +sub _start_child_collector { + my ($collector_name, $collector_sub, $device) = @_; + + debug(__LINE__, "Starting child collector: $collector_name"); + + my $pid = fork(); + unless (defined $pid) { + debug(__LINE__, "fork failed for $collector_name: $!"); + return undef; + } + + if ($pid == 0) { + $process_type = 'collector'; + debug(__LINE__, "In child process for $collector_name"); + $0 = "collector-$collector_name"; + $collector_sub->($device); + exit(0); + } + + debug(__LINE__, "Forked child PID $pid for $collector_name"); + return $pid; +} + +sub _stop_child_collectors { + debug(__LINE__, "Stopping all collectors"); + + my @pids = values %collectors; + + if (@pids) { + debug(__LINE__, "Sending SIGTERM to " . scalar(@pids) . " collector process(es)"); + foreach my $pid (@pids) { + if (kill(0, $pid)) { + kill('TERM', $pid); + debug(__LINE__, "Sent SIGTERM to collector PID $pid"); + } + } + + my $timeout = 2; + my $start = time(); + while (time() - $start < $timeout) { + my $any_alive = 0; + foreach my $pid (@pids) { + if (kill(0, $pid)) { $any_alive = 1; last; } + } + last unless $any_alive; + select(undef, undef, undef, 0.1); + } + + foreach my $pid (@pids) { + if (kill(0, $pid)) { + debug(__LINE__, "Force killing collector process $pid"); + kill('KILL', $pid); + } + } + } + + %collectors = (); + debug(__LINE__, "Cleared collector registry"); + + if (-f $state_file) { + unlink $state_file or debug(__LINE__, "Failed to remove $state_file: $!"); + } + + if (-d $pve_mod_working_dir) { + remove_tree($pve_mod_working_dir, { error => \my $err }); + debug(__LINE__, "Cleanup errors: @$err") if @$err; + } + + debug(__LINE__, "Cleanup complete"); +} + +# ============================================================================ +# END block — only the worker process performs cleanup +# ============================================================================ + +END { + if ($process_type eq 'worker') { + debug(__LINE__, "PVE Mod Worker END block: cleaning up"); + _stop_child_collectors(); + } elsif ($process_type eq 'collector') { + debug(__LINE__, "Collector ($0) END block: no cleanup needed"); + } else { + debug(__LINE__, "Main process END block: no cleanup needed"); + } +} + +1; diff --git a/PveSensorsV2/PVEMod_Store.pm b/PveSensorsV2/PVEMod_Store.pm new file mode 100644 index 0000000..436aa59 --- /dev/null +++ b/PveSensorsV2/PVEMod_Store.pm @@ -0,0 +1,166 @@ +package PVE::PVEMod::Store; + +use strict; +use warnings; +use Exporter 'import'; + +use File::Path qw(make_path); +use PVE::INotify; +use RRDs; + +use PVE::PVEMod::Config qw($RRD_SOCKET $RRD_BASE); +use PVE::PVEMod::Utils qw(debug); + +our @EXPORT_OK = qw( + get_nodename + gpu_rrd_path + update_intel_gpu_rrd + update_nvidia_gpu_rrd +); + +# ============================================================================ +# Node name +# ============================================================================ + +sub get_nodename { + return PVE::INotify::nodename(); +} + +# ============================================================================ +# RRD path helper +# ============================================================================ + +sub gpu_rrd_path { + my ($card) = @_; + return "$RRD_BASE/" . get_nodename() . "/$card"; +} + +# ============================================================================ +# Intel GPU RRD +# ============================================================================ + +sub _ensure_intel_gpu_rrd { + my ($card) = @_; + my $path = gpu_rrd_path($card); + return if -f $path; + + my $dir = "$RRD_BASE/" . get_nodename(); + make_path($dir, { mode => 0755 }) unless -d $dir; + + RRDs::create( + $path, + '--step', '1', + 'DS:freq_req:GAUGE:120:0:U', + 'DS:freq_act:GAUGE:120:0:U', + 'DS:rc6:GAUGE:120:0:100', + 'DS:power_gpu:GAUGE:120:0:U', + 'DS:power_pkg:GAUGE:120:0:U', + 'DS:render_busy:GAUGE:120:0:100', + 'DS:blitter_busy:GAUGE:120:0:100', + 'DS:video_busy:GAUGE:120:0:100', + 'DS:videnh_busy:GAUGE:120:0:100', + 'RRA:AVERAGE:0.5:1:1440', + 'RRA:AVERAGE:0.5:60:1440', + 'RRA:AVERAGE:0.5:1800:1344', + 'RRA:AVERAGE:0.5:21600:1464', + 'RRA:AVERAGE:0.5:604800:520', + 'RRA:MAX:0.5:1:1440', + 'RRA:MAX:0.5:60:1440', + 'RRA:MAX:0.5:1800:1344', + 'RRA:MAX:0.5:21600:1464', + 'RRA:MAX:0.5:604800:520', + ); + my $err = RRDs::error(); + debug(__LINE__, "Created Intel GPU RRD $path: " . ($err // 'OK')); +} + +sub update_intel_gpu_rrd { + my ($card, $stats) = @_; + _ensure_intel_gpu_rrd($card); + my $path = gpu_rrd_path($card); + + my $freq_req = $stats->{frequency}{requested} // 'U'; + my $freq_act = $stats->{frequency}{actual} // 'U'; + my $rc6 = $stats->{rc6}{value} // 'U'; + my $power_gpu = $stats->{power}{GPU} // 'U'; + my $power_pkg = $stats->{power}{Package} // 'U'; + my $render_busy = $stats->{engines}{'Render/3D'}{busy} // 'U'; + my $blitter = $stats->{engines}{Blitter}{busy} // 'U'; + my $video = $stats->{engines}{Video}{busy} // 'U'; + my $videnh = $stats->{engines}{VideoEnhance}{busy} // 'U'; + + my @daemon_args = (-S $RRD_SOCKET) ? ('--daemon', "unix:$RRD_SOCKET") : (); + RRDs::update( + $path, + @daemon_args, + "N:$freq_req:$freq_act:$rc6:$power_gpu:$power_pkg:$render_busy:$blitter:$video:$videnh", + ); + my $err = RRDs::error(); + debug(__LINE__, "RRD update intel $card: $err") if $err; +} + +# ============================================================================ +# NVIDIA GPU RRD +# ============================================================================ + +sub _ensure_nvidia_gpu_rrd { + my ($index) = @_; + my $card = "nvidia$index"; + my $path = gpu_rrd_path($card); + return if -f $path; + + my $dir = "$RRD_BASE/" . get_nodename(); + make_path($dir, { mode => 0755 }) unless -d $dir; + + RRDs::create( + $path, + '--step', '1', + 'DS:gpu_util:GAUGE:120:0:100', + 'DS:mem_util:GAUGE:120:0:100', + 'DS:mem_used:GAUGE:120:0:U', + 'DS:mem_total:GAUGE:120:0:U', + 'DS:power_draw:GAUGE:120:0:U', + 'DS:power_limit:GAUGE:120:0:U', + 'DS:temp_gpu:GAUGE:120:0:U', + 'DS:fan_speed:GAUGE:120:0:100', + 'RRA:AVERAGE:0.5:1:1440', + 'RRA:AVERAGE:0.5:60:1440', + 'RRA:AVERAGE:0.5:1800:1344', + 'RRA:AVERAGE:0.5:21600:1464', + 'RRA:AVERAGE:0.5:604800:520', + 'RRA:MAX:0.5:1:1440', + 'RRA:MAX:0.5:60:1440', + 'RRA:MAX:0.5:1800:1344', + 'RRA:MAX:0.5:21600:1464', + 'RRA:MAX:0.5:604800:520', + ); + my $err = RRDs::error(); + debug(__LINE__, "Created NVIDIA GPU RRD $path: " . ($err // 'OK')); +} + +sub update_nvidia_gpu_rrd { + my ($index, $stats) = @_; + _ensure_nvidia_gpu_rrd($index); + my $card = "nvidia$index"; + my $path = gpu_rrd_path($card); + + my $gpu_util = $stats->{utilization}{gpu} // 'U'; + my $mem_util = $stats->{utilization}{memory} // 'U'; + my $mem_used = $stats->{memory}{used} // 'U'; + my $mem_total = $stats->{memory}{total} // 'U'; + my $power_draw = $stats->{power}{draw} // 'U'; + my $power_limit = $stats->{power}{limit} // 'U'; + my $temp_gpu = $stats->{temperature}{gpu} // 'U'; + my $fan_speed = $stats->{fan}{speed} // 'U'; + + my @daemon_args = (-S $RRD_SOCKET) ? ('--daemon', "unix:$RRD_SOCKET") : (); + RRDs::update( + $path, + @daemon_args, + "N:$gpu_util:$mem_util:$mem_used:$mem_total:$power_draw:$power_limit:$temp_gpu:$fan_speed", + ); + my $err = RRDs::error(); + debug(__LINE__, "RRD update nvidia$index: $err") if $err; +} + +1; diff --git a/PveSensorsV2/PVEMod_Utils.pm b/PveSensorsV2/PVEMod_Utils.pm new file mode 100644 index 0000000..cc83615 --- /dev/null +++ b/PveSensorsV2/PVEMod_Utils.pm @@ -0,0 +1,259 @@ +package PVE::PVEMod::Utils; + +use strict; +use warnings; +use Exporter 'import'; + +use JSON; +use Fcntl qw(O_CREAT O_EXCL O_WRONLY); + +use PVE::PVEMod::Config qw($DEBUG_ENABLED $VERSION $pve_mod_working_dir); + +our @EXPORT_OK = qw( + debug + read_sysfs + is_process_alive + read_lock_pid + acquire_exclusive_lock + ensure_pve_mod_directory_exists + check_executable + pve_mod_hello + setup_collector_signals + safe_write_json + safe_read_json + parse_csv_line +); + +# ============================================================================ +# Debug +# ============================================================================ + +# debug function showing line number and call chain +# Usage: debug(__LINE__, "message") +sub debug { + return unless $DEBUG_ENABLED; + + my ($line, $message) = @_; + + my @caller1 = caller(1); # who called debug() + my @caller2 = caller(2); # parent of caller + + my $sub1 = $caller1[3] || 'main'; + my $sub2 = $caller2[3]; + + $sub1 =~ s/.*:://; + + if (defined $sub2) { + $sub2 =~ s/.*:://; + warn "[$sub2 -> $sub1:$line] $message\n"; + } else { + warn "[$sub1:$line] $message\n"; + } +} + +# ============================================================================ +# File / Process helpers +# ============================================================================ + +sub read_sysfs { + my ($path) = @_; + + return "unknown" unless defined $path && -f $path; + + if (open my $fh, '<', $path) { + my $value = <$fh>; + close $fh; + + if (defined $value) { + chomp $value; + $value =~ s/^\s+|\s+$//g; + return $value ne '' ? $value : "unknown"; + } + } + + return "unknown"; +} + +sub is_process_alive { + my ($pid) = @_; + return -d "/proc/$pid"; +} + +sub read_lock_pid { + my ($lock_path) = @_; + + return undef unless open(my $fh, '<', $lock_path); + + my $pid = <$fh>; + close($fh); + chomp $pid if defined $pid; + + return $pid; +} + +sub acquire_exclusive_lock { + my ($lock_path, $purpose) = @_; + $purpose //= 'lock'; + + my $fh; + + if (sysopen($fh, $lock_path, O_CREAT|O_EXCL|O_WRONLY, 0644)) { + debug(__LINE__, "Acquired $purpose on first try"); + return $fh; + } + + debug(__LINE__, ucfirst($purpose) . " exists, checking if stale"); + + my $lock_pid = read_lock_pid($lock_path); + + if (!defined $lock_pid) { + debug(__LINE__, "Could not read $purpose file: $!"); + return undef; + } + + if ($lock_pid eq '' || $lock_pid !~ /^\d+$/) { + debug(__LINE__, "Invalid PID in $purpose: '" . ($lock_pid // 'undefined') . "', removing"); + unlink($lock_path); + } elsif (is_process_alive($lock_pid)) { + debug(__LINE__, ucfirst($purpose) . " holder PID $lock_pid is still alive"); + return undef; + } else { + debug(__LINE__, ucfirst($purpose) . " holder PID $lock_pid is dead, removing stale lock"); + unlink($lock_path); + } + + unless (sysopen($fh, $lock_path, O_CREAT|O_EXCL|O_WRONLY, 0644)) { + debug(__LINE__, "Failed to acquire $purpose on retry: $!"); + return undef; + } + + debug(__LINE__, "Acquired $purpose after removing stale lock"); + return $fh; +} + +sub ensure_pve_mod_directory_exists { + unless (-d $pve_mod_working_dir) { + debug(__LINE__, "Creating directory $pve_mod_working_dir"); + unless (mkdir($pve_mod_working_dir, 0755)) { + debug(__LINE__, "Failed to create $pve_mod_working_dir: $!. PVE Mod cannot start."); + die "Failed to create $pve_mod_working_dir: $!"; + } + debug(__LINE__, "Directory $pve_mod_working_dir created"); + } else { + debug(__LINE__, "Directory $pve_mod_working_dir already exists"); + } +} + +# Returns 1 if executable exists, or debug mode is active with a debug file present. +# Returns 0 otherwise. +sub check_executable { + my ($exec_path, $type, $debug_mode_enabled, $debug_file) = @_; + + if (defined $debug_mode_enabled && $debug_mode_enabled) { + if (defined $debug_file && -f $debug_file) { + debug(__LINE__, "Debug mode enabled for $type, using debug file: $debug_file"); + return 1; + } elsif (defined $debug_file) { + debug(__LINE__, "Debug mode enabled for $type but debug file missing: $debug_file"); + return 0; + } else { + debug(__LINE__, "Debug mode enabled for $type, skipping executable check for $exec_path"); + return 1; + } + } + + unless (-x $exec_path) { + debug(__LINE__, "$type executable not found or not executable: $exec_path"); + return 0; + } + + debug(__LINE__, "$type executable found: $exec_path"); + return 1; +} + +sub pve_mod_hello { + debug(__LINE__, "PVE Mod is being started. Version $VERSION"); +} + +# Setup common TERM/INT signal handlers for collector processes. +# $shutdown_ref is a scalar ref that will be set to 1 on signal. +sub setup_collector_signals { + my ($name, $shutdown_ref, $extra_cleanup) = @_; + + $SIG{TERM} = sub { + debug(__LINE__, "Collector $name received SIGTERM"); + $$shutdown_ref = 1; + $extra_cleanup->() if $extra_cleanup; + }; + $SIG{INT} = sub { + debug(__LINE__, "Collector $name received SIGINT"); + $$shutdown_ref = 1; + $extra_cleanup->() if $extra_cleanup; + }; +} + +# ============================================================================ +# JSON helpers +# ============================================================================ + +sub safe_write_json { + my ($filepath, $data, $pretty) = @_; + $pretty //= 1; + + eval { + open my $fh, '>', $filepath or die "Failed to open $filepath: $!"; + my $json = $pretty ? JSON->new->pretty->encode($data) : encode_json($data); + print $fh $json; + close $fh; + debug(__LINE__, "Wrote JSON to $filepath"); + }; + if ($@) { + debug(__LINE__, "Error writing to $filepath: $@"); + return 0; + } + return 1; +} + +sub safe_read_json { + my ($filepath, $as_string) = @_; + + return unless -f $filepath; + + my $result; + eval { + open my $fh, '<', $filepath or die "Failed to open $filepath: $!"; + local $/; + my $json = <$fh>; + close $fh; + + if ($as_string) { + $result = $json; + } else { + $result = decode_json($json); + } + debug(__LINE__, "Read JSON from $filepath"); + }; + if ($@) { + debug(__LINE__, "Error reading $filepath: $@"); + return; + } + return $result; +} + +# ============================================================================ +# CSV helper +# ============================================================================ + +sub parse_csv_line { + my ($line, $expected_fields) = @_; + + return unless $line; + $line =~ s/^\s+|\s+$//g; + + my @values = map { s/^\s+|\s+$//gr } split(/,/, $line); + + return unless !$expected_fields || @values >= $expected_fields; + return @values; +} + +1; diff --git a/PveSensorsV2/PveMod_SensorInfo.pm b/PveSensorsV2/PveMod_SensorInfo.pm new file mode 100644 index 0000000..3c99107 --- /dev/null +++ b/PveSensorsV2/PveMod_SensorInfo.pm @@ -0,0 +1,201 @@ +package PVE::API2::PVEMod_SensorInfo; + +use strict; +use warnings; + +use PVE::PVEMod::Config qw(%config $VERSION $stats_dir $sensors_state_file $ups_state_file); +use PVE::PVEMod::Utils qw(debug safe_read_json); +use PVE::PVEMod::ProcessManager qw(pve_mod_starter notify_pve_mod_worker); + +# Per-endpoint state caches (module-level, reset on worker restart) +my $graphics_cache = { data => {}, mtime => 0 }; +my $sensors_cache = { data => '{}', mtime => 0 }; +my $ups_cache = { data => '{}', mtime => 0 }; + + +# ============================================================================ +# Internal helpers +# ============================================================================ + +sub _read_state_file_cached { + my ($files, $cache_ref, $reader, $empty_fallback) = @_; + + # Normalize scalar path to single-element arrayref + my @filepaths = ref($files) eq 'ARRAY' ? @$files : ($files); + + # Find newest mtime across all files + my $newest_mtime = 0; + my $any_exist = 0; + foreach my $fp (@filepaths) { + my @st = stat($fp); + if (@st) { + $any_exist = 1; + $newest_mtime = $st[9] if $st[9] > $newest_mtime; + } + } + + unless ($any_exist) { + debug(__LINE__, "No state files exist: " . join(', ', @filepaths)); + return $cache_ref->{data} // $empty_fallback; + } + + if ($newest_mtime == $cache_ref->{mtime} && defined $cache_ref->{data}) { + debug(__LINE__, "State files unchanged, returning cached data"); + return $cache_ref->{data}; + } + + my $data; + if (ref($reader) eq 'CODE') { + $data = $reader->(\@filepaths); + } else { + $data = safe_read_json($filepaths[0], $reader); + } + + if (!defined $data) { + debug(__LINE__, "Failed to read state file(s): " . join(', ', @filepaths)); + return $cache_ref->{data} // $empty_fallback; + } + + $cache_ref->{data} = $data; + $cache_ref->{mtime} = $newest_mtime; + return $cache_ref->{data}; +} + +sub _merge_graphics_files { + my ($filepaths) = @_; + + my $merged = { + Graphics => { + Intel => {}, + NVIDIA => {}, + AMD => {}, + } + }; + + foreach my $filepath (@$filepaths) { + my ($file) = $filepath =~ m{([^/]+)$}; + debug(__LINE__, "Reading device file: $filepath"); + + my $device_data = safe_read_json($filepath, 0); + if (!$device_data) { + debug(__LINE__, "Failed to read/parse $filepath"); + next; + } + + my $device_type = ($file =~ /^stats-card/) ? 'Intel' + : ($file =~ /^stats-nvidia/) ? 'NVIDIA' + : 'AMD'; + + foreach my $node_name (keys %$device_data) { + $merged->{Graphics}->{$device_type}->{$node_name} = $device_data->{$node_name}; + debug(__LINE__, "Merged $device_type node '$node_name' from $file"); + } + } + + return $merged; +} + +sub _load_graphics_data { + # Build filename patterns for enabled GPU types + my @patterns; + push @patterns, 'card\d+' if $config{gpu}{intel_enabled}; + push @patterns, 'nvidia\d+' if $config{gpu}{nvidia_enabled}; + push @patterns, 'amd\d+' if $config{gpu}{amd_enabled}; + + unless (@patterns) { + debug(__LINE__, "No GPU types enabled in config"); + return $graphics_cache->{data}; + } + + my $pattern = join('|', @patterns); + + # Find device stat files for enabled GPU types + my $dh; + unless (opendir($dh, $stats_dir)) { + debug(__LINE__, "Failed to open stats directory: $stats_dir: $!"); + return $graphics_cache->{data}; + } + + my @stat_files = grep { /^stats-(?:$pattern)\.json$/ } readdir($dh); + closedir($dh); + + unless (@stat_files) { + debug(__LINE__, "No device stat files found in $stats_dir"); + return $graphics_cache->{data}; + } + + debug(__LINE__, "Found " . scalar(@stat_files) . " device stat file(s): " . join(', ', @stat_files)); + + my @filepaths = map { "$stats_dir/$_" } @stat_files; + + my $data = _read_state_file_cached( + \@filepaths, + $graphics_cache, + \&_merge_graphics_files, + { Graphics => { Intel => {}, NVIDIA => {}, AMD => {} } } + ); + + my $intel_count = scalar(keys %{$data->{Graphics}{Intel} // {}}); + my $nvidia_count = scalar(keys %{$data->{Graphics}{NVIDIA} // {}}); + my $amd_count = scalar(keys %{$data->{Graphics}{AMD} // {}}); + debug(__LINE__, "Returning $intel_count Intel + $nvidia_count NVIDIA + $amd_count AMD device node(s)"); + + return $data; +} + +# ============================================================================ +# API calls +# ============================================================================ + +sub get_graphic_info { + debug(__LINE__, "get_graphic_info called"); + + # Start PVE Mod + pve_mod_starter(); + + my $data = _load_graphics_data(); + + # Notify pve_mod_worker of activity + notify_pve_mod_worker(); + + return $data; +} + +sub get_sensors_info { + debug(__LINE__, "get_sensors_info called"); + + # Start PVE Mod + pve_mod_starter(); + + my $data = _read_state_file_cached($sensors_state_file, $sensors_cache, 1, '{}'); + + # Notify pve_mod_worker of activity + notify_pve_mod_worker(); + + return $data; +} + +sub get_ups_info { + debug(__LINE__, "get_ups_info called"); + + # Start PVE Mod + pve_mod_starter(); + + my $data = _read_state_file_cached($ups_state_file, $ups_cache, 1, '{}'); + + # Notify pve_mod_worker of activity + notify_pve_mod_worker(); + + return $data; +} + +sub get_pve_mod_version { + debug(__LINE__, "get_pve_mod_version called"); + # Notify pve_mod_worker of activity + notify_pve_mod_worker(); + + return $VERSION; +} + + +1; diff --git a/PveMod_pvemanagerlib.js b/PveSensorsV2/PveMod_pvemanagerlib.js similarity index 100% rename from PveMod_pvemanagerlib.js rename to PveSensorsV2/PveMod_pvemanagerlib.js diff --git a/PveSensorsV2/pve-mod-gui-sensors-v2.sh b/PveSensorsV2/pve-mod-gui-sensors-v2.sh new file mode 100644 index 0000000..5d5d588 --- /dev/null +++ b/PveSensorsV2/pve-mod-gui-sensors-v2.sh @@ -0,0 +1,1025 @@ +#!/usr/bin/env bash +# +# This bash script installs a modification to the Proxmox Virtual Environment (PVE) web user interface (UI) to display sensors information. +# + +################### Configuration ############# +# Known CPU sensor names. They can be full or partial but should ensure unambiguous identification. +# Should new ones be added, also update logic in configure() function. +KNOWN_CPU_SENSORS=("coretemp-isa-" "k10temp-pci-") + +# Overwrite default backup location +BACKUP_DIR="/root/PVE-MOD2/Backup" + +##################### DO NOT EDIT BELOW ####################### +# Only to be used to debug on other systems. Save the "sensor -j" output into a json file. +# Information will be loaded for script configuration and presented in Proxmox. + +# DEV NOTE: lm-sensors version >3.6.0 breakes properly formatted JSON output using 'sensors -j'. This implements a workaround using uses a python3 for formatting + +DEBUG_REMOTE=false +DEBUG_JSON_FILE="/tmp/sensordata.json" +DEBUG_UPS_FILE="/tmp/upsc.txt" + +# This script's working directory +SCRIPT_CWD="$(dirname "$(readlink -f "$0")")" + +# Debug location +JSON_EXPORT_DIRECTORY="$SCRIPT_CWD" +JSON_EXPORT_FILENAME="sensorsdata.json" + +# File paths +PVE_MANAGER_LIB_JS_FILE="/usr/share/pve-manager/js/pvemanagerlib.js" +PVE_MOD_JS_SOURCE_FILE="$SCRIPT_CWD/PveSensors/PveMod_pvemanagerlib.js" +PVE_MOD_JS_TARGET_FILE="/usr/share/pve-manager/js/PveMod_PveNodeStatusView.js" +NODES_PM_FILE="/usr/share/perl5/PVE/API2/Nodes.pm" +PVE_SENSOR_INFO_MOD_FILE="/usr/share/perl5/PVE/API2/PveMod_SensorInfo.pm" +PVE_SENSOR_INFO_SOURCE_FILE="$SCRIPT_CWD/PveSensors/PveMod_SensorInfo.pm" +GPU_RRD_DIR="/var/lib/rrdcached/db/pve-mod-gpu" + +# New modular architecture paths +PVEMOD_SOURCES_DIR="$SCRIPT_CWD/PveSensors" +PVEMOD_INSTALL_DIR="/usr/share/perl5/PVE/PVEMod" +PVEMOD_COLLECTOR_DIR="$PVEMOD_INSTALL_DIR/Collector" +PVEMOD_CONFIG_FILE="$PVEMOD_INSTALL_DIR/Config.pm" + +#region message tools +# Section header (bold) +function msgb() { + local message="$1" + echo -e "\e[1m${message}\e[0m" +} + +# Info (green) +function info() { + local message="$1" + echo -e "\e[0;32m[info] ${message}\e[0m" +} + +# Warning (yellow) +function warn() { + local message="$1" + echo -e "\e[0;33m[warning] ${message}\e[0m" +} + +# Error (red) +function err() { + local message="$1" + echo -e "\e[0;31m[error] ${message}\e[0m" + exit 1 +} + +# Prompts (cyan or bold) +function ask() { + local prompt="$1" + local response + read -p $'\n\e[1;36m'"${prompt}:"$'\e[0m ' response + echo "$response" +} +#endregion message tools + +# Function to display usage information +function usage { + msgb "\nUsage:\n$0 [install | uninstall | save-sensors-data]\n" + exit 1 +} + +# System checks +function check_root_privileges() { + [[ $EUID -eq 0 ]] || err "This script must be run as root. Please run it with 'sudo $0'." + info "Root privileges verified." +} + +# Define a function to install packages +function install_packages { + # Check if the 'sensors' command is available on the system + if (! command -v sensors &>/dev/null); then + # If the 'sensors' command is not available, prompt the user to install lm-sensors + local choiceInstallLmSensors=$(ask "lm-sensors is not installed. Would you like to install it? (y/n)") + case "$choiceInstallLmSensors" in + [yY]) + # If the user chooses to install lm-sensors, update the package list and install the package + apt-get update + apt-get install lm-sensors + ;; + [nN]) + # If the user chooses not to install lm-sensors, exit the script with a zero status code + msgb "Decided to not install lm-sensors. The mod cannot run without it. Exiting..." + err "lm-sensors is required. Exiting..." + ;; + *) + # If the user enters an invalid input, print an error message and exit the script with a non-zero status code + err "Invalid input. Exiting..." + ;; + esac + fi + + # Check if lm-sensors is installed correctly and exit if not + if (! command -v sensors &>/dev/null); then + err "lm-sensors installation failed or 'sensors' command is not available. Please install lm-sensors manually and re-run the script." + fi +} + +# Main configuration function to detect sensors and set up parameters +function configure { + SENSORS_DETECTED=false + local sensorsOutput + local sanitisedSensorsOutput + local upsOutput + local modelName + local upsConnection + + install_packages + + #### Collect sensor data #### + #region sensors collection + if [ "$DEBUG_REMOTE" = true ]; then + warn "Remote debugging is used. Sensor readings from dump file $DEBUG_JSON_FILE will be used." + warn "Remote debugging is used. UPS readings from dump file $DEBUG_UPS_FILE will be used." + sensorsOutput=$(cat "$DEBUG_JSON_FILE") + else + sensorsOutput=$(sensors -j 2>/dev/null) + fi + + # Apply lm-sensors sanitization + sanitisedSensorsOutput=$(sanitize_sensors_output "$sensorsOutput") + + if [ $? -ne 0 ]; then + err "Sensor output error.\n\nCommand output:\n${sanitisedSensorsOutput}\n\nExiting..." + fi + #endregion sensors collection + + #### CPU #### + #region CPU setup + msgb "\n=== Detecting CPU temperature sensors ===" + ENABLE_CPU=false + local cpuList="" + local cpuCount=0 + + # Find all CPU sensors that match known patterns + for pattern in "${KNOWN_CPU_SENSORS[@]}"; do + found_sensors=$(echo "$sanitisedSensorsOutput" | grep -o "\"${pattern}[^\"]*\"" | sed 's/"//g') + if [ -n "$found_sensors" ]; then + while read -r sensor; do + if [ -n "$sensor" ]; then + cpuCount=$((cpuCount + 1)) + if [ -z "$cpuList" ]; then + cpuList="$sensor" + else + cpuList="$cpuList,$sensor" + fi + ENABLE_CPU=true + fi + done <<< "$found_sensors" + fi + done + + if [ "$ENABLE_CPU" = true ]; then + info "Detected CPU sensors ($cpuCount): $cpuList" + SENSORS_DETECTED=true + while true; do + local choice=$(ask "Display temperatures for all cores [C] or average per CPU [a] (some newer AMD variants support per die)? (C/a)") + case "$choice" in + [cC]|"") + CPU_TEMP_TARGET="Core" + info "Temperatures will be displayed for all cores." + break + ;; + [aA]) + CPU_TEMP_TARGET="Package" + info "An average temperature will be displayed per CPU." + break + ;; + *) + warn "Invalid input, please choose C or a." + ;; + esac + done + else + warn "No CPU temperature sensors found." + fi + #endregion cpu setup + + #### Graphics #### + #region Graphics setup + msgb "\n=== Detecting Graphics information ===" + + #region intel GPU setup + # Check for Intel GPU - ensure intel_gpu_top is installed + if command -v intel_gpu_top &>/dev/null; then + # detect all intel cards using intel_gpu_top -L. Show them line by line + local intelCards + + # Get the output from intel_gpu_top -L, skip empty lines + intelCards=$(intel_gpu_top -L 2>/dev/null | grep -E '^card[0-9]+' || true) + + if [[ -n "$intelCards" ]]; then + local cardCount=$(echo "$intelCards" | wc -l) + echo "Intel GPU(s) detected ($cardCount):" + echo "$intelCards" | while IFS= read -r line; do + # Extract card name, GPU model, and PCI info + if [[ $line =~ ^(card[0-9]+)[[:space:]]+(.+)[[:space:]]+pci:(.+)$ ]]; then + local cardName="${BASH_REMATCH[1]}" + local gpuModel="${BASH_REMATCH[2]}" + local pciInfo="${BASH_REMATCH[3]}" + + echo " - Card: $cardName" + echo " Model: $gpuModel" + echo " PCI: $pciInfo" + else + # Fallback: just show the line as-is + echo " $line" + fi + done + ENABLE_INTEL_GPU_INFO=true + ENABLE_GPU_INFO=true + else + warn "No Intel GPUs detected by intel_gpu_top." + ENABLE_INTEL_GPU_INFO=false + fi + else + warn "intel_gpu_top command not found. Skipping Intel GPU information detection." + ENABLE_INTEL_GPU_INFO=false + fi + #endregion intel GPU setup + + #region NVIDIA GPU setup + # Check for NVIDIA GPU - ensure nvidia-smi is installed + if command -v nvidia-smi &>/dev/null; then + # detect all NVIDIA cards using nvidia-smi -L + local nvidiaCards + + # Get the output from nvidia-smi -L (lists GPUs) + nvidiaCards=$(nvidia-smi -L 2>/dev/null || true) + + if [[ -n "$nvidiaCards" ]]; then + local cardCount=$(echo "$nvidiaCards" | wc -l) + echo "NVIDIA GPU(s) detected ($cardCount):" + echo "$nvidiaCards" | while IFS= read -r line; do + # Expected format: GPU 0: NVIDIA GeForce RTX 3080 (UUID: GPU-xxxxx) + if [[ $line =~ ^GPU\ ([0-9]+):\ (.+)\ \(UUID:\ (.+)\)$ ]]; then + local gpuIndex="${BASH_REMATCH[1]}" + local gpuModel="${BASH_REMATCH[2]}" + local gpuUUID="${BASH_REMATCH[3]}" + + echo " - GPU $gpuIndex" + echo " Model: $gpuModel" + echo " UUID: $gpuUUID" + else + # Fallback: just show the line as-is + echo " $line" + fi + done + ENABLE_NVIDIA_GPU_INFO=true + ENABLE_GPU_INFO=true + else + warn "No NVIDIA GPUs detected by nvidia-smi." + ENABLE_NVIDIA_GPU_INFO=false + fi + else + warn "nvidia-smi command not found. Skipping NVIDIA GPU information detection." + ENABLE_NVIDIA_GPU_INFO=false + fi + #endregion NVIDIA GPU setup + + #region AMD GPU setup + # not implemented yet + #endregion AMD GPU setup + + #endregion Graphics setup + + #### GPU Historical Data #### + #region gpu history setup + ENABLE_GPU_HISTORY=false + if [[ "$ENABLE_GPU_INFO" == true ]]; then + msgb "\n=== GPU Historical Data ===" + local choiceGpuHistory + choiceGpuHistory=$(ask "Store historical GPU data for graphs? (y/N)") + case "$choiceGpuHistory" in + [yY]) + ENABLE_GPU_HISTORY=true + info "Historical GPU data will be stored." + ;; + [nN]|"") + info "Historical GPU data will not be stored." + ;; + *) + warn "Invalid selection. Historical GPU data will not be stored." + ;; + esac + fi + #endregion gpu history setup + + #### RAM #### + #region ram setup + local ramList ramCount + msgb "\n=== Detecting RAM temperature sensors ===" + ramList=$(echo "$sanitisedSensorsOutput" | grep -o '"SODIMM[^"]*"' | sed 's/"//g' | paste -sd, -) + ramCount=$(grep -c '"SODIMM[^"]*"' <<<"$sanitisedSensorsOutput") + + if [ "$ramCount" -gt 0 ]; then + info "Detected RAM sensors ($ramCount): $ramList" + ENABLE_RAM_TEMP=true + SENSORS_DETECTED=true + else + warn "No RAM temperature sensors found." + ENABLE_RAM_TEMP=false + fi + #endregion ram setup + + #### HDD/SSD #### + #region hdd setup + msgb "\n=== Detecting HDD/SSD temperature sensors ===" + local hddList=($(echo "$sanitisedSensorsOutput" | grep -o '"drivetemp-scsi[^"]*"' | sed 's/"//g')) + if [ ${#hddList[@]} -gt 0 ]; then + info "Detected HDD/SSD sensors (${#hddList[@]}): $(IFS=,; echo "${hddList[*]}")" + ENABLE_HDD_TEMP=true + SENSORS_DETECTED=true + else + warn "No HDD/SSD temperature sensors found." + ENABLE_HDD_TEMP=false + fi + #endregion hdd setup + + #### NVMe #### + #region nvme setup + msgb "\n=== Detecting NVMe temperature sensors ===" + local nvmeList=($(echo "$sanitisedSensorsOutput" | grep -o '"nvme[^"]*"' | sed 's/"//g')) + if [ ${#nvmeList[@]} -gt 0 ]; then + info "Detected NVMe sensors (${#nvmeList[@]}): $(IFS=,; echo "${nvmeList[*]}")" + ENABLE_NVME_TEMP=true + SENSORS_DETECTED=true + else + warn "No NVMe temperature sensors found." + ENABLE_NVME_TEMP=false + fi + #endregion nvme setup + + #### Fans #### + #region fan setup + msgb "\n=== Detecting fan speed sensors ===" + + local fanList="" + local fanCount=0 + + # Find all fan names that have fan*_input entries + fanList=$(echo "$sanitisedSensorsOutput" | grep -B2 '"fan[0-9]\+_input"' | grep '".*": {' | sed 's/.*"\([^"]*\)": {.*/\1/' | sort -u | paste -sd, -) + fanCount=$(grep -c 'fan[0-9]\+_input' <<<"$sanitisedSensorsOutput") + + if [ "$fanCount" -gt 0 ]; then + info "Detected fan speed sensors ($fanCount): $fanList" + ENABLE_FAN_SPEED=true + SENSORS_DETECTED=true + + local choice + choice=$(ask "Display fans reporting zero speed? (Y/n)") + case "$choice" in + [yY]|"") + DISPLAY_ZERO_SPEED_FANS=true + info "Zero-speed fans will be displayed." + ;; + [nN]) + DISPLAY_ZERO_SPEED_FANS=false + info "Only active fans will be displayed." + ;; + *) + warn "Invalid input. Defaulting to show zero-speed fans." + DISPLAY_ZERO_SPEED_FANS=true + ;; + esac + else + warn "No fan speed sensors found." + ENABLE_FAN_SPEED=false + fi + #endregion fan setup + + #### Temperature Units #### + #region temp unit setup + msgb "\n=== Display temperature ===" + if [ "$SENSORS_DETECTED" = true ]; then + local unit=$(ask "Display temperatures in Celsius [C] or Fahrenheit [f]? (C/f)") + case "$unit" in + [cC]|"") + TEMP_UNIT="C" + info "Using Celsius." + ;; + [fF]) + TEMP_UNIT="F" + info "Using Fahrenheit." + ;; + *) + warn "Invalid selection. Defaulting to Celsius." + TEMP_UNIT="C" + ;; + esac + fi + #endregion temp unit setup + + #### UPS #### + #region ups setup + local choiceUPS=$(ask "Enable UPS information? (y/N)") + case "$choiceUPS" in + [yY]) + if [ "$DEBUG_REMOTE" = true ]; then + upsOutput=$(cat "$DEBUG_UPS_FILE") + info "Remote debugging: UPS readings from $DEBUG_UPS_FILE" + upsConnection="DEBUG_UPS" + else + upsConnection=$(ask "Enter UPS connection (e.g., upsname[@hostname[:port]])") + if ! command -v upsc &>/dev/null; then + err "The 'upsc' command is not available. Install 'nut-client'." + fi + upsOutput=$(upsc "$upsConnection" 2>&1) + fi + + if echo "$upsOutput" | grep -q "device.model:"; then + modelName=$(echo "$upsOutput" | grep "device.model:" | cut -d':' -f2- | xargs) + ENABLE_UPS=true + info "Connected to UPS model: $modelName at $upsConnection." + else + warn "Failed to connect to UPS at '$upsConnection'." + ENABLE_UPS=false + fi + ;; + [nN]|"") + ENABLE_UPS=false + info "UPS information will not be displayed." + ;; + *) + warn "Invalid selection. UPS info will not be displayed." + ENABLE_UPS=false + ;; + esac + #endregion ups setup + + #### System Info #### + #region system info setup + msgb "\n=== Detecting System Information ===" + for i in 1 2; do + echo "type ${i})" + dmidecode -t "$i" | awk -F': ' '/Manufacturer|Product Name|Serial Number/ {print $1": "$2}' + done + local choiceSysInfo=$(ask "Enable system information? (1/2/n)") + case "$choiceSysInfo" in + [1]|"") + ENABLE_SYSTEM_INFO=true + SYSTEM_INFO_TYPE=1 + info "System information will be displayed." + ;; + [2]) + ENABLE_SYSTEM_INFO=true + SYSTEM_INFO_TYPE=2 + info "Motherboard information will be displayed." + ;; + [nN]) + ENABLE_SYSTEM_INFO=false + info "System information will NOT be displayed." + ;; + *) + warn "Invalid selection. Defaulting to system information." + ENABLE_SYSTEM_INFO=true + SYSTEM_INFO_TYPE=1 + ;; + esac + #endregion system info setup + + #### Final Check #### + #region final check + if [ "$SENSORS_DETECTED" = false ] && [ "$ENABLE_UPS" = false ] && [ "$ENABLE_SYSTEM_INFO" = false ]; then + err "No sensors detected, UPS or system info enabled. Exiting." + fi + #endregion final check +} + + +# Function to install the modification +function install_mod { + msgb "\n=== Preparing mod installation ===" + check_root_privileges + check_mod_installation + configure + perform_backup + + #### Insert information retrieval code #### + msgb "\n=== Installing sensor info module ===" + install_sensor_monitor_module + insert_sensor_monitor_into_pve + insert_system_info_into_pve + + ## Historical GPU data ## + if [[ "$ENABLE_GPU_HISTORY" == true ]]; then + install_gpu_history + + # todo + + # add nodes.pm index list — add gpumeta after gpurrddata: + + fi + + #### Install UI modification module #### + msgb "\n=== Installing UI modification module ===" + install_node_status_view_module + + msgb "\n=== Finalizing installation ===" + + restart_proxy + info "Installation completed." + ask "Clear the browser cache to ensure all changes are visualized. (any key to continue)" +} + +# Sanitize sensors output to handle common lm-sensors parsing issues +sanitize_sensors_output() { + local input="$1" + + # Pipe the text into Perl: + # -0777 → "slurp mode": read the entire stream as one string so + # regexes can match across line breaks. + # -pe → loop over input, applying the script (-e) and printing. + # Apply python3 json.tool for proper formatting and validation + echo "$input" | perl -0777 -pe ' + # Replace ERROR lines with placeholder values + s/ERROR:.+\s(\w+):\s(.+)/"$1": 0.000,/g; + s/ERROR:.+\s(\w+)!/"$1": 0.000,/g; + + # Remove trailing commas before closing braces + s/,\s*(\})/$1/g; + + # Replace NaN values with null + s/\bNaN\b/null/g; + + # Fix duplicate SODIMM keys - handle both pretty and one-line JSON + s/"SODIMM"\s*:\s*\{\s*"temp(\d+)_input"/"SODIMM $1": {\n "temp$1_input"/g; + + # Fix duplicate fan keys - handle both pretty and one-line JSON + s/"([^"]*Fan[^"]*)"\s*:\s*\{\s*"fan(\d+)_input"/"$1 $2": {\n "fan$2_input"/g; + ' | python3 -m json.tool 2>/dev/null || echo "$input" +} + +#region Sensor Monitor Module Installation +# Install and configure the Sensor Monitor Perl module +install_sensor_monitor_module() { + local intel_enabled nvidia_enabled ups_enabled sensors_mode + + # Check if API facade source file exists + if [[ ! -f "$PVE_SENSOR_INFO_SOURCE_FILE" ]]; then + err "Source file not found: $PVE_SENSOR_INFO_SOURCE_FILE" + fi + + # Create PVEMod module directories + mkdir -p "$PVEMOD_INSTALL_DIR" || err "Failed to create directory: $PVEMOD_INSTALL_DIR" + mkdir -p "$PVEMOD_COLLECTOR_DIR" || err "Failed to create directory: $PVEMOD_COLLECTOR_DIR" + info "PVEMod module directories ready." + + # Copy the API facade module + cp "$PVE_SENSOR_INFO_SOURCE_FILE" "$PVE_SENSOR_INFO_MOD_FILE" || err "Failed to copy $PVE_SENSOR_INFO_SOURCE_FILE to $PVE_SENSOR_INFO_MOD_FILE" + info "Copied API facade to $PVE_SENSOR_INFO_MOD_FILE" + + # Copy the new modular support files + local -a module_files=( + "PVEMod_Config.pm:$PVEMOD_INSTALL_DIR/Config.pm" + "PVEMod_Utils.pm:$PVEMOD_INSTALL_DIR/Utils.pm" + "PVEMod_Store.pm:$PVEMOD_INSTALL_DIR/Store.pm" + "PVEMod_ProcessManager.pm:$PVEMOD_INSTALL_DIR/ProcessManager.pm" + "PVEMod_Collectors/Intel.pm:$PVEMOD_COLLECTOR_DIR/Intel.pm" + "PVEMod_Collectors/Nvidia.pm:$PVEMOD_COLLECTOR_DIR/Nvidia.pm" + "PVEMod_Collectors/LmSensors.pm:$PVEMOD_COLLECTOR_DIR/LmSensors.pm" + "PVEMod_Collectors/Amd.pm:$PVEMOD_COLLECTOR_DIR/Amd.pm" + "PVEMod_Collectors/Ups.pm:$PVEMOD_COLLECTOR_DIR/Ups.pm" + ) + for entry in "${module_files[@]}"; do + local src="$PVEMOD_SOURCES_DIR/${entry%%:*}" + local dst="${entry##*:}" + if [[ ! -f "$src" ]]; then + err "Module source file not found: $src" + fi + cp "$src" "$dst" || err "Failed to copy $src to $dst" + info "Copied $(basename "$src") to $dst" + done + + # Convert boolean flags to Perl format (1 or 0) + intel_enabled=$([[ "$ENABLE_INTEL_GPU_INFO" = true ]] && echo 1 || echo 0) + nvidia_enabled=$([[ "$ENABLE_NVIDIA_GPU_INFO" = true ]] && echo 1 || echo 0) + ups_enabled=$([[ "$ENABLE_UPS" = true ]] && echo 1 || echo 0) + + # Determine UPS device name + local ups_device="${upsConnection:-ups@localhost}" + + # Patch configuration values in Config.pm (the canonical config location) + sed -i " + /intel_enabled =>/ s/=> [01],/=> $intel_enabled,/ + /nvidia_enabled =>/ s/=> [01],/=> $nvidia_enabled,/ + /enabled =>/ s/=> [01],/=> $ups_enabled,/ + /device_name =>/ s|=> '[^']*',|=> '$ups_device',| + " "$PVEMOD_CONFIG_FILE" + + if [[ $? -eq 0 ]]; then + info "Config.pm patched successfully." + else + warn "Failed to patch Config.pm settings." + fi +} +#endregion Sensor Monitor Module Installation + +#region node info insertion +# Main insertion routine +insert_sensor_monitor_into_pve() { + #region PveSensorInfoMod heredoc + sed -i '/my \$dinfo = df('\''\/'\'', 1);/i\ + # Collect sensor data from PveMod_SensorInfo\ + # Bad practice to add use here, but cleaner implementation would require several extensive modifications.\ + use PVE::API2::PVEMod_SensorInfo;\ + $res->{PveMod_JsonSensorInfo} = PVE::API2::PVEMod_SensorInfo::get_sensors_info();\ + $res->{PveMod_graphicsInfo} = PVE::API2::PVEMod_SensorInfo::get_pve_mod_version();\ + $res->{PveMod_upsInfo} = PVE::API2::PVEMod_SensorInfo::get_ups_info();\ + ' "$NODES_PM_FILE" + #endregion PveSensorInfoMod heredoc + info "Sensor data retriever added to \"$NODES_PM_FILE\"." +} + +# Collect system information +insert_system_info_into_pve() { + local output_file="$1" + local systemInfoCmd + + if [[ $ENABLE_SYSTEM_INFO == false ]]; then + return + fi + + systemInfoCmd=$(dmidecode -t "${SYSTEM_INFO_TYPE}" \ + | awk -F': ' '/Manufacturer|Product Name|Serial Number/ {print $1": "$2}' \ + | awk '{$1=$1};1' \ + | sed 's/$/ |/' \ + | paste -sd " " - \ + | sed 's/ |$//') + #region system info heredoc + sed -i "/my \$dinfo = df('\/', 1);/i\\ + # Add system information to response\\ + \$res->{pveMod_sensorInfo_systemInfo} = \"$(echo "$systemInfoCmd")\";\\ +" "$NODES_PM_FILE" + #endregion system info heredoc + info "System information retriever added to \"$output_file\"." +} +#endregion node info insertion + +#region UI Module Installation +# Install the UI modification module +install_node_status_view_module() { + # Check if source file exists + if [[ ! -f "$PVE_MOD_JS_SOURCE_FILE" ]]; then + err "Source file not found: $PVE_MOD_JS_SOURCE_FILE" + fi + + # Copy the JavaScript module to PVE manager directory + cp "$PVE_MOD_JS_SOURCE_FILE" "$PVE_MOD_JS_TARGET_FILE" || err "Failed to copy $PVE_MOD_JS_SOURCE_FILE to $PVE_MOD_JS_TARGET_FILE" + info "Copied UI module to $PVE_MOD_JS_TARGET_FILE" + + # Comment out the original PVE.node.StatusView definition in pvemanagerlib.js + # This allows our custom module to provide the new definition + if grep -q "^Ext.define('PVE.node.StatusView'," "$PVE_MANAGER_LIB_JS_FILE" 2>/dev/null; then + # Find the start of the definition and comment it out until the matching closing brace + sed -i "/^Ext\.define('PVE\.node\.StatusView',/,/^});/s|^|// |" "$PVE_MANAGER_LIB_JS_FILE" + info "Commented out original StatusView definition in pvemanagerlib.js" + else + warn "Original StatusView definition not found in expected format in pvemanagerlib.js" + fi + + # Comment out the original PVE.node.Summary definition in pvemanagerlib.js + if grep -q "^Ext.define('PVE.node.Summary'," "$PVE_MANAGER_LIB_JS_FILE" 2>/dev/null; then + sed -i "/^Ext\.define('PVE\.node\.Summary',/,/^});/s|^|// |" "$PVE_MANAGER_LIB_JS_FILE" + info "Commented out original Summary definition in pvemanagerlib.js" + else + warn "Original Summary definition not found in expected format in pvemanagerlib.js" + fi + + # Add a dynamic script loader to load our custom module + # Insert before the commented-out Ext.define to load it + if ! grep -q "PveMod_PveNodeStatusView.js" "$PVE_MANAGER_LIB_JS_FILE" 2>/dev/null; then + # Use ExtJS Loader to dynamically load our custom module + sed -i "/^\/\/ Ext\.define('PVE\.node\.StatusView',/i\\ +// Load custom PVE.node.StatusView from external module\\ +Ext.Loader.loadScript({\\ + url: '/pve2/js/PveMod_PveNodeStatusView.js',\\ + onLoad: function() { },\\ + onError: function() { console.error('Failed to load PveMod_PveNodeStatusView.js'); }\\ +});\\ +" "$PVE_MANAGER_LIB_JS_FILE" + info "Added dynamic loader for custom UI module in pvemanagerlib.js" + else + info "Custom UI module loader already present in pvemanagerlib.js" + fi +} +#endregion UI Module Installation + + +#region historical GPU data +# Install GPU historical data storage: RRD directory + API endpoint in Nodes.pm +install_gpu_history() { + msgb "\n=== Installing GPU historical data support ===" + + # Create and configure the RRD directory + mkdir -p "$GPU_RRD_DIR" || err "Failed to create GPU RRD directory: $GPU_RRD_DIR" + chown www-data:www-data "$GPU_RRD_DIR" || err "Failed to set ownership on: $GPU_RRD_DIR" + info "GPU RRD directory ready: $GPU_RRD_DIR" + + # Register gpurrddata in the node method list + sed -i "s/{ name => 'rrddata' },/{ name => 'rrddata' },\n { name => 'gpurrddata' },/" "$NODES_PM_FILE" \ + || err "Failed to register gpurrddata method in $NODES_PM_FILE" + + # Insert the gpurrddata API method definition after the rrddata method + sed -i "/\"pve-node-9\.0\/\$param->{node}\", \$param->{timeframe}, \$param->{cf},/{ +n +n +a\\ +\\ +__PACKAGE__->register_method({\\ + name => 'gpurrddata',\\ + path => 'gpurrddata',\\ + method => 'GET',\\ + protected => 1,\\ + proxyto => 'node',\\ + permissions => {\\ + check => ['perm', '/nodes/{node}', ['Sys.Audit']],\\ + },\\ + description => \"Read GPU RRD statistics\",\\ + parameters => {\\ + additionalProperties => 0,\\ + properties => {\\ + node => get_standard_option('pve-node'),\\ + card => {\\ + description => \"The GPU card identifier (e.g. card0, nvidia0).\",\\ + type => 'string',\\ + pattern => '[a-zA-Z0-9]+',\\ + },\\ + timeframe => {\\ + description => \"Specify the time frame you are interested in.\",\\ + type => 'string',\\ + enum => ['hour', 'day', 'week', 'month', 'year', 'decade'],\\ + },\\ + cf => {\\ + description => \"The RRD consolidation function\",\\ + type => 'string',\\ + enum => ['AVERAGE', 'MAX'],\\ + optional => 1,\\ + },\\ + },\\ + },\\ + returns => {\\ + type => \"array\",\\ + items => {\\ + type => \"object\",\\ + properties => {},\\ + },\\ + },\\ + code => sub {\\ + my (\$param) = \@_;\\ + my \$nodename = PVE::INotify::nodename();\\ + my \$card = \$param->{card};\\ + die \"invalid card name\\n\" unless \$card =~ /^[a-zA-Z0-9]+\$/;\\ + return PVE::RRD::create_rrd_data(\\ + \"pve-mod-gpu/\$nodename/\$card\", \$param->{timeframe}, \$param->{cf},\\ + );\\ + },\\ +}); +}" "$NODES_PM_FILE" \ + || err "Failed to insert gpurrddata method in $NODES_PM_FILE" + + info "GPU historical data API endpoint added to $NODES_PM_FILE" +} +#endregion historical GPU data + +# Function to uninstall the modification +function uninstall_mod { + msgb "=== Uninstalling Mod ===" + + check_root_privileges + + if [[ -z $(grep -e "\$res->{PveMod_SensorInfo_JSON}" "$NODES_PM_FILE") ]] && [[ -z $(grep -e "\$res->{systemInfo}" "$NODES_PM_FILE") ]]; then + err "Mod is not installed." + fi + + set_backup_directory + info "Restoring modified files..." + + # Find the latest Nodes.pm file using the find command + local latest_nodes_pm=$(find "$BACKUP_DIR" -name "Nodes.pm.*" -type f -printf '%T+ %p\n' 2>/dev/null | sort -r | head -n 1 | awk '{print $2}') + + if [ -n "$latest_nodes_pm" ]; then + # Restore the latest Nodes.pm file + msgb "Restoring latest Nodes.pm from backup: $latest_nodes_pm to \"$NODES_PM_FILE\"." + cp "$latest_nodes_pm" "$NODES_PM_FILE" + info "Restored Nodes.pm successfully." + else + warn "No Nodes.pm backup files found." + fi + + # Restore original pvemanagerlib.js (uncomment the StatusView definition and remove loader) + local latest_pvemanagerlibjs=$(find "$BACKUP_DIR" -name "pvemanagerlib.js.*" -type f -printf '%T+ %p\n' 2>/dev/null | sort -r | head -n 1 | awk '{print $2}') + + if [ -n "$latest_pvemanagerlibjs" ]; then + # Restore the latest pvemanagerlib.js file + msgb "Restoring latest pvemanagerlib.js from backup: $latest_pvemanagerlibjs to \"$PVE_MANAGER_LIB_JS_FILE\"." + cp "$latest_pvemanagerlibjs" "$PVE_MANAGER_LIB_JS_FILE" + info "Restored pvemanagerlib.js successfully." + else + warn "No pvemanagerlib.js backup files found." + fi + + # Remove UI module files + if [ -f "$PVE_MOD_JS_TARGET_FILE" ]; then + msgb "Removing UI module: $PVE_MOD_JS_TARGET_FILE" + rm "$PVE_MOD_JS_TARGET_FILE" + info "Removed UI module successfully." + else + warn "UI module file not found: $PVE_MOD_JS_TARGET_FILE" + fi + + # Remove Sensor Info Perl module + local latest_sensor_info_pm=$(find "$BACKUP_DIR" -name "PveMod_SensorInfo.pm.*" -type f -printf '%T+ %p\n' 2>/dev/null | sort -r | head -n 1 | awk '{print $2}') + + if [ -n "$latest_sensor_info_pm" ]; then + # Restore the latest PveMod_SensorInfo.pm file (if there's a backup) + msgb "Restoring latest PveMod_SensorInfo.pm from backup: $latest_sensor_info_pm to \"$PVE_SENSOR_INFO_MOD_FILE\"." + cp "$latest_sensor_info_pm" "$PVE_SENSOR_INFO_MOD_FILE" + info "Restored PveMod_SensorInfo.pm successfully." + elif [ -f "$PVE_SENSOR_INFO_MOD_FILE" ]; then + # No backup found but file exists, remove it + msgb "No PveMod_SensorInfo.pm backup found. Removing installed module: $PVE_SENSOR_INFO_MOD_FILE" + rm "$PVE_SENSOR_INFO_MOD_FILE" + info "Removed PveMod_SensorInfo.pm successfully." + else + warn "No PveMod_SensorInfo.pm backup files found and module not installed." + fi + + # Remove PVEMod support modules (Config, Utils, Store, ProcessManager, Collectors) + if [[ -d "$PVEMOD_INSTALL_DIR" ]]; then + msgb "Removing PVEMod module directory: $PVEMOD_INSTALL_DIR" + rm -rf "$PVEMOD_INSTALL_DIR" + info "Removed PVEMod module directory." + fi + + # Remove GPU RRD directory (contains historical GPU data — destroyed permanently on uninstall) + if [[ -d "$GPU_RRD_DIR" ]]; then + msgb "Removing GPU RRD directory: $GPU_RRD_DIR" + rm -rf "$GPU_RRD_DIR" + info "Removed GPU RRD directory." + fi + + if [ -n "$latest_nodes_pm" ] || [ -n "$latest_pvemanagerlibjs" ] || [ -f "$PVE_MOD_JS_TARGET_FILE" ] || [ -n "$latest_sensor_info_pm" ] || [ -f "$PVE_SENSOR_INFO_MOD_FILE" ]; then + # At least one file was modified, restart the proxy + restart_proxy + fi + + ask "Clear the browser cache to ensure all changes are visualized. (any key to continue)" +} + +# Function to check if the modification is installed +check_mod_installation() { + if [[ -n $(grep -F 'use PVE::API2::PveMod_SensorInfo' "$NODES_PM_FILE") ]] || \ + [[ -n $(grep -F 'use PVE::API2::PVEMod_SensorInfo' "$NODES_PM_FILE") ]] || \ + [[ -n $(grep -F '$res->{sensorsJSONOutput}' "$NODES_PM_FILE") ]] || \ + [[ -n $(grep -F '$res->{systemInfo}' "$NODES_PM_FILE") ]] || \ + [[ -f "$PVE_MOD_JS_TARGET_FILE" ]]; then + err "Mod is already installed. Uninstall existing before installing." + fi +} + +function restart_proxy { + # Restart pveproxy + info "Restarting PVE proxy..." + systemctl restart pveproxy +} + +function save_sensors_data { + msgb "=== Saving Sensors Data ===" + + # Check if JSON_EXPORT_DIRECTORY exists and is writable + if [[ ! -d "$JSON_EXPORT_DIRECTORY" || ! -w "$JSON_EXPORT_DIRECTORY" ]]; then + err "Directory $JSON_EXPORT_DIRECTORY does not exist or is not writable. No file could be saved." + return + fi + + + # Check if command exists + if (command -v sensors &>/dev/null); then + # Save sensors output + local debug_save_filename="sensorsdata.json" + local filepath="${JSON_EXPORT_DIRECTORY}/${debug_save_filename}" + msgb "Sensors data will be saved in $filepath" + + # Prompt user for confirmation + local choiceContinue=$(ask "Do you wish to continue? (Y/n)") + case "$choiceContinue" in + [yY]|"") + echo "lm-sensors raw output:" >"$filepath" + sensorsOutput=$(sensors -j 2>/dev/null) + echo "$sensorsOutput" >>"$filepath" + echo -e "\n\nSanitised lm-sensors output:" >>"$filepath" + # Apply lm-sensors sanitization + sanitisedSensorsOutput=$(sanitize_sensors_output "$sensorsOutput") + echo "$sanitisedSensorsOutput" >>"$filepath" + info "Sensors data saved in $filepath." + ;; + *) + warn "Operation cancelled by user." + ;; + esac + else + err "Sensors is not installed. No file could be saved." + fi + echo +} + +function set_backup_directory { + # Check if the BACKUP_DIR variable is set, if not, use the default backup + if [[ -z "$BACKUP_DIR" ]]; then + # If not set, use the default backup directory, which is based on the home directory and PVE-MODS + BACKUP_DIR="$HOME/PVE-MODS" + info "Using default backup directory: $BACKUP_DIR" + else + # If set, ensure it is a valid directory + if [[ ! -d "$BACKUP_DIR" ]]; then + err "The specified backup directory does not exist: $BACKUP_DIR" + fi + info "Using custom backup directory: $BACKUP_DIR" + fi +} + +function create_backup_directory { + set_backup_directory + + # Create the backup directory if it does not exist + if [[ ! -d "$BACKUP_DIR" ]]; then + mkdir -p "$BACKUP_DIR" 2>/dev/null || { + err "Failed to create backup directory: $BACKUP_DIR. Please check permissions." + } + info "Created backup directory: $BACKUP_DIR" + else + info "Backup directory already exists: $BACKUP_DIR" + fi +} + +function create_file_backup() { + local source_file="$1" + local timestamp="$2" + local filename + + filename=$(basename "$source_file") + local backup_file="$BACKUP_DIR/${filename}.$timestamp" + + [[ -f "$source_file" ]] || err "Source file does not exist: $source_file" + [[ -r "$source_file" ]] || err "Cannot read source file: $source_file" + + cp "$source_file" "$backup_file" || err "Failed to create backup: $backup_file" + + # Verify backup integrity + if ! cmp -s "$source_file" "$backup_file"; then + err "Backup verification failed for: $backup_file" + fi + + info "Created backup: $backup_file" +} + +function perform_backup { + local timestamp + timestamp=$(date +%Y%m%d_%H%M%S) + + msgb "\n=== Creating backups of modified files ===" + + create_backup_directory + create_file_backup "$NODES_PM_FILE" "$timestamp" + create_file_backup "$PVE_MANAGER_LIB_JS_FILE" "$timestamp" + + # Backup Sensor Info module if it exists + if [[ -f "$PVE_SENSOR_INFO_MOD_FILE" ]]; then + create_file_backup "$PVE_SENSOR_INFO_MOD_FILE" "$timestamp" + fi +} + +# Process the arguments using a while loop and a case statement +executed=0 +while [[ $# -gt 0 ]]; do + case "$1" in + install) + executed=$(($executed + 1)) + msgb "\nInstalling the Proxmox VE sensors display mod..." + install_packages + install_mod + ;; + uninstall) + executed=$(($executed + 1)) + msgb "\nUninstalling the Proxmox VE sensors display mod..." + uninstall_mod + ;; + save-sensors-data) + executed=$(($executed + 1)) + msgb "\nSaving current sensor readings in a file for debugging..." + save_sensors_data + ;; + esac + shift +done + +# If no arguments were provided or all arguments have been processed, print the usage message +if [[ $executed -eq 0 ]]; then + usage +fi \ No newline at end of file diff --git a/pve-mod-gui-sensors.sh b/pve-mod-gui-sensors.sh index d7beb34..f72e81f 100644 --- a/pve-mod-gui-sensors.sh +++ b/pve-mod-gui-sensors.sh @@ -4,12 +4,20 @@ # ################### Configuration ############# + +# Display configuration for HDD, NVME, CPU +# Set to 0 to disable line breaks +# Note: use these settings only if the displayed layout is broken +CPU_ITEMS_PER_ROW=0 +NVME_ITEMS_PER_ROW=0 +HDD_ITEMS_PER_ROW=0 + # Known CPU sensor names. They can be full or partial but should ensure unambiguous identification. # Should new ones be added, also update logic in configure() function. KNOWN_CPU_SENSORS=("coretemp-isa-" "k10temp-pci-") # Overwrite default backup location -BACKUP_DIR="/root/PVE-MOD2/Backup" +BACKUP_DIR="" ##################### DO NOT EDIT BELOW ####################### # Only to be used to debug on other systems. Save the "sensor -j" output into a json file. @@ -30,12 +38,7 @@ JSON_EXPORT_FILENAME="sensorsdata.json" # File paths PVE_MANAGER_LIB_JS_FILE="/usr/share/pve-manager/js/pvemanagerlib.js" -PVE_MOD_JS_SOURCE_FILE="$SCRIPT_CWD/PveMod_PveNodeStatusView.js" -PVE_MOD_JS_TARGET_FILE="/usr/share/pve-manager/js/PveMod_PveNodeStatusView.js" NODES_PM_FILE="/usr/share/perl5/PVE/API2/Nodes.pm" -PVE_SENSOR_INFO_MOD_FILE="/usr/share/perl5/PVE/API2/PveMod_SensorInfo.pm" -PVE_SENSOR_INFO_SOURCE_FILE="$SCRIPT_CWD/PveMod_SensorInfo.pm" -GPU_RRD_DIR="/var/lib/rrdcached/db/pve-mod-gpu" #region message tools # Section header (bold) @@ -114,18 +117,16 @@ function install_packages { fi } -# Main configuration function to detect sensors and set up parameters function configure { SENSORS_DETECTED=false local sensorsOutput local sanitisedSensorsOutput local upsOutput local modelName - local upsConnection install_packages - #### Collect sensor data #### + #### Collect lm-sensors output #### #region sensors collection if [ "$DEBUG_REMOTE" = true ]; then warn "Remote debugging is used. Sensor readings from dump file $DEBUG_JSON_FILE will be used." @@ -144,7 +145,7 @@ function configure { #endregion sensors collection #### CPU #### - #region CPU setup + #region cpu setup msgb "\n=== Detecting CPU temperature sensors ===" ENABLE_CPU=false local cpuList="" @@ -193,123 +194,12 @@ function configure { warn "No CPU temperature sensors found." fi #endregion cpu setup - - #### Graphics #### - #region Graphics setup - msgb "\n=== Detecting Graphics information ===" - - #region intel GPU setup - # Check for Intel GPU - ensure intel_gpu_top is installed - if command -v intel_gpu_top &>/dev/null; then - # detect all intel cards using intel_gpu_top -L. Show them line by line - local intelCards - - # Get the output from intel_gpu_top -L, skip empty lines - intelCards=$(intel_gpu_top -L 2>/dev/null | grep -E '^card[0-9]+' || true) - - if [[ -n "$intelCards" ]]; then - local cardCount=$(echo "$intelCards" | wc -l) - echo "Intel GPU(s) detected ($cardCount):" - echo "$intelCards" | while IFS= read -r line; do - # Extract card name, GPU model, and PCI info - if [[ $line =~ ^(card[0-9]+)[[:space:]]+(.+)[[:space:]]+pci:(.+)$ ]]; then - local cardName="${BASH_REMATCH[1]}" - local gpuModel="${BASH_REMATCH[2]}" - local pciInfo="${BASH_REMATCH[3]}" - - echo " - Card: $cardName" - echo " Model: $gpuModel" - echo " PCI: $pciInfo" - else - # Fallback: just show the line as-is - echo " $line" - fi - done - ENABLE_INTEL_GPU_INFO=true - ENABLE_GPU_INFO=true - else - warn "No Intel GPUs detected by intel_gpu_top." - ENABLE_INTEL_GPU_INFO=false - fi - else - warn "intel_gpu_top command not found. Skipping Intel GPU information detection." - ENABLE_INTEL_GPU_INFO=false - fi - #endregion intel GPU setup - - #region NVIDIA GPU setup - # Check for NVIDIA GPU - ensure nvidia-smi is installed - if command -v nvidia-smi &>/dev/null; then - # detect all NVIDIA cards using nvidia-smi -L - local nvidiaCards - - # Get the output from nvidia-smi -L (lists GPUs) - nvidiaCards=$(nvidia-smi -L 2>/dev/null || true) - - if [[ -n "$nvidiaCards" ]]; then - local cardCount=$(echo "$nvidiaCards" | wc -l) - echo "NVIDIA GPU(s) detected ($cardCount):" - echo "$nvidiaCards" | while IFS= read -r line; do - # Expected format: GPU 0: NVIDIA GeForce RTX 3080 (UUID: GPU-xxxxx) - if [[ $line =~ ^GPU\ ([0-9]+):\ (.+)\ \(UUID:\ (.+)\)$ ]]; then - local gpuIndex="${BASH_REMATCH[1]}" - local gpuModel="${BASH_REMATCH[2]}" - local gpuUUID="${BASH_REMATCH[3]}" - - echo " - GPU $gpuIndex" - echo " Model: $gpuModel" - echo " UUID: $gpuUUID" - else - # Fallback: just show the line as-is - echo " $line" - fi - done - ENABLE_NVIDIA_GPU_INFO=true - ENABLE_GPU_INFO=true - else - warn "No NVIDIA GPUs detected by nvidia-smi." - ENABLE_NVIDIA_GPU_INFO=false - fi - else - warn "nvidia-smi command not found. Skipping NVIDIA GPU information detection." - ENABLE_NVIDIA_GPU_INFO=false - fi - #endregion NVIDIA GPU setup - - #region AMD GPU setup - # not implemented yet - #endregion AMD GPU setup - - #endregion Graphics setup - - #### GPU Historical Data #### - #region gpu history setup - ENABLE_GPU_HISTORY=false - if [[ "$ENABLE_GPU_INFO" == true ]]; then - msgb "\n=== GPU Historical Data ===" - local choiceGpuHistory - choiceGpuHistory=$(ask "Store historical GPU data for graphs? (y/N)") - case "$choiceGpuHistory" in - [yY]) - ENABLE_GPU_HISTORY=true - info "Historical GPU data will be stored." - ;; - [nN]|"") - info "Historical GPU data will not be stored." - ;; - *) - warn "Invalid selection. Historical GPU data will not be stored." - ;; - esac - fi - #endregion gpu history setup #### RAM #### #region ram setup - local ramList ramCount msgb "\n=== Detecting RAM temperature sensors ===" - ramList=$(echo "$sanitisedSensorsOutput" | grep -o '"SODIMM[^"]*"' | sed 's/"//g' | paste -sd, -) - ramCount=$(grep -c '"SODIMM[^"]*"' <<<"$sanitisedSensorsOutput") + local ramList=$(echo "$sanitisedSensorsOutput" | grep -o '"SODIMM[^"]*"' | sed 's/"//g' | paste -sd, -) + local ramCount=$(grep -c '"SODIMM[^"]*"' <<<"$sanitisedSensorsOutput") if [ "$ramCount" -gt 0 ]; then info "Detected RAM sensors ($ramCount): $ramList" @@ -488,6 +378,7 @@ function configure { # Function to install the modification function install_mod { + local upsConnection msgb "\n=== Preparing mod installation ===" check_root_privileges check_mod_installation @@ -495,24 +386,46 @@ function install_mod { perform_backup #### Insert information retrieval code #### - msgb "\n=== Installing sensor info module ===" - install_sensor_monitor_module - insert_sensor_monitor_into_pve - insert_system_info_into_pve + msgb "\n=== Inserting information retrieval code ===" + insert_node_info - ## Historical GPU data ## - if [[ "$ENABLE_GPU_HISTORY" == true ]]; then - install_gpu_history + #### Temperature helper parameters #### + msgb "\n=== Creating temperature conversion helper ===" + HELPERCTORPARAMS=$([[ "$TEMP_UNIT" = "F" ]] && \ + echo '{srcUnit: PVE.mod.TempHelper.CELSIUS, dstUnit: PVE.mod.TempHelper.FAHRENHEIT}' || \ + echo '{srcUnit: PVE.mod.TempHelper.CELSIUS, dstUnit: PVE.mod.TempHelper.CELSIUS}') + info "Temperature helper configured for $TEMP_UNIT." - # todo + #### Expand StatusView space #### + expand_statusview_space - # add nodes.pm index list — add gpumeta after gpurrddata: + #### Insert temperature helper #### + generate_and_insert_temp_helper - fi + #### Generate and insert widgets #### + msgb "\n=== Making visual adjustments ===" - #### Install UI modification module #### - msgb "\n=== Installing UI modification module ===" - install_node_status_view_module + generate_and_insert_widget "$ENABLE_SYSTEM_INFO" "generate_system_info" "system_info" + generate_and_insert_widget "$ENABLE_UPS" "generate_ups_widget" "ups" + generate_and_insert_widget "$ENABLE_HDD_TEMP" "generate_hdd_widget" "hdd" + generate_and_insert_widget "$ENABLE_NVME_TEMP" "generate_nvme_widget" "nvme" + + if [[ "$ENABLE_HDD_TEMP" = true || "$ENABLE_NVME_TEMP" = true ]]; then + generate_drive_header + info "Drive headers added." + fi + + generate_and_insert_widget "$ENABLE_FAN_SPEED" "generate_fan_widget" "fan" + generate_and_insert_widget "$ENABLE_RAM_TEMP" "generate_ram_widget" "ram" + generate_and_insert_widget "$ENABLE_CPU" "generate_cpu_widget" "cpu" + + #### Visual separation #### + add_visual_separator + info "Added visual separator for modified items." + + #### Node summary #### + setup_node_summary_container + info "Node summary box moved into its own container." msgb "\n=== Finalizing installation ===" @@ -541,7 +454,7 @@ sanitize_sensors_output() { # Replace NaN values with null s/\bNaN\b/null/g; - # Fix duplicate SODIMM keys - handle both pretty and one-line JSON + # Fix duplicate SODIMM keys - handle both pretty and one-line JSON s/"SODIMM"\s*:\s*\{\s*"temp(\d+)_input"/"SODIMM $1": {\n "temp$1_input"/g; # Fix duplicate fan keys - handle both pretty and one-line JSON @@ -549,76 +462,99 @@ sanitize_sensors_output() { ' | python3 -m json.tool 2>/dev/null || echo "$input" } -#region Sensor Monitor Module Installation -# Install and configure the Sensor Monitor Perl module -install_sensor_monitor_module() { - local intel_enabled nvidia_enabled ups_enabled sensors_mode - # Check if source file exists - if [[ ! -f "$PVE_SENSOR_INFO_SOURCE_FILE" ]]; then - err "Source file not found: $PVE_SENSOR_INFO_SOURCE_FILE" - fi - - # Copy the module file - cp "$PVE_SENSOR_INFO_SOURCE_FILE" "$PVE_SENSOR_INFO_MOD_FILE" || err "Failed to copy $PVE_SENSOR_INFO_SOURCE_FILE to $PVE_SENSOR_INFO_MOD_FILE" - info "Copied Sensor Monitor module to $PVE_SENSOR_INFO_MOD_FILE" - - # Convert boolean flags to Perl format (1 or 0) - intel_enabled=$([[ "$ENABLE_INTEL_GPU_INFO" = true ]] && echo 1 || echo 0) - nvidia_enabled=$([[ "$ENABLE_NVIDIA_GPU_INFO" = true ]] && echo 1 || echo 0) - ups_enabled=$([[ "$ENABLE_UPS" = true ]] && echo 1 || echo 0) - sensors_mode=$([[ "$DEBUG_REMOTE" = true ]] && echo 1 || echo 0) - - # Determine UPS device name - local ups_device="${upsConnection:-ups@localhost}" - - # Update configuration in the installed module - sed -i " - # Update GPU configuration - /intel_enabled =>/ s/=> [01],/=> $intel_enabled,/ - /nvidia_enabled =>/ s/=> [01],/=> $nvidia_enabled,/ - - # Update UPS configuration - /enabled =>/ { - /ups => {/,/},/ { - /enabled =>/ s/=> [01],/=> $ups_enabled,/ - } - } - /device_name =>/ s|=> '[^']*',|=> '$ups_device',| - " "$PVE_SENSOR_INFO_MOD_FILE" - - if [[ $? -eq 0 ]]; then - info "Sensor Monitor module configured successfully." - else - warn "Failed to configure Sensor Monitor module settings." - fi -} -#endregion Sensor Monitor Module Installation - #region node info insertion # Main insertion routine -insert_sensor_monitor_into_pve() { - #region PveSensorInfoMod heredoc - sed -i '/my \$dinfo = df('\''\/'\'', 1);/i\ - # Collect sensor data from PveMod_SensorInfo\ - # Bad practice to add use here, but cleaner implementation would require several extensive modifications.\ - use PVE::API2::PVEMod_SensorInfo;\ - $res->{PveMod_JsonSensorInfo} = PVE::API2::PVEMod_SensorInfo::get_sensors_info();\ - $res->{PveMod_graphicsInfo} = PVE::API2::PVEMod_SensorInfo::get_pve_mod_version();\ - $res->{PveMod_upsInfo} = PVE::API2::PVEMod_SensorInfo::get_ups_info();\ - ' "$NODES_PM_FILE" - #endregion PveSensorInfoMod heredoc - info "Sensor data retriever added to \"$NODES_PM_FILE\"." +insert_node_info() { + local output_file="$NODES_PM_FILE" + + collect_sensors_output "$output_file" + + if [[ $ENABLE_UPS == true ]]; then + collect_ups_output "$output_file" + fi + + if [[ $ENABLE_SYSTEM_INFO == true ]]; then + collect_system_info "$output_file" + fi } +# Collect lm-sensors data +collect_sensors_output() { + local output_file="$1" + local sensorsCmd + + if [[ $DEBUG_REMOTE == true ]]; then + sensorsCmd="cat \"$DEBUG_JSON_FILE\"" + else + # Note: sensors -f (Fahrenheit) breaks fan speeds + sensorsCmd="sensors -j 2>/dev/null" + fi + + # Remember to reflect this in sanitize_sensors_output() + #region sensors heredoc + sed -i '/my \$dinfo = df('\''\/'\'', 1);/i\ + # Collect sensor data from lm-sensors\ + $res->{sensorsOutput} = `'"$sensorsCmd"'`;\ + \ + # Sanitize JSON output to handle common lm-sensors parsing issues\ + # Replace ERROR lines with placeholder values\ + $res->{sensorsOutput} =~ s/ERROR:.+\\s(\\w+):\\s(.+)/\\"$1\\": 0.000,/g;\ + $res->{sensorsOutput} =~ s/ERROR:.+\\s(\\w+)!/\\"$1\\": 0.000,/g;\ + \ + # Remove trailing commas before closing braces\ + $res->{sensorsOutput} =~ s/,\\s*(\})/$1/g;\ + \ + # Replace NaN values with null for valid JSON\ + $res->{sensorsOutput} =~ s/\\bNaN\\b/null/g;\ + \ + # Fix duplicate SODIMM keys by appending temperature sensor number with a space - handle both pretty and one-line JSON\ + # Example: "SODIMM":{"temp3_input":34.0} becomes "SODIMM 3":{"temp3_input":34.0}\ + $res->{sensorsOutput} =~ s/"SODIMM"\\s*:\\s*\\{\\s*"temp(\\d+)_input"/"SODIMM $1": {\\n "temp$1_input"/g;\ + \ + # Fix duplicate fans keys by appending fan number with a space - handle both pretty and one-line JSON\ + # Example: "Processor Fan":{"fan2_input":1000,...} → "Processor Fan 2":{"fan2_input":1000,...}\ + $res->{sensorsOutput} =~ s/"([^"]+)"\\s*:\\s*\{\\s*"fan(\\d+)_input"/"$1 $2": {\\n "fan$2_input"/g;\ + \ + # Format JSON output properly (workaround for lm-sensors >3.6.0 issues)\ + $res->{sensorsOutput} =~ /^(.*)$/s;\ + $res->{sensorsOutput} = `echo \\Q$1\\E | python3 -m json.tool 2>/dev/null || echo \\Q$1\E`;\ + ' "$NODES_PM_FILE" + #endregion sensors heredoc + info "Sensors' retriever added to \"$output_file\"." +} + +# Collect UPS data +collect_ups_output() { + local output_file="$1" + local ups_cmd + + if [[ $DEBUG_REMOTE == true ]]; then + ups_cmd="cat \"$DEBUG_UPS_FILE\"" + else + ups_cmd="upsc \"$upsConnection\" 2>/dev/null" + fi + + # region ups heredoc + sed -i "/my \$dinfo = df('\/', 1);/i\\ + # Collect UPS status information\\ + sub get_upsc {\\ + my \$cmd = '$ups_cmd';\\ + my \$output = \`\\\$cmd\`;\\ + return \$output;\\ + }\\ + \$res->{upsc} = get_upsc();\\ +" "$NODES_PM_FILE" + # endregion ups heredoc + + info "UPS retriever added to \"$output_file\"." +} + + # Collect system information -insert_system_info_into_pve() { +collect_system_info() { local output_file="$1" local systemInfoCmd - if [[ $ENABLE_SYSTEM_INFO == false ]]; then - return - fi - systemInfoCmd=$(dmidecode -t "${SYSTEM_INFO_TYPE}" \ | awk -F': ' '/Manufacturer|Product Name|Serial Number/ {print $1": "$2}' \ | awk '{$1=$1};1' \ @@ -628,138 +564,1000 @@ insert_system_info_into_pve() { #region system info heredoc sed -i "/my \$dinfo = df('\/', 1);/i\\ # Add system information to response\\ - \$res->{pveMod_sensorInfo_systemInfo} = \"$(echo "$systemInfoCmd")\";\\ + \$res->{systemInfo} = \"$(echo "$systemInfoCmd")\";\\ " "$NODES_PM_FILE" #endregion system info heredoc info "System information retriever added to \"$output_file\"." } #endregion node info insertion -#region UI Module Installation -# Install the UI modification module -install_node_status_view_module() { - # Check if source file exists - if [[ ! -f "$PVE_MOD_JS_SOURCE_FILE" ]]; then - err "Source file not found: $PVE_MOD_JS_SOURCE_FILE" - fi +#region widget generation functions +# Helper function to insert widget after thermal items +insert_widget_after_thermal() { + local widget_file="$1" + sed -i "/^Ext.define('PVE.node.StatusView',/ { + :a + /items:/!{N;ba;} + :b + /'cpus.*},/!{N;bb;} + r $widget_file + }" "$PVE_MANAGER_LIB_JS_FILE" +} - # Copy the JavaScript module to PVE manager directory - cp "$PVE_MOD_JS_SOURCE_FILE" "$PVE_MOD_JS_TARGET_FILE" || err "Failed to copy $PVE_MOD_JS_SOURCE_FILE to $PVE_MOD_JS_TARGET_FILE" - info "Copied UI module to $PVE_MOD_JS_TARGET_FILE" +# Helper function to generate widget and insert it +generate_and_insert_widget() { + local enable_flag="$1" + local generator_func="$2" + local widget_name="$3" - # Comment out the original PVE.node.StatusView definition in pvemanagerlib.js - # This allows our custom module to provide the new definition - if grep -q "^Ext.define('PVE.node.StatusView'," "$PVE_MANAGER_LIB_JS_FILE" 2>/dev/null; then - # Find the start of the definition and comment it out until the matching closing brace - sed -i "/^Ext\.define('PVE\.node\.StatusView',/,/^});/s|^|// |" "$PVE_MANAGER_LIB_JS_FILE" - info "Commented out original StatusView definition in pvemanagerlib.js" - else - warn "Original StatusView definition not found in expected format in pvemanagerlib.js" - fi + if [ "$enable_flag" = true ]; then + local temp_js_file="/tmp/${widget_name}_widget.js" + "$generator_func" "$temp_js_file" + insert_widget_after_thermal "$temp_js_file" + rm "$temp_js_file" + info "Inserted $widget_name widget." + fi +} - # Comment out the original PVE.node.Summary definition in pvemanagerlib.js - if grep -q "^Ext.define('PVE.node.Summary'," "$PVE_MANAGER_LIB_JS_FILE" 2>/dev/null; then - sed -i "/^Ext\.define('PVE\.node\.Summary',/,/^});/s|^|// |" "$PVE_MANAGER_LIB_JS_FILE" - info "Commented out original Summary definition in pvemanagerlib.js" - else - warn "Original Summary definition not found in expected format in pvemanagerlib.js" - fi +# Function to generate drive header +generate_drive_header() { + if [ "$ENABLE_NVME_TEMP" = true ] || [ "$ENABLE_HDD_TEMP" = true ]; then + local temp_js_file="/tmp/drive_header.js" + #region drive header heredoc + cat > "$temp_js_file" <<'EOF' + { + xtype: 'box', + colspan: 2, + html: gettext('Drive(s)'), + }, +EOF +#endregion drive header heredoc + if [[ $? -ne 0 ]]; then + echo "Error: Failed to generate drive header code" >&2 + exit 1 + fi - # Add a dynamic script loader to load our custom module - # Insert before the commented-out Ext.define to load it - if ! grep -q "PveMod_PveNodeStatusView.js" "$PVE_MANAGER_LIB_JS_FILE" 2>/dev/null; then - # Use ExtJS Loader to dynamically load our custom module - sed -i "/^\/\/ Ext\.define('PVE\.node\.StatusView',/i\\ -// Load custom PVE.node.StatusView from external module\\ -Ext.Loader.loadScript({\\ - url: '/pve2/js/PveMod_PveNodeStatusView.js',\\ - onLoad: function() { },\\ - onError: function() { console.error('Failed to load PveMod_PveNodeStatusView.js'); }\\ -});\\ -" "$PVE_MANAGER_LIB_JS_FILE" - info "Added dynamic loader for custom UI module in pvemanagerlib.js" - else - info "Custom UI module loader already present in pvemanagerlib.js" + insert_widget_after_thermal "$temp_js_file" + rm "$temp_js_file" fi } -#endregion UI Module Installation +# Function to expand space and modify StatusView properties +expand_statusview_space() { + msgb "\n=== Expanding StatusView space ===" -#region historical GPU data -# Install GPU historical data storage: RRD directory + API endpoint in Nodes.pm -install_gpu_history() { - msgb "\n=== Installing GPU historical data support ===" + # Apply multiple modifications to the StatusView definition + sed -i "/Ext.define('PVE\.node\.StatusView'/,/\},/ { + s/\(bodyPadding:\) '[^']*'/\1 '20 15 20 15'/ + s/height: [0-9]\+/minHeight: 360,\n\tflex: 1,\n\tcollapsible: true,\n\ttitleCollapse: true/ + s/\(tableAttrs:.*$\)/trAttrs: \{ valign: 'top' \},\n\t\1/ + }" "$PVE_MANAGER_LIB_JS_FILE" - # Create and configure the RRD directory - mkdir -p "$GPU_RRD_DIR" || err "Failed to create GPU RRD directory: $GPU_RRD_DIR" - chown www-data:www-data "$GPU_RRD_DIR" || err "Failed to set ownership on: $GPU_RRD_DIR" - info "GPU RRD directory ready: $GPU_RRD_DIR" + if [[ $? -ne 0 ]]; then + echo "Error: Failed to expand StatusView space" >&2 + exit 1 + fi - # Register gpurrddata in the node method list - sed -i "s/{ name => 'rrddata' },/{ name => 'rrddata' },\n { name => 'gpurrddata' },/" "$NODES_PM_FILE" \ - || err "Failed to register gpurrddata method in $NODES_PM_FILE" + info "Expanded space in \"$PVE_MANAGER_LIB_JS_FILE\"." +} - # Insert the gpurrddata API method definition after the rrddata method - sed -i "/\"pve-node-9\.0\/\$param->{node}\", \$param->{timeframe}, \$param->{cf},/{ -n -n -a\\ -\\ -__PACKAGE__->register_method({\\ - name => 'gpurrddata',\\ - path => 'gpurrddata',\\ - method => 'GET',\\ - protected => 1,\\ - proxyto => 'node',\\ - permissions => {\\ - check => ['perm', '/nodes/{node}', ['Sys.Audit']],\\ - },\\ - description => \"Read GPU RRD statistics\",\\ - parameters => {\\ - additionalProperties => 0,\\ - properties => {\\ - node => get_standard_option('pve-node'),\\ - card => {\\ - description => \"The GPU card identifier (e.g. card0, nvidia0).\",\\ - type => 'string',\\ - pattern => '[a-zA-Z0-9]+',\\ - },\\ - timeframe => {\\ - description => \"Specify the time frame you are interested in.\",\\ - type => 'string',\\ - enum => ['hour', 'day', 'week', 'month', 'year', 'decade'],\\ - },\\ - cf => {\\ - description => \"The RRD consolidation function\",\\ - type => 'string',\\ - enum => ['AVERAGE', 'MAX'],\\ - optional => 1,\\ - },\\ - },\\ - },\\ - returns => {\\ - type => \"array\",\\ - items => {\\ - type => \"object\",\\ - properties => {},\\ - },\\ - },\\ - code => sub {\\ - my (\$param) = \@_;\\ - my \$nodename = PVE::INotify::nodename();\\ - my \$card = \$param->{card};\\ - die \"invalid card name\\n\" unless \$card =~ /^[a-zA-Z0-9]+\$/;\\ - return PVE::RRD::create_rrd_data(\\ - \"pve-mod-gpu/\$nodename/\$card\", \$param->{timeframe}, \$param->{cf},\\ - );\\ - },\\ +# Function to move node summary into its own container +setup_node_summary_container() { + # Move the node summary box into its own container + local temp_js_file="/tmp/summary_container.js" + #region summary container heredoc + cat > "$temp_js_file" <<'EOF' +{ + xtype: 'container', + itemId: 'summarycontainer', + layout: 'column', + minWidth: 700, + defaults: { + minHeight: 350, + padding: 5, + columnWidth: 1, + }, + items: [ + nodeStatus, + ] +}, +EOF +#endregion summary container heredoc + if [[ $? -ne 0 ]]; then + echo "Error: Failed to generate summary container code" >&2 + exit 1 + fi + + # Insert the new container after finding the nodeStatus and items pattern + sed -i "/^\s*nodeStatus: nodeStatus,/ { + :a + /items: \[/ !{N;ba;} + r $temp_js_file + }" "$PVE_MANAGER_LIB_JS_FILE" + + rm "$temp_js_file" + + # Deactivate the original box instance + sed -i "/^\s*nodeStatus: nodeStatus,/ { + :a + /itemId: 'itemcontainer',/ !{N;ba;} + n; + :b + /nodeStatus,/ !{N;bb;} + s/nodeStatus/\/\/nodeStatus/ + }" "$PVE_MANAGER_LIB_JS_FILE" + + if [[ $? -ne 0 ]]; then + echo "Error: Failed to deactivate original nodeStatus instance" >&2 + exit 1 + fi +} + +# Function to add visual spacing separator after the last widget +add_visual_separator() { + # Check for the presence of items in the reverse order of display + local lastItemId="" + + if [ "$ENABLE_UPS" = true ]; then + lastItemId="upsc" + elif [ "$ENABLE_HDD_TEMP" = true ]; then + lastItemId="thermalHdd" + elif [ "$ENABLE_NVME_TEMP" = true ]; then + lastItemId="thermalNvme" + elif [ "$ENABLE_FAN_SPEED" = true ]; then + lastItemId="speedFan" + else + lastItemId="thermalCpu" + fi + + if [ -n "$lastItemId" ]; then + local temp_js_file="/tmp/visual_separator.js" + + #region visual spacing heredoc + cat > "$temp_js_file" <<'EOF' + { + xtype: 'box', + colspan: 2, + padding: '0 0 20 0', + }, +EOF +#endregion visual spacing heredoc + if [[ $? -ne 0 ]]; then + echo "Error: Failed to generate visual separator code" >&2 + exit 1 + fi + + # Insert after the specific lastItemId (different pattern than thermal) + sed -i "/^Ext.define('PVE.node.StatusView',/ { + :a; + /^.*{.*'$lastItemId'.*},/!{N;ba;} + r $temp_js_file + }" "$PVE_MANAGER_LIB_JS_FILE" + + rm "$temp_js_file" + fi +} + +# Function to generate system info widget +generate_system_info() { + #region system info heredoc + cat > "$1" <<'EOF' + { + itemId: 'sysinfo', + colspan: 2, + printBar: false, + title: gettext('System Information'), + textField: 'systemInfo', + renderer: function(value){ + return value; + } + }, +EOF + #endregion system info heredoc + if [[ $? -ne 0 ]]; then + echo "Error: Failed to generate system info code" >&2 + exit 1 + fi +} + +# Function to generate and insert temperature conversion helper class +generate_and_insert_temp_helper() { + local temp_js_file="/tmp/temp_helper.js" + + msgb "\n=== Inserting temperature helper ===" + + #region temp helper heredoc + cat > "$temp_js_file" <<'EOF' +Ext.define('PVE.mod.TempHelper', { + //singleton: true, + + requires: ['Ext.util.Format'], + + statics: { + CELSIUS: 0, + FAHRENHEIT: 1 + }, + + srcUnit: null, + dstUnit: null, + + isValidUnit: function (unit) { + return ( + Ext.isNumber(unit) && (unit === this.self.CELSIUS || unit === this.self.FAHRENHEIT) + ); + }, + + constructor: function (config) { + this.srcUnit = config && this.isValidUnit(config.srcUnit) ? config.srcUnit : this.self.CELSIUS; + this.dstUnit = config && this.isValidUnit(config.dstUnit) ? config.dstUnit : this.self.CELSIUS; + }, + + toFahrenheit: function (tempCelsius) { + return Ext.isNumber(tempCelsius) + ? tempCelsius * 9 / 5 + 32 + : NaN; + }, + + toCelsius: function (tempFahrenheit) { + return Ext.isNumber(tempFahrenheit) + ? (tempFahrenheit - 32) * 5 / 9 + : NaN; + }, + + getTemp: function (value) { + if (this.srcUnit !== this.dstUnit) { + switch (this.srcUnit) { + case this.self.CELSIUS: + switch (this.dstUnit) { + case this.self.FAHRENHEIT: + return this.toFahrenheit(value); + + default: + Ext.raise({ + msg: + 'Unsupported destination temperature unit: ' + this.dstUnit, + }); + } + case this.self.FAHRENHEIT: + switch (this.dstUnit) { + case this.self.CELSIUS: + return this.toCelsius(value); + + default: + Ext.raise({ + msg: + 'Unsupported destination temperature unit: ' + this.dstUnit, + }); + } + default: + Ext.raise({ + msg: 'Unsupported source temperature unit: ' + this.srcUnit, + }); + } + } else { + return value; + } + }, + + getUnit: function(plainText) { + switch (this.dstUnit) { + case this.self.CELSIUS: + return plainText !== true ? '°C' : '\'C'; + + case this.self.FAHRENHEIT: + return plainText !== true ? '°F' : '\'F'; + + default: + Ext.raise({ + msg: 'Unsupported destination temperature unit: ' + this.srcUnit, + }); + } + }, }); -}" "$NODES_PM_FILE" \ - || err "Failed to insert gpurrddata method in $NODES_PM_FILE" +EOF + #endregion temp helper heredoc + if [[ $? -ne 0 ]]; then + echo "Error: Failed to generate temp helper code" >&2 + exit 1 + fi - info "GPU historical data API endpoint added to $NODES_PM_FILE" + sed -i "/^Ext.define('PVE.node.StatusView'/e cat /tmp/temp_helper.js" "$PVE_MANAGER_LIB_JS_FILE" + rm "$temp_js_file" + + info "Temperature helper inserted successfully." } -#endregion historical GPU data + +# Function to generate CPU widget +generate_cpu_widget() { + #region cpu widget heredoc + # use subshell to allow variable expansion + ( + export CPU_ITEMS_PER_ROW + export CPU_TEMP_TARGET + export HELPERCTORPARAMS + + cat <<'EOF' | envsubst '$CPU_ITEMS_PER_ROW $CPU_TEMP_TARGET $HELPERCTORPARAMS' > "$1" + { + itemId: 'thermalCpu', + colspan: 2, + printBar: false, + title: gettext('CPU Thermal State'), + iconCls: 'fa fa-fw fa-thermometer-half', + textField: 'sensorsOutput', + renderer: function(value){ + // sensors configuration + const cpuTempHelper = Ext.create('PVE.mod.TempHelper', $HELPERCTORPARAMS); + // display configuration + const itemsPerRow = $CPU_ITEMS_PER_ROW; + // --- + let objValue; + try { + objValue = JSON.parse(value) || {}; + } catch(e) { + objValue = {}; + } + const cpuKeysI = Object.keys(objValue).filter(item => String(item).startsWith('coretemp-isa-')).sort(); + const cpuKeysA = Object.keys(objValue).filter(item => String(item).startsWith('k10temp-pci-')).sort(); + const bINTEL = cpuKeysI.length > 0 ? true : false; + const INTELPackagePrefix = '$CPU_TEMP_TARGET' == 'Core' ? 'Core ' : 'Package id'; + const INTELPackageCaption = '$CPU_TEMP_TARGET' == 'Core' ? 'Core' : 'Package'; + let AMDPackagePrefix = 'Tccd'; + let AMDPackageCaption = 'CCD'; + + if (cpuKeysA.length > 0) { + let bTccd = false; + let bTctl = false; + let bTdie = false; + let bCpuCoreTemp = false; + cpuKeysA.forEach((cpuKey, cpuIndex) => { + let items = objValue[cpuKey]; + bTccd = Object.keys(items).findIndex(item => { return String(item).startsWith('Tccd'); }) >= 0; + bTctl = Object.keys(items).findIndex(item => { return String(item).startsWith('Tctl'); }) >= 0; + bTdie = Object.keys(items).findIndex(item => { return String(item).startsWith('Tdie'); }) >= 0; + bCpuCoreTemp = Object.keys(items).findIndex(item => { return String(item) === 'CPU Core Temp'; }) >= 0; + }); + if (bTccd && '$CPU_TEMP_TARGET' == 'Core') { + AMDPackagePrefix = 'Tccd'; + AMDPackageCaption = 'ccd'; + } else if (bCpuCoreTemp && '$CPU_TEMP_TARGET' == 'Package') { + AMDPackagePrefix = 'CPU Core Temp'; + AMDPackageCaption = 'CPU Core Temp'; + } else if (bTdie) { + AMDPackagePrefix = 'Tdie'; + AMDPackageCaption = 'die'; + } else if (bTctl) { + AMDPackagePrefix = 'Tctl'; + AMDPackageCaption = 'ctl'; + } else { + AMDPackagePrefix = 'temp'; + AMDPackageCaption = 'Temp'; + } + } + + const cpuKeys = bINTEL ? cpuKeysI : cpuKeysA; + const cpuItemPrefix = bINTEL ? INTELPackagePrefix : AMDPackagePrefix; + const cpuTempCaption = bINTEL ? INTELPackageCaption : AMDPackageCaption; + const formatTemp = bINTEL ? '0' : '0.0'; + const cpuCount = cpuKeys.length; + let temps = []; + + cpuKeys.forEach((cpuKey, cpuIndex) => { + let cpuTemps = []; + const items = objValue[cpuKey]; + const itemKeys = Object.keys(items).filter(item => { + if ('$CPU_TEMP_TARGET' == 'Core') { + // In Core mode: only show individual cores/CCDs, exclude overall CPU temp + return String(item).includes(cpuItemPrefix) || String(item).startsWith('Tccd'); + } else { + // In Package mode: show overall CPU temp and package-level readings + return String(item).includes(cpuItemPrefix) || String(item) === 'CPU Core Temp'; + } + }); + + itemKeys.forEach((coreKey) => { + try { + let tempVal = NaN, tempMax = NaN, tempCrit = NaN; + Object.keys(items[coreKey]).forEach((secondLevelKey) => { + if (secondLevelKey.endsWith('_input')) { + tempVal = cpuTempHelper.getTemp(parseFloat(items[coreKey][secondLevelKey])); + } else if (secondLevelKey.endsWith('_max')) { + tempMax = cpuTempHelper.getTemp(parseFloat(items[coreKey][secondLevelKey])); + } else if (secondLevelKey.endsWith('_crit')) { + tempCrit = cpuTempHelper.getTemp(parseFloat(items[coreKey][secondLevelKey])); + } + }); + + if (!isNaN(tempVal)) { + let tempStyle = ''; + if (!isNaN(tempMax) && tempVal >= tempMax) { + tempStyle = 'color: #FFC300; font-weight: bold;'; + } + if (!isNaN(tempCrit) && tempVal >= tempCrit) { + tempStyle = 'color: red; font-weight: bold;'; + } + + let tempStr = ''; + + // Enhanced parsing for AMD temperatures + if (coreKey.startsWith('Tccd')) { + let tempIndex = coreKey.match(/Tccd(\d+)/); + if (tempIndex !== null && tempIndex.length > 1) { + tempIndex = tempIndex[1]; + tempStr = `${cpuTempCaption} ${tempIndex}: ${Ext.util.Format.number(tempVal, formatTemp)}${cpuTempHelper.getUnit()}`; + } else { + tempStr = `${cpuTempCaption}: ${Ext.util.Format.number(tempVal, formatTemp)}${cpuTempHelper.getUnit()}`; + } + } + // Handle CPU Core Temp (single overall temperature) + else if (coreKey === 'CPU Core Temp') { + tempStr = `${cpuTempCaption}: ${Ext.util.Format.number(tempVal, formatTemp)}${cpuTempHelper.getUnit()}`; + } + // Enhanced parsing for Intel cores (P-Core, E-Core, regular Core) + else { + let tempIndex = coreKey.match(/(?:P\s+Core|E\s+Core|Core)\s*(\d+)/); + if (tempIndex !== null && tempIndex.length > 1) { + tempIndex = tempIndex[1]; + let coreType = coreKey.startsWith('P Core') ? 'P Core' : + coreKey.startsWith('E Core') ? 'E Core' : + cpuTempCaption; + tempStr = `${coreType} ${tempIndex}: ${Ext.util.Format.number(tempVal, formatTemp)}${cpuTempHelper.getUnit()}`; + } else { + // fallback for CPUs which do not have a core index + let coreType = coreKey.startsWith('P Core') ? 'P Core' : + coreKey.startsWith('E Core') ? 'E Core' : + cpuTempCaption; + tempStr = `${coreType}: ${Ext.util.Format.number(tempVal, formatTemp)}${cpuTempHelper.getUnit()}`; + } + } + + cpuTemps.push(tempStr); + } + } catch (e) { /*_*/ } + }); + + if(cpuTemps.length > 0) { + temps.push(cpuTemps); + } + }); + + let result = ''; + temps.forEach((cpuTemps, cpuIndex) => { + const strCoreTemps = cpuTemps.map((strTemp, index, arr) => { + return strTemp + (index + 1 < arr.length ? (itemsPerRow > 0 && (index + 1) % itemsPerRow === 0 ? '
' : ' | ') : ''); + }) + if(strCoreTemps.length > 0) { + result += (cpuCount > 1 ? `CPU ${cpuIndex+1}: ` : '') + strCoreTemps.join('') + (cpuIndex < cpuCount ? '
' : ''); + } + }); + + return '
' + (result.length > 0 ? result : 'N/A') + '
'; + } + }, +EOF + ) + #endregion cpu widget heredoc + if [[ $? -ne 0 ]]; then + echo "Error: Failed to generate cpu widget code" >&2 + exit 1 + fi +} + +# Function to generate nvme widget +generate_nvme_widget() { + #region nvme widget heredoc + # use subshell to allow variable expansion + ( + export HELPERCTORPARAMS + export NVME_ITEMS_PER_ROW + cat <<'EOF' | envsubst '$HELPERCTORPARAMS $NVME_ITEMS_PER_ROW' > "$1" + { + itemId: 'thermalNvme', + colspan: 2, + printBar: false, + title: gettext('NVMe Thermal State'), + iconCls: 'fa fa-fw fa-thermometer-half', + textField: 'sensorsOutput', + renderer: function(value) { + // sensors configuration + const addressPrefix = "nvme-pci-"; + const sensorName = "Composite"; + const tempHelper = Ext.create('PVE.mod.TempHelper', $HELPERCTORPARAMS); + // display configuration + const itemsPerRow = $NVME_ITEMS_PER_ROW; + // --- + let objValue; + try { + objValue = JSON.parse(value) || {}; + } catch(e) { + objValue = {}; + } + const nvmeKeys = Object.keys(objValue).filter(item => String(item).startsWith(addressPrefix)).sort(); + let temps = []; + nvmeKeys.forEach((nvmeKey, index) => { + try { + let tempVal = NaN, tempMax = NaN, tempCrit = NaN; + Object.keys(objValue[nvmeKey][sensorName]).forEach((secondLevelKey) => { + if (secondLevelKey.endsWith('_input')) { + tempVal = tempHelper.getTemp(parseFloat(objValue[nvmeKey][sensorName][secondLevelKey])); + } else if (secondLevelKey.endsWith('_max')) { + tempMax = tempHelper.getTemp(parseFloat(objValue[nvmeKey][sensorName][secondLevelKey])); + } else if (secondLevelKey.endsWith('_crit')) { + tempCrit = tempHelper.getTemp(parseFloat(objValue[nvmeKey][sensorName][secondLevelKey])); + } + }); + if (!isNaN(tempVal)) { + let tempStyle = ''; + if (!isNaN(tempMax) && tempVal >= tempMax) { + tempStyle = 'color: #FFC300; font-weight: bold;'; + } + if (!isNaN(tempCrit) && tempVal >= tempCrit) { + tempStyle = 'color: red; font-weight: bold;'; + } + const tempStr = `Drive ${index + 1}: ${Ext.util.Format.number(tempVal, '0.0')}${tempHelper.getUnit()}`; + temps.push(tempStr); + } + } catch(e) { /*_*/ } + }); + const result = temps.map((strTemp, index, arr) => { return strTemp + (index + 1 < arr.length ? ((index + 1) % itemsPerRow === 0 ? '
' : ' | ') : ''); }); + return '
' + (result.length > 0 ? result.join('') : 'N/A') + '
'; + } + }, +EOF + ) + #endregion nvme widget heredoc + if [[ $? -ne 0 ]]; then + echo "Error: Failed to generate nvme widget code" >&2 + exit 1 + fi +} + +# Function to generate Fan widget +generate_fan_widget() { + #region fan widget heredoc + # use subshell to allow variable expansion + ( + export DISPLAY_ZERO_SPEED_FANS + cat <<'EOF' | envsubst '$DISPLAY_ZERO_SPEED_FANS' > "$1" + { + xtype: 'box', + colspan: 2, + html: gettext('Cooling'), + }, + { + itemId: 'speedFan', + colspan: 2, + printBar: false, + title: gettext('Fan Speed(s)'), + iconCls: 'fa fa-fw fa-snowflake-o', + textField: 'sensorsOutput', + renderer: function(value) { + // --- + let objValue; + try { + objValue = JSON.parse(value) || {}; + } catch(e) { + objValue = {}; + } + + // Recursive function to find fan keys and values + function findFanKeys(obj, fanKeys, parentKey = null) { + Object.keys(obj).forEach(key => { + const value = obj[key]; + if (typeof value === 'object' && value !== null) { + // If the value is an object, recursively call the function + findFanKeys(value, fanKeys, key); + } else if (/^fan[0-9]+(_input)?$/.test(key)) { + if ($DISPLAY_ZERO_SPEED_FANS != true && value === 0) { + // Skip this fan if DISPLAY_ZERO_SPEED_FANS is false and value is 0 + return; + } + // If the key matches the pattern, add the parent key and value to the fanKeys array + fanKeys.push({ key: parentKey, value: value }); + } + }); + } + + let speeds = []; + // Loop through the parent keys + Object.keys(objValue).forEach(parentKey => { + const parentObj = objValue[parentKey]; + // Array to store fan keys and values + const fanKeys = []; + // Call the recursive function to find fan keys and values + findFanKeys(parentObj, fanKeys); + // Sort the fan keys + fanKeys.sort(); + // Process each fan key and value + fanKeys.forEach(({ key: fanKey, value: fanSpeed }) => { + try { + const fan = fanKey.charAt(0).toUpperCase() + fanKey.slice(1); // Capitalize the first letter of fanKey + speeds.push(`${fan}: ${fanSpeed} RPM`); + } catch(e) { + console.error(`Error retrieving fan speed for ${fanKey} in ${parentKey}:`, e); // Debug: Log specific error + } + }); + }); + return '
' + (speeds.length > 0 ? speeds.join(' | ') : 'N/A') + '
'; + } + }, +EOF + ) + #endregion fan widget heredoc + if [[ $? -ne 0 ]]; then + echo "Error: Failed to generate fan widget code" >&2 + exit 1 + fi +} + +# Function to generate UPS widget +generate_hdd_widget() { + #region hdd widget heredoc + # use subshell to allow variable expansion + ( + export HELPERCTORPARAMS + export HDD_ITEMS_PER_ROW + cat <<'EOF' | envsubst '$HDD_ITEMS_PER_ROW $HELPERCTORPARAMS' > "$1" + { + itemId: 'thermalHdd', + colspan: 2, + printBar: false, + title: gettext('HDD/SSD Thermal State'), + iconCls: 'fa fa-fw fa-thermometer-half', + textField: 'sensorsOutput', + renderer: function(value) { + // sensors configuration + const addressPrefix = "drivetemp-scsi-"; + const sensorName = "temp1"; + const tempHelper = Ext.create('PVE.mod.TempHelper', $HELPERCTORPARAMS); + // display configuration + const itemsPerRow = $HDD_ITEMS_PER_ROW; + // --- + let objValue; + try { + objValue = JSON.parse(value) || {}; + } catch(e) { + objValue = {}; + } + const drvKeys = Object.keys(objValue).filter(item => String(item).startsWith(addressPrefix)).sort((a, b) => a.localeCompare(b, undefined, { numeric: true, sensitivity: 'base' })); + let temps = []; + drvKeys.forEach((drvKey, index) => { + try { + let tempVal = NaN, tempMax = NaN, tempCrit = NaN; + Object.keys(objValue[drvKey][sensorName]).forEach((secondLevelKey) => { + if (secondLevelKey.endsWith('_input')) { + tempVal = tempHelper.getTemp(parseFloat(objValue[drvKey][sensorName][secondLevelKey])); + } else if (secondLevelKey.endsWith('_max')) { + tempMax = tempHelper.getTemp(parseFloat(objValue[drvKey][sensorName][secondLevelKey])); + } else if (secondLevelKey.endsWith('_crit')) { + tempCrit = tempHelper.getTemp(parseFloat(objValue[drvKey][sensorName][secondLevelKey])); + } + }); + if (!isNaN(tempVal)) { + let tempStyle = ''; + if (!isNaN(tempMax) && tempVal >= tempMax) { + tempStyle = 'color: #FFC300; font-weight: bold;'; + } + if (!isNaN(tempCrit) && tempVal >= tempCrit) { + tempStyle = 'color: red; font-weight: bold;'; + } + const tempStr = `Drive ${index + 1}: ${Ext.util.Format.number(tempVal, '0.0')}${tempHelper.getUnit()}`; + temps.push(tempStr); + } + } catch(e) { /*_*/ } + }); + const result = temps.map((strTemp, index, arr) => { return strTemp + (index + 1 < arr.length ? ((index + 1) % itemsPerRow === 0 ? '
' : ' | ') : ''); }); + return '
' + (result.length > 0 ? result.join('') : 'N/A') + '
'; + } + }, +EOF + ) + #endregion hdd widget heredoc + if [[ $? -ne 0 ]]; then + echo "Error: Failed to generate hhd widget code" >&2 + exit 1 + fi +} + +# Function to generate RAM widget +generate_ram_widget() { + #region ram widget heredoc + # use subshell to allow variable expansion + ( + export HELPERCTORPARAMS + cat <<'EOF' | envsubst '$HELPERCTORPARAMS' > "$1" + { + xtype: 'box', + colspan: 2, + html: gettext('RAM'), + }, + { + itemId: 'thermalRam', + colspan: 2, + printBar: false, + title: gettext('Thermal State'), + iconCls: 'fa fa-fw fa-thermometer-half', + textField: 'sensorsOutput', + renderer: function(value) { + const cpuTempHelper = Ext.create('PVE.mod.TempHelper', $HELPERCTORPARAMS); + + let objValue; + try { + objValue = JSON.parse(value) || {}; + } catch(e) { + objValue = {}; + } + + // Recursive function to find ram keys and values + function findRamKeys(obj, ramKeys, parentKey = null) { + Object.keys(obj).forEach(key => { + const value = obj[key]; + if (typeof value === 'object' && value !== null) { + // If the value is an object, recursively call the function + findRamKeys(value, ramKeys, key); + } else if (/^temp\d+_input$/.test(key) && parentKey && parentKey.startsWith("SODIMM")) { + if (value !== 0) { + ramKeys.push({ key: parentKey, value: value}); + } + } + }); + } + + let ramTemps = []; + // Loop through the parent keys + Object.keys(objValue).forEach(parentKey => { + const parentObj = objValue[parentKey]; + // Array to store ram keys and values + const ramKeys = []; + // Call the recursive function to find ram keys and values + findRamKeys(parentObj, ramKeys); + // Sort the ramKeys keys + ramKeys.sort(); + // Process each ram key and value + ramKeys.forEach(({ key: ramKey, value: ramTemp }) => { + try { + ramTemps.push(`${ramKey}: ${ramTemp}${cpuTempHelper.getUnit()}`); + } catch(e) { + console.error(`Error retrieving Ram Temp for ${ramTemps} in ${parentKey}:`, e); // Debug: Log specific error + } + }); + }); + return '
' + (ramTemps.length > 0 ? ramTemps.join(' | ') : 'N/A') + '
'; + } + }, +EOF + ) + #endregion ram widget heredoc + if [[ $? -ne 0 ]]; then + echo "Error: Failed to generate ram widget code" >&2 + exit 1 + fi +} + +# Function to generate UPS widget +generate_ups_widget() { + #region UPS widget heredoc + cat > "$1" <<'EOF' + { + xtype: 'box', + colspan: 2, + html: gettext('UPS'), + }, + { + itemId: 'upsc', + colspan: 2, + printBar: false, + title: gettext('Device'), + iconCls: 'fa fa-fw fa-battery-three-quarters', + textField: 'upsc', + renderer: function(value) { + let objValue = {}; + try { + // Parse the UPS data + if (typeof value === 'string') { + const lines = value.split('\n'); + lines.forEach(line => { + const colonIndex = line.indexOf(':'); + if (colonIndex > 0) { + const key = line.substring(0, colonIndex).trim(); + const val = line.substring(colonIndex + 1).trim(); + objValue[key] = val; + } + }); + } else if (typeof value === 'object') { + objValue = value || {}; + } + } catch(e) { + objValue = {}; + } + + // If objValue is null or empty, return N/A + if (!objValue || Object.keys(objValue).length === 0) { + return '
N/A
'; + } + + // Helper function to get status color + // Returns a CSS color string for non-default states, or null for default (no inline color) + function getStatusColor(status) { + if (!status) return '#999'; + const statusUpper = status.toUpperCase(); + if (statusUpper.includes('OL')) return null; // default (no explicit color) + if (statusUpper.includes('OB')) return '#d9534f'; // Red for on battery + if (statusUpper.includes('LB')) return '#d9534f'; // Red for low battery + return '#f0ad4e'; // Orange for other states + } + + // Helper function to get load/charge color + // Returns null for default/good values so no inline style is emitted + function getPercentageColor(value, isLoad = false) { + if (!value || isNaN(value)) return '#999'; + const num = parseFloat(value); + if (isLoad) { + if (num >= 80) return '#d9534f'; // Red for high load + if (num >= 60) return '#f0ad4e'; // Orange for medium load + return null; // default (no explicit color) + } else { + // For battery charge + if (num <= 20) return '#d9534f'; // Red for low charge + if (num <= 50) return '#f0ad4e'; // Orange for medium charge + return null; // default (no explicit color) + } + } + + // Helper function to format runtime + function formatRuntime(seconds) { + if (!seconds || isNaN(seconds)) return 'N/A'; + const mins = Math.floor(seconds / 60); + const secs = seconds % 60; + return `${mins}m ${secs}s`; + } + + // Extract key UPS information + const batteryCharge = objValue['battery.charge']; + const batteryRuntime = objValue['battery.runtime']; + const inputVoltage = objValue['input.voltage']; + const upsLoad = objValue['ups.load']; + const upsStatus = objValue['ups.status']; + const upsModel = objValue['ups.model'] || objValue['device.model']; + const testResult = objValue['ups.test.result']; + const batteryChargeLow = objValue['battery.charge.low']; + const batteryRuntimeLow = objValue['battery.runtime.low']; + const upsRealPowerNominal = objValue['ups.realpower.nominal']; + const batteryMfrDate = objValue['battery.mfr.date']; + + // Build the status display + let displayItems = []; + + // First line: Model info (no explicit color for default) + let modelLine = ''; + if (upsModel) { + modelLine = `${upsModel}`; + } else { + modelLine = `N/A`; + } + displayItems.push(modelLine); + + // Main status line with all metrics + let statusLine = ''; + + // Status + if (upsStatus) { + const statusUpper = upsStatus.toUpperCase(); + let statusText = 'Unknown'; + let statusColor = '#f0ad4e'; + + if (statusUpper.includes('OL')) { + statusText = 'Online'; + statusColor = null; // default (no explicit color) + } else if (statusUpper.includes('OB')) { + statusText = 'On Battery'; + statusColor = '#d9534f'; // Red for on battery + } else if (statusUpper.includes('LB')) { + statusText = 'Low Battery'; + statusColor = '#d9534f'; // Red for low battery + } else { + statusText = upsStatus; + statusColor = '#f0ad4e'; // Orange for unknown status + } + + let statusStyle = statusColor ? ('color: ' + statusColor + ';') : ''; + statusLine += 'Status: ' + statusText + ''; + } else { + statusLine += 'Status: N/A'; + } + + // Battery charge + if (statusLine) statusLine += ' | '; + if (batteryCharge) { + const chargeColor = getPercentageColor(batteryCharge, false); + let chargeStyle = chargeColor ? ('color: ' + chargeColor + ';') : ''; + statusLine += 'Battery: ' + batteryCharge + '%'; + } else { + statusLine += 'Battery: N/A'; + } + + // Load percentage + if (statusLine) statusLine += ' | '; + if (upsLoad) { + const loadColor = getPercentageColor(upsLoad, true); + let loadStyle = loadColor ? ('color: ' + loadColor + ';') : ''; + statusLine += 'Load: ' + upsLoad + '%'; + } else { + statusLine += 'Load: N/A'; + } + + // Runtime + if (statusLine) statusLine += ' | '; + if (batteryRuntime) { + const runtime = parseInt(batteryRuntime); + const runtimeLowThreshold = batteryRuntimeLow ? parseInt(batteryRuntimeLow) : 600; + let runtimeColor = null; + if (runtime <= runtimeLowThreshold / 2) runtimeColor = '#d9534f'; // Red if less than half of low threshold + else if (runtime <= runtimeLowThreshold) runtimeColor = '#f0ad4e'; // Orange if at low threshold + let runtimeStyle = runtimeColor ? ('color: ' + runtimeColor + ';') : ''; + statusLine += 'Runtime: ' + formatRuntime(runtime) + ''; + } else { + statusLine += 'Runtime: N/A'; + } + + // Input voltage + if (statusLine) statusLine += ' | '; + if (inputVoltage) { + statusLine += 'Input: ' + parseFloat(inputVoltage).toFixed(0) + 'V'; + } else { + statusLine += 'Input: N/A'; + } + + // Calculate actual watt usage + if (statusLine) statusLine += ' | '; + let actualWattage = null; + if (upsLoad && upsRealPowerNominal) { + const load = parseFloat(upsLoad); + const nominal = parseFloat(upsRealPowerNominal); + if (!isNaN(load) && !isNaN(nominal)) { + actualWattage = Math.round((load / 100) * nominal); + } + } + + // Real power (calculated watt usage) + if (actualWattage !== null) { + statusLine += 'Output: ' + actualWattage + 'W'; + } else { + statusLine += 'Output: N/A'; + } + + displayItems.push(statusLine); + + // Combined battery and test line + let batteryTestLine = ''; + if (batteryMfrDate) { + batteryTestLine += 'Battery MFD: ' + batteryMfrDate + ''; + } else { + batteryTestLine += 'Battery MFD: N/A'; + } + + if (testResult && !testResult.toLowerCase().includes('no test')) { + const testColor = testResult.toLowerCase().includes('passed') ? null : '#d9534f'; + let testStyle = testColor ? ('color: ' + testColor + ';') : ''; + batteryTestLine += ' | Test: ' + testResult + ''; + } else { + batteryTestLine += ' | Test: N/A'; + } + + displayItems.push(batteryTestLine); + + // Format the final output + return '
' + displayItems.join('
') + '
'; + } + }, +EOF + #endregion UPS widget heredoc + if [[ $? -ne 0 ]]; then + echo "Error: Failed to generate UPS widget code" >&2 + exit 1 + fi +} + +#endregion widget generation functions # Function to uninstall the modification function uninstall_mod { @@ -767,7 +1565,7 @@ function uninstall_mod { check_root_privileges - if [[ -z $(grep -e "\$res->{PveMod_SensorInfo_JSON}" "$NODES_PM_FILE") ]] && [[ -z $(grep -e "\$res->{systemInfo}" "$NODES_PM_FILE") ]]; then + if [[ -z $(grep -e "$res->{sensorsOutput}" "$NODES_PM_FILE") ]] && [[ -z $(grep -e "$res->{systemInfo}" "$NODES_PM_FILE") ]]; then err "Mod is not installed." fi @@ -778,7 +1576,7 @@ function uninstall_mod { local latest_nodes_pm=$(find "$BACKUP_DIR" -name "Nodes.pm.*" -type f -printf '%T+ %p\n' 2>/dev/null | sort -r | head -n 1 | awk '{print $2}') if [ -n "$latest_nodes_pm" ]; then - # Restore the latest Nodes.pm file + # Remove the latest Nodes.pm file msgb "Restoring latest Nodes.pm from backup: $latest_nodes_pm to \"$NODES_PM_FILE\"." cp "$latest_nodes_pm" "$NODES_PM_FILE" info "Restored Nodes.pm successfully." @@ -786,11 +1584,11 @@ function uninstall_mod { warn "No Nodes.pm backup files found." fi - # Restore original pvemanagerlib.js (uncomment the StatusView definition and remove loader) + # Find the latest pvemanagerlib.js file using the find command local latest_pvemanagerlibjs=$(find "$BACKUP_DIR" -name "pvemanagerlib.js.*" -type f -printf '%T+ %p\n' 2>/dev/null | sort -r | head -n 1 | awk '{print $2}') if [ -n "$latest_pvemanagerlibjs" ]; then - # Restore the latest pvemanagerlib.js file + # Remove the latest pvemanagerlib.js file msgb "Restoring latest pvemanagerlib.js from backup: $latest_pvemanagerlibjs to \"$PVE_MANAGER_LIB_JS_FILE\"." cp "$latest_pvemanagerlibjs" "$PVE_MANAGER_LIB_JS_FILE" info "Restored pvemanagerlib.js successfully." @@ -798,41 +1596,8 @@ function uninstall_mod { warn "No pvemanagerlib.js backup files found." fi - # Remove UI module files - if [ -f "$PVE_MOD_JS_TARGET_FILE" ]; then - msgb "Removing UI module: $PVE_MOD_JS_TARGET_FILE" - rm "$PVE_MOD_JS_TARGET_FILE" - info "Removed UI module successfully." - else - warn "UI module file not found: $PVE_MOD_JS_TARGET_FILE" - fi - - # Remove Sensor Info Perl module - local latest_sensor_info_pm=$(find "$BACKUP_DIR" -name "PveMod_SensorInfo.pm.*" -type f -printf '%T+ %p\n' 2>/dev/null | sort -r | head -n 1 | awk '{print $2}') - - if [ -n "$latest_sensor_info_pm" ]; then - # Restore the latest PveMod_SensorInfo.pm file (if there's a backup) - msgb "Restoring latest PveMod_SensorInfo.pm from backup: $latest_sensor_info_pm to \"$PVE_SENSOR_INFO_MOD_FILE\"." - cp "$latest_sensor_info_pm" "$PVE_SENSOR_INFO_MOD_FILE" - info "Restored PveMod_SensorInfo.pm successfully." - elif [ -f "$PVE_SENSOR_INFO_MOD_FILE" ]; then - # No backup found but file exists, remove it - msgb "No PveMod_SensorInfo.pm backup found. Removing installed module: $PVE_SENSOR_INFO_MOD_FILE" - rm "$PVE_SENSOR_INFO_MOD_FILE" - info "Removed PveMod_SensorInfo.pm successfully." - else - warn "No PveMod_SensorInfo.pm backup files found and module not installed." - fi - - # Remove GPU RRD directory (contains historical GPU data — destroyed permanently on uninstall) - if [[ -d "$GPU_RRD_DIR" ]]; then - msgb "Removing GPU RRD directory: $GPU_RRD_DIR" - rm -rf "$GPU_RRD_DIR" - info "Removed GPU RRD directory." - fi - - if [ -n "$latest_nodes_pm" ] || [ -n "$latest_pvemanagerlibjs" ] || [ -f "$PVE_MOD_JS_TARGET_FILE" ] || [ -n "$latest_sensor_info_pm" ] || [ -f "$PVE_SENSOR_INFO_MOD_FILE" ]; then - # At least one file was modified, restart the proxy + if [ -n "$latest_nodes_pm" ] || [ -n "$latest_pvemanagerlibjs" ]; then + # At least one of the variables is not empty, restart the proxy restart_proxy fi @@ -841,11 +1606,9 @@ function uninstall_mod { # Function to check if the modification is installed check_mod_installation() { - if [[ -n $(grep -F 'use PVE::API2::PveMod_SensorInfo' "$NODES_PM_FILE") ]] || \ - [[ -n $(grep -F 'use PVE::API2::PVEMod_SensorInfo' "$NODES_PM_FILE") ]] || \ - [[ -n $(grep -F '$res->{sensorsJSONOutput}' "$NODES_PM_FILE") ]] || \ - [[ -n $(grep -F '$res->{systemInfo}' "$NODES_PM_FILE") ]] || \ - [[ -f "$PVE_MOD_JS_TARGET_FILE" ]]; then + if [[ -n $(grep -F '$res->{sensorsOutput}' "$NODES_PM_FILE") ]] && \ + [[ -n $(grep -F '$res->{systemInfo}' "$NODES_PM_FILE") ]] && \ + [[ -n $(grep -E "itemId: 'thermal[[:alnum:]]*'" "$PVE_MANAGER_LIB_JS_FILE") ]]; then err "Mod is already installed. Uninstall existing before installing." fi } @@ -929,37 +1692,32 @@ function create_file_backup() { local source_file="$1" local timestamp="$2" local filename - + filename=$(basename "$source_file") local backup_file="$BACKUP_DIR/${filename}.$timestamp" - + [[ -f "$source_file" ]] || err "Source file does not exist: $source_file" [[ -r "$source_file" ]] || err "Cannot read source file: $source_file" - + cp "$source_file" "$backup_file" || err "Failed to create backup: $backup_file" - + # Verify backup integrity if ! cmp -s "$source_file" "$backup_file"; then err "Backup verification failed for: $backup_file" fi - + info "Created backup: $backup_file" } function perform_backup { local timestamp timestamp=$(date +%Y%m%d_%H%M%S) - + msgb "\n=== Creating backups of modified files ===" create_backup_directory create_file_backup "$NODES_PM_FILE" "$timestamp" create_file_backup "$PVE_MANAGER_LIB_JS_FILE" "$timestamp" - - # Backup Sensor Info module if it exists - if [[ -f "$PVE_SENSOR_INFO_MOD_FILE" ]]; then - create_file_backup "$PVE_SENSOR_INFO_MOD_FILE" "$timestamp" - fi } # Process the arguments using a while loop and a case statement