From b092a45f29d174016cec4e5362518eb8aebbe33e Mon Sep 17 00:00:00 2001 From: Lea9250 Date: Fri, 13 Sep 2024 17:41:14 +0200 Subject: [PATCH 1/3] feat(SnmpFork): add forking module for SNMP scanning --- etc/ocsinventory-agent/modules.conf | 1 + lib/Ocsinventory/Agent/Modules/SnmpFork.pm | 130 +++++++++++++++++++++ postinst.pl | 2 + 3 files changed, 133 insertions(+) create mode 100644 lib/Ocsinventory/Agent/Modules/SnmpFork.pm diff --git a/etc/ocsinventory-agent/modules.conf b/etc/ocsinventory-agent/modules.conf index 55dc52c9..c9d34a5b 100644 --- a/etc/ocsinventory-agent/modules.conf +++ b/etc/ocsinventory-agent/modules.conf @@ -6,6 +6,7 @@ use Ocsinventory::Agent::Modules::Download; use Ocsinventory::Agent::Modules::SnmpScan; #use Ocsinventory::Agent::Modules::LocalSnmpScan; +use Ocsinventory::Agent::Modules::SnmpFork; # DO NOT REMOVE the 1; 1; diff --git a/lib/Ocsinventory/Agent/Modules/SnmpFork.pm b/lib/Ocsinventory/Agent/Modules/SnmpFork.pm new file mode 100644 index 00000000..7a5c45e0 --- /dev/null +++ b/lib/Ocsinventory/Agent/Modules/SnmpFork.pm @@ -0,0 +1,130 @@ +############################################################################### +## OCSINVENTORY-NG +## Copyleft OCS Inventory NG Team +## Web : http://www.ocsinventory-ng.org +## +## Wrapper for SNMP scan (local and online mode) that handles the forking of the +## SNMP scan process +## +## This code is open source and may be copied and modified as long as the source +## code is always made freely available. +## Please refer to the General Public Licence http://www.gnu.org/ or Licence.txt +################################################################################ + +package Ocsinventory::Agent::Modules::SnmpFork; + +use strict; +no strict 'refs'; +no strict 'subs'; +use warnings; + +use XML::Simple; +use Digest::MD5; +use Net::Netmask; + + +# launch the SNMP scan in a forked process +# takes the native scan function to call, subnets to scan, nb of forks and self +sub fork_snmpscan { + my ($scan_function, $nets_to_scan, $fork_count, $self) = @_; + + my $logger = $self->{logger}; + + # get fork count from config or calculate it + $fork_count = $fork_count // 0; + if ($fork_count !~ /^\d+$/ || $fork_count <= 0) { + $logger->debug("Invalid fork_nb value in config: $fork_count. Falling back to calculated value."); + $fork_count = get_forks_nb(); + } + + # split nets_to_scan among forks + my @split_nets_to_scan = split_array_across_forks($nets_to_scan, $fork_count); + + my @pipes; + my @aggregated_content; + + # fork processes + for (my $i = 0; $i < $fork_count; $i++) { + # pipe + my ($reader, $writer); + pipe($reader, $writer); + $reader->autoflush(1); + $writer->autoflush(1); + push @pipes, $reader; + + my $pid = fork(); + if ($pid) { + # parent + close $writer; + } elsif (defined $pid) { + # child + close $reader; + my $subnets_to_scan = $split_nets_to_scan[$i]; + + # calling scan function + my $xml_result = $scan_function->($self, $subnets_to_scan); + + # write xml result to pipe + print $writer $xml_result; + close $writer; + exit 0; + } else { + $logger->error("Fork failed: $!"); + } + } + + # parent process: read and aggregate XML from pipes + foreach my $reader (@pipes) { + while (my $line = <$reader>) { + push @aggregated_content, $line; + } + close $reader; + } + + # wait for all child processes to finish + my $child_pid; + while (($child_pid = waitpid(-1, 0)) > 0) { + $logger->debug("Child process $child_pid finished with exit code $?"); + } + + # aggregated content into one content block + my $content_block = join("", @aggregated_content); + + # final XML structure + my $final_xml = <<"END_XML"; + + + + $content_block + + $self->{context}->{config}->{deviceid} + SNMP + +END_XML + + return $final_xml; +} + +# split the array of IPs into even portions for each fork +sub split_array_across_forks { + my ($nets_to_scan, $fork_count) = @_; + my @split_nets; + + my $i = 0; + foreach my $subnet (@$nets_to_scan) { + push(@{$split_nets[$i]}, $subnet); + $i = ($i + 1) % $fork_count; + } + + return @split_nets; +} + +# default nb of forks is nb of cores +sub get_forks_nb { + my $cores = `nproc`; + chomp($cores); + return $cores; +} + + +1; diff --git a/postinst.pl b/postinst.pl index 311f9c31..78d4eb4d 100644 --- a/postinst.pl +++ b/postinst.pl @@ -458,6 +458,8 @@ print MODULE "\n"; print MODULE "#use Ocsinventory::Agent::Modules::LocalSnmpScan;\n"; print MODULE "\n"; +print MODULE "use Ocsinventory::Agent::Modules::SnmpFork;\n"; +print MODULE "\n"; print MODULE "# DO NOT REMOVE THE 1;\n"; print MODULE "1;\n"; close MODULE; From 97e74a1ced5b73284e90d863e6b9f9b48b52927d Mon Sep 17 00:00:00 2001 From: Lea9250 Date: Fri, 13 Sep 2024 17:48:11 +0200 Subject: [PATCH 2/3] refactor(SnmpScan): implement SNMP forking into scanning module --- lib/Ocsinventory/Agent/Modules/SnmpScan.pm | 111 +++++++++++++++------ 1 file changed, 83 insertions(+), 28 deletions(-) diff --git a/lib/Ocsinventory/Agent/Modules/SnmpScan.pm b/lib/Ocsinventory/Agent/Modules/SnmpScan.pm index 3208de94..dc35e495 100644 --- a/lib/Ocsinventory/Agent/Modules/SnmpScan.pm +++ b/lib/Ocsinventory/Agent/Modules/SnmpScan.pm @@ -9,6 +9,7 @@ ################################################################################ package Ocsinventory::Agent::Modules::SnmpScan; +use Ocsinventory::Agent::Modules::SnmpFork (); use strict; no strict 'refs'; @@ -55,6 +56,12 @@ sub snmpscan_start_handler { $logger->debug("Calling snmp_start_handler"); + if ($config->{forking_enabled}) { + $logger->debug("SNMP Forking is enabled"); + } else { + $logger->debug("SNMP Forking is disabled"); + } + # Disabling module if local mode if ($config->{stdout} || $config->{local}) { $self->{disabled} = 1; @@ -154,23 +161,7 @@ sub snmpscan_end_handler { # We get the config my $config = $self->{context}->{config}; - # Load setting from the config file - my $configagent = new Ocsinventory::Agent::Config; - $configagent->loadUserParams(); - - my $communities=$self->{communities}; - if ( ! defined ($communities ) ) { - $logger->debug("We have no Community from server, we use default public community"); - $communities=[{VERSION=>"2c",NAME=>"public"}]; - } - - my ($name,$comm,$error,$system_oid); - - # Initalising the XML properties - my $snmp_inventory = $self->{inventory}; - $snmp_inventory->{xmlroot}->{QUERY} = ['SNMP']; - $snmp_inventory->{xmlroot}->{DEVICEID} = [$self->{context}->{config}->{deviceid}]; # Scanning network $logger->debug("Snmp: Scanning network"); @@ -182,15 +173,65 @@ sub snmpscan_end_handler { my $net_to_scan = []; $self->snmp_ip_scan($net_to_scan); } else { - foreach my $net_to_scan ( @$nets_to_scan ){ + foreach my $net_to_scan (@$nets_to_scan) { $self->snmp_ip_scan($net_to_scan); } } $logger->debug("Snmp: Ending Scanning network"); + my $xml_inventory; + if ($config->{forking_enabled}) { + $xml_inventory = Ocsinventory::Agent::Modules::SnmpFork::fork_snmpscan(\&perform_snmp_scan, $self->{netdevices}, $config->{fork_count}, $self); + } else { + $xml_inventory = $self->perform_snmp_scan(); + } + + $self->handleXml($xml_inventory); + + $logger->debug("End snmp_end_handler :)"); +} + +sub perform_snmp_scan { + my $self = shift; + my $logger = $self->{logger}; + my $common = $self->{context}->{common}; + my $network = $self->{context}->{network}; + # Begin scanning ip tables my $ip=$self->{netdevices}; + # identify the process + my $forked = 0; + if ($self->{context}->{config}->{forking_enabled}) { + $forked = 1; + $ip = shift; + } + + my $communities=$self->{communities}; + + if ( ! defined ($communities ) ) { + $logger->debug("We have no Community from server, we use default public community"); + $communities=[{VERSION=>"2c",NAME=>"public"}]; + } + my ($name,$comm,$error,$system_oid); + + # Load setting from the config file + my $configagent = new Ocsinventory::Agent::Config; + $configagent->loadUserParams(); + + # Initalising the XML properties + my $snmp_inventory = $self->{inventory}; + $snmp_inventory->{xmlroot}->{QUERY} = ['SNMP']; + $snmp_inventory->{xmlroot}->{DEVICEID} = [$self->{context}->{config}->{deviceid}]; + + + my $pidlog; + if ($forked) { + $pidlog = "[$$]"; + } else { + $pidlog = ""; + } + foreach my $device ( @$ip ) { my $session = undef; my $oid_condition = undef; @@ -200,7 +241,7 @@ sub snmpscan_end_handler { my $snmp_condition_value = undef; my $regex = undef; - $logger->debug("Scanning $device->{IPADDR} device"); + $logger->debug("$pidlog Scanning device $device->{IPADDR} device"); # Search for the good snmp community in the table community LIST_SNMP: foreach $comm ( @$communities ) { # Test if we use SNMP v3 @@ -272,7 +313,7 @@ sub snmpscan_end_handler { ); }; unless (defined($session)) { - $logger->error("Snmp INFO: $error"); + $logger->error("$pidlog Snmp INFO: $error"); } else { $self->{snmp_session}=$session; @@ -361,17 +402,31 @@ sub snmpscan_end_handler { } } - $logger->info("No more SNMP device to scan"); - - # Formatting the XML and sendig it to the server - my $content = XMLout( $snmp_inventory->{xmlroot}, RootName => 'REQUEST' , XMLDecl => '', SuppressEmpty => undef ); + $logger->info("$pidlog No more SNMP device to scan"); + my $clean_content; + my $content; + if ($forked) { + $content = XMLout($snmp_inventory->{xmlroot}, RootName => 'REQUEST', XMLDecl => '', SuppressEmpty => undef); + $content = extract_content_tag($content); + $logger->debug("$pidlog Sending XML content to parent process"); + } else { + # Formatting the XML and sendig it to the server + $content = XMLout( $snmp_inventory->{xmlroot}, RootName => 'REQUEST' , XMLDecl => '', SuppressEmpty => undef ); + } #Cleaning XML to delete unprintable characters - my $clean_content = $common->cleanXml($content); - - $self->handleXml($clean_content); + $clean_content = $common->cleanXml($content); + + return $clean_content; +} - $logger->debug("End snmp_end_handler :)"); +# extract CONTENT tag (used in perform_snmp_scan for forking) +sub extract_content_tag { + my ($xml_string) = @_; + if ($xml_string =~ m|(.*?)|s) { + return $1; + } + return ''; } sub snmp_ip_scan { @@ -402,7 +457,7 @@ sub snmp_ip_scan { $ping->close(); } elsif ($snmp_scan_type eq 'NMAP' && $common->can_load('Nmap::Parser')) { - $logger->debug("Scannig $net_to_scan with nmap"); + $logger->debug("Scanning $net_to_scan with nmap"); my $nmaparser = Nmap::Parser->new; $nmaparser->parsescan("nmap","-sn",$net_to_scan); From 16b3ca69919f227049222c9b835fcb483b5c1a49 Mon Sep 17 00:00:00 2001 From: Lea9250 Date: Tue, 17 Sep 2024 15:35:19 +0200 Subject: [PATCH 3/3] refactor(SnmpFork): remove unused dep --- lib/Ocsinventory/Agent/Modules/SnmpFork.pm | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/Ocsinventory/Agent/Modules/SnmpFork.pm b/lib/Ocsinventory/Agent/Modules/SnmpFork.pm index 7a5c45e0..a3c395a8 100644 --- a/lib/Ocsinventory/Agent/Modules/SnmpFork.pm +++ b/lib/Ocsinventory/Agent/Modules/SnmpFork.pm @@ -20,7 +20,6 @@ use warnings; use XML::Simple; use Digest::MD5; -use Net::Netmask; # launch the SNMP scan in a forked process