From b092a45f29d174016cec4e5362518eb8aebbe33e Mon Sep 17 00:00:00 2001
From: Lea9250 <lea.droguet@factorfx.com>
Date: Fri, 13 Sep 2024 17:41:14 +0200
Subject: [PATCH 1/3] feat(SnmpFork): add forking module for SNMP scanning
---
etc/ocsinventory-agent/modules.conf | 1 +
lib/Ocsinventory/Agent/Modules/SnmpFork.pm | 130 +++++++++++++++++++++
postinst.pl | 2 +
3 files changed, 133 insertions(+)
create mode 100644 lib/Ocsinventory/Agent/Modules/SnmpFork.pm
diff --git a/etc/ocsinventory-agent/modules.conf b/etc/ocsinventory-agent/modules.conf
index 55dc52c9..c9d34a5b 100644
--- a/etc/ocsinventory-agent/modules.conf
+++ b/etc/ocsinventory-agent/modules.conf
@@ -6,6 +6,7 @@
use Ocsinventory::Agent::Modules::Download;
use Ocsinventory::Agent::Modules::SnmpScan;
#use Ocsinventory::Agent::Modules::LocalSnmpScan;
+use Ocsinventory::Agent::Modules::SnmpFork;
# DO NOT REMOVE the 1;
1;
diff --git a/lib/Ocsinventory/Agent/Modules/SnmpFork.pm b/lib/Ocsinventory/Agent/Modules/SnmpFork.pm
new file mode 100644
index 00000000..7a5c45e0
--- /dev/null
+++ b/lib/Ocsinventory/Agent/Modules/SnmpFork.pm
@@ -0,0 +1,130 @@
+###############################################################################
+## OCSINVENTORY-NG
+## Copyleft OCS Inventory NG Team
+## Web : http://www.ocsinventory-ng.org
+##
+## Wrapper for SNMP scan (local and online mode) that handles the forking of the
+## SNMP scan process
+##
+## This code is open source and may be copied and modified as long as the source
+## code is always made freely available.
+## Please refer to the General Public Licence http://www.gnu.org/ or Licence.txt
+################################################################################
+
+package Ocsinventory::Agent::Modules::SnmpFork;
+
+use strict;
+no strict 'refs';
+no strict 'subs';
+use warnings;
+
+use XML::Simple;
+use Digest::MD5;
+use Net::Netmask;
+
+
+# launch the SNMP scan in a forked process
+# takes the native scan function to call, subnets to scan, nb of forks and self
+sub fork_snmpscan {
+ my ($scan_function, $nets_to_scan, $fork_count, $self) = @_;
+
+ my $logger = $self->{logger};
+
+ # get fork count from config or calculate it
+ $fork_count = $fork_count // 0;
+ if ($fork_count !~ /^\d+$/ || $fork_count <= 0) {
+ $logger->debug("Invalid fork_nb value in config: $fork_count. Falling back to calculated value.");
+ $fork_count = get_forks_nb();
+ }
+
+ # split nets_to_scan among forks
+ my @split_nets_to_scan = split_array_across_forks($nets_to_scan, $fork_count);
+
+ my @pipes;
+ my @aggregated_content;
+
+ # fork processes
+ for (my $i = 0; $i < $fork_count; $i++) {
+ # pipe
+ my ($reader, $writer);
+ pipe($reader, $writer);
+ $reader->autoflush(1);
+ $writer->autoflush(1);
+ push @pipes, $reader;
+
+ my $pid = fork();
+ if ($pid) {
+ # parent
+ close $writer;
+ } elsif (defined $pid) {
+ # child
+ close $reader;
+ my $subnets_to_scan = $split_nets_to_scan[$i];
+
+ # calling scan function
+ my $xml_result = $scan_function->($self, $subnets_to_scan);
+
+ # write xml result to pipe
+ print $writer $xml_result;
+ close $writer;
+ exit 0;
+ } else {
+ $logger->error("Fork failed: $!");
+ }
+ }
+
+ # parent process: read and aggregate XML from pipes
+ foreach my $reader (@pipes) {
+ while (my $line = <$reader>) {
+ push @aggregated_content, $line;
+ }
+ close $reader;
+ }
+
+ # wait for all child processes to finish
+ my $child_pid;
+ while (($child_pid = waitpid(-1, 0)) > 0) {
+ $logger->debug("Child process $child_pid finished with exit code $?");
+ }
+
+ # aggregated content into one content block
+ my $content_block = join("", @aggregated_content);
+
+ # final XML structure
+ my $final_xml = <<"END_XML";
+<?xml version="1.0" encoding="UTF-8"?>
+<REQUEST>
+ <CONTENT>
+ $content_block
+ </CONTENT>
+ <DEVICEID>$self->{context}->{config}->{deviceid}</DEVICEID>
+ <QUERY>SNMP</QUERY>
+</REQUEST>
+END_XML
+
+ return $final_xml;
+}
+
+# split the array of IPs into even portions for each fork
+sub split_array_across_forks {
+ my ($nets_to_scan, $fork_count) = @_;
+ my @split_nets;
+
+ my $i = 0;
+ foreach my $subnet (@$nets_to_scan) {
+ push(@{$split_nets[$i]}, $subnet);
+ $i = ($i + 1) % $fork_count;
+ }
+
+ return @split_nets;
+}
+
+# default nb of forks is nb of cores
+sub get_forks_nb {
+ my $cores = `nproc`;
+ chomp($cores);
+ return $cores;
+}
+
+
+1;
diff --git a/postinst.pl b/postinst.pl
index 311f9c31..78d4eb4d 100644
--- a/postinst.pl
+++ b/postinst.pl
@@ -458,6 +458,8 @@
print MODULE "\n";
print MODULE "#use Ocsinventory::Agent::Modules::LocalSnmpScan;\n";
print MODULE "\n";
+print MODULE "use Ocsinventory::Agent::Modules::SnmpFork;\n";
+print MODULE "\n";
print MODULE "# DO NOT REMOVE THE 1;\n";
print MODULE "1;\n";
close MODULE;
From 97e74a1ced5b73284e90d863e6b9f9b48b52927d Mon Sep 17 00:00:00 2001
From: Lea9250 <lea.droguet@factorfx.com>
Date: Fri, 13 Sep 2024 17:48:11 +0200
Subject: [PATCH 2/3] refactor(SnmpScan): implement SNMP forking into scanning
module
---
lib/Ocsinventory/Agent/Modules/SnmpScan.pm | 111 +++++++++++++++------
1 file changed, 83 insertions(+), 28 deletions(-)
diff --git a/lib/Ocsinventory/Agent/Modules/SnmpScan.pm b/lib/Ocsinventory/Agent/Modules/SnmpScan.pm
index 3208de94..dc35e495 100644
--- a/lib/Ocsinventory/Agent/Modules/SnmpScan.pm
+++ b/lib/Ocsinventory/Agent/Modules/SnmpScan.pm
@@ -9,6 +9,7 @@
################################################################################
package Ocsinventory::Agent::Modules::SnmpScan;
+use Ocsinventory::Agent::Modules::SnmpFork ();
use strict;
no strict 'refs';
@@ -55,6 +56,12 @@ sub snmpscan_start_handler {
$logger->debug("Calling snmp_start_handler");
+ if ($config->{forking_enabled}) {
+ $logger->debug("SNMP Forking is enabled");
+ } else {
+ $logger->debug("SNMP Forking is disabled");
+ }
+
# Disabling module if local mode
if ($config->{stdout} || $config->{local}) {
$self->{disabled} = 1;
@@ -154,23 +161,7 @@ sub snmpscan_end_handler {
# We get the config
my $config = $self->{context}->{config};
- # Load setting from the config file
- my $configagent = new Ocsinventory::Agent::Config;
- $configagent->loadUserParams();
-
- my $communities=$self->{communities};
- if ( ! defined ($communities ) ) {
- $logger->debug("We have no Community from server, we use default public community");
- $communities=[{VERSION=>"2c",NAME=>"public"}];
- }
-
- my ($name,$comm,$error,$system_oid);
-
- # Initalising the XML properties
- my $snmp_inventory = $self->{inventory};
- $snmp_inventory->{xmlroot}->{QUERY} = ['SNMP'];
- $snmp_inventory->{xmlroot}->{DEVICEID} = [$self->{context}->{config}->{deviceid}];
# Scanning network
$logger->debug("Snmp: Scanning network");
@@ -182,15 +173,65 @@ sub snmpscan_end_handler {
my $net_to_scan = [];
$self->snmp_ip_scan($net_to_scan);
} else {
- foreach my $net_to_scan ( @$nets_to_scan ){
+ foreach my $net_to_scan (@$nets_to_scan) {
$self->snmp_ip_scan($net_to_scan);
}
}
$logger->debug("Snmp: Ending Scanning network");
+ my $xml_inventory;
+ if ($config->{forking_enabled}) {
+ $xml_inventory = Ocsinventory::Agent::Modules::SnmpFork::fork_snmpscan(\&perform_snmp_scan, $self->{netdevices}, $config->{fork_count}, $self);
+ } else {
+ $xml_inventory = $self->perform_snmp_scan();
+ }
+
+ $self->handleXml($xml_inventory);
+
+ $logger->debug("End snmp_end_handler :)");
+}
+
+sub perform_snmp_scan {
+ my $self = shift;
+ my $logger = $self->{logger};
+ my $common = $self->{context}->{common};
+ my $network = $self->{context}->{network};
+
# Begin scanning ip tables
my $ip=$self->{netdevices};
+ # identify the process
+ my $forked = 0;
+ if ($self->{context}->{config}->{forking_enabled}) {
+ $forked = 1;
+ $ip = shift;
+ }
+
+ my $communities=$self->{communities};
+
+ if ( ! defined ($communities ) ) {
+ $logger->debug("We have no Community from server, we use default public community");
+ $communities=[{VERSION=>"2c",NAME=>"public"}];
+ }
+ my ($name,$comm,$error,$system_oid);
+
+ # Load setting from the config file
+ my $configagent = new Ocsinventory::Agent::Config;
+ $configagent->loadUserParams();
+
+ # Initalising the XML properties
+ my $snmp_inventory = $self->{inventory};
+ $snmp_inventory->{xmlroot}->{QUERY} = ['SNMP'];
+ $snmp_inventory->{xmlroot}->{DEVICEID} = [$self->{context}->{config}->{deviceid}];
+
+
+ my $pidlog;
+ if ($forked) {
+ $pidlog = "[$$]";
+ } else {
+ $pidlog = "";
+ }
+
foreach my $device ( @$ip ) {
my $session = undef;
my $oid_condition = undef;
@@ -200,7 +241,7 @@ sub snmpscan_end_handler {
my $snmp_condition_value = undef;
my $regex = undef;
- $logger->debug("Scanning $device->{IPADDR} device");
+ $logger->debug("$pidlog Scanning device $device->{IPADDR} device");
# Search for the good snmp community in the table community
LIST_SNMP: foreach $comm ( @$communities ) {
# Test if we use SNMP v3
@@ -272,7 +313,7 @@ sub snmpscan_end_handler {
);
};
unless (defined($session)) {
- $logger->error("Snmp INFO: $error");
+ $logger->error("$pidlog Snmp INFO: $error");
} else {
$self->{snmp_session}=$session;
@@ -361,17 +402,31 @@ sub snmpscan_end_handler {
}
}
- $logger->info("No more SNMP device to scan");
-
- # Formatting the XML and sendig it to the server
- my $content = XMLout( $snmp_inventory->{xmlroot}, RootName => 'REQUEST' , XMLDecl => '<?xml version="1.0" encoding="UTF-8"?>', SuppressEmpty => undef );
+ $logger->info("$pidlog No more SNMP device to scan");
+ my $clean_content;
+ my $content;
+ if ($forked) {
+ $content = XMLout($snmp_inventory->{xmlroot}, RootName => 'REQUEST', XMLDecl => '<?xml version="1.0" encoding="UTF-8"?>', SuppressEmpty => undef);
+ $content = extract_content_tag($content);
+ $logger->debug("$pidlog Sending XML content to parent process");
+ } else {
+ # Formatting the XML and sendig it to the server
+ $content = XMLout( $snmp_inventory->{xmlroot}, RootName => 'REQUEST' , XMLDecl => '<?xml version="1.0" encoding="UTF-8"?>', SuppressEmpty => undef );
+ }
#Cleaning XML to delete unprintable characters
- my $clean_content = $common->cleanXml($content);
-
- $self->handleXml($clean_content);
+ $clean_content = $common->cleanXml($content);
+
+ return $clean_content;
+}
- $logger->debug("End snmp_end_handler :)");
+# extract CONTENT tag (used in perform_snmp_scan for forking)
+sub extract_content_tag {
+ my ($xml_string) = @_;
+ if ($xml_string =~ m|<CONTENT>(.*?)</CONTENT>|s) {
+ return $1;
+ }
+ return '';
}
sub snmp_ip_scan {
@@ -402,7 +457,7 @@ sub snmp_ip_scan {
$ping->close();
} elsif ($snmp_scan_type eq 'NMAP' && $common->can_load('Nmap::Parser')) {
- $logger->debug("Scannig $net_to_scan with nmap");
+ $logger->debug("Scanning $net_to_scan with nmap");
my $nmaparser = Nmap::Parser->new;
$nmaparser->parsescan("nmap","-sn",$net_to_scan);
From 16b3ca69919f227049222c9b835fcb483b5c1a49 Mon Sep 17 00:00:00 2001
From: Lea9250 <lea.droguet@factorfx.com>
Date: Tue, 17 Sep 2024 15:35:19 +0200
Subject: [PATCH 3/3] refactor(SnmpFork): remove unused dep
---
lib/Ocsinventory/Agent/Modules/SnmpFork.pm | 1 -
1 file changed, 1 deletion(-)
diff --git a/lib/Ocsinventory/Agent/Modules/SnmpFork.pm b/lib/Ocsinventory/Agent/Modules/SnmpFork.pm
index 7a5c45e0..a3c395a8 100644
--- a/lib/Ocsinventory/Agent/Modules/SnmpFork.pm
+++ b/lib/Ocsinventory/Agent/Modules/SnmpFork.pm
@@ -20,7 +20,6 @@ use warnings;
use XML::Simple;
use Digest::MD5;
-use Net::Netmask;
# launch the SNMP scan in a forked process