Skip to content

Commit

Permalink
Merge pull request #5179 from wazuh/enhancement/vd-e2e-tests
Browse files Browse the repository at this point in the history 
Improvements for Vulnerability Detector E2E testing
  • Loading branch information
@davidjiglesias
davidjiglesias authored Apr 5, 2024
2 parents d784d5f + a5ba51d commit fc80e5b
Show file tree
Hide file tree
Showing 13 changed files with 924 additions and 478 deletions.
11 changes: 10 additions & 1 deletion CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ All notable changes to this project will be documented in this file.

### Added

- Add support for the installation/uninstallation of npm packages ([#5092](https://github.com/wazuh/wazuh-qa/pull/5092)) \- (Tests)
- Add alert.json file to Vulnerability Detector E2E test report ([#5147](https://github.com/wazuh/wazuh-qa/pull/5147)) \- (Framework)
- Add documentation about markers for system tests ([#5080](https://github.com/wazuh/wazuh-qa/pull/5080)) \- (Documentation)
- Add AWS Custom Buckets Integration tests ([#4675](https://github.com/wazuh/wazuh-qa/pull/4675)) \- (Framework + Tests)
- Add Vulnerability Detector end to end tests ([#4878](https://github.com/wazuh/wazuh-qa/pull/4878)) \- (Framework + Tests)
Expand All @@ -21,6 +23,7 @@ All notable changes to this project will be documented in this file.

### Changed

- Changes macOS packages with new ones that generate vulnerabilities ([#5174](https://github.com/wazuh/wazuh-qa/pull/5174)) \- (Tests)
- Refactor initial scan Vulnerability E2E tests ([#5081](https://github.com/wazuh/wazuh-qa/pull/5081)) \- (Framework + Tests)
- Update Packages in TestScanSyscollectorCases ([#4997](https://github.com/wazuh/wazuh-qa/pull/4997)) \- (Framework + Tests)
- Reduced test_shutdown_message runtime ([#4986](https://github.com/wazuh/wazuh-qa/pull/4986)) \- (Tests)
Expand Down Expand Up @@ -49,7 +52,13 @@ All notable changes to this project will be documented in this file.

### Fixed

- Fix enrollment cluster system tests ([#5134](https://github.com/wazuh/wazuh-qa/pull/5134/)) \- (Tests)
- Fix provision macOS endpoints with npm ([#5128](https://github.com/wazuh/wazuh-qa/pull/5158)) \- (Tests)
- Fix timestamps alerts and logs filter ([#5157](https://github.com/wazuh/wazuh-qa/pull/5157)) \- (Framework + Tests)
- Fix macOS and Windows agents timezone ([#5178](https://github.com/wazuh/wazuh-qa/pull/5178)) \- (Framework)
- Fix Vulnerability Detector E2E tests by adding description to all tests ([#5151](https://github.com/wazuh/wazuh-qa/pull/5151)) \- (Tests)
- Fix parser for non package vulnerabilities ([#5146](https://github.com/wazuh/wazuh-qa/pull/5146)) \- (Framework)
- Fix remote_operations_handler functions to Vulnerability Detector E2E tests ([#5155](https://github.com/wazuh/wazuh-qa/pull/5155)) \- (Framework)
- Fix enrollment cluster system tests ([#5134](https://github.com/wazuh/wazuh-qa/pull/5134)) \- (Tests)
- Fix `test_synchronization` system test ([#5089](https://github.com/wazuh/wazuh-qa/pull/5089)) \- (Framework + Tests)
- Fix number of files and their size for `test_zip_size_limit` ([#5133](https://github.com/wazuh/wazuh-qa/pull/5133)) \- (Tests)
- Fix test_shutdown_message system test ([#5087](https://github.com/wazuh/wazuh-qa/pull/5087)) \- (Tests)
Expand Down
22 changes: 22 additions & 0 deletions deps/wazuh_testing/wazuh_testing/end_to_end/indexer_api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,3 +93,25 @@ def get_indexer_values(host_manager: HostManager, credentials: dict = {'user': '
json=data)

return response.json()


def delete_index(host_manager: HostManager, credentials: dict = {'user': 'admin', 'password': 'changeme'},
index: str = 'wazuh-alerts*'):
"""
Delete index from the Wazuh Indexer API.
Args:
host_manager: An instance of the HostManager class containing information about hosts.
credentials (Optional): A dictionary containing the Indexer credentials. Defaults to
{'user': 'admin', 'password': 'changeme'}.
index (Optional): The Indexer index name. Defaults to 'wazuh-alerts*'.
"""
logging.info(f"Deleting {index} index")

url = f"https://{host_manager.get_master_ip()}:9200/{index}/"
headers = {
'Content-Type': 'application/json',
}

requests.delete(url=url, verify=False,
auth=requests.auth.HTTPBasicAuth(credentials['user'], credentials['password']), headers=headers)
19 changes: 19 additions & 0 deletions deps/wazuh_testing/wazuh_testing/end_to_end/logs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,29 @@ def get_hosts_logs(host_manager: HostManager, host_group: str = 'all') -> Dict[s
- host_manager (HostManager): An instance of the HostManager class for managing remote hosts.
- host_group (str, optional): The name of the host group where the files will be truncated.
Default is 'all'.
Returns:
- host_logs (Dict[str, str]): Dictionary containing the logs from the ossec.log file of each host
"""
host_logs = {}
for host in host_manager.get_group_hosts(host_group):
host_os_name = host_manager.get_host_variables(host)['os_name']
host_logs[host] = host_manager.get_file_content(host, logs_filepath_os[host_os_name])

return host_logs

def get_hosts_alerts(host_manager: HostManager) -> Dict[str, str]:
"""
Get the alerts in the alert.json file from the specified host group.
Parameters:
- host_manager (HostManager): An instance of the HostManager class for managing remote hosts.
Returns:
- host_alerts (Dict[str, str]): Dictionary containing the alerts from the alert.json file of each manager
"""
host_alerts = {}
for host in host_manager.get_group_hosts("manager"):
host_alerts[host] = host_manager.get_file_content(host, ALERTS_JSON_PATH)

return host_alerts
4 changes: 2 additions & 2 deletions deps/wazuh_testing/wazuh_testing/end_to_end/regex.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,10 @@
'parameters': ['HOST_NAME', 'CVE', 'PACKAGE_NAME', 'PACKAGE_VERSION', 'ARCHITECTURE']
},
'vuln_affected': {
'regex': 'CVE.*? affects.*"?'
'regex': 'CVE.* affects.*"?'
},
'vuln_mitigated': {
'regex': "The .* that affected .* was solved due to a package removal"
'regex': "The .* that affected .* was solved due to a package removal.*"
}
}

Expand Down
171 changes: 102 additions & 69 deletions deps/wazuh_testing/wazuh_testing/end_to_end/remote_operations_handler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
"""
import logging
from typing import Dict, List
from datetime import datetime
from datetime import datetime, timezone
from concurrent.futures import ThreadPoolExecutor

from wazuh_testing.end_to_end.waiters import wait_syscollector_and_vuln_scan
Expand Down Expand Up @@ -164,37 +164,47 @@ def install_package(host: str, operation_data: Dict[str, Dict], host_manager: Ho
package_id = None

if host_os_name in install_package_data:
if host_os_arch in install_package_data[host_os_name]:
package_id = install_package_data[host_os_name][host_os_arch]
else:
raise ValueError(f"Package for {host_os_name} and {host_os_arch} not found")
try:
if host_os_arch in install_package_data[host_os_name]:
package_id = install_package_data[host_os_name][host_os_arch]

package_data = load_packages_metadata()[package_id]
package_url = package_data['urls'][host_os_name][host_os_arch]
package_data = load_packages_metadata()[package_id]
package_url = package_data['urls'][host_os_name][host_os_arch]

logging.info(f"Installing package on {host}")
logging.info(f"Package URL: {package_url}")
logging.info(f"Installing package on {host}")
logging.info(f"Package URL: {package_url}")

current_datetime = datetime.utcnow().isoformat()
current_datetime = datetime.now(timezone.utc).isoformat()[:-6] # Delete timezone offset
use_npm = package_data.get('use_npm', False)

host_manager.install_package(host, package_url, system)
if use_npm:
host_manager.install_npm_package(host, package_url, system)
else:
host_manager.install_package(host, package_url, system)

logging.info(f"Package {package_url} installed on {host}")
logging.info(f"Package {package_url} installed on {host}")

logging.info(f"Package installed on {host}")
logging.info(f"Package installed on {host}")

results['checks']['all_successfull'] = True
results['checks']['all_successfull'] = True

wait_is_required = 'check' in operation_data and (operation_data['check']['alerts'] or
operation_data['check']['state_index'] or
operation_data['check']['no_alerts'] or
operation_data['check']['no_indices'])
wait_is_required = 'check' in operation_data and (operation_data['check']['alerts'] or
operation_data['check']['state_index'] or
operation_data['check']['no_alerts'] or
operation_data['check']['no_indices'])

if wait_is_required:
wait_syscollector_and_vuln_scan(host_manager, host, operation_data, current_datetime)
if wait_is_required:
wait_syscollector_and_vuln_scan(host_manager, host, operation_data, current_datetime)

check_vulnerability_alerts(results, operation_data['check'], current_datetime, host_manager, host,
package_data, operation='install')

else:
logging.error(f"Error: Package for {host_os_name} and {host_os_arch} not found")

except Exception as e:
logging.critical(f"Error searching package: {e}")

check_vulnerability_alerts(results, operation_data['check'], current_datetime, host_manager, host,
package_data, operation='install')
else:
logging.info(f"No operation to perform on {host}")

Expand Down Expand Up @@ -239,33 +249,42 @@ def remove_package(host: str, operation_data: Dict[str, Dict], host_manager: Hos
package_id = None

if host_os_name in package_data:
if host_os_arch in package_data[host_os_name]:
package_id = package_data[host_os_name][host_os_arch]
else:
raise ValueError(f"Package for {host_os_name} and {host_os_arch} not found")
try:
if host_os_arch in package_data[host_os_name]:
package_id = package_data[host_os_name][host_os_arch]

package_data = load_packages_metadata()[package_id]
use_npm = package_data.get('use_npm', False)

current_datetime = datetime.now(timezone.utc).isoformat()[:-6] # Delete timezone offset

package_data = load_packages_metadata()[package_id]
logging.info(f"Removing package on {host}")
if 'uninstall_name' in package_data:
uninstall_name = package_data['uninstall_name']
if use_npm:
host_manager.remove_npm_package(host, system, package_uninstall_name=uninstall_name)
else:
host_manager.remove_package(host, system, package_uninstall_name=uninstall_name)
elif 'uninstall_custom_playbook' in package_data:
host_manager.remove_package(host, system,
custom_uninstall_playbook=package_data['uninstall_custom_playbook'])

current_datetime = datetime.utcnow().isoformat()
wait_is_required = 'check' in operation_data and (operation_data['check']['alerts'] or
operation_data['check']['state_index'] or
operation_data['check']['no_alerts'] or
operation_data['check']['no_indices'])

logging.info(f"Removing package on {host}")
if 'uninstall_name' in package_data:
uninstall_name = package_data['uninstall_name']
host_manager.remove_package(host, system, package_uninstall_name=uninstall_name)
elif 'uninstall_custom_playbook' in package_data:
host_manager.remove_package(host, system,
custom_uninstall_playbook=package_data['uninstall_custom_playbook'])
if wait_is_required:
wait_syscollector_and_vuln_scan(host_manager, host, operation_data, current_datetime)

wait_is_required = 'check' in operation_data and (operation_data['check']['alerts'] or
operation_data['check']['state_index'] or
operation_data['check']['no_alerts'] or
operation_data['check']['no_indices'])
check_vulnerability_alerts(results, operation_data['check'], current_datetime, host_manager, host,
package_data, operation='remove')

if wait_is_required:
wait_syscollector_and_vuln_scan(host_manager, host, operation_data, current_datetime)
else:
logging.error(f"Error: Package for {host_os_name} and {host_os_arch} not found")

check_vulnerability_alerts(results, operation_data['check'], current_datetime, host_manager, host,
package_data, operation='remove')
except Exception as e:
logging.critical(f"Error searching package: {e}")

else:
logging.info(f"No operation to perform on {host}")
Expand Down Expand Up @@ -316,41 +335,55 @@ def update_package(host: str, operation_data: Dict[str, Dict], host_manager: Hos
package_id_to = None

if host_os_name in install_package_data_from:
if host_os_arch in install_package_data_from[host_os_name]:
package_id_from = install_package_data_from[host_os_name][host_os_arch]
else:
raise ValueError(f"Package for {host_os_name} and {host_os_arch} not found")
try:
if host_os_arch in install_package_data_from[host_os_name]:
package_id_from = install_package_data_from[host_os_name][host_os_arch]
else:
logging.error(f"Error: Package for {host_os_name} and {host_os_arch} not found")
except Exception as e:
logging.critical(f"Error searching package: {e}")

if host_os_name in install_package_data_to:
if host_os_arch in install_package_data_to[host_os_name]:
package_id_to = install_package_data_to[host_os_name][host_os_arch]
else:
raise ValueError(f"Package for {host_os_name} and {host_os_arch} not found")
try:
if host_os_arch in install_package_data_to[host_os_name]:
package_id_to = install_package_data_to[host_os_name][host_os_arch]

package_data_from = load_packages_metadata()[package_id_from]
package_data_to = load_packages_metadata()[package_id_to]

package_url_to = package_data_to['urls'][host_os_name][host_os_arch]

logging.info(f"Installing package on {host}")
logging.info(f"Package URL: {package_url_to}")

current_datetime = datetime.now(timezone.utc).isoformat()[:-6] # Delete timezone offset
use_npm = package_data_to.get('use_npm', False)

package_data_from = load_packages_metadata()[package_id_from]
package_data_to = load_packages_metadata()[package_id_to]
if use_npm:
host_manager.install_npm_package(host, package_url_to, system)
else:
host_manager.install_package(host, package_url_to, system)

package_url_to = package_data_to['urls'][host_os_name][host_os_arch]
logging.info(f"Package {package_url_to} installed on {host}")

logging.info(f"Installing package on {host}")
logging.info(f"Package URL: {package_url_to}")
logging.info(f"Package installed on {host}")

current_datetime = datetime.utcnow().isoformat()
host_manager.install_package(host, package_url_to, system)
wait_is_required = 'check' in operation_data and (operation_data['check']['alerts'] or
operation_data['check']['state_index'] or
operation_data['check']['no_alerts'] or
operation_data['check']['no_indices'])
if wait_is_required:
wait_syscollector_and_vuln_scan(host_manager, host, operation_data, current_datetime)

logging.info(f"Package {package_url_to} installed on {host}")
check_vulnerability_alerts(results, operation_data['check'], current_datetime, host_manager, host,
{'from': package_data_from, 'to': package_data_to}, operation='update')

logging.info(f"Package installed on {host}")
else:
logging.error(f"Error: Package for {host_os_name} and {host_os_arch} not found")

wait_is_required = 'check' in operation_data and (operation_data['check']['alerts'] or
operation_data['check']['state_index'] or
operation_data['check']['no_alerts'] or
operation_data['check']['no_indices'])
if wait_is_required:
wait_syscollector_and_vuln_scan(host_manager, host, operation_data, current_datetime)
except Exception as e:
logging.critical(f"Error searching package: {e}")

check_vulnerability_alerts(results, operation_data['check'], current_datetime, host_manager, host,
{'from': package_data_from, 'to': package_data_to}, operation='update')
else:
logging.info(f"No operation to perform on {host}")

Expand All @@ -369,7 +402,7 @@ def launch_remote_sequential_operation_on_agent(agent: str, task_list: List[Dict
host_manager (HostManager): An instance of the HostManager class containing information about hosts.
"""
# Convert datetime to Unix timestamp (integer)
timestamp = datetime.utcnow().isoformat()
timestamp = datetime.now(timezone.utc).isoformat()[:-6] # Delete timezone offset

if task_list:
for task in task_list:
Expand Down
Loading

0 comments on commit fc80e5b

Please sign in to comment.