diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index a99d024b7..26760863f 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -882,6 +882,100 @@ services: timeout: 15s retries: 3 start_period: 60s + netbox: + image: ghcr.io/mmguero/netbox:latest + restart: "no" + stdin_open: false + tty: true + hostname: netbox + networks: + - default + environment: + << : *process-variables + VIRTUAL_HOST : 'netbox.malcolm.local' + env_file: ./netbox/env/netbox.env + depends_on: + - netbox-postgres + - netbox-redis + - netbox-redis-cache + volumes: + - ./netbox/config/configuration:/etc/netbox/config:ro + - ./netbox/config/reports:/etc/netbox/reports:ro + - ./netbox/config/scripts:/etc/netbox/scripts:ro + - ./netbox/config/unit:/etc/unit:ro + - ./netbox/media:/opt/netbox/netbox/media:rw + healthcheck: + test: ["CMD", "curl", "--silent", "http://localhost:8080/assets/api/" ] + interval: 60s + timeout: 15s + retries: 3 + start_period: 120s + netbox-postgres: + image: ghcr.io/mmguero/postgres:14-alpine + restart: "no" + stdin_open: false + tty: true + hostname: netbox-postgres + networks: + - default + environment: + << : *process-variables + VIRTUAL_HOST : 'netbox-postgres.malcolm.local' + env_file: ./netbox/env/postgres.env + volumes: + - ./netbox/postgres:/var/lib/postgresql/data:rw + healthcheck: + test: [ "CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}" ] + interval: 60s + timeout: 15s + retries: 3 + start_period: 45s + netbox-redis: + image: ghcr.io/mmguero/redis:7-alpine + restart: "no" + stdin_open: false + tty: true + hostname: netbox-redis + networks: + - default + environment: + << : *process-variables + VIRTUAL_HOST : 'netbox-redis.malcolm.local' + env_file: ./netbox/env/redis.env + command: + - sh + - -c + - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD + volumes: + - ./netbox/redis:/data + healthcheck: + test: ["CMD-SHELL", "pidof redis-server || exit 1" ] + interval: 60s + timeout: 15s + retries: 3 + start_period: 45s + netbox-redis-cache: + image: ghcr.io/mmguero/redis:7-alpine + restart: "no" + stdin_open: false + tty: true + hostname: netbox-redis-cache + networks: + - default + environment: + << : *process-variables + VIRTUAL_HOST : 'netbox-redis-cache.malcolm.local' + env_file: ./netbox/env/redis-cache.env + command: + - sh + - -c + - redis-server --requirepass $$REDIS_PASSWORD + healthcheck: + test: ["CMD-SHELL", "pidof redis-server || exit 1" ] + interval: 60s + timeout: 15s + retries: 3 + start_period: 45s api: image: malcolmnetsec/api:6.4.0 command: gunicorn --bind 0:5000 manage:app diff --git a/docker-compose.yml b/docker-compose.yml index 6468a4653..28ca975fa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -942,6 +942,100 @@ services: timeout: 15s retries: 3 start_period: 60s + netbox: + image: ghcr.io/mmguero/netbox:latest + restart: "no" + stdin_open: false + tty: true + hostname: netbox + networks: + - default + environment: + << : *process-variables + VIRTUAL_HOST : 'netbox.malcolm.local' + env_file: ./netbox/env/netbox.env + depends_on: + - netbox-postgres + - netbox-redis + - netbox-redis-cache + volumes: + - ./netbox/config/configuration:/etc/netbox/config:ro + - ./netbox/config/reports:/etc/netbox/reports:ro + - ./netbox/config/scripts:/etc/netbox/scripts:ro + - ./netbox/config/unit:/etc/unit:ro + - ./netbox/media:/opt/netbox/netbox/media:rw + healthcheck: + test: ["CMD", "curl", "--silent", "http://localhost:8080/assets/api/" ] + interval: 60s + timeout: 15s + retries: 3 + start_period: 120s + netbox-postgres: + image: ghcr.io/mmguero/postgres:14-alpine + restart: "no" + stdin_open: false + tty: true + hostname: netbox-postgres + networks: + - default + environment: + << : *process-variables + VIRTUAL_HOST : 'netbox-postgres.malcolm.local' + env_file: ./netbox/env/postgres.env + volumes: + - ./netbox/postgres:/var/lib/postgresql/data:rw + healthcheck: + test: [ "CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}" ] + interval: 60s + timeout: 15s + retries: 3 + start_period: 45s + netbox-redis: + image: ghcr.io/mmguero/redis:7-alpine + restart: "no" + stdin_open: false + tty: true + hostname: netbox-redis + networks: + - default + environment: + << : *process-variables + VIRTUAL_HOST : 'netbox-redis.malcolm.local' + env_file: ./netbox/env/redis.env + command: + - sh + - -c + - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD + volumes: + - ./netbox/redis:/data + healthcheck: + test: ["CMD-SHELL", "pidof redis-server || exit 1" ] + interval: 60s + timeout: 15s + retries: 3 + start_period: 45s + netbox-redis-cache: + image: ghcr.io/mmguero/redis:7-alpine + restart: "no" + stdin_open: false + tty: true + hostname: netbox-redis-cache + networks: + - default + environment: + << : *process-variables + VIRTUAL_HOST : 'netbox-redis-cache.malcolm.local' + env_file: ./netbox/env/redis-cache.env + command: + - sh + - -c + - redis-server --requirepass $$REDIS_PASSWORD + healthcheck: + test: ["CMD-SHELL", "pidof redis-server || exit 1" ] + interval: 60s + timeout: 15s + retries: 3 + start_period: 45s api: image: malcolmnetsec/api:6.4.0 build: diff --git a/malcolm-iso/build.sh b/malcolm-iso/build.sh index 351602843..e13fb3bbc 100755 --- a/malcolm-iso/build.sh +++ b/malcolm-iso/build.sh @@ -96,6 +96,9 @@ if [ -d "$WORKDIR" ]; then mkdir -p "$MALCOLM_DEST_DIR/htadmin/" mkdir -p "$MALCOLM_DEST_DIR/logstash/certs/" mkdir -p "$MALCOLM_DEST_DIR/logstash/maps/" + mkdir -p "$MALCOLM_DEST_DIR/netbox/media/" + mkdir -p "$MALCOLM_DEST_DIR/netbox/postgres/" + mkdir -p "$MALCOLM_DEST_DIR/netbox/redis/" mkdir -p "$MALCOLM_DEST_DIR/nginx/ca-trust/" mkdir -p "$MALCOLM_DEST_DIR/nginx/certs/" mkdir -p "$MALCOLM_DEST_DIR/opensearch-backup/" @@ -108,8 +111,8 @@ if [ -d "$WORKDIR" ]; then mkdir -p "$MALCOLM_DEST_DIR/yara/rules/" mkdir -p "$MALCOLM_DEST_DIR/zeek-logs/current/" mkdir -p "$MALCOLM_DEST_DIR/zeek-logs/extract_files/" - mkdir -p "$MALCOLM_DEST_DIR/zeek-logs/processed/" mkdir -p "$MALCOLM_DEST_DIR/zeek-logs/live/" + mkdir -p "$MALCOLM_DEST_DIR/zeek-logs/processed/" mkdir -p "$MALCOLM_DEST_DIR/zeek-logs/upload/" mkdir -p "$MALCOLM_DEST_DIR/zeek/intel/MISP" mkdir -p "$MALCOLM_DEST_DIR/zeek/intel/STIX" diff --git a/netbox/.gitignore b/netbox/.gitignore new file mode 100644 index 000000000..4d047b820 --- /dev/null +++ b/netbox/.gitignore @@ -0,0 +1,3 @@ +media +postgres +redis \ No newline at end of file diff --git a/netbox/config/configuration/configuration.py b/netbox/config/configuration/configuration.py new file mode 100644 index 000000000..c8ddd14c2 --- /dev/null +++ b/netbox/config/configuration/configuration.py @@ -0,0 +1,256 @@ +#### +## We recommend to not edit this file. +## Create separate files to overwrite the settings. +## See `extra.py` as an example. +#### + +import re +from os import environ +from os.path import abspath, dirname, join + +# For reference see https://netbox.readthedocs.io/en/stable/configuration/ +# Based on https://github.com/netbox-community/netbox/blob/master/netbox/netbox/configuration.example.py + +# Read secret from file +def _read_secret(secret_name, default = None): + try: + f = open('/run/secrets/' + secret_name, 'r', encoding='utf-8') + except EnvironmentError: + return default + else: + with f: + return f.readline().strip() + +_BASE_DIR = dirname(dirname(abspath(__file__))) + +######################### +# # +# Required settings # +# # +######################### + +# This is a list of valid fully-qualified domain names (FQDNs) for the NetBox server. NetBox will not permit write +# access to the server via any other hostnames. The first FQDN in the list will be treated as the preferred name. +# +# Example: ALLOWED_HOSTS = ['netbox.example.com', 'netbox.internal.local'] +ALLOWED_HOSTS = environ.get('ALLOWED_HOSTS', '*').split(' ') + +# PostgreSQL database configuration. See the Django documentation for a complete list of available parameters: +# https://docs.djangoproject.com/en/stable/ref/settings/#databases +DATABASE = { + 'NAME': environ.get('DB_NAME', 'netbox'), # Database name + 'USER': environ.get('DB_USER', ''), # PostgreSQL username + 'PASSWORD': _read_secret('db_password', environ.get('DB_PASSWORD', '')), + # PostgreSQL password + 'HOST': environ.get('DB_HOST', 'localhost'), # Database server + 'PORT': environ.get('DB_PORT', ''), # Database port (leave blank for default) + 'OPTIONS': {'sslmode': environ.get('DB_SSLMODE', 'prefer')}, + # Database connection SSLMODE + 'CONN_MAX_AGE': int(environ.get('DB_CONN_MAX_AGE', '300')), + # Max database connection age + 'DISABLE_SERVER_SIDE_CURSORS': environ.get('DB_DISABLE_SERVER_SIDE_CURSORS', 'False').lower() == 'true', + # Disable the use of server-side cursors transaction pooling +} + +# Redis database settings. Redis is used for caching and for queuing background tasks such as webhook events. A separate +# configuration exists for each. Full connection details are required in both sections, and it is strongly recommended +# to use two separate database IDs. +REDIS = { + 'tasks': { + 'HOST': environ.get('REDIS_HOST', 'localhost'), + 'PORT': int(environ.get('REDIS_PORT', 6379)), + 'PASSWORD': _read_secret('redis_password', environ.get('REDIS_PASSWORD', '')), + 'DATABASE': int(environ.get('REDIS_DATABASE', 0)), + 'SSL': environ.get('REDIS_SSL', 'False').lower() == 'true', + 'INSECURE_SKIP_TLS_VERIFY': environ.get('REDIS_INSECURE_SKIP_TLS_VERIFY', 'False').lower() == 'true', + }, + 'caching': { + 'HOST': environ.get('REDIS_CACHE_HOST', environ.get('REDIS_HOST', 'localhost')), + 'PORT': int(environ.get('REDIS_CACHE_PORT', environ.get('REDIS_PORT', 6379))), + 'PASSWORD': _read_secret('redis_cache_password', environ.get('REDIS_CACHE_PASSWORD', environ.get('REDIS_PASSWORD', ''))), + 'DATABASE': int(environ.get('REDIS_CACHE_DATABASE', 1)), + 'SSL': environ.get('REDIS_CACHE_SSL', environ.get('REDIS_SSL', 'False')).lower() == 'true', + 'INSECURE_SKIP_TLS_VERIFY': environ.get('REDIS_CACHE_INSECURE_SKIP_TLS_VERIFY', environ.get('REDIS_INSECURE_SKIP_TLS_VERIFY', 'False')).lower() == 'true', + }, +} + +# This key is used for secure generation of random numbers and strings. It must never be exposed outside of this file. +# For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and +# symbols. NetBox will not run without this defined. For more information, see +# https://docs.djangoproject.com/en/stable/ref/settings/#std:setting-SECRET_KEY +SECRET_KEY = _read_secret('secret_key', environ.get('SECRET_KEY', '')) + + +######################### +# # +# Optional settings # +# # +######################### + +# Specify one or more name and email address tuples representing NetBox administrators. These people will be notified of +# application errors (assuming correct email settings are provided). +ADMINS = [ + # ['John Doe', 'jdoe@example.com'], +] + +# URL schemes that are allowed within links in NetBox +ALLOWED_URL_SCHEMES = ( + 'file', 'ftp', 'ftps', 'http', 'https', 'irc', 'mailto', 'sftp', 'ssh', 'tel', 'telnet', 'tftp', 'vnc', 'xmpp', +) + +# Optionally display a persistent banner at the top and/or bottom of every page. HTML is allowed. To display the same +# content in both banners, define BANNER_TOP and set BANNER_BOTTOM = BANNER_TOP. +BANNER_TOP = environ.get('BANNER_TOP', '') +BANNER_BOTTOM = environ.get('BANNER_BOTTOM', '') + +# Text to include on the login page above the login form. HTML is allowed. +BANNER_LOGIN = environ.get('BANNER_LOGIN', '') + +# Base URL path if accessing NetBox within a directory. For example, if installed at http://example.com/netbox/, set: +# BASE_PATH = 'netbox/' +BASE_PATH = environ.get('BASE_PATH', '') + +# Maximum number of days to retain logged changes. Set to 0 to retain changes indefinitely. (Default: 90) +CHANGELOG_RETENTION = int(environ.get('CHANGELOG_RETENTION', 90)) + +# API Cross-Origin Resource Sharing (CORS) settings. If CORS_ORIGIN_ALLOW_ALL is set to True, all origins will be +# allowed. Otherwise, define a list of allowed origins using either CORS_ORIGIN_WHITELIST or +# CORS_ORIGIN_REGEX_WHITELIST. For more information, see https://github.com/ottoyiu/django-cors-headers +CORS_ORIGIN_ALLOW_ALL = environ.get('CORS_ORIGIN_ALLOW_ALL', 'False').lower() == 'true' +CORS_ORIGIN_WHITELIST = list(filter(None, environ.get('CORS_ORIGIN_WHITELIST', 'https://localhost').split(' '))) +CORS_ORIGIN_REGEX_WHITELIST = [re.compile(r) for r in list(filter(None, environ.get('CORS_ORIGIN_REGEX_WHITELIST', '').split(' ')))] + +# Cross-Site-Request-Forgery-Attack settings. If Netbox is sitting behind a reverse proxy, you might need to set the CSRF_TRUSTED_ORIGINS flag. +# Django 4.0 requires to specify the URL Scheme in this setting. An example environment variable could be specified like: +# CSRF_TRUSTED_ORIGINS=https://demo.netbox.dev http://demo.netbox.dev +CSRF_TRUSTED_ORIGINS = list(filter(None, environ.get('CSRF_TRUSTED_ORIGINS', '').split(' '))) + +# Set to True to enable server debugging. WARNING: Debugging introduces a substantial performance penalty and may reveal +# sensitive information about your installation. Only enable debugging while performing testing. Never enable debugging +# on a production system. +DEBUG = environ.get('DEBUG', 'False').lower() == 'true' + +# Email settings +EMAIL = { + 'SERVER': environ.get('EMAIL_SERVER', 'localhost'), + 'PORT': int(environ.get('EMAIL_PORT', 25)), + 'USERNAME': environ.get('EMAIL_USERNAME', ''), + 'PASSWORD': _read_secret('email_password', environ.get('EMAIL_PASSWORD', '')), + 'USE_SSL': environ.get('EMAIL_USE_SSL', 'False').lower() == 'true', + 'USE_TLS': environ.get('EMAIL_USE_TLS', 'False').lower() == 'true', + 'SSL_CERTFILE': environ.get('EMAIL_SSL_CERTFILE', ''), + 'SSL_KEYFILE': environ.get('EMAIL_SSL_KEYFILE', ''), + 'TIMEOUT': int(environ.get('EMAIL_TIMEOUT', 10)), # seconds + 'FROM_EMAIL': environ.get('EMAIL_FROM', ''), +} + +# Enforcement of unique IP space can be toggled on a per-VRF basis. To enforce unique IP space within the global table +# (all prefixes and IP addresses not assigned to a VRF), set ENFORCE_GLOBAL_UNIQUE to True. +ENFORCE_GLOBAL_UNIQUE = environ.get('ENFORCE_GLOBAL_UNIQUE', 'False').lower() == 'true' + +# Exempt certain models from the enforcement of view permissions. Models listed here will be viewable by all users and +# by anonymous users. List models in the form `.`. Add '*' to this list to exempt all models. +EXEMPT_VIEW_PERMISSIONS = list(filter(None, environ.get('EXEMPT_VIEW_PERMISSIONS', '').split(' '))) + +# Enable GraphQL API. +GRAPHQL_ENABLED = environ.get('GRAPHQL_ENABLED', 'True').lower() == 'true' + +# Enable custom logging. Please see the Django documentation for detailed guidance on configuring custom logs: +# https://docs.djangoproject.com/en/stable/topics/logging/ +LOGGING = {} + +# Setting this to True will permit only authenticated users to access any part of NetBox. By default, anonymous users +# are permitted to access most data in NetBox (excluding secrets) but not make any changes. +LOGIN_REQUIRED = environ.get('LOGIN_REQUIRED', 'False').lower() == 'true' + +# The length of time (in seconds) for which a user will remain logged into the web UI before being prompted to +# re-authenticate. (Default: 1209600 [14 days]) +LOGIN_TIMEOUT = int(environ.get('LOGIN_TIMEOUT', 1209600)) + +# Setting this to True will display a "maintenance mode" banner at the top of every page. +MAINTENANCE_MODE = environ.get('MAINTENANCE_MODE', 'False').lower() == 'true' + +# Maps provider +MAPS_URL = environ.get('MAPS_URL', None) + +# An API consumer can request an arbitrary number of objects =by appending the "limit" parameter to the URL (e.g. +# "?limit=1000"). This setting defines the maximum limit. Setting it to 0 or None will allow an API consumer to request +# all objects by specifying "?limit=0". +MAX_PAGE_SIZE = int(environ.get('MAX_PAGE_SIZE', 1000)) + +# The file path where uploaded media such as image attachments are stored. A trailing slash is not needed. Note that +# the default value of this setting is derived from the installed location. +MEDIA_ROOT = environ.get('MEDIA_ROOT', join(_BASE_DIR, 'media')) + +# Expose Prometheus monitoring metrics at the HTTP endpoint '/metrics' +METRICS_ENABLED = environ.get('METRICS_ENABLED', 'False').lower() == 'true' + +# Credentials that NetBox will uses to authenticate to devices when connecting via NAPALM. +NAPALM_USERNAME = environ.get('NAPALM_USERNAME', '') +NAPALM_PASSWORD = _read_secret('napalm_password', environ.get('NAPALM_PASSWORD', '')) + +# NAPALM timeout (in seconds). (Default: 30) +NAPALM_TIMEOUT = int(environ.get('NAPALM_TIMEOUT', 30)) + +# NAPALM optional arguments (see http://napalm.readthedocs.io/en/latest/support/#optional-arguments). Arguments must +# be provided as a dictionary. +NAPALM_ARGS = {} + +# Determine how many objects to display per page within a list. (Default: 50) +PAGINATE_COUNT = int(environ.get('PAGINATE_COUNT', 50)) + +# Enable installed plugins. Add the name of each plugin to the list. +PLUGINS = [] + +# Plugins configuration settings. These settings are used by various plugins that the user may have installed. +# Each key in the dictionary is the name of an installed plugin and its value is a dictionary of settings. +PLUGINS_CONFIG = { +} + +# When determining the primary IP address for a device, IPv6 is preferred over IPv4 by default. Set this to True to +# prefer IPv4 instead. +PREFER_IPV4 = environ.get('PREFER_IPV4', 'False').lower() == 'true' + +# Rack elevation size defaults, in pixels. For best results, the ratio of width to height should be roughly 10:1. +RACK_ELEVATION_DEFAULT_UNIT_HEIGHT = int(environ.get('RACK_ELEVATION_DEFAULT_UNIT_HEIGHT', 22)) +RACK_ELEVATION_DEFAULT_UNIT_WIDTH = int(environ.get('RACK_ELEVATION_DEFAULT_UNIT_WIDTH', 220)) + +# Remote authentication support +REMOTE_AUTH_ENABLED = environ.get('REMOTE_AUTH_ENABLED', 'False').lower() == 'true' +REMOTE_AUTH_BACKEND = environ.get('REMOTE_AUTH_BACKEND', 'netbox.authentication.RemoteUserBackend') +REMOTE_AUTH_HEADER = environ.get('REMOTE_AUTH_HEADER', 'HTTP_REMOTE_USER') +REMOTE_AUTH_AUTO_CREATE_USER = environ.get('REMOTE_AUTH_AUTO_CREATE_USER', 'True').lower() == 'true' +REMOTE_AUTH_DEFAULT_GROUPS = list(filter(None, environ.get('REMOTE_AUTH_DEFAULT_GROUPS', '').split(' '))) + +# This repository is used to check whether there is a new release of NetBox available. Set to None to disable the +# version check or use the URL below to check for release in the official NetBox repository. +# https://api.github.com/repos/netbox-community/netbox/releases +RELEASE_CHECK_URL = environ.get('RELEASE_CHECK_URL', None) + +# The file path where custom reports will be stored. A trailing slash is not needed. Note that the default value of +# this setting is derived from the installed location. +REPORTS_ROOT = environ.get('REPORTS_ROOT', '/etc/netbox/reports') + +# Maximum execution time for background tasks, in seconds. +RQ_DEFAULT_TIMEOUT = int(environ.get('RQ_DEFAULT_TIMEOUT', 300)) + +# The file path where custom scripts will be stored. A trailing slash is not needed. Note that the default value of +# this setting is derived from the installed location. +SCRIPTS_ROOT = environ.get('SCRIPTS_ROOT', '/etc/netbox/scripts') + +# By default, NetBox will store session data in the database. Alternatively, a file path can be specified here to use +# local file storage instead. (This can be useful for enabling authentication on a standby instance with read-only +# database access.) Note that the user as which NetBox runs must have read and write permissions to this path. +SESSION_FILE_PATH = environ.get('SESSIONS_ROOT', None) + +# Time zone (default: UTC) +TIME_ZONE = environ.get('TIME_ZONE', 'UTC') + +# Date/time formatting. See the following link for supported formats: +# https://docs.djangoproject.com/en/stable/ref/templates/builtins/#date +DATE_FORMAT = environ.get('DATE_FORMAT', 'N j, Y') +SHORT_DATE_FORMAT = environ.get('SHORT_DATE_FORMAT', 'Y-m-d') +TIME_FORMAT = environ.get('TIME_FORMAT', 'g:i a') +SHORT_TIME_FORMAT = environ.get('SHORT_TIME_FORMAT', 'H:i:s') +DATETIME_FORMAT = environ.get('DATETIME_FORMAT', 'N j, Y g:i a') +SHORT_DATETIME_FORMAT = environ.get('SHORT_DATETIME_FORMAT', 'Y-m-d H:i') diff --git a/netbox/config/configuration/extra.py b/netbox/config/configuration/extra.py new file mode 100644 index 000000000..46f1877ed --- /dev/null +++ b/netbox/config/configuration/extra.py @@ -0,0 +1,55 @@ +#### +## This file contains extra configuration options that can't be configured +## directly through environment variables. +#### + +## Specify one or more name and email address tuples representing NetBox administrators. These people will be notified of +## application errors (assuming correct email settings are provided). +# ADMINS = [ +# # ['John Doe', 'jdoe@example.com'], +# ] + + +## URL schemes that are allowed within links in NetBox +# ALLOWED_URL_SCHEMES = ( +# 'file', 'ftp', 'ftps', 'http', 'https', 'irc', 'mailto', 'sftp', 'ssh', 'tel', 'telnet', 'tftp', 'vnc', 'xmpp', +# ) + + +## NAPALM optional arguments (see http://napalm.readthedocs.io/en/latest/support/#optional-arguments). Arguments must +## be provided as a dictionary. +# NAPALM_ARGS = {} + + +## Enable installed plugins. Add the name of each plugin to the list. +# from netbox.configuration.configuration import PLUGINS +# PLUGINS.append('my_plugin') + +## Plugins configuration settings. These settings are used by various plugins that the user may have installed. +## Each key in the dictionary is the name of an installed plugin and its value is a dictionary of settings. +# from netbox.configuration.configuration import PLUGINS_CONFIG +# PLUGINS_CONFIG['my_plugin'] = { +# 'foo': 'bar', +# 'buzz': 'bazz' +# } + + +## Remote authentication support +# REMOTE_AUTH_DEFAULT_PERMISSIONS = {} + + +## By default uploaded media is stored on the local filesystem. Using Django-storages is also supported. Provide the +## class path of the storage driver in STORAGE_BACKEND and any configuration options in STORAGE_CONFIG. For example: +# STORAGE_BACKEND = 'storages.backends.s3boto3.S3Boto3Storage' +# STORAGE_CONFIG = { +# 'AWS_ACCESS_KEY_ID': 'Key ID', +# 'AWS_SECRET_ACCESS_KEY': 'Secret', +# 'AWS_STORAGE_BUCKET_NAME': 'netbox', +# 'AWS_S3_REGION_NAME': 'eu-west-1', +# } + + +## This file can contain arbitrary Python code, e.g.: +# from datetime import datetime +# now = datetime.now().strftime("%d/%m/%Y %H:%M:%S") +# BANNER_TOP = f'This instance started on {now}.' diff --git a/netbox/config/configuration/logging.py b/netbox/config/configuration/logging.py new file mode 100644 index 000000000..d786768d6 --- /dev/null +++ b/netbox/config/configuration/logging.py @@ -0,0 +1,55 @@ +# # Remove first comment(#) on each line to implement this working logging example. +# # Add LOGLEVEL environment variable to netbox if you use this example & want a different log level. +# from os import environ + +# # Set LOGLEVEL in netbox.env or docker-compose.overide.yml to override a logging level of INFO. +# LOGLEVEL = environ.get('LOGLEVEL', 'INFO') + +# LOGGING = { + +# 'version': 1, +# 'disable_existing_loggers': False, +# 'formatters': { +# 'verbose': { +# 'format': '{levelname} {asctime} {module} {process:d} {thread:d} {message}', +# 'style': '{', +# }, +# 'simple': { +# 'format': '{levelname} {message}', +# 'style': '{', +# }, +# }, +# 'filters': { +# 'require_debug_false': { +# '()': 'django.utils.log.RequireDebugFalse', +# }, +# }, +# 'handlers': { +# 'console': { +# 'level': LOGLEVEL, +# 'filters': ['require_debug_false'], +# 'class': 'logging.StreamHandler', +# 'formatter': 'simple' +# }, +# 'mail_admins': { +# 'level': 'ERROR', +# 'class': 'django.utils.log.AdminEmailHandler', +# 'filters': ['require_debug_false'] +# } +# }, +# 'loggers': { +# 'django': { +# 'handlers': ['console'], +# 'propagate': True, +# }, +# 'django.request': { +# 'handlers': ['mail_admins'], +# 'level': 'ERROR', +# 'propagate': False, +# }, +# 'django_auth_ldap': { +# 'handlers': ['console',], +# 'level': LOGLEVEL, +# } +# } +# } diff --git a/netbox/config/configuration/plugins.py b/netbox/config/configuration/plugins.py new file mode 100644 index 000000000..c0b1a1fb5 --- /dev/null +++ b/netbox/config/configuration/plugins.py @@ -0,0 +1,13 @@ +# Add your plugins and plugin settings here. +# Of course uncomment this file out. + +# To learn how to build images with your required plugins +# See https://github.com/netbox-community/netbox-docker/wiki/Using-Netbox-Plugins + +# PLUGINS = ["netbox_bgp"] + +# PLUGINS_CONFIG = { +# "netbox_bgp": { +# ADD YOUR SETTINGS HERE +# } +# } diff --git a/netbox/config/reports/devices.py.example b/netbox/config/reports/devices.py.example new file mode 100644 index 000000000..670eeb616 --- /dev/null +++ b/netbox/config/reports/devices.py.example @@ -0,0 +1,46 @@ +from dcim.choices import DeviceStatusChoices +from dcim.models import ConsolePort, Device, PowerPort +from extras.reports import Report + + +class DeviceConnectionsReport(Report): + description = "Validate the minimum physical connections for each device" + + def test_console_connection(self): + + # Check that every console port for every active device has a connection defined. + active = DeviceStatusChoices.STATUS_ACTIVE + for console_port in ConsolePort.objects.prefetch_related('device').filter(device__status=active): + if console_port.connected_endpoint is None: + self.log_failure( + console_port.device, + "No console connection defined for {}".format(console_port.name) + ) + elif not console_port.connection_status: + self.log_warning( + console_port.device, + "Console connection for {} marked as planned".format(console_port.name) + ) + else: + self.log_success(console_port.device) + + def test_power_connections(self): + + # Check that every active device has at least two connected power supplies. + for device in Device.objects.filter(status=DeviceStatusChoices.STATUS_ACTIVE): + connected_ports = 0 + for power_port in PowerPort.objects.filter(device=device): + if power_port.connected_endpoint is not None: + connected_ports += 1 + if not power_port.connection_status: + self.log_warning( + device, + "Power connection for {} marked as planned".format(power_port.name) + ) + if connected_ports < 2: + self.log_failure( + device, + "{} connected power supplies found (2 needed)".format(connected_ports) + ) + else: + self.log_success(device) diff --git a/netbox/config/scripts/__init__.py b/netbox/config/scripts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/netbox/config/unit/nginx-unit.json b/netbox/config/unit/nginx-unit.json new file mode 100644 index 000000000..731bab304 --- /dev/null +++ b/netbox/config/unit/nginx-unit.json @@ -0,0 +1,47 @@ +{ + + "settings": { + "http": { + "discard_unsafe_fields": false + } + }, + + "listeners": { + "*:8080": { + "pass": "routes" + } + }, + + "routes": [ + { + "match": { + "uri": "/assets/static/*" + }, + "action": { + "share": "/opt/netbox/netbox${uri}" + } + }, + + { + "action": { + "pass": "applications/netbox" + } + } + ], + + "applications": { + "netbox": { + "type": "python 3", + "path": "/opt/netbox/netbox/", + "module": "netbox.wsgi", + "home": "/opt/netbox/venv", + "processes": { + "max": 4, + "spare": 1, + "idle_timeout": 120 + } + } + }, + + "access_log": "/dev/stdout" +} diff --git a/netbox/env/.gitignore b/netbox/env/.gitignore new file mode 100644 index 000000000..981ab9771 --- /dev/null +++ b/netbox/env/.gitignore @@ -0,0 +1,4 @@ +postgres.env +redis-cache.env +redis.env +netbox.env \ No newline at end of file diff --git a/netbox/env/netbox.env.example b/netbox/env/netbox.env.example new file mode 100644 index 000000000..335761e94 --- /dev/null +++ b/netbox/env/netbox.env.example @@ -0,0 +1,55 @@ +CORS_ORIGIN_ALLOW_ALL=True +CSRF_TRUSTED_ORIGINS=http://* https://* +BASE_PATH=assets +REMOTE_AUTH_ENABLED=True +REMOTE_AUTH_BACKEND=netbox.authentication.RemoteUserBackend +REMOTE_AUTH_HEADER=HTTP_X_REMOTE_AUTH +REMOTE_AUTH_AUTO_CREATE_USER=True +REMOTE_AUTH_DEFAULT_GROUPS=standard +REMOTE_AUTH_DEFAULT_PERMISSIONS=standard_permission +REMOTE_AUTH_STAFF_GROUPS=administrator +REMOTE_AUTH_STAFF_USERS= +REMOTE_AUTH_SUPERUSER_GROUPS=administrator +REMOTE_AUTH_SUPERUSERS= +EXEMPT_VIEW_PERMISSIONS=* +DB_HOST=netbox-postgres +DB_NAME=netbox +DB_PASSWORD=xxxxxxxxxxxxxxxx +DB_USER=netbox +EMAIL_FROM=netbox@bar.com +EMAIL_PASSWORD= +EMAIL_PORT=25 +EMAIL_SERVER=localhost +EMAIL_SSL_CERTFILE= +EMAIL_SSL_KEYFILE= +EMAIL_TIMEOUT=5 +EMAIL_USERNAME=netbox +# EMAIL_USE_SSL and EMAIL_USE_TLS are mutually exclusive, i.e. they can't both be `true`! +EMAIL_USE_SSL=false +EMAIL_USE_TLS=false +GRAPHQL_ENABLED=true +HOUSEKEEPING_INTERVAL=86400 +MAX_PAGE_SIZE=1000 +MEDIA_ROOT=/opt/netbox/netbox/media +METRICS_ENABLED=false +NAPALM_PASSWORD= +NAPALM_TIMEOUT=10 +NAPALM_USERNAME= +REDIS_CACHE_DATABASE=1 +REDIS_CACHE_HOST=netbox-redis-cache +REDIS_CACHE_INSECURE_SKIP_TLS_VERIFY=false +REDIS_CACHE_PASSWORD=xxxxxxxxxxxxxxxx +REDIS_CACHE_SSL=false +REDIS_DATABASE=0 +REDIS_HOST=netbox-redis +REDIS_INSECURE_SKIP_TLS_VERIFY=false +REDIS_PASSWORD=xxxxxxxxxxxxxxxx +REDIS_SSL=false +RELEASE_CHECK_URL=https://api.github.com/repos/netbox-community/netbox/releases +SECRET_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +SKIP_SUPERUSER=false +SUPERUSER_API_TOKEN=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +SUPERUSER_EMAIL=admin@example.com +SUPERUSER_NAME=admin +SUPERUSER_PASSWORD=admin +WEBHOOKS_ENABLED=true diff --git a/netbox/scripts/netbox_init.py b/netbox/scripts/netbox_init.py new file mode 100755 index 000000000..ad6093313 --- /dev/null +++ b/netbox/scripts/netbox_init.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import argparse +import json +import logging +import os +import pynetbox +import sys + +from slugify import slugify + +################################################################################################### +args = None +script_name = os.path.basename(__file__) +script_path = os.path.dirname(os.path.realpath(__file__)) +orig_path = os.getcwd() + +################################################################################################### +DEFAULT_GROUP_NAMES = ( + 'administrator', + 'standard', +) + +DEFAULT_PERMISSIONS = { + 'administrator_permission': { + 'name': 'administrator_permission', + 'enabled': True, + 'groups': ['administrator'], + 'actions': [ + 'view', + 'add', + 'change', + 'delete', + ], + 'exclude_objects': [], + }, + 'standard_permission': { + 'name': 'standard_permission', + 'enabled': True, + 'groups': ['standard'], + 'actions': [ + 'view', + 'add', + 'change', + 'delete', + ], + 'exclude_objects': [ + 'admin.logentry', + 'auth.group', + 'auth.permission', + 'auth.user', + 'users.admingroup', + 'users.adminuser', + 'users.objectpermission', + 'users.token', + 'users.userconfig', + ], + }, +} + +################################################################################################### +# main +def main(): + global args + + parser = argparse.ArgumentParser( + description='\n'.join([]), + formatter_class=argparse.RawTextHelpFormatter, + add_help=False, + usage='{} '.format(script_name), + ) + parser.add_argument('--verbose', '-v', action='count', default=1, help='Increase verbosity (e.g., -v, -vv, etc.)') + parser.add_argument( + '-u', + '--url', + dest='netboxUrl', + type=str, + default=None, + required=True, + help="Netbox Base URL", + ) + parser.add_argument( + '-t', + '--token', + dest='netboxToken', + type=str, + default=None, + required=True, + help="Netbox API Token", + ) + parser.add_argument( + '-s', + '--site', + dest='netboxSites', + nargs='*', + type=str, + default=[], + required=False, + help="Site(s) to create", + ) + try: + parser.error = parser.exit + args = parser.parse_args() + except SystemExit: + parser.print_help() + exit(2) + + args.verbose = logging.ERROR - (10 * args.verbose) if args.verbose > 0 else 0 + logging.basicConfig( + level=args.verbose, format='%(asctime)s %(levelname)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S' + ) + logging.debug(os.path.join(script_path, script_name)) + logging.debug("Arguments: {}".format(sys.argv[1:])) + logging.debug("Arguments: {}".format(args)) + if args.verbose > logging.DEBUG: + sys.tracebacklimit = 0 + + # create connection to netbox API + nb = pynetbox.api( + args.netboxUrl, + token=args.netboxToken, + ) + + ###### GROUPS ################################################################################################ + # list existing groups + groupsPreExisting = [x.name for x in nb.users.groups.all()] + logging.debug(groupsPreExisting) + + # create groups that don't already exist + for groupName in DEFAULT_GROUP_NAMES: + if groupName not in groupsPreExisting: + nb.users.groups.create({'name': groupName}) + + # get existing groups into name->id dictionary + groupNameIdDict = {x.name: x.id for x in nb.users.groups.all()} + logging.debug(groupNameIdDict) + + # ###### PERMISSIONS ########################################################################################### + # get all content types (for creating new permissions) + allContentTypeNames = [f'{x.app_label}.{x.model}' for x in nb.extras.content_types.all()] + + # get existing permissions + permsPreExisting = [x.name for x in nb.users.permissions.all()] + logging.debug(permsPreExisting) + + # create permissions that don't already exist + for permName, permConfig in DEFAULT_PERMISSIONS.items(): + if 'name' in permConfig and permConfig['name'] not in permsPreExisting: + permConfig['groups'] = [groupNameIdDict[x] for x in permConfig['groups']] + permConfig['object_types'] = [ct for ct in allContentTypeNames if ct not in permConfig['exclude_objects']] + permConfig.pop('exclude_objects', None) + nb.users.permissions.create(permConfig) + + logging.debug([x.name for x in nb.users.permissions.all()]) + + # ###### PERMISSIONS ########################################################################################### + # get existing sites + sitesPreExisting = [x.name for x in nb.dcim.sites.all()] + logging.debug(sitesPreExisting) + + # create sites that don't already exist + for siteName in args.netboxSites: + if siteName not in sitesPreExisting: + nb.dcim.sites.create( + { + "name": siteName, + "slug": slugify(siteName), + }, + ) + + logging.debug([f'{x.name} ({x.slug})' for x in nb.dcim.sites.all()]) + + +################################################################################################### +if __name__ == '__main__': + main() diff --git a/netbox/scripts/requirements.txt b/netbox/scripts/requirements.txt new file mode 100644 index 000000000..231cf5217 --- /dev/null +++ b/netbox/scripts/requirements.txt @@ -0,0 +1,2 @@ +pynetbox +python-slugify \ No newline at end of file diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 1b5617959..cdf090290 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -69,6 +69,10 @@ http { server name-map-ui:8080; } + upstream netbox { + server netbox:8080; + } + upstream extracted-file-http-server { server file-monitor:8440; } @@ -204,6 +208,17 @@ http { proxy_set_header Host file-monitor.malcolm.local; } + # netbox + location /assets { + proxy_pass http://netbox; + proxy_redirect off; + proxy_set_header Host netbox.malcolm.local; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Remote-Auth $authenticated_user; + } + # favicon, logos, banners, etc. include /etc/nginx/nginx_image_aliases.conf; diff --git a/scripts/control.py b/scripts/control.py index b2ad796bd..ddc151971 100755 --- a/scripts/control.py +++ b/scripts/control.py @@ -5,14 +5,17 @@ import argparse import errno +import fileinput import getpass import glob import json import os import platform import re +import secrets import shutil import stat +import string import sys from malcolm_common import * @@ -530,6 +533,10 @@ def start(): os.path.join(MalcolmPath, os.path.join('nginx', 'nginx_ldap.conf')), os.path.join(MalcolmPath, '.opensearch.primary.curlrc'), os.path.join(MalcolmPath, '.opensearch.secondary.curlrc'), + os.path.join(MalcolmPath, os.path.join('netbox', os.path.join('env', 'netbox.env'))), + os.path.join(MalcolmPath, os.path.join('netbox', os.path.join('env', 'postgres.env'))), + os.path.join(MalcolmPath, os.path.join('netbox', os.path.join('env', 'redis-cache.env'))), + os.path.join(MalcolmPath, os.path.join('netbox', os.path.join('env', 'redis.env'))), ]: # chmod 600 authFile os.chmod(authFile, stat.S_IRUSR | stat.S_IWUSR) @@ -539,6 +546,9 @@ def start(): os.path.join(MalcolmPath, 'opensearch'), os.path.join(MalcolmPath, 'opensearch-backup'), os.path.join(MalcolmPath, os.path.join('nginx', 'ca-trust')), + os.path.join(MalcolmPath, os.path.join('netbox', 'media')), + os.path.join(MalcolmPath, os.path.join('netbox', 'postgres')), + os.path.join(MalcolmPath, os.path.join('netbox', 'redis')), os.path.join(MalcolmPath, os.path.join('pcap', 'processed')), os.path.join(MalcolmPath, os.path.join('pcap', 'upload')), os.path.join(MalcolmPath, os.path.join('suricata-logs', 'live')), @@ -1050,6 +1060,79 @@ def authSetup(wipe=False): eprint("Failed to store email alert sender account variables:\n") eprint("\n".join(results)) + if YesOrNo('(Re)generate internal passwords for NetBox', default=True): + with pushd(os.path.join(MalcolmPath, os.path.join('netbox', 'env'))): + netboxPwAlphabet = string.ascii_letters + string.digits + '_' + netboxKeyAlphabet = string.ascii_letters + string.digits + '%@<=>?~^_-' + netboxPostGresPassword = ''.join(secrets.choice(netboxPwAlphabet) for i in range(24)) + netboxRedisPassword = ''.join(secrets.choice(netboxPwAlphabet) for i in range(24)) + netboxRedisCachePassword = ''.join(secrets.choice(netboxPwAlphabet) for i in range(24)) + netboxSuPassword = ''.join(secrets.choice(netboxPwAlphabet) for i in range(24)) + netboxSuToken = ''.join(secrets.choice(netboxPwAlphabet) for i in range(40)) + netboxSecretKey = ''.join(secrets.choice(netboxKeyAlphabet) for i in range(50)) + + with open('postgres.env', 'w') as f: + f.write('POSTGRES_DB=netbox\n') + f.write(f'POSTGRES_PASSWORD={netboxPostGresPassword}\n') + f.write('POSTGRES_USER=netbox\n') + os.chmod('postgres.env', stat.S_IRUSR | stat.S_IWUSR) + + with open('redis-cache.env', 'w') as f: + f.write(f'REDIS_PASSWORD={netboxRedisCachePassword}\n') + os.chmod('redis-cache.env', stat.S_IRUSR | stat.S_IWUSR) + + with open('redis.env', 'w') as f: + f.write(f'REDIS_PASSWORD={netboxRedisPassword}\n') + os.chmod('redis.env', stat.S_IRUSR | stat.S_IWUSR) + + if (not os.path.isfile('netbox.env')) and (os.path.isfile('netbox.env.example')): + shutil.copy2('netbox.env.example', 'netbox.env') + + with fileinput.FileInput('netbox.env', inplace=True, backup=None) as envFile: + for line in envFile: + line = line.rstrip("\n") + + if line.startswith('DB_PASSWORD'): + line = re.sub( + r'(DB_PASSWORD\s*=\s*)(\S+)', + fr"\g<1>{netboxPostGresPassword}", + line, + ) + elif line.startswith('REDIS_CACHE_PASSWORD'): + line = re.sub( + r'(REDIS_CACHE_PASSWORD\s*=\s*)(\S+)', + fr"\g<1>{netboxRedisCachePassword}", + line, + ) + elif line.startswith('REDIS_PASSWORD'): + line = re.sub( + r'(REDIS_PASSWORD\s*=\s*)(\S+)', + fr"\g<1>{netboxRedisPassword}", + line, + ) + elif line.startswith('SECRET_KEY'): + line = re.sub( + r'(SECRET_KEY\s*=\s*)(\S+)', + fr"\g<1>{netboxSecretKey}", + line, + ) + elif line.startswith('SUPERUSER_PASSWORD'): + line = re.sub( + r'(SUPERUSER_PASSWORD\s*=\s*)(\S+)', + fr"\g<1>{netboxSuPassword}", + line, + ) + elif line.startswith('SUPERUSER_API_TOKEN'): + line = re.sub( + r'(SUPERUSER_API_TOKEN\s*=\s*)(\S+)', + fr"\g<1>{netboxSuToken}", + line, + ) + + print(line) + + os.chmod('netbox.env', stat.S_IRUSR | stat.S_IWUSR) + ################################################################################################### # main diff --git a/scripts/malcolm_appliance_packager.sh b/scripts/malcolm_appliance_packager.sh index adb968b9f..da2576746 100755 --- a/scripts/malcolm_appliance_packager.sh +++ b/scripts/malcolm_appliance_packager.sh @@ -67,6 +67,9 @@ if mkdir "$DESTDIR"; then mkdir $VERBOSE -p "$DESTDIR/htadmin/" mkdir $VERBOSE -p "$DESTDIR/logstash/certs/" mkdir $VERBOSE -p "$DESTDIR/logstash/maps/" + mkdir $VERBOSE -p "$DESTDIR/netbox/media/" + mkdir $VERBOSE -p "$DESTDIR/netbox/postgres/" + mkdir $VERBOSE -p "$DESTDIR/netbox/redis/" mkdir $VERBOSE -p "$DESTDIR/nginx/ca-trust/" mkdir $VERBOSE -p "$DESTDIR/nginx/certs/" mkdir $VERBOSE -p "$DESTDIR/opensearch-backup/" diff --git a/scripts/malcolm_common.py b/scripts/malcolm_common.py index 14ce0d779..1e9a81f5a 100644 --- a/scripts/malcolm_common.py +++ b/scripts/malcolm_common.py @@ -687,6 +687,10 @@ def MalcolmAuthFilesExist(): and os.path.isfile(os.path.join(MalcolmPath, os.path.join('nginx', os.path.join('certs', 'cert.pem')))) and os.path.isfile(os.path.join(MalcolmPath, os.path.join('nginx', os.path.join('certs', 'key.pem')))) and os.path.isfile(os.path.join(MalcolmPath, os.path.join('htadmin', 'config.ini'))) + and os.path.isfile(os.path.join(MalcolmPath, os.path.join('netbox', os.path.join('env', 'netbox.env')))) + and os.path.isfile(os.path.join(MalcolmPath, os.path.join('netbox', os.path.join('env', 'postgres.env')))) + and os.path.isfile(os.path.join(MalcolmPath, os.path.join('netbox', os.path.join('env', 'redis-cache.env')))) + and os.path.isfile(os.path.join(MalcolmPath, os.path.join('netbox', os.path.join('env', 'redis.env')))) and os.path.isfile(os.path.join(MalcolmPath, 'auth.env')) and os.path.isfile(os.path.join(MalcolmPath, '.opensearch.primary.curlrc')) )