Merge pull request #21 from stanfrbd/advanced-config

Advanced config close #20

.dockerignore

@@ -0,0 +1,13 @@
+# Ignore temporary files
+*.tmp
+*.swp
+
+# Ignore OS generated files
+.DS_Store
+Thumbs.db
+
+# Ignore Python cache directories
+__pycache__
+
+# Ignore Git directory
+.git

Dockerfile

@@ -18,6 +18,9 @@ RUN pip install --no-cache-dir -r requirements.txt --trusted-host=pypi.python.or
 # Copy the rest of the application code from the host to the container
 COPY . .
 
+# Run file prod/advanced_config.py (does nothing if the required variables are not set in secrets.json)
+RUN python prod/advanced_config.py
+
 # Expose port 5000 to allow external access to the application
 EXPOSE 5000
 

README.md

@@ -153,9 +153,10 @@ Some misconfigurations may lead to **security issues**.
 
 * The API is available at `/api/` and can be accessed via the GUI or command-line.
 
-**There are currently two endpoints:**
+**There are currently 3 endpoints:**
 
 * `/api/analyze` - Analyze a text and return analysis ID (JSON).
+* `/api//is_analysis_complete/<analysis_id>` - Check if the analysis is complete (JSON).
 * `/api/results/<analysis_id>` - Retrieve the results of a previous analysis (JSON).
 
 ```bash
@@ -169,6 +170,16 @@ curl -X POST "http://localhost:5000/api/analyze" -H "Content-Type: application/j
 }
 ```
 
+```bash
+curl "http://localhost:5000/api/is_analysis_complete/e88de647-b153-4904-91e5-8f5c79174854"
+```
+
+```json
+{
+  "complete": true
+}
+```
+
 ```bash
 curl "http://localhost:5000/api/results/e88de647-b153-4904-91e5-8f5c79174854"
 ```
@@ -208,7 +219,8 @@ curl "http://localhost:5000/api/results/e88de647-b153-4904-91e5-8f5c79174854"
 ]
 ```
 
-
+> [!NOTE]
+> The [dedicated wiki page](https://github.com/stanfrbd/cyberbro/wiki/API-usage-and-engine-names) gives all the names of usable engines.
 
 # API and third-party services
 
@@ -233,7 +245,8 @@ curl "http://localhost:5000/api/results/e88de647-b153-4904-91e5-8f5c79174854"
 * [Grep.App](https://grep.app/)
 
 > [!NOTE]
-> Any questions? Check the [wiki](https://github.com/stanfrbd/cyberbro/wiki) or raise an [issue](https://github.com/stanfrbd/cyberbro/issues/new)
+> Any questions? Check the [wiki](https://github.com/stanfrbd/cyberbro/wiki) or raise an [issue](https://github.com/stanfrbd/cyberbro/issues/new) \
+> For the advanced config (tuning of `supervisord.conf` before deployment, selection of visible engines, change `/api/` prefix...), check the [dedicated wiki page](https://github.com/stanfrbd/cyberbro/wiki/Advanced-options-in-secrets.json).
 
 # Special thanks
 

app.py

@@ -22,18 +22,6 @@
 if not os.path.exists(DATA_DIR):
     os.makedirs(DATA_DIR)
 
-# Disable modification tracking to save memory
-app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
-
-# Set the size of the database connection pool
-app.config['SQLALCHEMY_POOL_SIZE'] = 10
-
-# Set the maximum overflow size of the connection pool
-app.config['SQLALCHEMY_MAX_OVERFLOW'] = 20
-
-# Enable the config page - not intended for public use since authentication is not implemented
-app.config['CONFIG_PAGE_ENABLED'] = False
-
 # Load secrets from a file
 SECRETS_FILE = os.path.join(BASE_DIR, 'secrets.json')
 if os.path.exists(SECRETS_FILE):
@@ -42,9 +30,27 @@
 else:
     secrets = {}
 
+# Define API_PREFIX if the variable api_prefix is set in secrets.json else it must be "api"
+API_PREFIX = secrets.get("api_prefix", "api")
+
+# Enable the config page - not intended for public use since authentication is not implemented - checks if the variable config_page_enabled is set in secrets.json
+app.config['CONFIG_PAGE_ENABLED'] = secrets.get('config_page_enabled', False)
+
+# Define gui_enabled_engines list if the variable of type list gui_enabled_engines is set in secrets.json
+GUI_ENABLED_ENGINES = secrets.get('gui_enabled_engines', [])
+
 # Update the database URI to use the data directory
 app.config['SQLALCHEMY_DATABASE_URI'] = f"sqlite:///{os.path.join(DATA_DIR, 'results.db')}"
 
+# Disable modification tracking to save memory
+app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
+
+# Set the size of the database connection pool
+app.config['SQLALCHEMY_POOL_SIZE'] = 10
+
+# Set the maximum overflow size of the connection pool
+app.config['SQLALCHEMY_MAX_OVERFLOW'] = 20
+
 # Initialize the database
 db.init_app(app)
 
@@ -55,7 +61,7 @@
 @app.route('/')
 def index():
     """Render the index page."""
-    return render_template('index.html', results=[])
+    return render_template('index.html', results=[], API_PREFIX=API_PREFIX, GUI_ENABLED_ENGINES=GUI_ENABLED_ENGINES)
 
 @app.route('/analyze', methods=['POST'])
 def analyze():
@@ -67,18 +73,18 @@ def analyze():
     analysis_id = str(uuid.uuid4())
     threading.Thread(target=perform_analysis, args=(app, observables, selected_engines, analysis_id)).start()
 
-    return render_template('waiting.html', analysis_id=analysis_id), 200
+    return render_template('waiting.html', analysis_id=analysis_id, API_PREFIX=API_PREFIX), 200
 
 @app.route('/results/<analysis_id>', methods=['GET'])
 def show_results(analysis_id):
     """Show the results of the analysis."""
     analysis_results = db.session.get(AnalysisResult, analysis_id)
     if analysis_results:
-        return render_template('index.html', analysis_results=analysis_results)
+        return render_template('index.html', analysis_results=analysis_results, API_PREFIX=API_PREFIX)
     else:
         return render_template('404.html'), 404
 
-@app.route('/is_analysis_complete/<analysis_id>', methods=['GET'])
+@app.route(f'/{API_PREFIX}/is_analysis_complete/<analysis_id>', methods=['GET'])
 def is_analysis_complete(analysis_id):
     """Check if the analysis is complete."""
     complete = not check_analysis_in_progress(analysis_id)
@@ -157,7 +163,7 @@ def update_config():
         message = "An error occurred while updating the configuration."
     return jsonify({'message': message})
 
-@app.route('/api/results/<analysis_id>', methods=['GET'])
+@app.route(f'/{API_PREFIX}/results/<analysis_id>', methods=['GET'])
 def get_results(analysis_id):
     """Get the results of the analysis."""
     analysis_results = db.session.get(AnalysisResult, analysis_id)
@@ -166,9 +172,9 @@ def get_results(analysis_id):
     else:
         return jsonify({'error': 'Analysis not found.'}), 404
 
-@app.route('/api/analyze', methods=['POST'])
+@app.route(f'/{API_PREFIX}/analyze', methods=['POST'])
 def analyze_api():
-    """Handle the analyze request."""
+    """Handle the analyze request via API. (Only JSON data is accepted)"""
     data = request.get_json()
     form_data = ioc_fanger.fang(data.get("text", ""))
     observables = extract_observables(form_data)

prod/advanced_config.py

@@ -0,0 +1,39 @@
+import os
+import json
+import configparser
+
+# Path to the secrets file
+secrets_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'secrets.json')
+
+# Path to the supervisord.conf file
+supervisord_conf_file = os.path.join(os.path.dirname(__file__), 'supervisord.conf')
+
+# Read the secrets file
+with open(secrets_file, 'r') as f:
+    secrets = json.load(f)
+
+# Read the existing supervisord.conf
+config = configparser.ConfigParser()
+config.read(supervisord_conf_file)
+
+supervisor_conf_edited = False
+
+# Update the supervisord.conf with the new parameters if they exist
+if 'supervisord_workers_count' in secrets:
+    config['program:cyberbro']['command'] = config['program:cyberbro']['command'].replace(
+        '-w ' + config['program:cyberbro']['command'].split('-w ')[1].split()[0],
+        f"-w {secrets['supervisord_workers_count']}"
+    )
+    supervisor_conf_edited = True
+
+if 'supervisord_threads_count' in secrets:
+    config['program:cyberbro']['command'] = config['program:cyberbro']['command'].replace(
+        '-t ' + config['program:cyberbro']['command'].split('-t ')[1].split()[0],
+        f"-t {secrets['supervisord_threads_count']}"
+    )
+    supervisor_conf_edited = True
+
+if supervisor_conf_edited:
+    # Write the updated supervisord.conf
+    with open(supervisord_conf_file, 'w') as configfile:
+        config.write(configfile)