From f8a1003df54fbe115d6df02e7fab2392b94b45dc Mon Sep 17 00:00:00 2001 From: stanfrbd Date: Sun, 19 Jan 2025 15:57:50 +0100 Subject: [PATCH 1/4] Add advanced non mandatory options in secrets.json to change supervisord conf - will add more options #13 --- .dockerignore | 13 +++++++++++++ Dockerfile | 3 +++ Dockerfile-test | 25 ------------------------- docker-compose-test.yml | 13 ------------- prod/advanced_config.py | 39 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 55 insertions(+), 38 deletions(-) create mode 100644 .dockerignore delete mode 100644 Dockerfile-test delete mode 100644 docker-compose-test.yml create mode 100644 prod/advanced_config.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..a6a9b96 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +# Ignore temporary files +*.tmp +*.swp + +# Ignore OS generated files +.DS_Store +Thumbs.db + +# Ignore Python cache directories +__pycache__ + +# Ignore Git directory +.git \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 32c8332..d0edf94 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,9 @@ RUN pip install --no-cache-dir -r requirements.txt --trusted-host=pypi.python.or # Copy the rest of the application code from the host to the container COPY . . +# Run file prod/advanced_config.py (does nothing if the required variables are not set in secrets.json) +RUN python prod/advanced_config.py + # Expose port 5000 to allow external access to the application EXPOSE 5000 diff --git a/Dockerfile-test b/Dockerfile-test deleted file mode 100644 index 0e6558c..0000000 --- a/Dockerfile-test +++ /dev/null @@ -1,25 +0,0 @@ -# Use a base Python image -FROM python:3.13-slim - -# Update the package list and install git - nano, then clean up the apt cache -RUN apt-get update && \ - apt-get install -y git nano && \ - rm -rf /var/lib/apt/lists/* - -# Set the working directory in the container -WORKDIR /app - -# Copy the requirements.txt file into the container -COPY requirements.txt . - -# Install Python dependencies -RUN pip install --no-cache-dir -r requirements.txt --trusted-host=pypi.python.org --trusted-host=pypi.org --trusted-host=files.pythonhosted.org - -# Copy the rest of your project files into the container -COPY . . - -# Expose port 5000 for Flask -EXPOSE 5000 - -# Start the Flask application -CMD python app.py \ No newline at end of file diff --git a/docker-compose-test.yml b/docker-compose-test.yml deleted file mode 100644 index 149c86c..0000000 --- a/docker-compose-test.yml +++ /dev/null @@ -1,13 +0,0 @@ -services: - web: - build: - context: . - dockerfile: Dockerfile-test - container_name: cyberbro - ports: - - "5000:5000" - environment: - - FLASK_ENV=development - restart: always - volumes: - - ./data:/app/data \ No newline at end of file diff --git a/prod/advanced_config.py b/prod/advanced_config.py new file mode 100644 index 0000000..dce972a --- /dev/null +++ b/prod/advanced_config.py @@ -0,0 +1,39 @@ +import os +import json +import configparser + +# Path to the secrets file +secrets_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'secrets.json') + +# Path to the supervisord.conf file +supervisord_conf_file = os.path.join(os.path.dirname(__file__), 'supervisord.conf') + +# Read the secrets file +with open(secrets_file, 'r') as f: + secrets = json.load(f) + +# Read the existing supervisord.conf +config = configparser.ConfigParser() +config.read(supervisord_conf_file) + +supervisor_conf_edited = False + +# Update the supervisord.conf with the new parameters if they exist +if 'supervisord_workers_count' in secrets: + config['program:cyberbro']['command'] = config['program:cyberbro']['command'].replace( + '-w ' + config['program:cyberbro']['command'].split('-w ')[1].split()[0], + f"-w {secrets['supervisord_workers_count']}" + ) + supervisor_conf_edited = True + +if 'supervisord_threads_count' in secrets: + config['program:cyberbro']['command'] = config['program:cyberbro']['command'].replace( + '-t ' + config['program:cyberbro']['command'].split('-t ')[1].split()[0], + f"-t {secrets['supervisord_threads_count']}" + ) + supervisor_conf_edited = True + +if supervisor_conf_edited: + # Write the updated supervisord.conf + with open(supervisord_conf_file, 'w') as configfile: + config.write(configfile) From 8aee1007a53282610cc8c2e6ea33b88bdef81603 Mon Sep 17 00:00:00 2001 From: stanfrbd Date: Mon, 20 Jan 2025 11:15:11 +0100 Subject: [PATCH 2/4] Ability for everything requested in #13 (docker only) and #20 for engine selection --- README.md | 13 ++++- app.py | 44 ++++++++------- templates/index-demo.html | 97 +++++++++++++++++---------------- templates/index.html | 111 +++++++++++++++++--------------------- templates/waiting.html | 2 +- 5 files changed, 134 insertions(+), 133 deletions(-) diff --git a/README.md b/README.md index 8bb01c9..ab34372 100644 --- a/README.md +++ b/README.md @@ -153,9 +153,10 @@ Some misconfigurations may lead to **security issues**. * The API is available at `/api/` and can be accessed via the GUI or command-line. -**There are currently two endpoints:** +**There are currently 3 endpoints:** * `/api/analyze` - Analyze a text and return analysis ID (JSON). +* `/api//is_analysis_complete/` - Check if the analysis is complete (JSON). * `/api/results/` - Retrieve the results of a previous analysis (JSON). ```bash @@ -169,6 +170,15 @@ curl -X POST "http://localhost:5000/api/analyze" -H "Content-Type: application/j } ``` +```bash +curl "http://localhost:5000/api/is_analysis_complete/e88de647-b153-4904-91e5-8f5c79174854" +``` + +```json +{ + "complete": true +} + ```bash curl "http://localhost:5000/api/results/e88de647-b153-4904-91e5-8f5c79174854" ``` @@ -234,6 +244,7 @@ curl "http://localhost:5000/api/results/e88de647-b153-4904-91e5-8f5c79174854" > [!NOTE] > Any questions? Check the [wiki](https://github.com/stanfrbd/cyberbro/wiki) or raise an [issue](https://github.com/stanfrbd/cyberbro/issues/new) +> For the advanced config (tuning of `supervisord.conf` before deployment, selection of visible engines, change /api/ prefix...), check the [wiki](https://github.com/stanfrbd/cyberbro/wiki). # Special thanks diff --git a/app.py b/app.py index af42556..b75aa04 100644 --- a/app.py +++ b/app.py @@ -22,18 +22,6 @@ if not os.path.exists(DATA_DIR): os.makedirs(DATA_DIR) -# Disable modification tracking to save memory -app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False - -# Set the size of the database connection pool -app.config['SQLALCHEMY_POOL_SIZE'] = 10 - -# Set the maximum overflow size of the connection pool -app.config['SQLALCHEMY_MAX_OVERFLOW'] = 20 - -# Enable the config page - not intended for public use since authentication is not implemented -app.config['CONFIG_PAGE_ENABLED'] = False - # Load secrets from a file SECRETS_FILE = os.path.join(BASE_DIR, 'secrets.json') if os.path.exists(SECRETS_FILE): @@ -42,9 +30,27 @@ else: secrets = {} +# Define API_PREFIX if the variable api_prefix is set in secrets.json else it must be "api" +API_PREFIX = secrets.get("api_prefix", "api") + +# Enable the config page - not intended for public use since authentication is not implemented - checks if the variable config_page_enabled is set in secrets.json +app.config['CONFIG_PAGE_ENABLED'] = secrets.get('config_page_enabled', False) + +# Define gui_enabled_engines list if the variable of type list gui_enabled_engines is set in secrets.json +GUI_ENABLED_ENGINES = secrets.get('gui_enabled_engines', []) + # Update the database URI to use the data directory app.config['SQLALCHEMY_DATABASE_URI'] = f"sqlite:///{os.path.join(DATA_DIR, 'results.db')}" +# Disable modification tracking to save memory +app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False + +# Set the size of the database connection pool +app.config['SQLALCHEMY_POOL_SIZE'] = 10 + +# Set the maximum overflow size of the connection pool +app.config['SQLALCHEMY_MAX_OVERFLOW'] = 20 + # Initialize the database db.init_app(app) @@ -55,7 +61,7 @@ @app.route('/') def index(): """Render the index page.""" - return render_template('index.html', results=[]) + return render_template('index.html', results=[], API_PREFIX=API_PREFIX, GUI_ENABLED_ENGINES=GUI_ENABLED_ENGINES) @app.route('/analyze', methods=['POST']) def analyze(): @@ -67,18 +73,18 @@ def analyze(): analysis_id = str(uuid.uuid4()) threading.Thread(target=perform_analysis, args=(app, observables, selected_engines, analysis_id)).start() - return render_template('waiting.html', analysis_id=analysis_id), 200 + return render_template('waiting.html', analysis_id=analysis_id, API_PREFIX=API_PREFIX), 200 @app.route('/results/', methods=['GET']) def show_results(analysis_id): """Show the results of the analysis.""" analysis_results = db.session.get(AnalysisResult, analysis_id) if analysis_results: - return render_template('index.html', analysis_results=analysis_results) + return render_template('index.html', analysis_results=analysis_results, API_PREFIX=API_PREFIX) else: return render_template('404.html'), 404 -@app.route('/is_analysis_complete/', methods=['GET']) +@app.route(f'/{API_PREFIX}/is_analysis_complete/', methods=['GET']) def is_analysis_complete(analysis_id): """Check if the analysis is complete.""" complete = not check_analysis_in_progress(analysis_id) @@ -157,7 +163,7 @@ def update_config(): message = "An error occurred while updating the configuration." return jsonify({'message': message}) -@app.route('/api/results/', methods=['GET']) +@app.route(f'/{API_PREFIX}/results/', methods=['GET']) def get_results(analysis_id): """Get the results of the analysis.""" analysis_results = db.session.get(AnalysisResult, analysis_id) @@ -166,9 +172,9 @@ def get_results(analysis_id): else: return jsonify({'error': 'Analysis not found.'}), 404 -@app.route('/api/analyze', methods=['POST']) +@app.route(f'/{API_PREFIX}/analyze', methods=['POST']) def analyze_api(): - """Handle the analyze request.""" + """Handle the analyze request via API. (Only JSON data is accepted)""" data = request.get_json() form_data = ioc_fanger.fang(data.get("text", "")) observables = extract_observables(form_data) diff --git a/templates/index-demo.html b/templates/index-demo.html index 7e053e1..a8f1a1a 100644 --- a/templates/index-demo.html +++ b/templates/index-demo.html @@ -385,53 +385,52 @@

DEMO - Cyberbro - Observable Analysis - All info is public!!!

Select the engines to use (use your own instance to get all engines, API limits may apply here)

-
-
-
-
-
-
-
-
-
-
-
-
-
-
-

+
@@ -511,13 +510,13 @@

Cyberbro - Analysis Results

- +
@@ -525,13 +510,13 @@

Cyberbro - Analysis Results

- +