Opencti engine ready - closes #14

Dockerfile-test

@@ -4,16 +4,6 @@ FROM python:3.13-slim
 # Set the working directory in the container
 WORKDIR /app
 
-# -----------------------------------------------------------------------------
-# Install system dependencies for Tor - Use only if you know what you are doing!
-# RUN apt-get update && \
-#    apt-get install -y tor && \
-#    rm -rf /var/lib/apt/lists/*
-
-# Configure Tor to allow control with cookie authentication
-# RUN echo "ControlPort 9051\nCookieAuthentication 1" >> /etc/tor/torrc
-# -----------------------------------------------------------------------------
-
 # Copy the requirements.txt file into the container
 COPY requirements.txt .
 
@@ -26,15 +16,5 @@ COPY . .
 # Expose port 5000 for Flask
 EXPOSE 5000
 
-# -----------------------------------------------------------------------------
-# Tor control port
-# EXPOSE 9051
-# -----------------------------------------------------------------------------
-
-# -----------------------------------------------------------------------------
-# Start Tor in the background and then the Flask application - Disabled by default
-# CMD tor & python app.py
-# -----------------------------------------------------------------------------
-
 # Start the Flask application
 CMD python app.py

README.md

@@ -27,6 +27,7 @@ without having to deploy a **complex** solution.
 * **Abuse Contact Lookup**: Accurately find abuse contacts for IPs, URLs, and domains.
 * **Export Options**: Export results to CSV and **autofiltered well formatted** Excel files.
 * **MDE Integration**: Check if observables are flagged on your Microsoft Defender for Endpoint (MDE) tenant.
+* **OpenCTI Integration**: Get stats (number of incidents, indicators) from OpenCTI and the latest Indicator if available.
 * **Proxy Support**: Use a proxy if required.
 * **Data Storage**: Store results in a SQLite database.
 * **Analysis History**: Maintain a history of analyses with easy retrieval and search functionality.
@@ -78,7 +79,9 @@ cp secrets-sample.json secrets.json
     "mde_tenant_id": "tenant_here",
     "mde_client_id": "client_id_here",
     "mde_client_secret": "client_secret_here",
-    "shodan": "token_here"
+    "shodan": "token_here",
+    "opencti_api_key": "token_here",
+    "opencti_url": "https://demo.opencti.io"
 }
 ```
 
@@ -225,6 +228,7 @@ curl "http://localhost:5000/api/results/e88de647-b153-4904-91e5-8f5c79174854"
 * [ThreatFox](https://threatfox.abuse.ch/api/)
 * [URLscan](https://urlscan.io/)
 * [Ioc.One](https://ioc.one/)
+* [OpenCTI](https://www.opencti.io/)
 
 > [!NOTE]
 > Any questions? Check the [wiki](https://github.com/stanfrbd/cyberbro/wiki) or raise an [issue](https://github.com/stanfrbd/cyberbro/issues/new)

engines/opencti.py

@@ -0,0 +1,205 @@
+import requests
+from urllib.parse import urljoin
+
+# Disable SSL warnings in case of proxies like Zscaler which break SSL...
+requests.packages.urllib3.disable_warnings()
+
+def query_opencti(observable, API_KEY, OPENCTI_URL, PROXIES):
+    """
+    Queries the OpenCTI API for information about a given observable.
+    Args:
+      observable (str): The observable to check.
+      api_key (str): The API key for authentication.
+      proxy (dict): The proxy settings.
+    Returns:
+      dict: A dictionary containing the response data.
+      None: If the response does not contain the expected data.
+    Raises:
+      requests.exceptions.RequestException: If there is an issue with the network request.
+      ValueError: If the response cannot be parsed as JSON.
+    """
+    try:
+      # Ensure the URL is properly formatted without trailing slashes
+      OPENCTI_URL = urljoin(OPENCTI_URL, '/')
+      OPENCTI_URL = OPENCTI_URL.rstrip('/')
+
+      # URL for the OpenCTI API
+      url = f"{OPENCTI_URL}/graphql"
+
+      # Headers including the API key
+      headers = {
+        "Authorization": f"Bearer {API_KEY}",
+        "Content-Type": "application/json"
+      }
+
+      # GraphQL query
+      query = """
+      query SearchStixCoreObjectsLinesPaginationQuery(
+        $types: [String]
+        $search: String
+        $count: Int!
+        $cursor: ID
+        $orderBy: StixCoreObjectsOrdering
+        $orderMode: OrderingMode
+        $filters: FilterGroup
+      ) {
+        globalSearch(types: $types, search: $search, first: $count, after: $cursor, orderBy: $orderBy, orderMode: $orderMode, filters: $filters) {
+        edges {
+          node {
+          id
+          entity_type
+          created_at
+          createdBy {
+            name
+            id
+          }
+          creators {
+            id
+            name
+          }
+          objectMarking {
+            id
+            definition_type
+            definition
+            x_opencti_order
+            x_opencti_color
+          }
+          }
+          cursor
+        }
+        pageInfo {
+          endCursor
+          hasNextPage
+          globalCount
+        }
+        }
+      }
+      """
+
+      # Filter by date desc and take the first 100 results
+      variables = {
+        "count": 100,
+        "orderMode": "desc",
+        "orderBy": "created_at",
+        "filters": {
+          "mode": "and",
+          "filters": [
+            {
+              "key": "entity_type",
+              "values": ["Stix-Core-Object"],
+              "operator": "eq",
+              "mode": "or"
+            }
+          ],
+          "filterGroups": []
+        },
+        "search": observable
+      }
+
+      # Payload for the POST request
+      payload = {
+        "id": "SearchStixCoreObjectsLinesPaginationQuery",
+        "query": query,
+        "variables": variables
+      }
+
+      # Define the search link
+      search_link = f"{OPENCTI_URL}/dashboard/search/knowledge/{observable}"
+
+      # Make the POST request to the API
+      response = requests.post(url, headers=headers, json=payload, proxies=PROXIES, verify=False)
+
+      # Parse the JSON response
+      data = response.json()
+
+      # Check if the response contains the expected data
+      if 'data' in data and 'globalSearch' in data['data']:
+        entity_counts = {}
+        edges = data['data']['globalSearch']['edges']
+        for edge in edges:
+          entity_type = edge['node']['entity_type']
+          if entity_type in entity_counts:
+            entity_counts[entity_type] += 1
+          else:
+            entity_counts[entity_type] = 1
+
+        global_count = data['data']['globalSearch']['pageInfo']['globalCount']
+
+        # Find the most recent element and check if it's an Indicator
+        first_element = edges[0]['node']
+        first_id = first_element['id']
+        latest_created_at = first_element['created_at']
+        latest_indicator_link = f"{OPENCTI_URL}/dashboard/observations/indicators/{first_id}" if first_element['entity_type'] == "Indicator" else None
+
+        # If the most recent element is not an Indicator, search for an Indicator in the data
+        if first_element['entity_type'] != "Indicator":
+          for edge in edges:
+            if edge['node']['entity_type'] == "Indicator":
+              first_element = edge['node']
+              first_id = first_element['id']
+              latest_created_at = first_element['created_at']
+              latest_indicator_link = f"{OPENCTI_URL}/dashboard/observations/indicators/{first_id}"
+              break
+
+        # If the most recent element is an Indicator, query for its additional attributes
+        x_opencti_score = None
+        revoked = None
+        valid_from = None
+        valid_until = None
+        confidence = None
+        name = None
+        if first_element['entity_type'] == "Indicator":
+          additional_query = """
+          query GetIndicator($id: String!) {
+            indicator(id: $id) {
+              name
+              x_opencti_score
+              revoked
+              valid_from
+              valid_until
+              confidence
+            }
+          }
+          """
+          additional_variables = {"id": first_id}
+          additional_payload = {
+            "query": additional_query,
+            "variables": additional_variables
+          }
+          additional_response = requests.post(url, headers=headers, json=additional_payload, proxies=PROXIES, verify=False)
+          additional_data = additional_response.json()
+          if 'data' in additional_data and 'indicator' in additional_data['data']:
+            indicator_data = additional_data['data']['indicator']
+            x_opencti_score = indicator_data.get('x_opencti_score')
+            revoked = indicator_data.get('revoked')
+            valid_from = indicator_data.get('valid_from')
+            valid_until = indicator_data.get('valid_until')
+            confidence = indicator_data.get('confidence')
+            name = indicator_data.get('name')
+
+        # Format dates to YYYY-MM-DD
+        if valid_from:
+          valid_from = valid_from.split("T")[0]
+        if valid_until:
+          valid_until = valid_until.split("T")[0]
+        if latest_created_at:
+          latest_created_at = latest_created_at.split("T")[0]  
+
+        return {
+            "entity_counts": entity_counts,
+            "global_count": global_count,
+            "search_link": search_link,
+            "latest_created_at": latest_created_at,
+            "latest_indicator_link": latest_indicator_link,
+            "latest_indicator_name": name,
+            "x_opencti_score": x_opencti_score,
+            "revoked": revoked,
+            "valid_from": valid_from,
+            "valid_until": valid_until,
+            "confidence": confidence
+        }
+    except Exception as e:
+      print(e)
+
+    return None
+

engines/spur_us_free.py

@@ -1,27 +1,18 @@
-from stem import Signal
-from stem.control import Controller
 import requests
 from bs4 import BeautifulSoup
 from fake_useragent import UserAgent
-import time
 
 ua = UserAgent()
 
 # Disable SSL warning in case of proxy like Zscaler which breaks SSL...
 requests.packages.urllib3.disable_warnings()
 
-def get_new_identity():
-    with Controller.from_port(port=9051) as controller:
-        controller.authenticate()  # Authentication using cookie
-        controller.signal(Signal.NEWNYM)
-
 def get_spur(ip, PROXIES):
     """
     Retrieves information about the given IP address from the spur.us website.
 
     This function makes an HTTP GET request to the spur.us website to fetch context information about the provided IP address.
-    If the TOR network is running, it waits for a second before making the request. The function parses the HTML response to
-    extract the anonymity status of the IP address from the page title.
+    The function parses the HTML response to extract the anonymity status of the IP address from the page title.
 
     Args:
         ip (str): The IP address to retrieve information for.
@@ -36,11 +27,6 @@ def get_spur(ip, PROXIES):
         None: If an error occurs during the request or parsing process.
     """
     try:
-        TOR_RUNNING = False
-        if PROXIES["http"] == "socks5h://127.0.0.1:9050":
-            TOR_RUNNING = True
-            #get_new_identity()
-            time.sleep(1)
         spur_url = f"https://spur.us/context/{ip}"
         spur_data = requests.get(spur_url, proxies=PROXIES, verify=False, headers={"User-Agent": ua.random})
         # print(spur_data.text)
@@ -56,12 +42,9 @@ def get_spur(ip, PROXIES):
             else:
                 content = "Not anonymous"
         else:
-            if TOR_RUNNING:
-                time.sleep(5)
-                get_new_identity()
-                get_spur(ip)
+            content = "Not anonymous"
         return {"link": f"https://spur.us/context/{ip}", "tunnels": content}
     except Exception as e:
         print(e)
     # Always return None in case of failure
-    return None
+    return None

requirements.txt

@@ -1,5 +1,4 @@
 requests
-requests[socks]
 flask
 pandas
 jwt
@@ -8,7 +7,6 @@ dnspython
 pycountry
 fake_useragent
 bs4
-stem
 pytest
 querycontacts
 flask_sqlalchemy

secrets-sample.json

@@ -7,5 +7,7 @@
     "mde_tenant_id": "tenant_here",
     "mde_client_id": "client_id_here",
     "mde_client_secret": "client_secret_here",
-    "shodan": "token_here"
+    "shodan": "token_here",
+    "opencti_api_key": "token_here",
+    "opencti_url": "https://demo.opencti.io"
 }