[Filebeat] Add ThreatQuotient to Threat Intel Module elastic#27423

CHANGELOG.next.asciidoc

@@ -92,6 +92,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove deprecated fields in Kafka module. {pull}27938[27938]
 - Remove deprecated fields in coredns module. {pull}28196[28196]
 - Remove old `httpjson` config implementation. {pull}28054[28054]
+- Added dataset `threatq` to the `threatintel` module to ingest indicators from ThreatQ {issue}27423[27423]
 
 *Heartbeat*
 

x-pack/filebeat/module/threatintel/_meta/config.yml

@@ -169,3 +169,28 @@
 
     # Set your API Token.
     var.api_token: "<RF_TOKEN>"
+
+  threatq:
+    enabled: true
+
+    # Input used for ingesting threat intel data
+    var.input: httpjson
+
+    # The URL used for ThreatQ ThreatLibrary API calls.
+    # Remember to put a slash at the end of the host URL
+    var.host: https://www.threatq.com/
+
+    # Oauth 2.0 Access Token URL
+    var.token_url: https://www.threatq.com/api/token
+
+    # Oauth 2.0 Client ID
+    var.client_id: "INSERT_CLIENT_ID"
+
+    # Oauth 2.0 Client Secret
+    var.client_secret: "INSERT_CLIENT_SECRET"
+
+    # The interval to poll the API for updates
+    var.interval: 1m
+
+    # The ID for the ThreatQ smart data collection
+    var.data_collection_id: "INSERT_THREATQ_DATA_COLLECTION_ID"

x-pack/filebeat/module/threatintel/module.yml

@@ -15,3 +15,5 @@ dashboards:
     file: Filebeat-threatintel-overview.json
   - id: 894dd3e0-df57-11eb-8f2b-753caedf727d
     file: Filebeat-threatintel-recordedfuture.json
+  - id: 3b605720-ff78-11eb-acb2-2960a7069ed1
+    file: Filebeat-threatintel-threatq.json

x-pack/filebeat/module/threatintel/threatq/_meta/fields.yml

@@ -0,0 +1,41 @@
+- name: threatq
+  type: group
+  description: >
+    Fields for ThreatQ Threat Library
+  fields:
+  - name: updated_at
+    type: date
+    description: >
+      Last modification time
+  - name: created_at
+    type: date
+    description: >
+      Object creation time
+  - name: expires_at
+    type: date
+    description: >
+      Expiration time
+  - name: expires_calculated_at
+    type: date
+    description: >
+      Expiration calculation time
+  - name: published_at
+    type: date
+    description: >
+      Object publication time
+  - name: status
+    type: keyword
+    description: >
+      Object status within the Threat Library
+  - name: indicator_value
+    type: keyword
+    description: >
+      Original indicator value
+  - name: adversaries
+    type: keyword
+    description: >
+      Adversaries that are linked to the object
+  - name: attributes
+    type: flattened
+    description: >
+      These provide additional context about an object

x-pack/filebeat/module/threatintel/threatq/config/config.yml

@@ -0,0 +1,64 @@
+{{ if eq .input "httpjson" }}
+
+type: httpjson
+config_version: "2"
+interval: {{ .interval }}
+
+auth.oauth2:
+  client.id: {{ .client_id }}
+  client.secret: {{ .client_secret }}
+  token_url: {{ .token_url }}
+request.method: GET
+{{ if .ssl }}
+  - request.ssl: {{ .ssl | tojson }}
+{{ end }}
+request.url: {{ .host }}api/indicators/query/hash/{{ .data_collection_id }}?sort=%2Bupdated_at&limit=1000
+request.transforms:
+  - set:
+      target: url.params.cursorMark
+      value: '[[.cursor.cursor_mark]]'
+      default: "*"
+cursor:
+  cursor_mark:
+    ignore_empty_value: true
+    value: '[[.last_response.body.nextCursorMark]]'
+response.request_body_on_pagination: true
+
+response.split:
+  target: body.data
+
+{{ else if eq .input "file" }}
+
+type: log
+paths:
+{{ range $i, $path := .paths }}
+  - {{$path}}
+{{ end }}
+exclude_files: [".gz$"]
+
+{{ end }}
+
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
+
+processors:
+  - decode_json_fields:
+      fields: [message]
+      target: json
+      process_array: true
+      max_depth: 5
+      overwrite_keys: true
+      add_error_key: true
+  - fingerprint:
+      fields: ["json.id"]
+      target_field: "@metadata._id"
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.10.0
+  - timestamp:
+      field: json.created_at
+      layouts:
+        - '2006-01-02 15:04:05'
+      test:
+        - '2021-06-17 13:55:02'