Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add additional processors in beats and pipelines #10

Merged
merged 10 commits into from
Dec 16, 2020
Merged

Add additional processors in beats and pipelines #10

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
0a8b1d4
Add additional processors in beats and pipelines
Dec 9, 2020
3392ef1
Add related.hosts fields
adriansr Dec 10, 2020
1cb5d4e
Don't populate related.hosts in ingest pipeline
adriansr Dec 10, 2020
38a41d0
append host.name in ingest pipeline
adriansr Dec 10, 2020
13ae589
Add server.domain to related.hosts
adriansr Dec 10, 2020
46b4071
add server.domain to related.hosts
adriansr Dec 10, 2020
40c4743
Yet another missing mapping for related.hosts
adriansr Dec 10, 2020
d1f4dcd
liblogparser.js: Ignore empty values in field mappings
adriansr Dec 11, 2020
2d06643
Add new *.top_level_domain field from ECS 1.7
adriansr Dec 14, 2020
3f709bb
Add missing ECS fields for packages
adriansr Dec 15, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 37 additions & 1 deletion layout/module/__module__/__fileset__/config/input.yml.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,43 @@ processors:
{{ if .community_id }}
- community_id: ~
{{ end }}
- registered_domain:
ignore_missing: true
ignore_failure: true
field: dns.question.name
target_field: dns.question.registered_domain
target_subdomain_field: dns.question.subdomain
- registered_domain:
ignore_missing: true
ignore_failure: true
field: client.domain
target_field: client.registered_domain
target_subdomain_field: client.subdomain
- registered_domain:
ignore_missing: true
ignore_failure: true
field: server.domain
target_field: server.registered_domain
target_subdomain_field: server.subdomain
- registered_domain:
ignore_missing: true
ignore_failure: true
field: destination.domain
target_field: destination.registered_domain
target_subdomain_field: destination.subdomain
- registered_domain:
ignore_missing: true
ignore_failure: true
field: source.domain
target_field: source.registered_domain
target_subdomain_field: source.subdomain
- registered_domain:
ignore_missing: true
ignore_failure: true
field: url.domain
target_field: url.registered_domain
target_subdomain_field: url.subdomain
- add_fields:
target: ''
fields:
ecs.version: 1.6.0
ecs.version: 1.7.0
45 changes: 45 additions & 0 deletions layout/module/__module__/__fileset__/ingest/pipeline.yml.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,51 @@ processors:
field: destination.as.organization_name
target_field: destination.as.organization.name
ignore_missing: true
- append:
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it's better to populate related.hosts in liblogparser.js, as the other related fields.

field: related.hosts
value: '{{url.domain}}'
allow_duplicates: false
if: ctx?.url?.domain != null && ctx?.url?.domain != ""
- append:
field: related.hosts
value: '{{server.domain}}'
allow_duplicates: false
if: ctx?.server?.domain != null && ctx?.server?.domain != ""
- append:
field: related.hosts
value: '{{host.name}}'
allow_duplicates: false
if: ctx?.host?.name != null && ctx.host?.name != ''
- append:
field: related.hosts
value: '{{host.hostname}}'
allow_duplicates: false
if: ctx?.host?.hostnamename != null && ctx.host?.hostname != ''
- append:
field: related.hosts
value: '{{destination.address}}'
allow_duplicates: false
if: ctx?.destination?.address != null && ctx.destination?.address != ''
- append:
field: related.hosts
value: '{{source.address}}'
allow_duplicates: false
if: ctx?.source?.address != null && ctx.source?.address != ''
- append:
field: related.hosts
value: '{{rsa.web.fqdn}}'
allow_duplicates: false
if: ctx?.rsa?.web?.fqdn != null && ctx.rsa?.web?.fqdn != ''
- append:
field: related.hosts
value: '{{rsa.misc.event_source}}'
allow_duplicates: false
if: ctx?.rsa?.misc?.event_source != null && ctx.rsa?.misc?.event_source != ''
- append:
field: related.hosts
value: '{{rsa.web.web_ref_domain}}'
allow_duplicates: false
if: ctx?.rsa?.web?.web_ref_domain != null && ctx?.rsa?.web?.web_ref_domain != ''
on_failure:
- append:
field: error.message
Expand Down
36 changes: 36 additions & 0 deletions ...ut/package/__module__/__version__/data_stream/__fileset__/agent/stream/stream.yml.hbs.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,42 @@ processors:
((- setvar "var_prefix" "" -))
((- getvar "extra_processors" -))
- community_id:
- registered_domain:
ignore_missing: true
ignore_failure: true
field: dns.question.name
target_field: dns.question.registered_domain
target_subdomain_field: dns.question.subdomain
- registered_domain:
ignore_missing: true
ignore_failure: true
field: client.domain
target_field: client.registered_domain
target_subdomain_field: client.subdomain
- registered_domain:
ignore_missing: true
ignore_failure: true
field: server.domain
target_field: server.registered_domain
target_subdomain_field: server.subdomain
- registered_domain:
ignore_missing: true
ignore_failure: true
field: destination.domain
target_field: destination.registered_domain
target_subdomain_field: destination.subdomain
- registered_domain:
ignore_missing: true
ignore_failure: true
field: source.domain
target_field: source.registered_domain
target_subdomain_field: source.subdomain
- registered_domain:
ignore_missing: true
ignore_failure: true
field: url.domain
target_field: url.registered_domain
target_subdomain_field: url.subdomain
- add_locale: ~
- add_fields:
target: ''
Expand Down
38 changes: 37 additions & 1 deletion layout/package/__module__/__version__/data_stream/__fileset__/agent/stream/tcp.yml.hbs.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,44 @@ processors:
((- setvar "var_prefix" "" -))
((- getvar "extra_processors" -))
- community_id:
- registered_domain:
ignore_missing: true
ignore_failure: true
field: dns.question.name
target_field: dns.question.registered_domain
target_subdomain_field: dns.question.subdomain
- registered_domain:
ignore_missing: true
ignore_failure: true
field: client.domain
target_field: client.registered_domain
target_subdomain_field: client.subdomain
- registered_domain:
ignore_missing: true
ignore_failure: true
field: server.domain
target_field: server.registered_domain
target_subdomain_field: server.subdomain
- registered_domain:
ignore_missing: true
ignore_failure: true
field: destination.domain
target_field: destination.registered_domain
target_subdomain_field: destination.subdomain
- registered_domain:
ignore_missing: true
ignore_failure: true
field: source.domain
target_field: source.registered_domain
target_subdomain_field: source.subdomain
- registered_domain:
ignore_missing: true
ignore_failure: true
field: url.domain
target_field: url.registered_domain
target_subdomain_field: url.subdomain
- add_locale: ~
- add_fields:
target: ''
fields:
ecs.version: 1.6.0
ecs.version: 1.7.0
36 changes: 36 additions & 0 deletions layout/package/__module__/__version__/data_stream/__fileset__/agent/stream/udp.yml.hbs.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,42 @@ processors:
((- setvar "var_prefix" "" -))
((- getvar "extra_processors" -))
- community_id:
- registered_domain:
ignore_missing: true
ignore_failure: true
field: dns.question.name
target_field: dns.question.registered_domain
target_subdomain_field: dns.question.subdomain
- registered_domain:
ignore_missing: true
ignore_failure: true
field: client.domain
target_field: client.registered_domain
target_subdomain_field: client.subdomain
- registered_domain:
ignore_missing: true
ignore_failure: true
field: server.domain
target_field: server.registered_domain
target_subdomain_field: server.subdomain
- registered_domain:
ignore_missing: true
ignore_failure: true
field: destination.domain
target_field: destination.registered_domain
target_subdomain_field: destination.subdomain
- registered_domain:
ignore_missing: true
ignore_failure: true
field: source.domain
target_field: source.registered_domain
target_subdomain_field: source.subdomain
- registered_domain:
ignore_missing: true
ignore_failure: true
field: url.domain
target_field: url.registered_domain
target_subdomain_field: url.subdomain
- add_locale: ~
- add_fields:
target: ''
Expand Down
45 changes: 45 additions & 0 deletions ...odule__/__version__/data_stream/__fileset__/elasticsearch/ingest_pipeline/default.yml.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,51 @@ processors:
field: destination.as.organization_name
target_field: destination.as.organization.name
ignore_missing: true
- append:
field: related.hosts
value: '{{url.domain}}'
allow_duplicates: false
if: ctx?.url?.domain != null && ctx?.url?.domain != ""
- append:
field: related.hosts
value: '{{server.domain}}'
allow_duplicates: false
if: ctx?.server?.domain != null && ctx?.server?.domain != ""
- append:
field: related.hosts
value: '{{host.name}}'
allow_duplicates: false
if: ctx?.host?.name != null && ctx.host?.name != ''
- append:
field: related.hosts
value: '{{host.hostname}}'
allow_duplicates: false
if: ctx?.host?.hostnamename != null && ctx.host?.hostname != ''
- append:
field: related.hosts
value: '{{destination.address}}'
allow_duplicates: false
if: ctx?.destination?.address != null && ctx.destination?.address != ''
- append:
field: related.hosts
value: '{{source.address}}'
allow_duplicates: false
if: ctx?.source?.address != null && ctx.source?.address != ''
- append:
field: related.hosts
value: '{{rsa.web.fqdn}}'
allow_duplicates: false
if: ctx?.rsa?.web?.fqdn != null && ctx.rsa?.web?.fqdn != ''
- append:
field: related.hosts
value: '{{rsa.misc.event_source}}'
allow_duplicates: false
if: ctx?.rsa?.misc?.event_source != null && ctx.rsa?.misc?.event_source != ''
- append:
field: related.hosts
value: '{{rsa.web.web_ref_domain}}'
allow_duplicates: false
if: ctx?.rsa?.web?.web_ref_domain != null && ctx?.rsa?.web?.web_ref_domain != ''
on_failure:
- append:
field: error.message
Expand Down