From ed8696729c0441ddfbc6a21a7d7be12360b56abc Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Wed, 9 Oct 2019 21:31:20 +0100 Subject: [PATCH 1/5] Add event.ingested as the ingest timestamp. --- code/go/ecs/event.go | 4 ++++ docs/field-details.asciidoc | 11 +++++++++++ generated/beats/fields.ecs.yml | 5 +++++ generated/csv/fields.csv | 1 + generated/ecs/ecs_flat.yml | 9 +++++++++ generated/ecs/ecs_nested.yml | 9 +++++++++ generated/elasticsearch/6/template.json | 3 +++ generated/elasticsearch/7/template.json | 3 +++ generated/legacy/template.json | 3 +++ schema.json | 10 ++++++++++ schemas/event.yml | 8 ++++++++ 11 files changed, 66 insertions(+) diff --git a/code/go/ecs/event.go b/code/go/ecs/event.go index 51b2dff995..30493215a8 100644 --- a/code/go/ecs/event.go +++ b/code/go/ecs/event.go @@ -164,4 +164,8 @@ type Event struct { // This is mainly useful if you use more than one system that assigns risk // scores, and you want to see a normalized value across all systems. RiskScoreNorm float64 `ecs:"risk_score_norm"` + + // Time when the event was ingested. This is different from `@timestamp` + // which is when the event originally occurred. + Ingested time.Time `ecs:"ingested"` } diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index e5aa5f247f..109faa9323 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -1222,6 +1222,17 @@ type: keyword example: `8a4f500d` +| core + +// =============================================================== + +| event.ingested +| Time when the event was ingested. This is different from `@timestamp` which is when the event originally occurred. + +type: date + + + | core // =============================================================== diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index b95ae54007..0b11b41191 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -987,6 +987,11 @@ ignore_above: 1024 description: Unique ID to describe the event. example: 8a4f500d + - name: ingested + level: core + type: date + description: Time when the event was ingested. This is different from `@timestamp` + which is when the event originally occurred. - name: kind level: extended type: keyword diff --git a/generated/csv/fields.csv b/generated/csv/fields.csv index 9859650b83..4e3b3d77d7 100644 --- a/generated/csv/fields.csv +++ b/generated/csv/fields.csv @@ -116,6 +116,7 @@ event.duration,long,core,,1.2.0-dev event.end,date,extended,,1.2.0-dev event.hash,keyword,extended,123456789012345678901234567890ABCD,1.2.0-dev event.id,keyword,core,8a4f500d,1.2.0-dev +event.ingested,date,core,,1.2.0-dev event.kind,keyword,extended,state,1.2.0-dev event.module,keyword,core,apache,1.2.0-dev event.original,keyword,core,Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232,1.2.0-dev diff --git a/generated/ecs/ecs_flat.yml b/generated/ecs/ecs_flat.yml index e00c05f372..6ebed5cc3b 100644 --- a/generated/ecs/ecs_flat.yml +++ b/generated/ecs/ecs_flat.yml @@ -1335,6 +1335,15 @@ event.id: order: 0 short: Unique ID to describe the event. type: keyword +event.ingested: + description: Time when the event was ingested. This is different from `@timestamp` + which is when the event originally occurred. + flat_name: event.ingested + level: core + name: ingested + order: 21 + short: Ingest timestamp + type: date event.kind: description: 'The kind of the event. diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index 942ee5dfef..e5dd6932d5 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -1547,6 +1547,15 @@ event: order: 0 short: Unique ID to describe the event. type: keyword + ingested: + description: Time when the event was ingested. This is different from `@timestamp` + which is when the event originally occurred. + flat_name: event.ingested + level: core + name: ingested + order: 21 + short: Ingest timestamp + type: date kind: description: 'The kind of the event. diff --git a/generated/elasticsearch/6/template.json b/generated/elasticsearch/6/template.json index d4940b74c5..2f9d7204b3 100644 --- a/generated/elasticsearch/6/template.json +++ b/generated/elasticsearch/6/template.json @@ -564,6 +564,9 @@ "ignore_above": 1024, "type": "keyword" }, + "ingested": { + "type": "date" + }, "kind": { "ignore_above": 1024, "type": "keyword" diff --git a/generated/elasticsearch/7/template.json b/generated/elasticsearch/7/template.json index 8ae0c64e9a..8f1eb4dd65 100644 --- a/generated/elasticsearch/7/template.json +++ b/generated/elasticsearch/7/template.json @@ -563,6 +563,9 @@ "ignore_above": 1024, "type": "keyword" }, + "ingested": { + "type": "date" + }, "kind": { "ignore_above": 1024, "type": "keyword" diff --git a/generated/legacy/template.json b/generated/legacy/template.json index d8bdc4e974..7e15dd0852 100644 --- a/generated/legacy/template.json +++ b/generated/legacy/template.json @@ -376,6 +376,9 @@ "ignore_above": 1024, "type": "keyword" }, + "ingested": { + "type": "date" + }, "kind": { "ignore_above": 1024, "type": "keyword" diff --git a/schema.json b/schema.json index 29136b9e6e..3fecf53aa5 100644 --- a/schema.json +++ b/schema.json @@ -882,6 +882,16 @@ "required": false, "type": "keyword" }, + "event.ingested": { + "description": "Time when the event was ingested. This is different from `@timestamp` which is when the event originally occurred.", + "example": "", + "footnote": "", + "group": 2, + "level": "core", + "name": "event.ingested", + "required": false, + "type": "date" + }, "event.kind": { "description": "The kind of the event.\nThis gives information about what type of information the event contains, without being specific to the contents of the event. Examples are `event`, `state`, `alarm`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution.", "example": "state", diff --git a/schemas/event.yml b/schemas/event.yml index d2b85cdbcd..afa54a3ebc 100644 --- a/schemas/event.yml +++ b/schemas/event.yml @@ -276,3 +276,11 @@ This is mainly useful if you use more than one system that assigns risk scores, and you want to see a normalized value across all systems. + + - name: ingested + level: core + type: date + short: Ingest timestamp + description: > + Time when the event was ingested. This is different from `@timestamp` + which is when the event originally occurred. From 443da9b7a2e9bdcd40abc161607458bde730551f Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Wed, 9 Oct 2019 21:57:33 +0100 Subject: [PATCH 2/5] Changelog --- CHANGELOG.next.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index edc5cf66f4..fa8d86a169 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -10,6 +10,7 @@ Thanks, you're awesome :-) --> ### Bugfixes ### Added +* Add `event.ingested` as the ingest timestamp. #582 ### Improvements From 18b1d0c83eb8c23c089d4b99bdd645cf99305fb8 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Wed, 13 Nov 2019 16:33:31 -0500 Subject: [PATCH 3/5] spacing tweak in changelog --- CHANGELOG.next.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index fa8d86a169..a6c9ec391a 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -10,6 +10,7 @@ Thanks, you're awesome :-) --> ### Bugfixes ### Added + * Add `event.ingested` as the ingest timestamp. #582 ### Improvements From 8e5023e04bb1d0358143d55622f98c7c4ef89827 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Wed, 13 Nov 2019 16:37:51 -0500 Subject: [PATCH 4/5] Flesh out the description wrt related timestamps --- code/go/ecs/event.go | 9 +++++++-- docs/field-details.asciidoc | 6 +++++- generated/beats/fields.ecs.yml | 10 ++++++++-- generated/ecs/ecs_flat.yml | 12 +++++++++--- generated/ecs/ecs_nested.yml | 12 +++++++++--- schema.json | 2 +- schemas/event.yml | 12 +++++++++--- 7 files changed, 48 insertions(+), 15 deletions(-) diff --git a/code/go/ecs/event.go b/code/go/ecs/event.go index 30493215a8..92bf942be0 100644 --- a/code/go/ecs/event.go +++ b/code/go/ecs/event.go @@ -165,7 +165,12 @@ type Event struct { // scores, and you want to see a normalized value across all systems. RiskScoreNorm float64 `ecs:"risk_score_norm"` - // Time when the event was ingested. This is different from `@timestamp` - // which is when the event originally occurred. + // Timestamp when an event arrived in the central data store. + // This is different from `@timestamp`, which is when the event originally + // occurred. It's also different from `event.created`, which is meant to + // capture the first time an agent saw the event. + // In normal conditions, assuming no tampering, the timestamps should + // chronologically look like this: `@timestamp` < `event.created` < + // `event.ingested`. Ingested time.Time `ecs:"ingested"` } diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index 109faa9323..6367e93cd4 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -1227,7 +1227,11 @@ example: `8a4f500d` // =============================================================== | event.ingested -| Time when the event was ingested. This is different from `@timestamp` which is when the event originally occurred. +| Timestamp when an event arrived in the central data store. + +This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. + +In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` < `event.created` < `event.ingested`. type: date diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index 0b11b41191..4b4c5fe0b2 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -990,8 +990,14 @@ - name: ingested level: core type: date - description: Time when the event was ingested. This is different from `@timestamp` - which is when the event originally occurred. + description: 'Timestamp when an event arrived in the central data store. + + This is different from `@timestamp`, which is when the event originally occurred. It''s + also different from `event.created`, which is meant to capture the first time + an agent saw the event. + + In normal conditions, assuming no tampering, the timestamps should chronologically + look like this: `@timestamp` < `event.created` < `event.ingested`.' - name: kind level: extended type: keyword diff --git a/generated/ecs/ecs_flat.yml b/generated/ecs/ecs_flat.yml index 6ebed5cc3b..c382008bd4 100644 --- a/generated/ecs/ecs_flat.yml +++ b/generated/ecs/ecs_flat.yml @@ -1336,13 +1336,19 @@ event.id: short: Unique ID to describe the event. type: keyword event.ingested: - description: Time when the event was ingested. This is different from `@timestamp` - which is when the event originally occurred. + description: 'Timestamp when an event arrived in the central data store. + + This is different from `@timestamp`, which is when the event originally occurred. It''s + also different from `event.created`, which is meant to capture the first time + an agent saw the event. + + In normal conditions, assuming no tampering, the timestamps should chronologically + look like this: `@timestamp` < `event.created` < `event.ingested`.' flat_name: event.ingested level: core name: ingested order: 21 - short: Ingest timestamp + short: Timestamp when an event arrived in the central data store. type: date event.kind: description: 'The kind of the event. diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index e5dd6932d5..4c5fa9d980 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -1548,13 +1548,19 @@ event: short: Unique ID to describe the event. type: keyword ingested: - description: Time when the event was ingested. This is different from `@timestamp` - which is when the event originally occurred. + description: 'Timestamp when an event arrived in the central data store. + + This is different from `@timestamp`, which is when the event originally occurred. It''s + also different from `event.created`, which is meant to capture the first time + an agent saw the event. + + In normal conditions, assuming no tampering, the timestamps should chronologically + look like this: `@timestamp` < `event.created` < `event.ingested`.' flat_name: event.ingested level: core name: ingested order: 21 - short: Ingest timestamp + short: Timestamp when an event arrived in the central data store. type: date kind: description: 'The kind of the event. diff --git a/schema.json b/schema.json index 3fecf53aa5..95f29f657b 100644 --- a/schema.json +++ b/schema.json @@ -883,7 +883,7 @@ "type": "keyword" }, "event.ingested": { - "description": "Time when the event was ingested. This is different from `@timestamp` which is when the event originally occurred.", + "description": "Timestamp when an event arrived in the central data store.\nThis is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event.\nIn normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` < `event.created` < `event.ingested`.", "example": "", "footnote": "", "group": 2, diff --git a/schemas/event.yml b/schemas/event.yml index afa54a3ebc..11da094f93 100644 --- a/schemas/event.yml +++ b/schemas/event.yml @@ -280,7 +280,13 @@ - name: ingested level: core type: date - short: Ingest timestamp + short: Timestamp when an event arrived in the central data store. description: > - Time when the event was ingested. This is different from `@timestamp` - which is when the event originally occurred. + Timestamp when an event arrived in the central data store. + + This is different from `@timestamp`, which is when the event originally + occurred. It's also different from `event.created`, which is meant + to capture the first time an agent saw the event. + + In normal conditions, assuming no tampering, the timestamps should + chronologically look like this: `@timestamp` < `event.created` < `event.ingested`. From b536afce4d0875fdda0cdcec63341c7f60d6d349 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Wed, 13 Nov 2019 16:45:49 -0500 Subject: [PATCH 5/5] Let's set the example by adding examples :-) --- docs/field-details.asciidoc | 4 ++-- generated/beats/fields.ecs.yml | 2 ++ generated/csv/fields.csv | 4 ++-- generated/ecs/ecs_flat.yml | 2 ++ generated/ecs/ecs_nested.yml | 2 ++ schema.json | 4 ++-- schemas/event.yml | 2 ++ 7 files changed, 14 insertions(+), 6 deletions(-) diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index 6367e93cd4..e2723425f8 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -1159,7 +1159,7 @@ In case the two timestamps are identical, @timestamp should be used. type: date - +example: `2016-05-23 08:05:34.857000` | core @@ -1235,7 +1235,7 @@ In normal conditions, assuming no tampering, the timestamps should chronological type: date - +example: `2016-05-23 08:05:35.101000` | core diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index 4b4c5fe0b2..0cefdb259c 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -945,6 +945,7 @@ your agent''s or pipeline''s ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used.' + example: 2016-05-23 08:05:34.857000 - name: dataset level: core type: keyword @@ -998,6 +999,7 @@ In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` < `event.created` < `event.ingested`.' + example: 2016-05-23 08:05:35.101000 - name: kind level: extended type: keyword diff --git a/generated/csv/fields.csv b/generated/csv/fields.csv index 4e3b3d77d7..4b5884cf71 100644 --- a/generated/csv/fields.csv +++ b/generated/csv/fields.csv @@ -110,13 +110,13 @@ error.type,keyword,extended,java.lang.NullPointerException,1.2.0-dev event.action,keyword,core,user-password-change,1.2.0-dev event.category,keyword,core,user-management,1.2.0-dev event.code,keyword,extended,4648,1.2.0-dev -event.created,date,core,,1.2.0-dev +event.created,date,core,2016-05-23 08:05:34.857000,1.2.0-dev event.dataset,keyword,core,apache.access,1.2.0-dev event.duration,long,core,,1.2.0-dev event.end,date,extended,,1.2.0-dev event.hash,keyword,extended,123456789012345678901234567890ABCD,1.2.0-dev event.id,keyword,core,8a4f500d,1.2.0-dev -event.ingested,date,core,,1.2.0-dev +event.ingested,date,core,2016-05-23 08:05:35.101000,1.2.0-dev event.kind,keyword,extended,state,1.2.0-dev event.module,keyword,core,apache,1.2.0-dev event.original,keyword,core,Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232,1.2.0-dev diff --git a/generated/ecs/ecs_flat.yml b/generated/ecs/ecs_flat.yml index c382008bd4..4e01ece547 100644 --- a/generated/ecs/ecs_flat.yml +++ b/generated/ecs/ecs_flat.yml @@ -1266,6 +1266,7 @@ event.created: agent''s or pipeline''s ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used.' + example: 2016-05-23 08:05:34.857000 flat_name: event.created level: core name: created @@ -1344,6 +1345,7 @@ event.ingested: In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` < `event.created` < `event.ingested`.' + example: 2016-05-23 08:05:35.101000 flat_name: event.ingested level: core name: ingested diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index 4c5fa9d980..0abc7804f9 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -1477,6 +1477,7 @@ event: your agent''s or pipeline''s ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used.' + example: 2016-05-23 08:05:34.857000 flat_name: event.created level: core name: created @@ -1556,6 +1557,7 @@ event: In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` < `event.created` < `event.ingested`.' + example: 2016-05-23 08:05:35.101000 flat_name: event.ingested level: core name: ingested diff --git a/schema.json b/schema.json index 95f29f657b..34c5de9bae 100644 --- a/schema.json +++ b/schema.json @@ -824,7 +824,7 @@ }, "event.created": { "description": "event.created contains the date/time when the event was first read by an agent, or by your pipeline.\nThis field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event.\nIn most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source.\nIn case the two timestamps are identical, @timestamp should be used.", - "example": "", + "example": "2016-05-23 08:05:34.857000", "footnote": "", "group": 2, "level": "core", @@ -884,7 +884,7 @@ }, "event.ingested": { "description": "Timestamp when an event arrived in the central data store.\nThis is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event.\nIn normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` < `event.created` < `event.ingested`.", - "example": "", + "example": "2016-05-23 08:05:35.101000", "footnote": "", "group": 2, "level": "core", diff --git a/schemas/event.yml b/schemas/event.yml index 11da094f93..9f9a2f4642 100644 --- a/schemas/event.yml +++ b/schemas/event.yml @@ -231,6 +231,7 @@ level: core type: date short: Time when the event was first read by an agent or by your pipeline. + example: 2016-05-23T08:05:34.857Z description: > event.created contains the date/time when the event was first read by an agent, or by your pipeline. @@ -281,6 +282,7 @@ level: core type: date short: Timestamp when an event arrived in the central data store. + example: 2016-05-23T08:05:35.101Z description: > Timestamp when an event arrived in the central data store.