Skip to content

Commit

Permalink
Several improvements to winlogbeat raw api: (#42297)
Browse files Browse the repository at this point in the history 
- Fingerprint templates by eventID+version
- Properly parse opcodes
- Improve handling of user data
  • Loading branch information
@marc-gr
marc-gr authored Jan 13, 2025
1 parent c3571bf commit f6d5acc
Show file tree
Hide file tree
Showing 3 changed files with 57 additions and 28 deletions.
63 changes: 43 additions & 20 deletions winlogbeat/sys/wineventlog/metadata_store.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package wineventlog

import (
"encoding/xml"
"fmt"
"strconv"
"strings"
Expand Down Expand Up @@ -51,10 +52,10 @@ type PublisherMetadataStore struct {
winevent.WinMeta

// Event ID to event metadata (message and event data param names).
Events map[uint16]*EventMetadata
Events map[uint32]*EventMetadata
// Event ID to map of fingerprints to event metadata. The fingerprint value
// is hash of the event data parameters count and types.
EventFingerprints map[uint16]map[uint64]*EventMetadata
EventFingerprints map[uint32]map[uint64]*EventMetadata
// Stores used messages by their ID. Message can be found in events as references
// such as %%1111. This need to be formatted the first time, and they are stored
// from that point after.
Expand All @@ -71,7 +72,7 @@ func NewPublisherMetadataStore(session EvtHandle, provider string, locale uint32
}
store := &PublisherMetadataStore{
Metadata: md,
EventFingerprints: map[uint16]map[uint64]*EventMetadata{},
EventFingerprints: map[uint32]map[uint64]*EventMetadata{},
MessagesByID: map[uint32]string{},
log: log.With("publisher", provider),
}
Expand Down Expand Up @@ -102,8 +103,8 @@ func NewEmptyPublisherMetadataStore(provider string, log *logp.Logger) *Publishe
Levels: map[uint8]string{},
Tasks: map[uint16]string{},
},
Events: map[uint16]*EventMetadata{},
EventFingerprints: map[uint16]map[uint64]*EventMetadata{},
Events: map[uint32]*EventMetadata{},
EventFingerprints: map[uint32]map[uint64]*EventMetadata{},
MessagesByID: map[uint32]string{},
log: log.With("publisher", provider, "empty", true),
}
Expand Down Expand Up @@ -137,7 +138,7 @@ func (s *PublisherMetadataStore) initOpcodes() error {
if val == "" {
val = opcodeMeta.Name
}
s.Opcodes[uint8(opcodeMeta.Mask)] = val
s.Opcodes[uint8(opcodeMeta.Opcode)] = val
}
return nil
}
Expand Down Expand Up @@ -182,23 +183,24 @@ func (s *PublisherMetadataStore) initEvents() error {
}
defer itr.Close()

s.Events = map[uint16]*EventMetadata{}
s.Events = map[uint32]*EventMetadata{}
for itr.Next() {
evt, err := newEventMetadataFromPublisherMetadata(itr, s.Metadata)
if err != nil {
s.log.Warnw("Failed to read event metadata from publisher. Continuing to next event.",
"error", err)
continue
}
s.Events[evt.EventID] = evt
s.Events[getEventCombinedID(evt.EventID, evt.Version)] = evt
}
return itr.Err()
}

func (s *PublisherMetadataStore) getEventMetadata(eventID uint16, eventDataFingerprint uint64, eventHandle EvtHandle) *EventMetadata {
func (s *PublisherMetadataStore) getEventMetadata(eventID uint16, version uint8, eventDataFingerprint uint64, eventHandle EvtHandle) *EventMetadata {
// Use a read lock to get a cached value.
s.mutex.RLock()
fingerprints, found := s.EventFingerprints[eventID]
combinedID := getEventCombinedID(eventID, version)
fingerprints, found := s.EventFingerprints[combinedID]
if found {
em, found := fingerprints[eventDataFingerprint]
if found {
Expand All @@ -212,10 +214,10 @@ func (s *PublisherMetadataStore) getEventMetadata(eventID uint16, eventDataFinge
s.mutex.Lock()
defer s.mutex.Unlock()

fingerprints, found = s.EventFingerprints[eventID]
fingerprints, found = s.EventFingerprints[combinedID]
if !found {
fingerprints = map[uint64]*EventMetadata{}
s.EventFingerprints[eventID] = fingerprints
s.EventFingerprints[combinedID] = fingerprints
}

em, found := fingerprints[eventDataFingerprint]
Expand All @@ -233,7 +235,7 @@ func (s *PublisherMetadataStore) getEventMetadata(eventID uint16, eventDataFinge
// metadata then we just associate the fingerprint with a pointer to the
// providers metadata for the event ID.

defaultEM := s.Events[eventID]
defaultEM := s.Events[combinedID]

// Use XML to get the parameters names.
em, err := newEventMetadataFromEventHandle(s.Metadata, eventHandle)
Expand All @@ -250,6 +252,15 @@ func (s *PublisherMetadataStore) getEventMetadata(eventID uint16, eventDataFinge
return defaultEM
}

// The first time we need to identify if the event has event data or
// user data from a handle, since this information is not available
// from the metadata or anywhere else. It is not ideal to update the defaultEM
// here but there is no way around it at the moment.
if em.EventData.IsUserData {
defaultEM.EventData.IsUserData = true
defaultEM.EventData.Name = em.EventData.Name
}

// Are the parameters the same as what the provider metadata listed?
// (This ignores the message values.)
if em.equal(defaultEM) {
Expand Down Expand Up @@ -324,13 +335,18 @@ func (s *PublisherMetadataStore) Close() error {
return nil
}

type EventDataParams struct {
IsUserData bool
Name xml.Name
Params []EventData
}

type EventMetadata struct {
EventID uint16 // Event ID.
Version uint8 // Event format version.
MsgStatic string // Used when the message has no parameters.
MsgTemplate *template.Template `json:"-"` // Template that expects an array of values as its data.
EventData []EventData // Names of parameters from XML template.
HasUserData bool // Event has a UserData section or not.
EventData EventDataParams // Names of parameters from XML template.
}

// newEventMetadataFromEventHandle collects metadata about an event type using
Expand All @@ -354,12 +370,13 @@ func newEventMetadataFromEventHandle(publisher *PublisherMetadata, eventHandle E
}
if len(event.EventData.Pairs) > 0 {
for _, pair := range event.EventData.Pairs {
em.EventData = append(em.EventData, EventData{Name: pair.Key})
em.EventData.Params = append(em.EventData.Params, EventData{Name: pair.Key})
}
} else {
em.HasUserData = true
em.EventData.IsUserData = true
em.EventData.Name = event.UserData.Name
for _, pair := range event.UserData.Pairs {
em.EventData = append(em.EventData, EventData{Name: pair.Key})
em.EventData.Params = append(em.EventData.Params, EventData{Name: pair.Key})
}
}

Expand Down Expand Up @@ -435,7 +452,7 @@ func (em *EventMetadata) initEventDataTemplate(itr *EventMetadataIterator) error
kv.Name = eventDataNameTransform.Replace(kv.Name)
}

em.EventData = tmpl.Data
em.EventData.Params = tmpl.Data
return nil
}

Expand Down Expand Up @@ -477,6 +494,10 @@ func (em *EventMetadata) setMessage(msg string) error {
return nil
}

func getEventCombinedID(eventID uint16, version uint8) uint32 {
return (uint32(eventID) << 16) | uint32(version)
}

// containsTemplatedValues traverses the template nodes to check if there are
// any dynamic values.
func containsTemplatedValues(tmpl *template.Template) bool {
Expand Down Expand Up @@ -513,7 +534,9 @@ func (em *EventMetadata) equal(other *EventMetadata) bool {

return em.EventID == other.EventID &&
em.Version == other.Version &&
eventDataNamesEqual(em.EventData, other.EventData)
em.EventData.IsUserData == other.EventData.IsUserData &&
em.EventData.Name == other.EventData.Name &&
eventDataNamesEqual(em.EventData.Params, other.EventData.Params)
}

type publisherMetadataCache struct {
Expand Down
9 changes: 7 additions & 2 deletions winlogbeat/sys/wineventlog/publisher_metadata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,8 @@ func NewMetadataKeyword(publisherMetadataHandle EvtHandle, arrayHandle EvtObject

type MetadataOpcode struct {
Name string
Mask uint32
Opcode uint16
Task uint16
MessageID uint32
Message string
}
Expand Down Expand Up @@ -292,11 +293,15 @@ func NewMetadataOpcode(publisherMetadataHandle EvtHandle, arrayHandle EvtObjectA
if err != nil {
return nil, err
}
// Mask high word contains the opcode value and the low word contains the task to which it belongs.
// If the low word is zero, the opcode is defined globally; otherwise, the opcode is task specific.
// Use the low word value to determine the task that defines the opcode.
valueMask := v.(uint32)

return &MetadataOpcode{
Name: name,
Mask: valueMask,
Opcode: uint16((valueMask >> 16) & 0xFFFF),
Task: uint16(valueMask & 0xFFFF),
MessageID: messageID,
Message: message,
}, nil
Expand Down
13 changes: 7 additions & 6 deletions winlogbeat/sys/wineventlog/renderer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ func (r *Renderer) Render(handle EvtHandle) (*winevent.Event, string, error) {
}

// Load cached event metadata or try to bootstrap it from the event's XML.
eventMeta := md.getEventMetadata(uint16(event.EventIdentifier.ID), fingerprint, handle)
eventMeta := md.getEventMetadata(uint16(event.EventIdentifier.ID), uint8(event.Version), fingerprint, handle)

// Associate key names with the event data values.
r.addEventData(eventMeta, eventData, event)
Expand Down Expand Up @@ -304,13 +304,13 @@ func (r *Renderer) addEventData(evtMeta *EventMetadata, values []interface{}, ev
r.log.Warnw("Event metadata not found.",
"provider", event.Provider.Name,
"event_id", event.EventIdentifier.ID)
} else if len(values) != len(evtMeta.EventData) {
} else if len(values) != len(evtMeta.EventData.Params) {
r.log.Warnw("The number of event data parameters doesn't match the number "+
"of parameters in the template.",
"provider", event.Provider.Name,
"event_id", event.EventIdentifier.ID,
"event_parameter_count", len(values),
"template_parameter_count", len(evtMeta.EventData),
"template_parameter_count", len(evtMeta.EventData.Params),
"template_version", evtMeta.Version,
"event_version", event.Version)
}
Expand All @@ -322,8 +322,8 @@ func (r *Renderer) addEventData(evtMeta *EventMetadata, values []interface{}, ev
// updated). If software was updated it could also be that this cached
// template is now stale.
paramName := func(idx int) string {
if evtMeta != nil && idx < len(evtMeta.EventData) {
return evtMeta.EventData[idx].Name
if evtMeta != nil && idx < len(evtMeta.EventData.Params) {
return evtMeta.EventData.Params[idx].Name
}
return "param" + strconv.Itoa(idx)
}
Expand All @@ -346,7 +346,8 @@ func (r *Renderer) addEventData(evtMeta *EventMetadata, values []interface{}, ev
}
}

if evtMeta != nil && evtMeta.HasUserData {
if evtMeta != nil && evtMeta.EventData.IsUserData {
event.UserData.Name = evtMeta.EventData.Name
event.UserData.Pairs = pairs
} else {
event.EventData.Pairs = pairs
Expand Down

0 comments on commit f6d5acc

Please sign in to comment.