Refactored message parsing code. (#3)

Also ignore useless xmlns attributes.

cmd/extract_windows.go

@@ -7,14 +7,13 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"path/filepath"
-	"regexp"
 	"strings"
 
 	_ "github.com/mattn/go-sqlite3"
 	"golang.org/x/sys/windows/registry"
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
 	"www.velocidex.com/golang/binparsergen/reader"
+	"www.velocidex.com/golang/evtx"
 	pe "www.velocidex.com/golang/go-pe"
 )
 
@@ -67,7 +66,7 @@ func walkProvider(cb func(provider string, message_table string) error) error {
 				continue
 			}
 
-			for _, message_file := range expandLocations(message_files) {
+			for _, message_file := range evtx.ExpandLocations(message_files) {
 				err = cb(provider_name, message_file)
 				if err != nil {
 					fmt.Printf("While processing %v (%v): %v\n",
@@ -82,85 +81,6 @@ func walkProvider(cb func(provider string, message_table string) error) error {
 	return nil
 }
 
-var system_root_re = regexp.MustCompile("(?i)%?SystemRoot%?")
-var windir_re = regexp.MustCompile("(?i)%windir%")
-var programfiles_re = regexp.MustCompile("(?i)%programfiles%")
-var system32_re = regexp.MustCompile(`(?i)\\System32\\`)
-
-// Produce a list of possible locations the message file may be. We
-// process all of them because sometimes event messages are split
-// across multiple dlls. For example, a generic message table may
-// exist in C:\Windows\System32\XXX.dll but a localized message table
-// also exists in C:\Windows\System32\en-us\XXX.dll.mui
-func expandLocations(message_file string) []string {
-
-	// Expand environment variables in paths.
-	replace_env_vars := func(paths []string) []string {
-		system_root := os.Getenv("SystemRoot")
-		windir := os.Getenv("WinDir")
-		programfiles := os.Getenv("programfiles")
-		programfiles_x86 := os.Getenv("ProgramFiles(x86)")
-
-		result := []string{}
-		for _, path := range paths {
-			path = system_root_re.ReplaceAllLiteralString(
-				path, system_root)
-
-			path = windir_re.ReplaceAllLiteralString(
-				path, windir)
-
-			if programfiles_re.FindString(path) != "" {
-				result = append(result,
-					programfiles_re.ReplaceAllLiteralString(
-						path, programfiles))
-				result = append(result,
-					programfiles_re.ReplaceAllLiteralString(
-						path, programfiles_x86))
-			} else {
-				result = append(result, path)
-			}
-		}
-		return result
-	}
-
-	// When paths refer to system32 the message table may instead
-	// reside in the 32 bit version.
-	split_system32 := func(paths []string) []string {
-		result := []string{}
-		for _, path := range paths {
-			result = append(result, path)
-
-			// Sometimes messages are found in the 32 bit folders.
-			if system32_re.FindString(path) != "" {
-				result = append(result, system32_re.ReplaceAllLiteralString(
-					path, "\\SysWow64\\"))
-			}
-		}
-
-		return result
-	}
-
-	include_muis := func(paths []string) []string {
-		result := []string{}
-		for _, path := range paths {
-			result = append(result, path)
-
-			// Sometimes messages are found in the MUI
-			// files include those as well.
-			dll_name := filepath.Base(path)
-			dir_name := filepath.Dir(path)
-
-			result = append(result, filepath.Join(
-				dir_name, "en-US", dll_name+".mui"))
-		}
-		return result
-	}
-
-	// Message file values may be separated by ;
-	return include_muis(split_system32(replace_env_vars(
-		strings.Split(message_file, ";"))))
-}
-
 func makeDatabase(filename string) (*sql.DB, error) {
 	database, err := sql.Open("sqlite3", filename)
 	if err != nil {

cmd/parse.go

@@ -62,7 +62,7 @@ func (self *parsingContext) Parse() {
 		for _, i := range records {
 			event_map, ok := i.Event.(*ordereddict.Dict)
 			if ok {
-				event, ok := GetMap(event_map, "Event")
+				event, ok := ordereddict.GetMap(event_map, "Event")
 				if !ok {
 					continue
 				}
@@ -77,13 +77,18 @@ func (self *parsingContext) Parse() {
 }
 
 func (self *parsingContext) maybeExpandMessage(event_map *ordereddict.Dict) {
+	// If not message database is loaded just ignore it.
+	if self.query == nil {
+		return
+	}
+
 	// Event.System.Provider.Name
-	name, ok := GetString(event_map, "System.Provider.Name")
+	name, ok := ordereddict.GetString(event_map, "System.Provider.Name")
 	if !ok {
 		return
 	}
 
-	event_id, ok := GetInt(event_map, "System.EventID.Value")
+	event_id, ok := ordereddict.GetInt(event_map, "System.EventID.Value")
 	if !ok {
 		return
 	}
@@ -97,7 +102,7 @@ func (self *parsingContext) maybeExpandMessage(event_map *ordereddict.Dict) {
 		var message string
 		err = rows.Scan(&message)
 		if err == nil {
-			event_map.Set("Message", self.expandMessage(event_map, message))
+			event_map.Set("Message", evtx.ExpandMessage(event_map, message))
 			return
 		}
 	}
@@ -108,9 +113,9 @@ var expansion_re = regexp.MustCompile(`\%[0-9n]+`)
 func (self *parsingContext) expandMessage(event_map *ordereddict.Dict, message string) string {
 	expansions := []string{}
 
-	data, pres := GetMap(event_map, "UserData.EventXML")
+	data, pres := ordereddict.GetMap(event_map, "UserData.EventXML")
 	if !pres {
-		data_any, pres := GetAny(event_map, "EventData.Data")
+		data_any, pres := ordereddict.GetAny(event_map, "EventData.Data")
 		if !pres {
 			return message
 		}
@@ -152,112 +157,6 @@ func (self *parsingContext) expandMessage(event_map *ordereddict.Dict, message s
 	})
 }
 
-func GetString(event_map *ordereddict.Dict, members string) (string, bool) {
-	var value interface{} = event_map
-	var pres bool
-
-	for _, member := range strings.Split(members, ".") {
-		if event_map == nil {
-			return "", false
-		}
-
-		value, pres = event_map.Get(member)
-		if !pres {
-			return "", false
-		}
-		event_map, pres = value.(*ordereddict.Dict)
-	}
-
-	value_str, ok := value.(string)
-	if ok {
-		return value_str, true
-	}
-
-	return "", false
-}
-
-func GetMap(event_map *ordereddict.Dict, members string) (*ordereddict.Dict, bool) {
-	var value interface{} = event_map
-	var pres bool
-
-	for _, member := range strings.Split(members, ".") {
-		if event_map == nil {
-			return nil, false
-		}
-
-		value, pres = event_map.Get(member)
-		if !pres {
-			return nil, false
-		}
-		event_map, pres = value.(*ordereddict.Dict)
-		if !pres {
-			return nil, false
-		}
-	}
-
-	return event_map, true
-}
-
-func GetAny(event_map *ordereddict.Dict, members string) (interface{}, bool) {
-	var value interface{} = event_map
-	var pres bool
-
-	for _, member := range strings.Split(members, ".") {
-		if event_map == nil {
-			return nil, false
-		}
-
-		value, pres = event_map.Get(member)
-		if !pres {
-			return nil, false
-		}
-		event_map, pres = value.(*ordereddict.Dict)
-	}
-
-	return value, true
-}
-
-func GetInt(event_map *ordereddict.Dict, members string) (int, bool) {
-	var value interface{} = event_map
-	var pres bool
-
-	for _, member := range strings.Split(members, ".") {
-		if event_map == nil {
-			return 0, false
-		}
-
-		value, pres = event_map.Get(member)
-		if !pres {
-			return 0, false
-		}
-		event_map, pres = value.(*ordereddict.Dict)
-	}
-
-	switch t := value.(type) {
-	case int:
-		return t, true
-	case uint8:
-		return int(t), true
-	case uint16:
-		return int(t), true
-	case uint32:
-		return int(t), true
-	case uint64:
-		return int(t), true
-	case int8:
-		return int(t), true
-	case int16:
-		return int(t), true
-	case int32:
-		return int(t), true
-	case int64:
-		return int(t), true
-
-	}
-
-	return 0, false
-}
-
 func doParse() {
 	ctx := NewParsingContext()
 	ctx.Parse()

evtx.go

@@ -232,7 +232,10 @@ func (self *TemplateNode) SetLiteral(key string, literal interface{}) {
 		self.NestedDict = ordereddict.NewDict() //make(map[string]*TemplateNode)
 	}
 
-	self.NestedDict.Set(key, &TemplateNode{Literal: literal})
+	// Ignore useless xmlsn attributes.
+	if key != "xmlns" {
+		self.NestedDict.Set(key, &TemplateNode{Literal: literal})
+	}
 }
 
 func (self *TemplateNode) SetExpansion(key string, id, type_id uint32) {

go.mod

@@ -1,17 +1,19 @@
 module www.velocidex.com/golang/evtx
 
 require (
-	github.com/Velocidex/ordereddict v0.0.0-20191103011020-3b5a5f6957d4
+	github.com/Velocidex/ordereddict v0.0.0-20191106020901-97c468e5e403
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751
 	github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d
 	github.com/davecgh/go-spew v1.1.1
 	github.com/mattn/go-sqlite3 v1.11.0
 	github.com/pkg/errors v0.8.1
-	golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c // indirect
+	golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
-	www.velocidex.com/golang/go-pe v0.1.1-0.20191103232346-ac12e8190bb6 // indirect
+	www.velocidex.com/golang/binparsergen v0.1.0
+	www.velocidex.com/golang/go-pe v0.1.1-0.20191103232346-ac12e8190bb6
 )
 
 // replace www.velocidex.com/golang/go-pe => /home/mic/projects/go-pe/
+//replace github.com/Velocidex/ordereddict => /home/mic/projects/ordereddict
 
 go 1.13

go.sum

@@ -1,5 +1,7 @@
 github.com/Velocidex/ordereddict v0.0.0-20191103011020-3b5a5f6957d4 h1:0sJvmFhuTT8W/qOlka2y3L1FrbRcP9ZxXMKejTGMCno=
 github.com/Velocidex/ordereddict v0.0.0-20191103011020-3b5a5f6957d4/go.mod h1:pxJpvN5ISMtDwrdIdqnJ3ZrjIngCw+WT6gfNil6Zjvo=
+github.com/Velocidex/ordereddict v0.0.0-20191106020901-97c468e5e403 h1:IyE0Jbkgftu+no3dqzrQOKsISicXDk5ob6IYdXaxlS8=
+github.com/Velocidex/ordereddict v0.0.0-20191106020901-97c468e5e403/go.mod h1:pxJpvN5ISMtDwrdIdqnJ3ZrjIngCw+WT6gfNil6Zjvo=
 github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=

messages.go

@@ -0,0 +1,65 @@
+package evtx
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+
+	"github.com/Velocidex/ordereddict"
+)
+
+var (
+	expansion_re    = regexp.MustCompile(`\%[0-9ntr]+`)
+	system_root_re  = regexp.MustCompile("(?i)%?SystemRoot%?")
+	windir_re       = regexp.MustCompile("(?i)%windir%")
+	programfiles_re = regexp.MustCompile("(?i)%programfiles%")
+	system32_re     = regexp.MustCompile(`(?i)\\System32\\`)
+)
+
+func flatten(dict *ordereddict.Dict) []interface{} {
+	result := []interface{}{}
+
+	for _, k := range dict.Keys() {
+		value, _ := dict.Get(k)
+
+		switch t := value.(type) {
+		case *ordereddict.Dict:
+			result = append(result, flatten(t)...)
+		default:
+			result = append(result, value)
+		}
+	}
+
+	return result
+}
+
+func ExpandMessage(event_map *ordereddict.Dict, message string) string {
+	data, pres := ordereddict.GetMap(event_map, "UserData")
+	if !pres {
+		data, pres = ordereddict.GetMap(event_map, "EventData")
+		if !pres {
+			return message
+		}
+	}
+	expansions := flatten(data)
+
+	return expansion_re.ReplaceAllStringFunc(message, func(match string) string {
+		switch match {
+		case "%n":
+			return "\n"
+		case "%r":
+			return ""
+		case "%t":
+			return "\t"
+		}
+
+		number, _ := strconv.Atoi(match[1:])
+
+		// Regex expansions start at 1
+		number -= 1
+		if number >= 0 && number < len(expansions) {
+			return fmt.Sprintf("%v", expansions[number])
+		}
+		return match
+	})
+}