support UUID type

build.sh

@@ -16,7 +16,7 @@ fi
 
 $DEP ensure $UPDATE
 mkdir -p build
-export ver="2.3.1"
+export ver="2.3.2"
 export version="v${ver}-$(date "+%Y%m%d")"
 env GOOS=darwin GOARCH=amd64 go build -ldflags "-X main.version=$version" -o build/keyhole-osx-x64 keyhole.go
 env GOOS=linux GOARCH=amd64 go build -ldflags "-X main.version=$version" -o build/keyhole-linux-x64 keyhole.go

examples/template.json

@@ -1,5 +1,7 @@
 {
-	"_id": "a1b2c3d4e5f6f7e8d9c0b6a8",
+	"_id": {
+        "$oid": "5ea73359754ad1b05b055f3d"
+    },
 	"email": "[email protected]",
 	"hostIP": "192.168.1.1",
 	"ssn": "599-63-1888",
@@ -8,8 +10,12 @@
 	"longString": "This is another string value. You can use any field names",
 	"number": 123,
 	"hex": "a1b2c3d4",
-	"objectId": "a1b2c3d4e5f6f7e8d9c0b6a8",
-	"lastUpdated": "2018-01-01T01:23:45Z",
+	"objectId": {
+        "$oid": "a1b2c3d4e5f6f7e8d9c0b6a8"
+    },
+	"lastUpdated": {
+        "$date": "2020-04-27T19:32:41.518Z"
+    },
 	"array1": [123, 456, 789],
 	"array2": [ "little", "cute", "girl" ],
 	"array3": [

mdb/mongo_cluster.go

@@ -191,7 +191,9 @@ func emptyBinData(firstDoc bson.M) bson.M {
 		}
 		t := reflect.TypeOf(v).String()
 		if t == "primitive.Binary" {
-			firstDoc[k] = primitive.Binary{}
+			if v.(primitive.Binary).Subtype != 4 { // empty data when it's not UUID
+				firstDoc[k] = primitive.Binary{Subtype: v.(primitive.Binary).Subtype}
+			}
 		} else {
 			// fmt.Println(v, t)
 		}

sim/schema.go

@@ -23,18 +23,23 @@ func GetSchema(c *mongo.Collection, verbose bool) (string, error) {
 	if buf, err = bson.MarshalExtJSON(doc, false, false); err != nil {
 		return "", err
 	}
-	json.Unmarshal(buf, &doc)
-	str := gox.Stringify(doc, "", "  ")
 	if verbose == true {
-		return str, err
+		json.Unmarshal(buf, &doc)
+		return gox.Stringify(doc, "", "  "), err
 	}
-	re := regexp.MustCompile(`{\s+"\$oid":\s?("[a-fA-F0-9]{24}")\s+}`)
+	re := regexp.MustCompile(`({"\$binary":{.*"subType":"04"}})`)
+	str := re.ReplaceAllString(string(buf), `"$$uuid"`)
+	json.Unmarshal([]byte(str), &doc)
+	str = gox.Stringify(doc, "", "  ")
+	re = regexp.MustCompile(`{\s+"\$oid":\s?("[a-fA-F0-9]{24}")\s+}`)
 	str = re.ReplaceAllString(str, "ObjectId($1)")
 	re = regexp.MustCompile(`{\s+"\$date":\s?("\S+")\s+}`)
 	str = re.ReplaceAllString(str, "ISODate($1)")
 	re = regexp.MustCompile(`{\s+"\$numberDecimal":\s?("\S+")\s+}`)
 	str = re.ReplaceAllString(str, "NumberDecimal($1)")
 	re = regexp.MustCompile(`{\s+"\$numberDouble":\s?("NaN")\s+}`)
 	str = re.ReplaceAllString(str, "NaN")
+	re = regexp.MustCompile(`"\$uuid"`)
+	str = re.ReplaceAllString(str, gox.GetRandomUUIDString())
 	return str, err
 }

sim/seed.go

@@ -403,7 +403,6 @@ func (f *Feeder) seedFromTemplate(client *mongo.Client) error {
 	}
 	log.Println("Seed data to collection", collName, "using", f.conns, "connections")
 	c := client.Database(f.database).Collection(collName)
-	c.InsertOne(ctx, sdoc)
 	if f.isDrop {
 		c.Drop(ctx)
 	}

sim/util/random_docs.go

@@ -3,7 +3,6 @@
 package util
 
 import (
-	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -14,10 +13,20 @@ import (
 	"strings"
 	"time"
 
+	"github.com/simagix/gox"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 )
 
+const metaEmail = "$email"
+const metaIP = "$ip"
+const metaSSN = "$ssn"
+const metaTEL = "$tel"
+const metaDate = "$date"
+const metaOID = "$oId"
+const numberDecimal = "$numberDecimal"
+const uuid = "$uuid"
+
 // GetDocByTemplate returns a bson.M document
 func GetDocByTemplate(filename string, meta bool) (bson.M, error) {
 	var buf []byte
@@ -26,19 +35,36 @@ func GetDocByTemplate(filename string, meta bool) (bson.M, error) {
 	if buf, err = ioutil.ReadFile(filename); err != nil {
 		return nil, err
 	}
+	v := bson.M{}
+	if err = json.Unmarshal(buf, &v); err != nil {
+		return nil, err
+	}
+	if buf, err = bson.MarshalExtJSON(v, false, false); err != nil {
+		return nil, err
+	}
 	return GetRandomizedDoc(buf, meta)
 }
 
 // GetRandomizedDoc returns a randomized doc from byte string
 func GetRandomizedDoc(buf []byte, meta bool) (bson.M, error) {
 	var err error
-	var str string
-	re := regexp.MustCompile(`ObjectId\(\S+\)`)
-	str = re.ReplaceAllString(string(buf), "\"$$oId\"")
+	str := string(buf)
+	str = strings.ReplaceAll(str, "NaN", "0.0")
+	re := regexp.MustCompile(`{"\$oid":"[a-fA-F0-9]{24}"}`)
+	str = re.ReplaceAllString(str, `"$$oId"`)
+	re = regexp.MustCompile(`{"\$binary":{"base64":"\S+","subType":"04"}}`)
+	str = re.ReplaceAllString(str, `"$$uuid"`)
+	re = regexp.MustCompile(`{"\$numberDecimal":"[-+]?[0-9]*\.?[0-9]*(E-)?[0-9]*"}`)
+	str = re.ReplaceAllString(str, `"$$numberDecimal"`)
+	// backward compatible
+	re = regexp.MustCompile(`ObjectId\(\S+\)`)
+	str = re.ReplaceAllString(str, "\"$$oId\"")
 	re = regexp.MustCompile(`NumberDecimal\("?([-+]?[0-9]*\.?[0-9]*)"?\)`)
 	str = re.ReplaceAllString(str, "{\"$$numberDecimal\": $1}")
-	re = regexp.MustCompile(`NumberLong\("?([-+]?[0-9]*\.?[0-9]*)"?\)`)
+	re = regexp.MustCompile(`numberDouble\("?([-+]?[0-9]*\.?[0-9]*)"?\)`)
+	str = re.ReplaceAllString(str, "{\"$$numberDouble\": $1}")
 	str = re.ReplaceAllString(str, "{\"$$numberLong\": $1}")
+	re = regexp.MustCompile(`NumberLong\("?([-+]?[0-9]*\.?[0-9]*)"?\)`)
 	re = regexp.MustCompile(`NumberInt\("?([-+]?[0-9]*\.?[0-9]*)"?\)`)
 	str = re.ReplaceAllString(str, "{\"$$numberInt\": $1}")
 	re = regexp.MustCompile(`ISODate\(\S+\)`)
@@ -88,6 +114,12 @@ func RandomizeDocument(doc *map[string]interface{}, f interface{}, meta bool) {
 				} else if value.(string) == metaOID || (len(value.(string)) == 24 && isHexString(value.(string))) {
 					(*doc)[key] = primitive.NewObjectID()
 					continue
+				} else if value.(string) == numberDecimal {
+					(*doc)[key] = primitive.NewDecimal128(rand.Uint64(), 0)
+					continue
+				} else if value.(string) == uuid {
+					(*doc)[key] = primitive.Binary{Subtype: 4, Data: []byte(gox.GetRandomDigitString(16))}
+					continue
 				}
 			}
 			(*doc)[key] = getMagicString(value.(string), meta)
@@ -121,6 +153,12 @@ func getArrayOfRandomDocs(obj []interface{}, doc *[]interface{}, meta bool) {
 				} else if value.(string) == metaOID || (len(value.(string)) == 24 && isHexString(value.(string))) {
 					(*doc)[key] = primitive.NewObjectID()
 					continue
+				} else if value.(string) == numberDecimal {
+					(*doc)[key] = primitive.NewDecimal128(rand.Uint64(), 0)
+					continue
+				} else if value.(string) == uuid {
+					(*doc)[key] = primitive.Binary{Subtype: 4, Data: []byte(gox.GetRandomDigitString(16))}
+					continue
 				}
 			}
 			(*doc)[key] = getMagicString(value.(string), meta)
@@ -138,13 +176,6 @@ func getArrayOfRandomDocs(obj []interface{}, doc *[]interface{}, meta bool) {
 	}
 }
 
-const metaEmail = "$email"
-const metaIP = "$ip"
-const metaSSN = "$ssn"
-const metaTEL = "$tel"
-const metaDate = "$date"
-const metaOID = "$oId"
-
 // Returns randomized string.  if meta is true, it intends to avoid future regex
 // actions by replacing the values with $email, $ip, and $date.
 func getMagicString(str string, meta bool) string {
@@ -161,6 +192,10 @@ func getMagicString(str string, meta bool) string {
 			return metaDate
 		} else if str == metaOID || (len(str) == 24 && isHexString(str)) {
 			return metaOID
+		} else if isHexString(str) {
+			return gox.GetRandomHexString(len(str))
+		} else if strings.HasPrefix(str, "$number") || str == uuid {
+			return str
 		}
 	} else if str == metaIP || isIP(str) {
 		return getIP()
@@ -170,8 +205,8 @@ func getMagicString(str string, meta bool) string {
 		// 	return getSSN()
 		// } else if str == metaTEL || isPhoneNumber(str) {
 		// 	return getPhoneNumber()
-		// } else if isHexString(str) {
-		// 	return getHexString(len(str))
+	} else if isHexString(str) {
+		return gox.GetRandomHexString(len(str))
 	} else if strings.HasPrefix(str, "$") { // could be a variable
 		return str
 	}
@@ -242,14 +277,6 @@ func isHexString(str string) bool {
 	return matched.MatchString(str)
 }
 
-func getHexString(n int) string {
-	bytes := make([]byte, n/2)
-	if _, err := rand.Read(bytes); err != nil {
-		return ""
-	}
-	return hex.EncodeToString(bytes)
-}
-
 func isDateString(str string) bool {
 	var matched = regexp.MustCompile(`^\d{4}\-(0?[1-9]|1[012])\-(0?[1-9]|[12][0-9]|3[01])T.*$`)
 	return matched.MatchString(str)

sim/util/random_docs_test.go

@@ -159,13 +159,6 @@ func TestIsHexString(t *testing.T) {
 	}
 }
 
-func TestGetHexString(t *testing.T) {
-	hex := getHexString(8)
-	if isHexString(hex) == false {
-		t.Fatal(hex)
-	}
-}
-
 func TestIsDateString(t *testing.T) {
 	var dstr = "2018-10-15T12:00:00Z"
 	if isDateString(dstr) == false {