From 5363ce0a886eaa14a926fc9b7842e8e706ca42be Mon Sep 17 00:00:00 2001 From: amrutasali Date: Mon, 23 Sep 2019 18:50:33 +0000 Subject: [PATCH 01/25] Initial support for table transformer --- src/translib/transformer/xlate_from_db.go | 110 +++++++++++++++++++++- 1 file changed, 109 insertions(+), 1 deletion(-) diff --git a/src/translib/transformer/xlate_from_db.go b/src/translib/transformer/xlate_from_db.go index f763990231..a861c2fd06 100644 --- a/src/translib/transformer/xlate_from_db.go +++ b/src/translib/transformer/xlate_from_db.go @@ -78,6 +78,17 @@ func validateHandlerFunc(inParams XfmrParams) (bool) { return ret[0].Interface().(bool) } +func xfmrTblHandlerFunc(xfmrTblNm string, inParams XfmrParams) []string { + //xpath, _ := RemoveXPATHPredicates(inParams.uri) + ret, err := XlateFuncCall(xfmrTblNm, inParams) + if err != nil { + return []string{} + } + return ret[0].Interface().([]string) + +} + + func DbValToInt(dbFldVal string, base int, size int, isUint bool) (interface{}, error) { var res interface{} var err error @@ -224,7 +235,71 @@ func tableNameAndKeyFromDbMapGet(dbDataMap map[string]map[string]db.Value) (stri return tableName, tableKey, nil } +/*db num ??, key ??*/ +func fillDbDataMapForTbl(uri string, xpath string, tblName string, tblKey string, cdb db.DBNum, dbs [db.MaxDB]*db.DB, resDbDataMap *map[string]map[string]db.Value) error { + var err error + var dbresult = make(map[db.DBNum]map[string]map[string]db.Value) + dbresult[cdb] = make(map[string]map[string]db.Value) + dbFormat := KeySpec{} + dbFormat.Ts.Name = tblName + dbFormat.dbNum = cdb + /* TODO - handle key + dbFormat.Key.Comp = append(dbFormat.Key.Comp, tblKey) */ + err = TraverseDb(dbs, dbFormat, &dbresult, nil) + if err != nil { + log.Errorf("TraverseDb() failure for tbl(DB num) %v(%v) for xpath %v", tblName, cdb, xpath) + return err + } + if _, ok := dbresult[cdb]; !ok { + logStr := fmt.Sprintf("TraverseDb() did not populate Db data for tbl(DB num) %v(%v) for xpath %v", tblName, cdb, xpath) + err = fmt.Errorf("%v", logStr) + return err + } + for tblNm := range(dbresult[cdb]) { + for tblky := range(dbresult[cdb][tblNm]) { + for fld, val := range(dbresult[cdb][tblNm]) { + (*resDbDataMap)[tblNm][tblky].Field[fld] = val + } + } + } + return err + +} + func yangListDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, resultMap map[string]interface{}, tbl string, tblKey string, cdb db.DBNum, validate bool) error { + + tblXfmr := false + var tblList []string + if tbl == "" && xSpecMap[xpath].xfmrTbl != nil { + lxfmrTblName := *xSpecMap[xpath].xfmrTbl + if len(lxfmrTblName) > 0 { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, "", dbDataMap, nil) + // call table transformer + tblList = xfmrTblHandlerFunc(inParams) + if len(tblList) == 0 { + log.Warningf("Table transformer did not return any table for xpath %v", xpath) + return nil + } + for _, tbl := range(tblList) { + var resDbDataMap = make(map[string]map[string]db.Value) + err := fillDbDataMapForTbl(uri, xpath, tbl, dbs, cdb, &resDbDataMap) + if err != nil { + log.Wraningf("Could not retrieve DB data for table %v for xpath %v", tbl, xpath) + continue + } + mapCopy((*dbDataMap), resDbDataMap) + } + tblXfmr = true + } else { + log.Warningf("Empty table transformer name for xpath %v", xpath) + return nil + } + } + if tbl != "" && !tblXfmr { + tblList[0] = tbl + } + + for _, tbl = range(tblList) { tblData, ok := (*dbDataMap)[cdb][tbl] if ok { @@ -258,6 +333,7 @@ func yangListDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, x log.Infof("Empty slice for (\"%v\").\r\n", uri) } } + }// end of tblList for return nil } @@ -356,6 +432,37 @@ func yangDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath } else { log.Infof("Empty container(\"%v\").\r\n", chldUri) } + } else if xSpecMap[chldXpath].xfmrTbl != nil { + lxfmrTblName := *xSpecMap[chldXpath].tableName + if len(lxfmrTblName) > 0 { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, "", dbDataMap, nil) + tblList = xfmrTblHandlerFunc(inParams) // should be a list containing a single table ??? + if len(tblList) == 0 { + log.Warningf("Table transformer did not return any table for xpath %v", chldXpath) + continue + } + // TODO - add check of single tbale in list ??? + if len(tblList) > 1 { + log.Warningf("Table transformer returned returned more than one table for container %v", chldXpath) + continue + } + for _, tbl := range(tblList) { + var resDbDataMap = make(map[string]map[string]db.Value) + //err := fillDbDataMapForTbl(chldUri, chldXpath, tbl, parentTblKey, dbs, cdb, &resDbDataMap) + err := fillDbDataMapForTbl(chldUri, chldXpath, tbl, tblKey, cdb, &resDbDataMap) + if err != nil { + log.Wraningf("Could not retrieve DB data for table %v for xpath %v", tbl, xpath) + continue + } + mapCopy((*dbDataMap), resDbDataMap) + } + tblXfmr = true + + } else { + log.Warningf("Table transformer name is empty for xpath %v", chldXpath) + } + + } else { cmap := make(map[string]interface{}) err = yangDataFill(dbs, ygRoot, chldUri, chldXpath, dbDataMap, cmap, tbl, tblKey, cdb, isValid) @@ -377,10 +484,11 @@ func yangDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath } } else { ynode, ok := xSpecMap[chldXpath] + lTblName := "" if ok && ynode.tableName != nil { lTblName := *ynode.tableName - yangListDataFill(dbs, ygRoot, chldUri, chldXpath, dbDataMap, resultMap, lTblName, "", cdb, isValid) } + yangListDataFill(dbs, ygRoot, chldUri, chldXpath, dbDataMap, resultMap, lTblName, "", cdb, isValid) } } else { return err From 4c5a210751d2b1d5105cc5bb3941221e41cbb9c5 Mon Sep 17 00:00:00 2001 From: amrutasali Date: Mon, 23 Sep 2019 23:32:36 +0000 Subject: [PATCH 02/25] Fix issues in initial commit for table transformer --- src/translib/transformer/xlate_from_db.go | 54 ++++++++++++----------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/src/translib/transformer/xlate_from_db.go b/src/translib/transformer/xlate_from_db.go index a861c2fd06..4dc7c7fcfa 100644 --- a/src/translib/transformer/xlate_from_db.go +++ b/src/translib/transformer/xlate_from_db.go @@ -244,7 +244,9 @@ func fillDbDataMapForTbl(uri string, xpath string, tblName string, tblKey string dbFormat.Ts.Name = tblName dbFormat.dbNum = cdb /* TODO - handle key - dbFormat.Key.Comp = append(dbFormat.Key.Comp, tblKey) */ + if tblKey != "" { + dbFormat.Key.Comp = append(dbFormat.Key.Comp, tblKey) + } */ err = TraverseDb(dbs, dbFormat, &dbresult, nil) if err != nil { log.Errorf("TraverseDb() failure for tbl(DB num) %v(%v) for xpath %v", tblName, cdb, xpath) @@ -257,7 +259,7 @@ func fillDbDataMapForTbl(uri string, xpath string, tblName string, tblKey string } for tblNm := range(dbresult[cdb]) { for tblky := range(dbresult[cdb][tblNm]) { - for fld, val := range(dbresult[cdb][tblNm]) { + for fld, val := range(dbresult[cdb][tblNm][tblky].Field) { (*resDbDataMap)[tblNm][tblky].Field[fld] = val } } @@ -273,21 +275,21 @@ func yangListDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, x if tbl == "" && xSpecMap[xpath].xfmrTbl != nil { lxfmrTblName := *xSpecMap[xpath].xfmrTbl if len(lxfmrTblName) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, "", dbDataMap, nil) + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, "", dbDataMap, nil) // call table transformer - tblList = xfmrTblHandlerFunc(inParams) + tblList = xfmrTblHandlerFunc(lxfmrTblName, inParams) if len(tblList) == 0 { log.Warningf("Table transformer did not return any table for xpath %v", xpath) return nil } for _, tbl := range(tblList) { var resDbDataMap = make(map[string]map[string]db.Value) - err := fillDbDataMapForTbl(uri, xpath, tbl, dbs, cdb, &resDbDataMap) + err := fillDbDataMapForTbl(uri, xpath, tbl, "", cdb, dbs, &resDbDataMap) if err != nil { - log.Wraningf("Could not retrieve DB data for table %v for xpath %v", tbl, xpath) + log.Warningf("Could not retrieve DB data for table %v for xpath %v", tbl, xpath) continue } - mapCopy((*dbDataMap), resDbDataMap) + mapCopy((*dbDataMap)[cdb], resDbDataMap) } tblXfmr = true } else { @@ -436,42 +438,42 @@ func yangDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath lxfmrTblName := *xSpecMap[chldXpath].tableName if len(lxfmrTblName) > 0 { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, "", dbDataMap, nil) - tblList = xfmrTblHandlerFunc(inParams) // should be a list containing a single table ??? + tblList := xfmrTblHandlerFunc(lxfmrTblName, inParams) // should be a list containing a single table ??? if len(tblList) == 0 { log.Warningf("Table transformer did not return any table for xpath %v", chldXpath) continue } - // TODO - add check of single tbale in list ??? + // TODO - add check of single table in list ??? if len(tblList) > 1 { log.Warningf("Table transformer returned returned more than one table for container %v", chldXpath) continue } for _, tbl := range(tblList) { var resDbDataMap = make(map[string]map[string]db.Value) - //err := fillDbDataMapForTbl(chldUri, chldXpath, tbl, parentTblKey, dbs, cdb, &resDbDataMap) - err := fillDbDataMapForTbl(chldUri, chldXpath, tbl, tblKey, cdb, &resDbDataMap) + /*TODO - check if there is a key transformer, if yes then get the key and supply else use incoming tblKey(key of parent)*/ + //err := fillDbDataMapForTbl(chldUri, chldXpath, tbl, parentTblKey, cdb, dbs, &resDbDataMap) + err := fillDbDataMapForTbl(chldUri, chldXpath, tbl, tblKey, cdb, dbs, &resDbDataMap) if err != nil { - log.Wraningf("Could not retrieve DB data for table %v for xpath %v", tbl, xpath) + log.Warningf("Could not retrieve DB data for table %v for xpath %v", tbl, xpath) continue } - mapCopy((*dbDataMap), resDbDataMap) + mapCopy((*dbDataMap)[cdb], resDbDataMap) } - tblXfmr = true + //tblXfmr = true } else { log.Warningf("Table transformer name is empty for xpath %v", chldXpath) + continue } - - - } else { - cmap := make(map[string]interface{}) - err = yangDataFill(dbs, ygRoot, chldUri, chldXpath, dbDataMap, cmap, tbl, tblKey, cdb, isValid) - if len(cmap) > 0 { - resultMap[cname] = cmap - } else { - log.Infof("Empty container(\"%v\").\r\n", chldUri) - } - } + } //else { + cmap := make(map[string]interface{}) + err = yangDataFill(dbs, ygRoot, chldUri, chldXpath, dbDataMap, cmap, tbl, tblKey, cdb, isValid) + if len(cmap) > 0 { + resultMap[cname] = cmap + } else { + log.Infof("Empty container(\"%v\").\r\n", chldUri) + } + //} } else if chldYangType == "list" { cdb = xSpecMap[chldXpath].dbIndex if len(xSpecMap[chldXpath].xfmrFunc) > 0 { @@ -486,7 +488,7 @@ func yangDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath ynode, ok := xSpecMap[chldXpath] lTblName := "" if ok && ynode.tableName != nil { - lTblName := *ynode.tableName + lTblName = *ynode.tableName } yangListDataFill(dbs, ygRoot, chldUri, chldXpath, dbDataMap, resultMap, lTblName, "", cdb, isValid) } From b7f5146c0f888d3703f82c1fc7797042ef34e720 Mon Sep 17 00:00:00 2001 From: s-mari Date: Tue, 24 Sep 2019 17:11:47 -0700 Subject: [PATCH 03/25] support for table transformer for crud and get operation --- src/translib/transformer/xlate_from_db.go | 660 ++++++++++------------ src/translib/transformer/xlate_to_db.go | 53 +- src/translib/transformer/xlate_utils.go | 113 ++-- src/translib/transformer/xspec.go | 349 ++++++------ 4 files changed, 586 insertions(+), 589 deletions(-) diff --git a/src/translib/transformer/xlate_from_db.go b/src/translib/transformer/xlate_from_db.go index 4dc7c7fcfa..7125c1caea 100644 --- a/src/translib/transformer/xlate_from_db.go +++ b/src/translib/transformer/xlate_from_db.go @@ -78,14 +78,12 @@ func validateHandlerFunc(inParams XfmrParams) (bool) { return ret[0].Interface().(bool) } -func xfmrTblHandlerFunc(xfmrTblNm string, inParams XfmrParams) []string { - //xpath, _ := RemoveXPATHPredicates(inParams.uri) - ret, err := XlateFuncCall(xfmrTblNm, inParams) +func xfmrTblHandlerFunc(xfmrTblFunc string, inParams XfmrParams) []string { + ret, err := XlateFuncCall(xfmrTblFunc, inParams) if err != nil { return []string{} } return ret[0].Interface().([]string) - } @@ -173,54 +171,54 @@ func processLfLstDbToYang(fieldXpath string, dbFldVal string) []interface{} { /* Traverse db map and create json for cvl yang */ func directDbToYangJsonCreate(dbDataMap map[string]map[string]db.Value, jsonData string, resultMap map[string]interface{}) error { var err error - for tblName, tblData := range dbDataMap { - var mapSlice []typeMapOfInterface - for keyStr, dbFldValData := range tblData { - curMap := make(map[string]interface{}) - for field, value := range dbFldValData.Field { - resField := field - if strings.HasSuffix(field, "@") { - fldVals := strings.Split(field, "@") - resField = fldVals[0] - } - fieldXpath := tblName + "/" + resField - xDbSpecMapEntry, ok := xDbSpecMap[fieldXpath] - if !ok { - log.Warningf("No entry found in xDbSpecMap for xpath %v", fieldXpath) - continue - } - if xDbSpecMapEntry.dbEntry == nil { - log.Warningf("Yang entry is nil in xDbSpecMap for xpath %v", fieldXpath) - continue - } - yangType := yangTypeGet(xDbSpecMapEntry.dbEntry) - if yangType == "leaf-list" { - /* this should never happen but just adding for safetty */ - if !strings.HasSuffix(field, "@") { - log.Warningf("Leaf-list in Sonic yang should also be a leaf-list in DB, its not for xpath %v", fieldXpath) - continue - } - resLst := processLfLstDbToYang(fieldXpath, value) - curMap[resField] = resLst - } else { /* yangType is leaf - there are only 2 types of yang terminal node leaf and leaf-list */ - yngTerminalNdDtType := xDbSpecMapEntry.dbEntry.Type.Kind - resVal, err := DbToYangType(yngTerminalNdDtType, fieldXpath, value) - if err != nil { - log.Warningf("Failure in converting Db value type to yang type for xpath", fieldXpath) - } else { - curMap[resField] = resVal - } - } - } //end of for - yangKeys := yangKeyFromEntryGet(xDbSpecMap[tblName].dbEntry) - sonicKeyDataAdd(yangKeys, keyStr, curMap) - if curMap != nil { - mapSlice = append(mapSlice, curMap) - } - } - resultMap[tblName] = mapSlice - } - return err + for tblName, tblData := range dbDataMap { + var mapSlice []typeMapOfInterface + for keyStr, dbFldValData := range tblData { + curMap := make(map[string]interface{}) + for field, value := range dbFldValData.Field { + resField := field + if strings.HasSuffix(field, "@") { + fldVals := strings.Split(field, "@") + resField = fldVals[0] + } + fieldXpath := tblName + "/" + resField + xDbSpecMapEntry, ok := xDbSpecMap[fieldXpath] + if !ok { + log.Warningf("No entry found in xDbSpecMap for xpath %v", fieldXpath) + continue + } + if xDbSpecMapEntry.dbEntry == nil { + log.Warningf("Yang entry is nil in xDbSpecMap for xpath %v", fieldXpath) + continue + } + yangType := yangTypeGet(xDbSpecMapEntry.dbEntry) + if yangType == "leaf-list" { + /* this should never happen but just adding for safetty */ + if !strings.HasSuffix(field, "@") { + log.Warningf("Leaf-list in Sonic yang should also be a leaf-list in DB, its not for xpath %v", fieldXpath) + continue + } + resLst := processLfLstDbToYang(fieldXpath, value) + curMap[resField] = resLst + } else { /* yangType is leaf - there are only 2 types of yang terminal node leaf and leaf-list */ + yngTerminalNdDtType := xDbSpecMapEntry.dbEntry.Type.Kind + resVal, err := DbToYangType(yngTerminalNdDtType, fieldXpath, value) + if err != nil { + log.Warningf("Failure in converting Db value type to yang type for xpath", fieldXpath) + } else { + curMap[resField] = resVal + } + } + } //end of for + yangKeys := yangKeyFromEntryGet(xDbSpecMap[tblName].dbEntry) + sonicKeyDataAdd(yangKeys, keyStr, curMap) + if curMap != nil { + mapSlice = append(mapSlice, curMap) + } + } + resultMap[tblName] = mapSlice + } + return err } func tableNameAndKeyFromDbMapGet(dbDataMap map[string]map[string]db.Value) (string, string, error) { @@ -235,338 +233,302 @@ func tableNameAndKeyFromDbMapGet(dbDataMap map[string]map[string]db.Value) (stri return tableName, tableKey, nil } -/*db num ??, key ??*/ -func fillDbDataMapForTbl(uri string, xpath string, tblName string, tblKey string, cdb db.DBNum, dbs [db.MaxDB]*db.DB, resDbDataMap *map[string]map[string]db.Value) error { - var err error - var dbresult = make(map[db.DBNum]map[string]map[string]db.Value) - dbresult[cdb] = make(map[string]map[string]db.Value) - dbFormat := KeySpec{} - dbFormat.Ts.Name = tblName - dbFormat.dbNum = cdb - /* TODO - handle key - if tblKey != "" { - dbFormat.Key.Comp = append(dbFormat.Key.Comp, tblKey) - } */ - err = TraverseDb(dbs, dbFormat, &dbresult, nil) - if err != nil { - log.Errorf("TraverseDb() failure for tbl(DB num) %v(%v) for xpath %v", tblName, cdb, xpath) - return err - } - if _, ok := dbresult[cdb]; !ok { - logStr := fmt.Sprintf("TraverseDb() did not populate Db data for tbl(DB num) %v(%v) for xpath %v", tblName, cdb, xpath) - err = fmt.Errorf("%v", logStr) - return err - } - for tblNm := range(dbresult[cdb]) { - for tblky := range(dbresult[cdb][tblNm]) { - for fld, val := range(dbresult[cdb][tblNm][tblky].Field) { - (*resDbDataMap)[tblNm][tblky].Field[fld] = val - } +func fillDbDataMapForTbl(uri string, xpath string, tblName string, tblKey string, cdb db.DBNum, dbs [db.MaxDB]*db.DB) (map[db.DBNum]map[string]map[string]db.Value, error) { + var err error + dbresult := make(map[db.DBNum]map[string]map[string]db.Value) + dbresult[cdb] = make(map[string]map[string]db.Value) + dbFormat := KeySpec{} + dbFormat.Ts.Name = tblName + dbFormat.dbNum = cdb + /* TODO - handle key + if tblKey != "" { + dbFormat.Key.Comp = append(dbFormat.Key.Comp, tblKey) + } */ + err = TraverseDb(dbs, dbFormat, &dbresult, nil) + if err != nil { + log.Errorf("TraverseDb() failure for tbl(DB num) %v(%v) for xpath %v", tblName, cdb, xpath) + return nil, err } - } - return err + if _, ok := dbresult[cdb]; !ok { + logStr := fmt.Sprintf("TraverseDb() did not populate Db data for tbl(DB num) %v(%v) for xpath %v", tblName, cdb, xpath) + err = fmt.Errorf("%v", logStr) + return nil, err + } + return dbresult, err } -func yangListDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, resultMap map[string]interface{}, tbl string, tblKey string, cdb db.DBNum, validate bool) error { +// Assumption: All tables are from the same DB +func dbDataFromTblXfmrGet(tbl string, inParams XfmrParams, dbDataMap *map[db.DBNum]map[string]map[string]db.Value) error { + xpath, _ := RemoveXPATHPredicates(inParams.uri) + curDbDataMap, err := fillDbDataMapForTbl(inParams.uri, xpath, tbl, "", inParams.curDb, inParams.dbs) + if err == nil { + mapCopy((*dbDataMap)[inParams.curDb], curDbDataMap[inParams.curDb]) + } + return nil +} - tblXfmr := false - var tblList []string - if tbl == "" && xSpecMap[xpath].xfmrTbl != nil { - lxfmrTblName := *xSpecMap[xpath].xfmrTbl - if len(lxfmrTblName) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, "", dbDataMap, nil) - // call table transformer - tblList = xfmrTblHandlerFunc(lxfmrTblName, inParams) - if len(tblList) == 0 { - log.Warningf("Table transformer did not return any table for xpath %v", xpath) - return nil - } - for _, tbl := range(tblList) { - var resDbDataMap = make(map[string]map[string]db.Value) - err := fillDbDataMapForTbl(uri, xpath, tbl, "", cdb, dbs, &resDbDataMap) - if err != nil { - log.Warningf("Could not retrieve DB data for table %v for xpath %v", tbl, xpath) - continue +func yangListDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, resultMap map[string]interface{}, tbl string, tblKey string, cdb db.DBNum, validate bool) error { + var tblList []string + tblXfmr := false + + if tbl == "" && xSpecMap[xpath].xfmrTbl != nil { + xfmrTblFunc := *xSpecMap[xpath].xfmrTbl + if len(xfmrTblFunc) > 0 { + tblXfmr = true + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, "", dbDataMap, nil) + tblList = xfmrTblHandlerFunc(xfmrTblFunc, inParams) + if len(tblList) != 0 { + for _, curTbl := range tblList { + dbDataFromTblXfmrGet(curTbl, inParams, dbDataMap) + } + } } - mapCopy((*dbDataMap)[cdb], resDbDataMap) - } - tblXfmr = true - } else { - log.Warningf("Empty table transformer name for xpath %v", xpath) - return nil + } else if tbl != "" && !tblXfmr { + tblList = append(tblList, tbl) } - } - if tbl != "" && !tblXfmr { - tblList[0] = tbl - } - for _, tbl = range(tblList) { - tblData, ok := (*dbDataMap)[cdb][tbl] - - if ok { - var mapSlice []typeMapOfInterface - for dbKey, _ := range tblData { - curMap := make(map[string]interface{}) - curKeyMap, curUri, _ := dbKeyToYangDataConvert(uri, xpath, dbKey) - if len(xSpecMap[xpath].xfmrFunc) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, curUri, GET, "", dbDataMap, nil) - cmap, _ := xfmrHandlerFunc(inParams) - if len(cmap) > 0 { - mapSlice = append(mapSlice, curMap) - } else { - log.Infof("Empty container returned from overloaded transformer for(\"%v\")", curUri) - } - } else { - _, keyFromCurUri, _ := xpathKeyExtract(dbs[cdb], ygRoot, GET, curUri) - if dbKey == keyFromCurUri { - for k, kv := range curKeyMap { - curMap[k] = kv - } - curXpath, _ := RemoveXPATHPredicates(curUri) - yangDataFill(dbs, ygRoot, curUri, curXpath, dbDataMap, curMap, tbl, dbKey, cdb, validate) - mapSlice = append(mapSlice, curMap) - } - } - } - if len(mapSlice) > 0 { - resultMap[xSpecMap[xpath].yangEntry.Name] = mapSlice - } else { - log.Infof("Empty slice for (\"%v\").\r\n", uri) - } - } - }// end of tblList for - return nil + for _, tbl = range(tblList) { + tblData, ok := (*dbDataMap)[cdb][tbl] + + if ok { + var mapSlice []typeMapOfInterface + for dbKey, _ := range tblData { + curMap := make(map[string]interface{}) + curKeyMap, curUri, _ := dbKeyToYangDataConvert(uri, xpath, dbKey) + if len(xSpecMap[xpath].xfmrFunc) > 0 { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, curUri, GET, "", dbDataMap, nil) + cmap, _ := xfmrHandlerFunc(inParams) + if len(cmap) > 0 { + mapSlice = append(mapSlice, curMap) + } else { + log.Infof("Empty container returned from overloaded transformer for(\"%v\")", curUri) + } + } else { + _, keyFromCurUri, _ := xpathKeyExtract(dbs[cdb], ygRoot, GET, curUri) + if dbKey == keyFromCurUri { + for k, kv := range curKeyMap { + curMap[k] = kv + } + curXpath, _ := RemoveXPATHPredicates(curUri) + yangDataFill(dbs, ygRoot, curUri, curXpath, dbDataMap, curMap, tbl, dbKey, cdb, validate) + mapSlice = append(mapSlice, curMap) + } + } + } + if len(mapSlice) > 0 { + resultMap[xSpecMap[xpath].yangEntry.Name] = mapSlice + } else { + log.Infof("Empty slice for (\"%v\").\r\n", uri) + } + } + }// end of tblList for + return nil } func terminalNodeProcess(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, tbl string, tblKey string) (map[string]interface{}, error) { - log.Infof("Received xpath - %v, uri - %v, dbDataMap - %v, table - %v, table key - %v", xpath, uri, (*dbDataMap), tbl, tblKey) - var err error + log.Infof("Received xpath - %v, uri - %v, dbDataMap - %v, table - %v, table key - %v", xpath, uri, (*dbDataMap), tbl, tblKey) + var err error resFldValMap := make(map[string]interface{}) - if xSpecMap[xpath].yangEntry == nil { + if xSpecMap[xpath].yangEntry == nil { logStr := fmt.Sprintf("No yang entry found for xpath %v.", xpath) - err = fmt.Errorf("%v", logStr) - return resFldValMap, err - } + err = fmt.Errorf("%v", logStr) + return resFldValMap, err + } - cdb := xSpecMap[xpath].dbIndex - if len(xSpecMap[xpath].xfmrFunc) > 0 { - _, key, _ := xpathKeyExtract(dbs[cdb], ygRoot, GET, uri) - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, key, dbDataMap, nil) + cdb := xSpecMap[xpath].dbIndex + if len(xSpecMap[xpath].xfmrFunc) > 0 { + _, key, _ := xpathKeyExtract(dbs[cdb], ygRoot, GET, uri) + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, key, dbDataMap, nil) fldValMap, err := leafXfmrHandlerFunc(inParams) - if err != nil { + if err != nil { logStr := fmt.Sprintf("%Failed to get data from overloaded function for %v -v.", uri, err) err = fmt.Errorf("%v", logStr) return resFldValMap, err - } - for lf, val := range fldValMap { - resFldValMap[lf] = val - } - } else { - dbFldName := xSpecMap[xpath].fieldName - /* if there is no transformer extension/annotation then it means leaf-list in yang is also leaflist in db */ - if len(dbFldName) > 0 && !xSpecMap[xpath].isKey { - yangType := yangTypeGet(xSpecMap[xpath].yangEntry) - if yangType == "leaf-list" { - dbFldName += "@" - val, ok := (*dbDataMap)[cdb][tbl][tblKey].Field[dbFldName] - if ok { - resLst := processLfLstDbToYang(xpath, val) - resFldValMap[xSpecMap[xpath].yangEntry.Name] = resLst - } - } else { - val, ok := (*dbDataMap)[cdb][tbl][tblKey].Field[dbFldName] - if ok { - yngTerminalNdDtType := xSpecMap[xpath].yangEntry.Type.Kind - resVal, err := DbToYangType(yngTerminalNdDtType, xpath, val) - if err != nil { - log.Error("Failure in converting Db value type to yang type for field", xpath) - } else { - resFldValMap[xSpecMap[xpath].yangEntry.Name] = resVal - } - } - } - } - } + } + for lf, val := range fldValMap { + resFldValMap[lf] = val + } + } else { + dbFldName := xSpecMap[xpath].fieldName + /* if there is no transformer extension/annotation then it means leaf-list in yang is also leaflist in db */ + if len(dbFldName) > 0 && !xSpecMap[xpath].isKey { + yangType := yangTypeGet(xSpecMap[xpath].yangEntry) + if yangType == "leaf-list" { + dbFldName += "@" + val, ok := (*dbDataMap)[cdb][tbl][tblKey].Field[dbFldName] + if ok { + resLst := processLfLstDbToYang(xpath, val) + resFldValMap[xSpecMap[xpath].yangEntry.Name] = resLst + } + } else { + val, ok := (*dbDataMap)[cdb][tbl][tblKey].Field[dbFldName] + if ok { + yngTerminalNdDtType := xSpecMap[xpath].yangEntry.Type.Kind + resVal, err := DbToYangType(yngTerminalNdDtType, xpath, val) + if err != nil { + log.Error("Failure in converting Db value type to yang type for field", xpath) + } else { + resFldValMap[xSpecMap[xpath].yangEntry.Name] = resVal + } + } + } + } + } return resFldValMap, err } func yangDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, resultMap map[string]interface{}, tbl string, tblKey string, cdb db.DBNum, validate bool) error { - var err error - isValid := validate - yangNode, ok := xSpecMap[xpath] - - if ok && yangNode.yangEntry != nil { - for yangChldName := range yangNode.yangEntry.Dir { - chldXpath := xpath+"/"+yangChldName - chldUri := uri+"/"+yangChldName - if xSpecMap[chldXpath] != nil && xSpecMap[chldXpath].yangEntry != nil { - _, key, _ := xpathKeyExtract(dbs[cdb], ygRoot, GET, chldUri) - if len(xSpecMap[chldXpath].validateFunc) > 0 && !validate { - // TODO - handle non CONFIG-DB - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, key, dbDataMap, nil) - res := validateHandlerFunc(inParams) - if res != true { - continue - } else { - isValid = res - } - } - chldYangType := yangTypeGet(xSpecMap[chldXpath].yangEntry) - cdb = xSpecMap[chldXpath].dbIndex - if chldYangType == "leaf" || chldYangType == "leaf-list" { - fldValMap, err := terminalNodeProcess(dbs, ygRoot, chldUri, chldXpath, dbDataMap, tbl, tblKey) - if err != nil { - log.Infof("Failed to get data(\"%v\").", chldUri) - } - for lf, val := range fldValMap { - resultMap[lf] = val - } - - } else if chldYangType == "container" { - cname := xSpecMap[chldXpath].yangEntry.Name - if len(xSpecMap[chldXpath].xfmrFunc) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, "", dbDataMap, nil) - cmap, _ := xfmrHandlerFunc(inParams) - if len(cmap) > 0 { - resultMap[cname] = cmap - } else { - log.Infof("Empty container(\"%v\").\r\n", chldUri) - } - } else if xSpecMap[chldXpath].xfmrTbl != nil { - lxfmrTblName := *xSpecMap[chldXpath].tableName - if len(lxfmrTblName) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, "", dbDataMap, nil) - tblList := xfmrTblHandlerFunc(lxfmrTblName, inParams) // should be a list containing a single table ??? - if len(tblList) == 0 { - log.Warningf("Table transformer did not return any table for xpath %v", chldXpath) - continue - } - // TODO - add check of single table in list ??? - if len(tblList) > 1 { - log.Warningf("Table transformer returned returned more than one table for container %v", chldXpath) - continue - } - for _, tbl := range(tblList) { - var resDbDataMap = make(map[string]map[string]db.Value) - /*TODO - check if there is a key transformer, if yes then get the key and supply else use incoming tblKey(key of parent)*/ - //err := fillDbDataMapForTbl(chldUri, chldXpath, tbl, parentTblKey, cdb, dbs, &resDbDataMap) - err := fillDbDataMapForTbl(chldUri, chldXpath, tbl, tblKey, cdb, dbs, &resDbDataMap) - if err != nil { - log.Warningf("Could not retrieve DB data for table %v for xpath %v", tbl, xpath) - continue + var err error + isValid := validate + yangNode, ok := xSpecMap[xpath] + + if ok && yangNode.yangEntry != nil { + for yangChldName := range yangNode.yangEntry.Dir { + chldXpath := xpath+"/"+yangChldName + chldUri := uri+"/"+yangChldName + if xSpecMap[chldXpath] != nil && xSpecMap[chldXpath].yangEntry != nil { + _, key, _ := xpathKeyExtract(dbs[cdb], ygRoot, GET, chldUri) + if len(xSpecMap[chldXpath].validateFunc) > 0 && !validate { + // TODO - handle non CONFIG-DB + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, key, dbDataMap, nil) + res := validateHandlerFunc(inParams) + if res != true { + continue + } else { + isValid = res + } + } + chldYangType := yangTypeGet(xSpecMap[chldXpath].yangEntry) + cdb = xSpecMap[chldXpath].dbIndex + if chldYangType == "leaf" || chldYangType == "leaf-list" { + fldValMap, err := terminalNodeProcess(dbs, ygRoot, chldUri, chldXpath, dbDataMap, tbl, tblKey) + if err != nil { + log.Infof("Failed to get data(\"%v\").", chldUri) + } + for lf, val := range fldValMap { + resultMap[lf] = val + } + } else if chldYangType == "container" { + cname := xSpecMap[chldXpath].yangEntry.Name + if len(xSpecMap[chldXpath].xfmrFunc) > 0 { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, "", dbDataMap, nil) + cmap, _ := xfmrHandlerFunc(inParams) + if len(cmap) > 0 { + resultMap[cname] = cmap + } else { + log.Infof("Empty container(\"%v\").\r\n", chldUri) + } + } else if xSpecMap[chldXpath].xfmrTbl != nil { + xfmrTblFunc := *xSpecMap[chldXpath].xfmrTbl + if len(xfmrTblFunc) > 0 { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, "", dbDataMap, nil) + tblList := xfmrTblHandlerFunc(xfmrTblFunc, inParams) + if len(tblList) > 1 { + log.Warningf("Table transformer returned more than one table for container %v", chldXpath) + } + dbDataFromTblXfmrGet(tblList[0], inParams, dbDataMap) + } + } + cmap := make(map[string]interface{}) + err = yangDataFill(dbs, ygRoot, chldUri, chldXpath, dbDataMap, cmap, tbl, tblKey, cdb, isValid) + if len(cmap) > 0 { + resultMap[cname] = cmap + } else { + log.Infof("Empty container(\"%v\").\r\n", chldUri) + } + } else if chldYangType == "list" { + cdb = xSpecMap[chldXpath].dbIndex + if len(xSpecMap[chldXpath].xfmrFunc) > 0 { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, "", dbDataMap, nil) + cmap, _ := xfmrHandlerFunc(inParams) + if len(cmap) > 0 { + resultMap = cmap + } else { + log.Infof("Empty list(\"%v\").\r\n", chldUri) + } + } else { + ynode, ok := xSpecMap[chldXpath] + lTblName := "" + if ok && ynode.tableName != nil { + lTblName = *ynode.tableName + } + yangListDataFill(dbs, ygRoot, chldUri, chldXpath, dbDataMap, resultMap, lTblName, "", cdb, isValid) + } + } else { + return err } - mapCopy((*dbDataMap)[cdb], resDbDataMap) - } - //tblXfmr = true - - } else { - log.Warningf("Table transformer name is empty for xpath %v", chldXpath) - continue } - } //else { - cmap := make(map[string]interface{}) - err = yangDataFill(dbs, ygRoot, chldUri, chldXpath, dbDataMap, cmap, tbl, tblKey, cdb, isValid) - if len(cmap) > 0 { - resultMap[cname] = cmap - } else { - log.Infof("Empty container(\"%v\").\r\n", chldUri) } - //} - } else if chldYangType == "list" { - cdb = xSpecMap[chldXpath].dbIndex - if len(xSpecMap[chldXpath].xfmrFunc) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, "", dbDataMap, nil) - cmap, _ := xfmrHandlerFunc(inParams) - if len(cmap) > 0 { - resultMap = cmap - } else { - log.Infof("Empty list(\"%v\").\r\n", chldUri) - } - } else { - ynode, ok := xSpecMap[chldXpath] - lTblName := "" - if ok && ynode.tableName != nil { - lTblName = *ynode.tableName - } - yangListDataFill(dbs, ygRoot, chldUri, chldXpath, dbDataMap, resultMap, lTblName, "", cdb, isValid) - } - } else { - return err - } - } - } - } - return err + } + return err } /* Traverse linear db-map data and add to nested json data */ func dbDataToYangJsonCreate(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, cdb db.DBNum) (string, error) { - jsonData := "" - resultMap := make(map[string]interface{}) + jsonData := "" + resultMap := make(map[string]interface{}) if isCvlYang(uri) { directDbToYangJsonCreate((*dbDataMap)[cdb], jsonData, resultMap) } else { - var d *db.DB - reqXpath, keyName, tableName := xpathKeyExtract(d, ygRoot, GET, uri) - yangNode, ok := xSpecMap[reqXpath] - - if ok { - yangType := yangTypeGet(yangNode.yangEntry) - if yangType == "leaf" || yangType == "leaf-list" { - //fldName := xSpecMap[reqXpath].fieldName - yangName := xSpecMap[reqXpath].yangEntry.Name - tbl, key, _ := tableNameAndKeyFromDbMapGet((*dbDataMap)[cdb]) - validateHandlerFlag := false - if len(xSpecMap[reqXpath].validateFunc) > 0 { - // TODO - handle non CONFIG-DB - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, key, dbDataMap, nil) - res := validateHandlerFunc(inParams) - if !res { - validateHandlerFlag = true - resultMap[yangName] = "" - } - } - if !validateHandlerFlag { - fldValMap, err := terminalNodeProcess(dbs, ygRoot, uri, reqXpath, dbDataMap, tbl, key) - //err := terminalNodeProcess(dbs, ygRoot, uri, reqXpath, dbDataMap, tbl, key) - if err != nil { - log.Infof("Empty terminal node (\"%v\").", uri) - } - resultMap = fldValMap + var d *db.DB + reqXpath, keyName, tableName := xpathKeyExtract(d, ygRoot, GET, uri) + yangNode, ok := xSpecMap[reqXpath] + if ok { + yangType := yangTypeGet(yangNode.yangEntry) + if yangType == "leaf" || yangType == "leaf-list" { + //fldName := xSpecMap[reqXpath].fieldName + yangName := xSpecMap[reqXpath].yangEntry.Name + tbl, key, _ := tableNameAndKeyFromDbMapGet((*dbDataMap)[cdb]) + validateHandlerFlag := false + if len(xSpecMap[reqXpath].validateFunc) > 0 { + // TODO - handle non CONFIG-DB + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, key, dbDataMap, nil) + res := validateHandlerFunc(inParams) + if !res { + validateHandlerFlag = true + resultMap[yangName] = "" + } + } + if !validateHandlerFlag { + fldValMap, err := terminalNodeProcess(dbs, ygRoot, uri, reqXpath, dbDataMap, tbl, key) + //err := terminalNodeProcess(dbs, ygRoot, uri, reqXpath, dbDataMap, tbl, key) + if err != nil { + log.Infof("Empty terminal node (\"%v\").", uri) + } + resultMap = fldValMap + } + } else if yangType == "container" { + cname := xSpecMap[reqXpath].yangEntry.Name + cmap := make(map[string]interface{}) + if len(xSpecMap[reqXpath].xfmrFunc) > 0 { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, "", dbDataMap, nil) + cmap, _ = xfmrHandlerFunc(inParams) + if len(cmap) > 0 { + resultMap[cname] = cmap + } else { + err := yangDataFill(dbs, ygRoot, uri, reqXpath, dbDataMap, resultMap, tableName, keyName, cdb, false) + if err != nil { + log.Infof("Empty container(\"%v\").\r\n", uri) + } + } + } else { + err := yangDataFill(dbs, ygRoot, uri, reqXpath, dbDataMap, resultMap, tableName, keyName, cdb, false) + if err != nil { + log.Infof("Empty container(\"%v\").\r\n", uri) + } + } + } else { + yangDataFill(dbs, ygRoot, uri, reqXpath, dbDataMap, resultMap, tableName, keyName, cdb, false) + } } + } - } else if yangType == "container" { - cname := xSpecMap[reqXpath].yangEntry.Name - cmap := make(map[string]interface{}) - if len(xSpecMap[reqXpath].xfmrFunc) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, "", dbDataMap, nil) - cmap, _ = xfmrHandlerFunc(inParams) - if len(cmap) > 0 { - resultMap[cname] = cmap - } else { - err := yangDataFill(dbs, ygRoot, uri, reqXpath, dbDataMap, resultMap, tableName, keyName, cdb, false) - if err != nil { - log.Infof("Empty container(\"%v\").\r\n", uri) - } - } - } else { - err := yangDataFill(dbs, ygRoot, uri, reqXpath, dbDataMap, resultMap, tableName, keyName, cdb, false) - if err != nil { - log.Infof("Empty container(\"%v\").\r\n", uri) - } - } - } else { - yangDataFill(dbs, ygRoot, uri, reqXpath, dbDataMap, resultMap, tableName, keyName, cdb, false) - } - } - } - - jsonMapData, _ := json.Marshal(resultMap) - jsonData = fmt.Sprintf("%v", string(jsonMapData)) - jsonDataPrint(jsonData) - return jsonData, nil + jsonMapData, _ := json.Marshal(resultMap) + jsonData = fmt.Sprintf("%v", string(jsonMapData)) + jsonDataPrint(jsonData) + return jsonData, nil } func jsonDataPrint(data string) { diff --git a/src/translib/transformer/xlate_to_db.go b/src/translib/transformer/xlate_to_db.go index f839e96ddd..4afa6cf0e1 100644 --- a/src/translib/transformer/xlate_to_db.go +++ b/src/translib/transformer/xlate_to_db.go @@ -46,8 +46,21 @@ func dataToDBMapAdd(tableName string, dbKey string, result map[string]map[string return } +func tblNameFromTblXfmrGet(xfmrTblFunc string, inParams XfmrParams) (string, error){ + tblList := xfmrTblHandlerFunc(xfmrTblFunc, inParams) + if len(tblList) != 1 { + logStr := fmt.Sprintf("Invalid return value(%v) from table transformer for (%v)", tblList, inParams.uri) + log.Error(logStr) + err := errors.New(logStr) + return "", err + } + return tblList[0], nil +} + /* Fill the redis-db map with data */ func mapFillData(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, dbKey string, result map[string]map[string]db.Value, xpathPrefix string, name string, value interface{}) error { + var dbs [db.MaxDB]*db.DB + var err error xpath := xpathPrefix + "/" + name xpathInfo := xSpecMap[xpath] log.Infof("name: \"%v\", xpathPrefix(\"%v\").", name, xpathPrefix) @@ -57,7 +70,7 @@ func mapFillData(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, dbKey st return errors.New("Invalid URI") } - if xpathInfo.tableName == nil { + if xpathInfo.tableName == nil && xpathInfo.xfmrTbl == nil{ log.Errorf("Table for yang-path(\"%v\") not found.", xpath) return errors.New("Invalid table name") } @@ -93,7 +106,6 @@ func mapFillData(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, dbKey st if nErr != nil { return nErr } - var dbs [db.MaxDB]*db.DB inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, uri, oper, "", nil, node[0].Data) ret, err := XlateFuncCall(yangToDbXfmrFunc(xSpecMap[xpath].xfmrFunc), inParams) if err != nil { @@ -134,15 +146,27 @@ func mapFillData(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, dbKey st log.Infof("leaf-list value after conversion to DB format %v : %v", fieldName, valueStr) } else { // xpath is a leaf - valueStr = fmt.Sprintf("%v", value) - if strings.Contains(valueStr, ":") { - valueStr = strings.Split(valueStr, ":")[1] - } + valueStr = fmt.Sprintf("%v", value) + if strings.Contains(valueStr, ":") { + valueStr = strings.Split(valueStr, ":")[1] + } } - dataToDBMapAdd(*xpathInfo.tableName, dbKey, result, fieldName, valueStr) - log.Infof("TblName: \"%v\", key: \"%v\", field: \"%v\", valueStr: \"%v\".", - *xpathInfo.tableName, dbKey, fieldName, valueStr) + tableName := "" + if xpathInfo.xfmrTbl != nil { + inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, uri, oper, "", nil, "") + // expecting only one table name from tbl-xfmr + tableName, err = tblNameFromTblXfmrGet(*xSpecMap[xpath].xfmrTbl, inParams) + if err != nil { + return err + } + } else { + tableName = *xpathInfo.tableName + } + + dataToDBMapAdd(tableName, dbKey, result, fieldName, valueStr) + log.Infof("TblName: \"%v\", key: \"%v\", field: \"%v\", valueStr: \"%v\".", tableName, dbKey, + fieldName, valueStr) return nil } @@ -305,7 +329,7 @@ func dbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string, jsonDat moduleNm := "/" + strings.Split(path, "/")[1] log.Infof("Module name for path %s is %s", path, moduleNm) if _, ok := xSpecMap[moduleNm]; ok { - if xSpecMap[moduleNm].yangDataType == "container" { + if xSpecMap[moduleNm].yangDataType == "container" && len(xSpecMap[moduleNm].xfmrPost) > 0 { log.Info("Invoke post transformer: ", xSpecMap[moduleNm].xfmrPost) dbDataMap := make(map[db.DBNum]map[string]map[string]db.Value) dbDataMap[db.ConfigDB] = result @@ -372,7 +396,7 @@ func yangReqToDbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, if (typeOfValue == reflect.Map || typeOfValue == reflect.Slice) && xSpecMap[xpath].yangDataType != "leaf-list" { if xSpecMap[xpath] != nil && len(xSpecMap[xpath].xfmrFunc) > 0 { /* subtree transformer present */ - inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curUri, oper, "", nil, nil) + inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curUri, oper, "", nil, nil) ret, err := XlateFuncCall(yangToDbXfmrFunc(xSpecMap[xpath].xfmrFunc), inParams) if err != nil { return nil @@ -440,7 +464,7 @@ func xpathKeyExtract(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string) (st if ok { if len(xSpecMap[yangXpath].xfmrKey) > 0 { xfmrFuncName := yangToDbXfmrFunc(xSpecMap[yangXpath].xfmrKey) - inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curPathWithKey, oper, "", nil, nil) + inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curPathWithKey, oper, "", nil, nil) ret, err := XlateFuncCall(xfmrFuncName, inParams) if err != nil { return "", "", "" @@ -461,7 +485,10 @@ func xpathKeyExtract(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string) (st tblPtr := xSpecMap[pfxPath].tableName if tblPtr != nil { tableName = *tblPtr - } + } else if xSpecMap[pfxPath].xfmrTbl != nil { + inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curPathWithKey, oper, "", nil, nil) + tableName, _ = tblNameFromTblXfmrGet(*xSpecMap[pfxPath].xfmrTbl, inParams) + } return pfxPath, keyStr, tableName } diff --git a/src/translib/transformer/xlate_utils.go b/src/translib/transformer/xlate_utils.go index 849f34e5f0..42e280ab5b 100644 --- a/src/translib/transformer/xlate_utils.go +++ b/src/translib/transformer/xlate_utils.go @@ -89,71 +89,76 @@ func yangTypeGet(entry *yang.Entry) string { } func dbKeyToYangDataConvert(uri string, xpath string, dbKey string) (map[string]interface{}, string, error) { - var err error - if len(uri) == 0 && len(xpath) == 0 && len(dbKey) == 0 { - err = fmt.Errorf("Insufficient input") - return nil, "", err - } + var err error + if len(uri) == 0 && len(xpath) == 0 && len(dbKey) == 0 { + err = fmt.Errorf("Insufficient input") + return nil, "", err + } - if _, ok := xSpecMap[xpath]; ok { - if xSpecMap[xpath].yangEntry == nil { - err = fmt.Errorf("Yang Entry not available for xpath ", xpath) - return nil, "", nil + if _, ok := xSpecMap[xpath]; ok { + if xSpecMap[xpath].yangEntry == nil { + err = fmt.Errorf("Yang Entry not available for xpath ", xpath) + return nil, "", nil + } } - } - var kLvlValList []string - keyDataList := strings.Split(dbKey, "|") - keyNameList := yangKeyFromEntryGet(xSpecMap[xpath].yangEntry) - id := xSpecMap[xpath].keyLevel - uriWithKey := fmt.Sprintf("%v", xpath) + var kLvlValList []string + keyDataList := strings.Split(dbKey, "|") + keyNameList := yangKeyFromEntryGet(xSpecMap[xpath].yangEntry) + id := xSpecMap[xpath].keyLevel + uriWithKey := fmt.Sprintf("%v", xpath) - /* if uri contins key, use it else use xpath */ - if strings.Contains(uri, "[") { - uriWithKey = fmt.Sprintf("%v", uri) - } + /* if uri contins key, use it else use xpath */ + if strings.Contains(uri, "[") { + uriWithKey = fmt.Sprintf("%v", uri) + } - if len(xSpecMap[xpath].xfmrKey) > 0 { - var dbs [db.MaxDB]*db.DB - inParams := formXfmrInputRequest(nil, dbs, db.MaxDB, nil, uri, GET, dbKey, nil, nil) - ret, err := XlateFuncCall(dbToYangXfmrFunc(xSpecMap[xpath].xfmrKey), inParams) - if err != nil { - return nil, "", err - } - rmap := ret[0].Interface().(map[string]interface{}) - for k, v := range rmap { - uriWithKey += fmt.Sprintf("[%v=%v]", k, v) - } - return rmap, uriWithKey, nil - } - kLvlValList = append(kLvlValList, keyDataList[id]) + if len(xSpecMap[xpath].xfmrKey) > 0 { + var dbs [db.MaxDB]*db.DB + inParams := formXfmrInputRequest(nil, dbs, db.MaxDB, nil, uri, GET, dbKey, nil, nil) + ret, err := XlateFuncCall(dbToYangXfmrFunc(xSpecMap[xpath].xfmrKey), inParams) + if err != nil { + return nil, "", err + } + rmap := ret[0].Interface().(map[string]interface{}) + for k, v := range rmap { + uriWithKey += fmt.Sprintf("[%v=%v]", k, v) + } + return rmap, uriWithKey, nil + } - if len(keyNameList) > 1 { - kLvlValList = strings.Split(keyDataList[id], "_") - } + if len(keyDataList) == 0 || len(keyNameList) == 0 { + return nil, "", nil + } - /* TODO: Need to add leaf-ref related code in here and remove this code*/ - kvalExceedFlag := false - chgId := -1 - if len(keyNameList) < len(kLvlValList) { - kvalExceedFlag = true - chgId = len(keyNameList) - 1 - } + kLvlValList = append(kLvlValList, keyDataList[id]) - rmap := make(map[string]interface{}) - for i, kname := range keyNameList { - kval := kLvlValList[i] + if len(keyNameList) > 1 { + kLvlValList = strings.Split(keyDataList[id], "_") + } - /* TODO: Need to add leaf-ref related code in here and remove this code*/ - if kvalExceedFlag && (i == chgId) { - kval = strings.Join(kLvlValList[chgId:], "_") - } + /* TODO: Need to add leaf-ref related code in here and remove this code*/ + kvalExceedFlag := false + chgId := -1 + if len(keyNameList) < len(kLvlValList) { + kvalExceedFlag = true + chgId = len(keyNameList) - 1 + } - uriWithKey += fmt.Sprintf("[%v=%v]", kname, kval) - rmap[kname] = kval - } + rmap := make(map[string]interface{}) + for i, kname := range keyNameList { + kval := kLvlValList[i] + + /* TODO: Need to add leaf-ref related code in here and remove this code*/ + if kvalExceedFlag && (i == chgId) { + kval = strings.Join(kLvlValList[chgId:], "_") + } + + uriWithKey += fmt.Sprintf("[%v=%v]", kname, kval) + rmap[kname] = kval + } - return rmap, uriWithKey, nil + return rmap, uriWithKey, nil } func contains(sl []string, str string) bool { diff --git a/src/translib/transformer/xspec.go b/src/translib/transformer/xspec.go index 80774588e4..2ec8c2f52d 100644 --- a/src/translib/transformer/xspec.go +++ b/src/translib/transformer/xspec.go @@ -42,68 +42,72 @@ var xDbSpecOrdTblMap map[string][]string //map of module-name to ordered list of /* update transformer spec with db-node */ func updateDbTableData (xpath string, xpathData *yangXpathInfo, tableName string) { - _, ok := xDbSpecMap[tableName] - if ok { + _, ok := xDbSpecMap[tableName] + if ok { xDbSpecMap[tableName].yangXpath = append(xDbSpecMap[tableName].yangXpath, xpath) - xpathData.dbEntry = xDbSpecMap[tableName].dbEntry - } + xpathData.dbEntry = xDbSpecMap[tableName].dbEntry + } } /* Recursive api to fill the map with yang details */ func yangToDbMapFill (keyLevel int, xSpecMap map[string]*yangXpathInfo, entry *yang.Entry, xpathPrefix string) { - xpath := "" - /* create the yang xpath */ - if xSpecMap[xpathPrefix] != nil && xSpecMap[xpathPrefix].yangDataType == "module" { - /* module name is separated from the rest of xpath with ":" */ - xpath = xpathPrefix + ":" + entry.Name - } else { - xpath = xpathPrefix + "/" + entry.Name - } + xpath := "" + /* create the yang xpath */ + if xSpecMap[xpathPrefix] != nil && xSpecMap[xpathPrefix].yangDataType == "module" { + /* module name is separated from the rest of xpath with ":" */ + xpath = xpathPrefix + ":" + entry.Name + } else { + xpath = xpathPrefix + "/" + entry.Name + } - xpathData, ok := xSpecMap[xpath] - if !ok { - xpathData = new(yangXpathInfo) - xSpecMap[xpath] = xpathData - xpathData.dbIndex = db.ConfigDB // default value - } else { - xpathData = xSpecMap[xpath] - } + xpathData, ok := xSpecMap[xpath] + if !ok { + xpathData = new(yangXpathInfo) + xSpecMap[xpath] = xpathData + xpathData.dbIndex = db.ConfigDB // default value + } else { + xpathData = xSpecMap[xpath] + } - xpathData.yangDataType = entry.Node.Statement().Keyword - if entry.Node.Statement().Keyword == "list" && xpathData.tableName != nil { - childToUpdateParent(xpath, *xpathData.tableName) - } + xpathData.yangDataType = entry.Node.Statement().Keyword + if entry.Node.Statement().Keyword == "list" && xpathData.tableName != nil { + childToUpdateParent(xpath, *xpathData.tableName) + } - parentXpathData, ok := xSpecMap[xpathPrefix] - /* init current xpath table data with its parent data, change only if needed. */ - if ok { - if xpathData.tableName == nil && parentXpathData.tableName != nil && xpathData.xfmrTbl == nil { - xpathData.tableName = parentXpathData.tableName - } else if xpathData.xfmrTbl == nil && parentXpathData.xfmrTbl != nil { - xpathData.xfmrTbl = parentXpathData.xfmrTbl - } - } + parentXpathData, ok := xSpecMap[xpathPrefix] + /* init current xpath table data with its parent data, change only if needed. */ + if ok { + if xpathData.tableName == nil && parentXpathData.tableName != nil && xpathData.xfmrTbl == nil { + xpathData.tableName = parentXpathData.tableName + } else if xpathData.xfmrTbl == nil && parentXpathData.xfmrTbl != nil { + xpathData.xfmrTbl = parentXpathData.xfmrTbl + } + } - if ok && parentXpathData.dbIndex != db.ConfigDB { - xpathData.dbIndex = parentXpathData.dbIndex - } + if ok && parentXpathData.dbIndex != db.ConfigDB { + xpathData.dbIndex = parentXpathData.dbIndex + } - if ok && len(parentXpathData.validateFunc) > 0 { - xpathData.validateFunc = parentXpathData.validateFunc - } + if ok && len(parentXpathData.validateFunc) > 0 { + xpathData.validateFunc = parentXpathData.validateFunc + } - if xpathData.yangDataType == "leaf" && len(xpathData.fieldName) == 0 { - if xpathData.tableName != nil && xDbSpecMap[*xpathData.tableName] != nil { + if ok && len(parentXpathData.xfmrFunc) > 0 && len(xpathData.xfmrFunc) == 0 { + xpathData.xfmrFunc = parentXpathData.xfmrFunc + } + + if xpathData.yangDataType == "leaf" && len(xpathData.fieldName) == 0 { + if xpathData.tableName != nil && xDbSpecMap[*xpathData.tableName] != nil { if xDbSpecMap[*xpathData.tableName].dbEntry.Dir[entry.Name] != nil { xpathData.fieldName = entry.Name } else if xDbSpecMap[*xpathData.tableName].dbEntry.Dir[strings.ToUpper(entry.Name)] != nil { xpathData.fieldName = strings.ToUpper(entry.Name) } } else if xpathData.xfmrTbl != nil { - /* table transformer present */ - xpathData.fieldName = entry.Name - } - } + /* table transformer present */ + xpathData.fieldName = entry.Name + } + } if xpathData.yangDataType == "leaf" && len(xpathData.fieldName) > 0 && xpathData.tableName != nil { dbPath := *xpathData.tableName + "/" + xpathData.fieldName @@ -112,44 +116,44 @@ func yangToDbMapFill (keyLevel int, xSpecMap map[string]*yangXpathInfo, entry *y } } - /* fill table with key data. */ - curKeyLevel := keyLevel - if len(entry.Key) != 0 { - parentKeyLen := 0 - - /* create list with current keys */ - keyXpath := make([]string, len(strings.Split(entry.Key, " "))) - for id, keyName := range(strings.Split(entry.Key, " ")) { - keyXpath[id] = xpath + "/" + keyName - keyXpathData := new(yangXpathInfo) - xSpecMap[xpath + "/" + keyName] = keyXpathData - xSpecMap[xpath + "/" + keyName].isKey = true - } + /* fill table with key data. */ + curKeyLevel := keyLevel + if len(entry.Key) != 0 { + parentKeyLen := 0 + + /* create list with current keys */ + keyXpath := make([]string, len(strings.Split(entry.Key, " "))) + for id, keyName := range(strings.Split(entry.Key, " ")) { + keyXpath[id] = xpath + "/" + keyName + keyXpathData := new(yangXpathInfo) + xSpecMap[xpath + "/" + keyName] = keyXpathData + xSpecMap[xpath + "/" + keyName].isKey = true + } - xpathData.keyXpath = make(map[int]*[]string, (parentKeyLen + 1)) - k := 0 - for ; k < parentKeyLen; k++ { - /* copy parent key-list to child key-list*/ - xpathData.keyXpath[k] = parentXpathData.keyXpath[k] - } - xpathData.keyXpath[k] = &keyXpath - xpathData.keyLevel = curKeyLevel - curKeyLevel++ - } else if parentXpathData != nil && parentXpathData.keyXpath != nil { - xpathData.keyXpath = parentXpathData.keyXpath - } + xpathData.keyXpath = make(map[int]*[]string, (parentKeyLen + 1)) + k := 0 + for ; k < parentKeyLen; k++ { + /* copy parent key-list to child key-list*/ + xpathData.keyXpath[k] = parentXpathData.keyXpath[k] + } + xpathData.keyXpath[k] = &keyXpath + xpathData.keyLevel = curKeyLevel + curKeyLevel++ + } else if parentXpathData != nil && parentXpathData.keyXpath != nil { + xpathData.keyXpath = parentXpathData.keyXpath + } - /* get current obj's children */ - var childList []string - for k := range entry.Dir { - childList = append(childList, k) - } + /* get current obj's children */ + var childList []string + for k := range entry.Dir { + childList = append(childList, k) + } - xpathData.yangEntry = entry - /* now recurse, filling the map with current node's children info */ - for _, child := range childList { - yangToDbMapFill(curKeyLevel, xSpecMap, entry.Dir[child], xpath) - } + xpathData.yangEntry = entry + /* now recurse, filling the map with current node's children info */ + for _, child := range childList { + yangToDbMapFill(curKeyLevel, xSpecMap, entry.Dir[child], xpath) + } } /* Build lookup table based of yang xpath */ @@ -195,7 +199,6 @@ func dbMapFill(prefixPath string, curPath string, moduleNm string, trkTpCnt bool xDbSpecMap[dbXpath].fieldType = entryType } - var childList []string for _, k := range entry.DirOKeys { childList = append(childList, k) @@ -215,108 +218,108 @@ func dbMapFill(prefixPath string, curPath string, moduleNm string, trkTpCnt bool /* Build redis db lookup map */ func dbMapBuild(entries []*yang.Entry) { - if entries == nil { - return - } - xDbSpecMap = make(map[string]*dbInfo) - xDbSpecOrdTblMap = make(map[string][]string) + if entries == nil { + return + } + xDbSpecMap = make(map[string]*dbInfo) + xDbSpecOrdTblMap = make(map[string][]string) - for _, e := range entries { - if e == nil || len(e.Dir) == 0 { - continue - } - moduleNm := e.Name - log.Info("Module name", moduleNm) - xDbSpecOrdTblMap[moduleNm] = []string{} - trkTpCnt := true - dbMapFill("", "", moduleNm, trkTpCnt, xDbSpecMap, e) - } + for _, e := range entries { + if e == nil || len(e.Dir) == 0 { + continue + } + moduleNm := e.Name + log.Info("Module name", moduleNm) + xDbSpecOrdTblMap[moduleNm] = []string{} + trkTpCnt := true + dbMapFill("", "", moduleNm, trkTpCnt, xDbSpecMap, e) + } } func childToUpdateParent( xpath string, tableName string) { - var xpathData *yangXpathInfo - parent := parentXpathGet(xpath) - if len(parent) == 0 || parent == "/" { - return - } + var xpathData *yangXpathInfo + parent := parentXpathGet(xpath) + if len(parent) == 0 || parent == "/" { + return + } - _, ok := xSpecMap[parent] - if !ok { - xpathData = new(yangXpathInfo) - xSpecMap[parent] = xpathData - } - xSpecMap[parent].childTable = append(xSpecMap[parent].childTable, tableName) - if xSpecMap[parent].yangEntry != nil && xSpecMap[parent].yangEntry.Node.Statement().Keyword == "list" { - return - } - childToUpdateParent(parent, tableName) + _, ok := xSpecMap[parent] + if !ok { + xpathData = new(yangXpathInfo) + xSpecMap[parent] = xpathData + } + xSpecMap[parent].childTable = append(xSpecMap[parent].childTable, tableName) + if xSpecMap[parent].yangEntry != nil && xSpecMap[parent].yangEntry.Node.Statement().Keyword == "list" { + return + } + childToUpdateParent(parent, tableName) } /* Build lookup map based on yang xpath */ func annotEntryFill(xSpecMap map[string]*yangXpathInfo, xpath string, entry *yang.Entry) { - xpathData := new(yangXpathInfo) - _, ok := xSpecMap[xpath] - if !ok { - fmt.Printf("Xpath not found(%v) \r\n", xpath) - } + xpathData := new(yangXpathInfo) + _, ok := xSpecMap[xpath] + if !ok { + fmt.Printf("Xpath not found(%v) \r\n", xpath) + } - xpathData.dbIndex = db.ConfigDB // default value - /* fill table with yang extension data. */ - if entry != nil && len(entry.Exts) > 0 { - for _, ext := range entry.Exts { - dataTagArr := strings.Split(ext.Keyword, ":") - tagType := dataTagArr[len(dataTagArr)-1] - switch tagType { - case "table-name" : - if xpathData.tableName == nil { - xpathData.tableName = new(string) - } - *xpathData.tableName = ext.NName() - updateDbTableData(xpath, xpathData, *xpathData.tableName) - //childToUpdateParent(xpath, *xpathData.tableName) - case "table-transformer" : - if xpathData.xfmrTbl == nil { - xpathData.xfmrTbl = new(string) - } - *xpathData.xfmrTbl = ext.NName() - case "field-name" : - xpathData.fieldName = ext.NName() - case "subtree-transformer" : - xpathData.xfmrFunc = ext.NName() - case "key-transformer" : - xpathData.xfmrKey = ext.NName() - case "key-delimiter" : - xpathData.delim = ext.NName() - case "field-transformer" : - xpathData.xfmrFunc = ext.NName() - case "post-transformer" : - xpathData.xfmrPost = ext.NName() - case "get-validate" : - xpathData.validateFunc = ext.NName() - case "use-self-key" : - xpathData.keyXpath = nil - case "redis-db-name" : - if ext.NName() == "APPL_DB" { - xpathData.dbIndex = db.ApplDB - } else if ext.NName() == "ASIC_DB" { - xpathData.dbIndex = db.AsicDB - } else if ext.NName() == "COUNTERS_DB" { - xpathData.dbIndex = db.CountersDB - } else if ext.NName() == "LOGLEVEL_DB" { - xpathData.dbIndex = db.LogLevelDB - } else if ext.NName() == "CONFIG_DB" { - xpathData.dbIndex = db.ConfigDB - } else if ext.NName() == "FLEX_COUNTER_DB" { - xpathData.dbIndex = db.FlexCounterDB - } else if ext.NName() == "STATE_DB" { - xpathData.dbIndex = db.StateDB - } else { - xpathData.dbIndex = db.ConfigDB - } - } - } - } - xSpecMap[xpath] = xpathData + xpathData.dbIndex = db.ConfigDB // default value + /* fill table with yang extension data. */ + if entry != nil && len(entry.Exts) > 0 { + for _, ext := range entry.Exts { + dataTagArr := strings.Split(ext.Keyword, ":") + tagType := dataTagArr[len(dataTagArr)-1] + switch tagType { + case "table-name" : + if xpathData.tableName == nil { + xpathData.tableName = new(string) + } + *xpathData.tableName = ext.NName() + updateDbTableData(xpath, xpathData, *xpathData.tableName) + //childToUpdateParent(xpath, *xpathData.tableName) + case "table-transformer" : + if xpathData.xfmrTbl == nil { + xpathData.xfmrTbl = new(string) + } + *xpathData.xfmrTbl = ext.NName() + case "field-name" : + xpathData.fieldName = ext.NName() + case "subtree-transformer" : + xpathData.xfmrFunc = ext.NName() + case "key-transformer" : + xpathData.xfmrKey = ext.NName() + case "key-delimiter" : + xpathData.delim = ext.NName() + case "field-transformer" : + xpathData.xfmrFunc = ext.NName() + case "post-transformer" : + xpathData.xfmrPost = ext.NName() + case "get-validate" : + xpathData.validateFunc = ext.NName() + case "use-self-key" : + xpathData.keyXpath = nil + case "redis-db-name" : + if ext.NName() == "APPL_DB" { + xpathData.dbIndex = db.ApplDB + } else if ext.NName() == "ASIC_DB" { + xpathData.dbIndex = db.AsicDB + } else if ext.NName() == "COUNTERS_DB" { + xpathData.dbIndex = db.CountersDB + } else if ext.NName() == "LOGLEVEL_DB" { + xpathData.dbIndex = db.LogLevelDB + } else if ext.NName() == "CONFIG_DB" { + xpathData.dbIndex = db.ConfigDB + } else if ext.NName() == "FLEX_COUNTER_DB" { + xpathData.dbIndex = db.FlexCounterDB + } else if ext.NName() == "STATE_DB" { + xpathData.dbIndex = db.StateDB + } else { + xpathData.dbIndex = db.ConfigDB + } + } + } + } + xSpecMap[xpath] = xpathData } /* Build xpath from yang-annotation */ From 045cbf90c92fbc20aa0a2dd3332c159e2ac26d1b Mon Sep 17 00:00:00 2001 From: s-mari Date: Wed, 25 Sep 2019 10:13:24 -0700 Subject: [PATCH 04/25] changed the xSpecMap to xYangSpecMap for code clarity and added new file with transformer-constants --- src/translib/transformer/xconst.go | 31 ++++++++++ src/translib/transformer/xlate.go | 8 +-- src/translib/transformer/xlate_from_db.go | 70 +++++++++++------------ src/translib/transformer/xlate_to_db.go | 42 +++++++------- src/translib/transformer/xlate_utils.go | 28 ++++----- src/translib/transformer/xspec.go | 55 +++++++++--------- 6 files changed, 133 insertions(+), 101 deletions(-) create mode 100644 src/translib/transformer/xconst.go diff --git a/src/translib/transformer/xconst.go b/src/translib/transformer/xconst.go new file mode 100644 index 0000000000..a20d297934 --- /dev/null +++ b/src/translib/transformer/xconst.go @@ -0,0 +1,31 @@ +package transformer + +const ( + YANG_MODULE = "module" + YANG_LIST = "list" + YANG_CONTAINER = "container" + YANG_LEAF = "leaf" + YANG_LEAF_LIST = "leaflist" + + YANG_ANNOT_DB_NAME = "redis-db-name" + YANG_ANNOT_TABLE_NAME = "table-name" + YANG_ANNOT_FIELD_NAME = "field-name" + YANG_ANNOT_KEY_DELIM = "key-delimiter" + YANG_ANNOT_TABLE_XFMR = "table-transformer" + YANG_ANNOT_FIELD_XFMR = "field-transformer" + YANG_ANNOT_KEY_XFMR = "key-transformer" + YANG_ANNOT_POST_XFMR = "post-transformer" + YANG_ANNOT_SUBTREE_XFMR = "subtree-transformer" + YANG_ANNOT_VALIDATE_FUNC = "get-validate" + + REDIS_DB_TYPE_APPLN = "APPL_DB" + REDIS_DB_TYPE_ASIC = "ASIC_DB" + REDIS_DB_TYPE_CONFIG = "CONFIG_DB" + REDIS_DB_TYPE_COUNTER = "COUNTERS_DB" + REDIS_DB_TYPE_LOG_LVL = "LOGLEVEL_DB" + REDIS_DB_TYPE_STATE = "STATE_DB" + REDIS_DB_TYPE_FLX_COUNTER = "FLEX_COUNTER_DB" + + XPATH_SEP_FWD_SLASH = "/" + XFMR_EMPTY_STRING = "" +) diff --git a/src/translib/transformer/xlate.go b/src/translib/transformer/xlate.go index f5489096f5..77007c30fc 100644 --- a/src/translib/transformer/xlate.go +++ b/src/translib/transformer/xlate.go @@ -128,12 +128,12 @@ func XlateUriToKeySpec(uri string, ygRoot *ygot.GoStruct, t *interface{}) (*[]Ke } func FillKeySpecs(yangXpath string , keyStr string, retdbFormat *[]KeySpec) ([]KeySpec){ - if xSpecMap == nil { + if xYangSpecMap == nil { return *retdbFormat } - _, ok := xSpecMap[yangXpath] + _, ok := xYangSpecMap[yangXpath] if ok { - xpathInfo := xSpecMap[yangXpath] + xpathInfo := xYangSpecMap[yangXpath] if xpathInfo.tableName != nil { dbFormat := KeySpec{} dbFormat.Ts.Name = *xpathInfo.tableName @@ -303,7 +303,7 @@ func XlateFromDb(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB, data m } } else { xpath, _ := RemoveXPATHPredicates(uri) - cdb = xSpecMap[xpath].dbIndex + cdb = xYangSpecMap[xpath].dbIndex } payload, err := dbDataToYangJsonCreate(uri, ygRoot, dbs, &dbData, cdb) log.Info("Payload generated:", payload) diff --git a/src/translib/transformer/xlate_from_db.go b/src/translib/transformer/xlate_from_db.go index 7125c1caea..fc8824222a 100644 --- a/src/translib/transformer/xlate_from_db.go +++ b/src/translib/transformer/xlate_from_db.go @@ -21,8 +21,8 @@ type typeMapOfInterface map[string]interface{} func xfmrHandlerFunc(inParams XfmrParams) (map[string]interface{}, error) { result := make(map[string]interface{}) xpath, _ := RemoveXPATHPredicates(inParams.uri) - log.Infof("Subtree transformer function(\"%v\") invoked for yang path(\"%v\").", xSpecMap[xpath].xfmrFunc, xpath) - _, err := XlateFuncCall(dbToYangXfmrFunc(xSpecMap[xpath].xfmrFunc), inParams) + log.Infof("Subtree transformer function(\"%v\") invoked for yang path(\"%v\").", xYangSpecMap[xpath].xfmrFunc, xpath) + _, err := XlateFuncCall(dbToYangXfmrFunc(xYangSpecMap[xpath].xfmrFunc), inParams) if err != nil { log.Infof("Failed to retrieve data for xpath(\"%v\") err(%v).", inParams.uri, err) return result, err @@ -61,7 +61,7 @@ func xfmrHandlerFunc(inParams XfmrParams) (map[string]interface{}, error) { func leafXfmrHandlerFunc(inParams XfmrParams) (map[string]interface{}, error) { xpath, _ := RemoveXPATHPredicates(inParams.uri) - ret, err := XlateFuncCall(dbToYangXfmrFunc(xSpecMap[xpath].xfmrFunc), inParams) + ret, err := XlateFuncCall(dbToYangXfmrFunc(xYangSpecMap[xpath].xfmrFunc), inParams) if err != nil { return nil, err } @@ -71,7 +71,7 @@ func leafXfmrHandlerFunc(inParams XfmrParams) (map[string]interface{}, error) { func validateHandlerFunc(inParams XfmrParams) (bool) { xpath, _ := RemoveXPATHPredicates(inParams.uri) - ret, err := XlateFuncCall(xSpecMap[xpath].validateFunc, inParams) + ret, err := XlateFuncCall(xYangSpecMap[xpath].validateFunc, inParams) if err != nil { return false } @@ -272,8 +272,8 @@ func yangListDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, x var tblList []string tblXfmr := false - if tbl == "" && xSpecMap[xpath].xfmrTbl != nil { - xfmrTblFunc := *xSpecMap[xpath].xfmrTbl + if tbl == "" && xYangSpecMap[xpath].xfmrTbl != nil { + xfmrTblFunc := *xYangSpecMap[xpath].xfmrTbl if len(xfmrTblFunc) > 0 { tblXfmr = true inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, "", dbDataMap, nil) @@ -296,7 +296,7 @@ func yangListDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, x for dbKey, _ := range tblData { curMap := make(map[string]interface{}) curKeyMap, curUri, _ := dbKeyToYangDataConvert(uri, xpath, dbKey) - if len(xSpecMap[xpath].xfmrFunc) > 0 { + if len(xYangSpecMap[xpath].xfmrFunc) > 0 { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, curUri, GET, "", dbDataMap, nil) cmap, _ := xfmrHandlerFunc(inParams) if len(cmap) > 0 { @@ -317,7 +317,7 @@ func yangListDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, x } } if len(mapSlice) > 0 { - resultMap[xSpecMap[xpath].yangEntry.Name] = mapSlice + resultMap[xYangSpecMap[xpath].yangEntry.Name] = mapSlice } else { log.Infof("Empty slice for (\"%v\").\r\n", uri) } @@ -330,14 +330,14 @@ func terminalNodeProcess(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string log.Infof("Received xpath - %v, uri - %v, dbDataMap - %v, table - %v, table key - %v", xpath, uri, (*dbDataMap), tbl, tblKey) var err error resFldValMap := make(map[string]interface{}) - if xSpecMap[xpath].yangEntry == nil { + if xYangSpecMap[xpath].yangEntry == nil { logStr := fmt.Sprintf("No yang entry found for xpath %v.", xpath) err = fmt.Errorf("%v", logStr) return resFldValMap, err } - cdb := xSpecMap[xpath].dbIndex - if len(xSpecMap[xpath].xfmrFunc) > 0 { + cdb := xYangSpecMap[xpath].dbIndex + if len(xYangSpecMap[xpath].xfmrFunc) > 0 { _, key, _ := xpathKeyExtract(dbs[cdb], ygRoot, GET, uri) inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, key, dbDataMap, nil) fldValMap, err := leafXfmrHandlerFunc(inParams) @@ -350,26 +350,26 @@ func terminalNodeProcess(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string resFldValMap[lf] = val } } else { - dbFldName := xSpecMap[xpath].fieldName + dbFldName := xYangSpecMap[xpath].fieldName /* if there is no transformer extension/annotation then it means leaf-list in yang is also leaflist in db */ - if len(dbFldName) > 0 && !xSpecMap[xpath].isKey { - yangType := yangTypeGet(xSpecMap[xpath].yangEntry) + if len(dbFldName) > 0 && !xYangSpecMap[xpath].isKey { + yangType := yangTypeGet(xYangSpecMap[xpath].yangEntry) if yangType == "leaf-list" { dbFldName += "@" val, ok := (*dbDataMap)[cdb][tbl][tblKey].Field[dbFldName] if ok { resLst := processLfLstDbToYang(xpath, val) - resFldValMap[xSpecMap[xpath].yangEntry.Name] = resLst + resFldValMap[xYangSpecMap[xpath].yangEntry.Name] = resLst } } else { val, ok := (*dbDataMap)[cdb][tbl][tblKey].Field[dbFldName] if ok { - yngTerminalNdDtType := xSpecMap[xpath].yangEntry.Type.Kind + yngTerminalNdDtType := xYangSpecMap[xpath].yangEntry.Type.Kind resVal, err := DbToYangType(yngTerminalNdDtType, xpath, val) if err != nil { log.Error("Failure in converting Db value type to yang type for field", xpath) } else { - resFldValMap[xSpecMap[xpath].yangEntry.Name] = resVal + resFldValMap[xYangSpecMap[xpath].yangEntry.Name] = resVal } } } @@ -382,15 +382,15 @@ func terminalNodeProcess(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string func yangDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, resultMap map[string]interface{}, tbl string, tblKey string, cdb db.DBNum, validate bool) error { var err error isValid := validate - yangNode, ok := xSpecMap[xpath] + yangNode, ok := xYangSpecMap[xpath] if ok && yangNode.yangEntry != nil { for yangChldName := range yangNode.yangEntry.Dir { chldXpath := xpath+"/"+yangChldName chldUri := uri+"/"+yangChldName - if xSpecMap[chldXpath] != nil && xSpecMap[chldXpath].yangEntry != nil { + if xYangSpecMap[chldXpath] != nil && xYangSpecMap[chldXpath].yangEntry != nil { _, key, _ := xpathKeyExtract(dbs[cdb], ygRoot, GET, chldUri) - if len(xSpecMap[chldXpath].validateFunc) > 0 && !validate { + if len(xYangSpecMap[chldXpath].validateFunc) > 0 && !validate { // TODO - handle non CONFIG-DB inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, key, dbDataMap, nil) res := validateHandlerFunc(inParams) @@ -400,8 +400,8 @@ func yangDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath isValid = res } } - chldYangType := yangTypeGet(xSpecMap[chldXpath].yangEntry) - cdb = xSpecMap[chldXpath].dbIndex + chldYangType := yangTypeGet(xYangSpecMap[chldXpath].yangEntry) + cdb = xYangSpecMap[chldXpath].dbIndex if chldYangType == "leaf" || chldYangType == "leaf-list" { fldValMap, err := terminalNodeProcess(dbs, ygRoot, chldUri, chldXpath, dbDataMap, tbl, tblKey) if err != nil { @@ -411,8 +411,8 @@ func yangDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath resultMap[lf] = val } } else if chldYangType == "container" { - cname := xSpecMap[chldXpath].yangEntry.Name - if len(xSpecMap[chldXpath].xfmrFunc) > 0 { + cname := xYangSpecMap[chldXpath].yangEntry.Name + if len(xYangSpecMap[chldXpath].xfmrFunc) > 0 { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, "", dbDataMap, nil) cmap, _ := xfmrHandlerFunc(inParams) if len(cmap) > 0 { @@ -420,8 +420,8 @@ func yangDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath } else { log.Infof("Empty container(\"%v\").\r\n", chldUri) } - } else if xSpecMap[chldXpath].xfmrTbl != nil { - xfmrTblFunc := *xSpecMap[chldXpath].xfmrTbl + } else if xYangSpecMap[chldXpath].xfmrTbl != nil { + xfmrTblFunc := *xYangSpecMap[chldXpath].xfmrTbl if len(xfmrTblFunc) > 0 { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, "", dbDataMap, nil) tblList := xfmrTblHandlerFunc(xfmrTblFunc, inParams) @@ -439,8 +439,8 @@ func yangDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath log.Infof("Empty container(\"%v\").\r\n", chldUri) } } else if chldYangType == "list" { - cdb = xSpecMap[chldXpath].dbIndex - if len(xSpecMap[chldXpath].xfmrFunc) > 0 { + cdb = xYangSpecMap[chldXpath].dbIndex + if len(xYangSpecMap[chldXpath].xfmrFunc) > 0 { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, "", dbDataMap, nil) cmap, _ := xfmrHandlerFunc(inParams) if len(cmap) > 0 { @@ -449,7 +449,7 @@ func yangDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath log.Infof("Empty list(\"%v\").\r\n", chldUri) } } else { - ynode, ok := xSpecMap[chldXpath] + ynode, ok := xYangSpecMap[chldXpath] lTblName := "" if ok && ynode.tableName != nil { lTblName = *ynode.tableName @@ -474,15 +474,15 @@ func dbDataToYangJsonCreate(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db } else { var d *db.DB reqXpath, keyName, tableName := xpathKeyExtract(d, ygRoot, GET, uri) - yangNode, ok := xSpecMap[reqXpath] + yangNode, ok := xYangSpecMap[reqXpath] if ok { yangType := yangTypeGet(yangNode.yangEntry) if yangType == "leaf" || yangType == "leaf-list" { - //fldName := xSpecMap[reqXpath].fieldName - yangName := xSpecMap[reqXpath].yangEntry.Name + //fldName := xYangSpecMap[reqXpath].fieldName + yangName := xYangSpecMap[reqXpath].yangEntry.Name tbl, key, _ := tableNameAndKeyFromDbMapGet((*dbDataMap)[cdb]) validateHandlerFlag := false - if len(xSpecMap[reqXpath].validateFunc) > 0 { + if len(xYangSpecMap[reqXpath].validateFunc) > 0 { // TODO - handle non CONFIG-DB inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, key, dbDataMap, nil) res := validateHandlerFunc(inParams) @@ -500,9 +500,9 @@ func dbDataToYangJsonCreate(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db resultMap = fldValMap } } else if yangType == "container" { - cname := xSpecMap[reqXpath].yangEntry.Name + cname := xYangSpecMap[reqXpath].yangEntry.Name cmap := make(map[string]interface{}) - if len(xSpecMap[reqXpath].xfmrFunc) > 0 { + if len(xYangSpecMap[reqXpath].xfmrFunc) > 0 { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, "", dbDataMap, nil) cmap, _ = xfmrHandlerFunc(inParams) if len(cmap) > 0 { diff --git a/src/translib/transformer/xlate_to_db.go b/src/translib/transformer/xlate_to_db.go index 4afa6cf0e1..2b98dd25e7 100644 --- a/src/translib/transformer/xlate_to_db.go +++ b/src/translib/transformer/xlate_to_db.go @@ -21,12 +21,12 @@ const SONIC_FIELD_INDEX = 3 /* Invoke the post tansformer */ func postXfmrHandlerFunc(inParams XfmrParams) (map[string]map[string]db.Value, error) { xpath, _ := RemoveXPATHPredicates(inParams.uri) - ret, err := XlateFuncCall(xSpecMap[xpath].xfmrPost, inParams) + ret, err := XlateFuncCall(xYangSpecMap[xpath].xfmrPost, inParams) if err != nil { return nil, err } retData := ret[0].Interface().(map[string]map[string]db.Value) - log.Info("Post Transformer function :", xSpecMap[xpath].xfmrPost, " Xpath: ", xpath, " retData: ", retData) + log.Info("Post Transformer function :", xYangSpecMap[xpath].xfmrPost, " Xpath: ", xpath, " retData: ", retData) return retData, err } @@ -62,7 +62,7 @@ func mapFillData(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, dbKey st var dbs [db.MaxDB]*db.DB var err error xpath := xpathPrefix + "/" + name - xpathInfo := xSpecMap[xpath] + xpathInfo := xYangSpecMap[xpath] log.Infof("name: \"%v\", xpathPrefix(\"%v\").", name, xpathPrefix) if xpathInfo == nil { @@ -107,7 +107,7 @@ func mapFillData(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, dbKey st return nErr } inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, uri, oper, "", nil, node[0].Data) - ret, err := XlateFuncCall(yangToDbXfmrFunc(xSpecMap[xpath].xfmrFunc), inParams) + ret, err := XlateFuncCall(yangToDbXfmrFunc(xYangSpecMap[xpath].xfmrFunc), inParams) if err != nil { return err } @@ -156,7 +156,7 @@ func mapFillData(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, dbKey st if xpathInfo.xfmrTbl != nil { inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, uri, oper, "", nil, "") // expecting only one table name from tbl-xfmr - tableName, err = tblNameFromTblXfmrGet(*xSpecMap[xpath].xfmrTbl, inParams) + tableName, err = tblNameFromTblXfmrGet(*xYangSpecMap[xpath].xfmrTbl, inParams) if err != nil { return err } @@ -247,7 +247,7 @@ func dbMapDelete(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string, jsonDat } else { xpathPrefix, keyName, tableName := xpathKeyExtract(d, ygRoot, oper, path) log.Infof("Delete req: path(\"%v\"), key(\"%v\"), xpathPrefix(\"%v\"), tableName(\"%v\").", path, keyName, xpathPrefix, tableName) - spec, ok := xSpecMap[xpathPrefix] + spec, ok := xYangSpecMap[xpathPrefix] if ok { if spec.tableName != nil { result[*spec.tableName] = make(map[string]db.Value) @@ -328,9 +328,9 @@ func dbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string, jsonDat if oper == CREATE { moduleNm := "/" + strings.Split(path, "/")[1] log.Infof("Module name for path %s is %s", path, moduleNm) - if _, ok := xSpecMap[moduleNm]; ok { - if xSpecMap[moduleNm].yangDataType == "container" && len(xSpecMap[moduleNm].xfmrPost) > 0 { - log.Info("Invoke post transformer: ", xSpecMap[moduleNm].xfmrPost) + if _, ok := xYangSpecMap[moduleNm]; ok { + if xYangSpecMap[moduleNm].yangDataType == "container" && len(xYangSpecMap[moduleNm].xfmrPost) > 0 { + log.Info("Invoke post transformer: ", xYangSpecMap[moduleNm].xfmrPost) dbDataMap := make(map[db.DBNum]map[string]map[string]db.Value) dbDataMap[db.ConfigDB] = result var dbs [db.MaxDB]*db.DB @@ -338,7 +338,7 @@ func dbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string, jsonDat result, err = postXfmrHandlerFunc(inParams) } } else { - log.Errorf("No Entry exists for module %s in XSpecMap. Unable to process post xfmr (\"%v\") path(\"%v\") error (\"%v\").", oper, path, err) + log.Errorf("No Entry exists for module %s in xYangSpecMap. Unable to process post xfmr (\"%v\") path(\"%v\") error (\"%v\").", oper, path, err) } } printDbData(result, "/tmp/yangToDbDataCreate.txt") @@ -362,10 +362,10 @@ func yangReqToDbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, for _, data := range dataMap { curKey := "" curUri, _ := uriWithKeyCreate(uri, xpathPrefix, data) - if len(xSpecMap[xpathPrefix].xfmrKey) > 0 { + if len(xYangSpecMap[xpathPrefix].xfmrKey) > 0 { /* key transformer present */ inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curUri, oper, "", nil, nil) - ret, err := XlateFuncCall(yangToDbXfmrFunc(xSpecMap[xpathPrefix].xfmrKey), inParams) + ret, err := XlateFuncCall(yangToDbXfmrFunc(xYangSpecMap[xpathPrefix].xfmrKey), inParams) if err != nil { return err } @@ -393,11 +393,11 @@ func yangReqToDbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, curUri = uri + "/" + pathAttr } - if (typeOfValue == reflect.Map || typeOfValue == reflect.Slice) && xSpecMap[xpath].yangDataType != "leaf-list" { - if xSpecMap[xpath] != nil && len(xSpecMap[xpath].xfmrFunc) > 0 { + if (typeOfValue == reflect.Map || typeOfValue == reflect.Slice) && xYangSpecMap[xpath].yangDataType != "leaf-list" { + if xYangSpecMap[xpath] != nil && len(xYangSpecMap[xpath].xfmrFunc) > 0 { /* subtree transformer present */ inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curUri, oper, "", nil, nil) - ret, err := XlateFuncCall(yangToDbXfmrFunc(xSpecMap[xpath].xfmrFunc), inParams) + ret, err := XlateFuncCall(yangToDbXfmrFunc(xYangSpecMap[xpath].xfmrFunc), inParams) if err != nil { return nil } @@ -460,10 +460,10 @@ func xpathKeyExtract(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string) (st keyStr += "|" } yangXpath, _ := RemoveXPATHPredicates(curPathWithKey) - _, ok := xSpecMap[yangXpath] + _, ok := xYangSpecMap[yangXpath] if ok { - if len(xSpecMap[yangXpath].xfmrKey) > 0 { - xfmrFuncName := yangToDbXfmrFunc(xSpecMap[yangXpath].xfmrKey) + if len(xYangSpecMap[yangXpath].xfmrKey) > 0 { + xfmrFuncName := yangToDbXfmrFunc(xYangSpecMap[yangXpath].xfmrKey) inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curPathWithKey, oper, "", nil, nil) ret, err := XlateFuncCall(xfmrFuncName, inParams) if err != nil { @@ -482,12 +482,12 @@ func xpathKeyExtract(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string) (st curPathWithKey += "/" } pfxPath, _ := RemoveXPATHPredicates(path) - tblPtr := xSpecMap[pfxPath].tableName + tblPtr := xYangSpecMap[pfxPath].tableName if tblPtr != nil { tableName = *tblPtr - } else if xSpecMap[pfxPath].xfmrTbl != nil { + } else if xYangSpecMap[pfxPath].xfmrTbl != nil { inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curPathWithKey, oper, "", nil, nil) - tableName, _ = tblNameFromTblXfmrGet(*xSpecMap[pfxPath].xfmrTbl, inParams) + tableName, _ = tblNameFromTblXfmrGet(*xYangSpecMap[pfxPath].xfmrTbl, inParams) } return pfxPath, keyStr, tableName diff --git a/src/translib/transformer/xlate_utils.go b/src/translib/transformer/xlate_utils.go index 42e280ab5b..87a1f67312 100644 --- a/src/translib/transformer/xlate_utils.go +++ b/src/translib/transformer/xlate_utils.go @@ -26,10 +26,10 @@ func keyFromXpathCreate(keyList []string) string { /* Create db key from data xpath(request) */ func keyCreate(keyPrefix string, xpath string, data interface{}) string { - _, ok := xSpecMap[xpath] + _, ok := xYangSpecMap[xpath] if ok { - if xSpecMap[xpath].yangEntry != nil { - yangEntry := xSpecMap[xpath].yangEntry + if xYangSpecMap[xpath].yangEntry != nil { + yangEntry := xYangSpecMap[xpath].yangEntry if len(keyPrefix) > 0 { keyPrefix += "|" } keyVal := "" for i, k := range (strings.Split(yangEntry.Key, " ")) { @@ -95,8 +95,8 @@ func dbKeyToYangDataConvert(uri string, xpath string, dbKey string) (map[string] return nil, "", err } - if _, ok := xSpecMap[xpath]; ok { - if xSpecMap[xpath].yangEntry == nil { + if _, ok := xYangSpecMap[xpath]; ok { + if xYangSpecMap[xpath].yangEntry == nil { err = fmt.Errorf("Yang Entry not available for xpath ", xpath) return nil, "", nil } @@ -104,8 +104,8 @@ func dbKeyToYangDataConvert(uri string, xpath string, dbKey string) (map[string] var kLvlValList []string keyDataList := strings.Split(dbKey, "|") - keyNameList := yangKeyFromEntryGet(xSpecMap[xpath].yangEntry) - id := xSpecMap[xpath].keyLevel + keyNameList := yangKeyFromEntryGet(xYangSpecMap[xpath].yangEntry) + id := xYangSpecMap[xpath].keyLevel uriWithKey := fmt.Sprintf("%v", xpath) /* if uri contins key, use it else use xpath */ @@ -113,10 +113,10 @@ func dbKeyToYangDataConvert(uri string, xpath string, dbKey string) (map[string] uriWithKey = fmt.Sprintf("%v", uri) } - if len(xSpecMap[xpath].xfmrKey) > 0 { + if len(xYangSpecMap[xpath].xfmrKey) > 0 { var dbs [db.MaxDB]*db.DB inParams := formXfmrInputRequest(nil, dbs, db.MaxDB, nil, uri, GET, dbKey, nil, nil) - ret, err := XlateFuncCall(dbToYangXfmrFunc(xSpecMap[xpath].xfmrKey), inParams) + ret, err := XlateFuncCall(dbToYangXfmrFunc(xYangSpecMap[xpath].xfmrKey), inParams) if err != nil { return nil, "", err } @@ -228,8 +228,8 @@ func yangToDbXfmrFunc(funcName string) string { func uriWithKeyCreate (uri string, xpathTmplt string, data interface{}) (string, error) { var err error - if _, ok := xSpecMap[xpathTmplt]; ok { - yangEntry := xSpecMap[xpathTmplt].yangEntry + if _, ok := xYangSpecMap[xpathTmplt]; ok { + yangEntry := xYangSpecMap[xpathTmplt].yangEntry if yangEntry != nil { for _, k := range (strings.Split(yangEntry.Key, " ")) { uri += fmt.Sprintf("[%v=%v]", k, data.(map[string]interface{})[k]) @@ -238,7 +238,7 @@ func uriWithKeyCreate (uri string, xpathTmplt string, data interface{}) (string, err = fmt.Errorf("Yang Entry not available for xpath ", xpathTmplt) } } else { - err = fmt.Errorf("No entry in xSpecMap for xpath ", xpathTmplt) + err = fmt.Errorf("No entry in xYangSpecMap for xpath ", xpathTmplt) } return uri, err } @@ -252,9 +252,9 @@ func xpathRootNameGet(path string) string { } func getDbNum(xpath string ) db.DBNum { - _, ok := xSpecMap[xpath] + _, ok := xYangSpecMap[xpath] if ok { - xpathInfo := xSpecMap[xpath] + xpathInfo := xYangSpecMap[xpath] return xpathInfo.dbIndex } // Default is ConfigDB diff --git a/src/translib/transformer/xspec.go b/src/translib/transformer/xspec.go index 2ec8c2f52d..2a42f87bf1 100644 --- a/src/translib/transformer/xspec.go +++ b/src/translib/transformer/xspec.go @@ -36,7 +36,7 @@ type dbInfo struct { yangXpath []string } -var xSpecMap map[string]*yangXpathInfo +var xYangSpecMap map[string]*yangXpathInfo var xDbSpecMap map[string]*dbInfo var xDbSpecOrdTblMap map[string][]string //map of module-name to ordered list of db tables { "sonic-acl" : ["ACL_TABLE", "ACL_RULE"] } @@ -50,23 +50,23 @@ func updateDbTableData (xpath string, xpathData *yangXpathInfo, tableName string } /* Recursive api to fill the map with yang details */ -func yangToDbMapFill (keyLevel int, xSpecMap map[string]*yangXpathInfo, entry *yang.Entry, xpathPrefix string) { +func yangToDbMapFill (keyLevel int, xYangSpecMap map[string]*yangXpathInfo, entry *yang.Entry, xpathPrefix string) { xpath := "" /* create the yang xpath */ - if xSpecMap[xpathPrefix] != nil && xSpecMap[xpathPrefix].yangDataType == "module" { + if xYangSpecMap[xpathPrefix] != nil && xYangSpecMap[xpathPrefix].yangDataType == "module" { /* module name is separated from the rest of xpath with ":" */ xpath = xpathPrefix + ":" + entry.Name } else { xpath = xpathPrefix + "/" + entry.Name } - xpathData, ok := xSpecMap[xpath] + xpathData, ok := xYangSpecMap[xpath] if !ok { xpathData = new(yangXpathInfo) - xSpecMap[xpath] = xpathData + xYangSpecMap[xpath] = xpathData xpathData.dbIndex = db.ConfigDB // default value } else { - xpathData = xSpecMap[xpath] + xpathData = xYangSpecMap[xpath] } xpathData.yangDataType = entry.Node.Statement().Keyword @@ -74,7 +74,7 @@ func yangToDbMapFill (keyLevel int, xSpecMap map[string]*yangXpathInfo, entry *y childToUpdateParent(xpath, *xpathData.tableName) } - parentXpathData, ok := xSpecMap[xpathPrefix] + parentXpathData, ok := xYangSpecMap[xpathPrefix] /* init current xpath table data with its parent data, change only if needed. */ if ok { if xpathData.tableName == nil && parentXpathData.tableName != nil && xpathData.xfmrTbl == nil { @@ -126,8 +126,8 @@ func yangToDbMapFill (keyLevel int, xSpecMap map[string]*yangXpathInfo, entry *y for id, keyName := range(strings.Split(entry.Key, " ")) { keyXpath[id] = xpath + "/" + keyName keyXpathData := new(yangXpathInfo) - xSpecMap[xpath + "/" + keyName] = keyXpathData - xSpecMap[xpath + "/" + keyName].isKey = true + xYangSpecMap[xpath + "/" + keyName] = keyXpathData + xYangSpecMap[xpath + "/" + keyName].isKey = true } xpathData.keyXpath = make(map[int]*[]string, (parentKeyLen + 1)) @@ -152,7 +152,7 @@ func yangToDbMapFill (keyLevel int, xSpecMap map[string]*yangXpathInfo, entry *y xpathData.yangEntry = entry /* now recurse, filling the map with current node's children info */ for _, child := range childList { - yangToDbMapFill(curKeyLevel, xSpecMap, entry.Dir[child], xpath) + yangToDbMapFill(curKeyLevel, xYangSpecMap, entry.Dir[child], xpath) } } @@ -162,8 +162,8 @@ func yangToDbMapBuild(entries map[string]*yang.Entry) { return } - if xSpecMap == nil { - xSpecMap = make(map[string]*yangXpathInfo) + if xYangSpecMap == nil { + xYangSpecMap = make(map[string]*yangXpathInfo) } for module, e := range entries { @@ -173,12 +173,12 @@ func yangToDbMapBuild(entries map[string]*yang.Entry) { /* Start to fill xpath based map with yang data */ keyLevel := 0 - yangToDbMapFill(keyLevel, xSpecMap, e, "") + yangToDbMapFill(keyLevel, xYangSpecMap, e, "") // Fill the ordered map of child tables list for oc yangs updateSchemaOrderedMap(module, e) } - mapPrint(xSpecMap, "/tmp/fullSpec.txt") + mapPrint(xYangSpecMap, "/tmp/fullSpec.txt") dbMapPrint() } @@ -243,22 +243,23 @@ func childToUpdateParent( xpath string, tableName string) { return } - _, ok := xSpecMap[parent] + _, ok := xYangSpecMap[parent] if !ok { xpathData = new(yangXpathInfo) - xSpecMap[parent] = xpathData + xYangSpecMap[parent] = xpathData } - xSpecMap[parent].childTable = append(xSpecMap[parent].childTable, tableName) - if xSpecMap[parent].yangEntry != nil && xSpecMap[parent].yangEntry.Node.Statement().Keyword == "list" { + xYangSpecMap[parent].childTable = append(xYangSpecMap[parent].childTable, tableName) + if xYangSpecMap[parent].yangEntry != nil && + xYangSpecMap[parent].yangEntry.Node.Statement().Keyword == "list" { return } childToUpdateParent(parent, tableName) } /* Build lookup map based on yang xpath */ -func annotEntryFill(xSpecMap map[string]*yangXpathInfo, xpath string, entry *yang.Entry) { +func annotEntryFill(xYangSpecMap map[string]*yangXpathInfo, xpath string, entry *yang.Entry) { xpathData := new(yangXpathInfo) - _, ok := xSpecMap[xpath] + _, ok := xYangSpecMap[xpath] if !ok { fmt.Printf("Xpath not found(%v) \r\n", xpath) } @@ -319,7 +320,7 @@ func annotEntryFill(xSpecMap map[string]*yangXpathInfo, xpath string, entry *yan } } } - xSpecMap[xpath] = xpathData + xYangSpecMap[xpath] = xpathData } /* Build xpath from yang-annotation */ @@ -336,8 +337,8 @@ func annotToDbMapBuild(annotEntries []*yang.Entry) { if annotEntries == nil { return } - if xSpecMap == nil { - xSpecMap = make(map[string]*yangXpathInfo) + if xYangSpecMap == nil { + xYangSpecMap = make(map[string]*yangXpathInfo) } for _, e := range annotEntries { @@ -348,14 +349,14 @@ func annotToDbMapBuild(annotEntries []*yang.Entry) { for i, deviate := range d.Deviate { if i == 2 { for _, ye := range deviate { - annotEntryFill(xSpecMap, xpath, ye) + annotEntryFill(xYangSpecMap, xpath, ye) } } } } } } - mapPrint(xSpecMap, "/tmp/annotSpec.txt") + mapPrint(xYangSpecMap, "/tmp/annotSpec.txt") } /* Debug function to print the yang xpath lookup map */ @@ -427,9 +428,9 @@ func updateSchemaOrderedMap(module string, entry *yang.Entry) { for _, dir := range entry.DirOKeys { // Gives the yang xpath for the top level container xpath := "/" + module + ":" + dir - _, ok := xSpecMap[xpath] + _, ok := xYangSpecMap[xpath] if ok { - yentry := xSpecMap[xpath].yangEntry + yentry := xYangSpecMap[xpath].yangEntry if yentry.Node.Statement().Keyword == "container" { var keyspec = make([]KeySpec, 0) keyspec = FillKeySpecs(xpath, "" , &keyspec) From ce598cfcd82a55116d8aca7c56d15f42f03f17f2 Mon Sep 17 00:00:00 2001 From: amrutasali Date: Wed, 25 Sep 2019 22:02:06 +0000 Subject: [PATCH 05/25] Enhanced transformer core to support user-defined key-delimiter/concatenater --- src/translib/transformer/xlate_from_db.go | 2 +- src/translib/transformer/xlate_to_db.go | 2 +- src/translib/transformer/xlate_utils.go | 25 ++++++++++++++++------- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/translib/transformer/xlate_from_db.go b/src/translib/transformer/xlate_from_db.go index fc8824222a..26f1ae7174 100644 --- a/src/translib/transformer/xlate_from_db.go +++ b/src/translib/transformer/xlate_from_db.go @@ -295,7 +295,7 @@ func yangListDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, x var mapSlice []typeMapOfInterface for dbKey, _ := range tblData { curMap := make(map[string]interface{}) - curKeyMap, curUri, _ := dbKeyToYangDataConvert(uri, xpath, dbKey) + curKeyMap, curUri, _ := dbKeyToYangDataConvert(uri, xpath, dbKey, dbs[cdb].Opts.KeySeparator) if len(xYangSpecMap[xpath].xfmrFunc) > 0 { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, curUri, GET, "", dbDataMap, nil) cmap, _ := xfmrHandlerFunc(inParams) diff --git a/src/translib/transformer/xlate_to_db.go b/src/translib/transformer/xlate_to_db.go index 2b98dd25e7..00c5f0d495 100644 --- a/src/translib/transformer/xlate_to_db.go +++ b/src/translib/transformer/xlate_to_db.go @@ -371,7 +371,7 @@ func yangReqToDbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, } curKey = ret[0].Interface().(string) } else { - curKey = keyCreate(keyName, xpathPrefix, data) + curKey = keyCreate(keyName, xpathPrefix, data, d.Opts.KeySeparator) } yangReqToDbMapCreate(d, ygRoot, oper, curUri, xpathPrefix, curKey, data, result) } diff --git a/src/translib/transformer/xlate_utils.go b/src/translib/transformer/xlate_utils.go index 87a1f67312..6820a56377 100644 --- a/src/translib/transformer/xlate_utils.go +++ b/src/translib/transformer/xlate_utils.go @@ -25,15 +25,21 @@ func keyFromXpathCreate(keyList []string) string { } /* Create db key from data xpath(request) */ -func keyCreate(keyPrefix string, xpath string, data interface{}) string { +func keyCreate(keyPrefix string, xpath string, data interface{}, dbKeySep string) string { _, ok := xYangSpecMap[xpath] if ok { if xYangSpecMap[xpath].yangEntry != nil { yangEntry := xYangSpecMap[xpath].yangEntry - if len(keyPrefix) > 0 { keyPrefix += "|" } + keyConcat := dbKeySep + if len(xYangSpecMap[xpath].delim) > 0 { + keyConcat = xYangSpecMap[xpath].delim + log.Infof("key concatenater(\"%v\") found for xpath %v ", keyConcat, xpath) + } + + if len(keyPrefix) > 0 { keyPrefix += dbKeySep } keyVal := "" for i, k := range (strings.Split(yangEntry.Key, " ")) { - if i > 0 { keyVal = keyVal + "_" } + if i > 0 { keyVal = keyVal + keyConcat } val := fmt.Sprint(data.(map[string]interface{})[k]) if strings.Contains(val, ":") { val = strings.Split(val, ":")[1] @@ -88,7 +94,7 @@ func yangTypeGet(entry *yang.Entry) string { return "" } -func dbKeyToYangDataConvert(uri string, xpath string, dbKey string) (map[string]interface{}, string, error) { +func dbKeyToYangDataConvert(uri string, xpath string, dbKey string, dbKeySep string) (map[string]interface{}, string, error) { var err error if len(uri) == 0 && len(xpath) == 0 && len(dbKey) == 0 { err = fmt.Errorf("Insufficient input") @@ -103,7 +109,7 @@ func dbKeyToYangDataConvert(uri string, xpath string, dbKey string) (map[string] } var kLvlValList []string - keyDataList := strings.Split(dbKey, "|") + keyDataList := strings.Split(dbKey, dbKeySep) keyNameList := yangKeyFromEntryGet(xYangSpecMap[xpath].yangEntry) id := xYangSpecMap[xpath].keyLevel uriWithKey := fmt.Sprintf("%v", xpath) @@ -112,6 +118,11 @@ func dbKeyToYangDataConvert(uri string, xpath string, dbKey string) (map[string] if strings.Contains(uri, "[") { uriWithKey = fmt.Sprintf("%v", uri) } + keyConcat := dbKeySep + if len(xYangSpecMap[xpath].delim) > 0 { + keyConcat = xYangSpecMap[xpath].delim + log.Infof("Found key concatenater(\"%v\") for xpath %v", keyConcat, xpath) + } if len(xYangSpecMap[xpath].xfmrKey) > 0 { var dbs [db.MaxDB]*db.DB @@ -134,7 +145,7 @@ func dbKeyToYangDataConvert(uri string, xpath string, dbKey string) (map[string] kLvlValList = append(kLvlValList, keyDataList[id]) if len(keyNameList) > 1 { - kLvlValList = strings.Split(keyDataList[id], "_") + kLvlValList = strings.Split(keyDataList[id], keyConcat) } /* TODO: Need to add leaf-ref related code in here and remove this code*/ @@ -151,7 +162,7 @@ func dbKeyToYangDataConvert(uri string, xpath string, dbKey string) (map[string] /* TODO: Need to add leaf-ref related code in here and remove this code*/ if kvalExceedFlag && (i == chgId) { - kval = strings.Join(kLvlValList[chgId:], "_") + kval = strings.Join(kLvlValList[chgId:], keyConcat) } uriWithKey += fmt.Sprintf("[%v=%v]", kname, kval) From 20d14e1543b02f780c6945691a0ee55b52b993f6 Mon Sep 17 00:00:00 2001 From: s-mari Date: Wed, 25 Sep 2019 17:12:28 -0700 Subject: [PATCH 06/25] basic support for sonic-yang annotation --- src/translib/transformer/transformer.go | 16 ++- src/translib/transformer/xspec.go | 154 +++++++++++++++++++----- 2 files changed, 131 insertions(+), 39 deletions(-) diff --git a/src/translib/transformer/transformer.go b/src/translib/transformer/transformer.go index e6649cff2c..c69ecabda6 100644 --- a/src/translib/transformer/transformer.go +++ b/src/translib/transformer/transformer.go @@ -117,13 +117,16 @@ func loadYangModules(files ...string) error { } } - sonic_entries := make([]*yang.Entry, len(names)) - oc_entries := make(map[string]*yang.Entry) - annot_entries := make([]*yang.Entry, len(names)) + sonic_entries := make([]*yang.Entry, len(names)) + oc_entries := make(map[string]*yang.Entry) + oc_annot_entries := make([]*yang.Entry, len(names)) + sonic_annot_entries := make([]*yang.Entry, len(names)) for _, n := range names { - if strings.Contains(n, "annot") { - annot_entries = append(annot_entries, yang.ToEntry(mods[n])) + if strings.Contains(n, "annot") && strings.Contains(n, "sonic") { + sonic_annot_entries = append(sonic_annot_entries, yang.ToEntry(mods[n])) + } else if strings.Contains(n, "annot") { + oc_annot_entries = append(oc_annot_entries, yang.ToEntry(mods[n])) } else if strings.Contains(n, "sonic") { sonic_entries = append(sonic_entries, yang.ToEntry(mods[n])) } else if oc_entries[n] == nil { @@ -132,7 +135,8 @@ func loadYangModules(files ...string) error { } dbMapBuild(sonic_entries) - annotToDbMapBuild(annot_entries) + annotDbSpecMap(sonic_annot_entries) + annotToDbMapBuild(oc_annot_entries) yangToDbMapBuild(oc_entries) return err diff --git a/src/translib/transformer/xspec.go b/src/translib/transformer/xspec.go index 2a42f87bf1..dde3804918 100644 --- a/src/translib/transformer/xspec.go +++ b/src/translib/transformer/xspec.go @@ -31,9 +31,11 @@ type yangXpathInfo struct { } type dbInfo struct { + dbIndex db.DBNum + keyName *string fieldType string dbEntry *yang.Entry - yangXpath []string + yangXpath []string } var xYangSpecMap map[string]*yangXpathInfo @@ -179,41 +181,44 @@ func yangToDbMapBuild(entries map[string]*yang.Entry) { updateSchemaOrderedMap(module, e) } mapPrint(xYangSpecMap, "/tmp/fullSpec.txt") - dbMapPrint() + dbMapPrint("/tmp/dbSpecMap.txt") } /* Fill the map with db details */ func dbMapFill(prefixPath string, curPath string, moduleNm string, trkTpCnt bool, xDbSpecMap map[string]*dbInfo, entry *yang.Entry) { - entryType := entry.Node.Statement().Keyword - if entryType == "list" { - prefixPath = entry.Name - } + entryType := entry.Node.Statement().Keyword + if entryType == "list" { + prefixPath = entry.Name + } - if !isYangResType(entryType) { - dbXpath := prefixPath - if entryType != "list" { - dbXpath = prefixPath + "/" + entry.Name - } - xDbSpecMap[dbXpath] = new(dbInfo) - xDbSpecMap[dbXpath].dbEntry = entry - xDbSpecMap[dbXpath].fieldType = entryType - } + if !isYangResType(entryType) { + dbXpath := prefixPath + if entryType != "list" { + dbXpath = prefixPath + "/" + entry.Name + } + xDbSpecMap[dbXpath] = new(dbInfo) + xDbSpecMap[dbXpath].dbEntry = entry + xDbSpecMap[dbXpath].fieldType = entryType + if entryType == "list" { + xDbSpecMap[dbXpath].dbIndex = db.ConfigDB + } + } - var childList []string - for _, k := range entry.DirOKeys { - childList = append(childList, k) - } + var childList []string + for _, k := range entry.DirOKeys { + childList = append(childList, k) + } - if entryType == "container" && trkTpCnt { - xDbSpecOrdTblMap[moduleNm] = childList - log.Info("xDbSpecOrdTblMap after appending ", xDbSpecOrdTblMap) - trkTpCnt = false - } + if entryType == "container" && trkTpCnt { + xDbSpecOrdTblMap[moduleNm] = childList + log.Info("xDbSpecOrdTblMap after appending ", xDbSpecOrdTblMap) + trkTpCnt = false + } - for _, child := range childList { - childPath := prefixPath + "/" + entry.Dir[child].Name - dbMapFill(prefixPath, childPath, moduleNm, trkTpCnt, xDbSpecMap, entry.Dir[child]) - } + for _, child := range childList { + childPath := prefixPath + "/" + entry.Dir[child].Name + dbMapFill(prefixPath, childPath, moduleNm, trkTpCnt, xDbSpecMap, entry.Dir[child]) + } } /* Build redis db lookup map */ @@ -359,6 +364,84 @@ func annotToDbMapBuild(annotEntries []*yang.Entry) { mapPrint(xYangSpecMap, "/tmp/annotSpec.txt") } +func annotDbSpecMapFill(xDbSpecMap map[string]*dbInfo, dbXpath string, entry *yang.Entry) error { + var err error + var dbXpathData *dbInfo + var ok bool + + //Currently sonic-yang annotation is supported for "list" type only. + listName := strings.Split(dbXpath, "/") + if len(listName) < 2 { + log.Errorf("Invalid list xpath length(%v) \r\n", dbXpath) + return err + } + dbXpathData, ok = xDbSpecMap[listName[2]] + if !ok { + log.Errorf("DB spec-map data not found(%v) \r\n", dbXpath) + return err + } + log.Infof("Annotate dbSpecMap for (%v)(listName:%v)\r\n", dbXpath, listName[2]) + dbXpathData.dbIndex = db.ConfigDB // default value + + /* fill table with cvl yang extension data. */ + if entry != nil && len(entry.Exts) > 0 { + for _, ext := range entry.Exts { + dataTagArr := strings.Split(ext.Keyword, ":") + tagType := dataTagArr[len(dataTagArr)-1] + switch tagType { + case "key-name" : + if dbXpathData.keyName == nil { + dbXpathData.keyName = new(string) + } + *dbXpathData.keyName = ext.NName() + case "db-name" : + if ext.NName() == "APPL_DB" { + dbXpathData.dbIndex = db.ApplDB + } else if ext.NName() == "ASIC_DB" { + dbXpathData.dbIndex = db.AsicDB + } else if ext.NName() == "COUNTERS_DB" { + dbXpathData.dbIndex = db.CountersDB + } else if ext.NName() == "LOGLEVEL_DB" { + dbXpathData.dbIndex = db.LogLevelDB + } else if ext.NName() == "CONFIG_DB" { + dbXpathData.dbIndex = db.ConfigDB + } else if ext.NName() == "FLEX_COUNTER_DB" { + dbXpathData.dbIndex = db.FlexCounterDB + } else if ext.NName() == "STATE_DB" { + dbXpathData.dbIndex = db.StateDB + } else { + dbXpathData.dbIndex = db.ConfigDB + } + default : + } + } + } + + dbMapPrint("/tmp/dbSpecMapFull.txt") + return err +} + +func annotDbSpecMap(annotEntries []*yang.Entry) { + if annotEntries == nil || xDbSpecMap == nil { + return + } + for _, e := range annotEntries { + if e != nil && len(e.Deviations) > 0 { + for _, d := range e.Deviations { + xpath := xpathFromDevCreate(d.Name) + xpath = "/" + strings.Replace(e.Name, "-annot", "", -1) + ":" + xpath + for i, deviate := range d.Deviate { + if i == 2 { + for _, ye := range deviate { + annotDbSpecMapFill(xDbSpecMap, xpath, ye) + } + } + } + } + } + } +} + /* Debug function to print the yang xpath lookup map */ func mapPrint(inMap map[string]*yangXpathInfo, fileName string) { fp, err := os.Create(fileName) @@ -405,8 +488,8 @@ func mapPrint(inMap map[string]*yangXpathInfo, fileName string) { } /* Debug function to print redis db lookup map */ -func dbMapPrint() { - fp, err := os.Create("/tmp/dbTmplt.txt") +func dbMapPrint( fname string) { + fp, err := os.Create(fname) if err != nil { return } @@ -414,9 +497,14 @@ func dbMapPrint() { fmt.Fprintf (fp, "-----------------------------------------------------------------\r\n") for k, v := range xDbSpecMap { fmt.Fprintf(fp, " field:%v \r\n", k) - fmt.Fprintf(fp, " type :%v \r\n", v.fieldType) - fmt.Fprintf(fp, " Yang :%v \r\n", v.yangXpath) - fmt.Fprintf(fp, " DB :%v \r\n", v.dbEntry) + fmt.Fprintf(fp, " type :%v \r\n", v.fieldType) + fmt.Fprintf(fp, " db-type :%v \r\n", v.dbIndex) + fmt.Fprintf(fp, " KeyName: ") + if v.keyName != nil { + fmt.Fprintf(fp, "%v", *v.keyName) + } + fmt.Fprintf(fp, "\r\n oc-yang :%v \r\n", v.yangXpath) + fmt.Fprintf(fp, " cvl-yang :%v \r\n", v.dbEntry) fmt.Fprintf (fp, "-----------------------------------------------------------------\r\n") } From 021209a571b95d4129d9c4cca9e74ab6698b6399 Mon Sep 17 00:00:00 2001 From: Ranjini Nagaraj Date: Wed, 25 Sep 2019 18:19:55 -0700 Subject: [PATCH 07/25] Fill cvl KeySpec with DbName read from annotation --- src/translib/transformer/xlate.go | 18 +++++++++-------- src/translib/transformer/xlate_to_db.go | 27 +++++++++++++------------ src/translib/transformer/xspec.go | 2 +- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/src/translib/transformer/xlate.go b/src/translib/transformer/xlate.go index 77007c30fc..844fb9b7a8 100644 --- a/src/translib/transformer/xlate.go +++ b/src/translib/transformer/xlate.go @@ -116,12 +116,12 @@ func XlateUriToKeySpec(uri string, ygRoot *ygot.GoStruct, t *interface{}) (*[]Ke // In case of CVL yang, the tablename and key info is available in the xpath if isCvlYang(uri) { /* Extract the xpath and key from input xpath */ - yangXpath, keyStr, tableName := sonicXpathKeyExtract(uri) - retdbFormat = fillCvlKeySpec(yangXpath, tableName, keyStr) + xpath, keyStr, tableName := sonicXpathKeyExtract(uri) + retdbFormat = fillCvlKeySpec(xpath, tableName, keyStr) } else { /* Extract the xpath and key from input xpath */ - yangXpath, keyStr, _ := xpathKeyExtract(nil, ygRoot, 0, uri) - retdbFormat = FillKeySpecs(yangXpath, keyStr, &retdbFormat) + xpath, keyStr, _ := xpathKeyExtract(nil, ygRoot, 0, uri) + retdbFormat = FillKeySpecs(xpath, keyStr, &retdbFormat) } return &retdbFormat, err @@ -170,29 +170,31 @@ func FillKeySpecs(yangXpath string , keyStr string, retdbFormat *[]KeySpec) ([]K return *retdbFormat } -func fillCvlKeySpec(yangXpath string , tableName string, keyStr string) ( []KeySpec ) { +func fillCvlKeySpec(xpath string , tableName string, keyStr string) ( []KeySpec ) { var retdbFormat = make([]KeySpec, 0) if tableName != "" { dbFormat := KeySpec{} dbFormat.Ts.Name = tableName - dbFormat.dbNum = db.ConfigDB + cdb := xDbSpecMap[xpath].dbIndex + dbFormat.dbNum = cdb if keyStr != "" { dbFormat.Key.Comp = append(dbFormat.Key.Comp, keyStr) } retdbFormat = append(retdbFormat, dbFormat) } else { // If table name not available in xpath get top container name - tokens:= strings.Split(yangXpath, ":") + tokens:= strings.Split(xpath, ":") container := "/" + tokens[len(tokens)-1] if xDbSpecMap[container] != nil { dbInfo := xDbSpecMap[container] if dbInfo.fieldType == "container" { for dir, _ := range dbInfo.dbEntry.Dir { + cdb := xDbSpecMap[dir].dbIndex dbFormat := KeySpec{} dbFormat.Ts.Name = dir - dbFormat.dbNum = db.ConfigDB + dbFormat.dbNum = cdb retdbFormat = append(retdbFormat, dbFormat) } } diff --git a/src/translib/transformer/xlate_to_db.go b/src/translib/transformer/xlate_to_db.go index 00c5f0d495..6b21d71f74 100644 --- a/src/translib/transformer/xlate_to_db.go +++ b/src/translib/transformer/xlate_to_db.go @@ -83,6 +83,19 @@ func mapFillData(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, dbKey st if xpathInfo.isKey { return nil } + + tableName := "" + if xpathInfo.xfmrTbl != nil { + inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, uri, oper, "", nil, "") + // expecting only one table name from tbl-xfmr + tableName, err = tblNameFromTblXfmrGet(*xYangSpecMap[xpath].xfmrTbl, inParams) + if err != nil { + return err + } + } else { + tableName = *xpathInfo.tableName + } + if len(xpathInfo.xfmrFunc) > 0 { uri = uri + "/" + name @@ -114,7 +127,7 @@ func mapFillData(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, dbKey st retData := ret[0].Interface().(map[string]string) log.Info("Transformer function :", xpathInfo.xfmrFunc, " Xpath: ", xpath, " retData: ", retData) for f, v := range retData { - dataToDBMapAdd(*xpathInfo.tableName, dbKey, result, f, v) + dataToDBMapAdd(tableName, dbKey, result, f, v) } return nil } @@ -152,18 +165,6 @@ func mapFillData(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, dbKey st } } - tableName := "" - if xpathInfo.xfmrTbl != nil { - inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, uri, oper, "", nil, "") - // expecting only one table name from tbl-xfmr - tableName, err = tblNameFromTblXfmrGet(*xYangSpecMap[xpath].xfmrTbl, inParams) - if err != nil { - return err - } - } else { - tableName = *xpathInfo.tableName - } - dataToDBMapAdd(tableName, dbKey, result, fieldName, valueStr) log.Infof("TblName: \"%v\", key: \"%v\", field: \"%v\", valueStr: \"%v\".", tableName, dbKey, fieldName, valueStr) diff --git a/src/translib/transformer/xspec.go b/src/translib/transformer/xspec.go index dde3804918..3327805223 100644 --- a/src/translib/transformer/xspec.go +++ b/src/translib/transformer/xspec.go @@ -371,7 +371,7 @@ func annotDbSpecMapFill(xDbSpecMap map[string]*dbInfo, dbXpath string, entry *ya //Currently sonic-yang annotation is supported for "list" type only. listName := strings.Split(dbXpath, "/") - if len(listName) < 2 { + if len(listName) < 3 { log.Errorf("Invalid list xpath length(%v) \r\n", dbXpath) return err } From 483030f8aeebe1f913750ade03c26be30cca9648 Mon Sep 17 00:00:00 2001 From: Ranjini Nagaraj Date: Thu, 26 Sep 2019 17:15:35 -0700 Subject: [PATCH 08/25] Fix table get in Traverse for App DB with ":" separator --- src/translib/transformer/xlate.go | 15 +++++++++++---- src/translib/transformer/xlate_to_db.go | 6 +++++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/translib/transformer/xlate.go b/src/translib/transformer/xlate.go index 844fb9b7a8..3563b93858 100644 --- a/src/translib/transformer/xlate.go +++ b/src/translib/transformer/xlate.go @@ -69,6 +69,10 @@ func XlateFuncCall(name string, params ...interface{}) (result []reflect.Value, func TraverseDb(dbs [db.MaxDB]*db.DB, spec KeySpec, result *map[db.DBNum]map[string]map[string]db.Value, parentKey *db.Key) error { var err error + separator := ":" + if spec.dbNum == db.ConfigDB { + separator = "|" + } if spec.Key.Len() > 0 { // get an entry with a specific key @@ -78,9 +82,9 @@ func TraverseDb(dbs [db.MaxDB]*db.DB, spec KeySpec, result *map[db.DBNum]map[str } if (*result)[spec.dbNum][spec.Ts.Name] == nil { - (*result)[spec.dbNum][spec.Ts.Name] = map[string]db.Value{strings.Join(spec.Key.Comp, "|"): data} + (*result)[spec.dbNum][spec.Ts.Name] = map[string]db.Value{strings.Join(spec.Key.Comp, separator): data} } else { - (*result)[spec.dbNum][spec.Ts.Name][strings.Join(spec.Key.Comp, "|")] = data + (*result)[spec.dbNum][spec.Ts.Name][strings.Join(spec.Key.Comp, separator)] = data } if len(spec.Child) > 0 { @@ -97,7 +101,7 @@ func TraverseDb(dbs [db.MaxDB]*db.DB, spec KeySpec, result *map[db.DBNum]map[str for i, _ := range keys { if parentKey != nil { // TODO - multi-depth with a custom delimiter - if strings.Index(strings.Join(keys[i].Comp, "|"), strings.Join((*parentKey).Comp, "|")) == -1 { + if strings.Index(strings.Join(keys[i].Comp, separator), strings.Join((*parentKey).Comp, "|")) == -1 { continue } } @@ -177,7 +181,10 @@ func fillCvlKeySpec(xpath string , tableName string, keyStr string) ( []KeySpec if tableName != "" { dbFormat := KeySpec{} dbFormat.Ts.Name = tableName - cdb := xDbSpecMap[xpath].dbIndex + cdb := db.ConfigDB + if _, ok := xDbSpecMap[xpath]; ok { + cdb = xDbSpecMap[xpath].dbIndex + } dbFormat.dbNum = cdb if keyStr != "" { dbFormat.Key.Comp = append(dbFormat.Key.Comp, keyStr) diff --git a/src/translib/transformer/xlate_to_db.go b/src/translib/transformer/xlate_to_db.go index 6b21d71f74..b34c87d5ec 100644 --- a/src/translib/transformer/xlate_to_db.go +++ b/src/translib/transformer/xlate_to_db.go @@ -436,7 +436,11 @@ func sonicXpathKeyExtract(path string) (string, string, string) { rgp := regexp.MustCompile(`\[([^\[\]]*)\]`) pathsubStr := strings.Split(path , "/") if len(pathsubStr) > SONIC_TABLE_INDEX { - tableName = strings.Split(pathsubStr[SONIC_TABLE_INDEX], "[")[0] + if strings.Contains(pathsubStr[2], ":") { + tableName = strings.Split(pathsubStr[SONIC_TABLE_INDEX], "[")[0] + } else { + tableName = pathsubStr[SONIC_TABLE_INDEX] + } for i, kname := range rgp.FindAllString(path, -1) { if i > 0 { keyStr += "|" } val := strings.Split(kname, "=")[1] From 60da3901fe5d172f4d5769d71d8f527d822b328b Mon Sep 17 00:00:00 2001 From: s-mari Date: Thu, 26 Sep 2019 18:16:22 -0700 Subject: [PATCH 09/25] added support to invoke container level key transformer --- src/translib/transformer/xlate_to_db.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/translib/transformer/xlate_to_db.go b/src/translib/transformer/xlate_to_db.go index b34c87d5ec..e47971dfc9 100644 --- a/src/translib/transformer/xlate_to_db.go +++ b/src/translib/transformer/xlate_to_db.go @@ -385,6 +385,7 @@ func yangReqToDbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, log.Infof("slice/map data: key(\"%v\"), xpathPrefix(\"%v\").", keyName, xpathPrefix) xpath := uri curUri := uri + curKey := keyName pathAttr := key.String() if len(xpathPrefix) > 0 { if strings.Contains(pathAttr, ":") { @@ -393,7 +394,17 @@ func yangReqToDbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, xpath = xpathPrefix + "/" + pathAttr curUri = uri + "/" + pathAttr } - + log.Infof("slice/map data: curKey(\"%v\"), xpath(\"%v\"), curUri(\"%v\").", + curKey, xpath, curUri) + if len(xYangSpecMap[xpath].xfmrKey) > 0 { + /* key transformer present */ + inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curUri, oper, "", nil, nil) + ret, err := XlateFuncCall(yangToDbXfmrFunc(xYangSpecMap[xpath].xfmrKey), inParams) + if err != nil { + return err + } + curKey = ret[0].Interface().(string) + } if (typeOfValue == reflect.Map || typeOfValue == reflect.Slice) && xYangSpecMap[xpath].yangDataType != "leaf-list" { if xYangSpecMap[xpath] != nil && len(xYangSpecMap[xpath].xfmrFunc) > 0 { /* subtree transformer present */ @@ -404,7 +415,7 @@ func yangReqToDbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, } mapCopy(result, ret[0].Interface().(map[string]map[string]db.Value)) } else { - yangReqToDbMapCreate(d, ygRoot, oper, curUri, xpath, keyName, jData.MapIndex(key).Interface(), result) + yangReqToDbMapCreate(d, ygRoot, oper, curUri, xpath, curKey, jData.MapIndex(key).Interface(), result) } } else { pathAttr := key.String() @@ -413,7 +424,7 @@ func yangReqToDbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, } value := jData.MapIndex(key).Interface() log.Infof("data field: key(\"%v\"), value(\"%v\").", key, value) - err := mapFillData(d, ygRoot, oper, uri, keyName, result, xpathPrefix, + err := mapFillData(d, ygRoot, oper, uri, curKey, result, xpathPrefix, pathAttr, value) if err != nil { log.Errorf("Failed constructing data for db write: key(\"%v\"), value(\"%v\"), path(\"%v\").", From 73b1ecf2d06760da27af3b58e29b2d9901732adb Mon Sep 17 00:00:00 2001 From: s-mari Date: Thu, 26 Sep 2019 22:52:14 -0700 Subject: [PATCH 10/25] added code to pass the current ygot node to the key xfmr function --- src/translib/transformer/xlate_to_db.go | 42 ++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/src/translib/transformer/xlate_to_db.go b/src/translib/transformer/xlate_to_db.go index e47971dfc9..7e4cb108b4 100644 --- a/src/translib/transformer/xlate_to_db.go +++ b/src/translib/transformer/xlate_to_db.go @@ -349,6 +349,28 @@ func dbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string, jsonDat return err } +func yangNodeForUriGet(uri string, ygRoot *ygot.GoStruct) (interface{}, error) { + path, _ := ygot.StringToPath(uri, ygot.StructuredPath, ygot.StringSlicePath) + for _, p := range path.Elem { + pathSlice := strings.Split(p.Name, ":") + p.Name = pathSlice[len(pathSlice)-1] + if len(p.Key) > 0 { + for ekey, ent := range p.Key { + eslice := strings.Split(ent, ":") + p.Key[ekey] = eslice[len(eslice)-1] + } + } + } + ocbSch, _ := ocbinds.Schema() + schRoot := ocbSch.RootSchema() + node, nErr := ytypes.GetNode(schRoot, (*ygRoot).(*ocbinds.Device), path) + log.Info("GetNode data: ", node[0].Data, " nErr :", nErr) + if nErr != nil { + return nil, nErr + } + return node[0].Data, nil +} + func yangReqToDbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, xpathPrefix string, keyName string, jsonData interface{}, result map[string]map[string]db.Value) error { log.Infof("key(\"%v\"), xpathPrefix(\"%v\").", keyName, xpathPrefix) var dbs [db.MaxDB]*db.DB @@ -365,10 +387,14 @@ func yangReqToDbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, curUri, _ := uriWithKeyCreate(uri, xpathPrefix, data) if len(xYangSpecMap[xpathPrefix].xfmrKey) > 0 { /* key transformer present */ - inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curUri, oper, "", nil, nil) + curYgotNode, nodeErr := yangNodeForUriGet(curUri, ygRoot) + if nodeErr != nil { + curYgotNode = nil + } + inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curUri, oper, "", nil, curYgotNode) ret, err := XlateFuncCall(yangToDbXfmrFunc(xYangSpecMap[xpathPrefix].xfmrKey), inParams) if err != nil { - return err + return err } curKey = ret[0].Interface().(string) } else { @@ -398,7 +424,11 @@ func yangReqToDbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, curKey, xpath, curUri) if len(xYangSpecMap[xpath].xfmrKey) > 0 { /* key transformer present */ - inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curUri, oper, "", nil, nil) + curYgotNode, nodeErr := yangNodeForUriGet(curUri, ygRoot) + if nodeErr != nil { + curYgotNode = nil + } + inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curUri, oper, "", nil, curYgotNode) ret, err := XlateFuncCall(yangToDbXfmrFunc(xYangSpecMap[xpath].xfmrKey), inParams) if err != nil { return err @@ -408,7 +438,11 @@ func yangReqToDbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, if (typeOfValue == reflect.Map || typeOfValue == reflect.Slice) && xYangSpecMap[xpath].yangDataType != "leaf-list" { if xYangSpecMap[xpath] != nil && len(xYangSpecMap[xpath].xfmrFunc) > 0 { /* subtree transformer present */ - inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curUri, oper, "", nil, nil) + curYgotNode, nodeErr := yangNodeForUriGet(curUri, ygRoot) + if nodeErr != nil { + curYgotNode = nil + } + inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curUri, oper, "", nil, curYgotNode) ret, err := XlateFuncCall(yangToDbXfmrFunc(xYangSpecMap[xpath].xfmrFunc), inParams) if err != nil { return nil From 98822f9fccee72dc4723ab6624557e1f59cd1f50 Mon Sep 17 00:00:00 2001 From: s-mari Date: Fri, 27 Sep 2019 12:46:57 -0700 Subject: [PATCH 11/25] added support for sonic yang non-config db get request --- src/translib/transformer/xlate_from_db.go | 70 +++++++++++++---------- src/translib/transformer/xlate_utils.go | 8 ++- 2 files changed, 45 insertions(+), 33 deletions(-) diff --git a/src/translib/transformer/xlate_from_db.go b/src/translib/transformer/xlate_from_db.go index 26f1ae7174..ac25687f64 100644 --- a/src/translib/transformer/xlate_from_db.go +++ b/src/translib/transformer/xlate_from_db.go @@ -169,38 +169,40 @@ func processLfLstDbToYang(fieldXpath string, dbFldVal string) []interface{} { } /* Traverse db map and create json for cvl yang */ -func directDbToYangJsonCreate(dbDataMap map[string]map[string]db.Value, jsonData string, resultMap map[string]interface{}) error { +func directDbToYangJsonCreate(dbDataMap *map[db.DBNum]map[string]map[string]db.Value, jsonData string, resultMap map[string]interface{}) error { var err error - for tblName, tblData := range dbDataMap { - var mapSlice []typeMapOfInterface - for keyStr, dbFldValData := range tblData { - curMap := make(map[string]interface{}) - for field, value := range dbFldValData.Field { - resField := field - if strings.HasSuffix(field, "@") { - fldVals := strings.Split(field, "@") - resField = fldVals[0] - } - fieldXpath := tblName + "/" + resField - xDbSpecMapEntry, ok := xDbSpecMap[fieldXpath] - if !ok { - log.Warningf("No entry found in xDbSpecMap for xpath %v", fieldXpath) - continue - } - if xDbSpecMapEntry.dbEntry == nil { - log.Warningf("Yang entry is nil in xDbSpecMap for xpath %v", fieldXpath) - continue - } - yangType := yangTypeGet(xDbSpecMapEntry.dbEntry) - if yangType == "leaf-list" { - /* this should never happen but just adding for safetty */ - if !strings.HasSuffix(field, "@") { - log.Warningf("Leaf-list in Sonic yang should also be a leaf-list in DB, its not for xpath %v", fieldXpath) + for curDbIdx := db.ApplDB; curDbIdx < db.MaxDB; curDbIdx++ { + dbTblData := (*dbDataMap)[curDbIdx] + for tblName, tblData := range dbTblData { + var mapSlice []typeMapOfInterface + for keyStr, dbFldValData := range tblData { + curMap := make(map[string]interface{}) + for field, value := range dbFldValData.Field { + resField := field + if strings.HasSuffix(field, "@") { + fldVals := strings.Split(field, "@") + resField = fldVals[0] + } + fieldXpath := tblName + "/" + resField + xDbSpecMapEntry, ok := xDbSpecMap[fieldXpath] + if !ok { + log.Warningf("No entry found in xDbSpecMap for xpath %v", fieldXpath) continue } - resLst := processLfLstDbToYang(fieldXpath, value) - curMap[resField] = resLst - } else { /* yangType is leaf - there are only 2 types of yang terminal node leaf and leaf-list */ + if xDbSpecMapEntry.dbEntry == nil { + log.Warningf("Yang entry is nil in xDbSpecMap for xpath %v", fieldXpath) + continue + } + yangType := yangTypeGet(xDbSpecMapEntry.dbEntry) + if yangType == "leaf-list" { + /* this should never happen but just adding for safetty */ + if !strings.HasSuffix(field, "@") { + log.Warningf("Leaf-list in Sonic yang should also be a leaf-list in DB, its not for xpath %v", fieldXpath) + continue + } + resLst := processLfLstDbToYang(fieldXpath, value) + curMap[resField] = resLst + } else { /* yangType is leaf - there are only 2 types of yang terminal node leaf and leaf-list */ yngTerminalNdDtType := xDbSpecMapEntry.dbEntry.Type.Kind resVal, err := DbToYangType(yngTerminalNdDtType, fieldXpath, value) if err != nil { @@ -210,14 +212,20 @@ func directDbToYangJsonCreate(dbDataMap map[string]map[string]db.Value, jsonData } } } //end of for + dbSpecData, ok := xDbSpecMap[tblName] + dbIndex := db.ConfigDB + if ok { + dbIndex = dbSpecData.dbIndex + } yangKeys := yangKeyFromEntryGet(xDbSpecMap[tblName].dbEntry) - sonicKeyDataAdd(yangKeys, keyStr, curMap) + sonicKeyDataAdd(dbIndex, yangKeys, keyStr, curMap) if curMap != nil { mapSlice = append(mapSlice, curMap) } } resultMap[tblName] = mapSlice } +} return err } @@ -470,7 +478,7 @@ func dbDataToYangJsonCreate(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db jsonData := "" resultMap := make(map[string]interface{}) if isCvlYang(uri) { - directDbToYangJsonCreate((*dbDataMap)[cdb], jsonData, resultMap) + directDbToYangJsonCreate(dbDataMap, jsonData, resultMap) } else { var d *db.DB reqXpath, keyName, tableName := xpathKeyExtract(d, ygRoot, GET, uri) diff --git a/src/translib/transformer/xlate_utils.go b/src/translib/transformer/xlate_utils.go index 6820a56377..f4dc44340d 100644 --- a/src/translib/transformer/xlate_utils.go +++ b/src/translib/transformer/xlate_utils.go @@ -222,8 +222,12 @@ func isCvlYang(path string) bool { return false } -func sonicKeyDataAdd(keyNameList []string, keyStr string, resultMap map[string]interface{}) { - keyValList := strings.Split(keyStr, "|") +func sonicKeyDataAdd(dbIndex db.DBNum, keyNameList []string, keyStr string, resultMap map[string]interface{}) { + keyValList := strings.Split(keyStr, ":") + if dbIndex == db.ConfigDB { + keyValList = strings.Split(keyStr, "|") + } + if len(keyNameList) != len(keyValList) { return } From 2a17ea20a094ce07f1bdb70e683507555fe70a25 Mon Sep 17 00:00:00 2001 From: amrutasali Date: Fri, 27 Sep 2019 23:54:25 +0000 Subject: [PATCH 12/25] Data base specific key-delimiter support in transformer core - Sonic Get case --- src/translib/transformer/xlate.go | 28 ++++++++++++++-------- src/translib/transformer/xlate_to_db.go | 14 ++++++++--- src/translib/transformer/xlate_utils.go | 32 +++++++++++++++++++++---- 3 files changed, 57 insertions(+), 17 deletions(-) diff --git a/src/translib/transformer/xlate.go b/src/translib/transformer/xlate.go index 3563b93858..7453631df9 100644 --- a/src/translib/transformer/xlate.go +++ b/src/translib/transformer/xlate.go @@ -69,10 +69,11 @@ func XlateFuncCall(name string, params ...interface{}) (result []reflect.Value, func TraverseDb(dbs [db.MaxDB]*db.DB, spec KeySpec, result *map[db.DBNum]map[string]map[string]db.Value, parentKey *db.Key) error { var err error - separator := ":" - if spec.dbNum == db.ConfigDB { - separator = "|" - } + var dbOpts db.Options + + dbOpts = getDBOptions(spec.dbNum) + separator := dbOpts.KeySeparator + log.Infof("key separator for table %v in Db %v is %v", spec.Ts.Name, spec.dbNum, separator) if spec.Key.Len() > 0 { // get an entry with a specific key @@ -101,7 +102,7 @@ func TraverseDb(dbs [db.MaxDB]*db.DB, spec KeySpec, result *map[db.DBNum]map[str for i, _ := range keys { if parentKey != nil { // TODO - multi-depth with a custom delimiter - if strings.Index(strings.Join(keys[i].Comp, separator), strings.Join((*parentKey).Comp, "|")) == -1 { + if strings.Index(strings.Join(keys[i].Comp, separator), strings.Join((*parentKey).Comp, separator)) == -1 { continue } } @@ -182,8 +183,8 @@ func fillCvlKeySpec(xpath string , tableName string, keyStr string) ( []KeySpec dbFormat := KeySpec{} dbFormat.Ts.Name = tableName cdb := db.ConfigDB - if _, ok := xDbSpecMap[xpath]; ok { - cdb = xDbSpecMap[xpath].dbIndex + if _, ok := xDbSpecMap[tableName]; ok { + cdb = xDbSpecMap[tableName].dbIndex } dbFormat.dbNum = cdb if keyStr != "" { @@ -296,14 +297,21 @@ func GetAndXlateFromDB(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB) func XlateFromDb(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB, data map[db.DBNum]map[string]map[string]db.Value) ([]byte, error) { var err error + var result []byte var dbData = make(map[db.DBNum]map[string]map[string]db.Value) var cdb db.DBNum = db.ConfigDB dbData = data if isCvlYang(uri) { - yangXpath, keyStr, tableName := sonicXpathKeyExtract(uri) + xpath, keyStr, tableName := sonicXpathKeyExtract(uri) if (tableName != "") { - tokens:= strings.Split(yangXpath, "/") + dbInfo, ok := xDbSpecMap[tableName] + if !ok { + log.Warningf("No entry in xDbSpecMap for xpath %v", tableName) + } else { + cdb = dbInfo.dbIndex + } + tokens:= strings.Split(xpath, "/") // Format /module:container/tableName[key]/fieldName if tokens[len(tokens)-2] == tableName { fieldName := tokens[len(tokens)-1] @@ -322,7 +330,7 @@ func XlateFromDb(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB, data m return nil, err } - result := []byte(payload) + result = []byte(payload) return result, err } diff --git a/src/translib/transformer/xlate_to_db.go b/src/translib/transformer/xlate_to_db.go index 7e4cb108b4..1cbe833915 100644 --- a/src/translib/transformer/xlate_to_db.go +++ b/src/translib/transformer/xlate_to_db.go @@ -364,7 +364,7 @@ func yangNodeForUriGet(uri string, ygRoot *ygot.GoStruct) (interface{}, error) { ocbSch, _ := ocbinds.Schema() schRoot := ocbSch.RootSchema() node, nErr := ytypes.GetNode(schRoot, (*ygRoot).(*ocbinds.Device), path) - log.Info("GetNode data: ", node[0].Data, " nErr :", nErr) + //log.Info("GetNode data: ", node[0].Data, " nErr :", nErr) if nErr != nil { return nil, nErr } @@ -481,13 +481,21 @@ func sonicXpathKeyExtract(path string) (string, string, string) { rgp := regexp.MustCompile(`\[([^\[\]]*)\]`) pathsubStr := strings.Split(path , "/") if len(pathsubStr) > SONIC_TABLE_INDEX { - if strings.Contains(pathsubStr[2], ":") { + if strings.Contains(pathsubStr[2], "[") { tableName = strings.Split(pathsubStr[SONIC_TABLE_INDEX], "[")[0] } else { tableName = pathsubStr[SONIC_TABLE_INDEX] } + dbInfo, ok := xDbSpecMap[tableName] + cdb := db.ConfigDB + if !ok { + log.Errorf("No entry in xDbSpecMap for xpath %v in order to fetch DB index.", tableName) + } else { + cdb = dbInfo.dbIndex + } + dbOpts := getDBOptions(cdb) for i, kname := range rgp.FindAllString(path, -1) { - if i > 0 { keyStr += "|" } + if i > 0 { keyStr += dbOpts.KeySeparator } val := strings.Split(kname, "=")[1] keyStr += strings.TrimRight(val, "]") } diff --git a/src/translib/transformer/xlate_utils.go b/src/translib/transformer/xlate_utils.go index f4dc44340d..bb21f339a5 100644 --- a/src/translib/transformer/xlate_utils.go +++ b/src/translib/transformer/xlate_utils.go @@ -223,10 +223,10 @@ func isCvlYang(path string) bool { } func sonicKeyDataAdd(dbIndex db.DBNum, keyNameList []string, keyStr string, resultMap map[string]interface{}) { - keyValList := strings.Split(keyStr, ":") - if dbIndex == db.ConfigDB { - keyValList = strings.Split(keyStr, "|") - } + var dbOpts db.Options + dbOpts = getDBOptions(dbIndex) + keySeparator := dbOpts.KeySeparator + keyValList := strings.Split(keyStr, keySeparator) if len(keyNameList) != len(keyValList) { return @@ -406,3 +406,27 @@ func findInMap(m map[string]string, str string) string { // str doesn't exist in map m. return "" } + +func getDBOptions(dbNo db.DBNum) db.Options { + var opt db.Options + + switch dbNo { + case db.ApplDB, db.CountersDB: + opt = getDBOptionsWithSeparator(dbNo, "", ":", ":") + break + case db.FlexCounterDB, db.AsicDB, db.LogLevelDB, db.ConfigDB, db.StateDB: + opt = getDBOptionsWithSeparator(dbNo, "", "|", "|") + break + } + + return opt +} + +func getDBOptionsWithSeparator(dbNo db.DBNum, initIndicator string, tableSeparator string, keySeparator string) db.Options { + return(db.Options { + DBNo : dbNo, + InitIndicator : initIndicator, + TableNameSeparator: tableSeparator, + KeySeparator : keySeparator, + }) +} From e99533e3726fdbbda1a87f3a3bbeed27883d0fb6 Mon Sep 17 00:00:00 2001 From: Ranjini Nagaraj Date: Mon, 30 Sep 2019 14:26:07 -0700 Subject: [PATCH 13/25] Add checks for xpath entry in xYangSpecMap and xDbSpecMap --- src/translib/transformer/xlate.go | 108 +++++++++++++----------- src/translib/transformer/xlate_to_db.go | 28 +++--- 2 files changed, 75 insertions(+), 61 deletions(-) diff --git a/src/translib/transformer/xlate.go b/src/translib/transformer/xlate.go index 7453631df9..17dd5bf44a 100644 --- a/src/translib/transformer/xlate.go +++ b/src/translib/transformer/xlate.go @@ -133,46 +133,50 @@ func XlateUriToKeySpec(uri string, ygRoot *ygot.GoStruct, t *interface{}) (*[]Ke } func FillKeySpecs(yangXpath string , keyStr string, retdbFormat *[]KeySpec) ([]KeySpec){ - if xYangSpecMap == nil { - return *retdbFormat - } - _, ok := xYangSpecMap[yangXpath] - if ok { - xpathInfo := xYangSpecMap[yangXpath] - if xpathInfo.tableName != nil { - dbFormat := KeySpec{} - dbFormat.Ts.Name = *xpathInfo.tableName - dbFormat.dbNum = xpathInfo.dbIndex - if keyStr != "" { - dbFormat.Key.Comp = append(dbFormat.Key.Comp, keyStr) - } - for _, child := range xpathInfo.childTable { - if xDbSpecMap != nil { - chlen := len(xDbSpecMap[child].yangXpath) - if chlen > 0 { - children := make([]KeySpec, 0) - for _, childXpath := range xDbSpecMap[child].yangXpath { - children = FillKeySpecs(childXpath, "", &children) - dbFormat.Child = append(dbFormat.Child, children...) + if xYangSpecMap == nil { + return *retdbFormat + } + _, ok := xYangSpecMap[yangXpath] + if ok { + xpathInfo := xYangSpecMap[yangXpath] + if xpathInfo.tableName != nil { + dbFormat := KeySpec{} + dbFormat.Ts.Name = *xpathInfo.tableName + dbFormat.dbNum = xpathInfo.dbIndex + if keyStr != "" { + dbFormat.Key.Comp = append(dbFormat.Key.Comp, keyStr) } - } - } - } - *retdbFormat = append(*retdbFormat, dbFormat) - } else { - for _, child := range xpathInfo.childTable { - if xDbSpecMap != nil { - chlen := len(xDbSpecMap[child].yangXpath) - if chlen > 0 { - for _, childXpath := range xDbSpecMap[child].yangXpath { - *retdbFormat = FillKeySpecs(childXpath, "", retdbFormat) - } - } - } - } - } - } - return *retdbFormat + for _, child := range xpathInfo.childTable { + if xDbSpecMap != nil { + if _, ok := xDbSpecMap[child]; ok { + chlen := len(xDbSpecMap[child].yangXpath) + if chlen > 0 { + children := make([]KeySpec, 0) + for _, childXpath := range xDbSpecMap[child].yangXpath { + children = FillKeySpecs(childXpath, "", &children) + dbFormat.Child = append(dbFormat.Child, children...) + } + } + } + } + } + *retdbFormat = append(*retdbFormat, dbFormat) + } else { + for _, child := range xpathInfo.childTable { + if xDbSpecMap != nil { + if _, ok := xDbSpecMap[child]; ok { + chlen := len(xDbSpecMap[child].yangXpath) + if chlen > 0 { + for _, childXpath := range xDbSpecMap[child].yangXpath { + *retdbFormat = FillKeySpecs(childXpath, "", retdbFormat) + } + } + } + } + } + } + } + return *retdbFormat } func fillCvlKeySpec(xpath string , tableName string, keyStr string) ( []KeySpec ) { @@ -195,15 +199,19 @@ func fillCvlKeySpec(xpath string , tableName string, keyStr string) ( []KeySpec // If table name not available in xpath get top container name tokens:= strings.Split(xpath, ":") container := "/" + tokens[len(tokens)-1] - if xDbSpecMap[container] != nil { - dbInfo := xDbSpecMap[container] - if dbInfo.fieldType == "container" { - for dir, _ := range dbInfo.dbEntry.Dir { - cdb := xDbSpecMap[dir].dbIndex - dbFormat := KeySpec{} - dbFormat.Ts.Name = dir - dbFormat.dbNum = cdb - retdbFormat = append(retdbFormat, dbFormat) + if xDbSpecMap != nil { + if _, ok := xDbSpecMap[container]; ok { + dbInfo := xDbSpecMap[container] + if dbInfo.fieldType == "container" { + for dir, _ := range dbInfo.dbEntry.Dir { + if _, ok := xDbSpecMap[dir]; ok { + cdb := xDbSpecMap[dir].dbIndex + dbFormat := KeySpec{} + dbFormat.Ts.Name = dir + dbFormat.dbNum = cdb + retdbFormat = append(retdbFormat, dbFormat) + } + } } } } @@ -320,7 +328,9 @@ func XlateFromDb(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB, data m } } else { xpath, _ := RemoveXPATHPredicates(uri) - cdb = xYangSpecMap[xpath].dbIndex + if _, ok := xYangSpecMap[xpath]; ok { + cdb = xYangSpecMap[xpath].dbIndex + } } payload, err := dbDataToYangJsonCreate(uri, ygRoot, dbs, &dbData, cdb) log.Info("Payload generated:", payload) diff --git a/src/translib/transformer/xlate_to_db.go b/src/translib/transformer/xlate_to_db.go index 1cbe833915..1f51ce5d2d 100644 --- a/src/translib/transformer/xlate_to_db.go +++ b/src/translib/transformer/xlate_to_db.go @@ -62,10 +62,10 @@ func mapFillData(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, dbKey st var dbs [db.MaxDB]*db.DB var err error xpath := xpathPrefix + "/" + name - xpathInfo := xYangSpecMap[xpath] + xpathInfo, ok := xYangSpecMap[xpath] log.Infof("name: \"%v\", xpathPrefix(\"%v\").", name, xpathPrefix) - if xpathInfo == nil { + if !ok || xpathInfo == nil { log.Errorf("Yang path(\"%v\") not found.", xpath) return errors.New("Invalid URI") } @@ -293,7 +293,8 @@ func cvlYangReqToDbMapDelete(xpathPrefix string, tableName string, keyName strin // If table name not available in xpath get top container name tokens:= strings.Split(xpathPrefix, ":") container := "/" + tokens[len(tokens)-1] - if xDbSpecMap[container] != nil { + _, ok := xDbSpecMap[container] + if ok && xDbSpecMap[container] != nil { dbInfo := xDbSpecMap[container] if dbInfo.fieldType == "container" { for dir, _ := range dbInfo.dbEntry.Dir { @@ -385,7 +386,8 @@ func yangReqToDbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, for _, data := range dataMap { curKey := "" curUri, _ := uriWithKeyCreate(uri, xpathPrefix, data) - if len(xYangSpecMap[xpathPrefix].xfmrKey) > 0 { + _, ok := xYangSpecMap[xpathPrefix] + if ok && len(xYangSpecMap[xpathPrefix].xfmrKey) > 0 { /* key transformer present */ curYgotNode, nodeErr := yangNodeForUriGet(curUri, ygRoot) if nodeErr != nil { @@ -420,9 +422,10 @@ func yangReqToDbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, xpath = xpathPrefix + "/" + pathAttr curUri = uri + "/" + pathAttr } - log.Infof("slice/map data: curKey(\"%v\"), xpath(\"%v\"), curUri(\"%v\").", + _, ok := xYangSpecMap[xpath] + log.Infof("slice/map data: curKey(\"%v\"), xpath(\"%v\"), curUri(\"%v\").", curKey, xpath, curUri) - if len(xYangSpecMap[xpath].xfmrKey) > 0 { + if ok && xYangSpecMap[xpath] != nil && len(xYangSpecMap[xpath].xfmrKey) > 0 { /* key transformer present */ curYgotNode, nodeErr := yangNodeForUriGet(curUri, ygRoot) if nodeErr != nil { @@ -436,7 +439,7 @@ func yangReqToDbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, curKey = ret[0].Interface().(string) } if (typeOfValue == reflect.Map || typeOfValue == reflect.Slice) && xYangSpecMap[xpath].yangDataType != "leaf-list" { - if xYangSpecMap[xpath] != nil && len(xYangSpecMap[xpath].xfmrFunc) > 0 { + if ok && xYangSpecMap[xpath] != nil && len(xYangSpecMap[xpath].xfmrFunc) > 0 { /* subtree transformer present */ curYgotNode, nodeErr := yangNodeForUriGet(curUri, ygRoot) if nodeErr != nil { @@ -540,14 +543,15 @@ func xpathKeyExtract(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string) (st curPathWithKey += "/" } pfxPath, _ := RemoveXPATHPredicates(path) - tblPtr := xYangSpecMap[pfxPath].tableName - if tblPtr != nil { - tableName = *tblPtr - } else if xYangSpecMap[pfxPath].xfmrTbl != nil { + if _, ok := xYangSpecMap[pfxPath]; ok { + tblPtr := xYangSpecMap[pfxPath].tableName + if tblPtr != nil { + tableName = *tblPtr + } else if xYangSpecMap[pfxPath].xfmrTbl != nil { inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curPathWithKey, oper, "", nil, nil) tableName, _ = tblNameFromTblXfmrGet(*xYangSpecMap[pfxPath].xfmrTbl, inParams) } - + } return pfxPath, keyStr, tableName } From 7753aad4444ac9c0fc194f2beffd9ed156809203 Mon Sep 17 00:00:00 2001 From: Ranjini Nagaraj Date: Mon, 30 Sep 2019 16:34:15 -0700 Subject: [PATCH 14/25] Chk for list type to create keySpec --- src/translib/transformer/xlate.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/translib/transformer/xlate.go b/src/translib/transformer/xlate.go index 17dd5bf44a..5baec6c5cb 100644 --- a/src/translib/transformer/xlate.go +++ b/src/translib/transformer/xlate.go @@ -204,7 +204,8 @@ func fillCvlKeySpec(xpath string , tableName string, keyStr string) ( []KeySpec dbInfo := xDbSpecMap[container] if dbInfo.fieldType == "container" { for dir, _ := range dbInfo.dbEntry.Dir { - if _, ok := xDbSpecMap[dir]; ok { + _, ok := xDbSpecMap[dir] + if ok && xDbSpecMap[dir].dbEntry.Node.Statement().Keyword == "list" { cdb := xDbSpecMap[dir].dbIndex dbFormat := KeySpec{} dbFormat.Ts.Name = dir From 2f5eb5bb8888d4c83b7711d1753f9fe877b4c654 Mon Sep 17 00:00:00 2001 From: amrutasali Date: Mon, 30 Sep 2019 23:58:24 +0000 Subject: [PATCH 15/25] workaround for VLAN_MEMBER_TABLE list level get due to translib.generateGetResponsePayload() failure --- src/translib/common_app.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/translib/common_app.go b/src/translib/common_app.go index 2ec8cf10bd..8ab5fac646 100644 --- a/src/translib/common_app.go +++ b/src/translib/common_app.go @@ -172,15 +172,15 @@ func (app *CommonApp) processGet(dbs [db.MaxDB]*db.DB) (GetResponse, error) { } } - payload, err = generateGetResponsePayload(app.pathInfo.Path, (*app.ygotRoot).(*ocbinds.Device), app.ygotTarget) + resPayload, err := generateGetResponsePayload(app.pathInfo.Path, (*app.ygotRoot).(*ocbinds.Device), app.ygotTarget) if err != nil { log.Error("generateGetResponsePayload() failed") return GetResponse{Payload: payload, ErrSrc: AppErr}, err } var dat map[string]interface{} - err = json.Unmarshal(payload, &dat) + err = json.Unmarshal(resPayload, &dat) - return GetResponse{Payload: payload}, err + return GetResponse{Payload: resPayload}, err } func (app *CommonApp) translateCRUDCommon(d *db.DB, opcode int) ([]db.WatchKeys, error) { From 6ca2b16a7996a6a03988a96063a935afe6e0b38b Mon Sep 17 00:00:00 2001 From: s-mari Date: Mon, 30 Sep 2019 18:05:25 -0700 Subject: [PATCH 16/25] added key-name support for oc yang and support to create and push table with just the keys and no field-value data in to redis. --- src/translib/transformer/xlate_from_db.go | 13 ++++++++----- src/translib/transformer/xlate_to_db.go | 13 ++++++++++++- src/translib/transformer/xspec.go | 10 ++++++++++ 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/translib/transformer/xlate_from_db.go b/src/translib/transformer/xlate_from_db.go index ac25687f64..ae8518b132 100644 --- a/src/translib/transformer/xlate_from_db.go +++ b/src/translib/transformer/xlate_from_db.go @@ -44,13 +44,13 @@ func xfmrHandlerFunc(inParams XfmrParams) (map[string]interface{}, error) { } } - nodeList, nErr := ytypes.GetNode(schRoot, device, path) - if nErr != nil { + nodeList, nodeErr := ytypes.GetNode(schRoot, device, path) + if nodeErr != nil { log.Infof("Failed to get node for xpath(\"%v\") err(%v).", inParams.uri, err) return result, err } node := nodeList[0].Data - nodeYgot, _:= (node).(ygot.ValidatedGoStruct) + nodeYgot, _ := (node).(ygot.ValidatedGoStruct) payload, err := ygot.EmitJSON(nodeYgot, &ygot.EmitJSONConfig{ Format: ygot.RFC7951, Indent: " ", SkipValidation: true, RFC7951Config: &ygot.RFC7951JSONConfig{ AppendModuleName: false, }, @@ -216,9 +216,9 @@ func directDbToYangJsonCreate(dbDataMap *map[db.DBNum]map[string]map[string]db.V dbIndex := db.ConfigDB if ok { dbIndex = dbSpecData.dbIndex + yangKeys := yangKeyFromEntryGet(xDbSpecMap[tblName].dbEntry) + sonicKeyDataAdd(dbIndex, yangKeys, keyStr, curMap) } - yangKeys := yangKeyFromEntryGet(xDbSpecMap[tblName].dbEntry) - sonicKeyDataAdd(dbIndex, yangKeys, keyStr, curMap) if curMap != nil { mapSlice = append(mapSlice, curMap) } @@ -359,6 +359,9 @@ func terminalNodeProcess(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string } } else { dbFldName := xYangSpecMap[xpath].fieldName + if dbFldName == "NONE" { + return resFldValMap, err + } /* if there is no transformer extension/annotation then it means leaf-list in yang is also leaflist in db */ if len(dbFldName) > 0 && !xYangSpecMap[xpath].isKey { yangType := yangTypeGet(xYangSpecMap[xpath].yangEntry) diff --git a/src/translib/transformer/xlate_to_db.go b/src/translib/transformer/xlate_to_db.go index 7e4cb108b4..089ebd8c96 100644 --- a/src/translib/transformer/xlate_to_db.go +++ b/src/translib/transformer/xlate_to_db.go @@ -42,6 +42,10 @@ func dataToDBMapAdd(tableName string, dbKey string, result map[string]map[string result[tableName][dbKey] = db.Value{Field: make(map[string]string)} } + if field == "NONE" { + result[tableName][dbKey].Field["NULL"] = "NULL" + } + result[tableName][dbKey].Field[field] = value return } @@ -397,6 +401,8 @@ func yangReqToDbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, return err } curKey = ret[0].Interface().(string) + } else if xYangSpecMap[xpathPrefix].keyName != nil { + curKey = *xYangSpecMap[xpathPrefix].keyName } else { curKey = keyCreate(keyName, xpathPrefix, data, d.Opts.KeySeparator) } @@ -434,7 +440,10 @@ func yangReqToDbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, return err } curKey = ret[0].Interface().(string) + } else if xYangSpecMap[xpath].keyName != nil { + curKey = *xYangSpecMap[xpath].keyName } + if (typeOfValue == reflect.Map || typeOfValue == reflect.Slice) && xYangSpecMap[xpath].yangDataType != "leaf-list" { if xYangSpecMap[xpath] != nil && len(xYangSpecMap[xpath].xfmrFunc) > 0 { /* subtree transformer present */ @@ -520,7 +529,9 @@ func xpathKeyExtract(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string) (st return "", "", "" } keyStr = ret[0].Interface().(string) - } else { + } else if xYangSpecMap[yangXpath].keyName != nil { + keyStr += *xYangSpecMap[yangXpath].keyName + } else { var keyl []string for _, kname := range rgp.FindAllString(k, -1) { keyl = append(keyl, strings.TrimRight(strings.TrimLeft(kname, "["), "]")) diff --git a/src/translib/transformer/xspec.go b/src/translib/transformer/xspec.go index 3327805223..ed16d03405 100644 --- a/src/translib/transformer/xspec.go +++ b/src/translib/transformer/xspec.go @@ -25,6 +25,7 @@ type yangXpathInfo struct { xfmrPost string validateFunc string xfmrKey string + keyName *string dbIndex db.DBNum keyLevel int isKey bool @@ -283,6 +284,11 @@ func annotEntryFill(xYangSpecMap map[string]*yangXpathInfo, xpath string, entry *xpathData.tableName = ext.NName() updateDbTableData(xpath, xpathData, *xpathData.tableName) //childToUpdateParent(xpath, *xpathData.tableName) + case "key-name" : + if xpathData.keyName == nil { + xpathData.keyName = new(string) + } + *xpathData.keyName = ext.NName() case "table-transformer" : if xpathData.xfmrTbl == nil { xpathData.xfmrTbl = new(string) @@ -462,6 +468,10 @@ func mapPrint(inMap map[string]*yangXpathInfo, fileName string) { if d.xfmrTbl != nil { fmt.Fprintf(fp, "%v", *d.xfmrTbl) } + fmt.Fprintf(fp, "\r\n keyName : ") + if d.keyName != nil { + fmt.Fprintf(fp, "%v", *d.keyName) + } fmt.Fprintf(fp, "\r\n childTbl : %v", d.childTable) fmt.Fprintf(fp, "\r\n FieldName: %v", d.fieldName) fmt.Fprintf(fp, "\r\n keyLevel : %v", d.keyLevel) From ae33c02f6846c60cd10a0c13a2d14cf623c8d255 Mon Sep 17 00:00:00 2001 From: amrutasali Date: Tue, 1 Oct 2019 08:27:15 +0000 Subject: [PATCH 17/25] OC Yang Get case - DB type based key-delimiter support in transformer core --- .../annotations/openconfig-acl-annot.yang | 1 - src/translib/transformer/xlate_to_db.go | 50 +++++++++++++------ src/translib/transformer/xlate_utils.go | 23 ++------- 3 files changed, 40 insertions(+), 34 deletions(-) diff --git a/models/yang/annotations/openconfig-acl-annot.yang b/models/yang/annotations/openconfig-acl-annot.yang index 28b98e9b4e..0c9d7edd42 100644 --- a/models/yang/annotations/openconfig-acl-annot.yang +++ b/models/yang/annotations/openconfig-acl-annot.yang @@ -18,7 +18,6 @@ module openconfig-acl-annot { deviation /oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set { deviate add { sonic-ext:table-name "ACL_TABLE"; - sonic-ext:key-delimiter "_"; } } diff --git a/src/translib/transformer/xlate_to_db.go b/src/translib/transformer/xlate_to_db.go index 6e2e189325..18ea1d905a 100644 --- a/src/translib/transformer/xlate_to_db.go +++ b/src/translib/transformer/xlate_to_db.go @@ -519,15 +519,30 @@ func sonicXpathKeyExtract(path string) (string, string, string) { func xpathKeyExtract(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string) (string, string, string) { keyStr := "" tableName := "" + pfxPath := "" rgp := regexp.MustCompile(`\[([^\[\]]*)\]`) curPathWithKey := "" + cdb := db.ConfigDB var dbs [db.MaxDB]*db.DB + pfxPath, _ = RemoveXPATHPredicates(path) + xpathInfo, ok := xYangSpecMap[pfxPath] + if !ok { + log.Errorf("No entry found in xYangSpecMap for xpath %v.", pfxPath) + return pfxPath, keyStr, tableName + } + cdb = xpathInfo.dbIndex + dbOpts := getDBOptions(cdb) + keySeparator := dbOpts.KeySeparator + if len(xpathInfo.delim) > 0 { + keySeparator = xpathInfo.delim + } + for _, k := range strings.Split(path, "/") { curPathWithKey += k if strings.Contains(k, "[") { if len(keyStr) > 0 { - keyStr += "|" + keyStr += keySeparator } yangXpath, _ := RemoveXPATHPredicates(curPathWithKey) _, ok := xYangSpecMap[yangXpath] @@ -541,27 +556,32 @@ func xpathKeyExtract(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string) (st } keyStr = ret[0].Interface().(string) } else if xYangSpecMap[yangXpath].keyName != nil { - keyStr += *xYangSpecMap[yangXpath].keyName - } else { - var keyl []string - for _, kname := range rgp.FindAllString(k, -1) { - keyl = append(keyl, strings.TrimRight(strings.TrimLeft(kname, "["), "]")) + keyStr += *xYangSpecMap[yangXpath].keyName + } else { + /* multi-leaf yang key together forms a single key-string in redis. + There should be key-transformer, if not then the yang key leaves + will be concatenated with respective default DB type key-delimiter + */ + for idx, kname := range rgp.FindAllString(k, -1) { + if idx > 0 { keyStr += keySeparator } + keyl := strings.TrimRight(strings.TrimLeft(kname, "["), "]") + if strings.Contains(keyl, ":") { + keyl = strings.Split(keyl, ":")[1] + } + keyStr += strings.Split(keyl, "=")[1] } - keyStr += keyFromXpathCreate(keyl) } } } curPathWithKey += "/" } - pfxPath, _ := RemoveXPATHPredicates(path) - if _, ok := xYangSpecMap[pfxPath]; ok { - tblPtr := xYangSpecMap[pfxPath].tableName - if tblPtr != nil { - tableName = *tblPtr - } else if xYangSpecMap[pfxPath].xfmrTbl != nil { + //pfxPath, _ := RemoveXPATHPredicates(path) + tblPtr := xpathInfo.tableName + if tblPtr != nil { + tableName = *tblPtr + } else if xpathInfo.xfmrTbl != nil { inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curPathWithKey, oper, "", nil, nil) - tableName, _ = tblNameFromTblXfmrGet(*xYangSpecMap[pfxPath].xfmrTbl, inParams) - } + tableName, _ = tblNameFromTblXfmrGet(*xpathInfo.xfmrTbl, inParams) } return pfxPath, keyStr, tableName } diff --git a/src/translib/transformer/xlate_utils.go b/src/translib/transformer/xlate_utils.go index bb21f339a5..81e94f89b8 100644 --- a/src/translib/transformer/xlate_utils.go +++ b/src/translib/transformer/xlate_utils.go @@ -11,35 +11,22 @@ import ( log "github.com/golang/glog" ) -/* Create db key from datd xpath(request) */ -func keyFromXpathCreate(keyList []string) string { - keyOut := "" - for i, k := range keyList { - if i > 0 { keyOut += "_" } - if strings.Contains(k, ":") { - k = strings.Split(k, ":")[1] - } - keyOut += strings.Split(k, "=")[1] - } - return keyOut -} - /* Create db key from data xpath(request) */ func keyCreate(keyPrefix string, xpath string, data interface{}, dbKeySep string) string { _, ok := xYangSpecMap[xpath] if ok { if xYangSpecMap[xpath].yangEntry != nil { yangEntry := xYangSpecMap[xpath].yangEntry - keyConcat := dbKeySep + delim := dbKeySep if len(xYangSpecMap[xpath].delim) > 0 { - keyConcat = xYangSpecMap[xpath].delim - log.Infof("key concatenater(\"%v\") found for xpath %v ", keyConcat, xpath) + delim = xYangSpecMap[xpath].delim + log.Infof("key concatenater(\"%v\") found for xpath %v ", delim, xpath) } - if len(keyPrefix) > 0 { keyPrefix += dbKeySep } + if len(keyPrefix) > 0 { keyPrefix += delim } keyVal := "" for i, k := range (strings.Split(yangEntry.Key, " ")) { - if i > 0 { keyVal = keyVal + keyConcat } + if i > 0 { keyVal = keyVal + delim } val := fmt.Sprint(data.(map[string]interface{})[k]) if strings.Contains(val, ":") { val = strings.Split(val, ":")[1] From 57eaddede74d2cc2cd1f2b543af054341978af6a Mon Sep 17 00:00:00 2001 From: Ranjini Nagaraj Date: Tue, 1 Oct 2019 12:35:00 -0700 Subject: [PATCH 18/25] Add check for no xfmrFunc defined case --- src/translib/transformer/xlate_from_db.go | 22 ++++++++++++------- src/translib/transformer/xlate_to_db.go | 26 ++++++++++++++++------- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/src/translib/transformer/xlate_from_db.go b/src/translib/transformer/xlate_from_db.go index ae8518b132..72e8c1aa43 100644 --- a/src/translib/transformer/xlate_from_db.go +++ b/src/translib/transformer/xlate_from_db.go @@ -65,8 +65,12 @@ func leafXfmrHandlerFunc(inParams XfmrParams) (map[string]interface{}, error) { if err != nil { return nil, err } - fldValMap := ret[0].Interface().(map[string]interface{}) - return fldValMap, nil + if ret != nil { + fldValMap := ret[0].Interface().(map[string]interface{}) + return fldValMap, nil + } else { + return nil, nil + } } func validateHandlerFunc(inParams XfmrParams) (bool) { @@ -307,7 +311,7 @@ func yangListDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, x if len(xYangSpecMap[xpath].xfmrFunc) > 0 { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, curUri, GET, "", dbDataMap, nil) cmap, _ := xfmrHandlerFunc(inParams) - if len(cmap) > 0 { + if cmap != nil && len(cmap) > 0 { mapSlice = append(mapSlice, curMap) } else { log.Infof("Empty container returned from overloaded transformer for(\"%v\")", curUri) @@ -354,9 +358,11 @@ func terminalNodeProcess(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string err = fmt.Errorf("%v", logStr) return resFldValMap, err } - for lf, val := range fldValMap { + if fldValMap != nil { + for lf, val := range fldValMap { resFldValMap[lf] = val - } + } + } } else { dbFldName := xYangSpecMap[xpath].fieldName if dbFldName == "NONE" { @@ -426,7 +432,7 @@ func yangDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath if len(xYangSpecMap[chldXpath].xfmrFunc) > 0 { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, "", dbDataMap, nil) cmap, _ := xfmrHandlerFunc(inParams) - if len(cmap) > 0 { + if cmap != nil && len(cmap) > 0 { resultMap[cname] = cmap } else { log.Infof("Empty container(\"%v\").\r\n", chldUri) @@ -454,7 +460,7 @@ func yangDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath if len(xYangSpecMap[chldXpath].xfmrFunc) > 0 { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, "", dbDataMap, nil) cmap, _ := xfmrHandlerFunc(inParams) - if len(cmap) > 0 { + if cmap != nil && len(cmap) > 0 { resultMap = cmap } else { log.Infof("Empty list(\"%v\").\r\n", chldUri) @@ -516,7 +522,7 @@ func dbDataToYangJsonCreate(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db if len(xYangSpecMap[reqXpath].xfmrFunc) > 0 { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, "", dbDataMap, nil) cmap, _ = xfmrHandlerFunc(inParams) - if len(cmap) > 0 { + if cmap != nil && len(cmap) > 0 { resultMap[cname] = cmap } else { err := yangDataFill(dbs, ygRoot, uri, reqXpath, dbDataMap, resultMap, tableName, keyName, cdb, false) diff --git a/src/translib/transformer/xlate_to_db.go b/src/translib/transformer/xlate_to_db.go index 18ea1d905a..39cddcf57f 100644 --- a/src/translib/transformer/xlate_to_db.go +++ b/src/translib/transformer/xlate_to_db.go @@ -128,10 +128,12 @@ func mapFillData(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, dbKey st if err != nil { return err } - retData := ret[0].Interface().(map[string]string) - log.Info("Transformer function :", xpathInfo.xfmrFunc, " Xpath: ", xpath, " retData: ", retData) - for f, v := range retData { - dataToDBMapAdd(tableName, dbKey, result, f, v) + if ret != nil { + retData := ret[0].Interface().(map[string]string) + log.Info("Transformer function :", xpathInfo.xfmrFunc, " Xpath: ", xpath, " retData: ", retData) + for f, v := range retData { + dataToDBMapAdd(tableName, dbKey, result, f, v) + } } return nil } @@ -402,7 +404,9 @@ func yangReqToDbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, if err != nil { return err } - curKey = ret[0].Interface().(string) + if ret != nil { + curKey = ret[0].Interface().(string) + } } else if xYangSpecMap[xpathPrefix].keyName != nil { curKey = *xYangSpecMap[xpathPrefix].keyName } else { @@ -442,7 +446,9 @@ func yangReqToDbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, if err != nil { return err } - curKey = ret[0].Interface().(string) + if ret != nil { + curKey = ret[0].Interface().(string) + } } else if xYangSpecMap[xpath].keyName != nil { curKey = *xYangSpecMap[xpath].keyName } @@ -459,7 +465,9 @@ func yangReqToDbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, if err != nil { return nil } - mapCopy(result, ret[0].Interface().(map[string]map[string]db.Value)) + if ret != nil { + mapCopy(result, ret[0].Interface().(map[string]map[string]db.Value)) + } } else { yangReqToDbMapCreate(d, ygRoot, oper, curUri, xpath, curKey, jData.MapIndex(key).Interface(), result) } @@ -554,7 +562,9 @@ func xpathKeyExtract(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string) (st if err != nil { return "", "", "" } - keyStr = ret[0].Interface().(string) + if ret != nil { + keyStr = ret[0].Interface().(string) + } } else if xYangSpecMap[yangXpath].keyName != nil { keyStr += *xYangSpecMap[yangXpath].keyName } else { From 2fc250f495cf698e06484132111241ad5dd3eb76 Mon Sep 17 00:00:00 2001 From: amrutasali Date: Wed, 2 Oct 2019 15:20:26 +0000 Subject: [PATCH 19/25] Changed processLeafList() function name to checkAndProcessLeafList() , added comments and removed unnecessary logs in common app --- src/translib/common_app.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/translib/common_app.go b/src/translib/common_app.go index 8ab5fac646..5648acd798 100644 --- a/src/translib/common_app.go +++ b/src/translib/common_app.go @@ -8,7 +8,7 @@ import ( "github.com/openconfig/ygot/ygot" "reflect" "translib/db" - "translib/ocbinds" + "translib/ocbinds" "translib/tlerr" "translib/transformer" "encoding/json" @@ -291,11 +291,11 @@ func (app *CommonApp) cmnAppCRUCommonDbOpn(d *db.DB, opcode int) error { case UPDATE: if existingEntry.IsPopulated() { log.Info("Entry already exists hence modifying it.") - /* Handle leaf-list merge + /* Handle leaf-list merge if any leaf-list exists A leaf-list field in redis has "@" suffix as per swsssdk convention. */ resTblRw := db.Value{Field: map[string]string{}} - resTblRw = processLeafList(existingEntry, tblRw, UPDATE, d, tblNm, tblKey) + resTblRw = checkAndProcessLeafList(existingEntry, tblRw, UPDATE, d, tblNm, tblKey) err = d.ModEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}, resTblRw) if err != nil { log.Error("UPDATE case - d.ModEntry() failure") @@ -395,8 +395,8 @@ func (app *CommonApp) cmnAppDelDbOpn(d *db.DB, opcode int) error { log.Info("Table Entry from which the fields are to be deleted does not exist") return err } - /*handle leaf-list merge*/ - resTblRw := processLeafList(existingEntry, tblRw, DELETE, d, tblNm, tblKey) + /* handle leaf-list merge if any leaf-list exists */ + resTblRw := checkAndProcessLeafList(existingEntry, tblRw, DELETE, d, tblNm, tblKey) err := d.DeleteEntryFields(cmnAppTs, db.Key{Comp: []string{tblKey}}, resTblRw) if err != nil { log.Error("DELETE case - d.DeleteEntryFields() failure") @@ -417,8 +417,8 @@ func (app *CommonApp) generateDbWatchKeys(d *db.DB, isDeleteOp bool) ([]db.Watch return keys, err } -func processLeafList(existingEntry db.Value, tblRw db.Value, opcode int, d *db.DB, tblNm string, tblKey string) db.Value { - log.Info("process leaf-list Fields in table row.") +/*check if any field is leaf-list , if yes perform merge*/ +func checkAndProcessLeafList(existingEntry db.Value, tblRw db.Value, opcode int, d *db.DB, tblNm string, tblKey string) db.Value { dbTblSpec := &db.TableSpec{Name: tblNm} mergeTblRw := db.Value{Field: map[string]string{}} for field, value := range tblRw.Field { From f793f74a6bcfb6bb8743ed978e1ce181d872a050 Mon Sep 17 00:00:00 2001 From: s-mari Date: Wed, 2 Oct 2019 11:15:02 -0700 Subject: [PATCH 20/25] fix for vlan get-req crash --- src/translib/transformer/xlate_from_db.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/translib/transformer/xlate_from_db.go b/src/translib/transformer/xlate_from_db.go index 72e8c1aa43..62716c71e6 100644 --- a/src/translib/transformer/xlate_from_db.go +++ b/src/translib/transformer/xlate_from_db.go @@ -437,6 +437,7 @@ func yangDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath } else { log.Infof("Empty container(\"%v\").\r\n", chldUri) } + continue } else if xYangSpecMap[chldXpath].xfmrTbl != nil { xfmrTblFunc := *xYangSpecMap[chldXpath].xfmrTbl if len(xfmrTblFunc) > 0 { @@ -445,6 +446,9 @@ func yangDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath if len(tblList) > 1 { log.Warningf("Table transformer returned more than one table for container %v", chldXpath) } + if len(tblList) == 0 { + continue + } dbDataFromTblXfmrGet(tblList[0], inParams, dbDataMap) } } From 3e79fc5fd296500b0961bc7c10a624a20c94264e Mon Sep 17 00:00:00 2001 From: Ranjini Nagaraj Date: Wed, 2 Oct 2019 13:19:55 -0700 Subject: [PATCH 21/25] sonic yang get case handling for table instance level --- src/translib/common_app.go | 20 +++++++++------- src/translib/transformer/xlate_from_db.go | 29 +++++++++++++++++++---- 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/src/translib/common_app.go b/src/translib/common_app.go index 5648acd798..d6229b0a30 100644 --- a/src/translib/common_app.go +++ b/src/translib/common_app.go @@ -155,6 +155,7 @@ func (app *CommonApp) processDelete(d *db.DB) (SetResponse, error) { func (app *CommonApp) processGet(dbs [db.MaxDB]*db.DB) (GetResponse, error) { var err error var payload []byte + var resPayload []byte log.Info("processGet:path =", app.pathInfo.Path) payload, err = transformer.GetAndXlateFromDB(app.pathInfo.Path, app.ygotRoot, dbs) @@ -170,17 +171,20 @@ func (app *CommonApp) processGet(dbs [db.MaxDB]*db.DB) (GetResponse, error) { log.Error("ocbinds.Unmarshal() failed. error:", err) return GetResponse{Payload: payload, ErrSrc: AppErr}, err } - } - resPayload, err := generateGetResponsePayload(app.pathInfo.Path, (*app.ygotRoot).(*ocbinds.Device), app.ygotTarget) - if err != nil { - log.Error("generateGetResponsePayload() failed") - return GetResponse{Payload: payload, ErrSrc: AppErr}, err + resPayload, err = generateGetResponsePayload(app.pathInfo.Path, (*app.ygotRoot).(*ocbinds.Device), app.ygotTarget) + if err != nil { + log.Error("generateGetResponsePayload() failed") + return GetResponse{Payload: payload, ErrSrc: AppErr}, err + } + var dat map[string]interface{} + err = json.Unmarshal(resPayload, &dat) + } else { + log.Warning("processGet. targetObj is null. Unable to Unmarshal payload") + resPayload = payload } - var dat map[string]interface{} - err = json.Unmarshal(resPayload, &dat) - return GetResponse{Payload: resPayload}, err + return GetResponse{Payload: resPayload}, err } func (app *CommonApp) translateCRUDCommon(d *db.DB, opcode int) ([]db.WatchKeys, error) { diff --git a/src/translib/transformer/xlate_from_db.go b/src/translib/transformer/xlate_from_db.go index 62716c71e6..81ff39e81c 100644 --- a/src/translib/transformer/xlate_from_db.go +++ b/src/translib/transformer/xlate_from_db.go @@ -173,8 +173,13 @@ func processLfLstDbToYang(fieldXpath string, dbFldVal string) []interface{} { } /* Traverse db map and create json for cvl yang */ -func directDbToYangJsonCreate(dbDataMap *map[db.DBNum]map[string]map[string]db.Value, jsonData string, resultMap map[string]interface{}) error { +func directDbToYangJsonCreate(uri string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value) (string, error) { var err error + resultMap := make(map[string]interface{}) + instanceMap := make(map[string]interface{}) + terminalNode := false + _, key, table := sonicXpathKeyExtract(uri) + for curDbIdx := db.ApplDB; curDbIdx < db.MaxDB; curDbIdx++ { dbTblData := (*dbDataMap)[curDbIdx] for tblName, tblData := range dbTblData { @@ -213,12 +218,18 @@ func directDbToYangJsonCreate(dbDataMap *map[db.DBNum]map[string]map[string]db.V log.Warningf("Failure in converting Db value type to yang type for xpath", fieldXpath) } else { curMap[resField] = resVal + if table != "" && key != "" && table == tblName && key == keyStr { + instanceMap = curMap + if strings.Contains(uri, resField) { + terminalNode = true + } + } } } } //end of for dbSpecData, ok := xDbSpecMap[tblName] dbIndex := db.ConfigDB - if ok { + if ok && !terminalNode { dbIndex = dbSpecData.dbIndex yangKeys := yangKeyFromEntryGet(xDbSpecMap[tblName].dbEntry) sonicKeyDataAdd(dbIndex, yangKeys, keyStr, curMap) @@ -229,8 +240,16 @@ func directDbToYangJsonCreate(dbDataMap *map[db.DBNum]map[string]map[string]db.V } resultMap[tblName] = mapSlice } -} - return err + } + jsonMapData, _ := json.Marshal(resultMap) + if table != "" && key != "" && len(instanceMap) > 0 { + jsonMapData, _ = json.Marshal(instanceMap) + } + + jsonData := fmt.Sprintf("%v", string(jsonMapData)) + jsonDataPrint(jsonData) + return jsonData, err + } func tableNameAndKeyFromDbMapGet(dbDataMap map[string]map[string]db.Value) (string, string, error) { @@ -491,7 +510,7 @@ func dbDataToYangJsonCreate(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db jsonData := "" resultMap := make(map[string]interface{}) if isCvlYang(uri) { - directDbToYangJsonCreate(dbDataMap, jsonData, resultMap) + return directDbToYangJsonCreate(uri, dbDataMap) } else { var d *db.DB reqXpath, keyName, tableName := xpathKeyExtract(d, ygRoot, GET, uri) From a79ee264264ea0f10263bd2f1493320961d726a2 Mon Sep 17 00:00:00 2001 From: Ranjini Nagaraj Date: Wed, 2 Oct 2019 15:45:22 -0700 Subject: [PATCH 22/25] Handle nested db-name annotations for oc --- src/translib/transformer/xspec.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/translib/transformer/xspec.go b/src/translib/transformer/xspec.go index ed16d03405..8587a575ef 100644 --- a/src/translib/transformer/xspec.go +++ b/src/translib/transformer/xspec.go @@ -87,7 +87,8 @@ func yangToDbMapFill (keyLevel int, xYangSpecMap map[string]*yangXpathInfo, entr } } - if ok && parentXpathData.dbIndex != db.ConfigDB { + // If DB Index is not annotated and parent DB index is annotated inherit the DB Index of the parent + if ok && xpathData.dbIndex == db.ConfigDB && parentXpathData.dbIndex != db.ConfigDB { xpathData.dbIndex = parentXpathData.dbIndex } From 8d567e02960e79e6887ec5d2da05754421a04faf Mon Sep 17 00:00:00 2001 From: Ranjini Nagaraj Date: Wed, 2 Oct 2019 17:50:17 -0700 Subject: [PATCH 23/25] Import module name of the annotated module in annotate template file --- goyang-modified-files/annotate.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/goyang-modified-files/annotate.go b/goyang-modified-files/annotate.go index e86d441fad..a98ec7cbee 100644 --- a/goyang-modified-files/annotate.go +++ b/goyang-modified-files/annotate.go @@ -82,6 +82,8 @@ func generate(w io.Writer, e *yang.Entry, path string) { fmt.Fprintf(w, " import %s { prefix %s; }\n", k, allimports[k]) } } + // Include the module for which annotation is being generated + fmt.Fprintf(w, " import %s { prefix %s; }\n", e.Name, e.Prefix.Name) fmt.Fprintln(w) } @@ -92,13 +94,10 @@ func generate(w io.Writer, e *yang.Entry, path string) { name = e.Prefix.Name + ":" + name } - delim := "" - if path != "" { - delim = "/" + if (e.Node.Kind() != "module") { + path = path + "/" + name + printDeviation(w, path) } - path = path + delim + name - - printDeviation(w, path) var names []string for k := range e.Dir { From 8148935e6a1911d343728a7f35212352e0bd26ff Mon Sep 17 00:00:00 2001 From: Ravi Vasanthm Date: Wed, 2 Oct 2019 18:04:43 -0700 Subject: [PATCH 24/25] Added table transformer method prototype. --- src/translib/transformer/xfmr_interface.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/translib/transformer/xfmr_interface.go b/src/translib/transformer/xfmr_interface.go index 7109cf4de8..f9c8c6d3a3 100644 --- a/src/translib/transformer/xfmr_interface.go +++ b/src/translib/transformer/xfmr_interface.go @@ -78,6 +78,14 @@ type ValidateCallpoint func (inParams XfmrParams) (bool) type PostXfmrFunc func (inParams XfmrParams) (map[string]map[string]db.Value, error) +/** + * TableXfmrFunc type is defined to use for table transformer function for dynamic derviation of redis table. + * Param: XfmrParams structure having database pointers, current db, operation, DB data in multidimensional map, YgotRoot, uri + * Return: List of table names, error + **/ +type TableXfmrFunc func (inParams XfmrParams) ([]string, error) + + /** * Xfmr validation interface for validating the callback registration of app modules * transformer methods. @@ -104,3 +112,6 @@ func (SubTreeXfmrYangToDb) xfmrInterfaceValiidate () { func (SubTreeXfmrDbToYang) xfmrInterfaceValiidate () { log.Info("xfmrInterfaceValiidate for SubTreeXfmrDbToYang") } +func (TableXfmrFunc) xfmrInterfaceValiidate () { + log.Info("xfmrInterfaceValiidate for TableXfmrFunc") +} From ff13d87510a8b94c801117026696d6be7a0d980b Mon Sep 17 00:00:00 2001 From: Ravi Vasanthm Date: Wed, 2 Oct 2019 18:08:10 -0700 Subject: [PATCH 25/25] Added table transformer extension --- models/yang/common/sonic-extensions.yang | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/models/yang/common/sonic-extensions.yang b/models/yang/common/sonic-extensions.yang index ff3c315b3d..ced7029938 100644 --- a/models/yang/common/sonic-extensions.yang +++ b/models/yang/common/sonic-extensions.yang @@ -75,4 +75,9 @@ module sonic-extensions { argument "db-name"; description "DB name that will indicate where data is stored. Eg: Config DB, App DB etc"; } + extension table-transformer { + argument "table-transformer-name"; + description "Db table transformer name.This can be applied to either transform yang value to some different format + or choose a specific DB table based on the type."; + } }