diff --git a/Makefile b/Makefile index 29d8c73b7c..f78cee25bb 100644 --- a/Makefile +++ b/Makefile @@ -94,6 +94,7 @@ codegen: yamlGen: $(MAKE) -C models/yang + $(MAKE) -C models/yang/sonic go-patch: go-deps cd $(BUILD_GOPATH)/src/github.com/openconfig/ygot/; git reset --hard HEAD; git checkout 724a6b18a9224343ef04fe49199dfb6020ce132a 2>/dev/null ; true; \ @@ -120,6 +121,8 @@ install: $(INSTALL) -d $(DESTDIR)/usr/sbin/schema/ $(INSTALL) -d $(DESTDIR)/usr/sbin/lib/ $(INSTALL) -d $(DESTDIR)/usr/models/yang/ + $(INSTALL) -D $(TOPDIR)/models/yang/sonic/*.yang $(DESTDIR)/usr/models/yang/ + $(INSTALL) -D $(TOPDIR)/models/yang/sonic/common/*.yang $(DESTDIR)/usr/models/yang/ $(INSTALL) -D $(TOPDIR)/src/cvl/schema/*.yin $(DESTDIR)/usr/sbin/schema/ $(INSTALL) -D $(TOPDIR)/src/cvl/testdata/schema/*.yin $(DESTDIR)/usr/sbin/schema/ $(INSTALL) -D $(TOPDIR)/src/cvl/schema/*.yang $(DESTDIR)/usr/models/yang/ diff --git a/src/cvl/schema/Makefile b/src/cvl/schema/Makefile index 8f1276b68b..3af437be35 100644 --- a/src/cvl/schema/Makefile +++ b/src/cvl/schema/Makefile @@ -28,7 +28,8 @@ out_common=$(patsubst %.yang, %.yin, $(shell ls -1 $(sonic_yang_common)/*.yang | out_tree=$(patsubst %.yang, %.tree, $(src_files)) search_path=$(sonic_yang):$(sonic_yang_common):$(sonic_yang_common)/ietf -all: yamlGen allyangs.tree allyangs_tree.html schema +#all: yamlGen allyangs.tree allyangs_tree.html schema +all: schema schema: $(out) $(out_common) diff --git a/src/rest/Makefile b/src/rest/Makefile index 4b3d0055ec..0eb6901f44 100644 --- a/src/rest/Makefile +++ b/src/rest/Makefile @@ -56,6 +56,7 @@ $(REST_BUILD_DIR)/: # Invokes yang and model make to generate swagger artifcats. $(REST_BIN): $(REST_SRCS) | $(REST_BUILD_DIR)/ $(MAKE) -C $(TOPDIR)/models/yang + $(MAKE) -C $(TOPDIR)/models/yang/sonic $(MAKE) -C $(TOPDIR)/models ifeq ($(SONIC_COVERAGE_ON),y) GOPATH=$(REST_GOPATH) $(GO) test -coverpkg=".././..." -c -o $@ main/main.go main/main_test.go diff --git a/src/translib/common_app.go b/src/translib/common_app.go index d6229b0a30..e48c71cdcc 100644 --- a/src/translib/common_app.go +++ b/src/translib/common_app.go @@ -453,7 +453,7 @@ func checkAndProcessLeafList(existingEntry db.Value, tblRw db.Value, opcode int, } /* delete specific item from leaf-list */ if opcode == DELETE { - if mergeTblRw.Field == nil { + if len(mergeTblRw.Field) == 0 { return tblRw } err := d.ModEntry(dbTblSpec, db.Key{Comp: []string{tblKey}}, mergeTblRw) diff --git a/src/translib/transformer/xconst.go b/src/translib/transformer/xconst.go index a20d297934..bcc2d0ffb3 100644 --- a/src/translib/transformer/xconst.go +++ b/src/translib/transformer/xconst.go @@ -28,4 +28,6 @@ const ( XPATH_SEP_FWD_SLASH = "/" XFMR_EMPTY_STRING = "" + SONIC_TABLE_INDEX = 2 + ) diff --git a/src/translib/transformer/xlate.go b/src/translib/transformer/xlate.go index 5baec6c5cb..3c675c63c9 100644 --- a/src/translib/transformer/xlate.go +++ b/src/translib/transformer/xlate.go @@ -53,7 +53,7 @@ func XlateFuncBind(name string, fn interface{}) (err error) { func XlateFuncCall(name string, params ...interface{}) (result []reflect.Value, err error) { if _, ok := XlateFuncs[name]; !ok { err = errors.New(name + " Xfmr function does not exist.") - return nil, nil + return nil, err } if len(params) != XlateFuncs[name].Type().NumIn() { err = ErrParamsNotAdapted @@ -118,11 +118,11 @@ func XlateUriToKeySpec(uri string, ygRoot *ygot.GoStruct, t *interface{}) (*[]Ke var err error var retdbFormat = make([]KeySpec, 0) - // In case of CVL yang, the tablename and key info is available in the xpath - if isCvlYang(uri) { + // In case of SONIC yang, the tablename and key info is available in the xpath + if isSonicYang(uri) { /* Extract the xpath and key from input xpath */ xpath, keyStr, tableName := sonicXpathKeyExtract(uri) - retdbFormat = fillCvlKeySpec(xpath, tableName, keyStr) + retdbFormat = fillSonicKeySpec(xpath, tableName, keyStr) } else { /* Extract the xpath and key from input xpath */ xpath, keyStr, _ := xpathKeyExtract(nil, ygRoot, 0, uri) @@ -179,7 +179,7 @@ func FillKeySpecs(yangXpath string , keyStr string, retdbFormat *[]KeySpec) ([]K return *retdbFormat } -func fillCvlKeySpec(xpath string , tableName string, keyStr string) ( []KeySpec ) { +func fillSonicKeySpec(xpath string , tableName string, keyStr string) ( []KeySpec ) { var retdbFormat = make([]KeySpec, 0) @@ -197,15 +197,14 @@ func fillCvlKeySpec(xpath string , tableName string, keyStr string) ( []KeySpec retdbFormat = append(retdbFormat, dbFormat) } else { // If table name not available in xpath get top container name - tokens:= strings.Split(xpath, ":") - container := "/" + tokens[len(tokens)-1] + container := xpath if xDbSpecMap != nil { if _, ok := xDbSpecMap[container]; ok { dbInfo := xDbSpecMap[container] if dbInfo.fieldType == "container" { for dir, _ := range dbInfo.dbEntry.Dir { _, ok := xDbSpecMap[dir] - if ok && xDbSpecMap[dir].dbEntry.Node.Statement().Keyword == "list" { + if ok && xDbSpecMap[dir].dbEntry.Node.Statement().Keyword == "container" { cdb := xDbSpecMap[dir].dbIndex dbFormat := KeySpec{} dbFormat.Ts.Name = dir @@ -311,7 +310,7 @@ func XlateFromDb(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB, data m var cdb db.DBNum = db.ConfigDB dbData = data - if isCvlYang(uri) { + if isSonicYang(uri) { xpath, keyStr, tableName := sonicXpathKeyExtract(uri) if (tableName != "") { dbInfo, ok := xDbSpecMap[tableName] @@ -321,10 +320,14 @@ func XlateFromDb(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB, data m cdb = dbInfo.dbIndex } tokens:= strings.Split(xpath, "/") - // Format /module:container/tableName[key]/fieldName - if tokens[len(tokens)-2] == tableName { + // Format /module:container/tableName/listname[key]/fieldName + if tokens[SONIC_TABLE_INDEX] == tableName { fieldName := tokens[len(tokens)-1] - dbData[cdb] = extractFieldFromDb(tableName, keyStr, fieldName, data[cdb]) + dbSpecField := tableName + "/" + fieldName + _, ok := xDbSpecMap[dbSpecField] + if ok && xDbSpecMap[dbSpecField].fieldType == "leaf" { + dbData[cdb] = extractFieldFromDb(tableName, keyStr, fieldName, data[cdb]) + } } } } else { diff --git a/src/translib/transformer/xlate_from_db.go b/src/translib/transformer/xlate_from_db.go index 81ff39e81c..78e6b19bca 100644 --- a/src/translib/transformer/xlate_from_db.go +++ b/src/translib/transformer/xlate_from_db.go @@ -61,6 +61,9 @@ func xfmrHandlerFunc(inParams XfmrParams) (map[string]interface{}, error) { func leafXfmrHandlerFunc(inParams XfmrParams) (map[string]interface{}, error) { xpath, _ := RemoveXPATHPredicates(inParams.uri) + if strings.Count(xpath, ":") > 1 { + xpath = stripAugmentedModuleNames(xpath) + } ret, err := XlateFuncCall(dbToYangXfmrFunc(xYangSpecMap[xpath].xfmrFunc), inParams) if err != nil { return nil, err @@ -172,84 +175,163 @@ func processLfLstDbToYang(fieldXpath string, dbFldVal string) []interface{} { return resLst } -/* Traverse db map and create json for cvl yang */ -func directDbToYangJsonCreate(uri string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value) (string, error) { - var err error - resultMap := make(map[string]interface{}) - instanceMap := make(map[string]interface{}) - terminalNode := false - _, key, table := sonicXpathKeyExtract(uri) - - for curDbIdx := db.ApplDB; curDbIdx < db.MaxDB; curDbIdx++ { - dbTblData := (*dbDataMap)[curDbIdx] - for tblName, tblData := range dbTblData { - var mapSlice []typeMapOfInterface - for keyStr, dbFldValData := range tblData { - curMap := make(map[string]interface{}) - for field, value := range dbFldValData.Field { - resField := field - if strings.HasSuffix(field, "@") { - fldVals := strings.Split(field, "@") - resField = fldVals[0] - } - fieldXpath := tblName + "/" + resField - xDbSpecMapEntry, ok := xDbSpecMap[fieldXpath] - if !ok { - log.Warningf("No entry found in xDbSpecMap for xpath %v", fieldXpath) - continue - } - if xDbSpecMapEntry.dbEntry == nil { - log.Warningf("Yang entry is nil in xDbSpecMap for xpath %v", fieldXpath) - continue +func sonicDbToYangTerminalNodeFill(tblName string, field string, value string, resultMap map[string]interface{}) { + resField := field + if len(value) == 0 { + return + } + if strings.HasSuffix(field, "@") { + fldVals := strings.Split(field, "@") + resField = fldVals[0] + } + fieldXpath := tblName + "/" + resField + xDbSpecMapEntry, ok := xDbSpecMap[fieldXpath] + if !ok { + log.Warningf("No entry found in xDbSpecMap for xpath %v", fieldXpath) + return + } + if xDbSpecMapEntry.dbEntry == nil { + log.Warningf("Yang entry is nil in xDbSpecMap for xpath %v", fieldXpath) + return + } + + yangType := yangTypeGet(xDbSpecMapEntry.dbEntry) + if yangType == "leaf-list" { + /* this should never happen but just adding for safetty */ + if !strings.HasSuffix(field, "@") { + log.Warningf("Leaf-list in Sonic yang should also be a leaf-list in DB, its not for xpath %v", fieldXpath) + return + } + resLst := processLfLstDbToYang(fieldXpath, value) + resultMap[resField] = resLst + } else { /* yangType is leaf - there are only 2 types of yang terminal node leaf and leaf-list */ + yngTerminalNdDtType := xDbSpecMapEntry.dbEntry.Type.Kind + resVal, err := DbToYangType(yngTerminalNdDtType, fieldXpath, value) + if err != nil { + log.Warningf("Failure in converting Db value type to yang type for xpath", fieldXpath) + } else { + resultMap[resField] = resVal + } + } + return +} + +func sonicDbToYangListFill(uri string, xpath string, dbIdx db.DBNum, table string, key string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value) []typeMapOfInterface { + var mapSlice []typeMapOfInterface + dbTblData := (*dbDataMap)[dbIdx][table] + + for keyStr, _ := range dbTblData { + curMap := make(map[string]interface{}) + sonicDbToYangDataFill(uri, xpath, dbIdx, table, keyStr, dbDataMap, curMap) + dbSpecData, ok := xDbSpecMap[table] + if ok && dbSpecData.keyName == nil { + yangKeys := yangKeyFromEntryGet(xDbSpecMap[xpath].dbEntry) + sonicKeyDataAdd(dbIdx, yangKeys, keyStr, curMap) + } + if curMap != nil { + mapSlice = append(mapSlice, curMap) + } + } + return mapSlice +} + +func sonicDbToYangDataFill(uri string, xpath string, dbIdx db.DBNum, table string, key string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, resultMap map[string]interface{}) { + yangNode, ok := xDbSpecMap[xpath] + + if ok && yangNode.dbEntry != nil { + xpathPrefix := table + if len(table) > 0 { xpathPrefix += "/" } + + for yangChldName := range yangNode.dbEntry.Dir { + chldXpath := xpathPrefix+yangChldName + if xDbSpecMap[chldXpath] != nil && xDbSpecMap[chldXpath].dbEntry != nil { + chldYangType := yangTypeGet(xDbSpecMap[chldXpath].dbEntry) + + if chldYangType == YANG_LEAF || chldYangType == YANG_LEAF_LIST { + log.Infof("tbl(%v), k(%v), yc(%v)", table, key, yangChldName) + sonicDbToYangTerminalNodeFill(table, yangChldName, (*dbDataMap)[dbIdx][table][key].Field[yangChldName], resultMap) + } else if chldYangType == YANG_CONTAINER { + curMap := make(map[string]interface{}) + curUri := xpath + "/" + yangChldName + // container can have a static key, so extract key for current container + _, curKey, curTable := sonicXpathKeyExtract(curUri) + // use table-name as xpath from now on + sonicDbToYangDataFill(curUri, curTable, xDbSpecMap[curTable].dbIndex, curTable, curKey, dbDataMap, curMap) + if len(curMap) > 0 { + resultMap[yangChldName] = curMap + } else { + log.Infof("Empty container for xpath(%v)", curUri) } - yangType := yangTypeGet(xDbSpecMapEntry.dbEntry) - if yangType == "leaf-list" { - /* this should never happen but just adding for safetty */ - if !strings.HasSuffix(field, "@") { - log.Warningf("Leaf-list in Sonic yang should also be a leaf-list in DB, its not for xpath %v", fieldXpath) - continue - } - resLst := processLfLstDbToYang(fieldXpath, value) - curMap[resField] = resLst - } else { /* yangType is leaf - there are only 2 types of yang terminal node leaf and leaf-list */ - yngTerminalNdDtType := xDbSpecMapEntry.dbEntry.Type.Kind - resVal, err := DbToYangType(yngTerminalNdDtType, fieldXpath, value) - if err != nil { - log.Warningf("Failure in converting Db value type to yang type for xpath", fieldXpath) + } else if chldYangType == YANG_LIST { + var mapSlice []typeMapOfInterface + curUri := xpath + "/" + yangChldName + mapSlice = sonicDbToYangListFill(curUri, curUri, dbIdx, table, key, dbDataMap) + if len(mapSlice) > 0 { + resultMap[yangChldName] = mapSlice } else { - curMap[resField] = resVal - if table != "" && key != "" && table == tblName && key == keyStr { - instanceMap = curMap - if strings.Contains(uri, resField) { - terminalNode = true - } - } + log.Infof("Empty list for xpath(%v)", curUri) } } - } //end of for - dbSpecData, ok := xDbSpecMap[tblName] - dbIndex := db.ConfigDB - if ok && !terminalNode { - dbIndex = dbSpecData.dbIndex - yangKeys := yangKeyFromEntryGet(xDbSpecMap[tblName].dbEntry) - sonicKeyDataAdd(dbIndex, yangKeys, keyStr, curMap) - } - if curMap != nil { - mapSlice = append(mapSlice, curMap) } } - resultMap[tblName] = mapSlice - } } - jsonMapData, _ := json.Marshal(resultMap) - if table != "" && key != "" && len(instanceMap) > 0 { - jsonMapData, _ = json.Marshal(instanceMap) - } + return +} - jsonData := fmt.Sprintf("%v", string(jsonMapData)) - jsonDataPrint(jsonData) - return jsonData, err +/* Traverse db map and create json for cvl yang */ +func directDbToYangJsonCreate(uri string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, resultMap map[string]interface{}) (string, error) { + xpath, key, table := sonicXpathKeyExtract(uri) + + if len(xpath) > 0 { + var dbNode *dbInfo + + if len(table) > 0 { + tokens:= strings.Split(xpath, "/") + if tokens[SONIC_TABLE_INDEX] == table { + fieldName := tokens[len(tokens)-1] + dbSpecField := table + "/" + fieldName + _, ok := xDbSpecMap[dbSpecField] + if ok && xDbSpecMap[dbSpecField].fieldType == "leaf" { + dbNode = xDbSpecMap[dbSpecField] + xpath = dbSpecField + } else { + dbNode = xDbSpecMap[table] + } + } + } else { + dbNode, _ = xDbSpecMap[xpath] + } + + if dbNode != nil && dbNode.dbEntry != nil { + cdb := db.ConfigDB + yangType := yangTypeGet(dbNode.dbEntry) + if len(table) > 0 { + cdb = xDbSpecMap[table].dbIndex + } + if yangType == YANG_LEAF || yangType == YANG_LEAF_LIST { + fldName := xDbSpecMap[xpath].dbEntry.Name + sonicDbToYangTerminalNodeFill(table, fldName, (*dbDataMap)[cdb][table][key].Field[fldName], resultMap) + } else if yangType == YANG_CONTAINER { + if len(table) > 0 { + xpath = table + } + sonicDbToYangDataFill(uri, xpath, cdb, table, key, dbDataMap, resultMap) + } else if yangType == YANG_LIST { + mapSlice := sonicDbToYangListFill(uri, xpath, cdb, table, key, dbDataMap) + if len(mapSlice) > 0 { + pathl := strings.Split(xpath, "/") + lname := pathl[len(pathl) - 1] + resultMap[lname] = mapSlice + } + } + } + } + + jsonMapData, _ := json.Marshal(resultMap) + jsonData := fmt.Sprintf("%v", string(jsonMapData)) + jsonDataPrint(jsonData) + return jsonData, nil } func tableNameAndKeyFromDbMapGet(dbDataMap map[string]map[string]db.Value) (string, string, error) { @@ -369,8 +451,7 @@ func terminalNodeProcess(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string cdb := xYangSpecMap[xpath].dbIndex if len(xYangSpecMap[xpath].xfmrFunc) > 0 { - _, key, _ := xpathKeyExtract(dbs[cdb], ygRoot, GET, uri) - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, key, dbDataMap, nil) + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, tblKey, dbDataMap, nil) fldValMap, err := leafXfmrHandlerFunc(inParams) if err != nil { logStr := fmt.Sprintf("%Failed to get data from overloaded function for %v -v.", uri, err) @@ -414,7 +495,6 @@ func terminalNodeProcess(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string return resFldValMap, err } - func yangDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, resultMap map[string]interface{}, tbl string, tblKey string, cdb db.DBNum, validate bool) error { var err error isValid := validate @@ -425,8 +505,9 @@ func yangDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath chldXpath := xpath+"/"+yangChldName chldUri := uri+"/"+yangChldName if xYangSpecMap[chldXpath] != nil && xYangSpecMap[chldXpath].yangEntry != nil { - _, key, _ := xpathKeyExtract(dbs[cdb], ygRoot, GET, chldUri) + cdb = xYangSpecMap[chldXpath].dbIndex if len(xYangSpecMap[chldXpath].validateFunc) > 0 && !validate { + _, key, _ := xpathKeyExtract(dbs[cdb], ygRoot, GET, chldUri) // TODO - handle non CONFIG-DB inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, key, dbDataMap, nil) res := validateHandlerFunc(inParams) @@ -437,7 +518,6 @@ func yangDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath } } chldYangType := yangTypeGet(xYangSpecMap[chldXpath].yangEntry) - cdb = xYangSpecMap[chldXpath].dbIndex if chldYangType == "leaf" || chldYangType == "leaf-list" { fldValMap, err := terminalNodeProcess(dbs, ygRoot, chldUri, chldXpath, dbDataMap, tbl, tblKey) if err != nil { @@ -448,16 +528,7 @@ func yangDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath } } else if chldYangType == "container" { cname := xYangSpecMap[chldXpath].yangEntry.Name - if len(xYangSpecMap[chldXpath].xfmrFunc) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, "", dbDataMap, nil) - cmap, _ := xfmrHandlerFunc(inParams) - if cmap != nil && len(cmap) > 0 { - resultMap[cname] = cmap - } else { - log.Infof("Empty container(\"%v\").\r\n", chldUri) - } - continue - } else if xYangSpecMap[chldXpath].xfmrTbl != nil { + if xYangSpecMap[chldXpath].xfmrTbl != nil { xfmrTblFunc := *xYangSpecMap[chldXpath].xfmrTbl if len(xfmrTblFunc) > 0 { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, "", dbDataMap, nil) @@ -469,14 +540,26 @@ func yangDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath continue } dbDataFromTblXfmrGet(tblList[0], inParams, dbDataMap) + tbl = tblList[0] } } - cmap := make(map[string]interface{}) - err = yangDataFill(dbs, ygRoot, chldUri, chldXpath, dbDataMap, cmap, tbl, tblKey, cdb, isValid) - if len(cmap) > 0 { - resultMap[cname] = cmap + if len(xYangSpecMap[chldXpath].xfmrFunc) > 0 { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, "", dbDataMap, nil) + cmap, _ := xfmrHandlerFunc(inParams) + if cmap != nil && len(cmap) > 0 { + resultMap[cname] = cmap + } else { + log.Infof("Empty container(\"%v\").\r\n", chldUri) + } + continue } else { - log.Infof("Empty container(\"%v\").\r\n", chldUri) + cmap := make(map[string]interface{}) + err = yangDataFill(dbs, ygRoot, chldUri, chldXpath, dbDataMap, cmap, tbl, tblKey, cdb, isValid) + if len(cmap) > 0 { + resultMap[cname] = cmap + } else { + log.Infof("Empty container(\"%v\").\r\n", chldUri) + } } } else if chldYangType == "list" { cdb = xYangSpecMap[chldXpath].dbIndex @@ -509,8 +592,8 @@ func yangDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath func dbDataToYangJsonCreate(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, cdb db.DBNum) (string, error) { jsonData := "" resultMap := make(map[string]interface{}) - if isCvlYang(uri) { - return directDbToYangJsonCreate(uri, dbDataMap) + if isSonicYang(uri) { + return directDbToYangJsonCreate(uri, dbDataMap, resultMap) } else { var d *db.DB reqXpath, keyName, tableName := xpathKeyExtract(d, ygRoot, GET, uri) @@ -542,18 +625,38 @@ func dbDataToYangJsonCreate(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db } else if yangType == "container" { cname := xYangSpecMap[reqXpath].yangEntry.Name cmap := make(map[string]interface{}) + IsYangDataFill := true if len(xYangSpecMap[reqXpath].xfmrFunc) > 0 { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, "", dbDataMap, nil) cmap, _ = xfmrHandlerFunc(inParams) if cmap != nil && len(cmap) > 0 { resultMap[cname] = cmap - } else { - err := yangDataFill(dbs, ygRoot, uri, reqXpath, dbDataMap, resultMap, tableName, keyName, cdb, false) - if err != nil { - log.Infof("Empty container(\"%v\").\r\n", uri) + IsYangDataFill = false + } + } else if xYangSpecMap[reqXpath].xfmrTbl != nil { + xfmrTblFunc := *xYangSpecMap[reqXpath].xfmrTbl + if len(xfmrTblFunc) > 0 { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, "", dbDataMap, nil) + tblList := xfmrTblHandlerFunc(xfmrTblFunc, inParams) + if len(tblList) > 1 { + log.Warningf("Table transformer returned more than one table for container %v", reqXpath) + IsYangDataFill = false + } + if len(tblList) != 0 { + dbDataFromTblXfmrGet(tblList[0], inParams, dbDataMap) + err := yangDataFill(dbs, ygRoot, uri, reqXpath, dbDataMap, resultMap, tableName, keyName, cdb, false) + if err != nil { + log.Infof("Empty container(\"%v\").\r\n", uri) + } + } + + } else { + log.Warningf("empty table transformer function name for xpath - %v", reqXpath) + IsYangDataFill = false } - } else { + } + if IsYangDataFill { err := yangDataFill(dbs, ygRoot, uri, reqXpath, dbDataMap, resultMap, tableName, keyName, cdb, false) if err != nil { log.Infof("Empty container(\"%v\").\r\n", uri) diff --git a/src/translib/transformer/xlate_to_db.go b/src/translib/transformer/xlate_to_db.go index 39cddcf57f..b6fe747d84 100644 --- a/src/translib/transformer/xlate_to_db.go +++ b/src/translib/transformer/xlate_to_db.go @@ -11,13 +11,11 @@ import ( "translib/db" "translib/ocbinds" "github.com/openconfig/ygot/ytypes" + "github.com/openconfig/goyang/pkg/yang" log "github.com/golang/glog" ) -const SONIC_TABLE_INDEX = 2 -const SONIC_FIELD_INDEX = 3 - /* Invoke the post tansformer */ func postXfmrHandlerFunc(inParams XfmrParams) (map[string]map[string]db.Value, error) { xpath, _ := RemoveXPATHPredicates(inParams.uri) @@ -177,80 +175,109 @@ func mapFillData(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, dbKey st return nil } -func cvlYangReqToDbMapCreate(jsonData interface{}, result map[string]map[string]db.Value) error { +func sonicYangReqToDbMapCreate(jsonData interface{}, result map[string]map[string]db.Value) error { if reflect.ValueOf(jsonData).Kind() == reflect.Map { data := reflect.ValueOf(jsonData) for _, key := range data.MapKeys() { _, ok := xDbSpecMap[key.String()] if ok { - directDbMapData(key.String(), data.MapIndex(key).Interface(), result) + directDbMapData("", key.String(), data.MapIndex(key).Interface(), result) } else { - cvlYangReqToDbMapCreate(data.MapIndex(key).Interface(), result) + sonicYangReqToDbMapCreate(data.MapIndex(key).Interface(), result) } } } return nil } -func directDbMapData(tableName string, jsonData interface{}, result map[string]map[string]db.Value) bool { - _, ok := xDbSpecMap[tableName] +func dbMapDataFill(uri string, tableName string, keyName string, d map[string]interface{}, result map[string]map[string]db.Value) { + result[tableName][keyName] = db.Value{Field: make(map[string]string)} + for field, value := range d { + fieldXpath := tableName + "/" + field + if _, fieldOk := xDbSpecMap[fieldXpath]; (fieldOk && (xDbSpecMap[fieldXpath].dbEntry != nil)) { + log.Info("Found non-nil yang entry in xDbSpecMap for field xpath = ", fieldXpath) + if xDbSpecMap[fieldXpath].dbEntry.IsLeafList() { + log.Info("Yang type is Leaflist for field = ", field) + field += "@" + fieldDt := reflect.ValueOf(value) + fieldValue := "" + for fidx := 0; fidx < fieldDt.Len(); fidx++ { + if fidx > 0 { + fieldValue += "," + } + fVal := fmt.Sprintf("%v", fieldDt.Index(fidx).Interface()) + fieldValue = fieldValue + fVal + } + result[tableName][keyName].Field[field] = fieldValue + continue + } + } else { + // should ideally never happen , just adding for safety + log.Info("Did not find entry in xDbSpecMap for field xpath = ", fieldXpath) + } + result[tableName][keyName].Field[field] = fmt.Sprintf("%v", value) + } + return +} - if ok && xDbSpecMap[tableName].dbEntry != nil { - dbSpecData := xDbSpecMap[tableName].dbEntry - tblKeyName := strings.Split(dbSpecData.Key, " ") - data := reflect.ValueOf(jsonData) - result[tableName] = make(map[string]db.Value) +func dbMapListDataFill(uri string, tableName string, dbEntry *yang.Entry, jsonData interface{}, result map[string]map[string]db.Value) { + data := reflect.ValueOf(jsonData) + tblKeyName := strings.Split(dbEntry.Key, " ") + for idx := 0; idx < data.Len(); idx++ { + keyName := "" + d := data.Index(idx).Interface().(map[string]interface{}) + for i, k := range tblKeyName { + if i > 0 { + keyName += "|" + } + keyName += fmt.Sprintf("%v", d[k]) + delete(d, k) + } + dbMapDataFill(uri, tableName, keyName, d, result) + } + return +} - for idx := 0; idx < data.Len(); idx++ { - keyName := "" - d := data.Index(idx).Interface().(map[string]interface{}) - for i, k := range tblKeyName { - if i > 0 { - keyName += "|" - } - keyName += fmt.Sprintf("%v", d[k]) - delete(d, k) - } +func directDbMapData(uri string, tableName string, jsonData interface{}, result map[string]map[string]db.Value) bool { + _, ok := xDbSpecMap[tableName] + if ok && xDbSpecMap[tableName].dbEntry != nil { + data := reflect.ValueOf(jsonData).Interface().(map[string]interface{}) + key := "" + dbSpecData := xDbSpecMap[tableName] + result[tableName] = make(map[string]db.Value) + + if dbSpecData.keyName != nil { + key = *dbSpecData.keyName + log.Infof("Fill data for container uri(%v), key(%v)", uri, key) + dbMapDataFill(uri, tableName, key, data, result) + return true + } - result[tableName][keyName] = db.Value{Field: make(map[string]string)} - for field, value := range d { - fieldXpath := tableName + "/" + field - if _, fieldOk := xDbSpecMap[fieldXpath]; (fieldOk && (xDbSpecMap[fieldXpath].dbEntry != nil)) { - log.Info("Found non-nil yang entry in xDbSpecMap for field xpath = ", fieldXpath) - if xDbSpecMap[fieldXpath].dbEntry.IsLeafList() { - log.Info("Yang type is Leaflist for field = ", field) - field += "@" - fieldDt := reflect.ValueOf(value) - fieldValue := "" - for fidx := 0; fidx < fieldDt.Len(); fidx++ { - if fidx > 0 { - fieldValue += "," - } - fVal := fmt.Sprintf("%v", fieldDt.Index(fidx).Interface()) - fieldValue = fieldValue + fVal - } - result[tableName][keyName].Field[field] = fieldValue - continue - } - } else { - // should ideally never happen , just adding for safety - log.Info("Did not find entry in xDbSpecMap for field xpath = ", fieldXpath) - } - result[tableName][keyName].Field[field] = fmt.Sprintf("%v", value) - } - } - return true - } - return false + for k, v := range data { + xpath := tableName + "/" + k + curDbSpecData, ok := xDbSpecMap[xpath] + if ok && curDbSpecData.dbEntry != nil { + eType := yangTypeGet(curDbSpecData.dbEntry) + switch eType { + case "list": + log.Infof("Fill data for list uri(%v)", uri) + dbMapListDataFill(uri, tableName, curDbSpecData.dbEntry, v, result) + default: + log.Infof("Invalid node type for uri(%v)", uri) + } + } + } + } + return true } /* Get the db table, key and field name for the incoming delete request */ func dbMapDelete(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string, jsonData interface{}, result map[string]map[string]db.Value) error { var err error - if isCvlYang(path) { + if isSonicYang(path) { xpathPrefix, keyName, tableName := sonicXpathKeyExtract(path) log.Infof("Delete req: path(\"%v\"), key(\"%v\"), xpathPrefix(\"%v\"), tableName(\"%v\").", path, keyName, xpathPrefix, tableName) - err = cvlYangReqToDbMapDelete(xpathPrefix, tableName, keyName, result) + err = sonicYangReqToDbMapDelete(xpathPrefix, tableName, keyName, result) } else { xpathPrefix, keyName, tableName := xpathKeyExtract(d, ygRoot, oper, path) log.Infof("Delete req: path(\"%v\"), key(\"%v\"), xpathPrefix(\"%v\"), tableName(\"%v\").", path, keyName, xpathPrefix, tableName) @@ -275,7 +302,7 @@ func dbMapDelete(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string, jsonDat return err } -func cvlYangReqToDbMapDelete(xpathPrefix string, tableName string, keyName string, result map[string]map[string]db.Value) error { +func sonicYangReqToDbMapDelete(xpathPrefix string, tableName string, keyName string, result map[string]map[string]db.Value) error { if (tableName != "") { // Specific table entry case result[tableName] = make(map[string]db.Value) @@ -283,13 +310,16 @@ func cvlYangReqToDbMapDelete(xpathPrefix string, tableName string, keyName strin // Specific key case var dbVal db.Value tokens:= strings.Split(xpathPrefix, "/") - // Format /module:container/tableName[key]/fieldName - if tokens[len(tokens)-2] == tableName { - // Specific leaf case - fieldName := tokens[len(tokens)-1] - dbVal.Field = make(map[string]string) - dbVal.Field[fieldName] = "" - } + if tokens[SONIC_TABLE_INDEX] == tableName { + fieldName := tokens[len(tokens)-1] + dbSpecField := tableName + "/" + fieldName + _, ok := xDbSpecMap[dbSpecField] + if ok && xDbSpecMap[dbSpecField].fieldType == "leaf" { + // Specific leaf case + dbVal.Field = make(map[string]string) + dbVal.Field[fieldName] = "" + } + } result[tableName][keyName] = dbVal } else { // Get all keys @@ -297,11 +327,9 @@ func cvlYangReqToDbMapDelete(xpathPrefix string, tableName string, keyName strin } else { // Get all table entries // If table name not available in xpath get top container name - tokens:= strings.Split(xpathPrefix, ":") - container := "/" + tokens[len(tokens)-1] - _, ok := xDbSpecMap[container] - if ok && xDbSpecMap[container] != nil { - dbInfo := xDbSpecMap[container] + _, ok := xDbSpecMap[xpathPrefix] + if ok && xDbSpecMap[xpathPrefix] != nil { + dbInfo := xDbSpecMap[xpathPrefix] if dbInfo.fieldType == "container" { for dir, _ := range dbInfo.dbEntry.Dir { result[dir] = make(map[string]db.Value) @@ -327,8 +355,8 @@ to write into redis-db */ func dbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string, jsonData interface{}, result map[string]map[string]db.Value) error { var err error root := xpathRootNameGet(path) - if isCvlYang(path) { - err = cvlYangReqToDbMapCreate(jsonData, result) + if isSonicYang(path) { + err = sonicYangReqToDbMapCreate(jsonData, result) } else { err = yangReqToDbMapCreate(d, ygRoot, oper, root, "", "", jsonData, result) } @@ -492,35 +520,35 @@ func yangReqToDbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, } func sonicXpathKeyExtract(path string) (string, string, string) { - xpath, keyStr, tableName := "", "", "" - var err error - xpath, err = RemoveXPATHPredicates(path) - if err != nil { - return xpath, keyStr, tableName - } - rgp := regexp.MustCompile(`\[([^\[\]]*)\]`) - pathsubStr := strings.Split(path , "/") - if len(pathsubStr) > SONIC_TABLE_INDEX { - if strings.Contains(pathsubStr[2], "[") { - tableName = strings.Split(pathsubStr[SONIC_TABLE_INDEX], "[")[0] - } else { - tableName = pathsubStr[SONIC_TABLE_INDEX] + xpath, keyStr, tableName := "", "", "" + var err error + xpath, err = RemoveXPATHPredicates(path) + if err != nil { + return xpath, keyStr, tableName } + rgp := regexp.MustCompile(`\[([^\[\]]*)\]`) + pathsubStr := strings.Split(path , "/") + if len(pathsubStr) > SONIC_TABLE_INDEX { + if strings.Contains(pathsubStr[2], "[") { + tableName = strings.Split(pathsubStr[SONIC_TABLE_INDEX], "[")[0] + } else { + tableName = pathsubStr[SONIC_TABLE_INDEX] + } dbInfo, ok := xDbSpecMap[tableName] - cdb := db.ConfigDB - if !ok { - log.Errorf("No entry in xDbSpecMap for xpath %v in order to fetch DB index.", tableName) - } else { + cdb := db.ConfigDB + if !ok { + log.Infof("No entry in xDbSpecMap for xpath %v in order to fetch DB index.", tableName) + } else { cdb = dbInfo.dbIndex } dbOpts := getDBOptions(cdb) - for i, kname := range rgp.FindAllString(path, -1) { - if i > 0 { keyStr += dbOpts.KeySeparator } - val := strings.Split(kname, "=")[1] - keyStr += strings.TrimRight(val, "]") - } - } - return xpath, keyStr, tableName + for i, kname := range rgp.FindAllString(path, -1) { + if i > 0 { keyStr += dbOpts.KeySeparator } + val := strings.Split(kname, "=")[1] + keyStr += strings.TrimRight(val, "]") + } + } + return xpath, keyStr, tableName } /* Extract key vars, create db key and xpath */ @@ -534,6 +562,9 @@ func xpathKeyExtract(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string) (st var dbs [db.MaxDB]*db.DB pfxPath, _ = RemoveXPATHPredicates(path) + if strings.Count(path, ":") > 1 { + pfxPath = stripAugmentedModuleNames(pfxPath) + } xpathInfo, ok := xYangSpecMap[pfxPath] if !ok { log.Errorf("No entry found in xYangSpecMap for xpath %v.", pfxPath) @@ -550,7 +581,7 @@ func xpathKeyExtract(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string) (st curPathWithKey += k if strings.Contains(k, "[") { if len(keyStr) > 0 { - keyStr += keySeparator + keyStr += keySeparator } yangXpath, _ := RemoveXPATHPredicates(curPathWithKey) _, ok := xYangSpecMap[yangXpath] @@ -585,12 +616,11 @@ func xpathKeyExtract(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string) (st } curPathWithKey += "/" } - //pfxPath, _ := RemoveXPATHPredicates(path) tblPtr := xpathInfo.tableName if tblPtr != nil { tableName = *tblPtr } else if xpathInfo.xfmrTbl != nil { - inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curPathWithKey, oper, "", nil, nil) + inParams := formXfmrInputRequest(d, dbs, cdb, ygRoot, curPathWithKey, oper, "", nil, nil) tableName, _ = tblNameFromTblXfmrGet(*xpathInfo.xfmrTbl, inParams) } return pfxPath, keyStr, tableName diff --git a/src/translib/transformer/xlate_utils.go b/src/translib/transformer/xlate_utils.go index 81e94f89b8..6afbeeaa74 100644 --- a/src/translib/transformer/xlate_utils.go +++ b/src/translib/transformer/xlate_utils.go @@ -202,7 +202,7 @@ func yangKeyFromEntryGet(entry *yang.Entry) []string { return keyList } -func isCvlYang(path string) bool { +func isSonicYang(path string) bool { if strings.HasPrefix(path, "/sonic") { return true } @@ -417,3 +417,17 @@ func getDBOptionsWithSeparator(dbNo db.DBNum, initIndicator string, tableSeparat KeySeparator : keySeparator, }) } + +func stripAugmentedModuleNames(xpath string) string { + pathList := strings.Split(xpath, "/") + pathList = pathList[1:] + for i, pvar := range pathList { + if (i > 0) && strings.Contains(pvar, ":") { + pvar = strings.Split(pvar,":")[1] + pathList[i] = pvar + } + } + path := "/" + strings.Join(pathList, "/") + return path +} + diff --git a/src/translib/transformer/xspec.go b/src/translib/transformer/xspec.go index 8587a575ef..11198a645f 100644 --- a/src/translib/transformer/xspec.go +++ b/src/translib/transformer/xspec.go @@ -39,8 +39,8 @@ type dbInfo struct { yangXpath []string } -var xYangSpecMap map[string]*yangXpathInfo -var xDbSpecMap map[string]*dbInfo +var xYangSpecMap map[string]*yangXpathInfo +var xDbSpecMap map[string]*dbInfo var xDbSpecOrdTblMap map[string][]string //map of module-name to ordered list of db tables { "sonic-acl" : ["ACL_TABLE", "ACL_RULE"] } /* update transformer spec with db-node */ @@ -87,8 +87,8 @@ func yangToDbMapFill (keyLevel int, xYangSpecMap map[string]*yangXpathInfo, entr } } - // If DB Index is not annotated and parent DB index is annotated inherit the DB Index of the parent - if ok && xpathData.dbIndex == db.ConfigDB && parentXpathData.dbIndex != db.ConfigDB { + if ok && xpathData.dbIndex == db.ConfigDB && parentXpathData.dbIndex != db.ConfigDB { + // If DB Index is not annotated and parent DB index is annotated inherit the DB Index of the parent xpathData.dbIndex = parentXpathData.dbIndex } @@ -187,23 +187,47 @@ func yangToDbMapBuild(entries map[string]*yang.Entry) { } /* Fill the map with db details */ -func dbMapFill(prefixPath string, curPath string, moduleNm string, trkTpCnt bool, xDbSpecMap map[string]*dbInfo, entry *yang.Entry) { +func dbMapFill(tableName string, curPath string, moduleNm string, trkTpCnt bool, xDbSpecMap map[string]*dbInfo, entry *yang.Entry) { entryType := entry.Node.Statement().Keyword - if entryType == "list" { - prefixPath = entry.Name - } - if !isYangResType(entryType) { - dbXpath := prefixPath - if entryType != "list" { - dbXpath = prefixPath + "/" + entry.Name + if entry.Name != moduleNm { + if entryType == "container" { + tableName = entry.Name } - xDbSpecMap[dbXpath] = new(dbInfo) - xDbSpecMap[dbXpath].dbEntry = entry - xDbSpecMap[dbXpath].fieldType = entryType - if entryType == "list" { - xDbSpecMap[dbXpath].dbIndex = db.ConfigDB + + if !isYangResType(entryType) { + dbXpath := tableName + if entryType != "container" { + dbXpath = tableName + "/" + entry.Name + } + xDbSpecMap[dbXpath] = new(dbInfo) + xDbSpecMap[dbXpath].dbIndex = db.MaxDB + xDbSpecMap[dbXpath].dbEntry = entry + xDbSpecMap[dbXpath].fieldType = entryType + if entryType == "container" { + xDbSpecMap[dbXpath].dbIndex = db.ConfigDB + if entry.Exts != nil && len(entry.Exts) > 0 { + for _, ext := range entry.Exts { + dataTagArr := strings.Split(ext.Keyword, ":") + tagType := dataTagArr[len(dataTagArr)-1] + switch tagType { + case "key-name" : + if xDbSpecMap[dbXpath].keyName == nil { + xDbSpecMap[dbXpath].keyName = new(string) + } + *xDbSpecMap[dbXpath].keyName = ext.NName() + default : + log.Infof("Unsupported ext type(%v) for xpath(%v).", tagType, dbXpath) + } + } + } + } } + } else { + moduleXpath := "/" + moduleNm + ":" + entry.Name + xDbSpecMap[moduleXpath] = new(dbInfo) + xDbSpecMap[moduleXpath].dbEntry = entry + xDbSpecMap[moduleXpath].fieldType = entryType } var childList []string @@ -218,8 +242,8 @@ func dbMapFill(prefixPath string, curPath string, moduleNm string, trkTpCnt bool } for _, child := range childList { - childPath := prefixPath + "/" + entry.Dir[child].Name - dbMapFill(prefixPath, childPath, moduleNm, trkTpCnt, xDbSpecMap, entry.Dir[child]) + childPath := tableName + "/" + entry.Dir[child].Name + dbMapFill(tableName, childPath, moduleNm, trkTpCnt, xDbSpecMap, entry.Dir[child]) } } @@ -236,7 +260,7 @@ func dbMapBuild(entries []*yang.Entry) { continue } moduleNm := e.Name - log.Info("Module name", moduleNm) + log.Infof("Module name(%v)", moduleNm) xDbSpecOrdTblMap[moduleNm] = []string{} trkTpCnt := true dbMapFill("", "", moduleNm, trkTpCnt, xDbSpecMap, e) @@ -522,23 +546,23 @@ func dbMapPrint( fname string) { } func updateSchemaOrderedMap(module string, entry *yang.Entry) { - var children []string - if entry.Node.Statement().Keyword == "module" { - for _, dir := range entry.DirOKeys { - // Gives the yang xpath for the top level container - xpath := "/" + module + ":" + dir - _, ok := xYangSpecMap[xpath] - if ok { - yentry := xYangSpecMap[xpath].yangEntry - if yentry.Node.Statement().Keyword == "container" { - var keyspec = make([]KeySpec, 0) - keyspec = FillKeySpecs(xpath, "" , &keyspec) - children = updateChildTable(keyspec, &children) - } - } + var children []string + if entry.Node.Statement().Keyword == "module" { + for _, dir := range entry.DirOKeys { + // Gives the yang xpath for the top level container + xpath := "/" + module + ":" + dir + _, ok := xYangSpecMap[xpath] + if ok { + yentry := xYangSpecMap[xpath].yangEntry + if yentry.Node.Statement().Keyword == "container" { + var keyspec = make([]KeySpec, 0) + keyspec = FillKeySpecs(xpath, "" , &keyspec) + children = updateChildTable(keyspec, &children) + } + } + } } - } - xDbSpecOrdTblMap[module] = children + xDbSpecOrdTblMap[module] = children } func updateChildTable(keyspec []KeySpec, chlist *[]string) ([]string) { @@ -548,7 +572,7 @@ func updateChildTable(keyspec []KeySpec, chlist *[]string) ([]string) { *chlist = append(*chlist, ks.Ts.Name) } } - *chlist = updateChildTable(ks.Child, chlist) + *chlist = updateChildTable(ks.Child, chlist) } return *chlist }