Broadcom sonic share to master merge

.gitignore

@@ -0,0 +1,15 @@
+
+.vscode
+*.code-workspace
+pkg/*
+build
+gopkgs
+__pycache__
+*.pyc
+*.rdb
+*.swp
+
+*.yin
+*.tree
+src/translib/ocbinds/ocbinds.go
+

Makefile

@@ -0,0 +1,204 @@
+################################################################################
+#                                                                              #
+#  Copyright 2019 Broadcom. The term Broadcom refers to Broadcom Inc. and/or   #
+#  its subsidiaries.                                                           #
+#                                                                              #
+#  Licensed under the Apache License, Version 2.0 (the "License");             #
+#  you may not use this file except in compliance with the License.            #
+#  You may obtain a copy of the License at                                     #
+#                                                                              #
+#     http://www.apache.org/licenses/LICENSE-2.0                               #
+#                                                                              #
+#  Unless required by applicable law or agreed to in writing, software         #
+#  distributed under the License is distributed on an "AS IS" BASIS,           #
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.    #
+#  See the License for the specific language governing permissions and         #
+#  limitations under the License.                                              #
+#                                                                              #
+################################################################################
+
+.PHONY: all clean cleanall codegen rest-server rest-clean yamlGen cli clitree
+
+TOPDIR := $(abspath .)
+BUILD_DIR := $(TOPDIR)/build
+export TOPDIR
+
+ifeq ($(BUILD_GOPATH),)
+export BUILD_GOPATH=$(TOPDIR)/gopkgs
+endif
+
+export GOPATH=$(BUILD_GOPATH):$(TOPDIR)
+
+ifeq ($(GO),)
+GO := /usr/local/go/bin/go 
+export GO
+endif
+
+INSTALL := /usr/bin/install
+
+MAIN_TARGET = sonic-mgmt-framework_1.0-01_amd64.deb
+
+GO_DEPS_LIST = github.com/gorilla/mux \
+               github.com/Workiva/go-datastructures/queue \
+               github.com/openconfig/goyang \
+               github.com/openconfig/ygot/ygot \
+               github.com/go-redis/redis \
+               github.com/golang/glog \
+               github.com/pkg/profile \
+               gopkg.in/go-playground/validator.v9 \
+               golang.org/x/crypto/ssh \
+               github.com/antchfx/jsonquery \
+               github.com/antchfx/xmlquery \
+               github.com/facette/natsort \
+	       github.com/philopon/go-toposort \
+               gopkg.in/godbus/dbus.v5 \
+               github.com/dgrijalva/jwt-go
+
+
+REST_BIN = $(BUILD_DIR)/rest_server/main
+CERTGEN_BIN = $(BUILD_DIR)/rest_server/generate_cert
+
+
+all: build-deps go-deps go-redis-patch go-patch translib rest-server cli
+
+build-deps:
+	mkdir -p $(BUILD_DIR)
+
+go-deps: $(GO_DEPS_LIST)
+
+go-redis-patch: go-deps
+	cd $(BUILD_GOPATH)/src/github.com/go-redis/redis; git checkout d19aba07b47683ef19378c4a4d43959672b7cec8 2>/dev/null ; true; \
+$(GO) install -v -gcflags "-N -l" $(BUILD_GOPATH)/src/github.com/go-redis/redis
+
+$(GO_DEPS_LIST):
+	$(GO) get -v $@
+
+cli: 
+	$(MAKE) -C src/CLI
+
+clitree:
+	 TGT_DIR=$(BUILD_DIR)/cli $(MAKE) -C src/CLI/clitree
+
+cvl: go-deps go-patch go-redis-patch
+	$(MAKE) -C src/cvl
+	$(MAKE) -C src/cvl/schema
+	$(MAKE) -C src/cvl/testdata/schema
+
+cvl-test:
+	$(MAKE) -C src/cvl gotest
+
+rest-server: translib
+	$(MAKE) -C src/rest
+
+rest-clean:
+	$(MAKE) -C src/rest clean
+
+translib: cvl
+	$(MAKE) -C src/translib
+
+codegen:
+	$(MAKE) -C models
+
+yamlGen:
+	$(MAKE) -C models/yang
+	$(MAKE) -C models/yang/sonic
+
+go-patch: go-deps
+	cd $(BUILD_GOPATH)/src/github.com/openconfig/ygot/; git reset --hard HEAD; git checkout 724a6b18a9224343ef04fe49199dfb6020ce132a 2>/dev/null ; true; \
+cd ../; cp $(TOPDIR)/ygot-modified-files/ygot.patch .; \
+patch -p1 < ygot.patch; rm -f ygot.patch; \
+$(GO) install -v -gcflags "-N -l" $(BUILD_GOPATH)/src/github.com/openconfig/ygot/ygot; \
+	cd $(BUILD_GOPATH)/src/github.com/openconfig/goyang/; git reset --hard HEAD; git checkout 064f9690516f4f72db189f4690b84622c13b7296 >/dev/null ; true; \
+cp $(TOPDIR)/goyang-modified-files/annotate.go $(BUILD_GOPATH)/src/github.com/openconfig/goyang/annotate.go; \
+cp $(TOPDIR)/goyang-modified-files/goyang.patch .; \
+patch -p1 < goyang.patch; rm -f goyang.patch; \
+$(GO) install -v -gcflags "-N -l" $(BUILD_GOPATH)/src/github.com/openconfig/goyang
+
+#Apply CVL related patches
+	$(apply_cvl_dep_patches)
+
+install:
+	$(INSTALL) -D $(REST_BIN) $(DESTDIR)/usr/sbin/rest_server
+	$(INSTALL) -D $(CERTGEN_BIN) $(DESTDIR)/usr/sbin/generate_cert
+	$(INSTALL) -d $(DESTDIR)/usr/sbin/schema/
+	$(INSTALL) -d $(DESTDIR)/usr/sbin/lib/
+	$(INSTALL) -d $(DESTDIR)/usr/models/yang/
+	$(INSTALL) -D $(TOPDIR)/models/yang/sonic/*.yang $(DESTDIR)/usr/models/yang/
+	$(INSTALL) -D $(TOPDIR)/models/yang/sonic/common/*.yang $(DESTDIR)/usr/models/yang/
+	$(INSTALL) -D $(TOPDIR)/models/yang/*.yang $(DESTDIR)/usr/models/yang/
+	$(INSTALL) -D $(TOPDIR)/config/transformer/models_list $(DESTDIR)/usr/models/yang/
+	$(INSTALL) -D $(TOPDIR)/models/yang/common/*.yang $(DESTDIR)/usr/models/yang/
+	$(INSTALL) -D $(TOPDIR)/models/yang/annotations/*.yang $(DESTDIR)/usr/models/yang/
+	cp -rf $(TOPDIR)/build/rest_server/dist/ui/ $(DESTDIR)/rest_ui/
+	cp -rf $(TOPDIR)/build/cli $(DESTDIR)/usr/sbin/
+	cp -rf $(TOPDIR)/build/swagger_client_py/ $(DESTDIR)/usr/sbin/lib/
+	cp -rf $(TOPDIR)/src/cvl/conf/cvl_cfg.json $(DESTDIR)/usr/sbin/cvl_cfg.json
+
+# Copy all CVL schema files
+	$(install_cvl_schema)
+
+	# Scripts for host service
+	$(INSTALL) -d $(DESTDIR)/usr/lib/sonic_host_service/host_modules
+	$(INSTALL) -D $(TOPDIR)/scripts/sonic_host_server.py $(DESTDIR)/usr/lib/sonic_host_service
+	$(INSTALL) -D $(TOPDIR)/scripts/host_modules/*.py $(DESTDIR)/usr/lib/sonic_host_service/host_modules
+	$(INSTALL) -d $(DESTDIR)/etc/dbus-1/system.d
+	$(INSTALL) -D $(TOPDIR)/scripts/org.sonic.hostservice.conf $(DESTDIR)/etc/dbus-1/system.d
+	$(INSTALL) -d $(DESTDIR)/lib/systemd/system
+	$(INSTALL) -D $(TOPDIR)/scripts/sonic-hostservice.service $(DESTDIR)/lib/systemd/system
+
+
+
+ifeq ($(SONIC_COVERAGE_ON),y)
+	echo "" > $(DESTDIR)/usr/sbin/.test
+endif
+
+$(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
+	mv $* $(DEST)/
+
+clean: rest-clean
+	$(MAKE) -C src/cvl clean
+	$(MAKE) -C src/translib clean
+	$(MAKE) -C src/cvl/schema clean
+	$(MAKE) -C src/cvl cleanall
+	rm -rf build/*
+	rm -rf debian/.debhelper
+	rm -rf $(BUILD_GOPATH)/src/github.com/openconfig/goyang/annotate.go
+
+cleanall:
+	$(MAKE) -C src/cvl cleanall
+	rm -rf build/*
+
+#Function to apply CVL related patches
+define apply_cvl_dep_patches
+
+	cd $(BUILD_GOPATH)/src/github.com/antchfx/jsonquery; git reset --hard HEAD; \
+	git checkout 3b69d31134d889b501e166a035a4d5ecb8c6c367; git apply $(TOPDIR)/patches/jsonquery.patch
+	cd $(BUILD_GOPATH)/src/github.com/antchfx/xmlquery; git reset --hard HEAD; \
+	git checkout fe009d4cc63c3011f05e1dfa75a27899acccdf11; git apply $(TOPDIR)/patches/xmlquery.patch
+
+endef
+
+#Function to install CVL schema files
+define install_cvl_schema
+	$(INSTALL) -d $(DESTDIR)/usr/sbin/schema/
+	$(INSTALL) -D $(TOPDIR)/src/cvl/schema/*.yin $(DESTDIR)/usr/sbin/schema/
+	$(INSTALL) -D $(TOPDIR)/src/cvl/testdata/schema/*.yin $(DESTDIR)/usr/sbin/schema/
+
+#Find all platform directories first and create them in destination
+	if [ -d $(TOPDIR)/src/cvl/schema/platform ] ; then \
+		for dd in $$(find $(TOPDIR)/src/cvl/schema/platform -type d) ; \
+		do  \
+		dds=$$(echo $${dd} | sed 's/^.*\(schema\/.*\)/\1/g') ; \
+		$(INSTALL) -d $(DESTDIR)/usr/sbin/$${dds} ; \
+		done  ; \
+	fi
+#For each platform directory, copy all platform YANG files to destination
+	if [ -d $(TOPDIR)/src/cvl/schema/platform ] ; then \
+		for ff in $$(find $(TOPDIR)/src/cvl/schema/platform -name '*.yin') ; \
+		do  \
+		ffs=$$(echo $${ff} | sed 's/^.*\(schema\/.*\)/\1/g') ; \
+		$(INSTALL) -T $(TOPDIR)/src/cvl/$${ffs} $(DESTDIR)/usr/sbin/$${ffs} ; \
+		done ; \
+	fi
+endef
+

README.md

@@ -0,0 +1,90 @@
+## SONiC Management Framework Repo
+
+### Build Instruction
+Please note that the build instruction in this guide has only been tested on Ubuntu 16.04.
+#### Pre-rerequisit
+##### User permissions:
+	`sudo usermod -aG sudo $USER`
+	`sudo usermod -aG docker $USER`
+
+##### Packages to be installed:
+	`sudo apt-get install git docker`
+
+#### Steps to build and create an installer
+1. git clone https://github.com/project-arlo/sonic-buildimage.git
+2. cd sonic-buildimage/
+3. sudo modprobe overlay
+4. make init
+5. make configure PLATFORM=broadcom
+6. Run the prefetch python script to download all binaries (see below for the script).
+7. To build mgmt-framework container:   
+	`BLDENV=stretch make target/docker-sonic-mgmt-framework.gz`
+8. To build Debian Stretch, if not already downloaded:   
+	`BLDENV=stretch make stretch`
+9. To build the ONIE installer:   
+	`BLDENV=stretch make target/sonic-broadcom.bin`
+
+#### Faster builds
+In order to speed up the process of build, you can prefetch the latest debian files from Azure server, and just build what you need.
+
+Here is a python script you could use to fetch latest prebuilt objects (deb, gz, ko, etc) from SONiC Jenkins cluster:
+
+    import os
+    import shutil
+    import urllib.request
+    from html.parser import HTMLParser
+
+    UPSTREAM_PREFIX = 'https://sonic-jenkins.westus2.cloudapp.azure.com/job/broadcom/job/buildimage-brcm-all/lastSuccessfulBuild/artifact/'
+
+    def get_all_bins(target_path, extension):
+        """Get all files matching the given extension from the target path"""
+        print('Fetching %s*%s' % (target_path, extension))
+        os.makedirs(target_path, exist_ok=True)
+
+        req = urllib.request.urlopen(UPSTREAM_PREFIX + target_path)
+        data = req.read().decode()
+
+        class Downloader(HTMLParser):
+            """Class to parse retrieved data, match against the given extension,
+               and download the matching files to the given target directory"""
+            def handle_starttag(self, tag, attrs):
+                """Handle only <a> tags"""
+                if tag == 'a':
+                    for attr, val in attrs:
+                        if attr == 'href' and val.endswith(extension):
+                            self.download_file(val)
+
+            @staticmethod
+            def download_file(path):
+                filename = os.path.join(target_path, path)
+                freq = urllib.request.urlopen(UPSTREAM_PREFIX + target_path + path)
+
+                print('\t%s' % path)
+                with open(filename, 'wb') as fp:
+                    shutil.copyfileobj(freq, fp)
+
+
+        parser = Downloader()
+        parser.feed(data)
+        print()
+
+    get_all_bins('target/debs/stretch/', '.deb')
+    get_all_bins('target/files/stretch/', '.ko')
+    get_all_bins('target/python-debs/', '.deb')
+    get_all_bins('target/python-wheels/', '.whl')
+    get_all_bins('target/', '.gz')
+
+
+
+##### Incremental builds 
+Just clean up the deb's/gz that require re-build, and build again. Here is an exmple:
+
+##### To build deb file for sonic-mgmt-framework
+
+	BLDENV=stretch make target/debs/stretch/sonic-mgmt-framework_1.0-01_amd64.deb-clean
+	BLDENV=stretch make target/debs/stretch/sonic-mgmt-framework_1.0-01_amd64.deb
+
+##### To build sonic-mgmt-framework docker alone
+
+	BLDENV=stretch make target/docker-sonic-mgmt-framework.gz-clean
+	BLDENV=stretch make target/docker-sonic-mgmt-framework.gz

config/transformer/models_list

@@ -0,0 +1,14 @@
+#List yang models transformer need to load
+
+openconfig-acl.yang
+openconfig-acl-annot.yang
+openconfig-network-instance.yang
+openconfig-network-instance-annot.yang
+sonic-vxlan.yang
+sonic-udld.yang
+sonic-udld-annot.yang
+ietf-ptp.yang
+ietf-ptp-annot.yang
+sonic-mclag.yang
+openconfig-system.yang
+openconfig-system-annot.yang

debian/.gitignore

@@ -0,0 +1,9 @@
+.debhelper/
+*.debhelper
+*.debhelper.log
+*.substvars
+sonic-mgmt-framework/
+sonic-host-service/
+tmp/
+files
+

debian/changelog

@@ -0,0 +1,5 @@
+sonic-mgmt-framework (1.0-01) UNRELEASED; urgency=low
+
+  * Initial release.
+
+ -- Prabhu Sreenivasan <[email protected]>  Tue, 18 Jun 2019 00:25:19 +0000

debian/compat

@@ -0,0 +1 @@
+9

debian/control

@@ -0,0 +1,18 @@
+Source: sonic-mgmt-framework 
+Maintainer: Prabhu Sreenivasan <[email protected]>
+Build-Depends: debhelper (>= 8.0.0),
+               dh-systemd
+Standards-Version: 3.9.3
+Section: net
+
+Package: sonic-mgmt-framework 
+Priority: extra  
+Architecture: amd64
+Depends: ${shlibs:Depends}, ${misc:Depends}  
+Description: SONiC Management Framework 
+
+Package: sonic-host-service
+Priority: extra
+Architecture: amd64
+Depends: python-dbus, python-gobject, python-systemd, ${shlibs:Depends}, ${misc:Depends}
+Description: SONiC Host Service

debian/rules

@@ -0,0 +1,7 @@
+#!/usr/bin/make -f  
+%:  
+	dh $@ --with systemd
+
+
+override_dh_shlibdeps:
+	dh_shlibdeps --dpkg-shlibdeps-params=--ignore-missing-info -l$(shell pwd)/build/cli/target/.libs/:$(shell pwd)/build/cli/.libs/

debian/sonic-host-service.install

@@ -0,0 +1,3 @@
+usr/lib/*
+etc/dbus-1/system.d/*
+lib/systemd/system/*

debian/sonic-mgmt-framework.install

@@ -0,0 +1,3 @@
+usr/sbin/*
+rest_ui/*
+usr/models/*