Create docker-base-buster and docker-config-engine-buster

dockers/docker-base-buster/Dockerfile.j2

@@ -0,0 +1,98 @@
+{% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
+{% if CONFIGURED_ARCH == "armhf" %}
+FROM multiarch/debian-debootstrap:armhf-buster
+{% elif CONFIGURED_ARCH == "arm64" %}
+FROM multiarch/debian-debootstrap:arm64-buster
+{% else %}
+FROM debian:buster
+{% endif %}
+
+# Clean documentation in FROM image
+RUN find /usr/share/doc -depth \( -type f -o -type l \) ! -name copyright | xargs rm || true
+
+# Clean doc directories that are empty or only contain empty directories
+RUN while [ -n "$(find /usr/share/doc -depth -type d -empty -print -exec rmdir {} +)" ]; do :; done && \
+    rm -rf               \
+    /usr/share/man/*     \
+    /usr/share/groff/*   \
+    /usr/share/info/*    \
+    /usr/share/lintian/* \
+    /usr/share/linda/*   \
+    /var/cache/man/*     \
+    /usr/share/locale/*
+
+# Make apt-get non-interactive
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Configure data sources for apt/dpkg
+COPY ["dpkg_01_drop", "/etc/dpkg/dpkg.cfg.d/01_drop"]
+{% if CONFIGURED_ARCH == "armhf" %}
+COPY ["sources.list.armhf", "/etc/apt/sources.list"]
+{% elif CONFIGURED_ARCH == "arm64" %}
+COPY ["sources.list.arm64", "/etc/apt/sources.list"]
+{% else %}
+COPY ["sources.list", "/etc/apt/sources.list"]
+{% endif %}
+COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"]
+COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"]
+
+# Update apt cache and
+# pre-install fundamental packages
+RUN apt-get update &&        \
+    apt-get -y install       \
+        less                 \
+        perl                 \
+        procps               \
+        python               \
+        rsyslog              \
+        vim-tiny             \
+# Install dependencies of supervisor
+        python-pkg-resources \
+        python-meld3         \
+# dependencies of redis-tools
+        libatomic1           \
+        libjemalloc2         \
+        liblua5.1-0          \
+        lua-bitop            \
+        lua-cjson            \
+# common dependencies
+        libpython2.7         \
+        libdaemon0           \
+        libdbus-1-3          \
+        libjansson4
+
+# ip and ifconfig utility missing in docker for arm arch
+RUN apt-get -y install       \
+        iproute2             \
+        net-tools
+
+RUN mkdir -p /etc/supervisor /var/log/supervisor
+
+RUN apt-get -y purge   \
+    exim4              \
+    exim4-base         \
+    exim4-config       \
+    exim4-daemon-light
+
+{% if docker_base_buster_debs.strip() -%}
+# Copy locally-built Debian package dependencies
+{{ copy_files("debs/", docker_base_buster_debs.split(' '), "/debs/") }}
+
+# Install built Debian packages and implicitly install their dependencies
+{{ install_debian_packages(docker_base_buster_debs.split(' ')) }}
+{%- endif %}
+
+# Clean up apt
+# Remove /var/lib/apt/lists/*, could be obsoleted for derived images
+RUN apt-get clean -y                     && \
+    apt-get autoclean -y                 && \
+    apt-get autoremove -y                && \
+    rm -rf /var/lib/apt/lists/* /tmp/*
+
+COPY ["etc/rsyslog.conf", "/etc/rsyslog.conf"]
+COPY ["etc/rsyslog.d/*", "/etc/rsyslog.d/"]
+COPY ["root/.vimrc", "/root/.vimrc"]
+
+RUN ln /usr/bin/vim.tiny /usr/bin/vim
+
+COPY ["etc/supervisor/supervisord.conf", "/etc/supervisor/"]

dockers/docker-base-buster/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2016 Microsoft, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

dockers/docker-base-buster/dpkg_01_drop

@@ -0,0 +1,30 @@
+## Drop unnecessary files
+## ref: https://wiki.ubuntu.com/ReducingDiskFootprint
+
+## Documentation
+path-exclude /usr/share/doc/*
+# we need to keep copyright files for legal reasons
+path-include /usr/share/doc/*/copyright
+path-exclude /usr/share/man/*
+path-exclude /usr/share/groff/*
+path-exclude /usr/share/info/*
+# lintian stuff is small, but really unnecessary
+path-exclude /usr/share/lintian/*
+path-exclude /usr/share/linda/*
+
+## Translations
+path-exclude /usr/share/locale/*
+
+## Landscape
+path-exclude /usr/share/pyshared/twisted/test*
+path-exclude /usr/lib/python*/dist-packages/twisted/test*
+path-exclude /usr/share/pyshared/twisted/*/test*
+path-exclude /usr/lib/python*/dist-packages/twisted/*/test*
+
+## install the configuration file if it’s currently missing
+force-confmiss
+## combined with confold: overwrite configuration files that you have not modified
+force-confdef
+## do not modify the current configuration file, the new version is installed with a .dpkg-dist suffix
+force-confold
+

dockers/docker-base-buster/etc/rsyslog.conf

@@ -0,0 +1,76 @@
+#
+#  /etc/rsyslog.conf    Configuration file for rsyslog.
+#
+#                       For more information see
+#                       /usr/share/doc/rsyslog-doc/html/rsyslog_conf.html
+
+
+#################
+#### MODULES ####
+#################
+
+$ModLoad imuxsock # provides support for local system logging
+
+#
+# Set a rate limit on messages from the container
+#
+$SystemLogRateLimitInterval 300
+$SystemLogRateLimitBurst 20000
+
+#$ModLoad imklog  # provides kernel logging support
+#$ModLoad immark  # provides --MARK-- message capability
+
+# provides UDP syslog reception
+#$ModLoad imudp
+#$UDPServerRun 514
+
+# provides TCP syslog reception
+#$ModLoad imtcp
+#$InputTCPServerRun 514
+
+
+###########################
+#### GLOBAL DIRECTIVES ####
+###########################
+
+# Set remote syslog server
+template (name="ForwardFormatInContainer" type="string" string="<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg%")
+*.* action(type="omfwd" target="127.0.0.1" port="514" protocol="udp" Template="ForwardFormatInContainer")
+
+#
+# Use traditional timestamp format.
+# To enable high precision timestamps, comment out the following line.
+#
+#$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
+
+# Define a custom template
+$template SONiCFileFormat,"%TIMESTAMP%.%timestamp:::date-subseconds% %HOSTNAME% %syslogseverity-text:::uppercase% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
+$ActionFileDefaultTemplate SONiCFileFormat
+
+#
+# Set the default permissions for all log files.
+#
+$FileOwner root
+$FileGroup adm
+$FileCreateMode 0640
+$DirCreateMode 0755
+$Umask 0022
+
+#
+# Where to place spool and state files
+#
+$WorkDirectory /var/spool/rsyslog
+
+#
+# Include all config files in /etc/rsyslog.d/
+#
+$IncludeConfig /etc/rsyslog.d/*.conf
+
+#
+# Suppress duplicate messages and report "message repeated n times"
+#
+$RepeatedMsgReduction on
+
+###############
+#### RULES ####
+###############

dockers/docker-base-buster/etc/rsyslog.d/supervisor.conf

@@ -0,0 +1,9 @@
+$ModLoad imfile
+
+$InputFileName /var/log/supervisor/supervisord.log
+$InputFileTag supervisord
+$InputFileStateFile state-supervisor
+$InputFileSeverity info
+$InputFileFacility local0
+$InputFilePersistStateInterval 1
+$InputRunFileMonitor

dockers/docker-base-buster/etc/supervisor/supervisord.conf

@@ -0,0 +1,33 @@
+; supervisor config file
+
+[unix_http_server]
+file=/var/run/supervisor.sock  ; (the path to the socket file)
+chmod=0700                     ; socket file mode (default 0700)
+username=dummy
+password=dummy
+
+[supervisord]
+logfile=/var/log/supervisor/supervisord.log  ; (main log file;default $CWD/supervisord.log)
+pidfile=/var/run/supervisord.pid             ; (supervisord pidfile;default supervisord.pid)
+childlogdir=/var/log/supervisor              ; ('AUTO' child log dir, default $TEMP)
+user=root
+
+; the below section must remain in the config file for RPC
+; (supervisorctl/web interface) to work, additional interfaces may be
+; added by defining them in separate rpcinterface: sections
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[supervisorctl]
+serverurl=unix:///var/run/supervisor.sock  ; use a unix:// URL  for a unix socket
+username=dummy
+password=dummy
+
+; The [include] section can just contain the "files" setting.  This
+; setting can list multiple files (separated by whitespace or
+; newlines).  It can also contain wildcards.  The filenames are
+; interpreted as relative to this file.  Included files *cannot*
+; include files themselves.
+
+[include]
+files = /etc/supervisor/conf.d/*.conf

dockers/docker-base-buster/no-check-valid-until

@@ -0,0 +1,4 @@
+# Instruct apt-get to NOT check the "Valid Until" date in Release files
+# Once the Debian team archives a repo, they stop updating this date
+
+Acquire::Check-Valid-Until "false";

dockers/docker-base-buster/no_install_recommend_suggest

@@ -0,0 +1,5 @@
+# Instruct apt-get to NOT install "recommended" or "suggested" packages by
+# default when installing a package.
+
+APT::Install-Recommends "false";
+APT::Install-Suggests "false";

dockers/docker-base-buster/root/.vimrc

@@ -0,0 +1,2 @@
+" enable vim features
+set nocompatible

dockers/docker-base-buster/sources.list

@@ -0,0 +1,8 @@
+## Debian mirror on Microsoft Azure
+## Ref: http://debian-archive.trafficmanager.net/
+
+deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster main contrib non-free
+deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster main contrib non-free
+deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free
+deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free
+deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster-backports main contrib non-free

dockers/docker-base-buster/sources.list.arm64

@@ -0,0 +1,7 @@
+## Debian mirror for ARM repo
+
+# ARM repo
+deb [arch=arm64] http://deb.debian.org/debian buster main contrib non-free
+deb-src [arch=arm64] http://deb.debian.org/debian buster main contrib non-free
+deb [arch=arm64] http://security.debian.org buster/updates main contrib non-free
+deb-src [arch=arm64] http://security.debian.org buster/updates main contrib non-free

dockers/docker-base-buster/sources.list.armhf

@@ -0,0 +1,7 @@
+## Debian mirror for ARM repo
+
+# ARM repo
+deb [arch=armhf] http://deb.debian.org/debian buster main contrib non-free
+deb-src [arch=armhf] http://deb.debian.org/debian buster main contrib non-free
+deb [arch=armhf] http://security.debian.org buster/updates main contrib non-free
+deb-src [arch=armhf] http://security.debian.org buster/updates main contrib non-free

dockers/docker-config-engine-buster/Dockerfile.j2

@@ -0,0 +1,50 @@
+{% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
+FROM docker-base-buster
+
+## Make apt-get non-interactive
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update         && \
+    apt-get install -y        \
+        # Dependencies for sonic-cfggen
+        python-lxml           \
+        python-yaml           \
+        python-bitarray       \
+        python-pip            \
+        python-dev            \
+        python-natsort        \
+        apt-utils             \
+        python-setuptools
+
+RUN pip install --upgrade pip
+
+RUN pip install             \
+        netaddr             \
+        ipaddr              \
+        jinja2              \
+        pyangbind==0.6.0
+
+{% if docker_config_engine_buster_debs.strip() %}
+# Copy locally-built Debian package dependencies
+{{ copy_files("debs/", docker_config_engine_buster_debs.split(' '), "/debs/") }}
+
+# Install locally-built Debian packages and implicitly install their dependencies
+{{ install_debian_packages(docker_config_engine_buster_debs.split(' ')) }}
+{% endif %}
+
+{% if docker_config_engine_buster_whls.strip() %}
+# Copy locally-built Python wheel dependencies
+{{ copy_files("python-wheels/", docker_config_engine_buster_whls.split(' '), "/python-wheels/") }}
+
+# Install locally-built Python wheel dependencies
+{{ install_python_wheels(docker_config_engine_buster_whls.split(' ')) }}
+{% endif %}
+
+## Clean up
+RUN apt-get purge -y               \
+        python-pip                 \
+        python-dev              && \
+    apt-get clean -y            && \
+    apt-get autoclean -y        && \
+    apt-get autoremove -y       && \
+    rm -rf /debs /python-wheels

dockers/docker-sonic-mgmt-framework/Dockerfile.j2

@@ -1,4 +1,4 @@
-FROM docker-config-engine-stretch
+FROM docker-config-engine-buster
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf
@@ -10,18 +10,12 @@ RUN apt-get update
 RUN pip install connexion==1.1.15 \
 		setuptools==21.0.0 \
 		grpcio-tools==1.20.0 \
-		pyangbind==0.6.0 \
 		certifi==2017.4.17 \
 		python-dateutil==2.6.0 \
 		six==1.11.0 \
 		urllib3==1.21.1
 
 
-
-## Install redis-tools dependencies
-## TODO: implicitly install dependencies
-RUN apt-get -y install libjemalloc1 libatomic1 liblua5.1-0 lua-bitop lua-cjson
-
 COPY \
 {% for deb in docker_sonic_mgmt_framework_debs.split(' ') -%}
 debs/{{ deb }}{{' '}}

rules/docker-base-buster.dep

@@ -0,0 +1,10 @@
+
+DPATH       := $($(DOCKER_BASE_BUSTER)_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/docker-base-buster.mk rules/docker-base-buster.dep   
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+DEP_FILES   += $(shell git ls-files $(DPATH))
+
+$(DOCKER_BASE_BUSTER)_CACHE_MODE  := GIT_CONTENT_SHA 
+$(DOCKER_BASE_BUSTER)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(DOCKER_BASE_BUSTER)_DEP_FILES   := $(DEP_FILES)
+