Update dockerfiles to do staged builds

Makefile.work

@@ -291,8 +291,8 @@ endif
 DOCKER_LOCKFILE_SAVE := $(DOCKER_LOCKDIR)/docker_save.lock
 $(shell mkdir -m 0777 -p $(DOCKER_LOCKDIR))
 $(shell [ -f $(DOCKER_LOCKFILE_SAVE) ] || (touch $(DOCKER_LOCKFILE_SAVE) && chmod 0777 $(DOCKER_LOCKFILE_SAVE)))
-$(shell [ -d $(DOCKER_ROOT) ] && docker run --rm -v $(DOCKER_ROOT)\:/mount debian sh -c 'rm -rf /mount/*')
-$(mkdir -p $(DOCKER_ROOT))
+$(shell [ -d $(DOCKER_ROOT) ] && docker run --rm -v $(DOCKER_ROOT)\:/mount $(DEFAULT_CONTAINER_REGISTRY)debian:bookworm sh -c 'rm -rf /mount/*')
+$(shell mkdir -p $(DOCKER_ROOT))
 
 ifeq ($(DOCKER_BUILDER_MOUNT),)
 override DOCKER_BUILDER_MOUNT := "$(PWD):/sonic"
@@ -302,6 +302,8 @@ ifeq ($(DOCKER_BUILDER_WORKDIR),)
 override DOCKER_BUILDER_WORKDIR := "/sonic"
 endif
 
+# Consider removing the --ulimit flag once nothing older
+# than Bullseye is being used as a slave container.
 DOCKER_RUN := docker run --rm=true --privileged --init \
     -v $(DOCKER_BUILDER_MOUNT) \
     -v "$(DOCKER_LOCKDIR):$(DOCKER_LOCKDIR)" \
@@ -310,6 +312,7 @@ DOCKER_RUN := docker run --rm=true --privileged --init \
     -e "https_proxy=$(https_proxy)" \
     -e "no_proxy=$(no_proxy)" \
     -i$(shell { if [ -t 0 ]; then echo t; fi }) \
+    --ulimit nofile=524288:524288 \
     $(SONIC_BUILDER_EXTRA_CMDLINE)
 
 # Mount the $(DOCKER_ROOT) to /var/lib/docker in the slave container, the overlay fs is not supported as dockerd root folder.

dockers/docker-base-bookworm/Dockerfile.j2

@@ -1,26 +1,14 @@
 {% set prefix = DEFAULT_CONTAINER_REGISTRY %}
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
 {% if CONFIGURED_ARCH == "armhf" and (MULTIARCH_QEMU_ENVIRON == "y" or CROSS_BUILD_ENVIRON == "y") %}
-FROM --platform=linux/arm/v7 {{ prefix }}debian:bookworm
+ARG BASE=--platform=linux/arm/v7 {{ prefix }}debian:bookworm
 {% elif CONFIGURED_ARCH == "arm64" and (MULTIARCH_QEMU_ENVIRON == "y" or CROSS_BUILD_ENVIRON == "y") %}
-FROM --platform=linux/arm64 {{ prefix }}debian:bookworm
+ARG BASE=--platform=linux/arm64 {{ prefix }}debian:bookworm
 {% else %}
-FROM {{ prefix }}{{DOCKER_BASE_ARCH}}/debian:bookworm
+ARG BASE={{ prefix }}{{DOCKER_BASE_ARCH}}/debian:bookworm
 {% endif %}
 
-# Clean documentation in FROM image
-RUN find /usr/share/doc -depth \( -type f -o -type l \) ! -name copyright | xargs rm || true
-
-# Clean doc directories that are empty or only contain empty directories
-RUN while [ -n "$(find /usr/share/doc -depth -type d -empty -print -exec rmdir {} +)" ]; do :; done && \
-    rm -rf               \
-    /usr/share/man/*     \
-    /usr/share/groff/*   \
-    /usr/share/info/*    \
-    /usr/share/lintian/* \
-    /usr/share/linda/*   \
-    /var/cache/man/*     \
-    /usr/share/locale/*
+FROM $BASE AS base
 
 # Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive
@@ -47,6 +35,8 @@ RUN apt update &&            \
         python-is-python3    \
         vim-tiny             \
         rsyslog              \
+# Install rsync for copying over only changes between layers
+        rsync                \
 # Install redis-tools
         redis-tools          \
 # common dependencies
@@ -98,17 +88,14 @@ RUN apt-get -y purge   \
 {{ install_debian_packages(docker_base_bookworm_debs.split(' ')) }}
 {%- endif %}
 
-# Clean up apt
-# Remove /var/lib/apt/lists/*, could be obsoleted for derived images
-RUN apt-get clean -y                     && \
-    apt-get autoclean -y                 && \
-    apt-get autoremove -y                && \
-    rm -rf /var/lib/apt/lists/* /tmp/* ~/.cache
-
 COPY ["etc/rsyslog.conf", "/etc/rsyslog.conf"]
 COPY ["etc/rsyslog.d/*", "/etc/rsyslog.d/"]
 COPY ["root/.vimrc", "/root/.vimrc"]
 
 RUN ln /usr/bin/vim.tiny /usr/bin/vim
 
 COPY ["etc/supervisor/supervisord.conf", "/etc/supervisor/"]
+
+FROM scratch
+
+COPY --from=base / /

dockers/docker-base-bullseye/Dockerfile.j2

@@ -1,26 +1,14 @@
 {% set prefix = DEFAULT_CONTAINER_REGISTRY %}
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
 {% if CONFIGURED_ARCH == "armhf" and (MULTIARCH_QEMU_ENVIRON == "y" or CROSS_BUILD_ENVIRON == "y") %}
-FROM {{ prefix }}multiarch/debian-debootstrap:armhf-bullseye
+ARG BASE={{ prefix }}multiarch/debian-debootstrap:armhf-bullseye
 {% elif CONFIGURED_ARCH == "arm64" and (MULTIARCH_QEMU_ENVIRON == "y" or CROSS_BUILD_ENVIRON == "y") %}
-FROM {{ prefix }}multiarch/debian-debootstrap:arm64-bullseye
+ARG BASE={{ prefix }}multiarch/debian-debootstrap:arm64-bullseye
 {% else %}
-FROM {{ prefix }}{{DOCKER_BASE_ARCH}}/debian:bullseye
+ARG BASE={{ prefix }}{{DOCKER_BASE_ARCH}}/debian:bullseye
 {% endif %}
 
-# Clean documentation in FROM image
-RUN find /usr/share/doc -depth \( -type f -o -type l \) ! -name copyright | xargs rm || true
-
-# Clean doc directories that are empty or only contain empty directories
-RUN while [ -n "$(find /usr/share/doc -depth -type d -empty -print -exec rmdir {} +)" ]; do :; done && \
-    rm -rf               \
-    /usr/share/man/*     \
-    /usr/share/groff/*   \
-    /usr/share/info/*    \
-    /usr/share/lintian/* \
-    /usr/share/linda/*   \
-    /var/cache/man/*     \
-    /usr/share/locale/*
+FROM $BASE AS base
 
 # Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive
@@ -45,6 +33,8 @@ RUN apt-get update &&        \
         python3-pip          \
         python-is-python3    \
         vim-tiny             \
+# Install rsync for copying over only changes between layers
+        rsync                \
 # Install redis-tools
         redis-tools          \
 # common dependencies
@@ -100,17 +90,14 @@ RUN apt-get -y purge   \
 {{ install_debian_packages(docker_base_bullseye_debs.split(' ')) }}
 {%- endif %}
 
-# Clean up apt
-# Remove /var/lib/apt/lists/*, could be obsoleted for derived images
-RUN apt-get clean -y                     && \
-    apt-get autoclean -y                 && \
-    apt-get autoremove -y                && \
-    rm -rf /var/lib/apt/lists/* /tmp/* ~/.cache
-
 COPY ["etc/rsyslog.conf", "/etc/rsyslog.conf"]
 COPY ["etc/rsyslog.d/*", "/etc/rsyslog.d/"]
 COPY ["root/.vimrc", "/root/.vimrc"]
 
 RUN ln /usr/bin/vim.tiny /usr/bin/vim
 
 COPY ["etc/supervisor/supervisord.conf", "/etc/supervisor/"]
+
+FROM scratch
+
+COPY --from=base / /

dockers/docker-config-engine-bookworm/Dockerfile.j2

@@ -1,5 +1,7 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-base-bookworm-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+ARG BASE=docker-base-bookworm-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+
+FROM $BASE as base
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive
@@ -50,10 +52,17 @@ COPY ["files/container_startup.py", "/usr/share/sonic/scripts/"]
 COPY ["00-load-omprog.conf", "/etc/rsyslog.d/"]
 
 ## Clean up
-RUN apt-get purge -y               \
-        python3-dev                \
-        build-essential         && \
-    apt-get clean -y            && \
-    apt-get autoclean -y        && \
-    apt-get autoremove -y       && \
-    rm -rf /debs /python-wheels ~/.cache
+
+{%- if CONFIGURED_ARCH == "armhf" or CONFIGURED_ARCH == "arm64" %}
+RUN apt-get purge -y        \
+        libxslt-dev         \
+        libz-dev
+{%- endif %}
+
+RUN apt-get purge -y        \
+        python3-dev         \
+        build-essential
+
+FROM $BASE
+
+RUN --mount=type=bind,from=base,target=/changes-to-image rsync -axAX --no-D --exclude=/sys --exclude=resolv.conf /changes-to-image/ /

dockers/docker-config-engine-bullseye/Dockerfile.j2

@@ -1,5 +1,7 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-base-bullseye-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+ARG BASE=docker-base-bullseye-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+
+FROM $BASE as base
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive
@@ -60,3 +62,7 @@ RUN apt-get purge -y               \
     apt-get autoclean -y        && \
     apt-get autoremove -y       && \
     rm -rf /debs /python-wheels ~/.cache
+
+FROM $BASE
+
+RUN --mount=type=bind,from=base,target=/changes-to-image rsync -axAX --no-D --exclude=/sys --exclude=resolv.conf /changes-to-image/ /

dockers/docker-database/Dockerfile.j2

@@ -1,5 +1,7 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-bookworm-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+ARG BASE=docker-config-engine-bookworm-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+
+FROM $BASE as base
 
 ARG docker_container_name
 
@@ -23,12 +25,8 @@ RUN pip3 install click
 {{ install_debian_packages(docker_database_debs.split(' ')) }}
 {%- endif %}
 
-# Clean up
-RUN apt-get clean -y                                  && \
-    apt-get autoclean -y                              && \
-    apt-get autoremove -y                             && \
-    rm -rf /debs ~/.cache                             && \
-    sed -ri 's/^# save ""$/save ""/g;                    \
+# Configure redis settings
+RUN sed -ri 's/^# save ""$/save ""/g;                    \
              s/^daemonize yes$/daemonize no/;            \
              s/^logfile .*$/logfile ""/;                 \
              s/^# syslog-enabled no$/syslog-enabled no/; \
@@ -50,4 +48,9 @@ COPY ["files/update_chassisdb_config", "/usr/local/bin/"]
 COPY ["flush_unused_database", "/usr/local/bin/"]
 COPY ["multi_database_config.json.j2", "/usr/share/sonic/templates/"]
 
+FROM $BASE
+
+RUN --mount=type=bind,from=base,target=/changes-to-image rsync -axAX --no-D --exclude=/sys --exclude=resolv.conf /changes-to-image/ /
+
+ENV DEBIAN_FRONTEND=noninteractive
 ENTRYPOINT ["/usr/local/bin/docker-database-init.sh"]

dockers/docker-dhcp-relay/Dockerfile.j2

@@ -1,15 +1,14 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-bookworm-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+ARG BASE=docker-config-engine-bookworm-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+
+FROM $BASE as base
 
 ARG docker_container_name
 ARG image_version
 
 # Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive
 
-# Pass the image_version to container
-ENV IMAGE_VERSION=$image_version
-
 # Update apt's cache of available packages
 RUN apt-get update
 
@@ -38,10 +37,6 @@ RUN pip3 install psutil
 # Clean up
 RUN apt-get remove -y build-essential     \
                 python3-dev
-RUN apt-get clean -y         && \
-    apt-get autoclean -y     && \
-    apt-get autoremove -y    && \
-    rm -rf /debs
 
 COPY ["docker_init.sh", "start.sh", "/usr/bin/"]
 COPY ["docker-dhcp-relay.supervisord.conf.j2", "port-name-alias-map.txt.j2", "wait_for_intf.sh.j2", "/usr/share/sonic/templates/"]
@@ -60,4 +55,12 @@ RUN rm -f /etc/rsyslog.d/rsyslog_plugin.conf.j2
 RUN rm -f /etc/rsyslog.d/events_info.json
 {% endif %}
 
+FROM $BASE
+
+RUN --mount=type=bind,from=base,target=/changes-to-image rsync -axAX --no-D --exclude=/sys --exclude=resolv.conf /changes-to-image/ /
+
+# Pass the image_version to container
+ENV IMAGE_VERSION=$image_version
+
+ENV DEBIAN_FRONTEND=noninteractive
 ENTRYPOINT ["/usr/bin/docker_init.sh"]

dockers/docker-dhcp-server/Dockerfile.j2

@@ -1,15 +1,14 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-bookworm-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+ARG BASE=docker-config-engine-bookworm-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+
+FROM $BASE as base
 
 ARG docker_container_name
 ARG image_version
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive
 
-# Pass the image_version to container
-ENV IMAGE_VERSION=$image_version
-
 RUN apt-get update &&       \
     apt-get install -f -y   \
         tcpdump             \
@@ -41,11 +40,6 @@ RUN pip3 install psutil
 RUN apt-get remove -y build-essential     \
                       python3-dev
 
-RUN apt-get clean -y        && \
-    apt-get autoclean -y    && \
-    apt-get autoremove -y   && \
-    rm -rf /debs
-
 COPY ["docker_init.sh", "start.sh", "/usr/bin/"]
 COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
 COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
@@ -55,4 +49,12 @@ COPY ["lease_update.sh", "/etc/kea/"]
 COPY ["kea-dhcp4-init.conf", "/etc/kea/kea-dhcp4.conf"]
 COPY ["cli", "/cli/"]
 
+FROM $BASE
+
+RUN --mount=type=bind,from=base,target=/changes-to-image rsync -axAX --no-D --exclude=/sys --exclude=resolv.conf /changes-to-image/ /
+
+# Pass the image_version to container
+ENV IMAGE_VERSION=$image_version
+
+ENV DEBIAN_FRONTEND=noninteractive
 ENTRYPOINT ["/usr/bin/docker_init.sh"]

dockers/docker-eventd/Dockerfile.j2

@@ -1,16 +1,14 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-bookworm-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+ARG BASE=docker-config-engine-bookworm-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+
+FROM $BASE as base
 
 ARG docker_container_name
 ARG image_version
-RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf
 
 # Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive
 
-# Pass the image_version to container
-ENV IMAGE_VERSION=$image_version
-
 # Update apt's cache of available packages
 RUN apt-get update
 
@@ -22,15 +20,17 @@ RUN apt-get update
 {{ install_debian_packages(docker_eventd_debs.split(' ')) }}
 {%- endif %}
 
-# Clean up
-RUN apt-get clean -y         && \
-    apt-get autoclean -y     && \
-    apt-get autoremove -y    && \
-    rm -rf /debs
-
 COPY ["start.sh", "/usr/bin/"]
 COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
 COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
 COPY ["critical_processes", "/etc/supervisor"]
 
+FROM $BASE
+
+RUN --mount=type=bind,from=base,target=/changes-to-image rsync -axAX --no-D --exclude=/sys --exclude=resolv.conf /changes-to-image/ /
+
+# Pass the image_version to container
+ENV IMAGE_VERSION=$image_version
+
+ENV DEBIAN_FRONTEND=noninteractive
 ENTRYPOINT ["/usr/local/bin/supervisord"]