Skip to content

Commit

Permalink
Hardened Nginx and S390x changes
Browse files Browse the repository at this point in the history
Signed-off-by: galal-hussein <[email protected]>
  • Loading branch information
galal-hussein authored and dereknola committed Oct 19, 2022
1 parent e60b521 commit ef72da0
Show file tree
Hide file tree
Showing 13 changed files with 342 additions and 186 deletions.
46 changes: 16 additions & 30 deletions build/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -34,46 +34,32 @@ mandatory=(
TAG
)

missing=false
for var in "${mandatory[@]}"; do
if [[ -z "${!var:-}" ]]; then
echo "Environment variable $var must be set"
exit 1
fi
done

export CGO_ENABLED=0
export GOARCH=${ARCH}

TARGETS_DIR="rootfs/bin/${ARCH}"
echo "Building targets for ${ARCH}, generated targets in ${TARGETS_DIR} directory."

echo "Building ${PKG}/cmd/nginx"

pushd /go/src/k8s.io/ingress-nginx
git config --add safe.directory /go/src/k8s.io/ingress-nginx
popd

${GO_BUILD_CMD} \
-trimpath -ldflags="-buildid= -w -s \
-X ${PKG}/version.RELEASE=${TAG} \
-X ${PKG}/version.COMMIT=${COMMIT_SHA} \
-X ${PKG}/version.REPO=${REPO_INFO}" \
-o "${TARGETS_DIR}/nginx-ingress-controller" "${PKG}/cmd/nginx"
if [ "$missing" = true ]; then
exit 1
fi

echo "Building ${PKG}/cmd/dbg"
export CGO_ENABLED=1
export GOARCH=${ARCH}

${GO_BUILD_CMD} \
-trimpath -ldflags="-buildid= -w -s \
export GO_LDFLAGS="-linkmode=external -buildid= \
-X ${PKG}/version.RELEASE=${TAG} \
-X ${PKG}/version.COMMIT=${COMMIT_SHA} \
-X ${PKG}/version.REPO=${REPO_INFO}" \
-o "${TARGETS_DIR}/dbg" "${PKG}/cmd/dbg"
-X ${PKG}/version.REPO=${REPO_INFO}"

echo "Building ${PKG}/cmd/waitshutdown"
go-build-static.sh -trimpath -o "rootfs/bin/${ARCH}/nginx-ingress-controller" "${PKG}/cmd/nginx"
go-build-static.sh -trimpath -o "rootfs/bin/${ARCH}/dbg" "${PKG}/cmd/dbg"
go-build-static.sh -trimpath -o "rootfs/bin/${ARCH}/wait-shutdown" "${PKG}/cmd/waitshutdown"

${GO_BUILD_CMD} \
-trimpath -ldflags="-buildid= -w -s \
-X ${PKG}/version.RELEASE=${TAG} \
-X ${PKG}/version.COMMIT=${COMMIT_SHA} \
-X ${PKG}/version.REPO=${REPO_INFO}" \
-o "${TARGETS_DIR}/wait-shutdown" "${PKG}/cmd/waitshutdown"
go-assert-static.sh rootfs/bin/${ARCH}/*
if [[ ${ARCH} != "s390x" ]]; then
go-assert-boring.sh rootfs/bin/${ARCH}/*
fi
strip rootfs/bin/${ARCH}/*
2 changes: 1 addition & 1 deletion build/dev-env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ docker tag "${REGISTRY}/controller:${TAG}" "${DEV_IMAGE}"

export K8S_VERSION=${K8S_VERSION:-v1.24.2@sha256:1f0cee2282f43150b52dc7933183ed96abdcfc8d293f30ec07082495874876f1}

KIND_CLUSTER_NAME="ingress-nginx-dev"
KIND_CLUSTER_NAME="ingress-nginx-dev-${RANDOM}"

if ! kind get clusters -q | grep -q ${KIND_CLUSTER_NAME}; then
echo "[dev-env] creating Kubernetes cluster with kind"
Expand Down
17 changes: 12 additions & 5 deletions hack/init-buildx.sh
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ fi
# Ensure qemu is in binfmt_misc
# Docker desktop already has these in versions recent enough to have buildx
# We only need to do this setup on linux hosts
if [ "$(uname)" == 'Linux' ]; then
# We don't need qemu in s390x, since we won't cross compile
if [ "$(uname)" == 'Linux' ] && [ ${ARCH} != "s390x" ]; then
# NOTE: this is pinned to a digest for a reason!
# Note2 (@rikatz) - Removing the pin, as apparently it's breaking new alpine builds
# docker run --rm --privileged multiarch/qemu-user-static@sha256:28ebe2e48220ae8fd5d04bb2c847293b24d7fbfad84f0b970246e0a4efd48ad6 --reset -p yes
Expand All @@ -51,7 +52,13 @@ if ! grep -q "^Driver: docker$" <<<"${current_builder}" && \
exit 0
fi


# Ensure we use a builder that can leverage it (the default on linux will not)
docker buildx rm ingress-nginx || true
docker buildx create --use --name=ingress-nginx
# Ensure qemu is in binfmt_misc
# Docker desktop already has these in versions recent enough to have buildx
# We only need to do this setup on linux hosts
# We don't need qemu in s390x, since we won't cross compile
if [ "$(uname)" == 'Linux' ] && [ ${ARCH} != "s390x" ]; then
# NOTE: this is pinned to a digest for a reason!
# https://github.com/docker/buildx/issues/542#issuecomment-778835576
docker run --rm --privileged tonistiigi/binfmt --uninstall qemu-aarch64 && docker run --rm --privileged tonistiigi/binfmt --install arm64
docker run --rm --privileged tonistiigi/binfmt
fi
12 changes: 9 additions & 3 deletions images/nginx/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,21 @@ IMAGE = $(REGISTRY)/nginx
export DOCKER_CLI_EXPERIMENTAL=enabled

# build with buildx
PLATFORMS?=linux/amd64,linux/arm,linux/arm64,linux/s390x
PLATFORMS ?= amd64 arm arm64 s390x

EMPTY :=
SPACE := $(EMPTY) $(EMPTY)
COMMA := ,

OUTPUT=
PROGRESS=plain
build: ensure-buildx
docker buildx build \
--platform=${PLATFORMS} $(OUTPUT) \
-f rootfs/Dockerfile.$(PLATFORMS) \
--platform=$(subst $(SPACE),$(COMMA),$(PLATFORMS)) $(OUTPUT) \
--progress=$(PROGRESS) \
--pull \
--tag $(IMAGE):$(TAG) rootfs
--tag $(IMAGE):$(TAG)-$(PLATFORMS) rootfs

# push the cross built image
push: OUTPUT=--push
Expand Down
74 changes: 0 additions & 74 deletions images/nginx/rootfs/Dockerfile

This file was deleted.

100 changes: 100 additions & 0 deletions images/nginx/rootfs/Dockerfile.amd64
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
# Copyright 2015 The Kubernetes Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


# Use a multi-stage build
FROM rockylinux:8 as builder

RUN yum -y update \
&& yum -y upgrade \
&& yum -y install epel-release 'dnf-command(config-manager)' \
&& yum -y upgrade \
&& yum config-manager --set-enabled powertools \
&& yum -y install dnf
RUN dnf -y install conntrack-tools findutils which

COPY etc /etc
COPY patches /patches
COPY build.sh /

RUN /build.sh

# Pull static components from alpine
FROM alpine:3.13 as extras

RUN apk update \
&& apk upgrade \
&& apk add -U --no-cache dumb-init

# With UBI as base image
FROM registry.access.redhat.com/ubi8/ubi-minimal

ENV PATH=$PATH:/usr/local/luajit/bin:/usr/local/nginx/sbin:/usr/local/nginx/bin

ENV LUA_PATH="/usr/local/share/luajit-2.1.0-beta3/?.lua;/usr/local/share/lua/5.1/?.lua;/usr/local/lib/lua/?.lua;;"
ENV LUA_CPATH="/usr/local/lib/lua/?/?.so;/usr/local/lib/lua/?.so;;"

COPY --from=builder /usr/local /usr/local
COPY --from=builder /opt /opt
COPY --from=builder /etc/nginx /etc/nginx
COPY --from=extras /usr/bin/dumb-init /usr/bin/dumb-init

RUN rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
RUN microdnf -y update && rm -rf /var/cache/yum
RUN microdnf -y install lmdb-libs || rpm -iv http://vault.centos.org/centos/8/BaseOS/x86_64/os/Packages/lmdb-libs-0.9.24-1.el8.x86_64.rpm
RUN microdnf -y install \
util-linux \
findutils \
which \
yajl \
GeoIP \
libmaxminddb \
wget

RUN microdnf -y install crypto-policies-scripts

RUN rm -rf /var/cache/yum

RUN ldDirs=" \
/usr/local/lib \
/usr/local/lib64 \
"; \
for dir in ${ldDirs}; do \
echo "${dir}" >>/etc/ld.so.conf.d/local.conf; \
done
RUN /sbin/ldconfig

RUN ln -s /usr/local/nginx/sbin/nginx /sbin/nginx
RUN groupadd -rg 101 www-data
RUN adduser -u 101 -M -d /usr/local/nginx -s /sbin/nologin -G www-data -g www-data www-data

RUN writeDirs=" \
/var/log/nginx \
/var/lib/nginx/body \
/var/lib/nginx/fastcgi \
/var/lib/nginx/proxy \
/var/lib/nginx/scgi \
/var/lib/nginx/uwsgi \
/var/log/audit \
"; \
for dir in ${writeDirs}; do \
mkdir -p ${dir}; \
chown -R www-data.www-data ${dir}; \
done

RUN microdnf clean all

EXPOSE 80 443

CMD ["nginx", "-g", "daemon off;"]
96 changes: 96 additions & 0 deletions images/nginx/rootfs/Dockerfile.s390x
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
# Copyright 2015 The Kubernetes Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


# Use a multi-stage build
FROM registry.opensuse.org/opensuse/leap-dnf:15.3 as builder

RUN dnf makecache \
&& dnf -y upgrade
RUN dnf -y install conntrack-tools findutils which microdnf libcap-progs

COPY etc /etc
COPY patches /patches
COPY build.sh /

RUN /build.sh

# Pull static components from alpine
FROM alpine:3.13 as extras

RUN apk update \
&& apk upgrade \
&& apk add -U --no-cache dumb-init

# With OpenSUSE Leap DNF as base image
FROM registry.opensuse.org/opensuse/leap-dnf:15.3

ENV PATH=$PATH:/usr/local/luajit/bin:/usr/local/nginx/sbin:/usr/local/nginx/bin
ENV LUA_PATH="/usr/local/share/luajit-2.1.0-beta3/?.lua;/usr/local/share/lua/5.1/?.lua;/usr/local/lib/lua/?.lua;;"
ENV LUA_CPATH="/usr/local/lib/lua/?/?.so;/usr/local/lib/lua/?.so;;"

COPY --from=builder /usr/local /usr/local
COPY --from=builder /opt /opt
COPY --from=builder /etc/nginx /etc/nginx
COPY --from=extras /usr/bin/dumb-init /usr/bin/dumb-init

RUN dnf -y install \
util-linux \
findutils \
which \
yajl \
GeoIP \
libmaxminddb0 \
lmdb \
wget \
libcap-progs

RUN dnf -y install 'dnf-command(config-manager)'
RUN dnf config-manager --add-repo https://download.opensuse.org/repositories/security:tls/openSUSE_Leap_15.3/security:tls.repo
RUN dnf -y install crypto-policies-scripts

RUN rm -rf /var/cache/yum

RUN ldDirs=" \
/usr/local/lib \
/usr/local/lib64 \
"; \
for dir in ${ldDirs}; do \
echo "${dir}" >>/etc/ld.so.conf.d/local.conf; \
done
RUN /sbin/ldconfig

RUN ln -s /usr/local/nginx/sbin/nginx /sbin/nginx
RUN groupadd -rg 101 www-data
RUN useradd -u 101 -M -d /usr/local/nginx -s /sbin/nologin -G www-data -g www-data www-data

RUN writeDirs=" \
/var/log/nginx \
/var/lib/nginx/body \
/var/lib/nginx/fastcgi \
/var/lib/nginx/proxy \
/var/lib/nginx/scgi \
/var/lib/nginx/uwsgi \
/var/log/audit \
"; \
for dir in ${writeDirs}; do \
mkdir -p ${dir}; \
chown -R www-data.www-data ${dir}; \
done

RUN dnf clean all

EXPOSE 80 443

CMD ["nginx", "-g", "daemon off;"]
Loading

0 comments on commit ef72da0

Please sign in to comment.