Use base image for all compiler version

azure-pipelines.yml

@@ -13,157 +13,46 @@ jobs:
       python build.py
     env:
       DOCKER_PASSWORD: $(DOCKER_PASSWORD)
+      DOCKER_CACHE: True
   strategy:
     matrix:
-      CentOS 6 GCC 7 x86_64:
-        GCC_VERSIONS: "7"
-        DOCKER_ARCHS: "x86_64"
-        DOCKER_DISTRO: "centos6,jnlp-slave-centos6"
-      CentOS 6 GCC 7 x86:
-        GCC_VERSIONS: "7"
-        DOCKER_ARCHS: "x86"
-        DOCKER_DISTRO: "centos6,jnlp-slave-centos6"
-      Ubuntu MinGW GCC 7 x86_64:
+      Ubuntu GCC 10 CCI:
+        GCC_VERSIONS: "10"
+        BUILD_CONAN_CCI: 1
+      Ubuntu GCC 9 CCI:
+        GCC_VERSIONS: "9"
+        BUILD_CONAN_CCI: 1
+      Ubuntu GCC 8 CCI:
+        GCC_VERSIONS: "8"
+        BUILD_CONAN_CCI: 1
+      Ubuntu GCC 7 CCI:
         GCC_VERSIONS: "7"
-        DOCKER_ARCHS: "x86_64"
-        DOCKER_DISTRO: "mingw"
-
-      Ubuntu GCC 4.6:
-        GCC_VERSIONS: "4.6"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu GCC 4.8:
-        GCC_VERSIONS: "4.8"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu GCC 4.8 x86:
-        GCC_VERSIONS: "4.8"
-        DOCKER_ARCHS: "x86"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu GCC 4.9:
-        GCC_VERSIONS: "4.9"
-        DOCKER_ARCHS: "x86_64,armv7,armv7hf"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu GCC 4.9 x86:
-        GCC_VERSIONS: "4.9"
-        DOCKER_ARCHS: "x86"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu GCC 5:
-        GCC_VERSIONS: "5"
-        DOCKER_ARCHS: "x86_64,armv7,armv7hf,armv8"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu GCC 5 x86:
-        GCC_VERSIONS: "5"
-        DOCKER_ARCHS: "x86"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu GCC 5.2 x86_64:
-        GCC_VERSIONS: "5.2"
-      Ubuntu GCC 5.3 x86_64:
-        GCC_VERSIONS: "5.3"
-      Ubuntu GCC 6:
-        GCC_VERSIONS: "6"
-        DOCKER_ARCHS: "x86_64,armv7,armv7hf,armv8"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu GCC 6 x86:
+        BUILD_CONAN_CCI: 1
+      Ubuntu GCC 6 CCI:
         GCC_VERSIONS: "6"
-        DOCKER_ARCHS: "x86"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu GCC 6.3 x86_64:
-        GCC_VERSIONS: "6.3"
-      Ubuntu GCC 6.4 x86_64:
-        GCC_VERSIONS: "6.4"
-      Ubuntu GCC 7:
-        GCC_VERSIONS: "7"
-        DOCKER_ARCHS: "x86_64,armv7,armv7hf,armv8"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu GCC 7 x86:
-        GCC_VERSIONS: "7"
-        DOCKER_ARCHS: "x86"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu GCC 7.2 x86_64:
-        GCC_VERSIONS: "7.2"
-      Ubuntu GCC 8:
-        GCC_VERSIONS: "8"
-        DOCKER_ARCHS: "x86_64,armv7,armv7hf,armv8"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu GCC 8 x86:
-        GCC_VERSIONS: "8"
-        DOCKER_ARCHS: "x86"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu GCC 9:
-        GCC_VERSIONS: "9"
-        DOCKER_ARCHS: "x86_64,armv7,armv7hf,armv8"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu GCC 9 x86:
-        GCC_VERSIONS: "9"
-        DOCKER_ARCHS: "x86"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu GCC 10:
-        GCC_VERSIONS: "10"
-        DOCKER_ARCHS: "x86_64,armv7,armv7hf"
-        DOCKER_DISTRO: "jnlp-slave"
+        BUILD_CONAN_CCI: 1
 
-      Ubuntu Clang 3.8 x86_64:
-        CLANG_VERSIONS: "3.8"
-      Ubuntu Clang 3.9 x86_64:
-        CLANG_VERSIONS: "3.9"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu Clang 3.9 x86:
-        CLANG_VERSIONS: "3.9"
-        DOCKER_ARCHS: "x86"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu Clang 4.0 x86_64:
-        CLANG_VERSIONS: "4.0"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu Clang 4.0 x86:
-        CLANG_VERSIONS: "4.0"
-        DOCKER_ARCHS: "x86"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu Clang 5.0 x86_64:
-        CLANG_VERSIONS: "5.0"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu Clang 5.0 x86:
-        CLANG_VERSIONS: "5.0"
-        DOCKER_ARCHS: "x86"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu Clang 6.0 x86_64:
-        CLANG_VERSIONS: "6.0"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu Clang 6.0 x86:
-        CLANG_VERSIONS: "6.0"
-        DOCKER_ARCHS: "x86"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu Clang 7 x86_64:
-        CLANG_VERSIONS: "7"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu Clang 7 x86:
-        CLANG_VERSIONS: "7"
-        DOCKER_ARCHS: "x86"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu Clang 8 x86_64:
-        CLANG_VERSIONS: "8"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu Clang 8 x86:
-        CLANG_VERSIONS: "8"
-        DOCKER_ARCHS: "x86"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu Clang 9 x86_64:
-        CLANG_VERSIONS: "9"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu Clang 9 x86:
-        CLANG_VERSIONS: "9"
-        DOCKER_ARCHS: "x86"
-        DOCKER_DISTRO: "jnlp-slave"
-      Ubuntu Clang 10 x86_64:
+      Ubuntu Clang 10 CCI:
         CLANG_VERSIONS: "10"
-        DOCKER_DISTRO: "jnlp-slave"
-
-      Android Clang 8:
-        DOCKER_CROSS: "android"
+        BUILD_CONAN_CCI: 1
+      Ubuntu Clang 9 CCI:
+        CLANG_VERSIONS: "9"
+        BUILD_CONAN_CCI: 1
+      Ubuntu Clang 8 CCI:
         CLANG_VERSIONS: "8"
-        DOCKER_ARCHS: "x86_64,x86,armv7,armv8"
-
-      Conan Server:
-        BUILD_CONAN_SERVER_IMAGE: 1
-
-      Conan Tests:
-        BUILD_CONAN_TESTS: 1
-        BUILD_CONAN_TEST_AZURE: 1
+        BUILD_CONAN_CCI: 1
+      Ubuntu Clang 7 CCI:
+        CLANG_VERSIONS: "7"
+        BUILD_CONAN_CCI: 1
+      Ubuntu Clang 6 CCI:
+        CLANG_VERSIONS: "6.0"
+        BUILD_CONAN_CCI: 1
+      Ubuntu Clang 5 CCI:
+        CLANG_VERSIONS: "5.0"
+        BUILD_CONAN_CCI: 1
+      Ubuntu Clang 4 CCI:
+        CLANG_VERSIONS: "4.0"
+        BUILD_CONAN_CCI: 1
+      Ubuntu Clang 3.9 CCI:
+        CLANG_VERSIONS: "3.9"
+        BUILD_CONAN_CCI: 1

build.py

@@ -50,6 +50,7 @@ def _get_variables(self):
         build_server = self._get_boolean_var("BUILD_CONAN_SERVER_IMAGE")
         build_tests = self._get_boolean_var("BUILD_CONAN_TESTS")
         build_test_azure = self._get_boolean_var("BUILD_CONAN_TEST_AZURE")
+        build_cci = self._get_boolean_var("BUILD_CONAN_CCI")
         docker_password = os.getenv("DOCKER_PASSWORD", "").replace('"', '\\"')
         docker_username = os.getenv("DOCKER_USERNAME", "conanio")
         docker_login_username = os.getenv("DOCKER_LOGIN_USERNAME", "lasote")
@@ -77,11 +78,12 @@ def _get_variables(self):
             "clang_versions, visual_versions, build_server, "
             "docker_build_tag, docker_archs, sudo_command, "
             "docker_upload_only_when_stable, docker_cross, docker_cache, "
-            "build_tests, build_test_azure docker_upload_retry")
+            "build_tests, build_test_azure docker_upload_retry, build_cci")
         return Variables(docker_upload, docker_password, docker_username, docker_login_username,
                          gcc_versions, docker_distro, clang_versions, visual_versions, build_server,
                          docker_build_tag, docker_archs, sudo_command, docker_upload_only_when_stable,
-                         docker_cross, docker_cache, build_tests, build_test_azure, docker_upload_retry)
+                         docker_cross, docker_cache, build_tests, build_test_azure, docker_upload_retry,
+                         build_cci)
 
     def _get_boolean_var(self, var, default="false"):
         """ Parse environment variable as boolean type
@@ -148,6 +150,8 @@ def build(self):
         """
         logging.info("Starting build for service %s." % self.service)
         no_cache = "" if self.variables.docker_cache else "--no-cache"
+        subprocess.call("docker pull %s" % self.tagged_image_name.replace(
+                                           self.variables.docker_build_tag, "latest"), shell=True)
         subprocess.check_call("docker-compose build %s %s" % (no_cache, self.service), shell=True)
 
         output = subprocess.check_output("docker image inspect %s --format '{{.Size}}'"
@@ -264,7 +268,9 @@ def test_linux(self, arch, compiler_name, compiler_version, distro):
             subprocess.check_call("docker exec %s conan user" % self.service, shell=True)
 
             if compiler_name == "clang" and compiler_version == "7":
-                compiler_version = "7.0"  # FIXME: Remove this when fixed in conan
+                compiler_version = "7.0"
+                if "cci" in self.service:
+                    compiler_version = "7.1"
 
         subprocess.check_call(
             "docker exec %s conan install lz4/1.9.2@ -s "
@@ -367,6 +373,12 @@ def tag(self):
             self.tagged_image_name.replace("clang7", "clang70")), shell=True)
             subprocess.check_call("docker tag %s/clang7 %s/clang70" %
             (self.variables.docker_username, self.variables.docker_username), shell=True)
+        elif self.service == "clang7-cci":
+            logging.info("Clang 7 CCI will produce the alias Clang 7.1")
+            subprocess.check_call("docker tag %s %s" % (self.created_image_name,
+            self.tagged_image_name.replace("clang7-cci", "clang71-cci")), shell=True)
+            subprocess.check_call("docker tag %s/clang7-cci %s/clang71-cci" %
+            (self.variables.docker_username, self.variables.docker_username), shell=True)
 
     def info(self):
         """Show Docker image info
@@ -420,6 +432,9 @@ def process_regular_images(self):
                     tag_arch = "" if arch == "x86_64" else "-%s" % arch
                     service = "%s%s%s%s" % (cross, compiler.name, version.replace(".", ""), tag_arch)
                     build_dir = "%s%s_%s%s" % (cross, compiler.name, version, tag_arch)
+                    if self.variables.build_cci:
+                        service = "%s%s-cci" % (compiler.name, version.replace(".", ""))
+                        build_dir = "%s-cci" % (compiler.name)
 
                     self.service = service
                     self.login()

clang-cci/.dockerignore

@@ -0,0 +1,7 @@
+.git/
+.idea/
+
+*.md
+*.py
+*.yml
+**/*.sh

clang-cci/Dockerfile

@@ -0,0 +1,103 @@
+FROM ubuntu:xenial as base
+
+LABEL maintainer="Conan.io <info.conan.io>"
+
+ARG COMPILER_VERSION
+
+ENV LLVM_VERSION=$COMPILER_VERSION \
+    CC=clang \
+    CXX=clang++ \
+    CMAKE_C_COMPILER=clang \
+    CMAKE_CXX_COMPILER=clang++ \
+    PYENV_ROOT=/opt/pyenv \
+    PYTHON_VERSION=3.8.2 \
+    PATH=/opt/pyenv/shims:${PATH} \
+    CMAKE_VERSION=3.17.2
+
+RUN dpkg --add-architecture i386 \
+    && apt-get -qq update \
+    # Install basics
+    && apt-get install -y --no-install-recommends \
+       sudo \
+       wget \
+       git \
+       subversion \
+       make \
+       gnupg \
+       ca-certificates \
+       dh-autoreconf \
+       apt-transport-https
+    # Install compiler toolset
+RUN apt-get install -y --no-install-recommends \
+       software-properties-common \
+    && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \
+    && add-apt-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-${LLVM_VERSION} main" \
+    && apt-get update
+
+RUN apt-get install -y --no-install-recommends \
+       clang-${LLVM_VERSION} \
+       lldb-${LLVM_VERSION} \
+       libc++-${LLVM_VERSION}-dev \
+       libc++abi-${LLVM_VERSION}-dev \
+    && apt-get remove -y lsb-release software-properties-common
+
+    # Install dependencies of Python
+RUN apt-get install -y --no-install-recommends \
+       libreadline-dev \
+       libsqlite3-dev \
+       libffi-dev \
+       libssl-dev \
+       zlib1g-dev \
+       libbz2-dev \
+       xz-utils \
+       curl \
+       libncurses5-dev \
+       libncursesw5-dev \
+       liblzma-dev \
+    && apt-get autoremove -y \
+    && apt-get clean all
+
+    # Update default compiler
+RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-${LLVM_VERSION} 100 \
+    && update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${LLVM_VERSION} 100 \
+    && update-alternatives --install /usr/bin/cc cc /usr/bin/clang-${LLVM_VERSION} 100 \
+    && update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-${LLVM_VERSION} 100 \
+    && update-alternatives --install /usr/bin/cpp cpp /usr/bin/clang++-${LLVM_VERSION} 100 \
+    && rm -rf /var/lib/apt/lists/* \
+    && groupadd 1001 -g 1001 \
+    && groupadd 1000 -g 1000 \
+    && groupadd 2000 -g 2000 \
+    && groupadd 999 -g 999 \
+    && useradd -ms /bin/bash conan -g 1001 -G 1000,2000,999 \
+    && printf "conan:conan" | chpasswd \
+    && adduser conan sudo \
+    && printf "conan ALL= NOPASSWD: ALL\\n" >> /etc/sudoers
+
+RUN wget --no-check-certificate --quiet -O /tmp/pyenv-installer https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer \
+    && chmod +x /tmp/pyenv-installer \
+    && /tmp/pyenv-installer \
+    && rm /tmp/pyenv-installer \
+    && update-alternatives --install /usr/bin/pyenv pyenv /opt/pyenv/bin/pyenv 100 \
+    && PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install ${PYTHON_VERSION} \
+    && pyenv global ${PYTHON_VERSION} \
+    && pip install -q --upgrade --no-cache-dir pip \
+    && pip install -q --no-cache-dir conan conan-package-tools cmake==${CMAKE_VERSION} \
+    && chown -R conan:1001 /opt/pyenv
+
+    # remove all __pycache__ directories created by pyenv
+RUN find /opt/pyenv -iname __pycache__ -print0 | xargs -0 rm -rf \
+    && update-alternatives --install /usr/bin/python python /opt/pyenv/shims/python 100 \
+    && update-alternatives --install /usr/bin/python3 python3 /opt/pyenv/shims/python3 100 \
+    && update-alternatives --install /usr/bin/pip pip /opt/pyenv/shims/pip 100 \
+    && update-alternatives --install /usr/bin/pip3 pip3 /opt/pyenv/shims/pip3 100
+
+USER conan
+WORKDIR /home/conan
+
+RUN mkdir -p /home/conan/.conan \
+    && printf 'eval "$(pyenv init -)"\n' >> ~/.bashrc \
+    && printf 'eval "$(pyenv virtualenv-init -)"\n' >> ~/.bashrc
+
+FROM base as release
+
+RUN pip install -U conan conan-package-tools