WIP - Redo CI and Include Automated Docker Deployment

.travis.yml

@@ -33,105 +33,106 @@ branches:
         - develop
         - release
         - /^v[0-9]/
-matrix:
-    include:
-        # Ubuntu 14.04 LTS "Trusty Tahr"
-        # https://en.wikipedia.org/wiki/List_of_Ubuntu_releases#Ubuntu_14.04_LTS_.28Trusty_Tahr.29
-        #
-        # TravisCI doesn't directly support any new Ubuntu releases.  These is
-        # some Docker support, which we should probably investigate, at least for
-        # Ubuntu 16.04 LTS "Xenial Xerus"
-        # See https://en.wikipedia.org/wiki/List_of_Ubuntu_releases#Ubuntu_16.04_LTS_.28Xenial_Xerus.29.
-        - os: linux
-          dist: trusty
-          sudo: required
-          compiler: gcc
-          env:
-              - ZIP_SUFFIX=ubuntu-trusty
 
-        - os: linux
-          dist: trusty
-          sudo: required
-          compiler: clang
-          env:
-              - ZIP_SUFFIX=ubuntu-trusty-clang
+env:
+    global:
+        - ENCRYPTION_LABEL="6d4541b72666"
+        - TRAVIS_BUILD_TYPE=RelWithDebInfo
+        - TRAVIS_DOCS=Off
+        - TRAVIS_EMSCRIPTEN=Off
+        - TRAVIS_INSTALL_DEPS=On
+        - TRAVIS_RELEASE=On
+        - NON_IPC_TESTS=On
+        - DOCKER_SOLC=Off
+        - OS_VERSION=
+        - IPC_TESTS=Off
 
+matrix:
+    include:
+        # Ubuntu 14.04 LTS "Trusty Tahr" clang compile
+      - os: linux
+        dist: trusty 
+        sudo: required
+        services:
+            - docker
+        compiler:
+          - clang
+        env:
+          - IPC_TESTS=On
+          - OS_VERSION="Ubuntu-Clang"
+        # Ubuntu 14.04 LTS "Trusty Tahr" gcc compile
+      - os: linux
+        dist: trusty 
+        sudo: required
+        services:
+            - docker
+        compiler:
+          - gcc
+        env:
+          - OS_VERSION="Ubuntu-Gcc"
+        # Docker image in Alpine
+      - os: linux
+        dist: trusty 
+        sudo: required
+        services:
+          - docker
+        env: 
+            - DOCKER_SOLC=On
+            - OS_VERSION="Docker"
+            - TRAVIS_RELEASE=Off
+            - TRAVIS_INSTALL_DEPS=Off
         # Documentation target, which generates documentation using Phoenix / ReadTheDocs.
-        - os: linux
-          dist: trusty
-          sudo: required
-          compiler: gcc
-          env:
-              - TRAVIS_DOCS=On
-              - TRAVIS_RELEASE=Off
-              - TRAVIS_TESTS=Off
-
+      - os: linux
+        dist: trusty
+        sudo: required
+        before_install: apt-get -y install python-sphinx
+        env:
+            - TRAVIS_DOCS=On
+            - TRAVIS_RELEASE=Off
+            - TRAVIS_TESTS=Off
+            - TRAVIS_INSTALL_DEPS=Off
         # Emscripten target, which compiles 'solc' to javascript and uploads the resulting .js
         # files to https://github.com/ethereum/solc-bin.  These binaries are used in Browser-Solidity
         # and in other Ethereum web-based development contexts.
-        - os: linux
-          dist: trusty
-          sudo: required
-          compiler: gcc
-          node_js: stable
-          services:
-              - docker
-          before_install:
-              - docker pull trzeci/emscripten:sdk-tag-1.35.4-64bit
-          env:
-              - TRAVIS_EMSCRIPTEN=On
-              - TRAVIS_INSTALL_DEPS=Off
-              - TRAVIS_RELEASE=Off
-              - TRAVIS_TESTS=Off
-
-        # OS X Mavericks (10.9)
-        # https://en.wikipedia.org/wiki/OS_X_Mavericks
-        #
-# Disabled because of problems on travis.
-#        - os: osx
-#          osx_image: beta-xcode6.2
-#          env:
-#              - ZIP_SUFFIX=osx-mavericks
-
-        # OS X Yosemite (10.10)
-        # https://en.wikipedia.org/wiki/OS_X_Yosemite
-        #
-#        - os: osx
-#          osx_image: xcode7.1
-#          env:
-#              # Workaround for "macOS - Yosemite, El Capitan and Sierra hanging?"
-#              # https://github.com/ethereum/solidity/issues/894
-#              - TRAVIS_TESTS=Off
-#              - ZIP_SUFFIX=osx-yosemite
-
-        # OS X El Capitan (10.11)
-        # https://en.wikipedia.org/wiki/OS_X_El_Capitan
-        #
-#        - os: osx
-#          osx_image: xcode7.3
-#          env:
-#              # The use of Debug config here ONLY for El Capitan is a workaround for "The Heisenbug"
-#              # See https://github.com/ethereum/webthree-umbrella/issues/565
-#              - TRAVIS_BUILD_TYPE=Debug
-#              # Workaround for "macOS - Yosemite, El Capitan and Sierra hanging?"
-#              # https://github.com/ethereum/solidity/issues/894
-#              - TRAVIS_TESTS=Off
-#              - ZIP_SUFFIX=osx-elcapitan
-
-        # macOS Sierra (10.12)
-        # https://en.wikipedia.org/wiki/MacOS_Sierra
-        #
-#       - os: osx
-#          osx_image: xcode8
-#          env:
-#              # Look like "The Heisenbug" is occurring here too, so we'll do the same workaround.
-#              # See https://travis-ci.org/ethereum/solidity/jobs/150240930
-#              - TRAVIS_BUILD_TYPE=Debug
-#              # Workaround for "macOS - Yosemite, El Capitan and Sierra hanging?"
-#              # https://github.com/ethereum/solidity/issues/894
-#              - TRAVIS_TESTS=Off
-#              - ZIP_SUFFIX=macos-sierra
+      - os: linux
+        dist: trusty
+        sudo: required
+        compiler: gcc
+        node_js: stable
+        services:
+            - docker
+        before_install:
+            - docker pull trzeci/emscripten:sdk-tag-1.35.4-64bit
+        env:
+            - TRAVIS_EMSCRIPTEN=On
+            - TRAVIS_INSTALL_DEPS=Off
+            - TRAVIS_RELEASE=Off
+            - TRAVIS_TESTS=Off
+        # OS X images (assumes clang)
 
+        # Yosemite
+      - os: osx
+        osx_image: xcode6.4
+        env:
+          - OS_VERSION="OSX-xcode6.4"
+        # El Capitan
+      - os: osx
+        osx_image: xcode7.3
+        env:
+          - OS_VERSION="OSX-xcode7.3"
+      - os: osx
+        osx_image: xcode8
+        env:
+          - OS_VERSION="OSX-xcode8"
+        # Sierra
+      - os: osx
+        osx_image: xcode8.1
+        env:
+          - OS_VERSION="OSX-xcode8.1"
+      - os: osx
+        osx_image: xcode8.2
+        env:
+          - OS_VERSION="OSX-xcode8.2"
 git:
     depth: 2
 
@@ -143,42 +144,18 @@ cache:
         - $HOME/.local
 
 install:
-    - test $TRAVIS_INSTALL_DEPS != On || ./scripts/install_deps.sh
-    - test "$TRAVIS_OS_NAME" != "linux" || ./scripts/install_cmake.sh
+    - test $TRAVIS_INSTALL_DEPS != On || ./scripts/travis/travis_install_deps.sh
     - echo -n "$TRAVIS_COMMIT" > commit_hash.txt
+    - test $IPC_TESTS != On || docker pull ethereum/client-cpp
+
 before_script:
-    - test $TRAVIS_EMSCRIPTEN != On || ./scripts/build_emscripten.sh
-    - test $TRAVIS_RELEASE != On || (mkdir -p build
-      && cd build
-      && cmake .. -DCMAKE_BUILD_TYPE=$TRAVIS_BUILD_TYPE
-      && make -j2
-      && cd ..
-      && ./scripts/release.sh $ZIP_SUFFIX
-      && ./scripts/create_source_tarball.sh )
+    - test $TRAVIS_EMSCRIPTEN != On || ./scripts/travis/build_emscripten.sh
+    - test $TRAVIS_RELEASE != On || ./scripts/build.sh $TRAVIS_BUILD_TYPE
+    - test $DOCKER_SOLC != On || docker build --build-arg TEST=On --build-arg BUILD=$TRAVIS_BUILD_TYPE -t ethereum/solc:build -f ./scripts/Dockerfile $TRAVIS_BUILD_DIR
 script:
     - test $TRAVIS_DOCS != On || ./scripts/docs.sh
-
-    # There are a variety of reliability issues with the Solidity unit-tests at the time of
-    # writing (especially on macOS), so within TravisCI we will try to run the unit-tests
-    # up to 3 times before giving up and declaring the tests as broken.
-    #
-    # We should aim to remove this "retry logic" as soon as we can, because it is a
-    # band-aid for issues which need solving at their root.  Some of those issues will be
-    # in Solidity's RPC setup and some will be in 'eth'.  It seems unlikely that Solidity
-    # itself is broken from the failure messages which we are seeing.
-    #
-    # More details on known issues at https://github.com/ethereum/solidity/issues/769
-    - test $TRAVIS_TESTS != On || (cd $TRAVIS_BUILD_DIR && (./scripts/tests.sh || ./scripts/tests.sh || ./scripts/tests.sh) )
-env:
-    global:
-        - ENCRYPTION_LABEL="6d4541b72666"
-        - TRAVIS_BUILD_TYPE=RelWithDebInfo
-        - TRAVIS_DOCS=Off
-        - TRAVIS_EMSCRIPTEN=Off
-        - TRAVIS_INSTALL_DEPS=On
-        - TRAVIS_RELEASE=On
-        - TRAVIS_TESTS=On
-
+    - test $NON_IPC_TESTS != On || ./scripts/run_non_ipc_tests_and_collect_binaries.sh
+    - test $IPC_TESTS != On || ./scripts/run_ipc_tests.sh
 deploy:
     # This is the deploy target for the Emscripten build.
     # It publishes the JS file which was compiled as part of the earlier 'build_emscripten.sh'
@@ -211,4 +188,4 @@ deploy:
       on:
           all_branches: true
           tags: true
-          condition: $TRAVIS_RELEASE == On
+          condition: $TRAVIS_RELEASE == On

scripts/Dockerfile

@@ -1,12 +1,21 @@
 FROM alpine
 MAINTAINER chriseth <[email protected]>
+#Official solidity docker image
 
-RUN \
-  apk --no-cache --update add build-base cmake boost-dev git                                                && \
-  sed -i -E -e 's/include <sys\/poll.h>/include <poll.h>/' /usr/include/boost/asio/detail/socket_types.hpp  && \
-  git clone --depth 1 --recursive -b develop https://github.com/ethereum/solidity                           && \
-  cd /solidity && cmake -DCMAKE_BUILD_TYPE=Release -DTESTS=0 -DSTATIC_LINKING=1                             && \
-  cd /solidity && make solc && install -s  solc/solc /usr/bin                                               && \
-  cd / && rm -rf solidity                                                                                   && \
-  apk del sed build-base git make cmake gcc g++ musl-dev curl-dev boost-dev                                 && \
-  rm -rf /var/cache/apk/*
+#Establish working directory as soliditiy
+WORKDIR /solidity
+#Determine whether or not we are in the testing suite
+ARG TEST=Off
+#Determine what build type we are using
+ARG BUILD=Release
+#Copy working directory on travis to the image
+COPY / $WORKDIR
+
+#Install dependencies, eliminate annoying warnings, test if we are in a test environment, if so, 
+#add in soltest to the binary. If not, assume this is basic solidity image, build it and ship it out.
+RUN ./scripts/install_deps.sh && sed -i -E -e 's/include <sys\/poll.h>/include <poll.h>/' /usr/include/boost/asio/detail/socket_types.hpp
+RUN test $TEST == On && cmake -DCMAKE_BUILD_TYPE=$BUILD -DSTATIC_LINKING=1 || cmake -DCMAKE_BUILD_TYPE=$BUILD -DTESTS=0 -DSTATIC_LINKING=1
+RUN test $TEST == On && (make && install -s solc/solc /usr/bin && install -s test/soltest /usr/bin) || (make solc && install -s  solc/solc /usr/bin)
+RUN cd / && rm -rf solidity
+RUN apk del sed build-base git make cmake gcc g++ musl-dev curl-dev boost-dev
+RUN rm -rf /var/cache/apk/*

scripts/build.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+if [ -z "$1" ]; then 
+    BUILD_TYPE=Release
+else 
+    BUILD_TYPE="$1"
+fi
+
+mkdir -p build
+cd build
+cmake .. -DCMAKE_BUILD_TYPE="$BUILD_TYPE"
+make -j2 && install -s solc/solc /usr/bin && install -s test/soltest
+cd ..

scripts/install_deps.sh

@@ -100,13 +100,6 @@ case $(uname -s) in
         brew install boost
         brew install cmake
 
-        # We should really 'brew install' our eth client here, but at the time of writing
-        # the bottle is known broken, so we will just cheat and use a hardcoded ZIP for
-        # the time being, which is good enough.   The cause of the breaks will go away
-        # when we commit the repository reorg changes anyway.
-        curl -L -O https://github.com/bobsummerwill/cpp-ethereum/releases/download/v1.3.0/cpp-ethereum-osx-mavericks-v1.3.0.zip
-        unzip cpp-ethereum-osx-mavericks-v1.3.0.zip
-
         ;;
 
 #------------------------------------------------------------------------------
@@ -156,8 +149,7 @@ case $(uname -s) in
                 # All our dependencies can be found in the Alpine Linux official repositories.
                 # See https://pkgs.alpinelinux.org/
 
-                apk update
-                apk add boost-dev build-base cmake
+                apk --no-cache --update add build-base cmake boost-dev git
 
                 ;;
 
@@ -205,7 +197,6 @@ case $(uname -s) in
                 # Install "normal packages"
                 sudo apt-get -y update
                 sudo apt-get -y install \
-                    python-sphinx \
                     build-essential \
                     cmake \
                     g++ \
@@ -309,18 +300,11 @@ case $(uname -s) in
 
                 sudo apt-get -y update
                 sudo apt-get -y install \
-                    python-sphinx \
                     build-essential \
                     cmake \
                     git \
                     libboost-all-dev
 
-                # Install 'eth', for use in the Solidity Tests-over-IPC.
-                sudo add-apt-repository -y ppa:ethereum/ethereum
-                sudo add-apt-repository -y ppa:ethereum/ethereum-dev
-                sudo apt-get -y update
-                sudo apt-get -y install eth
-
                 ;;
 
 #------------------------------------------------------------------------------

scripts/run_ipc_tests.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env sh
+
+#mount volume so that host can access the tmp directory and the geth.ipc process
+docker run -di \
+	--volume /tmp:/tmp      \
+	--name testEth          \
+	ethereum/client-cpp     \
+	--test -d /tmp/testeth  
+
+echo "--> Running tests without optimizer..."
+soltest --run_test=SolidityOptimizer,GasMeterTests,SolidityEndToEndTest -- --ipcpath=/tmp/testeth/geth.ipc && \
+	echo "--> Running tests WITH optimizer..." && \
+	soltest --run_test=SolidityOptimizer,GasMeterTests,SolidityEndToEndTest -- --optimize --ipcpath=/tmp/testeth/geth.ipc
+ERROR_CODE=$?
+docker kill testEth
+exit $ERROR_CODE