.gitlab-ci.yml

#############################################################################
# gitlab-runner:docker executor options
image: proteusdb/pelago-build:cuda11.3-llvm14.0.0

#############################################################################
variables:
 GIT_SUBMODULE_STRATEGY: recursive

#############################################################################
stages:
 - Code quality         # Environment configuration

 - Configure            # Environment configuration

 - Build                # Build the project

 - Install              # Install the software so it can be run for tests

 - Test                 # Functional tests

 - Benchmark            # Performance tests (need to be tagged in order to be
                        # run on a noise-free system)

 - Package              # Create archives to be installed

 - Publish              # Publish the packages

 - Build Profile        # Build the project with profiling information

 - Profile              # Profile using known workload

#############################################################################
# Helper Templates

before_script:
 # TODO: we should probably remove the next lines and fix that in executor
 - ln -s /data/ssbm100 tests/inputs/ssbm100
 - ln -s /data/tpch1 tests/inputs/tpch1
 - ulimit -c 0
 - export ccache_dir=$(pwd | sed "s/^\/builds/\/cache/g")/.ccache
 - export build_path=$(pwd | sed "s/^\/builds/\/cache/g")/${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHA}/cmake-build-debug
 - export CCACHE_DIR=${ccache_dir}
 - export CCACHE_NOHASHDIR=1
 - export CCACHE_BASEDIR=${build_path}
 - mkdir -p ${build_path}
 - export CWD=$(pwd)
 - ccache --zero-stats

# after_script:
#  - true


#----------------------------------------------------------------------------
# Caching policies templates

.cache: &cache
 tags:
  - pelago
 cache:
  key: "${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHA}"
  paths:
   - src/panorama
   - src/planner
   - src/SQLPlanner

.cache: &cache_profile
 cache:
  key: "${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHA}-profile"
  paths:
   - src/panorama
   - src/planner
   - src/SQLPlanner

#----------------------------------------------------------------------------
# Execution restrictions templates
.restrict_precommit: &restrict_precommit
 rules: # On anything without a tag
  # See https://gitlab.com/gitlab-org/gitlab-foss/-/issues/67456
  # for the condition is not checking against a simple empty ("") string
  - if: '$CI_COMMIT_TAG =~ "/^$/"'

.restrict_postcommit: &restrict_postcommit
 rules: # On the default branch, or tags
  - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
  - if: $CI_COMMIT_TAG

.restrict_latest: &restrict_latest
 rules: # Only on the non-tagged commits, in the default branch
  - if: '$CI_COMMIT_TAG =~ "/^$/" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'

.restrict_release: &restrict_release
 rules: # Only on tags
  - if: $CI_COMMIT_TAG

#----------------------------------------------------------------------------
# Job templates
.test: &test
 stage: Test
 cache:
  policy: pull
 <<: *cache
 <<: *restrict_precommit

.benchmark: &benchmark
 stage: Benchmark
 cache:
  policy: pull
 <<: *cache

#############################################################################
# Example jobs
#----------------------------------------------------------------------------

#----------------------------------------------------------------------------

codestyle:
 stage: Code quality
 script:
  # clang-format comes from the llvm repo: https://apt.llvm.org/
  # remember to update it every time we pop the LLVM version!
  - find . \( -name '*.cpp' -o -name '*.hpp' -o -name '*.cu' -o -name '*.cuh' -o -name '*.td' \) | xargs -n1 clang-format --dry-run --Werror
 <<: *cache
 <<: *restrict_precommit

licenses:
 stage: Code quality
 script:
  - python tools/check_licenses.py
 <<: *cache
 <<: *restrict_precommit

configure:
 stage: Configure
 script:
  - cd ${build_path} && cmake --preset "Proteus" -S $CWD -B ${build_path}
  - echo "Configure... done"
 <<: *cache
 <<: *restrict_precommit

#----------------------------------------------------------------------------
# Setup test environment
test-configuration:
 stage: Configure
 script:
  - clang++ --verbose
  - clang++ --verbose 2>&1 > /dev/null | grep "Found CUDA installation"
  - clang++ --verbose 2>&1 > /dev/null | grep "Found CUDA installation" | grep "version unknown" || exit 0
  - exit 1
 <<: *cache
 <<: *restrict_precommit


#----------------------------------------------------------------------------
# Setup test environment
build:
 stage: Build
 script:
  - cmake --build ${build_path} --target all -- -j`nproc --all`
  - ccache --show-stats
  - echo "Building... done"
 <<: *cache
 <<: *restrict_precommit

build panorama:
 stage: Build
 script:
  - cd tools/panorama
  - rm -rf node_modules
  - npm install yarn
  - export PATH=node_modules/.bin/:$PATH
#  latest LTS (18) does not work on ubuntu 18.04 due to a dependency on a newer libc
  - yarn global add node@16.19.0
  - export PATH=node_modules/.bin/:$(yarn global dir)/node_modules/node/bin:$PATH
  - yarn install
  # - export NODE_OPTIONS=--openssl-legacy-provider
  - ng build --prod
 <<: *cache
 <<: *restrict_precommit

install:
 stage: Install
 script:
  - cmake --build ${build_path} --target install -- -j`nproc --all`
  - echo "Installing... done"
 <<: *cache
 <<: *restrict_precommit

#----------------------------------------------------------------------------
# Functional Tests
Test 01 - Threads:
 <<: *test
 script:
  - cmake --build ${build_path} --target install -- -j`nproc --all`
  - cd ${build_path}/opt/pelago && ./unit-tests-threads --gtest_output="xml:$CWD/report-threads.xml"
    #- cd ${build_path}/opt/pelago && ./unit-tests-olap --gtest_filter='ThreadTest.*' --gtest_output="xml:$CWD/report-threads.xml"
 artifacts:
  reports:
   junit: report-threads.xml

Test 02 - Plan Parsing:
 <<: *test
 script:
  - cmake --build ${build_path} --target install -- -j`nproc --all`
  - cd ${build_path}/opt/pelago && ./unit-tests-plan-parsing --gtest_output="xml:$CWD/report-plan-parsing.xml"
    #- cd ${build_path}/opt/pelago && ./unit-tests-olap --gtest_filter='PlanTest.*' --gtest_output="xml:$CWD/report-plan-parsing.xml"
 artifacts:
  reports:
   junit: report-plan-parsing.xml

Test 03 - JSON:
 <<: *test
 script:
  - cmake --build ${build_path} --target install -- -j`nproc --all`
  - cd ${build_path}/opt/pelago && ./unit-json-test --gtest_output="xml:$CWD/report-json.xml"
 allow_failure: true
 artifacts:
  reports:
   junit: report-json.xml

Test 04 - Operators:
 <<: *test
 script:
  - cmake --build ${build_path} --target install -- -j`nproc --all`
  - cd ${build_path}/opt/pelago && ./unit-operator-tests --gtest_output="xml:$CWD/report-joins.xml"
 allow_failure: true
 artifacts:
  reports:
   junit: report-joins.xml

Test 05 - Output:
 <<: *test
 script:
  - cmake --build ${build_path} --target install -- -j`nproc --all`
  - cd ${build_path}/opt/pelago && ./unit-tests-output --gtest_output="xml:$CWD/report-output.xml"
    #- cd ${build_path}/opt/pelago && ./unit-tests-olap --gtest_filter='OutputTest.*' --gtest_output="xml:$CWD/report-output.xml"
 artifacts:
  reports:
   junit: report-output.xml

Test 06 - GPU SSB SF100:
 <<: *test
 script:
  - cmake --build ${build_path} --target install -- -j`nproc --all`
  - cd ${build_path}/opt/pelago && ./unit-tests-gpu-ssb --gtest_output="xml:$CWD/report-gpu.xml"
    #- cd ${build_path}/opt/pelago && ./unit-tests-olap --gtest_filter='GPUSSBTest.*' --gtest_output="xml:$CWD/report-gpu.xml"
 artifacts:
  reports:
   junit: report-gpu.xml

Test 07 - CPU SSB SF100:
 <<: *test
 script:
  - cmake --build ${build_path} --target install -- -j`nproc --all`
  - cd ${build_path}/opt/pelago && ./unit-tests-cpu-ssb --gtest_output="xml:$CWD/report-cpu.xml"
    #- cd ${build_path}/opt/pelago && ./unit-tests-olap --gtest_filter='CPUSSBTest.*' --gtest_output="xml:$CWD/report-cpu.xml"
 artifacts:
  reports:
   junit: report-cpu.xml

Test 08 - Hybrid SSB SF100:
 <<: *test
 script:
  - cmake --build ${build_path} --target install -- -j`nproc --all`
  - cd ${build_path}/opt/pelago && ./unit-tests-hyb-ssb --gtest_output="xml:$CWD/report-hyb.xml"
    #- cd ${build_path}/opt/pelago && ./unit-tests-olap --gtest_filter='HYBSSBTest.*' --gtest_output="xml:$CWD/report-hyb.xml"
 artifacts:
  reports:
   junit: report-hyb.xml

Test 09 - COW:
 <<: *test
 script:
  - cmake --build ${build_path} --target install -- -j`nproc --all`
  - cd ${build_path}/opt/pelago && ./unit-tests-cow --gtest_output="xml:$CWD/report-cow.xml"
    #- cd ${build_path}/opt/pelago && ./unit-tests-olap --gtest_filter='ThreadCOW.*' --gtest_output="xml:$CWD/report-cow.xml"
 artifacts:
  reports:
   junit: report-cow.xml

Test 10 - Planner:
 <<: *test
 script:
  - cmake --build ${build_path} --target install -- -j`nproc --all`
  - ln -s ${build_path}/opt/pelago/inputs ${build_path}/core/planner/CMakeFiles/planner.dir/inputs
  - export OUTPUT=0
  - cd ${build_path}/core/planner/CMakeFiles/planner.dir && sbt test || export OUTPUT=1
  - mv ${build_path}/core/planner/CMakeFiles/planner.dir/target/test-reports/TEST-*.xml $CWD
  - rm ${build_path}/core/planner/CMakeFiles/planner.dir/inputs || true
  - exit $OUTPUT
 allow_failure: true
 artifacts:
  reports:
   junit: TEST-*.xml

Test 11 - OLTP YCSB:
 <<: *test
 script:
  - cmake --build ${build_path} --target install -- -j`nproc --all`
  - cd ${build_path}/opt/pelago && ./unit-tests-ycsb --gtest_output="xml:$CWD/report-ycsb.xml"
  #- cd ${build_path}/opt/pelago && ./unit-tests-olap --gtest_filter='ThreadCOW.*' --gtest_output="xml:$CWD/report-cow.xml"
 artifacts:
  reports:
   junit: report-ycsb.xml

Test 12 - OLTP TPCC:
 <<: *test
 script:
  - cmake --build ${build_path} --target install -- -j`nproc --all`
  - cd ${build_path}/opt/pelago && ./unit-tests-tpcc --gtest_output="xml:$CWD/report-tpcc.xml"
  #- cd ${build_path}/opt/pelago && ./unit-tests-olap --gtest_filter='ThreadCOW.*' --gtest_output="xml:$CWD/report-cow.xml"
 artifacts:
  reports:
   junit: report-tpcc.xml

Test 13 - Platform - Memory Allocator:
 <<: *test
 script:
  - cmake --build ${build_path} --target install -- -j`nproc --all`
  - cd ${build_path}/opt/pelago && ./unit-memallocator-test --gtest_output="xml:$CWD/report-memallocator-test.xml"
 artifacts:
  reports:
   junit: report-memallocator-test.xml

Test 14 - Storage - File Request Interface:
 <<: *test
 script:
  - cmake --build ${build_path} --target install -- -j`nproc --all`
  - cd ${build_path}/opt/pelago && ./unit-request-interface-test --gtest_output="xml:$CWD/report-request-interface-test.xml"
 artifacts:
  reports:
   junit: report-request-interface-test.xml

Test 15 - CSV:
 <<: *test
 script:
  - cmake --build ${build_path} --target install -- -j`nproc --all`
  - cd ${build_path}/opt/pelago && ./unit-csv-test --gtest_output="xml:$CWD/report-json.xml"
 allow_failure: true
 artifacts:
  reports:
   junit: report-json.xml

#----------------------------------------------------------------------------
# Performance Tests
# unhide the jobs by removing the leading '.'
.bench:
 stage: Benchmark
 script:
  - echo "Bench... done"
 <<: *cache
 <<: *restrict_precommit

SSB100 Benchmark - CPU only:
 stage: Benchmark
 script:
  - cmake --build ${build_path} --target install -- -j`nproc --all`
  - cd ${build_path}/opt/pelago && java -jar ../../opt/pelago/proteusplanner.jar inputs/plans/schema.json & disown && sleep 30
  - echo "select sum(d_datekey) from ssbm_date;" > queries.sql
  - echo "alter session set hwmode=cpuonly;" >> queries.sql
  - for f in core/planner/src/test/resources/tests/ssb100/queries/q*.sql; do cat $f $f $f $f $f >> queries.sql ; done ;
  - echo "select /*+ QUERY_INFO(name='warmup') */ * from SessionTimings ;" >> queries.sql
  - JAVA_CLASSPATH=/avatica-1.13.0.jar sqlline --color=true --maxWidth=1024 -u "jdbc:avatica:remote:url=http://localhost:8081;serialization=PROTOBUF" < queries.sql
  - JAVA_AVATICA_CLASSPATH=/avatica-1.13.0.jar python ./tools/save_session_timings.py
 <<: *cache
 <<: *restrict_precommit

SSB100 Benchmark - GPU only:
 stage: Benchmark
 script:
  - cmake --build ${build_path} --target install -- -j`nproc --all`
  - cd ${build_path}/opt/pelago && java -jar ../../opt/pelago/proteusplanner.jar inputs/plans/schema.json & disown && sleep 30
  - echo "select /*+ QUERY_INFO(name='warmup') */ sum(d_datekey) from ssbm_date;" > queries.sql
  - echo "alter session set hwmode=gpuonly;" >> queries.sql
  - for f in core/planner/src/test/resources/tests/ssb100/queries/q*.sql; do cat $f $f $f $f $f >> queries.sql ; done ;
  - echo "select * from SessionTimings ;" >> queries.sql
  - JAVA_CLASSPATH=/avatica-1.13.0.jar sqlline --color=true --maxWidth=1024 -u "jdbc:avatica:remote:url=http://localhost:8081;serialization=PROTOBUF" < queries.sql
  - JAVA_AVATICA_CLASSPATH=/avatica-1.13.0.jar python ./tools/save_session_timings.py
 <<: *cache
 <<: *restrict_precommit

SSB100 Benchmark - Hybrid:
 stage: Benchmark
 script:
  - cmake --build ${build_path} --target install -- -j`nproc --all`
  - cd ${build_path}/opt/pelago && java -jar ../../opt/pelago/proteusplanner.jar inputs/plans/schema.json & disown && sleep 30
  - echo "select /*+ QUERY_INFO(name='warmup') */ sum(d_datekey) from ssbm_date;" > queries.sql
  - echo "alter session set hwmode=hybrid;" >> queries.sql
  - for f in core/planner/src/test/resources/tests/ssb100/queries/q*.sql; do cat $f $f $f $f $f >> queries.sql ; done ;
  - echo "select * from SessionTimings ;" >> queries.sql
  - JAVA_CLASSPATH=/avatica-1.13.0.jar sqlline --color=true --maxWidth=1024 -u "jdbc:avatica:remote:url=http://localhost:8081;serialization=PROTOBUF" < queries.sql
  - JAVA_AVATICA_CLASSPATH=/avatica-1.13.0.jar python ./tools/save_session_timings.py
 <<: *cache
 <<: *restrict_precommit

SSB100 Benchmark - Hybrid - Prepared Queries:
 stage: Benchmark
 script:
  - cmake --build ${build_path} --target install -- -j`nproc --all`
  - cd ${build_path}/opt/pelago
  - ./perftest-query-perf
  - JAVA_AVATICA_CLASSPATH=/avatica-1.13.0.jar python ${CWD}/tools/save_benchmark_session_timings.py
 <<: *cache
 <<: *restrict_precommit

#----------------------------------------------------------------------------
# Execution Profiling 
# unhide the jobs by removing the leading '.'
.build profile:
 stage: Build Profile
 script:
  - echo "Building Dependencies... done"
  - echo "Building... done"
  - echo "Installing... done"
 when: manual
 <<: *cache_profile
 <<: *restrict_precommit

.1/2 profile:
 stage: Profile
 script:
  - echo "Running workload 1... done"
  - echo "Extracting profiling statistics"
 when: manual
 cache:
  policy: pull
 <<: *cache_profile
 <<: *restrict_precommit

.2/2 profile:
 stage: Profile
 script:
  - echo "Running workload 2... done"
  - echo "Extracting profiling statistics"
 when: manual
 cache:
  policy: pull
 <<: *cache_profile
 <<: *restrict_precommit

#----------------------------------------------------------------------------
# Packaging & distribution
.tar:
 stage: Package
 script:
  - echo "Generating tar... done"
 <<: *cache
 <<: *restrict_postcommit

.tar:latest:
 stage: Publish
 script:
  - echo "Publishing HEAD... done"
 artifacts:
  name: "$CI_PROJECT_NAME-g$CI_COMMIT_SHA"
  paths:
   - pkg/
 cache:
  policy: pull
 <<: *cache
 <<: *restrict_latest

.tar:release:
 stage: Publish
 script:
  - echo "Publishing release... done"
 artifacts:
  name: "$CI_PROJECT_NAME-$CI_COMMIT_TAG"
  paths:
   - pkg/
 cache:
  policy: pull
 <<: *cache
 <<: *restrict_release