Skip to content

Commit

Permalink
test build docs
Browse files Browse the repository at this point in the history
  • Loading branch information
@yinweisu
yinweisu committed Nov 16, 2020
1 parent 7902c5a commit 07971ba
Show file tree
Hide file tree
Showing 2 changed files with 173 additions and 138 deletions.
33 changes: 33 additions & 0 deletions .github/workflows/build_docs.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#!/usr/bin/env bash

BRANCH=$(basename $1)

mkdir -p ~/.mxnet/datasets
for f in /mnt/efs/fs1/.mxnet/datasets/*; do
if [ -d "$f" ]; then
# Will not run if no directories are available
ln -s $f ~/.mxnet/datasets/$(basename "$f")
fi
done

export MXNET_CUDNN_AUTOTUNE_DEFAULT=0
cd docs
make html
sed -i.bak 's/33\\,150\\,243/23\\,141\\,201/g' build/html/_static/material-design-lite-1.3.0/material.blue-deep_orange.min.css;
sed -i.bak 's/2196f3/178dc9/g' build/html/_static/sphinx_materialdesign_theme.css;
sed -i.bak 's/pre{padding:1rem;margin:1.5rem\\s0;overflow:auto;overflow-y:hidden}/pre{padding:1rem;margin:1.5rem 0;overflow:auto;overflow-y:scroll}/g' build/html/_static/sphinx_materialdesign_theme.css

COMMAND_EXIT_CODE=$?
if [[ $BRANCH == master ]]; then
# aws s3 cp s3://gluon-cv.mxnet.io/coverage.svg build/html/coverage.svg
# aws s3 sync --delete build/html/ s3://gluon-cv.mxnet.io/ --acl public-read --cache-control max-age=7200
# aws s3 cp build/html/coverage.svg s3://gluon-cv.mxnet.io/coverage.svg --acl public-read --cache-control max-age=300
# echo "Uploaded doc to http://gluon-cv.mxnet.io"
echo master
elif [[ -d $SAVED_OUTPUT ]]; then
# aws s3 cp s3://gluon-vision-staging/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/coverage.svg build/html/coverage.svg
# aws s3 sync --delete build/html/ s3://gluon-vision-staging/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/ --acl public-read
# echo "Uploaded doc to http://gluon-vision-staging.s3-website-us-west-2.amazonaws.com/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/index.html"
echo $BRANCH
fi;
exit $COMMAND_EXIT_CODE
278 changes: 140 additions & 138 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,101 +3,140 @@ name: CI workflow
on: [push, pull_request_target]

jobs:
unittests:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [macos-latest, windows-latest, ubuntu-latest]
steps:
- name: checkout
uses: actions/checkout@v2
- name: Setup Miniconda
uses: conda-incubator/[email protected]
with:
auto-update-conda: true
python-version: 3.7
- name: sanity-check
shell: bash -l {0}
run: |
conda env create -n gluon_cv_lint -f ./tests/pylint.yml
conda env update -n gluon-cv-lint -f ./tests/pylint.yml --prune
conda activate gluon-cv-lint
conda list
make clean
make pylint
- name: unit-test
shell: bash -l {0}
run: |
conda env create -n gluon_cv_py3_test -f tests/py3_mxnet_ci.yml
conda env update -n gluon_cv_py3_test -f tests/py3_mxnet_ci.yml --prune
conda activate gluon_cv_py3_test
conda list
export CUDA_VISIBLE_DEVICES=0
export KMP_DUPLICATE_LIB_OK=TRUE
make clean
pip install --upgrade --force-reinstall --no-deps .
env
export LD_LIBRARY_PATH=/usr/local/cuda-10.0/lib64
export MPLBACKEND=Agg
export MXNET_CUDNN_AUTOTUNE_DEFAULT=0
export TINY_COCO=~/.mxnet/datasets/tiny_coco
export TINY_MOTORBIKE=~/.mxnet/datasets/tiny_motorbike
mkdir -p $TINY_COCO/annotations
curl -s https://gluoncv-ci.s3-us-west-2.amazonaws.com/mini_coco/sub_val.zip --output sub_val.zip
unzip -q sub_val.zip -d $TINY_COCO
mv $TINY_COCO/sub_val $TINY_COCO/val2017
curl -s https://gluoncv-ci.s3-us-west-2.amazonaws.com/mini_coco/instances_val2017_tiny.json --output instances_val2017_tiny.json
mv instances_val2017_tiny.json $TINY_COCO/annotations
curl -s https://gluoncv-ci.s3-us-west-2.amazonaws.com/tiny_motorbike.zip --output tiny_motorbike.zip
unzip -q tiny_motorbike.zip -d $TINY_MOTORBIKE
nosetests --with-timer --timer-ok 5 --timer-warning 20 -x --with-coverage --cover-package gluoncv -v tests/unittests
model_zoo_mxnet:
needs: unittests
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v2
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.GLUONCV_DEV_ACCESS_ID }}
aws-secret-access-key: ${{ secrets.GLUONCV_DEV_SECRET_ACCESS_KEY }}
aws-region: us-east-1
- name: Install dependencies
run: |
pip install --upgrade --force-reinstall --no-deps .
pip install boto3
- name: Test project on AWS Batch(For push)
shell: bash -l {0}
if: ${{ github.event_name == 'push' }}
run: |
echo "Start submitting job"
python ./tools/batch/submit-job.py --region us-east-1 \
--job-type g4dn.12x \
--name GluonCV-GPU-Test-${{ github.ref }} \
--source-ref ${{ github.ref }} \
--work-dir tests \
--remote https://github.com/${{ github.repository }} \
--command "nosetests --with-timer --timer-ok 5 --timer-warning 20 -x --with-coverage --cover-package gluoncv -v model_zoo" \
--wait
- name: Test project on AWS Batch(For pull request)
if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }}
run: |
echo "Start submitting job"
python ./tools/batch/submit-job.py --region us-east-1 \
--job-type g4dn.12x \
--name GluonCV-GPU-Test-PR#${{ github.event.number }} \
--source-ref ${{ github.event.pull_request.head.ref }} \
--work-dir tests \
--remote https://github.com/${{ github.event.pull_request.head.repo.full_name }} \
--command "nosetests --with-timer --timer-ok 5 --timer-warning 20 -x --with-coverage --cover-package gluoncv -v model_zoo" \
--wait
model_zoo_torch:
needs: unittests
# unittests:
# runs-on: ${{ matrix.os }}
# strategy:
# matrix:
# os: [macos-latest, windows-latest, ubuntu-latest]
# steps:
# - name: checkout
# uses: actions/checkout@v2
# - name: Setup Miniconda
# uses: conda-incubator/[email protected]
# with:
# auto-update-conda: true
# python-version: 3.7
# - name: sanity-check
# shell: bash -l {0}
# run: |
# conda env create -n gluon_cv_lint -f ./tests/pylint.yml
# conda env update -n gluon-cv-lint -f ./tests/pylint.yml --prune
# conda activate gluon-cv-lint
# conda list
# make clean
# make pylint
# - name: unit-test
# shell: bash -l {0}
# run: |
# conda env create -n gluon_cv_py3_test -f tests/py3_mxnet_ci.yml
# conda env update -n gluon_cv_py3_test -f tests/py3_mxnet_ci.yml --prune
# conda activate gluon_cv_py3_test
# conda list
# export CUDA_VISIBLE_DEVICES=0
# export KMP_DUPLICATE_LIB_OK=TRUE
# make clean
# pip install --upgrade --force-reinstall --no-deps .
# env
# export LD_LIBRARY_PATH=/usr/local/cuda-10.0/lib64
# export MPLBACKEND=Agg
# export MXNET_CUDNN_AUTOTUNE_DEFAULT=0
# export TINY_COCO=~/.mxnet/datasets/tiny_coco
# export TINY_MOTORBIKE=~/.mxnet/datasets/tiny_motorbike
# mkdir -p $TINY_COCO/annotations
# curl -s https://gluoncv-ci.s3-us-west-2.amazonaws.com/mini_coco/sub_val.zip --output sub_val.zip
# unzip -q sub_val.zip -d $TINY_COCO
# mv $TINY_COCO/sub_val $TINY_COCO/val2017
# curl -s https://gluoncv-ci.s3-us-west-2.amazonaws.com/mini_coco/instances_val2017_tiny.json --output instances_val2017_tiny.json
# mv instances_val2017_tiny.json $TINY_COCO/annotations
# curl -s https://gluoncv-ci.s3-us-west-2.amazonaws.com/tiny_motorbike.zip --output tiny_motorbike.zip
# unzip -q tiny_motorbike.zip -d $TINY_MOTORBIKE
# nosetests --with-timer --timer-ok 5 --timer-warning 20 -x --with-coverage --cover-package gluoncv -v tests/unittests
# model_zoo_mxnet:
# needs: unittests
# runs-on: ubuntu-latest
# steps:
# - name: checkout
# uses: actions/checkout@v2
# - name: Configure AWS Credentials
# uses: aws-actions/configure-aws-credentials@v1
# with:
# aws-access-key-id: ${{ secrets.GLUONCV_DEV_ACCESS_ID }}
# aws-secret-access-key: ${{ secrets.GLUONCV_DEV_SECRET_ACCESS_KEY }}
# aws-region: us-east-1
# - name: Install dependencies
# run: |
# python3 -m pip install --upgrade --force-reinstall --no-deps .
# python3 -m pip install boto3
# - name: Test model_zoo_mxnet on AWS Batch(For push)
# shell: bash -l {0}
# if: ${{ github.event_name == 'push' }}
# run: |
# echo "Start submitting job"
# python ./tools/batch/submit-job.py --region us-east-1 \
# --job-type g4dn.4x \
# --name GluonCV-GPU-Test-${{ github.ref }} \
# --source-ref ${{ github.ref }} \
# --work-dir tests \
# --remote https://github.com/${{ github.repository }} \
# --command "nosetests --with-timer --timer-ok 5 --timer-warning 20 -x --with-coverage --cover-package gluoncv -v model_zoo" \
# --wait
# - name: Test model_zoo_mxnet on AWS Batch(For pull request)
# if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }}
# run: |
# echo "Start submitting job"
# python ./tools/batch/submit-job.py --region us-east-1 \
# --job-type g4dn.4x \
# --name GluonCV-GPU-Test-PR#${{ github.event.number }} \
# --source-ref ${{ github.event.pull_request.head.ref }} \
# --work-dir tests \
# --remote https://github.com/${{ github.event.pull_request.head.repo.full_name }} \
# --command "nosetests --with-timer --timer-ok 5 --timer-warning 20 -x --with-coverage --cover-package gluoncv -v model_zoo" \
# --wait
# model_zoo_torch:
# needs: unittests
# runs-on: ubuntu-latest
# steps:
# - name: checkout
# uses: actions/checkout@v2
# - name: Configure AWS Credentials
# uses: aws-actions/configure-aws-credentials@v1
# with:
# aws-access-key-id: ${{ secrets.GLUONCV_DEV_ACCESS_ID }}
# aws-secret-access-key: ${{ secrets.GLUONCV_DEV_SECRET_ACCESS_KEY }}
# aws-region: us-east-1
# - name: Install dependencies
# run: |
# python3 -m pip install --upgrade --force-reinstall --no-deps .
# python3 -m pip install boto3
# - name: Test model_zoo_torch on AWS Batch(For push)
# shell: bash -l {0}
# if: ${{ github.event_name == 'push' }}
# run: |
# echo "Start submitting job"
# python ./tools/batch/submit-job.py --region us-east-1 \
# --job-type g4dn.4x \
# --name GluonCV-GPU-Test-${{ github.ref }} \
# --source-ref ${{ github.ref }} \
# --work-dir tests \
# --remote https://github.com/${{ github.repository }} \
# --command "nosetests --with-timer --timer-ok 5 --timer-warning 20 -x --with-coverage --cover-package gluoncv/torch -v model_zoo_torch" \
# --wait
# - name: Test model_zoo_torch on AWS Batch(For pull request)
# if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }}
# run: |
# echo "Start submitting job"
# python ./tools/batch/submit-job.py --region us-east-1 \
# --job-type g4dn.4x \
# --name GluonCV-GPU-Test-PR#${{ github.event.number }} \
# --source-ref ${{ github.event.pull_request.head.ref }} \
# --work-dir tests \
# --remote https://github.com/${{ github.event.pull_request.head.repo.full_name }} \
# --command "nosetests --with-timer --timer-ok 5 --timer-warning 20 -x --with-coverage --cover-package gluoncv/torch -v model_zoo_torch" \
# --wait
build-docs:
# needs: unittests, model_zoo_mxnet, model_zoo_torch
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v2
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1
with:
Expand All @@ -106,66 +145,29 @@ jobs:
aws-region: us-east-1
- name: Install dependencies
run: |
pip install --upgrade --force-reinstall --no-deps .
pip install boto3
- name: Test project on AWS Batch(For push)
python3 -m pip install --upgrade --force-reinstall --no-deps .
python3 -m pip install boto3 sphinx>=1.5.5 sphinx-gallery sphinx_rtd_theme matplotlib Image recommonmark scipy mxtheme
- name: Build docs on AWS Batch(For push)
shell: bash -l {0}
if: ${{ github.event_name == 'push' }}
run: |
echo "Start submitting job"
python ./tools/batch/submit-job.py --region us-east-1 \
--job-type g4dn.12x \
--job-type g4dn.4x \
--name GluonCV-GPU-Test-${{ github.ref }} \
--source-ref ${{ github.ref }} \
--work-dir tests \
--remote https://github.com/${{ github.repository }} \
--command "nosetests --with-timer --timer-ok 5 --timer-warning 20 -x --with-coverage --cover-package gluoncv/torch -v model_zoo_torch" \
--command "chmod +x ./github/workflow/build_docs.sh && ./github/workflow/build_docs.sh ${{ github.ref }}" \
--wait
- name: Test project on AWS Batch(For pull request)
- name: Build docs on AWS Batch(For pull request)
if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }}
run: |
echo "Start submitting job"
python ./tools/batch/submit-job.py --region us-east-1 \
--job-type g4dn.12x \
--job-type g4dn.4x \
--name GluonCV-GPU-Test-PR#${{ github.event.number }} \
--source-ref ${{ github.event.pull_request.head.ref }} \
--work-dir tests \
--remote https://github.com/${{ github.event.pull_request.head.repo.full_name }} \
--command "nosetests --with-timer --timer-ok 5 --timer-warning 20 -x --with-coverage --cover-package gluoncv/torch -v model_zoo_torch" \
--command "chmod +x ./github/workflow/build_docs.sh && ./github/workflow/build_docs.sh ${{ github.ref }}" \
--wait
# - name: Wait for job and copy files from AWS s3
# shell: bash -l {0}
# if: ${{ failure() || success() }}
# run: |
# echo batch done
# build-docs:
# needs: unittests, model_zoo_mxnet, model_zoo_torch
# runs-on: ubuntu-latest
# steps:
# - name: Configure AWS Credentials
# uses: aws-actions/configure-aws-credentials@v1
# with:
# aws-access-key-id: ${{ secrets.GLUONCV_DEV_ACCESS_ID }}
# aws-secret-access-key: ${{ secrets.GLUONCV_DEV_SECRET_ACCESS_KEY }}
# aws-region: us-east-1
# - name: Install dependencies
# run: |
# pip install --upgrade --force-reinstall --no-deps .
# pip install boto3
# - name: build-docs
# shell: bash -l {0}
# run: |
# conda env create -n gluon_vision_docs -f docs/build.yml
# conda env update -n gluon_vision_docs -f docs/build.yml --prune
# conda activate gluon_vision_docs
# export PYTHONPATH=\${PWD}
# export CUDA_VISIBLE_DEVICES=0
# env
# export LD_LIBRARY_PATH=/usr/local/cuda-10.0/lib64
# git submodule update --init --recursive
# git clean -fx
# pip install --upgrade --force-reinstall --no-deps .
# echo done pip install
# echo make file
# cd docs && make clean && make html

0 comments on commit 07971ba

Please sign in to comment.