diff --git a/.github/workflows/build_docs.sh b/.github/workflows/build_docs.sh new file mode 100644 index 0000000000..18a23bd621 --- /dev/null +++ b/.github/workflows/build_docs.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +BRANCH=$(basename $1) + +mkdir -p ~/.mxnet/datasets +for f in /mnt/efs/fs1/.mxnet/datasets/*; do + if [ -d "$f" ]; then + # Will not run if no directories are available + ln -s $f ~/.mxnet/datasets/$(basename "$f") + fi +done + +export MXNET_CUDNN_AUTOTUNE_DEFAULT=0 +cd docs +make html +sed -i.bak 's/33\\,150\\,243/23\\,141\\,201/g' build/html/_static/material-design-lite-1.3.0/material.blue-deep_orange.min.css; +sed -i.bak 's/2196f3/178dc9/g' build/html/_static/sphinx_materialdesign_theme.css; +sed -i.bak 's/pre{padding:1rem;margin:1.5rem\\s0;overflow:auto;overflow-y:hidden}/pre{padding:1rem;margin:1.5rem 0;overflow:auto;overflow-y:scroll}/g' build/html/_static/sphinx_materialdesign_theme.css + +COMMAND_EXIT_CODE=$? +if [[ $BRANCH == master ]]; then + # aws s3 cp s3://gluon-cv.mxnet.io/coverage.svg build/html/coverage.svg + # aws s3 sync --delete build/html/ s3://gluon-cv.mxnet.io/ --acl public-read --cache-control max-age=7200 + # aws s3 cp build/html/coverage.svg s3://gluon-cv.mxnet.io/coverage.svg --acl public-read --cache-control max-age=300 + # echo "Uploaded doc to http://gluon-cv.mxnet.io" + echo master +elif [[ -d $SAVED_OUTPUT ]]; then + # aws s3 cp s3://gluon-vision-staging/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/coverage.svg build/html/coverage.svg + # aws s3 sync --delete build/html/ s3://gluon-vision-staging/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/ --acl public-read + # echo "Uploaded doc to http://gluon-vision-staging.s3-website-us-west-2.amazonaws.com/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/index.html" + echo $BRANCH +fi; +exit $COMMAND_EXIT_CODE \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b2ec1bff0..2d7289ddc3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,101 +3,140 @@ name: CI workflow on: [push, pull_request_target] jobs: - unittests: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [macos-latest, windows-latest, ubuntu-latest] - steps: - - name: checkout - uses: actions/checkout@v2 - - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v1.7.0 - with: - auto-update-conda: true - python-version: 3.7 - - name: sanity-check - shell: bash -l {0} - run: | - conda env create -n gluon_cv_lint -f ./tests/pylint.yml - conda env update -n gluon-cv-lint -f ./tests/pylint.yml --prune - conda activate gluon-cv-lint - conda list - make clean - make pylint - - name: unit-test - shell: bash -l {0} - run: | - conda env create -n gluon_cv_py3_test -f tests/py3_mxnet_ci.yml - conda env update -n gluon_cv_py3_test -f tests/py3_mxnet_ci.yml --prune - conda activate gluon_cv_py3_test - conda list - export CUDA_VISIBLE_DEVICES=0 - export KMP_DUPLICATE_LIB_OK=TRUE - make clean - pip install --upgrade --force-reinstall --no-deps . - env - export LD_LIBRARY_PATH=/usr/local/cuda-10.0/lib64 - export MPLBACKEND=Agg - export MXNET_CUDNN_AUTOTUNE_DEFAULT=0 - export TINY_COCO=~/.mxnet/datasets/tiny_coco - export TINY_MOTORBIKE=~/.mxnet/datasets/tiny_motorbike - mkdir -p $TINY_COCO/annotations - curl -s https://gluoncv-ci.s3-us-west-2.amazonaws.com/mini_coco/sub_val.zip --output sub_val.zip - unzip -q sub_val.zip -d $TINY_COCO - mv $TINY_COCO/sub_val $TINY_COCO/val2017 - curl -s https://gluoncv-ci.s3-us-west-2.amazonaws.com/mini_coco/instances_val2017_tiny.json --output instances_val2017_tiny.json - mv instances_val2017_tiny.json $TINY_COCO/annotations - curl -s https://gluoncv-ci.s3-us-west-2.amazonaws.com/tiny_motorbike.zip --output tiny_motorbike.zip - unzip -q tiny_motorbike.zip -d $TINY_MOTORBIKE - nosetests --with-timer --timer-ok 5 --timer-warning 20 -x --with-coverage --cover-package gluoncv -v tests/unittests - model_zoo_mxnet: - needs: unittests - runs-on: ubuntu-latest - steps: - - name: checkout - uses: actions/checkout@v2 - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.GLUONCV_DEV_ACCESS_ID }} - aws-secret-access-key: ${{ secrets.GLUONCV_DEV_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - name: Install dependencies - run: | - pip install --upgrade --force-reinstall --no-deps . - pip install boto3 - - name: Test project on AWS Batch(For push) - shell: bash -l {0} - if: ${{ github.event_name == 'push' }} - run: | - echo "Start submitting job" - python ./tools/batch/submit-job.py --region us-east-1 \ - --job-type g4dn.12x \ - --name GluonCV-GPU-Test-${{ github.ref }} \ - --source-ref ${{ github.ref }} \ - --work-dir tests \ - --remote https://github.com/${{ github.repository }} \ - --command "nosetests --with-timer --timer-ok 5 --timer-warning 20 -x --with-coverage --cover-package gluoncv -v model_zoo" \ - --wait - - name: Test project on AWS Batch(For pull request) - if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} - run: | - echo "Start submitting job" - python ./tools/batch/submit-job.py --region us-east-1 \ - --job-type g4dn.12x \ - --name GluonCV-GPU-Test-PR#${{ github.event.number }} \ - --source-ref ${{ github.event.pull_request.head.ref }} \ - --work-dir tests \ - --remote https://github.com/${{ github.event.pull_request.head.repo.full_name }} \ - --command "nosetests --with-timer --timer-ok 5 --timer-warning 20 -x --with-coverage --cover-package gluoncv -v model_zoo" \ - --wait - model_zoo_torch: - needs: unittests + # unittests: + # runs-on: ${{ matrix.os }} + # strategy: + # matrix: + # os: [macos-latest, windows-latest, ubuntu-latest] + # steps: + # - name: checkout + # uses: actions/checkout@v2 + # - name: Setup Miniconda + # uses: conda-incubator/setup-miniconda@v1.7.0 + # with: + # auto-update-conda: true + # python-version: 3.7 + # - name: sanity-check + # shell: bash -l {0} + # run: | + # conda env create -n gluon_cv_lint -f ./tests/pylint.yml + # conda env update -n gluon-cv-lint -f ./tests/pylint.yml --prune + # conda activate gluon-cv-lint + # conda list + # make clean + # make pylint + # - name: unit-test + # shell: bash -l {0} + # run: | + # conda env create -n gluon_cv_py3_test -f tests/py3_mxnet_ci.yml + # conda env update -n gluon_cv_py3_test -f tests/py3_mxnet_ci.yml --prune + # conda activate gluon_cv_py3_test + # conda list + # export CUDA_VISIBLE_DEVICES=0 + # export KMP_DUPLICATE_LIB_OK=TRUE + # make clean + # pip install --upgrade --force-reinstall --no-deps . + # env + # export LD_LIBRARY_PATH=/usr/local/cuda-10.0/lib64 + # export MPLBACKEND=Agg + # export MXNET_CUDNN_AUTOTUNE_DEFAULT=0 + # export TINY_COCO=~/.mxnet/datasets/tiny_coco + # export TINY_MOTORBIKE=~/.mxnet/datasets/tiny_motorbike + # mkdir -p $TINY_COCO/annotations + # curl -s https://gluoncv-ci.s3-us-west-2.amazonaws.com/mini_coco/sub_val.zip --output sub_val.zip + # unzip -q sub_val.zip -d $TINY_COCO + # mv $TINY_COCO/sub_val $TINY_COCO/val2017 + # curl -s https://gluoncv-ci.s3-us-west-2.amazonaws.com/mini_coco/instances_val2017_tiny.json --output instances_val2017_tiny.json + # mv instances_val2017_tiny.json $TINY_COCO/annotations + # curl -s https://gluoncv-ci.s3-us-west-2.amazonaws.com/tiny_motorbike.zip --output tiny_motorbike.zip + # unzip -q tiny_motorbike.zip -d $TINY_MOTORBIKE + # nosetests --with-timer --timer-ok 5 --timer-warning 20 -x --with-coverage --cover-package gluoncv -v tests/unittests + # model_zoo_mxnet: + # needs: unittests + # runs-on: ubuntu-latest + # steps: + # - name: checkout + # uses: actions/checkout@v2 + # - name: Configure AWS Credentials + # uses: aws-actions/configure-aws-credentials@v1 + # with: + # aws-access-key-id: ${{ secrets.GLUONCV_DEV_ACCESS_ID }} + # aws-secret-access-key: ${{ secrets.GLUONCV_DEV_SECRET_ACCESS_KEY }} + # aws-region: us-east-1 + # - name: Install dependencies + # run: | + # python3 -m pip install --upgrade --force-reinstall --no-deps . + # python3 -m pip install boto3 + # - name: Test model_zoo_mxnet on AWS Batch(For push) + # shell: bash -l {0} + # if: ${{ github.event_name == 'push' }} + # run: | + # echo "Start submitting job" + # python ./tools/batch/submit-job.py --region us-east-1 \ + # --job-type g4dn.4x \ + # --name GluonCV-GPU-Test-${{ github.ref }} \ + # --source-ref ${{ github.ref }} \ + # --work-dir tests \ + # --remote https://github.com/${{ github.repository }} \ + # --command "nosetests --with-timer --timer-ok 5 --timer-warning 20 -x --with-coverage --cover-package gluoncv -v model_zoo" \ + # --wait + # - name: Test model_zoo_mxnet on AWS Batch(For pull request) + # if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} + # run: | + # echo "Start submitting job" + # python ./tools/batch/submit-job.py --region us-east-1 \ + # --job-type g4dn.4x \ + # --name GluonCV-GPU-Test-PR#${{ github.event.number }} \ + # --source-ref ${{ github.event.pull_request.head.ref }} \ + # --work-dir tests \ + # --remote https://github.com/${{ github.event.pull_request.head.repo.full_name }} \ + # --command "nosetests --with-timer --timer-ok 5 --timer-warning 20 -x --with-coverage --cover-package gluoncv -v model_zoo" \ + # --wait + # model_zoo_torch: + # needs: unittests + # runs-on: ubuntu-latest + # steps: + # - name: checkout + # uses: actions/checkout@v2 + # - name: Configure AWS Credentials + # uses: aws-actions/configure-aws-credentials@v1 + # with: + # aws-access-key-id: ${{ secrets.GLUONCV_DEV_ACCESS_ID }} + # aws-secret-access-key: ${{ secrets.GLUONCV_DEV_SECRET_ACCESS_KEY }} + # aws-region: us-east-1 + # - name: Install dependencies + # run: | + # python3 -m pip install --upgrade --force-reinstall --no-deps . + # python3 -m pip install boto3 + # - name: Test model_zoo_torch on AWS Batch(For push) + # shell: bash -l {0} + # if: ${{ github.event_name == 'push' }} + # run: | + # echo "Start submitting job" + # python ./tools/batch/submit-job.py --region us-east-1 \ + # --job-type g4dn.4x \ + # --name GluonCV-GPU-Test-${{ github.ref }} \ + # --source-ref ${{ github.ref }} \ + # --work-dir tests \ + # --remote https://github.com/${{ github.repository }} \ + # --command "nosetests --with-timer --timer-ok 5 --timer-warning 20 -x --with-coverage --cover-package gluoncv/torch -v model_zoo_torch" \ + # --wait + # - name: Test model_zoo_torch on AWS Batch(For pull request) + # if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} + # run: | + # echo "Start submitting job" + # python ./tools/batch/submit-job.py --region us-east-1 \ + # --job-type g4dn.4x \ + # --name GluonCV-GPU-Test-PR#${{ github.event.number }} \ + # --source-ref ${{ github.event.pull_request.head.ref }} \ + # --work-dir tests \ + # --remote https://github.com/${{ github.event.pull_request.head.repo.full_name }} \ + # --command "nosetests --with-timer --timer-ok 5 --timer-warning 20 -x --with-coverage --cover-package gluoncv/torch -v model_zoo_torch" \ + # --wait + build-docs: + # needs: unittests, model_zoo_mxnet, model_zoo_torch runs-on: ubuntu-latest steps: - - name: checkout - uses: actions/checkout@v2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: @@ -106,66 +145,29 @@ jobs: aws-region: us-east-1 - name: Install dependencies run: | - pip install --upgrade --force-reinstall --no-deps . - pip install boto3 - - name: Test project on AWS Batch(For push) + python3 -m pip install --upgrade --force-reinstall --no-deps . + python3 -m pip install boto3 sphinx>=1.5.5 sphinx-gallery sphinx_rtd_theme matplotlib Image recommonmark scipy mxtheme + - name: Build docs on AWS Batch(For push) shell: bash -l {0} if: ${{ github.event_name == 'push' }} run: | echo "Start submitting job" python ./tools/batch/submit-job.py --region us-east-1 \ - --job-type g4dn.12x \ + --job-type g4dn.4x \ --name GluonCV-GPU-Test-${{ github.ref }} \ --source-ref ${{ github.ref }} \ - --work-dir tests \ --remote https://github.com/${{ github.repository }} \ - --command "nosetests --with-timer --timer-ok 5 --timer-warning 20 -x --with-coverage --cover-package gluoncv/torch -v model_zoo_torch" \ + --command "chmod +x ./github/workflow/build_docs.sh && ./github/workflow/build_docs.sh ${{ github.ref }}" \ --wait - - name: Test project on AWS Batch(For pull request) + - name: Build docs on AWS Batch(For pull request) if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} run: | echo "Start submitting job" python ./tools/batch/submit-job.py --region us-east-1 \ - --job-type g4dn.12x \ + --job-type g4dn.4x \ --name GluonCV-GPU-Test-PR#${{ github.event.number }} \ --source-ref ${{ github.event.pull_request.head.ref }} \ - --work-dir tests \ --remote https://github.com/${{ github.event.pull_request.head.repo.full_name }} \ - --command "nosetests --with-timer --timer-ok 5 --timer-warning 20 -x --with-coverage --cover-package gluoncv/torch -v model_zoo_torch" \ + --command "chmod +x ./github/workflow/build_docs.sh && ./github/workflow/build_docs.sh ${{ github.ref }}" \ --wait - # - name: Wait for job and copy files from AWS s3 - # shell: bash -l {0} - # if: ${{ failure() || success() }} - # run: | - # echo batch done - # build-docs: - # needs: unittests, model_zoo_mxnet, model_zoo_torch - # runs-on: ubuntu-latest - # steps: - # - name: Configure AWS Credentials - # uses: aws-actions/configure-aws-credentials@v1 - # with: - # aws-access-key-id: ${{ secrets.GLUONCV_DEV_ACCESS_ID }} - # aws-secret-access-key: ${{ secrets.GLUONCV_DEV_SECRET_ACCESS_KEY }} - # aws-region: us-east-1 - # - name: Install dependencies - # run: | - # pip install --upgrade --force-reinstall --no-deps . - # pip install boto3 - # - name: build-docs - # shell: bash -l {0} - # run: | - # conda env create -n gluon_vision_docs -f docs/build.yml - # conda env update -n gluon_vision_docs -f docs/build.yml --prune - # conda activate gluon_vision_docs - # export PYTHONPATH=\${PWD} - # export CUDA_VISIBLE_DEVICES=0 - # env - # export LD_LIBRARY_PATH=/usr/local/cuda-10.0/lib64 - # git submodule update --init --recursive - # git clean -fx - # pip install --upgrade --force-reinstall --no-deps . - # echo done pip install - # echo make file - # cd docs && make clean && make html \ No newline at end of file