[dockerhub-client] - handle 429 errors with username:password #2982

Workflow file for this run

.github/workflows/on-push.yml at 8ce3ead

	name: CI - On Push

	on:
	push:
	branches:
	- master
	pull_request:
	branches:
	- "**"

	concurrency:
	group: tests-${{ github.head_ref \|\| github.ref }}
	cancel-in-progress: ${{ github.event_name == 'pull_request' }}

	jobs:
	validate-changelog:
	runs-on: ubuntu-latest
	if: github.ref_name != 'master'
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Python ${{ matrix.python-version }} - Setup Environment
	uses: ./.github/actions/setup_environment
	with:
	python-version: "3.10"

	- name: Validate Changelog
	env:
	PR_NUMBER: ${{ github.event.pull_request.number }}
	PR_TITLE: ${{ github.event.pull_request.title }}
	run: \|
	echo "Validating changelog for PR: $PR_NUMBER with Name: $PR_TITLE"
	poetry run python ./Utils/github_workflow_scripts/changelog_validation_scripts/validate_changelog.py -n "$PR_NUMBER" -t "$PR_TITLE"
	- name: Comment Changelog In PR Description
	continue-on-error: true
	env:
	PR_NUMBER: ${{ github.event.pull_request.number }}
	LATEST_COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
	run: \|
	echo "Commenting changelog for PR: $PR_NUMBER"
	poetry run python ./Utils/github_workflow_scripts/changelog_comment_scripts/comment_changelog.py -n "$PR_NUMBER" -lt "$LATEST_COMMIT_SHA" -ght ${{ secrets.GITHUB_TOKEN }}

	pre-commit-checks:
	runs-on: ubuntu-latest
	name: Pre Commit Checks
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Setup Python Environment
	uses: ./.github/actions/setup_environment
	with:
	python-version: "3.10"

	- name: Run Pre Commit
	uses: pre-commit/[email protected]
	with:
	extra_args: --all-files

	- name: Run Pytest collection
	run: \|
	poetry run pytest --collect-only .

	unit-tests:
	name: Unit Tests / Python ${{ matrix.python-version }}
	runs-on: ubuntu-latest
	strategy:
	matrix:
	python-version: ["3.8", "3.9", "3.10"]
	fail-fast: false
	defaults:
	run:
	shell: bash
	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Setup Test Environment (Python ${{ matrix.python-version }})
	uses: ./.github/actions/setup_test_environment
	with:
	python-version: ${{ matrix.python-version }}
	artifacts-dir: unit-tests

	- name: Run Unit Tests
	timeout-minutes: 60
	run: \|
	source "$(poetry env info --path)/bin/activate"

	# Due to race conditions in the tests bringing up and down the node server, have the server available
	# For all the tests.
	node demisto_sdk/commands/common/markdown_server/mdx-parse-server.js &
	node_pid=$!

	poetry run pytest -v . --ignore={demisto_sdk/commands/init/templates,demisto_sdk/tests/integration_tests,demisto_sdk/commands/content_graph,tests_end_to_end} --cov=demisto_sdk --cov-report=html:unit-tests/coverage --junitxml=unit-tests/junit.xml --force-sugar --durations 25 \|\| pytest_exit_code=$?
	echo "PYTEST_EXIT_CODE=$pytest_exit_code" >> $GITHUB_ENV

	kill $node_pid
	exit $pytest_exit_code

	- name: Python ${{ matrix.python-version }} - Test Summary Upload
	uses: ./.github/actions/test_summary
	if: always()
	with:
	artifact-name: unit-tests-python-${{ matrix.python-version }}-artifacts
	artifacts-path-dir: unit-tests
	junit-path: unit-tests/junit.xml

	integration-tests:
	name: Integration Tests / Python ${{ matrix.python-version }}
	runs-on: ubuntu-latest
	strategy:
	matrix:
	python-version: ["3.8", "3.9", "3.10"]
	fail-fast: false
	defaults:
	run:
	shell: bash
	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Setup Test Environment (Python ${{ matrix.python-version }})
	uses: ./.github/actions/setup_test_environment
	with:
	python-version: ${{ matrix.python-version }}
	artifacts-dir: integration-tests

	- name: Run Integration Tests
	timeout-minutes: 60
	run: \|
	source "$(poetry env info --path)/bin/activate"

	poetry run pytest -v demisto_sdk/tests/integration_tests --cov=demisto_sdk --cov-report=html:integration-tests/coverage --junitxml=integration-tests/junit.xml --force-sugar --durations 25 \|\| pytest_exit_code=$?
	echo "PYTEST_EXIT_CODE=$pytest_exit_code" >> $GITHUB_ENV

	exit $pytest_exit_code

	- name: Python ${{ matrix.python-version }} - Test Summary Upload
	uses: ./.github/actions/test_summary
	if: always()
	with:
	artifact-name: integration-tests-python-${{ matrix.python-version }}-artifacts
	artifacts-path-dir: integration-tests
	junit-path: integration-tests/junit.xml

	graph-tests:
	name: Graph Tests / Python ${{ matrix.python-version }}
	runs-on: ubuntu-latest
	strategy:
	matrix:
	python-version: ["3.8", "3.9", "3.10"]
	fail-fast: false
	defaults:
	run:
	shell: bash
	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Setup Test Environment (Python ${{ matrix.python-version }})
	uses: ./.github/actions/setup_test_environment
	with:
	python-version: ${{ matrix.python-version }}
	artifacts-dir: graph-tests

	- name: Run Graph Tests
	timeout-minutes: 60
	run: \|
	source "$(poetry env info --path)/bin/activate"

	poetry run pytest -v demisto_sdk/commands/content_graph --cov=demisto_sdk --cov-report=html:graph-tests/coverage --junitxml=graph-tests/junit.xml --force-sugar --durations 25 \|\| pytest_exit_code=$?
	echo "PYTEST_EXIT_CODE=$pytest_exit_code" >> $GITHUB_ENV

	exit $pytest_exit_code

	- name: Python ${{ matrix.python-version }} - Test Summary Upload
	uses: ./.github/actions/test_summary
	if: always()
	with:
	artifact-name: graph-tests-python-${{ matrix.python-version }}-artifacts
	artifacts-path-dir: graph-tests
	junit-path: graph-tests/junit.xml

	coverage:
	needs: [unit-tests, integration-tests, graph-tests]
	if: github.ref_name != 'master'
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- name: Set up Python 3.10
	uses: actions/setup-python@v5
	with:
	python-version: "3.10"
	- name: Download all artifacts
	uses: actions/download-artifact@v4
	- name: Run coverage
	run: \|
	pip install coverage
	coverage combine **/.coverage
	coverage report
	coverage xml
	- name: Coveralls-Action
	uses: coverallsapp/github-action@v2
	continue-on-error: true
	- name: Coveralls-Comment
	env:
	COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
	continue-on-error: true
	run: \|
	if [ -n "$COVERALLS_REPO_TOKEN" ]; then
	pip install coveralls
	coveralls
	else
	echo "Skipping coveralls"
	fi

	test-content-infrastructure:
	runs-on: ubuntu-latest
	name: Test Content Infrastructure
	steps:
	- name: Checkout SDK
	uses: actions/checkout@v4

	- name: Checkout content
	uses: actions/checkout@v4
	with:
	repository: demisto/content
	path: content

	- name: Get commit hash
	id: get-commit
	run: echo commit=$(git rev-parse HEAD) >> $GITHUB_OUTPUT

	- name: Python ${{ matrix.python-version }} - Setup Environment
	uses: ./.github/actions/setup_environment
	with:
	python-version: "3.10"
	working-dir: content
	poetry-install-arguments: "--with ci"

	- name: Un-install content version of SDK
	run: \|
	cd content
	poetry run pip uninstall demisto-sdk -y
	- name: Install current SDK branch
	run: \|
	cd content
	poetry run pip install git+https://github.com/demisto/demisto-sdk@${{ steps.get-commit.outputs.commit }}
	- name: Run content unit tests
	timeout-minutes: 60
	env:
	PYTHONPATH: ${{ github.workspace }}/content
	run: \|
	cd content
	poetry run pytest ./Tests/scripts/infrastructure_tests/ -v
	poetry run pytest ./Tests/Marketplace/Tests/ -v
	poetry run pytest ./Tests/tests -v
	poetry run pytest ./Tests/private_build/tests -v
	poetry run pytest Utils -v

	test-pre-commit-command:
	runs-on: ubuntu-latest
	name: Test Pre-Commit Command
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Setup Python Environment
	uses: ./.github/actions/setup_environment
	with:
	python-version: "3.10"

	- name: Checkout Content
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	repository: demisto/content
	path: content

	- name: Cache Pre commit
	id: cache-pre-commit
	uses: actions/cache@v4
	with:
	path: ~/.cache/pre-commit
	key: ${{ runner.os }}-pre-commit

	- name: Create SDK Config File - Ignore BA101
	run: \|
	# Run only a specific validation to make sure validate is triggered successfully in demisto-sdk pre-commit
	cd content
	echo -e "[validate]\nrun_specific_validations=BA101" > .demisto-sdk-conf

	- name: run pre-commit on changed files
	timeout-minutes: 60
	run: \|
	source $(poetry env info --path)/bin/activate
	cd content
	mkdir -p test-pre-commit-command
	echo "# test" >> Packs/HelloWorld/Integrations/HelloWorld/HelloWorld.yml
	echo "# test" >> Packs/CortexXDR/Integrations/CortexXDRIR/CortexXDRIR.yml
	echo "# test" >> Packs/QRadar/Integrations/QRadar_v3/QRadar_v3.yml
	echo "# test" >> Packs/QRadar/Integrations/QRadar_v3/QRadar_v3.py

	echo "# test" >> Tests/collect_tests.py
	git add .
	demisto-sdk pre-commit --validate --show-diff-on-failure --verbose --mode=ci

	- name: run pre-commit on input files
	timeout-minutes: 60
	run: \|
	source $(poetry env info --path)/bin/activate
	cd content
	demisto-sdk pre-commit --validate --show-diff-on-failure --verbose -i Packs/HelloWorld -i Packs/QRadar/Integrations/QRadar_v3 --mode=nightly

	- name: run pre-commit with default pre-commit config shipped with the SDK
	timeout-minutes: 60
	run: \|
	source $(poetry env info --path)/bin/activate
	cd content
	rm -rf .pre-commit-config_template.yaml
	demisto-sdk pre-commit --validate --show-diff-on-failure --verbose --mode=ci

	- name: Upload artifacts
	if: always()
	uses: ./.github/actions/upload_artifacts
	with:
	artifacts-path-dir: content/test-pre-commit-command
	artifact-name: test-demisto-sdk-pre-commit-command-artifacts

	test-validate-content-path:
	runs-on: ubuntu-latest
	name: Test validate-content-path
	steps:
	- name: Checkout SDK
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Check whether validate_content_path changed
	id: files-changed
	uses: tj-actions/changed-files@v44
	with:
	files: \|
	demisto_sdk/scripts/validate_content_path.py
	demisto_sdk/tests/validate_content_path_test.py

	- name: Checkout Content
	if: steps.files-changed.outputs.any_changed == 'true'
	uses: actions/checkout@v4
	with:
	fetch-depth: 1
	repository: demisto/content
	path: content

	- name: Setup Python Environment
	if: steps.files-changed.outputs.any_changed == 'true'
	uses: ./.github/actions/setup_environment
	with:
	python-version: "3.10"

	- name: Validate content master paths
	if: steps.files-changed.outputs.any_changed == 'true'
	run: \|
	source $(poetry env info --path)/bin/activate
	validate-content-path validate-all content --skip-depth-one-file --skip-depth-one-folder --skip-depth-zero-file --skip-integration-script-file-name --skip-integration-script-file-type --skip-markdown --skip-suffix


	test-graph-commands:
	runs-on: ubuntu-latest
	name: Test Graph Commands
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Setup Python Environment
	uses: ./.github/actions/setup_environment
	with:
	python-version: "3.10"

	- name: Checkout content
	uses: actions/checkout@v4
	with:
	repository: demisto/content
	path: content

	- name: Run Graph
	timeout-minutes: 60
	run: \|
	source $(poetry env info --path)/bin/activate
	cd content
	mkdir -p test-graph-commands
	# create content graph from scratch
	demisto-sdk create-content-graph
	# clean import folder
	sudo rm -rf /var/lib/neo4j/import
	# Update content graph from the bucket
	demisto-sdk update-content-graph -g -o ./test-graph-commands/content_graph

	# Update content graph from the the previous content graph that was created/built
	demisto-sdk update-content-graph -i ./test-graph-commands/content_graph/xsoar.zip -o ./test-graph-commands/content_graph

	- name: Upload artifacts
	if: always()
	uses: ./.github/actions/upload_artifacts
	with:
	artifacts-path-dir: content/test-graph-commands
	artifact-name: test-graph-commands-artifacts

	validate-files-old-validate:
	runs-on: ubuntu-latest
	name: Test Old Validate Command
	timeout-minutes: 60
	env:
	CI_COMMIT_BRANCH: "master"
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Run Validate
	uses: ./.github/actions/validate
	with:
	artifacts-path-dir: test-old-validate-command
	artifact-name: test-old-validate-command-artifacts
	validate-arguments: "-a --graph --skip-pack-dependencies --skip-new-validate --run-old-validate --prev-ver origin/master"

	validate-files-new-validate:
	runs-on: ubuntu-latest
	name: Test New Validate Command
	timeout-minutes: 60
	env:
	CI_COMMIT_BRANCH: "master"
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Run Validate
	uses: ./.github/actions/validate
	with:
	artifacts-path-dir: test-new-validate-command
	artifact-name: test-new-validate-command-artifacts
	validate-arguments: "-a --prev-ver origin/master --config-path sdk_validation_config.toml"

	test-lint:
	runs-on: ubuntu-latest
	name: Test Lint Command
	timeout-minutes: 60

	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Setup Python Environment
	uses: ./.github/actions/setup_environment
	with:
	python-version: "3.10"

	- name: Checkout content
	uses: actions/checkout@v4
	with:
	repository: demisto/content
	path: content

	- name: Run lint on python and powershell files
	run: \|
	source $(poetry env info --path)/bin/activate

	cd content
	mkdir -p test-lint

	# python file (CommonServerPython lint is running over python 3 and 2)
	demisto-sdk lint -i ./Packs/Base/Scripts/CommonServerPython --coverage-report test-lint/coverage-report
	if [[ -f test-lint/coverage-report/html/index.html && -f test-lint/coverage-report/coverage.xml ]]
	then
	echo "coverage files found."
	else
	echo "coverage files not found."
	exit 1
	fi
	# ps file
	demisto-sdk lint -i ./Packs/Base/Scripts/CommonServerPowerShell

	- name: Upload artifacts
	if: always()
	uses: ./.github/actions/upload_artifacts
	with:
	artifacts-path-dir: content/test-lint
	artifact-name: test-lint-artifacts

	slack-notify-master-failure:
	needs:
	[
	pre-commit-checks,
	unit-tests,
	integration-tests,
	graph-tests,
	test-content-infrastructure,
	test-pre-commit-command,
	test-graph-commands,
	validate-files-old-validate,
	validate-files-new-validate,
	test-lint,
	]
	if: failure() && github.ref_name == 'master'
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Setup Python Environment
	uses: ./.github/actions/setup_environment
	with:
	python-version: "3.10"

	- name: Download artifacts
	uses: actions/download-artifact@v4

	- name: Notify Failed Jobs
	run: \|
	poetry run python Utils/github_workflow_scripts/slack_notifier_master/slack_notifier.py --workflow-id ${{ github.run_id }} --slack-token ${{secrets.SLACK_TOKEN}} --github-token ${{ secrets.GITHUB_TOKEN }} --slack-channel ${{secrets.SLACK_CHANNEL}}
	generate_validate_docs:
	name: Generate Validate Docs
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Check whether validate files changed
	id: changed-files
	uses: tj-actions/changed-files@v44
	with:
	files: \|
	demisto_sdk/commands/validate/**
	demisto_sdk/commands/content_graph/objects/**
	demisto_sdk/commands/content_graph/parsers/**
	demisto_sdk/commands/common/**

	- name: Setup Environment
	if: steps.changed-files.outputs.any_changed == 'true'
	uses: ./.github/actions/setup_environment
	with:
	python-version: "3.10"

	- name: Generate Docs
	if: steps.changed-files.outputs.any_changed == 'true'
	run: poetry run python demisto_sdk/commands/validate/generate_validate_docs.py validation_docs.md

	- name: Upload artifacts
	if: steps.changed-files.outputs.any_changed == 'true'
	uses: actions/upload-artifact@v4
	with:
	path: validation_docs.md
	if-no-files-found: error
	name: validation_docs

	- name: Output to GitHub Action Summary
	if: steps.changed-files.outputs.any_changed == 'true'
	run: cat validation_docs.md >> $GITHUB_STEP_SUMMARY

	verify-pip-installation:
	name: Verify SDK Pip Installation
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 1

	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.10"
	cache: 'pip'

	- name: pip intsall current project
	run: pip install .

	- name: Verify Installation
	run: demisto-sdk --version

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[dockerhub-client] - handle 429 errors with username:password #2982

Workflow file

[dockerhub-client] - handle 429 errors with username:password #2982

Jobs

Run details

Workflow file for this run