Update for Spark 3.5 (#97) #172

	name: Validate code

	on:
	push:
	branches: [ main ]
	pull_request:
	branches: [ main ]

	jobs:
	validate_code:
	name: Validate code in spark container
	runs-on: ubuntu-latest
	if: ${{ github.ref != 'refs/heads/main' }}
	permissions:
	contents: read
	packages: read
	pull-requests: write

	container:
	image: ghcr.io/sneaksanddata/spark:v5.0.1
	credentials:
	username: USERNAME
	password: ${{ secrets.GHCR_TOKEN }}
	options: -u root -w /opt/bitnami/spark --mount type=tmpfs,destination=/home/spark

	steps:
	- uses: actions/checkout@v4
	- name: Install Poetry
	run: \|
	set -e

	curl -sSL https://install.python-poetry.org \| python3 -
	- name: Install Dependencies
	run: \|
	set -e

	/github/home/.local/bin/poetry install
	- name: Black
	shell: bash
	run: \|
	set -e
	/github/home/.local/bin/poetry run black . --check --diff
	- name: Lint
	run: \|
	set -e

	find ./hadoop_fs_wrapper -type f -name "*.py" \| xargs /github/home/.local/bin/poetry run pylint
	- name: Unit test
	shell: bash
	run: \|
	set -euxo pipefail

	/github/home/.local/bin/poetry run pytest ./test --doctest-modules --junitxml=junit/test-results.xml --cov=. --cov-report=term-missing:skip-covered \| tee pytest-coverage.txt
	- name: Publish Code Coverage
	uses: MishaKav/pytest-coverage-comment@main
	with:
	pytest-coverage-path: ./pytest-coverage.txt
	junitxml-path: ./junit/test-results.xml

Provide feedback