Update for Spark 3.5 (#97)

SneaksAndData · Oct 14, 2024 · 03f2710 · 03f2710
1 parent 164e913
commit 03f2710
Show file tree

Hide file tree

Showing 7 changed files with 173 additions and 363 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -11,16 +11,20 @@ jobs:
     name: Validate code in spark container
     runs-on: ubuntu-latest
     if: ${{ github.ref != 'refs/heads/main' }}
+    permissions:
+      contents: read
+      packages: read
+      pull-requests: write
 
     container:
-      image: esdcrdevelopment.azurecr.io/spark:v3.2.0-3-g1ee431d-bitnami-0ca3dee2-python-3.9.16
+      image: ghcr.io/sneaksanddata/spark:v5.0.1
       credentials:
-        username: ${{ secrets.AZCR_DEV_USER }}
-        password: ${{ secrets.AZCR_DEV_TOKEN }}
+        username: USERNAME
+        password: ${{ secrets.GHCR_TOKEN }}
       options: -u root -w /opt/bitnami/spark --mount type=tmpfs,destination=/home/spark
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Install Poetry
         run: |
           set -e

diff --git a/.github/workflows/prepare_release.yaml b/.github/workflows/prepare_release.yaml
@@ -9,11 +9,11 @@ jobs:
     if: ${{ github.ref == 'refs/heads/main' }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Create Release
-        uses: SneaksAndData/github-actions/semver_release@v0.0.2
+        uses: SneaksAndData/github-actions/semver_release@v0.1.9
         with:
           major_v: 0
-          minor_v: 6
+          minor_v: 7
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -7,12 +7,12 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
-          python-version: '3.9.x'
+          python-version: '3.11.x'
           architecture: 'x64'
       - name: Install Poetry and prepare version
         run: |

diff --git a/LICENSE b/LICENSE
@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2022 SneaksAndData
+Copyright (c) 2022-2025 SneaksAndData
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

diff --git a/README.md b/README.md
@@ -16,6 +16,7 @@ Select a version that matches hadoop version you are using:
 | 3.2.x / 3.2.x                  |                0.4.x                 |
 | 3.3.x / 3.3.x                  |             0.4.x, 0.5.x             |
 | 3.3.x / 3.4.x                  |                0.6.x                 |
+| 3.5.x / 3.5.x                  |                0.7.x                 |
 
 ## Usage
 Common use case is accessing Hadoop FileSystem from Spark session object:

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -9,14 +9,14 @@ readme = "README.md"
 repository = 'https://github.com/SneaksAndData/hadoop-fs-wrapper'
 
 [tool.poetry.dependencies]
-python = ">=3.9, <3.12"
-pyspark = "~3.4"
+python = ">=3.11, <3.13"
+pyspark = "~3.5"
 
 [tool.poetry.dev-dependencies]
 pytest = "^7.0"
 pytest-cov = "^3.0"
-pylint = "^2.12"
-black = "~22.12.0"
+pylint = "^3"
+black = "^23"
 
 [build-system]
 requires = ["poetry-core>=1.2.0"]