-
Notifications
You must be signed in to change notification settings - Fork 424
163 lines (124 loc) · 3.82 KB
/
python_build.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
name: python_build
on:
merge_group:
push:
branches: [main]
pull_request:
branches: [main]
defaults:
run:
working-directory: ./python
jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Setup Environment
uses: ./.github/actions/setup-env
- name: Check Python
run: |
uv pip install ruff==0.5.2 mypy==1.10.1 types-dataclasses typing-extensions --system
make check-python
- name: Check Rust
run: make check-rust
test-minimal:
name: Python Build (Python 3.9 PyArrow 16.0.0)
runs-on: ubuntu-latest
env:
RUSTFLAGS: "-C debuginfo=line-tables-only"
CARGO_INCREMENTAL: 0
steps:
- uses: actions/checkout@v3
- name: Setup Environment
uses: ./.github/actions/setup-env
with:
python-version: 3.9
- name: Build and install deltalake
run: |
# Install minimum PyArrow version
uv sync --extra devel --extra pandas --extra polars
uv pip install pyarrow==16.0.0
env:
RUSTFLAGS: "-C debuginfo=line-tables-only"
- name: Run tests
run: make unit-test
test:
name: Python Build (Python 3.10 PyArrow latest)
runs-on: ubuntu-latest
env:
RUSTFLAGS: "-C debuginfo=1"
CARGO_INCREMENTAL: 0
steps:
- uses: actions/checkout@v3
- name: Setup Environment
uses: ./.github/actions/setup-env
- name: Start emulated services
run: docker compose up -d
- name: Build and install deltalake
run: make develop
- name: Download Data Acceptance Tests (DAT) files
run: make setup-dat
- name: Run tests
run: uv run --no-sync pytest -m '((s3 or azure) and integration) or not integration and not benchmark' --doctest-modules
- name: Test without pandas
run: |
uv pip uninstall pandas
uv run --no-sync pytest -m "not pandas and not integration and not benchmark"
uv pip install pandas
test-lakefs:
name: Python Build (Python 3.10 LakeFS Integration tests)
runs-on: ubuntu-latest
env:
RUSTFLAGS: "-C debuginfo=1"
CARGO_INCREMENTAL: 0
steps:
- uses: actions/checkout@v3
- name: Setup Environment
uses: ./.github/actions/setup-env
- name: Start emulated services
run: docker compose -f ../docker-compose-lakefs.yml up -d
- name: Build and install deltalake
run: make develop
- name: Download Data Acceptance Tests (DAT) files
run: make setup-dat
- name: Run tests
run: uv run --no-sync pytest -m '(lakefs and integration)' --doctest-modules
test-pyspark:
name: PySpark Integration Tests
runs-on: ubuntu-latest
env:
RUSTFLAGS: "-C debuginfo=line-tables-only"
CARGO_INCREMENTAL: 0
steps:
- uses: actions/checkout@v3
- name: Setup Environment
uses: ./.github/actions/setup-env
- uses: actions/setup-java@v2
with:
distribution: "zulu"
java-version: "11"
- name: Build and install deltalake
run: make develop-pyspark
- name: Run tests
run: make test-pyspark
multi-python-running:
name: Running with Python ${{ matrix.python-version }}
runs-on: ubuntu-latest
env:
RUSTFLAGS: "-C debuginfo=line-tables-only"
CARGO_INCREMENTAL: 0
strategy:
matrix:
python-version: ["3.9", "3.10", "3.11", "3.12"]
steps:
- uses: actions/checkout@v3
- name: Setup Environment
uses: ./.github/actions/setup-env
with:
python-version: ${{ matrix.python-version }}
- name: Build and install deltalake
run: |
uv sync
- name: Run deltalake
run: |
uv run python -c 'import deltalake'