-
Notifications
You must be signed in to change notification settings - Fork 1
328 lines (320 loc) · 13.6 KB
/
build-vdb6.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
name: Build and Upload vdb6
on:
schedule:
- cron: "0 */6 * * *"
workflow_dispatch:
env:
REGISTRY: ghcr.io
IMAGE_NAME: appthreat/vdb
jobs:
app_builder:
runs-on: ubuntu-latest
permissions:
contents: write
packages: write
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v4
with:
repository: AppThreat/vulnerability-db
path: vulnerability-db
ref: "v6.2.3"
- uses: actions/checkout@v4
with:
repository: AppThreat/vuln-list
path: vuln-list
fetch-depth: "1"
sparse-checkout: |
nvd/2018
nvd/2019
nvd/2020
nvd/2021
nvd/2022
nvd/2023
nvd/2024
- uses: oras-project/setup-oras@v1
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Trim CI agent
run: |
chmod +x contrib/free_disk_space.sh
./contrib/free_disk_space.sh
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install setuptools wheel twine build
cd vulnerability-db && pip install ".[dev]"
- name: Build and upload app db - 2018
run: |
mkdir vdb_data vdb_cache
ls ./vuln-list/ ./vuln-list/nvd/
zip -q -r vuln-list.zip ./vuln-list/
mv vuln-list.zip vdb_cache/
rm -rf ./vuln-list/
python vulnerability-db/vdb/cli.py --cache-os
ls -lh vdb_data
ls -lh vdb_cache
cd vdb_data
zstd --ultra data.index.vdb6
zstd --ultra data.vdb6
tar -cvJf data.vdb6.tar.xz data.vdb6
tar -cvJf data.index.vdb6.tar.xz data.index.vdb6
echo $GITHUB_TOKEN | oras login ghcr.io -u $GITHUB_USERNAME --password-stdin
oras push ghcr.io/appthreat/vdbzst-app:v6 \
--artifact-type application/vnd.oras.config.v1+json \
./vdb.meta:application/vnd.appthreat.vdb.config.v1+json \
./data.vdb6.zst:application/vnd.appthreat.vdb.layer.v1+zst \
./data.index.vdb6.zst:application/vnd.appthreat.vdb.layer.v1+zst
oras push ghcr.io/appthreat/vdbxz-app:v6 \
--artifact-type application/vnd.oras.config.v1+json \
./vdb.meta:application/vnd.appthreat.vdb.config.v1+json \
./data.vdb6.tar.xz:application/vnd.appthreat.vdb.layer.v1+tar \
./data.index.vdb6.tar.xz:application/vnd.appthreat.vdb.layer.v1+tar
# echo $CODEBERG_TOKEN | oras login codeberg.org -u $CODEBERG_USERNAME --password-stdin
# oras push codeberg.org/appthreat/vdbzst-app:v6 \
# --artifact-type application/vnd.oras.config.v1+json \
# ./vdb.meta:application/vnd.appthreat.vdb.config.v1+json \
# ./data.vdb6.zst:application/vnd.appthreat.vdb.layer.v1+zst \
# ./data.index.vdb6.zst:application/vnd.appthreat.vdb.layer.v1+zst
# oras push codeberg.org/appthreat/vdbxz-app:v6 \
# --artifact-type application/vnd.oras.config.v1+json \
# ./vdb.meta:application/vnd.appthreat.vdb.config.v1+json \
# ./data.vdb6.tar.xz:application/vnd.appthreat.vdb.layer.v1+tar \
# ./data.index.vdb6.tar.xz:application/vnd.appthreat.vdb.layer.v1+tar
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
CODEBERG_TOKEN: ${{ secrets.CODEBERG_DEPLOY }}
PYTHONPATH: vulnerability-db
VDB_HOME: vdb_data
VDB_CACHE: vdb_cache
GITHUB_PAGE_COUNT: 30
NVD_START_YEAR: 2018
GITHUB_USERNAME: ${{ github.actor }}
CODEBERG_USERNAME: appthreat
VDB_METADATA_APP_ONLY: true
VDB_IGNORE_ALMALINUX: true
VDB_IGNORE_ALPINE: true
VDB_IGNORE_DEBIAN: true
VDB_IGNORE_ROCKYLINUX: true
continue-on-error: true
app_10y_builder:
runs-on: ubuntu-latest
permissions:
contents: write
packages: write
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v4
with:
repository: AppThreat/vulnerability-db
path: vulnerability-db
ref: "v6.2.3"
- uses: actions/checkout@v4
with:
repository: AppThreat/vuln-list
path: vuln-list
fetch-depth: "1"
sparse-checkout: |
nvd/2014
nvd/2015
nvd/2016
nvd/2017
nvd/2018
nvd/2019
nvd/2020
nvd/2021
nvd/2022
nvd/2023
nvd/2024
- uses: oras-project/setup-oras@v1
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Trim CI agent
run: |
chmod +x contrib/free_disk_space.sh
./contrib/free_disk_space.sh
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install setuptools wheel twine build
cd vulnerability-db && pip install ".[dev]"
- name: Build and upload app db - 2018
run: |
mkdir vdb_data vdb_cache
ls ./vuln-list/ ./vuln-list/nvd/
zip -q -r vuln-list.zip ./vuln-list/
mv vuln-list.zip vdb_cache/
rm -rf ./vuln-list/
python vulnerability-db/vdb/cli.py --cache-os
ls -lh vdb_data
ls -lh vdb_cache
cd vdb_data
zstd --ultra data.index.vdb6
zstd --ultra data.vdb6
tar -cvJf data.vdb6.tar.xz data.vdb6
tar -cvJf data.index.vdb6.tar.xz data.index.vdb6
echo $GITHUB_TOKEN | oras login ghcr.io -u $GITHUB_USERNAME --password-stdin
oras push ghcr.io/appthreat/vdbzst-app-10y:v6 \
--artifact-type application/vnd.oras.config.v1+json \
./vdb.meta:application/vnd.appthreat.vdb.config.v1+json \
./data.vdb6.zst:application/vnd.appthreat.vdb.layer.v1+zst \
./data.index.vdb6.zst:application/vnd.appthreat.vdb.layer.v1+zst
oras push ghcr.io/appthreat/vdbxz-app-10y:v6 \
--artifact-type application/vnd.oras.config.v1+json \
./vdb.meta:application/vnd.appthreat.vdb.config.v1+json \
./data.vdb6.tar.xz:application/vnd.appthreat.vdb.layer.v1+tar \
./data.index.vdb6.tar.xz:application/vnd.appthreat.vdb.layer.v1+tar
# echo $CODEBERG_TOKEN | oras login codeberg.org -u $CODEBERG_USERNAME --password-stdin
# oras push codeberg.org/appthreat/vdbzst-app-10y:v6 \
# --artifact-type application/vnd.oras.config.v1+json \
# ./vdb.meta:application/vnd.appthreat.vdb.config.v1+json \
# ./data.vdb6.zst:application/vnd.appthreat.vdb.layer.v1+zst \
# ./data.index.vdb6.zst:application/vnd.appthreat.vdb.layer.v1+zst
# oras push codeberg.org/appthreat/vdbxz-app-10y:v6 \
# --artifact-type application/vnd.oras.config.v1+json \
# ./vdb.meta:application/vnd.appthreat.vdb.config.v1+json \
# ./data.vdb6.tar.xz:application/vnd.appthreat.vdb.layer.v1+tar \
# ./data.index.vdb6.tar.xz:application/vnd.appthreat.vdb.layer.v1+tar
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
CODEBERG_TOKEN: ${{ secrets.CODEBERG_DEPLOY }}
PYTHONPATH: vulnerability-db
VDB_HOME: vdb_data
VDB_CACHE: vdb_cache
GITHUB_PAGE_COUNT: 30
NVD_START_YEAR: 2018
GITHUB_USERNAME: ${{ github.actor }}
CODEBERG_USERNAME: appthreat
VDB_METADATA_APP_ONLY: true
VDB_IGNORE_ALMALINUX: true
VDB_IGNORE_ALPINE: true
VDB_IGNORE_DEBIAN: true
VDB_IGNORE_ROCKYLINUX: true
continue-on-error: true
builder:
runs-on: [self-hosted, arm64]
permissions:
contents: write
packages: write
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v4
with:
repository: AppThreat/vulnerability-db
path: vulnerability-db
ref: "v6.2.3"
- uses: actions/checkout@v4
with:
repository: AppThreat/vuln-list
path: vuln-list
fetch-depth: "1"
- uses: oras-project/setup-oras@v1
- name: Install dependencies
run: |
python3 -m pip config set global.break-system-packages true
cd vulnerability-db && python3 -m pip install ".[dev]" --no-warn-script-location
- name: Build and upload db - 2018
run: |
mkdir vdb_data vdb_cache
rm -rf ./vuln-list/kevc ./vuln-list/mariner ./vuln-list/cvrf/suse/suse
zip -q -r vuln-list.zip ./vuln-list/
mv vuln-list.zip vdb_cache/
rm -rf ./vuln-list/
python3 vulnerability-db/vdb/cli.py --cache-os
ls -lh vdb_data
ls -lh vdb_cache
cd vdb_data
zstd --ultra data.index.vdb6
zstd --ultra data.vdb6
tar -cvJf data.vdb6.tar.xz data.vdb6
tar -cvJf data.index.vdb6.tar.xz data.index.vdb6
echo $GITHUB_TOKEN | oras login ghcr.io -u $GITHUB_USERNAME --password-stdin
oras push ghcr.io/appthreat/vdbzst:v6 \
--artifact-type application/vnd.oras.config.v1+json \
./vdb.meta:application/vnd.appthreat.vdb.config.v1+json \
./data.vdb6.zst:application/vnd.appthreat.vdb.layer.v1+zst \
./data.index.vdb6.zst:application/vnd.appthreat.vdb.layer.v1+zst
oras push ghcr.io/appthreat/vdbxz:v6 \
--artifact-type application/vnd.oras.config.v1+json \
./vdb.meta:application/vnd.appthreat.vdb.config.v1+json \
./data.vdb6.tar.xz:application/vnd.appthreat.vdb.layer.v1+tar \
./data.index.vdb6.tar.xz:application/vnd.appthreat.vdb.layer.v1+tar
# echo $CODEBERG_TOKEN | oras login codeberg.org -u $CODEBERG_USERNAME --password-stdin
# oras push codeberg.org/appthreat/vdbzst:v6 \
# --artifact-type application/vnd.oras.config.v1+json \
# ./vdb.meta:application/vnd.appthreat.vdb.config.v1+json \
# ./data.vdb6.zst:application/vnd.appthreat.vdb.layer.v1+zst \
# ./data.index.vdb6.zst:application/vnd.appthreat.vdb.layer.v1+zst
# oras push codeberg.org/appthreat/vdbxz:v6 \
# --artifact-type application/vnd.oras.config.v1+json \
# ./vdb.meta:application/vnd.appthreat.vdb.config.v1+json \
# ./data.vdb6.tar.xz:application/vnd.appthreat.vdb.layer.v1+tar \
# ./data.index.vdb6.tar.xz:application/vnd.appthreat.vdb.layer.v1+tar
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
CODEBERG_TOKEN: ${{ secrets.CODEBERG_DEPLOY }}
PYTHONPATH: vulnerability-db
VDB_HOME: vdb_data
VDB_CACHE: vdb_cache
GITHUB_PAGE_COUNT: 10
NVD_START_YEAR: 2018
GITHUB_USERNAME: ${{ github.actor }}
CODEBERG_USERNAME: appthreat
- uses: actions/checkout@v4
with:
repository: AppThreat/vuln-list
path: vuln-list2
fetch-depth: "1"
- name: Build and upload db - 2014
run: |
cd $GITHUB_WORKSPACE
rm -rf vdb_data vdb_cache
mkdir vdb_data vdb_cache
rm -rf ./vuln-list/kevc ./vuln-list/mariner ./vuln-list/cvrf/suse/suse
zip -q -r vuln-list.zip ./vuln-list2/
mv vuln-list.zip vdb_cache/
rm -rf ./vuln-list2/
python3 vulnerability-db/vdb/cli.py --cache-os
ls -lh vdb_data
ls -lh vdb_cache
cd vdb_data
zstd --ultra data.index.vdb6
zstd --ultra data.vdb6
tar -cvJf data.vdb6.tar.xz data.vdb6
tar -cvJf data.index.vdb6.tar.xz data.index.vdb6
echo $GITHUB_TOKEN | oras login ghcr.io -u $GITHUB_USERNAME --password-stdin
oras push ghcr.io/appthreat/vdbzst-10y:v6 \
--artifact-type application/vnd.oras.config.v1+json \
./vdb.meta:application/vnd.appthreat.vdb.config.v1+json \
./data.vdb6.zst:application/vnd.appthreat.vdb.layer.v1+zst \
./data.index.vdb6.zst:application/vnd.appthreat.vdb.layer.v1+zst
oras push ghcr.io/appthreat/vdbxz-10y:v6 \
--artifact-type application/vnd.oras.config.v1+json \
./vdb.meta:application/vnd.appthreat.vdb.config.v1+json \
./data.vdb6.tar.xz:application/vnd.appthreat.vdb.layer.v1+tar \
./data.index.vdb6.tar.xz:application/vnd.appthreat.vdb.layer.v1+tar
# echo $CODEBERG_TOKEN | oras login codeberg.org -u $CODEBERG_USERNAME --password-stdin
# oras push codeberg.org/appthreat/vdbzst-10y:v6 \
# --artifact-type application/vnd.oras.config.v1+json \
# ./vdb.meta:application/vnd.appthreat.vdb.config.v1+json \
# ./data.vdb6.zst:application/vnd.appthreat.vdb.layer.v1+zst \
# ./data.index.vdb6.zst:application/vnd.appthreat.vdb.layer.v1+zst
# oras push codeberg.org/appthreat/vdbxz-10y:v6 \
# --artifact-type application/vnd.oras.config.v1+json \
# ./vdb.meta:application/vnd.appthreat.vdb.config.v1+json \
# ./data.vdb6.tar.xz:application/vnd.appthreat.vdb.layer.v1+tar \
# ./data.index.vdb6.tar.xz:application/vnd.appthreat.vdb.layer.v1+tar
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
CODEBERG_TOKEN: ${{ secrets.CODEBERG_DEPLOY }}
PYTHONPATH: vulnerability-db
VDB_HOME: vdb_data
VDB_CACHE: vdb_cache
GITHUB_PAGE_COUNT: 20
NVD_START_YEAR: 2014
GITHUB_USERNAME: ${{ github.actor }}
CODEBERG_USERNAME: appthreat