Cache (#166)

cache source FWA gdb files on bcfishpass object storage

Makefile

@@ -73,18 +73,18 @@ clean_db:
 	$(PSQL) -f db/functions/postgisftw.sql  # pg_fs/pg_ts functions
 	touch $@
 
-# download and rename (so we do not have to unzip)
+# download
 data/FWA_BC.gdb.zip: .make/db
-	curl -o $@ ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/FWA_Public/FWA_BC.zip
+	curl -o $@ https://nrs.objectstore.gov.bc.ca/bchamp/fwapg/FWA_BC.gdb.zip
 
 data/FWA_LINEAR_BOUNDARIES_SP.gdb.zip: .make/db
-	curl -o $@ ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/FWA_Public/FWA_LINEAR_BOUNDARIES_SP.zip
+	curl -o $@ https://nrs.objectstore.gov.bc.ca/bchamp/fwapg/FWA_LINEAR_BOUNDARIES_SP.gdb.zip
 
 data/FWA_WATERSHEDS_POLY.gdb.zip: .make/db
-	curl -o $@ ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/FWA_Public/FWA_WATERSHEDS_POLY.zip
+	curl -o $@ https://nrs.objectstore.gov.bc.ca/bchamp/fwapg/FWA_WATERSHEDS_POLY.gdb.zip
 
 data/FWA_STREAM_NETWORKS_SP.gdb.zip: .make/db
-	curl -o $@ ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/FWA_Public/FWA_STREAM_NETWORKS_SP.zip
+	curl -o $@ https://nrs.objectstore.gov.bc.ca/bchamp/fwapg/FWA_STREAM_NETWORKS_SP.gdb.zip
 
 # load the larger tables per watershed group
 .make/fwa_stream_networks_sp: data/FWA_STREAM_NETWORKS_SP.gdb.zip .make/db

cache.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -euxo pipefail
+
+# cache fwa data to s3 for (much) faster downloads
+# also, adds .gdb to filename so files can be accessed directly via /vsizip//vsicurl
+
+curl -o /tmp/FWA_BC.gdb.zip ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/FWA_Public/FWA_BC.zip
+curl -o /tmp/FWA_LINEAR_BOUNDARIES_SP.gdb.zip ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/FWA_Public/FWA_LINEAR_BOUNDARIES_SP.zip
+curl -o /tmp/FWA_WATERSHEDS_POLY.gdb.zip ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/FWA_Public/FWA_WATERSHEDS_POLY.zip
+curl -o /tmp/FWA_STREAM_NETWORKS_SP.gdb.zip ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/FWA_Public/FWA_STREAM_NETWORKS_SP.zip
+
+aws s3 cp /tmp/FWA_BC.gdb.zip s3://bchamp/fwapg/FWA_BC.gdb.zip --acl public-read 
+aws s3 cp /tmp/FWA_LINEAR_BOUNDARIES_SP.gdb.zip s3://bchamp/fwapg/FWA_LINEAR_BOUNDARIES_SP.gdb.zip --acl public-read
+aws s3 cp /tmp/FWA_WATERSHEDS_POLY.gdb.zip s3://bchamp/fwapg/FWA_WATERSHEDS_POLY.gdb.zip --acl public-read
+aws s3 cp /tmp/FWA_STREAM_NETWORKS_SP.gdb.zip s3://bchamp/fwapg/FWA_STREAM_NETWORKS_SP.gdb.zip --acl public-read