From 4631a925da228b14a19ccd482c1f907f6716c750 Mon Sep 17 00:00:00 2001 From: Simon Norris Date: Wed, 20 Nov 2024 10:24:41 -0800 Subject: [PATCH] Cache (#166) cache source FWA gdb files on bcfishpass object storage --- Makefile | 10 +++++----- cache.sh | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) create mode 100755 cache.sh diff --git a/Makefile b/Makefile index cfd0d66..93892f1 100644 --- a/Makefile +++ b/Makefile @@ -73,18 +73,18 @@ clean_db: $(PSQL) -f db/functions/postgisftw.sql # pg_fs/pg_ts functions touch $@ -# download and rename (so we do not have to unzip) +# download data/FWA_BC.gdb.zip: .make/db - curl -o $@ ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/FWA_Public/FWA_BC.zip + curl -o $@ https://nrs.objectstore.gov.bc.ca/bchamp/fwapg/FWA_BC.gdb.zip data/FWA_LINEAR_BOUNDARIES_SP.gdb.zip: .make/db - curl -o $@ ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/FWA_Public/FWA_LINEAR_BOUNDARIES_SP.zip + curl -o $@ https://nrs.objectstore.gov.bc.ca/bchamp/fwapg/FWA_LINEAR_BOUNDARIES_SP.gdb.zip data/FWA_WATERSHEDS_POLY.gdb.zip: .make/db - curl -o $@ ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/FWA_Public/FWA_WATERSHEDS_POLY.zip + curl -o $@ https://nrs.objectstore.gov.bc.ca/bchamp/fwapg/FWA_WATERSHEDS_POLY.gdb.zip data/FWA_STREAM_NETWORKS_SP.gdb.zip: .make/db - curl -o $@ ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/FWA_Public/FWA_STREAM_NETWORKS_SP.zip + curl -o $@ https://nrs.objectstore.gov.bc.ca/bchamp/fwapg/FWA_STREAM_NETWORKS_SP.gdb.zip # load the larger tables per watershed group .make/fwa_stream_networks_sp: data/FWA_STREAM_NETWORKS_SP.gdb.zip .make/db diff --git a/cache.sh b/cache.sh new file mode 100755 index 0000000..dd55a3c --- /dev/null +++ b/cache.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -euxo pipefail + +# cache fwa data to s3 for (much) faster downloads +# also, adds .gdb to filename so files can be accessed directly via /vsizip//vsicurl + +curl -o /tmp/FWA_BC.gdb.zip ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/FWA_Public/FWA_BC.zip +curl -o /tmp/FWA_LINEAR_BOUNDARIES_SP.gdb.zip ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/FWA_Public/FWA_LINEAR_BOUNDARIES_SP.zip +curl -o /tmp/FWA_WATERSHEDS_POLY.gdb.zip ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/FWA_Public/FWA_WATERSHEDS_POLY.zip +curl -o /tmp/FWA_STREAM_NETWORKS_SP.gdb.zip ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/FWA_Public/FWA_STREAM_NETWORKS_SP.zip + +aws s3 cp /tmp/FWA_BC.gdb.zip s3://bchamp/fwapg/FWA_BC.gdb.zip --acl public-read +aws s3 cp /tmp/FWA_LINEAR_BOUNDARIES_SP.gdb.zip s3://bchamp/fwapg/FWA_LINEAR_BOUNDARIES_SP.gdb.zip --acl public-read +aws s3 cp /tmp/FWA_WATERSHEDS_POLY.gdb.zip s3://bchamp/fwapg/FWA_WATERSHEDS_POLY.gdb.zip --acl public-read +aws s3 cp /tmp/FWA_STREAM_NETWORKS_SP.gdb.zip s3://bchamp/fwapg/FWA_STREAM_NETWORKS_SP.gdb.zip --acl public-read