From a6877bac852af7f44b6554bbc855c2ebb47b5a0e Mon Sep 17 00:00:00 2001
From: Chang She <759245+changhiskhan@users.noreply.github.com>
Date: Wed, 14 Dec 2022 15:06:35 -0800
Subject: [PATCH] create and update tarball for pets (#372)

---
 python/tools/update_pet.sh | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/python/tools/update_pet.sh b/python/tools/update_pet.sh
index d56b422411..d269f800c2 100755
--- a/python/tools/update_pet.sh
+++ b/python/tools/update_pet.sh
@@ -15,15 +15,25 @@
 
 set -e
 
-PUBLIC_URI_ROOT="https://eto-public.s3.us-west-2.amazonaws.com/datasets/oxford_pet/"
+PUBLIC_URI_ROOT="https://eto-public.s3.us-west-2.amazonaws.com/datasets/oxford_pet"
 
 DATASET_ROOT=$1  # this is the root dir of the raw dataset
 OUTPUT_PATH=$2  # this is the root dir of the lance/parquet dataset
 
+rm -rf ${OUTPUT_PATH}
+rm -rf ${OUTPUT_PATH}.tar.gz
 
 python lance/data/convert/oxford_pet.py \
   $DATASET_ROOT --output-path $OUTPUT_PATH \
   --fmt lance --images-root $PUBLIC_URI_ROOT
 
+
+pushd ${OUTPUT_PATH}/../
+tar -cvf oxford_pet.lance.tar.gz oxford_pet.lance/
+popd
+
 aws s3 rm --recursive s3://eto-public/datasets/oxford_pet/oxford_pet.lance
-aws s3 cp --recursive $OUTPUT_PATH s3://eto-public/datasets/oxford_pet/oxford_pet.lance
\ No newline at end of file
+aws s3 cp --recursive $OUTPUT_PATH s3://eto-public/datasets/oxford_pet/oxford_pet.lance
+
+aws s3 rm s3://eto-public/datasets/oxford_pet/oxford_pet.lance.tar.gz
+aws s3 cp ${OUTPUT_PATH}.tar.gz s3://eto-public/datasets/oxford_pet/oxford_pet.lance.tar.gz