From 6c7d7427bf853ff603ad2ba43748f39ee7a52c6b Mon Sep 17 00:00:00 2001 From: dreimolo Date: Sat, 16 Dec 2023 14:14:58 +0100 Subject: [PATCH 1/5] Adds check for wget and curl, and valid sample archives --- run/download-samples.sh | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/run/download-samples.sh b/run/download-samples.sh index 66aa6125..09ddcf6b 100755 --- a/run/download-samples.sh +++ b/run/download-samples.sh @@ -2,16 +2,37 @@ set -e +# Check if wget exists +if command -v wget &> /dev/null; then + dl_prg="wget -O" +elif command -v curl &> /dev/null; then + dl_prg="curl -o" +else + echo "Neither wget nor curl found, exiting .." + exit 1 +fi + +case "$1" in +"s"|"m"|"l"|"xl") + ;; +*) + echo "Invalid argument. Must be one of 's', 'm', or 'l'." + exit 1 + ;; +esac + SAMPLE_NAME=crawl-${1:-m} SAMPLE_DIR="node-1/samples/${SAMPLE_NAME}/" function download_model { + + model=$1 url=$2 if [ ! -f $model ]; then echo "** Downloading $url" - wget -O $model $url + $dl_prg $model $url fi } @@ -23,7 +44,7 @@ fi mkdir -p node-1/samples/ SAMPLE_TARBALL=samples/${SAMPLE_NAME}.tar.gz -download_model ${SAMPLE_TARBALL} https://downloads.marginalia.nu/${SAMPLE_TARBALL} || rm ${SAMPLE_TARBALL} +download_model ${SAMPLE_TARBALL} https://downloads.marginalia.nu/${SAMPLE_TARBALL} if [ ! -f ${SAMPLE_TARBALL} ]; then echo "!! Failed" From 0b34d43804e683889ff8ce51d6e6c2284eccc52d Mon Sep 17 00:00:00 2001 From: dreimolo Date: Sat, 16 Dec 2023 14:22:11 +0100 Subject: [PATCH 2/5] workaround for failing mac on apple silicon deps --- code/api/index-api/build.gradle | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/code/api/index-api/build.gradle b/code/api/index-api/build.gradle index 37d76aa0..42cbc4c0 100644 --- a/code/api/index-api/build.gradle +++ b/code/api/index-api/build.gradle @@ -44,11 +44,18 @@ dependencies { protobuf { protoc { - artifact = "com.google.protobuf:protoc:3.0.2" + if (osdetector.os == "osx") { + artifact = "com.google.protobuf:protoc:3.0.2:osx-x86_64" + } else { + artifact = "com.google.protobuf:protoc:3.0.2" + } } plugins { grpc { - artifact = 'io.grpc:protoc-gen-grpc-java:1.1.2' + if (osdetector.os == "osx") { + artifact = "io.grpc:protoc-gen-grpc-java:1.1.2:osx-x86_64" + } else { + artifact = "io.grpc:protoc-gen-grpc-java:1.1.2" } } From c0cc05177fe36436289b8bb9b4c8b4b74de4739e Mon Sep 17 00:00:00 2001 From: dreimolo Date: Sat, 16 Dec 2023 14:24:41 +0100 Subject: [PATCH 3/5] corrects protobuf.plugins.grpc --- code/api/index-api/build.gradle | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/code/api/index-api/build.gradle b/code/api/index-api/build.gradle index 42cbc4c0..51205960 100644 --- a/code/api/index-api/build.gradle +++ b/code/api/index-api/build.gradle @@ -52,10 +52,11 @@ protobuf { } plugins { grpc { - if (osdetector.os == "osx") { - artifact = "io.grpc:protoc-gen-grpc-java:1.1.2:osx-x86_64" - } else { - artifact = "io.grpc:protoc-gen-grpc-java:1.1.2" + if (osdetector.os == "osx") { + artifact = "io.grpc:protoc-gen-grpc-java:1.1.2:osx-x86_64" + } else { + artifact = "io.grpc:protoc-gen-grpc-java:1.1.2" + } } } From 62954f98de158a83e5efc8c130c37351cd106333 Mon Sep 17 00:00:00 2001 From: dreimolo Date: Sat, 16 Dec 2023 19:41:41 +0100 Subject: [PATCH 4/5] adds xl to help output --- run/download-samples.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/run/download-samples.sh b/run/download-samples.sh index 09ddcf6b..d46dce56 100755 --- a/run/download-samples.sh +++ b/run/download-samples.sh @@ -16,7 +16,7 @@ case "$1" in "s"|"m"|"l"|"xl") ;; *) - echo "Invalid argument. Must be one of 's', 'm', or 'l'." + echo "Invalid argument. Must be one of 's', 'm', 'l' or 'xl'." exit 1 ;; esac @@ -25,8 +25,6 @@ SAMPLE_NAME=crawl-${1:-m} SAMPLE_DIR="node-1/samples/${SAMPLE_NAME}/" function download_model { - - model=$1 url=$2 From 128f550ee568f39b6775736e0481be4e4e8d0a4b Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Mon, 18 Dec 2023 18:28:17 +0100 Subject: [PATCH 5/5] (run) Download to a temporary file to avoid corruption from aborted downloads --- run/download-samples.sh | 2 +- run/setup.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/run/download-samples.sh b/run/download-samples.sh index d46dce56..bbae77e6 100755 --- a/run/download-samples.sh +++ b/run/download-samples.sh @@ -42,7 +42,7 @@ fi mkdir -p node-1/samples/ SAMPLE_TARBALL=samples/${SAMPLE_NAME}.tar.gz -download_model ${SAMPLE_TARBALL} https://downloads.marginalia.nu/${SAMPLE_TARBALL} +download_model ${SAMPLE_TARBALL}.tmp https://downloads.marginalia.nu/${SAMPLE_TARBALL} && mv ${SAMPLE_TARBALL}.tmp ${SAMPLE_TARBALL} if [ ! -f ${SAMPLE_TARBALL} ]; then echo "!! Failed" diff --git a/run/setup.sh b/run/setup.sh index f55817e3..1315893a 100755 --- a/run/setup.sh +++ b/run/setup.sh @@ -12,7 +12,8 @@ function download_model { if [ ! -f $model ]; then echo "** Downloading $url" - curl -s -o $model $url + curl -s -o $model.tmp $url + mv $model.tmp $model fi }