forked from lukaszett/Knack-Scraper
Use different embeddings model;
This commit is contained in:
parent
49239e7e25
commit
8fae350b34
10 changed files with 1846 additions and 57 deletions
16
transform/ensure_gte_model.sh
Normal file
16
transform/ensure_gte_model.sh
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [ -d "$GTE_MODEL_PATH" ] && find "$GTE_MODEL_PATH" -type f | grep -q .; then
|
||||
echo "GTE model already present at $GTE_MODEL_PATH"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Downloading GTE model to $GTE_MODEL_PATH"
|
||||
mkdir -p "$GTE_MODEL_PATH"
|
||||
curl -sL "https://huggingface.co/api/models/${GTE_MODEL_ID}" | jq -r '.siblings[].rfilename' | while read -r file; do
|
||||
target="${GTE_MODEL_PATH}/${file}"
|
||||
mkdir -p "$(dirname "$target")"
|
||||
echo "Downloading ${file}"
|
||||
curl -sL "https://huggingface.co/${GTE_MODEL_ID}/resolve/main/${file}" -o "$target"
|
||||
done
|
||||
Loading…
Add table
Add a link
Reference in a new issue