forked from lukaszett/Knack-Scraper
Makes transformer script executable via cli
This commit is contained in:
parent
8fae350b34
commit
7c2e34906e
11 changed files with 648 additions and 37 deletions
|
|
@ -1,16 +1,35 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [ -d "$GLINER_MODEL_PATH" ] && find "$GLINER_MODEL_PATH" -type f | grep -q .; then
|
||||
if [ -d "$GLINER_MODEL_PATH" ] && [ -f "$GLINER_MODEL_PATH/config.json" ]; then
|
||||
echo "GLiNER model already present at $GLINER_MODEL_PATH"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Downloading GLiNER model to $GLINER_MODEL_PATH"
|
||||
echo "Downloading GLiNER model $GLINER_MODEL_ID to $GLINER_MODEL_PATH"
|
||||
mkdir -p "$GLINER_MODEL_PATH"
|
||||
curl -sL "https://huggingface.co/api/models/${GLINER_MODEL_ID}" | jq -r '.siblings[].rfilename' | while read -r file; do
|
||||
target="${GLINER_MODEL_PATH}/${file}"
|
||||
mkdir -p "$(dirname "$target")"
|
||||
echo "Downloading ${file}"
|
||||
curl -sL "https://huggingface.co/${GLINER_MODEL_ID}/resolve/main/${file}" -o "$target"
|
||||
done
|
||||
|
||||
# Use Python with huggingface_hub for reliable model downloading
|
||||
python3 << 'EOF'
|
||||
import os
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
model_id = os.environ.get('GLINER_MODEL_ID')
|
||||
model_path = os.environ.get('GLINER_MODEL_PATH')
|
||||
|
||||
if not model_id or not model_path:
|
||||
raise ValueError(f"GLINER_MODEL_ID and GLINER_MODEL_PATH environment variables must be set")
|
||||
|
||||
try:
|
||||
print(f"Downloading model {model_id} to {model_path}")
|
||||
snapshot_download(
|
||||
repo_id=model_id,
|
||||
cache_dir=None, # Don't use cache, download directly
|
||||
local_dir=model_path,
|
||||
local_dir_use_symlinks=False # Don't use symlinks, copy files
|
||||
)
|
||||
print(f"Successfully downloaded GLiNER model to {model_path}")
|
||||
except Exception as e:
|
||||
print(f"Error downloading GLiNER model: {e}")
|
||||
exit(1)
|
||||
EOF
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue