linus/Knack-Scraper
1
0
Fork 0
forked from lukaszett/Knack-Scraper
Code Pull requests Activity

Use different embeddings model;

Browse source
This commit is contained in:
quorploop 2026-01-18 15:43:35 +01:00
parent 49239e7e25
commit 8fae350b34
10 changed files with 1846 additions and 57 deletions
Download patch file Download diff file Expand all files Collapse all files

12
transform/Dockerfile
View file

@ -17,8 +17,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
ENV GLINER_MODEL_ID=urchade/gliner_multi-v2.1
ENV GLINER_MODEL_PATH=/models/gliner_multi-v2.1
ENV MINILM_MODEL_ID=sentence-transformers/all-MiniLM-L6-v2
ENV MINILM_MODEL_PATH=/models/all-MiniLM-L6-v2
ENV GTE_MODEL_ID=thenlper/gte-large
ENV GTE_MODEL_PATH=/models/thenlper/gte-large
WORKDIR /app
COPY requirements.txt .
@ -31,16 +31,16 @@ RUN apt install -y cron locales
# Ensure GLiNER helper scripts are available
COPY ensure_gliner_model.sh /usr/local/bin/ensure_gliner_model.sh
# Ensure MiniLM helper scripts are available
COPY ensure_minilm_model.sh /usr/local/bin/ensure_minilm_model.sh
# Ensure GTE helper scripts are available
COPY ensure_gte_model.sh /usr/local/bin/ensure_gte_model.sh
COPY entrypoint.sh /usr/local/bin/entrypoint.sh
RUN chmod +x /usr/local/bin/ensure_gliner_model.sh /usr/local/bin/ensure_minilm_model.sh /usr/local/bin/entrypoint.sh
RUN chmod +x /usr/local/bin/ensure_gliner_model.sh /usr/local/bin/ensure_gte_model.sh /usr/local/bin/entrypoint.sh
COPY *.py .
# Create cron job that runs every weekend (Sunday at 3 AM) 0 3 * * 0
# Testing every 30 Minutes */30 * * * *
RUN echo "*/30 * * * * cd /app && /usr/local/bin/python main.py >> /proc/1/fd/1 2>&1" > /etc/cron.d/knack-transform
RUN echo "*/15 * * * * cd /app && /usr/local/bin/python main.py >> /proc/1/fd/1 2>&1" > /etc/cron.d/knack-transform
RUN chmod 0644 /etc/cron.d/knack-transform
RUN crontab /etc/cron.d/knack-transform