feat: add multi-stages docker and support platform arm (#274)

* feat: add multi-stages docker and support platform arm

* refactor: pre-commit

* fix: raise ImportError (fastembed) instead of auto install

* feat: add dependencies for local llm

* feat: free disk

* feat: update README

* feat: update README

* chore: fix typo

---------

Co-authored-by: cin-niko <niko@cinnamon.is>
This commit is contained in:
kan_cin
2024-09-12 20:25:03 +07:00
committed by GitHub
parent 73a476979e
commit d3fd75297f
6 changed files with 141 additions and 31 deletions

View File

@@ -1,14 +1,7 @@
# syntax=docker/dockerfile:1.0.0-experimental
FROM python:3.10-slim as base_image
# for additional file parsers
# tesseract-ocr \
# tesseract-ocr-jpn \
# libsm6 \
# libxext6 \
# ffmpeg \
# Lite version
FROM python:3.10-slim AS lite
# Common dependencies
RUN apt-get update -qqy && \
apt-get install -y --no-install-recommends \
ssh \
@@ -19,28 +12,75 @@ RUN apt-get update -qqy && \
libpoppler-dev \
unzip \
curl \
&& apt-get clean \
&& apt-get autoremove \
&& rm -rf /var/lib/apt/lists/*
cargo
# Set environment variables
ENV PYTHONDONTWRITEBYTECODE=1
ENV PYTHONUNBUFFERED=1
ENV PYTHONIOENCODING=UTF-8
# Create working directory
WORKDIR /app
FROM base_image as dev
# Download pdfjs
COPY scripts/download_pdfjs.sh /app/scripts/download_pdfjs.sh
RUN chmod +x /app/scripts/download_pdfjs.sh
ENV PDFJS_PREBUILT_DIR="/app/libs/ktem/ktem/assets/prebuilt/pdfjs-dist"
RUN bash scripts/download_pdfjs.sh $PDFJS_PREBUILT_DIR
# Copy contents
COPY . /app
RUN --mount=type=ssh pip install --no-cache-dir -e "libs/kotaemon[all]" \
&& pip install --no-cache-dir -e "libs/ktem" \
&& pip install --no-cache-dir graphrag future \
&& pip install --no-cache-dir "pdfservices-sdk@git+https://github.com/niallcm/pdfservices-python-sdk.git@bump-and-unfreeze-requirements"
# Install pip packages
RUN --mount=type=ssh \
--mount=type=cache,target=/root/.cache/pip \
pip install -e "libs/kotaemon[all]" \
&& pip install -e "libs/ktem" \
&& pip install graphrag future \
&& pip install "pdfservices-sdk@git+https://github.com/niallcm/pdfservices-python-sdk.git@bump-and-unfreeze-requirements"
# Clean up
RUN apt-get autoremove \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
&& rm -rf ~/.cache
CMD ["python", "app.py"]
# Full version
FROM lite AS full
# Additional dependencies for full version
RUN apt-get update -qqy && \
apt-get install -y --no-install-recommends \
tesseract-ocr \
tesseract-ocr-jpn \
libsm6 \
libxext6 \
libreoffice \
ffmpeg \
libmagic-dev
# Install torch and torchvision for unstructured
RUN --mount=type=ssh \
--mount=type=cache,target=/root/.cache/pip \
pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
# Copy contents
COPY . /app
# Install additional pip packages
RUN --mount=type=ssh \
--mount=type=cache,target=/root/.cache/pip \
pip install unstructured[all-docs]
# Clean up
RUN apt-get autoremove \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
&& rm -rf ~/.cache
# Download nltk packages as required for unstructured
RUN python -c "from unstructured.nlp.tokenize import _download_nltk_packages_if_not_present; _download_nltk_packages_if_not_present()"
CMD ["python", "app.py"]