feat: add multi-stages docker and support platform arm (#274)
* feat: add multi-stages docker and support platform arm * refactor: pre-commit * fix: raise ImportError (fastembed) instead of auto install * feat: add dependencies for local llm * feat: free disk * feat: update README * feat: update README * chore: fix typo --------- Co-authored-by: cin-niko <niko@cinnamon.is>
This commit is contained in:
80
Dockerfile
80
Dockerfile
@@ -1,14 +1,7 @@
|
||||
# syntax=docker/dockerfile:1.0.0-experimental
|
||||
FROM python:3.10-slim as base_image
|
||||
|
||||
# for additional file parsers
|
||||
|
||||
# tesseract-ocr \
|
||||
# tesseract-ocr-jpn \
|
||||
# libsm6 \
|
||||
# libxext6 \
|
||||
# ffmpeg \
|
||||
# Lite version
|
||||
FROM python:3.10-slim AS lite
|
||||
|
||||
# Common dependencies
|
||||
RUN apt-get update -qqy && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
ssh \
|
||||
@@ -19,28 +12,75 @@ RUN apt-get update -qqy && \
|
||||
libpoppler-dev \
|
||||
unzip \
|
||||
curl \
|
||||
&& apt-get clean \
|
||||
&& apt-get autoremove \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
cargo
|
||||
|
||||
# Set environment variables
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
ENV PYTHONIOENCODING=UTF-8
|
||||
|
||||
# Create working directory
|
||||
WORKDIR /app
|
||||
|
||||
FROM base_image as dev
|
||||
|
||||
# Download pdfjs
|
||||
COPY scripts/download_pdfjs.sh /app/scripts/download_pdfjs.sh
|
||||
RUN chmod +x /app/scripts/download_pdfjs.sh
|
||||
|
||||
ENV PDFJS_PREBUILT_DIR="/app/libs/ktem/ktem/assets/prebuilt/pdfjs-dist"
|
||||
RUN bash scripts/download_pdfjs.sh $PDFJS_PREBUILT_DIR
|
||||
|
||||
# Copy contents
|
||||
COPY . /app
|
||||
RUN --mount=type=ssh pip install --no-cache-dir -e "libs/kotaemon[all]" \
|
||||
&& pip install --no-cache-dir -e "libs/ktem" \
|
||||
&& pip install --no-cache-dir graphrag future \
|
||||
&& pip install --no-cache-dir "pdfservices-sdk@git+https://github.com/niallcm/pdfservices-python-sdk.git@bump-and-unfreeze-requirements"
|
||||
|
||||
# Install pip packages
|
||||
RUN --mount=type=ssh \
|
||||
--mount=type=cache,target=/root/.cache/pip \
|
||||
pip install -e "libs/kotaemon[all]" \
|
||||
&& pip install -e "libs/ktem" \
|
||||
&& pip install graphrag future \
|
||||
&& pip install "pdfservices-sdk@git+https://github.com/niallcm/pdfservices-python-sdk.git@bump-and-unfreeze-requirements"
|
||||
|
||||
# Clean up
|
||||
RUN apt-get autoremove \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& rm -rf ~/.cache
|
||||
|
||||
CMD ["python", "app.py"]
|
||||
|
||||
# Full version
|
||||
FROM lite AS full
|
||||
|
||||
# Additional dependencies for full version
|
||||
RUN apt-get update -qqy && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
tesseract-ocr \
|
||||
tesseract-ocr-jpn \
|
||||
libsm6 \
|
||||
libxext6 \
|
||||
libreoffice \
|
||||
ffmpeg \
|
||||
libmagic-dev
|
||||
|
||||
# Install torch and torchvision for unstructured
|
||||
RUN --mount=type=ssh \
|
||||
--mount=type=cache,target=/root/.cache/pip \
|
||||
pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
|
||||
|
||||
# Copy contents
|
||||
COPY . /app
|
||||
|
||||
# Install additional pip packages
|
||||
RUN --mount=type=ssh \
|
||||
--mount=type=cache,target=/root/.cache/pip \
|
||||
pip install unstructured[all-docs]
|
||||
|
||||
# Clean up
|
||||
RUN apt-get autoremove \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& rm -rf ~/.cache
|
||||
|
||||
# Download nltk packages as required for unstructured
|
||||
RUN python -c "from unstructured.nlp.tokenize import _download_nltk_packages_if_not_present; _download_nltk_packages_if_not_present()"
|
||||
|
||||
CMD ["python", "app.py"]
|
||||
|
Reference in New Issue
Block a user