From cc87aaa78358045c2f0c52bb9b2e560c9940aa21 Mon Sep 17 00:00:00 2001 From: "Albert (Quang)" Date: Wed, 6 Mar 2024 10:59:30 +0700 Subject: [PATCH] Add one-click installers for Linux, Windows, and MacOS (#146) * feat: Add installers for linux, windows, and macos * docs: Update README * pre-commit fix styles * Update installers and README * Remove env vars check and fix paths * Update installers: * Remove start.py and move install and launch part back to .sh/.bat * Add conda deactivate * Make messages more informative * Improve kotaemon based on insights from projects (#147) - Include static files in the package. - More reliable information panel. Faster & not breaking randomly. - Add directory upload. - Enable zip file to upload. - Allow setting endpoint for the OCR reader using environment variable. * feat: Add installers for linux, windows, and macos * docs: Update README * pre-commit fix styles * Update installers and README * Remove env vars check and fix paths * Update installers: * Remove start.py and move install and launch part back to .sh/.bat * Add conda deactivate * Make messages more informative * Make macOS installer runable and improve Windows, Linux installers * Minor fix macos commands * installation should pause before exit * Update Windows installer: add a new label to exit function with error * put install_dir to .gitignore * chore: Add comments to clarify the 'end' labels --------- Co-authored-by: Duc Nguyen (john) Co-authored-by: ian --- .gitignore | 3 + README.md | 20 +++-- scripts/run_linux.sh | 167 +++++++++++++++++++++++++++++++++++++++ scripts/run_macos.sh | 168 ++++++++++++++++++++++++++++++++++++++++ scripts/run_windows.bat | 152 ++++++++++++++++++++++++++++++++++++ 5 files changed, 505 insertions(+), 5 deletions(-) create mode 100755 scripts/run_linux.sh create mode 100644 scripts/run_macos.sh create mode 100644 scripts/run_windows.bat diff --git a/.gitignore b/.gitignore index 1948399..9e9e64c 100644 --- a/.gitignore +++ b/.gitignore @@ -464,3 +464,6 @@ S.gpg-agent* .vscode/settings.json examples/example1/assets storage/* + +# Conda and env storages +install_dir/ diff --git a/README.md b/README.md index 5389cff..b364055 100644 --- a/README.md +++ b/README.md @@ -7,13 +7,23 @@ projects. ## Install -```shell -pip install kotaemon@git+ssh://git@github.com/Cinnamon/kotaemon.git -``` +### Easy install -## Contribute +1. Clone the repository. +2. Navigate to the `scripts` folder and start an installer that matches your OS: + - Linux: `run_linux.sh` + - Windows: `run_windows.bat` + - macOS: `run_macos.sh` +3. After the installation, the installer will ask to launch the ktem's UI, answer to continue. +4. If launched, the application will be available at `http://localhost:7860/`. Let's start exploring! -### Setup +Here is the setup and update strategy: + +- **Run `run_*` script**: This setup environment, including downloading Miniconda (in case Conda is not available in your machine) and installing necessary dependencies in `install_dir` folder. +- **Launch the UI**: To launch the ktem's UI after initial setup or any changes, simply run `run_*` script again. +- **Reinstall dependencies**: Simply delete the `install_dir/env` folder and run `run_*` script. The script will recreate the folder with fresh dependencies. + +### Manual install - Create conda environment (suggest 3.10) diff --git a/scripts/run_linux.sh b/scripts/run_linux.sh new file mode 100755 index 0000000..723f90e --- /dev/null +++ b/scripts/run_linux.sh @@ -0,0 +1,167 @@ +#!/bin/bash + +# functions for better code organization +function check_path_for_spaces() { + if [[ $PWD =~ \ ]]; then + echo "The current workdir has whitespace which can lead to unintended behaviour. Please modify your path and continue later." + exit 1 + fi +} + +function install_miniconda() { + # Miniconda installer is limited to two main architectures: x86_64 and arm64 + local sys_arch=$(uname -m) + case "${sys_arch}" in + x86_64*) sys_arch="x86_64";; + arm64*) sys_arch="aarch64";; + aarch64*) sys_arch="aarch64";; + *) { + echo "Unknown system architecture: ${sys_arch}! This script runs only on x86_64 or arm64" + exit 1 + };; + esac + + # if miniconda has not been installed, download and install it + if ! "${conda_root}/bin/conda" --version &>/dev/null ; then + if [ ! -d "$install_dir/miniconda_installer.sh" ]; then + echo "Downloading Miniconda from $miniconda_url" + local miniconda_url="https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-${sys_arch}.sh" + + mkdir -p "$install_dir" + curl -Lk "$miniconda_url" > "$install_dir/miniconda_installer.sh" + fi + + echo "Installing Miniconda to $conda_root" + chmod u+x "$install_dir/miniconda_installer.sh" + bash "$install_dir/miniconda_installer.sh" -b -p "$conda_root" + rm -rf "$install_dir/miniconda_installer.sh" + fi + echo "Miniconda is installed at $conda_root" + + # test conda + echo "Conda version: " + "$conda_root/bin/conda" --version || { + echo "Conda not found. Will exit now..." + exit 1 + } +} + +function create_conda_env() { + local python_version="${1}" + + if [ ! -d "${env_dir}" ]; then + echo "Creating conda environment with python=$python_version in $env_dir" + "${conda_root}/bin/conda" create -y -k --prefix "$env_dir" python="$python_version" || { + echo "Failed to create conda environment." + echo "Will delete the ${env_dir} (if exist) and exit now..." + rm -rf $env_dir + exit 1 + } + else + echo "Conda environment exists at $env_dir" + fi +} + +function activate_conda_env() { + # deactivate the current env(s) to avoid conflicts + { conda deactivate && conda deactivate && conda deactivate; } 2> /dev/null + + # check if conda env is broken (because of interruption during creation) + if [ ! -f "$env_dir/bin/python" ]; then + echo "Conda environment appears to be broken. You may need to remove $env_dir and run the installer again." + exit 1 + fi + + source "$conda_root/etc/profile.d/conda.sh" # conda init + conda activate "$env_dir" || { + echo "Failed to activate environment. Please remove $env_dir and run the installer again" + exit 1 + } + echo "Activate conda environment at $CONDA_PREFIX" +} + +function deactivate_conda_env(){ + # Conda deactivate if we are in the right env + if [ "$CONDA_PREFIX" == "$env_dir" ]; then + conda deactivate + echo "Deactivate conda environment at $env_dir" + fi +} + +function install_dependencies() { + if pip list 2> /dev/null | grep -q "kotaemon"; then + echo "Requirements are already installed" + else + local kotaemon_root="$(pwd)/libs/kotaemon/.[dev]" + local ktem_root="$(pwd)/libs/ktem/" + + echo "" && echo "Install kotaemon's requirements" + python -m pip install -e "$kotaemon_root" + + echo "" && echo "Install ktem's requirements" + python -m pip install -e "$ktem_root" + + if ! pip list 2> /dev/null | grep -q "kotaemon"; then + echo "Installation failed. You may need to run the installer again." + deactivate_conda_env + exit 1 + else + print_highlight "Install finished successfully. Clear cache..." + conda clean --all -y + python -m pip cache purge + + print_highlight "Do you want to launch the web UI? [Y/N]" + read -p "Input> " launch + local launch=${launch,,} + if [[ "$launch" != "yes" && "$launch" != "y" && "$launch" != "true" ]]; then + echo "Will exit now..." + deactivate_conda_env + echo "Please run the installer again to launch the UI." + exit 0 + fi + fi + fi +} + +function launch_ui() { + gradio $(pwd)/libs/ktem/launch.py || { + echo "" && echo "Will exit now..." + exit 1 + } +} + +function print_highlight() { + local message="${1}" + echo "" && echo "******************************************************" + echo $message + echo "******************************************************" && echo "" +} + +# Main script execution + +# move two levels up from the dir where this script resides +cd "$(dirname "${BASH_SOURCE[0]}")" && cd .. + +install_dir="$(pwd)/install_dir" +conda_root="${install_dir}/conda" +env_dir="${install_dir}/env" +python_version="3.10" + +check_path_for_spaces + +print_highlight "Setup Anaconda/Miniconda" +install_miniconda + +print_highlight "Create and Activate conda environment" +create_conda_env "$python_version" +activate_conda_env + +print_highlight "Install requirements" +install_dependencies + +print_highlight "Launching web UI. Please wait..." +launch_ui + +deactivate_conda_env + +read -p "Press enter to continue" diff --git a/scripts/run_macos.sh b/scripts/run_macos.sh new file mode 100644 index 0000000..ba57339 --- /dev/null +++ b/scripts/run_macos.sh @@ -0,0 +1,168 @@ +#!/bin/bash + +# Functions for better code organization +function check_path_for_spaces() { + if [[ $PWD =~ \ ]]; then + echo "The current workdir has whitespace which can lead to unintended behaviour. Please modify your path and continue later." + exit 1 + fi +} + +function install_miniconda() { + # Miniconda installer is limited to two main architectures: x86_64 and arm64 + local sys_arch=$(uname -m) + case "${sys_arch}" in + x86_64*) sys_arch="x86_64";; + arm64*) sys_arch="arm64";; + *) { + echo "Unknown system architecture: ${sys_arch}! This script runs only on x86_64 or arm64" + exit 1 + };; + esac + + # if miniconda has not been installed, download and install it + if ! "${conda_root}/bin/conda" --version &>/dev/null ; then + if [ ! -d "$install_dir/miniconda_installer.sh" ]; then + local miniconda_url="https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-${sys_arch}.sh" + echo "Downloading Miniconda from $miniconda_url" + + mkdir -p "$install_dir" + curl -Lk "$miniconda_url" > "$install_dir/miniconda_installer.sh" + fi + + echo "Installing Miniconda to $conda_root" + chmod u+x "$install_dir/miniconda_installer.sh" + bash "$install_dir/miniconda_installer.sh" -b -p "$conda_root" + rm -rf "$install_dir/miniconda_installer.sh" + fi + echo "Miniconda is installed at $conda_root" + + # test conda + echo "Conda version: " + "$conda_root/bin/conda" --version || { + echo "Conda not found. Will exit now..." + exit 1 + } +} + +function create_conda_env() { + local python_version="${1}" + + if [ ! -d "${env_dir}" ]; then + echo "Creating conda environment with python=$python_version in $env_dir" + "${conda_root}/bin/conda" create -y -k --prefix "$env_dir" python="$python_version" || { + echo "Failed to create conda environment." + echo "Will delete the ${env_dir} (if exist) and exit now..." + rm -rf $env_dir + exit 1 + } + else + echo "Conda environment exists at $env_dir" + fi +} + +function activate_conda_env() { + # deactivate the current env(s) to avoid conflicts + { conda deactivate && conda deactivate && conda deactivate; } 2> /dev/null + + # check if conda env is broken (because of interruption during creation) + if [ ! -f "$env_dir/bin/python" ]; then + echo "Conda environment appears to be broken. You may need to remove $env_dir and run the installer again." + exit 1 + fi + + source "$conda_root/etc/profile.d/conda.sh" # conda init + conda activate "$env_dir" || { + echo "Failed to activate environment. Please remove $env_dir and run the installer again." + exit 1 + } + echo "Activate conda environment at $CONDA_PREFIX" +} + +function deactivate_conda_env(){ + # Conda deactivate if we are in the right env + if [[ "$CONDA_PREFIX" == "$env_dir" ]]; then + conda deactivate + echo "Deactivate conda environment at $env_dir" + fi +} + +function install_dependencies() { + # check if the env is already setup by finding 'kotaemon' in 'pip list' + if pip list 2> /dev/null | grep -q "kotaemon"; then + echo "Requirements are already installed" + else + local kotaemon_root="$(pwd)/libs/kotaemon/.[dev]" + local ktem_root="$(pwd)/libs/ktem/" + + echo "" && echo "Install kotaemon's requirements" + python -m pip install -e "$kotaemon_root" + + echo "" && echo "Install ktem's requirements" + python -m pip install -e "$ktem_root" + + if ! pip list 2> /dev/null | grep -q "kotaemon"; then + echo "Installation failed. You may need to run the installer again." + deactivate_conda_env + exit 1 + else + print_highlight "Install finished successfully. Clear cache..." + "$conda_root/bin/conda" clean --all -y + python -m pip cache purge + + print_highlight "Do you want to launch the web UI? [Y/N]" + read -p "Input (yes/no)> " launch + # Convert user input to lowercase + local launch=${launch:l} + if [[ "$launch" != "yes" && "$launch" != "y" && "$launch" != "true" ]]; then + echo "Will exit now..." + deactivate_conda_env + echo "Please run the installer again to launch the UI." + exit 0 + fi + fi + fi +} + +function launch_ui() { + gradio $(pwd)/libs/ktem/launch.py || { + echo "" && echo "Will exit now..." + exit 1 + } +} + +function print_highlight() { + local message="${1}" + echo "" && echo "******************************************************" + echo $message + echo "******************************************************" && echo "" +} + +# Main script execution + +# move two levels up from the dir where this script resides +cd "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" && cd .. + +install_dir="$(pwd)/install_dir" +conda_root="${install_dir}/conda" +env_dir="${install_dir}/env" +python_version="3.10" + +check_path_for_spaces + +print_highlight "Setup Anaconda/Miniconda" +install_miniconda + +print_highlight "Create and Activate conda environment" +create_conda_env "$python_version" +activate_conda_env + +print_highlight "Install requirements" +install_dependencies + +print_highlight "Launching web UI. Please wait..." +launch_ui + +deactivate_conda_env + +read -p "Press enter to continue" diff --git a/scripts/run_windows.bat b/scripts/run_windows.bat new file mode 100644 index 0000000..257e232 --- /dev/null +++ b/scripts/run_windows.bat @@ -0,0 +1,152 @@ +@ECHO off + +:: Main script execution +CD /D "%~dp0\.." + +SET install_dir=%CD%\install_dir +SET conda_root=%install_dir%\conda +SET env_dir=%install_dir%\env +SET python_version=3.10 +SET miniconda_download_url=https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe + +ECHO %CD%| FINDSTR /C:" " >nul 2>&1 +IF %ERRORLEVEL% EQU 0 ( + ECHO The current workdir has whitespace which can lead to unintended behaviour. Please modify your path and continue later. + GOTO :end +) +CALL :print_highlight "Setup Anaconda/Miniconda" +CALL :download_and_install_miniconda +:: check if function run fail, then exit the script +IF ERRORLEVEL 1 GOTO :end + +CALL :print_highlight "Create and Activate conda environment" +CALL :create_conda_environment +IF ERRORLEVEL 1 GOTO :end + +CALL :activate_environment +IF ERRORLEVEL 1 GOTO :end + +CALL :print_highlight "Install requirements" +CALL :install_dependencies +IF ERRORLEVEL 1 GOTO :end + +CALL :print_highlight "Launching web UI. Please wait..." +CALL :launch_ui + +CALL :deactivate_environment +GOTO :end_success + + +:download_and_install_miniconda +IF NOT EXIST "%install_dir%" ( MKDIR "%install_dir%" ) + +:: If conda has been installed at the %conda_root%, don't need to reinstall it +CALL "%conda_root%\_conda.exe" --version >nul 2>&1 +IF %ERRORLEVEL% NEQ 0 ( + IF NOT EXIST "%install_dir%\miniconda_installer.exe" ( + ECHO Downloading Miniconda from %miniconda_download_url% + CALL curl -Lk "%miniconda_download_url%" -o "%install_dir%\miniconda_installer.exe" || ( + ECHO. && ECHO Failed to download Miniconda. Aborting... + GOTO :exit_func_with_error + ) + ) + ECHO Installing Miniconda to %conda_root% + START /wait "" "%install_dir%\miniconda_installer.exe" /InstallationType=JustMe /NoShortcuts=1 /AddToPath=0 /RegisterPython=0 /NoRegistry=1 /S /D=%conda_root% + DEL "%install_dir%\miniconda_installer.exe" +) +ECHO Conda is installed at %conda_root% + +:: recheck conda +ECHO Conda version: +CALL "%conda_root%\_conda.exe" --version || ( ECHO. && ECHO Conda not found. Aborting... && GOTO :exit_func_with_error ) + +GOTO :eof + +:create_conda_environment +:: Create new conda environment if it doesn't exist +IF NOT EXIST %env_dir% ( + ECHO Creating conda environment with python=%python_version% in %env_dir% + :: Create conda environment. If the interruption happens, rollback and remove the env_dir + CALL "%conda_root%\_conda.exe" create --no-shortcuts -y -k --prefix %env_dir% python=%python_version% || ( + ECHO. && ECHO Failed to create conda environment. Will delete the %env_dir% and abort now... + RMDIR /s /q %env_dir% + GOTO :exit_func_with_error + ) + ECHO Conda environment created successfully +) ELSE ( + ECHO Conda environment exists at %env_dir% +) +GOTO :eof + +:activate_environment +:: deactivate existing conda env(s) to avoid conflicts +( CALL conda deactivate && CALL conda deactivate && CALL conda deactivate ) 2> nul + +CALL "%env_dir%\python.exe" --version >nul 2>&1 || ( + ECHO The environment appears to be broken. You may need to remove %env_dir% and run the installer again. + GOTO :exit_func_with_error +) + +CALL "%conda_root%\condabin\conda.bat" activate %env_dir% || ( + ECHO Failed to activate environment. You may need to remove %env_dir% and run the installer again. + GOTO :exit_func_with_error +) +ECHO Activate conda environment at %env_dir% + +GOTO :eof + +:deactivate_environment +:: Conda deactivate if we are in the right env +IF "%CONDA_PREFIX%" == "%env_dir%" ( + CALL "%conda_root%\condabin\conda.bat" deactivate + ECHO Deactivate conda environment at %env_dir% +) +GOTO :eof + +:install_dependencies +pip list | findstr /C:"kotaemon" >NUL 2>&1 +IF %ERRORLEVEL% == 0 ( + ECHO Dependencies are already installed +) ELSE ( + ECHO Install kotaemon's requirements + CALL python -m pip install -e "%CD%\libs\kotaemon\.[dev]" + + ECHO Install ktem's requirements + CALL python -m pip install -e "%CD%\libs\ktem" + + ( CALL pip list | findstr /C:"kotaemon" >NUL 2>&1 ) || ( + ECHO. && ECHO Installation failed. You may need to run the installer again. + CALL :deactivate_environment + GOTO :exit_func_with_error + ) + + CALL :print_highlight "Install successfully. Clear cache..." + CALL "%conda_root%\condabin\conda.bat" clean --all -y + CALL python -m pip cache purge +) +GOTO :eof + +:launch_ui +CALL gradio "%CD%\libs\ktem\launch.py" || ( ECHO. && ECHO Will exit now... && GOTO :exit_func_with_error ) +GOTO :eof + +:print_highlight +ECHO. && ECHO ****************************************************** +ECHO %~1 +ECHO ****************************************************** && ECHO. +GOTO :eof + +:exit_func_with_error +:: Called inside functions when error happens, then back to the main routine with error code 1 +EXIT /B 1 + +:end_success +:: Exit the script main routine with error code 0 (success) +ECHO Script completed successfully. +PAUSE +EXIT /B 0 + +:end +:: Exit the script main routine with error code 1 (fail) +PAUSE +EXIT /B 1