Public/kotaemon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feat/local endpoint llm (#148)

Browse Source 
* serve local model in a different process from the app
---------

Co-authored-by: albert <albert@cinnamon.is>
Co-authored-by: trducng <trungduc1992@gmail.com>
This commit is contained in:
ian_Cin
2024-03-15 16:17:33 +07:00
committed by GitHub
parent 2950e6ed02
commit df12dec732
20 changed files with 675 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
scripts/run_linux.sh
View File

@@ -12,23 +12,23 @@ function install_miniconda() {
# Miniconda installer is limited to two main architectures: x86_64 and arm64
local sys_arch=$(uname -m)
case "${sys_arch}" in
x86_64*) sys_arch="x86_64";;
arm64*) sys_arch="aarch64";;
aarch64*) sys_arch="aarch64";;
*) {
echo "Unknown system architecture: ${sys_arch}! This script runs only on x86_64 or arm64"
exit 1
};;
x86_64*) sys_arch="x86_64" ;;
arm64*) sys_arch="aarch64" ;;
aarch64*) sys_arch="aarch64" ;;
*) {
echo "Unknown system architecture: ${sys_arch}! This script runs only on x86_64 or arm64"
exit 1
} ;;
esac
# if miniconda has not been installed, download and install it
if ! "${conda_root}/bin/conda" --version &>/dev/null ; then
if ! "${conda_root}/bin/conda" --version &>/dev/null; then
if [ ! -d "$install_dir/miniconda_installer.sh" ]; then
echo "Downloading Miniconda from $miniconda_url"
local miniconda_url="https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-${sys_arch}.sh"
mkdir -p "$install_dir"
curl -Lk "$miniconda_url" > "$install_dir/miniconda_installer.sh"
curl -Lk "$miniconda_url" >"$install_dir/miniconda_installer.sh"
fi
echo "Installing Miniconda to $conda_root"
@@ -64,7 +64,7 @@ function create_conda_env() {
function activate_conda_env() {
# deactivate the current env(s) to avoid conflicts
{ conda deactivate && conda deactivate && conda deactivate; } 2> /dev/null
{ conda deactivate && conda deactivate && conda deactivate; } 2>/dev/null
# check if conda env is broken (because of interruption during creation)
if [ ! -f "$env_dir/bin/python" ]; then
@@ -80,7 +80,7 @@ function activate_conda_env() {
echo "Activate conda environment at $CONDA_PREFIX"
}
function deactivate_conda_env(){
function deactivate_conda_env() {
# Conda deactivate if we are in the right env
if [ "$CONDA_PREFIX" == "$env_dir" ]; then
conda deactivate
@@ -89,7 +89,7 @@ function deactivate_conda_env(){
}
function install_dependencies() {
if pip list 2> /dev/null | grep -q "kotaemon"; then
if pip list 2>/dev/null | grep -q "kotaemon"; then
echo "Requirements are already installed"
else
local kotaemon_root="$(pwd)/libs/kotaemon/.[dev]"
@@ -101,7 +101,7 @@ function install_dependencies() {
echo "" && echo "Install ktem's requirements"
python -m pip install -e "$ktem_root"
if ! pip list 2> /dev/null | grep -q "kotaemon"; then
if ! pip list 2>/dev/null | grep -q "kotaemon"; then
echo "Installation failed. You may need to run the installer again."
deactivate_conda_env
exit 1
@@ -123,6 +123,10 @@ function install_dependencies() {
fi
}
function setup_local_model() {
python $(pwd)/scripts/serve_local.py
}
function launch_ui() {
gradio $(pwd)/libs/ktem/launch.py || {
echo "" && echo "Will exit now..."
@@ -159,6 +163,9 @@ activate_conda_env
print_highlight "Install requirements"
install_dependencies
print_highlight "Setting up a local model"
setup_local_model
print_highlight "Launching web UI. Please wait..."
launch_ui