seemeai/llama-cpp repository overview

#!/bin/bash

data_dir=/data_folder_with_all_models

# Llama.cpp release container: https://hub.docker.com/repository/docker/seemeai/llama-cpp/general
image=seemeai/llama-cpp:b1632-cuda12.1

# do not change
llama_dir=/code/llama.cpp

# The name of the model. Should map to the folder with weights and config within the `data_dir`
model=mixtral-7b

# layers you want to offload on the GPU
ngl=40

bin_folder=""

# for legacy builds uncomment below
#bin_folder=/build/bin

# select desired quantizations 
quantizations=( q2_k q3_k_l q3_k_m q4_0 q4_1 q4_k_s q4_k_s q5_0 q5_1 q5_k_m q5_k_s q6_k q8_0 )

echo "start conversion"

docker run -it --shm-size=64G -v $data_dir:/models --entrypoint python $image $llama_dir/convert.py /models/llama-data/$model/ --vocab-dir /models

echo "model converted"
echo "starting quantizaion"

for quantize in "${quantizations[@]}"
do
        echo $quantize

        docker run -it --shm-size=64G -v $data_dir:/models --entrypoint $llama_dir$bin_folder/quantize $image /models/llama-data/$model/ggml-model-f16.gguf /models/llama-data/$model/ggml-model-$quantize.gguf $quantize

        echo "quant done"

        docker run -it --shm-size=64G -v $data_dir:/models --entrypoint $llama_dir$bin_folder/main $image -m /models/llama-data/$model/ggml-model-$quantize.gguf -p "Building a website can be done in 10 simple steps:\nStep 1:" -n 400 -e -t 16 # -ngl $ngl

done
echo "done"