llama.cpp

Voir aussi

ggml-org/llama.cpp

Téléchargement des modéles

Voir aussi

https://huggingface.co/ggml-org

Les modéles 7B fonctionne sur la 9070XT (16GB RAM) mais seulement 1 par 1, pour avoir 1 coder + 1 instruct j’utilise les 3B.

curl -JOL --output-dir ~/.local/share/llama/models https://huggingface.co/ggml-org/Qwen2.5-Coder-3B-Q8_0-GGUF/resolve/main/qwen2.5-coder-3b-q8_0.gguf
curl -JOL --output-dir ~/.local/share/llama/models https://huggingface.co/ggml-org/Qwen2.5-Coder-3B-Instruct-Q8_0-GGUF/resolve/main/qwen2.5-coder-3b-instruct-q8_0.gguf
#curl -JOL --output-dir ~/.local/share/llama/models https://huggingface.co/ggml-org/Qwen2.5-Coder-7B-Q8_0-GGUF/resolve/main/qwen2.5-coder-7b-q8_0.gguf
#curl -JOL  --output-dir ~/.local/share/llama/models https://huggingface.co/ggml-org/Qwen2.5-Coder-7B-Instruct-Q8_0-GGUF/resolve/main/qwen2.5-coder-7b-instruct-q8_0.gguf

Service

~/.config/systemd/user/llama.service

[Unit]
Description=LLM inference in C/C++

[Service]
Environment=LLAMA_ARG_PORT=11434
Environment=LLAMA_ARG_HOST=0.0.0.0
Environment=LLAMA_ARG_N_PARALLEL=3
Environment=LLAMA_ARG_LOG_COLORS=on
Environment=LLAMA_ARG_MODELS_AUTOLOAD=on
Environment=LLAMA_ARG_MODELS_DIR=/models
ExecStart=podman run --name llama --rm --device /dev/kfd --device /dev/dri -v %D/llama/models:/models -p ${LLAMA_ARG_PORT}:${LLAMA_ARG_PORT} --env 'LLAMA_ARG_*'  ghcr.io/ggml-org/llama.cpp:server-rocm --sleep-idle-seconds 60
ExecStop=podman stop llama

systemctl --user daemon-reload
systemctl --user start llama.service