TheCrazyInsanity/nixos-configv3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

i have to commit in order to get the patches to work

Browse Source
This commit is contained in:
TheCrazyInsanity
2026-04-05 18:08:32 -04:00
parent 00ebec0062
commit 77c77a298b
7 changed files with 331 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
machines/homepc/configuration.nix
View File

@@ -91,7 +91,10 @@
environment.etc."llama-swap/config.yaml".text = ''
models:
"Qwen3.5-35B-A3B-GGUF":
cmd: llama-server --port ''${PORT} -hf unsloth/Qwen3.5-35B-A3B-GGUF:UD-IQ4_XS --ctx-size 128000 --temp 0.6 --top-p 0.95 --top-k 20 --min-p 0.00 -np 1 --fit-target 256 --ubatch-size 1024 -fa on --slots --slot-save-path /home/laythe/llamapcache --jinja -kvu --no-mmproj --swa-checkpoints 32 --no-kv-offload
cmd: llama-server --port ''${PORT} -hf mudler/Qwen3.5-35B-A3B-APEX-GGUF:Mini --ctx-size 128000 --temp 0.6 --top-p 0.95 --top-k 20 --min-p 0.00 -np 1 --fit-target 256 --ubatch-size 1024 -fa on --slots --slot-save-path /home/laythe/llamapcache --jinja -kvu --no-mmproj --swa-checkpoints 32 --no-kv-offload
ttl: 2400
"Gemma4-26B-A4B":
cmd: llama-server --port ''${PORT} -hf unsloth/gemma-4-26B-A4B-it-GGUF:UD-IQ4_XS --ctx-size 128000 --temp 1.0 --top-p 0.95 --top-k 64 --min-p 0.00 -np 1 --fit-target 256 --ubatch-size 1024 -fa on --slots --slot-save-path /home/laythe/llamapcache --jinja -kvu --no-mmproj --swa-checkpoints 32 --no-kv-offload
ttl: 2400
"Qwen3-1.7B-GGUF":
cmd: llama-server --port ''${PORT} -hf unsloth/Qwen3-1.7B-GGUF:Q4_K_S --ctx-size 32768 --temp 0.6 --top-p 0.95 --top-k 20 --min-p 0.00 -fa on --jinja -kvu