ferrumox/fox repository overview

# Linux / macOS
curl -fsSL https://github.com/ferrumox/fox/releases/latest/download/install.sh | sh

# Windows
irm https://raw.githubusercontent.com/ferrumox/fox/main/install.ps1 | iex

# Pull a model and start
fox pull llama3.2
fox serve

# Ask something (OpenAI-compatible)
curl http://localhost:8080/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{"model":"llama3.2","messages":[{"role":"user","content":"Hello!"}],"stream":true}'

# If you already use Ollama — just change the port from 11434 to 8080. That's it.

from openai import OpenAI

client = OpenAI(base_url="http://localhost:8080/v1", api_key="sk-local")

resp = client.chat.completions.create(
    model="llama3.2",
    messages=[{"role": "user", "content": "Say hi in 5 words."}],
)
print(resp.choices[0].message.content)

import OpenAI from "openai";

const openai = new OpenAI({ baseURL: "http://localhost:8080/v1", apiKey: "sk-local" });

const resp = await openai.chat.completions.create({
  model: "llama3.2",
  messages: [{ role: "user", content: "Say hi in 5 words." }],
});
console.log(resp.choices[0].message?.content);

{ "github.copilot.advanced": { "serverUrl": "http://localhost:8080" } }

{
  "models": [{
    "title": "fox (local)",
    "provider": "openai",
    "model": "llama3.2",
    "apiBase": "http://localhost:8080/v1"
  }]
}

curl -fsSL https://github.com/ferrumox/fox/releases/latest/download/install.sh | sh

# Linux x86_64
curl -L https://github.com/ferrumox/fox/releases/latest/download/fox-linux-x86_64 -o fox && chmod +x fox

# macOS Apple Silicon
curl -L https://github.com/ferrumox/fox/releases/latest/download/fox-macos-arm64 -o fox && chmod +x fox

# macOS Intel
curl -L https://github.com/ferrumox/fox/releases/latest/download/fox-macos-x86_64 -o fox && chmod +x fox

irm https://raw.githubusercontent.com/ferrumox/fox/main/install.ps1 | iex

git clone --recurse-submodules https://github.com/ferrumox/fox
cd fox
cargo build --release

docker run -p 8080:8080 \
  -v ~/.cache/ferrumox/models:/root/.cache/ferrumox/models \
  ferrumox/fox serve

# Or with docker compose
docker compose up

# Search HuggingFace for GGUF models
fox search gemma
fox search qwen coder --limit 5

# Pull a model
fox pull llama3.2            # top result, balanced quantization
fox pull gemma3:12b          # specific size
fox pull gemma3:12b-q4       # specific quantization
fox pull bartowski/gemma-3-12b-it-GGUF  # specific HF repo

# Start the server
fox serve                    # lazy loading — no model needed upfront
fox serve --max-models 3     # keep up to 3 models loaded simultaneously

# Interactive REPL
fox run
fox run "Explain ownership in Rust"  # single-shot

# Manage models
fox list                     # list downloaded models
fox show llama3.2            # model info: architecture, quantization, size
fox ps                       # list currently loaded models

# Manage aliases
fox alias set llama3 Llama-3.2-3B-Instruct-Q4_K_M
fox alias list

# Benchmark
fox bench llama3.2
fox bench llama3.2 --runs 10

port = 8080
max_models = 3
keep_alive_secs = 300
system_prompt = "You are a helpful assistant."

[aliases]
"llama3"   = "Llama-3.2-3B-Instruct-Q4_K_M"
"mistral"  = "Mistral-7B-Instruct-v0.3-Q4_K_M"

# Compare fox vs Ollama side by side
./target/release/fox-bench \
  --url http://localhost:8080 \
  --compare-url http://localhost:11434 \
  --model llama3.2

# JSON output for CI
./target/release/fox-bench \
  --url http://localhost:8080 \
  --compare-url http://localhost:11434 \
  --model llama3.2 \
  --output json

# Reproducible benchmark (saves to benches/results.md)
./scripts/benchmark.sh llama3.2 4 50

┌─────────────────┬──────────────┬──────────────┬──────────┐
│ Metric          │     fox      │    ollama    │ Δ        │
├─────────────────┼──────────────┼──────────────┼──────────┤
│ TTFT P50        │          87ms│         310ms│ +72%     │
│ TTFT P95        │         134ms│         480ms│ +72%     │
│ Latency P50     │         412ms│         890ms│ +54%     │
│ Latency P95     │         823ms│        1740ms│ +53%     │
│ Latency P99     │        1204ms│        2600ms│ +54%     │
│ Throughput      │    312.4 t/s │    148.1 t/s │ +111%    │
└─────────────────┴──────────────┴──────────────┴──────────┘

fox/
├── src/
│   ├── main.rs              # Entry point, config, signal handling
│   ├── metrics.rs           # Prometheus metrics registry
│   ├── model_registry.rs    # Multi-model registry with LRU eviction
│   ├── config.rs            # Config file loading
│   ├── registry.rs          # Model discovery helpers
│   ├── api/                 # REST API (OpenAI + Ollama compat)
│   │   ├── router.rs        # Axum router setup
│   │   ├── types.rs         # Request/response types
│   │   ├── auth.rs          # API key middleware
│   │   ├── error.rs         # Unified error types
│   │   ├── pull_handler.rs  # POST /api/pull SSE streaming
│   │   ├── v1/              # OpenAI-compat handlers
│   │   │   ├── chat.rs
│   │   │   ├── completions.rs
│   │   │   ├── embeddings.rs
│   │   │   └── models.rs
│   │   ├── ollama/          # Ollama-compat handlers
│   │   │   ├── chat.rs
│   │   │   ├── generate.rs
│   │   │   ├── embed.rs
│   │   │   └── management.rs
│   │   └── shared/          # Shared inference + streaming helpers
│   │       ├── inference.rs
│   │       ├── streaming.rs
│   │       └── digest.rs
│   ├── scheduler/           # Continuous batching + prefix cache
│   ├── kv_cache/            # PageTable, ref-counted block manager
│   ├── engine/              # Inference engine, sampling, output filtering
│   └── cli/                 # Subcommands: serve, run, pull, list, show, ps
├── examples/
│   ├── curl.sh              # curl examples for all API routes
│   ├── langchain.py         # LangChain integration
│   └── openwebui.md         # Open WebUI setup guide
├── scripts/
│   └── benchmark.sh         # Reproducible benchmark vs Ollama
├── benches/
│   └── results.md           # Benchmark results (generated)
├── vendor/llama.cpp/        # Git submodule
├── Dockerfile
├── docker-compose.yml
├── fox.service              # systemd unit
├── install.sh               # One-liner installer
├── Makefile
├── CHANGELOG.md
└── Cargo.toml

make build           Compile release binaries (fox + fox-bench)
make run             Build and start the server
make dev             Start with RUST_LOG=debug
make test            Run unit tests
make check           Fast type-check (cargo check)
make bench           Run fox-bench against a running server
make docker          Build Docker image
make docker-run      Start via docker compose
make install-rust    Install Rust toolchain
make download-model  Download default model (Llama-3.2-3B Q4_K_M)

FOX_SKIP_LLAMA=1 cargo test --all