Quick Start Guide¶

from llcuda.api.multigpu import detect_gpus

gpus = detect_gpus()
print(f"✓ Detected {len(gpus)} GPUs")
for gpu in gpus:
    print(f"  GPU {gpu.id}: {gpu.name} ({gpu.memory_total_gb:.1f} GB)")

from llcuda.server import ServerManager, ServerConfig

config = ServerConfig(
    model_path="model.gguf",
    n_gpu_layers=99,
    flash_attn=True,
)

server = ServerManager()
server.start_with_config(config)
server.wait_until_ready()

print("✓ Server running at http://localhost:8080")

from llcuda.api import LlamaCppClient

client = LlamaCppClient("http://localhost:8080")
response = client.chat.completions.create(
    messages=[{"role": "user", "content": "Hello!"}],
    max_tokens=100
)

print(response.choices[0].message.content)