Benchmark Any LLM
With Real Data

# Evaluate a single model
llm-eval run --model gpt-4o-mini --benchmark mmlu --samples 100

╭──────────────────────────────────────╮
│  Evaluation: gpt-4o-mini             │
├──────────────────┬───────────────────┤
│ Accuracy         │ 78.00%            │
│ Avg Latency      │ 432 ms            │
│ P95 Latency      │ 1240 ms           │
│ Total Cost       │ $0.0023           │
│ Hallucination    │ 2.40%             │
│ Reasoning Score  │ 7.2 / 10          │
╰──────────────────┴───────────────────╯

# Compare 3 models
llm-eval compare \
  --models gpt-4o-mini \
  --models claude-3-haiku-20240307 \
  --models gemini/gemini-1.5-flash \
  --benchmark mmlu --samples 50

# Export results
llm-eval export --format csv --output results.csv

# Generate PDF report
llm-eval report --run-ids a3f92c1b --output ./reports/

# Start dashboard
llm-eval dashboard --port 8501

import asyncio
from llm_eval.core.evaluator import LLMEvaluator, EvaluationConfig
from llm_eval.benchmarks.mmlu import MMLUBenchmark

async def main():
    evaluator = LLMEvaluator()
    samples   = MMLUBenchmark().load(num_samples=100)

    config = EvaluationConfig(
        model="gpt-4o-mini",
        benchmark="mmlu",
        num_samples=100,
        temperature=0.0,
        concurrency=10,
    )

    result = await evaluator.evaluate(config, samples)
    print(f"Accuracy:   {result.accuracy:.2%}")
    print(f"P95 Latency:{result.p95_latency_ms:.0f}ms")
    print(f"Total Cost: ${result.total_cost_usd:.4f}")
    print(f"Hallucin.:  {result.hallucination_rate:.2%}")
    print(f"Reasoning:  {result.avg_reasoning_score:.1f}/10")

asyncio.run(main())

import asyncio
from llm_eval.core.evaluator import LLMEvaluator, EvaluationConfig
from llm_eval.benchmarks.mmlu import MMLUBenchmark

async def compare():
    evaluator = LLMEvaluator()
    samples   = MMLUBenchmark().load(num_samples=50)

    configs = [
        EvaluationConfig(model="gpt-4o-mini",            benchmark="mmlu", num_samples=50),
        EvaluationConfig(model="claude-3-haiku-20240307", benchmark="mmlu", num_samples=50),
        EvaluationConfig(model="gemini/gemini-1.5-flash",benchmark="mmlu", num_samples=50),
    ]

    results = await evaluator.evaluate_multiple(configs, samples)

    for r in sorted(results, key=lambda x: x.accuracy, reverse=True):
        print(f"{r.model:<35} {r.accuracy:>7.1%} {r.avg_latency_ms:>7.0f}ms ${r.cost_per_1k_tokens:.4f}")

asyncio.run(compare())

# Start server
uvicorn llm_eval.api.main:app --reload --port 8000

# Evaluate
curl -X POST http://localhost:8000/evaluate \
  -H "Content-Type: application/json" \
  -d '{"model":"gpt-4o-mini","benchmark":"mmlu","num_samples":50}'

{
  "run_id": "a3f92c1b",
  "accuracy": 0.78,
  "avg_latency_ms": 432.1,
  "p95_latency_ms": 1240.0,
  "total_cost_usd": 0.0012,
  "hallucination_rate": 0.024,
  "avg_reasoning_score": 7.2
}

# Generate PDF report
curl -X POST http://localhost:8000/report \
  -d '{"run_ids":["a3f92c1b"]}' -o report.pdf

# Clone and configure
git clone https://github.com/vignesh2027/LLM-Evaluation-Framework.git
cd LLM-Evaluation-Framework && cp .env.example .env

# Start API + Dashboard
docker-compose up -d

# API:       http://localhost:8000/docs
# Dashboard: http://localhost:8501

# View logs
docker-compose logs -f