Benchmark Any LLM
Like a Scientist

# Run a single model evaluation (100 MMLU samples)
llm-eval run --model gpt-4o-mini --benchmark mmlu --samples 100

╭──────────────────────────────────────╮
│  Evaluation: gpt-4o-mini             │
├──────────────────┬───────────────────┤
│ Accuracy         │ 78.00%            │
│ Avg Latency      │ 432 ms            │
│ P95 Latency      │ 1240 ms           │
│ Total Cost       │ $0.0023           │
│ Cost / 1K Tokens │ $0.0015           │
│ Hallucination    │ 2.40%             │
│ Reasoning Score  │ 7.2 / 10          │
│ Samples          │ 100               │
│ Run ID           │ a3f92c1b          │
╰──────────────────┴───────────────────╯

# Compare 3 models
llm-eval compare \
  --models gpt-4o-mini \
  --models claude-3-haiku-20240307 \
  --models gemini/gemini-1.5-flash \
  --benchmark mmlu --samples 50

# Show all stored results
llm-eval results --benchmark mmlu --limit 20

# Export results
llm-eval export --format csv --output results.csv

# Generate PDF report
llm-eval report --run-ids a3f92c1b --output ./reports/

# Launch dashboard
llm-eval dashboard

# Start REST API server
llm-eval serve --port 8000

import asyncio
from llm_eval.core.evaluator import LLMEvaluator, EvaluationConfig
from llm_eval.benchmarks.mmlu import MMLUBenchmark

async def main():
    # Initialize evaluator
    evaluator = LLMEvaluator()

    # Load benchmark
    samples = MMLUBenchmark().load(num_samples=100)

    # Configure run
    config = EvaluationConfig(
        model="gpt-4o-mini",
        benchmark="mmlu",
        num_samples=100,
        temperature=0.0,
        concurrency=10,     # 10 parallel calls
        timeout=30.0,
    )

    # Run evaluation
    result = await evaluator.evaluate(config, samples)

    # Access results
    print(f"Accuracy:     {result.accuracy:.2%}")
    print(f"Avg Latency:  {result.avg_latency_ms:.0f}ms")
    print(f"P95 Latency:  {result.p95_latency_ms:.0f}ms")
    print(f"Total Cost:   ${result.total_cost_usd:.4f}")
    print(f"Hallucinate:  {result.hallucination_rate:.2%}")
    print(f"Reasoning:    {result.avg_reasoning_score:.1f}/10")

    # Export to JSON
    import json
    print(json.dumps(result.to_dict(), indent=2))

asyncio.run(main())

import asyncio
from llm_eval.core.evaluator import LLMEvaluator, EvaluationConfig
from llm_eval.benchmarks.mmlu import MMLUBenchmark

async def compare_models():
    evaluator = LLMEvaluator()
    samples = MMLUBenchmark().load(num_samples=50)

    # All models run on the SAME samples — apples-to-apples comparison
    configs = [
        EvaluationConfig(model="gpt-4o-mini",             benchmark="mmlu", num_samples=50),
        EvaluationConfig(model="claude-3-haiku-20240307",  benchmark="mmlu", num_samples=50),
        EvaluationConfig(model="gemini/gemini-1.5-flash", benchmark="mmlu", num_samples=50),
        EvaluationConfig(model="mistral/mistral-small-latest", benchmark="mmlu", num_samples=50),
    ]

    # Runs all 4 evaluations in parallel
    results = await evaluator.evaluate_multiple(configs, samples)

    # Print leaderboard
    print(f"{'Model':<35} {'Accuracy':>10} {'Latency':>10} {'Cost/1K':>12} {'Reasoning':>12}")
    print("-" * 80)
    for r in sorted(results, key=lambda x: x.accuracy, reverse=True):
        print(f"{r.model:<35} {r.accuracy:>9.1%} {r.avg_latency_ms:>8.0f}ms ${r.cost_per_1k_tokens:>9.4f} {r.avg_reasoning_score:>9.1f}/10")

asyncio.run(compare_models())

# Start the API server
uvicorn llm_eval.api.main:app --reload --port 8000
# Open: http://localhost:8000/docs

# Evaluate a model
curl -X POST http://localhost:8000/evaluate \
  -H "Content-Type: application/json" \
  -d '{"model":"gpt-4o-mini","benchmark":"mmlu","num_samples":50}'

{
  "run_id": "a3f92c1b",
  "model": "gpt-4o-mini",
  "accuracy": 0.78,
  "avg_latency_ms": 432.1,
  "p95_latency_ms": 1240.0,
  "total_cost_usd": 0.0012,
  "cost_per_1k_tokens": 0.0015,
  "hallucination_rate": 0.024,
  "avg_reasoning_score": 7.2,
  "created_at": "2025-01-20T14:32:01"
}

# Compare models
curl -X POST http://localhost:8000/compare \
  -H "Content-Type: application/json" \
  -d '{"models":["gpt-4o-mini","claude-3-haiku-20240307"],"benchmark":"mmlu","num_samples":30}'

# Export CSV
curl http://localhost:8000/export/csv -o results.csv

# Generate PDF report
curl -X POST http://localhost:8000/report \
  -H "Content-Type: application/json" \
  -d '{"run_ids":["a3f92c1b"]}' \
  -o report.pdf

# Clone and configure
git clone https://github.com/vignesh2027/LLM-Evaluation-Framework.git
cd LLM-Evaluation-Framework
cp .env.example .env
# Edit .env — add your API keys

# Start API + Dashboard with one command
docker-compose up -d

# API:       http://localhost:8000/docs
# Dashboard: http://localhost:8501

# Build only the API image
docker build --target api -t llm-eval-api .
docker run -p 8000:8000 --env-file .env llm-eval-api

# Build only the Dashboard image
docker build --target dashboard -t llm-eval-dashboard .
docker run -p 8501:8501 --env-file .env llm-eval-dashboard

# View logs
docker-compose logs -f api
docker-compose logs -f dashboard