#!/usr/bin/env python3 """Generate an HTML report from a grading.json produced by run_evals.py. Usage: uv run python evals/generate_report.py evals/results//grading.json """ import argparse import html import json import subprocess from pathlib import Path def verdict_icon(passed: bool | None) -> tuple[str, str]: """Return (icon, css_class) for a passed value.""" if passed is True: return "✓", "pass" if passed is False: return "✗", "fail" return "—", "skip" def generate_html(data: dict) -> str: summary = data["summary"] results = data["results"] skill_name = data.get("skill_name", "authsome") timestamp = data.get("timestamp", "") font_url = "https://fonts.googleapis.com/css2?family=Poppins:wght@500;600&family=Lora:wght@400;500&display=swap" parts = [ f""" {html.escape(skill_name)} — Eval Report

{html.escape(skill_name)} — Eval Report

{html.escape(timestamp)}
Results: ✓ {summary["passed"]} passed ✗ {summary["failed"]} failed — {summary["skipped"]} skipped of {summary["total"]} total
""" ] for r in results: outcome = r.get("outcome", {}) traj = r.get("trajectory_efficiency", {}) o_icon, o_cls = verdict_icon(outcome.get("passed")) t_icon, t_cls = verdict_icon(traj.get("passed")) is_skipped = r.get("status") == "skipped" human = r.get("requires_human", False) human_badge = "human" if human else "" env_text = html.escape(r.get("environment", "")) state_text = html.escape(r.get("authsome_state", "")) prompt_text = html.escape(r["prompt"]) name_text = html.escape(r.get("name", r["prompt"])) agent_text = html.escape(r.get("agent", "claude")) if is_skipped: body_rows = ( "
" "skipped" "" "Rate limit hit — switch model and retry" "
" ) else: o_evidence = html.escape(outcome.get("evidence", "")) t_evidence = html.escape(traj.get("evidence", "")) body_rows = f"""
outcome {o_evidence}
trajectory {t_evidence}
""" parts.append(f"""
#{r["id"]} {name_text} {agent_text} {human_badge}
outcome {o_icon}
trajectory {t_icon}
prompt {prompt_text}
environment
{env_text}
authsome state at start
{state_text}
{body_rows}
""") parts.append("""
""") return "".join(parts) def main() -> None: parser = argparse.ArgumentParser(description="Generate HTML report from grading.json") parser.add_argument("grading_json", help="Path to grading.json") parser.add_argument( "-o", "--output", default=None, help="Output HTML path (default: report.html next to grading.json)", ) args = parser.parse_args() grading_path = Path(args.grading_json) data = json.loads(grading_path.read_text()) html_out = Path(args.output) if args.output else grading_path.parent / "report.html" html_out.write_text(generate_html(data)) print(f"Report written to {html_out}") subprocess.run(["open", str(html_out)], check=False) if __name__ == "__main__": main()