{"slug":"miosa-osa-validate-evaluator","source_name":"miosa-osa/validate-evaluator","name":"Miosa Osa/Validate Evaluator","description":"Calibrate LLM-as-Judge evaluators against human labels. Computes TPR, TNR, precision, recall, F1, and Cohen's kappa. Detects systematic biases and recommends prompt corrections. Produces a calibration report with confidence intervals. Triggers on: \"validate evaluator\", \"calibrate judge\", \"judge accuracy\", \"evaluator validation\", \"judge metrics\"","version":1,"lift":{"pass_rate_delta_pts":52.17,"pass_rate_pct":78.3,"total_cases":23,"passed_cases":18,"tokens_delta_pct":23.6,"turns_delta_pct":0,"verdict":"mixed","benchmark_model":"gemini-3.6-flash","grading_method":"judged","completed_at":"2026-08-27T08:58:52.115063+00:00"},"skill_score":0.7826,"benchmark_models":[{"model":"gemini-3.6-flash","headline":true,"delta_pts":52.17,"with_pass_pct":78.3,"without_pass_pct":26.1,"tokens_delta_pct":23.6,"turns_delta_pct":0,"total_cases":23,"cases_aggregated":22,"verdict":"mixed","never_hurt":false,"completed_at":"2026-08-27T08:58:52.115063+00:00","run_id":"5550cb70-dc97-459e-b839-29d32017f85e","latest_attempt":{"id":"5550cb70-dc97-459e-b839-29d32017f85e","runner_model":"gemini-3.6-flash","overall_verdict":"mixed","completed_at":"2026-08-27T08:58:52.115063+00:00","gate":null},"version_number":1,"is_latest_version":true,"gate":null}],"trust":{"skill_safety":"passed","safety_status":"clean","intent_verdict":"safe","content_status":"clean","indexable":true},"license":null,"install_count":0,"total_uses":0,"total_users":0,"manifest_hash":"cd7f752a31ab1da0169d5ac2abd9967d2a9b1dec5335fb1f86c5c8d3f4231945","install_name":"miosa-osa-validate-evaluator","version_id":"78a7e0ca-fd01-4bf1-a630-b23d45598778","body_sha256":"cd7f752a31ab1da0169d5ac2abd9967d2a9b1dec5335fb1f86c5c8d3f4231945","render_hash":"df8097a317d740c2fcb0126a19e7f8062f8b30e93df02e6dde4a98466357df6f","raw_url":"https://app.decimal.ai/s/miosa-osa-validate-evaluator/SKILL.md","scorecard_url":"https://app.decimal.ai/skills/miosa-osa-validate-evaluator"}