{ "models": { "anthropic_claude-opus-4-5-20251101": { "prs": [ { "url": "https://github.com/keycloak/keycloak/pull/37429", "language": "Java", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 3 }, "all": { "tp": 0, "fp": 1, "fn": 5 } }, "coderabbit": { "strict": { "tp": 0, "fp": 5, "fn": 1 }, "core": { "tp": 2, "fp": 5, "fn": 1 }, "all": { "tp": 4, "fp": 5, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 3 }, "all": { "tp": 2, "fp": 3, "fn": 3 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 1 }, "all": { "tp": 4, "fp": 3, "fn": 1 } }, "gitlab": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 6, "fn": 1 }, "core": { "tp": 2, "fp": 6, "fn": 1 }, "all": { "tp": 3, "fp": 6, "fn": 2 } }, "augment": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 1 }, "all": { "tp": 2, "fp": 3, "fn": 3 } }, "bugbot": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 3 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "claude-code": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 3 }, "all": { "tp": 2, "fp": 3, "fn": 3 } }, "cubic-v2": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 4 } }, "devin": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 4 } }, "macroscope": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 3 }, "all": { "tp": 0, "fp": 1, "fn": 5 } }, "qodo-v2": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 3 }, "all": { "tp": 0, "fp": 3, "fn": 5 } }, "sourcery": { "strict": { "tp": 0, "fp": 5, "fn": 1 }, "core": { "tp": 0, "fp": 5, "fn": 3 }, "all": { "tp": 2, "fp": 5, "fn": 3 } }, "gitar-v2": { "strict": { "tp": 0, "fp": 4, "fn": 1 }, "core": { "tp": 2, "fp": 4, "fn": 1 }, "all": { "tp": 2, "fp": 4, "fn": 3 } } } }, { "url": "https://github.com/keycloak/keycloak/pull/37634", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 3 }, "all": { "tp": 1, "fp": 2, "fn": 3 } }, "coderabbit": { "strict": { "tp": 3, "fp": 4, "fn": 0 }, "core": { "tp": 3, "fp": 4, "fn": 1 }, "all": { "tp": 3, "fp": 4, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 3, "fp": 0, "fn": 1 }, "all": { "tp": 3, "fp": 0, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "gitlab": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 3, "fp": 3, "fn": 1 }, "all": { "tp": 3, "fp": 3, "fn": 1 } }, "augment": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 2 } }, "bugbot": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "claude": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "claude-code": { "strict": { "tp": 0, "fp": 2, "fn": 3 }, "core": { "tp": 0, "fp": 2, "fn": 4 }, "all": { "tp": 0, "fp": 2, "fn": 4 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "devin": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "graphite": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 3 }, "all": { "tp": 1, "fp": 2, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "sourcery": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "gitar-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } } } }, { "url": "https://github.com/keycloak/keycloak/pull/38446", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 1, "fn": 2 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "coderabbit": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "gitlab": { "strict": { "tp": 1, "fp": 5, "fn": 1 }, "core": { "tp": 1, "fp": 5, "fn": 1 }, "all": { "tp": 1, "fp": 5, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 6, "fn": 1 }, "core": { "tp": 1, "fp": 6, "fn": 1 }, "all": { "tp": 1, "fp": 6, "fn": 1 } }, "augment": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "bugbot": { "strict": { "tp": 0, "fp": 3, "fn": 2 }, "core": { "tp": 0, "fp": 3, "fn": 2 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "claude": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "claude-code": { "strict": { "tp": 1, "fp": 5, "fn": 1 }, "core": { "tp": 1, "fp": 5, "fn": 1 }, "all": { "tp": 1, "fp": 5, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "devin": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 3, "fn": 0 }, "core": { "tp": 2, "fp": 3, "fn": 0 }, "all": { "tp": 2, "fp": 3, "fn": 0 } }, "sourcery": { "strict": { "tp": 1, "fp": 6, "fn": 1 }, "core": { "tp": 1, "fp": 6, "fn": 1 }, "all": { "tp": 1, "fp": 6, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } } } }, { "url": "https://github.com/keycloak/keycloak/pull/36882", "language": "Java", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "coderabbit": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "gitlab": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 2 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "augment": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "bugbot": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "devin": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "macroscope": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "sourcery": { "strict": { "tp": 0, "fp": 4, "fn": 1 }, "core": { "tp": 0, "fp": 4, "fn": 2 }, "all": { "tp": 0, "fp": 4, "fn": 2 } }, "gitar-v2": { "strict": { "tp": 0, "fp": 4, "fn": 1 }, "core": { "tp": 0, "fp": 4, "fn": 2 }, "all": { "tp": 0, "fp": 4, "fn": 2 } } } }, { "url": "https://github.com/keycloak/keycloak/pull/36880", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 3, "fp": 0, "fn": 1 }, "core": { "tp": 3, "fp": 0, "fn": 1 }, "all": { "tp": 3, "fp": 0, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 2, "fp": 0, "fn": 2 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 3 } }, "coderabbit": { "strict": { "tp": 1, "fp": 5, "fn": 3 }, "core": { "tp": 1, "fp": 5, "fn": 3 }, "all": { "tp": 2, "fp": 5, "fn": 3 } }, "qodo-extended-v2": { "strict": { "tp": 3, "fp": 0, "fn": 1 }, "core": { "tp": 3, "fp": 0, "fn": 1 }, "all": { "tp": 3, "fp": 0, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 3 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 3, "fn": 3 }, "core": { "tp": 1, "fp": 3, "fn": 3 }, "all": { "tp": 2, "fp": 3, "fn": 3 } }, "gitlab": { "strict": { "tp": 1, "fp": 6, "fn": 3 }, "core": { "tp": 1, "fp": 6, "fn": 3 }, "all": { "tp": 2, "fp": 6, "fn": 3 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 6, "fn": 2 }, "core": { "tp": 2, "fp": 6, "fn": 2 }, "all": { "tp": 2, "fp": 6, "fn": 3 } }, "augment": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 3 }, "all": { "tp": 1, "fp": 1, "fn": 4 } }, "bugbot": { "strict": { "tp": 3, "fp": 0, "fn": 1 }, "core": { "tp": 3, "fp": 0, "fn": 1 }, "all": { "tp": 4, "fp": 0, "fn": 1 } }, "claude": { "strict": { "tp": 1, "fp": 2, "fn": 3 }, "core": { "tp": 1, "fp": 2, "fn": 3 }, "all": { "tp": 2, "fp": 2, "fn": 3 } }, "claude-code": { "strict": { "tp": 0, "fp": 2, "fn": 4 }, "core": { "tp": 0, "fp": 2, "fn": 4 }, "all": { "tp": 0, "fp": 2, "fn": 5 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 3 }, "all": { "tp": 1, "fp": 1, "fn": 4 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 3 }, "core": { "tp": 1, "fp": 0, "fn": 3 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "macroscope": { "strict": { "tp": 2, "fp": 4, "fn": 2 }, "core": { "tp": 2, "fp": 4, "fn": 2 }, "all": { "tp": 2, "fp": 4, "fn": 3 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "sourcery": { "strict": { "tp": 1, "fp": 4, "fn": 3 }, "core": { "tp": 1, "fp": 4, "fn": 3 }, "all": { "tp": 2, "fp": 4, "fn": 3 } }, "gitar-v2": { "strict": { "tp": 2, "fp": 3, "fn": 2 }, "core": { "tp": 2, "fp": 3, "fn": 2 }, "all": { "tp": 3, "fp": 3, "fn": 2 } } } }, { "url": "https://github.com/keycloak/keycloak/pull/37038", "language": "Java", "pr_size": "small", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 5, "fn": 1 }, "core": { "tp": 1, "fp": 5, "fn": 1 }, "all": { "tp": 1, "fp": 5, "fn": 1 } }, "coderabbit": { "strict": { "tp": 2, "fp": 7, "fn": 0 }, "core": { "tp": 2, "fp": 7, "fn": 0 }, "all": { "tp": 2, "fp": 7, "fn": 0 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "gitlab": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 3, "fn": 0 }, "core": { "tp": 2, "fp": 3, "fn": 0 }, "all": { "tp": 2, "fp": 3, "fn": 0 } }, "augment": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "bugbot": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude-code": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "macroscope": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "sourcery": { "strict": { "tp": 1, "fp": 4, "fn": 1 }, "core": { "tp": 1, "fp": 4, "fn": 1 }, "all": { "tp": 1, "fp": 4, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } } } }, { "url": "https://github.com/keycloak/keycloak/pull/33832", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "coderabbit": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 3, "fn": 2 }, "core": { "tp": 1, "fp": 3, "fn": 2 }, "all": { "tp": 1, "fp": 3, "fn": 3 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "gitlab": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 3 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 4, "fn": 1 }, "core": { "tp": 2, "fp": 4, "fn": 1 }, "all": { "tp": 3, "fp": 4, "fn": 1 } }, "augment": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 3 } }, "bugbot": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "claude": { "strict": { "tp": 0, "fp": 5, "fn": 2 }, "core": { "tp": 1, "fp": 5, "fn": 2 }, "all": { "tp": 1, "fp": 5, "fn": 3 } }, "claude-code": { "strict": { "tp": 0, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 2 } }, "devin": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "macroscope": { "strict": { "tp": 2, "fp": 4, "fn": 0 }, "core": { "tp": 2, "fp": 4, "fn": 1 }, "all": { "tp": 2, "fp": 4, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "sourcery": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "gitar-v2": { "strict": { "tp": 0, "fp": 3, "fn": 2 }, "core": { "tp": 1, "fp": 3, "fn": 2 }, "all": { "tp": 2, "fp": 3, "fn": 2 } } } }, { "url": "https://github.com/keycloak/keycloak/pull/40940", "language": "Java", "pr_size": "small", "domain": "concurrency", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "coderabbit": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "gitlab": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "augment": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "bugbot": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude-code": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "graphite": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "macroscope": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "sourcery": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 2, "fp": 3, "fn": 0 }, "core": { "tp": 2, "fp": 3, "fn": 0 }, "all": { "tp": 2, "fp": 3, "fn": 0 } } } }, { "url": "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "coderabbit": { "strict": { "tp": 1, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 0 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "gitlab": { "strict": { "tp": 1, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 0 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "augment": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "bugbot": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 1 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude-code": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "devin": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "macroscope": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "sourcery": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } } } }, { "url": "https://github.com/getsentry/sentry/pull/93824", "language": "Python", "pr_size": "large", "domain": "concurrency", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 3, "fn": 3 }, "core": { "tp": 0, "fp": 3, "fn": 5 }, "all": { "tp": 0, "fp": 3, "fn": 5 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 2, "fp": 0, "fn": 3 }, "all": { "tp": 2, "fp": 0, "fn": 3 } }, "coderabbit": { "strict": { "tp": 0, "fp": 1, "fn": 3 }, "core": { "tp": 0, "fp": 1, "fn": 5 }, "all": { "tp": 0, "fp": 1, "fn": 5 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 3 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 4 }, "all": { "tp": 1, "fp": 1, "fn": 4 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 3 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "gitlab": { "strict": { "tp": 0, "fp": 2, "fn": 3 }, "core": { "tp": 0, "fp": 2, "fn": 5 }, "all": { "tp": 0, "fp": 2, "fn": 5 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 4, "fn": 2 }, "core": { "tp": 3, "fp": 4, "fn": 2 }, "all": { "tp": 3, "fp": 4, "fn": 2 } }, "augment": { "strict": { "tp": 3, "fp": 1, "fn": 0 }, "core": { "tp": 5, "fp": 1, "fn": 0 }, "all": { "tp": 5, "fp": 1, "fn": 0 } }, "bugbot": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 4 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "claude": { "strict": { "tp": 0, "fp": 2, "fn": 3 }, "core": { "tp": 0, "fp": 2, "fn": 5 }, "all": { "tp": 0, "fp": 2, "fn": 5 } }, "claude-code": { "strict": { "tp": 3, "fp": 3, "fn": 0 }, "core": { "tp": 3, "fp": 3, "fn": 2 }, "all": { "tp": 3, "fp": 3, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 1, "fn": 0 }, "core": { "tp": 5, "fp": 1, "fn": 0 }, "all": { "tp": 5, "fp": 1, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 4 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "graphite": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 4 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 5 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 5 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "macroscope": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 3, "fp": 1, "fn": 2 }, "all": { "tp": 3, "fp": 1, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 4, "fp": 3, "fn": 1 }, "all": { "tp": 4, "fp": 3, "fn": 1 } }, "sourcery": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 4, "fp": 2, "fn": 1 }, "all": { "tp": 4, "fp": 2, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 3 }, "all": { "tp": 2, "fp": 1, "fn": 3 } } } }, { "url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5", "language": "Python", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 3, "fp": 1, "fn": 3 }, "core": { "tp": 3, "fp": 1, "fn": 3 }, "all": { "tp": 3, "fp": 1, "fn": 3 } }, "codeant-v2": { "strict": { "tp": 2, "fp": 5, "fn": 4 }, "core": { "tp": 2, "fp": 5, "fn": 4 }, "all": { "tp": 2, "fp": 5, "fn": 4 } }, "coderabbit": { "strict": { "tp": 4, "fp": 13, "fn": 2 }, "core": { "tp": 4, "fp": 13, "fn": 2 }, "all": { "tp": 4, "fp": 13, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 0, "fn": 4 }, "core": { "tp": 2, "fp": 0, "fn": 4 }, "all": { "tp": 2, "fp": 0, "fn": 4 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 0, "fn": 5 }, "core": { "tp": 1, "fp": 0, "fn": 5 }, "all": { "tp": 1, "fp": 0, "fn": 5 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 1, "fn": 4 }, "core": { "tp": 2, "fp": 1, "fn": 4 }, "all": { "tp": 2, "fp": 1, "fn": 4 } }, "gitlab": { "strict": { "tp": 2, "fp": 1, "fn": 4 }, "core": { "tp": 2, "fp": 1, "fn": 4 }, "all": { "tp": 2, "fp": 1, "fn": 4 } }, "copilot-v2": { "strict": { "tp": 3, "fp": 5, "fn": 3 }, "core": { "tp": 3, "fp": 5, "fn": 3 }, "all": { "tp": 3, "fp": 5, "fn": 3 } }, "augment": { "strict": { "tp": 3, "fp": 2, "fn": 3 }, "core": { "tp": 3, "fp": 2, "fn": 3 }, "all": { "tp": 3, "fp": 2, "fn": 3 } }, "bugbot": { "strict": { "tp": 2, "fp": 0, "fn": 4 }, "core": { "tp": 2, "fp": 0, "fn": 4 }, "all": { "tp": 2, "fp": 0, "fn": 4 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 6 }, "core": { "tp": 0, "fp": 0, "fn": 6 }, "all": { "tp": 0, "fp": 0, "fn": 6 } }, "claude-code": { "strict": { "tp": 0, "fp": 2, "fn": 6 }, "core": { "tp": 0, "fp": 2, "fn": 6 }, "all": { "tp": 0, "fp": 2, "fn": 6 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 2, "fn": 4 }, "core": { "tp": 2, "fp": 2, "fn": 4 }, "all": { "tp": 2, "fp": 2, "fn": 4 } }, "devin": { "strict": { "tp": 2, "fp": 1, "fn": 4 }, "core": { "tp": 2, "fp": 1, "fn": 4 }, "all": { "tp": 2, "fp": 1, "fn": 4 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 6 }, "core": { "tp": 0, "fp": 0, "fn": 6 }, "all": { "tp": 0, "fp": 0, "fn": 6 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 6 }, "core": { "tp": 0, "fp": 0, "fn": 6 }, "all": { "tp": 0, "fp": 0, "fn": 6 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 1, "fn": 4 }, "core": { "tp": 2, "fp": 1, "fn": 4 }, "all": { "tp": 2, "fp": 1, "fn": 4 } }, "macroscope": { "strict": { "tp": 2, "fp": 3, "fn": 4 }, "core": { "tp": 2, "fp": 3, "fn": 4 }, "all": { "tp": 2, "fp": 3, "fn": 4 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 0, "fn": 4 }, "core": { "tp": 2, "fp": 0, "fn": 4 }, "all": { "tp": 2, "fp": 0, "fn": 4 } }, "sourcery": { "strict": { "tp": 2, "fp": 2, "fn": 4 }, "core": { "tp": 2, "fp": 2, "fn": 4 }, "all": { "tp": 2, "fp": 2, "fn": 4 } }, "gitar-v2": { "strict": { "tp": 3, "fp": 6, "fn": 3 }, "core": { "tp": 3, "fp": 6, "fn": 3 }, "all": { "tp": 3, "fp": 6, "fn": 3 } } } }, { "url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1", "language": "Python", "pr_size": "medium", "domain": "API", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 3 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 0, "fn": 3 }, "core": { "tp": 1, "fp": 0, "fn": 3 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "coderabbit": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 4, "fp": 0, "fn": 0 }, "core": { "tp": 4, "fp": 0, "fn": 0 }, "all": { "tp": 4, "fp": 0, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 3, "fp": 2, "fn": 1 }, "core": { "tp": 3, "fp": 2, "fn": 1 }, "all": { "tp": 3, "fp": 2, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "gitlab": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "augment": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "bugbot": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "claude": { "strict": { "tp": 2, "fp": 4, "fn": 2 }, "core": { "tp": 2, "fp": 4, "fn": 2 }, "all": { "tp": 2, "fp": 4, "fn": 2 } }, "claude-code": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 0, "fn": 1 }, "core": { "tp": 3, "fp": 0, "fn": 1 }, "all": { "tp": 3, "fp": 0, "fn": 1 } }, "devin": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "kg": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 3 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 4, "fn": 3 }, "core": { "tp": 1, "fp": 4, "fn": 3 }, "all": { "tp": 1, "fp": 4, "fn": 3 } }, "macroscope": { "strict": { "tp": 2, "fp": 0, "fn": 2 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 3, "fp": 0, "fn": 1 }, "core": { "tp": 3, "fp": 0, "fn": 1 }, "all": { "tp": 3, "fp": 0, "fn": 1 } }, "sourcery": { "strict": { "tp": 2, "fp": 3, "fn": 2 }, "core": { "tp": 2, "fp": 3, "fn": 2 }, "all": { "tp": 2, "fp": 3, "fn": 2 } }, "gitar-v2": { "strict": { "tp": 3, "fp": 5, "fn": 1 }, "core": { "tp": 3, "fp": 5, "fn": 1 }, "all": { "tp": 3, "fp": 5, "fn": 1 } } } }, { "url": "https://github.com/grafana/grafana/pull/97529", "language": "Go", "pr_size": "small", "domain": "concurrency", "change_type": "performance", "complexity": "complex", "difficulty": "very_subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "coderabbit": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "gitlab": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "augment": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "bugbot": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude-code": { "strict": { "tp": 1, "fp": 4, "fn": 1 }, "core": { "tp": 1, "fp": 4, "fn": 1 }, "all": { "tp": 1, "fp": 4, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "devin": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "macroscope": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 3, "fn": 0 }, "core": { "tp": 2, "fp": 3, "fn": 0 }, "all": { "tp": 2, "fp": 3, "fn": 0 } }, "sourcery": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 4, "fn": 1 }, "core": { "tp": 1, "fp": 4, "fn": 1 }, "all": { "tp": 1, "fp": 4, "fn": 1 } } } }, { "url": "https://github.com/getsentry/sentry/pull/80168", "language": "Python", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 3 }, "all": { "tp": 0, "fp": 1, "fn": 3 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 3 }, "all": { "tp": 0, "fp": 1, "fn": 3 } }, "coderabbit": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "gitlab": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "augment": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 3, "fp": 0, "fn": 0 }, "all": { "tp": 3, "fp": 0, "fn": 0 } }, "bugbot": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "claude-code": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 3, "fp": 1, "fn": 0 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 3, "fp": 1, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "macroscope": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "sourcery": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 3, "fp": 1, "fn": 0 } } } }, { "url": "https://github.com/getsentry/sentry/pull/80528", "language": "Python", "pr_size": "small", "domain": "scheduling", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "coderabbit": { "strict": { "tp": 1, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 5, "fn": 0 }, "core": { "tp": 1, "fp": 5, "fn": 1 }, "all": { "tp": 1, "fp": 5, "fn": 1 } }, "gitlab": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "augment": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "bugbot": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "graphite": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "kg": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "sourcery": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } } } }, { "url": "https://github.com/getsentry/sentry/pull/77754", "language": "Python", "pr_size": "medium", "domain": "serialization", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "coderabbit": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 3, "fp": 2, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 3 } }, "gitlab": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 3, "fp": 0, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "augment": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "bugbot": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "kg": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "sourcery": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 2, "fp": 3, "fn": 2 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 3 } } } }, { "url": "https://github.com/getsentry/sentry/pull/95633", "language": "Python", "pr_size": "medium", "domain": "concurrency", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 0, "fp": 1, "fn": 1 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "coderabbit": { "strict": { "tp": 0, "fp": 7, "fn": 0 }, "core": { "tp": 0, "fp": 7, "fn": 1 }, "all": { "tp": 0, "fp": 7, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 0, "fp": 4, "fn": 0 }, "core": { "tp": 1, "fp": 4, "fn": 0 }, "all": { "tp": 1, "fp": 4, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 5, "fn": 0 }, "core": { "tp": 0, "fp": 5, "fn": 1 }, "all": { "tp": 0, "fp": 5, "fn": 2 } }, "gitlab": { "strict": { "tp": 0, "fp": 5, "fn": 0 }, "core": { "tp": 1, "fp": 5, "fn": 0 }, "all": { "tp": 1, "fp": 5, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "augment": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "bugbot": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "devin": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 0, "fp": 4, "fn": 0 }, "core": { "tp": 0, "fp": 4, "fn": 1 }, "all": { "tp": 0, "fp": 4, "fn": 2 } }, "sourcery": { "strict": { "tp": 0, "fp": 4, "fn": 0 }, "core": { "tp": 1, "fp": 4, "fn": 0 }, "all": { "tp": 1, "fp": 4, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 0, "fp": 7, "fn": 0 }, "core": { "tp": 0, "fp": 7, "fn": 1 }, "all": { "tp": 0, "fp": 7, "fn": 2 } } } }, { "url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2", "language": "Python", "pr_size": "medium", "domain": "data_processing", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "coderabbit": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "qodo-extended-v2": { "strict": { "tp": 3, "fp": 3, "fn": 1 }, "core": { "tp": 3, "fp": 3, "fn": 1 }, "all": { "tp": 3, "fp": 3, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 3, "fp": 2, "fn": 1 }, "core": { "tp": 3, "fp": 2, "fn": 1 }, "all": { "tp": 3, "fp": 2, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 3, "fp": 3, "fn": 1 }, "core": { "tp": 3, "fp": 3, "fn": 1 }, "all": { "tp": 3, "fp": 3, "fn": 1 } }, "gitlab": { "strict": { "tp": 4, "fp": 1, "fn": 0 }, "core": { "tp": 4, "fp": 1, "fn": 0 }, "all": { "tp": 4, "fp": 1, "fn": 0 } }, "copilot-v2": { "strict": { "tp": 3, "fp": 8, "fn": 1 }, "core": { "tp": 3, "fp": 8, "fn": 1 }, "all": { "tp": 3, "fp": 8, "fn": 1 } }, "augment": { "strict": { "tp": 3, "fp": 2, "fn": 1 }, "core": { "tp": 3, "fp": 2, "fn": 1 }, "all": { "tp": 3, "fp": 2, "fn": 1 } }, "bugbot": { "strict": { "tp": 4, "fp": 0, "fn": 0 }, "core": { "tp": 4, "fp": 0, "fn": 0 }, "all": { "tp": 4, "fp": 0, "fn": 0 } }, "claude": { "strict": { "tp": 3, "fp": 5, "fn": 1 }, "core": { "tp": 3, "fp": 5, "fn": 1 }, "all": { "tp": 3, "fp": 5, "fn": 1 } }, "claude-code": { "strict": { "tp": 3, "fp": 3, "fn": 1 }, "core": { "tp": 3, "fp": 3, "fn": 1 }, "all": { "tp": 3, "fp": 3, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "devin": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "kg": { "strict": { "tp": 2, "fp": 2, "fn": 2 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 3, "fn": 2 }, "core": { "tp": 2, "fp": 3, "fn": 2 }, "all": { "tp": 2, "fp": 3, "fn": 2 } }, "macroscope": { "strict": { "tp": 3, "fp": 0, "fn": 1 }, "core": { "tp": 3, "fp": 0, "fn": 1 }, "all": { "tp": 3, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 4, "fp": 0, "fn": 0 }, "core": { "tp": 4, "fp": 0, "fn": 0 }, "all": { "tp": 4, "fp": 0, "fn": 0 } }, "sourcery": { "strict": { "tp": 3, "fp": 7, "fn": 1 }, "core": { "tp": 3, "fp": 7, "fn": 1 }, "all": { "tp": 3, "fp": 7, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 4, "fp": 4, "fn": 0 }, "core": { "tp": 4, "fp": 4, "fn": 0 }, "all": { "tp": 4, "fp": 4, "fn": 0 } } } }, { "url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3", "language": "Python", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 3, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 3, "fp": 1, "fn": 0 } }, "coderabbit": { "strict": { "tp": 2, "fp": 5, "fn": 1 }, "core": { "tp": 2, "fp": 5, "fn": 1 }, "all": { "tp": 2, "fp": 5, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 3, "fp": 0, "fn": 0 }, "core": { "tp": 3, "fp": 0, "fn": 0 }, "all": { "tp": 3, "fp": 0, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "gitlab": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 6, "fn": 2 }, "core": { "tp": 1, "fp": 6, "fn": 2 }, "all": { "tp": 1, "fp": 6, "fn": 2 } }, "augment": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "bugbot": { "strict": { "tp": 0, "fp": 3, "fn": 3 }, "core": { "tp": 0, "fp": 3, "fn": 3 }, "all": { "tp": 0, "fp": 3, "fn": 3 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "devin": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "macroscope": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "sourcery": { "strict": { "tp": 2, "fp": 6, "fn": 1 }, "core": { "tp": 2, "fp": 6, "fn": 1 }, "all": { "tp": 2, "fp": 6, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 0, "fp": 8, "fn": 3 }, "core": { "tp": 0, "fp": 8, "fn": 3 }, "all": { "tp": 0, "fp": 8, "fn": 3 } } } }, { "url": "https://github.com/grafana/grafana/pull/103633", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "coderabbit": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 1, "fn": 2 }, "core": { "tp": 0, "fp": 1, "fn": 3 }, "all": { "tp": 0, "fp": 1, "fn": 3 } }, "gitlab": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "augment": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "bugbot": { "strict": { "tp": 0, "fp": 1, "fn": 2 }, "core": { "tp": 0, "fp": 1, "fn": 3 }, "all": { "tp": 0, "fp": 1, "fn": 3 } }, "claude": { "strict": { "tp": 0, "fp": 6, "fn": 2 }, "core": { "tp": 0, "fp": 6, "fn": 3 }, "all": { "tp": 0, "fp": 6, "fn": 3 } }, "claude-code": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 3, "fp": 0, "fn": 0 }, "all": { "tp": 3, "fp": 0, "fn": 0 } }, "devin": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 3, "fp": 0, "fn": 0 }, "all": { "tp": 3, "fp": 0, "fn": 0 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 1, "fn": 2 }, "core": { "tp": 0, "fp": 1, "fn": 3 }, "all": { "tp": 0, "fp": 1, "fn": 3 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "sourcery": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 0, "fp": 2, "fn": 3 }, "all": { "tp": 0, "fp": 2, "fn": 3 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 6, "fn": 1 }, "core": { "tp": 2, "fp": 6, "fn": 1 }, "all": { "tp": 2, "fp": 6, "fn": 1 } } } }, { "url": "https://github.com/getsentry/sentry/pull/67876", "language": "Python", "pr_size": "small", "domain": "authentication", "change_type": "security_patch", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 1, "fn": 3 }, "core": { "tp": 0, "fp": 1, "fn": 3 }, "all": { "tp": 0, "fp": 1, "fn": 3 } }, "coderabbit": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "gitlab": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 13, "fn": 1 }, "core": { "tp": 2, "fp": 13, "fn": 1 }, "all": { "tp": 2, "fp": 13, "fn": 1 } }, "augment": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "bugbot": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "claude": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "claude-code": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 2, "fn": 0 }, "core": { "tp": 3, "fp": 2, "fn": 0 }, "all": { "tp": 3, "fp": 2, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "macroscope": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 1 }, "all": { "tp": 2, "fp": 3, "fn": 1 } }, "sourcery": { "strict": { "tp": 2, "fp": 7, "fn": 1 }, "core": { "tp": 2, "fp": 7, "fn": 1 }, "all": { "tp": 2, "fp": 7, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 2, "fp": 4, "fn": 1 }, "core": { "tp": 2, "fp": 4, "fn": 1 }, "all": { "tp": 2, "fp": 4, "fn": 1 } } } }, { "url": "https://github.com/keycloak/keycloak/pull/32918", "language": "Java", "pr_size": "small", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "coderabbit": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 3, "fn": 2 }, "core": { "tp": 0, "fp": 3, "fn": 2 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "gitlab": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "augment": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "bugbot": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "claude": { "strict": { "tp": 0, "fp": 1, "fn": 2 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "claude-code": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "devin": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "graphite": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "kg": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "sourcery": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } } } }, { "url": "https://github.com/grafana/grafana/pull/94942", "language": "Go", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "coderabbit": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "gitlab": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "augment": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "bugbot": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "claude-code": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "sourcery": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } } } }, { "url": "https://github.com/grafana/grafana/pull/90939", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "coderabbit": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "gitlab": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 4, "fn": 1 }, "core": { "tp": 1, "fp": 4, "fn": 1 }, "all": { "tp": 1, "fp": 4, "fn": 1 } }, "augment": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "bugbot": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude-code": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "sourcery": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } } } }, { "url": "https://github.com/grafana/grafana/pull/80329", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "performance", "complexity": "simple", "difficulty": "obvious", "risk": "low", "context": "local", "concern": "maintainability", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "coderabbit": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "gitlab": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "augment": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "bugbot": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "claude": { "strict": { "tp": 0, "fp": 4, "fn": 0 }, "core": { "tp": 2, "fp": 4, "fn": 0 }, "all": { "tp": 2, "fp": 4, "fn": 0 } }, "claude-code": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "cubic-v2": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "devin": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "macroscope": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "sourcery": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "gitar-v2": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 2, "fp": 3, "fn": 0 }, "all": { "tp": 2, "fp": 3, "fn": 0 } } } }, { "url": "https://github.com/grafana/grafana/pull/90045", "language": "Go", "pr_size": "medium", "domain": "logging", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "coderabbit": { "strict": { "tp": 3, "fp": 2, "fn": 0 }, "core": { "tp": 3, "fp": 2, "fn": 0 }, "all": { "tp": 3, "fp": 2, "fn": 0 } }, "qodo-extended-v2": { "strict": { "tp": 3, "fp": 2, "fn": 0 }, "core": { "tp": 3, "fp": 2, "fn": 0 }, "all": { "tp": 3, "fp": 2, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 3, "fp": 3, "fn": 0 }, "core": { "tp": 3, "fp": 3, "fn": 0 }, "all": { "tp": 3, "fp": 3, "fn": 0 } }, "gemini-v2": { "strict": { "tp": 3, "fp": 5, "fn": 0 }, "core": { "tp": 3, "fp": 5, "fn": 0 }, "all": { "tp": 3, "fp": 5, "fn": 0 } }, "gitlab": { "strict": { "tp": 3, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 3, "fp": 1, "fn": 0 } }, "copilot-v2": { "strict": { "tp": 3, "fp": 5, "fn": 0 }, "core": { "tp": 3, "fp": 5, "fn": 0 }, "all": { "tp": 3, "fp": 5, "fn": 0 } }, "augment": { "strict": { "tp": 2, "fp": 6, "fn": 1 }, "core": { "tp": 2, "fp": 6, "fn": 1 }, "all": { "tp": 2, "fp": 6, "fn": 1 } }, "bugbot": { "strict": { "tp": 2, "fp": 4, "fn": 1 }, "core": { "tp": 2, "fp": 4, "fn": 1 }, "all": { "tp": 2, "fp": 4, "fn": 1 } }, "claude": { "strict": { "tp": 3, "fp": 3, "fn": 0 }, "core": { "tp": 3, "fp": 3, "fn": 0 }, "all": { "tp": 3, "fp": 3, "fn": 0 } }, "claude-code": { "strict": { "tp": 3, "fp": 2, "fn": 0 }, "core": { "tp": 3, "fp": 2, "fn": 0 }, "all": { "tp": 3, "fp": 2, "fn": 0 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 2, "fn": 0 }, "core": { "tp": 3, "fp": 2, "fn": 0 }, "all": { "tp": 3, "fp": 2, "fn": 0 } }, "devin": { "strict": { "tp": 3, "fp": 2, "fn": 0 }, "core": { "tp": 3, "fp": 2, "fn": 0 }, "all": { "tp": 3, "fp": 2, "fn": 0 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 3, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 3, "fp": 1, "fn": 0 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 6, "fn": 1 }, "core": { "tp": 2, "fp": 6, "fn": 1 }, "all": { "tp": 2, "fp": 6, "fn": 1 } }, "sourcery": { "strict": { "tp": 3, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 3, "fp": 1, "fn": 0 } }, "gitar-v2": { "strict": { "tp": 3, "fp": 4, "fn": 0 }, "core": { "tp": 3, "fp": 4, "fn": 0 }, "all": { "tp": 3, "fp": 4, "fn": 0 } } } }, { "url": "https://github.com/grafana/grafana/pull/106778", "language": "Go", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "coderabbit": { "strict": { "tp": 1, "fp": 6, "fn": 1 }, "core": { "tp": 1, "fp": 6, "fn": 1 }, "all": { "tp": 1, "fp": 6, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 3, "fn": 2 }, "core": { "tp": 0, "fp": 3, "fn": 2 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "gitlab": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "augment": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "bugbot": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "claude": { "strict": { "tp": 0, "fp": 10, "fn": 2 }, "core": { "tp": 0, "fp": 10, "fn": 2 }, "all": { "tp": 0, "fp": 10, "fn": 2 } }, "claude-code": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "devin": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "sourcery": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "gitar-v2": { "strict": { "tp": 2, "fp": 4, "fn": 0 }, "core": { "tp": 2, "fp": 4, "fn": 0 }, "all": { "tp": 2, "fp": 4, "fn": 0 } } } }, { "url": "https://github.com/grafana/grafana/pull/107534", "language": "Go", "pr_size": "small", "domain": "testing", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "low", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "coderabbit": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 0, "fp": 1, "fn": 1 }, "all": { "tp": 0, "fp": 1, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 0, "fp": 1, "fn": 1 }, "all": { "tp": 0, "fp": 1, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 0, "fp": 3, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 0 }, "all": { "tp": 1, "fp": 3, "fn": 0 } }, "gitlab": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 1 } }, "augment": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "bugbot": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 0, "fp": 1, "fn": 1 }, "all": { "tp": 0, "fp": 1, "fn": 1 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "claude-code": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 0 }, "all": { "tp": 1, "fp": 3, "fn": 0 } }, "cubic-v2": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 0 } }, "devin": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 0, "fp": 1, "fn": 1 }, "all": { "tp": 0, "fp": 1, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "kg": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "macroscope": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 0, "fp": 1, "fn": 1 }, "all": { "tp": 0, "fp": 1, "fn": 1 } }, "sourcery": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 0, "fp": 3, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 0, "fp": 3, "fn": 1 } } } }, { "url": "https://github.com/grafana/grafana/pull/79265", "language": "Go", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 0, "fn": 3 }, "core": { "tp": 2, "fp": 0, "fn": 3 }, "all": { "tp": 2, "fp": 0, "fn": 3 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 4 }, "all": { "tp": 1, "fp": 1, "fn": 4 } }, "coderabbit": { "strict": { "tp": 2, "fp": 4, "fn": 2 }, "core": { "tp": 3, "fp": 4, "fn": 2 }, "all": { "tp": 3, "fp": 4, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 3 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "greptile-v4-1": { "strict": { "tp": 3, "fp": 2, "fn": 1 }, "core": { "tp": 4, "fp": 2, "fn": 1 }, "all": { "tp": 4, "fp": 2, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 3, "fn": 3 }, "core": { "tp": 1, "fp": 3, "fn": 4 }, "all": { "tp": 1, "fp": 3, "fn": 4 } }, "gitlab": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 2, "fp": 1, "fn": 3 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 3, "fn": 2 }, "core": { "tp": 2, "fp": 3, "fn": 3 }, "all": { "tp": 2, "fp": 3, "fn": 3 } }, "augment": { "strict": { "tp": 1, "fp": 0, "fn": 3 }, "core": { "tp": 2, "fp": 0, "fn": 3 }, "all": { "tp": 2, "fp": 0, "fn": 3 } }, "bugbot": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 4 }, "all": { "tp": 1, "fp": 1, "fn": 4 } }, "claude": { "strict": { "tp": 1, "fp": 2, "fn": 3 }, "core": { "tp": 1, "fp": 2, "fn": 4 }, "all": { "tp": 1, "fp": 2, "fn": 4 } }, "claude-code": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 3 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 4, "fp": 1, "fn": 1 }, "all": { "tp": 4, "fp": 1, "fn": 1 } }, "devin": { "strict": { "tp": 2, "fp": 0, "fn": 2 }, "core": { "tp": 2, "fp": 0, "fn": 3 }, "all": { "tp": 2, "fp": 0, "fn": 3 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 5 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kg": { "strict": { "tp": 1, "fp": 0, "fn": 3 }, "core": { "tp": 1, "fp": 0, "fn": 4 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 1, "fn": 4 }, "core": { "tp": 1, "fp": 1, "fn": 4 }, "all": { "tp": 1, "fp": 1, "fn": 4 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 3 }, "core": { "tp": 1, "fp": 0, "fn": 4 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 3, "fp": 1, "fn": 2 }, "all": { "tp": 3, "fp": 1, "fn": 2 } }, "sourcery": { "strict": { "tp": 2, "fp": 3, "fn": 2 }, "core": { "tp": 3, "fp": 3, "fn": 2 }, "all": { "tp": 3, "fp": 3, "fn": 2 } }, "gitar-v2": { "strict": { "tp": 2, "fp": 6, "fn": 2 }, "core": { "tp": 3, "fp": 6, "fn": 2 }, "all": { "tp": 3, "fp": 6, "fn": 2 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9", "language": "Ruby", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "reliability", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "coderabbit": { "strict": { "tp": 0, "fp": 4, "fn": 1 }, "core": { "tp": 0, "fp": 4, "fn": 2 }, "all": { "tp": 0, "fp": 4, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "gitlab": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "augment": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "bugbot": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 1, "fp": 4, "fn": 0 }, "core": { "tp": 1, "fp": 4, "fn": 1 }, "all": { "tp": 1, "fp": 4, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "devin": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "sourcery": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "gitar-v2": { "strict": { "tp": 0, "fp": 4, "fn": 1 }, "core": { "tp": 0, "fp": 4, "fn": 2 }, "all": { "tp": 0, "fp": 4, "fn": 2 } } } }, { "url": "https://github.com/grafana/grafana/pull/76186", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 1, "fn": 3 }, "core": { "tp": 0, "fp": 1, "fn": 3 }, "all": { "tp": 0, "fp": 1, "fn": 3 } }, "coderabbit": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "gitlab": { "strict": { "tp": 3, "fp": 0, "fn": 0 }, "core": { "tp": 3, "fp": 0, "fn": 0 }, "all": { "tp": 3, "fp": 0, "fn": 0 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 2, "fn": 3 }, "core": { "tp": 0, "fp": 2, "fn": 3 }, "all": { "tp": 0, "fp": 2, "fn": 3 } }, "augment": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "bugbot": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "claude-code": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 0, "fn": 0 }, "core": { "tp": 3, "fp": 0, "fn": 0 }, "all": { "tp": 3, "fp": 0, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "sourcery": { "strict": { "tp": 0, "fp": 3, "fn": 3 }, "core": { "tp": 0, "fp": 3, "fn": 3 }, "all": { "tp": 0, "fp": 3, "fn": 3 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10", "language": "Ruby", "pr_size": "large", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 2, "fp": 0, "fn": 5 }, "core": { "tp": 2, "fp": 0, "fn": 5 }, "all": { "tp": 2, "fp": 0, "fn": 5 } }, "codeant-v2": { "strict": { "tp": 4, "fp": 5, "fn": 3 }, "core": { "tp": 4, "fp": 5, "fn": 3 }, "all": { "tp": 4, "fp": 5, "fn": 3 } }, "coderabbit": { "strict": { "tp": 3, "fp": 8, "fn": 4 }, "core": { "tp": 3, "fp": 8, "fn": 4 }, "all": { "tp": 3, "fp": 8, "fn": 4 } }, "qodo-extended-v2": { "strict": { "tp": 5, "fp": 4, "fn": 2 }, "core": { "tp": 5, "fp": 4, "fn": 2 }, "all": { "tp": 5, "fp": 4, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 2, "fn": 5 }, "core": { "tp": 2, "fp": 2, "fn": 5 }, "all": { "tp": 2, "fp": 2, "fn": 5 } }, "gemini-v2": { "strict": { "tp": 4, "fp": 2, "fn": 3 }, "core": { "tp": 4, "fp": 2, "fn": 3 }, "all": { "tp": 4, "fp": 2, "fn": 3 } }, "gitlab": { "strict": { "tp": 2, "fp": 3, "fn": 5 }, "core": { "tp": 2, "fp": 3, "fn": 5 }, "all": { "tp": 2, "fp": 3, "fn": 5 } }, "copilot-v2": { "strict": { "tp": 6, "fp": 5, "fn": 1 }, "core": { "tp": 6, "fp": 5, "fn": 1 }, "all": { "tp": 6, "fp": 5, "fn": 1 } }, "augment": { "strict": { "tp": 3, "fp": 1, "fn": 4 }, "core": { "tp": 3, "fp": 1, "fn": 4 }, "all": { "tp": 3, "fp": 1, "fn": 4 } }, "bugbot": { "strict": { "tp": 5, "fp": 2, "fn": 2 }, "core": { "tp": 5, "fp": 2, "fn": 2 }, "all": { "tp": 5, "fp": 2, "fn": 2 } }, "claude": { "strict": { "tp": 1, "fp": 2, "fn": 6 }, "core": { "tp": 1, "fp": 2, "fn": 6 }, "all": { "tp": 1, "fp": 2, "fn": 6 } }, "claude-code": { "strict": { "tp": 2, "fp": 4, "fn": 5 }, "core": { "tp": 2, "fp": 4, "fn": 5 }, "all": { "tp": 2, "fp": 4, "fn": 5 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 1, "fn": 4 }, "core": { "tp": 3, "fp": 1, "fn": 4 }, "all": { "tp": 3, "fp": 1, "fn": 4 } }, "devin": { "strict": { "tp": 1, "fp": 1, "fn": 6 }, "core": { "tp": 1, "fp": 1, "fn": 6 }, "all": { "tp": 1, "fp": 1, "fn": 6 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 7 }, "core": { "tp": 0, "fp": 0, "fn": 7 }, "all": { "tp": 0, "fp": 0, "fn": 7 } }, "kg": { "strict": { "tp": 1, "fp": 1, "fn": 6 }, "core": { "tp": 1, "fp": 1, "fn": 6 }, "all": { "tp": 1, "fp": 1, "fn": 6 } }, "kodus-v2": { "strict": { "tp": 5, "fp": 4, "fn": 2 }, "core": { "tp": 5, "fp": 4, "fn": 2 }, "all": { "tp": 5, "fp": 4, "fn": 2 } }, "macroscope": { "strict": { "tp": 5, "fp": 2, "fn": 2 }, "core": { "tp": 5, "fp": 2, "fn": 2 }, "all": { "tp": 5, "fp": 2, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 3, "fp": 4, "fn": 4 }, "core": { "tp": 3, "fp": 4, "fn": 4 }, "all": { "tp": 3, "fp": 4, "fn": 4 } }, "sourcery": { "strict": { "tp": 5, "fp": 7, "fn": 2 }, "core": { "tp": 5, "fp": 7, "fn": 2 }, "all": { "tp": 5, "fp": 7, "fn": 2 } }, "gitar-v2": { "strict": { "tp": 5, "fp": 6, "fn": 2 }, "core": { "tp": 5, "fp": 6, "fn": 2 }, "all": { "tp": 5, "fp": 6, "fn": 2 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "bug_fix", "complexity": "simple", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 3, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 5, "fp": 1, "fn": 0 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 8, "fn": 3 }, "core": { "tp": 0, "fp": 8, "fn": 3 }, "all": { "tp": 1, "fp": 8, "fn": 4 } }, "coderabbit": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 4 } }, "qodo-extended-v2": { "strict": { "tp": 3, "fp": 2, "fn": 0 }, "core": { "tp": 3, "fp": 2, "fn": 0 }, "all": { "tp": 5, "fp": 2, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 4, "fp": 0, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 3, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 5, "fp": 1, "fn": 0 } }, "gitlab": { "strict": { "tp": 3, "fp": 0, "fn": 0 }, "core": { "tp": 3, "fp": 0, "fn": 0 }, "all": { "tp": 5, "fp": 0, "fn": 0 } }, "copilot-v2": { "strict": { "tp": 3, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 5, "fp": 1, "fn": 0 } }, "augment": { "strict": { "tp": 3, "fp": 2, "fn": 0 }, "core": { "tp": 3, "fp": 2, "fn": 0 }, "all": { "tp": 5, "fp": 2, "fn": 0 } }, "bugbot": { "strict": { "tp": 3, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 5, "fp": 1, "fn": 0 } }, "claude": { "strict": { "tp": 3, "fp": 4, "fn": 0 }, "core": { "tp": 3, "fp": 4, "fn": 0 }, "all": { "tp": 5, "fp": 4, "fn": 0 } }, "claude-code": { "strict": { "tp": 3, "fp": 0, "fn": 0 }, "core": { "tp": 3, "fp": 0, "fn": 0 }, "all": { "tp": 5, "fp": 0, "fn": 0 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 0, "fn": 0 }, "core": { "tp": 3, "fp": 0, "fn": 0 }, "all": { "tp": 5, "fp": 0, "fn": 0 } }, "devin": { "strict": { "tp": 3, "fp": 0, "fn": 0 }, "core": { "tp": 3, "fp": 0, "fn": 0 }, "all": { "tp": 5, "fp": 0, "fn": 0 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kodus-v2": { "strict": { "tp": 3, "fp": 0, "fn": 0 }, "core": { "tp": 3, "fp": 0, "fn": 0 }, "all": { "tp": 3, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 3, "fp": 0, "fn": 0 }, "core": { "tp": 3, "fp": 0, "fn": 0 }, "all": { "tp": 5, "fp": 0, "fn": 0 } }, "qodo-v2": { "strict": { "tp": 3, "fp": 2, "fn": 0 }, "core": { "tp": 3, "fp": 2, "fn": 0 }, "all": { "tp": 5, "fp": 2, "fn": 0 } }, "sourcery": { "strict": { "tp": 3, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 5, "fp": 1, "fn": 0 } }, "gitar-v2": { "strict": { "tp": 3, "fp": 3, "fn": 0 }, "core": { "tp": 3, "fp": 3, "fn": 0 }, "all": { "tp": 5, "fp": 3, "fn": 0 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8", "language": "Ruby", "pr_size": "medium", "domain": "API", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 2, "fp": 2, "fn": 4 }, "core": { "tp": 2, "fp": 2, "fn": 4 }, "all": { "tp": 2, "fp": 2, "fn": 4 } }, "codeant-v2": { "strict": { "tp": 5, "fp": 5, "fn": 1 }, "core": { "tp": 5, "fp": 5, "fn": 1 }, "all": { "tp": 5, "fp": 5, "fn": 1 } }, "coderabbit": { "strict": { "tp": 4, "fp": 6, "fn": 2 }, "core": { "tp": 4, "fp": 6, "fn": 2 }, "all": { "tp": 4, "fp": 6, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 3, "fp": 1, "fn": 3 }, "core": { "tp": 3, "fp": 1, "fn": 3 }, "all": { "tp": 3, "fp": 1, "fn": 3 } }, "greptile-v4-1": { "strict": { "tp": 4, "fp": 1, "fn": 2 }, "core": { "tp": 4, "fp": 1, "fn": 2 }, "all": { "tp": 4, "fp": 1, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 2, "fn": 4 }, "core": { "tp": 2, "fp": 2, "fn": 4 }, "all": { "tp": 2, "fp": 2, "fn": 4 } }, "gitlab": { "strict": { "tp": 4, "fp": 1, "fn": 2 }, "core": { "tp": 4, "fp": 1, "fn": 2 }, "all": { "tp": 4, "fp": 1, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 3, "fp": 9, "fn": 3 }, "core": { "tp": 3, "fp": 9, "fn": 3 }, "all": { "tp": 3, "fp": 9, "fn": 3 } }, "augment": { "strict": { "tp": 3, "fp": 2, "fn": 3 }, "core": { "tp": 3, "fp": 2, "fn": 3 }, "all": { "tp": 3, "fp": 2, "fn": 3 } }, "bugbot": { "strict": { "tp": 2, "fp": 1, "fn": 4 }, "core": { "tp": 2, "fp": 1, "fn": 4 }, "all": { "tp": 2, "fp": 1, "fn": 4 } }, "claude": { "strict": { "tp": 4, "fp": 3, "fn": 2 }, "core": { "tp": 4, "fp": 3, "fn": 2 }, "all": { "tp": 4, "fp": 3, "fn": 2 } }, "claude-code": { "strict": { "tp": 2, "fp": 1, "fn": 4 }, "core": { "tp": 2, "fp": 1, "fn": 4 }, "all": { "tp": 2, "fp": 1, "fn": 4 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 1, "fn": 3 }, "core": { "tp": 3, "fp": 1, "fn": 3 }, "all": { "tp": 3, "fp": 1, "fn": 3 } }, "devin": { "strict": { "tp": 2, "fp": 0, "fn": 4 }, "core": { "tp": 2, "fp": 0, "fn": 4 }, "all": { "tp": 2, "fp": 0, "fn": 4 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 6 }, "core": { "tp": 0, "fp": 0, "fn": 6 }, "all": { "tp": 0, "fp": 0, "fn": 6 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 6 }, "core": { "tp": 0, "fp": 0, "fn": 6 }, "all": { "tp": 0, "fp": 0, "fn": 6 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 6, "fn": 5 }, "core": { "tp": 1, "fp": 6, "fn": 5 }, "all": { "tp": 1, "fp": 6, "fn": 5 } }, "macroscope": { "strict": { "tp": 2, "fp": 2, "fn": 4 }, "core": { "tp": 2, "fp": 2, "fn": 4 }, "all": { "tp": 2, "fp": 2, "fn": 4 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 1, "fn": 4 }, "core": { "tp": 2, "fp": 1, "fn": 4 }, "all": { "tp": 2, "fp": 1, "fn": 4 } }, "sourcery": { "strict": { "tp": 4, "fp": 1, "fn": 2 }, "core": { "tp": 4, "fp": 1, "fn": 2 }, "all": { "tp": 4, "fp": 1, "fn": 2 } }, "gitar-v2": { "strict": { "tp": 3, "fp": 5, "fn": 3 }, "core": { "tp": 3, "fp": 5, "fn": 3 }, "all": { "tp": 3, "fp": 5, "fn": 3 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3", "language": "Ruby", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "coderabbit": { "strict": { "tp": 2, "fp": 5, "fn": 1 }, "core": { "tp": 2, "fp": 5, "fn": 1 }, "all": { "tp": 2, "fp": 5, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 1 }, "all": { "tp": 2, "fp": 3, "fn": 1 } }, "gitlab": { "strict": { "tp": 0, "fp": 2, "fn": 3 }, "core": { "tp": 0, "fp": 2, "fn": 3 }, "all": { "tp": 0, "fp": 2, "fn": 3 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 6, "fn": 1 }, "core": { "tp": 2, "fp": 6, "fn": 1 }, "all": { "tp": 2, "fp": 6, "fn": 1 } }, "augment": { "strict": { "tp": 1, "fp": 4, "fn": 2 }, "core": { "tp": 1, "fp": 4, "fn": 2 }, "all": { "tp": 1, "fp": 4, "fn": 2 } }, "bugbot": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "claude": { "strict": { "tp": 1, "fp": 3, "fn": 2 }, "core": { "tp": 1, "fp": 3, "fn": 2 }, "all": { "tp": 1, "fp": 3, "fn": 2 } }, "claude-code": { "strict": { "tp": 1, "fp": 4, "fn": 2 }, "core": { "tp": 1, "fp": 4, "fn": 2 }, "all": { "tp": 1, "fp": 4, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "devin": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "sourcery": { "strict": { "tp": 1, "fp": 3, "fn": 2 }, "core": { "tp": 1, "fp": 3, "fn": 2 }, "all": { "tp": 1, "fp": 3, "fn": 2 } }, "gitar-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5", "language": "Ruby", "pr_size": "small", "domain": "UI", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "coderabbit": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 7, "fn": 1 }, "core": { "tp": 2, "fp": 7, "fn": 1 }, "all": { "tp": 2, "fp": 7, "fn": 1 } }, "gitlab": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "augment": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "bugbot": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 1 }, "all": { "tp": 2, "fp": 3, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "devin": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "macroscope": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "sourcery": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 1 }, "all": { "tp": 2, "fp": 3, "fn": 1 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6", "language": "Ruby", "pr_size": "small", "domain": "serialization", "change_type": "feature", "complexity": "simple", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 0 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 0 }, "all": { "tp": 1, "fp": 3, "fn": 0 } }, "coderabbit": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 0 } }, "qodo-extended-v2": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 0 }, "all": { "tp": 1, "fp": 3, "fn": 0 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 5, "fn": 0 }, "core": { "tp": 1, "fp": 5, "fn": 0 }, "all": { "tp": 1, "fp": 5, "fn": 0 } }, "gitlab": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 0 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 0 }, "all": { "tp": 1, "fp": 3, "fn": 0 } }, "augment": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 0 } }, "bugbot": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 0 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "claude-code": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 0 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 0 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 0 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 0 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 0 } }, "sourcery": { "strict": { "tp": 1, "fp": 4, "fn": 0 }, "core": { "tp": 1, "fp": 4, "fn": 0 }, "all": { "tp": 1, "fp": 4, "fn": 0 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 5, "fn": 0 }, "core": { "tp": 1, "fp": 5, "fn": 0 }, "all": { "tp": 1, "fp": 5, "fn": 0 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4", "language": "Ruby", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 2, "fp": 2, "fn": 6 }, "core": { "tp": 2, "fp": 2, "fn": 6 }, "all": { "tp": 2, "fp": 2, "fn": 6 } }, "codeant-v2": { "strict": { "tp": 3, "fp": 16, "fn": 5 }, "core": { "tp": 3, "fp": 16, "fn": 5 }, "all": { "tp": 3, "fp": 16, "fn": 5 } }, "coderabbit": { "strict": { "tp": 6, "fp": 19, "fn": 2 }, "core": { "tp": 6, "fp": 19, "fn": 2 }, "all": { "tp": 6, "fp": 19, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 4, "fp": 1, "fn": 4 }, "core": { "tp": 4, "fp": 1, "fn": 4 }, "all": { "tp": 4, "fp": 1, "fn": 4 } }, "greptile-v4-1": { "strict": { "tp": 4, "fp": 1, "fn": 4 }, "core": { "tp": 4, "fp": 1, "fn": 4 }, "all": { "tp": 4, "fp": 1, "fn": 4 } }, "gemini-v2": { "strict": { "tp": 5, "fp": 4, "fn": 3 }, "core": { "tp": 5, "fp": 4, "fn": 3 }, "all": { "tp": 5, "fp": 4, "fn": 3 } }, "gitlab": { "strict": { "tp": 5, "fp": 0, "fn": 3 }, "core": { "tp": 5, "fp": 0, "fn": 3 }, "all": { "tp": 5, "fp": 0, "fn": 3 } }, "copilot-v2": { "strict": { "tp": 6, "fp": 2, "fn": 2 }, "core": { "tp": 6, "fp": 2, "fn": 2 }, "all": { "tp": 6, "fp": 2, "fn": 2 } }, "augment": { "strict": { "tp": 5, "fp": 0, "fn": 3 }, "core": { "tp": 5, "fp": 0, "fn": 3 }, "all": { "tp": 5, "fp": 0, "fn": 3 } }, "bugbot": { "strict": { "tp": 2, "fp": 3, "fn": 6 }, "core": { "tp": 2, "fp": 3, "fn": 6 }, "all": { "tp": 2, "fp": 3, "fn": 6 } }, "claude": { "strict": { "tp": 4, "fp": 3, "fn": 4 }, "core": { "tp": 4, "fp": 3, "fn": 4 }, "all": { "tp": 4, "fp": 3, "fn": 4 } }, "claude-code": { "strict": { "tp": 5, "fp": 0, "fn": 3 }, "core": { "tp": 5, "fp": 0, "fn": 3 }, "all": { "tp": 5, "fp": 0, "fn": 3 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 1, "fn": 5 }, "core": { "tp": 3, "fp": 1, "fn": 5 }, "all": { "tp": 3, "fp": 1, "fn": 5 } }, "devin": { "strict": { "tp": 3, "fp": 1, "fn": 5 }, "core": { "tp": 3, "fp": 1, "fn": 5 }, "all": { "tp": 3, "fp": 1, "fn": 5 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 8 }, "core": { "tp": 0, "fp": 0, "fn": 8 }, "all": { "tp": 0, "fp": 0, "fn": 8 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 8 }, "core": { "tp": 0, "fp": 0, "fn": 8 }, "all": { "tp": 0, "fp": 0, "fn": 8 } }, "kodus-v2": { "strict": { "tp": 4, "fp": 1, "fn": 4 }, "core": { "tp": 4, "fp": 1, "fn": 4 }, "all": { "tp": 4, "fp": 1, "fn": 4 } }, "macroscope": { "strict": { "tp": 2, "fp": 4, "fn": 6 }, "core": { "tp": 2, "fp": 4, "fn": 6 }, "all": { "tp": 2, "fp": 4, "fn": 6 } }, "qodo-v2": { "strict": { "tp": 6, "fp": 5, "fn": 2 }, "core": { "tp": 6, "fp": 5, "fn": 2 }, "all": { "tp": 6, "fp": 5, "fn": 2 } }, "sourcery": { "strict": { "tp": 3, "fp": 2, "fn": 5 }, "core": { "tp": 3, "fp": 2, "fn": 5 }, "all": { "tp": 3, "fp": 2, "fn": 5 } }, "gitar-v2": { "strict": { "tp": 4, "fp": 1, "fn": 4 }, "core": { "tp": 4, "fp": 1, "fn": 4 }, "all": { "tp": 4, "fp": 1, "fn": 4 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1", "language": "Ruby", "pr_size": "medium", "domain": "file_io", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 2, "fp": 0, "fn": 2 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "coderabbit": { "strict": { "tp": 3, "fp": 3, "fn": 1 }, "core": { "tp": 3, "fp": 3, "fn": 1 }, "all": { "tp": 3, "fp": 3, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 3, "fp": 0, "fn": 1 }, "core": { "tp": 3, "fp": 0, "fn": 1 }, "all": { "tp": 3, "fp": 0, "fn": 1 } }, "gitlab": { "strict": { "tp": 2, "fp": 0, "fn": 2 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "augment": { "strict": { "tp": 2, "fp": 2, "fn": 2 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 2 } }, "bugbot": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "claude": { "strict": { "tp": 2, "fp": 0, "fn": 2 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 3, "fp": 2, "fn": 1 }, "core": { "tp": 3, "fp": 2, "fn": 1 }, "all": { "tp": 3, "fp": 2, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 0, "fn": 1 }, "core": { "tp": 3, "fp": 0, "fn": 1 }, "all": { "tp": 3, "fp": 0, "fn": 1 } }, "devin": { "strict": { "tp": 2, "fp": 0, "fn": 2 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "kodus-v2": { "strict": { "tp": 3, "fp": 0, "fn": 1 }, "core": { "tp": 3, "fp": 0, "fn": 1 }, "all": { "tp": 3, "fp": 0, "fn": 1 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 3 }, "core": { "tp": 1, "fp": 0, "fn": 3 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 3, "fn": 2 }, "core": { "tp": 2, "fp": 3, "fn": 2 }, "all": { "tp": 2, "fp": 3, "fn": 2 } }, "sourcery": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "gitar-v2": { "strict": { "tp": 2, "fp": 3, "fn": 2 }, "core": { "tp": 2, "fp": 3, "fn": 2 }, "all": { "tp": 2, "fp": 3, "fn": 2 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 5, "fn": 0 }, "core": { "tp": 1, "fp": 5, "fn": 0 }, "all": { "tp": 1, "fp": 5, "fn": 1 } }, "coderabbit": { "strict": { "tp": 1, "fp": 5, "fn": 0 }, "core": { "tp": 1, "fp": 5, "fn": 0 }, "all": { "tp": 2, "fp": 5, "fn": 0 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 0 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "gitlab": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 4, "fn": 0 }, "core": { "tp": 1, "fp": 4, "fn": 0 }, "all": { "tp": 2, "fp": 4, "fn": 0 } }, "augment": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "bugbot": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "claude": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 1 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "claude-code": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "sourcery": { "strict": { "tp": 1, "fp": 5, "fn": 0 }, "core": { "tp": 1, "fp": 5, "fn": 0 }, "all": { "tp": 1, "fp": 5, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 5, "fn": 0 }, "core": { "tp": 1, "fp": 5, "fn": 0 }, "all": { "tp": 1, "fp": 5, "fn": 1 } } } }, { "url": "https://github.com/calcom/cal.com/pull/22532", "language": "TypeScript", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 2, "fn": 3 }, "core": { "tp": 1, "fp": 2, "fn": 3 }, "all": { "tp": 1, "fp": 2, "fn": 3 } }, "codeant-v2": { "strict": { "tp": 3, "fp": 2, "fn": 1 }, "core": { "tp": 3, "fp": 2, "fn": 1 }, "all": { "tp": 3, "fp": 2, "fn": 1 } }, "coderabbit": { "strict": { "tp": 2, "fp": 9, "fn": 2 }, "core": { "tp": 2, "fp": 9, "fn": 2 }, "all": { "tp": 2, "fp": 9, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 3, "fp": 2, "fn": 1 }, "core": { "tp": 3, "fp": 2, "fn": 1 }, "all": { "tp": 3, "fp": 2, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 4, "fp": 0, "fn": 0 }, "core": { "tp": 4, "fp": 0, "fn": 0 }, "all": { "tp": 4, "fp": 0, "fn": 0 } }, "gitlab": { "strict": { "tp": 2, "fp": 3, "fn": 2 }, "core": { "tp": 2, "fp": 3, "fn": 2 }, "all": { "tp": 2, "fp": 3, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 4, "fp": 2, "fn": 0 }, "core": { "tp": 4, "fp": 2, "fn": 0 }, "all": { "tp": 4, "fp": 2, "fn": 0 } }, "augment": { "strict": { "tp": 4, "fp": 2, "fn": 0 }, "core": { "tp": 4, "fp": 2, "fn": 0 }, "all": { "tp": 4, "fp": 2, "fn": 0 } }, "bugbot": { "strict": { "tp": 0, "fp": 1, "fn": 4 }, "core": { "tp": 0, "fp": 1, "fn": 4 }, "all": { "tp": 0, "fp": 1, "fn": 4 } }, "claude": { "strict": { "tp": 0, "fp": 1, "fn": 4 }, "core": { "tp": 0, "fp": 1, "fn": 4 }, "all": { "tp": 0, "fp": 1, "fn": 4 } }, "claude-code": { "strict": { "tp": 1, "fp": 3, "fn": 3 }, "core": { "tp": 1, "fp": 3, "fn": 3 }, "all": { "tp": 1, "fp": 3, "fn": 3 } }, "cubic-v2": { "strict": { "tp": 0, "fp": 2, "fn": 4 }, "core": { "tp": 0, "fp": 2, "fn": 4 }, "all": { "tp": 0, "fp": 2, "fn": 4 } }, "devin": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 3 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 3 }, "core": { "tp": 1, "fp": 0, "fn": 3 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "macroscope": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 3 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "qodo-v2": { "strict": { "tp": 3, "fp": 2, "fn": 1 }, "core": { "tp": 3, "fp": 2, "fn": 1 }, "all": { "tp": 3, "fp": 2, "fn": 1 } }, "sourcery": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 4, "fp": 7, "fn": 0 }, "core": { "tp": 4, "fp": 7, "fn": 0 }, "all": { "tp": 4, "fp": 7, "fn": 0 } } } }, { "url": "https://github.com/calcom/cal.com/pull/8330", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "coderabbit": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 3, "fn": 0 }, "core": { "tp": 2, "fp": 3, "fn": 0 }, "all": { "tp": 2, "fp": 3, "fn": 0 } }, "gitlab": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 4, "fn": 0 }, "core": { "tp": 2, "fp": 4, "fn": 0 }, "all": { "tp": 2, "fp": 4, "fn": 0 } }, "augment": { "strict": { "tp": 2, "fp": 3, "fn": 0 }, "core": { "tp": 2, "fp": 3, "fn": 0 }, "all": { "tp": 2, "fp": 3, "fn": 0 } }, "bugbot": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "claude": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "claude-code": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "devin": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "graphite": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "kg": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "macroscope": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "sourcery": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "gitar-v2": { "strict": { "tp": 2, "fp": 4, "fn": 0 }, "core": { "tp": 2, "fp": 4, "fn": 0 }, "all": { "tp": 2, "fp": 4, "fn": 0 } } } }, { "url": "https://github.com/calcom/cal.com/pull/14943", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "coderabbit": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "gitlab": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "augment": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "bugbot": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude-code": { "strict": { "tp": 0, "fp": 1, "fn": 2 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "graphite": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "kg": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "sourcery": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } } } }, { "url": "https://github.com/calcom/cal.com/pull/22345", "language": "TypeScript", "pr_size": "small", "domain": "database", "change_type": "migration", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "coderabbit": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "gitlab": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "augment": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "bugbot": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "devin": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 1 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "sourcery": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 4, "fn": 0 }, "core": { "tp": 1, "fp": 4, "fn": 0 }, "all": { "tp": 1, "fp": 4, "fn": 1 } } } }, { "url": "https://github.com/calcom/cal.com/pull/11059", "language": "TypeScript", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 6, "fp": 0, "fn": 3 }, "core": { "tp": 6, "fp": 0, "fn": 3 }, "all": { "tp": 6, "fp": 0, "fn": 3 } }, "codeant-v2": { "strict": { "tp": 7, "fp": 4, "fn": 2 }, "core": { "tp": 7, "fp": 4, "fn": 2 }, "all": { "tp": 7, "fp": 4, "fn": 2 } }, "coderabbit": { "strict": { "tp": 9, "fp": 10, "fn": 0 }, "core": { "tp": 9, "fp": 10, "fn": 0 }, "all": { "tp": 9, "fp": 10, "fn": 0 } }, "qodo-extended-v2": { "strict": { "tp": 7, "fp": 2, "fn": 2 }, "core": { "tp": 7, "fp": 2, "fn": 2 }, "all": { "tp": 7, "fp": 2, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 5, "fp": 2, "fn": 4 }, "core": { "tp": 5, "fp": 2, "fn": 4 }, "all": { "tp": 5, "fp": 2, "fn": 4 } }, "gemini-v2": { "strict": { "tp": 5, "fp": 1, "fn": 4 }, "core": { "tp": 5, "fp": 1, "fn": 4 }, "all": { "tp": 5, "fp": 1, "fn": 4 } }, "gitlab": { "strict": { "tp": 6, "fp": 2, "fn": 3 }, "core": { "tp": 6, "fp": 2, "fn": 3 }, "all": { "tp": 6, "fp": 2, "fn": 3 } }, "copilot-v2": { "strict": { "tp": 6, "fp": 2, "fn": 3 }, "core": { "tp": 6, "fp": 2, "fn": 3 }, "all": { "tp": 6, "fp": 2, "fn": 3 } }, "augment": { "strict": { "tp": 9, "fp": 1, "fn": 0 }, "core": { "tp": 9, "fp": 1, "fn": 0 }, "all": { "tp": 9, "fp": 1, "fn": 0 } }, "bugbot": { "strict": { "tp": 6, "fp": 4, "fn": 3 }, "core": { "tp": 6, "fp": 4, "fn": 3 }, "all": { "tp": 6, "fp": 4, "fn": 3 } }, "claude": { "strict": { "tp": 6, "fp": 0, "fn": 3 }, "core": { "tp": 6, "fp": 0, "fn": 3 }, "all": { "tp": 6, "fp": 0, "fn": 3 } }, "claude-code": { "strict": { "tp": 7, "fp": 0, "fn": 2 }, "core": { "tp": 7, "fp": 0, "fn": 2 }, "all": { "tp": 7, "fp": 0, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 4, "fp": 0, "fn": 5 }, "core": { "tp": 4, "fp": 0, "fn": 5 }, "all": { "tp": 4, "fp": 0, "fn": 5 } }, "devin": { "strict": { "tp": 5, "fp": 0, "fn": 4 }, "core": { "tp": 5, "fp": 0, "fn": 4 }, "all": { "tp": 5, "fp": 0, "fn": 4 } }, "graphite": { "strict": { "tp": 1, "fp": 0, "fn": 8 }, "core": { "tp": 1, "fp": 0, "fn": 8 }, "all": { "tp": 1, "fp": 0, "fn": 8 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 9 }, "core": { "tp": 0, "fp": 0, "fn": 9 }, "all": { "tp": 0, "fp": 0, "fn": 9 } }, "kodus-v2": { "strict": { "tp": 4, "fp": 3, "fn": 5 }, "core": { "tp": 4, "fp": 3, "fn": 5 }, "all": { "tp": 4, "fp": 3, "fn": 5 } }, "macroscope": { "strict": { "tp": 7, "fp": 6, "fn": 2 }, "core": { "tp": 7, "fp": 6, "fn": 2 }, "all": { "tp": 7, "fp": 6, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 8, "fp": 0, "fn": 1 }, "core": { "tp": 8, "fp": 0, "fn": 1 }, "all": { "tp": 8, "fp": 0, "fn": 1 } }, "sourcery": { "strict": { "tp": 5, "fp": 0, "fn": 4 }, "core": { "tp": 5, "fp": 0, "fn": 4 }, "all": { "tp": 5, "fp": 0, "fn": 4 } }, "gitar-v2": { "strict": { "tp": 8, "fp": 4, "fn": 1 }, "core": { "tp": 8, "fp": 4, "fn": 1 }, "all": { "tp": 8, "fp": 4, "fn": 1 } } } }, { "url": "https://github.com/calcom/cal.com/pull/7232", "language": "TypeScript", "pr_size": "medium", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "coderabbit": { "strict": { "tp": 2, "fp": 9, "fn": 1 }, "core": { "tp": 2, "fp": 9, "fn": 1 }, "all": { "tp": 2, "fp": 9, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 1 }, "all": { "tp": 2, "fp": 3, "fn": 1 } }, "gitlab": { "strict": { "tp": 0, "fp": 2, "fn": 3 }, "core": { "tp": 0, "fp": 2, "fn": 3 }, "all": { "tp": 0, "fp": 2, "fn": 3 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 3, "fn": 3 }, "core": { "tp": 0, "fp": 3, "fn": 3 }, "all": { "tp": 0, "fp": 3, "fn": 3 } }, "augment": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "bugbot": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "claude": { "strict": { "tp": 1, "fp": 3, "fn": 2 }, "core": { "tp": 1, "fp": 3, "fn": 2 }, "all": { "tp": 1, "fp": 3, "fn": 2 } }, "claude-code": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "devin": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 0, "fp": 1, "fn": 3 }, "core": { "tp": 0, "fp": 1, "fn": 3 }, "all": { "tp": 0, "fp": 1, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "macroscope": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "sourcery": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } } } }, { "url": "https://github.com/calcom/cal.com/pull/14740", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 1, "fn": 5 }, "core": { "tp": 1, "fp": 1, "fn": 5 }, "all": { "tp": 1, "fp": 1, "fn": 5 } }, "codeant-v2": { "strict": { "tp": 3, "fp": 4, "fn": 3 }, "core": { "tp": 3, "fp": 4, "fn": 3 }, "all": { "tp": 3, "fp": 4, "fn": 3 } }, "coderabbit": { "strict": { "tp": 5, "fp": 5, "fn": 1 }, "core": { "tp": 5, "fp": 5, "fn": 1 }, "all": { "tp": 5, "fp": 5, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 4, "fp": 1, "fn": 2 }, "core": { "tp": 4, "fp": 1, "fn": 2 }, "all": { "tp": 4, "fp": 1, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 3, "fp": 2, "fn": 3 }, "core": { "tp": 3, "fp": 2, "fn": 3 }, "all": { "tp": 3, "fp": 2, "fn": 3 } }, "gemini-v2": { "strict": { "tp": 3, "fp": 4, "fn": 3 }, "core": { "tp": 3, "fp": 4, "fn": 3 }, "all": { "tp": 3, "fp": 4, "fn": 3 } }, "gitlab": { "strict": { "tp": 3, "fp": 4, "fn": 3 }, "core": { "tp": 3, "fp": 4, "fn": 3 }, "all": { "tp": 3, "fp": 4, "fn": 3 } }, "copilot-v2": { "strict": { "tp": 5, "fp": 3, "fn": 1 }, "core": { "tp": 5, "fp": 3, "fn": 1 }, "all": { "tp": 5, "fp": 3, "fn": 1 } }, "augment": { "strict": { "tp": 4, "fp": 3, "fn": 2 }, "core": { "tp": 4, "fp": 3, "fn": 2 }, "all": { "tp": 4, "fp": 3, "fn": 2 } }, "bugbot": { "strict": { "tp": 3, "fp": 3, "fn": 3 }, "core": { "tp": 3, "fp": 3, "fn": 3 }, "all": { "tp": 3, "fp": 3, "fn": 3 } }, "claude": { "strict": { "tp": 2, "fp": 0, "fn": 4 }, "core": { "tp": 2, "fp": 0, "fn": 4 }, "all": { "tp": 2, "fp": 0, "fn": 4 } }, "claude-code": { "strict": { "tp": 2, "fp": 1, "fn": 4 }, "core": { "tp": 2, "fp": 1, "fn": 4 }, "all": { "tp": 2, "fp": 1, "fn": 4 } }, "cubic-v2": { "strict": { "tp": 4, "fp": 2, "fn": 2 }, "core": { "tp": 4, "fp": 2, "fn": 2 }, "all": { "tp": 4, "fp": 2, "fn": 2 } }, "devin": { "strict": { "tp": 2, "fp": 0, "fn": 4 }, "core": { "tp": 2, "fp": 0, "fn": 4 }, "all": { "tp": 2, "fp": 0, "fn": 4 } }, "graphite": { "strict": { "tp": 1, "fp": 0, "fn": 5 }, "core": { "tp": 1, "fp": 0, "fn": 5 }, "all": { "tp": 1, "fp": 0, "fn": 5 } }, "kg": { "strict": { "tp": 2, "fp": 0, "fn": 4 }, "core": { "tp": 2, "fp": 0, "fn": 4 }, "all": { "tp": 2, "fp": 0, "fn": 4 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 0, "fn": 4 }, "core": { "tp": 2, "fp": 0, "fn": 4 }, "all": { "tp": 2, "fp": 0, "fn": 4 } }, "macroscope": { "strict": { "tp": 3, "fp": 3, "fn": 3 }, "core": { "tp": 3, "fp": 3, "fn": 3 }, "all": { "tp": 3, "fp": 3, "fn": 3 } }, "qodo-v2": { "strict": { "tp": 5, "fp": 2, "fn": 1 }, "core": { "tp": 5, "fp": 2, "fn": 1 }, "all": { "tp": 5, "fp": 2, "fn": 1 } }, "sourcery": { "strict": { "tp": 3, "fp": 1, "fn": 3 }, "core": { "tp": 3, "fp": 1, "fn": 3 }, "all": { "tp": 3, "fp": 1, "fn": 3 } }, "gitar-v2": { "strict": { "tp": 5, "fp": 5, "fn": 1 }, "core": { "tp": 5, "fp": 5, "fn": 1 }, "all": { "tp": 5, "fp": 5, "fn": 1 } } } }, { "url": "https://github.com/calcom/cal.com/pull/10600", "language": "TypeScript", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 1, "fn": 2 }, "core": { "tp": 0, "fp": 1, "fn": 4 }, "all": { "tp": 0, "fp": 1, "fn": 5 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 3 }, "all": { "tp": 1, "fp": 2, "fn": 4 } }, "coderabbit": { "strict": { "tp": 1, "fp": 6, "fn": 1 }, "core": { "tp": 1, "fp": 6, "fn": 3 }, "all": { "tp": 2, "fp": 6, "fn": 3 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 3 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 3, "fp": 0, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 5, "fn": 2 }, "core": { "tp": 0, "fp": 5, "fn": 4 }, "all": { "tp": 1, "fp": 5, "fn": 4 } }, "gitlab": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 3 }, "all": { "tp": 2, "fp": 2, "fn": 3 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 4, "fn": 0 }, "core": { "tp": 4, "fp": 4, "fn": 0 }, "all": { "tp": 5, "fp": 4, "fn": 0 } }, "augment": { "strict": { "tp": 0, "fp": 3, "fn": 2 }, "core": { "tp": 1, "fp": 3, "fn": 3 }, "all": { "tp": 2, "fp": 3, "fn": 3 } }, "bugbot": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 3 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "claude": { "strict": { "tp": 0, "fp": 6, "fn": 2 }, "core": { "tp": 1, "fp": 6, "fn": 3 }, "all": { "tp": 2, "fp": 6, "fn": 3 } }, "claude-code": { "strict": { "tp": 0, "fp": 4, "fn": 2 }, "core": { "tp": 0, "fp": 4, "fn": 4 }, "all": { "tp": 1, "fp": 4, "fn": 4 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 3 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 3 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 3 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "macroscope": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 0, "fp": 2, "fn": 4 }, "all": { "tp": 0, "fp": 2, "fn": 5 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 3 } }, "sourcery": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 3, "fp": 1, "fn": 2 } }, "gitar-v2": { "strict": { "tp": 0, "fp": 6, "fn": 2 }, "core": { "tp": 0, "fp": 6, "fn": 4 }, "all": { "tp": 0, "fp": 6, "fn": 5 } } } }, { "url": "https://github.com/calcom/cal.com/pull/10967", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "bug_fix", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 1, "fn": 4 }, "core": { "tp": 1, "fp": 1, "fn": 4 }, "all": { "tp": 1, "fp": 1, "fn": 5 } }, "codeant-v2": { "strict": { "tp": 2, "fp": 5, "fn": 3 }, "core": { "tp": 2, "fp": 5, "fn": 3 }, "all": { "tp": 2, "fp": 5, "fn": 4 } }, "coderabbit": { "strict": { "tp": 5, "fp": 7, "fn": 0 }, "core": { "tp": 5, "fp": 7, "fn": 0 }, "all": { "tp": 5, "fp": 7, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 4, "fp": 2, "fn": 1 }, "core": { "tp": 4, "fp": 2, "fn": 1 }, "all": { "tp": 4, "fp": 2, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 1, "fn": 3 }, "core": { "tp": 2, "fp": 1, "fn": 3 }, "all": { "tp": 2, "fp": 1, "fn": 4 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 4, "fn": 3 }, "core": { "tp": 2, "fp": 4, "fn": 3 }, "all": { "tp": 2, "fp": 4, "fn": 4 } }, "gitlab": { "strict": { "tp": 4, "fp": 1, "fn": 1 }, "core": { "tp": 4, "fp": 1, "fn": 1 }, "all": { "tp": 4, "fp": 1, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 5, "fp": 6, "fn": 0 }, "core": { "tp": 5, "fp": 6, "fn": 0 }, "all": { "tp": 5, "fp": 6, "fn": 1 } }, "augment": { "strict": { "tp": 4, "fp": 0, "fn": 1 }, "core": { "tp": 4, "fp": 0, "fn": 1 }, "all": { "tp": 4, "fp": 0, "fn": 2 } }, "bugbot": { "strict": { "tp": 3, "fp": 3, "fn": 2 }, "core": { "tp": 3, "fp": 3, "fn": 2 }, "all": { "tp": 4, "fp": 3, "fn": 2 } }, "claude": { "strict": { "tp": 4, "fp": 3, "fn": 1 }, "core": { "tp": 4, "fp": 3, "fn": 1 }, "all": { "tp": 4, "fp": 3, "fn": 2 } }, "claude-code": { "strict": { "tp": 4, "fp": 3, "fn": 1 }, "core": { "tp": 4, "fp": 3, "fn": 1 }, "all": { "tp": 5, "fp": 3, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 2, "fn": 4 }, "core": { "tp": 1, "fp": 2, "fn": 4 }, "all": { "tp": 1, "fp": 2, "fn": 5 } }, "devin": { "strict": { "tp": 2, "fp": 3, "fn": 3 }, "core": { "tp": 2, "fp": 3, "fn": 3 }, "all": { "tp": 2, "fp": 3, "fn": 4 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 5 }, "core": { "tp": 0, "fp": 0, "fn": 5 }, "all": { "tp": 0, "fp": 0, "fn": 6 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 5 }, "core": { "tp": 0, "fp": 0, "fn": 5 }, "all": { "tp": 0, "fp": 0, "fn": 6 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 2, "fn": 3 }, "core": { "tp": 2, "fp": 2, "fn": 3 }, "all": { "tp": 2, "fp": 2, "fn": 4 } }, "macroscope": { "strict": { "tp": 3, "fp": 2, "fn": 2 }, "core": { "tp": 3, "fp": 2, "fn": 2 }, "all": { "tp": 3, "fp": 2, "fn": 3 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 3, "fn": 3 }, "core": { "tp": 2, "fp": 3, "fn": 3 }, "all": { "tp": 2, "fp": 3, "fn": 4 } }, "sourcery": { "strict": { "tp": 3, "fp": 2, "fn": 2 }, "core": { "tp": 3, "fp": 2, "fn": 2 }, "all": { "tp": 3, "fp": 2, "fn": 3 } }, "gitar-v2": { "strict": { "tp": 2, "fp": 5, "fn": 3 }, "core": { "tp": 2, "fp": 5, "fn": 3 }, "all": { "tp": 2, "fp": 5, "fn": 4 } } } }, { "url": "https://github.com/calcom/cal.com/pull/8087", "language": "TypeScript", "pr_size": "medium", "domain": "concurrency", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 0 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "coderabbit": { "strict": { "tp": 1, "fp": 6, "fn": 0 }, "core": { "tp": 1, "fp": 6, "fn": 0 }, "all": { "tp": 2, "fp": 6, "fn": 0 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "gitlab": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "augment": { "strict": { "tp": 1, "fp": 4, "fn": 0 }, "core": { "tp": 1, "fp": 4, "fn": 0 }, "all": { "tp": 1, "fp": 4, "fn": 1 } }, "bugbot": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "claude-code": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 10, "fn": 0 }, "core": { "tp": 1, "fp": 10, "fn": 0 }, "all": { "tp": 2, "fp": 10, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "sourcery": { "strict": { "tp": 1, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 0 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } } } } ], "tools": [ "augment", "baz", "bugbot", "claude", "claude-code", "codeant-v2", "coderabbit", "copilot-v2", "cubic-v2", "devin", "gemini-v2", "gitar-v2", "gitlab", "graphite", "greptile-v4-1", "kg", "kodus-v2", "macroscope", "qodo-extended-v2", "qodo-v2", "sourcery" ], "dimensions": { "language": [ "Go", "Java", "Python", "Ruby", "TypeScript" ], "pr_size": [ "small", "medium", "large" ], "domain": [ "API", "UI", "authentication", "caching", "concurrency", "configuration", "data_processing", "database", "file_io", "logging", "scheduling", "serialization", "testing" ], "change_type": [ "bug_fix", "feature", "migration", "performance", "refactoring", "security_patch" ], "complexity": [ "simple", "moderate", "complex" ], "difficulty": [ "obvious", "moderate", "subtle", "very_subtle" ], "risk": [ "low", "medium", "high", "critical" ], "context": [ "local", "file", "cross_file", "system" ], "concern": [ "correctness", "maintainability", "reliability", "security" ] }, "overall_metrics": { "coderabbit": { "strict": { "precision": 31.0, "recall": 62.6, "f1": 41.4, "f2": 52.0, "tp": 87, "fp": 194, "fn": 52, "num_prs": 50 }, "core": { "precision": 32.6, "recall": 59.5, "f1": 42.2, "f2": 51.1, "tp": 94, "fp": 194, "fn": 64, "num_prs": 50 }, "all": { "precision": 34.7, "recall": 59.5, "f1": 43.8, "f2": 52.1, "tp": 103, "fp": 194, "fn": 70, "num_prs": 50 } }, "cubic-v2": { "strict": { "precision": 58.3, "recall": 65.5, "f1": 61.7, "f2": 63.9, "tp": 91, "fp": 65, "fn": 48, "num_prs": 50 }, "core": { "precision": 61.5, "recall": 65.8, "f1": 63.6, "f2": 64.9, "tp": 104, "fp": 65, "fn": 54, "num_prs": 50 }, "all": { "precision": 62.4, "recall": 62.4, "f1": 62.4, "f2": 62.4, "tp": 108, "fp": 65, "fn": 65, "num_prs": 50 } }, "sourcery": { "strict": { "precision": 38.4, "recall": 56.1, "f1": 45.6, "f2": 51.4, "tp": 78, "fp": 125, "fn": 61, "num_prs": 50 }, "core": { "precision": 41.0, "recall": 55.1, "f1": 47.0, "f2": 51.5, "tp": 87, "fp": 125, "fn": 71, "num_prs": 50 }, "all": { "precision": 43.2, "recall": 54.9, "f1": 48.3, "f2": 52.1, "tp": 95, "fp": 125, "fn": 78, "num_prs": 50 } }, "graphite": { "strict": { "precision": 100.0, "recall": 7.9, "f1": 14.7, "f2": 9.7, "tp": 11, "fp": 0, "fn": 128, "num_prs": 50 }, "core": { "precision": 100.0, "recall": 7.6, "f1": 14.1, "f2": 9.3, "tp": 12, "fp": 0, "fn": 146, "num_prs": 50 }, "all": { "precision": 100.0, "recall": 6.9, "f1": 13.0, "f2": 8.5, "tp": 12, "fp": 0, "fn": 161, "num_prs": 50 } }, "claude-code": { "strict": { "precision": 44.3, "recall": 50.4, "f1": 47.1, "f2": 49.0, "tp": 70, "fp": 88, "fn": 69, "num_prs": 50 }, "core": { "precision": 46.3, "recall": 48.1, "f1": 47.2, "f2": 47.7, "tp": 76, "fp": 88, "fn": 82, "num_prs": 50 }, "all": { "precision": 48.8, "recall": 48.6, "f1": 48.7, "f2": 48.6, "tp": 84, "fp": 88, "fn": 89, "num_prs": 50 } }, "kodus-v2": { "strict": { "precision": 57.1, "recall": 40.3, "f1": 47.3, "f2": 42.8, "tp": 56, "fp": 42, "fn": 83, "num_prs": 50 }, "core": { "precision": 59.2, "recall": 38.6, "f1": 46.7, "f2": 41.5, "tp": 61, "fp": 42, "fn": 97, "num_prs": 50 }, "all": { "precision": 59.6, "recall": 35.8, "f1": 44.8, "f2": 38.9, "tp": 62, "fp": 42, "fn": 111, "num_prs": 50 } }, "devin": { "strict": { "precision": 67.4, "recall": 43.2, "f1": 52.6, "f2": 46.5, "tp": 60, "fp": 29, "fn": 79, "num_prs": 50 }, "core": { "precision": 69.1, "recall": 41.1, "f1": 51.6, "f2": 44.8, "tp": 65, "fp": 29, "fn": 93, "num_prs": 50 }, "all": { "precision": 70.1, "recall": 39.3, "f1": 50.4, "f2": 43.1, "tp": 68, "fp": 29, "fn": 105, "num_prs": 50 } }, "claude": { "strict": { "precision": 41.0, "recall": 39.6, "f1": 40.3, "f2": 39.9, "tp": 55, "fp": 79, "fn": 84, "num_prs": 50 }, "core": { "precision": 43.2, "recall": 38.0, "f1": 40.4, "f2": 38.9, "tp": 60, "fp": 79, "fn": 98, "num_prs": 50 }, "all": { "precision": 45.5, "recall": 38.2, "f1": 41.5, "f2": 39.4, "tp": 66, "fp": 79, "fn": 107, "num_prs": 50 } }, "macroscope": { "strict": { "precision": 60.2, "recall": 53.2, "f1": 56.5, "f2": 54.5, "tp": 74, "fp": 49, "fn": 65, "num_prs": 50 }, "core": { "precision": 61.1, "recall": 48.7, "f1": 54.2, "f2": 50.8, "tp": 77, "fp": 49, "fn": 81, "num_prs": 50 }, "all": { "precision": 62.3, "recall": 46.8, "f1": 53.5, "f2": 49.3, "tp": 81, "fp": 49, "fn": 92, "num_prs": 50 } }, "baz": { "strict": { "precision": 60.6, "recall": 41.0, "f1": 48.9, "f2": 43.8, "tp": 57, "fp": 37, "fn": 82, "num_prs": 50 }, "core": { "precision": 61.5, "recall": 37.3, "f1": 46.5, "f2": 40.5, "tp": 59, "fp": 37, "fn": 99, "num_prs": 50 }, "all": { "precision": 62.6, "recall": 35.8, "f1": 45.6, "f2": 39.2, "tp": 62, "fp": 37, "fn": 111, "num_prs": 50 } }, "augment": { "strict": { "precision": 56.5, "recall": 65.5, "f1": 60.7, "f2": 63.5, "tp": 91, "fp": 70, "fn": 48, "num_prs": 50 }, "core": { "precision": 59.5, "recall": 65.2, "f1": 62.2, "f2": 64.0, "tp": 103, "fp": 70, "fn": 55, "num_prs": 50 }, "all": { "precision": 60.7, "recall": 62.4, "f1": 61.5, "f2": 62.1, "tp": 108, "fp": 70, "fn": 65, "num_prs": 50 } }, "greptile-v4-1": { "strict": { "precision": 49.0, "recall": 54.0, "f1": 51.4, "f2": 52.9, "tp": 75, "fp": 78, "fn": 64, "num_prs": 50 }, "core": { "precision": 50.9, "recall": 51.3, "f1": 51.1, "f2": 51.2, "tp": 81, "fp": 78, "fn": 77, "num_prs": 50 }, "all": { "precision": 53.3, "recall": 51.4, "f1": 52.4, "f2": 51.8, "tp": 89, "fp": 78, "fn": 84, "num_prs": 50 } }, "bugbot": { "strict": { "precision": 54.8, "recall": 48.9, "f1": 51.7, "f2": 50.0, "tp": 68, "fp": 56, "fn": 71, "num_prs": 50 }, "core": { "precision": 56.9, "recall": 46.8, "f1": 51.4, "f2": 48.6, "tp": 74, "fp": 56, "fn": 84, "num_prs": 50 }, "all": { "precision": 58.2, "recall": 45.1, "f1": 50.8, "f2": 47.2, "tp": 78, "fp": 56, "fn": 95, "num_prs": 50 } }, "copilot-v2": { "strict": { "precision": 35.6, "recall": 66.9, "f1": 46.5, "f2": 56.9, "tp": 93, "fp": 168, "fn": 46, "num_prs": 50 }, "core": { "precision": 38.5, "recall": 66.5, "f1": 48.7, "f2": 58.0, "tp": 105, "fp": 168, "fn": 53, "num_prs": 50 }, "all": { "precision": 40.6, "recall": 66.5, "f1": 50.4, "f2": 59.0, "tp": 115, "fp": 168, "fn": 58, "num_prs": 50 } }, "codeant-v2": { "strict": { "precision": 39.0, "recall": 44.6, "f1": 41.6, "f2": 43.4, "tp": 62, "fp": 97, "fn": 77, "num_prs": 50 }, "core": { "precision": 40.1, "recall": 41.1, "f1": 40.6, "f2": 40.9, "tp": 65, "fp": 97, "fn": 93, "num_prs": 50 }, "all": { "precision": 41.2, "recall": 39.3, "f1": 40.2, "f2": 39.7, "tp": 68, "fp": 97, "fn": 105, "num_prs": 50 } }, "qodo-extended-v2": { "strict": { "precision": 64.8, "recall": 66.2, "f1": 65.5, "f2": 65.9, "tp": 92, "fp": 50, "fn": 47, "num_prs": 50 }, "core": { "precision": 67.1, "recall": 64.6, "f1": 65.8, "f2": 65.1, "tp": 102, "fp": 50, "fn": 56, "num_prs": 50 }, "all": { "precision": 67.9, "recall": 61.3, "f1": 64.4, "f2": 62.5, "tp": 106, "fp": 50, "fn": 67, "num_prs": 50 } }, "gemini-v2": { "strict": { "precision": 43.2, "recall": 57.6, "f1": 49.4, "f2": 54.0, "tp": 80, "fp": 105, "fn": 59, "num_prs": 50 }, "core": { "precision": 45.3, "recall": 55.1, "f1": 49.7, "f2": 52.8, "tp": 87, "fp": 105, "fn": 71, "num_prs": 50 }, "all": { "precision": 47.5, "recall": 54.9, "f1": 50.9, "f2": 53.3, "tp": 95, "fp": 105, "fn": 78, "num_prs": 50 } }, "qodo-v2": { "strict": { "precision": 53.0, "recall": 64.0, "f1": 58.0, "f2": 61.5, "tp": 89, "fp": 79, "fn": 50, "num_prs": 50 }, "core": { "precision": 55.4, "recall": 62.0, "f1": 58.5, "f2": 60.6, "tp": 98, "fp": 79, "fn": 60, "num_prs": 50 }, "all": { "precision": 56.4, "recall": 59.0, "f1": 57.6, "f2": 58.4, "tp": 102, "fp": 79, "fn": 71, "num_prs": 50 } }, "gitar-v2": { "strict": { "precision": 33.3, "recall": 63.3, "f1": 43.7, "f2": 53.7, "tp": 88, "fp": 176, "fn": 51, "num_prs": 50 }, "core": { "precision": 35.5, "recall": 61.4, "f1": 45.0, "f2": 53.6, "tp": 97, "fp": 176, "fn": 61, "num_prs": 50 }, "all": { "precision": 36.9, "recall": 59.5, "f1": 45.6, "f2": 53.0, "tp": 103, "fp": 176, "fn": 70, "num_prs": 50 } }, "gitlab": { "strict": { "precision": 51.0, "recall": 54.0, "f1": 52.4, "f2": 53.3, "tp": 75, "fp": 72, "fn": 64, "num_prs": 50 }, "core": { "precision": 53.5, "recall": 52.5, "f1": 53.0, "f2": 52.7, "tp": 83, "fp": 72, "fn": 75, "num_prs": 50 }, "all": { "precision": 55.8, "recall": 52.6, "f1": 54.2, "f2": 53.2, "tp": 91, "fp": 72, "fn": 82, "num_prs": 50 } }, "kg": { "strict": { "precision": 53.3, "recall": 17.3, "f1": 26.1, "f2": 20.0, "tp": 24, "fp": 21, "fn": 115, "num_prs": 50 }, "core": { "precision": 54.3, "recall": 15.8, "f1": 24.5, "f2": 18.4, "tp": 25, "fp": 21, "fn": 133, "num_prs": 50 }, "all": { "precision": 55.3, "recall": 15.0, "f1": 23.6, "f2": 17.6, "tp": 26, "fp": 21, "fn": 147, "num_prs": 50 } } } }, "anthropic_claude-sonnet-4-5-20250929": { "prs": [ { "url": "https://github.com/keycloak/keycloak/pull/37429", "language": "Java", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 1 }, "all": { "tp": 2, "fp": 3, "fn": 3 } }, "baz": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 3 }, "all": { "tp": 0, "fp": 1, "fn": 5 } }, "bugbot": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 3 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "claude-code": { "strict": { "tp": 0, "fp": 4, "fn": 1 }, "core": { "tp": 0, "fp": 4, "fn": 3 }, "all": { "tp": 1, "fp": 4, "fn": 4 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 3 }, "all": { "tp": 0, "fp": 1, "fn": 5 } }, "coderabbit": { "strict": { "tp": 0, "fp": 5, "fn": 1 }, "core": { "tp": 2, "fp": 5, "fn": 1 }, "all": { "tp": 4, "fp": 5, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 4 } }, "devin": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 4 } }, "macroscope": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 3 }, "all": { "tp": 0, "fp": 1, "fn": 5 } }, "qodo-extended-v2": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "qodo-v2": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 3 }, "all": { "tp": 0, "fp": 3, "fn": 5 } }, "sourcery": { "strict": { "tp": 0, "fp": 5, "fn": 1 }, "core": { "tp": 0, "fp": 5, "fn": 3 }, "all": { "tp": 1, "fp": 5, "fn": 4 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 3 }, "all": { "tp": 2, "fp": 3, "fn": 3 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 5, "fn": 1 }, "core": { "tp": 1, "fp": 5, "fn": 2 }, "all": { "tp": 2, "fp": 5, "fn": 3 } }, "gitlab": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 3 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 5, "fn": 1 }, "core": { "tp": 2, "fp": 5, "fn": 1 }, "all": { "tp": 3, "fp": 5, "fn": 2 } }, "gitar-v2": { "strict": { "tp": 0, "fp": 4, "fn": 1 }, "core": { "tp": 2, "fp": 4, "fn": 1 }, "all": { "tp": 2, "fp": 4, "fn": 3 } } } }, { "url": "https://github.com/keycloak/keycloak/pull/37634", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "baz": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "bugbot": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 2 } }, "claude": { "strict": { "tp": 1, "fp": 3, "fn": 2 }, "core": { "tp": 2, "fp": 3, "fn": 2 }, "all": { "tp": 2, "fp": 3, "fn": 2 } }, "claude-code": { "strict": { "tp": 0, "fp": 2, "fn": 3 }, "core": { "tp": 0, "fp": 2, "fn": 4 }, "all": { "tp": 0, "fp": 2, "fn": 4 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 3 }, "all": { "tp": 1, "fp": 2, "fn": 3 } }, "coderabbit": { "strict": { "tp": 3, "fp": 6, "fn": 0 }, "core": { "tp": 4, "fp": 6, "fn": 0 }, "all": { "tp": 4, "fp": 6, "fn": 0 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "devin": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "graphite": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 3 }, "all": { "tp": 1, "fp": 2, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "sourcery": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "gitlab": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 3, "fp": 3, "fn": 1 }, "all": { "tp": 3, "fp": 3, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 2 } } } }, { "url": "https://github.com/keycloak/keycloak/pull/38446", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "baz": { "strict": { "tp": 0, "fp": 1, "fn": 2 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "bugbot": { "strict": { "tp": 0, "fp": 3, "fn": 2 }, "core": { "tp": 0, "fp": 3, "fn": 2 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "claude": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "claude-code": { "strict": { "tp": 1, "fp": 5, "fn": 1 }, "core": { "tp": 1, "fp": 5, "fn": 1 }, "all": { "tp": 1, "fp": 5, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "coderabbit": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "devin": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "macroscope": { "strict": { "tp": 0, "fp": 3, "fn": 2 }, "core": { "tp": 0, "fp": 3, "fn": 2 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 4, "fn": 1 }, "core": { "tp": 1, "fp": 4, "fn": 1 }, "all": { "tp": 1, "fp": 4, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 3, "fn": 0 }, "core": { "tp": 2, "fp": 3, "fn": 0 }, "all": { "tp": 2, "fp": 3, "fn": 0 } }, "sourcery": { "strict": { "tp": 1, "fp": 6, "fn": 1 }, "core": { "tp": 1, "fp": 6, "fn": 1 }, "all": { "tp": 1, "fp": 6, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "gitlab": { "strict": { "tp": 1, "fp": 5, "fn": 1 }, "core": { "tp": 1, "fp": 5, "fn": 1 }, "all": { "tp": 1, "fp": 5, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 6, "fn": 1 }, "core": { "tp": 1, "fp": 6, "fn": 1 }, "all": { "tp": 1, "fp": 6, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } } } }, { "url": "https://github.com/keycloak/keycloak/pull/36882", "language": "Java", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "baz": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "bugbot": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "coderabbit": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "devin": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "macroscope": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "sourcery": { "strict": { "tp": 0, "fp": 4, "fn": 1 }, "core": { "tp": 0, "fp": 4, "fn": 2 }, "all": { "tp": 0, "fp": 4, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "gitlab": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 2 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "gitar-v2": { "strict": { "tp": 0, "fp": 4, "fn": 1 }, "core": { "tp": 0, "fp": 4, "fn": 2 }, "all": { "tp": 0, "fp": 4, "fn": 2 } } } }, { "url": "https://github.com/keycloak/keycloak/pull/36880", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 3 }, "all": { "tp": 1, "fp": 1, "fn": 4 } }, "baz": { "strict": { "tp": 2, "fp": 0, "fn": 2 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 3 } }, "bugbot": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 3, "fp": 1, "fn": 2 } }, "claude": { "strict": { "tp": 1, "fp": 3, "fn": 3 }, "core": { "tp": 1, "fp": 3, "fn": 3 }, "all": { "tp": 2, "fp": 3, "fn": 3 } }, "claude-code": { "strict": { "tp": 0, "fp": 2, "fn": 4 }, "core": { "tp": 0, "fp": 2, "fn": 4 }, "all": { "tp": 0, "fp": 2, "fn": 5 } }, "codeant-v2": { "strict": { "tp": 2, "fp": 0, "fn": 2 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 3 } }, "coderabbit": { "strict": { "tp": 1, "fp": 3, "fn": 3 }, "core": { "tp": 1, "fp": 3, "fn": 3 }, "all": { "tp": 1, "fp": 3, "fn": 4 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 3 }, "all": { "tp": 1, "fp": 1, "fn": 4 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 3 }, "core": { "tp": 1, "fp": 0, "fn": 3 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 2, "fn": 2 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 3 } }, "macroscope": { "strict": { "tp": 2, "fp": 4, "fn": 2 }, "core": { "tp": 2, "fp": 4, "fn": 2 }, "all": { "tp": 2, "fp": 4, "fn": 3 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 0, "fn": 2 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 3 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 2, "fn": 3 }, "core": { "tp": 1, "fp": 2, "fn": 3 }, "all": { "tp": 1, "fp": 2, "fn": 4 } }, "sourcery": { "strict": { "tp": 1, "fp": 4, "fn": 3 }, "core": { "tp": 1, "fp": 4, "fn": 3 }, "all": { "tp": 2, "fp": 4, "fn": 3 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 0, "fn": 3 }, "core": { "tp": 1, "fp": 0, "fn": 3 }, "all": { "tp": 2, "fp": 0, "fn": 3 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 3, "fn": 3 }, "core": { "tp": 1, "fp": 3, "fn": 3 }, "all": { "tp": 2, "fp": 3, "fn": 3 } }, "gitlab": { "strict": { "tp": 1, "fp": 6, "fn": 3 }, "core": { "tp": 1, "fp": 6, "fn": 3 }, "all": { "tp": 2, "fp": 6, "fn": 3 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 6, "fn": 3 }, "core": { "tp": 1, "fp": 6, "fn": 3 }, "all": { "tp": 1, "fp": 6, "fn": 4 } }, "gitar-v2": { "strict": { "tp": 3, "fp": 4, "fn": 1 }, "core": { "tp": 3, "fp": 4, "fn": 1 }, "all": { "tp": 4, "fp": 4, "fn": 1 } } } }, { "url": "https://github.com/keycloak/keycloak/pull/37038", "language": "Java", "pr_size": "small", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 2, "fp": 3, "fn": 0 }, "core": { "tp": 2, "fp": 3, "fn": 0 }, "all": { "tp": 2, "fp": 3, "fn": 0 } }, "baz": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "bugbot": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude-code": { "strict": { "tp": 1, "fp": 4, "fn": 1 }, "core": { "tp": 1, "fp": 4, "fn": 1 }, "all": { "tp": 1, "fp": 4, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 5, "fn": 1 }, "core": { "tp": 1, "fp": 5, "fn": 1 }, "all": { "tp": 1, "fp": 5, "fn": 1 } }, "coderabbit": { "strict": { "tp": 2, "fp": 8, "fn": 0 }, "core": { "tp": 2, "fp": 8, "fn": 0 }, "all": { "tp": 2, "fp": 8, "fn": 0 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "macroscope": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "sourcery": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "gitlab": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 3, "fn": 0 }, "core": { "tp": 2, "fp": 3, "fn": 0 }, "all": { "tp": 2, "fp": 3, "fn": 0 } }, "gitar-v2": { "strict": { "tp": 2, "fp": 4, "fn": 0 }, "core": { "tp": 2, "fp": 4, "fn": 0 }, "all": { "tp": 2, "fp": 4, "fn": 0 } } } }, { "url": "https://github.com/keycloak/keycloak/pull/33832", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 3 } }, "baz": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "bugbot": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "claude": { "strict": { "tp": 0, "fp": 5, "fn": 2 }, "core": { "tp": 1, "fp": 5, "fn": 2 }, "all": { "tp": 1, "fp": 5, "fn": 3 } }, "claude-code": { "strict": { "tp": 0, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 0, "fp": 2, "fn": 3 }, "all": { "tp": 1, "fp": 2, "fn": 3 } }, "coderabbit": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 2 } }, "devin": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "macroscope": { "strict": { "tp": 1, "fp": 5, "fn": 1 }, "core": { "tp": 1, "fp": 5, "fn": 2 }, "all": { "tp": 1, "fp": 5, "fn": 3 } }, "qodo-extended-v2": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 3 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "sourcery": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 3 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 3, "fn": 2 }, "core": { "tp": 1, "fp": 3, "fn": 2 }, "all": { "tp": 1, "fp": 3, "fn": 3 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 2 } }, "gitlab": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 3 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 4, "fn": 1 }, "core": { "tp": 2, "fp": 4, "fn": 1 }, "all": { "tp": 3, "fp": 4, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 3, "fp": 2, "fn": 1 } } } }, { "url": "https://github.com/keycloak/keycloak/pull/40940", "language": "Java", "pr_size": "small", "domain": "concurrency", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "baz": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "bugbot": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude-code": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "coderabbit": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "graphite": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "macroscope": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "sourcery": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "gitlab": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "gitar-v2": { "strict": { "tp": 2, "fp": 3, "fn": 0 }, "core": { "tp": 2, "fp": 3, "fn": 0 }, "all": { "tp": 2, "fp": 3, "fn": 0 } } } }, { "url": "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "baz": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "bugbot": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 1 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude-code": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "coderabbit": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "devin": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "macroscope": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "sourcery": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "gitlab": { "strict": { "tp": 1, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 0 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } } } }, { "url": "https://github.com/getsentry/sentry/pull/93824", "language": "Python", "pr_size": "large", "domain": "concurrency", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 3, "fp": 1, "fn": 0 }, "core": { "tp": 5, "fp": 1, "fn": 0 }, "all": { "tp": 5, "fp": 1, "fn": 0 } }, "baz": { "strict": { "tp": 0, "fp": 3, "fn": 3 }, "core": { "tp": 0, "fp": 3, "fn": 5 }, "all": { "tp": 0, "fp": 3, "fn": 5 } }, "bugbot": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 4 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "claude": { "strict": { "tp": 0, "fp": 2, "fn": 3 }, "core": { "tp": 0, "fp": 2, "fn": 5 }, "all": { "tp": 0, "fp": 2, "fn": 5 } }, "claude-code": { "strict": { "tp": 3, "fp": 2, "fn": 0 }, "core": { "tp": 3, "fp": 2, "fn": 2 }, "all": { "tp": 3, "fp": 2, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 2, "fp": 0, "fn": 3 }, "all": { "tp": 2, "fp": 0, "fn": 3 } }, "coderabbit": { "strict": { "tp": 1, "fp": 3, "fn": 2 }, "core": { "tp": 1, "fp": 3, "fn": 4 }, "all": { "tp": 1, "fp": 3, "fn": 4 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 1, "fn": 0 }, "core": { "tp": 5, "fp": 1, "fn": 0 }, "all": { "tp": 5, "fp": 1, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 4 }, "all": { "tp": 1, "fp": 1, "fn": 4 } }, "graphite": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 4 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 5 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 5 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "macroscope": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 3, "fp": 1, "fn": 2 }, "all": { "tp": 3, "fp": 1, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 3 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 4, "fp": 2, "fn": 1 }, "all": { "tp": 4, "fp": 2, "fn": 1 } }, "sourcery": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 4, "fp": 2, "fn": 1 }, "all": { "tp": 4, "fp": 2, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 4 }, "all": { "tp": 1, "fp": 1, "fn": 4 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 3 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "gitlab": { "strict": { "tp": 0, "fp": 2, "fn": 3 }, "core": { "tp": 0, "fp": 2, "fn": 5 }, "all": { "tp": 0, "fp": 2, "fn": 5 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 5, "fn": 2 }, "core": { "tp": 3, "fp": 5, "fn": 2 }, "all": { "tp": 3, "fp": 5, "fn": 2 } }, "gitar-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 3 }, "all": { "tp": 2, "fp": 1, "fn": 3 } } } }, { "url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5", "language": "Python", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 3, "fp": 2, "fn": 3 }, "core": { "tp": 3, "fp": 2, "fn": 3 }, "all": { "tp": 3, "fp": 2, "fn": 3 } }, "baz": { "strict": { "tp": 3, "fp": 1, "fn": 3 }, "core": { "tp": 3, "fp": 1, "fn": 3 }, "all": { "tp": 3, "fp": 1, "fn": 3 } }, "bugbot": { "strict": { "tp": 2, "fp": 0, "fn": 4 }, "core": { "tp": 2, "fp": 0, "fn": 4 }, "all": { "tp": 2, "fp": 0, "fn": 4 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 6 }, "core": { "tp": 0, "fp": 0, "fn": 6 }, "all": { "tp": 0, "fp": 0, "fn": 6 } }, "claude-code": { "strict": { "tp": 0, "fp": 2, "fn": 6 }, "core": { "tp": 0, "fp": 2, "fn": 6 }, "all": { "tp": 0, "fp": 2, "fn": 6 } }, "codeant-v2": { "strict": { "tp": 2, "fp": 5, "fn": 4 }, "core": { "tp": 2, "fp": 5, "fn": 4 }, "all": { "tp": 2, "fp": 5, "fn": 4 } }, "coderabbit": { "strict": { "tp": 4, "fp": 13, "fn": 2 }, "core": { "tp": 4, "fp": 13, "fn": 2 }, "all": { "tp": 4, "fp": 13, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 3, "fn": 5 }, "core": { "tp": 1, "fp": 3, "fn": 5 }, "all": { "tp": 1, "fp": 3, "fn": 5 } }, "devin": { "strict": { "tp": 2, "fp": 1, "fn": 4 }, "core": { "tp": 2, "fp": 1, "fn": 4 }, "all": { "tp": 2, "fp": 1, "fn": 4 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 6 }, "core": { "tp": 0, "fp": 0, "fn": 6 }, "all": { "tp": 0, "fp": 0, "fn": 6 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 6 }, "core": { "tp": 0, "fp": 0, "fn": 6 }, "all": { "tp": 0, "fp": 0, "fn": 6 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 1, "fn": 4 }, "core": { "tp": 2, "fp": 1, "fn": 4 }, "all": { "tp": 2, "fp": 1, "fn": 4 } }, "macroscope": { "strict": { "tp": 2, "fp": 3, "fn": 4 }, "core": { "tp": 2, "fp": 3, "fn": 4 }, "all": { "tp": 2, "fp": 3, "fn": 4 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 0, "fn": 4 }, "core": { "tp": 2, "fp": 0, "fn": 4 }, "all": { "tp": 2, "fp": 0, "fn": 4 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 0, "fn": 4 }, "core": { "tp": 2, "fp": 0, "fn": 4 }, "all": { "tp": 2, "fp": 0, "fn": 4 } }, "sourcery": { "strict": { "tp": 2, "fp": 2, "fn": 4 }, "core": { "tp": 2, "fp": 2, "fn": 4 }, "all": { "tp": 2, "fp": 2, "fn": 4 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 0, "fn": 5 }, "core": { "tp": 1, "fp": 0, "fn": 5 }, "all": { "tp": 1, "fp": 0, "fn": 5 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 1, "fn": 4 }, "core": { "tp": 2, "fp": 1, "fn": 4 }, "all": { "tp": 2, "fp": 1, "fn": 4 } }, "gitlab": { "strict": { "tp": 2, "fp": 1, "fn": 4 }, "core": { "tp": 2, "fp": 1, "fn": 4 }, "all": { "tp": 2, "fp": 1, "fn": 4 } }, "copilot-v2": { "strict": { "tp": 3, "fp": 5, "fn": 3 }, "core": { "tp": 3, "fp": 5, "fn": 3 }, "all": { "tp": 3, "fp": 5, "fn": 3 } }, "gitar-v2": { "strict": { "tp": 3, "fp": 6, "fn": 3 }, "core": { "tp": 3, "fp": 6, "fn": 3 }, "all": { "tp": 3, "fp": 6, "fn": 3 } } } }, { "url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1", "language": "Python", "pr_size": "medium", "domain": "API", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "baz": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 3 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "bugbot": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "claude": { "strict": { "tp": 2, "fp": 7, "fn": 2 }, "core": { "tp": 2, "fp": 7, "fn": 2 }, "all": { "tp": 2, "fp": 7, "fn": 2 } }, "claude-code": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 0, "fn": 3 }, "core": { "tp": 1, "fp": 0, "fn": 3 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "coderabbit": { "strict": { "tp": 3, "fp": 3, "fn": 1 }, "core": { "tp": 3, "fp": 3, "fn": 1 }, "all": { "tp": 3, "fp": 3, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "devin": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "kg": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 3 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 2, "fn": 3 }, "core": { "tp": 1, "fp": 2, "fn": 3 }, "all": { "tp": 1, "fp": 2, "fn": 3 } }, "macroscope": { "strict": { "tp": 2, "fp": 0, "fn": 2 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 3, "fp": 0, "fn": 1 }, "core": { "tp": 3, "fp": 0, "fn": 1 }, "all": { "tp": 3, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 2, "fn": 3 }, "core": { "tp": 1, "fp": 2, "fn": 3 }, "all": { "tp": 1, "fp": 2, "fn": 3 } }, "sourcery": { "strict": { "tp": 2, "fp": 3, "fn": 2 }, "core": { "tp": 2, "fp": 3, "fn": 2 }, "all": { "tp": 2, "fp": 3, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 3, "fp": 2, "fn": 1 }, "core": { "tp": 3, "fp": 2, "fn": 1 }, "all": { "tp": 3, "fp": 2, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 4, "fn": 2 }, "core": { "tp": 2, "fp": 4, "fn": 2 }, "all": { "tp": 2, "fp": 4, "fn": 2 } }, "gitlab": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "gitar-v2": { "strict": { "tp": 3, "fp": 6, "fn": 1 }, "core": { "tp": 3, "fp": 6, "fn": 1 }, "all": { "tp": 3, "fp": 6, "fn": 1 } } } }, { "url": "https://github.com/grafana/grafana/pull/97529", "language": "Go", "pr_size": "small", "domain": "concurrency", "change_type": "performance", "complexity": "complex", "difficulty": "very_subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "baz": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "bugbot": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude-code": { "strict": { "tp": 1, "fp": 4, "fn": 1 }, "core": { "tp": 1, "fp": 4, "fn": 1 }, "all": { "tp": 1, "fp": 4, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "coderabbit": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "devin": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "macroscope": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 4, "fn": 0 }, "core": { "tp": 2, "fp": 4, "fn": 0 }, "all": { "tp": 2, "fp": 4, "fn": 0 } }, "sourcery": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "gitlab": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 9, "fn": 1 }, "core": { "tp": 1, "fp": 9, "fn": 1 }, "all": { "tp": 1, "fp": 9, "fn": 1 } } } }, { "url": "https://github.com/getsentry/sentry/pull/80168", "language": "Python", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 3, "fp": 0, "fn": 0 }, "all": { "tp": 3, "fp": 0, "fn": 0 } }, "baz": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 3 }, "all": { "tp": 0, "fp": 1, "fn": 3 } }, "bugbot": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "claude-code": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 3, "fp": 1, "fn": 0 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 3 }, "all": { "tp": 0, "fp": 1, "fn": 3 } }, "coderabbit": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 3, "fp": 1, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "macroscope": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "sourcery": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "gitlab": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 3, "fp": 1, "fn": 0 } } } }, { "url": "https://github.com/getsentry/sentry/pull/80528", "language": "Python", "pr_size": "small", "domain": "scheduling", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "baz": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "bugbot": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "coderabbit": { "strict": { "tp": 1, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "graphite": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "kg": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "sourcery": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 4, "fn": 0 }, "core": { "tp": 2, "fp": 4, "fn": 0 }, "all": { "tp": 2, "fp": 4, "fn": 0 } }, "gitlab": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } } } }, { "url": "https://github.com/getsentry/sentry/pull/77754", "language": "Python", "pr_size": "medium", "domain": "serialization", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "baz": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "bugbot": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "coderabbit": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 3, "fp": 2, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "kg": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "sourcery": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 2, "fp": 3, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 3 } }, "gitlab": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 3 } } } }, { "url": "https://github.com/getsentry/sentry/pull/95633", "language": "Python", "pr_size": "medium", "domain": "concurrency", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 0, "fp": 6, "fn": 0 }, "core": { "tp": 0, "fp": 6, "fn": 1 }, "all": { "tp": 0, "fp": 6, "fn": 2 } }, "baz": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 0, "fp": 1, "fn": 1 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "bugbot": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 0, "fp": 1, "fn": 1 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "coderabbit": { "strict": { "tp": 0, "fp": 7, "fn": 0 }, "core": { "tp": 0, "fp": 7, "fn": 1 }, "all": { "tp": 0, "fp": 7, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "devin": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 0, "fp": 6, "fn": 0 }, "core": { "tp": 0, "fp": 6, "fn": 1 }, "all": { "tp": 0, "fp": 6, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 0, "fp": 4, "fn": 0 }, "core": { "tp": 0, "fp": 4, "fn": 1 }, "all": { "tp": 0, "fp": 4, "fn": 2 } }, "sourcery": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 0 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 7, "fn": 0 }, "core": { "tp": 0, "fp": 7, "fn": 1 }, "all": { "tp": 0, "fp": 7, "fn": 2 } }, "gitlab": { "strict": { "tp": 0, "fp": 5, "fn": 0 }, "core": { "tp": 1, "fp": 5, "fn": 0 }, "all": { "tp": 1, "fp": 5, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "gitar-v2": { "strict": { "tp": 0, "fp": 8, "fn": 0 }, "core": { "tp": 0, "fp": 8, "fn": 1 }, "all": { "tp": 0, "fp": 8, "fn": 2 } } } }, { "url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2", "language": "Python", "pr_size": "medium", "domain": "data_processing", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 3, "fp": 2, "fn": 1 }, "core": { "tp": 3, "fp": 2, "fn": 1 }, "all": { "tp": 3, "fp": 2, "fn": 1 } }, "baz": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "bugbot": { "strict": { "tp": 4, "fp": 0, "fn": 0 }, "core": { "tp": 4, "fp": 0, "fn": 0 }, "all": { "tp": 4, "fp": 0, "fn": 0 } }, "claude": { "strict": { "tp": 1, "fp": 6, "fn": 3 }, "core": { "tp": 1, "fp": 6, "fn": 3 }, "all": { "tp": 1, "fp": 6, "fn": 3 } }, "claude-code": { "strict": { "tp": 3, "fp": 3, "fn": 1 }, "core": { "tp": 3, "fp": 3, "fn": 1 }, "all": { "tp": 3, "fp": 3, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "coderabbit": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 0, "fn": 1 }, "core": { "tp": 3, "fp": 0, "fn": 1 }, "all": { "tp": 3, "fp": 0, "fn": 1 } }, "devin": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "kg": { "strict": { "tp": 1, "fp": 2, "fn": 3 }, "core": { "tp": 1, "fp": 2, "fn": 3 }, "all": { "tp": 1, "fp": 2, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 3, "fn": 2 }, "core": { "tp": 2, "fp": 3, "fn": 2 }, "all": { "tp": 2, "fp": 3, "fn": 2 } }, "macroscope": { "strict": { "tp": 3, "fp": 0, "fn": 1 }, "core": { "tp": 3, "fp": 0, "fn": 1 }, "all": { "tp": 3, "fp": 0, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 4, "fn": 2 }, "core": { "tp": 2, "fp": 4, "fn": 2 }, "all": { "tp": 2, "fp": 4, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "sourcery": { "strict": { "tp": 3, "fp": 5, "fn": 1 }, "core": { "tp": 3, "fp": 5, "fn": 1 }, "all": { "tp": 3, "fp": 5, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 3, "fn": 2 }, "core": { "tp": 2, "fp": 3, "fn": 2 }, "all": { "tp": 2, "fp": 3, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 3, "fp": 4, "fn": 1 }, "core": { "tp": 3, "fp": 4, "fn": 1 }, "all": { "tp": 3, "fp": 4, "fn": 1 } }, "gitlab": { "strict": { "tp": 4, "fp": 1, "fn": 0 }, "core": { "tp": 4, "fp": 1, "fn": 0 }, "all": { "tp": 4, "fp": 1, "fn": 0 } }, "copilot-v2": { "strict": { "tp": 3, "fp": 10, "fn": 1 }, "core": { "tp": 3, "fp": 10, "fn": 1 }, "all": { "tp": 3, "fp": 10, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 4, "fp": 4, "fn": 0 }, "core": { "tp": 4, "fp": 4, "fn": 0 }, "all": { "tp": 4, "fp": 4, "fn": 0 } } } }, { "url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3", "language": "Python", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "baz": { "strict": { "tp": 0, "fp": 3, "fn": 3 }, "core": { "tp": 0, "fp": 3, "fn": 3 }, "all": { "tp": 0, "fp": 3, "fn": 3 } }, "bugbot": { "strict": { "tp": 0, "fp": 3, "fn": 3 }, "core": { "tp": 0, "fp": 3, "fn": 3 }, "all": { "tp": 0, "fp": 3, "fn": 3 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 3, "fp": 2, "fn": 0 }, "core": { "tp": 3, "fp": 2, "fn": 0 }, "all": { "tp": 3, "fp": 2, "fn": 0 } }, "coderabbit": { "strict": { "tp": 2, "fp": 5, "fn": 1 }, "core": { "tp": 2, "fp": 5, "fn": 1 }, "all": { "tp": 2, "fp": 5, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "devin": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "macroscope": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 3, "fp": 0, "fn": 0 }, "core": { "tp": 3, "fp": 0, "fn": 0 }, "all": { "tp": 3, "fp": 0, "fn": 0 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "sourcery": { "strict": { "tp": 2, "fp": 5, "fn": 1 }, "core": { "tp": 2, "fp": 5, "fn": 1 }, "all": { "tp": 2, "fp": 5, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "gitlab": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 1 }, "all": { "tp": 2, "fp": 3, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 7, "fn": 2 }, "core": { "tp": 1, "fp": 7, "fn": 2 }, "all": { "tp": 1, "fp": 7, "fn": 2 } }, "gitar-v2": { "strict": { "tp": 0, "fp": 9, "fn": 3 }, "core": { "tp": 0, "fp": 9, "fn": 3 }, "all": { "tp": 0, "fp": 9, "fn": 3 } } } }, { "url": "https://github.com/grafana/grafana/pull/103633", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "baz": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "bugbot": { "strict": { "tp": 0, "fp": 1, "fn": 2 }, "core": { "tp": 0, "fp": 1, "fn": 3 }, "all": { "tp": 0, "fp": 1, "fn": 3 } }, "claude": { "strict": { "tp": 0, "fp": 5, "fn": 2 }, "core": { "tp": 0, "fp": 5, "fn": 3 }, "all": { "tp": 0, "fp": 5, "fn": 3 } }, "claude-code": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 0, "fp": 2, "fn": 3 }, "all": { "tp": 0, "fp": 2, "fn": 3 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "coderabbit": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "devin": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 1, "fn": 2 }, "core": { "tp": 0, "fp": 1, "fn": 3 }, "all": { "tp": 0, "fp": 1, "fn": 3 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "sourcery": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 0, "fp": 2, "fn": 3 }, "all": { "tp": 0, "fp": 2, "fn": 3 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 3, "fn": 2 }, "core": { "tp": 0, "fp": 3, "fn": 3 }, "all": { "tp": 0, "fp": 3, "fn": 3 } }, "gitlab": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "gitar-v2": { "strict": { "tp": 2, "fp": 4, "fn": 0 }, "core": { "tp": 3, "fp": 4, "fn": 0 }, "all": { "tp": 3, "fp": 4, "fn": 0 } } } }, { "url": "https://github.com/getsentry/sentry/pull/67876", "language": "Python", "pr_size": "small", "domain": "authentication", "change_type": "security_patch", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "baz": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "bugbot": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "claude": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "claude-code": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 1, "fn": 3 }, "core": { "tp": 0, "fp": 1, "fn": 3 }, "all": { "tp": 0, "fp": 1, "fn": 3 } }, "coderabbit": { "strict": { "tp": 1, "fp": 3, "fn": 2 }, "core": { "tp": 1, "fp": 3, "fn": 2 }, "all": { "tp": 1, "fp": 3, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 2, "fn": 0 }, "core": { "tp": 3, "fp": 2, "fn": 0 }, "all": { "tp": 3, "fp": 2, "fn": 0 } }, "devin": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "macroscope": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 3, "fp": 5, "fn": 0 }, "core": { "tp": 3, "fp": 5, "fn": 0 }, "all": { "tp": 3, "fp": 5, "fn": 0 } }, "sourcery": { "strict": { "tp": 1, "fp": 8, "fn": 2 }, "core": { "tp": 1, "fp": 8, "fn": 2 }, "all": { "tp": 1, "fp": 8, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "gitlab": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 13, "fn": 1 }, "core": { "tp": 2, "fp": 13, "fn": 1 }, "all": { "tp": 2, "fp": 13, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 3, "fp": 4, "fn": 0 }, "core": { "tp": 3, "fp": 4, "fn": 0 }, "all": { "tp": 3, "fp": 4, "fn": 0 } } } }, { "url": "https://github.com/keycloak/keycloak/pull/32918", "language": "Java", "pr_size": "small", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "baz": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "bugbot": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "claude": { "strict": { "tp": 0, "fp": 1, "fn": 2 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "claude-code": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "coderabbit": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "devin": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "graphite": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "kg": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "sourcery": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 3, "fn": 2 }, "core": { "tp": 0, "fp": 3, "fn": 2 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "gitlab": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } } } }, { "url": "https://github.com/grafana/grafana/pull/94942", "language": "Go", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "baz": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "bugbot": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "claude-code": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "coderabbit": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "devin": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "sourcery": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "gitlab": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } } } }, { "url": "https://github.com/grafana/grafana/pull/90939", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "baz": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "bugbot": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude-code": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "coderabbit": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "sourcery": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "gitlab": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } } } }, { "url": "https://github.com/grafana/grafana/pull/80329", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "performance", "complexity": "simple", "difficulty": "obvious", "risk": "low", "context": "local", "concern": "maintainability", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "baz": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "bugbot": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "claude": { "strict": { "tp": 0, "fp": 4, "fn": 0 }, "core": { "tp": 2, "fp": 4, "fn": 0 }, "all": { "tp": 2, "fp": 4, "fn": 0 } }, "claude-code": { "strict": { "tp": 0, "fp": 5, "fn": 0 }, "core": { "tp": 2, "fp": 5, "fn": 0 }, "all": { "tp": 2, "fp": 5, "fn": 0 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "coderabbit": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "devin": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "macroscope": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 2, "fp": 3, "fn": 0 }, "all": { "tp": 2, "fp": 3, "fn": 0 } }, "qodo-v2": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 2, "fp": 3, "fn": 0 }, "all": { "tp": 2, "fp": 3, "fn": 0 } }, "sourcery": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "gitlab": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "gitar-v2": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 2, "fp": 3, "fn": 0 }, "all": { "tp": 2, "fp": 3, "fn": 0 } } } }, { "url": "https://github.com/grafana/grafana/pull/90045", "language": "Go", "pr_size": "medium", "domain": "logging", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 3, "fp": 5, "fn": 0 }, "core": { "tp": 3, "fp": 5, "fn": 0 }, "all": { "tp": 3, "fp": 5, "fn": 0 } }, "baz": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "bugbot": { "strict": { "tp": 2, "fp": 4, "fn": 1 }, "core": { "tp": 2, "fp": 4, "fn": 1 }, "all": { "tp": 2, "fp": 4, "fn": 1 } }, "claude": { "strict": { "tp": 2, "fp": 5, "fn": 1 }, "core": { "tp": 2, "fp": 5, "fn": 1 }, "all": { "tp": 2, "fp": 5, "fn": 1 } }, "claude-code": { "strict": { "tp": 3, "fp": 4, "fn": 0 }, "core": { "tp": 3, "fp": 4, "fn": 0 }, "all": { "tp": 3, "fp": 4, "fn": 0 } }, "codeant-v2": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 1 }, "all": { "tp": 2, "fp": 3, "fn": 1 } }, "coderabbit": { "strict": { "tp": 3, "fp": 2, "fn": 0 }, "core": { "tp": 3, "fp": 2, "fn": 0 }, "all": { "tp": 3, "fp": 2, "fn": 0 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 2, "fn": 0 }, "core": { "tp": 3, "fp": 2, "fn": 0 }, "all": { "tp": 3, "fp": 2, "fn": 0 } }, "devin": { "strict": { "tp": 3, "fp": 2, "fn": 0 }, "core": { "tp": 3, "fp": 2, "fn": 0 }, "all": { "tp": 3, "fp": 2, "fn": 0 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 3, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 3, "fp": 1, "fn": 0 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 3, "fp": 6, "fn": 0 }, "core": { "tp": 3, "fp": 6, "fn": 0 }, "all": { "tp": 3, "fp": 6, "fn": 0 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 7, "fn": 2 }, "core": { "tp": 1, "fp": 7, "fn": 2 }, "all": { "tp": 1, "fp": 7, "fn": 2 } }, "sourcery": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 3, "fp": 3, "fn": 0 }, "core": { "tp": 3, "fp": 3, "fn": 0 }, "all": { "tp": 3, "fp": 3, "fn": 0 } }, "gemini-v2": { "strict": { "tp": 3, "fp": 5, "fn": 0 }, "core": { "tp": 3, "fp": 5, "fn": 0 }, "all": { "tp": 3, "fp": 5, "fn": 0 } }, "gitlab": { "strict": { "tp": 3, "fp": 2, "fn": 0 }, "core": { "tp": 3, "fp": 2, "fn": 0 }, "all": { "tp": 3, "fp": 2, "fn": 0 } }, "copilot-v2": { "strict": { "tp": 3, "fp": 6, "fn": 0 }, "core": { "tp": 3, "fp": 6, "fn": 0 }, "all": { "tp": 3, "fp": 6, "fn": 0 } }, "gitar-v2": { "strict": { "tp": 3, "fp": 4, "fn": 0 }, "core": { "tp": 3, "fp": 4, "fn": 0 }, "all": { "tp": 3, "fp": 4, "fn": 0 } } } }, { "url": "https://github.com/grafana/grafana/pull/106778", "language": "Go", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "baz": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "bugbot": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "claude": { "strict": { "tp": 0, "fp": 12, "fn": 2 }, "core": { "tp": 0, "fp": 12, "fn": 2 }, "all": { "tp": 0, "fp": 12, "fn": 2 } }, "claude-code": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "coderabbit": { "strict": { "tp": 1, "fp": 5, "fn": 1 }, "core": { "tp": 1, "fp": 5, "fn": 1 }, "all": { "tp": 1, "fp": 5, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "devin": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 6, "fn": 1 }, "core": { "tp": 1, "fp": 6, "fn": 1 }, "all": { "tp": 1, "fp": 6, "fn": 1 } }, "sourcery": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 3, "fn": 2 }, "core": { "tp": 0, "fp": 3, "fn": 2 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "gitlab": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "gitar-v2": { "strict": { "tp": 2, "fp": 4, "fn": 0 }, "core": { "tp": 2, "fp": 4, "fn": 0 }, "all": { "tp": 2, "fp": 4, "fn": 0 } } } }, { "url": "https://github.com/grafana/grafana/pull/107534", "language": "Go", "pr_size": "small", "domain": "testing", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "low", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "baz": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 1 } }, "bugbot": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 0, "fp": 1, "fn": 1 }, "all": { "tp": 0, "fp": 1, "fn": 1 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "claude-code": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "coderabbit": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 0, "fp": 1, "fn": 1 }, "all": { "tp": 0, "fp": 1, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 0 } }, "devin": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 0, "fp": 1, "fn": 1 }, "all": { "tp": 0, "fp": 1, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "kg": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "macroscope": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 0, "fp": 1, "fn": 1 }, "all": { "tp": 0, "fp": 1, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 0, "fp": 1, "fn": 1 }, "all": { "tp": 0, "fp": 1, "fn": 1 } }, "sourcery": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 0, "fp": 3, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 4, "fn": 0 }, "core": { "tp": 0, "fp": 4, "fn": 1 }, "all": { "tp": 0, "fp": 4, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 0 }, "all": { "tp": 1, "fp": 3, "fn": 0 } }, "gitlab": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 0, "fp": 1, "fn": 1 }, "all": { "tp": 0, "fp": 1, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 0, "fp": 3, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 0, "fp": 3, "fn": 1 } } } }, { "url": "https://github.com/grafana/grafana/pull/79265", "language": "Go", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 4 }, "all": { "tp": 1, "fp": 1, "fn": 4 } }, "baz": { "strict": { "tp": 1, "fp": 0, "fn": 3 }, "core": { "tp": 2, "fp": 0, "fn": 3 }, "all": { "tp": 2, "fp": 0, "fn": 3 } }, "bugbot": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 4 }, "all": { "tp": 1, "fp": 1, "fn": 4 } }, "claude": { "strict": { "tp": 1, "fp": 2, "fn": 3 }, "core": { "tp": 1, "fp": 2, "fn": 4 }, "all": { "tp": 1, "fp": 2, "fn": 4 } }, "claude-code": { "strict": { "tp": 1, "fp": 2, "fn": 3 }, "core": { "tp": 1, "fp": 2, "fn": 4 }, "all": { "tp": 1, "fp": 2, "fn": 4 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 4 }, "all": { "tp": 1, "fp": 1, "fn": 4 } }, "coderabbit": { "strict": { "tp": 2, "fp": 4, "fn": 2 }, "core": { "tp": 3, "fp": 4, "fn": 2 }, "all": { "tp": 3, "fp": 4, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 4, "fp": 1, "fn": 1 }, "all": { "tp": 4, "fp": 1, "fn": 1 } }, "devin": { "strict": { "tp": 2, "fp": 0, "fn": 2 }, "core": { "tp": 2, "fp": 0, "fn": 3 }, "all": { "tp": 2, "fp": 0, "fn": 3 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 5 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kg": { "strict": { "tp": 1, "fp": 0, "fn": 3 }, "core": { "tp": 1, "fp": 0, "fn": 4 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 1, "fn": 4 }, "core": { "tp": 1, "fp": 1, "fn": 4 }, "all": { "tp": 1, "fp": 1, "fn": 4 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 3 }, "core": { "tp": 1, "fp": 0, "fn": 4 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 3 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 2, "fn": 2 }, "core": { "tp": 2, "fp": 2, "fn": 3 }, "all": { "tp": 2, "fp": 2, "fn": 3 } }, "sourcery": { "strict": { "tp": 1, "fp": 4, "fn": 3 }, "core": { "tp": 2, "fp": 4, "fn": 3 }, "all": { "tp": 2, "fp": 4, "fn": 3 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 2, "fn": 2 }, "core": { "tp": 3, "fp": 2, "fn": 2 }, "all": { "tp": 3, "fp": 2, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 4, "fn": 4 }, "core": { "tp": 0, "fp": 4, "fn": 5 }, "all": { "tp": 0, "fp": 4, "fn": 5 } }, "gitlab": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 2, "fp": 1, "fn": 3 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 4, "fn": 3 }, "core": { "tp": 1, "fp": 4, "fn": 4 }, "all": { "tp": 1, "fp": 4, "fn": 4 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 6, "fn": 3 }, "core": { "tp": 2, "fp": 6, "fn": 3 }, "all": { "tp": 2, "fp": 6, "fn": 3 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9", "language": "Ruby", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "reliability", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "baz": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "bugbot": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 1, "fp": 5, "fn": 0 }, "core": { "tp": 1, "fp": 5, "fn": 1 }, "all": { "tp": 1, "fp": 5, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "coderabbit": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 2 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "devin": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "sourcery": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 2 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "gitlab": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "gitar-v2": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } } } }, { "url": "https://github.com/grafana/grafana/pull/76186", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "baz": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "bugbot": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "claude-code": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 1, "fn": 3 }, "core": { "tp": 0, "fp": 1, "fn": 3 }, "all": { "tp": 0, "fp": 1, "fn": 3 } }, "coderabbit": { "strict": { "tp": 1, "fp": 3, "fn": 2 }, "core": { "tp": 1, "fp": 3, "fn": 2 }, "all": { "tp": 1, "fp": 3, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 0, "fn": 0 }, "core": { "tp": 3, "fp": 0, "fn": 0 }, "all": { "tp": 3, "fp": 0, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "sourcery": { "strict": { "tp": 0, "fp": 3, "fn": 3 }, "core": { "tp": 0, "fp": 3, "fn": 3 }, "all": { "tp": 0, "fp": 3, "fn": 3 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "gitlab": { "strict": { "tp": 3, "fp": 0, "fn": 0 }, "core": { "tp": 3, "fp": 0, "fn": 0 }, "all": { "tp": 3, "fp": 0, "fn": 0 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 2, "fn": 3 }, "core": { "tp": 0, "fp": 2, "fn": 3 }, "all": { "tp": 0, "fp": 2, "fn": 3 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 3, "fn": 2 }, "core": { "tp": 1, "fp": 3, "fn": 2 }, "all": { "tp": 1, "fp": 3, "fn": 2 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10", "language": "Ruby", "pr_size": "large", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 3, "fp": 2, "fn": 4 }, "core": { "tp": 3, "fp": 2, "fn": 4 }, "all": { "tp": 3, "fp": 2, "fn": 4 } }, "baz": { "strict": { "tp": 2, "fp": 0, "fn": 5 }, "core": { "tp": 2, "fp": 0, "fn": 5 }, "all": { "tp": 2, "fp": 0, "fn": 5 } }, "bugbot": { "strict": { "tp": 5, "fp": 2, "fn": 2 }, "core": { "tp": 5, "fp": 2, "fn": 2 }, "all": { "tp": 5, "fp": 2, "fn": 2 } }, "claude": { "strict": { "tp": 1, "fp": 2, "fn": 6 }, "core": { "tp": 1, "fp": 2, "fn": 6 }, "all": { "tp": 1, "fp": 2, "fn": 6 } }, "claude-code": { "strict": { "tp": 1, "fp": 5, "fn": 6 }, "core": { "tp": 1, "fp": 5, "fn": 6 }, "all": { "tp": 1, "fp": 5, "fn": 6 } }, "codeant-v2": { "strict": { "tp": 2, "fp": 7, "fn": 5 }, "core": { "tp": 2, "fp": 7, "fn": 5 }, "all": { "tp": 2, "fp": 7, "fn": 5 } }, "coderabbit": { "strict": { "tp": 4, "fp": 10, "fn": 3 }, "core": { "tp": 4, "fp": 10, "fn": 3 }, "all": { "tp": 4, "fp": 10, "fn": 3 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 1, "fn": 4 }, "core": { "tp": 3, "fp": 1, "fn": 4 }, "all": { "tp": 3, "fp": 1, "fn": 4 } }, "devin": { "strict": { "tp": 1, "fp": 1, "fn": 6 }, "core": { "tp": 1, "fp": 1, "fn": 6 }, "all": { "tp": 1, "fp": 1, "fn": 6 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 7 }, "core": { "tp": 0, "fp": 0, "fn": 7 }, "all": { "tp": 0, "fp": 0, "fn": 7 } }, "kg": { "strict": { "tp": 1, "fp": 0, "fn": 6 }, "core": { "tp": 1, "fp": 0, "fn": 6 }, "all": { "tp": 1, "fp": 0, "fn": 6 } }, "kodus-v2": { "strict": { "tp": 5, "fp": 3, "fn": 2 }, "core": { "tp": 5, "fp": 3, "fn": 2 }, "all": { "tp": 5, "fp": 3, "fn": 2 } }, "macroscope": { "strict": { "tp": 3, "fp": 4, "fn": 4 }, "core": { "tp": 3, "fp": 4, "fn": 4 }, "all": { "tp": 3, "fp": 4, "fn": 4 } }, "qodo-extended-v2": { "strict": { "tp": 5, "fp": 4, "fn": 2 }, "core": { "tp": 5, "fp": 4, "fn": 2 }, "all": { "tp": 5, "fp": 4, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 5, "fn": 5 }, "core": { "tp": 2, "fp": 5, "fn": 5 }, "all": { "tp": 2, "fp": 5, "fn": 5 } }, "sourcery": { "strict": { "tp": 4, "fp": 7, "fn": 3 }, "core": { "tp": 4, "fp": 7, "fn": 3 }, "all": { "tp": 4, "fp": 7, "fn": 3 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 2, "fn": 5 }, "core": { "tp": 2, "fp": 2, "fn": 5 }, "all": { "tp": 2, "fp": 2, "fn": 5 } }, "gemini-v2": { "strict": { "tp": 4, "fp": 2, "fn": 3 }, "core": { "tp": 4, "fp": 2, "fn": 3 }, "all": { "tp": 4, "fp": 2, "fn": 3 } }, "gitlab": { "strict": { "tp": 2, "fp": 3, "fn": 5 }, "core": { "tp": 2, "fp": 3, "fn": 5 }, "all": { "tp": 2, "fp": 3, "fn": 5 } }, "copilot-v2": { "strict": { "tp": 6, "fp": 5, "fn": 1 }, "core": { "tp": 6, "fp": 5, "fn": 1 }, "all": { "tp": 6, "fp": 5, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 6, "fp": 5, "fn": 1 }, "core": { "tp": 6, "fp": 5, "fn": 1 }, "all": { "tp": 6, "fp": 5, "fn": 1 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "bug_fix", "complexity": "simple", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 3, "fp": 2, "fn": 0 }, "core": { "tp": 3, "fp": 2, "fn": 0 }, "all": { "tp": 5, "fp": 2, "fn": 0 } }, "baz": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 4, "fp": 1, "fn": 1 } }, "bugbot": { "strict": { "tp": 3, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 5, "fp": 1, "fn": 0 } }, "claude": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 4, "fp": 2, "fn": 1 } }, "claude-code": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 4, "fp": 2, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 9, "fn": 3 }, "core": { "tp": 0, "fp": 9, "fn": 3 }, "all": { "tp": 0, "fp": 9, "fn": 5 } }, "coderabbit": { "strict": { "tp": 0, "fp": 2, "fn": 3 }, "core": { "tp": 0, "fp": 2, "fn": 3 }, "all": { "tp": 0, "fp": 2, "fn": 5 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 2 } }, "devin": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 4, "fp": 2, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "macroscope": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 4, "fp": 0, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 1 }, "all": { "tp": 3, "fp": 3, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 3, "fn": 2 }, "core": { "tp": 1, "fp": 3, "fn": 2 }, "all": { "tp": 2, "fp": 3, "fn": 3 } }, "sourcery": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 3 } }, "gemini-v2": { "strict": { "tp": 3, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 5, "fp": 1, "fn": 0 } }, "gitlab": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 3, "fp": 0, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 3, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 5, "fp": 1, "fn": 0 } }, "gitar-v2": { "strict": { "tp": 3, "fp": 2, "fn": 0 }, "core": { "tp": 3, "fp": 2, "fn": 0 }, "all": { "tp": 5, "fp": 2, "fn": 0 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8", "language": "Ruby", "pr_size": "medium", "domain": "API", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 3, "fp": 2, "fn": 3 }, "core": { "tp": 3, "fp": 2, "fn": 3 }, "all": { "tp": 3, "fp": 2, "fn": 3 } }, "baz": { "strict": { "tp": 2, "fp": 3, "fn": 4 }, "core": { "tp": 2, "fp": 3, "fn": 4 }, "all": { "tp": 2, "fp": 3, "fn": 4 } }, "bugbot": { "strict": { "tp": 1, "fp": 1, "fn": 5 }, "core": { "tp": 1, "fp": 1, "fn": 5 }, "all": { "tp": 1, "fp": 1, "fn": 5 } }, "claude": { "strict": { "tp": 4, "fp": 3, "fn": 2 }, "core": { "tp": 4, "fp": 3, "fn": 2 }, "all": { "tp": 4, "fp": 3, "fn": 2 } }, "claude-code": { "strict": { "tp": 2, "fp": 1, "fn": 4 }, "core": { "tp": 2, "fp": 1, "fn": 4 }, "all": { "tp": 2, "fp": 1, "fn": 4 } }, "codeant-v2": { "strict": { "tp": 4, "fp": 9, "fn": 2 }, "core": { "tp": 4, "fp": 9, "fn": 2 }, "all": { "tp": 4, "fp": 9, "fn": 2 } }, "coderabbit": { "strict": { "tp": 4, "fp": 6, "fn": 2 }, "core": { "tp": 4, "fp": 6, "fn": 2 }, "all": { "tp": 4, "fp": 6, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 1, "fn": 3 }, "core": { "tp": 3, "fp": 1, "fn": 3 }, "all": { "tp": 3, "fp": 1, "fn": 3 } }, "devin": { "strict": { "tp": 2, "fp": 0, "fn": 4 }, "core": { "tp": 2, "fp": 0, "fn": 4 }, "all": { "tp": 2, "fp": 0, "fn": 4 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 6 }, "core": { "tp": 0, "fp": 0, "fn": 6 }, "all": { "tp": 0, "fp": 0, "fn": 6 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 6 }, "core": { "tp": 0, "fp": 0, "fn": 6 }, "all": { "tp": 0, "fp": 0, "fn": 6 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 6, "fn": 5 }, "core": { "tp": 1, "fp": 6, "fn": 5 }, "all": { "tp": 1, "fp": 6, "fn": 5 } }, "macroscope": { "strict": { "tp": 1, "fp": 3, "fn": 5 }, "core": { "tp": 1, "fp": 3, "fn": 5 }, "all": { "tp": 1, "fp": 3, "fn": 5 } }, "qodo-extended-v2": { "strict": { "tp": 3, "fp": 1, "fn": 3 }, "core": { "tp": 3, "fp": 1, "fn": 3 }, "all": { "tp": 3, "fp": 1, "fn": 3 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 1, "fn": 4 }, "core": { "tp": 2, "fp": 1, "fn": 4 }, "all": { "tp": 2, "fp": 1, "fn": 4 } }, "sourcery": { "strict": { "tp": 3, "fp": 2, "fn": 3 }, "core": { "tp": 3, "fp": 2, "fn": 3 }, "all": { "tp": 3, "fp": 2, "fn": 3 } }, "greptile-v4-1": { "strict": { "tp": 3, "fp": 1, "fn": 3 }, "core": { "tp": 3, "fp": 1, "fn": 3 }, "all": { "tp": 3, "fp": 1, "fn": 3 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 2, "fn": 4 }, "core": { "tp": 2, "fp": 2, "fn": 4 }, "all": { "tp": 2, "fp": 2, "fn": 4 } }, "gitlab": { "strict": { "tp": 3, "fp": 2, "fn": 3 }, "core": { "tp": 3, "fp": 2, "fn": 3 }, "all": { "tp": 3, "fp": 2, "fn": 3 } }, "copilot-v2": { "strict": { "tp": 3, "fp": 14, "fn": 3 }, "core": { "tp": 3, "fp": 14, "fn": 3 }, "all": { "tp": 3, "fp": 14, "fn": 3 } }, "gitar-v2": { "strict": { "tp": 3, "fp": 6, "fn": 3 }, "core": { "tp": 3, "fp": 6, "fn": 3 }, "all": { "tp": 3, "fp": 6, "fn": 3 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3", "language": "Ruby", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 1, "fp": 4, "fn": 2 }, "core": { "tp": 1, "fp": 4, "fn": 2 }, "all": { "tp": 1, "fp": 4, "fn": 2 } }, "baz": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "bugbot": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "claude": { "strict": { "tp": 1, "fp": 6, "fn": 2 }, "core": { "tp": 1, "fp": 6, "fn": 2 }, "all": { "tp": 1, "fp": 6, "fn": 2 } }, "claude-code": { "strict": { "tp": 1, "fp": 4, "fn": 2 }, "core": { "tp": 1, "fp": 4, "fn": 2 }, "all": { "tp": 1, "fp": 4, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 3, "fn": 2 }, "core": { "tp": 1, "fp": 3, "fn": 2 }, "all": { "tp": 1, "fp": 3, "fn": 2 } }, "coderabbit": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "devin": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "sourcery": { "strict": { "tp": 1, "fp": 3, "fn": 2 }, "core": { "tp": 1, "fp": 3, "fn": 2 }, "all": { "tp": 1, "fp": 3, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 3, "fn": 2 }, "core": { "tp": 1, "fp": 3, "fn": 2 }, "all": { "tp": 1, "fp": 3, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 1 }, "all": { "tp": 2, "fp": 3, "fn": 1 } }, "gitlab": { "strict": { "tp": 0, "fp": 2, "fn": 3 }, "core": { "tp": 0, "fp": 2, "fn": 3 }, "all": { "tp": 0, "fp": 2, "fn": 3 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 6, "fn": 1 }, "core": { "tp": 2, "fp": 6, "fn": 1 }, "all": { "tp": 2, "fp": 6, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5", "language": "Ruby", "pr_size": "small", "domain": "UI", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "baz": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "bugbot": { "strict": { "tp": 0, "fp": 1, "fn": 3 }, "core": { "tp": 0, "fp": 1, "fn": 3 }, "all": { "tp": 0, "fp": 1, "fn": 3 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 1 }, "all": { "tp": 2, "fp": 3, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "coderabbit": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "devin": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "macroscope": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "sourcery": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 7, "fn": 1 }, "core": { "tp": 2, "fp": 7, "fn": 1 }, "all": { "tp": 2, "fp": 7, "fn": 1 } }, "gitlab": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 3, "fp": 8, "fn": 0 }, "core": { "tp": 3, "fp": 8, "fn": 0 }, "all": { "tp": 3, "fp": 8, "fn": 0 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6", "language": "Ruby", "pr_size": "small", "domain": "serialization", "change_type": "feature", "complexity": "simple", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 0 } }, "baz": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 0 } }, "bugbot": { "strict": { "tp": 1, "fp": 4, "fn": 0 }, "core": { "tp": 1, "fp": 4, "fn": 0 }, "all": { "tp": 1, "fp": 4, "fn": 0 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "claude-code": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 0 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 0 } }, "coderabbit": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 0 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 0 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 0 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 0 } }, "qodo-extended-v2": { "strict": { "tp": 0, "fp": 4, "fn": 1 }, "core": { "tp": 0, "fp": 4, "fn": 1 }, "all": { "tp": 0, "fp": 4, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 0 } }, "sourcery": { "strict": { "tp": 1, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 0 }, "all": { "tp": 1, "fp": 3, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 0 }, "all": { "tp": 1, "fp": 3, "fn": 0 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 5, "fn": 0 }, "core": { "tp": 1, "fp": 5, "fn": 0 }, "all": { "tp": 1, "fp": 5, "fn": 0 } }, "gitlab": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 0 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 0 }, "all": { "tp": 1, "fp": 3, "fn": 0 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 5, "fn": 0 }, "core": { "tp": 1, "fp": 5, "fn": 0 }, "all": { "tp": 1, "fp": 5, "fn": 0 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4", "language": "Ruby", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 4, "fp": 1, "fn": 4 }, "core": { "tp": 4, "fp": 1, "fn": 4 }, "all": { "tp": 4, "fp": 1, "fn": 4 } }, "baz": { "strict": { "tp": 2, "fp": 2, "fn": 6 }, "core": { "tp": 2, "fp": 2, "fn": 6 }, "all": { "tp": 2, "fp": 2, "fn": 6 } }, "bugbot": { "strict": { "tp": 2, "fp": 3, "fn": 6 }, "core": { "tp": 2, "fp": 3, "fn": 6 }, "all": { "tp": 2, "fp": 3, "fn": 6 } }, "claude": { "strict": { "tp": 6, "fp": 2, "fn": 2 }, "core": { "tp": 6, "fp": 2, "fn": 2 }, "all": { "tp": 6, "fp": 2, "fn": 2 } }, "claude-code": { "strict": { "tp": 4, "fp": 0, "fn": 4 }, "core": { "tp": 4, "fp": 0, "fn": 4 }, "all": { "tp": 4, "fp": 0, "fn": 4 } }, "codeant-v2": { "strict": { "tp": 3, "fp": 16, "fn": 5 }, "core": { "tp": 3, "fp": 16, "fn": 5 }, "all": { "tp": 3, "fp": 16, "fn": 5 } }, "coderabbit": { "strict": { "tp": 5, "fp": 11, "fn": 3 }, "core": { "tp": 5, "fp": 11, "fn": 3 }, "all": { "tp": 5, "fp": 11, "fn": 3 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 4, "fn": 7 }, "core": { "tp": 1, "fp": 4, "fn": 7 }, "all": { "tp": 1, "fp": 4, "fn": 7 } }, "devin": { "strict": { "tp": 4, "fp": 1, "fn": 4 }, "core": { "tp": 4, "fp": 1, "fn": 4 }, "all": { "tp": 4, "fp": 1, "fn": 4 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 8 }, "core": { "tp": 0, "fp": 0, "fn": 8 }, "all": { "tp": 0, "fp": 0, "fn": 8 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 8 }, "core": { "tp": 0, "fp": 0, "fn": 8 }, "all": { "tp": 0, "fp": 0, "fn": 8 } }, "kodus-v2": { "strict": { "tp": 3, "fp": 2, "fn": 5 }, "core": { "tp": 3, "fp": 2, "fn": 5 }, "all": { "tp": 3, "fp": 2, "fn": 5 } }, "macroscope": { "strict": { "tp": 2, "fp": 5, "fn": 6 }, "core": { "tp": 2, "fp": 5, "fn": 6 }, "all": { "tp": 2, "fp": 5, "fn": 6 } }, "qodo-extended-v2": { "strict": { "tp": 4, "fp": 1, "fn": 4 }, "core": { "tp": 4, "fp": 1, "fn": 4 }, "all": { "tp": 4, "fp": 1, "fn": 4 } }, "qodo-v2": { "strict": { "tp": 6, "fp": 3, "fn": 2 }, "core": { "tp": 6, "fp": 3, "fn": 2 }, "all": { "tp": 6, "fp": 3, "fn": 2 } }, "sourcery": { "strict": { "tp": 3, "fp": 2, "fn": 5 }, "core": { "tp": 3, "fp": 2, "fn": 5 }, "all": { "tp": 3, "fp": 2, "fn": 5 } }, "greptile-v4-1": { "strict": { "tp": 5, "fp": 1, "fn": 3 }, "core": { "tp": 5, "fp": 1, "fn": 3 }, "all": { "tp": 5, "fp": 1, "fn": 3 } }, "gemini-v2": { "strict": { "tp": 4, "fp": 6, "fn": 4 }, "core": { "tp": 4, "fp": 6, "fn": 4 }, "all": { "tp": 4, "fp": 6, "fn": 4 } }, "gitlab": { "strict": { "tp": 4, "fp": 1, "fn": 4 }, "core": { "tp": 4, "fp": 1, "fn": 4 }, "all": { "tp": 4, "fp": 1, "fn": 4 } }, "copilot-v2": { "strict": { "tp": 5, "fp": 5, "fn": 3 }, "core": { "tp": 5, "fp": 5, "fn": 3 }, "all": { "tp": 5, "fp": 5, "fn": 3 } }, "gitar-v2": { "strict": { "tp": 4, "fp": 1, "fn": 4 }, "core": { "tp": 4, "fp": 1, "fn": 4 }, "all": { "tp": 4, "fp": 1, "fn": 4 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1", "language": "Ruby", "pr_size": "medium", "domain": "file_io", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 2, "fp": 3, "fn": 2 }, "core": { "tp": 2, "fp": 3, "fn": 2 }, "all": { "tp": 2, "fp": 3, "fn": 2 } }, "baz": { "strict": { "tp": 2, "fp": 0, "fn": 2 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "bugbot": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "claude": { "strict": { "tp": 2, "fp": 0, "fn": 2 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 2, "fp": 2, "fn": 2 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 2 } }, "coderabbit": { "strict": { "tp": 2, "fp": 3, "fn": 2 }, "core": { "tp": 2, "fp": 3, "fn": 2 }, "all": { "tp": 2, "fp": 3, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 0, "fn": 1 }, "core": { "tp": 3, "fp": 0, "fn": 1 }, "all": { "tp": 3, "fp": 0, "fn": 1 } }, "devin": { "strict": { "tp": 2, "fp": 0, "fn": 2 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "kodus-v2": { "strict": { "tp": 3, "fp": 3, "fn": 1 }, "core": { "tp": 3, "fp": 3, "fn": 1 }, "all": { "tp": 3, "fp": 3, "fn": 1 } }, "macroscope": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 3 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 3, "fn": 2 }, "core": { "tp": 2, "fp": 3, "fn": 2 }, "all": { "tp": 2, "fp": 3, "fn": 2 } }, "sourcery": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 3, "fp": 0, "fn": 1 }, "core": { "tp": 3, "fp": 0, "fn": 1 }, "all": { "tp": 3, "fp": 0, "fn": 1 } }, "gitlab": { "strict": { "tp": 2, "fp": 0, "fn": 2 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "gitar-v2": { "strict": { "tp": 2, "fp": 4, "fn": 2 }, "core": { "tp": 2, "fp": 4, "fn": 2 }, "all": { "tp": 2, "fp": 4, "fn": 2 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "baz": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "bugbot": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "claude": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "claude-code": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 5, "fn": 0 }, "core": { "tp": 1, "fp": 5, "fn": 0 }, "all": { "tp": 1, "fp": 5, "fn": 1 } }, "coderabbit": { "strict": { "tp": 1, "fp": 6, "fn": 0 }, "core": { "tp": 1, "fp": 6, "fn": 0 }, "all": { "tp": 1, "fp": 6, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 0 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "sourcery": { "strict": { "tp": 1, "fp": 5, "fn": 0 }, "core": { "tp": 1, "fp": 5, "fn": 0 }, "all": { "tp": 1, "fp": 5, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "gitlab": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 5, "fn": 0 }, "core": { "tp": 1, "fp": 5, "fn": 0 }, "all": { "tp": 1, "fp": 5, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 5, "fn": 0 }, "core": { "tp": 1, "fp": 5, "fn": 0 }, "all": { "tp": 1, "fp": 5, "fn": 1 } } } }, { "url": "https://github.com/calcom/cal.com/pull/22532", "language": "TypeScript", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 4, "fp": 2, "fn": 0 }, "core": { "tp": 4, "fp": 2, "fn": 0 }, "all": { "tp": 4, "fp": 2, "fn": 0 } }, "baz": { "strict": { "tp": 1, "fp": 2, "fn": 3 }, "core": { "tp": 1, "fp": 2, "fn": 3 }, "all": { "tp": 1, "fp": 2, "fn": 3 } }, "bugbot": { "strict": { "tp": 0, "fp": 1, "fn": 4 }, "core": { "tp": 0, "fp": 1, "fn": 4 }, "all": { "tp": 0, "fp": 1, "fn": 4 } }, "claude": { "strict": { "tp": 0, "fp": 1, "fn": 4 }, "core": { "tp": 0, "fp": 1, "fn": 4 }, "all": { "tp": 0, "fp": 1, "fn": 4 } }, "claude-code": { "strict": { "tp": 0, "fp": 5, "fn": 4 }, "core": { "tp": 0, "fp": 5, "fn": 4 }, "all": { "tp": 0, "fp": 5, "fn": 4 } }, "codeant-v2": { "strict": { "tp": 3, "fp": 2, "fn": 1 }, "core": { "tp": 3, "fp": 2, "fn": 1 }, "all": { "tp": 3, "fp": 2, "fn": 1 } }, "coderabbit": { "strict": { "tp": 2, "fp": 3, "fn": 2 }, "core": { "tp": 2, "fp": 3, "fn": 2 }, "all": { "tp": 2, "fp": 3, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 0, "fp": 2, "fn": 4 }, "core": { "tp": 0, "fp": 2, "fn": 4 }, "all": { "tp": 0, "fp": 2, "fn": 4 } }, "devin": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 3 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 3 }, "core": { "tp": 1, "fp": 0, "fn": 3 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "macroscope": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 3 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 3, "fp": 2, "fn": 1 }, "core": { "tp": 3, "fp": 2, "fn": 1 }, "all": { "tp": 3, "fp": 2, "fn": 1 } }, "sourcery": { "strict": { "tp": 2, "fp": 2, "fn": 2 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 3, "fp": 2, "fn": 1 }, "core": { "tp": 3, "fp": 2, "fn": 1 }, "all": { "tp": 3, "fp": 2, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "gitlab": { "strict": { "tp": 2, "fp": 3, "fn": 2 }, "core": { "tp": 2, "fp": 3, "fn": 2 }, "all": { "tp": 2, "fp": 3, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 3, "fp": 3, "fn": 1 }, "core": { "tp": 3, "fp": 3, "fn": 1 }, "all": { "tp": 3, "fp": 3, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 2, "fp": 9, "fn": 2 }, "core": { "tp": 2, "fp": 9, "fn": 2 }, "all": { "tp": 2, "fp": 9, "fn": 2 } } } }, { "url": "https://github.com/calcom/cal.com/pull/8330", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 2, "fp": 3, "fn": 0 }, "core": { "tp": 2, "fp": 3, "fn": 0 }, "all": { "tp": 2, "fp": 3, "fn": 0 } }, "baz": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "bugbot": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "claude": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "claude-code": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "coderabbit": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "devin": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "graphite": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "kg": { "strict": { "tp": 0, "fp": 3, "fn": 2 }, "core": { "tp": 0, "fp": 3, "fn": 2 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "macroscope": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "sourcery": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 3, "fn": 0 }, "core": { "tp": 2, "fp": 3, "fn": 0 }, "all": { "tp": 2, "fp": 3, "fn": 0 } }, "gitlab": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 4, "fn": 0 }, "core": { "tp": 2, "fp": 4, "fn": 0 }, "all": { "tp": 2, "fp": 4, "fn": 0 } }, "gitar-v2": { "strict": { "tp": 2, "fp": 4, "fn": 0 }, "core": { "tp": 2, "fp": 4, "fn": 0 }, "all": { "tp": 2, "fp": 4, "fn": 0 } } } }, { "url": "https://github.com/calcom/cal.com/pull/14943", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "baz": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "bugbot": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude-code": { "strict": { "tp": 0, "fp": 1, "fn": 2 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "coderabbit": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "graphite": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "kg": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "sourcery": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "gitlab": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } } } }, { "url": "https://github.com/calcom/cal.com/pull/22345", "language": "TypeScript", "pr_size": "small", "domain": "database", "change_type": "migration", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "baz": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "bugbot": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 1, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 0 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "coderabbit": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "devin": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 1 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "sourcery": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 5, "fn": 1 }, "core": { "tp": 0, "fp": 5, "fn": 1 }, "all": { "tp": 0, "fp": 5, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "gitlab": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 5, "fn": 1 }, "core": { "tp": 0, "fp": 5, "fn": 1 }, "all": { "tp": 1, "fp": 5, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 5, "fn": 0 }, "core": { "tp": 1, "fp": 5, "fn": 0 }, "all": { "tp": 1, "fp": 5, "fn": 1 } } } }, { "url": "https://github.com/calcom/cal.com/pull/11059", "language": "TypeScript", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 8, "fp": 2, "fn": 1 }, "core": { "tp": 8, "fp": 2, "fn": 1 }, "all": { "tp": 8, "fp": 2, "fn": 1 } }, "baz": { "strict": { "tp": 5, "fp": 0, "fn": 4 }, "core": { "tp": 5, "fp": 0, "fn": 4 }, "all": { "tp": 5, "fp": 0, "fn": 4 } }, "bugbot": { "strict": { "tp": 6, "fp": 1, "fn": 3 }, "core": { "tp": 6, "fp": 1, "fn": 3 }, "all": { "tp": 6, "fp": 1, "fn": 3 } }, "claude": { "strict": { "tp": 5, "fp": 1, "fn": 4 }, "core": { "tp": 5, "fp": 1, "fn": 4 }, "all": { "tp": 5, "fp": 1, "fn": 4 } }, "claude-code": { "strict": { "tp": 4, "fp": 0, "fn": 5 }, "core": { "tp": 4, "fp": 0, "fn": 5 }, "all": { "tp": 4, "fp": 0, "fn": 5 } }, "codeant-v2": { "strict": { "tp": 4, "fp": 4, "fn": 5 }, "core": { "tp": 4, "fp": 4, "fn": 5 }, "all": { "tp": 4, "fp": 4, "fn": 5 } }, "coderabbit": { "strict": { "tp": 9, "fp": 9, "fn": 0 }, "core": { "tp": 9, "fp": 9, "fn": 0 }, "all": { "tp": 9, "fp": 9, "fn": 0 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 0, "fn": 6 }, "core": { "tp": 3, "fp": 0, "fn": 6 }, "all": { "tp": 3, "fp": 0, "fn": 6 } }, "devin": { "strict": { "tp": 5, "fp": 0, "fn": 4 }, "core": { "tp": 5, "fp": 0, "fn": 4 }, "all": { "tp": 5, "fp": 0, "fn": 4 } }, "graphite": { "strict": { "tp": 1, "fp": 0, "fn": 8 }, "core": { "tp": 1, "fp": 0, "fn": 8 }, "all": { "tp": 1, "fp": 0, "fn": 8 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 9 }, "core": { "tp": 0, "fp": 0, "fn": 9 }, "all": { "tp": 0, "fp": 0, "fn": 9 } }, "kodus-v2": { "strict": { "tp": 4, "fp": 3, "fn": 5 }, "core": { "tp": 4, "fp": 3, "fn": 5 }, "all": { "tp": 4, "fp": 3, "fn": 5 } }, "macroscope": { "strict": { "tp": 5, "fp": 5, "fn": 4 }, "core": { "tp": 5, "fp": 5, "fn": 4 }, "all": { "tp": 5, "fp": 5, "fn": 4 } }, "qodo-extended-v2": { "strict": { "tp": 7, "fp": 2, "fn": 2 }, "core": { "tp": 7, "fp": 2, "fn": 2 }, "all": { "tp": 7, "fp": 2, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 8, "fp": 3, "fn": 1 }, "core": { "tp": 8, "fp": 3, "fn": 1 }, "all": { "tp": 8, "fp": 3, "fn": 1 } }, "sourcery": { "strict": { "tp": 5, "fp": 0, "fn": 4 }, "core": { "tp": 5, "fp": 0, "fn": 4 }, "all": { "tp": 5, "fp": 0, "fn": 4 } }, "greptile-v4-1": { "strict": { "tp": 5, "fp": 3, "fn": 4 }, "core": { "tp": 5, "fp": 3, "fn": 4 }, "all": { "tp": 5, "fp": 3, "fn": 4 } }, "gemini-v2": { "strict": { "tp": 4, "fp": 2, "fn": 5 }, "core": { "tp": 4, "fp": 2, "fn": 5 }, "all": { "tp": 4, "fp": 2, "fn": 5 } }, "gitlab": { "strict": { "tp": 5, "fp": 2, "fn": 4 }, "core": { "tp": 5, "fp": 2, "fn": 4 }, "all": { "tp": 5, "fp": 2, "fn": 4 } }, "copilot-v2": { "strict": { "tp": 6, "fp": 1, "fn": 3 }, "core": { "tp": 6, "fp": 1, "fn": 3 }, "all": { "tp": 6, "fp": 1, "fn": 3 } }, "gitar-v2": { "strict": { "tp": 8, "fp": 3, "fn": 1 }, "core": { "tp": 8, "fp": 3, "fn": 1 }, "all": { "tp": 8, "fp": 3, "fn": 1 } } } }, { "url": "https://github.com/calcom/cal.com/pull/7232", "language": "TypeScript", "pr_size": "medium", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "baz": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "bugbot": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "claude": { "strict": { "tp": 1, "fp": 3, "fn": 2 }, "core": { "tp": 1, "fp": 3, "fn": 2 }, "all": { "tp": 1, "fp": 3, "fn": 2 } }, "claude-code": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "coderabbit": { "strict": { "tp": 2, "fp": 7, "fn": 1 }, "core": { "tp": 2, "fp": 7, "fn": 1 }, "all": { "tp": 2, "fp": 7, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 1 }, "all": { "tp": 2, "fp": 3, "fn": 1 } }, "devin": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 0, "fp": 1, "fn": 3 }, "core": { "tp": 0, "fp": 1, "fn": 3 }, "all": { "tp": 0, "fp": 1, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "macroscope": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "sourcery": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 1 }, "all": { "tp": 2, "fp": 3, "fn": 1 } }, "gitlab": { "strict": { "tp": 0, "fp": 2, "fn": 3 }, "core": { "tp": 0, "fp": 2, "fn": 3 }, "all": { "tp": 0, "fp": 2, "fn": 3 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 3, "fn": 3 }, "core": { "tp": 0, "fp": 3, "fn": 3 }, "all": { "tp": 0, "fp": 3, "fn": 3 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } } } }, { "url": "https://github.com/calcom/cal.com/pull/14740", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 4, "fp": 3, "fn": 2 }, "core": { "tp": 4, "fp": 3, "fn": 2 }, "all": { "tp": 4, "fp": 3, "fn": 2 } }, "baz": { "strict": { "tp": 1, "fp": 1, "fn": 5 }, "core": { "tp": 1, "fp": 1, "fn": 5 }, "all": { "tp": 1, "fp": 1, "fn": 5 } }, "bugbot": { "strict": { "tp": 3, "fp": 3, "fn": 3 }, "core": { "tp": 3, "fp": 3, "fn": 3 }, "all": { "tp": 3, "fp": 3, "fn": 3 } }, "claude": { "strict": { "tp": 2, "fp": 0, "fn": 4 }, "core": { "tp": 2, "fp": 0, "fn": 4 }, "all": { "tp": 2, "fp": 0, "fn": 4 } }, "claude-code": { "strict": { "tp": 2, "fp": 1, "fn": 4 }, "core": { "tp": 2, "fp": 1, "fn": 4 }, "all": { "tp": 2, "fp": 1, "fn": 4 } }, "codeant-v2": { "strict": { "tp": 3, "fp": 4, "fn": 3 }, "core": { "tp": 3, "fp": 4, "fn": 3 }, "all": { "tp": 3, "fp": 4, "fn": 3 } }, "coderabbit": { "strict": { "tp": 4, "fp": 4, "fn": 2 }, "core": { "tp": 4, "fp": 4, "fn": 2 }, "all": { "tp": 4, "fp": 4, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 2, "fn": 3 }, "core": { "tp": 3, "fp": 2, "fn": 3 }, "all": { "tp": 3, "fp": 2, "fn": 3 } }, "devin": { "strict": { "tp": 2, "fp": 0, "fn": 4 }, "core": { "tp": 2, "fp": 0, "fn": 4 }, "all": { "tp": 2, "fp": 0, "fn": 4 } }, "graphite": { "strict": { "tp": 1, "fp": 0, "fn": 5 }, "core": { "tp": 1, "fp": 0, "fn": 5 }, "all": { "tp": 1, "fp": 0, "fn": 5 } }, "kg": { "strict": { "tp": 1, "fp": 1, "fn": 5 }, "core": { "tp": 1, "fp": 1, "fn": 5 }, "all": { "tp": 1, "fp": 1, "fn": 5 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 0, "fn": 4 }, "core": { "tp": 2, "fp": 0, "fn": 4 }, "all": { "tp": 2, "fp": 0, "fn": 4 } }, "macroscope": { "strict": { "tp": 3, "fp": 3, "fn": 3 }, "core": { "tp": 3, "fp": 3, "fn": 3 }, "all": { "tp": 3, "fp": 3, "fn": 3 } }, "qodo-extended-v2": { "strict": { "tp": 3, "fp": 2, "fn": 3 }, "core": { "tp": 3, "fp": 2, "fn": 3 }, "all": { "tp": 3, "fp": 2, "fn": 3 } }, "qodo-v2": { "strict": { "tp": 4, "fp": 2, "fn": 2 }, "core": { "tp": 4, "fp": 2, "fn": 2 }, "all": { "tp": 4, "fp": 2, "fn": 2 } }, "sourcery": { "strict": { "tp": 3, "fp": 2, "fn": 3 }, "core": { "tp": 3, "fp": 2, "fn": 3 }, "all": { "tp": 3, "fp": 2, "fn": 3 } }, "greptile-v4-1": { "strict": { "tp": 3, "fp": 2, "fn": 3 }, "core": { "tp": 3, "fp": 2, "fn": 3 }, "all": { "tp": 3, "fp": 2, "fn": 3 } }, "gemini-v2": { "strict": { "tp": 3, "fp": 4, "fn": 3 }, "core": { "tp": 3, "fp": 4, "fn": 3 }, "all": { "tp": 3, "fp": 4, "fn": 3 } }, "gitlab": { "strict": { "tp": 2, "fp": 5, "fn": 4 }, "core": { "tp": 2, "fp": 5, "fn": 4 }, "all": { "tp": 2, "fp": 5, "fn": 4 } }, "copilot-v2": { "strict": { "tp": 5, "fp": 5, "fn": 1 }, "core": { "tp": 5, "fp": 5, "fn": 1 }, "all": { "tp": 5, "fp": 5, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 5, "fp": 7, "fn": 1 }, "core": { "tp": 5, "fp": 7, "fn": 1 }, "all": { "tp": 5, "fp": 7, "fn": 1 } } } }, { "url": "https://github.com/calcom/cal.com/pull/10600", "language": "TypeScript", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 0, "fp": 3, "fn": 2 }, "core": { "tp": 1, "fp": 3, "fn": 3 }, "all": { "tp": 2, "fp": 3, "fn": 3 } }, "baz": { "strict": { "tp": 0, "fp": 1, "fn": 2 }, "core": { "tp": 0, "fp": 1, "fn": 4 }, "all": { "tp": 0, "fp": 1, "fn": 5 } }, "bugbot": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 3 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "claude": { "strict": { "tp": 0, "fp": 6, "fn": 2 }, "core": { "tp": 1, "fp": 6, "fn": 3 }, "all": { "tp": 2, "fp": 6, "fn": 3 } }, "claude-code": { "strict": { "tp": 0, "fp": 4, "fn": 2 }, "core": { "tp": 0, "fp": 4, "fn": 4 }, "all": { "tp": 1, "fp": 4, "fn": 4 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 3 }, "all": { "tp": 1, "fp": 2, "fn": 4 } }, "coderabbit": { "strict": { "tp": 1, "fp": 6, "fn": 1 }, "core": { "tp": 1, "fp": 6, "fn": 3 }, "all": { "tp": 2, "fp": 6, "fn": 3 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 3 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 3 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 1, "fn": 2 }, "core": { "tp": 0, "fp": 1, "fn": 4 }, "all": { "tp": 0, "fp": 1, "fn": 5 } }, "macroscope": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 0, "fp": 2, "fn": 4 }, "all": { "tp": 0, "fp": 2, "fn": 5 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 3 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 2 }, "all": { "tp": 2, "fp": 3, "fn": 3 } }, "sourcery": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 3, "fp": 2, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 3, "fp": 1, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 5, "fn": 2 }, "core": { "tp": 0, "fp": 5, "fn": 4 }, "all": { "tp": 1, "fp": 5, "fn": 4 } }, "gitlab": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 3 }, "all": { "tp": 2, "fp": 2, "fn": 3 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 4, "fn": 0 }, "core": { "tp": 4, "fp": 4, "fn": 0 }, "all": { "tp": 5, "fp": 4, "fn": 0 } }, "gitar-v2": { "strict": { "tp": 0, "fp": 6, "fn": 2 }, "core": { "tp": 0, "fp": 6, "fn": 4 }, "all": { "tp": 0, "fp": 6, "fn": 5 } } } }, { "url": "https://github.com/calcom/cal.com/pull/10967", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "bug_fix", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 3, "fp": 2, "fn": 2 }, "core": { "tp": 3, "fp": 2, "fn": 2 }, "all": { "tp": 3, "fp": 2, "fn": 3 } }, "baz": { "strict": { "tp": 1, "fp": 1, "fn": 4 }, "core": { "tp": 1, "fp": 1, "fn": 4 }, "all": { "tp": 1, "fp": 1, "fn": 5 } }, "bugbot": { "strict": { "tp": 2, "fp": 4, "fn": 3 }, "core": { "tp": 2, "fp": 4, "fn": 3 }, "all": { "tp": 2, "fp": 4, "fn": 4 } }, "claude": { "strict": { "tp": 4, "fp": 3, "fn": 1 }, "core": { "tp": 4, "fp": 3, "fn": 1 }, "all": { "tp": 4, "fp": 3, "fn": 2 } }, "claude-code": { "strict": { "tp": 3, "fp": 5, "fn": 2 }, "core": { "tp": 3, "fp": 5, "fn": 2 }, "all": { "tp": 3, "fp": 5, "fn": 3 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 7, "fn": 4 }, "core": { "tp": 1, "fp": 7, "fn": 4 }, "all": { "tp": 1, "fp": 7, "fn": 5 } }, "coderabbit": { "strict": { "tp": 5, "fp": 5, "fn": 0 }, "core": { "tp": 5, "fp": 5, "fn": 0 }, "all": { "tp": 5, "fp": 5, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 2, "fn": 4 }, "core": { "tp": 1, "fp": 2, "fn": 4 }, "all": { "tp": 1, "fp": 2, "fn": 5 } }, "devin": { "strict": { "tp": 3, "fp": 1, "fn": 2 }, "core": { "tp": 3, "fp": 1, "fn": 2 }, "all": { "tp": 3, "fp": 1, "fn": 3 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 5 }, "core": { "tp": 0, "fp": 0, "fn": 5 }, "all": { "tp": 0, "fp": 0, "fn": 6 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 5 }, "core": { "tp": 0, "fp": 0, "fn": 5 }, "all": { "tp": 0, "fp": 0, "fn": 6 } }, "kodus-v2": { "strict": { "tp": 3, "fp": 1, "fn": 2 }, "core": { "tp": 3, "fp": 1, "fn": 2 }, "all": { "tp": 3, "fp": 1, "fn": 3 } }, "macroscope": { "strict": { "tp": 3, "fp": 2, "fn": 2 }, "core": { "tp": 3, "fp": 2, "fn": 2 }, "all": { "tp": 3, "fp": 2, "fn": 3 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 4, "fn": 4 }, "core": { "tp": 1, "fp": 4, "fn": 4 }, "all": { "tp": 1, "fp": 4, "fn": 5 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 3, "fn": 3 }, "core": { "tp": 2, "fp": 3, "fn": 3 }, "all": { "tp": 2, "fp": 3, "fn": 4 } }, "sourcery": { "strict": { "tp": 2, "fp": 5, "fn": 3 }, "core": { "tp": 2, "fp": 5, "fn": 3 }, "all": { "tp": 2, "fp": 5, "fn": 4 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 2, "fn": 4 }, "core": { "tp": 1, "fp": 2, "fn": 4 }, "all": { "tp": 1, "fp": 2, "fn": 5 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 6, "fn": 4 }, "core": { "tp": 1, "fp": 6, "fn": 4 }, "all": { "tp": 1, "fp": 6, "fn": 5 } }, "gitlab": { "strict": { "tp": 3, "fp": 3, "fn": 2 }, "core": { "tp": 3, "fp": 3, "fn": 2 }, "all": { "tp": 3, "fp": 3, "fn": 3 } }, "copilot-v2": { "strict": { "tp": 4, "fp": 7, "fn": 1 }, "core": { "tp": 4, "fp": 7, "fn": 1 }, "all": { "tp": 4, "fp": 7, "fn": 2 } }, "gitar-v2": { "strict": { "tp": 2, "fp": 5, "fn": 3 }, "core": { "tp": 2, "fp": 5, "fn": 3 }, "all": { "tp": 2, "fp": 5, "fn": 4 } } } }, { "url": "https://github.com/calcom/cal.com/pull/8087", "language": "TypeScript", "pr_size": "medium", "domain": "concurrency", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": { "augment": { "strict": { "tp": 1, "fp": 6, "fn": 0 }, "core": { "tp": 1, "fp": 6, "fn": 0 }, "all": { "tp": 1, "fp": 6, "fn": 1 } }, "baz": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "bugbot": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "claude-code": { "strict": { "tp": 1, "fp": 6, "fn": 0 }, "core": { "tp": 1, "fp": 6, "fn": 0 }, "all": { "tp": 1, "fp": 6, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 4, "fn": 0 }, "core": { "tp": 1, "fp": 4, "fn": 0 }, "all": { "tp": 1, "fp": 4, "fn": 1 } }, "coderabbit": { "strict": { "tp": 1, "fp": 5, "fn": 0 }, "core": { "tp": 1, "fp": 5, "fn": 0 }, "all": { "tp": 1, "fp": 5, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 9, "fn": 0 }, "core": { "tp": 1, "fp": 9, "fn": 0 }, "all": { "tp": 2, "fp": 9, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 0 }, "all": { "tp": 2, "fp": 3, "fn": 0 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 1, "fp": 4, "fn": 0 }, "core": { "tp": 1, "fp": 4, "fn": 0 }, "all": { "tp": 1, "fp": 4, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "sourcery": { "strict": { "tp": 1, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 0 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "gitlab": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "gitar-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } } } } ], "tools": [ "augment", "baz", "bugbot", "claude", "claude-code", "codeant-v2", "coderabbit", "copilot-v2", "cubic-v2", "devin", "gemini-v2", "gitar-v2", "gitlab", "graphite", "greptile-v4-1", "kg", "kodus-v2", "macroscope", "qodo-extended-v2", "qodo-v2", "sourcery" ], "dimensions": { "language": [ "Go", "Java", "Python", "Ruby", "TypeScript" ], "pr_size": [ "small", "medium", "large" ], "domain": [ "API", "UI", "authentication", "caching", "concurrency", "configuration", "data_processing", "database", "file_io", "logging", "scheduling", "serialization", "testing" ], "change_type": [ "bug_fix", "feature", "migration", "performance", "refactoring", "security_patch" ], "complexity": [ "simple", "moderate", "complex" ], "difficulty": [ "obvious", "moderate", "subtle", "very_subtle" ], "risk": [ "low", "medium", "high", "critical" ], "context": [ "local", "file", "cross_file", "system" ], "concern": [ "correctness", "maintainability", "reliability", "security" ] }, "overall_metrics": { "coderabbit": { "strict": { "precision": 31.8, "recall": 60.4, "f1": 41.7, "f2": 51.2, "tp": 84, "fp": 180, "fn": 55, "num_prs": 50 }, "core": { "precision": 33.8, "recall": 58.2, "f1": 42.8, "f2": 50.9, "tp": 92, "fp": 180, "fn": 66, "num_prs": 50 }, "all": { "precision": 35.3, "recall": 56.6, "f1": 43.5, "f2": 50.5, "tp": 98, "fp": 180, "fn": 75, "num_prs": 50 } }, "sourcery": { "strict": { "precision": 34.2, "recall": 48.9, "f1": 40.2, "f2": 45.0, "tp": 68, "fp": 131, "fn": 71, "num_prs": 50 }, "core": { "precision": 37.0, "recall": 48.7, "f1": 42.1, "f2": 45.8, "tp": 77, "fp": 131, "fn": 81, "num_prs": 50 }, "all": { "precision": 38.8, "recall": 48.0, "f1": 42.9, "f2": 45.8, "tp": 83, "fp": 131, "fn": 90, "num_prs": 50 } }, "cubic-v2": { "strict": { "precision": 52.9, "recall": 59.7, "f1": 56.1, "f2": 58.2, "tp": 83, "fp": 74, "fn": 56, "num_prs": 50 }, "core": { "precision": 56.5, "recall": 60.8, "f1": 58.5, "f2": 59.9, "tp": 96, "fp": 74, "fn": 62, "num_prs": 50 }, "all": { "precision": 57.2, "recall": 57.2, "f1": 57.2, "f2": 57.2, "tp": 99, "fp": 74, "fn": 74, "num_prs": 50 } }, "graphite": { "strict": { "precision": 100.0, "recall": 7.9, "f1": 14.7, "f2": 9.7, "tp": 11, "fp": 0, "fn": 128, "num_prs": 50 }, "core": { "precision": 100.0, "recall": 7.6, "f1": 14.1, "f2": 9.3, "tp": 12, "fp": 0, "fn": 146, "num_prs": 50 }, "all": { "precision": 100.0, "recall": 6.9, "f1": 13.0, "f2": 8.5, "tp": 12, "fp": 0, "fn": 161, "num_prs": 50 } }, "claude-code": { "strict": { "precision": 34.3, "recall": 42.4, "f1": 37.9, "f2": 40.5, "tp": 59, "fp": 113, "fn": 80, "num_prs": 50 }, "core": { "precision": 36.5, "recall": 41.1, "f1": 38.7, "f2": 40.1, "tp": 65, "fp": 113, "fn": 93, "num_prs": 50 }, "all": { "precision": 38.3, "recall": 40.5, "f1": 39.3, "f2": 40.0, "tp": 70, "fp": 113, "fn": 103, "num_prs": 50 } }, "kodus-v2": { "strict": { "precision": 55.2, "recall": 38.1, "f1": 45.1, "f2": 40.6, "tp": 53, "fp": 43, "fn": 86, "num_prs": 50 }, "core": { "precision": 57.4, "recall": 36.7, "f1": 44.8, "f2": 39.6, "tp": 58, "fp": 43, "fn": 100, "num_prs": 50 }, "all": { "precision": 57.8, "recall": 34.1, "f1": 42.9, "f2": 37.2, "tp": 59, "fp": 43, "fn": 114, "num_prs": 50 } }, "devin": { "strict": { "precision": 66.0, "recall": 44.6, "f1": 53.2, "f2": 47.7, "tp": 62, "fp": 32, "fn": 77, "num_prs": 50 }, "core": { "precision": 67.3, "recall": 41.8, "f1": 51.6, "f2": 45.2, "tp": 66, "fp": 32, "fn": 92, "num_prs": 50 }, "all": { "precision": 68.3, "recall": 39.9, "f1": 50.4, "f2": 43.5, "tp": 69, "fp": 32, "fn": 104, "num_prs": 50 } }, "claude": { "strict": { "precision": 36.1, "recall": 37.4, "f1": 36.7, "f2": 37.1, "tp": 52, "fp": 92, "fn": 87, "num_prs": 50 }, "core": { "precision": 38.3, "recall": 36.1, "f1": 37.1, "f2": 36.5, "tp": 57, "fp": 92, "fn": 101, "num_prs": 50 }, "all": { "precision": 40.6, "recall": 36.4, "f1": 38.4, "f2": 37.2, "tp": 63, "fp": 92, "fn": 110, "num_prs": 50 } }, "macroscope": { "strict": { "precision": 50.4, "recall": 45.3, "f1": 47.7, "f2": 46.3, "tp": 63, "fp": 62, "fn": 76, "num_prs": 50 }, "core": { "precision": 51.6, "recall": 41.8, "f1": 46.2, "f2": 43.4, "tp": 66, "fp": 62, "fn": 92, "num_prs": 50 }, "all": { "precision": 53.0, "recall": 40.5, "f1": 45.9, "f2": 42.5, "tp": 70, "fp": 62, "fn": 103, "num_prs": 50 } }, "baz": { "strict": { "precision": 54.6, "recall": 38.1, "f1": 44.9, "f2": 40.6, "tp": 53, "fp": 44, "fn": 86, "num_prs": 50 }, "core": { "precision": 55.6, "recall": 34.8, "f1": 42.8, "f2": 37.6, "tp": 55, "fp": 44, "fn": 103, "num_prs": 50 }, "all": { "precision": 56.9, "recall": 33.5, "f1": 42.2, "f2": 36.5, "tp": 58, "fp": 44, "fn": 115, "num_prs": 50 } }, "augment": { "strict": { "precision": 49.4, "recall": 61.9, "f1": 55.0, "f2": 58.9, "tp": 86, "fp": 88, "fn": 53, "num_prs": 50 }, "core": { "precision": 52.4, "recall": 61.4, "f1": 56.6, "f2": 59.4, "tp": 97, "fp": 88, "fn": 61, "num_prs": 50 }, "all": { "precision": 53.7, "recall": 59.0, "f1": 56.2, "f2": 57.8, "tp": 102, "fp": 88, "fn": 71, "num_prs": 50 } }, "greptile-v4-1": { "strict": { "precision": 44.9, "recall": 50.4, "f1": 47.5, "f2": 49.2, "tp": 70, "fp": 86, "fn": 69, "num_prs": 50 }, "core": { "precision": 46.9, "recall": 48.1, "f1": 47.5, "f2": 47.9, "tp": 76, "fp": 86, "fn": 82, "num_prs": 50 }, "all": { "precision": 49.1, "recall": 48.0, "f1": 48.5, "f2": 48.2, "tp": 83, "fp": 86, "fn": 90, "num_prs": 50 } }, "bugbot": { "strict": { "precision": 52.0, "recall": 46.0, "f1": 48.9, "f2": 47.1, "tp": 64, "fp": 59, "fn": 75, "num_prs": 50 }, "core": { "precision": 54.3, "recall": 44.3, "f1": 48.8, "f2": 46.0, "tp": 70, "fp": 59, "fn": 88, "num_prs": 50 }, "all": { "precision": 55.3, "recall": 42.2, "f1": 47.9, "f2": 44.3, "tp": 73, "fp": 59, "fn": 100, "num_prs": 50 } }, "copilot-v2": { "strict": { "precision": 31.4, "recall": 61.9, "f1": 41.6, "f2": 51.8, "tp": 86, "fp": 188, "fn": 53, "num_prs": 50 }, "core": { "precision": 34.3, "recall": 62.0, "f1": 44.1, "f2": 53.4, "tp": 98, "fp": 188, "fn": 60, "num_prs": 50 }, "all": { "precision": 36.1, "recall": 61.3, "f1": 45.4, "f2": 53.8, "tp": 106, "fp": 188, "fn": 67, "num_prs": 50 } }, "codeant-v2": { "strict": { "precision": 31.1, "recall": 37.4, "f1": 34.0, "f2": 36.0, "tp": 52, "fp": 115, "fn": 87, "num_prs": 50 }, "core": { "precision": 32.4, "recall": 34.8, "f1": 33.5, "f2": 34.3, "tp": 55, "fp": 115, "fn": 103, "num_prs": 50 }, "all": { "precision": 33.1, "recall": 32.9, "f1": 33.0, "f2": 33.0, "tp": 57, "fp": 115, "fn": 116, "num_prs": 50 } }, "qodo-extended-v2": { "strict": { "precision": 53.6, "recall": 58.3, "f1": 55.9, "f2": 57.3, "tp": 81, "fp": 70, "fn": 58, "num_prs": 50 }, "core": { "precision": 56.2, "recall": 57.0, "f1": 56.6, "f2": 56.8, "tp": 90, "fp": 70, "fn": 68, "num_prs": 50 }, "all": { "precision": 56.8, "recall": 53.2, "f1": 54.9, "f2": 53.9, "tp": 92, "fp": 70, "fn": 81, "num_prs": 50 } }, "gemini-v2": { "strict": { "precision": 36.7, "recall": 51.8, "f1": 43.0, "f2": 47.9, "tp": 72, "fp": 124, "fn": 67, "num_prs": 50 }, "core": { "precision": 39.2, "recall": 50.6, "f1": 44.2, "f2": 47.8, "tp": 80, "fp": 124, "fn": 78, "num_prs": 50 }, "all": { "precision": 41.2, "recall": 50.3, "f1": 45.3, "f2": 48.2, "tp": 87, "fp": 124, "fn": 86, "num_prs": 50 } }, "qodo-v2": { "strict": { "precision": 45.1, "recall": 59.0, "f1": 51.1, "f2": 55.6, "tp": 82, "fp": 100, "fn": 57, "num_prs": 50 }, "core": { "precision": 47.1, "recall": 56.3, "f1": 51.3, "f2": 54.2, "tp": 89, "fp": 100, "fn": 69, "num_prs": 50 }, "all": { "precision": 47.9, "recall": 53.2, "f1": 50.4, "f2": 52.0, "tp": 92, "fp": 100, "fn": 81, "num_prs": 50 } }, "gitar-v2": { "strict": { "precision": 31.4, "recall": 64.7, "f1": 42.3, "f2": 53.4, "tp": 90, "fp": 197, "fn": 49, "num_prs": 50 }, "core": { "precision": 33.4, "recall": 62.7, "f1": 43.6, "f2": 53.3, "tp": 99, "fp": 197, "fn": 59, "num_prs": 50 }, "all": { "precision": 34.8, "recall": 60.7, "f1": 44.2, "f2": 52.8, "tp": 105, "fp": 197, "fn": 68, "num_prs": 50 } }, "gitlab": { "strict": { "precision": 44.7, "recall": 48.9, "f1": 46.7, "f2": 48.0, "tp": 68, "fp": 84, "fn": 71, "num_prs": 50 }, "core": { "precision": 47.5, "recall": 48.1, "f1": 47.8, "f2": 48.0, "tp": 76, "fp": 84, "fn": 82, "num_prs": 50 }, "all": { "precision": 49.7, "recall": 48.0, "f1": 48.8, "f2": 48.3, "tp": 83, "fp": 84, "fn": 90, "num_prs": 50 } }, "kg": { "strict": { "precision": 48.8, "recall": 15.1, "f1": 23.1, "f2": 17.5, "tp": 21, "fp": 22, "fn": 118, "num_prs": 50 }, "core": { "precision": 50.0, "recall": 13.9, "f1": 21.8, "f2": 16.3, "tp": 22, "fp": 22, "fn": 136, "num_prs": 50 }, "all": { "precision": 51.1, "recall": 13.3, "f1": 21.1, "f2": 15.6, "tp": 23, "fp": 22, "fn": 150, "num_prs": 50 } } } }, "openai_gpt-5.2": { "prs": [ { "url": "https://github.com/keycloak/keycloak/pull/37429", "language": "Java", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 3 }, "all": { "tp": 0, "fp": 2, "fn": 5 } }, "coderabbit": { "strict": { "tp": 0, "fp": 6, "fn": 1 }, "core": { "tp": 2, "fp": 6, "fn": 1 }, "all": { "tp": 4, "fp": 6, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 3 }, "all": { "tp": 2, "fp": 3, "fn": 3 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 5, "fn": 1 }, "core": { "tp": 1, "fp": 5, "fn": 2 }, "all": { "tp": 3, "fp": 5, "fn": 2 } }, "gitlab": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 6, "fn": 1 }, "core": { "tp": 2, "fp": 6, "fn": 1 }, "all": { "tp": 3, "fp": 6, "fn": 2 } }, "augment": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 1 }, "all": { "tp": 2, "fp": 3, "fn": 3 } }, "bugbot": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 3 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "claude-code": { "strict": { "tp": 0, "fp": 4, "fn": 1 }, "core": { "tp": 0, "fp": 4, "fn": 3 }, "all": { "tp": 2, "fp": 4, "fn": 3 } }, "cubic-v2": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 4 } }, "devin": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 2 }, "all": { "tp": 1, "fp": 3, "fn": 4 } }, "macroscope": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 3 }, "all": { "tp": 0, "fp": 2, "fn": 5 } }, "qodo-v2": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 3 }, "all": { "tp": 0, "fp": 3, "fn": 5 } }, "sourcery": { "strict": { "tp": 0, "fp": 13, "fn": 1 }, "core": { "tp": 0, "fp": 13, "fn": 3 }, "all": { "tp": 2, "fp": 13, "fn": 3 } } } }, { "url": "https://github.com/keycloak/keycloak/pull/37634", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 2, "fp": 5, "fn": 1 }, "core": { "tp": 2, "fp": 5, "fn": 2 }, "all": { "tp": 2, "fp": 5, "fn": 2 } }, "coderabbit": { "strict": { "tp": 3, "fp": 7, "fn": 0 }, "core": { "tp": 3, "fp": 7, "fn": 1 }, "all": { "tp": 3, "fp": 7, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 3, "fp": 0, "fn": 1 }, "all": { "tp": 3, "fp": 0, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 2 } }, "gitlab": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 3, "fp": 3, "fn": 1 }, "all": { "tp": 3, "fp": 3, "fn": 1 } }, "augment": { "strict": { "tp": 1, "fp": 5, "fn": 2 }, "core": { "tp": 1, "fp": 5, "fn": 3 }, "all": { "tp": 1, "fp": 5, "fn": 3 } }, "bugbot": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 2 } }, "claude": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 2 } }, "claude-code": { "strict": { "tp": 0, "fp": 2, "fn": 3 }, "core": { "tp": 0, "fp": 2, "fn": 4 }, "all": { "tp": 0, "fp": 2, "fn": 4 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "devin": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "graphite": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 3 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "kg": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 3 }, "all": { "tp": 1, "fp": 2, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "sourcery": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 2 } } } }, { "url": "https://github.com/keycloak/keycloak/pull/38446", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 1, "fn": 2 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 1, "fn": 2 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "coderabbit": { "strict": { "tp": 2, "fp": 3, "fn": 0 }, "core": { "tp": 2, "fp": 3, "fn": 0 }, "all": { "tp": 2, "fp": 3, "fn": 0 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 4, "fn": 2 }, "core": { "tp": 0, "fp": 4, "fn": 2 }, "all": { "tp": 0, "fp": 4, "fn": 2 } }, "gitlab": { "strict": { "tp": 0, "fp": 8, "fn": 2 }, "core": { "tp": 0, "fp": 8, "fn": 2 }, "all": { "tp": 0, "fp": 8, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 8, "fn": 1 }, "core": { "tp": 1, "fp": 8, "fn": 1 }, "all": { "tp": 1, "fp": 8, "fn": 1 } }, "augment": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "bugbot": { "strict": { "tp": 0, "fp": 3, "fn": 2 }, "core": { "tp": 0, "fp": 3, "fn": 2 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "claude": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "claude-code": { "strict": { "tp": 1, "fp": 5, "fn": 1 }, "core": { "tp": 1, "fp": 5, "fn": 1 }, "all": { "tp": 1, "fp": 5, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "devin": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "macroscope": { "strict": { "tp": 0, "fp": 4, "fn": 2 }, "core": { "tp": 0, "fp": 4, "fn": 2 }, "all": { "tp": 0, "fp": 4, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 3, "fn": 0 }, "core": { "tp": 2, "fp": 3, "fn": 0 }, "all": { "tp": 2, "fp": 3, "fn": 0 } }, "sourcery": { "strict": { "tp": 0, "fp": 7, "fn": 2 }, "core": { "tp": 0, "fp": 7, "fn": 2 }, "all": { "tp": 0, "fp": 7, "fn": 2 } } } }, { "url": "https://github.com/keycloak/keycloak/pull/36882", "language": "Java", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "coderabbit": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "gitlab": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 2 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "augment": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "bugbot": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "devin": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "macroscope": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "sourcery": { "strict": { "tp": 0, "fp": 4, "fn": 1 }, "core": { "tp": 0, "fp": 4, "fn": 2 }, "all": { "tp": 0, "fp": 4, "fn": 2 } } } }, { "url": "https://github.com/keycloak/keycloak/pull/36880", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "codeant-v2": { "strict": { "tp": 2, "fp": 0, "fn": 2 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 3 } }, "coderabbit": { "strict": { "tp": 1, "fp": 5, "fn": 3 }, "core": { "tp": 1, "fp": 5, "fn": 3 }, "all": { "tp": 2, "fp": 5, "fn": 3 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 0, "fn": 2 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 3 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 3 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 3, "fn": 3 }, "core": { "tp": 1, "fp": 3, "fn": 3 }, "all": { "tp": 2, "fp": 3, "fn": 3 } }, "gitlab": { "strict": { "tp": 1, "fp": 6, "fn": 3 }, "core": { "tp": 1, "fp": 6, "fn": 3 }, "all": { "tp": 2, "fp": 6, "fn": 3 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 7, "fn": 3 }, "core": { "tp": 1, "fp": 7, "fn": 3 }, "all": { "tp": 1, "fp": 7, "fn": 4 } }, "augment": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 3 }, "all": { "tp": 1, "fp": 1, "fn": 4 } }, "bugbot": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 3, "fp": 1, "fn": 2 } }, "claude": { "strict": { "tp": 1, "fp": 4, "fn": 3 }, "core": { "tp": 1, "fp": 4, "fn": 3 }, "all": { "tp": 2, "fp": 4, "fn": 3 } }, "claude-code": { "strict": { "tp": 0, "fp": 2, "fn": 4 }, "core": { "tp": 0, "fp": 2, "fn": 4 }, "all": { "tp": 0, "fp": 2, "fn": 5 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 3 }, "all": { "tp": 1, "fp": 1, "fn": 4 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 3 }, "core": { "tp": 1, "fp": 0, "fn": 3 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "macroscope": { "strict": { "tp": 2, "fp": 4, "fn": 2 }, "core": { "tp": 2, "fp": 4, "fn": 2 }, "all": { "tp": 2, "fp": 4, "fn": 3 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 3, "fn": 3 }, "core": { "tp": 1, "fp": 3, "fn": 3 }, "all": { "tp": 1, "fp": 3, "fn": 4 } }, "sourcery": { "strict": { "tp": 1, "fp": 4, "fn": 3 }, "core": { "tp": 1, "fp": 4, "fn": 3 }, "all": { "tp": 2, "fp": 4, "fn": 3 } } } }, { "url": "https://github.com/keycloak/keycloak/pull/37038", "language": "Java", "pr_size": "small", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 5, "fn": 1 }, "core": { "tp": 1, "fp": 5, "fn": 1 }, "all": { "tp": 1, "fp": 5, "fn": 1 } }, "coderabbit": { "strict": { "tp": 2, "fp": 8, "fn": 0 }, "core": { "tp": 2, "fp": 8, "fn": 0 }, "all": { "tp": 2, "fp": 8, "fn": 0 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "gitlab": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 3, "fn": 0 }, "core": { "tp": 2, "fp": 3, "fn": 0 }, "all": { "tp": 2, "fp": 3, "fn": 0 } }, "augment": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "bugbot": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude-code": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "macroscope": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "sourcery": { "strict": { "tp": 1, "fp": 4, "fn": 1 }, "core": { "tp": 1, "fp": 4, "fn": 1 }, "all": { "tp": 1, "fp": 4, "fn": 1 } } } }, { "url": "https://github.com/keycloak/keycloak/pull/33832", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "coderabbit": { "strict": { "tp": 2, "fp": 4, "fn": 0 }, "core": { "tp": 3, "fp": 4, "fn": 0 }, "all": { "tp": 4, "fp": 4, "fn": 0 } }, "qodo-extended-v2": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 3 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 3, "fn": 2 }, "core": { "tp": 1, "fp": 3, "fn": 2 }, "all": { "tp": 1, "fp": 3, "fn": 3 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "gitlab": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 3 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 6, "fn": 1 }, "core": { "tp": 2, "fp": 6, "fn": 1 }, "all": { "tp": 3, "fp": 6, "fn": 1 } }, "augment": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 3 } }, "bugbot": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "claude": { "strict": { "tp": 1, "fp": 4, "fn": 1 }, "core": { "tp": 2, "fp": 4, "fn": 1 }, "all": { "tp": 2, "fp": 4, "fn": 2 } }, "claude-code": { "strict": { "tp": 0, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 2 } }, "devin": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "macroscope": { "strict": { "tp": 2, "fp": 4, "fn": 0 }, "core": { "tp": 2, "fp": 4, "fn": 1 }, "all": { "tp": 2, "fp": 4, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 3 } }, "sourcery": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 2 } } } }, { "url": "https://github.com/keycloak/keycloak/pull/40940", "language": "Java", "pr_size": "small", "domain": "concurrency", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "coderabbit": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "gitlab": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "augment": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "bugbot": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude-code": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "graphite": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "macroscope": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "sourcery": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } } } }, { "url": "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "coderabbit": { "strict": { "tp": 1, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 0 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "gitlab": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "augment": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "bugbot": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude-code": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "devin": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "macroscope": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "sourcery": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } } } }, { "url": "https://github.com/getsentry/sentry/pull/93824", "language": "Python", "pr_size": "large", "domain": "concurrency", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 2, "fn": 3 }, "core": { "tp": 0, "fp": 2, "fn": 5 }, "all": { "tp": 0, "fp": 2, "fn": 5 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 1, "fn": 3 }, "core": { "tp": 2, "fp": 1, "fn": 3 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "coderabbit": { "strict": { "tp": 1, "fp": 3, "fn": 2 }, "core": { "tp": 1, "fp": 3, "fn": 4 }, "all": { "tp": 1, "fp": 3, "fn": 4 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 3 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 4 }, "all": { "tp": 1, "fp": 1, "fn": 4 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 3 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "gitlab": { "strict": { "tp": 0, "fp": 2, "fn": 3 }, "core": { "tp": 0, "fp": 2, "fn": 5 }, "all": { "tp": 0, "fp": 2, "fn": 5 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 4, "fn": 2 }, "core": { "tp": 3, "fp": 4, "fn": 2 }, "all": { "tp": 3, "fp": 4, "fn": 2 } }, "augment": { "strict": { "tp": 3, "fp": 1, "fn": 0 }, "core": { "tp": 5, "fp": 1, "fn": 0 }, "all": { "tp": 5, "fp": 1, "fn": 0 } }, "bugbot": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 4 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "claude": { "strict": { "tp": 0, "fp": 3, "fn": 3 }, "core": { "tp": 0, "fp": 3, "fn": 5 }, "all": { "tp": 0, "fp": 3, "fn": 5 } }, "claude-code": { "strict": { "tp": 3, "fp": 3, "fn": 0 }, "core": { "tp": 3, "fp": 3, "fn": 2 }, "all": { "tp": 3, "fp": 3, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 1, "fn": 0 }, "core": { "tp": 5, "fp": 1, "fn": 0 }, "all": { "tp": 5, "fp": 1, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 4 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "graphite": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 4 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 5 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 5 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "macroscope": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 3, "fp": 1, "fn": 2 }, "all": { "tp": 3, "fp": 1, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 4, "fp": 3, "fn": 1 }, "all": { "tp": 4, "fp": 3, "fn": 1 } }, "sourcery": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 4, "fp": 2, "fn": 1 }, "all": { "tp": 4, "fp": 2, "fn": 1 } } } }, { "url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5", "language": "Python", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 3, "fp": 1, "fn": 3 }, "core": { "tp": 3, "fp": 1, "fn": 3 }, "all": { "tp": 3, "fp": 1, "fn": 3 } }, "codeant-v2": { "strict": { "tp": 2, "fp": 5, "fn": 4 }, "core": { "tp": 2, "fp": 5, "fn": 4 }, "all": { "tp": 2, "fp": 5, "fn": 4 } }, "coderabbit": { "strict": { "tp": 4, "fp": 13, "fn": 2 }, "core": { "tp": 4, "fp": 13, "fn": 2 }, "all": { "tp": 4, "fp": 13, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 0, "fn": 4 }, "core": { "tp": 2, "fp": 0, "fn": 4 }, "all": { "tp": 2, "fp": 0, "fn": 4 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 0, "fn": 5 }, "core": { "tp": 1, "fp": 0, "fn": 5 }, "all": { "tp": 1, "fp": 0, "fn": 5 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 1, "fn": 4 }, "core": { "tp": 2, "fp": 1, "fn": 4 }, "all": { "tp": 2, "fp": 1, "fn": 4 } }, "gitlab": { "strict": { "tp": 2, "fp": 1, "fn": 4 }, "core": { "tp": 2, "fp": 1, "fn": 4 }, "all": { "tp": 2, "fp": 1, "fn": 4 } }, "copilot-v2": { "strict": { "tp": 3, "fp": 5, "fn": 3 }, "core": { "tp": 3, "fp": 5, "fn": 3 }, "all": { "tp": 3, "fp": 5, "fn": 3 } }, "augment": { "strict": { "tp": 3, "fp": 2, "fn": 3 }, "core": { "tp": 3, "fp": 2, "fn": 3 }, "all": { "tp": 3, "fp": 2, "fn": 3 } }, "bugbot": { "strict": { "tp": 2, "fp": 0, "fn": 4 }, "core": { "tp": 2, "fp": 0, "fn": 4 }, "all": { "tp": 2, "fp": 0, "fn": 4 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 6 }, "core": { "tp": 0, "fp": 0, "fn": 6 }, "all": { "tp": 0, "fp": 0, "fn": 6 } }, "claude-code": { "strict": { "tp": 0, "fp": 2, "fn": 6 }, "core": { "tp": 0, "fp": 2, "fn": 6 }, "all": { "tp": 0, "fp": 2, "fn": 6 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 2, "fn": 4 }, "core": { "tp": 2, "fp": 2, "fn": 4 }, "all": { "tp": 2, "fp": 2, "fn": 4 } }, "devin": { "strict": { "tp": 2, "fp": 1, "fn": 4 }, "core": { "tp": 2, "fp": 1, "fn": 4 }, "all": { "tp": 2, "fp": 1, "fn": 4 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 6 }, "core": { "tp": 0, "fp": 0, "fn": 6 }, "all": { "tp": 0, "fp": 0, "fn": 6 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 6 }, "core": { "tp": 0, "fp": 0, "fn": 6 }, "all": { "tp": 0, "fp": 0, "fn": 6 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 1, "fn": 4 }, "core": { "tp": 2, "fp": 1, "fn": 4 }, "all": { "tp": 2, "fp": 1, "fn": 4 } }, "macroscope": { "strict": { "tp": 2, "fp": 3, "fn": 4 }, "core": { "tp": 2, "fp": 3, "fn": 4 }, "all": { "tp": 2, "fp": 3, "fn": 4 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 0, "fn": 4 }, "core": { "tp": 2, "fp": 0, "fn": 4 }, "all": { "tp": 2, "fp": 0, "fn": 4 } }, "sourcery": { "strict": { "tp": 2, "fp": 2, "fn": 4 }, "core": { "tp": 2, "fp": 2, "fn": 4 }, "all": { "tp": 2, "fp": 2, "fn": 4 } } } }, { "url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1", "language": "Python", "pr_size": "medium", "domain": "API", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 3 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 3 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "coderabbit": { "strict": { "tp": 3, "fp": 2, "fn": 1 }, "core": { "tp": 3, "fp": 2, "fn": 1 }, "all": { "tp": 3, "fp": 2, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 3, "fp": 0, "fn": 1 }, "core": { "tp": 3, "fp": 0, "fn": 1 }, "all": { "tp": 3, "fp": 0, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 3, "fp": 2, "fn": 1 }, "core": { "tp": 3, "fp": 2, "fn": 1 }, "all": { "tp": 3, "fp": 2, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 3, "fp": 2, "fn": 1 }, "core": { "tp": 3, "fp": 2, "fn": 1 }, "all": { "tp": 3, "fp": 2, "fn": 1 } }, "gitlab": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 2, "fn": 2 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 2 } }, "augment": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "bugbot": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "claude": { "strict": { "tp": 2, "fp": 6, "fn": 2 }, "core": { "tp": 2, "fp": 6, "fn": 2 }, "all": { "tp": 2, "fp": 6, "fn": 2 } }, "claude-code": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 0, "fn": 1 }, "core": { "tp": 3, "fp": 0, "fn": 1 }, "all": { "tp": 3, "fp": 0, "fn": 1 } }, "devin": { "strict": { "tp": 3, "fp": 2, "fn": 1 }, "core": { "tp": 3, "fp": 2, "fn": 1 }, "all": { "tp": 3, "fp": 2, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "kg": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 3 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 4, "fn": 3 }, "core": { "tp": 1, "fp": 4, "fn": 3 }, "all": { "tp": 1, "fp": 4, "fn": 3 } }, "macroscope": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "sourcery": { "strict": { "tp": 2, "fp": 3, "fn": 2 }, "core": { "tp": 2, "fp": 3, "fn": 2 }, "all": { "tp": 2, "fp": 3, "fn": 2 } } } }, { "url": "https://github.com/grafana/grafana/pull/97529", "language": "Go", "pr_size": "small", "domain": "concurrency", "change_type": "performance", "complexity": "complex", "difficulty": "very_subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "coderabbit": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "gitlab": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 7, "fn": 1 }, "core": { "tp": 1, "fp": 7, "fn": 1 }, "all": { "tp": 1, "fp": 7, "fn": 1 } }, "augment": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "bugbot": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude-code": { "strict": { "tp": 1, "fp": 4, "fn": 1 }, "core": { "tp": 1, "fp": 4, "fn": 1 }, "all": { "tp": 1, "fp": 4, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "devin": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "macroscope": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 4, "fn": 0 }, "core": { "tp": 2, "fp": 4, "fn": 0 }, "all": { "tp": 2, "fp": 4, "fn": 0 } }, "sourcery": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } } } }, { "url": "https://github.com/getsentry/sentry/pull/80168", "language": "Python", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 3 }, "all": { "tp": 0, "fp": 1, "fn": 3 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 3 }, "all": { "tp": 0, "fp": 1, "fn": 3 } }, "coderabbit": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "gitlab": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "augment": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 3, "fp": 1, "fn": 0 } }, "bugbot": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "claude-code": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 3, "fp": 1, "fn": 0 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 3, "fp": 1, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "macroscope": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "sourcery": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } } } }, { "url": "https://github.com/getsentry/sentry/pull/80528", "language": "Python", "pr_size": "small", "domain": "scheduling", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 4, "fn": 1 }, "core": { "tp": 0, "fp": 4, "fn": 2 }, "all": { "tp": 0, "fp": 4, "fn": 2 } }, "coderabbit": { "strict": { "tp": 1, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 5, "fn": 0 }, "core": { "tp": 1, "fp": 5, "fn": 1 }, "all": { "tp": 1, "fp": 5, "fn": 1 } }, "gitlab": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "augment": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "bugbot": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "graphite": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "kg": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "sourcery": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } } } }, { "url": "https://github.com/getsentry/sentry/pull/77754", "language": "Python", "pr_size": "medium", "domain": "serialization", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "coderabbit": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 3, "fp": 2, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 3 } }, "gitlab": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "augment": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "bugbot": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "kg": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "sourcery": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 2, "fp": 3, "fn": 2 } } } }, { "url": "https://github.com/getsentry/sentry/pull/95633", "language": "Python", "pr_size": "medium", "domain": "concurrency", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 0, "fp": 1, "fn": 1 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "coderabbit": { "strict": { "tp": 0, "fp": 7, "fn": 0 }, "core": { "tp": 0, "fp": 7, "fn": 1 }, "all": { "tp": 0, "fp": 7, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 0, "fp": 8, "fn": 0 }, "core": { "tp": 0, "fp": 8, "fn": 1 }, "all": { "tp": 0, "fp": 8, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 5, "fn": 0 }, "core": { "tp": 0, "fp": 5, "fn": 1 }, "all": { "tp": 0, "fp": 5, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 11, "fn": 0 }, "core": { "tp": 0, "fp": 11, "fn": 1 }, "all": { "tp": 0, "fp": 11, "fn": 2 } }, "gitlab": { "strict": { "tp": 0, "fp": 5, "fn": 0 }, "core": { "tp": 1, "fp": 5, "fn": 0 }, "all": { "tp": 1, "fp": 5, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 6, "fn": 0 }, "core": { "tp": 0, "fp": 6, "fn": 1 }, "all": { "tp": 0, "fp": 6, "fn": 2 } }, "augment": { "strict": { "tp": 0, "fp": 4, "fn": 0 }, "core": { "tp": 1, "fp": 4, "fn": 0 }, "all": { "tp": 1, "fp": 4, "fn": 1 } }, "bugbot": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "devin": { "strict": { "tp": 0, "fp": 4, "fn": 0 }, "core": { "tp": 0, "fp": 4, "fn": 1 }, "all": { "tp": 0, "fp": 4, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 0, "fp": 4, "fn": 0 }, "core": { "tp": 0, "fp": 4, "fn": 1 }, "all": { "tp": 0, "fp": 4, "fn": 2 } }, "sourcery": { "strict": { "tp": 0, "fp": 4, "fn": 0 }, "core": { "tp": 1, "fp": 4, "fn": 0 }, "all": { "tp": 1, "fp": 4, "fn": 1 } } } }, { "url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2", "language": "Python", "pr_size": "medium", "domain": "data_processing", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 3, "fp": 2, "fn": 1 }, "core": { "tp": 3, "fp": 2, "fn": 1 }, "all": { "tp": 3, "fp": 2, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "coderabbit": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "qodo-extended-v2": { "strict": { "tp": 3, "fp": 4, "fn": 1 }, "core": { "tp": 3, "fp": 4, "fn": 1 }, "all": { "tp": 3, "fp": 4, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 4, "fn": 2 }, "core": { "tp": 2, "fp": 4, "fn": 2 }, "all": { "tp": 2, "fp": 4, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 3, "fp": 5, "fn": 1 }, "core": { "tp": 3, "fp": 5, "fn": 1 }, "all": { "tp": 3, "fp": 5, "fn": 1 } }, "gitlab": { "strict": { "tp": 4, "fp": 1, "fn": 0 }, "core": { "tp": 4, "fp": 1, "fn": 0 }, "all": { "tp": 4, "fp": 1, "fn": 0 } }, "copilot-v2": { "strict": { "tp": 3, "fp": 10, "fn": 1 }, "core": { "tp": 3, "fp": 10, "fn": 1 }, "all": { "tp": 3, "fp": 10, "fn": 1 } }, "augment": { "strict": { "tp": 3, "fp": 2, "fn": 1 }, "core": { "tp": 3, "fp": 2, "fn": 1 }, "all": { "tp": 3, "fp": 2, "fn": 1 } }, "bugbot": { "strict": { "tp": 4, "fp": 0, "fn": 0 }, "core": { "tp": 4, "fp": 0, "fn": 0 }, "all": { "tp": 4, "fp": 0, "fn": 0 } }, "claude": { "strict": { "tp": 3, "fp": 5, "fn": 1 }, "core": { "tp": 3, "fp": 5, "fn": 1 }, "all": { "tp": 3, "fp": 5, "fn": 1 } }, "claude-code": { "strict": { "tp": 3, "fp": 2, "fn": 1 }, "core": { "tp": 3, "fp": 2, "fn": 1 }, "all": { "tp": 3, "fp": 2, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 0, "fn": 1 }, "core": { "tp": 3, "fp": 0, "fn": 1 }, "all": { "tp": 3, "fp": 0, "fn": 1 } }, "devin": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "kg": { "strict": { "tp": 2, "fp": 2, "fn": 2 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 3, "fn": 2 }, "core": { "tp": 2, "fp": 3, "fn": 2 }, "all": { "tp": 2, "fp": 3, "fn": 2 } }, "macroscope": { "strict": { "tp": 3, "fp": 0, "fn": 1 }, "core": { "tp": 3, "fp": 0, "fn": 1 }, "all": { "tp": 3, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 4, "fp": 0, "fn": 0 }, "core": { "tp": 4, "fp": 0, "fn": 0 }, "all": { "tp": 4, "fp": 0, "fn": 0 } }, "sourcery": { "strict": { "tp": 3, "fp": 7, "fn": 1 }, "core": { "tp": 3, "fp": 7, "fn": 1 }, "all": { "tp": 3, "fp": 7, "fn": 1 } } } }, { "url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3", "language": "Python", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 6, "fn": 3 }, "core": { "tp": 0, "fp": 6, "fn": 3 }, "all": { "tp": 0, "fp": 6, "fn": 3 } }, "codeant-v2": { "strict": { "tp": 3, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 3, "fp": 1, "fn": 0 } }, "coderabbit": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 1 }, "all": { "tp": 2, "fp": 3, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 3, "fp": 0, "fn": 0 }, "core": { "tp": 3, "fp": 0, "fn": 0 }, "all": { "tp": 3, "fp": 0, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "gitlab": { "strict": { "tp": 2, "fp": 4, "fn": 1 }, "core": { "tp": 2, "fp": 4, "fn": 1 }, "all": { "tp": 2, "fp": 4, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 9, "fn": 2 }, "core": { "tp": 1, "fp": 9, "fn": 2 }, "all": { "tp": 1, "fp": 9, "fn": 2 } }, "augment": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "bugbot": { "strict": { "tp": 0, "fp": 3, "fn": 3 }, "core": { "tp": 0, "fp": 3, "fn": 3 }, "all": { "tp": 0, "fp": 3, "fn": 3 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "devin": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "macroscope": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "sourcery": { "strict": { "tp": 2, "fp": 6, "fn": 1 }, "core": { "tp": 2, "fp": 6, "fn": 1 }, "all": { "tp": 2, "fp": 6, "fn": 1 } } } }, { "url": "https://github.com/grafana/grafana/pull/103633", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "coderabbit": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 3, "fn": 2 }, "core": { "tp": 0, "fp": 3, "fn": 3 }, "all": { "tp": 0, "fp": 3, "fn": 3 } }, "gitlab": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "augment": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "bugbot": { "strict": { "tp": 0, "fp": 1, "fn": 2 }, "core": { "tp": 0, "fp": 1, "fn": 3 }, "all": { "tp": 0, "fp": 1, "fn": 3 } }, "claude": { "strict": { "tp": 0, "fp": 6, "fn": 2 }, "core": { "tp": 0, "fp": 6, "fn": 3 }, "all": { "tp": 0, "fp": 6, "fn": 3 } }, "claude-code": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 0, "fp": 2, "fn": 3 }, "all": { "tp": 0, "fp": 2, "fn": 3 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "devin": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 1, "fn": 2 }, "core": { "tp": 0, "fp": 1, "fn": 3 }, "all": { "tp": 0, "fp": 1, "fn": 3 } }, "macroscope": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "sourcery": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 0, "fp": 2, "fn": 3 }, "all": { "tp": 0, "fp": 2, "fn": 3 } } } }, { "url": "https://github.com/getsentry/sentry/pull/67876", "language": "Python", "pr_size": "small", "domain": "authentication", "change_type": "security_patch", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 2, "fn": 3 }, "core": { "tp": 0, "fp": 2, "fn": 3 }, "all": { "tp": 0, "fp": 2, "fn": 3 } }, "coderabbit": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "gitlab": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 13, "fn": 1 }, "core": { "tp": 2, "fp": 13, "fn": 1 }, "all": { "tp": 2, "fp": 13, "fn": 1 } }, "augment": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "bugbot": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "claude": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "claude-code": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 2, "fn": 0 }, "core": { "tp": 3, "fp": 2, "fn": 0 }, "all": { "tp": 3, "fp": 2, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "macroscope": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 5, "fn": 1 }, "core": { "tp": 2, "fp": 5, "fn": 1 }, "all": { "tp": 2, "fp": 5, "fn": 1 } }, "sourcery": { "strict": { "tp": 1, "fp": 8, "fn": 2 }, "core": { "tp": 1, "fp": 8, "fn": 2 }, "all": { "tp": 1, "fp": 8, "fn": 2 } } } }, { "url": "https://github.com/keycloak/keycloak/pull/32918", "language": "Java", "pr_size": "small", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "coderabbit": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 3, "fn": 2 }, "core": { "tp": 0, "fp": 3, "fn": 2 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "gitlab": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "augment": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "bugbot": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "claude": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "claude-code": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "devin": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "graphite": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "kg": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "sourcery": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } } } }, { "url": "https://github.com/grafana/grafana/pull/94942", "language": "Go", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "coderabbit": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "gitlab": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "augment": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "bugbot": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "claude": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "claude-code": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "devin": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "sourcery": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } } } }, { "url": "https://github.com/grafana/grafana/pull/90939", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "coderabbit": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "gitlab": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "augment": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "bugbot": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude-code": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "sourcery": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } } } }, { "url": "https://github.com/grafana/grafana/pull/80329", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "performance", "complexity": "simple", "difficulty": "obvious", "risk": "low", "context": "local", "concern": "maintainability", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "coderabbit": { "strict": { "tp": 0, "fp": 4, "fn": 0 }, "core": { "tp": 2, "fp": 4, "fn": 0 }, "all": { "tp": 2, "fp": 4, "fn": 0 } }, "qodo-extended-v2": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 2, "fp": 3, "fn": 0 }, "all": { "tp": 2, "fp": 3, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 2, "fp": 3, "fn": 0 }, "all": { "tp": 2, "fp": 3, "fn": 0 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 2, "fp": 3, "fn": 0 }, "all": { "tp": 2, "fp": 3, "fn": 0 } }, "gitlab": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 4, "fn": 0 }, "core": { "tp": 2, "fp": 4, "fn": 0 }, "all": { "tp": 2, "fp": 4, "fn": 0 } }, "augment": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "bugbot": { "strict": { "tp": 0, "fp": 4, "fn": 0 }, "core": { "tp": 2, "fp": 4, "fn": 0 }, "all": { "tp": 2, "fp": 4, "fn": 0 } }, "claude": { "strict": { "tp": 0, "fp": 4, "fn": 0 }, "core": { "tp": 2, "fp": 4, "fn": 0 }, "all": { "tp": 2, "fp": 4, "fn": 0 } }, "claude-code": { "strict": { "tp": 0, "fp": 4, "fn": 0 }, "core": { "tp": 2, "fp": 4, "fn": 0 }, "all": { "tp": 2, "fp": 4, "fn": 0 } }, "cubic-v2": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "devin": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 2, "fp": 3, "fn": 0 }, "all": { "tp": 2, "fp": 3, "fn": 0 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "macroscope": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 0, "fp": 5, "fn": 0 }, "core": { "tp": 2, "fp": 5, "fn": 0 }, "all": { "tp": 2, "fp": 5, "fn": 0 } }, "sourcery": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } } } }, { "url": "https://github.com/grafana/grafana/pull/90045", "language": "Go", "pr_size": "medium", "domain": "logging", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 1 }, "all": { "tp": 2, "fp": 3, "fn": 1 } }, "coderabbit": { "strict": { "tp": 3, "fp": 2, "fn": 0 }, "core": { "tp": 3, "fp": 2, "fn": 0 }, "all": { "tp": 3, "fp": 2, "fn": 0 } }, "qodo-extended-v2": { "strict": { "tp": 3, "fp": 4, "fn": 0 }, "core": { "tp": 3, "fp": 4, "fn": 0 }, "all": { "tp": 3, "fp": 4, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 3, "fp": 3, "fn": 0 }, "core": { "tp": 3, "fp": 3, "fn": 0 }, "all": { "tp": 3, "fp": 3, "fn": 0 } }, "gemini-v2": { "strict": { "tp": 3, "fp": 5, "fn": 0 }, "core": { "tp": 3, "fp": 5, "fn": 0 }, "all": { "tp": 3, "fp": 5, "fn": 0 } }, "gitlab": { "strict": { "tp": 3, "fp": 2, "fn": 0 }, "core": { "tp": 3, "fp": 2, "fn": 0 }, "all": { "tp": 3, "fp": 2, "fn": 0 } }, "copilot-v2": { "strict": { "tp": 3, "fp": 5, "fn": 0 }, "core": { "tp": 3, "fp": 5, "fn": 0 }, "all": { "tp": 3, "fp": 5, "fn": 0 } }, "augment": { "strict": { "tp": 2, "fp": 6, "fn": 1 }, "core": { "tp": 2, "fp": 6, "fn": 1 }, "all": { "tp": 2, "fp": 6, "fn": 1 } }, "bugbot": { "strict": { "tp": 3, "fp": 3, "fn": 0 }, "core": { "tp": 3, "fp": 3, "fn": 0 }, "all": { "tp": 3, "fp": 3, "fn": 0 } }, "claude": { "strict": { "tp": 3, "fp": 4, "fn": 0 }, "core": { "tp": 3, "fp": 4, "fn": 0 }, "all": { "tp": 3, "fp": 4, "fn": 0 } }, "claude-code": { "strict": { "tp": 3, "fp": 2, "fn": 0 }, "core": { "tp": 3, "fp": 2, "fn": 0 }, "all": { "tp": 3, "fp": 2, "fn": 0 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 2, "fn": 0 }, "core": { "tp": 3, "fp": 2, "fn": 0 }, "all": { "tp": 3, "fp": 2, "fn": 0 } }, "devin": { "strict": { "tp": 3, "fp": 2, "fn": 0 }, "core": { "tp": 3, "fp": 2, "fn": 0 }, "all": { "tp": 3, "fp": 2, "fn": 0 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 6, "fn": 1 }, "core": { "tp": 2, "fp": 6, "fn": 1 }, "all": { "tp": 2, "fp": 6, "fn": 1 } }, "sourcery": { "strict": { "tp": 3, "fp": 0, "fn": 0 }, "core": { "tp": 3, "fp": 0, "fn": 0 }, "all": { "tp": 3, "fp": 0, "fn": 0 } } } }, { "url": "https://github.com/grafana/grafana/pull/106778", "language": "Go", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "coderabbit": { "strict": { "tp": 1, "fp": 7, "fn": 1 }, "core": { "tp": 1, "fp": 7, "fn": 1 }, "all": { "tp": 1, "fp": 7, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 3, "fn": 2 }, "core": { "tp": 0, "fp": 3, "fn": 2 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "gitlab": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "augment": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "bugbot": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "claude": { "strict": { "tp": 0, "fp": 9, "fn": 2 }, "core": { "tp": 0, "fp": 9, "fn": 2 }, "all": { "tp": 0, "fp": 9, "fn": 2 } }, "claude-code": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "devin": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "sourcery": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } } } }, { "url": "https://github.com/grafana/grafana/pull/107534", "language": "Go", "pr_size": "small", "domain": "testing", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "low", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "coderabbit": { "strict": { "tp": 0, "fp": 1, "fn": 0 }, "core": { "tp": 0, "fp": 1, "fn": 1 }, "all": { "tp": 0, "fp": 1, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 0, "fp": 3, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 5, "fn": 0 }, "core": { "tp": 0, "fp": 5, "fn": 1 }, "all": { "tp": 0, "fp": 5, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 6, "fn": 0 }, "core": { "tp": 0, "fp": 6, "fn": 1 }, "all": { "tp": 0, "fp": 6, "fn": 1 } }, "gitlab": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 4, "fn": 0 }, "core": { "tp": 0, "fp": 4, "fn": 1 }, "all": { "tp": 0, "fp": 4, "fn": 1 } }, "augment": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "bugbot": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 1 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "claude-code": { "strict": { "tp": 0, "fp": 4, "fn": 0 }, "core": { "tp": 0, "fp": 4, "fn": 1 }, "all": { "tp": 0, "fp": 4, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 0 } }, "devin": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "kg": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "macroscope": { "strict": { "tp": 0, "fp": 0, "fn": 0 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 0, "fp": 2, "fn": 0 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 1 } }, "sourcery": { "strict": { "tp": 0, "fp": 3, "fn": 0 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 0, "fp": 3, "fn": 1 } } } }, { "url": "https://github.com/grafana/grafana/pull/79265", "language": "Go", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 1, "fp": 0, "fn": 4 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 2, "fn": 3 }, "core": { "tp": 1, "fp": 2, "fn": 4 }, "all": { "tp": 1, "fp": 2, "fn": 4 } }, "coderabbit": { "strict": { "tp": 3, "fp": 5, "fn": 1 }, "core": { "tp": 4, "fp": 5, "fn": 1 }, "all": { "tp": 4, "fp": 5, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 3 }, "all": { "tp": 2, "fp": 1, "fn": 3 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 2, "fn": 2 }, "core": { "tp": 3, "fp": 2, "fn": 2 }, "all": { "tp": 3, "fp": 2, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 4, "fn": 4 }, "core": { "tp": 0, "fp": 4, "fn": 5 }, "all": { "tp": 0, "fp": 4, "fn": 5 } }, "gitlab": { "strict": { "tp": 0, "fp": 2, "fn": 4 }, "core": { "tp": 1, "fp": 2, "fn": 4 }, "all": { "tp": 1, "fp": 2, "fn": 4 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 4, "fn": 3 }, "core": { "tp": 1, "fp": 4, "fn": 4 }, "all": { "tp": 1, "fp": 4, "fn": 4 } }, "augment": { "strict": { "tp": 1, "fp": 0, "fn": 3 }, "core": { "tp": 2, "fp": 0, "fn": 3 }, "all": { "tp": 2, "fp": 0, "fn": 3 } }, "bugbot": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 4 }, "all": { "tp": 1, "fp": 1, "fn": 4 } }, "claude": { "strict": { "tp": 1, "fp": 2, "fn": 3 }, "core": { "tp": 1, "fp": 2, "fn": 4 }, "all": { "tp": 1, "fp": 2, "fn": 4 } }, "claude-code": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 4 }, "all": { "tp": 1, "fp": 1, "fn": 4 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 4, "fp": 1, "fn": 1 }, "all": { "tp": 4, "fp": 1, "fn": 1 } }, "devin": { "strict": { "tp": 2, "fp": 0, "fn": 2 }, "core": { "tp": 2, "fp": 0, "fn": 3 }, "all": { "tp": 2, "fp": 0, "fn": 3 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 5 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kg": { "strict": { "tp": 1, "fp": 0, "fn": 3 }, "core": { "tp": 1, "fp": 0, "fn": 4 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 1, "fn": 4 }, "core": { "tp": 1, "fp": 1, "fn": 4 }, "all": { "tp": 1, "fp": 1, "fn": 4 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 3 }, "core": { "tp": 1, "fp": 0, "fn": 4 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 2, "fn": 3 }, "core": { "tp": 2, "fp": 2, "fn": 3 }, "all": { "tp": 2, "fp": 2, "fn": 3 } }, "sourcery": { "strict": { "tp": 2, "fp": 3, "fn": 2 }, "core": { "tp": 3, "fp": 3, "fn": 2 }, "all": { "tp": 3, "fp": 3, "fn": 2 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9", "language": "Ruby", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "reliability", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 2 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "coderabbit": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "gitlab": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 2 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "augment": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "bugbot": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 0, "fp": 5, "fn": 1 }, "core": { "tp": 0, "fp": 5, "fn": 2 }, "all": { "tp": 0, "fp": 5, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "devin": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "sourcery": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 2 }, "all": { "tp": 0, "fp": 2, "fn": 2 } } } }, { "url": "https://github.com/grafana/grafana/pull/76186", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 3, "fn": 3 }, "core": { "tp": 0, "fp": 3, "fn": 3 }, "all": { "tp": 0, "fp": 3, "fn": 3 } }, "coderabbit": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "gitlab": { "strict": { "tp": 3, "fp": 3, "fn": 0 }, "core": { "tp": 3, "fp": 3, "fn": 0 }, "all": { "tp": 3, "fp": 3, "fn": 0 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 2, "fn": 3 }, "core": { "tp": 0, "fp": 2, "fn": 3 }, "all": { "tp": 0, "fp": 2, "fn": 3 } }, "augment": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "bugbot": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "claude-code": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 0, "fn": 0 }, "core": { "tp": 3, "fp": 0, "fn": 0 }, "all": { "tp": 3, "fp": 0, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "sourcery": { "strict": { "tp": 0, "fp": 3, "fn": 3 }, "core": { "tp": 0, "fp": 3, "fn": 3 }, "all": { "tp": 0, "fp": 3, "fn": 3 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10", "language": "Ruby", "pr_size": "large", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 2, "fp": 1, "fn": 5 }, "core": { "tp": 2, "fp": 1, "fn": 5 }, "all": { "tp": 2, "fp": 1, "fn": 5 } }, "codeant-v2": { "strict": { "tp": 2, "fp": 7, "fn": 5 }, "core": { "tp": 2, "fp": 7, "fn": 5 }, "all": { "tp": 2, "fp": 7, "fn": 5 } }, "coderabbit": { "strict": { "tp": 5, "fp": 14, "fn": 2 }, "core": { "tp": 5, "fp": 14, "fn": 2 }, "all": { "tp": 5, "fp": 14, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 4, "fp": 6, "fn": 3 }, "core": { "tp": 4, "fp": 6, "fn": 3 }, "all": { "tp": 4, "fp": 6, "fn": 3 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 3, "fn": 5 }, "core": { "tp": 2, "fp": 3, "fn": 5 }, "all": { "tp": 2, "fp": 3, "fn": 5 } }, "gemini-v2": { "strict": { "tp": 4, "fp": 2, "fn": 3 }, "core": { "tp": 4, "fp": 2, "fn": 3 }, "all": { "tp": 4, "fp": 2, "fn": 3 } }, "gitlab": { "strict": { "tp": 2, "fp": 3, "fn": 5 }, "core": { "tp": 2, "fp": 3, "fn": 5 }, "all": { "tp": 2, "fp": 3, "fn": 5 } }, "copilot-v2": { "strict": { "tp": 6, "fp": 5, "fn": 1 }, "core": { "tp": 6, "fp": 5, "fn": 1 }, "all": { "tp": 6, "fp": 5, "fn": 1 } }, "augment": { "strict": { "tp": 4, "fp": 1, "fn": 3 }, "core": { "tp": 4, "fp": 1, "fn": 3 }, "all": { "tp": 4, "fp": 1, "fn": 3 } }, "bugbot": { "strict": { "tp": 5, "fp": 2, "fn": 2 }, "core": { "tp": 5, "fp": 2, "fn": 2 }, "all": { "tp": 5, "fp": 2, "fn": 2 } }, "claude": { "strict": { "tp": 1, "fp": 2, "fn": 6 }, "core": { "tp": 1, "fp": 2, "fn": 6 }, "all": { "tp": 1, "fp": 2, "fn": 6 } }, "claude-code": { "strict": { "tp": 1, "fp": 5, "fn": 6 }, "core": { "tp": 1, "fp": 5, "fn": 6 }, "all": { "tp": 1, "fp": 5, "fn": 6 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 1, "fn": 4 }, "core": { "tp": 3, "fp": 1, "fn": 4 }, "all": { "tp": 3, "fp": 1, "fn": 4 } }, "devin": { "strict": { "tp": 2, "fp": 0, "fn": 5 }, "core": { "tp": 2, "fp": 0, "fn": 5 }, "all": { "tp": 2, "fp": 0, "fn": 5 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 7 }, "core": { "tp": 0, "fp": 0, "fn": 7 }, "all": { "tp": 0, "fp": 0, "fn": 7 } }, "kg": { "strict": { "tp": 1, "fp": 0, "fn": 6 }, "core": { "tp": 1, "fp": 0, "fn": 6 }, "all": { "tp": 1, "fp": 0, "fn": 6 } }, "kodus-v2": { "strict": { "tp": 5, "fp": 4, "fn": 2 }, "core": { "tp": 5, "fp": 4, "fn": 2 }, "all": { "tp": 5, "fp": 4, "fn": 2 } }, "macroscope": { "strict": { "tp": 4, "fp": 3, "fn": 3 }, "core": { "tp": 4, "fp": 3, "fn": 3 }, "all": { "tp": 4, "fp": 3, "fn": 3 } }, "qodo-v2": { "strict": { "tp": 4, "fp": 6, "fn": 3 }, "core": { "tp": 4, "fp": 6, "fn": 3 }, "all": { "tp": 4, "fp": 6, "fn": 3 } }, "sourcery": { "strict": { "tp": 5, "fp": 7, "fn": 2 }, "core": { "tp": 5, "fp": 7, "fn": 2 }, "all": { "tp": 5, "fp": 7, "fn": 2 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "bug_fix", "complexity": "simple", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 3, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 5, "fp": 1, "fn": 0 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 9, "fn": 3 }, "core": { "tp": 0, "fp": 9, "fn": 3 }, "all": { "tp": 0, "fp": 9, "fn": 5 } }, "coderabbit": { "strict": { "tp": 0, "fp": 2, "fn": 3 }, "core": { "tp": 0, "fp": 2, "fn": 3 }, "all": { "tp": 0, "fp": 2, "fn": 5 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 3, "fp": 2, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 3, "fp": 0, "fn": 0 }, "core": { "tp": 3, "fp": 0, "fn": 0 }, "all": { "tp": 5, "fp": 0, "fn": 0 } }, "gemini-v2": { "strict": { "tp": 3, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 5, "fp": 1, "fn": 0 } }, "gitlab": { "strict": { "tp": 3, "fp": 0, "fn": 0 }, "core": { "tp": 3, "fp": 0, "fn": 0 }, "all": { "tp": 4, "fp": 0, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 3, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 5, "fp": 1, "fn": 0 } }, "augment": { "strict": { "tp": 3, "fp": 2, "fn": 0 }, "core": { "tp": 3, "fp": 2, "fn": 0 }, "all": { "tp": 5, "fp": 2, "fn": 0 } }, "bugbot": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 4, "fp": 1, "fn": 1 } }, "claude": { "strict": { "tp": 3, "fp": 3, "fn": 0 }, "core": { "tp": 3, "fp": 3, "fn": 0 }, "all": { "tp": 5, "fp": 3, "fn": 0 } }, "claude-code": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 4, "fp": 1, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 4, "fp": 1, "fn": 1 } }, "devin": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 1 }, "all": { "tp": 4, "fp": 3, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 3, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 3 } }, "qodo-v2": { "strict": { "tp": 3, "fp": 1, "fn": 0 }, "core": { "tp": 3, "fp": 1, "fn": 0 }, "all": { "tp": 5, "fp": 1, "fn": 0 } }, "sourcery": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 2 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8", "language": "Ruby", "pr_size": "medium", "domain": "API", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 2, "fp": 2, "fn": 4 }, "core": { "tp": 2, "fp": 2, "fn": 4 }, "all": { "tp": 2, "fp": 2, "fn": 4 } }, "codeant-v2": { "strict": { "tp": 5, "fp": 7, "fn": 1 }, "core": { "tp": 5, "fp": 7, "fn": 1 }, "all": { "tp": 5, "fp": 7, "fn": 1 } }, "coderabbit": { "strict": { "tp": 4, "fp": 6, "fn": 2 }, "core": { "tp": 4, "fp": 6, "fn": 2 }, "all": { "tp": 4, "fp": 6, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 3, "fp": 2, "fn": 3 }, "core": { "tp": 3, "fp": 2, "fn": 3 }, "all": { "tp": 3, "fp": 2, "fn": 3 } }, "greptile-v4-1": { "strict": { "tp": 4, "fp": 1, "fn": 2 }, "core": { "tp": 4, "fp": 1, "fn": 2 }, "all": { "tp": 4, "fp": 1, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 3, "fp": 2, "fn": 3 }, "core": { "tp": 3, "fp": 2, "fn": 3 }, "all": { "tp": 3, "fp": 2, "fn": 3 } }, "gitlab": { "strict": { "tp": 4, "fp": 1, "fn": 2 }, "core": { "tp": 4, "fp": 1, "fn": 2 }, "all": { "tp": 4, "fp": 1, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 3, "fp": 8, "fn": 3 }, "core": { "tp": 3, "fp": 8, "fn": 3 }, "all": { "tp": 3, "fp": 8, "fn": 3 } }, "augment": { "strict": { "tp": 3, "fp": 3, "fn": 3 }, "core": { "tp": 3, "fp": 3, "fn": 3 }, "all": { "tp": 3, "fp": 3, "fn": 3 } }, "bugbot": { "strict": { "tp": 2, "fp": 1, "fn": 4 }, "core": { "tp": 2, "fp": 1, "fn": 4 }, "all": { "tp": 2, "fp": 1, "fn": 4 } }, "claude": { "strict": { "tp": 4, "fp": 4, "fn": 2 }, "core": { "tp": 4, "fp": 4, "fn": 2 }, "all": { "tp": 4, "fp": 4, "fn": 2 } }, "claude-code": { "strict": { "tp": 2, "fp": 1, "fn": 4 }, "core": { "tp": 2, "fp": 1, "fn": 4 }, "all": { "tp": 2, "fp": 1, "fn": 4 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 1, "fn": 3 }, "core": { "tp": 3, "fp": 1, "fn": 3 }, "all": { "tp": 3, "fp": 1, "fn": 3 } }, "devin": { "strict": { "tp": 2, "fp": 0, "fn": 4 }, "core": { "tp": 2, "fp": 0, "fn": 4 }, "all": { "tp": 2, "fp": 0, "fn": 4 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 6 }, "core": { "tp": 0, "fp": 0, "fn": 6 }, "all": { "tp": 0, "fp": 0, "fn": 6 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 6 }, "core": { "tp": 0, "fp": 0, "fn": 6 }, "all": { "tp": 0, "fp": 0, "fn": 6 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 6, "fn": 5 }, "core": { "tp": 1, "fp": 6, "fn": 5 }, "all": { "tp": 1, "fp": 6, "fn": 5 } }, "macroscope": { "strict": { "tp": 2, "fp": 2, "fn": 4 }, "core": { "tp": 2, "fp": 2, "fn": 4 }, "all": { "tp": 2, "fp": 2, "fn": 4 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 2, "fn": 4 }, "core": { "tp": 2, "fp": 2, "fn": 4 }, "all": { "tp": 2, "fp": 2, "fn": 4 } }, "sourcery": { "strict": { "tp": 3, "fp": 2, "fn": 3 }, "core": { "tp": 3, "fp": 2, "fn": 3 }, "all": { "tp": 3, "fp": 2, "fn": 3 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3", "language": "Ruby", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 3, "fn": 2 }, "core": { "tp": 1, "fp": 3, "fn": 2 }, "all": { "tp": 1, "fp": 3, "fn": 2 } }, "coderabbit": { "strict": { "tp": 2, "fp": 5, "fn": 1 }, "core": { "tp": 2, "fp": 5, "fn": 1 }, "all": { "tp": 2, "fp": 5, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 3, "fn": 2 }, "core": { "tp": 1, "fp": 3, "fn": 2 }, "all": { "tp": 1, "fp": 3, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 1 }, "all": { "tp": 2, "fp": 3, "fn": 1 } }, "gitlab": { "strict": { "tp": 0, "fp": 2, "fn": 3 }, "core": { "tp": 0, "fp": 2, "fn": 3 }, "all": { "tp": 0, "fp": 2, "fn": 3 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 6, "fn": 1 }, "core": { "tp": 2, "fp": 6, "fn": 1 }, "all": { "tp": 2, "fp": 6, "fn": 1 } }, "augment": { "strict": { "tp": 1, "fp": 5, "fn": 2 }, "core": { "tp": 1, "fp": 5, "fn": 2 }, "all": { "tp": 1, "fp": 5, "fn": 2 } }, "bugbot": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "claude": { "strict": { "tp": 1, "fp": 4, "fn": 2 }, "core": { "tp": 1, "fp": 4, "fn": 2 }, "all": { "tp": 1, "fp": 4, "fn": 2 } }, "claude-code": { "strict": { "tp": 1, "fp": 4, "fn": 2 }, "core": { "tp": 1, "fp": 4, "fn": 2 }, "all": { "tp": 1, "fp": 4, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "devin": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 1 }, "all": { "tp": 2, "fp": 3, "fn": 1 } }, "sourcery": { "strict": { "tp": 1, "fp": 3, "fn": 2 }, "core": { "tp": 1, "fp": 3, "fn": 2 }, "all": { "tp": 1, "fp": 3, "fn": 2 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5", "language": "Ruby", "pr_size": "small", "domain": "UI", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "coderabbit": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 7, "fn": 1 }, "core": { "tp": 2, "fp": 7, "fn": 1 }, "all": { "tp": 2, "fp": 7, "fn": 1 } }, "gitlab": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "augment": { "strict": { "tp": 2, "fp": 0, "fn": 1 }, "core": { "tp": 2, "fp": 0, "fn": 1 }, "all": { "tp": 2, "fp": 0, "fn": 1 } }, "bugbot": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 1 }, "all": { "tp": 2, "fp": 3, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 1 }, "all": { "tp": 2, "fp": 2, "fn": 1 } }, "devin": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 1, "fp": 0, "fn": 2 }, "core": { "tp": 1, "fp": 0, "fn": 2 }, "all": { "tp": 1, "fp": 0, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "macroscope": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "sourcery": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6", "language": "Ruby", "pr_size": "small", "domain": "serialization", "change_type": "feature", "complexity": "simple", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 5, "fn": 1 }, "core": { "tp": 0, "fp": 5, "fn": 1 }, "all": { "tp": 0, "fp": 5, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 0 } }, "coderabbit": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 0 } }, "qodo-extended-v2": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 4, "fn": 1 }, "core": { "tp": 0, "fp": 4, "fn": 1 }, "all": { "tp": 0, "fp": 4, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 6, "fn": 1 }, "core": { "tp": 0, "fp": 6, "fn": 1 }, "all": { "tp": 0, "fp": 6, "fn": 1 } }, "gitlab": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 0 }, "all": { "tp": 1, "fp": 3, "fn": 0 } }, "augment": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 0, "fp": 3, "fn": 1 } }, "bugbot": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 0, "fp": 3, "fn": 1 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "claude-code": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 0 } }, "cubic-v2": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 1 } }, "devin": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 0, "fp": 3, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 1 } }, "macroscope": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 0 } }, "sourcery": { "strict": { "tp": 0, "fp": 5, "fn": 1 }, "core": { "tp": 0, "fp": 5, "fn": 1 }, "all": { "tp": 0, "fp": 5, "fn": 1 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4", "language": "Ruby", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 2, "fp": 2, "fn": 6 }, "core": { "tp": 2, "fp": 2, "fn": 6 }, "all": { "tp": 2, "fp": 2, "fn": 6 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 17, "fn": 7 }, "core": { "tp": 1, "fp": 17, "fn": 7 }, "all": { "tp": 1, "fp": 17, "fn": 7 } }, "coderabbit": { "strict": { "tp": 4, "fp": 23, "fn": 4 }, "core": { "tp": 4, "fp": 23, "fn": 4 }, "all": { "tp": 4, "fp": 23, "fn": 4 } }, "qodo-extended-v2": { "strict": { "tp": 3, "fp": 3, "fn": 5 }, "core": { "tp": 3, "fp": 3, "fn": 5 }, "all": { "tp": 3, "fp": 3, "fn": 5 } }, "greptile-v4-1": { "strict": { "tp": 4, "fp": 1, "fn": 4 }, "core": { "tp": 4, "fp": 1, "fn": 4 }, "all": { "tp": 4, "fp": 1, "fn": 4 } }, "gemini-v2": { "strict": { "tp": 4, "fp": 5, "fn": 4 }, "core": { "tp": 4, "fp": 5, "fn": 4 }, "all": { "tp": 4, "fp": 5, "fn": 4 } }, "gitlab": { "strict": { "tp": 3, "fp": 2, "fn": 5 }, "core": { "tp": 3, "fp": 2, "fn": 5 }, "all": { "tp": 3, "fp": 2, "fn": 5 } }, "copilot-v2": { "strict": { "tp": 5, "fp": 6, "fn": 3 }, "core": { "tp": 5, "fp": 6, "fn": 3 }, "all": { "tp": 5, "fp": 6, "fn": 3 } }, "augment": { "strict": { "tp": 3, "fp": 2, "fn": 5 }, "core": { "tp": 3, "fp": 2, "fn": 5 }, "all": { "tp": 3, "fp": 2, "fn": 5 } }, "bugbot": { "strict": { "tp": 2, "fp": 3, "fn": 6 }, "core": { "tp": 2, "fp": 3, "fn": 6 }, "all": { "tp": 2, "fp": 3, "fn": 6 } }, "claude": { "strict": { "tp": 3, "fp": 3, "fn": 5 }, "core": { "tp": 3, "fp": 3, "fn": 5 }, "all": { "tp": 3, "fp": 3, "fn": 5 } }, "claude-code": { "strict": { "tp": 3, "fp": 1, "fn": 5 }, "core": { "tp": 3, "fp": 1, "fn": 5 }, "all": { "tp": 3, "fp": 1, "fn": 5 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 3, "fn": 7 }, "core": { "tp": 1, "fp": 3, "fn": 7 }, "all": { "tp": 1, "fp": 3, "fn": 7 } }, "devin": { "strict": { "tp": 3, "fp": 1, "fn": 5 }, "core": { "tp": 3, "fp": 1, "fn": 5 }, "all": { "tp": 3, "fp": 1, "fn": 5 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 8 }, "core": { "tp": 0, "fp": 0, "fn": 8 }, "all": { "tp": 0, "fp": 0, "fn": 8 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 8 }, "core": { "tp": 0, "fp": 0, "fn": 8 }, "all": { "tp": 0, "fp": 0, "fn": 8 } }, "kodus-v2": { "strict": { "tp": 3, "fp": 2, "fn": 5 }, "core": { "tp": 3, "fp": 2, "fn": 5 }, "all": { "tp": 3, "fp": 2, "fn": 5 } }, "macroscope": { "strict": { "tp": 1, "fp": 5, "fn": 7 }, "core": { "tp": 1, "fp": 5, "fn": 7 }, "all": { "tp": 1, "fp": 5, "fn": 7 } }, "qodo-v2": { "strict": { "tp": 5, "fp": 13, "fn": 3 }, "core": { "tp": 5, "fp": 13, "fn": 3 }, "all": { "tp": 5, "fp": 13, "fn": 3 } }, "sourcery": { "strict": { "tp": 2, "fp": 3, "fn": 6 }, "core": { "tp": 2, "fp": 3, "fn": 6 }, "all": { "tp": 2, "fp": 3, "fn": 6 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1", "language": "Ruby", "pr_size": "medium", "domain": "file_io", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "coderabbit": { "strict": { "tp": 3, "fp": 4, "fn": 1 }, "core": { "tp": 3, "fp": 4, "fn": 1 }, "all": { "tp": 3, "fp": 4, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 3, "fp": 0, "fn": 1 }, "core": { "tp": 3, "fp": 0, "fn": 1 }, "all": { "tp": 3, "fp": 0, "fn": 1 } }, "gitlab": { "strict": { "tp": 2, "fp": 0, "fn": 2 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 2, "fn": 2 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 2 } }, "augment": { "strict": { "tp": 2, "fp": 4, "fn": 2 }, "core": { "tp": 2, "fp": 4, "fn": 2 }, "all": { "tp": 2, "fp": 4, "fn": 2 } }, "bugbot": { "strict": { "tp": 2, "fp": 2, "fn": 2 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 2 } }, "claude": { "strict": { "tp": 2, "fp": 0, "fn": 2 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 3, "fp": 2, "fn": 1 }, "core": { "tp": 3, "fp": 2, "fn": 1 }, "all": { "tp": 3, "fp": 2, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 0, "fn": 1 }, "core": { "tp": 3, "fp": 0, "fn": 1 }, "all": { "tp": 3, "fp": 0, "fn": 1 } }, "devin": { "strict": { "tp": 2, "fp": 0, "fn": 2 }, "core": { "tp": 2, "fp": 0, "fn": 2 }, "all": { "tp": 2, "fp": 0, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "kodus-v2": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } }, "macroscope": { "strict": { "tp": 1, "fp": 2, "fn": 3 }, "core": { "tp": 1, "fp": 2, "fn": 3 }, "all": { "tp": 1, "fp": 2, "fn": 3 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 4, "fn": 2 }, "core": { "tp": 2, "fp": 4, "fn": 2 }, "all": { "tp": 2, "fp": 4, "fn": 2 } }, "sourcery": { "strict": { "tp": 2, "fp": 1, "fn": 2 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 2, "fp": 1, "fn": 2 } } } }, { "url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 5, "fn": 0 }, "core": { "tp": 1, "fp": 5, "fn": 0 }, "all": { "tp": 1, "fp": 5, "fn": 1 } }, "coderabbit": { "strict": { "tp": 1, "fp": 7, "fn": 0 }, "core": { "tp": 1, "fp": 7, "fn": 0 }, "all": { "tp": 2, "fp": 7, "fn": 0 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "gitlab": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 4, "fn": 0 }, "core": { "tp": 1, "fp": 4, "fn": 0 }, "all": { "tp": 2, "fp": 4, "fn": 0 } }, "augment": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "bugbot": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "claude": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "claude-code": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 0 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "sourcery": { "strict": { "tp": 1, "fp": 5, "fn": 0 }, "core": { "tp": 1, "fp": 5, "fn": 0 }, "all": { "tp": 1, "fp": 5, "fn": 1 } } } }, { "url": "https://github.com/calcom/cal.com/pull/22532", "language": "TypeScript", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 2, "fn": 3 }, "core": { "tp": 1, "fp": 2, "fn": 3 }, "all": { "tp": 1, "fp": 2, "fn": 3 } }, "codeant-v2": { "strict": { "tp": 3, "fp": 3, "fn": 1 }, "core": { "tp": 3, "fp": 3, "fn": 1 }, "all": { "tp": 3, "fp": 3, "fn": 1 } }, "coderabbit": { "strict": { "tp": 2, "fp": 3, "fn": 2 }, "core": { "tp": 2, "fp": 3, "fn": 2 }, "all": { "tp": 2, "fp": 3, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 2, "fn": 2 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 3, "fn": 2 }, "core": { "tp": 2, "fp": 3, "fn": 2 }, "all": { "tp": 2, "fp": 3, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 4, "fp": 0, "fn": 0 }, "core": { "tp": 4, "fp": 0, "fn": 0 }, "all": { "tp": 4, "fp": 0, "fn": 0 } }, "gitlab": { "strict": { "tp": 2, "fp": 3, "fn": 2 }, "core": { "tp": 2, "fp": 3, "fn": 2 }, "all": { "tp": 2, "fp": 3, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 4, "fp": 2, "fn": 0 }, "core": { "tp": 4, "fp": 2, "fn": 0 }, "all": { "tp": 4, "fp": 2, "fn": 0 } }, "augment": { "strict": { "tp": 4, "fp": 2, "fn": 0 }, "core": { "tp": 4, "fp": 2, "fn": 0 }, "all": { "tp": 4, "fp": 2, "fn": 0 } }, "bugbot": { "strict": { "tp": 0, "fp": 1, "fn": 4 }, "core": { "tp": 0, "fp": 1, "fn": 4 }, "all": { "tp": 0, "fp": 1, "fn": 4 } }, "claude": { "strict": { "tp": 0, "fp": 1, "fn": 4 }, "core": { "tp": 0, "fp": 1, "fn": 4 }, "all": { "tp": 0, "fp": 1, "fn": 4 } }, "claude-code": { "strict": { "tp": 1, "fp": 3, "fn": 3 }, "core": { "tp": 1, "fp": 3, "fn": 3 }, "all": { "tp": 1, "fp": 3, "fn": 3 } }, "cubic-v2": { "strict": { "tp": 0, "fp": 2, "fn": 4 }, "core": { "tp": 0, "fp": 2, "fn": 4 }, "all": { "tp": 0, "fp": 2, "fn": 4 } }, "devin": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 3 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 4 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 4 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 3 }, "core": { "tp": 1, "fp": 0, "fn": 3 }, "all": { "tp": 1, "fp": 0, "fn": 3 } }, "macroscope": { "strict": { "tp": 1, "fp": 1, "fn": 3 }, "core": { "tp": 1, "fp": 1, "fn": 3 }, "all": { "tp": 1, "fp": 1, "fn": 3 } }, "qodo-v2": { "strict": { "tp": 3, "fp": 2, "fn": 1 }, "core": { "tp": 3, "fp": 2, "fn": 1 }, "all": { "tp": 3, "fp": 2, "fn": 1 } }, "sourcery": { "strict": { "tp": 3, "fp": 1, "fn": 1 }, "core": { "tp": 3, "fp": 1, "fn": 1 }, "all": { "tp": 3, "fp": 1, "fn": 1 } } } }, { "url": "https://github.com/calcom/cal.com/pull/8330", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "coderabbit": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 3, "fn": 0 }, "core": { "tp": 2, "fp": 3, "fn": 0 }, "all": { "tp": 2, "fp": 3, "fn": 0 } }, "gitlab": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 5, "fn": 0 }, "core": { "tp": 2, "fp": 5, "fn": 0 }, "all": { "tp": 2, "fp": 5, "fn": 0 } }, "augment": { "strict": { "tp": 2, "fp": 3, "fn": 0 }, "core": { "tp": 2, "fp": 3, "fn": 0 }, "all": { "tp": 2, "fp": 3, "fn": 0 } }, "bugbot": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "claude": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "claude-code": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "devin": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "graphite": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "kg": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "macroscope": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 0 }, "all": { "tp": 2, "fp": 2, "fn": 0 } }, "sourcery": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } } } }, { "url": "https://github.com/calcom/cal.com/pull/14943", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 3, "fn": 1 }, "core": { "tp": 1, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 2 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "coderabbit": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "gitlab": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "augment": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "bugbot": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 1, "fp": 2, "fn": 1 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "claude": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude-code": { "strict": { "tp": 0, "fp": 1, "fn": 2 }, "core": { "tp": 0, "fp": 1, "fn": 2 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 0, "fn": 0 }, "core": { "tp": 2, "fp": 0, "fn": 0 }, "all": { "tp": 2, "fp": 0, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "graphite": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "kg": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 1, "fp": 1, "fn": 1 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "macroscope": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } }, "sourcery": { "strict": { "tp": 2, "fp": 1, "fn": 0 }, "core": { "tp": 2, "fp": 1, "fn": 0 }, "all": { "tp": 2, "fp": 1, "fn": 0 } } } }, { "url": "https://github.com/calcom/cal.com/pull/22345", "language": "TypeScript", "pr_size": "small", "domain": "database", "change_type": "migration", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 2, "fn": 1 }, "core": { "tp": 0, "fp": 2, "fn": 1 }, "all": { "tp": 0, "fp": 2, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "coderabbit": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 0, "fp": 3, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "gitlab": { "strict": { "tp": 1, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 0 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "augment": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "bugbot": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "claude": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "claude-code": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "devin": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 0, "fp": 1, "fn": 1 }, "core": { "tp": 0, "fp": 1, "fn": 1 }, "all": { "tp": 0, "fp": 1, "fn": 2 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 0, "fp": 3, "fn": 1 }, "core": { "tp": 0, "fp": 3, "fn": 1 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "sourcery": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } } } }, { "url": "https://github.com/calcom/cal.com/pull/11059", "language": "TypeScript", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 4, "fp": 1, "fn": 5 }, "core": { "tp": 4, "fp": 1, "fn": 5 }, "all": { "tp": 4, "fp": 1, "fn": 5 } }, "codeant-v2": { "strict": { "tp": 4, "fp": 5, "fn": 5 }, "core": { "tp": 4, "fp": 5, "fn": 5 }, "all": { "tp": 4, "fp": 5, "fn": 5 } }, "coderabbit": { "strict": { "tp": 8, "fp": 13, "fn": 1 }, "core": { "tp": 8, "fp": 13, "fn": 1 }, "all": { "tp": 8, "fp": 13, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 7, "fp": 2, "fn": 2 }, "core": { "tp": 7, "fp": 2, "fn": 2 }, "all": { "tp": 7, "fp": 2, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 5, "fp": 4, "fn": 4 }, "core": { "tp": 5, "fp": 4, "fn": 4 }, "all": { "tp": 5, "fp": 4, "fn": 4 } }, "gemini-v2": { "strict": { "tp": 4, "fp": 2, "fn": 5 }, "core": { "tp": 4, "fp": 2, "fn": 5 }, "all": { "tp": 4, "fp": 2, "fn": 5 } }, "gitlab": { "strict": { "tp": 5, "fp": 5, "fn": 4 }, "core": { "tp": 5, "fp": 5, "fn": 4 }, "all": { "tp": 5, "fp": 5, "fn": 4 } }, "copilot-v2": { "strict": { "tp": 7, "fp": 3, "fn": 2 }, "core": { "tp": 7, "fp": 3, "fn": 2 }, "all": { "tp": 7, "fp": 3, "fn": 2 } }, "augment": { "strict": { "tp": 8, "fp": 5, "fn": 1 }, "core": { "tp": 8, "fp": 5, "fn": 1 }, "all": { "tp": 8, "fp": 5, "fn": 1 } }, "bugbot": { "strict": { "tp": 6, "fp": 2, "fn": 3 }, "core": { "tp": 6, "fp": 2, "fn": 3 }, "all": { "tp": 6, "fp": 2, "fn": 3 } }, "claude": { "strict": { "tp": 5, "fp": 1, "fn": 4 }, "core": { "tp": 5, "fp": 1, "fn": 4 }, "all": { "tp": 5, "fp": 1, "fn": 4 } }, "claude-code": { "strict": { "tp": 4, "fp": 0, "fn": 5 }, "core": { "tp": 4, "fp": 0, "fn": 5 }, "all": { "tp": 4, "fp": 0, "fn": 5 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 0, "fn": 6 }, "core": { "tp": 3, "fp": 0, "fn": 6 }, "all": { "tp": 3, "fp": 0, "fn": 6 } }, "devin": { "strict": { "tp": 5, "fp": 0, "fn": 4 }, "core": { "tp": 5, "fp": 0, "fn": 4 }, "all": { "tp": 5, "fp": 0, "fn": 4 } }, "graphite": { "strict": { "tp": 1, "fp": 1, "fn": 8 }, "core": { "tp": 1, "fp": 1, "fn": 8 }, "all": { "tp": 1, "fp": 1, "fn": 8 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 9 }, "core": { "tp": 0, "fp": 0, "fn": 9 }, "all": { "tp": 0, "fp": 0, "fn": 9 } }, "kodus-v2": { "strict": { "tp": 4, "fp": 3, "fn": 5 }, "core": { "tp": 4, "fp": 3, "fn": 5 }, "all": { "tp": 4, "fp": 3, "fn": 5 } }, "macroscope": { "strict": { "tp": 5, "fp": 5, "fn": 4 }, "core": { "tp": 5, "fp": 5, "fn": 4 }, "all": { "tp": 5, "fp": 5, "fn": 4 } }, "qodo-v2": { "strict": { "tp": 9, "fp": 2, "fn": 0 }, "core": { "tp": 9, "fp": 2, "fn": 0 }, "all": { "tp": 9, "fp": 2, "fn": 0 } }, "sourcery": { "strict": { "tp": 5, "fp": 0, "fn": 4 }, "core": { "tp": 5, "fp": 0, "fn": 4 }, "all": { "tp": 5, "fp": 0, "fn": 4 } } } }, { "url": "https://github.com/calcom/cal.com/pull/7232", "language": "TypeScript", "pr_size": "medium", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "coderabbit": { "strict": { "tp": 2, "fp": 12, "fn": 1 }, "core": { "tp": 2, "fp": 12, "fn": 1 }, "all": { "tp": 2, "fp": 12, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 2, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 1 }, "all": { "tp": 2, "fp": 1, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 1 }, "all": { "tp": 2, "fp": 3, "fn": 1 } }, "gitlab": { "strict": { "tp": 0, "fp": 3, "fn": 3 }, "core": { "tp": 0, "fp": 3, "fn": 3 }, "all": { "tp": 0, "fp": 3, "fn": 3 } }, "copilot-v2": { "strict": { "tp": 0, "fp": 3, "fn": 3 }, "core": { "tp": 0, "fp": 3, "fn": 3 }, "all": { "tp": 0, "fp": 3, "fn": 3 } }, "augment": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "bugbot": { "strict": { "tp": 1, "fp": 3, "fn": 2 }, "core": { "tp": 1, "fp": 3, "fn": 2 }, "all": { "tp": 1, "fp": 3, "fn": 2 } }, "claude": { "strict": { "tp": 1, "fp": 3, "fn": 2 }, "core": { "tp": 1, "fp": 3, "fn": 2 }, "all": { "tp": 1, "fp": 3, "fn": 2 } }, "claude-code": { "strict": { "tp": 1, "fp": 1, "fn": 2 }, "core": { "tp": 1, "fp": 1, "fn": 2 }, "all": { "tp": 1, "fp": 1, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 1 }, "all": { "tp": 2, "fp": 3, "fn": 1 } }, "devin": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 1 }, "all": { "tp": 2, "fp": 3, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 3 }, "core": { "tp": 0, "fp": 0, "fn": 3 }, "all": { "tp": 0, "fp": 0, "fn": 3 } }, "kg": { "strict": { "tp": 0, "fp": 1, "fn": 3 }, "core": { "tp": 0, "fp": 1, "fn": 3 }, "all": { "tp": 0, "fp": 1, "fn": 3 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 3, "fn": 2 }, "core": { "tp": 1, "fp": 3, "fn": 2 }, "all": { "tp": 1, "fp": 3, "fn": 2 } }, "macroscope": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } }, "qodo-v2": { "strict": { "tp": 2, "fp": 3, "fn": 1 }, "core": { "tp": 2, "fp": 3, "fn": 1 }, "all": { "tp": 2, "fp": 3, "fn": 1 } }, "sourcery": { "strict": { "tp": 1, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 2 }, "all": { "tp": 1, "fp": 2, "fn": 2 } } } }, { "url": "https://github.com/calcom/cal.com/pull/14740", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 1, "fn": 5 }, "core": { "tp": 1, "fp": 1, "fn": 5 }, "all": { "tp": 1, "fp": 1, "fn": 5 } }, "codeant-v2": { "strict": { "tp": 3, "fp": 4, "fn": 3 }, "core": { "tp": 3, "fp": 4, "fn": 3 }, "all": { "tp": 3, "fp": 4, "fn": 3 } }, "coderabbit": { "strict": { "tp": 4, "fp": 7, "fn": 2 }, "core": { "tp": 4, "fp": 7, "fn": 2 }, "all": { "tp": 4, "fp": 7, "fn": 2 } }, "qodo-extended-v2": { "strict": { "tp": 4, "fp": 1, "fn": 2 }, "core": { "tp": 4, "fp": 1, "fn": 2 }, "all": { "tp": 4, "fp": 1, "fn": 2 } }, "greptile-v4-1": { "strict": { "tp": 3, "fp": 2, "fn": 3 }, "core": { "tp": 3, "fp": 2, "fn": 3 }, "all": { "tp": 3, "fp": 2, "fn": 3 } }, "gemini-v2": { "strict": { "tp": 3, "fp": 4, "fn": 3 }, "core": { "tp": 3, "fp": 4, "fn": 3 }, "all": { "tp": 3, "fp": 4, "fn": 3 } }, "gitlab": { "strict": { "tp": 3, "fp": 4, "fn": 3 }, "core": { "tp": 3, "fp": 4, "fn": 3 }, "all": { "tp": 3, "fp": 4, "fn": 3 } }, "copilot-v2": { "strict": { "tp": 5, "fp": 5, "fn": 1 }, "core": { "tp": 5, "fp": 5, "fn": 1 }, "all": { "tp": 5, "fp": 5, "fn": 1 } }, "augment": { "strict": { "tp": 4, "fp": 3, "fn": 2 }, "core": { "tp": 4, "fp": 3, "fn": 2 }, "all": { "tp": 4, "fp": 3, "fn": 2 } }, "bugbot": { "strict": { "tp": 3, "fp": 3, "fn": 3 }, "core": { "tp": 3, "fp": 3, "fn": 3 }, "all": { "tp": 3, "fp": 3, "fn": 3 } }, "claude": { "strict": { "tp": 2, "fp": 0, "fn": 4 }, "core": { "tp": 2, "fp": 0, "fn": 4 }, "all": { "tp": 2, "fp": 0, "fn": 4 } }, "claude-code": { "strict": { "tp": 2, "fp": 1, "fn": 4 }, "core": { "tp": 2, "fp": 1, "fn": 4 }, "all": { "tp": 2, "fp": 1, "fn": 4 } }, "cubic-v2": { "strict": { "tp": 3, "fp": 3, "fn": 3 }, "core": { "tp": 3, "fp": 3, "fn": 3 }, "all": { "tp": 3, "fp": 3, "fn": 3 } }, "devin": { "strict": { "tp": 2, "fp": 0, "fn": 4 }, "core": { "tp": 2, "fp": 0, "fn": 4 }, "all": { "tp": 2, "fp": 0, "fn": 4 } }, "graphite": { "strict": { "tp": 1, "fp": 1, "fn": 5 }, "core": { "tp": 1, "fp": 1, "fn": 5 }, "all": { "tp": 1, "fp": 1, "fn": 5 } }, "kg": { "strict": { "tp": 2, "fp": 0, "fn": 4 }, "core": { "tp": 2, "fp": 0, "fn": 4 }, "all": { "tp": 2, "fp": 0, "fn": 4 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 0, "fn": 4 }, "core": { "tp": 2, "fp": 0, "fn": 4 }, "all": { "tp": 2, "fp": 0, "fn": 4 } }, "macroscope": { "strict": { "tp": 3, "fp": 3, "fn": 3 }, "core": { "tp": 3, "fp": 3, "fn": 3 }, "all": { "tp": 3, "fp": 3, "fn": 3 } }, "qodo-v2": { "strict": { "tp": 5, "fp": 3, "fn": 1 }, "core": { "tp": 5, "fp": 3, "fn": 1 }, "all": { "tp": 5, "fp": 3, "fn": 1 } }, "sourcery": { "strict": { "tp": 2, "fp": 3, "fn": 4 }, "core": { "tp": 2, "fp": 3, "fn": 4 }, "all": { "tp": 2, "fp": 3, "fn": 4 } } } }, { "url": "https://github.com/calcom/cal.com/pull/10600", "language": "TypeScript", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 0, "fp": 1, "fn": 2 }, "core": { "tp": 0, "fp": 1, "fn": 4 }, "all": { "tp": 0, "fp": 1, "fn": 5 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 4, "fn": 1 }, "core": { "tp": 1, "fp": 4, "fn": 3 }, "all": { "tp": 1, "fp": 4, "fn": 4 } }, "coderabbit": { "strict": { "tp": 1, "fp": 7, "fn": 1 }, "core": { "tp": 1, "fp": 7, "fn": 3 }, "all": { "tp": 2, "fp": 7, "fn": 3 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 2, "fn": 1 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 3 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 3, "fp": 1, "fn": 2 } }, "gemini-v2": { "strict": { "tp": 0, "fp": 5, "fn": 2 }, "core": { "tp": 0, "fp": 5, "fn": 4 }, "all": { "tp": 1, "fp": 5, "fn": 4 } }, "gitlab": { "strict": { "tp": 0, "fp": 2, "fn": 2 }, "core": { "tp": 1, "fp": 2, "fn": 3 }, "all": { "tp": 2, "fp": 2, "fn": 3 } }, "copilot-v2": { "strict": { "tp": 2, "fp": 3, "fn": 0 }, "core": { "tp": 4, "fp": 3, "fn": 0 }, "all": { "tp": 5, "fp": 3, "fn": 0 } }, "augment": { "strict": { "tp": 0, "fp": 3, "fn": 2 }, "core": { "tp": 1, "fp": 3, "fn": 3 }, "all": { "tp": 2, "fp": 3, "fn": 3 } }, "bugbot": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 3 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "claude": { "strict": { "tp": 0, "fp": 10, "fn": 2 }, "core": { "tp": 1, "fp": 10, "fn": 3 }, "all": { "tp": 2, "fp": 10, "fn": 3 } }, "claude-code": { "strict": { "tp": 0, "fp": 4, "fn": 2 }, "core": { "tp": 0, "fp": 4, "fn": 4 }, "all": { "tp": 1, "fp": 4, "fn": 4 } }, "cubic-v2": { "strict": { "tp": 2, "fp": 2, "fn": 0 }, "core": { "tp": 2, "fp": 2, "fn": 2 }, "all": { "tp": 2, "fp": 2, "fn": 3 } }, "devin": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 3 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 2 }, "core": { "tp": 0, "fp": 0, "fn": 4 }, "all": { "tp": 0, "fp": 0, "fn": 5 } }, "kodus-v2": { "strict": { "tp": 1, "fp": 0, "fn": 1 }, "core": { "tp": 1, "fp": 0, "fn": 3 }, "all": { "tp": 1, "fp": 0, "fn": 4 } }, "macroscope": { "strict": { "tp": 0, "fp": 3, "fn": 2 }, "core": { "tp": 0, "fp": 3, "fn": 4 }, "all": { "tp": 0, "fp": 3, "fn": 5 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 5, "fn": 1 }, "core": { "tp": 2, "fp": 5, "fn": 2 }, "all": { "tp": 2, "fp": 5, "fn": 3 } }, "sourcery": { "strict": { "tp": 1, "fp": 1, "fn": 1 }, "core": { "tp": 2, "fp": 1, "fn": 2 }, "all": { "tp": 3, "fp": 1, "fn": 2 } } } }, { "url": "https://github.com/calcom/cal.com/pull/10967", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "bug_fix", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 2, "fn": 4 }, "core": { "tp": 1, "fp": 2, "fn": 4 }, "all": { "tp": 1, "fp": 2, "fn": 5 } }, "codeant-v2": { "strict": { "tp": 2, "fp": 6, "fn": 3 }, "core": { "tp": 2, "fp": 6, "fn": 3 }, "all": { "tp": 2, "fp": 6, "fn": 4 } }, "coderabbit": { "strict": { "tp": 5, "fp": 15, "fn": 0 }, "core": { "tp": 5, "fp": 15, "fn": 0 }, "all": { "tp": 5, "fp": 15, "fn": 1 } }, "qodo-extended-v2": { "strict": { "tp": 3, "fp": 2, "fn": 2 }, "core": { "tp": 3, "fp": 2, "fn": 2 }, "all": { "tp": 3, "fp": 2, "fn": 3 } }, "greptile-v4-1": { "strict": { "tp": 2, "fp": 2, "fn": 3 }, "core": { "tp": 2, "fp": 2, "fn": 3 }, "all": { "tp": 2, "fp": 2, "fn": 4 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 6, "fn": 4 }, "core": { "tp": 1, "fp": 6, "fn": 4 }, "all": { "tp": 1, "fp": 6, "fn": 5 } }, "gitlab": { "strict": { "tp": 4, "fp": 1, "fn": 1 }, "core": { "tp": 4, "fp": 1, "fn": 1 }, "all": { "tp": 4, "fp": 1, "fn": 2 } }, "copilot-v2": { "strict": { "tp": 4, "fp": 8, "fn": 1 }, "core": { "tp": 4, "fp": 8, "fn": 1 }, "all": { "tp": 4, "fp": 8, "fn": 2 } }, "augment": { "strict": { "tp": 3, "fp": 2, "fn": 2 }, "core": { "tp": 3, "fp": 2, "fn": 2 }, "all": { "tp": 3, "fp": 2, "fn": 3 } }, "bugbot": { "strict": { "tp": 3, "fp": 3, "fn": 2 }, "core": { "tp": 3, "fp": 3, "fn": 2 }, "all": { "tp": 3, "fp": 3, "fn": 3 } }, "claude": { "strict": { "tp": 3, "fp": 4, "fn": 2 }, "core": { "tp": 3, "fp": 4, "fn": 2 }, "all": { "tp": 3, "fp": 4, "fn": 3 } }, "claude-code": { "strict": { "tp": 4, "fp": 3, "fn": 1 }, "core": { "tp": 4, "fp": 3, "fn": 1 }, "all": { "tp": 4, "fp": 3, "fn": 2 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 2, "fn": 4 }, "core": { "tp": 1, "fp": 2, "fn": 4 }, "all": { "tp": 1, "fp": 2, "fn": 5 } }, "devin": { "strict": { "tp": 3, "fp": 2, "fn": 2 }, "core": { "tp": 3, "fp": 2, "fn": 2 }, "all": { "tp": 3, "fp": 2, "fn": 3 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 5 }, "core": { "tp": 0, "fp": 0, "fn": 5 }, "all": { "tp": 0, "fp": 0, "fn": 6 } }, "kg": { "strict": { "tp": 0, "fp": 0, "fn": 5 }, "core": { "tp": 0, "fp": 0, "fn": 5 }, "all": { "tp": 0, "fp": 0, "fn": 6 } }, "kodus-v2": { "strict": { "tp": 2, "fp": 2, "fn": 3 }, "core": { "tp": 2, "fp": 2, "fn": 3 }, "all": { "tp": 2, "fp": 2, "fn": 4 } }, "macroscope": { "strict": { "tp": 2, "fp": 3, "fn": 3 }, "core": { "tp": 2, "fp": 3, "fn": 3 }, "all": { "tp": 2, "fp": 3, "fn": 4 } }, "qodo-v2": { "strict": { "tp": 3, "fp": 2, "fn": 2 }, "core": { "tp": 3, "fp": 2, "fn": 2 }, "all": { "tp": 3, "fp": 2, "fn": 3 } }, "sourcery": { "strict": { "tp": 2, "fp": 3, "fn": 3 }, "core": { "tp": 2, "fp": 3, "fn": 3 }, "all": { "tp": 2, "fp": 3, "fn": 4 } } } }, { "url": "https://github.com/calcom/cal.com/pull/8087", "language": "TypeScript", "pr_size": "medium", "domain": "concurrency", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": { "baz": { "strict": { "tp": 1, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 0 }, "all": { "tp": 1, "fp": 3, "fn": 1 } }, "codeant-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "coderabbit": { "strict": { "tp": 1, "fp": 7, "fn": 0 }, "core": { "tp": 1, "fp": 7, "fn": 0 }, "all": { "tp": 2, "fp": 7, "fn": 0 } }, "qodo-extended-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "greptile-v4-1": { "strict": { "tp": 1, "fp": 6, "fn": 0 }, "core": { "tp": 1, "fp": 6, "fn": 0 }, "all": { "tp": 1, "fp": 6, "fn": 1 } }, "gemini-v2": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "gitlab": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "copilot-v2": { "strict": { "tp": 1, "fp": 2, "fn": 0 }, "core": { "tp": 1, "fp": 2, "fn": 0 }, "all": { "tp": 1, "fp": 2, "fn": 1 } }, "augment": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "bugbot": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "claude": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "claude-code": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "cubic-v2": { "strict": { "tp": 1, "fp": 7, "fn": 0 }, "core": { "tp": 1, "fp": 7, "fn": 0 }, "all": { "tp": 2, "fp": 7, "fn": 0 } }, "devin": { "strict": { "tp": 1, "fp": 7, "fn": 0 }, "core": { "tp": 1, "fp": 7, "fn": 0 }, "all": { "tp": 1, "fp": 7, "fn": 1 } }, "graphite": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "kg": { "strict": { "tp": 1, "fp": 0, "fn": 0 }, "core": { "tp": 1, "fp": 0, "fn": 0 }, "all": { "tp": 1, "fp": 0, "fn": 1 } }, "kodus-v2": { "strict": { "tp": 0, "fp": 0, "fn": 1 }, "core": { "tp": 0, "fp": 0, "fn": 1 }, "all": { "tp": 0, "fp": 0, "fn": 2 } }, "macroscope": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "qodo-v2": { "strict": { "tp": 1, "fp": 1, "fn": 0 }, "core": { "tp": 1, "fp": 1, "fn": 0 }, "all": { "tp": 1, "fp": 1, "fn": 1 } }, "sourcery": { "strict": { "tp": 1, "fp": 3, "fn": 0 }, "core": { "tp": 1, "fp": 3, "fn": 0 }, "all": { "tp": 1, "fp": 3, "fn": 1 } } } } ], "tools": [ "augment", "baz", "bugbot", "claude", "claude-code", "codeant-v2", "coderabbit", "copilot-v2", "cubic-v2", "devin", "gemini-v2", "gitlab", "graphite", "greptile-v4-1", "kg", "kodus-v2", "macroscope", "qodo-extended-v2", "qodo-v2", "sourcery" ], "dimensions": { "language": [ "Go", "Java", "Python", "Ruby", "TypeScript" ], "pr_size": [ "small", "medium", "large" ], "domain": [ "API", "UI", "authentication", "caching", "concurrency", "configuration", "data_processing", "database", "file_io", "logging", "scheduling", "serialization", "testing" ], "change_type": [ "bug_fix", "feature", "migration", "performance", "refactoring", "security_patch" ], "complexity": [ "simple", "moderate", "complex" ], "difficulty": [ "obvious", "moderate", "subtle", "very_subtle" ], "risk": [ "low", "medium", "high", "critical" ], "context": [ "local", "file", "cross_file", "system" ], "concern": [ "correctness", "maintainability", "reliability", "security" ] }, "overall_metrics": { "coderabbit": { "strict": { "precision": 26.8, "recall": 61.9, "f1": 37.4, "f2": 49.0, "tp": 86, "fp": 235, "fn": 53, "num_prs": 50 }, "core": { "precision": 28.6, "recall": 59.5, "f1": 38.6, "f2": 48.9, "tp": 94, "fp": 235, "fn": 64, "num_prs": 50 }, "all": { "precision": 30.7, "recall": 60.1, "f1": 40.6, "f2": 50.4, "tp": 104, "fp": 235, "fn": 69, "num_prs": 50 } }, "cubic-v2": { "strict": { "precision": 53.2, "recall": 59.7, "f1": 56.3, "f2": 58.3, "tp": 83, "fp": 73, "fn": 56, "num_prs": 50 }, "core": { "precision": 56.8, "recall": 60.8, "f1": 58.7, "f2": 59.9, "tp": 96, "fp": 73, "fn": 62, "num_prs": 50 }, "all": { "precision": 57.8, "recall": 57.8, "f1": 57.8, "f2": 57.8, "tp": 100, "fp": 73, "fn": 73, "num_prs": 50 } }, "sourcery": { "strict": { "precision": 33.3, "recall": 50.4, "f1": 40.1, "f2": 45.7, "tp": 70, "fp": 140, "fn": 69, "num_prs": 50 }, "core": { "precision": 36.1, "recall": 50.0, "f1": 41.9, "f2": 46.4, "tp": 79, "fp": 140, "fn": 79, "num_prs": 50 }, "all": { "precision": 38.1, "recall": 49.7, "f1": 43.1, "f2": 46.8, "tp": 86, "fp": 140, "fn": 87, "num_prs": 50 } }, "graphite": { "strict": { "precision": 71.4, "recall": 7.2, "f1": 13.1, "f2": 8.8, "tp": 10, "fp": 4, "fn": 129, "num_prs": 50 }, "core": { "precision": 73.3, "recall": 7.0, "f1": 12.7, "f2": 8.5, "tp": 11, "fp": 4, "fn": 147, "num_prs": 50 }, "all": { "precision": 73.3, "recall": 6.4, "f1": 11.7, "f2": 7.8, "tp": 11, "fp": 4, "fn": 162, "num_prs": 50 } }, "claude-code": { "strict": { "precision": 38.2, "recall": 43.2, "f1": 40.5, "f2": 42.1, "tp": 60, "fp": 97, "fn": 79, "num_prs": 50 }, "core": { "precision": 40.1, "recall": 41.1, "f1": 40.6, "f2": 40.9, "tp": 65, "fp": 97, "fn": 93, "num_prs": 50 }, "all": { "precision": 42.3, "recall": 41.0, "f1": 41.6, "f2": 41.3, "tp": 71, "fp": 97, "fn": 102, "num_prs": 50 } }, "kodus-v2": { "strict": { "precision": 50.5, "recall": 36.0, "f1": 42.0, "f2": 38.2, "tp": 50, "fp": 49, "fn": 89, "num_prs": 50 }, "core": { "precision": 52.4, "recall": 34.2, "f1": 41.4, "f2": 36.7, "tp": 54, "fp": 49, "fn": 104, "num_prs": 50 }, "all": { "precision": 53.8, "recall": 32.9, "f1": 40.9, "f2": 35.7, "tp": 57, "fp": 49, "fn": 116, "num_prs": 50 } }, "devin": { "strict": { "precision": 56.6, "recall": 43.2, "f1": 49.0, "f2": 45.3, "tp": 60, "fp": 46, "fn": 79, "num_prs": 50 }, "core": { "precision": 58.6, "recall": 41.1, "f1": 48.3, "f2": 43.7, "tp": 65, "fp": 46, "fn": 93, "num_prs": 50 }, "all": { "precision": 59.3, "recall": 38.7, "f1": 46.9, "f2": 41.6, "tp": 67, "fp": 46, "fn": 106, "num_prs": 50 } }, "claude": { "strict": { "precision": 36.7, "recall": 38.8, "f1": 37.8, "f2": 38.4, "tp": 54, "fp": 93, "fn": 85, "num_prs": 50 }, "core": { "precision": 38.8, "recall": 37.3, "f1": 38.1, "f2": 37.6, "tp": 59, "fp": 93, "fn": 99, "num_prs": 50 }, "all": { "precision": 41.1, "recall": 37.6, "f1": 39.3, "f2": 38.2, "tp": 65, "fp": 93, "fn": 108, "num_prs": 50 } }, "macroscope": { "strict": { "precision": 47.7, "recall": 44.6, "f1": 46.1, "f2": 45.2, "tp": 62, "fp": 68, "fn": 77, "num_prs": 50 }, "core": { "precision": 48.9, "recall": 41.1, "f1": 44.7, "f2": 42.5, "tp": 65, "fp": 68, "fn": 93, "num_prs": 50 }, "all": { "precision": 50.0, "recall": 39.3, "f1": 44.0, "f2": 41.1, "tp": 68, "fp": 68, "fn": 105, "num_prs": 50 } }, "baz": { "strict": { "precision": 43.9, "recall": 36.0, "f1": 39.5, "f2": 37.3, "tp": 50, "fp": 64, "fn": 89, "num_prs": 50 }, "core": { "precision": 44.8, "recall": 32.9, "f1": 38.0, "f2": 34.8, "tp": 52, "fp": 64, "fn": 106, "num_prs": 50 }, "all": { "precision": 46.2, "recall": 31.8, "f1": 37.7, "f2": 33.9, "tp": 55, "fp": 64, "fn": 118, "num_prs": 50 } }, "augment": { "strict": { "precision": 45.9, "recall": 59.7, "f1": 51.9, "f2": 56.3, "tp": 83, "fp": 98, "fn": 56, "num_prs": 50 }, "core": { "precision": 49.5, "recall": 60.8, "f1": 54.5, "f2": 58.1, "tp": 96, "fp": 98, "fn": 62, "num_prs": 50 }, "all": { "precision": 50.8, "recall": 58.4, "f1": 54.3, "f2": 56.7, "tp": 101, "fp": 98, "fn": 72, "num_prs": 50 } }, "greptile-v4-1": { "strict": { "precision": 40.2, "recall": 50.4, "f1": 44.7, "f2": 47.9, "tp": 70, "fp": 104, "fn": 69, "num_prs": 50 }, "core": { "precision": 42.2, "recall": 48.1, "f1": 45.0, "f2": 46.8, "tp": 76, "fp": 104, "fn": 82, "num_prs": 50 }, "all": { "precision": 44.7, "recall": 48.6, "f1": 46.5, "f2": 47.7, "tp": 84, "fp": 104, "fn": 89, "num_prs": 50 } }, "bugbot": { "strict": { "precision": 49.6, "recall": 47.5, "f1": 48.5, "f2": 47.9, "tp": 66, "fp": 67, "fn": 73, "num_prs": 50 }, "core": { "precision": 51.8, "recall": 45.6, "f1": 48.5, "f2": 46.7, "tp": 72, "fp": 67, "fn": 86, "num_prs": 50 }, "all": { "precision": 52.8, "recall": 43.4, "f1": 47.6, "f2": 45.0, "tp": 75, "fp": 67, "fn": 98, "num_prs": 50 } }, "copilot-v2": { "strict": { "precision": 30.4, "recall": 63.3, "f1": 41.1, "f2": 52.1, "tp": 88, "fp": 201, "fn": 51, "num_prs": 50 }, "core": { "precision": 33.2, "recall": 63.3, "f1": 43.6, "f2": 53.6, "tp": 100, "fp": 201, "fn": 58, "num_prs": 50 }, "all": { "precision": 35.2, "recall": 63.0, "f1": 45.1, "f2": 54.4, "tp": 109, "fp": 201, "fn": 64, "num_prs": 50 } }, "codeant-v2": { "strict": { "precision": 29.3, "recall": 38.1, "f1": 33.1, "f2": 36.0, "tp": 53, "fp": 128, "fn": 86, "num_prs": 50 }, "core": { "precision": 30.4, "recall": 35.4, "f1": 32.7, "f2": 34.3, "tp": 56, "fp": 128, "fn": 102, "num_prs": 50 }, "all": { "precision": 31.2, "recall": 33.5, "f1": 32.3, "f2": 33.0, "tp": 58, "fp": 128, "fn": 115, "num_prs": 50 } }, "qodo-extended-v2": { "strict": { "precision": 55.9, "recall": 61.2, "f1": 58.4, "f2": 60.0, "tp": 85, "fp": 67, "fn": 54, "num_prs": 50 }, "core": { "precision": 58.4, "recall": 59.5, "f1": 58.9, "f2": 59.3, "tp": 94, "fp": 67, "fn": 64, "num_prs": 50 }, "all": { "precision": 58.9, "recall": 55.5, "f1": 57.1, "f2": 56.1, "tp": 96, "fp": 67, "fn": 77, "num_prs": 50 } }, "gemini-v2": { "strict": { "precision": 36.2, "recall": 54.0, "f1": 43.4, "f2": 49.1, "tp": 75, "fp": 132, "fn": 64, "num_prs": 50 }, "core": { "precision": 38.0, "recall": 51.3, "f1": 43.7, "f2": 47.9, "tp": 81, "fp": 132, "fn": 77, "num_prs": 50 }, "all": { "precision": 40.3, "recall": 51.4, "f1": 45.2, "f2": 48.7, "tp": 89, "fp": 132, "fn": 84, "num_prs": 50 } }, "qodo-v2": { "strict": { "precision": 44.1, "recall": 64.7, "f1": 52.5, "f2": 59.2, "tp": 90, "fp": 114, "fn": 49, "num_prs": 50 }, "core": { "precision": 46.5, "recall": 62.7, "f1": 53.4, "f2": 58.6, "tp": 99, "fp": 114, "fn": 59, "num_prs": 50 }, "all": { "precision": 47.5, "recall": 59.5, "f1": 52.8, "f2": 56.7, "tp": 103, "fp": 114, "fn": 70, "num_prs": 50 } }, "gitlab": { "strict": { "precision": 42.2, "recall": 48.9, "f1": 45.3, "f2": 47.4, "tp": 68, "fp": 93, "fn": 71, "num_prs": 50 }, "core": { "precision": 45.0, "recall": 48.1, "f1": 46.5, "f2": 47.4, "tp": 76, "fp": 93, "fn": 82, "num_prs": 50 }, "all": { "precision": 46.9, "recall": 47.4, "f1": 47.1, "f2": 47.3, "tp": 82, "fp": 93, "fn": 91, "num_prs": 50 } }, "kg": { "strict": { "precision": 53.5, "recall": 16.5, "f1": 25.3, "f2": 19.2, "tp": 23, "fp": 20, "fn": 116, "num_prs": 50 }, "core": { "precision": 54.5, "recall": 15.2, "f1": 23.8, "f2": 17.8, "tp": 24, "fp": 20, "fn": 134, "num_prs": 50 }, "all": { "precision": 55.6, "recall": 14.5, "f1": 22.9, "f2": 17.0, "tp": 25, "fp": 20, "fn": 148, "num_prs": 50 } } } } }, "predefined_filters": [ { "id": "high_precision", "label": "Highest Precision", "filters": {}, "sort": "precision", "description": "Tools ranked by precision - fewer false positives, more reliable findings", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "graphite", "best_score": 100.0 }, { "id": "tool_kg_language_ruby", "label": "Best for Ruby (Precision)", "filters": { "language": [ "Ruby" ] }, "description": "Ruby codebases with Rails patterns.", "best_model": "anthropic_claude-sonnet-4-5-20250929", "best_tool": "kg", "best_score": 100.0, "sort": "precision" }, { "id": "tool_gitar-v2_domain_ui", "label": "Best for Ui (Recall)", "filters": { "domain": [ "UI" ] }, "description": "User interface and frontend.", "best_model": "anthropic_claude-sonnet-4-5-20250929", "best_tool": "gitar-v2", "best_score": 91.7, "sort": "recall" }, { "id": "tool_sourcery_difficulty_moderate_context_file", "label": "Moderate Bugs + File Context (Recall)", "filters": { "difficulty": [ "moderate" ], "context": [ "file" ] }, "description": "Requires careful reading. Requires full file understanding.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "sourcery", "best_score": 90.0, "sort": "recall" }, { "id": "tool_coderabbit_language_typescript_domain_scheduling", "label": "Typescript + Scheduling (Recall)", "filters": { "language": [ "TypeScript" ], "domain": [ "scheduling" ] }, "description": "TypeScript codebases with frontend patterns. Task scheduling.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "coderabbit", "best_score": 88.9, "sort": "recall" }, { "id": "go_small", "label": "Best for Small Go PRs", "filters": { "language": [ "Go" ], "pr_size": [ "small" ] }, "description": "Go codebases with concurrency patterns. Small PRs with 1-2 files, easier to review thoroughly", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 88.2 }, { "id": "complexity_complex", "label": "Best for Complex Code", "filters": { "complexity": [ "complex" ] }, "description": "Deep logic and dependencies.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "augment", "best_score": 81.1 }, { "id": "domain_ui", "label": "Best for Ui", "filters": { "domain": [ "UI" ] }, "description": "User interface and frontend.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "bugbot", "best_score": 78.3 }, { "id": "risk_critical", "label": "Best for Critical Risk", "filters": { "risk": [ "critical" ] }, "description": "Critical security or data corruption risk.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-v2", "best_score": 75.9 }, { "id": "tool_codeant-v2_pr_size_small_change_type_performance", "label": "Small PRs + Performance Optimization (Precision)", "filters": { "pr_size": [ "small" ], "change_type": [ "performance" ] }, "description": "Small PRs with 1-2 files, easier to review thoroughly Performance optimization changes", "best_model": "anthropic_claude-sonnet-4-5-20250929", "best_tool": "codeant-v2", "best_score": 75.0, "sort": "precision" }, { "id": "tool_gitlab_change_type_bug_fix_context_cross_file", "label": "Bug Fixes + Cross-File", "filters": { "change_type": [ "bug_fix" ], "context": [ "cross_file" ] }, "description": "Bug fixes and issue resolution Spans multiple files.", "best_model": "openai_gpt-5.2", "best_tool": "gitlab", "best_score": 74.3 }, { "id": "tool_baz_language_java_domain_authentication", "label": "Java + Authentication", "filters": { "language": [ "Java" ], "domain": [ "authentication" ] }, "description": "Java codebases with OOP patterns. Auth and access control.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "baz", "best_score": 74.1 }, { "id": "python_medium", "label": "Best for Medium Python PRs", "filters": { "language": [ "Python" ], "pr_size": [ "medium" ] }, "description": "Python codebases with dynamic typing. Medium PRs with 3-5 files, typical feature development", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 72.7 }, { "id": "domain_concurrency", "label": "Best for Concurrency", "filters": { "domain": [ "concurrency" ] }, "description": "Threading and async operations.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "macroscope", "best_score": 69.6 }, { "id": "tool_claude_change_type_bug_fix", "label": "Best for Bug Fixes", "filters": { "change_type": [ "bug_fix" ] }, "description": "Bug fixes and issue resolution", "best_model": "anthropic_claude-sonnet-4-5-20250929", "best_tool": "claude", "best_score": 68.3 }, { "id": "tool_greptile-v4-1_language_ruby_pr_size_medium", "label": "Ruby + Medium PRs", "filters": { "language": [ "Ruby" ], "pr_size": [ "medium" ] }, "description": "Ruby codebases with Rails patterns. Medium PRs with 3-5 files, typical feature development", "best_model": "openai_gpt-5.2", "best_tool": "greptile-v4-1", "best_score": 66.7 }, { "id": "high_recall", "label": "Highest Recall", "filters": {}, "sort": "recall", "description": "Tools ranked by recall - catches more issues, may have more noise", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "copilot-v2", "best_score": 66.5 }, { "id": "tool_kodus-v2_language_ruby_change_type_feature", "label": "Ruby + Features", "filters": { "language": [ "Ruby" ], "change_type": [ "feature" ] }, "description": "Ruby codebases with Rails patterns. New feature implementation", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "kodus-v2", "best_score": 61.2 }, { "id": "difficulty_moderate", "label": "Best for Moderate Bugs", "filters": { "difficulty": [ "moderate" ] }, "description": "Requires careful reading.", "best_model": "anthropic_claude-sonnet-4-5-20250929", "best_tool": "devin", "best_score": 57.9 }, { "id": "tool_gemini-v2_language_ruby_difficulty_subtle", "label": "Ruby + Subtle Bugs", "filters": { "language": [ "Ruby" ], "difficulty": [ "subtle" ] }, "description": "Ruby codebases with Rails patterns. Non-obvious, needs domain knowledge.", "best_model": "openai_gpt-5.2", "best_tool": "gemini-v2", "best_score": 56.4 }, { "id": "language_go", "label": "Best for Go", "filters": { "language": [ "Go" ] }, "description": "Go codebases with concurrency patterns.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 84.2 }, { "id": "change_type_performance", "label": "Best for Performance Optimization", "filters": { "change_type": [ "performance" ] }, "description": "Performance optimization changes", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 82.9 }, { "id": "complex_subtle", "label": "Complex & Subtle", "filters": { "complexity": [ "complex" ], "difficulty": [ "subtle", "very_subtle" ] }, "description": "Deep logic and dependencies. Non-obvious, needs domain knowledge.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "augment", "best_score": 81.1 }, { "id": "pr_size_small", "label": "Best for Small PRs", "filters": { "pr_size": [ "small" ] }, "description": "Small PRs with 1-2 files, easier to review thoroughly", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 79.6 }, { "id": "domain_caching", "label": "Best for Caching", "filters": { "domain": [ "caching" ] }, "description": "Cache and memoization.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "augment", "best_score": 78.6 }, { "id": "pr_size_large", "label": "Best for Large PRs", "filters": { "pr_size": [ "large" ] }, "description": "Large PRs with 6+ files, complex changes requiring careful review", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "augment", "best_score": 74.5 }, { "id": "change_type_bug_fix", "label": "Best for Bug Fixes", "filters": { "change_type": [ "bug_fix" ] }, "description": "Bug fixes and issue resolution", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "augment", "best_score": 73.2 }, { "id": "language_python", "label": "Best for Python", "filters": { "language": [ "Python" ] }, "description": "Python codebases with dynamic typing.", "best_model": "openai_gpt-5.2", "best_tool": "cubic-v2", "best_score": 72.5 }, { "id": "high_risk_auth", "label": "High Risk Auth", "filters": { "risk": [ "high", "critical" ], "domain": [ "authentication" ] }, "description": "Significant impact, potential data loss. Auth and access control.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 71.9 }, { "id": "context_file", "label": "Best for File Context", "filters": { "context": [ "file" ] }, "description": "Requires full file understanding.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 71.6 }, { "id": "risk_high", "label": "Best for High Risk", "filters": { "risk": [ "high" ] }, "description": "Significant impact, potential data loss.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 71.3 }, { "id": "language_java", "label": "Best for Java", "filters": { "language": [ "Java" ] }, "description": "Java codebases with OOP patterns.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 70.8 }, { "id": "domain_authentication", "label": "Best for Authentication", "filters": { "domain": [ "authentication" ] }, "description": "Auth and access control.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 69.2 }, { "id": "ruby_medium", "label": "Best for Medium Ruby PRs", "filters": { "language": [ "Ruby" ], "pr_size": [ "medium" ] }, "description": "Ruby codebases with Rails patterns. Medium PRs with 3-5 files, typical feature development", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 68.8 }, { "id": "java_medium", "label": "Best for Medium Java PRs", "filters": { "language": [ "Java" ], "pr_size": [ "medium" ] }, "description": "Java codebases with OOP patterns. Medium PRs with 3-5 files, typical feature development", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 68.6 }, { "id": "domain_scheduling", "label": "Best for Scheduling", "filters": { "domain": [ "scheduling" ] }, "description": "Task scheduling.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 68.3 }, { "id": "language_typescript", "label": "Best for Typescript", "filters": { "language": [ "TypeScript" ] }, "description": "TypeScript codebases with frontend patterns.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "augment", "best_score": 68.2 }, { "id": "context_cross_file", "label": "Best for Cross-File", "filters": { "context": [ "cross_file" ] }, "description": "Spans multiple files.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 67.5 }, { "id": "pr_size_medium", "label": "Best for Medium PRs", "filters": { "pr_size": [ "medium" ] }, "description": "Medium PRs with 3-5 files, typical feature development", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 67.1 }, { "id": "concern_reliability", "label": "Best for Reliability", "filters": { "concern": [ "reliability" ] }, "description": "Error handling and system stability", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-v2", "best_score": 66.7 }, { "id": "difficulty_subtle", "label": "Best for Subtle Bugs", "filters": { "difficulty": [ "subtle" ] }, "description": "Non-obvious, needs domain knowledge.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 66.4 }, { "id": "concern_correctness", "label": "Best for Correctness", "filters": { "concern": [ "correctness" ] }, "description": "Logical correctness and expected behavior", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 66.4 }, { "id": "high_f1", "label": "Highest Score", "filters": {}, "sort": "f1", "description": "Tools ranked by F-beta score (default F2, recall-weighted)", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 65.8 }, { "id": "concern_security", "label": "Best for Security", "filters": { "concern": [ "security" ] }, "description": "Security vulnerabilities and attack vectors", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 65.2 }, { "id": "security_critical", "label": "Security Critical", "filters": { "concern": [ "security" ], "risk": [ "high", "critical" ] }, "description": "Security vulnerabilities and attack vectors Significant impact, potential data loss.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 65.2 }, { "id": "language_ruby", "label": "Best for Ruby", "filters": { "language": [ "Ruby" ] }, "description": "Ruby codebases with Rails patterns.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 63.8 }, { "id": "complexity_moderate", "label": "Best for Moderate Code", "filters": { "complexity": [ "moderate" ] }, "description": "Some abstraction.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 63.4 }, { "id": "change_type_feature", "label": "Best for Features", "filters": { "change_type": [ "feature" ] }, "description": "New feature implementation", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 62.9 }, { "id": "risk_medium", "label": "Best for Medium Risk", "filters": { "risk": [ "medium" ] }, "description": "Moderate user impact.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 59.5 } ], "tool_display_names": { "graphite": "Graphite", "qodo": "Qodo", "gemini": "Gemini", "claude": "Claude Code", "augment": "Augment", "bugbot": "Cursor Bugbot", "coderabbit": "CodeRabbit", "propel": "Propel", "copilot": "GitHub Copilot", "baz": "Baz", "greptile": "Greptile", "kg": "KG", "entelligence": "Entelligence", "cubic-v2": "Cubic v2", "sourcery": "Sourcery", "mesa": "Mesa", "codeant-v2": "CodeAnt", "claude-code": "Claude Code (CLI)", "devin": "Devin", "kodus-v2": "Kodus", "greptile-v4-1": "Greptile v4", "gemini-v2": "Gemini", "gitlab": "GitLab Duo Code Review", "qodo-v2": "Qodo v2", "qodo-extended-v2": "Qodo Extended", "macroscope": "Macroscope", "gitar-v2": "Gitar" }, "tool_colors": { "graphite": "#6366f1", "qodo": "#8b5cf6", "gemini": "#06b6d4", "claude": "#f59e0b", "augment": "#10b981", "bugbot": "#3b82f6", "coderabbit": "#ec4899", "propel": "#14b8a6", "propel-v2": "#0d9488", "copilot": "#6b7280", "baz": "#f97316", "greptile": "#22c55e", "kg": "#a855f7", "entelligence": "#0ea5e9", "cubic-dev": "#d946ef", "sourcery": "#84cc16", "mesa": "#f43f5e", "codeant": "#e11d48", "codeant-v2": "#fb7185", "claude-code": "#d97706", "devin": "#7c3aed", "kodus-v2": "#059669", "greptile-v4": "#16a34a", "qodo-v2": "#7c3aed", "qodo-extended-v2": "#6d28d9", "macroscope": "#0891b2", "cubic-v2": "#c026d3" }, "default_model": "anthropic_claude-opus-4-5-20251101", "min_prs_threshold": 5, "profiles": { "strict": { "categories": [ "api", "bug", "concurrency", "data", "security" ], "description": "Bug, Security, Concurrency, Data, API" }, "core": { "categories": [ "api", "bug", "concurrency", "data", "doc_defect", "perf", "security", "test_gap" ], "description": "Bug, Security, Concurrency, Data, API, Perf, Test Gap, Doc Defect" }, "all": { "categories": [ "api", "bug", "concurrency", "data", "doc_defect", "perf", "security", "speculative", "style", "test_gap" ], "description": "All categories (includes Style, Speculative)" } }, "default_profile": "core", "default_beta": 2.0 }