{"schema":"hugging-bay.agent-evals.v1","status":"measured","generatedAt":"2026-11-09T23:30:10.848615+00:00","source":"2f9258f3ce9332fec6c33b5a521bc23fff446b93","target":"https://huggingbay.xyz/api/mcp?profile=core","scope":"public-live","harnessSha256":"a7132a1acca1245264c0e4ff42013fa5863d03d11e7e288ee2b099fee1237888","workingTreeDirty":false,"taskSetSha256":"b66acbd83e455fdce9deee0fc02fc23612c326800d558a68351561b5778906e3","clients":[{"client":"claude","kind":"agent","version":"claude-haiku-5-5","planned":6,"attempted":6,"unavailable":0,"firstCallSuccess":5,"toolChoiceCorrect":4,"taskSuccess":2,"tokens":31248,"medianLatencyMs":5846,"results":[{"task":"m2-chat","attempted":true,"calls":1,"firstCallSuccess":false,"toolChoiceCorrect":false,"taskSuccess":false,"argumentsCorrect":false,"tokens":3656,"latencyMs":5163,"error":null,"model":"claude-haiku-5-5","stopReason":"end_turn","completed":true},{"task":"generic-no-proof","attempted":true,"calls":2,"firstCallSuccess":true,"toolChoiceCorrect":false,"taskSuccess":false,"argumentsCorrect":true,"tokens":6825,"latencyMs":8225,"error":null,"model":"claude-haiku-5-5","stopReason":"end_turn","completed":true},{"task":"search-qwen","attempted":true,"calls":1,"firstCallSuccess":true,"toolChoiceCorrect":true,"taskSuccess":true,"argumentsCorrect":true,"tokens":6182,"latencyMs":4414,"error":null,"model":"claude-haiku-5-5","stopReason":"end_turn","completed":true},{"task":"chat-8gb","attempted":true,"calls":1,"firstCallSuccess":true,"toolChoiceCorrect":true,"taskSuccess":false,"argumentsCorrect":true,"tokens":5376,"latencyMs":5554,"error":null,"model":"claude-haiku-5-5","stopReason":"end_turn","completed":true},{"task":"known-id","attempted":true,"calls":1,"firstCallSuccess":true,"toolChoiceCorrect":true,"taskSuccess":false,"argumentsCorrect":true,"tokens":4609,"latencyMs":6829,"error":null,"model":"claude-haiku-5-5","stopReason":"end_turn","completed":true},{"task":"serve-known","attempted":true,"calls":1,"firstCallSuccess":true,"toolChoiceCorrect":true,"taskSuccess":true,"argumentsCorrect":true,"tokens":4600,"latencyMs":6139,"error":null,"model":"claude-haiku-5-5","stopReason":"end_turn","completed":true}]},{"client":"codex","kind":"agent","version":"codex-cli 0.165.0","planned":6,"attempted":6,"unavailable":0,"firstCallSuccess":4,"toolChoiceCorrect":2,"taskSuccess":1,"tokens":551773,"medianLatencyMs":19620,"results":[{"task":"m2-chat","attempted":true,"calls":1,"firstCallSuccess":false,"toolChoiceCorrect":false,"taskSuccess":false,"argumentsCorrect":false,"tokens":62789,"latencyMs":14299,"error":null,"exitCode":0,"model":"unreported_client_default","completed":true},{"task":"generic-no-proof","attempted":true,"calls":3,"firstCallSuccess":false,"toolChoiceCorrect":false,"taskSuccess":false,"argumentsCorrect":true,"tokens":112066,"latencyMs":26408,"error":null,"exitCode":0,"model":"unreported_client_default","completed":true},{"task":"search-qwen","attempted":true,"calls":1,"firstCallSuccess":true,"toolChoiceCorrect":true,"taskSuccess":false,"argumentsCorrect":true,"tokens":63038,"latencyMs":13971,"error":null,"exitCode":0,"model":"unreported_client_default","completed":true},{"task":"chat-8gb","attempted":true,"calls":2,"firstCallSuccess":true,"toolChoiceCorrect":true,"taskSuccess":false,"argumentsCorrect":true,"tokens":114977,"latencyMs":21657,"error":null,"exitCode":0,"model":"unreported_client_default","completed":true},{"task":"known-id","attempted":true,"calls":2,"firstCallSuccess":true,"toolChoiceCorrect":false,"taskSuccess":false,"argumentsCorrect":false,"tokens":86744,"latencyMs":17583,"error":null,"exitCode":0,"model":"unreported_client_default","completed":true},{"task":"serve-known","attempted":true,"calls":3,"firstCallSuccess":true,"toolChoiceCorrect":false,"taskSuccess":true,"argumentsCorrect":false,"tokens":112159,"latencyMs":22304,"error":null,"exitCode":0,"model":"unreported_client_default","completed":true}]},{"client":"jsonrpc","kind":"transport","version":"stdlib-jsonrpc-v1","planned":6,"attempted":6,"unavailable":0,"firstCallSuccess":4,"toolChoiceCorrect":null,"taskSuccess":4,"tokens":null,"medianLatencyMs":824,"results":[{"task":"m2-chat","attempted":true,"calls":1,"firstCallSuccess":false,"toolChoiceCorrect":true,"taskSuccess":false,"argumentsCorrect":true,"tokens":null,"latencyMs":1068,"error":null},{"task":"generic-no-proof","attempted":true,"calls":1,"firstCallSuccess":false,"toolChoiceCorrect":true,"taskSuccess":false,"argumentsCorrect":true,"tokens":null,"latencyMs":204,"error":null},{"task":"search-qwen","attempted":true,"calls":1,"firstCallSuccess":true,"toolChoiceCorrect":true,"taskSuccess":true,"argumentsCorrect":true,"tokens":null,"latencyMs":615,"error":null},{"task":"chat-8gb","attempted":true,"calls":1,"firstCallSuccess":true,"toolChoiceCorrect":true,"taskSuccess":true,"argumentsCorrect":true,"tokens":null,"latencyMs":738,"error":null},{"task":"known-id","attempted":true,"calls":1,"firstCallSuccess":true,"toolChoiceCorrect":true,"taskSuccess":true,"argumentsCorrect":true,"tokens":null,"latencyMs":1762,"error":null},{"task":"serve-known","attempted":true,"calls":1,"firstCallSuccess":true,"toolChoiceCorrect":true,"taskSuccess":true,"argumentsCorrect":true,"tokens":null,"latencyMs":910,"error":null}]}],"taskIds":["m2-chat","generic-no-proof","search-qwen","chat-8gb","known-id","serve-known"],"clientRunTimes":{"claude":"2026-11-09T23:27:25.642225+00:00","codex":"2026-11-09T23:28:08.202257+00:00","jsonrpc":"2026-11-09T23:30:10.848615+00:00"},"schedule":{"runner":"mac-user-launchd","dueUtc":"09:00","clientTaskInvocationsMax":18,"claudeApiRequestsMax":6,"observedMcpCalls":25,"internalClientRequestsInstrumented":false,"budgetOverrun":false},"note":"Six fixed tasks per client. Failed and unavailable attempts are retained. Internal client HTTP retries are not instrumented; the three-tool prompt limit is not a hard provider-side cap.","suiteExitCodes":{"claude":1,"codex":1,"jsonrpc":1},"delivery":{"source":"public-gcs","url":"https://storage.googleapis.com/hb-desktop-releases-barneyserver/agent-evals/latest.json","fetchedAt":"2026-11-13T21:31:04.793Z","stale":true}}