From 9e5285dbd7d6af67722b9214c42f4a4b1a0330ef Mon Sep 17 00:00:00 2001 From: rzonyx <77541804+rzonyx@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:17:35 +0700 Subject: [PATCH 1/7] A job summary that says how many tasks were solved, at what cost The summary had each eval's mean reward and pass@k. It now reads the job's config.json, lock.json and each trial's result.json too: - a row per eval: solved trials and the pass rate (a trial is solved when its reward is 1), mean reward, pass@k, errors, the agent's time in total and per trial, tokens in and out, and the cost in total and per solved trial; - what ran: hbench and its results epoch, the agent and model, the tasks with their registry and manifest digest, attempts, concurrency and when the job ran; - the tasks' own metrics, the errors by type, and a row per trial, errors and failures first. New outputs: n-solved, pass-rate and cost-usd. tests/test_report.py checks the report on two job directories, one an agent's with a solved, a failed and a timed-out trial, one the oracle's with reward.txt. --- .github/workflows/test.yml | 12 +- README.md | 6 +- action.yml | 9 + docs/REFERENCE.md | 33 +- scripts/__pycache__/report.cpython-314.pyc | Bin 0 -> 25433 bytes scripts/report.py | 332 ++++++++++++++---- tests/__pycache__/test_report.cpython-314.pyc | Bin 0 -> 8433 bytes tests/report/agent-job/config.json | 17 + .../agent-job/count-words__b2/result.json | 30 ++ .../agent-job/fix-median__a1/result.json | 30 ++ .../agent-job/hello-world__c3/result.json | 30 ++ tests/report/agent-job/lock.json | 8 + tests/report/agent-job/result.json | 38 ++ tests/report/oracle-job/config.json | 13 + .../oracle-job/hello-world__d4/result.json | 26 ++ tests/report/oracle-job/lock.json | 6 + tests/report/oracle-job/result.json | 27 ++ tests/test_report.py | 90 +++++ 18 files changed, 635 insertions(+), 72 deletions(-) create mode 100644 scripts/__pycache__/report.cpython-314.pyc create mode 100644 tests/__pycache__/test_report.cpython-314.pyc create mode 100644 tests/report/agent-job/config.json create mode 100644 tests/report/agent-job/count-words__b2/result.json create mode 100644 tests/report/agent-job/fix-median__a1/result.json create mode 100644 tests/report/agent-job/hello-world__c3/result.json create mode 100644 tests/report/agent-job/lock.json create mode 100644 tests/report/agent-job/result.json create mode 100644 tests/report/oracle-job/config.json create mode 100644 tests/report/oracle-job/hello-world__d4/result.json create mode 100644 tests/report/oracle-job/lock.json create mode 100644 tests/report/oracle-job/result.json create mode 100644 tests/test_report.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2e7ed24..a9963dd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ defaults: jobs: lint: - name: actionlint, shellcheck and the report's syntax + name: actionlint, shellcheck and the report's tests runs-on: ubuntu-24.04 timeout-minutes: 10 env: @@ -41,8 +41,8 @@ jobs: run: '"$RUNNER_TEMP/actionlint" -color' - name: Lint the scripts run: shellcheck scripts/*.sh - - name: Compile the report - run: python3 -m py_compile scripts/report.py + - name: Test the report + run: python3 -m unittest discover --start-directory tests --pattern 'test_*.py' --verbose install: name: Install ${{ matrix.version }} on ${{ matrix.os }} @@ -99,10 +99,13 @@ jobs: MEAN_REWARD: ${{ steps.hbench.outputs.mean-reward }} N_TRIALS: ${{ steps.hbench.outputs.n-trials }} N_ERRORS: ${{ steps.hbench.outputs.n-errors }} + N_SOLVED: ${{ steps.hbench.outputs.n-solved }} + PASS_RATE: ${{ steps.hbench.outputs.pass-rate }} RESULT_PATH: ${{ steps.hbench.outputs.result-path }} ARTIFACT_ID: ${{ steps.hbench.outputs.artifact-id }} run: | [[ "$MEAN_REWARD" == 1.0 && "$N_TRIALS" == 1 && "$N_ERRORS" == 0 ]] + [[ "$N_SOLVED" == 1 && "$PASS_RATE" == 1.0 ]] [[ -f "$RESULT_PATH" && -n "$ARTIFACT_ID" ]] run-below-threshold: @@ -128,8 +131,9 @@ jobs: OUTCOME: ${{ steps.hbench.outcome }} MEAN_REWARD: ${{ steps.hbench.outputs.mean-reward }} N_ERRORS: ${{ steps.hbench.outputs.n-errors }} + N_SOLVED: ${{ steps.hbench.outputs.n-solved }} run: | - [[ "$OUTCOME" == failure && "$MEAN_REWARD" == 0.0 && "$N_ERRORS" == 0 ]] + [[ "$OUTCOME" == failure && "$MEAN_REWARD" == 0.0 && "$N_ERRORS" == 0 && "$N_SOLVED" == 0 ]] run-oci-dataset: name: Run the oracle on an OCI dataset, locked by digest diff --git a/README.md b/README.md index 6830a52..94e6001 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,11 @@ For hbench itself, the docs are at [nodeq.tech/docs](https://nodeq.tech/docs): ``` With none of `path`, `dataset` or `config`, the action only installs hbench, and later steps can call -`hbench` themselves. Useful outputs: `mean-reward`, `n-trials`, `n-errors` and `job-dir`. +`hbench` themselves. Useful outputs: `pass-rate`, `n-solved`, `mean-reward`, `cost-usd` and `job-dir`. + +The job summary shows, for each agent and model, how many tasks it solved, its mean reward and pass@k, its +errors, the time it spent, its tokens and its cost per solved task; then what ran (the hbench release, the +agent, the model, the tasks and their digest), the errors by type, and a row per task. A `registry.json` lists datasets and where each of their tasks is: a git repository at a commit, or a folder in your repository, read from `working-directory`: diff --git a/action.yml b/action.yml index 01b0a3b..53695f5 100644 --- a/action.yml +++ b/action.yml @@ -96,6 +96,15 @@ outputs: n-errors: description: Trials that ended in an error. value: ${{ steps.run.outputs.n-errors }} + n-solved: + description: Trials solved, with a reward of 1. + value: ${{ steps.run.outputs.n-solved }} + pass-rate: + description: Solved trials over all trials, from 0 to 1. + value: ${{ steps.run.outputs.pass-rate }} + cost-usd: + description: The agents' cost in US dollars, where hbench knows it. Empty for agents that report none, such as the oracle. + value: ${{ steps.run.outputs.cost-usd }} artifact-id: description: ID of the uploaded artifact. value: ${{ steps.upload.outputs.artifact-id }} diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index dfbb7da..10c6446 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -47,10 +47,26 @@ The [README](../README.md) has the short version. | `mean-reward` | Mean reward over every trial of every eval. | | `n-trials` | Trials run. | | `n-errors` | Trials that ended in an error. | +| `n-solved` | Trials solved, with a reward of 1. | +| `pass-rate` | Solved trials over all trials, from 0 to 1. | +| `cost-usd` | The agents' cost in US dollars, where hbench knows it. Empty for agents that report none, such as the oracle. | | `artifact-id` | ID of the uploaded artifact. | -`mean-reward` weighs each eval's mean reward by its trials. The job summary has a row per eval and lists the -trials below a reward of 1. +A trial is solved when its reward is 1, so `pass-rate` is the share of trials that got full marks. +`mean-reward` weighs each eval's mean reward by its trials; it differs from `pass-rate` when tasks give +partial rewards. + +The job summary has: + +- a row per eval (agent, model and tasks): solved trials and the pass rate, mean reward, pass@k, errors, + the agent's time in total and per trial (median), tokens in and out, and the cost in total and per + solved trial; +- whether the mean reward reached `min-mean-reward`; +- what ran: the hbench release and results epoch, the agent and model, the tasks with their registry and + manifest digest, attempts and concurrency, and when the job ran; +- the tasks' own metrics, the other keys of their `reward.json`, as means; +- the errors by type, with the trials they hit; +- a row per trial, errors and failures first: result, reward, agent time, tokens and cost. ## More examples @@ -162,15 +178,16 @@ Pass any other `hbench run` flag through `args`, one argument per line: action.yml the inputs, outputs and three steps: install, run, upload scripts/install.sh download, checksum, provenance, tool cache scripts/run.sh build the hbench run command line, run it, report -scripts/report.py result.json to outputs, job summary and the threshold +scripts/report.py the job's results to outputs, job summary and the threshold +tests/test_report.py the report, on the two job directories in tests/report/ tests/tasks/ a one-task folder the tests run ``` -`.github/workflows/test.yml` runs on GitHub-hosted runners only: actionlint and shellcheck; installing -`latest` and a pinned release on Linux x86_64 and arm64 and macOS arm64 and x86_64, with provenance checked; -the oracle on the local task with `min-mean-reward: '1'`; the `nop` agent, which must fail the step on its -score; two tasks of an OCI dataset with `--lock-write`; and a dataset listed in a `registry.json`, by path and -by URL. +`.github/workflows/test.yml` runs on GitHub-hosted runners only: actionlint, shellcheck and the report's unit +tests; installing `latest` and a pinned release on Linux x86_64 and arm64 and macOS arm64 and x86_64, with +provenance checked; the oracle on the local task with `min-mean-reward: '1'`; the `nop` agent, which must fail +the step on its score; two tasks of an OCI dataset with `--lock-write`; and a dataset listed in a +`registry.json`, by path and by URL. To try the scripts outside GitHub, set the variables the runner sets (`RUNNER_TEMP`, `GITHUB_ACTION_PATH`, `GITHUB_OUTPUT`, `GITHUB_PATH`, `GITHUB_STEP_SUMMARY`) and the `INPUT_*` ones `action.yml` passes, then run diff --git a/scripts/__pycache__/report.cpython-314.pyc b/scripts/__pycache__/report.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d7070382e60b698bb780ddf61a12428caca710c5 GIT binary patch literal 25433 zcmcJ13v?UTdFBk>#2X~|eu1P&5fmR1B~cV9>P@{x(+~+-G9?HiKoSxk@Box#T5(vX z*-(jFQLa4IT>omfQEIBv)P_K&Dre@#SnpCHG`Q3V zjV=vBlS_-x?9w5$xUvvhU3!EzmjPk6%ZMe=C^N z<@jm4_b1b&n5(G1NQyJe7K{&hq`&1Dr7HS!mU_)fhUw!1VZX!cI6r=FeaON2LlYz6 zkRv?q2oL)mp|C%Q)bWXMa002`F`pweF*@qyo~)-Y1H=A-3j}i1;T`e@#zJAoXke^i z)bAZ@;QW`ooUc)@?+tl}{B0;YIL?I|gHJkKohN#a^>~h)c%)0Oclo`(kYo6qe{5h_ zZb0Mt(D;~G#W{Z{;PX2IVcH_5rgMlFmM4K@(K5!xV=JyT^JHlMRJCa^rqkfN= z>tWNosI}1ET;8 zk6-YQiA4;Ihr*4Hz5bE$OOAo@Q9qjM4Wqmko!4-5^5F}AeVV3^eZR?f-Q;I`b2^cIZ8m5 zs+eM?UoAnofHGFNL)QQGVTq>vwAR#odl)gdbf9ELX11vxEdor;SZ5k*TmbKhJi{=PfR>{ zZXytFY8(Kf`rmao+F59L|8+0;cbkP4Bt>X6^^dWiB`^luyr&}ImDTQGwK>u6{Oa`d| zPj{C1H#KxAT`Fl(gDfVI?nM+@jwRu{P06?vbF&i7rIT%z(jPX#PIcF$v9`al-}h9@ z7o7dR4U>xgO>y?rq-Mj$jcpq@I&~cBRa_Ygb8!XdM`Yq$T#NY`@d6ukK9CwhMdGGG zZhSNl8Xp|zM!jK<;FeRtkv_(?AONGIeqV^7YmI{=U5RIrT)?T(AeW2xD*Qqo1ejSv z?u`HDiI}0{j-diGd)I8eHu%-S8Bf$d+bfti$IR`E=Jxr03k8Dt$kf5a#OKE>jzxv?WtgHRT%bcRNiW05vHL5nLLUH|47^Oe~kty zE3aFNBGx=Ed5WZ|N?ofNb0{Q8=`UB|X6031PFAj4A;0I7_r8ky6qLz?Y+DlT!p4l@ zcPhK$ddvq0BI?1?HVg*F0-<60;#7-Bpz;LXhK5B$gJ5XlRgd28 zL2w^srqfn3Cv<-Qo>qrtmO@=!Sf<+&bhVxJuoH?2rH7HmD##|OZ@0YMNe$vt!6X;p zgmOaRqW`2NUygBhFdGlbrUn8C`E}9t#&Fu zQBQ;i8(QCB<2o=lQPGT`N4WVt7SoKB^leUdU%;Gbx`TjbQ9cP^AK^PFWar$thCCwE|mk?po{A% z!{`Z&4UWfEV_pz^HBE_&jG*{I6B?i3uq&u3`xjSuJZZBOc|w7Ta^KpBdvb}drMr=xRD?(7fwgm9pG&J zmxJ8Scrjt7Js12>Htqng4pwkyqx^9Rd7BTnk(cM2v#D zI%eL$n>Wm6$4na+O&jM*1k=u#X+Lk;zfc)79$qvazFi|2A4Q%-;gIub zP90b=Sgviqx_!~!8J(DOqpyO!Q!sS$s!q;L;(Y zjQg1URl+SR)5#uqoMD2BewiI|%ex)0m31q(qjr(AJz4;~P<9heaLwbA^JH8}nR0yU z+6$^4QqQNXt|lg^en>90)Za|43u=1UOst$s({GWw>6TmT(mtdPU4}Zq0g#=-jd4sB z&s|v$r0Dx?a$Ty7BdnLom;Sm8ZcRo=qYPo{&yh>bJa1+RR{HC*xYZd;waO5-hd^Y@ z5IGNlu*(p+DgDh{U!DXj{cVy*pd46x^o&TC+$`v0{-mi6G7HgSP>n!bPf3J_5=zTO z$S;EdKj#r42=`0`%uw7Y64gYq*?)P!A0%Q>g3J;E5?2vg3T_!8Eskr)h{iQy5pgvY zsN*SFiCi5wi3KP4thhyd^A(B^(HwZq(N696X z_9$QI)W%KFzj%BB&OZC!Qq_Xo#?Ugfz?4 z*+BRpk602%ef!@Hs7Wdt=${i`nh-hlT6| zvFu}f_OaWQ)7lkl-VObAePpL#t&3SV^45*B=cd&w_M#g{t{>scn*{sDn7xI!BmZ>P ziq3q^bk+0=Yu-&=G*2j~Mec~^W&I2KXuD9*6e~E$7aW{xnXg~od~n5@AIXavmkS$K ztOYmMN7ui;;gt=u!$RdYp{SL7EPT22g;IX~HlYx&<*${`<$bs4?IM21abZ)JQ1Qr$ zwJfTBJ?oV$zOGGhv_J4#9?gBd@RdS-!w#XM1MRq}6~Slg1jkmPpk+1JU^4$sY0A?7 z#=;al%C6Qi#_Xx%5AHIA7ofY$AR>fwb1`WYiao+4X{FDRpks_nu?C7wfr2h45HG z!rLb0NN`~#Os0KVlo%%WG2iN0VIlA%Ua2qhE(L9ABj zAJfG`HI*j0VdV1S7dnDqig~}XRxppeQWMF2>&V>1_d-ADoF9qRAANRWspkyu@huIW zkM&%*HZeW%O3lj_F`WUplwCW&}SVIqi03@ZCcDfDqGx@Bd6lqL#MfE}NI>D-;PGxGV1}WrG zFg0#%`WMs<4x84U71SnCj#xxUsgn3DjaX3MFSB&SUXl^U(4)-6Ry8qTp;YHsm+m1W zoh8+hGSYM@l&6k@%x?noz1a z1^+a^AO7_kdrcEcqEI*hJq*RepSq3xWXaHnu;58tNolo7sy8vW6`0%9Udi9jDUL#3 zA~QUp44Q&Qw9}H(&dfM(kxEaQiNyFpJpR2_RaxG{h)^ehawxAVl%y#P$=A^98RLr!y zv!r$GTT9MmJU`C5$a=_4fu$Ake#|bEpnAEt5GSDo{i}MJPB4tvPie-$#4C>ejX0WaWfgFrZfDxS z+8D*xM5;0f%#h=lfdJzy0~e0nEfm)#OkLc5j1@<=udX!78su8ke;h+Z)_;KTrSJGs z1EsDc79NQ68A&HnA~u^CRf$f-srVHk5?x?p2)puzf@{Gak)ZejS_EJ1k7Gcg={% zy~C#Zge{%7OXzj54G=_@`hZ5ekg<#MV;pVA+NN_s0FYmZprfd`+4{CBIuU8Par(K_ z-&TEmpA7Lcx%n*DhzNHIL0mVMVDsY|(fUYI5-}Aq$x(hVZW@Obkn<#_FrGyWrqF|X zF2waoF(Ph};~prv!y!7}am^6t^+CWfh;kO?Vt+AXEN%4i|J%q{z=4u4e#r zAE+Y^S9RBq>gxGs)ecABT5HO0~KAl9`aEQmxj?$ zTsZ=Zh93!(ljXo=F;tq3Te@9`kMD7vgn78fb$H(iYHkS!LSb0P;~Ei9oY`@;h*$)7 zx$~r46-5CmVK|ZM5^an{`~a3Z?R$T4fTxrm6mB0Z`t>uCFiCx()kU2)O*tc;d*-i`|Z1g zyuo+3%vIkmU*30eDZgQ6BI1eVH}Lrl?{1kk@OguGbMh7m-4gP42st~ZbubAp+Ce1* z`&PlS746X(uNkfyRu#D>GZCa!t4hRvVJ(PMzwCU$89gEtHN}c{@I^b`**q8i_V!gZ zKCfyRTVWc&7QSf9+@@76`KV)T`4EJ)uNz)5%pRH>`tF6dFD%puEr$ii5uxDdY8C;~ zGd9OR)YQ-Ree3iar{DA})pUHMR#j&Imzs_j^bzGse$i`9i}{;AQZu==|IpGg@Bj7~ zjnC}6nIFylM%f+v`uA%(?vX6322!4#v6XyJv!H%2>sHq74Z?1BY)DPZ#Y+r$$JT7EBkKy%}}&`sk~`n|7s~oEMsit?`?QZA9HNF z zhi#T)O6z|%>q%H@Mfl$O{9{?x+pPu1EY>v2M&x6%Vcx%ofXJFDDuCt(x6BmD2|fg= zGSwO;!2$l8P2X?0@BtNaq}ete^R+me9_#B4APd;t(5BVfn9+#D9a?ob1hRI zw=$LU5`Djt3Ipg0E}`RHbcaqG7@3Qx!*`jB05eBf_iVQ!!99Sjquc}ZIHhn$!)yp4K zBhNFooQLo}1U=+dgToP9-IraTBCI5O0EYEbdq78si;$*}L6NNIvX* z^^MD~7^14_@U<&nz4E5i()>c!$aVXNM_h^RB(9VErRWT-v0NEbhlhI!nVuz8 zjUs?Prgev4IsdL|E@$Du4-HG!dcNPopB;|%2NwGSc%8|aaeuuyG7znOX;iS*zpI)$ z#9PkaHCY$QI11K{f@$N_fe$;6KD&95v{^8>3WnDCmSw{pUe(J#+Kcexduqmdfem2* znJ*jnY*GCw30}`bkfyOw;X7SLWwdE-MqDR^rG!FJ;bx7DOWCY6QRQFPZwcrxr~V|) zHpbaO5m^!*DjGPi!IMigB-V%uKy_$fobz+9BGYsDg~U2m?1j%ApWe!=cFb1Kv2!(Z z1@pGqr+8;OQaA_T(y&=>;Gf|OS?0+GCYMmsKP$+j2%PAEB;1Qwl=4k+W=Gn$jVazB zdMXIfuEWZsGBGZ7vkC)}T*4xR5`&|`FcAnXgzm3_Ie= zjm^F|J36Ttb;Q{ajIU}AYs(kMz+Hp|6h=|4WNc6m!w4J#*8=Yzh(v)g;$pW3!*sq0 z_$a~>!TklQ`BVHtBXopTwB~2N_`bz{*PJtR@M~jJ2k&MVMDnBSgzWWyaCEAZuou4Q zULk+|qRKvX@I$R}B|Cp6Kca|ii*AlIM4#X-PG0ByP-|YvE}SWhR7U!uCnCpZZMR zz)gpegt&SnFd7KMv@$U`7`U89Az&b(4xQY81Uy0C5HUW3ZPOZ``2xoFgThCynqX}( zSpRtEN?!5TN2B}r+}hc^SskCV>5g&JZ`4dtH~W!>FnK}6#5=E4djXhq{dH;Q6vvF~wq-K4Uyc@V*t#F}K|8t*8j++>$YtVXSGd}PGS zg}A>$+FSUAps1pqGClJ&I<-<>J!`x9)byj%7iR{hM|eXqui8CVF?V2o^V|-8<8Hi( zy&yuR^pH56@tufZdCKk~KDeZnFlT&jCY5lz)90DR!cowEL3`j-{mO?zxH$xIwwf*< ziNtIpY3fVE<0Jkw3ZsP}F+e!_?*rg3@e5TVm|{NMuyJ}|X8#|Zk8E8wJEA$fsXF>7 zZ&=T(+UNWT9-Iy$vD1z;A|#IKF1C^A^1)WXzJ3)_GyIS;DiZ0I?^30*6q)=($TXAZ zLFV8=)g1LQQNx0+aA{J@^*br+An$~0ze{=#LWNR1;uJ1jYAGpDSZ!evNJ_4m;FftKR4Hdji?#uRnhno^Qb%}QFcDYXunO~P&gb(9*)*wp&cV3Q@e zEX~S#5EAJLs*@^kw9=YdgOm#@euc}H`o%BSsX?9Km&9i+jFcN=oV}(_m<%%7-E&Ze zJ+&U>%|%+?nlvNQ^4Fvl^vk_kn`14sw4b5$!W8waBJ;rlKoz^m4pxkEO4ig`>{ih) zY8q+~mNw@~J*T$&XGPwN+V0CH`>r@Tp}=!oahwqS^RGCLCw=FUEIQ94LY?M9i0qNw z=kd}b`p*Z(98KiV0#f$D`0cndx$a6sLj(D>$w69#e*Q#J??_`B2*PZ}?>@0s%FHB| zxVZbcK&Q~96uv!ak-@H`O0bvMttfj5x=U4r$~e0_&N_&Wkn9NwHkq42oV$)7Z3{_g z3MmIPK+aP<=s*Pu&xEQGP#(F|z1Y$FcF)|SQ8ro;&57=L%|E;6)#2!=<+6=JM{hzg z=cr`4CrO2bx!}HlU{Z(E2k;@ScW_UWS1p|NWqlb(v^Sq9F{)HU^@+k!3wY{`;B!)K zA@?Pc$w&_mBs4OgD6hb2zmcSC&SEO+>Zp#6^0||$x;m7haw46w*C-sZ2~v&1aRt84 zZHRJzNCLu`&>fWSIHDSd&}nh>v?4E!U3knN;=V!BNyj8NIqDIur`xb6?7s|0d8#-f z(Fa`-vgBdr7EuP-wL{;*);GocLg!u4HLj|aCe2jmM>@u8pH{Be3vL{~et5CeJ%3`M zc)8{ZJcE&7~OP0zNYf)73 zTFY$J?AUT$hfuYXuiQ0%nJ+uEY&|riziTg!?2Bq%Ix5&}(O!d@w>XxG-*>}`F*nkD zb0|8%*R?OhU#+ZzFWosm!WSN0HXfZmyrMH*GhQ{Wm~AsvGlO5>FjE>CMBfU&c{=*Y z+~&E^eCv0Bjfzx;3~sD!P@TH@@7Jjmaji0 z*s6Hzy6C>ytmsp7RdWOLwcijPW$dQZv58J8$+?i z-HVO87i#%Ej|z=0-hFDR@zk@;*S1~Vc5TQpY z1u}D2^2|NRCBa)z31#E z2@$7=jJf63$UR=B^P3r3_L5AgYXgDDY-G+Mo3&GNWPfT7*}pQF^^h_j#7QtEm?glc zbJ`e98`9EzPRR&2tB)HqLSs&Y`iIOX4BP~3Q;gGET5FCwiax$MRL2mge+lsx1;u^Hipx)tb#&=mwN$6rPw9PsOy)H&=Li}fGJ8dC16p6)taKZtF-a%} zO59}Cm5Qr5^GK)2PMMvt%A^)ce?K!vxdbWw<^IWSl3TWyjFH{W(6Qj({oRsh3g`iV6SeVwy9t`W~h#2;L{JGyr!=c%62Z}v zgMp5-L{&%*2u~VOy>~1yh&bvad%V~NDaYBkehh|p*ivy2G@di&!Qrxj364WQ5{CRq z3#nmG0)$kOB_1C}o!l)_px%K~%Q(*qgD3}A7KS4Vl1V+u=JJk_9Omki4b`I!)jmh{ zp|a=i#&x&irli}oS)xZVr zE&=>0NmtWZq31-n9V9`E(~@L7i82Vh43v0Y90HE(TX$48u)ZPy;V`q<2{U@DK9s)-J%83#d(W>{*qrmsx? z!Q_lHrY*UnEm_%p;642<{q0?`-H+ed{Wzap8Eu(uTCQs2E87>XJ9zH^l!YR{J@bXA zd-m{hbql|4+q`=25}&(kNw<5&mVcw@dQs$%V5^SVHt;r_Q@LZ?fLtx@-wnJSSTM#~ zj^AlHKI4pZM(dYLn)u?4bL^tEnLp8+T6FL0Cto=^dr_!viB<3AtM|^=^6U03Y+C4D zILPN7`?>BoG?Ur&u*Jep8h+N1^Fod@VN!MpIZbe@He_FGzuG=?F-F9evK4#rjic9( zM)iVyL(G1jx1XQA^sT4fcxry9&~!vhxIOf<3qQKRpCXpe>CTmXN8W3{)qeZQ*uIl@ z_CZq|(Ly+}@cFf~?CknE=W^W+-q|t#=%RfW?>)y4oaZk*u{^->=R$la%zGwK0!-t_ zt{z*-E{jY=hInhuht9^CzDRH6V06=BPW9a0-zr&K>o3~&e1F5c8y1FRZI9e(dt|0R zx_-8HxpFgKv1M-0A{@ti`+lod*iN$ltW?2xsb|c&*B-z6cqA`oax9x1iAo|JZxziQ zoImiLM+E17zP5AWQNHkC%y{TGxr}`yEW6gcYm-+eBlUu1eauqNTk2;wO&waX<=!Z` zUceVM3AT+fTQ6_xojdT|BX1w!cl8KcdZ&)8sItCt><^C3?25KV>t+x0wwBn*f%(1j zlk+3D>lWL3`M`x`+sKk?^e#?i*1TT#O5JR~P}wFFw14m9!o~L{Z%y(pk8tp;(BZ|g zOs)Awt1PP#O@0|)uyIZ`_t?Cj-?D$gg@w$#C!cM{=hH~SG0 zu3%lyELUM?5;<^=;=7B_U*vNgGq zy9QiHwHOEbpzJh#XA%nB6!hT~(n;&{Fg!FRc5U!nYadtY zxt~uhDRl=&+w;;SAhJK_rz!2g=P^0>-47ywDPhfdhIEWL!vJq&4lF6(OHiW6-x>Be zdxn@#()eh+r`H@bHCg0mkn>&79PE=zCHL;6W`a*pY|>H`FYDAuN$P| zF$`4Kkh5!YH#DNyiJ@6@QSO9B=shRDuimtXDn?Uvr*Z=A=mf#}U%nXq_53x*1gx3N zFK>Q1aUY8K`qHxwa&ridZ%|F+F@R8IB6pj_{+u8Z{?7>FS~}=(98^<12IU&W`6GM<2i#}qK~%{@E3xaR%pdo!fFX*d5d2p#Yxfka zdWhKWEV(o8CG+~JgDb|InXczg^Tw*F&R^(`tmxJy^ojR%Om;rD-=-~decyWet*2+U zMYcxsU)sqR)h*gKES)$B9b)Xn**hoB-rB(rayaQEsuQgUU3EnB^@f@9s4D7? z9-7_vj(W~Nf9N~o!o~x9)4_$%Pg?mNR3BLGIWHW&z#kcbvTl^Mt=NiZjFEk>X=l~5 zgMV%k94&nLwmGVaYyW4eh%=x|A7wK-!}PWWQvE0Am$VLvdAWFVAiDk=BX6CU z%Y74y>1}*zE8lvU&pooFI|^&4q`V`@B|U_=frvBro3X&5yptC#*>1?zv6hj?>4hW= z`w$o%t_4swDpCykG68#pbt${pV3tee)`9G(DdE&4H<`Y#NXj*Q)us~v6mqL;&I&>e z;)hg4jZj%fQjbYN#YhCfJ~ufQ6*Tndael!dk)FgAG6rl}L_EX!1D6@Xjip(gW}wpd zy@(Ceie9Kv#2y?TlCC;LwE z8nki0e#jW&t}nO6RhZiQ4C9oYvQf*5q%(TDxzdvXuAxGwBZ6W;?*Zztp;FaU0q`r_XFgeupl73!k#d>&dYla zj&2i#OZ+7m272ik+3M4;l5@n_AubA-Nn;fn)t9&rjRXEM+${uYqoqXxo#D^dcFX;e zG(d6I(Z&4`G0AvK_$}(&on{pgJp{?17UGw*(Uu1mj}5n|DMz~|P zaoMq5I5zw-_a3RFq$lI`Z4TV)B-vFN&O!r_$mnqouC$O+#E0rB_Vu|d|a3$ z{xJ!Dih!yW$%RgF4e@UPuS`gZc(&j<68jSp5ZkzTnC272;wr)j)8jW#NRcObY5*~j zKh-9>&5<>2Boos|CG@+N>_tLugzQa%1*(X(?<=ONsEar5e9Jkv|IH>u=9>|KYtiBgn?_8R{ENpY}EhqRBz9m1DFyjJDvc4zyC&IjLVpYkgjZhNlY(%$Z z%f(S`TSd&ajkj%!`W9=pfsxf&W*+_4fyMmVd7K#7&+mtY7(T3Cz84nW0Y1mKtn)#c zq{^D%B0FcBd6VlM!~B8U`-P2zru8qz4COyJl&|C!-3VL{ES3ju z@8i2Y%g4^fjtwpz8(cazER+X?yudUVaofx}zNB?NXWspviswyyC)FUGyTD%<;fqH3 z+_7cj*jGD$qlGc~H#%nP_GQ!H+xurv@ZJ5($4z8%wr_~=+tY0*iPVb*tC;JTSn^(^d{Ec2H z-6z=hO?Tei+B%>6?NbYt?|5cUW~Ns#6i0l^hDu&l`SC{;jCqI+5kvS7RGSZF>(X3<2_dG8`!5jrw^`kZ_DE(j zK-|+bGYaBs=TdAVb>YcyV;F3(nKUoSd%|!zD$zzMCDxg|!&wr)&$ufL5aKP3aKB}} zTqWw~su1ALFHXSUxYj>*F~GsgwupSHCQ0f`40?dL7MkZU@kIm| z#c*{9M3-IJ?VKvXNzh1E^if2}MQx^QZCBf_?Yz1(f-Bc*r}lrSHP9=;BFgAc%+$DS zYMiTCF=i8;Xe58hSn)et3f8n}YW$6o(HTE9H3Eh=HGUGpX2ASS_MQ^;trEkYhOD&h zgYYVJDwhzZ%3w*w2|P|OLGgnTz~m&Rf?<*uCL}LH=+>dMHBVAe2=f^HrBZHPG_C=~ zE6%;*^mZx-ElFNDOFZ|{WkQ}U6yadNl+E-(U4OBZ+Fv5Yx{0-7 z&6BZiSyB36-7eV-7R>6ekP0K$lex7SBo%!M4DL$K2xI6ci-JDnCgqGU#(uIoL>Rrx z&_g`zjBZ`RUEg%v&@WK{n10X5qXQ^&56sgz%bsj2?qAMS2cT>{#25;Lwp5rehhnVs zy5_ucvK~lLL_W=N8#A`Zn1lqBz1bqo9&Rik>zO`%>-zfoh9S9V#rL2orO&x;Q%W9r z1tB$oB``{6iE3%L&W{psw)$!-RPa8`dFq;T<@Ik!REUw1a-&uGkjx6;`z_-gF-2El zB|&{z$c-Wou8maXAEoz5Uy?ZIFrg_txV_HS*Hk`%p_v@2tiIKt}h{r^`z_vs@-tKWy}lP9yS z4R5HTI16;6__^XYRia=SHyu3ObEtQ(=aJqX+?6~DTQa#atAT7q9g`+K5!@0tcy}aN zWc^r!$Y^&=>Jb;2?T*iLXmOI7Fu4g4daarkE)sD>L7=Ec;Z@*$2UIU@!%SsCvnI)6ovtS<9f+# zx=BFRD&hZ$b)8;=MbG+*`~o@nCEmkDm95VPIElV%MIba zX1E-1kCSgyj}6Z+CPhG@vI&(^uxB%zo|KyuOLX9i+FrUMWY{HiIMYTe1Z7jEu&uyIDi`yFw+i(OL7lNq-Ln&g`{F@4LPuI)l7tH+n zokGDb@v^RkErRvX)Zw)P&;#!V9Gb&260BgZ8c?1}zoN@u(WBinqfm*j%U>z2c(MOF z_twN*&-|u0pXT#+B6*j6#a=SghFF>Xu04N|psJAHBH%)?oNY*LO@7b)ux01-y^+@C z;>J04DYtoHFK<5xkVC9}rKs%1`s;h&>YQ6Q&%W8k=d{l^ubMPjyFOB9sWl(jVUwP5 zM|~|k^>gc%n)hN#cx&@JTj%ZH zZl7!5cR|5F!2A6C$wA&cw4@uxB)xAiat<>8I**6Uu*cIzcZRMmEYgGcKt)<7`zu0C zomHHJpbj=6hN~d~MN^8~gt6kjLV^kc@-HOz??^y+EYXXjoWuzFi}%fWNbDR5D0V+j zVsrrz#9r$R{(X{0rSv$7-6sJN#>J&TR|dgN(hcHqgXO34F?_&eeZZJ$X#0TC|H7IxWm?Hz zH)Xk}(XjS=*;)j470llE%xX6Ko>j#jKr|oGEi8MQy{A*K4fpg_to7b8R>`{AkBlta zaZjgb3-1+a*t~n$N_G!>4~aYO>GD|Hy=DdLXYc7t5#7z^BHUQcR^8jrs@TeV=5p4w vx}RlK#_7o=W#NiS`;|k_9J<;wQ+1>EdTpfn>-BIu`MJsgK3`Qt=k)&oAz{q- literal 0 HcmV?d00001 diff --git a/scripts/report.py b/scripts/report.py index 844b932..7ea192f 100644 --- a/scripts/report.py +++ b/scripts/report.py @@ -1,43 +1,137 @@ -"""Write a job's scores to the step's outputs and summary, and check them against min-mean-reward. +"""Write a job's results to the step's outputs and summary, and check them against min-mean-reward. Usage: report.py RESULT_JSON -Reads hbench's result.json. Each eval (agent, model and dataset) gets a row with its trials, errors, mean -reward and pass@k; the trials that did not reach a reward of 1 are listed below. An eval's mean reward is -its first metric's `reward` (a task's reward.json) or `mean` (a task's reward.txt). mean-reward is the mean -over every trial of every eval. Exits 1 when INPUT_MIN_MEAN_REWARD is set and mean-reward is below it. +Reads hbench's result.json, and beside it the job's config.json and lock.json and each trial's result.json. +A trial is solved when its reward is 1. Each eval (agent, model and tasks) gets a row: solved trials and the +pass rate, mean reward, pass@k, errors, agent time, tokens and cost. Below come what ran (hbench, agent, +model, tasks and digest), the tasks' own metrics, the errors by type, and a row per trial, errors and +failures first. mean-reward is the trial-weighted mean of the evals' mean rewards, as hbench computes them +(a task's reward.json `reward`, or its reward.txt). Exits 1 when INPUT_MIN_MEAN_REWARD is set and +mean-reward is below it. """ +import datetime import json import os +import pathlib +import re +import statistics import sys -LISTED_TRIALS = 50 +LISTED_TRIALS = 500 +PRIMARY_METRICS = ("reward", "mean") def numeric(value): + if isinstance(value, bool): + return None try: return float(value) except (TypeError, ValueError): return None -def eval_rows(stats): - rows = [] - for name, entry in sorted((stats.get("evals") or {}).items()): - metrics = entry.get("metrics") or [{}] - rewards = (entry.get("reward_stats") or {}).get("reward") or {} - below_one = sorted( - trial for value, trials in rewards.items() if (numeric(value) or 0.0) < 1.0 for trial in trials +def timestamp(value): + if not isinstance(value, str): + return None + trimmed = re.sub(r"(\.\d{6})\d+", r"\1", value).replace("Z", "+00:00") + try: + return datetime.datetime.fromisoformat(trimmed) + except ValueError: + return None + + +def seconds_between(span): + if not isinstance(span, dict): + return None + started, finished = timestamp(span.get("started_at")), timestamp(span.get("finished_at")) + return (finished - started).total_seconds() if started and finished else None + + +def read_json(path): + try: + with open(path, encoding="utf-8") as stream: + return json.load(stream) + except (OSError, ValueError): + return {} + + +def eval_key(trial): + agent = (trial.get("agent_info") or {}).get("name") or "agent" + model = ((trial.get("agent_info") or {}).get("model_info") or {}).get("name") + source = trial.get("source") or "tasks" + return "__".join(part for part in (agent, model, source) if part) + + +def load_trials(job_directory): + trials = [] + for result_path in sorted(pathlib.Path(job_directory).glob("*/result.json")): + result = read_json(result_path) + if "trial_name" not in result: + continue + rewards = (result.get("verifier_result") or {}).get("rewards") or {} + usage = result.get("agent_result") or {} + exception = result.get("exception_info") or {} + trials.append( + { + "eval": eval_key(result), + "task": result.get("task_name") or result["trial_name"], + "trial": result["trial_name"], + "reward": numeric(rewards.get("reward")), + "error": exception.get("exception_type"), + "agent_seconds": seconds_between(result.get("agent_execution")), + "input_tokens": usage.get("n_input_tokens"), + "cache_tokens": usage.get("n_cache_tokens"), + "output_tokens": usage.get("n_output_tokens"), + "cost": numeric(usage.get("cost_usd")), + } ) + return trials + + +def solved(trial): + return trial["error"] is None and trial["reward"] is not None and trial["reward"] >= 1.0 + + +def total(values): + present = [value for value in values if isinstance(value, (int, float)) and not isinstance(value, bool)] + return sum(present) if present else None + + +def eval_rows(stats, trials): + evals = stats.get("evals") or {} + rows = [] + for name, entry in sorted(evals.items()): + metrics = (entry.get("metrics") or [{}])[0] + mine = [trial for trial in trials if trial["eval"] == name] + if not mine and len(evals) == 1: + mine = trials + agent_times = [trial["agent_seconds"] for trial in mine if trial["agent_seconds"] is not None] + n_solved = sum(1 for trial in mine if solved(trial)) + cost = total(trial["cost"] for trial in mine) rows.append( { "name": name, - "trials": entry.get("n_trials") or 0, + "trials": entry.get("n_trials") or len(mine), "errors": entry.get("n_errors") or 0, - "mean": numeric(metrics[0].get("reward", metrics[0].get("mean"))), - "pass_at_k": entry.get("pass_at_k") or {}, - "below_one": below_one, + "mean": numeric(metrics.get("reward", metrics.get("mean"))), + "other_metrics": { + key: numeric(value) + for key, value in sorted(metrics.items()) + if key not in PRIMARY_METRICS and numeric(value) is not None + }, + "pass_at_k": {k: numeric(v) for k, v in sorted((entry.get("pass_at_k") or {}).items())}, + "exceptions": {kind: sorted(names) for kind, names in sorted((entry.get("exception_stats") or {}).items())}, + "solved": n_solved, + "graded": len(mine), + "agent_total": sum(agent_times) if agent_times else None, + "agent_median": statistics.median(agent_times) if agent_times else None, + "input_tokens": total(trial["input_tokens"] for trial in mine), + "cache_tokens": total(trial["cache_tokens"] for trial in mine), + "output_tokens": total(trial["output_tokens"] for trial in mine), + "cost": cost, + "cost_per_solved": cost / n_solved if cost is not None and n_solved else None, } ) return rows @@ -49,48 +143,159 @@ def overall_mean(rows): return sum(mean * count for mean, count in weighted) / trials if trials else None -def text(value, digits=3): +def score(value, digits=3): return "–" if value is None else f"{value:.{digits}f}" -def summary(result, rows, mean, threshold): - stats = result.get("stats") or {} +def duration(seconds): + if seconds is None: + return "–" + seconds = round(seconds) + hours, rest = divmod(seconds, 3600) + minutes, seconds = divmod(rest, 60) + if hours: + return f"{hours}h {minutes:02d}m" + if minutes: + return f"{minutes}m {seconds:02d}s" + return f"{seconds}s" + + +def tokens(count): + if count is None: + return "–" + for limit, suffix in ((1_000_000_000, "B"), (1_000_000, "M"), (1_000, "K")): + if count >= limit: + return f"{count / limit:.1f}{suffix}" + return str(count) + + +def dollars(amount): + if amount is None: + return "–" + return f"${amount:.2f}" if amount >= 1 else f"${amount:.4f}" + + +def percent(part, whole): + return f"{100 * part / whole:.1f}%" if whole else "–" + + +def scorecard(rows): lines = [ - "## hbench", - "", - "| Eval | Trials | Errors | Mean reward | pass@k |", - "|---|---:|---:|---:|---|", + "| Eval | Solved | Mean reward | pass@k | Errors | Agent time | Tokens in / out | Cost |", + "|---|---:|---:|---|---:|---|---|---|", ] for row in rows: - pass_at_k = ", ".join(f"pass@{k} {text(numeric(v))}" for k, v in sorted(row["pass_at_k"].items())) or "–" - lines.append(f"| `{row['name']}` | {row['trials']} | {row['errors']} | {text(row['mean'])} | {pass_at_k} |") - lines.append("") - verdict = f"Mean reward **{text(mean)}** over {sum(row['trials'] for row in rows)} trials" - if threshold is not None: - verdict += f", {'at or above' if mean is not None and mean >= threshold else 'below'} min-mean-reward {threshold:g}" - lines.append(verdict + ".") - tokens = [ - f"{label} {stats[key]:,}" - for key, label in (("n_input_tokens", "input"), ("n_cache_tokens", "cached"), ("n_output_tokens", "output")) - if isinstance(stats.get(key), int) + pass_at_k = ", ".join(f"@{k} {score(v)}" for k, v in row["pass_at_k"].items()) or "–" + agent_time = "–" + if row["agent_total"] is not None: + agent_time = f"{duration(row['agent_total'])} total, {duration(row['agent_median'])} median" + token_text = "–" + if row["input_tokens"] is not None or row["output_tokens"] is not None: + token_text = f"{tokens(row['input_tokens'])} / {tokens(row['output_tokens'])}" + if row["cache_tokens"]: + token_text += f" ({tokens(row['cache_tokens'])} cached)" + cost_text = dollars(row["cost"]) + if row["cost_per_solved"] is not None: + cost_text += f" ({dollars(row['cost_per_solved'])} per solved)" + lines.append( + f"| `{row['name']}` | **{row['solved']} / {row['trials']}** ({percent(row['solved'], row['trials'])}) " + f"| {score(row['mean'])} | {pass_at_k} | {row['errors']} | {agent_time} | {token_text} | {cost_text} |" + ) + return lines + + +def shown_path(path): + absolute, here = os.path.abspath(path), os.getcwd() + return os.path.relpath(absolute, here) if os.path.commonpath([absolute, here]) == here else path + + +def run_details(result, config, lock): + hbench = lock.get("hbench") or {} + lines = [] + if hbench.get("version"): + epoch = f", results epoch {hbench['results_epoch']}" if hbench.get("results_epoch") else "" + lines.append(f"- **hbench** {hbench['version']}{epoch}") + for agent in config.get("agents") or []: + model = f", model `{agent['model_name']}`" if agent.get("model_name") else "" + lines.append(f"- **Agent** `{agent.get('name') or 'oracle'}`{model}") + for dataset in config.get("datasets") or []: + if dataset.get("name"): + where = dataset.get("registry") or dataset.get("registry_url") or dataset.get("registry_path") + label = f"`{dataset['name']}@{dataset['version']}`" if dataset.get("version") else f"`{dataset['name']}`" + lines.append( + f"- **Tasks** {label}" + + (f" from `{where}`" if where else "") + + (f", manifest `{dataset['digest']}`" if dataset.get("digest") else "") + ) + elif dataset.get("path"): + lines.append(f"- **Tasks** `{shown_path(dataset['path'])}`") + attempts = config.get("n_attempts") or 1 + concurrent = lock.get("n_concurrent_trials") or config.get("n_concurrent_trials") + lines.append(f"- **Attempts** {attempts} per task" + (f", {concurrent} at a time" if concurrent else "")) + started, finished = timestamp(result.get("started_at")), timestamp(result.get("finished_at")) + if started and finished: + lines.append( + f"- **Ran** {started:%Y-%m-%d %H:%M} to {finished:%H:%M} UTC, " + f"{duration((finished - started).total_seconds())}" + ) + return lines + + +def trial_table(trials): + def order(trial): + return (0 if trial["error"] else 1 if not solved(trial) else 2, trial["task"], trial["trial"]) + + ordered = sorted(trials, key=order) + shown = ordered[:LISTED_TRIALS] + lines = [ + "| Task | Result | Reward | Agent time | Tokens in / out | Cost |", + "|---|---|---:|---|---|---|", ] - if tokens: - lines.append(f"Tokens: {', '.join(tokens)}.") - if isinstance(stats.get("cost_usd"), (int, float)): - lines.append(f"Cost: ${stats['cost_usd']:.4f}.") + for trial in shown: + result = f"⚠️ {trial['error']}" if trial["error"] else "✅ solved" if solved(trial) else "❌ failed" + token_text = "–" + if trial["input_tokens"] is not None or trial["output_tokens"] is not None: + token_text = f"{tokens(trial['input_tokens'])} / {tokens(trial['output_tokens'])}" + lines.append( + f"| `{trial['task']}` | {result} | {score(trial['reward'])} | {duration(trial['agent_seconds'])} " + f"| {token_text} | {dollars(trial['cost'])} |" + ) + if len(ordered) > len(shown): + lines.append(f"| and {len(ordered) - len(shown)} more | | | | | |") + return lines + + +def summary(result, config, lock, rows, trials, mean, threshold): + n_trials = sum(row["trials"] for row in rows) + n_solved = sum(row["solved"] for row in rows) + lines = ["## Harness Benchmark", "", *scorecard(rows), ""] + verdict = ( + f"**{n_solved} of {n_trials}** trials solved ({percent(n_solved, n_trials)}), mean reward **{score(mean)}**." + ) + if threshold is not None: + passed = mean is not None and mean >= threshold + verdict += f" {'✅ At or above' if passed else '❌ Below'} min-mean-reward {threshold:g}." + lines += [verdict, "", "### What ran", "", *run_details(result, config, lock)] for row in rows: - if row["below_one"]: - shown = row["below_one"][:LISTED_TRIALS] - more = len(row["below_one"]) - len(shown) - lines += [ - "", - f"
{len(row['below_one'])} trials of {row['name']} below a reward of 1", - "", - *[f"- `{trial}`" for trial in shown], - *([f"- and {more} more"] if more > 0 else []), - "", - "
", - ] + if row["other_metrics"]: + metrics = ", ".join(f"`{key}` {score(value)}" for key, value in row["other_metrics"].items()) + lines += ["", f"The tasks' own metrics for `{row['name']}`, as means over its trials: {metrics}."] + errors = [(row["name"], kind, names) for row in rows for kind, names in row["exceptions"].items()] + if errors: + lines += ["", "### Errors", ""] + for name, kind, names in errors: + shown = ", ".join(f"`{trial}`" for trial in names[:10]) + more = f" and {len(names) - 10} more" if len(names) > 10 else "" + lines.append(f"- **{kind}** in `{name}`: {len(names)} ({shown}{more})") + if trials: + lines += [ + "", + f"
{len(trials)} trials, errors and failures first", + "", + *trial_table(trials), + "", + "
", + ] return "\n".join(lines) + "\n" @@ -105,23 +310,32 @@ def main(arguments): if len(arguments) != 1: print(__doc__, file=sys.stderr) return 2 - with open(arguments[0], encoding="utf-8") as stream: - result = json.load(stream) - rows = eval_rows(result.get("stats") or {}) + result_path = pathlib.Path(arguments[0]) + result = read_json(result_path) + job_directory = result_path.parent + config, lock = read_json(job_directory / "config.json"), read_json(job_directory / "lock.json") + trials = load_trials(job_directory) + rows = eval_rows(result.get("stats") or {}, trials) mean = overall_mean(rows) threshold = numeric(os.environ.get("INPUT_MIN_MEAN_REWARD") or None) + n_trials = sum(row["trials"] for row in rows) + n_solved = sum(row["solved"] for row in rows) + cost = total(row["cost"] for row in rows) append( "GITHUB_OUTPUT", - f"result-path={arguments[0]}\n" + f"result-path={result_path}\n" f"mean-reward={'' if mean is None else round(mean, 6)}\n" - f"n-trials={sum(row['trials'] for row in rows)}\n" - f"n-errors={sum(row['errors'] for row in rows)}\n", + f"n-trials={n_trials}\n" + f"n-errors={sum(row['errors'] for row in rows)}\n" + f"n-solved={n_solved}\n" + f"pass-rate={round(n_solved / n_trials, 6) if n_trials else ''}\n" + f"cost-usd={'' if cost is None else round(cost, 6)}\n", ) - report = summary(result, rows, mean, threshold) + report = summary(result, config, lock, rows, trials, mean, threshold) append("GITHUB_STEP_SUMMARY", report) print(report) if threshold is not None and (mean is None or mean < threshold): - print(f"::error title=hbench::Mean reward {text(mean)} is below min-mean-reward {threshold:g}") + print(f"::error title=hbench::Mean reward {score(mean)} is below min-mean-reward {threshold:g}") return 1 return 0 diff --git a/tests/__pycache__/test_report.cpython-314.pyc b/tests/__pycache__/test_report.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6b91abf6b94b07c9436e5ec98fecca1a4a56336d GIT binary patch literal 8433 zcmdrxZERCncGus}jzc~m$piv2%i4jKVDp@LFjk5qCK`k;qfyALi6Yx zVhp25R^JBMKNZuydMBZlEOx_mRLf%%{3UA}6b;KsyTG6y=yjBxme7B)#doT2X zpYaCuT?%^(mGQ<>tXh_x)8_H58}Z}YH{#Br=^E(ESxF9K>aHqsXrt*hs<>8l*0pl& zw39KHQZkmZnzghh*KMG-m9{hG)b-3JhH64_gS@!`wIRmdgc!%Pb-c4UF0IsSQcH3J zp{xQ*=SwIXr<<7Sy>BBF-GWdH0&2e|t@v(3+FG~5ZoA0OUGsZPFy=NW=7bQrq}Zg0 z7)!~LVioz6Aj%m#cY#mJo{Pdvrb^_cbX@jcl!Ro)A&6Wg&f|tiVIfIZc%ktG9$Nv< zqbY*KV7!#7fGrMbzkxNRjcF6B4>Dz(GHcA^R;^Tgx1mWRgS1V21zLkfD;{EuQ+7?# zq)Af<##Dke>sa%o!DmWYrid|Nu5qWvTYQg$fvtb0baNQ5U*Cq%q(vLg#<{nmDcDH$ zsie?tXzC5Uy@G)h*HxQE#Xeee89TI6@eO?x*9d8Hyauh5vN9I@r>QF_)w=qWz9x*d z2~C>D+ltb)lBSa5E)7$BYi+fnf<2iC}wyGRpM2s7>f(3bZ(ns71DAlErWTb6A4bds#G5x4UJ5Ek3BgN z8lMO$n+w%bq2Y1%)WnGsL(CbaVRQ`I*b}2;?1|x_F_sxVJ;WRyixj*mnS}*l5Ppvj z1K0_Zm`5RW1bpe`y75vvLW~fD&go`w1~h6>j0+JiE**F6W!vD^2cf|HOYu*iXE-gEqM96-wq4sq?WouhfU21Wro4dN?NX5m|s$ zx1~6FHXfT%%;Qi{Op+{$*jXtSXo$=z=4dP;D@H+5Y<%+Fm?$I_M?^@<{1q8<)rfqQ z$YLcq3OlM8V}e4RlEql^!stm6?*+I~E|yg6BF{xx%%?cWI+FEsQ%b>yvUng$O(K?L z6Fi5PV2%@`id_{CLltp=7AY*TsxXuU_+iB=$()>q@u>tyhv^-~H5I!6 zu^E2Se*>6DU)k&O_VzFA?Rk4g&fc-?{%UL6C*2=)FI~OUncceoSEoOl`kSfG-76F4 zvIp2j+t!k{OLOnoNb>5m*)O#@PX5{#6Gl{n;Z*OPaJ4d{Rd~?Kl=guKKn8A z$=MIjE^o_M_pMd;{Z%GE@cl0azMo@4d1m?xW;)BT`GN4-K=`YMZ8zWfqjx@aEnUj) z>{)3GW*d4JjyyG?y_^Af|GB<#wQ=&!`F#J8)&3*d{*iqDsnz~dx&Dc4<7Bpea>4eX zreSd|PX|`%K$Z@!)C8Y4Ajj@6>>W=x5g7yFXCMdupFORC2AJrIz2lj50OtQELkEX_ zmcMW58E&!vdQ;o5z53Ta3YQ0lY|wDeUNziYcW(o{2lw385 zWG7>HA-6+u4&iY7mM{|n%PF&TtmL4p6zLNJt2PESr3Y5cJo||L{Aj^(9z+s5c}52?}_+WBDyia`gI#l2NEAYFX28rR4ubeygZRo>PyG# zCAghHBsBv7ed`(g)tPM1(N)T~VE%E{V(5>jmU{nu@G&%^%9HyBx|oN-S8`r})u)6d zsLwh54AA;~K7Gm>7q~)0cS+=Aj9N7QmQqY*Qp~R@#Zo54`kGQ~Wl~;uEL52k`)iKH zQ6{CTD1~u?(p8IiCo@#i1I`nizG4XCiX_C}<)exr@XVg#B*`OkGM}OR-XM7}1uu$% zC_SsfUelv`&1WXRx9=Gh5hU4@mZBM}-y7%&_KvwtN)-s~>ciz@IOKErGB8DEpn^fEri4616$)OB?QXdfNy>Y_V?L2q!EJJS;A zNzj2liIxSKi+9t31ReBCbOO!?Tr#tx*YEfGj{#4h*V}iD?(FF!b#hRO0(ZNrqI9QL zNbw@AYMguQwo#!}y&X*w_baZ~*CC>JQbd4r4ICTTSx$ljg(R`qURXJn03clAlaj8t zEOel&G$&nB#y0Bl|3;n{>-U(Z-yc-JU%k@kUhe(H;O)U2wHGu@MfKi^{&oB><2mY( zj_SW_{>=GHXO0@yQ3a~|)S(w>;EoprivK?v_#8_yzf^PK^5M2-c6sQoF7-qX$_swf z>+h!3Tf9WW{XQ}a$ezp~CL;;~RN+XROGkN6M2Pa?ZaM+4I34Da@~kMNVi8}VSxBWN zPoJkRobh7r5Ga*|x%n$xA{FP~40!!v`n)J4=&;CNh`~wwDoGrKGaaNi86VVT%o3on z_RAokn-0>POmj3=;fx>CF~J9%q$fg$!1`448xy|^ z8o0q){KQ)3W;q$`BgrPY1TV4bqOcr3ZLwsI#8&VqB;~+HCj_X<;TA4hXpPX5poXJrU9ed<+^KYHM0dttpQ3wCl{148*1Jnr}HS&4;Z@Yif zeV;o00t>?E6SqSAE=`<k9KMph4b#`?Z$4W3x4R$EWP3`rhQ5}@3tz%W&TUvJq7q^J z*^lY-9ApBbAO*s&-w!OFa+dS4D;_mw6ONDnpb&(C1N;}}fK#M;3%A6QQT~eJP$Tnk zUXt8q(GTJj3(+RUhC54(JY3PqDVAg<@WoucF9lRpB&S`XbZkKh%jlJLi}ii5>bFCFKh>||MdLBfF%YBel-J|;?# z@JjMYfn~)2UW!2shA=pc!BGrOV!!}U8d%7PadIreax!??nKUdEh^ixwT#|5BEdnl^ zWg(MLU4jHQ?7iM3CL(rYc^I_8Pr}{)=XF?yM@EBX#PG;!vmAI-Z?+sZJaSqrZyO$M zGFpa=k0^`9{S=A^9wGolBZ-U%uH&n5AEV!02vdwba4Pz zv3V&dQ?K6Z0&oE74B#6ZTpHkf z(F@jFP2l&UIjSh?dAp)Sf6-6fN(d9 z+u;i@EV!6ho4jKO6GEb^vF^1taz0Umu=KRjQ{Xuoz;>b_fTi4dA_beyYs`t-3o>;34y-!+p8xAb)&2MgB-Q4~J yLFGxvP-XBhj^wxOTHUhi34+SQ7G!oVWOAnF2WIE>6Yrh)!;}ARLguYxn*RfcfWoo> literal 0 HcmV?d00001 diff --git a/tests/report/agent-job/config.json b/tests/report/agent-job/config.json new file mode 100644 index 0000000..6c6acf1 --- /dev/null +++ b/tests/report/agent-job/config.json @@ -0,0 +1,17 @@ +{ + "job_name": "agent-job", + "n_attempts": null, + "agents": [ + { + "name": "claude-code", + "model_name": "anthropic/claude-opus-5-5" + } + ], + "datasets": [ + { + "name": "example", + "version": "1.0", + "registry_path": "registry.json" + } + ] +} diff --git a/tests/report/agent-job/count-words__b2/result.json b/tests/report/agent-job/count-words__b2/result.json new file mode 100644 index 0000000..319dedb --- /dev/null +++ b/tests/report/agent-job/count-words__b2/result.json @@ -0,0 +1,30 @@ +{ + "trial_name": "count-words__b2", + "task_name": "example/count-words", + "source": "example", + "agent_info": { + "name": "claude-code", + "version": "2.1.0", + "model_info": { + "name": "claude-opus-5-5", + "provider": "anthropic" + } + }, + "agent_result": { + "n_input_tokens": 80000, + "n_cache_tokens": 0, + "n_output_tokens": 2000, + "cost_usd": 0.0434 + }, + "verifier_result": { + "rewards": { + "reward": 0.0, + "field_score": 0.5 + } + }, + "exception_info": null, + "agent_execution": { + "started_at": "2026-10-22T05:22:00.123456789Z", + "finished_at": "2026-10-22T05:22:45.123456789Z" + } +} diff --git a/tests/report/agent-job/fix-median__a1/result.json b/tests/report/agent-job/fix-median__a1/result.json new file mode 100644 index 0000000..5f2a856 --- /dev/null +++ b/tests/report/agent-job/fix-median__a1/result.json @@ -0,0 +1,30 @@ +{ + "trial_name": "fix-median__a1", + "task_name": "example/fix-median", + "source": "example", + "agent_info": { + "name": "claude-code", + "version": "2.1.0", + "model_info": { + "name": "claude-opus-5-5", + "provider": "anthropic" + } + }, + "agent_result": { + "n_input_tokens": 120000, + "n_cache_tokens": 100000, + "n_output_tokens": 3500, + "cost_usd": 0.06 + }, + "verifier_result": { + "rewards": { + "reward": 1.0, + "field_score": 1.0 + } + }, + "exception_info": null, + "agent_execution": { + "started_at": "2026-10-22T05:22:00.123456789Z", + "finished_at": "2026-10-22T05:23:30.123456789Z" + } +} diff --git a/tests/report/agent-job/hello-world__c3/result.json b/tests/report/agent-job/hello-world__c3/result.json new file mode 100644 index 0000000..cacc71c --- /dev/null +++ b/tests/report/agent-job/hello-world__c3/result.json @@ -0,0 +1,30 @@ +{ + "trial_name": "hello-world__c3", + "task_name": "example/hello-world", + "source": "example", + "agent_info": { + "name": "claude-code", + "version": "2.1.0", + "model_info": { + "name": "claude-opus-5-5", + "provider": "anthropic" + } + }, + "agent_result": { + "n_input_tokens": 200000, + "n_cache_tokens": 150000, + "n_output_tokens": 9000, + "cost_usd": 0.02 + }, + "verifier_result": null, + "exception_info": { + "exception_type": "AgentTimeoutError", + "exception_message": "agent timed out", + "exception_traceback": "", + "occurred_at": "2026-10-22T05:32:00Z" + }, + "agent_execution": { + "started_at": "2026-10-22T05:22:00.123456789Z", + "finished_at": "2026-10-22T05:32:00.123456789Z" + } +} diff --git a/tests/report/agent-job/lock.json b/tests/report/agent-job/lock.json new file mode 100644 index 0000000..3bb1bd6 --- /dev/null +++ b/tests/report/agent-job/lock.json @@ -0,0 +1,8 @@ +{ + "schema_version": 3, + "hbench": { + "version": "0.3.0", + "results_epoch": "0.2" + }, + "n_concurrent_trials": 3 +} diff --git a/tests/report/agent-job/result.json b/tests/report/agent-job/result.json new file mode 100644 index 0000000..f20f0c9 --- /dev/null +++ b/tests/report/agent-job/result.json @@ -0,0 +1,38 @@ +{ + "started_at": "2026-10-22T05:21:31.249358767Z", + "finished_at": "2026-10-22T05:33:01.100000000Z", + "n_total_trials": 3, + "stats": { + "n_completed_trials": 3, + "n_errored_trials": 1, + "cost_usd": 0.1234, + "evals": { + "claude-code__claude-opus-5-5__example": { + "n_trials": 3, + "n_errors": 1, + "metrics": [ + { + "reward": 0.5, + "field_score": 0.75 + } + ], + "pass_at_k": {}, + "reward_stats": { + "reward": { + "1.0": [ + "fix-median__a1" + ], + "0.0": [ + "count-words__b2" + ] + } + }, + "exception_stats": { + "AgentTimeoutError": [ + "hello-world__c3" + ] + } + } + } + } +} diff --git a/tests/report/oracle-job/config.json b/tests/report/oracle-job/config.json new file mode 100644 index 0000000..9028060 --- /dev/null +++ b/tests/report/oracle-job/config.json @@ -0,0 +1,13 @@ +{ + "agents": [ + { + "name": "oracle", + "model_name": null + } + ], + "datasets": [ + { + "path": "tasks" + } + ] +} diff --git a/tests/report/oracle-job/hello-world__d4/result.json b/tests/report/oracle-job/hello-world__d4/result.json new file mode 100644 index 0000000..8501b7e --- /dev/null +++ b/tests/report/oracle-job/hello-world__d4/result.json @@ -0,0 +1,26 @@ +{ + "trial_name": "hello-world__d4", + "task_name": "hbench/hello-world", + "source": "tasks", + "agent_info": { + "name": "oracle", + "version": null, + "model_info": null + }, + "agent_result": { + "n_input_tokens": null, + "n_cache_tokens": null, + "n_output_tokens": null, + "cost_usd": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "agent_execution": { + "started_at": "2026-10-22T06:00:02Z", + "finished_at": "2026-10-22T06:00:03Z" + } +} diff --git a/tests/report/oracle-job/lock.json b/tests/report/oracle-job/lock.json new file mode 100644 index 0000000..4b37c71 --- /dev/null +++ b/tests/report/oracle-job/lock.json @@ -0,0 +1,6 @@ +{ + "hbench": { + "version": "0.3.0", + "results_epoch": "0.2" + } +} diff --git a/tests/report/oracle-job/result.json b/tests/report/oracle-job/result.json new file mode 100644 index 0000000..38cbc7f --- /dev/null +++ b/tests/report/oracle-job/result.json @@ -0,0 +1,27 @@ +{ + "started_at": "2026-10-22T06:00:00Z", + "finished_at": "2026-10-22T06:00:12Z", + "n_total_trials": 1, + "stats": { + "evals": { + "oracle__tasks": { + "n_trials": 1, + "n_errors": 0, + "metrics": [ + { + "mean": 1.0 + } + ], + "pass_at_k": {}, + "reward_stats": { + "reward": { + "1.0": [ + "hello-world__d4" + ] + } + }, + "exception_stats": {} + } + } + } +} diff --git a/tests/test_report.py b/tests/test_report.py new file mode 100644 index 0000000..af47222 --- /dev/null +++ b/tests/test_report.py @@ -0,0 +1,90 @@ +import contextlib +import io +import os +import pathlib +import sys +import tempfile +import unittest +from unittest import mock + +HERE = pathlib.Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parent / "scripts")) + +import report # noqa: E402 + +AGENT_JOB = HERE / "report" / "agent-job" / "result.json" +ORACLE_JOB = HERE / "report" / "oracle-job" / "result.json" + + +def run_report(result_path, min_mean_reward=""): + with tempfile.TemporaryDirectory() as directory: + output, summary = pathlib.Path(directory, "output"), pathlib.Path(directory, "summary") + environment = { + "GITHUB_OUTPUT": str(output), + "GITHUB_STEP_SUMMARY": str(summary), + "INPUT_MIN_MEAN_REWARD": min_mean_reward, + } + with mock.patch.dict(os.environ, environment), contextlib.redirect_stdout(io.StringIO()): + status = report.main([str(result_path)]) + outputs = dict(line.split("=", 1) for line in output.read_text().splitlines()) + return status, outputs, summary.read_text() + + +class AgentJobTest(unittest.TestCase): + def setUp(self): + self.status, self.outputs, self.summary = run_report(AGENT_JOB) + + def test_outputs_count_solved_trials_and_cost(self): + self.assertEqual(self.status, 0) + self.assertEqual(self.outputs["n-trials"], "3") + self.assertEqual(self.outputs["n-solved"], "1") + self.assertEqual(self.outputs["pass-rate"], "0.333333") + self.assertEqual(self.outputs["n-errors"], "1") + self.assertEqual(self.outputs["mean-reward"], "0.5") + self.assertEqual(self.outputs["cost-usd"], "0.1234") + + def test_scorecard_has_pass_rate_time_tokens_and_cost_per_solved_task(self): + self.assertIn("**1 / 3** (33.3%)", self.summary) + self.assertIn("12m 15s total, 1m 30s median", self.summary) + self.assertIn("400.0K / 14.5K (250.0K cached)", self.summary) + self.assertIn("$0.1234 ($0.1234 per solved)", self.summary) + + def test_what_ran_names_hbench_agent_model_and_tasks(self): + self.assertIn("- **hbench** 0.3.0, results epoch 0.2", self.summary) + self.assertIn("- **Agent** `claude-code`, model `anthropic/claude-opus-5-5`", self.summary) + self.assertIn("- **Tasks** `example@1.0` from `registry.json`", self.summary) + self.assertIn("- **Attempts** 1 per task, 3 at a time", self.summary) + self.assertIn("- **Ran** 2026-10-22 05:21 to 05:33 UTC, 11m 30s", self.summary) + + def test_tasks_metrics_and_errors_are_listed(self): + self.assertIn("`field_score` 0.750", self.summary) + self.assertIn("- **AgentTimeoutError** in `claude-code__claude-opus-5-5__example`: 1 (`hello-world__c3`)", self.summary) + + def test_trials_list_errors_then_failures_then_solved(self): + error = self.summary.index("| `example/hello-world` | ⚠️ AgentTimeoutError |") + failure = self.summary.index("| `example/count-words` | ❌ failed | 0.000 |") + solved = self.summary.index("| `example/fix-median` | ✅ solved | 1.000 |") + self.assertLess(error, failure) + self.assertLess(failure, solved) + + def test_mean_reward_below_the_threshold_fails(self): + self.assertEqual(run_report(AGENT_JOB, "0.6")[0], 1) + self.assertEqual(run_report(AGENT_JOB, "0.5")[0], 0) + + +class OracleJobTest(unittest.TestCase): + def test_reward_txt_mean_and_no_usage(self): + status, outputs, summary = run_report(ORACLE_JOB, "1") + self.assertEqual(status, 0) + self.assertEqual(outputs["mean-reward"], "1.0") + self.assertEqual(outputs["pass-rate"], "1.0") + self.assertEqual(outputs["cost-usd"], "") + self.assertIn("**1 / 1** (100.0%)", summary) + self.assertIn("- **Agent** `oracle`", summary) + self.assertIn("- **Tasks** `tasks`", summary) + self.assertIn("✅ At or above min-mean-reward 1.", summary) + self.assertNotIn("### Errors", summary) + + +if __name__ == "__main__": + unittest.main() From 70fef33e323a03c1d7aff01a4d03d2c9fc335a71 Mon Sep 17 00:00:00 2001 From: rzonyx <77541804+rzonyx@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:17:48 +0700 Subject: [PATCH 2/7] Ignore Python's bytecode caches The previous commit added the __pycache__ the unit tests wrote. --- .gitignore | 1 + scripts/__pycache__/report.cpython-314.pyc | Bin 25433 -> 0 bytes tests/__pycache__/test_report.cpython-314.pyc | Bin 8433 -> 0 bytes 3 files changed, 1 insertion(+) create mode 100644 .gitignore delete mode 100644 scripts/__pycache__/report.cpython-314.pyc delete mode 100644 tests/__pycache__/test_report.cpython-314.pyc diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c18dd8d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/scripts/__pycache__/report.cpython-314.pyc b/scripts/__pycache__/report.cpython-314.pyc deleted file mode 100644 index d7070382e60b698bb780ddf61a12428caca710c5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 25433 zcmcJ13v?UTdFBk>#2X~|eu1P&5fmR1B~cV9>P@{x(+~+-G9?HiKoSxk@Box#T5(vX z*-(jFQLa4IT>omfQEIBv)P_K&Dre@#SnpCHG`Q3V zjV=vBlS_-x?9w5$xUvvhU3!EzmjPk6%ZMe=C^N z<@jm4_b1b&n5(G1NQyJe7K{&hq`&1Dr7HS!mU_)fhUw!1VZX!cI6r=FeaON2LlYz6 zkRv?q2oL)mp|C%Q)bWXMa002`F`pweF*@qyo~)-Y1H=A-3j}i1;T`e@#zJAoXke^i z)bAZ@;QW`ooUc)@?+tl}{B0;YIL?I|gHJkKohN#a^>~h)c%)0Oclo`(kYo6qe{5h_ zZb0Mt(D;~G#W{Z{;PX2IVcH_5rgMlFmM4K@(K5!xV=JyT^JHlMRJCa^rqkfN= z>tWNosI}1ET;8 zk6-YQiA4;Ihr*4Hz5bE$OOAo@Q9qjM4Wqmko!4-5^5F}AeVV3^eZR?f-Q;I`b2^cIZ8m5 zs+eM?UoAnofHGFNL)QQGVTq>vwAR#odl)gdbf9ELX11vxEdor;SZ5k*TmbKhJi{=PfR>{ zZXytFY8(Kf`rmao+F59L|8+0;cbkP4Bt>X6^^dWiB`^luyr&}ImDTQGwK>u6{Oa`d| zPj{C1H#KxAT`Fl(gDfVI?nM+@jwRu{P06?vbF&i7rIT%z(jPX#PIcF$v9`al-}h9@ z7o7dR4U>xgO>y?rq-Mj$jcpq@I&~cBRa_Ygb8!XdM`Yq$T#NY`@d6ukK9CwhMdGGG zZhSNl8Xp|zM!jK<;FeRtkv_(?AONGIeqV^7YmI{=U5RIrT)?T(AeW2xD*Qqo1ejSv z?u`HDiI}0{j-diGd)I8eHu%-S8Bf$d+bfti$IR`E=Jxr03k8Dt$kf5a#OKE>jzxv?WtgHRT%bcRNiW05vHL5nLLUH|47^Oe~kty zE3aFNBGx=Ed5WZ|N?ofNb0{Q8=`UB|X6031PFAj4A;0I7_r8ky6qLz?Y+DlT!p4l@ zcPhK$ddvq0BI?1?HVg*F0-<60;#7-Bpz;LXhK5B$gJ5XlRgd28 zL2w^srqfn3Cv<-Qo>qrtmO@=!Sf<+&bhVxJuoH?2rH7HmD##|OZ@0YMNe$vt!6X;p zgmOaRqW`2NUygBhFdGlbrUn8C`E}9t#&Fu zQBQ;i8(QCB<2o=lQPGT`N4WVt7SoKB^leUdU%;Gbx`TjbQ9cP^AK^PFWar$thCCwE|mk?po{A% z!{`Z&4UWfEV_pz^HBE_&jG*{I6B?i3uq&u3`xjSuJZZBOc|w7Ta^KpBdvb}drMr=xRD?(7fwgm9pG&J zmxJ8Scrjt7Js12>Htqng4pwkyqx^9Rd7BTnk(cM2v#D zI%eL$n>Wm6$4na+O&jM*1k=u#X+Lk;zfc)79$qvazFi|2A4Q%-;gIub zP90b=Sgviqx_!~!8J(DOqpyO!Q!sS$s!q;L;(Y zjQg1URl+SR)5#uqoMD2BewiI|%ex)0m31q(qjr(AJz4;~P<9heaLwbA^JH8}nR0yU z+6$^4QqQNXt|lg^en>90)Za|43u=1UOst$s({GWw>6TmT(mtdPU4}Zq0g#=-jd4sB z&s|v$r0Dx?a$Ty7BdnLom;Sm8ZcRo=qYPo{&yh>bJa1+RR{HC*xYZd;waO5-hd^Y@ z5IGNlu*(p+DgDh{U!DXj{cVy*pd46x^o&TC+$`v0{-mi6G7HgSP>n!bPf3J_5=zTO z$S;EdKj#r42=`0`%uw7Y64gYq*?)P!A0%Q>g3J;E5?2vg3T_!8Eskr)h{iQy5pgvY zsN*SFiCi5wi3KP4thhyd^A(B^(HwZq(N696X z_9$QI)W%KFzj%BB&OZC!Qq_Xo#?Ugfz?4 z*+BRpk602%ef!@Hs7Wdt=${i`nh-hlT6| zvFu}f_OaWQ)7lkl-VObAePpL#t&3SV^45*B=cd&w_M#g{t{>scn*{sDn7xI!BmZ>P ziq3q^bk+0=Yu-&=G*2j~Mec~^W&I2KXuD9*6e~E$7aW{xnXg~od~n5@AIXavmkS$K ztOYmMN7ui;;gt=u!$RdYp{SL7EPT22g;IX~HlYx&<*${`<$bs4?IM21abZ)JQ1Qr$ zwJfTBJ?oV$zOGGhv_J4#9?gBd@RdS-!w#XM1MRq}6~Slg1jkmPpk+1JU^4$sY0A?7 z#=;al%C6Qi#_Xx%5AHIA7ofY$AR>fwb1`WYiao+4X{FDRpks_nu?C7wfr2h45HG z!rLb0NN`~#Os0KVlo%%WG2iN0VIlA%Ua2qhE(L9ABj zAJfG`HI*j0VdV1S7dnDqig~}XRxppeQWMF2>&V>1_d-ADoF9qRAANRWspkyu@huIW zkM&%*HZeW%O3lj_F`WUplwCW&}SVIqi03@ZCcDfDqGx@Bd6lqL#MfE}NI>D-;PGxGV1}WrG zFg0#%`WMs<4x84U71SnCj#xxUsgn3DjaX3MFSB&SUXl^U(4)-6Ry8qTp;YHsm+m1W zoh8+hGSYM@l&6k@%x?noz1a z1^+a^AO7_kdrcEcqEI*hJq*RepSq3xWXaHnu;58tNolo7sy8vW6`0%9Udi9jDUL#3 zA~QUp44Q&Qw9}H(&dfM(kxEaQiNyFpJpR2_RaxG{h)^ehawxAVl%y#P$=A^98RLr!y zv!r$GTT9MmJU`C5$a=_4fu$Ake#|bEpnAEt5GSDo{i}MJPB4tvPie-$#4C>ejX0WaWfgFrZfDxS z+8D*xM5;0f%#h=lfdJzy0~e0nEfm)#OkLc5j1@<=udX!78su8ke;h+Z)_;KTrSJGs z1EsDc79NQ68A&HnA~u^CRf$f-srVHk5?x?p2)puzf@{Gak)ZejS_EJ1k7Gcg={% zy~C#Zge{%7OXzj54G=_@`hZ5ekg<#MV;pVA+NN_s0FYmZprfd`+4{CBIuU8Par(K_ z-&TEmpA7Lcx%n*DhzNHIL0mVMVDsY|(fUYI5-}Aq$x(hVZW@Obkn<#_FrGyWrqF|X zF2waoF(Ph};~prv!y!7}am^6t^+CWfh;kO?Vt+AXEN%4i|J%q{z=4u4e#r zAE+Y^S9RBq>gxGs)ecABT5HO0~KAl9`aEQmxj?$ zTsZ=Zh93!(ljXo=F;tq3Te@9`kMD7vgn78fb$H(iYHkS!LSb0P;~Ei9oY`@;h*$)7 zx$~r46-5CmVK|ZM5^an{`~a3Z?R$T4fTxrm6mB0Z`t>uCFiCx()kU2)O*tc;d*-i`|Z1g zyuo+3%vIkmU*30eDZgQ6BI1eVH}Lrl?{1kk@OguGbMh7m-4gP42st~ZbubAp+Ce1* z`&PlS746X(uNkfyRu#D>GZCa!t4hRvVJ(PMzwCU$89gEtHN}c{@I^b`**q8i_V!gZ zKCfyRTVWc&7QSf9+@@76`KV)T`4EJ)uNz)5%pRH>`tF6dFD%puEr$ii5uxDdY8C;~ zGd9OR)YQ-Ree3iar{DA})pUHMR#j&Imzs_j^bzGse$i`9i}{;AQZu==|IpGg@Bj7~ zjnC}6nIFylM%f+v`uA%(?vX6322!4#v6XyJv!H%2>sHq74Z?1BY)DPZ#Y+r$$JT7EBkKy%}}&`sk~`n|7s~oEMsit?`?QZA9HNF z zhi#T)O6z|%>q%H@Mfl$O{9{?x+pPu1EY>v2M&x6%Vcx%ofXJFDDuCt(x6BmD2|fg= zGSwO;!2$l8P2X?0@BtNaq}ete^R+me9_#B4APd;t(5BVfn9+#D9a?ob1hRI zw=$LU5`Djt3Ipg0E}`RHbcaqG7@3Qx!*`jB05eBf_iVQ!!99Sjquc}ZIHhn$!)yp4K zBhNFooQLo}1U=+dgToP9-IraTBCI5O0EYEbdq78si;$*}L6NNIvX* z^^MD~7^14_@U<&nz4E5i()>c!$aVXNM_h^RB(9VErRWT-v0NEbhlhI!nVuz8 zjUs?Prgev4IsdL|E@$Du4-HG!dcNPopB;|%2NwGSc%8|aaeuuyG7znOX;iS*zpI)$ z#9PkaHCY$QI11K{f@$N_fe$;6KD&95v{^8>3WnDCmSw{pUe(J#+Kcexduqmdfem2* znJ*jnY*GCw30}`bkfyOw;X7SLWwdE-MqDR^rG!FJ;bx7DOWCY6QRQFPZwcrxr~V|) zHpbaO5m^!*DjGPi!IMigB-V%uKy_$fobz+9BGYsDg~U2m?1j%ApWe!=cFb1Kv2!(Z z1@pGqr+8;OQaA_T(y&=>;Gf|OS?0+GCYMmsKP$+j2%PAEB;1Qwl=4k+W=Gn$jVazB zdMXIfuEWZsGBGZ7vkC)}T*4xR5`&|`FcAnXgzm3_Ie= zjm^F|J36Ttb;Q{ajIU}AYs(kMz+Hp|6h=|4WNc6m!w4J#*8=Yzh(v)g;$pW3!*sq0 z_$a~>!TklQ`BVHtBXopTwB~2N_`bz{*PJtR@M~jJ2k&MVMDnBSgzWWyaCEAZuou4Q zULk+|qRKvX@I$R}B|Cp6Kca|ii*AlIM4#X-PG0ByP-|YvE}SWhR7U!uCnCpZZMR zz)gpegt&SnFd7KMv@$U`7`U89Az&b(4xQY81Uy0C5HUW3ZPOZ``2xoFgThCynqX}( zSpRtEN?!5TN2B}r+}hc^SskCV>5g&JZ`4dtH~W!>FnK}6#5=E4djXhq{dH;Q6vvF~wq-K4Uyc@V*t#F}K|8t*8j++>$YtVXSGd}PGS zg}A>$+FSUAps1pqGClJ&I<-<>J!`x9)byj%7iR{hM|eXqui8CVF?V2o^V|-8<8Hi( zy&yuR^pH56@tufZdCKk~KDeZnFlT&jCY5lz)90DR!cowEL3`j-{mO?zxH$xIwwf*< ziNtIpY3fVE<0Jkw3ZsP}F+e!_?*rg3@e5TVm|{NMuyJ}|X8#|Zk8E8wJEA$fsXF>7 zZ&=T(+UNWT9-Iy$vD1z;A|#IKF1C^A^1)WXzJ3)_GyIS;DiZ0I?^30*6q)=($TXAZ zLFV8=)g1LQQNx0+aA{J@^*br+An$~0ze{=#LWNR1;uJ1jYAGpDSZ!evNJ_4m;FftKR4Hdji?#uRnhno^Qb%}QFcDYXunO~P&gb(9*)*wp&cV3Q@e zEX~S#5EAJLs*@^kw9=YdgOm#@euc}H`o%BSsX?9Km&9i+jFcN=oV}(_m<%%7-E&Ze zJ+&U>%|%+?nlvNQ^4Fvl^vk_kn`14sw4b5$!W8waBJ;rlKoz^m4pxkEO4ig`>{ih) zY8q+~mNw@~J*T$&XGPwN+V0CH`>r@Tp}=!oahwqS^RGCLCw=FUEIQ94LY?M9i0qNw z=kd}b`p*Z(98KiV0#f$D`0cndx$a6sLj(D>$w69#e*Q#J??_`B2*PZ}?>@0s%FHB| zxVZbcK&Q~96uv!ak-@H`O0bvMttfj5x=U4r$~e0_&N_&Wkn9NwHkq42oV$)7Z3{_g z3MmIPK+aP<=s*Pu&xEQGP#(F|z1Y$FcF)|SQ8ro;&57=L%|E;6)#2!=<+6=JM{hzg z=cr`4CrO2bx!}HlU{Z(E2k;@ScW_UWS1p|NWqlb(v^Sq9F{)HU^@+k!3wY{`;B!)K zA@?Pc$w&_mBs4OgD6hb2zmcSC&SEO+>Zp#6^0||$x;m7haw46w*C-sZ2~v&1aRt84 zZHRJzNCLu`&>fWSIHDSd&}nh>v?4E!U3knN;=V!BNyj8NIqDIur`xb6?7s|0d8#-f z(Fa`-vgBdr7EuP-wL{;*);GocLg!u4HLj|aCe2jmM>@u8pH{Be3vL{~et5CeJ%3`M zc)8{ZJcE&7~OP0zNYf)73 zTFY$J?AUT$hfuYXuiQ0%nJ+uEY&|riziTg!?2Bq%Ix5&}(O!d@w>XxG-*>}`F*nkD zb0|8%*R?OhU#+ZzFWosm!WSN0HXfZmyrMH*GhQ{Wm~AsvGlO5>FjE>CMBfU&c{=*Y z+~&E^eCv0Bjfzx;3~sD!P@TH@@7Jjmaji0 z*s6Hzy6C>ytmsp7RdWOLwcijPW$dQZv58J8$+?i z-HVO87i#%Ej|z=0-hFDR@zk@;*S1~Vc5TQpY z1u}D2^2|NRCBa)z31#E z2@$7=jJf63$UR=B^P3r3_L5AgYXgDDY-G+Mo3&GNWPfT7*}pQF^^h_j#7QtEm?glc zbJ`e98`9EzPRR&2tB)HqLSs&Y`iIOX4BP~3Q;gGET5FCwiax$MRL2mge+lsx1;u^Hipx)tb#&=mwN$6rPw9PsOy)H&=Li}fGJ8dC16p6)taKZtF-a%} zO59}Cm5Qr5^GK)2PMMvt%A^)ce?K!vxdbWw<^IWSl3TWyjFH{W(6Qj({oRsh3g`iV6SeVwy9t`W~h#2;L{JGyr!=c%62Z}v zgMp5-L{&%*2u~VOy>~1yh&bvad%V~NDaYBkehh|p*ivy2G@di&!Qrxj364WQ5{CRq z3#nmG0)$kOB_1C}o!l)_px%K~%Q(*qgD3}A7KS4Vl1V+u=JJk_9Omki4b`I!)jmh{ zp|a=i#&x&irli}oS)xZVr zE&=>0NmtWZq31-n9V9`E(~@L7i82Vh43v0Y90HE(TX$48u)ZPy;V`q<2{U@DK9s)-J%83#d(W>{*qrmsx? z!Q_lHrY*UnEm_%p;642<{q0?`-H+ed{Wzap8Eu(uTCQs2E87>XJ9zH^l!YR{J@bXA zd-m{hbql|4+q`=25}&(kNw<5&mVcw@dQs$%V5^SVHt;r_Q@LZ?fLtx@-wnJSSTM#~ zj^AlHKI4pZM(dYLn)u?4bL^tEnLp8+T6FL0Cto=^dr_!viB<3AtM|^=^6U03Y+C4D zILPN7`?>BoG?Ur&u*Jep8h+N1^Fod@VN!MpIZbe@He_FGzuG=?F-F9evK4#rjic9( zM)iVyL(G1jx1XQA^sT4fcxry9&~!vhxIOf<3qQKRpCXpe>CTmXN8W3{)qeZQ*uIl@ z_CZq|(Ly+}@cFf~?CknE=W^W+-q|t#=%RfW?>)y4oaZk*u{^->=R$la%zGwK0!-t_ zt{z*-E{jY=hInhuht9^CzDRH6V06=BPW9a0-zr&K>o3~&e1F5c8y1FRZI9e(dt|0R zx_-8HxpFgKv1M-0A{@ti`+lod*iN$ltW?2xsb|c&*B-z6cqA`oax9x1iAo|JZxziQ zoImiLM+E17zP5AWQNHkC%y{TGxr}`yEW6gcYm-+eBlUu1eauqNTk2;wO&waX<=!Z` zUceVM3AT+fTQ6_xojdT|BX1w!cl8KcdZ&)8sItCt><^C3?25KV>t+x0wwBn*f%(1j zlk+3D>lWL3`M`x`+sKk?^e#?i*1TT#O5JR~P}wFFw14m9!o~L{Z%y(pk8tp;(BZ|g zOs)Awt1PP#O@0|)uyIZ`_t?Cj-?D$gg@w$#C!cM{=hH~SG0 zu3%lyELUM?5;<^=;=7B_U*vNgGq zy9QiHwHOEbpzJh#XA%nB6!hT~(n;&{Fg!FRc5U!nYadtY zxt~uhDRl=&+w;;SAhJK_rz!2g=P^0>-47ywDPhfdhIEWL!vJq&4lF6(OHiW6-x>Be zdxn@#()eh+r`H@bHCg0mkn>&79PE=zCHL;6W`a*pY|>H`FYDAuN$P| zF$`4Kkh5!YH#DNyiJ@6@QSO9B=shRDuimtXDn?Uvr*Z=A=mf#}U%nXq_53x*1gx3N zFK>Q1aUY8K`qHxwa&ridZ%|F+F@R8IB6pj_{+u8Z{?7>FS~}=(98^<12IU&W`6GM<2i#}qK~%{@E3xaR%pdo!fFX*d5d2p#Yxfka zdWhKWEV(o8CG+~JgDb|InXczg^Tw*F&R^(`tmxJy^ojR%Om;rD-=-~decyWet*2+U zMYcxsU)sqR)h*gKES)$B9b)Xn**hoB-rB(rayaQEsuQgUU3EnB^@f@9s4D7? z9-7_vj(W~Nf9N~o!o~x9)4_$%Pg?mNR3BLGIWHW&z#kcbvTl^Mt=NiZjFEk>X=l~5 zgMV%k94&nLwmGVaYyW4eh%=x|A7wK-!}PWWQvE0Am$VLvdAWFVAiDk=BX6CU z%Y74y>1}*zE8lvU&pooFI|^&4q`V`@B|U_=frvBro3X&5yptC#*>1?zv6hj?>4hW= z`w$o%t_4swDpCykG68#pbt${pV3tee)`9G(DdE&4H<`Y#NXj*Q)us~v6mqL;&I&>e z;)hg4jZj%fQjbYN#YhCfJ~ufQ6*Tndael!dk)FgAG6rl}L_EX!1D6@Xjip(gW}wpd zy@(Ceie9Kv#2y?TlCC;LwE z8nki0e#jW&t}nO6RhZiQ4C9oYvQf*5q%(TDxzdvXuAxGwBZ6W;?*Zztp;FaU0q`r_XFgeupl73!k#d>&dYla zj&2i#OZ+7m272ik+3M4;l5@n_AubA-Nn;fn)t9&rjRXEM+${uYqoqXxo#D^dcFX;e zG(d6I(Z&4`G0AvK_$}(&on{pgJp{?17UGw*(Uu1mj}5n|DMz~|P zaoMq5I5zw-_a3RFq$lI`Z4TV)B-vFN&O!r_$mnqouC$O+#E0rB_Vu|d|a3$ z{xJ!Dih!yW$%RgF4e@UPuS`gZc(&j<68jSp5ZkzTnC272;wr)j)8jW#NRcObY5*~j zKh-9>&5<>2Boos|CG@+N>_tLugzQa%1*(X(?<=ONsEar5e9Jkv|IH>u=9>|KYtiBgn?_8R{ENpY}EhqRBz9m1DFyjJDvc4zyC&IjLVpYkgjZhNlY(%$Z z%f(S`TSd&ajkj%!`W9=pfsxf&W*+_4fyMmVd7K#7&+mtY7(T3Cz84nW0Y1mKtn)#c zq{^D%B0FcBd6VlM!~B8U`-P2zru8qz4COyJl&|C!-3VL{ES3ju z@8i2Y%g4^fjtwpz8(cazER+X?yudUVaofx}zNB?NXWspviswyyC)FUGyTD%<;fqH3 z+_7cj*jGD$qlGc~H#%nP_GQ!H+xurv@ZJ5($4z8%wr_~=+tY0*iPVb*tC;JTSn^(^d{Ec2H z-6z=hO?Tei+B%>6?NbYt?|5cUW~Ns#6i0l^hDu&l`SC{;jCqI+5kvS7RGSZF>(X3<2_dG8`!5jrw^`kZ_DE(j zK-|+bGYaBs=TdAVb>YcyV;F3(nKUoSd%|!zD$zzMCDxg|!&wr)&$ufL5aKP3aKB}} zTqWw~su1ALFHXSUxYj>*F~GsgwupSHCQ0f`40?dL7MkZU@kIm| z#c*{9M3-IJ?VKvXNzh1E^if2}MQx^QZCBf_?Yz1(f-Bc*r}lrSHP9=;BFgAc%+$DS zYMiTCF=i8;Xe58hSn)et3f8n}YW$6o(HTE9H3Eh=HGUGpX2ASS_MQ^;trEkYhOD&h zgYYVJDwhzZ%3w*w2|P|OLGgnTz~m&Rf?<*uCL}LH=+>dMHBVAe2=f^HrBZHPG_C=~ zE6%;*^mZx-ElFNDOFZ|{WkQ}U6yadNl+E-(U4OBZ+Fv5Yx{0-7 z&6BZiSyB36-7eV-7R>6ekP0K$lex7SBo%!M4DL$K2xI6ci-JDnCgqGU#(uIoL>Rrx z&_g`zjBZ`RUEg%v&@WK{n10X5qXQ^&56sgz%bsj2?qAMS2cT>{#25;Lwp5rehhnVs zy5_ucvK~lLL_W=N8#A`Zn1lqBz1bqo9&Rik>zO`%>-zfoh9S9V#rL2orO&x;Q%W9r z1tB$oB``{6iE3%L&W{psw)$!-RPa8`dFq;T<@Ik!REUw1a-&uGkjx6;`z_-gF-2El zB|&{z$c-Wou8maXAEoz5Uy?ZIFrg_txV_HS*Hk`%p_v@2tiIKt}h{r^`z_vs@-tKWy}lP9yS z4R5HTI16;6__^XYRia=SHyu3ObEtQ(=aJqX+?6~DTQa#atAT7q9g`+K5!@0tcy}aN zWc^r!$Y^&=>Jb;2?T*iLXmOI7Fu4g4daarkE)sD>L7=Ec;Z@*$2UIU@!%SsCvnI)6ovtS<9f+# zx=BFRD&hZ$b)8;=MbG+*`~o@nCEmkDm95VPIElV%MIba zX1E-1kCSgyj}6Z+CPhG@vI&(^uxB%zo|KyuOLX9i+FrUMWY{HiIMYTe1Z7jEu&uyIDi`yFw+i(OL7lNq-Ln&g`{F@4LPuI)l7tH+n zokGDb@v^RkErRvX)Zw)P&;#!V9Gb&260BgZ8c?1}zoN@u(WBinqfm*j%U>z2c(MOF z_twN*&-|u0pXT#+B6*j6#a=SghFF>Xu04N|psJAHBH%)?oNY*LO@7b)ux01-y^+@C z;>J04DYtoHFK<5xkVC9}rKs%1`s;h&>YQ6Q&%W8k=d{l^ubMPjyFOB9sWl(jVUwP5 zM|~|k^>gc%n)hN#cx&@JTj%ZH zZl7!5cR|5F!2A6C$wA&cw4@uxB)xAiat<>8I**6Uu*cIzcZRMmEYgGcKt)<7`zu0C zomHHJpbj=6hN~d~MN^8~gt6kjLV^kc@-HOz??^y+EYXXjoWuzFi}%fWNbDR5D0V+j zVsrrz#9r$R{(X{0rSv$7-6sJN#>J&TR|dgN(hcHqgXO34F?_&eeZZJ$X#0TC|H7IxWm?Hz zH)Xk}(XjS=*;)j470llE%xX6Ko>j#jKr|oGEi8MQy{A*K4fpg_to7b8R>`{AkBlta zaZjgb3-1+a*t~n$N_G!>4~aYO>GD|Hy=DdLXYc7t5#7z^BHUQcR^8jrs@TeV=5p4w vx}RlK#_7o=W#NiS`;|k_9J<;wQ+1>EdTpfn>-BIu`MJsgK3`Qt=k)&oAz{q- diff --git a/tests/__pycache__/test_report.cpython-314.pyc b/tests/__pycache__/test_report.cpython-314.pyc deleted file mode 100644 index 6b91abf6b94b07c9436e5ec98fecca1a4a56336d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8433 zcmdrxZERCncGus}jzc~m$piv2%i4jKVDp@LFjk5qCK`k;qfyALi6Yx zVhp25R^JBMKNZuydMBZlEOx_mRLf%%{3UA}6b;KsyTG6y=yjBxme7B)#doT2X zpYaCuT?%^(mGQ<>tXh_x)8_H58}Z}YH{#Br=^E(ESxF9K>aHqsXrt*hs<>8l*0pl& zw39KHQZkmZnzghh*KMG-m9{hG)b-3JhH64_gS@!`wIRmdgc!%Pb-c4UF0IsSQcH3J zp{xQ*=SwIXr<<7Sy>BBF-GWdH0&2e|t@v(3+FG~5ZoA0OUGsZPFy=NW=7bQrq}Zg0 z7)!~LVioz6Aj%m#cY#mJo{Pdvrb^_cbX@jcl!Ro)A&6Wg&f|tiVIfIZc%ktG9$Nv< zqbY*KV7!#7fGrMbzkxNRjcF6B4>Dz(GHcA^R;^Tgx1mWRgS1V21zLkfD;{EuQ+7?# zq)Af<##Dke>sa%o!DmWYrid|Nu5qWvTYQg$fvtb0baNQ5U*Cq%q(vLg#<{nmDcDH$ zsie?tXzC5Uy@G)h*HxQE#Xeee89TI6@eO?x*9d8Hyauh5vN9I@r>QF_)w=qWz9x*d z2~C>D+ltb)lBSa5E)7$BYi+fnf<2iC}wyGRpM2s7>f(3bZ(ns71DAlErWTb6A4bds#G5x4UJ5Ek3BgN z8lMO$n+w%bq2Y1%)WnGsL(CbaVRQ`I*b}2;?1|x_F_sxVJ;WRyixj*mnS}*l5Ppvj z1K0_Zm`5RW1bpe`y75vvLW~fD&go`w1~h6>j0+JiE**F6W!vD^2cf|HOYu*iXE-gEqM96-wq4sq?WouhfU21Wro4dN?NX5m|s$ zx1~6FHXfT%%;Qi{Op+{$*jXtSXo$=z=4dP;D@H+5Y<%+Fm?$I_M?^@<{1q8<)rfqQ z$YLcq3OlM8V}e4RlEql^!stm6?*+I~E|yg6BF{xx%%?cWI+FEsQ%b>yvUng$O(K?L z6Fi5PV2%@`id_{CLltp=7AY*TsxXuU_+iB=$()>q@u>tyhv^-~H5I!6 zu^E2Se*>6DU)k&O_VzFA?Rk4g&fc-?{%UL6C*2=)FI~OUncceoSEoOl`kSfG-76F4 zvIp2j+t!k{OLOnoNb>5m*)O#@PX5{#6Gl{n;Z*OPaJ4d{Rd~?Kl=guKKn8A z$=MIjE^o_M_pMd;{Z%GE@cl0azMo@4d1m?xW;)BT`GN4-K=`YMZ8zWfqjx@aEnUj) z>{)3GW*d4JjyyG?y_^Af|GB<#wQ=&!`F#J8)&3*d{*iqDsnz~dx&Dc4<7Bpea>4eX zreSd|PX|`%K$Z@!)C8Y4Ajj@6>>W=x5g7yFXCMdupFORC2AJrIz2lj50OtQELkEX_ zmcMW58E&!vdQ;o5z53Ta3YQ0lY|wDeUNziYcW(o{2lw385 zWG7>HA-6+u4&iY7mM{|n%PF&TtmL4p6zLNJt2PESr3Y5cJo||L{Aj^(9z+s5c}52?}_+WBDyia`gI#l2NEAYFX28rR4ubeygZRo>PyG# zCAghHBsBv7ed`(g)tPM1(N)T~VE%E{V(5>jmU{nu@G&%^%9HyBx|oN-S8`r})u)6d zsLwh54AA;~K7Gm>7q~)0cS+=Aj9N7QmQqY*Qp~R@#Zo54`kGQ~Wl~;uEL52k`)iKH zQ6{CTD1~u?(p8IiCo@#i1I`nizG4XCiX_C}<)exr@XVg#B*`OkGM}OR-XM7}1uu$% zC_SsfUelv`&1WXRx9=Gh5hU4@mZBM}-y7%&_KvwtN)-s~>ciz@IOKErGB8DEpn^fEri4616$)OB?QXdfNy>Y_V?L2q!EJJS;A zNzj2liIxSKi+9t31ReBCbOO!?Tr#tx*YEfGj{#4h*V}iD?(FF!b#hRO0(ZNrqI9QL zNbw@AYMguQwo#!}y&X*w_baZ~*CC>JQbd4r4ICTTSx$ljg(R`qURXJn03clAlaj8t zEOel&G$&nB#y0Bl|3;n{>-U(Z-yc-JU%k@kUhe(H;O)U2wHGu@MfKi^{&oB><2mY( zj_SW_{>=GHXO0@yQ3a~|)S(w>;EoprivK?v_#8_yzf^PK^5M2-c6sQoF7-qX$_swf z>+h!3Tf9WW{XQ}a$ezp~CL;;~RN+XROGkN6M2Pa?ZaM+4I34Da@~kMNVi8}VSxBWN zPoJkRobh7r5Ga*|x%n$xA{FP~40!!v`n)J4=&;CNh`~wwDoGrKGaaNi86VVT%o3on z_RAokn-0>POmj3=;fx>CF~J9%q$fg$!1`448xy|^ z8o0q){KQ)3W;q$`BgrPY1TV4bqOcr3ZLwsI#8&VqB;~+HCj_X<;TA4hXpPX5poXJrU9ed<+^KYHM0dttpQ3wCl{148*1Jnr}HS&4;Z@Yif zeV;o00t>?E6SqSAE=`<k9KMph4b#`?Z$4W3x4R$EWP3`rhQ5}@3tz%W&TUvJq7q^J z*^lY-9ApBbAO*s&-w!OFa+dS4D;_mw6ONDnpb&(C1N;}}fK#M;3%A6QQT~eJP$Tnk zUXt8q(GTJj3(+RUhC54(JY3PqDVAg<@WoucF9lRpB&S`XbZkKh%jlJLi}ii5>bFCFKh>||MdLBfF%YBel-J|;?# z@JjMYfn~)2UW!2shA=pc!BGrOV!!}U8d%7PadIreax!??nKUdEh^ixwT#|5BEdnl^ zWg(MLU4jHQ?7iM3CL(rYc^I_8Pr}{)=XF?yM@EBX#PG;!vmAI-Z?+sZJaSqrZyO$M zGFpa=k0^`9{S=A^9wGolBZ-U%uH&n5AEV!02vdwba4Pz zv3V&dQ?K6Z0&oE74B#6ZTpHkf z(F@jFP2l&UIjSh?dAp)Sf6-6fN(d9 z+u;i@EV!6ho4jKO6GEb^vF^1taz0Umu=KRjQ{Xuoz;>b_fTi4dA_beyYs`t-3o>;34y-!+p8xAb)&2MgB-Q4~J yLFGxvP-XBhj^wxOTHUhi34+SQ7G!oVWOAnF2WIE>6Yrh)!;}ARLguYxn*RfcfWoo> From 935a7f9b00a59de45823a287c5b270f082d14f81 Mon Sep 17 00:00:00 2001 From: rzonyx <77541804+rzonyx@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:23:49 +0700 Subject: [PATCH 3/7] Put the command that opens the job in hbench's dashboard in the summary hbench's dashboard (hbench view) is a web app with its own server, so it cannot run in a job summary, which renders sanitised Markdown only. When the job is uploaded, the summary now says "View interactive dashboard by hbench:" with the command that downloads this run's artifact into hbench-jobs/ and opens it: hbench view needs the folder above the job, or it takes the trials for jobs. --- README.md | 5 +++-- action.yml | 1 + docs/REFERENCE.md | 2 ++ scripts/report.py | 27 +++++++++++++++++++++++---- scripts/run.sh | 6 +++++- tests/test_report.py | 16 +++++++++++++++- 6 files changed, 49 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 94e6001..1d67d72 100644 --- a/README.md +++ b/README.md @@ -119,8 +119,9 @@ With none of `path`, `dataset` or `config`, the action only installs hbench, and `hbench` themselves. Useful outputs: `pass-rate`, `n-solved`, `mean-reward`, `cost-usd` and `job-dir`. The job summary shows, for each agent and model, how many tasks it solved, its mean reward and pass@k, its -errors, the time it spent, its tokens and its cost per solved task; then what ran (the hbench release, the -agent, the model, the tasks and their digest), the errors by type, and a row per task. +errors, the time it spent, its tokens and its cost per solved task; the command that opens the job in +hbench's interactive dashboard on your machine; what ran (the hbench release, the agent, the model, the tasks +and their digest); the errors by type; and a row per task. A `registry.json` lists datasets and where each of their tasks is: a git repository at a commit, or a folder in your repository, read from `working-directory`: diff --git a/action.yml b/action.yml index 53695f5..34ea22f 100644 --- a/action.yml +++ b/action.yml @@ -143,6 +143,7 @@ runs: INPUT_JOBS_DIR: ${{ inputs.jobs-dir }} INPUT_MIN_MEAN_REWARD: ${{ inputs.min-mean-reward }} INPUT_ARTIFACT_NAME: ${{ inputs.artifact-name }} + INPUT_UPLOAD_ARTIFACT: ${{ inputs.upload-artifact }} run: '"$GITHUB_ACTION_PATH/scripts/run.sh"' - name: Upload the job directory diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index 10c6446..f582276 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -62,6 +62,8 @@ The job summary has: the agent's time in total and per trial (median), tokens in and out, and the cost in total and per solved trial; - whether the mean reward reached `min-mean-reward`; +- when the job is uploaded, the command that downloads it and opens it in hbench's interactive dashboard, + `gh run download -R -n -D hbench-jobs/ && hbench view hbench-jobs`; - what ran: the hbench release and results epoch, the agent and model, the tasks with their registry and manifest digest, attempts and concurrency, and when the job ran; - the tasks' own metrics, the other keys of their `reward.json`, as means; diff --git a/scripts/report.py b/scripts/report.py index 7ea192f..021d1a3 100644 --- a/scripts/report.py +++ b/scripts/report.py @@ -5,7 +5,8 @@ Reads hbench's result.json, and beside it the job's config.json and lock.json and each trial's result.json. A trial is solved when its reward is 1. Each eval (agent, model and tasks) gets a row: solved trials and the pass rate, mean reward, pass@k, errors, agent time, tokens and cost. Below come what ran (hbench, agent, -model, tasks and digest), the tasks' own metrics, the errors by type, and a row per trial, errors and +model, tasks and digest), the command that opens the job in hbench's dashboard when the job is uploaded +(HBENCH_ARTIFACT_NAME), the tasks' own metrics, the errors by type, and a row per trial, errors and failures first. mean-reward is the trial-weighted mean of the evals' mean rewards, as hbench computes them (a task's reward.json `reward`, or its reward.txt). Exits 1 when INPUT_MIN_MEAN_REWARD is set and mean-reward is below it. @@ -265,7 +266,21 @@ def order(trial): return lines -def summary(result, config, lock, rows, trials, mean, threshold): +def dashboard_command(job_name): + artifact = os.environ.get("HBENCH_ARTIFACT_NAME") + repository, run_id = os.environ.get("GITHUB_REPOSITORY"), os.environ.get("GITHUB_RUN_ID") + if not (artifact and repository and run_id): + return [] + return [ + "View interactive dashboard by hbench:", + "", + "```bash", + f"gh run download {run_id} -R {repository} -n {artifact} -D hbench-jobs/{job_name} && hbench view hbench-jobs", + "```", + ] + + +def summary(result, config, lock, rows, trials, mean, threshold, job_name): n_trials = sum(row["trials"] for row in rows) n_solved = sum(row["solved"] for row in rows) lines = ["## Harness Benchmark", "", *scorecard(rows), ""] @@ -275,7 +290,11 @@ def summary(result, config, lock, rows, trials, mean, threshold): if threshold is not None: passed = mean is not None and mean >= threshold verdict += f" {'✅ At or above' if passed else '❌ Below'} min-mean-reward {threshold:g}." - lines += [verdict, "", "### What ran", "", *run_details(result, config, lock)] + lines += [verdict, ""] + command = dashboard_command(job_name) + if command: + lines += [*command, ""] + lines += ["### What ran", "", *run_details(result, config, lock)] for row in rows: if row["other_metrics"]: metrics = ", ".join(f"`{key}` {score(value)}" for key, value in row["other_metrics"].items()) @@ -331,7 +350,7 @@ def main(arguments): f"pass-rate={round(n_solved / n_trials, 6) if n_trials else ''}\n" f"cost-usd={'' if cost is None else round(cost, 6)}\n", ) - report = summary(result, config, lock, rows, trials, mean, threshold) + report = summary(result, config, lock, rows, trials, mean, threshold, job_directory.name) append("GITHUB_STEP_SUMMARY", report) print(report) if threshold is not None and (mean is None or mean < threshold): diff --git a/scripts/run.sh b/scripts/run.sh index b04fdff..ecc57cc 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -65,7 +65,8 @@ arguments+=(--jobs-dir "$jobs_directory" --job-name "$job_name") add_lines "" "$INPUT_ARGS" echo "job-dir=$job_directory" >> "$GITHUB_OUTPUT" -echo "artifact-name=${INPUT_ARTIFACT_NAME:-hbench-$job_name}" >> "$GITHUB_OUTPUT" +artifact_name="${INPUT_ARTIFACT_NAME:-hbench-$job_name}" +echo "artifact-name=$artifact_name" >> "$GITHUB_OUTPUT" printf 'hbench' printf ' %q' "${arguments[@]}" @@ -78,6 +79,9 @@ if [[ ! -f "$job_directory/result.json" ]]; then fi report_status=0 +if [[ "${INPUT_UPLOAD_ARTIFACT:-true}" == true ]]; then + export HBENCH_ARTIFACT_NAME="$artifact_name" +fi python3 "$GITHUB_ACTION_PATH/scripts/report.py" "$job_directory/result.json" || report_status=$? if [[ "$status" -ne 0 ]]; then fail "hbench run exited $status" diff --git a/tests/test_report.py b/tests/test_report.py index af47222..68378a4 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -16,13 +16,16 @@ ORACLE_JOB = HERE / "report" / "oracle-job" / "result.json" -def run_report(result_path, min_mean_reward=""): +def run_report(result_path, min_mean_reward="", artifact=""): with tempfile.TemporaryDirectory() as directory: output, summary = pathlib.Path(directory, "output"), pathlib.Path(directory, "summary") environment = { "GITHUB_OUTPUT": str(output), "GITHUB_STEP_SUMMARY": str(summary), "INPUT_MIN_MEAN_REWARD": min_mean_reward, + "HBENCH_ARTIFACT_NAME": artifact, + "GITHUB_REPOSITORY": "you/tasks", + "GITHUB_RUN_ID": "123456", } with mock.patch.dict(os.environ, environment), contextlib.redirect_stdout(io.StringIO()): status = report.main([str(result_path)]) @@ -67,6 +70,17 @@ def test_trials_list_errors_then_failures_then_solved(self): self.assertLess(error, failure) self.assertLess(failure, solved) + def test_an_uploaded_job_gets_the_command_that_opens_it_in_the_dashboard(self): + summary = run_report(AGENT_JOB, artifact="hbench-bench")[2] + self.assertIn("View interactive dashboard by hbench:", summary) + self.assertIn( + "gh run download 123456 -R you/tasks -n hbench-bench -D hbench-jobs/agent-job && hbench view hbench-jobs", + summary, + ) + + def test_a_job_that_is_not_uploaded_gets_no_dashboard_command(self): + self.assertNotIn("View interactive dashboard", self.summary) + def test_mean_reward_below_the_threshold_fails(self): self.assertEqual(run_report(AGENT_JOB, "0.6")[0], 1) self.assertEqual(run_report(AGENT_JOB, "0.5")[0], 0) From e5894620d31f6b2dc5c456980ab1ca4c46e0f2ac Mon Sep 17 00:00:00 2001 From: rzonyx <77541804+rzonyx@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:26:16 +0700 Subject: [PATCH 4/7] Chart the trials by outcome and the mean reward by eval Job summaries render Mermaid, so the summary now draws a pie of the trials solved, failed and ended in an error (an empty slice left out) and a bar chart of each eval's mean reward, from 0 to 1. --- README.md | 3 ++- docs/REFERENCE.md | 2 ++ scripts/report.py | 34 ++++++++++++++++++++++++++++++++-- tests/test_report.py | 8 ++++++++ 4 files changed, 44 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1d67d72..753aec2 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,8 @@ With none of `path`, `dataset` or `config`, the action only installs hbench, and `hbench` themselves. Useful outputs: `pass-rate`, `n-solved`, `mean-reward`, `cost-usd` and `job-dir`. The job summary shows, for each agent and model, how many tasks it solved, its mean reward and pass@k, its -errors, the time it spent, its tokens and its cost per solved task; the command that opens the job in +errors, the time it spent, its tokens and its cost per solved task; charts of the trials by outcome and of +each eval's mean reward; the command that opens the job in hbench's interactive dashboard on your machine; what ran (the hbench release, the agent, the model, the tasks and their digest); the errors by type; and a row per task. diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index f582276..c02e7ed 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -62,6 +62,8 @@ The job summary has: the agent's time in total and per trial (median), tokens in and out, and the cost in total and per solved trial; - whether the mean reward reached `min-mean-reward`; +- two Mermaid charts: a pie of the trials solved, failed and ended in an error, and a bar of each eval's + mean reward; - when the job is uploaded, the command that downloads it and opens it in hbench's interactive dashboard, `gh run download -R -n -D hbench-jobs/ && hbench view hbench-jobs`; - what ran: the hbench release and results epoch, the agent and model, the tasks with their registry and diff --git a/scripts/report.py b/scripts/report.py index 021d1a3..cc9563d 100644 --- a/scripts/report.py +++ b/scripts/report.py @@ -4,7 +4,8 @@ Reads hbench's result.json, and beside it the job's config.json and lock.json and each trial's result.json. A trial is solved when its reward is 1. Each eval (agent, model and tasks) gets a row: solved trials and the -pass rate, mean reward, pass@k, errors, agent time, tokens and cost. Below come what ran (hbench, agent, +pass rate, mean reward, pass@k, errors, agent time, tokens and cost. Mermaid charts follow: a pie of solved, +failed and errored trials, and a bar of each eval's mean reward. Below come what ran (hbench, agent, model, tasks and digest), the command that opens the job in hbench's dashboard when the job is uploaded (HBENCH_ARTIFACT_NAME), the tasks' own metrics, the errors by type, and a row per trial, errors and failures first. mean-reward is the trial-weighted mean of the evals' mean rewards, as hbench computes them @@ -266,6 +267,35 @@ def order(trial): return lines +def chart_label(name): + return re.sub(r'["\[\]]', "", name.replace("__", " / ")) + + +def charts(rows, trials): + n_errors = sum(1 for trial in trials if trial["error"]) if trials else sum(row["errors"] for row in rows) + n_solved = sum(row["solved"] for row in rows) + n_failed = (len(trials) if trials else sum(row["trials"] for row in rows)) - n_solved - n_errors + slices = [(label, count) for label, count in (("Solved", n_solved), ("Failed", n_failed), ("Error", n_errors)) if count > 0] + lines = [] + if slices: + lines += ["```mermaid", "pie showData", " title Trials", *[f' "{label}" : {count}' for label, count in slices], "```", ""] + scored = [row for row in rows if row["mean"] is not None] + if scored: + labels = ", ".join(f'"{chart_label(row["name"])}"' for row in scored) + values = ", ".join(f"{row['mean']:.3f}" for row in scored) + lines += [ + "```mermaid", + "xychart-beta", + ' title "Mean reward by eval"', + f" x-axis [{labels}]", + ' y-axis "Mean reward" 0 --> 1', + f" bar [{values}]", + "```", + "", + ] + return lines + + def dashboard_command(job_name): artifact = os.environ.get("HBENCH_ARTIFACT_NAME") repository, run_id = os.environ.get("GITHUB_REPOSITORY"), os.environ.get("GITHUB_RUN_ID") @@ -290,7 +320,7 @@ def summary(result, config, lock, rows, trials, mean, threshold, job_name): if threshold is not None: passed = mean is not None and mean >= threshold verdict += f" {'✅ At or above' if passed else '❌ Below'} min-mean-reward {threshold:g}." - lines += [verdict, ""] + lines += [verdict, "", *charts(rows, trials)] command = dashboard_command(job_name) if command: lines += [*command, ""] diff --git a/tests/test_report.py b/tests/test_report.py index 68378a4..ce0c051 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -70,6 +70,11 @@ def test_trials_list_errors_then_failures_then_solved(self): self.assertLess(error, failure) self.assertLess(failure, solved) + def test_charts_show_the_trials_by_outcome_and_the_mean_reward_by_eval(self): + self.assertIn('```mermaid\npie showData\n title Trials\n "Solved" : 1\n "Failed" : 1\n "Error" : 1\n```', self.summary) + self.assertIn(' x-axis ["claude-code / claude-opus-5-5 / example"]', self.summary) + self.assertIn(" bar [0.500]", self.summary) + def test_an_uploaded_job_gets_the_command_that_opens_it_in_the_dashboard(self): summary = run_report(AGENT_JOB, artifact="hbench-bench")[2] self.assertIn("View interactive dashboard by hbench:", summary) @@ -98,6 +103,9 @@ def test_reward_txt_mean_and_no_usage(self): self.assertIn("- **Tasks** `tasks`", summary) self.assertIn("✅ At or above min-mean-reward 1.", summary) self.assertNotIn("### Errors", summary) + self.assertIn(' "Solved" : 1\n```', summary) + self.assertNotIn('"Failed"', summary) + self.assertNotIn('"Error"', summary) if __name__ == "__main__": From 6bb79ca27c9a3e703543dea00bcffc6654a89677 Mon Sep 17 00:00:00 2001 From: rzonyx <77541804+rzonyx@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:34:56 +0700 Subject: [PATCH 5/7] Smaller charts, in hbench's colours The charts stretched to the summary's full width in Mermaid's default theme. They now keep a fixed size (useMaxWidth off; the bar chart 360 px wide per few evals, 220 px high) and use the viewer's palette from viewer/src/tokens.css: good, bad and warn for solved, failed and errored trials, the accent for the bars, the muted ink for text. Slices are sorted by count so each colour stays on its label whether or not Mermaid sorts them. --- docs/REFERENCE.md | 4 +-- scripts/report.py | 65 +++++++++++++++++++++++++++++++++++++++++--- tests/test_report.py | 8 +++++- 3 files changed, 70 insertions(+), 7 deletions(-) diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index c02e7ed..2b3c6c3 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -62,8 +62,8 @@ The job summary has: the agent's time in total and per trial (median), tokens in and out, and the cost in total and per solved trial; - whether the mean reward reached `min-mean-reward`; -- two Mermaid charts: a pie of the trials solved, failed and ended in an error, and a bar of each eval's - mean reward; +- two small Mermaid charts in hbench's colours: a pie of the trials solved (green), failed (red) and ended + in an error (amber), and a bar of each eval's mean reward in hbench's blue; - when the job is uploaded, the command that downloads it and opens it in hbench's interactive dashboard, `gh run download -R -n -D hbench-jobs/ && hbench view hbench-jobs`; - what ran: the hbench release and results epoch, the agent and model, the tasks with their registry and diff --git a/scripts/report.py b/scripts/report.py index cc9563d..d9e4801 100644 --- a/scripts/report.py +++ b/scripts/report.py @@ -4,8 +4,8 @@ Reads hbench's result.json, and beside it the job's config.json and lock.json and each trial's result.json. A trial is solved when its reward is 1. Each eval (agent, model and tasks) gets a row: solved trials and the -pass rate, mean reward, pass@k, errors, agent time, tokens and cost. Mermaid charts follow: a pie of solved, -failed and errored trials, and a bar of each eval's mean reward. Below come what ran (hbench, agent, +pass rate, mean reward, pass@k, errors, agent time, tokens and cost. Mermaid charts follow, small and in +hbench's colours: a pie of solved, failed and errored trials, and a bar of each eval's mean reward. Below come what ran (hbench, agent, model, tasks and digest), the command that opens the job in hbench's dashboard when the job is uploaded (HBENCH_ARTIFACT_NAME), the tasks' own metrics, the errors by type, and a row per trial, errors and failures first. mean-reward is the trial-weighted mean of the evals' mean rewards, as hbench computes them @@ -23,6 +23,10 @@ LISTED_TRIALS = 500 PRIMARY_METRICS = ("reward", "mean") +HBENCH_ACCENT = "#076be3" +HBENCH_MUTED = "#5e646d" +HBENCH_RULE = "#dce0e6" +OUTCOME_COLORS = {"Solved": "#137d41", "Failed": "#c52c2a", "Error": "#ad7300"} def numeric(value): @@ -275,16 +279,69 @@ def charts(rows, trials): n_errors = sum(1 for trial in trials if trial["error"]) if trials else sum(row["errors"] for row in rows) n_solved = sum(row["solved"] for row in rows) n_failed = (len(trials) if trials else sum(row["trials"] for row in rows)) - n_solved - n_errors - slices = [(label, count) for label, count in (("Solved", n_solved), ("Failed", n_failed), ("Error", n_errors)) if count > 0] + counts = (("Solved", n_solved), ("Failed", n_failed), ("Error", n_errors)) + slices = sorted(((label, count) for label, count in counts if count > 0), key=lambda item: -item[1]) lines = [] if slices: - lines += ["```mermaid", "pie showData", " title Trials", *[f' "{label}" : {count}' for label, count in slices], "```", ""] + pie_theme = { + "theme": "base", + "pie": {"useMaxWidth": False, "textPosition": 0.7}, + "themeVariables": { + **{f"pie{index}": OUTCOME_COLORS[label] for index, (label, _) in enumerate(slices, start=1)}, + "pieOpacity": "1", + "pieStrokeColor": "#ffffff", + "pieOuterStrokeWidth": "0px", + "pieTitleTextSize": "14px", + "pieTitleTextColor": HBENCH_MUTED, + "pieSectionTextSize": "12px", + "pieSectionTextColor": "#ffffff", + "pieLegendTextSize": "12px", + "pieLegendTextColor": HBENCH_MUTED, + }, + } + lines += [ + "```mermaid", + f"%%{{init: {json.dumps(pie_theme)}}}%%", + "pie showData", + " title Trials", + *[f' "{label}" : {count}' for label, count in slices], + "```", + "", + ] scored = [row for row in rows if row["mean"] is not None] if scored: labels = ", ".join(f'"{chart_label(row["name"])}"' for row in scored) values = ", ".join(f"{row['mean']:.3f}" for row in scored) + axis = {"labelFontSize": 11, "titleFontSize": 12} + bar_theme = { + "theme": "base", + "xyChart": { + "width": min(900, max(360, 140 * len(scored))), + "height": 220, + "useMaxWidth": False, + "titleFontSize": 14, + "xAxis": axis, + "yAxis": axis, + }, + "themeVariables": { + "xyChart": { + "backgroundColor": "transparent", + "plotColorPalette": HBENCH_ACCENT, + "titleColor": HBENCH_MUTED, + "xAxisLabelColor": HBENCH_MUTED, + "xAxisTitleColor": HBENCH_MUTED, + "xAxisLineColor": HBENCH_RULE, + "xAxisTickColor": HBENCH_RULE, + "yAxisLabelColor": HBENCH_MUTED, + "yAxisTitleColor": HBENCH_MUTED, + "yAxisLineColor": HBENCH_RULE, + "yAxisTickColor": HBENCH_RULE, + } + }, + } lines += [ "```mermaid", + f"%%{{init: {json.dumps(bar_theme)}}}%%", "xychart-beta", ' title "Mean reward by eval"', f" x-axis [{labels}]", diff --git a/tests/test_report.py b/tests/test_report.py index ce0c051..2a7f317 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -71,10 +71,16 @@ def test_trials_list_errors_then_failures_then_solved(self): self.assertLess(failure, solved) def test_charts_show_the_trials_by_outcome_and_the_mean_reward_by_eval(self): - self.assertIn('```mermaid\npie showData\n title Trials\n "Solved" : 1\n "Failed" : 1\n "Error" : 1\n```', self.summary) + self.assertIn('pie showData\n title Trials\n "Solved" : 1\n "Failed" : 1\n "Error" : 1\n```', self.summary) self.assertIn(' x-axis ["claude-code / claude-opus-5-5 / example"]', self.summary) self.assertIn(" bar [0.500]", self.summary) + def test_charts_are_small_and_in_hbench_colours(self): + self.assertIn('"pie1": "#137d41", "pie2": "#c52c2a", "pie3": "#ad7300"', self.summary) + self.assertIn('"plotColorPalette": "#076be3"', self.summary) + self.assertIn('"width": 360, "height": 220, "useMaxWidth": false', self.summary) + self.assertIn('"pie": {"useMaxWidth": false', self.summary) + def test_an_uploaded_job_gets_the_command_that_opens_it_in_the_dashboard(self): summary = run_report(AGENT_JOB, artifact="hbench-bench")[2] self.assertIn("View interactive dashboard by hbench:", summary) From 8b5ee76698ff9929807ec77f1323abcbadbfd066 Mon Sep 17 00:00:00 2001 From: rzonyx <77541804+rzonyx@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:40:13 +0700 Subject: [PATCH 6/7] Show the trials by outcome as a table with share bars, not a pie A row each for solved, failed and errored trials: the count, and the share as a bar of up to 20 blocks with its percentage. It reads as numbers first and renders anywhere Markdown does; any outcome above 0 gets at least one block. The mean reward by eval stays a Mermaid bar chart. --- README.md | 4 ++-- docs/REFERENCE.md | 4 ++-- scripts/report.py | 42 +++++++++++------------------------------- tests/test_report.py | 15 ++++++++------- 4 files changed, 23 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 753aec2..2808fb8 100644 --- a/README.md +++ b/README.md @@ -119,8 +119,8 @@ With none of `path`, `dataset` or `config`, the action only installs hbench, and `hbench` themselves. Useful outputs: `pass-rate`, `n-solved`, `mean-reward`, `cost-usd` and `job-dir`. The job summary shows, for each agent and model, how many tasks it solved, its mean reward and pass@k, its -errors, the time it spent, its tokens and its cost per solved task; charts of the trials by outcome and of -each eval's mean reward; the command that opens the job in +errors, the time it spent, its tokens and its cost per solved task; the trials by outcome, with a bar for +each share, and a chart of each eval's mean reward; the command that opens the job in hbench's interactive dashboard on your machine; what ran (the hbench release, the agent, the model, the tasks and their digest); the errors by type; and a row per task. diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index 2b3c6c3..fc38ae4 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -62,8 +62,8 @@ The job summary has: the agent's time in total and per trial (median), tokens in and out, and the cost in total and per solved trial; - whether the mean reward reached `min-mean-reward`; -- two small Mermaid charts in hbench's colours: a pie of the trials solved (green), failed (red) and ended - in an error (amber), and a bar of each eval's mean reward in hbench's blue; +- a table of the trials solved, failed and ended in an error, each with its share as a bar of up to 20 + blocks, and a small Mermaid bar chart of each eval's mean reward in hbench's blue; - when the job is uploaded, the command that downloads it and opens it in hbench's interactive dashboard, `gh run download -R -n -D hbench-jobs/ && hbench view hbench-jobs`; - what ran: the hbench release and results epoch, the agent and model, the tasks with their registry and diff --git a/scripts/report.py b/scripts/report.py index d9e4801..d582bf3 100644 --- a/scripts/report.py +++ b/scripts/report.py @@ -4,8 +4,9 @@ Reads hbench's result.json, and beside it the job's config.json and lock.json and each trial's result.json. A trial is solved when its reward is 1. Each eval (agent, model and tasks) gets a row: solved trials and the -pass rate, mean reward, pass@k, errors, agent time, tokens and cost. Mermaid charts follow, small and in -hbench's colours: a pie of solved, failed and errored trials, and a bar of each eval's mean reward. Below come what ran (hbench, agent, +pass rate, mean reward, pass@k, errors, agent time, tokens and cost. Charts follow, small and in +hbench's colours: a table of solved, failed and errored trials with a bar of each share, and a Mermaid +bar chart of each eval's mean reward. Below come what ran (hbench, agent, model, tasks and digest), the command that opens the job in hbench's dashboard when the job is uploaded (HBENCH_ARTIFACT_NAME), the tasks' own metrics, the errors by type, and a row per trial, errors and failures first. mean-reward is the trial-weighted mean of the evals' mean rewards, as hbench computes them @@ -26,7 +27,7 @@ HBENCH_ACCENT = "#076be3" HBENCH_MUTED = "#5e646d" HBENCH_RULE = "#dce0e6" -OUTCOME_COLORS = {"Solved": "#137d41", "Failed": "#c52c2a", "Error": "#ad7300"} +SHARE_BAR_WIDTH = 20 def numeric(value): @@ -279,35 +280,14 @@ def charts(rows, trials): n_errors = sum(1 for trial in trials if trial["error"]) if trials else sum(row["errors"] for row in rows) n_solved = sum(row["solved"] for row in rows) n_failed = (len(trials) if trials else sum(row["trials"] for row in rows)) - n_solved - n_errors - counts = (("Solved", n_solved), ("Failed", n_failed), ("Error", n_errors)) - slices = sorted(((label, count) for label, count in counts if count > 0), key=lambda item: -item[1]) + n_all = n_solved + n_failed + n_errors lines = [] - if slices: - pie_theme = { - "theme": "base", - "pie": {"useMaxWidth": False, "textPosition": 0.7}, - "themeVariables": { - **{f"pie{index}": OUTCOME_COLORS[label] for index, (label, _) in enumerate(slices, start=1)}, - "pieOpacity": "1", - "pieStrokeColor": "#ffffff", - "pieOuterStrokeWidth": "0px", - "pieTitleTextSize": "14px", - "pieTitleTextColor": HBENCH_MUTED, - "pieSectionTextSize": "12px", - "pieSectionTextColor": "#ffffff", - "pieLegendTextSize": "12px", - "pieLegendTextColor": HBENCH_MUTED, - }, - } - lines += [ - "```mermaid", - f"%%{{init: {json.dumps(pie_theme)}}}%%", - "pie showData", - " title Trials", - *[f' "{label}" : {count}' for label, count in slices], - "```", - "", - ] + if n_all: + lines += ["| Outcome | Trials | Share |", "|---|---:|---|"] + for label, count in (("Solved", n_solved), ("Failed", n_failed), ("Error", n_errors)): + cells = max(1, round(SHARE_BAR_WIDTH * count / n_all)) if count else 0 + lines.append(f"| {label} | {count} | {'█' * cells} {percent(count, n_all)} |") + lines.append("") scored = [row for row in rows if row["mean"] is not None] if scored: labels = ", ".join(f'"{chart_label(row["name"])}"' for row in scored) diff --git a/tests/test_report.py b/tests/test_report.py index 2a7f317..71075d7 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -71,15 +71,16 @@ def test_trials_list_errors_then_failures_then_solved(self): self.assertLess(failure, solved) def test_charts_show_the_trials_by_outcome_and_the_mean_reward_by_eval(self): - self.assertIn('pie showData\n title Trials\n "Solved" : 1\n "Failed" : 1\n "Error" : 1\n```', self.summary) + self.assertIn("| Solved | 1 | ███████ 33.3% |", self.summary) + self.assertIn("| Failed | 1 | ███████ 33.3% |", self.summary) + self.assertIn("| Error | 1 | ███████ 33.3% |", self.summary) + self.assertNotIn("pie showData", self.summary) self.assertIn(' x-axis ["claude-code / claude-opus-5-5 / example"]', self.summary) self.assertIn(" bar [0.500]", self.summary) - def test_charts_are_small_and_in_hbench_colours(self): - self.assertIn('"pie1": "#137d41", "pie2": "#c52c2a", "pie3": "#ad7300"', self.summary) + def test_the_bar_chart_is_small_and_in_hbench_blue(self): self.assertIn('"plotColorPalette": "#076be3"', self.summary) self.assertIn('"width": 360, "height": 220, "useMaxWidth": false', self.summary) - self.assertIn('"pie": {"useMaxWidth": false', self.summary) def test_an_uploaded_job_gets_the_command_that_opens_it_in_the_dashboard(self): summary = run_report(AGENT_JOB, artifact="hbench-bench")[2] @@ -109,9 +110,9 @@ def test_reward_txt_mean_and_no_usage(self): self.assertIn("- **Tasks** `tasks`", summary) self.assertIn("✅ At or above min-mean-reward 1.", summary) self.assertNotIn("### Errors", summary) - self.assertIn(' "Solved" : 1\n```', summary) - self.assertNotIn('"Failed"', summary) - self.assertNotIn('"Error"', summary) + self.assertIn("| Solved | 1 | ████████████████████ 100.0% |", summary) + self.assertIn("| Failed | 0 | 0.0% |", summary) + self.assertIn("| Error | 0 | 0.0% |", summary) if __name__ == "__main__": From 6afffdad093c8a3395e62536146def2c2686a1dc Mon Sep 17 00:00:00 2001 From: rzonyx <77541804+rzonyx@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:41:57 +0700 Subject: [PATCH 7/7] Show each eval's mean reward as a table with bars too Like the trials by outcome: a row per eval with its mean reward as a bar of up to 20 blocks and the number. The summary no longer needs Mermaid, so it reads the same wherever Markdown renders. --- README.md | 4 ++-- docs/REFERENCE.md | 4 ++-- scripts/report.py | 54 +++++++------------------------------------- tests/test_report.py | 10 +++----- 4 files changed, 15 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 2808fb8..5752f46 100644 --- a/README.md +++ b/README.md @@ -119,8 +119,8 @@ With none of `path`, `dataset` or `config`, the action only installs hbench, and `hbench` themselves. Useful outputs: `pass-rate`, `n-solved`, `mean-reward`, `cost-usd` and `job-dir`. The job summary shows, for each agent and model, how many tasks it solved, its mean reward and pass@k, its -errors, the time it spent, its tokens and its cost per solved task; the trials by outcome, with a bar for -each share, and a chart of each eval's mean reward; the command that opens the job in +errors, the time it spent, its tokens and its cost per solved task; the trials by outcome and each eval's +mean reward, each with a bar; the command that opens the job in hbench's interactive dashboard on your machine; what ran (the hbench release, the agent, the model, the tasks and their digest); the errors by type; and a row per task. diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index fc38ae4..3ef72aa 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -62,8 +62,8 @@ The job summary has: the agent's time in total and per trial (median), tokens in and out, and the cost in total and per solved trial; - whether the mean reward reached `min-mean-reward`; -- a table of the trials solved, failed and ended in an error, each with its share as a bar of up to 20 - blocks, and a small Mermaid bar chart of each eval's mean reward in hbench's blue; +- two tables with bars of up to 20 blocks: the trials solved, failed and ended in an error, with each + share, and each eval's mean reward; - when the job is uploaded, the command that downloads it and opens it in hbench's interactive dashboard, `gh run download -R -n -D hbench-jobs/ && hbench view hbench-jobs`; - what ran: the hbench release and results epoch, the agent and model, the tasks with their registry and diff --git a/scripts/report.py b/scripts/report.py index d582bf3..17a18e6 100644 --- a/scripts/report.py +++ b/scripts/report.py @@ -4,9 +4,8 @@ Reads hbench's result.json, and beside it the job's config.json and lock.json and each trial's result.json. A trial is solved when its reward is 1. Each eval (agent, model and tasks) gets a row: solved trials and the -pass rate, mean reward, pass@k, errors, agent time, tokens and cost. Charts follow, small and in -hbench's colours: a table of solved, failed and errored trials with a bar of each share, and a Mermaid -bar chart of each eval's mean reward. Below come what ran (hbench, agent, +pass rate, mean reward, pass@k, errors, agent time, tokens and cost. Two tables follow with bars of up to +20 blocks: the solved, failed and errored trials with each share, and each eval's mean reward. Below come what ran (hbench, agent, model, tasks and digest), the command that opens the job in hbench's dashboard when the job is uploaded (HBENCH_ARTIFACT_NAME), the tasks' own metrics, the errors by type, and a row per trial, errors and failures first. mean-reward is the trial-weighted mean of the evals' mean rewards, as hbench computes them @@ -24,9 +23,6 @@ LISTED_TRIALS = 500 PRIMARY_METRICS = ("reward", "mean") -HBENCH_ACCENT = "#076be3" -HBENCH_MUTED = "#5e646d" -HBENCH_RULE = "#dce0e6" SHARE_BAR_WIDTH = 20 @@ -290,46 +286,12 @@ def charts(rows, trials): lines.append("") scored = [row for row in rows if row["mean"] is not None] if scored: - labels = ", ".join(f'"{chart_label(row["name"])}"' for row in scored) - values = ", ".join(f"{row['mean']:.3f}" for row in scored) - axis = {"labelFontSize": 11, "titleFontSize": 12} - bar_theme = { - "theme": "base", - "xyChart": { - "width": min(900, max(360, 140 * len(scored))), - "height": 220, - "useMaxWidth": False, - "titleFontSize": 14, - "xAxis": axis, - "yAxis": axis, - }, - "themeVariables": { - "xyChart": { - "backgroundColor": "transparent", - "plotColorPalette": HBENCH_ACCENT, - "titleColor": HBENCH_MUTED, - "xAxisLabelColor": HBENCH_MUTED, - "xAxisTitleColor": HBENCH_MUTED, - "xAxisLineColor": HBENCH_RULE, - "xAxisTickColor": HBENCH_RULE, - "yAxisLabelColor": HBENCH_MUTED, - "yAxisTitleColor": HBENCH_MUTED, - "yAxisLineColor": HBENCH_RULE, - "yAxisTickColor": HBENCH_RULE, - } - }, - } - lines += [ - "```mermaid", - f"%%{{init: {json.dumps(bar_theme)}}}%%", - "xychart-beta", - ' title "Mean reward by eval"', - f" x-axis [{labels}]", - ' y-axis "Mean reward" 0 --> 1', - f" bar [{values}]", - "```", - "", - ] + lines += ["| Eval | Mean reward |", "|---|---|"] + for row in scored: + share = min(max(row["mean"], 0.0), 1.0) + cells = max(1, round(SHARE_BAR_WIDTH * share)) if share > 0 else 0 + lines.append(f"| `{chart_label(row['name'])}` | {'█' * cells} {score(row['mean'])} |") + lines.append("") return lines diff --git a/tests/test_report.py b/tests/test_report.py index 71075d7..349bff0 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -74,13 +74,8 @@ def test_charts_show_the_trials_by_outcome_and_the_mean_reward_by_eval(self): self.assertIn("| Solved | 1 | ███████ 33.3% |", self.summary) self.assertIn("| Failed | 1 | ███████ 33.3% |", self.summary) self.assertIn("| Error | 1 | ███████ 33.3% |", self.summary) - self.assertNotIn("pie showData", self.summary) - self.assertIn(' x-axis ["claude-code / claude-opus-5-5 / example"]', self.summary) - self.assertIn(" bar [0.500]", self.summary) - - def test_the_bar_chart_is_small_and_in_hbench_blue(self): - self.assertIn('"plotColorPalette": "#076be3"', self.summary) - self.assertIn('"width": 360, "height": 220, "useMaxWidth": false', self.summary) + self.assertIn("| `claude-code / claude-opus-5-5 / example` | ██████████ 0.500 |", self.summary) + self.assertNotIn("```mermaid", self.summary) def test_an_uploaded_job_gets_the_command_that_opens_it_in_the_dashboard(self): summary = run_report(AGENT_JOB, artifact="hbench-bench")[2] @@ -113,6 +108,7 @@ def test_reward_txt_mean_and_no_usage(self): self.assertIn("| Solved | 1 | ████████████████████ 100.0% |", summary) self.assertIn("| Failed | 0 | 0.0% |", summary) self.assertIn("| Error | 0 | 0.0% |", summary) + self.assertIn("| `oracle / tasks` | ████████████████████ 1.000 |", summary) if __name__ == "__main__":