{ "description": "Existing aggregate reports only; no predictions, prompts or hosted Jev results. No new evaluation was run for this post. Intervals are group-clustered bootstrap CI95, conditional on each checkpoint, not across seeds.", "source_repository": "https://github.com/nullsilver-labs/alpha-sys-1", "source_files": { "runs/cmp/metrics.json": { "sha256": "3006346d440031e5d6ced935d11c7bf95f369c9fe6d163c5e7504fb6c272144b" }, "runs/p2/gate-7env-450M.json": { "sha256": "a9200677ecf2dfb9e13dc364bd188a4ab0034f9b1c16c5b92ab23dc6fb9cc1d5" }, "runs/p2/gate-7env-1.6B.json": { "sha256": "24dadd794fd3954cb19e1c6e05a67875ef0363e35ce5822f1c2f7175cfa73aa5" }, "runs/p2/gate-7env-3B.json": { "sha256": "72139e3384c641c7abd902083fbf162e9dc70d9838ac637d24efa2003594f444" } }, "release_seeds": { "450M": 1, "1.6B": 1, "3B": 1 }, "release_revisions": { "450M": "260919", "1.6B": "260919", "3B": "260920" }, "trained_environment_baselines": { "450M": { "mcq": "LFM2.5-VL-450M +T", "chaosnli": "zs-LFM2.5-VL-450M-prefill +T", "civilcomments": "zs-LFM2.5-VL-450M-prefill +T", "stsb": "zs-LFM2.5-VL-450M-prefill +T", "folktables": "zs-LFM2.5-VL-450M-prefill +T" }, "1.6B": { "mcq": "zs-LFM2.5-VL-1.6B-prefill +T", "chaosnli": "zs-LFM2.5-VL-1.6B-prefill +T", "civilcomments": "LFM2.5-VL-1.6B +T", "stsb": "zs-LFM2.5-VL-1.6B-prefill +T", "folktables": "LFM2.5-VL-1.6B +T" }, "3B": { "mcq": "zs-LFM2.5-VL-3B-prefill +T", "chaosnli": "zs-LFM2.5-VL-3B-prefill +T", "civilcomments": "zs-LFM2.5-VL-3B-prefill +T", "stsb": "LFM2.5-VL-3B +T", "folktables": "LFM2.5-VL-3B +T" } }, "unseen_baseline_policy": "Fixed first-assistant-token LFM readout + dev-fitted temperature. Not the lowest test CE of two readouts.", "metrics": [ { "env": "boolq", "slice": "all", "system": "base rate", "metrics": { "n": { "value": 8129.0, "lo": 7984.475, "hi": 8270.575 }, "ce": { "value": 0.6646544945813517, "lo": 0.6585364928682439, "hi": 0.6720828659936188 }, "kl": { "value": 0.6646544945813517, "lo": 0.6585364928682439, "hi": 0.6720828659936188 }, "jsd": { "value": 0.20396171787752962, "lo": 0.2023067701872962, "hi": 0.20597112661708397 }, "brier": { "value": 0.4717530550578688, "lo": 0.4657881407089929, "hi": 0.47899555045156544 }, "acc": { "value": 0.6198794439660474, "lo": 0.6065282404082177, "hi": 0.6308754862619356 }, "conf": { "value": 0.635615028306742, "lo": 0.635615028306742, "hi": 0.6356150283067422 }, "ece": { "value": 0.015735584340694553, "lo": 0.004739542044806519, "hi": 0.029086787898524334 }, "reliability": { "value": 0.0002476086145431116, "lo": 2.2486951592956432e-05, "hi": 0.0008460416293660667 }, "resolution": { "value": 0.0, "lo": 0.0, "hi": 0.0 }, "uncertainty": { "value": 0.2356289189143913, "lo": 0.23287158340290345, "hi": 0.23865173359641664 }, "auroc": { "value": 0.5, "lo": 0.5, "hi": 0.5 }, "acc@80": { "value": 0.6226357065969552, "lo": 0.6052843777392773, "hi": 0.6335721802831055 } } }, { "env": "boolq", "slice": "all", "system": "laya-0.1.6", "metrics": { "n": { "value": 8129.0, "lo": 7984.475, "hi": 8270.575 }, "ce": { "value": 0.16732241000325002, "lo": 0.15388610376208772, "hi": 0.1802310830968935 }, "kl": { "value": 0.16732241000325002, "lo": 0.15388610376208772, "hi": 0.1802310830968935 }, "jsd": { "value": 0.041991452965761206, "lo": 0.039388778184086294, "hi": 0.044628730457879705 }, "brier": { "value": 0.08883623468224963, "lo": 0.08079539790469732, "hi": 0.09667981589340952 }, "acc": { "value": 0.9453807356378399, "lo": 0.9399843400447427, "hi": 0.9504083064841119 }, "conf": { "value": 0.9455707663465563, "lo": 0.9437346472223621, "hi": 0.9474041161979702 }, "ece": { "value": 0.00826415120785431, "lo": 0.006670393035826101, "hi": 0.014237934086454946 }, "reliability": { "value": 0.00015128253894199073, "lo": 7.579447670100361e-05, "hi": 0.0004981111013192051 }, "resolution": { "value": 0.0067872674046922975, "lo": 0.005359476950057658, "hi": 0.008495357238412838 }, "uncertainty": { "value": 0.0516360003226966, "lo": 0.04713235430009503, "hi": 0.056413780293513655 }, "auroc": { "value": 0.8366656995316722, "lo": 0.8175586056341942, "hi": 0.8558524171817646 }, "acc@80": { "value": 0.9781639243426111, "lo": 0.9741500219637983, "hi": 0.9818828417002816 } } }, { "env": "boolq", "slice": "all", "system": "laya-0.1.6 +T", "metrics": { "n": { "value": 8129.0, "lo": 7984.475, "hi": 8270.575 }, "ce": { "value": 0.16842074844908808, "lo": 0.15377416039538502, "hi": 0.1826377502844351 }, "kl": { "value": 0.16842074844908808, "lo": 0.15377416039538502, "hi": 0.1826377502844351 }, "jsd": { "value": 0.039848000608692, "lo": 0.03717194335215928, "hi": 0.04255754852113439 }, "brier": { "value": 0.08911969369012436, "lo": 0.0807434104771179, "hi": 0.09722725633968095 }, "acc": { "value": 0.9453807356378399, "lo": 0.9399843400447427, "hi": 0.9504083064841119 }, "conf": { "value": 0.9547045580034629, "lo": 0.9529434187871263, "hi": 0.9564472385719919 }, "ece": { "value": 0.010118889872109627, "lo": 0.008057267182127393, "hi": 0.016097098115100707 }, "reliability": { "value": 0.00029446399075773173, "lo": 0.00014625249070317913, "hi": 0.0007716198702540654 }, "resolution": { "value": 0.006787507272524246, "lo": 0.0053595468591614666, "hi": 0.008492841986105507 }, "uncertainty": { "value": 0.0516360003226966, "lo": 0.04713235430009503, "hi": 0.056413780293513655 }, "auroc": { "value": 0.836659545036253, "lo": 0.8175513786555788, "hi": 0.8558356541920423 }, "acc@80": { "value": 0.9781639243426111, "lo": 0.9741500219637983, "hi": 0.9818828417002816 } } }, { "env": "chaosnli", "slice": "all", "system": "base rate", "metrics": { "n": { "value": 2763.0, "lo": 2732.475, "hi": 2798.525 }, "ce": { "value": 0.9379373340497879, "lo": 0.9285802317593829, "hi": 0.9475822940904634 }, "kl": { "value": 0.40757726398052585, "lo": 0.39747565842988464, "hi": 0.41904554967110874 }, "jsd": { "value": 0.11304809900072296, "lo": 0.11032258933459423, "hi": 0.11615128759372625 }, "brier": { "value": 0.26981895182793947, "lo": 0.2626011497084805, "hi": 0.2773451971013373 }, "acc": { "value": 0.48968512486427795, "lo": 0.47177673148510973, "hi": 0.5089376292721363 }, "conf": { "value": 0.4905093503234308, "lo": 0.4888078368966116, "hi": 0.4923186354261798 }, "ece": { "value": 0.022808835759865427, "lo": 0.008771611429390938, "hi": 0.04095054029804827 }, "reliability": { "value": 0.0005754631118155022, "lo": 0.00011636701405239725, "hi": 0.0018846778177793791 }, "resolution": { "value": 0.0003291497616916151, "lo": 2.4129556481931592e-05, "hi": 0.0016044554135521544 }, "uncertainty": { "value": 0.24989360335093447, "lo": 0.24920342118429026, "hi": 0.24999970359945467 }, "auroc": { "value": 0.5195357833655706, "lo": 0.4991706758215456, "hi": 0.5404608926184091 }, "acc@80": { "value": 0.49366515837104075, "lo": 0.4756861233826964, "hi": 0.5150407931169876 } } }, { "env": "chaosnli", "slice": "all", "system": "laya-0.1.6", "metrics": { "n": { "value": 2763.0, "lo": 2732.475, "hi": 2798.525 }, "ce": { "value": 1.0285447436275288, "lo": 1.0033821554904991, "hi": 1.0545519423823069 }, "kl": { "value": 0.498184673558267, "lo": 0.47789097962855864, "hi": 0.51921965502745 }, "jsd": { "value": 0.10085241061940411, "lo": 0.09746306336749984, "hi": 0.10453247078825088 }, "brier": { "value": 0.2719412201157911, "lo": 0.26132247894674693, "hi": 0.2831178041460505 }, "acc": { "value": 0.6568946796959826, "lo": 0.6389596974115177, "hi": 0.6737617308149204 }, "conf": { "value": 0.7850386756747001, "lo": 0.7782867783386168, "hi": 0.7904756883129792 }, "ece": { "value": 0.12836848797463352, "lo": 0.11764994557569582, "hi": 0.1486388444337131 }, "reliability": { "value": 0.02160372386829519, "lo": 0.018122040872542048, "hi": 0.02785991771997649 }, "resolution": { "value": 0.011270827854638914, "lo": 0.009226852780802366, "hi": 0.016455992401153814 }, "uncertainty": { "value": 0.22538405948309503, "lo": 0.2198068206209398, "hi": 0.23069019528126342 }, "auroc": { "value": 0.62926793830131, "lo": 0.6093513301105226, "hi": 0.6516518175130473 }, "acc@80": { "value": 0.6909502262443439, "lo": 0.6722383500218386, "hi": 0.7085911083203138 } } }, { "env": "chaosnli", "slice": "all", "system": "laya-0.1.6 +T", "metrics": { "n": { "value": 2763.0, "lo": 2732.475, "hi": 2798.525 }, "ce": { "value": 0.8251350047631987, "lo": 0.8152288405803177, "hi": 0.835546065711284 }, "kl": { "value": 0.29477493469393684, "lo": 0.2862985778747161, "hi": 0.3033341949032095 }, "jsd": { "value": 0.08114470031563259, "lo": 0.07900899539493031, "hi": 0.08340997116660731 }, "brier": { "value": 0.19441446983318122, "lo": 0.18809944062540326, "hi": 0.20078332541152275 }, "acc": { "value": 0.6568946796959826, "lo": 0.6389596974115177, "hi": 0.6737617308149204 }, "conf": { "value": 0.5986317700597011, "lo": 0.595133404666914, "hi": 0.6016171878298637 }, "ece": { "value": 0.0627802665034211, "lo": 0.05626098050792744, "hi": 0.08661036301274243 }, "reliability": { "value": 0.005811743129342419, "lo": 0.004608585782396878, "hi": 0.010239545437114193 }, "resolution": { "value": 0.01383844026256437, "lo": 0.01116274735607004, "hi": 0.019402317467878596 }, "uncertainty": { "value": 0.22538405948309503, "lo": 0.2198068206209398, "hi": 0.23069019528126342 }, "auroc": { "value": 0.6400919436017249, "lo": 0.6191042100717233, "hi": 0.6627523607997489 }, "acc@80": { "value": 0.702262443438914, "lo": 0.683554107513062, "hi": 0.7210812093966812 } } }, { "env": "civilcomments", "slice": "all", "system": "base rate", "metrics": { "n": { "value": 10581.0, "lo": 10261.425, "hi": 10904.525 }, "ce": { "value": 0.4254538278813023, "lo": 0.4167216532262873, "hi": 0.43439171663848053 }, "kl": { "value": 0.1939710191119612, "lo": 0.1898765552212037, "hi": 0.19867000295695353 }, "jsd": { "value": 0.05293960990922591, "lo": 0.05205862775499703, "hi": 0.05401522441712282 }, "brier": { "value": 0.10156534944654734, "lo": 0.09792364241321323, "hi": 0.10541567724997143 }, "acc": { "value": 0.9116340610528305, "lo": 0.906076390941178, "hi": 0.9167691033774958 }, "conf": { "value": 0.8593235781658778, "lo": 0.8593235781658777, "hi": 0.859323578165878 }, "ece": { "value": 0.052310482886952725, "lo": 0.046752812775300176, "hi": 0.05744552521161804 }, "reliability": { "value": 0.002736386619866174, "lo": 0.0021858255137981748, "hi": 0.0032999884123147837 }, "resolution": { "value": 0.0, "lo": 0.0, "hi": 0.0 }, "uncertainty": { "value": 0.0805573997811546, "lo": 0.07630351442444205, "hi": 0.08510196470879164 }, "auroc": { "value": 0.5, "lo": 0.5, "hi": 0.5 }, "acc@80": { "value": 0.9141169521559362, "lo": 0.9055380777445347, "hi": 0.9173038439837854 } } }, { "env": "civilcomments", "slice": "all", "system": "laya-0.1.6", "metrics": { "n": { "value": 10581.0, "lo": 10261.425, "hi": 10904.525 }, "ce": { "value": 0.3810125971120588, "lo": 0.37484793949536976, "hi": 0.3860455755495298 }, "kl": { "value": 0.14952978834271774, "lo": 0.14694071031932254, "hi": 0.15214002028455306 }, "jsd": { "value": 0.048026748840901134, "lo": 0.04717718855909775, "hi": 0.04884049072928548 }, "brier": { "value": 0.07264455665626256, "lo": 0.07091948864137936, "hi": 0.07446755140854741 }, "acc": { "value": 0.9363954257631604, "lo": 0.9312894156481802, "hi": 0.9412408224641303 }, "conf": { "value": 0.7462928145292993, "lo": 0.7432110057199227, "hi": 0.7495198525874994 }, "ece": { "value": 0.19010261123386113, "lo": 0.18572869743920553, "hi": 0.19434341943002456 }, "reliability": { "value": 0.042625286127443054, "lo": 0.04090385585377438, "hi": 0.044403189351862014 }, "resolution": { "value": 0.013717049821998603, "lo": 0.011881205041954722, "hi": 0.01598063638905128 }, "uncertainty": { "value": 0.059559032372989984, "lo": 0.05530653638439067, "hi": 0.06398943971092431 }, "auroc": { "value": 0.9171025739927692, "lo": 0.9087929385965824, "hi": 0.9240171107172537 }, "acc@80": { "value": 0.9878322504430006, "lo": 0.9847184534863704, "hi": 0.9904090341015473 } } }, { "env": "civilcomments", "slice": "all", "system": "laya-0.1.6 +T", "metrics": { "n": { "value": 10581.0, "lo": 10261.425, "hi": 10904.525 }, "ce": { "value": 0.31924507592150614, "lo": 0.3115205115811762, "hi": 0.32654836174716545 }, "kl": { "value": 0.08776226715216504, "lo": 0.08515573370008006, "hi": 0.09058161011803512 }, "jsd": { "value": 0.024394595342128112, "lo": 0.023688361737283142, "hi": 0.025138113681980403 }, "brier": { "value": 0.04172181358871891, "lo": 0.040081896395935374, "hi": 0.04352301107833134 }, "acc": { "value": 0.9363954257631604, "lo": 0.9312894156481802, "hi": 0.9412408224641303 }, "conf": { "value": 0.8608904153831163, "lo": 0.8574604423635073, "hi": 0.8642629707355846 }, "ece": { "value": 0.07550501038004401, "lo": 0.0718289183674466, "hi": 0.07971665523074521 }, "reliability": { "value": 0.009249599786670091, "lo": 0.00825301093897232, "hi": 0.010361731443868353 }, "resolution": { "value": 0.013717049821998603, "lo": 0.011852341507289721, "hi": 0.01598063638905128 }, "uncertainty": { "value": 0.059559032372989984, "lo": 0.05530653638439067, "hi": 0.06398943971092431 }, "auroc": { "value": 0.9170985248536161, "lo": 0.9087881215216295, "hi": 0.9240207230291888 }, "acc@80": { "value": 0.9878322504430006, "lo": 0.9847184534863704, "hi": 0.9904090341015473 } } }, { "env": "folktables", "slice": "all", "system": "base rate", "metrics": { "n": { "value": 15003.0, "lo": 14855.8, "hi": 15157.0 }, "ce": { "value": 0.6832870834397348, "lo": 0.6811854548087254, "hi": 0.6856621653799928 }, "kl": { "value": 0.6832870834397348, "lo": 0.6811854548087254, "hi": 0.6856621653799928 }, "jsd": { "value": 0.21218818553265442, "lo": 0.211602993626582, "hi": 0.21284537334357836 }, "brier": { "value": 0.49029879579639624, "lo": 0.4882531508828007, "hi": 0.4926056356460426 }, "acc": { "value": 0.6536026128107711, "lo": 0.6451184668229782, "hi": 0.661609588316768 }, "conf": { "value": 0.5285609544757716, "lo": 0.5272976034555974, "hi": 0.5298136221174654 }, "ece": { "value": 0.12504165833499967, "lo": 0.11694450254962907, "hi": 0.13293473708420933 }, "reliability": { "value": 0.018745024303201244, "lo": 0.016591111933824162, "hi": 0.02095019662130334 }, "resolution": { "value": 1.863742707408628e-06, "lo": 1.2375750882431937e-08, "hi": 9.316471375822222e-05 }, "uncertainty": { "value": 0.22640623733770432, "lo": 0.22388234090058073, "hi": 0.2289406268460248 }, "auroc": { "value": 0.5012058742372862, "lo": 0.49426236500946596, "hi": 0.5081097296397117 }, "acc@80": { "value": 0.6346442259623396, "lo": 0.64362479180161, "hi": 0.6630202666626369 } } }, { "env": "folktables", "slice": "all", "system": "laya-0.1.6", "metrics": { "n": { "value": 15003.0, "lo": 14855.8, "hi": 15157.0 }, "ce": { "value": 0.720976727971998, "lo": 0.7187275568573948, "hi": 0.7231376246748215 }, "kl": { "value": 0.720976727971998, "lo": 0.7187275568573948, "hi": 0.7231376246748215 }, "jsd": { "value": 0.22245330451912865, "lo": 0.22185146115893456, "hi": 0.22303308672417152 }, "brier": { "value": 0.5273739882830246, "lo": 0.5251673418866659, "hi": 0.5294821933633662 }, "acc": { "value": 0.4391121775644871, "lo": 0.4309876348072268, "hi": 0.4470842753273963 }, "conf": { "value": 0.5516842689218898, "lo": 0.5510766443317116, "hi": 0.5522637898698728 }, "ece": { "value": 0.11439187478790329, "lo": 0.10666349685369207, "hi": 0.12235665432272713 }, "reliability": { "value": 0.019709030457048863, "lo": 0.017700443867459795, "hi": 0.022151985907463968 }, "resolution": { "value": 0.002366054834422019, "lo": 0.001883000473039606, "hi": 0.003348470989642238 }, "uncertainty": { "value": 0.24629267307906147, "lo": 0.24523729330617874, "hi": 0.24719992563234872 }, "auroc": { "value": 0.44683860462548985, "lo": 0.43852322856093934, "hi": 0.4566225509724676 }, "acc@80": { "value": 0.4237627062156307, "lo": 0.4155537377531345, "hi": 0.43319309264175665 } } }, { "env": "folktables", "slice": "all", "system": "laya-0.1.6 +T", "metrics": { "n": { "value": 15003.0, "lo": 14855.8, "hi": 15157.0 }, "ce": { "value": 0.6932795192425513, "lo": 0.6932643598725224, "hi": 0.6932936000889979 }, "kl": { "value": 0.6932795192425513, "lo": 0.6932643598725224, "hi": 0.6932936000889979 }, "jsd": { "value": 0.21579786082603852, "lo": 0.21579369688263556, "hi": 0.21580172783369722 }, "brier": { "value": 0.500132338595746, "lo": 0.5001171792366494, "hi": 0.5001464194336274 }, "acc": { "value": 0.4391121775644871, "lo": 0.4309876348072268, "hi": 0.4470842753273963 }, "conf": { "value": 0.5003518570537647, "lo": 0.5003476356222193, "hi": 0.500355839523348 }, "ece": { "value": 0.06343668058765774, "lo": 0.05648981460136645, "hi": 0.0717494721729754 }, "reliability": { "value": 0.00613941541133633, "lo": 0.005200797478803559, "hi": 0.007625049705925511 }, "resolution": { "value": 0.0023660704975580994, "lo": 0.0018832496360224036, "hi": 0.0033462913468053673 }, "uncertainty": { "value": 0.24629267307906147, "lo": 0.24523729330617874, "hi": 0.24719992563234872 }, "auroc": { "value": 0.44683114584539635, "lo": 0.4385128537020856, "hi": 0.45661674335469843 }, "acc@80": { "value": 0.42367938676887185, "lo": 0.4155120588620882, "hi": 0.43323697795480515 } } }, { "env": "mcq", "slice": "all", "system": "base rate", "metrics": { "n": { "value": 5064.0, "lo": 5064.0, "hi": 5064.0 }, "ce": { "value": 1.4388183798049623, "lo": 1.4361334266007217, "hi": 1.4412926929165724 }, "kl": { "value": 1.4388183798049623, "lo": 1.4361334266007217, "hi": 1.4412926929165724 }, "jsd": { "value": 0.39036904499300623, "lo": 0.3898617116314181, "hi": 0.39084013172225024 }, "brier": { "value": 0.7617430226434967, "lo": 0.7611488776987889, "hi": 0.7623009643233282 }, "acc": { "value": 0.2401263823064771, "lo": 0.228870458135861, "hi": 0.2513921800947867 }, "conf": { "value": 0.23825697735650342, "lo": 0.23769903567667192, "hi": 0.23885112230121117 }, "ece": { "value": 0.0018694049499736714, "lo": 0.0012241640337019553, "hi": 0.014866870721432324 }, "reliability": { "value": 8.073839816032738e-05, "lo": 1.2232715420221261e-05, "hi": 0.0005723834895640047 }, "resolution": { "value": 0.0006050685727678989, "lo": 0.0001906606623764816, "hi": 0.0014383353490246106 }, "uncertainty": { "value": 0.1824657028268807, "lo": 0.17648877152854212, "hi": 0.18819411298413483 }, "auroc": { "value": 0.5260262080780173, "lo": 0.5131244477517856, "hi": 0.5401024591295606 }, "acc@80": { "value": 0.2512959763021476, "lo": 0.23684892619106396, "hi": 0.26289804986423104 } } }, { "env": "mcq", "slice": "all", "system": "laya-0.1.6", "metrics": { "n": { "value": 5064.0, "lo": 5064.0, "hi": 5064.0 }, "ce": { "value": 1.1993251171399195, "lo": 1.176848157016128, "hi": 1.2253531746697932 }, "kl": { "value": 1.1993251171399195, "lo": 1.176848157016128, "hi": 1.2253531746697932 }, "jsd": { "value": 0.3027824241850154, "lo": 0.298267374595891, "hi": 0.30751843367633935 }, "brier": { "value": 0.6312290009042739, "lo": 0.6193712176952356, "hi": 0.6439454266304352 }, "acc": { "value": 0.5078988941548184, "lo": 0.4945645734597156, "hi": 0.5199447077409163 }, "conf": { "value": 0.5662472816198876, "lo": 0.5608019186667127, "hi": 0.5720023136665401 }, "ece": { "value": 0.06579449038075703, "lo": 0.058592720388318593, "hi": 0.08158590762440474 }, "reliability": { "value": 0.007559025677366429, "lo": 0.0060630002870996486, "hi": 0.01051744168424885 }, "resolution": { "value": 0.022468520685816, "lo": 0.019473980735299188, "hi": 0.026768319076778915 }, "uncertainty": { "value": 0.24993760747113097, "lo": 0.24960220863312943, "hi": 0.24999996100466945 }, "auroc": { "value": 0.667545944846214, "lo": 0.6533370729818624, "hi": 0.6822790215851574 }, "acc@80": { "value": 0.5573932362379659, "lo": 0.5412182177240187, "hi": 0.5717106887188349 } } }, { "env": "mcq", "slice": "all", "system": "laya-0.1.6 +T", "metrics": { "n": { "value": 5064.0, "lo": 5064.0, "hi": 5064.0 }, "ce": { "value": 1.1787822993133397, "lo": 1.1625312940057417, "hi": 1.1965745610113971 }, "kl": { "value": 1.1787822993133397, "lo": 1.1625312940057417, "hi": 1.1965745610113971 }, "jsd": { "value": 0.31641723619898476, "lo": 0.31287769510313346, "hi": 0.32002992408234404 }, "brier": { "value": 0.6249379778249716, "lo": 0.6157089328560164, "hi": 0.6343573188963025 }, "acc": { "value": 0.5078988941548184, "lo": 0.4945645734597156, "hi": 0.5199447077409163 }, "conf": { "value": 0.47546104819028784, "lo": 0.4711100074624925, "hi": 0.4799136990918725 }, "ece": { "value": 0.0418016715069125, "lo": 0.034856876299361336, "hi": 0.056485500357719466 }, "reliability": { "value": 0.0024197159470087408, "lo": 0.001863533060433718, "hi": 0.0045085865560292124 }, "resolution": { "value": 0.02228669792844079, "lo": 0.019275008490941038, "hi": 0.026687375477648448 }, "uncertainty": { "value": 0.24993760747113097, "lo": 0.24960220863312943, "hi": 0.24999996100466945 }, "auroc": { "value": 0.6674585735005205, "lo": 0.6531285905081086, "hi": 0.6821516095991993 }, "acc@80": { "value": 0.5568995309800049, "lo": 0.5412182177240187, "hi": 0.5697358676869909 } } }, { "env": "stsb", "slice": "all", "system": "base rate", "metrics": { "n": { "value": 3676.0, "lo": 3150.95, "hi": 4793.674999999996 }, "ce": { "value": 1.7269663984515509, "lo": 1.705291022662975, "hi": 1.7506263143965053 }, "kl": { "value": 1.3246625755892953, "lo": 1.2976795900285005, "hi": 1.3546742057651315 }, "jsd": { "value": 0.35163188140152485, "lo": 0.345662174043538, "hi": 0.3583488621411241 }, "brier": { "value": 0.537933427501715, "lo": 0.5253465273948369, "hi": 0.5509865061000444 }, "acc": { "value": 0.24183895538628944, "lo": 0.2057694485131953, "hi": 0.27246244499642425 }, "conf": { "value": 0.23596666484615023, "lo": 0.2314976047091242, "hi": 0.23947197861216712 }, "ece": { "value": 0.022163881119620442, "lo": 0.009124136796962445, "hi": 0.04684933352937424 }, "reliability": { "value": 0.0004970487369445307, "lo": 0.0001251405423291301, "hi": 0.00260283556603496 }, "resolution": { "value": 0.001650002222171168, "lo": 0.0001834599765092166, "hi": 0.00383292294125337 }, "uncertainty": { "value": 0.18335287504395775, "lo": 0.16342811435849147, "hi": 0.19822650163650427 }, "auroc": { "value": 0.5479298268556043, "lo": 0.515657963419467, "hi": 0.5753557516756425 }, "acc@80": { "value": 0.2471948316899014, "lo": 0.20855173667103932, "hi": 0.2797579046695503 } } }, { "env": "stsb", "slice": "all", "system": "laya-0.1.6", "metrics": { "n": { "value": 3676.0, "lo": 3150.95, "hi": 4793.674999999996 }, "ce": { "value": 1.6736087922378693, "lo": 1.6454919880841663, "hi": 1.7003488444969814 }, "kl": { "value": 1.2713049693756135, "lo": 1.2456231458092684, "hi": 1.3005210492731487 }, "jsd": { "value": 0.31901269603382404, "lo": 0.3090945377346262, "hi": 0.32777996558039413 }, "brier": { "value": 0.5437303126659617, "lo": 0.5326542358620336, "hi": 0.5563186476397121 }, "acc": { "value": 0.28917301414581065, "lo": 0.2619405832349143, "hi": 0.3228819749910248 }, "conf": { "value": 0.45360173306897106, "lo": 0.43005428967076675, "hi": 0.486999864881288 }, "ece": { "value": 0.16442871892316038, "lo": 0.15040931743295344, "hi": 0.17809551965658568 }, "reliability": { "value": 0.0333625174345907, "lo": 0.02894269456847073, "hi": 0.040629612785721775 }, "resolution": { "value": 0.016751470987512153, "lo": 0.012073706086188617, "hi": 0.02212108688598159 }, "uncertainty": { "value": 0.20555198203563743, "lo": 0.1933277138691961, "hi": 0.21862919761213467 }, "auroc": { "value": 0.6473404019773771, "lo": 0.6229505005582147, "hi": 0.6655828886182101 }, "acc@80": { "value": 0.3138388303298198, "lo": 0.2825246786719, "hi": 0.35257716166005415 } } }, { "env": "stsb", "slice": "all", "system": "laya-0.1.6 +T", "metrics": { "n": { "value": 3676.0, "lo": 3150.95, "hi": 4793.674999999996 }, "ce": { "value": 1.6062735114762483, "lo": 1.5746073747409106, "hi": 1.6321932664093877 }, "kl": { "value": 1.2039696886139928, "lo": 1.1755094929077654, "hi": 1.2319700579142734 }, "jsd": { "value": 0.3226950954777939, "lo": 0.31440668412059836, "hi": 0.32976379597199573 }, "brier": { "value": 0.5022439281949195, "lo": 0.49011437005747943, "hi": 0.5151003741658788 }, "acc": { "value": 0.28917301414581065, "lo": 0.2619405832349143, "hi": 0.3228819749910248 }, "conf": { "value": 0.3376014139837463, "lo": 0.3221646652656687, "hi": 0.3595460530741712 }, "ece": { "value": 0.04980471678034271, "lo": 0.04039839900599371, "hi": 0.06840069816459508 }, "reliability": { "value": 0.0032059262837748044, "lo": 0.00261795207411194, "hi": 0.006214947101106022 }, "resolution": { "value": 0.0168465680778093, "lo": 0.012226649179908476, "hi": 0.02242040477662038 }, "uncertainty": { "value": 0.20555198203563743, "lo": 0.1933277138691961, "hi": 0.21862919761213467 }, "auroc": { "value": 0.6481324472506849, "lo": 0.6241612697813905, "hi": 0.6662312023612728 }, "acc@80": { "value": 0.31553893233594016, "lo": 0.28261010446385143, "hi": 0.35400641765209856 } } }, { "env": "yelp", "slice": "all", "system": "base rate", "metrics": { "n": { "value": 5000.0, "lo": 5000.0, "hi": 5000.0 }, "ce": { "value": 1.6094379124340998, "lo": 1.6094379124340998, "hi": 1.6094379124340998 }, "kl": { "value": 1.6094379124340998, "lo": 1.6094379124340998, "hi": 1.6094379124340998 }, "jsd": { "value": 0.4228104552401622, "lo": 0.4228104552401622, "hi": 0.4228104552401622 }, "brier": { "value": 0.8000000000000004, "lo": 0.8000000000000003, "hi": 0.8000000000000004 }, "acc": { "value": 0.2, "lo": 0.18939, "hi": 0.211505 }, "conf": { "value": 0.19999999999999998, "lo": 0.19999999999999998, "hi": 0.19999999999999998 }, "ece": { "value": 2.7755575615628914e-17, "lo": 0.0003999999999999837, "hi": 0.012600000000000028 }, "reliability": { "value": 7.703719777548943e-34, "lo": 1.5999999999998695e-07, "hi": 0.0001587600000000007 }, "resolution": { "value": 0.0, "lo": 0.0, "hi": 0.0 }, "uncertainty": { "value": 0.16000000000000003, "lo": 0.15352138799999998, "hi": 0.166770625 }, "auroc": { "value": 0.5, "lo": 0.5, "hi": 0.5 }, "acc@80": { "value": 0.25, "lo": 0.18761875, "hi": 0.2127625 } } }, { "env": "yelp", "slice": "all", "system": "laya-0.1.6", "metrics": { "n": { "value": 5000.0, "lo": 5000.0, "hi": 5000.0 }, "ce": { "value": 1.1845529684192286, "lo": 1.1431186098319637, "hi": 1.221813557092446 }, "kl": { "value": 1.1845529684192286, "lo": 1.1431186098319637, "hi": 1.221813557092446 }, "jsd": { "value": 0.2648056976124352, "lo": 0.25859518272782595, "hi": 0.27073400111055734 }, "brier": { "value": 0.5980090902700684, "lo": 0.5806929323199541, "hi": 0.6144723507243347 }, "acc": { "value": 0.5624, "lo": 0.548095, "hi": 0.576505 }, "conf": { "value": 0.6928905481040478, "lo": 0.6877329366482795, "hi": 0.6983667400699853 }, "ece": { "value": 0.13049054810404778, "lo": 0.1181634066042304, "hi": 0.14539070894256234 }, "reliability": { "value": 0.021255738396660482, "lo": 0.01769159733360709, "hi": 0.026098285027633436 }, "resolution": { "value": 0.02356813782596968, "lo": 0.020821619069272403, "hi": 0.02775570260559374 }, "uncertainty": { "value": 0.24610624, "lo": 0.24414697500000002, "hi": 0.247686861 }, "auroc": { "value": 0.6714477454939786, "lo": 0.6578699212962908, "hi": 0.6860522557179326 }, "acc@80": { "value": 0.6115, "lo": 0.59625, "hi": 0.6278812500000001 } } }, { "env": "yelp", "slice": "all", "system": "laya-0.1.6 +T", "metrics": { "n": { "value": 5000.0, "lo": 5000.0, "hi": 5000.0 }, "ce": { "value": 1.0798171229857636, "lo": 1.0550082317900924, "hi": 1.1030971551084312 }, "kl": { "value": 1.0798171229857636, "lo": 1.0550082317900924, "hi": 1.1030971551084312 }, "jsd": { "value": 0.28105126217439047, "lo": 0.27631426430754846, "hi": 0.2855167871962946 }, "brier": { "value": 0.5691607375831819, "lo": 0.5563139269221727, "hi": 0.5804567829613643 }, "acc": { "value": 0.5624, "lo": 0.548095, "hi": 0.576505 }, "conf": { "value": 0.5649238940779658, "lo": 0.5602919850396543, "hi": 0.569808322521585 }, "ece": { "value": 0.023761205308505857, "lo": 0.022986379472374863, "hi": 0.043442903484244774 }, "reliability": { "value": 0.0008859507295626262, "lo": 0.0008034458816503666, "hi": 0.0028251749879963177 }, "resolution": { "value": 0.0240286258589128, "lo": 0.02139084002659621, "hi": 0.02833310805383938 }, "uncertainty": { "value": 0.24610624, "lo": 0.24414697500000002, "hi": 0.247686861 }, "auroc": { "value": 0.673008209787773, "lo": 0.6592879328838852, "hi": 0.6876161819228981 }, "acc@80": { "value": 0.60925, "lo": 0.59436875, "hi": 0.6258812500000001 } } }, { "env": "boolq", "slice": "all", "system": "qwen3.8-27b", "metrics": { "n": { "value": 8129.0, "lo": 7984.475, "hi": 8270.575 }, "ce": { "value": 0.38917437609910194, "lo": 0.36478176193977996, "hi": 0.4117113811120201 }, "kl": { "value": 0.38917437609910194, "lo": 0.36478176193977996, "hi": 0.4117113811120201 }, "jsd": { "value": 0.07719183345624069, "lo": 0.07339144015470589, "hi": 0.08092392101720004 }, "brier": { "value": 0.20377173136231924, "lo": 0.19210978848275312, "hi": 0.21488258715437442 }, "acc": { "value": 0.8737852134333867, "lo": 0.8662839492571259, "hi": 0.8814999082101197 }, "conf": { "value": 0.9438369756317792, "lo": 0.9414265075098082, "hi": 0.9461442786298321 }, "ece": { "value": 0.07005176219839243, "lo": 0.06288961645106496, "hi": 0.07734588351195741 }, "reliability": { "value": 0.007471879521472837, "lo": 0.006190243666681738, "hi": 0.00926393843997429 }, "resolution": { "value": 0.015923177972556827, "lo": 0.013562333586149574, "hi": 0.018614559575419718 }, "uncertainty": { "value": 0.11028461421855758, "lo": 0.10445781851909179, "hi": 0.11583606612118956 }, "auroc": { "value": 0.7923272680269353, "lo": 0.7754385655654847, "hi": 0.806692445445645 }, "acc@80": { "value": 0.9298785176072581, "lo": 0.9236048483392075, "hi": 0.9361573239432275 } } }, { "env": "boolq", "slice": "all", "system": "qwen3.8-27b +T", "metrics": { "n": { "value": 8129.0, "lo": 7984.475, "hi": 8270.575 }, "ce": { "value": 0.31589675014113305, "lo": 0.3022209294312514, "hi": 0.3288799637631647 }, "kl": { "value": 0.31589675014113305, "lo": 0.3022209294312514, "hi": 0.3288799637631647 }, "jsd": { "value": 0.08575448318811, "lo": 0.08262998191879009, "hi": 0.08875348138449343 }, "brier": { "value": 0.1893347868894183, "lo": 0.1799530596178881, "hi": 0.19822644898122355 }, "acc": { "value": 0.8737852134333867, "lo": 0.8662839492571259, "hi": 0.8814999082101197 }, "conf": { "value": 0.8793572729627128, "lo": 0.876571291263446, "hi": 0.882064924721078 }, "ece": { "value": 0.015191298573106632, "lo": 0.01243495301594864, "hi": 0.022962742675946134 }, "reliability": { "value": 0.0003958249808374941, "lo": 0.00024787725473796393, "hi": 0.0009551273630832529 }, "resolution": { "value": 0.015923177972556827, "lo": 0.013562333586149574, "hi": 0.018614559575419718 }, "uncertainty": { "value": 0.11028461421855758, "lo": 0.10445781851909179, "hi": 0.11583606612118956 }, "auroc": { "value": 0.7923272680269353, "lo": 0.7754385655654847, "hi": 0.806692445445645 }, "acc@80": { "value": 0.9298785176072581, "lo": 0.9236048483392075, "hi": 0.9361573239432275 } } }, { "env": "chaosnli", "slice": "all", "system": "qwen3.8-27b", "metrics": { "n": { "value": 2763.0, "lo": 2732.475, "hi": 2798.525 }, "ce": { "value": 1.1717923312186944, "lo": 1.138586864694005, "hi": 1.20638100834521 }, "kl": { "value": 0.6414322611494326, "lo": 0.6127829502553543, "hi": 0.672345413595422 }, "jsd": { "value": 0.07691632010275923, "lo": 0.07386184330292474, "hi": 0.0798222751056858 }, "brier": { "value": 0.18196941143197648, "lo": 0.1719848329072861, "hi": 0.1914884979845514 }, "acc": { "value": 0.7604053564965617, "lo": 0.7432688129163535, "hi": 0.7773603885998843 }, "conf": { "value": 0.8891980318490847, "lo": 0.8833185849628782, "hi": 0.8948141919312281 }, "ece": { "value": 0.12880139700185403, "lo": 0.11318601239529825, "hi": 0.14434011515062262 }, "reliability": { "value": 0.022344589657931863, "lo": 0.017880110274394165, "hi": 0.02871049996449803 }, "resolution": { "value": 0.03430066836980699, "lo": 0.029370157179605578, "hi": 0.04077879134663945 }, "uncertainty": { "value": 0.18218905030789861, "lo": 0.17307121157914737, "hi": 0.1908196954129349 }, "auroc": { "value": 0.7887252653390487, "lo": 0.7712352266239298, "hi": 0.8048739902785704 }, "acc@80": { "value": 0.8307692307692308, "lo": 0.8124315473649547, "hi": 0.848554618482514 } } }, { "env": "chaosnli", "slice": "all", "system": "qwen3.8-27b +T", "metrics": { "n": { "value": 2763.0, "lo": 2732.475, "hi": 2798.525 }, "ce": { "value": 0.7060486628596031, "lo": 0.6928300845033976, "hi": 0.7187495942850437 }, "kl": { "value": 0.1756885927903413, "lo": 0.16822572806269168, "hi": 0.18281272188519052 }, "jsd": { "value": 0.04626507343288381, "lo": 0.04436686621685745, "hi": 0.047964706815352545 }, "brier": { "value": 0.09934163650724467, "lo": 0.09412590074193818, "hi": 0.1043263450093202 }, "acc": { "value": 0.7604053564965617, "lo": 0.7432688129163535, "hi": 0.7773603885998843 }, "conf": { "value": 0.6842320865266962, "lo": 0.6781952964143029, "hi": 0.6904375285745871 }, "ece": { "value": 0.07858697628390005, "lo": 0.06939970982409929, "hi": 0.09493126035293394 }, "reliability": { "value": 0.007777248597400178, "lo": 0.0062565383718330165, "hi": 0.011154558191824553 }, "resolution": { "value": 0.03757447751388948, "lo": 0.03259328935851462, "hi": 0.04455272036935069 }, "uncertainty": { "value": 0.18218905030789861, "lo": 0.17307121157914737, "hi": 0.1908196954129349 }, "auroc": { "value": 0.801060781012063, "lo": 0.7834037982866692, "hi": 0.8174479544838923 }, "acc@80": { "value": 0.8380090497737557, "lo": 0.819881352229595, "hi": 0.8555053007886539 } } }, { "env": "civilcomments", "slice": "all", "system": "qwen3.8-27b", "metrics": { "n": { "value": 10581.0, "lo": 10261.425, "hi": 10904.525 }, "ce": { "value": 0.5176786123346133, "lo": 0.505453019906735, "hi": 0.530276195831063 }, "kl": { "value": 0.28619580356527224, "lo": 0.27642535835701254, "hi": 0.29615644767286525 }, "jsd": { "value": 0.06800278814294652, "lo": 0.06598507291823116, "hi": 0.07004084242617473 }, "brier": { "value": 0.17101015392468566, "lo": 0.1643534221508666, "hi": 0.17728369471419955 }, "acc": { "value": 0.8048388621113316, "lo": 0.7953190817461367, "hi": 0.8132878036263456 }, "conf": { "value": 0.841425401683969, "lo": 0.8382532675599482, "hi": 0.8447274094378437 }, "ece": { "value": 0.040925964535554055, "lo": 0.036076279908205756, "hi": 0.04868327493816005 }, "reliability": { "value": 0.0031079957315440234, "lo": 0.002517658730804907, "hi": 0.004257526650648568 }, "resolution": { "value": 0.019630506626700843, "lo": 0.01802207771597479, "hi": 0.021900322431707055 }, "uncertainty": { "value": 0.15707326814666855, "lo": 0.15185071675929868, "hi": 0.16278661948606002 }, "auroc": { "value": 0.7538875689913418, "lo": 0.7437286645947476, "hi": 0.763803056702711 }, "acc@80": { "value": 0.8555227406969876, "lo": 0.8464496598759651, "hi": 0.8634669478649846 } } }, { "env": "civilcomments", "slice": "all", "system": "qwen3.8-27b +T", "metrics": { "n": { "value": 10581.0, "lo": 10261.425, "hi": 10904.525 }, "ce": { "value": 0.47297702112235573, "lo": 0.46389071412431704, "hi": 0.48134516777486236 }, "kl": { "value": 0.24149421235301474, "lo": 0.2352554985703877, "hi": 0.24806821852400143 }, "jsd": { "value": 0.0695681637074148, "lo": 0.06792337761642556, "hi": 0.07123778931909612 }, "brier": { "value": 0.15797512535439767, "lo": 0.15300866671717778, "hi": 0.16292719893400803 }, "acc": { "value": 0.8048388621113316, "lo": 0.7953190817461367, "hi": 0.8132878036263456 }, "conf": { "value": 0.7659403516461888, "lo": 0.7629512527726343, "hi": 0.7694311478600653 }, "ece": { "value": 0.03889851046514287, "lo": 0.032308723391677986, "hi": 0.04674378130222563 }, "reliability": { "value": 0.002204465811645611, "lo": 0.001631123972190602, "hi": 0.003318884561341544 }, "resolution": { "value": 0.019630506626700843, "lo": 0.01802207771597479, "hi": 0.021900322431707055 }, "uncertainty": { "value": 0.15707326814666855, "lo": 0.15185071675929868, "hi": 0.16278661948606002 }, "auroc": { "value": 0.7538875974237925, "lo": 0.7437286645947476, "hi": 0.7638030715603317 }, "acc@80": { "value": 0.8555227406969876, "lo": 0.8464496598759651, "hi": 0.8634669478649846 } } }, { "env": "folktables", "slice": "all", "system": "qwen3.8-27b", "metrics": { "n": { "value": 15003.0, "lo": 14855.8, "hi": 15157.0 }, "ce": { "value": 0.5152170611479078, "lo": 0.5013345977707182, "hi": 0.5271323194698895 }, "kl": { "value": 0.5152170611479078, "lo": 0.5013345977707182, "hi": 0.5271323194698895 }, "jsd": { "value": 0.13197355737727537, "lo": 0.1292066427756764, "hi": 0.13466268637823997 }, "brier": { "value": 0.3300800164591847, "lo": 0.32136915186904, "hi": 0.3379782638183155 }, "acc": { "value": 0.7638472305538893, "lo": 0.756615411131695, "hi": 0.7708374099107584 }, "conf": { "value": 0.8475244302376784, "lo": 0.8447128637395995, "hi": 0.8499934222016116 }, "ece": { "value": 0.0836771996837894, "lo": 0.07735320245887115, "hi": 0.09023618321654706 }, "reliability": { "value": 0.008968703120597428, "lo": 0.007738443309191269, "hi": 0.010453365822199101 }, "resolution": { "value": 0.024364708249844, "lo": 0.02252781376785893, "hi": 0.026466105696258707 }, "uncertainty": { "value": 0.1803846389290428, "lo": 0.17664708837189344, "hi": 0.18414851315321487 }, "auroc": { "value": 0.7486387015864431, "lo": 0.7404284055591798, "hi": 0.7573377305552835 }, "acc@80": { "value": 0.8189468421929679, "lo": 0.8115530142839764, "hi": 0.8258698916099144 } } }, { "env": "folktables", "slice": "all", "system": "qwen3.8-27b +T", "metrics": { "n": { "value": 15003.0, "lo": 14855.8, "hi": 15157.0 }, "ce": { "value": 0.4718461850593996, "lo": 0.46285303778129794, "hi": 0.47994812655713986 }, "kl": { "value": 0.4718461850593996, "lo": 0.46285303778129794, "hi": 0.47994812655713986 }, "jsd": { "value": 0.13745501746238115, "lo": 0.13529276541690255, "hi": 0.13957112105282737 }, "brier": { "value": 0.3131132197819545, "lo": 0.3064709953074027, "hi": 0.3195434958143069 }, "acc": { "value": 0.7638472305538893, "lo": 0.756615411131695, "hi": 0.7708374099107584 }, "conf": { "value": 0.7789741979156999, "lo": 0.7761000959587006, "hi": 0.7813742400176061 }, "ece": { "value": 0.01823993969565023, "lo": 0.01435277097834691, "hi": 0.025505661267580974 }, "reliability": { "value": 0.0005702561447984441, "lo": 0.00037617808288516083, "hi": 0.0011406738456206653 }, "resolution": { "value": 0.024364708249844, "lo": 0.02252781376785893, "hi": 0.026466105696258707 }, "uncertainty": { "value": 0.1803846389290428, "lo": 0.17664708837189344, "hi": 0.18414851315321487 }, "auroc": { "value": 0.7486387139008708, "lo": 0.7404284055591798, "hi": 0.7573377305552835 }, "acc@80": { "value": 0.8189468421929679, "lo": 0.8115530142839764, "hi": 0.8258698916099144 } } }, { "env": "mcq", "slice": "all", "system": "qwen3.8-27b", "metrics": { "n": { "value": 5064.0, "lo": 5064.0, "hi": 5064.0 }, "ce": { "value": 0.16152235565574208, "lo": 0.1436146447662271, "hi": 0.18117535842745647 }, "kl": { "value": 0.16152235565574208, "lo": 0.1436146447662271, "hi": 0.18117535842745647 }, "jsd": { "value": 0.0339809006728758, "lo": 0.03120830008883931, "hi": 0.03714233939422152 }, "brier": { "value": 0.07759509006027641, "lo": 0.07031103397817423, "hi": 0.08625582464906033 }, "acc": { "value": 0.9506319115323855, "lo": 0.9444065955766193, "hi": 0.9558698657187993 }, "conf": { "value": 0.9612343825055764, "lo": 0.9584609354556813, "hi": 0.964068326400297 }, "ece": { "value": 0.011042265252385032, "lo": 0.008490387787188679, "hi": 0.01711441036696952 }, "reliability": { "value": 0.00033699148728081663, "lo": 0.00018894192159061082, "hi": 0.0009361550731056858 }, "resolution": { "value": 0.010201227058651932, "lo": 0.007894381945680142, "hi": 0.01323264157178327 }, "uncertainty": { "value": 0.04693088030866832, "lo": 0.04218265580506332, "hi": 0.05250276808353859 }, "auroc": { "value": 0.9187461570419609, "lo": 0.9038582629555388, "hi": 0.9315262910876058 }, "acc@80": { "value": 0.9918538632436436, "lo": 0.9892557393236238, "hi": 0.994569242162429 } } }, { "env": "mcq", "slice": "all", "system": "qwen3.8-27b +T", "metrics": { "n": { "value": 5064.0, "lo": 5064.0, "hi": 5064.0 }, "ce": { "value": 0.15856058240573354, "lo": 0.14420100372376268, "hi": 0.17388267441183627 }, "kl": { "value": 0.15856058240573354, "lo": 0.14420100372376268, "hi": 0.17388267441183627 }, "jsd": { "value": 0.040311464109205934, "lo": 0.03765378380738312, "hi": 0.043271018983395255 }, "brier": { "value": 0.07782049892680105, "lo": 0.07090973446475114, "hi": 0.08542435214527039 }, "acc": { "value": 0.9506319115323855, "lo": 0.9444065955766193, "hi": 0.9558698657187993 }, "conf": { "value": 0.9368245108445022, "lo": 0.9332560573742827, "hi": 0.9402748198546733 }, "ece": { "value": 0.014467744564029262, "lo": 0.01030803101457266, "hi": 0.02049353649834128 }, "reliability": { "value": 0.0006845060520818079, "lo": 0.00031037112769545055, "hi": 0.0015105244764829953 }, "resolution": { "value": 0.010048250389244446, "lo": 0.007674740410042408, "hi": 0.012867026968922805 }, "uncertainty": { "value": 0.04693088030866832, "lo": 0.04218265580506332, "hi": 0.05250276808353859 }, "auroc": { "value": 0.9181204819277109, "lo": 0.9034286574202369, "hi": 0.9307128031294514 }, "acc@80": { "value": 0.992100715872624, "lo": 0.9893853369538386, "hi": 0.9948160947914095 } } }, { "env": "stsb", "slice": "all", "system": "qwen3.8-27b", "metrics": { "n": { "value": 3676.0, "lo": 3150.95, "hi": 4793.674999999996 }, "ce": { "value": 1.6476253216148637, "lo": 1.5775778269475285, "hi": 1.7187834538113418 }, "kl": { "value": 1.2453214987526078, "lo": 1.1781597293366053, "hi": 1.3132812315415179 }, "jsd": { "value": 0.2531612317538707, "lo": 0.23836222277454266, "hi": 0.2660907919098357 }, "brier": { "value": 0.49555985949446896, "lo": 0.4665357822966389, "hi": 0.5207911062026427 }, "acc": { "value": 0.43307943416757344, "lo": 0.4084860412975492, "hi": 0.4593722714036901 }, "conf": { "value": 0.6409933912226389, "lo": 0.6288105489244771, "hi": 0.656202620137405 }, "ece": { "value": 0.20791395705506535, "lo": 0.1920427844589958, "hi": 0.22585973683509933 }, "reliability": { "value": 0.05260596751028478, "lo": 0.04485848169615247, "hi": 0.0640769550047139 }, "resolution": { "value": 0.0210368610762396, "lo": 0.01579295412315698, "hi": 0.028579310257640548 }, "uncertainty": { "value": 0.24552163786866787, "lo": 0.2416251851428358, "hi": 0.24834907728376013 }, "auroc": { "value": 0.6278471291196868, "lo": 0.5916084195966954, "hi": 0.6613640926526575 }, "acc@80": { "value": 0.458347500850051, "lo": 0.4284039479890641, "hi": 0.49020417161416496 } } }, { "env": "stsb", "slice": "all", "system": "qwen3.8-27b +T", "metrics": { "n": { "value": 3676.0, "lo": 3150.95, "hi": 4793.674999999996 }, "ce": { "value": 1.346727264947815, "lo": 1.2981053078549638, "hi": 1.387067630833167 }, "kl": { "value": 0.9444234420855593, "lo": 0.9002371630659755, "hi": 0.9827978015240808 }, "jsd": { "value": 0.2544173722121313, "lo": 0.2431917264063818, "hi": 0.2638236801564275 }, "brier": { "value": 0.4137424475287545, "lo": 0.3945117940173643, "hi": 0.43107894651028067 }, "acc": { "value": 0.43307943416757344, "lo": 0.4084860412975492, "hi": 0.4593722714036901 }, "conf": { "value": 0.43821688614180715, "lo": 0.4277928172327064, "hi": 0.45204815469211607 }, "ece": { "value": 0.04762515082161155, "lo": 0.041572815509467266, "hi": 0.0712501592630694 }, "reliability": { "value": 0.0034786243859689096, "lo": 0.002723084325544467, "hi": 0.006703017478029771 }, "resolution": { "value": 0.02259646870336115, "lo": 0.017410538777569182, "hi": 0.03078626929564409 }, "uncertainty": { "value": 0.24552163786866787, "lo": 0.2416251851428358, "hi": 0.24834907728376013 }, "auroc": { "value": 0.6288149601172851, "lo": 0.5940100615585853, "hi": 0.6597205074364987 }, "acc@80": { "value": 0.4546072764365862, "lo": 0.42664841306830836, "hi": 0.4858779236978728 } } }, { "env": "yelp", "slice": "all", "system": "qwen3.8-27b", "metrics": { "n": { "value": 5000.0, "lo": 5000.0, "hi": 5000.0 }, "ce": { "value": 1.0750614394949205, "lo": 1.0249181485562968, "hi": 1.1214973037621498 }, "kl": { "value": 1.0750614394949205, "lo": 1.0249181485562968, "hi": 1.1214973037621498 }, "jsd": { "value": 0.1992499097762599, "lo": 0.1921623077584074, "hi": 0.20578540527137232 }, "brier": { "value": 0.5048983976118063, "lo": 0.48400116815630245, "hi": 0.52369317141654 }, "acc": { "value": 0.6704, "lo": 0.658095, "hi": 0.684305 }, "conf": { "value": 0.8447078410241782, "lo": 0.8402829839710233, "hi": 0.8494419245794594 }, "ece": { "value": 0.17707699045033023, "lo": 0.1655175251053024, "hi": 0.18917956895743576 }, "reliability": { "value": 0.040149556280830494, "lo": 0.035376448880104855, "hi": 0.04608360773679248 }, "resolution": { "value": 0.02467955631511751, "lo": 0.021987147303851054, "hi": 0.028814119240178446 }, "uncertainty": { "value": 0.22096384, "lo": 0.216031657, "hi": 0.22500596099999998 }, "auroc": { "value": 0.6987530629445976, "lo": 0.6840858584822799, "hi": 0.7139086803877877 }, "acc@80": { "value": 0.71575, "lo": 0.7025, "hi": 0.7306312500000001 } } }, { "env": "yelp", "slice": "all", "system": "qwen3.8-27b +T", "metrics": { "n": { "value": 5000.0, "lo": 5000.0, "hi": 5000.0 }, "ce": { "value": 0.8579646852100378, "lo": 0.8307245312991125, "hi": 0.8845571792743167 }, "kl": { "value": 0.8579646852100378, "lo": 0.8307245312991125, "hi": 0.8845571792743167 }, "jsd": { "value": 0.21839012188284068, "lo": 0.21317031798038433, "hi": 0.22358432096686606 }, "brier": { "value": 0.4560912008380844, "lo": 0.4403772714428241, "hi": 0.4699115792262956 }, "acc": { "value": 0.6704, "lo": 0.658095, "hi": 0.684305 }, "conf": { "value": 0.6933824652455798, "lo": 0.6882470527288771, "hi": 0.6985853923447576 }, "ece": { "value": 0.062183613836976986, "lo": 0.05147542975040488, "hi": 0.0739244817342388 }, "reliability": { "value": 0.005187854546029306, "lo": 0.0040598069598425345, "hi": 0.007699148668794739 }, "resolution": { "value": 0.023317555991440544, "lo": 0.020700572010863134, "hi": 0.02715480823596157 }, "uncertainty": { "value": 0.22096384, "lo": 0.216031657, "hi": 0.22500596099999998 }, "auroc": { "value": 0.6935266874435202, "lo": 0.6782681124440355, "hi": 0.707951431911175 }, "acc@80": { "value": 0.71275, "lo": 0.69911875, "hi": 0.72813125 } } }, { "env": "boolq", "slice": "all", "system": "LFM2.5-VL-450M", "metrics": { "n": { "value": 8129.0, "lo": 7984.475, "hi": 8270.575 }, "ce": { "value": 1.8442211704383502, "lo": 1.7851897229303701, "hi": 1.9091822456500516 }, "kl": { "value": 1.8442211704383502, "lo": 1.7851897229303701, "hi": 1.9091822456500516 }, "jsd": { "value": 0.24937887226282604, "lo": 0.2419793140327026, "hi": 0.2580996156272248 }, "brier": { "value": 0.7293012228537639, "lo": 0.707313065987665, "hi": 0.7551299460192672 }, "acc": { "value": 0.6223397711895682, "lo": 0.6091156416180727, "hi": 0.6334462215543152 }, "conf": { "value": 0.9874027616233605, "lo": 0.9865438820518876, "hi": 0.9880995577569468 }, "ece": { "value": 0.3650629904337923, "lo": 0.35360355814649796, "hi": 0.3783027822129629 }, "reliability": { "value": 0.15031460025611365, "lo": 0.1416393213713674, "hi": 0.16045640817015988 }, "resolution": { "value": 0.021194817232827602, "lo": 0.018512684938401948, "hi": 0.024949449951594642 }, "uncertainty": { "value": 0.2350329803852841, "lo": 0.23219210586793973, "hi": 0.23809377671631937 }, "auroc": { "value": 0.6765497423561583, "lo": 0.6622366769117792, "hi": 0.6908041518723789 }, "acc@80": { "value": 0.6730739658619099, "lo": 0.6583250142349811, "hi": 0.6852436048190568 } } }, { "env": "boolq", "slice": "all", "system": "LFM2.5-VL-450M +T", "metrics": { "n": { "value": 8129.0, "lo": 7984.475, "hi": 8270.575 }, "ce": { "value": 0.6462882006413323, "lo": 0.6391522517704625, "hi": 0.6546312670639313 }, "kl": { "value": 0.6462882006413323, "lo": 0.6391522517704625, "hi": 0.6546312670639313 }, "jsd": { "value": 0.19800455056845404, "lo": 0.19609560000277734, "hi": 0.2002744331262142 }, "brier": { "value": 0.4548536484309796, "lo": 0.4480256042879124, "hi": 0.4628228606147715 }, "acc": { "value": 0.6223397711895682, "lo": 0.6091156416180727, "hi": 0.6334462215543152 }, "conf": { "value": 0.6498392437994798, "lo": 0.6489468585458049, "hi": 0.6506301063148058 }, "ece": { "value": 0.09650537872482522, "lo": 0.0878912971113201, "hi": 0.10872639955944483 }, "reliability": { "value": 0.013666581300217133, "lo": 0.011609939079836616, "hi": 0.016744569783581506 }, "resolution": { "value": 0.021194817232827602, "lo": 0.018512684938401948, "hi": 0.024949449951594642 }, "uncertainty": { "value": 0.2350329803852841, "lo": 0.23219210586793973, "hi": 0.23809377671631937 }, "auroc": { "value": 0.6765497423561583, "lo": 0.6622366769117792, "hi": 0.6908041518723789 }, "acc@80": { "value": 0.6730739658619099, "lo": 0.6583250142349811, "hi": 0.6852436048190568 } } }, { "env": "chaosnli", "slice": "all", "system": "LFM2.5-VL-450M", "metrics": { "n": { "value": 2763.0, "lo": 2732.475, "hi": 2798.525 }, "ce": { "value": 1.338092801429914, "lo": 1.3015494891945418, "hi": 1.3777673557125674 }, "kl": { "value": 0.8077327313606523, "lo": 0.7694633892661862, "hi": 0.8466904649156057 }, "jsd": { "value": 0.15568317018900518, "lo": 0.14978620375965646, "hi": 0.16185003866414396 }, "brier": { "value": 0.45064982413103327, "lo": 0.4318968382561482, "hi": 0.4702720495376779 }, "acc": { "value": 0.47050307636626854, "lo": 0.45173159322760664, "hi": 0.49106711293539973 }, "conf": { "value": 0.793341954419719, "lo": 0.7883808439698059, "hi": 0.7978205623973785 }, "ece": { "value": 0.32283887805345046, "lo": 0.30244659738864715, "hi": 0.3422140118566659 }, "reliability": { "value": 0.11115614403633266, "lo": 0.09952787023352347, "hi": 0.12475005518126144 }, "resolution": { "value": 0.004140842407372943, "lo": 0.0030534059588853087, "hi": 0.007733036003736438 }, "uncertainty": { "value": 0.24912993149614582, "lo": 0.2476698559729648, "hi": 0.24992017489438081 }, "auroc": { "value": 0.5560857563489142, "lo": 0.5350635361967264, "hi": 0.5771278261325536 }, "acc@80": { "value": 0.4850678733031674, "lo": 0.4625494747149383, "hi": 0.5074677074352085 } } }, { "env": "chaosnli", "slice": "all", "system": "LFM2.5-VL-450M +T", "metrics": { "n": { "value": 2763.0, "lo": 2732.475, "hi": 2798.525 }, "ce": { "value": 0.9470154748727558, "lo": 0.9377579590201713, "hi": 0.9565186291632289 }, "kl": { "value": 0.41665540480349406, "lo": 0.4075258234291402, "hi": 0.4265268103403057 }, "jsd": { "value": 0.11686510490943085, "lo": 0.11437385167077567, "hi": 0.11960367577491623 }, "brier": { "value": 0.2754697165443986, "lo": 0.2682097526227291, "hi": 0.283347714769842 }, "acc": { "value": 0.47050307636626854, "lo": 0.45173159322760664, "hi": 0.49106711293539973 }, "conf": { "value": 0.4756218893714501, "lo": 0.4715470519749072, "hi": 0.4795668726741519 }, "ece": { "value": 0.060558357153253424, "lo": 0.05080237045491872, "hi": 0.08167849404299185 }, "reliability": { "value": 0.005019142385552081, "lo": 0.0037667096489379685, "hi": 0.009072622700655618 }, "resolution": { "value": 0.004701090799118026, "lo": 0.0034839186490559088, "hi": 0.008955117248874217 }, "uncertainty": { "value": 0.24912993149614582, "lo": 0.2476698559729648, "hi": 0.24992017489438081 }, "auroc": { "value": 0.5551511646248488, "lo": 0.5348637119219112, "hi": 0.5765463281844294 }, "acc@80": { "value": 0.48733031674208144, "lo": 0.46429486455981944, "hi": 0.5089358074753532 } } }, { "env": "civilcomments", "slice": "all", "system": "LFM2.5-VL-450M", "metrics": { "n": { "value": 10581.0, "lo": 10261.425, "hi": 10904.525 }, "ce": { "value": 1.4821756429077173, "lo": 1.4715134964963283, "hi": 1.4919811813196462 }, "kl": { "value": 1.250692834138376, "lo": 1.234751864466174, "hi": 1.2657587399506547 }, "jsd": { "value": 0.3094179506553737, "lo": 0.30548679187966254, "hi": 0.3131230305948588 }, "brier": { "value": 0.9676093443593095, "lo": 0.9558303989434425, "hi": 0.9785138985363868 }, "acc": { "value": 0.08902750212645308, "lo": 0.08389836855315358, "hi": 0.09475320608057158 }, "conf": { "value": 0.8087093981021451, "lo": 0.8069045121845806, "hi": 0.8105242254083899 }, "ece": { "value": 0.7196818959756921, "lo": 0.7141072087205469, "hi": 0.7246391429567834 }, "reliability": { "value": 0.5196409469902246, "lo": 0.5114804735713947, "hi": 0.5269996362257217 }, "resolution": { "value": 0.0014444372335602875, "lo": 0.0011473712943158997, "hi": 0.0020672249708752147 }, "uncertainty": { "value": 0.08110160599157748, "lo": 0.0768594322623401, "hi": 0.08577496996869058 }, "auroc": { "value": 0.6267196427993231, "lo": 0.6090423391336587, "hi": 0.6445086359013343 }, "acc@80": { "value": 0.09982279976373302, "lo": 0.09379804174920786, "hi": 0.10631157687181092 } } }, { "env": "civilcomments", "slice": "all", "system": "LFM2.5-VL-450M +T", "metrics": { "n": { "value": 10581.0, "lo": 10261.425, "hi": 10904.525 }, "ce": { "value": 0.6966061569918292, "lo": 0.696547118275331, "hi": 0.6966604611465038 }, "kl": { "value": 0.4651233482224881, "lo": 0.45887698441033087, "hi": 0.4719685436354903 }, "jsd": { "value": 0.14202715322565218, "lo": 0.1400542553069198, "hi": 0.14422450005879134 }, "brier": { "value": 0.3479509513394226, "lo": 0.34365343805139087, "hi": 0.35265884466994574 }, "acc": { "value": 0.08902750212645308, "lo": 0.08389836855315358, "hi": 0.09475320608057158 }, "conf": { "value": 0.5025412600611476, "lo": 0.502522134743794, "hi": 0.502559541257449 }, "ece": { "value": 0.41351375793469464, "lo": 0.4077915773159254, "hi": 0.41864092887115295 }, "reliability": { "value": 0.17238127768567704, "lo": 0.16785753907234766, "hi": 0.17680060406378054 }, "resolution": { "value": 0.0014444372335602875, "lo": 0.0011473712943158997, "hi": 0.0020672249708752147 }, "uncertainty": { "value": 0.08110160599157748, "lo": 0.0768594322623401, "hi": 0.08577496996869058 }, "auroc": { "value": 0.6267196427993231, "lo": 0.6090423391336587, "hi": 0.6445086359013343 }, "acc@80": { "value": 0.09982279976373302, "lo": 0.09379804174920786, "hi": 0.10631157687181092 } } }, { "env": "folktables", "slice": "all", "system": "LFM2.5-VL-450M", "metrics": { "n": { "value": 15003.0, "lo": 14855.8, "hi": 15157.0 }, "ce": { "value": 1.13187423963659, "lo": 1.1199427317069248, "hi": 1.143924365145015 }, "kl": { "value": 1.13187423963659, "lo": 1.1199427317069248, "hi": 1.143924365145015 }, "jsd": { "value": 0.2976529663502538, "lo": 0.2947501180040379, "hi": 0.30064246013239343 }, "brier": { "value": 0.8518198364734217, "lo": 0.8421069098053824, "hi": 0.8617736268712928 }, "acc": { "value": 0.35112977404519097, "lo": 0.3427286002698279, "hi": 0.3591625122920405 }, "conf": { "value": 0.8052857982343986, "lo": 0.8038829735505919, "hi": 0.8066028173974609 }, "ece": { "value": 0.4541560241892077, "lo": 0.44676883916665083, "hi": 0.46226413029702884 }, "reliability": { "value": 0.2133990714594856, "lo": 0.2065251674018482, "hi": 0.2212758548271539 }, "resolution": { "value": 0.01625963369096802, "lo": 0.014593422216554746, "hi": 0.01834318497783375 }, "uncertainty": { "value": 0.2278376558241641, "lo": 0.2252656850700038, "hi": 0.23016480033951636 }, "auroc": { "value": 0.6484255122164856, "lo": 0.6393501068451605, "hi": 0.6577473978427627 }, "acc@80": { "value": 0.3798533577737044, "lo": 0.37053804935340834, "hi": 0.3899598578179518 } } }, { "env": "folktables", "slice": "all", "system": "LFM2.5-VL-450M +T", "metrics": { "n": { "value": 15003.0, "lo": 14855.8, "hi": 15157.0 }, "ce": { "value": 0.69433492276209, "lo": 0.6942494606398926, "hi": 0.6944197895761097 }, "kl": { "value": 0.69433492276209, "lo": 0.6942494606398926, "hi": 0.6944197895761097 }, "jsd": { "value": 0.21608632861145755, "lo": 0.21606285112182436, "hi": 0.21610963849604864 }, "brier": { "value": 0.5011877355086589, "lo": 0.5011022745545566, "hi": 0.5012726015528075 }, "acc": { "value": 0.35112977404519097, "lo": 0.3427286002698279, "hi": 0.3591625122920405 }, "conf": { "value": 0.5024821010314376, "lo": 0.5024680409691173, "hi": 0.5024953831742714 }, "ece": { "value": 0.17631529944337335, "lo": 0.1679947607190661, "hi": 0.18368582074778786 }, "reliability": { "value": 0.0390112009131181, "lo": 0.03597731804506837, "hi": 0.04218598137939827 }, "resolution": { "value": 0.01625963369096802, "lo": 0.014593422216554746, "hi": 0.01834318497783375 }, "uncertainty": { "value": 0.2278376558241641, "lo": 0.2252656850700038, "hi": 0.23016480033951636 }, "auroc": { "value": 0.6484255122164856, "lo": 0.6393501068451605, "hi": 0.6577473978427627 }, "acc@80": { "value": 0.3798533577737044, "lo": 0.37053804935340834, "hi": 0.3899598578179518 } } }, { "env": "mcq", "slice": "all", "system": "LFM2.5-VL-450M", "metrics": { "n": { "value": 5064.0, "lo": 5064.0, "hi": 5064.0 }, "ce": { "value": 1.1238747937701299, "lo": 1.0841670112525117, "hi": 1.1706346398444945 }, "kl": { "value": 1.1238747937701299, "lo": 1.0841670112525117, "hi": 1.1706346398444945 }, "jsd": { "value": 0.22378367348235934, "lo": 0.21775409032394605, "hi": 0.23094043551278276 }, "brier": { "value": 0.53661718405505, "lo": 0.5205883125570585, "hi": 0.5559808042874332 }, "acc": { "value": 0.6330963665086888, "lo": 0.6201471169036334, "hi": 0.6453396524486572 }, "conf": { "value": 0.8085802885546315, "lo": 0.8031755723428469, "hi": 0.8137682487780602 }, "ece": { "value": 0.17548392204594274, "lo": 0.16367946177070716, "hi": 0.18919804894349182 }, "reliability": { "value": 0.03901419698912007, "lo": 0.03459275241008605, "hi": 0.045940238397651 }, "resolution": { "value": 0.04566062336067558, "lo": 0.04170762143482616, "hi": 0.051254038565977274 }, "uncertainty": { "value": 0.23228535722218477, "lo": 0.22887638542610353, "hi": 0.23556458277959963 }, "auroc": { "value": 0.7638296936516368, "lo": 0.7515525862224941, "hi": 0.7775176429487639 }, "acc@80": { "value": 0.7018020241915577, "lo": 0.6856023204147124, "hi": 0.7147679585287584 } } }, { "env": "mcq", "slice": "all", "system": "LFM2.5-VL-450M +T", "metrics": { "n": { "value": 5064.0, "lo": 5064.0, "hi": 5064.0 }, "ce": { "value": 0.9293923372332724, "lo": 0.9085442498482007, "hi": 0.9541396485598952 }, "kl": { "value": 0.9293923372332724, "lo": 0.9085442498482007, "hi": 0.9541396485598952 }, "jsd": { "value": 0.24308830993178895, "lo": 0.23876913010109363, "hi": 0.24814049221299603 }, "brier": { "value": 0.483246252779501, "lo": 0.4721787294287204, "hi": 0.49679611510778227 }, "acc": { "value": 0.6330963665086888, "lo": 0.6201471169036334, "hi": 0.6453396524486572 }, "conf": { "value": 0.6332911854783634, "lo": 0.6279506724075, "hi": 0.6386419969695399 }, "ece": { "value": 0.025710280334627914, "lo": 0.022769106257303048, "hi": 0.041568619142218766 }, "reliability": { "value": 0.0009101891355786433, "lo": 0.0008029880638276326, "hi": 0.0025178788028172865 }, "resolution": { "value": 0.047357823574182736, "lo": 0.04354459092559669, "hi": 0.05275982801643855 }, "uncertainty": { "value": 0.23228535722218477, "lo": 0.22887638542610353, "hi": 0.23556458277959963 }, "auroc": { "value": 0.7685087567914574, "lo": 0.7560508452734865, "hi": 0.7819789848383302 }, "acc@80": { "value": 0.7069859294001481, "lo": 0.6906813132559863, "hi": 0.7193285608491731 } } }, { "env": "stsb", "slice": "all", "system": "LFM2.5-VL-450M", "metrics": { "n": { "value": 3676.0, "lo": 3150.95, "hi": 4793.674999999996 }, "ce": { "value": 2.769908074626373, "lo": 2.7249642687168563, "hi": 2.812541851185826 }, "kl": { "value": 2.367604251764118, "lo": 2.3196694467977057, "hi": 2.412967225632331 }, "jsd": { "value": 0.42961846772895235, "lo": 0.41779523612791825, "hi": 0.44160509059057923 }, "brier": { "value": 0.9222239189081299, "lo": 0.8803932211717393, "hi": 0.9668684382412573 }, "acc": { "value": 0.23449401523394994, "lo": 0.20829456008503736, "hi": 0.26186580018791106 }, "conf": { "value": 0.7663902332428619, "lo": 0.7597274453661149, "hi": 0.7754193678634801 }, "ece": { "value": 0.5318962180089121, "lo": 0.4991755148723419, "hi": 0.5669433273802466 }, "reliability": { "value": 0.2979379142039276, "lo": 0.26392840232646725, "hi": 0.3386203519337424 }, "resolution": { "value": 0.005745888820674186, "lo": 0.0037674743621111855, "hi": 0.008934000963257147 }, "uncertainty": { "value": 0.17950657205341, "lo": 0.16490790178615253, "hi": 0.19329210120204962 }, "auroc": { "value": 0.3864619972725039, "lo": 0.35313031786271104, "hi": 0.420721035327209 }, "acc@80": { "value": 0.20843250595035703, "lo": 0.1813403726698036, "hi": 0.23840618798127763 } } }, { "env": "stsb", "slice": "all", "system": "LFM2.5-VL-450M +T", "metrics": { "n": { "value": 3676.0, "lo": 3150.95, "hi": 4793.674999999996 }, "ce": { "value": 1.7879169759051958, "lo": 1.7854613143177502, "hi": 1.790315991369925 }, "kl": { "value": 1.3856131530429403, "lo": 1.3744897439790065, "hi": 1.3960821204982374 }, "jsd": { "value": 0.36946552416133693, "lo": 0.3673007075324218, "hi": 0.3715704740325432 }, "brier": { "value": 0.557530205458868, "lo": 0.5499179112941774, "hi": 0.5648913387038841 }, "acc": { "value": 0.23449401523394994, "lo": 0.20829456008503736, "hi": 0.26186580018791106 }, "conf": { "value": 0.1945217882583341, "lo": 0.19428222565926243, "hi": 0.19481388059383264 }, "ece": { "value": 0.06466293144934368, "lo": 0.05635712184785718, "hi": 0.08360482969247139 }, "reliability": { "value": 0.007829707597464845, "lo": 0.005758262436951522, "hi": 0.01262360517332117 }, "resolution": { "value": 0.005915938610681793, "lo": 0.00409133744752459, "hi": 0.00926977298543524 }, "uncertainty": { "value": 0.17950657205341, "lo": 0.16490790178615253, "hi": 0.19329210120204962 }, "auroc": { "value": 0.3916183088534787, "lo": 0.36758265019595104, "hi": 0.42022600196576926 }, "acc@80": { "value": 0.20775246514790888, "lo": 0.1796780074561421, "hi": 0.23912592842405617 } } }, { "env": "yelp", "slice": "all", "system": "LFM2.5-VL-450M", "metrics": { "n": { "value": 5000.0, "lo": 5000.0, "hi": 5000.0 }, "ce": { "value": 2.0268496797689064, "lo": 1.9757696337111814, "hi": 2.0727043198643322 }, "kl": { "value": 2.0268496797689064, "lo": 1.9757696337111814, "hi": 2.0727043198643322 }, "jsd": { "value": 0.38751170604058394, "lo": 0.3800926360793931, "hi": 0.3943478975315039 }, "brier": { "value": 0.9752197802933239, "lo": 0.9535710225405619, "hi": 0.9955246274883223 }, "acc": { "value": 0.3654, "lo": 0.351295, "hi": 0.37961 }, "conf": { "value": 0.772893846130371, "lo": 0.7679695500971377, "hi": 0.7770327220417559 }, "ece": { "value": 0.4074938461303711, "lo": 0.3921759519062936, "hi": 0.42181115289151666 }, "reliability": { "value": 0.18062167840384138, "lo": 0.16894849842848417, "hi": 0.1937890895997626 }, "resolution": { "value": 0.002703533747639855, "lo": 0.0019412275991941928, "hi": 0.005002970304404658 }, "uncertainty": { "value": 0.23188284000000003, "lo": 0.22788681300000002, "hi": 0.23550620800000002 }, "auroc": { "value": 0.5561342098449372, "lo": 0.540110963525977, "hi": 0.5729905033824033 }, "acc@80": { "value": 0.38675, "lo": 0.37110625, "hi": 0.40163125 } } }, { "env": "yelp", "slice": "all", "system": "LFM2.5-VL-450M +T", "metrics": { "n": { "value": 5000.0, "lo": 5000.0, "hi": 5000.0 }, "ce": { "value": 1.4661425923220464, "lo": 1.4517260160875394, "hi": 1.4799758798314981 }, "kl": { "value": 1.4661425923220464, "lo": 1.4517260160875394, "hi": 1.4799758798314981 }, "jsd": { "value": 0.38481973653617163, "lo": 0.38190879602015265, "hi": 0.38753674066981 }, "brier": { "value": 0.7442832037214115, "lo": 0.7368222931464771, "hi": 0.7510716412102759 }, "acc": { "value": 0.3654, "lo": 0.351295, "hi": 0.37961 }, "conf": { "value": 0.3929922703588896, "lo": 0.3906625462630033, "hi": 0.3950085689472873 }, "ece": { "value": 0.03837137950549759, "lo": 0.03302673934354592, "hi": 0.05504690977148236 }, "reliability": { "value": 0.002792990203293114, "lo": 0.00206872995279149, "hi": 0.004910302827656831 }, "resolution": { "value": 0.0026951055589721448, "lo": 0.0019255613761626623, "hi": 0.004816427312298835 }, "uncertainty": { "value": 0.23188284000000003, "lo": 0.22788681300000002, "hi": 0.23550620800000002 }, "auroc": { "value": 0.5551914061428608, "lo": 0.5389410313371229, "hi": 0.5711703153645804 }, "acc@80": { "value": 0.387, "lo": 0.3715, "hi": 0.4027625 } } }, { "env": "boolq", "slice": "all", "system": "zs-LFM2.5-VL-450M-prefill", "metrics": { "n": { "value": 8129.0, "lo": 7984.475, "hi": 8270.575 }, "ce": { "value": 2.19728081495343, "lo": 2.124304608559749, "hi": 2.2815227998032035 }, "kl": { "value": 2.19728081495343, "lo": 2.124304608559749, "hi": 2.2815227998032035 }, "jsd": { "value": 0.25085200285439146, "lo": 0.24325014396883182, "hi": 0.2596134570244947 }, "brier": { "value": 0.7293619975390971, "lo": 0.7070458137496666, "hi": 0.7550569114874577 }, "acc": { "value": 0.6255381965801452, "lo": 0.6123595475426337, "hi": 0.6370788578916808 }, "conf": { "value": 0.9897076468330451, "lo": 0.9886793475548706, "hi": 0.9906145027240567 }, "ece": { "value": 0.36416945025289993, "lo": 0.3526348168184627, "hi": 0.3770581720829012 }, "reliability": { "value": 0.1483079643460137, "lo": 0.1392692926735881, "hi": 0.15822142319007584 }, "resolution": { "value": 0.019428780862266886, "lo": 0.016427198057464343, "hi": 0.023126909746466903 }, "uncertainty": { "value": 0.23424016119940483, "lo": 0.2312093865609688, "hi": 0.237375277593653 }, "auroc": { "value": 0.6688632601878447, "lo": 0.6535178089270295, "hi": 0.683143197873873 }, "acc@80": { "value": 0.6763032446563124, "lo": 0.6616667289131235, "hi": 0.6883528348868468 } } }, { "env": "boolq", "slice": "all", "system": "zs-LFM2.5-VL-450M-prefill +T", "metrics": { "n": { "value": 8129.0, "lo": 7984.475, "hi": 8270.575 }, "ce": { "value": 0.6456473889972213, "lo": 0.6386110520713849, "hi": 0.6540006891041504 }, "kl": { "value": 0.6456473889972213, "lo": 0.6386110520713849, "hi": 0.6540006891041504 }, "jsd": { "value": 0.1977480080233696, "lo": 0.19585256710888305, "hi": 0.2000011000174901 }, "brier": { "value": 0.45409670547516723, "lo": 0.44733937324373924, "hi": 0.4621110447796479 }, "acc": { "value": 0.6255381965801452, "lo": 0.6123595475426337, "hi": 0.6370788578916808 }, "conf": { "value": 0.6503467435365148, "lo": 0.6494268606073704, "hi": 0.6511364278149309 }, "ece": { "value": 0.09484015765890995, "lo": 0.08239948310358765, "hi": 0.10614650821261719 }, "reliability": { "value": 0.012131099068849526, "lo": 0.009954431487962349, "hi": 0.015130793748406441 }, "resolution": { "value": 0.019428780862266886, "lo": 0.016427198057464343, "hi": 0.023126909746466903 }, "uncertainty": { "value": 0.23424016119940483, "lo": 0.2312093865609688, "hi": 0.237375277593653 }, "auroc": { "value": 0.6688632601878447, "lo": 0.6535178089270295, "hi": 0.683143197873873 }, "acc@80": { "value": 0.6763032446563124, "lo": 0.6616667289131235, "hi": 0.6883528348868468 } } }, { "env": "chaosnli", "slice": "all", "system": "zs-LFM2.5-VL-450M-prefill", "metrics": { "n": { "value": 2763.0, "lo": 2732.475, "hi": 2798.525 }, "ce": { "value": 1.5707520474601657, "lo": 1.5275747663629993, "hi": 1.6173270858002489 }, "kl": { "value": 1.0403919773909036, "lo": 0.9966209037282945, "hi": 1.0856548039288445 }, "jsd": { "value": 0.17964967086268427, "lo": 0.17321191794730745, "hi": 0.18619182691973712 }, "brier": { "value": 0.5303528884034465, "lo": 0.5096208612341412, "hi": 0.5517010621282801 }, "acc": { "value": 0.47701773434672456, "lo": 0.45927291945657245, "hi": 0.4972083924734921 }, "conf": { "value": 0.8680714446049517, "lo": 0.8640764415692318, "hi": 0.8719903504246821 }, "ece": { "value": 0.3910537102582271, "lo": 0.36983254462392084, "hi": 0.409753591755807 }, "reliability": { "value": 0.16377605405545875, "lo": 0.1487661016563696, "hi": 0.1803656095749142 }, "resolution": { "value": 0.0037965216513373306, "lo": 0.0027810791537164403, "hi": 0.007693189873688164 }, "uncertainty": { "value": 0.24947181546544228, "lo": 0.24834130451031894, "hi": 0.24998980524401404 }, "auroc": { "value": 0.542554515334653, "lo": 0.5215917921541139, "hi": 0.5615865777972047 }, "acc@80": { "value": 0.4832579185520362, "lo": 0.4612578412985837, "hi": 0.5049284446588672 } } }, { "env": "chaosnli", "slice": "all", "system": "zs-LFM2.5-VL-450M-prefill +T", "metrics": { "n": { "value": 2763.0, "lo": 2732.475, "hi": 2798.525 }, "ce": { "value": 0.9411396746035926, "lo": 0.9316432723286321, "hi": 0.9510239685529941 }, "kl": { "value": 0.4107796045343307, "lo": 0.40118868149631065, "hi": 0.42065309138964746 }, "jsd": { "value": 0.11444784812915831, "lo": 0.11182792046593194, "hi": 0.11709707186892492 }, "brier": { "value": 0.2715981745898122, "lo": 0.2638461718757901, "hi": 0.2790434599477199 }, "acc": { "value": 0.47701773434672456, "lo": 0.45927291945657245, "hi": 0.4972083924734921 }, "conf": { "value": 0.4964434871110875, "lo": 0.4928107712749071, "hi": 0.5000973279827943 }, "ece": { "value": 0.04567342424405743, "lo": 0.039554822050559726, "hi": 0.0703830773513986 }, "reliability": { "value": 0.003249705978136044, "lo": 0.0024085959787744924, "hi": 0.007313038874674322 }, "resolution": { "value": 0.006150872707803075, "lo": 0.004592221659527366, "hi": 0.010323172091024698 }, "uncertainty": { "value": 0.24947181546544228, "lo": 0.24834130451031894, "hi": 0.24998980524401404 }, "auroc": { "value": 0.5787131598153856, "lo": 0.558184534930305, "hi": 0.5992775062158711 }, "acc@80": { "value": 0.5027149321266968, "lo": 0.48357814607970573, "hi": 0.5246717077591531 } } }, { "env": "civilcomments", "slice": "all", "system": "zs-LFM2.5-VL-450M-prefill", "metrics": { "n": { "value": 10581.0, "lo": 10261.425, "hi": 10904.525 }, "ce": { "value": 0.5900203309821959, "lo": 0.5838787321375208, "hi": 0.5954055604731321 }, "kl": { "value": 0.3585375222128549, "lo": 0.3526599668328442, "hi": 0.3646889333871 }, "jsd": { "value": 0.10899092024426695, "lo": 0.10722699098880814, "hi": 0.11086045531598936 }, "brier": { "value": 0.24589472982697438, "lo": 0.24149032242307591, "hi": 0.25058992920458706 }, "acc": { "value": 0.7690199414044041, "lo": 0.7604835999526309, "hi": 0.7782208280198362 }, "conf": { "value": 0.6458412299376108, "lo": 0.643821715178355, "hi": 0.6479629327616658 }, "ece": { "value": 0.1231787114667933, "lo": 0.11508685145064275, "hi": 0.13185081974310223 }, "reliability": { "value": 0.01598794750263893, "lo": 0.01415087215041063, "hi": 0.018437871557114755 }, "resolution": { "value": 0.005509658827805386, "lo": 0.00455771339467265, "hi": 0.006841205371086549 }, "uncertainty": { "value": 0.177628271126771, "lo": 0.17259316108493603, "hi": 0.18214829408492678 }, "auroc": { "value": 0.6114104019001925, "lo": 0.5994015242532421, "hi": 0.6240694745868953 }, "acc@80": { "value": 0.7942114589486119, "lo": 0.7824591945486796, "hi": 0.8012385112850875 } } }, { "env": "civilcomments", "slice": "all", "system": "zs-LFM2.5-VL-450M-prefill +T", "metrics": { "n": { "value": 10581.0, "lo": 10261.425, "hi": 10904.525 }, "ce": { "value": 0.5814737265683071, "lo": 0.5732353596092621, "hi": 0.5890529160665945 }, "kl": { "value": 0.3499909177989659, "lo": 0.3433920464430891, "hi": 0.35686043475600315 }, "jsd": { "value": 0.10305108396492324, "lo": 0.10123033439323824, "hi": 0.10503232184467196 }, "brier": { "value": 0.2374545346605293, "lo": 0.2319786130597003, "hi": 0.24301453536356327 }, "acc": { "value": 0.7690199414044041, "lo": 0.7604835999526309, "hi": 0.7782208280198362 }, "conf": { "value": 0.6966262697833534, "lo": 0.6940898809225147, "hi": 0.6992943953368023 }, "ece": { "value": 0.07537646986127365, "lo": 0.06830649884157658, "hi": 0.08475949545171395 }, "reliability": { "value": 0.007450847038259102, "lo": 0.006266291426017038, "hi": 0.009229124500393811 }, "resolution": { "value": 0.005509658827805386, "lo": 0.00455771339467265, "hi": 0.006841205371086549 }, "uncertainty": { "value": 0.177628271126771, "lo": 0.17259316108493603, "hi": 0.18214829408492678 }, "auroc": { "value": 0.6114104019001925, "lo": 0.5994015242532421, "hi": 0.6240694745868953 }, "acc@80": { "value": 0.7942114589486119, "lo": 0.7824591945486796, "hi": 0.8012385112850875 } } }, { "env": "folktables", "slice": "all", "system": "zs-LFM2.5-VL-450M-prefill", "metrics": { "n": { "value": 15003.0, "lo": 14855.8, "hi": 15157.0 }, "ce": { "value": 0.8381660962858019, "lo": 0.8311775782412278, "hi": 0.8453279801409322 }, "kl": { "value": 0.8381660962858019, "lo": 0.8311775782412278, "hi": 0.8453279801409322 }, "jsd": { "value": 0.24380680319635128, "lo": 0.24198023830936183, "hi": 0.24565457418496592 }, "brier": { "value": 0.6295013098606138, "lo": 0.6233081387799438, "hi": 0.6359303666896823 }, "acc": { "value": 0.42118243018063056, "lo": 0.41307011683913974, "hi": 0.4287435405329441 }, "conf": { "value": 0.6760865042251419, "lo": 0.6744624158494803, "hi": 0.6777140352172191 }, "ece": { "value": 0.25705756810492836, "lo": 0.24951568640967292, "hi": 0.26552571260322566 }, "reliability": { "value": 0.07892091224121713, "lo": 0.07498311484665345, "hi": 0.08387523515382338 }, "resolution": { "value": 0.008360669342411055, "lo": 0.007178693961924451, "hi": 0.010094067568160319 }, "uncertainty": { "value": 0.2437877906877688, "lo": 0.2424431754297249, "hi": 0.24492251689546507 }, "auroc": { "value": 0.5193538780959998, "lo": 0.5098087090592502, "hi": 0.5285425283334804 }, "acc@80": { "value": 0.4035160806532245, "lo": 0.3946298976790928, "hi": 0.41245959992217585 } } }, { "env": "folktables", "slice": "all", "system": "zs-LFM2.5-VL-450M-prefill +T", "metrics": { "n": { "value": 15003.0, "lo": 14855.8, "hi": 15157.0 }, "ce": { "value": 0.6934844593415295, "lo": 0.6934336599007378, "hi": 0.693534520171246 }, "kl": { "value": 0.6934844593415295, "lo": 0.6934336599007378, "hi": 0.693534520171246 }, "jsd": { "value": 0.21585368991239473, "lo": 0.21583973124532319, "hi": 0.21586743937159877 }, "brier": { "value": 0.5003372788995907, "lo": 0.5002864798773798, "hi": 0.5003873393791819 }, "acc": { "value": 0.42118243018063056, "lo": 0.41307011683913974, "hi": 0.4287435405329441 }, "conf": { "value": 0.5013043170496617, "lo": 0.5012913479575357, "hi": 0.5013185173855981 }, "ece": { "value": 0.11145470539128079, "lo": 0.10300510627501262, "hi": 0.12032274662126989 }, "reliability": { "value": 0.014739607237335837, "lo": 0.012980927601164585, "hi": 0.017113136480238565 }, "resolution": { "value": 0.008360669342411055, "lo": 0.007178693961924451, "hi": 0.010094067568160319 }, "uncertainty": { "value": 0.2437877906877688, "lo": 0.2424431754297249, "hi": 0.24492251689546507 }, "auroc": { "value": 0.5193538780959998, "lo": 0.5098087090592502, "hi": 0.5285425283334804 }, "acc@80": { "value": 0.4035160806532245, "lo": 0.3946298976790928, "hi": 0.41245959992217585 } } }, { "env": "mcq", "slice": "all", "system": "zs-LFM2.5-VL-450M-prefill", "metrics": { "n": { "value": 5064.0, "lo": 5064.0, "hi": 5064.0 }, "ce": { "value": 1.2006815423871746, "lo": 1.1556908086451052, "hi": 1.2525439528494509 }, "kl": { "value": 1.2006815423871746, "lo": 1.1556908086451052, "hi": 1.2525439528494509 }, "jsd": { "value": 0.22489259461281508, "lo": 0.21893657284624224, "hi": 0.23211634280046434 }, "brier": { "value": 0.5490602447358751, "lo": 0.5313253217366278, "hi": 0.5696447789893158 }, "acc": { "value": 0.6317140600315956, "lo": 0.618088467614534, "hi": 0.6442584913112164 }, "conf": { "value": 0.8235576231368062, "lo": 0.8179018961868909, "hi": 0.8288325505291434 }, "ece": { "value": 0.1918435631052107, "lo": 0.1803097437399235, "hi": 0.20564171560182226 }, "reliability": { "value": 0.04628672012359426, "lo": 0.041624676116715796, "hi": 0.05335272961327013 }, "resolution": { "value": 0.04594570047395249, "lo": 0.04148231527304997, "hi": 0.05110003801118209 }, "uncertainty": { "value": 0.23265140638999324, "lo": 0.22918947796015116, "hi": 0.23605511381645117 }, "auroc": { "value": 0.7650734520757576, "lo": 0.7519250235530622, "hi": 0.7782467152900953 }, "acc@80": { "value": 0.6998272031597137, "lo": 0.6853801530486299, "hi": 0.7143915082695631 } } }, { "env": "mcq", "slice": "all", "system": "zs-LFM2.5-VL-450M-prefill +T", "metrics": { "n": { "value": 5064.0, "lo": 5064.0, "hi": 5064.0 }, "ce": { "value": 0.9334720356419071, "lo": 0.914514870062201, "hi": 0.9582322834004864 }, "kl": { "value": 0.9334720356419071, "lo": 0.914514870062201, "hi": 0.9582322834004864 }, "jsd": { "value": 0.24389875313034248, "lo": 0.23958877153740263, "hi": 0.24891649684701952 }, "brier": { "value": 0.4846589365397375, "lo": 0.47352779484961377, "hi": 0.4980496994037511 }, "acc": { "value": 0.6317140600315956, "lo": 0.618088467614534, "hi": 0.6442584913112164 }, "conf": { "value": 0.6321695269208738, "lo": 0.6264940350916997, "hi": 0.6378483184065368 }, "ece": { "value": 0.022256455628038435, "lo": 0.01999505672676786, "hi": 0.039566132825991016 }, "reliability": { "value": 0.0007071546382037909, "lo": 0.000646030699779827, "hi": 0.002216121494060057 }, "resolution": { "value": 0.046863271490700284, "lo": 0.04270781407907646, "hi": 0.05233557744229133 }, "uncertainty": { "value": 0.23265140638999324, "lo": 0.22918947796015116, "hi": 0.23605511381645117 }, "auroc": { "value": 0.7688532860889001, "lo": 0.7557556679608567, "hi": 0.782075236828284 }, "acc@80": { "value": 0.7010614663046162, "lo": 0.6855097506788447, "hi": 0.7151320661565046 } } }, { "env": "stsb", "slice": "all", "system": "zs-LFM2.5-VL-450M-prefill", "metrics": { "n": { "value": 3676.0, "lo": 3150.95, "hi": 4793.674999999996 }, "ce": { "value": 2.9252958890973293, "lo": 2.8542557464838847, "hi": 2.990760903901766 }, "kl": { "value": 2.5229920662350733, "lo": 2.4474793436751017, "hi": 2.5950072765919248 }, "jsd": { "value": 0.4405102692623737, "lo": 0.42611439867612205, "hi": 0.45603016319697026 }, "brier": { "value": 1.0110556365336225, "lo": 0.9646760809995163, "hi": 1.061952511853829 }, "acc": { "value": 0.23449401523394994, "lo": 0.20829456008503736, "hi": 0.26186580018791106 }, "conf": { "value": 0.8349484990840379, "lo": 0.8289625878220371, "hi": 0.8429096477418141 }, "ece": { "value": 0.6004544838500879, "lo": 0.5682166316242653, "hi": 0.6344356695474724 }, "reliability": { "value": 0.3760320688752832, "lo": 0.33785596690004494, "hi": 0.42015100287903556 }, "resolution": { "value": 0.006868195834272034, "lo": 0.004344999612472737, "hi": 0.009792027695906771 }, "uncertainty": { "value": 0.17950657205341, "lo": 0.16490790178615253, "hi": 0.19329210120204962 }, "auroc": { "value": 0.3793089161418628, "lo": 0.3468237587038948, "hi": 0.4127735986935907 }, "acc@80": { "value": 0.20707242434546072, "lo": 0.17569723088852932, "hi": 0.23624905508074437 } } }, { "env": "stsb", "slice": "all", "system": "zs-LFM2.5-VL-450M-prefill +T", "metrics": { "n": { "value": 3676.0, "lo": 3150.95, "hi": 4793.674999999996 }, "ce": { "value": 1.7818855315091495, "lo": 1.776248229095906, "hi": 1.786696177576579 }, "kl": { "value": 1.379581708646894, "lo": 1.36617426785534, "hi": 1.3919406261312177 }, "jsd": { "value": 0.36735733088340217, "lo": 0.36466329704064493, "hi": 0.3699104853924312 }, "brier": { "value": 0.5556155232652956, "lo": 0.5470766212231325, "hi": 0.5634638237610552 }, "acc": { "value": 0.23449401523394994, "lo": 0.20829456008503736, "hi": 0.26186580018791106 }, "conf": { "value": 0.21819528485698236, "lo": 0.21767100327886024, "hi": 0.21884265661817287 }, "ece": { "value": 0.06393672301189443, "lo": 0.05676741520385015, "hi": 0.08194139559936113 }, "reliability": { "value": 0.007255352947775862, "lo": 0.005509254823106484, "hi": 0.01145279891547225 }, "resolution": { "value": 0.006273143829640579, "lo": 0.004612689575726061, "hi": 0.009656098696653346 }, "uncertainty": { "value": 0.17950657205341, "lo": 0.16490790178615253, "hi": 0.19329210120204962 }, "auroc": { "value": 0.38472659902344425, "lo": 0.35672093422911605, "hi": 0.4155279687971298 }, "acc@80": { "value": 0.20877252635158108, "lo": 0.1788658812922375, "hi": 0.2373745525689766 } } }, { "env": "yelp", "slice": "all", "system": "zs-LFM2.5-VL-450M-prefill", "metrics": { "n": { "value": 5000.0, "lo": 5000.0, "hi": 5000.0 }, "ce": { "value": 1.909529184309992, "lo": 1.8689111656271569, "hi": 1.947252798147701 }, "kl": { "value": 1.909529184309992, "lo": 1.8689111656271569, "hi": 1.947252798147701 }, "jsd": { "value": 0.3865241606078184, "lo": 0.3793586320580127, "hi": 0.3933433205318795 }, "brier": { "value": 0.9623211190083816, "lo": 0.9417940605110796, "hi": 0.9803091733259967 }, "acc": { "value": 0.3508, "lo": 0.336295, "hi": 0.3652 }, "conf": { "value": 0.7536129065871239, "lo": 0.7485283327189088, "hi": 0.7582657927136123 }, "ece": { "value": 0.4028129065871239, "lo": 0.3893723001004755, "hi": 0.41627636068210006 }, "reliability": { "value": 0.17593504065367982, "lo": 0.16507473345946477, "hi": 0.18765280021593223 }, "resolution": { "value": 0.004805350649765269, "lo": 0.0038690200729840376, "hi": 0.007448485908352989 }, "uncertainty": { "value": 0.22773936, "lo": 0.22320066299999997, "hi": 0.23182896000000003 }, "auroc": { "value": 0.5829090764108584, "lo": 0.5658942332599112, "hi": 0.599821896088401 }, "acc@80": { "value": 0.37275, "lo": 0.35911875, "hi": 0.38938125 } } }, { "env": "yelp", "slice": "all", "system": "zs-LFM2.5-VL-450M-prefill +T", "metrics": { "n": { "value": 5000.0, "lo": 5000.0, "hi": 5000.0 }, "ce": { "value": 1.4546720351361775, "lo": 1.4407163563912457, "hi": 1.4679473982561566 }, "kl": { "value": 1.4546720351361775, "lo": 1.4407163563912457, "hi": 1.4679473982561566 }, "jsd": { "value": 0.3824816280037077, "lo": 0.3795358432372821, "hi": 0.3852028864033026 }, "brier": { "value": 0.7421864816303968, "lo": 0.734649832104407, "hi": 0.7490823032617606 }, "acc": { "value": 0.3508, "lo": 0.336295, "hi": 0.3652 }, "conf": { "value": 0.4019106739742869, "lo": 0.3991919691836869, "hi": 0.40414484885959845 }, "ece": { "value": 0.05146108965461193, "lo": 0.042050024498915935, "hi": 0.06641135791941125 }, "reliability": { "value": 0.0035359386025178735, "lo": 0.00256163157206236, "hi": 0.006047043118079161 }, "resolution": { "value": 0.0047180908266350186, "lo": 0.0036765392781320312, "hi": 0.0073615620756998425 }, "uncertainty": { "value": 0.22773936, "lo": 0.22320066299999997, "hi": 0.23182896000000003 }, "auroc": { "value": 0.5833198091010706, "lo": 0.5654694588668958, "hi": 0.6004176415078453 }, "acc@80": { "value": 0.376, "lo": 0.36136875, "hi": 0.3925 } } }, { "env": "boolq", "slice": "all", "system": "LFM2.5-VL-1.6B", "metrics": { "n": { "value": 8129.0, "lo": 7984.475, "hi": 8270.575 }, "ce": { "value": 0.49785330640044806, "lo": 0.47560878566083276, "hi": 0.5168941670796923 }, "kl": { "value": 0.49785330640044806, "lo": 0.47560878566083276, "hi": 0.5168941670796923 }, "jsd": { "value": 0.1289900321502576, "lo": 0.12450703835114382, "hi": 0.13344787634126143 }, "brier": { "value": 0.31364132154159263, "lo": 0.2992261822655051, "hi": 0.32655412303134806 }, "acc": { "value": 0.7834912043301759, "lo": 0.7727635141427726, "hi": 0.7942986561315352 }, "conf": { "value": 0.8385633099546935, "lo": 0.8342370984319535, "hi": 0.8421433070067155 }, "ece": { "value": 0.05572035842341299, "lo": 0.04829541495473734, "hi": 0.06622252562054536 }, "reliability": { "value": 0.004336838053223081, "lo": 0.003321665032817823, "hi": 0.0059174203416841745 }, "resolution": { "value": 0.017281614936288374, "lo": 0.015054374950629739, "hi": 0.019808555621349623 }, "uncertainty": { "value": 0.16963273706742643, "lo": 0.16338830068774005, "hi": 0.17559999038847476 }, "auroc": { "value": 0.720295839934912, "lo": 0.7059285762321004, "hi": 0.73328066474667 }, "acc@80": { "value": 0.8342303552206674, "lo": 0.8219057567913703, "hi": 0.8448184058249903 } } }, { "env": "boolq", "slice": "all", "system": "LFM2.5-VL-1.6B +T", "metrics": { "n": { "value": 8129.0, "lo": 7984.475, "hi": 8270.575 }, "ce": { "value": 0.47054909267913464, "lo": 0.4549301610766264, "hi": 0.48505115783006075 }, "kl": { "value": 0.47054909267913464, "lo": 0.4549301610766264, "hi": 0.48505115783006075 }, "jsd": { "value": 0.134867241463773, "lo": 0.13130031093067707, "hi": 0.1385972596511396 }, "brier": { "value": 0.3060432354808317, "lo": 0.29394893888705187, "hi": 0.31721282900461956 }, "acc": { "value": 0.7834912043301759, "lo": 0.7727635141427726, "hi": 0.7942986561315352 }, "conf": { "value": 0.7873545146192261, "lo": 0.7830510087868503, "hi": 0.7909054094918448 }, "ece": { "value": 0.02205472772536668, "lo": 0.017562225471831002, "hi": 0.03129979479071034 }, "reliability": { "value": 0.0006079931936090638, "lo": 0.0004712707639423926, "hi": 0.001431221786267658 }, "resolution": { "value": 0.017281614936288374, "lo": 0.015054374950629739, "hi": 0.019808555621349623 }, "uncertainty": { "value": 0.16963273706742643, "lo": 0.16338830068774005, "hi": 0.17559999038847476 }, "auroc": { "value": 0.720295839934912, "lo": 0.7059285762321004, "hi": 0.73328066474667 }, "acc@80": { "value": 0.8342303552206674, "lo": 0.8219057567913703, "hi": 0.8448184058249903 } } }, { "env": "chaosnli", "slice": "all", "system": "LFM2.5-VL-1.6B", "metrics": { "n": { "value": 2763.0, "lo": 2732.475, "hi": 2798.525 }, "ce": { "value": 0.9602479808993487, "lo": 0.9361369244725344, "hi": 0.9843172571819331 }, "kl": { "value": 0.4298879108300868, "lo": 0.41134151220678244, "hi": 0.4520748675028292 }, "jsd": { "value": 0.09649697096993072, "lo": 0.09281219393386245, "hi": 0.10082829836391946 }, "brier": { "value": 0.26577383186749354, "lo": 0.2546888605608322, "hi": 0.27974481953511815 }, "acc": { "value": 0.579080709373869, "lo": 0.5594891914460797, "hi": 0.5986598971194417 }, "conf": { "value": 0.7066341453904998, "lo": 0.7019352197997519, "hi": 0.7123606447036466 }, "ece": { "value": 0.12755343601663088, "lo": 0.11026264951346527, "hi": 0.14841342371534058 }, "reliability": { "value": 0.019211434563092787, "lo": 0.015482553590135497, "hi": 0.026132798028573642 }, "resolution": { "value": 0.011672201689484832, "lo": 0.009373052472500086, "hi": 0.016802626077061344 }, "uncertainty": { "value": 0.24374624140492565, "lo": 0.24026622284850968, "hi": 0.24646102634681313 }, "auroc": { "value": 0.6226829858125538, "lo": 0.6014047939769757, "hi": 0.6428758549104638 }, "acc@80": { "value": 0.6131221719457014, "lo": 0.5899383468834688, "hi": 0.6332811174567478 } } }, { "env": "chaosnli", "slice": "all", "system": "LFM2.5-VL-1.6B +T", "metrics": { "n": { "value": 2763.0, "lo": 2732.475, "hi": 2798.525 }, "ce": { "value": 0.8519077402493126, "lo": 0.8405353646498452, "hi": 0.8627873330566642 }, "kl": { "value": 0.3215476701800507, "lo": 0.3130729840825459, "hi": 0.3318221108548847 }, "jsd": { "value": 0.08853265402241807, "lo": 0.08620149330702792, "hi": 0.09129035095982928 }, "brier": { "value": 0.21624319391507632, "lo": 0.2094406116514185, "hi": 0.22423506366859275 }, "acc": { "value": 0.579080709373869, "lo": 0.5594891914460797, "hi": 0.5986598971194417 }, "conf": { "value": 0.5528968678800066, "lo": 0.5493274220417315, "hi": 0.5568968168888413 }, "ece": { "value": 0.04217048857273199, "lo": 0.03485165644347747, "hi": 0.06340023380791397 }, "reliability": { "value": 0.0028069758520380406, "lo": 0.0019621215165353534, "hi": 0.0060221874232393195 }, "resolution": { "value": 0.011836966536868603, "lo": 0.009035809731571719, "hi": 0.01629494941176776 }, "uncertainty": { "value": 0.24374624140492565, "lo": 0.24026622284850968, "hi": 0.24646102634681313 }, "auroc": { "value": 0.6158762360275151, "lo": 0.5942953451246389, "hi": 0.6370132106595583 }, "acc@80": { "value": 0.6176470588235294, "lo": 0.5971840436530517, "hi": 0.638120808284715 } } }, { "env": "civilcomments", "slice": "all", "system": "LFM2.5-VL-1.6B", "metrics": { "n": { "value": 10581.0, "lo": 10261.425, "hi": 10904.525 }, "ce": { "value": 0.628772068815973, "lo": 0.6216998765989009, "hi": 0.6351669416229918 }, "kl": { "value": 0.397289260046632, "lo": 0.39052833898608946, "hi": 0.40449370764458126 }, "jsd": { "value": 0.11740113018175644, "lo": 0.11548964447189249, "hi": 0.11934102760312093 }, "brier": { "value": 0.280118354904552, "lo": 0.2744577113120764, "hi": 0.2855380627483798 }, "acc": { "value": 0.6832057461487572, "lo": 0.6741245818307544, "hi": 0.6931629642589503 }, "conf": { "value": 0.6608092393206801, "lo": 0.6586314404096306, "hi": 0.6627601171445902 }, "ece": { "value": 0.044680318138615394, "lo": 0.040354284677392396, "hi": 0.05704754718799394 }, "reliability": { "value": 0.00442753772069274, "lo": 0.003552214328484263, "hi": 0.00812415256967844 }, "resolution": { "value": 0.004326625220736689, "lo": 0.003684353149117342, "hi": 0.00702849888079183 }, "uncertainty": { "value": 0.21643565457807715, "lo": 0.21268806824183523, "hi": 0.2196806077426754 }, "auroc": { "value": 0.570141816424234, "lo": 0.55657289916221, "hi": 0.5825056692924083 }, "acc@80": { "value": 0.6973419964559953, "lo": 0.685354588080431, "hi": 0.7075284918032322 } } }, { "env": "civilcomments", "slice": "all", "system": "LFM2.5-VL-1.6B +T", "metrics": { "n": { "value": 10581.0, "lo": 10261.425, "hi": 10904.525 }, "ce": { "value": 0.6286493075695474, "lo": 0.6216425549183775, "hi": 0.6349876756300533 }, "kl": { "value": 0.39716649880020644, "lo": 0.3903972556891716, "hi": 0.4043454095770948 }, "jsd": { "value": 0.11748060993828376, "lo": 0.11557479409577277, "hi": 0.11941577090821023 }, "brier": { "value": 0.2801003072251209, "lo": 0.27448260909947786, "hi": 0.28549944743997047 }, "acc": { "value": 0.6832057461487572, "lo": 0.6741245818307544, "hi": 0.6931629642589503 }, "conf": { "value": 0.6594996163310518, "lo": 0.6573364393569618, "hi": 0.6614357378208103 }, "ece": { "value": 0.04538447461741798, "lo": 0.040671909795305515, "hi": 0.05725331864949602 }, "reliability": { "value": 0.004429914427853016, "lo": 0.003549175662120752, "hi": 0.008105769448894466 }, "resolution": { "value": 0.004326625220736689, "lo": 0.003684353149117342, "hi": 0.00702849888079183 }, "uncertainty": { "value": 0.21643565457807715, "lo": 0.21268806824183523, "hi": 0.2196806077426754 }, "auroc": { "value": 0.570141816424234, "lo": 0.55657289916221, "hi": 0.5825056692924083 }, "acc@80": { "value": 0.6973419964559953, "lo": 0.685354588080431, "hi": 0.7075284918032322 } } }, { "env": "folktables", "slice": "all", "system": "LFM2.5-VL-1.6B", "metrics": { "n": { "value": 15003.0, "lo": 14855.8, "hi": 15157.0 }, "ce": { "value": 0.7461344360201306, "lo": 0.7318165582948096, "hi": 0.7625133777479692 }, "kl": { "value": 0.7461344360201306, "lo": 0.7318165582948096, "hi": 0.7625133777479692 }, "jsd": { "value": 0.18684880414111565, "lo": 0.18358249706605045, "hi": 0.1905574729419387 }, "brier": { "value": 0.5087217043706055, "lo": 0.4979805961161123, "hi": 0.5205494304915599 }, "acc": { "value": 0.6536026128107711, "lo": 0.6451184668229782, "hi": 0.661609588316768 }, "conf": { "value": 0.8728890623528585, "lo": 0.8713695262973349, "hi": 0.8741989386944872 }, "ece": { "value": 0.21928644954208723, "lo": 0.2120593261305226, "hi": 0.2270807691928449 }, "reliability": { "value": 0.05965806026922896, "lo": 0.05587235775469685, "hi": 0.06374055446160587 }, "resolution": { "value": 0.03126160888216562, "lo": 0.029147991406127175, "hi": 0.034045515786957116 }, "uncertainty": { "value": 0.22640623733770432, "lo": 0.22388234090058073, "hi": 0.2289406268460248 }, "auroc": { "value": 0.7191105974276959, "lo": 0.710611030947418, "hi": 0.7276068851737252 }, "acc@80": { "value": 0.7234627562072988, "lo": 0.715027466679644, "hi": 0.7316353539303582 } } }, { "env": "folktables", "slice": "all", "system": "LFM2.5-VL-1.6B +T", "metrics": { "n": { "value": 15003.0, "lo": 14855.8, "hi": 15157.0 }, "ce": { "value": 0.6121096088424054, "lo": 0.605738667381172, "hi": 0.6194423479030153 }, "kl": { "value": 0.6121096088424054, "lo": 0.605738667381172, "hi": 0.6194423479030153 }, "jsd": { "value": 0.18423216443856238, "lo": 0.182536459553405, "hi": 0.1861922559361493 }, "brier": { "value": 0.42469090692255734, "lo": 0.4187951793424778, "hi": 0.43151763460653886 }, "acc": { "value": 0.6536026128107711, "lo": 0.6451184668229782, "hi": 0.661609588316768 }, "conf": { "value": 0.7066941692618047, "lo": 0.7055402831071527, "hi": 0.7076874142613194 }, "ece": { "value": 0.09931113401963056, "lo": 0.0927887134946938, "hi": 0.10663302728364965 }, "reliability": { "value": 0.017519382831588975, "lo": 0.015838382170399252, "hi": 0.019669253791422578 }, "resolution": { "value": 0.03126160888216562, "lo": 0.029147991406127175, "hi": 0.034045515786957116 }, "uncertainty": { "value": 0.22640623733770432, "lo": 0.22388234090058073, "hi": 0.2289406268460248 }, "auroc": { "value": 0.7191105974276959, "lo": 0.710611030947418, "hi": 0.7276068851737252 }, "acc@80": { "value": 0.7234627562072988, "lo": 0.715027466679644, "hi": 0.7316353539303582 } } }, { "env": "mcq", "slice": "all", "system": "LFM2.5-VL-1.6B", "metrics": { "n": { "value": 5064.0, "lo": 5064.0, "hi": 5064.0 }, "ce": { "value": 0.572165947776608, "lo": 0.5452410135475054, "hi": 0.600168726395567 }, "kl": { "value": 0.572165947776608, "lo": 0.5452410135475054, "hi": 0.600168726395567 }, "jsd": { "value": 0.1321212551223715, "lo": 0.12725109212655403, "hi": 0.137655877383627 }, "brier": { "value": 0.2907395378003839, "lo": 0.277269503680997, "hi": 0.3036812445034888 }, "acc": { "value": 0.7940363349131122, "lo": 0.7818868483412322, "hi": 0.80451224328594 }, "conf": { "value": 0.8386109491830801, "lo": 0.8331356306114539, "hi": 0.8433986392359162 }, "ece": { "value": 0.04461313826789694, "lo": 0.03691930355427648, "hi": 0.05483356854248656 }, "reliability": { "value": 0.0031711189146454073, "lo": 0.0023652986228275426, "hi": 0.004927669784027484 }, "resolution": { "value": 0.0415554142512013, "lo": 0.037302450812327796, "hi": 0.04679214675977566 }, "uncertainty": { "value": 0.1635426337508641, "lo": 0.1572722547911223, "hi": 0.1705397950077866 }, "auroc": { "value": 0.8434959034579483, "lo": 0.8314073112444279, "hi": 0.8549349683519096 }, "acc@80": { "value": 0.8765736855097507, "lo": 0.8666872377190817, "hi": 0.8871883485559121 } } }, { "env": "mcq", "slice": "all", "system": "LFM2.5-VL-1.6B +T", "metrics": { "n": { "value": 5064.0, "lo": 5064.0, "hi": 5064.0 }, "ce": { "value": 0.5558825055009654, "lo": 0.5348871978845787, "hi": 0.5784931928840349 }, "kl": { "value": 0.5558825055009654, "lo": 0.5348871978845787, "hi": 0.5784931928840349 }, "jsd": { "value": 0.14306107981735358, "lo": 0.13869103713638892, "hi": 0.14781201028160543 }, "brier": { "value": 0.28702519231409684, "lo": 0.275253304876988, "hi": 0.2992549336460319 }, "acc": { "value": 0.7940363349131122, "lo": 0.7818868483412322, "hi": 0.80451224328594 }, "conf": { "value": 0.7848499106116366, "lo": 0.7784763165245263, "hi": 0.7901990883085657 }, "ece": { "value": 0.013609003904996479, "lo": 0.010953061439293286, "hi": 0.025846386110621757 }, "reliability": { "value": 0.0002893597635578177, "lo": 0.00021791850356241846, "hi": 0.0014217689143205163 }, "resolution": { "value": 0.041273284636219036, "lo": 0.037026098960848956, "hi": 0.04646639681337491 }, "uncertainty": { "value": 0.1635426337508641, "lo": 0.1572722547911223, "hi": 0.1705397950077866 }, "auroc": { "value": 0.8431811608422989, "lo": 0.8313809563709517, "hi": 0.8543887509495475 }, "acc@80": { "value": 0.8775610960256727, "lo": 0.8678042458652184, "hi": 0.8874475438163416 } } }, { "env": "stsb", "slice": "all", "system": "LFM2.5-VL-1.6B", "metrics": { "n": { "value": 3676.0, "lo": 3150.95, "hi": 4793.674999999996 }, "ce": { "value": 2.3010640888069385, "lo": 2.219348265834835, "hi": 2.3945236861110115 }, "kl": { "value": 1.8987602659446827, "lo": 1.812395668802827, "hi": 1.9966693748004591 }, "jsd": { "value": 0.3890836419430554, "lo": 0.37413400486300785, "hi": 0.40635039337373485 }, "brier": { "value": 0.7514212698668175, "lo": 0.7248395195004066, "hi": 0.774923581452912 }, "acc": { "value": 0.23449401523394994, "lo": 0.20829456008503736, "hi": 0.26186580018791106 }, "conf": { "value": 0.6277639134356324, "lo": 0.610452234221094, "hi": 0.6395318968402306 }, "ece": { "value": 0.3932698982016825, "lo": 0.3754240981422377, "hi": 0.40815711587016956 }, "reliability": { "value": 0.16204072167674188, "lo": 0.14865970239664933, "hi": 0.17464263148459871 }, "resolution": { "value": 0.015115334255708575, "lo": 0.010829255968167148, "hi": 0.021201278220300646 }, "uncertainty": { "value": 0.17950657205341, "lo": 0.16490790178615253, "hi": 0.19329210120204962 }, "auroc": { "value": 0.6323157167427694, "lo": 0.5873000880206326, "hi": 0.6841052866583409 }, "acc@80": { "value": 0.2862971778306698, "lo": 0.2554955400741242, "hi": 0.3120790916209867 } } }, { "env": "stsb", "slice": "all", "system": "LFM2.5-VL-1.6B +T", "metrics": { "n": { "value": 3676.0, "lo": 3150.95, "hi": 4793.674999999996 }, "ce": { "value": 1.7525020624452756, "lo": 1.7332842234632335, "hi": 1.7758527468562777 }, "kl": { "value": 1.35019823958302, "lo": 1.3248534405741286, "hi": 1.3783798358965833 }, "jsd": { "value": 0.3586514174344883, "lo": 0.3534942812011378, "hi": 0.36449631932752286 }, "brier": { "value": 0.5466196704914243, "lo": 0.5345524094586155, "hi": 0.5583900186117372 }, "acc": { "value": 0.23449401523394994, "lo": 0.20829456008503736, "hi": 0.26186580018791106 }, "conf": { "value": 0.25416640367558696, "lo": 0.2516250643543517, "hi": 0.2560133568043584 }, "ece": { "value": 0.10395879835710332, "lo": 0.08579567132893845, "hi": 0.1290127644194208 }, "reliability": { "value": 0.014588727222329737, "lo": 0.010642276200674787, "hi": 0.021232165150903908 }, "resolution": { "value": 0.016003579625248947, "lo": 0.011654112067711338, "hi": 0.022233263244754886 }, "uncertainty": { "value": 0.17950657205341, "lo": 0.16490790178615253, "hi": 0.19329210120204962 }, "auroc": { "value": 0.6143144898642353, "lo": 0.5670806578704596, "hi": 0.6697986028198386 }, "acc@80": { "value": 0.28357701462087725, "lo": 0.2540760524470549, "hi": 0.3115851173964403 } } }, { "env": "yelp", "slice": "all", "system": "LFM2.5-VL-1.6B", "metrics": { "n": { "value": 5000.0, "lo": 5000.0, "hi": 5000.0 }, "ce": { "value": 1.2672820743302236, "lo": 1.2379575231452973, "hi": 1.296280505283855 }, "kl": { "value": 1.2672820743302236, "lo": 1.2379575231452973, "hi": 1.296280505283855 }, "jsd": { "value": 0.3070395576017841, "lo": 0.30149149729949076, "hi": 0.31204754471832824 }, "brier": { "value": 0.6588944098752754, "lo": 0.6449593526627326, "hi": 0.6723762128925471 }, "acc": { "value": 0.4702, "lo": 0.4572, "hi": 0.48362 }, "conf": { "value": 0.6046248717546463, "lo": 0.5987460925602913, "hi": 0.610359503082037 }, "ece": { "value": 0.1344248717546463, "lo": 0.12321233036130667, "hi": 0.14781036010198295 }, "reliability": { "value": 0.024641956620509477, "lo": 0.021457088746067842, "hi": 0.029497053853111736 }, "resolution": { "value": 0.024504319425293557, "lo": 0.02105900824547472, "hi": 0.02885907580372959 }, "uncertainty": { "value": 0.24911196000000002, "lo": 0.24816815999999997, "hi": 0.24973153599999998 }, "auroc": { "value": 0.663396490477615, "lo": 0.6478424101253941, "hi": 0.6779021032724923 }, "acc@80": { "value": 0.51325, "lo": 0.49825, "hi": 0.52725 } } }, { "env": "yelp", "slice": "all", "system": "LFM2.5-VL-1.6B +T", "metrics": { "n": { "value": 5000.0, "lo": 5000.0, "hi": 5000.0 }, "ce": { "value": 1.226441656161317, "lo": 1.203748108594829, "hi": 1.2478335056564203 }, "kl": { "value": 1.226441656161317, "lo": 1.203748108594829, "hi": 1.2478335056564203 }, "jsd": { "value": 0.3188183609293161, "lo": 0.3142419866773621, "hi": 0.3229560394339662 }, "brier": { "value": 0.6366238617464098, "lo": 0.6249159437916981, "hi": 0.6473524135460489 }, "acc": { "value": 0.4702, "lo": 0.4572, "hi": 0.48362 }, "conf": { "value": 0.5124118862896443, "lo": 0.5070885174999723, "hi": 0.5181019518532565 }, "ece": { "value": 0.052707036505233985, "lo": 0.04610336615075152, "hi": 0.06913588327222285 }, "reliability": { "value": 0.004453180531319464, "lo": 0.0034316601113259246, "hi": 0.006909642281025447 }, "resolution": { "value": 0.024464507157936383, "lo": 0.02122354112907011, "hi": 0.028876068000622108 }, "uncertainty": { "value": 0.24911196000000002, "lo": 0.24816815999999997, "hi": 0.24973153599999998 }, "auroc": { "value": 0.6644650863009548, "lo": 0.6492578389442697, "hi": 0.6792196427477908 }, "acc@80": { "value": 0.513, "lo": 0.49836875, "hi": 0.52814375 } } }, { "env": "boolq", "slice": "all", "system": "zs-LFM2.5-VL-1.6B-prefill", "metrics": { "n": { "value": 8129.0, "lo": 7984.475, "hi": 8270.575 }, "ce": { "value": 0.45809081848121175, "lo": 0.4459957462368789, "hi": 0.47074164998784135 }, "kl": { "value": 0.45809081848121175, "lo": 0.4459957462368789, "hi": 0.47074164998784135 }, "jsd": { "value": 0.13492659985478037, "lo": 0.13182005876327219, "hi": 0.13822748077897493 }, "brier": { "value": 0.29719946416782766, "lo": 0.2876802262296285, "hi": 0.30709230436429497 }, "acc": { "value": 0.7844753352195842, "lo": 0.7741087364795196, "hi": 0.7952461097461098 }, "conf": { "value": 0.7698005057228845, "lo": 0.7658782172520326, "hi": 0.7734927521904338 }, "ece": { "value": 0.017457544466331448, "lo": 0.015434384582842886, "hi": 0.029130645298296235 }, "reliability": { "value": 0.00043783593969573645, "lo": 0.00035828944809354607, "hi": 0.0012241892843678525 }, "resolution": { "value": 0.020778926965138774, "lo": 0.01864961651885824, "hi": 0.02375247698322183 }, "uncertainty": { "value": 0.16907378365170517, "lo": 0.16282972658385433, "hi": 0.17486439983689395 }, "auroc": { "value": 0.7424820344660427, "lo": 0.7301943723927901, "hi": 0.7563721693720247 }, "acc@80": { "value": 0.8422266646163309, "lo": 0.8309900400093337, "hi": 0.8519479967797275 } } }, { "env": "boolq", "slice": "all", "system": "zs-LFM2.5-VL-1.6B-prefill +T", "metrics": { "n": { "value": 8129.0, "lo": 7984.475, "hi": 8270.575 }, "ce": { "value": 0.4574282843520093, "lo": 0.44385529922606226, "hi": 0.47141889239347284 }, "kl": { "value": 0.4574282843520093, "lo": 0.44385529922606226, "hi": 0.47141889239347284 }, "jsd": { "value": 0.13152558032176487, "lo": 0.128123333032652, "hi": 0.13509654541911817 }, "brier": { "value": 0.2966681446394566, "lo": 0.286429025745625, "hi": 0.3072686901546995 }, "acc": { "value": 0.7844753352195842, "lo": 0.7741087364795196, "hi": 0.7952461097461098 }, "conf": { "value": 0.7898864226759795, "lo": 0.7857718409246228, "hi": 0.7936401831584182 }, "ece": { "value": 0.010848552134439696, "lo": 0.010703686129022155, "hi": 0.02327767605806169 }, "reliability": { "value": 0.00017561707931481276, "lo": 0.00019444645099059272, "hi": 0.0008448418020550343 }, "resolution": { "value": 0.020778926965138774, "lo": 0.01864961651885824, "hi": 0.02375247698322183 }, "uncertainty": { "value": 0.16907378365170517, "lo": 0.16282972658385433, "hi": 0.17486439983689395 }, "auroc": { "value": 0.7424820344660427, "lo": 0.7301943723927901, "hi": 0.7563721693720247 }, "acc@80": { "value": 0.8422266646163309, "lo": 0.8309900400093337, "hi": 0.8519479967797275 } } }, { "env": "chaosnli", "slice": "all", "system": "zs-LFM2.5-VL-1.6B-prefill", "metrics": { "n": { "value": 2763.0, "lo": 2732.475, "hi": 2798.525 }, "ce": { "value": 0.9518350511907218, "lo": 0.9283674771218429, "hi": 0.9747391947504885 }, "kl": { "value": 0.4214749811214598, "lo": 0.4022965505307716, "hi": 0.4429000525633131 }, "jsd": { "value": 0.09261838684960924, "lo": 0.08886798776282236, "hi": 0.09671742531030467 }, "brier": { "value": 0.25716765006067627, "lo": 0.2452494254966154, "hi": 0.26993159240426684 }, "acc": { "value": 0.6058631921824105, "lo": 0.5882627514525123, "hi": 0.623616533456431 }, "conf": { "value": 0.7285714013353017, "lo": 0.7234689812577547, "hi": 0.7342638359808698 }, "ece": { "value": 0.12270820915289131, "lo": 0.10870676878938139, "hi": 0.14057291935152566 }, "reliability": { "value": 0.018860741503653168, "lo": 0.0153226981630458, "hi": 0.02550495911405989 }, "resolution": { "value": 0.013356532305657276, "lo": 0.010626462844700021, "hi": 0.018205897395722793 }, "uncertainty": { "value": 0.23879298454095002, "lo": 0.234718782714523, "hi": 0.242209681238806 }, "auroc": { "value": 0.6302217899643772, "lo": 0.6092174144235512, "hi": 0.6487577196468025 }, "acc@80": { "value": 0.644343891402715, "lo": 0.6252236392170022, "hi": 0.6633401205261578 } } }, { "env": "chaosnli", "slice": "all", "system": "zs-LFM2.5-VL-1.6B-prefill +T", "metrics": { "n": { "value": 2763.0, "lo": 2732.475, "hi": 2798.525 }, "ce": { "value": 0.8347258254601914, "lo": 0.8226522855169598, "hi": 0.8456516996319347 }, "kl": { "value": 0.3043657553909295, "lo": 0.29452635459737536, "hi": 0.31487484659061205 }, "jsd": { "value": 0.08303659510340684, "lo": 0.08057666355029988, "hi": 0.08582469297651862 }, "brier": { "value": 0.2044370114518093, "lo": 0.19698183969822086, "hi": 0.21250505908078474 }, "acc": { "value": 0.6058631921824105, "lo": 0.5882627514525123, "hi": 0.623616533456431 }, "conf": { "value": 0.5782082809309398, "lo": 0.5741746859961775, "hi": 0.5825492156087732 }, "ece": { "value": 0.03619162070865312, "lo": 0.03401204560331832, "hi": 0.06117888941289102 }, "reliability": { "value": 0.002101243035652229, "lo": 0.001923223605356818, "hi": 0.005806942654502659 }, "resolution": { "value": 0.012847809609936024, "lo": 0.010548778386390832, "hi": 0.01850531944868528 }, "uncertainty": { "value": 0.23879298454095002, "lo": 0.234718782714523, "hi": 0.242209681238806 }, "auroc": { "value": 0.630666390197182, "lo": 0.6093681868939779, "hi": 0.6500422528507463 }, "acc@80": { "value": 0.6475113122171946, "lo": 0.6277296313785591, "hi": 0.6668965468301858 } } }, { "env": "civilcomments", "slice": "all", "system": "zs-LFM2.5-VL-1.6B-prefill", "metrics": { "n": { "value": 10581.0, "lo": 10261.425, "hi": 10904.525 }, "ce": { "value": 0.6502473360734663, "lo": 0.6447503319703678, "hi": 0.6555783078692586 }, "kl": { "value": 0.41876452730412533, "lo": 0.41192029397300456, "hi": 0.42577051054253134 }, "jsd": { "value": 0.12545754668024597, "lo": 0.12342769463728959, "hi": 0.12751187936910485 }, "brier": { "value": 0.3021940400965815, "lo": 0.29682435114805916, "hi": 0.30791112680685717 }, "acc": { "value": 0.6406766846233816, "lo": 0.6309652583155962, "hi": 0.6504697018498695 }, "conf": { "value": 0.6305742715462069, "lo": 0.6286774044784078, "hi": 0.6323416374505513 }, "ece": { "value": 0.03474427794995554, "lo": 0.03138912308177586, "hi": 0.04557950196961081 }, "reliability": { "value": 0.0026833209329026487, "lo": 0.002160425696622576, "hi": 0.003972453846252947 }, "resolution": { "value": 0.0033928532405596472, "lo": 0.002752364853399364, "hi": 0.005004430122386786 }, "uncertainty": { "value": 0.23021007040337363, "lo": 0.22735886827276156, "hi": 0.23284807778724742 }, "auroc": { "value": 0.5576777550250918, "lo": 0.5472387826952736, "hi": 0.5695659157795933 }, "acc@80": { "value": 0.6537507383343177, "lo": 0.6412779888193662, "hi": 0.6635854397027292 } } }, { "env": "civilcomments", "slice": "all", "system": "zs-LFM2.5-VL-1.6B-prefill +T", "metrics": { "n": { "value": 10581.0, "lo": 10261.425, "hi": 10904.525 }, "ce": { "value": 0.6502288177519915, "lo": 0.6447428064421498, "hi": 0.6555474234689279 }, "kl": { "value": 0.4187460089826506, "lo": 0.41190585434423493, "hi": 0.4257500233279973 }, "jsd": { "value": 0.12546885045847098, "lo": 0.1234402570863187, "hi": 0.1275238311101115 }, "brier": { "value": 0.3021844160960673, "lo": 0.2968225989617081, "hi": 0.3078932463327301 }, "acc": { "value": 0.6406766846233816, "lo": 0.6309652583155962, "hi": 0.6504697018498695 }, "conf": { "value": 0.6303210618646045, "lo": 0.6284271536306199, "hi": 0.6320855428907367 }, "ece": { "value": 0.034696598410399876, "lo": 0.031366328712397716, "hi": 0.04553824327196812 }, "reliability": { "value": 0.0026774160508352897, "lo": 0.002155129499152936, "hi": 0.0039658681084932335 }, "resolution": { "value": 0.0033928532405596472, "lo": 0.002752364853399364, "hi": 0.005004430122386786 }, "uncertainty": { "value": 0.23021007040337363, "lo": 0.22735886827276156, "hi": 0.23284807778724742 }, "auroc": { "value": 0.5576777550250918, "lo": 0.5472387826952736, "hi": 0.5695659157795933 }, "acc@80": { "value": 0.6537507383343177, "lo": 0.6412779888193662, "hi": 0.6635854397027292 } } }, { "env": "folktables", "slice": "all", "system": "zs-LFM2.5-VL-1.6B-prefill", "metrics": { "n": { "value": 15003.0, "lo": 14855.8, "hi": 15157.0 }, "ce": { "value": 0.8197066959026252, "lo": 0.8030192075311231, "hi": 0.8382327868642858 }, "kl": { "value": 0.8197066959026252, "lo": 0.8030192075311231, "hi": 0.8382327868642858 }, "jsd": { "value": 0.1938454187339975, "lo": 0.1902161653193078, "hi": 0.19782609427049014 }, "brier": { "value": 0.5447034447983794, "lo": 0.533082550576082, "hi": 0.5573901888815646 }, "acc": { "value": 0.6536026128107711, "lo": 0.6451184668229782, "hi": 0.661609588316768 }, "conf": { "value": 0.9027565910737848, "lo": 0.901633137398959, "hi": 0.9037675302880445 }, "ece": { "value": 0.24915397826301364, "lo": 0.2419406006958448, "hi": 0.25708588876344846 }, "reliability": { "value": 0.07734302902894433, "lo": 0.07311883088930139, "hi": 0.08198136422133916 }, "resolution": { "value": 0.031099067312709942, "lo": 0.028930295369946016, "hi": 0.033717553045725074 }, "uncertainty": { "value": 0.22640623733770432, "lo": 0.22388234090058073, "hi": 0.2289406268460248 }, "auroc": { "value": 0.7213528267123783, "lo": 0.7130525215248933, "hi": 0.7297975155813541 }, "acc@80": { "value": 0.7195467422096318, "lo": 0.7107755151525855, "hi": 0.7283890372652869 } } }, { "env": "folktables", "slice": "all", "system": "zs-LFM2.5-VL-1.6B-prefill +T", "metrics": { "n": { "value": 15003.0, "lo": 14855.8, "hi": 15157.0 }, "ce": { "value": 0.6167797650432055, "lo": 0.6105579967897169, "hi": 0.6238108651779094 }, "kl": { "value": 0.6167797650432055, "lo": 0.6105579967897169, "hi": 0.6238108651779094 }, "jsd": { "value": 0.1860303969380419, "lo": 0.18435871097790205, "hi": 0.1879150410105986 }, "brier": { "value": 0.4287724273270234, "lo": 0.42292768984183027, "hi": 0.4353708842047484 }, "acc": { "value": 0.6536026128107711, "lo": 0.6451184668229782, "hi": 0.661609588316768 }, "conf": { "value": 0.7026956371286125, "lo": 0.7017580592058847, "hi": 0.7035779274954069 }, "ece": { "value": 0.11113349067205512, "lo": 0.10483935635734597, "hi": 0.11866877782849507 }, "reliability": { "value": 0.019291929561390565, "lo": 0.01740403521074185, "hi": 0.021570290379250588 }, "resolution": { "value": 0.031099067312709942, "lo": 0.028930295369946016, "hi": 0.033717553045725074 }, "uncertainty": { "value": 0.22640623733770432, "lo": 0.22388234090058073, "hi": 0.2289406268460248 }, "auroc": { "value": 0.7213528267123783, "lo": 0.7130525215248933, "hi": 0.7297975155813541 }, "acc@80": { "value": 0.7195467422096318, "lo": 0.7107755151525855, "hi": 0.7283890372652869 } } }, { "env": "mcq", "slice": "all", "system": "zs-LFM2.5-VL-1.6B-prefill", "metrics": { "n": { "value": 5064.0, "lo": 5064.0, "hi": 5064.0 }, "ce": { "value": 0.567471638295474, "lo": 0.5355246464459119, "hi": 0.5998490333572554 }, "kl": { "value": 0.567471638295474, "lo": 0.5355246464459119, "hi": 0.5998490333572554 }, "jsd": { "value": 0.12043067454541828, "lo": 0.11526442191156205, "hi": 0.12609448505334858 }, "brier": { "value": 0.27981321356917266, "lo": 0.26538361838399294, "hi": 0.2954207827576226 }, "acc": { "value": 0.8104265402843602, "lo": 0.8001579778830964, "hi": 0.8204038309636651 }, "conf": { "value": 0.8767070750842727, "lo": 0.8721753720947398, "hi": 0.8810695888627598 }, "ece": { "value": 0.0663454392910192, "lo": 0.05677600162439224, "hi": 0.0766042745597131 }, "reliability": { "value": 0.007613211206775605, "lo": 0.005713414187734919, "hi": 0.01029174037611074 }, "resolution": { "value": 0.03825688826641818, "lo": 0.034504085352924735, "hi": 0.04331040736302365 }, "uncertainty": { "value": 0.15363536308708248, "lo": 0.14734137537934652, "hi": 0.1599051883131306 }, "auroc": { "value": 0.8459082094704353, "lo": 0.8333766358908723, "hi": 0.8573552144558576 }, "acc@80": { "value": 0.8908911379906196, "lo": 0.8797827696864972, "hi": 0.8996605776351518 } } }, { "env": "mcq", "slice": "all", "system": "zs-LFM2.5-VL-1.6B-prefill +T", "metrics": { "n": { "value": 5064.0, "lo": 5064.0, "hi": 5064.0 }, "ce": { "value": 0.5269270169220264, "lo": 0.5048392107778062, "hi": 0.5501583389985965 }, "kl": { "value": 0.5269270169220264, "lo": 0.5048392107778062, "hi": 0.5501583389985965 }, "jsd": { "value": 0.13504835666232706, "lo": 0.13062475564306758, "hi": 0.1398825440824005 }, "brier": { "value": 0.2693563783138094, "lo": 0.2577888016783025, "hi": 0.28193399649932643 }, "acc": { "value": 0.8104265402843602, "lo": 0.8001579778830964, "hi": 0.8204038309636651 }, "conf": { "value": 0.8002951632916763, "lo": 0.7950182083441955, "hi": 0.8050914600543674 }, "ece": { "value": 0.020351312816510297, "lo": 0.01712466743891839, "hi": 0.030929943004074326 }, "reliability": { "value": 0.00048451933467936613, "lo": 0.0004253497776299618, "hi": 0.0013789222022809087 }, "resolution": { "value": 0.03855886462790291, "lo": 0.03457260919057467, "hi": 0.043692728924570434 }, "uncertainty": { "value": 0.15363536308708248, "lo": 0.14734137537934652, "hi": 0.1599051883131306 }, "auroc": { "value": 0.8465923743096166, "lo": 0.8337513744168854, "hi": 0.8583663363199558 }, "acc@80": { "value": 0.8913848432485806, "lo": 0.8813811404591458, "hi": 0.9008948407800543 } } }, { "env": "stsb", "slice": "all", "system": "zs-LFM2.5-VL-1.6B-prefill", "metrics": { "n": { "value": 3676.0, "lo": 3150.95, "hi": 4793.674999999996 }, "ce": { "value": 2.4949856820898892, "lo": 2.403966048289237, "hi": 2.5998394086267878 }, "kl": { "value": 2.092681859227634, "lo": 1.997292570579143, "hi": 2.201441225034531 }, "jsd": { "value": 0.3968087037540929, "lo": 0.38052443084813153, "hi": 0.4154332315379364 }, "brier": { "value": 0.824537909273267, "lo": 0.794885561707944, "hi": 0.850379683918623 }, "acc": { "value": 0.23449401523394994, "lo": 0.20829456008503736, "hi": 0.26186580018791106 }, "conf": { "value": 0.7117134036984745, "lo": 0.6963201652904776, "hi": 0.7221477089054564 }, "ece": { "value": 0.4772193884645245, "lo": 0.4590508668907983, "hi": 0.4923052034116568 }, "reliability": { "value": 0.23199332828556152, "lo": 0.21560957449199486, "hi": 0.24716219570305237 }, "resolution": { "value": 0.015095316428784285, "lo": 0.01132341751077712, "hi": 0.020081745389184037 }, "uncertainty": { "value": 0.17950657205341, "lo": 0.16490790178615253, "hi": 0.19329210120204962 }, "auroc": { "value": 0.6818591827076088, "lo": 0.6393946118136222, "hi": 0.729803709649928 }, "acc@80": { "value": 0.286977218633118, "lo": 0.2565269320815032, "hi": 0.3127481120442128 } } }, { "env": "stsb", "slice": "all", "system": "zs-LFM2.5-VL-1.6B-prefill +T", "metrics": { "n": { "value": 3676.0, "lo": 3150.95, "hi": 4793.674999999996 }, "ce": { "value": 1.749101240806852, "lo": 1.7283041885675015, "hi": 1.774187438110359 }, "kl": { "value": 1.3467974179445963, "lo": 1.3195490845977145, "hi": 1.3764656306697425 }, "jsd": { "value": 0.3572562508931146, "lo": 0.3517192084644762, "hi": 0.36339969260996136 }, "brier": { "value": 0.5444155979698277, "lo": 0.5319484618563186, "hi": 0.5569032296162743 }, "acc": { "value": 0.23449401523394994, "lo": 0.20829456008503736, "hi": 0.26186580018791106 }, "conf": { "value": 0.26412474790906104, "lo": 0.26161907123082884, "hi": 0.2658817355503822 }, "ece": { "value": 0.09120896808132307, "lo": 0.07239989869480402, "hi": 0.11569162818968533 }, "reliability": { "value": 0.013670878123295767, "lo": 0.009786623552048445, "hi": 0.01985571136586088 }, "resolution": { "value": 0.01538556123039212, "lo": 0.011344999347197309, "hi": 0.02065854047173325 }, "uncertainty": { "value": 0.17950657205341, "lo": 0.16490790178615253, "hi": 0.19329210120204962 }, "auroc": { "value": 0.6692634771122841, "lo": 0.6256132861111295, "hi": 0.7196155209077784 }, "acc@80": { "value": 0.2852771166269976, "lo": 0.2552077935193685, "hi": 0.31323414252153486 } } }, { "env": "yelp", "slice": "all", "system": "zs-LFM2.5-VL-1.6B-prefill", "metrics": { "n": { "value": 5000.0, "lo": 5000.0, "hi": 5000.0 }, "ce": { "value": 1.2795574641538, "lo": 1.2517782952036682, "hi": 1.3058245794645162 }, "kl": { "value": 1.2795574641538, "lo": 1.2517782952036682, "hi": 1.3058245794645162 }, "jsd": { "value": 0.31409994204249014, "lo": 0.3085733024183901, "hi": 0.3189092146477358 }, "brier": { "value": 0.6701537111869355, "lo": 0.6550587774857013, "hi": 0.6837073153760602 }, "acc": { "value": 0.4778, "lo": 0.463495, "hi": 0.4916 }, "conf": { "value": 0.6009585103601217, "lo": 0.5957442806113511, "hi": 0.6065007946558297 }, "ece": { "value": 0.13250732738077642, "lo": 0.12254822125673294, "hi": 0.14733026904314755 }, "reliability": { "value": 0.02536507914825806, "lo": 0.021953854325576682, "hi": 0.030694678524850767 }, "resolution": { "value": 0.019443062968837098, "lo": 0.01639166898892588, "hi": 0.02351216928237807 }, "uncertainty": { "value": 0.24950716, "lo": 0.248667375, "hi": 0.24992943999999997 }, "auroc": { "value": 0.6292576132885325, "lo": 0.6118137052534594, "hi": 0.6438647698650272 }, "acc@80": { "value": 0.504, "lo": 0.48886875, "hi": 0.5198812500000001 } } }, { "env": "yelp", "slice": "all", "system": "zs-LFM2.5-VL-1.6B-prefill +T", "metrics": { "n": { "value": 5000.0, "lo": 5000.0, "hi": 5000.0 }, "ce": { "value": 1.2467077560785884, "lo": 1.2253371183672128, "hi": 1.2665288437688031 }, "kl": { "value": 1.2467077560785884, "lo": 1.2253371183672128, "hi": 1.2665288437688031 }, "jsd": { "value": 0.3246998353081678, "lo": 0.3201356534839991, "hi": 0.3287172198525013 }, "brier": { "value": 0.6477096486623636, "lo": 0.6355266875009784, "hi": 0.6583932973272311 }, "acc": { "value": 0.4778, "lo": 0.463495, "hi": 0.4916 }, "conf": { "value": 0.512566381017171, "lo": 0.5078738480498601, "hi": 0.5175937098496847 }, "ece": { "value": 0.07436672394824685, "lo": 0.06416638063069739, "hi": 0.09020058880391907 }, "reliability": { "value": 0.006848674886070009, "lo": 0.005376357899329489, "hi": 0.009904528379284263 }, "resolution": { "value": 0.019817199380698064, "lo": 0.016920157765347506, "hi": 0.02387091143686241 }, "uncertainty": { "value": 0.24950716, "lo": 0.248667375, "hi": 0.24992943999999997 }, "auroc": { "value": 0.6293416990518429, "lo": 0.6121168328862725, "hi": 0.6439684894481816 }, "acc@80": { "value": 0.50325, "lo": 0.48861875, "hi": 0.5188812500000001 } } }, { "env": "boolq", "slice": "all", "system": "LFM2.5-VL-3B", "metrics": { "n": { "value": 8129.0, "lo": 7984.475, "hi": 8270.575 }, "ce": { "value": 0.5328583476623585, "lo": 0.5034779386975547, "hi": 0.5645709102211561 }, "kl": { "value": 0.5328583476623585, "lo": 0.5034779386975547, "hi": 0.5645709102211561 }, "jsd": { "value": 0.10291671049966716, "lo": 0.09828475981022308, "hi": 0.10777257902109055 }, "brier": { "value": 0.2784675294234334, "lo": 0.26475431357101853, "hi": 0.2925826290127473 }, "acc": { "value": 0.8294993234100135, "lo": 0.8200077032444733, "hi": 0.8384865413882864 }, "conf": { "value": 0.9367556756295641, "lo": 0.9339936626756461, "hi": 0.939232940598248 }, "ece": { "value": 0.10725635221955052, "lo": 0.0988807214514619, "hi": 0.11561362088654317 }, "reliability": { "value": 0.016289727611228377, "lo": 0.014055659763456274, "hi": 0.01910470563320159 }, "resolution": { "value": 0.018139487624438805, "lo": 0.016009183270894112, "hi": 0.020861142138065423 }, "uncertainty": { "value": 0.14143019587234332, "lo": 0.1354268563237357, "hi": 0.14759506919989493 }, "auroc": { "value": 0.7673957322852474, "lo": 0.754213355432779, "hi": 0.7805694196890747 }, "acc@80": { "value": 0.8829770874980778, "lo": 0.8731227133062054, "hi": 0.8917302633474256 } } }, { "env": "boolq", "slice": "all", "system": "LFM2.5-VL-3B +T", "metrics": { "n": { "value": 8129.0, "lo": 7984.475, "hi": 8270.575 }, "ce": { "value": 0.39341402832620836, "lo": 0.3793506116651023, "hi": 0.40836095134841677 }, "kl": { "value": 0.39341402832620836, "lo": 0.3793506116651023, "hi": 0.40836095134841677 }, "jsd": { "value": 0.1096984527809455, "lo": 0.10643592813274551, "hi": 0.11335693290663253 }, "brier": { "value": 0.24814633560253155, "lo": 0.2377302397014397, "hi": 0.25936932864393564 }, "acc": { "value": 0.8294993234100135, "lo": 0.8200077032444733, "hi": 0.8384865413882864 }, "conf": { "value": 0.8452801117521198, "lo": 0.8419216373999594, "hi": 0.8483471561336406 }, "ece": { "value": 0.02446798654186068, "lo": 0.020447147595451493, "hi": 0.03533475737885574 }, "reliability": { "value": 0.0011106918261193821, "lo": 0.0007329247301666026, "hi": 0.002098926828915632 }, "resolution": { "value": 0.018139487624438805, "lo": 0.016009183270894112, "hi": 0.020861142138065423 }, "uncertainty": { "value": 0.14143019587234332, "lo": 0.1354268563237357, "hi": 0.14759506919989493 }, "auroc": { "value": 0.7673957322852474, "lo": 0.754213355432779, "hi": 0.7805694196890747 }, "acc@80": { "value": 0.8829770874980778, "lo": 0.8731227133062054, "hi": 0.8917302633474256 } } }, { "env": "chaosnli", "slice": "all", "system": "LFM2.5-VL-3B", "metrics": { "n": { "value": 2763.0, "lo": 2732.475, "hi": 2798.525 }, "ce": { "value": 1.4463869206613593, "lo": 1.3969440713345973, "hi": 1.496035654471535 }, "kl": { "value": 0.9160268505920974, "lo": 0.8695692282664632, "hi": 0.9636039120549132 }, "jsd": { "value": 0.12339426634731591, "lo": 0.11785326462702327, "hi": 0.1285760716912485 }, "brier": { "value": 0.3256343876098984, "lo": 0.30917852060517886, "hi": 0.3416501268543098 }, "acc": { "value": 0.66847629388346, "lo": 0.6527744206804789, "hi": 0.685302583509242 }, "conf": { "value": 0.8955592298977969, "lo": 0.8907141895012433, "hi": 0.901283667606361 }, "ece": { "value": 0.22708293601433696, "lo": 0.21198514666343163, "hi": 0.2429271002362792 }, "reliability": { "value": 0.057846081841530395, "lo": 0.05095878852089352, "hi": 0.06702085714378257 }, "resolution": { "value": 0.01831117311529517, "lo": 0.015418712607187562, "hi": 0.02377916072160715 }, "uncertainty": { "value": 0.22161573839929402, "lo": 0.21566295244340217, "hi": 0.22665997607375962 }, "auroc": { "value": 0.6697400245411537, "lo": 0.649998624891882, "hi": 0.6903150321308094 }, "acc@80": { "value": 0.716289592760181, "lo": 0.6996120589974848, "hi": 0.7357207615593835 } } }, { "env": "chaosnli", "slice": "all", "system": "LFM2.5-VL-3B +T", "metrics": { "n": { "value": 2763.0, "lo": 2732.475, "hi": 2798.525 }, "ce": { "value": 0.7944889838081668, "lo": 0.7817771260528582, "hi": 0.8063845883673213 }, "kl": { "value": 0.26412891373890496, "lo": 0.2543928784905818, "hi": 0.274557945037211 }, "jsd": { "value": 0.07146401553311572, "lo": 0.06888100386968574, "hi": 0.07407442993902372 }, "brier": { "value": 0.1728068690748953, "lo": 0.16526219560988412, "hi": 0.1810891866882665 }, "acc": { "value": 0.66847629388346, "lo": 0.6527744206804789, "hi": 0.685302583509242 }, "conf": { "value": 0.6224814034196495, "lo": 0.618021338576969, "hi": 0.6276350420443061 }, "ece": { "value": 0.04958543734846641, "lo": 0.04367292327058699, "hi": 0.0705175026939239 }, "reliability": { "value": 0.0036959336504425217, "lo": 0.003028344555863666, "hi": 0.0073011007642513155 }, "resolution": { "value": 0.0162851595288571, "lo": 0.013386979538948246, "hi": 0.02120530567234291 }, "uncertainty": { "value": 0.22161573839929402, "lo": 0.21566295244340217, "hi": 0.22665997607375962 }, "auroc": { "value": 0.6575504240323622, "lo": 0.6363788080021665, "hi": 0.6796131873409706 }, "acc@80": { "value": 0.7180995475113122, "lo": 0.6998164435534286, "hi": 0.7346567620671158 } } }, { "env": "civilcomments", "slice": "all", "system": "LFM2.5-VL-3B", "metrics": { "n": { "value": 10581.0, "lo": 10261.425, "hi": 10904.525 }, "ce": { "value": 0.5077022501274139, "lo": 0.4961021782471906, "hi": 0.5180279389922728 }, "kl": { "value": 0.27621944135807286, "lo": 0.26730500664356843, "hi": 0.2835696862919652 }, "jsd": { "value": 0.07257758135861772, "lo": 0.07055121962272641, "hi": 0.07418253016567902 }, "brier": { "value": 0.17210397486649306, "lo": 0.16552973030690482, "hi": 0.17740769611117288 }, "acc": { "value": 0.8038937718552122, "lo": 0.795871605523184, "hi": 0.8129462172944398 }, "conf": { "value": 0.8094657350832546, "lo": 0.8063933901096233, "hi": 0.8122725926216197 }, "ece": { "value": 0.02503634923212628, "lo": 0.023475550226927357, "hi": 0.03463331161127155 }, "reliability": { "value": 0.0014436016102529808, "lo": 0.0010643736014497817, "hi": 0.002367815593946149 }, "resolution": { "value": 0.012833172824697326, "lo": 0.011369842633845467, "hi": 0.014824249058854993 }, "uncertainty": { "value": 0.15764857542761226, "lo": 0.15206466487253756, "hi": 0.1624599924504572 }, "auroc": { "value": 0.6998776200499152, "lo": 0.6875208159261627, "hi": 0.7120482908431496 }, "acc@80": { "value": 0.8473715298287064, "lo": 0.8399738848348878, "hi": 0.8569233002961383 } } }, { "env": "civilcomments", "slice": "all", "system": "LFM2.5-VL-3B +T", "metrics": { "n": { "value": 10581.0, "lo": 10261.425, "hi": 10904.525 }, "ce": { "value": 0.49676150002708036, "lo": 0.4872737952681122, "hi": 0.5053113122868408 }, "kl": { "value": 0.2652786912577394, "lo": 0.25799358231107855, "hi": 0.2712157832203521 }, "jsd": { "value": 0.07525692086753505, "lo": 0.07339140236779222, "hi": 0.07673566141619703 }, "brier": { "value": 0.16868331824137972, "lo": 0.1629474417311238, "hi": 0.17327347459413364 }, "acc": { "value": 0.8038937718552122, "lo": 0.795871605523184, "hi": 0.8129462172944398 }, "conf": { "value": 0.7700251446366264, "lo": 0.767086855986615, "hi": 0.7728444308829732 }, "ece": { "value": 0.035339698457243184, "lo": 0.030404589401963122, "hi": 0.044576461852605574 }, "reliability": { "value": 0.002158982108121902, "lo": 0.0015709597177534181, "hi": 0.003239906426687996 }, "resolution": { "value": 0.012833172824697326, "lo": 0.011369842633845467, "hi": 0.014824249058854993 }, "uncertainty": { "value": 0.15764857542761226, "lo": 0.15206466487253756, "hi": 0.1624599924504572 }, "auroc": { "value": 0.6998776200499152, "lo": 0.6875208159261627, "hi": 0.7120482908431496 }, "acc@80": { "value": 0.8473715298287064, "lo": 0.8399738848348878, "hi": 0.8569233002961383 } } }, { "env": "folktables", "slice": "all", "system": "LFM2.5-VL-3B", "metrics": { "n": { "value": 15003.0, "lo": 14855.8, "hi": 15157.0 }, "ce": { "value": 0.6757370971930118, "lo": 0.6632898334485878, "hi": 0.6906677526353899 }, "kl": { "value": 0.6757370971930118, "lo": 0.6632898334485878, "hi": 0.6906677526353899 }, "jsd": { "value": 0.17894972031199147, "lo": 0.1762759338754224, "hi": 0.18216940431510975 }, "brier": { "value": 0.44920946997907557, "lo": 0.44049943112591494, "hi": 0.4595456620134782 }, "acc": { "value": 0.6783976538025728, "lo": 0.6702337979068954, "hi": 0.6859475904305837 }, "conf": { "value": 0.7932900791326173, "lo": 0.7909126609219911, "hi": 0.7954442834879843 }, "ece": { "value": 0.11489242533004451, "lo": 0.10689556856059901, "hi": 0.122851500909695 }, "reliability": { "value": 0.015857651914882486, "lo": 0.01431326500314493, "hi": 0.018158490729361654 }, "resolution": { "value": 0.009356267506864168, "lo": 0.008159686778144225, "hi": 0.010873125948180637 }, "uncertainty": { "value": 0.2181742771177374, "lo": 0.21542349156276827, "hi": 0.2210204522057745 }, "auroc": { "value": 0.6186877619818016, "lo": 0.609356710370421, "hi": 0.6281315833357176 }, "acc@80": { "value": 0.717380436593901, "lo": 0.7081394662921349, "hi": 0.7246021461791023 } } }, { "env": "folktables", "slice": "all", "system": "LFM2.5-VL-3B +T", "metrics": { "n": { "value": 15003.0, "lo": 14855.8, "hi": 15157.0 }, "ce": { "value": 0.609676541729926, "lo": 0.60345749975659, "hi": 0.6172599009517398 }, "kl": { "value": 0.609676541729926, "lo": 0.60345749975659, "hi": 0.6172599009517398 }, "jsd": { "value": 0.18367213312589367, "lo": 0.1820172407129862, "hi": 0.18564696915229778 }, "brier": { "value": 0.4202375772553369, "lo": 0.4146823417587002, "hi": 0.42692297904544924 }, "acc": { "value": 0.6783976538025728, "lo": 0.6702337979068954, "hi": 0.6859475904305837 }, "conf": { "value": 0.6819142642418174, "lo": 0.6801775258058251, "hi": 0.6835818231655569 }, "ece": { "value": 0.02830506018556748, "lo": 0.02470489534954933, "hi": 0.03731011481623176 }, "reliability": { "value": 0.0014208268752148375, "lo": 0.0010898369561207933, "hi": 0.002237282144687026 }, "resolution": { "value": 0.009356267506864168, "lo": 0.008159686778144225, "hi": 0.010873125948180637 }, "uncertainty": { "value": 0.2181742771177374, "lo": 0.21542349156276827, "hi": 0.2210204522057745 }, "auroc": { "value": 0.6186877619818016, "lo": 0.609356710370421, "hi": 0.6281315833357176 }, "acc@80": { "value": 0.717380436593901, "lo": 0.7081394662921349, "hi": 0.7246021461791023 } } }, { "env": "mcq", "slice": "all", "system": "LFM2.5-VL-3B", "metrics": { "n": { "value": 5064.0, "lo": 5064.0, "hi": 5064.0 }, "ce": { "value": 0.5759872670241797, "lo": 0.5329584569816519, "hi": 0.6265682381052227 }, "kl": { "value": 0.5759872670241797, "lo": 0.5329584569816519, "hi": 0.6265682381052227 }, "jsd": { "value": 0.08353110019922566, "lo": 0.07820357437215822, "hi": 0.08958165164902972 }, "brier": { "value": 0.22348060549884158, "lo": 0.20822744166423812, "hi": 0.2404070146904414 }, "acc": { "value": 0.868088467614534, "lo": 0.858901066350711, "hi": 0.8766834518167457 }, "conf": { "value": 0.9615081339474255, "lo": 0.9586090640788736, "hi": 0.9641287516267002 }, "ece": { "value": 0.09343844024232026, "lo": 0.08509777928960008, "hi": 0.1014441097433286 }, "reliability": { "value": 0.019725029048978552, "lo": 0.016627496437260518, "hi": 0.023582789751029707 }, "resolution": { "value": 0.028799536189915965, "lo": 0.02509962954635008, "hi": 0.03310776419602363 }, "uncertainty": { "value": 0.11451088000918419, "lo": 0.1081095674029609, "hi": 0.12119001484786204 }, "auroc": { "value": 0.8771137206932813, "lo": 0.866687062555456, "hi": 0.8878373819364807 }, "acc@80": { "value": 0.9427301900765244, "lo": 0.9355714638360898, "hi": 0.9497716613181931 } } }, { "env": "mcq", "slice": "all", "system": "LFM2.5-VL-3B +T", "metrics": { "n": { "value": 5064.0, "lo": 5064.0, "hi": 5064.0 }, "ce": { "value": 0.3906034227460842, "lo": 0.37137167691447304, "hi": 0.41206851196394295 }, "kl": { "value": 0.3906034227460842, "lo": 0.37137167691447304, "hi": 0.41206851196394295 }, "jsd": { "value": 0.10002890622380546, "lo": 0.09610960090860976, "hi": 0.10440735944956796 }, "brier": { "value": 0.1937977066232963, "lo": 0.18263927510816466, "hi": 0.20566609163289462 }, "acc": { "value": 0.868088467614534, "lo": 0.858901066350711, "hi": 0.8766834518167457 }, "conf": { "value": 0.8558151643390487, "lo": 0.8511140746036796, "hi": 0.8593811069517472 }, "ece": { "value": 0.0305649517438755, "lo": 0.026309415051930726, "hi": 0.039549654257689214 }, "reliability": { "value": 0.0010577246490419798, "lo": 0.0008757950986678834, "hi": 0.0019207445946941867 }, "resolution": { "value": 0.029804711091153086, "lo": 0.026167586198610596, "hi": 0.03420775328193042 }, "uncertainty": { "value": 0.11451088000918419, "lo": 0.1081095674029609, "hi": 0.12119001484786204 }, "auroc": { "value": 0.8805730781385364, "lo": 0.8702694072406397, "hi": 0.8911781380761858 }, "acc@80": { "value": 0.9449518637373489, "lo": 0.9370525796099728, "hi": 0.9511231794618613 } } }, { "env": "stsb", "slice": "all", "system": "LFM2.5-VL-3B", "metrics": { "n": { "value": 3676.0, "lo": 3150.95, "hi": 4793.674999999996 }, "ce": { "value": 2.6978885169981255, "lo": 2.5657299873689126, "hi": 2.8476372111272323 }, "kl": { "value": 2.2955846941358695, "lo": 2.15679378594023, "hi": 2.4465032494598424 }, "jsd": { "value": 0.3275760175289124, "lo": 0.3162216290936204, "hi": 0.3390716580405639 }, "brier": { "value": 0.6919414239196017, "lo": 0.6638624637074109, "hi": 0.7198471346121803 }, "acc": { "value": 0.3520130576713819, "lo": 0.33738188605734065, "hi": 0.36734544487166837 }, "conf": { "value": 0.7248023496748184, "lo": 0.7112682362621677, "hi": 0.7419135614849961 }, "ece": { "value": 0.3727892920034365, "lo": 0.34990632614727657, "hi": 0.39600605273239675 }, "reliability": { "value": 0.1495211764503258, "lo": 0.13412612551018585, "hi": 0.1686786364778362 }, "resolution": { "value": 0.007239650807423352, "lo": 0.005789911237959588, "hi": 0.010394805507148573 }, "uncertainty": { "value": 0.22809986490022627, "lo": 0.22355534383864203, "hi": 0.23240276779454366 }, "auroc": { "value": 0.5737491191665466, "lo": 0.5525762701641901, "hi": 0.5929816932597282 }, "acc@80": { "value": 0.3651819109146549, "lo": 0.34728186363682, "hi": 0.3814624881124586 } } }, { "env": "stsb", "slice": "all", "system": "LFM2.5-VL-3B +T", "metrics": { "n": { "value": 3676.0, "lo": 3150.95, "hi": 4793.674999999996 }, "ce": { "value": 1.5758103744712262, "lo": 1.552264621029743, "hi": 1.6003796429076587 }, "kl": { "value": 1.1735065516089704, "lo": 1.1455278721099897, "hi": 1.2018428465852689 }, "jsd": { "value": 0.3105009562586476, "lo": 0.3048929847451175, "hi": 0.31570691507361504 }, "brier": { "value": 0.48641483403122787, "lo": 0.4735061300168612, "hi": 0.49741827930201415 }, "acc": { "value": 0.3520130576713819, "lo": 0.33738188605734065, "hi": 0.36734544487166837 }, "conf": { "value": 0.3528447149650681, "lo": 0.3482181155596457, "hi": 0.35906599390698796 }, "ece": { "value": 0.04800463541314871, "lo": 0.041971453587293386, "hi": 0.07054283841584154 }, "reliability": { "value": 0.0058868450744030375, "lo": 0.0038970035004699415, "hi": 0.008971929159702999 }, "resolution": { "value": 0.012971840177239595, "lo": 0.009013087833499103, "hi": 0.017507180870015102 }, "uncertainty": { "value": 0.22809986490022627, "lo": 0.22355534383864203, "hi": 0.23240276779454366 }, "auroc": { "value": 0.6127025592510548, "lo": 0.5903542080481157, "hi": 0.633554195636134 }, "acc@80": { "value": 0.37810268616116965, "lo": 0.3606827313509045, "hi": 0.39444786464976245 } } }, { "env": "yelp", "slice": "all", "system": "LFM2.5-VL-3B", "metrics": { "n": { "value": 5000.0, "lo": 5000.0, "hi": 5000.0 }, "ce": { "value": 1.3860599847549677, "lo": 1.3290156657155492, "hi": 1.4319837141033827 }, "kl": { "value": 1.3860599847549677, "lo": 1.3290156657155492, "hi": 1.4319837141033827 }, "jsd": { "value": 0.24637008755393536, "lo": 0.23853545745746285, "hi": 0.2536433407530511 }, "brier": { "value": 0.6418344653090541, "lo": 0.6195384048344682, "hi": 0.6631553474924564 }, "acc": { "value": 0.5928, "lo": 0.578695, "hi": 0.6074 }, "conf": { "value": 0.8411075247108937, "lo": 0.8362002471122145, "hi": 0.8460747388541698 }, "ece": { "value": 0.24830752471089362, "lo": 0.2347119203391671, "hi": 0.26253560655146835 }, "reliability": { "value": 0.07551728217756373, "lo": 0.06811680137418837, "hi": 0.08436615957078013 }, "resolution": { "value": 0.021735186667385942, "lo": 0.018947388334327648, "hi": 0.025708930635887053 }, "uncertainty": { "value": 0.24138816, "lo": 0.23846524, "hi": 0.24380708699999998 }, "auroc": { "value": 0.6629740249066068, "lo": 0.6476209612008791, "hi": 0.6784682418750948 }, "acc@80": { "value": 0.626, "lo": 0.61036875, "hi": 0.64305 } } }, { "env": "yelp", "slice": "all", "system": "LFM2.5-VL-3B +T", "metrics": { "n": { "value": 5000.0, "lo": 5000.0, "hi": 5000.0 }, "ce": { "value": 0.9724022468050197, "lo": 0.9483220949408991, "hi": 0.9937898554692086 }, "kl": { "value": 0.9724022468050197, "lo": 0.9483220949408991, "hi": 0.9937898554692086 }, "jsd": { "value": 0.2549887608925926, "lo": 0.2497078453139047, "hi": 0.2595903865139499 }, "brier": { "value": 0.5386607794010954, "lo": 0.5238223524042669, "hi": 0.5520623252485813 }, "acc": { "value": 0.5928, "lo": 0.578695, "hi": 0.6074 }, "conf": { "value": 0.6407830723416755, "lo": 0.6355987860389466, "hi": 0.6457528767574988 }, "ece": { "value": 0.06264510225858502, "lo": 0.054522714912741445, "hi": 0.07988027057516923 }, "reliability": { "value": 0.005896093809502374, "lo": 0.004641341937083263, "hi": 0.008890860101156363 }, "resolution": { "value": 0.020065581733110356, "lo": 0.017560380981010818, "hi": 0.02410640542155473 }, "uncertainty": { "value": 0.24138816, "lo": 0.23846524, "hi": 0.24380708699999998 }, "auroc": { "value": 0.6625109533127059, "lo": 0.6478243237246345, "hi": 0.6778121607764245 }, "acc@80": { "value": 0.63075, "lo": 0.61384375, "hi": 0.64675 } } }, { "env": "boolq", "slice": "all", "system": "zs-LFM2.5-VL-3B-prefill", "metrics": { "n": { "value": 8129.0, "lo": 7984.475, "hi": 8270.575 }, "ce": { "value": 0.5816780320630307, "lo": 0.5487705254824645, "hi": 0.6153245192490632 }, "kl": { "value": 0.5816780320630307, "lo": 0.5487705254824645, "hi": 0.6153245192490632 }, "jsd": { "value": 0.10986828522452227, "lo": 0.1050696462488618, "hi": 0.1150266473853312 }, "brier": { "value": 0.2965075321276258, "lo": 0.2821260469188488, "hi": 0.3115855130827731 }, "acc": { "value": 0.8144913273465371, "lo": 0.8049283683473497, "hi": 0.8234987300629023 }, "conf": { "value": 0.9305177249875012, "lo": 0.9276152868000491, "hi": 0.9331465415011992 }, "ece": { "value": 0.11602639764096409, "lo": 0.10767501255807296, "hi": 0.12471576912306866 }, "reliability": { "value": 0.016211774414308433, "lo": 0.013994393412881936, "hi": 0.019032058298974353 }, "resolution": { "value": 0.018600505289024332, "lo": 0.016417993663277192, "hi": 0.021259464248592964 }, "uncertainty": { "value": 0.15109520502381327, "lo": 0.14534857005625082, "hi": 0.15701869003971955 }, "auroc": { "value": 0.7468794030888777, "lo": 0.7331697018418196, "hi": 0.7595890184078872 }, "acc@80": { "value": 0.8699061971397817, "lo": 0.8604725626403062, "hi": 0.8787990101196874 } } }, { "env": "boolq", "slice": "all", "system": "zs-LFM2.5-VL-3B-prefill +T", "metrics": { "n": { "value": 8129.0, "lo": 7984.475, "hi": 8270.575 }, "ce": { "value": 0.42064213095297337, "lo": 0.40627805325118327, "hi": 0.43623354700168643 }, "kl": { "value": 0.42064213095297337, "lo": 0.40627805325118327, "hi": 0.43623354700168643 }, "jsd": { "value": 0.11765313022620086, "lo": 0.1142472051321885, "hi": 0.12136347298891204 }, "brier": { "value": 0.2654743249822674, "lo": 0.25476302978534976, "hi": 0.2768470924704069 }, "acc": { "value": 0.8144913273465371, "lo": 0.8049283683473497, "hi": 0.8234987300629023 }, "conf": { "value": 0.8300160892799145, "lo": 0.8266725368018208, "hi": 0.832904834456254 }, "ece": { "value": 0.017711542259073728, "lo": 0.01568471163053336, "hi": 0.0289449105887779 }, "reliability": { "value": 0.0006015042336070265, "lo": 0.00040831449968761535, "hi": 0.001444573067852351 }, "resolution": { "value": 0.018600505289024332, "lo": 0.016417993663277192, "hi": 0.021259464248592964 }, "uncertainty": { "value": 0.15109520502381327, "lo": 0.14534857005625082, "hi": 0.15701869003971955 }, "auroc": { "value": 0.7468794030888777, "lo": 0.7331697018418196, "hi": 0.7595890184078872 }, "acc@80": { "value": 0.8699061971397817, "lo": 0.8604725626403062, "hi": 0.8787990101196874 } } }, { "env": "chaosnli", "slice": "all", "system": "zs-LFM2.5-VL-3B-prefill", "metrics": { "n": { "value": 2763.0, "lo": 2732.475, "hi": 2798.525 }, "ce": { "value": 1.4204363394402675, "lo": 1.3710480649425658, "hi": 1.46618904115787 }, "kl": { "value": 0.8900762693710054, "lo": 0.8454883049121936, "hi": 0.9338853046818825 }, "jsd": { "value": 0.12223944325440186, "lo": 0.11642119374214775, "hi": 0.127179793442689 }, "brier": { "value": 0.3237633895884616, "lo": 0.3063289695870303, "hi": 0.3388995012898196 }, "acc": { "value": 0.664133188563156, "lo": 0.6471050323227849, "hi": 0.6809085836106766 }, "conf": { "value": 0.8941022409095827, "lo": 0.8889790748660467, "hi": 0.899864060352879 }, "ece": { "value": 0.22996905234642664, "lo": 0.214183808314927, "hi": 0.24590967789885282 }, "reliability": { "value": 0.058616589135346245, "lo": 0.05185238690359973, "hi": 0.06779369685426515 }, "resolution": { "value": 0.02073116596346457, "lo": 0.01743926586421297, "hi": 0.026394245867025382 }, "uncertainty": { "value": 0.2230602964120915, "lo": 0.21727207848417857, "hi": 0.22836008361661222 }, "auroc": { "value": 0.679469193366532, "lo": 0.6591451012025751, "hi": 0.6992481806264657 }, "acc@80": { "value": 0.7167420814479638, "lo": 0.6975548414267926, "hi": 0.7350212868900042 } } }, { "env": "chaosnli", "slice": "all", "system": "zs-LFM2.5-VL-3B-prefill +T", "metrics": { "n": { "value": 2763.0, "lo": 2732.475, "hi": 2798.525 }, "ce": { "value": 0.7949680342778785, "lo": 0.7820818091143783, "hi": 0.8070480612599928 }, "kl": { "value": 0.2646079642086167, "lo": 0.25460557149217655, "hi": 0.2749783513874871 }, "jsd": { "value": 0.07147578627026423, "lo": 0.06895103092046842, "hi": 0.07409254429129038 }, "brier": { "value": 0.17245520194055936, "lo": 0.16471963076202234, "hi": 0.18086821914340606 }, "acc": { "value": 0.664133188563156, "lo": 0.6471050323227849, "hi": 0.6809085836106766 }, "conf": { "value": 0.6241185789225275, "lo": 0.6198024413713349, "hi": 0.6292444607739884 }, "ece": { "value": 0.056334159207648, "lo": 0.045386576939123006, "hi": 0.0740956807468349 }, "reliability": { "value": 0.004457731467115396, "lo": 0.003090377892814316, "hi": 0.0074727726597290005 }, "resolution": { "value": 0.018693482513854446, "lo": 0.015458552568498226, "hi": 0.02357529165847566 }, "uncertainty": { "value": 0.2230602964120915, "lo": 0.21727207848417857, "hi": 0.22836008361661222 }, "auroc": { "value": 0.6649402776472799, "lo": 0.6420207504509805, "hi": 0.686683594932958 }, "acc@80": { "value": 0.7149321266968326, "lo": 0.6973889241402683, "hi": 0.7339677960187573 } } }, { "env": "civilcomments", "slice": "all", "system": "zs-LFM2.5-VL-3B-prefill", "metrics": { "n": { "value": 10581.0, "lo": 10261.425, "hi": 10904.525 }, "ce": { "value": 0.43727784798305874, "lo": 0.42678145217047764, "hi": 0.4476415325693049 }, "kl": { "value": 0.20579503921371767, "lo": 0.1987235586387011, "hi": 0.21235390816668306 }, "jsd": { "value": 0.05116968131305278, "lo": 0.049437446831806064, "hi": 0.052595588753110684 }, "brier": { "value": 0.11855361891618092, "lo": 0.11343503373455538, "hi": 0.12313664944780268 }, "acc": { "value": 0.8596540969662603, "lo": 0.8527050254805175, "hi": 0.8673564438680864 }, "conf": { "value": 0.8579861902861635, "lo": 0.8548736530748258, "hi": 0.8611949410419385 }, "ece": { "value": 0.0115804810761031, "lo": 0.009709700819945096, "hi": 0.020409191830464038 }, "reliability": { "value": 0.00031894320172582623, "lo": 0.0001891966330327208, "hi": 0.000863846155247233 }, "resolution": { "value": 0.01624237040360173, "lo": 0.014398505478895042, "hi": 0.018297819452531085 }, "uncertainty": { "value": 0.12064893053538385, "lo": 0.11504924303043536, "hi": 0.1255991644744319 }, "auroc": { "value": 0.79042595405832, "lo": 0.7795293025825059, "hi": 0.8014689978834265 }, "acc@80": { "value": 0.9135262847017129, "lo": 0.9061254603062078, "hi": 0.920151691035287 } } }, { "env": "civilcomments", "slice": "all", "system": "zs-LFM2.5-VL-3B-prefill +T", "metrics": { "n": { "value": 10581.0, "lo": 10261.425, "hi": 10904.525 }, "ce": { "value": 0.422683145843368, "lo": 0.4138486310025811, "hi": 0.4310942217711616 }, "kl": { "value": 0.19120033707402692, "lo": 0.18542010246649468, "hi": 0.19586281686802368 }, "jsd": { "value": 0.053667711340155445, "lo": 0.05219516817925039, "hi": 0.05481715442924367 }, "brier": { "value": 0.11366267360512225, "lo": 0.10908920692600414, "hi": 0.1175331613875349 }, "acc": { "value": 0.8596540969662603, "lo": 0.8527050254805175, "hi": 0.8673564438680864 }, "conf": { "value": 0.8151815127133928, "lo": 0.8119748811039861, "hi": 0.8183424007286001 }, "ece": { "value": 0.04447258425286756, "lo": 0.038576093724351536, "hi": 0.05067963280226556 }, "reliability": { "value": 0.0021773724847827768, "lo": 0.0017861691523396307, "hi": 0.003071245325309536 }, "resolution": { "value": 0.01624237040360173, "lo": 0.014398505478895042, "hi": 0.018297819452531085 }, "uncertainty": { "value": 0.12064893053538385, "lo": 0.11504924303043536, "hi": 0.1255991644744319 }, "auroc": { "value": 0.79042595405832, "lo": 0.7795293025825059, "hi": 0.8014689978834265 }, "acc@80": { "value": 0.9135262847017129, "lo": 0.9061254603062078, "hi": 0.920151691035287 } } }, { "env": "folktables", "slice": "all", "system": "zs-LFM2.5-VL-3B-prefill", "metrics": { "n": { "value": 15003.0, "lo": 14855.8, "hi": 15157.0 }, "ce": { "value": 0.9620101471971333, "lo": 0.9402995881311484, "hi": 0.9863653242467205 }, "kl": { "value": 0.9620101471971333, "lo": 0.9402995881311484, "hi": 0.9863653242467205 }, "jsd": { "value": 0.1987697950575701, "lo": 0.19457230876065962, "hi": 0.20330264801141593 }, "brier": { "value": 0.5668222757523853, "lo": 0.5533690394685322, "hi": 0.5807961863890092 }, "acc": { "value": 0.6554689062187562, "lo": 0.6476506602729011, "hi": 0.6634009913565764 }, "conf": { "value": 0.9244254015488902, "lo": 0.9230365547995866, "hi": 0.9257004334141172 }, "ece": { "value": 0.26895649533013394, "lo": 0.2608665257868461, "hi": 0.2770311776931512 }, "reliability": { "value": 0.0779897735496795, "lo": 0.07362743547269779, "hi": 0.08285164159173998 }, "resolution": { "value": 0.020396192527323317, "lo": 0.01845828721270402, "hi": 0.02260715738277575 }, "uncertainty": { "value": 0.2258294191991436, "lo": 0.22330011591143803, "hi": 0.22819928249323598 }, "auroc": { "value": 0.6630820901485849, "lo": 0.6535759412768337, "hi": 0.6720372414084176 }, "acc@80": { "value": 0.7175470754874188, "lo": 0.7077399384103128, "hi": 0.7245951236863847 } } }, { "env": "folktables", "slice": "all", "system": "zs-LFM2.5-VL-3B-prefill +T", "metrics": { "n": { "value": 15003.0, "lo": 14855.8, "hi": 15157.0 }, "ce": { "value": 0.617889507782528, "lo": 0.6118568964850071, "hi": 0.6249296516130413 }, "kl": { "value": 0.617889507782528, "lo": 0.6118568964850071, "hi": 0.6249296516130413 }, "jsd": { "value": 0.1869655766677398, "lo": 0.18535809436615105, "hi": 0.18885473665697913 }, "brier": { "value": 0.4285013151213943, "lo": 0.42284947531250267, "hi": 0.43501332745244986 }, "acc": { "value": 0.6554689062187562, "lo": 0.6476506602729011, "hi": 0.6634009913565764 }, "conf": { "value": 0.6880828157315765, "lo": 0.6869715589806281, "hi": 0.6891711087324444 }, "ece": { "value": 0.06847379056915753, "lo": 0.0614792476384789, "hi": 0.07600070537683982 }, "reliability": { "value": 0.009036866185085667, "lo": 0.00782397103727989, "hi": 0.010700940911057373 }, "resolution": { "value": 0.020396192527323317, "lo": 0.01845828721270402, "hi": 0.02260715738277575 }, "uncertainty": { "value": 0.2258294191991436, "lo": 0.22330011591143803, "hi": 0.22819928249323598 }, "auroc": { "value": 0.6630820901485849, "lo": 0.6535759412768337, "hi": 0.6720372414084176 }, "acc@80": { "value": 0.7175470754874188, "lo": 0.7077399384103128, "hi": 0.7245951236863847 } } }, { "env": "mcq", "slice": "all", "system": "zs-LFM2.5-VL-3B-prefill", "metrics": { "n": { "value": 5064.0, "lo": 5064.0, "hi": 5064.0 }, "ce": { "value": 0.53918624449217, "lo": 0.49918554138227583, "hi": 0.5816754544353302 }, "kl": { "value": 0.53918624449217, "lo": 0.49918554138227583, "hi": 0.5816754544353302 }, "jsd": { "value": 0.0828151557503379, "lo": 0.07740562729541314, "hi": 0.08866953235726405 }, "brier": { "value": 0.2200081923930159, "lo": 0.20467040691603539, "hi": 0.23641613618774399 }, "acc": { "value": 0.868088467614534, "lo": 0.8587035939968405, "hi": 0.8769747235387045 }, "conf": { "value": 0.9587305887038768, "lo": 0.955746518491398, "hi": 0.9612954949833582 }, "ece": { "value": 0.09069410544977557, "lo": 0.08256884661310399, "hi": 0.09929659935744335 }, "reliability": { "value": 0.01824473815330227, "lo": 0.015307869170038992, "hi": 0.021990593540975493 }, "resolution": { "value": 0.029541577003898572, "lo": 0.025870115631190058, "hi": 0.03445812620111184 }, "uncertainty": { "value": 0.11451088000918419, "lo": 0.10789005781291729, "hi": 0.1213317219292893 }, "auroc": { "value": 0.8802924746503353, "lo": 0.869610276682273, "hi": 0.8906437648543074 }, "acc@80": { "value": 0.9439644532214269, "lo": 0.9363120217230313, "hi": 0.9516168847198223 } } }, { "env": "mcq", "slice": "all", "system": "zs-LFM2.5-VL-3B-prefill +T", "metrics": { "n": { "value": 5064.0, "lo": 5064.0, "hi": 5064.0 }, "ce": { "value": 0.38602727384245317, "lo": 0.3674236940481484, "hi": 0.4071443140733702 }, "kl": { "value": 0.38602727384245317, "lo": 0.3674236940481484, "hi": 0.4071443140733702 }, "jsd": { "value": 0.09900290314036149, "lo": 0.09492639363834217, "hi": 0.10347017607384866 }, "brier": { "value": 0.1923688280406221, "lo": 0.1813023317296054, "hi": 0.20459157811467585 }, "acc": { "value": 0.868088467614534, "lo": 0.8587035939968405, "hi": 0.8769747235387045 }, "conf": { "value": 0.8574229183584634, "lo": 0.8528227884766457, "hi": 0.8610242711011652 }, "ece": { "value": 0.03227782449687891, "lo": 0.02757931630534842, "hi": 0.04139280942821726 }, "reliability": { "value": 0.0011589286567429886, "lo": 0.0008960357913958104, "hi": 0.002189673488796682 }, "resolution": { "value": 0.029976965872117813, "lo": 0.026465285875255894, "hi": 0.03492715316660374 }, "uncertainty": { "value": 0.11451088000918419, "lo": 0.10789005781291729, "hi": 0.1213317219292893 }, "auroc": { "value": 0.8834581178861567, "lo": 0.8733877344785387, "hi": 0.8935237529435539 }, "acc@80": { "value": 0.9454455689953098, "lo": 0.9380399901258949, "hi": 0.9521105899777833 } } }, { "env": "stsb", "slice": "all", "system": "zs-LFM2.5-VL-3B-prefill", "metrics": { "n": { "value": 3676.0, "lo": 3150.95, "hi": 4793.674999999996 }, "ce": { "value": 2.7571148750759926, "lo": 2.634763394031497, "hi": 2.8926179822485656 }, "kl": { "value": 2.354811052213737, "lo": 2.2283260410334957, "hi": 2.4908382476713715 }, "jsd": { "value": 0.33952460202990814, "lo": 0.3276339706798127, "hi": 0.35131355014479676 }, "brier": { "value": 0.7369648478638614, "lo": 0.707818783439776, "hi": 0.7640002657837547 }, "acc": { "value": 0.3291621327529924, "lo": 0.3153417244862836, "hi": 0.3446662586052942 }, "conf": { "value": 0.7506276568776894, "lo": 0.7392975738806389, "hi": 0.7643049989491109 }, "ece": { "value": 0.42146552412469707, "lo": 0.40124266760385313, "hi": 0.44190863029437977 }, "reliability": { "value": 0.1858167116582256, "lo": 0.17023990225473562, "hi": 0.2029554849167609 }, "resolution": { "value": 0.005736625759717607, "lo": 0.004384957131903068, "hi": 0.009514040670213124 }, "uncertainty": { "value": 0.2208144231144938, "lo": 0.21590132128117429, "hi": 0.2258713780811604 }, "auroc": { "value": 0.5797091686607281, "lo": 0.5610394723800967, "hi": 0.5973409348331422 }, "acc@80": { "value": 0.3498809928595716, "lo": 0.33342935112189204, "hi": 0.3666129559917872 } } }, { "env": "stsb", "slice": "all", "system": "zs-LFM2.5-VL-3B-prefill +T", "metrics": { "n": { "value": 3676.0, "lo": 3150.95, "hi": 4793.674999999996 }, "ce": { "value": 1.5891963814976409, "lo": 1.5669328002567837, "hi": 1.6120440647134835 }, "kl": { "value": 1.1868925586353853, "lo": 1.1599661816058862, "hi": 1.2138312690286734 }, "jsd": { "value": 0.3146720961655134, "lo": 0.3091232327268351, "hi": 0.3198014719773035 }, "brier": { "value": 0.49317121724101637, "lo": 0.4802501732716859, "hi": 0.5042819420802195 }, "acc": { "value": 0.3291621327529924, "lo": 0.3153417244862836, "hi": 0.3446662586052942 }, "conf": { "value": 0.3511478778518168, "lo": 0.3477794827058472, "hi": 0.3554372821668198 }, "ece": { "value": 0.05635346653229976, "lo": 0.04942946070230882, "hi": 0.07853436982282196 }, "reliability": { "value": 0.006045812371941183, "lo": 0.004487231318838066, "hi": 0.009358210914887517 }, "resolution": { "value": 0.011532073268055585, "lo": 0.008924448325187958, "hi": 0.016059102874527314 }, "uncertainty": { "value": 0.2208144231144938, "lo": 0.21590132128117429, "hi": 0.2258713780811604 }, "auroc": { "value": 0.6206665192066652, "lo": 0.5980069035086956, "hi": 0.6401835762902832 }, "acc@80": { "value": 0.3594015640938456, "lo": 0.34184327632218964, "hi": 0.3758668855010105 } } }, { "env": "yelp", "slice": "all", "system": "zs-LFM2.5-VL-3B-prefill", "metrics": { "n": { "value": 5000.0, "lo": 5000.0, "hi": 5000.0 }, "ce": { "value": 1.2918572553652283, "lo": 1.2399163291387825, "hi": 1.333798228136019 }, "kl": { "value": 1.2918572553652283, "lo": 1.2399163291387825, "hi": 1.333798228136019 }, "jsd": { "value": 0.24252614524047386, "lo": 0.23506189151301068, "hi": 0.24962484157285553 }, "brier": { "value": 0.627556481522769, "lo": 0.6061041358647522, "hi": 0.6478518299527196 }, "acc": { "value": 0.5952, "lo": 0.580695, "hi": 0.6106 }, "conf": { "value": 0.8340039846897125, "lo": 0.8292275972224772, "hi": 0.8386646578364074 }, "ece": { "value": 0.23880398468971253, "lo": 0.22476129868015646, "hi": 0.25315469246715305 }, "reliability": { "value": 0.07083242179295059, "lo": 0.06359410323551529, "hi": 0.07945601400217328 }, "resolution": { "value": 0.02347230412776531, "lo": 0.02062576803127584, "hi": 0.027707121829315607 }, "uncertainty": { "value": 0.24093696, "lo": 0.23776764, "hi": 0.243488307 }, "auroc": { "value": 0.6694239024182923, "lo": 0.6536613432275953, "hi": 0.683653191239962 }, "acc@80": { "value": 0.63175, "lo": 0.6155, "hi": 0.6495125 } } }, { "env": "yelp", "slice": "all", "system": "zs-LFM2.5-VL-3B-prefill +T", "metrics": { "n": { "value": 5000.0, "lo": 5000.0, "hi": 5000.0 }, "ce": { "value": 0.9605660163022847, "lo": 0.9369846035904397, "hi": 0.9816685557479368 }, "kl": { "value": 0.9605660163022847, "lo": 0.9369846035904397, "hi": 0.9816685557479368 }, "jsd": { "value": 0.2516838502783082, "lo": 0.24637970558942096, "hi": 0.25627033242451014 }, "brier": { "value": 0.5332790437612978, "lo": 0.5184867031910769, "hi": 0.5463571151359354 }, "acc": { "value": 0.5952, "lo": 0.580695, "hi": 0.6106 }, "conf": { "value": 0.6486309407859533, "lo": 0.6437934317585855, "hi": 0.6531680733905414 }, "ece": { "value": 0.06880398054217841, "lo": 0.058410055797531384, "hi": 0.08362566469544623 }, "reliability": { "value": 0.007127271545934169, "lo": 0.005628349175403814, "hi": 0.010483590238076811 }, "resolution": { "value": 0.022972801829853808, "lo": 0.019711298467703607, "hi": 0.02692298937206552 }, "uncertainty": { "value": 0.24093696, "lo": 0.23776764, "hi": 0.243488307 }, "auroc": { "value": 0.6694089607505631, "lo": 0.6543323056358303, "hi": 0.6836065841514896 }, "acc@80": { "value": 0.63325, "lo": 0.61686875, "hi": 0.6505375 } } }, { "env": "boolq", "slice": "all", "system": "mix7-1.6B-s0", "metrics": { "n": { "value": 8129.0, "lo": 7984.475, "hi": 8270.575 }, "ce": { "value": 0.5082761310374166, "lo": 0.49592209899738315, "hi": 0.5199124539145769 }, "kl": { "value": 0.5082761310374166, "lo": 0.49592209899738315, "hi": 0.5199124539145769 }, "jsd": { "value": 0.1506662270607344, "lo": 0.1475305328937042, "hi": 0.1537566314506511 }, "brier": { "value": 0.34082382639915726, "lo": 0.3305859817855617, "hi": 0.35073283205382355 }, "acc": { "value": 0.7293640054127198, "lo": 0.7184596840938764, "hi": 0.7395318200163514 }, "conf": { "value": 0.7481786015949092, "lo": 0.7447704591796487, "hi": 0.7514176330865708 }, "ece": { "value": 0.03149522683147957, "lo": 0.026041321058523508, "hi": 0.041646159942767116 }, "reliability": { "value": 0.0016323515032443912, "lo": 0.0012245269658096858, "hi": 0.0029887046246071846 }, "resolution": { "value": 0.028251831702324568, "lo": 0.025786177624655605, "hi": 0.03159886042170951 }, "uncertainty": { "value": 0.19739215302103383, "lo": 0.19262450715283344, "hi": 0.20227536629943227 }, "auroc": { "value": 0.7446593400696117, "lo": 0.7326277586422778, "hi": 0.756000760333661 }, "acc@80": { "value": 0.7887129017376595, "lo": 0.7769549200023308, "hi": 0.8002794750066544 } } }, { "env": "chaosnli", "slice": "all", "system": "mix7-1.6B-s0", "metrics": { "n": { "value": 2763.0, "lo": 2732.475, "hi": 2798.525 }, "ce": { "value": 0.7654613715471141, "lo": 0.7494300351864432, "hi": 0.7808923184162763 }, "kl": { "value": 0.23510130147785244, "lo": 0.2222539775728993, "hi": 0.24790096655683252 }, "jsd": { "value": 0.05892347080822899, "lo": 0.05614685702665513, "hi": 0.06198860929577264 }, "brier": { "value": 0.1552037163302883, "lo": 0.1462400606765066, "hi": 0.16456635180066206 }, "acc": { "value": 0.7082880926529135, "lo": 0.6925717712394138, "hi": 0.7251270878721859 }, "conf": { "value": 0.6869863378065374, "lo": 0.681784769246876, "hi": 0.6923667213332382 }, "ece": { "value": 0.030578552813707745, "lo": 0.027840654001020076, "hi": 0.05437526135243692 }, "reliability": { "value": 0.0017135782171140309, "lo": 0.0012915285493501388, "hi": 0.004640477300022285 }, "resolution": { "value": 0.01976698573550043, "lo": 0.016656680968617162, "hi": 0.02524448990397694 }, "uncertainty": { "value": 0.20661607045901134, "lo": 0.19931779430618907, "hi": 0.21291611270607952 }, "auroc": { "value": 0.6902250748410934, "lo": 0.6678661067691528, "hi": 0.7123250156117648 }, "acc@80": { "value": 0.7529411764705882, "lo": 0.7348948919282172, "hi": 0.7722481802085382 } } }, { "env": "civilcomments", "slice": "all", "system": "mix7-1.6B-s0", "metrics": { "n": { "value": 10581.0, "lo": 10261.425, "hi": 10904.525 }, "ce": { "value": 0.3291432395939985, "lo": 0.3200820913634004, "hi": 0.3372185842634208 }, "kl": { "value": 0.09766043082465743, "lo": 0.09463633276139745, "hi": 0.10064454636867035 }, "jsd": { "value": 0.024450077575365353, "lo": 0.02376066723067054, "hi": 0.025111864127835334 }, "brier": { "value": 0.04824225276040297, "lo": 0.04608759857299234, "hi": 0.05039442895462397 }, "acc": { "value": 0.9314809564313392, "lo": 0.926637093159785, "hi": 0.9358851114843256 }, "conf": { "value": 0.8949943995861078, "lo": 0.8925153914028515, "hi": 0.8978972249872205 }, "ece": { "value": 0.04325364368555168, "lo": 0.039369716140716667, "hi": 0.04738038014011293 }, "reliability": { "value": 0.0025325939682709357, "lo": 0.0021693777868036232, "hi": 0.0031092331245068244 }, "resolution": { "value": 0.01635684924521661, "lo": 0.014259526254035334, "hi": 0.01878172823720776 }, "uncertainty": { "value": 0.06382418423709675, "lo": 0.06000416947908488, "hi": 0.06798078845455381 }, "auroc": { "value": 0.9217205693013882, "lo": 0.913332991673519, "hi": 0.9293901686812233 }, "acc@80": { "value": 0.9864146485528648, "lo": 0.9840827523103477, "hi": 0.9893232853040609 } } }, { "env": "folktables", "slice": "all", "system": "mix7-1.6B-s0", "metrics": { "n": { "value": 15003.0, "lo": 14855.8, "hi": 15157.0 }, "ce": { "value": 0.4429014798100703, "lo": 0.4338010939037464, "hi": 0.452151049808509 }, "kl": { "value": 0.4429014798100703, "lo": 0.4338010939037464, "hi": 0.452151049808509 }, "jsd": { "value": 0.1264743363940025, "lo": 0.12427204099846796, "hi": 0.12871952108589504 }, "brier": { "value": 0.29192483263490315, "lo": 0.28545596286425384, "hi": 0.2991082407855318 }, "acc": { "value": 0.7856428714257149, "lo": 0.7790224321133412, "hi": 0.7922019791889412 }, "conf": { "value": 0.8061644708856792, "lo": 0.8034163905686282, "hi": 0.8086048746732889 }, "ece": { "value": 0.025429919649106476, "lo": 0.020018190591396846, "hi": 0.03176325508409786 }, "reliability": { "value": 0.0009121385172829448, "lo": 0.0006445539894268695, "hi": 0.0015913915011417576 }, "resolution": { "value": 0.0233436276816435, "lo": 0.02173771272662814, "hi": 0.025429969496669848 }, "uncertainty": { "value": 0.1684081500036725, "lo": 0.16461800164228868, "hi": 0.1721464467827975 }, "auroc": { "value": 0.760233718887534, "lo": 0.7515386177960786, "hi": 0.7686806860142983 }, "acc@80": { "value": 0.8390268288618563, "lo": 0.831536650563158, "hi": 0.8459624211130234 } } }, { "env": "mcq", "slice": "all", "system": "mix7-1.6B-s0", "metrics": { "n": { "value": 5064.0, "lo": 5064.0, "hi": 5064.0 }, "ce": { "value": 0.4002523221645569, "lo": 0.3782136455299466, "hi": 0.42385603103379427 }, "kl": { "value": 0.4002523221645569, "lo": 0.3782136455299466, "hi": 0.42385603103379427 }, "jsd": { "value": 0.09391323294325246, "lo": 0.08972082333599198, "hi": 0.09825219031625738 }, "brier": { "value": 0.20633200495477827, "lo": 0.19470067976198033, "hi": 0.21846857959263977 }, "acc": { "value": 0.8554502369668247, "lo": 0.8450780015797789, "hi": 0.8641390205371248 }, "conf": { "value": 0.8843396415677963, "lo": 0.8795300210097814, "hi": 0.8890400585438909 }, "ece": { "value": 0.0288912705417964, "lo": 0.02272542280852676, "hi": 0.037411134562191764 }, "reliability": { "value": 0.0014989374336935519, "lo": 0.0010164991828886306, "hi": 0.002780364073085358 }, "resolution": { "value": 0.033634874124788244, "lo": 0.029467195088500375, "hi": 0.03864765960070014 }, "uncertainty": { "value": 0.1236551290402282, "lo": 0.11740277372226343, "hi": 0.1309211631012456 }, "auroc": { "value": 0.8781354225007443, "lo": 0.866011040404468, "hi": 0.8890746426752913 }, "acc@80": { "value": 0.9360651690940508, "lo": 0.9281658849666748, "hi": 0.9434707479634659 } } }, { "env": "stsb", "slice": "all", "system": "mix7-1.6B-s0", "metrics": { "n": { "value": 3676.0, "lo": 3150.95, "hi": 4793.674999999996 }, "ce": { "value": 1.031748552907778, "lo": 0.9704824263024805, "hi": 1.0889858187347534 }, "kl": { "value": 0.6294447300455221, "lo": 0.572015076742756, "hi": 0.6832772809458031 }, "jsd": { "value": 0.1479128524201686, "lo": 0.13660829407083266, "hi": 0.15784509493722043 }, "brier": { "value": 0.290818102811832, "lo": 0.2668247655785668, "hi": 0.3128366138275768 }, "acc": { "value": 0.6131664853101197, "lo": 0.587906790991269, "hi": 0.6378792112198279 }, "conf": { "value": 0.6504272062481422, "lo": 0.6443878978823263, "hi": 0.6562695090876604 }, "ece": { "value": 0.052070733139512326, "lo": 0.04119822485513943, "hi": 0.07324350903062836 }, "reliability": { "value": 0.004317376438774994, "lo": 0.0029592961485554003, "hi": 0.00820963383335681 }, "resolution": { "value": 0.013739284081678127, "lo": 0.011211957772358483, "hi": 0.017058838178443402 }, "uncertainty": { "value": 0.23719334660255445, "lo": 0.23098927933949753, "hi": 0.24227237669441876 }, "auroc": { "value": 0.6128823020677726, "lo": 0.5911372722639222, "hi": 0.6308539874239646 }, "acc@80": { "value": 0.6399183951037062, "lo": 0.6111209029066172, "hi": 0.6684116940898375 } } }, { "env": "yelp", "slice": "all", "system": "mix7-1.6B-s0", "metrics": { "n": { "value": 5000.0, "lo": 5000.0, "hi": 5000.0 }, "ce": { "value": 1.1007355521982354, "lo": 1.0810604797933898, "hi": 1.1197186777887949 }, "kl": { "value": 1.1007355521982354, "lo": 1.0810604797933898, "hi": 1.1197186777887949 }, "jsd": { "value": 0.2967980785000652, "lo": 0.2926581255188195, "hi": 0.3006860390537452 }, "brier": { "value": 0.6052061326700343, "lo": 0.5939172279744793, "hi": 0.6159469872070467 }, "acc": { "value": 0.4936, "lo": 0.4808, "hi": 0.507905 }, "conf": { "value": 0.5416603589355946, "lo": 0.5379687575377524, "hi": 0.5453135892054439 }, "ece": { "value": 0.055525891262292876, "lo": 0.04558760949969293, "hi": 0.06793060291126372 }, "reliability": { "value": 0.0037097829303959985, "lo": 0.0029671397564004914, "hi": 0.005947604738688515 }, "resolution": { "value": 0.021382927360414306, "lo": 0.0185277120501153, "hi": 0.025603394336124526 }, "uncertainty": { "value": 0.24995903999999997, "lo": 0.24963136, "hi": 0.24999996 }, "auroc": { "value": 0.6502098103753319, "lo": 0.6356542944329878, "hi": 0.6653851325094023 }, "acc@80": { "value": 0.5315, "lo": 0.5173687499999999, "hi": 0.548 } } }, { "env": "boolq", "slice": "all", "system": "mix7-1.6B-s1", "metrics": { "n": { "value": 8129.0, "lo": 7984.475, "hi": 8270.575 }, "ce": { "value": 0.47855710362970805, "lo": 0.4687038073149669, "hi": 0.48872637673033503 }, "kl": { "value": 0.47855710362970805, "lo": 0.4687038073149669, "hi": 0.48872637673033503 }, "jsd": { "value": 0.1457436762811214, "lo": 0.14307058632434067, "hi": 0.1485167780754198 }, "brier": { "value": 0.3130394035594896, "lo": 0.30479280208146486, "hi": 0.32176081123172 }, "acc": { "value": 0.7684832082666995, "lo": 0.7575482036495192, "hi": 0.7778365736030838 }, "conf": { "value": 0.7263465951562207, "lo": 0.7231753169988112, "hi": 0.7296469446485713 }, "ece": { "value": 0.04808772066331964, "lo": 0.04045296450417986, "hi": 0.058733391696430805 }, "reliability": { "value": 0.00284671903813426, "lo": 0.0022353916304481353, "hi": 0.004130891358852014 }, "resolution": { "value": 0.024018337935267288, "lo": 0.021604361539690864, "hi": 0.0270778336935212 }, "uncertainty": { "value": 0.17791676687882008, "lo": 0.17280683360798862, "hi": 0.18366891095742832 }, "auroc": { "value": 0.7496162663923529, "lo": 0.7382809384344194, "hi": 0.7626390674411822 }, "acc@80": { "value": 0.8236198677533446, "lo": 0.8130154888594103, "hi": 0.8347216561229348 } } }, { "env": "chaosnli", "slice": "all", "system": "mix7-1.6B-s1", "metrics": { "n": { "value": 2763.0, "lo": 2732.475, "hi": 2798.525 }, "ce": { "value": 0.7837159885599414, "lo": 0.7690258664923734, "hi": 0.7995680854416652 }, "kl": { "value": 0.25335591849067945, "lo": 0.24210794053634344, "hi": 0.26627656912373626 }, "jsd": { "value": 0.06430561168056584, "lo": 0.061732091418810385, "hi": 0.06738181425264714 }, "brier": { "value": 0.17074606097407263, "lo": 0.16227614013310068, "hi": 0.1806546230159187 }, "acc": { "value": 0.6800579080709374, "lo": 0.6616313268009419, "hi": 0.6974956007895736 }, "conf": { "value": 0.6740838724376929, "lo": 0.6684434502085462, "hi": 0.6796477558247118 }, "ece": { "value": 0.02181350362935101, "lo": 0.022559406720764993, "hi": 0.0476477255996329 }, "reliability": { "value": 0.0008491005387559562, "lo": 0.0008246849919371727, "hi": 0.0033061108419428875 }, "resolution": { "value": 0.017039685512144916, "lo": 0.01404967003529775, "hi": 0.02205710108209954 }, "uncertainty": { "value": 0.21757914974111786, "lo": 0.21099517127486303, "hi": 0.2238753103132754 }, "auroc": { "value": 0.6697049311393612, "lo": 0.6491910484953667, "hi": 0.6894518935356425 }, "acc@80": { "value": 0.7217194570135747, "lo": 0.7032542171221171, "hi": 0.7411502385227859 } } }, { "env": "civilcomments", "slice": "all", "system": "mix7-1.6B-s1", "metrics": { "n": { "value": 10581.0, "lo": 10261.425, "hi": 10904.525 }, "ce": { "value": 0.3231331506493501, "lo": 0.31510608432322923, "hi": 0.33053231851393033 }, "kl": { "value": 0.09165034188000905, "lo": 0.08921166635656741, "hi": 0.09423752245204604 }, "jsd": { "value": 0.02525167459760998, "lo": 0.02464559411887002, "hi": 0.025894354411833548 }, "brier": { "value": 0.04397100215369724, "lo": 0.04207733690750005, "hi": 0.04582011475573585 }, "acc": { "value": 0.93346564596919, "lo": 0.9287920101088212, "hi": 0.9380227139566573 }, "conf": { "value": 0.8726835281555306, "lo": 0.8698799446038222, "hi": 0.8757637342502513 }, "ece": { "value": 0.0636185640232424, "lo": 0.060011936270282715, "hi": 0.06774610592440335 }, "reliability": { "value": 0.005551196569429456, "lo": 0.0049828496460246514, "hi": 0.006304045412355839 }, "resolution": { "value": 0.01528220459349804, "lo": 0.01310670354760269, "hi": 0.017633533336201036 }, "uncertainty": { "value": 0.06210753376451282, "lo": 0.05813609646680648, "hi": 0.06613740925205552 }, "auroc": { "value": 0.9217670673143299, "lo": 0.9137446684305477, "hi": 0.9294152745063432 }, "acc@80": { "value": 0.9871234494979326, "lo": 0.9845517587908574, "hi": 0.9891930614162803 } } }, { "env": "folktables", "slice": "all", "system": "mix7-1.6B-s1", "metrics": { "n": { "value": 15003.0, "lo": 14855.8, "hi": 15157.0 }, "ce": { "value": 0.44795320351265, "lo": 0.43988770426826734, "hi": 0.45616144497070304 }, "kl": { "value": 0.44795320351265, "lo": 0.43988770426826734, "hi": 0.45616144497070304 }, "jsd": { "value": 0.1314335571367777, "lo": 0.12918742495131863, "hi": 0.13364927526286144 }, "brier": { "value": 0.29621629260680155, "lo": 0.28995125809083827, "hi": 0.3029004037214525 }, "acc": { "value": 0.7784443111377725, "lo": 0.7708365555562754, "hi": 0.7857097418027347 }, "conf": { "value": 0.7826558687977256, "lo": 0.7799437584074044, "hi": 0.7850506899493145 }, "ece": { "value": 0.013490881572951727, "lo": 0.010313860277834775, "hi": 0.02133988882347297 }, "reliability": { "value": 0.0002916299564866727, "lo": 0.00018029928017010852, "hi": 0.0007644078426685735 }, "resolution": { "value": 0.024531181398278194, "lo": 0.022702030203598764, "hi": 0.026773431321743524 }, "uncertainty": { "value": 0.17246876559501137, "lo": 0.16836993443494178, "hi": 0.17664756015099178 }, "auroc": { "value": 0.760897827041841, "lo": 0.7527185587025165, "hi": 0.7690317796217346 }, "acc@80": { "value": 0.8350274954174304, "lo": 0.8268750344847557, "hi": 0.8415655775646039 } } }, { "env": "mcq", "slice": "all", "system": "mix7-1.6B-s1", "metrics": { "n": { "value": 5064.0, "lo": 5064.0, "hi": 5064.0 }, "ce": { "value": 0.42684009817659985, "lo": 0.40534061865938326, "hi": 0.4487317974627279 }, "kl": { "value": 0.42684009817659985, "lo": 0.40534061865938326, "hi": 0.4487317974627279 }, "jsd": { "value": 0.1047209966086494, "lo": 0.10048340589225171, "hi": 0.10896929767667661 }, "brier": { "value": 0.21781855156334679, "lo": 0.2069730368823189, "hi": 0.22866386525479698 }, "acc": { "value": 0.846958925750395, "lo": 0.8374802527646129, "hi": 0.8564474723538705 }, "conf": { "value": 0.8572383006007703, "lo": 0.8524303276069745, "hi": 0.8613331286549292 }, "ece": { "value": 0.012092256853813047, "lo": 0.010227614650458664, "hi": 0.023183345897741164 }, "reliability": { "value": 0.00033246617151455745, "lo": 0.0002320958444557757, "hi": 0.001265645465417821 }, "resolution": { "value": 0.036602240533722234, "lo": 0.03218982097016048, "hi": 0.04174564202942582 }, "uncertainty": { "value": 0.1296195038421319, "lo": 0.12294516055469451, "hi": 0.13610707899393298 }, "auroc": { "value": 0.8766589700584391, "lo": 0.8656314490404682, "hi": 0.8881121225579844 }, "acc@80": { "value": 0.9313749691434213, "lo": 0.9230992347568501, "hi": 0.9381695877561097 } } }, { "env": "stsb", "slice": "all", "system": "mix7-1.6B-s1", "metrics": { "n": { "value": 3676.0, "lo": 3150.95, "hi": 4793.674999999996 }, "ce": { "value": 1.0325978650062761, "lo": 0.9812421226139131, "hi": 1.0807375838768523 }, "kl": { "value": 0.6302940421440205, "lo": 0.5805065812874415, "hi": 0.675634602555858 }, "jsd": { "value": 0.15664148876714856, "lo": 0.14586758188198548, "hi": 0.16587076106736395 }, "brier": { "value": 0.30203148903766214, "lo": 0.2793681708337265, "hi": 0.3214175970283663 }, "acc": { "value": 0.5715451577801959, "lo": 0.5432189681593734, "hi": 0.5975937224782812 }, "conf": { "value": 0.6035776622552866, "lo": 0.5966974025467761, "hi": 0.610775968141833 }, "ece": { "value": 0.04524248022022652, "lo": 0.03777464454166938, "hi": 0.07000553128061206 }, "reliability": { "value": 0.0029649726855611597, "lo": 0.002174174413508668, "hi": 0.0068987481306255465 }, "resolution": { "value": 0.013546995487004788, "lo": 0.010450661463123909, "hi": 0.01863973032653182 }, "uncertainty": { "value": 0.24488129039820689, "lo": 0.24047544437763765, "hi": 0.248132120610992 }, "auroc": { "value": 0.621027477467268, "lo": 0.5938392011645692, "hi": 0.6485523592260055 }, "acc@80": { "value": 0.6028561713702822, "lo": 0.5703271629887985, "hi": 0.6363633066398043 } } }, { "env": "yelp", "slice": "all", "system": "mix7-1.6B-s1", "metrics": { "n": { "value": 5000.0, "lo": 5000.0, "hi": 5000.0 }, "ce": { "value": 1.1098901509100594, "lo": 1.0902504703014342, "hi": 1.1298374929111197 }, "kl": { "value": 1.1098901509100594, "lo": 1.0902504703014342, "hi": 1.1298374929111197 }, "jsd": { "value": 0.29727174008141743, "lo": 0.29302459407091197, "hi": 0.3012404960787604 }, "brier": { "value": 0.5931743975488044, "lo": 0.582613042214466, "hi": 0.60352119345443 }, "acc": { "value": 0.5326, "lo": 0.518685, "hi": 0.5468 }, "conf": { "value": 0.5210890755355359, "lo": 0.5175688868150115, "hi": 0.5246254901348054 }, "ece": { "value": 0.031602025562524796, "lo": 0.027794011172354218, "hi": 0.049214339447766535 }, "reliability": { "value": 0.0015905999807410446, "lo": 0.0012336936696333237, "hi": 0.0034744437984095895 }, "resolution": { "value": 0.022269785719372066, "lo": 0.019280771406242864, "hi": 0.02686939814799849 }, "uncertainty": { "value": 0.24893724, "lo": 0.24780976000000002, "hi": 0.24965078100000002 }, "auroc": { "value": 0.6616861342240318, "lo": 0.6469168791201212, "hi": 0.677234165187451 }, "acc@80": { "value": 0.57175, "lo": 0.5550812500000001, "hi": 0.589 } } }, { "env": "boolq", "slice": "all", "system": "mix7-1.6B-s2", "metrics": { "n": { "value": 8129.0, "lo": 7984.475, "hi": 8270.575 }, "ce": { "value": 0.5184885789058133, "lo": 0.5073414469680039, "hi": 0.5299303650960395 }, "kl": { "value": 0.5184885789058133, "lo": 0.5073414469680039, "hi": 0.5299303650960395 }, "jsd": { "value": 0.15550945721477388, "lo": 0.1527205926314921, "hi": 0.15841855987947642 }, "brier": { "value": 0.3463320106654194, "lo": 0.33707788650964454, "hi": 0.35558891417168853 }, "acc": { "value": 0.7291179726903678, "lo": 0.7178177140234293, "hi": 0.7394308067209925 }, "conf": { "value": 0.7274356591758699, "lo": 0.7244902567629894, "hi": 0.7306223464699275 }, "ece": { "value": 0.03144833778070925, "lo": 0.026431504587073013, "hi": 0.04174819339554388 }, "reliability": { "value": 0.0014419736102994397, "lo": 0.0010707162982793146, "hi": 0.002594793016103829 }, "resolution": { "value": 0.025398090908325222, "lo": 0.022671965366050328, "hi": 0.028545983813892145 }, "uncertainty": { "value": 0.19750495459025585, "lo": 0.1926728720725454, "hi": 0.20255541854484804 }, "auroc": { "value": 0.7306054651913142, "lo": 0.7181527314846667, "hi": 0.7417360677234859 }, "acc@80": { "value": 0.7879440258342304, "lo": 0.7763449134687196, "hi": 0.798086545338904 } } }, { "env": "chaosnli", "slice": "all", "system": "mix7-1.6B-s2", "metrics": { "n": { "value": 2763.0, "lo": 2732.475, "hi": 2798.525 }, "ce": { "value": 0.7585695789984018, "lo": 0.7426975509629593, "hi": 0.7754835285504206 }, "kl": { "value": 0.22820950892914002, "lo": 0.21553206414863885, "hi": 0.24147519530133094 }, "jsd": { "value": 0.05680358621993229, "lo": 0.053984252957542944, "hi": 0.05979957873716277 }, "brier": { "value": 0.14927606706706414, "lo": 0.1402168151341663, "hi": 0.1586389948864686 }, "acc": { "value": 0.7278320665942816, "lo": 0.7109009257791143, "hi": 0.7449838551772038 }, "conf": { "value": 0.6846000137663043, "lo": 0.679424621606799, "hi": 0.6903244356008484 }, "ece": { "value": 0.056012450127855294, "lo": 0.04558204311582116, "hi": 0.07314523965095096 }, "reliability": { "value": 0.003940287369283433, "lo": 0.0031526820419978735, "hi": 0.007212572967384259 }, "resolution": { "value": 0.014707128696814295, "lo": 0.011672780792140328, "hi": 0.019684878450526758 }, "uncertainty": { "value": 0.19809254943137886, "lo": 0.1899829070334599, "hi": 0.20552079463651074 }, "auroc": { "value": 0.6720206417893078, "lo": 0.6508045463960422, "hi": 0.6950214079589969 }, "acc@80": { "value": 0.7737556561085973, "lo": 0.7553810211428797, "hi": 0.790371275790134 } } }, { "env": "civilcomments", "slice": "all", "system": "mix7-1.6B-s2", "metrics": { "n": { "value": 10581.0, "lo": 10261.425, "hi": 10904.525 }, "ce": { "value": 0.3163261854007762, "lo": 0.3083993356944391, "hi": 0.3236248116520279 }, "kl": { "value": 0.08484337663143518, "lo": 0.0824035445799588, "hi": 0.08768283157281037 }, "jsd": { "value": 0.022882750186549245, "lo": 0.022234285074514283, "hi": 0.0235707239570925 }, "brier": { "value": 0.039377961884232315, "lo": 0.03774909977347542, "hi": 0.04112938297858045 }, "acc": { "value": 0.9425385124279368, "lo": 0.9377526041043914, "hi": 0.9467558718433233 }, "conf": { "value": 0.8647348193914562, "lo": 0.8613458292080601, "hi": 0.8683777216504087 }, "ece": { "value": 0.07780369303648059, "lo": 0.07279760445520497, "hi": 0.08162182332821356 }, "reliability": { "value": 0.010334194443595839, "lo": 0.00929202710467513, "hi": 0.011360849811607052 }, "resolution": { "value": 0.01176697872619398, "lo": 0.010198096047943774, "hi": 0.014251711287922499 }, "uncertainty": { "value": 0.054159665018068785, "lo": 0.05040919081874604, "hi": 0.058372644645337014 }, "auroc": { "value": 0.9198111216072871, "lo": 0.9123547956416179, "hi": 0.927071480238284 }, "acc@80": { "value": 0.9890135853514471, "lo": 0.9865763365502729, "hi": 0.9912093413196587 } } }, { "env": "folktables", "slice": "all", "system": "mix7-1.6B-s2", "metrics": { "n": { "value": 15003.0, "lo": 14855.8, "hi": 15157.0 }, "ce": { "value": 0.4356590369299968, "lo": 0.4255747789775578, "hi": 0.44521351969369516 }, "kl": { "value": 0.4356590369299968, "lo": 0.4255747789775578, "hi": 0.44521351969369516 }, "jsd": { "value": 0.12351583298705354, "lo": 0.12126312193035323, "hi": 0.12589004464231174 }, "brier": { "value": 0.28450758265424486, "lo": 0.27780617593704, "hi": 0.29142304276175635 }, "acc": { "value": 0.7911084449776711, "lo": 0.784140465204479, "hi": 0.7979199537449984 }, "conf": { "value": 0.8088698498306295, "lo": 0.8060494976775232, "hi": 0.8115879565893355 }, "ece": { "value": 0.01979037306272671, "lo": 0.016355303584789916, "hi": 0.02714447983071726 }, "reliability": { "value": 0.000565135572096387, "lo": 0.00041583292822958926, "hi": 0.0010895546936945067 }, "resolution": { "value": 0.023620479731631654, "lo": 0.021853692097286808, "hi": 0.02577255861621752 }, "uncertainty": { "value": 0.16525587326268226, "lo": 0.16124369616200576, "hi": 0.16926419587388697 }, "auroc": { "value": 0.7657509738706254, "lo": 0.7575488803450572, "hi": 0.773991327057773 }, "acc@80": { "value": 0.8474420929845026, "lo": 0.840006326543138, "hi": 0.8532372466567733 } } }, { "env": "mcq", "slice": "all", "system": "mix7-1.6B-s2", "metrics": { "n": { "value": 5064.0, "lo": 5064.0, "hi": 5064.0 }, "ce": { "value": 0.41635328038970587, "lo": 0.39123240415937593, "hi": 0.44061227208142756 }, "kl": { "value": 0.41635328038970587, "lo": 0.39123240415937593, "hi": 0.44061227208142756 }, "jsd": { "value": 0.09177637644129429, "lo": 0.0874378979350158, "hi": 0.09609393418574062 }, "brier": { "value": 0.2106166671020026, "lo": 0.19884213068170087, "hi": 0.22260596854163442 }, "acc": { "value": 0.8578199052132701, "lo": 0.8488299763033176, "hi": 0.8672146524486571 }, "conf": { "value": 0.9008063965758064, "lo": 0.8966689786369108, "hi": 0.904753290160337 }, "ece": { "value": 0.0429864913625363, "lo": 0.036125414531183606, "hi": 0.051237418890399566 }, "reliability": { "value": 0.00413420947410155, "lo": 0.0028608661129630114, "hi": 0.005898513586923716 }, "resolution": { "value": 0.032241344066748535, "lo": 0.027679393936887287, "hi": 0.03666930523992034 }, "uncertainty": { "value": 0.12196491543316638, "lo": 0.11515331150686695, "hi": 0.12831763790776637 }, "auroc": { "value": 0.8752545017393084, "lo": 0.8638134438304621, "hi": 0.8857928861927227 }, "acc@80": { "value": 0.9333497901752653, "lo": 0.9259442113058504, "hi": 0.9411318193038757 } } }, { "env": "stsb", "slice": "all", "system": "mix7-1.6B-s2", "metrics": { "n": { "value": 3676.0, "lo": 3150.95, "hi": 4793.674999999996 }, "ce": { "value": 1.1350048984988437, "lo": 1.059055504232052, "hi": 1.2076017137835424 }, "kl": { "value": 0.732701075636588, "lo": 0.6608848588343845, "hi": 0.8045771003578774 }, "jsd": { "value": 0.1485437192618815, "lo": 0.1363266804954933, "hi": 0.15994733514458628 }, "brier": { "value": 0.3191667956646663, "lo": 0.29302127439037406, "hi": 0.34358278105281215 }, "acc": { "value": 0.5998367791077258, "lo": 0.5756276123301985, "hi": 0.6217297290540421 }, "conf": { "value": 0.721857658172912, "lo": 0.7139604273591086, "hi": 0.7298799076292077 }, "ece": { "value": 0.12202087906518622, "lo": 0.10624456008692325, "hi": 0.14103889926102078 }, "reliability": { "value": 0.01820899459314394, "lo": 0.015093344534630931, "hi": 0.02479185368774445 }, "resolution": { "value": 0.015581056345493581, "lo": 0.011424916235943822, "hi": 0.021824644160194848 }, "uncertainty": { "value": 0.24003261753739516, "lo": 0.23518186354343104, "hi": 0.2442793155504878 }, "auroc": { "value": 0.6290156941997284, "lo": 0.600513688902862, "hi": 0.6555364844263627 }, "acc@80": { "value": 0.6290377422645359, "lo": 0.5976708068202223, "hi": 0.6559320254312636 } } }, { "env": "yelp", "slice": "all", "system": "mix7-1.6B-s2", "metrics": { "n": { "value": 5000.0, "lo": 5000.0, "hi": 5000.0 }, "ce": { "value": 1.1317714985484708, "lo": 1.1070318971483937, "hi": 1.1563624192840993 }, "kl": { "value": 1.1317714985484708, "lo": 1.1070318971483937, "hi": 1.1563624192840993 }, "jsd": { "value": 0.28937557216683035, "lo": 0.28447826678600413, "hi": 0.29423930364960643 }, "brier": { "value": 0.6464389177537861, "lo": 0.6331029659381819, "hi": 0.6603220935009174 }, "acc": { "value": 0.484, "lo": 0.4708, "hi": 0.497305 }, "conf": { "value": 0.6399741260707379, "lo": 0.6360008893069625, "hi": 0.6436980085825921 }, "ece": { "value": 0.15597412607073785, "lo": 0.14327579551830888, "hi": 0.168145623575598 }, "reliability": { "value": 0.026895021035463172, "lo": 0.02383132064565059, "hi": 0.03218924544504216 }, "resolution": { "value": 0.014863111326895757, "lo": 0.012713097310915848, "hi": 0.01878409197471433 }, "uncertainty": { "value": 0.249744, "lo": 0.24914736, "hi": 0.24999216 }, "auroc": { "value": 0.628967823050804, "lo": 0.6138308150497146, "hi": 0.6436885618805029 }, "acc@80": { "value": 0.519, "lo": 0.5033687499999999, "hi": 0.5341312500000001 } } }, { "env": "boolq", "slice": "all", "system": "mix7-450M-s0", "metrics": { "n": { "value": 8129.0, "lo": 7984.475, "hi": 8270.575 }, "ce": { "value": 0.7136019924352867, "lo": 0.6992330314186738, "hi": 0.7284831141138027 }, "kl": { "value": 0.7136019924352867, "lo": 0.6992330314186738, "hi": 0.7284831141138027 }, "jsd": { "value": 0.1997060328901781, "lo": 0.19629395072043002, "hi": 0.2034049058664722 }, "brier": { "value": 0.50396008392508, "lo": 0.4924207562288041, "hi": 0.516100548797697 }, "acc": { "value": 0.5961372862590725, "lo": 0.5833206151993764, "hi": 0.6072701905987213 }, "conf": { "value": 0.7473454058632256, "lo": 0.7440006008262551, "hi": 0.750466218302769 }, "ece": { "value": 0.15120811960415317, "lo": 0.14059334256144637, "hi": 0.1628602029115415 }, "reliability": { "value": 0.025790954004519116, "lo": 0.022849653178952548, "hi": 0.030032748048394644 }, "resolution": { "value": 0.014443398893949458, "lo": 0.012634056254531721, "hi": 0.01717559926160552 }, "uncertainty": { "value": 0.24075762219074115, "lo": 0.23849310329488035, "hi": 0.24305724425008088 }, "auroc": { "value": 0.6325759999517268, "lo": 0.6216105301613887, "hi": 0.6446417077529917 }, "acc@80": { "value": 0.6287867138243888, "lo": 0.6141491061740227, "hi": 0.6399709257842388 } } }, { "env": "chaosnli", "slice": "all", "system": "mix7-450M-s0", "metrics": { "n": { "value": 2763.0, "lo": 2732.475, "hi": 2798.525 }, "ce": { "value": 0.8982318895558363, "lo": 0.8809683495606556, "hi": 0.9142790524229913 }, "kl": { "value": 0.3678718194865745, "lo": 0.35238190600853664, "hi": 0.384989532036147 }, "jsd": { "value": 0.09134736494105902, "lo": 0.08804191118690168, "hi": 0.09513120937645976 }, "brier": { "value": 0.24486443443340125, "lo": 0.23463086135651814, "hi": 0.25658701777405735 }, "acc": { "value": 0.5696706478465436, "lo": 0.5515324548447733, "hi": 0.5888673090601938 }, "conf": { "value": 0.6321799852064542, "lo": 0.627721041997912, "hi": 0.6365111712155478 }, "ece": { "value": 0.07636093434284788, "lo": 0.06381985569042921, "hi": 0.09822724828004374 }, "reliability": { "value": 0.008408304299897924, "lo": 0.006805944793064514, "hi": 0.014170574047492654 }, "resolution": { "value": 0.00602359624024682, "lo": 0.004797682776253241, "hi": 0.01106953877313359 }, "uncertainty": { "value": 0.2451460008286429, "lo": 0.24210259247819907, "hi": 0.24734439344902914 }, "auroc": { "value": 0.5786139997841288, "lo": 0.5569645510149233, "hi": 0.6020625222577087 }, "acc@80": { "value": 0.5923076923076923, "lo": 0.5700111036152837, "hi": 0.6124531330235895 } } }, { "env": "civilcomments", "slice": "all", "system": "mix7-450M-s0", "metrics": { "n": { "value": 10581.0, "lo": 10261.425, "hi": 10904.525 }, "ce": { "value": 0.3362536541996374, "lo": 0.32705962616964074, "hi": 0.3443111084010556 }, "kl": { "value": 0.1047708454302963, "lo": 0.10134550620320375, "hi": 0.10809404029584808 }, "jsd": { "value": 0.0258384918133153, "lo": 0.025078699358151307, "hi": 0.02657033880076522 }, "brier": { "value": 0.05199782543309763, "lo": 0.04963794515746586, "hi": 0.05439872462029978 }, "acc": { "value": 0.9291182307910405, "lo": 0.9243398642858486, "hi": 0.933591687041009 }, "conf": { "value": 0.8974095615014813, "lo": 0.8949546016420573, "hi": 0.9002264280977454 }, "ece": { "value": 0.03717635887555369, "lo": 0.033073985492604104, "hi": 0.04094511516000459 }, "reliability": { "value": 0.0018100862451617744, "lo": 0.0015027982301821941, "hi": 0.002249096987843986 }, "resolution": { "value": 0.01567649580728653, "lo": 0.013568188613809378, "hi": 0.01819872012798344 }, "uncertainty": { "value": 0.0658575440027673, "lo": 0.06199824016764933, "hi": 0.06993567843806958 }, "auroc": { "value": 0.9130753060048147, "lo": 0.904983118439983, "hi": 0.9213973689142747 }, "acc@80": { "value": 0.9845245126993503, "lo": 0.9811782355859234, "hi": 0.9873262110974187 } } }, { "env": "folktables", "slice": "all", "system": "mix7-450M-s0", "metrics": { "n": { "value": 15003.0, "lo": 14855.8, "hi": 15157.0 }, "ce": { "value": 0.43932493729908706, "lo": 0.4306063873085365, "hi": 0.44795527539062224 }, "kl": { "value": 0.43932493729908706, "lo": 0.4306063873085365, "hi": 0.44795527539062224 }, "jsd": { "value": 0.12698998066490708, "lo": 0.1247512801296757, "hi": 0.1291667144425798 }, "brier": { "value": 0.28949881585732284, "lo": 0.28289448648859516, "hi": 0.2962732345100782 }, "acc": { "value": 0.7843098047057255, "lo": 0.7771967479746235, "hi": 0.7909794763309138 }, "conf": { "value": 0.7981700179879287, "lo": 0.795316481521177, "hi": 0.8008636433840136 }, "ece": { "value": 0.016622621027984567, "lo": 0.012594631006075246, "hi": 0.023673511842104707 }, "reliability": { "value": 0.00041057214653177586, "lo": 0.00028476222813153264, "hi": 0.0009173206104042607 }, "resolution": { "value": 0.024776907383988914, "lo": 0.022891319809217686, "hi": 0.02678481059917013 }, "uncertainty": { "value": 0.1691679349481922, "lo": 0.16533094318620042, "hi": 0.1731619621170642 }, "auroc": { "value": 0.7666138137673784, "lo": 0.7587729096798848, "hi": 0.7744423019809946 }, "acc@80": { "value": 0.8434427595400766, "lo": 0.8364590299690824, "hi": 0.8505335929509873 } } }, { "env": "mcq", "slice": "all", "system": "mix7-450M-s0", "metrics": { "n": { "value": 5064.0, "lo": 5064.0, "hi": 5064.0 }, "ce": { "value": 0.747297009316037, "lo": 0.7213950937827313, "hi": 0.7766705758301444 }, "kl": { "value": 0.747297009316037, "lo": 0.7213950937827313, "hi": 0.7766705758301444 }, "jsd": { "value": 0.1767239562795029, "lo": 0.17163378414726665, "hi": 0.18206793350034872 }, "brier": { "value": 0.38947030443786845, "lo": 0.37626890786001316, "hi": 0.4035176613209252 }, "acc": { "value": 0.7156398104265402, "lo": 0.7043838862559242, "hi": 0.7273943522906793 }, "conf": { "value": 0.7855074125037516, "lo": 0.7796935032984745, "hi": 0.7909096821161254 }, "ece": { "value": 0.06986760207721124, "lo": 0.060243255898023915, "hi": 0.08024064675863614 }, "reliability": { "value": 0.006471904151593135, "lo": 0.005258512902660889, "hi": 0.00894751923819154 }, "resolution": { "value": 0.048900232407484534, "lo": 0.04468738679701923, "hi": 0.05468343522446478 }, "uncertainty": { "value": 0.2034994721592058, "lo": 0.19829179882184186, "hi": 0.20822722703892543 }, "auroc": { "value": 0.8112850096578367, "lo": 0.7978697372086834, "hi": 0.8238597453353185 }, "acc@80": { "value": 0.7965934337200691, "lo": 0.7846149098987903, "hi": 0.8086892125401135 } } }, { "env": "stsb", "slice": "all", "system": "mix7-450M-s0", "metrics": { "n": { "value": 3676.0, "lo": 3150.95, "hi": 4793.674999999996 }, "ce": { "value": 1.1119125656921782, "lo": 1.053280421966601, "hi": 1.1685552061846858 }, "kl": { "value": 0.7096087428299227, "lo": 0.65358620171179, "hi": 0.766083341781686 }, "jsd": { "value": 0.17320132983767303, "lo": 0.1616274604120623, "hi": 0.18393198896815666 }, "brier": { "value": 0.3273993070991984, "lo": 0.3053723911340578, "hi": 0.3496214053801269 }, "acc": { "value": 0.5525027203482046, "lo": 0.5235303024329845, "hi": 0.5782422389407145 }, "conf": { "value": 0.5877989736487997, "lo": 0.5764306252561174, "hi": 0.6025828143294766 }, "ece": { "value": 0.04876845997228197, "lo": 0.03908335157774449, "hi": 0.06955720642802253 }, "reliability": { "value": 0.0029641416367139854, "lo": 0.002345042505377347, "hi": 0.006676650159377232 }, "resolution": { "value": 0.013711604731963535, "lo": 0.011535638417900448, "hi": 0.01873196110357933 }, "uncertainty": { "value": 0.24724346435603822, "lo": 0.2438781452883429, "hi": 0.24944630452685188 }, "auroc": { "value": 0.6062123708655655, "lo": 0.58165551110918, "hi": 0.6276101089025438 }, "acc@80": { "value": 0.5725943556613396, "lo": 0.5416357437692293, "hi": 0.6052191508252037 } } }, { "env": "yelp", "slice": "all", "system": "mix7-450M-s0", "metrics": { "n": { "value": 5000.0, "lo": 5000.0, "hi": 5000.0 }, "ce": { "value": 1.4732754178012324, "lo": 1.4513124485509135, "hi": 1.4966784315799533 }, "kl": { "value": 1.4732754178012324, "lo": 1.4513124485509135, "hi": 1.4966784315799533 }, "jsd": { "value": 0.3673805956929661, "lo": 0.362933915344585, "hi": 0.37171122393183115 }, "brier": { "value": 0.7798598577553912, "lo": 0.767312370573482, "hi": 0.7911860416874497 }, "acc": { "value": 0.3442, "lo": 0.3314, "hi": 0.35841 }, "conf": { "value": 0.5286268295913935, "lo": 0.5250381398198753, "hi": 0.532158047472313 }, "ece": { "value": 0.18442682959139348, "lo": 0.17059637910090386, "hi": 0.1975069882760942 }, "reliability": { "value": 0.04259200737759276, "lo": 0.03767236290099163, "hi": 0.04908729373114615 }, "resolution": { "value": 0.0042384781061300825, "lo": 0.0033056711060300992, "hi": 0.006562855173325714 }, "uncertainty": { "value": 0.22572636, "lo": 0.22157404, "hi": 0.22995223199999998 }, "auroc": { "value": 0.5572634051246829, "lo": 0.5406379019247921, "hi": 0.5753965855237008 }, "acc@80": { "value": 0.36725, "lo": 0.3512125, "hi": 0.3832625 } } }, { "env": "boolq", "slice": "all", "system": "mix7-450M-s1", "metrics": { "n": { "value": 8129.0, "lo": 7984.475, "hi": 8270.575 }, "ce": { "value": 0.6613666478181621, "lo": 0.6487013682148409, "hi": 0.6745199642192672 }, "kl": { "value": 0.6613666478181621, "lo": 0.6487013682148409, "hi": 0.6745199642192672 }, "jsd": { "value": 0.19066056494487488, "lo": 0.18749099463660807, "hi": 0.19401811688218176 }, "brier": { "value": 0.4651263812087598, "lo": 0.45469805941530567, "hi": 0.4759976378084498 }, "acc": { "value": 0.6204945257719277, "lo": 0.6092188247359599, "hi": 0.6309430661660926 }, "conf": { "value": 0.7238843866872207, "lo": 0.7205719296032946, "hi": 0.7267868473831358 }, "ece": { "value": 0.10346502359685661, "lo": 0.09424817112916596, "hi": 0.11470731441404307 }, "reliability": { "value": 0.0130333503843448, "lo": 0.01105717569199151, "hi": 0.01607895916415323 }, "resolution": { "value": 0.015713242092913044, "lo": 0.013837220264048737, "hi": 0.0182541093085967 }, "uncertainty": { "value": 0.23548106925899825, "lo": 0.2328539124458066, "hi": 0.23807119371158414 }, "auroc": { "value": 0.6412828374486046, "lo": 0.6283801720039986, "hi": 0.6536368927242742 }, "acc@80": { "value": 0.6513916653852069, "lo": 0.6387216888372156, "hi": 0.6638142819075533 } } }, { "env": "chaosnli", "slice": "all", "system": "mix7-450M-s1", "metrics": { "n": { "value": 2763.0, "lo": 2732.475, "hi": 2798.525 }, "ce": { "value": 0.9017071316185276, "lo": 0.8833701893435971, "hi": 0.9209804894390085 }, "kl": { "value": 0.37134706154926567, "lo": 0.3545081413060039, "hi": 0.39115978645605487 }, "jsd": { "value": 0.09162369502806199, "lo": 0.08801208138497119, "hi": 0.09608325834988683 }, "brier": { "value": 0.24518685638804466, "lo": 0.23340681051390524, "hi": 0.25813472844184493 }, "acc": { "value": 0.5754614549402823, "lo": 0.5559670321869261, "hi": 0.5955889039843612 }, "conf": { "value": 0.6433491620774119, "lo": 0.6386417505573803, "hi": 0.6481729690973095 }, "ece": { "value": 0.07894042577598563, "lo": 0.06663756892339306, "hi": 0.09771193973001406 }, "reliability": { "value": 0.010354522866363586, "lo": 0.008124692146806126, "hi": 0.015591151725612823 }, "resolution": { "value": 0.00784090507888757, "lo": 0.005835860267174734, "hi": 0.012494261150418014 }, "uncertainty": { "value": 0.24430556881829574, "lo": 0.24086262703703576, "hi": 0.2468676430223986 }, "auroc": { "value": 0.5909461843255213, "lo": 0.5695110838859726, "hi": 0.6109242883847428 }, "acc@80": { "value": 0.5972850678733032, "lo": 0.5770786718246081, "hi": 0.6213368257061813 } } }, { "env": "civilcomments", "slice": "all", "system": "mix7-450M-s1", "metrics": { "n": { "value": 10581.0, "lo": 10261.425, "hi": 10904.525 }, "ce": { "value": 0.32425900287305687, "lo": 0.3163181434463966, "hi": 0.33175254743750576 }, "kl": { "value": 0.0927761941037158, "lo": 0.0899606203264132, "hi": 0.09570525052304658 }, "jsd": { "value": 0.02541242196432821, "lo": 0.02467779413471958, "hi": 0.026152304085951525 }, "brier": { "value": 0.044416175920525686, "lo": 0.04240057062755955, "hi": 0.04647481358668125 }, "acc": { "value": 0.9396087326339665, "lo": 0.9352888967187092, "hi": 0.9439752403673666 }, "conf": { "value": 0.8663100878296226, "lo": 0.8631051099045708, "hi": 0.8698803377815415 }, "ece": { "value": 0.07329864480434391, "lo": 0.06895776154217659, "hi": 0.07696218223674356 }, "reliability": { "value": 0.007928800038177412, "lo": 0.0070054302122457, "hi": 0.00882283596655696 }, "resolution": { "value": 0.011163723718570301, "lo": 0.009838755300416056, "hi": 0.012906397767968134 }, "uncertainty": { "value": 0.05674416219195776, "lo": 0.05288598575935794, "hi": 0.06052357633528109 }, "auroc": { "value": 0.9061359484383446, "lo": 0.8971158509009561, "hi": 0.9145749425838893 }, "acc@80": { "value": 0.9866509155345541, "lo": 0.9836346548870232, "hi": 0.9891896272006314 } } }, { "env": "folktables", "slice": "all", "system": "mix7-450M-s1", "metrics": { "n": { "value": 15003.0, "lo": 14855.8, "hi": 15157.0 }, "ce": { "value": 0.4356261342600311, "lo": 0.4262985371348895, "hi": 0.44383308741446337 }, "kl": { "value": 0.4356261342600311, "lo": 0.4262985371348895, "hi": 0.44383308741446337 }, "jsd": { "value": 0.12577303288797503, "lo": 0.12356107449550632, "hi": 0.12789032184153792 }, "brier": { "value": 0.2863458999243068, "lo": 0.2798399479718523, "hi": 0.29287672179440166 }, "acc": { "value": 0.789375458241685, "lo": 0.782430143058245, "hi": 0.7961437083938346 }, "conf": { "value": 0.7976119624434539, "lo": 0.7947091525830844, "hi": 0.8002880792451132 }, "ece": { "value": 0.013219735506875569, "lo": 0.011337052212381292, "hi": 0.020845035677029018 }, "reliability": { "value": 0.0003257695476476921, "lo": 0.00024292459016608795, "hi": 0.0007882323891700785 }, "resolution": { "value": 0.02332887340217792, "lo": 0.02159768656050772, "hi": 0.0255286686995298 }, "uncertainty": { "value": 0.16626184416741482, "lo": 0.16229890348613865, "hi": 0.17023321077601278 }, "auroc": { "value": 0.7633103515723115, "lo": 0.7554848240124551, "hi": 0.7721751096809657 }, "acc@80": { "value": 0.8421929678386936, "lo": 0.8359579924822174, "hi": 0.848862181247759 } } }, { "env": "mcq", "slice": "all", "system": "mix7-450M-s1", "metrics": { "n": { "value": 5064.0, "lo": 5064.0, "hi": 5064.0 }, "ce": { "value": 0.7268390369943714, "lo": 0.701661708648479, "hi": 0.7544727097001674 }, "kl": { "value": 0.7268390369943714, "lo": 0.701661708648479, "hi": 0.7544727097001674 }, "jsd": { "value": 0.1737407686543726, "lo": 0.1686769082277473, "hi": 0.17902775793235276 }, "brier": { "value": 0.38048971460670034, "lo": 0.36697716722425716, "hi": 0.39513580937699744 }, "acc": { "value": 0.7215639810426541, "lo": 0.7087282780410743, "hi": 0.733214849921011 }, "conf": { "value": 0.7886344792283071, "lo": 0.7825879776870941, "hi": 0.7938439071054848 }, "ece": { "value": 0.06707049818565308, "lo": 0.05728493071724713, "hi": 0.07796081099659204 }, "reliability": { "value": 0.006713542783910689, "lo": 0.005178319951393914, "hi": 0.00933968993179277 }, "resolution": { "value": 0.05029871507191342, "lo": 0.04609185458371129, "hi": 0.05585772583013485 }, "uncertainty": { "value": 0.20090940230453042, "lo": 0.1956108337763203, "hi": 0.20643250594600798 }, "auroc": { "value": 0.8156515544996836, "lo": 0.8048366961594599, "hi": 0.8276977452680627 }, "acc@80": { "value": 0.800049370525796, "lo": 0.7878116514440878, "hi": 0.8131449024932116 } } }, { "env": "stsb", "slice": "all", "system": "mix7-450M-s1", "metrics": { "n": { "value": 3676.0, "lo": 3150.95, "hi": 4793.674999999996 }, "ce": { "value": 1.1069691649903395, "lo": 1.0446012821079593, "hi": 1.165738121532312 }, "kl": { "value": 0.7046653421280841, "lo": 0.6449676716730225, "hi": 0.760569363090012 }, "jsd": { "value": 0.16901927548315193, "lo": 0.1571037350910772, "hi": 0.17973577357612286 }, "brier": { "value": 0.32481970890073175, "lo": 0.3027704832669146, "hi": 0.3472728203806762 }, "acc": { "value": 0.5620239390642002, "lo": 0.5360869489613143, "hi": 0.5870780451619467 }, "conf": { "value": 0.6037559004194459, "lo": 0.5937292598986336, "hi": 0.6165383115026002 }, "ece": { "value": 0.04551832948662383, "lo": 0.036274910117460646, "hi": 0.06540883176745567 }, "reliability": { "value": 0.0029292827612687512, "lo": 0.0019677464215818396, "hi": 0.005977527992258667 }, "resolution": { "value": 0.013574186923772965, "lo": 0.010217866698878853, "hi": 0.017759200429510277 }, "uncertainty": { "value": 0.24615303098296037, "lo": 0.2424174069956571, "hi": 0.24869771644786673 }, "auroc": { "value": 0.6223150024351675, "lo": 0.6008536046440507, "hi": 0.640362722022482 }, "acc@80": { "value": 0.5946956817409045, "lo": 0.5682574797266854, "hi": 0.6205814035115087 } } }, { "env": "yelp", "slice": "all", "system": "mix7-450M-s1", "metrics": { "n": { "value": 5000.0, "lo": 5000.0, "hi": 5000.0 }, "ce": { "value": 1.4667452588920318, "lo": 1.444240469510126, "hi": 1.49047082016875 }, "kl": { "value": 1.4667452588920318, "lo": 1.444240469510126, "hi": 1.49047082016875 }, "jsd": { "value": 0.36672095432867907, "lo": 0.36240916995821554, "hi": 0.3710108191970698 }, "brier": { "value": 0.7609335954239111, "lo": 0.7495009082898206, "hi": 0.7717066495198419 }, "acc": { "value": 0.3754, "lo": 0.36138, "hi": 0.3894 }, "conf": { "value": 0.5026116242468357, "lo": 0.499141373424083, "hi": 0.5060167957246303 }, "ece": { "value": 0.1303246833860874, "lo": 0.11905975172311067, "hi": 0.1465369994214177 }, "reliability": { "value": 0.0262975168219169, "lo": 0.022168531149260436, "hi": 0.031952602452765634 }, "resolution": { "value": 0.0025656878872772973, "lo": 0.002004726317232737, "hi": 0.004824254096524662 }, "uncertainty": { "value": 0.23447484000000002, "lo": 0.230784336, "hi": 0.23776764000000003 }, "auroc": { "value": 0.5356010052080641, "lo": 0.5200740038979366, "hi": 0.5516852145424147 }, "acc@80": { "value": 0.39225, "lo": 0.3755, "hi": 0.40675 } } }, { "env": "boolq", "slice": "all", "system": "mix7-450M-s2", "metrics": { "n": { "value": 8129.0, "lo": 7984.475, "hi": 8270.575 }, "ce": { "value": 0.7508329721009208, "lo": 0.7358536079983113, "hi": 0.7664551576724759 }, "kl": { "value": 0.7508329721009208, "lo": 0.7358536079983113, "hi": 0.7664551576724759 }, "jsd": { "value": 0.20942762939546353, "lo": 0.20584896383359036, "hi": 0.2130091640498956 }, "brier": { "value": 0.5348680335227326, "lo": 0.5232469861530633, "hi": 0.5472430128018364 }, "acc": { "value": 0.5651371632427112, "lo": 0.5530735884451423, "hi": 0.575905109150735 }, "conf": { "value": 0.7431906673693836, "lo": 0.740020551374251, "hi": 0.7465173845156629 }, "ece": { "value": 0.1780535041266723, "lo": 0.16736057480662753, "hi": 0.1895971520382431 }, "reliability": { "value": 0.03491287469138259, "lo": 0.031337234242882296, "hi": 0.039678006536083436 }, "resolution": { "value": 0.013110929383974334, "lo": 0.011345961017750624, "hi": 0.0159477605978508 }, "uncertainty": { "value": 0.2457571499646924, "lo": 0.24423841095816076, "hi": 0.2471831856925081 }, "auroc": { "value": 0.6219674022878374, "lo": 0.6097153466302697, "hi": 0.6349263420934658 }, "acc@80": { "value": 0.5918806704597878, "lo": 0.5782243718484255, "hi": 0.6048652525465069 } } }, { "env": "chaosnli", "slice": "all", "system": "mix7-450M-s2", "metrics": { "n": { "value": 2763.0, "lo": 2732.475, "hi": 2798.525 }, "ce": { "value": 0.9068941962366361, "lo": 0.8888516301850681, "hi": 0.9230321660093607 }, "kl": { "value": 0.37653412616737425, "lo": 0.3590756962948154, "hi": 0.3930393550067818 }, "jsd": { "value": 0.0937054334217359, "lo": 0.08994113864237777, "hi": 0.09745959474970158 }, "brier": { "value": 0.2526296104819719, "lo": 0.24055254013616717, "hi": 0.26465888636043466 }, "acc": { "value": 0.5798045602605864, "lo": 0.5618061180489594, "hi": 0.5990943704525836 }, "conf": { "value": 0.642501125010381, "lo": 0.6378141976322809, "hi": 0.647348896192297 }, "ece": { "value": 0.08573853579112858, "lo": 0.0736734226832399, "hi": 0.10743256827409897 }, "reliability": { "value": 0.01218772358814043, "lo": 0.009377742264503228, "hi": 0.01817624780187894 }, "resolution": { "value": 0.004357190061475463, "lo": 0.0034560381761563077, "hi": 0.008340115579019522 }, "uncertainty": { "value": 0.24363123216161445, "lo": 0.24018030445412428, "hi": 0.24617989033616938 }, "auroc": { "value": 0.5601164995091192, "lo": 0.5395851619829724, "hi": 0.5819451806207654 }, "acc@80": { "value": 0.602262443438914, "lo": 0.5819354162148589, "hi": 0.6237648972781059 } } }, { "env": "civilcomments", "slice": "all", "system": "mix7-450M-s2", "metrics": { "n": { "value": 10581.0, "lo": 10261.425, "hi": 10904.525 }, "ce": { "value": 0.3247535194409554, "lo": 0.316974842136389, "hi": 0.3318598707986615 }, "kl": { "value": 0.09327071067161433, "lo": 0.09064621848006946, "hi": 0.0961752930409639 }, "jsd": { "value": 0.02606528108395236, "lo": 0.025350973955439525, "hi": 0.02679507349136137 }, "brier": { "value": 0.044632186213394674, "lo": 0.04271824048677188, "hi": 0.04659245419611563 }, "acc": { "value": 0.9398922597108024, "lo": 0.934916151775364, "hi": 0.9446732982719718 }, "conf": { "value": 0.8583259791454062, "lo": 0.8553666134039802, "hi": 0.8618230064894931 }, "ece": { "value": 0.08156628056539615, "lo": 0.07692101049155381, "hi": 0.08594324081028143 }, "reliability": { "value": 0.00975207879263131, "lo": 0.008637521595815463, "hi": 0.010896552611447664 }, "resolution": { "value": 0.010355037855520537, "lo": 0.008795191276641622, "hi": 0.012097052840457853 }, "uncertainty": { "value": 0.056494799846524, "lo": 0.05226565402111658, "hi": 0.06084793886484167 }, "auroc": { "value": 0.8993091721449102, "lo": 0.8904418349475692, "hi": 0.9084257351286236 }, "acc@80": { "value": 0.9846426461901949, "lo": 0.9819117897378651, "hi": 0.9873751089781088 } } }, { "env": "folktables", "slice": "all", "system": "mix7-450M-s2", "metrics": { "n": { "value": 15003.0, "lo": 14855.8, "hi": 15157.0 }, "ce": { "value": 0.4384948531105458, "lo": 0.4294832468311023, "hi": 0.4474398319850143 }, "kl": { "value": 0.4384948531105458, "lo": 0.4294832468311023, "hi": 0.4474398319850143 }, "jsd": { "value": 0.12703740170169311, "lo": 0.12481733848097287, "hi": 0.12926476086816707 }, "brier": { "value": 0.28842804801350497, "lo": 0.2818311021471689, "hi": 0.2952518595623806 }, "acc": { "value": 0.7847097247217223, "lo": 0.7777182306995476, "hi": 0.7918898360105096 }, "conf": { "value": 0.7962835168970399, "lo": 0.7936710437870885, "hi": 0.7987139368304775 }, "ece": { "value": 0.014699667606768206, "lo": 0.01157972838630544, "hi": 0.02266268690188863 }, "reliability": { "value": 0.00039307823531009166, "lo": 0.00023782628922572584, "hi": 0.0009264038908527687 }, "resolution": { "value": 0.0249705180780406, "lo": 0.02304098520361409, "hi": 0.027294810654141508 }, "uncertainty": { "value": 0.16894037264888112, "lo": 0.16480031233278103, "hi": 0.1728725842837765 }, "auroc": { "value": 0.7679356316954442, "lo": 0.7603918536552557, "hi": 0.7756067883381488 }, "acc@80": { "value": 0.8422762872854525, "lo": 0.8344919684580364, "hi": 0.849076861428603 } } }, { "env": "mcq", "slice": "all", "system": "mix7-450M-s2", "metrics": { "n": { "value": 5064.0, "lo": 5064.0, "hi": 5064.0 }, "ce": { "value": 0.7446275684259486, "lo": 0.718017758419548, "hi": 0.7752686950616806 }, "kl": { "value": 0.7446275684259486, "lo": 0.718017758419548, "hi": 0.7752686950616806 }, "jsd": { "value": 0.17277025351490105, "lo": 0.1681006773300853, "hi": 0.17823748981647008 }, "brier": { "value": 0.3832624003293176, "lo": 0.37073998564183014, "hi": 0.3978390423209227 }, "acc": { "value": 0.7278830963665087, "lo": 0.7161285545023697, "hi": 0.7391488941548183 }, "conf": { "value": 0.79739615916735, "lo": 0.7916504391466119, "hi": 0.8023993313515567 }, "ece": { "value": 0.06951306280084134, "lo": 0.059637876585902756, "hi": 0.07954645595597148 }, "reliability": { "value": 0.006545407602718125, "lo": 0.005115935871504275, "hi": 0.009169499928122849 }, "resolution": { "value": 0.04375928005498686, "lo": 0.0397319930327455, "hi": 0.04924385825133258 }, "uncertainty": { "value": 0.1980692943904125, "lo": 0.1928077675266853, "hi": 0.20328843820425568 }, "auroc": { "value": 0.800425865098159, "lo": 0.788415624556946, "hi": 0.8130848853706965 }, "acc@80": { "value": 0.8047395704764255, "lo": 0.7931374969143421, "hi": 0.8157306837817824 } } }, { "env": "stsb", "slice": "all", "system": "mix7-450M-s2", "metrics": { "n": { "value": 3676.0, "lo": 3150.95, "hi": 4793.674999999996 }, "ce": { "value": 1.1153752379761714, "lo": 1.052273713920008, "hi": 1.174999009193296 }, "kl": { "value": 0.7130714151139158, "lo": 0.6529503982280231, "hi": 0.7707624273377698 }, "jsd": { "value": 0.17712156250081665, "lo": 0.16455952178985314, "hi": 0.18869218834487786 }, "brier": { "value": 0.33239855464142265, "lo": 0.30888337549417993, "hi": 0.35571402833468196 }, "acc": { "value": 0.5424374319912949, "lo": 0.5119142542368635, "hi": 0.572282029555273 }, "conf": { "value": 0.5774308672505183, "lo": 0.567075724870965, "hi": 0.591399122720293 }, "ece": { "value": 0.040822435904119915, "lo": 0.03244471963777056, "hi": 0.06495263875134669 }, "reliability": { "value": 0.002350762670456841, "lo": 0.0015834587515945622, "hi": 0.005964010156505599 }, "resolution": { "value": 0.013943037849111384, "lo": 0.011632704935385763, "hi": 0.017970457315596322 }, "uncertainty": { "value": 0.2481990643659842, "lo": 0.2447753076204106, "hi": 0.24985804768021877 }, "auroc": { "value": 0.6181128104885405, "lo": 0.5952025262220156, "hi": 0.6359539621774973 }, "acc@80": { "value": 0.5702142128527712, "lo": 0.536247408370524, "hi": 0.6019217497658009 } } }, { "env": "yelp", "slice": "all", "system": "mix7-450M-s2", "metrics": { "n": { "value": 5000.0, "lo": 5000.0, "hi": 5000.0 }, "ce": { "value": 1.2990671352362657, "lo": 1.2826223818640479, "hi": 1.316544163088931 }, "kl": { "value": 1.2990671352362657, "lo": 1.2826223818640479, "hi": 1.316544163088931 }, "jsd": { "value": 0.3433666298098565, "lo": 0.34000996210799006, "hi": 0.34651044451864993 }, "brier": { "value": 0.6990184567747504, "lo": 0.6901114858907645, "hi": 0.7073333773478532 }, "acc": { "value": 0.3984, "lo": 0.384085, "hi": 0.4118 }, "conf": { "value": 0.4726655150473118, "lo": 0.4689257402203232, "hi": 0.4758530365492403 }, "ece": { "value": 0.07504558734297752, "lo": 0.06929116827212274, "hi": 0.09375943623945117 }, "reliability": { "value": 0.012138016938554872, "lo": 0.009904491197695403, "hi": 0.01594855405546701 }, "resolution": { "value": 0.005249490304021365, "lo": 0.0038488391332653954, "hi": 0.008041268008154764 }, "uncertainty": { "value": 0.23967744, "lo": 0.236563623, "hi": 0.24222076000000003 }, "auroc": { "value": 0.5681968231970435, "lo": 0.5512457926399172, "hi": 0.5857333428553422 }, "acc@80": { "value": 0.42575, "lo": 0.4105, "hi": 0.44125 } } }, { "env": "boolq", "slice": "all", "system": "mix7-3B-s0", "metrics": { "n": { "value": 8129.0, "lo": 7984.475, "hi": 8270.575 }, "ce": { "value": 0.4015344595969336, "lo": 0.3860123018209743, "hi": 0.4162750405789503 }, "kl": { "value": 0.4015344595969336, "lo": 0.3860123018209743, "hi": 0.4162750405789503 }, "jsd": { "value": 0.11153371062325852, "lo": 0.10802042083492568, "hi": 0.11504278763351378 }, "brier": { "value": 0.2557792622318184, "lo": 0.24457311973185, "hi": 0.26649771244224674 }, "acc": { "value": 0.8148603764300651, "lo": 0.8049397055807423, "hi": 0.8241163617325865 }, "conf": { "value": 0.8385382051000971, "lo": 0.8348062592992268, "hi": 0.842005228281196 }, "ece": { "value": 0.023677828670031976, "lo": 0.018086918447659944, "hi": 0.033850399052529526 }, "reliability": { "value": 0.0009166159624454497, "lo": 0.0005855294696240854, "hi": 0.0018833355470140156 }, "resolution": { "value": 0.02349288559872494, "lo": 0.021108489281470456, "hi": 0.02688732478426397 }, "uncertainty": { "value": 0.15086294335431769, "lo": 0.14494858385545084, "hi": 0.15701177536230493 }, "auroc": { "value": 0.7821920590784343, "lo": 0.7696202531580901, "hi": 0.794582540631279 }, "acc@80": { "value": 0.8782100568968169, "lo": 0.8675312644742936, "hi": 0.8860653702855604 } } }, { "env": "chaosnli", "slice": "all", "system": "mix7-3B-s0", "metrics": { "n": { "value": 2763.0, "lo": 2732.475, "hi": 2798.525 }, "ce": { "value": 0.7317921943962872, "lo": 0.7175234800133125, "hi": 0.7461812755771366 }, "kl": { "value": 0.2014321243270253, "lo": 0.1915986327201315, "hi": 0.2125726256760473 }, "jsd": { "value": 0.050709724346277796, "lo": 0.04841450046627548, "hi": 0.05330863049440536 }, "brier": { "value": 0.13177767445592195, "lo": 0.1241462323637343, "hi": 0.1397989300330173 }, "acc": { "value": 0.7321751719145856, "lo": 0.7150233697658727, "hi": 0.7488255251608713 }, "conf": { "value": 0.6842451099248369, "lo": 0.6790877096119059, "hi": 0.689623098340747 }, "ece": { "value": 0.048137297006435174, "lo": 0.03969545172528373, "hi": 0.06678392948039072 }, "reliability": { "value": 0.003264699173978197, "lo": 0.002404396547416299, "hi": 0.006193114064328454 }, "resolution": { "value": 0.021631081394147284, "lo": 0.018310029205294103, "hi": 0.02754729414658824 }, "uncertainty": { "value": 0.19609468954643264, "lo": 0.18808585802554073, "hi": 0.20376483602107143 }, "auroc": { "value": 0.7111210939065611, "lo": 0.6915367421288887, "hi": 0.7322106870758519 }, "acc@80": { "value": 0.783710407239819, "lo": 0.7668429473017556, "hi": 0.8015500885942087 } } }, { "env": "civilcomments", "slice": "all", "system": "mix7-3B-s0", "metrics": { "n": { "value": 10581.0, "lo": 10261.425, "hi": 10904.525 }, "ce": { "value": 0.32493529455042947, "lo": 0.31631111338723766, "hi": 0.332856731464495 }, "kl": { "value": 0.09345248578108843, "lo": 0.09049580801852516, "hi": 0.09640686219627244 }, "jsd": { "value": 0.023236853210795962, "lo": 0.022569521394711495, "hi": 0.023902308587587625 }, "brier": { "value": 0.044440062724136684, "lo": 0.0424329646687974, "hi": 0.04654394576966348 }, "acc": { "value": 0.9376240430961157, "lo": 0.9325575502601535, "hi": 0.9421614171606318 }, "conf": { "value": 0.8938100220361445, "lo": 0.8909953417291749, "hi": 0.8969365017590876 }, "ece": { "value": 0.045806885644400594, "lo": 0.04272027448871869, "hi": 0.049717523521862775 }, "reliability": { "value": 0.003369591571482668, "lo": 0.002855248382434492, "hi": 0.003956748929645738 }, "resolution": { "value": 0.013996847595005229, "lo": 0.011918922910384185, "hi": 0.016271498684583937 }, "uncertainty": { "value": 0.05848519690420908, "lo": 0.05449328087635878, "hi": 0.06289396568079476 }, "auroc": { "value": 0.9210726863433244, "lo": 0.9125079832657689, "hi": 0.9294178166364858 }, "acc@80": { "value": 0.9868871825162433, "lo": 0.9844468765677032, "hi": 0.9895334115529497 } } }, { "env": "folktables", "slice": "all", "system": "mix7-3B-s0", "metrics": { "n": { "value": 15003.0, "lo": 14855.8, "hi": 15157.0 }, "ce": { "value": 0.44509115726019133, "lo": 0.43615898719736423, "hi": 0.4543582098797216 }, "kl": { "value": 0.44509115726019133, "lo": 0.43615898719736423, "hi": 0.4543582098797216 }, "jsd": { "value": 0.12672637436350415, "lo": 0.12465214829656616, "hi": 0.12904600548269632 }, "brier": { "value": 0.2930003292994701, "lo": 0.28668903733955103, "hi": 0.3004513879616006 }, "acc": { "value": 0.7876424715056989, "lo": 0.7805317861129476, "hi": 0.7939606685270852 }, "conf": { "value": 0.8070413636770326, "lo": 0.8041954014356815, "hi": 0.8095585776890852 }, "ece": { "value": 0.02779868638635452, "lo": 0.022337478545157424, "hi": 0.0340497470428343 }, "reliability": { "value": 0.001027915756892152, "lo": 0.0007314647635620185, "hi": 0.0016595918413771723 }, "resolution": { "value": 0.02169735023637832, "lo": 0.020282061718895836, "hi": 0.02374580577398444 }, "uncertainty": { "value": 0.1672618085860932, "lo": 0.16358712397960315, "hi": 0.17130191668681444 }, "auroc": { "value": 0.7532512981367189, "lo": 0.7450062741277153, "hi": 0.761987673941276 }, "acc@80": { "value": 0.8386935510748209, "lo": 0.8307700002943383, "hi": 0.8448743534482759 } } }, { "env": "mcq", "slice": "all", "system": "mix7-3B-s0", "metrics": { "n": { "value": 5064.0, "lo": 5064.0, "hi": 5064.0 }, "ce": { "value": 0.28394581130659097, "lo": 0.26603788540962414, "hi": 0.3050438775655096 }, "kl": { "value": 0.28394581130659097, "lo": 0.26603788540962414, "hi": 0.3050438775655096 }, "jsd": { "value": 0.06816561102544832, "lo": 0.06459715585964569, "hi": 0.0721479748574227 }, "brier": { "value": 0.14707832873591406, "lo": 0.13726122315282865, "hi": 0.1568657334447049 }, "acc": { "value": 0.8992890995260664, "lo": 0.8912766587677725, "hi": 0.9069905213270142 }, "conf": { "value": 0.9096089693560231, "lo": 0.9052909305017577, "hi": 0.9133925156906176 }, "ece": { "value": 0.01118656555058266, "lo": 0.009677040375810336, "hi": 0.02007273871548722 }, "reliability": { "value": 0.00036228744679989394, "lo": 0.00023085968401454236, "hi": 0.001134681554697722 }, "resolution": { "value": 0.02394559257678946, "lo": 0.020469756669410887, "hi": 0.028077017761401986 }, "uncertainty": { "value": 0.09056821499966304, "lo": 0.08435871554996521, "hi": 0.09690248878338312 }, "auroc": { "value": 0.9004529523711109, "lo": 0.8896641740533964, "hi": 0.9115282046258159 }, "acc@80": { "value": 0.9674154529745742, "lo": 0.9612441372500617, "hi": 0.9728462108121452 } } }, { "env": "stsb", "slice": "all", "system": "mix7-3B-s0", "metrics": { "n": { "value": 3676.0, "lo": 3150.95, "hi": 4793.674999999996 }, "ce": { "value": 1.0320934313543655, "lo": 0.9737943844535284, "hi": 1.0907567401781189 }, "kl": { "value": 0.6297896084921099, "lo": 0.5771812394309076, "hi": 0.6852677359838089 }, "jsd": { "value": 0.14011290764967546, "lo": 0.13022503773077626, "hi": 0.14963892944676882 }, "brier": { "value": 0.2879302523060823, "lo": 0.26417715261978636, "hi": 0.31065636616289766 }, "acc": { "value": 0.6235038084874864, "lo": 0.5980606588589551, "hi": 0.6456225171251262 }, "conf": { "value": 0.6865655293564542, "lo": 0.6821742615009048, "hi": 0.6905385006638595 }, "ece": { "value": 0.06937201697302331, "lo": 0.05758364124893235, "hi": 0.09149823202160497 }, "reliability": { "value": 0.006808533333740608, "lo": 0.004744926620305128, "hi": 0.011386983417516254 }, "resolution": { "value": 0.012280547113417957, "lo": 0.009840436482986279, "hi": 0.01631852055148593 }, "uncertainty": { "value": 0.23474680928908628, "lo": 0.22879368686549964, "hi": 0.24038410235088176 }, "auroc": { "value": 0.6192176040815504, "lo": 0.5996142966251977, "hi": 0.6389117903837324 }, "acc@80": { "value": 0.6501190071404285, "lo": 0.622900699736252, "hi": 0.6762869012258145 } } }, { "env": "yelp", "slice": "all", "system": "mix7-3B-s0", "metrics": { "n": { "value": 5000.0, "lo": 5000.0, "hi": 5000.0 }, "ce": { "value": 1.0391471499843858, "lo": 1.0134554300556946, "hi": 1.0625187943526921 }, "kl": { "value": 1.0391471499843858, "lo": 1.0134554300556946, "hi": 1.0625187943526921 }, "jsd": { "value": 0.2630772132910125, "lo": 0.2576900758880535, "hi": 0.2682183793453246 }, "brier": { "value": 0.5788665208432717, "lo": 0.563677305243213, "hi": 0.5927627160220487 }, "acc": { "value": 0.5574, "lo": 0.5436949999999999, "hi": 0.571705 }, "conf": { "value": 0.6398223581820727, "lo": 0.6342675311058015, "hi": 0.6450378332845866 }, "ece": { "value": 0.08723655652105806, "lo": 0.07699143006928265, "hi": 0.10208049490526318 }, "reliability": { "value": 0.011098361250199508, "lo": 0.009022912139590654, "hi": 0.014836317323023748 }, "resolution": { "value": 0.019314236421571273, "lo": 0.01676013430935074, "hi": 0.023318424272665134 }, "uncertainty": { "value": 0.24670524, "lo": 0.24485838299999999, "hi": 0.248090737 }, "auroc": { "value": 0.650048049242894, "lo": 0.633433680844322, "hi": 0.6650909921020128 }, "acc@80": { "value": 0.59225, "lo": 0.57625, "hi": 0.6096312500000001 } } }, { "env": "boolq", "slice": "all", "system": "mix7-3B-s1", "metrics": { "n": { "value": 8129.0, "lo": 7984.475, "hi": 8270.575 }, "ce": { "value": 0.4049574630856653, "lo": 0.39007587939008853, "hi": 0.419696224864637 }, "kl": { "value": 0.4049574630856653, "lo": 0.39007587939008853, "hi": 0.419696224864637 }, "jsd": { "value": 0.111901545028253, "lo": 0.10854569451679102, "hi": 0.11533485690992372 }, "brier": { "value": 0.25681659383982436, "lo": 0.24637147985817529, "hi": 0.2673096727568516 }, "acc": { "value": 0.8173207036535859, "lo": 0.8080316120551718, "hi": 0.8275816552524383 }, "conf": { "value": 0.8375277741176204, "lo": 0.8336681292483965, "hi": 0.841110267633026 }, "ece": { "value": 0.020491637790873277, "lo": 0.016299947854084208, "hi": 0.03069316298435388 }, "reliability": { "value": 0.0007266617534696402, "lo": 0.00047220778666538565, "hi": 0.001557464036775021 }, "resolution": { "value": 0.02125015517324278, "lo": 0.018781052645828994, "hi": 0.023955819660760647 }, "uncertainty": { "value": 0.14930757103279313, "lo": 0.14269003212759473, "hi": 0.1551165258621473 }, "auroc": { "value": 0.7717524431552125, "lo": 0.759722393146674, "hi": 0.7835680600374859 }, "acc@80": { "value": 0.8769798554513302, "lo": 0.8675635293138366, "hi": 0.8852583732057416 } } }, { "env": "chaosnli", "slice": "all", "system": "mix7-3B-s1", "metrics": { "n": { "value": 2763.0, "lo": 2732.475, "hi": 2798.525 }, "ce": { "value": 0.7349974377531698, "lo": 0.7208577981229234, "hi": 0.7501208449342154 }, "kl": { "value": 0.204637367683908, "lo": 0.19473334216220856, "hi": 0.21571769509786942 }, "jsd": { "value": 0.05172847852322033, "lo": 0.04934382396665588, "hi": 0.05433960084166812 }, "brier": { "value": 0.13334485416438893, "lo": 0.12546612960853043, "hi": 0.14151518877841704 }, "acc": { "value": 0.7285559174809989, "lo": 0.7118080916048567, "hi": 0.7442746559724079 }, "conf": { "value": 0.687175090378538, "lo": 0.6818579669141838, "hi": 0.6933242753211061 }, "ece": { "value": 0.04855649530822884, "lo": 0.04099712850132304, "hi": 0.06923226641689625 }, "reliability": { "value": 0.003739122579211648, "lo": 0.0027199493628244638, "hi": 0.006664422066402824 }, "resolution": { "value": 0.023876320718372028, "lo": 0.019966860404635328, "hi": 0.030223145045202605 }, "uncertainty": { "value": 0.19776219258441882, "lo": 0.19032988663427408, "hi": 0.20513721000332208 }, "auroc": { "value": 0.7173164431197218, "lo": 0.6983102158074311, "hi": 0.7388496103712955 }, "acc@80": { "value": 0.783710407239819, "lo": 0.7663283716734326, "hi": 0.8005933503074701 } } }, { "env": "civilcomments", "slice": "all", "system": "mix7-3B-s1", "metrics": { "n": { "value": 10581.0, "lo": 10261.425, "hi": 10904.525 }, "ce": { "value": 0.31942936538814876, "lo": 0.31149573393331353, "hi": 0.3271525389195797 }, "kl": { "value": 0.0879465566188077, "lo": 0.08520028414159837, "hi": 0.09096505372521008 }, "jsd": { "value": 0.022278551457989914, "lo": 0.021592106042922107, "hi": 0.022963349235797917 }, "brier": { "value": 0.040234408557455296, "lo": 0.03843925900728954, "hi": 0.04211302940769042 }, "acc": { "value": 0.9445232019657878, "lo": 0.9398753114478805, "hi": 0.9490178170496311 }, "conf": { "value": 0.877979220542731, "lo": 0.8746057752849715, "hi": 0.8814415317216445 }, "ece": { "value": 0.06654398142305668, "lo": 0.062062010685223434, "hi": 0.07060110024882085 }, "reliability": { "value": 0.00795537710407801, "lo": 0.007002408528562913, "hi": 0.0090299316599831 }, "resolution": { "value": 0.010679213571236502, "lo": 0.009024108954011732, "hi": 0.012601257559940677 }, "uncertainty": { "value": 0.05239912291408346, "lo": 0.04838299931780303, "hi": 0.056509704103497885 }, "auroc": { "value": 0.913231925526696, "lo": 0.9052468216388458, "hi": 0.9223523500697764 }, "acc@80": { "value": 0.9890135853514471, "lo": 0.9868124670718026, "hi": 0.9914803908055612 } } }, { "env": "folktables", "slice": "all", "system": "mix7-3B-s1", "metrics": { "n": { "value": 15003.0, "lo": 14855.8, "hi": 15157.0 }, "ce": { "value": 0.44111304791622236, "lo": 0.43174813491236325, "hi": 0.44934647479135614 }, "kl": { "value": 0.44111304791622236, "lo": 0.43174813491236325, "hi": 0.44934647479135614 }, "jsd": { "value": 0.12770348267111706, "lo": 0.12550513145803793, "hi": 0.12981442786045344 }, "brier": { "value": 0.289130483213222, "lo": 0.2827122997681443, "hi": 0.2951485319528612 }, "acc": { "value": 0.7869759381457042, "lo": 0.779830205792653, "hi": 0.7937276150183592 }, "conf": { "value": 0.7849255870844709, "lo": 0.7818482000939939, "hi": 0.7876585101401754 }, "ece": { "value": 0.014096905742098072, "lo": 0.011880030806329907, "hi": 0.021862798972842573 }, "reliability": { "value": 0.0002827487042510449, "lo": 0.0002090908089736323, "hi": 0.0006698803904785933 }, "resolution": { "value": 0.02329595916455912, "lo": 0.02145050436541092, "hi": 0.025442393969833454 }, "uncertainty": { "value": 0.16764481092539296, "lo": 0.16372408614990336, "hi": 0.17169505174081806 }, "auroc": { "value": 0.7613656299221321, "lo": 0.7528968817560899, "hi": 0.7702832216209622 }, "acc@80": { "value": 0.8421096483919347, "lo": 0.8348447237671797, "hi": 0.8485784365383892 } } }, { "env": "mcq", "slice": "all", "system": "mix7-3B-s1", "metrics": { "n": { "value": 5064.0, "lo": 5064.0, "hi": 5064.0 }, "ce": { "value": 0.2891957645267851, "lo": 0.27197159366670415, "hi": 0.30715282098789554 }, "kl": { "value": 0.2891957645267851, "lo": 0.27197159366670415, "hi": 0.30715282098789554 }, "jsd": { "value": 0.07248578135348922, "lo": 0.0688045261914869, "hi": 0.07633694894831154 }, "brier": { "value": 0.1517139060824241, "lo": 0.14185286472944145, "hi": 0.16164069421183636 }, "acc": { "value": 0.8937598736176935, "lo": 0.8855598341232228, "hi": 0.9019599131121643 }, "conf": { "value": 0.8991462239086345, "lo": 0.8944063981877284, "hi": 0.9031266109305794 }, "ece": { "value": 0.00769100147542233, "lo": 0.006277996731118698, "hi": 0.01696278155835766 }, "reliability": { "value": 0.00025283248472168286, "lo": 9.270644217381292e-05, "hi": 0.0009927991395790466 }, "resolution": { "value": 0.027283589988228798, "lo": 0.023354736970078813, "hi": 0.031692906294089315 }, "uncertainty": { "value": 0.09495316192857806, "lo": 0.08842821852640076, "hi": 0.1013436045864124 }, "auroc": { "value": 0.9045484002385228, "lo": 0.8941789210892567, "hi": 0.9148721676895425 }, "acc@80": { "value": 0.9696371266353987, "lo": 0.9637126635398667, "hi": 0.9743273265860282 } } }, { "env": "stsb", "slice": "all", "system": "mix7-3B-s1", "metrics": { "n": { "value": 3676.0, "lo": 3150.95, "hi": 4793.674999999996 }, "ce": { "value": 0.9606397550106914, "lo": 0.9136520337197678, "hi": 1.0053183174223232 }, "kl": { "value": 0.5583359321484358, "lo": 0.5134575243067591, "hi": 0.5998446529037125 }, "jsd": { "value": 0.1417157022674515, "lo": 0.13153584188974404, "hi": 0.15075761011863018 }, "brier": { "value": 0.2696511629637947, "lo": 0.25035290256773923, "hi": 0.2883582189865905 }, "acc": { "value": 0.6109902067464635, "lo": 0.5904998647433761, "hi": 0.62975556849448 }, "conf": { "value": 0.6308347471342123, "lo": 0.621701280045729, "hi": 0.6416677557116188 }, "ece": { "value": 0.03286231362683471, "lo": 0.027581300777750294, "hi": 0.053216983102494 }, "reliability": { "value": 0.0019507831759346804, "lo": 0.0013248013918855113, "hi": 0.004471052294358259 }, "resolution": { "value": 0.01910747870379169, "lo": 0.01575617488715062, "hi": 0.02309175560386782 }, "uncertainty": { "value": 0.23768117400637728, "lo": 0.23316348711925408, "hi": 0.24180977137399856 }, "auroc": { "value": 0.6517820336386676, "lo": 0.6311177764729875, "hi": 0.6689101056191983 }, "acc@80": { "value": 0.6501190071404285, "lo": 0.627964313123561, "hi": 0.6712623429020483 } } }, { "env": "yelp", "slice": "all", "system": "mix7-3B-s1", "metrics": { "n": { "value": 5000.0, "lo": 5000.0, "hi": 5000.0 }, "ce": { "value": 0.9725467655960507, "lo": 0.9492864944117893, "hi": 0.9923747008198048 }, "kl": { "value": 0.9725467655960507, "lo": 0.9492864944117893, "hi": 0.9923747008198048 }, "jsd": { "value": 0.2595922501934123, "lo": 0.2547513860074246, "hi": 0.2642575432437123 }, "brier": { "value": 0.5467747525784498, "lo": 0.5338918415026284, "hi": 0.5591425403359902 }, "acc": { "value": 0.5768, "lo": 0.5646, "hi": 0.59242 }, "conf": { "value": 0.6021929107874632, "lo": 0.5970300256822259, "hi": 0.6071611789501459 }, "ece": { "value": 0.035365436366200466, "lo": 0.030351453499123455, "hi": 0.0507533370282501 }, "reliability": { "value": 0.002083417082839355, "lo": 0.0016458693872491098, "hi": 0.003948515192838469 }, "resolution": { "value": 0.019561498181295066, "lo": 0.01691330508509431, "hi": 0.023520177786882694 }, "uncertainty": { "value": 0.24410176, "lo": 0.241458384, "hi": 0.24582684 }, "auroc": { "value": 0.6583241349837051, "lo": 0.6431085844867885, "hi": 0.6750408391933727 }, "acc@80": { "value": 0.6155, "lo": 0.59975, "hi": 0.632 } } }, { "env": "boolq", "slice": "all", "system": "mix7-3B-s2", "metrics": { "n": { "value": 8129.0, "lo": 7984.475, "hi": 8270.575 }, "ce": { "value": 0.4131032401681462, "lo": 0.3973496692894577, "hi": 0.427998296376232 }, "kl": { "value": 0.4131032401681462, "lo": 0.3973496692894577, "hi": 0.427998296376232 }, "jsd": { "value": 0.11348010140326135, "lo": 0.10986484531222152, "hi": 0.11709720283232482 }, "brier": { "value": 0.26307555605300936, "lo": 0.25154510471816366, "hi": 0.27411998163707557 }, "acc": { "value": 0.8106778201500799, "lo": 0.801569355404868, "hi": 0.8200104361128396 }, "conf": { "value": 0.8402179098830245, "lo": 0.8364491696032379, "hi": 0.8435558076027064 }, "ece": { "value": 0.02967430733860104, "lo": 0.022061073993597972, "hi": 0.03808999907093463 }, "reliability": { "value": 0.001150064358198883, "lo": 0.0007762966791525043, "hi": 0.002134593047143699 }, "resolution": { "value": 0.02284729456198637, "lo": 0.02037014493284156, "hi": 0.02564082887246402 }, "uncertainty": { "value": 0.1534792920667946, "lo": 0.14759312013249243, "hi": 0.15905592089447454 }, "auroc": { "value": 0.7737325244207016, "lo": 0.7613846273950495, "hi": 0.7844834670571107 }, "acc@80": { "value": 0.8726741503921267, "lo": 0.8627757865114729, "hi": 0.8818397680978345 } } }, { "env": "chaosnli", "slice": "all", "system": "mix7-3B-s2", "metrics": { "n": { "value": 2763.0, "lo": 2732.475, "hi": 2798.525 }, "ce": { "value": 0.7314480272773598, "lo": 0.7165841531560584, "hi": 0.7462843837267661 }, "kl": { "value": 0.2010879572080979, "lo": 0.19093257237017797, "hi": 0.21267601193959595 }, "jsd": { "value": 0.05125472097302959, "lo": 0.048962697249526135, "hi": 0.053971984200325965 }, "brier": { "value": 0.13283454092153876, "lo": 0.12531100763357664, "hi": 0.14145555307972574 }, "acc": { "value": 0.7350705754614549, "lo": 0.7182138184335632, "hi": 0.7539765682599965 }, "conf": { "value": 0.6815311866871395, "lo": 0.6761925889266218, "hi": 0.6868314750200839 }, "ece": { "value": 0.054777696259505855, "lo": 0.04402893164749189, "hi": 0.07315946449562966 }, "reliability": { "value": 0.00425957846132254, "lo": 0.003025259707686656, "hi": 0.007968014331708758 }, "resolution": { "value": 0.019088587588990097, "lo": 0.015589169144610369, "hi": 0.0244941432336972 }, "uncertainty": { "value": 0.19474182455222042, "lo": 0.1854955533308363, "hi": 0.20238271613029457 }, "auroc": { "value": 0.698795715588703, "lo": 0.6777626810697348, "hi": 0.7203468860875333 }, "acc@80": { "value": 0.783710407239819, "lo": 0.7646371632765255, "hi": 0.8012821438681834 } } }, { "env": "civilcomments", "slice": "all", "system": "mix7-3B-s2", "metrics": { "n": { "value": 10581.0, "lo": 10261.425, "hi": 10904.525 }, "ce": { "value": 0.3162963367925811, "lo": 0.30857301423871375, "hi": 0.32356174573071533 }, "kl": { "value": 0.08481352802324012, "lo": 0.08228301201192431, "hi": 0.087507702002961 }, "jsd": { "value": 0.02294056032259847, "lo": 0.02228275480633488, "hi": 0.02362822533140682 }, "brier": { "value": 0.0392586704145921, "lo": 0.037557024866026884, "hi": 0.040924470986403415 }, "acc": { "value": 0.9453737831962953, "lo": 0.9408058678375908, "hi": 0.9497721239851789 }, "conf": { "value": 0.8633439165355465, "lo": 0.8598479996963536, "hi": 0.8670561465722972 }, "ece": { "value": 0.08202986666074875, "lo": 0.07739240662636059, "hi": 0.0860125247996312 }, "reliability": { "value": 0.011687426094487264, "lo": 0.01055066906885218, "hi": 0.012810255865789797 }, "resolution": { "value": 0.010853550782185535, "lo": 0.009294702048276536, "hi": 0.012908751438985054 }, "uncertainty": { "value": 0.05164219324141937, "lo": 0.04770503161703058, "hi": 0.055690186487304075 }, "auroc": { "value": 0.9157116186943225, "lo": 0.9075945311556154, "hi": 0.9243755856561098 }, "acc@80": { "value": 0.9898405197873598, "lo": 0.9877442295024116, "hi": 0.9921699911982781 } } }, { "env": "folktables", "slice": "all", "system": "mix7-3B-s2", "metrics": { "n": { "value": 15003.0, "lo": 14855.8, "hi": 15157.0 }, "ce": { "value": 0.45390238176418163, "lo": 0.44257713882119504, "hi": 0.46506275189519375 }, "kl": { "value": 0.45390238176418163, "lo": 0.44257713882119504, "hi": 0.46506275189519375 }, "jsd": { "value": 0.1233984339381717, "lo": 0.12091792149914202, "hi": 0.12588460310110852 }, "brier": { "value": 0.2935044644711353, "lo": 0.2862414068257359, "hi": 0.30096003595232784 }, "acc": { "value": 0.7881090448576952, "lo": 0.7811839122725273, "hi": 0.7946900863642335 }, "conf": { "value": 0.8258664168088204, "lo": 0.8230876402888855, "hi": 0.8284361155496249 }, "ece": { "value": 0.03934656827201734, "lo": 0.035264071697279034, "hi": 0.046831063758928815 }, "reliability": { "value": 0.0023381925082443748, "lo": 0.0018911525496485661, "hi": 0.003290204609039463 }, "resolution": { "value": 0.022477558484952493, "lo": 0.02105558637455978, "hi": 0.024568491105061697 }, "uncertainty": { "value": 0.1669931782711866, "lo": 0.16315775020668882, "hi": 0.17093560734410718 }, "auroc": { "value": 0.7578438227483217, "lo": 0.7501047484842124, "hi": 0.7660957165442223 }, "acc@80": { "value": 0.8403599400099984, "lo": 0.832566749894723, "hi": 0.8466900172226277 } } }, { "env": "mcq", "slice": "all", "system": "mix7-3B-s2", "metrics": { "n": { "value": 5064.0, "lo": 5064.0, "hi": 5064.0 }, "ce": { "value": 0.2861016989829702, "lo": 0.2676710966396172, "hi": 0.3077958136689403 }, "kl": { "value": 0.2861016989829702, "lo": 0.2676710966396172, "hi": 0.3077958136689403 }, "jsd": { "value": 0.06600847510575883, "lo": 0.06225906479814482, "hi": 0.06988313589145256 }, "brier": { "value": 0.1476629415774208, "lo": 0.13761717537202953, "hi": 0.15737659063233683 }, "acc": { "value": 0.8996840442338072, "lo": 0.8914840047393365, "hi": 0.9069905213270142 }, "conf": { "value": 0.9201619169346718, "lo": 0.9160170786120998, "hi": 0.9236272624636431 }, "ece": { "value": 0.02074630266588068, "lo": 0.015070425714167513, "hi": 0.027729098887387272 }, "reliability": { "value": 0.0009633181351610409, "lo": 0.0005118152598797653, "hi": 0.0019473326818066304 }, "resolution": { "value": 0.0237194446043811, "lo": 0.020530420995619435, "hi": 0.027802868385090936 }, "uncertainty": { "value": 0.09025266478490804, "lo": 0.08435871554996521, "hi": 0.09674026430879061 }, "auroc": { "value": 0.9013879335375001, "lo": 0.8896879285924512, "hi": 0.9122964877806818 }, "acc@80": { "value": 0.9686497161194767, "lo": 0.9623488027647494, "hi": 0.9743273265860282 } } }, { "env": "stsb", "slice": "all", "system": "mix7-3B-s2", "metrics": { "n": { "value": 3676.0, "lo": 3150.95, "hi": 4793.674999999996 }, "ce": { "value": 1.0124180703390282, "lo": 0.9578298609416918, "hi": 1.0643383631981576 }, "kl": { "value": 0.6101142474767726, "lo": 0.559022520022428, "hi": 0.6599987950545193 }, "jsd": { "value": 0.13817419605804432, "lo": 0.12871165392582945, "hi": 0.1469010911344988 }, "brier": { "value": 0.28177827479458084, "lo": 0.2600940330569067, "hi": 0.30225653590456275 }, "acc": { "value": 0.6191512513601741, "lo": 0.5956881764551564, "hi": 0.6384247164223111 }, "conf": { "value": 0.6741650431245663, "lo": 0.6692797965312662, "hi": 0.679254919631878 }, "ece": { "value": 0.0637306943779023, "lo": 0.05506032962232484, "hi": 0.08412400401234231 }, "reliability": { "value": 0.00621485549369787, "lo": 0.004667391451410416, "hi": 0.010454610787192975 }, "resolution": { "value": 0.015622626577521078, "lo": 0.012215964784041453, "hi": 0.01979589089674316 }, "uncertainty": { "value": 0.2358029792993046, "lo": 0.23083859508637722, "hi": 0.24084376999801121 }, "auroc": { "value": 0.6282491212653779, "lo": 0.603501593465401, "hi": 0.6475211116319775 }, "acc@80": { "value": 0.6558993539612377, "lo": 0.6289360689362621, "hi": 0.6771410206860542 } } }, { "env": "yelp", "slice": "all", "system": "mix7-3B-s2", "metrics": { "n": { "value": 5000.0, "lo": 5000.0, "hi": 5000.0 }, "ce": { "value": 1.0011337405188199, "lo": 0.9753346307884541, "hi": 1.024778413367375 }, "kl": { "value": 1.0011337405188199, "lo": 0.9753346307884541, "hi": 1.024778413367375 }, "jsd": { "value": 0.2552398454410999, "lo": 0.24980107122494624, "hi": 0.26006344367899425 }, "brier": { "value": 0.5655031047936078, "lo": 0.5503120153958699, "hi": 0.5792040081646198 }, "acc": { "value": 0.5726, "lo": 0.55879, "hi": 0.58741 }, "conf": { "value": 0.649690758112073, "lo": 0.6439095934392512, "hi": 0.6554846212086826 }, "ece": { "value": 0.08159133845865724, "lo": 0.07506652643799783, "hi": 0.09863834241949022 }, "reliability": { "value": 0.011495647789278435, "lo": 0.009415492261957151, "hi": 0.01509313122028412 }, "resolution": { "value": 0.016553226261710813, "lo": 0.014624989329846983, "hi": 0.020412400144338892 }, "uncertainty": { "value": 0.24472924, "lo": 0.242359452, "hi": 0.246543696 }, "auroc": { "value": 0.6421032484716579, "lo": 0.6284965560499978, "hi": 0.6575292856478184 }, "acc@80": { "value": 0.60775, "lo": 0.59211875, "hi": 0.62563125 } } } ], "paired_gate_results": { "450M": [ { "tier": "B", "env": "mcq", "seed": 0, "ce_single": 0.7381567415998964, "ce": 0.747297009316037, "diff": { "value": 0.009140267716140581, "lo": -0.006196410089797683, "hi": 0.024727763838233493 }, "pass": true, "ce_base": 0.9293923372332724, "diff_base": { "value": -0.18209532791723532, "lo": -0.20456379334472086, "hi": -0.1581666467099493 }, "acc": 0.7156398104265402, "auroc": 0.8112850096578367, "ece": 0.06986760207721124, "conf": 0.7855074125037516, "flip": 0.16686413902053712, "brier": 0.38947030443786845 }, { "tier": "B", "env": "mcq", "seed": 1, "ce_single": 0.7256265333493006, "ce": 0.7268390369943714, "diff": { "value": 0.0012125036450708123, "lo": -0.016725134202668655, "hi": 0.019177147258237007 }, "pass": true, "ce_base": 0.9293923372332724, "diff_base": { "value": -0.20255330023890092, "lo": -0.22482715328826855, "hi": -0.17969118748438198 }, "acc": 0.7215639810426541, "auroc": 0.8156515544996836, "ece": 0.06707049818565308, "conf": 0.7886344792283071, "flip": 0.14533965244865718, "brier": 0.38048971460670034 }, { "tier": "B", "env": "mcq", "seed": 2, "ce_single": 0.7281025572024337, "ce": 0.7446275684259486, "diff": { "value": 0.016525011223514807, "lo": -0.000855911760031846, "hi": 0.034942436657513926 }, "pass": true, "ce_base": 0.9293923372332724, "diff_base": { "value": -0.18476476880732384, "lo": -0.20814323123273681, "hi": -0.1604615926601847 }, "acc": 0.7278830963665087, "auroc": 0.800425865098159, "ece": 0.06951306280084134, "conf": 0.79739615916735, "flip": 0.15916271721958924, "brier": 0.3832624003293176 }, { "tier": "B", "env": "chaosnli", "seed": 0, "ce_single": 0.9844640133367495, "ce": 0.8982318895558363, "diff": { "value": -0.08623212378091323, "lo": -0.10652471262333044, "hi": -0.06621382766939787 }, "pass": true, "ce_base": 0.9411396746035926, "diff_base": { "value": -0.042907785047756254, "lo": -0.05749265348496446, "hi": -0.028103012772523146 }, "acc": 0.5696706478465436, "auroc": 0.5786139997841288, "ece": 0.07636093434284788, "conf": 0.6321799852064542, "flip": 0.1237785016286645, "brier": 0.24486443443340125 }, { "tier": "B", "env": "chaosnli", "seed": 1, "ce_single": 0.8871500921895148, "ce": 0.9017071316185276, "diff": { "value": 0.014557039429012689, "lo": -0.0004969397159415857, "hi": 0.029173974675597524 }, "pass": true, "ce_base": 0.9411396746035926, "diff_base": { "value": -0.03943254298506512, "lo": -0.055159777065724384, "hi": -0.023642907355823452 }, "acc": 0.5754614549402823, "auroc": 0.5909461843255213, "ece": 0.07894042577598563, "conf": 0.6433491620774119, "flip": 0.1610568222946073, "brier": 0.24518685638804466 }, { "tier": "B", "env": "chaosnli", "seed": 2, "ce_single": 0.8954622802568517, "ce": 0.9068941962366361, "diff": { "value": 0.011431915979784435, "lo": -0.006660753236571492, "hi": 0.028242286674602532 }, "pass": true, "ce_base": 0.9411396746035926, "diff_base": { "value": -0.034245478366956496, "lo": -0.04885892896992744, "hi": -0.020397057125051746 }, "acc": 0.5798045602605864, "auroc": 0.5601164995091192, "ece": 0.08573853579112858, "conf": 0.642501125010381, "flip": 0.1986970684039088, "brier": 0.2526296104819719 }, { "tier": "B", "env": "cifar10", "seed": 0, "ce_single": 0.5567829186931413, "ce": 0.5611000733190353, "diff": { "value": 0.0043171546258939705, "lo": -0.02197660365861151, "hi": 0.03097862715820516 }, "pass": true, "ce_base": 0.8037275317717176, "diff_base": { "value": -0.24262745845268224, "lo": -0.30102144441524353, "hi": -0.18111650743956448 }, "acc": 0.8472521739130435, "auroc": 0.9243227687215418, "ece": 0.05521569651339365, "conf": 0.9024678704264372, "flip": 0.06149565217391304, "brier": 0.2198787598683563 }, { "tier": "B", "env": "cifar10", "seed": 1, "ce_single": 0.5469380003104549, "ce": 0.5091627016145139, "diff": { "value": -0.037775298695941054, "lo": -0.05927893644139918, "hi": -0.017854071484835844 }, "pass": true, "ce_base": 0.8037275317717176, "diff_base": { "value": -0.29456483015720364, "lo": -0.36004579887557103, "hi": -0.23134719299077566 }, "acc": 0.8509217391304348, "auroc": 0.9181642373285862, "ece": 0.04019794063930929, "conf": 0.8910135569981906, "flip": 0.08700869565217391, "brier": 0.21224998635895223 }, { "tier": "B", "env": "cifar10", "seed": 2, "ce_single": 0.5655811216366115, "ce": 0.58316263146589, "diff": { "value": 0.017581509829278405, "lo": -0.004978112560093911, "hi": 0.04073811228309107 }, "pass": true, "ce_base": 0.8037275317717176, "diff_base": { "value": -0.2205649003058277, "lo": -0.2781528600006001, "hi": -0.16262958098419275 }, "acc": 0.8423652173913043, "auroc": 0.9255523829062392, "ece": 0.0568483020759147, "conf": 0.8992134633206803, "flip": 0.05718260869565217, "brier": 0.22618299444310025 }, { "tier": "B", "env": "civilcomments", "seed": 0, "ce_single": 0.3190687141729324, "ce": 0.3362536541996374, "diff": { "value": 0.01718494002670492, "lo": 0.015083710602497, "hi": 0.01932160736486855 }, "pass": false, "ce_base": 0.5814737265683071, "diff_base": { "value": -0.2452200723686696, "lo": -0.25360402874176485, "hi": -0.23702421070301385 }, "acc": 0.9291182307910405, "auroc": 0.9130753060048147, "ece": 0.03717635887555369, "conf": 0.8974095615014813, "flip": null, "brier": 0.05199782543309763 }, { "tier": "B", "env": "civilcomments", "seed": 1, "ce_single": 0.3173171481792371, "ce": 0.32425900287305687, "diff": { "value": 0.006941854693819749, "lo": 0.005367884280422213, "hi": 0.008546991347228855 }, "pass": false, "ce_base": 0.5814737265683071, "diff_base": { "value": -0.2572147236952501, "lo": -0.2651383155764999, "hi": -0.24950532969871045 }, "acc": 0.9396087326339665, "auroc": 0.9061359484383446, "ece": 0.07329864480434391, "conf": 0.8663100878296226, "flip": null, "brier": 0.044416175920525686 }, { "tier": "B", "env": "civilcomments", "seed": 2, "ce_single": 0.3207698949153425, "ce": 0.3247535194409554, "diff": { "value": 0.00398362452561285, "lo": 0.0023974518068756983, "hi": 0.005536747774107673 }, "pass": false, "ce_base": 0.5814737265683071, "diff_base": { "value": -0.2567202071273516, "lo": -0.26446706818505655, "hi": -0.24923490914482513 }, "acc": 0.9398922597108024, "auroc": 0.8993091721449102, "ece": 0.08156628056539615, "conf": 0.8583259791454062, "flip": null, "brier": 0.044632186213394674 }, { "tier": "B", "env": "stsb", "seed": 0, "ce_single": 1.1361109137841074, "ce": 1.1119125656921782, "diff": { "value": -0.02419834809192916, "lo": -0.04609115024581878, "hi": 0.0016605323503595525 }, "pass": true, "ce_base": 1.7818855315091495, "diff_base": { "value": -0.669972965816971, "lo": -0.7339895191263983, "hi": -0.6063226809511448 }, "acc": 0.5525027203482046, "auroc": 0.6062123708655655, "ece": 0.04876845997228197, "conf": 0.5877989736487997, "flip": null, "brier": 0.3273993070991984 }, { "tier": "B", "env": "stsb", "seed": 1, "ce_single": 1.1748568926745255, "ce": 1.1069691649903395, "diff": { "value": -0.0678877276841858, "lo": -0.09197059477343385, "hi": -0.03933164131398174 }, "pass": true, "ce_base": 1.7818855315091495, "diff_base": { "value": -0.6749163665188097, "lo": -0.7423604856159355, "hi": -0.611100239422524 }, "acc": 0.5620239390642002, "auroc": 0.6223150024351675, "ece": 0.04551832948662383, "conf": 0.6037559004194459, "flip": null, "brier": 0.32481970890073175 }, { "tier": "B", "env": "stsb", "seed": 2, "ce_single": 1.226978893861858, "ce": 1.1153752379761714, "diff": { "value": -0.11160365588568648, "lo": -0.1390004700182866, "hi": -0.08355764749961739 }, "pass": true, "ce_base": 1.7818855315091495, "diff_base": { "value": -0.6665102935329779, "lo": -0.7348370252655096, "hi": -0.6021282107648279 }, "acc": 0.5424374319912949, "auroc": 0.6181128104885405, "ece": 0.040822435904119915, "conf": 0.5774308672505183, "flip": null, "brier": 0.33239855464142265 }, { "tier": "B", "env": "folktables", "seed": 0, "ce_single": 0.44187247322055384, "ce": 0.43932493729908706, "diff": { "value": -0.0025475359214667897, "lo": -0.00612057476814358, "hi": 0.0010668040060764063 }, "pass": true, "ce_base": 0.6934844593415295, "diff_base": { "value": -0.25415952204244235, "lo": -0.26311984755805745, "hi": -0.24539613398456228 }, "acc": 0.7843098047057255, "auroc": 0.7666138137673784, "ece": 0.016622621027984567, "conf": 0.7981700179879287, "flip": null, "brier": 0.28949881585732284 }, { "tier": "B", "env": "folktables", "seed": 1, "ce_single": 0.4481469455246447, "ce": 0.4356261342600311, "diff": { "value": -0.012520811264613562, "lo": -0.01634915127214725, "hi": -0.008893121878611332 }, "pass": true, "ce_base": 0.6934844593415295, "diff_base": { "value": -0.25785832508149825, "lo": -0.2670034281428999, "hi": -0.24913489819862952 }, "acc": 0.789375458241685, "auroc": 0.7633103515723115, "ece": 0.013219735506875569, "conf": 0.7976119624434539, "flip": null, "brier": 0.2863458999243068 }, { "tier": "B", "env": "folktables", "seed": 2, "ce_single": 0.44233523833101085, "ce": 0.4384948531105458, "diff": { "value": -0.0038403852204651023, "lo": -0.007681409785079287, "hi": 3.4361367331577135e-05 }, "pass": true, "ce_base": 0.6934844593415295, "diff_base": { "value": -0.2549896062309836, "lo": -0.26381183562964033, "hi": -0.24609928291102195 }, "acc": 0.7847097247217223, "auroc": 0.7679356316954442, "ece": 0.014699667606768206, "conf": 0.7962835168970399, "flip": null, "brier": 0.28842804801350497 }, { "tier": "B", "env": "camelyon17", "seed": 0, "ce_single": 0.20837835979747724, "ce": 0.19917673998548172, "diff": { "value": -0.009201619811995519, "lo": -0.030910606864758623, "hi": 0.01116204998378419 }, "pass": true, "ce_base": 0.6931978275120406, "diff_base": { "value": -0.4940210875265588, "lo": -0.5312363334452189, "hi": -0.4495808424622838 }, "acc": 0.9294615384615385, "auroc": 0.8643750951592453, "ece": 0.01728559489892081, "conf": 0.9467440238870107, "flip": null, "brier": 0.1093529491210613 }, { "tier": "B", "env": "camelyon17", "seed": 1, "ce_single": 0.18892128246007237, "ce": 0.2022852801095065, "diff": { "value": 0.01336399764943412, "lo": -0.03134999841335524, "hi": 0.052091479403583205 }, "pass": true, "ce_base": 0.6931978275120406, "diff_base": { "value": -0.490912547402534, "lo": -0.5306599014776814, "hi": -0.45281031031847896 }, "acc": 0.930076923076923, "auroc": 0.8874975786388497, "ece": 0.032629293684776, "conf": 0.9625870088293003, "flip": null, "brier": 0.11040756872368296 }, { "tier": "B", "env": "camelyon17", "seed": 2, "ce_single": 0.22308319161557216, "ce": 0.2580159262717116, "diff": { "value": 0.03493273465613941, "lo": 0.01103169983968305, "hi": 0.06797831187697921 }, "pass": false, "ce_base": 0.6931978275120406, "diff_base": { "value": -0.435181901240329, "lo": -0.49185439548196314, "hi": -0.3594040908925146 }, "acc": 0.9143846153846154, "auroc": 0.8487005253347428, "ece": 0.03692966717940113, "conf": 0.9513100121938265, "flip": null, "brier": 0.13598931412826326 } ], "1.6B": [ { "tier": "B", "env": "mcq", "seed": 0, "ce_single": 0.40774162897171706, "ce": 0.4002523221645569, "diff": { "value": -0.007489306807160107, "lo": -0.02144890226507013, "hi": 0.00687239297739766 }, "pass": true, "ce_base": 0.5269270169220264, "diff_base": { "value": -0.12667469475746948, "lo": -0.14512021534410527, "hi": -0.10755897094871683 }, "acc": 0.8554502369668247, "auroc": 0.8781354225007443, "ece": 0.0288912705417964, "conf": 0.8843396415677963, "flip": 0.08807266982622433, "brier": 0.20633200495477827 }, { "tier": "B", "env": "mcq", "seed": 1, "ce_single": 0.39632642582156424, "ce": 0.42684009817659985, "diff": { "value": 0.03051367235503558, "lo": 0.01535934910813931, "hi": 0.04480322164845855 }, "pass": false, "ce_base": 0.5269270169220264, "diff_base": { "value": -0.10008691874542655, "lo": -0.11721972238829916, "hi": -0.08171584887331773 }, "acc": 0.846958925750395, "auroc": 0.8766589700584391, "ece": 0.012092256853813047, "conf": 0.8572383006007703, "flip": 0.10624012638230648, "brier": 0.21781855156334679 }, { "tier": "B", "env": "mcq", "seed": 2, "ce_single": 0.4040138206383986, "ce": 0.41635328038970587, "diff": { "value": 0.012339459751307244, "lo": -0.003366449172156456, "hi": 0.027517208823090694 }, "pass": true, "ce_base": 0.5269270169220264, "diff_base": { "value": -0.11057373653232058, "lo": -0.13183961328275404, "hi": -0.090193696888094 }, "acc": 0.8578199052132701, "auroc": 0.8752545017393084, "ece": 0.0429864913625363, "conf": 0.9008063965758064, "flip": 0.08708530805687204, "brier": 0.2106166671020026 }, { "tier": "B", "env": "chaosnli", "seed": 0, "ce_single": 0.7432735836870603, "ce": 0.7654613715471141, "diff": { "value": 0.022187787860053886, "lo": 0.014892751113375909, "hi": 0.029058804683116835 }, "pass": false, "ce_base": 0.8347258254601914, "diff_base": { "value": -0.06926445391307712, "lo": -0.07996195673824386, "hi": -0.058263186156225916 }, "acc": 0.7082880926529135, "auroc": 0.6902250748410934, "ece": 0.030578552813707745, "conf": 0.6869863378065374, "flip": 0.12015924719507781, "brier": 0.1552037163302883 }, { "tier": "B", "env": "chaosnli", "seed": 1, "ce_single": 0.7425795222297821, "ce": 0.7837159885599414, "diff": { "value": 0.04113646633015925, "lo": 0.03226748686400686, "hi": 0.04967784705488425 }, "pass": false, "ce_base": 0.8347258254601914, "diff_base": { "value": -0.0510098369002501, "lo": -0.06193706511579653, "hi": -0.0398299530236231 }, "acc": 0.6800579080709374, "auroc": 0.6697049311393612, "ece": 0.02181350362935101, "conf": 0.6740838724376929, "flip": 0.31921824104234525, "brier": 0.17074606097407263 }, { "tier": "B", "env": "chaosnli", "seed": 2, "ce_single": 0.7426378492384095, "ce": 0.7585695789984018, "diff": { "value": 0.015931729759992495, "lo": 0.00928400184774719, "hi": 0.023151134788389538 }, "pass": false, "ce_base": 0.8347258254601914, "diff_base": { "value": -0.07615624646178952, "lo": -0.08783875216411097, "hi": -0.06443452143714924 }, "acc": 0.7278320665942816, "auroc": 0.6720206417893078, "ece": 0.056012450127855294, "conf": 0.6846000137663043, "flip": 0.09554831704668838, "brier": 0.14927606706706414 }, { "tier": "B", "env": "cifar10", "seed": 0, "ce_single": 0.2299739719242439, "ce": 0.2519872646207342, "diff": { "value": 0.022013292696490255, "lo": 0.0030736171336475703, "hi": 0.03992000338242685 }, "pass": false, "ce_base": 0.525821451871081, "diff_base": { "value": -0.27383418725034675, "lo": -0.3237711272866988, "hi": -0.22284108838891117 }, "acc": 0.9315304347826087, "auroc": 0.9566540211436751, "ece": 0.028470759793986423, "conf": 0.9599102325045544, "flip": 0.027443478260869566, "brier": 0.10188192213687446 }, { "tier": "B", "env": "cifar10", "seed": 1, "ce_single": 0.2501114400700646, "ce": 0.2533726154562614, "diff": { "value": 0.0032611753861967957, "lo": -0.011275453788412921, "hi": 0.016673796957155318 }, "pass": true, "ce_base": 0.525821451871081, "diff_base": { "value": -0.2724488364148195, "lo": -0.32245688367507885, "hi": -0.22221876717115202 }, "acc": 0.9217565217391305, "auroc": 0.9468035615074913, "ece": 0.006458782019822494, "conf": 0.9263151420883511, "flip": 0.04678260869565217, "brier": 0.11110962959453999 }, { "tier": "B", "env": "cifar10", "seed": 2, "ce_single": 0.2321976805761593, "ce": 0.334982548927015, "diff": { "value": 0.10278486835085567, "lo": 0.0748897647062431, "hi": 0.1358331625647502 }, "pass": false, "ce_base": 0.525821451871081, "diff_base": { "value": -0.1908389029440659, "lo": -0.23785761084554316, "hi": -0.1401684523329021 }, "acc": 0.9176521739130434, "auroc": 0.936990951591359, "ece": 0.04628719613396603, "conf": 0.9639267829972765, "flip": 0.03248695652173913, "brier": 0.1294624052588854 }, { "tier": "B", "env": "civilcomments", "seed": 0, "ce_single": 0.31429617484808725, "ce": 0.3291432395939985, "diff": { "value": 0.014847064745911189, "lo": 0.013104040172484954, "hi": 0.01664621848809903 }, "pass": false, "ce_base": 0.6286493075695474, "diff_base": { "value": -0.299506067975549, "lo": -0.3074177196746435, "hi": -0.2915985168841146 }, "acc": 0.9314809564313392, "auroc": 0.9217205693013882, "ece": 0.04325364368555168, "conf": 0.8949943995861078, "flip": null, "brier": 0.04824225276040297 }, { "tier": "B", "env": "civilcomments", "seed": 1, "ce_single": 0.3138288534673895, "ce": 0.3231331506493501, "diff": { "value": 0.009304297181960621, "lo": 0.007857961523336704, "hi": 0.010786769292632555 }, "pass": false, "ce_base": 0.6286493075695474, "diff_base": { "value": -0.30551615692019735, "lo": -0.31285911893489743, "hi": -0.2983232581327826 }, "acc": 0.93346564596919, "auroc": 0.9217670673143299, "ece": 0.0636185640232424, "conf": 0.8726835281555306, "flip": null, "brier": 0.04397100215369724 }, { "tier": "B", "env": "civilcomments", "seed": 2, "ce_single": 0.3146981139398467, "ce": 0.3163261854007762, "diff": { "value": 0.00162807146092949, "lo": 0.0005988585567799004, "hi": 0.002661235833755524 }, "pass": false, "ce_base": 0.6286493075695474, "diff_base": { "value": -0.31232312216877123, "lo": -0.31959109428057375, "hi": -0.3052725327859585 }, "acc": 0.9425385124279368, "auroc": 0.9198111216072871, "ece": 0.07780369303648059, "conf": 0.8647348193914562, "flip": null, "brier": 0.039377961884232315 }, { "tier": "B", "env": "stsb", "seed": 0, "ce_single": 1.055144585762775, "ce": 1.031748552907778, "diff": { "value": -0.023396032854997285, "lo": -0.04090748294174989, "hi": -0.0036253573610600352 }, "pass": true, "ce_base": 1.749101240806852, "diff_base": { "value": -0.7173526878990744, "lo": -0.8007907469966399, "hi": -0.6449286724678583 }, "acc": 0.6131664853101197, "auroc": 0.6128823020677726, "ece": 0.052070733139512326, "conf": 0.6504272062481422, "flip": null, "brier": 0.290818102811832 }, { "tier": "B", "env": "stsb", "seed": 1, "ce_single": 1.0539440524395638, "ce": 1.0325978650062761, "diff": { "value": -0.021346187433287877, "lo": -0.05153571897527513, "hi": 0.008768368946593146 }, "pass": true, "ce_base": 1.749101240806852, "diff_base": { "value": -0.716503375800576, "lo": -0.7897844611529958, "hi": -0.6517002411893676 }, "acc": 0.5715451577801959, "auroc": 0.621027477467268, "ece": 0.04524248022022652, "conf": 0.6035776622552866, "flip": null, "brier": 0.30203148903766214 }, { "tier": "B", "env": "stsb", "seed": 2, "ce_single": 1.0425603202621663, "ce": 1.1350048984988437, "diff": { "value": 0.09244457823667729, "lo": 0.05544914470844475, "hi": 0.13377877956106016 }, "pass": false, "ce_base": 1.749101240806852, "diff_base": { "value": -0.6140963423080084, "lo": -0.7119540303207893, "hi": -0.525878957121933 }, "acc": 0.5998367791077258, "auroc": 0.6290156941997284, "ece": 0.12202087906518622, "conf": 0.721857658172912, "flip": null, "brier": 0.3191667956646663 }, { "tier": "B", "env": "folktables", "seed": 0, "ce_single": 0.43414523920300574, "ce": 0.4429014798100703, "diff": { "value": 0.008756240607064609, "lo": 0.005746677057294381, "hi": 0.011726213381559743 }, "pass": false, "ce_base": 0.6121096088424054, "diff_base": { "value": -0.1692081290323351, "lo": -0.1793469097741068, "hi": -0.15912578607336217 }, "acc": 0.7856428714257149, "auroc": 0.760233718887534, "ece": 0.025429919649106476, "conf": 0.8061644708856792, "flip": null, "brier": 0.29192483263490315 }, { "tier": "B", "env": "folktables", "seed": 1, "ce_single": 0.4378669224129474, "ce": 0.44795320351265, "diff": { "value": 0.010086281099702563, "lo": 0.0053899754819800244, "hi": 0.014885390200504426 }, "pass": false, "ce_base": 0.6121096088424054, "diff_base": { "value": -0.16415640532975542, "lo": -0.17344494183186426, "hi": -0.15514808425113147 }, "acc": 0.7784443111377725, "auroc": 0.760897827041841, "ece": 0.013490881572951727, "conf": 0.7826558687977256, "flip": null, "brier": 0.29621629260680155 }, { "tier": "B", "env": "folktables", "seed": 2, "ce_single": 0.43541280816255606, "ce": 0.4356590369299968, "diff": { "value": 0.0002462287674407353, "lo": -0.002834348848603956, "hi": 0.0033126552922933578 }, "pass": true, "ce_base": 0.6121096088424054, "diff_base": { "value": -0.1764505719124086, "lo": -0.1862956406214327, "hi": -0.16675564239316432 }, "acc": 0.7911084449776711, "auroc": 0.7657509738706254, "ece": 0.01979037306272671, "conf": 0.8088698498306295, "flip": null, "brier": 0.28450758265424486 }, { "tier": "B", "env": "camelyon17", "seed": 0, "ce_single": 0.14715429696058982, "ce": 0.16948480361084028, "diff": { "value": 0.022330506650250492, "lo": 0.005246748162873034, "hi": 0.0401075197620554 }, "pass": false, "ce_base": 0.6492519817213229, "diff_base": { "value": -0.4797671781104826, "lo": -0.5181835922538937, "hi": -0.4500954663723073 }, "acc": 0.9452307692307692, "auroc": 0.8774072311344218, "ece": 0.024357909413484414, "conf": 0.9695761512517929, "flip": null, "brier": 0.08766813727769751 }, { "tier": "B", "env": "camelyon17", "seed": 1, "ce_single": 0.15899906997961716, "ce": 0.15577225450669627, "diff": { "value": -0.0032268154729209056, "lo": -0.02846167486233926, "hi": 0.018547685804321406 }, "pass": true, "ce_base": 0.6492519817213229, "diff_base": { "value": -0.4934797272146266, "lo": -0.5387851743221831, "hi": -0.4622773848595618 }, "acc": 0.9384615384615385, "auroc": 0.907457530737705, "ece": 0.013398746646367605, "conf": 0.9395801230302224, "flip": null, "brier": 0.0900247205040645 }, { "tier": "B", "env": "camelyon17", "seed": 2, "ce_single": 0.1814494749853387, "ce": 0.16058040959806322, "diff": { "value": -0.020869065387275485, "lo": -0.033328658363837894, "hi": -0.009703955758107462 }, "pass": true, "ce_base": 0.6492519817213229, "diff_base": { "value": -0.4886715721232597, "lo": -0.5259585444754835, "hi": -0.45916087180637793 }, "acc": 0.941923076923077, "auroc": 0.8967018839964414, "ece": 0.020103888387863443, "conf": 0.9619018222047733, "flip": null, "brier": 0.09038797778880224 } ], "3B": [ { "tier": "B", "env": "mcq", "seed": 0, "ce": 0.28394581130659097, "ce_base": 0.38602727384245317, "diff_base": { "value": -0.10208146253586213, "lo": -0.11673248732151378, "hi": -0.08761654927776354 }, "acc": 0.8992890995260664, "auroc": 0.9004529523711109, "ece": 0.01118656555058266, "conf": 0.9096089693560231, "brier": 0.14707832873591406 }, { "tier": "B", "env": "mcq", "seed": 1, "ce": 0.2891957645267851, "ce_base": 0.38602727384245317, "diff_base": { "value": -0.09683150931566804, "lo": -0.11046495538913444, "hi": -0.08322018033501698 }, "acc": 0.8937598736176935, "auroc": 0.9045484002385228, "ece": 0.00769100147542233, "conf": 0.8991462239086345, "brier": 0.1517139060824241 }, { "tier": "B", "env": "mcq", "seed": 2, "ce": 0.2861016989829702, "ce_base": 0.38602727384245317, "diff_base": { "value": -0.09992557485948292, "lo": -0.11570205023000027, "hi": -0.08457066767730967 }, "acc": 0.8996840442338072, "auroc": 0.9013879335375001, "ece": 0.02074630266588068, "conf": 0.9201619169346718, "brier": 0.1476629415774208 }, { "tier": "B", "env": "chaosnli", "seed": 0, "ce": 0.7317921943962872, "ce_base": 0.7949680342778785, "diff_base": { "value": -0.06317583988159142, "lo": -0.07211168590286045, "hi": -0.05446602001900818 }, "acc": 0.7321751719145856, "auroc": 0.7111210939065611, "ece": 0.048137297006435174, "conf": 0.6842451099248369, "brier": 0.13177767445592195 }, { "tier": "B", "env": "chaosnli", "seed": 1, "ce": 0.7349974377531698, "ce_base": 0.7949680342778785, "diff_base": { "value": -0.059970596524708736, "lo": -0.06850646719340128, "hi": -0.05143582608454075 }, "acc": 0.7285559174809989, "auroc": 0.7173164431197218, "ece": 0.04855649530822884, "conf": 0.687175090378538, "brier": 0.13334485416438893 }, { "tier": "B", "env": "chaosnli", "seed": 2, "ce": 0.7314480272773598, "ce_base": 0.7949680342778785, "diff_base": { "value": -0.06352000700051882, "lo": -0.0721910035084413, "hi": -0.054702471241274524 }, "acc": 0.7350705754614549, "auroc": 0.698795715588703, "ece": 0.054777696259505855, "conf": 0.6815311866871395, "brier": 0.13283454092153876 }, { "tier": "B", "env": "cifar10", "seed": 0, "ce": 0.1926190952540813, "ce_base": 0.2611563624941145, "diff_base": { "value": -0.06853726724003315, "lo": -0.09402452041400289, "hi": -0.04432714693000657 }, "acc": 0.9390434782608695, "auroc": 0.9529923188587097, "ece": 0.016654349529743186, "conf": 0.9556232028479161, "brier": 0.08987807640548674 }, { "tier": "B", "env": "cifar10", "seed": 1, "ce": 0.19700957925621565, "ce_base": 0.2611563624941145, "diff_base": { "value": -0.06414678323789884, "lo": -0.08776735215450746, "hi": -0.041930136281348514 }, "acc": 0.9375652173913044, "auroc": 0.9508026273025971, "ece": 0.01768210469406584, "conf": 0.9533248674587064, "brier": 0.0916005966374745 }, { "tier": "B", "env": "cifar10", "seed": 2, "ce": 0.24005205260956894, "ce_base": 0.2611563624941145, "diff_base": { "value": -0.02110430988454551, "lo": -0.04498645729999961, "hi": 0.0030856171380235138 }, "acc": 0.9355478260869565, "auroc": 0.9478078197050045, "ece": 0.029083070768221565, "conf": 0.9646026096185912, "brier": 0.09852223326178403 }, { "tier": "B", "env": "civilcomments", "seed": 0, "ce": 0.32493529455042947, "ce_base": 0.422683145843368, "diff_base": { "value": -0.09774785129293849, "lo": -0.10272723555639415, "hi": -0.09211969004446136 }, "acc": 0.9376240430961157, "auroc": 0.9210726863433244, "ece": 0.045806885644400594, "conf": 0.8938100220361445, "brier": 0.044440062724136684 }, { "tier": "B", "env": "civilcomments", "seed": 1, "ce": 0.31942936538814876, "ce_base": 0.422683145843368, "diff_base": { "value": -0.10325378045521923, "lo": -0.10829973730458461, "hi": -0.09770082750583385 }, "acc": 0.9445232019657878, "auroc": 0.913231925526696, "ece": 0.06654398142305668, "conf": 0.877979220542731, "brier": 0.040234408557455296 }, { "tier": "B", "env": "civilcomments", "seed": 2, "ce": 0.3162963367925811, "ce_base": 0.422683145843368, "diff_base": { "value": -0.1063868090507868, "lo": -0.1111116179990059, "hi": -0.1012086043872098 }, "acc": 0.9453737831962953, "auroc": 0.9157116186943225, "ece": 0.08202986666074875, "conf": 0.8633439165355465, "brier": 0.0392586704145921 }, { "tier": "B", "env": "stsb", "seed": 0, "ce": 1.0320934313543655, "ce_base": 1.5758103744712262, "diff_base": { "value": -0.5437169431168607, "lo": -0.6207747093223118, "hi": -0.4716511781174488 }, "acc": 0.6235038084874864, "auroc": 0.6192176040815504, "ece": 0.06937201697302331, "conf": 0.6865655293564542, "brier": 0.2879302523060823 }, { "tier": "B", "env": "stsb", "seed": 1, "ce": 0.9606397550106914, "ce_base": 1.5758103744712262, "diff_base": { "value": -0.6151706194605348, "lo": -0.6804369290481779, "hi": -0.5581021087930027 }, "acc": 0.6109902067464635, "auroc": 0.6517820336386676, "ece": 0.03286231362683471, "conf": 0.6308347471342123, "brier": 0.2696511629637947 }, { "tier": "B", "env": "stsb", "seed": 2, "ce": 1.0124180703390282, "ce_base": 1.5758103744712262, "diff_base": { "value": -0.563392304132198, "lo": -0.6365432112290809, "hi": -0.49533936739389856 }, "acc": 0.6191512513601741, "auroc": 0.6282491212653779, "ece": 0.0637306943779023, "conf": 0.6741650431245663, "brier": 0.28177827479458084 }, { "tier": "B", "env": "folktables", "seed": 0, "ce": 0.44509115726019133, "ce_base": 0.609676541729926, "diff_base": { "value": -0.16458538446973475, "lo": -0.17439749195793422, "hi": -0.15483688868750595 }, "acc": 0.7876424715056989, "auroc": 0.7532512981367189, "ece": 0.02779868638635452, "conf": 0.8070413636770326, "brier": 0.2930003292994701 }, { "tier": "B", "env": "folktables", "seed": 1, "ce": 0.44111304791622236, "ce_base": 0.609676541729926, "diff_base": { "value": -0.16856349381370364, "lo": -0.17746318378000042, "hi": -0.16005416847899803 }, "acc": 0.7869759381457042, "auroc": 0.7613656299221321, "ece": 0.014096905742098072, "conf": 0.7849255870844709, "brier": 0.289130483213222 }, { "tier": "B", "env": "folktables", "seed": 2, "ce": 0.45390238176418163, "ce_base": 0.609676541729926, "diff_base": { "value": -0.15577415996574437, "lo": -0.16661552004733296, "hi": -0.1447391492070872 }, "acc": 0.7881090448576952, "auroc": 0.7578438227483217, "ece": 0.03934656827201734, "conf": 0.8258664168088204, "brier": 0.2935044644711353 }, { "tier": "B", "env": "camelyon17", "seed": 0, "ce": 0.1406058934525335, "ce_base": 0.6861919247772764, "diff_base": { "value": -0.5455860313247429, "lo": -0.6275696224676554, "hi": -0.4919595356019622 }, "acc": 0.9468461538461539, "auroc": 0.9067836424796653, "ece": 0.011683198690414417, "conf": 0.9584915467592386, "brier": 0.07937519631993027 }, { "tier": "B", "env": "camelyon17", "seed": 1, "ce": 0.19288188233974013, "ce_base": 0.6861919247772764, "diff_base": { "value": -0.4933100424375363, "lo": -0.5776901705796982, "hi": -0.43618574916489894 }, "acc": 0.942076923076923, "auroc": 0.8713108698544598, "ece": 0.02991748259159234, "conf": 0.9719926106517132, "brier": 0.09324804513698431 }, { "tier": "B", "env": "camelyon17", "seed": 2, "ce": 0.1634219955634534, "ce_base": 0.6861919247772764, "diff_base": { "value": -0.522769929213823, "lo": -0.5845630301942749, "hi": -0.48434549592554427 }, "acc": 0.938076923076923, "auroc": 0.8971406670588445, "ece": 0.012466411677690642, "conf": 0.9502902042086308, "brier": 0.09485444071675608 } ] }, "missing_values": "Non-finite values in the source reports (e.g. unmeasured flip rates) are represented as null." }