{ "schemaVersion": "1.3.0", "implementationVersion": "vm-01-vm-07.5", "manifest": { "generatedAt": "2026-07-10T13:15:59.487Z", "dataSha256": "1aa1306720b94850ca696befd7532b1408ff2e90734dd9def30b635c012acc14", "configSha256": "c98bf32b10dc106be2cc141ba032a727f5c131dc2575dcba9e44676dc0d9de80", "sourceDescriptor": "data/elo-backtest-db-snapshot.json (maps-with-last-observed-roster-v4)", "sourceSnapshotSha256": "1aa1306720b94850ca696befd7532b1408ff2e90734dd9def30b635c012acc14", "sourceSnapshotFileSha256": "1003de9038d4fd024d442b14c508c307b445278e8b3bb8f1941595af74316bd3", "sourceSnapshotExportedAt": "2026-07-09T21:25:41.060Z", "evaluatedModelVersion": "map-elo-v1.0.0", "sourceCodeSha256": "57f19dc87cfcb41adfb7626c4cfe7d80bbb3742546b1642e65f16784db59b5c8", "sourceFiles": [ "lib/elo/backtest.ts", "lib/elo/elo-calculator.ts", "scripts/run-elo-backtest.ts" ], "gitCommit": "dfd4a7a0261a4505846d581a97adbc4d19a931e2", "gitWorkingTree": "dirty", "provenanceCapturedAt": "2026-07-10T13:15:59.487Z", "provenanceCapturePoint": "before input reads and artifact writes", "sourceExtraction": { "scope": "maps where processed = true; required winner and loser team joins; completed_at and score validity are checked during normalization", "totalMaps": 3818, "processedMaps": 3818, "unprocessedMaps": 0, "mapsWithoutCompletedAt": 0, "processedMapsWithoutCompletedAt": 0, "extractedProcessedRows": 3818, "processedRowsExcludedByRequiredTeamJoins": 0 }, "source": "snapshot", "acceptedRows": 3818, "earliestCompletedAt": "2023-02-13T12:10:00.000Z", "latestCompletedAt": "2026-06-21T10:20:00.000Z" }, "dataQuality": { "receivedRows": 3818, "acceptedRows": 3818, "rejectedRows": 0, "issueCounts": { "missing_required": 0, "invalid_value": 0, "duplicate_source_id": 0, "duplicate_map": 0, "source_sequence_gap": 0 }, "issues": [], "schemaMissingColumns": [], "missingRequiredRows": 0, "duplicateRows": 0, "sourceSequenceGapCount": 0, "sourceSequenceGaps": [], "seriesIdCoverage": { "rows": 3818, "rate": 1, "status": "complete" }, "rosterCoverage": { "rowsWithBothRosters": 3818, "rate": 1, "minimumPlayersPerTeam": 3 } }, "split": { "boundaries": { "trainEndExclusive": "2025-01-01T00:00:00.000Z", "validationEndExclusive": "2025-09-01T00:00:00.000Z", "testEndExclusive": null }, "boundarySemantics": "[start, end) in UTC", "counts": { "train": 1799, "validation": 1184, "test": 835 }, "seriesCounts": { "train": 712, "validation": 464, "test": 319 }, "observedRanges": { "train": { "firstCompletedAt": "2023-02-13T12:10:00.000Z", "lastCompletedAt": "2024-08-25T03:00:00.000Z" }, "validation": { "firstCompletedAt": "2025-01-11T04:00:00.000Z", "lastCompletedAt": "2025-08-31T17:00:00.000Z" }, "test": { "firstCompletedAt": "2025-09-12T09:00:00.000Z", "lastCompletedAt": "2026-06-21T10:20:00.000Z" } }, "excludedAtOrAfterTestEnd": 0 }, "methodology": { "unit": "map", "timezone": "UTC", "ordering": "completed_at, then source series/match id, game_number, source row id; all tie breaks are deterministic", "orientation": "team A is the lexical minimum normalized team id; the winner field sets only the binary outcome after orientation", "predictionTiming": "all maps sharing completed_at are predicted from state strictly before that timestamp, then outcomes update in deterministic series/game order", "splitPolicy": "within each run, training only warms state; validation alone selects one experimental challenger from the candidate grid; fixed references are not selection candidates; test is not evaluated until selection is frozen", "selectionMetric": "validation log loss", "holdoutPolicy": "within-run fixed plain Elo, pre-holdout frozen production, analysis-time production, 0.5 reference, and the one validation-selected experimental challenger are evaluated after selection; candidate tables contain validation metrics only; no persistent ledger prevents later reruns", "bootstrap": { "method": "deterministic block bootstrap", "blockDefinition": "source series/match id", "iterations": 500, "seed": 20250308 }, "rosterPolicy": "player appearances become observable only after every prediction at their map timestamp; regression is labeled a last-observed roster proxy and applies only at strictly later timestamps", "metricDefinitions": { "brier": "mean squared error of team-A win probability; lower is better", "logLoss": "mean negative Bernoulli log likelihood with probabilities clipped to [1e-15, 1-1e-15]; lower is better", "accuracy": "share correct using probabilityTeamA >= 0.5 as the deterministic team-A decision; higher is better", "ece": "10-bin expected calibration error weighted by bin sample size; lower is better" } }, "evaluationContext": { "studyDesign": "retrospective chronological replay", "prospectivelyRegistered": false, "candidateGridStatus": "The candidate grid was implemented at analysis time and was not prospectively preregistered before the test period.", "productionConfiguration": { "status": "deployed at analysis time", "knownRepositoryChangeAt": "2025-10-04T02:22:24.000Z", "knownRepositoryChangeCommit": "99a0a87a240f47d1aa0c894ffb730fe9ea4ddd23", "changePredatesTestStart": false, "note": "The repository changed ratingScale from 2000 to 1000 at the recorded commit. The evaluated configuration is an analysis-time retrospective comparator, not a configuration asserted to predate the test period." }, "preHoldoutFrozenProductionConfiguration": { "status": "repository snapshot predating chronological test start", "repositoryCommit": "2f134f187c0717dbfcf18b1a07c4eccbef94dcdb", "repositoryCommitAt": "2025-08-26T15:49:51-05:00", "predatesTestStart": true, "sourceFiles": [ "lib/elo/elo-calculator.ts", "lib/predictions/calculations.ts", "db/elo/elo-processor.ts" ], "sourceBlobs": [ { "path": "lib/elo/elo-calculator.ts", "gitBlob": "cdf7c5361a88a1099049058181e986fde4cc8f29" }, { "path": "lib/predictions/calculations.ts", "gitBlob": "11343604323dcf6ea414d2215e75be7327d9e533" }, { "path": "db/elo/elo-processor.ts", "gitBlob": "0c3d10548e2ef966dd831dcafaa9636378562d14" } ], "reconstruction": "Map rating updates use initial=1000, ratingScale=2000, K=74, production MOV scale=1, and hard annual reset; forecast probabilities separately use predictionScale=1000.", "interpretation": "This repository-frozen comparator is retrospectively out of sample after the chronological test start. It does not prove the original parameter choice was leakage-free and does not restore an untouched holdout." } }, "references": { "plainElo": { "id": "plain-overall-elo", "label": "Plain overall-team Elo", "family": "plain-elo", "ratingScope": "overall", "initialRating": 1000, "ratingScale": 400, "predictionScale": 400, "kFactor": 32, "marginFormula": "none", "marginScale": 0, "seasonCarry": null, "coldStartPriorGames": 0, "rosterRegressionStrength": 0 }, "preHoldoutFrozenProductionMapElo": { "id": "production-map-elo-pre-holdout-frozen", "label": "Pre-holdout frozen production map Elo", "family": "production-map-elo", "ratingScope": "map", "initialRating": 1000, "ratingScale": 2000, "predictionScale": 1000, "kFactor": 74, "marginFormula": "production", "marginScale": 1, "seasonCarry": 0, "coldStartPriorGames": 0, "rosterRegressionStrength": 0 }, "productionMapElo": { "id": "production-map-elo-fixed", "label": "Production map Elo at analysis time", "family": "production-map-elo", "ratingScope": "map", "initialRating": 1000, "ratingScale": 1000, "predictionScale": 1000, "kFactor": 74, "marginFormula": "production", "marginScale": 1, "seasonCarry": 0, "coldStartPriorGames": 0, "rosterRegressionStrength": 0 }, "naiveProbability": 0.5 }, "evaluatedModelExamples": { "modelId": "production-map-elo-fixed", "equalRating": 1000, "updates": [ { "winnerScore": 13, "loserScore": 11, "winnerChange": 86.30980246398896, "loserChange": -86.30980246398896 }, { "winnerScore": 13, "loserScore": 0, "winnerChange": 114.8080357215112, "loserChange": -114.8080357215112 } ] }, "experiments": [ { "family": "season-carry", "status": "available", "reason": "At least one season boundary exists before the holdout.", "evidence": { "seasonsBeforeHoldout": 3, "seasonYears": "2023,2024,2025" }, "candidates": [ { "model": { "id": "season-carry-25", "label": "Production map Elo with 25% season carry", "family": "season-carry", "ratingScope": "map", "initialRating": 1000, "ratingScale": 1000, "predictionScale": 1000, "kFactor": 74, "marginFormula": "production", "marginScale": 1, "seasonCarry": 0.25, "coldStartPriorGames": 0, "rosterRegressionStrength": 0 }, "validation": { "metrics": { "n": 1184, "brier": 0.2521969334325719, "logLoss": 0.6985253032923229, "accuracy": 0.5396959459459459, "ece": 0.04851343923712215, "reliabilityBins": [ { "lower": 0, "upper": 0.1, "upperInclusive": false, "n": 1, "meanPredictedProbability": 0.06371515392848515, "observedTeamAWinRate": 0, "observedRateWilson95": { "lower95": 0, "upper95": 0.7934506856227626 } }, { "lower": 0.1, "upper": 0.2, "upperInclusive": false, "n": 8, "meanPredictedProbability": 0.17956523393537505, "observedTeamAWinRate": 0.375, "observedRateWilson95": { "lower95": 0.13684428582359737, "upper95": 0.6942576053973728 } }, { "lower": 0.2, "upper": 0.3, "upperInclusive": false, "n": 38, "meanPredictedProbability": 0.25696958540935944, "observedTeamAWinRate": 0.4473684210526316, "observedRateWilson95": { "lower95": 0.3014640099805125, "upper95": 0.6029370287800753 } }, { "lower": 0.3, "upper": 0.4, "upperInclusive": false, "n": 140, "meanPredictedProbability": 0.35807692316335926, "observedTeamAWinRate": 0.4928571428571429, "observedRateWilson95": { "lower95": 0.411345857883062, "upper95": 0.5747499449793583 } }, { "lower": 0.4, "upper": 0.5, "upperInclusive": false, "n": 350, "meanPredictedProbability": 0.4576506410709341, "observedTeamAWinRate": 0.5, "observedRateWilson95": { "lower95": 0.44790287743932533, "upper95": 0.5520971225606747 } }, { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 432, "meanPredictedProbability": 0.5416794940442617, "observedTeamAWinRate": 0.5439814814814815, "observedRateWilson95": { "lower95": 0.4968330216981095, "upper95": 0.5903546453559173 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 148, "meanPredictedProbability": 0.6384531007569392, "observedTeamAWinRate": 0.6013513513513513, "observedRateWilson95": { "lower95": 0.5208675133747347, "upper95": 0.6767069846115252 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 60, "meanPredictedProbability": 0.7373499937408872, "observedTeamAWinRate": 0.6, "observedRateWilson95": { "lower95": 0.4736605349204075, "upper95": 0.7143050946511744 } }, { "lower": 0.8, "upper": 0.9, "upperInclusive": false, "n": 7, "meanPredictedProbability": 0.8343514959713063, "observedTeamAWinRate": 0.8571428571428571, "observedRateWilson95": { "lower95": 0.4868721707256811, "upper95": 0.9743203756552565 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null } ], "accuracyByConfidenceBand": [ { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 782, "meanConfidence": 0.5419793056934704, "accuracy": 0.5242966751918159, "accuracyWilson95": { "lower95": 0.4892606751124145, "upper95": 0.5590951345403701 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 288, "meanConfidence": 0.640139894684572, "accuracy": 0.5555555555555556, "accuracyWilson95": { "lower95": 0.49781011672903974, "upper95": 0.6118384579776152 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 98, "meanConfidence": 0.7395526059071179, "accuracy": 0.5816326530612245, "accuracyWilson95": { "lower95": 0.48269880400178744, "upper95": 0.6744081363865095 } }, { "lower": 0.8, "upper": 0.8999999999999999, "upperInclusive": false, "n": 15, "meanConfidence": 0.8269292400210764, "accuracy": 0.7333333333333333, "accuracyWilson95": { "lower95": 0.4804956594401943, "upper95": 0.8910254667430761 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 1, "meanConfidence": 0.9362848460715149, "accuracy": 1, "accuracyWilson95": { "lower95": 0.20654931437723745, "upper95": 1 } } ], "sampleSizeByMap": [ { "mapName": "Abyss", "n": 44 }, { "mapName": "Ascent", "n": 114 }, { "mapName": "Bind", "n": 97 }, { "mapName": "Corrode", "n": 57 }, { "mapName": "Fracture", "n": 88 }, { "mapName": "Haven", "n": 175 }, { "mapName": "Icebox", "n": 118 }, { "mapName": "Lotus", "n": 217 }, { "mapName": "Pearl", "n": 97 }, { "mapName": "Split", "n": 104 }, { "mapName": "Sunset", "n": 73 } ] }, "intervals95": { "brier": { "estimate": 0.2521969334325719, "lower95": 0.24632984988810608, "upper95": 0.2579172849099358, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2719143829 }, "logLoss": { "estimate": 0.6985253032923229, "lower95": 0.6862274946028525, "upper95": 0.7111341939445983, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2719143829 }, "accuracy": { "estimate": 0.5396959459459459, "lower95": 0.5112109952018135, "upper95": 0.5690770229052041, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2719143829 }, "ece": { "estimate": 0.04851343923712215, "lower95": 0.03581168309709093, "upper95": 0.08053068249646567, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2719143829 } }, "pairedDeltaVsPlainElo": { "brier": { "estimate": 0.008527135708996603, "lower95": -0.000035027910856625376, "upper95": 0.01620268057013134, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 325204824 }, "logLoss": { "estimate": 0.01698691405404107, "lower95": -0.002007102111612219, "upper95": 0.033748683578489036, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 325204824 }, "accuracy": { "estimate": -0.029560810810810856, "lower95": -0.06280386280386277, "upper95": 0.005899830371463768, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 325204824 }, "ece": { "estimate": -0.017010729660937682, "lower95": -0.04346776885736107, "upper95": 0.012931157264497601, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 325204824 } } } }, { "model": { "id": "season-carry-50", "label": "Production map Elo with 50% season carry", "family": "season-carry", "ratingScope": "map", "initialRating": 1000, "ratingScale": 1000, "predictionScale": 1000, "kFactor": 74, "marginFormula": "production", "marginScale": 1, "seasonCarry": 0.5, "coldStartPriorGames": 0, "rosterRegressionStrength": 0 }, "validation": { "metrics": { "n": 1184, "brier": 0.25293441690764884, "logLoss": 0.7004372806048953, "accuracy": 0.5430743243243243, "ece": 0.054282057899609576, "reliabilityBins": [ { "lower": 0, "upper": 0.1, "upperInclusive": false, "n": 1, "meanPredictedProbability": 0.05299212795272118, "observedTeamAWinRate": 0, "observedRateWilson95": { "lower95": 0, "upper95": 0.7934506856227626 } }, { "lower": 0.1, "upper": 0.2, "upperInclusive": false, "n": 12, "meanPredictedProbability": 0.1731740797055826, "observedTeamAWinRate": 0.3333333333333333, "observedRateWilson95": { "lower95": 0.13812009109121315, "upper95": 0.6093779111272004 } }, { "lower": 0.2, "upper": 0.3, "upperInclusive": false, "n": 41, "meanPredictedProbability": 0.2525378072953051, "observedTeamAWinRate": 0.3902439024390244, "observedRateWilson95": { "lower95": 0.2565614506012352, "upper95": 0.5427314306587621 } }, { "lower": 0.3, "upper": 0.4, "upperInclusive": false, "n": 148, "meanPredictedProbability": 0.35713535241329053, "observedTeamAWinRate": 0.5, "observedRateWilson95": { "lower95": 0.4204714779657883, "upper95": 0.5795285220342117 } }, { "lower": 0.4, "upper": 0.5, "upperInclusive": false, "n": 341, "meanPredictedProbability": 0.45402672529910837, "observedTeamAWinRate": 0.501466275659824, "observedRateWilson95": { "lower95": 0.4486775822201764, "upper95": 0.554222301115835 } }, { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 383, "meanPredictedProbability": 0.5429634277625348, "observedTeamAWinRate": 0.5456919060052219, "observedRateWilson95": { "lower95": 0.4956190875322942, "upper95": 0.5948572541367173 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 177, "meanPredictedProbability": 0.640142388548762, "observedTeamAWinRate": 0.5875706214689266, "observedRateWilson95": { "lower95": 0.5139392946718131, "upper95": 0.6574815742050344 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 70, "meanPredictedProbability": 0.7402120024322425, "observedTeamAWinRate": 0.6285714285714286, "observedRateWilson95": { "lower95": 0.5114720115842711, "upper95": 0.7322934866530887 } }, { "lower": 0.8, "upper": 0.9, "upperInclusive": false, "n": 11, "meanPredictedProbability": 0.8330180776895876, "observedTeamAWinRate": 0.7272727272727273, "observedRateWilson95": { "lower95": 0.43435469882387084, "upper95": 0.902539407099751 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null } ], "accuracyByConfidenceBand": [ { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 724, "meanConfidence": 0.5443810490415122, "accuracy": 0.5234806629834254, "accuracyWilson95": { "lower95": 0.4870720951552407, "upper95": 0.559641374636779 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 325, "meanConfidence": 0.6413820634337347, "accuracy": 0.5476923076923077, "accuracyWilson95": { "lower95": 0.49333751347426746, "upper95": 0.6009328382969892 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 111, "meanConfidence": 0.7428900006409863, "accuracy": 0.6216216216216216, "accuracyWilson95": { "lower95": 0.5287598467882427, "upper95": 0.7063468845025537 } }, { "lower": 0.8, "upper": 0.8999999999999999, "upperInclusive": false, "n": 23, "meanConfidence": 0.8297873868747163, "accuracy": 0.6956521739130435, "accuracyWilson95": { "lower95": 0.49134243731240795, "upper95": 0.8439597554678597 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 1, "meanConfidence": 0.9470078720472788, "accuracy": 1, "accuracyWilson95": { "lower95": 0.20654931437723745, "upper95": 1 } } ], "sampleSizeByMap": [ { "mapName": "Abyss", "n": 44 }, { "mapName": "Ascent", "n": 114 }, { "mapName": "Bind", "n": 97 }, { "mapName": "Corrode", "n": 57 }, { "mapName": "Fracture", "n": 88 }, { "mapName": "Haven", "n": 175 }, { "mapName": "Icebox", "n": 118 }, { "mapName": "Lotus", "n": 217 }, { "mapName": "Pearl", "n": 97 }, { "mapName": "Split", "n": 104 }, { "mapName": "Sunset", "n": 73 } ] }, "intervals95": { "brier": { "estimate": 0.25293441690764884, "lower95": 0.2463196364215153, "upper95": 0.2592839381567076, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 31781525 }, "logLoss": { "estimate": 0.7004372806048953, "lower95": 0.685990216774477, "upper95": 0.7149003037918065, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 31781525 }, "accuracy": { "estimate": 0.5430743243243243, "lower95": 0.5131804137774901, "upper95": 0.573182917857446, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 31781525 }, "ece": { "estimate": 0.054282057899609576, "lower95": 0.03707787718202196, "upper95": 0.08650547046892532, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 31781525 } }, "pairedDeltaVsPlainElo": { "brier": { "estimate": 0.009264619184073564, "lower95": 0.0008673923048873277, "upper95": 0.01699786163958942, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 4123754072 }, "logLoss": { "estimate": 0.01889889136661349, "lower95": 0.0003706719674938469, "upper95": 0.03558720883338847, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 4123754072 }, "accuracy": { "estimate": -0.026182432432432456, "lower95": -0.062110000271304165, "upper95": 0.010046690981674045, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 4123754072 }, "ece": { "estimate": -0.011242110998450253, "lower95": -0.042876284070883106, "upper95": 0.01920457450905918, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 4123754072 } } } }, { "model": { "id": "season-carry-75", "label": "Production map Elo with 75% season carry", "family": "season-carry", "ratingScope": "map", "initialRating": 1000, "ratingScale": 1000, "predictionScale": 1000, "kFactor": 74, "marginFormula": "production", "marginScale": 1, "seasonCarry": 0.75, "coldStartPriorGames": 0, "rosterRegressionStrength": 0 }, "validation": { "metrics": { "n": 1184, "brier": 0.2553793535207225, "logLoss": 0.7066407929821915, "accuracy": 0.5388513513513513, "ece": 0.07093530161498078, "reliabilityBins": [ { "lower": 0, "upper": 0.1, "upperInclusive": false, "n": 2, "meanPredictedProbability": 0.0680521498476852, "observedTeamAWinRate": 0.5, "observedRateWilson95": { "lower95": 0.09453120573423074, "upper95": 0.9054687942657693 } }, { "lower": 0.1, "upper": 0.2, "upperInclusive": false, "n": 17, "meanPredictedProbability": 0.1634834442783928, "observedTeamAWinRate": 0.29411764705882354, "observedRateWilson95": { "lower95": 0.1327998961029523, "upper95": 0.5313311006653985 } }, { "lower": 0.2, "upper": 0.3, "upperInclusive": false, "n": 56, "meanPredictedProbability": 0.2511857360874696, "observedTeamAWinRate": 0.44642857142857145, "observedRateWilson95": { "lower95": 0.3238672125478337, "upper95": 0.5758678521006054 } }, { "lower": 0.3, "upper": 0.4, "upperInclusive": false, "n": 180, "meanPredictedProbability": 0.3594262956745848, "observedTeamAWinRate": 0.4888888888888889, "observedRateWilson95": { "lower95": 0.41686216064501447, "upper95": 0.5613799615310212 } }, { "lower": 0.4, "upper": 0.5, "upperInclusive": false, "n": 279, "meanPredictedProbability": 0.4538065766385367, "observedTeamAWinRate": 0.5161290322580645, "observedRateWilson95": { "lower95": 0.45766967947077297, "upper95": 0.5741502667193134 } }, { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 348, "meanPredictedProbability": 0.5434835253785026, "observedTeamAWinRate": 0.5431034482758621, "observedRateWilson95": { "lower95": 0.4905802228692232, "upper95": 0.5946854536106162 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 189, "meanPredictedProbability": 0.6416633968894094, "observedTeamAWinRate": 0.5661375661375662, "observedRateWilson95": { "lower95": 0.49485813457815375, "upper95": 0.634782037990989 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 90, "meanPredictedProbability": 0.7419851554599527, "observedTeamAWinRate": 0.6444444444444445, "observedRateWilson95": { "lower95": 0.5415017311001208, "upper95": 0.7355613104035182 } }, { "lower": 0.8, "upper": 0.9, "upperInclusive": false, "n": 23, "meanPredictedProbability": 0.830112019731633, "observedTeamAWinRate": 0.5652173913043478, "observedRateWilson95": { "lower95": 0.3681137508501646, "upper95": 0.743653646743258 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null } ], "accuracyByConfidenceBand": [ { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 627, "meanConfidence": 0.5446893651508253, "accuracy": 0.5167464114832536, "accuracyWilson95": { "lower95": 0.47764882352146365, "upper95": 0.5556400475670945 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 369, "meanConfidence": 0.6411318395411197, "accuracy": 0.5392953929539296, "accuracyWilson95": { "lower95": 0.48829355260881946, "upper95": 0.5894874969872412 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 146, "meanConfidence": 0.7446045395239554, "accuracy": 0.6095890410958904, "accuracyWilson95": { "lower95": 0.528618110885002, "upper95": 0.6849409418087681 } }, { "lower": 0.8, "upper": 0.8999999999999999, "upperInclusive": false, "n": 40, "meanConfidence": 0.8328339475273723, "accuracy": 0.625, "accuracyWilson95": { "lower95": 0.47032439137301096, "upper95": 0.7577702083276673 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 2, "meanConfidence": 0.9319478501523148, "accuracy": 0.5, "accuracyWilson95": { "lower95": 0.09453120573423074, "upper95": 0.9054687942657693 } } ], "sampleSizeByMap": [ { "mapName": "Abyss", "n": 44 }, { "mapName": "Ascent", "n": 114 }, { "mapName": "Bind", "n": 97 }, { "mapName": "Corrode", "n": 57 }, { "mapName": "Fracture", "n": 88 }, { "mapName": "Haven", "n": 175 }, { "mapName": "Icebox", "n": 118 }, { "mapName": "Lotus", "n": 217 }, { "mapName": "Pearl", "n": 97 }, { "mapName": "Split", "n": 104 }, { "mapName": "Sunset", "n": 73 } ] }, "intervals95": { "brier": { "estimate": 0.2553793535207225, "lower95": 0.24786563972102754, "upper95": 0.26302381756080695, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3181182350 }, "logLoss": { "estimate": 0.7066407929821915, "lower95": 0.6896638660300884, "upper95": 0.7238821017766714, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3181182350 }, "accuracy": { "estimate": 0.5388513513513513, "lower95": 0.5077002567545776, "upper95": 0.5668070575961871, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3181182350 }, "ece": { "estimate": 0.07093530161498078, "lower95": 0.0550156016754763, "upper95": 0.1068843054750466, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3181182350 } }, "pairedDeltaVsPlainElo": { "brier": { "estimate": 0.01170955579714722, "lower95": 0.0034764744212124944, "upper95": 0.019953913903587413, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1845731679 }, "logLoss": { "estimate": 0.025102403743909707, "lower95": 0.006967260736269803, "upper95": 0.04335868112722652, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1845731679 }, "accuracy": { "estimate": -0.030405405405405483, "lower95": -0.06439904081432908, "upper95": 0.004716411487902765, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1845731679 }, "ece": { "estimate": 0.005411132716920952, "lower95": -0.026979596517121656, "upper95": 0.03524135132230127, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1845731679 } } } }, { "model": { "id": "season-carry-100", "label": "Production map Elo with 100% season carry", "family": "season-carry", "ratingScope": "map", "initialRating": 1000, "ratingScale": 1000, "predictionScale": 1000, "kFactor": 74, "marginFormula": "production", "marginScale": 1, "seasonCarry": 1, "coldStartPriorGames": 0, "rosterRegressionStrength": 0 }, "validation": { "metrics": { "n": 1184, "brier": 0.2599979127193587, "logLoss": 0.7193216806400914, "accuracy": 0.5380067567567568, "ece": 0.09124066974323898, "reliabilityBins": [ { "lower": 0, "upper": 0.1, "upperInclusive": false, "n": 6, "meanPredictedProbability": 0.07469937871791271, "observedTeamAWinRate": 0.3333333333333333, "observedRateWilson95": { "lower95": 0.09677141110578041, "upper95": 0.700006684861608 } }, { "lower": 0.1, "upper": 0.2, "upperInclusive": false, "n": 27, "meanPredictedProbability": 0.15936447497744755, "observedTeamAWinRate": 0.37037037037037035, "observedRateWilson95": { "lower95": 0.21532512960965597, "upper95": 0.5577076558789761 } }, { "lower": 0.2, "upper": 0.3, "upperInclusive": false, "n": 80, "meanPredictedProbability": 0.2562469288072505, "observedTeamAWinRate": 0.475, "observedRateWilson95": { "lower95": 0.36924733297407564, "upper95": 0.5830435735027005 } }, { "lower": 0.3, "upper": 0.4, "upperInclusive": false, "n": 183, "meanPredictedProbability": 0.35277772946851366, "observedTeamAWinRate": 0.46994535519125685, "observedRateWilson95": { "lower95": 0.39899646866147, "upper95": 0.5421300882096347 } }, { "lower": 0.4, "upper": 0.5, "upperInclusive": false, "n": 231, "meanPredictedProbability": 0.4524537283537523, "observedTeamAWinRate": 0.5367965367965368, "observedRateWilson95": { "lower95": 0.47241664546939055, "upper95": 0.59997261699461 } }, { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 314, "meanPredictedProbability": 0.5463782022940654, "observedTeamAWinRate": 0.5382165605095541, "observedRateWilson95": { "lower95": 0.4829451586480277, "upper95": 0.592564185189773 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 195, "meanPredictedProbability": 0.6461846788138824, "observedTeamAWinRate": 0.5743589743589743, "observedRateWilson95": { "lower95": 0.5041834627262513, "upper95": 0.6416613735166311 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 100, "meanPredictedProbability": 0.7432570996645971, "observedTeamAWinRate": 0.56, "observedRateWilson95": { "lower95": 0.4622810465167699, "upper95": 0.6532797336983921 } }, { "lower": 0.8, "upper": 0.9, "upperInclusive": false, "n": 46, "meanPredictedProbability": 0.8399595733419049, "observedTeamAWinRate": 0.6956521739130435, "observedRateWilson95": { "lower95": 0.5519435355159235, "upper95": 0.8092015919723277 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 2, "meanPredictedProbability": 0.911130945248691, "observedTeamAWinRate": 0.5, "observedRateWilson95": { "lower95": 0.09453120573423074, "upper95": 0.9054687942657693 } } ], "accuracyByConfidenceBand": [ { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 545, "meanConfidence": 0.5468732922396691, "accuracy": 0.5064220183486239, "accuracyWilson95": { "lower95": 0.464549843017048, "upper95": 0.5482042955187708 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 378, "meanConfidence": 0.646687004962881, "accuracy": 0.5529100529100529, "accuracyWilson95": { "lower95": 0.5025058511572499, "upper95": 0.6022496673334315 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 180, "meanConfidence": 0.7434775314548868, "accuracy": 0.5444444444444444, "accuracyWilson95": { "lower95": 0.47151949265004, "upper95": 0.6155120186458166 } }, { "lower": 0.8, "upper": 0.8999999999999999, "upperInclusive": false, "n": 73, "meanConfidence": 0.8402095828676239, "accuracy": 0.6712328767123288, "accuracyWilson95": { "lower95": 0.5572900614018867, "upper95": 0.7680551413529999 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 8, "meanConfidence": 0.9217582022737382, "accuracy": 0.625, "accuracyWilson95": { "lower95": 0.3057423946026273, "upper95": 0.8631557141764027 } } ], "sampleSizeByMap": [ { "mapName": "Abyss", "n": 44 }, { "mapName": "Ascent", "n": 114 }, { "mapName": "Bind", "n": 97 }, { "mapName": "Corrode", "n": 57 }, { "mapName": "Fracture", "n": 88 }, { "mapName": "Haven", "n": 175 }, { "mapName": "Icebox", "n": 118 }, { "mapName": "Lotus", "n": 217 }, { "mapName": "Pearl", "n": 97 }, { "mapName": "Split", "n": 104 }, { "mapName": "Sunset", "n": 73 } ] }, "intervals95": { "brier": { "estimate": 0.2599979127193587, "lower95": 0.2513107407177876, "upper95": 0.2689469287359341, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3919279175 }, "logLoss": { "estimate": 0.7193216806400914, "lower95": 0.6995294301735893, "upper95": 0.739482773607306, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3919279175 }, "accuracy": { "estimate": 0.5380067567567568, "lower95": 0.5050657889104808, "upper95": 0.5687591152163345, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3919279175 }, "ece": { "estimate": 0.09124066974323898, "lower95": 0.0710659534647804, "upper95": 0.12295664941361709, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3919279175 } }, "pairedDeltaVsPlainElo": { "brier": { "estimate": 0.016328114995783405, "lower95": 0.007521658565212664, "upper95": 0.02527782056295188, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1904722150 }, "logLoss": { "estimate": 0.03778329140180958, "lower95": 0.017136617381964706, "upper95": 0.05798588581679377, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1904722150 }, "accuracy": { "estimate": -0.03125, "lower95": -0.06428639526113747, "upper95": 0.004705674752846962, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1904722150 }, "ece": { "estimate": 0.025716500845179152, "lower95": -0.0065396686363654345, "upper95": 0.053815779164754905, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1904722150 } } } } ], "bestOnValidationModelId": "season-carry-25" }, { "family": "roster-regression", "status": "available", "reason": "Prior-map player appearances provide adequate last-observed roster coverage and observed changes.", "evidence": { "developmentRows": 2983, "rowsWithBothRosters": 2983, "coverageRate": 1, "observedRosterChanges": 315, "minimumCoverageRate": 0.6, "minimumCompleteRows": 30 }, "candidates": [ { "model": { "id": "roster-regression-25", "label": "Last-observed roster proxy (25% strength)", "family": "roster-regression", "ratingScope": "map", "initialRating": 1000, "ratingScale": 1000, "predictionScale": 1000, "kFactor": 74, "marginFormula": "production", "marginScale": 1, "seasonCarry": 0, "coldStartPriorGames": 0, "rosterRegressionStrength": 0.25 }, "validation": { "metrics": { "n": 1184, "brier": 0.25258925391038106, "logLoss": 0.6991810114702315, "accuracy": 0.5388513513513513, "ece": 0.0537085742561131, "reliabilityBins": [ { "lower": 0, "upper": 0.1, "upperInclusive": false, "n": 1, "meanPredictedProbability": 0.07758848936112012, "observedTeamAWinRate": 0, "observedRateWilson95": { "lower95": 0, "upper95": 0.7934506856227626 } }, { "lower": 0.1, "upper": 0.2, "upperInclusive": false, "n": 5, "meanPredictedProbability": 0.17982374506767704, "observedTeamAWinRate": 0.6, "observedRateWilson95": { "lower95": 0.2307242812760128, "upper95": 0.882379225767352 } }, { "lower": 0.2, "upper": 0.3, "upperInclusive": false, "n": 40, "meanPredictedProbability": 0.26800779829255406, "observedTeamAWinRate": 0.425, "observedRateWilson95": { "lower95": 0.28509392559950375, "upper95": 0.5780493145800892 } }, { "lower": 0.3, "upper": 0.4, "upperInclusive": false, "n": 137, "meanPredictedProbability": 0.3622018527515037, "observedTeamAWinRate": 0.48905109489051096, "observedRateWilson95": { "lower95": 0.406793178139353, "upper95": 0.5719062756640947 } }, { "lower": 0.4, "upper": 0.5, "upperInclusive": false, "n": 305, "meanPredictedProbability": 0.45514677642193224, "observedTeamAWinRate": 0.5016393442622951, "observedRateWilson95": { "lower95": 0.4458557145325085, "upper95": 0.5573821927219723 } }, { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 497, "meanPredictedProbability": 0.5333173706276055, "observedTeamAWinRate": 0.5492957746478874, "observedRateWilson95": { "lower95": 0.505340081748531, "upper95": 0.5924952694130479 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 147, "meanPredictedProbability": 0.6373745077115789, "observedTeamAWinRate": 0.564625850340136, "observedRateWilson95": { "lower95": 0.48384052172838504, "upper95": 0.642119543551714 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 46, "meanPredictedProbability": 0.7372619573503418, "observedTeamAWinRate": 0.6304347826086957, "observedRateWilson95": { "lower95": 0.48600096423389705, "upper95": 0.7547624540916037 } }, { "lower": 0.8, "upper": 0.9, "upperInclusive": false, "n": 6, "meanPredictedProbability": 0.8291924578725368, "observedTeamAWinRate": 0.8333333333333334, "observedRateWilson95": { "lower95": 0.43649717781352987, "upper95": 0.9699466302516935 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null } ], "accuracyByConfidenceBand": [ { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 802, "meanConfidence": 0.5377044468743523, "accuracy": 0.529925187032419, "accuracyWilson95": { "lower95": 0.4953223342404829, "upper95": 0.5642427321563808 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 284, "meanConfidence": 0.6375788690374866, "accuracy": 0.5387323943661971, "accuracyWilson95": { "lower95": 0.48062483766563746, "upper95": 0.5958061257200244 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 86, "meanConfidence": 0.7348109082141112, "accuracy": 0.6046511627906976, "accuracyWilson95": { "lower95": 0.4989772079672809, "upper95": 0.7013757274845405 } }, { "lower": 0.8, "upper": 0.8999999999999999, "upperInclusive": false, "n": 11, "meanConfidence": 0.8250941838088033, "accuracy": 0.6363636363636364, "accuracyWilson95": { "lower95": 0.35380117450784887, "upper95": 0.8483352890463243 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 1, "meanConfidence": 0.9224115106388799, "accuracy": 1, "accuracyWilson95": { "lower95": 0.20654931437723745, "upper95": 1 } } ], "sampleSizeByMap": [ { "mapName": "Abyss", "n": 44 }, { "mapName": "Ascent", "n": 114 }, { "mapName": "Bind", "n": 97 }, { "mapName": "Corrode", "n": 57 }, { "mapName": "Fracture", "n": 88 }, { "mapName": "Haven", "n": 175 }, { "mapName": "Icebox", "n": 118 }, { "mapName": "Lotus", "n": 217 }, { "mapName": "Pearl", "n": 97 }, { "mapName": "Split", "n": 104 }, { "mapName": "Sunset", "n": 73 } ] }, "intervals95": { "brier": { "estimate": 0.25258925391038106, "lower95": 0.24681981377006135, "upper95": 0.25867310047503106, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3654353433 }, "logLoss": { "estimate": 0.6991810114702315, "lower95": 0.6867587090397479, "upper95": 0.711935376816224, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3654353433 }, "accuracy": { "estimate": 0.5388513513513513, "lower95": 0.5111843580125455, "upper95": 0.567898953660922, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3654353433 }, "ece": { "estimate": 0.0537085742561131, "lower95": 0.035540396889882656, "upper95": 0.08051871300100574, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3654353433 } }, "pairedDeltaVsPlainElo": { "brier": { "estimate": 0.008919456186805785, "lower95": 0.00040847227947248433, "upper95": 0.016777609525894004, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2313121388 }, "logLoss": { "estimate": 0.01764262223194968, "lower95": -0.0011149407546416409, "upper95": 0.03426550624178745, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2313121388 }, "accuracy": { "estimate": -0.030405405405405483, "lower95": -0.07311049636097497, "upper95": 0.008493906212360115, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2313121388 }, "ece": { "estimate": -0.01181559464194673, "lower95": -0.043295185276612816, "upper95": 0.01482043796269089, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2313121388 } } } }, { "model": { "id": "roster-regression-50", "label": "Last-observed roster proxy (50% strength)", "family": "roster-regression", "ratingScope": "map", "initialRating": 1000, "ratingScale": 1000, "predictionScale": 1000, "kFactor": 74, "marginFormula": "production", "marginScale": 1, "seasonCarry": 0, "coldStartPriorGames": 0, "rosterRegressionStrength": 0.5 }, "validation": { "metrics": { "n": 1184, "brier": 0.25251333365076134, "logLoss": 0.6989599946317091, "accuracy": 0.5371621621621622, "ece": 0.05744790136986284, "reliabilityBins": [ { "lower": 0, "upper": 0.1, "upperInclusive": false, "n": 1, "meanPredictedProbability": 0.08169516864398763, "observedTeamAWinRate": 0, "observedRateWilson95": { "lower95": 0, "upper95": 0.7934506856227626 } }, { "lower": 0.1, "upper": 0.2, "upperInclusive": false, "n": 3, "meanPredictedProbability": 0.18286819600255003, "observedTeamAWinRate": 0.3333333333333333, "observedRateWilson95": { "lower95": 0.06149194472039615, "upper95": 0.7923403991979523 } }, { "lower": 0.2, "upper": 0.3, "upperInclusive": false, "n": 38, "meanPredictedProbability": 0.2635848244373665, "observedTeamAWinRate": 0.47368421052631576, "observedRateWilson95": { "lower95": 0.32479023322526057, "upper95": 0.6274102861550331 } }, { "lower": 0.3, "upper": 0.4, "upperInclusive": false, "n": 129, "meanPredictedProbability": 0.3611003591502897, "observedTeamAWinRate": 0.4883720930232558, "observedRateWilson95": { "lower95": 0.40370476711526254, "upper95": 0.5737119217209439 } }, { "lower": 0.4, "upper": 0.5, "upperInclusive": false, "n": 317, "meanPredictedProbability": 0.4548127263866795, "observedTeamAWinRate": 0.501577287066246, "observedRateWilson95": { "lower95": 0.4468478976405923, "upper95": 0.5562689065395023 } }, { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 507, "meanPredictedProbability": 0.5336370407768062, "observedTeamAWinRate": 0.5542406311637081, "observedRateWilson95": { "lower95": 0.5107281223259185, "upper95": 0.5969373755508017 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 141, "meanPredictedProbability": 0.6377156774306036, "observedTeamAWinRate": 0.5531914893617021, "observedRateWilson95": { "lower95": 0.470802945074666, "upper95": 0.6327585636406612 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 42, "meanPredictedProbability": 0.7346460515139119, "observedTeamAWinRate": 0.5952380952380952, "observedRateWilson95": { "lower95": 0.44494312750039344, "upper95": 0.7295713885251867 } }, { "lower": 0.8, "upper": 0.9, "upperInclusive": false, "n": 6, "meanPredictedProbability": 0.826672740432119, "observedTeamAWinRate": 0.8333333333333334, "observedRateWilson95": { "lower95": 0.43649717781352987, "upper95": 0.9699466302516935 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null } ], "accuracyByConfidenceBand": [ { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 824, "meanConfidence": 0.5380805162733777, "accuracy": 0.5327669902912622, "accuracyWilson95": { "lower95": 0.4986278284788598, "upper95": 0.5666020527074801 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 270, "meanConfidence": 0.6382813488419542, "accuracy": 0.5333333333333333, "accuracyWilson95": { "lower95": 0.4737756158080121, "upper95": 0.5919558482205448 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 80, "meanConfidence": 0.7354863854370546, "accuracy": 0.5625, "accuracyWilson95": { "lower95": 0.45341134296949726, "upper95": 0.6658613908385618 } }, { "lower": 0.8, "upper": 0.8999999999999999, "upperInclusive": false, "n": 9, "meanConfidence": 0.8234924282872295, "accuracy": 0.7777777777777778, "accuracyWilson95": { "lower95": 0.4525889691069889, "upper95": 0.9367748928821533 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 1, "meanConfidence": 0.9183048313560124, "accuracy": 1, "accuracyWilson95": { "lower95": 0.20654931437723745, "upper95": 1 } } ], "sampleSizeByMap": [ { "mapName": "Abyss", "n": 44 }, { "mapName": "Ascent", "n": 114 }, { "mapName": "Bind", "n": 97 }, { "mapName": "Corrode", "n": 57 }, { "mapName": "Fracture", "n": 88 }, { "mapName": "Haven", "n": 175 }, { "mapName": "Icebox", "n": 118 }, { "mapName": "Lotus", "n": 217 }, { "mapName": "Pearl", "n": 97 }, { "mapName": "Split", "n": 104 }, { "mapName": "Sunset", "n": 73 } ] }, "intervals95": { "brier": { "estimate": 0.25251333365076134, "lower95": 0.24731457200622392, "upper95": 0.258264646241335, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 538022889 }, "logLoss": { "estimate": 0.6989599946317091, "lower95": 0.687883740088973, "upper95": 0.7114134879155062, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 538022889 }, "accuracy": { "estimate": 0.5371621621621622, "lower95": 0.5095625348300294, "upper95": 0.5616963891190279, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 538022889 }, "ece": { "estimate": 0.05744790136986284, "lower95": 0.039509244402414025, "upper95": 0.08781308499842941, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 538022889 } }, "pairedDeltaVsPlainElo": { "brier": { "estimate": 0.008843535927186064, "lower95": 0.00025812483672023464, "upper95": 0.01722036594574603, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3992455804 }, "logLoss": { "estimate": 0.017421605393427297, "lower95": -0.0008927506858974032, "upper95": 0.035432767431866866, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3992455804 }, "accuracy": { "estimate": -0.03209459459459463, "lower95": -0.06914324176940273, "upper95": 0.00714272665441275, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3992455804 }, "ece": { "estimate": -0.008076267528196991, "lower95": -0.04375400974883162, "upper95": 0.01788397545252586, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3992455804 } } } }, { "model": { "id": "roster-regression-75", "label": "Last-observed roster proxy (75% strength)", "family": "roster-regression", "ratingScope": "map", "initialRating": 1000, "ratingScale": 1000, "predictionScale": 1000, "kFactor": 74, "marginFormula": "production", "marginScale": 1, "seasonCarry": 0, "coldStartPriorGames": 0, "rosterRegressionStrength": 0.75 }, "validation": { "metrics": { "n": 1184, "brier": 0.2525006462455276, "logLoss": 0.6988856306393176, "accuracy": 0.5346283783783784, "ece": 0.053984639858583254, "reliabilityBins": [ { "lower": 0, "upper": 0.1, "upperInclusive": false, "n": 1, "meanPredictedProbability": 0.08596700596113638, "observedTeamAWinRate": 0, "observedRateWilson95": { "lower95": 0, "upper95": 0.7934506856227626 } }, { "lower": 0.1, "upper": 0.2, "upperInclusive": false, "n": 2, "meanPredictedProbability": 0.18394080457069842, "observedTeamAWinRate": 0.5, "observedRateWilson95": { "lower95": 0.09453120573423074, "upper95": 0.9054687942657693 } }, { "lower": 0.2, "upper": 0.3, "upperInclusive": false, "n": 37, "meanPredictedProbability": 0.2631851366431334, "observedTeamAWinRate": 0.4594594594594595, "observedRateWilson95": { "lower95": 0.3103856980437303, "upper95": 0.6161595310782052 } }, { "lower": 0.3, "upper": 0.4, "upperInclusive": false, "n": 127, "meanPredictedProbability": 0.3624494811026418, "observedTeamAWinRate": 0.4881889763779528, "observedRateWilson95": { "lower95": 0.4028856656610888, "upper95": 0.5741858220230158 } }, { "lower": 0.4, "upper": 0.5, "upperInclusive": false, "n": 326, "meanPredictedProbability": 0.4563769710426575, "observedTeamAWinRate": 0.5061349693251533, "observedRateWilson95": { "lower95": 0.45210833729898253, "upper95": 0.5600187009887495 } }, { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 506, "meanPredictedProbability": 0.5335291818609245, "observedTeamAWinRate": 0.549407114624506, "observedRateWilson95": { "lower95": 0.5058445775852763, "upper95": 0.5922251245445861 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 144, "meanPredictedProbability": 0.6382110362160917, "observedTeamAWinRate": 0.5833333333333334, "observedRateWilson95": { "lower95": 0.5016686220726789, "upper95": 0.6606674384034816 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 35, "meanPredictedProbability": 0.7359169635351945, "observedTeamAWinRate": 0.5142857142857142, "observedRateWilson95": { "lower95": 0.3556880459586857, "upper95": 0.6700576399749025 } }, { "lower": 0.8, "upper": 0.9, "upperInclusive": false, "n": 6, "meanPredictedProbability": 0.824360334290103, "observedTeamAWinRate": 0.8333333333333334, "observedRateWilson95": { "lower95": 0.43649717781352987, "upper95": 0.9699466302516935 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null } ], "accuracyByConfidenceBand": [ { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 832, "meanConfidence": 0.5374842229107228, "accuracy": 0.5276442307692307, "accuracyWilson95": { "lower95": 0.4936721889451414, "upper95": 0.561362171356754 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 271, "meanConfidence": 0.6379014948896006, "accuracy": 0.5498154981549815, "accuracyWilson95": { "lower95": 0.4902970417840027, "upper95": 0.6079414121808875 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 72, "meanConfidence": 0.7363783842768872, "accuracy": 0.5277777777777778, "accuracyWilson95": { "lower95": 0.4140067049007661, "upper95": 0.6387348966216531 } }, { "lower": 0.8, "upper": 0.8999999999999999, "upperInclusive": false, "n": 8, "meanConfidence": 0.8222850495749027, "accuracy": 0.75, "accuracyWilson95": { "lower95": 0.40927543031016883, "upper95": 0.9285207872478909 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 1, "meanConfidence": 0.9140329940388636, "accuracy": 1, "accuracyWilson95": { "lower95": 0.20654931437723745, "upper95": 1 } } ], "sampleSizeByMap": [ { "mapName": "Abyss", "n": 44 }, { "mapName": "Ascent", "n": 114 }, { "mapName": "Bind", "n": 97 }, { "mapName": "Corrode", "n": 57 }, { "mapName": "Fracture", "n": 88 }, { "mapName": "Haven", "n": 175 }, { "mapName": "Icebox", "n": 118 }, { "mapName": "Lotus", "n": 217 }, { "mapName": "Pearl", "n": 97 }, { "mapName": "Split", "n": 104 }, { "mapName": "Sunset", "n": 73 } ] }, "intervals95": { "brier": { "estimate": 0.2525006462455276, "lower95": 0.24729461592728047, "upper95": 0.2577113115929582, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2423300826 }, "logLoss": { "estimate": 0.6988856306393176, "lower95": 0.6879267885171985, "upper95": 0.7100109533031982, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2423300826 }, "accuracy": { "estimate": 0.5346283783783784, "lower95": 0.5078014112357115, "upper95": 0.5616780645430868, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2423300826 }, "ece": { "estimate": 0.053984639858583254, "lower95": 0.033206933337260956, "upper95": 0.08254845605185235, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2423300826 } }, "pairedDeltaVsPlainElo": { "brier": { "estimate": 0.008830848521952317, "lower95": 0.00009339946423854987, "upper95": 0.016420582353586625, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2458546219 }, "logLoss": { "estimate": 0.01734724140103583, "lower95": -0.002549555739359532, "upper95": 0.03462621352139458, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2458546219 }, "accuracy": { "estimate": -0.0346283783783784, "lower95": -0.06979187723258778, "upper95": 0.003828707031013714, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2458546219 }, "ece": { "estimate": -0.011539529039476575, "lower95": -0.04783166003000405, "upper95": 0.014918890459855755, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2458546219 } } } } ], "bestOnValidationModelId": "roster-regression-75" }, { "family": "cold-start-uncertainty", "status": "available", "reason": "Every row has observable prior map counts; shrinkage uses only counts strictly before prediction.", "evidence": { "candidatePriorGameCounts": "3,8,15" }, "candidates": [ { "model": { "id": "cold-start-prior-3", "label": "Production map Elo with 3-game uncertainty prior", "family": "cold-start-uncertainty", "ratingScope": "map", "initialRating": 1000, "ratingScale": 1000, "predictionScale": 1000, "kFactor": 74, "marginFormula": "production", "marginScale": 1, "seasonCarry": 0, "coldStartPriorGames": 3, "rosterRegressionStrength": 0 }, "validation": { "metrics": { "n": 1184, "brier": 0.24834045477458444, "logLoss": 0.689864809698639, "accuracy": 0.5413851351351351, "ece": 0.03928062470202826, "reliabilityBins": [ { "lower": 0, "upper": 0.1, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.1, "upper": 0.2, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.2, "upper": 0.3, "upperInclusive": false, "n": 5, "meanPredictedProbability": 0.26617999933177244, "observedTeamAWinRate": 0.8, "observedRateWilson95": { "lower95": 0.3755346297625253, "upper95": 0.9637758913675698 } }, { "lower": 0.3, "upper": 0.4, "upperInclusive": false, "n": 51, "meanPredictedProbability": 0.36793597949387086, "observedTeamAWinRate": 0.3137254901960784, "observedRateWilson95": { "lower95": 0.20327678465525453, "upper95": 0.45026999530694345 } }, { "lower": 0.4, "upper": 0.5, "upperInclusive": false, "n": 329, "meanPredictedProbability": 0.4671064896841575, "observedTeamAWinRate": 0.5075987841945289, "observedRateWilson95": { "lower95": 0.45380174466437884, "upper95": 0.5612204223989302 } }, { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 748, "meanPredictedProbability": 0.5172248769354357, "observedTeamAWinRate": 0.5508021390374331, "observedRateWilson95": { "lower95": 0.5149865476520136, "upper95": 0.5860985935148625 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 43, "meanPredictedProbability": 0.6370787165774576, "observedTeamAWinRate": 0.5813953488372093, "observedRateWilson95": { "lower95": 0.4332857738696422, "upper95": 0.7161544907215082 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 8, "meanPredictedProbability": 0.7197146336704981, "observedTeamAWinRate": 0.75, "observedRateWilson95": { "lower95": 0.40927543031016883, "upper95": 0.9285207872478909 } }, { "lower": 0.8, "upper": 0.9, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null } ], "accuracyByConfidenceBand": [ { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 1077, "meanConfidence": 0.5220113025456066, "accuracy": 0.5329619312906221, "accuracyWilson95": { "lower95": 0.5031011002287746, "upper95": 0.5625884599010413 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 94, "meanConfidence": 0.6343579772196091, "accuracy": 0.6382978723404256, "accuracyWilson95": { "lower95": 0.5375052079060172, "upper95": 0.7282308135452578 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 13, "meanConfidence": 0.725139774823471, "accuracy": 0.5384615384615384, "accuracyWilson95": { "lower95": 0.29143795714506004, "upper95": 0.7679393219046169 } }, { "lower": 0.8, "upper": 0.8999999999999999, "upperInclusive": false, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null } ], "sampleSizeByMap": [ { "mapName": "Abyss", "n": 44 }, { "mapName": "Ascent", "n": 114 }, { "mapName": "Bind", "n": 97 }, { "mapName": "Corrode", "n": 57 }, { "mapName": "Fracture", "n": 88 }, { "mapName": "Haven", "n": 175 }, { "mapName": "Icebox", "n": 118 }, { "mapName": "Lotus", "n": 217 }, { "mapName": "Pearl", "n": 97 }, { "mapName": "Split", "n": 104 }, { "mapName": "Sunset", "n": 73 } ] }, "intervals95": { "brier": { "estimate": 0.24834045477458444, "lower95": 0.2456862715579088, "upper95": 0.2509111012143804, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 4012818894 }, "logLoss": { "estimate": 0.689864809698639, "lower95": 0.6843753992082988, "upper95": 0.6951210820744083, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 4012818894 }, "accuracy": { "estimate": 0.5413851351351351, "lower95": 0.5149524497761793, "upper95": 0.5707195275874504, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 4012818894 }, "ece": { "estimate": 0.03928062470202826, "lower95": 0.016646164792236386, "upper95": 0.07164340096456079, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 4012818894 } }, "pairedDeltaVsPlainElo": { "brier": { "estimate": 0.004670657051009164, "lower95": -0.004297805628159496, "upper95": 0.01197836399446764, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3205564831 }, "logLoss": { "estimate": 0.008326420460357142, "lower95": -0.011702911072607604, "upper95": 0.024340977236113094, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3205564831 }, "accuracy": { "estimate": -0.02787162162162171, "lower95": -0.060649894491924455, "upper95": 0.007704165635194183, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3205564831 }, "ece": { "estimate": -0.026243544196031567, "lower95": -0.0633074207654427, "upper95": -0.0004960027675334942, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3205564831 } } } }, { "model": { "id": "cold-start-prior-8", "label": "Production map Elo with 8-game uncertainty prior", "family": "cold-start-uncertainty", "ratingScope": "map", "initialRating": 1000, "ratingScale": 1000, "predictionScale": 1000, "kFactor": 74, "marginFormula": "production", "marginScale": 1, "seasonCarry": 0, "coldStartPriorGames": 8, "rosterRegressionStrength": 0 }, "validation": { "metrics": { "n": 1184, "brier": 0.2483543842832202, "logLoss": 0.6898503861574465, "accuracy": 0.5413851351351351, "ece": 0.03233234152559392, "reliabilityBins": [ { "lower": 0, "upper": 0.1, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.1, "upper": 0.2, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.2, "upper": 0.3, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.3, "upper": 0.4, "upperInclusive": false, "n": 13, "meanPredictedProbability": 0.3665345813495317, "observedTeamAWinRate": 0.5384615384615384, "observedRateWilson95": { "lower95": 0.29143795714506004, "upper95": 0.7679393219046169 } }, { "lower": 0.4, "upper": 0.5, "upperInclusive": false, "n": 372, "meanPredictedProbability": 0.47575104598708695, "observedTeamAWinRate": 0.4838709677419355, "observedRateWilson95": { "lower95": 0.4335124895186582, "upper95": 0.5345591542067221 } }, { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 784, "meanPredictedProbability": 0.5121667301294971, "observedTeamAWinRate": 0.5535714285714286, "observedRateWilson95": { "lower95": 0.5185962871697138, "upper95": 0.5880241490354818 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 15, "meanPredictedProbability": 0.6376364968379438, "observedTeamAWinRate": 0.6, "observedRateWilson95": { "lower95": 0.357468301206246, "upper95": 0.8017550385865841 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.8, "upper": 0.9, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null } ], "accuracyByConfidenceBand": [ { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 1156, "meanConfidence": 0.5160547814137795, "accuracy": 0.5415224913494809, "accuracyWilson95": { "lower95": 0.5127088036992963, "upper95": 0.5700611294410581 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 28, "meanConfidence": 0.6356999248223302, "accuracy": 0.5357142857142857, "accuracyWilson95": { "lower95": 0.358126892344679, "upper95": 0.7046843001720462 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null }, { "lower": 0.8, "upper": 0.8999999999999999, "upperInclusive": false, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null } ], "sampleSizeByMap": [ { "mapName": "Abyss", "n": 44 }, { "mapName": "Ascent", "n": 114 }, { "mapName": "Bind", "n": 97 }, { "mapName": "Corrode", "n": 57 }, { "mapName": "Fracture", "n": 88 }, { "mapName": "Haven", "n": 175 }, { "mapName": "Icebox", "n": 118 }, { "mapName": "Lotus", "n": 217 }, { "mapName": "Pearl", "n": 97 }, { "mapName": "Split", "n": 104 }, { "mapName": "Sunset", "n": 73 } ] }, "intervals95": { "brier": { "estimate": 0.2483543842832202, "lower95": 0.24661714128207948, "upper95": 0.25020276429014987, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2141677177 }, "logLoss": { "estimate": 0.6898503861574465, "lower95": 0.6863456242118213, "upper95": 0.6936087528455444, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2141677177 }, "accuracy": { "estimate": 0.5413851351351351, "lower95": 0.5130082519673564, "upper95": 0.5687942593204502, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2141677177 }, "ece": { "estimate": 0.03233234152559392, "lower95": 0.013754098384133624, "upper95": 0.06492379328522738, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2141677177 } }, "pairedDeltaVsPlainElo": { "brier": { "estimate": 0.004684586559644932, "lower95": -0.003933981368930021, "upper95": 0.01274999946971556, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1217566668 }, "logLoss": { "estimate": 0.008311996919164732, "lower95": -0.011459432191697682, "upper95": 0.02686104183833876, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1217566668 }, "accuracy": { "estimate": -0.02787162162162171, "lower95": -0.06324048500101309, "upper95": 0.006719603614451903, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1217566668 }, "ece": { "estimate": -0.033191827372465905, "lower95": -0.07064582232751518, "upper95": -0.005785365401993063, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1217566668 } } } }, { "model": { "id": "cold-start-prior-15", "label": "Production map Elo with 15-game uncertainty prior", "family": "cold-start-uncertainty", "ratingScope": "map", "initialRating": 1000, "ratingScale": 1000, "predictionScale": 1000, "kFactor": 74, "marginFormula": "production", "marginScale": 1, "seasonCarry": 0, "coldStartPriorGames": 15, "rosterRegressionStrength": 0 }, "validation": { "metrics": { "n": 1184, "brier": 0.24870642961480088, "logLoss": 0.6905552412680424, "accuracy": 0.5413851351351351, "ece": 0.031690816704723054, "reliabilityBins": [ { "lower": 0, "upper": 0.1, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.1, "upper": 0.2, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.2, "upper": 0.3, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.3, "upper": 0.4, "upperInclusive": false, "n": 3, "meanPredictedProbability": 0.37875752284776115, "observedTeamAWinRate": 0.6666666666666666, "observedRateWilson95": { "lower95": 0.20765960080204765, "upper95": 0.9385080552796037 } }, { "lower": 0.4, "upper": 0.5, "upperInclusive": false, "n": 382, "meanPredictedProbability": 0.48302879652290054, "observedTeamAWinRate": 0.48429319371727747, "observedRateWilson95": { "lower95": 0.4345839317656578, "upper95": 0.5343152113262292 } }, { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 795, "meanPredictedProbability": 0.5086070803043766, "observedTeamAWinRate": 0.5534591194968553, "observedRateWilson95": { "lower95": 0.5187271270084762, "upper95": 0.5876769648941685 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 4, "meanPredictedProbability": 0.6205428348342973, "observedTeamAWinRate": 0.75, "observedRateWilson95": { "lower95": 0.30064184258240184, "upper95": 0.9544127391902995 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.8, "upper": 0.9, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null } ], "accuracyByConfidenceBand": [ { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 1177, "meanConfidence": 0.5113216895244105, "accuracy": 0.5412064570943076, "accuracyWilson95": { "lower95": 0.5126508889003258, "upper95": 0.5694939234073941 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 7, "meanConfidence": 0.6208426815419864, "accuracy": 0.5714285714285714, "accuracyWilson95": { "lower95": 0.2504583645276572, "upper95": 0.8417801447485302 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null }, { "lower": 0.8, "upper": 0.8999999999999999, "upperInclusive": false, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null } ], "sampleSizeByMap": [ { "mapName": "Abyss", "n": 44 }, { "mapName": "Ascent", "n": 114 }, { "mapName": "Bind", "n": 97 }, { "mapName": "Corrode", "n": 57 }, { "mapName": "Fracture", "n": 88 }, { "mapName": "Haven", "n": 175 }, { "mapName": "Icebox", "n": 118 }, { "mapName": "Lotus", "n": 217 }, { "mapName": "Pearl", "n": 97 }, { "mapName": "Split", "n": 104 }, { "mapName": "Sunset", "n": 73 } ] }, "intervals95": { "brier": { "estimate": 0.24870642961480088, "lower95": 0.24771149314274238, "upper95": 0.24964944571079667, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3934204877 }, "logLoss": { "estimate": 0.6905552412680424, "lower95": 0.6885581851433256, "upper95": 0.6924406180537361, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3934204877 }, "accuracy": { "estimate": 0.5413851351351351, "lower95": 0.5152440919670899, "upper95": 0.5666169002489032, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3934204877 }, "ece": { "estimate": 0.031690816704723054, "lower95": 0.014194652251723984, "upper95": 0.06273824650075184, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3934204877 } }, "pairedDeltaVsPlainElo": { "brier": { "estimate": 0.005036631891225607, "lower95": -0.003174889874049681, "upper95": 0.013481471815179492, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3154451008 }, "logLoss": { "estimate": 0.009016852029760547, "lower95": -0.009076569835677926, "upper95": 0.027887141061411234, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3154451008 }, "accuracy": { "estimate": -0.02787162162162171, "lower95": -0.06082913940067651, "upper95": 0.003412389086989023, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3154451008 }, "ece": { "estimate": -0.033833352193336774, "lower95": -0.06827192070044255, "upper95": -0.0014111386533780797, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3154451008 } } } } ], "bestOnValidationModelId": "cold-start-prior-8" }, { "family": "margin-of-victory", "status": "available", "reason": "Valid score margins are present before the holdout.", "evidence": { "formulas": "none,log1p,sqrt,linear-capped" }, "candidates": [ { "model": { "id": "mov-none", "label": "Map Elo without MOV", "family": "margin-of-victory", "ratingScope": "map", "initialRating": 1000, "ratingScale": 1000, "predictionScale": 1000, "kFactor": 74, "marginFormula": "none", "marginScale": 0, "seasonCarry": 0, "coldStartPriorGames": 0, "rosterRegressionStrength": 0 }, "validation": { "metrics": { "n": 1184, "brier": 0.24873231761128192, "logLoss": 0.6905913058170543, "accuracy": 0.5337837837837838, "ece": 0.03263382508489537, "reliabilityBins": [ { "lower": 0, "upper": 0.1, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.1, "upper": 0.2, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.2, "upper": 0.3, "upperInclusive": false, "n": 2, "meanPredictedProbability": 0.2659541097584587, "observedTeamAWinRate": 0.5, "observedRateWilson95": { "lower95": 0.09453120573423074, "upper95": 0.9054687942657693 } }, { "lower": 0.3, "upper": 0.4, "upperInclusive": false, "n": 28, "meanPredictedProbability": 0.37504911756449283, "observedTeamAWinRate": 0.32142857142857145, "observedRateWilson95": { "lower95": 0.17933246922494958, "upper95": 0.5066115681914238 } }, { "lower": 0.4, "upper": 0.5, "upperInclusive": false, "n": 442, "meanPredictedProbability": 0.46547118182405267, "observedTeamAWinRate": 0.5090497737556561, "observedRateWilson95": { "lower95": 0.46256759426040367, "upper95": 0.5553760039441261 } }, { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 676, "meanPredictedProbability": 0.5249248243302905, "observedTeamAWinRate": 0.5502958579881657, "observedRateWilson95": { "lower95": 0.5126162435299509, "upper95": 0.5874070767864624 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 36, "meanPredictedProbability": 0.6317659599518083, "observedTeamAWinRate": 0.6388888888888888, "observedRateWilson95": { "lower95": 0.4757507548555804, "upper95": 0.7752440705348462 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.8, "upper": 0.9, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null } ], "accuracyByConfidenceBand": [ { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 1118, "meanConfidence": 0.5287217521297369, "accuracy": 0.5268336314847942, "accuracyWilson95": { "lower95": 0.49752524742569604, "upper95": 0.5559582457437161 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 64, "meanConfidence": 0.6287843635384264, "accuracy": 0.65625, "accuracyWilson95": { "lower95": 0.5340364020991819, "upper95": 0.7607685792155665 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 2, "meanConfidence": 0.7340458902415413, "accuracy": 0.5, "accuracyWilson95": { "lower95": 0.09453120573423074, "upper95": 0.9054687942657693 } }, { "lower": 0.8, "upper": 0.8999999999999999, "upperInclusive": false, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null } ], "sampleSizeByMap": [ { "mapName": "Abyss", "n": 44 }, { "mapName": "Ascent", "n": 114 }, { "mapName": "Bind", "n": 97 }, { "mapName": "Corrode", "n": 57 }, { "mapName": "Fracture", "n": 88 }, { "mapName": "Haven", "n": 175 }, { "mapName": "Icebox", "n": 118 }, { "mapName": "Lotus", "n": 217 }, { "mapName": "Pearl", "n": 97 }, { "mapName": "Split", "n": 104 }, { "mapName": "Sunset", "n": 73 } ] }, "intervals95": { "brier": { "estimate": 0.24873231761128192, "lower95": 0.24608292752459043, "upper95": 0.2513449869906186, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2154553093 }, "logLoss": { "estimate": 0.6905913058170543, "lower95": 0.6851693733389619, "upper95": 0.695901074422489, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2154553093 }, "accuracy": { "estimate": 0.5337837837837838, "lower95": 0.507369905179885, "upper95": 0.562987173841242, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2154553093 }, "ece": { "estimate": 0.03263382508489537, "lower95": 0.012232086879650342, "upper95": 0.06204045127267789, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2154553093 } }, "pairedDeltaVsPlainElo": { "brier": { "estimate": 0.005062519887706646, "lower95": -0.0036202396961244505, "upper95": 0.011481379973991287, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2783029576 }, "logLoss": { "estimate": 0.009052916578772496, "lower95": -0.010460567392084612, "upper95": 0.023267832084885733, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2783029576 }, "accuracy": { "estimate": -0.03547297297297303, "lower95": -0.06949152542372883, "upper95": -0.0029006475169420474, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2783029576 }, "ece": { "estimate": -0.03289034381316446, "lower95": -0.07325433439488274, "upper95": -0.009049207636022077, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2783029576 } } } }, { "model": { "id": "mov-log1p", "label": "Map Elo with log1p MOV", "family": "margin-of-victory", "ratingScope": "map", "initialRating": 1000, "ratingScale": 1000, "predictionScale": 1000, "kFactor": 74, "marginFormula": "log1p", "marginScale": 0.75, "seasonCarry": 0, "coldStartPriorGames": 0, "rosterRegressionStrength": 0 }, "validation": { "metrics": { "n": 1184, "brier": 0.25096909316007054, "logLoss": 0.6955537747960054, "accuracy": 0.5447635135135135, "ece": 0.05704754665353165, "reliabilityBins": [ { "lower": 0, "upper": 0.1, "upperInclusive": false, "n": 1, "meanPredictedProbability": 0.09170983072616151, "observedTeamAWinRate": 0, "observedRateWilson95": { "lower95": 0, "upper95": 0.7934506856227626 } }, { "lower": 0.1, "upper": 0.2, "upperInclusive": false, "n": 4, "meanPredictedProbability": 0.18295940318185858, "observedTeamAWinRate": 0.75, "observedRateWilson95": { "lower95": 0.30064184258240184, "upper95": 0.9544127391902995 } }, { "lower": 0.2, "upper": 0.3, "upperInclusive": false, "n": 26, "meanPredictedProbability": 0.2649739542874381, "observedTeamAWinRate": 0.3076923076923077, "observedRateWilson95": { "lower95": 0.16501315595463767, "upper95": 0.4998825832219837 } }, { "lower": 0.3, "upper": 0.4, "upperInclusive": false, "n": 130, "meanPredictedProbability": 0.35836701879044514, "observedTeamAWinRate": 0.49230769230769234, "observedRateWilson95": { "lower95": 0.4078305354043078, "upper95": 0.5772264116909912 } }, { "lower": 0.4, "upper": 0.5, "upperInclusive": false, "n": 328, "meanPredictedProbability": 0.45385843339597604, "observedTeamAWinRate": 0.49390243902439024, "observedRateWilson95": { "lower95": 0.44018063428177345, "upper95": 0.5477654167806366 } }, { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 521, "meanPredictedProbability": 0.5337417379310662, "observedTeamAWinRate": 0.564299424184261, "observedRateWilson95": { "lower95": 0.5214049939071042, "upper95": 0.6062526041617438 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 136, "meanPredictedProbability": 0.6405222008566861, "observedTeamAWinRate": 0.5294117647058824, "observedRateWilson95": { "lower95": 0.44587284305424035, "upper95": 0.6113347981379019 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 35, "meanPredictedProbability": 0.7416568889132694, "observedTeamAWinRate": 0.6857142857142857, "observedRateWilson95": { "lower95": 0.5202023638793081, "upper95": 0.8144915532573418 } }, { "lower": 0.8, "upper": 0.9, "upperInclusive": false, "n": 3, "meanPredictedProbability": 0.8208468461647082, "observedTeamAWinRate": 1, "observedRateWilson95": { "lower95": 0.43850296824495455, "upper95": 1 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null } ], "accuracyByConfidenceBand": [ { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 849, "meanConfidence": 0.5385322488907004, "accuracy": 0.541813898704358, "accuracyWilson95": { "lower95": 0.5081855110826454, "upper95": 0.5750656011442139 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 266, "meanConfidence": 0.6410650634351561, "accuracy": 0.518796992481203, "accuracyWilson95": { "lower95": 0.4589136477606637, "upper95": 0.5781451498151562 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 61, "meanConfidence": 0.7388306278769022, "accuracy": 0.6885245901639344, "accuracyWilson95": { "lower95": 0.5640856563035855, "upper95": 0.7906256721938473 } }, { "lower": 0.8, "upper": 0.8999999999999999, "upperInclusive": false, "n": 7, "meanConfidence": 0.8186718465380985, "accuracy": 0.5714285714285714, "accuracyWilson95": { "lower95": 0.2504583645276572, "upper95": 0.8417801447485302 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 1, "meanConfidence": 0.9082901692738385, "accuracy": 1, "accuracyWilson95": { "lower95": 0.20654931437723745, "upper95": 1 } } ], "sampleSizeByMap": [ { "mapName": "Abyss", "n": 44 }, { "mapName": "Ascent", "n": 114 }, { "mapName": "Bind", "n": 97 }, { "mapName": "Corrode", "n": 57 }, { "mapName": "Fracture", "n": 88 }, { "mapName": "Haven", "n": 175 }, { "mapName": "Icebox", "n": 118 }, { "mapName": "Lotus", "n": 217 }, { "mapName": "Pearl", "n": 97 }, { "mapName": "Split", "n": 104 }, { "mapName": "Sunset", "n": 73 } ] }, "intervals95": { "brier": { "estimate": 0.25096909316007054, "lower95": 0.2456679823198144, "upper95": 0.25648568314800657, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3280605116 }, "logLoss": { "estimate": 0.6955537747960054, "lower95": 0.6843179876741319, "upper95": 0.7072747103394463, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3280605116 }, "accuracy": { "estimate": 0.5447635135135135, "lower95": 0.5163301411275755, "upper95": 0.5710858585858586, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3280605116 }, "ece": { "estimate": 0.05704754665353165, "lower95": 0.034425259184123626, "upper95": 0.08678986234516775, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3280605116 } }, "pairedDeltaVsPlainElo": { "brier": { "estimate": 0.007299295436495268, "lower95": -0.0013769589899255254, "upper95": 0.015478510281150423, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2124332177 }, "logLoss": { "estimate": 0.014015385557723636, "lower95": -0.004888435782803135, "upper95": 0.03214579782879256, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2124332177 }, "accuracy": { "estimate": -0.024493243243243312, "lower95": -0.060066966616460445, "upper95": 0.010131514613289174, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2124332177 }, "ece": { "estimate": -0.008476622244528179, "lower95": -0.045666691445154076, "upper95": 0.01581788535631039, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2124332177 } } } }, { "model": { "id": "mov-sqrt", "label": "Map Elo with square-root MOV", "family": "margin-of-victory", "ratingScope": "map", "initialRating": 1000, "ratingScale": 1000, "predictionScale": 1000, "kFactor": 74, "marginFormula": "sqrt", "marginScale": 0.75, "seasonCarry": 0, "coldStartPriorGames": 0, "rosterRegressionStrength": 0 }, "validation": { "metrics": { "n": 1184, "brier": 0.24923657730187385, "logLoss": 0.6917531129749324, "accuracy": 0.5481418918918919, "ece": 0.048132627428721014, "reliabilityBins": [ { "lower": 0, "upper": 0.1, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.1, "upper": 0.2, "upperInclusive": false, "n": 3, "meanPredictedProbability": 0.168513037994709, "observedTeamAWinRate": 0.3333333333333333, "observedRateWilson95": { "lower95": 0.06149194472039615, "upper95": 0.7923403991979523 } }, { "lower": 0.2, "upper": 0.3, "upperInclusive": false, "n": 17, "meanPredictedProbability": 0.26233369180319793, "observedTeamAWinRate": 0.29411764705882354, "observedRateWilson95": { "lower95": 0.1327998961029523, "upper95": 0.5313311006653985 } }, { "lower": 0.3, "upper": 0.4, "upperInclusive": false, "n": 113, "meanPredictedProbability": 0.361010786028957, "observedTeamAWinRate": 0.49557522123893805, "observedRateWilson95": { "lower95": 0.4050633172767508, "upper95": 0.5863780768132133 } }, { "lower": 0.4, "upper": 0.5, "upperInclusive": false, "n": 362, "meanPredictedProbability": 0.45585447398315887, "observedTeamAWinRate": 0.4861878453038674, "observedRateWilson95": { "lower95": 0.4351166475211748, "upper95": 0.537549107660768 } }, { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 549, "meanPredictedProbability": 0.5330806001057546, "observedTeamAWinRate": 0.5664845173041895, "observedRateWilson95": { "lower95": 0.524711015390169, "upper95": 0.6073340742739123 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 116, "meanPredictedProbability": 0.6402828978598835, "observedTeamAWinRate": 0.5517241379310345, "observedRateWilson95": { "lower95": 0.4610121324415494, "upper95": 0.6391201599895224 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 23, "meanPredictedProbability": 0.737917370806056, "observedTeamAWinRate": 0.6956521739130435, "observedRateWilson95": { "lower95": 0.49134243731240795, "upper95": 0.8439597554678597 } }, { "lower": 0.8, "upper": 0.9, "upperInclusive": false, "n": 1, "meanPredictedProbability": 0.8158850688066634, "observedTeamAWinRate": 1, "observedRateWilson95": { "lower95": 0.20654931437723745, "upper95": 1 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null } ], "accuracyByConfidenceBand": [ { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 911, "meanConfidence": 0.5374774202811806, "accuracy": 0.5455543358946213, "accuracyWilson95": { "lower95": 0.5130972249271306, "upper95": 0.5776288776119847 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 229, "meanConfidence": 0.6396445298273988, "accuracy": 0.5283842794759825, "accuracyWilson95": { "lower95": 0.46379524002504624, "upper95": 0.5920367413834322 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 40, "meanConfidence": 0.7378106691971232, "accuracy": 0.7, "accuracyWilson95": { "lower95": 0.5456998118185507, "upper95": 0.8192515477025347 } }, { "lower": 0.8, "upper": 0.8999999999999999, "upperInclusive": false, "n": 4, "meanConfidence": 0.8275864887056341, "accuracy": 0.75, "accuracyWilson95": { "lower95": 0.30064184258240184, "upper95": 0.9544127391902995 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null } ], "sampleSizeByMap": [ { "mapName": "Abyss", "n": 44 }, { "mapName": "Ascent", "n": 114 }, { "mapName": "Bind", "n": 97 }, { "mapName": "Corrode", "n": 57 }, { "mapName": "Fracture", "n": 88 }, { "mapName": "Haven", "n": 175 }, { "mapName": "Icebox", "n": 118 }, { "mapName": "Lotus", "n": 217 }, { "mapName": "Pearl", "n": 97 }, { "mapName": "Split", "n": 104 }, { "mapName": "Sunset", "n": 73 } ] }, "intervals95": { "brier": { "estimate": 0.24923657730187385, "lower95": 0.24453828565949504, "upper95": 0.25414743478360263, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 31832929 }, "logLoss": { "estimate": 0.6917531129749324, "lower95": 0.6818899871171163, "upper95": 0.7022538115652873, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 31832929 }, "accuracy": { "estimate": 0.5481418918918919, "lower95": 0.5183568379967269, "upper95": 0.5747612473781087, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 31832929 }, "ece": { "estimate": 0.048132627428721014, "lower95": 0.028338497394360652, "upper95": 0.07819781811561814, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 31832929 } }, "pairedDeltaVsPlainElo": { "brier": { "estimate": 0.00556677957829857, "lower95": -0.001973959575914984, "upper95": 0.013490329772374944, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 822614820 }, "logLoss": { "estimate": 0.010214723736650555, "lower95": -0.006271554575941831, "upper95": 0.02821955304148867, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 822614820 }, "accuracy": { "estimate": -0.021114864864864913, "lower95": -0.0572281310112224, "upper95": 0.015593043046592432, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 822614820 }, "ece": { "estimate": -0.017391541469338814, "lower95": -0.053214247052549714, "upper95": 0.008131011856535057, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 822614820 } } } }, { "model": { "id": "mov-linear-capped", "label": "Map Elo with capped-linear MOV", "family": "margin-of-victory", "ratingScope": "map", "initialRating": 1000, "ratingScale": 1000, "predictionScale": 1000, "kFactor": 74, "marginFormula": "linear-capped", "marginScale": 0.25, "seasonCarry": 0, "coldStartPriorGames": 0, "rosterRegressionStrength": 0 }, "validation": { "metrics": { "n": 1184, "brier": 0.24849042123281165, "logLoss": 0.6901045784321616, "accuracy": 0.5396959459459459, "ece": 0.030151214873430445, "reliabilityBins": [ { "lower": 0, "upper": 0.1, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.1, "upper": 0.2, "upperInclusive": false, "n": 1, "meanPredictedProbability": 0.18291072000924302, "observedTeamAWinRate": 0, "observedRateWilson95": { "lower95": 0, "upper95": 0.7934506856227626 } }, { "lower": 0.2, "upper": 0.3, "upperInclusive": false, "n": 5, "meanPredictedProbability": 0.27729437499187254, "observedTeamAWinRate": 0.6, "observedRateWilson95": { "lower95": 0.2307242812760128, "upper95": 0.882379225767352 } }, { "lower": 0.3, "upper": 0.4, "upperInclusive": false, "n": 67, "meanPredictedProbability": 0.3704703061701376, "observedTeamAWinRate": 0.40298507462686567, "observedRateWilson95": { "lower95": 0.2939049277975166, "upper95": 0.5225867111888554 } }, { "lower": 0.4, "upper": 0.5, "upperInclusive": false, "n": 420, "meanPredictedProbability": 0.462503875110244, "observedTeamAWinRate": 0.5047619047619047, "observedRateWilson95": { "lower95": 0.45711985182106624, "upper95": 0.552317639289713 } }, { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 618, "meanPredictedProbability": 0.529573133863355, "observedTeamAWinRate": 0.551779935275081, "observedRateWilson95": { "lower95": 0.5123713468314757, "upper95": 0.5905487770878499 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 66, "meanPredictedProbability": 0.6327846073963035, "observedTeamAWinRate": 0.6363636363636364, "observedRateWilson95": { "lower95": 0.5157964063076522, "upper95": 0.7419301690522909 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 7, "meanPredictedProbability": 0.7120402227853029, "observedTeamAWinRate": 0.7142857142857143, "observedRateWilson95": { "lower95": 0.35893445183261935, "upper95": 0.9177810759959433 } }, { "lower": 0.8, "upper": 0.9, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null } ], "accuracyByConfidenceBand": [ { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 1038, "meanConfidence": 0.5327789683827084, "accuracy": 0.5289017341040463, "accuracyWilson95": { "lower95": 0.4984846608223204, "upper95": 0.5591056754923016 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 133, "meanConfidence": 0.6311449140959156, "accuracy": 0.6165413533834586, "accuracyWilson95": { "lower95": 0.5317374963621947, "upper95": 0.6948020353496082 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 12, "meanConfidence": 0.7164841403781465, "accuracy": 0.5833333333333334, "accuracyWilson95": { "lower95": 0.31951131254954973, "upper95": 0.8067396863412435 } }, { "lower": 0.8, "upper": 0.8999999999999999, "upperInclusive": false, "n": 1, "meanConfidence": 0.817089279990757, "accuracy": 1, "accuracyWilson95": { "lower95": 0.20654931437723745, "upper95": 1 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null } ], "sampleSizeByMap": [ { "mapName": "Abyss", "n": 44 }, { "mapName": "Ascent", "n": 114 }, { "mapName": "Bind", "n": 97 }, { "mapName": "Corrode", "n": 57 }, { "mapName": "Fracture", "n": 88 }, { "mapName": "Haven", "n": 175 }, { "mapName": "Icebox", "n": 118 }, { "mapName": "Lotus", "n": 217 }, { "mapName": "Pearl", "n": 97 }, { "mapName": "Split", "n": 104 }, { "mapName": "Sunset", "n": 73 } ] }, "intervals95": { "brier": { "estimate": 0.24849042123281165, "lower95": 0.2451297030281436, "upper95": 0.2520847779565373, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 254908922 }, "logLoss": { "estimate": 0.6901045784321616, "lower95": 0.6831947727234706, "upper95": 0.697558443665057, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 254908922 }, "accuracy": { "estimate": 0.5396959459459459, "lower95": 0.5113233272735335, "upper95": 0.565090770035825, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 254908922 }, "ece": { "estimate": 0.030151214873430445, "lower95": 0.013193396934504154, "upper95": 0.06362607126794957, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 254908922 } }, "pairedDeltaVsPlainElo": { "brier": { "estimate": 0.004820623509236371, "lower95": -0.003091063529654264, "upper95": 0.013649973869103556, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2145373515 }, "logLoss": { "estimate": 0.008566189193879747, "lower95": -0.009105659956315192, "upper95": 0.027775296049737108, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2145373515 }, "accuracy": { "estimate": -0.029560810810810856, "lower95": -0.06477648908687691, "upper95": 0.0063603322949116955, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2145373515 }, "ece": { "estimate": -0.03537295402462938, "lower95": -0.06811406606176282, "upper95": -0.007198233566748783, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2145373515 } } } } ], "bestOnValidationModelId": "mov-linear-capped" } ], "selection": { "eligibleCandidateIds": [ "season-carry-25", "season-carry-50", "season-carry-75", "season-carry-100", "roster-regression-25", "roster-regression-50", "roster-regression-75", "cold-start-prior-3", "cold-start-prior-8", "cold-start-prior-15", "mov-none", "mov-log1p", "mov-sqrt", "mov-linear-capped" ], "selectedChallengerId": "cold-start-prior-8", "rule": "Choose the lowest validation log loss among experimental map-Elo candidates only; fixed references are not eligible and can score better. Ties break lexically by model id. Test labels are inaccessible to this choice within the execution.", "candidateTestMetricsPublished": false, "testEvaluationPassesWithinRun": 1 }, "finalModels": [ { "role": "plain-elo-baseline", "model": { "id": "plain-overall-elo", "label": "Plain overall-team Elo", "family": "plain-elo", "ratingScope": "overall", "initialRating": 1000, "ratingScale": 400, "predictionScale": 400, "kFactor": 32, "marginFormula": "none", "marginScale": 0, "seasonCarry": null, "coldStartPriorGames": 0, "rosterRegressionStrength": 0 }, "selectionRule": "Comparator fixed in code for this run; no parameter search inside the runner.", "provenance": { "basis": "runner-fixed-reference", "repositoryCommit": null, "repositoryCommitAt": null, "sourceFiles": [ "lib/elo/backtest.ts" ], "sourceBlobs": [], "note": "A conventional overall-team Elo reference fixed by the backtest implementation." }, "validation": { "metrics": { "n": 1184, "brier": 0.24366979772357528, "logLoss": 0.6815383892382818, "accuracy": 0.5692567567567568, "ece": 0.06552416889805983, "reliabilityBins": [ { "lower": 0, "upper": 0.1, "upperInclusive": false, "n": 2, "meanPredictedProbability": 0.08667239222905254, "observedTeamAWinRate": 0, "observedRateWilson95": { "lower95": 0, "upper95": 0.6576197724933469 } }, { "lower": 0.1, "upper": 0.2, "upperInclusive": false, "n": 21, "meanPredictedProbability": 0.1555326854295948, "observedTeamAWinRate": 0.23809523809523808, "observedRateWilson95": { "lower95": 0.10628013022636468, "upper95": 0.4509117368295075 } }, { "lower": 0.2, "upper": 0.3, "upperInclusive": false, "n": 91, "meanPredictedProbability": 0.2556841307050949, "observedTeamAWinRate": 0.34065934065934067, "observedRateWilson95": { "lower95": 0.25151369352131475, "upper95": 0.44271285718129555 } }, { "lower": 0.3, "upper": 0.4, "upperInclusive": false, "n": 141, "meanPredictedProbability": 0.35651622225080826, "observedTeamAWinRate": 0.3900709219858156, "observedRateWilson95": { "lower95": 0.31349773454130114, "upper95": 0.4724751474470229 } }, { "lower": 0.4, "upper": 0.5, "upperInclusive": false, "n": 235, "meanPredictedProbability": 0.45033734087491056, "observedTeamAWinRate": 0.5617021276595745, "observedRateWilson95": { "lower95": 0.4977757837094084, "upper95": 0.6236436722895238 } }, { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 305, "meanPredictedProbability": 0.5498471075086752, "observedTeamAWinRate": 0.5573770491803278, "observedRateWilson95": { "lower95": 0.5012636227354222, "upper95": 0.6120631311714103 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 225, "meanPredictedProbability": 0.6503249128824475, "observedTeamAWinRate": 0.5333333333333333, "observedRateWilson95": { "lower95": 0.46813401735317156, "upper95": 0.5974135459131205 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 119, "meanPredictedProbability": 0.7366903837922818, "observedTeamAWinRate": 0.7226890756302521, "observedRateWilson95": { "lower95": 0.6362542159994129, "upper95": 0.7951962121281266 } }, { "lower": 0.8, "upper": 0.9, "upperInclusive": false, "n": 43, "meanPredictedProbability": 0.8354619713348143, "observedTeamAWinRate": 0.7209302325581395, "observedRateWilson95": { "lower95": 0.5730897131812114, "upper95": 0.832533862137625 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 2, "meanPredictedProbability": 0.9140089374572402, "observedTeamAWinRate": 0, "observedRateWilson95": { "lower95": 0, "upper95": 0.6576197724933469 } } ], "accuracyByConfidenceBand": [ { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 540, "meanConfidence": 0.5497668383047077, "accuracy": 0.5055555555555555, "accuracyWilson95": { "lower95": 0.4634963841063396, "upper95": 0.5475362429629737 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 366, "meanConfidence": 0.6476893936097994, "accuracy": 0.5628415300546448, "accuracyWilson95": { "lower95": 0.5116309117083634, "upper95": 0.6127467071412896 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 210, "meanConfidence": 0.7399947608434188, "accuracy": 0.6952380952380952, "accuracyWilson95": { "lower95": 0.6299363181289208, "upper95": 0.7535253382412637 } }, { "lower": 0.8, "upper": 0.8999999999999999, "upperInclusive": false, "n": 64, "meanConfidence": 0.8384168495839927, "accuracy": 0.734375, "accuracyWilson95": { "lower95": 0.6151712627143486, "upper95": 0.8270362092577739 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 4, "meanConfidence": 0.9136682726140938, "accuracy": 0.5, "accuracyWilson95": { "lower95": 0.15003898915214953, "upper95": 0.8499610108478505 } } ], "sampleSizeByMap": [ { "mapName": "Abyss", "n": 44 }, { "mapName": "Ascent", "n": 114 }, { "mapName": "Bind", "n": 97 }, { "mapName": "Corrode", "n": 57 }, { "mapName": "Fracture", "n": 88 }, { "mapName": "Haven", "n": 175 }, { "mapName": "Icebox", "n": 118 }, { "mapName": "Lotus", "n": 217 }, { "mapName": "Pearl", "n": 97 }, { "mapName": "Split", "n": 104 }, { "mapName": "Sunset", "n": 73 } ] }, "intervals95": { "brier": { "estimate": 0.24366979772357528, "lower95": 0.23439414530414887, "upper95": 0.25293728937932775, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1014119777 }, "logLoss": { "estimate": 0.6815383892382818, "lower95": 0.6611554000347355, "upper95": 0.7018010621574586, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1014119777 }, "accuracy": { "estimate": 0.5692567567567568, "lower95": 0.5401411379789338, "upper95": 0.5968193384223919, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1014119777 }, "ece": { "estimate": 0.06552416889805983, "lower95": 0.04817507701022898, "upper95": 0.09980090355550143, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1014119777 } }, "pairedDeltaVsPlainElo": null }, "test": { "metrics": { "n": 835, "brier": 0.25125727292733124, "logLoss": 0.6992107340148396, "accuracy": 0.5760479041916168, "ece": 0.05781128922716304, "reliabilityBins": [ { "lower": 0, "upper": 0.1, "upperInclusive": false, "n": 3, "meanPredictedProbability": 0.08391698227405092, "observedTeamAWinRate": 0.3333333333333333, "observedRateWilson95": { "lower95": 0.06149194472039615, "upper95": 0.7923403991979523 } }, { "lower": 0.1, "upper": 0.2, "upperInclusive": false, "n": 18, "meanPredictedProbability": 0.15334382217485346, "observedTeamAWinRate": 0.5555555555555556, "observedRateWilson95": { "lower95": 0.33716415642042774, "upper95": 0.7544048187299437 } }, { "lower": 0.2, "upper": 0.3, "upperInclusive": false, "n": 55, "meanPredictedProbability": 0.2617431856498862, "observedTeamAWinRate": 0.32727272727272727, "observedRateWilson95": { "lower95": 0.21813065595168937, "upper95": 0.45896776443421317 } }, { "lower": 0.3, "upper": 0.4, "upperInclusive": false, "n": 131, "meanPredictedProbability": 0.3562700710204479, "observedTeamAWinRate": 0.4351145038167939, "observedRateWilson95": { "lower95": 0.3532632308838599, "upper95": 0.520662784536918 } }, { "lower": 0.4, "upper": 0.5, "upperInclusive": false, "n": 190, "meanPredictedProbability": 0.4477146905757832, "observedTeamAWinRate": 0.4105263157894737, "observedRateWilson95": { "lower95": 0.34302554030191634, "upper95": 0.4815733860224545 } }, { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 254, "meanPredictedProbability": 0.5532292683072572, "observedTeamAWinRate": 0.5433070866141733, "observedRateWilson95": { "lower95": 0.4818580371511403, "upper95": 0.6034657121815807 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 124, "meanPredictedProbability": 0.643616033967825, "observedTeamAWinRate": 0.6048387096774194, "observedRateWilson95": { "lower95": 0.5168839177124075, "upper95": 0.6864929855226117 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 54, "meanPredictedProbability": 0.7422550402752164, "observedTeamAWinRate": 0.5370370370370371, "observedRateWilson95": { "lower95": 0.40605356021011496, "upper95": 0.6631009888201891 } }, { "lower": 0.8, "upper": 0.9, "upperInclusive": false, "n": 6, "meanPredictedProbability": 0.8540941635401725, "observedTeamAWinRate": 1, "observedRateWilson95": { "lower95": 0.6096657120978346, "upper95": 1 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null } ], "accuracyByConfidenceBand": [ { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 444, "meanConfidence": 0.5528253219383883, "accuracy": 0.5630630630630631, "accuracyWilson95": { "lower95": 0.516580754944525, "upper95": 0.6084634965355978 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 255, "meanConfidence": 0.6436745447385555, "accuracy": 0.5843137254901961, "accuracyWilson95": { "lower95": 0.523009891197656, "upper95": 0.6431149645187042 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 109, "meanConfidence": 0.7402375868267697, "accuracy": 0.6055045871559633, "accuracyWilson95": { "lower95": 0.5116648839074484, "upper95": 0.6921609106220886 } }, { "lower": 0.8, "upper": 0.8999999999999999, "upperInclusive": false, "n": 24, "meanConfidence": 0.8485156742539028, "accuracy": 0.5833333333333334, "accuracyWilson95": { "lower95": 0.3883466586522876, "upper95": 0.7553239739018824 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 3, "meanConfidence": 0.9160830177259491, "accuracy": 0.6666666666666666, "accuracyWilson95": { "lower95": 0.20765960080204765, "upper95": 0.9385080552796037 } } ], "sampleSizeByMap": [ { "mapName": "Abyss", "n": 72 }, { "mapName": "Ascent", "n": 36 }, { "mapName": "Bind", "n": 80 }, { "mapName": "Breeze", "n": 98 }, { "mapName": "Corrode", "n": 57 }, { "mapName": "Fracture", "n": 45 }, { "mapName": "Haven", "n": 124 }, { "mapName": "Lotus", "n": 97 }, { "mapName": "Pearl", "n": 90 }, { "mapName": "Split", "n": 125 }, { "mapName": "Sunset", "n": 11 } ] }, "intervals95": { "brier": { "estimate": 0.25125727292733124, "lower95": 0.24017463067133435, "upper95": 0.2602788816689738, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1143947186 }, "logLoss": { "estimate": 0.6992107340148396, "lower95": 0.6751938689485715, "upper95": 0.7191565506361657, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1143947186 }, "accuracy": { "estimate": 0.5760479041916168, "lower95": 0.5466073743613208, "upper95": 0.6144242120740725, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1143947186 }, "ece": { "estimate": 0.05781128922716304, "lower95": 0.03741674249707493, "upper95": 0.09577249174408534, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1143947186 } }, "pairedDeltaVsPlainElo": null } }, { "role": "pre-holdout-frozen-production", "model": { "id": "production-map-elo-pre-holdout-frozen", "label": "Pre-holdout frozen production map Elo", "family": "production-map-elo", "ratingScope": "map", "initialRating": 1000, "ratingScale": 2000, "predictionScale": 1000, "kFactor": 74, "marginFormula": "production", "marginScale": 1, "seasonCarry": 0, "coldStartPriorGames": 0, "rosterRegressionStrength": 0 }, "selectionRule": "Externally frozen by repository history before the chronological test start; not selected or tuned by this backtest run.", "provenance": { "basis": "pre-holdout-repository-snapshot", "repositoryCommit": "2f134f187c0717dbfcf18b1a07c4eccbef94dcdb", "repositoryCommitAt": "2025-08-26T15:49:51-05:00", "sourceFiles": [ "lib/elo/elo-calculator.ts", "lib/predictions/calculations.ts", "db/elo/elo-processor.ts" ], "sourceBlobs": [ { "path": "lib/elo/elo-calculator.ts", "gitBlob": "cdf7c5361a88a1099049058181e986fde4cc8f29" }, { "path": "lib/predictions/calculations.ts", "gitBlob": "11343604323dcf6ea414d2215e75be7327d9e533" }, { "path": "db/elo/elo-processor.ts", "gitBlob": "0c3d10548e2ef966dd831dcafaa9636378562d14" } ], "note": "The historical update path used ratingScale=2000, K=74, production MOV scale=1, and annual reset; the historical forecast path separately used a 1000 divisor." }, "validation": { "metrics": { "n": 1184, "brier": 0.2543164812947671, "logLoss": 0.7035810472874201, "accuracy": 0.5380067567567568, "ece": 0.06436832526009967, "reliabilityBins": [ { "lower": 0, "upper": 0.1, "upperInclusive": false, "n": 1, "meanPredictedProbability": 0.0509152448931422, "observedTeamAWinRate": 0, "observedRateWilson95": { "lower95": 0, "upper95": 0.7934506856227626 } }, { "lower": 0.1, "upper": 0.2, "upperInclusive": false, "n": 10, "meanPredictedProbability": 0.15980713895867518, "observedTeamAWinRate": 0.4, "observedRateWilson95": { "lower95": 0.16818032970623614, "upper95": 0.6873262302663417 } }, { "lower": 0.2, "upper": 0.3, "upperInclusive": false, "n": 53, "meanPredictedProbability": 0.26115167488836993, "observedTeamAWinRate": 0.4716981132075472, "observedRateWilson95": { "lower95": 0.3438224347819615, "upper95": 0.603399187212937 } }, { "lower": 0.3, "upper": 0.4, "upperInclusive": false, "n": 143, "meanPredictedProbability": 0.3625606961619098, "observedTeamAWinRate": 0.5174825174825175, "observedRateWilson95": { "lower95": 0.436202145886806, "upper95": 0.5978481832243031 } }, { "lower": 0.4, "upper": 0.5, "upperInclusive": false, "n": 282, "meanPredictedProbability": 0.45552239379063497, "observedTeamAWinRate": 0.48936170212765956, "observedRateWilson95": { "lower95": 0.4315539985510224, "upper95": 0.5474553445136455 } }, { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 463, "meanPredictedProbability": 0.5312372090616999, "observedTeamAWinRate": 0.550755939524838, "observedRateWilson95": { "lower95": 0.5052147911039679, "upper95": 0.595461785733763 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 168, "meanPredictedProbability": 0.6370403923345244, "observedTeamAWinRate": 0.5595238095238095, "observedRateWilson95": { "lower95": 0.4839554939660739, "upper95": 0.6324308550398943 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 49, "meanPredictedProbability": 0.7396280186168448, "observedTeamAWinRate": 0.6122448979591837, "observedRateWilson95": { "lower95": 0.4724599944352045, "upper95": 0.7357098828826433 } }, { "lower": 0.8, "upper": 0.9, "upperInclusive": false, "n": 15, "meanPredictedProbability": 0.8401370395325969, "observedTeamAWinRate": 0.6666666666666666, "observedRateWilson95": { "lower95": 0.41713547738519774, "upper95": 0.8482367556028526 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null } ], "accuracyByConfidenceBand": [ { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 745, "meanConfidence": 0.536249010398131, "accuracy": 0.5355704697986577, "accuracyWilson95": { "lower95": 0.49966671556019293, "upper95": 0.5711092803141095 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 311, "meanConfidence": 0.6372238146657452, "accuracy": 0.5241157556270096, "accuracyWilson95": { "lower95": 0.468655375295927, "upper95": 0.5789876513124037 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 102, "meanConfidence": 0.7392228837562916, "accuracy": 0.5686274509803921, "accuracyWilson95": { "lower95": 0.4717497773240239, "upper95": 0.6605235317931281 } }, { "lower": 0.8, "upper": 0.8999999999999999, "upperInclusive": false, "n": 25, "meanConfidence": 0.840159368136088, "accuracy": 0.64, "accuracyWilson95": { "lower95": 0.44518497748164865, "upper95": 0.7975211922554178 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 1, "meanConfidence": 0.9490847551068577, "accuracy": 1, "accuracyWilson95": { "lower95": 0.20654931437723745, "upper95": 1 } } ], "sampleSizeByMap": [ { "mapName": "Abyss", "n": 44 }, { "mapName": "Ascent", "n": 114 }, { "mapName": "Bind", "n": 97 }, { "mapName": "Corrode", "n": 57 }, { "mapName": "Fracture", "n": 88 }, { "mapName": "Haven", "n": 175 }, { "mapName": "Icebox", "n": 118 }, { "mapName": "Lotus", "n": 217 }, { "mapName": "Pearl", "n": 97 }, { "mapName": "Split", "n": 104 }, { "mapName": "Sunset", "n": 73 } ] }, "intervals95": { "brier": { "estimate": 0.2543164812947671, "lower95": 0.2479161752989521, "upper95": 0.26151721863102195, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 705265939 }, "logLoss": { "estimate": 0.7035810472874201, "lower95": 0.6895219234832212, "upper95": 0.7191912613504238, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 705265939 }, "accuracy": { "estimate": 0.5380067567567568, "lower95": 0.508852672102736, "upper95": 0.5653247179751331, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 705265939 }, "ece": { "estimate": 0.06436832526009967, "lower95": 0.043537880348870545, "upper95": 0.09422512950562892, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 705265939 } }, "pairedDeltaVsPlainElo": { "brier": { "estimate": 0.010646683571191806, "lower95": 0.0021994987668685025, "upper95": 0.02014409606836845, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1036059314 }, "logLoss": { "estimate": 0.022042658049138297, "lower95": 0.003712843138362002, "upper95": 0.042699613931836244, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1036059314 }, "accuracy": { "estimate": -0.03125, "lower95": -0.07014280491966905, "upper95": 0.0063567927170867945, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1036059314 }, "ece": { "estimate": -0.0011558436379601605, "lower95": -0.037485014452214055, "upper95": 0.023178825083504988, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1036059314 } } }, "test": { "metrics": { "n": 835, "brier": 0.25662970325551515, "logLoss": 0.7083957798981126, "accuracy": 0.5437125748502994, "ece": 0.05123947987132716, "reliabilityBins": [ { "lower": 0, "upper": 0.1, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.1, "upper": 0.2, "upperInclusive": false, "n": 5, "meanPredictedProbability": 0.15707908440094207, "observedTeamAWinRate": 0.4, "observedRateWilson95": { "lower95": 0.11762077423264783, "upper95": 0.769275718723987 } }, { "lower": 0.2, "upper": 0.3, "upperInclusive": false, "n": 32, "meanPredictedProbability": 0.2687702991197859, "observedTeamAWinRate": 0.34375, "observedRateWilson95": { "lower95": 0.20410437924433175, "upper95": 0.5168891202103777 } }, { "lower": 0.3, "upper": 0.4, "upperInclusive": false, "n": 114, "meanPredictedProbability": 0.3574569210782965, "observedTeamAWinRate": 0.47368421052631576, "observedRateWilson95": { "lower95": 0.38438778617578623, "upper95": 0.5646963472009949 } }, { "lower": 0.4, "upper": 0.5, "upperInclusive": false, "n": 207, "meanPredictedProbability": 0.45295291603858334, "observedTeamAWinRate": 0.4396135265700483, "observedRateWilson95": { "lower95": 0.3737086281366463, "upper95": 0.5077188665324133 } }, { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 341, "meanPredictedProbability": 0.5295790763027952, "observedTeamAWinRate": 0.5395894428152492, "observedRateWilson95": { "lower95": 0.48653967620516847, "upper95": 0.5917571738671398 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 112, "meanPredictedProbability": 0.6382682969453091, "observedTeamAWinRate": 0.5267857142857143, "observedRateWilson95": { "lower95": 0.4349725864921189, "upper95": 0.6168223413435806 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 18, "meanPredictedProbability": 0.7362829120122424, "observedTeamAWinRate": 0.3888888888888889, "observedRateWilson95": { "lower95": 0.20305246572163624, "upper95": 0.613809583977621 } }, { "lower": 0.8, "upper": 0.9, "upperInclusive": false, "n": 6, "meanPredictedProbability": 0.8345282357488825, "observedTeamAWinRate": 0.6666666666666666, "observedRateWilson95": { "lower95": 0.299993315138392, "upper95": 0.9032285888942195 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null } ], "accuracyByConfidenceBand": [ { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 548, "meanConfidence": 0.5361773930643545, "accuracy": 0.5474452554744526, "accuracyWilson95": { "lower95": 0.5055851112689724, "upper95": 0.5886448512842688 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 226, "meanConfidence": 0.6404246028980035, "accuracy": 0.5265486725663717, "accuracyWilson95": { "lower95": 0.4615542299064052, "upper95": 0.5906556719171014 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 50, "meanConfidence": 0.7330488568877443, "accuracy": 0.56, "accuracyWilson95": { "lower95": 0.4230602802165686, "upper95": 0.6883780078874284 } }, { "lower": 0.8, "upper": 0.8999999999999999, "upperInclusive": false, "n": 11, "meanConfidence": 0.838343090226235, "accuracy": 0.6363636363636364, "accuracyWilson95": { "lower95": 0.35380117450784887, "upper95": 0.8483352890463243 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null } ], "sampleSizeByMap": [ { "mapName": "Abyss", "n": 72 }, { "mapName": "Ascent", "n": 36 }, { "mapName": "Bind", "n": 80 }, { "mapName": "Breeze", "n": 98 }, { "mapName": "Corrode", "n": 57 }, { "mapName": "Fracture", "n": 45 }, { "mapName": "Haven", "n": 124 }, { "mapName": "Lotus", "n": 97 }, { "mapName": "Pearl", "n": 90 }, { "mapName": "Split", "n": 125 }, { "mapName": "Sunset", "n": 11 } ] }, "intervals95": { "brier": { "estimate": 0.25662970325551515, "lower95": 0.24865561771470834, "upper95": 0.2645117547508362, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 20172004 }, "logLoss": { "estimate": 0.7083957798981126, "lower95": 0.6912495206636196, "upper95": 0.7253873479041488, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 20172004 }, "accuracy": { "estimate": 0.5437125748502994, "lower95": 0.5087270753211417, "upper95": 0.5773642513627105, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 20172004 }, "ece": { "estimate": 0.05123947987132716, "lower95": 0.03679971450544202, "upper95": 0.09193673080538192, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 20172004 } }, "pairedDeltaVsPlainElo": { "brier": { "estimate": 0.005372430328183908, "lower95": -0.005170617291777188, "upper95": 0.015983251374596587, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 4021683241 }, "logLoss": { "estimate": 0.009185045883273069, "lower95": -0.01328305774219916, "upper95": 0.032516671942085004, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 4021683241 }, "accuracy": { "estimate": -0.03233532934131733, "lower95": -0.08116114975755975, "upper95": 0.015024429791373861, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 4021683241 }, "ece": { "estimate": -0.0065718093558358795, "lower95": -0.03954446122800112, "upper95": 0.0324925567641602, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 4021683241 } } } }, { "role": "fixed-production", "model": { "id": "production-map-elo-fixed", "label": "Production map Elo at analysis time", "family": "production-map-elo", "ratingScope": "map", "initialRating": 1000, "ratingScale": 1000, "predictionScale": 1000, "kFactor": 74, "marginFormula": "production", "marginScale": 1, "seasonCarry": 0, "coldStartPriorGames": 0, "rosterRegressionStrength": 0 }, "selectionRule": "Configuration deployed at analysis time; evaluated retrospectively and not frozen before the test period.", "provenance": { "basis": "analysis-time-deployment", "repositoryCommit": "99a0a87a240f47d1aa0c894ffb730fe9ea4ddd23", "repositoryCommitAt": "2025-10-04T02:22:24.000Z", "sourceFiles": [ "lib/elo/elo-calculator.ts" ], "sourceBlobs": [], "note": "The analysis-time implementation uses a 1000 divisor for both rating updates and forecasts." }, "validation": { "metrics": { "n": 1184, "brier": 0.2527435899371852, "logLoss": 0.6995864055574936, "accuracy": 0.5422297297297297, "ece": 0.05497824282118526, "reliabilityBins": [ { "lower": 0, "upper": 0.1, "upperInclusive": false, "n": 1, "meanPredictedProbability": 0.07364199865358073, "observedTeamAWinRate": 0, "observedRateWilson95": { "lower95": 0, "upper95": 0.7934506856227626 } }, { "lower": 0.1, "upper": 0.2, "upperInclusive": false, "n": 5, "meanPredictedProbability": 0.16956858811833425, "observedTeamAWinRate": 0.6, "observedRateWilson95": { "lower95": 0.2307242812760128, "upper95": 0.882379225767352 } }, { "lower": 0.2, "upper": 0.3, "upperInclusive": false, "n": 45, "meanPredictedProbability": 0.268365000047894, "observedTeamAWinRate": 0.4888888888888889, "observedRateWilson95": { "lower95": 0.349570237528785, "upper95": 0.6299553535541681 } }, { "lower": 0.3, "upper": 0.4, "upperInclusive": false, "n": 138, "meanPredictedProbability": 0.36131975781129533, "observedTeamAWinRate": 0.4782608695652174, "observedRateWilson95": { "lower95": 0.39664111777602556, "upper95": 0.5610581328529124 } }, { "lower": 0.4, "upper": 0.5, "upperInclusive": false, "n": 293, "meanPredictedProbability": 0.45323000076564246, "observedTeamAWinRate": 0.49146757679180886, "observedRateWilson95": { "lower95": 0.4347066650540538, "upper95": 0.5484493266330095 } }, { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 495, "meanPredictedProbability": 0.5328991145269272, "observedTeamAWinRate": 0.5515151515151515, "observedRateWilson95": { "lower95": 0.5074731567000941, "upper95": 0.59476373459713 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 154, "meanPredictedProbability": 0.6394113611197857, "observedTeamAWinRate": 0.564935064935065, "observedRateWilson95": { "lower95": 0.4859968415353618, "upper95": 0.6407125803708292 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 45, "meanPredictedProbability": 0.740497684952403, "observedTeamAWinRate": 0.6444444444444445, "observedRateWilson95": { "lower95": 0.4983581419637687, "upper95": 0.7678091739578413 } }, { "lower": 0.8, "upper": 0.9, "upperInclusive": false, "n": 8, "meanPredictedProbability": 0.8245873127740609, "observedTeamAWinRate": 0.75, "observedRateWilson95": { "lower95": 0.40927543031016883, "upper95": 0.9285207872478909 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null } ], "accuracyByConfidenceBand": [ { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 788, "meanConfidence": 0.5380566896782942, "accuracy": 0.5355329949238579, "accuracyWilson95": { "lower95": 0.5006226089940932, "upper95": 0.570098618565022 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 292, "meanConfidence": 0.6390658323098911, "accuracy": 0.5445205479452054, "accuracyWilson95": { "lower95": 0.48719026348684424, "upper95": 0.6006946465715471 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 90, "meanConfidence": 0.7360663424522546, "accuracy": 0.5777777777777777, "accuracyWilson95": { "lower95": 0.4746118562531433, "upper95": 0.6745759353257391 } }, { "lower": 0.8, "upper": 0.8999999999999999, "upperInclusive": false, "n": 13, "meanConfidence": 0.8268350432000628, "accuracy": 0.6153846153846154, "accuracyWilson95": { "lower95": 0.35522891512665694, "upper95": 0.8229029220223743 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 1, "meanConfidence": 0.9263580013464192, "accuracy": 1, "accuracyWilson95": { "lower95": 0.20654931437723745, "upper95": 1 } } ], "sampleSizeByMap": [ { "mapName": "Abyss", "n": 44 }, { "mapName": "Ascent", "n": 114 }, { "mapName": "Bind", "n": 97 }, { "mapName": "Corrode", "n": 57 }, { "mapName": "Fracture", "n": 88 }, { "mapName": "Haven", "n": 175 }, { "mapName": "Icebox", "n": 118 }, { "mapName": "Lotus", "n": 217 }, { "mapName": "Pearl", "n": 97 }, { "mapName": "Split", "n": 104 }, { "mapName": "Sunset", "n": 73 } ] }, "intervals95": { "brier": { "estimate": 0.2527435899371852, "lower95": 0.24689443218997487, "upper95": 0.2590576455324173, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1852137397 }, "logLoss": { "estimate": 0.6995864055574936, "lower95": 0.6872303732005904, "upper95": 0.713139481662413, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1852137397 }, "accuracy": { "estimate": 0.5422297297297297, "lower95": 0.5141864334495535, "upper95": 0.570590510093212, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1852137397 }, "ece": { "estimate": 0.05497824282118526, "lower95": 0.036881934485191975, "upper95": 0.08313701497837339, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1852137397 } }, "pairedDeltaVsPlainElo": { "brier": { "estimate": 0.009073792213609932, "lower95": 0.001056562413957099, "upper95": 0.017550273426054037, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2252062840 }, "logLoss": { "estimate": 0.018048016319211824, "lower95": -0.00040205441518620085, "upper95": 0.03655445037667566, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2252062840 }, "accuracy": { "estimate": -0.027027027027027084, "lower95": -0.06415950394740731, "upper95": 0.010987081994767708, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2252062840 }, "ece": { "estimate": -0.010545926076874566, "lower95": -0.04475294332413793, "upper95": 0.01628919379247422, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2252062840 } } }, "test": { "metrics": { "n": 835, "brier": 0.25544817411250825, "logLoss": 0.705281828557604, "accuracy": 0.5425149700598803, "ece": 0.05031042156239309, "reliabilityBins": [ { "lower": 0, "upper": 0.1, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.1, "upper": 0.2, "upperInclusive": false, "n": 2, "meanPredictedProbability": 0.18041740920950003, "observedTeamAWinRate": 0.5, "observedRateWilson95": { "lower95": 0.09453120573423074, "upper95": 0.9054687942657693 } }, { "lower": 0.2, "upper": 0.3, "upperInclusive": false, "n": 26, "meanPredictedProbability": 0.2709783103229796, "observedTeamAWinRate": 0.34615384615384615, "observedRateWilson95": { "lower95": 0.1941223032294052, "upper95": 0.5377942881118918 } }, { "lower": 0.3, "upper": 0.4, "upperInclusive": false, "n": 119, "meanPredictedProbability": 0.36089446547771037, "observedTeamAWinRate": 0.48739495798319327, "observedRateWilson95": { "lower95": 0.3993973588188823, "upper95": 0.5761809188342759 } }, { "lower": 0.4, "upper": 0.5, "upperInclusive": false, "n": 212, "meanPredictedProbability": 0.45373858889185514, "observedTeamAWinRate": 0.42924528301886794, "observedRateWilson95": { "lower95": 0.3644599740660836, "upper95": 0.4965491195347807 } }, { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 353, "meanPredictedProbability": 0.531251440591354, "observedTeamAWinRate": 0.5354107648725213, "observedRateWilson95": { "lower95": 0.48328075497211903, "upper95": 0.5867783689946452 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 105, "meanPredictedProbability": 0.6389282598056109, "observedTeamAWinRate": 0.5238095238095238, "observedRateWilson95": { "lower95": 0.42913839724135777, "upper95": 0.6167999800477322 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 15, "meanPredictedProbability": 0.7398501576052282, "observedTeamAWinRate": 0.4, "observedRateWilson95": { "lower95": 0.1982449614134158, "upper95": 0.6425316987937539 } }, { "lower": 0.8, "upper": 0.9, "upperInclusive": false, "n": 3, "meanPredictedProbability": 0.8280425589102208, "observedTeamAWinRate": 1, "observedRateWilson95": { "lower95": 0.43850296824495455, "upper95": 1 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null } ], "accuracyByConfidenceBand": [ { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 565, "meanConfidence": 0.536883500325088, "accuracy": 0.5486725663716814, "accuracyWilson95": { "lower95": 0.5074489896389298, "upper95": 0.589238759024862 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 224, "meanConfidence": 0.6390224369988464, "accuracy": 0.5178571428571429, "accuracyWilson95": { "lower95": 0.45267331801393645, "upper95": 0.5824388167196102 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 41, "meanConfidence": 0.7329833242849014, "accuracy": 0.5609756097560976, "accuracyWilson95": { "lower95": 0.4104052722210608, "upper95": 0.7010986826344964 } }, { "lower": 0.8, "upper": 0.8999999999999999, "upperInclusive": false, "n": 5, "meanConfidence": 0.8246585716623326, "accuracy": 0.8, "accuracyWilson95": { "lower95": 0.3755346297625253, "upper95": 0.9637758913675698 } }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null } ], "sampleSizeByMap": [ { "mapName": "Abyss", "n": 72 }, { "mapName": "Ascent", "n": 36 }, { "mapName": "Bind", "n": 80 }, { "mapName": "Breeze", "n": 98 }, { "mapName": "Corrode", "n": 57 }, { "mapName": "Fracture", "n": 45 }, { "mapName": "Haven", "n": 124 }, { "mapName": "Lotus", "n": 97 }, { "mapName": "Pearl", "n": 90 }, { "mapName": "Split", "n": 125 }, { "mapName": "Sunset", "n": 11 } ] }, "intervals95": { "brier": { "estimate": 0.25544817411250825, "lower95": 0.24867960786880486, "upper95": 0.26279260881482563, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 785782382 }, "logLoss": { "estimate": 0.705281828557604, "lower95": 0.6909013821580605, "upper95": 0.7206204726085332, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 785782382 }, "accuracy": { "estimate": 0.5425149700598803, "lower95": 0.5085225384295804, "upper95": 0.5757286666189008, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 785782382 }, "ece": { "estimate": 0.05031042156239309, "lower95": 0.035785155097237425, "upper95": 0.09133258955288609, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 785782382 } }, "pairedDeltaVsPlainElo": { "brier": { "estimate": 0.0041909011851770095, "lower95": -0.005995987262899939, "upper95": 0.014874305013644527, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1877816383 }, "logLoss": { "estimate": 0.006071094542764444, "lower95": -0.016487821822353366, "upper95": 0.029817134351208738, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1877816383 }, "accuracy": { "estimate": -0.033532934131736525, "lower95": -0.07929907363977487, "upper95": 0.015387540335230553, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1877816383 }, "ece": { "estimate": -0.007500867664769949, "lower95": -0.03823876111871882, "upper95": 0.03086407835264114, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1877816383 } } } }, { "role": "naive-0.5-reference", "model": { "id": "naive-half", "label": "Uninformative 0.5 probability", "family": "naive", "ratingScope": "overall", "initialRating": 1000, "ratingScale": 400, "predictionScale": 400, "kFactor": 0, "marginFormula": "none", "marginScale": 0, "seasonCarry": null, "coldStartPriorGames": 0, "rosterRegressionStrength": 0 }, "selectionRule": "Uninformative comparator fixed in code for this run.", "provenance": { "basis": "runner-fixed-reference", "repositoryCommit": null, "repositoryCommitAt": null, "sourceFiles": [ "lib/elo/backtest.ts" ], "sourceBlobs": [], "note": "A constant probability reference carrying no team-strength information." }, "validation": { "metrics": { "n": 1184, "brier": 0.25, "logLoss": 0.6931471805599256, "accuracy": 0.5320945945945946, "ece": 0.03209459459459463, "reliabilityBins": [ { "lower": 0, "upper": 0.1, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.1, "upper": 0.2, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.2, "upper": 0.3, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.3, "upper": 0.4, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.4, "upper": 0.5, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 1184, "meanPredictedProbability": 0.5, "observedTeamAWinRate": 0.5320945945945946, "observedRateWilson95": { "lower95": 0.5036151908996981, "upper95": 0.5603664115571264 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.8, "upper": 0.9, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null } ], "accuracyByConfidenceBand": [ { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 1184, "meanConfidence": 0.5, "accuracy": 0.5320945945945946, "accuracyWilson95": { "lower95": 0.5036151908996981, "upper95": 0.5603664115571264 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null }, { "lower": 0.8, "upper": 0.8999999999999999, "upperInclusive": false, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null } ], "sampleSizeByMap": [ { "mapName": "Abyss", "n": 44 }, { "mapName": "Ascent", "n": 114 }, { "mapName": "Bind", "n": 97 }, { "mapName": "Corrode", "n": 57 }, { "mapName": "Fracture", "n": 88 }, { "mapName": "Haven", "n": 175 }, { "mapName": "Icebox", "n": 118 }, { "mapName": "Lotus", "n": 217 }, { "mapName": "Pearl", "n": 97 }, { "mapName": "Split", "n": 104 }, { "mapName": "Sunset", "n": 73 } ] }, "intervals95": { "brier": { "estimate": 0.25, "lower95": 0.25, "upper95": 0.25, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3306828247 }, "logLoss": { "estimate": 0.6931471805599256, "lower95": 0.693147180559925, "upper95": 0.6931471805599264, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3306828247 }, "accuracy": { "estimate": 0.5320945945945946, "lower95": 0.5018870633087567, "upper95": 0.5637705558751087, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3306828247 }, "ece": { "estimate": 0.03209459459459463, "lower95": 0.002542164728999469, "upper95": 0.06377055587510878, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3306828247 } }, "pairedDeltaVsPlainElo": { "brier": { "estimate": 0.0063302022764247234, "lower95": -0.0027646776417798846, "upper95": 0.01516291191720847, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2945902582 }, "logLoss": { "estimate": 0.01160879132164383, "lower95": -0.008474049328402938, "upper95": 0.031121889207286233, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2945902582 }, "accuracy": { "estimate": -0.03716216216216217, "lower95": -0.07618304224629588, "upper95": 0.0025612800357339965, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2945902582 }, "ece": { "estimate": -0.0334295743034652, "lower95": -0.07270728480812552, "upper95": -0.007712221489996372, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2945902582 } } }, "test": { "metrics": { "n": 835, "brier": 0.25, "logLoss": 0.6931471805599404, "accuracy": 0.4934131736526946, "ece": 0.006586826347305397, "reliabilityBins": [ { "lower": 0, "upper": 0.1, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.1, "upper": 0.2, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.2, "upper": 0.3, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.3, "upper": 0.4, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.4, "upper": 0.5, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 835, "meanPredictedProbability": 0.5, "observedTeamAWinRate": 0.4934131736526946, "observedRateWilson95": { "lower95": 0.45961032795475804, "upper95": 0.5272763478475269 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.8, "upper": 0.9, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null } ], "accuracyByConfidenceBand": [ { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 835, "meanConfidence": 0.5, "accuracy": 0.4934131736526946, "accuracyWilson95": { "lower95": 0.45961032795475804, "upper95": 0.5272763478475269 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null }, { "lower": 0.8, "upper": 0.8999999999999999, "upperInclusive": false, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null } ], "sampleSizeByMap": [ { "mapName": "Abyss", "n": 72 }, { "mapName": "Ascent", "n": 36 }, { "mapName": "Bind", "n": 80 }, { "mapName": "Breeze", "n": 98 }, { "mapName": "Corrode", "n": 57 }, { "mapName": "Fracture", "n": 45 }, { "mapName": "Haven", "n": 124 }, { "mapName": "Lotus", "n": 97 }, { "mapName": "Pearl", "n": 90 }, { "mapName": "Split", "n": 125 }, { "mapName": "Sunset", "n": 11 } ] }, "intervals95": { "brier": { "estimate": 0.25, "lower95": 0.25, "upper95": 0.25, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3412416336 }, "logLoss": { "estimate": 0.6931471805599404, "lower95": 0.693147180559939, "upper95": 0.693147180559942, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3412416336 }, "accuracy": { "estimate": 0.4934131736526946, "lower95": 0.45463545159053975, "upper95": 0.5265788698673095, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3412416336 }, "ece": { "estimate": 0.006586826347305397, "lower95": 0.0006016157971029127, "upper95": 0.04613500468004204, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3412416336 } }, "pairedDeltaVsPlainElo": { "brier": { "estimate": -0.0012572729273312433, "lower95": -0.01222257388105984, "upper95": 0.008158255380611438, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3445940773 }, "logLoss": { "estimate": -0.006063553454899173, "lower95": -0.03126851590292622, "upper95": 0.014872185948920097, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3445940773 }, "accuracy": { "estimate": -0.08263473053892217, "lower95": -0.12892361579854233, "upper95": -0.03615065243179125, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3445940773 }, "ece": { "estimate": -0.05122446287985764, "lower95": -0.08199839851446261, "upper95": -0.021965609631333947, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 3445940773 } } } }, { "role": "validation-selected-challenger", "model": { "id": "cold-start-prior-8", "label": "Production map Elo with 8-game uncertainty prior", "family": "cold-start-uncertainty", "ratingScope": "map", "initialRating": 1000, "ratingScale": 1000, "predictionScale": 1000, "kFactor": 74, "marginFormula": "production", "marginScale": 1, "seasonCarry": 0, "coldStartPriorGames": 8, "rosterRegressionStrength": 0 }, "selectionRule": "Lowest validation log loss among the experimental map-Elo candidate grid only; fixed references were not eligible and may score better. Ties break by model id. The grid was not prospectively preregistered.", "provenance": { "basis": "validation-selected-experimental-candidate", "repositoryCommit": null, "repositoryCommitAt": null, "sourceFiles": [ "lib/elo/backtest.ts" ], "sourceBlobs": [], "note": "Selected inside this run from experimental candidates using validation log loss; it is not the best model across fixed references and candidates." }, "validation": { "metrics": { "n": 1184, "brier": 0.2483543842832202, "logLoss": 0.6898503861574465, "accuracy": 0.5413851351351351, "ece": 0.03233234152559392, "reliabilityBins": [ { "lower": 0, "upper": 0.1, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.1, "upper": 0.2, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.2, "upper": 0.3, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.3, "upper": 0.4, "upperInclusive": false, "n": 13, "meanPredictedProbability": 0.3665345813495317, "observedTeamAWinRate": 0.5384615384615384, "observedRateWilson95": { "lower95": 0.29143795714506004, "upper95": 0.7679393219046169 } }, { "lower": 0.4, "upper": 0.5, "upperInclusive": false, "n": 372, "meanPredictedProbability": 0.47575104598708695, "observedTeamAWinRate": 0.4838709677419355, "observedRateWilson95": { "lower95": 0.4335124895186582, "upper95": 0.5345591542067221 } }, { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 784, "meanPredictedProbability": 0.5121667301294971, "observedTeamAWinRate": 0.5535714285714286, "observedRateWilson95": { "lower95": 0.5185962871697138, "upper95": 0.5880241490354818 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 15, "meanPredictedProbability": 0.6376364968379438, "observedTeamAWinRate": 0.6, "observedRateWilson95": { "lower95": 0.357468301206246, "upper95": 0.8017550385865841 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.8, "upper": 0.9, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null } ], "accuracyByConfidenceBand": [ { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 1156, "meanConfidence": 0.5160547814137795, "accuracy": 0.5415224913494809, "accuracyWilson95": { "lower95": 0.5127088036992963, "upper95": 0.5700611294410581 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 28, "meanConfidence": 0.6356999248223302, "accuracy": 0.5357142857142857, "accuracyWilson95": { "lower95": 0.358126892344679, "upper95": 0.7046843001720462 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null }, { "lower": 0.8, "upper": 0.8999999999999999, "upperInclusive": false, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null } ], "sampleSizeByMap": [ { "mapName": "Abyss", "n": 44 }, { "mapName": "Ascent", "n": 114 }, { "mapName": "Bind", "n": 97 }, { "mapName": "Corrode", "n": 57 }, { "mapName": "Fracture", "n": 88 }, { "mapName": "Haven", "n": 175 }, { "mapName": "Icebox", "n": 118 }, { "mapName": "Lotus", "n": 217 }, { "mapName": "Pearl", "n": 97 }, { "mapName": "Split", "n": 104 }, { "mapName": "Sunset", "n": 73 } ] }, "intervals95": { "brier": { "estimate": 0.2483543842832202, "lower95": 0.24661714128207948, "upper95": 0.25020276429014987, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2141677177 }, "logLoss": { "estimate": 0.6898503861574465, "lower95": 0.6863456242118213, "upper95": 0.6936087528455444, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2141677177 }, "accuracy": { "estimate": 0.5413851351351351, "lower95": 0.5130082519673564, "upper95": 0.5687942593204502, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2141677177 }, "ece": { "estimate": 0.03233234152559392, "lower95": 0.013754098384133624, "upper95": 0.06492379328522738, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2141677177 } }, "pairedDeltaVsPlainElo": { "brier": { "estimate": 0.004684586559644932, "lower95": -0.003933981368930021, "upper95": 0.01274999946971556, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1217566668 }, "logLoss": { "estimate": 0.008311996919164732, "lower95": -0.011459432191697682, "upper95": 0.02686104183833876, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1217566668 }, "accuracy": { "estimate": -0.02787162162162171, "lower95": -0.06324048500101309, "upper95": 0.006719603614451903, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1217566668 }, "ece": { "estimate": -0.033191827372465905, "lower95": -0.07064582232751518, "upper95": -0.005785365401993063, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 1217566668 } } }, "test": { "metrics": { "n": 835, "brier": 0.24982989034913441, "logLoss": 0.6928405062234989, "accuracy": 0.5461077844311377, "ece": 0.03300053483972225, "reliabilityBins": [ { "lower": 0, "upper": 0.1, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.1, "upper": 0.2, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.2, "upper": 0.3, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.3, "upper": 0.4, "upperInclusive": false, "n": 12, "meanPredictedProbability": 0.37129335257513524, "observedTeamAWinRate": 0.4166666666666667, "observedRateWilson95": { "lower95": 0.19326031365875668, "upper95": 0.6804886874504503 } }, { "lower": 0.4, "upper": 0.5, "upperInclusive": false, "n": 278, "meanPredictedProbability": 0.47660703894763096, "observedTeamAWinRate": 0.4244604316546763, "observedRateWilson95": { "lower95": 0.36777738100107793, "upper95": 0.4832026693586524 } }, { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 540, "meanPredictedProbability": 0.5104100734650276, "observedTeamAWinRate": 0.5314814814814814, "observedRateWilson95": { "lower95": 0.48931939368859967, "upper95": 0.573198826370842 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 5, "meanPredictedProbability": 0.6271299331486503, "observedTeamAWinRate": 0.4, "observedRateWilson95": { "lower95": 0.11762077423264783, "upper95": 0.769275718723987 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.8, "upper": 0.9, "upperInclusive": false, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanPredictedProbability": null, "observedTeamAWinRate": null, "observedRateWilson95": null } ], "accuracyByConfidenceBand": [ { "lower": 0.5, "upper": 0.6, "upperInclusive": false, "n": 818, "meanConfidence": 0.5148223506646373, "accuracy": 0.5464547677261614, "accuracyWilson95": { "lower95": 0.5122007033712255, "upper95": 0.5802745534855613 } }, { "lower": 0.6, "upper": 0.7, "upperInclusive": false, "n": 17, "meanConfidence": 0.6282429079318606, "accuracy": 0.5294117647058824, "accuracyWilson95": { "lower95": 0.3096323506000443, "upper95": 0.7383489355759056 } }, { "lower": 0.7, "upper": 0.8, "upperInclusive": false, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null }, { "lower": 0.8, "upper": 0.8999999999999999, "upperInclusive": false, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null }, { "lower": 0.9, "upper": 1, "upperInclusive": true, "n": 0, "meanConfidence": null, "accuracy": null, "accuracyWilson95": null } ], "sampleSizeByMap": [ { "mapName": "Abyss", "n": 72 }, { "mapName": "Ascent", "n": 36 }, { "mapName": "Bind", "n": 80 }, { "mapName": "Breeze", "n": 98 }, { "mapName": "Corrode", "n": 57 }, { "mapName": "Fracture", "n": 45 }, { "mapName": "Haven", "n": 124 }, { "mapName": "Lotus", "n": 97 }, { "mapName": "Pearl", "n": 90 }, { "mapName": "Split", "n": 125 }, { "mapName": "Sunset", "n": 11 } ] }, "intervals95": { "brier": { "estimate": 0.24982989034913441, "lower95": 0.2477542800501171, "upper95": 0.2518166484913484, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2910422810 }, "logLoss": { "estimate": 0.6928405062234989, "lower95": 0.6886732634728681, "upper95": 0.6968904605609417, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2910422810 }, "accuracy": { "estimate": 0.5461077844311377, "lower95": 0.513060141509434, "upper95": 0.5788616096349224, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2910422810 }, "ece": { "estimate": 0.03300053483972225, "lower95": 0.012182848771033362, "upper95": 0.06689745424716363, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 2910422810 } }, "pairedDeltaVsPlainElo": { "brier": { "estimate": -0.0014273825781968286, "lower95": -0.01008601348217987, "upper95": 0.00860354565131208, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 222236779 }, "logLoss": { "estimate": -0.006370227791340666, "lower95": -0.02562914870241878, "upper95": 0.015854260443179596, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 222236779 }, "accuracy": { "estimate": -0.029940119760479056, "lower95": -0.07267790097341228, "upper95": 0.019837823780580792, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 222236779 }, "ece": { "estimate": -0.024810754387440792, "lower95": -0.06567476530542173, "upper95": 0.01405154443499508, "method": "deterministic-block-bootstrap-percentile", "iterations": 500, "seed": 222236779 } } } } ], "conclusion": { "productionModelChanged": false, "statement": "Backtest results are evidence for review only. The runner never writes Elo configuration or adopts a challenger." }, "limitations": [ "This is an observational temporal backtest, not evidence of causal model improvement.", "The reported chronological test period is retrospective and can be rerun; it is not a prospectively untouched holdout, and future event mixes and patches may differ.", "The pre-holdout production comparator is externally frozen by repository history before the chronological test start, making its later rows retrospectively out of sample; this does not prove its original parameter selection was leakage-free or restore an untouched holdout after these results have been viewed.", "Roster change is a last-observed appearance proxy, not a confirmed pre-match roster announcement.", "Data-quality rejection counts start from maps marked processed that survive required team joins; they do not measure events never scraped, source-universe completeness, or all upstream ETL failures.", "Confidence intervals quantify resampling uncertainty for the observed series blocks and do not cover source bias or model-selection uncertainty.", "Validation-only selection and one test evaluation are enforced within each run; there is no persistent access ledger that can detect reruns after holdout results have been viewed.", "No experimental result automatically changes the production model; adoption requires an explicit review and calibration decision." ] }