BenchBenchmark bench.benchmark@1

Generated at from openvibe-contracts v0.419.0 and openvibe-sdk v0.161.0.

Version
1.0.0
Owner
bench
Visibility
first-party
Status
active
Compatibility
backward
Schema
https://openvibe.network/contracts/bench/benchmark.v1.json

bench.benchmark@1: a versioned, forkable benchmark on OpenVibe.Bench (spec 2026-10-30, stage B1). It holds the tasks, the judges, how they combine into one score, the limits a run may not pass, the default model and harness matrix, and the visibility of the set. A version is immutable once a run names it; a fork records its parent. Holdout tasks are hidden from everyone but the executor.

Fields

FieldTypeRequiredDescriptionConstraints
idstringyesThe benchmark id; the same id carries every version.
  • pattern ^bmk_[0-9A-HJKMNP-TV-Z]{26}$
ownerstringyesWho owns the set: a person (usr_…), a project (prj_…) or a service.
  • minLength 1
  • maxLength 120
titlestringyes
  • minLength 1
  • maxLength 160
descriptionstringyes
  • maxLength 4000
kindstringyesOne of Bench's kinds; the kind sets the renderer.
  • minLength 1
  • maxLength 80
tagsarray of stringyes
  • maxItems 32
  • items: minLength 1
  • items: maxLength 40
visibilityenumyespublic appears in the registry; unlisted is reachable by id only.
  • one of "public", "unlisted"
versionintegeryesThe version number this document is.
  • minimum 1
parentstring | nullyesThe benchmark id@version this one was forked from, or null.
  • maxLength 120
taskshttps://openvibe.network/contracts/bench/benchmark.v1.json#/$defs/taskyes
  • maxItems 200
judgeshttps://openvibe.network/contracts/bench/benchmark.v1.json#/$defs/judgeyes
  • maxItems 20
scoringhttps://openvibe.network/contracts/bench/benchmark.v1.json#/$defs/scoringyes
limitshttps://openvibe.network/contracts/bench/benchmark.v1.json#/$defs/limitsyes
matrixhttps://openvibe.network/contracts/bench/benchmark.v1.json#/$defs/matrixyes
created_atstringyes
  • format date-time

Examples

From the contract's own test fixtures: valid ones validate, rejected ones must fail.

Valid: playable-games
{
  "id": "bmk_01JAB2C3D4E5F6G7H8J9K0MNPS",
  "owner": "openvibe",
  "title": "Playable games arena",
  "description": "One-screen arcade games, judged by play and by the frame.",
  "kind": "games.play",
  "tags": [
    "games",
    "arcade"
  ],
  "visibility": "public",
  "version": 1,
  "parent": null,
  "tasks": [
    {
      "id": "t1",
      "prompt": "Build a one-screen arcade game with a paddle, bricks, a ball and a score.",
      "holdout": false
    },
    {
      "id": "t2",
      "prompt": "Build a snake game on a 20x20 grid.",
      "holdout": true,
      "expected": {
        "score": "food eaten"
      }
    }
  ],
  "judges": [
    {
      "type": "human_pairwise",
      "version": "1.0",
      "config": {
        "blind": true
      }
    },
    {
      "type": "game_outcome",
      "version": "1.0",
      "config": {}
    }
  ],
  "scoring": {
    "method": "mean",
    "scale": {
      "min": 0,
      "max": 100
    },
    "weights": {
      "human_pairwise": 1
    }
  },
  "limits": {
    "ms": 120000,
    "tokens": 200000,
    "usd": 0.5
  },
  "matrix": {
    "models": [
      "claude-opus-4",
      "gpt-5"
    ],
    "harnesses": [
      "claude-code",
      "opencode"
    ]
  },
  "created_at": "2026-10-30T12:00:00.000Z"
}
Rejected: bad-task
{
  "id": "bmk_01JAB2C3D4E5F6G7H8J9K0MNPS",
  "owner": "openvibe",
  "title": "Playable games arena",
  "description": "One-screen arcade games, judged by play and by the frame.",
  "kind": "games.play",
  "tags": [
    "games",
    "arcade"
  ],
  "visibility": "public",
  "version": 1,
  "parent": null,
  "tasks": [
    {
      "id": "t1",
      "prompt": "x"
    }
  ],
  "judges": [
    {
      "type": "human_pairwise",
      "version": "1.0",
      "config": {
        "blind": true
      }
    },
    {
      "type": "game_outcome",
      "version": "1.0",
      "config": {}
    }
  ],
  "scoring": {
    "method": "mean",
    "scale": {
      "min": 0,
      "max": 100
    },
    "weights": {
      "human_pairwise": 1
    }
  },
  "limits": {
    "ms": 120000,
    "tokens": 200000,
    "usd": 0.5
  },
  "matrix": {
    "models": [
      "claude-opus-4",
      "gpt-5"
    ],
    "harnesses": [
      "claude-code",
      "opencode"
    ]
  },
  "created_at": "2026-10-30T12:00:00.000Z"
}
Rejected: bad-visibility
{
  "id": "bmk_01JAB2C3D4E5F6G7H8J9K0MNPS",
  "owner": "openvibe",
  "title": "Playable games arena",
  "description": "One-screen arcade games, judged by play and by the frame.",
  "kind": "games.play",
  "tags": [
    "games",
    "arcade"
  ],
  "visibility": "secret",
  "version": 1,
  "parent": null,
  "tasks": [
    {
      "id": "t1",
      "prompt": "Build a one-screen arcade game with a paddle, bricks, a ball and a score.",
      "holdout": false
    },
    {
      "id": "t2",
      "prompt": "Build a snake game on a 20x20 grid.",
      "holdout": true,
      "expected": {
        "score": "food eaten"
      }
    }
  ],
  "judges": [
    {
      "type": "human_pairwise",
      "version": "1.0",
      "config": {
        "blind": true
      }
    },
    {
      "type": "game_outcome",
      "version": "1.0",
      "config": {}
    }
  ],
  "scoring": {
    "method": "mean",
    "scale": {
      "min": 0,
      "max": 100
    },
    "weights": {
      "human_pairwise": 1
    }
  },
  "limits": {
    "ms": 120000,
    "tokens": 200000,
    "usd": 0.5
  },
  "matrix": {
    "models": [
      "claude-opus-4",
      "gpt-5"
    ],
    "harnesses": [
      "claude-code",
      "opencode"
    ]
  },
  "created_at": "2026-10-30T12:00:00.000Z"
}

Validate

const contracts = require('openvibe-contracts');
contracts.validate('bench.benchmark@1', value);   // { valid, errors: [{ path, message }] }