AiVideoAnalysis ai.video-analysis@1

Generated at from openvibe-contracts v0.443.0 and openvibe-sdk v0.177.0.

Version
1.0.0
Owner
ai
Visibility
public
Status
active
Compatibility
backward
Schema
https://openvibe.network/contracts/ai/video-analysis.v1.json

A video analysis: what OpenVibe.AI (workflow video.analyze) found in one video. The document holds the video it read, what ffprobe measured, the keyframes it took (each with what the vision route saw in it and the text on screen), the transcript with its word times, the chapters, the summary, the people, objects and places that were named (only what was shown or said, each with the time it was seen or heard), the tags, the answer to the question that was asked, the explicit-content verdict and the receipt of every step and model call. Nothing is made up: a step that failed says so in the receipt.

Fields

FieldTypeRequiredDescriptionConstraints
media_idstring | nullyes
  • pattern ^med_[0-9A-HJKMNP-TV-Z]{26}$
duration_snumberyes
probeobjectyes
  • no other fields
probe.duration_snumberyes
probe.size_bytesnumberyes
probe.videoobject | nullyes
probe.audioobject | nullyes
keyframesarray of objectyes
  • maxItems 200
  • items: no other fields
keyframes[].tnumberyes
keyframes[].urlstringyes
  • maxLength 2048
keyframes[].captionstringyes
  • maxLength 1200
keyframes[].textstringyes
  • maxLength 1200
transcriptobjectyes
  • no other fields
transcript.langstringyes
  • maxLength 8
transcript.wordsarray of objectyes
  • maxItems 20000
  • items: no other fields
transcript.words[].t0numberyes
transcript.words[].t1numberyes
transcript.words[].wstringyes
  • maxLength 200
transcript.textstringyes
  • maxLength 200000
chaptersarray of objectyes
  • maxItems 60
  • items: no other fields
chapters[].startnumberyes
chapters[].endnumberyes
chapters[].titlestringyes
  • maxLength 200
summarystringyes
  • maxLength 6000
entitiesarray of objectyes
  • maxItems 120
  • items: no other fields
entities[].namestringyes
  • maxLength 200
entities[].kindenumyes
  • one of "person", "object", "place", "organization", "other"
entities[].evidencearray of objectyes
  • maxItems 20
  • items: no other fields
entities[].evidence[].tnumberyes
entities[].evidence[].fromenumyes
  • one of "frame", "speech"
tagsarray of stringyes
  • maxItems 40
  • items: maxLength 60
answerstring
  • maxLength 4000
explicitobjectyes
  • no other fields
explicit.categoryenumyes
  • one of "fine", "sex_acts", "minor_sexual"
explicit.reasonstringyes
  • maxLength 300
receiptobjectyes
  • no other fields
receipt.stepsarray of objectyes
  • maxItems 16
  • items: no other fields
receipt.steps[].keyenumyes
  • one of "probe", "keyframes", "transcript", "describe", "analyse", "explicit", "report"
receipt.steps[].statusenumyes
  • one of "done", "skipped", "failed"
receipt.steps[].detailstring | null
  • maxLength 300
receipt.steps[].cost_usdnumber
  • minimum 0
receipt.steps[].msinteger
  • minimum 0
receipt.modelsarray of objectyes
  • maxItems 400
  • items: no other fields
receipt.models[].stepenumyes
  • one of "describe", "analyse", "explicit"
receipt.models[].providerstring | nullyes
  • maxLength 80
receipt.models[].modelstring | nullyes
  • maxLength 200
receipt.models[].tokens_inintegeryes
  • minimum 0
receipt.models[].tokens_outintegeryes
  • minimum 0
receipt.models[].cost_usdnumberyes
  • minimum 0
receipt.cost_usdnumberyes
  • minimum 0

Examples

From the contract's own test fixtures: valid ones validate, rejected ones must fail.

Valid: screen-recording
{
  "media_id": "med_01JAB2C3D4E5F6G7H8J9K0MNPQ",
  "duration_s": 94.5,
  "probe": {
    "duration_s": 94.5,
    "size_bytes": 41943040,
    "video": {
      "codec": "h264",
      "width": 1920,
      "height": 1080,
      "fps": 29.97,
      "rotation": 0
    },
    "audio": {
      "codec": "aac",
      "sample_rate": 48000,
      "channels": 2
    }
  },
  "keyframes": [
    {
      "t": 0,
      "url": "/api/v1/video-analyses/run_01JAB2C3D4E5F6G7H8J9K0MNPQ/frames/0",
      "caption": "A desk with a laptop showing a README file and a coffee cup beside it.",
      "text": "README.md"
    },
    {
      "t": 47.25,
      "url": "/api/v1/video-analyses/run_01JAB2C3D4E5F6G7H8J9K0MNPQ/frames/1",
      "caption": "A terminal window with a test run passing.",
      "text": "12 passing"
    }
  ],
  "transcript": {
    "lang": "en",
    "words": [
      {
        "t0": 3.2,
        "t1": 3.8,
        "w": "we"
      },
      {
        "t0": 3.8,
        "t1": 4.4,
        "w": "open"
      },
      {
        "t0": 4.4,
        "t1": 5.1,
        "w": "the"
      },
      {
        "t0": 5.1,
        "t1": 5.7,
        "w": "readme"
      }
    ],
    "text": "we open the readme"
  },
  "chapters": [
    {
      "start": 0,
      "end": 48,
      "title": "The README"
    },
    {
      "start": 48,
      "end": 94.5,
      "title": "The test run"
    }
  ],
  "summary": "A screen recording of a developer opening a README file and running the tests.",
  "entities": [
    {
      "name": "README.md",
      "kind": "object",
      "evidence": [
        {
          "t": 0,
          "from": "frame"
        }
      ]
    },
    {
      "name": "readme",
      "kind": "object",
      "evidence": [
        {
          "t": 5.1,
          "from": "speech"
        }
      ]
    }
  ],
  "tags": [
    "screen recording",
    "developer"
  ],
  "answer": "It shows a README file being edited and the tests passing.",
  "explicit": {
    "category": "fine",
    "reason": "the text describes neither sex acts nor a child with nudity"
  },
  "receipt": {
    "steps": [
      {
        "key": "probe",
        "status": "done",
        "detail": "94.5 s, 1920x1080 h264, aac audio",
        "cost_usd": 0,
        "ms": 412
      },
      {
        "key": "keyframes",
        "status": "done",
        "detail": "2 frames (at most 12)",
        "cost_usd": 0,
        "ms": 2140
      },
      {
        "key": "transcript",
        "status": "done",
        "detail": "4 words",
        "cost_usd": 0,
        "ms": 3100
      },
      {
        "key": "describe",
        "status": "done",
        "detail": "2 of 2 frames read",
        "cost_usd": 0.0012,
        "ms": 2600
      },
      {
        "key": "analyse",
        "status": "done",
        "detail": "2 chapters, 2 entities",
        "cost_usd": 0.0021,
        "ms": 1800
      },
      {
        "key": "explicit",
        "status": "done",
        "detail": "fine: the text describes neither sex acts nor a child with nudity",
        "cost_usd": 0,
        "ms": 1
      }
    ],
    "models": [
      {
        "step": "describe",
        "provider": "openai",
        "model": "gpt-4o-mini",
        "tokens_in": 1046,
        "tokens_out": 174,
        "cost_usd": 0.0012
      },
      {
        "step": "analyse",
        "provider": "openai",
        "model": "gpt-4o-mini",
        "tokens_in": 2210,
        "tokens_out": 402,
        "cost_usd": 0.0021
      }
    ],
    "cost_usd": 0.0033
  }
}
Rejected: missing-summary
{
  "media_id": null,
  "duration_s": 12,
  "probe": {
    "duration_s": 12,
    "size_bytes": 1024,
    "video": {
      "codec": "h264",
      "width": 640,
      "height": 360,
      "fps": 25,
      "rotation": 0
    },
    "audio": null
  },
  "keyframes": [],
  "transcript": {
    "lang": "und",
    "words": [],
    "text": ""
  },
  "chapters": [],
  "entities": [],
  "tags": [],
  "explicit": {
    "category": "fine",
    "reason": "there is no text to judge"
  },
  "receipt": {
    "steps": [],
    "models": [],
    "cost_usd": 0
  }
}

Validate

const contracts = require('openvibe-contracts');
contracts.validate('ai.video-analysis@1', value);   // { valid, errors: [{ path, message }] }