{
  "sample_id": "talo-demo-001",
  "demonstration": true,
  "schema_version": "0.3",
  "task": {
    "name": "pour_water",
    "description": "Pick up the bottle with the right hand and pour water into the cup."
  },
  "clip": {
    "file": "clip-pour.mp4",
    "duration_s": 8.0,
    "fps": 24,
    "width": 480,
    "height": 360,
    "modality": "RGB video, head camera",
    "view": "Teleoperated humanoid robot"
  },
  "bbox_format": "xywh_pixels",
  "interpolation": "linear",
  "tracks": [
    {
      "id": "trk_cup",
      "label": "cup",
      "category": "container",
      "keyframes": [
        {
          "t": 0.0,
          "bbox": [
            178,
            152,
            36,
            48
          ]
        },
        {
          "t": 0.5,
          "bbox": [
            178,
            152,
            36,
            48
          ]
        },
        {
          "t": 1.0,
          "bbox": [
            178,
            152,
            36,
            48
          ]
        },
        {
          "t": 1.5,
          "bbox": [
            178,
            152,
            36,
            48
          ]
        },
        {
          "t": 2.0,
          "bbox": [
            178,
            152,
            36,
            48
          ]
        },
        {
          "t": 2.5,
          "bbox": [
            178,
            152,
            36,
            48
          ]
        },
        {
          "t": 3.0,
          "bbox": [
            178,
            152,
            36,
            48
          ]
        },
        {
          "t": 3.5,
          "bbox": [
            178,
            152,
            36,
            48
          ]
        },
        {
          "t": 4.0,
          "bbox": [
            178,
            152,
            36,
            48
          ]
        },
        {
          "t": 4.5,
          "bbox": [
            178,
            152,
            36,
            48
          ]
        },
        {
          "t": 5.0,
          "bbox": [
            178,
            152,
            36,
            48
          ]
        },
        {
          "t": 5.5,
          "bbox": [
            178,
            152,
            36,
            48
          ]
        },
        {
          "t": 6.0,
          "bbox": [
            178,
            152,
            36,
            48
          ]
        },
        {
          "t": 6.5,
          "bbox": [
            178,
            152,
            36,
            48
          ]
        },
        {
          "t": 7.0,
          "bbox": [
            178,
            152,
            36,
            48
          ]
        },
        {
          "t": 7.5,
          "bbox": [
            178,
            152,
            36,
            48
          ]
        },
        {
          "t": 8.0,
          "bbox": [
            178,
            152,
            36,
            48
          ]
        }
      ]
    },
    {
      "id": "trk_bottle",
      "label": "bottle",
      "category": "object",
      "keyframes": [
        {
          "t": 0.0,
          "bbox": [
            230,
            82,
            30,
            64
          ]
        },
        {
          "t": 0.5,
          "bbox": [
            231,
            82,
            30,
            62
          ]
        },
        {
          "t": 1.0,
          "bbox": [
            232,
            82,
            32,
            60
          ]
        },
        {
          "t": 1.5,
          "bbox": [
            222,
            90,
            30,
            60
          ]
        },
        {
          "t": 2.0,
          "bbox": [
            210,
            92,
            32,
            66
          ]
        },
        {
          "t": 2.5,
          "bbox": [
            232,
            86,
            32,
            62
          ]
        },
        {
          "t": 2.75,
          "bbox": [
            238,
            88,
            36,
            50
          ]
        },
        {
          "t": 3.0,
          "bbox": [
            236,
            90,
            48,
            40
          ]
        },
        {
          "t": 3.25,
          "bbox": [
            228,
            100,
            50,
            30
          ]
        },
        {
          "t": 3.5,
          "bbox": [
            214,
            92,
            58,
            42
          ]
        },
        {
          "t": 3.75,
          "bbox": [
            208,
            92,
            58,
            34
          ]
        },
        {
          "t": 4.0,
          "bbox": [
            196,
            84,
            68,
            34
          ]
        },
        {
          "t": 4.25,
          "bbox": [
            193,
            95,
            63,
            33
          ]
        },
        {
          "t": 4.5,
          "bbox": [
            186,
            104,
            62,
            32
          ]
        },
        {
          "t": 4.75,
          "bbox": [
            190,
            104,
            56,
            38
          ]
        },
        {
          "t": 5.0,
          "bbox": [
            188,
            100,
            56,
            50
          ]
        },
        {
          "t": 5.25,
          "bbox": [
            188,
            100,
            55,
            51
          ]
        },
        {
          "t": 5.5,
          "bbox": [
            186,
            98,
            58,
            54
          ]
        },
        {
          "t": 5.75,
          "bbox": [
            190,
            98,
            53,
            53
          ]
        },
        {
          "t": 6.0,
          "bbox": [
            190,
            94,
            54,
            58
          ]
        },
        {
          "t": 6.25,
          "bbox": [
            190,
            95,
            50,
            50
          ]
        },
        {
          "t": 6.5,
          "bbox": [
            186,
            90,
            54,
            48
          ]
        },
        {
          "t": 6.75,
          "bbox": [
            192,
            88,
            63,
            38
          ]
        },
        {
          "t": 7.0,
          "bbox": [
            196,
            82,
            72,
            32
          ]
        },
        {
          "t": 7.25,
          "bbox": [
            222,
            68,
            53,
            54
          ]
        },
        {
          "t": 7.5,
          "bbox": [
            246,
            60,
            36,
            72
          ]
        },
        {
          "t": 7.75,
          "bbox": [
            260,
            60,
            36,
            78
          ]
        },
        {
          "t": 8.0,
          "bbox": [
            270,
            67,
            34,
            75
          ]
        }
      ]
    },
    {
      "id": "trk_right_hand",
      "label": "right hand",
      "category": "end_effector",
      "keyframes": [
        {
          "t": 0.0,
          "bbox": [
            224,
            98,
            98,
            94
          ]
        },
        {
          "t": 0.5,
          "bbox": [
            222,
            103,
            96,
            88
          ]
        },
        {
          "t": 1.0,
          "bbox": [
            220,
            108,
            92,
            84
          ]
        },
        {
          "t": 1.5,
          "bbox": [
            214,
            112,
            92,
            84
          ]
        },
        {
          "t": 2.0,
          "bbox": [
            206,
            118,
            96,
            84
          ]
        },
        {
          "t": 2.5,
          "bbox": [
            224,
            108,
            96,
            80
          ]
        },
        {
          "t": 2.75,
          "bbox": [
            238,
            98,
            94,
            88
          ]
        },
        {
          "t": 3.0,
          "bbox": [
            242,
            84,
            104,
            98
          ]
        },
        {
          "t": 3.25,
          "bbox": [
            242,
            80,
            100,
            100
          ]
        },
        {
          "t": 3.5,
          "bbox": [
            240,
            76,
            98,
            104
          ]
        },
        {
          "t": 3.75,
          "bbox": [
            240,
            65,
            96,
            112
          ]
        },
        {
          "t": 4.0,
          "bbox": [
            236,
            54,
            100,
            122
          ]
        },
        {
          "t": 4.25,
          "bbox": [
            232,
            54,
            97,
            126
          ]
        },
        {
          "t": 4.5,
          "bbox": [
            224,
            50,
            98,
            136
          ]
        },
        {
          "t": 4.75,
          "bbox": [
            219,
            48,
            102,
            143
          ]
        },
        {
          "t": 5.0,
          "bbox": [
            212,
            46,
            110,
            150
          ]
        },
        {
          "t": 5.25,
          "bbox": [
            209,
            43,
            112,
            153
          ]
        },
        {
          "t": 5.5,
          "bbox": [
            204,
            40,
            116,
            156
          ]
        },
        {
          "t": 5.75,
          "bbox": [
            204,
            39,
            117,
            157
          ]
        },
        {
          "t": 6.0,
          "bbox": [
            204,
            38,
            118,
            158
          ]
        },
        {
          "t": 6.25,
          "bbox": [
            206,
            33,
            115,
            160
          ]
        },
        {
          "t": 6.5,
          "bbox": [
            208,
            26,
            112,
            164
          ]
        },
        {
          "t": 6.75,
          "bbox": [
            224,
            29,
            109,
            154
          ]
        },
        {
          "t": 7.0,
          "bbox": [
            238,
            32,
            108,
            144
          ]
        },
        {
          "t": 7.25,
          "bbox": [
            240,
            50,
            107,
            131
          ]
        },
        {
          "t": 7.5,
          "bbox": [
            240,
            70,
            110,
            116
          ]
        },
        {
          "t": 7.75,
          "bbox": [
            247,
            85,
            113,
            107
          ]
        },
        {
          "t": 8.0,
          "bbox": [
            250,
            94,
            118,
            106
          ]
        }
      ]
    }
  ],
  "segments": [
    {
      "id": "seg_01",
      "start_s": 0.0,
      "end_s": 2.5,
      "action": "grasp",
      "description": "Right hand closes on the bottle."
    },
    {
      "id": "seg_02",
      "start_s": 2.5,
      "end_s": 4.5,
      "action": "tilt",
      "description": "Bottle is lifted and tilted toward the cup."
    },
    {
      "id": "seg_03",
      "start_s": 4.5,
      "end_s": 6.5,
      "action": "pour",
      "description": "Bottle is held over the cup."
    },
    {
      "id": "seg_04",
      "start_s": 6.5,
      "end_s": 8.0,
      "action": "return",
      "description": "Bottle is brought upright and moved back."
    }
  ],
  "provenance": "Labels added by Talo for this website. Not customer work.",
  "review": {
    "method": "Two passes by one annotator. Pass one drew boxes at 0.5 s. Pass two burned the interpolated boxes onto 28 frames (every 0.5 s to 2.5 s, then every 0.25 s through the motion), compared each against the frame, and set a measured keyframe at every frame checked on the bottle and hand tracks. The cup did not move and keeps 0.5 s keyframes. The 8.0 s keyframe was checked against the clip's last frame, at 7.96 s.",
    "independent": false,
    "frames_checked": 28,
    "checked_t": [
      0.0,
      0.5,
      1.0,
      1.5,
      2.0,
      2.5,
      2.75,
      3.0,
      3.25,
      3.5,
      3.75,
      4.0,
      4.25,
      4.5,
      4.75,
      5.0,
      5.25,
      5.5,
      5.75,
      6.0,
      6.25,
      6.5,
      6.75,
      7.0,
      7.25,
      7.5,
      7.75,
      8.0
    ],
    "threshold_px": 4,
    "corrections": [
      {
        "track": "trk_bottle",
        "t": 2.75,
        "issue": "Left edge 4 px off.",
        "fix": "Keyframe set to [238, 88, 36, 50]."
      },
      {
        "track": "trk_bottle",
        "t": 3.25,
        "issue": "Top edge 9 px off.",
        "fix": "Keyframe set to [228, 100, 50, 30]."
      },
      {
        "track": "trk_bottle",
        "t": 3.75,
        "issue": "Top edge 4 px off.",
        "fix": "Keyframe set to [208, 92, 58, 34]."
      },
      {
        "track": "trk_bottle",
        "t": 7.75,
        "issue": "Top edge 4 px off.",
        "fix": "Keyframe set to [260, 60, 36, 78]."
      },
      {
        "track": "trk_bottle",
        "t": 8.0,
        "issue": "Bottom edge 4 px off.",
        "fix": "Keyframe set to [270, 67, 34, 75]."
      },
      {
        "track": "trk_right_hand",
        "t": 2.75,
        "issue": "Left edge 5 px off.",
        "fix": "Keyframe set to [238, 98, 94, 88]."
      },
      {
        "track": "trk_right_hand",
        "t": 7.5,
        "issue": "Top edge 6 px off.",
        "fix": "Keyframe set to [240, 70, 110, 116]."
      },
      {
        "track": "trk_right_hand",
        "t": 7.75,
        "issue": "Top edge 8 px off.",
        "fix": "Keyframe set to [247, 85, 113, 107]."
      },
      {
        "track": "trk_right_hand",
        "t": 8.0,
        "issue": "Top edge 4 px off.",
        "fix": "Keyframe set to [250, 94, 118, 106]."
      }
    ],
    "limits": "The same annotator ran both passes, so this is a self-check and not an independent review. 28 of the clip's 192 frames were checked. Boxes were measured by eye against a 60 px grid, so expect errors of a few pixels. No agreement score or IoU is reported because there is no second annotator."
  },
  "permitted_use": "Labels may be used to evaluate Talo's delivery format.",
  "not_included": "Robot state and operator action streams. This package holds video and labels only.",
  "instructions": {
    "version": "1.0",
    "file": "INSTRUCTIONS.md",
    "objects": "Label every object the task acts on and the effector that acts on it. Here: bottle, cup, right hand.",
    "boxes": "Draw the tightest axis-aligned box around the visible extent of the object. Do not extend a box over parts hidden behind another object.",
    "keyframes": "Place a keyframe every 0.5 s while an object is still or moving slowly and every 0.25 s while it is moving or rotating. Positions between keyframes are linearly interpolated.",
    "segments": "One segment per action, contiguous, no gaps. A boundary goes on the first sampled frame where the motion changes: the bottle starts to rotate, stops over the cup, or starts back.",
    "ambiguity": [
      "Occlusion: when the hand hides part of the bottle, box the visible part only.",
      "Effector extent: the hand box ends at the wrist joint. The forearm is not labelled.",
      "Idle effectors: an effector that takes no part in the task is not labelled. The left hand is in frame throughout and has no track.",
      "Transparent objects: the cup is boxed on its rim and base outline, not on reflections on the table.",
      "Unobservable events: liquid is not visible at this resolution, so the pour segment is defined by bottle pose over the cup, not by flow.",
      "Boundary tolerance: segment boundaries are accurate to the sampling step, plus or minus 0.25 s."
    ],
    "brief": [
      {
        "rule": "Objects",
        "text": "Bottle, cup and right hand: the objects acted on and the hand acting."
      },
      {
        "rule": "Boxes",
        "text": "Tightest box around the visible part only."
      },
      {
        "rule": "Keyframes",
        "text": "Every 0.5 s when still or slow, every 0.25 s when moving fast, linear in between."
      },
      {
        "rule": "Segments",
        "text": "One per action, no gaps, with the boundary where the motion changes."
      }
    ],
    "ambiguity_brief": [
      {
        "rule": "Occlusion",
        "text": "Box the visible part only."
      },
      {
        "rule": "Hand extent",
        "text": "The hand box stops at the wrist."
      },
      {
        "rule": "Idle hand",
        "text": "The left hand is in frame but unused, so it has no track."
      },
      {
        "rule": "Transparent cup",
        "text": "Boxed on its outline, not its reflections."
      },
      {
        "rule": "Liquid not visible",
        "text": "The pour is defined by bottle pose over the cup."
      },
      {
        "rule": "Boundary tolerance",
        "text": "Plus or minus 0.25 s."
      }
    ]
  }
}