{
  "name": "mercury-instant-fall-detection-benchmark",
  "status": "scored_development_benchmark",
  "scored_at": "2026-09-03",
  "scope": "development_evaluation_used_for_checkpoint_selection",
  "split_unit": "complete_camera_view",
  "selection_rule": "Reserve every evaluated image from each selected camera view. Keep views from the same visual environment together.",
  "test_camera_count": 23,
  "test_environment_group_count": 22,
  "test_image_count": 25814,
  "linked_environment_groups": [
    {
      "group_id": "shared-environment-01",
      "camera_ids": [
        "camera-11",
        "camera-21"
      ],
      "camera_count": 2,
      "note": "The original split review grouped these two views as the same bedroom; both were withheld from training."
    }
  ],
  "class_counts": {
    "Away": 7137,
    "Fall": 3438,
    "In bed": 5101,
    "Sit": 7085,
    "Stand": 3053
  },
  "scenario_coverage": [
    "bedroom and care room",
    "living and open room",
    "hallway and sparse transition view",
    "kitchen and dining view",
    "exterior night view",
    "assistance and mobility equipment",
    "multiple people",
    "furniture and bedding occlusion",
    "distant or small subject",
    "recliner and seated posture boundary",
    "floor sitting, crouching, kneeling, and reclining near furniture"
  ],
  "evaluation_protocol": {
    "classes": [
      "Away",
      "Fall",
      "In bed",
      "Sit",
      "Stand"
    ],
    "same_images_for_both_systems": true,
    "mercury_checkpoint_epoch": 8,
    "checkpoint_selected_using_test_metrics": true,
    "claude_model": "claude-opus-5",
    "comparator_prompt_version": "mercury-five-state-v1",
    "primary_metric": "Fall recall",
    "secondary_metrics": [
      "Fall precision",
      "per class F1",
      "macro F1",
      "accuracy"
    ],
    "checkpoint_selection_metric": "instant evaluation accuracy",
    "main_model_selection_used_separate_evaluation_set": true,
    "main_selection_camera_count": 6,
    "benchmark_cameras_in_earlier_exploratory_cross_validation": 9,
    "checkpoint_selection_scope": "The benchmark run selected epoch 8 by evaluation accuracy; main model development used a separate six-camera set.",
    "separate_validation_split": false,
    "untouched_final_test": false,
    "comparator_prompt_file": "claude-prompt-v1.txt",
    "comparator_prompt_provenance": "Recovered from the v1 controller source. Full historical API request/response records are not present in the prediction cache.",
    "annotation_provenance": "Most annotations were labeled manually. The researcher confirms the legacy annotation template was not used.",
    "annotation_rules_status": "Rules for ambiguous postures and multiple people await documentation; no corrected-prompt result is claimed."
  },
  "confidential_fields_omitted": [
    "training corpus composition",
    "training image count",
    "model architecture",
    "model parameter count",
    "initialization and pretraining source",
    "exact optimization recipe"
  ],
  "image_availability": "Underlying images are not yet distributed publicly and require privacy and licensing review.",
  "revision_date": "2026-09-04",
  "artifact_scope": "Camera inventory and aggregate counts; not a full image membership manifest.",
  "cameras": [
    {
      "camera_id": "camera-01",
      "environment_group_id": "environment-01",
      "image_count": 1647,
      "class_counts": {
        "away": 1122,
        "fall": 10,
        "in bed": 245,
        "sit": 13,
        "stand": 257
      }
    },
    {
      "camera_id": "camera-02",
      "environment_group_id": "environment-02",
      "image_count": 1255,
      "class_counts": {
        "away": 487,
        "fall": 14,
        "in bed": 5,
        "sit": 610,
        "stand": 139
      }
    },
    {
      "camera_id": "camera-03",
      "environment_group_id": "environment-03",
      "image_count": 1994,
      "class_counts": {
        "away": 299,
        "fall": 524,
        "in bed": 303,
        "sit": 663,
        "stand": 205
      }
    },
    {
      "camera_id": "camera-04",
      "environment_group_id": "environment-04",
      "image_count": 3528,
      "class_counts": {
        "away": 1141,
        "fall": 534,
        "in bed": 509,
        "sit": 731,
        "stand": 613
      }
    },
    {
      "camera_id": "camera-05",
      "environment_group_id": "environment-05",
      "image_count": 3964,
      "class_counts": {
        "away": 583,
        "fall": 511,
        "in bed": 1453,
        "sit": 757,
        "stand": 660
      }
    },
    {
      "camera_id": "camera-06",
      "environment_group_id": "environment-06",
      "image_count": 3051,
      "class_counts": {
        "away": 240,
        "fall": 521,
        "in bed": 812,
        "sit": 1275,
        "stand": 203
      }
    },
    {
      "camera_id": "camera-07",
      "environment_group_id": "environment-07",
      "image_count": 4125,
      "class_counts": {
        "away": 298,
        "fall": 513,
        "in bed": 1583,
        "sit": 1519,
        "stand": 212
      }
    },
    {
      "camera_id": "camera-08",
      "environment_group_id": "environment-08",
      "image_count": 1406,
      "class_counts": {
        "away": 362,
        "fall": 251,
        "in bed": 5,
        "sit": 620,
        "stand": 168
      }
    },
    {
      "camera_id": "camera-09",
      "environment_group_id": "environment-09",
      "image_count": 3379,
      "class_counts": {
        "away": 2137,
        "fall": 503,
        "in bed": 24,
        "sit": 529,
        "stand": 186
      }
    },
    {
      "camera_id": "camera-10",
      "environment_group_id": "environment-10",
      "image_count": 130,
      "class_counts": {
        "away": 12,
        "fall": 0,
        "in bed": 2,
        "sit": 69,
        "stand": 47
      }
    },
    {
      "camera_id": "camera-11",
      "environment_group_id": "environment-11",
      "image_count": 14,
      "class_counts": {
        "away": 2,
        "fall": 2,
        "in bed": 0,
        "sit": 0,
        "stand": 10
      }
    },
    {
      "camera_id": "camera-12",
      "environment_group_id": "environment-12",
      "image_count": 35,
      "class_counts": {
        "away": 6,
        "fall": 10,
        "in bed": 0,
        "sit": 5,
        "stand": 14
      }
    },
    {
      "camera_id": "camera-13",
      "environment_group_id": "environment-13",
      "image_count": 208,
      "class_counts": {
        "away": 185,
        "fall": 0,
        "in bed": 21,
        "sit": 2,
        "stand": 0
      }
    },
    {
      "camera_id": "camera-14",
      "environment_group_id": "environment-14",
      "image_count": 71,
      "class_counts": {
        "away": 22,
        "fall": 4,
        "in bed": 5,
        "sit": 11,
        "stand": 29
      }
    },
    {
      "camera_id": "camera-15",
      "environment_group_id": "environment-15",
      "image_count": 167,
      "class_counts": {
        "away": 8,
        "fall": 4,
        "in bed": 41,
        "sit": 45,
        "stand": 69
      }
    },
    {
      "camera_id": "camera-16",
      "environment_group_id": "environment-16",
      "image_count": 373,
      "class_counts": {
        "away": 174,
        "fall": 9,
        "in bed": 72,
        "sit": 57,
        "stand": 61
      }
    },
    {
      "camera_id": "camera-17",
      "environment_group_id": "environment-17",
      "image_count": 13,
      "class_counts": {
        "away": 1,
        "fall": 9,
        "in bed": 0,
        "sit": 2,
        "stand": 1
      }
    },
    {
      "camera_id": "camera-18",
      "environment_group_id": "environment-18",
      "image_count": 11,
      "class_counts": {
        "away": 11,
        "fall": 0,
        "in bed": 0,
        "sit": 0,
        "stand": 0
      }
    },
    {
      "camera_id": "camera-19",
      "environment_group_id": "environment-19",
      "image_count": 69,
      "class_counts": {
        "away": 1,
        "fall": 0,
        "in bed": 0,
        "sit": 9,
        "stand": 59
      }
    },
    {
      "camera_id": "camera-20",
      "environment_group_id": "environment-20",
      "image_count": 73,
      "class_counts": {
        "away": 1,
        "fall": 0,
        "in bed": 16,
        "sit": 19,
        "stand": 37
      }
    },
    {
      "camera_id": "camera-21",
      "environment_group_id": "environment-11",
      "image_count": 56,
      "class_counts": {
        "away": 20,
        "fall": 17,
        "in bed": 4,
        "sit": 4,
        "stand": 11
      }
    },
    {
      "camera_id": "camera-22",
      "environment_group_id": "environment-21",
      "image_count": 214,
      "class_counts": {
        "away": 14,
        "fall": 0,
        "in bed": 0,
        "sit": 138,
        "stand": 62
      }
    },
    {
      "camera_id": "camera-23",
      "environment_group_id": "environment-22",
      "image_count": 31,
      "class_counts": {
        "away": 11,
        "fall": 2,
        "in bed": 1,
        "sit": 7,
        "stand": 10
      }
    }
  ],
  "results_file": "benchmark-results.json",
  "scoring_script": "score-benchmark.mjs",
  "evidence_gaps": [
    "Full image membership and image-to-prediction mapping",
    "Per-camera Claude confusion matrices",
    "Independent incident identifiers and cluster-aware uncertainty",
    "Versioned manual annotation rules and reviewer/adjudication history",
    "Matched end-to-end latency measurements",
    "A new evaluation set unused for model or prompt selection"
  ]
}