[ { "sample_id": "b_d5be48474254", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0001.JPG", "source_split": "train", "source_frame_id": "0001", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0001.JPG", "context_ids": [ "b_1aede9f60837", "b_a38e3ce498d5", "b_63ebbf87a7c1", "b_442bffc55440" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_d5be48474254", "target_sha256": "c8e0464012d765884090d74c36c513738c8d3ff0de56534b10d97666e0dfae81", "context_ids": [ "b_1aede9f60837", "b_a38e3ce498d5", "b_63ebbf87a7c1", "b_442bffc55440" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "red umbrella", "grounding_queries": [ "red umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_1aede9f60837", "observation": "red umbrella partially obscured" }, { "context_id": "b_a38e3ce498d5", "observation": "red umbrella fully visible" }, { "context_id": "b_63ebbf87a7c1", "observation": "red umbrella partially obscured" }, { "context_id": "b_442bffc5540", "observation": "red umbrella fully visible" } ], "uncertainty": "red umbrella's visibility varies across views", "fallback_box_norm_xyxy": null, "routing_eligible": true, "consistency_filter": "passed" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "black and white umbrella", "grounding_queries": [ "black and white umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_1aede9f60837", "observation": "black and white umbrella partially obscured" }, { "context_id": "b_a38e3ce498d5", "observation": "black and white umbrella fully visible" }, { "context_id": "b_63ebbf87a7c1", "observation": "black and white umbrella partially obscured" }, { "context_id": "b_442bffc5540", "observation": "black and white umbrella fully visible" } ], "uncertainty": "black and white umbrella's visibility varies across views", "fallback_box_norm_xyxy": null, "routing_eligible": true, "consistency_filter": "passed" }, { "region_id": "r3", "cause": "transient_occluder", "semantic_description": "mannequin shirt", "grounding_queries": [ "mannequin shirt" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_1aede9f60837", "observation": "mannequin shirt partially obscured" }, { "context_id": "b_a38e3ce498d5", "observation": "mannequin shirt fully visible" }, { "context_id": "b_63ebbf87a7c1", "observation": "mannequin shirt partially obscured" }, { "context_id": "b_442bffc5540", "observation": "mannequin shirt fully visible" } ], "uncertainty": "mannequin shirt's visibility varies across views", "fallback_box_norm_xyxy": null, "routing_eligible": true, "consistency_filter": "passed" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"red umbrella\",\n \"grounding_queries\": [\"red umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_1aede9f60837\", \"observation\": \"red umbrella partially obscured\"},\n {\"context_id\": \"b_a38e3ce498d5\", \"observation\": \"red umbrella fully visible\"},\n {\"context_id\": \"b_63ebbf87a7c1\", \"observation\": \"red umbrella partially obscured\"},\n {\"context_id\": \"b_442bffc5540\", \"observation\": \"red umbrella fully visible\"}\n ],\n \"uncertainty\": \"red umbrella's visibility varies across views\",\n \"fallback_box_norm_xyxy\": [292, 275, 400, 370]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"black and white umbrella\",\n \"grounding_queries\": [\"black and white umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_1aede9f60837\", \"observation\": \"black and white umbrella partially obscured\"},\n {\"context_id\": \"b_a38e3ce498d5\", \"observation\": \"black and white umbrella fully visible\"},\n {\"context_id\": \"b_63ebbf87a7c1\", \"observation\": \"black and white umbrella partially obscured\"},\n {\"context_id\": \"b_442bffc5540\", \"observation\": \"black and white umbrella fully visible\"}\n ],\n \"uncertainty\": \"black and white umbrella's visibility varies across views\",\n \"fallback_box_norm_xyxy\": [1, 320, 170, 448]\n },\n {\n \"region_id\": \"r3\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"mannequin shirt\",\n \"grounding_queries\": [\"mannequin shirt\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_1aede9f60837\", \"observation\": \"mannequin shirt partially obscured\"},\n {\"context_id\": \"b_a38e3ce498d5\", \"observation\": \"mannequin shirt fully visible\"},\n {\"context_id\": \"b_63ebbf87a7c1\", \"observation\": \"mannequin shirt partially obscured\"},\n {\"context_id\": \"b_442bffc5540\", \"observation\": \"mannequin shirt fully visible\"}\n ],\n \"uncertainty\": \"mannequin shirt's visibility varies across views\",\n \"fallback_box_norm_xyxy\": [170, 280, 290, 448]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_d5be48474254", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "red umbrella", "queries": [ "red umbrella" ], "source": "grounding", "boxes_xyxy": [ [ 113.717529296875, 702.1360473632812, 436.87554931640625, 1007.55078125 ], [ 753.3574829101562, 598.4653930664062, 1055.838623046875, 894.8519287109375 ] ], "box_scores": [ 0.3146287798881531, 0.2932315170764923 ], "box_queries": [ "red umbrella", "red umbrella" ], "sam_predicted_iou": [ 0.99609375, 0.97265625 ], "mask_path": "masks/b_d5be48474254_r1.png" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "black and white umbrella", "queries": [ "black and white umbrella" ], "source": "grounding", "boxes_xyxy": [ [ 113.71705627441406, 701.9348754882812, 437.05767822265625, 1007.9161376953125 ] ], "box_scores": [ 0.27239349484443665 ], "box_queries": [ "black and white umbrella" ], "sam_predicted_iou": [ 0.99609375 ], "mask_path": "masks/b_d5be48474254_r2.png" }, { "region_id": "r3", "cause": "transient_occluder", "semantic_description": "mannequin shirt", "queries": [ "mannequin shirt" ], "source": "grounding", "boxes_xyxy": [], "box_scores": [], "box_queries": [], "sam_predicted_iou": [], "mask_path": "masks/b_d5be48474254_r3.png" } ], "final_mask_path": "masks/b_d5be48474254.png", "routing_mask_path": "masks/b_d5be48474254_routing.png", "predicted_fraction": 0.045033856966976794 }, "raw_mask_metrics": { "iou": 0.5666007355440863, "dice": 0.7233505291918604, "precision": 0.7930572077430518, "recall": 0.6649076995611649, "safety_recall": 0.6649076995611649, "leakage": 0.3350923004388351, "overmask": 0.00984842402570801, "boundary_f5": 0.12364454052526745, "iou_ignore_boundary5": 0.6084722406694116, "predicted_fraction": 0.045033856966976794, "gt_fraction": 0.05371335733320866 }, "routing_mask_metrics": { "iou": 0.5847536110654697, "dice": 0.7379741645419885, "precision": 0.7683739462398305, "recall": 0.7098882965219598, "safety_recall": 0.7098882965219598, "leakage": 0.2901117034780401, "overmask": 0.012146869975687733, "boundary_f5": 0.337117142558952, "iou_ignore_boundary5": 0.6080899308224442, "predicted_fraction": 0.04962490454595041, "gt_fraction": 0.05371335733320866 }, "box_metrics": { "gt_pixel_coverage": 0.7976734486635477, "area_precision": 0.46590830857548365, "box_fraction": 0.0919616976015709 }, "frame_detected": true, "gt_ambiguity": { "median_rgb_max_difference": 0.00784313678741455, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_1aede9f60837", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0002.JPG", "source_split": "train", "source_frame_id": "0002", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0002.JPG", "context_ids": [ "b_d5be48474254", "b_a38e3ce498d5", "b_63ebbf87a7c1", "b_442bffc55440" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_1aede9f60837", "target_sha256": "8fa7c6b67704a3493ad59f7ce40137e6dd08cf91d93521af022d99e65c613084", "context_ids": [ "b_d5be48474254", "b_a38e3ce498d5", "b_63ebbf87a7c1", "b_442bffc55440" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "red umbrella", "grounding_queries": [ "red umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_d5be48474254", "observation": "red umbrella is visible" }, { "context_id": "b_a38e3ce498d5", "observation": "red umbrella is visible" }, { "context_id": "b_63ebbf87a7c1", "observation": "red umbrella is visible" }, { "context_id": "b_442bffc55440", "observation": "red umbrella is visible" } ], "uncertainty": "red umbrella is consistently visible across all context images", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "black and white umbrella", "grounding_queries": [ "black and white umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_d5be48474254", "observation": "black and white umbrella is visible" }, { "context_id": "b_a38e3ce498d5", "observation": "black and white umbrella is visible" }, { "context_id": "b_63ebbf87a7c1", "observation": "black and white umbrella is visible" }, { "context_id": "b_442bffc55440", "observation": "black and white umbrella is visible" } ], "uncertainty": "black and white umbrella is consistently visible across all context images", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r3", "cause": "transient_occluder", "semantic_description": "mannequin shirt", "grounding_queries": [ "mannequin shirt" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_a38e3ce498d5", "observation": "mannequin shirt is visible" }, { "context_id": "b_63ebbf87a7c1", "observation": "mannequin shirt is visible" }, { "context_id": "b_442bffc55440", "observation": "mannequin shirt is visible" } ], "uncertainty": "mannequin shirt is consistently visible across two context images", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"red umbrella\",\n \"grounding_queries\": [\"red umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_d5be48474254\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_a38e3ce498d5\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_63ebbf87a7c1\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_442bffc55440\", \"observation\": \"red umbrella is visible\"}\n ],\n \"uncertainty\": \"red umbrella is consistently visible across all context images\",\n \"fallback_box_norm_xyxy\": [280, 258, 380, 348]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"black and white umbrella\",\n \"grounding_queries\": [\"black and white umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_d5be48474254\", \"observation\": \"black and white umbrella is visible\"},\n {\"context_id\": \"b_a38e3ce498d5\", \"observation\": \"black and white umbrella is visible\"},\n {\"context_id\": \"b_63ebbf87a7c1\", \"observation\": \"black and white umbrella is visible\"},\n {\"context_id\": \"b_442bffc55440\", \"observation\": \"black and white umbrella is visible\"}\n ],\n \"uncertainty\": \"black and white umbrella is consistently visible across all context images\",\n \"fallback_box_norm_xyxy\": [0, 138, 280, 476]\n },\n {\n \"region_id\": \"r3\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"mannequin shirt\",\n \"grounding_queries\": [\"mannequin shirt\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_a38e3ce498d5\", \"observation\": \"mannequin shirt is visible\"},\n {\"context_id\": \"b_63ebbf87a7c1\", \"observation\": \"mannequin shirt is visible\"},\n {\"context_id\": \"b_442bffc55440\", \"observation\": \"mannequin shirt is visible\"}\n ],\n \"uncertainty\": \"mannequin shirt is consistently visible across two context images\",\n \"fallback_box_norm_xyxy\": [168, 308, 310, 476]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_1aede9f60837", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [], "final_mask_path": "masks/b_1aede9f60837.png", "routing_mask_path": "masks/b_1aede9f60837_routing.png", "predicted_fraction": 0.0 }, "raw_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.24996883133074632 }, "routing_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.24996883133074632 }, "box_metrics": { "gt_pixel_coverage": 0.0, "area_precision": 0.0, "box_fraction": 0.0 }, "frame_detected": false, "gt_ambiguity": { "median_rgb_max_difference": 0.011764705181121826, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_a38e3ce498d5", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0003.JPG", "source_split": "train", "source_frame_id": "0003", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0003.JPG", "context_ids": [ "b_63ebbf87a7c1", "b_1aede9f60837", "b_442bffc55440", "b_6c74696a698a" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_a38e3ce498d5", "target_sha256": "1f9ba7f01a8ec6049827973e61b97173ff02690eff8dcb5da68e289ff5445ea2", "context_ids": [ "b_63ebbf87a7c1", "b_1aede9f60837", "b_442bffc55440", "b_6c74696a698a" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "mannequin with shirt", "grounding_queries": [ "mannequin with shirt" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_63ebbf87a7c1", "observation": "mannequin with shirt is visible" }, { "context_id": "b_1aede9f60837", "observation": "mannequin with shirt is visible" }, { "context_id": "b_442bffc55440", "observation": "mannequin with shirt is visible" }, { "context_id": "b_6c74696a698a", "observation": "mannequin with shirt is visible" } ], "uncertainty": "mannequin with shirt is consistently visible across all context images", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "red umbrella", "grounding_queries": [ "red umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_63ebbf87a7c1", "observation": "red umbrella is visible" }, { "context_id": "b_1aede9f60837", "observation": "red umbrella is visible" }, { "context_id": "b_442bffc55440", "observation": "red umbrella is visible" }, { "context_id": "b_6c74696a698a", "observation": "red umbrella is visible" } ], "uncertainty": "red umbrella is consistently visible across all context images", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r3", "cause": "transient_occluder", "semantic_description": "sign with Chinese characters", "grounding_queries": [ "sign with Chinese characters" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_63ebbf87a7c1", "observation": "sign with Chinese characters is visible" }, { "context_id": "b_1aede9f60837", "observation": "sign with Chinese characters is visible" }, { "context_id": "b_442bffc55440", "observation": "sign with Chinese characters is visible" }, { "context_id": "b_6c74696a698a", "observation": "sign with Chinese characters is visible" } ], "uncertainty": "sign with Chinese characters is consistently visible across all context images", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"mannequin with shirt\",\n \"grounding_queries\": [\"mannequin with shirt\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_63ebbf87a7c1\", \"observation\": \"mannequin with shirt is visible\"},\n {\"context_id\": \"b_1aede9f60837\", \"observation\": \"mannequin with shirt is visible\"},\n {\"context_id\": \"b_442bffc55440\", \"observation\": \"mannequin with shirt is visible\"},\n {\"context_id\": \"b_6c74696a698a\", \"observation\": \"mannequin with shirt is visible\"}\n ],\n \"uncertainty\": \"mannequin with shirt is consistently visible across all context images\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"red umbrella\",\n \"grounding_queries\": [\"red umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_63ebbf87a7c1\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_1aede9f60837\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_442bffc55440\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_6c74696a698a\", \"observation\": \"red umbrella is visible\"}\n ],\n \"uncertainty\": \"red umbrella is consistently visible across all context images\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r3\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"sign with Chinese characters\",\n \"grounding_queries\": [\"sign with Chinese characters\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_63ebbf87a7c1\", \"observation\": \"sign with Chinese characters is visible\"},\n {\"context_id\": \"b_1aede9f60837\", \"observation\": \"sign with Chinese characters is visible\"},\n {\"context_id\": \"b_442bffc55440\", \"observation\": \"sign with Chinese characters is visible\"},\n {\"context_id\": \"b_6c74696a698a\", \"observation\": \"sign with Chinese characters is visible\"}\n ],\n \"uncertainty\": \"sign with Chinese characters is consistently visible across all context images\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_a38e3ce498d5", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [], "final_mask_path": "masks/b_a38e3ce498d5.png", "routing_mask_path": "masks/b_a38e3ce498d5_routing.png", "predicted_fraction": 0.0 }, "raw_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.12777303754266212 }, "routing_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.12777303754266212 }, "box_metrics": { "gt_pixel_coverage": 0.0, "area_precision": 0.0, "box_fraction": 0.0 }, "frame_detected": false, "gt_ambiguity": { "median_rgb_max_difference": 0.00784313678741455, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_63ebbf87a7c1", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0004.JPG", "source_split": "train", "source_frame_id": "0004", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0004.JPG", "context_ids": [ "b_442bffc55440", "b_a38e3ce498d5", "b_6c74696a698a", "b_e21902361190" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_63ebbf87a7c1", "target_sha256": "2e7c8d2413cc7d1ba3ba9cc698dc06124e65069ca702187a418ca895b8afc20a", "context_ids": [ "b_442bffc55440", "b_a38e3ce498d5", "b_6c74696a698a", "b_e21902361190" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "red umbrella", "grounding_queries": [ "red umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_442bffc55440", "observation": "umbrella is visible" }, { "context_id": "b_a38e3ce498d5", "observation": "umbrella is visible" }, { "context_id": "b_6c74696a698a", "observation": "umbrella is visible" }, { "context_id": "b_e21902361190", "observation": "umbrella is visible" } ], "uncertainty": "umbrella is consistently visible across all contexts", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "sign with Chinese characters", "grounding_queries": [ "sign with Chinese characters" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_442bffc55440", "observation": "sign is visible" }, { "context_id": "b_a38e3ce498d5", "observation": "sign is visible" }, { "context_id": "b_6c74696a698a", "observation": "sign is visible" }, { "context_id": "b_e21902361190", "observation": "sign is visible" } ], "uncertainty": "sign is consistently visible across all contexts", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"red umbrella\",\n \"grounding_queries\": [\"red umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_442bffc55440\", \"observation\": \"umbrella is visible\"},\n {\"context_id\": \"b_a38e3ce498d5\", \"observation\": \"umbrella is visible\"},\n {\"context_id\": \"b_6c74696a698a\", \"observation\": \"umbrella is visible\"},\n {\"context_id\": \"b_e21902361190\", \"observation\": \"umbrella is visible\"}\n ],\n \"uncertainty\": \"umbrella is consistently visible across all contexts\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"sign with Chinese characters\",\n \"grounding_queries\": [\"sign with Chinese characters\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_442bffc55440\", \"observation\": \"sign is visible\"},\n {\"context_id\": \"b_a38e3ce498d5\", \"observation\": \"sign is visible\"},\n {\"context_id\": \"b_6c74696a698a\", \"observation\": \"sign is visible\"},\n {\"context_id\": \"b_e21902361190\", \"observation\": \"sign is visible\"}\n ],\n \"uncertainty\": \"sign is consistently visible across all contexts\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_63ebbf87a7c1", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [], "final_mask_path": "masks/b_63ebbf87a7c1.png", "routing_mask_path": "masks/b_63ebbf87a7c1_routing.png", "predicted_fraction": 0.0 }, "raw_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.23910216700172987 }, "routing_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.23910216700172987 }, "box_metrics": { "gt_pixel_coverage": 0.0, "area_precision": 0.0, "box_fraction": 0.0 }, "frame_detected": false, "gt_ambiguity": { "median_rgb_max_difference": 0.011764705181121826, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_6c74696a698a", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0005.JPG", "source_split": "train", "source_frame_id": "0005", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0005.JPG", "context_ids": [ "b_c7168d6511cd", "b_63ebbf87a7c1", "b_e21902361190", "b_442bffc55440" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_6c74696a698a", "target_sha256": "692c1ff802d24d43d30ea964cbb5401b228209287c7e3cb45fe06d378039eadc", "context_ids": [ "b_c7168d6511cd", "b_63ebbf87a7c1", "b_e21902361190", "b_442bffc55440" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "red umbrella", "grounding_queries": [ "red umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_c7168d6511cd", "observation": "red umbrella is visible" }, { "context_id": "b_63ebbf87a7c1", "observation": "red umbrella is visible" }, { "context_id": "b_e21902361190", "observation": "red umbrella is visible" }, { "context_id": "b_442bffc55440", "observation": "red umbrella is visible" } ], "uncertainty": "red umbrella is consistently visible across all context images", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "white sign with red text", "grounding_queries": [ "white sign with red text" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_c7168d6511cd", "observation": "white sign with red text is visible" }, { "context_id": "b_63ebbf87a7c1", "observation": "white sign with red text is visible" }, { "context_id": "b_e21902361190", "observation": "white sign with red text is visible" }, { "context_id": "b_442bffc55440", "observation": "white sign with red text is visible" } ], "uncertainty": "white sign with red text is consistently visible across all context images", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"red umbrella\",\n \"grounding_queries\": [\"red umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_c7168d6511cd\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_63ebbf87a7c1\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_e21902361190\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_442bffc55440\", \"observation\": \"red umbrella is visible\"}\n ],\n \"uncertainty\": \"red umbrella is consistently visible across all context images\",\n \"fallback_box_norm_xyxy\": [278, 250, 382, 340]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"white sign with red text\",\n \"grounding_queries\": [\"white sign with red text\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_c7168d6511cd\", \"observation\": \"white sign with red text is visible\"},\n {\"context_id\": \"b_63ebbf87a7c1\", \"observation\": \"white sign with red text is visible\"},\n {\"context_id\": \"b_e21902361190\", \"observation\": \"white sign with red text is visible\"},\n {\"context_id\": \"b_442bffc55440\", \"observation\": \"white sign with red text is visible\"}\n ],\n \"uncertainty\": \"white sign with red text is consistently visible across all context images\",\n \"fallback_box_norm_xyxy\": [170, 250, 234, 300]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_6c74696a698a", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [], "final_mask_path": "masks/b_6c74696a698a.png", "routing_mask_path": "masks/b_6c74696a698a_routing.png", "predicted_fraction": 0.0 }, "raw_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.020396971963782005 }, "routing_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.020396971963782005 }, "box_metrics": { "gt_pixel_coverage": 0.0, "area_precision": 0.0, "box_fraction": 0.0 }, "frame_detected": false, "gt_ambiguity": { "median_rgb_max_difference": 0.00784313678741455, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_c7168d6511cd", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0006.JPG", "source_split": "train", "source_frame_id": "0006", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0006.JPG", "context_ids": [ "b_6c74696a698a", "b_e21902361190", "b_f2d05c389498", "b_f9646cc9c954" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_c7168d6511cd", "target_sha256": "a6ed36e8527621f968cb684b2277d0e217f8744eec4d5bbc6be1c9a93d1e5b11", "context_ids": [ "b_6c74696a698a", "b_e21902361190", "b_f2d05c389498", "b_f9646cc9c954" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "red umbrella", "grounding_queries": [ "red umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_6c74696a698a", "observation": "red umbrella is visible" }, { "context_id": "b_e21902361190", "observation": "red umbrella is visible" }, { "context_id": "b_f2d05c389498", "observation": "red umbrella is visible" }, { "context_id": "b_f9646cc9c954", "observation": "red umbrella is visible" } ], "uncertainty": "red umbrella is consistently visible across all context images", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "black and white umbrella", "grounding_queries": [ "black and white umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_f9646cc9c954", "observation": "black and white umbrella is visible" } ], "uncertainty": "black and white umbrella is only visible in one context image", "fallback_box_norm_xyxy": null, "routing_eligible": true, "consistency_filter": "passed" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"red umbrella\",\n \"grounding_queries\": [\"red umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_6c74696a698a\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_e21902361190\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_f2d05c389498\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_f9646cc9c954\", \"observation\": \"red umbrella is visible\"}\n ],\n \"uncertainty\": \"red umbrella is consistently visible across all context images\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"black and white umbrella\",\n \"grounding_queries\": [\"black and white umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_f9646cc9c954\", \"observation\": \"black and white umbrella is visible\"}\n ],\n \"uncertainty\": \"black and white umbrella is only visible in one context image\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_c7168d6511cd", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [ { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "black and white umbrella", "queries": [ "black and white umbrella" ], "source": "grounding", "boxes_xyxy": [], "box_scores": [], "box_queries": [], "sam_predicted_iou": [], "mask_path": "masks/b_c7168d6511cd_r2.png" } ], "final_mask_path": "masks/b_c7168d6511cd.png", "routing_mask_path": "masks/b_c7168d6511cd_routing.png", "predicted_fraction": 0.0 }, "raw_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.45545412751102593 }, "routing_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.45545412751102593 }, "box_metrics": { "gt_pixel_coverage": 0.0, "area_precision": 0.0, "box_fraction": 0.0 }, "frame_detected": false, "gt_ambiguity": { "median_rgb_max_difference": 0.027450978755950928, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_f9646cc9c954", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0007.JPG", "source_split": "train", "source_frame_id": "0007", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0007.JPG", "context_ids": [ "b_f2d05c389498", "b_1ea940c8ac4e", "b_39a6a990391f", "b_c7168d6511cd" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_f9646cc9c954", "target_sha256": "0f951d7fb7e2f9b4f41f6adca3e1ca2cb66f91a97ab2476d57248824398134e6", "context_ids": [ "b_f2d05c389498", "b_1ea940c8ac4e", "b_39a6a990391f", "b_c7168d6511cd" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "black and white umbrella", "grounding_queries": [ "black and white umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_f2d05c389498", "observation": "umbrella is visible" }, { "context_id": "b_1ea940c8ac4e", "observation": "umbrella is visible" }, { "context_id": "b_39a6a990391f", "observation": "umbrella is visible" }, { "context_id": "b_c7168d6511cd", "observation": "umbrella is visible" } ], "uncertainty": "umbrella is consistently visible across all contexts", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "red umbrella", "grounding_queries": [ "red umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_f2d05c389498", "observation": "umbrella is visible" }, { "context_id": "b_1ea940c8ac4e", "observation": "umbrella is visible" }, { "context_id": "b_39a6a990391f", "observation": "umbrella is visible" }, { "context_id": "b_c7168d6511cd", "observation": "umbrella is visible" } ], "uncertainty": "umbrella is consistently visible across all contexts", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r3", "cause": "transient_occluder", "semantic_description": "flower arrangement", "grounding_queries": [ "flower arrangement" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_f2d05c389498", "observation": "flower arrangement is visible" }, { "context_id": "b_1ea940c8ac4e", "observation": "flower arrangement is visible" }, { "context_id": "b_39a6a990391f", "observation": "flower arrangement is visible" }, { "context_id": "b_c7168d6511cd", "observation": "flower arrangement is visible" } ], "uncertainty": "flower arrangement is consistently visible across all contexts", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"black and white umbrella\",\n \"grounding_queries\": [\"black and white umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_f2d05c389498\", \"observation\": \"umbrella is visible\"},\n {\"context_id\": \"b_1ea940c8ac4e\", \"observation\": \"umbrella is visible\"},\n {\"context_id\": \"b_39a6a990391f\", \"observation\": \"umbrella is visible\"},\n {\"context_id\": \"b_c7168d6511cd\", \"observation\": \"umbrella is visible\"}\n ],\n \"uncertainty\": \"umbrella is consistently visible across all contexts\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"red umbrella\",\n \"grounding_queries\": [\"red umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_f2d05c389498\", \"observation\": \"umbrella is visible\"},\n {\"context_id\": \"b_1ea940c8ac4e\", \"observation\": \"umbrella is visible\"},\n {\"context_id\": \"b_39a6a990391f\", \"observation\": \"umbrella is visible\"},\n {\"context_id\": \"b_c7168d6511cd\", \"observation\": \"umbrella is visible\"}\n ],\n \"uncertainty\": \"umbrella is consistently visible across all contexts\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r3\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"flower arrangement\",\n \"grounding_queries\": [\"flower arrangement\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_f2d05c389498\", \"observation\": \"flower arrangement is visible\"},\n {\"context_id\": \"b_1ea940c8ac4e\", \"observation\": \"flower arrangement is visible\"},\n {\"context_id\": \"b_39a6a990391f\", \"observation\": \"flower arrangement is visible\"},\n {\"context_id\": \"b_c7168d6511cd\", \"observation\": \"flower arrangement is visible\"}\n ],\n \"uncertainty\": \"flower arrangement is consistently visible across all contexts\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_f9646cc9c954", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [], "final_mask_path": "masks/b_f9646cc9c954.png", "routing_mask_path": "masks/b_f9646cc9c954_routing.png", "predicted_fraction": 0.0 }, "raw_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.3984047485467608 }, "routing_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.3984047485467608 }, "box_metrics": { "gt_pixel_coverage": 0.0, "area_precision": 0.0, "box_fraction": 0.0 }, "frame_detected": false, "gt_ambiguity": { "median_rgb_max_difference": 0.019607841968536377, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_1ea940c8ac4e", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0008.JPG", "source_split": "train", "source_frame_id": "0008", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0008.JPG", "context_ids": [ "b_39a6a990391f", "b_31faa8f0af47", "b_f9646cc9c954", "b_0ed8205374dd" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_1ea940c8ac4e", "target_sha256": "5939628718d60686b21ce724529ae903572b899fa39971c25aa12f559a30b34d", "context_ids": [ "b_39a6a990391f", "b_31faa8f0af47", "b_f9646cc9c954", "b_0ed8205374dd" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "red umbrella", "grounding_queries": [ "red umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_39a6a990391f", "observation": "red umbrella is visible" }, { "context_id": "b_31faa8f0af47", "observation": "red umbrella is visible" }, { "context_id": "b_f9646cc9c954", "observation": "red umbrella is visible" }, { "context_id": "b_0ed8205374dd", "observation": "red umbrella is visible" } ], "uncertainty": "red umbrella is consistently visible across all context images", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "black umbrella", "grounding_queries": [ "black umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_39a6a990391f", "observation": "black umbrella is visible" }, { "context_id": "b_31faa8f0af47", "observation": "black umbrella is visible" }, { "context_id": "b_f9646cc9c954", "observation": "black umbrella is visible" }, { "context_id": "b_0ed8205374dd", "observation": "black umbrella is visible" } ], "uncertainty": "black umbrella is consistently visible across all context images", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r3", "cause": "transient_occluder", "semantic_description": "green wall", "grounding_queries": [ "green wall" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_39a6a990391f", "observation": "green wall is visible" }, { "context_id": "b_31faa8f0af47", "observation": "green wall is visible" }, { "context_id": "b_f9646cc9c954", "observation": "green wall is visible" }, { "context_id": "b_0ed8205374dd", "observation": "green wall is visible" } ], "uncertainty": "green wall is consistently visible across all context images", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"red umbrella\",\n \"grounding_queries\": [\"red umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_39a6a990391f\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_31faa8f0af47\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_f9646cc9c954\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_0ed8205374dd\", \"observation\": \"red umbrella is visible\"}\n ],\n \"uncertainty\": \"red umbrella is consistently visible across all context images\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"black umbrella\",\n \"grounding_queries\": [\"black umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_39a6a990391f\", \"observation\": \"black umbrella is visible\"},\n {\"context_id\": \"b_31faa8f0af47\", \"observation\": \"black umbrella is visible\"},\n {\"context_id\": \"b_f9646cc9c954\", \"observation\": \"black umbrella is visible\"},\n {\"context_id\": \"b_0ed8205374dd\", \"observation\": \"black umbrella is visible\"}\n ],\n \"uncertainty\": \"black umbrella is consistently visible across all context images\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r3\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"green wall\",\n \"grounding_queries\": [\"green wall\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_39a6a990391f\", \"observation\": \"green wall is visible\"},\n {\"context_id\": \"b_31faa8f0af47\", \"observation\": \"green wall is visible\"},\n {\"context_id\": \"b_f9646cc9c954\", \"observation\": \"green wall is visible\"},\n {\"context_id\": \"b_0ed8205374dd\", \"observation\": \"green wall is visible\"}\n ],\n \"uncertainty\": \"green wall is consistently visible across all context images\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_1ea940c8ac4e", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [], "final_mask_path": "masks/b_1ea940c8ac4e.png", "routing_mask_path": "masks/b_1ea940c8ac4e_routing.png", "predicted_fraction": 0.0 }, "raw_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.11082263858369566 }, "routing_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.11082263858369566 }, "box_metrics": { "gt_pixel_coverage": 0.0, "area_precision": 0.0, "box_fraction": 0.0 }, "frame_detected": false, "gt_ambiguity": { "median_rgb_max_difference": 0.011764705181121826, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_31faa8f0af47", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0009.JPG", "source_split": "train", "source_frame_id": "0009", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0009.JPG", "context_ids": [ "b_0ed8205374dd", "b_1ea940c8ac4e", "b_39a6a990391f", "b_57cf54a3bbc0" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_31faa8f0af47", "target_sha256": "8522b8e9cf92d396f1857e8197dbba361ef53ce127af536586946128b5e576d0", "context_ids": [ "b_0ed8205374dd", "b_1ea940c8ac4e", "b_39a6a990391f", "b_57cf54a3bbc0" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "black curtain", "grounding_queries": [ "black curtain" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_0ed8205374dd", "observation": "black curtain is absent" }, { "context_id": "b_1ea940c8ac4e", "observation": "black curtain is absent" }, { "context_id": "b_39a6a990391f", "observation": "black curtain is absent" }, { "context_id": "b_57cf54a3bbc0", "observation": "black curtain is absent" } ], "uncertainty": "black curtain is consistently absent across all context images", "fallback_box_norm_xyxy": null, "routing_eligible": true, "consistency_filter": "passed" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"black curtain\",\n \"grounding_queries\": [\"black curtain\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_0ed8205374dd\", \"observation\": \"black curtain is absent\"},\n {\"context_id\": \"b_1ea940c8ac4e\", \"observation\": \"black curtain is absent\"},\n {\"context_id\": \"b_39a6a990391f\", \"observation\": \"black curtain is absent\"},\n {\"context_id\": \"b_57cf54a3bbc0\", \"observation\": \"black curtain is absent\"}\n ],\n \"uncertainty\": \"black curtain is consistently absent across all context images\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_31faa8f0af47", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "black curtain", "queries": [ "black curtain" ], "source": "grounding", "boxes_xyxy": [], "box_scores": [], "box_queries": [], "sam_predicted_iou": [], "mask_path": "masks/b_31faa8f0af47_r1.png" } ], "final_mask_path": "masks/b_31faa8f0af47.png", "routing_mask_path": "masks/b_31faa8f0af47_routing.png", "predicted_fraction": 0.0 }, "raw_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.38210548256892174 }, "routing_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.38210548256892174 }, "box_metrics": { "gt_pixel_coverage": 0.0, "area_precision": 0.0, "box_fraction": 0.0 }, "frame_detected": false, "gt_ambiguity": { "median_rgb_max_difference": 0.019607841968536377, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_0ed8205374dd", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0010.JPG", "source_split": "train", "source_frame_id": "0010", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0010.JPG", "context_ids": [ "b_31faa8f0af47", "b_57cf54a3bbc0", "b_c7dd5015d023", "b_1ea940c8ac4e" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_0ed8205374dd", "target_sha256": "9fc4d940f95eac42ee267d66fd919fd55d45c2ccc498dcf4dfece29c22bbdabd", "context_ids": [ "b_31faa8f0af47", "b_57cf54a3bbc0", "b_c7dd5015d023", "b_1ea940c8ac4e" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "black curtain", "grounding_queries": [ "black curtain" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_31faa8f0af47", "observation": "black curtain partially obscures the teddy bear" }, { "context_id": "b_57cf54a3bbc0", "observation": "black curtain fully covers the teddy bear" }, { "context_id": "b_c7dd5015d023", "observation": "black curtain is not visible" }, { "context_id": "b_1ea940c8ac4e", "observation": "black curtain is not visible" } ], "uncertainty": "black curtain is only visible in one context image", "fallback_box_norm_xyxy": null, "routing_eligible": true, "consistency_filter": "passed" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "brown cushion", "grounding_queries": [ "brown cushion" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_31faa8f0af47", "observation": "brown cushion is visible" }, { "context_id": "b_57cf54a3bbc0", "observation": "brown cushion is visible" }, { "context_id": "b_c7dd5015d023", "observation": "brown cushion is visible" }, { "context_id": "b_1ea940c8ac4e", "observation": "brown cushion is visible" } ], "uncertainty": "brown cushion is consistently visible across all context images", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r3", "cause": "transient_occluder", "semantic_description": "red umbrella", "grounding_queries": [ "red umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_31faa8f0af47", "observation": "red umbrella is visible" }, { "context_id": "b_57cf54a3bbc0", "observation": "red umbrella is visible" }, { "context_id": "b_c7dd5015d023", "observation": "red umbrella is visible" }, { "context_id": "b_1ea940c8ac4e", "observation": "red umbrella is visible" } ], "uncertainty": "red umbrella is consistently visible across all context images", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"black curtain\",\n \"grounding_queries\": [\"black curtain\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_31faa8f0af47\", \"observation\": \"black curtain partially obscures the teddy bear\"},\n {\"context_id\": \"b_57cf54a3bbc0\", \"observation\": \"black curtain fully covers the teddy bear\"},\n {\"context_id\": \"b_c7dd5015d023\", \"observation\": \"black curtain is not visible\"},\n {\"context_id\": \"b_1ea940c8ac4e\", \"observation\": \"black curtain is not visible\"}\n ],\n \"uncertainty\": \"black curtain is only visible in one context image\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"brown cushion\",\n \"grounding_queries\": [\"brown cushion\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_31faa8f0af47\", \"observation\": \"brown cushion is visible\"},\n {\"context_id\": \"b_57cf54a3bbc0\", \"observation\": \"brown cushion is visible\"},\n {\"context_id\": \"b_c7dd5015d023\", \"observation\": \"brown cushion is visible\"},\n {\"context_id\": \"b_1ea940c8ac4e\", \"observation\": \"brown cushion is visible\"}\n ],\n \"uncertainty\": \"brown cushion is consistently visible across all context images\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r3\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"red umbrella\",\n \"grounding_queries\": [\"red umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_31faa8f0af47\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_57cf54a3bbc0\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_c7dd5015d023\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_1ea940c8ac4e\", \"observation\": \"red umbrella is visible\"}\n ],\n \"uncertainty\": \"red umbrella is consistently visible across all context images\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_0ed8205374dd", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "black curtain", "queries": [ "black curtain" ], "source": "grounding", "boxes_xyxy": [], "box_scores": [], "box_queries": [], "sam_predicted_iou": [], "mask_path": "masks/b_0ed8205374dd_r1.png" } ], "final_mask_path": "masks/b_0ed8205374dd.png", "routing_mask_path": "masks/b_0ed8205374dd_routing.png", "predicted_fraction": 0.0 }, "raw_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.10644295354309848 }, "routing_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.10644295354309848 }, "box_metrics": { "gt_pixel_coverage": 0.0, "area_precision": 0.0, "box_fraction": 0.0 }, "frame_detected": false, "gt_ambiguity": { "median_rgb_max_difference": 0.011764705181121826, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_c7dd5015d023", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0011.JPG", "source_split": "train", "source_frame_id": "0011", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0011.JPG", "context_ids": [ "b_3fad5029fb0d", "b_57cf54a3bbc0", "b_0a52973d1b8d", "b_0ed8205374dd" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_c7dd5015d023", "target_sha256": "1a7c6617a66bee68a33a1c94e36afbcc21f1f40af2a2d54f760cdb68d0fe29c3", "context_ids": [ "b_3fad5029fb0d", "b_57cf54a3bbc0", "b_0a52973d1b8d", "b_0ed8205374dd" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "red umbrella", "grounding_queries": [ "red umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_3fad5029fb0d", "observation": "red umbrella partially obscured" }, { "context_id": "b_57cf54a3bbc0", "observation": "red umbrella fully visible" }, { "context_id": "b_0a52973d1b8d", "observation": "red umbrella fully visible" }, { "context_id": "b_0ed8205374dd", "observation": "red umbrella fully visible" } ], "uncertainty": "red umbrella consistently visible across all contexts", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "green curtain", "grounding_queries": [ "green curtain" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_3fad5029fb0d", "observation": "green curtain partially obscures the teddy bear" }, { "context_id": "b_57cf54a3bbc0", "observation": "green curtain fully visible" }, { "context_id": "b_0a52973d1b8d", "observation": "green curtain fully visible" }, { "context_id": "b_0ed8205374dd", "observation": "green curtain fully visible" } ], "uncertainty": "green curtain consistently visible across all contexts", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"red umbrella\",\n \"grounding_queries\": [\"red umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_3fad5029fb0d\", \"observation\": \"red umbrella partially obscured\"},\n {\"context_id\": \"b_57cf54a3bbc0\", \"observation\": \"red umbrella fully visible\"},\n {\"context_id\": \"b_0a52973d1b8d\", \"observation\": \"red umbrella fully visible\"},\n {\"context_id\": \"b_0ed8205374dd\", \"observation\": \"red umbrella fully visible\"}\n ],\n \"uncertainty\": \"red umbrella consistently visible across all contexts\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"green curtain\",\n \"grounding_queries\": [\"green curtain\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_3fad5029fb0d\", \"observation\": \"green curtain partially obscures the teddy bear\"},\n {\"context_id\": \"b_57cf54a3bbc0\", \"observation\": \"green curtain fully visible\"},\n {\"context_id\": \"b_0a52973d1b8d\", \"observation\": \"green curtain fully visible\"},\n {\"context_id\": \"b_0ed8205374dd\", \"observation\": \"green curtain fully visible\"}\n ],\n \"uncertainty\": \"green curtain consistently visible across all contexts\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_c7dd5015d023", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [], "final_mask_path": "masks/b_c7dd5015d023.png", "routing_mask_path": "masks/b_c7dd5015d023_routing.png", "predicted_fraction": 0.0 }, "raw_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.15088606682562689 }, "routing_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.15088606682562689 }, "box_metrics": { "gt_pixel_coverage": 0.0, "area_precision": 0.0, "box_fraction": 0.0 }, "frame_detected": false, "gt_ambiguity": { "median_rgb_max_difference": 0.011764705181121826, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_3fad5029fb0d", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0012.JPG", "source_split": "train", "source_frame_id": "0012", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0012.JPG", "context_ids": [ "b_6125c5000375", "b_48b647235f33", "b_c7dd5015d023", "b_0a52973d1b8d" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_3fad5029fb0d", "target_sha256": "e986422f654342f73f45ef0a77076801f4c6ba007336b65b72a5b0254189af77", "context_ids": [ "b_6125c5000375", "b_48b647235f33", "b_c7dd5015d023", "b_0a52973d1b8d" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "curtain", "grounding_queries": [ "curtain" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_6125c5000375", "observation": "curtain partially visible" }, { "context_id": "b_48b647235f33", "observation": "curtain fully visible" }, { "context_id": "b_c7dd5015d023", "observation": "curtain fully visible" }, { "context_id": "b_0a52973d1b8d", "observation": "curtain fully visible" } ], "uncertainty": "curtain is consistently visible across all contexts", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"curtain\",\n \"grounding_queries\": [\"curtain\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_6125c5000375\", \"observation\": \"curtain partially visible\"},\n {\"context_id\": \"b_48b647235f33\", \"observation\": \"curtain fully visible\"},\n {\"context_id\": \"b_c7dd5015d023\", \"observation\": \"curtain fully visible\"},\n {\"context_id\": \"b_0a52973d1b8d\", \"observation\": \"curtain fully visible\"}\n ],\n \"uncertainty\": \"curtain is consistently visible across all contexts\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_3fad5029fb0d", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [], "final_mask_path": "masks/b_3fad5029fb0d.png", "routing_mask_path": "masks/b_3fad5029fb0d_routing.png", "predicted_fraction": 0.0 }, "raw_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.19238763694734054 }, "routing_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.19238763694734054 }, "box_metrics": { "gt_pixel_coverage": 0.0, "area_precision": 0.0, "box_fraction": 0.0 }, "frame_detected": false, "gt_ambiguity": { "median_rgb_max_difference": 0.0156862735748291, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_6125c5000375", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0013.JPG", "source_split": "train", "source_frame_id": "0013", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0013.JPG", "context_ids": [ "b_3fad5029fb0d", "b_48b647235f33", "b_2962b6747c5c", "b_0a52973d1b8d" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_6125c5000375", "target_sha256": "ceef144931ee2abde82fa3b7d28510e0c313a7be5c40c58dde0e35f1ce5d52ef", "context_ids": [ "b_3fad5029fb0d", "b_48b647235f33", "b_2962b6747c5c", "b_0a52973d1b8d" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "red cloth", "grounding_queries": [ "red cloth" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_3fad5029fb0d", "observation": "red cloth partially visible" }, { "context_id": "b_48b647235f33", "observation": "red cloth completely absent" }, { "context_id": "b_2962b6747c5c", "observation": "red cloth partially visible" }, { "context_id": "b_0a52973d1b8d", "observation": "red cloth partially visible" } ], "uncertainty": "red cloth may be partially obscured in some views", "fallback_box_norm_xyxy": [ 0.0, 0.0, 0.3, 1.0 ], "routing_eligible": true, "consistency_filter": "passed" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"red cloth\",\n \"grounding_queries\": [\"red cloth\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_3fad5029fb0d\", \"observation\": \"red cloth partially visible\"},\n {\"context_id\": \"b_48b647235f33\", \"observation\": \"red cloth completely absent\"},\n {\"context_id\": \"b_2962b6747c5c\", \"observation\": \"red cloth partially visible\"},\n {\"context_id\": \"b_0a52973d1b8d\", \"observation\": \"red cloth partially visible\"}\n ],\n \"uncertainty\": \"red cloth may be partially obscured in some views\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 0.3, 1.0]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_6125c5000375", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "red cloth", "queries": [ "red cloth" ], "source": "grounding", "boxes_xyxy": [ [ 106.89315795898438, 247.7123260498047, 676.0635986328125, 1165.3028564453125 ] ], "box_scores": [ 0.3131753206253052 ], "box_queries": [ "red cloth" ], "sam_predicted_iou": [ 0.99609375 ], "mask_path": "masks/b_6125c5000375_r1.png" } ], "final_mask_path": "masks/b_6125c5000375.png", "routing_mask_path": "masks/b_6125c5000375_routing.png", "predicted_fraction": 0.23016406408278398 }, "raw_mask_metrics": { "iou": 0.8724448636983735, "dice": 0.9318777611161885, "precision": 1.0, "recall": 0.8724448636983735, "safety_recall": 0.8724448636983735, "leakage": 0.12755513630162654, "overmask": 0.0, "boundary_f5": 0.025880600641205082, "iou_ignore_boundary5": 0.9033726206814819, "predicted_fraction": 0.23016406408278398, "gt_fraction": 0.26381502563623044 }, "routing_mask_metrics": { "iou": 0.8967687269478278, "dice": 0.9455751923861133, "precision": 0.9999938244666398, "recall": 0.896773693333801, "safety_recall": 0.896773693333801, "leakage": 0.103226306666199, "overmask": 1.9845981949417884e-06, "boundary_f5": 0.45161147598369206, "iou_ignore_boundary5": 0.9236515442871507, "predicted_fraction": 0.23658383592812504, "gt_fraction": 0.26381502563623044 }, "box_metrics": { "gt_pixel_coverage": 0.9123726470039856, "area_precision": 0.9472222594883215, "box_fraction": 0.25410890722645596 }, "frame_detected": true, "gt_ambiguity": { "median_rgb_max_difference": 0.011764705181121826, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_2962b6747c5c", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0014.JPG", "source_split": "train", "source_frame_id": "0014", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0014.JPG", "context_ids": [ "b_2d3a471eb3b2", "b_6125c5000375", "b_48b647235f33", "b_42d9140f0598" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_2962b6747c5c", "target_sha256": "694505997ce3fd02c783fc8b4875ecd868966c155a15378d4caa53c76d6f7fef", "context_ids": [ "b_2d3a471eb3b2", "b_6125c5000375", "b_48b647235f33", "b_42d9140f0598" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "black and white umbrella", "grounding_queries": [ "black and white umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_2d3a471eb3b2", "observation": "absent" }, { "context_id": "b_6125c5000375", "observation": "absent" }, { "context_id": "b_48b647235f33", "observation": "absent" }, { "context_id": "b_42d9140f0598", "observation": "absent" } ], "uncertainty": "no other views confirm absence", "fallback_box_norm_xyxy": null, "routing_eligible": true, "consistency_filter": "passed" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "red umbrella", "grounding_queries": [ "red umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_2d3a471eb3b2", "observation": "present" }, { "context_id": "b_6125c5000375", "observation": "absent" }, { "context_id": "b_48b647235f33", "observation": "present" }, { "context_id": "b_42d9140f0598", "observation": "absent" } ], "uncertainty": "no other views confirm absence", "fallback_box_norm_xyxy": null, "routing_eligible": true, "consistency_filter": "passed" }, { "region_id": "r3", "cause": "transient_occluder", "semantic_description": "blue umbrella", "grounding_queries": [ "blue umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_2d3a471eb3b2", "observation": "absent" }, { "context_id": "b_6125c5000375", "observation": "absent" }, { "context_id": "b_48b647235f33", "observation": "absent" }, { "context_id": "b_42d9140f0598", "observation": "absent" } ], "uncertainty": "no other views confirm absence", "fallback_box_norm_xyxy": null, "routing_eligible": true, "consistency_filter": "passed" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"black and white umbrella\",\n \"grounding_queries\": [\"black and white umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_2d3a471eb3b2\", \"observation\": \"absent\"},\n {\"context_id\": \"b_6125c5000375\", \"observation\": \"absent\"},\n {\"context_id\": \"b_48b647235f33\", \"observation\": \"absent\"},\n {\"context_id\": \"b_42d9140f0598\", \"observation\": \"absent\"}\n ],\n \"uncertainty\": \"no other views confirm absence\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"red umbrella\",\n \"grounding_queries\": [\"red umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_2d3a471eb3b2\", \"observation\": \"present\"},\n {\"context_id\": \"b_6125c5000375\", \"observation\": \"absent\"},\n {\"context_id\": \"b_48b647235f33\", \"observation\": \"present\"},\n {\"context_id\": \"b_42d9140f0598\", \"observation\": \"absent\"}\n ],\n \"uncertainty\": \"no other views confirm absence\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r3\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"blue umbrella\",\n \"grounding_queries\": [\"blue umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_2d3a471eb3b2\", \"observation\": \"absent\"},\n {\"context_id\": \"b_6125c5000375\", \"observation\": \"absent\"},\n {\"context_id\": \"b_48b647235f33\", \"observation\": \"absent\"},\n {\"context_id\": \"b_42d9140f0598\", \"observation\": \"absent\"}\n ],\n \"uncertainty\": \"no other views confirm absence\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_2962b6747c5c", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "black and white umbrella", "queries": [ "black and white umbrella" ], "source": "grounding", "boxes_xyxy": [], "box_scores": [], "box_queries": [], "sam_predicted_iou": [], "mask_path": "masks/b_2962b6747c5c_r1.png" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "red umbrella", "queries": [ "red umbrella" ], "source": "grounding", "boxes_xyxy": [ [ 646.9236450195312, 822.9298706054688, 966.198486328125, 1159.1925048828125 ] ], "box_scores": [ 0.28780749440193176 ], "box_queries": [ "red umbrella" ], "sam_predicted_iou": [ 0.984375 ], "mask_path": "masks/b_2962b6747c5c_r2.png" }, { "region_id": "r3", "cause": "transient_occluder", "semantic_description": "blue umbrella", "queries": [ "blue umbrella" ], "source": "grounding", "boxes_xyxy": [ [ 646.8992309570312, 822.8531494140625, 966.4400024414062, 1159.5849609375 ] ], "box_scores": [ 0.2961234748363495 ], "box_queries": [ "blue umbrella" ], "sam_predicted_iou": [ 0.984375 ], "mask_path": "masks/b_2962b6747c5c_r3.png" } ], "final_mask_path": "masks/b_2962b6747c5c.png", "routing_mask_path": "masks/b_2962b6747c5c_routing.png", "predicted_fraction": 0.037878212320974956 }, "raw_mask_metrics": { "iou": 0.3983456099984043, "dice": 0.5697384211030045, "precision": 0.9949856641423557, "recall": 0.39914689938673725, "safety_recall": 0.39914689938673725, "leakage": 0.6008531006132628, "overmask": 0.00020973797273729028, "boundary_f5": 0.06760471954187672, "iou_ignore_boundary5": 0.5162138060558565, "predicted_fraction": 0.037878212320974956, "gt_fraction": 0.09442207443078218 }, "routing_mask_metrics": { "iou": 0.42483968604114297, "dice": 0.5963333141309983, "precision": 0.9874915040005724, "recall": 0.42713829617136284, "safety_recall": 0.42713829617136284, "leakage": 0.5728617038286371, "overmask": 0.0005641413676959424, "boundary_f5": 0.12949185840615218, "iou_ignore_boundary5": 0.5377809870228029, "predicted_fraction": 0.040842157962815775, "gt_fraction": 0.09442207443078218 }, "box_metrics": { "gt_pixel_coverage": 0.46893713153944944, "area_precision": 0.8457252355748212, "box_fraction": 0.052355085168388736 }, "frame_detected": true, "gt_ambiguity": { "median_rgb_max_difference": 0.011764705181121826, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_2d3a471eb3b2", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0015.JPG", "source_split": "train", "source_frame_id": "0015", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0015.JPG", "context_ids": [ "b_2962b6747c5c", "b_3d50f98d854f", "b_42d9140f0598", "b_66a730d6ef90" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_2d3a471eb3b2", "target_sha256": "f014481f7f0078f0da0b52176b2bee217e86d6f0eadbe198bdd21db93b673616", "context_ids": [ "b_2962b6747c5c", "b_3d50f98d854f", "b_42d9140f0598", "b_66a730d6ef90" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "red umbrella", "grounding_queries": [ "red umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_2962b6747c5c", "observation": "red umbrella is visible" }, { "context_id": "b_3d50f98d854f", "observation": "red umbrella is visible" }, { "context_id": "b_42d9140f0598", "observation": "red umbrella is visible" }, { "context_id": "b_66a730d6ef90", "observation": "red umbrella is visible" } ], "uncertainty": "red umbrella is consistently visible across all context images", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "black and white umbrella", "grounding_queries": [ "black and white umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_2962b6747c5c", "observation": "black and white umbrella is visible" }, { "context_id": "b_3d50f98d854f", "observation": "black and white umbrella is visible" }, { "context_id": "b_42d9140f0598", "observation": "black and white umbrella is visible" }, { "context_id": "b_66a730d6ef90", "observation": "black and white umbrella is visible" } ], "uncertainty": "black and white umbrella is consistently visible across all context images", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"red umbrella\",\n \"grounding_queries\": [\"red umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_2962b6747c5c\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_3d50f98d854f\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_42d9140f0598\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_66a730d6ef90\", \"observation\": \"red umbrella is visible\"}\n ],\n \"uncertainty\": \"red umbrella is consistently visible across all context images\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"black and white umbrella\",\n \"grounding_queries\": [\"black and white umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_2962b6747c5c\", \"observation\": \"black and white umbrella is visible\"},\n {\"context_id\": \"b_3d50f98d854f\", \"observation\": \"black and white umbrella is visible\"},\n {\"context_id\": \"b_42d9140f0598\", \"observation\": \"black and white umbrella is visible\"},\n {\"context_id\": \"b_66a730d6ef90\", \"observation\": \"black and white umbrella is visible\"}\n ],\n \"uncertainty\": \"black and white umbrella is consistently visible across all context images\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_2d3a471eb3b2", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [], "final_mask_path": "masks/b_2d3a471eb3b2.png", "routing_mask_path": "masks/b_2d3a471eb3b2_routing.png", "predicted_fraction": 0.0 }, "raw_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.2572437935387349 }, "routing_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.2572437935387349 }, "box_metrics": { "gt_pixel_coverage": 0.0, "area_precision": 0.0, "box_fraction": 0.0 }, "frame_detected": false, "gt_ambiguity": { "median_rgb_max_difference": 0.011764705181121826, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_3d50f98d854f", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0016.JPG", "source_split": "train", "source_frame_id": "0016", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0016.JPG", "context_ids": [ "b_42d9140f0598", "b_66a730d6ef90", "b_4dd28c1b3b60", "b_2d3a471eb3b2" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_3d50f98d854f", "target_sha256": "b54ae4644bc87c5f8c5e614d9d4fd35577effb77828e182c45f19a0f901fdbbb", "context_ids": [ "b_42d9140f0598", "b_66a730d6ef90", "b_4dd28c1b3b60", "b_2d3a471eb3b2" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "mannequin", "grounding_queries": [ "mannequin" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_42d9140f0598", "observation": "mannequin appears in the foreground" }, { "context_id": "b_66a730d6ef90", "observation": "mannequin appears in the foreground" }, { "context_id": "b_4dd28c1b3b60", "observation": "mannequin appears in the foreground" }, { "context_id": "b_2d3a471eb3b2", "observation": "mannequin appears in the foreground" } ], "uncertainty": "mannequin is consistently visible across all context images", "fallback_box_norm_xyxy": [ 0.7, 0.3, 1.0, 0.8 ], "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "mirror", "grounding_queries": [ "mirror" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_4dd28c1b3b60", "observation": "mirror appears in the foreground" } ], "uncertainty": "mirror is only visible in one context image", "fallback_box_norm_xyxy": [ 0.6, 0.5, 0.9, 0.8 ], "routing_eligible": true, "consistency_filter": "passed" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"mannequin\",\n \"grounding_queries\": [\"mannequin\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_42d9140f0598\", \"observation\": \"mannequin appears in the foreground\"},\n {\"context_id\": \"b_66a730d6ef90\", \"observation\": \"mannequin appears in the foreground\"},\n {\"context_id\": \"b_4dd28c1b3b60\", \"observation\": \"mannequin appears in the foreground\"},\n {\"context_id\": \"b_2d3a471eb3b2\", \"observation\": \"mannequin appears in the foreground\"}\n ],\n \"uncertainty\": \"mannequin is consistently visible across all context images\",\n \"fallback_box_norm_xyxy\": [0.7, 0.3, 1.0, 0.8]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"mirror\",\n \"grounding_queries\": [\"mirror\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_4dd28c1b3b60\", \"observation\": \"mirror appears in the foreground\"}\n ],\n \"uncertainty\": \"mirror is only visible in one context image\",\n \"fallback_box_norm_xyxy\": [0.6, 0.5, 0.9, 0.8]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_3d50f98d854f", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [ { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "mirror", "queries": [ "mirror" ], "source": "grounding", "boxes_xyxy": [ [ 707.57080078125, 487.44549560546875, 1005.8277587890625, 673.9874877929688 ], [ 939.605712890625, 390.13330078125, 1174.2569580078125, 565.254150390625 ] ], "box_scores": [ 0.3109780251979828, 0.2871554493904114 ], "box_queries": [ "mirror", "mirror" ], "sam_predicted_iou": [ 0.9453125, 0.94921875 ], "mask_path": "masks/b_3d50f98d854f_r2.png" } ], "final_mask_path": "masks/b_3d50f98d854f.png", "routing_mask_path": "masks/b_3d50f98d854f_routing.png", "predicted_fraction": 0.02520717424844546 }, "raw_mask_metrics": { "iou": 0.06761076176326317, "dice": 0.12665807461812656, "precision": 0.7376494909097935, "recall": 0.0692766056277716, "safety_recall": 0.0692766056277716, "leakage": 0.9307233943722284, "overmask": 0.009039288519657573, "boundary_f5": 0.18058851221590533, "iou_ignore_boundary5": 0.05759626117444637, "predicted_fraction": 0.02520717424844546, "gt_fraction": 0.26840315115246155 }, "routing_mask_metrics": { "iou": 0.07158868030260578, "dice": 0.1336122368937116, "precision": 0.6724126587275479, "recall": 0.07417568460092465, "safety_recall": 0.07417568460092465, "leakage": 0.9258243153990754, "overmask": 0.013257711919692189, "boundary_f5": 0.15425897400045405, "iou_ignore_boundary5": 0.05783324938404651, "predicted_fraction": 0.02960828774915455, "gt_fraction": 0.26840315115246155 }, "box_metrics": { "gt_pixel_coverage": 0.07870279646685682, "area_precision": 0.4738986976662879, "box_fraction": 0.04457509311639939 }, "frame_detected": true, "gt_ambiguity": { "median_rgb_max_difference": 0.0156862735748291, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_4dd28c1b3b60", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0017.JPG", "source_split": "train", "source_frame_id": "0017", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0017.JPG", "context_ids": [ "b_d95203c68daa", "b_3d50f98d854f", "b_8ffaf4db41bf", "b_17642d7e1164" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_4dd28c1b3b60", "target_sha256": "b1bcc7035d539193f181a7bb44e8eae918a065ce8bb46ebee596b324287323c8", "context_ids": [ "b_d95203c68daa", "b_3d50f98d854f", "b_8ffaf4db41bf", "b_17642d7e1164" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "mirror", "grounding_queries": [ "mirror" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_d95203c68daa", "observation": "mirror is visible" }, { "context_id": "b_3d50f98d854f", "observation": "mirror is visible" }, { "context_id": "b_8ffaf4db41bf", "observation": "mirror is visible" }, { "context_id": "b_17642d7e1164", "observation": "mirror is visible" } ], "uncertainty": "mirror is consistently visible across all contexts", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "mirror", "grounding_queries": [ "mirror" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_d95203c68daa", "observation": "mirror is visible" }, { "context_id": "b_3d50f98d854f", "observation": "mirror is visible" }, { "context_id": "b_8ffaf4db41bf", "observation": "mirror is visible" }, { "context_id": "b_17642d7e1164", "observation": "mirror is visible" } ], "uncertainty": "mirror is consistently visible across all contexts", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"mirror\",\n \"grounding_queries\": [\"mirror\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_d95203c68daa\", \"observation\": \"mirror is visible\"},\n {\"context_id\": \"b_3d50f98d854f\", \"observation\": \"mirror is visible\"},\n {\"context_id\": \"b_8ffaf4db41bf\", \"observation\": \"mirror is visible\"},\n {\"context_id\": \"b_17642d7e1164\", \"observation\": \"mirror is visible\"}\n ],\n \"uncertainty\": \"mirror is consistently visible across all contexts\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"mirror\",\n \"grounding_queries\": [\"mirror\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_d95203c68daa\", \"observation\": \"mirror is visible\"},\n {\"context_id\": \"b_3d50f98d854f\", \"observation\": \"mirror is visible\"},\n {\"context_id\": \"b_8ffaf4db41bf\", \"observation\": \"mirror is visible\"},\n {\"context_id\": \"b_17642d7e1164\", \"observation\": \"mirror is visible\"}\n ],\n \"uncertainty\": \"mirror is consistently visible across all contexts\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_4dd28c1b3b60", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [], "final_mask_path": "masks/b_4dd28c1b3b60.png", "routing_mask_path": "masks/b_4dd28c1b3b60_routing.png", "predicted_fraction": 0.0 }, "raw_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.11984012420714697 }, "routing_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.11984012420714697 }, "box_metrics": { "gt_pixel_coverage": 0.0, "area_precision": 0.0, "box_fraction": 0.0 }, "frame_detected": false, "gt_ambiguity": { "median_rgb_max_difference": 0.00784313678741455, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_d95203c68daa", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0018.JPG", "source_split": "train", "source_frame_id": "0018", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0018.JPG", "context_ids": [ "b_4dd28c1b3b60", "b_17642d7e1164", "b_8ffaf4db41bf", "b_42d3a74baab7" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_d95203c68daa", "target_sha256": "01bc9f3c3918db7763a12902c0f0e86339b42f1d5c6a592d7fe1465a67e76508", "context_ids": [ "b_4dd28c1b3b60", "b_17642d7e1164", "b_8ffaf4db41bf", "b_42d3a74baab7" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "mirror", "grounding_queries": [ "mirror" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_4dd28c1b3b60", "observation": "mirror partially visible" }, { "context_id": "b_17642d7e1164", "observation": "mirror fully visible" }, { "context_id": "b_8ffaf4db41bf", "observation": "mirror partially visible" }, { "context_id": "b_42d3a74baab7", "observation": "mirror fully visible" } ], "uncertainty": "mirror's position varies slightly across views", "fallback_box_norm_xyxy": null, "routing_eligible": true, "consistency_filter": "passed" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "person holding mirror", "grounding_queries": [ "person holding mirror" ], "is_spatial": true, "confidence": 0.8, "context_evidence": [ { "context_id": "b_4dd28c1b3b60", "observation": "person not visible" }, { "context_id": "b_17642d7e1164", "observation": "person not visible" }, { "context_id": "b_8ffaf4db41bf", "observation": "person not visible" }, { "context_id": "b_42d3a74baab7", "observation": "person partially visible" } ], "uncertainty": "person's visibility is inconsistent", "fallback_box_norm_xyxy": null, "routing_eligible": true, "consistency_filter": "passed" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"mirror\",\n \"grounding_queries\": [\"mirror\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_4dd28c1b3b60\", \"observation\": \"mirror partially visible\"},\n {\"context_id\": \"b_17642d7e1164\", \"observation\": \"mirror fully visible\"},\n {\"context_id\": \"b_8ffaf4db41bf\", \"observation\": \"mirror partially visible\"},\n {\"context_id\": \"b_42d3a74baab7\", \"observation\": \"mirror fully visible\"}\n ],\n \"uncertainty\": \"mirror's position varies slightly across views\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"person holding mirror\",\n \"grounding_queries\": [\"person holding mirror\"],\n \"is_spatial\": true,\n \"confidence\": 0.8,\n \"context_evidence\": [\n {\"context_id\": \"b_4dd28c1b3b60\", \"observation\": \"person not visible\"},\n {\"context_id\": \"b_17642d7e1164\", \"observation\": \"person not visible\"},\n {\"context_id\": \"b_8ffaf4db41bf\", \"observation\": \"person not visible\"},\n {\"context_id\": \"b_42d3a74baab7\", \"observation\": \"person partially visible\"}\n ],\n \"uncertainty\": \"person's visibility is inconsistent\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_d95203c68daa", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "mirror", "queries": [ "mirror" ], "source": "grounding", "boxes_xyxy": [ [ 1013.426025390625, 278.7547912597656, 1307.5675048828125, 1166.7349853515625 ], [ 639.6337280273438, 490.9345703125, 934.0805053710938, 699.3328247070312 ] ], "box_scores": [ 0.3015284538269043, 0.26977986097335815 ], "box_queries": [ "mirror", "mirror" ], "sam_predicted_iou": [ 0.9296875, 0.9296875 ], "mask_path": "masks/b_d95203c68daa_r1.png" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "person holding mirror", "queries": [ "person holding mirror" ], "source": "grounding", "boxes_xyxy": [ [ 1014.04736328125, 278.1952209472656, 1307.2713623046875, 1166.5721435546875 ] ], "box_scores": [ 0.314229816198349 ], "box_queries": [ "person holding mirror" ], "sam_predicted_iou": [ 0.9296875 ], "mask_path": "masks/b_d95203c68daa_r2.png" } ], "final_mask_path": "masks/b_d95203c68daa.png", "routing_mask_path": "masks/b_d95203c68daa_routing.png", "predicted_fraction": 0.09696378200632724 }, "raw_mask_metrics": { "iou": 0.6082124257635363, "dice": 0.756383194185026, "precision": 0.8440833751883475, "recall": 0.6851918294124842, "safety_recall": 0.6851918294124842, "leakage": 0.3148081705875158, "overmask": 0.017169098167442262, "boundary_f5": 0.18696034604215656, "iou_ignore_boundary5": 0.643403242943595, "predicted_fraction": 0.09696378200632724, "gt_fraction": 0.11944905481010488 }, "routing_mask_metrics": { "iou": 0.6857349874527269, "dice": 0.81357389216786, "precision": 0.8374446962339485, "recall": 0.7910262160068496, "safety_recall": 0.7910262160068496, "leakage": 0.20897378399315042, "overmask": 0.020828793511769982, "boundary_f5": 0.3946239744743433, "iou_ignore_boundary5": 0.699450611476952, "predicted_fraction": 0.11282814764598625, "gt_fraction": 0.11944905481010488 }, "box_metrics": { "gt_pixel_coverage": 0.8608839238390346, "area_precision": 0.6528934308374947, "box_fraction": 0.15750161687471753 }, "frame_detected": true, "gt_ambiguity": { "median_rgb_max_difference": 0.00784313678741455, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_c0cd1b9b9c0e", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0019.JPG", "source_split": "train", "source_frame_id": "0019", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0019.JPG", "context_ids": [ "b_bc84988e0e0a", "b_42d3a74baab7", "b_c0a10b3aef20", "b_17642d7e1164" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_c0cd1b9b9c0e", "target_sha256": "fcb5d7fcc27dc2c89475f6d2923b6f666a06af139f14a1713be5fe487be7896b", "context_ids": [ "b_bc84988e0e0a", "b_42d3a74baab7", "b_c0a10b3aef20", "b_17642d7e1164" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "red umbrella", "grounding_queries": [ "red umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_bc84988e0e0a", "observation": "red umbrella is visible" }, { "context_id": "b_42d3a74baab7", "observation": "red umbrella is partially obscured" }, { "context_id": "b_c0a10b3aef20", "observation": "red umbrella is fully obscured" }, { "context_id": "b_17642d7e1164", "observation": "red umbrella is not visible" } ], "uncertainty": "red umbrella is only visible in one context", "fallback_box_norm_xyxy": [ 0.5, 0.4, 0.7, 0.5 ], "routing_eligible": true, "consistency_filter": "passed" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "scroll painting", "grounding_queries": [ "scroll painting" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_bc84988e0e0a", "observation": "scroll painting is visible" }, { "context_id": "b_42d3a74baab7", "observation": "scroll painting is partially obscured" }, { "context_id": "b_c0a10b3aef20", "observation": "scroll painting is fully obscured" }, { "context_id": "b_17642d7e1164", "observation": "scroll painting is not visible" } ], "uncertainty": "scroll painting is only visible in one context", "fallback_box_norm_xyxy": [ 0.6, 0.4, 0.8, 0.5 ], "routing_eligible": true, "consistency_filter": "passed" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"red umbrella\",\n \"grounding_queries\": [\"red umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_bc84988e0e0a\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_42d3a74baab7\", \"observation\": \"red umbrella is partially obscured\"},\n {\"context_id\": \"b_c0a10b3aef20\", \"observation\": \"red umbrella is fully obscured\"},\n {\"context_id\": \"b_17642d7e1164\", \"observation\": \"red umbrella is not visible\"}\n ],\n \"uncertainty\": \"red umbrella is only visible in one context\",\n \"fallback_box_norm_xyxy\": [0.5, 0.4, 0.7, 0.5]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"scroll painting\",\n \"grounding_queries\": [\"scroll painting\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_bc84988e0e0a\", \"observation\": \"scroll painting is visible\"},\n {\"context_id\": \"b_42d3a74baab7\", \"observation\": \"scroll painting is partially obscured\"},\n {\"context_id\": \"b_c0a10b3aef20\", \"observation\": \"scroll painting is fully obscured\"},\n {\"context_id\": \"b_17642d7e1164\", \"observation\": \"scroll painting is not visible\"}\n ],\n \"uncertainty\": \"scroll painting is only visible in one context\",\n \"fallback_box_norm_xyxy\": [0.6, 0.4, 0.8, 0.5]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_c0cd1b9b9c0e", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "red umbrella", "queries": [ "red umbrella" ], "source": "grounding", "boxes_xyxy": [ [ 974.6791381835938, 756.0464477539062, 1211.8345947265625, 1101.6326904296875 ] ], "box_scores": [ 0.27580294013023376 ], "box_queries": [ "red umbrella" ], "sam_predicted_iou": [ 0.98046875 ], "mask_path": "masks/b_c0cd1b9b9c0e_r1.png" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "scroll painting", "queries": [ "scroll painting" ], "source": "vlm_fallback", "boxes_xyxy": [ [ 1054.8, 467.20000000000005, 1406.4, 584.0 ] ], "box_scores": [ 0.9 ], "box_queries": [ "vlm_fallback" ], "sam_predicted_iou": [ 0.8828125 ], "mask_path": "masks/b_c0cd1b9b9c0e_r2.png" } ], "final_mask_path": "masks/b_c0cd1b9b9c0e.png", "routing_mask_path": "masks/b_c0cd1b9b9c0e_routing.png", "predicted_fraction": 0.04364003303878941 }, "raw_mask_metrics": { "iou": 0.1663498400744051, "dice": 0.28524861813980806, "precision": 0.8183644317471654, "recall": 0.1727271442502014, "safety_recall": 0.1727271442502014, "leakage": 0.8272728557497986, "overmask": 0.00999269396300321, "boundary_f5": 0.06648116416003601, "iou_ignore_boundary5": 0.21833857589573732, "predicted_fraction": 0.04364003303878941, "gt_fraction": 0.20676223759876572 }, "routing_mask_metrics": { "iou": 0.1860315780667422, "dice": 0.31370425797596013, "precision": 0.8228340550649559, "recall": 0.19379395789463766, "safety_recall": 0.19379395789463766, "leakage": 0.8062060421053623, "overmask": 0.010876171882194759, "boundary_f5": 0.06240564545849471, "iou_ignore_boundary5": 0.23662842039513968, "predicted_fraction": 0.048696662614739664, "gt_fraction": 0.20676223759876572 }, "box_metrics": { "gt_pixel_coverage": 0.22794509061273713, "area_precision": 0.7863474961200627, "box_fraction": 0.05993588994342886 }, "frame_detected": true, "gt_ambiguity": { "median_rgb_max_difference": 0.011764705181121826, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_42d3a74baab7", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0020.JPG", "source_split": "train", "source_frame_id": "0020", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0020.JPG", "context_ids": [ "b_17642d7e1164", "b_c0cd1b9b9c0e", "b_d95203c68daa", "b_bc84988e0e0a" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_42d3a74baab7", "target_sha256": "beecb1ddf9871d8ad3aff7649bc170553336bbfad9551fe94b82dfc99715c5ab", "context_ids": [ "b_17642d7e1164", "b_c0cd1b9b9c0e", "b_d95203c68daa", "b_bc84988e0e0a" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "person holding mirror", "grounding_queries": [ "person holding mirror" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_17642d7e1164", "observation": "mirror is visible" }, { "context_id": "b_c0cd1b9b9c0e", "observation": "mirror is visible" }, { "context_id": "b_d95203c68daa", "observation": "mirror is visible" }, { "context_id": "b_bc84988e0e0a", "observation": "mirror is visible" } ], "uncertainty": "mirror is consistently visible across all contexts", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "red umbrella", "grounding_queries": [ "red umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_17642d7e1164", "observation": "red umbrella is visible" }, { "context_id": "b_c0cd1b9b9c0e", "observation": "red umbrella is visible" }, { "context_id": "b_d95203c68daa", "observation": "red umbrella is visible" }, { "context_id": "b_bc84988e0e0a", "observation": "red umbrella is visible" } ], "uncertainty": "red umbrella is consistently visible across all contexts", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r3", "cause": "transient_occluder", "semantic_description": "color chart", "grounding_queries": [ "color chart" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_17642d7e1164", "observation": "color chart is visible" }, { "context_id": "b_c0cd1b9b9c0e", "observation": "color chart is visible" }, { "context_id": "b_d95203c68daa", "observation": "color chart is visible" }, { "context_id": "b_bc84988e0e0a", "observation": "color chart is visible" } ], "uncertainty": "color chart is consistently visible across all contexts", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"person holding mirror\",\n \"grounding_queries\": [\"person holding mirror\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_17642d7e1164\", \"observation\": \"mirror is visible\"},\n {\"context_id\": \"b_c0cd1b9b9c0e\", \"observation\": \"mirror is visible\"},\n {\"context_id\": \"b_d95203c68daa\", \"observation\": \"mirror is visible\"},\n {\"context_id\": \"b_bc84988e0e0a\", \"observation\": \"mirror is visible\"}\n ],\n \"uncertainty\": \"mirror is consistently visible across all contexts\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"red umbrella\",\n \"grounding_queries\": [\"red umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_17642d7e1164\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_c0cd1b9b9c0e\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_d95203c68daa\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_bc84988e0e0a\", \"observation\": \"red umbrella is visible\"}\n ],\n \"uncertainty\": \"red umbrella is consistently visible across all contexts\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r3\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"color chart\",\n \"grounding_queries\": [\"color chart\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_17642d7e1164\", \"observation\": \"color chart is visible\"},\n {\"context_id\": \"b_c0cd1b9b9c0e\", \"observation\": \"color chart is visible\"},\n {\"context_id\": \"b_d95203c68daa\", \"observation\": \"color chart is visible\"},\n {\"context_id\": \"b_bc84988e0e0a\", \"observation\": \"color chart is visible\"}\n ],\n \"uncertainty\": \"color chart is consistently visible across all contexts\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_42d3a74baab7", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [], "final_mask_path": "masks/b_42d3a74baab7.png", "routing_mask_path": "masks/b_42d3a74baab7_routing.png", "predicted_fraction": 0.0 }, "raw_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.22075599607274768 }, "routing_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.22075599607274768 }, "box_metrics": { "gt_pixel_coverage": 0.0, "area_precision": 0.0, "box_fraction": 0.0 }, "frame_detected": false, "gt_ambiguity": { "median_rgb_max_difference": 0.011764705181121826, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_bc84988e0e0a", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0021.JPG", "source_split": "train", "source_frame_id": "0021", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0021.JPG", "context_ids": [ "b_c0cd1b9b9c0e", "b_45dcf0b2cf9e", "b_42d3a74baab7", "b_c0a10b3aef20" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_bc84988e0e0a", "target_sha256": "b2a22676fb275f1471a6a57d62769047f357b1dc24ac5d355f90ef4037ea737b", "context_ids": [ "b_c0cd1b9b9c0e", "b_45dcf0b2cf9e", "b_42d3a74baab7", "b_c0a10b3aef20" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "scroll with cranes", "grounding_queries": [ "scroll with cranes" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_c0cd1b9b9c0e", "observation": "scroll with cranes is visible" }, { "context_id": "b_45dcf0b2cf9e", "observation": "scroll with cranes is visible" }, { "context_id": "b_42d3a74baab7", "observation": "scroll with cranes is visible" }, { "context_id": "b_c0a10b3aef20", "observation": "scroll with cranes is visible" } ], "uncertainty": "scroll with cranes is consistently visible across all context images", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "red umbrella", "grounding_queries": [ "red umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_c0cd1b9b9c0e", "observation": "red umbrella is visible" }, { "context_id": "b_45dcf0b2cf9e", "observation": "red umbrella is visible" }, { "context_id": "b_42d3a74baab7", "observation": "red umbrella is visible" }, { "context_id": "b_c0a10b3aef20", "observation": "red umbrella is visible" } ], "uncertainty": "red umbrella is consistently visible across all context images", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r3", "cause": "transient_occluder", "semantic_description": "color chart", "grounding_queries": [ "color chart" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_c0cd1b9b9c0e", "observation": "color chart is visible" }, { "context_id": "b_45dcf0b2cf9e", "observation": "color chart is visible" }, { "context_id": "b_42d3a74baab7", "observation": "color chart is visible" }, { "context_id": "b_c0a10b3aef20", "observation": "color chart is visible" } ], "uncertainty": "color chart is consistently visible across all context images", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"scroll with cranes\",\n \"grounding_queries\": [\"scroll with cranes\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_c0cd1b9b9c0e\", \"observation\": \"scroll with cranes is visible\"},\n {\"context_id\": \"b_45dcf0b2cf9e\", \"observation\": \"scroll with cranes is visible\"},\n {\"context_id\": \"b_42d3a74baab7\", \"observation\": \"scroll with cranes is visible\"},\n {\"context_id\": \"b_c0a10b3aef20\", \"observation\": \"scroll with cranes is visible\"}\n ],\n \"uncertainty\": \"scroll with cranes is consistently visible across all context images\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"red umbrella\",\n \"grounding_queries\": [\"red umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_c0cd1b9b9c0e\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_45dcf0b2cf9e\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_42d3a74baab7\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_c0a10b3aef20\", \"observation\": \"red umbrella is visible\"}\n ],\n \"uncertainty\": \"red umbrella is consistently visible across all context images\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r3\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"color chart\",\n \"grounding_queries\": [\"color chart\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_c0cd1b9b9c0e\", \"observation\": \"color chart is visible\"},\n {\"context_id\": \"b_45dcf0b2cf9e\", \"observation\": \"color chart is visible\"},\n {\"context_id\": \"b_42d3a74baab7\", \"observation\": \"color chart is visible\"},\n {\"context_id\": \"b_c0a10b3aef20\", \"observation\": \"color chart is visible\"}\n ],\n \"uncertainty\": \"color chart is consistently visible across all context images\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_bc84988e0e0a", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [], "final_mask_path": "masks/b_bc84988e0e0a.png", "routing_mask_path": "masks/b_bc84988e0e0a_routing.png", "predicted_fraction": 0.0 }, "raw_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.24241140305764644 }, "routing_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.24241140305764644 }, "box_metrics": { "gt_pixel_coverage": 0.0, "area_precision": 0.0, "box_fraction": 0.0 }, "frame_detected": false, "gt_ambiguity": { "median_rgb_max_difference": 0.011764705181121826, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_45dcf0b2cf9e", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0022.JPG", "source_split": "train", "source_frame_id": "0022", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0022.JPG", "context_ids": [ "b_bc84988e0e0a", "b_c0cd1b9b9c0e", "b_2f68fc214982", "b_c0a10b3aef20" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_45dcf0b2cf9e", "target_sha256": "6094aaf918724214a99944de0f905327768498a637303147495f79f41fb064e1", "context_ids": [ "b_bc84988e0e0a", "b_c0cd1b9b9c0e", "b_2f68fc214982", "b_c0a10b3aef20" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "black curtain", "grounding_queries": [ "black curtain" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_c0a10b3aef20", "observation": "black curtain partially obscures the view" } ], "uncertainty": "curtain may be a reflection or specular effect", "fallback_box_norm_xyxy": null, "routing_eligible": true, "consistency_filter": "passed" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "red umbrella", "grounding_queries": [ "red umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_bc84988e0e0a", "observation": "red umbrella is visible" }, { "context_id": "b_c0cd1b9b9c0e", "observation": "red umbrella is visible" }, { "context_id": "b_2f68fc214982", "observation": "red umbrella is visible" }, { "context_id": "b_c0a10b3aef20", "observation": "red umbrella is not visible" } ], "uncertainty": "umbrella may be a reflection or specular effect", "fallback_box_norm_xyxy": null, "routing_eligible": true, "consistency_filter": "passed" }, { "region_id": "r3", "cause": "transient_occluder", "semantic_description": "white sign with red text", "grounding_queries": [ "white sign with red text" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_bc84988e0e0a", "observation": "white sign with red text is visible" }, { "context_id": "b_c0cd1b9b9c0e", "observation": "white sign with red text is visible" }, { "context_id": "b_2f68fc214982", "observation": "white sign with red text is visible" }, { "context_id": "b_c0a10b3aef20", "observation": "white sign with red text is not visible" } ], "uncertainty": "sign may be a reflection or specular effect", "fallback_box_norm_xyxy": null, "routing_eligible": true, "consistency_filter": "passed" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"black curtain\",\n \"grounding_queries\": [\"black curtain\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_c0a10b3aef20\", \"observation\": \"black curtain partially obscures the view\"}\n ],\n \"uncertainty\": \"curtain may be a reflection or specular effect\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"red umbrella\",\n \"grounding_queries\": [\"red umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_bc84988e0e0a\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_c0cd1b9b9c0e\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_2f68fc214982\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_c0a10b3aef20\", \"observation\": \"red umbrella is not visible\"}\n ],\n \"uncertainty\": \"umbrella may be a reflection or specular effect\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r3\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"white sign with red text\",\n \"grounding_queries\": [\"white sign with red text\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_bc84988e0e0a\", \"observation\": \"white sign with red text is visible\"},\n {\"context_id\": \"b_c0cd1b9b9c0e\", \"observation\": \"white sign with red text is visible\"},\n {\"context_id\": \"b_2f68fc214982\", \"observation\": \"white sign with red text is visible\"},\n {\"context_id\": \"b_c0a10b3aef20\", \"observation\": \"white sign with red text is not visible\"}\n ],\n \"uncertainty\": \"sign may be a reflection or specular effect\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_45dcf0b2cf9e", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "black curtain", "queries": [ "black curtain" ], "source": "grounding", "boxes_xyxy": [], "box_scores": [], "box_queries": [], "sam_predicted_iou": [], "mask_path": "masks/b_45dcf0b2cf9e_r1.png" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "red umbrella", "queries": [ "red umbrella" ], "source": "grounding", "boxes_xyxy": [ [ 281.3736572265625, 890.8401489257812, 416.388916015625, 1016.050048828125 ], [ 1122.8323974609375, 582.1143188476562, 1406.656494140625, 736.9505615234375 ] ], "box_scores": [ 0.2556205093860626, 0.2505408525466919 ], "box_queries": [ "red umbrella", "red umbrella" ], "sam_predicted_iou": [ 0.97265625, 0.97265625 ], "mask_path": "masks/b_45dcf0b2cf9e_r2.png" }, { "region_id": "r3", "cause": "transient_occluder", "semantic_description": "white sign with red text", "queries": [ "white sign with red text" ], "source": "grounding", "boxes_xyxy": [ [ 1122.9132080078125, 581.886474609375, 1406.521484375, 736.3436889648438 ] ], "box_scores": [ 0.32850027084350586 ], "box_queries": [ "white sign with red text" ], "sam_predicted_iou": [ 0.97265625 ], "mask_path": "masks/b_45dcf0b2cf9e_r3.png" } ], "final_mask_path": "masks/b_45dcf0b2cf9e.png", "routing_mask_path": "masks/b_45dcf0b2cf9e_routing.png", "predicted_fraction": 0.017133027880374647 }, "raw_mask_metrics": { "iou": 0.004627504683937668, "dice": 0.009212379040714222, "precision": 0.05244457077885162, "recall": 0.005049703312824331, "safety_recall": 0.005049703312824331, "leakage": 0.9949502966871757, "overmask": 0.01974850353322559, "boundary_f5": 0.0751433096427025, "iou_ignore_boundary5": 0.002886513071102152, "predicted_fraction": 0.017133027880374647, "gt_fraction": 0.17793803668552371 }, "routing_mask_metrics": { "iou": 0.0105129527512199, "dice": 0.020807160804020102, "precision": 0.10051204248056135, "recall": 0.011604738236517703, "safety_recall": 0.011604738236517703, "leakage": 0.9883952617634822, "overmask": 0.022478992592311738, "boundary_f5": 0.01760604361304794, "iou_ignore_boundary5": 0.00862226301161281, "predicted_fraction": 0.020544049121822745, "gt_fraction": 0.17793803668552371 }, "box_metrics": { "gt_pixel_coverage": 0.032419368964988725, "area_precision": 0.19451514902701372, "box_fraction": 0.029656501784406316 }, "frame_detected": true, "gt_ambiguity": { "median_rgb_max_difference": 0.011764705181121826, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_2f68fc214982", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0023.JPG", "source_split": "train", "source_frame_id": "0023", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0023.JPG", "context_ids": [ "b_45dcf0b2cf9e", "b_c0a10b3aef20", "b_bc84988e0e0a", "b_c0cd1b9b9c0e" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_2f68fc214982", "target_sha256": "9aeb3f64365e0a88f38e6c4fe971124b99abdc8889daa561865c2178a8458f76", "context_ids": [ "b_45dcf0b2cf9e", "b_c0a10b3aef20", "b_bc84988e0e0a", "b_c0cd1b9b9c0e" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "bear on floor", "grounding_queries": [ "bear on floor" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_45dcf0b2cf9e", "observation": "bear partially obscured" }, { "context_id": "b_c0a10b3aef20", "observation": "bear fully visible" }, { "context_id": "b_bc84988e0e0a", "observation": "bear partially obscured" }, { "context_id": "b_c0cd1b9b9c0e", "observation": "bear fully visible" } ], "uncertainty": "bear's position varies slightly across views", "fallback_box_norm_xyxy": [ 0.2, 0.5, 0.6, 0.8 ], "routing_eligible": true, "consistency_filter": "passed" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "umbrella on floor", "grounding_queries": [ "umbrella on floor" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_45dcf0b2cf9e", "observation": "umbrella partially obscured" }, { "context_id": "b_c0a10b3aef20", "observation": "umbrella fully visible" }, { "context_id": "b_bc84988e0e0a", "observation": "umbrella partially obscured" }, { "context_id": "b_c0cd1b9b9c0e", "observation": "umbrella fully visible" } ], "uncertainty": "umbrella's position varies slightly across views", "fallback_box_norm_xyxy": [ 0.5, 0.5, 0.7, 0.8 ], "routing_eligible": true, "consistency_filter": "passed" }, { "region_id": "r3", "cause": "transient_occluder", "semantic_description": "scroll on floor", "grounding_queries": [ "scroll on floor" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_45dcf0b2cf9e", "observation": "scroll partially obscured" }, { "context_id": "b_c0a10b3aef20", "observation": "scroll fully visible" }, { "context_id": "b_bc84988e0e0a", "observation": "scroll partially obscured" }, { "context_id": "b_c0cd1b9b9c0e", "observation": "scroll fully visible" } ], "uncertainty": "scroll's position varies slightly across views", "fallback_box_norm_xyxy": [ 0.7, 0.5, 0.9, 0.8 ], "routing_eligible": true, "consistency_filter": "passed" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"bear on floor\",\n \"grounding_queries\": [\"bear on floor\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_45dcf0b2cf9e\", \"observation\": \"bear partially obscured\"},\n {\"context_id\": \"b_c0a10b3aef20\", \"observation\": \"bear fully visible\"},\n {\"context_id\": \"b_bc84988e0e0a\", \"observation\": \"bear partially obscured\"},\n {\"context_id\": \"b_c0cd1b9b9c0e\", \"observation\": \"bear fully visible\"}\n ],\n \"uncertainty\": \"bear's position varies slightly across views\",\n \"fallback_box_norm_xyxy\": [0.2, 0.5, 0.6, 0.8]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"umbrella on floor\",\n \"grounding_queries\": [\"umbrella on floor\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_45dcf0b2cf9e\", \"observation\": \"umbrella partially obscured\"},\n {\"context_id\": \"b_c0a10b3aef20\", \"observation\": \"umbrella fully visible\"},\n {\"context_id\": \"b_bc84988e0e0a\", \"observation\": \"umbrella partially obscured\"},\n {\"context_id\": \"b_c0cd1b9b9c0e\", \"observation\": \"umbrella fully visible\"}\n ],\n \"uncertainty\": \"umbrella's position varies slightly across views\",\n \"fallback_box_norm_xyxy\": [0.5, 0.5, 0.7, 0.8]\n },\n {\n \"region_id\": \"r3\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"scroll on floor\",\n \"grounding_queries\": [\"scroll on floor\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_45dcf0b2cf9e\", \"observation\": \"scroll partially obscured\"},\n {\"context_id\": \"b_c0a10b3aef20\", \"observation\": \"scroll fully visible\"},\n {\"context_id\": \"b_bc84988e0e0a\", \"observation\": \"scroll partially obscured\"},\n {\"context_id\": \"b_c0cd1b9b9c0e\", \"observation\": \"scroll fully visible\"}\n ],\n \"uncertainty\": \"scroll's position varies slightly across views\",\n \"fallback_box_norm_xyxy\": [0.7, 0.5, 0.9, 0.8]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_2f68fc214982", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "bear on floor", "queries": [ "bear on floor" ], "source": "vlm_fallback", "boxes_xyxy": [ [ 351.6, 584.0, 1054.8, 934.4000000000001 ] ], "box_scores": [ 0.9 ], "box_queries": [ "vlm_fallback" ], "sam_predicted_iou": [ 0.478515625 ], "mask_path": "masks/b_2f68fc214982_r1.png" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "umbrella on floor", "queries": [ "umbrella on floor" ], "source": "grounding", "boxes_xyxy": [ [ 138.65228271484375, 864.3546752929688, 303.8629150390625, 1014.9258422851562 ] ], "box_scores": [ 0.2737511098384857 ], "box_queries": [ "umbrella on floor" ], "sam_predicted_iou": [ 0.97265625 ], "mask_path": "masks/b_2f68fc214982_r2.png" }, { "region_id": "r3", "cause": "transient_occluder", "semantic_description": "scroll on floor", "queries": [ "scroll on floor" ], "source": "vlm_fallback", "boxes_xyxy": [ [ 1230.6, 584.0, 1582.2, 934.4000000000001 ] ], "box_scores": [ 0.9 ], "box_queries": [ "vlm_fallback" ], "sam_predicted_iou": [ 0.83203125 ], "mask_path": "masks/b_2f68fc214982_r3.png" } ], "final_mask_path": "masks/b_2f68fc214982.png", "routing_mask_path": "masks/b_2f68fc214982_routing.png", "predicted_fraction": 0.015903813486683186 }, "raw_mask_metrics": { "iou": 0.03125341260692835, "dice": 0.06061247841676888, "precision": 0.3155009799118079, "recall": 0.03352673385246675, "safety_recall": 0.03352673385246675, "leakage": 0.9664732661475333, "overmask": 0.012802134204487076, "boundary_f5": 0.09002814402788738, "iou_ignore_boundary5": 0.033187962332306586, "predicted_fraction": 0.015903813486683186, "gt_fraction": 0.1496617225365063 }, "routing_mask_metrics": { "iou": 0.05218082441094991, "dice": 0.09918603951020051, "precision": 0.31205331771918643, "recall": 0.05896383746546613, "safety_recall": 0.05896383746546613, "leakage": 0.9410361625345338, "overmask": 0.02287866751964592, "boundary_f5": 0.1333250484638646, "iou_ignore_boundary5": 0.04658328395496693, "predicted_fraction": 0.02827923621175994, "gt_fraction": 0.1496617225365063 }, "box_metrics": { "gt_pixel_coverage": 0.259701210841276, "area_precision": 0.20265353021088583, "box_fraction": 0.19179202315832125 }, "frame_detected": true, "gt_ambiguity": { "median_rgb_max_difference": 0.011764705181121826, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_c0a10b3aef20", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0024.JPG", "source_split": "train", "source_frame_id": "0024", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0024.JPG", "context_ids": [ "b_c0cd1b9b9c0e", "b_42d3a74baab7", "b_bc84988e0e0a", "b_2f68fc214982" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_c0a10b3aef20", "target_sha256": "8d91c5d290a5064fe0a6fccead5e94c7d4909c3820c3252916712b856df52c85", "context_ids": [ "b_c0cd1b9b9c0e", "b_42d3a74baab7", "b_bc84988e0e0a", "b_2f68fc214982" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "black curtain", "grounding_queries": [ "black curtain" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_c0cd1b9b9c0e", "observation": "black curtain partially visible" }, { "context_id": "b_42d3a74baab7", "observation": "black curtain fully visible" }, { "context_id": "b_bc84988e0e0a", "observation": "black curtain partially visible" }, { "context_id": "b_2f68fc214982", "observation": "black curtain partially visible" } ], "uncertainty": "black curtain is consistently visible across all contexts.", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "red umbrella", "grounding_queries": [ "red umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_c0cd1b9b9c0e", "observation": "red umbrella partially visible" }, { "context_id": "b_42d3a74baab7", "observation": "red umbrella fully visible" }, { "context_id": "b_bc84988e0e0a", "observation": "red umbrella partially visible" }, { "context_id": "b_2f68fc214982", "observation": "red umbrella partially visible" } ], "uncertainty": "red umbrella is consistently visible across all contexts.", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r3", "cause": "transient_occluder", "semantic_description": "white signboard", "grounding_queries": [ "white signboard" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_c0cd1b9b9c0e", "observation": "white signboard partially visible" }, { "context_id": "b_42d3a74baab7", "observation": "white signboard fully visible" }, { "context_id": "b_bc84988e0e0a", "observation": "white signboard partially visible" }, { "context_id": "b_2f68fc214982", "observation": "white signboard partially visible" } ], "uncertainty": "white signboard is consistently visible across all contexts.", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"black curtain\",\n \"grounding_queries\": [\"black curtain\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_c0cd1b9b9c0e\", \"observation\": \"black curtain partially visible\"},\n {\"context_id\": \"b_42d3a74baab7\", \"observation\": \"black curtain fully visible\"},\n {\"context_id\": \"b_bc84988e0e0a\", \"observation\": \"black curtain partially visible\"},\n {\"context_id\": \"b_2f68fc214982\", \"observation\": \"black curtain partially visible\"}\n ],\n \"uncertainty\": \"black curtain is consistently visible across all contexts.\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"red umbrella\",\n \"grounding_queries\": [\"red umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_c0cd1b9b9c0e\", \"observation\": \"red umbrella partially visible\"},\n {\"context_id\": \"b_42d3a74baab7\", \"observation\": \"red umbrella fully visible\"},\n {\"context_id\": \"b_bc84988e0e0a\", \"observation\": \"red umbrella partially visible\"},\n {\"context_id\": \"b_2f68fc214982\", \"observation\": \"red umbrella partially visible\"}\n ],\n \"uncertainty\": \"red umbrella is consistently visible across all contexts.\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r3\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"white signboard\",\n \"grounding_queries\": [\"white signboard\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_c0cd1b9b9c0e\", \"observation\": \"white signboard partially visible\"},\n {\"context_id\": \"b_42d3a74baab7\", \"observation\": \"white signboard fully visible\"},\n {\"context_id\": \"b_bc84988e0e0a\", \"observation\": \"white signboard partially visible\"},\n {\"context_id\": \"b_2f68fc214982\", \"observation\": \"white signboard partially visible\"}\n ],\n \"uncertainty\": \"white signboard is consistently visible across all contexts.\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_c0a10b3aef20", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [], "final_mask_path": "masks/b_c0a10b3aef20.png", "routing_mask_path": "masks/b_c0a10b3aef20_routing.png", "predicted_fraction": 0.0 }, "raw_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.2129696728848162 }, "routing_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.2129696728848162 }, "box_metrics": { "gt_pixel_coverage": 0.0, "area_precision": 0.0, "box_fraction": 0.0 }, "frame_detected": false, "gt_ambiguity": { "median_rgb_max_difference": 0.011764705181121826, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_8ffaf4db41bf", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0025.JPG", "source_split": "train", "source_frame_id": "0025", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0025.JPG", "context_ids": [ "b_d95203c68daa", "b_4dd28c1b3b60", "b_17642d7e1164", "b_42d3a74baab7" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_8ffaf4db41bf", "target_sha256": "215efed0325c4c5bfc4cab03cf298ffef068a601d17f729ebdc3803b1505db7d", "context_ids": [ "b_d95203c68daa", "b_4dd28c1b3b60", "b_17642d7e1164", "b_42d3a74baab7" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "mirror", "grounding_queries": [ "mirror" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_d95203c68daa", "observation": "mirror is visible" }, { "context_id": "b_4dd28c1b3b60", "observation": "mirror is visible" }, { "context_id": "b_17642d7e1164", "observation": "mirror is visible" }, { "context_id": "b_42d3a74baab7", "observation": "mirror is partially visible" } ], "uncertainty": "mirror is partially visible in some views", "fallback_box_norm_xyxy": null, "routing_eligible": true, "consistency_filter": "passed" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "person holding mirror", "grounding_queries": [ "person holding mirror" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_d95203c68daa", "observation": "person holding mirror" }, { "context_id": "b_4dd28c1b3b60", "observation": "person holding mirror" }, { "context_id": "b_17642d7e1164", "observation": "person holding mirror" }, { "context_id": "b_42d3a74baab7", "observation": "person holding mirror" } ], "uncertainty": "person holding mirror is partially visible", "fallback_box_norm_xyxy": null, "routing_eligible": true, "consistency_filter": "passed" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"mirror\",\n \"grounding_queries\": [\"mirror\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_d95203c68daa\", \"observation\": \"mirror is visible\"},\n {\"context_id\": \"b_4dd28c1b3b60\", \"observation\": \"mirror is visible\"},\n {\"context_id\": \"b_17642d7e1164\", \"observation\": \"mirror is visible\"},\n {\"context_id\": \"b_42d3a74baab7\", \"observation\": \"mirror is partially visible\"}\n ],\n \"uncertainty\": \"mirror is partially visible in some views\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"person holding mirror\",\n \"grounding_queries\": [\"person holding mirror\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_d95203c68daa\", \"observation\": \"person holding mirror\"},\n {\"context_id\": \"b_4dd28c1b3b60\", \"observation\": \"person holding mirror\"},\n {\"context_id\": \"b_17642d7e1164\", \"observation\": \"person holding mirror\"},\n {\"context_id\": \"b_42d3a74baab7\", \"observation\": \"person holding mirror\"}\n ],\n \"uncertainty\": \"person holding mirror is partially visible\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_8ffaf4db41bf", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "mirror", "queries": [ "mirror" ], "source": "grounding", "boxes_xyxy": [ [ 1185.1329345703125, 284.9571533203125, 1387.48193359375, 921.6734619140625 ] ], "box_scores": [ 0.2762335240840912 ], "box_queries": [ "mirror" ], "sam_predicted_iou": [ 0.93359375 ], "mask_path": "masks/b_8ffaf4db41bf_r1.png" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "person holding mirror", "queries": [ "person holding mirror" ], "source": "grounding", "boxes_xyxy": [ [ 1185.721435546875, 284.9627380371094, 1387.1141357421875, 921.4432983398438 ] ], "box_scores": [ 0.2745721936225891 ], "box_queries": [ "person holding mirror" ], "sam_predicted_iou": [ 0.93359375 ], "mask_path": "masks/b_8ffaf4db41bf_r2.png" } ], "final_mask_path": "masks/b_8ffaf4db41bf.png", "routing_mask_path": "masks/b_8ffaf4db41bf_routing.png", "predicted_fraction": 0.03442384714884598 }, "raw_mask_metrics": { "iou": 0.13937883928024547, "dice": 0.24465758793325, "precision": 0.9979910587968989, "recall": 0.13941795543258065, "safety_recall": 0.13941795543258065, "leakage": 0.8605820445674194, "overmask": 9.176867314777535e-05, "boundary_f5": 0.08186245031215608, "iou_ignore_boundary5": 0.14958321922497603, "predicted_fraction": 0.03442384714884598, "gt_fraction": 0.2464151160253713 }, "routing_mask_metrics": { "iou": 0.17070194491065258, "dice": 0.2916232362177898, "precision": 0.9945356448514271, "recall": 0.17086219674885122, "safety_recall": 0.17086219674885122, "leakage": 0.8291378032511487, "overmask": 0.0003069726742619246, "boundary_f5": 0.15888570258090945, "iou_ignore_boundary5": 0.1763979700123237, "predicted_fraction": 0.04233435800333505, "gt_fraction": 0.2464151160253713 }, "box_metrics": { "gt_pixel_coverage": 0.2290152675527447, "area_precision": 0.9005393474983291, "box_fraction": 0.0626655835554101 }, "frame_detected": true, "gt_ambiguity": { "median_rgb_max_difference": 0.011764705181121826, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_42d9140f0598", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0026.JPG", "source_split": "train", "source_frame_id": "0026", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0026.JPG", "context_ids": [ "b_66a730d6ef90", "b_3d50f98d854f", "b_2d3a471eb3b2", "b_4dd28c1b3b60" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_42d9140f0598", "target_sha256": "267fc28ff59ccf3e906e9f6236c98d8f7ab103a5101b3228b4fed69b043b83f1", "context_ids": [ "b_66a730d6ef90", "b_3d50f98d854f", "b_2d3a471eb3b2", "b_4dd28c1b3b60" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "mannequin torso", "grounding_queries": [ "mannequin torso" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_66a730d6ef90", "observation": "mannequin torso is visible" }, { "context_id": "b_3d50f98d854f", "observation": "mannequin torso is visible" }, { "context_id": "b_2d3a471eb3b2", "observation": "mannequin torso is visible" }, { "context_id": "b_4dd28c1b3b60", "observation": "mannequin torso is partially visible" } ], "uncertainty": "mannequin torso is partially visible in some views", "fallback_box_norm_xyxy": [ 0.5, 0.0, 1.0, 0.5 ], "routing_eligible": true, "consistency_filter": "passed" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "red umbrella", "grounding_queries": [ "red umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_66a730d6ef90", "observation": "red umbrella is visible" }, { "context_id": "b_3d50f98d854f", "observation": "red umbrella is visible" }, { "context_id": "b_2d3a471eb3b2", "observation": "red umbrella is visible" }, { "context_id": "b_4dd28c1b3b60", "observation": "red umbrella is partially visible" } ], "uncertainty": "red umbrella is partially visible in some views", "fallback_box_norm_xyxy": [ 0.5, 0.5, 0.7, 0.7 ], "routing_eligible": true, "consistency_filter": "passed" }, { "region_id": "r3", "cause": "transient_occluder", "semantic_description": "color chart", "grounding_queries": [ "color chart" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_66a730d6ef90", "observation": "color chart is visible" }, { "context_id": "b_3d50f98d854f", "observation": "color chart is visible" }, { "context_id": "b_2d3a471eb3b2", "observation": "color chart is visible" }, { "context_id": "b_4dd28c1b3b60", "observation": "color chart is partially visible" } ], "uncertainty": "color chart is partially visible in some views", "fallback_box_norm_xyxy": [ 0.0, 0.5, 0.1, 0.6 ], "routing_eligible": true, "consistency_filter": "passed" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"mannequin torso\",\n \"grounding_queries\": [\"mannequin torso\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_66a730d6ef90\", \"observation\": \"mannequin torso is visible\"},\n {\"context_id\": \"b_3d50f98d854f\", \"observation\": \"mannequin torso is visible\"},\n {\"context_id\": \"b_2d3a471eb3b2\", \"observation\": \"mannequin torso is visible\"},\n {\"context_id\": \"b_4dd28c1b3b60\", \"observation\": \"mannequin torso is partially visible\"}\n ],\n \"uncertainty\": \"mannequin torso is partially visible in some views\",\n \"fallback_box_norm_xyxy\": [0.5, 0.0, 1.0, 0.5]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"red umbrella\",\n \"grounding_queries\": [\"red umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_66a730d6ef90\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_3d50f98d854f\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_2d3a471eb3b2\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_4dd28c1b3b60\", \"observation\": \"red umbrella is partially visible\"}\n ],\n \"uncertainty\": \"red umbrella is partially visible in some views\",\n \"fallback_box_norm_xyxy\": [0.5, 0.5, 0.7, 0.7]\n },\n {\n \"region_id\": \"r3\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"color chart\",\n \"grounding_queries\": [\"color chart\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_66a730d6ef90\", \"observation\": \"color chart is visible\"},\n {\"context_id\": \"b_3d50f98d854f\", \"observation\": \"color chart is visible\"},\n {\"context_id\": \"b_2d3a471eb3b2\", \"observation\": \"color chart is visible\"},\n {\"context_id\": \"b_4dd28c1b3b60\", \"observation\": \"color chart is partially visible\"}\n ],\n \"uncertainty\": \"color chart is partially visible in some views\",\n \"fallback_box_norm_xyxy\": [0.0, 0.5, 0.1, 0.6]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_42d9140f0598", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "mannequin torso", "queries": [ "mannequin torso" ], "source": "vlm_fallback", "boxes_xyxy": [ [ 879.0, 0.0, 1758.0, 584.0 ] ], "box_scores": [ 0.9 ], "box_queries": [ "vlm_fallback" ], "sam_predicted_iou": [ 0.890625 ], "mask_path": "masks/b_42d9140f0598_r1.png" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "red umbrella", "queries": [ "red umbrella" ], "source": "vlm_fallback", "boxes_xyxy": [ [ 879.0, 584.0, 1230.6, 817.5999999999999 ] ], "box_scores": [ 0.9 ], "box_queries": [ "vlm_fallback" ], "sam_predicted_iou": [ 0.80078125 ], "mask_path": "masks/b_42d9140f0598_r2.png" }, { "region_id": "r3", "cause": "transient_occluder", "semantic_description": "color chart", "queries": [ "color chart" ], "source": "vlm_fallback", "boxes_xyxy": [ [ 0.0, 584.0, 175.8, 700.8 ] ], "box_scores": [ 0.9 ], "box_queries": [ "vlm_fallback" ], "sam_predicted_iou": [ 0.86328125 ], "mask_path": "masks/b_42d9140f0598_r3.png" } ], "final_mask_path": "masks/b_42d9140f0598.png", "routing_mask_path": "masks/b_42d9140f0598_routing.png", "predicted_fraction": 0.05020201193760032 }, "raw_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.06332833457943811, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.05020201193760032, "gt_fraction": 0.20727408558916577 }, "routing_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.08112848558644845, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.06431265292128352, "gt_fraction": 0.20727408558916577 }, "box_metrics": { "gt_pixel_coverage": 0.27558181882261723, "area_precision": 0.19031277178498643, "box_fraction": 0.3001425966618355 }, "frame_detected": true, "gt_ambiguity": { "median_rgb_max_difference": 0.011764705181121826, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_48b647235f33", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0027.JPG", "source_split": "train", "source_frame_id": "0027", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0027.JPG", "context_ids": [ "b_0a52973d1b8d", "b_3fad5029fb0d", "b_6125c5000375", "b_2962b6747c5c" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_48b647235f33", "target_sha256": "4478eb05fc8017524c6130aaf321ae48dcba6afa4f0b8a6f6bfcc8a7608b3d88", "context_ids": [ "b_0a52973d1b8d", "b_3fad5029fb0d", "b_6125c5000375", "b_2962b6747c5c" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "black and white umbrella", "grounding_queries": [ "black and white umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_2962b6747c5c", "observation": "appears in one view" } ], "uncertainty": "insufficient evidence for persistence", "fallback_box_norm_xyxy": null, "routing_eligible": true, "consistency_filter": "passed" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "red cloth", "grounding_queries": [ "red cloth" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_6125c5000375", "observation": "appears in one view" } ], "uncertainty": "insufficient evidence for persistence", "fallback_box_norm_xyxy": null, "routing_eligible": true, "consistency_filter": "passed" }, { "region_id": "r3", "cause": "transient_occluder", "semantic_description": "green curtain", "grounding_queries": [ "green curtain" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_3fad5029fb0d", "observation": "appears in one view" } ], "uncertainty": "insufficient evidence for persistence", "fallback_box_norm_xyxy": null, "routing_eligible": true, "consistency_filter": "passed" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"black and white umbrella\",\n \"grounding_queries\": [\"black and white umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_2962b6747c5c\", \"observation\": \"appears in one view\"}\n ],\n \"uncertainty\": \"insufficient evidence for persistence\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"red cloth\",\n \"grounding_queries\": [\"red cloth\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_6125c5000375\", \"observation\": \"appears in one view\"}\n ],\n \"uncertainty\": \"insufficient evidence for persistence\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r3\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"green curtain\",\n \"grounding_queries\": [\"green curtain\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_3fad5029fb0d\", \"observation\": \"appears in one view\"}\n ],\n \"uncertainty\": \"insufficient evidence for persistence\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_48b647235f33", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "black and white umbrella", "queries": [ "black and white umbrella" ], "source": "grounding", "boxes_xyxy": [], "box_scores": [], "box_queries": [], "sam_predicted_iou": [], "mask_path": "masks/b_48b647235f33_r1.png" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "red cloth", "queries": [ "red cloth" ], "source": "grounding", "boxes_xyxy": [], "box_scores": [], "box_queries": [], "sam_predicted_iou": [], "mask_path": "masks/b_48b647235f33_r2.png" }, { "region_id": "r3", "cause": "transient_occluder", "semantic_description": "green curtain", "queries": [ "green curtain" ], "source": "grounding", "boxes_xyxy": [], "box_scores": [], "box_queries": [], "sam_predicted_iou": [], "mask_path": "masks/b_48b647235f33_r3.png" } ], "final_mask_path": "masks/b_48b647235f33.png", "routing_mask_path": "masks/b_48b647235f33_routing.png", "predicted_fraction": 0.0 }, "raw_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.02949919740676672 }, "routing_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.02949919740676672 }, "box_metrics": { "gt_pixel_coverage": 0.0, "area_precision": 0.0, "box_fraction": 0.0 }, "frame_detected": false, "gt_ambiguity": { "median_rgb_max_difference": 0.00784313678741455, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_57cf54a3bbc0", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0028.JPG", "source_split": "train", "source_frame_id": "0028", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0028.JPG", "context_ids": [ "b_0ed8205374dd", "b_c7dd5015d023", "b_31faa8f0af47", "b_3fad5029fb0d" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_57cf54a3bbc0", "target_sha256": "afcf569e721329b5cb0850ec96c8f3bced66556ebadc4e3191fdb396f9f7cdcd", "context_ids": [ "b_0ed8205374dd", "b_c7dd5015d023", "b_31faa8f0af47", "b_3fad5029fb0d" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "black curtain", "grounding_queries": [ "black curtain" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_31faa8f0af47", "observation": "curtain partially visible" }, { "context_id": "b_3fad5029fb0d", "observation": "curtain fully visible" } ], "uncertainty": "curtain may be partially obscured in some views", "fallback_box_norm_xyxy": null, "routing_eligible": true, "consistency_filter": "passed" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "red umbrella", "grounding_queries": [ "red umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_0ed8205374dd", "observation": "umbrella present" }, { "context_id": "b_c7dd5015d023", "observation": "umbrella present" }, { "context_id": "b_31faa8f0af47", "observation": "umbrella present" }, { "context_id": "b_3fad5029fb0d", "observation": "umbrella present" } ], "uncertainty": "umbrella consistently visible", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"black curtain\",\n \"grounding_queries\": [\"black curtain\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_31faa8f0af47\", \"observation\": \"curtain partially visible\"},\n {\"context_id\": \"b_3fad5029fb0d\", \"observation\": \"curtain fully visible\"}\n ],\n \"uncertainty\": \"curtain may be partially obscured in some views\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"red umbrella\",\n \"grounding_queries\": [\"red umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_0ed8205374dd\", \"observation\": \"umbrella present\"},\n {\"context_id\": \"b_c7dd5015d023\", \"observation\": \"umbrella present\"},\n {\"context_id\": \"b_31faa8f0af47\", \"observation\": \"umbrella present\"},\n {\"context_id\": \"b_3fad5029fb0d\", \"observation\": \"umbrella present\"}\n ],\n \"uncertainty\": \"umbrella consistently visible\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_57cf54a3bbc0", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "black curtain", "queries": [ "black curtain" ], "source": "grounding", "boxes_xyxy": [], "box_scores": [], "box_queries": [], "sam_predicted_iou": [], "mask_path": "masks/b_57cf54a3bbc0_r1.png" } ], "final_mask_path": "masks/b_57cf54a3bbc0.png", "routing_mask_path": "masks/b_57cf54a3bbc0_routing.png", "predicted_fraction": 0.0 }, "raw_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.14584502158430346 }, "routing_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.14584502158430346 }, "box_metrics": { "gt_pixel_coverage": 0.0, "area_precision": 0.0, "box_fraction": 0.0 }, "frame_detected": false, "gt_ambiguity": { "median_rgb_max_difference": 0.011764705181121826, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_39a6a990391f", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0029.JPG", "source_split": "train", "source_frame_id": "0029", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0029.JPG", "context_ids": [ "b_1ea940c8ac4e", "b_31faa8f0af47", "b_f9646cc9c954", "b_0ed8205374dd" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_39a6a990391f", "target_sha256": "f219da94ff53ac815426911c9fd887e919db470e608cea05b3894efd77048001", "context_ids": [ "b_1ea940c8ac4e", "b_31faa8f0af47", "b_f9646cc9c954", "b_0ed8205374dd" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "mirror", "grounding_queries": [ "mirror" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_1ea940c8ac4e", "observation": "mirror is visible" }, { "context_id": "b_31faa8f0af47", "observation": "mirror is visible" }, { "context_id": "b_f9646cc9c954", "observation": "mirror is visible" }, { "context_id": "b_0ed8205374dd", "observation": "mirror is visible" } ], "uncertainty": "mirror is consistently visible across all views", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "umbrella", "grounding_queries": [ "umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_1ea940c8ac4e", "observation": "umbrella is visible" }, { "context_id": "b_31faa8f0af47", "observation": "umbrella is visible" }, { "context_id": "b_f9646cc9c954", "observation": "umbrella is visible" }, { "context_id": "b_0ed8205374dd", "observation": "umbrella is visible" } ], "uncertainty": "umbrella is consistently visible across all views", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r3", "cause": "transient_occluder", "semantic_description": "pillow", "grounding_queries": [ "pillow" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_1ea940c8ac4e", "observation": "pillow is visible" }, { "context_id": "b_31faa8f0af47", "observation": "pillow is visible" }, { "context_id": "b_f9646cc9c954", "observation": "pillow is visible" }, { "context_id": "b_0ed8205374dd", "observation": "pillow is visible" } ], "uncertainty": "pillow is consistently visible across all views", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"mirror\",\n \"grounding_queries\": [\"mirror\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_1ea940c8ac4e\", \"observation\": \"mirror is visible\"},\n {\"context_id\": \"b_31faa8f0af47\", \"observation\": \"mirror is visible\"},\n {\"context_id\": \"b_f9646cc9c954\", \"observation\": \"mirror is visible\"},\n {\"context_id\": \"b_0ed8205374dd\", \"observation\": \"mirror is visible\"}\n ],\n \"uncertainty\": \"mirror is consistently visible across all views\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"umbrella\",\n \"grounding_queries\": [\"umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_1ea940c8ac4e\", \"observation\": \"umbrella is visible\"},\n {\"context_id\": \"b_31faa8f0af47\", \"observation\": \"umbrella is visible\"},\n {\"context_id\": \"b_f9646cc9c954\", \"observation\": \"umbrella is visible\"},\n {\"context_id\": \"b_0ed8205374dd\", \"observation\": \"umbrella is visible\"}\n ],\n \"uncertainty\": \"umbrella is consistently visible across all views\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r3\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"pillow\",\n \"grounding_queries\": [\"pillow\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_1ea940c8ac4e\", \"observation\": \"pillow is visible\"},\n {\"context_id\": \"b_31faa8f0af47\", \"observation\": \"pillow is visible\"},\n {\"context_id\": \"b_f9646cc9c954\", \"observation\": \"pillow is visible\"},\n {\"context_id\": \"b_0ed8205374dd\", \"observation\": \"pillow is visible\"}\n ],\n \"uncertainty\": \"pillow is consistently visible across all views\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_39a6a990391f", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [], "final_mask_path": "masks/b_39a6a990391f.png", "routing_mask_path": "masks/b_39a6a990391f_routing.png", "predicted_fraction": 0.0 }, "raw_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.15133022036249164 }, "routing_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.15133022036249164 }, "box_metrics": { "gt_pixel_coverage": 0.0, "area_precision": 0.0, "box_fraction": 0.0 }, "frame_detected": false, "gt_ambiguity": { "median_rgb_max_difference": 0.011764705181121826, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_e21902361190", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/train/0030.JPG", "source_split": "train", "source_frame_id": "0030", "clean_reference_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/val/0030.JPG", "context_ids": [ "b_6c74696a698a", "b_c7168d6511cd", "b_442bffc55440", "b_f2d05c389498" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_e21902361190", "target_sha256": "7aad75333082d3e970f7577b36d73f84a6996115d0084e245bc54f56316fa30a", "context_ids": [ "b_6c74696a698a", "b_c7168d6511cd", "b_442bffc55440", "b_f2d05c389498" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "red umbrella", "grounding_queries": [ "red umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_6c74696a698a", "observation": "red umbrella is visible" }, { "context_id": "b_c7168d6511cd", "observation": "red umbrella is visible" }, { "context_id": "b_442bffc55440", "observation": "red umbrella is visible" }, { "context_id": "b_f2d05c389498", "observation": "red umbrella is visible" } ], "uncertainty": "red umbrella is consistently visible across all context images", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "white flower bouquet", "grounding_queries": [ "white flower bouquet" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_6c74696a698a", "observation": "white flower bouquet is visible" }, { "context_id": "b_c7168d6511cd", "observation": "white flower bouquet is visible" }, { "context_id": "b_442bffc55440", "observation": "white flower bouquet is visible" }, { "context_id": "b_f2d05c389498", "observation": "white flower bouquet is visible" } ], "uncertainty": "white flower bouquet is consistently visible across all context images", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r3", "cause": "transient_occluder", "semantic_description": "green wall", "grounding_queries": [ "green wall" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_6c74696a698a", "observation": "green wall is visible" }, { "context_id": "b_c7168d6511cd", "observation": "green wall is visible" }, { "context_id": "b_442bffc55440", "observation": "green wall is visible" }, { "context_id": "b_f2d05c389498", "observation": "green wall is visible" } ], "uncertainty": "green wall is consistently visible across all context images", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"red umbrella\",\n \"grounding_queries\": [\"red umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_6c74696a698a\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_c7168d6511cd\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_442bffc55440\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_f2d05c389498\", \"observation\": \"red umbrella is visible\"}\n ],\n \"uncertainty\": \"red umbrella is consistently visible across all context images\",\n \"fallback_box_norm_xyxy\": [300, 270, 440, 380]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"white flower bouquet\",\n \"grounding_queries\": [\"white flower bouquet\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_6c74696a698a\", \"observation\": \"white flower bouquet is visible\"},\n {\"context_id\": \"b_c7168d6511cd\", \"observation\": \"white flower bouquet is visible\"},\n {\"context_id\": \"b_442bffc55440\", \"observation\": \"white flower bouquet is visible\"},\n {\"context_id\": \"b_f2d05c389498\", \"observation\": \"white flower bouquet is visible\"}\n ],\n \"uncertainty\": \"white flower bouquet is consistently visible across all context images\",\n \"fallback_box_norm_xyxy\": [250, 100, 350, 200]\n },\n {\n \"region_id\": \"r3\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"green wall\",\n \"grounding_queries\": [\"green wall\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_6c74696a698a\", \"observation\": \"green wall is visible\"},\n {\"context_id\": \"b_c7168d6511cd\", \"observation\": \"green wall is visible\"},\n {\"context_id\": \"b_442bffc55440\", \"observation\": \"green wall is visible\"},\n {\"context_id\": \"b_f2d05c389498\", \"observation\": \"green wall is visible\"}\n ],\n \"uncertainty\": \"green wall is consistently visible across all context images\",\n \"fallback_box_norm_xyxy\": [0, 0, 300, 300]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_e21902361190", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [], "final_mask_path": "masks/b_e21902361190.png", "routing_mask_path": "masks/b_e21902361190_routing.png", "predicted_fraction": 0.0 }, "raw_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.21333736577991802 }, "routing_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 0.0, "safety_recall": 0.0, "leakage": 1.0, "overmask": 0.0, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.21333736577991802 }, "box_metrics": { "gt_pixel_coverage": 0.0, "area_precision": 0.0, "box_fraction": 0.0 }, "frame_detected": false, "gt_ambiguity": { "median_rgb_max_difference": 0.011764705181121826, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_442bffc55440", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/test/0031.JPG", "source_split": "test", "source_frame_id": "0031", "clean_reference_path": null, "context_ids": [ "b_63ebbf87a7c1", "b_a38e3ce498d5", "b_e21902361190", "b_6c74696a698a" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_442bffc55440", "target_sha256": "ea5e203e121b9e94bef7b63ba762139d4a7bfd98071db955ef86ed4c1923dbc0", "context_ids": [ "b_63ebbf87a7c1", "b_a38e3ce498d5", "b_e21902361190", "b_6c74696a698a" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "red umbrella", "grounding_queries": [ "red umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_63ebbf87a7c1", "observation": "umbrella is visible" }, { "context_id": "b_a38e3ce498d5", "observation": "umbrella is visible" }, { "context_id": "b_e21902361190", "observation": "umbrella is visible" }, { "context_id": "b_6c74696a698a", "observation": "umbrella is visible" } ], "uncertainty": "umbrella is consistently visible across all contexts", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "sign with Chinese characters", "grounding_queries": [ "sign with Chinese characters" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_63ebbf87a7c1", "observation": "sign is visible" }, { "context_id": "b_a38e3ce498d5", "observation": "sign is visible" }, { "context_id": "b_e21902361190", "observation": "sign is visible" }, { "context_id": "b_6c74696a698a", "observation": "sign is visible" } ], "uncertainty": "sign is consistently visible across all contexts", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"red umbrella\",\n \"grounding_queries\": [\"red umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_63ebbf87a7c1\", \"observation\": \"umbrella is visible\"},\n {\"context_id\": \"b_a38e3ce498d5\", \"observation\": \"umbrella is visible\"},\n {\"context_id\": \"b_e21902361190\", \"observation\": \"umbrella is visible\"},\n {\"context_id\": \"b_6c74696a698a\", \"observation\": \"umbrella is visible\"}\n ],\n \"uncertainty\": \"umbrella is consistently visible across all contexts\",\n \"fallback_box_norm_xyxy\": [294, 267, 400, 370]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"sign with Chinese characters\",\n \"grounding_queries\": [\"sign with Chinese characters\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_63ebbf87a7c1\", \"observation\": \"sign is visible\"},\n {\"context_id\": \"b_a38e3ce498d5\", \"observation\": \"sign is visible\"},\n {\"context_id\": \"b_e21902361190\", \"observation\": \"sign is visible\"},\n {\"context_id\": \"b_6c74696a698a\", \"observation\": \"sign is visible\"}\n ],\n \"uncertainty\": \"sign is consistently visible across all contexts\",\n \"fallback_box_norm_xyxy\": [220, 400, 290, 476]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_442bffc55440", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [], "final_mask_path": "masks/b_442bffc55440.png", "routing_mask_path": "masks/b_442bffc55440_routing.png", "predicted_fraction": 0.0 }, "raw_mask_metrics": { "iou": 1.0, "dice": 1.0, "precision": 1.0, "recall": 1.0, "safety_recall": 1.0, "leakage": 0.0, "overmask": 0.0, "boundary_f5": 1.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.0 }, "routing_mask_metrics": { "iou": 1.0, "dice": 1.0, "precision": 1.0, "recall": 1.0, "safety_recall": 1.0, "leakage": 0.0, "overmask": 0.0, "boundary_f5": 1.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.0 }, "box_metrics": { "gt_pixel_coverage": 1.0, "area_precision": 1.0, "box_fraction": 0.0 }, "frame_detected": false, "gt_ambiguity": { "median_rgb_max_difference": 0.0, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_f2d05c389498", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/test/0032.JPG", "source_split": "test", "source_frame_id": "0032", "clean_reference_path": null, "context_ids": [ "b_f9646cc9c954", "b_e21902361190", "b_c7168d6511cd", "b_1ea940c8ac4e" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_f2d05c389498", "target_sha256": "ce4ed46f67474b370e54193ad6ad1e8f2e0bf59b35a2c5e4e38dfe5a707dd6b6", "context_ids": [ "b_f9646cc9c954", "b_e21902361190", "b_c7168d6511cd", "b_1ea940c8ac4e" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "red umbrella", "grounding_queries": [ "red umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_f9646cc9c954", "observation": "red umbrella is visible" }, { "context_id": "b_e21902361190", "observation": "red umbrella is visible" }, { "context_id": "b_c7168d6511cd", "observation": "red umbrella is visible" }, { "context_id": "b_1ea940c8ac4e", "observation": "red umbrella is visible" } ], "uncertainty": "red umbrella is consistently visible across all context images", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "black and white umbrella", "grounding_queries": [ "black and white umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_f9646cc9c954", "observation": "black and white umbrella is visible" }, { "context_id": "b_e21902361190", "observation": "black and white umbrella is visible" }, { "context_id": "b_c7168d6511cd", "observation": "black and white umbrella is visible" }, { "context_id": "b_1ea940c8ac4e", "observation": "black and white umbrella is visible" } ], "uncertainty": "black and white umbrella is consistently visible across all context images", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"red umbrella\",\n \"grounding_queries\": [\"red umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_f9646cc9c954\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_e21902361190\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_c7168d6511cd\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_1ea940c8ac4e\", \"observation\": \"red umbrella is visible\"}\n ],\n \"uncertainty\": \"red umbrella is consistently visible across all context images\",\n \"fallback_box_norm_xyxy\": [320, 224, 450, 345]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"black and white umbrella\",\n \"grounding_queries\": [\"black and white umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_f9646cc9c954\", \"observation\": \"black and white umbrella is visible\"},\n {\"context_id\": \"b_e21902361190\", \"observation\": \"black and white umbrella is visible\"},\n {\"context_id\": \"b_c7168d6511cd\", \"observation\": \"black and white umbrella is visible\"},\n {\"context_id\": \"b_1ea940c8ac4e\", \"observation\": \"black and white umbrella is visible\"}\n ],\n \"uncertainty\": \"black and white umbrella is consistently visible across all context images\",\n \"fallback_box_norm_xyxy\": [140, 300, 260, 420]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_f2d05c389498", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [], "final_mask_path": "masks/b_f2d05c389498.png", "routing_mask_path": "masks/b_f2d05c389498_routing.png", "predicted_fraction": 0.0 }, "raw_mask_metrics": { "iou": 1.0, "dice": 1.0, "precision": 1.0, "recall": 1.0, "safety_recall": 1.0, "leakage": 0.0, "overmask": 0.0, "boundary_f5": 1.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.0 }, "routing_mask_metrics": { "iou": 1.0, "dice": 1.0, "precision": 1.0, "recall": 1.0, "safety_recall": 1.0, "leakage": 0.0, "overmask": 0.0, "boundary_f5": 1.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.0 }, "box_metrics": { "gt_pixel_coverage": 1.0, "area_precision": 1.0, "box_fraction": 0.0 }, "frame_detected": false, "gt_ambiguity": { "median_rgb_max_difference": 0.0, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_0a52973d1b8d", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/test/0033.JPG", "source_split": "test", "source_frame_id": "0033", "clean_reference_path": null, "context_ids": [ "b_48b647235f33", "b_3fad5029fb0d", "b_6125c5000375", "b_c7dd5015d023" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_0a52973d1b8d", "target_sha256": "02d3286042e0c5392fa100b8c69e3a41868ea2bbd5a3cc9429cbb1b3789f65d7", "context_ids": [ "b_48b647235f33", "b_3fad5029fb0d", "b_6125c5000375", "b_c7dd5015d023" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "red umbrella", "grounding_queries": [ "red umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_48b647235f33", "observation": "red umbrella is visible" }, { "context_id": "b_3fad5029fb0d", "observation": "red umbrella is visible" }, { "context_id": "b_6125c5000375", "observation": "red umbrella is visible" }, { "context_id": "b_c7dd5015d023", "observation": "red umbrella is not visible" } ], "uncertainty": "red umbrella is not visible in the last context image", "fallback_box_norm_xyxy": null, "routing_eligible": true, "consistency_filter": "passed" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "green curtain", "grounding_queries": [ "green curtain" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_48b647235f33", "observation": "green curtain is visible" }, { "context_id": "b_3fad5029fb0d", "observation": "green curtain is visible" }, { "context_id": "b_6125c5000375", "observation": "green curtain is visible" }, { "context_id": "b_c7dd5015d023", "observation": "green curtain is not visible" } ], "uncertainty": "green curtain is not visible in the last context image", "fallback_box_norm_xyxy": null, "routing_eligible": true, "consistency_filter": "passed" }, { "region_id": "r3", "cause": "transient_occluder", "semantic_description": "black chair", "grounding_queries": [ "black chair" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_48b647235f33", "observation": "black chair is visible" }, { "context_id": "b_3fad5029fb0d", "observation": "black chair is visible" }, { "context_id": "b_6125c5000375", "observation": "black chair is visible" }, { "context_id": "b_c7dd5015d023", "observation": "black chair is not visible" } ], "uncertainty": "black chair is not visible in the last context image", "fallback_box_norm_xyxy": null, "routing_eligible": true, "consistency_filter": "passed" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"red umbrella\",\n \"grounding_queries\": [\"red umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_48b647235f33\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_3fad5029fb0d\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_6125c5000375\", \"observation\": \"red umbrella is visible\"},\n {\"context_id\": \"b_c7dd5015d023\", \"observation\": \"red umbrella is not visible\"}\n ],\n \"uncertainty\": \"red umbrella is not visible in the last context image\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"green curtain\",\n \"grounding_queries\": [\"green curtain\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_48b647235f33\", \"observation\": \"green curtain is visible\"},\n {\"context_id\": \"b_3fad5029fb0d\", \"observation\": \"green curtain is visible\"},\n {\"context_id\": \"b_6125c5000375\", \"observation\": \"green curtain is visible\"},\n {\"context_id\": \"b_c7dd5015d023\", \"observation\": \"green curtain is not visible\"}\n ],\n \"uncertainty\": \"green curtain is not visible in the last context image\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r3\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"black chair\",\n \"grounding_queries\": [\"black chair\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_48b647235f33\", \"observation\": \"black chair is visible\"},\n {\"context_id\": \"b_3fad5029fb0d\", \"observation\": \"black chair is visible\"},\n {\"context_id\": \"b_6125c5000375\", \"observation\": \"black chair is visible\"},\n {\"context_id\": \"b_c7dd5015d023\", \"observation\": \"black chair is not visible\"}\n ],\n \"uncertainty\": \"black chair is not visible in the last context image\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_0a52973d1b8d", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "red umbrella", "queries": [ "red umbrella" ], "source": "grounding", "boxes_xyxy": [], "box_scores": [], "box_queries": [], "sam_predicted_iou": [], "mask_path": "masks/b_0a52973d1b8d_r1.png" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "green curtain", "queries": [ "green curtain" ], "source": "grounding", "boxes_xyxy": [], "box_scores": [], "box_queries": [], "sam_predicted_iou": [], "mask_path": "masks/b_0a52973d1b8d_r2.png" }, { "region_id": "r3", "cause": "transient_occluder", "semantic_description": "black chair", "queries": [ "black chair" ], "source": "grounding", "boxes_xyxy": [], "box_scores": [], "box_queries": [], "sam_predicted_iou": [], "mask_path": "masks/b_0a52973d1b8d_r3.png" } ], "final_mask_path": "masks/b_0a52973d1b8d.png", "routing_mask_path": "masks/b_0a52973d1b8d_routing.png", "predicted_fraction": 0.0 }, "raw_mask_metrics": { "iou": 1.0, "dice": 1.0, "precision": 1.0, "recall": 1.0, "safety_recall": 1.0, "leakage": 0.0, "overmask": 0.0, "boundary_f5": 1.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.0 }, "routing_mask_metrics": { "iou": 1.0, "dice": 1.0, "precision": 1.0, "recall": 1.0, "safety_recall": 1.0, "leakage": 0.0, "overmask": 0.0, "boundary_f5": 1.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.0 }, "box_metrics": { "gt_pixel_coverage": 1.0, "area_precision": 1.0, "box_fraction": 0.0 }, "frame_detected": false, "gt_ambiguity": { "median_rgb_max_difference": 0.0, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_66a730d6ef90", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/test/0034.JPG", "source_split": "test", "source_frame_id": "0034", "clean_reference_path": null, "context_ids": [ "b_42d9140f0598", "b_3d50f98d854f", "b_2d3a471eb3b2", "b_2962b6747c5c" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_66a730d6ef90", "target_sha256": "2caa6d5ab56862347537483d461fa637f435a1d2209c4e4106f78cab764e5592", "context_ids": [ "b_42d9140f0598", "b_3d50f98d854f", "b_2d3a471eb3b2", "b_2962b6747c5c" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "mannequin", "grounding_queries": [ "mannequin" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_42d9140f0598", "observation": "mannequin partially obscures the teddy bear" }, { "context_id": "b_3d50f98d854f", "observation": "mannequin partially obscures the teddy bear" }, { "context_id": "b_2d3a471eb3b2", "observation": "mannequin partially obscures the teddy bear" }, { "context_id": "b_2962b6747c5c", "observation": "mannequin partially obscures the teddy bear" } ], "uncertainty": "mannequin is partially visible in all context images, indicating it is not a spatial nuisance.", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" }, { "region_id": "r2", "cause": "transient_occluder", "semantic_description": "umbrella", "grounding_queries": [ "umbrella" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_42d9140f0598", "observation": "umbrella partially obscures the teddy bear" }, { "context_id": "b_3d50f98d854f", "observation": "umbrella partially obscures the teddy bear" }, { "context_id": "b_2d3a471eb3b2", "observation": "umbrella partially obscures the teddy bear" }, { "context_id": "b_2962b6747c5c", "observation": "umbrella partially obscures the teddy bear" } ], "uncertainty": "umbrella is partially visible in all context images, indicating it is not a spatial nuisance.", "fallback_box_norm_xyxy": null, "routing_eligible": false, "consistency_filter": "rejected_explicitly_persistent" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"mannequin\",\n \"grounding_queries\": [\"mannequin\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_42d9140f0598\", \"observation\": \"mannequin partially obscures the teddy bear\"},\n {\"context_id\": \"b_3d50f98d854f\", \"observation\": \"mannequin partially obscures the teddy bear\"},\n {\"context_id\": \"b_2d3a471eb3b2\", \"observation\": \"mannequin partially obscures the teddy bear\"},\n {\"context_id\": \"b_2962b6747c5c\", \"observation\": \"mannequin partially obscures the teddy bear\"}\n ],\n \"uncertainty\": \"mannequin is partially visible in all context images, indicating it is not a spatial nuisance.\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n },\n {\n \"region_id\": \"r2\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"umbrella\",\n \"grounding_queries\": [\"umbrella\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_42d9140f0598\", \"observation\": \"umbrella partially obscures the teddy bear\"},\n {\"context_id\": \"b_3d50f98d854f\", \"observation\": \"umbrella partially obscures the teddy bear\"},\n {\"context_id\": \"b_2d3a471eb3b2\", \"observation\": \"umbrella partially obscures the teddy bear\"},\n {\"context_id\": \"b_2962b6747c5c\", \"observation\": \"umbrella partially obscures the teddy bear\"}\n ],\n \"uncertainty\": \"umbrella is partially visible in all context images, indicating it is not a spatial nuisance.\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_66a730d6ef90", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [], "final_mask_path": "masks/b_66a730d6ef90.png", "routing_mask_path": "masks/b_66a730d6ef90_routing.png", "predicted_fraction": 0.0 }, "raw_mask_metrics": { "iou": 1.0, "dice": 1.0, "precision": 1.0, "recall": 1.0, "safety_recall": 1.0, "leakage": 0.0, "overmask": 0.0, "boundary_f5": 1.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.0 }, "routing_mask_metrics": { "iou": 1.0, "dice": 1.0, "precision": 1.0, "recall": 1.0, "safety_recall": 1.0, "leakage": 0.0, "overmask": 0.0, "boundary_f5": 1.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.0, "gt_fraction": 0.0 }, "box_metrics": { "gt_pixel_coverage": 1.0, "area_precision": 1.0, "box_fraction": 0.0 }, "frame_detected": false, "gt_ambiguity": { "median_rgb_max_difference": 0.0, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } }, { "sample_id": "b_17642d7e1164", "source_path": "/local/scratch-2/fy277/temp/diag3dgs/data/realx3d/data_4/dynamic/Ujikintoki/test/0035.JPG", "source_split": "test", "source_frame_id": "0035", "clean_reference_path": null, "context_ids": [ "b_42d3a74baab7", "b_d95203c68daa", "b_c0cd1b9b9c0e", "b_8ffaf4db41bf" ], "vlm": { "schema_version": "diag.region.v1", "sample_id": "b_17642d7e1164", "target_sha256": "ee1fbf84bece876f54c8a63e2d4bbff0086d2ee90c0ed535b19e4040d6381f54", "context_ids": [ "b_42d3a74baab7", "b_d95203c68daa", "b_c0cd1b9b9c0e", "b_8ffaf4db41bf" ], "model": { "provider": "local", "name": "qwen2.5-vl-7b-instruct", "version": "local_snapshot", "quantization": "nf4" }, "prompt_sha256": "15497acb4f0a5266ff625a2745e66a38aa75c362c8f6e3fe7b4bfb2bc5003a36", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "mirror", "grounding_queries": [ "mirror" ], "is_spatial": true, "confidence": 0.9, "context_evidence": [ { "context_id": "b_42d3a74baab7", "observation": "mirror partially obscures the teddy bear" }, { "context_id": "b_d95203c68daa", "observation": "mirror fully obscures the teddy bear" }, { "context_id": "b_c0cd1b9b9c0e", "observation": "mirror partially obscures the teddy bear" }, { "context_id": "b_8ffaf4db41bf", "observation": "mirror fully obscures the teddy bear" } ], "uncertainty": "mirror's position varies across views", "fallback_box_norm_xyxy": null, "routing_eligible": true, "consistency_filter": "passed" } ], "global_causes": [], "parse_error": null, "format_repair": "wrapped_top_level_region_list", "validation_errors": [] }, "vlm_raw": "```json\n[\n {\n \"region_id\": \"r1\",\n \"cause\": \"transient_occluder\",\n \"semantic_description\": \"mirror\",\n \"grounding_queries\": [\"mirror\"],\n \"is_spatial\": true,\n \"confidence\": 0.9,\n \"context_evidence\": [\n {\"context_id\": \"b_42d3a74baab7\", \"observation\": \"mirror partially obscures the teddy bear\"},\n {\"context_id\": \"b_d95203c68daa\", \"observation\": \"mirror fully obscures the teddy bear\"},\n {\"context_id\": \"b_c0cd1b9b9c0e\", \"observation\": \"mirror partially obscures the teddy bear\"},\n {\"context_id\": \"b_8ffaf4db41bf\", \"observation\": \"mirror fully obscures the teddy bear\"}\n ],\n \"uncertainty\": \"mirror's position varies across views\",\n \"fallback_box_norm_xyxy\": [0.0, 0.0, 1.0, 1.0]\n }\n]\n```", "grounding": { "schema_version": "diag.ground_sam.v1", "sample_id": "b_17642d7e1164", "thresholds": { "vlm_confidence": 0.35, "box": 0.25, "text": 0.2, "nms": 0.5 }, "grounding_model": "/local/scratch-2/fy277/temp/diag3dgs/models/grounding-dino-tiny", "sam_checkpoint": "/local/scratch-2/fy277/temp/diag3dgs/models/sam2.1/sam2.1_hiera_small.pt", "sam_checkpoint_sha256": "6d1aa6f30de5c92224f8172114de081d104bbd23dd9dc5c58996f0cad5dc4d38", "regions": [ { "region_id": "r1", "cause": "transient_occluder", "semantic_description": "mirror", "queries": [ "mirror" ], "source": "grounding", "boxes_xyxy": [ [ 592.1898193359375, 452.15679931640625, 883.3391723632812, 680.7554931640625 ] ], "box_scores": [ 0.3116818070411682 ], "box_queries": [ "mirror" ], "sam_predicted_iou": [ 0.9296875 ], "mask_path": "masks/b_17642d7e1164_r1.png" } ], "final_mask_path": "masks/b_17642d7e1164.png", "routing_mask_path": "masks/b_17642d7e1164_routing.png", "predicted_fraction": 0.015087096950145713 }, "raw_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 1.0, "safety_recall": 1.0, "leakage": 0.0, "overmask": 0.015087096950145713, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.015087096950145713, "gt_fraction": 0.0 }, "routing_mask_metrics": { "iou": 0.0, "dice": 0.0, "precision": 0.0, "recall": 1.0, "safety_recall": 1.0, "leakage": 0.0, "overmask": 0.018559968519644053, "boundary_f5": 0.0, "iou_ignore_boundary5": 0.0, "predicted_fraction": 0.018559968519644053, "gt_fraction": 0.0 }, "box_metrics": { "gt_pixel_coverage": 1.0, "area_precision": 0.0, "box_fraction": 0.03245388984992286 }, "frame_detected": true, "gt_ambiguity": { "median_rgb_max_difference": 0.0, "note": "paired-reference pseudo-GT may include lighting, shadow, reflection, and registration differences" } } ]