{ "run_id": "pact_eval100_sample_eval_20260521", "timestamp": "2026-05-21T12:17:39+00:00", "selection_policy": "source-aware random: one random sample per Dataset 2.1 source plus one extra random sample from the remaining 96", "seed": 20260521, "pact_steps": 8, "dataset_validation": { "dataset_root": "/data/250010098/PAct-Transporter/datasets/pact_eval100_2_1_coverage_20260521", "num_records": 100, "unique_sample_ids": 100, "by_source": { "ArtVIP": 25, "GAPartNet": 25, "GRScenes": 25, "PartNetMobility": 25 }, "by_category": { "Clock": 4, "Dispenser": 4, "architectural_fixtures": 11, "electronics": 5, "household_fixtures": 13, "household_items": 6, "large_furniture": 4, "major_appliances": 22, "small_appliances": 16, "small_furniture": 9, "storage": 6 }, "by_problem_tag": { "cross_source_domain_gap": 75, "hard_motion": 74, "internal_or_occluded_part_risk": 48, "known_quality_or_conversion_issue": 40, "many_parts": 56, "mixed_joint_types": 37, "non_pm_generalization": 75, "part_decomposition": 100, "prismatic_motion": 46, "revolute_motion": 91, "single_view_conditioning": 100, "thin_structure": 9, "unseen_category": 15 }, "parsed_sdf_count": 100, "preview_count": 100, "issues": [], "ok": true }, "sample_records": [ { "sample_name": "002_ArtVIP_major_appliances_major_appliances_model_dishwasher_3", "benchmark_index": 2, "sample_id": "ArtVIP/major_appliances/model_dishwasher_3", "source_dataset": "ArtVIP", "category": "major_appliances", "object_id": "major_appliances/model_dishwasher_3", "num_parts": 4, "movable_parts": 3, "pact_object_json": "/data/250010098/PAct-Transporter/reports/20260513_pact_transporter_exploration/runs/pact_eval100_sample_eval_20260521/pact_outputs/seed42_slatcfg7.0_sscfg7.0_sssteps8_slatsteps8_artioutmean_feature_regression_steps/exported_arti_objects/002_ArtVIP_major_appliances_major_appliances_model_dishwasher_3@002_ArtVIP_major_appliances_major_appliances_model_dishwasher_3_synthetic_processed.@ng/object.json", "pact_exploded_png": "/data/250010098/PAct-Transporter/reports/20260513_pact_transporter_exploration/runs/pact_eval100_sample_eval_20260521/pact_outputs/seed42_slatcfg7.0_sscfg7.0_sssteps8_slatsteps8_artioutmean_feature_regression_steps/00__exploded_part.png", "metric_groups": { "pipeline_availability": { "inference_success": { "value": 1, "status": "measured", "note": "" }, "object_json_valid": { "value": 1, "status": "measured", "note": "" }, "glb_export_success": { "value": 0, "status": "not_provided", "note": "This smoke run exports articulated object JSON/PLY, not textured GLB." }, "part_asset_completeness": { "value": 1.0, "status": "measured", "note": "" }, "runtime_seconds": { "value": 19.840816974639893, "status": "measured", "note": "" }, "peak_gpu_memory_mb": { "value": null, "status": "not_provided", "note": "Not sampled in this run." }, "reproducible_seed_match": { "value": 1, "status": "measured", "note": "" } }, "geometry": { "chamfer_l2": { "value": null, "status": "not_provided", "note": "PLY-level Chamfer hook present; omitted in smoke run for speed." }, "fscore_tau_0_01": { "value": null, "status": "not_provided", "note": "Requires dense surface sampling." }, "normal_consistency": { "value": null, "status": "not_provided", "note": "Requires dense surface sampling." }, "scale_consistency": { "value": 0.4695195165659668, "status": "measured", "note": "" }, "mesh_manifold_sanity": { "value": null, "status": "not_provided", "note": "Requires mesh repair/manifold pass." }, "texture_or_material_coverage": { "value": 0.0, "status": "not_provided", "note": "PAct smoke export is untextured PLY." } }, "part_decomposition": { "part_count_mae": { "value": 2, "status": "measured", "note": "" }, "part_count_accuracy": { "value": 0.5, "status": "measured", "note": "" }, "part_segmentation_miou_if_masks_available": { "value": null, "status": "requires_gt", "note": "2D mask GT is conditioning input; not a predicted mask output." }, "semantic_part_accuracy": { "value": null, "status": "not_provided", "note": "PAct output names are not semantic labels in this export." }, "parent_child_accuracy": { "value": 1.0, "status": "measured", "note": "" }, "tree_edit_distance": { "value": 2, "status": "measured", "note": "" }, "orphan_duplicate_part_rate": { "value": 0.0, "status": "measured", "note": "" } }, "kinematics": { "joint_count_mae": { "value": 3, "status": "measured", "note": "" }, "joint_precision": { "value": 0.0, "status": "measured", "note": "" }, "joint_recall": { "value": 0.0, "status": "measured", "note": "" }, "joint_f1": { "value": 0.0, "status": "measured", "note": "" }, "joint_type_accuracy": { "value": 0.0, "status": "measured", "note": "" }, "axis_angular_error_deg": { "value": 90.0, "status": "measured", "note": "" }, "origin_distance_error": { "value": 0.0, "status": "measured", "note": "" }, "joint_limit_error": { "value": 0.0, "status": "measured", "note": "" }, "kinematic_tree_validity": { "value": 1, "status": "measured", "note": "" } }, "motion_consistency": { "detachment_distance": { "value": null, "status": "requires_simulation", "note": "" }, "collision_rate": { "value": null, "status": "requires_simulation", "note": "" }, "self_collision_rate": { "value": null, "status": "requires_simulation", "note": "" }, "closed_state_assembly_error": { "value": 0.5304804834340332, "status": "measured", "note": "" }, "open_state_plausibility": { "value": 0.0, "status": "measured", "note": "" }, "trajectory_smoothness": { "value": null, "status": "requires_simulation", "note": "" } }, "ot_routing_diagnostics": { "ot_modules_active": { "value": 0, "status": "not_applicable", "note": "Raw PAct baseline; no OT module." }, "ot_gate_mean": { "value": null, "status": "not_applicable", "note": "" }, "ot_plan_marginal_error": { "value": null, "status": "not_applicable", "note": "" }, "ot_assignment_entropy": { "value": null, "status": "not_applicable", "note": "" }, "ot_edge_delta": { "value": null, "status": "not_applicable", "note": "" }, "ot_mask_prior_beta": { "value": null, "status": "not_applicable", "note": "" }, "ot_mask_dropout_delta": { "value": null, "status": "not_applicable", "note": "" }, "ot_virtual_mass": { "value": null, "status": "not_applicable", "note": "" }, "part_patch_compactness": { "value": null, "status": "not_applicable", "note": "" } }, "debug": { "pred_part_count": 2, "gt_part_count": 4, "pred_joint_count": 0, "gt_joint_count": 3, "matches": [], "pred_tree": { "root_count": 1, "duplicate_ids": 0, "dangling_parent_count": 0, "cycle_node_count": 0, "valid": 1 }, "ref_tree": { "root_count": 1, "duplicate_ids": 0, "dangling_parent_count": 0, "cycle_node_count": 0, "valid": 1 } }, "robustness_breakdown": { "macro_score_by_source_dataset": { "value": { "ArtVIP": 33.72558544108934, "GAPartNet": 23.258877226688433, "GRScenes": 65.82505051979625, "PartNetMobility": 23.46849844636604 }, "status": "measured", "note": "5-sample source-aware random smoke subset" }, "macro_score_by_category": { "value": { "electronics": 23.363687836527237, "major_appliances": 47.1455740748926, "small_appliances": 38.985073252189736 }, "status": "measured", "note": "computed on 5-sample subset" }, "macro_score_by_problem_tag": { "value": { "cross_source_domain_gap": 39.133774657165844, "hard_motion": 39.18617996208525, "internal_or_occluded_part_risk": 44.42540713399165, "known_quality_or_conversion_issue": 30.306556442848244, "many_parts": 25.06449110101447, "mixed_joint_types": 47.1455740748926, "non_pm_generalization": 39.133774657165844, "part_decomposition": 36.00071941500588, "prismatic_motion": 35.254630955709914, "revolute_motion": 44.42540713399165, "single_view_conditioning": 36.00071941500588 }, "status": "measured", "note": "computed on 5-sample subset" }, "worst_10pct_score": { "value": 23.258877226688433, "status": "measured", "note": "minimum sample score because n=5" }, "failure_cluster_count": { "value": null, "status": "not_provided", "note": "requires larger batch clustering" } } }, "group_scores": { "pipeline_availability": 100.0, "geometry": 46.95195165659668, "part_decomposition": 50.0, "kinematics": 0.0, "motion_consistency": 0.0, "ot_routing_diagnostics": 0.0 }, "weighted_score": 28.466097629988948, "vlm_card": "/data/250010098/PAct-Transporter/reports/20260513_pact_transporter_exploration/runs/pact_eval100_sample_eval_20260521/vlm_cards/002_ArtVIP_major_appliances_major_appliances_model_dishwasher_3.png", "vlm": { "ok": true, "model": "gemini-2.5-flash", "text": "- 几何形状基本准确，但PAct输出的物体边缘有明显的像素化，不如输入平滑。\n- Part分解严重不足，GT有4个部件，PAct只识别出2个，且分解方式与GT不符，未能正确分离底部结构。\n- 关节和运动信息完全缺失，GT有3个关节，PAct识别为0，导致joint_f1为0.00，无法实现铰链物体的运动。", "status_code": 200 }, "pact_animation_mp4": "/data/250010098/PAct-Transporter/reports/20260513_pact_transporter_exploration/runs/pact_eval100_sample_eval_20260521/pact_outputs/seed42_slatcfg7.0_sscfg7.0_sssteps8_slatsteps8_artioutmean_feature_regression_steps/00__articulation_animation.mp4" }, { "sample_name": "036_GAPartNet_electronics_electronics_104007", "benchmark_index": 36, "sample_id": "GAPartNet/electronics/104007", "source_dataset": "GAPartNet", "category": "electronics", "object_id": "electronics/104007", "num_parts": 12, "movable_parts": 11, "pact_object_json": "/data/250010098/PAct-Transporter/reports/20260513_pact_transporter_exploration/runs/pact_eval100_sample_eval_20260521/pact_outputs/seed42_slatcfg7.0_sscfg7.0_sssteps8_slatsteps8_artioutmean_feature_regression_steps/exported_arti_objects/036_GAPartNet_electronics_electronics_104007@036_GAPartNet_electronics_electronics_104007_synthetic_processed.@ng/object.json", "pact_exploded_png": "/data/250010098/PAct-Transporter/reports/20260513_pact_transporter_exploration/runs/pact_eval100_sample_eval_20260521/pact_outputs/seed42_slatcfg7.0_sscfg7.0_sssteps8_slatsteps8_artioutmean_feature_regression_steps/01__exploded_part.png", "metric_groups": { "pipeline_availability": { "inference_success": { "value": 1, "status": "measured", "note": "" }, "object_json_valid": { "value": 1, "status": "measured", "note": "" }, "glb_export_success": { "value": 0, "status": "not_provided", "note": "This smoke run exports articulated object JSON/PLY, not textured GLB." }, "part_asset_completeness": { "value": 1.0, "status": "measured", "note": "" }, "runtime_seconds": { "value": 19.840816974639893, "status": "measured", "note": "" }, "peak_gpu_memory_mb": { "value": null, "status": "not_provided", "note": "Not sampled in this run." }, "reproducible_seed_match": { "value": 1, "status": "measured", "note": "" } }, "geometry": { "chamfer_l2": { "value": null, "status": "not_provided", "note": "PLY-level Chamfer hook present; omitted in smoke run for speed." }, "fscore_tau_0_01": { "value": null, "status": "not_provided", "note": "Requires dense surface sampling." }, "normal_consistency": { "value": null, "status": "not_provided", "note": "Requires dense surface sampling." }, "scale_consistency": { "value": 0.5841733354680451, "status": "measured", "note": "" }, "mesh_manifold_sanity": { "value": null, "status": "not_provided", "note": "Requires mesh repair/manifold pass." }, "texture_or_material_coverage": { "value": 0.0, "status": "not_provided", "note": "PAct smoke export is untextured PLY." } }, "part_decomposition": { "part_count_mae": { "value": 10, "status": "measured", "note": "" }, "part_count_accuracy": { "value": 0.16666666666666663, "status": "measured", "note": "" }, "part_segmentation_miou_if_masks_available": { "value": null, "status": "requires_gt", "note": "2D mask GT is conditioning input; not a predicted mask output." }, "semantic_part_accuracy": { "value": null, "status": "not_provided", "note": "PAct output names are not semantic labels in this export." }, "parent_child_accuracy": { "value": 1.0, "status": "measured", "note": "" }, "tree_edit_distance": { "value": 10, "status": "measured", "note": "" }, "orphan_duplicate_part_rate": { "value": 0.0, "status": "measured", "note": "" } }, "kinematics": { "joint_count_mae": { "value": 10, "status": "measured", "note": "" }, "joint_precision": { "value": 0.0, "status": "measured", "note": "" }, "joint_recall": { "value": 0.0, "status": "measured", "note": "" }, "joint_f1": { "value": 0.0, "status": "measured", "note": "" }, "joint_type_accuracy": { "value": 0.0, "status": "measured", "note": "" }, "axis_angular_error_deg": { "value": 90.0, "status": "measured", "note": "" }, "origin_distance_error": { "value": 0.0, "status": "measured", "note": "" }, "joint_limit_error": { "value": 0.0, "status": "measured", "note": "" }, "kinematic_tree_validity": { "value": 1, "status": "measured", "note": "" } }, "motion_consistency": { "detachment_distance": { "value": null, "status": "requires_simulation", "note": "" }, "collision_rate": { "value": null, "status": "requires_simulation", "note": "" }, "self_collision_rate": { "value": null, "status": "requires_simulation", "note": "" }, "closed_state_assembly_error": { "value": 0.41582666453195494, "status": "measured", "note": "" }, "open_state_plausibility": { "value": 0.0, "status": "measured", "note": "" }, "trajectory_smoothness": { "value": null, "status": "requires_simulation", "note": "" } }, "ot_routing_diagnostics": { "ot_modules_active": { "value": 0, "status": "not_applicable", "note": "Raw PAct baseline; no OT module." }, "ot_gate_mean": { "value": null, "status": "not_applicable", "note": "" }, "ot_plan_marginal_error": { "value": null, "status": "not_applicable", "note": "" }, "ot_assignment_entropy": { "value": null, "status": "not_applicable", "note": "" }, "ot_edge_delta": { "value": null, "status": "not_applicable", "note": "" }, "ot_mask_prior_beta": { "value": null, "status": "not_applicable", "note": "" }, "ot_mask_dropout_delta": { "value": null, "status": "not_applicable", "note": "" }, "ot_virtual_mass": { "value": null, "status": "not_applicable", "note": "" }, "part_patch_compactness": { "value": null, "status": "not_applicable", "note": "" } }, "debug": { "pred_part_count": 2, "gt_part_count": 12, "pred_joint_count": 1, "gt_joint_count": 11, "matches": [ { "type_mismatch": 1.0, "axis_error_deg": 46.791789455707395, "origin_error": 0.4578715952642887, "limit_error": 11.847575332641602, "accepted": false } ], "pred_tree": { "root_count": 1, "duplicate_ids": 0, "dangling_parent_count": 0, "cycle_node_count": 0, "valid": 1 }, "ref_tree": { "root_count": 1, "duplicate_ids": 0, "dangling_parent_count": 0, "cycle_node_count": 0, "valid": 1 } }, "robustness_breakdown": { "macro_score_by_source_dataset": { "value": { "ArtVIP": 33.72558544108934, "GAPartNet": 23.258877226688433, "GRScenes": 65.82505051979625, "PartNetMobility": 23.46849844636604 }, "status": "measured", "note": "5-sample source-aware random smoke subset" }, "macro_score_by_category": { "value": { "electronics": 23.363687836527237, "major_appliances": 47.1455740748926, "small_appliances": 38.985073252189736 }, "status": "measured", "note": "computed on 5-sample subset" }, "macro_score_by_problem_tag": { "value": { "cross_source_domain_gap": 39.133774657165844, "hard_motion": 39.18617996208525, "internal_or_occluded_part_risk": 44.42540713399165, "known_quality_or_conversion_issue": 30.306556442848244, "many_parts": 25.06449110101447, "mixed_joint_types": 47.1455740748926, "non_pm_generalization": 39.133774657165844, "part_decomposition": 36.00071941500588, "prismatic_motion": 35.254630955709914, "revolute_motion": 44.42540713399165, "single_view_conditioning": 36.00071941500588 }, "status": "measured", "note": "computed on 5-sample subset" }, "worst_10pct_score": { "value": 23.258877226688433, "status": "measured", "note": "minimum sample score because n=5" }, "failure_cluster_count": { "value": null, "status": "not_provided", "note": "requires larger batch clustering" } } }, "group_scores": { "pipeline_availability": 100.0, "geometry": 58.41733354680451, "part_decomposition": 16.666666666666664, "kinematics": 0.0, "motion_consistency": 0.0, "ot_routing_diagnostics": 0.0 }, "weighted_score": 23.258877226688433, "vlm_card": "/data/250010098/PAct-Transporter/reports/20260513_pact_transporter_exploration/runs/pact_eval100_sample_eval_20260521/vlm_cards/036_GAPartNet_electronics_electronics_104007.png", "vlm": { "ok": true, "model": "gemini-2.5-flash", "text": "- 几何形状基本还原，但细节缺失，如按钮和屏幕。\n- 部件分解严重不足，将12个GT部件合并为2个，导致关节信息几乎完全丢失。\n- 关节检测失败，仅识别出1个关节，与GT的11个关节相去甚远，joint_f1为0.00表明运动学理解能力极差。", "status_code": 200 }, "pact_animation_mp4": "/data/250010098/PAct-Transporter/reports/20260513_pact_transporter_exploration/runs/pact_eval100_sample_eval_20260521/pact_outputs/seed42_slatcfg7.0_sscfg7.0_sssteps8_slatsteps8_artioutmean_feature_regression_steps/01__articulation_animation.mp4" }, { "sample_name": "047_GRScenes_major_appliances_major_appliances_grscenes_washingmachine_8f10d8438747_cdff8c", "benchmark_index": 47, "sample_id": "GRScenes/major_appliances/grscenes_washingmachine_8f10d8438747_cdff8c", "source_dataset": "GRScenes", "category": "major_appliances", "object_id": "major_appliances/grscenes_washingmachine_8f10d8438747_cdff8c", "num_parts": 3, "movable_parts": 2, "pact_object_json": "/data/250010098/PAct-Transporter/reports/20260513_pact_transporter_exploration/runs/pact_eval100_sample_eval_20260521/pact_outputs/seed42_slatcfg7.0_sscfg7.0_sssteps8_slatsteps8_artioutmean_feature_regression_steps/exported_arti_objects/047_GRScenes_major_appliances_major_appliances_grscenes_washingmachine_8f10d8438747_cdff8c@047_GRScenes_major_appliances_major_appliances_grscenes_washingmachine_8f10d8438747_cdff8c_synthetic_processed.@ng/object.json", "pact_exploded_png": "/data/250010098/PAct-Transporter/reports/20260513_pact_transporter_exploration/runs/pact_eval100_sample_eval_20260521/pact_outputs/seed42_slatcfg7.0_sscfg7.0_sssteps8_slatsteps8_artioutmean_feature_regression_steps/02__exploded_part.png", "metric_groups": { "pipeline_availability": { "inference_success": { "value": 1, "status": "measured", "note": "" }, "object_json_valid": { "value": 1, "status": "measured", "note": "" }, "glb_export_success": { "value": 0, "status": "not_provided", "note": "This smoke run exports articulated object JSON/PLY, not textured GLB." }, "part_asset_completeness": { "value": 1.0, "status": "measured", "note": "" }, "runtime_seconds": { "value": 19.840816974639893, "status": "measured", "note": "" }, "peak_gpu_memory_mb": { "value": null, "status": "not_provided", "note": "Not sampled in this run." }, "reproducible_seed_match": { "value": 1, "status": "measured", "note": "" } }, "geometry": { "chamfer_l2": { "value": null, "status": "not_provided", "note": "PLY-level Chamfer hook present; omitted in smoke run for speed." }, "fscore_tau_0_01": { "value": null, "status": "not_provided", "note": "Requires dense surface sampling." }, "normal_consistency": { "value": null, "status": "not_provided", "note": "Requires dense surface sampling." }, "scale_consistency": { "value": 0.8355865329204293, "status": "measured", "note": "" }, "mesh_manifold_sanity": { "value": null, "status": "not_provided", "note": "Requires mesh repair/manifold pass." }, "texture_or_material_coverage": { "value": 0.0, "status": "not_provided", "note": "PAct smoke export is untextured PLY." } }, "part_decomposition": { "part_count_mae": { "value": 0, "status": "measured", "note": "" }, "part_count_accuracy": { "value": 1.0, "status": "measured", "note": "" }, "part_segmentation_miou_if_masks_available": { "value": null, "status": "requires_gt", "note": "2D mask GT is conditioning input; not a predicted mask output." }, "semantic_part_accuracy": { "value": null, "status": "not_provided", "note": "PAct output names are not semantic labels in this export." }, "parent_child_accuracy": { "value": 1.0, "status": "measured", "note": "" }, "tree_edit_distance": { "value": 0, "status": "measured", "note": "" }, "orphan_duplicate_part_rate": { "value": 0.0, "status": "measured", "note": "" } }, "kinematics": { "joint_count_mae": { "value": 0, "status": "measured", "note": "" }, "joint_precision": { "value": 0.5, "status": "measured", "note": "" }, "joint_recall": { "value": 0.5, "status": "measured", "note": "" }, "joint_f1": { "value": 0.5, "status": "measured", "note": "" }, "joint_type_accuracy": { "value": 1.0, "status": "measured", "note": "" }, "axis_angular_error_deg": { "value": 18.450859777423986, "status": "measured", "note": "" }, "origin_distance_error": { "value": 0.0658873053636274, "status": "measured", "note": "" }, "joint_limit_error": { "value": 0.265873958170414, "status": "measured", "note": "" }, "kinematic_tree_validity": { "value": 1, "status": "measured", "note": "" } }, "motion_consistency": { "detachment_distance": { "value": null, "status": "requires_simulation", "note": "" }, "collision_rate": { "value": null, "status": "requires_simulation", "note": "" }, "self_collision_rate": { "value": null, "status": "requires_simulation", "note": "" }, "closed_state_assembly_error": { "value": 0.16441346707957072, "status": "measured", "note": "" }, "open_state_plausibility": { "value": 0.5, "status": "measured", "note": "" }, "trajectory_smoothness": { "value": null, "status": "requires_simulation", "note": "" } }, "ot_routing_diagnostics": { "ot_modules_active": { "value": 0, "status": "not_applicable", "note": "Raw PAct baseline; no OT module." }, "ot_gate_mean": { "value": null, "status": "not_applicable", "note": "" }, "ot_plan_marginal_error": { "value": null, "status": "not_applicable", "note": "" }, "ot_assignment_entropy": { "value": null, "status": "not_applicable", "note": "" }, "ot_edge_delta": { "value": null, "status": "not_applicable", "note": "" }, "ot_mask_prior_beta": { "value": null, "status": "not_applicable", "note": "" }, "ot_mask_dropout_delta": { "value": null, "status": "not_applicable", "note": "" }, "ot_virtual_mass": { "value": null, "status": "not_applicable", "note": "" }, "part_patch_compactness": { "value": null, "status": "not_applicable", "note": "" } }, "debug": { "pred_part_count": 3, "gt_part_count": 3, "pred_joint_count": 2, "gt_joint_count": 2, "matches": [ { "type_mismatch": 0.0, "axis_error_deg": 90.0, "origin_error": 0.8750167555822178, "limit_error": 56.47970645030471, "accepted": false }, { "type_mismatch": 0.0, "axis_error_deg": 18.450859777423986, "origin_error": 0.0658873053636274, "limit_error": 0.265873958170414, "accepted": true } ], "pred_tree": { "root_count": 1, "duplicate_ids": 0, "dangling_parent_count": 0, "cycle_node_count": 0, "valid": 1 }, "ref_tree": { "root_count": 1, "duplicate_ids": 0, "dangling_parent_count": 0, "cycle_node_count": 0, "valid": 1 } }, "robustness_breakdown": { "macro_score_by_source_dataset": { "value": { "ArtVIP": 33.72558544108934, "GAPartNet": 23.258877226688433, "GRScenes": 65.82505051979625, "PartNetMobility": 23.46849844636604 }, "status": "measured", "note": "5-sample source-aware random smoke subset" }, "macro_score_by_category": { "value": { "electronics": 23.363687836527237, "major_appliances": 47.1455740748926, "small_appliances": 38.985073252189736 }, "status": "measured", "note": "computed on 5-sample subset" }, "macro_score_by_problem_tag": { "value": { "cross_source_domain_gap": 39.133774657165844, "hard_motion": 39.18617996208525, "internal_or_occluded_part_risk": 44.42540713399165, "known_quality_or_conversion_issue": 30.306556442848244, "many_parts": 25.06449110101447, "mixed_joint_types": 47.1455740748926, "non_pm_generalization": 39.133774657165844, "part_decomposition": 36.00071941500588, "prismatic_motion": 35.254630955709914, "revolute_motion": 44.42540713399165, "single_view_conditioning": 36.00071941500588 }, "status": "measured", "note": "computed on 5-sample subset" }, "worst_10pct_score": { "value": 23.258877226688433, "status": "measured", "note": "minimum sample score because n=5" }, "failure_cluster_count": { "value": null, "status": "not_provided", "note": "requires larger batch clustering" } } }, "group_scores": { "pipeline_availability": 100.0, "geometry": 83.55865329204293, "part_decomposition": 100.0, "kinematics": 50.0, "motion_consistency": 50.0, "ot_routing_diagnostics": 0.0 }, "weighted_score": 65.82505051979625, "vlm_card": "/data/250010098/PAct-Transporter/reports/20260513_pact_transporter_exploration/runs/pact_eval100_sample_eval_20260521/vlm_cards/047_GRScenes_major_appliances_major_appliances_grscenes_washingmachine_8f10d8438747_cdff8c.png", "vlm": { "ok": true, "model": "gemini-2.5-flash", "text": "- 几何形状基本准确，但PAct输出的部件边缘略显粗糙，不如原始输入平滑。\n- 部件分解与GT一致，PAct成功识别并分离了3个部件，且与2D part mask的分割结果吻合。\n- 关节识别数量正确，但joint_f1分数较低，表明关节位置或类型可能存在偏差，影响了运动的准确性。", "status_code": 200 }, "pact_animation_mp4": "/data/250010098/PAct-Transporter/reports/20260513_pact_transporter_exploration/runs/pact_eval100_sample_eval_20260521/pact_outputs/seed42_slatcfg7.0_sscfg7.0_sssteps8_slatsteps8_artioutmean_feature_regression_steps/02__articulation_animation.mp4" }, { "sample_name": "073_PartNetMobility_electronics_electronics_3811", "benchmark_index": 73, "sample_id": "PartNetMobility/electronics/3811", "source_dataset": "PartNetMobility", "category": "electronics", "object_id": "electronics/3811", "num_parts": 31, "movable_parts": 30, "pact_object_json": "/data/250010098/PAct-Transporter/reports/20260513_pact_transporter_exploration/runs/pact_eval100_sample_eval_20260521/pact_outputs/seed42_slatcfg7.0_sscfg7.0_sssteps8_slatsteps8_artioutmean_feature_regression_steps/exported_arti_objects/073_PartNetMobility_electronics_electronics_3811@073_PartNetMobility_electronics_electronics_3811_synthetic_processed.@ng/object.json", "pact_exploded_png": "/data/250010098/PAct-Transporter/reports/20260513_pact_transporter_exploration/runs/pact_eval100_sample_eval_20260521/pact_outputs/seed42_slatcfg7.0_sscfg7.0_sssteps8_slatsteps8_artioutmean_feature_regression_steps/04__exploded_part.png", "metric_groups": { "pipeline_availability": { "inference_success": { "value": 1, "status": "measured", "note": "" }, "object_json_valid": { "value": 1, "status": "measured", "note": "" }, "glb_export_success": { "value": 0, "status": "not_provided", "note": "This smoke run exports articulated object JSON/PLY, not textured GLB." }, "part_asset_completeness": { "value": 1.0, "status": "measured", "note": "" }, "runtime_seconds": { "value": 19.840816974639893, "status": "measured", "note": "" }, "peak_gpu_memory_mb": { "value": null, "status": "not_provided", "note": "Not sampled in this run." }, "reproducible_seed_match": { "value": 1, "status": "measured", "note": "" } }, "geometry": { "chamfer_l2": { "value": null, "status": "not_provided", "note": "PLY-level Chamfer hook present; omitted in smoke run for speed." }, "fscore_tau_0_01": { "value": null, "status": "not_provided", "note": "Requires dense surface sampling." }, "normal_consistency": { "value": null, "status": "not_provided", "note": "Requires dense surface sampling." }, "scale_consistency": { "value": 0.5185962994526451, "status": "measured", "note": "" }, "mesh_manifold_sanity": { "value": null, "status": "not_provided", "note": "Requires mesh repair/manifold pass." }, "texture_or_material_coverage": { "value": 0.0, "status": "not_provided", "note": "PAct smoke export is untextured PLY." } }, "part_decomposition": { "part_count_mae": { "value": 24, "status": "measured", "note": "" }, "part_count_accuracy": { "value": 0.22580645161290325, "status": "measured", "note": "" }, "part_segmentation_miou_if_masks_available": { "value": null, "status": "requires_gt", "note": "2D mask GT is conditioning input; not a predicted mask output." }, "semantic_part_accuracy": { "value": null, "status": "not_provided", "note": "PAct output names are not semantic labels in this export." }, "parent_child_accuracy": { "value": 1.0, "status": "measured", "note": "" }, "tree_edit_distance": { "value": 24, "status": "measured", "note": "" }, "orphan_duplicate_part_rate": { "value": 0.0, "status": "measured", "note": "" } }, "kinematics": { "joint_count_mae": { "value": 27, "status": "measured", "note": "" }, "joint_precision": { "value": 0.0, "status": "measured", "note": "" }, "joint_recall": { "value": 0.0, "status": "measured", "note": "" }, "joint_f1": { "value": 0.0, "status": "measured", "note": "" }, "joint_type_accuracy": { "value": 0.06666666666666667, "status": "measured", "note": "" }, "axis_angular_error_deg": { "value": 90.0, "status": "measured", "note": "" }, "origin_distance_error": { "value": 0.0, "status": "measured", "note": "" }, "joint_limit_error": { "value": 0.0, "status": "measured", "note": "" }, "kinematic_tree_validity": { "value": 1, "status": "measured", "note": "" } }, "motion_consistency": { "detachment_distance": { "value": null, "status": "requires_simulation", "note": "" }, "collision_rate": { "value": null, "status": "requires_simulation", "note": "" }, "self_collision_rate": { "value": null, "status": "requires_simulation", "note": "" }, "closed_state_assembly_error": { "value": 0.48140370054735493, "status": "measured", "note": "" }, "open_state_plausibility": { "value": 0.0, "status": "measured", "note": "" }, "trajectory_smoothness": { "value": null, "status": "requires_simulation", "note": "" } }, "ot_routing_diagnostics": { "ot_modules_active": { "value": 0, "status": "not_applicable", "note": "Raw PAct baseline; no OT module." }, "ot_gate_mean": { "value": null, "status": "not_applicable", "note": "" }, "ot_plan_marginal_error": { "value": null, "status": "not_applicable", "note": "" }, "ot_assignment_entropy": { "value": null, "status": "not_applicable", "note": "" }, "ot_edge_delta": { "value": null, "status": "not_applicable", "note": "" }, "ot_mask_prior_beta": { "value": null, "status": "not_applicable", "note": "" }, "ot_mask_dropout_delta": { "value": null, "status": "not_applicable", "note": "" }, "ot_virtual_mass": { "value": null, "status": "not_applicable", "note": "" }, "part_patch_compactness": { "value": null, "status": "not_applicable", "note": "" } }, "debug": { "pred_part_count": 7, "gt_part_count": 31, "pred_joint_count": 3, "gt_joint_count": 30, "matches": [ { "type_mismatch": 0.0, "axis_error_deg": 38.19733907558288, "origin_error": 0.0, "limit_error": 0.007768755175173283, "accepted": false }, { "type_mismatch": 1.0, "axis_error_deg": 29.993704044077234, "origin_error": 0.5397910693574457, "limit_error": 50.476250793457034, "accepted": false }, { "type_mismatch": 0.0, "axis_error_deg": 43.82082509252977, "origin_error": 0.0, "limit_error": 0.06770512056350708, "accepted": false } ], "pred_tree": { "root_count": 1, "duplicate_ids": 0, "dangling_parent_count": 0, "cycle_node_count": 0, "valid": 1 }, "ref_tree": { "root_count": 1, "duplicate_ids": 0, "dangling_parent_count": 0, "cycle_node_count": 0, "valid": 1 } }, "robustness_breakdown": { "macro_score_by_source_dataset": { "value": { "ArtVIP": 33.72558544108934, "GAPartNet": 23.258877226688433, "GRScenes": 65.82505051979625, "PartNetMobility": 23.46849844636604 }, "status": "measured", "note": "5-sample source-aware random smoke subset" }, "macro_score_by_category": { "value": { "electronics": 23.363687836527237, "major_appliances": 47.1455740748926, "small_appliances": 38.985073252189736 }, "status": "measured", "note": "computed on 5-sample subset" }, "macro_score_by_problem_tag": { "value": { "cross_source_domain_gap": 39.133774657165844, "hard_motion": 39.18617996208525, "internal_or_occluded_part_risk": 44.42540713399165, "known_quality_or_conversion_issue": 30.306556442848244, "many_parts": 25.06449110101447, "mixed_joint_types": 47.1455740748926, "non_pm_generalization": 39.133774657165844, "part_decomposition": 36.00071941500588, "prismatic_motion": 35.254630955709914, "revolute_motion": 44.42540713399165, "single_view_conditioning": 36.00071941500588 }, "status": "measured", "note": "computed on 5-sample subset" }, "worst_10pct_score": { "value": 23.258877226688433, "status": "measured", "note": "minimum sample score because n=5" }, "failure_cluster_count": { "value": null, "status": "not_provided", "note": "requires larger batch clustering" } } }, "group_scores": { "pipeline_availability": 100.0, "geometry": 51.85962994526451, "part_decomposition": 22.580645161290324, "kinematics": 0.0, "motion_consistency": 0.0, "ot_routing_diagnostics": 0.0 }, "weighted_score": 23.46849844636604, "vlm_card": "/data/250010098/PAct-Transporter/reports/20260513_pact_transporter_exploration/runs/pact_eval100_sample_eval_20260521/vlm_cards/073_PartNetMobility_electronics_electronics_3811.png", "vlm": { "ok": true, "model": "gemini-2.5-flash", "text": "- PAct生成的几何形状与输入基本一致，但细节缺失，如侧面纹理和底部轮子。\n- 部件分解严重不足，PAct仅识别出7个部件，远低于GT的31个，且未能正确区分可动部件。\n- 关节和运动信息几乎完全丢失，PAct只检测到3个关节，导致joint_f1得分为0，无法实现铰接物体的正确运动。", "status_code": 200 }, "pact_animation_mp4": "/data/250010098/PAct-Transporter/reports/20260513_pact_transporter_exploration/runs/pact_eval100_sample_eval_20260521/pact_outputs/seed42_slatcfg7.0_sscfg7.0_sssteps8_slatsteps8_artioutmean_feature_regression_steps/04__articulation_animation.mp4" }, { "sample_name": "059_ArtVIP_small_appliances_small_appliances_model_electric_fan_6", "benchmark_index": 59, "sample_id": "ArtVIP/small_appliances/model_electric_fan_6", "source_dataset": "ArtVIP", "category": "small_appliances", "object_id": "small_appliances/model_electric_fan_6", "num_parts": 3, "movable_parts": 2, "pact_object_json": "/data/250010098/PAct-Transporter/reports/20260513_pact_transporter_exploration/runs/pact_eval100_sample_eval_20260521/pact_outputs/seed42_slatcfg7.0_sscfg7.0_sssteps8_slatsteps8_artioutmean_feature_regression_steps/exported_arti_objects/059_ArtVIP_small_appliances_small_appliances_model_electric_fan_6@059_ArtVIP_small_appliances_small_appliances_model_electric_fan_6_synthetic_processed.@ng/object.json", "pact_exploded_png": "/data/250010098/PAct-Transporter/reports/20260513_pact_transporter_exploration/runs/pact_eval100_sample_eval_20260521/pact_outputs/seed42_slatcfg7.0_sscfg7.0_sssteps8_slatsteps8_artioutmean_feature_regression_steps/03__exploded_part.png", "metric_groups": { "pipeline_availability": { "inference_success": { "value": 1, "status": "measured", "note": "" }, "object_json_valid": { "value": 1, "status": "measured", "note": "" }, "glb_export_success": { "value": 0, "status": "not_provided", "note": "This smoke run exports articulated object JSON/PLY, not textured GLB." }, "part_asset_completeness": { "value": 1.0, "status": "measured", "note": "" }, "runtime_seconds": { "value": 19.840816974639893, "status": "measured", "note": "" }, "peak_gpu_memory_mb": { "value": null, "status": "not_provided", "note": "Not sampled in this run." }, "reproducible_seed_match": { "value": 1, "status": "measured", "note": "" } }, "geometry": { "chamfer_l2": { "value": null, "status": "not_provided", "note": "PLY-level Chamfer hook present; omitted in smoke run for speed." }, "fscore_tau_0_01": { "value": null, "status": "not_provided", "note": "Requires dense surface sampling." }, "normal_consistency": { "value": null, "status": "not_provided", "note": "Requires dense surface sampling." }, "scale_consistency": { "value": 0.4690546393053497, "status": "measured", "note": "" }, "mesh_manifold_sanity": { "value": null, "status": "not_provided", "note": "Requires mesh repair/manifold pass." }, "texture_or_material_coverage": { "value": 0.0, "status": "not_provided", "note": "PAct smoke export is untextured PLY." } }, "part_decomposition": { "part_count_mae": { "value": 0, "status": "measured", "note": "" }, "part_count_accuracy": { "value": 1.0, "status": "measured", "note": "" }, "part_segmentation_miou_if_masks_available": { "value": null, "status": "requires_gt", "note": "2D mask GT is conditioning input; not a predicted mask output." }, "semantic_part_accuracy": { "value": null, "status": "not_provided", "note": "PAct output names are not semantic labels in this export." }, "parent_child_accuracy": { "value": 0.0, "status": "measured", "note": "" }, "tree_edit_distance": { "value": 2, "status": "measured", "note": "" }, "orphan_duplicate_part_rate": { "value": 0.0, "status": "measured", "note": "" } }, "kinematics": { "joint_count_mae": { "value": 0, "status": "measured", "note": "" }, "joint_precision": { "value": 0.0, "status": "measured", "note": "" }, "joint_recall": { "value": 0.0, "status": "measured", "note": "" }, "joint_f1": { "value": 0.0, "status": "measured", "note": "" }, "joint_type_accuracy": { "value": 0.5, "status": "measured", "note": "" }, "axis_angular_error_deg": { "value": 90.0, "status": "measured", "note": "" }, "origin_distance_error": { "value": 0.0, "status": "measured", "note": "" }, "joint_limit_error": { "value": 0.0, "status": "measured", "note": "" }, "kinematic_tree_validity": { "value": 0, "status": "measured", "note": "" } }, "motion_consistency": { "detachment_distance": { "value": null, "status": "requires_simulation", "note": "" }, "collision_rate": { "value": null, "status": "requires_simulation", "note": "" }, "self_collision_rate": { "value": null, "status": "requires_simulation", "note": "" }, "closed_state_assembly_error": { "value": 0.5309453606946504, "status": "measured", "note": "" }, "open_state_plausibility": { "value": 0.0, "status": "measured", "note": "" }, "trajectory_smoothness": { "value": null, "status": "requires_simulation", "note": "" } }, "ot_routing_diagnostics": { "ot_modules_active": { "value": 0, "status": "not_applicable", "note": "Raw PAct baseline; no OT module." }, "ot_gate_mean": { "value": null, "status": "not_applicable", "note": "" }, "ot_plan_marginal_error": { "value": null, "status": "not_applicable", "note": "" }, "ot_assignment_entropy": { "value": null, "status": "not_applicable", "note": "" }, "ot_edge_delta": { "value": null, "status": "not_applicable", "note": "" }, "ot_mask_prior_beta": { "value": null, "status": "not_applicable", "note": "" }, "ot_mask_dropout_delta": { "value": null, "status": "not_applicable", "note": "" }, "ot_virtual_mass": { "value": null, "status": "not_applicable", "note": "" }, "part_patch_compactness": { "value": null, "status": "not_applicable", "note": "" } }, "debug": { "pred_part_count": 3, "gt_part_count": 3, "pred_joint_count": 2, "gt_joint_count": 2, "matches": [ { "type_mismatch": 0.0, "axis_error_deg": 90.0, "origin_error": 0.7740856891240943, "limit_error": 4.6514293564672915, "accepted": false }, { "type_mismatch": 1.0, "axis_error_deg": 68.41363536367565, "origin_error": 0.24119337436425747, "limit_error": 3.223309287922927, "accepted": false } ], "pred_tree": { "root_count": 3, "duplicate_ids": 0, "dangling_parent_count": 0, "cycle_node_count": 0, "valid": 0 }, "ref_tree": { "root_count": 1, "duplicate_ids": 0, "dangling_parent_count": 0, "cycle_node_count": 0, "valid": 1 } }, "robustness_breakdown": { "macro_score_by_source_dataset": { "value": { "ArtVIP": 33.72558544108934, "GAPartNet": 23.258877226688433, "GRScenes": 65.82505051979625, "PartNetMobility": 23.46849844636604 }, "status": "measured", "note": "5-sample source-aware random smoke subset" }, "macro_score_by_category": { "value": { "electronics": 23.363687836527237, "major_appliances": 47.1455740748926, "small_appliances": 38.985073252189736 }, "status": "measured", "note": "computed on 5-sample subset" }, "macro_score_by_problem_tag": { "value": { "cross_source_domain_gap": 39.133774657165844, "hard_motion": 39.18617996208525, "internal_or_occluded_part_risk": 44.42540713399165, "known_quality_or_conversion_issue": 30.306556442848244, "many_parts": 25.06449110101447, "mixed_joint_types": 47.1455740748926, "non_pm_generalization": 39.133774657165844, "part_decomposition": 36.00071941500588, "prismatic_motion": 35.254630955709914, "revolute_motion": 44.42540713399165, "single_view_conditioning": 36.00071941500588 }, "status": "measured", "note": "computed on 5-sample subset" }, "worst_10pct_score": { "value": 23.258877226688433, "status": "measured", "note": "minimum sample score because n=5" }, "failure_cluster_count": { "value": null, "status": "not_provided", "note": "requires larger batch clustering" } } }, "group_scores": { "pipeline_availability": 100.0, "geometry": 46.90546393053497, "part_decomposition": 100.0, "kinematics": 0.0, "motion_consistency": 0.0, "ot_routing_diagnostics": 0.0 }, "weighted_score": 38.985073252189736, "vlm_card": "/data/250010098/PAct-Transporter/reports/20260513_pact_transporter_exploration/runs/pact_eval100_sample_eval_20260521/vlm_cards/059_ArtVIP_small_appliances_small_appliances_model_electric_fan_6.png", "vlm": { "ok": true, "model": "gemini-2.5-flash", "text": "- 几何形状与输入基本一致，但细节如风扇叶片网格部分有缺失。\n- 部件分解数量正确，但风扇叶片与外壳的分解边界不准确，导致部分叶片被错误地划分为外壳。\n- 关节检测完全失败（joint_f1=0.00），未能识别出风扇叶片的旋转关节，导致无法模拟其运动。", "status_code": 200 }, "pact_animation_mp4": "/data/250010098/PAct-Transporter/reports/20260513_pact_transporter_exploration/runs/pact_eval100_sample_eval_20260521/pact_outputs/seed42_slatcfg7.0_sscfg7.0_sssteps8_slatsteps8_artioutmean_feature_regression_steps/03__articulation_animation.mp4" } ], "aggregate": { "mean_weighted_score": 36.00071941500588, "by_source": { "ArtVIP": 33.72558544108934, "GAPartNet": 23.258877226688433, "GRScenes": 65.82505051979625, "PartNetMobility": 23.46849844636604 }, "mean_joint_f1": 0.1, "mean_part_count_mae": 7.2 } }