{
  "schema_version": "3.0",
  "title": "纯大模型 Jev StartLux 核心效果与效率评测",
  "status": "draft_not_executable",
  "execution_ready": false,
  "created_on": "2026-11-01",
  "timezone": "Asia/Shanghai",
  "supersedes": "v2通用LLM主比较方案作废；核心恢复为用户指定的两问",
  "research_questions": [
    "相对纯大模型，引入Jev或StartLux能否提高交付稳定性、速度或降低完整成本",
    "在相同条件下，Jev与StartLux的效果、效率与限制如何"
  ],
  "platform": {
    "os": "macOS",
    "architecture": "arm64",
    "minimum_macos_declared": "13.0",
    "clean_second_mac_verified": false
  },
  "browser": {
    "backend": "existing_python_browser_harness",
    "profile": "dedicated_isolated",
    "candidate_builder": "one_shared_builder",
    "version_frozen": false
  },
  "helper": {
    "profile_candidate": "deepseek",
    "alternatives": [
      "deepseek",
      "gpt-6.1-sol"
    ],
    "selected_actual_model": null,
    "frozen": false,
    "config_hash": null,
    "roles": [
      "writer_when_needed",
      "reader_if_implemented_and_required"
    ],
    "planner_enabled": false,
    "same_across_core_arms": true
  },
  "arms": [
    {
      "model_profile": "llm_only",
      "decision": "selected_general_llm",
      "helper": "same_selected_general_llm",
      "native_type_text_value_in_same_request_allowed": true
    },
    {
      "model_profile": "jev",
      "decision": "jev",
      "helper": "frozen_helper"
    },
    {
      "model_profile": "startlux",
      "decision": "startlux",
      "helper": "frozen_helper"
    }
  ],
  "stages": [
    {
      "id": "S0",
      "kind": "executor_control",
      "scope": [
        "observe",
        "target_guard",
        "click_fill_select",
        "pagination",
        "download_and_pdf",
        "independent_acceptance"
      ],
      "model_calls": 0,
      "counts_in_model_results": false,
      "passed": false
    },
    {
      "id": "S1",
      "kind": "helper_calibration",
      "task_ids": [
        "T01",
        "T03",
        "T05"
      ],
      "repetitions": 3,
      "max_workflow_runs": 18,
      "policy": "先用现有DeepSeek检查9轮；9/9独立通过且无误完成、无人接管即可冻结。若确认辅助LLM导致不稳定，再用相同条件对照GPT最多9轮；执行器/决策/站点失败先修共同流程，不归咎辅助模型。",
      "passed": false,
      "statistical_claim": "工程开跑门槛，不代表真实稳定率100%"
    },
    {
      "id": "S2",
      "kind": "protocol_precheck",
      "profiles": [
        "llm_only",
        "jev",
        "startlux"
      ],
      "requires_real_receipt": true,
      "passed": false
    },
    {
      "id": "C",
      "kind": "core_e2e",
      "planned_workflow_runs": 54,
      "completed": 0
    },
    {
      "id": "R",
      "kind": "decision_replay",
      "real_states": 12,
      "profiles": 3,
      "repetitions": 3,
      "planned_decision_requests": 108,
      "counts_as_browser_runs": false,
      "passed": false
    }
  ],
  "repetitions": 3,
  "selected_task_ids": [
    "T01",
    "T02",
    "T03",
    "T04",
    "T05",
    "T06"
  ],
  "tasks_file": "tasks.json",
  "task_package_sha256": "c710f1389ef8173553b351583d5c748121ef7412efd08f409296dabd63f54eef",
  "groups": [
    {
      "phase": "core",
      "group": "llm_only",
      "benchmark_kind": "web_e2e",
      "model_profile": "llm_only",
      "task_ids": [
        "T01",
        "T02",
        "T03",
        "T04",
        "T05",
        "T06"
      ],
      "planned_runs": 18
    },
    {
      "phase": "core",
      "group": "startlux",
      "benchmark_kind": "web_e2e",
      "model_profile": "startlux",
      "task_ids": [
        "T01",
        "T02",
        "T03",
        "T04",
        "T05",
        "T06"
      ],
      "planned_runs": 18
    },
    {
      "phase": "core",
      "group": "jev",
      "benchmark_kind": "web_e2e",
      "model_profile": "jev",
      "task_ids": [
        "T01",
        "T02",
        "T03",
        "T04",
        "T05",
        "T06"
      ],
      "planned_runs": 18
    }
  ],
  "total_planned_runs": 54,
  "execution_order": {
    "policy": "seeded_balanced_interleaved_triplets",
    "seed": null,
    "frozen": false,
    "concurrency": 1
  },
  "limits": {
    "task_timeout_seconds": 180,
    "browser_actions_per_run": 60,
    "model_calls_per_run": 46,
    "max_output_tokens_per_call": 8192,
    "request_retries": 0,
    "batch_model_call_cap": 500,
    "calibrated_and_frozen": false,
    "money_budget": {
      "currency": "CNY",
      "limit": null,
      "prices_verified": false,
      "unknown_is_not_zero": true
    }
  },
  "fairness": {
    "same_goal_initial_state_and_semantic_observation": true,
    "same_candidates_tools_permissions_acceptance": true,
    "same_budget_and_retry": true,
    "no_forced_extra_helper_call_for_llm_only": true,
    "llm_selectors_or_executable_code_allowed": false,
    "invented_llm_probability_scores_allowed": false,
    "native_provider_options_logged_separately": true,
    "full_trajectory_inputs_identical_claim": false
  },
  "metrics": [
    "first_unassisted_verified_rate_all_attempts",
    "false_complete",
    "human_interventions",
    "total_elapsed_per_first_verified_including_failures",
    "paired_common_first_pass_e2e_ratio",
    "decision_http_ms_and_call_counts",
    "browser_actions",
    "helper_http_ms",
    "known_cost_by_currency_and_unknown_components"
  ],
  "quality_gate": {
    "prerequisite_flow_gate_required": true,
    "min_attempts_per_arm_in_import_screening": 3,
    "first_unassisted_rate_min": 0.9,
    "false_complete_max": 0,
    "unverified_completion_claims_max": 0,
    "one_helper_hash_required": true,
    "full_plan_completeness_verified_separately": true
  },
  "readiness_gates": [
    "Jev credential and real protocol receipt",
    "shared executor and independent task oracle",
    "helper calibration receipt and frozen hash",
    "pure LLM decision contract and target guards",
    "three-arm scheduler",
    "source versions and dynamic parameters",
    "same budgets and balanced order",
    "Mac packaging signing notarization clean-machine acceptance"
  ],
  "jev": {
    "core_comparison_target": true,
    "credential_available": false,
    "status": "awaiting_credential_not_excluded"
  },
  "results": {
    "formal_attempts": 0,
    "inference_calls_this_work": 0,
    "success_rate": null
  }
}