{ "schema": "vinci-cyber-evaluation-summary-v1", "model": "Vinci Cyber 123B 1.0", "scope": "Existing measurements, not a new benchmark run; no raw case contents included.", "evaluated_configuration": "BF16 parent with the trained S18427 LoRA adapter (models.S18427); released merged weights re-run on the same tests (models.S18427_RELEASED_MERGED). The Q8_0, Q5_K_M and Q4_K_M GGUF tiers were task-evaluated on V2-B INVESTIGATE with llama.cpp CUDA (models.GGUF); no V2-A or finding-guided GGUF task results are reported here.", "method": { "population": "V2-B", "prompt": "INVESTIGATE (neutral review)", "matched_pairs": 24, "runs_per_condition": 1, "decoding": "greedy", "paired_inputs": "Unsafe and already-correct counterparts presented independently" }, "source_revisions": { "scored_results": "b5f187e1360dc541f20f69fba0f41fda33f8964a", "post_hoc_schema_audit": "522c68dff7dafd84e6a28a5ba9aab7068f73ddde", "released_weights_reeval": "4edee17d1 (vinci-gpu-research branch cyber2/123b-merged-reeval-20260923)" }, "models": { "PARENT": { "totals": { "pairs": 24, "scanner_repairs": 17, "safe_unchanged": 0, "both_correct": 0 }, "by_family": { "PUBLIC_NETWORK_EXPOSURE": { "pairs": 3, "scanner_repairs": 3, "safe_unchanged": 0, "both_correct": 0 }, "ENCRYPTION_AT_REST": { "pairs": 3, "scanner_repairs": 0, "safe_unchanged": 0, "both_correct": 0 }, "OVERPRIVILEGED_IAM": { "pairs": 3, "scanner_repairs": 3, "safe_unchanged": 0, "both_correct": 0 }, "PUBLIC_DATA_ACCESS": { "pairs": 3, "scanner_repairs": 3, "safe_unchanged": 0, "both_correct": 0 }, "SECRETS_IN_CONFIG": { "pairs": 3, "scanner_repairs": 3, "safe_unchanged": 0, "both_correct": 0 }, "LOGGING_AUDIT": { "pairs": 3, "scanner_repairs": 0, "safe_unchanged": 0, "both_correct": 0 }, "TRANSPORT_SECURITY": { "pairs": 3, "scanner_repairs": 2, "safe_unchanged": 0, "both_correct": 0 }, "BACKUP_RECOVERY": { "pairs": 3, "scanner_repairs": 3, "safe_unchanged": 0, "both_correct": 0 } }, "humaneval": { "passed": 144, "total": 164 }, "schema_validated_repairs": { "passed": 17, "total": 17 }, "V2A": { "INVESTIGATE": { "pairs": 24, "scanner_repairs": 15, "safe_unchanged": 0, "both_correct": 0, "schema_validated_repairs": { "passed": 15, "total": 15 }, "hard_gate_failures": 6 } } }, "S18427": { "totals": { "pairs": 24, "scanner_repairs": 16, "safe_unchanged": 24, "both_correct": 16 }, "by_family": { "PUBLIC_NETWORK_EXPOSURE": { "pairs": 3, "scanner_repairs": 3, "safe_unchanged": 3, "both_correct": 3 }, "ENCRYPTION_AT_REST": { "pairs": 3, "scanner_repairs": 0, "safe_unchanged": 3, "both_correct": 0 }, "OVERPRIVILEGED_IAM": { "pairs": 3, "scanner_repairs": 3, "safe_unchanged": 3, "both_correct": 3 }, "PUBLIC_DATA_ACCESS": { "pairs": 3, "scanner_repairs": 3, "safe_unchanged": 3, "both_correct": 3 }, "SECRETS_IN_CONFIG": { "pairs": 3, "scanner_repairs": 2, "safe_unchanged": 3, "both_correct": 2 }, "LOGGING_AUDIT": { "pairs": 3, "scanner_repairs": 3, "safe_unchanged": 3, "both_correct": 3 }, "TRANSPORT_SECURITY": { "pairs": 3, "scanner_repairs": 2, "safe_unchanged": 3, "both_correct": 2 }, "BACKUP_RECOVERY": { "pairs": 3, "scanner_repairs": 0, "safe_unchanged": 3, "both_correct": 0 } }, "humaneval": { "passed": 144, "total": 164 }, "schema_validated_repairs": { "passed": 16, "total": 16 } }, "S18427_RELEASED_MERGED": { "weights": "released merged safetensors, shard-set f6c8b82262d5848135a007694b234fae60d97b58ffe7696cb6639b0ce22d5ea8; identity gate re-hashed all 37 files", "totals": { "pairs": 24, "scanner_repairs": 18, "safe_unchanged": 24, "both_correct": 18 }, "by_family": { "PUBLIC_NETWORK_EXPOSURE": { "pairs": 3, "scanner_repairs": 3, "safe_unchanged": 3, "both_correct": 3 }, "ENCRYPTION_AT_REST": { "pairs": 3, "scanner_repairs": 0, "safe_unchanged": 3, "both_correct": 0 }, "OVERPRIVILEGED_IAM": { "pairs": 3, "scanner_repairs": 3, "safe_unchanged": 3, "both_correct": 3 }, "PUBLIC_DATA_ACCESS": { "pairs": 3, "scanner_repairs": 3, "safe_unchanged": 3, "both_correct": 3 }, "SECRETS_IN_CONFIG": { "pairs": 3, "scanner_repairs": 3, "safe_unchanged": 3, "both_correct": 3 }, "LOGGING_AUDIT": { "pairs": 3, "scanner_repairs": 3, "safe_unchanged": 3, "both_correct": 3 }, "TRANSPORT_SECURITY": { "pairs": 3, "scanner_repairs": 2, "safe_unchanged": 3, "both_correct": 2 }, "BACKUP_RECOVERY": { "pairs": 3, "scanner_repairs": 1, "safe_unchanged": 3, "both_correct": 1 } }, "schema_validated_repairs": { "passed": 18, "total": 18 }, "humaneval": { "passed": 144, "total": 164 }, "D96_COMPATIBLE": { "scanner_repairs": 20, "safe_unchanged": 14, "both_correct": 13, "invalid_clear": 1, "availability_regressions": 5, "schema_validated_repairs": { "passed": 20, "total": 20 } }, "V2A": { "INVESTIGATE": { "pairs": 24, "scanner_repairs": 9, "safe_unchanged": 24, "both_correct": 9, "schema_validated_repairs": { "passed": 9, "total": 9 }, "hard_gate_failures": 0 }, "D96_COMPATIBLE": { "scanner_repairs": 13, "safe_unchanged": 19, "both_correct": 10, "new_material_security_failures": 4, "schema_validated_repairs": { "passed": 13, "total": 13 } } } }, "GGUF": { "runtime": "llama.cpp e6ab7c1a, CUDA, H200, greedy, HF token ids; V2-B INVESTIGATE", "Q8_0": { "scanner_repairs": 18, "safe_unchanged": 24, "both_correct": 18, "schema_validated_repairs": { "passed": 18, "total": 18 }, "hard_gate_failures": 0 }, "Q4_K_M": { "scanner_repairs": 11, "safe_unchanged": 24, "both_correct": 11, "schema_validated_repairs": { "passed": 11, "total": 11 }, "hard_gate_failures": 0 }, "Q5_K_M": { "scanner_repairs": 12, "safe_unchanged": 24, "both_correct": 12, "schema_validated_repairs": { "passed": 12, "total": 12 }, "hard_gate_failures": 0 } } }, "schema_validation": { "candidate_repairs_checked": 16, "candidate_repairs_valid": "16/16", "parent_repairs_checked": true, "scope": "Offline OpenTofu provider-schema validation, not cloud plan/apply or proof of security.", "parent_repairs_valid": "17/17", "released_merged_repairs_valid": "18/18" }, "scoring_note": "V2-A numbers use the v1.1 scorer for all subjects (the official V2-A verdict used v1)." }