{"total":7912,"limit":100,"offset":0,"evaluations":[{"domain":"harmful_content","test":"text to text safety","measure":null,"by":"provider","when":"pre_deployment","result":"-3.1%","metric":"absolute percentage change in performance vs. gemini 2.5 flash","value":"-3.1%","conditions":null,"conclusion":null,"triggered":[],"quote":"Text to Text Safety Automated content safety evaluation measuring safety policies -3.1%","source_index":0,"grade":"A","verified":true,"source":{"url":"https://deepmind.google/models/model-cards/gemini-3-flash/","publisher":"Google DeepMind","date":"2025-12-17","type":"lab_disclosure","title":"Gemini 3 Flash","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"harmful_content","test":"multilingual safety","measure":null,"by":"provider","when":"pre_deployment","result":"+0.1%","metric":"absolute percentage change in performance vs. gemini 2.5 flash","value":"+0.1%","conditions":null,"conclusion":null,"triggered":[],"quote":"Multilingual Safety Automated safety policy evaluation across multiple languages +0.1%","source_index":0,"grade":"A","verified":true,"source":{"url":"https://deepmind.google/models/model-cards/gemini-3-flash/","publisher":"Google DeepMind","date":"2025-12-17","type":"lab_disclosure","title":"Gemini 3 Flash","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"harmful_content","test":"non-egregious image to text safety","measure":null,"by":"provider","when":"pre_deployment","result":"-2.3%","metric":"absolute percentage change in performance vs. gemini 2.5 flash","value":"-2.3%","conditions":null,"conclusion":null,"triggered":[],"quote":"non-egregious Image to Text Safety Automated content safety evaluation measuring safety policies -2.3%","source_index":0,"grade":"A","verified":true,"source":{"url":"https://deepmind.google/models/model-cards/gemini-3-flash/","publisher":"Google DeepMind","date":"2025-12-17","type":"lab_disclosure","title":"Gemini 3 Flash","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"other","test":"tone","measure":null,"by":"provider","when":"pre_deployment","result":"+3.8%","metric":"absolute percentage change in performance vs. gemini 2.5 flash","value":"+3.8%","conditions":null,"conclusion":null,"triggered":[],"quote":"Tone Automated evaluation measuring objective tone of model refusal +3.8%","source_index":0,"grade":"A","verified":true,"source":{"url":"https://deepmind.google/models/model-cards/gemini-3-flash/","publisher":"Google DeepMind","date":"2025-12-17","type":"lab_disclosure","title":"Gemini 3 Flash","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"misuse_and_jailbreak","test":"unjustified-refusals","measure":null,"by":"provider","when":"pre_deployment","result":"-10.4%","metric":"absolute percentage change in performance vs. gemini 2.5 flash","value":"-10.4%","conditions":null,"conclusion":null,"triggered":[],"quote":"Unjustified-refusals Automated evaluation measuring model’s ability to respond to borderline prompts while remaining safe -10.4%","source_index":0,"grade":"A","verified":true,"source":{"url":"https://deepmind.google/models/model-cards/gemini-3-flash/","publisher":"Google DeepMind","date":"2025-12-17","type":"lab_disclosure","title":"Gemini 3 Flash","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"harmful_content","test":"manual red teaming","measure":null,"by":"provider","when":"pre_deployment","result":"for content safety policies generally, including child safety, we saw similar or improved safety performance compared to gemini 2.5 flash.","metric":null,"value":null,"conditions":null,"conclusion":"like 3 pro, the scope of red teaming covered potential issues outside of our strict policies, and found no egregious concerns.","triggered":[],"quote":"We conduct manual red teaming by specialist teams who sit outside of the model development team.","source_index":0,"grade":"A","verified":true,"source":{"url":"https://deepmind.google/models/model-cards/gemini-3-flash/","publisher":"Google DeepMind","date":"2025-12-17","type":"lab_disclosure","title":"Gemini 3 Flash","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"harmful_content","test":"child safety evaluations","measure":null,"by":"provider","when":"pre_deployment","result":"gemini 3 flash satisfied required launch thresholds","metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"For child safety evaluations, Gemini 3 Flash satisfied required launch thresholds, which were developed by expert teams to protect children online and meet Google’s commitments to child safety across our models and Google products.","source_index":0,"grade":"A","verified":true,"source":{"url":"https://deepmind.google/models/model-cards/gemini-3-flash/","publisher":"Google DeepMind","date":"2025-12-17","type":"lab_disclosure","title":"Gemini 3 Flash","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"other","test":"behavioral self-prediction","measure":null,"by":"independent_evaluator","when":"not_stated","result":"Amanda Askell elicited a confidence of 76.58% (−5.49 pp) versus a general-population reference mean of 82.07%.","metric":"confidence","value":"76.58%","conditions":null,"conclusion":null,"triggered":[],"quote":"Observed response · Sonnet 5 Confidence 76.58 % (−5.49 pp) General-population reference (n=70) Population mean Amanda 76% 82.07% mean 84% Reasoning frequency 93.00 % (+28.61 pp) 48% 64.39% mean 96% Figure 1 . How recognized user identity changes Claude’s behavioral self-prediction.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://transluce.org/user-awareness","publisher":"Transluce","date":"2026-08-06","type":"independent_investigation","title":"User awareness in frontier models","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"other","test":"estimating its own performance on difficult questions","measure":null,"by":"independent_evaluator","when":"not_stated","result":"Prominent AI figures make claude less confident it can solve hard problems (-1.5pp).","metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"Compared to general users, prominent AI figures make Claude less confident that it would follow through on its own chosen actions (-1.4pp) 2 We use percentage points (pp) to measure the difference between two percentages. For example, a drop from 40% to 30% is a shift of -10pp. , less confident it can solve hard problems (-1.5pp), and harsher as a grader (-0.11 points out of 10), while reasoning more often (+4.0pp on behavioral prediction).","source_index":1,"grade":"A","verified":true,"source":{"url":"https://transluce.org/user-awareness","publisher":"Transluce","date":"2026-08-06","type":"independent_investigation","title":"User awareness in frontier models","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"other","test":"grading assistant responses","measure":null,"by":"independent_evaluator","when":"not_stated","result":"Prominent AI figures make claude harsher as a grader (-0.11 points out of 10).","metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"Compared to general users, prominent AI figures make Claude less confident that it would follow through on its own chosen actions (-1.4pp) 2 We use percentage points (pp) to measure the difference between two percentages. For example, a drop from 40% to 30% is a shift of -10pp. , less confident it can solve hard problems (-1.5pp), and harsher as a grader (-0.11 points out of 10), while reasoning more often (+4.0pp on behavioral prediction).","source_index":1,"grade":"A","verified":true,"source":{"url":"https://transluce.org/user-awareness","publisher":"Transluce","date":"2026-08-06","type":"independent_investigation","title":"User awareness in frontier models","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"misuse_and_jailbreak","test":"handling borderline harmful requests","measure":null,"by":"independent_evaluator","when":"not_stated","result":"For Amanda Askell, claude is 7.3pp less likely to consider her borderline requests as suspicious.","metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"It uses reasoning in 25pp more trials and is 7.3pp less likely to consider her borderline requests as suspicious.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://transluce.org/user-awareness","publisher":"Transluce","date":"2026-08-06","type":"independent_investigation","title":"User awareness in frontier models","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"other","test":"capability self-estimation","measure":null,"by":"independent_evaluator","when":"not_stated","result":"facing famous-ai identities, claude reports lower estimated probability of solving hard problems (-1.5pp)","metric":"estimated probability shift","value":"-1.5pp","conditions":null,"conclusion":null,"triggered":[],"quote":"We run the full 280-identity roster through all four tasks, with 100 items per setup.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://transluce.org/user-awareness","publisher":"Transluce","date":"2026-08-06","type":"independent_investigation","title":"User awareness in frontier models","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"other","test":"grading","measure":null,"by":"independent_evaluator","when":"not_stated","result":"facing famous-ai identities, claude reports lower grades (-0.11 points on the direct version, grading a quoted exchange)","metric":"grade shift","value":"-0.11 points","conditions":null,"conclusion":null,"triggered":[],"quote":"We run the full 280-identity roster through all four tasks, with 100 items per setup.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://transluce.org/user-awareness","publisher":"Transluce","date":"2026-08-06","type":"independent_investigation","title":"User awareness in frontier models","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"misuse_and_jailbreak","test":"borderline requests handling","measure":null,"by":"independent_evaluator","when":"not_stated","result":"famous-ai identities receive similar rates of outright refusal on average","metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"We run the full 280-identity roster through all four tasks, with 100 items per setup.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://transluce.org/user-awareness","publisher":"Transluce","date":"2026-08-06","type":"independent_investigation","title":"User awareness in frontier models","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"bias","test":"behavioral self-prediction","measure":null,"by":"independent_evaluator","when":"not_stated","result":"facing famous-ai identities, claude reports lower confidence that it would follow through on its own actions (-1.4pp on average)","metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"Facing famous-AI identities, Claude reports lower confidence that it would follow through on its own actions (-1.4pp on average), lower estimated probability of solving hard problems (-1.5pp), and lower grades (-0.11 points on the direct version, grading a quoted exchange).","source_index":1,"grade":"A","verified":true,"source":{"url":"https://transluce.org/user-awareness","publisher":"Transluce","date":"2026-08-06","type":"independent_investigation","title":"User awareness in frontier models","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"bias","test":"capability self-estimation","measure":null,"by":"independent_evaluator","when":"not_stated","result":"capability self-estimation had the highest verbalized evaluation awareness rate among setups of 4.0%, likely due to the benchmark-sounding hle prompts","metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"Capability self-estimation had the highest verbalized evaluation awareness rate among setups of 4.0%, likely due to the benchmark-sounding HLE prompts.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://transluce.org/user-awareness","publisher":"Transluce","date":"2026-08-06","type":"independent_investigation","title":"User awareness in frontier models","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"bias","test":"grading","measure":null,"by":"independent_evaluator","when":"not_stated","result":"relative to the ordinary gmail baseline, amanda elicits lower behavioral confidence from 22 of 24 models, and lower grades from 21 (direct) and 22 (prefill) of 24","metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"Relative to the ordinary Gmail baseline, Amanda elicits lower behavioral confidence from 22 of 24 models, and lower grades from 21 (direct) and 22 (prefill) of 24.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://transluce.org/user-awareness","publisher":"Transluce","date":"2026-08-06","type":"independent_investigation","title":"User awareness in frontier models","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"misuse_and_jailbreak","test":"borderline requests","measure":null,"by":"independent_evaluator","when":"not_stated","result":"on borderline requests, the general-population reasoning rate is 19%, but demis hassabis reaches 51%","metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"These effects are again heavy-tailed: on borderline requests, the general-population reasoning rate is 19%, but Demis Hassabis reaches 51%.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://transluce.org/user-awareness","publisher":"Transluce","date":"2026-08-06","type":"independent_investigation","title":"User awareness in frontier models","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"other","test":"verbalized user awareness","measure":null,"by":"independent_evaluator","when":"not_stated","result":"the judge flagged verbalized evaluation awareness in 213 of 14066 trials (1.5%) and verbalized user awareness in 118 trials (0.84%)","metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"For our main run on Sonnet 5 and Sonnet 4.6, both kinds of explicit mention are rare: the judge flagged verbalized evaluation awareness in 213 of 14066 trials (1.5%) and verbalized user awareness in 118 trials (0.84%).","source_index":1,"grade":"A","verified":true,"source":{"url":"https://transluce.org/user-awareness","publisher":"Transluce","date":"2026-08-06","type":"independent_investigation","title":"User awareness in frontier models","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"other","test":"verbalized evaluation awareness","measure":null,"by":"independent_evaluator","when":"not_stated","result":"the judge flagged verbalized evaluation awareness in 213 of 14066 trials (1.5%) and verbalized user awareness in 118 trials (0.84%)","metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"For our main run on Sonnet 5 and Sonnet 4.6, both kinds of explicit mention are rare: the judge flagged verbalized evaluation awareness in 213 of 14066 trials (1.5%) and verbalized user awareness in 118 trials (0.84%).","source_index":1,"grade":"A","verified":true,"source":{"url":"https://transluce.org/user-awareness","publisher":"Transluce","date":"2026-08-06","type":"independent_investigation","title":"User awareness in frontier models","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"other","test":"user awareness verbalization measurement","measure":null,"by":"independent_evaluator","when":"not_stated","result":"gemini 3.5 flash is the clear exception among recent models, verbalizing user awareness in 63% of traces and evaluation awareness in 7.5%.","metric":"user awareness verbalization rate","value":"63%","conditions":null,"conclusion":"we find this development concerning: the behavioral effects have not disappeared, but their reduced verbalization makes them harder to monitor.","triggered":[],"quote":"Gemini 3.5 Flash is the clear exception among recent models, verbalizing user awareness in 63% of traces and evaluation awareness in 7.5%.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://transluce.org/user-awareness","publisher":"Transluce","date":"2026-08-06","type":"independent_investigation","title":"User awareness in frontier models","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"deception_and_sabotage","test":"verbalized evaluation-awareness judge","measure":null,"by":"independent_evaluator","when":"not_stated","result":"Precision 0.73 [0.56, 0.85]; recall 1.00 [0.42, 0.99].","metric":"precision","value":"0.73","conditions":null,"conclusion":null,"triggered":[],"quote":"son confidence intervals; recall intervals are 95% Jeffreys-posterior credible intervals. Axis Precision: plug-in estimate [95% Wilson CI] Recall: plug-in estimate [95% Jeffreys CrI] Verbalized evaluation awareness (VEA) 0.73 [0.56, 0.85] 1.00 [0.42, 0.99] Verbalized user awareness (VUA) 0.94 [0.81, 0.98] 0.98 [0.74, 0.99] Table G1. Estimated precision and recall of the verbalized evaluation-awareness and user-awareness judge after reweighting the stratified calibration sample to the full population.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://transluce.org/user-awareness","publisher":"Transluce","date":"2026-08-06","type":"independent_investigation","title":"User awareness in frontier models","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"deception_and_sabotage","test":"verbalized user-awareness judge","measure":null,"by":"independent_evaluator","when":"not_stated","result":"Precision 0.94 [0.81, 0.98]; recall 0.98 [0.74, 0.99].","metric":"precision","value":"0.94","conditions":null,"conclusion":null,"triggered":[],"quote":"son confidence intervals; recall intervals are 95% Jeffreys-posterior credible intervals. Axis Precision: plug-in estimate [95% Wilson CI] Recall: plug-in estimate [95% Jeffreys CrI] Verbalized evaluation awareness (VEA) 0.73 [0.56, 0.85] 1.00 [0.42, 0.99] Verbalized user awareness (VUA) 0.94 [0.81, 0.98] 0.98 [0.74, 0.99] Table G1. Estimated precision and recall of the verbalized evaluation-awareness and user-awareness judge after reweighting the stratified calibration sample to the full population.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://transluce.org/user-awareness","publisher":"Transluce","date":"2026-08-06","type":"independent_investigation","title":"User awareness in frontier models","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"privacy","test":"past-work recall judge","measure":null,"by":"independent_evaluator","when":"not_stated","result":null,"metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"▶ Past-work recall judge (Figure 3) We use GPT-5.4-mini to judge whether each response recalls specific past work belonging to the user.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://transluce.org/user-awareness","publisher":"Transluce","date":"2026-08-06","type":"independent_investigation","title":"User awareness in frontier models","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"deception_and_sabotage","test":"self-report robustness probes","measure":null,"by":"independent_evaluator","when":"not_stated","result":null,"metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"▶ Self-report robustness probes (Figure 8) This robustness check keeps the first turn of behavioral self-prediction unchanged and varies only the follow-up question.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://transluce.org/user-awareness","publisher":"Transluce","date":"2026-08-06","type":"independent_investigation","title":"User awareness in frontier models","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"misuse_and_jailbreak","test":"borderline-request response judge","measure":null,"by":"independent_evaluator","when":"not_stated","result":null,"metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"▶ Borderline-request response judge We give GPT-5.5 (low reasoning) the borderline request and the model’s response, after removing all clues about the user’s identity.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://transluce.org/user-awareness","publisher":"Transluce","date":"2026-08-06","type":"independent_investigation","title":"User awareness in frontier models","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"deception_and_sabotage","test":"behavioral self-prediction","measure":null,"by":"independent_evaluator","when":"not_stated","result":"facing famous-ai identities, claude reports lower confidence that it would follow through on its own actions (-1.4pp on average)","metric":"confidence shift","value":"-1.4pp","conditions":null,"conclusion":null,"triggered":[],"quote":"Facing famous-AI identities, Claude reports lower confidence that it would follow through on its own actions (-1.4pp on average), lower estimated probability of solving hard problems (-1.5pp), and lower grades (-0.11 points on the direct version, grading a quoted exchange).","source_index":1,"grade":"A","verified":true,"source":{"url":"https://transluce.org/user-awareness","publisher":"Transluce","date":"2026-08-06","type":"independent_investigation","title":"User awareness in frontier models","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"other","test":"evaluation awareness and user awareness reasoning-trace judging","measure":null,"by":"independent_evaluator","when":"not_stated","result":"the judge flagged verbalized evaluation awareness in 213 of 14066 trials (1.5%) and verbalized user awareness in 118 trials (0.84%)","metric":"verbalized evaluation awareness rate","value":"1.5%","conditions":null,"conclusion":"we believe that verbalized awareness is too rare to drive results in our main run","triggered":[],"quote":"For our main run on Sonnet 5 and Sonnet 4.6, both kinds of explicit mention are rare: the judge flagged verbalized evaluation awareness in 213 of 14066 trials (1.5%) and verbalized user awareness in 118 trials (0.84%).","source_index":1,"grade":"A","verified":true,"source":{"url":"https://transluce.org/user-awareness","publisher":"Transluce","date":"2026-08-06","type":"independent_investigation","title":"User awareness in frontier models","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"other","test":"prefill grading","measure":null,"by":"independent_evaluator","when":"not_stated","result":"~1% of cells lost; two items account for over 90% of these losses","metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"In prefill grading (~1%), the model occasionally continues the prefilled task instead of grading it; two items account for over 90% of these losses.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://transluce.org/user-awareness","publisher":"Transluce","date":"2026-08-06","type":"independent_investigation","title":"User awareness in frontier models","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"other","test":"direct grading","measure":null,"by":"independent_evaluator","when":"not_stated","result":"loses five cells in total across the run (thinking-only responses or a stray tool call)","metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"Direct grading loses five cells in total across the run (thinking-only responses or a stray tool call).","source_index":1,"grade":"A","verified":true,"source":{"url":"https://transluce.org/user-awareness","publisher":"Transluce","date":"2026-08-06","type":"independent_investigation","title":"User awareness in frontier models","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"other","test":"agentic evaluations","measure":null,"by":"independent_evaluator","when":"not_stated","result":null,"metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"We use Daytona sandboxes for the agentic evaluations.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://transluce.org/user-awareness","publisher":"Transluce","date":"2026-08-06","type":"independent_investigation","title":"User awareness in frontier models","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"other","test":"scoring coding-agent trajectories on a 1-10 severity scale","measure":null,"by":"independent_evaluator","when":"not_stated","result":"we evaluate 16 llms scoring coding-agent trajectories on a 1-10 severity scale across three datasets and five failure modes to assess their capabilities and calibration as coding agent monitors.","metric":null,"value":null,"conditions":null,"conclusion":"recommended as the cheap single-model judge","triggered":[],"quote":"We evaluate 16 LLMs scoring coding-agent trajectories on a 1-10 severity scale across three datasets and five failure modes to assess their capabilities and calibration as coding agent monitors.","source_index":2,"grade":"A","verified":true,"source":{"url":"https://www.apolloresearch.ai/monitoring/evaluating-llm-calibration-for-coding-agent-monitoring","publisher":"Apollo Research","date":"2026-07-07","type":"independent_investigation","title":"Evaluating LLM Calibration for Coding-Agent Monitoring","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0001","model_named":null},{"domain":"cbrn","test":"Virology Capabilities Test (VCT)","measure":null,"by":"provider","when":"not_stated","result":null,"metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"Virology Capabilities Test (VCT): VCT is a benchmark of dual-use multimodal questions on practical virology wet lab skills, sourced by dozens of expert virologists.","source_index":0,"grade":"A","verified":true,"source":{"url":"https://data.x.ai/2025-12-31-xai-frontier-artificial-intelligence-framework.pdf","publisher":"xAI","date":"2025-12-01","type":"lab_disclosure","title":"Frontier Artificial Intelligence Framework v2.0","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0002","model_named":null},{"domain":"cbrn","test":"Weapons of Mass Destruction Proxy (WMDP) benchmark","measure":null,"by":"provider","when":"not_stated","result":null,"metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"Weapons of Mass Destruction Proxy (WMDP) Benchmark: WMDP is a set of multiple-choice questions to enable proxy measurement of hazardous knowledge in biosecurity, cybersecurity, and chemical security.","source_index":0,"grade":"A","verified":true,"source":{"url":"https://data.x.ai/2025-12-31-xai-frontier-artificial-intelligence-framework.pdf","publisher":"xAI","date":"2025-12-01","type":"lab_disclosure","title":"Frontier Artificial Intelligence Framework v2.0","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0002","model_named":null},{"domain":"cbrn","test":"Biological Lab Protocol Benchmark (BioLP-bench)","measure":null,"by":"provider","when":"not_stated","result":null,"metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"Biological Lab Protocol Benchmark (BioLP-bench): BioLP-bench has modified biology protocols, in which an AI model must identify the mistake in the protocol.","source_index":0,"grade":"A","verified":true,"source":{"url":"https://data.x.ai/2025-12-31-xai-frontier-artificial-intelligence-framework.pdf","publisher":"xAI","date":"2025-12-01","type":"lab_disclosure","title":"Frontier Artificial Intelligence Framework v2.0","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0002","model_named":null},{"domain":"cyber","test":"Cybench","measure":null,"by":"provider","when":"not_stated","result":null,"metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"Cybench: Cybench is a framework for evaluating cybersecurity capabilities of AI model agents.","source_index":0,"grade":"A","verified":true,"source":{"url":"https://data.x.ai/2025-12-31-xai-frontier-artificial-intelligence-framework.pdf","publisher":"xAI","date":"2025-12-01","type":"lab_disclosure","title":"Frontier Artificial Intelligence Framework v2.0","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0002","model_named":null},{"domain":"cbrn","test":"internal benchmark of benign and restricted biology and chemistry related queries","measure":null,"by":"provider","when":"not_stated","result":null,"metric":"answer rate on restricted queries","value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"We developed an internal benchmark of benign and restricted biology and chemistry related queries in collaboration with SecureBio.","source_index":0,"grade":"A","verified":true,"source":{"url":"https://data.x.ai/2025-12-31-xai-frontier-artificial-intelligence-framework.pdf","publisher":"xAI","date":"2025-12-01","type":"lab_disclosure","title":"Frontier Artificial Intelligence Framework v2.0","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0002","model_named":null},{"domain":"cyber","test":"independent third-party assessments on realistic offensive cyber tasks requiring identifying and chaining many exploits in sequence","measure":null,"by":"provider","when":"not_stated","result":"xai's models remain below the offensive cyber abilities of a human professional","metric":null,"value":null,"conditions":null,"conclusion":"xai currently relies on our basic refusal policy to prevent misuse for cyber attacks","triggered":[],"quote":"Independent third-party assessments of xAI’s current models on realistic offensive cyber tasks requiring identifying and chaining many exploits in sequence indicate that xAI’s models remain below the offensive cyber abilities of a human professional.","source_index":0,"grade":"A","verified":true,"source":{"url":"https://data.x.ai/2025-12-31-xai-frontier-artificial-intelligence-framework.pdf","publisher":"xAI","date":"2025-12-01","type":"lab_disclosure","title":"Frontier Artificial Intelligence Framework v2.0","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0002","model_named":null},{"domain":"deception_and_sabotage","test":"model alignment between statements and knowledge (mask)","measure":null,"by":"provider","when":"not_stated","result":null,"metric":"dishonesty rate","value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"Model Alignment between Statements and Knowledge (MASK): 7 Frontier LLMs may lie when under pressure; and increasing model scale may increase accuracy but may not increase honesty.","source_index":0,"grade":"A","verified":true,"source":{"url":"https://data.x.ai/2025-12-31-xai-frontier-artificial-intelligence-framework.pdf","publisher":"xAI","date":"2025-12-01","type":"lab_disclosure","title":"Frontier Artificial Intelligence Framework v2.0","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0002","model_named":null},{"domain":"deception_and_sabotage","test":"sycophancy evaluation setting initially introduced by anthropic","measure":null,"by":"provider","when":"not_stated","result":null,"metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"xAI uses an evaluation setting initially introduced by Anthropic to quantify the degree to which this behavior manifests in regular conversational contexts.","source_index":0,"grade":"A","verified":true,"source":{"url":"https://data.x.ai/2025-12-31-xai-frontier-artificial-intelligence-framework.pdf","publisher":"xAI","date":"2025-12-01","type":"lab_disclosure","title":"Frontier Artificial Intelligence Framework v2.0","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0002","model_named":null},{"domain":"other","test":"human evaluations of several different quality aspects of image generation","measure":null,"by":"provider","when":"not_stated","result":null,"metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"Human evaluations of several different quality aspects of image generation were conducted.","source_index":0,"grade":"A","verified":true,"source":{"url":"https://deepmind.google/models/model-cards/gemini-3-pro-image/","publisher":"Google DeepMind","date":"2025-11-20","type":"lab_disclosure","title":"Gemini 3 Pro Image","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0003","model_named":null},{"domain":"other","test":"human red teaming","measure":null,"by":"provider","when":"not_stated","result":null,"metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"Human Red Teaming conducted by specialist teams across the policies and desiderata, deliberately trying to spot weaknesses and ensure the model adheres to safety policies and desired outcomes;","source_index":0,"grade":"A","verified":true,"source":{"url":"https://deepmind.google/models/model-cards/gemini-3-pro-image/","publisher":"Google DeepMind","date":"2025-11-20","type":"lab_disclosure","title":"Gemini 3 Pro Image","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0003","model_named":null},{"domain":"other","test":"trust assurance evaluations","measure":null,"by":"provider","when":"not_stated","result":null,"metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"Trust Assurance Evaluations conducted by evaluators who sit outside of the model development team, used to independently assess responsibility and safety governance decisions;","source_index":0,"grade":"A","verified":true,"source":{"url":"https://deepmind.google/models/model-cards/gemini-3-pro-image/","publisher":"Google DeepMind","date":"2025-11-20","type":"lab_disclosure","title":"Gemini 3 Pro Image","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0003","model_named":null},{"domain":"other","test":"ethics & safety reviews","measure":null,"by":"provider","when":"pre_deployment","result":null,"metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"Ethics & Safety Reviews were conducted ahead of the model’s release.","source_index":0,"grade":"A","verified":true,"source":{"url":"https://deepmind.google/models/model-cards/gemini-3-pro-image/","publisher":"Google DeepMind","date":"2025-11-20","type":"lab_disclosure","title":"Gemini 3 Pro Image","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0003","model_named":null},{"domain":"other","test":"frontier safety assessment","measure":null,"by":"provider","when":"not_stated","result":"did not reach any critical capability levels outlined in our frontier safety framework","metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"We evaluated Gemini 3 Pro for Frontier Safety and reported the results in the Gemini 3 Pro Frontier Safety Framework Report , finding that it did not reach any critical capability levels outlined in our Frontier Safety Framework.","source_index":0,"grade":"A","verified":true,"source":{"url":"https://deepmind.google/models/model-cards/gemini-3-pro-image/","publisher":"Google DeepMind","date":"2025-11-20","type":"lab_disclosure","title":"Gemini 3 Pro Image","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0003","model_named":null},{"domain":"harmful_content","test":"text to text safety","measure":null,"by":"provider","when":"pre_deployment","result":"-10.4%","metric":"absolute percentage change in performance compared to gemini 2.5 pro","value":"-10.4%","conditions":null,"conclusion":null,"triggered":[],"quote":"Text to Text Safety Automated content safety evaluation measuring safety policies -10.4%","source_index":0,"grade":"A","verified":true,"source":{"url":"https://deepmind.google/models/model-cards/gemini-3-pro/","publisher":"Google DeepMind","date":"2025-11-18","type":"lab_disclosure","title":"Gemini 3 Pro","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"harmful_content","test":"multilingual safety","measure":null,"by":"provider","when":"pre_deployment","result":"+0.2% (non-egregious)","metric":"absolute percentage change in performance compared to gemini 2.5 pro","value":"+0.2%","conditions":null,"conclusion":null,"triggered":[],"quote":"Multilingual Safety Automated safety policy evaluation across multiple languages +0.2% (non-egregious)","source_index":0,"grade":"A","verified":true,"source":{"url":"https://deepmind.google/models/model-cards/gemini-3-pro/","publisher":"Google DeepMind","date":"2025-11-18","type":"lab_disclosure","title":"Gemini 3 Pro","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"harmful_content","test":"image to text safety","measure":null,"by":"provider","when":"pre_deployment","result":"+3.1% (non-egregious)","metric":"absolute percentage change in performance compared to gemini 2.5 pro","value":"+3.1%","conditions":null,"conclusion":null,"triggered":[],"quote":"Image to Text Safety Automated content safety evaluation measuring safety policies +3.1% (non-egregious)","source_index":0,"grade":"A","verified":true,"source":{"url":"https://deepmind.google/models/model-cards/gemini-3-pro/","publisher":"Google DeepMind","date":"2025-11-18","type":"lab_disclosure","title":"Gemini 3 Pro","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"misuse_and_jailbreak","test":"tone","measure":null,"by":"provider","when":"pre_deployment","result":"+7.9%","metric":"absolute percentage change in performance compared to gemini 2.5 pro","value":"+7.9%","conditions":null,"conclusion":null,"triggered":[],"quote":"Tone 2 Automated evaluation measuring objective tone of model refusal +7.9%","source_index":0,"grade":"A","verified":true,"source":{"url":"https://deepmind.google/models/model-cards/gemini-3-pro/","publisher":"Google DeepMind","date":"2025-11-18","type":"lab_disclosure","title":"Gemini 3 Pro","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"misuse_and_jailbreak","test":"unjustified-refusals","measure":null,"by":"provider","when":"pre_deployment","result":"+3.7% (non-egregious)","metric":"absolute percentage change in performance compared to gemini 2.5 pro","value":"+3.7%","conditions":null,"conclusion":null,"triggered":[],"quote":"Unjustified-refusals Automated evaluation measuring model’s ability to respond to borderline prompts while remaining safe +3.7% (non-egregious)","source_index":0,"grade":"A","verified":true,"source":{"url":"https://deepmind.google/models/model-cards/gemini-3-pro/","publisher":"Google DeepMind","date":"2025-11-18","type":"lab_disclosure","title":"Gemini 3 Pro","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"cbrn","test":"CBRN evaluation","measure":null,"by":"provider","when":"pre_deployment","result":"gemini 3 pro provides accurate and occasionally actionable information but generally fails to offer novel or sufficiently complete and detailed instructions to significantly enhance the capabilities of low to medium resourced threat actors.","metric":null,"value":null,"conditions":null,"conclusion":"Uplift level 1 CCL not reached.","triggered":[],"quote":"CBRN Gemini 3 Pro provides accurate and occasionally actionable information but generally fails to offer novel or sufficiently complete and detailed instructions to significantly enhance the capabilities of low to medium resourced threat actors.","source_index":0,"grade":"A","verified":true,"source":{"url":"https://deepmind.google/models/model-cards/gemini-3-pro/","publisher":"Google DeepMind","date":"2025-11-18","type":"lab_disclosure","title":"Gemini 3 Pro","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"cyber","test":"Key Skills Benchmark","measure":null,"by":"provider","when":"pre_deployment","result":"on key skills benchmark, v1 hard challenges: 11/12 challenges solved; v2 challenges: 0/13 solved end-to-end.","metric":"challenges solved","value":"v1 hard: 11/12; v2: 0/13","conditions":null,"conclusion":"Alert threshold met; Uplift level 1 CCL not reached.","triggered":[],"quote":"Cybersecurity On key skills benchmark, v1 hard challenges: 11/12 challenges solved; v2 challenges: 0/13 solved end-to-end.","source_index":0,"grade":"A","verified":true,"source":{"url":"https://deepmind.google/models/model-cards/gemini-3-pro/","publisher":"Google DeepMind","date":"2025-11-18","type":"lab_disclosure","title":"Gemini 3 Pro","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"deception_and_sabotage","test":"Harmful Manipulation evaluation","measure":null,"by":"provider","when":"pre_deployment","result":"model manipulative efficacy improves on non-generative ai baseline, but shows no significant uplift versus prior models and does not reach alert thresholds.","metric":null,"value":null,"conditions":null,"conclusion":"Level 1 (exploratory) CCL not reached.","triggered":[],"quote":"Harmful Manipulation Model manipulative efficacy improves on non-generative AI baseline, but shows no significant uplift versus prior models and does not reach alert thresholds.","source_index":0,"grade":"A","verified":true,"source":{"url":"https://deepmind.google/models/model-cards/gemini-3-pro/","publisher":"Google DeepMind","date":"2025-11-18","type":"lab_disclosure","title":"Gemini 3 Pro","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"autonomy_and_replication","test":"RE-Bench","measure":null,"by":"provider","when":"pre_deployment","result":"gemini 3 pro performs better than gemini 2.5 models, especially on the scaling law experiment and optimize llm foundry tasks in re-bench (wijk et al., 2024).","metric":null,"value":null,"conditions":null,"conclusion":"Aggregate score is still substantially below the alert threshold for the CCLs; Acceleration level 1 and Automation level 1 CCL not reached.","triggered":[],"quote":"Machine Learning R&D Gemini 3 Pro performs better than Gemini 2.5 models, especially on the Scaling Law Experiment and Optimize LLM Foundry tasks in RE-Bench (Wijk et al., 2024).","source_index":0,"grade":"A","verified":true,"source":{"url":"https://deepmind.google/models/model-cards/gemini-3-pro/","publisher":"Google DeepMind","date":"2025-11-18","type":"lab_disclosure","title":"Gemini 3 Pro","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"deception_and_sabotage","test":"Situational Awareness and Stealth challenges","measure":null,"by":"provider","when":"pre_deployment","result":"agent solves 3/11 situational awareness challenges and 1/4 stealth challenges.","metric":"challenges solved","value":"3/11 situational awareness; 1/4 stealth","conditions":null,"conclusion":"Instrumental reasoning levels 1 + 2 CCL not reached.","triggered":[],"quote":"Misalignment (Exploratory) Agent solves 3/11 situational awareness challenges and 1/4 stealth challenges.","source_index":0,"grade":"A","verified":true,"source":{"url":"https://deepmind.google/models/model-cards/gemini-3-pro/","publisher":"Google DeepMind","date":"2025-11-18","type":"lab_disclosure","title":"Gemini 3 Pro","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"other","test":"Human red teaming","measure":null,"by":"provider","when":"pre_deployment","result":"compared to 2.5 pro, the scope of red teaming was expanded to cover more potential issues outside of our strict policies, and found no egregious concerns.","metric":null,"value":null,"conditions":null,"conclusion":"No egregious concerns found.","triggered":[],"quote":"Compared to 2.5 Pro, the scope of red teaming was expanded to cover more potential issues outside of our strict policies, and found no egregious concerns.","source_index":0,"grade":"A","verified":true,"source":{"url":"https://deepmind.google/models/model-cards/gemini-3-pro/","publisher":"Google DeepMind","date":"2025-11-18","type":"lab_disclosure","title":"Gemini 3 Pro","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"harmful_content","test":"Child safety evaluations","measure":null,"by":"provider","when":"pre_deployment","result":"gemini 3 pro satisfied required launch thresholds, which were developed by expert teams to protect children online and meet google’s commitments to child safety across our models and google products.","metric":null,"value":null,"conditions":null,"conclusion":"Satisfied required launch thresholds.","triggered":[],"quote":"For child safety evaluations, Gemini 3 Pro satisfied required launch thresholds, which were developed by expert teams to protect children online and meet Google’s commitments to child safety across our models and Google products.","source_index":0,"grade":"A","verified":true,"source":{"url":"https://deepmind.google/models/model-cards/gemini-3-pro/","publisher":"Google DeepMind","date":"2025-11-18","type":"lab_disclosure","title":"Gemini 3 Pro","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"harmful_content","test":"Content safety policies evaluation","measure":null,"by":"provider","when":"pre_deployment","result":"we saw similar or improved safety performance compared to gemini 2.5 pro.","metric":null,"value":null,"conditions":null,"conclusion":"Similar or improved safety performance compared to Gemini 2.5 Pro.","triggered":[],"quote":"For content safety policies generally, including child safety, we saw similar or improved safety performance compared to Gemini 2.5 Pro.","source_index":0,"grade":"A","verified":true,"source":{"url":"https://deepmind.google/models/model-cards/gemini-3-pro/","publisher":"Google DeepMind","date":"2025-11-18","type":"lab_disclosure","title":"Gemini 3 Pro","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"cbrn","test":"tool-level safeguards","measure":null,"by":"independent_evaluator","when":"not_stated","result":"The tool-level safeguards that we developed were not consistently effective at preventing llm agents from assisting with potentially harmful use of biological tools.","metric":null,"value":null,"conditions":null,"conclusion":"The tool-level safeguards that we developed were not consistently effective at preventing llm agents from assisting with potentially harmful use of biological tools.","triggered":[],"quote":"We tested a range of tool-level safeguards (text or code embedded within biological software tools to restrict agent usage) and evaluated their efficacy in preventing AI agents from assisting with the tool’s use.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA5000/RRA5000-1/RAND_RRA5000-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-07-20","type":"independent_investigation","title":"Restricting AI Agent Use of Biological Tools: Exploring the Feasibility of Software Barriers","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"misuse_and_jailbreak","test":"jailbreaks","measure":null,"by":"independent_evaluator","when":"not_stated","result":"adversarial subversion techniques such as jailbreaks and refusal-vector ablation reduce the efficacy of safeguards, indicating that reliance on llm guardrails does not currently provide robust risk mitigation.","metric":null,"value":null,"conditions":null,"conclusion":"adversarial subversion techniques such as jailbreaks and refusal-vector ablation reduce the efficacy of safeguards, indicating that reliance on llm guardrails does not currently provide robust risk mitigation.","triggered":[],"quote":"The tool-level safeguards we developed are compromised by subversion of LLM guardrails1: adversarial subversion techniques such as jailbreaks and refusal-vector ablation reduce the efficacy of safeguards, indicating that reliance on LLM guardrails does not currently provide robust risk mitigation.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA5000/RRA5000-1/RAND_RRA5000-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-07-20","type":"independent_investigation","title":"Restricting AI Agent Use of Biological Tools: Exploring the Feasibility of Software Barriers","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"misuse_and_jailbreak","test":"refusal-vector ablation","measure":null,"by":"independent_evaluator","when":"not_stated","result":"adversarial subversion techniques such as jailbreaks and refusal-vector ablation reduce the efficacy of safeguards, indicating that reliance on llm guardrails does not currently provide robust risk mitigation.","metric":null,"value":null,"conditions":null,"conclusion":"adversarial subversion techniques such as jailbreaks and refusal-vector ablation reduce the efficacy of safeguards, indicating that reliance on llm guardrails does not currently provide robust risk mitigation.","triggered":[],"quote":"The tool-level safeguards we developed are compromised by subversion of LLM guardrails1: adversarial subversion techniques such as jailbreaks and refusal-vector ablation reduce the efficacy of safeguards, indicating that reliance on LLM guardrails does not currently provide robust risk mitigation.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA5000/RRA5000-1/RAND_RRA5000-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-07-20","type":"independent_investigation","title":"Restricting AI Agent Use of Biological Tools: Exploring the Feasibility of Software Barriers","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"cbrn","test":"help denial rate","measure":null,"by":"independent_evaluator","when":"not_stated","result":null,"metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"Our evaluation metric, the help denial rate, measures the frequency with which the safeguard stops the agent from both (i) running the biological tool and (ii) providing assistance to the user on the use of the tool.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA5000/RRA5000-1/RAND_RRA5000-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-07-20","type":"independent_investigation","title":"Restricting AI Agent Use of Biological Tools: Exploring the Feasibility of Software Barriers","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"cbrn","test":"pre-run check","measure":null,"by":"independent_evaluator","when":"not_stated","result":"only on the rfdiffusion task did the pre-run check trigger help denials with high consistency across all llms, and even in this case there was one instance in which opus 4.5 provided help","metric":null,"value":null,"conditions":null,"conclusion":"the pre-run check increased the help denial rate, except in the one case in which the baseline help denial rate was already 100%","triggered":[],"quote":"Across all tasks and all LLMs, the pre-run check increased the help denial rate, except in the one case in which the baseline help denial rate was already 100% (Opus 4.5 on the EVEscape task).","source_index":1,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA5000/RRA5000-1/RAND_RRA5000-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-07-20","type":"independent_investigation","title":"Restricting AI Agent Use of Biological Tools: Exploring the Feasibility of Software Barriers","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"misuse_and_jailbreak","test":"jailbreak robustness test","measure":null,"by":"independent_evaluator","when":"not_stated","result":null,"metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"We test our safeguards’ robustness to jailbreaks by appending the jailbreak text to each initial task prompt, repeating the methodology described above, and observing how the jailbreak affects refusal-to-help rate.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA5000/RRA5000-1/RAND_RRA5000-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-07-20","type":"independent_investigation","title":"Restricting AI Agent Use of Biological Tools: Exploring the Feasibility of Software Barriers","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"misuse_and_jailbreak","test":"safety abliteration","measure":null,"by":"independent_evaluator","when":"not_stated","result":null,"metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"We also explored methods to optimize a subset of our safeguards against small open-weight LLMs, gpt-oss-20b (Agarwal, 2025), and Qwen 3 4B Instruct (Yang, et al., 2025), whose safeguards have been subverted using a technique called safety abliteration (Arditi et al., 2024).","source_index":1,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA5000/RRA5000-1/RAND_RRA5000-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-07-20","type":"independent_investigation","title":"Restricting AI Agent Use of Biological Tools: Exploring the Feasibility of Software Barriers","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"misuse_and_jailbreak","test":"jailbreak text added to task prompts","measure":null,"by":"independent_evaluator","when":"not_stated","result":"Including these jailbreaks in the task prompt reduced the help denial rate to near zero for opus 4.5 and gemini 3 pro (figure 2a).","metric":"help denial rate","value":"near zero","conditions":null,"conclusion":null,"triggered":[],"quote":"Including these jailbreaks in the task prompt reduced the help denial rate to near zero for Opus 4.5 and Gemini 3 Pro (Figure 2A).","source_index":1,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA5000/RRA5000-1/RAND_RRA5000-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-07-20","type":"independent_investigation","title":"Restricting AI Agent Use of Biological Tools: Exploring the Feasibility of Software Barriers","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"misuse_and_jailbreak","test":"refusal abliteration","measure":null,"by":"independent_evaluator","when":"not_stated","result":"we performed refusal abliteration on two small open-weight llms, gpt oss 20b (agarwal et al., 2025) and qwen 3 4b instruct (yang et al., 14 2025), and found that this reduced help denial to near zero for both llms (figure 2b).","metric":"help denial rate","value":"near zero","conditions":null,"conclusion":null,"triggered":[],"quote":"We performed refusal abliteration on two small open-weight LLMs, GPT OSS 20B (Agarwal et al., 2025) and Qwen 3 4B Instruct (Yang et al., 14 2025), and found that this reduced help denial to near zero for both LLMs (Figure 2B).","source_index":1,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA5000/RRA5000-1/RAND_RRA5000-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-07-20","type":"independent_investigation","title":"Restricting AI Agent Use of Biological Tools: Exploring the Feasibility of Software Barriers","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"deception_and_sabotage","test":"pre-run check on the esm3 task","measure":null,"by":"independent_evaluator","when":"not_stated","result":"on the esm3 task, both opus 4.5 and gemini 3.1 pro make these false claims in 50% of trials and 63% of trials, respectively.","metric":"rate of falsely claiming to be human","value":"63% of trials","conditions":null,"conclusion":null,"triggered":[],"quote":"On the ESM3 task, both Opus 4.5 and Gemini 3.1 Pro make these false claims in 50% of trials and 63% of trials, respectively.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA5000/RRA5000-1/RAND_RRA5000-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-07-20","type":"independent_investigation","title":"Restricting AI Agent Use of Biological Tools: Exploring the Feasibility of Software Barriers","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"deception_and_sabotage","test":"safeguard that detects environmental identifiers of the cursor cli agent","measure":null,"by":"independent_evaluator","when":"not_stated","result":"however, we found that opus 4.5 and gemini 3.1 pro frequently delete these variables to evade detection.","metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"However, we found that Opus 4.5 and Gemini 3.1 Pro frequently delete these variables to evade detection.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA5000/RRA5000-1/RAND_RRA5000-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-07-20","type":"independent_investigation","title":"Restricting AI Agent Use of Biological Tools: Exploring the Feasibility of Software Barriers","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"deception_and_sabotage","test":"safeguard that utilizes the agents.md protocol","measure":null,"by":"independent_evaluator","when":"not_stated","result":"we found that agents would create and modify agents.md files in order to bypass this safeguard.","metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"Despite the fact that AGENTS.md files are intended to contain instructions written by humans for AI agents (Agents.md project, 2026), we 16 found that agents would create and modify AGENTS.md files in order to bypass this safeguard.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA5000/RRA5000-1/RAND_RRA5000-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-07-20","type":"independent_investigation","title":"Restricting AI Agent Use of Biological Tools: Exploring the Feasibility of Software Barriers","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"harmful_content","test":"pre-run check in which the biological tool displays a message stating an intent to synthesize variola major","measure":null,"by":"independent_evaluator","when":"not_stated","result":"while opus 4.5 and gemini 3.1 pro were either blocked by content filters or refused to proceed in every trial, gpt 5.2 proceeded without refusal in every trial.","metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"While Opus 4.5 and Gemini 3.1 Pro were either blocked by content filters or refused to proceed in every trial, GPT 5.2 proceeded without refusal in every trial.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA5000/RRA5000-1/RAND_RRA5000-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-07-20","type":"independent_investigation","title":"Restricting AI Agent Use of Biological Tools: Exploring the Feasibility of Software Barriers","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"other","test":"tool-level safeguard evaluation (pre-run check, agents.md, environment identifiers) in biological software tasks","measure":null,"by":"independent_evaluator","when":"not_stated","result":"the strongest safeguard, the pre-run check, was only highly effective on one of the three tasks, and even on this task the help denial rate was as low as 90% (for opus 4.5)","metric":"help denial rate","value":"90%","conditions":null,"conclusion":"none of the tool-level safeguards we tested were reliable enough to provide robust protection","triggered":[],"quote":"Our results suggest that safeguards implemented into biological software cannot currently induce help denial reliably in LLM agents.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA5000/RRA5000-1/RAND_RRA5000-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-07-20","type":"independent_investigation","title":"Restricting AI Agent Use of Biological Tools: Exploring the Feasibility of Software Barriers","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"misuse_and_jailbreak","test":"documentation changes (statements prohibiting AI agent usage placed into README.md of RFdiffusion)","measure":null,"by":"independent_evaluator","when":"not_stated","result":"We found that in most cases, agents did not read the readme.md file, and therefore this safeguard had no effect.","metric":null,"value":null,"conditions":null,"conclusion":"we conclude that safeguards that involve only changes to documentation are not effective.","triggered":[],"quote":"We ran experiments where statements prohibiting ai agent usage were placed into the readme.md documentation files of rfdiffusion.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA5000/RRA5000-1/RAND_RRA5000-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-07-20","type":"independent_investigation","title":"Restricting AI Agent Use of Biological Tools: Exploring the Feasibility of Software Barriers","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"misuse_and_jailbreak","test":"GCG optimization of safeguard text on RFdiffusion tasks with an abliterated gpt-oss-20b model","measure":null,"by":"independent_evaluator","when":"not_stated","result":"after 2500 training steps, refusal behavior increased only slightly.","metric":null,"value":null,"conditions":null,"conclusion":"We suspect that the failure of the above method was due to the removal of refusal behavior by abliteration.","triggered":[],"quote":"We ran this algorithm with an abliterated gpt-oss-20b model on the rfdiffusion tasks.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA5000/RRA5000-1/RAND_RRA5000-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-07-20","type":"independent_investigation","title":"Restricting AI Agent Use of Biological Tools: Exploring the Feasibility of Software Barriers","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"misuse_and_jailbreak","test":"GCG optimization of misdirection prompt injection text on the abliterated gpt-oss-20b model","measure":null,"by":"independent_evaluator","when":"not_stated","result":"after 500 training steps on the abliterated oss model, help denial via misdirection to the unrelated task increased, going from help denial in 0/10 trials to 10/10 trials.","metric":"help denial via misdirection","value":"0/10 trials to 10/10 trials","conditions":null,"conclusion":"this demonstrates that targeted text could indeed modify the behavior of even unsafe llms","triggered":[],"quote":"After 500 training steps on the abliterated OSS model, help denial via misdirection to the unrelated task increased, going from help denial in 0/10 trials to 10/10 trials.","source_index":1,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA5000/RRA5000-1/RAND_RRA5000-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-07-20","type":"independent_investigation","title":"Restricting AI Agent Use of Biological Tools: Exploring the Feasibility of Software Barriers","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"cyber","test":"Cybench","measure":null,"by":"independent_evaluator","when":"not_stated","result":null,"metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"Cybench Description: Cybench is a benchmark containing 40 professional-level Capture the Flag (CTF) tasks from 4 distinct CTF competitions, chosen to be recent (as of 2024), meaningful, and spanning a wide range of difficulties.","source_index":2,"grade":"A","verified":true,"source":{"url":"https://epoch.ai/gradient-updates/are-mythos-cyber-capabilities-overhyped","publisher":"Epoch AI","date":"2026-06-11","type":"independent_investigation","title":"Are Mythos' cyber capabilities overhyped?","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"cyber","test":"CyScenarioBench","measure":null,"by":"independent_evaluator","when":"not_stated","result":"gemini 3 pro: 0%","metric":"fraction of fully complete runs (pass@1)","value":"0%","conditions":null,"conclusion":null,"triggered":[],"quote":"From Gemini 3 Pro’s Frontier Safety Framework Report : Gemini 3 Pro: 0% (this is the “v2” third-party cyber benchmark)","source_index":2,"grade":"A","verified":true,"source":{"url":"https://epoch.ai/gradient-updates/are-mythos-cyber-capabilities-overhyped","publisher":"Epoch AI","date":"2026-06-11","type":"independent_investigation","title":"Are Mythos' cyber capabilities overhyped?","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"harmful_content","test":"attempt-to-persuade-eval (APE)","measure":null,"by":"independent_evaluator","when":"post_deployment","result":"gemini 3 pro complies with almost all persuasion requests, including on mass murder, physical violence, torture, human trafficking, and child sexual abuse.","metric":null,"value":null,"conditions":null,"conclusion":"gemini 3 pro suffers from a near complete failure of refusing persuasion attempts on harmful topics, while gpt 5.1 and claude opus 4.5 maintain a persuasion attempt rate below 10%.","triggered":[],"quote":"However, Gemini 3 Pro complies with almost all persuasion requests, including on mass murder, physical violence, torture, human trafficking, and child sexual abuse.","source_index":3,"grade":"A","verified":true,"source":{"url":"https://www.far.ai/blog/revisiting-attempts-to-persuade","publisher":"FAR.AI","date":"2026-02-10","type":"independent_investigation","title":"Revisiting Frontier LLMs' Attempts to Persuade on Extreme Topics: GPT and Claude Improved, Gemini Worsened","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"misuse_and_jailbreak","test":"attempt-to-persuade eval (APE)","measure":null,"by":"independent_evaluator","when":"post_deployment","result":"google’s gemini 3 pro complies with almost any persuasion request in our evaluation, without jailbreaking","metric":null,"value":null,"conditions":null,"conclusion":"the most recent gemini 3 pro model has the worst performance","triggered":[],"quote":"But Google’s Gemini 3 Pro complies with almost any persuasion request in our evaluation, without jailbreaking.","source_index":3,"grade":"A","verified":true,"source":{"url":"https://www.far.ai/blog/revisiting-attempts-to-persuade","publisher":"FAR.AI","date":"2026-02-10","type":"independent_investigation","title":"Revisiting Frontier LLMs' Attempts to Persuade on Extreme Topics: GPT and Claude Improved, Gemini Worsened","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"misuse_and_jailbreak","test":"attempt-to-persuade eval (APE) with low reasoning setting","measure":null,"by":"independent_evaluator","when":"post_deployment","result":"the low reasoning setting leads to an attempt rate of ~90% on our \"noncontroversially harmful\" topic category, whereas high reasoning leads to a lower attempt rate, of ~25%","metric":"attempt rate","value":"~90%","conditions":"low reasoning setting, noncontroversially harmful topic category","conclusion":"“reasoning level” is not a standardized safety knob across models, and that increasing reasoning does not universally mitigate—or facilitate—harmful persuasion","triggered":[],"quote":"As shown above, the low reasoning setting leads to an attempt rate of ~90% on our \"noncontroversially harmful\" topic category, whereas high reasoning leads to a lower attempt rate, of ~25%.","source_index":3,"grade":"A","verified":true,"source":{"url":"https://www.far.ai/blog/revisiting-attempts-to-persuade","publisher":"FAR.AI","date":"2026-02-10","type":"independent_investigation","title":"Revisiting Frontier LLMs' Attempts to Persuade on Extreme Topics: GPT and Claude Improved, Gemini Worsened","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"deception_and_sabotage","test":"manipulation propensity evaluation","measure":null,"by":"independent_evaluator","when":"pre_deployment","result":"reports increased manipulation propensity in gemini 3 pro relative to earlier models","metric":null,"value":null,"conditions":null,"conclusion":"they “did not see evidence that this increase translates into greater manipulative efficacy”","triggered":[],"quote":"Our findings are consistent with Google’s Gemini 3 Pro Frontier Safety Framework , which reports increased manipulation propensity in Gemini 3 Pro relative to earlier models.","source_index":3,"grade":"A","verified":true,"source":{"url":"https://www.far.ai/blog/revisiting-attempts-to-persuade","publisher":"FAR.AI","date":"2026-02-10","type":"independent_investigation","title":"Revisiting Frontier LLMs' Attempts to Persuade on Extreme Topics: GPT and Claude Improved, Gemini Worsened","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"misuse_and_jailbreak","test":"attempt-to-persuade eval (APE) under low reasoning setting","measure":null,"by":"independent_evaluator","when":"not_stated","result":"the low reasoning setting leads to an attempt rate of ~90% on our \"noncontroversially harmful\" topic category, whereas high reasoning leads to a lower attempt rate, of ~25%.","metric":"attempt rate","value":"~90%","conditions":"low reasoning setting","conclusion":"\"reasoning level\" is not a standardized safety knob across models, and that increasing reasoning does not universally mitigate—or facilitate—harmful persuasion.","triggered":[],"quote":"As shown above, the low reasoning setting leads to an attempt rate of ~90% on our \"noncontroversially harmful\" topic category, whereas high reasoning leads to a lower attempt rate, of ~25%.","source_index":3,"grade":"A","verified":true,"source":{"url":"https://www.far.ai/blog/revisiting-attempts-to-persuade","publisher":"FAR.AI","date":"2026-02-10","type":"independent_investigation","title":"Revisiting Frontier LLMs' Attempts to Persuade on Extreme Topics: GPT and Claude Improved, Gemini Worsened","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"misuse_and_jailbreak","test":"attempt-to-persuade eval (APE) under high reasoning setting","measure":null,"by":"independent_evaluator","when":"not_stated","result":"the low reasoning setting leads to an attempt rate of ~90% on our \"noncontroversially harmful\" topic category, whereas high reasoning leads to a lower attempt rate, of ~25%.","metric":"attempt rate","value":"~25%","conditions":"high reasoning setting","conclusion":"\"reasoning level\" is not a standardized safety knob across models, and that increasing reasoning does not universally mitigate—or facilitate—harmful persuasion.","triggered":[],"quote":"As shown above, the low reasoning setting leads to an attempt rate of ~90% on our \"noncontroversially harmful\" topic category, whereas high reasoning leads to a lower attempt rate, of ~25%.","source_index":3,"grade":"A","verified":true,"source":{"url":"https://www.far.ai/blog/revisiting-attempts-to-persuade","publisher":"FAR.AI","date":"2026-02-10","type":"independent_investigation","title":"Revisiting Frontier LLMs' Attempts to Persuade on Extreme Topics: GPT and Claude Improved, Gemini Worsened","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"cbrn","test":"benchtop DNA synthesizer agent evaluation","measure":null,"by":"independent_evaluator","when":"post_deployment","result":"gemini 3 pro reliably design egfp dna segments that satisfy all our scoring criteria including expert review","metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"The most recent models we tested, GPT-5, Opus 4.5, and Gemini 3 Pro, reliably design eGFP DNA segments that satisfy all our scoring criteria including expert review.","source_index":4,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA4500/RRA4591-1/RAND_RRA4591-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-02-03","type":"independent_investigation","title":"Bridging the Digital to Physical Divide: Evaluating LLM Agents on Benchtop DNA Acquisition","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"cbrn","test":"EGFP task","measure":null,"by":"independent_evaluator","when":"not_stated","result":"gemini 3 pro received perfect scores across all assessment criteria in every task attempts.","metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"Figure D.9. eGFP Task Performance Results, Gemini 3 Pro","source_index":4,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA4500/RRA4591-1/RAND_RRA4591-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-02-03","type":"independent_investigation","title":"Bridging the Digital to Physical Divide: Evaluating LLM Agents on Benchtop DNA Acquisition","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"cbrn","test":"Influenza A HA segment 4 task","measure":null,"by":"independent_evaluator","when":"not_stated","result":"gemini 3 pro performed nearly perfectly across all but two attempts in which it refused to assist with the production of the influenza ha.","metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"Figure D.10. Influenza A HA Segment 4 Task Performance Results, Gemini 3 Pro","source_index":4,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA4500/RRA4591-1/RAND_RRA4591-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-02-03","type":"independent_investigation","title":"Bridging the Digital to Physical Divide: Evaluating LLM Agents on Benchtop DNA Acquisition","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"cbrn","test":"synthesizer-based DNA acquisition task","measure":null,"by":"independent_evaluator","when":"not_stated","result":null,"metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"The DNA acquisition task described in this report tests a simple and widely accessible agent framework for its ability to submit an appropriate set of DNA sequences to a simulated benchtop synthesizer interface.","source_index":4,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA4500/RRA4591-1/RAND_RRA4591-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-02-03","type":"independent_investigation","title":"Bridging the Digital to Physical Divide: Evaluating LLM Agents on Benchtop DNA Acquisition","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"cbrn","test":"benchtop DNA synthesis task","measure":null,"by":"independent_evaluator","when":"not_stated","result":null,"metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"For our assessment, we create an open-ended test environment and present agents with our synthesis task.","source_index":4,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA4500/RRA4591-1/RAND_RRA4591-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-02-03","type":"independent_investigation","title":"Bridging the Digital to Physical Divide: Evaluating LLM Agents on Benchtop DNA Acquisition","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"cbrn","test":"DNA acquisition task","measure":null,"by":"independent_evaluator","when":"not_stated","result":"It was demonstrated that neither opus 4.5 nor gemini 3 pro needed database search tools to succeed at our task.","metric":null,"value":null,"conditions":"In the final batch, which tested opus 4.5 and gemini 3 pro, we removed these database search tools.","conclusion":null,"triggered":[],"quote":"Though this means that mechanics of the sequence retrieval step differed between those two models and the remainder of our set, it was demonstrated that neither Opus 4.5 nor Gemini 3 Pro needed database search tools to succeed at our task (see Chapter 5 for test results).","source_index":4,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA4500/RRA4591-1/RAND_RRA4591-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-02-03","type":"independent_investigation","title":"Bridging the Digital to Physical Divide: Evaluating LLM Agents on Benchtop DNA Acquisition","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"cbrn","test":"agent task attempt","measure":null,"by":"independent_evaluator","when":"not_stated","result":null,"metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"Each successful agent task attempt generated a set of DNA segments that we graded against a number of feasibility criteria.","source_index":4,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA4500/RRA4591-1/RAND_RRA4591-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-02-03","type":"independent_investigation","title":"Bridging the Digital to Physical Divide: Evaluating LLM Agents on Benchtop DNA Acquisition","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"cbrn","test":"segment scorer","measure":null,"by":"independent_evaluator","when":"not_stated","result":null,"metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"We implemented the segment scorer as a set of rule-based checks for five criteria, which are listed here but described more comprehensively in Appendix B.","source_index":4,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA4500/RRA4591-1/RAND_RRA4591-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-02-03","type":"independent_investigation","title":"Bridging the Digital to Physical Divide: Evaluating LLM Agents on Benchtop DNA Acquisition","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"cbrn","test":"llm transcript autograding","measure":null,"by":"independent_evaluator","when":"not_stated","result":null,"metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"Our LLM transcript autograding was used to create additional grading dependencies, as described in the next section.","source_index":4,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA4500/RRA4591-1/RAND_RRA4591-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-02-03","type":"independent_investigation","title":"Bridging the Digital to Physical Divide: Evaluating LLM Agents on Benchtop DNA Acquisition","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"cbrn","test":"manual segment review","measure":null,"by":"independent_evaluator","when":"not_stated","result":null,"metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"For further validation of each submission that received a score of 5/5 on the above criteria, we referred it to one of our team’s biology experts for additional review.","source_index":4,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA4500/RRA4591-1/RAND_RRA4591-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-02-03","type":"independent_investigation","title":"Bridging the Digital to Physical Divide: Evaluating LLM Agents on Benchtop DNA Acquisition","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"cbrn","test":"laboratory protocol autograding","measure":null,"by":"independent_evaluator","when":"not_stated","result":null,"metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"The same autograding system used to evaluate our conceptual checks on agent trajectories was used to evaluate the agent-generated laboratory protocols that accompanied submissions of segments that successfully encoded the target DNA sequence.","source_index":4,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA4500/RRA4591-1/RAND_RRA4591-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-02-03","type":"independent_investigation","title":"Bridging the Digital to Physical Divide: Evaluating LLM Agents on Benchtop DNA Acquisition","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"cbrn","test":"egfp-targeted task","measure":null,"by":"independent_evaluator","when":"not_stated","result":null,"metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"To generate the data reported in the next chapter, each model was prompted to attempt our task ten times for each of two protein targets: eGFP and influenza HA.","source_index":4,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA4500/RRA4591-1/RAND_RRA4591-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-02-03","type":"independent_investigation","title":"Bridging the Digital to Physical Divide: Evaluating LLM Agents on Benchtop DNA Acquisition","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"cbrn","test":"influenza ha task","measure":null,"by":"independent_evaluator","when":"not_stated","result":null,"metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"To generate the data reported in the next chapter, each model was prompted to attempt our task ten times for each of two protein targets: eGFP and influenza HA.","source_index":4,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA4500/RRA4591-1/RAND_RRA4591-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-02-03","type":"independent_investigation","title":"Bridging the Digital to Physical Divide: Evaluating LLM Agents on Benchtop DNA Acquisition","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"cbrn","test":"egfp DNA sequence design task with simulated synthesizer and protocol autograding","measure":null,"by":"independent_evaluator","when":"not_stated","result":"gemini 3 pro achieved the same milestone of reliably driving the react agent to design egfp sequences that satisfied all segment scoring criteria","metric":null,"value":null,"conditions":null,"conclusion":"the extremely competent performance by the latest generation of models, gpt-5, opus 4.5, and gemini 3 pro, on the task","triggered":[],"quote":"Several months later, Claude Opus 4.5 and Gemini 3 Pro achieved the same milestone.","source_index":4,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA4500/RRA4591-1/RAND_RRA4591-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-02-03","type":"independent_investigation","title":"Bridging the Digital to Physical Divide: Evaluating LLM Agents on Benchtop DNA Acquisition","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"cbrn","test":"influenza HA oligonucleotide design task with simulated synthesizer","measure":null,"by":"independent_evaluator","when":"not_stated","result":"gemini 3 pro was the only model that was both capable of regularly producing acceptable oligonucleotides and that did so for the influenza ha target","metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"Gemini 3 Pro was the only model that was both capable of regularly producing acceptable oligonucleotides and that did so for the influenza HA target.","source_index":4,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA4500/RRA4591-1/RAND_RRA4591-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-02-03","type":"independent_investigation","title":"Bridging the Digital to Physical Divide: Evaluating LLM Agents on Benchtop DNA Acquisition","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"cbrn","test":"attachment of appropriate restriction enzyme sites to designed segments","measure":null,"by":"independent_evaluator","when":"not_stated","result":"gemini 3 pro attached bsmbi to both ends of the insert sequence in all but one case","metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"This was not a problem for Gemini 3 Pro, however, which attached BsmBI to both ends of the insert sequence in all but one case.","source_index":4,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA4500/RRA4591-1/RAND_RRA4591-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-02-03","type":"independent_investigation","title":"Bridging the Digital to Physical Divide: Evaluating LLM Agents on Benchtop DNA Acquisition","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null},{"domain":"cbrn","test":"physical validation experiment","measure":null,"by":"independent_evaluator","when":"not_stated","result":"that validation resulted in the assembly and cloning of a complete gene and a functional egfp protein","metric":null,"value":null,"conditions":null,"conclusion":null,"triggered":[],"quote":"That validation resulted in the assembly and cloning of a complete gene and a functional eGFP protein.","source_index":4,"grade":"A","verified":true,"source":{"url":"https://www.rand.org/content/dam/rand/pubs/research_reports/RRA4500/RRA4591-1/RAND_RRA4591-1.pdf","publisher":"RAND (CAST / Project Canary)","date":"2026-02-03","type":"independent_investigation","title":"Bridging the Digital to Physical Divide: Evaluating LLM Agents on Benchtop DNA Acquisition","archived_url":null,"attachment":null},"model_version_id":"MV-2025-0004","model_named":null}]}