FEAT: Add Anthropic model-written evals dataset

doc/bibliography.md

@@ -5,5 +5,5 @@ All academic papers, research blogs, and technical reports referenced throughout
 :::{dropdown} Citation Keys
 :class: hidden-citations
 
-[@aakanksha2024multilingual; @abughallous2026semguard; @adversaai2023universal; @ahn2025puzzled; @andriushchenko2024tense; @anthropic2024manyshot; @aqrawi2024singleturncrescendo; @atr2026; @banerjee2025safeinfer; @bethany2024mathprompt; @bhardwaj2023harmfulqa; @bhardwaj2024homer; @boucher2023trojan; @brahman2024coconot; @bryan2025agentictaxonomy; @bullwinkel2025airtlessons; @bullwinkel2025repeng; @bullwinkel2026trigger; @chao2023pair; @chao2024jailbreakbench; @choi2026xlsafetybench; @cui2024orbench; @darkbench2025; @derczynski2024garak; @ding2023wolf; @dong2025sata; @embracethered2024unicode; @embracethered2025sneakybits; @evtimov2025wasp; @gehman2020realtoxicityprompts; @ghosh2025aegis; @ghosh2025ailuminate; @gong2025figstep; @gupta2024walledeval; @haider2024phi3safety; @han2024medsafetybench; @han2024wildguard; @hiddenlayer2025policypuppetry; @hines2024spotlighting; @huang2024bijectionlearning; @hughes2024bestofn; @inie2025summon; @ji2023beavertails; @ji2024pkusaferlhf; @jiang2025sosbench; @jones2025computeruse; @kim2023propile; @kingma2014adam; @knight2025fortress; @li2024drattack; @li2024mossbench; @li2024saladbench; @li2024wmdp; @lin2023toxicchat; @liu2024flipattack; @liu2024mmsafetybench; @lopez2024pyrit; @luo2024jailbreakv; @lutz2026pyrit; @lv2024codechameleon; @mazeika2023tdc; @mazeika2024harmbench; @mckee2024transparency; @mehrotra2023tap; @microsoft2024skeletonkey; @odin2024; @palaskar2025vlsu; @pavlova2024goat; @pfohl2024equitymedqa; @promptfoo2025ccp; @ren2024codeattack; @robustintelligence2024bypass; @roccia2024promptintel; @rottger2023xstest; @rottger2025msts; @russinovich2024crescendo; @russinovich2025cca; @russinovich2025price; @scheuerman2025transphobia; @shaikh2022second; @shayegani2025computeruse; @shen2023donotanything; @sheshadri2024lat; @souly2024strongreject; @stok2023ansi; @tan2026comicjailbreak; @tang2025multilingual; @tedeschi2024alert; @vantaylor2024socialbias; @vidgen2023simplesafetytests; @wang2023decodingtrust; @wang2023donotanswer; @wang2025siuo; @wang2026visualleakbench; @wei2023jailbroken; @xie2024sorrybench; @yu2023gptfuzzer; @yuan2023cipherchat; @zeng2024persuasion; @zeng2024shieldgemma; @zhang2024cbtbench; @ziems2022mic; @zong2024vlguard; @zou2023gcg]
+[@aakanksha2024multilingual; @abughallous2026semguard; @adversaai2023universal; @ahn2025puzzled; @andriushchenko2024tense; @anthropic2024manyshot; @aqrawi2024singleturncrescendo; @atr2026; @banerjee2025safeinfer; @bethany2024mathprompt; @bhardwaj2023harmfulqa; @bhardwaj2024homer; @boucher2023trojan; @brahman2024coconot; @bryan2025agentictaxonomy; @bullwinkel2025airtlessons; @bullwinkel2025repeng; @bullwinkel2026trigger; @chao2023pair; @chao2024jailbreakbench; @choi2026xlsafetybench; @cui2024orbench; @darkbench2025; @derczynski2024garak; @ding2023wolf; @dong2025sata; @embracethered2024unicode; @embracethered2025sneakybits; @evtimov2025wasp; @gehman2020realtoxicityprompts; @ghosh2025aegis; @ghosh2025ailuminate; @gong2025figstep; @gupta2024walledeval; @haider2024phi3safety; @han2024medsafetybench; @han2024wildguard; @hiddenlayer2025policypuppetry; @hines2024spotlighting; @huang2024bijectionlearning; @hughes2024bestofn; @inie2025summon; @ji2023beavertails; @ji2024pkusaferlhf; @jiang2025sosbench; @jones2025computeruse; @kim2023propile; @kingma2014adam; @knight2025fortress; @li2024drattack; @li2024mossbench; @li2024saladbench; @li2024wmdp; @lin2023toxicchat; @liu2024flipattack; @liu2024mmsafetybench; @lopez2024pyrit; @luo2024jailbreakv; @lutz2026pyrit; @lv2024codechameleon; @mazeika2023tdc; @mazeika2024harmbench; @mckee2024transparency; @mehrotra2023tap; @microsoft2024skeletonkey; @odin2024; @palaskar2025vlsu; @pavlova2024goat; @perez2022modelwritten; @pfohl2024equitymedqa; @promptfoo2025ccp; @ren2024codeattack; @robustintelligence2024bypass; @roccia2024promptintel; @rottger2023xstest; @rottger2025msts; @russinovich2024crescendo; @russinovich2025cca; @russinovich2025price; @scheuerman2025transphobia; @shaikh2022second; @shayegani2025computeruse; @shen2023donotanything; @sheshadri2024lat; @souly2024strongreject; @stok2023ansi; @tan2026comicjailbreak; @tang2025multilingual; @tedeschi2024alert; @vantaylor2024socialbias; @vidgen2023simplesafetytests; @wang2023decodingtrust; @wang2023donotanswer; @wang2025siuo; @wang2026visualleakbench; @wei2023jailbroken; @xie2024sorrybench; @yu2023gptfuzzer; @yuan2023cipherchat; @zeng2024persuasion; @zeng2024shieldgemma; @zhang2024cbtbench; @ziems2022mic; @zong2024vlguard; @zou2023gcg]
 :::

doc/code/datasets/1_loading_datasets.ipynb

@@ -18,6 +18,7 @@
     "Aegis [@ghosh2025aegis],\n",
     "Agent Threat Rules [@atr2026],\n",
     "ALERT [@tedeschi2024alert],\n",
+    "Anthropic Model-Written Evals [@perez2022modelwritten],\n",
     "BeaverTails [@ji2023beavertails],\n",
     "CBT-Bench [@zhang2024cbtbench],\n",
     "CategoricalHarmfulQA (CatQA) [@bhardwaj2024homer],\n",
@@ -94,6 +95,7 @@
        " '0din_technical_field_guide',\n",
        " '0din_threatfeed',\n",
        " 'adv_bench',\n",
+       " 'adversarial_benchmark_v1',\n",
        " 'aegis_content_safety',\n",
        " 'agent_threat_rules',\n",
        " 'airt_fairness',\n",
@@ -110,6 +112,7 @@
        " 'airt_scams',\n",
        " 'airt_sexual',\n",
        " 'airt_violence',\n",
+       " 'anthropic_model_written_evals',\n",
        " 'aya_redteaming',\n",
        " 'babelscape_alert',\n",
        " 'beaver_tails',\n",

doc/code/datasets/1_loading_datasets.py

@@ -22,6 +22,7 @@
 # Aegis [@ghosh2025aegis],
 # Agent Threat Rules [@atr2026],
 # ALERT [@tedeschi2024alert],
+# Anthropic Model-Written Evals [@perez2022modelwritten],
 # BeaverTails [@ji2023beavertails],
 # CBT-Bench [@zhang2024cbtbench],
 # CategoricalHarmfulQA (CatQA) [@bhardwaj2024homer],

doc/references.bib

@@ -897,3 +897,12 @@ @article{kim2023propile
   year      = {2023},
   url       = {https://arxiv.org/abs/2307.01881},
 }
+
+@inproceedings{perez2022modelwritten,
+  title     = {Discovering Language Model Behaviors with Model-Written Evaluations},
+  author    = {Ethan Perez and Sam Ringer and Kamil{\.e} Luko{\v{s}}i{\=u}t{\.e} and Karina Nguyen and Edwin Chen and Scott Heiner and Craig Pettit and Catherine Olsson and Sandipan Kundu and Saurav Kadavath and others},
+  booktitle = {Findings of the Association for Computational Linguistics: ACL 2023},
+  pages     = {13387--13434},
+  year      = {2023},
+  url       = {https://arxiv.org/abs/2212.09251},
+}

pyrit/datasets/seed_datasets/remote/__init__.py

@@ -23,6 +23,10 @@
         ATRVariationType,
         _AgentThreatRulesDataset,
     )
+    from pyrit.datasets.seed_datasets.remote.anthropic_model_written_evals_dataset import (
+        AnthropicModelWrittenEvalCategory,
+        _AnthropicModelWrittenEvalsDataset,
+    )
     from pyrit.datasets.seed_datasets.remote.aya_redteaming_dataset import _AyaRedteamingDataset
     from pyrit.datasets.seed_datasets.remote.babelscape_alert_dataset import _BabelscapeAlertDataset
     from pyrit.datasets.seed_datasets.remote.beaver_tails_dataset import _BeaverTailsDataset
@@ -158,6 +162,7 @@
 
 _LAZY_EXPORTS: dict[str, str | tuple[str, str | None]] = {
     "AegisHarmCategory": "pyrit.datasets.seed_datasets.remote.aegis_ai_content_safety_dataset",
+    "AnthropicModelWrittenEvalCategory": "pyrit.datasets.seed_datasets.remote.anthropic_model_written_evals_dataset",
     "CoCoNotCategory": "pyrit.datasets.seed_datasets.remote.coconot_dataset",
     "CoCoNotSplit": "pyrit.datasets.seed_datasets.remote.coconot_dataset",
     "DecodingTrustToxicitySubset": "pyrit.datasets.seed_datasets.remote.decoding_trust_toxicity_dataset",
@@ -186,6 +191,7 @@
     "WildGuardMixPromptHarmLabel": "pyrit.datasets.seed_datasets.remote.wildguardmix_dataset",
     "WildGuardMixSplit": "pyrit.datasets.seed_datasets.remote.wildguardmix_dataset",
     "_AegisContentSafetyDataset": "pyrit.datasets.seed_datasets.remote.aegis_ai_content_safety_dataset",
+    "_AnthropicModelWrittenEvalsDataset": "pyrit.datasets.seed_datasets.remote.anthropic_model_written_evals_dataset",
     "ATRCategory": "pyrit.datasets.seed_datasets.remote.agent_threat_rules_dataset",
     "ATRDetectionField": "pyrit.datasets.seed_datasets.remote.agent_threat_rules_dataset",
     "ATRVariationType": "pyrit.datasets.seed_datasets.remote.agent_threat_rules_dataset",