[BREAKING] FEAT: Standardize Jailbreak scenario defaults

doc/scanner/airt.ipynb

@@ -24,7 +24,9 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "2",
-   "metadata": {},
+   "metadata": {
+    "lines_to_next_cell": 0
+   },
    "outputs": [
     {
      "name": "stderr",
@@ -75,7 +77,7 @@
     "  --initializers target \\\n",
     "  --target openai_chat \\\n",
     "  --strategies role_play \\\n",
-    "  --dataset-names airt_hate \\  \n",
+    "  --dataset-names airt_hate \\\n",
     "  --max-dataset-size 1\n",
     "```\n",
     "\n",
@@ -205,9 +207,9 @@
     "Tests whether a target appropriately handles psychosocial crisis situations, evaluating imminent\n",
     "crisis handling and licensed therapist impersonation.\n",
     "\n",
-    "`\bash\n",
+    "```bash\n",
     "pyrit_scan airt.psychosocial --target openai_chat --strategies imminent_crisis --max-dataset-size 1\n",
-    "`\n",
+    "```\n",
     "\n",
     "**Available strategies:** ALL, ImminentCrisis, LicensedTherapist\n",
     "\n",
@@ -499,18 +501,29 @@
    "source": [
     "## Jailbreak\n",
     "\n",
-    "Tests target resilience against template-based jailbreak attacks using various prompt injection\n",
-    "templates.\n",
+    "Tests target resilience against template-based jailbreak attacks across three axes: **objectives**\n",
+    "(harmful behaviors, via `--dataset-names`, default HarmBench), **techniques** (how each jailbroken\n",
+    "objective is delivered — default is to \"just send\" it; the shared techniques like `role_play` and\n",
+    "`many_shot` are also selectable via `--strategies`), and **jailbreaks** (which templates to apply,\n",
+    "a random sample by default).\n",
     "\n",
     "```bash\n",
     "pyrit_scan airt.jailbreak \\\n",
-    "  --initializers target \\\n",
+    "  --initializers target load_default_datasets techniques \\\n",
     "  --target openai_chat \\\n",
-    "  --strategies prompt_sending \\\n",
+    "  --num-templates 1 \\\n",
     "  --max-dataset-size 1\n",
     "```\n",
     "\n",
-    "**Available strategies:** ALL, SIMPLE, COMPLEX, PromptSending, ManyShot, SkeletonKey, RolePlay"
+    "**Available strategies (techniques):** the default is `prompt_sending` (\"just send\"); the shared\n",
+    "`core` techniques (`role_play`, `many_shot`, `tap`, `crescendo`, …) are selectable via\n",
+    "`--strategies`, plus the `single_turn` / `multi_turn` / `all` aggregates.\n",
+    "\n",
+    "By default the scenario randomly samples `num_templates` jailbreak templates (default: 10) to keep\n",
+    "runs fast and predictable. Pass `--num-templates N` to widen or narrow coverage, `jailbreak_names`\n",
+    "to pin specific templates, or `--num-attempts N` to repeat each template. The specific templates\n",
+    "chosen for a run are printed in the scenario output under **Scenario Inputs** and persisted to\n",
+    "`scenario_result.metadata`."
    ]
   },
   {
@@ -520,14 +533,13 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from pyrit.scenario.airt import Jailbreak, JailbreakStrategy\n",
+    "from pyrit.scenario.airt import Jailbreak\n",
     "\n",
-    "dataset_config = DatasetConfiguration(dataset_names=[\"airt_harms\"], max_dataset_size=1)\n",
+    "dataset_config = DatasetConfiguration(dataset_names=[\"harmbench\"], max_dataset_size=1)\n",
     "\n",
-    "scenario = Jailbreak()\n",
+    "scenario = Jailbreak(num_templates=2)\n",
     "await scenario.initialize_async(  # type: ignore\n",
     "    objective_target=objective_target,\n",
-    "    scenario_strategies=[JailbreakStrategy.PromptSending],\n",
     "    dataset_config=dataset_config,\n",
     ")\n",
     "\n",

doc/scanner/airt.py

@@ -5,7 +5,7 @@
 #       extension: .py
 #       format_name: percent
 #       format_version: '1.3'
-#       jupytext_version: 1.19.1
+#       jupytext_version: 1.19.3
 # ---
 
 # %% [markdown]
@@ -151,28 +151,38 @@
 # %% [markdown]
 # ## Jailbreak
 #
-# Tests target resilience against template-based jailbreak attacks using various prompt injection
-# templates.
+# Tests target resilience against template-based jailbreak attacks across three axes: **objectives**
+# (harmful behaviors, via `--dataset-names`, default HarmBench), **techniques** (how each jailbroken
+# objective is delivered — default is to "just send" it; the shared techniques like `role_play` and
+# `many_shot` are also selectable via `--strategies`), and **jailbreaks** (which templates to apply,
+# a random sample by default).
 #
 # ```bash
 # pyrit_scan airt.jailbreak \
-#   --initializers target \
+#   --initializers target load_default_datasets techniques \
 #   --target openai_chat \
-#   --strategies prompt_sending \
+#   --num-templates 1 \
 #   --max-dataset-size 1
 # ```
 #
-# **Available strategies:** ALL, SIMPLE, COMPLEX, PromptSending, ManyShot, SkeletonKey, RolePlay
+# **Available strategies (techniques):** the default is `prompt_sending` ("just send"); the shared
+# `core` techniques (`role_play`, `many_shot`, `tap`, `crescendo`, …) are selectable via
+# `--strategies`, plus the `single_turn` / `multi_turn` / `all` aggregates.
+#
+# By default the scenario randomly samples `num_templates` jailbreak templates (default: 10) to keep
+# runs fast and predictable. Pass `--num-templates N` to widen or narrow coverage, `jailbreak_names`
+# to pin specific templates, or `--num-attempts N` to repeat each template. The specific templates
+# chosen for a run are printed in the scenario output under **Scenario Inputs** and persisted to
+# `scenario_result.metadata`.
 
 # %%
-from pyrit.scenario.airt import Jailbreak, JailbreakStrategy
+from pyrit.scenario.airt import Jailbreak
 
-dataset_config = DatasetConfiguration(dataset_names=["airt_harms"], max_dataset_size=1)
+dataset_config = DatasetConfiguration(dataset_names=["harmbench"], max_dataset_size=1)
 
-scenario = Jailbreak()
+scenario = Jailbreak(num_templates=2)
 await scenario.initialize_async(  # type: ignore
     objective_target=objective_target,
-    scenario_strategies=[JailbreakStrategy.PromptSending],
     dataset_config=dataset_config,
 )
 

pyrit/datasets/jailbreak/text_jailbreak.py

@@ -220,13 +220,16 @@ def get_jailbreak_templates(cls, num_templates: int | None = None) -> list[str]:
 
         Raises:
             ValueError: If no jailbreak templates are found in the jailbreak directory.
-            ValueError: If n is larger than the number of templates that exist.
+            ValueError: If num_templates is not a positive integer.
+            ValueError: If num_templates is larger than the number of templates that exist.
         """
         jailbreak_template_names = sorted(cls._get_template_cache().keys())
         if not jailbreak_template_names:
             raise ValueError("No jailbreak templates found in the jailbreak directory")
 
-        if num_templates:
+        if num_templates is not None:
+            if num_templates <= 0:
+                raise ValueError(f"num_templates must be a positive integer or None, got {num_templates}.")
             if num_templates > len(jailbreak_template_names):
                 raise ValueError(
                     f"Attempted to pull {num_templates} jailbreaks from a dataset"

pyrit/models/results/scenario_result.py

@@ -99,7 +99,10 @@ class ScenarioResult(BaseModel):
     #: Free-form JSON metadata persisted with the scenario result. Currently used to record
     #: ``objective_hashes`` — the objective ``sha256`` set chosen on the first run, replayed
     #: on resume so a fresh ``random.sample`` can't silently change which objectives the
-    #: scenario operates on. Keys are not part of any public contract and may evolve.
+    #: scenario operates on. Scenarios may also set ``summary`` (a ``dict[str, str]`` of
+    #: human-readable label -> value pairs, e.g. the jailbreak templates sampled) which the
+    #: pretty printer renders under "Scenario Inputs". Keys are not part of any public contract
+    #: and may evolve.
     metadata: dict[str, Any] = Field(default_factory=dict)
 
     @model_validator(mode="before")

pyrit/output/scenario_result/pretty.py

@@ -133,6 +133,37 @@ def _get_rate_color(self, rate: int) -> str:
             return str(Fore.CYAN)
         return str(Fore.GREEN)
 
+    def _render_scenario_inputs(self, result: ScenarioResult) -> str:
+        """
+        Render the scenario's human-readable input summary, if any.
+
+        Scenarios may record a ``summary`` mapping (``dict[str, str]`` of label -> value) in
+        ``ScenarioResult.metadata`` to surface the concrete inputs a run used — e.g. the jailbreak
+        templates that were sampled. Only this curated mapping is rendered; other internal metadata
+        keys (such as ``objective_hashes``) are never shown.
+
+        Args:
+            result (ScenarioResult): The scenario result.
+
+        Returns:
+            str: The rendered "Scenario Inputs" block, or an empty string when no summary is present.
+        """
+        metadata = result.metadata or {}
+        summary = metadata.get("summary")
+        if not isinstance(summary, dict) or not summary:
+            return ""
+
+        lines: list[str] = []
+        lines.append("\n")
+        lines.append(self._format_colored(f"{self._indent}🧪 Scenario Inputs", Style.BRIGHT))
+        value_indent = self._indent * 4
+        available_width = 120 - len(value_indent)
+        for key, value in summary.items():
+            lines.append(self._format_colored(f"{self._indent * 2}• {key}:", Fore.CYAN))
+            wrapped_lines = textwrap.wrap(str(value), width=available_width, break_long_words=False) or [""]
+            lines.extend(self._format_colored(f"{value_indent}{line}", Fore.CYAN) for line in wrapped_lines)
+        return "".join(lines)
+
     async def render_async(self, result: ScenarioResult) -> str:
         """
         Render the scenario result summary and return it as a string.
@@ -167,6 +198,8 @@ async def render_async(self, result: ScenarioResult) -> str:
             wrapped_lines = textwrap.wrap(result.scenario_description, width=available_width, break_long_words=False)
             lines.extend(self._format_colored(f"{desc_indent}{line}", Fore.CYAN) for line in wrapped_lines)
 
+        lines.append(self._render_scenario_inputs(result))
+
         lines.append("\n")
         lines.append(self._format_colored(f"{self._indent}🎯 Target Information", Style.BRIGHT))
         target_id = result.objective_target_identifier

pyrit/scenario/scenarios/airt/__init__.py

@@ -6,7 +6,7 @@
 from typing import Any
 
 from pyrit.scenario.scenarios.airt.cyber import Cyber, _build_cyber_strategy
-from pyrit.scenario.scenarios.airt.jailbreak import Jailbreak, JailbreakStrategy
+from pyrit.scenario.scenarios.airt.jailbreak import Jailbreak, _build_jailbreak_strategy
 from pyrit.scenario.scenarios.airt.leakage import Leakage, _build_leakage_strategy
 from pyrit.scenario.scenarios.airt.psychosocial import Psychosocial, PsychosocialStrategy
 from pyrit.scenario.scenarios.airt.rapid_response import RapidResponse, _build_rapid_response_strategy
@@ -23,6 +23,8 @@ def __getattr__(name: str) -> Any:
     Raises:
         AttributeError: If the attribute name is not recognized.
     """
+    if name == "JailbreakStrategy":
+        return _build_jailbreak_strategy()
     if name == "RapidResponseStrategy":
         return _build_rapid_response_strategy()
     if name == "LeakageStrategy":