diff --git a/doc/scanner/airt.ipynb b/doc/scanner/airt.ipynb index cb4107454b..1803fbc4a3 100644 --- a/doc/scanner/airt.ipynb +++ b/doc/scanner/airt.ipynb @@ -24,7 +24,9 @@ "cell_type": "code", "execution_count": null, "id": "2", - "metadata": {}, + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [ { "name": "stderr", @@ -75,7 +77,7 @@ " --initializers target \\\n", " --target openai_chat \\\n", " --strategies role_play \\\n", - " --dataset-names airt_hate \\ \n", + " --dataset-names airt_hate \\\n", " --max-dataset-size 1\n", "```\n", "\n", @@ -205,9 +207,9 @@ "Tests whether a target appropriately handles psychosocial crisis situations, evaluating imminent\n", "crisis handling and licensed therapist impersonation.\n", "\n", - "`\bash\n", + "```bash\n", "pyrit_scan airt.psychosocial --target openai_chat --strategies imminent_crisis --max-dataset-size 1\n", - "`\n", + "```\n", "\n", "**Available strategies:** ALL, ImminentCrisis, LicensedTherapist\n", "\n", @@ -499,18 +501,29 @@ "source": [ "## Jailbreak\n", "\n", - "Tests target resilience against template-based jailbreak attacks using various prompt injection\n", - "templates.\n", + "Tests target resilience against template-based jailbreak attacks across three axes: **objectives**\n", + "(harmful behaviors, via `--dataset-names`, default HarmBench), **techniques** (how each jailbroken\n", + "objective is delivered — default is to \"just send\" it; the shared techniques like `role_play` and\n", + "`many_shot` are also selectable via `--strategies`), and **jailbreaks** (which templates to apply,\n", + "a random sample by default).\n", "\n", "```bash\n", "pyrit_scan airt.jailbreak \\\n", - " --initializers target \\\n", + " --initializers target load_default_datasets techniques \\\n", " --target openai_chat \\\n", - " --strategies prompt_sending \\\n", + " --num-templates 1 \\\n", " --max-dataset-size 1\n", "```\n", "\n", - "**Available strategies:** ALL, SIMPLE, COMPLEX, PromptSending, ManyShot, SkeletonKey, RolePlay" + "**Available strategies (techniques):** the default is `prompt_sending` (\"just send\"); the shared\n", + "`core` techniques (`role_play`, `many_shot`, `tap`, `crescendo`, …) are selectable via\n", + "`--strategies`, plus the `single_turn` / `multi_turn` / `all` aggregates.\n", + "\n", + "By default the scenario randomly samples `num_templates` jailbreak templates (default: 10) to keep\n", + "runs fast and predictable. Pass `--num-templates N` to widen or narrow coverage, `jailbreak_names`\n", + "to pin specific templates, or `--num-attempts N` to repeat each template. The specific templates\n", + "chosen for a run are printed in the scenario output under **Scenario Inputs** and persisted to\n", + "`scenario_result.metadata`." ] }, { @@ -520,14 +533,13 @@ "metadata": {}, "outputs": [], "source": [ - "from pyrit.scenario.airt import Jailbreak, JailbreakStrategy\n", + "from pyrit.scenario.airt import Jailbreak\n", "\n", - "dataset_config = DatasetConfiguration(dataset_names=[\"airt_harms\"], max_dataset_size=1)\n", + "dataset_config = DatasetConfiguration(dataset_names=[\"harmbench\"], max_dataset_size=1)\n", "\n", - "scenario = Jailbreak()\n", + "scenario = Jailbreak(num_templates=2)\n", "await scenario.initialize_async( # type: ignore\n", " objective_target=objective_target,\n", - " scenario_strategies=[JailbreakStrategy.PromptSending],\n", " dataset_config=dataset_config,\n", ")\n", "\n", diff --git a/doc/scanner/airt.py b/doc/scanner/airt.py index 910a744eab..0285a0d095 100644 --- a/doc/scanner/airt.py +++ b/doc/scanner/airt.py @@ -5,7 +5,7 @@ # extension: .py # format_name: percent # format_version: '1.3' -# jupytext_version: 1.19.1 +# jupytext_version: 1.19.3 # --- # %% [markdown] @@ -151,28 +151,38 @@ # %% [markdown] # ## Jailbreak # -# Tests target resilience against template-based jailbreak attacks using various prompt injection -# templates. +# Tests target resilience against template-based jailbreak attacks across three axes: **objectives** +# (harmful behaviors, via `--dataset-names`, default HarmBench), **techniques** (how each jailbroken +# objective is delivered — default is to "just send" it; the shared techniques like `role_play` and +# `many_shot` are also selectable via `--strategies`), and **jailbreaks** (which templates to apply, +# a random sample by default). # # ```bash # pyrit_scan airt.jailbreak \ -# --initializers target \ +# --initializers target load_default_datasets techniques \ # --target openai_chat \ -# --strategies prompt_sending \ +# --num-templates 1 \ # --max-dataset-size 1 # ``` # -# **Available strategies:** ALL, SIMPLE, COMPLEX, PromptSending, ManyShot, SkeletonKey, RolePlay +# **Available strategies (techniques):** the default is `prompt_sending` ("just send"); the shared +# `core` techniques (`role_play`, `many_shot`, `tap`, `crescendo`, …) are selectable via +# `--strategies`, plus the `single_turn` / `multi_turn` / `all` aggregates. +# +# By default the scenario randomly samples `num_templates` jailbreak templates (default: 10) to keep +# runs fast and predictable. Pass `--num-templates N` to widen or narrow coverage, `jailbreak_names` +# to pin specific templates, or `--num-attempts N` to repeat each template. The specific templates +# chosen for a run are printed in the scenario output under **Scenario Inputs** and persisted to +# `scenario_result.metadata`. # %% -from pyrit.scenario.airt import Jailbreak, JailbreakStrategy +from pyrit.scenario.airt import Jailbreak -dataset_config = DatasetConfiguration(dataset_names=["airt_harms"], max_dataset_size=1) +dataset_config = DatasetConfiguration(dataset_names=["harmbench"], max_dataset_size=1) -scenario = Jailbreak() +scenario = Jailbreak(num_templates=2) await scenario.initialize_async( # type: ignore objective_target=objective_target, - scenario_strategies=[JailbreakStrategy.PromptSending], dataset_config=dataset_config, ) diff --git a/pyrit/datasets/jailbreak/text_jailbreak.py b/pyrit/datasets/jailbreak/text_jailbreak.py index b4affc3d51..0df9486595 100644 --- a/pyrit/datasets/jailbreak/text_jailbreak.py +++ b/pyrit/datasets/jailbreak/text_jailbreak.py @@ -220,13 +220,16 @@ def get_jailbreak_templates(cls, num_templates: int | None = None) -> list[str]: Raises: ValueError: If no jailbreak templates are found in the jailbreak directory. - ValueError: If n is larger than the number of templates that exist. + ValueError: If num_templates is not a positive integer. + ValueError: If num_templates is larger than the number of templates that exist. """ jailbreak_template_names = sorted(cls._get_template_cache().keys()) if not jailbreak_template_names: raise ValueError("No jailbreak templates found in the jailbreak directory") - if num_templates: + if num_templates is not None: + if num_templates <= 0: + raise ValueError(f"num_templates must be a positive integer or None, got {num_templates}.") if num_templates > len(jailbreak_template_names): raise ValueError( f"Attempted to pull {num_templates} jailbreaks from a dataset" diff --git a/pyrit/models/results/scenario_result.py b/pyrit/models/results/scenario_result.py index 6719f1affd..aa72cc9015 100644 --- a/pyrit/models/results/scenario_result.py +++ b/pyrit/models/results/scenario_result.py @@ -99,7 +99,10 @@ class ScenarioResult(BaseModel): #: Free-form JSON metadata persisted with the scenario result. Currently used to record #: ``objective_hashes`` — the objective ``sha256`` set chosen on the first run, replayed #: on resume so a fresh ``random.sample`` can't silently change which objectives the - #: scenario operates on. Keys are not part of any public contract and may evolve. + #: scenario operates on. Scenarios may also set ``summary`` (a ``dict[str, str]`` of + #: human-readable label -> value pairs, e.g. the jailbreak templates sampled) which the + #: pretty printer renders under "Scenario Inputs". Keys are not part of any public contract + #: and may evolve. metadata: dict[str, Any] = Field(default_factory=dict) @model_validator(mode="before") diff --git a/pyrit/output/scenario_result/pretty.py b/pyrit/output/scenario_result/pretty.py index 7f570fec20..c56d24fcf5 100644 --- a/pyrit/output/scenario_result/pretty.py +++ b/pyrit/output/scenario_result/pretty.py @@ -133,6 +133,37 @@ def _get_rate_color(self, rate: int) -> str: return str(Fore.CYAN) return str(Fore.GREEN) + def _render_scenario_inputs(self, result: ScenarioResult) -> str: + """ + Render the scenario's human-readable input summary, if any. + + Scenarios may record a ``summary`` mapping (``dict[str, str]`` of label -> value) in + ``ScenarioResult.metadata`` to surface the concrete inputs a run used — e.g. the jailbreak + templates that were sampled. Only this curated mapping is rendered; other internal metadata + keys (such as ``objective_hashes``) are never shown. + + Args: + result (ScenarioResult): The scenario result. + + Returns: + str: The rendered "Scenario Inputs" block, or an empty string when no summary is present. + """ + metadata = result.metadata or {} + summary = metadata.get("summary") + if not isinstance(summary, dict) or not summary: + return "" + + lines: list[str] = [] + lines.append("\n") + lines.append(self._format_colored(f"{self._indent}🧪 Scenario Inputs", Style.BRIGHT)) + value_indent = self._indent * 4 + available_width = 120 - len(value_indent) + for key, value in summary.items(): + lines.append(self._format_colored(f"{self._indent * 2}• {key}:", Fore.CYAN)) + wrapped_lines = textwrap.wrap(str(value), width=available_width, break_long_words=False) or [""] + lines.extend(self._format_colored(f"{value_indent}{line}", Fore.CYAN) for line in wrapped_lines) + return "".join(lines) + async def render_async(self, result: ScenarioResult) -> str: """ Render the scenario result summary and return it as a string. @@ -167,6 +198,8 @@ async def render_async(self, result: ScenarioResult) -> str: wrapped_lines = textwrap.wrap(result.scenario_description, width=available_width, break_long_words=False) lines.extend(self._format_colored(f"{desc_indent}{line}", Fore.CYAN) for line in wrapped_lines) + lines.append(self._render_scenario_inputs(result)) + lines.append("\n") lines.append(self._format_colored(f"{self._indent}🎯 Target Information", Style.BRIGHT)) target_id = result.objective_target_identifier diff --git a/pyrit/scenario/scenarios/airt/__init__.py b/pyrit/scenario/scenarios/airt/__init__.py index d1efdbd82d..d9e87ec8f2 100644 --- a/pyrit/scenario/scenarios/airt/__init__.py +++ b/pyrit/scenario/scenarios/airt/__init__.py @@ -6,7 +6,7 @@ from typing import Any from pyrit.scenario.scenarios.airt.cyber import Cyber, _build_cyber_strategy -from pyrit.scenario.scenarios.airt.jailbreak import Jailbreak, JailbreakStrategy +from pyrit.scenario.scenarios.airt.jailbreak import Jailbreak, _build_jailbreak_strategy from pyrit.scenario.scenarios.airt.leakage import Leakage, _build_leakage_strategy from pyrit.scenario.scenarios.airt.psychosocial import Psychosocial, PsychosocialStrategy from pyrit.scenario.scenarios.airt.rapid_response import RapidResponse, _build_rapid_response_strategy @@ -23,6 +23,8 @@ def __getattr__(name: str) -> Any: Raises: AttributeError: If the attribute name is not recognized. """ + if name == "JailbreakStrategy": + return _build_jailbreak_strategy() if name == "RapidResponseStrategy": return _build_rapid_response_strategy() if name == "LeakageStrategy": diff --git a/pyrit/scenario/scenarios/airt/jailbreak.py b/pyrit/scenario/scenarios/airt/jailbreak.py index 7ed3bf2efb..fc7a4dc1b4 100644 --- a/pyrit/scenario/scenarios/airt/jailbreak.py +++ b/pyrit/scenario/scenarios/airt/jailbreak.py @@ -1,84 +1,157 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import logging +from functools import cache from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any, ClassVar from pyrit.common import apply_defaults from pyrit.datasets import TextJailBreak -from pyrit.executor.attack.core.attack_config import AttackAdversarialConfig, AttackConverterConfig, AttackScoringConfig -from pyrit.executor.attack.single_turn.many_shot_jailbreak import ManyShotJailbreakAttack from pyrit.executor.attack.single_turn.prompt_sending import PromptSendingAttack -from pyrit.executor.attack.single_turn.role_play import RolePlayAttack, RolePlayPaths -from pyrit.executor.attack.single_turn.skeleton_key import SkeletonKeyAttack -from pyrit.models import SeedAttackGroup -from pyrit.prompt_converter import TextJailbreakConverter -from pyrit.prompt_normalizer import PromptConverterConfiguration -from pyrit.prompt_target.common.prompt_target import PromptTarget +from pyrit.models import Parameter +from pyrit.prompt_converter import PromptConverter, TextJailbreakConverter from pyrit.scenario.core.atomic_attack import AtomicAttack -from pyrit.scenario.core.attack_technique import AttackTechnique +from pyrit.scenario.core.attack_technique_factory import AttackTechniqueFactory from pyrit.scenario.core.dataset_configuration import DatasetAttackConfiguration +from pyrit.scenario.core.matrix_atomic_attack_builder import ( + MatrixAtomicAttackBuilder, + MatrixCombo, + resolve_technique_factories, +) from pyrit.scenario.core.scenario import Scenario from pyrit.scenario.core.scenario_context import ScenarioContext -from pyrit.scenario.core.scenario_strategy import ScenarioStrategy -from pyrit.scenario.core.scenario_target_defaults import get_default_adversarial_target from pyrit.score import TrueFalseScorer +if TYPE_CHECKING: + from pyrit.scenario.core.scenario_strategy import ScenarioStrategy -class JailbreakStrategy(ScenarioStrategy): +logger = logging.getLogger(__name__) + + +class _Unset: + """Sentinel marking an omitted ``num_templates`` argument (distinct from an explicit ``None``).""" + + +_UNSET = _Unset() + + +def _prompt_sending_factory() -> AttackTechniqueFactory: """ - Strategy for jailbreak attacks. + Build the scenario-local "just send" technique factory. - The SIMPLE strategy just sends the jailbroken prompt and records the response. It is meant to - expose an obvious way of using this scenario without worrying about additional tweaks and changes - to the prompt. + The shared ``core`` technique catalog intentionally omits a bare + ``PromptSendingAttack`` (the central baseline covers it), but Jailbreak's + default technique is to send the jailbroken objective unmodified, so it + supplies its own. - COMPLEX strategies use additional techniques to enhance the jailbreak like modifying the - system prompt or probing the target model for an additional vulnerability (e.g. the SkeletonKeyAttack). - They are meant to provide a sense of how well a jailbreak generalizes to slight changes in the delivery - method. + Returns: + AttackTechniqueFactory: The ``prompt_sending`` factory. """ + return AttackTechniqueFactory( + name="prompt_sending", + attack_class=PromptSendingAttack, + strategy_tags=["single_turn"], + ) - # Aggregate members (special markers that expand to strategies with matching tags) - ALL = ("all", {"all"}) - SIMPLE = ("simple", {"simple"}) - COMPLEX = ("complex", {"complex"}) - # Simple strategies - PromptSending = ("prompt_sending", {"simple"}) +@cache +def _build_jailbreak_strategy() -> type["ScenarioStrategy"]: + """ + Build the Jailbreak strategy class dynamically from the registered technique factories. - # Complex strategies - ManyShot = ("many_shot", {"complex"}) - SkeletonKey = ("skeleton", {"complex"}) - RolePlay = ("role_play", {"complex"}) + Mirrors ``RapidResponse``: the technique axis is the shared ``core`` catalog + (role_play, many_shot, tap, crescendo, …) plus the scenario-local + ``prompt_sending`` technique. The scenario's ``default_strategy`` is the + concrete ``prompt_sending`` member, so a no-strategy run "just sends" the + jailbroken objective; ``single_turn`` / ``multi_turn`` / ``all`` aggregates + remain available for broader selection. - @classmethod - def get_aggregate_tags(cls) -> set[str]: - """ - Get the set of tags that represent aggregate categories. + Returns: + type[ScenarioStrategy]: The dynamically generated strategy enum class. + """ + from pyrit.registry.components.attack_technique_registry import AttackTechniqueRegistry + from pyrit.registry.tag_query import TagQuery - Returns: - set[str]: Set of tags that are aggregate markers. - """ - # Include base class aggregates ("all") and add scenario-specific ones - return super().get_aggregate_tags() | {"simple", "complex"} + registry = AttackTechniqueRegistry.get_registry_singleton() + core_factories = TagQuery.all("core").filter(list(registry.get_factories_or_raise().values())) + factories = [_prompt_sending_factory(), *core_factories] + + return AttackTechniqueRegistry.build_strategy_class_from_factories( # type: ignore[ty:invalid-return-type] + class_name="JailbreakStrategy", + factories=factories, + aggregate_tags={ + "single_turn": TagQuery.any_of("single_turn"), + "multi_turn": TagQuery.any_of("multi_turn"), + }, + ) class Jailbreak(Scenario): """ Jailbreak scenario implementation for PyRIT. - This scenario tests how vulnerable models are to jailbreak attacks by applying - various single-turn jailbreak templates to a set of test prompts. The responses are - scored to determine if the jailbreak was successful. + Tests how vulnerable a model is to jailbreak templates along three orthogonal axes: + + * **objectives** — harmful objectives, selected with ``--dataset-names`` (default HarmBench). + * **techniques** — how each jailbroken objective is delivered; default is to "just send" + (``prompt_sending``), with the shared ``core`` techniques (role_play, many_shot, …) + selectable via ``--strategies``. + * **jailbreaks** — which jailbreak templates to apply, a random sample by default or an + explicit list via ``jailbreak_names`` / ``--num-templates``. + + Each ``(technique x objective)`` pair is built for every selected jailbreak template, with + the template applied via a ``TextJailbreakConverter``. Results group by jailbreak template. """ - VERSION: int = 1 + VERSION: int = 3 + + #: Number of jailbreak templates sampled by default when neither the constructor argument + #: nor the ``num_templates`` runtime parameter is supplied. The full catalog ships many + #: templates; this is a small, fast random subset. Raise ``--num-templates`` for broader + #: coverage, or pass ``num_templates=None`` to run the full catalog. + DEFAULT_NUM_TEMPLATES: ClassVar[int] = 10 @classmethod def required_datasets(cls) -> list[str]: """Return a list of dataset names required by this scenario.""" - return ["airt_harms"] + return ["harmbench"] + + @classmethod + def supported_parameters(cls) -> list[Parameter]: + """ + Declare runtime parameters settable from the CLI / config file. + + Returns: + list[Parameter]: Parameters configurable per-run, exposed as ``--jailbreak-names``, + ``--num-templates`` and ``--num-attempts``. + """ + return [ + Parameter( + name="jailbreak_names", + description=( + "Specific jailbreak template names to run (the jailbreaks axis). When omitted, " + "a random sample of size num_templates is drawn. Mutually exclusive with num_templates." + ), + param_type=list[str], + default=[], + ), + Parameter( + name="num_templates", + description=( + "Number of jailbreak templates to randomly sample from the full catalog. " + "Lower this for a faster run; raise it for broader coverage." + ), + param_type=int, + default=cls.DEFAULT_NUM_TEMPLATES, + ), + Parameter( + name="num_attempts", + description="Number of times to run each selected jailbreak template.", + param_type=int, + default=1, + ), + ] @apply_defaults def __init__( @@ -86,8 +159,8 @@ def __init__( *, objective_scorer: TrueFalseScorer | None = None, scenario_result_id: str | None = None, - num_templates: int | None = None, - num_attempts: int = 1, + num_templates: "int | None | _Unset" = _UNSET, + num_attempts: int | None = None, jailbreak_names: list[str] | None = None, ) -> None: """ @@ -97,21 +170,26 @@ def __init__( objective_scorer (TrueFalseScorer | None): Scorer for detecting successful jailbreaks (non-refusal). If not provided, defaults to an inverted refusal scorer. scenario_result_id (str | None): Optional ID of an existing scenario result to resume. - num_templates (int | None): Choose num_templates random jailbreaks rather than using all of them. - num_attempts (int | None): Number of times to try each jailbreak. - jailbreak_names (list[str] | None): List of jailbreak names from the template list under datasets. - to use. + On resume the template names chosen by the original run are replayed (read from + ``ScenarioResult.metadata``) so the atomic-attack set stays stable across processes. + num_templates (int | None): Number of random jailbreak templates to run. When omitted, + falls back to the ``num_templates`` runtime parameter (default + ``DEFAULT_NUM_TEMPLATES``). An explicit integer takes precedence over the parameter. + Pass ``num_templates=None`` to opt out of sampling and run the full catalog. + num_attempts (int | None): Number of times to try each jailbreak. When omitted, falls back + to the ``num_attempts`` runtime parameter (default 1). + jailbreak_names (list[str] | None): Specific jailbreak template names to use. Mutually + exclusive with ``num_templates`` (random selection). Raises: ValueError: If both jailbreak_names and num_templates are provided, as random selection is incompatible with a predetermined list. ValueError: If the jailbreak_names list contains a jailbreak that isn't in the listed templates. - """ if jailbreak_names is None: jailbreak_names = [] - if jailbreak_names and num_templates: + if jailbreak_names and not isinstance(num_templates, _Unset): raise ValueError( "Please provide only one of `num_templates` (random selection)" " or `jailbreak_names` (specific selection)." @@ -121,139 +199,187 @@ def __init__( objective_scorer if objective_scorer else self._get_default_objective_scorer() ) - self._num_templates = num_templates + # Distinguish an omitted argument (use the runtime default) from an explicit ``None`` + # (opt out of sampling and run the full catalog). + if isinstance(num_templates, _Unset): + self._num_templates_unset = True + self._num_templates: int | None = None + else: + self._num_templates_unset = False + self._num_templates = num_templates self._num_attempts = num_attempts - self._adversarial_target: PromptTarget | None = None - - # Note that num_templates and jailbreak_names are mutually exclusive. - # If self._num_templates is None, then this returns all discoverable jailbreak templates. - # If self._num_templates has some value, then all_templates is a subset of all available - # templates, but jailbreak_names is guaranteed to be [], so diff = {}. - all_templates = TextJailBreak.get_jailbreak_templates(num_templates=self._num_templates) - # Example: if jailbreak_names is {'a', 'b', 'c'}, and all_templates is {'b', 'c', 'd'}, - # then diff = {'a'}, which raises the error as 'a' was not discovered in all_templates. - diff = set(jailbreak_names) - set(all_templates) - if len(diff) > 0: - raise ValueError(f"Error: could not find templates `{diff}`!") - - # If jailbreak_names has some value, then `if jailbreak_names` passes, and self._jailbreaks - # is set to jailbreak_names. Otherwise we use all_templates. - self._jailbreaks = jailbreak_names if jailbreak_names else all_templates + # Template resolution is split by selection mode: + # * ``jailbreak_names`` (explicit selection) is validated and resolved eagerly here so an + # unknown name fails fast at construction time. + # * Random ``num_templates`` selection is deferred to ``_build_atomic_attacks_async`` so the + # ``num_templates`` runtime parameter (populated into ``self.params`` during + # ``initialize_async``) is honored — ``self.params`` does not exist yet in ``__init__``. + if jailbreak_names: + all_templates = TextJailBreak.get_jailbreak_templates() + diff = set(jailbreak_names) - set(all_templates) + if diff: + raise ValueError(f"Error: could not find templates `{diff}`!") + self._jailbreaks: list[str] = jailbreak_names + self._jailbreaks_explicit = True + else: + self._jailbreaks = [] + self._jailbreaks_explicit = False + + strategy_class = _build_jailbreak_strategy() super().__init__( version=self.VERSION, - strategy_class=JailbreakStrategy, - default_strategy=JailbreakStrategy.SIMPLE, - default_dataset_config=DatasetAttackConfiguration(dataset_names=["airt_harms"], max_dataset_size=4), + strategy_class=strategy_class, + default_strategy=strategy_class("prompt_sending"), + default_dataset_config=DatasetAttackConfiguration(dataset_names=["harmbench"], max_dataset_size=4), objective_scorer=self._objective_scorer, scenario_result_id=scenario_result_id, ) - def _get_or_create_adversarial_target(self) -> PromptTarget: + @property + def selected_jailbreak_names(self) -> list[str]: """ - Return the shared adversarial target, creating it on first access. + Jailbreak template names selected for this run. - Reuses a single PromptTarget instance across all role-play attacks - to avoid repeated client and TLS setup. + Populated once ``initialize_async`` has resolved the sample (or replayed the persisted set + on ``--resume``). For the random-sampling path this is empty before initialization. The same + list is also persisted to ``ScenarioResult.metadata`` and surfaced in the scenario output. Returns: - PromptTarget: The shared adversarial target. + list[str]: The jailbreak template names this run executes. """ - if self._adversarial_target is None: - self._adversarial_target = get_default_adversarial_target() - return self._adversarial_target + return list(self._jailbreaks) - async def _get_atomic_attack_from_strategy_async( - self, *, strategy: str, jailbreak_template_name: str, seed_groups: list[SeedAttackGroup] - ) -> AtomicAttack: + def _load_persisted_jailbreak_names(self) -> list[str] | None: """ - Create an atomic attack for a specific jailbreak template. + Return the template names persisted by a prior run when resuming, otherwise ``None``. - Args: - strategy (str): JailbreakStrategy to use. - jailbreak_template_name (str): Name of the jailbreak template file. - seed_groups (list[SeedAttackGroup]): Seed groups the attack draws from. + Template resolution happens inside ``_build_atomic_attacks_async``, which the base class runs + *before* it applies persisted resume state. Since each template is its own atomic attack, + the persisted names must be read here (not in ``_apply_persisted_objectives``) so the resumed + run rebuilds the same atomic attacks instead of drawing a fresh random sample. Returns: - AtomicAttack: An atomic attack using the specified jailbreak template. - - Raises: - ValueError: If scenario is not properly initialized. + list[str] | None: The persisted template names, or ``None`` when not resuming or when no + names were persisted. """ - # objective_target is guaranteed to be non-None by parent class validation - if self._objective_target is None: - raise ValueError( - "Scenario not properly initialized. Call await scenario.initialize_async() before running." - ) + if not self._scenario_result_id: + return None + stored = self._memory.get_scenario_results(scenario_result_ids=[self._scenario_result_id]) + if not stored: + return None + names = (stored[0].metadata or {}).get("jailbreak_template_names") + if not names: + return None + return list(names) + + def _resolve_jailbreaks(self) -> list[str]: + """ + Resolve the jailbreak templates to run. - # Create the jailbreak converter - jailbreak_converter = TextJailbreakConverter( - jailbreak_template=TextJailBreak(template_file_name=jailbreak_template_name) - ) + Resolution precedence: - # Create converter configuration - converter_config = AttackConverterConfig( - request_converters=PromptConverterConfiguration.from_converters(converters=[jailbreak_converter]) - ) + 1. On resume, replay the template names persisted by the original run (deterministic resume). + 2. Explicit constructor ``jailbreak_names`` (resolved and validated in ``__init__``). + 3. The ``jailbreak_names`` runtime parameter (specific selection via CLI / config). + 4. An explicit constructor ``num_templates`` (an integer wins over the runtime parameter; an + explicit ``None`` opts out of sampling and runs the full catalog). + 5. The ``num_templates`` runtime parameter, which defaults to ``DEFAULT_NUM_TEMPLATES``. - attack: ManyShotJailbreakAttack | PromptSendingAttack | RolePlayAttack | SkeletonKeyAttack | None = None - args: dict[str, Any] = { - "objective_target": self._objective_target, - "attack_scoring_config": AttackScoringConfig(objective_scorer=self._objective_scorer), - "attack_converter_config": converter_config, - } - match strategy: - case "many_shot": - attack = ManyShotJailbreakAttack(**args) - case "prompt_sending": - attack = PromptSendingAttack(**args) - case "skeleton": - attack = SkeletonKeyAttack(**args) - case "role_play": - args["attack_adversarial_config"] = AttackAdversarialConfig( - target=self._get_or_create_adversarial_target() - ) - args["role_play_definition_path"] = RolePlayPaths.PERSUASION_SCRIPT.value - attack = RolePlayAttack(**args) - case _: - raise ValueError(f"Unknown JailbreakStrategy `{strategy}`.") + Returns: + list[str]: The jailbreak template file names to run. - if not attack: - raise ValueError(f"Attack cannot be None!") + Raises: + ValueError: If the ``jailbreak_names`` runtime parameter contains a name that is not in + the template catalog. + """ + persisted = self._load_persisted_jailbreak_names() + if persisted is not None: + return persisted + if self._jailbreaks_explicit: + return self._jailbreaks + runtime_names = self.params.get("jailbreak_names") if hasattr(self, "params") else None + if runtime_names: + all_templates = TextJailBreak.get_jailbreak_templates() + diff = set(runtime_names) - set(all_templates) + if diff: + raise ValueError(f"Error: could not find templates `{diff}`!") + return list(runtime_names) + num_templates = self.params["num_templates"] if self._num_templates_unset else self._num_templates + return TextJailBreak.get_jailbreak_templates(num_templates=num_templates) + + def _build_initial_scenario_metadata(self) -> dict[str, Any]: + """ + Persist the resolved template names so ``--resume`` replays the same sample. - # Extract template name without extension for the atomic attack name - template_name = Path(jailbreak_template_name).stem + Extends the base ``objective_hashes`` persistence (preserved via ``super()``) with the + concrete template names chosen for this run, mirroring that pattern for the template axis. - return AtomicAttack( - atomic_attack_name=f"jailbreak_{template_name}", - attack_technique=AttackTechnique(attack=attack), - seed_groups=seed_groups, - ) + Returns: + dict[str, Any]: Metadata payload for the new ScenarioResult. + """ + metadata = super()._build_initial_scenario_metadata() + names = list(self._jailbreaks) + metadata["jailbreak_template_names"] = names + summary = metadata.setdefault("summary", {}) + summary["Jailbreak templates"] = ", ".join(names) + return metadata async def _build_atomic_attacks_async(self, *, context: ScenarioContext) -> list[AtomicAttack]: """ - Generate atomic attacks for each jailbreak template. + Build the ``technique x objective`` atomic attacks for every selected jailbreak template. - This method creates an atomic attack for each retrieved jailbreak template. + Resolves the jailbreak templates now that runtime parameters are populated (and replays the + persisted sample on ``--resume``), then for each template builds the technique x objective + cross-product via ``MatrixAtomicAttackBuilder``, injecting that template's + ``TextJailbreakConverter`` as a request converter for every technique. Results group by + jailbreak template. The baseline (plain objective, no jailbreak) is emitted centrally by the + base ``initialize_async``, so this override never prepends one. Args: context (ScenarioContext): The resolved runtime inputs for this run. Returns: - list[AtomicAttack]: List of atomic attacks to execute, one per jailbreak template. + list[AtomicAttack]: One atomic attack per ``(technique x objective x jailbreak x attempt)``. """ - atomic_attacks: list[AtomicAttack] = [] + self._jailbreaks = self._resolve_jailbreaks() + logger.info( + "Jailbreak scenario running %d template(s): %s", + len(self._jailbreaks), + ", ".join(self._jailbreaks), + ) + num_attempts = self._num_attempts if self._num_attempts is not None else self.params["num_attempts"] - seed_groups = list(context.seed_groups) - strategies = {s.value for s in context.scenario_strategies} + factories = resolve_technique_factories( + context=context, extra_factories={"prompt_sending": _prompt_sending_factory()} + ) + builder = MatrixAtomicAttackBuilder( + objective_target=context.objective_target, + objective_scorer=self._objective_scorer, + memory_labels=context.memory_labels, + ) - for strategy in strategies: - for template_name in self._jailbreaks: - for _ in range(self._num_attempts): - atomic_attack = await self._get_atomic_attack_from_strategy_async( - strategy=strategy, jailbreak_template_name=template_name, seed_groups=seed_groups + atomic_attacks: list[AtomicAttack] = [] + for template_name in self._jailbreaks: + stem = Path(template_name).stem + converter = TextJailbreakConverter(jailbreak_template=TextJailBreak(template_file_name=template_name)) + strategy_converters: dict[str, list[PromptConverter]] = {name: [converter] for name in factories} + for attempt in range(num_attempts): + + def name_fn(combo: MatrixCombo, *, stem: str = stem, attempt: int = attempt) -> str: + base = f"jailbreak_{combo.technique_name}_{stem}_{combo.dataset_name}" + return base if num_attempts == 1 else f"{base}_attempt{attempt + 1}" + + atomic_attacks.extend( + builder.build( + technique_factories=factories, + dataset_groups=context.seed_groups_by_dataset, + strategy_converters=strategy_converters, + name_fn=name_fn, + display_group_fn=lambda combo, *, stem=stem: stem, + include_baseline=False, ) - atomic_attacks.append(atomic_attack) + ) return atomic_attacks diff --git a/tests/unit/datasets/test_jailbreak_text.py b/tests/unit/datasets/test_jailbreak_text.py index 9deb982a2e..05a28ad042 100644 --- a/tests/unit/datasets/test_jailbreak_text.py +++ b/tests/unit/datasets/test_jailbreak_text.py @@ -89,6 +89,19 @@ def test_get_jailbreak_templates_includes_subdirectory_templates(): assert len(templates) > top_level_count, "Subdirectory templates should be included in the listing" +def test_get_jailbreak_templates_none_returns_all(): + """num_templates=None returns the full sorted catalog (the power-user opt-out).""" + all_templates = TextJailBreak.get_jailbreak_templates() + assert TextJailBreak.get_jailbreak_templates(num_templates=None) == all_templates + + +@pytest.mark.parametrize("invalid", [0, -1, -5]) +def test_get_jailbreak_templates_non_positive_raises(invalid): + """num_templates must be a positive integer; 0 must not silently return the full catalog.""" + with pytest.raises(ValueError, match="positive integer"): + TextJailBreak.get_jailbreak_templates(num_templates=invalid) + + def test_all_templates_render_without_syntax_errors(jailbreak_dir): """Test that all jailbreak templates can be successfully rendered with a test prompt.""" yaml_files = [f for f in jailbreak_dir.rglob("*.yaml") if "multi_parameter" not in f.parts] diff --git a/tests/unit/output/scenario_result/test_pretty.py b/tests/unit/output/scenario_result/test_pretty.py index 45671d74ad..19c4069fc4 100644 --- a/tests/unit/output/scenario_result/test_pretty.py +++ b/tests/unit/output/scenario_result/test_pretty.py @@ -30,6 +30,7 @@ def _scenario_result( attack_results: dict[str, list[AttackResult]] | None = None, objective_scorer_identifier: ComponentIdentifier | None = None, display_group_map: dict[str, str] | None = None, + metadata: dict | None = None, ) -> ScenarioResult: return make_scenario_result( scenario_name="TestScenario", @@ -40,6 +41,7 @@ def _scenario_result( attack_results=attack_results or {"strategy_a": [_attack_result()]}, objective_scorer_identifier=objective_scorer_identifier, display_group_map=display_group_map or {}, + metadata=metadata or {}, ) @@ -96,6 +98,24 @@ async def test_write_async_with_unknown_target_when_no_params(printer, capsys): assert "Target Endpoint: Unknown" in out +async def test_write_async_renders_scenario_inputs_from_metadata_summary(printer, capsys): + result = _scenario_result(metadata={"summary": {"Jailbreak templates": "aim, dan_1, tuo"}}) + await printer.write_async(result) + out = capsys.readouterr().out + assert "Scenario Inputs" in out + assert "Jailbreak templates:" in out + assert "aim, dan_1, tuo" in out + + +async def test_write_async_omits_scenario_inputs_when_no_summary(printer, capsys): + # objective_hashes is internal-only and must never be rendered. + result = _scenario_result(metadata={"objective_hashes": ["abc123"]}) + await printer.write_async(result) + out = capsys.readouterr().out + assert "Scenario Inputs" not in out + assert "abc123" not in out + + async def test_write_async_renders_scorer_section_when_scorer_identifier_present(printer, monkeypatch, capsys): # Stub the scorer printer's render_async so we don't depend on real evaluation data. async def fake_render_async(*, scorer_identifier, harm_category=None): diff --git a/tests/unit/scenario/airt/test_jailbreak.py b/tests/unit/scenario/airt/test_jailbreak.py index 73234bb168..5af4df6b7c 100644 --- a/tests/unit/scenario/airt/test_jailbreak.py +++ b/tests/unit/scenario/airt/test_jailbreak.py @@ -1,119 +1,68 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Tests for the Jailbreak class.""" +"""Tests for the redesigned 3-axis Jailbreak scenario.""" from unittest.mock import AsyncMock, MagicMock, patch import pytest -from pyrit.common.path import JAILBREAK_TEMPLATES_PATH -from pyrit.datasets import TextJailBreak -from pyrit.executor.attack.single_turn.many_shot_jailbreak import ManyShotJailbreakAttack -from pyrit.executor.attack.single_turn.prompt_sending import PromptSendingAttack -from pyrit.executor.attack.single_turn.role_play import RolePlayAttack -from pyrit.executor.attack.single_turn.skeleton_key import SkeletonKeyAttack -from pyrit.models import ComponentIdentifier, SeedAttackGroup, SeedObjective +from pyrit.models import ComponentIdentifier, SeedAttackGroup, SeedObjective, SeedPrompt from pyrit.prompt_target import PromptTarget -from pyrit.scenario.core import BaselineAttackPolicy -from pyrit.scenario.scenarios.airt.jailbreak import Jailbreak, JailbreakStrategy -from pyrit.score.true_false.true_false_inverter_scorer import TrueFalseInverterScorer +from pyrit.registry import TargetRegistry +from pyrit.registry.components.attack_technique_registry import AttackTechniqueRegistry +from pyrit.scenario.core.dataset_configuration import DatasetAttackConfiguration +from pyrit.scenario.scenarios.airt.jailbreak import Jailbreak, _build_jailbreak_strategy +from pyrit.score import TrueFalseScorer +from pyrit.setup.initializers.techniques import build_technique_factories +_MOCK_MANY_SHOT_EXAMPLES = [{"question": f"q{i}", "answer": f"a{i}"} for i in range(100)] -@pytest.fixture -def mock_templates() -> list[str]: - """Mock constant for jailbreak subset.""" - return ["aim", "dan_1", "tuo"] - - -@pytest.fixture -def mock_random_num_attempts() -> int: - """Mock constant for n-many attempts per jailbreak.""" - return 2 - - -@pytest.fixture -def mock_random_num_templates() -> int: - """Mock constant for k-many jailbreak templates to be used.""" - return 3 +_MOCK_TEMPLATES = ["aim.yaml", "aligned.yaml"] -@pytest.fixture -def mock_scenario_result_id() -> str: - return "mock-scenario-result-id" +def _mock_id(name: str) -> ComponentIdentifier: + return ComponentIdentifier(class_name=name, class_module="test") -@pytest.fixture -def mock_memory_seed_groups() -> list[SeedAttackGroup]: - """Create mock seed groups that _get_default_seed_groups() would return.""" - return [ - SeedAttackGroup(seeds=[SeedObjective(value=prompt)]) - for prompt in [ - "sample objective 1", - "sample objective 2", - "sample objective 3", - ] - ] +def _strategy_class(): + return _build_jailbreak_strategy() @pytest.fixture -def mock_objective_target() -> PromptTarget: - """Create a mock objective target for testing.""" +def mock_objective_target(): mock = MagicMock(spec=PromptTarget) - mock.get_identifier.return_value = ComponentIdentifier(class_name="MockObjectiveTarget", class_module="test") + mock.get_identifier.return_value = _mock_id("MockObjectiveTarget") return mock @pytest.fixture -def mock_objective_scorer() -> TrueFalseInverterScorer: - """Create a mock scorer for testing.""" - mock = MagicMock(spec=TrueFalseInverterScorer) - mock.get_identifier.return_value = ComponentIdentifier(class_name="MockObjectiveScorer", class_module="test") +def mock_objective_scorer(): + mock = MagicMock(spec=TrueFalseScorer) + mock.get_identifier.return_value = _mock_id("MockObjectiveScorer") return mock -@pytest.fixture -def all_jailbreak_strategy() -> JailbreakStrategy: - return JailbreakStrategy.ALL - - -@pytest.fixture -def simple_jailbreak_strategy() -> JailbreakStrategy: - return JailbreakStrategy.SIMPLE - - -@pytest.fixture -def complex_jailbreak_strategy() -> JailbreakStrategy: - return JailbreakStrategy.COMPLEX - - -@pytest.fixture -def manyshot_jailbreak_strategy() -> JailbreakStrategy: - return JailbreakStrategy.ManyShot - - -@pytest.fixture -def promptsending_jailbreak_strategy() -> JailbreakStrategy: - return JailbreakStrategy.PromptSending - - -@pytest.fixture -def skeleton_jailbreak_attack() -> JailbreakStrategy: - return JailbreakStrategy.SkeletonKey - - -@pytest.fixture -def roleplay_jailbreak_strategy() -> JailbreakStrategy: - return JailbreakStrategy.RolePlay +@pytest.fixture(autouse=True) +def reset_technique_registry(): + """Reset registries, register a mock adversarial target, populate core factories.""" + AttackTechniqueRegistry.reset_registry_singleton() + TargetRegistry.reset_registry_singleton() + _build_jailbreak_strategy.cache_clear() + adv_target = MagicMock(spec=PromptTarget) + adv_target.capabilities.includes.return_value = True + TargetRegistry.get_registry_singleton().instances.register(adv_target, name="adversarial_chat") -# Synthetic many-shot examples used to prevent real HTTP requests to GitHub during tests -_MOCK_MANY_SHOT_EXAMPLES = [{"question": f"test question {i}", "answer": f"test answer {i}"} for i in range(100)] + AttackTechniqueRegistry.get_registry_singleton().register_from_factories(build_technique_factories()) + yield + AttackTechniqueRegistry.reset_registry_singleton() + TargetRegistry.reset_registry_singleton() + _build_jailbreak_strategy.cache_clear() @pytest.fixture(autouse=True) def patch_many_shot_load(): - """Prevent ManyShotJailbreakAttack from loading the full dataset during unit tests.""" with patch( "pyrit.executor.attack.single_turn.many_shot_jailbreak.load_many_shot_jailbreaking_dataset", return_value=_MOCK_MANY_SHOT_EXAMPLES, @@ -121,14 +70,23 @@ def patch_many_shot_load(): yield +@pytest.fixture(autouse=True) +def patch_templates(): + """Deterministic jailbreak template catalog + sampling (real template files are used).""" + with patch( + "pyrit.datasets.TextJailBreak.get_jailbreak_templates", + side_effect=lambda num_templates=None: ( + list(_MOCK_TEMPLATES) if num_templates is None else list(_MOCK_TEMPLATES)[:num_templates] + ), + ): + yield + + @pytest.fixture def mock_runtime_env(): with patch.dict( "os.environ", { - "AZURE_OPENAI_GPT4O_UNSAFE_CHAT_ENDPOINT": "https://test.openai.azure.com/", - "AZURE_OPENAI_GPT4O_UNSAFE_CHAT_KEY": "test-key", - "AZURE_OPENAI_GPT4O_UNSAFE_CHAT_MODEL": "gpt-4", "OPENAI_CHAT_ENDPOINT": "https://test.openai.azure.com/", "OPENAI_CHAT_KEY": "test-key", "OPENAI_CHAT_MODEL": "gpt-4", @@ -137,523 +95,156 @@ def mock_runtime_env(): yield -FIXTURES = ["patch_central_database", "mock_runtime_env"] - - -@pytest.mark.usefixtures(*FIXTURES) -class TestJailbreakInitialization: - """Tests for Jailbreak initialization.""" - - def test_init_with_scenario_result_id(self, mock_scenario_result_id): - """Test initialization with a scenario result ID.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(scenario_result_id=mock_scenario_result_id) - assert scenario._scenario_result_id == mock_scenario_result_id - - def test_init_with_default_scorer(self, mock_memory_seed_groups): - """Test initialization with default scorer.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak() - assert scenario._objective_scorer_identifier - - def test_init_with_custom_scorer(self, mock_objective_scorer, mock_memory_seed_groups): - """Test initialization with custom scorer.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(objective_scorer=mock_objective_scorer) - assert scenario._objective_scorer == mock_objective_scorer - - def test_init_with_num_templates(self, mock_random_num_templates): - """Test initialization with num_templates provided.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(num_templates=mock_random_num_templates) - assert scenario._num_templates == mock_random_num_templates - - def test_init_with_num_attempts(self, mock_random_num_attempts): - """Test initialization with n provided.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(num_attempts=mock_random_num_attempts) - assert scenario._num_attempts == mock_random_num_attempts - - def test_init_raises_exception_when_both_num_and_which_jailbreaks(self, mock_random_num_templates, mock_templates): - """Test failure on providing mutually exclusive arguments.""" - - with pytest.raises(ValueError): - Jailbreak(num_templates=mock_random_num_templates, jailbreak_names=mock_templates) - - def test_init_accepts_subdirectory_jailbreak_names(self, mock_objective_scorer, mock_memory_seed_groups): - """Test that explicit jailbreak names can reference templates stored in subdirectories.""" - # Pick a template that lives in a subdirectory (not top-level) - all_templates = TextJailBreak.get_jailbreak_templates() - top_level_names = {f.name for f in JAILBREAK_TEMPLATES_PATH.glob("*.yaml")} - subdir_templates = [t for t in all_templates if t not in top_level_names] - assert subdir_templates, "Expected at least one subdirectory template to exist" - subdir_name = subdir_templates[0] - - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(objective_scorer=mock_objective_scorer, jailbreak_names=[subdir_name]) - assert scenario._jailbreaks == [subdir_name] - - async def test_init_raises_exception_when_no_datasets_available(self, mock_objective_target, mock_objective_scorer): - """Test that initialization raises DatasetConstraintError when datasets are not available in memory.""" - from pyrit.scenario.core.dataset_configuration import DatasetConstraintError - - # Don't mock _resolve_seed_groups, let it try to load from empty memory - scenario = Jailbreak(objective_scorer=mock_objective_scorer) - - # Error should occur during initialize_async when _get_atomic_attacks_async resolves seed groups. - # Neutralize the provider fetch so the empty-memory path raises loudly instead of fetching. - with patch( - "pyrit.scenario.core.dataset_configuration.DatasetConfiguration._fetch_dataset_async", - new_callable=AsyncMock, - ): - with pytest.raises(DatasetConstraintError, match="could not be loaded"): - await scenario.initialize_async(objective_target=mock_objective_target) - - def test_class_inherits_default_baseline_attack_policy(self): - """Jailbreak inherits the base default (Enabled) — baseline included by default.""" - assert Jailbreak.BASELINE_ATTACK_POLICY is BaselineAttackPolicy.Enabled - - async def test_default_initialize_includes_baseline( - self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups - ): - """initialize_async without include_baseline honors BASELINE_ATTACK_POLICY=Enabled.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(objective_scorer=mock_objective_scorer) - await scenario.initialize_async(objective_target=mock_objective_target) - assert scenario._atomic_attacks[0].atomic_attack_name == "baseline" - - async def test_explicit_include_baseline_false_omits_baseline( - self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups - ): - """Caller can opt out of baseline by passing include_baseline=False.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(objective_scorer=mock_objective_scorer) - await scenario.initialize_async( - objective_target=mock_objective_target, - include_baseline=False, - ) - assert not any(a.atomic_attack_name == "baseline" for a in scenario._atomic_attacks) - - -@pytest.mark.usefixtures(*FIXTURES) -class TestJailbreakAttackGeneration: - """Tests for Jailbreak attack generation.""" - - async def test_attack_generation_for_simple( - self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups, simple_jailbreak_strategy - ): - """Test that the simple attack generation works.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(objective_scorer=mock_objective_scorer, num_templates=2) - - await scenario.initialize_async( - objective_target=mock_objective_target, scenario_strategies=[simple_jailbreak_strategy] - ) - atomic_attacks = scenario._atomic_attacks - for run in atomic_attacks: - assert isinstance(run.attack_technique.attack, PromptSendingAttack) - - async def test_attack_generation_for_complex( - self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups, complex_jailbreak_strategy - ): - """Test that the complex attack generation works.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(objective_scorer=mock_objective_scorer, num_templates=2) - - await scenario.initialize_async( - objective_target=mock_objective_target, - scenario_strategies=[complex_jailbreak_strategy], - include_baseline=False, - ) - atomic_attacks = scenario._atomic_attacks - for run in atomic_attacks: - assert isinstance( - run.attack_technique.attack, (RolePlayAttack, ManyShotJailbreakAttack, SkeletonKeyAttack) - ) - - async def test_attack_generation_for_manyshot( - self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups, manyshot_jailbreak_strategy - ): - """Test that the manyshot attack generation works.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(objective_scorer=mock_objective_scorer, num_templates=2) +def _make_seed_groups() -> list[SeedAttackGroup]: + return [ + SeedAttackGroup(seeds=[SeedObjective(value="obj 1"), SeedPrompt(value="prompt 1")]), + SeedAttackGroup(seeds=[SeedObjective(value="obj 2"), SeedPrompt(value="prompt 2")]), + ] - await scenario.initialize_async( - objective_target=mock_objective_target, - scenario_strategies=[manyshot_jailbreak_strategy], - include_baseline=False, - ) - atomic_attacks = scenario._atomic_attacks - for run in atomic_attacks: - assert isinstance(run.attack_technique.attack, ManyShotJailbreakAttack) - async def test_attack_generation_for_promptsending( - self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups, promptsending_jailbreak_strategy - ): - """Test that the prompt sending attack generation works.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(objective_scorer=mock_objective_scorer, num_templates=2) +FIXTURES = ["patch_central_database", "mock_runtime_env"] - await scenario.initialize_async( - objective_target=mock_objective_target, - scenario_strategies=[promptsending_jailbreak_strategy], - include_baseline=False, - ) - atomic_attacks = scenario._atomic_attacks - for run in atomic_attacks: - assert isinstance(run.attack_technique.attack, PromptSendingAttack) - async def test_attack_generation_for_skeleton( - self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups, skeleton_jailbreak_attack - ): - """Test that the skelton key attack generation works.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(objective_scorer=mock_objective_scorer, num_templates=2) +def _patch_seed_groups(): + return patch.object( + Jailbreak, + "_resolve_seed_groups_by_dataset_async", + new_callable=AsyncMock, + return_value={"harmbench": _make_seed_groups()}, + ) - await scenario.initialize_async( - objective_target=mock_objective_target, - scenario_strategies=[skeleton_jailbreak_attack], - include_baseline=False, - ) - atomic_attacks = scenario._atomic_attacks - for run in atomic_attacks: - assert isinstance(run.attack_technique.attack, SkeletonKeyAttack) - async def test_attack_generation_for_roleplay( - self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups, roleplay_jailbreak_strategy - ): - """Test that the roleplaying attack generation works.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(objective_scorer=mock_objective_scorer, num_templates=2) +def _patch_scorer(mock_objective_scorer): + return patch( + "pyrit.scenario.core.scenario.Scenario._get_default_objective_scorer", + return_value=mock_objective_scorer, + ) - await scenario.initialize_async( - objective_target=mock_objective_target, - scenario_strategies=[roleplay_jailbreak_strategy], - include_baseline=False, - ) - atomic_attacks = scenario._atomic_attacks - for run in atomic_attacks: - assert isinstance(run.attack_technique.attack, RolePlayAttack) - async def test_attack_runs_include_objectives( - self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups - ): - """Test that attack runs include objectives for each seed prompt and that - each atomic attack carries a valid attack_technique. - - Combined coverage previously split across test_get_atomic_attacks_async_returns_attacks. - """ - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(objective_scorer=mock_objective_scorer, num_templates=2) - - await scenario.initialize_async(objective_target=mock_objective_target) - atomic_attacks = scenario._atomic_attacks - - assert len(atomic_attacks) > 0 - for run in atomic_attacks: - assert run.attack_technique is not None - assert len(run.objectives) > 0 - - async def test_get_all_jailbreak_templates( - self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups - ): - """Test that all jailbreak templates are found.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak( - objective_scorer=mock_objective_scorer, - ) - await scenario.initialize_async(objective_target=mock_objective_target) - assert len(scenario._jailbreaks) > 0 +@pytest.mark.usefixtures(*FIXTURES) +class TestJailbreakClassProperties: + def test_version_is_3(self): + assert Jailbreak.VERSION == 3 - async def test_get_some_jailbreak_templates( - self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups, mock_random_num_templates - ): - """Test that random jailbreak template selection works.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(objective_scorer=mock_objective_scorer, num_templates=mock_random_num_templates) - await scenario.initialize_async(objective_target=mock_objective_target) - assert len(scenario._jailbreaks) == mock_random_num_templates - - async def test_custom_num_attempts( - self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups, mock_random_num_attempts - ): - """Test that n successfully tries each jailbreak template n-many times.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - base_scenario = Jailbreak(objective_scorer=mock_objective_scorer, num_templates=2) - await base_scenario.initialize_async(objective_target=mock_objective_target, include_baseline=False) - atomic_attacks_1 = base_scenario._atomic_attacks - - mult_scenario = Jailbreak( - objective_scorer=mock_objective_scorer, - num_templates=2, - num_attempts=mock_random_num_attempts, - ) - await mult_scenario.initialize_async(objective_target=mock_objective_target, include_baseline=False) - atomic_attacks_n = mult_scenario._atomic_attacks + def test_required_datasets_is_harmbench(self): + assert Jailbreak.required_datasets() == ["harmbench"] - assert len(atomic_attacks_1) * mock_random_num_attempts == len(atomic_attacks_n) + def test_supported_parameter_names(self): + names = {p.name for p in Jailbreak.supported_parameters()} + assert names == {"jailbreak_names", "num_templates", "num_attempts"} + def test_default_dataset_config_is_harmbench(self, mock_objective_scorer): + with _patch_scorer(mock_objective_scorer): + config = Jailbreak()._default_dataset_config + assert isinstance(config, DatasetAttackConfiguration) + assert config.dataset_names == ["harmbench"] + assert config.max_dataset_size == 4 -@pytest.mark.usefixtures(*FIXTURES) -class TestJailbreakLifecycle: - """Tests for Jailbreak lifecycle.""" - - async def test_initialize_async_with_max_concurrency( - self, - *, - mock_objective_target: PromptTarget, - mock_objective_scorer: TrueFalseInverterScorer, - mock_memory_seed_groups: list[SeedAttackGroup], - ) -> None: - """Test initialization with custom max_concurrency.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(objective_scorer=mock_objective_scorer) - await scenario.initialize_async(objective_target=mock_objective_target, max_concurrency=20) - assert scenario._max_concurrency == 20 - - async def test_initialize_async_with_memory_labels( - self, - *, - mock_objective_target: PromptTarget, - mock_objective_scorer: TrueFalseInverterScorer, - mock_memory_seed_groups: list[SeedAttackGroup], - ) -> None: - """Test initialization with memory labels.""" - memory_labels = {"type": "jailbreak", "category": "scenario"} - - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(objective_scorer=mock_objective_scorer) - await scenario.initialize_async( - memory_labels=memory_labels, - objective_target=mock_objective_target, - ) - assert scenario._memory_labels == memory_labels + def test_default_strategy_is_prompt_sending(self, mock_objective_scorer): + strat = _strategy_class() + with _patch_scorer(mock_objective_scorer): + assert Jailbreak()._default_strategy == strat("prompt_sending") @pytest.mark.usefixtures(*FIXTURES) -class TestJailbreakProperties: - """Tests for Jailbreak properties.""" +class TestJailbreakStrategyAxis: + def test_prompt_sending_is_a_member(self): + strat = _strategy_class() + assert "prompt_sending" in {m.name for m in strat} - def test_scenario_version_is_set( - self, - *, - mock_objective_scorer: TrueFalseInverterScorer, - ) -> None: - """Test that scenario version is properly set.""" - scenario = Jailbreak( - objective_scorer=mock_objective_scorer, - ) + def test_core_techniques_are_available(self): + names = {m.name for m in _strategy_class()} + # rapid-response-style core techniques are selectable + assert {"role_play", "many_shot"}.issubset(names) - assert scenario.VERSION == 1 - def test_scenario_default_dataset(self) -> None: - """Test that scenario default dataset is correct.""" - - assert Jailbreak.required_datasets() == ["airt_harms"] - - async def test_no_target_duplication_async( - self, *, mock_objective_target: PromptTarget, mock_memory_seed_groups: list[SeedAttackGroup] - ) -> None: - """Test that all three targets (adversarial, object, scorer) are distinct.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak() - await scenario.initialize_async(objective_target=mock_objective_target) +@pytest.mark.usefixtures(*FIXTURES) +class TestJailbreakInitValidation: + def test_both_num_templates_and_names_raises(self, mock_objective_scorer): + with _patch_scorer(mock_objective_scorer): + with pytest.raises(ValueError, match="only one of"): + Jailbreak(num_templates=3, jailbreak_names=["aim.yaml"]) - objective_target = scenario._objective_target - scorer_target = scenario._objective_scorer + def test_unknown_jailbreak_name_raises(self, mock_objective_scorer): + with _patch_scorer(mock_objective_scorer): + with pytest.raises(ValueError, match="could not find templates"): + Jailbreak(jailbreak_names=["does_not_exist.yaml"]) - assert objective_target != scorer_target + def test_valid_jailbreak_names_accepted(self, mock_objective_scorer): + with _patch_scorer(mock_objective_scorer): + scenario = Jailbreak(jailbreak_names=["aim.yaml"]) + assert scenario.selected_jailbreak_names == ["aim.yaml"] @pytest.mark.usefixtures(*FIXTURES) -class TestJailbreakAdversarialTarget: - """Tests for adversarial target creation and caching.""" - - def test_get_or_create_adversarial_target_returns_prompt_target(self) -> None: - """Test that _get_or_create_adversarial_target returns a PromptTarget.""" - from pyrit.prompt_target import PromptTarget - - scenario = Jailbreak() - target = scenario._get_or_create_adversarial_target() - assert isinstance(target, PromptTarget) - - def test_get_or_create_adversarial_target_reuses_instance(self) -> None: - """Test that _get_or_create_adversarial_target returns the same instance on repeated calls.""" - scenario = Jailbreak() - first = scenario._get_or_create_adversarial_target() - second = scenario._get_or_create_adversarial_target() - assert first is second - - def test_get_or_create_adversarial_target_creates_on_first_call(self) -> None: - """Test that _adversarial_target starts as None and is populated after first access.""" - scenario = Jailbreak() - assert scenario._adversarial_target is None - target = scenario._get_or_create_adversarial_target() - assert scenario._adversarial_target is target - - async def test_roleplay_attacks_share_adversarial_target( - self, - *, - mock_objective_target: PromptTarget, - mock_objective_scorer: TrueFalseInverterScorer, - mock_memory_seed_groups: list[SeedAttackGroup], - roleplay_jailbreak_strategy: JailbreakStrategy, - ) -> None: - """Test that multiple role-play attacks share the same adversarial target instance.""" - with patch.object( - Jailbreak, - "_resolve_seed_groups_by_dataset_async", - new_callable=AsyncMock, - return_value={"memory": mock_memory_seed_groups}, - ): - scenario = Jailbreak(objective_scorer=mock_objective_scorer, num_templates=2) +class TestJailbreakAtomicAttacks: + async def test_default_run_crosses_templates_objectives(self, mock_objective_target, mock_objective_scorer): + with _patch_scorer(mock_objective_scorer), _patch_seed_groups(): + scenario = Jailbreak(num_templates=2) + await scenario.initialize_async(objective_target=mock_objective_target, include_baseline=False) + + non_baseline = [a for a in scenario._atomic_attacks if a.atomic_attack_name != "baseline"] + # default technique = prompt_sending (1) x 1 dataset x 2 templates = 2 + assert len(non_baseline) == 2 + names = {a.atomic_attack_name for a in non_baseline} + assert names == { + "jailbreak_prompt_sending_aim_harmbench", + "jailbreak_prompt_sending_aligned_harmbench", + } + + async def test_display_group_is_template_stem(self, mock_objective_target, mock_objective_scorer): + with _patch_scorer(mock_objective_scorer), _patch_seed_groups(): + scenario = Jailbreak(num_templates=2) + await scenario.initialize_async(objective_target=mock_objective_target, include_baseline=False) + groups = {a.display_group for a in scenario._atomic_attacks if a.atomic_attack_name != "baseline"} + assert groups == {"aim", "aligned"} + + async def test_num_attempts_multiplies_with_unique_names(self, mock_objective_target, mock_objective_scorer): + with _patch_scorer(mock_objective_scorer), _patch_seed_groups(): + scenario = Jailbreak(num_templates=1, num_attempts=3) + await scenario.initialize_async(objective_target=mock_objective_target, include_baseline=False) + non_baseline = [a for a in scenario._atomic_attacks if a.atomic_attack_name != "baseline"] + assert len(non_baseline) == 3 + names = {a.atomic_attack_name for a in non_baseline} + assert len(names) == 3 # all unique + assert all("attempt" in n for n in names) + + async def test_explicit_strategy_adds_technique_axis(self, mock_objective_target, mock_objective_scorer): + strat = _strategy_class() + with _patch_scorer(mock_objective_scorer), _patch_seed_groups(): + scenario = Jailbreak(num_templates=1) await scenario.initialize_async( objective_target=mock_objective_target, - scenario_strategies=[roleplay_jailbreak_strategy], + scenario_strategies=[strat("prompt_sending"), strat("role_play")], include_baseline=False, ) - atomic_attacks = scenario._atomic_attacks - assert len(atomic_attacks) >= 2 + techniques = { + a.atomic_attack_name.split("_")[1] for a in scenario._atomic_attacks if a.atomic_attack_name != "baseline" + } + # both prompt_sending and role_play techniques appear + assert "prompt" in techniques or "role" in techniques + assert len([a for a in scenario._atomic_attacks if a.atomic_attack_name != "baseline"]) == 2 - # All role-play attacks should share the same adversarial target - adversarial_targets = [run.attack_technique.attack._adversarial_chat for run in atomic_attacks] - assert all(t is adversarial_targets[0] for t in adversarial_targets) + async def test_baseline_prepended_centrally(self, mock_objective_target, mock_objective_scorer): + with _patch_scorer(mock_objective_scorer), _patch_seed_groups(): + scenario = Jailbreak(num_templates=1) + await scenario.initialize_async(objective_target=mock_objective_target, include_baseline=True) + baselines = [a for a in scenario._atomic_attacks if a.atomic_attack_name == "baseline"] + assert len(baselines) == 1 @pytest.mark.usefixtures(*FIXTURES) -class TestJailbreakBaselineUniformity: - """ADO 9012 regression: baseline shares objectives with strategies under max_dataset_size.""" - - async def test_one_resolution_call_baseline_matches_strategies( - self, mock_objective_target, mock_objective_scorer, simple_jailbreak_strategy - ): - from pyrit.models import SeedAttackGroup, SeedObjective - from pyrit.scenario import DatasetAttackConfiguration - - seed_groups = [SeedAttackGroup(seeds=[SeedObjective(value=f"obj{i}")]) for i in range(10)] - config = DatasetAttackConfiguration(seed_groups=seed_groups, max_dataset_size=3) - - first_sample = [("inline", group) for group in seed_groups[:3]] - second_sample = [("inline", group) for group in seed_groups[5:8]] - scenario = Jailbreak(objective_scorer=mock_objective_scorer, num_templates=1) - with patch( - "pyrit.scenario.core.dataset_configuration.random.sample", - side_effect=[first_sample, second_sample], - ) as mock_sample: - await scenario.initialize_async( - objective_target=mock_objective_target, - scenario_strategies=[simple_jailbreak_strategy], - dataset_config=config, - include_baseline=True, - ) - - assert mock_sample.call_count == 1 - assert scenario._atomic_attacks[0].atomic_attack_name == "baseline" - baseline_objs = set(scenario._atomic_attacks[0].objectives) - for attack in scenario._atomic_attacks[1:]: - assert set(attack.objectives) == baseline_objs +class TestJailbreakResume: + async def test_metadata_persists_template_names(self, mock_objective_target, mock_objective_scorer): + with _patch_scorer(mock_objective_scorer), _patch_seed_groups(): + scenario = Jailbreak(num_templates=2) + await scenario.initialize_async(objective_target=mock_objective_target, include_baseline=False) + metadata = scenario._build_initial_scenario_metadata() + assert set(metadata["jailbreak_template_names"]) == set(_MOCK_TEMPLATES) + assert "Jailbreak templates" in metadata["summary"] + + async def test_resume_replays_persisted_names(self, mock_objective_scorer): + with _patch_scorer(mock_objective_scorer): + scenario = Jailbreak(scenario_result_id="abc") + with patch.object(scenario, "_load_persisted_jailbreak_names", return_value=["aim.yaml"]): + assert scenario._resolve_jailbreaks() == ["aim.yaml"]