FIX Stop JSON retries for adversarial refusals

pyrit/executor/attack/component/adversarial_conversation_manager.py

@@ -13,8 +13,11 @@
 from uuid import uuid4
 
 from pyrit.exceptions import (
+    BadRequestException,
     ComponentRole,
+    EmptyResponseException,
     InvalidJsonException,
+    PyritException,
     execution_context,
     remove_markdown_json,
 )
@@ -29,6 +32,7 @@
     JsonResponseConfig,
     JsonSchemaDefinition,
     Message,
+    MessagePiece,
     Score,
     SeedPrompt,
     get_common_json_schema,
@@ -138,6 +142,22 @@ def _joined_text_value(message: Message) -> str:
     return "\n".join(piece.converted_value for piece in pieces if piece.converted_value)
 
 
+def _first_error_piece(message: Message) -> MessagePiece | None:
+    """
+    Find the first piece that represents an error.
+
+    Args:
+        message: The message to scan for an errored piece.
+
+    Returns:
+        The first errored piece, or None when no piece represents an error.
+    """
+    for piece in message.message_pieces:
+        if piece.has_error() or piece.converted_value_data_type == "error":
+            return piece
+    return None
+
+
 def _first_response_error(message: Message) -> str:
     """
     Find the response-error code of the first errored piece.
@@ -148,10 +168,67 @@ def _first_response_error(message: Message) -> str:
     Returns:
         The first errored piece's response-error code, or ``"none"`` when no piece errored.
     """
-    for piece in message.message_pieces:
-        if piece.has_error():
-            return piece.response_error
-    return "none"
+    error_piece = _first_error_piece(message)
+    return error_piece.response_error if error_piece else "none"
+
+
+def _get_error_payload(response_value: str) -> tuple[int | None, str]:
+    """
+    Extract a serialized exception's status code and message from a response.
+
+    Args:
+        response_value: The errored response value to inspect.
+
+    Returns:
+        The optional status code and human-readable message. Non-object or non-JSON
+        payloads are returned unchanged as the message.
+    """
+    try:
+        payload = json.loads(response_value)
+    except json.JSONDecodeError:
+        return None, response_value
+
+    if not isinstance(payload, dict):
+        return None, response_value
+
+    raw_status_code = payload.get("status_code")
+    status_code = (
+        raw_status_code if isinstance(raw_status_code, int) and not isinstance(raw_status_code, bool) else None
+    )
+    raw_message = payload.get("message")
+    message = raw_message if isinstance(raw_message, str) else response_value
+    return status_code, message
+
+
+def _raise_for_adversarial_error(response: Message) -> None:
+    """
+    Raise a terminal exception that preserves the adversarial response's error category.
+
+    Args:
+        response: The adversarial-chat response to inspect.
+
+    Raises:
+        BadRequestException: If the response was blocked.
+        EmptyResponseException: If the response was empty.
+        PyritException: If the response carries another error category.
+    """
+    if not response.is_error():
+        return
+
+    error_piece = _first_error_piece(response)
+    if error_piece is None:
+        raise PyritException(message="Adversarial chat returned an unknown error.")
+
+    response_error = error_piece.response_error
+    response_value = error_piece.converted_value
+    if response_error == "blocked":
+        status_code, message = _get_error_payload(response_value)
+        raise BadRequestException(status_code=status_code if status_code is not None else 400, message=message)
+    if response_error == "empty":
+        raise EmptyResponseException(message="The adversarial chat returned an empty response.")
+
+    normalized_error = response_error if response_error != "none" else "unknown"
+    raise PyritException(message=f"Adversarial chat returned a {normalized_error} error: {response_value}")
 
 
 def _build_adversarial_feedback_text(
@@ -755,6 +832,7 @@ async def generate_adversarial_reply_async(
             AdversarialReply: ``next_message`` plus the parsed ``rationale`` / ``last_response_summary``.
 
         Raises:
+            PyritException: If the adversarial chat returns an errored response.
             ValueError: If no response is received from the adversarial chat.
             InvalidJsonException: If the reply is not valid JSON after the retry budget is exhausted.
         """
@@ -831,6 +909,7 @@ async def _send_and_parse_async(
             AdversarialReply: ``next_message`` plus the parsed ``rationale`` / ``last_response_summary``.
 
         Raises:
+            PyritException: If the adversarial chat returns an errored response.
             ValueError: If no response is received from the adversarial chat.
             InvalidJsonException: If the reply is not valid JSON after the retry budget is exhausted.
         """
@@ -853,6 +932,7 @@ async def _send_and_parse_async(
         schema = self._response_json_schema
 
         def _parse(response: Message) -> AdversarialReply:
+            _raise_for_adversarial_error(response)
             return _parse_adversarial_reply(response.get_value(), schema=schema)
 
         with execution_context(

tests/unit/executor/attack/component/test_adversarial_conversation_manager.py

@@ -1,13 +1,14 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
 
+import json
 import logging
 from types import SimpleNamespace
 from unittest.mock import AsyncMock, MagicMock
 
 import pytest
 
-from pyrit.exceptions import InvalidJsonException
+from pyrit.exceptions import BadRequestException, EmptyResponseException, InvalidJsonException, PyritException
 from pyrit.executor.attack.component.adversarial_conversation_manager import (
     _BLOCKED_FEEDBACK_TEXT,
     _DEFAULT_ADVERSARIAL_SCHEMA_NAME,
@@ -93,6 +94,18 @@ def _response_message(value: str = "target said hi", *, data_type: str = "text",
     return Message(message_pieces=[piece])
 
 
+def _blocked_adversarial_response(refusal: str = "I cannot assist with that request.") -> Message:
+    payload = json.dumps({"status_code": 200, "message": refusal})
+    piece = MessagePiece(
+        role="assistant",
+        original_value=payload,
+        original_value_data_type="error",
+        response_error="blocked",
+    )
+    piece.mark_as_structured_refusal(refusal=refusal)
+    return Message(message_pieces=[piece])
+
+
 def _seed_message(value: str = "seed prompt") -> Message:
     return Message(message_pieces=[MessagePiece(role="user", original_value=value, original_value_data_type="text")])
 
@@ -549,6 +562,92 @@ async def test_invalid_reply_raises(self):
         with pytest.raises(InvalidJsonException):
             await manager.get_next_message_async(turn_index=1, last_response=_response_message())
 
+    async def test_blocked_reply_does_not_retry_json_parsing(self) -> None:
+        refusal = "I cannot assist with that request."
+        normalizer = _normalizer(None)
+        normalizer.send_prompt_async.return_value = _blocked_adversarial_response(refusal)
+        manager = _manager(
+            adversarial_system_prompt=_system_prompt(schema=SCHEMA),
+            prompt_normalizer=normalizer,
+        )
+
+        with pytest.raises(BadRequestException) as exc_info:
+            await manager.get_next_message_async(turn_index=1, last_response=_response_message())
+
+        assert exc_info.value.status_code == 200
+        assert exc_info.value.message == refusal
+        normalizer.send_prompt_async.assert_awaited_once()
+
+    @pytest.mark.parametrize("response_error", ["processing", "unknown"])
+    async def test_non_blocked_error_preserves_category_without_retry(self, response_error: str) -> None:
+        normalizer = _normalizer(None)
+        normalizer.send_prompt_async.return_value = _response_message(
+            "provider failed",
+            data_type="error",
+            error=response_error,
+        )
+        manager = _manager(
+            adversarial_system_prompt=_system_prompt(schema=SCHEMA),
+            prompt_normalizer=normalizer,
+        )
+
+        with pytest.raises(PyritException) as exc_info:
+            await manager.get_next_message_async(turn_index=1, last_response=_response_message())
+
+        assert type(exc_info.value) is PyritException
+        assert response_error in exc_info.value.message
+        assert "provider failed" in exc_info.value.message
+        normalizer.send_prompt_async.assert_awaited_once()
+
+    async def test_blocked_reply_uses_error_piece_when_clean_piece_precedes_it(self) -> None:
+        refusal = "I cannot assist with that request."
+        blocked_piece = _blocked_adversarial_response(refusal).get_piece()
+        response = Message.from_prompt(prompt="partial content", role="assistant")
+        response.message_pieces.append(blocked_piece)
+        normalizer = _normalizer(None)
+        normalizer.send_prompt_async.return_value = response
+        manager = _manager(
+            adversarial_system_prompt=_system_prompt(schema=SCHEMA),
+            prompt_normalizer=normalizer,
+        )
+
+        with pytest.raises(BadRequestException) as exc_info:
+            await manager.get_next_message_async(turn_index=1, last_response=_response_message())
+
+        assert exc_info.value.status_code == 200
+        assert exc_info.value.message == refusal
+        normalizer.send_prompt_async.assert_awaited_once()
+
+    async def test_empty_error_preserves_category_without_retry(self) -> None:
+        normalizer = _normalizer(None)
+        normalizer.send_prompt_async.return_value = _response_message("", error="empty")
+        manager = _manager(
+            adversarial_system_prompt=_system_prompt(schema=SCHEMA),
+            prompt_normalizer=normalizer,
+        )
+
+        with pytest.raises(EmptyResponseException):
+            await manager.get_next_message_async(turn_index=1, last_response=_response_message())
+
+        normalizer.send_prompt_async.assert_awaited_once()
+
+    async def test_malformed_non_error_reply_retries_then_succeeds(self) -> None:
+        normalizer = _normalizer(None)
+        normalizer.send_prompt_async.side_effect = [
+            Message.from_prompt(prompt="totally not json", role="assistant"),
+            Message.from_prompt(prompt=VALID_JSON, role="assistant"),
+        ]
+        manager = _manager(
+            adversarial_system_prompt=_system_prompt(schema=SCHEMA),
+            prompt_normalizer=normalizer,
+        )
+
+        turn = await manager.get_next_message_async(turn_index=1, last_response=_response_message())
+
+        assert turn.reply is not None
+        assert turn.reply.next_message == "hello target"
+        assert normalizer.send_prompt_async.call_count == 2
+
     async def test_non_object_reply_retries_then_succeeds(self) -> None:
         normalizer = _normalizer(None)
         normalizer.send_prompt_async.side_effect = [