TEST: skip Azure key-auth-disabled integration tests

doc/contributing/10_release_process.md

@@ -14,6 +14,7 @@ Before starting the release process, verify the codebase is in a healthy state.
 - **Check for pending changes.** Ask other PyRIT maintainers whether they have any in-flight changes that should land before the release.
 - **Verify build pipelines.** Confirm that all integration tests and end-to-end tests are passing in the CI pipelines. If any tests are failing, fix them before proceeding.
   - **Partner integration tests.** Ensure the partner integration tests are also passing. These tests validate that we are not breaking contracts with partner teams (e.g., Foundry). If any are failing, coordinate with the affected partner teams before proceeding with the release.
+  - **Azure key-based auth is disabled in our tenant.** Our Azure subscription has API-key (local) auth turned off, so Azure targets authenticate with Microsoft Entra ID (Entra auth) only. Integration tests and notebooks that exercise Azure targets with API keys are deliberately skipped; otherwise they fail with HTTP 403 `AuthenticationTypeDisabled` ("Key based authentication is disabled for this resource"). The same endpoints are covered by the Entra-auth tests in `tests/integration/targets/test_entra_auth_targets.py`, so this is expected and not a coverage gap. Do not re-enable these for our tenant. When validating a release manually, authenticate Azure targets with Entra (`az login`) rather than API keys.
 - **Update scorer metrics.** Run `python .\build_scripts\evaluate_scorers.py` and commit the results so that scorer evaluation metrics are up to date.
 
 ## 2. Decide the Next Version
@@ -180,6 +181,8 @@ Additionally, verify that your environment file includes all the test secrets ne
 
 In the new location, run all notebooks that are currently skipped by integration tests (there are less than 10) in VS Code. These are listed in `skipped_files` in each `tests/integration/<folder>/test_notebooks_*.py` file and are located in the doc folder that you copied into your new `releases\releasevx.y.z` folder. Note that some of these notebooks have known issues and it may make sense to skip testing them until those are fixed. Check with the last person to deploy or look for the relevant release work item for more information. In running the notebooks, you may also see exceptions. If this happens, make sure to look for existing bugs open on the ADO board or create a new one if it does not exist! If it is easy to fix, we prefer to fix the issue before the release continues.
 
+Some notebooks that use Azure targets with API-key (local) auth are skipped by the integration tests for a separate reason: key-based auth is disabled in our Azure subscription (see the note under *Release Readiness* above). These are tracked in the `_azure_key_auth_notebooks` set in `tests/integration/targets/test_notebooks_targets.py` (distinct from the long-standing `skipped_files` list). Validate them with Entra auth (`az login`) rather than API keys, or rely on the equivalent Entra-auth integration tests; running them with API keys fails with HTTP 403 `AuthenticationTypeDisabled`.
+
 A reminder that you should ensure that the integration tests pass in the version you are releasing in addition to the skipped files.
 
 Note: copying the `doc` folder elsewhere is essential since we store data files

tests/integration/embeddings/test_openai_embedding.py

@@ -8,12 +8,22 @@
 from pyrit.auth import get_azure_openai_auth
 from pyrit.embedding import OpenAITextEmbedding
 
+_AZURE_KEY_AUTH_DISABLED_REASON = (
+    "Azure key-based (local) auth is disabled in our tenant; "
+    "covered by the Entra-auth tests (test_entra_auth_targets.py)."
+)
+
 
 @pytest.mark.run_only_if_all_tests
 @pytest.mark.parametrize(
     "endpoint_env,key_env,model_env",
     [
-        ("OPENAI_EMBEDDING_ENDPOINT", "OPENAI_EMBEDDING_KEY", "OPENAI_EMBEDDING_MODEL"),
+        pytest.param(
+            "OPENAI_EMBEDDING_ENDPOINT",
+            "OPENAI_EMBEDDING_KEY",
+            "OPENAI_EMBEDDING_MODEL",
+            marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON),
+        ),
         ("PLATFORM_OPENAI_EMBEDDING_ENDPOINT", "PLATFORM_OPENAI_EMBEDDING_KEY", "PLATFORM_OPENAI_EMBEDDING_MODEL"),
     ],
 )

tests/integration/score/test_azure_content_filter_integration.py

@@ -12,6 +12,11 @@
 from pyrit.memory import CentralMemory, MemoryInterface
 from pyrit.score import AzureContentFilterScorer
 
+_AZURE_KEY_AUTH_DISABLED_REASON = (
+    "Azure key-based (local) auth is disabled in our tenant; "
+    "covered by the Entra-auth tests (test_entra_auth_targets.py)."
+)
+
 
 @pytest.fixture
 def memory() -> Generator[MemoryInterface, None, None]:
@@ -80,6 +85,7 @@ async def test_azure_content_filter_scorer_long_text_chunking_integration(memory
 
 
 @pytest.mark.run_only_if_all_tests
+@pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON)
 async def test_azure_content_filter_scorer_image_with_api_key(memory) -> None:
     """
     Integration test for Azure Content Filter Scorer image scoring with explicit API key auth.
@@ -108,6 +114,7 @@ async def test_azure_content_filter_scorer_image_with_api_key(memory) -> None:
 
 
 @pytest.mark.run_only_if_all_tests
+@pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON)
 async def test_azure_content_filter_scorer_text_with_api_key(memory) -> None:
     """
     Integration test for Azure Content Filter Scorer text scoring with explicit API key auth.

tests/integration/targets/test_notebooks_targets.py

@@ -19,11 +19,34 @@
     "10_3_websocket_copilot_target.ipynb",  # WebSocket Copilot target requires manual pasting tokens
 ]
 
-
-@pytest.mark.parametrize(
-    "file_name",
-    [file for file in os.listdir(nb_directory_path) if file.endswith(".ipynb") and file not in skipped_files],
+_AZURE_KEY_AUTH_DISABLED_REASON = (
+    "Azure key-based (local) auth is disabled in our tenant; "
+    "covered by the Entra-auth tests (test_entra_auth_targets.py)."
 )
+
+# Notebooks whose targets use Azure key-based (local) auth, which is disabled in our tenant.
+_azure_key_auth_notebooks = {
+    "10_http_target.ipynb",
+    "5_openai_tts_target.ipynb",
+    "6_custom_targets.ipynb",
+    "9_rate_limiting.ipynb",
+    "round_robin_target.ipynb",
+}
+
+
+def _notebook_params():
+    params = []
+    for file in os.listdir(nb_directory_path):
+        if not file.endswith(".ipynb") or file in skipped_files:
+            continue
+        if file in _azure_key_auth_notebooks:
+            params.append(pytest.param(file, marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON)))
+        else:
+            params.append(file)
+    return params
+
+
+@pytest.mark.parametrize("file_name", _notebook_params())
 def test_execute_notebooks(file_name):
     nb_path = pathlib.Path(nb_directory_path, file_name).resolve()
     with open(nb_path, encoding="utf-8") as f:

tests/integration/targets/test_openai_responses_gpt5.py

@@ -12,6 +12,13 @@
 from pyrit.models import MessagePiece
 from pyrit.prompt_target import OpenAIResponseTarget
 
+_AZURE_KEY_AUTH_DISABLED_REASON = (
+    "Azure key-based (local) auth is disabled in our tenant; "
+    "covered by the Entra-auth tests (test_entra_auth_targets.py)."
+)
+
+pytestmark = pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON)
+
 
 @pytest.fixture()
 def gpt5_args():

tests/integration/targets/test_target_filters.py

@@ -13,6 +13,11 @@
     OpenAIVideoTarget,
 )
 
+_AZURE_KEY_AUTH_DISABLED_REASON = (
+    "Azure key-based (local) auth is disabled in our tenant; "
+    "covered by the Entra-auth tests (test_entra_auth_targets.py)."
+)
+
 
 @pytest.mark.parametrize(
     ("endpoint", "api_key", "model_name"),
@@ -24,6 +29,7 @@
         ),
     ],
 )
+@pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON)
 async def test_azure_content_filters(sqlite_instance, endpoint, api_key, model_name):
     args = {
         "endpoint": os.getenv(endpoint),
@@ -62,6 +68,7 @@ async def test_azure_content_filters(sqlite_instance, endpoint, api_key, model_n
         ),
     ],
 )
+@pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON)
 async def test_azure_content_filters_response_api(sqlite_instance, endpoint, api_key, model_name):
     endpoint_val = os.getenv(endpoint)
     api_key_val = os.getenv(api_key)
@@ -95,6 +102,7 @@ async def test_azure_content_filters_response_api(sqlite_instance, endpoint, api
     ("endpoint", "api_key", "model_name"),
     [("OPENAI_IMAGE_STRICT_FILTER_ENDPOINT", "OPENAI_IMAGE_STRICT_FILTER_KEY", "OPENAI_IMAGE_STRICT_FILTER_MODEL")],
 )
+@pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON)
 async def test_image_input_filters(sqlite_instance, endpoint, api_key, model_name):
     target = OpenAIImageTarget(
         endpoint=os.getenv(endpoint), api_key=os.getenv(api_key), model_name=os.getenv(model_name)
@@ -119,6 +127,7 @@ async def test_image_input_filters(sqlite_instance, endpoint, api_key, model_nam
     ("endpoint", "api_key", "model_name"),
     [("AZURE_OPENAI_VIDEO_ENDPOINT", "AZURE_OPENAI_VIDEO_KEY", "AZURE_OPENAI_VIDEO_MODEL")],
 )
+@pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON)
 async def test_video_input_filters(sqlite_instance, endpoint, api_key, model_name):
     target = OpenAIVideoTarget(
         endpoint=os.getenv(endpoint),

tests/integration/targets/test_targets_and_secrets.py

@@ -23,6 +23,11 @@
     RealtimeTarget,
 )
 
+_AZURE_KEY_AUTH_DISABLED_REASON = (
+    "Azure key-based (local) auth is disabled in our tenant; "
+    "covered by the Entra-auth tests (test_entra_auth_targets.py)."
+)
+
 
 def _get_required_env_var(env_var_name: str) -> str:
     """
@@ -94,42 +99,77 @@ async def _assert_can_send_video_prompt(target):
 @pytest.mark.parametrize(
     ("endpoint", "api_key", "model_name", "supports_seed"),
     [
-        ("OPENAI_CHAT_ENDPOINT", "OPENAI_CHAT_KEY", "OPENAI_CHAT_MODEL", True),
+        pytest.param(
+            "OPENAI_CHAT_ENDPOINT",
+            "OPENAI_CHAT_KEY",
+            "OPENAI_CHAT_MODEL",
+            True,
+            marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON),
+        ),
         ("PLATFORM_OPENAI_CHAT_ENDPOINT", "PLATFORM_OPENAI_CHAT_KEY", "PLATFORM_OPENAI_CHAT_MODEL", True),
-        ("AZURE_OPENAI_GPT4O_ENDPOINT", "AZURE_OPENAI_GPT4O_KEY", "AZURE_OPENAI_GPT4O_MODEL", True),
-        (
+        pytest.param(
+            "AZURE_OPENAI_GPT4O_ENDPOINT",
+            "AZURE_OPENAI_GPT4O_KEY",
+            "AZURE_OPENAI_GPT4O_MODEL",
+            True,
+            marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON),
+        ),
+        pytest.param(
             "AZURE_OPENAI_INTEGRATION_TEST_ENDPOINT",
             "AZURE_OPENAI_INTEGRATION_TEST_KEY",
             "AZURE_OPENAI_INTEGRATION_TEST_MODEL",
             True,
+            marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON),
         ),
-        (
+        pytest.param(
             "AZURE_OPENAI_GPT4O_UNSAFE_CHAT_ENDPOINT",
             "AZURE_OPENAI_GPT4O_UNSAFE_CHAT_KEY",
             "AZURE_OPENAI_GPT4O_UNSAFE_CHAT_MODEL",
             True,
+            marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON),
         ),
-        (
+        pytest.param(
             "AZURE_OPENAI_GPT4O_UNSAFE_CHAT_ENDPOINT2",
             "AZURE_OPENAI_GPT4O_UNSAFE_CHAT_KEY2",
             "AZURE_OPENAI_GPT4O_UNSAFE_CHAT_MODEL2",
             True,
+            marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON),
         ),
-        ("AZURE_OPENAI_GPT3_5_CHAT_ENDPOINT", "AZURE_OPENAI_GPT3_5_CHAT_KEY", "AZURE_OPENAI_GPT3_5_CHAT_MODEL", True),
-        ("AZURE_OPENAI_GPT4_CHAT_ENDPOINT", "AZURE_OPENAI_GPT4_CHAT_KEY", "AZURE_OPENAI_GPT4_CHAT_MODEL", True),
-        ("AZURE_OPENAI_GPTV_CHAT_ENDPOINT", "AZURE_OPENAI_GPTV_CHAT_KEY", "AZURE_OPENAI_GPTV_CHAT_MODEL", True),
-        (
+        pytest.param(
+            "AZURE_OPENAI_GPT3_5_CHAT_ENDPOINT",
+            "AZURE_OPENAI_GPT3_5_CHAT_KEY",
+            "AZURE_OPENAI_GPT3_5_CHAT_MODEL",
+            True,
+            marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON),
+        ),
+        pytest.param(
+            "AZURE_OPENAI_GPT4_CHAT_ENDPOINT",
+            "AZURE_OPENAI_GPT4_CHAT_KEY",
+            "AZURE_OPENAI_GPT4_CHAT_MODEL",
+            True,
+            marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON),
+        ),
+        pytest.param(
+            "AZURE_OPENAI_GPTV_CHAT_ENDPOINT",
+            "AZURE_OPENAI_GPTV_CHAT_KEY",
+            "AZURE_OPENAI_GPTV_CHAT_MODEL",
+            True,
+            marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON),
+        ),
+        pytest.param(
             "AZURE_OPENAI_GPT5_COMPLETIONS_ENDPOINT",
             "AZURE_OPENAI_GPT5_COMPLETIONS_KEY",
             "AZURE_OPENAI_GPT5_COMPLETIONS_MODEL",
             True,
+            marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON),
         ),
         ("AZURE_FOUNDRY_DEEPSEEK_ENDPOINT", "AZURE_FOUNDRY_DEEPSEEK_KEY", "AZURE_FOUNDRY_DEEPSEEK_MODEL", True),
-        (
+        pytest.param(
             "AZURE_FOUNDRY_MISTRAL_LARGE_ENDPOINT",
             "AZURE_FOUNDRY_MISTRAL_LARGE_KEY",
             "AZURE_FOUNDRY_MISTRAL_LARGE_MODEL",
             False,
+            marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON),
         ),
         ("AZURE_FOUNDRY_PHI4_ENDPOINT", "AZURE_CHAT_PHI4_KEY", "AZURE_CHAT_PHI4_MODEL", True),
         ("GOOGLE_GEMINI_ENDPOINT", "GOOGLE_GEMINI_API_KEY", "GOOGLE_GEMINI_MODEL", False),
@@ -166,16 +206,23 @@ async def test_connect_required_openai_text_targets(sqlite_instance, endpoint, a
             "PLATFORM_OPENAI_RESPONSES_KEY",
             "PLATFORM_OPENAI_RESPONSES_MODEL",
         ),
-        ("AZURE_OPENAI_RESPONSES_ENDPOINT", "AZURE_OPENAI_RESPONSES_KEY", "AZURE_OPENAI_RESPONSES_MODEL"),
-        (
+        pytest.param(
+            "AZURE_OPENAI_RESPONSES_ENDPOINT",
+            "AZURE_OPENAI_RESPONSES_KEY",
+            "AZURE_OPENAI_RESPONSES_MODEL",
+            marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON),
+        ),
+        pytest.param(
             "AZURE_OPENAI_GPT41_RESPONSES_ENDPOINT",
             "AZURE_OPENAI_GPT41_RESPONSES_KEY",
             "AZURE_OPENAI_GPT41_RESPONSES_MODEL",
+            marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON),
         ),
-        (
+        pytest.param(
             "AZURE_OPENAI_GPT5_RESPONSES_ENDPOINT",
             "AZURE_OPENAI_GPT5_KEY",
             "AZURE_OPENAI_GPT5_MODEL",
+            marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON),
         ),
         ("AWS_ENDPOINT", "AWS_KEY", "AWS_RESPONSES_MODEL"),
     ],
@@ -294,8 +341,18 @@ async def test_connect_openai_completion(sqlite_instance):
 @pytest.mark.parametrize(
     ("endpoint", "api_key", "model_name"),
     [
-        ("OPENAI_IMAGE_ENDPOINT1", "OPENAI_IMAGE_API_KEY1", "OPENAI_IMAGE_MODEL1"),  # gpt-image-1.5
-        ("OPENAI_IMAGE_ENDPOINT2", "OPENAI_IMAGE_API_KEY2", "OPENAI_IMAGE_MODEL2"),  # gpt-image-1
+        pytest.param(
+            "OPENAI_IMAGE_ENDPOINT1",
+            "OPENAI_IMAGE_API_KEY1",
+            "OPENAI_IMAGE_MODEL1",
+            marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON),
+        ),  # gpt-image-1.5
+        pytest.param(
+            "OPENAI_IMAGE_ENDPOINT2",
+            "OPENAI_IMAGE_API_KEY2",
+            "OPENAI_IMAGE_MODEL2",
+            marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON),
+        ),  # gpt-image-1
         ("PLATFORM_OPENAI_IMAGE_ENDPOINT", "PLATFORM_OPENAI_IMAGE_KEY", "PLATFORM_OPENAI_IMAGE_MODEL"),  # gpt-image-1.5
     ],
 )
@@ -332,6 +389,7 @@ async def test_connect_image(sqlite_instance, endpoint, api_key, model_name):
 
 
 @pytest.mark.run_only_if_all_tests
+@pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON)
 async def test_image_editing_single_image_api_key(sqlite_instance):
     """
     Test image editing with a single image input using API key authentication.
@@ -380,6 +438,7 @@ async def test_image_editing_single_image_api_key(sqlite_instance):
 
 
 @pytest.mark.run_only_if_all_tests
+@pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON)
 async def test_image_editing_multiple_images_api_key(sqlite_instance):
     """
     Test image editing with multiple image inputs using API key authentication.
@@ -440,6 +499,7 @@ async def test_image_editing_multiple_images_api_key(sqlite_instance):
         ("OPENAI_TTS_ENDPOINT2", "OPENAI_TTS_KEY2", "OPENAI_TTS_MODEL2"),
     ],
 )
+@pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON)
 async def test_connect_tts(sqlite_instance, endpoint, api_key, model_name):
     endpoint_value = _get_required_env_var(endpoint)
     api_key_value = _get_required_env_var(api_key)
@@ -464,6 +524,7 @@ async def test_connect_tts(sqlite_instance, endpoint, api_key, model_name):
         #  "PLATFORM_OPENAI_VIDEO_MODEL"),
     ],
 )
+@pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON)
 async def test_connect_video(sqlite_instance, endpoint, api_key, model_name):
     """Test OpenAIVideoTarget with video API."""
     endpoint_value = _get_required_env_var(endpoint)
@@ -482,6 +543,7 @@ async def test_connect_video(sqlite_instance, endpoint, api_key, model_name):
 
 
 @pytest.mark.run_only_if_all_tests
+@pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON)
 async def test_video_multiple_prompts_create_separate_files(sqlite_instance):
     """
     Test that sending multiple prompts to video API using PromptSendingAttack