AI Foundry changes

Author: pamelafox

evals/safety_evaluation.py

@@ -8,85 +8,79 @@
 from typing import Optional
 
 import requests
-from azure.ai.evaluation import AzureAIProject
 from azure.ai.evaluation.red_team import AttackStrategy, RedTeam, RiskCategory
 from azure.identity import AzureDeveloperCliCredential
 from dotenv_azd import load_azd_env
-from rich.logging import RichHandler
-
-logger = logging.getLogger("ragapp")
-
-# Configure logging to capture and display warnings with tracebacks
-logging.captureWarnings(True)  # Capture warnings as log messages
 
 root_dir = pathlib.Path(__file__).parent
 
 
 def get_azure_credential():
     AZURE_TENANT_ID = os.getenv("AZURE_TENANT_ID")
     if AZURE_TENANT_ID:
-        logger.info("Setting up Azure credential using AzureDeveloperCliCredential with tenant_id %s", AZURE_TENANT_ID)
+        print("Setting up Azure credential using AzureDeveloperCliCredential with tenant_id %s", AZURE_TENANT_ID)
         azure_credential = AzureDeveloperCliCredential(tenant_id=AZURE_TENANT_ID, process_timeout=60)
     else:
-        logger.info("Setting up Azure credential using AzureDeveloperCliCredential for home tenant")
+        print("Setting up Azure credential using AzureDeveloperCliCredential for home tenant")
         azure_credential = AzureDeveloperCliCredential(process_timeout=60)
     return azure_credential
 
 
-async def callback(
-    messages: list,
+def callback(
+    question: str,
     target_url: str = "http://127.0.0.1:8000/chat",
 ):
-    query = messages[-1].content
     headers = {"Content-Type": "application/json"}
     body = {
-        "messages": [{"content": query, "role": "user"}],
+        "messages": [{"content": question, "role": "user"}],
         "stream": False,
-        "context": {"overrides": {"use_advanced_flow": True, "top": 3, "retrieval_mode": "hybrid", "temperature": 0.3}},
+        "context": {
+            "overrides": {"use_advanced_flow": False, "top": 3, "retrieval_mode": "hybrid", "temperature": 0.3}
+        },
     }
     url = target_url
     r = requests.post(url, headers=headers, json=body)
     response = r.json()
     if "error" in response:
-        message = {"content": response["error"], "role": "assistant"}
+        return f"Error received: {response['error']}"
     else:
-        message = response["message"]
-    return {"messages": messages + [message]}
+        return response["message"]["content"]
 
 
-async def run_simulator(target_url: str, max_simulations: int, scan_name: Optional[str] = None):
-    credential = get_azure_credential()
-    azure_ai_project: AzureAIProject = {
-        "subscription_id": os.getenv("AZURE_SUBSCRIPTION_ID"),
-        "resource_group_name": os.getenv("AZURE_RESOURCE_GROUP"),
-        "project_name": "pf-testprojforaisaety",
-    }
+async def run_redteaming(target_url: str, questions_per_category: int = 1, scan_name: Optional[str] = None):
+    AZURE_AI_FOUNDRY = os.getenv("AZURE_AI_FOUNDRY")
+    AZURE_AI_PROJECT = os.getenv("AZURE_AI_PROJECT")
     model_red_team = RedTeam(
-        azure_ai_project=azure_ai_project,
-        credential=credential,
+        azure_ai_project=f"https://{AZURE_AI_FOUNDRY}.services.ai.azure.com/api/projects/{AZURE_AI_PROJECT}",
+        credential=get_azure_credential(),
         risk_categories=[
             RiskCategory.Violence,
             RiskCategory.HateUnfairness,
             RiskCategory.Sexual,
             RiskCategory.SelfHarm,
         ],
-        num_objectives=1,
+        num_objectives=questions_per_category,
     )
+
     if scan_name is None:
         timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
         scan_name = f"Safety evaluation {timestamp}"
+
     await model_red_team.scan(
-        target=lambda messages, stream=False, session_state=None, context=None: callback(messages, target_url),
         scan_name=scan_name,
+        output_path=f"{root_dir}/redteams/{scan_name}.json",
         attack_strategies=[
-            AttackStrategy.DIFFICULT,
             AttackStrategy.Baseline,
-            AttackStrategy.UnicodeConfusable,  # Use confusable Unicode characters
-            AttackStrategy.Morse,  # Encode prompts in Morse code
-            AttackStrategy.Leetspeak,  # Use Leetspeak
-            AttackStrategy.Url,  # Use URLs in prompts
+            # Easy Complexity:
+            AttackStrategy.Morse,
+            AttackStrategy.UnicodeConfusable,
+            AttackStrategy.Url,
+            # Moderate Complexity:
+            AttackStrategy.Tense,
+            # Difficult Complexity:
+            AttackStrategy.Compose([AttackStrategy.Tense, AttackStrategy.Url]),
         ],
-        output_path="Advanced-Callback-Scan.json",
+        target=lambda query: callback(query, target_url),
     )
 
 
@@ -96,31 +90,17 @@ async def run_simulator(target_url: str, max_simulations: int, scan_name: Option
         "--target_url", type=str, default="http://127.0.0.1:8000/chat", help="Target URL for the callback."
     )
     parser.add_argument(
-        "--max_simulations", type=int, default=200, help="Maximum number of simulations (question/response pairs)."
+        "--questions_per_category",
+        type=int,
+        default=1,
+        help="Number of questions per risk category to ask during the scan.",
     )
-    # argument for the name
     parser.add_argument("--scan_name", type=str, default=None, help="Name of the safety evaluation (optional).")
     args = parser.parse_args()
 
-    # Configure logging to show tracebacks for warnings and above
-    logging.basicConfig(
-        level=logging.WARNING,
-        format="%(message)s",
-        datefmt="[%X]",
-        handlers=[RichHandler(rich_tracebacks=False, show_path=True)],
-    )
-
-    # Set urllib3 and azure libraries to WARNING level to see connection issues
-    logging.getLogger("urllib3").setLevel(logging.WARNING)
-    logging.getLogger("azure").setLevel(logging.WARNING)
-
-    # Set our application logger to INFO level
-    logger.setLevel(logging.INFO)
-
     load_azd_env()
-
     try:
-        asyncio.run(run_simulator(args.target_url, args.max_simulations, args.scan_name))
+        asyncio.run(run_redteaming(args.target_url, args.questions_per_category, args.scan_name))
     except Exception:
         logging.exception("Unhandled exception in safety evaluation")
         sys.exit(1)

infra/core/ai/ai-foundry.bicep

@@ -0,0 +1,117 @@
+@minLength(1)
+@description('Primary location for all resources')
+param location string
+
+@description('The AI Foundry resource name.')
+param foundryName string
+
+@description('The AI Project resource name.')
+param projectName string = foundryName
+
+param projectDescription string = ''
+param projectDisplayName string = projectName
+
+@description('The Storage Account resource name.')
+param storageAccountName string
+
+param principalId string
+param principalType string
+
+param tags object = {}
+
+// Step 1: Create an AI Foundry resource
+resource account 'Microsoft.CognitiveServices/accounts@2025-04-01-preview' = {
+  name: foundryName
+  location: location
+  tags: tags
+  sku: {
+    name: 'S0'
+  }
+  kind: 'AIServices'
+  identity: {
+    type: 'SystemAssigned'
+  }
+  properties: {
+    allowProjectManagement: true
+    customSubDomainName: toLower(foundryName)
+    networkAcls: {
+      defaultAction: 'Allow'
+      virtualNetworkRules: []
+      ipRules: []
+    }
+    publicNetworkAccess: 'Enabled'
+    disableLocalAuth: false
+  }
+}
+
+// Step 2: Create an AI Foundry project
+resource project 'Microsoft.CognitiveServices/accounts/projects@2025-04-01-preview' = {
+  parent: account
+  name: projectName
+  location: location
+  tags: tags
+  identity: {
+    type: 'SystemAssigned'
+  }
+  properties: {
+    description: projectDescription
+    displayName: projectDisplayName
+  }
+}
+
+// Step 4: Create a storage account, needed for evaluations
+resource storageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' existing = {
+  name: storageAccountName
+}
+
+// Create a storage account connection for the foundry resource
+resource storageAccountConnection 'Microsoft.CognitiveServices/accounts/connections@2025-04-01-preview' = {
+  parent: account
+  name: 'default-storage'
+  properties: {
+    authType: 'AAD'
+    category: 'AzureStorageAccount'
+    isSharedToAll: true
+    target: storageAccount.properties.primaryEndpoints.blob
+    metadata: {
+      ApiType: 'Azure'
+      ResourceId: storageAccount.id
+    }
+  }
+}
+
+// Assign a role to the project's managed identity for the storage account
+resource storageRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
+  name: guid(storageAccount.id, 'Storage Blob Data Contributor', project.name)
+  scope: storageAccount
+  properties: {
+    roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe') // Storage Blob Data Contributor
+    principalId: project.identity.principalId
+    principalType: 'ServicePrincipal'
+  }
+}
+
+// Assign a role to the calling user for the AI Foundry project (needed for projects (including agents) API)
+resource projectRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
+  name: guid(project.id, 'Azure AI User', principalId)
+  scope: project
+  properties: {
+    roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '53ca6127-db72-4b80-b1b0-d745d6d5456d') // Azure AI User
+    principalId: principalId
+    principalType: 'User'
+  }
+}
+
+// Assign a role to the calling user for the AI Foundry account (needed for Azure OpenAI API)
+resource accountRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
+  name: guid(account.id, 'Azure AI User', principalId)
+  scope: account
+  properties: {
+    roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '53ca6127-db72-4b80-b1b0-d745d6d5456d') // Azure AI User
+    principalId: principalId
+    principalType: 'User'
+  }
+}
+
+output foundryName string = account.name
+output projectName string = project.name

infra/main.bicep

@@ -472,16 +472,17 @@ module storage 'br/public:avm/res/storage/storage-account:0.9.1' = if (useAiProj
   }
 }
 
-module ai 'core/ai/ai-environment.bicep' = if (useAiProject) {
+module ai 'core/ai/ai-foundry.bicep' = if (useAiProject) {
   name: 'ai'
   scope: resourceGroup
   params: {
     location: 'swedencentral'
     tags: tags
-    hubName: 'aihub-${resourceToken}'
-    projectName: 'aiproj-${resourceToken}'
-    applicationInsightsId: monitoring.outputs.applicationInsightsId
-    storageAccountId: storage.outputs.resourceId
+    foundryName: 'aifoundry-${resourceToken}'
+    projectName: 'aiproject-${resourceToken}'
+    storageAccountName: storage.outputs.name
+    principalId: principalId
+    principalType: empty(runningOnGh) ? 'User' : 'ServicePrincipal'
   }
 }
 
@@ -491,11 +492,22 @@ module openAIRoleUser 'core/security/role.bicep' = {
   name: 'openai-role-user'
   params: {
     principalId: principalId
-    roleDefinitionId: '5e0bd9bd-7b93-4f28-af87-19fc36ad61bd'
+    roleDefinitionId: '5e0bd9bd-7b93-4f28-af87-19fc36ad61bd' // Cognitive Services OpenAI User
     principalType: empty(runningOnGh) ? 'User' : 'ServicePrincipal'
   }
 }
 
+module azureAiUserRole 'core/security/role.bicep' = if (useAiProject && resourceGroup.name != openAIResourceGroup.name) {
+  name: 'azureai-role-user'
+  scope: resourceGroup
+  params: {
+    principalId: principalId
+    roleDefinitionId: '53ca6127-db72-4b80-b1b0-d745d6d5456d' // Azure AI User
+    principalType: empty(runningOnGh) ? 'User' : 'ServicePrincipal'
+  }
+}
+
+
 // Backend roles
 module openAIRoleBackend 'core/security/role.bicep' = {
   scope: openAIResourceGroup
@@ -560,6 +572,7 @@ output AZURE_OPENAI_EVAL_DEPLOYMENT_CAPACITY string = deployAzureOpenAI ? evalDe
 output AZURE_OPENAI_EVAL_DEPLOYMENT_SKU string = deployAzureOpenAI ? evalDeploymentSku : ''
 output AZURE_OPENAI_EVAL_MODEL string = deployAzureOpenAI ? evalModelName : ''
 
+output AZURE_AI_FOUNDRY string = useAiProject ? ai.outputs.foundryName : ''
 output AZURE_AI_PROJECT string = useAiProject ? ai.outputs.projectName : ''
 
 output POSTGRES_HOST string = postgresServer.outputs.POSTGRES_DOMAIN_NAME