Azure Aigateway

# Verify identity is enabled
az apim show --name <apim> --resource-group <rg> --query "identity" -o json

# Check role assignments
AOAI_ID=$(az cognitiveservices account show --name <aoai> --resource-group <rg> --query id -o tsv)
az role assignment list --scope "$AOAI_ID" --query "[?principalType=='ServicePrincipal'].{role:roleDefinitionName, principal:principalId}" -o table

<!-- Per-user instead of global -->
<azure-openai-token-limit
    tokens-per-minute="50000"
    counter-key="@(context.Request.Headers.GetValueOrDefault("X-User-Id", context.Subscription.Id))"
    estimate-prompt-tokens="true" />

<retry condition="@(context.Response.StatusCode == 429)" count="3" interval="10">
    <forward-request />
</retry>

# Check cache-related headers in response
curl -v -X POST "${GATEWAY_URL}/openai/deployments/<deployment>/chat/completions?api-version=2024-02-01" \
  -H "Content-Type: application/json" \
  -H "Ocp-Apim-Subscription-Key: <key>" \
  -d '{"messages": [{"role": "user", "content": "What is Azure?"}], "max_tokens": 100}'

# Look for: x-cache-status header in response

<!-- Shorter TTL for frequently changing knowledge -->
<azure-openai-semantic-cache-store duration="300" />  <!-- 5 minutes -->

<llm-content-safety backend-id="contentsafety-backend">
    <category name="Hate" threshold="5" />      <!-- Was 4, now less strict -->
    <category name="Sexual" threshold="5" />
    <category name="SelfHarm" threshold="5" />
    <category name="Violence" threshold="5" />
</llm-content-safety>

<on-error>
    <choose>
        <when condition="@(context.LastError.Source == "llm-content-safety")">
            <trace source="content-safety" severity="warning">
                @{
                    return new JObject(
                        new JProperty("blocked", true),
                        new JProperty("subscription", context.Subscription.Id),
                        new JProperty("timestamp", DateTime.UtcNow)
                    ).ToString();
                }
            </trace>
            <return-response>
                <set-status code="400" reason="Content Filtered" />
                <set-body>{"error": "Content filtered by safety policy"}</set-body>
            </return-response>
        </when>
    </choose>
</on-error>

# Verify backend exists
az apim backend list --service-name <apim> --resource-group <rg> \
  --query "[].{id:name, url:url}" -o table

# Check backend ID matches policy reference

<backend>
    <!-- Default is 30s, increase for large AI requests -->
    <forward-request timeout="120" />
</backend>

# Get tracing subscription key
az apim subscription list --service-name <apim> --resource-group <rg> \
  --query "[?displayName=='Built-in all-access subscription'].primaryKey" -o tsv

# Send request with tracing
curl -X POST "${GATEWAY_URL}/..." \
  -H "Ocp-Apim-Trace: true" \
  -H "Ocp-Apim-Subscription-Key: <built-in-key>"

// Failed AI gateway requests
requests
| where success == false
| where url contains "openai"
| project timestamp, resultCode, duration, url
| order by timestamp desc
| take 20

// Token metrics over time
customMetrics
| where name == "Total Tokens"
| summarize TotalTokens = sum(value) by bin(timestamp, 1h)
| render timechart

// Content safety blocks
traces
| where message contains "content-safety"
| project timestamp, message, customDimensions
| order by timestamp desc

# 1. Check APIM is running
az apim show --name <apim> --resource-group <rg> --query "provisioningState" -o tsv
# Expected: Succeeded

# 2. Check backends
az apim backend list --service-name <apim> --resource-group <rg> -o table

# 3. Test endpoint
curl -s -o /dev/null -w "%{http_code}" "${GATEWAY_URL}/openai/deployments/<deployment>/chat/completions?api-version=2024-02-01" \
  -H "Ocp-Apim-Subscription-Key: <key>" \
  -H "Content-Type: application/json" \
  -d '{"messages": [{"role": "user", "content": "ping"}], "max_tokens": 5}'
# Expected: 200