Multi-Tenant Resilience at Scale: Operational Strategies for ISVs on AWS

# Emit per-tenant, per-model telemetry via CloudWatch EMF (aws-embedded-metrics)
from aws_embedded_metrics import metric_scope

@metric_scope
def record_invocation(tenant_id, tier, model_id, ttft_ms, tokens, throttled, metrics):
metrics.set_namespace("ISV/MultiTenant")
metrics.set_dimensions({"TenantId": tenant_id, "Tier": tier, "ModelId": model_id})
metrics.put_metric("TimeToFirstToken", ttft_ms, "Milliseconds")
metrics.put_metric("TokenConsumption", tokens, "Count")
metrics.put_metric("ThrottledRequests", 1 if throttled else 0, "Count")

import boto3, datetime

sq = boto3.client("service-quotas")
cw = boto3.client("cloudwatch")

def hours_to_exhaustion(service_code, quota_code, usage_metric):
quota = sq.get_service_quota(ServiceCode=service_code, QuotaCode=quota_code)
ceiling = quota["Quota"]["Value"]

now = datetime.datetime.utcnow()
stats = cw.get_metric_statistics(
Namespace=usage_metric["namespace"],
MetricName=usage_metric["name"],
Dimensions=usage_metric["dimensions"],
StartTime=now - datetime.timedelta(hours=6),
EndTime=now, Period=3600, Statistics=["Maximum"],
)["Datapoints"]
points = sorted(stats, key=lambda d: d["Timestamp"])
if len(points) < 2:
return None
current = points[-1]["Maximum"]
velocity = (points[-1]["Maximum"] - points[0]["Maximum"]) / max(len(points) - 1, 1)
if velocity <= 0:
return float("inf")
return (ceiling - current) / velocity # hours of headroom remaining

{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "ReadOnlyDiagnostics",
"Effect": "Allow",
"Action": [
"logs:StartQuery",
"logs:GetQueryResults",
"cloudwatch:GetMetricData",
"cloudwatch:GetMetricStatistics",
"xray:GetTraceSummaries",
"xray:BatchGetTraces",
"servicequotas:GetServiceQuota"
],
"Resource": "*"
},
{
"Sid": "ScopedQuotaIncrease",
"Effect": "Allow",
"Action": "servicequotas:RequestServiceQuotaIncrease",
"Resource": "*",
"Condition": {
"StringEquals": { "servicequotas:service": "bedrock" }
}
}
]
}