Engineering Low-Latency, Resilient AWS Architectures: Nitro Enclaves, ENA Express, Aurora Limitless, and Route 53 ARC

// Cargo.toml dependencies:
// aws-nitro-enclaves-nsm-api = "0.4.0"
// vsock = "0.4.0"
// rsa = { version = "0.9.6", features = ["sha2"] }
// cms = "0.2.2"
// serde_bytes = "0.11.14"

use aws_nitro_enclaves_nsm_api::api::{Request, Response};
use aws_nitro_enclaves_nsm_api::driver::{nsm_exit, nsm_init, nsm_process_request};
use rsa::{Oaep, RsaPrivateKey, pkcs8::EncodePublicKey};
use serde_bytes::ByteBuf;
use sha2::Sha256;
use std::io::{Read, Write};
use std::time::Duration;
use vsock::{VsockAddr, VsockStream, VMADDR_CID_HOST};

const MAX_CMS_RESPONSE_BYTES: usize = 64 * 1024; // 64 KiB upper bound guard

struct NsmGuard(i32);
impl NsmGuard {
    fn open() -> Result<Self, String> {
        let fd = nsm_init();
        if fd < 0 { Err("Failed to open /dev/nsm".into()) } else { Ok(Self(fd)) }
    }
}
impl Drop for NsmGuard {
    fn drop(&mut self) { nsm_exit(self.0); }
}

pub fn request_and_unwrap_kms_key(
    vsock_port: u32,
    kms_key_id: &str,
    encrypted_dek_blob: &[u8],
    nonce: Vec<u8>,
    ephemeral_privkey: &RsaPrivateKey,
) -> Result<Vec<u8>, String> {
    let nsm = NsmGuard::open()?;
    let pubkey_der = ephemeral_privkey
        .to_public_key()
        .to_public_key_der()
        .map_err(|e| format!("DER encoding error: {e}"))?;

    let req = Request::Attestation {
        user_data: Some(ByteBuf::from(kms_key_id.as_bytes().to_vec())),
        nonce: Some(ByteBuf::from(nonce)),
        public_key: Some(ByteBuf::from(pubkey_der.as_bytes().to_vec())),
    };

    let attestation_cbor = match nsm_process_request(nsm.0, req) {
        Response::Attestation { document } => document,
        Response::Error(err) => return Err(format!("NSM driver error: {err:?}")),
        _ => return Err("Unexpected NSM response variant".into()),
    };

    // Connect to parent EC2 vsock proxy (CID 3) with explicit I/O timeouts
    let mut stream = VsockStream::connect(&VsockAddr::new(VMADDR_CID_HOST, vsock_port))
        .map_err(|e| format!("VSOCK connect error: {e}"))?;
    stream.set_read_timeout(Some(Duration::from_secs(3))).map_err(|e| e.to_string())?;
    stream.set_write_timeout(Some(Duration::from_secs(3))).map_err(|e| e.to_string())?;

    // Frame format: [u32 attestation_len][attestation_cbor][u32 ciphertext_len][ciphertext]
    stream.write_all(&(attestation_cbor.len() as u32).to_be_bytes()).map_err(|e| e.to_string())?;
    stream.write_all(&attestation_cbor).map_err(|e| e.to_string())?;
    stream.write_all(&(encrypted_dek_blob.len() as u32).to_be_bytes()).map_err(|e| e.to_string())?;
    stream.write_all(encrypted_dek_blob).map_err(|e| e.to_string())?;

    let mut len_buf = [0u8; 4];
    stream.read_exact(&mut len_buf).map_err(|e| format!("Proxy read header failed: {e}"))?;
    let resp_len = u32::from_be_bytes(len_buf) as usize;
    if resp_len == 0 || resp_len > MAX_CMS_RESPONSE_BYTES {
        return Err(format!("Rejected out-of-bounds CMS response size: {resp_len} bytes"));
    }

    let mut rsa_wrapped_cek = vec![0u8; resp_len];
    stream.read_exact(&mut rsa_wrapped_cek).map_err(|e| e.to_string())?;

    // Unwrap the RecipientInfo OAEP-SHA256 payload inside enclave memory
    ephemeral_privkey
        .decrypt(Oaep::new::<Sha256>(), &rsa_wrapped_cek)
        .map_err(|e| format!("Enclave RSA-OAEP-SHA256 unwrap failed: {e}"))
}

terraform {
  required_version = ">= 1.8.0"
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 5.70"
    }
  }
}

variable "vpc_cidr" {
  type    = string
  default = "10.42.0.0/16"
}

data "aws_ami" "al2023_arm64" {
  most_recent = true
  owners      = ["amazon"]
  filter {
    name   = "name"
    values = ["al2023-ami-2023.*-kernel-6.1-arm64"]
  }
}

resource "aws_vpc" "compute_plane" {
  cidr_block           = var.vpc_cidr
  enable_dns_support   = true
  enable_dns_hostnames = true
}

resource "aws_subnet" "private_az1" {
  vpc_id               = aws_vpc.compute_plane.id
  cidr_block           = cidrsubnet(var.vpc_cidr, 4, 1)
  availability_zone_id = "use1-az1"
}

resource "aws_security_group" "lattice_service_sg" {
  name        = "lattice-srd-service-sg"
  vpc_id      = aws_vpc.compute_plane.id
  ingress {
    description = "Allow intra-subnet SRD and VPC Lattice link-local ingress"
    from_port   = 8443
    to_port     = 8443
    protocol    = "tcp"
    cidr_blocks = [aws_subnet.private_az1.cidr_block, "169.254.171.0/24"]
  }
}

resource "aws_placement_group" "cluster_srd" {
  name     = "ledger-srd-cluster"
  strategy = "cluster"
}

# Supported ENA Express instance family (c7gn features 2nd-gen Nitro + ENA Express support)
resource "aws_network_interface" "srd_eni" {
  subnet_id       = aws_subnet.private_az1.id
  security_groups = [aws_security_group.lattice_service_sg.id]

  ena_srd_specification {
    ena_srd_enabled = true
    ena_srd_udp_specification {
      ena_srd_udp_enabled = true
    }
  }
}

resource "aws_instance" "ledger_node" {
  ami               = data.aws_ami.al2023_arm64.id
  instance_type     = "c7gn.4xlarge"
  placement_group   = aws_placement_group.cluster_srd.id
  network_interface {
    network_interface_id = aws_network_interface.srd_eni.id
    device_index         = 0
  }
}

resource "aws_vpclattice_service_network" "core_mesh" {
  name      = "prod-service-lattice"
  auth_type = "AWS_IAM"
}

resource "aws_vpclattice_service_network_vpc_association" "compute_assoc" {
  vpc_identifier             = aws_vpc.compute_plane.id
  service_network_identifier = aws_vpclattice_service_network.core_mesh.id
  security_group_ids         = [aws_security_group.lattice_service_sg.id]
}

resource "aws_vpclattice_service" "ledger_api" {
  name      = "ledger-settlement-svc"
  auth_type = "AWS_IAM"
}

resource "aws_vpclattice_service_network_service_association" "ledger_bind" {
  service_identifier         = aws_vpclattice_service.ledger_api.id
  service_network_identifier = aws_vpclattice_service_network.core_mesh.id
}

// Build prerequisite: install aws/clockbound daemon & libclockbound headers
// gcc -std=c11 -O2 clockbound_timestamp_probe.c -lclockbound -o clockbound_probe

#include <stdint.h>
#include <stdio.h>
#include <stdbool.h>
#include "clockbound.h"

typedef struct {
    uint64_t earliest_ns;
    uint64_t latest_ns;
    uint64_t max_drift_bound_ns;
    bool     is_synchronized;
} verified_time_envelope_t;

int read_aws_clockbound_envelope(
    const char *shm_path,
    uint64_t max_acceptable_error_ns,
    verified_time_envelope_t *out
) {
    if (!out) return -1;
    clockbound_err cb_err = {0};

    clockbound_ctx *ctx = clockbound_open(shm_path ? shm_path : CLOCKBOUND_SHM_DEFAULT_PATH, &cb_err);
    if (!ctx) {
        fprintf(stderr, "clockbound_open failed (kind=%d, errno=%d)\n", cb_err.kind, cb_err.sys_errno);
        return -2;
    }

    clockbound_now_result now_res = {0};
    if (clockbound_now(ctx, &now_res, &cb_err) != 0) {
        fprintf(stderr, "clockbound_now read error (kind=%d)\n", cb_err.kind);
        clockbound_close(ctx, &cb_err);
        return -3;
    }
    clockbound_close(ctx, &cb_err);

    // Reject timestamps if chronyd / PHC synchronization is degraded or free-running
    if (now_res.clock_status != CLOCK_STATUS_SYNCHRONIZED) {
        out->is_synchronized = false;
        return -4;
    }

    uint64_t earliest = (uint64_t)now_res.earliest.tv_sec * 1000000000ULL + (uint64_t)now_res.earliest.tv_nsec;
    uint64_t latest   = (uint64_t)now_res.latest.tv_sec   * 1000000000ULL + (uint64_t)now_res.latest.tv_nsec;

    if (latest < earliest) {
        return -5; // Guard against monotonic warp or corrupted interval
    }

    uint64_t half_width_bound = (latest - earliest) / 2;
    if (half_width_bound > max_acceptable_error_ns) {
        // Caller must halt commit-wait or fall back to quorum consensus if bound exceeds SLA
        return -6;
    }

    out->earliest_ns        = earliest;
    out->latest_ns          = latest;
    out->max_drift_bound_ns = half_width_bound;
    out->is_synchronized    = true;
    return 0;
}

// package.json: "@aws-sdk/client-route53-recovery-cluster": "^3.650.0"
import {
  Route53RecoveryClusterClient,
  UpdateRoutingControlStatesCommand,
  GetRoutingControlStateCommand,
  ConflictException,
  ValidationException,
  AccessDeniedException,
} from "@aws-sdk/client-route53-recovery-cluster";

export interface ArcClusterEndpoint {
  region: string;
  // Format: https://<cluster-guid>.route53-recovery-cluster.<region>.amazonaws.com/v1
  url: string;
}

export async function shiftCellTrafficWithArc(
  clusterEndpoints: ArcClusterEndpoint[],
  impairControlArn: string,
  standbyControlArn: string,
  perAttemptTimeoutMs = 2500
): Promise<{ succeededEndpoint: string; elapsedMs: number }> {
  if (clusterEndpoints.length !== 5) {
    throw new Error("Expected all 5 regional ARC cluster endpoints from DescribeCluster");
  }

  const startMs = performance.now();

  for (const ep of clusterEndpoints) {
    const client = new Route53RecoveryClusterClient({
      region: ep.region,
      endpoint: ep.url,
      maxAttempts: 1, // Fail over across regional endpoints rather than stalling on one
    });

    try {
      // 1. Verify standby cell control is currently reachable before shifting
      await client.send(
        new GetRoutingControlStateCommand({ RoutingControlArn: standbyControlArn }),
        { abortSignal: AbortSignal.timeout(perAttemptTimeoutMs) }
      );

      // 2. Atomically enable standby cell and disable impaired cell (respecting Safety Rules)
      await client.send(
        new UpdateRoutingControlStatesCommand({
          UpdateRoutingControlStatesEntries: [
            { RoutingControlArn: standbyControlArn, RoutingControlState: "On" },
            { RoutingControlArn: impairControlArn, RoutingControlState: "Off" },
          ],
        }),
        { abortSignal: AbortSignal.timeout(perAttemptTimeoutMs) }
      );

      return {
        succeededEndpoint: ep.region,
        elapsedMs: Math.round(performance.now() - startMs),
      };
    } catch (err) {
      // Do NOT retry on terminal IAM or Safety Rule rejection errors
      if (
        err instanceof AccessDeniedException ||
        err instanceof ValidationException ||
        err instanceof ConflictException
      ) {
        throw new Error(`Terminal ARC Safety/Auth rejection at ${ep.region}: ${err.message}`);
      }
      // Transient network / timeout on this regional endpoint: try next quorum region
      continue;
    }
  }

  throw new Error("All 5 regional Route 53 ARC cluster endpoints timed out");
}