Stop Paying Hourly for Idle SFTP Servers: Switch to S3 Pre-Signed URLs and Save 99% with Terraform💸(10-Minute Setup)

# modules/s3-file-upload/main.tf

# S3 bucket for file uploads
resource "aws_s3_bucket" "uploads" {
bucket = "partner-file-uploads"

tags = {
Name = "partner-uploads"
}
}

# Block public access (security first!)
resource "aws_s3_bucket_public_access_block" "uploads" {
bucket = aws_s3_bucket.uploads.id

block_public_acls = true
block_public_policy = true
ignore_public_acls = true
restrict_public_buckets = true
}

# Lifecycle rule to move old files to cheaper storage
resource "aws_s3_bucket_lifecycle_configuration" "uploads" {
bucket = aws_s3_bucket.uploads.id

rule {
id = "archive-old-uploads"
status = "Enabled"

transition {
days = 30
storage_class = "INTELLIGENT_TIERING"
}

transition {
days = 90
storage_class = "GLACIER"
}
}
}

# Lambda function to generate pre-signed URLs
resource "aws_lambda_function" "generate_upload_url" {
filename = data.archive_file.lambda.output_path
function_name = "generate-upload-url"
role = aws_iam_role.lambda.arn
handler = "index.handler"
runtime = "python3.11"
timeout = 10
source_code_hash = data.archive_file.lambda.output_base64sha256

environment {
variables = {
BUCKET_NAME = aws_s3_bucket.uploads.id
}
}
}

# Lambda code
data "archive_file" "lambda" {
type = "zip"
output_path = "${path.module}/lambda.zip"

source {
content = <<-EOF
import json
import boto3
import os
from datetime import timedelta

s3_client = boto3.client('s3')
BUCKET_NAME = os.environ['BUCKET_NAME']

def handler(event, context):
"""Generate pre-signed URL for file upload"""

try:
# Get filename from request
body = json.loads(event.get('body', '{}'))
filename = body.get('filename')
partner_id = body.get('partner_id')

if not filename or not partner_id:
return {
'statusCode': 400,
'body': json.dumps({'error': 'filename and partner_id required'})
}

# Create S3 key with partner prefix
s3_key = f"{partner_id}/{filename}"

# Generate pre-signed URL (valid for 1 hour)
presigned_url = s3_client.generate_presigned_url(
'put_object',
Params={
'Bucket': BUCKET_NAME,
'Key': s3_key,
'ContentType': body.get('content_type', 'application/octet-stream')
},
ExpiresIn=3600 # 1 hour
)

return {
'statusCode': 200,
'headers': {
'Content-Type': 'application/json',
'Access-Control-Allow-Origin': '*'
},
'body': json.dumps({
'upload_url': presigned_url,
's3_key': s3_key,
'expires_in': 3600
})
}

except Exception as e:
print(f"Error: {str(e)}")
return {
'statusCode': 500,
'body': json.dumps({'error': str(e)})
}
EOF
filename = "index.py"
}
}

# IAM role for Lambda
resource "aws_iam_role" "lambda" {
name = "upload-url-generator-role"

assume_role_policy = jsonencode({
Version = "2012-10-17"
Statement = [{
Action = "sts:AssumeRole"
Effect = "Allow"
Principal = {
Service = "lambda.amazonaws.com"
}
}]
})
}

# Lambda permissions
resource "aws_iam_role_policy" "lambda_s3" {
role = aws_iam_role.lambda.id

policy = jsonencode({
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Action = [
"s3:PutObject",
"s3:PutObjectAcl"
]
Resource = "${aws_s3_bucket.uploads.arn}/*"
},
{
Effect = "Allow"
Action = [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
]
Resource = "arn:aws:logs:*:*:*"
}
]
})
}

# API Gateway REST API
resource "aws_api_gateway_rest_api" "upload" {
name = "file-upload-api"
description = "API for generating S3 upload URLs"
}

resource "aws_api_gateway_resource" "upload" {
rest_api_id = aws_api_gateway_rest_api.upload.id
parent_id = aws_api_gateway_rest_api.upload.root_resource_id
path_part = "upload"
}

resource "aws_api_gateway_method" "post" {
rest_api_id = aws_api_gateway_rest_api.upload.id
resource_id = aws_api_gateway_resource.upload.id
http_method = "POST"
authorization = "AWS_IAM" # Use IAM auth or API keys
}

resource "aws_api_gateway_integration" "lambda" {
rest_api_id = aws_api_gateway_rest_api.upload.id
resource_id = aws_api_gateway_resource.upload.id
http_method = aws_api_gateway_method.post.http_method

integration_http_method = "POST"
type = "AWS_PROXY"
uri = aws_lambda_function.generate_upload_url.invoke_arn
}

resource "aws_api_gateway_deployment" "upload" {
rest_api_id = aws_api_gateway_rest_api.upload.id

depends_on = [
aws_api_gateway_integration.lambda
]

lifecycle {
create_before_destroy = true
}
}

resource "aws_api_gateway_stage" "prod" {
deployment_id = aws_api_gateway_deployment.upload.id
rest_api_id = aws_api_gateway_rest_api.upload.id
stage_name = "prod"
}

# Allow API Gateway to invoke Lambda
resource "aws_lambda_permission" "api_gateway" {
statement_id = "AllowAPIGatewayInvoke"
action = "lambda:InvokeFunction"
function_name = aws_lambda_function.generate_upload_url.function_name
principal = "apigateway.amazonaws.com"
source_arn = "${aws_api_gateway_rest_api.upload.execution_arn}/*/*"
}

# S3 event notification to trigger processing
resource "aws_lambda_function" "process_upload" {
filename = data.archive_file.processor.output_path
function_name = "process-uploaded-file"
role = aws_iam_role.processor.arn
handler = "index.handler"
runtime = "python3.11"
timeout = 60
source_code_hash = data.archive_file.processor.output_base64sha256
}

data "archive_file" "processor" {
type = "zip"
output_path = "${path.module}/processor.zip"

source {
content = <<-EOF
import json

def handler(event, context):
"""Process uploaded file"""

for record in event['Records']:
bucket = record['s3']['bucket']['name']
key = record['s3']['object']['key']

print(f"New file uploaded: s3://{bucket}/{key}")

# Add your processing logic here:
# - Parse CSV/JSON
# - Trigger ETL pipeline
# - Send notification
# - Validate file format

return {'statusCode': 200}
EOF
filename = "index.py"
}
}

resource "aws_iam_role" "processor" {
name = "upload-processor-role"

assume_role_policy = jsonencode({
Version = "2012-10-17"
Statement = [{
Action = "sts:AssumeRole"
Effect = "Allow"
Principal = {
Service = "lambda.amazonaws.com"
}
}]
})
}

resource "aws_iam_role_policy" "processor_s3" {
role = aws_iam_role.processor.id

policy = jsonencode({
Version = "2012-10-17"
Statement = [{
Effect = "Allow"
Action = [
"s3:GetObject",
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
]
Resource = "*"
}]
})
}

resource "aws_s3_bucket_notification" "upload_trigger" {
bucket = aws_s3_bucket.uploads.id

lambda_function {
lambda_function_arn = aws_lambda_function.process_upload.arn
events = ["s3:ObjectCreated:*"]
}

depends_on = [aws_lambda_permission.s3]
}

resource "aws_lambda_permission" "s3" {
statement_id = "AllowS3Invoke"
action = "lambda:InvokeFunction"
function_name = aws_lambda_function.process_upload.function_name
principal = "s3.amazonaws.com"
source_arn = aws_s3_bucket.uploads.arn
}

# Outputs
output "api_endpoint" {
value = "${aws_api_gateway_stage.prod.invoke_url}/upload"
}

output "bucket_name" {
value = aws_s3_bucket.uploads.id
}