Terraform Modules
Comprehensive guide to the Terraform infrastructure modules for the NorthBuilt RAG System.
Note: This system uses S3 Vectors with Bedrock Knowledge Base for vector storage. The Chat Lambda retrieves documents directly from Bedrock Knowledge Base - there is no separate Query Lambda.
S3 Vectors Configuration: The index is configured with
AMAZON_BEDROCK_TEXTandAMAZON_BEDROCK_METADATAas non-filterable metadata keys to avoid the 2KB filterable metadata limit. LLM parsing is disabled; metadata is provided via sidecar.metadata.jsonfiles.
Module Structure
terraform/
├── main.tf # Root module - orchestrates all resources
├── variables.tf # Input variables
├── outputs.tf # Output values
├── versions.tf # Provider versions
├── backend.tf # S3 + DynamoDB backend
├── modules/
│ ├── api_gateway/ # HTTP API + routes
│ ├── auth/ # Cognito user pool
│ ├── lambda/ # Lambda functions
│ ├── storage/ # S3 + DynamoDB
│ ├── secrets/ # Secrets Manager
│ └── web/ # CloudFront + S3 web hosting
└── terraform.tfvars # Variable values (gitignored)
Root Module
main.tf
Purpose: Orchestrate all infrastructure components
terraform {
required_version = ">= 1.13.0"
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 5.0"
}
}
backend "s3" {
bucket = "nb-rag-sys-terraform-state"
key = "terraform.tfstate"
region = "us-east-1"
dynamodb_table = "nb-rag-sys-terraform-locks"
encrypt = true
}
}
provider "aws" {
region = var.aws_region
default_tags {
tags = {
Project = "NorthBuilt-RAG"
Environment = var.environment
ManagedBy = "terraform"
}
}
}
locals {
project_prefix = "nb-rag-sys"
region = var.aws_region
}
# Secrets Manager secrets
module "secrets" {
source = "./modules/secrets"
project_prefix = local.project_prefix
fathom_api_key = var.fathom_api_key_value
helpscout_api_key = var.helpscout_api_key_value
linear_api_key = var.linear_api_key_value
google_client_secret = var.google_client_secret
}
# DynamoDB tables
module "storage" {
source = "./modules/storage"
project_prefix = local.project_prefix
}
# Cognito user pool
module "auth" {
source = "./modules/auth"
project_prefix = local.project_prefix
google_client_id = var.google_client_id
google_client_secret = module.secrets.google_client_secret_arn
callback_urls = var.cognito_callback_urls
}
# Lambda functions
module "lambda" {
source = "./modules/lambda"
project_prefix = local.project_prefix
region = local.region
knowledge_base_id = var.knowledge_base_id
fathom_api_key_arn = module.secrets.fathom_api_key_arn
helpscout_api_key_arn = module.secrets.helpscout_api_key_arn
linear_api_key_arn = module.secrets.linear_api_key_arn
classify_table_name = module.storage.classify_table_name
}
# API Gateway
module "api_gateway" {
source = "./modules/api_gateway"
project_prefix = local.project_prefix
cognito_user_pool_id = module.auth.user_pool_id
cognito_user_pool_client_id = module.auth.user_pool_client_id
chat_lambda_arn = module.lambda.chat_lambda_arn
chat_lambda_name = module.lambda.chat_lambda_name
fathom_webhook_lambda_arn = module.lambda.fathom_webhook_lambda_arn
fathom_webhook_lambda_name = module.lambda.fathom_webhook_lambda_name
# ... other webhook lambdas
}
# Web hosting
module "web" {
source = "./modules/web"
project_prefix = local.project_prefix
domain_name = var.domain_name
acm_certificate_arn = var.acm_certificate_arn
api_gateway_url = module.api_gateway.api_endpoint
}
variables.tf
variable "aws_region" {
description = "AWS region for all resources"
type = string
default = "us-east-1"
}
variable "environment" {
description = "Environment name (dev, staging, production)"
type = string
default = "production"
}
variable "knowledge_base_id" {
description = "Bedrock Knowledge Base ID"
type = string
}
variable "fathom_api_key_value" {
description = "Fathom API key"
type = string
sensitive = true
default = ""
}
variable "helpscout_api_key_value" {
description = "HelpScout API key"
type = string
sensitive = true
default = ""
}
variable "linear_api_key_value" {
description = "Linear API key"
type = string
sensitive = true
default = ""
}
variable "google_client_id" {
description = "Google OAuth client ID"
type = string
}
variable "google_client_secret" {
description = "Google OAuth client secret"
type = string
sensitive = true
}
variable "cognito_callback_urls" {
description = "Cognito callback URLs"
type = list(string)
default = ["http://localhost:8080/callback.html"]
}
variable "domain_name" {
description = "Custom domain name for web UI (optional)"
type = string
default = ""
}
variable "acm_certificate_arn" {
description = "ACM certificate ARN for custom domain (optional)"
type = string
default = ""
}
outputs.tf
output "api_endpoint" {
description = "API Gateway endpoint URL"
value = module.api_gateway.api_endpoint
}
output "web_url" {
description = "Web UI URL"
value = module.web.cloudfront_url
}
output "cognito_user_pool_id" {
description = "Cognito user pool ID"
value = module.auth.user_pool_id
}
output "cognito_client_id" {
description = "Cognito client ID"
value = module.auth.user_pool_client_id
}
output "cognito_domain" {
description = "Cognito hosted UI domain"
value = module.auth.user_pool_domain
}
Lambda Module
modules/lambda/main.tf
Purpose: Create and configure all Lambda functions
# Chat Lambda
resource "aws_lambda_function" "chat" {
function_name = "${var.project_prefix}-chat"
role = aws_iam_role.chat.arn
handler = "handler.lambda_handler"
runtime = "python3.13"
timeout = 60
memory_size = 1024
filename = data.archive_file.chat.output_path
source_code_hash = data.archive_file.chat.output_base64sha256
environment {
variables = {
KNOWLEDGE_BASE_ID = var.knowledge_base_id
BEDROCK_LLM_MODEL = "us.anthropic.claude-sonnet-4-5-20250929-v1:0"
DYNAMODB_TABLE = var.classify_table_name
ENABLE_QUERY_UNDERSTANDING = "true"
AWS_REGION = var.region
}
}
tracing_config {
mode = "Active" # X-Ray tracing
}
reserved_concurrent_executions = 10 # Prevent runaway costs
tags = {
Component = "compute"
}
}
# Package Lambda code
data "archive_file" "chat" {
type = "zip"
source_dir = "${path.root}/../lambda/chat"
output_path = "${path.module}/chat.zip"
excludes = [
"__pycache__",
"*.pyc",
".venv",
"venv",
"tests"
]
}
# IAM role for Chat Lambda
resource "aws_iam_role" "chat" {
name = "${var.project_prefix}-chat-lambda-role"
assume_role_policy = jsonencode({
Version = "2012-10-17"
Statement = [
{
Action = "sts:AssumeRole"
Effect = "Allow"
Principal = {
Service = "lambda.amazonaws.com"
}
}
]
})
}
# IAM policy for Chat Lambda
resource "aws_iam_role_policy" "chat" {
name = "${var.project_prefix}-chat-lambda-policy"
role = aws_iam_role.chat.id
policy = jsonencode({
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Action = [
"bedrock:InvokeModel"
]
Resource = [
"arn:aws:bedrock:${var.region}::foundation-model/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
"arn:aws:bedrock:${var.region}::foundation-model/us.anthropic.claude-3-5-haiku-20241022-v1:0"
]
},
{
Effect = "Allow"
Action = [
"bedrock:Retrieve"
]
Resource = [
"arn:aws:bedrock:${var.region}:*:knowledge-base/${var.knowledge_base_id}"
]
},
{
Effect = "Allow"
Action = [
"dynamodb:Query",
"dynamodb:Scan"
]
Resource = [
"arn:aws:dynamodb:${var.region}:*:table/${var.project_prefix}-classify",
"arn:aws:dynamodb:${var.region}:*:table/${var.project_prefix}-classify/index/*"
]
},
{
Effect = "Allow"
Action = [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
]
Resource = [
"arn:aws:logs:${var.region}:*:log-group:/aws/lambda/${var.project_prefix}-chat:*"
]
},
{
Effect = "Allow"
Action = [
"xray:PutTraceSegments",
"xray:PutTelemetryRecords"
]
Resource = ["*"]
}
]
})
}
# CloudWatch log group
resource "aws_cloudwatch_log_group" "chat" {
name = "/aws/lambda/${var.project_prefix}-chat"
retention_in_days = 7
tags = {
Component = "logs"
}
}
modules/lambda/variables.tf
variable "project_prefix" {
description = "Project prefix for resource naming"
type = string
}
variable "region" {
description = "AWS region"
type = string
}
variable "knowledge_base_id" {
description = "Bedrock Knowledge Base ID"
type = string
}
variable "fathom_api_key_arn" {
description = "ARN of Fathom API key secret"
type = string
}
variable "helpscout_api_key_arn" {
description = "ARN of HelpScout API key secret"
type = string
}
variable "linear_api_key_arn" {
description = "ARN of Linear API key secret"
type = string
}
variable "classify_table_name" {
description = "DynamoDB classify table name"
type = string
}
modules/lambda/outputs.tf
output "chat_lambda_arn" {
description = "Chat Lambda ARN"
value = aws_lambda_function.chat.arn
}
output "chat_lambda_name" {
description = "Chat Lambda function name"
value = aws_lambda_function.chat.function_name
}
output "classify_lambda_arn" {
description = "Classify Lambda ARN"
value = aws_lambda_function.classify.arn
}
output "fathom_webhook_lambda_arn" {
description = "Fathom webhook Lambda ARN"
value = aws_lambda_function.fathom_webhook.arn
}
output "fathom_webhook_lambda_name" {
description = "Fathom webhook Lambda function name"
value = aws_lambda_function.fathom_webhook.function_name
}
API Gateway Module
modules/api_gateway/main.tf
Purpose: Create HTTP API with routes and authorizer
# HTTP API
resource "aws_apigatewayv2_api" "main" {
name = "${var.project_prefix}-api"
protocol_type = "HTTP"
cors_configuration {
allow_origins = var.cors_allow_origins
allow_methods = ["POST", "OPTIONS"]
allow_headers = ["Authorization", "Content-Type"]
max_age = 300
allow_credentials = true
}
tags = {
Component = "api"
}
}
# JWT Authorizer (Cognito)
resource "aws_apigatewayv2_authorizer" "cognito" {
api_id = aws_apigatewayv2_api.main.id
authorizer_type = "JWT"
identity_sources = ["$request.header.Authorization"]
name = "cognito-authorizer"
jwt_configuration {
audience = [var.cognito_user_pool_client_id]
issuer = "https://cognito-idp.${data.aws_region.current.name}.amazonaws.com/${var.cognito_user_pool_id}"
}
}
# Chat route
resource "aws_apigatewayv2_integration" "chat" {
api_id = aws_apigatewayv2_api.main.id
integration_type = "AWS_PROXY"
integration_method = "POST"
integration_uri = var.chat_lambda_arn
payload_format_version = "2.0"
}
resource "aws_apigatewayv2_route" "chat" {
api_id = aws_apigatewayv2_api.main.id
route_key = "POST /chat"
authorization_type = "JWT"
authorizer_id = aws_apigatewayv2_authorizer.cognito.id
target = "integrations/${aws_apigatewayv2_integration.chat.id}"
}
# Lambda permission for API Gateway
resource "aws_lambda_permission" "chat" {
statement_id = "AllowAPIGatewayInvoke"
action = "lambda:InvokeFunction"
function_name = var.chat_lambda_name
principal = "apigateway.amazonaws.com"
source_arn = "${aws_apigatewayv2_api.main.execution_arn}/*/*/chat"
}
# Production stage
resource "aws_apigatewayv2_stage" "production" {
api_id = aws_apigatewayv2_api.main.id
name = "production"
auto_deploy = true
default_route_settings {
throttling_rate_limit = var.throttle_rate_limit
throttling_burst_limit = var.throttle_burst_limit
}
access_log_settings {
destination_arn = aws_cloudwatch_log_group.api.arn
format = jsonencode({
requestId = "$context.requestId"
ip = "$context.identity.sourceIp"
requestTime = "$context.requestTime"
httpMethod = "$context.httpMethod"
routeKey = "$context.routeKey"
status = "$context.status"
protocol = "$context.protocol"
responseLength = "$context.responseLength"
integrationErrorMessage = "$context.integrationErrorMessage"
errorMessage = "$context.error.message"
})
}
tags = {
Component = "api"
}
}
# CloudWatch log group for API Gateway
resource "aws_cloudwatch_log_group" "api" {
name = "/aws/apigateway/${var.project_prefix}"
retention_in_days = 7
tags = {
Component = "logs"
}
}
data "aws_region" "current" {}
modules/api_gateway/variables.tf
variable "project_prefix" {
description = "Project prefix for resource naming"
type = string
}
variable "cognito_user_pool_id" {
description = "Cognito user pool ID"
type = string
}
variable "cognito_user_pool_client_id" {
description = "Cognito user pool client ID"
type = string
}
variable "chat_lambda_arn" {
description = "Chat Lambda ARN"
type = string
}
variable "chat_lambda_name" {
description = "Chat Lambda function name"
type = string
}
variable "cors_allow_origins" {
description = "CORS allowed origins"
type = list(string)
default = ["http://localhost:8080"]
}
variable "throttle_rate_limit" {
description = "API Gateway throttle rate limit (requests per second)"
type = number
default = 10
}
variable "throttle_burst_limit" {
description = "API Gateway throttle burst limit"
type = number
default = 20
}
modules/api_gateway/outputs.tf
output "api_endpoint" {
description = "API Gateway endpoint URL"
value = aws_apigatewayv2_stage.production.invoke_url
}
output "api_id" {
description = "API Gateway ID"
value = aws_apigatewayv2_api.main.id
}
Storage Module
modules/storage/main.tf
Purpose: Create DynamoDB tables
# Classify results table
resource "aws_dynamodb_table" "classify" {
name = "${var.project_prefix}-classify"
billing_mode = "PAY_PER_REQUEST"
hash_key = "document_id"
attribute {
name = "document_id"
type = "S"
}
point_in_time_recovery {
enabled = true
}
server_side_encryption {
enabled = true
}
tags = {
Component = "database"
}
}
# Terraform state lock table (created separately during bootstrap)
# This is just documentation - actual table created by bootstrap script
modules/storage/outputs.tf
output "classify_table_name" {
description = "DynamoDB classify table name"
value = aws_dynamodb_table.classify.name
}
output "classify_table_arn" {
description = "DynamoDB classify table ARN"
value = aws_dynamodb_table.classify.arn
}
Secrets Module
modules/secrets/main.tf
Purpose: Store API keys and credentials securely
# Fathom API key
resource "aws_secretsmanager_secret" "fathom" {
count = var.fathom_api_key != "" ? 1 : 0
name = "${var.project_prefix}-fathom-api-key"
description = "Fathom API key for video transcripts"
recovery_window_in_days = 7
tags = {
Component = "secrets"
}
}
resource "aws_secretsmanager_secret_version" "fathom" {
count = var.fathom_api_key != "" ? 1 : 0
secret_id = aws_secretsmanager_secret.fathom[0].id
secret_string = jsonencode({
api_key = var.fathom_api_key
})
}
# Similar for HelpScout, Linear, Google OAuth secret...
modules/secrets/outputs.tf
output "fathom_api_key_arn" {
description = "Fathom API key secret ARN"
value = var.fathom_api_key != "" ? aws_secretsmanager_secret.fathom[0].arn : ""
}
output "google_client_secret_arn" {
description = "Google OAuth client secret ARN"
value = aws_secretsmanager_secret.google_oauth.arn
}
Best Practices
Module Design Principles
- Single Responsibility: Each module manages one logical component
- Loose Coupling: Modules communicate via outputs/variables
- DRY (Don’t Repeat Yourself): Reusable modules for similar resources
- Versioning: Use Git tags for module versions
- Documentation: README.md in each module directory
Naming Conventions
Resource naming: ${project_prefix}-${component}-${resource_type}
Examples:
- nb-rag-sys-chat-lambda
- nb-rag-sys-api-gw
- nb-rag-sys-classify-table
Tagging Strategy
default_tags {
tags = {
Project = "NorthBuilt-RAG"
Environment = var.environment
ManagedBy = "terraform"
Component = "api|compute|storage|logs|secrets"
CostCenter = "engineering"
}
}
State Management
Remote State with Locking:
backend "s3" {
bucket = "nb-rag-sys-terraform-state"
key = "terraform.tfstate"
region = "us-east-1"
dynamodb_table = "nb-rag-sys-terraform-locks"
encrypt = true
}
State Locking:
- Prevents concurrent modifications
- Uses DynamoDB for atomic operations
- Automatically handled by Terraform
State Encryption:
- S3 server-side encryption (AES-256)
- Secrets stored encrypted
- State file versioning enabled
Drift Detection
Manual Drift Check:
terraform plan -detailed-exitcode
# Exit code 2 = drift detected
Automated Drift Detection (GitHub Actions):
name: Terraform Drift Detection
on:
schedule:
- cron: '0 0 * * *' # Daily at midnight
jobs:
drift:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: aws-actions/configure-aws-credentials@v4
- name: Terraform Plan
run: terraform plan -detailed-exitcode
- name: Notify on Drift
if: failure()
run: |
# Send Slack notification
curl -X POST $ \
-d '{"text": "Terraform drift detected!"}'
Last updated: 2025-12-31