Cross-Account Log Streaming

Prerequisites

AWS accounts with appropriate permissions
Terraform >= 1.0
Cross-account IAM roles configured

Step 1: Create Destination in Central Account

In your central logging account:

# Central logging account
module "central_logs" {
  source  = "registry.patterneddesigns.ca/essentials/cloudwatch-logs/aws"
  version = "1.3.0"

  log_group_name    = "/central/aggregated"
  retention_in_days = 365
  kms_key_arn       = module.logging_kms.key_arn
}

resource "aws_cloudwatch_log_destination" "central" {
  name       = "central-log-destination"
  role_arn   = aws_iam_role.logs_destination.arn
  target_arn = aws_kinesis_stream.logs.arn
}

resource "aws_cloudwatch_log_destination_policy" "central" {
  destination_name = aws_cloudwatch_log_destination.central.name
  access_policy    = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect = "Allow"
        Principal = {
          AWS = var.source_account_ids
        }
        Action   = "logs:PutSubscriptionFilter"
        Resource = aws_cloudwatch_log_destination.central.arn
      }
    ]
  })
}

Step 2: Create Log Group in Source Account

In each source account:

# Source account
module "app_logs" {
  source  = "registry.patterneddesigns.ca/essentials/cloudwatch-logs/aws"
  version = "1.3.0"

  log_group_name    = "/app/my-service"
  retention_in_days = 30
}

resource "aws_cloudwatch_log_subscription_filter" "central" {
  name            = "stream-to-central"
  log_group_name  = module.app_logs.log_group_name
  filter_pattern  = ""
  destination_arn = "arn:aws:logs:${var.region}:${var.central_account_id}:destination:central-log-destination"
  distribution    = "ByLogStream"
}

Step 3: Process Logs with Kinesis

# Central account - Kinesis stream and Lambda processor
resource "aws_kinesis_stream" "logs" {
  name             = "central-logs"
  shard_count      = 2
  retention_period = 24
}

resource "aws_lambda_function" "log_processor" {
  function_name = "log-processor"
  runtime       = "python3.12"
  handler       = "main.handler"
  # ... additional configuration
}

resource "aws_lambda_event_source_mapping" "kinesis" {
  event_source_arn  = aws_kinesis_stream.logs.arn
  function_name     = aws_lambda_function.log_processor.arn
  starting_position = "LATEST"
}

Step 4: Verify Streaming

Check that logs are flowing to the central account:

# In source account - generate test log
aws logs put-log-events \
  --log-group-name "/app/my-service" \
  --log-stream-name "test-stream" \
  --log-events timestamp=$(date +%s000),message="Test log message"

# In central account - verify log arrival
aws logs filter-log-events \
  --log-group-name "/central/aggregated" \
  --filter-pattern "Test log message"