Deploy Kafka to Cloud

# AWS MSK Cluster Configuration using Terraform
resource "aws_msk_cluster" "kafka" {
  cluster_name           = "production-kafka"
  kafka_version          = "3.4.0"
  number_of_broker_nodes = 3

  broker_node_group_info {
    instance_type   = "kafka.m5.large"
    client_subnets  = aws_subnet.private[*].id
    security_groups = [aws_security_group.kafka.id]
    storage_info {
      ebs_storage_info {
        volume_size = 1000
      }
    }
  }

  encryption_info {
    encryption_at_rest_kms_key_arn = aws_kms_key.kafka.arn
    encryption_in_cluster {
      data_volume_kms_key_arn = aws_kms_key.kafka.arn
    }
  }

  configuration_info {
    arn      = aws_msk_configuration.kafka.arn
    revision = 1
  }

  logging_info {
    broker_logs {
      cloudwatch_logs {
        enabled   = true
        log_group = aws_cloudwatch_log_group.kafka.name
      }
      s3 {
        enabled = true
        bucket  = aws_s3_bucket.logs.id
        prefix  = "kafka-logs/"
      }
    }
  }

  tags = {
    Environment = "production"
    Project     = "event-streaming"
  }
}

# MSK Configuration
resource "aws_msk_configuration" "kafka" {
  name           = "kafka-config"
  kafka_versions = ["3.4.0"]

  server_properties = <<PROPERTIES
auto.create.topics.enable=false
default.replication.factor=3
min.insync.replicas=2
log.retention.hours=168
num.partitions=12
PROPERTIES
}