I'm struggling to deploy my EKS node/node-group using terraform-CodePudding

I was getting this error first

NodeCreationFailure: Instances failed to 
join the kubernetes cluster

and I didn't have my private subnets tagged right. I found examples online where they tagged their vpc and subnet a certain way, so I copied that and now I'm getting this error

Error: Cycle: aws_eks_cluster.eks, aws_subnet.private_subnet

This is frustrating, but here's my main.tf file condensed to all of the relevant resource blocks. This is my entire vpc section, since I feel like it could be anything in here based off other posts. Also for context, I'm trying to deploy the cluster inside private subnets.

resource "aws_vpc" "vpc" {
  cidr_block = "10.1.0.0/16"

  tags = {
    "kubernetes.io/cluster/${var.cluster_name}" = "shared"
  }

}
resource "aws_subnet" "public_subnet" {
  count                   = length(var.azs)
  vpc_id                  = aws_vpc.vpc.id
  cidr_block              = var.public_cidrs[count.index]
  availability_zone       = var.azs[count.index]
  map_public_ip_on_launch = true

  tags = {
    Name = "${var.name}-public-subnet-${count.index   1}"
  }
}

resource "aws_subnet" "private_subnet" {
  count                   = length(var.azs)
  vpc_id                  = aws_vpc.vpc.id
  cidr_block              = var.private_cidrs[count.index]
  availability_zone       = var.azs[count.index]
  map_public_ip_on_launch = false

  tags = {
    "kubernetes.io/cluster/${aws_eks_cluster.eks.name}" = "shared"
    "kubernetes.io/role/internal-elb"           = "1"
  }
}

resource "aws_internet_gateway" "internet_gateway" {
  vpc_id = aws_vpc.vpc.id

  tags = {
    Name = "${var.name}-internet-gateway"
  }
}

resource "aws_route_table" "public_rt" {
  vpc_id = aws_vpc.vpc.id

  tags = {
    Name = "${var.name}-public-rt"
  }
}

resource "aws_route" "default_route" {
  route_table_id         = aws_route_table.public_rt.id
  destination_cidr_block = "0.0.0.0/0"
  gateway_id             = aws_internet_gateway.internet_gateway.id
}

resource "aws_route_table_association" "public_assoc" {
  count          = length(var.public_cidrs)
  subnet_id      = aws_subnet.public_subnet[count.index].id
  route_table_id = aws_route_table.public_rt.id
}

resource "aws_eip" "nat_eip" {
  count      = length(var.public_cidrs)
  vpc        = true
  depends_on = [aws_internet_gateway.internet_gateway]

  tags = {
    Name = "${var.name}-nat-eip-${count.index   1}"
  }
}

resource "aws_nat_gateway" "nat_gateway" {
  count         = length(var.public_cidrs)
  allocation_id = aws_eip.nat_eip[count.index].id
  subnet_id     = aws_subnet.public_subnet[count.index].id
  depends_on    = [aws_internet_gateway.internet_gateway]

  tags = {
    Name = "${var.name}-NAT-gateway-${count.index   1}"
  }
}

Here's all of my source blocks related to my cluster and nodes

resource "aws_iam_role" "eks_cluster" {
  name = "${var.name}-eks-cluster-role"

  assume_role_policy = <<POLICY
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Principal": {
        "Service": "eks.amazonaws.com"
      },
      "Action": "sts:AssumeRole"
    }
  ]
}
POLICY
}

resource "aws_iam_role_policy_attachment" "amazon_eks_cluster_policy" {
  policy_arn = "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy"
  role       = aws_iam_role.eks_cluster.name
}

resource "aws_eks_cluster" "eks" {
  name     = var.cluster_name
  role_arn = aws_iam_role.eks_cluster.arn

  ## k8s Version
  version = var.k8s_version

  vpc_config {
    endpoint_private_access = true
    endpoint_public_access  = false
    subnet_ids              = [
        aws_subnet.private_subnet[0].id,
        aws_subnet.private_subnet[1].id,
        aws_subnet.private_subnet[2].id,
    ]
  }
  depends_on = [
    aws_iam_role_policy_attachment.amazon_eks_cluster_policy
  ]
}

resource "aws_iam_role" "nodes_eks" {
  name               = "role-node-group-eks"
  assume_role_policy = <<POLICY
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Principal": {
        "Service": "ec2.amazonaws.com"
      }, 
      "Action": "sts:AssumeRole"
    }
  ]
}
POLICY
}

resource "aws_iam_role_policy_attachment" "amazon_eks_worker_node_policy_eks" {
  policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy"
  role       = aws_iam_role.nodes_eks.name
}

resource "aws_iam_role_policy_attachment" "amazon_eks_cni_policy_eks" {
  policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"
  role       = aws_iam_role.nodes_eks.name
}

resource "aws_iam_role_policy_attachment" "amazon_ec2_container_registry_read_only" {
  policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
  role       = aws_iam_role.nodes_eks.name
}


resource "aws_eks_node_group" "nodes_eks" {
  cluster_name    = aws_eks_cluster.eks.name
  node_group_name = "${var.name}-node-group"
  node_role_arn   = aws_iam_role.nodes_eks.arn
  subnet_ids      = [
        aws_subnet.private_subnet[0].id,
        aws_subnet.private_subnet[1].id,
        aws_subnet.private_subnet[2].id,
  ]
  remote_access {
    ec2_ssh_key = aws_key_pair.bastion_auth.id

  }

  scaling_config {
    desired_size = 3
    max_size     = 6
    min_size     = 3
  }

  ami_type       = "AL2_x86_64"
  capacity_type  = "ON_DEMAND"
  disk_size      = 20
  instance_types = [var.instance_type]
  labels = {
    role = "nodes-group-1"
  }

  version = var.k8s_version

  depends_on = [
    aws_iam_role_policy_attachment.amazon_eks_worker_node_policy_eks,
    aws_iam_role_policy_attachment.amazon_eks_cni_policy_eks,
    aws_iam_role_policy_attachment.amazon_ec2_container_registry_read_only,
  ]
}

CodePudding user response：

In the private-subnet resource, you are referencing your EKS cluster in the tag: ${aws_eks_cluster.eks.name}, which creates a dependency for this resource on EKS cluster.

resource "aws_subnet" "private_subnet" {
  count                   = length(var.azs)
  vpc_id                  = aws_vpc.vpc.id
  cidr_block              = var.private_cidrs[count.index]
  availability_zone       = var.azs[count.index]
  map_public_ip_on_launch = false

  tags = {
    "kubernetes.io/cluster/${aws_eks_cluster.eks.name}" = "shared" <- this creates dependency
    "kubernetes.io/role/internal-elb"                   = "1"
  }
}

On the other side, you are referencing the same private subnet, when you create the EKS cluster, which now creates a dependency for this resource on the private subnet.

resource "aws_eks_cluster" "eks" {
  name     = var.cluster_name
  role_arn = aws_iam_role.eks_cluster.arn

  ## k8s Version
  version = var.k8s_version

  vpc_config {
    endpoint_private_access = true
    endpoint_public_access  = false
    subnet_ids              = [
        aws_subnet.private_subnet[0].id, <- this creates dependency
        aws_subnet.private_subnet[1].id, <- this creates dependency
        aws_subnet.private_subnet[2].id, <- this creates dependency
    ]
  }
  depends_on = [
    aws_iam_role_policy_attachment.amazon_eks_cluster_policy
  ]
}

And in a result, you get a dependency cycle that causes your error.

To solve it, update the tag for the private subnet to:

  tags = {
    "kubernetes.io/cluster/${var.cluster_name}" = "shared"
    "kubernetes.io/role/internal-elb"           = "1"
  }