AKS - I was expecting the POD IPs and Cluster IPs in the "aks-subnet" range ["10.2.4.-CodePudding

I've created a Kubernetes cluster in Azure using the following Terraform. Azure Kubernetes Service Cluster got created with Application Gateway.

# Locals block for hardcoded names
locals {
  backend_address_pool_name      = "appgateway-beap"
  frontend_port_name             = "appgateway-feport"
  frontend_ip_configuration_name = "appgateway-feip"
  http_setting_name              = "appgateway-be-htst"
  listener_name                  = "appgateway-httplstn"
  request_routing_rule_name      = "appgateway-rqrt"
  app_gateway_subnet_name        = "appgateway-subnet"
}

resource "azurerm_subnet" "aks_subnet" {
  name                 = "aks-subnet"
  resource_group_name  = azurerm_resource_group.ipz12-dat-np-connection-rg.name
  virtual_network_name = azurerm_virtual_network.spoke_vnet.name
  address_prefixes     = ["10.2.4.0/24"]
  private_endpoint_network_policies_enabled = false
  
  depends_on = [
    azurerm_virtual_network.spoke_vnet
  ]  
}

resource "azurerm_subnet" "applicationgateway_subnet" {
  name                 = "appgateway-subnet"
  resource_group_name  = azurerm_resource_group.ipz12-dat-np-connection-rg.name
  virtual_network_name = azurerm_virtual_network.spoke_vnet.name
  address_prefixes     = ["10.2.5.0/24"]
  private_endpoint_network_policies_enabled = false
  
  depends_on = [
    azurerm_virtual_network.spoke_vnet
  ]  
}

# Create Resource Group for Kubernetes Cluster
module "resource_group_kubernetes_cluster" {
  source                  = "./modules/resource_group"
  count                   = var.enable_kubernetes == true ? 1 : 0
  #name_override          = "rg-aks-spoke-dev-westus3-001"
  app_or_service_name     = "aks"                                   # var.app_or_service_name
  subscription_type       = var.subscription_type                   # "spoke"   
  environment             = var.environment                         # "dev"    
  location                = var.location                            # "westus3"
  instance_number         = var.instance_number                     # "001"    
  tags                    = var.tags
}

resource "azurerm_user_assigned_identity" "identity_uami" {
  location            = var.location
  name                = "appgw-uami"
  resource_group_name = module.resource_group_kubernetes_cluster[0].name
}

# Application Gateway Public Ip 
resource "azurerm_public_ip" "test" {
  name                = "publicIp1"
  location            = var.location
  resource_group_name = module.resource_group_kubernetes_cluster[0].name
  allocation_method   = "Static"
  sku                 = "Standard"
}

resource "azurerm_application_gateway" "network" {
  name                = var.app_gateway_name
  resource_group_name = module.resource_group_kubernetes_cluster[0].name
  location            = var.location

  sku {
    name     = var.app_gateway_sku
    tier     = "Standard_v2"
    capacity = 2
  }

  identity {
    type = "UserAssigned"
    identity_ids = [
      azurerm_user_assigned_identity.identity_uami.id
    ]
  }

  gateway_ip_configuration {
    name      = "appGatewayIpConfig"
    subnet_id = azurerm_subnet.appgateway-subnet.id
  }

  frontend_port {
    name = local.frontend_port_name
    port = 80
  }

  frontend_port {
    name = "httpsPort"
    port = 443
  }

  frontend_ip_configuration {
    name                 = local.frontend_ip_configuration_name
    public_ip_address_id = azurerm_public_ip.test.id
  }

  backend_address_pool {
    name = local.backend_address_pool_name
  }

  backend_http_settings {
    name                  = local.http_setting_name
    cookie_based_affinity = "Disabled"
    port                  = 80
    protocol              = "Http"
    request_timeout       = 1
  }

  http_listener {
    name                           = local.listener_name
    frontend_ip_configuration_name = local.frontend_ip_configuration_name
    frontend_port_name             = local.frontend_port_name
    protocol                       = "Http"
  }

  request_routing_rule {
    name                       = local.request_routing_rule_name
    rule_type                  = "Basic"
    http_listener_name         = local.listener_name
    backend_address_pool_name  = local.backend_address_pool_name
    backend_http_settings_name = local.http_setting_name
    priority                   = 100
  }

  tags = var.tags

  depends_on = [azurerm_public_ip.test]

  lifecycle {
    ignore_changes = [
      backend_address_pool,
      backend_http_settings,
      request_routing_rule,
      http_listener,
      probe,
      tags,
      frontend_port
    ]
  }
}

# Create the Azure Kubernetes Service (AKS) Cluster
resource "azurerm_kubernetes_cluster" "kubernetes_cluster" {
  count                         = var.enable_kubernetes == true ? 1 : 0
  name                          = "aks-prjx-${var.subscription_type}-${var.environment}-${var.location}-${var.instance_number}"    
  location                      = var.location
  resource_group_name           = module.resource_group_kubernetes_cluster[0].name  # "rg-aks-spoke-dev-westus3-001"
  dns_prefix                    = "dns-aks-prjx-${var.subscription_type}-${var.environment}-${var.location}-${var.instance_number}" #"dns-prjxcluster"
  private_cluster_enabled       = false
  local_account_disabled        = true

  default_node_pool {
    name                        = "npprjx${var.subscription_type}" #"prjxsyspool" # NOTE: "name must start with a lowercase letter, have max length of 12, and only have characters a-z0-9."
    vm_size                     = "Standard_B8ms"
    vnet_subnet_id              = azurerm_subnet.aks-subnet.id
    # zones                     = ["1", "2", "3"]
    enable_auto_scaling         = true
    max_count                   = 3
    min_count                   = 1
    # node_count                = 3
    os_disk_size_gb             = 50
    type                        = "VirtualMachineScaleSets"
    enable_node_public_ip       = false
    enable_host_encryption      = false

    node_labels = {
      "node_pool_type"          = "npprjx${var.subscription_type}"
      "node_pool_os"            = "linux"
      "environment"             = "${var.environment}"
      "app"                     = "prjx_${var.subscription_type}_app"
    }
    tags = var.tags
  }

  ingress_application_gateway {
    gateway_id = azurerm_application_gateway.network.id
  }

  # Enabled the cluster configuration to the Azure kubernets with RBAC
  azure_active_directory_role_based_access_control { 
    managed                     = true
    admin_group_object_ids      = var.active_directory_role_based_access_control_admin_group_object_ids
    azure_rbac_enabled          = true #false
  }

  network_profile {
    network_plugin              = "azure"
    network_policy              = "azure"
    outbound_type               = "userDefinedRouting"
  }

  identity {
    type = "SystemAssigned"
  }  

  oms_agent {
    log_analytics_workspace_id  = module.log_analytics_workspace[0].id
  }

  timeouts {
    create = "20m"
    delete = "20m"
  }

  depends_on = [
    azurerm_application_gateway.network
  ]
}

I was expecting the POD IPs and Cluster IPs in the "aks-subnet" range ["10.2.4.0/24"] as I have used the Azure CNI networking. However, the POD IP is different. Am I missing something?

CodePudding user response：

In your above screenshot, you have listed Kubernetes services and not pods. In Kubernetes, pods and services are different.

$ kubectl get services #command will list services.

$ kubectl get pods #command will list pods

Kindly refer below screenshot.

While creating the Kubernetes cluster, you have options to define cluster subnet range and Kubernetes service address range. All the cluster Pods and Nodes will become part of the cluster subnet (in your case, it is 10.2.4.0/24). All the cluster services will become part of the Kubernetes Service address range (or service CIDR). In the above terraform code, it seems you didn’t mention anything about Service CIDR address range. So, it creates default range 10.0.0.0/16. To see the service CIDR range. navigate to your k8s cluster > networking in left blade as shown below.

enter image description here

Kubernetes services will pick IPs from the service CIDR range and not from the cluster subnet.