502 Bad Gateway from Azure Application Gateway Connecting to Azure Container Instance-CodePudding

I am working on learning Terraform and Azure Web Services. After following a series of tutorials, I've been working on getting an Azure Container Instance setup that talks to a CosmosDB instance within a virtual network, and I want an Application Gateway setup that will allow HTTP connections to the Azure Container Instance.

Currently, when I call the IP address assigned to the Application Gateway, I receive a 502 Bad Gateway. I've verified that the image I'm running in the Azure Container Instance works locally. I have a feeling that the issues I'm facing are in relation to the back-end address pool I've configured, and possibly an issue with the rules I've setup in my network security group (nsg-myapp).

I was wondering if someone could look at my Terraform and identify what I've not configured correctly? The closest question I found similar to my scenario on StackOverflow as this unresolved question from last year.

network.tf

resource "azurerm_virtual_network" "myappdb" {
  name                = "myappdb-vnet"
  address_space       = ["10.7.0.0/16"]
  location            = azurerm_resource_group.rg.location
  resource_group_name = azurerm_resource_group.rg.name
}

resource "azurerm_subnet" "internal" {
  name                 = "internal"
  resource_group_name  = azurerm_resource_group.rg.name
  virtual_network_name = azurerm_virtual_network.myappdb.name
  address_prefixes     = ["10.7.2.0/24"]
  service_endpoints    = ["Microsoft.AzureCosmosDB"]

  delegation {
    name = "acidelegationservice"

    service_delegation {
      name    = "Microsoft.ContainerInstance/containerGroups"
      actions = ["Microsoft.Network/virtualNetworks/subnets/join/action", "Microsoft.Network/virtualNetworks/subnets/prepareNetworkPolicies/action"]
    }
  }

  enforce_private_link_endpoint_network_policies = true
}

resource "azurerm_subnet" "frontend" {
  name                 = "myapp-frontend"
  resource_group_name  = azurerm_resource_group.rg.name
  virtual_network_name = azurerm_virtual_network.myappdb.name
  address_prefixes     = ["10.7.0.0/24"]
}

resource "azurerm_network_security_group" "nsg-myapp" {
  name                = "nsg-aci"
  location            = azurerm_resource_group.rg.location
  resource_group_name = azurerm_resource_group.rg.name

  security_rule {
    name              = "from-gateway-subnet"
    priority          = 100
    direction         = "Inbound"
    access            = "Allow"
    protocol          = "Tcp"
    source_port_range = "*"

    destination_port_ranges    = [22, 80, 443, 445, 8000]
    source_address_prefixes    = azurerm_subnet.internal.address_prefixes
    destination_address_prefix = azurerm_subnet.internal.address_prefixes[0]
  }

  security_rule {
    name                       = "DenyAllInBound-Override"
    priority                   = 900
    direction                  = "Inbound"
    access                     = "Deny"
    protocol                   = "*"
    source_port_range          = "*"
    destination_port_range     = "*"
    source_address_prefix      = "*"
    destination_address_prefix = "*"
  }


  security_rule {
    name              = "to-internet"
    priority          = 100
    direction         = "Outbound"
    access            = "Allow"
    protocol          = "Tcp"
    source_port_range = "*"

    destination_port_ranges    = [80, 443, 445]
    source_address_prefix      = "*"
    destination_address_prefix = "*"
  }

  security_rule {
    name                       = "DenyAllOutBound-Override"
    priority                   = 900
    direction                  = "Outbound"
    access                     = "Deny"
    protocol                   = "*"
    source_port_range          = "*"
    destination_port_range     = "*"
    source_address_prefix      = "*"
    destination_address_prefix = "*"
  }
}

resource "azurerm_subnet_network_security_group_association" "sn-nsg-aci" {
  subnet_id                 = azurerm_subnet.internal.id
  network_security_group_id = azurerm_network_security_group.nsg-myapp.id
}

resource "azurerm_network_profile" "containergroup_profile" {
  name                = "acg-profile"
  location            = azurerm_resource_group.rg.location
  resource_group_name = azurerm_resource_group.rg.name

  container_network_interface {
    name = "acg-nic"

    ip_configuration {
      name      = "aciipconfig"
      subnet_id = azurerm_subnet.internal.id
    }
  }
}

resource "azurerm_public_ip" "myappip" {
  name                = "myappip"
  resource_group_name = azurerm_resource_group.rg.name
  location            = azurerm_resource_group.rg.location
  allocation_method   = "Static"
  sku                 = "Standard"
}

locals {
  backend_address_pool_name      = "${azurerm_virtual_network.myappdb.name}-beap"
  frontend_port_name             = "${azurerm_virtual_network.myappdb.name}-feport"
  frontend_ip_configuration_name = "${azurerm_virtual_network.myappdb.name}-feip"
  http_setting_name              = "${azurerm_virtual_network.myappdb.name}-be-htst"
  listener_name                  = "${azurerm_virtual_network.myappdb.name}-httplstn"
  request_routing_rule_name      = "${azurerm_virtual_network.myappdb.name}-rqrt"
  redirect_configuration_name    = "${azurerm_virtual_network.myappdb.name}-rdrcfg"
}

resource "azurerm_application_gateway" "network" {
  name                = "myapp-appgateway"
  resource_group_name = azurerm_resource_group.rg.name
  location            = azurerm_resource_group.rg.location

  sku {
    name     = "Standard_v2"
    tier     = "Standard_v2"
    capacity = 2
  }

  gateway_ip_configuration {
    name      = "my-gateway-ip-configuration"
    subnet_id = azurerm_subnet.frontend.id
  }

  frontend_port {
    name = local.frontend_port_name
    port = 80
  }

  frontend_ip_configuration {
    name                 = local.frontend_ip_configuration_name
    public_ip_address_id = azurerm_public_ip.myappip.id
  }

  backend_address_pool {
    name  = local.backend_address_pool_name
    ip_addresses = [azurerm_container_group.tf_cg_sampleapi.ip_address]
  }

  backend_http_settings {
    name                  = local.http_setting_name
    cookie_based_affinity = "Disabled"
    path                  = "/path1/"
    port                  = 80
    protocol              = "Http"
    request_timeout       = 60
  }

  http_listener {
    name                           = local.listener_name
    frontend_ip_configuration_name = local.frontend_ip_configuration_name
    frontend_port_name             = local.frontend_port_name
    protocol                       = "Http"
  }

  request_routing_rule {
    name                       = local.request_routing_rule_name
    rule_type                  = "Basic"
    http_listener_name         = local.listener_name
    backend_address_pool_name  = local.backend_address_pool_name
    backend_http_settings_name = local.http_setting_name
  }
}

container.tf

resource "azurerm_container_group" "tf_cg_sampleapi" {
  depends_on = [azurerm_cosmosdb_account.db]

  name                = "cg_myapp"
  location            = azurerm_resource_group.rg.location
  resource_group_name = azurerm_resource_group.rg.name

  network_profile_id = azurerm_network_profile.containergroup_profile.id

  ip_address_type = "Private"
  # dns_name_label  = "sampleapitf"
  os_type         = "Linux"

  identity {
    type = "SystemAssigned"
  }

  container {
    name   = "myapp"
    image  = "sample/myapp"
    cpu    = 1
    memory = 1

    ports {
      port     = 80
      protocol = "TCP"
    }

    ports {
      port     = 443
      protocol = "TCP"
    }

    secure_environment_variables = {
      "MYAPP_CONNECTION_STRING" = azurerm_cosmosdb_account.db.connection_strings[0]
    }
  }
}

CodePudding user response：

I met the similar issue and in my case(containers on top of Azure App Service) I needed to put the depends_on block inside the application gateway resource creation with regards to app services being created in the first place. So in your case should be:

resource "azurerm_application_gateway" "network" {
  name                = "myapp-appgateway"
  resource_group_name = azurerm_resource_group.rg.name
  location            = azurerm_resource_group.rg.location

  sku {
    name     = "Standard_v2"
    tier     = "Standard_v2"
    capacity = 2
  }

  gateway_ip_configuration {
    name      = "my-gateway-ip-configuration"
    subnet_id = azurerm_subnet.frontend.id
  }

  frontend_port {
    name = local.frontend_port_name
    port = 80
  }

  frontend_ip_configuration {
    name                 = local.frontend_ip_configuration_name
    public_ip_address_id = azurerm_public_ip.myappip.id
  }

  backend_address_pool {
    name  = local.backend_address_pool_name
    ip_addresses = [azurerm_container_group.tf_cg_sampleapi.ip_address]
  }

  backend_http_settings {
    name                  = local.http_setting_name
    cookie_based_affinity = "Disabled"
    path                  = "/path1/"
    port                  = 80
    protocol              = "Http"
    request_timeout       = 60
  }

  http_listener {
    name                           = local.listener_name
    frontend_ip_configuration_name = local.frontend_ip_configuration_name
    frontend_port_name             = local.frontend_port_name
    protocol                       = "Http"
  }

  request_routing_rule {
    name                       = local.request_routing_rule_name
    rule_type                  = "Basic"
    http_listener_name         = local.listener_name
    backend_address_pool_name  = local.backend_address_pool_name
    backend_http_settings_name = local.http_setting_name
  }
  depends_on = [ azurerm_container_group.tf_cg_sampleapi, ]
}

CodePudding user response：

I figured out the root cause of my 502 Gateway error was due to health checks not being setup / not working. Consequently, I setup custom probes that would go to an API endpoint to return a 200 OK response. Of course, I will configure this endpoint to actually check to see if I can connect to my services, but this was just a test to verify this was the issue.

I also removed the DenyAllInBound-Override and DenyAllOutBound-Override rules within my nsg-aci security group, as this was causing issues with my ACI to connect to my Cosmos DB.

This was my resulting network.tf and container.tf files:

network.tf

resource "azurerm_virtual_network" "myappdb" {
  name                = "myappdb-vnet"
  address_space       = ["10.7.0.0/16"]
  location            = azurerm_resource_group.rg.location
  resource_group_name = azurerm_resource_group.rg.name
}

resource "azurerm_subnet" "internal" {
  name                 = "internal"
  resource_group_name  = azurerm_resource_group.rg.name
  virtual_network_name = azurerm_virtual_network.myappdb.name
  address_prefixes     = ["10.7.2.0/24"]
  service_endpoints    = ["Microsoft.AzureCosmosDB"]

  delegation {
    name = "acidelegationservice"

    service_delegation {
      name    = "Microsoft.ContainerInstance/containerGroups"
      actions = ["Microsoft.Network/virtualNetworks/subnets/join/action", "Microsoft.Network/virtualNetworks/subnets/prepareNetworkPolicies/action"]
    }
  }

  enforce_private_link_endpoint_network_policies = true
}

resource "azurerm_subnet" "frontend" {
  name                 = "myapp-frontend"
  resource_group_name  = azurerm_resource_group.rg.name
  virtual_network_name = azurerm_virtual_network.myappdb.name
  address_prefixes     = ["10.7.0.0/24"]
}

resource "azurerm_network_security_group" "nsg-myapp" {
  name                = "nsg-aci"
  location            = azurerm_resource_group.rg.location
  resource_group_name = azurerm_resource_group.rg.name

  security_rule {
    name              = "from-gateway-subnet"
    priority          = 100
    direction         = "Inbound"
    access            = "Allow"
    protocol          = "Tcp"
    source_port_range = "*"

    destination_port_ranges      = [22, 80, 443, 445, 8000]
    source_address_prefixes      = azurerm_subnet.internal.address_prefixes
    destination_address_prefixes = azurerm_subnet.internal.address_prefixes
  }

  security_rule {
    name              = "to-internet"
    priority          = 100
    direction         = "Outbound"
    access            = "Allow"
    protocol          = "Tcp"
    source_port_range = "*"

    destination_port_ranges    = [80, 443, 445]
    source_address_prefix      = "*"
    destination_address_prefix = "*"
  }
}

resource "azurerm_subnet_network_security_group_association" "sn-nsg-aci" {
  subnet_id                 = azurerm_subnet.internal.id
  network_security_group_id = azurerm_network_security_group.nsg-myapp.id
}

resource "azurerm_network_profile" "containergroup_profile" {
  name                = "acg-profile"
  location            = azurerm_resource_group.rg.location
  resource_group_name = azurerm_resource_group.rg.name

  container_network_interface {
    name = "acg-nic"

    ip_configuration {
      name      = "aciipconfig"
      subnet_id = azurerm_subnet.internal.id
    }
  }
}

resource "azurerm_public_ip" "myappip" {
  name                = "myappip"
  resource_group_name = azurerm_resource_group.rg.name
  location            = azurerm_resource_group.rg.location
  allocation_method   = "Static"
  sku                 = "Standard"
}

locals {
  backend_address_pool_name      = "${azurerm_virtual_network.myappdb.name}-beap"
  frontend_port_name             = "${azurerm_virtual_network.myappdb.name}-feport"
  frontend_ip_configuration_name = "${azurerm_virtual_network.myappdb.name}-feip"
  http_setting_name              = "${azurerm_virtual_network.myappdb.name}-be-htst"
  listener_name                  = "${azurerm_virtual_network.myappdb.name}-httplstn"
  request_routing_rule_name      = "${azurerm_virtual_network.myappdb.name}-rqrt"
  redirect_configuration_name    = "${azurerm_virtual_network.myappdb.name}-rdrcfg"
}

resource "azurerm_application_gateway" "network" {
  name                = "myapp-appgateway"
  resource_group_name = azurerm_resource_group.rg.name
  location            = azurerm_resource_group.rg.location

  sku {
    name     = "Standard_v2"
    tier     = "Standard_v2"
    capacity = 2
  }

  gateway_ip_configuration {
    name      = "my-gateway-ip-configuration"
    subnet_id = azurerm_subnet.frontend.id
  }

  frontend_port {
    name = local.frontend_port_name
    port = 80
  }

  frontend_ip_configuration {
    name                 = local.frontend_ip_configuration_name
    public_ip_address_id = azurerm_public_ip.myappip.id
  }

  backend_address_pool {
    name         = local.backend_address_pool_name
    ip_addresses = [azurerm_container_group.tf_cg_sampleapi.ip_address]
  }

  probe {
    interval            = 60
    timeout             = 60
    name                = "status"
    protocol            = "Http"
    path                = "/api/status/"
    unhealthy_threshold = 3
    host                = "127.0.0.1"
  }

  backend_http_settings {
    name                  = local.http_setting_name
    cookie_based_affinity = "Disabled"
    path                  = "/"
    port                  = 80
    protocol              = "Http"
    request_timeout       = 60
    probe_name            = "status"
  }

  http_listener {
    name                           = local.listener_name
    frontend_ip_configuration_name = local.frontend_ip_configuration_name
    frontend_port_name             = local.frontend_port_name
    protocol                       = "Http"
  }

  request_routing_rule {
    name                       = local.request_routing_rule_name
    rule_type                  = "Basic"
    http_listener_name         = local.listener_name
    backend_address_pool_name  = local.backend_address_pool_name
    backend_http_settings_name = local.http_setting_name
  }

  depends_on = [azurerm_container_group.tf_cg_sampleapi, ]
}

container.tf

resource "azurerm_container_group" "tf_cg_sampleapi" {
  depends_on = [azurerm_cosmosdb_account.db]

  name                = "cg_myapp"
  location            = azurerm_resource_group.rg.location
  resource_group_name = azurerm_resource_group.rg.name

  network_profile_id = azurerm_network_profile.containergroup_profile.id

  ip_address_type = "Private"
  # dns_name_label  = "sampleapitf"
  os_type = "Linux"

  container {
    name   = "myapp"
    image  = "sample/myapp"
    cpu    = 1
    memory = 1

    ports {
      port     = 80
      protocol = "TCP"
    }

    ports {
      port     = 443
      protocol = "TCP"
    }

    secure_environment_variables = {
      "MYAPP_CONNECTION_STRING" = azurerm_cosmosdb_account.db.connection_strings[0]
    }
  }
}