azure terraform linux aci 连接器 CrashLoopBackOff

Question

我希望有人能帮我解决这个问题。

我正在尝试使用虚拟节点部署 Azure kubernetes 集群，terraform apply 运行并正确配置所有节点池，但 aci 连接器 pod 抛出 CrashLoopBackOff 并出现以下错误：

level=fatal msg="error creating provider: error setting up network: error while looking up subnet: GET 403: 403 Forbidden\nERROR CODE: AuthorizationFailed\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"AuthorizationFailed\",\n    \"message\": \"The client does not have authorization to perform action 'Microsoft.Network/virtualNetworks/subnets/read' over scope /aks-dev-vnet/subnets/aks-dev-aci' or the scope is invalid. If access was recently granted, please refresh your credentials.\"\n  }\n}\n--------------------------------------------------------------------------------\n"

我的地形中确实有以下配置：

虚拟网络和子网：

resource "azurerm_virtual_network" "aks_virtual_network" {
  address_space       = [""]
  location            = var.location
  name                = "aks-${var.environment}-${var.region}-vnet"
  resource_group_name = azurerm_resource_group.kubernetes_resource_group.name
}

# System Pool Subnet
resource "azurerm_subnet" "system_pool_subnet" {
  address_prefixes     = [""]
  name                 = "aks-${var.environment}-${var.region}-system-pool"
  resource_group_name  = azurerm_resource_group.kubernetes_resource_group.name
  virtual_network_name = azurerm_virtual_network.aks_virtual_network.name
}

# System Pool Subnet
resource "azurerm_subnet" "linux_pool_subnet" {
  address_prefixes     = [""]
  name                 = "aks-${var.environment}-${var.region}-linux-pool"
  resource_group_name  = azurerm_resource_group.kubernetes_resource_group.name
  virtual_network_name = azurerm_virtual_network.aks_virtual_network.name
}


resource "azurerm_subnet" "aci_subnet" {
  address_prefixes     = [""]
  name                 = "aks-${var.environment}-${var.region}-aci"
  resource_group_name  = azurerm_resource_group.kubernetes_resource_group.name
  virtual_network_name = azurerm_virtual_network.aks_virtual_network.name
  delegation {
    name = "aci-subnet"

    service_delegation {
      name    = "Microsoft.ContainerInstance/containerGroups"
      actions = ["Microsoft.Network/virtualNetworks/subnets/action"]
    }
  }
}

服务主体配置：

resource "azuread_application" "aks_application" {
  display_name = "azure-app-${var.environment}-${var.region}-principle"
}

resource "azuread_service_principal" "aks_service_principle" {
  application_id = azuread_application.aks_application.application_id
}

resource "azuread_service_principal_password" "main" {
  service_principal_id = azuread_service_principal.aks_service_principle.id
}

# Grant AKS cluster access to join AKS subnet
resource "azurerm_role_assignment" "linux_subnet_aci" {
  scope                = azurerm_subnet.linux_pool_subnet.id
  role_definition_name = "Network Contributor"
  principal_id         = azuread_service_principal.aks_service_principle.id
}

# Grant AKS cluster access to join AKS subnet
resource "azurerm_role_assignment" "system_subnet_aci" {
  scope                = azurerm_subnet.system_pool_subnet.id
  role_definition_name = "Network Contributor"
  principal_id         = azuread_service_principal.aks_service_principle.id
}

resource "azurerm_role_assignment" "aci_subnet" {
  scope                = azurerm_subnet.aci_subnet.id
  role_definition_name = "Network Contributor"
  principal_id         = azuread_service_principal.aks_service_principle.id
}

以及我的集群配置：

default_node_pool {
    name                 = "systempool"
    vm_size              = "Standard_D2_V2"
    orchestrator_version = data.azurerm_kubernetes_service_versions.current.latest_version
    zones                = [1]
    enable_auto_scaling  = true
    min_count            = 1
    max_count            = 2
    os_disk_size_gb      = 64
    type                 = "VirtualMachineScaleSets"
    vnet_subnet_id       = azurerm_subnet.system_pool_subnet.id
    node_labels = {
      "nodepool-type" = "System"
      "environment"   = "${var.environment}"
      "nodepoolOS"    = "linux"
      "app"           = "System"
      "nodepool" = "systempool00"
    }
    upgrade_settings {
      max_surge = "10%"
    }


  }
  identity {
    type = "UserAssigned"
    identity_ids = [data.azurerm_user_assigned_identity.user_assigned.id]
  }
  kubelet_identity {
    client_id = data.azurerm_user_assigned_identity.user_assigned.client_id
    object_id = data.azurerm_user_assigned_identity.user_assigned.principal_id
    user_assigned_identity_id = data.azurerm_user_assigned_identity.user_assigned.id
  }
  oidc_issuer_enabled = true

  # Add On Profile
  aci_connector_linux {
    subnet_name = azurerm_subnet.aci_subnet.name
  }

此时我知道我应该在集群配置中使用服务原理：

#  service_principal {
#    client_id     = azuread_application.aks_application.application_id
#    client_secret = azuread_service_principal_password.main.value
#  }

但是当我尝试启用它时，我收到错误消息，只能启用一个身份（身份或服务主体）。 在我的具体情况下，我需要为平台的另一部分的 dns 权限配置身份。

当我在 aci 连接器中收到错误时。引用的对象 ID 是自动创建的托管标识，问题是该托管标识没有在子网上执行操作的权限。

我现在真的很困惑，我无法考虑此配置的任何解决方法来向 aci 托管身份授予权限并启用我的身份。

有人可以在这里遮挡一些光线吗？..如果我的查询不是 100% 清楚，请告诉我，我可以提供更多信息。

非常感谢您提供的任何帮助。

Answer 1

Azure Terraform Linux ACI 连接器 CrashLoopBackOff 问题

Azure Kubernetes 服务 (AKS) 部署中 ACI 连接器的问题源于托管标识访问子网的权限。您似乎正在为 AKS 群集使用用户分配的托管标识，并尝试应用角色分配来提供网络活动所需的权限。然而，ACI 连接器需要不同的权限才能与指定的子网交互，并且该错误表明这些权限尚未正确建立。

我尝试了更新的 terraform 配置，并根据要求进行了更改，并且我能够成功地配置要求。

我的 Terraform 配置：

provider "azurerm" {
  features {}
}

provider "azuread" {}

data "azuread_client_config" "current" {}

resource "azurerm_resource_group" "kubernetes_resource_group" {
  name     = "testvksb-rg"
  location = "East US2"
}

resource "azuread_application" "aks_application" {
  display_name = "aksApp"
}

resource "azuread_service_principal" "aks_service_principal" {
  client_id                    = azuread_application.aks_application.client_id
  app_role_assignment_required = false
  owners                       = [data.azuread_client_config.current.object_id]
}

resource "azuread_service_principal_password" "aks_sp_password" {
  service_principal_id = azuread_service_principal.aks_service_principal.id
  end_date_relative    = "8760h" # 1 year
}

resource "azurerm_virtual_network" "aks_virtual_network" {
  name                = "myVnet"
  address_space       = ["10.0.0.0/8"]
  location            = azurerm_resource_group.kubernetes_resource_group.location
  resource_group_name = azurerm_resource_group.kubernetes_resource_group.name
}

resource "azurerm_subnet" "system_pool_subnet" {
  name                 = "systemSubnet"
  resource_group_name  = azurerm_resource_group.kubernetes_resource_group.name
  virtual_network_name = azurerm_virtual_network.aks_virtual_network.name
  address_prefixes     = ["10.240.0.0/16"]
}

resource "azurerm_subnet" "aci_subnet" {
  name                 = "aciSubnet"
  resource_group_name  = azurerm_resource_group.kubernetes_resource_group.name
  virtual_network_name = azurerm_virtual_network.aks_virtual_network.name
  address_prefixes     = ["10.241.0.0/16"]
  delegation {
    name = "aciDelegation"
    service_delegation {
      name    = "Microsoft.ContainerInstance/containerGroups"
      actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"]
    }
  }
}


resource "azurerm_role_assignment" "aci_subnet_contributor" {
  scope                = azurerm_subnet.aci_subnet.id
  role_definition_name = "Network Contributor"
  principal_id         = azuread_service_principal.aks_service_principal.id
}

resource "azurerm_kubernetes_cluster" "aks_cluster" {
  name                = "vkAKSCluster"
  location            = azurerm_resource_group.kubernetes_resource_group.location
  resource_group_name = azurerm_resource_group.kubernetes_resource_group.name
  dns_prefix          = "myaksdns"

  default_node_pool {
    name           = "default"
    node_count     = 1
    vm_size        = "Standard_DS2_v2"
    vnet_subnet_id = azurerm_subnet.system_pool_subnet.id
  }

  service_principal {
    client_id     = azuread_application.aks_application.application_id
    client_secret = azuread_service_principal_password.aks_sp_password.value
  }

  network_profile {
    network_plugin    = "azure"
    service_cidr      = "10.10.0.0/16"
    dns_service_ip    = "10.10.0.10"
   // docker_bridge_cidr = "172.17.0.1/16"
  }
  
 aci_connector_linux {
    subnet_name = azurerm_subnet.aci_subnet.name
  }
}

部署成功：