AKS 通过 Terraform 错误：代码 =“CustomRouteTableWithUnsupportedMSIType”

Question

尝试使用现有的 vnet 和子网通过 terraform 创建私有 aks，但突然出现以下错误时能够创建集群。

│ 错误：创建托管 Kubernetes 集群“demo-azwe-aks-cluster”（资源组“demo-azwe-aks-rg”）：containerservice.ManagedClustersClient#CreateOrUpdate：发送请求失败：StatusCode=0 -- 原始错误: Code="CustomRouteTableWithUnsupportedMSIType" Message="使用托管标识类型 SystemAssigned 的集群不支持自带路由表。有关详细信息，请参阅https://aka.ms/aks/customrt" │ │ 使用 azurerm_kubernetes_cluster.aks_cluster， │ 在 aks_cluster.tf 第 30 行，资源“azurerm_kubernetes_cluster”“aks_cluster”中： │ 30: 资源 "azurerm_kubernetes_cluster" "aks_cluster" {

# Provision AKS Cluster
resource "azurerm_kubernetes_cluster" "aks_cluster" {
  name                = "${var.global-prefix}-${var.cluster-id}-${var.environment}-azwe-aks-cluster"
  location            = "${var.location}"
  resource_group_name = azurerm_resource_group.aks_rg.name
  dns_prefix          = "${var.global-prefix}-${var.cluster-id}-${var.environment}-azwe-aks-cluster"
  kubernetes_version  = data.azurerm_kubernetes_service_versions.current.latest_version
  node_resource_group = "${var.global-prefix}-${var.cluster-id}-${var.environment}-azwe-aks-nrg"
  private_cluster_enabled = true


  default_node_pool {
    name                 = "dpool"
    vm_size              = "Standard_DS2_v2"
    orchestrator_version = data.azurerm_kubernetes_service_versions.current.latest_version
    availability_zones   = [1, 2, 3]
    enable_auto_scaling  = true
    max_count            = 2
    min_count            = 1
    os_disk_size_gb      = 30
    type                 = "VirtualMachineScaleSets"
    vnet_subnet_id       = data.azurerm_subnet.aks.id
    node_labels = {
      "nodepool-type"    = "system"
      "environment"      = "${var.environment}"
      "nodepoolos"       = "${var.nodepool-os}"
      "app"              = "system-apps" 
    } 
   tags = {
      "nodepool-type"    = "system"
      "environment"      = "dev"
      "nodepoolos"       = "linux"
      "app"              = "system-apps" 
   } 
  }

# Identity (System Assigned or Service Principal)
  identity {
    type = "SystemAssigned"
  }

# Add On Profiles
  addon_profile {
    azure_policy {enabled =  true}
    oms_agent {
      enabled =  true
      log_analytics_workspace_id = azurerm_log_analytics_workspace.insights.id
    }
  }

# Create Azure AD Group in Active Directory for AKS Admins
resource "azuread_group" "aks_administrators" {
  name        = "${azurerm_resource_group.aks_rg.name}-cluster-administrators"
 description = "Azure AKS Kubernetes administrators for the ${azurerm_resource_group.aks_rg.name}-cluster."
}

 RBAC and Azure AD Integration Block
  role_based_access_control {
    enabled = true
    azure_active_directory {
      managed = true
      admin_group_object_ids = [azuread_group.aks_administrators.id]
    }
  }



# Linux Profile
  linux_profile {
    admin_username = "ubuntu"
    ssh_key {
      key_data = file(var.ssh_public_key)
    }
  }

# Network Profile
  network_profile {
    network_plugin = "kubenet"
    load_balancer_sku = "Standard"
  }

  tags = {
    Environment = "prod"
  }
}

Answer 1

您正在尝试为 AKS 和防火墙创建具有现有 Vnet 和现有子网的私有 AKS 群集，因此根据错误 "CustomRouteTableWithUnsupportedMSIType" 您需要托管标识来创建路由表和分配给它的角色，即 @ 987654328@.

当您使用 azure vnet 及其子网时，网络配置文件将是 azure 而不是 kubenet。

您可以根据您的要求添加，但请确保您已将数据块用于工作区，否则您可以直接提供资源 ID。所以，而不是

log_analytics_workspace_id = azurerm_log_analytics_workspace.insights.id

你可以使用

log_analytics_workspace_id = "/subscriptions/SubscriptionID/resourcegroups/resourcegroupname/providers/microsoft.operationalinsights/workspaces/workspacename"

使用现有 vnet 和子网创建私有集群的示例（我没有添加插件）：

provider "azurerm" {
  features {}
}

#resource group as this will be referred to in managed identity creation
data "azurerm_resource_group" "base" {
  name     = "resourcegroupname"
}

#exisiting vnet
data "azurerm_virtual_network" "base" {
  name                = "ansuman-vnet"
  resource_group_name = data.azurerm_resource_group.base.name
}

#exisiting subnets

data "azurerm_subnet" "aks" {
  name                 = "akssubnet"
  resource_group_name  = data.azurerm_resource_group.base.name
  virtual_network_name = data.azurerm_virtual_network.base.name
}

data "azurerm_subnet" "firewall" {
  name                 = "AzureFirewallSubnet"
  resource_group_name  = data.azurerm_resource_group.base.name
  virtual_network_name = data.azurerm_virtual_network.base.name
}

#user assigned identity required to create route table

resource "azurerm_user_assigned_identity" "base" {
  resource_group_name = data.azurerm_resource_group.base.name
  location            = data.azurerm_resource_group.base.location
  name                = "mi-name"
}

#role assignment required to create route table

resource "azurerm_role_assignment" "base" {
  scope                = data.azurerm_resource_group.base.id
  role_definition_name = "Network Contributor"
  principal_id         = azurerm_user_assigned_identity.base.principal_id
}

#route table

resource "azurerm_route_table" "base" {
  name                = "rt-aksroutetable"
  location            = data.azurerm_resource_group.base.location
  resource_group_name = data.azurerm_resource_group.base.name
}

#route 

resource "azurerm_route" "base" {
  name                   = "dg-aksroute"
  resource_group_name    = data.azurerm_resource_group.base.name
  route_table_name       = azurerm_route_table.base.name
  address_prefix         = "0.0.0.0/0"
  next_hop_type          = "VirtualAppliance"
  next_hop_in_ip_address = azurerm_firewall.base.ip_configuration.0.private_ip_address
}

#route table association

resource "azurerm_subnet_route_table_association" "base" {
  subnet_id      = data.azurerm_subnet.aks.id
  route_table_id = azurerm_route_table.base.id
}

#firewall

resource "azurerm_public_ip" "base" {
  name                = "pip-firewall"
  location            = data.azurerm_resource_group.base.location
  resource_group_name = data.azurerm_resource_group.base.name
  allocation_method   = "Static"
  sku                 = "Standard"
}

resource "azurerm_firewall" "base" {
  name                = "fw-akscluster"
  location            = data.azurerm_resource_group.base.location
  resource_group_name = data.azurerm_resource_group.base.name

  ip_configuration {
    name                 = "ip-firewallakscluster"
    subnet_id            = data.azurerm_subnet.firewall.id
    public_ip_address_id = azurerm_public_ip.base.id
  }
}

#kubernetes_cluster

resource "azurerm_kubernetes_cluster" "base" {
  name                    = "testakscluster"
  location                = data.azurerm_resource_group.base.location
  resource_group_name     = data.azurerm_resource_group.base.name
  dns_prefix              = "dns-testakscluster"
  private_cluster_enabled = true

  network_profile {
    network_plugin = "azure"
    outbound_type  = "userDefinedRouting"
  }


  default_node_pool {
    name           = "default"
    node_count     = 1
    vm_size        = "Standard_D2_v2"
    vnet_subnet_id = data.azurerm_subnet.aks.id
  }

  identity {
    type                      = "UserAssigned"
    user_assigned_identity_id = azurerm_user_assigned_identity.base.id
  }
  depends_on = [
      azurerm_route.base,
      azurerm_role_assignment.base
    ]
}

输出：

（地形规划）

（地形应用）

（Azure 门户）

注意：默认情况下，azure 要求防火墙的子网名称为 AzureFirewallSubnet。如果您使用任何其他名称的子网来创建防火墙，那么它将出错。所以，请确保将防火墙使用的现有子网命名为AzureFirewallSubnet。