Terraform - 如何在 AKS 集群上安装 AZURE CNI 并设置 POD IP 范围？

Question

我正在尝试使用 terraform 在 azure 中创建一个 AKS 集群。我的要求如下：

1. 创建站点到站点 VPN 连接，其中网关位于 172.30.0.0/16 范围的子网中 -这个完成了

2. 使用 Azure CNI 安装 Azure AKS 集群，pod 应该在 VPN CIDR (172.30.0.0/16) 的范围内。

   这是我的地形代码。我读到如果你使用azure作为你的network_policy和network_plugin，你不能设置pod_cidr - source

   那么我该怎么做才能让我的 POD 可以通过站点到站点的 vpn 访问内部部署网络？

   
   resource "azurerm_kubernetes_cluster" "k8s_cluster" {
     lifecycle {
       ignore_changes = [
         default_node_pool[0].node_count
       ]
       prevent_destroy = false
     }
   
     name                = var.cluster_name
     location            = var.location
     resource_group_name = var.rg_name
     dns_prefix          = var.dns_prefix
     kubernetes_version  = var.kubernetes_version
     # node_resource_group = var.resource_group_name
   
     default_node_pool {
       name                  = var.default_node_pool.name
       node_count            = var.default_node_pool.node_count
       max_count             = var.default_node_pool.max_count
       min_count             = var.default_node_pool.min_count
       vm_size               = var.default_node_pool.vm_size
       os_disk_size_gb       = var.default_node_pool.os_disk_size_gb
       # vnet_subnet_id        = var.vnet_subnet_id
       max_pods              = var.default_node_pool.max_pods
       type                  = var.default_node_pool.agent_pool_type
       enable_node_public_ip = var.default_node_pool.enable_node_public_ip
       enable_auto_scaling   = var.default_node_pool.enable_auto_scaling
   
       tags = merge(var.common_tags)
     }
   
     identity {
       type = var.identity
     }
   
     network_profile {
       network_plugin     = var.network_plugin #azure
       network_policy     = var.network_policy #"azure"
       load_balancer_sku  = var.load_balancer_sku #"standard"
       # pod_cidr  = var.pod_cidr | When network_plugin is set to azure - the vnet_subnet_id field in the default_node_pool block must be set and pod_cidr must not be set.
     }
     
   
     tags = merge(var.common_tags)
   }
   

   # AKS cluster related variables
   cluster_name       = "test-cluster"
   dns_prefix         = "testjana"
   kubernetes_version = "1.22.15"
   default_node_pool = {
     name                  = "masternp" # for system pods
     node_count            = 1
     vm_size               = "standard_e4bds_v5" # 4 vcpu and 32 Gb of memory
     enable_auto_scaling   = false
     enable_node_public_ip = false
     min_count             = null
     max_count             = null
     max_pods              = 100
     os_disk_size_gb       = 80
     agent_pool_type       = "VirtualMachineScaleSets"
   }
   admin_username    = "jananathadmin"
   ssh_public_key    = "public_key"
   identity          = "SystemAssigned"
   network_plugin    = "azure"
   network_policy    = "azure"
   load_balancer_sku = "standard"
   
   

Answer 1

默认情况下，AKS 中的所有 Pod 都会相互通信，当我们想要限制流量时，可以使用网络策略来允许或拒绝 pod 之间的流量。

这里是教程link

通过 terraform 使用以下代码 sn-p 将集群与 azure CNI 和 vnet 网关重现，该网关通过站点到站点 VPN 将我们的本地环境链接到 azure。

步骤1： 主要tf文件如下

resource "azurerm_resource_group" "example" {
  name     = "*****-****"
  location = "East US"
}

resource "azurerm_role_assignment" "role_acrpull" {
  scope                            = azurerm_container_registry.acr.id
  role_definition_name             = "AcrPull"
  principal_id                     = azurerm_kubernetes_cluster.demo.kubelet_identity.0.object_id
}

resource "azurerm_container_registry" "acr" {
  name                = "acrswarna"
  resource_group_name = azurerm_resource_group.example.name
  location            = azurerm_resource_group.example.location
  sku                 = "Standard"
  admin_enabled       = false
}
resource "azurerm_virtual_network" "puvnet" {
    name                = "Publics_VNET"
   resource_group_name = azurerm_resource_group.example.name
  location            = azurerm_resource_group.example.location
  address_space       = ["10.19.0.0/16"]
}
resource "azurerm_subnet" "example" {
  name                 = "GatewaySubnet"
   resource_group_name = azurerm_resource_group.example.name
  virtual_network_name = azurerm_virtual_network.puvnet.name
  address_prefixes     = ["10.19.3.0/24"]
}
resource "azurerm_subnet" "osubnet" {
  name                 = "Outer_Subnet"
  resource_group_name = azurerm_resource_group.example.name
  address_prefixes     = ["10.19.1.0/24"]
  virtual_network_name = azurerm_virtual_network.puvnet.name
}

resource "azurerm_kubernetes_cluster" "demo" {
  name                = "demo-aksnew"
  location            = azurerm_resource_group.example.location
  resource_group_name = azurerm_resource_group.example.name
  dns_prefix          = "demo-aks"

  default_node_pool {
    name                = "default"
    node_count          = 2
    vm_size             = "standard_e4bds_v5"
    type                = "VirtualMachineScaleSets"
    enable_auto_scaling = false
    min_count             = null
    max_count             = null
    max_pods              = 100
    //vnet_subnet_id = azurerm_subnet.osubnet.id
  }
  
  identity {
    type = "SystemAssigned"
  }

  network_profile {
    network_plugin    = "azure"
    load_balancer_sku = "standard"
    network_policy    = "azure"
  }

  tags = {
    Environment = "Development"
  }
}
resource "azurerm_public_ip" "example" {
  name                = "pips-firewall"
   resource_group_name = azurerm_resource_group.example.name
  location            = azurerm_resource_group.example.location
   allocation_method = "Static"
  sku                 = "Standard"
}
resource "azurerm_virtual_network_gateway" "example" {
  name                = "test"
  location            = azurerm_resource_group.example.location
  resource_group_name = azurerm_resource_group.example.name

  type     = "Vpn"
  vpn_type = "RouteBased"

  active_active = false
  enable_bgp    = false
  sku           = "VpnGw1"

  ip_configuration {
    name                          = "vnetGatewayConfig"
    public_ip_address_id          = azurerm_public_ip.example.id
    private_ip_address_allocation = "Dynamic"
    subnet_id                     = azurerm_subnet.example.id
  }

  vpn_client_configuration {
    address_space = ["172.30.0.0/16"]

    root_certificate {
      name = "******-****-ID-Root-CA"

      public_cert_data = <<EOF
   **Use certificate here**
   EOF
    }
    revoked_certificate {
      name       = "*****-Global-Root-CA"
      thumbprint = "****************"
    }
  }
}

注意：在上面的代码中自行更新根证书配置。

Provider tf文件如下

terraform {
  required_providers {
    azurerm = {
      source  = "hashicorp/azurerm"
      version = "=3.0.0"
    }
  }

}
provider "azurerm" {
  features {}
  
  skip_provider_registration = true
}

跑步时

 terraform plan 
 terraform apply -auto-approve

Vnet 和子网配置

虚拟网络网关配置如下。

在集群上部署示例 pod