我使用以下 Terraform 在 Azure 中创建了一个 Kubernetes 集群
# Locals block for hardcoded names
locals {
backend_address_pool_name = "appgateway-beap"
frontend_port_name = "appgateway-feport"
frontend_ip_configuration_name = "appgateway-feip"
http_setting_name = "appgateway-be-htst"
listener_name = "appgateway-httplstn"
request_routing_rule_name = "appgateway-rqrt"
app_gateway_subnet_name = "appgateway-subnet"
}
data "azurerm_subnet" "aks-subnet" {
name = "aks-subnet"
virtual_network_name = "np-dat-spoke-vnet"
resource_group_name = "ipz12-dat-np-connect-rg"
}
data "azurerm_subnet" "appgateway-subnet" {
name = "appgateway-subnet"
virtual_network_name = "np-dat-spoke-vnet"
resource_group_name = "ipz12-dat-np-connect-rg"
}
# Create Resource Group for Kubernetes Cluster
module "resource_group_kubernetes_cluster" {
source = "./modules/resource_group"
count = var.enable_kubernetes == true ? 1 : 0
#name_override = "rg-aks-spoke-dev-westus3-001"
app_or_service_name = "aks" # var.app_or_service_name
subscription_type = var.subscription_type # "spoke"
environment = var.environment # "dev"
location = var.location # "westus3"
instance_number = var.instance_number # "001"
tags = var.tags
}
resource "azurerm_user_assigned_identity" "identity_uami" {
location = var.location
name = "appgw-uami"
resource_group_name = module.resource_group_kubernetes_cluster[0].name
}
# Application Gateway Public Ip
resource "azurerm_public_ip" "test" {
name = "publicIp1"
location = var.location
resource_group_name = module.resource_group_kubernetes_cluster[0].name
allocation_method = "Static"
sku = "Standard"
}
resource "azurerm_application_gateway" "network" {
name = var.app_gateway_name
resource_group_name = module.resource_group_kubernetes_cluster[0].name
location = var.location
sku {
name = var.app_gateway_sku
tier = "Standard_v2"
capacity = 2
}
identity {
type = "UserAssigned"
identity_ids = [
azurerm_user_assigned_identity.identity_uami.id
]
}
gateway_ip_configuration {
name = "appGatewayIpConfig"
subnet_id = data.azurerm_subnet.appgateway-subnet.id
}
frontend_port {
name = local.frontend_port_name
port = 80
}
frontend_port {
name = "httpsPort"
port = 443
}
frontend_ip_configuration {
name = local.frontend_ip_configuration_name
public_ip_address_id = azurerm_public_ip.test.id
}
backend_address_pool {
name = local.backend_address_pool_name
}
backend_http_settings {
name = local.http_setting_name
cookie_based_affinity = "Disabled"
port = 80
protocol = "Http"
request_timeout = 1
}
http_listener {
name = local.listener_name
frontend_ip_configuration_name = local.frontend_ip_configuration_name
frontend_port_name = local.frontend_port_name
protocol = "Http"
}
request_routing_rule {
name = local.request_routing_rule_name
rule_type = "Basic"
http_listener_name = local.listener_name
backend_address_pool_name = local.backend_address_pool_name
backend_http_settings_name = local.http_setting_name
priority = 100
}
tags = var.tags
depends_on = [azurerm_public_ip.test]
lifecycle {
ignore_changes = [
backend_address_pool,
backend_http_settings,
request_routing_rule,
http_listener,
probe,
tags,
frontend_port
]
}
}
# Create the Azure Kubernetes Service (AKS) Cluster
resource "azurerm_kubernetes_cluster" "kubernetes_cluster" {
count = var.enable_kubernetes == true ? 1 : 0
name = "aks-prjx-${var.subscription_type}-${var.environment}-${var.location}-${var.instance_number}"
location = var.location
resource_group_name = module.resource_group_kubernetes_cluster[0].name # "rg-aks-spoke-dev-westus3-001"
dns_prefix = "dns-aks-prjx-${var.subscription_type}-${var.environment}-${var.location}-${var.instance_number}" #"dns-prjxcluster"
private_cluster_enabled = false
local_account_disabled = true
default_node_pool {
name = "npprjx${var.subscription_type}" #"prjxsyspool" # NOTE: "name must start with a lowercase letter, have max length of 12, and only have characters a-z0-9."
vm_size = "Standard_B8ms"
vnet_subnet_id = data.azurerm_subnet.aks-subnet.id
# zones = ["1", "2", "3"]
enable_auto_scaling = true
max_count = 3
min_count = 1
# node_count = 3
os_disk_size_gb = 50
type = "VirtualMachineScaleSets"
enable_node_public_ip = false
enable_host_encryption = false
node_labels = {
"node_pool_type" = "npprjx${var.subscription_type}"
"node_pool_os" = "linux"
"environment" = "${var.environment}"
"app" = "prjx_${var.subscription_type}_app"
}
tags = var.tags
}
ingress_application_gateway {
gateway_id = azurerm_application_gateway.network.id
}
# Enabled the cluster configuration to the Azure kubernets with RBAC
azure_active_directory_role_based_access_control {
managed = true
admin_group_object_ids = var.active_directory_role_based_access_control_admin_group_object_ids
azure_rbac_enabled = true #false
}
network_profile {
network_plugin = "azure"
network_policy = "azure"
outbound_type = "userDefinedRouting"
}
identity {
type = "SystemAssigned"
}
oms_agent {
log_analytics_workspace_id = module.log_analytics_workspace[0].id
}
timeouts {
create = "20m"
delete = "20m"
}
depends_on = [
azurerm_application_gateway.network
]
}
并提供必要的权限
# Get the AKS Agent Pool SystemAssigned Identity
data "azurerm_user_assigned_identity" "aks-identity" {
name = "${azurerm_kubernetes_cluster.kubernetes_cluster[0].name}-agentpool"
resource_group_name = "MC_${module.resource_group_kubernetes_cluster[0].name}_aks-prjx-spoke-dev-eastus-001_eastus"
}
# Get the AKS SystemAssigned Identity
data "azuread_service_principal" "aks-sp" {
display_name = azurerm_kubernetes_cluster.kubernetes_cluster[0].name
}
# Provide ACR Pull permission to AKS SystemAssigned Identity
resource "azurerm_role_assignment" "acrpull_role" {
scope = module.container_registry[0].id
role_definition_name = "AcrPull"
principal_id = data.azurerm_user_assigned_identity.aks-identity.principal_id
skip_service_principal_aad_check = true
depends_on = [
data.azurerm_user_assigned_identity.aks-identity
]
}
resource "azurerm_role_assignment" "aks_id_network_contributor_subnet" {
scope = data.azurerm_subnet.aks-subnet.id
role_definition_name = "Network Contributor"
principal_id = data.azurerm_user_assigned_identity.aks-identity.principal_id
depends_on = [data.azurerm_user_assigned_identity.aks-identity]
}
resource "azurerm_role_assignment" "akssp_network_contributor_subnet" {
scope = data.azurerm_subnet.aks-subnet.id
role_definition_name = "Network Contributor"
principal_id = data.azuread_service_principal.aks-sp.object_id
depends_on = [data.azuread_service_principal.aks-sp]
}
resource "azurerm_role_assignment" "aks_id_contributor_agw" {
scope = data.azurerm_subnet.appgateway-subnet.id
role_definition_name = "Network Contributor"
principal_id = data.azurerm_user_assigned_identity.aks-identity.principal_id
depends_on = [data.azurerm_user_assigned_identity.aks-identity]
}
resource "azurerm_role_assignment" "akssp_contributor_agw" {
scope = data.azurerm_subnet.appgateway-subnet.id
role_definition_name = "Network Contributor"
principal_id = data.azuread_service_principal.aks-sp.object_id
depends_on = [data.azuread_service_principal.aks-sp]
}
resource "azurerm_role_assignment" "aks_ingressid_contributor_on_agw" {
scope = azurerm_application_gateway.network.id
role_definition_name = "Contributor"
principal_id = azurerm_kubernetes_cluster.kubernetes_cluster[0].ingress_application_gateway[0].ingress_application_gateway_identity[0].object_id
depends_on = [azurerm_application_gateway.network,azurerm_kubernetes_cluster.kubernetes_cluster]
skip_service_principal_aad_check = true
}
resource "azurerm_role_assignment" "aks_ingressid_contributor_on_uami" {
scope = azurerm_user_assigned_identity.identity_uami.id
role_definition_name = "Contributor"
principal_id = azurerm_kubernetes_cluster.kubernetes_cluster[0].ingress_application_gateway[0].ingress_application_gateway_identity[0].object_id
depends_on = [azurerm_application_gateway.network,azurerm_kubernetes_cluster.kubernetes_cluster]
skip_service_principal_aad_check = true
}
resource "azurerm_role_assignment" "uami_contributor_on_agw" {
scope = azurerm_application_gateway.network.id
role_definition_name = "Contributor"
principal_id = azurerm_user_assigned_identity.identity_uami.principal_id
depends_on = [azurerm_application_gateway.network,azurerm_user_assigned_identity.identity_uami]
skip_service_principal_aad_check = true
}
并部署了下面提到的应用程序
apiVersion: apps/v1
kind: Deployment
metadata:
name: aks-helloworld
spec:
replicas: 1
selector:
matchLabels:
app: aks-helloworld-two
template:
metadata:
labels:
app: aks-helloworld-two
spec:
containers:
- name: aks-helloworld-two
image: mcr.microsoft.com/azuredocs/aks-helloworld:v1
ports:
- containerPort: 80
env:
- name: TITLE
value: "AKS Ingress Demo"
---
apiVersion: v1
kind: Service
metadata:
name: aks-helloworld
spec:
type: LoadBalancer
ports:
- port: 80
selector:
app: aks-helloworld-two
外部IP已分配
但是我无法访问外部IP
注意:我没有像 Microsoft Article 中提到的那样单独部署任何 Ingress Controller 。因为我不确定这是必需的
最佳答案
我尝试在我的环境中重现相同的内容，以使用应用程序网关创建 Kubernetes 服务集群:
关注Stack link使用Ingress应用程序网关创建Kubernetes服务集群。
如果在 Azure Kubernetes 服务 (AKS) 中部署后无法使用外部负载均衡器 IP 访问应用程序，请验证 AKS 群集中的以下设置。
1.使用以下命令检查负载均衡器的状态。
kubectl get service <your service name>
确保外部 -IP 字段未设置为待处理状态。
请按照以下步骤检查AKS集群中的NSG安全规则。
转到 Azure 门户 > Kubernetes 服务 > 选择您的 Kubernetes 服务 > 属性 > 在基础结构资源组 > 概述下选择您的资源组 > 选择您的 NSG 组。
我在网络安全组中禁用了入站http规则进行测试，得到了同样的错误。
应用程序状态，一旦禁用NSG中的端口 80。
关于azure - AKS 服务部署已完成，但无法访问外部 IP，我们在Stack Overflow上找到一个类似的问题： https://stackoverflow.com/questions/75165929/