[PRODENG-3929] Scope example ingress, split node roles, stop inlining creds

examples/terraform/aws-complete/README.md

@@ -9,4 +9,43 @@
 - This configuration uses Launchpad terraform provider for installing MCR, MKE and MSR 2.9x on the provisioned infrstructure
 - After installing MKE, the configuration uses MKE terraform provider to create a kubernetes context to install kubernetes workloads
 - After creating a kubernetes context the same is used for installing AWS cloud controller manager and AWS EBS CSI driver
-- The content of test.tf file is commented out and the resources in that file can be used testing the functionality of AWS cloud controller manager and AWS EBS CSI driver
+- The content of test.tf file is commented out and the resources in that file can be used testing the functionality of AWS cloud controller manager and AWS EBS CSI driver
+
+## Inputs worth knowing about
+
+### Network exposure — `admin_cidrs`
+
+`admin_cidrs` lists the CIDRs allowed to reach the cluster from outside it:
+SSH, WinRM over HTTPS, and the MKE and Kubernetes API ports.
+
+It defaults to `["0.0.0.0/0"]` — open to the whole internet. That is a real
+risk, and it is the exposure this example has always had; it is the default so
+that an existing `terraform.tfvars` keeps working and a first run needs no
+extra arguments. Narrow it whenever you can:
+
+```hcl
+admin_cidrs = ["198.51.100.17/32", "203.0.113.0/24"]
+```
+
+Node-to-node traffic does not pass through these rules, so narrowing this does
+not affect the cluster talking to itself.
+
+### Credentials
+
+The MKE admin password (`launchpad.mke_connect.password`) and
+`windows_password` are ordinary terraform variables. Setting them in
+`terraform.tfvars` is all that is required, and the generated
+`launchpad_yaml` output is usable as-is.
+
+To keep a password off disk, use terraform's own environment mechanism —
+terraform reads any input variable from `TF_VAR_<name>`:
+
+```sh
+export TF_VAR_launchpad='{"drain":false,"mcr_version":"25.0.13","mke_version":"3.8.8","msr_version":"","mke_connect":{"username":"admin","password":"...","insecure":false},"skip_create":false,"skip_destroy":true}'
+export TF_VAR_windows_password='...'
+```
+
+The MKE password is nested inside the `launchpad` object, so `TF_VAR_launchpad`
+carries that whole object as JSON. Either route works; the environment is an
+option for keeping secrets out of a file, not a requirement for running the
+example.

examples/terraform/aws-complete/iam.tf

@@ -1,5 +1,24 @@
-resource "aws_iam_role" "common_role" {
-  name               = "common-iam-role-${var.name}"
+// Manager and worker nodes carry separate roles so that the cloud-controller
+// permissions, which only the control plane uses, are not held by every
+// worker in the cluster.  Both roles share the same trust policy.
+//
+// APPLYING THIS TO AN EXISTING STACK REPLACES ITS IAM RESOURCES. The previous
+// common_role, common_profile and their policy attachments are destroyed and
+// the new profiles are attached to the launch templates, not to instances
+// that are already running: provision-aws does not refresh instances on a
+// launch-template change, so nodes provisioned before this change keep
+// running with an instance profile whose role no longer exists, and the
+// cloud controller manager and EBS CSI driver on them lose their permissions
+// until the nodes are replaced. Replace the nodes, or recreate the stack,
+// rather than re-applying in place against a live cluster.
+resource "aws_iam_role" "manager_role" {
+  name               = "manager-iam-role-${var.name}"
+  path               = "/"
+  assume_role_policy = data.aws_iam_policy_document.assume_role.json
+}
+
+resource "aws_iam_role" "worker_role" {
+  name               = "worker-iam-role-${var.name}"
   path               = "/"
   assume_role_policy = data.aws_iam_policy_document.assume_role.json
 }
@@ -17,7 +36,12 @@ data "aws_iam_policy_document" "assume_role" {
   }
 }
 
-resource "aws_iam_instance_profile" "common_profile" {
-  name = "common-instance-profile-${var.name}"
-  role = aws_iam_role.common_role.name
+resource "aws_iam_instance_profile" "manager_profile" {
+  name = "manager-instance-profile-${var.name}"
+  role = aws_iam_role.manager_role.name
+}
+
+resource "aws_iam_instance_profile" "worker_profile" {
+  name = "worker-instance-profile-${var.name}"
+  role = aws_iam_role.worker_role.name
 }

examples/terraform/aws-complete/launchpad.tf

@@ -52,28 +52,31 @@ locals {
     }
   }
 
-  // standard MCR/MKE/MSR firewall rules [here we just leave it open until we can figure this out]
+  // MCR/MKE/MSR firewall rules.  The load-balanced ports accept the VPC CIDR
+  // as well as var.admin_cidrs, because the ingress load balancers reach the
+  // manager nodes from inside the VPC rather than from an administrator's
+  // address, and their health checks would fail without it.
   launchpad_securitygroups = {
     "manager" = {
       description = "Common security group for manager nodes"
       nodegroups  = [for n, ng in var.nodegroups : n if ng.role == "manager"]
       tags        = local.common_tags
       ingress_ipv4 = [
         {
-          description : "Allow https traffic from anywhere"
+          description : "Allow https traffic from administrators and the load balancers"
           from_port : 443
           to_port : 443
           protocol : "tcp"
           self : false
-          cidr_blocks : ["0.0.0.0/0"]
+          cidr_blocks : concat(var.admin_cidrs, [var.network.cidr])
         },
         {
-          description : "Allow https traffic from anywhere for kube api server"
+          description : "Allow kube api server traffic from administrators and the load balancer"
           from_port : 6443
           to_port : 6443
           protocol : "tcp"
           self : false
-          cidr_blocks : ["0.0.0.0/0"]
+          cidr_blocks : concat(var.admin_cidrs, [var.network.cidr])
         },
       ]
     }
@@ -107,7 +110,7 @@ locals {
 
     winrm_address : ngn.public_ip
     winrm_user : ng.winrm_user
-    winrm_password : var.windows_password
+    winrm_password : local.windows_password
     winrm_useHTTPS : ng.winrm_useHTTPS
     winrm_insecure : ng.winrm_insecure
   } if contains(local.launchpad_roles, ng.role) && ng.connection == "winrm" }]...)
@@ -182,6 +185,23 @@ resource "launchpad_config" "cluster" {
 }
 
 // ------- Ye old launchpad yaml (just for debugging)
+//
+// The credentials are written from the terraform variables that carry them:
+// var.launchpad.mke_connect.password and var.windows_password.  The document
+// this produces is usable as-is, with nothing exported into the environment
+// first.
+//
+// To keep a password out of a tfvars file on disk, supply the variable
+// through terraform's own environment mechanism instead -- TF_VAR_<name>,
+// which terraform reads for any input variable:
+//
+//   export TF_VAR_windows_password='...'
+//   export TF_VAR_launchpad='{"drain":false,...,"mke_connect":{"username":"admin","password":"...","insecure":true}}'
+//
+// TF_VAR_launchpad takes the whole object as JSON because the MKE password
+// is nested inside it; test/terraform.go does exactly this.  Either route
+// produces the same configuration -- the environment is an option for
+// keeping secrets off disk, not a requirement for running the example.
 output "launchpad_yaml" {
   description = "launchpad config file yaml (for debugging)"
   sensitive   = true

examples/terraform/aws-complete/main.tf

@@ -24,12 +24,17 @@ locals {
 
 
 module "aws_ccm" {
-  source        = "./modules/aws-ccm"
-  iam_role_name = aws_iam_role.common_role.name
-  cluster_name  = var.name
+  source = "./modules/aws-ccm"
+  // The control-plane policy goes to managers only; the worker policy, which
+  // is ECR pull access, is needed by every node that runs containers.
+  control_plane_role_name = aws_iam_role.manager_role.name
+  worker_role_names       = [aws_iam_role.manager_role.name, aws_iam_role.worker_role.name]
+  cluster_name            = var.name
 }
 
 module "aws_ebs_csi_driver" {
-  source        = "./modules/ebs-csi-driver"
-  iam_role_name = aws_iam_role.common_role.name
+  source = "./modules/ebs-csi-driver"
+  // Attached to both roles, as it was to the single role before: the CSI
+  // controller runs on a manager and the node plugin runs everywhere.
+  iam_role_names = [aws_iam_role.manager_role.name, aws_iam_role.worker_role.name]
 }