diff --git a/bosh/gcp_labels_ops.go b/bosh/gcp_labels_ops.go index 76d350de9..1152b9c3a 100644 --- a/bosh/gcp_labels_ops.go +++ b/bosh/gcp_labels_ops.go @@ -1,13 +1,16 @@ package bosh import ( + "fmt" + "sort" + "gopkg.in/yaml.v2" ) type gcpLabelsOp struct { - Type string `yaml:"type"` - Path string `yaml:"path"` - Value map[string]string `yaml:"value"` + Type string `yaml:"type"` + Path string `yaml:"path"` + Value interface{} `yaml:"value"` } // GCPLabelsOps renders an ops file that applies GCP resource labels to the @@ -23,3 +26,35 @@ func GCPLabelsOps(labels map[string]string) ([]byte, error) { }, }) } + +// GCPLabelsRuntimeConfigOps renders an ops file that applies the GCP resource +// labels as runtime config tags. The director combines runtime config tags with +// the tags of each deployment, so every VM deployed to the environment is +// labelled, not only the VMs that bbl creates itself. +// +// One replace operation is emitted per label key rather than a single replace of +// the whole /tags map, so that tags defined by other ops files in the same +// runtime config are preserved. The `?` marks tags and the key as optional so +// the operations also create them when they do not exist yet. +// +// BOSH enforces its own reserved VM tags (director, deployment, instance_group, +// job, id, name, index and created_at) and will ignore a label that uses one of +// those keys on deployment VMs. +func GCPLabelsRuntimeConfigOps(labels map[string]string) ([]byte, error) { + keys := make([]string, 0, len(labels)) + for key := range labels { + keys = append(keys, key) + } + sort.Strings(keys) + + ops := make([]gcpLabelsOp, 0, len(keys)) + for _, key := range keys { + ops = append(ops, gcpLabelsOp{ + Type: "replace", + Path: fmt.Sprintf("/tags?/%s?", key), + Value: labels[key], + }) + } + + return yaml.Marshal(ops) +} diff --git a/bosh/gcp_labels_ops_test.go b/bosh/gcp_labels_ops_test.go index ebc8dba02..b97b2267d 100644 --- a/bosh/gcp_labels_ops_test.go +++ b/bosh/gcp_labels_ops_test.go @@ -23,3 +23,21 @@ var _ = Describe("GCPLabelsOps", func() { `)) }) }) + +var _ = Describe("GCPLabelsRuntimeConfigOps", func() { + It("renders one replace operation per label so unrelated tags are preserved", func() { + ops, err := bosh.GCPLabelsRuntimeConfigOps(map[string]string{ + "pipeline": "bosh-deployment", + "owner": "fiwg", + }) + Expect(err).NotTo(HaveOccurred()) + + Expect(string(ops)).To(Equal(`- type: replace + path: /tags?/owner? + value: fiwg +- type: replace + path: /tags?/pipeline? + value: bosh-deployment +`)) + }) +}) diff --git a/docs/advanced-configuration.md b/docs/advanced-configuration.md index 93c95ce52..4f8c90dd7 100644 --- a/docs/advanced-configuration.md +++ b/docs/advanced-configuration.md @@ -127,18 +127,37 @@ bbl up The labels are applied to: * the BOSH director and jumpbox VMs (through the google CPI `labels` cloud property) +* every other VM deployed to the environment: `bbl` sets the labels as a `tags` block in the + director's runtime config, and the director combines runtime config tags with the tags of each + deployment * the GCP resources that `bbl` creates with terraform that support labels (static addresses and the managed DNS zone) +Runtime config tags require a BOSH director running bosh-release v260 or newer. Deployments that are +already running pick the tags up on their next deploy, which updates the VM metadata without +recreating the VMs. Tags that a deployment sets itself take precedence over the runtime config tags, so +a deployment can override a label by setting the same key in its own `tags`. + +BOSH enforces its own reserved VM tags (`director`, `deployment`, `instance_group`, `job`, `id`, +`name`, `index` and `created_at`). A label that uses one of those keys is still applied to the GCP +resources that `bbl` labels directly, but the director will not apply it to deployment VMs, so avoid +reserved keys when the label is meant to reach VMs. + +The director only allows `tags` in a single runtime config. If another runtime config on the director +already defines `tags`, `bbl up` fails with `Runtime config 'tags' key cannot be defined in multiple +runtime configs.` Remove the other definition before enabling `--gcp-label`. + +Persistent disks are only labelled when the CPI implements `set_disk_metadata`, which the google CPI +does not, so disks are not labelled on GCP. + Load balancer forwarding rules are intentionally not labelled: they are not billed, and Google Cloud only supports labels on some forwarding-rule types. Some GCP resources (VPC networks, subnets, firewall rules, routers and target pools) do not support labels and are therefore not labelled. -Note: when labels are propagated to deployment VMs through the BOSH director, the google CPI normalizes -`_` in label keys to `-`. Use `-` in a key if you need the same key on both `bbl`-managed resources and -deployment VMs. +Note: the google CPI normalizes `_` in label keys to `-`. Use `-` in a key if you need the same key on +both `bbl`-managed resources and deployment VMs. Labels are stored in the environment state, so they persist across `bbl plan`/`bbl up` runs. To inspect labelled instances: diff --git a/runtimeconfig/manager.go b/runtimeconfig/manager.go index 258b0028e..e1a7ed40d 100644 --- a/runtimeconfig/manager.go +++ b/runtimeconfig/manager.go @@ -2,6 +2,7 @@ package runtimeconfig import ( "fmt" + "os" "path/filepath" "strings" @@ -22,6 +23,7 @@ type fs interface { fileio.DirReader fileio.Stater fileio.FileReader + fileio.Remover } type logger interface { @@ -83,6 +85,10 @@ func (m Manager) Update(state storage.State) error { return fmt.Errorf("could not find runtime-config directory: %s", err) } + if err := m.syncGCPLabelsOpsFile(dir, state); err != nil { + return fmt.Errorf("failed to sync gcp labels ops file: %s", err) + } + opsFiles := []string{} files, err := m.fs.ReadDir(dir) if err != nil { @@ -105,3 +111,31 @@ func (m Manager) Update(state storage.State) error { return nil } + +// syncGCPLabelsOpsFile keeps the ops file that applies the GCP resource labels +// as runtime config tags in sync with the environment state. The director +// combines runtime config tags with the tags of every deployment, so all VMs +// deployed to the environment are labelled, not only the VMs that bbl creates +// itself. The file is removed when the environment has no GCP labels so that +// stale tags are not left behind. +func (m Manager) syncGCPLabelsOpsFile(dir string, state storage.State) error { + path := filepath.Join(dir, "gcp-labels.yml") + + if state.IAAS != "gcp" || len(state.GCP.Labels) == 0 { + if err := m.fs.Remove(path); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove stale ops file: %s", err) + } + return nil + } + + contents, err := bosh.GCPLabelsRuntimeConfigOps(state.GCP.Labels) + if err != nil { + return fmt.Errorf("marshal ops file: %s", err) + } + + if err := m.fs.WriteFile(path, contents, 0600); err != nil { + return fmt.Errorf("write ops file: %s", err) + } + + return nil +} diff --git a/runtimeconfig/manager_test.go b/runtimeconfig/manager_test.go index 5e1aa46a0..ec1789393 100644 --- a/runtimeconfig/manager_test.go +++ b/runtimeconfig/manager_test.go @@ -130,6 +130,68 @@ var _ = Describe("Manager", func() { Expect(configUpdater.UpdateRuntimeConfigCall.Receives.Name).To(Equal("dns")) }) + Context("when the environment has gcp labels", func() { + BeforeEach(func() { + incomingState.GCP = storage.GCP{ + Labels: map[string]string{"pipeline": "bosh-bootloader"}, + } + // The generated ops file is written before the directory is read, + // so it appears in the listing and is passed to the update. + fileIO.ReadDirCall.Returns.FileInfos = []os.FileInfo{ + fakes.FileInfo{FileName: "runtime-config.yml"}, + fakes.FileInfo{FileName: "gcp-labels.yml"}, + } + }) + + It("writes an ops file that applies the labels as runtime config tags and passes it to the director", func() { + err := manager.Update(incomingState) + Expect(err).NotTo(HaveOccurred()) + + Expect(fileIO.WriteFileCall.CallCount).To(Equal(1)) + Expect(fileIO.WriteFileCall.Receives[0].Filename).To(Equal(filepath.Join("some-runtime-config-dir", "gcp-labels.yml"))) + Expect(fileIO.WriteFileCall.Receives[0].Contents).To(MatchYAML([]byte(`- type: replace + path: /tags?/pipeline? + value: bosh-bootloader +`))) + + Expect(configUpdater.UpdateRuntimeConfigCall.Receives.OpsFilepaths).To(ContainElement( + filepath.Join("some-runtime-config-dir", "gcp-labels.yml"), + )) + }) + + Context("when writing the ops file fails", func() { + BeforeEach(func() { + fileIO.WriteFileCall.Returns = []fakes.WriteFileReturn{{Error: errors.New("some-error")}} + }) + + It("returns an error", func() { + err := manager.Update(incomingState) + Expect(err).To(MatchError("failed to sync gcp labels ops file: write ops file: some-error")) + }) + }) + }) + + Context("when the environment has no gcp labels", func() { + It("removes a stale ops file", func() { + err := manager.Update(incomingState) + Expect(err).NotTo(HaveOccurred()) + + Expect(fileIO.RemoveCall.CallCount).To(Equal(1)) + Expect(fileIO.RemoveCall.Receives[0].Name).To(Equal(filepath.Join("some-runtime-config-dir", "gcp-labels.yml"))) + }) + + Context("when removing a stale ops file fails", func() { + BeforeEach(func() { + fileIO.RemoveCall.Returns = []fakes.RemoveReturn{{Error: errors.New("some-error")}} + }) + + It("returns an error", func() { + err := manager.Update(incomingState) + Expect(err).To(MatchError("failed to sync gcp labels ops file: remove stale ops file: some-error")) + }) + }) + }) + Context("failure cases", func() { Context("when the config updater fails to initialize the authenticated bosh cli", func() { BeforeEach(func() { diff --git a/storage/file_ownership.go b/storage/file_ownership.go index 5a55270f1..a65cca8ae 100644 --- a/storage/file_ownership.go +++ b/storage/file_ownership.go @@ -33,6 +33,7 @@ var bblManaged = []string{ // runtime config "runtime-config/runtime-config.yml", + "runtime-config/gcp-labels.yml", // directories "jumpbox-deployment", diff --git a/storage/garbage_collector_test.go b/storage/garbage_collector_test.go index f6b19bdd8..3fd5b211f 100644 --- a/storage/garbage_collector_test.go +++ b/storage/garbage_collector_test.go @@ -106,18 +106,23 @@ var _ = Describe("garbage collector", func() { }) Describe("runtime-config", func() { - var runtimeConfig string + var ( + runtimeConfig string + gcpLabels string + ) BeforeEach(func() { runtimeConfig = filepath.Join("some-dir", "runtime-config", "runtime-config.yml") + gcpLabels = filepath.Join("some-dir", "runtime-config", "gcp-labels.yml") fileIO.StatCall.Returns.FileInfo = &fakes.DirFileInfo{} }) - It("removes the runtime-config file; removes the directory if there are no user-provided files", func() { + It("removes the runtime-config and generated gcp-labels files; removes the directory if there are no user-provided files", func() { err := gc.Remove("some-dir") Expect(err).NotTo(HaveOccurred()) Expect(fileIO.RemoveAllCall.Receives).To(ContainElement(fakes.RemoveAllReceive{Path: runtimeConfig})) + Expect(fileIO.RemoveAllCall.Receives).To(ContainElement(fakes.RemoveAllReceive{Path: gcpLabels})) // don't remove populated, relevant dirs Expect(fileIO.RemoveCall.Receives).To(ContainElement(fakes.RemoveReceive{ Name: filepath.Join("some-dir", "runtime-config"),