From f224d84994f4c48096874cd528c5de00d9b5edf6 Mon Sep 17 00:00:00 2001 From: Kalibh Halford Date: Mon, 6 Jul 2026 09:39:34 +0100 Subject: [PATCH 1/4] MAINT: Update influx1 to influx2 influx1 is being retired and all the data has been copied over to influx2 which is now running on Rocky 9. --- .../roles/grafana/templates/cloud_datasource.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grafana_monitoring/roles/grafana/templates/cloud_datasource.yaml.j2 b/grafana_monitoring/roles/grafana/templates/cloud_datasource.yaml.j2 index b4ef6189..5a8f16d5 100644 --- a/grafana_monitoring/roles/grafana/templates/cloud_datasource.yaml.j2 +++ b/grafana_monitoring/roles/grafana/templates/cloud_datasource.yaml.j2 @@ -37,7 +37,7 @@ datasources: access: proxy uid: "openstack_grafana" user: {{ prod_influx_username }} - url: "http://influx1.nubes.rl.ac.uk:8086" + url: "http://influx2.nubes.rl.ac.uk:8086" jsonData: dbName : "cloud" httpMode: POST From 8770475491dc5f30c8e4be0f65ce82b0432876e9 Mon Sep 17 00:00:00 2001 From: Kalibh Halford Date: Thu, 3 Sep 2026 09:11:15 +0100 Subject: [PATCH 2/4] BUG: Remove mail option from certbot Let's encrypt no longer sends emails and this flag breaks the command. --- grafana_monitoring/roles/certbot/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grafana_monitoring/roles/certbot/tasks/main.yml b/grafana_monitoring/roles/certbot/tasks/main.yml index c1c4b9e5..677da700 100644 --- a/grafana_monitoring/roles/certbot/tasks/main.yml +++ b/grafana_monitoring/roles/certbot/tasks/main.yml @@ -37,7 +37,7 @@ - name: Generate the certificate for the first time become: true - ansible.builtin.shell: "certbot certonly --standalone --non-interactive --agree-tos --domains {{ inventory_hostname }} -m cloud-support@stfc.ac.uk" + ansible.builtin.shell: "certbot certonly --standalone --non-interactive --agree-tos --domains {{ inventory_hostname }} -v" when: not certificate_file.stat.exists - name: Copy certificate for the first time From 5b303b3ea397fb7c8fdd4862bbac4ad082b65a80 Mon Sep 17 00:00:00 2001 From: Kalibh Halford Date: Thu, 3 Sep 2026 09:11:51 +0100 Subject: [PATCH 3/4] BUG: Sometimes the repo doesn't get cleaned properly Make sure the repo is set back to the origin --- .../roles/grafana/templates/provision_dashboards.sh.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/grafana_monitoring/roles/grafana/templates/provision_dashboards.sh.j2 b/grafana_monitoring/roles/grafana/templates/provision_dashboards.sh.j2 index 87ae2ff0..361587d5 100644 --- a/grafana_monitoring/roles/grafana/templates/provision_dashboards.sh.j2 +++ b/grafana_monitoring/roles/grafana/templates/provision_dashboards.sh.j2 @@ -14,6 +14,7 @@ BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD) if [[ "$BRANCH_NAME" == "{{ grafana_dashboards_branch }}" ]]; then git fetch && git reset --h origin/{{ grafana_dashboards_branch }} + git clean -fd fi {% if inventory_hostname.startswith("grafana") %} From 73a1357555c516e0590a980ced2d5a787052d2ee Mon Sep 17 00:00:00 2001 From: Kalibh Halford Date: Thu, 3 Sep 2026 09:12:37 +0100 Subject: [PATCH 4/4] MAINT: Update datasources Remove unused or broken datasources. Update the uid of datasources to represent what they are. The dashboards have also been updated in the cloud-grafana-dashboards repository --- grafana_monitoring/group_vars/grafana/vars | 13 +-- grafana_monitoring/group_vars/grafana/vault | 101 ++++++++---------- .../templates/cloud_datasource.yaml.j2 | 70 ++---------- 3 files changed, 60 insertions(+), 124 deletions(-) diff --git a/grafana_monitoring/group_vars/grafana/vars b/grafana_monitoring/group_vars/grafana/vars index 6cc2d206..6c93802b 100644 --- a/grafana_monitoring/group_vars/grafana/vars +++ b/grafana_monitoring/group_vars/grafana/vars @@ -7,17 +7,12 @@ client_id: "{{ vault_client_id }}" client_secret: "{{ vault_client_secret }}" weekly_reports_token: "{{ vault_weekly_reports_token }}" weekly_reports_token_dw: "{{ vault_weekly_reports_token_dw }}" -prod_opensearch_password: "{{ vault_prod_opensearch_password }}" -prod_influx_username: "{{ vault_prod_influx_username }}" -prod_influx_password: "{{ vault_prod_influx_password }}" -dev_influx_username: "{{ vault_dev_influx_username }}" -dev_influx_password: "{{ vault_dev_influx_password }}" +prod_cloud_influx_username: "{{ vault_prod_influx_username }}" +prod_cloud_influx_password: "{{ vault_prod_influx_password }}" grafana_admin_password: "{{ vault_grafana_admin_password }}" -meerkat_db_username: "{{ vault_meerkat_db_username }}" -meerkat_db_password: "{{ vault_meerkat_db_password }}" dev_chatops_prometheus_username: "{{ vault_dev_chatops_prometheus_username }}" dev_chatops_prometheus_password: "{{ vault_dev_chatops_prometheus_password }}" prod_chatops_prometheus_username: "{{ vault_prod_chatops_prometheus_username }}" prod_chatops_prometheus_password: "{{ vault_prod_chatops_prometheus_password }}" -remote_write_prometheus_username: "{{ vault_remote_write_prometheus_username }}" -remote_write_prometheus_password: "{{ vault_remote_write_prometheus_password }}" +power_metrics_prometheus_username: "{{ vault_power_metrics_prometheus_username }}" +power_metrics_prometheus_password: "{{ vault_power_metrics_prometheus_password }}" diff --git a/grafana_monitoring/group_vars/grafana/vault b/grafana_monitoring/group_vars/grafana/vault index 5c5e3e86..e39ae9c3 100644 --- a/grafana_monitoring/group_vars/grafana/vault +++ b/grafana_monitoring/group_vars/grafana/vault @@ -1,56 +1,47 @@ $ANSIBLE_VAULT;1.1;AES256 -36663065393566663035323862623033326432646565333866326465356130653265643435626636 -3532396262326336643236356536666463613631613639390a343363346632363638353030376137 -31343137396666323731376235626662346532306632626331343333636465313533343334373339 -3036313264363261650a633336343238386237343337633533643037373236323064623038366162 -66623264613365626162383261613065396138393634356631376563316434316461353134333836 -33616364653866323837376434636431643234383164303631633835303462666633393837346239 -32393030326637396563386566346233343366633263653133653834326538376439333630323763 -34366130363338393034633434393136666163653161646137616564623733396632316165646539 -61353931643430366238316337363761636563373333653563326163643934373932373137643634 -34643435616333636638316562303439356464626337323235323366353139636566623635613563 -37393734316637623932613564643130376665323065623062366439323462333234373439643134 -38666564323362626263393266336531613865303964383930303133656632656635313234333439 -39323164346338633235643837346435346264346339643166336662663661353733643933643961 -36323434353963373532653436306137303133386437616133666263333733333134323163663935 -33646532343838343030643237346566666461316437316361393165316638346564313938343161 -65623938623431623832376436393561353666353930623362633262376134386266613330383132 -61373338633963353461383462633139373161663439376436343930613061326163643638653134 -32333561636461366632336637373666386538616432303530363534313633333838343935383933 -65393430333237313031306531326233356339643533626338396534666139333465313339333862 -32316364613137393061333632393837363331653230316662303661373032393330363264613036 -34353435643165636463386564376433366262373536633966653261343162363436663534306362 -38333538646262333561383133376163333837303463306133633539653363623333333037646136 -33646432626332616231346565663330353763623566346639653065396637306664616462653733 -64383937643536323464363937363833303132363935313265656233633266663835376337373964 -35353438303766653164326133316633336263353530313730396632346337393437613138613930 -31643637306262333837353936613233666539656536616236616234333866643566353733366632 -36353863353837353163326264323738373935656332636564333338363039333462316164386161 -30616639343637393236663165643630396538616137383864306337613865613465366166663237 -62643166643734376138643461343062306233373038633062656439306230316261336164653961 -30376534646536363838326566666438373133373537373138373132353561303361663165353664 -61303539633965643438663863623832393437383138636532626164393430343261306637313365 -35663733306363353565373961373334636433636235656532383034623966346239343433643236 -30333232656230613233363037363837666531346661306563346166353062623033333034636563 -66353337643134633238343237643936353137663237613865633435646262613064343636346363 -31666264653239326463393733653333323762373936646166623361363032333462326263313730 -39643432623239323762623366633363313536643836653238363539663466646563306638663235 -34313861633937346135373366373436646136306636376461626161363737363130663161383335 -62666639333163386666313062316166386334626231396231306664393932303934653934633336 -36386638653061373564643261633463306631336231353462303331393737623330646438306265 -38653534363364633137343833376334616335333030393833656333303930316530336337636662 -32383530623731383032303531313234303566306132303839336331306431623733656131383237 -38396465326232333033313530663265333434363263356237613161646565353361313232336530 -62396533393632626235623365366634616631326531313230633236303436366338333963343338 -39623564663566623131653632613163343437313536643962646366646262373138313531333230 -37353564613663373261636636356435626631303737393939636532663161376564623731393436 -61323937356135633234343365356231313363383937383939626235383766343263643337356265 -31333663653334383761343631616537366238313939326562356131643434323461633664306133 -62323837323838666635363337376365366365643337623866386362613337633837636533313362 -35643232373164303666313264316561343736303135316533663834646336623332636363326633 -37376261313538616533633334343863306233313763363063616534623937353730333532303237 -33643466666564643633303366333031353764373638393832656365303662326661643536333433 -34303031373866316165363336623063346165633831303461356139353766373364353338383434 -38663232623165333833386462363133346336316339643964376463336339633836666362383438 -65333033326130373937666361346539646663396465303738303133396362346561353363316633 -383330646433353437363135626464643031 +31313631356561373838353363616237336361326533636261613531633637363161633136356364 +3864323735363466396462386136396436376464366563660a386238343032326432633261623732 +38653233323438643635656466363261613061653362306333626335623964363936393138376664 +6636393730366361370a393065623838343466346666666531393934663336636534386138666334 +32333737633764663836616434643239306461633036303638373965363637343965303265653535 +33386566613164336164633836653231613631656264633764323339303065386232336132343432 +38656538356461313135393466336337306464336164373363373939373538363039326265316162 +33383535626461366632313365663363343637643630626362373130623834616137333266366634 +30313935353133616230396439376236303732646664656365666636383739613432326435303134 +61666338363137303332613365623634643662333334393666303532653533333039383933333239 +62396539353336643934643462303334663137656562313731313733313965303237643031353165 +33653935626364646163623361633638616264316665336232373936633430633861666565343537 +30353931613962353136363137616463383537306436303165653163643935663331653735653866 +62303737333637333435306637336336613165646631616638333362633463313766633237303565 +64326335326239366661613363643863386533383436366261643839353765373161346131646330 +36353436323463623234663637623038373963383563396133643237633930356332303566656430 +36323237396333633635366232383734323366316431396562373134323238346439313038336530 +35373033313636353864313239316439326231666238343932306536383763383634633964666164 +64616430326664626662623266666337333366616434343736313437343461613363346265356565 +34333539343031353530633034663833653661373633666161626436653761373435303033666433 +38646137353765316363303066383036633031313136333235613235636337643564303963663262 +30363533333966666435376335383363613434316539663163333432653330663334323763323565 +35376337343062636264373761303863646266616635613434336662666264636362616163323931 +33316534386533373530323166333066623837616664336330666537386339386637363131383836 +66363364666661613030316466653636393435653435313032666238373135313737366461393164 +61396230373866653134393065373262373138623031383762373939306365316630356339383739 +31663030336538306230633166323565303434313064333162303537316361643037363639383233 +62373135346564306461336664383362363565343566623966376133383833623862383332396238 +61393736643939303662363537346262613937306663316430333162653438353231646165333632 +64653337616633396366616232663666316337336466303139663731366166366664666335643232 +39636333623737323662386234633466363530653337623038653434373264313038323535323332 +63653066633462363264303637616338363062343338313535356535626534313162346561383235 +39303538616234613833363861316537333430373265316438636465326164306137343664653463 +65346261646632393930323034633366643762366237386234306137346362323433623034303363 +66356431363039393262616230613766633338326466653830346230613433383338326431613836 +33303635646333663031383933336432633232343862323538333734316633663333613838366137 +62663934333362633265613364623735316335366236373733663538333638656432663537313461 +39666366336233633337386330323764383964653561613234356363376438343934643339643763 +30373763323366366338623166316539643833653832626666613263306434656661663063383238 +32636639346262373664343935306462353566653932376364633762333136626665396139383331 +61303161336162306266336637613536653962336332393134376231373735366639636134343166 +32306132653234346366326337653337373562313563616563613638653961313466643165383532 +35663230313631663635623931663537633330633966646261366339643037313830353931346535 +37313261663430653038336330316238646631393936366361303830316631323031316665316331 +38643763393038336238396437373963626364643132336430383163396366633663376464616135 +6135353937316464343164336163386636326531313133336533 diff --git a/grafana_monitoring/roles/grafana/templates/cloud_datasource.yaml.j2 b/grafana_monitoring/roles/grafana/templates/cloud_datasource.yaml.j2 index 5a8f16d5..c5e3007c 100644 --- a/grafana_monitoring/roles/grafana/templates/cloud_datasource.yaml.j2 +++ b/grafana_monitoring/roles/grafana/templates/cloud_datasource.yaml.j2 @@ -2,67 +2,37 @@ apiVersion: 1 prune: true datasources: # These are our datasources safe for public query - - name: RemoteWritePrometheus + - name: PowerMetricsPrometheus type: prometheus - uid: "remote_write_prometheus" + uid: "power_metrics_prometheus" access: proxy url: http://130.246.83.213:9090 isDefault: false basicAuth: true - basicAuthUser: {{ remote_write_prometheus_username }} + basicAuthUser: {{ power_metrics_prometheus_username }} jsonData: timeout: 120 secureJsonData: - basicAuthPassword: {{ remote_write_prometheus_password }} + basicAuthPassword: {{ power_metrics_prometheus_password }} - - name: DevCloudInfluxDB + - name: CloudInfluxDB type: influxdb access: proxy - uid: "dev_openstack_grafana" - user: {{ dev_influx_username }} - url: "http://172.16.110.169:8086" + uid: "prod_cloud_influx" + user: {{ prod_cloud_influx_username }} + url: "https://influx2.nubes.rl.ac.uk:8086" jsonData: + tlsSkipVerify: true dbName : "cloud" httpMode: POST timeInterval: "60" maxSeries: 200 secureJsonData: - password: {{ dev_influx_password }} + password: {{ prod_cloud_influx_password }} # These datasources should not be queried by the public. # That's why they are only on the dev instance {% if inventory_hostname.startswith("dev") %} - - name: CloudInfluxDB - type: influxdb - access: proxy - uid: "openstack_grafana" - user: {{ prod_influx_username }} - url: "http://influx2.nubes.rl.ac.uk:8086" - jsonData: - dbName : "cloud" - httpMode: POST - timeInterval: "60" - maxSeries: 200 - secureJsonData: - password: {{ prod_influx_password }} - - - name: grafana-power-metrics - type: grafana-opensearch-datasource - access: proxy - uid: "opensearch-grafana" - url: https://172.16.114.193:9200 - basicAuth: true - basicAuthUser: admin - jsonData: - tlsSkipVerify: true - database: "[cloud-ipmi-power-]YYYY.MM.DD" - interval: Daily - timeField: "@timestamp" - version: "2.3.0" - flavor: "opensearch" - secureJsonData: - basicAuthPassword: {{ prod_opensearch_password }} - - name: WeeklyReportsInfluxDB type: influxdb access: proxy @@ -78,17 +48,6 @@ datasources: secureJsonData: token: {{ weekly_reports_token }} - - name: Meerkat DB - type: prometheus - uid: "meerkat-db" - access: proxy - url: https://meerkatdb.nubes.rl.ac.uk:8247 - isDefault: false - basicAuth: true - basicAuthUser: {{ meerkat_db_username }} - secureJsonData: - basicAuthPassword: {{ meerkat_db_password }} - - name: DevChatOps type: prometheus uid: "dev-chatops" @@ -110,13 +69,4 @@ datasources: basicAuthUser: {{ prod_chatops_prometheus_username }} secureJsonData: basicAuthPassword: {{ prod_chatops_prometheus_password }} - - - name: HarborPrometheus - type: prometheus - uid: "harbor-prometheus" - access: proxy - url: http://130.246.83.216:9090 - isDefault: false - basicAuth: false - {% endif %}