diff --git a/OpenStack-Rally-Tester/.gitignore b/OpenStack-Rally-Tester/.gitignore new file mode 100644 index 00000000..ad28918c --- /dev/null +++ b/OpenStack-Rally-Tester/.gitignore @@ -0,0 +1,220 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[codz] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py.cover +*.lcov +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +# Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +# poetry.lock +# poetry.toml + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. +# https://pdm-project.org/en/latest/usage/project/#working-with-version-control +# pdm.lock +# pdm.toml +.pdm-python +.pdm-build/ + +# pixi +# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. +# pixi.lock +# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one +# in the .venv directory. It is recommended not to include this directory in version control. +.pixi/* +!.pixi/config.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule* +celerybeat.pid + +# Redis +*.rdb +*.aof +*.pid + +# RabbitMQ +mnesia/ +rabbitmq/ +rabbitmq-data/ + +# ActiveMQ +activemq-data/ + +# SageMath parsed files +*.sage.py + +# Environments +.env +.envrc +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +# .idea/ + +# Abstra +# Abstra is an AI-powered process automation framework. +# Ignore directories containing user credentials, local state, and settings. +# Learn more at https://abstra.io/docs +.abstra/ + +# Visual Studio Code +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore +# and can be added to the global gitignore or merged into this file. However, if you prefer, +# you could uncomment the following to ignore the entire vscode folder +# .vscode/ +# Temporary file for partial code execution +tempCodeRunnerFile.py + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc + +# Marimo +marimo/_static/ +marimo/_lsp/ +__marimo__/ + +# Streamlit +.streamlit/secrets.toml \ No newline at end of file diff --git a/OpenStack-Rally-Tester/README.md b/OpenStack-Rally-Tester/README.md new file mode 100644 index 00000000..89d3429b --- /dev/null +++ b/OpenStack-Rally-Tester/README.md @@ -0,0 +1,25 @@ +# Rally Tester Script + +This script scrapes outputted task reports of rally metrics and feeds them to influxdb + +## Prerequisites +- An InfluxDB server running HTTPs (using self-signed certs) +- A Rally VM that is generating reports - that we will deploy this script onto + +## Installation + +setup a python3 virtual environment +```commandline +dnf install python3.12 +python3.12 -m venv opt/extract-rally-venv +``` + +## Developer Notes - WIP + +several bash scripts exist in aquilon that would be better served to be moved here +as those bash scripts reference this python script. Changing the python script here, +will require changing the bash scripts - it would be much easier if they were all managed from a central source + +if placed here, we can run CI/CD jobs against them + +TODO: tests diff --git a/OpenStack-Rally-Tester/pyproject.toml b/OpenStack-Rally-Tester/pyproject.toml new file mode 100644 index 00000000..0228ef6b --- /dev/null +++ b/OpenStack-Rally-Tester/pyproject.toml @@ -0,0 +1,7 @@ +[project] +name = "openstack-rally-tester" +version = "1.0.0" +description = "Add your description here" +readme = "README.md" +requires-python = ">=3.12" +dependencies = ["influxdb"] diff --git a/OpenStack-Rally-Tester/etc/openstack-utils/rally-tester.example.conf b/OpenStack-Rally-Tester/rally-tester.example.conf similarity index 58% rename from OpenStack-Rally-Tester/etc/openstack-utils/rally-tester.example.conf rename to OpenStack-Rally-Tester/rally-tester.example.conf index 9e36601b..e51f030b 100644 --- a/OpenStack-Rally-Tester/etc/openstack-utils/rally-tester.example.conf +++ b/OpenStack-Rally-Tester/rally-tester.example.conf @@ -1,10 +1,7 @@ -[auth] -password= -username= - [cloud] instance= [db] database= host= +port= \ No newline at end of file diff --git a/OpenStack-Rally-Tester/rally_extract_results.py b/OpenStack-Rally-Tester/rally_extract_results.py new file mode 100644 index 00000000..e716c101 --- /dev/null +++ b/OpenStack-Rally-Tester/rally_extract_results.py @@ -0,0 +1,101 @@ +import sys +from typing import Union, Dict, List +from influxdb import InfluxDBClient +from influxdb.exceptions import InfluxDBClientError, InfluxDBServerError + +from utils import load_data, load_config, create_metric, add_tags, is_workhour + +TIMEOUT = 30 # how long to wait to connect to influxdb before timing out (in seconds) + + +def create_metrics(data: Union[Dict, List[Dict]]) -> List[Dict]: + """Convert rally data into a list of metrics.""" + + # if only one rally metric is found + if isinstance(data, dict): + return [ + create_metric( + measurement=key, + fields={"success": 0}, + ) + for key in data + ] + + # if a list of rally metrics are found + metrics = [] + for test in data: + measurement = test["key"]["name"] + + for result in test["result"]: + fields = { + "success": int(all(sla["success"] for sla in test["sla"])), + "duration": result["duration"], + **result["atomic_actions"], + "timestamp": result["timestamp"], + } + + if measurement == "VMTasks.boot_runcommand_delete": + fields.update({ + "image": f'"{test["key"]["kw"]["args"]["image"]["name"]}"', + "network": f'"{test["key"]["kw"]["args"]["fixednetwork"]}"', + }) + + metrics.append( + create_metric( + measurement=measurement, + fields=fields, + ) + ) + return metrics + + +def metrics_to_string(metrics: Dict) -> str: + """Convert metrics to the required output format.""" + lines = [] + + for metric in metrics: + measurement = metric["measurement"].replace(".", "-") + instance = metric["tags"]["instance"] + + for field, value in metric["fields"].items(): + field = field.replace(".", "-") + + lines.append( + f"{measurement}," + f"instance={instance}," + f"workhours={is_workhour()} " + f"{field}={value}" + ) + + return "\n".join(lines) + "\n" + + +def main(): + config = load_config("config.ini") + + influx_client = InfluxDBClient( + host=config["host"], + port=config["port"], + username=config["username"], + password=config["password"], + database=config["database"], + ssl=True, + verify_ssl=False, + timeout=TIMEOUT, + ) + + data = load_data(sys.argv[1]) + + metrics = create_metrics(data) + metrics = add_tags(metrics, config["instance"]) + + data_string = metrics_to_string(metrics) + lines = [line for line in data_string.splitlines() if line.strip()] + + if not lines: + return + + try: + influx_client.write_points(data_string) + except (InfluxDBClientError, InfluxDBServerError) as exc: + raise RuntimeError(f"Failed to write to InfluxDB: {exc}") from exc diff --git a/OpenStack-Rally-Tester/requirements.txt b/OpenStack-Rally-Tester/requirements.txt deleted file mode 100644 index 663bd1f6..00000000 --- a/OpenStack-Rally-Tester/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -requests \ No newline at end of file diff --git a/OpenStack-Rally-Tester/run_rally_tests.sh b/OpenStack-Rally-Tester/run_rally_tests.sh new file mode 100644 index 00000000..e69de29b diff --git a/OpenStack-Rally-Tester/usr/local/bin/rally-run-test.sh b/OpenStack-Rally-Tester/usr/local/bin/rally-run-test.sh deleted file mode 100644 index c7b1f944..00000000 --- a/OpenStack-Rally-Tester/usr/local/bin/rally-run-test.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2023 United Kingdom Research and Innovation - -# Exit on error -set -e - -rally_test=$1 - -echo "This may take some time..." -rally_results_cmd=$(rally task start "$rally_test" | grep "rally task report" | grep "json" | sed 's/output.json//g') -echo "$rally_results_cmd" - -rally_task_id=$(echo "$rally_results_cmd" | cut -d" " -f 4) -rally_results_path="/tmp/results/$rally_task_id" - -if [[ $rally_results_cmd == *'task'* ]] -then - echo "$rally_task_id" - mkdir -p /tmp/results - echo "$rally_results_path" - rally task results "$rally_task_id" > "$rally_results_path"; -else - echo "$rally_results_path" -fi - -echo "$rally_results_path" -rally_extract_command="/usr/local/bin/rally_extract_results.py $rally_results_path" -echo "$rally_extract_command" -"$rally_extract_command" diff --git a/OpenStack-Rally-Tester/usr/local/bin/rally_extract_results.py b/OpenStack-Rally-Tester/usr/local/bin/rally_extract_results.py deleted file mode 100644 index c2d0f148..00000000 --- a/OpenStack-Rally-Tester/usr/local/bin/rally_extract_results.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/python3 -# SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2023 United Kingdom Research and Innovation -""" -Parses results from the rally task report and sends them to influxdb -""" - -import json -import sys -import time -from configparser import ConfigParser - -import requests - -# Read from config file -parser = ConfigParser() -parser.read("/etc/openstack-utils/rally-tester.conf") -host = parser.get("db", "host") -database = parser.get("db", "database") -username = parser.get("auth", "username") -password = parser.get("auth", "password") -instance = parser.get("cloud", "instance") - -url = "http://" + host + "/write?db=" + database - -nowtime = time.localtime() - -workhours = 9 <= nowtime.tm_hour <= 16 - -with open(sys.argv[1], encoding="utf-8") as data_file: - data = json.load(data_file) - -# pylint: disable=invalid-name -datastring = "" -metrics = [] - -if isinstance(data, dict): - print(data) - for key in data.keys(): - metric = {} - metric["fields"] = {} - metric["measurement"] = key - metric["fields"]["success"] = 0 - metrics.append(metric) -else: - for test in data: - for result in test["result"]: - metric = {} - metric["fields"] = {} - metric["measurement"] = test["key"]["name"] - - metric["fields"]["success"] = 1 - for sla in test["sla"]: - if not sla["success"]: - metric["fields"]["success"] = 0 - - metric["fields"]["duration"] = result["duration"] - # metrics.append(metric) - for atomic_action in result["atomic_actions"]: - metric["fields"][atomic_action] = result["atomic_actions"][ - atomic_action - ] - - metric["fields"]["timestamp"] = result["timestamp"] - - if test["key"]["name"] == "VMTasks.boot_runcommand_delete": - metric["fields"]["image"] = ( - '"' + test["key"]["kw"]["args"]["image"]["name"] + '"' - ) - metric["fields"]["network"] = ( - '"' + test["key"]["kw"]["args"]["fixednetwork"] + '"' - ) - - metrics.append(metric) - -json_metrics = [] -for metric in metrics: - # print metric - metric["tags"] = {"instance": instance} - json_metrics.append(metric) - for field in metric["fields"]: - datastring += ( - metric["measurement"].replace(".", "-") - + ",instance=" - + metric["tags"]["instance"] - + ",workhours=" - + str(workhours) - ) - datastring += " " + field.replace(".", "-") + "=" + str(metric["fields"][field]) - datastring += "\n" - -print(json.dumps(json_metrics, indent=4, sort_keys=True)) - -print(datastring) - -r = requests.post(url, data=datastring, auth=(username, password), timeout=10) -print(r.text) -print(r) diff --git a/OpenStack-Rally-Tester/utils.py b/OpenStack-Rally-Tester/utils.py new file mode 100644 index 00000000..10be5eff --- /dev/null +++ b/OpenStack-Rally-Tester/utils.py @@ -0,0 +1,58 @@ +import json +import os +import time +from configparser import ConfigParser + +def load_data(filename): + """Load JSON data from a file.""" + with open(filename, encoding="utf-8") as data_file: + return json.load(data_file) + + +def load_config(config_path: str): + """ loads influx config from config file and environment variables """ + + config = {} + + parser = ConfigParser() + if not parser.read(config_path, encoding="utf-8"): + raise FileNotFoundError(f"Config file {config_path} not found") + + config["host"] = parser.get("db", "host") + config["port"] = parser.get("db", "port") + config["database"] = parser.get("db", "database") + config["instance"] = parser.get("cloud", "instance") + + def require_env(name: str) -> str: + """ + helper to load in environment variables + :param name: environment variable name + :return: environment variable value + """ + value = os.getenv(name) + if not value: + raise ValueError(f"{name}: influxdb environment variable not set") + return value + + config["username"] = require_env("INFLUXDB_USERNAME") + config["password"] = require_env("INFLUXDB_PASSWORD") + return config + + +def create_metric(measurement, fields): + """Create a metric dictionary.""" + return { + "measurement": measurement, + "fields": fields, + } + +def add_tags(metrics, instance): + """Add instance tag to each metric.""" + for metric in metrics: + metric["tags"] = {"instance": instance} + return metrics + +def is_workhour() -> bool: + """Returns if it is currently a work hour.""" + nowtime = time.localtime() + return 9 <= nowtime.tm_hour <= 16