Merge pull request #88 from stfc/Ci_cd_rally

CI + CD For Rally

Author: apdibbo

.github/workflows/Openstack-Rally-Tester.yaml

@@ -0,0 +1,41 @@
+name: Openstack-Rally-Tester
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    paths:
+      - ".github/workflows/Openstack-Rally-Tester.yaml"
+      - "OpenStack-Rally-Tester/**"
+
+jobs:
+  pylint:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+      - name: Install dependencies
+        run: |
+          cd OpenStack-Rally-Tester && python -m pip install -r requirements.txt
+          python -m pip install pylint
+      
+      - name: Analyse with pylint
+        run: |
+          cd OpenStack-Rally-Tester/usr/local/bin && pylint rally_extract_results.py
+
+  shellcheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Run ShellCheck
+        uses: ludeeus/action-shellcheck@master
+        with:
+          scandir: "./OpenStack-Rally-Tester"

.github/workflows/black.yaml

@@ -31,3 +31,8 @@ jobs:
         uses: psf/black@stable
         with:
             src: "dns_entry_checker"
+
+      - name: Openstack-Rally-Tester
+        uses: psf/black@stable
+        with:
+            src: "OpenStack-Rally-Tester"

OpenStack-Rally-Tester/requirements.txt

@@ -0,0 +1 @@
+requests

OpenStack-Rally-Tester/usr/local/bin/rally-extract-results

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Exit on error
+set -e
+
+rally_test=$1
+
+echo "This may take some time..."
+rally_results_cmd=$(rally task start "$rally_test" | grep "rally task report" | grep "json" | sed 's/output.json//g')
+echo "$rally_results_cmd"
+
+rally_task_id=$(echo "$rally_results_cmd" | cut -d" " -f 4)
+rally_results_path="/tmp/results/$rally_task_id"
+
+if [[ $rally_results_cmd == *'task'* ]]
+then
+    echo "$rally_task_id"
+    mkdir -p /tmp/results
+    echo "$rally_results_path"
+    rally task results "$rally_task_id" > "$rally_results_path";
+else
+    echo "$rally_results_path"
+fi
+
+echo "$rally_results_path"
+rally_extract_command="/usr/local/bin/rally_extract_results.py $rally_results_path"
+echo "$rally_extract_command"
+"$rally_extract_command"

OpenStack-Rally-Tester/usr/local/bin/rally-run-test

@@ -0,0 +1,95 @@
+#!/usr/bin/python3
+"""
+Parses results from the rally task report and sends them to influxdb
+"""
+import json
+import sys
+import time
+from configparser import ConfigParser
+
+import requests
+
+# Read from config file
+parser = ConfigParser()
+parser.read("/etc/openstack-utils/rally-tester.conf")
+host = parser.get("db", "host")
+database = parser.get("db", "database")
+username = parser.get("auth", "username")
+password = parser.get("auth", "password")
+instance = parser.get("cloud", "instance")
+
+url = "http://" + host + "/write?db=" + database
+
+nowtime = time.localtime()
+
+workhours = 9 <= nowtime.tm_hour <= 16
+
+with open(sys.argv[1], encoding="utf-8") as data_file:
+    data = json.load(data_file)
+
+# pylint: disable=invalid-name
+datastring = ""
+metrics = []
+
+if isinstance(data, dict):
+    print(data)
+    for key in data.keys():
+        metric = {}
+        metric["fields"] = {}
+        metric["measurement"] = key
+        metric["fields"]["success"] = 0
+        metrics.append(metric)
+else:
+    for test in data:
+        for result in test["result"]:
+            metric = {}
+            metric["fields"] = {}
+            metric["measurement"] = test["key"]["name"]
+
+            metric["fields"]["success"] = 1
+            for sla in test["sla"]:
+                if not sla["success"]:
+                    metric["fields"]["success"] = 0
+
+            metric["fields"]["duration"] = result["duration"]
+            # metrics.append(metric)
+            for atomic_action in result["atomic_actions"]:
+                metric["fields"][atomic_action] = result["atomic_actions"][
+                    atomic_action
+                ]
+
+            metric["fields"]["timestamp"] = result["timestamp"]
+
+            if test["key"]["name"] == "VMTasks.boot_runcommand_delete":
+                metric["fields"]["image"] = (
+                    '"' + test["key"]["kw"]["args"]["image"]["name"] + '"'
+                )
+                metric["fields"]["network"] = (
+                    '"' + test["key"]["kw"]["args"]["fixednetwork"] + '"'
+                )
+
+            metrics.append(metric)
+
+json_metrics = []
+for metric in metrics:
+    # print metric
+    metric["tags"] = {"instance": instance}
+    json_metrics.append(metric)
+    for field in metric["fields"]:
+        datastring += (
+            metric["measurement"].replace(".", "-")
+            + ",instance="
+            + metric["tags"]["instance"]
+            + ",workhours="
+            + str(workhours)
+        )
+        datastring += " " + field.replace(".", "-") + "=" + str(metric["fields"][field])
+        datastring += "\n"
+
+print(json.dumps(json_metrics, indent=4, sort_keys=True))
+
+print(datastring)
+
+r = requests.post(url, data=datastring, auth=(username, password), timeout=10)
+print(r.text)
+print(r)