Send nightly builds statistics to monitoring #1335 (#1363)

Author: bissquit

.github/workflows/collect-statistics.yml

@@ -62,6 +62,9 @@ env:
   data_path: monitoring/data
   monitoring_properties: monitoring/monitoring.properties
   push_script: monitoring/push_with_rebase.sh
+  PUSHGATEWAY_HOSTNAME: monitoring.utbot.org
+  PUSHGATEWAY_ADDITIONAL_PATH: /pushgateway-custom
+  PROM_ADDITIONAL_LABELS: /service/github
 
 jobs:
   setup_matrix:
@@ -136,7 +139,7 @@ jobs:
         id: insert
         shell: bash
         run: |
-          OUT_FILE="$data_path/$date-$branch-$short_commit-${{ matrix.value }}.json"
+          OUT_FILE="$data_path/$date-$short_commit-${{ matrix.value }}.json"
           echo "output=$OUT_FILE" >> $GITHUB_OUTPUT
 
           INPUTS=($(seq ${{ inputs.run_number }}))
@@ -184,7 +187,9 @@ jobs:
         if: ${{ inputs.send_to_grafana }}
         run: |
           python monitoring/prepare_metrics.py --stats_file $stats_file --output_file grafana_metrics.json
-          echo "TODO send metrics to grafana"
+          chmod +x scripts/project/json_to_prometheus.py
+          python3 scripts/project/json_to_prometheus.py grafana_metrics.json | curl -u "${{ secrets.PUSHGATEWAY_USER }}:${{ secrets.PUSHGATEWAY_PASSWORD }}" --data-binary @- "https://${PUSHGATEWAY_HOSTNAME}${PUSHGATEWAY_ADDITIONAL_PATH}/metrics/job/pushgateway-custom/instance/run-${{ matrix.value }}${PROM_ADDITIONAL_LABELS}"
+          echo "Please visit Grafana to check metrics: https://monitoring.utbot.org/d/m6bagaD4z/utbot-nightly-statistic"
         env:
           stats_file: ${{ steps.insert.outputs.output }}
 

.github/workflows/run-chosen-tests-from-branch.yml

@@ -30,6 +30,7 @@ on:
 env:
   # Environment variable setting gradle options.
   GRADLE_OPTS: "-XX:MaxHeapSize=2048m -Dorg.gradle.jvmargs='-XX:MaxHeapSize=2048m -XX:MaxPermSize=512m -Dorg.gradle.daemon=false' -Dorg.gradle.daemon=false"
+  PUSHGATEWAY_HOSTNAME: monitoring.utbot.org
 
 jobs:
     run-chosen-tests:
@@ -46,7 +47,7 @@ jobs:
         run: |
           echo Find your Prometheus metrics using label {instance=\"${GITHUB_RUN_ID}-${HOSTNAME}\"}
           chmod +x ./scripts/project/monitoring.sh
-          ./scripts/project/monitoring.sh ${{ secrets.PUSHGATEWAY_HOSTNAME }} ${{ secrets.PUSHGATEWAY_USER }} ${{ secrets.PUSHGATEWAY_PASSWORD }}
+          ./scripts/project/monitoring.sh ${PUSHGATEWAY_HOSTNAME} ${{ secrets.PUSHGATEWAY_USER }} ${{ secrets.PUSHGATEWAY_PASSWORD }}
           
       - name: Run chosen tests
         run: |

monitoring/prepare_metrics.py

@@ -82,14 +82,14 @@ def build_metrics_from_data_array(metrics: List[dict], labels: dict) -> List[dic
     return result
 
 
-def build_metrics_from_target(target: dict, runner: str) -> List[dict]:
+def build_metrics_from_target(target: dict, run_id: str) -> List[dict]:
     result = []
     project = target["target"]
 
     result.extend(build_metrics_from_data_array(
         target["summarised"],
         {
-            "runner": runner,
+            "run_id": run_id,
             "project": project,
             "class": "All"
         }
@@ -100,7 +100,7 @@ def build_metrics_from_target(target: dict, runner: str) -> List[dict]:
         result.extend(build_metrics_from_data_array(
             class_item["data"],
             {
-                "runner": runner,
+                "run_id": run_id,
                 "project": project,
                 "class": class_name
             }
@@ -109,10 +109,10 @@ def build_metrics_from_target(target: dict, runner: str) -> List[dict]:
     return result
 
 
-def build_metrics_from_targets(targets: List[dict], runner: str) -> List[dict]:
+def build_metrics_from_targets(targets: List[dict], run_id: str) -> List[dict]:
     metrics = []
     for target in targets:
-        metrics.extend(build_metrics_from_target(target, runner))
+        metrics.extend(build_metrics_from_target(target, run_id))
     return metrics
 
 
@@ -131,11 +131,16 @@ def get_args():
     return args
 
 
+def extract_run_id(text: str):
+    idx = text.find('-')
+    return "run" + text[idx:]
+
+
 def main():
     args = get_args()
     stats = load(args.stats_file)
-    runner = stats["metadata"]["environment"]["host"]
-    metrics = build_metrics_from_targets(stats["targets"], runner)
+    run_id = extract_run_id(stats["metadata"]["source"]["id"])
+    metrics = build_metrics_from_targets(stats["targets"], run_id)
     metrics.sort(key=lambda x: x["metric"])
     with open(args.output_file, "w") as f:
         json.dump(metrics, f, indent=4)

scripts/project/json_to_prometheus.py

@@ -0,0 +1,36 @@
+import sys
+import json
+
+with open(sys.argv[1]) as metrics_raw:
+    metrics_json = json.load(metrics_raw)
+
+# metrics is a json list e.g.:
+#     [
+#         {
+#             "metric": "total_classes",
+#             "labels": {
+#                 "project": "guava",
+#                 "fuzzing_ratio": 0.1
+#             },
+#             "value": 20
+#         },
+#         {
+#             "metric": "testcases_generated",
+#             "labels": {
+#                 "project": "guava",
+#                 "fuzzing_ratio": 0.1
+#             },
+#             "value": 1042
+#         }
+#     ]
+#
+# the loop below iterates over each list item and constructs metrics set
+metrics_set_str = ""
+for metric in metrics_json:
+    labels_set_str = ""
+    comma = ""
+    for label, value in metric['labels'].items():
+        labels_set_str = f'{labels_set_str}{comma}{label}=\"{value}\"'
+        comma = ","
+    metrics_set_str += f'{metric["metric"]}{{{labels_set_str}}} {metric["value"]}\n'
+print(metrics_set_str)