Push agent v2 logs to QE grafana (#1185)

Author: john-david3

.github/actions/start-promtail/action.yml

@@ -0,0 +1,30 @@
+name: Start Promtail
+description: Start promtail in a Docker container to send test results to loki
+inputs:
+    loki-dashboard-url:
+        description: "URL of the Loki dashboard to send the logs"
+        required: true
+runs:
+    using: "composite"
+    steps:
+        - name: Start Promtail container
+          shell: bash
+          run: |
+              docker run -d \
+               --name=promtail \
+               -v "${{ github.workspace }}/test/dashboard/prep/promtail.yaml:/etc/promtail/config.yaml" \
+               -v "${{ github.workspace }}/test/dashboard/logs:/var/log" \
+               -e TEST_OUTDIR=test/dashboard/logs \
+               -e GITHUB_RUN_ID=${{ github.run_id }} \
+               -e GITHUB_JOB=${{ github.job }} \
+               -e GITHUB_WORKFLOW=${{ github.workflow }} \
+               -e GITHUB_EVENT_NAME=${{ github.event_name }} \
+               -e GITHUB_SERVER_URL=${{ github.server_url }} \
+               -e GITHUB_REPOSITORY=${{ github.repository }} \
+               -e GITHUB_HEAD_REF=${{ github.head_ref }} \
+               -e GITHUB_SHA=${{ github.sha }} \
+               -e GITHUB_ACTOR=${{ github.actor }} \
+               -e LOKI_DASHBOARD_URL=${{ inputs.loki-dashboard-url }} \
+               grafana/promtail:3.4.4 \
+               -config.file=/etc/promtail/config.yaml \
+               -config.expand-env=true

.github/workflows/ci.yml

@@ -57,8 +57,19 @@ jobs:
       - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
         with:
           go-version-file: 'go.mod'
+      - name: Set Start Time
+        run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
+      - name: Create Directory
+        run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/
+      - name: Start Promtail
+        uses: ./.github/actions/start-promtail
+        with:
+            loki-dashboard-url: ${{ secrets.LOKI_DASHBOARD_URL }}
       - name: Run Unit Tests
-        run: make unit-test
+        run: make unit-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+      - name: Generate Test Results
+        if: always()
+        run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}} ${{github.workspace}}
       - name: Upload Test Coverage
         uses: codecov/codecov-action@84508663e988701840491b86de86b666e8a86bed # v4.3.0
         with:
@@ -161,18 +172,29 @@ jobs:
             with:
                 name: nginx-agent-unsigned-snapshots
                 path: build
+          - name: Set Start Time
+            run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
+          - name: Create Directory
+            run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/
+          - name: Start Promtail
+            uses: ./.github/actions/start-promtail
+            with:
+                loki-dashboard-url: ${{ secrets.LOKI_DASHBOARD_URL }}
           - name: Run Integration Tests
             run: |
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
                 OS_RELEASE="${{ matrix.container.image }}" OS_VERSION="${{ matrix.container.version }}" \
-                  make integration-test
+                  make integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+          - name: Generate Test Results
+            if: always()
+            run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}/${{matrix.container.image}}${{matrix.container.version}} ${{github.workspace}}
           - name: Container Output Logs
             if: failure()
             run: |
                 docker ps -a
                 dockerid=$(docker ps -a --format "{{.ID}}")
                 docker logs "$dockerid"
-
+                
           - name: Archive integration test logs
             if: success() || failure()
             uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
@@ -214,11 +236,22 @@ jobs:
         with:
           name: nginx-agent-unsigned-snapshots
           path: build
+      - name: Set Start Time
+        run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
+      - name: Create Directory
+        run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/
+      - name: Start Promtail
+        uses: ./.github/actions/start-promtail
+        with:
+            loki-dashboard-url: ${{ secrets.LOKI_DASHBOARD_URL }}
       - name: Run Integration Tests
         run: |
           go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
           CONTAINER_NGINX_IMAGE_REGISTRY="${{ env.NGINX_OSS_REGISTRY }}" TAG="${{ matrix.container.version }}-${{ matrix.container.image }}" OS_RELEASE="${{ matrix.container.release }}"\
-            make official-image-integration-test
+            make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+      - name: Generate Test Results
+        if: always()
+        run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}/${{matrix.container.image}}${{matrix.container.version}} ${{github.workspace}}
       - name: Container Output Logs 
         if: failure()
         run: |
@@ -281,12 +314,23 @@ jobs:
                 registry: ${{ secrets.REGISTRY_URL }}
                 username: ${{ secrets.REGISTRY_USERNAME }}
                 password: ${{ secrets.REGISTRY_PASSWORD }}
+          - name: Set Start Time
+            run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
+          - name: Create Directory
+            run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/
+          - name: Start Promtail
+            uses: ./.github/actions/start-promtail
+            with:
+                loki-dashboard-url: ${{ secrets.LOKI_DASHBOARD_URL }}
           - name: Run Integration Tests
             run: |
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
                 CONTAINER_NGINX_IMAGE_REGISTRY="${{ secrets.REGISTRY_URL }}" TAG="${{ matrix.container.plus }}-${{ matrix.container.image }}-${{ matrix.container.version }}" \
                 OS_RELEASE="${{ matrix.container.release }}" IMAGE_PATH="${{ matrix.container.path }}" \
-                  make official-image-integration-test
+                  make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+          - name: Generate Test Results
+            if: always()
+            run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}/${{matrix.container.image}}${{matrix.container.version}} ${{github.workspace}}
           - name: Container Output Logs
             if: failure()
             run: |

Makefile

@@ -193,16 +193,16 @@ unit-test: $(TEST_BUILD_DIR) test-core test-plugins test-sdk test-extensions ##
 	@printf "\nTotal code coverage: " && $(GOTOOL) cover -func=$(TEST_BUILD_DIR)/coverage.out | grep 'total:' | awk '{print $$3}'
 
 test-core: $(TEST_BUILD_DIR) ## Run core unit tests
-	GOWORK=off CGO_ENABLED=0 go test -count=1 -coverprofile=$(TEST_BUILD_DIR)/core_coverage.out -covermode count ./src/core/...
+	GOWORK=off CGO_ENABLED=0 go test -v -count=1 -coverprofile=$(TEST_BUILD_DIR)/core_coverage.out -covermode count ./src/core/...
 
 test-plugins: $(TEST_BUILD_DIR) ## Run plugins unit tests
-	GOWORK=off CGO_ENABLED=0 go test -count=1 -coverprofile=$(TEST_BUILD_DIR)/plugins_coverage.out -covermode count ./src/plugins/...
+	GOWORK=off CGO_ENABLED=0 go test -v -count=1 -coverprofile=$(TEST_BUILD_DIR)/plugins_coverage.out -covermode count ./src/plugins/...
 
 test-extensions: $(TEST_BUILD_DIR) ## Run extensions unit tests
-	GOWORK=off CGO_ENABLED=0 go test -count=1 -coverprofile=$(TEST_BUILD_DIR)/extensions_coverage.out -covermode count ./src/extensions/...
+	GOWORK=off CGO_ENABLED=0 go test -v -count=1 -coverprofile=$(TEST_BUILD_DIR)/extensions_coverage.out -covermode count ./src/extensions/...
 
 test-sdk: $(TEST_BUILD_DIR) ## Run sdk unit tests from root directory
-	cd sdk && GOWORK=off CGO_ENABLED=0 go test -count=1 -coverprofile=../$(TEST_BUILD_DIR)/sdk_coverage.out -covermode count ./...
+	cd sdk && GOWORK=off CGO_ENABLED=0 go test -v -count=1 -coverprofile=../$(TEST_BUILD_DIR)/sdk_coverage.out -covermode count ./...
 
 # Component tests
 component-test: test-component-build test-component-run ## Run component tests

docs/proto/proto.md

@@ -18,9 +18,6 @@
     - [AgentConnectStatus.StatusCode](#f5-nginx-agent-sdk-AgentConnectStatus-StatusCode)
     - [AgentLogging.Level](#f5-nginx-agent-sdk-AgentLogging-Level)
 
-- [command_svc.proto](#command_svc-proto)
-    - [Commander](#f5-nginx-agent-sdk-Commander)
-  
 - [command.proto](#command-proto)
     - [AgentActivityStatus](#f5-nginx-agent-sdk-AgentActivityStatus)
     - [ChunkedResourceChunk](#f5-nginx-agent-sdk-ChunkedResourceChunk)
@@ -42,6 +39,9 @@
     - [NginxConfigStatus.Status](#f5-nginx-agent-sdk-NginxConfigStatus-Status)
     - [UploadStatus.TransferStatus](#f5-nginx-agent-sdk-UploadStatus-TransferStatus)
 
+- [command_svc.proto](#command_svc-proto)
+    - [Commander](#f5-nginx-agent-sdk-Commander)
+  
 - [common.proto](#common-proto)
     - [CertificateDates](#f5-nginx-agent-sdk-CertificateDates)
     - [CertificateName](#f5-nginx-agent-sdk-CertificateName)
@@ -341,34 +341,6 @@ Log level enum
 
 
 
-<a name="command_svc-proto"></a>
-<p align="right"><a href="#top">Top</a></p>
-
-## command_svc.proto
-
-
- 
-
- 
-
- 
-
-
-<a name="f5-nginx-agent-sdk-Commander"></a>
-
-### Commander
-Represents a service used to sent command messages between the management server and the agent.
-
-| Method Name | Request Type | Response Type | Description |
-| ----------- | ------------ | ------------- | ------------|
-| CommandChannel | [Command](#f5-nginx-agent-sdk-Command) stream | [Command](#f5-nginx-agent-sdk-Command) stream | A Bidirectional streaming RPC established by the agent and is kept open |
-| Download | [DownloadRequest](#f5-nginx-agent-sdk-DownloadRequest) | [DataChunk](#f5-nginx-agent-sdk-DataChunk) stream | A streaming RPC established by the agent and is used to download resources associated with commands The download stream will be kept open for the duration of the data transfer and will be closed when its done. The transfer is a stream of chunks as follows: header -&gt; data chunk 1 -&gt; data chunk N. Each data chunk is of a size smaller than the maximum gRPC payload |
-| Upload | [DataChunk](#f5-nginx-agent-sdk-DataChunk) stream | [UploadStatus](#f5-nginx-agent-sdk-UploadStatus) | A streaming RPC established by the agent and is used to upload resources associated with commands |
-
- 
-
-
-
 <a name="command-proto"></a>
 <p align="right"><a href="#top">Top</a></p>
 
@@ -680,6 +652,34 @@ Transfer status enum
 
 
 
+<a name="command_svc-proto"></a>
+<p align="right"><a href="#top">Top</a></p>
+
+## command_svc.proto
+
+
+ 
+
+ 
+
+ 
+
+
+<a name="f5-nginx-agent-sdk-Commander"></a>
+
+### Commander
+Represents a service used to sent command messages between the management server and the agent.
+
+| Method Name | Request Type | Response Type | Description |
+| ----------- | ------------ | ------------- | ------------|
+| CommandChannel | [Command](#f5-nginx-agent-sdk-Command) stream | [Command](#f5-nginx-agent-sdk-Command) stream | A Bidirectional streaming RPC established by the agent and is kept open |
+| Download | [DownloadRequest](#f5-nginx-agent-sdk-DownloadRequest) | [DataChunk](#f5-nginx-agent-sdk-DataChunk) stream | A streaming RPC established by the agent and is used to download resources associated with commands The download stream will be kept open for the duration of the data transfer and will be closed when its done. The transfer is a stream of chunks as follows: header -&gt; data chunk 1 -&gt; data chunk N. Each data chunk is of a size smaller than the maximum gRPC payload |
+| Upload | [DataChunk](#f5-nginx-agent-sdk-DataChunk) stream | [UploadStatus](#f5-nginx-agent-sdk-UploadStatus) | A streaming RPC established by the agent and is used to upload resources associated with commands |
+
+ 
+
+
+
 <a name="common-proto"></a>
 <p align="right"><a href="#top">Top</a></p>
 

scripts/workflow/generate_results.sh

@@ -0,0 +1,115 @@
+#!/bin/bash
+
+# Script to process test logs and generate formatted JSON result files
+# Usage: ./generate_results.sh <job_result> <start_time> <test_type> <workspace>
+
+set -euo pipefail
+
+# Check if the correct number of arguments are provided
+if [ "$#" -ne 4 ]; then
+    echo "Usage: $0 <job_result> <start_time> <test_type> <workspace>"
+    exit 1
+fi
+
+JOB_RESULT="$1"
+START_TIME="$2"
+TEST_TYPE="$3"
+WORKSPACE="$4"
+
+INPUT_FILE="$WORKSPACE/test/dashboard/logs/$TEST_TYPE/raw_logs.log"
+OUTPUT_PATH="$WORKSPACE/test/dashboard/logs/$TEST_TYPE"
+
+# Validate input file exists
+if [ ! -f "$INPUT_FILE" ]; then
+    echo "Error: Input file not found: $INPUT_FILE"
+    exit 1
+fi
+
+END_TIME="`date "+%Y-%m-%dT%H:%M:%S.%NZ"`"
+START_SECONDS=$(date -d "$START_TIME" +%s.%N)
+END_SECONDS=$(date -d "$END_TIME" +%s.%N)
+DURATION=$(echo "$END_SECONDS - $START_SECONDS" | bc)
+
+MSG=""        # individual test msg
+FAIL_MSG=""   # msg for entire job run
+RESULT=""
+HAS_FAILED=false
+IS_RUNNING=false
+
+load_job_status(){
+    if [ "$JOB_RESULT" == "success" ]; then
+        RESULT="pass"
+    elif [ "$JOB_RESULT" == "failure" ]; then
+        RESULT="fail"
+    else
+        RESULT="skip"
+    fi
+}
+
+format_logs_to_json(){
+    line="$1"
+    json="{"
+    
+    while [[ "$line" =~ ([a-zA-Z0-9_]+)=((\"([^\"\\]|\\.)*\")|[^[:space:]]+) ]]; do
+        key="${BASH_REMATCH[1]}"
+        value="${BASH_REMATCH[2]}"
+        line="${line#*"${key}=${value}"}"
+        
+        if [[ "$value" == \"*\" ]]; then
+            value="${value:1:${#value}-2}"
+            value="${value//\"/\\\"}"
+        fi
+        json+="\"$key\":\"$value\","
+    done
+    
+    json="${json%,}}"    
+    echo "$json"
+}
+
+format_results(){
+    while IFS= read -r line; do
+        
+        if [[ "$line" =~ ^===\ RUN[[:space:]]+(.+) ]]; then
+            TEST_NAME="${BASH_REMATCH[1]}"
+            IS_RUNNING=true
+            MSG=""
+            TEST_START=""
+            TEST_END=""
+            mkdir -p "$OUTPUT_PATH/$TEST_NAME/"
+            RESULT_FILE="$OUTPUT_PATH/$TEST_NAME/result.json"
+            LOG_FILE="$OUTPUT_PATH/$TEST_NAME/test.log"
+        elif [[ "$line" =~ ([0-9T:\.\-Z]+)[[:space:]]+testing ]]; then
+            TEST_START="${BASH_REMATCH[1]}"
+        elif [[ "$line" =~ ([0-9T:\.\-Z]+)[[:space:]]+finished[[:space:]]testing ]]; then
+            TEST_END="${BASH_REMATCH[1]}"
+        elif [[ "$line" == "FAIL" ]]; then
+            HAS_FAILED=false
+            MSG="$MSG_STR"
+            FAIL_MSG+="$MSG"
+            HAS_FAILED=false
+            echo "{\"start_at\": \"$START_TIME\", \"end_at\": \"$END_TIME\", \"duration_seconds\": \"$DURATION\", \"result\": \"$TEST_RES\", \"msg\": \"$MSG\"}" > $RESULT_FILE
+        elif [[ "$line" == "--- PASS"* ]]; then
+            TEST_RES="pass"
+            IS_RUNNING=false
+            echo "{\"start_at\": \"$START_TIME\", \"end_at\": \"$END_TIME\", \"duration_seconds\": \"$DURATION\", \"result\": \"$TEST_RES\", \"msg\": \"$MSG\"}" > $RESULT_FILE
+        elif [[ "$line" == "--- FAIL"* ]]; then
+            TEST_RES="fail"
+            HAS_FAILED=true
+            IS_RUNNING=false
+        elif [[ "$line" == time=* && "$line" == *level=* ]]; then
+            LOG_LINE=$(format_logs_to_json "$line")
+            echo "$LOG_LINE" >> "$LOG_FILE"
+        fi
+        
+        if [ $HAS_FAILED == true ]; then
+            MSG_STR+="$line"
+        fi
+
+    done < "$INPUT_FILE"
+}
+
+# Main body of the script
+{
+    load_job_status
+    format_results
+}