Merge pull request #303 from stackql/feature/new-testing-interface

general-kroll-4-life · web-flow · commit 25f2497e3ee3 · 2025-04-28T08:39:17.000+10:00
new-testing-interface
diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml
@@ -15,16 +15,62 @@ on:
       - integration*
 
 env:
+  IS_TAG: ${{ github.ref_type == 'tag' }}
   GO_VERSION: '^1.22'
   STACKQL_CORE_REPOSITORY: ${{ vars.STACKQL_CORE_REPOSITORY != '' && vars.STACKQL_CORE_REPOSITORY || 'stackql/stackql' }}
   STACKQL_CORE_REF: ${{ vars.STACKQL_CORE_REF != '' && vars.STACKQL_CORE_REF || 'main' }}
   STACKQL_ANY_SDK_REPOSITORY: ${{ vars.STACKQL_ANY_SDK_REPOSITORY != '' && vars.STACKQL_ANY_SDK_REPOSITORY || 'stackql/any-sdk' }}
   STACKQL_ANY_SDK_REF: ${{ vars.STACKQL_ANY_SDK_REF != '' && vars.STACKQL_ANY_SDK_REF || 'main' }}
 
 jobs:
+
+  test_python_package_build:
+    # id: test_python_package_build
+    name: Test Python Package Build
+    runs-on: ubuntu-22.04
+    timeout-minutes: ${{ vars.DEFAULT_JOB_TIMEOUT_MIN == '' && 120 || vars.DEFAULT_JOB_TIMEOUT_MIN }}
+    steps:
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v4.1.1
+      with:
+        repository: ${{ env.STACKQL_CORE_REPOSITORY }}
+        ref: ${{ env.STACKQL_CORE_REF }}
+        token: ${{ secrets.CI_STACKQL_PACKAGE_DOWNLOAD_TOKEN  }}
+        path: stackql-core-pkg
+
+    - name: Setup Python
+      uses: actions/setup-python@v5.0.0
+      with:
+        cache: pip
+        python-version: '3.12' 
+
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+      with:
+        version: 1.8.3
+        virtualenvs-create: true
+        virtualenvs-in-project: false
+        virtualenvs-path: stackql-core-pkg/my-custom-path
+        installer-parallel: true
+
+    
+    - name: Build package
+      working-directory: stackql-core-pkg
+      run: |
+        cicd/util/01-build-robot-lib.sh
+    
+    - name: Upload python package artifact
+      uses: actions/upload-artifact@v4.3.1
+      with:
+        name: python-package-dist-folder
+        path: stackql-core-pkg/test/dist
+
   regression-testing:
     name: regression-testing
     runs-on: ubuntu-latest
+    needs: test_python_package_build
+    timeout-minutes: ${{ vars.DEFAULT_JOB_TIMEOUT_MIN == '' && 120 || vars.DEFAULT_JOB_TIMEOUT_MIN }}
     permissions:
       id-token: write
       contents: read
@@ -72,7 +118,20 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v5.0.0
         with:
-          python-version: '3.11'
+          python-version: '3.12'
+    
+      - name: Download python package dist folder
+        uses: actions/download-artifact@v4.1.2
+        with:
+          name: python-package-dist-folder
+          path: test/dist
+
+      - name: Install python testing package
+        run: |
+          echo "Inspecting python package"
+          for file in test/dist/*.whl; do
+            pip3 install "$file" --force-reinstall
+          done
 
       - name: Add dependencies
         working-directory: stackql-core
@@ -115,15 +174,15 @@ jobs:
       - name: Generate rewritten registry for simulations
         working-directory: stackql-core
         run: |
-          python3 test/python/registry-rewrite.py
+          python3 test/python/stackql_test_tooling/registry_rewrite.py --srcdir "$(pwd)/test/registry/src" --destdir "$(pwd)/test/registry-mocked/src"
 
 
       - name: Prepare load balancing materials
         working-directory: stackql-core
         run: |
           sudo cp /etc/hosts /etc/hosts.bak
-          python3 test/python/tcp_lb.py --generate-hosts-entries | sudo tee -a /etc/hosts
-          python3 test/python/tcp_lb.py --generate-nginx-lb > test/tcp/reverse-proxy/nginx/dynamic-sni-proxy.conf
+          python3 test/python/stackql_test_tooling/tcp_lb.py --generate-hosts-entries | sudo tee -a /etc/hosts
+          python3 test/python/stackql_test_tooling/tcp_lb.py --generate-nginx-lb > test/tcp/reverse-proxy/nginx/dynamic-sni-proxy.conf
 
       
       - name: Install and run nginx load balancer
@@ -152,8 +211,8 @@ jobs:
         working-directory: stackql-core
         run: |
           pgrep -f flask | xargs kill -9 || true
-          flask --app=./test/python/flask/gcp/app          run --cert=./test/server/mtls/credentials/pg_server_cert.pem --key=./test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1080 & 
-          flask --app=./test/python/flask/oauth2/token_srv run --cert=./test/server/mtls/credentials/pg_server_cert.pem --key=./test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 2091 & 
+          flask --app=./test/python/stackql_test_tooling/flask/gcp/app          run --cert=./test/server/mtls/credentials/pg_server_cert.pem --key=./test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1080 & 
+          flask --app=./test/python/stackql_test_tooling/flask/oauth2/token_srv run --cert=./test/server/mtls/credentials/pg_server_cert.pem --key=./test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 2091 & 
     
       - name: Run any-sdk cli mocked testing
         working-directory: stackql-core
@@ -181,12 +240,22 @@ jobs:
         run: |
           providerRoot="$(realpath $(pwd)/../providers)"
           sundryCfg='SUNDRY_CONFIG:{"registry_path": "'"${providerRoot}"'"}'
-          robot \
-            --variable "${sundryCfg}" \
-            --variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \
-            --include registry \
-            -d test/robot/reports \
-            test/robot/functional  || true
+          # Only tag runs non blocking on fail, hard stop on fail for branch checks
+          if [ "${{ env.IS_TAG }}" = "true" ]; then
+            robot \
+              --variable "${sundryCfg}" \
+              --variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \
+              --include registry \
+              -d test/robot/reports \
+              test/robot/functional  || true
+          else
+            robot \
+              --variable "${sundryCfg}" \
+              --variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \
+              --include registry \
+              -d test/robot/reports \
+              test/robot/functional
+          fi
       
       - name: Output from core proxied functional tests
         if: always()
@@ -211,11 +280,20 @@ jobs:
         run: |
           providerRoot="$(realpath $(pwd)/providers)"
           sundryCfg='SUNDRY_CONFIG:{"registry_path": "'"${providerRoot}"'"}'
-          robot \
-            --variable "${sundryCfg}" \
-            --variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \
-            -d test/robot/reports/mocked \
-            test/robot/stackql/mocked || true
+          # Only tag runs non blocking on fail, hard stop on fail for branch checks
+          if [ "${{ env.IS_TAG }}" = "true" ]; then
+            robot \
+              --variable "${sundryCfg}" \
+              --variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \
+              -d test/robot/reports/mocked \
+              test/robot/stackql/mocked || true
+          else
+            robot \
+              --variable "${sundryCfg}" \
+              --variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \
+              -d test/robot/reports/mocked \
+              test/robot/stackql/mocked
+          fi
       
       - name: Output from local registry mocked functional tests
         if: always()
diff --git a/.gitignore b/.gitignore
@@ -7,6 +7,7 @@ stackql*.sh
 stackql-zip
 .stackql/
 stackql-core/
+stackql-core-pkg/
 stackql-any-sdk/
 .venv/
 *.py[co]
diff --git a/docs/test-developer-guide.md b/docs/test-developer-guide.md
@@ -0,0 +1,38 @@
+
+# Test Developer Guide
+
+
+## Local Setup
+
+You will need to have these on your machine:
+
+- `python` at a version `>= 3.11`.  To see if it is already present and what version, type this in a terminal `python --version`.  Hopefully you will see something like `Python 3.13.2` appear.  You can also try `python3 --version`; although if only this latter works then you will want to set up an alias (outside scope of this doc).  If it is not present at all, then install it per [python downloads page](https://www.python.org/downloads/)
+- `go` at a version `>= 1.19`.  To see if it is already present and what version, type this in a terminal `go version`.  Hopefully you will see something like `go version go1.22.0 darwin/amd64` appear.  If it is missing or some inadequate verison, then install it per [the official go install page](https://go.dev/doc/install).
+- `openssl`, prefererable version 3 sometging, although earlier versions will probably work.  To see if it is already present and what version, type this in a terminal `openssl --version`.  Hopefully you will see something like `OpenSSL 3.4.1 11 Feb 2025 (Library: OpenSSL 3.4.1 11 Feb 2025)` appear.   If not present, then, on Mac, use [homebrew](https://brew.sh/) to install it with `brew install openssl`.
+
+In addition, you will need shell scripts with exported credentials in the files (relative to repository root):
+
+- `scripts/sec/sec-ro-stackql.sh`.
+- `scripts/sec/sec-rw-stackql.sh`.
+
+Then, once all this is in place, test setup can be done as a "once-off" (run again when you want to update dependencies) with:
+
+```bash
+
+scripts/local/ci/01-gather.sh
+
+scripts/local/ci/02-setup.sh
+
+```
+
+All sorts of stuff will happen here and may take a little while.
+
+Then, to run readonly tests locally: `scripts/local/ci/03-run-live-readonly.sh`.
+
+To run readwrite tests locally (more dangerous): `scripts/local/ci/04-run-live-readwrite.sh`.
+
+
+You can now develop new test cases.  Once they are working locally, you can test them remotely (once you are in the appropriate `github` group) by pushing a `git` tag that begins
+with `robot`, `regression`, or `integration`.
+
+
diff --git a/scripts/local/ci/02-setup.sh b/scripts/local/ci/02-setup.sh
@@ -15,7 +15,7 @@ if [ ! -d "${STACKQL_CORE_DIR}/.venv" ]; then
   >&2 echo "Virtual environment created."
 fi
 
-source "${REPOSITORY_ROOT_DIR}/.venv/bin/activate"
+source "${STACKQL_CORE_DIR}/.venv/bin/activate"
 
 pip install -r "${STACKQL_CORE_DIR}/cicd/requirements.txt"
 
@@ -25,7 +25,7 @@ cd "${STACKQL_CORE_DIR}"
 
 python cicd/python/build.py --build
 
-python test/python/registry-rewrite.py
+python test/python/stackql_test_tooling/registry_rewrite.py --srcdir "$(pwd)/test/registry/src" --destdir "$(pwd)/test/registry-mocked/src"
 
 openssl req -x509 -keyout test/server/mtls/credentials/pg_server_key.pem -out test/server/mtls/credentials/pg_server_cert.pem -config test/server/mtls/openssl.cnf -days 365
 openssl req -x509 -keyout test/server/mtls/credentials/pg_client_key.pem -out test/server/mtls/credentials/pg_client_cert.pem -config test/server/mtls/openssl.cnf -days 365
diff --git a/scripts/local/ci/03-run-live-readonly.sh b/scripts/local/ci/03-run-live-readonly.sh
@@ -6,14 +6,16 @@ SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
 REPOSITORY_ROOT_DIR="$(realpath ${SCRIPT_DIR}/../../..)"
 
-_SEC_FILE="${REPOSITORY_ROOT_DIR}/scripts/sec/sec-readwrite.sh"
+STACKQL_CORE_DIR="${REPOSITORY_ROOT_DIR}/stackql-core"
+
+_SEC_FILE="${REPOSITORY_ROOT_DIR}/scripts/sec/sec-rw-stackql.sh"
 
 if [ -f "${_SEC_FILE}" ]; then
   source "${_SEC_FILE}"
 fi
 
-if [ -f "scripts/sec/aws-ro-stackql.sh" ]; then
-  source "scripts/sec/aws-ro-stackql.sh"
+if [ -f "scripts/sec/sec-ro-stackql.sh" ]; then
+  source "scripts/sec/sec-ro-stackql.sh"
 fi
 
 if [ "${AWS_ACCESS_KEY_ID}" = "" ]; then
@@ -33,7 +35,9 @@ fi
 
 cd "${REPOSITORY_ROOT_DIR}"
 
-source "${REPOSITORY_ROOT_DIR}/.venv/bin/activate"
+source "${STACKQL_CORE_DIR}/.venv/bin/activate"
+
+export PYTHONPATH="${PYTHONPATH}:${STACKQL_CORE_DIR}/test/python"
 
 robot -d test/robot/reports/readonly test/robot/stackql/live/readonly
 
diff --git a/scripts/local/ci/04-run-live-readwrite.sh b/scripts/local/ci/04-run-live-readwrite.sh
@@ -6,7 +6,9 @@ SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
 REPOSITORY_ROOT_DIR="$(realpath ${SCRIPT_DIR}/../../..)"
 
-_SEC_FILE="${REPOSITORY_ROOT_DIR}/scripts/sec/sec-readwrite.sh"
+STACKQL_CORE_DIR="${REPOSITORY_ROOT_DIR}/stackql-core"
+
+_SEC_FILE="${REPOSITORY_ROOT_DIR}/scripts/sec/sec-rw-stackql.sh"
 
 if [ -f "${_SEC_FILE}" ]; then
   source "${_SEC_FILE}"
@@ -19,7 +21,9 @@ fi
 
 cd "${REPOSITORY_ROOT_DIR}"
 
-source "${REPOSITORY_ROOT_DIR}/.venv/bin/activate"
+source "${STACKQL_CORE_DIR}/.venv/bin/activate"
+
+export PYTHONPATH="${PYTHONPATH}:${STACKQL_CORE_DIR}/test/python"
 
 robot -d test/robot/reports/readwrite test/robot/stackql/live/readwrite
 
diff --git a/scripts/local/ci/99-clean.sh b/scripts/local/ci/99-clean.sh
@@ -0,0 +1,13 @@
+#! /usr/bin/env bash
+
+>&2 echo "cleaning dependent repositories from local file system"
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+REPOSITORY_ROOT_DIR="$(realpath ${SCRIPT_DIR}/../../..)"
+
+cd "${REPOSITORY_ROOT_DIR}"
+
+rm -rf stackql-core    || true
+rm -rf stackql-any-sdk || true
+
diff --git a/test/.gitignore b/test/.gitignore
@@ -0,0 +1 @@
+dist/
diff --git a/test/robot/stackql/live/readonly/live_readonly.robot b/test/robot/stackql/live/readonly/live_readonly.robot
@@ -18,6 +18,21 @@ Simple Google Buckets List With Date Logic Contains Exemplifies Use of SQLite Ma
     ...    ${EMPTY}
     ...    Google-Buckets-List-With-Date-Logic-Contains-Exemplifies-Use-of-SQLite-Math-Functions
 
+Simple Google IAM Service Accounts List
+    Pass Execution If    "${SQL_BACKEND}" == "postgres_tcp"    This is a valid case where the test is targetted at SQLite only
+    [Tags]   google   iam    service_accounts   gooogle.iam    google.iam.service_accounts    tier_1
+    ${inputStr}=    Catenate
+    ...    select email 
+    ...    from google.iam.service_accounts 
+    ...    where projectsId = 'stackql-robot' 
+    ...    order by email desc
+    ...    ;
+    Stock Stackql Exec Inline Contains Both Streams
+    ...    ${inputStr}
+    ...    stackql\-robot\-rw\-sa@stackql\-robot.iam.gserviceaccount.com
+    ...    ${EMPTY}
+    ...    Google-Buckets-List-With-Date-Logic-Contains-Exemplifies-Use-of-SQLite-Math-Functions
+
 AWS Route53 List Record Sets Simple
     [Documentation]    It is fine for this to dump 404 infor to stderr. So long as the empty reusult is represented with a header row, all good.
     [Tags]   aws   route53    resource_record_sets   aws.route53    aws.route53.resource_record_sets    tier_1
diff --git a/test/robot/stackql/live/readonly/stackql.resource b/test/robot/stackql/live/readonly/stackql.resource
@@ -1,7 +1,7 @@
 *** Variables ***
 ${REPOSITORY_ROOT}                     ${CURDIR}${/}..${/}..${/}..${/}..${/}..
 ${CORE_REPOSITORY_ROOT}                ${REPOSITORY_ROOT}${/}stackql-core
-${CORE_LIB_HOME}                       ${CORE_REPOSITORY_ROOT}${/}test${/}robot${/}lib
+${CORE_LIB_HOME}                       ${CORE_REPOSITORY_ROOT}${/}test${/}python${/}stackql_test_tooling
 ${EXECUTION_PLATFORM}                  native   # to be overridden from command line, eg "docker"
 ${SQL_BACKEND}                         sqlite_embedded   # to be overridden from command line, eg "postgres_tcp"
 ${IS_WSL}                              false   # to be overridden from command line, with string "true"
@@ -17,19 +17,24 @@ ${SUNDRY_CONFIG}=  SEPARATOR=
 ...        "AWS_RECORD_SET_REGION": "us-east-1",
 ...        "registry_path": "${CURDIR}${/}..${/}..${/}..${/}..${/}..${/}providers"
 ...    }
+${CWD_FOR_EXEC}                        ${CORE_REPOSITORY_ROOT}  # works for self repository, can be overwritten when shared
+${WEB_SERVICE_LIBRARY}                 stackql_test_tooling.web_service_keywords
+${STACKQL_INTERFACE_LIBRARY}           stackql_test_tooling.StackQLInterfaces
+${CLOUD_INTEGRATION_LIBRARY}           stackql_test_tooling.CloudIntegration
 
 *** Settings ***
 Library           Process
 Library           OperatingSystem
 # Variable first defined clobbers later defined therefore most specific variable file first.
 Variables         ${CURDIR}${/}readonly_variables.py    ${SUNDRY_CONFIG}
-Variables         ${CORE_LIB_HOME}/stackql_context.py    ${EXECUTION_PLATFORM}    ${SQL_BACKEND}    ${USE_STACKQL_PREINSTALLED}
+Variables         ${CORE_LIB_HOME}${/}stackql_context.py    ${CORE_REPOSITORY_ROOT}    ${EXECUTION_PLATFORM}    ${SQL_BACKEND}    ${USE_STACKQL_PREINSTALLED}
 ...               ${SUNDRY_CONFIG}
 Library           Process
 Library           OperatingSystem
 Library           String
-Library           ${CORE_LIB_HOME}/StackQLInterfaces.py    ${EXECUTION_PLATFORM}    ${SQL_BACKEND}    ${CONCURRENCY_LIMIT}
-Library           ${CORE_LIB_HOME}/CloudIntegration.py
+Library           ${STACKQL_INTERFACE_LIBRARY}    ${EXECUTION_PLATFORM}    ${SQL_BACKEND}    ${CONCURRENCY_LIMIT}
+Library           ${CLOUD_INTEGRATION_LIBRARY}
+Library           ${WEB_SERVICE_LIBRARY}          ${CWD_FOR_EXEC}
 
 *** Keywords ***
 
diff --git a/test/robot/stackql/live/readwrite/stackql.resource b/test/robot/stackql/live/readwrite/stackql.resource
@@ -1,5 +1,7 @@
 *** Variables ***
-${CORE_LIB_HOME}                       ${CURDIR}${/}..${/}..${/}..${/}..${/}..${/}stackql-core${/}test${/}robot${/}lib
+${REPOSITORY_ROOT}                     ${CURDIR}${/}..${/}..${/}..${/}..${/}..
+${CORE_REPOSITORY_ROOT}                ${REPOSITORY_ROOT}${/}stackql-core
+${CORE_LIB_HOME}                       ${CORE_REPOSITORY_ROOT}${/}test${/}python${/}stackql_test_tooling
 ${EXECUTION_PLATFORM}                  native   # to be overridden from command line, eg "docker"
 ${SQL_BACKEND}                         sqlite_embedded   # to be overridden from command line, eg "postgres_tcp"
 ${IS_WSL}                              false   # to be overridden from command line, with string "true"
@@ -15,18 +17,23 @@ ${SUNDRY_CONFIG}=  SEPARATOR=
 ...        "AWS_RECORD_SET_REGION": "us-east-1",
 ...        "registry_path": "${CURDIR}${/}..${/}..${/}..${/}..${/}..${/}providers"
 ...    }
+${CWD_FOR_EXEC}                        ${CORE_REPOSITORY_ROOT}  # works for self repository, can be overwritten when shared
+${WEB_SERVICE_LIBRARY}                 stackql_test_tooling.web_service_keywords
+${STACKQL_INTERFACE_LIBRARY}           stackql_test_tooling.StackQLInterfaces
+${CLOUD_INTEGRATION_LIBRARY}           stackql_test_tooling.CloudIntegration
 
 *** Settings ***
 Library           Process
 Library           OperatingSystem
 Variables         ${CURDIR}${/}readwrite_variables.py    ${SUNDRY_CONFIG}
-Variables         ${CORE_LIB_HOME}${/}stackql_context.py    ${EXECUTION_PLATFORM}    ${SQL_BACKEND}    ${USE_STACKQL_PREINSTALLED}
+Variables         ${CORE_LIB_HOME}${/}stackql_context.py    ${CORE_REPOSITORY_ROOT}    ${EXECUTION_PLATFORM}    ${SQL_BACKEND}    ${USE_STACKQL_PREINSTALLED}
 ...               ${SUNDRY_CONFIG}
 Library           Process
 Library           OperatingSystem
 Library           String
-Library           ${CORE_LIB_HOME}/StackQLInterfaces.py    ${EXECUTION_PLATFORM}    ${SQL_BACKEND}    ${CONCURRENCY_LIMIT}
-Library           ${CORE_LIB_HOME}/CloudIntegration.py
+Library           ${STACKQL_INTERFACE_LIBRARY}    ${EXECUTION_PLATFORM}    ${SQL_BACKEND}    ${CONCURRENCY_LIMIT}
+Library           ${CLOUD_INTEGRATION_LIBRARY}
+Library           ${WEB_SERVICE_LIBRARY}          ${CWD_FOR_EXEC}
 
 *** Keywords ***
 
diff --git a/test/robot/stackql/mocked/stackql.resource b/test/robot/stackql/mocked/stackql.resource
