Skip to content

Rerun failed tests #20

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

Rerun failed tests #20

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
74cea2a
option to run test based on past status
darkknight13 May 15, 2024
ba57288
fixed temp dir creation process
darkknight13 May 15, 2024
78af7dc
synced the result out, fixed name
darkknight13 May 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
/node_modules
/results
/tmp
353 changes: 87 additions & 266 deletions cql-tests-runner.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,296 +1,117 @@
#!/usr/bin/node

const { format } = require('date-fns');
const fs = require('fs');
const path = require('path');
const { format } = require('date-fns');
const loadTests = require('./loadTests');
const colors = require('colors/safe');
const currentDate = format(new Date(), 'yyyyMMddhhmm');
const axios = require('axios');
// TODO: Read server-url from environment path...

// Setup for running both $cql and Library/$evaluate
// Expand outputType to allow Parameters representation

// Test Container Structure:
/*
class Tests {
name: String
version: String // The version in which the capability being tested was introduced
description: String
reference: String // A reference to the section of the spec being tested
notes: String
group: TestGroup[]
}

class TestGroup {
name: String
version: String // The version in which the capability being tested was introduced
description: String
reference: String // A reference to the section of the spec being tested
notes: String
test: Test[]
}

class Test {
name: String
version: String // The version in which the capability being tested was introduced
description: String
reference: String // A reference to the section of the spec being tested
inputFile: String // Input data, if any
predicate: Boolean // True if this test represents a predicate
mode: String // strict | loose
ordered: Boolean // Whether the results are expected to be ordered, false if not present
checkOrderedFunctions: Boolean // Whether to ensure that attempting to use ordered functions with an unordered input should throw (e.g., using .skip() on an unordered list)
expression: String | { text: String, invalid: false, semantic, true }
output: String([]) | { text: String, type: boolean | code | date | dateTime | decimal | integer | long | quantity | string | time }([])
}
*/

class Result {
testStatus; // String: pass | fail | skip | error
responseStatus; // Integer
actual; // String
expected; // String
error; // Error
constructor(testsName, groupName, test) {
this.testsName = testsName;
this.groupName = groupName;
this.testName = test.name;

if (typeof test.expression !== 'string') {
this.invalid = test.expression.invalid;
this.expression = test.expression.text;
}
else {
this.invalid = 'false';
this.expression = test.expression;
}

if (test.output !== undefined) {
if (typeof test.output !== 'string') {
// TODO: Structure the result if it can be structured (i.e. is one of the expected types)
this.expected = test.output.text;
const CQLTest = require('./lib/CQLTest');
const CQLTests = require('./lib/CQLTests');

/**
* Loads tests from cache based on test status.
* @param {string} tmpPath - Path to the temporary directory.
* @param {Array<string>} testStatus - Array of test statuses.
* @returns {Promise<CQLTests>} - A promise resolving to CQLTests instance.
*/
async function loadTestsFromCache(tmpPath, testStatus) {
let preparedTests = new CQLTests(tmpPath);
for (const status of testStatus) {
const testListByStatusPath = path.join(tmpPath, CQLTests.CACHE_DIR, status);
if (fs.existsSync(testListByStatusPath)) {
const testList = fs.readdirSync(testListByStatusPath);
for (const testCachedFile of testList) {
const testData = fs.readFileSync(path.join(testListByStatusPath, testCachedFile));
const testInstance = CQLTest.fromJSON(JSON.parse(testData));
preparedTests.add(testInstance);
}
else {
this.expected = test.output;
}
}
else {
this.testStatus = 'skip';
}
}
return preparedTests;
}

// Iterate through tests
async function main() {
const args = process.argv.slice(2);
let apiUrl = 'https://cloud.alphora.com/sandbox/r4/cds/fhir/$cql';
let environmentPath = './environment/globals.json';
let outputPath = './results'
if (args.length > 0) {
for (const arg of args) {
let prefix = arg.slice(0, 4);
switch (prefix) {
case '-au=':
apiUrl = arg.slice(4);
break;
case '-ep=':
environmentPath = arg.slice(4);
break;
case '-op=':
outputPath = arg.slice(4);
break;
}
};
}

/**
* Loads all tests.
* @param {string} tmpPath - Path to the temporary directory.
* @param {boolean} quickTest - Whether to load only one group for testing.
* @returns {Promise<CQLTests>} - A promise resolving to CQLTests instance.
*/
async function loadAllTests(tmpPath, quickTest) {
const preparedTests = new CQLTests(tmpPath);
const tests = loadTests.load();

// Set this to true to run only the first group of tests
const quickTest = false;

let results = [];
for (const ts of tests) {
console.log('Tests: ' + ts.name);
// console.log('Tests: ' + ts.name);
for (const group of ts.group) {
console.log(' Group: ' + group.name);
let test = group.test;
// console.log(' Group: ' + group.name);
const test = group.test;
if (test != undefined) {
for (const t of test) {
console.log(' Test: ' + t.name);
results.push(new Result(ts.name, group.name, t));
// console.log(' Test: ' + t.name);
preparedTests.add(new CQLTest(ts.name, group.name, t));
}
}
if (quickTest) {
break; // Only load 1 group for testing
}
if (quickTest) break; // Only load 1 group for testing
}
if (quickTest) {
break; // Only load 1 test set for testing
}
}

for (let r of results) {
await runTest(r, apiUrl);
if (quickTest) break; // Only load 1 test set for testing
}
return preparedTests;
}

logResults(results, outputPath);
};

main();

async function runTest(result, apiUrl) {
if (result.testStatus !== 'skip') {
const data = {
"resourceType": "Parameters",
"parameter": [{
"name": "expression",
"valueString": result.expression
}]
};

try {
console.log('Running test %s:%s:%s', result.testsName, result.groupName, result.name);
const response = await axios.post(apiUrl, data, {
headers: {
'Content-Type': 'application/json',
}
});

result.responseStatus = response.status;

const responseBody = response.data;
result.actual = extractResult(responseBody);

const invalid = result.invalid;
if (invalid === 'true' || invalid === 'semantic') {
// TODO: Validate the error message is as expected...
result.testStatus = response.status === 200 ? 'fail' : 'pass';
}
else {
if (response.status === 200) {
result.testStatus = result.expected === result.actual ? 'pass' : 'fail';
}
else {
result.testStatus = 'fail';
}
}
}
catch (error) {
result.testStatus = 'error';
result.error = error;
};
}

console.log('Test %s:%s:%s status: %s expected: %s actual: %s', result.testsName, result.groupName, result.name, result.testStatus, result.expected, result.actual);
return result;
};


function extractResult(response) {
var result;
if (response.hasOwnProperty('resourceType') && response.resourceType === 'Parameters') {
for (let p of response.parameter) {
if (p.name === 'return') {
if (result === undefined) {
if (p.hasOwnProperty("valueBoolean")) {
result = p.valueBoolean.toString();
}
else if (p.hasOwnProperty("valueInteger")) {
result = p.valueInteger.toString();
}
else if (p.hasOwnProperty("valueString")) {
result = p.valueString;
}
else if (p.hasOwnProperty("valueDecimal")) {
result = p.valueDecimal;
}
else if (p.hasOwnProperty("valueDate")) {
result = p.valueDate;
}
else if (p.hasOwnProperty("valueDateTime")) {
result = p.valueDateTime;
}
else if (p.hasOwnProperty("valueTime")) {
result = p.valueTime;
}
else if (p.hasOwnProperty("valueQuantity")) {
result = p.valueQuantity.value.toString() + " '" + p.valueQuantity.code + "'";
}

// Any other type isn't handled yet...
}
else {
// Can't handle list-valued results yet...
result = undefined;
break;
const DEFAULT_API_URL = 'https://cloud.alphora.com/sandbox/r4/cds/fhir/$cql';
const DEFAULT_ENVIRONMENT_PATH = './environment/globals.json';
const DEFAULT_OUTPUT_PATH = './results';
const DEFAULT_TMP_PATH = './tmp';
/**
* Main function to execute the CQL test runner.
*/
async function main() {
let validTestStautsList = Object.values(CQLTest.STATUS);
const args = process.argv.slice(2);
let apiUrl = DEFAULT_API_URL;
let environmentPath = DEFAULT_ENVIRONMENT_PATH;
let outputPath = DEFAULT_OUTPUT_PATH;
let tmpPath = DEFAULT_TMP_PATH;
let testStatus = [];

for (const arg of args) {
let prefix = arg.slice(0, 4);
switch (prefix) {
case '-au=':
apiUrl = arg.slice(4);
break;
case '-ep=':
environmentPath = arg.slice(4);
break;
case '-op=':
outputPath = arg.slice(4);
break;
case '-status=':
let testWithStatus = arg.slice(4);
testStatus = testWithStatus.split(',').filter((value) => (
validTestStautsList.includes(value)
));
if (testStatus.length === 0) {
console.log(`Valid list of test status not passed. CQL Test Runner will run the tests!`);
}
}
}

if (result !== undefined) {
return result;
break;
}
}

// Anything that can't be structured directly, return as the actual output...
return JSON.stringify(response);
}

// Output test results
let preparedTests;

function logResult(result, outputPath) {
const fileName = `${result.testsName}_${result.groupName}_${result.testName}_${currentDate}_results.json`;
if (!fs.existsSync(outputPath)) {
fs.mkdirSync(outputPath, { recursive: true });
if (testStatus.length > 0 && fs.existsSync(tmpPath)) {
console.log(`Running following tests with status - ${testStatus}`);
preparedTests = await loadTestsFromCache(tmpPath, testStatus);
} else {
console.log(`Running all tests!!`);
const quickTest = true;
preparedTests = await loadAllTests(tmpPath, quickTest);
}
const filePath = path.join(outputPath, fileName);
fs.writeFileSync(filePath, JSON.stringify(result, null, 2), (error) => {
if (error) throw error;
});
}

function logResults(results, outputPath) {
const fileName = `${currentDate}_results.json`;
if (!fs.existsSync(outputPath)) {
fs.mkdirSync(outputPath, { recursive: true });
}
const filePath = path.join(outputPath, fileName);
const result = {
summary: summarizeResults(results),
results: results
};
fs.writeFileSync(filePath, JSON.stringify(result, null, 2), (error) => {
if (error) throw error;
});
let results = await preparedTests.run(apiUrl);
results.save(outputPath);
}

function summarizeResults(results) {
let passCount = 0;
let skipCount = 0;
let failCount = 0;
let errorCount = 0;
for (let r of results) {
if (r.testStatus === 'pass') {
passCount++;
}
else if (r.testStatus === 'skip') {
skipCount++;
}
else if (r.testStatus === 'fail') {
failCount++;
}
else if (r.testStatus === 'error') {
errorCount++;
}
}
console.log("pass: %d skip: %d fail: %d error: %d", passCount, skipCount, failCount, errorCount);

return {
pass: passCount,
skip: skipCount,
fail: failCount,
error: errorCount
}
}
main().catch(err => {
console.error('Error:', err);
});
Loading