diff --git a/.gitignore b/.gitignore index c334b79..bb8cef5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /node_modules /results +/tmp \ No newline at end of file diff --git a/cql-tests-runner.js b/cql-tests-runner.js index 14c5826..ea15c00 100644 --- a/cql-tests-runner.js +++ b/cql-tests-runner.js @@ -1,296 +1,117 @@ #!/usr/bin/node +const { format } = require('date-fns'); const fs = require('fs'); const path = require('path'); -const { format } = require('date-fns'); const loadTests = require('./loadTests'); -const colors = require('colors/safe'); -const currentDate = format(new Date(), 'yyyyMMddhhmm'); -const axios = require('axios'); -// TODO: Read server-url from environment path... - -// Setup for running both $cql and Library/$evaluate -// Expand outputType to allow Parameters representation - -// Test Container Structure: -/* -class Tests { - name: String - version: String // The version in which the capability being tested was introduced - description: String - reference: String // A reference to the section of the spec being tested - notes: String - group: TestGroup[] -} - -class TestGroup { - name: String - version: String // The version in which the capability being tested was introduced - description: String - reference: String // A reference to the section of the spec being tested - notes: String - test: Test[] -} - -class Test { - name: String - version: String // The version in which the capability being tested was introduced - description: String - reference: String // A reference to the section of the spec being tested - inputFile: String // Input data, if any - predicate: Boolean // True if this test represents a predicate - mode: String // strict | loose - ordered: Boolean // Whether the results are expected to be ordered, false if not present - checkOrderedFunctions: Boolean // Whether to ensure that attempting to use ordered functions with an unordered input should throw (e.g., using .skip() on an unordered list) - expression: String | { text: String, invalid: false, semantic, true } - output: String([]) | { text: String, type: boolean | code | date | dateTime | decimal | integer | long | quantity | string | time }([]) -} -*/ - -class Result { - testStatus; // String: pass | fail | skip | error - responseStatus; // Integer - actual; // String - expected; // String - error; // Error - constructor(testsName, groupName, test) { - this.testsName = testsName; - this.groupName = groupName; - this.testName = test.name; - - if (typeof test.expression !== 'string') { - this.invalid = test.expression.invalid; - this.expression = test.expression.text; - } - else { - this.invalid = 'false'; - this.expression = test.expression; - } - - if (test.output !== undefined) { - if (typeof test.output !== 'string') { - // TODO: Structure the result if it can be structured (i.e. is one of the expected types) - this.expected = test.output.text; +const CQLTest = require('./lib/CQLTest'); +const CQLTests = require('./lib/CQLTests'); + +/** + * Loads tests from cache based on test status. + * @param {string} tmpPath - Path to the temporary directory. + * @param {Array} testStatus - Array of test statuses. + * @returns {Promise} - A promise resolving to CQLTests instance. + */ +async function loadTestsFromCache(tmpPath, testStatus) { + let preparedTests = new CQLTests(tmpPath); + for (const status of testStatus) { + const testListByStatusPath = path.join(tmpPath, CQLTests.CACHE_DIR, status); + if (fs.existsSync(testListByStatusPath)) { + const testList = fs.readdirSync(testListByStatusPath); + for (const testCachedFile of testList) { + const testData = fs.readFileSync(path.join(testListByStatusPath, testCachedFile)); + const testInstance = CQLTest.fromJSON(JSON.parse(testData)); + preparedTests.add(testInstance); } - else { - this.expected = test.output; - } - } - else { - this.testStatus = 'skip'; } } + return preparedTests; } -// Iterate through tests -async function main() { - const args = process.argv.slice(2); - let apiUrl = 'https://cloud.alphora.com/sandbox/r4/cds/fhir/$cql'; - let environmentPath = './environment/globals.json'; - let outputPath = './results' - if (args.length > 0) { - for (const arg of args) { - let prefix = arg.slice(0, 4); - switch (prefix) { - case '-au=': - apiUrl = arg.slice(4); - break; - case '-ep=': - environmentPath = arg.slice(4); - break; - case '-op=': - outputPath = arg.slice(4); - break; - } - }; - } - +/** + * Loads all tests. + * @param {string} tmpPath - Path to the temporary directory. + * @param {boolean} quickTest - Whether to load only one group for testing. + * @returns {Promise} - A promise resolving to CQLTests instance. + */ +async function loadAllTests(tmpPath, quickTest) { + const preparedTests = new CQLTests(tmpPath); const tests = loadTests.load(); - - // Set this to true to run only the first group of tests - const quickTest = false; - - let results = []; for (const ts of tests) { - console.log('Tests: ' + ts.name); + // console.log('Tests: ' + ts.name); for (const group of ts.group) { - console.log(' Group: ' + group.name); - let test = group.test; + // console.log(' Group: ' + group.name); + const test = group.test; if (test != undefined) { for (const t of test) { - console.log(' Test: ' + t.name); - results.push(new Result(ts.name, group.name, t)); + // console.log(' Test: ' + t.name); + preparedTests.add(new CQLTest(ts.name, group.name, t)); } } - if (quickTest) { - break; // Only load 1 group for testing - } + if (quickTest) break; // Only load 1 group for testing } - if (quickTest) { - break; // Only load 1 test set for testing - } - } - - for (let r of results) { - await runTest(r, apiUrl); + if (quickTest) break; // Only load 1 test set for testing } + return preparedTests; +} - logResults(results, outputPath); -}; - -main(); - -async function runTest(result, apiUrl) { - if (result.testStatus !== 'skip') { - const data = { - "resourceType": "Parameters", - "parameter": [{ - "name": "expression", - "valueString": result.expression - }] - }; - - try { - console.log('Running test %s:%s:%s', result.testsName, result.groupName, result.name); - const response = await axios.post(apiUrl, data, { - headers: { - 'Content-Type': 'application/json', - } - }); - - result.responseStatus = response.status; - - const responseBody = response.data; - result.actual = extractResult(responseBody); - - const invalid = result.invalid; - if (invalid === 'true' || invalid === 'semantic') { - // TODO: Validate the error message is as expected... - result.testStatus = response.status === 200 ? 'fail' : 'pass'; - } - else { - if (response.status === 200) { - result.testStatus = result.expected === result.actual ? 'pass' : 'fail'; - } - else { - result.testStatus = 'fail'; - } - } - } - catch (error) { - result.testStatus = 'error'; - result.error = error; - }; - } - - console.log('Test %s:%s:%s status: %s expected: %s actual: %s', result.testsName, result.groupName, result.name, result.testStatus, result.expected, result.actual); - return result; -}; - - -function extractResult(response) { - var result; - if (response.hasOwnProperty('resourceType') && response.resourceType === 'Parameters') { - for (let p of response.parameter) { - if (p.name === 'return') { - if (result === undefined) { - if (p.hasOwnProperty("valueBoolean")) { - result = p.valueBoolean.toString(); - } - else if (p.hasOwnProperty("valueInteger")) { - result = p.valueInteger.toString(); - } - else if (p.hasOwnProperty("valueString")) { - result = p.valueString; - } - else if (p.hasOwnProperty("valueDecimal")) { - result = p.valueDecimal; - } - else if (p.hasOwnProperty("valueDate")) { - result = p.valueDate; - } - else if (p.hasOwnProperty("valueDateTime")) { - result = p.valueDateTime; - } - else if (p.hasOwnProperty("valueTime")) { - result = p.valueTime; - } - else if (p.hasOwnProperty("valueQuantity")) { - result = p.valueQuantity.value.toString() + " '" + p.valueQuantity.code + "'"; - } - // Any other type isn't handled yet... - } - else { - // Can't handle list-valued results yet... - result = undefined; - break; +const DEFAULT_API_URL = 'https://cloud.alphora.com/sandbox/r4/cds/fhir/$cql'; +const DEFAULT_ENVIRONMENT_PATH = './environment/globals.json'; +const DEFAULT_OUTPUT_PATH = './results'; +const DEFAULT_TMP_PATH = './tmp'; +/** + * Main function to execute the CQL test runner. + */ +async function main() { + let validTestStautsList = Object.values(CQLTest.STATUS); + const args = process.argv.slice(2); + let apiUrl = DEFAULT_API_URL; + let environmentPath = DEFAULT_ENVIRONMENT_PATH; + let outputPath = DEFAULT_OUTPUT_PATH; + let tmpPath = DEFAULT_TMP_PATH; + let testStatus = []; + + for (const arg of args) { + let prefix = arg.slice(0, 4); + switch (prefix) { + case '-au=': + apiUrl = arg.slice(4); + break; + case '-ep=': + environmentPath = arg.slice(4); + break; + case '-op=': + outputPath = arg.slice(4); + break; + case '-status=': + let testWithStatus = arg.slice(4); + testStatus = testWithStatus.split(',').filter((value) => ( + validTestStautsList.includes(value) + )); + if (testStatus.length === 0) { + console.log(`Valid list of test status not passed. CQL Test Runner will run the tests!`); } - } - } - - if (result !== undefined) { - return result; + break; } } - // Anything that can't be structured directly, return as the actual output... - return JSON.stringify(response); -} - -// Output test results + let preparedTests; -function logResult(result, outputPath) { - const fileName = `${result.testsName}_${result.groupName}_${result.testName}_${currentDate}_results.json`; - if (!fs.existsSync(outputPath)) { - fs.mkdirSync(outputPath, { recursive: true }); + if (testStatus.length > 0 && fs.existsSync(tmpPath)) { + console.log(`Running following tests with status - ${testStatus}`); + preparedTests = await loadTestsFromCache(tmpPath, testStatus); + } else { + console.log(`Running all tests!!`); + const quickTest = true; + preparedTests = await loadAllTests(tmpPath, quickTest); } - const filePath = path.join(outputPath, fileName); - fs.writeFileSync(filePath, JSON.stringify(result, null, 2), (error) => { - if (error) throw error; - }); -} -function logResults(results, outputPath) { - const fileName = `${currentDate}_results.json`; - if (!fs.existsSync(outputPath)) { - fs.mkdirSync(outputPath, { recursive: true }); - } - const filePath = path.join(outputPath, fileName); - const result = { - summary: summarizeResults(results), - results: results - }; - fs.writeFileSync(filePath, JSON.stringify(result, null, 2), (error) => { - if (error) throw error; - }); + let results = await preparedTests.run(apiUrl); + results.save(outputPath); } -function summarizeResults(results) { - let passCount = 0; - let skipCount = 0; - let failCount = 0; - let errorCount = 0; - for (let r of results) { - if (r.testStatus === 'pass') { - passCount++; - } - else if (r.testStatus === 'skip') { - skipCount++; - } - else if (r.testStatus === 'fail') { - failCount++; - } - else if (r.testStatus === 'error') { - errorCount++; - } - } - console.log("pass: %d skip: %d fail: %d error: %d", passCount, skipCount, failCount, errorCount); - - return { - pass: passCount, - skip: skipCount, - fail: failCount, - error: errorCount - } -} \ No newline at end of file +main().catch(err => { + console.error('Error:', err); +}); diff --git a/lib/CQLResults.js b/lib/CQLResults.js new file mode 100644 index 0000000..f7c0f24 --- /dev/null +++ b/lib/CQLResults.js @@ -0,0 +1,101 @@ +const fs = require('fs'); +const path = require('path'); +const CQLTestResult = require('./CQLTestResult'); +const { format } = require('date-fns'); +const currentDate = format(new Date(), 'yyyyMMddhhmm'); + +/** + * Represents the results of running CQL tests. + */ +class CQLResults { + /** + * Initializes CQLResults object with counts and results array. + */ + constructor() { + /** + * @type {Object} counts - Object containing counts of test statuses. + * @property {number} pass - Number of passed tests. + * @property {number} skip - Number of skipped tests. + * @property {number} fail - Number of failed tests. + * @property {number} error - Number of tests with errors. + */ + this.counts = { + pass: 0, + skip: 0, + fail: 0, + error: 0 + }; + /** + * @type {Array} results - Array containing CQLTestResult objects. + */ + this.results = []; + } + + /** + * Adds a test result to the counts and results array. + * @param {CQLTestResult} result - The test result to add. + */ + add(result) { + if (result instanceof CQLTestResult) { + this.counts[result.testStatus]++; + this.results.push(result); + } + } + + /** + * Displays the summary of test counts. + * @returns {Object} The counts object. + */ + summaryCount() { + console.log(`pass: ${this.counts.pass} skip: ${this.counts.skip} fail: ${this.counts.fail} error: ${this.counts.error}`); + return this.counts; + } + + /** + * Summarizes the test results and updates counts accordingly. + * @returns {Object} The counts object. + */ + summarize() { + this.counts = { + pass: 0, + skip: 0, + fail: 0, + error: 0 + }; + + for (const result of this.results) { + this.counts[result.testStatus]++; + } + + return this.summaryCount(); + } + + /** + * Converts CQLResults object to JSON format. + * @returns {Object} JSON representation of CQLResults. + */ + toJSON() { + return { + summary: this.counts, + results: this.results + }; + } + + /** + * Saves the CQLResults object to a JSON file. + * @param {string} outputPath - The directory path where the file will be saved. + * @returns {string} The path of the saved file. + */ + save(outputPath) { + const fileName = `${currentDate}_results.json`; // File name based on current date + if (!fs.existsSync(outputPath)) { + fs.mkdirSync(outputPath, { recursive: true }); // Create directory if not exists + } + const filePath = path.join(outputPath, fileName); // Construct full file path + fs.writeFileSync(filePath, JSON.stringify(this, null, 2)); // Write JSON to file + console.log(`Results saved to: ${filePath}`); // Log file path + return filePath; // Return file path + } +} + +module.exports = CQLResults; \ No newline at end of file diff --git a/lib/CQLTest.js b/lib/CQLTest.js new file mode 100644 index 0000000..2362499 --- /dev/null +++ b/lib/CQLTest.js @@ -0,0 +1,207 @@ +const fs = require('fs'); +const path = require('path'); +const axios = require('axios'); +const CQLTestResult = require('./CQLTestResult'); +const crypto = require('crypto'); + +/** + * Represents a CQL Test. + */ +class CQLTest { + /** + * Enum representing possible test statuses. + * @enum {string} + */ + static STATUS = { + 'PASS': 'pass', + 'FAIL': 'fail', + 'SKIP': 'skip', + 'ERROR': 'error', + } + + /** + * Creates an instance of CQLTest. + * @param {string} tests - The type of test. + * @param {string} group - The group of test. + * @param {object} test - The test details. + */ + constructor(tests, group, test) { + /** + * Configuration object for the test. + * @type {object} + */ + this.config = {}; + if (tests && group && test) { + this.config = { + tests, + group, + test, + options: {} + } + this._prepareTest(); + } + } + + /** + * Loads configuration for the test. + * @param {object} config - The configuration object. + */ + loadConfig(config) { + let { checksum, ...restConfig } = config; + if (checksum) { + let calculatedChecksum = CQLTest.generateCheckSum(restConfig); + if (calculatedChecksum === checksum) { + this.config = config; + this.prepared = true; + this.cacheFile = `${this.id}.json`; + } else { + throw new Error(`Loading configuration failed due to a mismatched checksum.`); + } + } else { + throw new Error(`Loading configuration failed due to a missing checksum.`); + } + return this; + } + + get id(){ + return this.config?.id ?? null; + } + + /** + * Prepares the test configuration. + * @private + */ + _prepareTest() { + this.config['name'] = this.config.test.name; + this.config.id = `${this.config.tests}_${this.config.group}_${this.config.name}`; + if (typeof this.config.test?.expression !== 'string') { + this.config['invalid'] = this.config.test.expression.invalid; + this.config['expression'] = this.config.test.expression.text; + } + else { + this.config['invalid'] = 'false'; + this.config['expression'] = this.config.test.expression; + } + + if (this.config.test.output !== undefined) { + if (typeof this.config.test.output !== 'string') { + // @todo: Structure the result if it can be structured (i.e. is one of the expected types) + this.config['expected'] = this.config.test.output.text; + } + else { + this.config['expected'] = this.config.test.output; + } + } + else { + this.status = CQLTest.STATUS.SKIP; + } + this.config['checksum'] = CQLTest.generateCheckSum(this.config); + this.cacheFile = `${this.id}.json`; + this.prepared = true; + } + + /** + * Saves the test configuration to a file. + * @param {string} directory - The directory to save the file. + * @returns {string|null} - The file save path, or null if directory is not provided. + */ + save(directory, force = false) { + let fileSavePath = null; + if (directory && this.id) { + if (!fs.existsSync(directory)) { + fs.mkdirSync(directory, { recursive: true }); + } + fileSavePath = path.join(directory, this.cacheFile); + if (!fs.existsSync(fileSavePath) || force) { + fs.writeFileSync(fileSavePath, JSON.stringify(this,0,4)); + } + } + return fileSavePath; + } + + /** + * Runs the test. + * @param {string} apiUrl - The API URL to run the test against. + * @returns {Promise} - The result of the test. + */ + async run(apiUrl) { + if (this.prepared) { + let result = new CQLTestResult(this?.status ?? CQLTest.STATUS.FAIL, null); + result.testsName = this.config.tests; + result.testName = this.config.name; + result.groupName = this.config.group; + result.expected = this.config?.expected; + result.invalid = this.config.invalid; + + try { + console.log('Running test %s:%s:%s', this.config.tests, this.config.group, this.config.name); + + const data = { + "resourceType": "Parameters", + "parameter": [{ + "name": "expression", + "valueString": this.config.expression + }] + }; + + const response = await axios.post(apiUrl, data, { + headers: { + 'Content-Type': 'application/json', + } + }); + + result.responseStatus = response.status; + + const responseBody = response.data; + result.actual = CQLTestResult.compute(responseBody); + + if (response.status === 200) { + if (['true', 'semantic'].includes(this.config.invalid)) { + result.testStatus = CQLTest.STATUS.PASS; + } else if (this.config?.expected === result.actual) { + result.testStatus = CQLTest.STATUS.PASS; + } + } + } + catch (error) { + result.testStatus = CQLTest.STATUS.ERROR; + result.error = error; + } + finally { + console.log('Test %s:%s:%s status: %s expected: %s actual: %s', this.config.tests, this.config.group, this.config.name, result.testStatus, this.config.expected, result.actual); + } + return result; + } else { + throw new Error(`A valid test configuration was not found. Unable to execute the test.`) + } + + } + + /** + * Returns the JSON representation of the test. + * @returns {object} - The JSON representation of the test. + */ + toJSON() { + return this.config; + } + + /** + * Generates a checksum for the given configuration. + * @param {object} config - The configuration object. + * @returns {string} - The checksum. + */ + static generateCheckSum(config) { + return crypto.createHash('sha256').update(JSON.stringify(config)).digest('hex').toString(); + } + + /** + * Creates a CQLTest instance from a JSON configuration. + * @param {object} config - The JSON configuration. + * @returns {CQLTest} - The CQLTest instance. + */ + static fromJSON(config) { + return new CQLTest().loadConfig(config); + } +} + +module.exports = CQLTest; diff --git a/lib/CQLTestResult.js b/lib/CQLTestResult.js new file mode 100644 index 0000000..34641f3 --- /dev/null +++ b/lib/CQLTestResult.js @@ -0,0 +1,48 @@ +class CQLTestResult { + testStatus; // String: pass | fail | skip | error + responseStatus; // Integer + actual; // String + expected; // String + error; // Error + testsName; + groupName; + testName; + invalid; + expression; + test; + + constructor(testStatus){ + if(testStatus){ + this.testStatus = testStatus; + } + } + + static compute(response) { + if (response?.resourceType !== 'Parameters') { + return JSON.stringify(response); // Return JSON string representation if resourceType is not 'Parameters' + } + + const returnValue = response.parameter.find(p => p.name === 'return'); + + if (!returnValue) { + return undefined; // No 'return' parameter found + } + + const valueTypes = ['valueBoolean', 'valueInteger', 'valueString', 'valueDecimal', 'valueDate', 'valueDateTime', 'valueTime', 'valueQuantity']; + for (const valueType of valueTypes) { + if (typeof returnValue?.[valueType] === 'boolean' || returnValue?.[valueType]) { + if (valueType === 'valueQuantity') { + const { value, code } = returnValue[valueType]; + return `${value.toString()} '${code}'`; + } else { + return returnValue[valueType].toString(); // Convert value to string + } + } + } + + // If none of the expected value types are found, return undefined + return undefined; + } +} + +module.exports = CQLTestResult; \ No newline at end of file diff --git a/lib/CQLTests.js b/lib/CQLTests.js new file mode 100644 index 0000000..274880f --- /dev/null +++ b/lib/CQLTests.js @@ -0,0 +1,123 @@ +const fs = require('fs'); +const path = require('path'); +const CQLTest = require('./CQLTest'); +const CQLResults = require('./CQLResults'); + +/** + * Represents a collection of CQL tests. + * @class + */ +class CQLTests { + /** + * Directory name for caching tests. + * @type {string} + * @static + */ + static CACHE_DIR = 'tests'; + + /** + * Creates an instance of CQLTests. + * @param {string} [savePath=null] - The path to save test results. + */ + constructor(savePath = null) { + /** + * Path where test results are saved. + * @type {string} + * @private + */ + this.savePath = null; + this._prepareSavePath(savePath); + /** + * Array to store added tests. + * @type {CQLTest[]} + */ + this.tests = []; + } + + /** + * Prepares the save path for test results. + * @private + * @param {string} savePath - The path to save test results. + */ + _prepareSavePath(savePath) { + console.log(savePath); + if (savePath) { + if(!fs.existsSync(savePath)){ + fs.mkdirSync(savePath, { recursive: true }); + } + if(fs.existsSync(savePath)){ + const testSavePath = path.join(savePath, CQLTests.CACHE_DIR); + for (const status of Object.values(CQLTest.STATUS)) { + const statusPath = path.join(testSavePath, status); + fs.mkdirSync(statusPath, { recursive: true }); + } + this.savePath = testSavePath; + } + } + } + + /** + * Gets the save path based on test status. + * @private + * @param {string} status - The status of the test. + * @returns {string} The save path. + */ + _getSavePathOnStatus(status) { + if (Object.values(CQLTest.STATUS).includes(status)) { + return path.join(this.savePath, status); + } + } + + /** + * Caches the test result. + * @private + * @param {CQLTest} test - The test to cache. + * @param {string} testStatus - The status of the test. + * @param {boolean} force - Flag indicating whether to force caching. + */ + _cacheTest(test, testStatus, force) { + console.log(testStatus) + const savePath = this._getSavePathOnStatus(testStatus); + if (fs.existsSync(savePath)) { + test.save(savePath, force); + } + for (const status of Object.values(CQLTest.STATUS)) { + if (status !== testStatus) { + const otherPath = this._getSavePathOnStatus(status); + const testFilePath = path.join(otherPath, test.cacheFile); + if (fs.existsSync(testFilePath)) { + fs.unlinkSync(testFilePath); + } + } + } + } + + /** + * Adds a test to the collection. + * @param {CQLTest} test - The test to add. + */ + add(test) { + if (test instanceof CQLTest) { + this.tests.push(test); + } + } + + /** + * Runs all tests. + * @param {string} apiUrl - The API URL for tests. + * @param {boolean} [force=false] - Flag indicating whether to force test cache. + * @returns {Promise} The results of all tests. + */ + async run(apiUrl, force = false) { + const results = new CQLResults(); + for (const test of this.tests) { + const result = await test.run(apiUrl); + this._cacheTest(test, result?.testStatus, force); + results.add(result); + } + results.summaryCount(); + return results; + } +} + +module.exports = CQLTests; diff --git a/loadTests.js b/loadTests.js index c9628b0..793c7cc 100644 --- a/loadTests.js +++ b/loadTests.js @@ -9,29 +9,31 @@ const alwaysArray = [ "tests.group", "tests.group.test" ]; - + const options = { - ignoreAttributes : false, - attributeNamePrefix : '', - parseTagValue : false, - isArray: (name, jpath, isLeafNode, isAttribute) => { + ignoreAttributes: false, + attributeNamePrefix: '', + parseTagValue: false, + isArray: (name, jpath, isLeafNode, isAttribute) => { if (alwaysArray.indexOf(jpath) !== -1) return true; }, - textNodeName : 'text' + textNodeName: 'text' }; const parser = new XMLParser(options); function load() { const tests = []; fs.readdirSync(testsPath).forEach(file => { - console.log('Loading tests from ' + file); + // console.log('Loading tests from ' + file); let testsContainer = parser.parse(fs.readFileSync(path.join(testsPath, file))); tests.push(testsContainer.tests); }); return tests; -} +} module.exports = { load -} \ No newline at end of file +} + +