From 13e8db2000c9956524d861f59d910527bbbd8cdb Mon Sep 17 00:00:00 2001 From: Mariusz Jasuwienas Date: Tue, 12 Aug 2025 09:52:18 +0200 Subject: [PATCH 1/3] test: batch 2 conformity tests (#3823) Signed-off-by: Mariusz Jasuwienas --- .../tests/acceptance/conformityTests.spec.ts | 110 ++++++++---------- 1 file changed, 48 insertions(+), 62 deletions(-) diff --git a/packages/server/tests/acceptance/conformityTests.spec.ts b/packages/server/tests/acceptance/conformityTests.spec.ts index 49c5f353a1..ff2b255126 100644 --- a/packages/server/tests/acceptance/conformityTests.spec.ts +++ b/packages/server/tests/acceptance/conformityTests.spec.ts @@ -1,19 +1,14 @@ // SPDX-License-Identifier: Apache-2.0 -// import { -// JSONSchemaObject, -// MethodObject, -// MethodOrReference, -// OpenrpcDocument, -// } from '@open-rpc/meta-schema'; -// import { parseOpenRPCDocument } from '@open-rpc/schema-utils-js'; -// import { expect } from 'chai'; +import { JSONSchemaObject, MethodObject, MethodOrReference, OpenrpcDocument } from '@open-rpc/meta-schema'; +import { parseOpenRPCDocument } from '@open-rpc/schema-utils-js'; +import { expect } from 'chai'; import fs from 'fs'; import path from 'path'; // import WebSocket from 'ws'; import openRpcData from '../../../../docs/openrpc.json'; -// import CallerContract from '../contracts/Caller.json'; -// import LogsContract from '../contracts/Logs.json'; +import CallerContract from '../contracts/Caller.json'; +import LogsContract from '../contracts/Logs.json'; import { chainId, gasLimit, @@ -26,9 +21,8 @@ import { setTransaction1559_2930AndBlockHash, setTransaction1559AndBlockHash, setTransaction2930AndBlockHash, - // WS_RELAY_URL, } from './data/conformity/utils/constants'; -// import { TestCases, UpdateParamFunction } from './data/conformity/utils/interfaces'; +import { TestCases, UpdateParamFunction } from './data/conformity/utils/interfaces'; import { processFileContent, splitReqAndRes } from './data/conformity/utils/processors'; import { createContractLegacyTransaction, @@ -37,53 +31,45 @@ import { transaction1559_2930, transaction2930, } from './data/conformity/utils/transactions'; -import { - getLatestBlockHash, - // sendRequestToRelay, - signAndSendRawTransaction, -} from './data/conformity/utils/utils'; -// import { hasResponseFormatIssues, isResponseValid } from './data/conformity/utils/validations'; +import { getLatestBlockHash, sendRequestToRelay, signAndSendRawTransaction } from './data/conformity/utils/utils'; +import { isResponseValid } from './data/conformity/utils/validations'; const directoryPath = path.resolve(__dirname, '../../../../node_modules/execution-apis/tests'); const overwritesDirectoryPath = path.resolve(__dirname, 'data/conformity/overwrites'); -// let relayOpenRpcData: OpenrpcDocument; -// (async () => { -// relayOpenRpcData = await parseOpenRPCDocument(JSON.stringify(openRpcData)); -// })().catch((error) => console.error('Error parsing OpenRPC document:', error)); - -// const synthesizeTestCases = function (testCases: TestCases, updateParamIfNeeded: UpdateParamFunction) { -// for (const testName in testCases) { -// it(`${testName}`, async function () { -// const isErrorStatusExpected: boolean = -// (testCases[testName]?.status && testCases[testName].status != 200) || -// !!JSON.parse(testCases[testName].response).error; -// const method = relayOpenRpcData.methods.find( -// (m: MethodOrReference): m is MethodObject => 'name' in m && m.name === testName.split(' ')[0], -// ); -// const schema: JSONSchemaObject | undefined = -// method?.result && 'schema' in method.result && typeof method.result.schema === 'object' -// ? method.result.schema -// : undefined; -// try { -// const req = updateParamIfNeeded(testName, JSON.parse(testCases[testName].request)); -// const res = await sendRequestToRelay(RELAY_URL, req, false); -// const isResFormatInvalid: boolean = hasResponseFormatIssues(res, JSON.parse(testCases[testName].response)); -// -// if (schema && schema.pattern) { -// const check = isResponseValid(schema, res); -// expect(check).to.be.true; -// } -// -// expect(isResFormatInvalid).to.be.false; -// expect(isErrorStatusExpected).to.be.false; -// } catch (e: any) { -// expect(isErrorStatusExpected).to.be.true; -// expect(e?.response?.status).to.equal(testCases[testName].status); -// } -// }); -// } -// }; +let relayOpenRpcData: OpenrpcDocument; +(async () => { + relayOpenRpcData = await parseOpenRPCDocument(JSON.stringify(openRpcData)); +})().catch((error) => console.error('Error parsing OpenRPC document:', error)); + +const synthesizeTestCases = function (testCases: TestCases, updateParamIfNeeded: UpdateParamFunction) { + for (const testName in testCases) { + it(`${testName}`, async function () { + const isErrorStatusExpected: boolean = + (testCases[testName]?.status && testCases[testName].status != 200) || + !!JSON.parse(testCases[testName].response).error; + const method = relayOpenRpcData.methods.find( + (m: MethodOrReference): m is MethodObject => 'name' in m && m.name === testName.split(' ')[0], + ); + const schema: JSONSchemaObject | undefined = + method?.result && 'schema' in method.result && typeof method.result.schema === 'object' + ? method.result.schema + : undefined; + try { + const req = updateParamIfNeeded(testName, JSON.parse(testCases[testName].request)); + const res = await sendRequestToRelay(RELAY_URL, req, false); + if (schema && schema.pattern) { + const check = isResponseValid(schema, res); + expect(check).to.be.true; + } + expect(isErrorStatusExpected).to.be.false; + } catch (e: any) { + expect(isErrorStatusExpected).to.be.true; + expect(e?.response?.status).to.equal(testCases[testName].status); + } + }); + } +}; /** * To run the Ethereum Execution API tests as defined in the repository ethereum/execution-apis, it’s necessary @@ -152,7 +138,7 @@ describe('@api-conformity', async function () { // // These test suites must be un-skipped. The code requires refactoring to resolve the // static analysis issues before they can be re-enabled. - /* describe.skip('@conformity-batch-2 Ethereum execution apis tests', async function () { + describe('@conformity-batch-2 Ethereum execution apis tests', async function () { this.timeout(240 * 1000); let existingBlockFilter: string; @@ -222,7 +208,7 @@ describe('@api-conformity', async function () { synthesizeTestCases(TEST_CASES_BATCH_2, updateParamIfNeeded); }); - describe.skip('@conformity-batch-3 Ethereum execution apis tests', async function () { + describe('@conformity-batch-3 Ethereum execution apis tests', async function () { this.timeout(240 * 1000); let txHash: any; @@ -254,7 +240,7 @@ describe('@api-conformity', async function () { synthesizeTestCases(TEST_CASES_BATCH_3['server'], updateParamIfNeeded); - describe('ws related rpc methods', async function () { + /* describe('ws related rpc methods', async function () { let webSocket: WebSocket; let contractAddress: string | null; let existingFilter: string; @@ -340,10 +326,10 @@ describe('@api-conformity', async function () { }; synthesizeWsTestCases(TEST_CASES_BATCH_3['ws-server'], updateParamIfNeeded); - }); + });*/ }); - describe.skip('@conformity-batch-4 Ethereum execution apis tests', async function () { + describe('@conformity-batch-4 Ethereum execution apis tests', async function () { this.timeout(240 * 1000); let existingCallerContractAddress: string | null; @@ -543,12 +529,12 @@ describe('@api-conformity', async function () { synthesizeTestCases(TEST_CASES_BATCH_4, updateParamIfNeeded); }); - describe.skip('@conformity-batch-5 Ethereum execution apis tests', async function () { + describe('@conformity-batch-5 Ethereum execution apis tests', async function () { this.timeout(240 * 1000); // eslint-disable-next-line @typescript-eslint/no-var-requires const TEST_CASES_BATCH_5 = require('./data/conformity-tests-batch-5.json'); const updateParamIfNeeded = (_testName: any, request: any) => request; synthesizeTestCases(TEST_CASES_BATCH_5, updateParamIfNeeded); - });*/ + }); }); From 65694d30b60f59f3f5186386cf505715c5dff3fe Mon Sep 17 00:00:00 2001 From: Mariusz Jasuwienas Date: Sun, 17 Aug 2025 20:06:22 +0200 Subject: [PATCH 2/3] test: bringing back ws tests and applying codacy suggestion (#3823) Signed-off-by: Mariusz Jasuwienas --- .../tests/acceptance/conformityTests.spec.ts | 62 ++++++++++++++----- .../data/conformity/utils/validations.ts | 47 ++++++++++++-- 2 files changed, 87 insertions(+), 22 deletions(-) diff --git a/packages/server/tests/acceptance/conformityTests.spec.ts b/packages/server/tests/acceptance/conformityTests.spec.ts index ff2b255126..246720c525 100644 --- a/packages/server/tests/acceptance/conformityTests.spec.ts +++ b/packages/server/tests/acceptance/conformityTests.spec.ts @@ -4,8 +4,8 @@ import { parseOpenRPCDocument } from '@open-rpc/schema-utils-js'; import { expect } from 'chai'; import fs from 'fs'; import path from 'path'; +import WebSocket from 'ws'; -// import WebSocket from 'ws'; import openRpcData from '../../../../docs/openrpc.json'; import CallerContract from '../contracts/Caller.json'; import LogsContract from '../contracts/Logs.json'; @@ -21,8 +21,9 @@ import { setTransaction1559_2930AndBlockHash, setTransaction1559AndBlockHash, setTransaction2930AndBlockHash, + WS_RELAY_URL, } from './data/conformity/utils/constants'; -import { TestCases, UpdateParamFunction } from './data/conformity/utils/interfaces'; +import { TestCase, TestCases, UpdateParamFunction } from './data/conformity/utils/interfaces'; import { processFileContent, splitReqAndRes } from './data/conformity/utils/processors'; import { createContractLegacyTransaction, @@ -32,7 +33,7 @@ import { transaction2930, } from './data/conformity/utils/transactions'; import { getLatestBlockHash, sendRequestToRelay, signAndSendRawTransaction } from './data/conformity/utils/utils'; -import { isResponseValid } from './data/conformity/utils/validations'; +import { getMissingKeys, isResponseValid } from './data/conformity/utils/validations'; const directoryPath = path.resolve(__dirname, '../../../../node_modules/execution-apis/tests'); const overwritesDirectoryPath = path.resolve(__dirname, 'data/conformity/overwrites'); @@ -42,19 +43,48 @@ let relayOpenRpcData: OpenrpcDocument; relayOpenRpcData = await parseOpenRPCDocument(JSON.stringify(openRpcData)); })().catch((error) => console.error('Error parsing OpenRPC document:', error)); +/** + * Determines whether a given test case is expected to return an error response. + * + * @param testCase - The test case to evaluate. + * @returns {boolean} `true` if an error response is expected, otherwise `false`. + * + * @example + * ```typescript + * const tc = { status: 404, response: '{"error": "Not found"}' }; + * console.log(isErrorResponseExpected(tc)); // true + * ``` + */ +const isErrorResponseExpected = function (testCase: TestCase): boolean { + return (testCase?.status && testCase.status != 200) || !!JSON.parse(testCase.response).error; +}; + +/** + * Retrieves the JSON schema object for the result of a given method name from the OpenRPC data. + * + * @param name - The name of the method to look up. + * @returns {JSONSchemaObject | undefined} The method's result schema, or `undefined` if not found or invalid. + * + * @example + * ```typescript + * const schema = getMethodSchema("eth_getBalance"); + * console.log(schema); // JSON schema object or undefined + * ``` + */ +const getMethodSchema = function (name: string): JSONSchemaObject | undefined { + const method = relayOpenRpcData.methods.find( + (m: MethodOrReference): m is MethodObject => 'name' in m && m.name === name, + ); + return method?.result && 'schema' in method.result && typeof method.result.schema === 'object' + ? method.result.schema + : undefined; +}; + const synthesizeTestCases = function (testCases: TestCases, updateParamIfNeeded: UpdateParamFunction) { for (const testName in testCases) { it(`${testName}`, async function () { - const isErrorStatusExpected: boolean = - (testCases[testName]?.status && testCases[testName].status != 200) || - !!JSON.parse(testCases[testName].response).error; - const method = relayOpenRpcData.methods.find( - (m: MethodOrReference): m is MethodObject => 'name' in m && m.name === testName.split(' ')[0], - ); - const schema: JSONSchemaObject | undefined = - method?.result && 'schema' in method.result && typeof method.result.schema === 'object' - ? method.result.schema - : undefined; + const isErrorStatusExpected = isErrorResponseExpected(testCases[testName]); + const schema = getMethodSchema(testName.split(' ')[0]); try { const req = updateParamIfNeeded(testName, JSON.parse(testCases[testName].request)); const res = await sendRequestToRelay(RELAY_URL, req, false); @@ -240,7 +270,7 @@ describe('@api-conformity', async function () { synthesizeTestCases(TEST_CASES_BATCH_3['server'], updateParamIfNeeded); - /* describe('ws related rpc methods', async function () { + describe('ws related rpc methods', async function () { let webSocket: WebSocket; let contractAddress: string | null; let existingFilter: string; @@ -319,14 +349,14 @@ describe('@api-conformity', async function () { }); await new Promise((r) => setTimeout(r, 500)); - const hasMissingKeys: boolean = hasResponseFormatIssues(response, JSON.parse(testCases[testName].response)); + const hasMissingKeys = getMissingKeys(response, JSON.parse(testCases[testName].response)).length > 0; expect(hasMissingKeys).to.be.false; }); } }; synthesizeWsTestCases(TEST_CASES_BATCH_3['ws-server'], updateParamIfNeeded); - });*/ + }); }); describe('@conformity-batch-4 Ethereum execution apis tests', async function () { diff --git a/packages/server/tests/acceptance/data/conformity/utils/validations.ts b/packages/server/tests/acceptance/data/conformity/utils/validations.ts index 2f3bc23eb2..5e3053a6eb 100644 --- a/packages/server/tests/acceptance/data/conformity/utils/validations.ts +++ b/packages/server/tests/acceptance/data/conformity/utils/validations.ts @@ -84,12 +84,7 @@ export function hasResponseFormatIssues( } return true; } - - const actualResponseKeys = extractKeys(actualResponse as Record); - const expectedResponseKeys = extractKeys(parsedExpectedResponse as Record); - const filteredExpectedKeys = expectedResponseKeys.filter((key) => !wildcards.includes(key)); - const missingKeys = filteredExpectedKeys.filter((key) => !actualResponseKeys.includes(key)); - + const missingKeys = getMissingKeys(actualResponse as Record, parsedExpectedResponse, wildcards); if (missingKeys.length > 0) { console.log(`Missing keys in response: ${JSON.stringify(missingKeys)}`); return true; @@ -98,6 +93,46 @@ export function hasResponseFormatIssues( return hasValuesMismatch(actualResponse, parsedExpectedResponse, wildcards); } +/** + * Returns the list of keys that exist in `expectedResponse` but are missing in `actualResponse`, + * excluding any keys listed in `wildcards`. + * + * This function compares only key presence (as produced by `extractKeys`)—it does not parse, + * validate error states, or compare values. Any keys in `expectedResponse` that match entries + * in `wildcards` are ignored. + * + * @param actualResponse - The actual response object whose keys will be checked. + * @param expectedResponse - The reference object whose keys are considered required. + * @param wildcards - Array of key paths to ignore during the check (default: []). + * @returns {string[]} Array of key names/paths that are required by `expectedResponse` + * but absent from `actualResponse`. + * + * @example + * ```typescript + * const actual = { result: "0x123" }; + * const expected = { result: "0x123", id: 1 }; + * const missing = getMissingKeys(actual, expected); + * console.log(missing); // ["id"] + * ``` + * + * @example + * ```typescript + * const actual = { result: "0x123", timestamp: "2023-01-01" }; + * const expected = { result: "0x123", timestamp: "2023-01-02", id: 1 }; + * const missing = getMissingKeys(actual, expected, ["timestamp"]); + * console.log(missing); // ["id"] // "timestamp" is ignored by wildcard + * ``` + */ +export function getMissingKeys( + actualResponse: Record, + expectedResponse: Record, + wildcards: string[] = [], +): string[] { + const actualResponseKeys = extractKeys(actualResponse); + const expectedResponseKeys = extractKeys(expectedResponse); + return expectedResponseKeys.filter((key) => !actualResponseKeys.includes(key) && !wildcards.includes(key)); +} + /** * Checks if the actual response is missing required error properties * From 9356d97beb9bb30bab9b890798af032fb70aec25 Mon Sep 17 00:00:00 2001 From: Mariusz Jasuwienas Date: Mon, 18 Aug 2025 12:28:05 +0200 Subject: [PATCH 3/3] test: forcing codacy to rerun checks (#3823) Signed-off-by: Mariusz Jasuwienas --- .../tests/acceptance/conformityTests.spec.ts | 7 +- .../data/conformity/utils/validations.ts | 64 ++++++++----------- 2 files changed, 33 insertions(+), 38 deletions(-) diff --git a/packages/server/tests/acceptance/conformityTests.spec.ts b/packages/server/tests/acceptance/conformityTests.spec.ts index 246720c525..2b18f70d36 100644 --- a/packages/server/tests/acceptance/conformityTests.spec.ts +++ b/packages/server/tests/acceptance/conformityTests.spec.ts @@ -349,7 +349,12 @@ describe('@api-conformity', async function () { }); await new Promise((r) => setTimeout(r, 500)); - const hasMissingKeys = getMissingKeys(response, JSON.parse(testCases[testName].response)).length > 0; + const hasMissingKeys = + getMissingKeys({ + actual: response, + expected: JSON.parse(testCases[testName].response), + wildcards: [], + }).length > 0; expect(hasMissingKeys).to.be.false; }); } diff --git a/packages/server/tests/acceptance/data/conformity/utils/validations.ts b/packages/server/tests/acceptance/data/conformity/utils/validations.ts index 5e3053a6eb..0fe5dc5b60 100644 --- a/packages/server/tests/acceptance/data/conformity/utils/validations.ts +++ b/packages/server/tests/acceptance/data/conformity/utils/validations.ts @@ -20,7 +20,7 @@ addFormats(ajv); * Validates response format by comparing actual response against expected response structure. * * @param actualResponse - The actual response received from the API call - * @param expectedResponse - The expected response structure to validate against (can be object, string, or ErrorResponse) + * @param expectedResponse - The expected response structure to validate against (can be an object, string, or ErrorResponse) * @param wildcards - Array of property paths to ignore during validation (default: empty array) * @returns {boolean} Returns true if the response format has issues (validation failed), false if format is valid * @@ -84,7 +84,11 @@ export function hasResponseFormatIssues( } return true; } - const missingKeys = getMissingKeys(actualResponse as Record, parsedExpectedResponse, wildcards); + const missingKeys = getMissingKeys({ + actual: actualResponse as Record, + expected: parsedExpectedResponse, + wildcards, + }); if (missingKeys.length > 0) { console.log(`Missing keys in response: ${JSON.stringify(missingKeys)}`); return true; @@ -94,45 +98,31 @@ export function hasResponseFormatIssues( } /** - * Returns the list of keys that exist in `expectedResponse` but are missing in `actualResponse`, - * excluding any keys listed in `wildcards`. - * - * This function compares only key presence (as produced by `extractKeys`)—it does not parse, - * validate error states, or compare values. Any keys in `expectedResponse` that match entries - * in `wildcards` are ignored. - * - * @param actualResponse - The actual response object whose keys will be checked. - * @param expectedResponse - The reference object whose keys are considered required. - * @param wildcards - Array of key paths to ignore during the check (default: []). - * @returns {string[]} Array of key names/paths that are required by `expectedResponse` - * but absent from `actualResponse`. - * - * @example - * ```typescript - * const actual = { result: "0x123" }; - * const expected = { result: "0x123", id: 1 }; - * const missing = getMissingKeys(actual, expected); - * console.log(missing); // ["id"] - * ``` + * Gets the list of expected keys that are missing from the actual response, + * excluding any wildcard keys that should be ignored. * - * @example - * ```typescript - * const actual = { result: "0x123", timestamp: "2023-01-01" }; - * const expected = { result: "0x123", timestamp: "2023-01-02", id: 1 }; - * const missing = getMissingKeys(actual, expected, ["timestamp"]); - * console.log(missing); // ["id"] // "timestamp" is ignored by wildcard - * ``` + * @param response - The object containing: + * - actual: the actual response object to check + * - expected: the expected response object + * - wildcards: a list of keys to ignore during comparison + * @returns {string[]} - An array of keys that are expected but missing in the actual response */ -export function getMissingKeys( - actualResponse: Record, - expectedResponse: Record, - wildcards: string[] = [], -): string[] { - const actualResponseKeys = extractKeys(actualResponse); - const expectedResponseKeys = extractKeys(expectedResponse); - return expectedResponseKeys.filter((key) => !actualResponseKeys.includes(key) && !wildcards.includes(key)); +export function getMissingKeys(response: { + actual: Record; + expected: Record; + wildcards: string[]; +}): string[] { + const skipKeys = new Set([...extractKeys(response.actual), ...response.wildcards]); + return extractKeys(response.expected).filter(notIn(skipKeys)); } +/** + * Predicate factory that checks whether a given key is not in a provided Set. + * + * @param set - A Set of keys to exclude + */ +const notIn = (set: ReadonlySet) => (k: string) => !set.has(k); + /** * Checks if the actual response is missing required error properties *