Capture separate logs for each lsptoolshost test suite

Directory.Build.props

@@ -5,6 +5,7 @@
             Defines the lowest supported target framework for the extension.
             Used by server download / integration tests to ensure they run when only this SDK is installed.
         -->
+        <CheckEolTargetFramework>false</CheckEolTargetFramework>
         <LowestSupportedTargetFramework>net6.0</LowestSupportedTargetFramework>
     </PropertyGroup>
-</Project>
+</Project>

azure-pipelines/test-matrix.yml

@@ -21,10 +21,9 @@ jobs:
       UnitTests:
         npmCommand: test:unit
         isIntegration: false
-      ${{ if ne(parameters.os, 'windows') }}:
-        CSharpIntegrationTests:
-          npmCommand: test:integration:csharp
-          isIntegration: true
+      CSharpIntegrationTests:
+        npmCommand: test:integration:csharp
+        isIntegration: true
       DevKitTests:
         npmCommand: test:integration:devkit
         isIntegration: true

test/jestHelpers.ts

@@ -0,0 +1,55 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as vscode from 'vscode';
+import { EOL, platform } from 'os';
+import { describe, expect, test } from '@jest/globals';
+import { usingDevKit } from './lsptoolshost/integrationTests/integrationHelpers';
+
+export async function expectText(document: vscode.TextDocument, expectedLines: string[]) {
+    const expectedText = expectedLines.join(EOL);
+    expect(document.getText()).toBe(expectedText);
+}
+
+export function expectPath(expected: vscode.Uri, actual: vscode.Uri) {
+    if (isLinux()) {
+        expect(actual.path).toBe(expected.path);
+    } else {
+        const actualPath = actual.path.toLowerCase();
+        const expectedPath = expected.path.toLocaleLowerCase();
+        expect(actualPath).toBe(expectedPath);
+    }
+}
+
+export const describeIfCSharp = describeIf(!usingDevKit());
+export const describeIfDevKit = describeIf(usingDevKit());
+export const describeIfNotMacOS = describeIf(!isMacOS());
+export const describeIfWindows = describeIf(isWindows());
+export const testIfCSharp = testIf(!usingDevKit());
+export const testIfDevKit = testIf(usingDevKit());
+export const testIfNotMacOS = testIf(!isMacOS());
+export const testIfWindows = testIf(isWindows());
+
+function describeIf(condition: boolean) {
+    return condition ? describe : describe.skip;
+}
+
+function testIf(condition: boolean) {
+    return condition ? test : test.skip;
+}
+
+function isMacOS() {
+    const currentPlatform = platform();
+    return currentPlatform === 'darwin';
+}
+
+function isWindows() {
+    const currentPlatform = platform();
+    return currentPlatform === 'win32';
+}
+
+function isLinux() {
+    return !(isMacOS() || isWindows());
+}

test/lsptoolshost/integrationTests/buildDiagnostics.integration.test.ts

@@ -12,6 +12,7 @@ import {
 } from '../../../src/lsptoolshost/diagnostics/buildDiagnosticsService';
 import * as integrationHelpers from './integrationHelpers';
 import path from 'path';
+
 describe(`Build and live diagnostics dedupe`, () => {
     beforeAll(async () => {
         await integrationHelpers.activateCSharpExtension();

test/lsptoolshost/integrationTests/codeactions.integration.test.ts

@@ -7,12 +7,8 @@ import * as vscode from 'vscode';
 import * as path from 'path';
 import { describe, beforeAll, beforeEach, afterAll, test, expect, afterEach } from '@jest/globals';
 import testAssetWorkspace from './testAssets/testAssetWorkspace';
-import {
-    activateCSharpExtension,
-    closeAllEditorsAsync,
-    expectText,
-    openFileInWorkspaceAsync,
-} from './integrationHelpers';
+import { activateCSharpExtension, closeAllEditorsAsync, openFileInWorkspaceAsync } from './integrationHelpers';
+import { expectText } from '../../jestHelpers';
 
 describe(`Code Actions Tests`, () => {
     beforeAll(async () => {

test/lsptoolshost/integrationTests/commandEnablement.integration.test.ts

@@ -5,14 +5,15 @@
 
 import { expect, beforeAll, afterAll, describe } from '@jest/globals';
 import * as vscode from 'vscode';
-import { activateCSharpExtension, testIfCSharp, testIfDevKit } from './integrationHelpers';
+import { activateCSharpExtension } from './integrationHelpers';
 import testAssetWorkspace from './testAssets/testAssetWorkspace';
 import {
     RoslynDevKitCommands,
     RoslynStandaloneCommands,
     UnexpectedRoslynDevKitCommands,
     UnexpectedRoslynStandaloneCommands,
 } from './expectedCommands';
+import { testIfCSharp, testIfDevKit } from '../../jestHelpers';
 
 describe(`Command Enablement Tests`, () => {
     beforeAll(async () => {

test/lsptoolshost/integrationTests/documentDiagnostics.integration.test.ts

@@ -9,12 +9,8 @@ import testAssetWorkspace from './testAssets/testAssetWorkspace';
 import { AnalysisSetting } from '../../../src/lsptoolshost/diagnostics/buildDiagnosticsService';
 import path from 'path';
 import { getCode, setDiagnosticSettings, waitForExpectedDiagnostics } from './diagnosticsHelpers';
-import {
-    activateCSharpExtension,
-    closeAllEditorsAsync,
-    describeIfCSharp,
-    openFileInWorkspaceAsync,
-} from './integrationHelpers';
+import { activateCSharpExtension, closeAllEditorsAsync, openFileInWorkspaceAsync } from './integrationHelpers';
+import { describeIfCSharp } from '../../jestHelpers';
 
 // Restarting the server is required for these tests, but not supported with C# Dev Kit.
 describeIfCSharp(`Document Diagnostics Tests`, () => {

test/lsptoolshost/integrationTests/formatting.integration.test.ts

@@ -6,14 +6,10 @@
 import * as vscode from 'vscode';
 import * as path from 'path';
 import testAssetWorkspace from './testAssets/testAssetWorkspace';
-import {
-    activateCSharpExtension,
-    closeAllEditorsAsync,
-    expectText,
-    openFileInWorkspaceAsync,
-} from './integrationHelpers';
+import { activateCSharpExtension, closeAllEditorsAsync, openFileInWorkspaceAsync } from './integrationHelpers';
 import { describe, beforeEach, afterAll, test, afterEach } from '@jest/globals';
 import { formatDocumentAsync, formatOnTypeAsync, formatRangeAsync } from './formattingTestHelpers';
+import { expectText } from '../../jestHelpers';
 
 describe(`Formatting Tests`, () => {
     beforeEach(async () => {

test/lsptoolshost/integrationTests/formattingEditorConfig.integration.test.ts

@@ -6,14 +6,10 @@
 import * as vscode from 'vscode';
 import * as path from 'path';
 import testAssetWorkspace from './testAssets/testAssetWorkspace';
-import {
-    activateCSharpExtension,
-    closeAllEditorsAsync,
-    expectText,
-    openFileInWorkspaceAsync,
-} from './integrationHelpers';
+import { activateCSharpExtension, closeAllEditorsAsync, openFileInWorkspaceAsync } from './integrationHelpers';
 import { describe, beforeAll, beforeEach, afterAll, test, afterEach } from '@jest/globals';
 import { formatDocumentAsync, formatOnTypeAsync, formatRangeAsync } from './formattingTestHelpers';
+import { expectText } from '../../jestHelpers';
 
 describe(`Formatting With EditorConfig Tests`, () => {
     beforeAll(async () => {

test/lsptoolshost/integrationTests/gotoDefinition.integration.test.ts

@@ -12,10 +12,9 @@ import {
     findRangeOfString,
     navigate,
     openFileInWorkspaceAsync,
-    testIfCSharp,
-    testIfDevKit,
 } from './integrationHelpers';
 import { describe, beforeAll, beforeEach, afterAll, test, expect, afterEach } from '@jest/globals';
+import { testIfCSharp, testIfDevKit } from '../../jestHelpers';
 
 describe(`Go To Definition Tests`, () => {
     beforeAll(async () => {

test/lsptoolshost/integrationTests/index.ts

@@ -3,11 +3,73 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
+import * as vscode from 'vscode';
+import * as fsExtra from 'fs-extra';
+import path from 'path';
+import { AggregatedResult } from '@jest/test-result';
 import { runIntegrationTests } from '../../runIntegrationTests';
 import { jestIntegrationTestProjectName } from './jest.config';
+import { activateCSharpExtension } from './integrationHelpers';
 
 export async function run() {
     process.env.RUNNING_INTEGRATION_TESTS = 'true';
 
-    await runIntegrationTests(jestIntegrationTestProjectName);
+    await activateCSharpExtension();
+    await moveLogs('activated');
+
+    let anyFailures = false;
+    let results: AggregatedResult | undefined;
+    if (process.env.TEST_FILE_FILTER) {
+        results = await runIntegrationTests(jestIntegrationTestProjectName);
+        await moveLogs(path.basename(process.env.TEST_FILE_FILTER, '.integration.test.ts'));
+        anyFailures = anyFailures || !results.success;
+    } else {
+        const workingDirectory = process.cwd();
+
+        const testFiles = (await fsExtra.readdir(__dirname))
+            .filter((file) => file.endsWith('.integration.test.js'))
+            .map((file) => path.join(__dirname, file));
+
+        for (let file of testFiles) {
+            // We have to fix up the file path because the test file was discovered in the /out/ directory.
+            file = file.substring(0, file.length - 2) + 'ts';
+            file = file.replace(/[\\/]out[\\/]/, path.sep);
+            file = workingDirectory[0] + file.substring(1);
+
+            console.log('');
+            console.log(`-- Running integration tests for ${path.basename(file)} --`);
+            console.log('');
+
+            process.env.TEST_FILE_FILTER = file;
+
+            results = await runIntegrationTests(jestIntegrationTestProjectName);
+            await moveLogs(path.basename(process.env.TEST_FILE_FILTER, '.integration.test.ts'));
+            anyFailures = anyFailures || !results.success;
+        }
+    }
+
+    // Explicitly exit the process - VSCode likes to write a bunch of cancellation errors to the console after this
+    // which make it look like the tests always fail.  We're done with the tests at this point, so just exit.
+    process.exit(anyFailures ? 1 : 0);
+}
+
+async function moveLogs(name: string) {
+    const exports = vscode.extensions.getExtension('ms-dotnettools.csharp')?.exports;
+    if (!exports) {
+        throw new Error('Failed to get C# extension exports for cleanup');
+    }
+
+    if (!exports.logDirectory) {
+        console.warn(`Failed to get log directory from C# extension exports`);
+        return;
+    }
+
+    const targetLogDir = path.join(
+        path.dirname(exports.logDirectory),
+        `${name ?? 'unknown'}_${path.basename(exports.logDirectory)}`
+    );
+    await fsExtra.copy(exports.logDirectory, targetLogDir);
+    console.log(`Copied extension logs from ${exports.logDirectory} to ${targetLogDir}`);
+
+    await new Promise((resolve) => fsExtra.rm(path.join(exports.logDirectory, '*.log'), resolve));
 }

test/lsptoolshost/integrationTests/integrationHelpers.ts

@@ -10,8 +10,6 @@ import { CSharpExtensionExports } from '../../../src/csharpExtensionExports';
 import { existsSync } from 'fs';
 import { ServerState } from '../../../src/lsptoolshost/server/languageServerEvents';
 import testAssetWorkspace from './testAssets/testAssetWorkspace';
-import { EOL, platform } from 'os';
-import { describe, expect, test } from '@jest/globals';
 
 export async function activateCSharpExtension(): Promise<void> {
     const csharpExtension = vscode.extensions.getExtension<CSharpExtensionExports>('ms-dotnettools.csharp');
@@ -191,8 +189,6 @@ export async function navigate(
 
     // Navigation happens asynchronously when a different file is opened, so we need to wait for the window to change.
     await windowChanged;
-
-    expect(vscode.window.activeTextEditor?.document.fileName).toContain(expectedFileName);
 }
 
 export function sortLocations(locations: vscode.Location[]): vscode.Location[] {
@@ -253,49 +249,3 @@ export async function waitForExpectedResult<T>(
 export async function sleep(ms = 0) {
     return new Promise((r) => setTimeout(r, ms));
 }
-
-export async function expectText(document: vscode.TextDocument, expectedLines: string[]) {
-    const expectedText = expectedLines.join(EOL);
-    expect(document.getText()).toBe(expectedText);
-}
-
-export function expectPath(expected: vscode.Uri, actual: vscode.Uri) {
-    if (isLinux()) {
-        expect(actual.path).toBe(expected.path);
-    } else {
-        const actualPath = actual.path.toLowerCase();
-        const expectedPath = expected.path.toLocaleLowerCase();
-        expect(actualPath).toBe(expectedPath);
-    }
-}
-
-export const describeIfCSharp = describeIf(!usingDevKit());
-export const describeIfDevKit = describeIf(usingDevKit());
-export const describeIfNotMacOS = describeIf(!isMacOS());
-export const describeIfWindows = describeIf(isWindows());
-export const testIfCSharp = testIf(!usingDevKit());
-export const testIfDevKit = testIf(usingDevKit());
-export const testIfNotMacOS = testIf(!isMacOS());
-export const testIfWindows = testIf(isWindows());
-
-function describeIf(condition: boolean) {
-    return condition ? describe : describe.skip;
-}
-
-function testIf(condition: boolean) {
-    return condition ? test : test.skip;
-}
-
-function isMacOS() {
-    const currentPlatform = platform();
-    return currentPlatform === 'darwin';
-}
-
-function isWindows() {
-    const currentPlatform = platform();
-    return currentPlatform === 'win32';
-}
-
-function isLinux() {
-    return !(isMacOS() || isWindows());
-}

test/lsptoolshost/integrationTests/unitTests.integration.test.ts

@@ -10,11 +10,11 @@ import testAssetWorkspace from './testAssets/testAssetWorkspace';
 import {
     activateCSharpExtension,
     closeAllEditorsAsync,
-    describeIfCSharp,
     getCodeLensesAsync,
     openFileInWorkspaceAsync,
 } from './integrationHelpers';
 import { TestProgress } from '../../../src/lsptoolshost/server/roslynProtocol';
+import { describeIfCSharp } from '../../jestHelpers';
 
 describeIfCSharp(`Unit Testing Tests`, () => {
     beforeAll(async () => {

test/lsptoolshost/integrationTests/workspaceDiagnostics.integration.test.ts

@@ -8,7 +8,8 @@ import { describe, test, expect, beforeAll, afterAll } from '@jest/globals';
 import testAssetWorkspace from './testAssets/testAssetWorkspace';
 import { AnalysisSetting } from '../../../src/lsptoolshost/diagnostics/buildDiagnosticsService';
 import { getCode, setDiagnosticSettings, waitForExpectedDiagnostics } from './diagnosticsHelpers';
-import { activateCSharpExtension, describeIfCSharp } from './integrationHelpers';
+import { activateCSharpExtension } from './integrationHelpers';
+import { describeIfCSharp } from '../../jestHelpers';
 
 // Restarting the server is required for these tests, but not supported with C# Dev Kit.
 describeIfCSharp(`Workspace Diagnostic Tests`, () => {

test/omnisharp/omnisharpIntegrationTests/index.ts

@@ -9,5 +9,9 @@ import { jestIntegrationTestProjectName } from './jest.config';
 export async function run() {
     process.env.RUNNING_INTEGRATION_TESTS = 'true';
 
-    await runIntegrationTests(jestIntegrationTestProjectName);
+    const results = await runIntegrationTests(jestIntegrationTestProjectName);
+
+    // Explicitly exit the process - VSCode likes to write a bunch of cancellation errors to the console after this
+    // which make it look like the tests always fail.  We're done with the tests at this point, so just exit.
+    process.exit(results.success ? 0 : 1);
 }

test/razor/razorIntegrationTests/index.ts

@@ -9,5 +9,9 @@ import { jestIntegrationTestProjectName } from './jest.config';
 export async function run() {
     process.env.RUNNING_INTEGRATION_TESTS = 'true';
 
-    await runIntegrationTests(jestIntegrationTestProjectName);
+    const results = await runIntegrationTests(jestIntegrationTestProjectName);
+
+    // Explicitly exit the process - VSCode likes to write a bunch of cancellation errors to the console after this
+    // which make it look like the tests always fail.  We're done with the tests at this point, so just exit.
+    process.exit(results.success ? 0 : 1);
 }

test/razor/razorIntegrationTests/reference.integration.test.ts

@@ -8,6 +8,7 @@ import * as vscode from 'vscode';
 import { beforeAll, afterAll, test, expect, beforeEach, describe } from '@jest/globals';
 import testAssetWorkspace from './testAssets/testAssetWorkspace';
 import * as integrationHelpers from '../../lsptoolshost/integrationTests/integrationHelpers';
+import * as jestHelpers from '../../jestHelpers';
 
 describe(`Razor References ${testAssetWorkspace.description}`, function () {
     beforeAll(async function () {
@@ -151,9 +152,9 @@ describe(`Razor References ${testAssetWorkspace.description}`, function () {
                 const razorFile = integrationHelpers.getFilePath(path.join('Pages', 'References.razor'));
                 const csharpFile = integrationHelpers.getFilePath(path.join('Pages', 'References.razor.cs'));
 
-                integrationHelpers.expectPath(razorFile, sortedLocations[0].uri);
-                integrationHelpers.expectPath(csharpFile, sortedLocations[1].uri);
-                integrationHelpers.expectPath(csharpFile, sortedLocations[2].uri);
+                jestHelpers.expectPath(razorFile, sortedLocations[0].uri);
+                jestHelpers.expectPath(csharpFile, sortedLocations[1].uri);
+                jestHelpers.expectPath(csharpFile, sortedLocations[2].uri);
             }
         );
     });