chore(cli): speedup internal tests (#698)

* step 1 - no browser & preview if not test on page

* test with devtools-json

* step by step :)

* add dummy flavor

* more to test pnpm

* kitOnly

* installCmd

* Revert changes back to fe61de7 state

This reverts all changes made after commit fe61de7
(step 1 - no browser & preview if not test on page)

Reverted commits:
- cfe2965: test with devtools-json
- 1db5b78: step by step :)
- 167fdbd: add dummy flavor
- b376118: more to test pnpm
- 41cb8ff: kitOnly
- a6b528c: installCmd

* install & build only ts (when no extra things after)

* speed up by using async exec

* rework playwright test

* rename `skipBrowser` and don't run `prepareServer` when `browser` is disabled

* these tests need to have the browser and server running

* tweak storybook test

* fix drizzle test

* use locators

* oops

* ok let these be sync

* maybe this

* disable file parallelism

* test also limiting concurrency

* ugh

* remove concurrency only for tests that run shell commands

* revert before i lose my sanity

* pipe io

* skip storybook for now

* now lets do async shell cmd only on build and install

* yea i though as much - reverting

* dont skip storybook

---------

Co-authored-by: AdrianGonz97 <31664583+AdrianGonz97@users.noreply.github.com>

Author: jycouet

.changeset/dull-islands-lead.md

@@ -0,0 +1,5 @@
+---
+'sv': patch
+---
+
+chore(cli): speedup internal tests

packages/addons/_tests/_setup/suite.ts

@@ -1,6 +1,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
-import { execSync } from 'node:child_process';
+import { promisify } from 'node:util';
+import { exec, execSync } from 'node:child_process';
 import * as vitest from 'vitest';
 import { installAddon, type AddonMap, type OptionMap } from 'sv';
 import {
@@ -10,29 +11,38 @@ import {
 	type CreateProject,
 	type ProjectVariant
 } from 'sv/testing';
-import { chromium, type Browser, type Page } from '@playwright/test';
+import { chromium, type Browser, type BrowserContext, type Page } from '@playwright/test';
 
 const cwd = vitest.inject('testDir');
 const templatesDir = vitest.inject('templatesDir');
 const variants = vitest.inject('variants');
 
+export const execAsync = promisify(exec);
+
 type Fixtures<Addons extends AddonMap> = {
 	page: Page;
 	run(variant: ProjectVariant, options: OptionMap<Addons>): Promise<string>;
 };
 
-export function setupTest<Addons extends AddonMap>(addons: Addons) {
+export function setupTest<Addons extends AddonMap>(
+	addons: Addons,
+	options?: { browser?: boolean }
+) {
 	const test = vitest.test.extend<Fixtures<Addons>>({} as any);
 
+	const withBrowser = options?.browser ?? true;
+
 	let create: CreateProject;
 	let browser: Browser;
 
-	vitest.beforeAll(async () => {
-		browser = await chromium.launch();
-		return async () => {
-			await browser.close();
-		};
-	});
+	if (withBrowser) {
+		vitest.beforeAll(async () => {
+			browser = await chromium.launch();
+			return async () => {
+				await browser.close();
+			};
+		});
+	}
 
 	vitest.beforeAll(({ name }) => {
 		const testName = path.dirname(name).split('/').at(-1)!;
@@ -59,8 +69,11 @@ export function setupTest<Addons extends AddonMap>(addons: Addons) {
 
 	// runs before each test case
 	vitest.beforeEach<Fixtures<Addons>>(async (ctx) => {
-		const browserCtx = await browser.newContext();
-		ctx.page = await browserCtx.newPage();
+		let browserCtx: BrowserContext;
+		if (withBrowser) {
+			browserCtx = await browser.newContext();
+			ctx.page = await browserCtx.newPage();
+		}
 		ctx.run = async (variant, options) => {
 			const cwd = create({ testId: ctx.task.id, variant });
 
@@ -81,7 +94,9 @@ export function setupTest<Addons extends AddonMap>(addons: Addons) {
 		};
 
 		return async () => {
-			await browserCtx.close();
+			if (withBrowser) {
+				await browserCtx.close();
+			}
 			// ...other tear downs
 		};
 	});

packages/addons/_tests/devtools-json/test.ts

@@ -4,17 +4,11 @@ import devtoolsJson from '../../devtools-json/index.ts';
 import fs from 'node:fs';
 import path from 'node:path';
 
-const { test, variants, prepareServer } = setupTest({ devtoolsJson } as {
-	devtoolsJson?: typeof devtoolsJson;
-});
+const { test, variants } = setupTest({ devtoolsJson }, { browser: false });
 
-test.concurrent.for(variants)('default - %s', async (variant, { page, ...ctx }) => {
+test.concurrent.for(variants)('default - %s', async (variant, ctx) => {
 	const cwd = await ctx.run(variant, { devtoolsJson: {} });
 
-	const { close } = await prepareServer({ cwd, page });
-	// kill server process when we're done
-	ctx.onTestFinished(async () => await close());
-
 	const ext = variant.includes('ts') ? 'ts' : 'js';
 	const viteFile = path.resolve(cwd, `vite.config.${ext}`);
 	const viteContent = fs.readFileSync(viteFile, 'utf8');
@@ -26,25 +20,3 @@ test.concurrent.for(variants)('default - %s', async (variant, { page, ...ctx })
 	// Check if it's called
 	expect(viteContent).toContain(`devtoolsJson()`);
 });
-
-test.concurrent.for(variants)(
-	'without selecting the addon specifically - %s',
-	async (variant, { page, ...ctx }) => {
-		const cwd = await ctx.run(variant, {});
-
-		const { close } = await prepareServer({ cwd, page });
-		// kill server process when we're done
-		ctx.onTestFinished(async () => await close());
-
-		const ext = variant.includes('ts') ? 'ts' : 'js';
-		const viteFile = path.resolve(cwd, `vite.config.${ext}`);
-		const viteContent = fs.readFileSync(viteFile, 'utf8');
-
-		// Check if we have the import part
-		expect(viteContent).toContain(`import devtoolsJson from`);
-		expect(viteContent).toContain(`vite-plugin-devtools-json`);
-
-		// Check if it's called
-		expect(viteContent).toContain(`devtoolsJson()`);
-	}
-);

packages/addons/_tests/drizzle/test.ts

@@ -3,7 +3,7 @@ import path from 'node:path';
 import process from 'node:process';
 import { fileURLToPath } from 'node:url';
 import { execSync } from 'node:child_process';
-import * as vitest from 'vitest';
+import { beforeAll } from 'vitest';
 import { expect } from '@playwright/test';
 import { setupTest } from '../_setup/suite.ts';
 import drizzle from '../../drizzle/index.ts';
@@ -14,7 +14,7 @@ const { test, variants, prepareServer } = setupTest({ drizzle });
 // only linux is supported for running docker containers in github runners
 const noDocker = process.env.CI && process.platform !== 'linux';
 
-vitest.beforeAll(() => {
+beforeAll(() => {
 	if (noDocker) return;
 	const cwd = path.dirname(fileURLToPath(import.meta.url));
 	execSync('docker compose up --detach', { cwd, stdio: 'pipe' });
@@ -64,6 +64,6 @@ test.concurrent.for(testCases)(
 		// kill server process when we're done
 		ctx.onTestFinished(async () => await close());
 
-		expect(await page.$('[data-testid]')).toBeTruthy();
+		expect(page.locator('[data-testid]')).toBeTruthy();
 	}
 );

packages/addons/_tests/eslint/test.ts

@@ -1,25 +1,22 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { execSync } from 'node:child_process';
-import { expect } from '@playwright/test';
 import { setupTest } from '../_setup/suite.ts';
 import eslint from '../../eslint/index.ts';
 
-const { test, variants, prepareServer } = setupTest({ eslint });
+const { test, variants } = setupTest({ eslint }, { browser: false });
 
-test.concurrent.for(variants)('core - %s', async (variant, { page, ...ctx }) => {
+test.concurrent.for(variants)('core - %s', async (variant, { expect, ...ctx }) => {
 	const cwd = await ctx.run(variant, { eslint: {} });
 
-	const { close } = await prepareServer({ cwd, page });
-	// kill server process when we're done
-	ctx.onTestFinished(async () => await close());
-
 	const unlintedFile = 'let foo = "";\nif (Boolean(foo)) {\n//\n}';
 	fs.writeFileSync(path.resolve(cwd, 'src/lib/foo.js'), unlintedFile, 'utf8');
 
-	expect(() => execSync('pnpm lint', { cwd, stdio: 'pipe' })).toThrowError();
+	expect(() => execSync('pnpm install', { cwd, stdio: 'pipe' })).not.toThrow();
+
+	expect(() => execSync('pnpm lint', { cwd, stdio: 'pipe' })).toThrow();
 
-	expect(() => execSync('pnpm eslint --fix .', { cwd, stdio: 'inherit' })).not.toThrowError();
+	expect(() => execSync('pnpm eslint --fix .', { cwd, stdio: 'pipe' })).not.toThrow();
 
-	expect(() => execSync('pnpm lint', { cwd, stdio: 'pipe' })).not.toThrowError();
+	expect(() => execSync('pnpm lint', { cwd, stdio: 'pipe' })).not.toThrow();
 });

packages/addons/_tests/mdsvex/test.ts

@@ -20,9 +20,9 @@ test.concurrent.for(variants)('core - %s', async (variant, { page, ...ctx }) =>
 	// kill server process when we're done
 	ctx.onTestFinished(async () => await close());
 
-	expect(await page.$('.mdsvex h1')).toBeTruthy();
-	expect(await page.$('.mdsvex h2')).toBeTruthy();
-	expect(await page.$('.mdsvex p')).toBeTruthy();
+	expect(page.locator('.mdsvex h1')).toBeTruthy();
+	expect(page.locator('.mdsvex h2')).toBeTruthy();
+	expect(page.locator('.mdsvex p')).toBeTruthy();
 });
 
 function addFixture(cwd: string, variant: string) {

packages/addons/_tests/playwright/test.ts

@@ -1,15 +1,21 @@
-import { expect } from '@playwright/test';
+import fs from 'node:fs';
+import path from 'node:path';
 import { setupTest } from '../_setup/suite.ts';
 import playwright from '../../playwright/index.ts';
 
-const { test, variants, prepareServer } = setupTest({ playwright });
+const { test, variants } = setupTest({ playwright }, { browser: false });
 
-test.concurrent.for(variants)('core - %s', async (variant, { page, ...ctx }) => {
+test.concurrent.for(variants)('core - %s', async (variant, { expect, ...ctx }) => {
 	const cwd = await ctx.run(variant, { playwright: {} });
 
-	const { close } = await prepareServer({ cwd, page });
-	// kill server process when we're done
-	ctx.onTestFinished(async () => await close());
+	const ext = variant.includes('ts') ? 'ts' : 'js';
+	const playwrightConfig = path.resolve(cwd, `playwright.config.${ext}`);
+	const configContent = fs.readFileSync(playwrightConfig, 'utf8');
 
-	expect(true).toBe(true);
+	// Check if we have the imports
+	expect(configContent).toContain(`import { defineConfig } from`);
+	expect(configContent).toContain(`@playwright/test`);
+
+	// Check if it's called
+	expect(configContent).toContain(`export default defineConfig({`);
 });

packages/addons/_tests/prettier/test.ts

@@ -1,25 +1,22 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { execSync } from 'node:child_process';
-import { expect } from '@playwright/test';
 import { setupTest } from '../_setup/suite.ts';
 import prettier from '../../prettier/index.ts';
 
-const { test, variants, prepareServer } = setupTest({ prettier });
+const { test, variants } = setupTest({ prettier }, { browser: false });
 
-test.concurrent.for(variants)('core - %s', async (variant, { page, ...ctx }) => {
+test.concurrent.for(variants)('core - %s', async (variant, { expect, ...ctx }) => {
 	const cwd = await ctx.run(variant, { prettier: {} });
 
-	const { close } = await prepareServer({ cwd, page });
-	// kill server process when we're done
-	ctx.onTestFinished(async () => await close());
-
 	const unformattedFile = 'const foo = "bar"';
 	fs.writeFileSync(path.resolve(cwd, 'src/lib/foo.js'), unformattedFile, 'utf8');
 
-	expect(() => execSync('pnpm lint', { cwd, stdio: 'pipe' })).toThrowError();
+	expect(() => execSync('pnpm install', { cwd, stdio: 'pipe' })).not.toThrow();
+
+	expect(() => execSync('pnpm lint', { cwd, stdio: 'pipe' })).toThrow();
 
-	expect(() => execSync('pnpm format', { cwd, stdio: 'pipe' })).not.toThrowError();
+	expect(() => execSync('pnpm format', { cwd, stdio: 'pipe' })).not.toThrow();
 
-	expect(() => execSync('pnpm lint', { cwd, stdio: 'pipe' })).not.toThrowError();
+	expect(() => execSync('pnpm lint', { cwd, stdio: 'pipe' })).not.toThrow();
 });

packages/addons/_tests/storybook/test.ts

@@ -10,17 +10,18 @@ import eslint from '../../eslint/index.ts';
 const { test, variants, prepareServer } = setupTest({ storybook, eslint });
 
 let port = 6006;
+const CI = Boolean(process.env.CI);
 
 beforeAll(() => {
-	if (process.env.CI) {
+	if (CI) {
 		// prefetch the storybook cli during ci to reduce fetching errors in tests
 		execSync('pnpx create-storybook@latest --version');
 	}
 });
 
 test.for(variants)(
 	'storybook loaded - %s',
-	{ concurrent: !process.env.CI },
+	{ concurrent: !CI },
 	async (variant, { page, ...ctx }) => {
 		const cwd = await ctx.run(variant, { storybook: {}, eslint: {} });
 
@@ -33,7 +34,7 @@ test.for(variants)(
 		// kill server process when we're done
 		ctx.onTestFinished(async () => await close());
 
-		expect(await page.$('main .sb-bar')).toBeTruthy();
-		expect(await page.$('#storybook-preview-wrapper')).toBeTruthy();
+		expect(page.locator('main .sb-bar')).toBeTruthy();
+		expect(page.locator('#storybook-preview-wrapper')).toBeTruthy();
 	}
 );

packages/addons/_tests/sveltekit-adapter/test.ts

@@ -1,6 +1,6 @@
-import { expect } from '@playwright/test';
-import { readFile } from 'node:fs/promises';
 import { join } from 'node:path';
+import { readFile } from 'node:fs/promises';
+import { expect } from '@playwright/test';
 import sveltekitAdapter from '../../sveltekit-adapter/index.ts';
 import { setupTest } from '../_setup/suite.ts';