diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 7c6f926dc1..4369a922b6 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -50,7 +50,9 @@ jobs: os: [ubuntu-24.04, macos-14, windows-2025] node-version: ['22'] # Must include the minimum deno version from the `DENO_VERSION_RANGE` constant in `node/bridge.ts`. - deno-version: ['v1.39.0', 'v2.2.4'] + # We're adding v2.4.2 here because it's needed for the upcoming nimble release, so we can test + # those workflows ahead of time before we can update the base version across the board. + deno-version: ['v1.39.0', 'v2.2.4', 'v2.4.2'] include: - os: ubuntu-24.04 # Earliest supported version diff --git a/package-lock.json b/package-lock.json index df5ac5d1e6..0003bb37f5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24365,6 +24365,7 @@ "parse-imports": "^2.2.1", "path-key": "^4.0.0", "semver": "^7.3.8", + "tar": "^7.4.3", "tmp-promise": "^3.0.3", "urlpattern-polyfill": "8.0.2", "uuid": "^11.0.0" @@ -24380,7 +24381,6 @@ "cpy": "^11.1.0", "nock": "^14.0.0", "npm-run-all2": "^6.0.0", - "tar": "^7.0.0", "typescript": "^5.0.0", "vitest": "^3.0.0" }, @@ -24544,7 +24544,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", - "dev": true, "license": "BlueOak-1.0.0", "engines": { "node": ">=18" @@ -24590,7 +24589,6 @@ "version": "7.1.2", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "dev": true, "license": "ISC", "engines": { "node": ">=16 || 14 >=14.17" @@ -24600,7 +24598,6 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.2.tgz", "integrity": "sha512-oG62iEk+CYt5Xj2YqI5Xi9xWUeZhDI8jjQmC5oThVH5JGCTgIjr7ciJDzC7MBzYd//WvR1OTmP5Q38Q8ShQtVA==", - "dev": true, "license": "MIT", "dependencies": { "minipass": "^7.1.2" @@ -24613,7 +24610,6 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==", - "dev": true, "license": "MIT", "bin": { "mkdirp": "dist/cjs/src/bin.js" @@ -24661,7 +24657,6 @@ "version": "7.4.3", "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz", "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==", - "dev": true, "license": "ISC", "dependencies": { "@isaacs/fs-minipass": "^4.0.0", @@ -24797,7 +24792,6 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", - "dev": true, "license": "BlueOak-1.0.0", "engines": { "node": ">=18" diff --git a/packages/edge-bundler/deno/lib/common.ts b/packages/edge-bundler/deno/lib/common.ts index e7ea78efc8..104bedabe5 100644 --- a/packages/edge-bundler/deno/lib/common.ts +++ b/packages/edge-bundler/deno/lib/common.ts @@ -42,7 +42,7 @@ const loadWithRetry = (specifier: string, delay = 1000, maxTry = 3) => { maxTry, }); } catch (error) { - if (isTooManyTries(error as Error)) { + if (error instanceof Error && isTooManyTries(error)) { console.error(`Loading ${specifier} failed after ${maxTry} tries.`); } throw error; diff --git a/packages/edge-bundler/node/bridge.ts b/packages/edge-bundler/node/bridge.ts index 25c4a67835..85fb386d1f 100644 --- a/packages/edge-bundler/node/bridge.ts +++ b/packages/edge-bundler/node/bridge.ts @@ -7,6 +7,7 @@ import pathKey from 'path-key' import semver from 'semver' import { download } from './downloader.js' +import { FeatureFlags } from './feature_flags.js' import { getPathInHome } from './home_path.js' import { getLogger, Logger } from './logger.js' import { getBinaryExtension } from './platform.js' @@ -16,15 +17,18 @@ const DENO_VERSION_FILE = 'version.txt' // When updating DENO_VERSION_RANGE, ensure that the deno version // on the netlify/buildbot build image satisfies this range! // https://github.com/netlify/buildbot/blob/f9c03c9dcb091d6570e9d0778381560d469e78ad/build-image/noble/Dockerfile#L410 -const DENO_VERSION_RANGE = '1.39.0 - 2.2.4' +export const DENO_VERSION_RANGE = '1.39.0 - 2.2.4' -type OnBeforeDownloadHook = () => void | Promise -type OnAfterDownloadHook = (error?: Error) => void | Promise +const NEXT_DENO_VERSION_RANGE = '^2.4.2' -interface DenoOptions { +export type OnBeforeDownloadHook = () => void | Promise +export type OnAfterDownloadHook = (error?: Error) => void | Promise + +export interface DenoOptions { cacheDirectory?: string debug?: boolean denoDir?: string + featureFlags?: FeatureFlags logger?: Logger onAfterDownload?: OnAfterDownloadHook onBeforeDownload?: OnBeforeDownloadHook @@ -32,11 +36,12 @@ interface DenoOptions { versionRange?: string } -interface ProcessRef { +export interface ProcessRef { ps?: ExecaChildProcess } interface RunOptions { + cwd?: string env?: NodeJS.ProcessEnv extendEnv?: boolean pipeOutput?: boolean @@ -45,7 +50,7 @@ interface RunOptions { rejectOnExitCode?: boolean } -class DenoBridge { +export class DenoBridge { cacheDirectory: string currentDownload?: ReturnType debug: boolean @@ -64,7 +69,9 @@ class DenoBridge { this.onAfterDownload = options.onAfterDownload this.onBeforeDownload = options.onBeforeDownload this.useGlobal = options.useGlobal ?? true - this.versionRange = options.versionRange ?? DENO_VERSION_RANGE + this.versionRange = + options.versionRange ?? + (options.featureFlags?.edge_bundler_generate_tarball ? NEXT_DENO_VERSION_RANGE : DENO_VERSION_RANGE) } private async downloadBinary() { @@ -245,11 +252,11 @@ class DenoBridge { // process, awaiting its execution. async run( args: string[], - { env: inputEnv, extendEnv = true, rejectOnExitCode = true, stderr, stdout }: RunOptions = {}, + { cwd, env: inputEnv, extendEnv = true, rejectOnExitCode = true, stderr, stdout }: RunOptions = {}, ) { const { path: binaryPath } = await this.getBinaryPath() const env = this.getEnvironmentVariables(inputEnv) - const options: Options = { env, extendEnv, reject: rejectOnExitCode } + const options: Options = { cwd, env, extendEnv, reject: rejectOnExitCode } return DenoBridge.runWithBinary(binaryPath, args, { options, stderr, stdout }) } @@ -271,6 +278,3 @@ class DenoBridge { } } } - -export { DENO_VERSION_RANGE, DenoBridge } -export type { DenoOptions, OnAfterDownloadHook, OnBeforeDownloadHook, ProcessRef } diff --git a/packages/edge-bundler/node/bundle.ts b/packages/edge-bundler/node/bundle.ts index 522130a704..9f460bebd5 100644 --- a/packages/edge-bundler/node/bundle.ts +++ b/packages/edge-bundler/node/bundle.ts @@ -1,6 +1,7 @@ export enum BundleFormat { ESZIP2 = 'eszip2', JS = 'js', + TARBALL = 'tar', } export interface Bundle { diff --git a/packages/edge-bundler/node/bundler.test.ts b/packages/edge-bundler/node/bundler.test.ts index c73601c1c1..15fc9c6659 100644 --- a/packages/edge-bundler/node/bundler.test.ts +++ b/packages/edge-bundler/node/bundler.test.ts @@ -1,14 +1,16 @@ import { Buffer } from 'buffer' +import { execSync } from 'node:child_process' import { access, readdir, readFile, rm, writeFile } from 'fs/promises' import { join, resolve } from 'path' import process from 'process' import { pathToFileURL } from 'url' +import { lt } from 'semver' import tmp from 'tmp-promise' -import { test, expect, vi } from 'vitest' +import { test, expect, vi, describe } from 'vitest' import { importMapSpecifier } from '../shared/consts.js' -import { runESZIP, useFixture } from '../test/util.js' +import { runESZIP, runTarball, useFixture } from '../test/util.js' import { BundleError } from './bundle_error.js' import { bundle, BundleOptions } from './bundler.js' @@ -48,7 +50,6 @@ test('Produces an ESZIP bundle', async () => { expect(importMapURL).toBe(importMapSpecifier) const bundlePath = join(distPath, bundles[0].asset) - const { func1, func2, func3 } = await runESZIP(bundlePath) expect(func1).toBe('HELLO, JANE DOE!') @@ -499,7 +500,7 @@ test('Loads npm modules from bare specifiers', async () => { const { func1 } = await runESZIP(bundlePath, vendorDirectory.path) expect(func1).toBe( - `JavaScript, APIs${process.cwd()}, Markup${process.cwd()}`, + `JavaScript, APIs${process.cwd()}, Markup${process.cwd()}, TmV0bGlmeQ==`, ) await cleanup() @@ -692,3 +693,98 @@ test('Loads edge functions from the Frameworks API', async () => { await cleanup() }) + +const denoVersion = execSync('deno eval --no-lock "console.log(Deno.version.deno)"').toString() + +describe.skipIf(lt(denoVersion, '2.4.2'))( + 'Produces a tarball bundle', + () => { + test('With only local imports', async () => { + const systemLogger = vi.fn() + const { basePath, cleanup, distPath } = await useFixture('imports_node_builtin', { copyDirectory: true }) + const declarations: Declaration[] = [ + { + function: 'func1', + path: '/func1', + }, + ] + const vendorDirectory = await tmp.dir() + + await bundle([join(basePath, 'netlify/edge-functions')], distPath, declarations, { + basePath, + configPath: join(basePath, '.netlify/edge-functions/config.json'), + featureFlags: { + edge_bundler_generate_tarball: true, + }, + systemLogger, + }) + + expect( + systemLogger.mock.calls.find((call) => call[0] === 'Could not track dependencies in edge function:'), + ).toBeUndefined() + + const expectedOutput = { + func1: 'ok', + } + + const manifestFile = await readFile(resolve(distPath, 'manifest.json'), 'utf8') + const manifest = JSON.parse(manifestFile) + + const tarballPath = join(distPath, manifest.bundles[0].asset) + const tarballResult = await runTarball(tarballPath) + expect(tarballResult).toStrictEqual(expectedOutput) + + const eszipPath = join(distPath, manifest.bundles[1].asset) + const eszipResult = await runESZIP(eszipPath) + expect(eszipResult).toStrictEqual(expectedOutput) + + await cleanup() + await rm(vendorDirectory.path, { force: true, recursive: true }) + }) + + // TODO: https://github.com/denoland/deno/issues/30187 + test.todo('Using npm modules', async () => { + const systemLogger = vi.fn() + const { basePath, cleanup, distPath } = await useFixture('imports_npm_module', { copyDirectory: true }) + const sourceDirectory = join(basePath, 'functions') + const declarations: Declaration[] = [ + { + function: 'func1', + path: '/func1', + }, + ] + const vendorDirectory = await tmp.dir() + + await bundle([sourceDirectory], distPath, declarations, { + basePath, + featureFlags: { + edge_bundler_generate_tarball: true, + }, + importMapPaths: [join(basePath, 'import_map.json')], + vendorDirectory: vendorDirectory.path, + systemLogger, + }) + + expect( + systemLogger.mock.calls.find((call) => call[0] === 'Could not track dependencies in edge function:'), + ).toBeUndefined() + + const expectedOutput = `JavaScript, APIs${process.cwd()}, Markup${process.cwd()}, TmV0bGlmeQ==` + + const manifestFile = await readFile(resolve(distPath, 'manifest.json'), 'utf8') + const manifest = JSON.parse(manifestFile) + + const tarballPath = join(distPath, manifest.bundles[0].asset) + const tarballResult = await runTarball(tarballPath) + expect(tarballResult.func1).toBe(expectedOutput) + + const eszipPath = join(distPath, manifest.bundles[1].asset) + const eszipResult = await runESZIP(eszipPath, vendorDirectory.path) + expect(eszipResult.func1).toBe(expectedOutput) + + await cleanup() + await rm(vendorDirectory.path, { force: true, recursive: true }) + }) + }, + 10_000, +) diff --git a/packages/edge-bundler/node/bundler.ts b/packages/edge-bundler/node/bundler.ts index 6de583622c..20e309ec37 100644 --- a/packages/edge-bundler/node/bundler.ts +++ b/packages/edge-bundler/node/bundler.ts @@ -15,6 +15,7 @@ import { EdgeFunction } from './edge_function.js' import { FeatureFlags, getFlags } from './feature_flags.js' import { findFunctions } from './finder.js' import { bundle as bundleESZIP } from './formats/eszip.js' +import { bundle as bundleTarball } from './formats/tarball.js' import { ImportMap } from './import_map.js' import { getLogger, LogFunction, Logger } from './logger.js' import { writeManifest } from './manifest.js' @@ -66,6 +67,7 @@ export const bundle = async ( const options: DenoOptions = { debug, cacheDirectory, + featureFlags, logger, onAfterDownload, onBeforeDownload, @@ -114,27 +116,47 @@ export const bundle = async ( vendorDirectory, }) + const bundles: Bundle[] = [] + + if (featureFlags.edge_bundler_generate_tarball) { + bundles.push( + await bundleTarball({ + basePath, + buildID, + debug, + deno, + distDirectory, + functions, + featureFlags, + importMap: importMap.clone(), + vendorDirectory: vendor?.directory, + }), + ) + } + if (vendor) { importMap.add(vendor.importMap) } - const functionBundle = await bundleESZIP({ - basePath, - buildID, - debug, - deno, - distDirectory, - externals, - functions, - featureFlags, - importMap, - vendorDirectory: vendor?.directory, - }) + bundles.push( + await bundleESZIP({ + basePath, + buildID, + debug, + deno, + distDirectory, + externals, + functions, + featureFlags, + importMap, + vendorDirectory: vendor?.directory, + }), + ) // The final file name of the bundles contains a SHA256 hash of the contents, // which we can only compute now that the files have been generated. So let's // rename the bundles to their permanent names. - await createFinalBundles([functionBundle], distDirectory, buildID) + await createFinalBundles(bundles, distDirectory, buildID) // Retrieving a configuration object for each function. // Run `getFunctionConfig` in parallel as it is a non-trivial operation and spins up deno @@ -165,7 +187,7 @@ export const bundle = async ( }) const manifest = await writeManifest({ - bundles: [functionBundle], + bundles, declarations, distDirectory, featureFlags, diff --git a/packages/edge-bundler/node/feature_flags.ts b/packages/edge-bundler/node/feature_flags.ts index bbb1af7173..eb27ce1528 100644 --- a/packages/edge-bundler/node/feature_flags.ts +++ b/packages/edge-bundler/node/feature_flags.ts @@ -1,4 +1,6 @@ -const defaultFlags = {} +const defaultFlags = { + edge_bundler_generate_tarball: false, +} type FeatureFlag = keyof typeof defaultFlags type FeatureFlags = Partial> diff --git a/packages/edge-bundler/node/formats/tarball.ts b/packages/edge-bundler/node/formats/tarball.ts new file mode 100644 index 0000000000..20173fce76 --- /dev/null +++ b/packages/edge-bundler/node/formats/tarball.ts @@ -0,0 +1,157 @@ +import { promises as fs } from 'fs' +import path from 'path' + +import commonPathPrefix from 'common-path-prefix' +import * as tar from 'tar' +import tmp from 'tmp-promise' + +import { DenoBridge } from '../bridge.js' +import { Bundle, BundleFormat } from '../bundle.js' +import { EdgeFunction } from '../edge_function.js' +import { FeatureFlags } from '../feature_flags.js' +import { ImportMap } from '../import_map.js' +import { getDirectoryHash, getStringHash } from '../utils/sha256.js' + +const TARBALL_EXTENSION = '.tar' + +interface Manifest { + functions: Record + version: number +} + +interface BundleTarballOptions { + basePath: string + buildID: string + debug?: boolean + deno: DenoBridge + distDirectory: string + featureFlags: FeatureFlags + functions: EdgeFunction[] + importMap: ImportMap + vendorDirectory?: string +} + +const getUnixPath = (input: string) => input.split(path.sep).join('/') + +export const bundle = async ({ + basePath, + buildID, + deno, + distDirectory, + functions, + importMap, + vendorDirectory, +}: BundleTarballOptions): Promise => { + const sideFilesDir = await tmp.dir({ unsafeCleanup: true }) + const cleanup = [sideFilesDir.cleanup] + + let denoDir = vendorDirectory ? path.join(vendorDirectory, 'deno_dir') : undefined + + if (!denoDir) { + const tmpDir = await tmp.dir({ unsafeCleanup: true }) + + denoDir = tmpDir.path + + cleanup.push(tmpDir.cleanup) + } + + const manifest: Manifest = { + functions: {}, + version: 1, + } + const entryPoints = functions.map((func) => func.path) + + // `deno bundle` does not return the paths of the files it emits, so we have + // to infer them. When multiple entry points are supplied, it will find the + // common path prefix and use that as the base directory in `outdir`. When + // using a single entry point, `commonPathPrefix` returns an empty string, + // so we use the path of the first entry point's directory. + const commonPath = commonPathPrefix(entryPoints) || path.dirname(entryPoints[0]) + + for (const func of functions) { + const relativePath = path.relative(commonPath, func.path) + const bundledPath = path.format({ + ...path.parse(relativePath), + base: undefined, + ext: '.js', + }) + + manifest.functions[func.name] = getUnixPath(bundledPath) + } + + await deno.run( + [ + 'bundle', + '--import-map', + importMap.withNodeBuiltins().toDataURL(), + '--quiet', + '--code-splitting', + '--outdir', + distDirectory, + ...functions.map((func) => func.path), + ], + { + cwd: basePath, + }, + ) + + const manifestPath = path.join(sideFilesDir.path, 'manifest.json') + const manifestContents = JSON.stringify(manifest) + await fs.writeFile(manifestPath, manifestContents) + + const denoConfigPath = path.join(sideFilesDir.path, 'deno.json') + const denoConfigContents = JSON.stringify(importMap.getContentsWithRelativePaths()) + await fs.writeFile(denoConfigPath, denoConfigContents) + + const rootLevel = await fs.readdir(distDirectory) + const hash = await getDirectoryHash(distDirectory) + const tarballPath = path.join(distDirectory, buildID + TARBALL_EXTENSION) + + await fs.mkdir(path.dirname(tarballPath), { recursive: true }) + + // Adding all the bundled files. + await tar.create( + { + cwd: distDirectory, + file: tarballPath, + onWriteEntry(entry) { + entry.path = getUnixPath(`./${entry.path}`) + }, + }, + rootLevel, + ) + + // Adding `deno.json`. + await tar.update( + { + cwd: distDirectory, + file: tarballPath, + onWriteEntry(entry) { + entry.path = './deno.json' + }, + }, + [denoConfigPath], + ) + + // Adding the manifest file. + await tar.update( + { + cwd: distDirectory, + file: tarballPath, + onWriteEntry(entry) { + entry.path = './___netlify-edge-functions.json' + }, + }, + [manifestPath], + ) + + await Promise.all(cleanup) + + const finalHash = [hash, getStringHash(manifestContents), getStringHash(denoConfigContents)].join('') + + return { + extension: TARBALL_EXTENSION, + format: BundleFormat.TARBALL, + hash: finalHash, + } +} diff --git a/packages/edge-bundler/node/import_map.ts b/packages/edge-bundler/node/import_map.ts index c3cabfe75d..10dc64776f 100644 --- a/packages/edge-bundler/node/import_map.ts +++ b/packages/edge-bundler/node/import_map.ts @@ -1,7 +1,8 @@ -import { Buffer } from 'buffer' -import { promises as fs } from 'fs' -import { dirname, relative } from 'path' -import { fileURLToPath, pathToFileURL } from 'url' +import { Buffer } from 'node:buffer' +import { promises as fs } from 'node:fs' +import { builtinModules } from 'node:module' +import { dirname, relative } from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' import { parse, ParsedImportMap } from '@import-maps/resolve' @@ -200,6 +201,29 @@ export class ImportMap { } } + getContentsWithRelativePaths() { + let imports: Imports = {} + let scopes: Record = {} + + this.sources.forEach((file) => { + imports = { ...imports, ...file.imports } + scopes = { ...scopes, ...file.scopes } + }) + + // Internal imports must come last, because we need to guarantee that + // `netlify:edge` isn't user-defined. + Object.entries(INTERNAL_IMPORTS).forEach((internalImport) => { + const [specifier, url] = internalImport + + imports[specifier] = url + }) + + return { + imports, + scopes, + } + } + // The same as `getContents`, but the URLs are represented as URL objects // instead of strings. This is compatible with the `ParsedImportMap` type // from the `@import-maps/resolve` library. @@ -256,6 +280,23 @@ export class ImportMap { return `data:application/json;base64,${encodedImportMap}` } + // Adds an import map source mapping Node.js built-in modules to their prefixed + // version (e.g. "path" => "node:path"). + withNodeBuiltins() { + const imports: Record = {} + + for (const name of builtinModules) { + imports[name] = `node:${name}` + } + + this.sources.push({ + baseURL: new URL(import.meta.url), + imports, + }) + + return this + } + async writeToFile(path: string) { const distDirectory = dirname(path) diff --git a/packages/edge-bundler/node/npm_dependencies.ts b/packages/edge-bundler/node/npm_dependencies.ts index 8d016c4a95..4c72a927f0 100644 --- a/packages/edge-bundler/node/npm_dependencies.ts +++ b/packages/edge-bundler/node/npm_dependencies.ts @@ -12,8 +12,7 @@ import tmp from 'tmp-promise' import { ImportMap } from './import_map.js' import { Logger } from './logger.js' import { pathsBetween } from './utils/fs.js' - -const TYPESCRIPT_EXTENSIONS = new Set(['.ts', '.tsx', '.cts', '.ctsx', '.mts', '.mtsx']) +import { TYPESCRIPT_EXTENSIONS } from './utils/typescript.js' const slugifyFileName = (specifier: string) => { return specifier.replace(/\//g, '_') diff --git a/packages/edge-bundler/node/utils/sha256.ts b/packages/edge-bundler/node/utils/sha256.ts index 03c835e64d..7c27316ba8 100644 --- a/packages/edge-bundler/node/utils/sha256.ts +++ b/packages/edge-bundler/node/utils/sha256.ts @@ -1,13 +1,37 @@ -import crypto from 'crypto' -import fs from 'fs' +import crypto from 'node:crypto' +import { createReadStream, promises as fs } from 'node:fs' +import path from 'node:path' -const getFileHash = (path: string): Promise => { +export const getDirectoryHash = async (dirPath: string): Promise => { + const entries: string[] = [] + + async function walk(currentPath: string): Promise { + const dirents = await fs.readdir(currentPath, { withFileTypes: true }) + for (const dirent of dirents) { + const fullPath = path.join(currentPath, dirent.name) + const relativePath = path.relative(dirPath, fullPath) + + if (dirent.isDirectory()) { + await walk(fullPath) + } else if (dirent.isFile() || dirent.isSymbolicLink()) { + const fileHash = await getFileHash(fullPath) + entries.push(`${relativePath}:${fileHash}`) + } + } + } + + await walk(dirPath) + + return getStringHash(entries.sort((a, b) => a.localeCompare(b)).join('\n')) +} + +export const getFileHash = (path: string): Promise => { const hash = crypto.createHash('sha256') hash.setEncoding('hex') return new Promise((resolve, reject) => { - const file = fs.createReadStream(path) + const file = createReadStream(path) file.on('end', () => { hash.end() @@ -20,4 +44,11 @@ const getFileHash = (path: string): Promise => { }) } -export { getFileHash } +export const getStringHash = (input: string) => { + const hash = crypto.createHash('sha256') + + hash.setEncoding('hex') + hash.update(input) + + return hash.digest('hex') +} diff --git a/packages/edge-bundler/node/utils/typescript.ts b/packages/edge-bundler/node/utils/typescript.ts new file mode 100644 index 0000000000..be8bf82295 --- /dev/null +++ b/packages/edge-bundler/node/utils/typescript.ts @@ -0,0 +1 @@ +export const TYPESCRIPT_EXTENSIONS = new Set(['.ts', '.tsx', '.cts', '.ctsx', '.mts', '.mtsx']) diff --git a/packages/edge-bundler/package.json b/packages/edge-bundler/package.json index 25bb7b1751..263a3ced52 100644 --- a/packages/edge-bundler/package.json +++ b/packages/edge-bundler/package.json @@ -52,7 +52,6 @@ "cpy": "^11.1.0", "nock": "^14.0.0", "npm-run-all2": "^6.0.0", - "tar": "^7.0.0", "typescript": "^5.0.0", "vitest": "^3.0.0" }, @@ -78,6 +77,7 @@ "parse-imports": "^2.2.1", "path-key": "^4.0.0", "semver": "^7.3.8", + "tar": "^7.4.3", "tmp-promise": "^3.0.3", "urlpattern-polyfill": "8.0.2", "uuid": "^11.0.0" diff --git a/packages/edge-bundler/test/fixtures/imports_npm_module/functions/func1.ts b/packages/edge-bundler/test/fixtures/imports_npm_module/functions/func1.ts index cb73fdbf0d..e673b6dea1 100644 --- a/packages/edge-bundler/test/fixtures/imports_npm_module/functions/func1.ts +++ b/packages/edge-bundler/test/fixtures/imports_npm_module/functions/func1.ts @@ -1,14 +1,10 @@ import parent1 from 'parent-1' -import parent3 from './lib/util.ts' +import parent3 from './helpers/util.ts' import { echo, parent2 } from 'alias:helper' -import { HTMLRewriter } from 'html-rewriter' - -await Promise.resolve() - -new HTMLRewriter() +import { encode as base64Encode } from "https://deno.land/std@0.194.0/encoding/base64.ts"; export default async () => { - const text = [parent1('JavaScript'), parent2('APIs'), parent3('Markup')].join(', ') + const text = [parent1('JavaScript'), parent2('APIs'), parent3('Markup'), base64Encode("Netlify")].join(', ') return new Response(echo(text)) } diff --git a/packages/edge-bundler/test/fixtures/imports_npm_module/functions/lib/util.ts b/packages/edge-bundler/test/fixtures/imports_npm_module/functions/helpers/util.ts similarity index 100% rename from packages/edge-bundler/test/fixtures/imports_npm_module/functions/lib/util.ts rename to packages/edge-bundler/test/fixtures/imports_npm_module/functions/helpers/util.ts diff --git a/packages/edge-bundler/test/fixtures/imports_npm_module/import_map.json b/packages/edge-bundler/test/fixtures/imports_npm_module/import_map.json index 5ef2121b01..a72e1a7d9f 100644 --- a/packages/edge-bundler/test/fixtures/imports_npm_module/import_map.json +++ b/packages/edge-bundler/test/fixtures/imports_npm_module/import_map.json @@ -1,6 +1,5 @@ { "imports": { - "alias:helper": "./helper.ts", - "html-rewriter": "https://ghuc.cc/worker-tools/html-rewriter/index.ts" + "alias:helper": "./helper.ts" } } diff --git a/packages/edge-bundler/test/fixtures/imports_npm_module/node_modules/parent-4/index.js b/packages/edge-bundler/test/fixtures/imports_npm_module/node_modules/parent-4/index.js new file mode 100644 index 0000000000..35a9b77a4d --- /dev/null +++ b/packages/edge-bundler/test/fixtures/imports_npm_module/node_modules/parent-4/index.js @@ -0,0 +1,4 @@ +export default () => { + throw new Error("I should not be loaded") +} + diff --git a/packages/edge-bundler/test/fixtures/imports_npm_module/node_modules/parent-4/package.json b/packages/edge-bundler/test/fixtures/imports_npm_module/node_modules/parent-4/package.json new file mode 100644 index 0000000000..483924df90 --- /dev/null +++ b/packages/edge-bundler/test/fixtures/imports_npm_module/node_modules/parent-4/package.json @@ -0,0 +1,5 @@ +{ + "name": "parent-4", + "version": "1.0.0", + "main": "index.js" +} \ No newline at end of file diff --git a/packages/edge-bundler/test/fixtures/imports_npm_module/package.json b/packages/edge-bundler/test/fixtures/imports_npm_module/package.json index 3dbc1ca591..3541812a7b 100644 --- a/packages/edge-bundler/test/fixtures/imports_npm_module/package.json +++ b/packages/edge-bundler/test/fixtures/imports_npm_module/package.json @@ -1,3 +1,7 @@ { - "type": "module" + "type": "module", + "dependencies": { + "parent-1": "1.0.0", + "parent-3": "1.0.0" + } } diff --git a/packages/edge-bundler/test/util.ts b/packages/edge-bundler/test/util.ts index 12c71b7d91..0b54dc55d1 100644 --- a/packages/edge-bundler/test/util.ts +++ b/packages/edge-bundler/test/util.ts @@ -3,33 +3,52 @@ import { join, resolve } from 'path' import { stderr, stdout } from 'process' import { fileURLToPath, pathToFileURL } from 'url' +import cpy from 'cpy' import { execa } from 'execa' +import * as tar from 'tar' import tmp from 'tmp-promise' import { getLogger } from '../node/logger.js' import type { Manifest } from '../node/manifest.js' -const testLogger = getLogger(() => { +export const testLogger = getLogger(() => { // no-op }) const url = new URL(import.meta.url) const dirname = fileURLToPath(url) -const fixturesDir = resolve(dirname, '..', 'fixtures') +export const fixturesDir = resolve(dirname, '..', 'fixtures') -const useFixture = async (fixtureName: string) => { - const tmpDir = await tmp.dir({ unsafeCleanup: true }) +interface UseFixtureOptions { + copyDirectory?: boolean +} + +export const useFixture = async (fixtureName: string, { copyDirectory }: UseFixtureOptions = {}) => { + const tmpDistDir = await tmp.dir({ unsafeCleanup: true }) const fixtureDir = resolve(fixturesDir, fixtureName) - const distPath = join(tmpDir.path, '.netlify', 'edge-functions-dist') + const distPath = join(tmpDistDir.path, '.netlify', 'edge-functions-dist') + + if (copyDirectory) { + const tmpFixtureDir = await tmp.dir({ unsafeCleanup: true }) + + // TODO: Replace with `fs.cp` once the Node.js version range allows. + await cpy(`${fixtureDir}/**`, tmpFixtureDir.path) + + return { + basePath: tmpFixtureDir.path, + cleanup: () => Promise.allSettled([tmpDistDir.cleanup, tmpFixtureDir.cleanup]), + distPath, + } + } return { basePath: fixtureDir, - cleanup: tmpDir.cleanup, + cleanup: tmpDistDir.cleanup, distPath, } } -const inspectFunction = (path: string) => ` +const inspectESZIPFunction = (path: string) => ` import { functions } from "${pathToFileURL(path)}.js"; const responses = {}; @@ -44,7 +63,26 @@ const inspectFunction = (path: string) => ` console.log(JSON.stringify(responses)); ` -const getRouteMatcher = (manifest: Manifest) => (candidate: string) => +const inspectTarballFunction = () => ` +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import manifest from "./___netlify-edge-functions.json" with { type: "json" }; + +const responses = {}; + +for (const functionName in manifest.functions) { + const req = new Request("https://test.netlify"); + const entrypoint = path.resolve(manifest.functions[functionName]); + const func = await import(pathToFileURL(entrypoint)) + const res = await func.default(req); + + responses[functionName] = await res.text(); +} + +console.log(JSON.stringify(responses)); +` + +export const getRouteMatcher = (manifest: Manifest) => (candidate: string) => manifest.routes.find((route) => { const regex = new RegExp(route.pattern) @@ -62,13 +100,14 @@ const getRouteMatcher = (manifest: Manifest) => (candidate: string) => return !isExcluded }) -const runESZIP = async (eszipPath: string, vendorDirectory?: string) => { +export const runESZIP = async (eszipPath: string, vendorDirectory?: string) => { const tmpDir = await tmp.dir({ unsafeCleanup: true }) // Extract ESZIP into temporary directory. const extractCommand = execa('deno', [ 'run', '--allow-all', + '--no-lock', 'https://deno.land/x/eszip@v0.55.2/eszip.ts', 'x', eszipPath, @@ -99,7 +138,7 @@ const runESZIP = async (eszipPath: string, vendorDirectory?: string) => { await fs.rename(stage2Path, `${stage2Path}.js`) // Run function that imports the extracted stage 2 and invokes each function. - const evalCommand = execa('deno', ['eval', '--import-map', importMapPath, inspectFunction(stage2Path)]) + const evalCommand = execa('deno', ['eval', '--import-map', importMapPath, inspectESZIPFunction(stage2Path)]) evalCommand.stderr?.pipe(stderr) @@ -110,4 +149,23 @@ const runESZIP = async (eszipPath: string, vendorDirectory?: string) => { return JSON.parse(result.stdout) } -export { fixturesDir, getRouteMatcher, testLogger, runESZIP, useFixture } +export const runTarball = async (tarballPath: string) => { + const tmpDir = await tmp.dir({ unsafeCleanup: true }) + + await tar.extract({ + cwd: tmpDir.path, + file: tarballPath, + }) + + const evalCommand = execa('deno', ['eval', inspectTarballFunction()], { + cwd: tmpDir.path, + }) + + evalCommand.stderr?.pipe(stderr) + + const result = await evalCommand + + await tmpDir.cleanup() + + return JSON.parse(result.stdout) +}