From 8c48dd1bbd5d99451c499bfe56bb6331fc411601 Mon Sep 17 00:00:00 2001 From: Lionel Henry Date: Mon, 12 May 2025 16:16:41 +0200 Subject: [PATCH 1/4] Add support for github refs when downloading Ark Log output Run Ark to check it's runnable Build for release --- .github/workflows/prevent-repo-references.yml | 35 +++++ extensions/positron-r/scripts/README.md | 68 +++++++++ .../positron-r/scripts/install-kernel.ts | 143 +++++++++++++++++- 3 files changed, 242 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/prevent-repo-references.yml create mode 100644 extensions/positron-r/scripts/README.md diff --git a/.github/workflows/prevent-repo-references.yml b/.github/workflows/prevent-repo-references.yml new file mode 100644 index 000000000000..c82a7bf4278d --- /dev/null +++ b/.github/workflows/prevent-repo-references.yml @@ -0,0 +1,35 @@ +name: Prevent GitHub Repo References in Package.json + +on: + pull_request: + branches: [main, release/*] + paths: + - "extensions/positron-r/package.json" + push: + branches: [main, release/*] + paths: + - "extensions/positron-r/package.json" + +jobs: + check-repo-references: + name: Check for GitHub Repo References in Ark Version + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Check for repo references in package.json + run: | + echo "Checking for GitHub repo references in package.json" + + # Extract the Ark version from package.json using jq + ARK_VERSION=$(jq -r '.positron.binaryDependencies.ark // empty' extensions/positron-r/package.json) + + # Check if the extracted version follows the GitHub reference pattern + if [[ "$ARK_VERSION" =~ ^[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+@[a-zA-Z0-9._\/-]+ ]] then + echo "::error::GitHub repo reference found in extensions/positron-r/package.json: $ARK_VERSION" + echo "GitHub repo references (org/repo@revision format) are only for development and should not be used in main or release branches." + exit 1 + else + echo "No GitHub repo references found in extensions/positron-r/package.json" + fi diff --git a/extensions/positron-r/scripts/README.md b/extensions/positron-r/scripts/README.md new file mode 100644 index 000000000000..b43f80c07d34 --- /dev/null +++ b/extensions/positron-r/scripts/README.md @@ -0,0 +1,68 @@ +# Positron R Extension Scripts + +## `install-kernel.ts` + +This script handles downloading and installing the Ark R kernel, which is used by the Positron R extension to execute R code. + + +### Installation Methods + +#### Release Mode (Production Use) + +- Downloads pre-built binaries from GitHub releases +- Uses a semantic version number like `"0.1.182"` +- Example in package.json: + ```json + "positron": { + "binaryDependencies": { + "ark": "0.1.182" + } + } + ``` + + +#### Local development mode + +For kernel developers working directly on the Ark kernel, the script will check for locally built versions in a sibling `ark` directory before attempting to download or build from source. + +Note that this has precedence over downloading Ark based on the version specified in `package.json` (both release and github references). + + +#### CI development Mode + +- Clones and builds the Ark kernel from source using a GitHub repositoryreference +- Uses the format `"org/repo@branch_or_revision"` +- Examples in package.json: + ```json + "positron": { + "binaryDependencies": { + "ark": "posit-dev/ark@main" // Use the main branch + "ark": "posit-dev/ark@experimental-feature" // Use a feature branch + "ark": "posit-dev/ark@a1b2c3d" // Use a specific commit + "ark": "posit-dev/ark@v0.1.183" // Use a specific tag + } + } + ``` + +The repository reference format (`org/repo@branch_or_revision`) should only be used during development and never be merged into main or release branches. A GitHub Action (`prevent-repo-references.yml`) enforces this restriction by checking pull requests to main and release branches for this pattern. + + +### Authentication + +When accessing GitHub repositories or releases, the script attempts to find a GitHub Personal Access Token (PAT) in the following order: + +1. The `GITHUB_PAT` environment variable +2. The `POSITRON_GITHUB_PAT` environment variable +3. The git config setting `credential.https://api.github.com.token` + +Providing a PAT is recommended to avoid rate limiting and to access private repositories. + + +## `compile-syntax.ts` + +This script compiles TextMate grammar files for syntax highlighting. + + +## `post-install.ts` + +This script performs additional setup steps after the extension is installed. diff --git a/extensions/positron-r/scripts/install-kernel.ts b/extensions/positron-r/scripts/install-kernel.ts index bdf8dffeba78..6d5cb3668e60 100644 --- a/extensions/positron-r/scripts/install-kernel.ts +++ b/extensions/positron-r/scripts/install-kernel.ts @@ -7,6 +7,7 @@ import * as decompress from 'decompress'; import * as fs from 'fs'; import { IncomingMessage } from 'http'; import * as https from 'https'; +import * as os from 'os'; import { platform, arch } from 'os'; import * as path from 'path'; import { promisify } from 'util'; @@ -16,6 +17,7 @@ import { promisify } from 'util'; const readFileAsync = promisify(fs.readFile); const writeFileAsync = promisify(fs.writeFile); const existsAsync = promisify(fs.exists); +const mkdtempAsync = promisify(fs.mkdtemp); // Create a promisified version of https.get. We can't use the built-in promisify // because the callback doesn't follow the promise convention of (error, result). @@ -64,19 +66,29 @@ async function getLocalArkVersion(): Promise { * * @param command The command to execute. * @param stdin Optional stdin to pass to the command. + * @param cwd Optional working directory for the command * @returns A promise that resolves with the stdout and stderr of the command. */ -async function executeCommand(command: string, stdin?: string): - Promise<{ stdout: string; stderr: string }> { +async function executeCommand( + command: string, + stdin?: string, + cwd?: string +): Promise<{ stdout: string; stderr: string }> { const { exec } = require('child_process'); return new Promise((resolve, reject) => { - const process = exec(command, (error: any, stdout: string, stderr: string) => { + const options: { cwd?: string } = {}; + if (cwd) { + options.cwd = cwd; + } + + const process = exec(command, options, (error: any, stdout: string, stderr: string) => { if (error) { reject(error); } else { resolve({ stdout, stderr }); } }); + if (stdin) { process.stdin.write(stdin); process.stdin.end(); @@ -214,6 +226,100 @@ async function downloadAndReplaceArk(version: string, } } +/** + * Downloads and builds Ark from a GitHub repository at a specific branch or revision. + * + * This function supports development workflows by allowing developers to: + * - Test changes from non-released branches + * - Use experimental features not yet in a release + * - Develop against the latest code in a repository + * + * IMPORTANT: This feature is for DEVELOPMENT ONLY and should not be used in + * production environments or merged to main branches. A GitHub Action enforces + * this restriction by blocking PRs with repo references in package.json. + * + * @param ref The GitHub repo reference in the format 'org/repo@branch_or_revision' + * @param githubPat An optional Github Personal Access Token + */ +async function downloadFromGitHubRepository( + ref: string, + githubPat: string | undefined +): Promise { + const { org, repo, revision } = parseGitHubRepoReference(ref); + + console.log(`Downloading and building Ark from GitHub repo: ${org}/${repo} at revision: ${revision}`); + + // Create a temporary directory for cloning the repo + const tempDir = await mkdtempAsync(path.join(os.tmpdir(), 'ark-build-')); + + try { + console.log(`Created temporary build directory: ${tempDir}`); + + // Set up git command with credentials if available + let gitCloneCommand = `git clone https://github.com/${org}/${repo}.git ${tempDir}`; + if (githubPat) { + gitCloneCommand = `git clone https://x-access-token:${githubPat}@github.com/${org}/${repo}.git ${tempDir}`; + } + + // Clone the repository + console.log('Cloning repository...'); + await executeCommand(gitCloneCommand); + + // Checkout the specific revision + console.log(`Checking out revision: ${revision}`); + await executeCommand(`git checkout ${revision}`, undefined, tempDir); + + // Verify that we have a valid Ark repository structure + const cargoTomlPath = path.join(tempDir, 'Cargo.toml'); + if (!await existsAsync(cargoTomlPath)) { + throw new Error(`Invalid Ark repository: Cargo.toml not found at the repository root`); + } + + console.log('Building Ark from source...'); + + const buildOutput = await executeCommand('cargo build --release', undefined, tempDir); + console.log('Ark build stdout:', buildOutput.stdout); + console.log('Ark build stderr:', buildOutput.stderr); + + // Determine the location of the built binary + const kernelName = platform() === 'win32' ? 'ark.exe' : 'ark'; + const binaryPath = path.join(tempDir, 'target', 'release', kernelName); + + // Ensure the binary was built successfully + if (!fs.existsSync(binaryPath)) { + throw new Error(`Failed to build Ark binary at ${binaryPath}`); + } + + // Run the binary and check output. An error will be thrown if this fails. + const { stdout: versionStdout, stderr: versionStderr } = await executeCommand(`${binaryPath}`); + console.log('Ark stdout:', versionStdout); + console.log('Ark stderr:', versionStderr); + + // Create the resources/ark directory if it doesn't exist + const arkDir = path.join('resources', 'ark'); + if (!await existsAsync(arkDir)) { + await fs.promises.mkdir(arkDir, { recursive: true }); + } + + // Copy the binary to the resources directory + await fs.promises.copyFile(binaryPath, path.join(arkDir, kernelName)); + console.log(`Successfully built and installed Ark from ${org}/${repo}@${revision}`); + + // Write the version information to VERSION file + await writeFileAsync(path.join(arkDir, 'VERSION'), ref); + + } catch (err) { + throw new Error(`Error building Ark from GitHub repository: ${err}`); + } finally { + // Clean up the temporary directory + try { + await fs.promises.rm(tempDir, { recursive: true, force: true }); + } catch (err) { + console.warn(`Warning: Failed to clean up temporary directory ${tempDir}: ${err}`); + } + } +} + async function main() { const kernelName = platform() === 'win32' ? 'ark.exe' : 'ark'; @@ -252,6 +358,7 @@ async function main() { console.log(`package.json version: ${packageJsonVersion} `); console.log(`Downloaded ark version: ${localArkVersion ? localArkVersion : 'Not found'} `); + // Skip installation if versions match if (packageJsonVersion === localArkVersion) { console.log('Versions match. No action required.'); return; @@ -293,7 +400,35 @@ async function main() { } } - await downloadAndReplaceArk(packageJsonVersion, githubPat); + // Check if the version is a GitHub repo reference + if (isGitHubRepoReference(packageJsonVersion)) { + await downloadFromGitHubRepository(packageJsonVersion, githubPat); + } else { + await downloadAndReplaceArk(packageJsonVersion, githubPat); + } +} + +/** + * Check if the version string follows the format 'org/repo@branch_or_revision'. + */ +function isGitHubRepoReference(version: string): boolean { + return /^[a-zA-Z0-9_-]+\/[a-zA-Z0-9_-]+@[a-zA-Z0-9._\/-]+$/.test(version); +} + +/** + * Parse a GitHub repo reference in the format 'org/repo@branch_or_revision'. + */ +function parseGitHubRepoReference(reference: string): { org: string; repo: string; revision: string } { + const orgRepoMatch = reference.match(/^([a-zA-Z0-9_-]+)\/([a-zA-Z0-9_-]+)@([a-zA-Z0-9._\/-]+)$/); + if (!orgRepoMatch) { + throw new Error(`Invalid GitHub repo reference: ${reference}`); + } + + return { + org: orgRepoMatch[1], + repo: orgRepoMatch[2], + revision: orgRepoMatch[3] + }; } main().catch((error) => { From e2af8ea7bc95c85530c2dc76adfbfeb955b9ddc9 Mon Sep 17 00:00:00 2001 From: Lionel Henry Date: Thu, 7 Aug 2025 04:47:38 -0400 Subject: [PATCH 2/4] Test against release Ark --- extensions/positron-r/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/positron-r/package.json b/extensions/positron-r/package.json index 5061a50be671..d29c9dc314d3 100644 --- a/extensions/positron-r/package.json +++ b/extensions/positron-r/package.json @@ -966,7 +966,7 @@ }, "positron": { "binaryDependencies": { - "ark": "0.1.201" + "ark": "posit-dev/ark@0.1.201" }, "minimumRVersion": "4.2.0", "minimumRenvVersion": "1.0.9" From f7a592c2d88c08ae2bfe6223193f7312287ffe9b Mon Sep 17 00:00:00 2001 From: Lionel Henry Date: Thu, 7 Aug 2025 12:06:18 -0400 Subject: [PATCH 3/4] Revert "Test against release Ark" This reverts commit 901b38be3529b0a860d12d6da448a5661ae44725. --- extensions/positron-r/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/positron-r/package.json b/extensions/positron-r/package.json index d29c9dc314d3..5061a50be671 100644 --- a/extensions/positron-r/package.json +++ b/extensions/positron-r/package.json @@ -966,7 +966,7 @@ }, "positron": { "binaryDependencies": { - "ark": "posit-dev/ark@0.1.201" + "ark": "0.1.201" }, "minimumRVersion": "4.2.0", "minimumRenvVersion": "1.0.9" From 3615e4e75dfb8b3aec0a0d168fc811546edf7499 Mon Sep 17 00:00:00 2001 From: Lionel Henry Date: Fri, 8 Aug 2025 04:49:28 -0400 Subject: [PATCH 4/4] Also allow on prereleases --- .github/workflows/prevent-repo-references.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/prevent-repo-references.yml b/.github/workflows/prevent-repo-references.yml index c82a7bf4278d..b54ccd6f4f11 100644 --- a/.github/workflows/prevent-repo-references.yml +++ b/.github/workflows/prevent-repo-references.yml @@ -2,11 +2,11 @@ name: Prevent GitHub Repo References in Package.json on: pull_request: - branches: [main, release/*] + branches: [main, release/*, prerelease/*] paths: - "extensions/positron-r/package.json" push: - branches: [main, release/*] + branches: [main, release/*, prerelease/*] paths: - "extensions/positron-r/package.json"