Add support for github refs when downloading Ark

.github/workflows/prevent-repo-references.yml

@@ -0,0 +1,35 @@
+name: Prevent GitHub Repo References in Package.json
+
+on:
+  pull_request:
+    branches: [main, release/*, prerelease/*]
+    paths:
+      - "extensions/positron-r/package.json"
+  push:
+    branches: [main, release/*, prerelease/*]
+    paths:
+      - "extensions/positron-r/package.json"
+
+jobs:
+  check-repo-references:
+    name: Check for GitHub Repo References in Ark Version
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Check for repo references in package.json
+        run: |
+          echo "Checking for GitHub repo references in package.json"
+
+          # Extract the Ark version from package.json using jq
+          ARK_VERSION=$(jq -r '.positron.binaryDependencies.ark // empty' extensions/positron-r/package.json)
+
+          # Check if the extracted version follows the GitHub reference pattern
+          if [[ "$ARK_VERSION" =~ ^[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+@[a-zA-Z0-9._\/-]+ ]] then
+            echo "::error::GitHub repo reference found in extensions/positron-r/package.json: $ARK_VERSION"
+            echo "GitHub repo references (org/repo@revision format) are only for development and should not be used in main or release branches."
+            exit 1
+          else
+            echo "No GitHub repo references found in extensions/positron-r/package.json"
+          fi

extensions/positron-r/scripts/README.md

@@ -0,0 +1,68 @@
+# Positron R Extension Scripts
+
+## `install-kernel.ts`
+
+This script handles downloading and installing the Ark R kernel, which is used by the Positron R extension to execute R code.
+
+
+### Installation Methods
+
+#### Release Mode (Production Use)
+
+- Downloads pre-built binaries from GitHub releases
+- Uses a semantic version number like `"0.1.182"`
+- Example in package.json:
+  ```json
+  "positron": {
+    "binaryDependencies": {
+      "ark": "0.1.182"
+    }
+  }
+  ```
+
+
+#### Local development mode
+
+For kernel developers working directly on the Ark kernel, the script will check for locally built versions in a sibling `ark` directory before attempting to download or build from source.
+
+Note that this has precedence over downloading Ark based on the version specified in `package.json` (both release and github references).
+
+
+#### CI development Mode
+
+- Clones and builds the Ark kernel from source using a GitHub repositoryreference
+- Uses the format `"org/repo@branch_or_revision"`
+- Examples in package.json:
+  ```json
+  "positron": {
+    "binaryDependencies": {
+      "ark": "posit-dev/ark@main"                  // Use the main branch
+      "ark": "posit-dev/ark@experimental-feature"  // Use a feature branch
+      "ark": "posit-dev/ark@a1b2c3d"               // Use a specific commit
+      "ark": "posit-dev/ark@v0.1.183"              // Use a specific tag
+    }
+  }
+  ```
+
+The repository reference format (`org/repo@branch_or_revision`) should only be used during development and never be merged into main or release branches. A GitHub Action (`prevent-repo-references.yml`) enforces this restriction by checking pull requests to main and release branches for this pattern.
+
+
+### Authentication
+
+When accessing GitHub repositories or releases, the script attempts to find a GitHub Personal Access Token (PAT) in the following order:
+
+1. The `GITHUB_PAT` environment variable
+2. The `POSITRON_GITHUB_PAT` environment variable
+3. The git config setting `credential.https://api.github.com.token`
+
+Providing a PAT is recommended to avoid rate limiting and to access private repositories.
+
+
+## `compile-syntax.ts`
+
+This script compiles TextMate grammar files for syntax highlighting.
+
+
+## `post-install.ts`
+
+This script performs additional setup steps after the extension is installed.

extensions/positron-r/scripts/install-kernel.ts

@@ -7,6 +7,7 @@ import * as decompress from 'decompress';
 import * as fs from 'fs';
 import { IncomingMessage } from 'http';
 import * as https from 'https';
+import * as os from 'os';
 import { platform, arch } from 'os';
 import * as path from 'path';
 import { promisify } from 'util';
@@ -16,6 +17,7 @@ import { promisify } from 'util';
 const readFileAsync = promisify(fs.readFile);
 const writeFileAsync = promisify(fs.writeFile);
 const existsAsync = promisify(fs.exists);
+const mkdtempAsync = promisify(fs.mkdtemp);
 
 // Create a promisified version of https.get. We can't use the built-in promisify
 // because the callback doesn't follow the promise convention of (error, result).
@@ -64,19 +66,29 @@ async function getLocalArkVersion(): Promise<string | null> {
  *
  * @param command The command to execute.
  * @param stdin Optional stdin to pass to the command.
+ * @param cwd Optional working directory for the command
  * @returns A promise that resolves with the stdout and stderr of the command.
  */
-async function executeCommand(command: string, stdin?: string):
-	Promise<{ stdout: string; stderr: string }> {
+async function executeCommand(
+	command: string,
+	stdin?: string,
+	cwd?: string
+): Promise<{ stdout: string; stderr: string }> {
 	const { exec } = require('child_process');
 	return new Promise((resolve, reject) => {
-		const process = exec(command, (error: any, stdout: string, stderr: string) => {
+		const options: { cwd?: string } = {};
+		if (cwd) {
+			options.cwd = cwd;
+		}
+
+		const process = exec(command, options, (error: any, stdout: string, stderr: string) => {
 			if (error) {
 				reject(error);
 			} else {
 				resolve({ stdout, stderr });
 			}
 		});
+
 		if (stdin) {
 			process.stdin.write(stdin);
 			process.stdin.end();
@@ -214,6 +226,100 @@ async function downloadAndReplaceArk(version: string,
 	}
 }
 
+/**
+ * Downloads and builds Ark from a GitHub repository at a specific branch or revision.
+ *
+ * This function supports development workflows by allowing developers to:
+ * - Test changes from non-released branches
+ * - Use experimental features not yet in a release
+ * - Develop against the latest code in a repository
+ *
+ * IMPORTANT: This feature is for DEVELOPMENT ONLY and should not be used in
+ * production environments or merged to main branches. A GitHub Action enforces
+ * this restriction by blocking PRs with repo references in package.json.
+ *
+ * @param ref The GitHub repo reference in the format 'org/repo@branch_or_revision'
+ * @param githubPat An optional Github Personal Access Token
+ */
+async function downloadFromGitHubRepository(
+	ref: string,
+	githubPat: string | undefined
+): Promise<void> {
+	const { org, repo, revision } = parseGitHubRepoReference(ref);
+
+	console.log(`Downloading and building Ark from GitHub repo: ${org}/${repo} at revision: ${revision}`);
+
+	// Create a temporary directory for cloning the repo
+	const tempDir = await mkdtempAsync(path.join(os.tmpdir(), 'ark-build-'));
+
+	try {
+		console.log(`Created temporary build directory: ${tempDir}`);
+
+		// Set up git command with credentials if available
+		let gitCloneCommand = `git clone https://github.com/${org}/${repo}.git ${tempDir}`;
+		if (githubPat) {
+			gitCloneCommand = `git clone https://x-access-token:${githubPat}@github.com/${org}/${repo}.git ${tempDir}`;
+		}
+
+		// Clone the repository
+		console.log('Cloning repository...');
+		await executeCommand(gitCloneCommand);
+
+		// Checkout the specific revision
+		console.log(`Checking out revision: ${revision}`);
+		await executeCommand(`git checkout ${revision}`, undefined, tempDir);
+
+		// Verify that we have a valid Ark repository structure
+		const cargoTomlPath = path.join(tempDir, 'Cargo.toml');
+		if (!await existsAsync(cargoTomlPath)) {
+			throw new Error(`Invalid Ark repository: Cargo.toml not found at the repository root`);
+		}
+
+		console.log('Building Ark from source...');
+
+		const buildOutput = await executeCommand('cargo build --release', undefined, tempDir);
+		console.log('Ark build stdout:', buildOutput.stdout);
+		console.log('Ark build stderr:', buildOutput.stderr);
+
+		// Determine the location of the built binary
+		const kernelName = platform() === 'win32' ? 'ark.exe' : 'ark';
+		const binaryPath = path.join(tempDir, 'target', 'release', kernelName);
+
+		// Ensure the binary was built successfully
+		if (!fs.existsSync(binaryPath)) {
+			throw new Error(`Failed to build Ark binary at ${binaryPath}`);
+		}
+
+		// Run the binary and check output. An error will be thrown if this fails.
+		const { stdout: versionStdout, stderr: versionStderr } = await executeCommand(`${binaryPath}`);
+		console.log('Ark stdout:', versionStdout);
+		console.log('Ark stderr:', versionStderr);
+
+		// Create the resources/ark directory if it doesn't exist
+		const arkDir = path.join('resources', 'ark');
+		if (!await existsAsync(arkDir)) {
+			await fs.promises.mkdir(arkDir, { recursive: true });
+		}
+
+		// Copy the binary to the resources directory
+		await fs.promises.copyFile(binaryPath, path.join(arkDir, kernelName));
+		console.log(`Successfully built and installed Ark from ${org}/${repo}@${revision}`);
+
+		// Write the version information to VERSION file
+		await writeFileAsync(path.join(arkDir, 'VERSION'), ref);
+
+	} catch (err) {
+		throw new Error(`Error building Ark from GitHub repository: ${err}`);
+	} finally {
+		// Clean up the temporary directory
+		try {
+			await fs.promises.rm(tempDir, { recursive: true, force: true });
+		} catch (err) {
+			console.warn(`Warning: Failed to clean up temporary directory ${tempDir}: ${err}`);
+		}
+	}
+}
+
 async function main() {
 	const kernelName = platform() === 'win32' ? 'ark.exe' : 'ark';
 
@@ -252,6 +358,7 @@ async function main() {
 	console.log(`package.json version: ${packageJsonVersion} `);
 	console.log(`Downloaded ark version: ${localArkVersion ? localArkVersion : 'Not found'} `);
 
+	// Skip installation if versions match
 	if (packageJsonVersion === localArkVersion) {
 		console.log('Versions match. No action required.');
 		return;
@@ -293,7 +400,35 @@ async function main() {
 		}
 	}
 
-	await downloadAndReplaceArk(packageJsonVersion, githubPat);
+	// Check if the version is a GitHub repo reference
+	if (isGitHubRepoReference(packageJsonVersion)) {
+		await downloadFromGitHubRepository(packageJsonVersion, githubPat);
+	} else {
+		await downloadAndReplaceArk(packageJsonVersion, githubPat);
+	}
+}
+
+/**
+ * Check if the version string follows the format 'org/repo@branch_or_revision'.
+ */
+function isGitHubRepoReference(version: string): boolean {
+	return /^[a-zA-Z0-9_-]+\/[a-zA-Z0-9_-]+@[a-zA-Z0-9._\/-]+$/.test(version);
+}
+
+/**
+ * Parse a GitHub repo reference in the format 'org/repo@branch_or_revision'.
+ */
+function parseGitHubRepoReference(reference: string): { org: string; repo: string; revision: string } {
+	const orgRepoMatch = reference.match(/^([a-zA-Z0-9_-]+)\/([a-zA-Z0-9_-]+)@([a-zA-Z0-9._\/-]+)$/);
+	if (!orgRepoMatch) {
+		throw new Error(`Invalid GitHub repo reference: ${reference}`);
+	}
+
+	return {
+		org: orgRepoMatch[1],
+		repo: orgRepoMatch[2],
+		revision: orgRepoMatch[3]
+	};
 }
 
 main().catch((error) => {