diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11c0f08deb..73c5936cd9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,35 +147,48 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 256 # get a bit more of the history - - name: install josh-proxy - run: cargo +stable install josh-proxy --git https://github.com/josh-project/josh --tag r24.10.04 + - name: install josh-sync + run: cargo +stable install --locked --git https://github.com/rust-lang/josh-sync - name: setup bot git name and email run: | git config --global user.name 'The Miri Cronjob Bot' git config --global user.email 'miri@cron.bot' - name: Install nightly toolchain run: rustup toolchain install nightly --profile minimal - - name: get changes from rustc - run: ./miri rustc-pull - name: Install rustup-toolchain-install-master run: cargo install -f rustup-toolchain-install-master - - name: format changes (if any) + - name: Push changes to a branch and create PR run: | + # Make it easier to see what happens. + set -x + # Temporarily disable early exit to examine the status code of rustc-josh-sync + set +e + rustc-josh-sync pull + exitcode=$? + set -e + + # If there were no changes to pull, rustc-josh-sync returns status code 2. + # In that case, skip the rest of the job. + if [ $exitcode -eq 2 ]; then + echo "Nothing changed in rustc, skipping PR" + exit 0 + elif [ $exitcode -ne 0 ]; then + # If return code was not 0 or 2, rustc-josh-sync actually failed + echo "error: rustc-josh-sync failed" + exit ${exitcode} + fi + + # Format changes ./miri toolchain ./miri fmt --check || (./miri fmt && git commit -am "fmt") - - name: Push changes to a branch and create PR - run: | - # `git diff --exit-code` "succeeds" if the diff is empty. - if git diff --exit-code HEAD^; then echo "Nothing changed in rustc, skipping PR"; exit 0; fi - # The diff is non-empty, create a PR. + + # Create a PR BRANCH="rustup-$(date -u +%Y-%m-%d)" git switch -c $BRANCH git push -u origin $BRANCH gh pr create -B master --title 'Automatic Rustup' --body 'Please close and re-open this PR to trigger CI, then enable auto-merge.' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ZULIP_BOT_EMAIL: ${{ secrets.ZULIP_BOT_EMAIL }} - ZULIP_API_TOKEN: ${{ secrets.ZULIP_API_TOKEN }} cron-fail-notify: name: cronjob failure notification diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 637c0dd2fd..7d78fdddba 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -297,14 +297,14 @@ You can also directly run Miri on a Rust source file: ## Advanced topic: Syncing with the rustc repo -We use the [`josh` proxy](https://github.com/josh-project/josh) to transmit changes between the +We use the [`josh-sync`](https://github.com/rust-lang/josh-sync) tool to transmit changes between the rustc and Miri repositories. You can install it as follows: ```sh -cargo +stable install josh-proxy --git https://github.com/josh-project/josh --tag r24.10.04 +cargo install --locked --git https://github.com/rust-lang/josh-sync ``` -Josh will automatically be started and stopped by `./miri`. +The commands below will automatically install and manage the [Josh](https://github.com/josh-project/josh) proxy that performs the actual work. ### Importing changes from the rustc repo @@ -312,10 +312,12 @@ Josh will automatically be started and stopped by `./miri`. We assume we start on an up-to-date master branch in the Miri repo. +1) First, create a branch for the pull, e.g. `git checkout -b rustup` +2) Then run the following: ```sh # Fetch and merge rustc side of the history. Takes ca 5 min the first time. # This will also update the `rustc-version` file. -./miri rustc-pull +rustc-josh-sync pull # Update local toolchain and apply formatting. ./miri toolchain && ./miri fmt git commit -am "rustup" @@ -328,12 +330,12 @@ needed. ### Exporting changes to the rustc repo -We will use the josh proxy to push to your fork of rustc. Run the following in the Miri repo, +We will use the `josh-sync` tool to push to your fork of rustc. Run the following in the Miri repo, assuming we are on an up-to-date master branch: ```sh # Push the Miri changes to your rustc fork (substitute your github handle for YOUR_NAME). -./miri rustc-push YOUR_NAME miri +rustc-josh-sync push miri YOUR_NAME ``` This will create a new branch called `miri` in your fork, and the output should include a link that diff --git a/josh-sync.toml b/josh-sync.toml new file mode 100644 index 0000000000..86208b3742 --- /dev/null +++ b/josh-sync.toml @@ -0,0 +1,2 @@ +repo = "miri" +filter = ":rev(75dd959a3a40eb5b4574f8d2e23aa6efbeb33573:prefix=src/tools/miri):/src/tools/miri" diff --git a/miri-script/Cargo.lock b/miri-script/Cargo.lock index a049bfcbcc..044a678869 100644 --- a/miri-script/Cargo.lock +++ b/miri-script/Cargo.lock @@ -116,27 +116,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" -[[package]] -name = "directories" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d" -dependencies = [ - "dirs-sys", -] - -[[package]] -name = "dirs-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" -dependencies = [ - "libc", - "option-ext", - "redox_users", - "windows-sys 0.60.2", -] - [[package]] name = "dunce" version = "1.0.5" @@ -165,17 +144,6 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" -[[package]] -name = "getrandom" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" -dependencies = [ - "cfg-if", - "libc", - "wasi 0.11.1+wasi-snapshot-preview1", -] - [[package]] name = "getrandom" version = "0.3.3" @@ -185,7 +153,7 @@ dependencies = [ "cfg-if", "libc", "r-efi", - "wasi 0.14.2+wasi-0.2.4", + "wasi", ] [[package]] @@ -221,16 +189,6 @@ version = "0.2.174" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" -[[package]] -name = "libredox" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1580801010e535496706ba011c15f8532df6b42297d2e471fec38ceadd8c0638" -dependencies = [ - "bitflags", - "libc", -] - [[package]] name = "linux-raw-sys" version = "0.9.4" @@ -249,7 +207,6 @@ version = "0.1.0" dependencies = [ "anyhow", "clap", - "directories", "dunce", "itertools", "path_macro", @@ -275,12 +232,6 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" -[[package]] -name = "option-ext" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" - [[package]] name = "path_macro" version = "1.0.0" @@ -311,17 +262,6 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" -[[package]] -name = "redox_users" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd6f9d3d47bdd2ad6945c5015a226ec6155d0bcdfd8f7cd29f86b71f8de99d2b" -dependencies = [ - "getrandom 0.2.16", - "libredox", - "thiserror", -] - [[package]] name = "rustc_version" version = "0.4.1" @@ -427,32 +367,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" dependencies = [ "fastrand", - "getrandom 0.3.3", + "getrandom", "once_cell", "rustix", "windows-sys 0.59.0", ] -[[package]] -name = "thiserror" -version = "2.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "unicode-ident" version = "1.0.18" @@ -475,12 +395,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - [[package]] name = "wasi" version = "0.14.2+wasi-0.2.4" diff --git a/miri-script/Cargo.toml b/miri-script/Cargo.toml index b3f82cd1d5..39858880e8 100644 --- a/miri-script/Cargo.toml +++ b/miri-script/Cargo.toml @@ -22,7 +22,6 @@ anyhow = "1.0" xshell = "0.2.6" rustc_version = "0.4" dunce = "1.0.4" -directories = "6" serde = "1" serde_json = "1" serde_derive = "1" diff --git a/miri-script/src/commands.rs b/miri-script/src/commands.rs index 9aaad9ca04..017e461cd8 100644 --- a/miri-script/src/commands.rs +++ b/miri-script/src/commands.rs @@ -2,11 +2,9 @@ use std::collections::BTreeMap; use std::ffi::{OsStr, OsString}; use std::fmt::Write as _; use std::fs::{self, File}; -use std::io::{self, BufRead, BufReader, BufWriter, Write as _}; -use std::ops::Not; +use std::io::{self, BufRead, BufReader, BufWriter}; use std::path::PathBuf; -use std::time::Duration; -use std::{env, net, process}; +use std::{env, process}; use anyhow::{Context, Result, anyhow, bail}; use path_macro::path; @@ -18,11 +16,6 @@ use xshell::{Shell, cmd}; use crate::Command; use crate::util::*; -/// Used for rustc syncs. -const JOSH_FILTER: &str = - ":rev(75dd959a3a40eb5b4574f8d2e23aa6efbeb33573:prefix=src/tools/miri):/src/tools/miri"; -const JOSH_PORT: u16 = 42042; - impl MiriEnv { /// Prepares the environment: builds miri and cargo-miri and a sysroot. /// Returns the location of the sysroot. @@ -99,66 +92,6 @@ impl Command { Ok(()) } - fn start_josh() -> Result { - // Determine cache directory. - let local_dir = { - let user_dirs = - directories::ProjectDirs::from("org", "rust-lang", "miri-josh").unwrap(); - user_dirs.cache_dir().to_owned() - }; - - // Start josh, silencing its output. - let mut cmd = process::Command::new("josh-proxy"); - cmd.arg("--local").arg(local_dir); - cmd.arg("--remote").arg("https://github.com"); - cmd.arg("--port").arg(JOSH_PORT.to_string()); - cmd.arg("--no-background"); - cmd.stdout(process::Stdio::null()); - cmd.stderr(process::Stdio::null()); - let josh = cmd.spawn().context("failed to start josh-proxy, make sure it is installed")?; - - // Create a wrapper that stops it on drop. - struct Josh(process::Child); - impl Drop for Josh { - fn drop(&mut self) { - #[cfg(unix)] - { - // Try to gracefully shut it down. - process::Command::new("kill") - .args(["-s", "INT", &self.0.id().to_string()]) - .output() - .expect("failed to SIGINT josh-proxy"); - // Sadly there is no "wait with timeout"... so we just give it some time to finish. - std::thread::sleep(Duration::from_millis(100)); - // Now hopefully it is gone. - if self.0.try_wait().expect("failed to wait for josh-proxy").is_some() { - return; - } - } - // If that didn't work (or we're not on Unix), kill it hard. - eprintln!( - "I have to kill josh-proxy the hard way, let's hope this does not break anything." - ); - self.0.kill().expect("failed to SIGKILL josh-proxy"); - } - } - - // Wait until the port is open. We try every 10ms until 1s passed. - for _ in 0..100 { - // This will generally fail immediately when the port is still closed. - let josh_ready = net::TcpStream::connect_timeout( - &net::SocketAddr::from(([127, 0, 0, 1], JOSH_PORT)), - Duration::from_millis(1), - ); - if josh_ready.is_ok() { - return Ok(Josh(josh)); - } - // Not ready yet. - std::thread::sleep(Duration::from_millis(10)); - } - bail!("Even after waiting for 1s, josh-proxy is still not available.") - } - pub fn exec(self) -> Result<()> { // First, and crucially only once, run the auto-actions -- but not for all commands. match &self { @@ -170,11 +103,7 @@ impl Command { | Command::Fmt { .. } | Command::Doc { .. } | Command::Clippy { .. } => Self::auto_actions()?, - | Command::Toolchain { .. } - | Command::Bench { .. } - | Command::RustcPull { .. } - | Command::RustcPush { .. } - | Command::Squash => {} + | Command::Toolchain { .. } | Command::Bench { .. } | Command::Squash => {} } // Then run the actual command. match self { @@ -191,8 +120,6 @@ impl Command { Command::Bench { target, no_install, save_baseline, load_baseline, benches } => Self::bench(target, no_install, save_baseline, load_baseline, benches), Command::Toolchain { flags } => Self::toolchain(flags), - Command::RustcPull { commit } => Self::rustc_pull(commit.clone()), - Command::RustcPush { github_user, branch } => Self::rustc_push(github_user, branch), Command::Squash => Self::squash(), } } @@ -233,156 +160,6 @@ impl Command { Ok(()) } - fn rustc_pull(commit: Option) -> Result<()> { - let sh = Shell::new()?; - sh.change_dir(miri_dir()?); - let commit = commit.map(Result::Ok).unwrap_or_else(|| { - let rust_repo_head = - cmd!(sh, "git ls-remote https://github.com/rust-lang/rust/ HEAD").read()?; - rust_repo_head - .split_whitespace() - .next() - .map(|front| front.trim().to_owned()) - .ok_or_else(|| anyhow!("Could not obtain Rust repo HEAD from remote.")) - })?; - // Make sure the repo is clean. - if cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty().not() { - bail!("working directory must be clean before running `./miri rustc-pull`"); - } - // Make sure josh is running. - let josh = Self::start_josh()?; - let josh_url = - format!("http://localhost:{JOSH_PORT}/rust-lang/rust.git@{commit}{JOSH_FILTER}.git"); - - // Update rust-version file. As a separate commit, since making it part of - // the merge has confused the heck out of josh in the past. - // We pass `--no-verify` to avoid running git hooks like `./miri fmt` that could in turn - // trigger auto-actions. - // We do this before the merge so that if there are merge conflicts, we have - // the right rust-version file while resolving them. - sh.write_file("rust-version", format!("{commit}\n"))?; - const PREPARING_COMMIT_MESSAGE: &str = "Preparing for merge from rustc"; - cmd!(sh, "git commit rust-version --no-verify -m {PREPARING_COMMIT_MESSAGE}") - .run() - .context("FAILED to commit rust-version file, something went wrong")?; - - // Fetch given rustc commit. - cmd!(sh, "git fetch {josh_url}") - .run() - .inspect_err(|_| { - // Try to un-do the previous `git commit`, to leave the repo in the state we found it. - cmd!(sh, "git reset --hard HEAD^") - .run() - .expect("FAILED to clean up again after failed `git fetch`, sorry for that"); - }) - .context("FAILED to fetch new commits, something went wrong (committing the rust-version file has been undone)")?; - - // This should not add any new root commits. So count those before and after merging. - let num_roots = || -> Result { - Ok(cmd!(sh, "git rev-list HEAD --max-parents=0 --count") - .read() - .context("failed to determine the number of root commits")? - .parse::()?) - }; - let num_roots_before = num_roots()?; - - // Merge the fetched commit. - const MERGE_COMMIT_MESSAGE: &str = "Merge from rustc"; - cmd!(sh, "git merge FETCH_HEAD --no-verify --no-ff -m {MERGE_COMMIT_MESSAGE}") - .run() - .context("FAILED to merge new commits, something went wrong")?; - - // Check that the number of roots did not increase. - if num_roots()? != num_roots_before { - bail!("Josh created a new root commit. This is probably not the history you want."); - } - - drop(josh); - Ok(()) - } - - fn rustc_push(github_user: String, branch: String) -> Result<()> { - let sh = Shell::new()?; - sh.change_dir(miri_dir()?); - let base = sh.read_file("rust-version")?.trim().to_owned(); - // Make sure the repo is clean. - if cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty().not() { - bail!("working directory must be clean before running `./miri rustc-push`"); - } - // Make sure josh is running. - let josh = Self::start_josh()?; - let josh_url = - format!("http://localhost:{JOSH_PORT}/{github_user}/rust.git{JOSH_FILTER}.git"); - - // Find a repo we can do our preparation in. - if let Ok(rustc_git) = env::var("RUSTC_GIT") { - // If rustc_git is `Some`, we'll use an existing fork for the branch updates. - sh.change_dir(rustc_git); - } else { - // Otherwise, do this in the local Miri repo. - println!( - "This will pull a copy of the rust-lang/rust history into this Miri checkout, growing it by about 1GB." - ); - print!( - "To avoid that, abort now and set the `RUSTC_GIT` environment variable to an existing rustc checkout. Proceed? [y/N] " - ); - std::io::stdout().flush()?; - let mut answer = String::new(); - std::io::stdin().read_line(&mut answer)?; - if answer.trim().to_lowercase() != "y" { - std::process::exit(1); - } - }; - // Prepare the branch. Pushing works much better if we use as base exactly - // the commit that we pulled from last time, so we use the `rust-version` - // file to find out which commit that would be. - println!("Preparing {github_user}/rust (base: {base})..."); - if cmd!(sh, "git fetch https://github.com/{github_user}/rust {branch}") - .ignore_stderr() - .read() - .is_ok() - { - println!( - "The branch '{branch}' seems to already exist in 'https://github.com/{github_user}/rust'. Please delete it and try again." - ); - std::process::exit(1); - } - cmd!(sh, "git fetch https://github.com/rust-lang/rust {base}").run()?; - cmd!(sh, "git push https://github.com/{github_user}/rust {base}:refs/heads/{branch}") - .ignore_stdout() - .ignore_stderr() // silence the "create GitHub PR" message - .run()?; - println!(); - - // Do the actual push. - sh.change_dir(miri_dir()?); - println!("Pushing miri changes..."); - cmd!(sh, "git push {josh_url} HEAD:{branch}").run()?; - println!(); - - // Do a round-trip check to make sure the push worked as expected. - cmd!(sh, "git fetch {josh_url} {branch}").ignore_stderr().read()?; - let head = cmd!(sh, "git rev-parse HEAD").read()?; - let fetch_head = cmd!(sh, "git rev-parse FETCH_HEAD").read()?; - if head != fetch_head { - bail!( - "Josh created a non-roundtrip push! Do NOT merge this into rustc!\n\ - Expected {head}, got {fetch_head}." - ); - } - println!( - "Confirmed that the push round-trips back to Miri properly. Please create a rustc PR:" - ); - println!( - // Open PR with `subtree update` title to silence the `no-merges` triagebot check - // See https://github.com/rust-lang/rust/pull/114157 - " https://github.com/rust-lang/rust/compare/{github_user}:{branch}?quick_pull=1&title=Miri+subtree+update&body=r?+@ghost" - ); - - drop(josh); - Ok(()) - } - fn squash() -> Result<()> { let sh = Shell::new()?; sh.change_dir(miri_dir()?); diff --git a/miri-script/src/main.rs b/miri-script/src/main.rs index e41df662ca..761ec5979f 100644 --- a/miri-script/src/main.rs +++ b/miri-script/src/main.rs @@ -142,25 +142,6 @@ pub enum Command { #[arg(trailing_var_arg = true, allow_hyphen_values = true)] flags: Vec, }, - /// Pull and merge Miri changes from the rustc repo. - /// - /// The fetched commit is stored in the `rust-version` file, so the next `./miri toolchain` will - /// install the rustc that just got pulled. - RustcPull { - /// The commit to fetch (default: latest rustc commit). - commit: Option, - }, - /// Push Miri changes back to the rustc repo. - /// - /// This will pull a copy of the rustc history into the Miri repo, unless you set the RUSTC_GIT - /// env var to an existing clone of the rustc repo. - RustcPush { - /// The Github user that owns the rustc fork to which we should push. - github_user: String, - /// The branch to push to. - #[arg(default_value = "miri-sync")] - branch: String, - }, /// Squash the commits of the current feature branch into one. Squash, } @@ -184,8 +165,7 @@ impl Command { flags.extend(remainder); Ok(()) } - Self::Bench { .. } | Self::RustcPull { .. } | Self::RustcPush { .. } | Self::Squash => - bail!("unexpected \"--\" found in arguments"), + Self::Bench { .. } | Self::Squash => bail!("unexpected \"--\" found in arguments"), } } }