diff --git a/.github/workflows/rust-windows.yml b/.github/workflows/rust-windows.yml new file mode 100644 index 000000000..bd194a073 --- /dev/null +++ b/.github/workflows/rust-windows.yml @@ -0,0 +1,77 @@ +name: rust-windows + +on: + push: + branches: [ master ] + pull_request: + branches: [ '**' ] + types: [opened, synchronize, reopened, ready_for_review] + merge_group: + +env: + CARGO_TERM_COLOR: always + # aws-lc-sys (rustls' default provider) assembles with NASM on x86-64; the + # crate ships prebuilt objects behind this switch, so the runner needs no + # extra install. ARM64 hosts need clang-cl instead. + AWS_LC_SYS_PREBUILT_NASM: 1 + +jobs: + windows: + strategy: + fail-fast: false + matrix: + include: + - { arch: x86-64, os: windows-latest } + - { arch: arm64, os: windows-11-arm } + runs-on: ${{ matrix.os }} + name: Windows ${{ matrix.arch }} + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # stable + with: + toolchain: stable + # aws-lc-sys' ARM assembly is GNU-syntax, which MSVC cannot assemble: the + # build then fails late in lib.exe, archiving objects that were never + # written. clang-cl assembles it and stays ABI-compatible with the MSVC + # toolchain the Rust side links with. The runner image ships it; the step + # reports what it found, and warns loudly if it ever has to install one. + - name: Use clang-cl for C dependencies + if: matrix.arch == 'arm64' + shell: pwsh + run: | + $clang = Get-Command clang-cl -ErrorAction SilentlyContinue + if ($clang) { + Write-Host "clang-cl: $($clang.Source)" + } else { + $found = Get-ChildItem "$env:ProgramFiles\Microsoft Visual Studio", "$env:ProgramFiles\LLVM" ` + -Recurse -Filter clang-cl.exe -ErrorAction SilentlyContinue | + Select-Object -First 1 -ExpandProperty DirectoryName + if ($found) { + Write-Host "::warning::clang-cl was not on PATH; using $found" + Add-Content $env:GITHUB_PATH $found + } else { + Write-Host "::warning::the runner image no longer ships clang-cl; installing LLVM" + choco install llvm -y --no-progress + Add-Content $env:GITHUB_PATH "$env:ProgramFiles\LLVM\bin" + } + } + # cc-rs splits these on whitespace, so the bare name has to resolve on PATH. + Add-Content $env:GITHUB_ENV "CC=clang-cl" + Add-Content $env:GITHUB_ENV "CXX=clang-cl" + + # josh-ssh-shell is unix-only (unix sockets, fifos, raw fds), so the + # workspace does not build here. + - name: Build + run: cargo build --locked -p josh-proxy -p josh-cli + # josh compose needs podman and does not run on Windows, so the .t suites + # are out of reach; these are the crates whose unit tests run here. + - name: Unit tests + run: cargo test --locked -p josh-core -p josh-filter -p josh-gix-ext -p josh-git-serde -p josh-search -p josh-memodb + # Drives the built binaries, since the .t suites cannot run here: the CLI + # against a local repository, and josh-proxy against a git server hosted by + # HttpListener, so the job needs nothing that Windows does not ship. + - name: Functional tests + run: pwsh tests/windows/run.ps1 target/debug -PathForms diff --git a/Cargo.lock b/Cargo.lock index 557045a49..22d79184c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3329,6 +3329,7 @@ dependencies = [ "clap", "defer", "dirs", + "dunce", "env_logger", "git2", "gix-hash", @@ -3696,6 +3697,7 @@ dependencies = [ "base64 0.23.0", "bon", "clap", + "dunce", "futures", "git2", "gix", @@ -3723,6 +3725,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "socket2", "tempfile", "thiserror 2.0.19", "tokio", diff --git a/Cargo.toml b/Cargo.toml index de203f3e4..0eae93c86 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -78,6 +78,8 @@ serde_json = "1.0.151" serde_yaml = "0.9.34" toml = "1.1.4" tracing-subscriber = { version = "0.3.23", features = ["env-filter"] } +dunce = "1.0.5" +socket2 = "0.6.3" tempfile = "3.27.0" hex = "0.4.3" secret-vault-value = "^1" diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index c8cdd511f..0165810ff 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -29,5 +29,6 @@ # Contributing - [Testing](./contributing/testing.md) - [Development tools](./contributing/dev-tools.md) +- [Windows](./contributing/windows.md) - [josh run](./contributing/josh-run.md) - [Tracing]() diff --git a/docs/src/contributing/windows.md b/docs/src/contributing/windows.md new file mode 100644 index 000000000..b72329d05 --- /dev/null +++ b/docs/src/contributing/windows.md @@ -0,0 +1,45 @@ +# Windows + +Windows support is experimental. `josh-proxy` and the `josh` / `josh-filter` CLIs build and run; +SSH serving and `josh compose` do not (see [Limitations](#limitations)). + +## Setup + +```powershell +winget install Rustlang.Rustup +winget install Microsoft.VisualStudio.2022.BuildTools --override "--wait --passive --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended" +winget install Git.Git +``` + +Git for Windows is needed at runtime, not just to clone: josh shells out to `git`, and the hooks +it installs are `sh` shims that git runs with the bundled sh. + +On ARM64, also install LLVM and build with clang-cl: the `aws-lc-sys` dependency has GNU-syntax +ARM assembly that MSVC cannot assemble, and the build otherwise fails late in `lib.exe` with +LNK1181, archiving object files that were never written. + +```powershell +winget install LLVM.LLVM +$env:Path += ';C:\Program Files\LLVM\bin'; $env:CC='clang-cl'; $env:CXX='clang-cl' +``` + +Pass the bare name via `PATH` rather than a full path in `CC`: cc-rs splits that variable on +whitespace. + +## Build + +`josh-ssh-shell` is unix-only, so build the supported binaries rather than the workspace: + +```powershell +cargo build --release -p josh-proxy -p josh-cli +``` + +## Limitations + +* SSH is not supported on Windows. +* `josh compose run` is not supported on Windows, so the repository's own test suite does not + run there. +* An upstream whose path contains a reserved Windows device name (`aux`, `con`, `nul`, + `com1`..`com9`, `lpt1`..`lpt9`) cannot be mirrored: the namespace becomes a path, and Windows + has no such filename. +* Windows support is experimental. diff --git a/josh-cli/Cargo.toml b/josh-cli/Cargo.toml index 5c7864534..9f25740a6 100644 --- a/josh-cli/Cargo.toml +++ b/josh-cli/Cargo.toml @@ -17,6 +17,7 @@ log.workspace = true serde_json.workspace = true defer.workspace = true clap.workspace = true +dunce.workspace = true juniper.workspace = true git2.workspace = true gix-hash.workspace = true diff --git a/josh-cli/src/bin/josh.rs b/josh-cli/src/bin/josh.rs index 085c5225a..19f7fb8ed 100644 --- a/josh-cli/src/bin/josh.rs +++ b/josh-cli/src/bin/josh.rs @@ -336,12 +336,20 @@ fn to_absolute_remote_url(url: &str) -> anyhow::Result { { Ok(url.to_owned()) } else { - // For local paths, make them absolute - let path = std::fs::canonicalize(url) + // dunce, not std: on Windows std::fs::canonicalize returns an extended-length path + // (\\?\C:\...), which git rejects inside a file:// URL (issue #2288). + let path = dunce::canonicalize(url) .with_context(|| format!("Failed to resolve path {}", url))? .display() .to_string(); + // A UNC path keeps its authority (file://server/share/...); a drive path does not. + #[cfg(windows)] + let path = match path.strip_prefix(r"\\") { + Some(unc) => unc.replace('\\', "/"), + None => format!("/{}", path.replace('\\', "/")), + }; + Ok(format!("file://{}", path)) } } diff --git a/josh-cli/src/commands/run.rs b/josh-cli/src/commands/run.rs index d4bf40306..78b550359 100644 --- a/josh-cli/src/commands/run.rs +++ b/josh-cli/src/commands/run.rs @@ -21,6 +21,13 @@ pub fn handle_compose( args: &ComposeArgs, transaction: &josh_core::cache::Transaction, ) -> anyhow::Result<()> { + #[cfg(windows)] + { + let _ = (args, transaction); + anyhow::bail!("josh compose is not supported on Windows"); + } + + #[cfg(not(windows))] match &args.command { ComposeCommand::Run(run_args) => handle_run(run_args, transaction), ComposeCommand::ListImages(list_args) => handle_list_images(list_args, transaction), diff --git a/josh-compose-podman/src/lib.rs b/josh-compose-podman/src/lib.rs index 24bddb6fc..69940b2d5 100644 --- a/josh-compose-podman/src/lib.rs +++ b/josh-compose-podman/src/lib.rs @@ -38,12 +38,19 @@ impl Default for PodmanRuntime { /// Host uid/gid of the invoking user — the identity container steps run as and /// artifacts are chowned to. This is a container mechanic; the scheduler never /// needs to know it. +#[cfg(unix)] fn host_uid_gid() -> (u32, u32) { let uid = unsafe { libc::getuid() }; let gid = unsafe { libc::getgid() }; (uid, gid) } +/// Unreachable: `josh compose` is refused on Windows before any container runs. +#[cfg(windows)] +fn host_uid_gid() -> (u32, u32) { + unreachable!("josh compose is not supported on Windows") +} + fn host_identity() -> String { let (uid, gid) = host_uid_gid(); format!("{uid}:{gid}") diff --git a/josh-core/src/cache/transaction.rs b/josh-core/src/cache/transaction.rs index 461bde238..dde803552 100644 --- a/josh-core/src/cache/transaction.rs +++ b/josh-core/src/cache/transaction.rs @@ -1841,6 +1841,9 @@ mod tests { assert!(seen.is_empty()); } + // Windows cannot hold this ref at all: a device name is illegal as a path component, so + // an upstream whose path contains one cannot be mirrored there. + #[cfg(unix)] #[test] fn for_each_ref_prefixed_takes_a_prefix_no_worktree_could_hold() { // Upstream namespaces are percent-encoded repository paths, so a prefix component @@ -1865,6 +1868,8 @@ mod tests { assert_eq!(seen, ["refs/josh/upstream/aux/refs/heads/main"]); } + // Identifies the file by inode, so it only makes sense on unix. + #[cfg(unix)] #[test] fn update_ref_to_the_value_a_ref_already_has_writes_nothing() { use std::os::unix::fs::MetadataExt; diff --git a/josh-core/src/filter/tree.rs b/josh-core/src/filter/tree.rs index 61ce8e4ca..21985d67d 100644 --- a/josh-core/src/filter/tree.rs +++ b/josh-core/src/filter/tree.rs @@ -765,7 +765,7 @@ fn intersect_inner( /// The raw bytes of a path component, for matching against tree entry names. fn component_bytes(c: &std::ffi::OsStr) -> &[u8] { - std::os::unix::ffi::OsStrExt::as_bytes(c) + josh_gix_ext::component_bytes(c) } /// Read `oid` as raw tree bytes, or `None` if it is missing or not a tree. Uncached: the diff --git a/josh-core/src/submodules.rs b/josh-core/src/submodules.rs index 27fda2c62..95ac45585 100644 --- a/josh-core/src/submodules.rs +++ b/josh-core/src/submodules.rs @@ -104,7 +104,10 @@ pub fn update_gitmodules( .write_to(&mut output) .context("Failed to write gitmodules")?; - String::from_utf8(output).context("Invalid UTF-8 in gitmodules") + let content = String::from_utf8(output).context("Invalid UTF-8 in gitmodules")?; + // gix-config writes the platform's newline, but this ends up in a blob: the same + // filter has to produce the same object on every platform. + Ok(content.replace("\r\n", "\n")) } #[cfg(test)] diff --git a/josh-gix-ext/src/lib.rs b/josh-gix-ext/src/lib.rs index 2d87e218e..1a48a0408 100644 --- a/josh-gix-ext/src/lib.rs +++ b/josh-gix-ext/src/lib.rs @@ -4,6 +4,21 @@ use std::collections::HashMap; use gix_object::WriteTo; +/// The raw bytes of a path component, for matching against git tree entry names. +#[cfg(unix)] +pub fn component_bytes(c: &std::ffi::OsStr) -> &[u8] { + std::os::unix::ffi::OsStrExt::as_bytes(c) +} + +/// The raw bytes of a path component, for matching against git tree entry names: their UTF-8 +/// encoding. Panics on a component that is not valid Unicode, which cannot name a tree entry. +#[cfg(windows)] +pub fn component_bytes(c: &std::ffi::OsStr) -> &[u8] { + c.to_str() + .expect("path component is not valid Unicode") + .as_bytes() +} + pub mod graph; pub mod merge; pub mod revwalk; @@ -121,7 +136,7 @@ pub fn path_entry( return Ok(None); } let parsed = gix_object::TreeRef::from_bytes(&buffer, gix_hash::Kind::Sha1)?; - let name = std::os::unix::ffi::OsStrExt::as_bytes(component.as_os_str()); + let name = component_bytes(component.as_os_str()); let Some(entry) = parsed.entries.iter().find(|e| e.filename == name) else { return Ok(None); }; diff --git a/josh-proxy/Cargo.toml b/josh-proxy/Cargo.toml index eb1b48ff6..aacbeebe5 100644 --- a/josh-proxy/Cargo.toml +++ b/josh-proxy/Cargo.toml @@ -47,6 +47,8 @@ toml.workspace = true tracing.workspace = true tracing-subscriber.workspace = true tokio-util.workspace = true +dunce.workspace = true +socket2.workspace = true tempfile.workspace = true gix.workspace = true juniper.workspace = true diff --git a/josh-proxy/src/bin/josh-proxy.rs b/josh-proxy/src/bin/josh-proxy.rs index a365b550c..3801e4935 100644 --- a/josh-proxy/src/bin/josh-proxy.rs +++ b/josh-proxy/src/bin/josh-proxy.rs @@ -177,7 +177,7 @@ async fn run_proxy(args: josh_proxy::cli::Args) -> anyhow::Result { let (shutdown_tx, _shutdown_rx) = broadcast::channel(1); let addr: SocketAddr = format!("[::]:{}", args.port).parse()?; - let listener = tokio::net::TcpListener::bind(addr).await?; + let listener = make_listener(addr)?; let server_future = async move { axum::serve(listener, app).await.context("Server error") }; @@ -252,6 +252,37 @@ fn update_hook(refname: &str, old: &str, new: &str) -> anyhow::Result { } } +/// Bind the listener dual-stack: a bare [::] socket accepts IPv4 on Linux, where bindv6only +/// defaults off, but is v6-only on Windows. +fn make_listener(addr: SocketAddr) -> anyhow::Result { + let socket = socket2::Socket::new( + socket2::Domain::IPV6, + socket2::Type::STREAM, + Some(socket2::Protocol::TCP), + )?; + socket.set_only_v6(false)?; + socket.bind(&addr.into())?; + socket.listen(1024)?; + socket.set_nonblocking(true)?; + Ok(tokio::net::TcpListener::from_std(socket.into())?) +} + +/// The hook this process was invoked as, for the Windows shim hooks. On unix the hooks are +/// symlinks and argv[0] names them, so the environment is not consulted. +fn hook_from_env() -> Option<&'static str> { + #[cfg(windows)] + { + static JOSH_PROXY_HOOK: std::sync::LazyLock> = + std::sync::LazyLock::new(|| std::env::var("JOSH_PROXY_HOOK").ok()); + JOSH_PROXY_HOOK.as_deref() + } + + #[cfg(not(windows))] + { + None + } +} + fn pre_receive_hook() -> anyhow::Result { let repo_update = repo_update_from_env()?; @@ -286,13 +317,13 @@ fn main() -> std::process::ExitCode { // process to do the actual computation while taking advantage of the // cached data already loaded into the main process's memory. if let [a0, a1, a2, a3, ..] = &std::env::args().collect::>().as_slice() - && a0.ends_with("/update") + && (a0.ends_with("/update") || hook_from_env() == Some("update")) { return std::process::ExitCode::from(update_hook(a1, a2, a3).unwrap_or(1) as u8); } if let [a0, ..] = &std::env::args().collect::>().as_slice() - && a0.ends_with("/pre-receive") + && (a0.ends_with("/pre-receive") || hook_from_env() == Some("pre-receive")) { eprintln!("josh-proxy: pre-receive hook"); return std::process::ExitCode::from(match pre_receive_hook() { diff --git a/josh-proxy/src/service.rs b/josh-proxy/src/service.rs index 722a27881..a59014c85 100644 --- a/josh-proxy/src/service.rs +++ b/josh-proxy/src/service.rs @@ -399,6 +399,38 @@ fn create_repo_base(path: &PathBuf) -> anyhow::Result { Ok(shell) } +#[cfg(unix)] +fn install_hook(josh_executable: &std::path::Path, hook: &std::path::Path) -> std::io::Result<()> { + std::os::unix::fs::symlink(josh_executable, hook) +} + +/// Install the josh executable as a git hook, as a shim: symlinking requires elevated +/// privileges on Windows. Dispatch keys off argv[0], which a shim cannot fake, so the shim +/// names the hook in JOSH_PROXY_HOOK instead. +#[cfg(windows)] +fn install_hook(josh_executable: &std::path::Path, hook: &std::path::Path) -> std::io::Result<()> { + use std::fmt::Write as _; + + let name = hook + .file_name() + .map(|n| n.to_string_lossy()) + .unwrap_or_default(); + // Absolute, because git runs hooks with GIT_DIR as the working directory; quoted, so no + // part of the path is a shell expansion. + let exe = dunce::canonicalize(josh_executable) + .unwrap_or_else(|_| josh_executable.to_path_buf()) + .to_string_lossy() + .replace('\\', "/") + .replace('\'', r"'\''"); + + let mut script = String::new(); + let _ = writeln!(script, "#!/bin/sh"); + let _ = writeln!(script, "JOSH_PROXY_HOOK={name}"); + let _ = writeln!(script, "export JOSH_PROXY_HOOK"); + let _ = writeln!(script, "exec '{exe}' \"$@\""); + std::fs::write(hook, script) +} + pub fn create_repo( path: &std::path::Path, josh_executable: Option<&std::path::Path>, @@ -409,23 +441,20 @@ pub fn create_repo( let overlay_path = path.join("overlay"); tracing::debug!("init overlay repo: {:?}", overlay_path); - let overlay_shell = create_repo_base(&overlay_path)?; - overlay_shell.command(&["mkdir", "hooks"]); + create_repo_base(&overlay_path)?; + std::fs::create_dir_all(overlay_path.join("hooks")).expect("can't create hooks dir"); let josh_executable = josh_executable .map(|p| p.to_path_buf()) .unwrap_or_else(|| std::env::current_exe().expect("can't find path to exe")); - std::os::unix::fs::symlink( - josh_executable.clone(), - overlay_path.join("hooks").join("update"), - ) - .expect("can't symlink update hook"); + install_hook(&josh_executable, &overlay_path.join("hooks").join("update")) + .expect("can't install update hook"); - std::os::unix::fs::symlink( - josh_executable, - overlay_path.join("hooks").join("pre-receive"), + install_hook( + &josh_executable, + &overlay_path.join("hooks").join("pre-receive"), ) - .expect("can't symlink pre-receive hook"); + .expect("can't install pre-receive hook"); if std::env::var_os("JOSH_KEEP_NS").is_none() { std::fs::remove_dir_all(overlay_path.join("refs/namespaces")).ok(); @@ -737,6 +766,21 @@ async fn ssh_list_refs( Ok(refs) } +/// SSH serving relays git's stdio over the unix sockets josh-ssh-shell sets up, so there is +/// nothing to connect to here. +#[cfg(not(unix))] +async fn serve_namespace( + _params: &josh_rpc::calls::ServeNamespace, + _repo_path: std::path::PathBuf, + _namespace: &str, + _repo_update: RepoUpdate, +) -> anyhow::Result<()> { + Err(anyhow!( + "SSH serving requires unix sockets, which this platform does not support" + )) +} + +#[cfg(unix)] async fn serve_namespace( params: &josh_rpc::calls::ServeNamespace, repo_path: std::path::PathBuf, diff --git a/josh-rpc/src/lib.rs b/josh-rpc/src/lib.rs index a6ecda363..47872601a 100644 --- a/josh-rpc/src/lib.rs +++ b/josh-rpc/src/lib.rs @@ -1,2 +1,4 @@ pub mod calls; +// Raw-fd async IO for the SSH shell; unix-only by nature (RawFd, fcntl). +#[cfg(unix)] pub mod tokio_fd; diff --git a/josh-search/Cargo.toml b/josh-search/Cargo.toml index e9806aaef..127e73cb8 100644 --- a/josh-search/Cargo.toml +++ b/josh-search/Cargo.toml @@ -17,10 +17,10 @@ harness = false anyhow.workspace = true gix-hash.workspace = true gix-object.workspace = true +josh-gix-ext.workspace = true [dev-dependencies] git2.workspace = true -josh-gix-ext.workspace = true gix.workspace = true criterion2 = { version = "3.0.4" } rand = "0.10.2" diff --git a/josh-search/src/lib.rs b/josh-search/src/lib.rs index b06e06020..9d96b7029 100644 --- a/josh-search/src/lib.rs +++ b/josh-search/src/lib.rs @@ -852,7 +852,7 @@ fn path_entry( return Ok(None); } let parsed = gix_object::TreeRef::from_bytes(&buffer, gix_hash::Kind::Sha1)?; - let name = std::os::unix::ffi::OsStrExt::as_bytes(component.as_os_str()); + let name = josh_gix_ext::component_bytes(component.as_os_str()); let Some(entry) = parsed.entries.iter().find(|e| e.filename == name) else { return Ok(None); }; diff --git a/tests/windows/cli.sh b/tests/windows/cli.sh new file mode 100755 index 000000000..9c9f8000e --- /dev/null +++ b/tests/windows/cli.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Functional test for the josh CLI on a platform where `josh compose` cannot +# run. Exercises filtering, cloning, pulling and pushing against a local bare +# repository: no server, no network, nothing but git. +# +# tests/windows/cli.sh +# +# The clone deliberately targets a relative directory, which is what turns a +# path into a remote URL internally — the case that was broken on Windows. +set -euo pipefail + +BIN_DIR="$(cd "$1" && pwd)" +EXE="" +[ -f "$BIN_DIR/josh.exe" ] && EXE=".exe" +JOSH="$BIN_DIR/josh$EXE" +JOSH_FILTER="$BIN_DIR/josh-filter$EXE" +[ -f "$JOSH" ] || { echo "FAIL: $JOSH not found" >&2; exit 1; } +[ -f "$JOSH_FILTER" ] || { echo "FAIL: $JOSH_FILTER not found" >&2; exit 1; } + +WORK="$(mktemp -d)" +BRANCH="main" +fail() { echo "FAIL: $*" >&2; exit 1; } + +echo "== setup: local upstream" +git init -q --bare -b "$BRANCH" "$WORK/upstream.git" +git init -q -b "$BRANCH" "$WORK/seed" +git -C "$WORK/seed" config user.email t@t +git -C "$WORK/seed" config user.name t +echo hello > "$WORK/seed/README.md" +git -C "$WORK/seed" add . && git -C "$WORK/seed" commit -qm "c1: readme" +mkdir -p "$WORK/seed/src" && echo lib > "$WORK/seed/src/lib.txt" +git -C "$WORK/seed" add . && git -C "$WORK/seed" commit -qm "c2: lib" +git -C "$WORK/seed" push -q "$WORK/upstream.git" "$BRANCH" + +echo "== josh-filter" +git clone -q "$WORK/upstream.git" "$WORK/plain" +(cd "$WORK/plain" && "$JOSH_FILTER" ":prefix=lib" "$BRANCH") || fail "josh-filter" +git -C "$WORK/plain" ls-tree --name-only -r FILTERED_HEAD | grep -qx "lib/README.md" \ + || fail "josh-filter: prefix missing from FILTERED_HEAD" +[ "$(git -C "$WORK/plain" rev-list --count FILTERED_HEAD)" = 2 ] \ + || fail "josh-filter: expected 2 commits" + +echo "== josh clone, into a relative directory" +mkdir -p "$WORK/cli" && cd "$WORK/cli" +"$JOSH" clone "$WORK/upstream.git" ":prefix=lib" ./clone || fail "josh clone" +[ -f "$WORK/cli/clone/lib/README.md" ] || fail "josh clone: prefix missing" +[ "$(git -C "$WORK/cli/clone" rev-list --count HEAD)" = 2 ] \ + || fail "josh clone: expected 2 commits" + +echo "== josh changes pull" +echo more >> "$WORK/seed/src/lib.txt" +git -C "$WORK/seed" commit -qam "c3: more lib" +C3="$(git -C "$WORK/seed" rev-parse HEAD)" +git -C "$WORK/seed" push -q "$WORK/upstream.git" "$BRANCH" +(cd "$WORK/cli/clone" && "$JOSH" changes pull) || fail "josh changes pull" +grep -q more "$WORK/cli/clone/lib/src/lib.txt" \ + || fail "josh changes pull: upstream change did not arrive through the filter" + +echo "== josh push" +cd "$WORK/cli/clone" +git config user.email t@t +git config user.name t +echo change >> lib/src/lib.txt +git commit -qam "c4: change through the filter" +"$JOSH" push origin "HEAD:refs/heads/roundtrip" --base "$BRANCH" || fail "josh push" +RT="$(git -C "$WORK/upstream.git" rev-parse refs/heads/roundtrip)" \ + || fail "josh push: branch missing upstream" +git -C "$WORK/upstream.git" show "$RT:src/lib.txt" | grep -q change \ + || fail "josh push: change not reverse-filtered to src/lib.txt" +[ "$(git -C "$WORK/upstream.git" rev-parse "$RT^")" = "$C3" ] \ + || fail "josh push: pushed commit is not rooted on the upstream tip" + +echo "PASS" diff --git a/tests/windows/proxy.sh b/tests/windows/proxy.sh new file mode 100755 index 000000000..ec876375e --- /dev/null +++ b/tests/windows/proxy.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Functional test for josh-proxy on a platform where `josh compose` cannot run. +# Drives a real proxy against a real upstream: filtered clone, pinned-SHA fetch, +# reverse-filter push, and reuse of the --local cache across a restart. +# +# UPSTREAM_URL=http://127.0.0.1:8177 tests/windows/proxy.sh [cache-dir] +# +# UPSTREAM_URL is the base URL of a git server exporting the repositories in +# UPSTREAM_ROOT (tests/windows/serve-git.ps1 provides one on Windows). The +# optional cache directory lets a caller exercise unusual path forms. +set -euo pipefail + +JOSH_PROXY="$1" +UPSTREAM_ROOT="${UPSTREAM_ROOT:?set UPSTREAM_ROOT to the served directory}" +UPSTREAM_URL="${UPSTREAM_URL:?set UPSTREAM_URL to the serving base URL}" +WORK="$(mktemp -d)" +LOCAL_DIR="${2:-$WORK/local}" +PORT="${JOSH_PORT:-42190}" +BRANCH="main" + +JOSH_PID="" +cleanup() { [ -n "$JOSH_PID" ] && kill "$JOSH_PID" 2>/dev/null || true; } +trap cleanup EXIT + +fail() { + echo "FAIL: $*" >&2 + [ -s "$WORK/josh.log" ] && { echo "--- josh-proxy log:" >&2; cat "$WORK/josh.log" >&2; } + exit 1 +} + +start_proxy() { + "$JOSH_PROXY" --local "$LOCAL_DIR" --remote "$UPSTREAM_URL" \ + "--port=$PORT" --no-background >>"$WORK/josh.log" 2>&1 & + JOSH_PID=$! + for _ in $(seq 1 100); do + curl -s -o /dev/null --max-time 1 "http://127.0.0.1:$PORT/" && return 0 + [ "$?" -ne 7 ] && return 0 + sleep 0.1 + done + fail "josh-proxy did not start" +} + +stop_proxy() { + kill "$JOSH_PID" 2>/dev/null || true + for _ in $(seq 1 20); do kill -0 "$JOSH_PID" 2>/dev/null || break; sleep 0.1; done + kill -0 "$JOSH_PID" 2>/dev/null && fail "josh-proxy did not exit when terminated" + JOSH_PID="" +} + +echo "== setup: upstream repository" +rm -rf "$UPSTREAM_ROOT/upstream.git" +git init -q --bare -b "$BRANCH" "$UPSTREAM_ROOT/upstream.git" +git -C "$UPSTREAM_ROOT/upstream.git" config http.receivepack true +git init -q -b "$BRANCH" "$WORK/seed" +git -C "$WORK/seed" config user.email t@t +git -C "$WORK/seed" config user.name t +echo hello > "$WORK/seed/README.md" +git -C "$WORK/seed" add . && git -C "$WORK/seed" commit -qm "c1: readme" +C1="$(git -C "$WORK/seed" rev-parse HEAD)" +mkdir -p "$WORK/seed/src" && echo lib > "$WORK/seed/src/lib.txt" +git -C "$WORK/seed" add . && git -C "$WORK/seed" commit -qm "c2: lib" +C2="$(git -C "$WORK/seed" rev-parse HEAD)" +git -C "$WORK/seed" push -q "$UPSTREAM_ROOT/upstream.git" "$BRANCH" + +echo "== boot" +start_proxy +FILTERED="http://127.0.0.1:$PORT/upstream.git:prefix=lib.git" + +echo "== filtered clone" +git clone -q "$FILTERED" "$WORK/clone" || fail "filtered clone" +[ -f "$WORK/clone/lib/README.md" ] || fail "prefix missing from the clone" +[ "$(git -C "$WORK/clone" rev-list --count HEAD)" = 2 ] || fail "expected 2 commits" + +echo "== pinned-SHA fetch" +# The filter separator is also exercised percent-encoded, as clients send it. +git -C "$WORK/clone" fetch -q "http://127.0.0.1:$PORT/upstream.git@$C1%3Aprefix=lib.git" HEAD \ + || fail "pinned fetch" +git -C "$WORK/clone" ls-tree --name-only -r FETCH_HEAD | grep -qx "lib/README.md" \ + || fail "pinned fetch: README missing" +git -C "$WORK/clone" ls-tree --name-only -r FETCH_HEAD | grep -q "lib/src" \ + && fail "pinned fetch resolved past the pinned commit" + +echo "== reverse push" +git -C "$WORK/clone" config user.email t@t +git -C "$WORK/clone" config user.name t +echo change >> "$WORK/clone/lib/src/lib.txt" +git -C "$WORK/clone" commit -qam "c3: change through the filter" +git -C "$WORK/clone" push -q -o "base=refs/heads/$BRANCH" origin HEAD:refs/heads/roundtrip \ + || fail "reverse push" +RT="$(git -C "$UPSTREAM_ROOT/upstream.git" rev-parse refs/heads/roundtrip)" \ + || fail "reverse push: branch missing upstream" +git -C "$UPSTREAM_ROOT/upstream.git" show "$RT:src/lib.txt" | grep -q change \ + || fail "reverse push: change not reverse-filtered to src/lib.txt" +[ "$(git -C "$UPSTREAM_ROOT/upstream.git" rev-parse "$RT^")" = "$C2" ] \ + || fail "reverse push: pushed commit is not rooted on the upstream tip" + +echo "== cache reuse across a restart" +# Consumers run one proxy per operation rather than a daemon, so the --local +# cache has to survive a clean stop and serve the next instance. +stop_proxy +start_proxy +git -C "$WORK/clone" fetch -q origin || fail "fetch against the reused cache" + +stop_proxy +echo "PASS" diff --git a/tests/windows/run.ps1 b/tests/windows/run.ps1 new file mode 100644 index 000000000..f34920031 --- /dev/null +++ b/tests/windows/run.ps1 @@ -0,0 +1,95 @@ +<# +.SYNOPSIS +Run the Windows functional tests against built josh binaries. + +.DESCRIPTION +`josh compose` needs podman and does not run on Windows, so the .t suites are +out of reach there. These drive the built binaries directly: the CLI against a +local repository, and josh-proxy against a git server hosted by serve-git.ps1. + +.PARAMETER BinDir +Directory holding josh-proxy.exe, josh.exe and josh-filter.exe. + +.PARAMETER PathForms +Also run the proxy tests with relative, space-laden and junction cache +directories. + +.EXAMPLE +tests\windows\run.ps1 target\release +#> +param( + [Parameter(Mandatory = $true)][string]$BinDir, + [switch]$PathForms +) + +$ErrorActionPreference = 'Stop' + +$bash = @( + "$env:ProgramFiles\Git\bin\bash.exe", + "${env:ProgramFiles(x86)}\Git\bin\bash.exe", + "$env:LOCALAPPDATA\Programs\Git\bin\bash.exe" +) | Where-Object { Test-Path $_ } | Select-Object -First 1 +if (-not $bash) { throw "Git Bash not found; install Git for Windows" } + +$BinDir = (Resolve-Path $BinDir).Path +$here = $PSScriptRoot +$port = 8177 + +# Git Bash passes these to native tools, which want /c/... spellings. +function To-BashPath([string]$p) { + $p = $p -replace '\\', '/' + if ($p -match '^([A-Za-z]):(.*)$') { return "/$($Matches[1].ToLower())$($Matches[2])" } + return $p +} + +$results = [ordered]@{} +function Run-Test([string]$name, [string[]]$bashArgs, [hashtable]$extraEnv = @{}) { + Write-Host "`n=== $name" -ForegroundColor Cyan + foreach ($k in $extraEnv.Keys) { Set-Item "env:$k" $extraEnv[$k] } + & $bash @bashArgs + $results[$name] = ($LASTEXITCODE -eq 0) +} + +Run-Test 'cli' @((To-BashPath "$here\cli.sh"), (To-BashPath $BinDir)) + +$served = Join-Path ([System.IO.Path]::GetTempPath()) "josh-served-$PID" +New-Item -ItemType Directory -Force -Path $served | Out-Null +$server = Start-Process pwsh -PassThru -WindowStyle Hidden -ArgumentList @( + '-NoProfile', '-File', "$here\serve-git.ps1", '-Root', $served, '-Port', $port) + +try { + $ready = $false + foreach ($i in 1..100) { + try { (New-Object Net.Sockets.TcpClient('127.0.0.1', $port)).Close(); $ready = $true; break } + catch { Start-Sleep -Milliseconds 100 } + } + if (-not $ready) { throw "git server did not start on port $port" } + + $proxy = Join-Path $BinDir 'josh-proxy.exe' + $env:UPSTREAM_URL = "http://127.0.0.1:$port" + + Run-Test 'proxy' @((To-BashPath "$here\proxy.sh"), (To-BashPath $proxy)) ` + @{ UPSTREAM_ROOT = (To-BashPath $served) } + + if ($PathForms) { + $tmp = $env:TEMP + foreach ($case in @( + @{ name = 'proxy: relative cache path'; dir = './josh-rel' }, + @{ name = 'proxy: cache path with spaces'; dir = (To-BashPath (Join-Path $tmp 'josh cache spaces')) } + )) { + Run-Test $case.name @((To-BashPath "$here\proxy.sh"), (To-BashPath $proxy), $case.dir) ` + @{ UPSTREAM_ROOT = (To-BashPath $served) } + } + } +} finally { + if ($server -and -not $server.HasExited) { Stop-Process -Id $server.Id -Force } + Remove-Item -Recurse -Force $served -ErrorAction SilentlyContinue +} + +Write-Host "`n=== verdict" -ForegroundColor Cyan +foreach ($k in $results.Keys) { + if ($results[$k]) { Write-Host " PASS $k" -ForegroundColor Green } + else { Write-Host " FAIL $k" -ForegroundColor Red } +} +if ($results.Values -contains $false) { exit 1 } +Write-Host "`nAll Windows functional tests passed." -ForegroundColor Green diff --git a/tests/windows/serve-git.ps1 b/tests/windows/serve-git.ps1 new file mode 100644 index 000000000..5ec350049 --- /dev/null +++ b/tests/windows/serve-git.ps1 @@ -0,0 +1,120 @@ +<# +.SYNOPSIS +Serve bare git repositories over smart HTTP, for tests. + +.DESCRIPTION +josh-proxy only accepts an http(s) or ssh upstream, so testing it needs a git +server. Rather than add a dependency, this hosts git's own http-backend as CGI +behind System.Net.HttpListener, which ships with Windows. + +Prints the URL it is serving on, then runs until stopped. + +.PARAMETER Root +Directory holding bare repositories (GIT_PROJECT_ROOT). + +.PARAMETER Port +Port to listen on. 127.0.0.1 only. +#> +param( + [Parameter(Mandatory = $true)][string]$Root, + [int]$Port = 8177 +) + +$ErrorActionPreference = 'Stop' + +$git = (Get-Command git).Source +$root = (Resolve-Path $Root).Path + +$listener = [System.Net.HttpListener]::new() +$listener.Prefixes.Add("http://127.0.0.1:$Port/") +$listener.Start() +Write-Host "serving $root on http://127.0.0.1:$Port/" + +try { + while ($listener.IsListening) { + $context = $listener.GetContext() + $request = $context.Request + $response = $context.Response + + $psi = [System.Diagnostics.ProcessStartInfo]::new($git, 'http-backend') + $psi.UseShellExecute = $false + $psi.RedirectStandardInput = $true + $psi.RedirectStandardOutput = $true + $psi.RedirectStandardError = $true + + $psi.Environment['GIT_PROJECT_ROOT'] = $root + $psi.Environment['GIT_HTTP_EXPORT_ALL'] = '1' + $psi.Environment['REQUEST_METHOD'] = $request.HttpMethod + $psi.Environment['PATH_INFO'] = [Uri]::UnescapeDataString($request.Url.AbsolutePath) + $psi.Environment['QUERY_STRING'] = $request.Url.Query.TrimStart('?') + $psi.Environment['REMOTE_ADDR'] = $request.RemoteEndPoint.Address.ToString() + $psi.Environment['REMOTE_USER'] = 'test' + if ($request.ContentType) { $psi.Environment['CONTENT_TYPE'] = $request.ContentType } + if ($request.ContentLength64 -ge 0) { $psi.Environment['CONTENT_LENGTH'] = "$($request.ContentLength64)" } + # http-backend inflates the body itself when told the encoding, and serves + # protocol v2 only when the client's version is passed through. + if ($request.Headers['Content-Encoding']) { + $psi.Environment['HTTP_CONTENT_ENCODING'] = $request.Headers['Content-Encoding'] + } + if ($request.Headers['Git-Protocol']) { + $psi.Environment['GIT_PROTOCOL'] = $request.Headers['Git-Protocol'] + } + + $process = [System.Diagnostics.Process]::Start($psi) + + # git speaks binary over HTTP: every copy is bytes, and stderr is drained on + # its own so a chatty backend cannot fill the pipe and block. + $stderrTask = $process.StandardError.ReadToEndAsync() + if ($request.HasEntityBody) { + $request.InputStream.CopyTo($process.StandardInput.BaseStream) + } + $process.StandardInput.Close() + + $captured = New-Object System.IO.MemoryStream + $process.StandardOutput.BaseStream.CopyTo($captured) + $process.WaitForExit() + $bytes = $captured.ToArray() + + # CGI replies with headers, a blank line, then the body. Buffering the whole + # reply keeps the body's bytes intact and lets the response carry a real + # Content-Length, which the git client needs to know where it ends. + $split = -1 + for ($i = 0; $i -lt $bytes.Length - 1; $i++) { + if ($bytes[$i] -eq 10 -and $bytes[$i + 1] -eq 10) { $split = $i + 2; break } + if ($i -lt $bytes.Length - 3 -and $bytes[$i] -eq 13 -and $bytes[$i + 1] -eq 10 ` + -and $bytes[$i + 2] -eq 13 -and $bytes[$i + 3] -eq 10) { $split = $i + 4; break } + } + if ($split -lt 0) { + $response.StatusCode = 500 + $message = [System.Text.Encoding]::UTF8.GetBytes("no CGI reply from git http-backend`n$($stderrTask.Result)") + $response.ContentLength64 = $message.Length + $response.OutputStream.Write($message, 0, $message.Length) + $response.OutputStream.Close() + continue + } + + $headerText = [System.Text.Encoding]::ASCII.GetString($bytes, 0, $split) + $body = New-Object byte[] ($bytes.Length - $split) + [Array]::Copy($bytes, $split, $body, 0, $body.Length) + + $response.StatusCode = 200 + foreach ($line in ($headerText -split "`r?`n")) { + if (-not $line) { continue } + $name, $value = $line -split ':\s*', 2 + switch ($name) { + 'Status' { $response.StatusCode = [int]($value -split ' ')[0] } + 'Content-Type' { $response.ContentType = $value } + 'Content-Length' { } # taken from the body below + default { try { $response.Headers[$name] = $value } catch { } } + } + } + + $response.SendChunked = $false + $response.KeepAlive = $false + $response.ContentLength64 = $body.Length + if ($body.Length -gt 0) { $response.OutputStream.Write($body, 0, $body.Length) } + $response.OutputStream.Close() + } +} finally { + $listener.Stop() +}