WIP: arm64 windows CI leg

.github/workflows/rust-windows.yml

@@ -0,0 +1,77 @@
+name: rust-windows
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ '**' ]
+    types: [opened, synchronize, reopened, ready_for_review]
+  merge_group:
+
+env:
+  CARGO_TERM_COLOR: always
+  # aws-lc-sys (rustls' default provider) assembles with NASM on x86-64; the
+  # crate ships prebuilt objects behind this switch, so the runner needs no
+  # extra install. ARM64 hosts need clang-cl instead.
+  AWS_LC_SYS_PREBUILT_NASM: 1
+
+jobs:
+  windows:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - { arch: x86-64, os: windows-latest }
+          - { arch: arm64, os: windows-11-arm }
+    runs-on: ${{ matrix.os }}
+    name: Windows ${{ matrix.arch }}
+    if: github.event_name != 'pull_request' || github.event.pull_request.draft == false
+    steps:
+    - name: Checkout
+      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+    - name: Install Rust toolchain
+      uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # stable
+      with:
+        toolchain: stable
+    # aws-lc-sys' ARM assembly is GNU-syntax, which MSVC cannot assemble: the
+    # build then fails late in lib.exe, archiving objects that were never
+    # written. clang-cl assembles it and stays ABI-compatible with the MSVC
+    # toolchain the Rust side links with. The runner image ships it; the step
+    # reports what it found, and warns loudly if it ever has to install one.
+    - name: Use clang-cl for C dependencies
+      if: matrix.arch == 'arm64'
+      shell: pwsh
+      run: |
+        $clang = Get-Command clang-cl -ErrorAction SilentlyContinue
+        if ($clang) {
+          Write-Host "clang-cl: $($clang.Source)"
+        } else {
+          $found = Get-ChildItem "$env:ProgramFiles\Microsoft Visual Studio", "$env:ProgramFiles\LLVM" `
+            -Recurse -Filter clang-cl.exe -ErrorAction SilentlyContinue |
+            Select-Object -First 1 -ExpandProperty DirectoryName
+          if ($found) {
+            Write-Host "::warning::clang-cl was not on PATH; using $found"
+            Add-Content $env:GITHUB_PATH $found
+          } else {
+            Write-Host "::warning::the runner image no longer ships clang-cl; installing LLVM"
+            choco install llvm -y --no-progress
+            Add-Content $env:GITHUB_PATH "$env:ProgramFiles\LLVM\bin"
+          }
+        }
+        # cc-rs splits these on whitespace, so the bare name has to resolve on PATH.
+        Add-Content $env:GITHUB_ENV "CC=clang-cl"
+        Add-Content $env:GITHUB_ENV "CXX=clang-cl"
+
+    # josh-ssh-shell is unix-only (unix sockets, fifos, raw fds), so the
+    # workspace does not build here.
+    - name: Build
+      run: cargo build --locked -p josh-proxy -p josh-cli
+    # josh compose needs podman and does not run on Windows, so the .t suites
+    # are out of reach; these are the crates whose unit tests run here.
+    - name: Unit tests
+      run: cargo test --locked -p josh-core -p josh-filter -p josh-gix-ext -p josh-git-serde -p josh-search -p josh-memodb
+    # Drives the built binaries, since the .t suites cannot run here: the CLI
+    # against a local repository, and josh-proxy against a git server hosted by
+    # HttpListener, so the job needs nothing that Windows does not ship.
+    - name: Functional tests
+      run: pwsh tests/windows/run.ps1 target/debug -PathForms

Cargo.toml

@@ -78,6 +78,8 @@ serde_json = "1.0.151"
 serde_yaml = "0.9.34"
 toml = "1.1.4"
 tracing-subscriber = { version = "0.3.23", features = ["env-filter"] }
+dunce = "1.0.5"
+socket2 = "0.6.3"
 tempfile = "3.27.0"
 hex = "0.4.3"
 secret-vault-value = "^1"

docs/src/SUMMARY.md

@@ -29,5 +29,6 @@
 # Contributing
 - [Testing](./contributing/testing.md)
 - [Development tools](./contributing/dev-tools.md)
+- [Windows](./contributing/windows.md)
 - [josh run](./contributing/josh-run.md)
 - [Tracing]()

docs/src/contributing/windows.md

@@ -0,0 +1,45 @@
+# Windows
+
+Windows support is experimental. `josh-proxy` and the `josh` / `josh-filter` CLIs build and run;
+SSH serving and `josh compose` do not (see [Limitations](#limitations)).
+
+## Setup
+
+```powershell
+winget install Rustlang.Rustup
+winget install Microsoft.VisualStudio.2022.BuildTools --override "--wait --passive --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended"
+winget install Git.Git
+```
+
+Git for Windows is needed at runtime, not just to clone: josh shells out to `git`, and the hooks
+it installs are `sh` shims that git runs with the bundled sh.
+
+On ARM64, also install LLVM and build with clang-cl: the `aws-lc-sys` dependency has GNU-syntax
+ARM assembly that MSVC cannot assemble, and the build otherwise fails late in `lib.exe` with
+LNK1181, archiving object files that were never written.
+
+```powershell
+winget install LLVM.LLVM
+$env:Path += ';C:\Program Files\LLVM\bin'; $env:CC='clang-cl'; $env:CXX='clang-cl'
+```
+
+Pass the bare name via `PATH` rather than a full path in `CC`: cc-rs splits that variable on
+whitespace.
+
+## Build
+
+`josh-ssh-shell` is unix-only, so build the supported binaries rather than the workspace:
+
+```powershell
+cargo build --release -p josh-proxy -p josh-cli
+```
+
+## Limitations
+
+* SSH is not supported on Windows.
+* `josh compose run` is not supported on Windows, so the repository's own test suite does not
+  run there.
+* An upstream whose path contains a reserved Windows device name (`aux`, `con`, `nul`,
+  `com1`..`com9`, `lpt1`..`lpt9`) cannot be mirrored: the namespace becomes a path, and Windows
+  has no such filename.
+* Windows support is experimental.

josh-cli/Cargo.toml

@@ -17,6 +17,7 @@ log.workspace = true
 serde_json.workspace = true
 defer.workspace = true
 clap.workspace = true
+dunce.workspace = true
 juniper.workspace = true
 git2.workspace = true
 gix-hash.workspace = true

josh-cli/src/bin/josh.rs

@@ -336,12 +336,20 @@ fn to_absolute_remote_url(url: &str) -> anyhow::Result<String> {
     {
         Ok(url.to_owned())
     } else {
-        // For local paths, make them absolute
-        let path = std::fs::canonicalize(url)
+        // dunce, not std: on Windows std::fs::canonicalize returns an extended-length path
+        // (\\?\C:\...), which git rejects inside a file:// URL (issue #2288).
+        let path = dunce::canonicalize(url)
             .with_context(|| format!("Failed to resolve path {}", url))?
             .display()
             .to_string();
 
+        // A UNC path keeps its authority (file://server/share/...); a drive path does not.
+        #[cfg(windows)]
+        let path = match path.strip_prefix(r"\\") {
+            Some(unc) => unc.replace('\\', "/"),
+            None => format!("/{}", path.replace('\\', "/")),
+        };
+
         Ok(format!("file://{}", path))
     }
 }

josh-cli/src/commands/run.rs

@@ -21,6 +21,13 @@ pub fn handle_compose(
     args: &ComposeArgs,
     transaction: &josh_core::cache::Transaction,
 ) -> anyhow::Result<()> {
+    #[cfg(windows)]
+    {
+        let _ = (args, transaction);
+        anyhow::bail!("josh compose is not supported on Windows");
+    }
+
+    #[cfg(not(windows))]
     match &args.command {
         ComposeCommand::Run(run_args) => handle_run(run_args, transaction),
         ComposeCommand::ListImages(list_args) => handle_list_images(list_args, transaction),

josh-compose-podman/src/lib.rs

@@ -38,12 +38,19 @@ impl Default for PodmanRuntime {
 /// Host uid/gid of the invoking user — the identity container steps run as and
 /// artifacts are chowned to. This is a container mechanic; the scheduler never
 /// needs to know it.
+#[cfg(unix)]
 fn host_uid_gid() -> (u32, u32) {
     let uid = unsafe { libc::getuid() };
     let gid = unsafe { libc::getgid() };
     (uid, gid)
 }
 
+/// Unreachable: `josh compose` is refused on Windows before any container runs.
+#[cfg(windows)]
+fn host_uid_gid() -> (u32, u32) {
+    unreachable!("josh compose is not supported on Windows")
+}
+
 fn host_identity() -> String {
     let (uid, gid) = host_uid_gid();
     format!("{uid}:{gid}")

josh-core/src/cache/transaction.rs

@@ -1841,6 +1841,9 @@ mod tests {
         assert!(seen.is_empty());
     }
 
+    // Windows cannot hold this ref at all: a device name is illegal as a path component, so
+    // an upstream whose path contains one cannot be mirrored there.
+    #[cfg(unix)]
     #[test]
     fn for_each_ref_prefixed_takes_a_prefix_no_worktree_could_hold() {
         // Upstream namespaces are percent-encoded repository paths, so a prefix component
@@ -1865,6 +1868,8 @@ mod tests {
         assert_eq!(seen, ["refs/josh/upstream/aux/refs/heads/main"]);
     }
 
+    // Identifies the file by inode, so it only makes sense on unix.
+    #[cfg(unix)]
     #[test]
     fn update_ref_to_the_value_a_ref_already_has_writes_nothing() {
         use std::os::unix::fs::MetadataExt;

josh-core/src/filter/tree.rs

@@ -765,7 +765,7 @@ fn intersect_inner(
 
 /// The raw bytes of a path component, for matching against tree entry names.
 fn component_bytes(c: &std::ffi::OsStr) -> &[u8] {
-    std::os::unix::ffi::OsStrExt::as_bytes(c)
+    josh_gix_ext::component_bytes(c)
 }
 
 /// Read `oid` as raw tree bytes, or `None` if it is missing or not a tree. Uncached: the

josh-core/src/submodules.rs

@@ -104,7 +104,10 @@ pub fn update_gitmodules(
         .write_to(&mut output)
         .context("Failed to write gitmodules")?;
 
-    String::from_utf8(output).context("Invalid UTF-8 in gitmodules")
+    let content = String::from_utf8(output).context("Invalid UTF-8 in gitmodules")?;
+    // gix-config writes the platform's newline, but this ends up in a blob: the same
+    // filter has to produce the same object on every platform.
+    Ok(content.replace("\r\n", "\n"))
 }
 
 #[cfg(test)]

josh-gix-ext/src/lib.rs

@@ -4,6 +4,21 @@ use std::collections::HashMap;
 
 use gix_object::WriteTo;
 
+/// The raw bytes of a path component, for matching against git tree entry names.
+#[cfg(unix)]
+pub fn component_bytes(c: &std::ffi::OsStr) -> &[u8] {
+    std::os::unix::ffi::OsStrExt::as_bytes(c)
+}
+
+/// The raw bytes of a path component, for matching against git tree entry names: their UTF-8
+/// encoding. Panics on a component that is not valid Unicode, which cannot name a tree entry.
+#[cfg(windows)]
+pub fn component_bytes(c: &std::ffi::OsStr) -> &[u8] {
+    c.to_str()
+        .expect("path component is not valid Unicode")
+        .as_bytes()
+}
+
 pub mod graph;
 pub mod merge;
 pub mod revwalk;
@@ -121,7 +136,7 @@ pub fn path_entry(
             return Ok(None);
         }
         let parsed = gix_object::TreeRef::from_bytes(&buffer, gix_hash::Kind::Sha1)?;
-        let name = std::os::unix::ffi::OsStrExt::as_bytes(component.as_os_str());
+        let name = component_bytes(component.as_os_str());
         let Some(entry) = parsed.entries.iter().find(|e| e.filename == name) else {
             return Ok(None);
         };

josh-proxy/Cargo.toml

@@ -47,6 +47,8 @@ toml.workspace = true
 tracing.workspace = true
 tracing-subscriber.workspace = true
 tokio-util.workspace = true
+dunce.workspace = true
+socket2.workspace = true
 tempfile.workspace = true
 gix.workspace = true
 juniper.workspace = true

josh-proxy/src/bin/josh-proxy.rs

@@ -177,7 +177,7 @@ async fn run_proxy(args: josh_proxy::cli::Args) -> anyhow::Result<i32> {
     let (shutdown_tx, _shutdown_rx) = broadcast::channel(1);
 
     let addr: SocketAddr = format!("[::]:{}", args.port).parse()?;
-    let listener = tokio::net::TcpListener::bind(addr).await?;
+    let listener = make_listener(addr)?;
 
     let server_future = async move { axum::serve(listener, app).await.context("Server error") };
 
@@ -252,6 +252,37 @@ fn update_hook(refname: &str, old: &str, new: &str) -> anyhow::Result<i32> {
     }
 }
 
+/// Bind the listener dual-stack: a bare [::] socket accepts IPv4 on Linux, where bindv6only
+/// defaults off, but is v6-only on Windows.
+fn make_listener(addr: SocketAddr) -> anyhow::Result<tokio::net::TcpListener> {
+    let socket = socket2::Socket::new(
+        socket2::Domain::IPV6,
+        socket2::Type::STREAM,
+        Some(socket2::Protocol::TCP),
+    )?;
+    socket.set_only_v6(false)?;
+    socket.bind(&addr.into())?;
+    socket.listen(1024)?;
+    socket.set_nonblocking(true)?;
+    Ok(tokio::net::TcpListener::from_std(socket.into())?)
+}
+
+/// The hook this process was invoked as, for the Windows shim hooks. On unix the hooks are
+/// symlinks and argv[0] names them, so the environment is not consulted.
+fn hook_from_env() -> Option<&'static str> {
+    #[cfg(windows)]
+    {
+        static JOSH_PROXY_HOOK: std::sync::LazyLock<Option<String>> =
+            std::sync::LazyLock::new(|| std::env::var("JOSH_PROXY_HOOK").ok());
+        JOSH_PROXY_HOOK.as_deref()
+    }
+
+    #[cfg(not(windows))]
+    {
+        None
+    }
+}
+
 fn pre_receive_hook() -> anyhow::Result<i32> {
     let repo_update = repo_update_from_env()?;
 
@@ -286,13 +317,13 @@ fn main() -> std::process::ExitCode {
     // process to do the actual computation while taking advantage of the
     // cached data already loaded into the main process's memory.
     if let [a0, a1, a2, a3, ..] = &std::env::args().collect::<Vec<_>>().as_slice()
-        && a0.ends_with("/update")
+        && (a0.ends_with("/update") || hook_from_env() == Some("update"))
     {
         return std::process::ExitCode::from(update_hook(a1, a2, a3).unwrap_or(1) as u8);
     }
 
     if let [a0, ..] = &std::env::args().collect::<Vec<_>>().as_slice()
-        && a0.ends_with("/pre-receive")
+        && (a0.ends_with("/pre-receive") || hook_from_env() == Some("pre-receive"))
     {
         eprintln!("josh-proxy: pre-receive hook");
         return std::process::ExitCode::from(match pre_receive_hook() {