From 0f058dfb94fdd2a6b4ef32bd5a3be45bb5540fd2 Mon Sep 17 00:00:00 2001 From: Yuan Tong <13075180+tongyuantongyu@users.noreply.github.com> Date: Tue, 21 Jul 2026 01:42:35 -0700 Subject: [PATCH 1/2] Build arm wheel natively --- .github/workflows/CI.yml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index c2a0ca2..ebb1659 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -45,9 +45,9 @@ jobs: target: x86_64 - runner: ubuntu-22.04 target: x86 - - runner: ubuntu-22.04 + - runner: ubuntu-22.04-arm target: aarch64 - - runner: ubuntu-22.04 + - runner: ubuntu-22.04-arm target: armv7 - runner: ubuntu-22.04 target: ppc64le @@ -58,9 +58,6 @@ jobs: python-version: 3.x - name: Build wheels uses: PyO3/maturin-action@db323e2cf5679b7feb8bcb561a36b27a0bc19e79 # v1 - env: - # Ensure ring's ARM assembly sees an explicit architecture on aarch64 (glibc) - CFLAGS_aarch64_unknown_linux_gnu: -D__ARM_ARCH=8 with: target: ${{ matrix.platform.target }} args: --release --out dist --find-interpreter @@ -83,9 +80,9 @@ jobs: target: x86_64 - runner: ubuntu-22.04 target: x86 - - runner: ubuntu-22.04 + - runner: ubuntu-22.04-arm target: aarch64 - - runner: ubuntu-22.04 + - runner: ubuntu-22.04-arm target: armv7 steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 @@ -94,9 +91,6 @@ jobs: python-version: 3.x - name: Build wheels uses: PyO3/maturin-action@db323e2cf5679b7feb8bcb561a36b27a0bc19e79 # v1 - env: - # Ensure ring's ARM assembly sees an explicit architecture on aarch64 (musl) - CFLAGS_aarch64_unknown_linux_musl: -D__ARM_ARCH=8 with: target: ${{ matrix.platform.target }} args: --release --out dist --find-interpreter From f704ac26ec974365de379e9146927ce5b417cf9a Mon Sep 17 00:00:00 2001 From: Yuan Tong <13075180+tongyuantongyu@users.noreply.github.com> Date: Tue, 21 Jul 2026 01:53:00 -0700 Subject: [PATCH 2/2] Fix clippy & rustfmt error --- src/tiktoken.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/tiktoken.rs b/src/tiktoken.rs index 9c5a290..f4eb1bc 100644 --- a/src/tiktoken.rs +++ b/src/tiktoken.rs @@ -305,12 +305,7 @@ impl CoreBPE { let token_is_all_space = |token| { self.decoder .get(token) - .map(|token_bytes| { - token_bytes - .iter() - .rev() - .all(|&b| [b' ', b'\n', b'\t'].contains(&b)) - }) + .map(|token_bytes| token_bytes.iter().rev().all(|&b| b" \n\t".contains(&b))) .unwrap_or(false) }; if last_piece_token_len > 0