diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index c2a0ca2..ebb1659 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -45,9 +45,9 @@ jobs: target: x86_64 - runner: ubuntu-22.04 target: x86 - - runner: ubuntu-22.04 + - runner: ubuntu-22.04-arm target: aarch64 - - runner: ubuntu-22.04 + - runner: ubuntu-22.04-arm target: armv7 - runner: ubuntu-22.04 target: ppc64le @@ -58,9 +58,6 @@ jobs: python-version: 3.x - name: Build wheels uses: PyO3/maturin-action@db323e2cf5679b7feb8bcb561a36b27a0bc19e79 # v1 - env: - # Ensure ring's ARM assembly sees an explicit architecture on aarch64 (glibc) - CFLAGS_aarch64_unknown_linux_gnu: -D__ARM_ARCH=8 with: target: ${{ matrix.platform.target }} args: --release --out dist --find-interpreter @@ -83,9 +80,9 @@ jobs: target: x86_64 - runner: ubuntu-22.04 target: x86 - - runner: ubuntu-22.04 + - runner: ubuntu-22.04-arm target: aarch64 - - runner: ubuntu-22.04 + - runner: ubuntu-22.04-arm target: armv7 steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 @@ -94,9 +91,6 @@ jobs: python-version: 3.x - name: Build wheels uses: PyO3/maturin-action@db323e2cf5679b7feb8bcb561a36b27a0bc19e79 # v1 - env: - # Ensure ring's ARM assembly sees an explicit architecture on aarch64 (musl) - CFLAGS_aarch64_unknown_linux_musl: -D__ARM_ARCH=8 with: target: ${{ matrix.platform.target }} args: --release --out dist --find-interpreter diff --git a/src/tiktoken.rs b/src/tiktoken.rs index 9c5a290..f4eb1bc 100644 --- a/src/tiktoken.rs +++ b/src/tiktoken.rs @@ -305,12 +305,7 @@ impl CoreBPE { let token_is_all_space = |token| { self.decoder .get(token) - .map(|token_bytes| { - token_bytes - .iter() - .rev() - .all(|&b| [b' ', b'\n', b'\t'].contains(&b)) - }) + .map(|token_bytes| token_bytes.iter().rev().all(|&b| b" \n\t".contains(&b))) .unwrap_or(false) }; if last_piece_token_len > 0