From 5b3e6994c466b96b193459a7a8f746b30757c5c3 Mon Sep 17 00:00:00 2001 From: FrederikLeed <37104276+FrederikLeed@users.noreply.github.com> Date: Thu, 12 Mar 2026 14:04:00 +0100 Subject: [PATCH 1/2] Add Python streaming repacker (PyPsiRepacker) Streaming line-by-line NTLM hash converter that runs with near-zero RAM, enabling conversion of 70GB+ Pwned Passwords files on machines with as little as 16GB. Includes sort-order verification to guard against unsorted input. Updated README with both Python and C++ usage. Co-Authored-By: Claude Opus 4.6 --- README.md | 53 ++++++++++++++++-- pypsirepacker/__init__.py | 0 pypsirepacker/__main__.py | 46 +++++++++++++++ pypsirepacker/repacker.py | 114 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 208 insertions(+), 5 deletions(-) create mode 100644 pypsirepacker/__init__.py create mode 100644 pypsirepacker/__main__.py create mode 100644 pypsirepacker/repacker.py diff --git a/README.md b/README.md index 7b79be5..223547f 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,58 @@ # PsiRepacker -Repacking NT hash files from Troy Hunt to binary format for Get-Badpasswords solution -# Usage -Simply run the executable with two parameters: +Repacking NT hash files from Troy Hunt's [Pwned Passwords](https://haveibeenpwned.com/Passwords) to compact binary format for the [Get-Badpasswords](https://github.com/yourlink) solution. + +## Versions + +### PyPsiRepacker (Python — recommended) + +A streaming Python implementation that converts NTLM hash text files to binary with **near-zero memory usage**. Unlike the C++ version, it does not load all entries into RAM — it processes line-by-line, relying on the input already being sorted (as produced by [PwnedPasswordsDownloader](https://github.com/HaveIBeenPwned/PwnedPasswordsDownloader)). + +**Requirements:** Python 3.6+ (no external dependencies) + +**RAM usage:** ~0 (streams line by line) ``` -PsiRepacker +python -m pypsirepacker ``` For example: ``` -PsiRepacker "C:\pwned-passwords.txt" "C:\pwned-passwords.bin" +python -m pypsirepacker "C:\pwned-passwords-ntlm.txt" "C:\pwned-passwords-ntlm.bin" +``` + +Options: + +| Flag | Description | +|------|-------------| +| `--no-verify` | Skip sort-order verification (not recommended) | + +Sort-order verification is enabled by default. If a hash is found out of order, conversion aborts immediately — this means the input was not pre-sorted and a streaming approach cannot produce a valid binary file. + +### PsiRepacker (C++ — original) + +The original C++ implementation loads all hashes into memory, sorts them, and writes the binary output. This requires **~50 GB RAM** for a full Pwned Passwords NTLM file (~1.7 billion entries) and is Windows-only. + +``` +PsiRepacker ``` + +Use this version if your input file is **not pre-sorted** and you have sufficient RAM. + +## Binary output format + +The output `.bin` file has the following structure: + +| Offset | Size | Description | +|--------|------|-------------| +| 0 | 8 bytes | Entry count (uint64, little-endian) | +| 8 | 16 bytes each | NTLM hashes packed as raw binary, sorted | + +Each 32-character hex NTLM hash (e.g. `A4F49C406510BDCAB6824EE7C30FD852`) is stored as 16 raw bytes — halving the size of the text representation and enabling fast binary search. + +## Workflow + +1. Download NTLM hashes using [PwnedPasswordsDownloader](https://github.com/HaveIBeenPwned/PwnedPasswordsDownloader) with the `-n` (NTLM) flag +2. Run PsiRepacker (Python or C++) to convert to binary +3. Use the `.bin` file with Get-Badpasswords to audit Active Directory passwords diff --git a/pypsirepacker/__init__.py b/pypsirepacker/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pypsirepacker/__main__.py b/pypsirepacker/__main__.py new file mode 100644 index 0000000..3508538 --- /dev/null +++ b/pypsirepacker/__main__.py @@ -0,0 +1,46 @@ +""" +Command-line entry point for PyPsiRepacker. + +Usage: + python -m pypsirepacker [--no-verify] +""" + +import argparse +import sys +import time + +from .repacker import repack + + +def main(): + parser = argparse.ArgumentParser( + prog="pypsirepacker", + description=( + "Convert Troy Hunt Pwned Passwords NTLM hash files from text to " + "compact binary format for Get-Badpasswords." + ), + ) + parser.add_argument("input", help="Path to NTLM hash text file (HASH:count per line)") + parser.add_argument("output", help="Path for output binary file") + parser.add_argument( + "--no-verify", + action="store_true", + help="Skip sort-order verification (not recommended)", + ) + + args = parser.parse_args() + + start = time.perf_counter() + + try: + count = repack(args.input, args.output, verify_sort=not args.no_verify) + except (FileNotFoundError, ValueError) as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + + elapsed = time.perf_counter() - start + print(f"Completed in {elapsed:.1f} seconds.") + + +if __name__ == "__main__": + main() diff --git a/pypsirepacker/repacker.py b/pypsirepacker/repacker.py new file mode 100644 index 0000000..9f8e8bd --- /dev/null +++ b/pypsirepacker/repacker.py @@ -0,0 +1,114 @@ +""" +Streaming NTLM hash repacker for Troy Hunt's Pwned Passwords lists. + +Converts text-based NTLM hash files (HASH:count format) to compact binary +format for use with Get-Badpasswords. Processes line-by-line with near-zero +memory usage, relying on the input being pre-sorted (as produced by +PwnedPasswordsDownloader). +""" + +import struct +import sys +import os + + +HASH_HEX_LEN = 32 +HASH_BIN_LEN = 16 +HEADER_SIZE = 8 # uint64 little-endian + + +def count_lines(filepath: str) -> int: + """Count lines in a text file efficiently using a raw byte buffer.""" + count = 0 + buf_size = 1024 * 1024 # 1 MB chunks + with open(filepath, "rb") as f: + while True: + buf = f.read(buf_size) + if not buf: + break + count += buf.count(b"\n") + return count + + +def repack(input_path: str, output_path: str, *, verify_sort: bool = True) -> int: + """ + Convert a Troy Hunt NTLM hash text file to sorted binary format. + + The input file must have lines in the format: <32-char NTLM hash>: + The output is a binary file: 8-byte entry count (uint64 LE) followed by + packed 16-byte hash entries. + + Args: + input_path: Path to the NTLM text file (e.g. pwned-passwords-ntlm.txt). + output_path: Path for the output binary file. + verify_sort: If True, verify that input hashes are in sorted order and + abort if not. Default True. + + Returns: + Number of entries written. + + Raises: + FileNotFoundError: If input file does not exist. + ValueError: If the file format is invalid or sort order is broken. + """ + if not os.path.isfile(input_path): + raise FileNotFoundError(f"Input file not found: {input_path}") + + # Validate format by peeking at first line + with open(input_path, "r") as f: + first_line = f.readline().strip() + + if len(first_line) < HASH_HEX_LEN or first_line[HASH_HEX_LEN] != ":": + raise ValueError( + f"Not a valid Troy Hunt NTLM hash file. " + f"Expected format: [32-char NTLM hash]:[count]. " + f"Got: {first_line[:50]}" + ) + + # Count entries for the binary header + print(f"Counting entries in {input_path}...") + total = count_lines(input_path) + print(f"Found {total:,} entries.") + + # Stream convert + print(f"Converting to binary: {output_path}") + written = 0 + prev_hash = None + + with open(input_path, "r") as fin, open(output_path, "wb") as fout: + fout.write(struct.pack("14,} / {total:,} ({pct:.1f}%)") + + print(f"Done. Wrote {written:,} entries ({written * HASH_BIN_LEN:,} bytes + {HEADER_SIZE} byte header).") + return written From 0f60d1545abea1993001eea6700223c45884fd21 Mon Sep 17 00:00:00 2001 From: FrederikLeed <37104276+FrederikLeed@users.noreply.github.com> Date: Thu, 12 Mar 2026 15:24:11 +0100 Subject: [PATCH 2/2] Update README with Windows prerequisites, usage examples, and benchmark data Add step-by-step Windows setup instructions, cross-platform usage examples, and real-world benchmark from converting ~2 billion NTLM hashes (70 GB text to 30.6 GB binary in ~14.5 minutes). Co-Authored-By: Claude Opus 4.6 --- README.md | 51 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 223547f..999168e 100644 --- a/README.md +++ b/README.md @@ -12,17 +12,41 @@ A streaming Python implementation that converts NTLM hash text files to binary w **RAM usage:** ~0 (streams line by line) +#### Prerequisites (Windows) + +1. **Install Python 3.6 or later** — download from [python.org](https://www.python.org/downloads/) + - During installation, check **"Add Python to PATH"** +2. **Verify** by opening Command Prompt or PowerShell: + ``` + python --version + ``` +3. **Clone or download** this repository: + ``` + git clone https://github.com/improsec/PsiRepacker.git + cd PsiRepacker + ``` + +#### Usage + +Run from the repository root: + ``` python -m pypsirepacker ``` -For example: +**Windows example (Command Prompt or PowerShell):** ``` -python -m pypsirepacker "C:\pwned-passwords-ntlm.txt" "C:\pwned-passwords-ntlm.bin" +python -m pypsirepacker "D:\output\hashes\pwnedpasswords_ntlm.txt" "D:\output\bin\pwnedpasswords_ntlm.bin" ``` -Options: +**Linux / macOS example:** + +``` +python3 -m pypsirepacker /data/pwnedpasswords_ntlm.txt /data/pwnedpasswords_ntlm.bin +``` + +#### Options | Flag | Description | |------|-------------| @@ -30,6 +54,27 @@ Options: Sort-order verification is enabled by default. If a hash is found out of order, conversion aborts immediately — this means the input was not pre-sorted and a streaming approach cannot produce a valid binary file. +#### Example run (Pwned Passwords v10, ~2 billion NTLM hashes) + +| | File | Size | +|---|---|---| +| **Input** | `pwnedpasswords_ntlm.txt` | ~70 GB | +| **Output** | `pwnedpasswords_ntlm.bin` | ~30.6 GB | + +``` +Counting entries in pwnedpasswords_ntlm.txt... +Found 2,052,742,897 entries. +Converting to binary: pwnedpasswords_ntlm.bin + 1,800,000,000 / 2,052,742,897 (87.7%) + 1,850,000,000 / 2,052,742,897 (90.1%) + 1,900,000,000 / 2,052,742,897 (92.6%) + 1,950,000,000 / 2,052,742,897 (95.0%) + 2,000,000,000 / 2,052,742,897 (97.4%) + 2,050,000,000 / 2,052,742,897 (99.9%) +Done. Wrote 2,052,742,897 entries (32,843,886,352 bytes + 8 byte header). +Completed in 870.2 seconds. +``` + ### PsiRepacker (C++ — original) The original C++ implementation loads all hashes into memory, sorts them, and writes the binary output. This requires **~50 GB RAM** for a full Pwned Passwords NTLM file (~1.7 billion entries) and is Windows-only.