test(base64-rs): add GNU differential parity harness
Runs both binaries over 154 cases and compares stdout, stderr and exit code: encoding at every wrap width, decode round-trips, canonical-tail rules, padding placement, concatenated streams, every prefix of a known encoding, usage errors and file operands. Locates a genuine GNU base64 rather than trusting /usr/bin/base64, since distributions increasingly ship uutils there and it disagrees with GNU on most malformed input — running against it fails 51 of the 154 cases, which doubles as a negative control for the harness itself. The three known differences are asserted as expected rather than ignored.
This commit is contained in:
@@ -0,0 +1,204 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Differential test: this base64 against GNU coreutils base64.
|
||||
|
||||
Runs both binaries over the same inputs and compares stdout, stderr and
|
||||
exit code. Linux-only — it needs a real GNU base64 to diff against, so it
|
||||
cannot run on the Windows machine the tool is actually built for.
|
||||
|
||||
cargo build --release && python3 tests/parity.py
|
||||
|
||||
Note that many distributions now ship uutils coreutils as /usr/bin/base64,
|
||||
which differs from GNU on malformed input. This script looks for a genuine
|
||||
GNU build and refuses to run against anything else.
|
||||
"""
|
||||
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
MINE = os.path.join(ROOT, "target", "release", "base64")
|
||||
|
||||
# Differences that are intentional; see README.md.
|
||||
EXPECTED_DIFF = {
|
||||
# We skip '\r' alongside '\n' so CRLF base64 files work on Windows.
|
||||
"decode CRLF wrapped",
|
||||
# Our --help omits GNU's own support URLs.
|
||||
"help",
|
||||
"help after other flags",
|
||||
}
|
||||
|
||||
|
||||
def find_gnu():
|
||||
"""Locate a genuine GNU base64, ignoring uutils drop-in replacements."""
|
||||
for name in ("gnubase64", "base64", "gbase64"):
|
||||
path = shutil.which(name)
|
||||
if not path:
|
||||
continue
|
||||
try:
|
||||
banner = subprocess.run(
|
||||
[path, "--version"], capture_output=True, text=True, timeout=10
|
||||
).stdout
|
||||
except OSError:
|
||||
continue
|
||||
if "GNU coreutils" in banner:
|
||||
return path, banner.splitlines()[0]
|
||||
return None, None
|
||||
|
||||
|
||||
GNU, GNU_BANNER = find_gnu()
|
||||
|
||||
|
||||
def norm(b, gnu_name):
|
||||
"""Strip argv[0] and locale noise so the two are comparable."""
|
||||
b = b.replace(gnu_name.encode(), b"base64")
|
||||
b = b.replace(os.path.basename(gnu_name).encode(), b"base64")
|
||||
# GNU quotes with U+2018/2019 outside the C locale; we always use ASCII.
|
||||
b = b.replace("‘".encode(), b"'").replace("’".encode(), b"'")
|
||||
return b
|
||||
|
||||
|
||||
def run(exe, args, data):
|
||||
# C locale: GNU otherwise uses en_GB spelling and Unicode quotes, neither
|
||||
# of which a Windows console would produce.
|
||||
env = dict(os.environ, LC_ALL="C")
|
||||
p = subprocess.run([exe] + args, input=data, capture_output=True, env=env)
|
||||
return p.returncode, norm(p.stdout, GNU), norm(p.stderr, GNU)
|
||||
|
||||
|
||||
results = []
|
||||
|
||||
|
||||
def t(name, args, data=b""):
|
||||
mine = run(MINE, args, data)
|
||||
gnu = run(GNU, args, data)
|
||||
ok = mine == gnu
|
||||
if not ok:
|
||||
tag = "DIFF(expected)" if name in EXPECTED_DIFF else "FAIL"
|
||||
print(f"[{tag}] {name} args={args}")
|
||||
if mine[0] != gnu[0]:
|
||||
print(f" exit: mine={mine[0]} gnu={gnu[0]}")
|
||||
if mine[1] != gnu[1]:
|
||||
print(f" out : mine={mine[1][:90]!r}")
|
||||
print(f" gnu ={gnu[1][:90]!r}")
|
||||
if mine[2] != gnu[2]:
|
||||
print(f" err : mine={mine[2][:120]!r}")
|
||||
print(f" gnu ={gnu[2][:120]!r}")
|
||||
results.append((name, ok or name in EXPECTED_DIFF))
|
||||
|
||||
|
||||
def main():
|
||||
random.seed(7)
|
||||
big = bytes(random.getrandbits(8) for _ in range(200_000))
|
||||
|
||||
# ── encoding ──────────────────────────────────────────────────────
|
||||
t("encode empty", [], b"")
|
||||
for n in (1, 2, 3, 4, 5, 56, 57, 58, 76, 1000):
|
||||
t(f"encode {n} bytes", [], (bytes(range(256)) * 10)[:n])
|
||||
t("encode all byte values", [], bytes(range(256)))
|
||||
t("encode 200KB binary", [], big)
|
||||
sample = b"hello world, this is a longer sample string for wrapping"
|
||||
for w in ("0", "1", "2", "3", "4", "5", "76", "77", "1000"):
|
||||
t(f"encode wrap {w}", ["-w", w], sample)
|
||||
t(f"encode wrap {w} joined", [f"-w{w}"], sample)
|
||||
t(f"encode wrap {w} long", ["--wrap=" + w], big[:5000])
|
||||
|
||||
# ── decoding ──────────────────────────────────────────────────────
|
||||
for n in (0, 1, 2, 3, 4, 5, 100, 1000):
|
||||
enc = subprocess.run([GNU], input=big[:n], capture_output=True).stdout
|
||||
t(f"decode roundtrip {n}", ["-d"], enc)
|
||||
t("decode 200KB roundtrip", ["-d"],
|
||||
subprocess.run([GNU], input=big, capture_output=True).stdout)
|
||||
t("decode unwrapped", ["-d"], b"aGVsbG8gd29ybGQ=")
|
||||
t("decode no trailing newline", ["-d"], b"aGVsbG8=")
|
||||
t("decode with newlines", ["-d"], b"aGVs\nbG8=\n")
|
||||
t("decode CRLF wrapped", ["-d"], b"aGVs\r\nbG8=\r\n")
|
||||
t("decode empty", ["-d"], b"")
|
||||
t("decode only newline", ["-d"], b"\n")
|
||||
t("decode spaces garbage", ["-d"], b"aGVs bG8=")
|
||||
t("decode spaces garbage -i", ["-di"], b"aGVs bG8=")
|
||||
t("decode junk", ["-d"], b"aG!!Vs*bG8=")
|
||||
t("decode junk -i", ["-di"], b"aG!!Vs*bG8=")
|
||||
t("decode junk only -i", ["-di"], b"!!!!")
|
||||
t("decode nul bytes -i", ["-di"], b"aGVs\x00bG8=")
|
||||
t("decode high bytes", ["-d"], b"aGVs\xffbG8=")
|
||||
t("decode high bytes -i", ["-di"], b"aGVs\xffbG8=")
|
||||
t("decode urlsafe chars", ["-d"], b"_-_-")
|
||||
|
||||
# Unpadded tails are legal only when the leftover bits are zero.
|
||||
for s in (b"a", b"aA", b"aQ", b"ag", b"aG", b"aGA", b"aGE", b"aGV",
|
||||
b"aB", b"aC", b"aGB", b"aGC", b"aGVs", b"aGVsb", b"aGVsbG",
|
||||
b"aGVsbG8"):
|
||||
t(f"decode tail {s.decode()}", ["-d"], s)
|
||||
|
||||
# Padding placement, and streams concatenated after padding.
|
||||
for s in (b"=", b"====", b"=aGVsbG8=", b"aG=sbG8=", b"aGVsbG8===",
|
||||
b"aA==", b"aQ==", b"aG==", b"aGA=", b"aGV=", b"aA=", b"aA=x",
|
||||
b"aG=x", b"aGA=x", b"aA==x", b"aA==aGVs", b"aGVsbG8=x",
|
||||
b"aGVsbG8=aGVs", b"aGVsbG8=\naGVs", b"aGVsbG8=aGVsbG8="):
|
||||
t(f"decode pad {s.decode()}", ["-d"], s)
|
||||
|
||||
# Every prefix of a known-good encoding, strict and lenient.
|
||||
enc = subprocess.run([GNU], input=b"hello world!", capture_output=True).stdout.strip()
|
||||
for i in range(1, len(enc) + 1):
|
||||
t(f"decode prefix {i}", ["-d"], enc[:i])
|
||||
t(f"decode prefix {i} -i", ["-di"], enc[:i])
|
||||
|
||||
# ── options and usage errors ──────────────────────────────────────
|
||||
t("help", ["--help"])
|
||||
t("help after other flags", ["--decode", "--help"])
|
||||
t("abbrev --dec", ["--dec"], b"aGVsbG8=")
|
||||
t("abbrev --ign", ["--ign", "-d"], b"aGV s bG8=")
|
||||
t("abbrev --w", ["--w", "4"], b"hello world")
|
||||
t("bare --", ["--"], b"hello")
|
||||
t("cluster -di", ["-di"], b"aG!Vs bG8=")
|
||||
t("cluster -dw0", ["-dw0"], b"aGVsbG8=")
|
||||
t("wrap missing arg", ["-w"])
|
||||
t("wrap bad value", ["-w", "abc"])
|
||||
t("wrap negative", ["-w", "-5"])
|
||||
t("unknown short", ["-z"])
|
||||
t("unknown long", ["--nope"])
|
||||
t("ambiguous long", ["--i"])
|
||||
t("extra operand", ["a.txt", "b.txt"])
|
||||
t("missing file", ["/nonexistent/nope.txt"])
|
||||
t("directory as file", ["/tmp"])
|
||||
t("dash is stdin", ["-"], b"hello")
|
||||
t("double dash then dash", ["--", "-"], b"hello")
|
||||
|
||||
# ── file operands ─────────────────────────────────────────────────
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
raw = os.path.join(tmp, "sample.bin")
|
||||
with open(raw, "wb") as fh:
|
||||
fh.write(big[:9999])
|
||||
t("encode from file", [raw])
|
||||
t("encode from file wrap 0", ["-w0", raw])
|
||||
|
||||
encoded = os.path.join(tmp, "sample.b64")
|
||||
with open(encoded, "wb") as fh:
|
||||
fh.write(subprocess.run([GNU], input=big[:9999], capture_output=True).stdout)
|
||||
t("decode from file", ["-d", encoded])
|
||||
|
||||
passed = sum(1 for _, ok in results if ok)
|
||||
print(f"\n{passed}/{len(results)} cases match GNU base64")
|
||||
failed = [n for n, ok in results if not ok]
|
||||
if failed:
|
||||
print("failing:", ", ".join(failed))
|
||||
return 1
|
||||
print("all parity checks passed")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if not os.path.exists(MINE):
|
||||
sys.exit(f"build it first: cargo build --release (looked for {MINE})")
|
||||
if GNU is None:
|
||||
sys.exit(
|
||||
"no GNU coreutils base64 found. Distributions increasingly ship "
|
||||
"uutils as /usr/bin/base64, which differs on malformed input; "
|
||||
"install GNU coreutils to run this comparison."
|
||||
)
|
||||
print(f"comparing {MINE}\n against {GNU} ({GNU_BANNER})\n")
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user