Files
meshcore-open/tools/aeic/eval/compare_codecs.py
T
zjs81 aafd541b0f Add an AEIC quality and rate evaluation harness
tools/aeic/eval runs a folder of photos through the same ONNX graphs the
app downloads, counts exact bytes with the app's own entropy layer and
rANS coder (aeic_bytes.dart, checked byte-for-byte against the golden
recordings with --selftest), and scores what the receiver sees with
LPIPS, DISTS, CLIP, MS-SSIM and PSNR.

Variants cover preprocessing (stretch, crops, letterbox), subject
selection, the send sheet's re-roll tweaks, quantisation-step rate
control (fixed steps and per-image byte budgets) and two-part encodes.
compare_codecs.py compares AEIC with JPEG XL and AVIF at the same byte
budgets.
2026-11-14 20:58:43 -07:00

257 lines
10 KiB
Python

#!/usr/bin/env python3
"""Rate-distortion comparison: AEIC (as shipped and with step rate control)
against JPEG XL and AVIF at the same byte budgets.
python -I tools/aeic/eval/compare_codecs.py --images ~/aeic-data/kodak --out ~/aeic-runs/rd
AEIC is swept over every quantisation step in STEP_LADDER (step 1.0 is the
stream the app sends today) and also fitted per image to 1 and 2 packets.
JPEG XL and AVIF are fitted per image to each byte budget: for every candidate
size the quality setting is searched to fit, the result is decoded and
upscaled to the display size, and the size with the lowest LPIPS is kept. That
choice uses the original, which the sender has, so it is a fair best case for
the conventional codecs rather than an oracle.
AVIF is counted twice. `avif` is the real .avif file, whose ftyp+meta boxes
cost about 275 bytes before any image data. `avif_av1` counts only the AV1
payload (the mdat contents), as if the receiver rebuilt the fixed container
from a template; that is AVIF's best case for a custom wire format.
Everything is scored against the source at the same display size (long side
512, the AEIC output size). Needs `imagecodecs` with JPEG XL support on top of
the eval.py requirements.
"""
from __future__ import annotations
import argparse
import importlib.util
import json
import struct
import sys
from pathlib import Path
import numpy as np
from PIL import Image
HERE = Path(__file__).resolve().parent
_spec = importlib.util.spec_from_file_location("aeic_eval", HERE / "eval.py")
ev = importlib.util.module_from_spec(_spec)
sys.modules["aeic_eval"] = ev # dataclasses look their module up here
_spec.loader.exec_module(ev) # type: ignore[union-attr]
import imagecodecs # noqa: E402 (after eval.py, which loads torch first)
from concurrent.futures import ThreadPoolExecutor # noqa: E402
POOL = ThreadPoolExecutor(max_workers=12)
PACKET_BUDGETS = {"1 packet": ev.CHUNK_FIRST_CAPACITY,
"2 packets": ev.CHUNK_FIRST_CAPACITY + ev.CHUNK_CAPACITY}
BUDGETS = [100, 157, 200, 250, 315, 400, 500, 700]
SIZES = [16, 24, 32, 40, 48, 56, 64, 80, 96, 128, 160, 192]
JXL_EFFORT = 9
AVIF_SPEED = 2
DISPLAY = ev.SIZE
def display_size(img: Image.Image, long_side: int) -> tuple[int, int]:
s = long_side / max(img.size)
return (max(1, round(img.width * s)), max(1, round(img.height * s)))
def mdat_payload(avif: bytes) -> int:
"""Size of the AV1 data in an .avif file (its mdat box contents)."""
i = 0
while i + 8 <= len(avif):
size, kind = struct.unpack(">I4s", avif[i:i + 8])
header = 8
if size == 1:
size = struct.unpack(">Q", avif[i + 8:i + 16])[0]
header = 16
elif size == 0:
size = len(avif) - i
if kind == b"mdat":
return size - header
i += size
raise ValueError("no mdat box")
def _jxl(arr: np.ndarray, q: float) -> bytes:
return imagecodecs.jpegxl_encode(arr, distance=q, effort=JXL_EFFORT)
def _avif(arr: np.ndarray, q: float) -> bytes:
return imagecodecs.avif_encode(arr, level=int(q), speed=AVIF_SPEED)
# name -> (encode(arr, q), decode, bytes counted, q for the smallest file,
# q for the largest, integer q?)
CODECS = {
"jxl": (_jxl, imagecodecs.jpegxl_decode, len, 25.0, 0.05, False),
"avif": (_avif, imagecodecs.avif_decode, len, 0, 100, True),
"avif_av1": (_avif, imagecodecs.avif_decode, mdat_payload, 0, 100, True),
}
def codec_fit(name: str, src: Image.Image, long_side: int, budget: int) -> tuple[int, Image.Image] | None:
"""Best stream of codec `name` for `src` at `long_side` px within `budget`.
Returns (counted bytes, decoded image), or None if even the lowest quality
does not fit.
"""
enc, dec, count, q_small, q_large, integer = CODECS[name]
small = np.asarray(src.resize(display_size(src, long_side), Image.LANCZOS))
data = enc(small, q_small)
if count(data) > budget:
return None
best = data
lo, hi = q_small, q_large # lo always fits
for _ in range(12):
if integer:
if abs(hi - lo) <= 1:
break
mid = (lo + hi) // 2
else:
mid = (lo * hi) ** 0.5 # JPEG XL distance acts roughly logarithmically
data = enc(small, mid)
if count(data) <= budget:
best, lo = data, mid
else:
hi = mid
decoded = Image.fromarray(np.asarray(dec(best))[..., :3], "RGB")
return count(best), decoded
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--images", type=Path, required=True)
ap.add_argument("--out", type=Path, required=True)
ap.add_argument("--models", type=Path, default=ev.DEFAULT_MODELS)
ap.add_argument("--limit", type=int)
args = ap.parse_args()
images = ev.find_images(args.images, args.limit)
out: Path = args.out
work = out / "tensors"
(out / "img").mkdir(parents=True, exist_ok=True)
work.mkdir(parents=True, exist_ok=True)
import torch
gpu = torch.cuda.is_available()
codec = ev.Codec(args.models, args.models / ev.DECODER_GRAPH, gpu)
metrics = ev.Metrics("cuda" if gpu else "cpu")
# ---- AEIC: every step on the ladder, one Dart run for all byte counts.
srcs, aeic_y = {}, {}
files = []
for path in images:
stem = path.stem
src = ev.load_source(path)
srcs[stem] = src
side = codec.encode_side(ev.variant_stretch(src).square)
for step in ev.STEP_LADDER:
t = codec.encode_with_step(side, step)
tf = work / f"{stem}.s{step:.2f}.f32"
ev.write_tensor_file(tf, t)
aeic_y[(stem, step)] = t["y_hat"]
files.append(tf)
print(f"encoded {stem}", file=sys.stderr)
nbytes = {k: v["bytes"] for k, v in ev.count_bytes(args.models / ev.CDF_TABLE, files).items()}
for tf in files:
tf.unlink()
rows = []
emb = {}
def score(stem: str, shown: Image.Image) -> dict:
src = srcs[stem]
ref = src.resize(display_size(src, DISPLAY), Image.LANCZOS)
if stem not in emb:
emb[stem] = metrics.clip_embed(src)
s = metrics.score(ref, shown.resize(ref.size, Image.BICUBIC), emb[stem])
s.pop("clip_full")
return s
aeic_shown = {}
for path in images:
stem = path.stem
src = srcs[stem]
for step in ev.STEP_LADDER:
recon = codec.decode(aeic_y[(stem, step)])
shown = recon.resize(display_size(src, DISPLAY), Image.LANCZOS)
aeic_shown[(stem, step)] = shown
b = nbytes[str(work / f"{stem}.s{step:.2f}.f32")]
rows.append({"image": stem, "codec": "aeic", "step": step, "bytes": b, **score(stem, shown)})
print(f"scored AEIC {stem}", file=sys.stderr)
# ---- Conventional codecs: per image and budget, the best size that fits.
for name in CODECS:
for path in images:
stem = path.stem
src = srcs[stem]
for budget in BUDGETS:
best = None
# imagecodecs releases the GIL, so the size search runs in threads.
fits = list(POOL.map(lambda ls: codec_fit(name, src, ls, budget), SIZES))
for long_side, fit in zip(SIZES, fits):
if fit is None:
continue
nb, small = fit
shown = small.resize(display_size(src, DISPLAY), Image.BICUBIC)
ref = src.resize(shown.size, Image.LANCZOS)
lp = float(metrics.lpips(metrics._t(shown), metrics._t(ref)))
if best is None or lp < best[0]:
best = (lp, long_side, nb, shown)
if best is None:
continue
_, long_side, nb, shown = best
rows.append({"image": stem, "codec": name, "budget": budget, "px": long_side,
"bytes": nb, **score(stem, shown)})
if budget in PACKET_BUDGETS.values():
shown.save(out / "img" / f"{stem}.{name}{budget}.png")
print(f"scored {name} {stem}", file=sys.stderr)
# ---- AEIC fitted to the packet budgets, picked from the sweep.
for stem in srcs:
for label, budget in PACKET_BUDGETS.items():
cands = [r for r in rows if r["codec"] == "aeic" and r["image"] == stem and r["bytes"] <= budget]
if not cands:
cands = [max((r for r in rows if r["codec"] == "aeic" and r["image"] == stem),
key=lambda r: r["step"])]
pick = min(cands, key=lambda r: r["step"])
rows.append({**pick, "codec": f"aeic_fit{budget}", "budget": budget})
aeic_shown[(stem, pick["step"])].save(out / "img" / f"{stem}.aeic{budget}.png")
aeic_shown[(stem, 1.0)].save(out / "img" / f"{stem}.aeic_today.png")
srcs[stem].resize(display_size(srcs[stem], DISPLAY), Image.LANCZOS).save(out / "img" / f"{stem}.src.png")
(out / "rows.json").write_text(json.dumps(rows, indent=0))
def mean(rs, k):
return float(np.mean([r[k] for r in rs]))
def line(name, rs):
return (f"{name:<22}{mean(rs, 'bytes'):>7.0f}{mean(rs, 'lpips'):>8.3f}{mean(rs, 'dists'):>8.3f}"
f"{mean(rs, 'clip'):>7.3f}{mean(rs, 'ms_ssim'):>8.3f}{mean(rs, 'psnr'):>7.2f}")
print(f"\n{'':<22}{'bytes':>7}{'lpips':>8}{'dists':>8}{'clip':>7}{'msssim':>8}{'psnr':>7}")
print(line("AEIC today (step 1)", [r for r in rows if r["codec"] == "aeic" and r["step"] == 1.0]))
for label, budget in PACKET_BUDGETS.items():
print(f"-- {label} ({budget} B)")
print(line(" AEIC step-fit", [r for r in rows if r["codec"] == f"aeic_fit{budget}"]))
for name in CODECS:
rs = [r for r in rows if r["codec"] == name and r["budget"] == budget]
print(line(f" {name} ({len(rs)} img)", rs) if rs else f" {name}: does not fit")
print("\nAEIC curve")
for step in ev.STEP_LADDER:
print(line(f" step {step:.2f}", [r for r in rows if r["codec"] == "aeic" and r["step"] == step]))
for name in CODECS:
print(f"{name} curve")
for budget in BUDGETS:
rs = [r for r in rows if r["codec"] == name and r["budget"] == budget]
if rs:
print(line(f" <= {budget} B ({len(rs)} img)", rs))
if __name__ == "__main__":
main()