From aafd541b0f6b1f3f25b64cf3b3a868296559d922 Mon Sep 17 00:00:00 2001 From: zjs81 Date: Sat, 14 Nov 2026 20:58:43 -0700 Subject: [PATCH] Add an AEIC quality and rate evaluation harness tools/aeic/eval runs a folder of photos through the same ONNX graphs the app downloads, counts exact bytes with the app's own entropy layer and rANS coder (aeic_bytes.dart, checked byte-for-byte against the golden recordings with --selftest), and scores what the receiver sees with LPIPS, DISTS, CLIP, MS-SSIM and PSNR. Variants cover preprocessing (stretch, crops, letterbox), subject selection, the send sheet's re-roll tweaks, quantisation-step rate control (fixed steps and per-image byte budgets) and two-part encodes. compare_codecs.py compares AEIC with JPEG XL and AVIF at the same byte budgets. --- tools/aeic/README.md | 4 + tools/aeic/eval/README.md | 115 +++++ tools/aeic/eval/aeic_bytes.dart | 240 +++++++++ tools/aeic/eval/compare_codecs.py | 256 +++++++++ tools/aeic/eval/eval.py | 831 ++++++++++++++++++++++++++++++ tools/aeic/eval/requirements.txt | 10 + tools/aeic/eval/rois/kodak.json | 20 + 7 files changed, 1476 insertions(+) create mode 100644 tools/aeic/eval/README.md create mode 100644 tools/aeic/eval/aeic_bytes.dart create mode 100644 tools/aeic/eval/compare_codecs.py create mode 100644 tools/aeic/eval/eval.py create mode 100644 tools/aeic/eval/requirements.txt create mode 100644 tools/aeic/eval/rois/kodak.json diff --git a/tools/aeic/README.md b/tools/aeic/README.md index a246f15d..c3ce2c76 100644 --- a/tools/aeic/README.md +++ b/tools/aeic/README.md @@ -9,6 +9,10 @@ Dart side is checked against. They are committed here so `test/services/golden/` is reproducible. Nothing in the app runs them. +To measure reconstruction quality and byte cost instead, use the +[evaluation harness](eval/README.md), which runs on the downloaded model bundle +and needs no research checkout. + | script | produces | used by | |---|---|---| | `export_golden.py` | `golden/aeic_cdf_ft32.bin`, `golden/vectors/*.gv` | `rans_coder_test.dart`, `entropy_tables_test.dart` | diff --git a/tools/aeic/eval/README.md b/tools/aeic/eval/README.md new file mode 100644 index 00000000..effd2c10 --- /dev/null +++ b/tools/aeic/eval/README.md @@ -0,0 +1,115 @@ +# AEIC quality evaluation + +`eval.py` measures how good received images are and how many bytes they cost, +so codec changes can be compared with numbers rather than by eye. It runs the +same ONNX graphs the app downloads. Unlike the fixture scripts in the parent +directory, it needs no research checkout. + +For each photo and each preprocessing variant it: + +1. prepares the 512×512 codec input (the `stretch` variant mirrors the app's `_toSquareRgb`), +2. runs `aeic_entropy_side_fp32_op17.onnx` on CPU, +3. counts the exact bytes with `aeic_bytes.dart`, which calls the app's own entropy layer and rANS coder, +4. runs the decoder on `y_hat` (GPU when available), +5. restores the aspect ratio the way the receiver does and scores the result against the region that was sent. + +## Setup + +```bash +python3 -m venv ~/.venvs/aeic-eval # or: uv venv ~/.venvs/aeic-eval +~/.venvs/aeic-eval/bin/pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 +~/.venvs/aeic-eval/bin/pip install -r tools/aeic/eval/requirements.txt +``` + +You also need `dart` on PATH (`flutter pub get` already run), and the model +bundle. By default the bundle is read from `~/Documents/image_codec_models`; +use `--models` to point elsewhere. The first run downloads the LPIPS, DISTS and +CLIP weights into the torch and Hugging Face caches. + +Check that the byte counter matches the reference coder: + +```bash +dart run tools/aeic/eval/aeic_bytes.dart --selftest test/services/golden/e2e/*.aeicrec +``` + +## Running + +From the repository root: + +```bash +# Kodak (24 images, downloaded once) +~/.venvs/aeic-eval/bin/python -I tools/aeic/eval/eval.py \ + --download-kodak ~/aeic-data/kodak --out ~/aeic-runs/kodak-baseline \ + --variants stretch,crop,crop43,pad + +# Your own photos; Kodak alone is not representative of what people send +~/.venvs/aeic-eval/bin/python -I tools/aeic/eval/eval.py \ + --images ~/aeic-data/mesh-photos --out ~/aeic-runs/mesh-baseline +``` + +| option | meaning | +|---|---| +| `--variants` | comma list of variants to compare (below) | +| `--rois FILE` | JSON `{image stem: [l, t, r, b]}` as fractions; adds subject-region (`roi_*`) scores and limits the run to listed images. `rois/kodak.json` has 17 Kodak subjects | +| `--display N` | long side of the receiver view that is scored (default 512; use 1024 for two-part variants) | +| `--decoder` | score a different decoder export, e.g. fp16 instead of int8 | +| `--limit N` | first N images only | +| `--cpu` | run the decoder on CPU (much slower) | + +| variant | what is sent | +|---|---| +| `stretch` | the whole photo stretched to the square, as the app does | +| `crop`, `crop43`, `pad` | centre square crop; crop to at most 4:3 then stretch; letterbox on grey | +| `reroll1`..`reroll5` | the whole photo with the send sheet's "Try another version" tweaks (`kImageSendRerollTweaks`; keep the two tables in step) | +| `select`, `select_sq` | only the `--rois` box; the box grown to a square | +| `q` | quantisation step `` instead of 1 (e.g. `q0.7` spends more bytes) | +| `fit` | per image, the finest step from the 16-step ladder that fits `` (e.g. `fit157` = one packet) | +| `hd_tiles`, `hd_detail` | two encodes per photo: two overlapping halves; or the whole photo plus the `--rois` box, blended | + +`q`/`fit` re-run the four context stages with the decode-side entropy graph; +at step 1 they reproduce the app's bytes exactly. Two-part byte counts include +the extra stream header such a format would need. + +To add an experiment, write a function returning `Prepared` and register it in +`VARIANTS`. + +## Comparing with JPEG XL and AVIF + +`compare_codecs.py` sweeps AEIC over every quantisation step and fits JPEG XL +and AVIF to the same budgets (16–192 px thumbnails, best size per image, scored +after upscaling). AVIF is counted both as a real `.avif` file and as its AV1 +data alone. Needs `imagecodecs` (in `requirements.txt`). + +```bash +~/.venvs/aeic-eval/bin/python -I tools/aeic/eval/compare_codecs.py \ + --images ~/aeic-data/kodak --out ~/aeic-runs/rd +``` + +## Output + +| file | contents | +|---|---| +| `report.html` | source next to each variant's reconstruction, with bytes, packets, LPIPS and CLIP; a red border marks CLIP < 0.80 | +| `results.csv` | one row per image × variant | +| `summary.json` | per-variant mean/median/min/max, packet histogram, run metadata | +| `recon/`, `ref/` | the scored images, at the receiver's aspect ratio | +| `tensors/*.aeic` | the bitstreams, exactly as the app would send them | + +## Reading the metrics + +| metric | better | what it catches | +|---|---|---| +| `bytes`, `data_chunks` | lower | airtime; one chunk carries up to 157 image bytes | +| `lpips`, `dists` | lower | perceptual difference; the main quality score for a generative decoder | +| `clip` | higher | whether the reconstruction shows the same *things* as the sent region; low values are the "fox came back as a sheep" failures | +| `clip_full` | higher | the same, against the whole original photo, so a crop pays for what it dropped | +| `psnr`, `ms_ssim` | higher | pixel fidelity; included for reference, but weak signals for this codec | + +`1-pkt` in the console summary is the share of images that fit in a single data +chunk. Each image also gets one parity packet when recovery is on. + +## Caveats + +- Desktop ORT on CPU is not bit-identical to ORT on a phone, so an occasional rounding difference can change a byte count by one. The byte counter itself matches the reference coder exactly (see `--selftest`). +- The `stretch` resize approximates `dart:ui`'s `FilterQuality.medium`; it is not pixel-identical. +- CLIP and LPIPS are proxies. Look at the images flagged in `report.html` before drawing conclusions. diff --git a/tools/aeic/eval/aeic_bytes.dart b/tools/aeic/eval/aeic_bytes.dart new file mode 100644 index 00000000..be3423c9 --- /dev/null +++ b/tools/aeic/eval/aeic_bytes.dart @@ -0,0 +1,240 @@ +// Byte counter for the AEIC evaluation harness (`eval.py`). +// +// Python runs the encode-side ONNX graph; this turns its tensors into the exact +// bitstream the app would send, using the app's own entropy layer and rANS +// coder rather than a port. A port could drift; this cannot, because it is the +// shipping code. +// +// Usage (from the repository root): +// +// dart run tools/aeic/eval/aeic_bytes.dart ... +// dart run tools/aeic/eval/aeic_bytes.dart --selftest ... +// +// Each `.f32` file holds little-endian float32 tensors back to back, in this +// order: z_q [1,128,4,4], yq0..yq3 [1,256,16,16], sc0..sc3 [1,256,16,16]. The +// bitstream is written next to it as `.aeic`, and one JSON object per input is +// printed to stdout. +// +// `--selftest` replays the golden `.aeicrec` recordings in +// `test/services/golden/e2e/` and checks that the output is byte-identical to +// the reference C++ bitstream, which shows the harness counts what the app +// sends. + +import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:meshcore_open/services/entropy_tables.dart'; +import 'package:meshcore_open/services/image_chunk_transport.dart'; +import 'package:meshcore_open/services/image_codec_entropy.dart'; +import 'package:meshcore_open/services/rans_coder.dart'; + +const int _resolution = 512; + +Future main(List args) async { + if (args.isEmpty) { + stderr.writeln( + 'usage: aeic_bytes.dart ...\n' + ' aeic_bytes.dart --selftest ...', + ); + exitCode = 64; + return; + } + if (args.first == '--selftest') { + exitCode = await _selfTest(args.skip(1).toList()); + return; + } + + final tables = EntropyTables.parse(File(args.first).readAsBytesSync()); + final geometry = AeicEntropyGeometry.forResolution(_resolution); + for (final path in args.skip(1)) { + final tensors = _readTensorFile(path, geometry); + final stream = await _encode(tables, geometry, tensors); + final outPath = path.replaceFirst(RegExp(r'\.f32$'), '.aeic'); + File(outPath).writeAsBytesSync(stream); + stdout.writeln( + jsonEncode({ + 'file': path, + 'bitstream': outPath, + 'bytes': stream.length, + 'data_chunks': imageDataChunkCount(stream.length), + }), + ); + } +} + +Future _encode( + EntropyTables tables, + AeicEntropyGeometry geometry, + AeicEncodeSideTensors tensors, +) { + final codec = AeicEntropyCodec( + geometry: geometry, + network: _FixedNetwork(tensors), + coders: _Coders(tables), + ); + // The network ignores its input; encode() only needs a correctly sized RGB + // buffer to get past its own conversion step. + return codec.encode(Uint8List(_resolution * _resolution * 3)); +} + +AeicEncodeSideTensors _readTensorFile( + String path, + AeicEntropyGeometry geometry, +) { + final bytes = File(path).readAsBytesSync(); + final z = geometry.zElements; + final y = geometry.yElements; + final expected = (z + 8 * y) * 4; + if (bytes.length != expected) { + throw FormatException( + '$path is ${bytes.length} bytes, expected $expected ' + '(z_q + 4 yq + 4 sc as float32)', + ); + } + final all = Float32List.sublistView(bytes); + Float32List at(int offset, int length) => + Float32List.sublistView(all, offset, offset + length); + return AeicEncodeSideTensors( + zQ: at(0, z), + yQ: [for (var i = 0; i < 4; i++) at(z + i * y, y)], + scales: [for (var i = 0; i < 4; i++) at(z + (4 + i) * y, y)], + ); +} + +Future _selfTest(List recordings) async { + if (recordings.isEmpty) { + stderr.writeln('--selftest needs at least one .aeicrec file'); + return 64; + } + final geometry = AeicEntropyGeometry.forResolution(_resolution); + var failures = 0; + for (final path in recordings) { + final r = _Recording.load(path); + final tables = EntropyTables.parse( + File( + '${File(path).parent.parent.path}/${r.meta['cdf_table']}', + ).readAsBytesSync(), + ); + final tensors = AeicEncodeSideTensors( + zQ: r.f32('enc/z_q'), + yQ: [for (var i = 0; i < 4; i++) r.f32('enc/yq$i')], + scales: [for (var i = 0; i < 4; i++) r.f32('enc/sc$i')], + ); + final got = await _encode(tables, geometry, tensors); + final want = r.u8('enc/bitstream'); + final ok = + got.length == want.length && + Iterable.generate(got.length).every((i) => got[i] == want[i]); + if (!ok) failures++; + stdout.writeln( + '${ok ? 'ok ' : 'FAIL'} ${r.meta['stem']}: ${got.length} B ' + '(reference ${want.length} B)', + ); + } + return failures == 0 ? 0 : 1; +} + +class _FixedNetwork implements AeicEntropyNetwork { + _FixedNetwork(this.tensors); + + final AeicEncodeSideTensors tensors; + + @override + bool get supportsDecodeSide => false; + + @override + Future runEncodeSide(Float32List imageChw) async => + tensors; + + @override + Future runHyperSynthesis(Float32List zQ) => + throw UnsupportedError('encode only'); + + @override + Future runStage(int stage, Float32List base) => + throw UnsupportedError('encode only'); +} + +class _Coders implements AeicRansCoderFactory { + _Coders(this.tables); + + final EntropyTables tables; + + @override + AeicRansEncoder createEncoder() => _Encoder(RansEncoder(tables)); + + @override + AeicRansDecoder createDecoder(Uint8List bitstream) => + throw UnsupportedError('encode only'); +} + +class _Encoder implements AeicRansEncoder { + _Encoder(this._encoder); + + final RansEncoder _encoder; + + @override + void pushSymbols(Int16List symbols, Int16List indexes, int cdfGroup) => + _encoder.encodeWithIndexes(symbols, indexes, cdfGroup); + + @override + Uint8List finish() => _encoder.finish(); +} + +/// Minimal reader for the `.aeicrec` container (see +/// `test/services/image_codec_e2e_test.dart` for the full one). +class _Recording { + _Recording(this.index, this.bytes) + : _entries = >{ + for (final e + in (index['entries'] as List).cast>()) + e['name'] as String: e, + }; + + final Map index; + final Uint8List bytes; + final Map> _entries; + + static _Recording load(String path) { + final bytes = File(path).readAsBytesSync(); + final bd = ByteData.sublistView(bytes); + if (ascii.decode(bytes.sublist(0, 8)) != 'AEICREC1') { + throw FormatException('bad .aeicrec magic in $path'); + } + final indexOffset = bd.getUint64(16, Endian.little); + final indexLength = bd.getUint32(24, Endian.little); + final index = + jsonDecode( + utf8.decode( + bytes.sublist(indexOffset, indexOffset + indexLength), + ), + ) + as Map; + return _Recording(index, bytes); + } + + Map get meta => index['meta'] as Map; + + Map _entry(String name) { + final e = _entries[name]; + if (e == null) throw StateError('no entry "$name" in recording'); + return e; + } + + Float32List f32(String name) { + final e = _entry(name); + final offset = e['offset'] as int; + return Float32List.sublistView( + bytes, + offset, + offset + (e['length'] as int), + ); + } + + Uint8List u8(String name) { + final e = _entry(name); + final offset = e['offset'] as int; + return Uint8List.sublistView(bytes, offset, offset + (e['length'] as int)); + } +} diff --git a/tools/aeic/eval/compare_codecs.py b/tools/aeic/eval/compare_codecs.py new file mode 100644 index 00000000..8271c923 --- /dev/null +++ b/tools/aeic/eval/compare_codecs.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +"""Rate-distortion comparison: AEIC (as shipped and with step rate control) +against JPEG XL and AVIF at the same byte budgets. + + python -I tools/aeic/eval/compare_codecs.py --images ~/aeic-data/kodak --out ~/aeic-runs/rd + +AEIC is swept over every quantisation step in STEP_LADDER (step 1.0 is the +stream the app sends today) and also fitted per image to 1 and 2 packets. +JPEG XL and AVIF are fitted per image to each byte budget: for every candidate +size the quality setting is searched to fit, the result is decoded and +upscaled to the display size, and the size with the lowest LPIPS is kept. That +choice uses the original, which the sender has, so it is a fair best case for +the conventional codecs rather than an oracle. + +AVIF is counted twice. `avif` is the real .avif file, whose ftyp+meta boxes +cost about 275 bytes before any image data. `avif_av1` counts only the AV1 +payload (the mdat contents), as if the receiver rebuilt the fixed container +from a template; that is AVIF's best case for a custom wire format. + +Everything is scored against the source at the same display size (long side +512, the AEIC output size). Needs `imagecodecs` with JPEG XL support on top of +the eval.py requirements. +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import struct +import sys +from pathlib import Path + +import numpy as np +from PIL import Image + +HERE = Path(__file__).resolve().parent +_spec = importlib.util.spec_from_file_location("aeic_eval", HERE / "eval.py") +ev = importlib.util.module_from_spec(_spec) +sys.modules["aeic_eval"] = ev # dataclasses look their module up here +_spec.loader.exec_module(ev) # type: ignore[union-attr] + +import imagecodecs # noqa: E402 (after eval.py, which loads torch first) +from concurrent.futures import ThreadPoolExecutor # noqa: E402 + +POOL = ThreadPoolExecutor(max_workers=12) + +PACKET_BUDGETS = {"1 packet": ev.CHUNK_FIRST_CAPACITY, + "2 packets": ev.CHUNK_FIRST_CAPACITY + ev.CHUNK_CAPACITY} +BUDGETS = [100, 157, 200, 250, 315, 400, 500, 700] +SIZES = [16, 24, 32, 40, 48, 56, 64, 80, 96, 128, 160, 192] +JXL_EFFORT = 9 +AVIF_SPEED = 2 +DISPLAY = ev.SIZE + + +def display_size(img: Image.Image, long_side: int) -> tuple[int, int]: + s = long_side / max(img.size) + return (max(1, round(img.width * s)), max(1, round(img.height * s))) + + +def mdat_payload(avif: bytes) -> int: + """Size of the AV1 data in an .avif file (its mdat box contents).""" + i = 0 + while i + 8 <= len(avif): + size, kind = struct.unpack(">I4s", avif[i:i + 8]) + header = 8 + if size == 1: + size = struct.unpack(">Q", avif[i + 8:i + 16])[0] + header = 16 + elif size == 0: + size = len(avif) - i + if kind == b"mdat": + return size - header + i += size + raise ValueError("no mdat box") + + +def _jxl(arr: np.ndarray, q: float) -> bytes: + return imagecodecs.jpegxl_encode(arr, distance=q, effort=JXL_EFFORT) + + +def _avif(arr: np.ndarray, q: float) -> bytes: + return imagecodecs.avif_encode(arr, level=int(q), speed=AVIF_SPEED) + + +# name -> (encode(arr, q), decode, bytes counted, q for the smallest file, +# q for the largest, integer q?) +CODECS = { + "jxl": (_jxl, imagecodecs.jpegxl_decode, len, 25.0, 0.05, False), + "avif": (_avif, imagecodecs.avif_decode, len, 0, 100, True), + "avif_av1": (_avif, imagecodecs.avif_decode, mdat_payload, 0, 100, True), +} + + +def codec_fit(name: str, src: Image.Image, long_side: int, budget: int) -> tuple[int, Image.Image] | None: + """Best stream of codec `name` for `src` at `long_side` px within `budget`. + + Returns (counted bytes, decoded image), or None if even the lowest quality + does not fit. + """ + enc, dec, count, q_small, q_large, integer = CODECS[name] + small = np.asarray(src.resize(display_size(src, long_side), Image.LANCZOS)) + data = enc(small, q_small) + if count(data) > budget: + return None + best = data + lo, hi = q_small, q_large # lo always fits + for _ in range(12): + if integer: + if abs(hi - lo) <= 1: + break + mid = (lo + hi) // 2 + else: + mid = (lo * hi) ** 0.5 # JPEG XL distance acts roughly logarithmically + data = enc(small, mid) + if count(data) <= budget: + best, lo = data, mid + else: + hi = mid + decoded = Image.fromarray(np.asarray(dec(best))[..., :3], "RGB") + return count(best), decoded + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--images", type=Path, required=True) + ap.add_argument("--out", type=Path, required=True) + ap.add_argument("--models", type=Path, default=ev.DEFAULT_MODELS) + ap.add_argument("--limit", type=int) + args = ap.parse_args() + + images = ev.find_images(args.images, args.limit) + out: Path = args.out + work = out / "tensors" + (out / "img").mkdir(parents=True, exist_ok=True) + work.mkdir(parents=True, exist_ok=True) + + import torch + gpu = torch.cuda.is_available() + codec = ev.Codec(args.models, args.models / ev.DECODER_GRAPH, gpu) + metrics = ev.Metrics("cuda" if gpu else "cpu") + + # ---- AEIC: every step on the ladder, one Dart run for all byte counts. + srcs, aeic_y = {}, {} + files = [] + for path in images: + stem = path.stem + src = ev.load_source(path) + srcs[stem] = src + side = codec.encode_side(ev.variant_stretch(src).square) + for step in ev.STEP_LADDER: + t = codec.encode_with_step(side, step) + tf = work / f"{stem}.s{step:.2f}.f32" + ev.write_tensor_file(tf, t) + aeic_y[(stem, step)] = t["y_hat"] + files.append(tf) + print(f"encoded {stem}", file=sys.stderr) + nbytes = {k: v["bytes"] for k, v in ev.count_bytes(args.models / ev.CDF_TABLE, files).items()} + for tf in files: + tf.unlink() + + rows = [] + emb = {} + + def score(stem: str, shown: Image.Image) -> dict: + src = srcs[stem] + ref = src.resize(display_size(src, DISPLAY), Image.LANCZOS) + if stem not in emb: + emb[stem] = metrics.clip_embed(src) + s = metrics.score(ref, shown.resize(ref.size, Image.BICUBIC), emb[stem]) + s.pop("clip_full") + return s + + aeic_shown = {} + for path in images: + stem = path.stem + src = srcs[stem] + for step in ev.STEP_LADDER: + recon = codec.decode(aeic_y[(stem, step)]) + shown = recon.resize(display_size(src, DISPLAY), Image.LANCZOS) + aeic_shown[(stem, step)] = shown + b = nbytes[str(work / f"{stem}.s{step:.2f}.f32")] + rows.append({"image": stem, "codec": "aeic", "step": step, "bytes": b, **score(stem, shown)}) + print(f"scored AEIC {stem}", file=sys.stderr) + + # ---- Conventional codecs: per image and budget, the best size that fits. + for name in CODECS: + for path in images: + stem = path.stem + src = srcs[stem] + for budget in BUDGETS: + best = None + # imagecodecs releases the GIL, so the size search runs in threads. + fits = list(POOL.map(lambda ls: codec_fit(name, src, ls, budget), SIZES)) + for long_side, fit in zip(SIZES, fits): + if fit is None: + continue + nb, small = fit + shown = small.resize(display_size(src, DISPLAY), Image.BICUBIC) + ref = src.resize(shown.size, Image.LANCZOS) + lp = float(metrics.lpips(metrics._t(shown), metrics._t(ref))) + if best is None or lp < best[0]: + best = (lp, long_side, nb, shown) + if best is None: + continue + _, long_side, nb, shown = best + rows.append({"image": stem, "codec": name, "budget": budget, "px": long_side, + "bytes": nb, **score(stem, shown)}) + if budget in PACKET_BUDGETS.values(): + shown.save(out / "img" / f"{stem}.{name}{budget}.png") + print(f"scored {name} {stem}", file=sys.stderr) + + # ---- AEIC fitted to the packet budgets, picked from the sweep. + for stem in srcs: + for label, budget in PACKET_BUDGETS.items(): + cands = [r for r in rows if r["codec"] == "aeic" and r["image"] == stem and r["bytes"] <= budget] + if not cands: + cands = [max((r for r in rows if r["codec"] == "aeic" and r["image"] == stem), + key=lambda r: r["step"])] + pick = min(cands, key=lambda r: r["step"]) + rows.append({**pick, "codec": f"aeic_fit{budget}", "budget": budget}) + aeic_shown[(stem, pick["step"])].save(out / "img" / f"{stem}.aeic{budget}.png") + aeic_shown[(stem, 1.0)].save(out / "img" / f"{stem}.aeic_today.png") + srcs[stem].resize(display_size(srcs[stem], DISPLAY), Image.LANCZOS).save(out / "img" / f"{stem}.src.png") + + (out / "rows.json").write_text(json.dumps(rows, indent=0)) + + def mean(rs, k): + return float(np.mean([r[k] for r in rs])) + + def line(name, rs): + return (f"{name:<22}{mean(rs, 'bytes'):>7.0f}{mean(rs, 'lpips'):>8.3f}{mean(rs, 'dists'):>8.3f}" + f"{mean(rs, 'clip'):>7.3f}{mean(rs, 'ms_ssim'):>8.3f}{mean(rs, 'psnr'):>7.2f}") + + print(f"\n{'':<22}{'bytes':>7}{'lpips':>8}{'dists':>8}{'clip':>7}{'msssim':>8}{'psnr':>7}") + print(line("AEIC today (step 1)", [r for r in rows if r["codec"] == "aeic" and r["step"] == 1.0])) + for label, budget in PACKET_BUDGETS.items(): + print(f"-- {label} ({budget} B)") + print(line(" AEIC step-fit", [r for r in rows if r["codec"] == f"aeic_fit{budget}"])) + for name in CODECS: + rs = [r for r in rows if r["codec"] == name and r["budget"] == budget] + print(line(f" {name} ({len(rs)} img)", rs) if rs else f" {name}: does not fit") + print("\nAEIC curve") + for step in ev.STEP_LADDER: + print(line(f" step {step:.2f}", [r for r in rows if r["codec"] == "aeic" and r["step"] == step])) + for name in CODECS: + print(f"{name} curve") + for budget in BUDGETS: + rs = [r for r in rows if r["codec"] == name and r["budget"] == budget] + if rs: + print(line(f" <= {budget} B ({len(rs)} img)", rs)) + + +if __name__ == "__main__": + main() diff --git a/tools/aeic/eval/eval.py b/tools/aeic/eval/eval.py new file mode 100644 index 00000000..e5dd446c --- /dev/null +++ b/tools/aeic/eval/eval.py @@ -0,0 +1,831 @@ +#!/usr/bin/env python3 +"""Quality/rate evaluation harness for the AEIC image codec. + +Runs a folder of photos through the same three ONNX graphs the app ships, +counts the exact bytes with the app's own Dart rANS coder (aeic_bytes.dart), +and scores what the receiver would see. + + python -I tools/aeic/eval/eval.py --images ~/photos --out /tmp/aeic-eval + +See README.md in this directory for setup and how to read the output. +""" + +from __future__ import annotations + +import argparse +import csv +import datetime as dt +import hashlib +import html +import json +import math +import os +import shutil +import subprocess +import sys +import time +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Callable + +import numpy as np +from PIL import Image, ImageOps + +# torch must be imported before onnxruntime so the CUDA libraries that ship in +# the torch wheels are already loaded when ORT's CUDA provider looks for them. +import torch # noqa: E402 (import order is deliberate) +import onnxruntime as ort # noqa: E402 + +SIZE = 512 +REPO = Path(__file__).resolve().parents[3] +DART_TOOL = Path(__file__).resolve().parent / "aeic_bytes.dart" +DEFAULT_MODELS = Path.home() / "Documents" / "image_codec_models" +SIDE_GRAPH = "aeic_entropy_side_fp32_op17.onnx" +DECODER_GRAPH = "aeic_decoder_qdq_conv_pct.onnx" +CDF_TABLE = "aeic_cdf_ft32.bin" +CONTEXT_GRAPH = "aeic_entropy_decode_fp32_op17.onnx" + + +def _stage_masks() -> list[np.ndarray]: + """The four checkerboard masks, as AeicMaskSet.isLive defines them.""" + perm = [0, 3, 2, 1] + c = np.arange(256)[:, None, None] // 64 + yy = np.arange(16)[None, :, None] + xx = np.arange(16)[None, None, :] + micro = ((yy & 1) << 1) | (xx & 1) + return [((c ^ perm[s]) == micro)[None] for s in range(4)] + + +STAGE_MASKS = _stage_masks() + +# Quantisation steps a stream could signal in 4 bits (the metadata byte's rate +# and resolution fields). 1.0 is today's ft32 stream. +STEP_LADDER = [0.40, 0.45, 0.50, 0.55, 0.60, 0.66, 0.72, 0.79, + 0.86, 0.93, 1.00, 1.10, 1.20, 1.32, 1.45, 1.60] +IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".webp", ".bmp", ".tif", ".tiff"} + +# Mirrors lib/services/image_chunk_transport.dart. aeic_bytes.dart reports the +# authoritative count; these are only used for the summary's packet totals. +PARITY_CHUNKS = 1 + + +# -------------------------------------------------------------------------- +# Preprocessing variants +# -------------------------------------------------------------------------- +# +# A variant turns the (EXIF-rotated) source photo into the 512x512 codec input +# and says which part of the source it represents, so the reconstruction can +# be compared with that region at its true aspect ratio, which is what the +# receiver displays. + + +@dataclass +class Prepared: + square: Image.Image # 512x512 RGB, what the encoder sees + region: tuple[int, int, int, int] # source box (l, t, r, b) it represents + content_box: tuple[int, int, int, int] # box inside `square` holding it + # For multi-part variants: width in source pixels of the cross-fade along + # this part's edges that lie inside the picture (not on its border). + feather: int = 0 + + +def _resize(img: Image.Image, size: tuple[int, int]) -> Image.Image: + # The app draws with dart:ui FilterQuality.medium: bilinear sampling over + # mipmaps. A box pre-reduce followed by bilinear is the closest PIL match. + return img.resize(size, Image.BILINEAR, reducing_gap=2.0) + + +def _full(img: Image.Image) -> tuple[int, int, int, int]: + return (0, 0, img.width, img.height) + + +def variant_stretch(img: Image.Image) -> Prepared: + """What the app does today: the whole frame squeezed into the square.""" + return Prepared(_resize(img, (SIZE, SIZE)), _full(img), (0, 0, SIZE, SIZE)) + + +def _centre_crop_box(img: Image.Image, max_ratio: float) -> tuple[int, int, int, int]: + w, h = img.size + ratio = w / h + if ratio > max_ratio: + cw = round(h * max_ratio) + left = (w - cw) // 2 + return (left, 0, left + cw, h) + if ratio < 1 / max_ratio: + ch = round(w * max_ratio) + top = (h - ch) // 2 + return (0, top, w, top + ch) + return _full(img) + + +def variant_crop(img: Image.Image) -> Prepared: + """Centre square crop: no distortion, loses the edges of the frame.""" + box = _centre_crop_box(img, 1.0) + return Prepared(_resize(img.crop(box), (SIZE, SIZE)), box, (0, 0, SIZE, SIZE)) + + +def variant_crop43(img: Image.Image) -> Prepared: + """Crop to at most 4:3 (or 3:4), then stretch what is left.""" + box = _centre_crop_box(img, 4 / 3) + return Prepared(_resize(img.crop(box), (SIZE, SIZE)), box, (0, 0, SIZE, SIZE)) + + +def variant_pad(img: Image.Image) -> Prepared: + """Letterbox on mid-grey: no distortion, spends latent area on the bars.""" + w, h = img.size + scale = SIZE / max(w, h) + nw, nh = max(1, round(w * scale)), max(1, round(h * scale)) + canvas = Image.new("RGB", (SIZE, SIZE), (128, 128, 128)) + left, top = (SIZE - nw) // 2, (SIZE - nh) // 2 + canvas.paste(_resize(img, (nw, nh)), (left, top)) + return Prepared(canvas, _full(img), (left, top, left + nw, top + nh)) + + +# Re-roll tweaks: (zoom, dx, dy) as fractions of the source. Mirrors +# kImageSendRerollTweaks / rerollSourceRect in +# lib/widgets/image_send_codec_binding.dart; keep the two in step. +REROLL_TWEAKS = [ + (1.0, 0.0, 0.0), + (1.02, 0.005, 0.0), + (1.02, -0.005, 0.005), + (1.04, 0.0, -0.005), + (1.03, 0.01, 0.01), + (1.03, -0.01, -0.01), +] + + +def reroll_source_rect(w: int, h: int, variant: int) -> tuple[float, float, float, float]: + """Source rect for re-roll `variant`: a near-invisible zoom and shift.""" + zoom, dx, dy = REROLL_TWEAKS[variant] + cw, ch = w / zoom, h / zoom + left = min(max((w - cw) / 2 + dx * w, 0.0), w - cw) + top = min(max((h - ch) / 2 + dy * h, 0.0), h - ch) + return (left, top, left + cw, top + ch) + + +def variant_reroll(img: Image.Image, variant: int) -> Prepared: + """Whole photo with re-roll tweak `variant`, stretched like `stretch`. + + Scored against the full frame: the tweak only drops a percent or two at + the edges, which is what the sender sees as "the whole photo". + """ + l, t, r, b = reroll_source_rect(img.width, img.height, variant) + # PIL's resize box takes float source coordinates, like drawImageRect. + square = img.resize((SIZE, SIZE), Image.BILINEAR, box=(l, t, r, b), reducing_gap=2.0) + return Prepared(square, _full(img), (0, 0, SIZE, SIZE)) + + +VARIANTS: dict[str, Callable[[Image.Image], Prepared]] = { + "stretch": variant_stretch, + "crop": variant_crop, + "crop43": variant_crop43, + "pad": variant_pad, + **{f"reroll{k}": (lambda img, k=k: variant_reroll(img, k)) for k in range(1, len(REROLL_TWEAKS))}, +} + +# Need a box per image (--rois), so they are not in VARIANTS. +SELECT = "select" +SELECT_SQUARE = "select_sq" +SELECT_VARIANTS = (SELECT, SELECT_SQUARE) + +Box = tuple[int, int, int, int] + + +def roi_box(img: Image.Image, frac: list[float]) -> Box: + """Fractional [l, t, r, b] -> pixel box in the (EXIF-rotated) source.""" + l, t, r, b = frac + w, h = img.size + return (round(l * w), round(t * h), round(r * w), round(b * h)) + + +def variant_select(img: Image.Image, box: Box) -> Prepared: + """The user's selection, stretched into the square like `stretch`. + + The aspect nibble restores the box's shape on the receiver, so this is the + app's existing pipeline applied to a crop the user chose. + """ + return Prepared(_resize(img.crop(box), (SIZE, SIZE)), box, (0, 0, SIZE, SIZE)) + + +def square_box(img: Image.Image, box: Box) -> Box: + """Grow `box` to a square around its centre, shifted to stay in the image. + + Models a selection tool locked to 1:1, so the codec never sees a stretch. + Capped at the image's short side. + """ + l, t, r, b = box + side = min(max(r - l, b - t), img.width, img.height) + cx, cy = (l + r) / 2, (t + b) / 2 + nl = int(round(min(max(cx - side / 2, 0), img.width - side))) + nt = int(round(min(max(cy - side / 2, 0), img.height - side))) + return (nl, nt, nl + side, nt + side) + + +# -------------------------------------------------------------------------- +# HD: two encoded parts per photo, aimed at a two-packet budget +# -------------------------------------------------------------------------- +# +# The model is fixed at 512x512 / ft32, so "HD" spends the second packet on a +# second 512x512 encode. The parts travel as one stream; the receiver decodes +# both and blends them. Overheads below are what the stream would need on top +# of the bitstreams (a hypothetical layout, not implemented in the app). + +HD_TILES = "hd_tiles" +HD_DETAIL = "hd_detail" +HD_VARIANTS = (HD_TILES, HD_DETAIL) +HD_OVERHEAD = { + HD_TILES: 1, # length of part 0, so the receiver can split the stream + HD_DETAIL: 1 + 4, # that, plus the detail box quantised to 4 bytes +} +TILE_OVERLAP = 0.08 # of the long side, cross-faded at the seam + + +def variant_hd_tiles(img: Image.Image) -> list[Prepared]: + """Two overlapping halves along the long side, each encoded at 512x512. + + Doubles the pixel count. A 3:2 photo splits into two near-square halves, + so each half is also stretched far less than the whole frame would be. + """ + w, h = img.size + long_side = max(w, h) + ov = round(long_side * TILE_OVERLAP) + half = (long_side + ov) // 2 + if w >= h: + boxes = [(0, 0, half, h), (w - half, 0, w, h)] + else: + boxes = [(0, 0, w, half), (0, h - half, w, h)] + return [Prepared(_resize(img.crop(b), (SIZE, SIZE)), b, (0, 0, SIZE, SIZE), feather=ov) + for b in boxes] + + +def variant_hd_detail(img: Image.Image, box: Box) -> list[Prepared]: + """The whole photo, plus the selected subject blended in at higher detail.""" + detail = variant_select(img, box) + detail.feather = max(4, round(0.08 * min(box[2] - box[0], box[3] - box[1]))) + return [variant_stretch(img), detail] + + +def composite(src: Image.Image, parts: list[Prepared], recons: list[Image.Image], + long_side: int) -> tuple[Image.Image, Image.Image, Box]: + """Reference and receiver view for a variant made of one or more parts. + + The view covers the union of the parts' regions at `long_side` pixels on + its long edge. Parts are laid down in order; each later part fades in over + `feather` pixels along its edges that are inside the union, so seams + cross-fade instead of cutting hard. + """ + region = (min(p.region[0] for p in parts), min(p.region[1] for p in parts), + max(p.region[2] for p in parts), max(p.region[3] for p in parts)) + rw, rh = region[2] - region[0], region[3] - region[1] + s = long_side / max(rw, rh) + size = (max(1, round(rw * s)), max(1, round(rh * s))) + ref = src.crop(region).resize(size, Image.LANCZOS) + canvas = np.zeros((size[1], size[0], 3), dtype=np.float64) + for i, (p, recon) in enumerate(zip(parts, recons)): + l, t, r, b = p.region + x0, y0 = round((l - region[0]) * s), round((t - region[1]) * s) + x1, y1 = round((r - region[0]) * s), round((b - region[1]) * s) + tile = np.asarray(recon.crop(p.content_box).resize((x1 - x0, y1 - y0), Image.LANCZOS), + dtype=np.float64) + alpha = np.ones((y1 - y0, x1 - x0), dtype=np.float64) + f = min(round(p.feather * s), alpha.shape[0], alpha.shape[1]) + if i > 0 and f > 0: + ramp = np.arange(1, f + 1, dtype=np.float64) / f + if l > region[0]: + alpha[:, :f] *= ramp[None, :] + if r < region[2]: + alpha[:, -f:] *= ramp[::-1][None, :] + if t > region[1]: + alpha[:f, :] *= ramp[:, None] + if b < region[3]: + alpha[-f:, :] *= ramp[::-1][:, None] + a = alpha[..., None] + canvas[y0:y1, x0:x1] = canvas[y0:y1, x0:x1] * (1 - a) + tile * a + shown = Image.fromarray(np.clip(np.rint(canvas), 0, 255).astype(np.uint8), "RGB") + return ref, shown, region + + +# Mirrors imageDataChunkCount() in image_chunk_transport.dart. +CHUNK_FIRST_CAPACITY = 157 +CHUNK_CAPACITY = 158 + + +def parse_rate_variant(v: str) -> dict | None: + """`q0.7` -> fixed step 0.7; `fit315` -> per-image step to fit 315 bytes.""" + try: + if v.startswith("q"): + return {"steps": [float(v[1:])], "budget": None} + if v.startswith("fit"): + return {"steps": STEP_LADDER, "budget": int(v[3:])} + except ValueError: + return None + return None + + +def data_chunks(payload_bytes: int) -> int: + n = max(payload_bytes, 0) + if n <= CHUNK_FIRST_CAPACITY: + return 1 + return 1 + math.ceil((n - CHUNK_FIRST_CAPACITY) / CHUNK_CAPACITY) + + +# -------------------------------------------------------------------------- +# Codec +# -------------------------------------------------------------------------- + + +class Codec: + def __init__(self, models: Path, decoder: Path, gpu: bool) -> None: + cpu = ["CPUExecutionProvider"] + cuda = ["CUDAExecutionProvider", "CPUExecutionProvider"] if gpu else cpu + # The entropy graph stays on CPU: its rounded outputs decide the bytes, + # and CPU fp32 is the closest stand-in for the phone. It costs ~0.1 s. + self.side = ort.InferenceSession(str(models / SIDE_GRAPH), providers=cpu) + self.context = ort.InferenceSession(str(models / CONTEXT_GRAPH), providers=cpu) + self.decoder = ort.InferenceSession(str(decoder), providers=cuda) + self.decoder_provider = self.decoder.get_providers()[0] + + def encode_side(self, square: Image.Image) -> dict[str, np.ndarray]: + rgb = np.asarray(square, dtype=np.uint8) + # Same float32 steps as aeicRgbToChw in image_codec_entropy.dart. + chw = rgb.transpose(2, 0, 1).astype(np.float32) / np.float32(255.0) + chw = (chw - np.float32(0.5)) * np.float32(2.0) + names = (["z_q", "y_hat", "y", "base0"] + + [f"yq{i}" for i in range(4)] + [f"sc{i}" for i in range(4)]) + out = self.side.run(names, {"image": chw[None]}) + return dict(zip(names, out)) + + def encode_with_step(self, side: dict[str, np.ndarray], step: float) -> dict[str, np.ndarray]: + """Re-quantise y with step `step` instead of 1 (rate control). + + Replays the four-stage context loop the receiver runs + (AeicEntropyCodec.decodeToLatent) with the decode-side graph, coding + round((y - mean) / step) against scale / step. The receiver's + reconstruction becomes symbol * step + mean. step < 1 spends more + bytes, step > 1 fewer. At step == 1 this reproduces the side graph's + symbols, which the harness's q1 check relies on. + """ + st = np.float32(step) + y, z_q = side["y"], side["z_q"] + base = side["base0"].copy() + out: dict[str, np.ndarray] = {"z_q": z_q} + for stage in range(4): + means, scales = self.context.run( + ["means", "scales"], + {"z_q": z_q, "base": base, "stage": np.array([stage], dtype=np.int32)}) + m = STAGE_MASKS[stage] + sym = np.where(m, np.rint((y - means) / st), np.float32(0)).astype(np.float32) + base = np.where(m, sym * st + means, base).astype(np.float32) + out[f"yq{stage}"] = sym + out[f"sc{stage}"] = np.where(m, scales / st, np.float32(0)).astype(np.float32) + out["y_hat"] = base + return out + + def decode(self, y_hat: np.ndarray) -> Image.Image: + (out,) = self.decoder.run(["image"], {"y_hat": y_hat.astype(np.float32)}) + # Same mapping as _chwFloatsToRgb in image_codec_backend.dart: clamp, + # then round half away from zero (Dart's double.round()). + scaled = (out[0].astype(np.float64) + 1.0) * 127.5 + rgb = np.floor(np.clip(scaled, 0, 255) + 0.5).astype(np.uint8) + return Image.fromarray(rgb.transpose(1, 2, 0), "RGB") + + +def write_tensor_file(path: Path, t: dict[str, np.ndarray]) -> None: + parts = [t["z_q"]] + [t[f"yq{i}"] for i in range(4)] + [t[f"sc{i}"] for i in range(4)] + with open(path, "wb") as f: + for p in parts: + f.write(np.ascontiguousarray(p, dtype=" dict[str, dict]: + if not tensor_files: + return {} + dart = shutil.which("dart") + if dart is None: + sys.exit("dart is not on PATH; it is needed to run aeic_bytes.dart") + cmd = [dart, "run", str(DART_TOOL), str(cdf), *map(str, tensor_files)] + proc = subprocess.run(cmd, cwd=REPO, capture_output=True, text=True) + if proc.returncode != 0: + sys.exit(f"aeic_bytes.dart failed:\n{proc.stderr}") + results = {} + for line in proc.stdout.splitlines(): + line = line.strip() + if line.startswith("{"): + row = json.loads(line) + results[row["file"]] = row + return results + + +# -------------------------------------------------------------------------- +# Metrics +# -------------------------------------------------------------------------- + + +class Metrics: + """Full-reference scores. Higher is better except LPIPS and DISTS.""" + + def __init__(self, device: str) -> None: + import open_clip + import piq + + self.piq = piq + self.device = device + self.lpips = piq.LPIPS(reduction="none").to(device).eval() + self.dists = piq.DISTS(reduction="none").to(device).eval() + model, _, preprocess = open_clip.create_model_and_transforms( + "ViT-B-32", pretrained="laion2b_s34b_b79k" + ) + self.clip = model.to(device).eval() + self.clip_pre = preprocess + + def _t(self, img: Image.Image) -> torch.Tensor: + a = np.asarray(img, dtype=np.float32) / 255.0 + return torch.from_numpy(a).permute(2, 0, 1)[None].to(self.device) + + @torch.no_grad() + def clip_embed(self, img: Image.Image) -> torch.Tensor: + # CLIP's own preprocessing centre-crops to a square, which would hide + # the frame edges -- exactly what crop variants throw away. Letterbox + # first so the whole image is scored. + side = max(img.size) + boxed = Image.new("RGB", (side, side), (0, 0, 0)) + boxed.paste(img, ((side - img.width) // 2, (side - img.height) // 2)) + x = self.clip_pre(boxed)[None].to(self.device) + e = self.clip.encode_image(x) + return e / e.norm(dim=-1, keepdim=True) + + @torch.no_grad() + def score(self, ref: Image.Image, out: Image.Image, full_src_emb: torch.Tensor) -> dict: + r, o = self._t(ref), self._t(out) + out_emb = self.clip_embed(out) + return { + "psnr": float(self.piq.psnr(o, r, data_range=1.0)), + "ms_ssim": float(self.piq.multi_scale_ssim(o, r, data_range=1.0)), + "lpips": float(self.lpips(o, r)), + "dists": float(self.dists(o, r)), + # Semantic agreement with exactly what was sent ... + "clip": float((self.clip_embed(ref) * out_emb).sum()), + # ... and with the whole original photo, so cropping is charged + # for what it threw away. + "clip_full": float((full_src_emb * out_emb).sum()), + } + + +def roi_pair(src: Image.Image, region: Box, shown: Image.Image, box: Box) -> tuple[Image.Image, Image.Image] | None: + """The ROI as the receiver would see it after zooming in, plus its reference. + + `shown` covers `region` of the source; the ROI is mapped into it and + both crops are scaled to long side 512. For a full-frame variant that is an + upscale of a few dozen pixels, which is exactly what zooming in shows. + Returns None when the variant did not send the whole ROI. + """ + rl, rt, rr, rb = region + l, t, r, b = box + if l < rl or t < rt or r > rr or b > rb: + return None + sx = shown.width / (rr - rl) + sy = shown.height / (rb - rt) + out = shown.crop((round((l - rl) * sx), round((t - rt) * sy), + round((r - rl) * sx), round((b - rt) * sy))) + ref = src.crop(box) + scale = SIZE / max(ref.size) + size = (max(1, round(ref.width * scale)), max(1, round(ref.height * scale))) + return ref.resize(size, Image.LANCZOS), out.resize(size, Image.BICUBIC) + + +# -------------------------------------------------------------------------- +# Corpus +# -------------------------------------------------------------------------- + +KODAK_URL = "https://r0k.us/graphics/kodak/kodak/kodim{:02d}.png" + + +def download_kodak(dest: Path) -> None: + dest.mkdir(parents=True, exist_ok=True) + for i in range(1, 25): + target = dest / f"kodim{i:02d}.png" + if target.exists(): + continue + print(f"downloading {target.name}", file=sys.stderr) + tmp = target.with_suffix(".part") + with urllib.request.urlopen(KODAK_URL.format(i), timeout=60) as r, open(tmp, "wb") as f: + shutil.copyfileobj(r, f) + tmp.rename(target) + + +def load_source(path: Path) -> Image.Image: + img = Image.open(path) + # Picked photos are shown upright in the app, so honour EXIF orientation. + img = ImageOps.exif_transpose(img) + return img.convert("RGB") + + +def find_images(folder: Path, limit: int | None) -> list[Path]: + files = sorted(p for p in folder.rglob("*") if p.suffix.lower() in IMAGE_SUFFIXES) + return files[:limit] if limit else files + + +# -------------------------------------------------------------------------- +# Report +# -------------------------------------------------------------------------- + +METRIC_COLS = ["bytes", "data_chunks", "psnr", "ms_ssim", "lpips", "dists", "clip", "clip_full"] +ROI_COLS = ["roi_psnr", "roi_ms_ssim", "roi_lpips", "roi_dists", "roi_clip"] +LOWER_IS_BETTER = {"bytes", "data_chunks", "lpips", "dists", "roi_lpips", "roi_dists"} + + +def summarise(rows: list[dict], variants: list[str]) -> dict: + summary = {} + for v in variants: + vr = [r for r in rows if r["variant"] == v] + if not vr: + continue + s: dict = {"images": len(vr)} + for m in METRIC_COLS + ROI_COLS: + vals = np.array([r[m] for r in vr if r.get(m) not in (None, "")], dtype=np.float64) + if vals.size == 0: + continue + s[m] = {"mean": float(vals.mean()), "median": float(np.median(vals)), + "min": float(vals.min()), "max": float(vals.max())} + chunks = [r["data_chunks"] for r in vr] + s["chunk_histogram"] = {str(k): chunks.count(k) for k in sorted(set(chunks))} + s["packets_mean_with_parity"] = float(np.mean(chunks)) + PARITY_CHUNKS + # The share of images whose CLIP agreement with the source is low. + # This is the "fox came back as a sheep" bucket; worth eyeballing. + s["clip_below_0.80"] = sum(1 for r in vr if r["clip"] < 0.80) + summary[v] = s + return summary + + +def print_summary(summary: dict) -> None: + head = (f"{'variant':<10}" + "".join(f"{m:>12}" for m in METRIC_COLS) + + f"{'1-pkt':>8}{'<=2-pkt':>9}{'clip<.8':>9}") + print(head) + print("-" * len(head)) + for v, s in summary.items(): + line = f"{v:<10}" + "".join(f"{s[m]['mean']:>12.4g}" for m in METRIC_COLS) + one = s["chunk_histogram"].get("1", 0) + two = one + s["chunk_histogram"].get("2", 0) + line += f"{one / s['images']:>7.0%} {two / s['images']:>8.0%} {s['clip_below_0.80']:>8}" + print(line) + print("means; lower is better for bytes, data_chunks, lpips, dists") + if any("roi_lpips" in s for s in summary.values()): + print() + print(f"{'variant':<10}" + "".join(f"{m:>13}" for m in ROI_COLS)) + for v, s in summary.items(): + if "roi_lpips" in s: + print(f"{v:<10}" + "".join(f"{s[m]['mean']:>13.4g}" for m in ROI_COLS)) + print("scored on the selected region only, as the receiver sees it after zooming in") + + +def write_html(out: Path, rows: list[dict], summary: dict, meta: dict) -> None: + def fmt(v: float, m: str) -> str: + return f"{v:.1f}" if m in ("bytes", "data_chunks") else f"{v:.3f}" + + variants = list(summary) + stems = list(dict.fromkeys(r["image"] for r in rows)) + by = {(r["image"], r["variant"]): r for r in rows} + + sum_rows = "".join( + f"{html.escape(v)}" + + "".join(f"{fmt(summary[v][m]['mean'], m)}" for m in METRIC_COLS) + + "" + for v in variants + ) + cards = [] + for stem in stems: + cells = [] + first = by[(stem, variants[0])] + cells.append( + f'
' + f"
source
" + ) + for v in variants: + r = by.get((stem, v)) + if not r: + continue + warn = " warn" if r["clip"] < 0.80 else "" + cells.append( + f'
' + f"
{html.escape(v)} {r['bytes']} B / {r['data_chunks']} pkt
" + f"LPIPS {r['lpips']:.3f} · CLIP {r['clip']:.3f}
" + ) + cards.append(f"

{html.escape(stem)}

{''.join(cells)}
") + + doc = f""" + +AEIC eval + +

AEIC eval

+

{html.escape(json.dumps(meta))}

+

Means

+{''.join(f'' for m in METRIC_COLS)}{sum_rows}
{m}
+

Images are shown at the aspect ratio the receiver restores. A red border means CLIP similarity to the sent region is below 0.80, so check it by eye.

+{''.join(cards)} +""" + (out / "report.html").write_text(doc, encoding="utf-8") + + +# -------------------------------------------------------------------------- +# Main +# -------------------------------------------------------------------------- + + +def sha256(path: Path) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + for block in iter(lambda: f.read(1 << 20), b""): + h.update(block) + return h.hexdigest() + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--images", type=Path, help="folder of photos (searched recursively)") + ap.add_argument("--download-kodak", type=Path, metavar="DIR", + help="download the 24 Kodak images to DIR and use them") + ap.add_argument("--out", type=Path, required=True, help="output folder") + ap.add_argument("--models", type=Path, default=DEFAULT_MODELS, + help=f"model bundle folder (default {DEFAULT_MODELS})") + ap.add_argument("--decoder", type=Path, help="alternative decoder .onnx (e.g. an fp16 export)") + ap.add_argument("--variants", default="stretch", + help=f"comma list from: {', '.join(VARIANTS)} (default stretch)") + ap.add_argument("--limit", type=int, help="only the first N images") + ap.add_argument("--cpu", action="store_true", help="run the decoder on CPU too") + ap.add_argument("--rois", type=Path, + help="JSON {image stem: [l, t, r, b] fractions}; enables the select and " + "hd_detail variants and ROI scores, and limits the run to listed images") + ap.add_argument("--display", type=int, default=SIZE, + help="long side in pixels of the receiver view that is scored (default 512; " + "use 1024 when comparing HD variants)") + args = ap.parse_args() + + if args.download_kodak: + download_kodak(args.download_kodak) + args.images = args.images or args.download_kodak + if not args.images: + ap.error("give --images or --download-kodak") + variants = [v.strip() for v in args.variants.split(",") if v.strip()] + unknown = [v for v in variants + if v not in VARIANTS and v not in SELECT_VARIANTS and v not in HD_VARIANTS + and parse_rate_variant(v) is None] + if unknown: + ap.error(f"unknown variants: {unknown}") + rois: dict[str, list[float]] = {} + if args.rois: + rois = {k: v for k, v in json.loads(args.rois.read_text()).items() if not k.startswith("_")} + if any(v in SELECT_VARIANTS + (HD_DETAIL,) for v in variants) and not rois: + ap.error("the select and hd_detail variants need --rois") + + images = find_images(args.images, None) + if rois: + images = [p for p in images if p.relative_to(args.images).with_suffix("").as_posix() in rois] + images = images[: args.limit] if args.limit else images + if not images: + sys.exit(f"no images under {args.images}") + + out: Path = args.out + work = out / "tensors" + dirs = [work] + [out / k / v for k in ("recon", "ref", "roi") for v in variants] + for d in dirs: + d.mkdir(parents=True, exist_ok=True) + + decoder_path = args.decoder or (args.models / DECODER_GRAPH) + gpu = not args.cpu and torch.cuda.is_available() + codec = Codec(args.models, decoder_path, gpu) + metrics = Metrics("cuda" if gpu else "cpu") + print(f"{len(images)} images x {len(variants)} variants; decoder on {codec.decoder_provider}", + file=sys.stderr) + + # Pass 1: preprocess + encode side, so every byte count is one Dart run. + jobs = [] + for path in images: + src = load_source(path) + key = path.relative_to(args.images).with_suffix("").as_posix() + stem = key.replace("/", "__") + box = roi_box(src, rois[key]) if key in rois else None + for v in variants: + if (v in SELECT_VARIANTS or v == HD_DETAIL) and box is None: + continue + if v == SELECT: + parts = [variant_select(src, box)] + elif v == SELECT_SQUARE: + parts = [variant_select(src, square_box(src, box))] + elif v == HD_TILES: + parts = variant_hd_tiles(src) + elif v == HD_DETAIL: + parts = variant_hd_detail(src, box) + elif parse_rate_variant(v) is not None: + parts = [variant_stretch(src)] + else: + parts = [VARIANTS[v](src)] + rate = parse_rate_variant(v) + steps = rate["steps"] if rate else [None] + t0 = time.perf_counter() + encoded = [] # per part: [(step, y_hat, tensor file), ...] + for i, p in enumerate(parts): + side = codec.encode_side(p.square) + cands = [] + for step in steps: + tensors = side if step is None else codec.encode_with_step(side, step) + tag = "" if step is None else f".s{step:.2f}" + tf = work / f"{stem}.{v}.p{i}{tag}.f32" + write_tensor_file(tf, tensors) + cands.append((step, tensors["y_hat"], tf)) + encoded.append(cands) + enc_s = time.perf_counter() - t0 + jobs.append((path, stem, v, src, parts, box, encoded, enc_s)) + byte_rows = count_bytes(args.models / CDF_TABLE, + [tf for j in jobs for cands in j[6] for _, _, tf in cands]) + + # Pick one candidate per part. A fit variant takes the finest step whose + # stream fits N bytes, or the coarsest step if none does. + for j in jobs: + rate = parse_rate_variant(j[2]) + chosen = [] + for cands in j[6]: + if rate and rate["budget"] is not None: + fitting = [c for c in cands if byte_rows[str(c[2])]["bytes"] <= rate["budget"]] + chosen.append(min(fitting, key=lambda c: c[0]) if fitting + else max(cands, key=lambda c: c[0])) + else: + chosen.append(cands[0]) + j[6][:] = chosen + + # Pass 2: decode + score. + rows = [] + emb_cache: dict[str, torch.Tensor] = {} + for n, (path, stem, v, src, parts, box, encoded, enc_s) in enumerate(jobs, 1): + t0 = time.perf_counter() + recons = [codec.decode(y_hat) for _, y_hat, _ in encoded] + dec_s = time.perf_counter() - t0 + ref, shown, region = composite(src, parts, recons, args.display) + part_bytes = [byte_rows[str(tf)]["bytes"] for _, _, tf in encoded] + total_bytes = sum(part_bytes) + HD_OVERHEAD.get(v, 0) + chunks = data_chunks(total_bytes) + if len(parts) == 1: + assert chunks == byte_rows[str(encoded[0][2])]["data_chunks"], "chunk mirror drifted" + if stem not in emb_cache: + emb_cache[stem] = metrics.clip_embed(src) + scores = metrics.score(ref, shown, emb_cache[stem]) + roi = roi_pair(src, region, shown, box) if box else None + if roi: + roi_scores = metrics.score(roi[0], roi[1], emb_cache[stem]) + scores.update({f"roi_{k}": roi_scores[k] for k in ("psnr", "ms_ssim", "lpips", "dists", "clip")}) + roi[1].save(out / "roi" / v / f"{stem}.png") + roi[0].save(out / "roi" / v / f"{stem}.ref.png") + else: + scores.update({c: "" for c in ROI_COLS}) + out_png = Path("recon") / v / f"{stem}.png" + ref_png = Path("ref") / v / f"{stem}.png" + shown.save(out / out_png) + ref.save(out / ref_png) + row = { + "image": stem, "variant": v, "source": str(path), + "width": src.width, "height": src.height, + "bytes": total_bytes, "data_chunks": chunks, + "part_bytes": "+".join(map(str, part_bytes)), + "step": "+".join("1.00" if st is None else f"{st:.2f}" for st, _, _ in encoded), + **scores, + "encode_s": round(enc_s, 3), "decode_s": round(dec_s, 3), + "out_png": out_png.as_posix(), "ref_png": ref_png.as_posix(), + "bitstream": ";".join(os.path.relpath(byte_rows[str(tf)]["bitstream"], out) + for _, _, tf in encoded), + } + rows.append(row) + print(f"[{n}/{len(jobs)}] {stem} {v}: {row['bytes']} B, LPIPS {row['lpips']:.3f}, " + f"CLIP {row['clip']:.3f}, decode {dec_s:.2f}s", file=sys.stderr) + + summary = summarise(rows, variants) + meta = { + "date": dt.datetime.now().isoformat(timespec="seconds"), + "images": str(args.images), "count": len(images), "variants": variants, + "decoder": decoder_path.name, "decoder_sha256": sha256(decoder_path)[:16], + "decoder_provider": codec.decoder_provider, + "onnxruntime": ort.__version__, + } + with open(out / "results.csv", "w", newline="") as f: + w = csv.DictWriter(f, fieldnames=list(rows[0])) + w.writeheader() + w.writerows(rows) + (out / "summary.json").write_text(json.dumps({"meta": meta, "variants": summary}, indent=1)) + write_html(out, rows, summary, meta) + for tf in work.glob("*.f32"): # 2 MB each; the .aeic bitstreams are kept + tf.unlink() + print() + print_summary(summary) + print(f"\nreport: {out / 'report.html'}") + + +if __name__ == "__main__": + main() diff --git a/tools/aeic/eval/requirements.txt b/tools/aeic/eval/requirements.txt new file mode 100644 index 00000000..6abe164e --- /dev/null +++ b/tools/aeic/eval/requirements.txt @@ -0,0 +1,10 @@ +# For the CUDA build of torch, install it first from the PyTorch index, e.g. +# pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 +torch +torchvision +onnxruntime-gpu # or onnxruntime for CPU only +numpy +pillow +piq +open_clip_torch +imagecodecs # compare_codecs.py: JPEG XL and AVIF diff --git a/tools/aeic/eval/rois/kodak.json b/tools/aeic/eval/rois/kodak.json new file mode 100644 index 00000000..ce6727fc --- /dev/null +++ b/tools/aeic/eval/rois/kodak.json @@ -0,0 +1,20 @@ +{ + "_comment": "Subject boxes for the Kodak set, as fractions of width/height: [left, top, right, bottom]. Stand-ins for a user's 'select' crop.", + "kodim02": [0.15, 0.15, 0.70, 0.75], + "kodim03": [0.15, 0.05, 0.65, 0.65], + "kodim04": [0.20, 0.15, 0.80, 0.65], + "kodim05": [0.00, 0.00, 0.50, 1.00], + "kodim06": [0.15, 0.15, 0.80, 0.85], + "kodim09": [0.35, 0.10, 0.95, 0.75], + "kodim11": [0.35, 0.30, 0.85, 0.85], + "kodim12": [0.25, 0.20, 0.65, 0.85], + "kodim14": [0.00, 0.15, 0.90, 0.85], + "kodim15": [0.35, 0.00, 0.90, 0.75], + "kodim16": [0.40, 0.35, 0.90, 0.60], + "kodim17": [0.25, 0.00, 0.75, 0.45], + "kodim18": [0.25, 0.15, 0.85, 1.00], + "kodim19": [0.35, 0.00, 0.85, 0.75], + "kodim21": [0.30, 0.05, 0.70, 0.65], + "kodim22": [0.05, 0.10, 0.60, 0.70], + "kodim23": [0.50, 0.00, 1.00, 0.75] +}