Files
meshcore-open/tools/aeic/eval/eval.py
T
zjs81 aafd541b0f Add an AEIC quality and rate evaluation harness
tools/aeic/eval runs a folder of photos through the same ONNX graphs the
app downloads, counts exact bytes with the app's own entropy layer and
rANS coder (aeic_bytes.dart, checked byte-for-byte against the golden
recordings with --selftest), and scores what the receiver sees with
LPIPS, DISTS, CLIP, MS-SSIM and PSNR.

Variants cover preprocessing (stretch, crops, letterbox), subject
selection, the send sheet's re-roll tweaks, quantisation-step rate
control (fixed steps and per-image byte budgets) and two-part encodes.
compare_codecs.py compares AEIC with JPEG XL and AVIF at the same byte
budgets.
2026-11-14 20:58:43 -07:00

832 lines
35 KiB
Python

#!/usr/bin/env python3
"""Quality/rate evaluation harness for the AEIC image codec.
Runs a folder of photos through the same three ONNX graphs the app ships,
counts the exact bytes with the app's own Dart rANS coder (aeic_bytes.dart),
and scores what the receiver would see.
python -I tools/aeic/eval/eval.py --images ~/photos --out /tmp/aeic-eval
See README.md in this directory for setup and how to read the output.
"""
from __future__ import annotations
import argparse
import csv
import datetime as dt
import hashlib
import html
import json
import math
import os
import shutil
import subprocess
import sys
import time
import urllib.request
from dataclasses import dataclass
from pathlib import Path
from typing import Callable
import numpy as np
from PIL import Image, ImageOps
# torch must be imported before onnxruntime so the CUDA libraries that ship in
# the torch wheels are already loaded when ORT's CUDA provider looks for them.
import torch # noqa: E402 (import order is deliberate)
import onnxruntime as ort # noqa: E402
SIZE = 512
REPO = Path(__file__).resolve().parents[3]
DART_TOOL = Path(__file__).resolve().parent / "aeic_bytes.dart"
DEFAULT_MODELS = Path.home() / "Documents" / "image_codec_models"
SIDE_GRAPH = "aeic_entropy_side_fp32_op17.onnx"
DECODER_GRAPH = "aeic_decoder_qdq_conv_pct.onnx"
CDF_TABLE = "aeic_cdf_ft32.bin"
CONTEXT_GRAPH = "aeic_entropy_decode_fp32_op17.onnx"
def _stage_masks() -> list[np.ndarray]:
"""The four checkerboard masks, as AeicMaskSet.isLive defines them."""
perm = [0, 3, 2, 1]
c = np.arange(256)[:, None, None] // 64
yy = np.arange(16)[None, :, None]
xx = np.arange(16)[None, None, :]
micro = ((yy & 1) << 1) | (xx & 1)
return [((c ^ perm[s]) == micro)[None] for s in range(4)]
STAGE_MASKS = _stage_masks()
# Quantisation steps a stream could signal in 4 bits (the metadata byte's rate
# and resolution fields). 1.0 is today's ft32 stream.
STEP_LADDER = [0.40, 0.45, 0.50, 0.55, 0.60, 0.66, 0.72, 0.79,
0.86, 0.93, 1.00, 1.10, 1.20, 1.32, 1.45, 1.60]
IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".webp", ".bmp", ".tif", ".tiff"}
# Mirrors lib/services/image_chunk_transport.dart. aeic_bytes.dart reports the
# authoritative count; these are only used for the summary's packet totals.
PARITY_CHUNKS = 1
# --------------------------------------------------------------------------
# Preprocessing variants
# --------------------------------------------------------------------------
#
# A variant turns the (EXIF-rotated) source photo into the 512x512 codec input
# and says which part of the source it represents, so the reconstruction can
# be compared with that region at its true aspect ratio, which is what the
# receiver displays.
@dataclass
class Prepared:
square: Image.Image # 512x512 RGB, what the encoder sees
region: tuple[int, int, int, int] # source box (l, t, r, b) it represents
content_box: tuple[int, int, int, int] # box inside `square` holding it
# For multi-part variants: width in source pixels of the cross-fade along
# this part's edges that lie inside the picture (not on its border).
feather: int = 0
def _resize(img: Image.Image, size: tuple[int, int]) -> Image.Image:
# The app draws with dart:ui FilterQuality.medium: bilinear sampling over
# mipmaps. A box pre-reduce followed by bilinear is the closest PIL match.
return img.resize(size, Image.BILINEAR, reducing_gap=2.0)
def _full(img: Image.Image) -> tuple[int, int, int, int]:
return (0, 0, img.width, img.height)
def variant_stretch(img: Image.Image) -> Prepared:
"""What the app does today: the whole frame squeezed into the square."""
return Prepared(_resize(img, (SIZE, SIZE)), _full(img), (0, 0, SIZE, SIZE))
def _centre_crop_box(img: Image.Image, max_ratio: float) -> tuple[int, int, int, int]:
w, h = img.size
ratio = w / h
if ratio > max_ratio:
cw = round(h * max_ratio)
left = (w - cw) // 2
return (left, 0, left + cw, h)
if ratio < 1 / max_ratio:
ch = round(w * max_ratio)
top = (h - ch) // 2
return (0, top, w, top + ch)
return _full(img)
def variant_crop(img: Image.Image) -> Prepared:
"""Centre square crop: no distortion, loses the edges of the frame."""
box = _centre_crop_box(img, 1.0)
return Prepared(_resize(img.crop(box), (SIZE, SIZE)), box, (0, 0, SIZE, SIZE))
def variant_crop43(img: Image.Image) -> Prepared:
"""Crop to at most 4:3 (or 3:4), then stretch what is left."""
box = _centre_crop_box(img, 4 / 3)
return Prepared(_resize(img.crop(box), (SIZE, SIZE)), box, (0, 0, SIZE, SIZE))
def variant_pad(img: Image.Image) -> Prepared:
"""Letterbox on mid-grey: no distortion, spends latent area on the bars."""
w, h = img.size
scale = SIZE / max(w, h)
nw, nh = max(1, round(w * scale)), max(1, round(h * scale))
canvas = Image.new("RGB", (SIZE, SIZE), (128, 128, 128))
left, top = (SIZE - nw) // 2, (SIZE - nh) // 2
canvas.paste(_resize(img, (nw, nh)), (left, top))
return Prepared(canvas, _full(img), (left, top, left + nw, top + nh))
# Re-roll tweaks: (zoom, dx, dy) as fractions of the source. Mirrors
# kImageSendRerollTweaks / rerollSourceRect in
# lib/widgets/image_send_codec_binding.dart; keep the two in step.
REROLL_TWEAKS = [
(1.0, 0.0, 0.0),
(1.02, 0.005, 0.0),
(1.02, -0.005, 0.005),
(1.04, 0.0, -0.005),
(1.03, 0.01, 0.01),
(1.03, -0.01, -0.01),
]
def reroll_source_rect(w: int, h: int, variant: int) -> tuple[float, float, float, float]:
"""Source rect for re-roll `variant`: a near-invisible zoom and shift."""
zoom, dx, dy = REROLL_TWEAKS[variant]
cw, ch = w / zoom, h / zoom
left = min(max((w - cw) / 2 + dx * w, 0.0), w - cw)
top = min(max((h - ch) / 2 + dy * h, 0.0), h - ch)
return (left, top, left + cw, top + ch)
def variant_reroll(img: Image.Image, variant: int) -> Prepared:
"""Whole photo with re-roll tweak `variant`, stretched like `stretch`.
Scored against the full frame: the tweak only drops a percent or two at
the edges, which is what the sender sees as "the whole photo".
"""
l, t, r, b = reroll_source_rect(img.width, img.height, variant)
# PIL's resize box takes float source coordinates, like drawImageRect.
square = img.resize((SIZE, SIZE), Image.BILINEAR, box=(l, t, r, b), reducing_gap=2.0)
return Prepared(square, _full(img), (0, 0, SIZE, SIZE))
VARIANTS: dict[str, Callable[[Image.Image], Prepared]] = {
"stretch": variant_stretch,
"crop": variant_crop,
"crop43": variant_crop43,
"pad": variant_pad,
**{f"reroll{k}": (lambda img, k=k: variant_reroll(img, k)) for k in range(1, len(REROLL_TWEAKS))},
}
# Need a box per image (--rois), so they are not in VARIANTS.
SELECT = "select"
SELECT_SQUARE = "select_sq"
SELECT_VARIANTS = (SELECT, SELECT_SQUARE)
Box = tuple[int, int, int, int]
def roi_box(img: Image.Image, frac: list[float]) -> Box:
"""Fractional [l, t, r, b] -> pixel box in the (EXIF-rotated) source."""
l, t, r, b = frac
w, h = img.size
return (round(l * w), round(t * h), round(r * w), round(b * h))
def variant_select(img: Image.Image, box: Box) -> Prepared:
"""The user's selection, stretched into the square like `stretch`.
The aspect nibble restores the box's shape on the receiver, so this is the
app's existing pipeline applied to a crop the user chose.
"""
return Prepared(_resize(img.crop(box), (SIZE, SIZE)), box, (0, 0, SIZE, SIZE))
def square_box(img: Image.Image, box: Box) -> Box:
"""Grow `box` to a square around its centre, shifted to stay in the image.
Models a selection tool locked to 1:1, so the codec never sees a stretch.
Capped at the image's short side.
"""
l, t, r, b = box
side = min(max(r - l, b - t), img.width, img.height)
cx, cy = (l + r) / 2, (t + b) / 2
nl = int(round(min(max(cx - side / 2, 0), img.width - side)))
nt = int(round(min(max(cy - side / 2, 0), img.height - side)))
return (nl, nt, nl + side, nt + side)
# --------------------------------------------------------------------------
# HD: two encoded parts per photo, aimed at a two-packet budget
# --------------------------------------------------------------------------
#
# The model is fixed at 512x512 / ft32, so "HD" spends the second packet on a
# second 512x512 encode. The parts travel as one stream; the receiver decodes
# both and blends them. Overheads below are what the stream would need on top
# of the bitstreams (a hypothetical layout, not implemented in the app).
HD_TILES = "hd_tiles"
HD_DETAIL = "hd_detail"
HD_VARIANTS = (HD_TILES, HD_DETAIL)
HD_OVERHEAD = {
HD_TILES: 1, # length of part 0, so the receiver can split the stream
HD_DETAIL: 1 + 4, # that, plus the detail box quantised to 4 bytes
}
TILE_OVERLAP = 0.08 # of the long side, cross-faded at the seam
def variant_hd_tiles(img: Image.Image) -> list[Prepared]:
"""Two overlapping halves along the long side, each encoded at 512x512.
Doubles the pixel count. A 3:2 photo splits into two near-square halves,
so each half is also stretched far less than the whole frame would be.
"""
w, h = img.size
long_side = max(w, h)
ov = round(long_side * TILE_OVERLAP)
half = (long_side + ov) // 2
if w >= h:
boxes = [(0, 0, half, h), (w - half, 0, w, h)]
else:
boxes = [(0, 0, w, half), (0, h - half, w, h)]
return [Prepared(_resize(img.crop(b), (SIZE, SIZE)), b, (0, 0, SIZE, SIZE), feather=ov)
for b in boxes]
def variant_hd_detail(img: Image.Image, box: Box) -> list[Prepared]:
"""The whole photo, plus the selected subject blended in at higher detail."""
detail = variant_select(img, box)
detail.feather = max(4, round(0.08 * min(box[2] - box[0], box[3] - box[1])))
return [variant_stretch(img), detail]
def composite(src: Image.Image, parts: list[Prepared], recons: list[Image.Image],
long_side: int) -> tuple[Image.Image, Image.Image, Box]:
"""Reference and receiver view for a variant made of one or more parts.
The view covers the union of the parts' regions at `long_side` pixels on
its long edge. Parts are laid down in order; each later part fades in over
`feather` pixels along its edges that are inside the union, so seams
cross-fade instead of cutting hard.
"""
region = (min(p.region[0] for p in parts), min(p.region[1] for p in parts),
max(p.region[2] for p in parts), max(p.region[3] for p in parts))
rw, rh = region[2] - region[0], region[3] - region[1]
s = long_side / max(rw, rh)
size = (max(1, round(rw * s)), max(1, round(rh * s)))
ref = src.crop(region).resize(size, Image.LANCZOS)
canvas = np.zeros((size[1], size[0], 3), dtype=np.float64)
for i, (p, recon) in enumerate(zip(parts, recons)):
l, t, r, b = p.region
x0, y0 = round((l - region[0]) * s), round((t - region[1]) * s)
x1, y1 = round((r - region[0]) * s), round((b - region[1]) * s)
tile = np.asarray(recon.crop(p.content_box).resize((x1 - x0, y1 - y0), Image.LANCZOS),
dtype=np.float64)
alpha = np.ones((y1 - y0, x1 - x0), dtype=np.float64)
f = min(round(p.feather * s), alpha.shape[0], alpha.shape[1])
if i > 0 and f > 0:
ramp = np.arange(1, f + 1, dtype=np.float64) / f
if l > region[0]:
alpha[:, :f] *= ramp[None, :]
if r < region[2]:
alpha[:, -f:] *= ramp[::-1][None, :]
if t > region[1]:
alpha[:f, :] *= ramp[:, None]
if b < region[3]:
alpha[-f:, :] *= ramp[::-1][:, None]
a = alpha[..., None]
canvas[y0:y1, x0:x1] = canvas[y0:y1, x0:x1] * (1 - a) + tile * a
shown = Image.fromarray(np.clip(np.rint(canvas), 0, 255).astype(np.uint8), "RGB")
return ref, shown, region
# Mirrors imageDataChunkCount() in image_chunk_transport.dart.
CHUNK_FIRST_CAPACITY = 157
CHUNK_CAPACITY = 158
def parse_rate_variant(v: str) -> dict | None:
"""`q0.7` -> fixed step 0.7; `fit315` -> per-image step to fit 315 bytes."""
try:
if v.startswith("q"):
return {"steps": [float(v[1:])], "budget": None}
if v.startswith("fit"):
return {"steps": STEP_LADDER, "budget": int(v[3:])}
except ValueError:
return None
return None
def data_chunks(payload_bytes: int) -> int:
n = max(payload_bytes, 0)
if n <= CHUNK_FIRST_CAPACITY:
return 1
return 1 + math.ceil((n - CHUNK_FIRST_CAPACITY) / CHUNK_CAPACITY)
# --------------------------------------------------------------------------
# Codec
# --------------------------------------------------------------------------
class Codec:
def __init__(self, models: Path, decoder: Path, gpu: bool) -> None:
cpu = ["CPUExecutionProvider"]
cuda = ["CUDAExecutionProvider", "CPUExecutionProvider"] if gpu else cpu
# The entropy graph stays on CPU: its rounded outputs decide the bytes,
# and CPU fp32 is the closest stand-in for the phone. It costs ~0.1 s.
self.side = ort.InferenceSession(str(models / SIDE_GRAPH), providers=cpu)
self.context = ort.InferenceSession(str(models / CONTEXT_GRAPH), providers=cpu)
self.decoder = ort.InferenceSession(str(decoder), providers=cuda)
self.decoder_provider = self.decoder.get_providers()[0]
def encode_side(self, square: Image.Image) -> dict[str, np.ndarray]:
rgb = np.asarray(square, dtype=np.uint8)
# Same float32 steps as aeicRgbToChw in image_codec_entropy.dart.
chw = rgb.transpose(2, 0, 1).astype(np.float32) / np.float32(255.0)
chw = (chw - np.float32(0.5)) * np.float32(2.0)
names = (["z_q", "y_hat", "y", "base0"]
+ [f"yq{i}" for i in range(4)] + [f"sc{i}" for i in range(4)])
out = self.side.run(names, {"image": chw[None]})
return dict(zip(names, out))
def encode_with_step(self, side: dict[str, np.ndarray], step: float) -> dict[str, np.ndarray]:
"""Re-quantise y with step `step` instead of 1 (rate control).
Replays the four-stage context loop the receiver runs
(AeicEntropyCodec.decodeToLatent) with the decode-side graph, coding
round((y - mean) / step) against scale / step. The receiver's
reconstruction becomes symbol * step + mean. step < 1 spends more
bytes, step > 1 fewer. At step == 1 this reproduces the side graph's
symbols, which the harness's q1 check relies on.
"""
st = np.float32(step)
y, z_q = side["y"], side["z_q"]
base = side["base0"].copy()
out: dict[str, np.ndarray] = {"z_q": z_q}
for stage in range(4):
means, scales = self.context.run(
["means", "scales"],
{"z_q": z_q, "base": base, "stage": np.array([stage], dtype=np.int32)})
m = STAGE_MASKS[stage]
sym = np.where(m, np.rint((y - means) / st), np.float32(0)).astype(np.float32)
base = np.where(m, sym * st + means, base).astype(np.float32)
out[f"yq{stage}"] = sym
out[f"sc{stage}"] = np.where(m, scales / st, np.float32(0)).astype(np.float32)
out["y_hat"] = base
return out
def decode(self, y_hat: np.ndarray) -> Image.Image:
(out,) = self.decoder.run(["image"], {"y_hat": y_hat.astype(np.float32)})
# Same mapping as _chwFloatsToRgb in image_codec_backend.dart: clamp,
# then round half away from zero (Dart's double.round()).
scaled = (out[0].astype(np.float64) + 1.0) * 127.5
rgb = np.floor(np.clip(scaled, 0, 255) + 0.5).astype(np.uint8)
return Image.fromarray(rgb.transpose(1, 2, 0), "RGB")
def write_tensor_file(path: Path, t: dict[str, np.ndarray]) -> None:
parts = [t["z_q"]] + [t[f"yq{i}"] for i in range(4)] + [t[f"sc{i}"] for i in range(4)]
with open(path, "wb") as f:
for p in parts:
f.write(np.ascontiguousarray(p, dtype="<f4").tobytes())
def count_bytes(cdf: Path, tensor_files: list[Path]) -> dict[str, dict]:
if not tensor_files:
return {}
dart = shutil.which("dart")
if dart is None:
sys.exit("dart is not on PATH; it is needed to run aeic_bytes.dart")
cmd = [dart, "run", str(DART_TOOL), str(cdf), *map(str, tensor_files)]
proc = subprocess.run(cmd, cwd=REPO, capture_output=True, text=True)
if proc.returncode != 0:
sys.exit(f"aeic_bytes.dart failed:\n{proc.stderr}")
results = {}
for line in proc.stdout.splitlines():
line = line.strip()
if line.startswith("{"):
row = json.loads(line)
results[row["file"]] = row
return results
# --------------------------------------------------------------------------
# Metrics
# --------------------------------------------------------------------------
class Metrics:
"""Full-reference scores. Higher is better except LPIPS and DISTS."""
def __init__(self, device: str) -> None:
import open_clip
import piq
self.piq = piq
self.device = device
self.lpips = piq.LPIPS(reduction="none").to(device).eval()
self.dists = piq.DISTS(reduction="none").to(device).eval()
model, _, preprocess = open_clip.create_model_and_transforms(
"ViT-B-32", pretrained="laion2b_s34b_b79k"
)
self.clip = model.to(device).eval()
self.clip_pre = preprocess
def _t(self, img: Image.Image) -> torch.Tensor:
a = np.asarray(img, dtype=np.float32) / 255.0
return torch.from_numpy(a).permute(2, 0, 1)[None].to(self.device)
@torch.no_grad()
def clip_embed(self, img: Image.Image) -> torch.Tensor:
# CLIP's own preprocessing centre-crops to a square, which would hide
# the frame edges -- exactly what crop variants throw away. Letterbox
# first so the whole image is scored.
side = max(img.size)
boxed = Image.new("RGB", (side, side), (0, 0, 0))
boxed.paste(img, ((side - img.width) // 2, (side - img.height) // 2))
x = self.clip_pre(boxed)[None].to(self.device)
e = self.clip.encode_image(x)
return e / e.norm(dim=-1, keepdim=True)
@torch.no_grad()
def score(self, ref: Image.Image, out: Image.Image, full_src_emb: torch.Tensor) -> dict:
r, o = self._t(ref), self._t(out)
out_emb = self.clip_embed(out)
return {
"psnr": float(self.piq.psnr(o, r, data_range=1.0)),
"ms_ssim": float(self.piq.multi_scale_ssim(o, r, data_range=1.0)),
"lpips": float(self.lpips(o, r)),
"dists": float(self.dists(o, r)),
# Semantic agreement with exactly what was sent ...
"clip": float((self.clip_embed(ref) * out_emb).sum()),
# ... and with the whole original photo, so cropping is charged
# for what it threw away.
"clip_full": float((full_src_emb * out_emb).sum()),
}
def roi_pair(src: Image.Image, region: Box, shown: Image.Image, box: Box) -> tuple[Image.Image, Image.Image] | None:
"""The ROI as the receiver would see it after zooming in, plus its reference.
`shown` covers `region` of the source; the ROI is mapped into it and
both crops are scaled to long side 512. For a full-frame variant that is an
upscale of a few dozen pixels, which is exactly what zooming in shows.
Returns None when the variant did not send the whole ROI.
"""
rl, rt, rr, rb = region
l, t, r, b = box
if l < rl or t < rt or r > rr or b > rb:
return None
sx = shown.width / (rr - rl)
sy = shown.height / (rb - rt)
out = shown.crop((round((l - rl) * sx), round((t - rt) * sy),
round((r - rl) * sx), round((b - rt) * sy)))
ref = src.crop(box)
scale = SIZE / max(ref.size)
size = (max(1, round(ref.width * scale)), max(1, round(ref.height * scale)))
return ref.resize(size, Image.LANCZOS), out.resize(size, Image.BICUBIC)
# --------------------------------------------------------------------------
# Corpus
# --------------------------------------------------------------------------
KODAK_URL = "https://r0k.us/graphics/kodak/kodak/kodim{:02d}.png"
def download_kodak(dest: Path) -> None:
dest.mkdir(parents=True, exist_ok=True)
for i in range(1, 25):
target = dest / f"kodim{i:02d}.png"
if target.exists():
continue
print(f"downloading {target.name}", file=sys.stderr)
tmp = target.with_suffix(".part")
with urllib.request.urlopen(KODAK_URL.format(i), timeout=60) as r, open(tmp, "wb") as f:
shutil.copyfileobj(r, f)
tmp.rename(target)
def load_source(path: Path) -> Image.Image:
img = Image.open(path)
# Picked photos are shown upright in the app, so honour EXIF orientation.
img = ImageOps.exif_transpose(img)
return img.convert("RGB")
def find_images(folder: Path, limit: int | None) -> list[Path]:
files = sorted(p for p in folder.rglob("*") if p.suffix.lower() in IMAGE_SUFFIXES)
return files[:limit] if limit else files
# --------------------------------------------------------------------------
# Report
# --------------------------------------------------------------------------
METRIC_COLS = ["bytes", "data_chunks", "psnr", "ms_ssim", "lpips", "dists", "clip", "clip_full"]
ROI_COLS = ["roi_psnr", "roi_ms_ssim", "roi_lpips", "roi_dists", "roi_clip"]
LOWER_IS_BETTER = {"bytes", "data_chunks", "lpips", "dists", "roi_lpips", "roi_dists"}
def summarise(rows: list[dict], variants: list[str]) -> dict:
summary = {}
for v in variants:
vr = [r for r in rows if r["variant"] == v]
if not vr:
continue
s: dict = {"images": len(vr)}
for m in METRIC_COLS + ROI_COLS:
vals = np.array([r[m] for r in vr if r.get(m) not in (None, "")], dtype=np.float64)
if vals.size == 0:
continue
s[m] = {"mean": float(vals.mean()), "median": float(np.median(vals)),
"min": float(vals.min()), "max": float(vals.max())}
chunks = [r["data_chunks"] for r in vr]
s["chunk_histogram"] = {str(k): chunks.count(k) for k in sorted(set(chunks))}
s["packets_mean_with_parity"] = float(np.mean(chunks)) + PARITY_CHUNKS
# The share of images whose CLIP agreement with the source is low.
# This is the "fox came back as a sheep" bucket; worth eyeballing.
s["clip_below_0.80"] = sum(1 for r in vr if r["clip"] < 0.80)
summary[v] = s
return summary
def print_summary(summary: dict) -> None:
head = (f"{'variant':<10}" + "".join(f"{m:>12}" for m in METRIC_COLS)
+ f"{'1-pkt':>8}{'<=2-pkt':>9}{'clip<.8':>9}")
print(head)
print("-" * len(head))
for v, s in summary.items():
line = f"{v:<10}" + "".join(f"{s[m]['mean']:>12.4g}" for m in METRIC_COLS)
one = s["chunk_histogram"].get("1", 0)
two = one + s["chunk_histogram"].get("2", 0)
line += f"{one / s['images']:>7.0%} {two / s['images']:>8.0%} {s['clip_below_0.80']:>8}"
print(line)
print("means; lower is better for bytes, data_chunks, lpips, dists")
if any("roi_lpips" in s for s in summary.values()):
print()
print(f"{'variant':<10}" + "".join(f"{m:>13}" for m in ROI_COLS))
for v, s in summary.items():
if "roi_lpips" in s:
print(f"{v:<10}" + "".join(f"{s[m]['mean']:>13.4g}" for m in ROI_COLS))
print("scored on the selected region only, as the receiver sees it after zooming in")
def write_html(out: Path, rows: list[dict], summary: dict, meta: dict) -> None:
def fmt(v: float, m: str) -> str:
return f"{v:.1f}" if m in ("bytes", "data_chunks") else f"{v:.3f}"
variants = list(summary)
stems = list(dict.fromkeys(r["image"] for r in rows))
by = {(r["image"], r["variant"]): r for r in rows}
sum_rows = "".join(
f"<tr><th>{html.escape(v)}</th>"
+ "".join(f"<td>{fmt(summary[v][m]['mean'], m)}</td>" for m in METRIC_COLS)
+ "</tr>"
for v in variants
)
cards = []
for stem in stems:
cells = []
first = by[(stem, variants[0])]
cells.append(
f'<figure><img loading="lazy" src="{html.escape(first["ref_png"])}">'
f"<figcaption>source</figcaption></figure>"
)
for v in variants:
r = by.get((stem, v))
if not r:
continue
warn = " warn" if r["clip"] < 0.80 else ""
cells.append(
f'<figure class="{warn.strip()}"><img loading="lazy" src="{html.escape(r["out_png"])}">'
f"<figcaption><b>{html.escape(v)}</b> {r['bytes']} B / {r['data_chunks']} pkt<br>"
f"LPIPS {r['lpips']:.3f} · CLIP {r['clip']:.3f}</figcaption></figure>"
)
cards.append(f"<section><h3>{html.escape(stem)}</h3><div class=row>{''.join(cells)}</div></section>")
doc = f"""<!doctype html><html lang=en><meta charset=utf-8>
<meta name=viewport content="width=device-width,initial-scale=1">
<title>AEIC eval</title>
<style>
:root{{color-scheme:light dark;--bg:#fff;--fg:#111;--muted:#666;--line:#ddd;--warn:#c62828}}
@media (prefers-color-scheme:dark){{:root{{--bg:#121212;--fg:#eee;--muted:#aaa;--line:#333;--warn:#ef5350}}}}
body{{background:var(--bg);color:var(--fg);font:15px/1.4 system-ui,sans-serif;margin:16px}}
table{{border-collapse:collapse;margin:8px 0 24px}}td,th{{border:1px solid var(--line);padding:4px 8px;text-align:right}}
.row{{display:flex;gap:8px;flex-wrap:wrap}}figure{{margin:0;width:256px}}
figure img{{width:256px;height:auto;display:block;border:2px solid transparent}}
figure.warn img{{border-color:var(--warn)}}figcaption{{font-size:13px;color:var(--muted)}}
code{{font-size:13px}}
</style>
<h1>AEIC eval</h1>
<p><code>{html.escape(json.dumps(meta))}</code></p>
<h2>Means</h2>
<table><tr><th></th>{''.join(f'<th>{m}</th>' for m in METRIC_COLS)}</tr>{sum_rows}</table>
<p>Images are shown at the aspect ratio the receiver restores. A red border means CLIP similarity to the sent region is below 0.80, so check it by eye.</p>
{''.join(cards)}
</html>"""
(out / "report.html").write_text(doc, encoding="utf-8")
# --------------------------------------------------------------------------
# Main
# --------------------------------------------------------------------------
def sha256(path: Path) -> str:
h = hashlib.sha256()
with open(path, "rb") as f:
for block in iter(lambda: f.read(1 << 20), b""):
h.update(block)
return h.hexdigest()
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--images", type=Path, help="folder of photos (searched recursively)")
ap.add_argument("--download-kodak", type=Path, metavar="DIR",
help="download the 24 Kodak images to DIR and use them")
ap.add_argument("--out", type=Path, required=True, help="output folder")
ap.add_argument("--models", type=Path, default=DEFAULT_MODELS,
help=f"model bundle folder (default {DEFAULT_MODELS})")
ap.add_argument("--decoder", type=Path, help="alternative decoder .onnx (e.g. an fp16 export)")
ap.add_argument("--variants", default="stretch",
help=f"comma list from: {', '.join(VARIANTS)} (default stretch)")
ap.add_argument("--limit", type=int, help="only the first N images")
ap.add_argument("--cpu", action="store_true", help="run the decoder on CPU too")
ap.add_argument("--rois", type=Path,
help="JSON {image stem: [l, t, r, b] fractions}; enables the select and "
"hd_detail variants and ROI scores, and limits the run to listed images")
ap.add_argument("--display", type=int, default=SIZE,
help="long side in pixels of the receiver view that is scored (default 512; "
"use 1024 when comparing HD variants)")
args = ap.parse_args()
if args.download_kodak:
download_kodak(args.download_kodak)
args.images = args.images or args.download_kodak
if not args.images:
ap.error("give --images or --download-kodak")
variants = [v.strip() for v in args.variants.split(",") if v.strip()]
unknown = [v for v in variants
if v not in VARIANTS and v not in SELECT_VARIANTS and v not in HD_VARIANTS
and parse_rate_variant(v) is None]
if unknown:
ap.error(f"unknown variants: {unknown}")
rois: dict[str, list[float]] = {}
if args.rois:
rois = {k: v for k, v in json.loads(args.rois.read_text()).items() if not k.startswith("_")}
if any(v in SELECT_VARIANTS + (HD_DETAIL,) for v in variants) and not rois:
ap.error("the select and hd_detail variants need --rois")
images = find_images(args.images, None)
if rois:
images = [p for p in images if p.relative_to(args.images).with_suffix("").as_posix() in rois]
images = images[: args.limit] if args.limit else images
if not images:
sys.exit(f"no images under {args.images}")
out: Path = args.out
work = out / "tensors"
dirs = [work] + [out / k / v for k in ("recon", "ref", "roi") for v in variants]
for d in dirs:
d.mkdir(parents=True, exist_ok=True)
decoder_path = args.decoder or (args.models / DECODER_GRAPH)
gpu = not args.cpu and torch.cuda.is_available()
codec = Codec(args.models, decoder_path, gpu)
metrics = Metrics("cuda" if gpu else "cpu")
print(f"{len(images)} images x {len(variants)} variants; decoder on {codec.decoder_provider}",
file=sys.stderr)
# Pass 1: preprocess + encode side, so every byte count is one Dart run.
jobs = []
for path in images:
src = load_source(path)
key = path.relative_to(args.images).with_suffix("").as_posix()
stem = key.replace("/", "__")
box = roi_box(src, rois[key]) if key in rois else None
for v in variants:
if (v in SELECT_VARIANTS or v == HD_DETAIL) and box is None:
continue
if v == SELECT:
parts = [variant_select(src, box)]
elif v == SELECT_SQUARE:
parts = [variant_select(src, square_box(src, box))]
elif v == HD_TILES:
parts = variant_hd_tiles(src)
elif v == HD_DETAIL:
parts = variant_hd_detail(src, box)
elif parse_rate_variant(v) is not None:
parts = [variant_stretch(src)]
else:
parts = [VARIANTS[v](src)]
rate = parse_rate_variant(v)
steps = rate["steps"] if rate else [None]
t0 = time.perf_counter()
encoded = [] # per part: [(step, y_hat, tensor file), ...]
for i, p in enumerate(parts):
side = codec.encode_side(p.square)
cands = []
for step in steps:
tensors = side if step is None else codec.encode_with_step(side, step)
tag = "" if step is None else f".s{step:.2f}"
tf = work / f"{stem}.{v}.p{i}{tag}.f32"
write_tensor_file(tf, tensors)
cands.append((step, tensors["y_hat"], tf))
encoded.append(cands)
enc_s = time.perf_counter() - t0
jobs.append((path, stem, v, src, parts, box, encoded, enc_s))
byte_rows = count_bytes(args.models / CDF_TABLE,
[tf for j in jobs for cands in j[6] for _, _, tf in cands])
# Pick one candidate per part. A fit<N> variant takes the finest step whose
# stream fits N bytes, or the coarsest step if none does.
for j in jobs:
rate = parse_rate_variant(j[2])
chosen = []
for cands in j[6]:
if rate and rate["budget"] is not None:
fitting = [c for c in cands if byte_rows[str(c[2])]["bytes"] <= rate["budget"]]
chosen.append(min(fitting, key=lambda c: c[0]) if fitting
else max(cands, key=lambda c: c[0]))
else:
chosen.append(cands[0])
j[6][:] = chosen
# Pass 2: decode + score.
rows = []
emb_cache: dict[str, torch.Tensor] = {}
for n, (path, stem, v, src, parts, box, encoded, enc_s) in enumerate(jobs, 1):
t0 = time.perf_counter()
recons = [codec.decode(y_hat) for _, y_hat, _ in encoded]
dec_s = time.perf_counter() - t0
ref, shown, region = composite(src, parts, recons, args.display)
part_bytes = [byte_rows[str(tf)]["bytes"] for _, _, tf in encoded]
total_bytes = sum(part_bytes) + HD_OVERHEAD.get(v, 0)
chunks = data_chunks(total_bytes)
if len(parts) == 1:
assert chunks == byte_rows[str(encoded[0][2])]["data_chunks"], "chunk mirror drifted"
if stem not in emb_cache:
emb_cache[stem] = metrics.clip_embed(src)
scores = metrics.score(ref, shown, emb_cache[stem])
roi = roi_pair(src, region, shown, box) if box else None
if roi:
roi_scores = metrics.score(roi[0], roi[1], emb_cache[stem])
scores.update({f"roi_{k}": roi_scores[k] for k in ("psnr", "ms_ssim", "lpips", "dists", "clip")})
roi[1].save(out / "roi" / v / f"{stem}.png")
roi[0].save(out / "roi" / v / f"{stem}.ref.png")
else:
scores.update({c: "" for c in ROI_COLS})
out_png = Path("recon") / v / f"{stem}.png"
ref_png = Path("ref") / v / f"{stem}.png"
shown.save(out / out_png)
ref.save(out / ref_png)
row = {
"image": stem, "variant": v, "source": str(path),
"width": src.width, "height": src.height,
"bytes": total_bytes, "data_chunks": chunks,
"part_bytes": "+".join(map(str, part_bytes)),
"step": "+".join("1.00" if st is None else f"{st:.2f}" for st, _, _ in encoded),
**scores,
"encode_s": round(enc_s, 3), "decode_s": round(dec_s, 3),
"out_png": out_png.as_posix(), "ref_png": ref_png.as_posix(),
"bitstream": ";".join(os.path.relpath(byte_rows[str(tf)]["bitstream"], out)
for _, _, tf in encoded),
}
rows.append(row)
print(f"[{n}/{len(jobs)}] {stem} {v}: {row['bytes']} B, LPIPS {row['lpips']:.3f}, "
f"CLIP {row['clip']:.3f}, decode {dec_s:.2f}s", file=sys.stderr)
summary = summarise(rows, variants)
meta = {
"date": dt.datetime.now().isoformat(timespec="seconds"),
"images": str(args.images), "count": len(images), "variants": variants,
"decoder": decoder_path.name, "decoder_sha256": sha256(decoder_path)[:16],
"decoder_provider": codec.decoder_provider,
"onnxruntime": ort.__version__,
}
with open(out / "results.csv", "w", newline="") as f:
w = csv.DictWriter(f, fieldnames=list(rows[0]))
w.writeheader()
w.writerows(rows)
(out / "summary.json").write_text(json.dumps({"meta": meta, "variants": summary}, indent=1))
write_html(out, rows, summary, meta)
for tf in work.glob("*.f32"): # 2 MB each; the .aeic bitstreams are kept
tf.unlink()
print()
print_summary(summary)
print(f"\nreport: {out / 'report.html'}")
if __name__ == "__main__":
main()