From aafd541b0f6b1f3f25b64cf3b3a868296559d922 Mon Sep 17 00:00:00 2001 From: zjs81 Date: Sat, 14 Nov 2026 20:58:43 -0700 Subject: [PATCH 1/2] Add an AEIC quality and rate evaluation harness tools/aeic/eval runs a folder of photos through the same ONNX graphs the app downloads, counts exact bytes with the app's own entropy layer and rANS coder (aeic_bytes.dart, checked byte-for-byte against the golden recordings with --selftest), and scores what the receiver sees with LPIPS, DISTS, CLIP, MS-SSIM and PSNR. Variants cover preprocessing (stretch, crops, letterbox), subject selection, the send sheet's re-roll tweaks, quantisation-step rate control (fixed steps and per-image byte budgets) and two-part encodes. compare_codecs.py compares AEIC with JPEG XL and AVIF at the same byte budgets. --- tools/aeic/README.md | 4 + tools/aeic/eval/README.md | 115 +++++ tools/aeic/eval/aeic_bytes.dart | 240 +++++++++ tools/aeic/eval/compare_codecs.py | 256 +++++++++ tools/aeic/eval/eval.py | 831 ++++++++++++++++++++++++++++++ tools/aeic/eval/requirements.txt | 10 + tools/aeic/eval/rois/kodak.json | 20 + 7 files changed, 1476 insertions(+) create mode 100644 tools/aeic/eval/README.md create mode 100644 tools/aeic/eval/aeic_bytes.dart create mode 100644 tools/aeic/eval/compare_codecs.py create mode 100644 tools/aeic/eval/eval.py create mode 100644 tools/aeic/eval/requirements.txt create mode 100644 tools/aeic/eval/rois/kodak.json diff --git a/tools/aeic/README.md b/tools/aeic/README.md index a246f15d..c3ce2c76 100644 --- a/tools/aeic/README.md +++ b/tools/aeic/README.md @@ -9,6 +9,10 @@ Dart side is checked against. They are committed here so `test/services/golden/` is reproducible. Nothing in the app runs them. +To measure reconstruction quality and byte cost instead, use the +[evaluation harness](eval/README.md), which runs on the downloaded model bundle +and needs no research checkout. + | script | produces | used by | |---|---|---| | `export_golden.py` | `golden/aeic_cdf_ft32.bin`, `golden/vectors/*.gv` | `rans_coder_test.dart`, `entropy_tables_test.dart` | diff --git a/tools/aeic/eval/README.md b/tools/aeic/eval/README.md new file mode 100644 index 00000000..effd2c10 --- /dev/null +++ b/tools/aeic/eval/README.md @@ -0,0 +1,115 @@ +# AEIC quality evaluation + +`eval.py` measures how good received images are and how many bytes they cost, +so codec changes can be compared with numbers rather than by eye. It runs the +same ONNX graphs the app downloads. Unlike the fixture scripts in the parent +directory, it needs no research checkout. + +For each photo and each preprocessing variant it: + +1. prepares the 512×512 codec input (the `stretch` variant mirrors the app's `_toSquareRgb`), +2. runs `aeic_entropy_side_fp32_op17.onnx` on CPU, +3. counts the exact bytes with `aeic_bytes.dart`, which calls the app's own entropy layer and rANS coder, +4. runs the decoder on `y_hat` (GPU when available), +5. restores the aspect ratio the way the receiver does and scores the result against the region that was sent. + +## Setup + +```bash +python3 -m venv ~/.venvs/aeic-eval # or: uv venv ~/.venvs/aeic-eval +~/.venvs/aeic-eval/bin/pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 +~/.venvs/aeic-eval/bin/pip install -r tools/aeic/eval/requirements.txt +``` + +You also need `dart` on PATH (`flutter pub get` already run), and the model +bundle. By default the bundle is read from `~/Documents/image_codec_models`; +use `--models` to point elsewhere. The first run downloads the LPIPS, DISTS and +CLIP weights into the torch and Hugging Face caches. + +Check that the byte counter matches the reference coder: + +```bash +dart run tools/aeic/eval/aeic_bytes.dart --selftest test/services/golden/e2e/*.aeicrec +``` + +## Running + +From the repository root: + +```bash +# Kodak (24 images, downloaded once) +~/.venvs/aeic-eval/bin/python -I tools/aeic/eval/eval.py \ + --download-kodak ~/aeic-data/kodak --out ~/aeic-runs/kodak-baseline \ + --variants stretch,crop,crop43,pad + +# Your own photos; Kodak alone is not representative of what people send +~/.venvs/aeic-eval/bin/python -I tools/aeic/eval/eval.py \ + --images ~/aeic-data/mesh-photos --out ~/aeic-runs/mesh-baseline +``` + +| option | meaning | +|---|---| +| `--variants` | comma list of variants to compare (below) | +| `--rois FILE` | JSON `{image stem: [l, t, r, b]}` as fractions; adds subject-region (`roi_*`) scores and limits the run to listed images. `rois/kodak.json` has 17 Kodak subjects | +| `--display N` | long side of the receiver view that is scored (default 512; use 1024 for two-part variants) | +| `--decoder` | score a different decoder export, e.g. fp16 instead of int8 | +| `--limit N` | first N images only | +| `--cpu` | run the decoder on CPU (much slower) | + +| variant | what is sent | +|---|---| +| `stretch` | the whole photo stretched to the square, as the app does | +| `crop`, `crop43`, `pad` | centre square crop; crop to at most 4:3 then stretch; letterbox on grey | +| `reroll1`..`reroll5` | the whole photo with the send sheet's "Try another version" tweaks (`kImageSendRerollTweaks`; keep the two tables in step) | +| `select`, `select_sq` | only the `--rois` box; the box grown to a square | +| `q` | quantisation step `` instead of 1 (e.g. `q0.7` spends more bytes) | +| `fit` | per image, the finest step from the 16-step ladder that fits `` (e.g. `fit157` = one packet) | +| `hd_tiles`, `hd_detail` | two encodes per photo: two overlapping halves; or the whole photo plus the `--rois` box, blended | + +`q`/`fit` re-run the four context stages with the decode-side entropy graph; +at step 1 they reproduce the app's bytes exactly. Two-part byte counts include +the extra stream header such a format would need. + +To add an experiment, write a function returning `Prepared` and register it in +`VARIANTS`. + +## Comparing with JPEG XL and AVIF + +`compare_codecs.py` sweeps AEIC over every quantisation step and fits JPEG XL +and AVIF to the same budgets (16–192 px thumbnails, best size per image, scored +after upscaling). AVIF is counted both as a real `.avif` file and as its AV1 +data alone. Needs `imagecodecs` (in `requirements.txt`). + +```bash +~/.venvs/aeic-eval/bin/python -I tools/aeic/eval/compare_codecs.py \ + --images ~/aeic-data/kodak --out ~/aeic-runs/rd +``` + +## Output + +| file | contents | +|---|---| +| `report.html` | source next to each variant's reconstruction, with bytes, packets, LPIPS and CLIP; a red border marks CLIP < 0.80 | +| `results.csv` | one row per image × variant | +| `summary.json` | per-variant mean/median/min/max, packet histogram, run metadata | +| `recon/`, `ref/` | the scored images, at the receiver's aspect ratio | +| `tensors/*.aeic` | the bitstreams, exactly as the app would send them | + +## Reading the metrics + +| metric | better | what it catches | +|---|---|---| +| `bytes`, `data_chunks` | lower | airtime; one chunk carries up to 157 image bytes | +| `lpips`, `dists` | lower | perceptual difference; the main quality score for a generative decoder | +| `clip` | higher | whether the reconstruction shows the same *things* as the sent region; low values are the "fox came back as a sheep" failures | +| `clip_full` | higher | the same, against the whole original photo, so a crop pays for what it dropped | +| `psnr`, `ms_ssim` | higher | pixel fidelity; included for reference, but weak signals for this codec | + +`1-pkt` in the console summary is the share of images that fit in a single data +chunk. Each image also gets one parity packet when recovery is on. + +## Caveats + +- Desktop ORT on CPU is not bit-identical to ORT on a phone, so an occasional rounding difference can change a byte count by one. The byte counter itself matches the reference coder exactly (see `--selftest`). +- The `stretch` resize approximates `dart:ui`'s `FilterQuality.medium`; it is not pixel-identical. +- CLIP and LPIPS are proxies. Look at the images flagged in `report.html` before drawing conclusions. diff --git a/tools/aeic/eval/aeic_bytes.dart b/tools/aeic/eval/aeic_bytes.dart new file mode 100644 index 00000000..be3423c9 --- /dev/null +++ b/tools/aeic/eval/aeic_bytes.dart @@ -0,0 +1,240 @@ +// Byte counter for the AEIC evaluation harness (`eval.py`). +// +// Python runs the encode-side ONNX graph; this turns its tensors into the exact +// bitstream the app would send, using the app's own entropy layer and rANS +// coder rather than a port. A port could drift; this cannot, because it is the +// shipping code. +// +// Usage (from the repository root): +// +// dart run tools/aeic/eval/aeic_bytes.dart ... +// dart run tools/aeic/eval/aeic_bytes.dart --selftest ... +// +// Each `.f32` file holds little-endian float32 tensors back to back, in this +// order: z_q [1,128,4,4], yq0..yq3 [1,256,16,16], sc0..sc3 [1,256,16,16]. The +// bitstream is written next to it as `.aeic`, and one JSON object per input is +// printed to stdout. +// +// `--selftest` replays the golden `.aeicrec` recordings in +// `test/services/golden/e2e/` and checks that the output is byte-identical to +// the reference C++ bitstream, which shows the harness counts what the app +// sends. + +import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:meshcore_open/services/entropy_tables.dart'; +import 'package:meshcore_open/services/image_chunk_transport.dart'; +import 'package:meshcore_open/services/image_codec_entropy.dart'; +import 'package:meshcore_open/services/rans_coder.dart'; + +const int _resolution = 512; + +Future main(List args) async { + if (args.isEmpty) { + stderr.writeln( + 'usage: aeic_bytes.dart ...\n' + ' aeic_bytes.dart --selftest ...', + ); + exitCode = 64; + return; + } + if (args.first == '--selftest') { + exitCode = await _selfTest(args.skip(1).toList()); + return; + } + + final tables = EntropyTables.parse(File(args.first).readAsBytesSync()); + final geometry = AeicEntropyGeometry.forResolution(_resolution); + for (final path in args.skip(1)) { + final tensors = _readTensorFile(path, geometry); + final stream = await _encode(tables, geometry, tensors); + final outPath = path.replaceFirst(RegExp(r'\.f32$'), '.aeic'); + File(outPath).writeAsBytesSync(stream); + stdout.writeln( + jsonEncode({ + 'file': path, + 'bitstream': outPath, + 'bytes': stream.length, + 'data_chunks': imageDataChunkCount(stream.length), + }), + ); + } +} + +Future _encode( + EntropyTables tables, + AeicEntropyGeometry geometry, + AeicEncodeSideTensors tensors, +) { + final codec = AeicEntropyCodec( + geometry: geometry, + network: _FixedNetwork(tensors), + coders: _Coders(tables), + ); + // The network ignores its input; encode() only needs a correctly sized RGB + // buffer to get past its own conversion step. + return codec.encode(Uint8List(_resolution * _resolution * 3)); +} + +AeicEncodeSideTensors _readTensorFile( + String path, + AeicEntropyGeometry geometry, +) { + final bytes = File(path).readAsBytesSync(); + final z = geometry.zElements; + final y = geometry.yElements; + final expected = (z + 8 * y) * 4; + if (bytes.length != expected) { + throw FormatException( + '$path is ${bytes.length} bytes, expected $expected ' + '(z_q + 4 yq + 4 sc as float32)', + ); + } + final all = Float32List.sublistView(bytes); + Float32List at(int offset, int length) => + Float32List.sublistView(all, offset, offset + length); + return AeicEncodeSideTensors( + zQ: at(0, z), + yQ: [for (var i = 0; i < 4; i++) at(z + i * y, y)], + scales: [for (var i = 0; i < 4; i++) at(z + (4 + i) * y, y)], + ); +} + +Future _selfTest(List recordings) async { + if (recordings.isEmpty) { + stderr.writeln('--selftest needs at least one .aeicrec file'); + return 64; + } + final geometry = AeicEntropyGeometry.forResolution(_resolution); + var failures = 0; + for (final path in recordings) { + final r = _Recording.load(path); + final tables = EntropyTables.parse( + File( + '${File(path).parent.parent.path}/${r.meta['cdf_table']}', + ).readAsBytesSync(), + ); + final tensors = AeicEncodeSideTensors( + zQ: r.f32('enc/z_q'), + yQ: [for (var i = 0; i < 4; i++) r.f32('enc/yq$i')], + scales: [for (var i = 0; i < 4; i++) r.f32('enc/sc$i')], + ); + final got = await _encode(tables, geometry, tensors); + final want = r.u8('enc/bitstream'); + final ok = + got.length == want.length && + Iterable.generate(got.length).every((i) => got[i] == want[i]); + if (!ok) failures++; + stdout.writeln( + '${ok ? 'ok ' : 'FAIL'} ${r.meta['stem']}: ${got.length} B ' + '(reference ${want.length} B)', + ); + } + return failures == 0 ? 0 : 1; +} + +class _FixedNetwork implements AeicEntropyNetwork { + _FixedNetwork(this.tensors); + + final AeicEncodeSideTensors tensors; + + @override + bool get supportsDecodeSide => false; + + @override + Future runEncodeSide(Float32List imageChw) async => + tensors; + + @override + Future runHyperSynthesis(Float32List zQ) => + throw UnsupportedError('encode only'); + + @override + Future runStage(int stage, Float32List base) => + throw UnsupportedError('encode only'); +} + +class _Coders implements AeicRansCoderFactory { + _Coders(this.tables); + + final EntropyTables tables; + + @override + AeicRansEncoder createEncoder() => _Encoder(RansEncoder(tables)); + + @override + AeicRansDecoder createDecoder(Uint8List bitstream) => + throw UnsupportedError('encode only'); +} + +class _Encoder implements AeicRansEncoder { + _Encoder(this._encoder); + + final RansEncoder _encoder; + + @override + void pushSymbols(Int16List symbols, Int16List indexes, int cdfGroup) => + _encoder.encodeWithIndexes(symbols, indexes, cdfGroup); + + @override + Uint8List finish() => _encoder.finish(); +} + +/// Minimal reader for the `.aeicrec` container (see +/// `test/services/image_codec_e2e_test.dart` for the full one). +class _Recording { + _Recording(this.index, this.bytes) + : _entries = >{ + for (final e + in (index['entries'] as List).cast>()) + e['name'] as String: e, + }; + + final Map index; + final Uint8List bytes; + final Map> _entries; + + static _Recording load(String path) { + final bytes = File(path).readAsBytesSync(); + final bd = ByteData.sublistView(bytes); + if (ascii.decode(bytes.sublist(0, 8)) != 'AEICREC1') { + throw FormatException('bad .aeicrec magic in $path'); + } + final indexOffset = bd.getUint64(16, Endian.little); + final indexLength = bd.getUint32(24, Endian.little); + final index = + jsonDecode( + utf8.decode( + bytes.sublist(indexOffset, indexOffset + indexLength), + ), + ) + as Map; + return _Recording(index, bytes); + } + + Map get meta => index['meta'] as Map; + + Map _entry(String name) { + final e = _entries[name]; + if (e == null) throw StateError('no entry "$name" in recording'); + return e; + } + + Float32List f32(String name) { + final e = _entry(name); + final offset = e['offset'] as int; + return Float32List.sublistView( + bytes, + offset, + offset + (e['length'] as int), + ); + } + + Uint8List u8(String name) { + final e = _entry(name); + final offset = e['offset'] as int; + return Uint8List.sublistView(bytes, offset, offset + (e['length'] as int)); + } +} diff --git a/tools/aeic/eval/compare_codecs.py b/tools/aeic/eval/compare_codecs.py new file mode 100644 index 00000000..8271c923 --- /dev/null +++ b/tools/aeic/eval/compare_codecs.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +"""Rate-distortion comparison: AEIC (as shipped and with step rate control) +against JPEG XL and AVIF at the same byte budgets. + + python -I tools/aeic/eval/compare_codecs.py --images ~/aeic-data/kodak --out ~/aeic-runs/rd + +AEIC is swept over every quantisation step in STEP_LADDER (step 1.0 is the +stream the app sends today) and also fitted per image to 1 and 2 packets. +JPEG XL and AVIF are fitted per image to each byte budget: for every candidate +size the quality setting is searched to fit, the result is decoded and +upscaled to the display size, and the size with the lowest LPIPS is kept. That +choice uses the original, which the sender has, so it is a fair best case for +the conventional codecs rather than an oracle. + +AVIF is counted twice. `avif` is the real .avif file, whose ftyp+meta boxes +cost about 275 bytes before any image data. `avif_av1` counts only the AV1 +payload (the mdat contents), as if the receiver rebuilt the fixed container +from a template; that is AVIF's best case for a custom wire format. + +Everything is scored against the source at the same display size (long side +512, the AEIC output size). Needs `imagecodecs` with JPEG XL support on top of +the eval.py requirements. +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import struct +import sys +from pathlib import Path + +import numpy as np +from PIL import Image + +HERE = Path(__file__).resolve().parent +_spec = importlib.util.spec_from_file_location("aeic_eval", HERE / "eval.py") +ev = importlib.util.module_from_spec(_spec) +sys.modules["aeic_eval"] = ev # dataclasses look their module up here +_spec.loader.exec_module(ev) # type: ignore[union-attr] + +import imagecodecs # noqa: E402 (after eval.py, which loads torch first) +from concurrent.futures import ThreadPoolExecutor # noqa: E402 + +POOL = ThreadPoolExecutor(max_workers=12) + +PACKET_BUDGETS = {"1 packet": ev.CHUNK_FIRST_CAPACITY, + "2 packets": ev.CHUNK_FIRST_CAPACITY + ev.CHUNK_CAPACITY} +BUDGETS = [100, 157, 200, 250, 315, 400, 500, 700] +SIZES = [16, 24, 32, 40, 48, 56, 64, 80, 96, 128, 160, 192] +JXL_EFFORT = 9 +AVIF_SPEED = 2 +DISPLAY = ev.SIZE + + +def display_size(img: Image.Image, long_side: int) -> tuple[int, int]: + s = long_side / max(img.size) + return (max(1, round(img.width * s)), max(1, round(img.height * s))) + + +def mdat_payload(avif: bytes) -> int: + """Size of the AV1 data in an .avif file (its mdat box contents).""" + i = 0 + while i + 8 <= len(avif): + size, kind = struct.unpack(">I4s", avif[i:i + 8]) + header = 8 + if size == 1: + size = struct.unpack(">Q", avif[i + 8:i + 16])[0] + header = 16 + elif size == 0: + size = len(avif) - i + if kind == b"mdat": + return size - header + i += size + raise ValueError("no mdat box") + + +def _jxl(arr: np.ndarray, q: float) -> bytes: + return imagecodecs.jpegxl_encode(arr, distance=q, effort=JXL_EFFORT) + + +def _avif(arr: np.ndarray, q: float) -> bytes: + return imagecodecs.avif_encode(arr, level=int(q), speed=AVIF_SPEED) + + +# name -> (encode(arr, q), decode, bytes counted, q for the smallest file, +# q for the largest, integer q?) +CODECS = { + "jxl": (_jxl, imagecodecs.jpegxl_decode, len, 25.0, 0.05, False), + "avif": (_avif, imagecodecs.avif_decode, len, 0, 100, True), + "avif_av1": (_avif, imagecodecs.avif_decode, mdat_payload, 0, 100, True), +} + + +def codec_fit(name: str, src: Image.Image, long_side: int, budget: int) -> tuple[int, Image.Image] | None: + """Best stream of codec `name` for `src` at `long_side` px within `budget`. + + Returns (counted bytes, decoded image), or None if even the lowest quality + does not fit. + """ + enc, dec, count, q_small, q_large, integer = CODECS[name] + small = np.asarray(src.resize(display_size(src, long_side), Image.LANCZOS)) + data = enc(small, q_small) + if count(data) > budget: + return None + best = data + lo, hi = q_small, q_large # lo always fits + for _ in range(12): + if integer: + if abs(hi - lo) <= 1: + break + mid = (lo + hi) // 2 + else: + mid = (lo * hi) ** 0.5 # JPEG XL distance acts roughly logarithmically + data = enc(small, mid) + if count(data) <= budget: + best, lo = data, mid + else: + hi = mid + decoded = Image.fromarray(np.asarray(dec(best))[..., :3], "RGB") + return count(best), decoded + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--images", type=Path, required=True) + ap.add_argument("--out", type=Path, required=True) + ap.add_argument("--models", type=Path, default=ev.DEFAULT_MODELS) + ap.add_argument("--limit", type=int) + args = ap.parse_args() + + images = ev.find_images(args.images, args.limit) + out: Path = args.out + work = out / "tensors" + (out / "img").mkdir(parents=True, exist_ok=True) + work.mkdir(parents=True, exist_ok=True) + + import torch + gpu = torch.cuda.is_available() + codec = ev.Codec(args.models, args.models / ev.DECODER_GRAPH, gpu) + metrics = ev.Metrics("cuda" if gpu else "cpu") + + # ---- AEIC: every step on the ladder, one Dart run for all byte counts. + srcs, aeic_y = {}, {} + files = [] + for path in images: + stem = path.stem + src = ev.load_source(path) + srcs[stem] = src + side = codec.encode_side(ev.variant_stretch(src).square) + for step in ev.STEP_LADDER: + t = codec.encode_with_step(side, step) + tf = work / f"{stem}.s{step:.2f}.f32" + ev.write_tensor_file(tf, t) + aeic_y[(stem, step)] = t["y_hat"] + files.append(tf) + print(f"encoded {stem}", file=sys.stderr) + nbytes = {k: v["bytes"] for k, v in ev.count_bytes(args.models / ev.CDF_TABLE, files).items()} + for tf in files: + tf.unlink() + + rows = [] + emb = {} + + def score(stem: str, shown: Image.Image) -> dict: + src = srcs[stem] + ref = src.resize(display_size(src, DISPLAY), Image.LANCZOS) + if stem not in emb: + emb[stem] = metrics.clip_embed(src) + s = metrics.score(ref, shown.resize(ref.size, Image.BICUBIC), emb[stem]) + s.pop("clip_full") + return s + + aeic_shown = {} + for path in images: + stem = path.stem + src = srcs[stem] + for step in ev.STEP_LADDER: + recon = codec.decode(aeic_y[(stem, step)]) + shown = recon.resize(display_size(src, DISPLAY), Image.LANCZOS) + aeic_shown[(stem, step)] = shown + b = nbytes[str(work / f"{stem}.s{step:.2f}.f32")] + rows.append({"image": stem, "codec": "aeic", "step": step, "bytes": b, **score(stem, shown)}) + print(f"scored AEIC {stem}", file=sys.stderr) + + # ---- Conventional codecs: per image and budget, the best size that fits. + for name in CODECS: + for path in images: + stem = path.stem + src = srcs[stem] + for budget in BUDGETS: + best = None + # imagecodecs releases the GIL, so the size search runs in threads. + fits = list(POOL.map(lambda ls: codec_fit(name, src, ls, budget), SIZES)) + for long_side, fit in zip(SIZES, fits): + if fit is None: + continue + nb, small = fit + shown = small.resize(display_size(src, DISPLAY), Image.BICUBIC) + ref = src.resize(shown.size, Image.LANCZOS) + lp = float(metrics.lpips(metrics._t(shown), metrics._t(ref))) + if best is None or lp < best[0]: + best = (lp, long_side, nb, shown) + if best is None: + continue + _, long_side, nb, shown = best + rows.append({"image": stem, "codec": name, "budget": budget, "px": long_side, + "bytes": nb, **score(stem, shown)}) + if budget in PACKET_BUDGETS.values(): + shown.save(out / "img" / f"{stem}.{name}{budget}.png") + print(f"scored {name} {stem}", file=sys.stderr) + + # ---- AEIC fitted to the packet budgets, picked from the sweep. + for stem in srcs: + for label, budget in PACKET_BUDGETS.items(): + cands = [r for r in rows if r["codec"] == "aeic" and r["image"] == stem and r["bytes"] <= budget] + if not cands: + cands = [max((r for r in rows if r["codec"] == "aeic" and r["image"] == stem), + key=lambda r: r["step"])] + pick = min(cands, key=lambda r: r["step"]) + rows.append({**pick, "codec": f"aeic_fit{budget}", "budget": budget}) + aeic_shown[(stem, pick["step"])].save(out / "img" / f"{stem}.aeic{budget}.png") + aeic_shown[(stem, 1.0)].save(out / "img" / f"{stem}.aeic_today.png") + srcs[stem].resize(display_size(srcs[stem], DISPLAY), Image.LANCZOS).save(out / "img" / f"{stem}.src.png") + + (out / "rows.json").write_text(json.dumps(rows, indent=0)) + + def mean(rs, k): + return float(np.mean([r[k] for r in rs])) + + def line(name, rs): + return (f"{name:<22}{mean(rs, 'bytes'):>7.0f}{mean(rs, 'lpips'):>8.3f}{mean(rs, 'dists'):>8.3f}" + f"{mean(rs, 'clip'):>7.3f}{mean(rs, 'ms_ssim'):>8.3f}{mean(rs, 'psnr'):>7.2f}") + + print(f"\n{'':<22}{'bytes':>7}{'lpips':>8}{'dists':>8}{'clip':>7}{'msssim':>8}{'psnr':>7}") + print(line("AEIC today (step 1)", [r for r in rows if r["codec"] == "aeic" and r["step"] == 1.0])) + for label, budget in PACKET_BUDGETS.items(): + print(f"-- {label} ({budget} B)") + print(line(" AEIC step-fit", [r for r in rows if r["codec"] == f"aeic_fit{budget}"])) + for name in CODECS: + rs = [r for r in rows if r["codec"] == name and r["budget"] == budget] + print(line(f" {name} ({len(rs)} img)", rs) if rs else f" {name}: does not fit") + print("\nAEIC curve") + for step in ev.STEP_LADDER: + print(line(f" step {step:.2f}", [r for r in rows if r["codec"] == "aeic" and r["step"] == step])) + for name in CODECS: + print(f"{name} curve") + for budget in BUDGETS: + rs = [r for r in rows if r["codec"] == name and r["budget"] == budget] + if rs: + print(line(f" <= {budget} B ({len(rs)} img)", rs)) + + +if __name__ == "__main__": + main() diff --git a/tools/aeic/eval/eval.py b/tools/aeic/eval/eval.py new file mode 100644 index 00000000..e5dd446c --- /dev/null +++ b/tools/aeic/eval/eval.py @@ -0,0 +1,831 @@ +#!/usr/bin/env python3 +"""Quality/rate evaluation harness for the AEIC image codec. + +Runs a folder of photos through the same three ONNX graphs the app ships, +counts the exact bytes with the app's own Dart rANS coder (aeic_bytes.dart), +and scores what the receiver would see. + + python -I tools/aeic/eval/eval.py --images ~/photos --out /tmp/aeic-eval + +See README.md in this directory for setup and how to read the output. +""" + +from __future__ import annotations + +import argparse +import csv +import datetime as dt +import hashlib +import html +import json +import math +import os +import shutil +import subprocess +import sys +import time +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Callable + +import numpy as np +from PIL import Image, ImageOps + +# torch must be imported before onnxruntime so the CUDA libraries that ship in +# the torch wheels are already loaded when ORT's CUDA provider looks for them. +import torch # noqa: E402 (import order is deliberate) +import onnxruntime as ort # noqa: E402 + +SIZE = 512 +REPO = Path(__file__).resolve().parents[3] +DART_TOOL = Path(__file__).resolve().parent / "aeic_bytes.dart" +DEFAULT_MODELS = Path.home() / "Documents" / "image_codec_models" +SIDE_GRAPH = "aeic_entropy_side_fp32_op17.onnx" +DECODER_GRAPH = "aeic_decoder_qdq_conv_pct.onnx" +CDF_TABLE = "aeic_cdf_ft32.bin" +CONTEXT_GRAPH = "aeic_entropy_decode_fp32_op17.onnx" + + +def _stage_masks() -> list[np.ndarray]: + """The four checkerboard masks, as AeicMaskSet.isLive defines them.""" + perm = [0, 3, 2, 1] + c = np.arange(256)[:, None, None] // 64 + yy = np.arange(16)[None, :, None] + xx = np.arange(16)[None, None, :] + micro = ((yy & 1) << 1) | (xx & 1) + return [((c ^ perm[s]) == micro)[None] for s in range(4)] + + +STAGE_MASKS = _stage_masks() + +# Quantisation steps a stream could signal in 4 bits (the metadata byte's rate +# and resolution fields). 1.0 is today's ft32 stream. +STEP_LADDER = [0.40, 0.45, 0.50, 0.55, 0.60, 0.66, 0.72, 0.79, + 0.86, 0.93, 1.00, 1.10, 1.20, 1.32, 1.45, 1.60] +IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".webp", ".bmp", ".tif", ".tiff"} + +# Mirrors lib/services/image_chunk_transport.dart. aeic_bytes.dart reports the +# authoritative count; these are only used for the summary's packet totals. +PARITY_CHUNKS = 1 + + +# -------------------------------------------------------------------------- +# Preprocessing variants +# -------------------------------------------------------------------------- +# +# A variant turns the (EXIF-rotated) source photo into the 512x512 codec input +# and says which part of the source it represents, so the reconstruction can +# be compared with that region at its true aspect ratio, which is what the +# receiver displays. + + +@dataclass +class Prepared: + square: Image.Image # 512x512 RGB, what the encoder sees + region: tuple[int, int, int, int] # source box (l, t, r, b) it represents + content_box: tuple[int, int, int, int] # box inside `square` holding it + # For multi-part variants: width in source pixels of the cross-fade along + # this part's edges that lie inside the picture (not on its border). + feather: int = 0 + + +def _resize(img: Image.Image, size: tuple[int, int]) -> Image.Image: + # The app draws with dart:ui FilterQuality.medium: bilinear sampling over + # mipmaps. A box pre-reduce followed by bilinear is the closest PIL match. + return img.resize(size, Image.BILINEAR, reducing_gap=2.0) + + +def _full(img: Image.Image) -> tuple[int, int, int, int]: + return (0, 0, img.width, img.height) + + +def variant_stretch(img: Image.Image) -> Prepared: + """What the app does today: the whole frame squeezed into the square.""" + return Prepared(_resize(img, (SIZE, SIZE)), _full(img), (0, 0, SIZE, SIZE)) + + +def _centre_crop_box(img: Image.Image, max_ratio: float) -> tuple[int, int, int, int]: + w, h = img.size + ratio = w / h + if ratio > max_ratio: + cw = round(h * max_ratio) + left = (w - cw) // 2 + return (left, 0, left + cw, h) + if ratio < 1 / max_ratio: + ch = round(w * max_ratio) + top = (h - ch) // 2 + return (0, top, w, top + ch) + return _full(img) + + +def variant_crop(img: Image.Image) -> Prepared: + """Centre square crop: no distortion, loses the edges of the frame.""" + box = _centre_crop_box(img, 1.0) + return Prepared(_resize(img.crop(box), (SIZE, SIZE)), box, (0, 0, SIZE, SIZE)) + + +def variant_crop43(img: Image.Image) -> Prepared: + """Crop to at most 4:3 (or 3:4), then stretch what is left.""" + box = _centre_crop_box(img, 4 / 3) + return Prepared(_resize(img.crop(box), (SIZE, SIZE)), box, (0, 0, SIZE, SIZE)) + + +def variant_pad(img: Image.Image) -> Prepared: + """Letterbox on mid-grey: no distortion, spends latent area on the bars.""" + w, h = img.size + scale = SIZE / max(w, h) + nw, nh = max(1, round(w * scale)), max(1, round(h * scale)) + canvas = Image.new("RGB", (SIZE, SIZE), (128, 128, 128)) + left, top = (SIZE - nw) // 2, (SIZE - nh) // 2 + canvas.paste(_resize(img, (nw, nh)), (left, top)) + return Prepared(canvas, _full(img), (left, top, left + nw, top + nh)) + + +# Re-roll tweaks: (zoom, dx, dy) as fractions of the source. Mirrors +# kImageSendRerollTweaks / rerollSourceRect in +# lib/widgets/image_send_codec_binding.dart; keep the two in step. +REROLL_TWEAKS = [ + (1.0, 0.0, 0.0), + (1.02, 0.005, 0.0), + (1.02, -0.005, 0.005), + (1.04, 0.0, -0.005), + (1.03, 0.01, 0.01), + (1.03, -0.01, -0.01), +] + + +def reroll_source_rect(w: int, h: int, variant: int) -> tuple[float, float, float, float]: + """Source rect for re-roll `variant`: a near-invisible zoom and shift.""" + zoom, dx, dy = REROLL_TWEAKS[variant] + cw, ch = w / zoom, h / zoom + left = min(max((w - cw) / 2 + dx * w, 0.0), w - cw) + top = min(max((h - ch) / 2 + dy * h, 0.0), h - ch) + return (left, top, left + cw, top + ch) + + +def variant_reroll(img: Image.Image, variant: int) -> Prepared: + """Whole photo with re-roll tweak `variant`, stretched like `stretch`. + + Scored against the full frame: the tweak only drops a percent or two at + the edges, which is what the sender sees as "the whole photo". + """ + l, t, r, b = reroll_source_rect(img.width, img.height, variant) + # PIL's resize box takes float source coordinates, like drawImageRect. + square = img.resize((SIZE, SIZE), Image.BILINEAR, box=(l, t, r, b), reducing_gap=2.0) + return Prepared(square, _full(img), (0, 0, SIZE, SIZE)) + + +VARIANTS: dict[str, Callable[[Image.Image], Prepared]] = { + "stretch": variant_stretch, + "crop": variant_crop, + "crop43": variant_crop43, + "pad": variant_pad, + **{f"reroll{k}": (lambda img, k=k: variant_reroll(img, k)) for k in range(1, len(REROLL_TWEAKS))}, +} + +# Need a box per image (--rois), so they are not in VARIANTS. +SELECT = "select" +SELECT_SQUARE = "select_sq" +SELECT_VARIANTS = (SELECT, SELECT_SQUARE) + +Box = tuple[int, int, int, int] + + +def roi_box(img: Image.Image, frac: list[float]) -> Box: + """Fractional [l, t, r, b] -> pixel box in the (EXIF-rotated) source.""" + l, t, r, b = frac + w, h = img.size + return (round(l * w), round(t * h), round(r * w), round(b * h)) + + +def variant_select(img: Image.Image, box: Box) -> Prepared: + """The user's selection, stretched into the square like `stretch`. + + The aspect nibble restores the box's shape on the receiver, so this is the + app's existing pipeline applied to a crop the user chose. + """ + return Prepared(_resize(img.crop(box), (SIZE, SIZE)), box, (0, 0, SIZE, SIZE)) + + +def square_box(img: Image.Image, box: Box) -> Box: + """Grow `box` to a square around its centre, shifted to stay in the image. + + Models a selection tool locked to 1:1, so the codec never sees a stretch. + Capped at the image's short side. + """ + l, t, r, b = box + side = min(max(r - l, b - t), img.width, img.height) + cx, cy = (l + r) / 2, (t + b) / 2 + nl = int(round(min(max(cx - side / 2, 0), img.width - side))) + nt = int(round(min(max(cy - side / 2, 0), img.height - side))) + return (nl, nt, nl + side, nt + side) + + +# -------------------------------------------------------------------------- +# HD: two encoded parts per photo, aimed at a two-packet budget +# -------------------------------------------------------------------------- +# +# The model is fixed at 512x512 / ft32, so "HD" spends the second packet on a +# second 512x512 encode. The parts travel as one stream; the receiver decodes +# both and blends them. Overheads below are what the stream would need on top +# of the bitstreams (a hypothetical layout, not implemented in the app). + +HD_TILES = "hd_tiles" +HD_DETAIL = "hd_detail" +HD_VARIANTS = (HD_TILES, HD_DETAIL) +HD_OVERHEAD = { + HD_TILES: 1, # length of part 0, so the receiver can split the stream + HD_DETAIL: 1 + 4, # that, plus the detail box quantised to 4 bytes +} +TILE_OVERLAP = 0.08 # of the long side, cross-faded at the seam + + +def variant_hd_tiles(img: Image.Image) -> list[Prepared]: + """Two overlapping halves along the long side, each encoded at 512x512. + + Doubles the pixel count. A 3:2 photo splits into two near-square halves, + so each half is also stretched far less than the whole frame would be. + """ + w, h = img.size + long_side = max(w, h) + ov = round(long_side * TILE_OVERLAP) + half = (long_side + ov) // 2 + if w >= h: + boxes = [(0, 0, half, h), (w - half, 0, w, h)] + else: + boxes = [(0, 0, w, half), (0, h - half, w, h)] + return [Prepared(_resize(img.crop(b), (SIZE, SIZE)), b, (0, 0, SIZE, SIZE), feather=ov) + for b in boxes] + + +def variant_hd_detail(img: Image.Image, box: Box) -> list[Prepared]: + """The whole photo, plus the selected subject blended in at higher detail.""" + detail = variant_select(img, box) + detail.feather = max(4, round(0.08 * min(box[2] - box[0], box[3] - box[1]))) + return [variant_stretch(img), detail] + + +def composite(src: Image.Image, parts: list[Prepared], recons: list[Image.Image], + long_side: int) -> tuple[Image.Image, Image.Image, Box]: + """Reference and receiver view for a variant made of one or more parts. + + The view covers the union of the parts' regions at `long_side` pixels on + its long edge. Parts are laid down in order; each later part fades in over + `feather` pixels along its edges that are inside the union, so seams + cross-fade instead of cutting hard. + """ + region = (min(p.region[0] for p in parts), min(p.region[1] for p in parts), + max(p.region[2] for p in parts), max(p.region[3] for p in parts)) + rw, rh = region[2] - region[0], region[3] - region[1] + s = long_side / max(rw, rh) + size = (max(1, round(rw * s)), max(1, round(rh * s))) + ref = src.crop(region).resize(size, Image.LANCZOS) + canvas = np.zeros((size[1], size[0], 3), dtype=np.float64) + for i, (p, recon) in enumerate(zip(parts, recons)): + l, t, r, b = p.region + x0, y0 = round((l - region[0]) * s), round((t - region[1]) * s) + x1, y1 = round((r - region[0]) * s), round((b - region[1]) * s) + tile = np.asarray(recon.crop(p.content_box).resize((x1 - x0, y1 - y0), Image.LANCZOS), + dtype=np.float64) + alpha = np.ones((y1 - y0, x1 - x0), dtype=np.float64) + f = min(round(p.feather * s), alpha.shape[0], alpha.shape[1]) + if i > 0 and f > 0: + ramp = np.arange(1, f + 1, dtype=np.float64) / f + if l > region[0]: + alpha[:, :f] *= ramp[None, :] + if r < region[2]: + alpha[:, -f:] *= ramp[::-1][None, :] + if t > region[1]: + alpha[:f, :] *= ramp[:, None] + if b < region[3]: + alpha[-f:, :] *= ramp[::-1][:, None] + a = alpha[..., None] + canvas[y0:y1, x0:x1] = canvas[y0:y1, x0:x1] * (1 - a) + tile * a + shown = Image.fromarray(np.clip(np.rint(canvas), 0, 255).astype(np.uint8), "RGB") + return ref, shown, region + + +# Mirrors imageDataChunkCount() in image_chunk_transport.dart. +CHUNK_FIRST_CAPACITY = 157 +CHUNK_CAPACITY = 158 + + +def parse_rate_variant(v: str) -> dict | None: + """`q0.7` -> fixed step 0.7; `fit315` -> per-image step to fit 315 bytes.""" + try: + if v.startswith("q"): + return {"steps": [float(v[1:])], "budget": None} + if v.startswith("fit"): + return {"steps": STEP_LADDER, "budget": int(v[3:])} + except ValueError: + return None + return None + + +def data_chunks(payload_bytes: int) -> int: + n = max(payload_bytes, 0) + if n <= CHUNK_FIRST_CAPACITY: + return 1 + return 1 + math.ceil((n - CHUNK_FIRST_CAPACITY) / CHUNK_CAPACITY) + + +# -------------------------------------------------------------------------- +# Codec +# -------------------------------------------------------------------------- + + +class Codec: + def __init__(self, models: Path, decoder: Path, gpu: bool) -> None: + cpu = ["CPUExecutionProvider"] + cuda = ["CUDAExecutionProvider", "CPUExecutionProvider"] if gpu else cpu + # The entropy graph stays on CPU: its rounded outputs decide the bytes, + # and CPU fp32 is the closest stand-in for the phone. It costs ~0.1 s. + self.side = ort.InferenceSession(str(models / SIDE_GRAPH), providers=cpu) + self.context = ort.InferenceSession(str(models / CONTEXT_GRAPH), providers=cpu) + self.decoder = ort.InferenceSession(str(decoder), providers=cuda) + self.decoder_provider = self.decoder.get_providers()[0] + + def encode_side(self, square: Image.Image) -> dict[str, np.ndarray]: + rgb = np.asarray(square, dtype=np.uint8) + # Same float32 steps as aeicRgbToChw in image_codec_entropy.dart. + chw = rgb.transpose(2, 0, 1).astype(np.float32) / np.float32(255.0) + chw = (chw - np.float32(0.5)) * np.float32(2.0) + names = (["z_q", "y_hat", "y", "base0"] + + [f"yq{i}" for i in range(4)] + [f"sc{i}" for i in range(4)]) + out = self.side.run(names, {"image": chw[None]}) + return dict(zip(names, out)) + + def encode_with_step(self, side: dict[str, np.ndarray], step: float) -> dict[str, np.ndarray]: + """Re-quantise y with step `step` instead of 1 (rate control). + + Replays the four-stage context loop the receiver runs + (AeicEntropyCodec.decodeToLatent) with the decode-side graph, coding + round((y - mean) / step) against scale / step. The receiver's + reconstruction becomes symbol * step + mean. step < 1 spends more + bytes, step > 1 fewer. At step == 1 this reproduces the side graph's + symbols, which the harness's q1 check relies on. + """ + st = np.float32(step) + y, z_q = side["y"], side["z_q"] + base = side["base0"].copy() + out: dict[str, np.ndarray] = {"z_q": z_q} + for stage in range(4): + means, scales = self.context.run( + ["means", "scales"], + {"z_q": z_q, "base": base, "stage": np.array([stage], dtype=np.int32)}) + m = STAGE_MASKS[stage] + sym = np.where(m, np.rint((y - means) / st), np.float32(0)).astype(np.float32) + base = np.where(m, sym * st + means, base).astype(np.float32) + out[f"yq{stage}"] = sym + out[f"sc{stage}"] = np.where(m, scales / st, np.float32(0)).astype(np.float32) + out["y_hat"] = base + return out + + def decode(self, y_hat: np.ndarray) -> Image.Image: + (out,) = self.decoder.run(["image"], {"y_hat": y_hat.astype(np.float32)}) + # Same mapping as _chwFloatsToRgb in image_codec_backend.dart: clamp, + # then round half away from zero (Dart's double.round()). + scaled = (out[0].astype(np.float64) + 1.0) * 127.5 + rgb = np.floor(np.clip(scaled, 0, 255) + 0.5).astype(np.uint8) + return Image.fromarray(rgb.transpose(1, 2, 0), "RGB") + + +def write_tensor_file(path: Path, t: dict[str, np.ndarray]) -> None: + parts = [t["z_q"]] + [t[f"yq{i}"] for i in range(4)] + [t[f"sc{i}"] for i in range(4)] + with open(path, "wb") as f: + for p in parts: + f.write(np.ascontiguousarray(p, dtype=" dict[str, dict]: + if not tensor_files: + return {} + dart = shutil.which("dart") + if dart is None: + sys.exit("dart is not on PATH; it is needed to run aeic_bytes.dart") + cmd = [dart, "run", str(DART_TOOL), str(cdf), *map(str, tensor_files)] + proc = subprocess.run(cmd, cwd=REPO, capture_output=True, text=True) + if proc.returncode != 0: + sys.exit(f"aeic_bytes.dart failed:\n{proc.stderr}") + results = {} + for line in proc.stdout.splitlines(): + line = line.strip() + if line.startswith("{"): + row = json.loads(line) + results[row["file"]] = row + return results + + +# -------------------------------------------------------------------------- +# Metrics +# -------------------------------------------------------------------------- + + +class Metrics: + """Full-reference scores. Higher is better except LPIPS and DISTS.""" + + def __init__(self, device: str) -> None: + import open_clip + import piq + + self.piq = piq + self.device = device + self.lpips = piq.LPIPS(reduction="none").to(device).eval() + self.dists = piq.DISTS(reduction="none").to(device).eval() + model, _, preprocess = open_clip.create_model_and_transforms( + "ViT-B-32", pretrained="laion2b_s34b_b79k" + ) + self.clip = model.to(device).eval() + self.clip_pre = preprocess + + def _t(self, img: Image.Image) -> torch.Tensor: + a = np.asarray(img, dtype=np.float32) / 255.0 + return torch.from_numpy(a).permute(2, 0, 1)[None].to(self.device) + + @torch.no_grad() + def clip_embed(self, img: Image.Image) -> torch.Tensor: + # CLIP's own preprocessing centre-crops to a square, which would hide + # the frame edges -- exactly what crop variants throw away. Letterbox + # first so the whole image is scored. + side = max(img.size) + boxed = Image.new("RGB", (side, side), (0, 0, 0)) + boxed.paste(img, ((side - img.width) // 2, (side - img.height) // 2)) + x = self.clip_pre(boxed)[None].to(self.device) + e = self.clip.encode_image(x) + return e / e.norm(dim=-1, keepdim=True) + + @torch.no_grad() + def score(self, ref: Image.Image, out: Image.Image, full_src_emb: torch.Tensor) -> dict: + r, o = self._t(ref), self._t(out) + out_emb = self.clip_embed(out) + return { + "psnr": float(self.piq.psnr(o, r, data_range=1.0)), + "ms_ssim": float(self.piq.multi_scale_ssim(o, r, data_range=1.0)), + "lpips": float(self.lpips(o, r)), + "dists": float(self.dists(o, r)), + # Semantic agreement with exactly what was sent ... + "clip": float((self.clip_embed(ref) * out_emb).sum()), + # ... and with the whole original photo, so cropping is charged + # for what it threw away. + "clip_full": float((full_src_emb * out_emb).sum()), + } + + +def roi_pair(src: Image.Image, region: Box, shown: Image.Image, box: Box) -> tuple[Image.Image, Image.Image] | None: + """The ROI as the receiver would see it after zooming in, plus its reference. + + `shown` covers `region` of the source; the ROI is mapped into it and + both crops are scaled to long side 512. For a full-frame variant that is an + upscale of a few dozen pixels, which is exactly what zooming in shows. + Returns None when the variant did not send the whole ROI. + """ + rl, rt, rr, rb = region + l, t, r, b = box + if l < rl or t < rt or r > rr or b > rb: + return None + sx = shown.width / (rr - rl) + sy = shown.height / (rb - rt) + out = shown.crop((round((l - rl) * sx), round((t - rt) * sy), + round((r - rl) * sx), round((b - rt) * sy))) + ref = src.crop(box) + scale = SIZE / max(ref.size) + size = (max(1, round(ref.width * scale)), max(1, round(ref.height * scale))) + return ref.resize(size, Image.LANCZOS), out.resize(size, Image.BICUBIC) + + +# -------------------------------------------------------------------------- +# Corpus +# -------------------------------------------------------------------------- + +KODAK_URL = "https://r0k.us/graphics/kodak/kodak/kodim{:02d}.png" + + +def download_kodak(dest: Path) -> None: + dest.mkdir(parents=True, exist_ok=True) + for i in range(1, 25): + target = dest / f"kodim{i:02d}.png" + if target.exists(): + continue + print(f"downloading {target.name}", file=sys.stderr) + tmp = target.with_suffix(".part") + with urllib.request.urlopen(KODAK_URL.format(i), timeout=60) as r, open(tmp, "wb") as f: + shutil.copyfileobj(r, f) + tmp.rename(target) + + +def load_source(path: Path) -> Image.Image: + img = Image.open(path) + # Picked photos are shown upright in the app, so honour EXIF orientation. + img = ImageOps.exif_transpose(img) + return img.convert("RGB") + + +def find_images(folder: Path, limit: int | None) -> list[Path]: + files = sorted(p for p in folder.rglob("*") if p.suffix.lower() in IMAGE_SUFFIXES) + return files[:limit] if limit else files + + +# -------------------------------------------------------------------------- +# Report +# -------------------------------------------------------------------------- + +METRIC_COLS = ["bytes", "data_chunks", "psnr", "ms_ssim", "lpips", "dists", "clip", "clip_full"] +ROI_COLS = ["roi_psnr", "roi_ms_ssim", "roi_lpips", "roi_dists", "roi_clip"] +LOWER_IS_BETTER = {"bytes", "data_chunks", "lpips", "dists", "roi_lpips", "roi_dists"} + + +def summarise(rows: list[dict], variants: list[str]) -> dict: + summary = {} + for v in variants: + vr = [r for r in rows if r["variant"] == v] + if not vr: + continue + s: dict = {"images": len(vr)} + for m in METRIC_COLS + ROI_COLS: + vals = np.array([r[m] for r in vr if r.get(m) not in (None, "")], dtype=np.float64) + if vals.size == 0: + continue + s[m] = {"mean": float(vals.mean()), "median": float(np.median(vals)), + "min": float(vals.min()), "max": float(vals.max())} + chunks = [r["data_chunks"] for r in vr] + s["chunk_histogram"] = {str(k): chunks.count(k) for k in sorted(set(chunks))} + s["packets_mean_with_parity"] = float(np.mean(chunks)) + PARITY_CHUNKS + # The share of images whose CLIP agreement with the source is low. + # This is the "fox came back as a sheep" bucket; worth eyeballing. + s["clip_below_0.80"] = sum(1 for r in vr if r["clip"] < 0.80) + summary[v] = s + return summary + + +def print_summary(summary: dict) -> None: + head = (f"{'variant':<10}" + "".join(f"{m:>12}" for m in METRIC_COLS) + + f"{'1-pkt':>8}{'<=2-pkt':>9}{'clip<.8':>9}") + print(head) + print("-" * len(head)) + for v, s in summary.items(): + line = f"{v:<10}" + "".join(f"{s[m]['mean']:>12.4g}" for m in METRIC_COLS) + one = s["chunk_histogram"].get("1", 0) + two = one + s["chunk_histogram"].get("2", 0) + line += f"{one / s['images']:>7.0%} {two / s['images']:>8.0%} {s['clip_below_0.80']:>8}" + print(line) + print("means; lower is better for bytes, data_chunks, lpips, dists") + if any("roi_lpips" in s for s in summary.values()): + print() + print(f"{'variant':<10}" + "".join(f"{m:>13}" for m in ROI_COLS)) + for v, s in summary.items(): + if "roi_lpips" in s: + print(f"{v:<10}" + "".join(f"{s[m]['mean']:>13.4g}" for m in ROI_COLS)) + print("scored on the selected region only, as the receiver sees it after zooming in") + + +def write_html(out: Path, rows: list[dict], summary: dict, meta: dict) -> None: + def fmt(v: float, m: str) -> str: + return f"{v:.1f}" if m in ("bytes", "data_chunks") else f"{v:.3f}" + + variants = list(summary) + stems = list(dict.fromkeys(r["image"] for r in rows)) + by = {(r["image"], r["variant"]): r for r in rows} + + sum_rows = "".join( + f"{html.escape(v)}" + + "".join(f"{fmt(summary[v][m]['mean'], m)}" for m in METRIC_COLS) + + "" + for v in variants + ) + cards = [] + for stem in stems: + cells = [] + first = by[(stem, variants[0])] + cells.append( + f'
' + f"
source
" + ) + for v in variants: + r = by.get((stem, v)) + if not r: + continue + warn = " warn" if r["clip"] < 0.80 else "" + cells.append( + f'
' + f"
{html.escape(v)} {r['bytes']} B / {r['data_chunks']} pkt
" + f"LPIPS {r['lpips']:.3f} · CLIP {r['clip']:.3f}
" + ) + cards.append(f"

{html.escape(stem)}

{''.join(cells)}
") + + doc = f""" + +AEIC eval + +

AEIC eval

+

{html.escape(json.dumps(meta))}

+

Means

+{''.join(f'' for m in METRIC_COLS)}{sum_rows}
{m}
+

Images are shown at the aspect ratio the receiver restores. A red border means CLIP similarity to the sent region is below 0.80, so check it by eye.

+{''.join(cards)} +""" + (out / "report.html").write_text(doc, encoding="utf-8") + + +# -------------------------------------------------------------------------- +# Main +# -------------------------------------------------------------------------- + + +def sha256(path: Path) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + for block in iter(lambda: f.read(1 << 20), b""): + h.update(block) + return h.hexdigest() + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--images", type=Path, help="folder of photos (searched recursively)") + ap.add_argument("--download-kodak", type=Path, metavar="DIR", + help="download the 24 Kodak images to DIR and use them") + ap.add_argument("--out", type=Path, required=True, help="output folder") + ap.add_argument("--models", type=Path, default=DEFAULT_MODELS, + help=f"model bundle folder (default {DEFAULT_MODELS})") + ap.add_argument("--decoder", type=Path, help="alternative decoder .onnx (e.g. an fp16 export)") + ap.add_argument("--variants", default="stretch", + help=f"comma list from: {', '.join(VARIANTS)} (default stretch)") + ap.add_argument("--limit", type=int, help="only the first N images") + ap.add_argument("--cpu", action="store_true", help="run the decoder on CPU too") + ap.add_argument("--rois", type=Path, + help="JSON {image stem: [l, t, r, b] fractions}; enables the select and " + "hd_detail variants and ROI scores, and limits the run to listed images") + ap.add_argument("--display", type=int, default=SIZE, + help="long side in pixels of the receiver view that is scored (default 512; " + "use 1024 when comparing HD variants)") + args = ap.parse_args() + + if args.download_kodak: + download_kodak(args.download_kodak) + args.images = args.images or args.download_kodak + if not args.images: + ap.error("give --images or --download-kodak") + variants = [v.strip() for v in args.variants.split(",") if v.strip()] + unknown = [v for v in variants + if v not in VARIANTS and v not in SELECT_VARIANTS and v not in HD_VARIANTS + and parse_rate_variant(v) is None] + if unknown: + ap.error(f"unknown variants: {unknown}") + rois: dict[str, list[float]] = {} + if args.rois: + rois = {k: v for k, v in json.loads(args.rois.read_text()).items() if not k.startswith("_")} + if any(v in SELECT_VARIANTS + (HD_DETAIL,) for v in variants) and not rois: + ap.error("the select and hd_detail variants need --rois") + + images = find_images(args.images, None) + if rois: + images = [p for p in images if p.relative_to(args.images).with_suffix("").as_posix() in rois] + images = images[: args.limit] if args.limit else images + if not images: + sys.exit(f"no images under {args.images}") + + out: Path = args.out + work = out / "tensors" + dirs = [work] + [out / k / v for k in ("recon", "ref", "roi") for v in variants] + for d in dirs: + d.mkdir(parents=True, exist_ok=True) + + decoder_path = args.decoder or (args.models / DECODER_GRAPH) + gpu = not args.cpu and torch.cuda.is_available() + codec = Codec(args.models, decoder_path, gpu) + metrics = Metrics("cuda" if gpu else "cpu") + print(f"{len(images)} images x {len(variants)} variants; decoder on {codec.decoder_provider}", + file=sys.stderr) + + # Pass 1: preprocess + encode side, so every byte count is one Dart run. + jobs = [] + for path in images: + src = load_source(path) + key = path.relative_to(args.images).with_suffix("").as_posix() + stem = key.replace("/", "__") + box = roi_box(src, rois[key]) if key in rois else None + for v in variants: + if (v in SELECT_VARIANTS or v == HD_DETAIL) and box is None: + continue + if v == SELECT: + parts = [variant_select(src, box)] + elif v == SELECT_SQUARE: + parts = [variant_select(src, square_box(src, box))] + elif v == HD_TILES: + parts = variant_hd_tiles(src) + elif v == HD_DETAIL: + parts = variant_hd_detail(src, box) + elif parse_rate_variant(v) is not None: + parts = [variant_stretch(src)] + else: + parts = [VARIANTS[v](src)] + rate = parse_rate_variant(v) + steps = rate["steps"] if rate else [None] + t0 = time.perf_counter() + encoded = [] # per part: [(step, y_hat, tensor file), ...] + for i, p in enumerate(parts): + side = codec.encode_side(p.square) + cands = [] + for step in steps: + tensors = side if step is None else codec.encode_with_step(side, step) + tag = "" if step is None else f".s{step:.2f}" + tf = work / f"{stem}.{v}.p{i}{tag}.f32" + write_tensor_file(tf, tensors) + cands.append((step, tensors["y_hat"], tf)) + encoded.append(cands) + enc_s = time.perf_counter() - t0 + jobs.append((path, stem, v, src, parts, box, encoded, enc_s)) + byte_rows = count_bytes(args.models / CDF_TABLE, + [tf for j in jobs for cands in j[6] for _, _, tf in cands]) + + # Pick one candidate per part. A fit variant takes the finest step whose + # stream fits N bytes, or the coarsest step if none does. + for j in jobs: + rate = parse_rate_variant(j[2]) + chosen = [] + for cands in j[6]: + if rate and rate["budget"] is not None: + fitting = [c for c in cands if byte_rows[str(c[2])]["bytes"] <= rate["budget"]] + chosen.append(min(fitting, key=lambda c: c[0]) if fitting + else max(cands, key=lambda c: c[0])) + else: + chosen.append(cands[0]) + j[6][:] = chosen + + # Pass 2: decode + score. + rows = [] + emb_cache: dict[str, torch.Tensor] = {} + for n, (path, stem, v, src, parts, box, encoded, enc_s) in enumerate(jobs, 1): + t0 = time.perf_counter() + recons = [codec.decode(y_hat) for _, y_hat, _ in encoded] + dec_s = time.perf_counter() - t0 + ref, shown, region = composite(src, parts, recons, args.display) + part_bytes = [byte_rows[str(tf)]["bytes"] for _, _, tf in encoded] + total_bytes = sum(part_bytes) + HD_OVERHEAD.get(v, 0) + chunks = data_chunks(total_bytes) + if len(parts) == 1: + assert chunks == byte_rows[str(encoded[0][2])]["data_chunks"], "chunk mirror drifted" + if stem not in emb_cache: + emb_cache[stem] = metrics.clip_embed(src) + scores = metrics.score(ref, shown, emb_cache[stem]) + roi = roi_pair(src, region, shown, box) if box else None + if roi: + roi_scores = metrics.score(roi[0], roi[1], emb_cache[stem]) + scores.update({f"roi_{k}": roi_scores[k] for k in ("psnr", "ms_ssim", "lpips", "dists", "clip")}) + roi[1].save(out / "roi" / v / f"{stem}.png") + roi[0].save(out / "roi" / v / f"{stem}.ref.png") + else: + scores.update({c: "" for c in ROI_COLS}) + out_png = Path("recon") / v / f"{stem}.png" + ref_png = Path("ref") / v / f"{stem}.png" + shown.save(out / out_png) + ref.save(out / ref_png) + row = { + "image": stem, "variant": v, "source": str(path), + "width": src.width, "height": src.height, + "bytes": total_bytes, "data_chunks": chunks, + "part_bytes": "+".join(map(str, part_bytes)), + "step": "+".join("1.00" if st is None else f"{st:.2f}" for st, _, _ in encoded), + **scores, + "encode_s": round(enc_s, 3), "decode_s": round(dec_s, 3), + "out_png": out_png.as_posix(), "ref_png": ref_png.as_posix(), + "bitstream": ";".join(os.path.relpath(byte_rows[str(tf)]["bitstream"], out) + for _, _, tf in encoded), + } + rows.append(row) + print(f"[{n}/{len(jobs)}] {stem} {v}: {row['bytes']} B, LPIPS {row['lpips']:.3f}, " + f"CLIP {row['clip']:.3f}, decode {dec_s:.2f}s", file=sys.stderr) + + summary = summarise(rows, variants) + meta = { + "date": dt.datetime.now().isoformat(timespec="seconds"), + "images": str(args.images), "count": len(images), "variants": variants, + "decoder": decoder_path.name, "decoder_sha256": sha256(decoder_path)[:16], + "decoder_provider": codec.decoder_provider, + "onnxruntime": ort.__version__, + } + with open(out / "results.csv", "w", newline="") as f: + w = csv.DictWriter(f, fieldnames=list(rows[0])) + w.writeheader() + w.writerows(rows) + (out / "summary.json").write_text(json.dumps({"meta": meta, "variants": summary}, indent=1)) + write_html(out, rows, summary, meta) + for tf in work.glob("*.f32"): # 2 MB each; the .aeic bitstreams are kept + tf.unlink() + print() + print_summary(summary) + print(f"\nreport: {out / 'report.html'}") + + +if __name__ == "__main__": + main() diff --git a/tools/aeic/eval/requirements.txt b/tools/aeic/eval/requirements.txt new file mode 100644 index 00000000..6abe164e --- /dev/null +++ b/tools/aeic/eval/requirements.txt @@ -0,0 +1,10 @@ +# For the CUDA build of torch, install it first from the PyTorch index, e.g. +# pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 +torch +torchvision +onnxruntime-gpu # or onnxruntime for CPU only +numpy +pillow +piq +open_clip_torch +imagecodecs # compare_codecs.py: JPEG XL and AVIF diff --git a/tools/aeic/eval/rois/kodak.json b/tools/aeic/eval/rois/kodak.json new file mode 100644 index 00000000..ce6727fc --- /dev/null +++ b/tools/aeic/eval/rois/kodak.json @@ -0,0 +1,20 @@ +{ + "_comment": "Subject boxes for the Kodak set, as fractions of width/height: [left, top, right, bottom]. Stand-ins for a user's 'select' crop.", + "kodim02": [0.15, 0.15, 0.70, 0.75], + "kodim03": [0.15, 0.05, 0.65, 0.65], + "kodim04": [0.20, 0.15, 0.80, 0.65], + "kodim05": [0.00, 0.00, 0.50, 1.00], + "kodim06": [0.15, 0.15, 0.80, 0.85], + "kodim09": [0.35, 0.10, 0.95, 0.75], + "kodim11": [0.35, 0.30, 0.85, 0.85], + "kodim12": [0.25, 0.20, 0.65, 0.85], + "kodim14": [0.00, 0.15, 0.90, 0.85], + "kodim15": [0.35, 0.00, 0.90, 0.75], + "kodim16": [0.40, 0.35, 0.90, 0.60], + "kodim17": [0.25, 0.00, 0.75, 0.45], + "kodim18": [0.25, 0.15, 0.85, 1.00], + "kodim19": [0.35, 0.00, 0.85, 0.75], + "kodim21": [0.30, 0.05, 0.70, 0.65], + "kodim22": [0.05, 0.10, 0.60, 0.70], + "kodim23": [0.50, 0.00, 1.00, 0.75] +} From 53e4e457ca8b6621097282a4cb2560d6a21dc163 Mon Sep 17 00:00:00 2001 From: zjs81 Date: Sat, 14 Nov 2026 20:58:43 -0700 Subject: [PATCH 2/2] Preview image versions before sending, and repair unheard chunks Send sheet: opens on what recipients will see (the encoded photo decoded on this device) instead of the original. "Try another version" encodes the whole photo again with a near-invisible zoom/shift; the generative decoder reconstructs each version differently, and on Kodak the best of three raised CLIP agreement from 0.828 to 0.853. The user picks the version to send; packet count and airtime follow the selection. Version 1 is byte-identical to the previous encode, so the wire format is unchanged. The sender's bubble shows the chosen version and carries the AI-reconstructed label. Repeater echoes: the raw RX log handler now decrypts GRP_DATA as well as GRP_TXT and counts repeaters re-flooding each chunk of our own images. The bubble shows the repeat count like channel text, and Packet info lists each packet. About 15 s after sending, chunks no repeater echoed are resent once, but only when another chunk of the same image was echoed, so senders with no repeater in earshot do not double their airtime. Outgoing entries now record their payload size. --- documentation/design.md | 1 + documentation/image-messages.md | 11 +- lib/connector/meshcore_connector.dart | 189 ++++++++- lib/l10n/app_en.arb | 38 ++ lib/l10n/app_localizations.dart | 42 ++ lib/l10n/app_localizations_bg.dart | 50 +++ lib/l10n/app_localizations_cs.dart | 50 +++ lib/l10n/app_localizations_de.dart | 50 +++ lib/l10n/app_localizations_en.dart | 50 +++ lib/l10n/app_localizations_es.dart | 50 +++ lib/l10n/app_localizations_fr.dart | 50 +++ lib/l10n/app_localizations_hu.dart | 50 +++ lib/l10n/app_localizations_it.dart | 50 +++ lib/l10n/app_localizations_ja.dart | 50 +++ lib/l10n/app_localizations_ko.dart | 50 +++ lib/l10n/app_localizations_nl.dart | 50 +++ lib/l10n/app_localizations_pl.dart | 50 +++ lib/l10n/app_localizations_pt.dart | 50 +++ lib/l10n/app_localizations_ru.dart | 50 +++ lib/l10n/app_localizations_sk.dart | 50 +++ lib/l10n/app_localizations_sl.dart | 50 +++ lib/l10n/app_localizations_sv.dart | 50 +++ lib/l10n/app_localizations_uk.dart | 50 +++ lib/l10n/app_localizations_zh.dart | 50 +++ lib/main.dart | 2 + lib/screens/channel_chat_screen.dart | 79 +++- lib/services/image_codec_service.dart | 37 +- lib/services/received_image_store.dart | 72 +++- lib/widgets/image_send_codec_binding.dart | 87 +++- lib/widgets/image_send_preview_sheet.dart | 399 ++++++++++++++---- lib/widgets/received_image_message.dart | 11 +- test/connector/image_echo_test.dart | 141 +++++++ test/services/received_image_store_test.dart | 67 +++ .../image_send_codec_binding_test.dart | 57 +++ .../image_send_preview_sheet_test.dart | 141 +++++++ untranslated.json | 192 ++++++++- 36 files changed, 2366 insertions(+), 150 deletions(-) create mode 100644 test/connector/image_echo_test.dart create mode 100644 test/widgets/image_send_codec_binding_test.dart diff --git a/documentation/design.md b/documentation/design.md index 231aaaf3..423b53aa 100644 --- a/documentation/design.md +++ b/documentation/design.md @@ -118,6 +118,7 @@ Icon sizes in use: 24 for tile and banner icons, 20 for menu icons and small act | Settings navigation row | `MeshNavTile` | `onTap: null` dims it to 50 % and reports it as disabled; use `subtitle` to say why | | Settings switch | `SwitchListTile` inside `MeshSettingsTileTheme` | Wrap the settings screen body once in `MeshSettingsTileTheme` | | Radio-style choice in a sheet | `MeshSheetOption` | Used by the language picker and map time filter | +| Choosing between images | Selectable thumbnail tiles in a centred `Wrap` | Image send versions (`_versionTile` in `image_send_preview_sheet.dart`). `InkWell` with `MeshRadii.sm`; selected tile gets a 3px `primary` border **and** a `primary` check badge (never colour alone); caption under the tile in `bodySmall`, w600 when selected; figures such as packet counts in `MeshTheme.mono` 12; wrap the tile in `Semantics(button: true, selected:, label:)` | | Popup menu row | **always** `PopupMenuItem(child: MeshMenuItemRow(icon:, label:))` | Label wraps to 2 lines then ellipsizes; `leading:` for `ThemedIcon`; no icon keeps the 20px slot | | Segmented button label | **always** `ButtonSegment(label: MeshSegmentLabel(text))` | Shrinks to 12 (before system scale), then wraps at word boundaries onto 2 centred lines; never breaks a word | | Status | `StatusChip` | Mono label, coloured dot or icon | diff --git a/documentation/image-messages.md b/documentation/image-messages.md index 2cda8bf8..935dd398 100644 --- a/documentation/image-messages.md +++ b/documentation/image-messages.md @@ -15,9 +15,14 @@ Downloads use the configured Hugging Face model bundle, support resume, and veri 1. Open channel chat and tap the image button beside the composer. 2. Pick a photo from your gallery. -3. Wait for encoding. Inspect the source preview and, if available, the reconstructed preview. -4. Review transmitted bytes, packet count, and estimated send duration. Estimates use the current radio settings and are unavailable if those settings are unknown. -5. Choose whether to include a recovery packet, then send. The chat shows per-packet progress. +3. Wait for the preview. The sheet shows **What recipients see**: the photo encoded and then decoded on your device, which is the picture recipients reconstruct. Switch to **Original** to compare. +4. If the reconstruction got something wrong, tap **Try another version**. Each version encodes the whole photo with a tiny, invisible change of framing, and the model reconstructs each one a little differently. Tap the version you want to send; the packet count and time on air follow your choice. Up to six versions can be made, one at a time. +5. Review transmitted bytes, packet count, and estimated send duration. Estimates use the current radio settings and are unavailable if those settings are unknown. +6. Choose whether to include a recovery packet, then send. The chat shows per-packet progress, and your own copy of the image shows the version you chose. + +After sending, your image shows how many times repeaters were heard passing it on, like a channel message (Packet info lists each packet). If repeaters passed on some packets but not others, the missing ones are sent once more automatically. + +On a device that cannot decode, the sheet shows the original photo with a note instead, and versions are not offered. Each new version costs one encode and one decode, so it can take a few seconds on older phones. The current build ships one quality/model setting: **ft32**, with a 512×512 codec reconstruction and metadata used to restore a supported aspect ratio. Reserved rate/resolution codes are not additional selectable quality options. diff --git a/lib/connector/meshcore_connector.dart b/lib/connector/meshcore_connector.dart index 0c1e668f..4b4168e4 100644 --- a/lib/connector/meshcore_connector.dart +++ b/lib/connector/meshcore_connector.dart @@ -29,11 +29,14 @@ import '../services/linux_ble_pairing_service_stub.dart' if (dart.library.io) '../services/linux_ble_pairing_service.dart'; import '../services/image_chunk_transport.dart' show + ImageChunkHeader, ImageChunkOutcome, ImageChunkTransport, buildSendChannelDataFrame, dataTypeAeicImage, + kImageReassemblyTtl, outPathUnknown, + parseImageChunkHeader, respCodeChannelDataRecv, senderPrefixFromKey; import '../services/image_codec_service.dart'; @@ -5714,7 +5717,9 @@ class MeshCoreConnector extends ChangeNotifier { /// queued a packet to the radio. It says nothing about transmission or /// delivery — GRP_DATA is unacknowledged flood traffic, there is no /// PUSH_CODE_SEND_CONFIRMED for it, and a chunk may still be lost on air. - /// That is what the XOR parity chunk is for. + /// That is what the XOR parity chunk is for, and why repeater echoes are + /// counted afterwards ([ownImageEchoes]). With [repairUnheardChunks], chunks + /// no repeater echoed are resent once (see [_repairUnheardImageChunks]). /// /// Returns false when the transport is unavailable (disconnected or firmware /// too old); throws whatever [_sendFrameAndWaitForCommandAck] throws, i.e. @@ -5727,9 +5732,47 @@ class MeshCoreConnector extends ChangeNotifier { required int channelIndex, required Duration interChunkDelay, void Function(int sent, int total)? onProgress, + bool repairUnheardChunks = true, }) async { if (blobs.isEmpty) return false; if (!supportsChannelData) return false; + final header = parseImageChunkHeader(blobs.first); + final key = header == null ? null : (channelIndex, header.imgId); + if (key != null && repairUnheardChunks) { + // Index == chunk index: data chunks first, the parity chunk (index == + // total) last, so blobs.length slots cover every chunk we send. + _ownImageEchoes.remove(key); + _ownImageEchoes[key] = List.filled(blobs.length, 0); + while (_ownImageEchoes.length > _maxTrackedOwnImages) { + _ownImageEchoes.remove(_ownImageEchoes.keys.first); + } + } + final startedAt = DateTime.now(); + final sentAll = await _transmitImageBlobs( + blobs, + channelIndex: channelIndex, + interChunkDelay: interChunkDelay, + onProgress: onProgress, + ); + if (sentAll && key != null && repairUnheardChunks) { + unawaited( + _repairUnheardImageChunks( + key, + blobs, + interChunkDelay: interChunkDelay, + startedAt: startedAt, + ), + ); + } + return sentAll; + } + + Future _transmitImageBlobs( + List blobs, { + required int channelIndex, + required Duration interChunkDelay, + void Function(int sent, int total)? onProgress, + }) async { // `return` inside the scoped callback only exits the callback, so a // disconnect mid-send used to fall through to `return true` below and the // caller would record a complete outgoing image and tell the user "sent" @@ -5760,6 +5803,142 @@ class MeshCoreConnector extends ChangeNotifier { return sentAll; } + // ---- repeater echoes of our own image chunks ------------------------------- + // + // GRP_DATA is never acknowledged. What we can observe is a repeater + // re-flooding one of our chunks: the radio reports it in the raw RX log, the + // same way repeats of channel text are counted. A chunk nobody echoed while + // other chunks of the same image were echoed was most likely lost, so it is + // sent once more. + + /// How long to listen for echoes after the last chunk before repairing. + static const Duration _imageEchoWait = Duration(seconds: 15); + + /// Receivers drop a partial image [kImageReassemblyTtl] after its first + /// chunk. A repair that cannot land well inside that window is skipped. + static const Duration _imageRepairDeadline = Duration(seconds: 45); + + static const int _maxTrackedOwnImages = 16; + + final Map<(int, int), List> _ownImageEchoes = {}; + final Set<(int, int)> _imageRepairsInFlight = {}; + + /// Repeater echoes heard for each chunk of one of our own images sent this + /// session (data chunks, then the parity chunk), or null if not tracked. + List? ownImageEchoes(int channelIndex, int imgId) { + final echoes = _ownImageEchoes[(channelIndex, imgId)]; + return echoes == null ? null : List.unmodifiable(echoes); + } + + /// Non-null while unheard chunks of that image are being resent. + ({int resends, int maxResends})? imageResendProgress( + int channelIndex, + int imgId, + ) => _imageRepairsInFlight.contains((channelIndex, imgId)) + ? (resends: 1, maxResends: 1) + : null; + + /// A GRP_DATA packet from the raw RX log. Other people's images arrive + /// already decrypted through RESP_CODE_CHANNEL_DATA_RECV; here we only look + /// for repeaters re-flooding one of our own chunks. + void _handleImageEcho(_RawPacket packet) { + final self = senderPrefixFromKey(_selfPublicKey); + if (self == null || packet.payload.length < 2) return; + final channelHash = packet.payload[0]; + final encrypted = Uint8List.fromList(packet.payload.sublist(1)); + final channels = _channels.isNotEmpty ? _channels : _cachedChannels; + for (final channel in channels) { + if (channel.isEmpty) continue; + if (_computeChannelHash(channel.psk) != channelHash) continue; + // Null unless the MAC matches this channel's key. + final plain = _decryptPayload(channel.psk, encrypted); + if (plain == null || plain.length < 3) continue; + // Plaintext: data_type (uint16 LE), data_len, blob (padded to 16). + final dataType = plain[0] | (plain[1] << 8); + final length = plain[2]; + if (dataType != dataTypeAeicImage || plain.length < 3 + length) return; + final header = parseImageChunkHeader( + Uint8List.sublistView(plain, 3, 3 + length), + ); + if (header == null || header.senderPrefix != self) return; + final channelId = channel.storageId; + final slot = _channelMapConfirmed && channelId != null + ? _slotForChannelId(channelId) + : channel.index; + if (slot == null) return; + _recordOwnImageEcho(slot, header); + return; + } + } + + void _recordOwnImageEcho(int slot, ImageChunkHeader header) { + final echoes = _ownImageEchoes[(slot, header.imgId)]; + if (echoes == null || header.index >= echoes.length) return; + echoes[header.index]++; + unawaited( + _receivedImageStore?.setOutgoingEchoes( + channelIndex: slot, + senderPrefix: header.senderPrefix, + imgId: header.imgId, + echoes: List.of(echoes), + ), + ); + notifyListeners(); + } + + /// One repair round: resend the chunks no repeater was heard echoing, but + /// only when some other chunk of the image was echoed. With no echo at all + /// we cannot tell a lost image from a sender with no repeater in earshot, + /// and resending would double every such user's airtime for nothing. + /// + /// The resent chunk is byte-identical, so a repeater that did forward it + /// (and we just missed the echo) drops the copy as already seen. Receivers + /// missing it merge it into the image they are still holding. + Future _repairUnheardImageChunks( + (int, int) key, + List blobs, { + required Duration interChunkDelay, + required DateTime startedAt, + }) async { + await Future.delayed(_imageEchoWait); + final echoes = _ownImageEchoes[key]; + if (echoes == null || echoes.length != blobs.length || !isConnected) { + return; + } + final unheard = [ + for (var i = 0; i < blobs.length; i++) + if (echoes[i] == 0) blobs[i], + ]; + if (unheard.isEmpty || unheard.length == blobs.length) return; + if (DateTime.now().difference(startedAt) > _imageRepairDeadline) { + _appDebugLogService?.info( + 'Image ${key.$2}: ${unheard.length} chunk(s) unheard, too late to ' + 'resend before receivers give up', + tag: 'Image resend', + ); + return; + } + _appDebugLogService?.info( + 'Image ${key.$2}: resending ${unheard.length} of ${blobs.length} ' + 'chunk(s) no repeater echoed', + tag: 'Image resend', + ); + _imageRepairsInFlight.add(key); + notifyListeners(); + try { + await _transmitImageBlobs( + unheard, + channelIndex: key.$1, + interChunkDelay: interChunkDelay, + ); + } catch (e) { + _appDebugLogService?.warn('Image resend failed: $e', tag: 'Image resend'); + } finally { + _imageRepairsInFlight.remove(key); + notifyListeners(); + } + } + Future _runScopedChannelSend( Future Function() action, { required String region, @@ -8396,7 +8575,12 @@ class MeshCoreConnector extends ChangeNotifier { final raw = reader.readRemainingBytes(); final packet = _parseRawPacket(raw); - if (packet == null || packet.payloadType != _payloadTypeGroupText) return; + if (packet == null) return; + if (packet.payloadType == _payloadTypeGroupData) { + _handleImageEcho(packet); + return; + } + if (packet.payloadType != _payloadTypeGroupText) return; final payload = BufferReader(packet.payload); final channelHash = payload.readByte(); @@ -10477,6 +10661,7 @@ const int _routeFlood = 0x01; const int _routeTransportDirect = 0x03; const int _payloadTypeGroupText = 0x05; +const int _payloadTypeGroupData = 0x06; const int _cipherMacSize = 2; bool _isPathLenValidForMode( diff --git a/lib/l10n/app_en.arb b/lib/l10n/app_en.arb index cda9324f..26d43828 100644 --- a/lib/l10n/app_en.arb +++ b/lib/l10n/app_en.arb @@ -3496,6 +3496,44 @@ "imageSend_viewOriginal": "Original", "imageSend_viewReconstruction": "What recipients see", "imageSend_reconstructionUnavailable": "This device can't preview the reconstruction.", + "receivedImage_packetEchoes": "Packet {number}: {count, plural, =0{no repeater heard} =1{repeated once} other{repeated {count} times}}", + "@receivedImage_packetEchoes": { + "placeholders": { + "number": { + "type": "int" + }, + "count": { + "type": "int" + } + } + }, + "receivedImage_recoveryPacketEchoes": "Recovery packet: {count, plural, =0{no repeater heard} =1{repeated once} other{repeated {count} times}}", + "@receivedImage_recoveryPacketEchoes": { + "placeholders": { + "count": { + "type": "int" + } + } + }, + "imageSend_preparingPreview": "Preparing preview…", + "imageSend_tryAnother": "Try another version", + "imageSend_versionsHint": "Each version is reconstructed a little differently. Pick the one that looks most like your photo.", + "imageSend_versionLabel": "Version {number}", + "@imageSend_versionLabel": { + "placeholders": { + "number": { + "type": "int" + } + } + }, + "imageSend_versionPackets": "{count, plural, =1{1 packet} other{{count} packets}}", + "@imageSend_versionPackets": { + "placeholders": { + "count": { + "type": "int" + } + } + }, "imageSend_modelNotDownloaded": "The image model isn't downloaded yet. Download it in Settings to send images.", "imageSend_originalSize": "Original", "imageSend_onAirSize": "On air", diff --git a/lib/l10n/app_localizations.dart b/lib/l10n/app_localizations.dart index bdd028de..e0582ade 100644 --- a/lib/l10n/app_localizations.dart +++ b/lib/l10n/app_localizations.dart @@ -10565,6 +10565,48 @@ abstract class AppLocalizations { /// **'This device can\'t preview the reconstruction.'** String get imageSend_reconstructionUnavailable; + /// No description provided for @receivedImage_packetEchoes. + /// + /// In en, this message translates to: + /// **'Packet {number}: {count, plural, =0{no repeater heard} =1{repeated once} other{repeated {count} times}}'** + String receivedImage_packetEchoes(int number, int count); + + /// No description provided for @receivedImage_recoveryPacketEchoes. + /// + /// In en, this message translates to: + /// **'Recovery packet: {count, plural, =0{no repeater heard} =1{repeated once} other{repeated {count} times}}'** + String receivedImage_recoveryPacketEchoes(int count); + + /// No description provided for @imageSend_preparingPreview. + /// + /// In en, this message translates to: + /// **'Preparing preview…'** + String get imageSend_preparingPreview; + + /// No description provided for @imageSend_tryAnother. + /// + /// In en, this message translates to: + /// **'Try another version'** + String get imageSend_tryAnother; + + /// No description provided for @imageSend_versionsHint. + /// + /// In en, this message translates to: + /// **'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'** + String get imageSend_versionsHint; + + /// No description provided for @imageSend_versionLabel. + /// + /// In en, this message translates to: + /// **'Version {number}'** + String imageSend_versionLabel(int number); + + /// No description provided for @imageSend_versionPackets. + /// + /// In en, this message translates to: + /// **'{count, plural, =1{1 packet} other{{count} packets}}'** + String imageSend_versionPackets(int count); + /// No description provided for @imageSend_modelNotDownloaded. /// /// In en, this message translates to: diff --git a/lib/l10n/app_localizations_bg.dart b/lib/l10n/app_localizations_bg.dart index f45c475d..7d7e88bc 100644 --- a/lib/l10n/app_localizations_bg.dart +++ b/lib/l10n/app_localizations_bg.dart @@ -6245,6 +6245,56 @@ class AppLocalizationsBg extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Това устройство не може да покаже реконструкцията.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'Моделът за изображения още не е изтеглен. Изтеглете го в настройките, за да изпращате изображения.'; diff --git a/lib/l10n/app_localizations_cs.dart b/lib/l10n/app_localizations_cs.dart index b015322c..c3cea98a 100644 --- a/lib/l10n/app_localizations_cs.dart +++ b/lib/l10n/app_localizations_cs.dart @@ -6231,6 +6231,56 @@ class AppLocalizationsCs extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Toto zařízení nemůže zobrazit náhled rekonstrukce.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'Model obrázků ještě není stažen. Pro odesílání obrázků jej stáhněte v Nastavení.'; diff --git a/lib/l10n/app_localizations_de.dart b/lib/l10n/app_localizations_de.dart index c8d2226d..9317d6a1 100644 --- a/lib/l10n/app_localizations_de.dart +++ b/lib/l10n/app_localizations_de.dart @@ -6263,6 +6263,56 @@ class AppLocalizationsDe extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Dieses Gerät kann die Rekonstruktion nicht in der Vorschau anzeigen.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'Das Bildmodell wurde noch nicht heruntergeladen. Laden Sie es in den Einstellungen herunter, um Bilder zu senden.'; diff --git a/lib/l10n/app_localizations_en.dart b/lib/l10n/app_localizations_en.dart index 880a8a4b..e0c6670e 100644 --- a/lib/l10n/app_localizations_en.dart +++ b/lib/l10n/app_localizations_en.dart @@ -6159,6 +6159,56 @@ class AppLocalizationsEn extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'This device can\'t preview the reconstruction.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'The image model isn\'t downloaded yet. Download it in Settings to send images.'; diff --git a/lib/l10n/app_localizations_es.dart b/lib/l10n/app_localizations_es.dart index c14c8fcf..7af2dbf8 100644 --- a/lib/l10n/app_localizations_es.dart +++ b/lib/l10n/app_localizations_es.dart @@ -6247,6 +6247,56 @@ class AppLocalizationsEs extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Este dispositivo no puede previsualizar la reconstrucción.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'El modelo de imagen aún no se ha descargado. Descárgalo en Configuración para enviar imágenes.'; diff --git a/lib/l10n/app_localizations_fr.dart b/lib/l10n/app_localizations_fr.dart index 61effc54..833c9e59 100644 --- a/lib/l10n/app_localizations_fr.dart +++ b/lib/l10n/app_localizations_fr.dart @@ -6270,6 +6270,56 @@ class AppLocalizationsFr extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Cet appareil ne peut pas prévisualiser la reconstitution.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'Le modèle d\'image n\'est pas encore téléchargé. Téléchargez-le dans Paramètres pour envoyer des images.'; diff --git a/lib/l10n/app_localizations_hu.dart b/lib/l10n/app_localizations_hu.dart index b99d6fe9..5a26abff 100644 --- a/lib/l10n/app_localizations_hu.dart +++ b/lib/l10n/app_localizations_hu.dart @@ -6244,6 +6244,56 @@ class AppLocalizationsHu extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Ezen az eszközön nem jeleníthető meg a rekonstrukció előnézete.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'A képmodell még nincs letöltve. Töltse le a Beállításokban a képek küldéséhez.'; diff --git a/lib/l10n/app_localizations_it.dart b/lib/l10n/app_localizations_it.dart index 058c8a27..9c863c67 100644 --- a/lib/l10n/app_localizations_it.dart +++ b/lib/l10n/app_localizations_it.dart @@ -6252,6 +6252,56 @@ class AppLocalizationsIt extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Questo dispositivo non può mostrare un\'anteprima della ricostruzione.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'Il modello di immagine non è ancora stato scaricato. Scaricalo nelle Impostazioni per inviare immagini.'; diff --git a/lib/l10n/app_localizations_ja.dart b/lib/l10n/app_localizations_ja.dart index 51064d0a..d4979730 100644 --- a/lib/l10n/app_localizations_ja.dart +++ b/lib/l10n/app_localizations_ja.dart @@ -5944,6 +5944,56 @@ class AppLocalizationsJa extends AppLocalizations { @override String get imageSend_reconstructionUnavailable => 'このデバイスでは復元結果をプレビューできません。'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => '画像モデルがまだダウンロードされていません。設定でダウンロードすると画像を送信できます。'; diff --git a/lib/l10n/app_localizations_ko.dart b/lib/l10n/app_localizations_ko.dart index 9544d808..6dcd999c 100644 --- a/lib/l10n/app_localizations_ko.dart +++ b/lib/l10n/app_localizations_ko.dart @@ -5960,6 +5960,56 @@ class AppLocalizationsKo extends AppLocalizations { String get imageSend_reconstructionUnavailable => '이 장치에서는 복원 결과를 미리 볼 수 없습니다.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => '이미지 모델이 아직 다운로드되지 않았습니다. 설정에서 다운로드하면 이미지를 보낼 수 있습니다.'; diff --git a/lib/l10n/app_localizations_nl.dart b/lib/l10n/app_localizations_nl.dart index 565e79a4..48a8b16a 100644 --- a/lib/l10n/app_localizations_nl.dart +++ b/lib/l10n/app_localizations_nl.dart @@ -6216,6 +6216,56 @@ class AppLocalizationsNl extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Dit apparaat kan geen voorbeeld van de reconstructie tonen.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'Het afbeeldingsmodel is nog niet gedownload. Download het in Instellingen om afbeeldingen te verzenden.'; diff --git a/lib/l10n/app_localizations_pl.dart b/lib/l10n/app_localizations_pl.dart index 5f5c9b00..8889a984 100644 --- a/lib/l10n/app_localizations_pl.dart +++ b/lib/l10n/app_localizations_pl.dart @@ -6290,6 +6290,56 @@ class AppLocalizationsPl extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'To urządzenie nie może wyświetlić podglądu rekonstrukcji.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'Model obrazowy nie jest jeszcze pobrany. Pobierz go w Ustawieniach, aby wysyłać obrazy.'; diff --git a/lib/l10n/app_localizations_pt.dart b/lib/l10n/app_localizations_pt.dart index 0c0749b6..a0274faa 100644 --- a/lib/l10n/app_localizations_pt.dart +++ b/lib/l10n/app_localizations_pt.dart @@ -6241,6 +6241,56 @@ class AppLocalizationsPt extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Este dispositivo não pode pré-visualizar a reconstrução.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'O modelo de imagem ainda não foi transferido. Transfira-o nas Configurações para enviar imagens.'; diff --git a/lib/l10n/app_localizations_ru.dart b/lib/l10n/app_localizations_ru.dart index 5a6119ee..5da20d8c 100644 --- a/lib/l10n/app_localizations_ru.dart +++ b/lib/l10n/app_localizations_ru.dart @@ -6266,6 +6266,56 @@ class AppLocalizationsRu extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Это устройство не может показать реконструкцию.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'Модель изображения ещё не загружена. Загрузите её в настройках, чтобы отправлять изображения.'; diff --git a/lib/l10n/app_localizations_sk.dart b/lib/l10n/app_localizations_sk.dart index 109fdbd5..cbc100b5 100644 --- a/lib/l10n/app_localizations_sk.dart +++ b/lib/l10n/app_localizations_sk.dart @@ -6240,6 +6240,56 @@ class AppLocalizationsSk extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Toto zariadenie nedokáže zobraziť náhľad rekonštrukcie.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'Model obrazu ešte nie je stiahnutý. Stiahnite ho v Nastaveniach, aby ste mohli posielať obrázky.'; diff --git a/lib/l10n/app_localizations_sl.dart b/lib/l10n/app_localizations_sl.dart index aedc64ea..e2a05bf1 100644 --- a/lib/l10n/app_localizations_sl.dart +++ b/lib/l10n/app_localizations_sl.dart @@ -6232,6 +6232,56 @@ class AppLocalizationsSl extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Ta naprava ne more prikazati predogleda rekonstrukcije.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'Obrazni model še ni prenesen. Prenesite ga v Nastavitvah, da boste lahko pošiljali slike.'; diff --git a/lib/l10n/app_localizations_sv.dart b/lib/l10n/app_localizations_sv.dart index 50dc42c8..dfecda49 100644 --- a/lib/l10n/app_localizations_sv.dart +++ b/lib/l10n/app_localizations_sv.dart @@ -6199,6 +6199,56 @@ class AppLocalizationsSv extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Enheten kan inte förhandsgranska rekonstruktionen.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'Bildmodellen har ännu inte laddats ner. Ladda ner den i Inställningar för att skicka bilder.'; diff --git a/lib/l10n/app_localizations_uk.dart b/lib/l10n/app_localizations_uk.dart index a656a790..10daf118 100644 --- a/lib/l10n/app_localizations_uk.dart +++ b/lib/l10n/app_localizations_uk.dart @@ -6272,6 +6272,56 @@ class AppLocalizationsUk extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Цей пристрій не може показати відтворення.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'Модель зображення ще не завантажена. Завантажте її в налаштуваннях, щоб надсилати зображення.'; diff --git a/lib/l10n/app_localizations_zh.dart b/lib/l10n/app_localizations_zh.dart index a1661572..322c287c 100644 --- a/lib/l10n/app_localizations_zh.dart +++ b/lib/l10n/app_localizations_zh.dart @@ -5807,6 +5807,56 @@ class AppLocalizationsZh extends AppLocalizations { @override String get imageSend_reconstructionUnavailable => '此设备无法预览重建结果。'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => '图像模型尚未下载。在设置中下载后才能发送图像。'; diff --git a/lib/main.dart b/lib/main.dart index 751e79d2..8f1cef8d 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -140,6 +140,8 @@ void main() async { [blob], channelIndex: channelIndex, interChunkDelay: Duration.zero, + // One blob at a time: echo tracking needs the whole chunk set. + repairUnheardChunks: false, ), // Overwritten by onImageSenderPrefix as soon as SELF_INFO lands. senderPrefix: 0, diff --git a/lib/screens/channel_chat_screen.dart b/lib/screens/channel_chat_screen.dart index fc276315..ebf4b0c6 100644 --- a/lib/screens/channel_chat_screen.dart +++ b/lib/screens/channel_chat_screen.dart @@ -1892,9 +1892,8 @@ class _ChannelChatScreenState extends State { final connector = context.read(); // The picked bytes are kept past the preview on purpose: the sender's own - // bubble renders from them (see [_registerOutgoingImage]). The bitstream - // cannot serve — turning it back into pixels is a ~2.16 GiB, ~1 s decode - // of an image the sender is already holding. + // bubble falls back to them (see [_registerOutgoingImage]) when the sheet + // could not decode the chosen version on this device. final XFile? picked; try { picked = await ImagePicker().pickImage( @@ -2070,8 +2069,11 @@ class _ChannelChatScreenState extends State { /// GRP_DATA is not echoed back to the sender and an image is not a /// [ChannelMessage], so without this the sender sees a "sent" snack bar and /// an otherwise empty conversation. The entry lands in `decoded` directly — - /// an outgoing image is never queued for inference, so showing your own - /// photo costs no model memory. + /// an outgoing image is never queued for inference. + /// + /// The bubble shows the version the user picked, decoded as recipients see + /// it ([ImageSendPreviewResult.previewPng]). Only when this device could not decode does it fall back + /// to the stretched original. /// /// Best effort throughout: a failure here has already been preceded by a /// successful transmission, so it must not turn into an error the user sees. @@ -2087,7 +2089,8 @@ class _ChannelChatScreenState extends State { } on ProviderNotFoundException { return; // screen test without the store } - final previewPng = await _squarePreviewPng(sourceBytes); + final previewPng = + result.previewPng ?? await _squarePreviewPng(sourceBytes); if (previewPng == null) return; try { await store.registerOutgoing( @@ -2101,6 +2104,18 @@ class _ChannelChatScreenState extends State { rate: aeicRatePointForUi(result.rate), chunkCount: chunkSet.dataChunkCount, aspectCode: await _aspectCodeOf(sourceBytes), + // The decoded version is model output and carries the AI label; the + // stretched-original fallback is the sender's own pixels. + synthesized: result.previewPng != null, + payloadBytes: result.payload.length, + // Repeaters may already have echoed the first chunks while the rest + // were still going out. + chunkEchoes: mounted + ? context.read().ownImageEchoes( + widget.channel.index, + chunkSet.imgId, + ) + : null, ); } on Exception catch (error) { debugPrint('outgoing image registration failed: $error'); @@ -2128,11 +2143,9 @@ class _ChannelChatScreenState extends State { /// [imageBytes] stretched to 512x512, as PNG. /// - /// This is what the preview sheet showed (`BoxFit.fill` on a 1:1 box is - /// exactly the stretch the codec applies) and what the codec actually - /// encoded, so the sender's bubble matches both. Deliberately not a decode of the - /// bitstream: that would cost ~2.16 GiB and ~1 s to reproduce, less well, an - /// image this device is already holding in memory. + /// Fallback for the sender's bubble when the preview sheet had no decoded + /// version to hand over (a device that cannot decode). It is the input the + /// codec stretched, not what recipients see, so it is only a stand-in. /// /// Returns null rather than throwing on an undecodable source — the send has /// already happened by the time this runs. @@ -2375,12 +2388,35 @@ class _ChannelChatScreenState extends State { right: 8, bottom: 4, ), - child: Text( - formatChatTime(context, entry.firstSeen), - style: MeshTheme.mono( - fontSize: 12 * textScale, - color: metaColor, - ), + child: Row( + mainAxisSize: MainAxisSize.min, + children: [ + Text( + formatChatTime(context, entry.firstSeen), + style: MeshTheme.mono( + fontSize: 12 * textScale, + color: metaColor, + ), + ), + if (isOutgoing) ...[ + const SizedBox(width: 4), + // Same status as channel text: repeats heard, here + // for the least-heard data chunk. + MessageStatusIcon( + size: 16 * textScale, + isAcked: false, + isChannel: true, + repeatCount: entry.minDataChunkEchoes ?? 0, + resendProgress: context + .watch() + .imageResendProgress( + entry.channelIndex, + entry.imgId, + ), + onColor: metaColor, + ), + ], + ], ), ), ], @@ -2513,6 +2549,15 @@ class _ChannelChatScreenState extends State { if (entry.recoveredWithParity) Text(l10n.receivedImage_parityRecovered), if (decodeMs != null) Text(l10n.receivedImage_decodeTime(decodeMs)), + if (entry.chunkEchoes case final echoes?) ...[ + const SizedBox(height: 8), + for (var i = 0; i < echoes.length; i++) + Text( + i < entry.totalChunks + ? l10n.receivedImage_packetEchoes(i + 1, echoes[i]) + : l10n.receivedImage_recoveryPacketEchoes(echoes[i]), + ), + ], ], ), actions: [ diff --git a/lib/services/image_codec_service.dart b/lib/services/image_codec_service.dart index edf5e0e5..932d90d7 100644 --- a/lib/services/image_codec_service.dart +++ b/lib/services/image_codec_service.dart @@ -970,15 +970,17 @@ class ImageCodecService extends ChangeNotifier implements ImageSendCodec { /// [ImageSendCodec] entry point used by `ImageSendPreviewSheet`. /// - /// Takes the raw picked file (JPEG/PNG/HEIC/…), centre-crops and scales it to - /// [kImageCodecSquareSize] square, then encodes. Throws on failure so the + /// Takes the raw picked file (JPEG/PNG/HEIC/…), stretches the part of it + /// that [variant] selects (see [rerollSourceRect]; 0 is the whole photo) to + /// a [kImageCodecSquareSize] square, then encodes. Throws on failure so the /// sheet can surface it — unlike the nullable service-level methods below, /// whose contract is "null means not available". @override Future encode( Uint8List imageBytes, - ImageCodecRatePoint rate, - ) async { + ImageCodecRatePoint rate, { + int variant = 0, + }) async { if (!kImageCodecBitstreamPathAvailable) { // Fail here rather than after a 512x512 raster and a session load, and // fail with the specific exception so the sheet can say why. @@ -990,7 +992,7 @@ class ImageCodecService extends ChangeNotifier implements ImageSendCodec { throw const ImageCodecBundleIncomplete(); } final result = await encodeImage( - rgbBytes: await _toSquareRgb(imageBytes), + rgbBytes: await _toSquareRgb(imageBytes, variant: variant), ratePoint: aeicRatePointForUi(rate), ); final bitstream = result?.bitstream; @@ -1300,14 +1302,18 @@ class ImageCodecService extends ChangeNotifier implements ImageSendCodec { static const int _bytesPerPixel = 3; - /// Centre-crops [imageBytes] to a square and scales it to - /// [kImageCodecSquareSize], returning packed 8-bit RGB. + /// Stretches the [variant] region of [imageBytes] (see [rerollSourceRect]; + /// 0 is the whole frame) to a [kImageCodecSquareSize] square, returning + /// packed 8-bit RGB. /// /// Uses `dart:ui` rather than a decode package so this adds no dependency. /// It runs on the root isolate, which is acceptable: it is a single 512x512 /// raster, three orders of magnitude cheaper than the 1385 GFLOP inference /// that follows in the worker. - Future _toSquareRgb(Uint8List imageBytes) async { + Future _toSquareRgb( + Uint8List imageBytes, { + int variant = 0, + }) async { final codec = await ui.instantiateImageCodec(imageBytes); final frame = await codec.getNextFrame(); final source = frame.image; @@ -1317,16 +1323,11 @@ class ImageCodecService extends ChangeNotifier implements ImageSendCodec { // Aspect ratio is deliberately not preserved: the codec's graph is // statically 512x512, so something has to give, and losing 25% of a 4:3 // photo (44% of 16:9, or the subject's head in a portrait) is worse than - // distorting it. Nothing outside the frame is discarded this way. - // NOTE: the aspect ratio is NOT transmitted, so the receiver cannot undo - // the stretch — see the note in image_chunk_transport.dart on the - // metadata byte. - final src = ui.Rect.fromLTWH( - 0, - 0, - source.width.toDouble(), - source.height.toDouble(), - ); + // distorting it. The source shape travels in the metadata byte's aspect + // nibble, so the receiver un-stretches it. Re-roll variants trim at most + // ~2% at an edge, so they keep the same shape and aspect code. + final r = rerollSourceRect(source.width, source.height, variant); + final src = ui.Rect.fromLTWH(r.left, r.top, r.width, r.height); final dstSize = kImageCodecSquareSize.toDouble(); final recorder = ui.PictureRecorder(); ui.Canvas(recorder).drawImageRect( diff --git a/lib/services/received_image_store.dart b/lib/services/received_image_store.dart index 94ee9ee1..401204da 100644 --- a/lib/services/received_image_store.dart +++ b/lib/services/received_image_store.dart @@ -197,13 +197,30 @@ class ReceivedImageEntry { /// One chunk was rebuilt from the XOR parity chunk. final bool recoveredWithParity; - /// This is the local user's own send. Its PNG is their real 512x512 crop, so - /// [synthesized] is false and the R6 label must NOT be shown. + /// This is the local user's own send. final bool isOutgoing; - /// Whether the pixels are generative-model output. Derived, never supplied: - /// `synthesized == !isOutgoing`. - bool get synthesized => !isOutgoing; + /// Whether the pixels are generative-model output, so the R6 label must be + /// shown. Always true for received images. For the user's own send it is + /// true when the PNG is the version they picked, decoded as recipients see + /// it, and false only when it is the stretched original (a device that + /// could not decode). + final bool synthesized; + + /// Our own image only: repeater echoes heard for each chunk, data chunks + /// first and the parity chunk last. Null when not tracked. + final List? chunkEchoes; + + /// The fewest echoes any data chunk got: every data chunk was heard + /// repeated at least this many times. Null when not tracked. + int? get minDataChunkEchoes { + final echoes = chunkEchoes; + if (echoes == null || echoes.isEmpty) return null; + final data = echoes.length > totalChunks + ? echoes.sublist(0, totalChunks) + : echoes; + return data.reduce((a, b) => a < b ? a : b); + } final int? decodeMs; final String? error; @@ -237,6 +254,8 @@ class ReceivedImageEntry { this.metadataAssumed = true, this.recoveredWithParity = false, this.isOutgoing = false, + bool? synthesized, + this.chunkEchoes, this.decodeMs, this.error, this.needsManualDecode = false, @@ -245,7 +264,7 @@ class ReceivedImageEntry { this.bitstreamByteCount = 0, this.pngByteCount = 0, this.pngBytes, - }); + }) : synthesized = synthesized ?? !isOutgoing; ImageStreamKey get key => ImageStreamKey( senderPrefix: senderPrefix, @@ -282,6 +301,7 @@ class ReceivedImageEntry { int? bitstreamByteCount, int? pngByteCount, Object? pngBytes = _unset, + List? chunkEchoes, }) { return ReceivedImageEntry( streamId: streamId, @@ -299,6 +319,8 @@ class ReceivedImageEntry { metadataAssumed: metadataAssumed ?? this.metadataAssumed, recoveredWithParity: recoveredWithParity ?? this.recoveredWithParity, isOutgoing: isOutgoing, + synthesized: synthesized, + chunkEchoes: chunkEchoes ?? this.chunkEchoes, decodeMs: identical(decodeMs, _unset) ? this.decodeMs : decodeMs as int?, error: identical(error, _unset) ? this.error : error as String?, needsManualDecode: needsManualDecode ?? this.needsManualDecode, @@ -329,6 +351,7 @@ class ReceivedImageEntry { 'recoveredWithParity': recoveredWithParity, 'isOutgoing': isOutgoing, 'synthesized': synthesized, + if (chunkEchoes != null) 'chunkEchoes': chunkEchoes, 'needsManualDecode': needsManualDecode, 'bitstreamStored': bitstreamStored, 'pngStored': pngStored, @@ -357,6 +380,10 @@ class ReceivedImageEntry { metadataAssumed: json['metadataAssumed'] as bool? ?? true, recoveredWithParity: json['recoveredWithParity'] as bool? ?? false, isOutgoing: json['isOutgoing'] as bool? ?? false, + synthesized: json['synthesized'] as bool?, + chunkEchoes: (json['chunkEchoes'] as List?) + ?.whereType() + .toList(), decodeMs: json['decodeMs'] as int?, error: json['error'] as String?, needsManualDecode: json['needsManualDecode'] as bool? ?? false, @@ -1041,6 +1068,9 @@ class ReceivedImageStore extends ChangeNotifier { required AeicRatePoint rate, required int chunkCount, int aspectCode = 0, + bool synthesized = false, + List? chunkEchoes, + int payloadBytes = 0, DateTime? at, }) async { final now = at ?? _clock(); @@ -1069,6 +1099,11 @@ class ReceivedImageStore extends ChangeNotifier { aspectCode: aspectCode, metadataAssumed: false, isOutgoing: true, + synthesized: synthesized, + chunkEchoes: chunkEchoes, + // The size of what was sent, for the caption and packet info. The + // bitstream itself is not kept, so bitstreamStored stays false. + bitstreamByteCount: payloadBytes, pngStored: pngWritten, pngByteCount: pngWritten ? previewPng.length : 0, pngBytes: previewPng, @@ -1078,6 +1113,31 @@ class ReceivedImageStore extends ChangeNotifier { return stored; } + /// Records repeater echoes of the user's own image (see + /// `MeshCoreConnector.ownImageEchoes`). Echoes that land before the image is + /// registered are passed to [registerOutgoing] instead. + Future setOutgoingEchoes({ + required int channelIndex, + required int senderPrefix, + required int imgId, + required List echoes, + }) async { + ReceivedImageEntry? match; + for (final entry in _entries.values) { + if (!entry.isOutgoing || + entry.imgId != imgId || + entry.channelIndex != channelIndex || + entry.senderPrefix != senderPrefix) { + continue; + } + if (match == null || entry.firstSeen.isAfter(match.firstSeen)) { + match = entry; + } + } + if (match == null) return; + await _store(match.copyWith(chunkEchoes: List.unmodifiable(echoes))); + } + // ---- decode scheduling --------------------------------------------------- /// App lifecycle gate. Backgrounding cancels an in-flight decode and returns diff --git a/lib/widgets/image_send_codec_binding.dart b/lib/widgets/image_send_codec_binding.dart index fe99a99b..c0b952ef 100644 --- a/lib/widgets/image_send_codec_binding.dart +++ b/lib/widgets/image_send_codec_binding.dart @@ -98,6 +98,54 @@ class ImageCodecRateStats { } } +/// Input tweaks for "Try another version", as `(zoom, dx, dy)` fractions of +/// the source photo. Index == version variant; 0 is the untouched photo. +/// +/// The decoder is generative and very sensitive to its input: a near-identical +/// photo can come back with different invented detail. Measured on Kodak, the +/// best of three versions raised CLIP agreement with the source from 0.828 to +/// 0.853 and fixed 3 of 8 photos that came back as something else +/// (`tools/aeic/eval`, `reroll*` variants, which mirror this table). The tweaks +/// drop at most ~2% at an edge, so every version still shows the whole photo +/// and keeps the same aspect code. +const List<(double, double, double)> kImageSendRerollTweaks = [ + (1.0, 0.0, 0.0), + (1.02, 0.005, 0.0), + (1.02, -0.005, 0.005), + (1.04, 0.0, -0.005), + (1.03, 0.01, 0.01), + (1.03, -0.01, -0.01), +]; + +/// Most versions the send sheet will make of one photo. +final int kImageSendMaxVersions = kImageSendRerollTweaks.length; + +/// The part of a [width] x [height] photo that version [variant] encodes. +/// +/// Variant 0 is the full frame, so the first version is exactly what the app +/// sent before versions existed. Other variants zoom in by `zoom`, shift by +/// `dx`/`dy` of the photo size, and are clamped to stay inside the photo. +({double left, double top, double width, double height}) rerollSourceRect( + int width, + int height, + int variant, +) { + if (variant < 0 || variant >= kImageSendRerollTweaks.length) { + throw RangeError.range( + variant, + 0, + kImageSendRerollTweaks.length - 1, + 'variant', + ); + } + final (zoom, dx, dy) = kImageSendRerollTweaks[variant]; + final w = width / zoom; + final h = height / zoom; + final left = ((width - w) / 2 + dx * width).clamp(0.0, width - w); + final top = ((height - h) / 2 + dy * height).clamp(0.0, height - h); + return (left: left, top: top, width: w, height: h); +} + /// The encode interface the preview sheet consumes. abstract class ImageSendCodec { /// Current availability. The sheet renders a non-interactive explanation for @@ -118,9 +166,14 @@ abstract class ImageSendCodec { /// Encode [imageBytes] (any common still format) at [rate], returning the /// compressed bitstream that will be chunked onto the air. /// - /// Implementations are expected to centre-crop to - /// [kImageCodecSquareSize] square first. - Future encode(Uint8List imageBytes, ImageCodecRatePoint rate); + /// Implementations stretch the part of the photo given by + /// [rerollSourceRect] for [variant] into a [kImageCodecSquareSize] square + /// first. Variant 0 is the whole photo. + Future encode( + Uint8List imageBytes, + ImageCodecRatePoint rate, { + int variant = 0, + }); /// Decodes [payload] to PNG exactly as a recipient would, or null when this /// device cannot decode. @@ -142,26 +195,44 @@ class FakeImageSendCodec implements ImageSendCodec { final Duration latency; + /// Returned by [decodePreview]; null means "this device cannot decode". + final Uint8List? previewPng; + const FakeImageSendCodec({ this.availability = ImageCodecAvailability.ready, this.unavailableReason, this.latency = const Duration(milliseconds: 250), + this.previewPng, }); + /// Payload size the fake produces for [variant]: the measured mean, plus a + /// little per variant so versions are told apart. + static int payloadBytesFor(ImageCodecRatePoint rate, int variant) => + ImageCodecRateStats.forRate(rate).meanBytes + variant * 3; + @override Future encode( Uint8List imageBytes, - ImageCodecRatePoint rate, - ) async { + ImageCodecRatePoint rate, { + int variant = 0, + }) async { if (latency > Duration.zero) { await Future.delayed(latency); } - final size = ImageCodecRateStats.forRate(rate).meanBytes; + final size = payloadBytesFor(rate, variant); return Uint8List.fromList( - List.generate(size, (i) => (i * 31 + rate.index) & 0xFF), + List.generate( + size, + (i) => (i * 31 + rate.index + variant * 7) & 0xFF, + ), ); } @override - Future decodePreview(Uint8List payload) async => null; + Future decodePreview(Uint8List payload) async { + if (latency > Duration.zero) { + await Future.delayed(latency); + } + return previewPng; + } } diff --git a/lib/widgets/image_send_preview_sheet.dart b/lib/widgets/image_send_preview_sheet.dart index 8797da5d..f20cd842 100644 --- a/lib/widgets/image_send_preview_sheet.dart +++ b/lib/widgets/image_send_preview_sheet.dart @@ -6,6 +6,7 @@ import 'package:provider/provider.dart'; import '../connector/meshcore_connector.dart'; import '../l10n/l10n.dart'; +import '../theme/mesh_theme.dart'; import '../services/image_chunk_transport.dart'; import '../utils/lora_airtime.dart'; import 'image_send_codec_binding.dart'; @@ -37,6 +38,11 @@ class ImageSendPreviewResult { /// actually shown. Null when the radio settings were unknown. final Duration? wallClock; + /// The chosen version decoded as recipients will see it (a + /// 512x512 PNG, stretched like the codec input), or null if this device + /// could not decode. Used for the sender's own chat bubble. + final Uint8List? previewPng; + const ImageSendPreviewResult({ required this.payload, required this.rate, @@ -44,6 +50,7 @@ class ImageSendPreviewResult { required this.packetCount, required this.airtime, required this.wallClock, + this.previewPng, }); } @@ -141,29 +148,53 @@ class _RateEstimate { bool get isExact => payloadBytes != null; } +/// One encoded version of the photo: the whole photo with re-roll tweak +/// [variant] (see [rerollSourceRect]), its bitstream, and what recipients will +/// see when they decode it. +class _Version { + _Version(this.variant); + + final int variant; + Uint8List? payload; + Uint8List? png; + bool encoding = true; + bool decoding = false; + + /// The encode failed; there is nothing to send for this version. + bool get failed => !encoding && payload == null; +} + class _ImageSendPreviewSheetState extends State { - /// The encoded ft32 bitstream, once the encode finishes. - Uint8List? _encoded; + /// Versions made so far, in the order they were made. Version 1 (variant 0) + /// is the photo exactly as the app has always encoded it; more are added only + /// when the user taps "Try another version". + final List<_Version> _versions = []; + int _selected = 0; late bool _parity; - bool _encoding = false; bool _sending = false; - bool _failed = false; /// Latest radio params seen in build; used by [_onSend] so the confirm path /// does not have to touch an InheritedWidget after an await. ImageSendRadio? _radio; - /// Source `width / height`. Recipients un-stretch the 512x512 square back to - /// this shape, so both preview views use it. + /// Source `width / height`, used when the original is shown. double _aspect = 1; // The shape recipients restore: the nearest supported ratio, or square. double _recipientAspect = 1; - bool _showReconstruction = false; - Uint8List? _reconstruction; - bool _reconstructing = false; - bool _reconstructionUnavailable = false; + /// The main preview shows the original photo instead of the reconstruction. + bool _showOriginal = false; + + /// This device cannot decode, so there is no reconstruction to show and + /// making more versions would be pointless. + bool _previewUnavailable = false; + + _Version? get _current => _versions.isEmpty ? null : _versions[_selected]; + + /// A version is being encoded or decoded. Codec jobs run one at a time, so + /// "Try another version" waits for this to clear. + bool get _working => _versions.any((v) => v.encoding || v.decoding); @override void initState() { @@ -171,7 +202,7 @@ class _ImageSendPreviewSheetState extends State { _parity = widget.initialParity; _probeAspect(); if (widget.codec.availability == ImageCodecAvailability.ready) { - _encode(); + _addVersion(); } } @@ -197,59 +228,56 @@ class _ImageSendPreviewSheetState extends State { } catch (_) {} } - Future _selectView(bool reconstruction) async { - setState(() => _showReconstruction = reconstruction); - final payload = _encoded; - if (!reconstruction || - payload == null || - _reconstruction != null || - _reconstructing) { - return; + /// Encodes the next version, selects it, then decodes it so the user sees + /// what recipients will get. + Future _addVersion() async { + final version = _Version(_versions.length); + setState(() { + _versions.add(version); + _selected = _versions.length - 1; + _showOriginal = false; + }); + Uint8List? payload; + try { + payload = await widget.codec.encode( + widget.imageBytes, + kImageSendRatePoint, + variant: version.variant, + ); + } catch (_) { + payload = null; } - setState(() => _reconstructing = true); + if (!mounted) return; + setState(() { + version.encoding = false; + version.payload = payload; + version.decoding = payload != null && !_previewUnavailable; + }); + if (!version.decoding) return; Uint8List? png; try { - png = await widget.codec.decodePreview(payload); + png = await widget.codec.decodePreview(payload!); } catch (_) { png = null; } if (!mounted) return; setState(() { - _reconstructing = false; - _reconstruction = png; - if (png == null) { - _reconstructionUnavailable = true; - _showReconstruction = false; + version.decoding = false; + version.png = png; + if (png == null && !_versions.any((v) => v.png != null)) { + // Nothing has ever decoded here: treat the device as unable to + // preview and fall back to showing the original. + _previewUnavailable = true; + _showOriginal = true; } }); } - Future _encode() async { - final cached = _encoded; - if (cached != null) return cached; + void _select(int index) { setState(() { - _encoding = true; - _failed = false; + _selected = index; + _showOriginal = false; }); - try { - final bytes = await widget.codec.encode( - widget.imageBytes, - kImageSendRatePoint, - ); - if (!mounted) return bytes; - setState(() { - _encoded = bytes; - _encoding = false; - }); - return bytes; - } catch (_) { - if (!mounted) return null; - setState(() { - _encoding = false; - _failed = true; - }); - return null; - } } ImageSendRadio _radioParams(BuildContext context) { @@ -264,7 +292,7 @@ class _ImageSendPreviewSheetState extends State { } _RateEstimate _estimateFor(ImageSendRadio radio) { - final bytes = _encoded; + final bytes = _current?.payload; if (bytes != null) { final exact = _estimate(bytes.length, radio); return _RateEstimate( @@ -293,14 +321,11 @@ class _ImageSendPreviewSheetState extends State { ); } - Future _onSend() async { + void _onSend() { + final version = _current; + final payload = version?.payload; + if (version == null || payload == null) return; setState(() => _sending = true); - final payload = await _encode(); - if (!mounted) return; - if (payload == null) { - setState(() => _sending = false); - return; - } final radio = _radio ?? const ImageSendRadio(); final estimate = _estimate(payload.length, radio); Navigator.of(context).pop( @@ -311,6 +336,7 @@ class _ImageSendPreviewSheetState extends State { packetCount: estimate.chunkCount, airtime: estimate.totalAirtime, wallClock: estimate.pacedWallClock, + previewPng: version.png, ), ); } @@ -375,7 +401,7 @@ class _ImageSendPreviewSheetState extends State { ..._warnings(theme, colors, radio, selected), const SizedBox(height: 4), _parityTile(theme, colors), - if (_failed) ...[ + if (_current?.failed ?? false) ...[ const SizedBox(height: 12), _banner( colors: colors, @@ -408,6 +434,9 @@ class _ImageSendPreviewSheetState extends State { ), ); + /// The main preview: by default the selected version as recipients + /// will see it, with the original one tap away for comparison. + /// /// The preview is deliberately capped in height: the packet count and airtime /// below it are the reason this screen exists and must not be pushed off the /// first screenful by a full-width square. @@ -419,59 +448,96 @@ class _ImageSendPreviewSheetState extends State { ) { final l10n = context.l10n; final maxSide = MediaQuery.sizeOf(context).height * 0.28; - final reconstruction = _showReconstruction ? _reconstruction : null; + final version = _current; + final canPreview = ready && !_previewUnavailable; + final showReconstruction = canPreview && !_showOriginal && version != null; + final png = showReconstruction ? version.png : null; + final pending = + showReconstruction && + png == null && + (version.encoding || version.decoding); + final broken = showReconstruction && png == null && !pending; + + Widget content; + if (pending) { + content = Center( + child: Padding( + padding: const EdgeInsets.all(12), + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + const CircularProgressIndicator(), + const SizedBox(height: 12), + Text( + l10n.imageSend_preparingPreview, + textAlign: TextAlign.center, + style: theme.textTheme.bodyMedium?.copyWith( + color: colors.onSurfaceVariant, + ), + ), + ], + ), + ), + ); + } else if (broken) { + content = Center( + child: Icon( + Icons.broken_image_outlined, + size: 48, + color: colors.onSurfaceVariant, + ), + ); + } else { + content = Image.memory( + png ?? widget.imageBytes, + // The reconstruction is the stretched square; fill undoes the stretch + // into the recipient's shape. + fit: png != null ? BoxFit.fill : BoxFit.contain, + alignment: Alignment.center, + gaplessPlayback: true, + errorBuilder: (context, error, stack) => Center( + child: Icon( + Icons.broken_image_outlined, + size: 48, + color: colors.onSurfaceVariant, + ), + ), + ); + } + return Column( children: [ - if (ready) ...[ + if (canPreview) ...[ SegmentedButton( segments: [ ButtonSegment( value: false, - label: MeshSegmentLabel(l10n.imageSend_viewOriginal), + label: MeshSegmentLabel(l10n.imageSend_viewReconstruction), ), ButtonSegment( value: true, - label: MeshSegmentLabel(l10n.imageSend_viewReconstruction), - enabled: _encoded != null && !_reconstructionUnavailable, + label: MeshSegmentLabel(l10n.imageSend_viewOriginal), ), ], - selected: {_showReconstruction}, - onSelectionChanged: (s) => _selectView(s.first), + selected: {_showOriginal}, + onSelectionChanged: (s) => setState(() => _showOriginal = s.first), ), const SizedBox(height: 12), ], ConstrainedBox( constraints: BoxConstraints(maxHeight: maxSide, maxWidth: maxSide), child: ClipRRect( - borderRadius: BorderRadius.circular(12), + borderRadius: BorderRadius.circular(MeshRadii.md), child: AspectRatio( - aspectRatio: reconstruction != null ? _recipientAspect : _aspect, - child: Container( + aspectRatio: showReconstruction ? _recipientAspect : _aspect, + child: ColoredBox( color: colors.surfaceContainerHighest, - child: _showReconstruction && _reconstructing - ? const Center(child: CircularProgressIndicator()) - : Image.memory( - reconstruction ?? widget.imageBytes, - // The reconstruction is the stretched square; fill - // undoes the stretch into the recipient's shape. - fit: reconstruction != null - ? BoxFit.fill - : BoxFit.contain, - alignment: Alignment.center, - gaplessPlayback: true, - errorBuilder: (context, error, stack) => Center( - child: Icon( - Icons.broken_image_outlined, - size: 48, - color: colors.onSurfaceVariant, - ), - ), - ), + child: content, ), ), ), ), - if (_reconstructionUnavailable) ...[ + if (_previewUnavailable) ...[ const SizedBox(height: 8), Text( l10n.imageSend_reconstructionUnavailable, @@ -481,6 +547,10 @@ class _ImageSendPreviewSheetState extends State { ), ), ], + if (canPreview && _versions.isNotEmpty) ...[ + const SizedBox(height: 16), + _versionsRow(theme, colors), + ], const SizedBox(height: 8), Text( l10n.imageSend_lossyNote( @@ -496,6 +566,152 @@ class _ImageSendPreviewSheetState extends State { ); } + /// The versions made so far, plus "Try another version". + Widget _versionsRow(ThemeData theme, ColorScheme colors) { + final l10n = context.l10n; + return Column( + children: [ + Wrap( + alignment: WrapAlignment.center, + spacing: 8, + runSpacing: 8, + children: [ + for (var i = 0; i < _versions.length; i++) + _versionTile(theme, colors, i), + ], + ), + if (_versions.length < kImageSendMaxVersions) ...[ + const SizedBox(height: 12), + OutlinedButton.icon( + onPressed: (_working || _sending) ? null : _addVersion, + icon: const Icon(Icons.refresh), + label: Text(l10n.imageSend_tryAnother), + ), + ], + const SizedBox(height: 8), + Text( + l10n.imageSend_versionsHint, + textAlign: TextAlign.center, + style: theme.textTheme.bodySmall?.copyWith( + color: colors.onSurfaceVariant, + ), + ), + ], + ); + } + + /// One selectable version: its reconstruction, "Version n" and its packet + /// count. Selection shows as a thick primary border plus a check badge, so + /// it never relies on colour alone. + Widget _versionTile(ThemeData theme, ColorScheme colors, int index) { + final l10n = context.l10n; + final version = _versions[index]; + final selected = index == _selected; + final label = l10n.imageSend_versionLabel(index + 1); + const maxSide = 80.0; + final aspect = _recipientAspect; + final width = aspect >= 1 ? maxSide : maxSide * aspect; + final height = aspect >= 1 ? maxSide / aspect : maxSide; + final payload = version.payload; + final packets = payload == null + ? null + : _estimate( + payload.length, + _radio ?? const ImageSendRadio(), + ).chunkCount; + + final Widget thumb; + if (version.png != null) { + thumb = Image.memory( + version.png!, + fit: BoxFit.fill, + gaplessPlayback: true, + ); + } else if (version.encoding || version.decoding) { + thumb = const Center( + child: SizedBox( + width: 20, + height: 20, + child: CircularProgressIndicator(strokeWidth: 2), + ), + ); + } else { + thumb = Icon(Icons.broken_image_outlined, color: colors.onSurfaceVariant); + } + + return Semantics( + button: true, + selected: selected, + label: label, + excludeSemantics: true, + child: InkWell( + borderRadius: BorderRadius.circular(MeshRadii.sm), + onTap: _sending ? null : () => _select(index), + child: Padding( + padding: const EdgeInsets.all(4), + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + Stack( + children: [ + Container( + width: width.clamp(48.0, maxSide), + height: height.clamp(48.0, maxSide), + clipBehavior: Clip.antiAlias, + decoration: BoxDecoration( + color: colors.surfaceContainerHighest, + borderRadius: BorderRadius.circular(MeshRadii.sm), + border: Border.all( + color: selected + ? colors.primary + : colors.outlineVariant, + width: selected ? 3 : 1, + ), + ), + child: thumb, + ), + if (selected) + Positioned( + top: 4, + right: 4, + child: Container( + padding: const EdgeInsets.all(2), + decoration: BoxDecoration( + color: colors.primary, + shape: BoxShape.circle, + ), + child: Icon( + Icons.check, + size: 16, + color: colors.onPrimary, + ), + ), + ), + ], + ), + const SizedBox(height: 4), + Text( + label, + style: theme.textTheme.bodySmall?.copyWith( + color: selected ? colors.onSurface : colors.onSurfaceVariant, + fontWeight: selected ? FontWeight.w600 : null, + ), + ), + if (packets != null) + Text( + l10n.imageSend_versionPackets(packets), + style: MeshTheme.mono( + fontSize: 12, + color: colors.onSurfaceVariant, + ), + ), + ], + ), + ), + ), + ); + } + Widget _sizeRow(ThemeData theme, ColorScheme colors, _RateEstimate estimate) { return Row( mainAxisAlignment: MainAxisAlignment.center, @@ -794,7 +1010,8 @@ class _ImageSendPreviewSheetState extends State { Widget _actions(ThemeData theme, bool ready) { final localizations = context.l10n; - final busy = _sending || _encoding; + final busy = _sending || (_current?.encoding ?? false); + final canSend = ready && !_sending && _current?.payload != null; return Padding( padding: const EdgeInsets.fromLTRB(16, 8, 16, 16), child: Row( @@ -810,7 +1027,7 @@ class _ImageSendPreviewSheetState extends State { const SizedBox(width: 12), Expanded( child: FilledButton.icon( - onPressed: (!ready || busy) ? null : _onSend, + onPressed: canSend ? _onSend : null, icon: busy ? const SizedBox( width: 16, diff --git a/lib/widgets/received_image_message.dart b/lib/widgets/received_image_message.dart index 6c4debdf..7fc4d55b 100644 --- a/lib/widgets/received_image_message.dart +++ b/lib/widgets/received_image_message.dart @@ -19,10 +19,13 @@ /// /// Both are private to this file, are not parameterised, and are reused /// verbatim by the full-screen viewer this widget opens on tap, so there is no -/// route in the app that can show the pixels without them. The only case in -/// which they are absent is [ReceivedImageEntry.isOutgoing], where the PNG is -/// the sender's own 512x512 crop and the label would be a falsehood — that is -/// read from the store's sidecar, never from a constructor argument. +/// route in the app that can show the pixels without them. They are absent +/// only when [ReceivedImageEntry.synthesized] is false: the sender's own image +/// on a device that could not decode, where the PNG is their stretched +/// original and the label would be a falsehood. The sender's own image is +/// otherwise the decoded version they picked, so it is labelled like any +/// other. That is read from the store's sidecar, never from a constructor +/// argument. /// /// The widget deliberately exposes no share/save action: an exported raster /// would have to have the caption composited into it first, and that belongs diff --git a/test/connector/image_echo_test.dart b/test/connector/image_echo_test.dart new file mode 100644 index 00000000..49531f88 --- /dev/null +++ b/test/connector/image_echo_test.dart @@ -0,0 +1,141 @@ +import 'dart:typed_data'; + +import 'package:crypto/crypto.dart' as crypto; +import 'package:flutter_test/flutter_test.dart'; +import 'package:meshcore_open/connector/meshcore_connector.dart'; +import 'package:meshcore_open/connector/meshcore_protocol.dart'; +import 'package:meshcore_open/models/channel.dart'; +import 'package:meshcore_open/review_mode/review_radio.dart'; +import 'package:meshcore_open/review_mode/review_script.dart'; +import 'package:meshcore_open/services/image_chunk_transport.dart'; +import 'package:meshcore_open/services/message_retry_service.dart'; +import 'package:meshcore_open/services/path_history_service.dart'; +import 'package:meshcore_open/services/storage_service.dart'; +import 'package:meshcore_open/storage/prefs_manager.dart'; +import 'package:meshcore_open/widgets/image_send_codec_binding.dart'; +import 'package:pointycastle/export.dart'; +import 'package:shared_preferences/shared_preferences.dart'; + +Future _waitFor(bool Function() condition, {int seconds = 8}) async { + final end = DateTime.now().add(Duration(seconds: seconds)); + while (!condition()) { + if (DateTime.now().isAfter(end)) fail('Timed out waiting for condition'); + await Future.delayed(const Duration(milliseconds: 20)); + } +} + +Uint8List _psk(int slot) => + Channel.parsePskHex(ReviewScript.channels[slot].pskHex); + +/// A raw-log frame for a repeater re-flooding [blob] as GRP_DATA on [psk]'s +/// channel, encrypted the way the firmware's createGroupDatagram does. +Uint8List _logRxGroupData(Uint8List psk, Uint8List blob, {int hops = 1}) { + final plain = [ + dataTypeAeicImage & 0xFF, + dataTypeAeicImage >> 8, + blob.length, + ...blob, + ]; + final padded = Uint8List(((plain.length + 15) ~/ 16) * 16) + ..setRange(0, plain.length, plain); + final cipher = ECBBlockCipher(AESEngine())..init(true, KeyParameter(psk)); + final encrypted = Uint8List(padded.length); + for (var i = 0; i < padded.length; i += 16) { + cipher.processBlock(padded, i, encrypted, i); + } + final key32 = Uint8List(32)..setRange(0, 16, psk); + final mac = crypto.Hmac(crypto.sha256, key32).convert(encrypted).bytes; + final channelHash = crypto.sha256.convert(psk).bytes[0]; + return Uint8List.fromList([ + pushCodeLogRxData, + 40, // snr + 0xA0, // rssi + 0x19, // flood route, GRP_DATA payload + hops, // path length: one 1-byte hash per hop + for (var i = 0; i < hops; i++) 0x42 + i, + channelHash, + mac[0], + mac[1], + ...encrypted, + ]); +} + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + + late MeshCoreConnector connector; + + // One connector for the file: the notification service it starts accepts a + // single listener per process. + setUpAll(() async { + SharedPreferences.setMockInitialValues({}); + PrefsManager.reset(); + await PrefsManager.initialize(); + connector = MeshCoreConnector(); + final retryService = MessageRetryService(); + connector.initialize( + retryService: retryService, + pathHistoryService: PathHistoryService(StorageService()), + ); + final radio = ReviewRadio( + scheduledMessageInterval: const Duration(hours: 1), + ackDelay: const Duration(milliseconds: 50), + replyDelay: const Duration(hours: 1), + ); + connector.reviewRadioFactoryForTest = () => radio; + await connector.connectReview(); + await _waitFor(() => connector.channelMapConfirmed); + }); + + tearDownAll(() async { + await connector.disconnect(); + connector.dispose(); + }); + + Future sendImage({ + required int slot, + required int imgId, + }) async { + final chunkSet = buildImageChunks( + payload: Uint8List.fromList(List.generate(200, (i) => i & 0xFF)), + metadata: const ImageStreamMetadata(rate: ImageCodecRatePoint.standard), + senderPrefix: senderPrefixFromKey(connector.selfPublicKey)!, + imgId: imgId, + ); + expect( + await connector.sendImageChunks( + chunkSet.blobs, + channelIndex: slot, + interChunkDelay: Duration.zero, + ), + isTrue, + ); + return chunkSet; + } + + test('a repeater echo of our own chunk is counted for that chunk', () async { + final chunkSet = await sendImage(slot: 0, imgId: 77); + expect(chunkSet.blobs.length, 3, reason: '2 data chunks + parity'); + expect(connector.ownImageEchoes(0, 77), [0, 0, 0]); + + connector.handleFrameForTest(_logRxGroupData(_psk(0), chunkSet.blobs[1])); + connector.handleFrameForTest(_logRxGroupData(_psk(0), chunkSet.blobs[1])); + connector.handleFrameForTest(_logRxGroupData(_psk(0), chunkSet.blobs[2])); + + expect(connector.ownImageEchoes(0, 77), [0, 2, 1]); + }); + + test('chunks from someone else, or on another channel key, are not ' + 'counted', () async { + final chunkSet = await sendImage(slot: 0, imgId: 78); + + // Same image id, different sender. + final foreign = Uint8List.fromList(chunkSet.blobs[0]); + foreign[0] ^= 0xFF; + connector.handleFrameForTest(_logRxGroupData(_psk(0), foreign)); + // Our chunk, but encrypted for a different channel than it was sent on. + connector.handleFrameForTest(_logRxGroupData(_psk(1), chunkSet.blobs[0])); + + expect(connector.ownImageEchoes(0, 78), [0, 0, 0]); + }); +} diff --git a/test/services/received_image_store_test.dart b/test/services/received_image_store_test.dart index 8dd9f0c8..6a72569e 100644 --- a/test/services/received_image_store_test.dart +++ b/test/services/received_image_store_test.dart @@ -826,6 +826,73 @@ void main() { expect(h.decoder.calls, 0); }); + test('a decoded preview is labelled synthesized and keeps its byte count ' + 'and echoes', () async { + final h = _build(); + final entry = await h.store.registerOutgoing( + channelIndex: 1, + senderPrefix: kSelf, + imgId: 4, + previewPng: Uint8List.fromList(List.filled(64, 1)), + rate: AeicRatePoint.ft32, + chunkCount: 2, + synthesized: true, + payloadBytes: 171, + chunkEchoes: const [1, 0, 2], + ); + expect(entry.synthesized, isTrue); + expect(entry.bitstreamByteCount, 171); + expect(entry.bitstreamStored, isFalse); + expect(entry.minDataChunkEchoes, 0); + + final restored = ReceivedImageEntry.fromJson(entry.toJson()); + expect(restored.synthesized, isTrue); + expect(restored.chunkEchoes, [1, 0, 2]); + }); + + test( + 'setOutgoingEchoes updates the newest matching outgoing image', + () async { + final h = _build(); + final older = await h.store.registerOutgoing( + channelIndex: 1, + senderPrefix: kSelf, + imgId: 9, + previewPng: Uint8List.fromList(List.filled(64, 1)), + rate: AeicRatePoint.ft32, + chunkCount: 1, + ); + h.advance(const Duration(seconds: 5)); + final newer = await h.store.registerOutgoing( + channelIndex: 1, + senderPrefix: kSelf, + imgId: 9, + previewPng: Uint8List.fromList(List.filled(64, 2)), + rate: AeicRatePoint.ft32, + chunkCount: 1, + ); + + await h.store.setOutgoingEchoes( + channelIndex: 1, + senderPrefix: kSelf, + imgId: 9, + echoes: const [3, 1], + ); + expect(h.store.entryFor(newer.streamId)!.chunkEchoes, [3, 1]); + expect(h.store.entryFor(newer.streamId)!.minDataChunkEchoes, 3); + expect(h.store.entryFor(older.streamId)!.chunkEchoes, isNull); + + // Another channel's image with the same id is left alone. + await h.store.setOutgoingEchoes( + channelIndex: 2, + senderPrefix: kSelf, + imgId: 9, + echoes: const [7, 7], + ); + expect(h.store.entryFor(newer.streamId)!.chunkEchoes, [3, 1]); + }, + ); + test('spent outgoing entries count toward maxEntries', () async { final h = _build(maxEntries: 2, maxAge: const Duration(minutes: 10)); final ids = []; diff --git a/test/widgets/image_send_codec_binding_test.dart b/test/widgets/image_send_codec_binding_test.dart new file mode 100644 index 00000000..da5aca7f --- /dev/null +++ b/test/widgets/image_send_codec_binding_test.dart @@ -0,0 +1,57 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:meshcore_open/widgets/image_send_codec_binding.dart'; + +void main() { + group('rerollSourceRect', () { + const sizes = [ + (4000, 3000), + (3000, 4000), + (1920, 1080), + (512, 512), + (7, 3), + ]; + + test('variant 0 is the whole photo', () { + for (final (w, h) in sizes) { + final r = rerollSourceRect(w, h, 0); + expect(r.left, 0); + expect(r.top, 0); + expect(r.width, w); + expect(r.height, h); + } + }); + + test('every variant stays inside the photo and keeps its shape', () { + for (final (w, h) in sizes) { + for (var v = 0; v < kImageSendMaxVersions; v++) { + final r = rerollSourceRect(w, h, v); + expect(r.left, greaterThanOrEqualTo(0), reason: '$w x $h v$v'); + expect(r.top, greaterThanOrEqualTo(0), reason: '$w x $h v$v'); + expect(r.left + r.width, lessThanOrEqualTo(w + 1e-9)); + expect(r.top + r.height, lessThanOrEqualTo(h + 1e-9)); + // Same aspect, so the version keeps the photo's aspect code. + expect(r.width / r.height, closeTo(w / h, (w / h) * 0.01)); + // "The whole photo": at most a few percent trimmed. + expect(r.width, greaterThanOrEqualTo(w * 0.95)); + } + } + }); + + test('variants differ from each other and are deterministic', () { + final rects = [ + for (var v = 0; v < kImageSendMaxVersions; v++) + rerollSourceRect(4000, 3000, v), + ]; + expect(rects.toSet().length, kImageSendMaxVersions); + expect(rerollSourceRect(4000, 3000, 3), rects[3]); + }); + + test('rejects a variant outside the table', () { + expect(() => rerollSourceRect(10, 10, -1), throwsRangeError); + expect( + () => rerollSourceRect(10, 10, kImageSendMaxVersions), + throwsRangeError, + ); + }); + }); +} diff --git a/test/widgets/image_send_preview_sheet_test.dart b/test/widgets/image_send_preview_sheet_test.dart index 749690b4..23271e3e 100644 --- a/test/widgets/image_send_preview_sheet_test.dart +++ b/test/widgets/image_send_preview_sheet_test.dart @@ -336,4 +336,145 @@ void main() { final l10n = await AppLocalizations.delegate.load(const Locale('en')); expect(_texts(tester), contains(l10n.imageSend_codecUnavailable)); }); + + group('versions', () { + final previewCodec = FakeImageSendCodec( + latency: Duration.zero, + previewPng: _onePixelPng, + ); + + Future tapText(WidgetTester tester, String text) async { + final finder = find.text(text); + await tester.tap(finder); + await tester.pumpAndSettle(); + } + + Future send( + WidgetTester tester, + ImageSendPreviewResult? Function() result, + ) async { + await tester.tap(find.byType(FilledButton)); + await tester.pumpAndSettle(); + return result(); + } + + testWidgets('opens on what recipients see, as Version 1', (tester) async { + _tallScreen(tester); + await _openSheet(tester, radio: _knownRadio, codec: previewCodec); + + final l10n = await AppLocalizations.delegate.load(const Locale('en')); + final texts = _texts(tester); + expect(texts, contains(l10n.imageSend_versionLabel(1))); + expect(texts, contains(l10n.imageSend_tryAnother)); + expect(texts, isNot(contains(l10n.imageSend_reconstructionUnavailable))); + final segments = tester.widget>( + find.byType(SegmentedButton), + ); + expect(segments.selected, { + false, + }, reason: 'the reconstruction, not the original, is the default view'); + }); + + testWidgets('trying another version selects it and sends its payload and ' + 'reconstruction', (tester) async { + _tallScreen(tester); + ImageSendPreviewResult? result; + await tester.pumpWidget(_host(previewCodec, (r) => result = r)); + await tester.tap(find.text('open')); + await tester.pumpAndSettle(); + final l10n = await AppLocalizations.delegate.load(const Locale('en')); + + await tapText(tester, l10n.imageSend_tryAnother); + expect(_texts(tester), contains(l10n.imageSend_versionLabel(2))); + + final sent = await send(tester, () => result); + expect( + sent!.payload.length, + FakeImageSendCodec.payloadBytesFor(kImageSendRatePoint, 1), + ); + expect(sent.previewPng, isNotNull); + }); + + testWidgets('going back to Version 1 sends Version 1', (tester) async { + _tallScreen(tester); + ImageSendPreviewResult? result; + await tester.pumpWidget(_host(previewCodec, (r) => result = r)); + await tester.tap(find.text('open')); + await tester.pumpAndSettle(); + final l10n = await AppLocalizations.delegate.load(const Locale('en')); + + await tapText(tester, l10n.imageSend_tryAnother); + await tapText(tester, l10n.imageSend_versionLabel(1)); + + final sent = await send(tester, () => result); + expect( + sent!.payload.length, + FakeImageSendCodec.payloadBytesFor(kImageSendRatePoint, 0), + ); + }); + + testWidgets('stops offering versions at the maximum', (tester) async { + _tallScreen(tester); + await _openSheet(tester, radio: _knownRadio, codec: previewCodec); + final l10n = await AppLocalizations.delegate.load(const Locale('en')); + + for (var i = 1; i < kImageSendMaxVersions; i++) { + await tapText(tester, l10n.imageSend_tryAnother); + } + final texts = _texts(tester); + expect( + texts, + contains(l10n.imageSend_versionLabel(kImageSendMaxVersions)), + ); + expect(texts, isNot(contains(l10n.imageSend_tryAnother))); + }); + + testWidgets('a device that cannot decode shows the original and no ' + 'versions', (tester) async { + _tallScreen(tester); + await _openSheet(tester, radio: _knownRadio); + + final l10n = await AppLocalizations.delegate.load(const Locale('en')); + final texts = _texts(tester); + expect(texts, contains(l10n.imageSend_reconstructionUnavailable)); + expect(texts, isNot(contains(l10n.imageSend_versionLabel(1)))); + expect(texts, isNot(contains(l10n.imageSend_tryAnother))); + expect(find.byType(SegmentedButton), findsNothing); + }); + }); } + +/// Room for the whole sheet, so version tiles and "Try another version" are +/// on screen without scrolling the draggable sheet. +void _tallScreen(WidgetTester tester) { + tester.view.physicalSize = const Size(1000, 2400); + tester.view.devicePixelRatio = 1.0; + addTearDown(tester.view.reset); +} + +/// A host app whose button opens the sheet and reports what it returned. +Widget _host( + ImageSendCodec codec, + void Function(ImageSendPreviewResult?) onResult, +) => MaterialApp( + localizationsDelegates: AppLocalizations.localizationsDelegates, + supportedLocales: AppLocalizations.supportedLocales, + home: Builder( + builder: (context) => Scaffold( + body: Center( + child: ElevatedButton( + onPressed: () async => onResult( + await showImageSendPreviewSheet( + context: context, + imageBytes: _onePixelPng, + originalFileBytes: 1024, + codec: codec, + radio: _knownRadio, + ), + ), + child: const Text('open'), + ), + ), + ), + ), +); diff --git a/untranslated.json b/untranslated.json index 9e26dfee..3119859d 100644 --- a/untranslated.json +++ b/untranslated.json @@ -1 +1,191 @@ -{} \ No newline at end of file +{ + "bg": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "cs": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "de": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "es": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "fr": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "hu": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "it": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "ja": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "ko": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "nl": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "pl": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "pt": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "pt_PT": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "ru": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "sk": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "sl": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "sv": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "uk": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "zh": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ] +}