Merge branch 'dev' into ez_bookmark

This commit is contained in:
zjs81 authored and GitHub committed 2026-11-15 15:16:18 -07:00
commit 89071473a5
49 files changed
+3932 -149

No files matched your search

+1
View File
@@ -118,6 +118,7 @@ Icon sizes in use: 24 for tile and banner icons, 20 for menu icons and small act
| Settings navigation row | `MeshNavTile` | `onTap: null` dims it to 50 % and reports it as disabled; use `subtitle` to say why |
| Settings switch | `SwitchListTile` inside `MeshSettingsTileTheme` | Wrap the settings screen body once in `MeshSettingsTileTheme` |
| Radio-style choice in a sheet | `MeshSheetOption` | Used by the language picker and map time filter |
| Choosing between images | Selectable thumbnail tiles in a centred `Wrap` | Image send versions (`_versionTile` in `image_send_preview_sheet.dart`). `InkWell` with `MeshRadii.sm`; selected tile gets a 3px `primary` border **and** a `primary` check badge (never colour alone); caption under the tile in `bodySmall`, w600 when selected; figures such as packet counts in `MeshTheme.mono` 12; wrap the tile in `Semantics(button: true, selected:, label:)` |
| Popup menu row | **always** `PopupMenuItem(child: MeshMenuItemRow(icon:, label:))` | Label wraps to 2 lines then ellipsizes; `leading:` for `ThemedIcon`; no icon keeps the 20px slot |
| Segmented button label | **always** `ButtonSegment(label: MeshSegmentLabel(text))` | Shrinks to 12 (before system scale), then wraps at word boundaries onto 2 centred lines; never breaks a word |
| Status | `StatusChip` | Mono label, coloured dot or icon |
+8 -3
View File
@@ -15,9 +15,14 @@ Downloads use the configured Hugging Face model bundle, support resume, and veri
1. Open channel chat and tap the image button beside the composer.
2. Pick a photo from your gallery.
3. Wait for encoding. Inspect the source preview and, if available, the reconstructed preview.
4. Review transmitted bytes, packet count, and estimated send duration. Estimates use the current radio settings and are unavailable if those settings are unknown.
5. Choose whether to include a recovery packet, then send. The chat shows per-packet progress.
3. Wait for the preview. The sheet shows **What recipients see**: the photo encoded and then decoded on your device, which is the picture recipients reconstruct. Switch to **Original** to compare.
4. If the reconstruction got something wrong, tap **Try another version**. Each version encodes the whole photo with a tiny, invisible change of framing, and the model reconstructs each one a little differently. Tap the version you want to send; the packet count and time on air follow your choice. Up to six versions can be made, one at a time.
5. Review transmitted bytes, packet count, and estimated send duration. Estimates use the current radio settings and are unavailable if those settings are unknown.
6. Choose whether to include a recovery packet, then send. The chat shows per-packet progress, and your own copy of the image shows the version you chose.
After sending, your image shows how many times repeaters were heard passing it on, like a channel message (Packet info lists each packet). If repeaters passed on some packets but not others, the missing ones are sent once more automatically.
On a device that cannot decode, the sheet shows the original photo with a note instead, and versions are not offered. Each new version costs one encode and one decode, so it can take a few seconds on older phones.
The current build ships one quality/model setting: **ft32**, with a 512×512 codec reconstruction and metadata used to restore a supported aspect ratio. Reserved rate/resolution codes are not additional selectable quality options.
+189 -2
View File
@@ -30,11 +30,14 @@ import '../services/linux_ble_pairing_service_stub.dart'
if (dart.library.io) '../services/linux_ble_pairing_service.dart';
import '../services/image_chunk_transport.dart'
show
ImageChunkHeader,
ImageChunkOutcome,
ImageChunkTransport,
buildSendChannelDataFrame,
dataTypeAeicImage,
kImageReassemblyTtl,
outPathUnknown,
parseImageChunkHeader,
respCodeChannelDataRecv,
senderPrefixFromKey;
import '../services/image_codec_service.dart';
@@ -2196,6 +2199,8 @@ class MeshCoreConnector extends ChangeNotifier {
notifyListeners();
}
Future<void> reloadChannelOrder() => _loadChannelOrder();
/// History is stored by channel ID, so it may carry the slot the channel
/// sat in when it was saved; rebind it to the slot it is shown in now.
List<ChannelMessage> _bindToSlot(List<ChannelMessage> messages, int slot) => [
@@ -5777,7 +5782,9 @@ class MeshCoreConnector extends ChangeNotifier {
/// queued a packet to the radio. It says nothing about transmission or
/// delivery — GRP_DATA is unacknowledged flood traffic, there is no
/// PUSH_CODE_SEND_CONFIRMED for it, and a chunk may still be lost on air.
/// That is what the XOR parity chunk is for.
/// That is what the XOR parity chunk is for, and why repeater echoes are
/// counted afterwards ([ownImageEchoes]). With [repairUnheardChunks], chunks
/// no repeater echoed are resent once (see [_repairUnheardImageChunks]).
///
/// Returns false when the transport is unavailable (disconnected or firmware
/// too old); throws whatever [_sendFrameAndWaitForCommandAck] throws, i.e.
@@ -5790,9 +5797,47 @@ class MeshCoreConnector extends ChangeNotifier {
required int channelIndex,
required Duration interChunkDelay,
void Function(int sent, int total)? onProgress,
bool repairUnheardChunks = true,
}) async {
if (blobs.isEmpty) return false;
if (!supportsChannelData) return false;
final header = parseImageChunkHeader(blobs.first);
final key = header == null ? null : (channelIndex, header.imgId);
if (key != null && repairUnheardChunks) {
// Index == chunk index: data chunks first, the parity chunk (index ==
// total) last, so blobs.length slots cover every chunk we send.
_ownImageEchoes.remove(key);
_ownImageEchoes[key] = List<int>.filled(blobs.length, 0);
while (_ownImageEchoes.length > _maxTrackedOwnImages) {
_ownImageEchoes.remove(_ownImageEchoes.keys.first);
}
}
final startedAt = DateTime.now();
final sentAll = await _transmitImageBlobs(
blobs,
channelIndex: channelIndex,
interChunkDelay: interChunkDelay,
onProgress: onProgress,
);
if (sentAll && key != null && repairUnheardChunks) {
unawaited(
_repairUnheardImageChunks(
key,
blobs,
interChunkDelay: interChunkDelay,
startedAt: startedAt,
),
);
}
return sentAll;
}
Future<bool> _transmitImageBlobs(
List<Uint8List> blobs, {
required int channelIndex,
required Duration interChunkDelay,
void Function(int sent, int total)? onProgress,
}) async {
// `return` inside the scoped callback only exits the callback, so a
// disconnect mid-send used to fall through to `return true` below and the
// caller would record a complete outgoing image and tell the user "sent"
@@ -5823,6 +5868,142 @@ class MeshCoreConnector extends ChangeNotifier {
return sentAll;
}
// ---- repeater echoes of our own image chunks -------------------------------
//
// GRP_DATA is never acknowledged. What we can observe is a repeater
// re-flooding one of our chunks: the radio reports it in the raw RX log, the
// same way repeats of channel text are counted. A chunk nobody echoed while
// other chunks of the same image were echoed was most likely lost, so it is
// sent once more.
/// How long to listen for echoes after the last chunk before repairing.
static const Duration _imageEchoWait = Duration(seconds: 15);
/// Receivers drop a partial image [kImageReassemblyTtl] after its first
/// chunk. A repair that cannot land well inside that window is skipped.
static const Duration _imageRepairDeadline = Duration(seconds: 45);
static const int _maxTrackedOwnImages = 16;
final Map<(int, int), List<int>> _ownImageEchoes = {};
final Set<(int, int)> _imageRepairsInFlight = {};
/// Repeater echoes heard for each chunk of one of our own images sent this
/// session (data chunks, then the parity chunk), or null if not tracked.
List<int>? ownImageEchoes(int channelIndex, int imgId) {
final echoes = _ownImageEchoes[(channelIndex, imgId)];
return echoes == null ? null : List<int>.unmodifiable(echoes);
}
/// Non-null while unheard chunks of that image are being resent.
({int resends, int maxResends})? imageResendProgress(
int channelIndex,
int imgId,
) => _imageRepairsInFlight.contains((channelIndex, imgId))
? (resends: 1, maxResends: 1)
: null;
/// A GRP_DATA packet from the raw RX log. Other people's images arrive
/// already decrypted through RESP_CODE_CHANNEL_DATA_RECV; here we only look
/// for repeaters re-flooding one of our own chunks.
void _handleImageEcho(_RawPacket packet) {
final self = senderPrefixFromKey(_selfPublicKey);
if (self == null || packet.payload.length < 2) return;
final channelHash = packet.payload[0];
final encrypted = Uint8List.fromList(packet.payload.sublist(1));
final channels = _channels.isNotEmpty ? _channels : _cachedChannels;
for (final channel in channels) {
if (channel.isEmpty) continue;
if (_computeChannelHash(channel.psk) != channelHash) continue;
// Null unless the MAC matches this channel's key.
final plain = _decryptPayload(channel.psk, encrypted);
if (plain == null || plain.length < 3) continue;
// Plaintext: data_type (uint16 LE), data_len, blob (padded to 16).
final dataType = plain[0] | (plain[1] << 8);
final length = plain[2];
if (dataType != dataTypeAeicImage || plain.length < 3 + length) return;
final header = parseImageChunkHeader(
Uint8List.sublistView(plain, 3, 3 + length),
);
if (header == null || header.senderPrefix != self) return;
final channelId = channel.storageId;
final slot = _channelMapConfirmed && channelId != null
? _slotForChannelId(channelId)
: channel.index;
if (slot == null) return;
_recordOwnImageEcho(slot, header);
return;
}
}
void _recordOwnImageEcho(int slot, ImageChunkHeader header) {
final echoes = _ownImageEchoes[(slot, header.imgId)];
if (echoes == null || header.index >= echoes.length) return;
echoes[header.index]++;
unawaited(
_receivedImageStore?.setOutgoingEchoes(
channelIndex: slot,
senderPrefix: header.senderPrefix,
imgId: header.imgId,
echoes: List<int>.of(echoes),
),
);
notifyListeners();
}
/// One repair round: resend the chunks no repeater was heard echoing, but
/// only when some other chunk of the image was echoed. With no echo at all
/// we cannot tell a lost image from a sender with no repeater in earshot,
/// and resending would double every such user's airtime for nothing.
///
/// The resent chunk is byte-identical, so a repeater that did forward it
/// (and we just missed the echo) drops the copy as already seen. Receivers
/// missing it merge it into the image they are still holding.
Future<void> _repairUnheardImageChunks(
(int, int) key,
List<Uint8List> blobs, {
required Duration interChunkDelay,
required DateTime startedAt,
}) async {
await Future<void>.delayed(_imageEchoWait);
final echoes = _ownImageEchoes[key];
if (echoes == null || echoes.length != blobs.length || !isConnected) {
return;
}
final unheard = <Uint8List>[
for (var i = 0; i < blobs.length; i++)
if (echoes[i] == 0) blobs[i],
];
if (unheard.isEmpty || unheard.length == blobs.length) return;
if (DateTime.now().difference(startedAt) > _imageRepairDeadline) {
_appDebugLogService?.info(
'Image ${key.$2}: ${unheard.length} chunk(s) unheard, too late to '
'resend before receivers give up',
tag: 'Image resend',
);
return;
}
_appDebugLogService?.info(
'Image ${key.$2}: resending ${unheard.length} of ${blobs.length} '
'chunk(s) no repeater echoed',
tag: 'Image resend',
);
_imageRepairsInFlight.add(key);
notifyListeners();
try {
await _transmitImageBlobs(
unheard,
channelIndex: key.$1,
interChunkDelay: interChunkDelay,
);
} catch (e) {
_appDebugLogService?.warn('Image resend failed: $e', tag: 'Image resend');
} finally {
_imageRepairsInFlight.remove(key);
notifyListeners();
}
}
Future<void> _runScopedChannelSend(
Future<void> Function() action, {
required String region,
@@ -8459,7 +8640,12 @@ class MeshCoreConnector extends ChangeNotifier {
final raw = reader.readRemainingBytes();
final packet = _parseRawPacket(raw);
if (packet == null || packet.payloadType != _payloadTypeGroupText) return;
if (packet == null) return;
if (packet.payloadType == _payloadTypeGroupData) {
_handleImageEcho(packet);
return;
}
if (packet.payloadType != _payloadTypeGroupText) return;
final payload = BufferReader(packet.payload);
final channelHash = payload.readByte();
@@ -10541,6 +10727,7 @@ const int _routeFlood = 0x01;
const int _routeTransportDirect = 0x03;
const int _payloadTypeGroupText = 0x05;
const int _payloadTypeGroupData = 0x06;
const int _cipherMacSize = 2;
bool _isPathLenValidForMode(
+38
View File
@@ -3509,6 +3509,44 @@
"imageSend_viewOriginal": "Original",
"imageSend_viewReconstruction": "What recipients see",
"imageSend_reconstructionUnavailable": "This device can't preview the reconstruction.",
"receivedImage_packetEchoes": "Packet {number}: {count, plural, =0{no repeater heard} =1{repeated once} other{repeated {count} times}}",
"@receivedImage_packetEchoes": {
"placeholders": {
"number": {
"type": "int"
},
"count": {
"type": "int"
}
}
},
"receivedImage_recoveryPacketEchoes": "Recovery packet: {count, plural, =0{no repeater heard} =1{repeated once} other{repeated {count} times}}",
"@receivedImage_recoveryPacketEchoes": {
"placeholders": {
"count": {
"type": "int"
}
}
},
"imageSend_preparingPreview": "Preparing preview…",
"imageSend_tryAnother": "Try another version",
"imageSend_versionsHint": "Each version is reconstructed a little differently. Pick the one that looks most like your photo.",
"imageSend_versionLabel": "Version {number}",
"@imageSend_versionLabel": {
"placeholders": {
"number": {
"type": "int"
}
}
},
"imageSend_versionPackets": "{count, plural, =1{1 packet} other{{count} packets}}",
"@imageSend_versionPackets": {
"placeholders": {
"count": {
"type": "int"
}
}
},
"imageSend_modelNotDownloaded": "The image model isn't downloaded yet. Download it in Settings to send images.",
"imageSend_originalSize": "Original",
"imageSend_onAirSize": "On air",
+42
View File
@@ -10643,6 +10643,48 @@ abstract class AppLocalizations {
/// **'This device can\'t preview the reconstruction.'**
String get imageSend_reconstructionUnavailable;
/// No description provided for @receivedImage_packetEchoes.
///
/// In en, this message translates to:
/// **'Packet {number}: {count, plural, =0{no repeater heard} =1{repeated once} other{repeated {count} times}}'**
String receivedImage_packetEchoes(int number, int count);
/// No description provided for @receivedImage_recoveryPacketEchoes.
///
/// In en, this message translates to:
/// **'Recovery packet: {count, plural, =0{no repeater heard} =1{repeated once} other{repeated {count} times}}'**
String receivedImage_recoveryPacketEchoes(int count);
/// No description provided for @imageSend_preparingPreview.
///
/// In en, this message translates to:
/// **'Preparing preview…'**
String get imageSend_preparingPreview;
/// No description provided for @imageSend_tryAnother.
///
/// In en, this message translates to:
/// **'Try another version'**
String get imageSend_tryAnother;
/// No description provided for @imageSend_versionsHint.
///
/// In en, this message translates to:
/// **'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'**
String get imageSend_versionsHint;
/// No description provided for @imageSend_versionLabel.
///
/// In en, this message translates to:
/// **'Version {number}'**
String imageSend_versionLabel(int number);
/// No description provided for @imageSend_versionPackets.
///
/// In en, this message translates to:
/// **'{count, plural, =1{1 packet} other{{count} packets}}'**
String imageSend_versionPackets(int count);
/// No description provided for @imageSend_modelNotDownloaded.
///
/// In en, this message translates to:
+50
View File
@@ -6285,6 +6285,56 @@ class AppLocalizationsBg extends AppLocalizations {
String get imageSend_reconstructionUnavailable =>
'Това устройство не може да покаже реконструкцията.';
@override
String receivedImage_packetEchoes(int number, int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Packet $number: $_temp0';
}
@override
String receivedImage_recoveryPacketEchoes(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Recovery packet: $_temp0';
}
@override
String get imageSend_preparingPreview => 'Preparing preview…';
@override
String get imageSend_tryAnother => 'Try another version';
@override
String get imageSend_versionsHint =>
'Each version is reconstructed a little differently. Pick the one that looks most like your photo.';
@override
String imageSend_versionLabel(int number) {
return 'Version $number';
}
@override
String imageSend_versionPackets(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: '$count packets',
one: '1 packet',
);
return '$_temp0';
}
@override
String get imageSend_modelNotDownloaded =>
'Моделът за изображения още не е изтеглен. Изтеглете го в настройките, за да изпращате изображения.';
+50
View File
@@ -6271,6 +6271,56 @@ class AppLocalizationsCs extends AppLocalizations {
String get imageSend_reconstructionUnavailable =>
'Toto zařízení nemůže zobrazit náhled rekonstrukce.';
@override
String receivedImage_packetEchoes(int number, int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Packet $number: $_temp0';
}
@override
String receivedImage_recoveryPacketEchoes(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Recovery packet: $_temp0';
}
@override
String get imageSend_preparingPreview => 'Preparing preview…';
@override
String get imageSend_tryAnother => 'Try another version';
@override
String get imageSend_versionsHint =>
'Each version is reconstructed a little differently. Pick the one that looks most like your photo.';
@override
String imageSend_versionLabel(int number) {
return 'Version $number';
}
@override
String imageSend_versionPackets(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: '$count packets',
one: '1 packet',
);
return '$_temp0';
}
@override
String get imageSend_modelNotDownloaded =>
'Model obrázků ještě není stažen. Pro odesílání obrázků jej stáhněte v Nastavení.';
+50
View File
@@ -6303,6 +6303,56 @@ class AppLocalizationsDe extends AppLocalizations {
String get imageSend_reconstructionUnavailable =>
'Dieses Gerät kann die Rekonstruktion nicht in der Vorschau anzeigen.';
@override
String receivedImage_packetEchoes(int number, int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Packet $number: $_temp0';
}
@override
String receivedImage_recoveryPacketEchoes(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Recovery packet: $_temp0';
}
@override
String get imageSend_preparingPreview => 'Preparing preview…';
@override
String get imageSend_tryAnother => 'Try another version';
@override
String get imageSend_versionsHint =>
'Each version is reconstructed a little differently. Pick the one that looks most like your photo.';
@override
String imageSend_versionLabel(int number) {
return 'Version $number';
}
@override
String imageSend_versionPackets(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: '$count packets',
one: '1 packet',
);
return '$_temp0';
}
@override
String get imageSend_modelNotDownloaded =>
'Das Bildmodell wurde noch nicht heruntergeladen. Laden Sie es in den Einstellungen herunter, um Bilder zu senden.';
+50
View File
@@ -6199,6 +6199,56 @@ class AppLocalizationsEn extends AppLocalizations {
String get imageSend_reconstructionUnavailable =>
'This device can\'t preview the reconstruction.';
@override
String receivedImage_packetEchoes(int number, int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Packet $number: $_temp0';
}
@override
String receivedImage_recoveryPacketEchoes(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Recovery packet: $_temp0';
}
@override
String get imageSend_preparingPreview => 'Preparing preview…';
@override
String get imageSend_tryAnother => 'Try another version';
@override
String get imageSend_versionsHint =>
'Each version is reconstructed a little differently. Pick the one that looks most like your photo.';
@override
String imageSend_versionLabel(int number) {
return 'Version $number';
}
@override
String imageSend_versionPackets(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: '$count packets',
one: '1 packet',
);
return '$_temp0';
}
@override
String get imageSend_modelNotDownloaded =>
'The image model isn\'t downloaded yet. Download it in Settings to send images.';
+50
View File
@@ -6287,6 +6287,56 @@ class AppLocalizationsEs extends AppLocalizations {
String get imageSend_reconstructionUnavailable =>
'Este dispositivo no puede previsualizar la reconstrucción.';
@override
String receivedImage_packetEchoes(int number, int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Packet $number: $_temp0';
}
@override
String receivedImage_recoveryPacketEchoes(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Recovery packet: $_temp0';
}
@override
String get imageSend_preparingPreview => 'Preparing preview…';
@override
String get imageSend_tryAnother => 'Try another version';
@override
String get imageSend_versionsHint =>
'Each version is reconstructed a little differently. Pick the one that looks most like your photo.';
@override
String imageSend_versionLabel(int number) {
return 'Version $number';
}
@override
String imageSend_versionPackets(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: '$count packets',
one: '1 packet',
);
return '$_temp0';
}
@override
String get imageSend_modelNotDownloaded =>
'El modelo de imagen aún no se ha descargado. Descárgalo en Configuración para enviar imágenes.';
+50
View File
@@ -6310,6 +6310,56 @@ class AppLocalizationsFr extends AppLocalizations {
String get imageSend_reconstructionUnavailable =>
'Cet appareil ne peut pas prévisualiser la reconstitution.';
@override
String receivedImage_packetEchoes(int number, int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Packet $number: $_temp0';
}
@override
String receivedImage_recoveryPacketEchoes(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Recovery packet: $_temp0';
}
@override
String get imageSend_preparingPreview => 'Preparing preview…';
@override
String get imageSend_tryAnother => 'Try another version';
@override
String get imageSend_versionsHint =>
'Each version is reconstructed a little differently. Pick the one that looks most like your photo.';
@override
String imageSend_versionLabel(int number) {
return 'Version $number';
}
@override
String imageSend_versionPackets(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: '$count packets',
one: '1 packet',
);
return '$_temp0';
}
@override
String get imageSend_modelNotDownloaded =>
'Le modèle d\'image n\'est pas encore téléchargé. Téléchargez-le dans Paramètres pour envoyer des images.';
+50
View File
@@ -6284,6 +6284,56 @@ class AppLocalizationsHu extends AppLocalizations {
String get imageSend_reconstructionUnavailable =>
'Ezen az eszközön nem jeleníthető meg a rekonstrukció előnézete.';
@override
String receivedImage_packetEchoes(int number, int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Packet $number: $_temp0';
}
@override
String receivedImage_recoveryPacketEchoes(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Recovery packet: $_temp0';
}
@override
String get imageSend_preparingPreview => 'Preparing preview…';
@override
String get imageSend_tryAnother => 'Try another version';
@override
String get imageSend_versionsHint =>
'Each version is reconstructed a little differently. Pick the one that looks most like your photo.';
@override
String imageSend_versionLabel(int number) {
return 'Version $number';
}
@override
String imageSend_versionPackets(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: '$count packets',
one: '1 packet',
);
return '$_temp0';
}
@override
String get imageSend_modelNotDownloaded =>
'A képmodell még nincs letöltve. Töltse le a Beállításokban a képek küldéséhez.';
+50
View File
@@ -6292,6 +6292,56 @@ class AppLocalizationsIt extends AppLocalizations {
String get imageSend_reconstructionUnavailable =>
'Questo dispositivo non può mostrare un\'anteprima della ricostruzione.';
@override
String receivedImage_packetEchoes(int number, int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Packet $number: $_temp0';
}
@override
String receivedImage_recoveryPacketEchoes(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Recovery packet: $_temp0';
}
@override
String get imageSend_preparingPreview => 'Preparing preview…';
@override
String get imageSend_tryAnother => 'Try another version';
@override
String get imageSend_versionsHint =>
'Each version is reconstructed a little differently. Pick the one that looks most like your photo.';
@override
String imageSend_versionLabel(int number) {
return 'Version $number';
}
@override
String imageSend_versionPackets(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: '$count packets',
one: '1 packet',
);
return '$_temp0';
}
@override
String get imageSend_modelNotDownloaded =>
'Il modello di immagine non è ancora stato scaricato. Scaricalo nelle Impostazioni per inviare immagini.';
+50
View File
@@ -5984,6 +5984,56 @@ class AppLocalizationsJa extends AppLocalizations {
@override
String get imageSend_reconstructionUnavailable => 'このデバイスでは復元結果をプレビューできません。';
@override
String receivedImage_packetEchoes(int number, int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Packet $number: $_temp0';
}
@override
String receivedImage_recoveryPacketEchoes(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Recovery packet: $_temp0';
}
@override
String get imageSend_preparingPreview => 'Preparing preview…';
@override
String get imageSend_tryAnother => 'Try another version';
@override
String get imageSend_versionsHint =>
'Each version is reconstructed a little differently. Pick the one that looks most like your photo.';
@override
String imageSend_versionLabel(int number) {
return 'Version $number';
}
@override
String imageSend_versionPackets(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: '$count packets',
one: '1 packet',
);
return '$_temp0';
}
@override
String get imageSend_modelNotDownloaded =>
'画像モデルがまだダウンロードされていません。設定でダウンロードすると画像を送信できます。';
+50
View File
@@ -6000,6 +6000,56 @@ class AppLocalizationsKo extends AppLocalizations {
String get imageSend_reconstructionUnavailable =>
'이 장치에서는 복원 결과를 미리 볼 수 없습니다.';
@override
String receivedImage_packetEchoes(int number, int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Packet $number: $_temp0';
}
@override
String receivedImage_recoveryPacketEchoes(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Recovery packet: $_temp0';
}
@override
String get imageSend_preparingPreview => 'Preparing preview…';
@override
String get imageSend_tryAnother => 'Try another version';
@override
String get imageSend_versionsHint =>
'Each version is reconstructed a little differently. Pick the one that looks most like your photo.';
@override
String imageSend_versionLabel(int number) {
return 'Version $number';
}
@override
String imageSend_versionPackets(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: '$count packets',
one: '1 packet',
);
return '$_temp0';
}
@override
String get imageSend_modelNotDownloaded =>
'이미지 모델이 아직 다운로드되지 않았습니다. 설정에서 다운로드하면 이미지를 보낼 수 있습니다.';
+50
View File
@@ -6256,6 +6256,56 @@ class AppLocalizationsNl extends AppLocalizations {
String get imageSend_reconstructionUnavailable =>
'Dit apparaat kan geen voorbeeld van de reconstructie tonen.';
@override
String receivedImage_packetEchoes(int number, int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Packet $number: $_temp0';
}
@override
String receivedImage_recoveryPacketEchoes(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Recovery packet: $_temp0';
}
@override
String get imageSend_preparingPreview => 'Preparing preview…';
@override
String get imageSend_tryAnother => 'Try another version';
@override
String get imageSend_versionsHint =>
'Each version is reconstructed a little differently. Pick the one that looks most like your photo.';
@override
String imageSend_versionLabel(int number) {
return 'Version $number';
}
@override
String imageSend_versionPackets(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: '$count packets',
one: '1 packet',
);
return '$_temp0';
}
@override
String get imageSend_modelNotDownloaded =>
'Het afbeeldingsmodel is nog niet gedownload. Download het in Instellingen om afbeeldingen te verzenden.';
+50
View File
@@ -6330,6 +6330,56 @@ class AppLocalizationsPl extends AppLocalizations {
String get imageSend_reconstructionUnavailable =>
'To urządzenie nie może wyświetlić podglądu rekonstrukcji.';
@override
String receivedImage_packetEchoes(int number, int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Packet $number: $_temp0';
}
@override
String receivedImage_recoveryPacketEchoes(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Recovery packet: $_temp0';
}
@override
String get imageSend_preparingPreview => 'Preparing preview…';
@override
String get imageSend_tryAnother => 'Try another version';
@override
String get imageSend_versionsHint =>
'Each version is reconstructed a little differently. Pick the one that looks most like your photo.';
@override
String imageSend_versionLabel(int number) {
return 'Version $number';
}
@override
String imageSend_versionPackets(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: '$count packets',
one: '1 packet',
);
return '$_temp0';
}
@override
String get imageSend_modelNotDownloaded =>
'Model obrazowy nie jest jeszcze pobrany. Pobierz go w Ustawieniach, aby wysyłać obrazy.';
+50
View File
@@ -6281,6 +6281,56 @@ class AppLocalizationsPt extends AppLocalizations {
String get imageSend_reconstructionUnavailable =>
'Este dispositivo não pode pré-visualizar a reconstrução.';
@override
String receivedImage_packetEchoes(int number, int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Packet $number: $_temp0';
}
@override
String receivedImage_recoveryPacketEchoes(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Recovery packet: $_temp0';
}
@override
String get imageSend_preparingPreview => 'Preparing preview…';
@override
String get imageSend_tryAnother => 'Try another version';
@override
String get imageSend_versionsHint =>
'Each version is reconstructed a little differently. Pick the one that looks most like your photo.';
@override
String imageSend_versionLabel(int number) {
return 'Version $number';
}
@override
String imageSend_versionPackets(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: '$count packets',
one: '1 packet',
);
return '$_temp0';
}
@override
String get imageSend_modelNotDownloaded =>
'O modelo de imagem ainda não foi transferido. Transfira-o nas Configurações para enviar imagens.';
+50
View File
@@ -6306,6 +6306,56 @@ class AppLocalizationsRu extends AppLocalizations {
String get imageSend_reconstructionUnavailable =>
'Это устройство не может показать реконструкцию.';
@override
String receivedImage_packetEchoes(int number, int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Packet $number: $_temp0';
}
@override
String receivedImage_recoveryPacketEchoes(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Recovery packet: $_temp0';
}
@override
String get imageSend_preparingPreview => 'Preparing preview…';
@override
String get imageSend_tryAnother => 'Try another version';
@override
String get imageSend_versionsHint =>
'Each version is reconstructed a little differently. Pick the one that looks most like your photo.';
@override
String imageSend_versionLabel(int number) {
return 'Version $number';
}
@override
String imageSend_versionPackets(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: '$count packets',
one: '1 packet',
);
return '$_temp0';
}
@override
String get imageSend_modelNotDownloaded =>
'Модель изображения ещё не загружена. Загрузите её в настройках, чтобы отправлять изображения.';
+50
View File
@@ -6280,6 +6280,56 @@ class AppLocalizationsSk extends AppLocalizations {
String get imageSend_reconstructionUnavailable =>
'Toto zariadenie nedokáže zobraziť náhľad rekonštrukcie.';
@override
String receivedImage_packetEchoes(int number, int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Packet $number: $_temp0';
}
@override
String receivedImage_recoveryPacketEchoes(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Recovery packet: $_temp0';
}
@override
String get imageSend_preparingPreview => 'Preparing preview…';
@override
String get imageSend_tryAnother => 'Try another version';
@override
String get imageSend_versionsHint =>
'Each version is reconstructed a little differently. Pick the one that looks most like your photo.';
@override
String imageSend_versionLabel(int number) {
return 'Version $number';
}
@override
String imageSend_versionPackets(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: '$count packets',
one: '1 packet',
);
return '$_temp0';
}
@override
String get imageSend_modelNotDownloaded =>
'Model obrazu ešte nie je stiahnutý. Stiahnite ho v Nastaveniach, aby ste mohli posielať obrázky.';
+50
View File
@@ -6272,6 +6272,56 @@ class AppLocalizationsSl extends AppLocalizations {
String get imageSend_reconstructionUnavailable =>
'Ta naprava ne more prikazati predogleda rekonstrukcije.';
@override
String receivedImage_packetEchoes(int number, int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Packet $number: $_temp0';
}
@override
String receivedImage_recoveryPacketEchoes(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Recovery packet: $_temp0';
}
@override
String get imageSend_preparingPreview => 'Preparing preview…';
@override
String get imageSend_tryAnother => 'Try another version';
@override
String get imageSend_versionsHint =>
'Each version is reconstructed a little differently. Pick the one that looks most like your photo.';
@override
String imageSend_versionLabel(int number) {
return 'Version $number';
}
@override
String imageSend_versionPackets(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: '$count packets',
one: '1 packet',
);
return '$_temp0';
}
@override
String get imageSend_modelNotDownloaded =>
'Obrazni model še ni prenesen. Prenesite ga v Nastavitvah, da boste lahko pošiljali slike.';
+50
View File
@@ -6239,6 +6239,56 @@ class AppLocalizationsSv extends AppLocalizations {
String get imageSend_reconstructionUnavailable =>
'Enheten kan inte förhandsgranska rekonstruktionen.';
@override
String receivedImage_packetEchoes(int number, int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Packet $number: $_temp0';
}
@override
String receivedImage_recoveryPacketEchoes(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Recovery packet: $_temp0';
}
@override
String get imageSend_preparingPreview => 'Preparing preview…';
@override
String get imageSend_tryAnother => 'Try another version';
@override
String get imageSend_versionsHint =>
'Each version is reconstructed a little differently. Pick the one that looks most like your photo.';
@override
String imageSend_versionLabel(int number) {
return 'Version $number';
}
@override
String imageSend_versionPackets(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: '$count packets',
one: '1 packet',
);
return '$_temp0';
}
@override
String get imageSend_modelNotDownloaded =>
'Bildmodellen har ännu inte laddats ner. Ladda ner den i Inställningar för att skicka bilder.';
+50
View File
@@ -6312,6 +6312,56 @@ class AppLocalizationsUk extends AppLocalizations {
String get imageSend_reconstructionUnavailable =>
'Цей пристрій не може показати відтворення.';
@override
String receivedImage_packetEchoes(int number, int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Packet $number: $_temp0';
}
@override
String receivedImage_recoveryPacketEchoes(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Recovery packet: $_temp0';
}
@override
String get imageSend_preparingPreview => 'Preparing preview…';
@override
String get imageSend_tryAnother => 'Try another version';
@override
String get imageSend_versionsHint =>
'Each version is reconstructed a little differently. Pick the one that looks most like your photo.';
@override
String imageSend_versionLabel(int number) {
return 'Version $number';
}
@override
String imageSend_versionPackets(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: '$count packets',
one: '1 packet',
);
return '$_temp0';
}
@override
String get imageSend_modelNotDownloaded =>
'Модель зображення ще не завантажена. Завантажте її в налаштуваннях, щоб надсилати зображення.';
+50
View File
@@ -5847,6 +5847,56 @@ class AppLocalizationsZh extends AppLocalizations {
@override
String get imageSend_reconstructionUnavailable => '此设备无法预览重建结果。';
@override
String receivedImage_packetEchoes(int number, int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Packet $number: $_temp0';
}
@override
String receivedImage_recoveryPacketEchoes(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: 'repeated $count times',
one: 'repeated once',
zero: 'no repeater heard',
);
return 'Recovery packet: $_temp0';
}
@override
String get imageSend_preparingPreview => 'Preparing preview…';
@override
String get imageSend_tryAnother => 'Try another version';
@override
String get imageSend_versionsHint =>
'Each version is reconstructed a little differently. Pick the one that looks most like your photo.';
@override
String imageSend_versionLabel(int number) {
return 'Version $number';
}
@override
String imageSend_versionPackets(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: '$count packets',
one: '1 packet',
);
return '$_temp0';
}
@override
String get imageSend_modelNotDownloaded => '图像模型尚未下载。在设置中下载后才能发送图像。';
+2
View File
@@ -140,6 +140,8 @@ void main() async {
<Uint8List>[blob],
channelIndex: channelIndex,
interChunkDelay: Duration.zero,
// One blob at a time: echo tracking needs the whole chunk set.
repairUnheardChunks: false,
),
// Overwritten by onImageSenderPrefix as soon as SELF_INFO lands.
senderPrefix: 0,
+3
View File
@@ -16,6 +16,7 @@ import '../services/app_settings_service.dart';
import '../services/backup_service.dart';
import '../services/image_codec_service.dart';
import '../services/translation_service.dart';
import '../services/ui_view_state_service.dart';
import '../services/message_filter/message_filter_service.dart';
import '../utils/platform_info.dart';
import '../utils/share_origin.dart';
@@ -437,6 +438,7 @@ class _BackupRestoreScreenState extends State<BackupRestoreScreen> {
final l10n = context.l10n;
final connector = context.read<MeshCoreConnector>();
final appSettingsService = context.read<AppSettingsService>();
final uiViewStateService = context.read<UiViewStateService>();
final messageFilterService = context.read<MessageFilterService>();
final translationService = context.read<TranslationService>();
final imageCodecService = context.read<ImageCodecService>();
@@ -459,6 +461,7 @@ class _BackupRestoreScreenState extends State<BackupRestoreScreen> {
)) {
await appSettingsService.loadSettings();
if (_selectedItems.contains(BackupItem.appSettings)) {
await uiViewStateService.reload();
await _offerMissingModels(
appSettingsService,
translationService,
+62 -17
View File
@@ -1917,9 +1917,8 @@ class _ChannelChatScreenState extends State<ChannelChatScreen> {
final connector = context.read<MeshCoreConnector>();
// The picked bytes are kept past the preview on purpose: the sender's own
// bubble renders from them (see [_registerOutgoingImage]). The bitstream
// cannot serve — turning it back into pixels is a ~2.16 GiB, ~1 s decode
// of an image the sender is already holding.
// bubble falls back to them (see [_registerOutgoingImage]) when the sheet
// could not decode the chosen version on this device.
final XFile? picked;
try {
picked = await ImagePicker().pickImage(
@@ -2095,8 +2094,11 @@ class _ChannelChatScreenState extends State<ChannelChatScreen> {
/// GRP_DATA is not echoed back to the sender and an image is not a
/// [ChannelMessage], so without this the sender sees a "sent" snack bar and
/// an otherwise empty conversation. The entry lands in `decoded` directly —
/// an outgoing image is never queued for inference, so showing your own
/// photo costs no model memory.
/// an outgoing image is never queued for inference.
///
/// The bubble shows the version the user picked, decoded as recipients see
/// it ([ImageSendPreviewResult.previewPng]). Only when this device could not decode does it fall back
/// to the stretched original.
///
/// Best effort throughout: a failure here has already been preceded by a
/// successful transmission, so it must not turn into an error the user sees.
@@ -2112,7 +2114,8 @@ class _ChannelChatScreenState extends State<ChannelChatScreen> {
} on ProviderNotFoundException {
return; // screen test without the store
}
final previewPng = await _squarePreviewPng(sourceBytes);
final previewPng =
result.previewPng ?? await _squarePreviewPng(sourceBytes);
if (previewPng == null) return;
try {
await store.registerOutgoing(
@@ -2126,6 +2129,18 @@ class _ChannelChatScreenState extends State<ChannelChatScreen> {
rate: aeicRatePointForUi(result.rate),
chunkCount: chunkSet.dataChunkCount,
aspectCode: await _aspectCodeOf(sourceBytes),
// The decoded version is model output and carries the AI label; the
// stretched-original fallback is the sender's own pixels.
synthesized: result.previewPng != null,
payloadBytes: result.payload.length,
// Repeaters may already have echoed the first chunks while the rest
// were still going out.
chunkEchoes: mounted
? context.read<MeshCoreConnector>().ownImageEchoes(
widget.channel.index,
chunkSet.imgId,
)
: null,
);
} on Exception catch (error) {
debugPrint('outgoing image registration failed: $error');
@@ -2153,11 +2168,9 @@ class _ChannelChatScreenState extends State<ChannelChatScreen> {
/// [imageBytes] stretched to 512x512, as PNG.
///
/// This is what the preview sheet showed (`BoxFit.fill` on a 1:1 box is
/// exactly the stretch the codec applies) and what the codec actually
/// encoded, so the sender's bubble matches both. Deliberately not a decode of the
/// bitstream: that would cost ~2.16 GiB and ~1 s to reproduce, less well, an
/// image this device is already holding in memory.
/// Fallback for the sender's bubble when the preview sheet had no decoded
/// version to hand over (a device that cannot decode). It is the input the
/// codec stretched, not what recipients see, so it is only a stand-in.
///
/// Returns null rather than throwing on an undecodable source — the send has
/// already happened by the time this runs.
@@ -2411,12 +2424,35 @@ class _ChannelChatScreenState extends State<ChannelChatScreen> {
right: 8,
bottom: 4,
),
child: Text(
formatChatTime(context, entry.firstSeen),
style: MeshTheme.mono(
fontSize: 12 * textScale,
color: metaColor,
),
child: Row(
mainAxisSize: MainAxisSize.min,
children: [
Text(
formatChatTime(context, entry.firstSeen),
style: MeshTheme.mono(
fontSize: 12 * textScale,
color: metaColor,
),
),
if (isOutgoing) ...[
const SizedBox(width: 4),
// Same status as channel text: repeats heard, here
// for the least-heard data chunk.
MessageStatusIcon(
size: 16 * textScale,
isAcked: false,
isChannel: true,
repeatCount: entry.minDataChunkEchoes ?? 0,
resendProgress: context
.watch<MeshCoreConnector>()
.imageResendProgress(
entry.channelIndex,
entry.imgId,
),
onColor: metaColor,
),
],
],
),
),
],
@@ -2549,6 +2585,15 @@ class _ChannelChatScreenState extends State<ChannelChatScreen> {
if (entry.recoveredWithParity)
Text(l10n.receivedImage_parityRecovered),
if (decodeMs != null) Text(l10n.receivedImage_decodeTime(decodeMs)),
if (entry.chunkEchoes case final echoes?) ...[
const SizedBox(height: 8),
for (var i = 0; i < echoes.length; i++)
Text(
i < entry.totalChunks
? l10n.receivedImage_packetEchoes(i + 1, echoes[i])
: l10n.receivedImage_recoveryPacketEchoes(echoes[i]),
),
],
],
),
actions: [
+6
View File
@@ -487,6 +487,12 @@ class BackupService {
);
}
}
if (selectedItems.contains(BackupItem.appSettings)) {
await appSettingsService.loadSettings();
}
if (selectedItems.contains(BackupItem.channelSettings)) {
await connector.reloadChannelOrder();
}
if (!importingIdentity &&
selectedItems.contains(BackupItem.discoveredContacts)) {
await connector.reloadDiscoveredContacts();
+19 -18
View File
@@ -970,15 +970,17 @@ class ImageCodecService extends ChangeNotifier implements ImageSendCodec {
/// [ImageSendCodec] entry point used by `ImageSendPreviewSheet`.
///
/// Takes the raw picked file (JPEG/PNG/HEIC/…), centre-crops and scales it to
/// [kImageCodecSquareSize] square, then encodes. Throws on failure so the
/// Takes the raw picked file (JPEG/PNG/HEIC/…), stretches the part of it
/// that [variant] selects (see [rerollSourceRect]; 0 is the whole photo) to
/// a [kImageCodecSquareSize] square, then encodes. Throws on failure so the
/// sheet can surface it — unlike the nullable service-level methods below,
/// whose contract is "null means not available".
@override
Future<Uint8List> encode(
Uint8List imageBytes,
ImageCodecRatePoint rate,
) async {
ImageCodecRatePoint rate, {
int variant = 0,
}) async {
if (!kImageCodecBitstreamPathAvailable) {
// Fail here rather than after a 512x512 raster and a session load, and
// fail with the specific exception so the sheet can say why.
@@ -990,7 +992,7 @@ class ImageCodecService extends ChangeNotifier implements ImageSendCodec {
throw const ImageCodecBundleIncomplete();
}
final result = await encodeImage(
rgbBytes: await _toSquareRgb(imageBytes),
rgbBytes: await _toSquareRgb(imageBytes, variant: variant),
ratePoint: aeicRatePointForUi(rate),
);
final bitstream = result?.bitstream;
@@ -1300,14 +1302,18 @@ class ImageCodecService extends ChangeNotifier implements ImageSendCodec {
static const int _bytesPerPixel = 3;
/// Centre-crops [imageBytes] to a square and scales it to
/// [kImageCodecSquareSize], returning packed 8-bit RGB.
/// Stretches the [variant] region of [imageBytes] (see [rerollSourceRect];
/// 0 is the whole frame) to a [kImageCodecSquareSize] square, returning
/// packed 8-bit RGB.
///
/// Uses `dart:ui` rather than a decode package so this adds no dependency.
/// It runs on the root isolate, which is acceptable: it is a single 512x512
/// raster, three orders of magnitude cheaper than the 1385 GFLOP inference
/// that follows in the worker.
Future<Uint8List> _toSquareRgb(Uint8List imageBytes) async {
Future<Uint8List> _toSquareRgb(
Uint8List imageBytes, {
int variant = 0,
}) async {
final codec = await ui.instantiateImageCodec(imageBytes);
final frame = await codec.getNextFrame();
final source = frame.image;
@@ -1317,16 +1323,11 @@ class ImageCodecService extends ChangeNotifier implements ImageSendCodec {
// Aspect ratio is deliberately not preserved: the codec's graph is
// statically 512x512, so something has to give, and losing 25% of a 4:3
// photo (44% of 16:9, or the subject's head in a portrait) is worse than
// distorting it. Nothing outside the frame is discarded this way.
// NOTE: the aspect ratio is NOT transmitted, so the receiver cannot undo
// the stretch — see the note in image_chunk_transport.dart on the
// metadata byte.
final src = ui.Rect.fromLTWH(
0,
0,
source.width.toDouble(),
source.height.toDouble(),
);
// distorting it. The source shape travels in the metadata byte's aspect
// nibble, so the receiver un-stretches it. Re-roll variants trim at most
// ~2% at an edge, so they keep the same shape and aspect code.
final r = rerollSourceRect(source.width, source.height, variant);
final src = ui.Rect.fromLTWH(r.left, r.top, r.width, r.height);
final dstSize = kImageCodecSquareSize.toDouble();
final recorder = ui.PictureRecorder();
ui.Canvas(recorder).drawImageRect(
+66 -6
View File
@@ -197,13 +197,30 @@ class ReceivedImageEntry {
/// One chunk was rebuilt from the XOR parity chunk.
final bool recoveredWithParity;
/// This is the local user's own send. Its PNG is their real 512x512 crop, so
/// [synthesized] is false and the R6 label must NOT be shown.
/// This is the local user's own send.
final bool isOutgoing;
/// Whether the pixels are generative-model output. Derived, never supplied:
/// `synthesized == !isOutgoing`.
bool get synthesized => !isOutgoing;
/// Whether the pixels are generative-model output, so the R6 label must be
/// shown. Always true for received images. For the user's own send it is
/// true when the PNG is the version they picked, decoded as recipients see
/// it, and false only when it is the stretched original (a device that
/// could not decode).
final bool synthesized;
/// Our own image only: repeater echoes heard for each chunk, data chunks
/// first and the parity chunk last. Null when not tracked.
final List<int>? chunkEchoes;
/// The fewest echoes any data chunk got: every data chunk was heard
/// repeated at least this many times. Null when not tracked.
int? get minDataChunkEchoes {
final echoes = chunkEchoes;
if (echoes == null || echoes.isEmpty) return null;
final data = echoes.length > totalChunks
? echoes.sublist(0, totalChunks)
: echoes;
return data.reduce((a, b) => a < b ? a : b);
}
final int? decodeMs;
final String? error;
@@ -237,6 +254,8 @@ class ReceivedImageEntry {
this.metadataAssumed = true,
this.recoveredWithParity = false,
this.isOutgoing = false,
bool? synthesized,
this.chunkEchoes,
this.decodeMs,
this.error,
this.needsManualDecode = false,
@@ -245,7 +264,7 @@ class ReceivedImageEntry {
this.bitstreamByteCount = 0,
this.pngByteCount = 0,
this.pngBytes,
});
}) : synthesized = synthesized ?? !isOutgoing;
ImageStreamKey get key => ImageStreamKey(
senderPrefix: senderPrefix,
@@ -282,6 +301,7 @@ class ReceivedImageEntry {
int? bitstreamByteCount,
int? pngByteCount,
Object? pngBytes = _unset,
List<int>? chunkEchoes,
}) {
return ReceivedImageEntry(
streamId: streamId,
@@ -299,6 +319,8 @@ class ReceivedImageEntry {
metadataAssumed: metadataAssumed ?? this.metadataAssumed,
recoveredWithParity: recoveredWithParity ?? this.recoveredWithParity,
isOutgoing: isOutgoing,
synthesized: synthesized,
chunkEchoes: chunkEchoes ?? this.chunkEchoes,
decodeMs: identical(decodeMs, _unset) ? this.decodeMs : decodeMs as int?,
error: identical(error, _unset) ? this.error : error as String?,
needsManualDecode: needsManualDecode ?? this.needsManualDecode,
@@ -329,6 +351,7 @@ class ReceivedImageEntry {
'recoveredWithParity': recoveredWithParity,
'isOutgoing': isOutgoing,
'synthesized': synthesized,
if (chunkEchoes != null) 'chunkEchoes': chunkEchoes,
'needsManualDecode': needsManualDecode,
'bitstreamStored': bitstreamStored,
'pngStored': pngStored,
@@ -357,6 +380,10 @@ class ReceivedImageEntry {
metadataAssumed: json['metadataAssumed'] as bool? ?? true,
recoveredWithParity: json['recoveredWithParity'] as bool? ?? false,
isOutgoing: json['isOutgoing'] as bool? ?? false,
synthesized: json['synthesized'] as bool?,
chunkEchoes: (json['chunkEchoes'] as List<dynamic>?)
?.whereType<int>()
.toList(),
decodeMs: json['decodeMs'] as int?,
error: json['error'] as String?,
needsManualDecode: json['needsManualDecode'] as bool? ?? false,
@@ -1041,6 +1068,9 @@ class ReceivedImageStore extends ChangeNotifier {
required AeicRatePoint rate,
required int chunkCount,
int aspectCode = 0,
bool synthesized = false,
List<int>? chunkEchoes,
int payloadBytes = 0,
DateTime? at,
}) async {
final now = at ?? _clock();
@@ -1069,6 +1099,11 @@ class ReceivedImageStore extends ChangeNotifier {
aspectCode: aspectCode,
metadataAssumed: false,
isOutgoing: true,
synthesized: synthesized,
chunkEchoes: chunkEchoes,
// The size of what was sent, for the caption and packet info. The
// bitstream itself is not kept, so bitstreamStored stays false.
bitstreamByteCount: payloadBytes,
pngStored: pngWritten,
pngByteCount: pngWritten ? previewPng.length : 0,
pngBytes: previewPng,
@@ -1078,6 +1113,31 @@ class ReceivedImageStore extends ChangeNotifier {
return stored;
}
/// Records repeater echoes of the user's own image (see
/// `MeshCoreConnector.ownImageEchoes`). Echoes that land before the image is
/// registered are passed to [registerOutgoing] instead.
Future<void> setOutgoingEchoes({
required int channelIndex,
required int senderPrefix,
required int imgId,
required List<int> echoes,
}) async {
ReceivedImageEntry? match;
for (final entry in _entries.values) {
if (!entry.isOutgoing ||
entry.imgId != imgId ||
entry.channelIndex != channelIndex ||
entry.senderPrefix != senderPrefix) {
continue;
}
if (match == null || entry.firstSeen.isAfter(match.firstSeen)) {
match = entry;
}
}
if (match == null) return;
await _store(match.copyWith(chunkEchoes: List<int>.unmodifiable(echoes)));
}
// ---- decode scheduling ---------------------------------------------------
/// App lifecycle gate. Backgrounding cancels an in-flight decode and returns
+5
View File
@@ -109,6 +109,11 @@ class UiViewStateService extends ChangeNotifier {
}
}
Future<void> reload() async {
await initialize();
notifyListeners();
}
void setContactsSelectedGroupName(String value) {
if (_contactsSelectedGroupName == value) return;
_contactsSelectedGroupName = value;
+79 -8
View File
@@ -98,6 +98,54 @@ class ImageCodecRateStats {
}
}
/// Input tweaks for "Try another version", as `(zoom, dx, dy)` fractions of
/// the source photo. Index == version variant; 0 is the untouched photo.
///
/// The decoder is generative and very sensitive to its input: a near-identical
/// photo can come back with different invented detail. Measured on Kodak, the
/// best of three versions raised CLIP agreement with the source from 0.828 to
/// 0.853 and fixed 3 of 8 photos that came back as something else
/// (`tools/aeic/eval`, `reroll*` variants, which mirror this table). The tweaks
/// drop at most ~2% at an edge, so every version still shows the whole photo
/// and keeps the same aspect code.
const List<(double, double, double)> kImageSendRerollTweaks = [
(1.0, 0.0, 0.0),
(1.02, 0.005, 0.0),
(1.02, -0.005, 0.005),
(1.04, 0.0, -0.005),
(1.03, 0.01, 0.01),
(1.03, -0.01, -0.01),
];
/// Most versions the send sheet will make of one photo.
final int kImageSendMaxVersions = kImageSendRerollTweaks.length;
/// The part of a [width] x [height] photo that version [variant] encodes.
///
/// Variant 0 is the full frame, so the first version is exactly what the app
/// sent before versions existed. Other variants zoom in by `zoom`, shift by
/// `dx`/`dy` of the photo size, and are clamped to stay inside the photo.
({double left, double top, double width, double height}) rerollSourceRect(
int width,
int height,
int variant,
) {
if (variant < 0 || variant >= kImageSendRerollTweaks.length) {
throw RangeError.range(
variant,
0,
kImageSendRerollTweaks.length - 1,
'variant',
);
}
final (zoom, dx, dy) = kImageSendRerollTweaks[variant];
final w = width / zoom;
final h = height / zoom;
final left = ((width - w) / 2 + dx * width).clamp(0.0, width - w);
final top = ((height - h) / 2 + dy * height).clamp(0.0, height - h);
return (left: left, top: top, width: w, height: h);
}
/// The encode interface the preview sheet consumes.
abstract class ImageSendCodec {
/// Current availability. The sheet renders a non-interactive explanation for
@@ -118,9 +166,14 @@ abstract class ImageSendCodec {
/// Encode [imageBytes] (any common still format) at [rate], returning the
/// compressed bitstream that will be chunked onto the air.
///
/// Implementations are expected to centre-crop to
/// [kImageCodecSquareSize] square first.
Future<Uint8List> encode(Uint8List imageBytes, ImageCodecRatePoint rate);
/// Implementations stretch the part of the photo given by
/// [rerollSourceRect] for [variant] into a [kImageCodecSquareSize] square
/// first. Variant 0 is the whole photo.
Future<Uint8List> encode(
Uint8List imageBytes,
ImageCodecRatePoint rate, {
int variant = 0,
});
/// Decodes [payload] to PNG exactly as a recipient would, or null when this
/// device cannot decode.
@@ -142,26 +195,44 @@ class FakeImageSendCodec implements ImageSendCodec {
final Duration latency;
/// Returned by [decodePreview]; null means "this device cannot decode".
final Uint8List? previewPng;
const FakeImageSendCodec({
this.availability = ImageCodecAvailability.ready,
this.unavailableReason,
this.latency = const Duration(milliseconds: 250),
this.previewPng,
});
/// Payload size the fake produces for [variant]: the measured mean, plus a
/// little per variant so versions are told apart.
static int payloadBytesFor(ImageCodecRatePoint rate, int variant) =>
ImageCodecRateStats.forRate(rate).meanBytes + variant * 3;
@override
Future<Uint8List> encode(
Uint8List imageBytes,
ImageCodecRatePoint rate,
) async {
ImageCodecRatePoint rate, {
int variant = 0,
}) async {
if (latency > Duration.zero) {
await Future<void>.delayed(latency);
}
final size = ImageCodecRateStats.forRate(rate).meanBytes;
final size = payloadBytesFor(rate, variant);
return Uint8List.fromList(
List<int>.generate(size, (i) => (i * 31 + rate.index) & 0xFF),
List<int>.generate(
size,
(i) => (i * 31 + rate.index + variant * 7) & 0xFF,
),
);
}
@override
Future<Uint8List?> decodePreview(Uint8List payload) async => null;
Future<Uint8List?> decodePreview(Uint8List payload) async {
if (latency > Duration.zero) {
await Future<void>.delayed(latency);
}
return previewPng;
}
}
+308 -91
View File
@@ -6,6 +6,7 @@ import 'package:provider/provider.dart';
import '../connector/meshcore_connector.dart';
import '../l10n/l10n.dart';
import '../theme/mesh_theme.dart';
import '../services/image_chunk_transport.dart';
import '../utils/lora_airtime.dart';
import 'image_send_codec_binding.dart';
@@ -37,6 +38,11 @@ class ImageSendPreviewResult {
/// actually shown. Null when the radio settings were unknown.
final Duration? wallClock;
/// The chosen version decoded as recipients will see it (a
/// 512x512 PNG, stretched like the codec input), or null if this device
/// could not decode. Used for the sender's own chat bubble.
final Uint8List? previewPng;
const ImageSendPreviewResult({
required this.payload,
required this.rate,
@@ -44,6 +50,7 @@ class ImageSendPreviewResult {
required this.packetCount,
required this.airtime,
required this.wallClock,
this.previewPng,
});
}
@@ -141,29 +148,53 @@ class _RateEstimate {
bool get isExact => payloadBytes != null;
}
/// One encoded version of the photo: the whole photo with re-roll tweak
/// [variant] (see [rerollSourceRect]), its bitstream, and what recipients will
/// see when they decode it.
class _Version {
_Version(this.variant);
final int variant;
Uint8List? payload;
Uint8List? png;
bool encoding = true;
bool decoding = false;
/// The encode failed; there is nothing to send for this version.
bool get failed => !encoding && payload == null;
}
class _ImageSendPreviewSheetState extends State<ImageSendPreviewSheet> {
/// The encoded ft32 bitstream, once the encode finishes.
Uint8List? _encoded;
/// Versions made so far, in the order they were made. Version 1 (variant 0)
/// is the photo exactly as the app has always encoded it; more are added only
/// when the user taps "Try another version".
final List<_Version> _versions = [];
int _selected = 0;
late bool _parity;
bool _encoding = false;
bool _sending = false;
bool _failed = false;
/// Latest radio params seen in build; used by [_onSend] so the confirm path
/// does not have to touch an InheritedWidget after an await.
ImageSendRadio? _radio;
/// Source `width / height`. Recipients un-stretch the 512x512 square back to
/// this shape, so both preview views use it.
/// Source `width / height`, used when the original is shown.
double _aspect = 1;
// The shape recipients restore: the nearest supported ratio, or square.
double _recipientAspect = 1;
bool _showReconstruction = false;
Uint8List? _reconstruction;
bool _reconstructing = false;
bool _reconstructionUnavailable = false;
/// The main preview shows the original photo instead of the reconstruction.
bool _showOriginal = false;
/// This device cannot decode, so there is no reconstruction to show and
/// making more versions would be pointless.
bool _previewUnavailable = false;
_Version? get _current => _versions.isEmpty ? null : _versions[_selected];
/// A version is being encoded or decoded. Codec jobs run one at a time, so
/// "Try another version" waits for this to clear.
bool get _working => _versions.any((v) => v.encoding || v.decoding);
@override
void initState() {
@@ -171,7 +202,7 @@ class _ImageSendPreviewSheetState extends State<ImageSendPreviewSheet> {
_parity = widget.initialParity;
_probeAspect();
if (widget.codec.availability == ImageCodecAvailability.ready) {
_encode();
_addVersion();
}
}
@@ -197,59 +228,56 @@ class _ImageSendPreviewSheetState extends State<ImageSendPreviewSheet> {
} catch (_) {}
}
Future<void> _selectView(bool reconstruction) async {
setState(() => _showReconstruction = reconstruction);
final payload = _encoded;
if (!reconstruction ||
payload == null ||
_reconstruction != null ||
_reconstructing) {
return;
/// Encodes the next version, selects it, then decodes it so the user sees
/// what recipients will get.
Future<void> _addVersion() async {
final version = _Version(_versions.length);
setState(() {
_versions.add(version);
_selected = _versions.length - 1;
_showOriginal = false;
});
Uint8List? payload;
try {
payload = await widget.codec.encode(
widget.imageBytes,
kImageSendRatePoint,
variant: version.variant,
);
} catch (_) {
payload = null;
}
setState(() => _reconstructing = true);
if (!mounted) return;
setState(() {
version.encoding = false;
version.payload = payload;
version.decoding = payload != null && !_previewUnavailable;
});
if (!version.decoding) return;
Uint8List? png;
try {
png = await widget.codec.decodePreview(payload);
png = await widget.codec.decodePreview(payload!);
} catch (_) {
png = null;
}
if (!mounted) return;
setState(() {
_reconstructing = false;
_reconstruction = png;
if (png == null) {
_reconstructionUnavailable = true;
_showReconstruction = false;
version.decoding = false;
version.png = png;
if (png == null && !_versions.any((v) => v.png != null)) {
// Nothing has ever decoded here: treat the device as unable to
// preview and fall back to showing the original.
_previewUnavailable = true;
_showOriginal = true;
}
});
}
Future<Uint8List?> _encode() async {
final cached = _encoded;
if (cached != null) return cached;
void _select(int index) {
setState(() {
_encoding = true;
_failed = false;
_selected = index;
_showOriginal = false;
});
try {
final bytes = await widget.codec.encode(
widget.imageBytes,
kImageSendRatePoint,
);
if (!mounted) return bytes;
setState(() {
_encoded = bytes;
_encoding = false;
});
return bytes;
} catch (_) {
if (!mounted) return null;
setState(() {
_encoding = false;
_failed = true;
});
return null;
}
}
ImageSendRadio _radioParams(BuildContext context) {
@@ -264,7 +292,7 @@ class _ImageSendPreviewSheetState extends State<ImageSendPreviewSheet> {
}
_RateEstimate _estimateFor(ImageSendRadio radio) {
final bytes = _encoded;
final bytes = _current?.payload;
if (bytes != null) {
final exact = _estimate(bytes.length, radio);
return _RateEstimate(
@@ -293,14 +321,11 @@ class _ImageSendPreviewSheetState extends State<ImageSendPreviewSheet> {
);
}
Future<void> _onSend() async {
void _onSend() {
final version = _current;
final payload = version?.payload;
if (version == null || payload == null) return;
setState(() => _sending = true);
final payload = await _encode();
if (!mounted) return;
if (payload == null) {
setState(() => _sending = false);
return;
}
final radio = _radio ?? const ImageSendRadio();
final estimate = _estimate(payload.length, radio);
Navigator.of(context).pop(
@@ -311,6 +336,7 @@ class _ImageSendPreviewSheetState extends State<ImageSendPreviewSheet> {
packetCount: estimate.chunkCount,
airtime: estimate.totalAirtime,
wallClock: estimate.pacedWallClock,
previewPng: version.png,
),
);
}
@@ -375,7 +401,7 @@ class _ImageSendPreviewSheetState extends State<ImageSendPreviewSheet> {
..._warnings(theme, colors, radio, selected),
const SizedBox(height: 4),
_parityTile(theme, colors),
if (_failed) ...[
if (_current?.failed ?? false) ...[
const SizedBox(height: 12),
_banner(
colors: colors,
@@ -408,6 +434,9 @@ class _ImageSendPreviewSheetState extends State<ImageSendPreviewSheet> {
),
);
/// The main preview: by default the selected version as recipients
/// will see it, with the original one tap away for comparison.
///
/// The preview is deliberately capped in height: the packet count and airtime
/// below it are the reason this screen exists and must not be pushed off the
/// first screenful by a full-width square.
@@ -419,59 +448,96 @@ class _ImageSendPreviewSheetState extends State<ImageSendPreviewSheet> {
) {
final l10n = context.l10n;
final maxSide = MediaQuery.sizeOf(context).height * 0.28;
final reconstruction = _showReconstruction ? _reconstruction : null;
final version = _current;
final canPreview = ready && !_previewUnavailable;
final showReconstruction = canPreview && !_showOriginal && version != null;
final png = showReconstruction ? version.png : null;
final pending =
showReconstruction &&
png == null &&
(version.encoding || version.decoding);
final broken = showReconstruction && png == null && !pending;
Widget content;
if (pending) {
content = Center(
child: Padding(
padding: const EdgeInsets.all(12),
child: Column(
mainAxisSize: MainAxisSize.min,
children: [
const CircularProgressIndicator(),
const SizedBox(height: 12),
Text(
l10n.imageSend_preparingPreview,
textAlign: TextAlign.center,
style: theme.textTheme.bodyMedium?.copyWith(
color: colors.onSurfaceVariant,
),
),
],
),
),
);
} else if (broken) {
content = Center(
child: Icon(
Icons.broken_image_outlined,
size: 48,
color: colors.onSurfaceVariant,
),
);
} else {
content = Image.memory(
png ?? widget.imageBytes,
// The reconstruction is the stretched square; fill undoes the stretch
// into the recipient's shape.
fit: png != null ? BoxFit.fill : BoxFit.contain,
alignment: Alignment.center,
gaplessPlayback: true,
errorBuilder: (context, error, stack) => Center(
child: Icon(
Icons.broken_image_outlined,
size: 48,
color: colors.onSurfaceVariant,
),
),
);
}
return Column(
children: [
if (ready) ...[
if (canPreview) ...[
SegmentedButton<bool>(
segments: [
ButtonSegment(
value: false,
label: MeshSegmentLabel(l10n.imageSend_viewOriginal),
label: MeshSegmentLabel(l10n.imageSend_viewReconstruction),
),
ButtonSegment(
value: true,
label: MeshSegmentLabel(l10n.imageSend_viewReconstruction),
enabled: _encoded != null && !_reconstructionUnavailable,
label: MeshSegmentLabel(l10n.imageSend_viewOriginal),
),
],
selected: {_showReconstruction},
onSelectionChanged: (s) => _selectView(s.first),
selected: {_showOriginal},
onSelectionChanged: (s) => setState(() => _showOriginal = s.first),
),
const SizedBox(height: 12),
],
ConstrainedBox(
constraints: BoxConstraints(maxHeight: maxSide, maxWidth: maxSide),
child: ClipRRect(
borderRadius: BorderRadius.circular(12),
borderRadius: BorderRadius.circular(MeshRadii.md),
child: AspectRatio(
aspectRatio: reconstruction != null ? _recipientAspect : _aspect,
child: Container(
aspectRatio: showReconstruction ? _recipientAspect : _aspect,
child: ColoredBox(
color: colors.surfaceContainerHighest,
child: _showReconstruction && _reconstructing
? const Center(child: CircularProgressIndicator())
: Image.memory(
reconstruction ?? widget.imageBytes,
// The reconstruction is the stretched square; fill
// undoes the stretch into the recipient's shape.
fit: reconstruction != null
? BoxFit.fill
: BoxFit.contain,
alignment: Alignment.center,
gaplessPlayback: true,
errorBuilder: (context, error, stack) => Center(
child: Icon(
Icons.broken_image_outlined,
size: 48,
color: colors.onSurfaceVariant,
),
),
),
child: content,
),
),
),
),
if (_reconstructionUnavailable) ...[
if (_previewUnavailable) ...[
const SizedBox(height: 8),
Text(
l10n.imageSend_reconstructionUnavailable,
@@ -481,6 +547,10 @@ class _ImageSendPreviewSheetState extends State<ImageSendPreviewSheet> {
),
),
],
if (canPreview && _versions.isNotEmpty) ...[
const SizedBox(height: 16),
_versionsRow(theme, colors),
],
const SizedBox(height: 8),
Text(
l10n.imageSend_lossyNote(
@@ -496,6 +566,152 @@ class _ImageSendPreviewSheetState extends State<ImageSendPreviewSheet> {
);
}
/// The versions made so far, plus "Try another version".
Widget _versionsRow(ThemeData theme, ColorScheme colors) {
final l10n = context.l10n;
return Column(
children: [
Wrap(
alignment: WrapAlignment.center,
spacing: 8,
runSpacing: 8,
children: [
for (var i = 0; i < _versions.length; i++)
_versionTile(theme, colors, i),
],
),
if (_versions.length < kImageSendMaxVersions) ...[
const SizedBox(height: 12),
OutlinedButton.icon(
onPressed: (_working || _sending) ? null : _addVersion,
icon: const Icon(Icons.refresh),
label: Text(l10n.imageSend_tryAnother),
),
],
const SizedBox(height: 8),
Text(
l10n.imageSend_versionsHint,
textAlign: TextAlign.center,
style: theme.textTheme.bodySmall?.copyWith(
color: colors.onSurfaceVariant,
),
),
],
);
}
/// One selectable version: its reconstruction, "Version n" and its packet
/// count. Selection shows as a thick primary border plus a check badge, so
/// it never relies on colour alone.
Widget _versionTile(ThemeData theme, ColorScheme colors, int index) {
final l10n = context.l10n;
final version = _versions[index];
final selected = index == _selected;
final label = l10n.imageSend_versionLabel(index + 1);
const maxSide = 80.0;
final aspect = _recipientAspect;
final width = aspect >= 1 ? maxSide : maxSide * aspect;
final height = aspect >= 1 ? maxSide / aspect : maxSide;
final payload = version.payload;
final packets = payload == null
? null
: _estimate(
payload.length,
_radio ?? const ImageSendRadio(),
).chunkCount;
final Widget thumb;
if (version.png != null) {
thumb = Image.memory(
version.png!,
fit: BoxFit.fill,
gaplessPlayback: true,
);
} else if (version.encoding || version.decoding) {
thumb = const Center(
child: SizedBox(
width: 20,
height: 20,
child: CircularProgressIndicator(strokeWidth: 2),
),
);
} else {
thumb = Icon(Icons.broken_image_outlined, color: colors.onSurfaceVariant);
}
return Semantics(
button: true,
selected: selected,
label: label,
excludeSemantics: true,
child: InkWell(
borderRadius: BorderRadius.circular(MeshRadii.sm),
onTap: _sending ? null : () => _select(index),
child: Padding(
padding: const EdgeInsets.all(4),
child: Column(
mainAxisSize: MainAxisSize.min,
children: [
Stack(
children: [
Container(
width: width.clamp(48.0, maxSide),
height: height.clamp(48.0, maxSide),
clipBehavior: Clip.antiAlias,
decoration: BoxDecoration(
color: colors.surfaceContainerHighest,
borderRadius: BorderRadius.circular(MeshRadii.sm),
border: Border.all(
color: selected
? colors.primary
: colors.outlineVariant,
width: selected ? 3 : 1,
),
),
child: thumb,
),
if (selected)
Positioned(
top: 4,
right: 4,
child: Container(
padding: const EdgeInsets.all(2),
decoration: BoxDecoration(
color: colors.primary,
shape: BoxShape.circle,
),
child: Icon(
Icons.check,
size: 16,
color: colors.onPrimary,
),
),
),
],
),
const SizedBox(height: 4),
Text(
label,
style: theme.textTheme.bodySmall?.copyWith(
color: selected ? colors.onSurface : colors.onSurfaceVariant,
fontWeight: selected ? FontWeight.w600 : null,
),
),
if (packets != null)
Text(
l10n.imageSend_versionPackets(packets),
style: MeshTheme.mono(
fontSize: 12,
color: colors.onSurfaceVariant,
),
),
],
),
),
),
);
}
Widget _sizeRow(ThemeData theme, ColorScheme colors, _RateEstimate estimate) {
return Row(
mainAxisAlignment: MainAxisAlignment.center,
@@ -794,7 +1010,8 @@ class _ImageSendPreviewSheetState extends State<ImageSendPreviewSheet> {
Widget _actions(ThemeData theme, bool ready) {
final localizations = context.l10n;
final busy = _sending || _encoding;
final busy = _sending || (_current?.encoding ?? false);
final canSend = ready && !_sending && _current?.payload != null;
return Padding(
padding: const EdgeInsets.fromLTRB(16, 8, 16, 16),
child: Row(
@@ -810,7 +1027,7 @@ class _ImageSendPreviewSheetState extends State<ImageSendPreviewSheet> {
const SizedBox(width: 12),
Expanded(
child: FilledButton.icon(
onPressed: (!ready || busy) ? null : _onSend,
onPressed: canSend ? _onSend : null,
icon: busy
? const SizedBox(
width: 16,
+7 -4
View File
@@ -19,10 +19,13 @@
///
/// Both are private to this file, are not parameterised, and are reused
/// verbatim by the full-screen viewer this widget opens on tap, so there is no
/// route in the app that can show the pixels without them. The only case in
/// which they are absent is [ReceivedImageEntry.isOutgoing], where the PNG is
/// the sender's own 512x512 crop and the label would be a falsehood — that is
/// read from the store's sidecar, never from a constructor argument.
/// route in the app that can show the pixels without them. They are absent
/// only when [ReceivedImageEntry.synthesized] is false: the sender's own image
/// on a device that could not decode, where the PNG is their stretched
/// original and the label would be a falsehood. The sender's own image is
/// otherwise the decoded version they picked, so it is labelled like any
/// other. That is read from the store's sidecar, never from a constructor
/// argument.
///
/// The widget deliberately exposes no share/save action: an exported raster
/// would have to have the caption composited into it first, and that belongs
+13
View File
@@ -259,6 +259,19 @@ void main() {
},
);
test('reloads a restored custom channel order', () async {
await _connect(_TestRadio());
final restoredOrder = [_id(_psk(1)), _id(_psk(0)), _id(_psk(2))];
await PrefsManager.instance.setString(
'channel_order_$_scope',
jsonEncode(restoredOrder),
);
await _connector.reloadChannelOrder();
expect(_connector.channels.first.storageId, restoredOrder.first);
});
test('deleting a channel clears its stored history', () async {
await _connect(_TestRadio());
await _waitFor(() => _texts(1).contains(_queuedHikingText));
+141
View File
@@ -0,0 +1,141 @@
import 'dart:typed_data';
import 'package:crypto/crypto.dart' as crypto;
import 'package:flutter_test/flutter_test.dart';
import 'package:meshcore_open/connector/meshcore_connector.dart';
import 'package:meshcore_open/connector/meshcore_protocol.dart';
import 'package:meshcore_open/models/channel.dart';
import 'package:meshcore_open/review_mode/review_radio.dart';
import 'package:meshcore_open/review_mode/review_script.dart';
import 'package:meshcore_open/services/image_chunk_transport.dart';
import 'package:meshcore_open/services/message_retry_service.dart';
import 'package:meshcore_open/services/path_history_service.dart';
import 'package:meshcore_open/services/storage_service.dart';
import 'package:meshcore_open/storage/prefs_manager.dart';
import 'package:meshcore_open/widgets/image_send_codec_binding.dart';
import 'package:pointycastle/export.dart';
import 'package:shared_preferences/shared_preferences.dart';
Future<void> _waitFor(bool Function() condition, {int seconds = 8}) async {
final end = DateTime.now().add(Duration(seconds: seconds));
while (!condition()) {
if (DateTime.now().isAfter(end)) fail('Timed out waiting for condition');
await Future<void>.delayed(const Duration(milliseconds: 20));
}
}
Uint8List _psk(int slot) =>
Channel.parsePskHex(ReviewScript.channels[slot].pskHex);
/// A raw-log frame for a repeater re-flooding [blob] as GRP_DATA on [psk]'s
/// channel, encrypted the way the firmware's createGroupDatagram does.
Uint8List _logRxGroupData(Uint8List psk, Uint8List blob, {int hops = 1}) {
final plain = <int>[
dataTypeAeicImage & 0xFF,
dataTypeAeicImage >> 8,
blob.length,
...blob,
];
final padded = Uint8List(((plain.length + 15) ~/ 16) * 16)
..setRange(0, plain.length, plain);
final cipher = ECBBlockCipher(AESEngine())..init(true, KeyParameter(psk));
final encrypted = Uint8List(padded.length);
for (var i = 0; i < padded.length; i += 16) {
cipher.processBlock(padded, i, encrypted, i);
}
final key32 = Uint8List(32)..setRange(0, 16, psk);
final mac = crypto.Hmac(crypto.sha256, key32).convert(encrypted).bytes;
final channelHash = crypto.sha256.convert(psk).bytes[0];
return Uint8List.fromList([
pushCodeLogRxData,
40, // snr
0xA0, // rssi
0x19, // flood route, GRP_DATA payload
hops, // path length: one 1-byte hash per hop
for (var i = 0; i < hops; i++) 0x42 + i,
channelHash,
mac[0],
mac[1],
...encrypted,
]);
}
void main() {
TestWidgetsFlutterBinding.ensureInitialized();
late MeshCoreConnector connector;
// One connector for the file: the notification service it starts accepts a
// single listener per process.
setUpAll(() async {
SharedPreferences.setMockInitialValues({});
PrefsManager.reset();
await PrefsManager.initialize();
connector = MeshCoreConnector();
final retryService = MessageRetryService();
connector.initialize(
retryService: retryService,
pathHistoryService: PathHistoryService(StorageService()),
);
final radio = ReviewRadio(
scheduledMessageInterval: const Duration(hours: 1),
ackDelay: const Duration(milliseconds: 50),
replyDelay: const Duration(hours: 1),
);
connector.reviewRadioFactoryForTest = () => radio;
await connector.connectReview();
await _waitFor(() => connector.channelMapConfirmed);
});
tearDownAll(() async {
await connector.disconnect();
connector.dispose();
});
Future<ImageChunkSet> sendImage({
required int slot,
required int imgId,
}) async {
final chunkSet = buildImageChunks(
payload: Uint8List.fromList(List<int>.generate(200, (i) => i & 0xFF)),
metadata: const ImageStreamMetadata(rate: ImageCodecRatePoint.standard),
senderPrefix: senderPrefixFromKey(connector.selfPublicKey)!,
imgId: imgId,
);
expect(
await connector.sendImageChunks(
chunkSet.blobs,
channelIndex: slot,
interChunkDelay: Duration.zero,
),
isTrue,
);
return chunkSet;
}
test('a repeater echo of our own chunk is counted for that chunk', () async {
final chunkSet = await sendImage(slot: 0, imgId: 77);
expect(chunkSet.blobs.length, 3, reason: '2 data chunks + parity');
expect(connector.ownImageEchoes(0, 77), [0, 0, 0]);
connector.handleFrameForTest(_logRxGroupData(_psk(0), chunkSet.blobs[1]));
connector.handleFrameForTest(_logRxGroupData(_psk(0), chunkSet.blobs[1]));
connector.handleFrameForTest(_logRxGroupData(_psk(0), chunkSet.blobs[2]));
expect(connector.ownImageEchoes(0, 77), [0, 2, 1]);
});
test('chunks from someone else, or on another channel key, are not '
'counted', () async {
final chunkSet = await sendImage(slot: 0, imgId: 78);
// Same image id, different sender.
final foreign = Uint8List.fromList(chunkSet.blobs[0]);
foreign[0] ^= 0xFF;
connector.handleFrameForTest(_logRxGroupData(_psk(0), foreign));
// Our chunk, but encrypted for a different channel than it was sent on.
connector.handleFrameForTest(_logRxGroupData(_psk(1), chunkSet.blobs[0]));
expect(connector.ownImageEchoes(0, 78), [0, 0, 0]);
});
}
@@ -410,6 +410,11 @@ void main() {
test('mesh copies of one message are stored and filtered once', () async {
await _connect();
await _waitFor(
() => ReviewScript.seedChannelMessages.every(
(message) => _find(message.channelIndex, message.text) != null,
),
);
const text = 'Weather net starts at seven tonight on the hill';
final ts = _nowSecs();
final cleanBefore = _filter.recentCleanCount;
+60
View File
@@ -12,6 +12,7 @@ import 'package:meshcore_open/services/image_codec_settings_store.dart';
import 'package:meshcore_open/services/translation_file_store.dart';
import 'package:meshcore_open/services/translation_service.dart';
import 'package:meshcore_open/services/backup_service.dart';
import 'package:meshcore_open/services/ui_view_state_service.dart';
import 'package:meshcore_open/storage/prefs_manager.dart';
import 'package:shared_preferences/shared_preferences.dart';
@@ -211,6 +212,65 @@ void main() {
expect(restored['map_tile_api_key'], 'existing-stadia');
},
);
test('applies restored app settings to the live service', () async {
SharedPreferences.setMockInitialValues({
'app_settings': jsonEncode({'evict_discovered_contacts_enabled': true}),
});
PrefsManager.reset();
await PrefsManager.initialize();
addTearDown(PrefsManager.reset);
final connector = MeshCoreConnector();
final settingsService = AppSettingsService();
addTearDown(connector.dispose);
addTearDown(settingsService.dispose);
await settingsService.loadSettings();
final backup = await BackupService.encodeBackup({
'format': 'meshcore-open-backup',
'version': 1,
'sourcePublicKey': '',
'items': [BackupItem.appSettings.name],
'sections': {
'appSettings': {
'preferences': {
'app_settings': {'evict_discovered_contacts_enabled': false},
},
},
},
}, password: '');
await const BackupService().restoreBackup(
contents: backup,
password: '',
items: {BackupItem.appSettings},
connector: connector,
appSettingsService: settingsService,
);
expect(settingsService.settings.evictDiscoveredContactsEnabled, isFalse);
});
});
group('UiViewStateService reload', () {
test('notifies listeners with restored sort preferences', () async {
SharedPreferences.setMockInitialValues({});
PrefsManager.reset();
await PrefsManager.initialize();
addTearDown(PrefsManager.reset);
final viewState = UiViewStateService();
addTearDown(viewState.dispose);
await viewState.initialize();
var notifications = 0;
viewState.addListener(() => notifications++);
await PrefsManager.instance.setString('ui_contacts_sort_option', 'name');
await PrefsManager.instance.setString('ui_channels_sort_option', 'name');
await viewState.reload();
expect(viewState.contactsSortOption.name, 'name');
expect(viewState.channelsSortOption, ChannelSortOption.name);
expect(notifications, 1);
});
});
group('restored model registry reconciliation', () {
@@ -826,6 +826,73 @@ void main() {
expect(h.decoder.calls, 0);
});
test('a decoded preview is labelled synthesized and keeps its byte count '
'and echoes', () async {
final h = _build();
final entry = await h.store.registerOutgoing(
channelIndex: 1,
senderPrefix: kSelf,
imgId: 4,
previewPng: Uint8List.fromList(List<int>.filled(64, 1)),
rate: AeicRatePoint.ft32,
chunkCount: 2,
synthesized: true,
payloadBytes: 171,
chunkEchoes: const [1, 0, 2],
);
expect(entry.synthesized, isTrue);
expect(entry.bitstreamByteCount, 171);
expect(entry.bitstreamStored, isFalse);
expect(entry.minDataChunkEchoes, 0);
final restored = ReceivedImageEntry.fromJson(entry.toJson());
expect(restored.synthesized, isTrue);
expect(restored.chunkEchoes, [1, 0, 2]);
});
test(
'setOutgoingEchoes updates the newest matching outgoing image',
() async {
final h = _build();
final older = await h.store.registerOutgoing(
channelIndex: 1,
senderPrefix: kSelf,
imgId: 9,
previewPng: Uint8List.fromList(List<int>.filled(64, 1)),
rate: AeicRatePoint.ft32,
chunkCount: 1,
);
h.advance(const Duration(seconds: 5));
final newer = await h.store.registerOutgoing(
channelIndex: 1,
senderPrefix: kSelf,
imgId: 9,
previewPng: Uint8List.fromList(List<int>.filled(64, 2)),
rate: AeicRatePoint.ft32,
chunkCount: 1,
);
await h.store.setOutgoingEchoes(
channelIndex: 1,
senderPrefix: kSelf,
imgId: 9,
echoes: const [3, 1],
);
expect(h.store.entryFor(newer.streamId)!.chunkEchoes, [3, 1]);
expect(h.store.entryFor(newer.streamId)!.minDataChunkEchoes, 3);
expect(h.store.entryFor(older.streamId)!.chunkEchoes, isNull);
// Another channel's image with the same id is left alone.
await h.store.setOutgoingEchoes(
channelIndex: 2,
senderPrefix: kSelf,
imgId: 9,
echoes: const [7, 7],
);
expect(h.store.entryFor(newer.streamId)!.chunkEchoes, [3, 1]);
},
);
test('spent outgoing entries count toward maxEntries', () async {
final h = _build(maxEntries: 2, maxAge: const Duration(minutes: 10));
final ids = <String>[];
@@ -0,0 +1,57 @@
import 'package:flutter_test/flutter_test.dart';
import 'package:meshcore_open/widgets/image_send_codec_binding.dart';
void main() {
group('rerollSourceRect', () {
const sizes = [
(4000, 3000),
(3000, 4000),
(1920, 1080),
(512, 512),
(7, 3),
];
test('variant 0 is the whole photo', () {
for (final (w, h) in sizes) {
final r = rerollSourceRect(w, h, 0);
expect(r.left, 0);
expect(r.top, 0);
expect(r.width, w);
expect(r.height, h);
}
});
test('every variant stays inside the photo and keeps its shape', () {
for (final (w, h) in sizes) {
for (var v = 0; v < kImageSendMaxVersions; v++) {
final r = rerollSourceRect(w, h, v);
expect(r.left, greaterThanOrEqualTo(0), reason: '$w x $h v$v');
expect(r.top, greaterThanOrEqualTo(0), reason: '$w x $h v$v');
expect(r.left + r.width, lessThanOrEqualTo(w + 1e-9));
expect(r.top + r.height, lessThanOrEqualTo(h + 1e-9));
// Same aspect, so the version keeps the photo's aspect code.
expect(r.width / r.height, closeTo(w / h, (w / h) * 0.01));
// "The whole photo": at most a few percent trimmed.
expect(r.width, greaterThanOrEqualTo(w * 0.95));
}
}
});
test('variants differ from each other and are deterministic', () {
final rects = [
for (var v = 0; v < kImageSendMaxVersions; v++)
rerollSourceRect(4000, 3000, v),
];
expect(rects.toSet().length, kImageSendMaxVersions);
expect(rerollSourceRect(4000, 3000, 3), rects[3]);
});
test('rejects a variant outside the table', () {
expect(() => rerollSourceRect(10, 10, -1), throwsRangeError);
expect(
() => rerollSourceRect(10, 10, kImageSendMaxVersions),
throwsRangeError,
);
});
});
}
@@ -336,4 +336,145 @@ void main() {
final l10n = await AppLocalizations.delegate.load(const Locale('en'));
expect(_texts(tester), contains(l10n.imageSend_codecUnavailable));
});
group('versions', () {
final previewCodec = FakeImageSendCodec(
latency: Duration.zero,
previewPng: _onePixelPng,
);
Future<void> tapText(WidgetTester tester, String text) async {
final finder = find.text(text);
await tester.tap(finder);
await tester.pumpAndSettle();
}
Future<ImageSendPreviewResult?> send(
WidgetTester tester,
ImageSendPreviewResult? Function() result,
) async {
await tester.tap(find.byType(FilledButton));
await tester.pumpAndSettle();
return result();
}
testWidgets('opens on what recipients see, as Version 1', (tester) async {
_tallScreen(tester);
await _openSheet(tester, radio: _knownRadio, codec: previewCodec);
final l10n = await AppLocalizations.delegate.load(const Locale('en'));
final texts = _texts(tester);
expect(texts, contains(l10n.imageSend_versionLabel(1)));
expect(texts, contains(l10n.imageSend_tryAnother));
expect(texts, isNot(contains(l10n.imageSend_reconstructionUnavailable)));
final segments = tester.widget<SegmentedButton<bool>>(
find.byType(SegmentedButton<bool>),
);
expect(segments.selected, {
false,
}, reason: 'the reconstruction, not the original, is the default view');
});
testWidgets('trying another version selects it and sends its payload and '
'reconstruction', (tester) async {
_tallScreen(tester);
ImageSendPreviewResult? result;
await tester.pumpWidget(_host(previewCodec, (r) => result = r));
await tester.tap(find.text('open'));
await tester.pumpAndSettle();
final l10n = await AppLocalizations.delegate.load(const Locale('en'));
await tapText(tester, l10n.imageSend_tryAnother);
expect(_texts(tester), contains(l10n.imageSend_versionLabel(2)));
final sent = await send(tester, () => result);
expect(
sent!.payload.length,
FakeImageSendCodec.payloadBytesFor(kImageSendRatePoint, 1),
);
expect(sent.previewPng, isNotNull);
});
testWidgets('going back to Version 1 sends Version 1', (tester) async {
_tallScreen(tester);
ImageSendPreviewResult? result;
await tester.pumpWidget(_host(previewCodec, (r) => result = r));
await tester.tap(find.text('open'));
await tester.pumpAndSettle();
final l10n = await AppLocalizations.delegate.load(const Locale('en'));
await tapText(tester, l10n.imageSend_tryAnother);
await tapText(tester, l10n.imageSend_versionLabel(1));
final sent = await send(tester, () => result);
expect(
sent!.payload.length,
FakeImageSendCodec.payloadBytesFor(kImageSendRatePoint, 0),
);
});
testWidgets('stops offering versions at the maximum', (tester) async {
_tallScreen(tester);
await _openSheet(tester, radio: _knownRadio, codec: previewCodec);
final l10n = await AppLocalizations.delegate.load(const Locale('en'));
for (var i = 1; i < kImageSendMaxVersions; i++) {
await tapText(tester, l10n.imageSend_tryAnother);
}
final texts = _texts(tester);
expect(
texts,
contains(l10n.imageSend_versionLabel(kImageSendMaxVersions)),
);
expect(texts, isNot(contains(l10n.imageSend_tryAnother)));
});
testWidgets('a device that cannot decode shows the original and no '
'versions', (tester) async {
_tallScreen(tester);
await _openSheet(tester, radio: _knownRadio);
final l10n = await AppLocalizations.delegate.load(const Locale('en'));
final texts = _texts(tester);
expect(texts, contains(l10n.imageSend_reconstructionUnavailable));
expect(texts, isNot(contains(l10n.imageSend_versionLabel(1))));
expect(texts, isNot(contains(l10n.imageSend_tryAnother)));
expect(find.byType(SegmentedButton<bool>), findsNothing);
});
});
}
/// Room for the whole sheet, so version tiles and "Try another version" are
/// on screen without scrolling the draggable sheet.
void _tallScreen(WidgetTester tester) {
tester.view.physicalSize = const Size(1000, 2400);
tester.view.devicePixelRatio = 1.0;
addTearDown(tester.view.reset);
}
/// A host app whose button opens the sheet and reports what it returned.
Widget _host(
ImageSendCodec codec,
void Function(ImageSendPreviewResult?) onResult,
) => MaterialApp(
localizationsDelegates: AppLocalizations.localizationsDelegates,
supportedLocales: AppLocalizations.supportedLocales,
home: Builder(
builder: (context) => Scaffold(
body: Center(
child: ElevatedButton(
onPressed: () async => onResult(
await showImageSendPreviewSheet(
context: context,
imageBytes: _onePixelPng,
originalFileBytes: 1024,
codec: codec,
radio: _knownRadio,
),
),
child: const Text('open'),
),
),
),
),
);
+4
View File
@@ -9,6 +9,10 @@ Dart side is checked against.
They are committed here so `test/services/golden/` is reproducible. Nothing in
the app runs them.
To measure reconstruction quality and byte cost instead, use the
[evaluation harness](eval/README.md), which runs on the downloaded model bundle
and needs no research checkout.
| script | produces | used by |
|---|---|---|
| `export_golden.py` | `golden/aeic_cdf_ft32.bin`, `golden/vectors/*.gv` | `rans_coder_test.dart`, `entropy_tables_test.dart` |
+115
View File
@@ -0,0 +1,115 @@
# AEIC quality evaluation
`eval.py` measures how good received images are and how many bytes they cost,
so codec changes can be compared with numbers rather than by eye. It runs the
same ONNX graphs the app downloads. Unlike the fixture scripts in the parent
directory, it needs no research checkout.
For each photo and each preprocessing variant it:
1. prepares the 512×512 codec input (the `stretch` variant mirrors the app's `_toSquareRgb`),
2. runs `aeic_entropy_side_fp32_op17.onnx` on CPU,
3. counts the exact bytes with `aeic_bytes.dart`, which calls the app's own entropy layer and rANS coder,
4. runs the decoder on `y_hat` (GPU when available),
5. restores the aspect ratio the way the receiver does and scores the result against the region that was sent.
## Setup
```bash
python3 -m venv ~/.venvs/aeic-eval # or: uv venv ~/.venvs/aeic-eval
~/.venvs/aeic-eval/bin/pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128
~/.venvs/aeic-eval/bin/pip install -r tools/aeic/eval/requirements.txt
```
You also need `dart` on PATH (`flutter pub get` already run), and the model
bundle. By default the bundle is read from `~/Documents/image_codec_models`;
use `--models` to point elsewhere. The first run downloads the LPIPS, DISTS and
CLIP weights into the torch and Hugging Face caches.
Check that the byte counter matches the reference coder:
```bash
dart run tools/aeic/eval/aeic_bytes.dart --selftest test/services/golden/e2e/*.aeicrec
```
## Running
From the repository root:
```bash
# Kodak (24 images, downloaded once)
~/.venvs/aeic-eval/bin/python -I tools/aeic/eval/eval.py \
--download-kodak ~/aeic-data/kodak --out ~/aeic-runs/kodak-baseline \
--variants stretch,crop,crop43,pad
# Your own photos; Kodak alone is not representative of what people send
~/.venvs/aeic-eval/bin/python -I tools/aeic/eval/eval.py \
--images ~/aeic-data/mesh-photos --out ~/aeic-runs/mesh-baseline
```
| option | meaning |
|---|---|
| `--variants` | comma list of variants to compare (below) |
| `--rois FILE` | JSON `{image stem: [l, t, r, b]}` as fractions; adds subject-region (`roi_*`) scores and limits the run to listed images. `rois/kodak.json` has 17 Kodak subjects |
| `--display N` | long side of the receiver view that is scored (default 512; use 1024 for two-part variants) |
| `--decoder` | score a different decoder export, e.g. fp16 instead of int8 |
| `--limit N` | first N images only |
| `--cpu` | run the decoder on CPU (much slower) |
| variant | what is sent |
|---|---|
| `stretch` | the whole photo stretched to the square, as the app does |
| `crop`, `crop43`, `pad` | centre square crop; crop to at most 4:3 then stretch; letterbox on grey |
| `reroll1`..`reroll5` | the whole photo with the send sheet's "Try another version" tweaks (`kImageSendRerollTweaks`; keep the two tables in step) |
| `select`, `select_sq` | only the `--rois` box; the box grown to a square |
| `q<step>` | quantisation step `<step>` instead of 1 (e.g. `q0.7` spends more bytes) |
| `fit<bytes>` | per image, the finest step from the 16-step ladder that fits `<bytes>` (e.g. `fit157` = one packet) |
| `hd_tiles`, `hd_detail` | two encodes per photo: two overlapping halves; or the whole photo plus the `--rois` box, blended |
`q`/`fit` re-run the four context stages with the decode-side entropy graph;
at step 1 they reproduce the app's bytes exactly. Two-part byte counts include
the extra stream header such a format would need.
To add an experiment, write a function returning `Prepared` and register it in
`VARIANTS`.
## Comparing with JPEG XL and AVIF
`compare_codecs.py` sweeps AEIC over every quantisation step and fits JPEG XL
and AVIF to the same budgets (16–192 px thumbnails, best size per image, scored
after upscaling). AVIF is counted both as a real `.avif` file and as its AV1
data alone. Needs `imagecodecs` (in `requirements.txt`).
```bash
~/.venvs/aeic-eval/bin/python -I tools/aeic/eval/compare_codecs.py \
--images ~/aeic-data/kodak --out ~/aeic-runs/rd
```
## Output
| file | contents |
|---|---|
| `report.html` | source next to each variant's reconstruction, with bytes, packets, LPIPS and CLIP; a red border marks CLIP < 0.80 |
| `results.csv` | one row per image × variant |
| `summary.json` | per-variant mean/median/min/max, packet histogram, run metadata |
| `recon/`, `ref/` | the scored images, at the receiver's aspect ratio |
| `tensors/*.aeic` | the bitstreams, exactly as the app would send them |
## Reading the metrics
| metric | better | what it catches |
|---|---|---|
| `bytes`, `data_chunks` | lower | airtime; one chunk carries up to 157 image bytes |
| `lpips`, `dists` | lower | perceptual difference; the main quality score for a generative decoder |
| `clip` | higher | whether the reconstruction shows the same *things* as the sent region; low values are the "fox came back as a sheep" failures |
| `clip_full` | higher | the same, against the whole original photo, so a crop pays for what it dropped |
| `psnr`, `ms_ssim` | higher | pixel fidelity; included for reference, but weak signals for this codec |
`1-pkt` in the console summary is the share of images that fit in a single data
chunk. Each image also gets one parity packet when recovery is on.
## Caveats
- Desktop ORT on CPU is not bit-identical to ORT on a phone, so an occasional rounding difference can change a byte count by one. The byte counter itself matches the reference coder exactly (see `--selftest`).
- The `stretch` resize approximates `dart:ui`'s `FilterQuality.medium`; it is not pixel-identical.
- CLIP and LPIPS are proxies. Look at the images flagged in `report.html` before drawing conclusions.
+240
View File
@@ -0,0 +1,240 @@
// Byte counter for the AEIC evaluation harness (`eval.py`).
//
// Python runs the encode-side ONNX graph; this turns its tensors into the exact
// bitstream the app would send, using the app's own entropy layer and rANS
// coder rather than a port. A port could drift; this cannot, because it is the
// shipping code.
//
// Usage (from the repository root):
//
// dart run tools/aeic/eval/aeic_bytes.dart <cdf_table.bin> <tensors.f32>...
// dart run tools/aeic/eval/aeic_bytes.dart --selftest <recording.aeicrec>...
//
// Each `.f32` file holds little-endian float32 tensors back to back, in this
// order: z_q [1,128,4,4], yq0..yq3 [1,256,16,16], sc0..sc3 [1,256,16,16]. The
// bitstream is written next to it as `.aeic`, and one JSON object per input is
// printed to stdout.
//
// `--selftest` replays the golden `.aeicrec` recordings in
// `test/services/golden/e2e/` and checks that the output is byte-identical to
// the reference C++ bitstream, which shows the harness counts what the app
// sends.
import 'dart:convert';
import 'dart:io';
import 'dart:typed_data';
import 'package:meshcore_open/services/entropy_tables.dart';
import 'package:meshcore_open/services/image_chunk_transport.dart';
import 'package:meshcore_open/services/image_codec_entropy.dart';
import 'package:meshcore_open/services/rans_coder.dart';
const int _resolution = 512;
Future<void> main(List<String> args) async {
if (args.isEmpty) {
stderr.writeln(
'usage: aeic_bytes.dart <cdf_table.bin> <tensors.f32>...\n'
' aeic_bytes.dart --selftest <recording.aeicrec>...',
);
exitCode = 64;
return;
}
if (args.first == '--selftest') {
exitCode = await _selfTest(args.skip(1).toList());
return;
}
final tables = EntropyTables.parse(File(args.first).readAsBytesSync());
final geometry = AeicEntropyGeometry.forResolution(_resolution);
for (final path in args.skip(1)) {
final tensors = _readTensorFile(path, geometry);
final stream = await _encode(tables, geometry, tensors);
final outPath = path.replaceFirst(RegExp(r'\.f32$'), '.aeic');
File(outPath).writeAsBytesSync(stream);
stdout.writeln(
jsonEncode(<String, Object>{
'file': path,
'bitstream': outPath,
'bytes': stream.length,
'data_chunks': imageDataChunkCount(stream.length),
}),
);
}
}
Future<Uint8List> _encode(
EntropyTables tables,
AeicEntropyGeometry geometry,
AeicEncodeSideTensors tensors,
) {
final codec = AeicEntropyCodec(
geometry: geometry,
network: _FixedNetwork(tensors),
coders: _Coders(tables),
);
// The network ignores its input; encode() only needs a correctly sized RGB
// buffer to get past its own conversion step.
return codec.encode(Uint8List(_resolution * _resolution * 3));
}
AeicEncodeSideTensors _readTensorFile(
String path,
AeicEntropyGeometry geometry,
) {
final bytes = File(path).readAsBytesSync();
final z = geometry.zElements;
final y = geometry.yElements;
final expected = (z + 8 * y) * 4;
if (bytes.length != expected) {
throw FormatException(
'$path is ${bytes.length} bytes, expected $expected '
'(z_q + 4 yq + 4 sc as float32)',
);
}
final all = Float32List.sublistView(bytes);
Float32List at(int offset, int length) =>
Float32List.sublistView(all, offset, offset + length);
return AeicEncodeSideTensors(
zQ: at(0, z),
yQ: <Float32List>[for (var i = 0; i < 4; i++) at(z + i * y, y)],
scales: <Float32List>[for (var i = 0; i < 4; i++) at(z + (4 + i) * y, y)],
);
}
Future<int> _selfTest(List<String> recordings) async {
if (recordings.isEmpty) {
stderr.writeln('--selftest needs at least one .aeicrec file');
return 64;
}
final geometry = AeicEntropyGeometry.forResolution(_resolution);
var failures = 0;
for (final path in recordings) {
final r = _Recording.load(path);
final tables = EntropyTables.parse(
File(
'${File(path).parent.parent.path}/${r.meta['cdf_table']}',
).readAsBytesSync(),
);
final tensors = AeicEncodeSideTensors(
zQ: r.f32('enc/z_q'),
yQ: <Float32List>[for (var i = 0; i < 4; i++) r.f32('enc/yq$i')],
scales: <Float32List>[for (var i = 0; i < 4; i++) r.f32('enc/sc$i')],
);
final got = await _encode(tables, geometry, tensors);
final want = r.u8('enc/bitstream');
final ok =
got.length == want.length &&
Iterable<int>.generate(got.length).every((i) => got[i] == want[i]);
if (!ok) failures++;
stdout.writeln(
'${ok ? 'ok ' : 'FAIL'} ${r.meta['stem']}: ${got.length} B '
'(reference ${want.length} B)',
);
}
return failures == 0 ? 0 : 1;
}
class _FixedNetwork implements AeicEntropyNetwork {
_FixedNetwork(this.tensors);
final AeicEncodeSideTensors tensors;
@override
bool get supportsDecodeSide => false;
@override
Future<AeicEncodeSideTensors> runEncodeSide(Float32List imageChw) async =>
tensors;
@override
Future<Float32List> runHyperSynthesis(Float32List zQ) =>
throw UnsupportedError('encode only');
@override
Future<AeicStageParams> runStage(int stage, Float32List base) =>
throw UnsupportedError('encode only');
}
class _Coders implements AeicRansCoderFactory {
_Coders(this.tables);
final EntropyTables tables;
@override
AeicRansEncoder createEncoder() => _Encoder(RansEncoder(tables));
@override
AeicRansDecoder createDecoder(Uint8List bitstream) =>
throw UnsupportedError('encode only');
}
class _Encoder implements AeicRansEncoder {
_Encoder(this._encoder);
final RansEncoder _encoder;
@override
void pushSymbols(Int16List symbols, Int16List indexes, int cdfGroup) =>
_encoder.encodeWithIndexes(symbols, indexes, cdfGroup);
@override
Uint8List finish() => _encoder.finish();
}
/// Minimal reader for the `.aeicrec` container (see
/// `test/services/image_codec_e2e_test.dart` for the full one).
class _Recording {
_Recording(this.index, this.bytes)
: _entries = <String, Map<String, dynamic>>{
for (final e
in (index['entries'] as List<dynamic>).cast<Map<String, dynamic>>())
e['name'] as String: e,
};
final Map<String, dynamic> index;
final Uint8List bytes;
final Map<String, Map<String, dynamic>> _entries;
static _Recording load(String path) {
final bytes = File(path).readAsBytesSync();
final bd = ByteData.sublistView(bytes);
if (ascii.decode(bytes.sublist(0, 8)) != 'AEICREC1') {
throw FormatException('bad .aeicrec magic in $path');
}
final indexOffset = bd.getUint64(16, Endian.little);
final indexLength = bd.getUint32(24, Endian.little);
final index =
jsonDecode(
utf8.decode(
bytes.sublist(indexOffset, indexOffset + indexLength),
),
)
as Map<String, dynamic>;
return _Recording(index, bytes);
}
Map<String, dynamic> get meta => index['meta'] as Map<String, dynamic>;
Map<String, dynamic> _entry(String name) {
final e = _entries[name];
if (e == null) throw StateError('no entry "$name" in recording');
return e;
}
Float32List f32(String name) {
final e = _entry(name);
final offset = e['offset'] as int;
return Float32List.sublistView(
bytes,
offset,
offset + (e['length'] as int),
);
}
Uint8List u8(String name) {
final e = _entry(name);
final offset = e['offset'] as int;
return Uint8List.sublistView(bytes, offset, offset + (e['length'] as int));
}
}
+256
View File
@@ -0,0 +1,256 @@
#!/usr/bin/env python3
"""Rate-distortion comparison: AEIC (as shipped and with step rate control)
against JPEG XL and AVIF at the same byte budgets.
python -I tools/aeic/eval/compare_codecs.py --images ~/aeic-data/kodak --out ~/aeic-runs/rd
AEIC is swept over every quantisation step in STEP_LADDER (step 1.0 is the
stream the app sends today) and also fitted per image to 1 and 2 packets.
JPEG XL and AVIF are fitted per image to each byte budget: for every candidate
size the quality setting is searched to fit, the result is decoded and
upscaled to the display size, and the size with the lowest LPIPS is kept. That
choice uses the original, which the sender has, so it is a fair best case for
the conventional codecs rather than an oracle.
AVIF is counted twice. `avif` is the real .avif file, whose ftyp+meta boxes
cost about 275 bytes before any image data. `avif_av1` counts only the AV1
payload (the mdat contents), as if the receiver rebuilt the fixed container
from a template; that is AVIF's best case for a custom wire format.
Everything is scored against the source at the same display size (long side
512, the AEIC output size). Needs `imagecodecs` with JPEG XL support on top of
the eval.py requirements.
"""
from __future__ import annotations
import argparse
import importlib.util
import json
import struct
import sys
from pathlib import Path
import numpy as np
from PIL import Image
HERE = Path(__file__).resolve().parent
_spec = importlib.util.spec_from_file_location("aeic_eval", HERE / "eval.py")
ev = importlib.util.module_from_spec(_spec)
sys.modules["aeic_eval"] = ev # dataclasses look their module up here
_spec.loader.exec_module(ev) # type: ignore[union-attr]
import imagecodecs # noqa: E402 (after eval.py, which loads torch first)
from concurrent.futures import ThreadPoolExecutor # noqa: E402
POOL = ThreadPoolExecutor(max_workers=12)
PACKET_BUDGETS = {"1 packet": ev.CHUNK_FIRST_CAPACITY,
"2 packets": ev.CHUNK_FIRST_CAPACITY + ev.CHUNK_CAPACITY}
BUDGETS = [100, 157, 200, 250, 315, 400, 500, 700]
SIZES = [16, 24, 32, 40, 48, 56, 64, 80, 96, 128, 160, 192]
JXL_EFFORT = 9
AVIF_SPEED = 2
DISPLAY = ev.SIZE
def display_size(img: Image.Image, long_side: int) -> tuple[int, int]:
s = long_side / max(img.size)
return (max(1, round(img.width * s)), max(1, round(img.height * s)))
def mdat_payload(avif: bytes) -> int:
"""Size of the AV1 data in an .avif file (its mdat box contents)."""
i = 0
while i + 8 <= len(avif):
size, kind = struct.unpack(">I4s", avif[i:i + 8])
header = 8
if size == 1:
size = struct.unpack(">Q", avif[i + 8:i + 16])[0]
header = 16
elif size == 0:
size = len(avif) - i
if kind == b"mdat":
return size - header
i += size
raise ValueError("no mdat box")
def _jxl(arr: np.ndarray, q: float) -> bytes:
return imagecodecs.jpegxl_encode(arr, distance=q, effort=JXL_EFFORT)
def _avif(arr: np.ndarray, q: float) -> bytes:
return imagecodecs.avif_encode(arr, level=int(q), speed=AVIF_SPEED)
# name -> (encode(arr, q), decode, bytes counted, q for the smallest file,
# q for the largest, integer q?)
CODECS = {
"jxl": (_jxl, imagecodecs.jpegxl_decode, len, 25.0, 0.05, False),
"avif": (_avif, imagecodecs.avif_decode, len, 0, 100, True),
"avif_av1": (_avif, imagecodecs.avif_decode, mdat_payload, 0, 100, True),
}
def codec_fit(name: str, src: Image.Image, long_side: int, budget: int) -> tuple[int, Image.Image] | None:
"""Best stream of codec `name` for `src` at `long_side` px within `budget`.
Returns (counted bytes, decoded image), or None if even the lowest quality
does not fit.
"""
enc, dec, count, q_small, q_large, integer = CODECS[name]
small = np.asarray(src.resize(display_size(src, long_side), Image.LANCZOS))
data = enc(small, q_small)
if count(data) > budget:
return None
best = data
lo, hi = q_small, q_large # lo always fits
for _ in range(12):
if integer:
if abs(hi - lo) <= 1:
break
mid = (lo + hi) // 2
else:
mid = (lo * hi) ** 0.5 # JPEG XL distance acts roughly logarithmically
data = enc(small, mid)
if count(data) <= budget:
best, lo = data, mid
else:
hi = mid
decoded = Image.fromarray(np.asarray(dec(best))[..., :3], "RGB")
return count(best), decoded
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--images", type=Path, required=True)
ap.add_argument("--out", type=Path, required=True)
ap.add_argument("--models", type=Path, default=ev.DEFAULT_MODELS)
ap.add_argument("--limit", type=int)
args = ap.parse_args()
images = ev.find_images(args.images, args.limit)
out: Path = args.out
work = out / "tensors"
(out / "img").mkdir(parents=True, exist_ok=True)
work.mkdir(parents=True, exist_ok=True)
import torch
gpu = torch.cuda.is_available()
codec = ev.Codec(args.models, args.models / ev.DECODER_GRAPH, gpu)
metrics = ev.Metrics("cuda" if gpu else "cpu")
# ---- AEIC: every step on the ladder, one Dart run for all byte counts.
srcs, aeic_y = {}, {}
files = []
for path in images:
stem = path.stem
src = ev.load_source(path)
srcs[stem] = src
side = codec.encode_side(ev.variant_stretch(src).square)
for step in ev.STEP_LADDER:
t = codec.encode_with_step(side, step)
tf = work / f"{stem}.s{step:.2f}.f32"
ev.write_tensor_file(tf, t)
aeic_y[(stem, step)] = t["y_hat"]
files.append(tf)
print(f"encoded {stem}", file=sys.stderr)
nbytes = {k: v["bytes"] for k, v in ev.count_bytes(args.models / ev.CDF_TABLE, files).items()}
for tf in files:
tf.unlink()
rows = []
emb = {}
def score(stem: str, shown: Image.Image) -> dict:
src = srcs[stem]
ref = src.resize(display_size(src, DISPLAY), Image.LANCZOS)
if stem not in emb:
emb[stem] = metrics.clip_embed(src)
s = metrics.score(ref, shown.resize(ref.size, Image.BICUBIC), emb[stem])
s.pop("clip_full")
return s
aeic_shown = {}
for path in images:
stem = path.stem
src = srcs[stem]
for step in ev.STEP_LADDER:
recon = codec.decode(aeic_y[(stem, step)])
shown = recon.resize(display_size(src, DISPLAY), Image.LANCZOS)
aeic_shown[(stem, step)] = shown
b = nbytes[str(work / f"{stem}.s{step:.2f}.f32")]
rows.append({"image": stem, "codec": "aeic", "step": step, "bytes": b, **score(stem, shown)})
print(f"scored AEIC {stem}", file=sys.stderr)
# ---- Conventional codecs: per image and budget, the best size that fits.
for name in CODECS:
for path in images:
stem = path.stem
src = srcs[stem]
for budget in BUDGETS:
best = None
# imagecodecs releases the GIL, so the size search runs in threads.
fits = list(POOL.map(lambda ls: codec_fit(name, src, ls, budget), SIZES))
for long_side, fit in zip(SIZES, fits):
if fit is None:
continue
nb, small = fit
shown = small.resize(display_size(src, DISPLAY), Image.BICUBIC)
ref = src.resize(shown.size, Image.LANCZOS)
lp = float(metrics.lpips(metrics._t(shown), metrics._t(ref)))
if best is None or lp < best[0]:
best = (lp, long_side, nb, shown)
if best is None:
continue
_, long_side, nb, shown = best
rows.append({"image": stem, "codec": name, "budget": budget, "px": long_side,
"bytes": nb, **score(stem, shown)})
if budget in PACKET_BUDGETS.values():
shown.save(out / "img" / f"{stem}.{name}{budget}.png")
print(f"scored {name} {stem}", file=sys.stderr)
# ---- AEIC fitted to the packet budgets, picked from the sweep.
for stem in srcs:
for label, budget in PACKET_BUDGETS.items():
cands = [r for r in rows if r["codec"] == "aeic" and r["image"] == stem and r["bytes"] <= budget]
if not cands:
cands = [max((r for r in rows if r["codec"] == "aeic" and r["image"] == stem),
key=lambda r: r["step"])]
pick = min(cands, key=lambda r: r["step"])
rows.append({**pick, "codec": f"aeic_fit{budget}", "budget": budget})
aeic_shown[(stem, pick["step"])].save(out / "img" / f"{stem}.aeic{budget}.png")
aeic_shown[(stem, 1.0)].save(out / "img" / f"{stem}.aeic_today.png")
srcs[stem].resize(display_size(srcs[stem], DISPLAY), Image.LANCZOS).save(out / "img" / f"{stem}.src.png")
(out / "rows.json").write_text(json.dumps(rows, indent=0))
def mean(rs, k):
return float(np.mean([r[k] for r in rs]))
def line(name, rs):
return (f"{name:<22}{mean(rs, 'bytes'):>7.0f}{mean(rs, 'lpips'):>8.3f}{mean(rs, 'dists'):>8.3f}"
f"{mean(rs, 'clip'):>7.3f}{mean(rs, 'ms_ssim'):>8.3f}{mean(rs, 'psnr'):>7.2f}")
print(f"\n{'':<22}{'bytes':>7}{'lpips':>8}{'dists':>8}{'clip':>7}{'msssim':>8}{'psnr':>7}")
print(line("AEIC today (step 1)", [r for r in rows if r["codec"] == "aeic" and r["step"] == 1.0]))
for label, budget in PACKET_BUDGETS.items():
print(f"-- {label} ({budget} B)")
print(line(" AEIC step-fit", [r for r in rows if r["codec"] == f"aeic_fit{budget}"]))
for name in CODECS:
rs = [r for r in rows if r["codec"] == name and r["budget"] == budget]
print(line(f" {name} ({len(rs)} img)", rs) if rs else f" {name}: does not fit")
print("\nAEIC curve")
for step in ev.STEP_LADDER:
print(line(f" step {step:.2f}", [r for r in rows if r["codec"] == "aeic" and r["step"] == step]))
for name in CODECS:
print(f"{name} curve")
for budget in BUDGETS:
rs = [r for r in rows if r["codec"] == name and r["budget"] == budget]
if rs:
print(line(f" <= {budget} B ({len(rs)} img)", rs))
if __name__ == "__main__":
main()
+831
View File
@@ -0,0 +1,831 @@
#!/usr/bin/env python3
"""Quality/rate evaluation harness for the AEIC image codec.
Runs a folder of photos through the same three ONNX graphs the app ships,
counts the exact bytes with the app's own Dart rANS coder (aeic_bytes.dart),
and scores what the receiver would see.
python -I tools/aeic/eval/eval.py --images ~/photos --out /tmp/aeic-eval
See README.md in this directory for setup and how to read the output.
"""
from __future__ import annotations
import argparse
import csv
import datetime as dt
import hashlib
import html
import json
import math
import os
import shutil
import subprocess
import sys
import time
import urllib.request
from dataclasses import dataclass
from pathlib import Path
from typing import Callable
import numpy as np
from PIL import Image, ImageOps
# torch must be imported before onnxruntime so the CUDA libraries that ship in
# the torch wheels are already loaded when ORT's CUDA provider looks for them.
import torch # noqa: E402 (import order is deliberate)
import onnxruntime as ort # noqa: E402
SIZE = 512
REPO = Path(__file__).resolve().parents[3]
DART_TOOL = Path(__file__).resolve().parent / "aeic_bytes.dart"
DEFAULT_MODELS = Path.home() / "Documents" / "image_codec_models"
SIDE_GRAPH = "aeic_entropy_side_fp32_op17.onnx"
DECODER_GRAPH = "aeic_decoder_qdq_conv_pct.onnx"
CDF_TABLE = "aeic_cdf_ft32.bin"
CONTEXT_GRAPH = "aeic_entropy_decode_fp32_op17.onnx"
def _stage_masks() -> list[np.ndarray]:
"""The four checkerboard masks, as AeicMaskSet.isLive defines them."""
perm = [0, 3, 2, 1]
c = np.arange(256)[:, None, None] // 64
yy = np.arange(16)[None, :, None]
xx = np.arange(16)[None, None, :]
micro = ((yy & 1) << 1) | (xx & 1)
return [((c ^ perm[s]) == micro)[None] for s in range(4)]
STAGE_MASKS = _stage_masks()
# Quantisation steps a stream could signal in 4 bits (the metadata byte's rate
# and resolution fields). 1.0 is today's ft32 stream.
STEP_LADDER = [0.40, 0.45, 0.50, 0.55, 0.60, 0.66, 0.72, 0.79,
0.86, 0.93, 1.00, 1.10, 1.20, 1.32, 1.45, 1.60]
IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".webp", ".bmp", ".tif", ".tiff"}
# Mirrors lib/services/image_chunk_transport.dart. aeic_bytes.dart reports the
# authoritative count; these are only used for the summary's packet totals.
PARITY_CHUNKS = 1
# --------------------------------------------------------------------------
# Preprocessing variants
# --------------------------------------------------------------------------
#
# A variant turns the (EXIF-rotated) source photo into the 512x512 codec input
# and says which part of the source it represents, so the reconstruction can
# be compared with that region at its true aspect ratio, which is what the
# receiver displays.
@dataclass
class Prepared:
square: Image.Image # 512x512 RGB, what the encoder sees
region: tuple[int, int, int, int] # source box (l, t, r, b) it represents
content_box: tuple[int, int, int, int] # box inside `square` holding it
# For multi-part variants: width in source pixels of the cross-fade along
# this part's edges that lie inside the picture (not on its border).
feather: int = 0
def _resize(img: Image.Image, size: tuple[int, int]) -> Image.Image:
# The app draws with dart:ui FilterQuality.medium: bilinear sampling over
# mipmaps. A box pre-reduce followed by bilinear is the closest PIL match.
return img.resize(size, Image.BILINEAR, reducing_gap=2.0)
def _full(img: Image.Image) -> tuple[int, int, int, int]:
return (0, 0, img.width, img.height)
def variant_stretch(img: Image.Image) -> Prepared:
"""What the app does today: the whole frame squeezed into the square."""
return Prepared(_resize(img, (SIZE, SIZE)), _full(img), (0, 0, SIZE, SIZE))
def _centre_crop_box(img: Image.Image, max_ratio: float) -> tuple[int, int, int, int]:
w, h = img.size
ratio = w / h
if ratio > max_ratio:
cw = round(h * max_ratio)
left = (w - cw) // 2
return (left, 0, left + cw, h)
if ratio < 1 / max_ratio:
ch = round(w * max_ratio)
top = (h - ch) // 2
return (0, top, w, top + ch)
return _full(img)
def variant_crop(img: Image.Image) -> Prepared:
"""Centre square crop: no distortion, loses the edges of the frame."""
box = _centre_crop_box(img, 1.0)
return Prepared(_resize(img.crop(box), (SIZE, SIZE)), box, (0, 0, SIZE, SIZE))
def variant_crop43(img: Image.Image) -> Prepared:
"""Crop to at most 4:3 (or 3:4), then stretch what is left."""
box = _centre_crop_box(img, 4 / 3)
return Prepared(_resize(img.crop(box), (SIZE, SIZE)), box, (0, 0, SIZE, SIZE))
def variant_pad(img: Image.Image) -> Prepared:
"""Letterbox on mid-grey: no distortion, spends latent area on the bars."""
w, h = img.size
scale = SIZE / max(w, h)
nw, nh = max(1, round(w * scale)), max(1, round(h * scale))
canvas = Image.new("RGB", (SIZE, SIZE), (128, 128, 128))
left, top = (SIZE - nw) // 2, (SIZE - nh) // 2
canvas.paste(_resize(img, (nw, nh)), (left, top))
return Prepared(canvas, _full(img), (left, top, left + nw, top + nh))
# Re-roll tweaks: (zoom, dx, dy) as fractions of the source. Mirrors
# kImageSendRerollTweaks / rerollSourceRect in
# lib/widgets/image_send_codec_binding.dart; keep the two in step.
REROLL_TWEAKS = [
(1.0, 0.0, 0.0),
(1.02, 0.005, 0.0),
(1.02, -0.005, 0.005),
(1.04, 0.0, -0.005),
(1.03, 0.01, 0.01),
(1.03, -0.01, -0.01),
]
def reroll_source_rect(w: int, h: int, variant: int) -> tuple[float, float, float, float]:
"""Source rect for re-roll `variant`: a near-invisible zoom and shift."""
zoom, dx, dy = REROLL_TWEAKS[variant]
cw, ch = w / zoom, h / zoom
left = min(max((w - cw) / 2 + dx * w, 0.0), w - cw)
top = min(max((h - ch) / 2 + dy * h, 0.0), h - ch)
return (left, top, left + cw, top + ch)
def variant_reroll(img: Image.Image, variant: int) -> Prepared:
"""Whole photo with re-roll tweak `variant`, stretched like `stretch`.
Scored against the full frame: the tweak only drops a percent or two at
the edges, which is what the sender sees as "the whole photo".
"""
l, t, r, b = reroll_source_rect(img.width, img.height, variant)
# PIL's resize box takes float source coordinates, like drawImageRect.
square = img.resize((SIZE, SIZE), Image.BILINEAR, box=(l, t, r, b), reducing_gap=2.0)
return Prepared(square, _full(img), (0, 0, SIZE, SIZE))
VARIANTS: dict[str, Callable[[Image.Image], Prepared]] = {
"stretch": variant_stretch,
"crop": variant_crop,
"crop43": variant_crop43,
"pad": variant_pad,
**{f"reroll{k}": (lambda img, k=k: variant_reroll(img, k)) for k in range(1, len(REROLL_TWEAKS))},
}
# Need a box per image (--rois), so they are not in VARIANTS.
SELECT = "select"
SELECT_SQUARE = "select_sq"
SELECT_VARIANTS = (SELECT, SELECT_SQUARE)
Box = tuple[int, int, int, int]
def roi_box(img: Image.Image, frac: list[float]) -> Box:
"""Fractional [l, t, r, b] -> pixel box in the (EXIF-rotated) source."""
l, t, r, b = frac
w, h = img.size
return (round(l * w), round(t * h), round(r * w), round(b * h))
def variant_select(img: Image.Image, box: Box) -> Prepared:
"""The user's selection, stretched into the square like `stretch`.
The aspect nibble restores the box's shape on the receiver, so this is the
app's existing pipeline applied to a crop the user chose.
"""
return Prepared(_resize(img.crop(box), (SIZE, SIZE)), box, (0, 0, SIZE, SIZE))
def square_box(img: Image.Image, box: Box) -> Box:
"""Grow `box` to a square around its centre, shifted to stay in the image.
Models a selection tool locked to 1:1, so the codec never sees a stretch.
Capped at the image's short side.
"""
l, t, r, b = box
side = min(max(r - l, b - t), img.width, img.height)
cx, cy = (l + r) / 2, (t + b) / 2
nl = int(round(min(max(cx - side / 2, 0), img.width - side)))
nt = int(round(min(max(cy - side / 2, 0), img.height - side)))
return (nl, nt, nl + side, nt + side)
# --------------------------------------------------------------------------
# HD: two encoded parts per photo, aimed at a two-packet budget
# --------------------------------------------------------------------------
#
# The model is fixed at 512x512 / ft32, so "HD" spends the second packet on a
# second 512x512 encode. The parts travel as one stream; the receiver decodes
# both and blends them. Overheads below are what the stream would need on top
# of the bitstreams (a hypothetical layout, not implemented in the app).
HD_TILES = "hd_tiles"
HD_DETAIL = "hd_detail"
HD_VARIANTS = (HD_TILES, HD_DETAIL)
HD_OVERHEAD = {
HD_TILES: 1, # length of part 0, so the receiver can split the stream
HD_DETAIL: 1 + 4, # that, plus the detail box quantised to 4 bytes
}
TILE_OVERLAP = 0.08 # of the long side, cross-faded at the seam
def variant_hd_tiles(img: Image.Image) -> list[Prepared]:
"""Two overlapping halves along the long side, each encoded at 512x512.
Doubles the pixel count. A 3:2 photo splits into two near-square halves,
so each half is also stretched far less than the whole frame would be.
"""
w, h = img.size
long_side = max(w, h)
ov = round(long_side * TILE_OVERLAP)
half = (long_side + ov) // 2
if w >= h:
boxes = [(0, 0, half, h), (w - half, 0, w, h)]
else:
boxes = [(0, 0, w, half), (0, h - half, w, h)]
return [Prepared(_resize(img.crop(b), (SIZE, SIZE)), b, (0, 0, SIZE, SIZE), feather=ov)
for b in boxes]
def variant_hd_detail(img: Image.Image, box: Box) -> list[Prepared]:
"""The whole photo, plus the selected subject blended in at higher detail."""
detail = variant_select(img, box)
detail.feather = max(4, round(0.08 * min(box[2] - box[0], box[3] - box[1])))
return [variant_stretch(img), detail]
def composite(src: Image.Image, parts: list[Prepared], recons: list[Image.Image],
long_side: int) -> tuple[Image.Image, Image.Image, Box]:
"""Reference and receiver view for a variant made of one or more parts.
The view covers the union of the parts' regions at `long_side` pixels on
its long edge. Parts are laid down in order; each later part fades in over
`feather` pixels along its edges that are inside the union, so seams
cross-fade instead of cutting hard.
"""
region = (min(p.region[0] for p in parts), min(p.region[1] for p in parts),
max(p.region[2] for p in parts), max(p.region[3] for p in parts))
rw, rh = region[2] - region[0], region[3] - region[1]
s = long_side / max(rw, rh)
size = (max(1, round(rw * s)), max(1, round(rh * s)))
ref = src.crop(region).resize(size, Image.LANCZOS)
canvas = np.zeros((size[1], size[0], 3), dtype=np.float64)
for i, (p, recon) in enumerate(zip(parts, recons)):
l, t, r, b = p.region
x0, y0 = round((l - region[0]) * s), round((t - region[1]) * s)
x1, y1 = round((r - region[0]) * s), round((b - region[1]) * s)
tile = np.asarray(recon.crop(p.content_box).resize((x1 - x0, y1 - y0), Image.LANCZOS),
dtype=np.float64)
alpha = np.ones((y1 - y0, x1 - x0), dtype=np.float64)
f = min(round(p.feather * s), alpha.shape[0], alpha.shape[1])
if i > 0 and f > 0:
ramp = np.arange(1, f + 1, dtype=np.float64) / f
if l > region[0]:
alpha[:, :f] *= ramp[None, :]
if r < region[2]:
alpha[:, -f:] *= ramp[::-1][None, :]
if t > region[1]:
alpha[:f, :] *= ramp[:, None]
if b < region[3]:
alpha[-f:, :] *= ramp[::-1][:, None]
a = alpha[..., None]
canvas[y0:y1, x0:x1] = canvas[y0:y1, x0:x1] * (1 - a) + tile * a
shown = Image.fromarray(np.clip(np.rint(canvas), 0, 255).astype(np.uint8), "RGB")
return ref, shown, region
# Mirrors imageDataChunkCount() in image_chunk_transport.dart.
CHUNK_FIRST_CAPACITY = 157
CHUNK_CAPACITY = 158
def parse_rate_variant(v: str) -> dict | None:
"""`q0.7` -> fixed step 0.7; `fit315` -> per-image step to fit 315 bytes."""
try:
if v.startswith("q"):
return {"steps": [float(v[1:])], "budget": None}
if v.startswith("fit"):
return {"steps": STEP_LADDER, "budget": int(v[3:])}
except ValueError:
return None
return None
def data_chunks(payload_bytes: int) -> int:
n = max(payload_bytes, 0)
if n <= CHUNK_FIRST_CAPACITY:
return 1
return 1 + math.ceil((n - CHUNK_FIRST_CAPACITY) / CHUNK_CAPACITY)
# --------------------------------------------------------------------------
# Codec
# --------------------------------------------------------------------------
class Codec:
def __init__(self, models: Path, decoder: Path, gpu: bool) -> None:
cpu = ["CPUExecutionProvider"]
cuda = ["CUDAExecutionProvider", "CPUExecutionProvider"] if gpu else cpu
# The entropy graph stays on CPU: its rounded outputs decide the bytes,
# and CPU fp32 is the closest stand-in for the phone. It costs ~0.1 s.
self.side = ort.InferenceSession(str(models / SIDE_GRAPH), providers=cpu)
self.context = ort.InferenceSession(str(models / CONTEXT_GRAPH), providers=cpu)
self.decoder = ort.InferenceSession(str(decoder), providers=cuda)
self.decoder_provider = self.decoder.get_providers()[0]
def encode_side(self, square: Image.Image) -> dict[str, np.ndarray]:
rgb = np.asarray(square, dtype=np.uint8)
# Same float32 steps as aeicRgbToChw in image_codec_entropy.dart.
chw = rgb.transpose(2, 0, 1).astype(np.float32) / np.float32(255.0)
chw = (chw - np.float32(0.5)) * np.float32(2.0)
names = (["z_q", "y_hat", "y", "base0"]
+ [f"yq{i}" for i in range(4)] + [f"sc{i}" for i in range(4)])
out = self.side.run(names, {"image": chw[None]})
return dict(zip(names, out))
def encode_with_step(self, side: dict[str, np.ndarray], step: float) -> dict[str, np.ndarray]:
"""Re-quantise y with step `step` instead of 1 (rate control).
Replays the four-stage context loop the receiver runs
(AeicEntropyCodec.decodeToLatent) with the decode-side graph, coding
round((y - mean) / step) against scale / step. The receiver's
reconstruction becomes symbol * step + mean. step < 1 spends more
bytes, step > 1 fewer. At step == 1 this reproduces the side graph's
symbols, which the harness's q1 check relies on.
"""
st = np.float32(step)
y, z_q = side["y"], side["z_q"]
base = side["base0"].copy()
out: dict[str, np.ndarray] = {"z_q": z_q}
for stage in range(4):
means, scales = self.context.run(
["means", "scales"],
{"z_q": z_q, "base": base, "stage": np.array([stage], dtype=np.int32)})
m = STAGE_MASKS[stage]
sym = np.where(m, np.rint((y - means) / st), np.float32(0)).astype(np.float32)
base = np.where(m, sym * st + means, base).astype(np.float32)
out[f"yq{stage}"] = sym
out[f"sc{stage}"] = np.where(m, scales / st, np.float32(0)).astype(np.float32)
out["y_hat"] = base
return out
def decode(self, y_hat: np.ndarray) -> Image.Image:
(out,) = self.decoder.run(["image"], {"y_hat": y_hat.astype(np.float32)})
# Same mapping as _chwFloatsToRgb in image_codec_backend.dart: clamp,
# then round half away from zero (Dart's double.round()).
scaled = (out[0].astype(np.float64) + 1.0) * 127.5
rgb = np.floor(np.clip(scaled, 0, 255) + 0.5).astype(np.uint8)
return Image.fromarray(rgb.transpose(1, 2, 0), "RGB")
def write_tensor_file(path: Path, t: dict[str, np.ndarray]) -> None:
parts = [t["z_q"]] + [t[f"yq{i}"] for i in range(4)] + [t[f"sc{i}"] for i in range(4)]
with open(path, "wb") as f:
for p in parts:
f.write(np.ascontiguousarray(p, dtype="<f4").tobytes())
def count_bytes(cdf: Path, tensor_files: list[Path]) -> dict[str, dict]:
if not tensor_files:
return {}
dart = shutil.which("dart")
if dart is None:
sys.exit("dart is not on PATH; it is needed to run aeic_bytes.dart")
cmd = [dart, "run", str(DART_TOOL), str(cdf), *map(str, tensor_files)]
proc = subprocess.run(cmd, cwd=REPO, capture_output=True, text=True)
if proc.returncode != 0:
sys.exit(f"aeic_bytes.dart failed:\n{proc.stderr}")
results = {}
for line in proc.stdout.splitlines():
line = line.strip()
if line.startswith("{"):
row = json.loads(line)
results[row["file"]] = row
return results
# --------------------------------------------------------------------------
# Metrics
# --------------------------------------------------------------------------
class Metrics:
"""Full-reference scores. Higher is better except LPIPS and DISTS."""
def __init__(self, device: str) -> None:
import open_clip
import piq
self.piq = piq
self.device = device
self.lpips = piq.LPIPS(reduction="none").to(device).eval()
self.dists = piq.DISTS(reduction="none").to(device).eval()
model, _, preprocess = open_clip.create_model_and_transforms(
"ViT-B-32", pretrained="laion2b_s34b_b79k"
)
self.clip = model.to(device).eval()
self.clip_pre = preprocess
def _t(self, img: Image.Image) -> torch.Tensor:
a = np.asarray(img, dtype=np.float32) / 255.0
return torch.from_numpy(a).permute(2, 0, 1)[None].to(self.device)
@torch.no_grad()
def clip_embed(self, img: Image.Image) -> torch.Tensor:
# CLIP's own preprocessing centre-crops to a square, which would hide
# the frame edges -- exactly what crop variants throw away. Letterbox
# first so the whole image is scored.
side = max(img.size)
boxed = Image.new("RGB", (side, side), (0, 0, 0))
boxed.paste(img, ((side - img.width) // 2, (side - img.height) // 2))
x = self.clip_pre(boxed)[None].to(self.device)
e = self.clip.encode_image(x)
return e / e.norm(dim=-1, keepdim=True)
@torch.no_grad()
def score(self, ref: Image.Image, out: Image.Image, full_src_emb: torch.Tensor) -> dict:
r, o = self._t(ref), self._t(out)
out_emb = self.clip_embed(out)
return {
"psnr": float(self.piq.psnr(o, r, data_range=1.0)),
"ms_ssim": float(self.piq.multi_scale_ssim(o, r, data_range=1.0)),
"lpips": float(self.lpips(o, r)),
"dists": float(self.dists(o, r)),
# Semantic agreement with exactly what was sent ...
"clip": float((self.clip_embed(ref) * out_emb).sum()),
# ... and with the whole original photo, so cropping is charged
# for what it threw away.
"clip_full": float((full_src_emb * out_emb).sum()),
}
def roi_pair(src: Image.Image, region: Box, shown: Image.Image, box: Box) -> tuple[Image.Image, Image.Image] | None:
"""The ROI as the receiver would see it after zooming in, plus its reference.
`shown` covers `region` of the source; the ROI is mapped into it and
both crops are scaled to long side 512. For a full-frame variant that is an
upscale of a few dozen pixels, which is exactly what zooming in shows.
Returns None when the variant did not send the whole ROI.
"""
rl, rt, rr, rb = region
l, t, r, b = box
if l < rl or t < rt or r > rr or b > rb:
return None
sx = shown.width / (rr - rl)
sy = shown.height / (rb - rt)
out = shown.crop((round((l - rl) * sx), round((t - rt) * sy),
round((r - rl) * sx), round((b - rt) * sy)))
ref = src.crop(box)
scale = SIZE / max(ref.size)
size = (max(1, round(ref.width * scale)), max(1, round(ref.height * scale)))
return ref.resize(size, Image.LANCZOS), out.resize(size, Image.BICUBIC)
# --------------------------------------------------------------------------
# Corpus
# --------------------------------------------------------------------------
KODAK_URL = "https://r0k.us/graphics/kodak/kodak/kodim{:02d}.png"
def download_kodak(dest: Path) -> None:
dest.mkdir(parents=True, exist_ok=True)
for i in range(1, 25):
target = dest / f"kodim{i:02d}.png"
if target.exists():
continue
print(f"downloading {target.name}", file=sys.stderr)
tmp = target.with_suffix(".part")
with urllib.request.urlopen(KODAK_URL.format(i), timeout=60) as r, open(tmp, "wb") as f:
shutil.copyfileobj(r, f)
tmp.rename(target)
def load_source(path: Path) -> Image.Image:
img = Image.open(path)
# Picked photos are shown upright in the app, so honour EXIF orientation.
img = ImageOps.exif_transpose(img)
return img.convert("RGB")
def find_images(folder: Path, limit: int | None) -> list[Path]:
files = sorted(p for p in folder.rglob("*") if p.suffix.lower() in IMAGE_SUFFIXES)
return files[:limit] if limit else files
# --------------------------------------------------------------------------
# Report
# --------------------------------------------------------------------------
METRIC_COLS = ["bytes", "data_chunks", "psnr", "ms_ssim", "lpips", "dists", "clip", "clip_full"]
ROI_COLS = ["roi_psnr", "roi_ms_ssim", "roi_lpips", "roi_dists", "roi_clip"]
LOWER_IS_BETTER = {"bytes", "data_chunks", "lpips", "dists", "roi_lpips", "roi_dists"}
def summarise(rows: list[dict], variants: list[str]) -> dict:
summary = {}
for v in variants:
vr = [r for r in rows if r["variant"] == v]
if not vr:
continue
s: dict = {"images": len(vr)}
for m in METRIC_COLS + ROI_COLS:
vals = np.array([r[m] for r in vr if r.get(m) not in (None, "")], dtype=np.float64)
if vals.size == 0:
continue
s[m] = {"mean": float(vals.mean()), "median": float(np.median(vals)),
"min": float(vals.min()), "max": float(vals.max())}
chunks = [r["data_chunks"] for r in vr]
s["chunk_histogram"] = {str(k): chunks.count(k) for k in sorted(set(chunks))}
s["packets_mean_with_parity"] = float(np.mean(chunks)) + PARITY_CHUNKS
# The share of images whose CLIP agreement with the source is low.
# This is the "fox came back as a sheep" bucket; worth eyeballing.
s["clip_below_0.80"] = sum(1 for r in vr if r["clip"] < 0.80)
summary[v] = s
return summary
def print_summary(summary: dict) -> None:
head = (f"{'variant':<10}" + "".join(f"{m:>12}" for m in METRIC_COLS)
+ f"{'1-pkt':>8}{'<=2-pkt':>9}{'clip<.8':>9}")
print(head)
print("-" * len(head))
for v, s in summary.items():
line = f"{v:<10}" + "".join(f"{s[m]['mean']:>12.4g}" for m in METRIC_COLS)
one = s["chunk_histogram"].get("1", 0)
two = one + s["chunk_histogram"].get("2", 0)
line += f"{one / s['images']:>7.0%} {two / s['images']:>8.0%} {s['clip_below_0.80']:>8}"
print(line)
print("means; lower is better for bytes, data_chunks, lpips, dists")
if any("roi_lpips" in s for s in summary.values()):
print()
print(f"{'variant':<10}" + "".join(f"{m:>13}" for m in ROI_COLS))
for v, s in summary.items():
if "roi_lpips" in s:
print(f"{v:<10}" + "".join(f"{s[m]['mean']:>13.4g}" for m in ROI_COLS))
print("scored on the selected region only, as the receiver sees it after zooming in")
def write_html(out: Path, rows: list[dict], summary: dict, meta: dict) -> None:
def fmt(v: float, m: str) -> str:
return f"{v:.1f}" if m in ("bytes", "data_chunks") else f"{v:.3f}"
variants = list(summary)
stems = list(dict.fromkeys(r["image"] for r in rows))
by = {(r["image"], r["variant"]): r for r in rows}
sum_rows = "".join(
f"<tr><th>{html.escape(v)}</th>"
+ "".join(f"<td>{fmt(summary[v][m]['mean'], m)}</td>" for m in METRIC_COLS)
+ "</tr>"
for v in variants
)
cards = []
for stem in stems:
cells = []
first = by[(stem, variants[0])]
cells.append(
f'<figure><img loading="lazy" src="{html.escape(first["ref_png"])}">'
f"<figcaption>source</figcaption></figure>"
)
for v in variants:
r = by.get((stem, v))
if not r:
continue
warn = " warn" if r["clip"] < 0.80 else ""
cells.append(
f'<figure class="{warn.strip()}"><img loading="lazy" src="{html.escape(r["out_png"])}">'
f"<figcaption><b>{html.escape(v)}</b> {r['bytes']} B / {r['data_chunks']} pkt<br>"
f"LPIPS {r['lpips']:.3f} · CLIP {r['clip']:.3f}</figcaption></figure>"
)
cards.append(f"<section><h3>{html.escape(stem)}</h3><div class=row>{''.join(cells)}</div></section>")
doc = f"""<!doctype html><html lang=en><meta charset=utf-8>
<meta name=viewport content="width=device-width,initial-scale=1">
<title>AEIC eval</title>
<style>
:root{{color-scheme:light dark;--bg:#fff;--fg:#111;--muted:#666;--line:#ddd;--warn:#c62828}}
@media (prefers-color-scheme:dark){{:root{{--bg:#121212;--fg:#eee;--muted:#aaa;--line:#333;--warn:#ef5350}}}}
body{{background:var(--bg);color:var(--fg);font:15px/1.4 system-ui,sans-serif;margin:16px}}
table{{border-collapse:collapse;margin:8px 0 24px}}td,th{{border:1px solid var(--line);padding:4px 8px;text-align:right}}
.row{{display:flex;gap:8px;flex-wrap:wrap}}figure{{margin:0;width:256px}}
figure img{{width:256px;height:auto;display:block;border:2px solid transparent}}
figure.warn img{{border-color:var(--warn)}}figcaption{{font-size:13px;color:var(--muted)}}
code{{font-size:13px}}
</style>
<h1>AEIC eval</h1>
<p><code>{html.escape(json.dumps(meta))}</code></p>
<h2>Means</h2>
<table><tr><th></th>{''.join(f'<th>{m}</th>' for m in METRIC_COLS)}</tr>{sum_rows}</table>
<p>Images are shown at the aspect ratio the receiver restores. A red border means CLIP similarity to the sent region is below 0.80, so check it by eye.</p>
{''.join(cards)}
</html>"""
(out / "report.html").write_text(doc, encoding="utf-8")
# --------------------------------------------------------------------------
# Main
# --------------------------------------------------------------------------
def sha256(path: Path) -> str:
h = hashlib.sha256()
with open(path, "rb") as f:
for block in iter(lambda: f.read(1 << 20), b""):
h.update(block)
return h.hexdigest()
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--images", type=Path, help="folder of photos (searched recursively)")
ap.add_argument("--download-kodak", type=Path, metavar="DIR",
help="download the 24 Kodak images to DIR and use them")
ap.add_argument("--out", type=Path, required=True, help="output folder")
ap.add_argument("--models", type=Path, default=DEFAULT_MODELS,
help=f"model bundle folder (default {DEFAULT_MODELS})")
ap.add_argument("--decoder", type=Path, help="alternative decoder .onnx (e.g. an fp16 export)")
ap.add_argument("--variants", default="stretch",
help=f"comma list from: {', '.join(VARIANTS)} (default stretch)")
ap.add_argument("--limit", type=int, help="only the first N images")
ap.add_argument("--cpu", action="store_true", help="run the decoder on CPU too")
ap.add_argument("--rois", type=Path,
help="JSON {image stem: [l, t, r, b] fractions}; enables the select and "
"hd_detail variants and ROI scores, and limits the run to listed images")
ap.add_argument("--display", type=int, default=SIZE,
help="long side in pixels of the receiver view that is scored (default 512; "
"use 1024 when comparing HD variants)")
args = ap.parse_args()
if args.download_kodak:
download_kodak(args.download_kodak)
args.images = args.images or args.download_kodak
if not args.images:
ap.error("give --images or --download-kodak")
variants = [v.strip() for v in args.variants.split(",") if v.strip()]
unknown = [v for v in variants
if v not in VARIANTS and v not in SELECT_VARIANTS and v not in HD_VARIANTS
and parse_rate_variant(v) is None]
if unknown:
ap.error(f"unknown variants: {unknown}")
rois: dict[str, list[float]] = {}
if args.rois:
rois = {k: v for k, v in json.loads(args.rois.read_text()).items() if not k.startswith("_")}
if any(v in SELECT_VARIANTS + (HD_DETAIL,) for v in variants) and not rois:
ap.error("the select and hd_detail variants need --rois")
images = find_images(args.images, None)
if rois:
images = [p for p in images if p.relative_to(args.images).with_suffix("").as_posix() in rois]
images = images[: args.limit] if args.limit else images
if not images:
sys.exit(f"no images under {args.images}")
out: Path = args.out
work = out / "tensors"
dirs = [work] + [out / k / v for k in ("recon", "ref", "roi") for v in variants]
for d in dirs:
d.mkdir(parents=True, exist_ok=True)
decoder_path = args.decoder or (args.models / DECODER_GRAPH)
gpu = not args.cpu and torch.cuda.is_available()
codec = Codec(args.models, decoder_path, gpu)
metrics = Metrics("cuda" if gpu else "cpu")
print(f"{len(images)} images x {len(variants)} variants; decoder on {codec.decoder_provider}",
file=sys.stderr)
# Pass 1: preprocess + encode side, so every byte count is one Dart run.
jobs = []
for path in images:
src = load_source(path)
key = path.relative_to(args.images).with_suffix("").as_posix()
stem = key.replace("/", "__")
box = roi_box(src, rois[key]) if key in rois else None
for v in variants:
if (v in SELECT_VARIANTS or v == HD_DETAIL) and box is None:
continue
if v == SELECT:
parts = [variant_select(src, box)]
elif v == SELECT_SQUARE:
parts = [variant_select(src, square_box(src, box))]
elif v == HD_TILES:
parts = variant_hd_tiles(src)
elif v == HD_DETAIL:
parts = variant_hd_detail(src, box)
elif parse_rate_variant(v) is not None:
parts = [variant_stretch(src)]
else:
parts = [VARIANTS[v](src)]
rate = parse_rate_variant(v)
steps = rate["steps"] if rate else [None]
t0 = time.perf_counter()
encoded = [] # per part: [(step, y_hat, tensor file), ...]
for i, p in enumerate(parts):
side = codec.encode_side(p.square)
cands = []
for step in steps:
tensors = side if step is None else codec.encode_with_step(side, step)
tag = "" if step is None else f".s{step:.2f}"
tf = work / f"{stem}.{v}.p{i}{tag}.f32"
write_tensor_file(tf, tensors)
cands.append((step, tensors["y_hat"], tf))
encoded.append(cands)
enc_s = time.perf_counter() - t0
jobs.append((path, stem, v, src, parts, box, encoded, enc_s))
byte_rows = count_bytes(args.models / CDF_TABLE,
[tf for j in jobs for cands in j[6] for _, _, tf in cands])
# Pick one candidate per part. A fit<N> variant takes the finest step whose
# stream fits N bytes, or the coarsest step if none does.
for j in jobs:
rate = parse_rate_variant(j[2])
chosen = []
for cands in j[6]:
if rate and rate["budget"] is not None:
fitting = [c for c in cands if byte_rows[str(c[2])]["bytes"] <= rate["budget"]]
chosen.append(min(fitting, key=lambda c: c[0]) if fitting
else max(cands, key=lambda c: c[0]))
else:
chosen.append(cands[0])
j[6][:] = chosen
# Pass 2: decode + score.
rows = []
emb_cache: dict[str, torch.Tensor] = {}
for n, (path, stem, v, src, parts, box, encoded, enc_s) in enumerate(jobs, 1):
t0 = time.perf_counter()
recons = [codec.decode(y_hat) for _, y_hat, _ in encoded]
dec_s = time.perf_counter() - t0
ref, shown, region = composite(src, parts, recons, args.display)
part_bytes = [byte_rows[str(tf)]["bytes"] for _, _, tf in encoded]
total_bytes = sum(part_bytes) + HD_OVERHEAD.get(v, 0)
chunks = data_chunks(total_bytes)
if len(parts) == 1:
assert chunks == byte_rows[str(encoded[0][2])]["data_chunks"], "chunk mirror drifted"
if stem not in emb_cache:
emb_cache[stem] = metrics.clip_embed(src)
scores = metrics.score(ref, shown, emb_cache[stem])
roi = roi_pair(src, region, shown, box) if box else None
if roi:
roi_scores = metrics.score(roi[0], roi[1], emb_cache[stem])
scores.update({f"roi_{k}": roi_scores[k] for k in ("psnr", "ms_ssim", "lpips", "dists", "clip")})
roi[1].save(out / "roi" / v / f"{stem}.png")
roi[0].save(out / "roi" / v / f"{stem}.ref.png")
else:
scores.update({c: "" for c in ROI_COLS})
out_png = Path("recon") / v / f"{stem}.png"
ref_png = Path("ref") / v / f"{stem}.png"
shown.save(out / out_png)
ref.save(out / ref_png)
row = {
"image": stem, "variant": v, "source": str(path),
"width": src.width, "height": src.height,
"bytes": total_bytes, "data_chunks": chunks,
"part_bytes": "+".join(map(str, part_bytes)),
"step": "+".join("1.00" if st is None else f"{st:.2f}" for st, _, _ in encoded),
**scores,
"encode_s": round(enc_s, 3), "decode_s": round(dec_s, 3),
"out_png": out_png.as_posix(), "ref_png": ref_png.as_posix(),
"bitstream": ";".join(os.path.relpath(byte_rows[str(tf)]["bitstream"], out)
for _, _, tf in encoded),
}
rows.append(row)
print(f"[{n}/{len(jobs)}] {stem} {v}: {row['bytes']} B, LPIPS {row['lpips']:.3f}, "
f"CLIP {row['clip']:.3f}, decode {dec_s:.2f}s", file=sys.stderr)
summary = summarise(rows, variants)
meta = {
"date": dt.datetime.now().isoformat(timespec="seconds"),
"images": str(args.images), "count": len(images), "variants": variants,
"decoder": decoder_path.name, "decoder_sha256": sha256(decoder_path)[:16],
"decoder_provider": codec.decoder_provider,
"onnxruntime": ort.__version__,
}
with open(out / "results.csv", "w", newline="") as f:
w = csv.DictWriter(f, fieldnames=list(rows[0]))
w.writeheader()
w.writerows(rows)
(out / "summary.json").write_text(json.dumps({"meta": meta, "variants": summary}, indent=1))
write_html(out, rows, summary, meta)
for tf in work.glob("*.f32"): # 2 MB each; the .aeic bitstreams are kept
tf.unlink()
print()
print_summary(summary)
print(f"\nreport: {out / 'report.html'}")
if __name__ == "__main__":
main()
+10
View File
@@ -0,0 +1,10 @@
# For the CUDA build of torch, install it first from the PyTorch index, e.g.
# pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128
torch
torchvision
onnxruntime-gpu # or onnxruntime for CPU only
numpy
pillow
piq
open_clip_torch
imagecodecs # compare_codecs.py: JPEG XL and AVIF
+20
View File
@@ -0,0 +1,20 @@
{
"_comment": "Subject boxes for the Kodak set, as fractions of width/height: [left, top, right, bottom]. Stand-ins for a user's 'select' crop.",
"kodim02": [0.15, 0.15, 0.70, 0.75],
"kodim03": [0.15, 0.05, 0.65, 0.65],
"kodim04": [0.20, 0.15, 0.80, 0.65],
"kodim05": [0.00, 0.00, 0.50, 1.00],
"kodim06": [0.15, 0.15, 0.80, 0.85],
"kodim09": [0.35, 0.10, 0.95, 0.75],
"kodim11": [0.35, 0.30, 0.85, 0.85],
"kodim12": [0.25, 0.20, 0.65, 0.85],
"kodim14": [0.00, 0.15, 0.90, 0.85],
"kodim15": [0.35, 0.00, 0.90, 0.75],
"kodim16": [0.40, 0.35, 0.90, 0.60],
"kodim17": [0.25, 0.00, 0.75, 0.45],
"kodim18": [0.25, 0.15, 0.85, 1.00],
"kodim19": [0.35, 0.00, 0.85, 0.75],
"kodim21": [0.30, 0.05, 0.70, 0.65],
"kodim22": [0.05, 0.10, 0.60, 0.70],
"kodim23": [0.50, 0.00, 1.00, 0.75]
}
+187
View File
@@ -301,5 +301,192 @@
"settings_exportBookmarksSubtitle",
"settings_exportBookmarksSuccess",
"settings_exportBookmarksNoBookmarks"
"receivedImage_packetEchoes",
"receivedImage_recoveryPacketEchoes",
"imageSend_preparingPreview",
"imageSend_tryAnother",
"imageSend_versionsHint",
"imageSend_versionLabel",
"imageSend_versionPackets"
],
"cs": [
"receivedImage_packetEchoes",
"receivedImage_recoveryPacketEchoes",
"imageSend_preparingPreview",
"imageSend_tryAnother",
"imageSend_versionsHint",
"imageSend_versionLabel",
"imageSend_versionPackets"
],
"de": [
"receivedImage_packetEchoes",
"receivedImage_recoveryPacketEchoes",
"imageSend_preparingPreview",
"imageSend_tryAnother",
"imageSend_versionsHint",
"imageSend_versionLabel",
"imageSend_versionPackets"
],
"es": [
"receivedImage_packetEchoes",
"receivedImage_recoveryPacketEchoes",
"imageSend_preparingPreview",
"imageSend_tryAnother",
"imageSend_versionsHint",
"imageSend_versionLabel",
"imageSend_versionPackets"
],
"fr": [
"receivedImage_packetEchoes",
"receivedImage_recoveryPacketEchoes",
"imageSend_preparingPreview",
"imageSend_tryAnother",
"imageSend_versionsHint",
"imageSend_versionLabel",
"imageSend_versionPackets"
],
"hu": [
"receivedImage_packetEchoes",
"receivedImage_recoveryPacketEchoes",
"imageSend_preparingPreview",
"imageSend_tryAnother",
"imageSend_versionsHint",
"imageSend_versionLabel",
"imageSend_versionPackets"
],
"it": [
"receivedImage_packetEchoes",
"receivedImage_recoveryPacketEchoes",
"imageSend_preparingPreview",
"imageSend_tryAnother",
"imageSend_versionsHint",
"imageSend_versionLabel",
"imageSend_versionPackets"
],
"ja": [
"receivedImage_packetEchoes",
"receivedImage_recoveryPacketEchoes",
"imageSend_preparingPreview",
"imageSend_tryAnother",
"imageSend_versionsHint",
"imageSend_versionLabel",
"imageSend_versionPackets"
],
"ko": [
"receivedImage_packetEchoes",
"receivedImage_recoveryPacketEchoes",
"imageSend_preparingPreview",
"imageSend_tryAnother",
"imageSend_versionsHint",
"imageSend_versionLabel",
"imageSend_versionPackets"
],
"nl": [
"receivedImage_packetEchoes",
"receivedImage_recoveryPacketEchoes",
"imageSend_preparingPreview",
"imageSend_tryAnother",
"imageSend_versionsHint",
"imageSend_versionLabel",
"imageSend_versionPackets"
],
"pl": [
"receivedImage_packetEchoes",
"receivedImage_recoveryPacketEchoes",
"imageSend_preparingPreview",
"imageSend_tryAnother",
"imageSend_versionsHint",
"imageSend_versionLabel",
"imageSend_versionPackets"
],
"pt": [
"receivedImage_packetEchoes",
"receivedImage_recoveryPacketEchoes",
"imageSend_preparingPreview",
"imageSend_tryAnother",
"imageSend_versionsHint",
"imageSend_versionLabel",
"imageSend_versionPackets"
],
"pt_PT": [
"receivedImage_packetEchoes",
"receivedImage_recoveryPacketEchoes",
"imageSend_preparingPreview",
"imageSend_tryAnother",
"imageSend_versionsHint",
"imageSend_versionLabel",
"imageSend_versionPackets"
],
"ru": [
"receivedImage_packetEchoes",
"receivedImage_recoveryPacketEchoes",
"imageSend_preparingPreview",
"imageSend_tryAnother",
"imageSend_versionsHint",
"imageSend_versionLabel",
"imageSend_versionPackets"
],
"sk": [
"receivedImage_packetEchoes",
"receivedImage_recoveryPacketEchoes",
"imageSend_preparingPreview",
"imageSend_tryAnother",
"imageSend_versionsHint",
"imageSend_versionLabel",
"imageSend_versionPackets"
],
"sl": [
"receivedImage_packetEchoes",
"receivedImage_recoveryPacketEchoes",
"imageSend_preparingPreview",
"imageSend_tryAnother",
"imageSend_versionsHint",
"imageSend_versionLabel",
"imageSend_versionPackets"
],
"sv": [
"receivedImage_packetEchoes",
"receivedImage_recoveryPacketEchoes",
"imageSend_preparingPreview",
"imageSend_tryAnother",
"imageSend_versionsHint",
"imageSend_versionLabel",
"imageSend_versionPackets"
],
"uk": [
"receivedImage_packetEchoes",
"receivedImage_recoveryPacketEchoes",
"imageSend_preparingPreview",
"imageSend_tryAnother",
"imageSend_versionsHint",
"imageSend_versionLabel",
"imageSend_versionPackets"
],
"zh": [
"receivedImage_packetEchoes",
"receivedImage_recoveryPacketEchoes",
"imageSend_preparingPreview",
"imageSend_tryAnother",
"imageSend_versionsHint",
"imageSend_versionLabel",
"imageSend_versionPackets"
]
}