diff --git a/documentation/design.md b/documentation/design.md index 231aaaf3..423b53aa 100644 --- a/documentation/design.md +++ b/documentation/design.md @@ -118,6 +118,7 @@ Icon sizes in use: 24 for tile and banner icons, 20 for menu icons and small act | Settings navigation row | `MeshNavTile` | `onTap: null` dims it to 50 % and reports it as disabled; use `subtitle` to say why | | Settings switch | `SwitchListTile` inside `MeshSettingsTileTheme` | Wrap the settings screen body once in `MeshSettingsTileTheme` | | Radio-style choice in a sheet | `MeshSheetOption` | Used by the language picker and map time filter | +| Choosing between images | Selectable thumbnail tiles in a centred `Wrap` | Image send versions (`_versionTile` in `image_send_preview_sheet.dart`). `InkWell` with `MeshRadii.sm`; selected tile gets a 3px `primary` border **and** a `primary` check badge (never colour alone); caption under the tile in `bodySmall`, w600 when selected; figures such as packet counts in `MeshTheme.mono` 12; wrap the tile in `Semantics(button: true, selected:, label:)` | | Popup menu row | **always** `PopupMenuItem(child: MeshMenuItemRow(icon:, label:))` | Label wraps to 2 lines then ellipsizes; `leading:` for `ThemedIcon`; no icon keeps the 20px slot | | Segmented button label | **always** `ButtonSegment(label: MeshSegmentLabel(text))` | Shrinks to 12 (before system scale), then wraps at word boundaries onto 2 centred lines; never breaks a word | | Status | `StatusChip` | Mono label, coloured dot or icon | diff --git a/documentation/image-messages.md b/documentation/image-messages.md index 2cda8bf8..935dd398 100644 --- a/documentation/image-messages.md +++ b/documentation/image-messages.md @@ -15,9 +15,14 @@ Downloads use the configured Hugging Face model bundle, support resume, and veri 1. Open channel chat and tap the image button beside the composer. 2. Pick a photo from your gallery. -3. Wait for encoding. Inspect the source preview and, if available, the reconstructed preview. -4. Review transmitted bytes, packet count, and estimated send duration. Estimates use the current radio settings and are unavailable if those settings are unknown. -5. Choose whether to include a recovery packet, then send. The chat shows per-packet progress. +3. Wait for the preview. The sheet shows **What recipients see**: the photo encoded and then decoded on your device, which is the picture recipients reconstruct. Switch to **Original** to compare. +4. If the reconstruction got something wrong, tap **Try another version**. Each version encodes the whole photo with a tiny, invisible change of framing, and the model reconstructs each one a little differently. Tap the version you want to send; the packet count and time on air follow your choice. Up to six versions can be made, one at a time. +5. Review transmitted bytes, packet count, and estimated send duration. Estimates use the current radio settings and are unavailable if those settings are unknown. +6. Choose whether to include a recovery packet, then send. The chat shows per-packet progress, and your own copy of the image shows the version you chose. + +After sending, your image shows how many times repeaters were heard passing it on, like a channel message (Packet info lists each packet). If repeaters passed on some packets but not others, the missing ones are sent once more automatically. + +On a device that cannot decode, the sheet shows the original photo with a note instead, and versions are not offered. Each new version costs one encode and one decode, so it can take a few seconds on older phones. The current build ships one quality/model setting: **ft32**, with a 512×512 codec reconstruction and metadata used to restore a supported aspect ratio. Reserved rate/resolution codes are not additional selectable quality options. diff --git a/lib/connector/meshcore_connector.dart b/lib/connector/meshcore_connector.dart index 0c1e668f..4b4168e4 100644 --- a/lib/connector/meshcore_connector.dart +++ b/lib/connector/meshcore_connector.dart @@ -29,11 +29,14 @@ import '../services/linux_ble_pairing_service_stub.dart' if (dart.library.io) '../services/linux_ble_pairing_service.dart'; import '../services/image_chunk_transport.dart' show + ImageChunkHeader, ImageChunkOutcome, ImageChunkTransport, buildSendChannelDataFrame, dataTypeAeicImage, + kImageReassemblyTtl, outPathUnknown, + parseImageChunkHeader, respCodeChannelDataRecv, senderPrefixFromKey; import '../services/image_codec_service.dart'; @@ -5714,7 +5717,9 @@ class MeshCoreConnector extends ChangeNotifier { /// queued a packet to the radio. It says nothing about transmission or /// delivery — GRP_DATA is unacknowledged flood traffic, there is no /// PUSH_CODE_SEND_CONFIRMED for it, and a chunk may still be lost on air. - /// That is what the XOR parity chunk is for. + /// That is what the XOR parity chunk is for, and why repeater echoes are + /// counted afterwards ([ownImageEchoes]). With [repairUnheardChunks], chunks + /// no repeater echoed are resent once (see [_repairUnheardImageChunks]). /// /// Returns false when the transport is unavailable (disconnected or firmware /// too old); throws whatever [_sendFrameAndWaitForCommandAck] throws, i.e. @@ -5727,9 +5732,47 @@ class MeshCoreConnector extends ChangeNotifier { required int channelIndex, required Duration interChunkDelay, void Function(int sent, int total)? onProgress, + bool repairUnheardChunks = true, }) async { if (blobs.isEmpty) return false; if (!supportsChannelData) return false; + final header = parseImageChunkHeader(blobs.first); + final key = header == null ? null : (channelIndex, header.imgId); + if (key != null && repairUnheardChunks) { + // Index == chunk index: data chunks first, the parity chunk (index == + // total) last, so blobs.length slots cover every chunk we send. + _ownImageEchoes.remove(key); + _ownImageEchoes[key] = List.filled(blobs.length, 0); + while (_ownImageEchoes.length > _maxTrackedOwnImages) { + _ownImageEchoes.remove(_ownImageEchoes.keys.first); + } + } + final startedAt = DateTime.now(); + final sentAll = await _transmitImageBlobs( + blobs, + channelIndex: channelIndex, + interChunkDelay: interChunkDelay, + onProgress: onProgress, + ); + if (sentAll && key != null && repairUnheardChunks) { + unawaited( + _repairUnheardImageChunks( + key, + blobs, + interChunkDelay: interChunkDelay, + startedAt: startedAt, + ), + ); + } + return sentAll; + } + + Future _transmitImageBlobs( + List blobs, { + required int channelIndex, + required Duration interChunkDelay, + void Function(int sent, int total)? onProgress, + }) async { // `return` inside the scoped callback only exits the callback, so a // disconnect mid-send used to fall through to `return true` below and the // caller would record a complete outgoing image and tell the user "sent" @@ -5760,6 +5803,142 @@ class MeshCoreConnector extends ChangeNotifier { return sentAll; } + // ---- repeater echoes of our own image chunks ------------------------------- + // + // GRP_DATA is never acknowledged. What we can observe is a repeater + // re-flooding one of our chunks: the radio reports it in the raw RX log, the + // same way repeats of channel text are counted. A chunk nobody echoed while + // other chunks of the same image were echoed was most likely lost, so it is + // sent once more. + + /// How long to listen for echoes after the last chunk before repairing. + static const Duration _imageEchoWait = Duration(seconds: 15); + + /// Receivers drop a partial image [kImageReassemblyTtl] after its first + /// chunk. A repair that cannot land well inside that window is skipped. + static const Duration _imageRepairDeadline = Duration(seconds: 45); + + static const int _maxTrackedOwnImages = 16; + + final Map<(int, int), List> _ownImageEchoes = {}; + final Set<(int, int)> _imageRepairsInFlight = {}; + + /// Repeater echoes heard for each chunk of one of our own images sent this + /// session (data chunks, then the parity chunk), or null if not tracked. + List? ownImageEchoes(int channelIndex, int imgId) { + final echoes = _ownImageEchoes[(channelIndex, imgId)]; + return echoes == null ? null : List.unmodifiable(echoes); + } + + /// Non-null while unheard chunks of that image are being resent. + ({int resends, int maxResends})? imageResendProgress( + int channelIndex, + int imgId, + ) => _imageRepairsInFlight.contains((channelIndex, imgId)) + ? (resends: 1, maxResends: 1) + : null; + + /// A GRP_DATA packet from the raw RX log. Other people's images arrive + /// already decrypted through RESP_CODE_CHANNEL_DATA_RECV; here we only look + /// for repeaters re-flooding one of our own chunks. + void _handleImageEcho(_RawPacket packet) { + final self = senderPrefixFromKey(_selfPublicKey); + if (self == null || packet.payload.length < 2) return; + final channelHash = packet.payload[0]; + final encrypted = Uint8List.fromList(packet.payload.sublist(1)); + final channels = _channels.isNotEmpty ? _channels : _cachedChannels; + for (final channel in channels) { + if (channel.isEmpty) continue; + if (_computeChannelHash(channel.psk) != channelHash) continue; + // Null unless the MAC matches this channel's key. + final plain = _decryptPayload(channel.psk, encrypted); + if (plain == null || plain.length < 3) continue; + // Plaintext: data_type (uint16 LE), data_len, blob (padded to 16). + final dataType = plain[0] | (plain[1] << 8); + final length = plain[2]; + if (dataType != dataTypeAeicImage || plain.length < 3 + length) return; + final header = parseImageChunkHeader( + Uint8List.sublistView(plain, 3, 3 + length), + ); + if (header == null || header.senderPrefix != self) return; + final channelId = channel.storageId; + final slot = _channelMapConfirmed && channelId != null + ? _slotForChannelId(channelId) + : channel.index; + if (slot == null) return; + _recordOwnImageEcho(slot, header); + return; + } + } + + void _recordOwnImageEcho(int slot, ImageChunkHeader header) { + final echoes = _ownImageEchoes[(slot, header.imgId)]; + if (echoes == null || header.index >= echoes.length) return; + echoes[header.index]++; + unawaited( + _receivedImageStore?.setOutgoingEchoes( + channelIndex: slot, + senderPrefix: header.senderPrefix, + imgId: header.imgId, + echoes: List.of(echoes), + ), + ); + notifyListeners(); + } + + /// One repair round: resend the chunks no repeater was heard echoing, but + /// only when some other chunk of the image was echoed. With no echo at all + /// we cannot tell a lost image from a sender with no repeater in earshot, + /// and resending would double every such user's airtime for nothing. + /// + /// The resent chunk is byte-identical, so a repeater that did forward it + /// (and we just missed the echo) drops the copy as already seen. Receivers + /// missing it merge it into the image they are still holding. + Future _repairUnheardImageChunks( + (int, int) key, + List blobs, { + required Duration interChunkDelay, + required DateTime startedAt, + }) async { + await Future.delayed(_imageEchoWait); + final echoes = _ownImageEchoes[key]; + if (echoes == null || echoes.length != blobs.length || !isConnected) { + return; + } + final unheard = [ + for (var i = 0; i < blobs.length; i++) + if (echoes[i] == 0) blobs[i], + ]; + if (unheard.isEmpty || unheard.length == blobs.length) return; + if (DateTime.now().difference(startedAt) > _imageRepairDeadline) { + _appDebugLogService?.info( + 'Image ${key.$2}: ${unheard.length} chunk(s) unheard, too late to ' + 'resend before receivers give up', + tag: 'Image resend', + ); + return; + } + _appDebugLogService?.info( + 'Image ${key.$2}: resending ${unheard.length} of ${blobs.length} ' + 'chunk(s) no repeater echoed', + tag: 'Image resend', + ); + _imageRepairsInFlight.add(key); + notifyListeners(); + try { + await _transmitImageBlobs( + unheard, + channelIndex: key.$1, + interChunkDelay: interChunkDelay, + ); + } catch (e) { + _appDebugLogService?.warn('Image resend failed: $e', tag: 'Image resend'); + } finally { + _imageRepairsInFlight.remove(key); + notifyListeners(); + } + } + Future _runScopedChannelSend( Future Function() action, { required String region, @@ -8396,7 +8575,12 @@ class MeshCoreConnector extends ChangeNotifier { final raw = reader.readRemainingBytes(); final packet = _parseRawPacket(raw); - if (packet == null || packet.payloadType != _payloadTypeGroupText) return; + if (packet == null) return; + if (packet.payloadType == _payloadTypeGroupData) { + _handleImageEcho(packet); + return; + } + if (packet.payloadType != _payloadTypeGroupText) return; final payload = BufferReader(packet.payload); final channelHash = payload.readByte(); @@ -10477,6 +10661,7 @@ const int _routeFlood = 0x01; const int _routeTransportDirect = 0x03; const int _payloadTypeGroupText = 0x05; +const int _payloadTypeGroupData = 0x06; const int _cipherMacSize = 2; bool _isPathLenValidForMode( diff --git a/lib/l10n/app_en.arb b/lib/l10n/app_en.arb index cda9324f..26d43828 100644 --- a/lib/l10n/app_en.arb +++ b/lib/l10n/app_en.arb @@ -3496,6 +3496,44 @@ "imageSend_viewOriginal": "Original", "imageSend_viewReconstruction": "What recipients see", "imageSend_reconstructionUnavailable": "This device can't preview the reconstruction.", + "receivedImage_packetEchoes": "Packet {number}: {count, plural, =0{no repeater heard} =1{repeated once} other{repeated {count} times}}", + "@receivedImage_packetEchoes": { + "placeholders": { + "number": { + "type": "int" + }, + "count": { + "type": "int" + } + } + }, + "receivedImage_recoveryPacketEchoes": "Recovery packet: {count, plural, =0{no repeater heard} =1{repeated once} other{repeated {count} times}}", + "@receivedImage_recoveryPacketEchoes": { + "placeholders": { + "count": { + "type": "int" + } + } + }, + "imageSend_preparingPreview": "Preparing preview…", + "imageSend_tryAnother": "Try another version", + "imageSend_versionsHint": "Each version is reconstructed a little differently. Pick the one that looks most like your photo.", + "imageSend_versionLabel": "Version {number}", + "@imageSend_versionLabel": { + "placeholders": { + "number": { + "type": "int" + } + } + }, + "imageSend_versionPackets": "{count, plural, =1{1 packet} other{{count} packets}}", + "@imageSend_versionPackets": { + "placeholders": { + "count": { + "type": "int" + } + } + }, "imageSend_modelNotDownloaded": "The image model isn't downloaded yet. Download it in Settings to send images.", "imageSend_originalSize": "Original", "imageSend_onAirSize": "On air", diff --git a/lib/l10n/app_localizations.dart b/lib/l10n/app_localizations.dart index bdd028de..e0582ade 100644 --- a/lib/l10n/app_localizations.dart +++ b/lib/l10n/app_localizations.dart @@ -10565,6 +10565,48 @@ abstract class AppLocalizations { /// **'This device can\'t preview the reconstruction.'** String get imageSend_reconstructionUnavailable; + /// No description provided for @receivedImage_packetEchoes. + /// + /// In en, this message translates to: + /// **'Packet {number}: {count, plural, =0{no repeater heard} =1{repeated once} other{repeated {count} times}}'** + String receivedImage_packetEchoes(int number, int count); + + /// No description provided for @receivedImage_recoveryPacketEchoes. + /// + /// In en, this message translates to: + /// **'Recovery packet: {count, plural, =0{no repeater heard} =1{repeated once} other{repeated {count} times}}'** + String receivedImage_recoveryPacketEchoes(int count); + + /// No description provided for @imageSend_preparingPreview. + /// + /// In en, this message translates to: + /// **'Preparing preview…'** + String get imageSend_preparingPreview; + + /// No description provided for @imageSend_tryAnother. + /// + /// In en, this message translates to: + /// **'Try another version'** + String get imageSend_tryAnother; + + /// No description provided for @imageSend_versionsHint. + /// + /// In en, this message translates to: + /// **'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'** + String get imageSend_versionsHint; + + /// No description provided for @imageSend_versionLabel. + /// + /// In en, this message translates to: + /// **'Version {number}'** + String imageSend_versionLabel(int number); + + /// No description provided for @imageSend_versionPackets. + /// + /// In en, this message translates to: + /// **'{count, plural, =1{1 packet} other{{count} packets}}'** + String imageSend_versionPackets(int count); + /// No description provided for @imageSend_modelNotDownloaded. /// /// In en, this message translates to: diff --git a/lib/l10n/app_localizations_bg.dart b/lib/l10n/app_localizations_bg.dart index f45c475d..7d7e88bc 100644 --- a/lib/l10n/app_localizations_bg.dart +++ b/lib/l10n/app_localizations_bg.dart @@ -6245,6 +6245,56 @@ class AppLocalizationsBg extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Това устройство не може да покаже реконструкцията.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'Моделът за изображения още не е изтеглен. Изтеглете го в настройките, за да изпращате изображения.'; diff --git a/lib/l10n/app_localizations_cs.dart b/lib/l10n/app_localizations_cs.dart index b015322c..c3cea98a 100644 --- a/lib/l10n/app_localizations_cs.dart +++ b/lib/l10n/app_localizations_cs.dart @@ -6231,6 +6231,56 @@ class AppLocalizationsCs extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Toto zařízení nemůže zobrazit náhled rekonstrukce.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'Model obrázků ještě není stažen. Pro odesílání obrázků jej stáhněte v Nastavení.'; diff --git a/lib/l10n/app_localizations_de.dart b/lib/l10n/app_localizations_de.dart index c8d2226d..9317d6a1 100644 --- a/lib/l10n/app_localizations_de.dart +++ b/lib/l10n/app_localizations_de.dart @@ -6263,6 +6263,56 @@ class AppLocalizationsDe extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Dieses Gerät kann die Rekonstruktion nicht in der Vorschau anzeigen.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'Das Bildmodell wurde noch nicht heruntergeladen. Laden Sie es in den Einstellungen herunter, um Bilder zu senden.'; diff --git a/lib/l10n/app_localizations_en.dart b/lib/l10n/app_localizations_en.dart index 880a8a4b..e0c6670e 100644 --- a/lib/l10n/app_localizations_en.dart +++ b/lib/l10n/app_localizations_en.dart @@ -6159,6 +6159,56 @@ class AppLocalizationsEn extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'This device can\'t preview the reconstruction.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'The image model isn\'t downloaded yet. Download it in Settings to send images.'; diff --git a/lib/l10n/app_localizations_es.dart b/lib/l10n/app_localizations_es.dart index c14c8fcf..7af2dbf8 100644 --- a/lib/l10n/app_localizations_es.dart +++ b/lib/l10n/app_localizations_es.dart @@ -6247,6 +6247,56 @@ class AppLocalizationsEs extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Este dispositivo no puede previsualizar la reconstrucción.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'El modelo de imagen aún no se ha descargado. Descárgalo en Configuración para enviar imágenes.'; diff --git a/lib/l10n/app_localizations_fr.dart b/lib/l10n/app_localizations_fr.dart index 61effc54..833c9e59 100644 --- a/lib/l10n/app_localizations_fr.dart +++ b/lib/l10n/app_localizations_fr.dart @@ -6270,6 +6270,56 @@ class AppLocalizationsFr extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Cet appareil ne peut pas prévisualiser la reconstitution.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'Le modèle d\'image n\'est pas encore téléchargé. Téléchargez-le dans Paramètres pour envoyer des images.'; diff --git a/lib/l10n/app_localizations_hu.dart b/lib/l10n/app_localizations_hu.dart index b99d6fe9..5a26abff 100644 --- a/lib/l10n/app_localizations_hu.dart +++ b/lib/l10n/app_localizations_hu.dart @@ -6244,6 +6244,56 @@ class AppLocalizationsHu extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Ezen az eszközön nem jeleníthető meg a rekonstrukció előnézete.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'A képmodell még nincs letöltve. Töltse le a Beállításokban a képek küldéséhez.'; diff --git a/lib/l10n/app_localizations_it.dart b/lib/l10n/app_localizations_it.dart index 058c8a27..9c863c67 100644 --- a/lib/l10n/app_localizations_it.dart +++ b/lib/l10n/app_localizations_it.dart @@ -6252,6 +6252,56 @@ class AppLocalizationsIt extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Questo dispositivo non può mostrare un\'anteprima della ricostruzione.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'Il modello di immagine non è ancora stato scaricato. Scaricalo nelle Impostazioni per inviare immagini.'; diff --git a/lib/l10n/app_localizations_ja.dart b/lib/l10n/app_localizations_ja.dart index 51064d0a..d4979730 100644 --- a/lib/l10n/app_localizations_ja.dart +++ b/lib/l10n/app_localizations_ja.dart @@ -5944,6 +5944,56 @@ class AppLocalizationsJa extends AppLocalizations { @override String get imageSend_reconstructionUnavailable => 'このデバイスでは復元結果をプレビューできません。'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => '画像モデルがまだダウンロードされていません。設定でダウンロードすると画像を送信できます。'; diff --git a/lib/l10n/app_localizations_ko.dart b/lib/l10n/app_localizations_ko.dart index 9544d808..6dcd999c 100644 --- a/lib/l10n/app_localizations_ko.dart +++ b/lib/l10n/app_localizations_ko.dart @@ -5960,6 +5960,56 @@ class AppLocalizationsKo extends AppLocalizations { String get imageSend_reconstructionUnavailable => '이 장치에서는 복원 결과를 미리 볼 수 없습니다.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => '이미지 모델이 아직 다운로드되지 않았습니다. 설정에서 다운로드하면 이미지를 보낼 수 있습니다.'; diff --git a/lib/l10n/app_localizations_nl.dart b/lib/l10n/app_localizations_nl.dart index 565e79a4..48a8b16a 100644 --- a/lib/l10n/app_localizations_nl.dart +++ b/lib/l10n/app_localizations_nl.dart @@ -6216,6 +6216,56 @@ class AppLocalizationsNl extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Dit apparaat kan geen voorbeeld van de reconstructie tonen.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'Het afbeeldingsmodel is nog niet gedownload. Download het in Instellingen om afbeeldingen te verzenden.'; diff --git a/lib/l10n/app_localizations_pl.dart b/lib/l10n/app_localizations_pl.dart index 5f5c9b00..8889a984 100644 --- a/lib/l10n/app_localizations_pl.dart +++ b/lib/l10n/app_localizations_pl.dart @@ -6290,6 +6290,56 @@ class AppLocalizationsPl extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'To urządzenie nie może wyświetlić podglądu rekonstrukcji.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'Model obrazowy nie jest jeszcze pobrany. Pobierz go w Ustawieniach, aby wysyłać obrazy.'; diff --git a/lib/l10n/app_localizations_pt.dart b/lib/l10n/app_localizations_pt.dart index 0c0749b6..a0274faa 100644 --- a/lib/l10n/app_localizations_pt.dart +++ b/lib/l10n/app_localizations_pt.dart @@ -6241,6 +6241,56 @@ class AppLocalizationsPt extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Este dispositivo não pode pré-visualizar a reconstrução.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'O modelo de imagem ainda não foi transferido. Transfira-o nas Configurações para enviar imagens.'; diff --git a/lib/l10n/app_localizations_ru.dart b/lib/l10n/app_localizations_ru.dart index 5a6119ee..5da20d8c 100644 --- a/lib/l10n/app_localizations_ru.dart +++ b/lib/l10n/app_localizations_ru.dart @@ -6266,6 +6266,56 @@ class AppLocalizationsRu extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Это устройство не может показать реконструкцию.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'Модель изображения ещё не загружена. Загрузите её в настройках, чтобы отправлять изображения.'; diff --git a/lib/l10n/app_localizations_sk.dart b/lib/l10n/app_localizations_sk.dart index 109fdbd5..cbc100b5 100644 --- a/lib/l10n/app_localizations_sk.dart +++ b/lib/l10n/app_localizations_sk.dart @@ -6240,6 +6240,56 @@ class AppLocalizationsSk extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Toto zariadenie nedokáže zobraziť náhľad rekonštrukcie.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'Model obrazu ešte nie je stiahnutý. Stiahnite ho v Nastaveniach, aby ste mohli posielať obrázky.'; diff --git a/lib/l10n/app_localizations_sl.dart b/lib/l10n/app_localizations_sl.dart index aedc64ea..e2a05bf1 100644 --- a/lib/l10n/app_localizations_sl.dart +++ b/lib/l10n/app_localizations_sl.dart @@ -6232,6 +6232,56 @@ class AppLocalizationsSl extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Ta naprava ne more prikazati predogleda rekonstrukcije.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'Obrazni model še ni prenesen. Prenesite ga v Nastavitvah, da boste lahko pošiljali slike.'; diff --git a/lib/l10n/app_localizations_sv.dart b/lib/l10n/app_localizations_sv.dart index 50dc42c8..dfecda49 100644 --- a/lib/l10n/app_localizations_sv.dart +++ b/lib/l10n/app_localizations_sv.dart @@ -6199,6 +6199,56 @@ class AppLocalizationsSv extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Enheten kan inte förhandsgranska rekonstruktionen.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'Bildmodellen har ännu inte laddats ner. Ladda ner den i Inställningar för att skicka bilder.'; diff --git a/lib/l10n/app_localizations_uk.dart b/lib/l10n/app_localizations_uk.dart index a656a790..10daf118 100644 --- a/lib/l10n/app_localizations_uk.dart +++ b/lib/l10n/app_localizations_uk.dart @@ -6272,6 +6272,56 @@ class AppLocalizationsUk extends AppLocalizations { String get imageSend_reconstructionUnavailable => 'Цей пристрій не може показати відтворення.'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => 'Модель зображення ще не завантажена. Завантажте її в налаштуваннях, щоб надсилати зображення.'; diff --git a/lib/l10n/app_localizations_zh.dart b/lib/l10n/app_localizations_zh.dart index a1661572..322c287c 100644 --- a/lib/l10n/app_localizations_zh.dart +++ b/lib/l10n/app_localizations_zh.dart @@ -5807,6 +5807,56 @@ class AppLocalizationsZh extends AppLocalizations { @override String get imageSend_reconstructionUnavailable => '此设备无法预览重建结果。'; + @override + String receivedImage_packetEchoes(int number, int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Packet $number: $_temp0'; + } + + @override + String receivedImage_recoveryPacketEchoes(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: 'repeated $count times', + one: 'repeated once', + zero: 'no repeater heard', + ); + return 'Recovery packet: $_temp0'; + } + + @override + String get imageSend_preparingPreview => 'Preparing preview…'; + + @override + String get imageSend_tryAnother => 'Try another version'; + + @override + String get imageSend_versionsHint => + 'Each version is reconstructed a little differently. Pick the one that looks most like your photo.'; + + @override + String imageSend_versionLabel(int number) { + return 'Version $number'; + } + + @override + String imageSend_versionPackets(int count) { + String _temp0 = intl.Intl.pluralLogic( + count, + locale: localeName, + other: '$count packets', + one: '1 packet', + ); + return '$_temp0'; + } + @override String get imageSend_modelNotDownloaded => '图像模型尚未下载。在设置中下载后才能发送图像。'; diff --git a/lib/main.dart b/lib/main.dart index 751e79d2..8f1cef8d 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -140,6 +140,8 @@ void main() async { [blob], channelIndex: channelIndex, interChunkDelay: Duration.zero, + // One blob at a time: echo tracking needs the whole chunk set. + repairUnheardChunks: false, ), // Overwritten by onImageSenderPrefix as soon as SELF_INFO lands. senderPrefix: 0, diff --git a/lib/screens/channel_chat_screen.dart b/lib/screens/channel_chat_screen.dart index fc276315..ebf4b0c6 100644 --- a/lib/screens/channel_chat_screen.dart +++ b/lib/screens/channel_chat_screen.dart @@ -1892,9 +1892,8 @@ class _ChannelChatScreenState extends State { final connector = context.read(); // The picked bytes are kept past the preview on purpose: the sender's own - // bubble renders from them (see [_registerOutgoingImage]). The bitstream - // cannot serve — turning it back into pixels is a ~2.16 GiB, ~1 s decode - // of an image the sender is already holding. + // bubble falls back to them (see [_registerOutgoingImage]) when the sheet + // could not decode the chosen version on this device. final XFile? picked; try { picked = await ImagePicker().pickImage( @@ -2070,8 +2069,11 @@ class _ChannelChatScreenState extends State { /// GRP_DATA is not echoed back to the sender and an image is not a /// [ChannelMessage], so without this the sender sees a "sent" snack bar and /// an otherwise empty conversation. The entry lands in `decoded` directly — - /// an outgoing image is never queued for inference, so showing your own - /// photo costs no model memory. + /// an outgoing image is never queued for inference. + /// + /// The bubble shows the version the user picked, decoded as recipients see + /// it ([ImageSendPreviewResult.previewPng]). Only when this device could not decode does it fall back + /// to the stretched original. /// /// Best effort throughout: a failure here has already been preceded by a /// successful transmission, so it must not turn into an error the user sees. @@ -2087,7 +2089,8 @@ class _ChannelChatScreenState extends State { } on ProviderNotFoundException { return; // screen test without the store } - final previewPng = await _squarePreviewPng(sourceBytes); + final previewPng = + result.previewPng ?? await _squarePreviewPng(sourceBytes); if (previewPng == null) return; try { await store.registerOutgoing( @@ -2101,6 +2104,18 @@ class _ChannelChatScreenState extends State { rate: aeicRatePointForUi(result.rate), chunkCount: chunkSet.dataChunkCount, aspectCode: await _aspectCodeOf(sourceBytes), + // The decoded version is model output and carries the AI label; the + // stretched-original fallback is the sender's own pixels. + synthesized: result.previewPng != null, + payloadBytes: result.payload.length, + // Repeaters may already have echoed the first chunks while the rest + // were still going out. + chunkEchoes: mounted + ? context.read().ownImageEchoes( + widget.channel.index, + chunkSet.imgId, + ) + : null, ); } on Exception catch (error) { debugPrint('outgoing image registration failed: $error'); @@ -2128,11 +2143,9 @@ class _ChannelChatScreenState extends State { /// [imageBytes] stretched to 512x512, as PNG. /// - /// This is what the preview sheet showed (`BoxFit.fill` on a 1:1 box is - /// exactly the stretch the codec applies) and what the codec actually - /// encoded, so the sender's bubble matches both. Deliberately not a decode of the - /// bitstream: that would cost ~2.16 GiB and ~1 s to reproduce, less well, an - /// image this device is already holding in memory. + /// Fallback for the sender's bubble when the preview sheet had no decoded + /// version to hand over (a device that cannot decode). It is the input the + /// codec stretched, not what recipients see, so it is only a stand-in. /// /// Returns null rather than throwing on an undecodable source — the send has /// already happened by the time this runs. @@ -2375,12 +2388,35 @@ class _ChannelChatScreenState extends State { right: 8, bottom: 4, ), - child: Text( - formatChatTime(context, entry.firstSeen), - style: MeshTheme.mono( - fontSize: 12 * textScale, - color: metaColor, - ), + child: Row( + mainAxisSize: MainAxisSize.min, + children: [ + Text( + formatChatTime(context, entry.firstSeen), + style: MeshTheme.mono( + fontSize: 12 * textScale, + color: metaColor, + ), + ), + if (isOutgoing) ...[ + const SizedBox(width: 4), + // Same status as channel text: repeats heard, here + // for the least-heard data chunk. + MessageStatusIcon( + size: 16 * textScale, + isAcked: false, + isChannel: true, + repeatCount: entry.minDataChunkEchoes ?? 0, + resendProgress: context + .watch() + .imageResendProgress( + entry.channelIndex, + entry.imgId, + ), + onColor: metaColor, + ), + ], + ], ), ), ], @@ -2513,6 +2549,15 @@ class _ChannelChatScreenState extends State { if (entry.recoveredWithParity) Text(l10n.receivedImage_parityRecovered), if (decodeMs != null) Text(l10n.receivedImage_decodeTime(decodeMs)), + if (entry.chunkEchoes case final echoes?) ...[ + const SizedBox(height: 8), + for (var i = 0; i < echoes.length; i++) + Text( + i < entry.totalChunks + ? l10n.receivedImage_packetEchoes(i + 1, echoes[i]) + : l10n.receivedImage_recoveryPacketEchoes(echoes[i]), + ), + ], ], ), actions: [ diff --git a/lib/services/image_codec_service.dart b/lib/services/image_codec_service.dart index edf5e0e5..932d90d7 100644 --- a/lib/services/image_codec_service.dart +++ b/lib/services/image_codec_service.dart @@ -970,15 +970,17 @@ class ImageCodecService extends ChangeNotifier implements ImageSendCodec { /// [ImageSendCodec] entry point used by `ImageSendPreviewSheet`. /// - /// Takes the raw picked file (JPEG/PNG/HEIC/…), centre-crops and scales it to - /// [kImageCodecSquareSize] square, then encodes. Throws on failure so the + /// Takes the raw picked file (JPEG/PNG/HEIC/…), stretches the part of it + /// that [variant] selects (see [rerollSourceRect]; 0 is the whole photo) to + /// a [kImageCodecSquareSize] square, then encodes. Throws on failure so the /// sheet can surface it — unlike the nullable service-level methods below, /// whose contract is "null means not available". @override Future encode( Uint8List imageBytes, - ImageCodecRatePoint rate, - ) async { + ImageCodecRatePoint rate, { + int variant = 0, + }) async { if (!kImageCodecBitstreamPathAvailable) { // Fail here rather than after a 512x512 raster and a session load, and // fail with the specific exception so the sheet can say why. @@ -990,7 +992,7 @@ class ImageCodecService extends ChangeNotifier implements ImageSendCodec { throw const ImageCodecBundleIncomplete(); } final result = await encodeImage( - rgbBytes: await _toSquareRgb(imageBytes), + rgbBytes: await _toSquareRgb(imageBytes, variant: variant), ratePoint: aeicRatePointForUi(rate), ); final bitstream = result?.bitstream; @@ -1300,14 +1302,18 @@ class ImageCodecService extends ChangeNotifier implements ImageSendCodec { static const int _bytesPerPixel = 3; - /// Centre-crops [imageBytes] to a square and scales it to - /// [kImageCodecSquareSize], returning packed 8-bit RGB. + /// Stretches the [variant] region of [imageBytes] (see [rerollSourceRect]; + /// 0 is the whole frame) to a [kImageCodecSquareSize] square, returning + /// packed 8-bit RGB. /// /// Uses `dart:ui` rather than a decode package so this adds no dependency. /// It runs on the root isolate, which is acceptable: it is a single 512x512 /// raster, three orders of magnitude cheaper than the 1385 GFLOP inference /// that follows in the worker. - Future _toSquareRgb(Uint8List imageBytes) async { + Future _toSquareRgb( + Uint8List imageBytes, { + int variant = 0, + }) async { final codec = await ui.instantiateImageCodec(imageBytes); final frame = await codec.getNextFrame(); final source = frame.image; @@ -1317,16 +1323,11 @@ class ImageCodecService extends ChangeNotifier implements ImageSendCodec { // Aspect ratio is deliberately not preserved: the codec's graph is // statically 512x512, so something has to give, and losing 25% of a 4:3 // photo (44% of 16:9, or the subject's head in a portrait) is worse than - // distorting it. Nothing outside the frame is discarded this way. - // NOTE: the aspect ratio is NOT transmitted, so the receiver cannot undo - // the stretch — see the note in image_chunk_transport.dart on the - // metadata byte. - final src = ui.Rect.fromLTWH( - 0, - 0, - source.width.toDouble(), - source.height.toDouble(), - ); + // distorting it. The source shape travels in the metadata byte's aspect + // nibble, so the receiver un-stretches it. Re-roll variants trim at most + // ~2% at an edge, so they keep the same shape and aspect code. + final r = rerollSourceRect(source.width, source.height, variant); + final src = ui.Rect.fromLTWH(r.left, r.top, r.width, r.height); final dstSize = kImageCodecSquareSize.toDouble(); final recorder = ui.PictureRecorder(); ui.Canvas(recorder).drawImageRect( diff --git a/lib/services/received_image_store.dart b/lib/services/received_image_store.dart index 94ee9ee1..401204da 100644 --- a/lib/services/received_image_store.dart +++ b/lib/services/received_image_store.dart @@ -197,13 +197,30 @@ class ReceivedImageEntry { /// One chunk was rebuilt from the XOR parity chunk. final bool recoveredWithParity; - /// This is the local user's own send. Its PNG is their real 512x512 crop, so - /// [synthesized] is false and the R6 label must NOT be shown. + /// This is the local user's own send. final bool isOutgoing; - /// Whether the pixels are generative-model output. Derived, never supplied: - /// `synthesized == !isOutgoing`. - bool get synthesized => !isOutgoing; + /// Whether the pixels are generative-model output, so the R6 label must be + /// shown. Always true for received images. For the user's own send it is + /// true when the PNG is the version they picked, decoded as recipients see + /// it, and false only when it is the stretched original (a device that + /// could not decode). + final bool synthesized; + + /// Our own image only: repeater echoes heard for each chunk, data chunks + /// first and the parity chunk last. Null when not tracked. + final List? chunkEchoes; + + /// The fewest echoes any data chunk got: every data chunk was heard + /// repeated at least this many times. Null when not tracked. + int? get minDataChunkEchoes { + final echoes = chunkEchoes; + if (echoes == null || echoes.isEmpty) return null; + final data = echoes.length > totalChunks + ? echoes.sublist(0, totalChunks) + : echoes; + return data.reduce((a, b) => a < b ? a : b); + } final int? decodeMs; final String? error; @@ -237,6 +254,8 @@ class ReceivedImageEntry { this.metadataAssumed = true, this.recoveredWithParity = false, this.isOutgoing = false, + bool? synthesized, + this.chunkEchoes, this.decodeMs, this.error, this.needsManualDecode = false, @@ -245,7 +264,7 @@ class ReceivedImageEntry { this.bitstreamByteCount = 0, this.pngByteCount = 0, this.pngBytes, - }); + }) : synthesized = synthesized ?? !isOutgoing; ImageStreamKey get key => ImageStreamKey( senderPrefix: senderPrefix, @@ -282,6 +301,7 @@ class ReceivedImageEntry { int? bitstreamByteCount, int? pngByteCount, Object? pngBytes = _unset, + List? chunkEchoes, }) { return ReceivedImageEntry( streamId: streamId, @@ -299,6 +319,8 @@ class ReceivedImageEntry { metadataAssumed: metadataAssumed ?? this.metadataAssumed, recoveredWithParity: recoveredWithParity ?? this.recoveredWithParity, isOutgoing: isOutgoing, + synthesized: synthesized, + chunkEchoes: chunkEchoes ?? this.chunkEchoes, decodeMs: identical(decodeMs, _unset) ? this.decodeMs : decodeMs as int?, error: identical(error, _unset) ? this.error : error as String?, needsManualDecode: needsManualDecode ?? this.needsManualDecode, @@ -329,6 +351,7 @@ class ReceivedImageEntry { 'recoveredWithParity': recoveredWithParity, 'isOutgoing': isOutgoing, 'synthesized': synthesized, + if (chunkEchoes != null) 'chunkEchoes': chunkEchoes, 'needsManualDecode': needsManualDecode, 'bitstreamStored': bitstreamStored, 'pngStored': pngStored, @@ -357,6 +380,10 @@ class ReceivedImageEntry { metadataAssumed: json['metadataAssumed'] as bool? ?? true, recoveredWithParity: json['recoveredWithParity'] as bool? ?? false, isOutgoing: json['isOutgoing'] as bool? ?? false, + synthesized: json['synthesized'] as bool?, + chunkEchoes: (json['chunkEchoes'] as List?) + ?.whereType() + .toList(), decodeMs: json['decodeMs'] as int?, error: json['error'] as String?, needsManualDecode: json['needsManualDecode'] as bool? ?? false, @@ -1041,6 +1068,9 @@ class ReceivedImageStore extends ChangeNotifier { required AeicRatePoint rate, required int chunkCount, int aspectCode = 0, + bool synthesized = false, + List? chunkEchoes, + int payloadBytes = 0, DateTime? at, }) async { final now = at ?? _clock(); @@ -1069,6 +1099,11 @@ class ReceivedImageStore extends ChangeNotifier { aspectCode: aspectCode, metadataAssumed: false, isOutgoing: true, + synthesized: synthesized, + chunkEchoes: chunkEchoes, + // The size of what was sent, for the caption and packet info. The + // bitstream itself is not kept, so bitstreamStored stays false. + bitstreamByteCount: payloadBytes, pngStored: pngWritten, pngByteCount: pngWritten ? previewPng.length : 0, pngBytes: previewPng, @@ -1078,6 +1113,31 @@ class ReceivedImageStore extends ChangeNotifier { return stored; } + /// Records repeater echoes of the user's own image (see + /// `MeshCoreConnector.ownImageEchoes`). Echoes that land before the image is + /// registered are passed to [registerOutgoing] instead. + Future setOutgoingEchoes({ + required int channelIndex, + required int senderPrefix, + required int imgId, + required List echoes, + }) async { + ReceivedImageEntry? match; + for (final entry in _entries.values) { + if (!entry.isOutgoing || + entry.imgId != imgId || + entry.channelIndex != channelIndex || + entry.senderPrefix != senderPrefix) { + continue; + } + if (match == null || entry.firstSeen.isAfter(match.firstSeen)) { + match = entry; + } + } + if (match == null) return; + await _store(match.copyWith(chunkEchoes: List.unmodifiable(echoes))); + } + // ---- decode scheduling --------------------------------------------------- /// App lifecycle gate. Backgrounding cancels an in-flight decode and returns diff --git a/lib/widgets/image_send_codec_binding.dart b/lib/widgets/image_send_codec_binding.dart index fe99a99b..c0b952ef 100644 --- a/lib/widgets/image_send_codec_binding.dart +++ b/lib/widgets/image_send_codec_binding.dart @@ -98,6 +98,54 @@ class ImageCodecRateStats { } } +/// Input tweaks for "Try another version", as `(zoom, dx, dy)` fractions of +/// the source photo. Index == version variant; 0 is the untouched photo. +/// +/// The decoder is generative and very sensitive to its input: a near-identical +/// photo can come back with different invented detail. Measured on Kodak, the +/// best of three versions raised CLIP agreement with the source from 0.828 to +/// 0.853 and fixed 3 of 8 photos that came back as something else +/// (`tools/aeic/eval`, `reroll*` variants, which mirror this table). The tweaks +/// drop at most ~2% at an edge, so every version still shows the whole photo +/// and keeps the same aspect code. +const List<(double, double, double)> kImageSendRerollTweaks = [ + (1.0, 0.0, 0.0), + (1.02, 0.005, 0.0), + (1.02, -0.005, 0.005), + (1.04, 0.0, -0.005), + (1.03, 0.01, 0.01), + (1.03, -0.01, -0.01), +]; + +/// Most versions the send sheet will make of one photo. +final int kImageSendMaxVersions = kImageSendRerollTweaks.length; + +/// The part of a [width] x [height] photo that version [variant] encodes. +/// +/// Variant 0 is the full frame, so the first version is exactly what the app +/// sent before versions existed. Other variants zoom in by `zoom`, shift by +/// `dx`/`dy` of the photo size, and are clamped to stay inside the photo. +({double left, double top, double width, double height}) rerollSourceRect( + int width, + int height, + int variant, +) { + if (variant < 0 || variant >= kImageSendRerollTweaks.length) { + throw RangeError.range( + variant, + 0, + kImageSendRerollTweaks.length - 1, + 'variant', + ); + } + final (zoom, dx, dy) = kImageSendRerollTweaks[variant]; + final w = width / zoom; + final h = height / zoom; + final left = ((width - w) / 2 + dx * width).clamp(0.0, width - w); + final top = ((height - h) / 2 + dy * height).clamp(0.0, height - h); + return (left: left, top: top, width: w, height: h); +} + /// The encode interface the preview sheet consumes. abstract class ImageSendCodec { /// Current availability. The sheet renders a non-interactive explanation for @@ -118,9 +166,14 @@ abstract class ImageSendCodec { /// Encode [imageBytes] (any common still format) at [rate], returning the /// compressed bitstream that will be chunked onto the air. /// - /// Implementations are expected to centre-crop to - /// [kImageCodecSquareSize] square first. - Future encode(Uint8List imageBytes, ImageCodecRatePoint rate); + /// Implementations stretch the part of the photo given by + /// [rerollSourceRect] for [variant] into a [kImageCodecSquareSize] square + /// first. Variant 0 is the whole photo. + Future encode( + Uint8List imageBytes, + ImageCodecRatePoint rate, { + int variant = 0, + }); /// Decodes [payload] to PNG exactly as a recipient would, or null when this /// device cannot decode. @@ -142,26 +195,44 @@ class FakeImageSendCodec implements ImageSendCodec { final Duration latency; + /// Returned by [decodePreview]; null means "this device cannot decode". + final Uint8List? previewPng; + const FakeImageSendCodec({ this.availability = ImageCodecAvailability.ready, this.unavailableReason, this.latency = const Duration(milliseconds: 250), + this.previewPng, }); + /// Payload size the fake produces for [variant]: the measured mean, plus a + /// little per variant so versions are told apart. + static int payloadBytesFor(ImageCodecRatePoint rate, int variant) => + ImageCodecRateStats.forRate(rate).meanBytes + variant * 3; + @override Future encode( Uint8List imageBytes, - ImageCodecRatePoint rate, - ) async { + ImageCodecRatePoint rate, { + int variant = 0, + }) async { if (latency > Duration.zero) { await Future.delayed(latency); } - final size = ImageCodecRateStats.forRate(rate).meanBytes; + final size = payloadBytesFor(rate, variant); return Uint8List.fromList( - List.generate(size, (i) => (i * 31 + rate.index) & 0xFF), + List.generate( + size, + (i) => (i * 31 + rate.index + variant * 7) & 0xFF, + ), ); } @override - Future decodePreview(Uint8List payload) async => null; + Future decodePreview(Uint8List payload) async { + if (latency > Duration.zero) { + await Future.delayed(latency); + } + return previewPng; + } } diff --git a/lib/widgets/image_send_preview_sheet.dart b/lib/widgets/image_send_preview_sheet.dart index 8797da5d..f20cd842 100644 --- a/lib/widgets/image_send_preview_sheet.dart +++ b/lib/widgets/image_send_preview_sheet.dart @@ -6,6 +6,7 @@ import 'package:provider/provider.dart'; import '../connector/meshcore_connector.dart'; import '../l10n/l10n.dart'; +import '../theme/mesh_theme.dart'; import '../services/image_chunk_transport.dart'; import '../utils/lora_airtime.dart'; import 'image_send_codec_binding.dart'; @@ -37,6 +38,11 @@ class ImageSendPreviewResult { /// actually shown. Null when the radio settings were unknown. final Duration? wallClock; + /// The chosen version decoded as recipients will see it (a + /// 512x512 PNG, stretched like the codec input), or null if this device + /// could not decode. Used for the sender's own chat bubble. + final Uint8List? previewPng; + const ImageSendPreviewResult({ required this.payload, required this.rate, @@ -44,6 +50,7 @@ class ImageSendPreviewResult { required this.packetCount, required this.airtime, required this.wallClock, + this.previewPng, }); } @@ -141,29 +148,53 @@ class _RateEstimate { bool get isExact => payloadBytes != null; } +/// One encoded version of the photo: the whole photo with re-roll tweak +/// [variant] (see [rerollSourceRect]), its bitstream, and what recipients will +/// see when they decode it. +class _Version { + _Version(this.variant); + + final int variant; + Uint8List? payload; + Uint8List? png; + bool encoding = true; + bool decoding = false; + + /// The encode failed; there is nothing to send for this version. + bool get failed => !encoding && payload == null; +} + class _ImageSendPreviewSheetState extends State { - /// The encoded ft32 bitstream, once the encode finishes. - Uint8List? _encoded; + /// Versions made so far, in the order they were made. Version 1 (variant 0) + /// is the photo exactly as the app has always encoded it; more are added only + /// when the user taps "Try another version". + final List<_Version> _versions = []; + int _selected = 0; late bool _parity; - bool _encoding = false; bool _sending = false; - bool _failed = false; /// Latest radio params seen in build; used by [_onSend] so the confirm path /// does not have to touch an InheritedWidget after an await. ImageSendRadio? _radio; - /// Source `width / height`. Recipients un-stretch the 512x512 square back to - /// this shape, so both preview views use it. + /// Source `width / height`, used when the original is shown. double _aspect = 1; // The shape recipients restore: the nearest supported ratio, or square. double _recipientAspect = 1; - bool _showReconstruction = false; - Uint8List? _reconstruction; - bool _reconstructing = false; - bool _reconstructionUnavailable = false; + /// The main preview shows the original photo instead of the reconstruction. + bool _showOriginal = false; + + /// This device cannot decode, so there is no reconstruction to show and + /// making more versions would be pointless. + bool _previewUnavailable = false; + + _Version? get _current => _versions.isEmpty ? null : _versions[_selected]; + + /// A version is being encoded or decoded. Codec jobs run one at a time, so + /// "Try another version" waits for this to clear. + bool get _working => _versions.any((v) => v.encoding || v.decoding); @override void initState() { @@ -171,7 +202,7 @@ class _ImageSendPreviewSheetState extends State { _parity = widget.initialParity; _probeAspect(); if (widget.codec.availability == ImageCodecAvailability.ready) { - _encode(); + _addVersion(); } } @@ -197,59 +228,56 @@ class _ImageSendPreviewSheetState extends State { } catch (_) {} } - Future _selectView(bool reconstruction) async { - setState(() => _showReconstruction = reconstruction); - final payload = _encoded; - if (!reconstruction || - payload == null || - _reconstruction != null || - _reconstructing) { - return; + /// Encodes the next version, selects it, then decodes it so the user sees + /// what recipients will get. + Future _addVersion() async { + final version = _Version(_versions.length); + setState(() { + _versions.add(version); + _selected = _versions.length - 1; + _showOriginal = false; + }); + Uint8List? payload; + try { + payload = await widget.codec.encode( + widget.imageBytes, + kImageSendRatePoint, + variant: version.variant, + ); + } catch (_) { + payload = null; } - setState(() => _reconstructing = true); + if (!mounted) return; + setState(() { + version.encoding = false; + version.payload = payload; + version.decoding = payload != null && !_previewUnavailable; + }); + if (!version.decoding) return; Uint8List? png; try { - png = await widget.codec.decodePreview(payload); + png = await widget.codec.decodePreview(payload!); } catch (_) { png = null; } if (!mounted) return; setState(() { - _reconstructing = false; - _reconstruction = png; - if (png == null) { - _reconstructionUnavailable = true; - _showReconstruction = false; + version.decoding = false; + version.png = png; + if (png == null && !_versions.any((v) => v.png != null)) { + // Nothing has ever decoded here: treat the device as unable to + // preview and fall back to showing the original. + _previewUnavailable = true; + _showOriginal = true; } }); } - Future _encode() async { - final cached = _encoded; - if (cached != null) return cached; + void _select(int index) { setState(() { - _encoding = true; - _failed = false; + _selected = index; + _showOriginal = false; }); - try { - final bytes = await widget.codec.encode( - widget.imageBytes, - kImageSendRatePoint, - ); - if (!mounted) return bytes; - setState(() { - _encoded = bytes; - _encoding = false; - }); - return bytes; - } catch (_) { - if (!mounted) return null; - setState(() { - _encoding = false; - _failed = true; - }); - return null; - } } ImageSendRadio _radioParams(BuildContext context) { @@ -264,7 +292,7 @@ class _ImageSendPreviewSheetState extends State { } _RateEstimate _estimateFor(ImageSendRadio radio) { - final bytes = _encoded; + final bytes = _current?.payload; if (bytes != null) { final exact = _estimate(bytes.length, radio); return _RateEstimate( @@ -293,14 +321,11 @@ class _ImageSendPreviewSheetState extends State { ); } - Future _onSend() async { + void _onSend() { + final version = _current; + final payload = version?.payload; + if (version == null || payload == null) return; setState(() => _sending = true); - final payload = await _encode(); - if (!mounted) return; - if (payload == null) { - setState(() => _sending = false); - return; - } final radio = _radio ?? const ImageSendRadio(); final estimate = _estimate(payload.length, radio); Navigator.of(context).pop( @@ -311,6 +336,7 @@ class _ImageSendPreviewSheetState extends State { packetCount: estimate.chunkCount, airtime: estimate.totalAirtime, wallClock: estimate.pacedWallClock, + previewPng: version.png, ), ); } @@ -375,7 +401,7 @@ class _ImageSendPreviewSheetState extends State { ..._warnings(theme, colors, radio, selected), const SizedBox(height: 4), _parityTile(theme, colors), - if (_failed) ...[ + if (_current?.failed ?? false) ...[ const SizedBox(height: 12), _banner( colors: colors, @@ -408,6 +434,9 @@ class _ImageSendPreviewSheetState extends State { ), ); + /// The main preview: by default the selected version as recipients + /// will see it, with the original one tap away for comparison. + /// /// The preview is deliberately capped in height: the packet count and airtime /// below it are the reason this screen exists and must not be pushed off the /// first screenful by a full-width square. @@ -419,59 +448,96 @@ class _ImageSendPreviewSheetState extends State { ) { final l10n = context.l10n; final maxSide = MediaQuery.sizeOf(context).height * 0.28; - final reconstruction = _showReconstruction ? _reconstruction : null; + final version = _current; + final canPreview = ready && !_previewUnavailable; + final showReconstruction = canPreview && !_showOriginal && version != null; + final png = showReconstruction ? version.png : null; + final pending = + showReconstruction && + png == null && + (version.encoding || version.decoding); + final broken = showReconstruction && png == null && !pending; + + Widget content; + if (pending) { + content = Center( + child: Padding( + padding: const EdgeInsets.all(12), + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + const CircularProgressIndicator(), + const SizedBox(height: 12), + Text( + l10n.imageSend_preparingPreview, + textAlign: TextAlign.center, + style: theme.textTheme.bodyMedium?.copyWith( + color: colors.onSurfaceVariant, + ), + ), + ], + ), + ), + ); + } else if (broken) { + content = Center( + child: Icon( + Icons.broken_image_outlined, + size: 48, + color: colors.onSurfaceVariant, + ), + ); + } else { + content = Image.memory( + png ?? widget.imageBytes, + // The reconstruction is the stretched square; fill undoes the stretch + // into the recipient's shape. + fit: png != null ? BoxFit.fill : BoxFit.contain, + alignment: Alignment.center, + gaplessPlayback: true, + errorBuilder: (context, error, stack) => Center( + child: Icon( + Icons.broken_image_outlined, + size: 48, + color: colors.onSurfaceVariant, + ), + ), + ); + } + return Column( children: [ - if (ready) ...[ + if (canPreview) ...[ SegmentedButton( segments: [ ButtonSegment( value: false, - label: MeshSegmentLabel(l10n.imageSend_viewOriginal), + label: MeshSegmentLabel(l10n.imageSend_viewReconstruction), ), ButtonSegment( value: true, - label: MeshSegmentLabel(l10n.imageSend_viewReconstruction), - enabled: _encoded != null && !_reconstructionUnavailable, + label: MeshSegmentLabel(l10n.imageSend_viewOriginal), ), ], - selected: {_showReconstruction}, - onSelectionChanged: (s) => _selectView(s.first), + selected: {_showOriginal}, + onSelectionChanged: (s) => setState(() => _showOriginal = s.first), ), const SizedBox(height: 12), ], ConstrainedBox( constraints: BoxConstraints(maxHeight: maxSide, maxWidth: maxSide), child: ClipRRect( - borderRadius: BorderRadius.circular(12), + borderRadius: BorderRadius.circular(MeshRadii.md), child: AspectRatio( - aspectRatio: reconstruction != null ? _recipientAspect : _aspect, - child: Container( + aspectRatio: showReconstruction ? _recipientAspect : _aspect, + child: ColoredBox( color: colors.surfaceContainerHighest, - child: _showReconstruction && _reconstructing - ? const Center(child: CircularProgressIndicator()) - : Image.memory( - reconstruction ?? widget.imageBytes, - // The reconstruction is the stretched square; fill - // undoes the stretch into the recipient's shape. - fit: reconstruction != null - ? BoxFit.fill - : BoxFit.contain, - alignment: Alignment.center, - gaplessPlayback: true, - errorBuilder: (context, error, stack) => Center( - child: Icon( - Icons.broken_image_outlined, - size: 48, - color: colors.onSurfaceVariant, - ), - ), - ), + child: content, ), ), ), ), - if (_reconstructionUnavailable) ...[ + if (_previewUnavailable) ...[ const SizedBox(height: 8), Text( l10n.imageSend_reconstructionUnavailable, @@ -481,6 +547,10 @@ class _ImageSendPreviewSheetState extends State { ), ), ], + if (canPreview && _versions.isNotEmpty) ...[ + const SizedBox(height: 16), + _versionsRow(theme, colors), + ], const SizedBox(height: 8), Text( l10n.imageSend_lossyNote( @@ -496,6 +566,152 @@ class _ImageSendPreviewSheetState extends State { ); } + /// The versions made so far, plus "Try another version". + Widget _versionsRow(ThemeData theme, ColorScheme colors) { + final l10n = context.l10n; + return Column( + children: [ + Wrap( + alignment: WrapAlignment.center, + spacing: 8, + runSpacing: 8, + children: [ + for (var i = 0; i < _versions.length; i++) + _versionTile(theme, colors, i), + ], + ), + if (_versions.length < kImageSendMaxVersions) ...[ + const SizedBox(height: 12), + OutlinedButton.icon( + onPressed: (_working || _sending) ? null : _addVersion, + icon: const Icon(Icons.refresh), + label: Text(l10n.imageSend_tryAnother), + ), + ], + const SizedBox(height: 8), + Text( + l10n.imageSend_versionsHint, + textAlign: TextAlign.center, + style: theme.textTheme.bodySmall?.copyWith( + color: colors.onSurfaceVariant, + ), + ), + ], + ); + } + + /// One selectable version: its reconstruction, "Version n" and its packet + /// count. Selection shows as a thick primary border plus a check badge, so + /// it never relies on colour alone. + Widget _versionTile(ThemeData theme, ColorScheme colors, int index) { + final l10n = context.l10n; + final version = _versions[index]; + final selected = index == _selected; + final label = l10n.imageSend_versionLabel(index + 1); + const maxSide = 80.0; + final aspect = _recipientAspect; + final width = aspect >= 1 ? maxSide : maxSide * aspect; + final height = aspect >= 1 ? maxSide / aspect : maxSide; + final payload = version.payload; + final packets = payload == null + ? null + : _estimate( + payload.length, + _radio ?? const ImageSendRadio(), + ).chunkCount; + + final Widget thumb; + if (version.png != null) { + thumb = Image.memory( + version.png!, + fit: BoxFit.fill, + gaplessPlayback: true, + ); + } else if (version.encoding || version.decoding) { + thumb = const Center( + child: SizedBox( + width: 20, + height: 20, + child: CircularProgressIndicator(strokeWidth: 2), + ), + ); + } else { + thumb = Icon(Icons.broken_image_outlined, color: colors.onSurfaceVariant); + } + + return Semantics( + button: true, + selected: selected, + label: label, + excludeSemantics: true, + child: InkWell( + borderRadius: BorderRadius.circular(MeshRadii.sm), + onTap: _sending ? null : () => _select(index), + child: Padding( + padding: const EdgeInsets.all(4), + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + Stack( + children: [ + Container( + width: width.clamp(48.0, maxSide), + height: height.clamp(48.0, maxSide), + clipBehavior: Clip.antiAlias, + decoration: BoxDecoration( + color: colors.surfaceContainerHighest, + borderRadius: BorderRadius.circular(MeshRadii.sm), + border: Border.all( + color: selected + ? colors.primary + : colors.outlineVariant, + width: selected ? 3 : 1, + ), + ), + child: thumb, + ), + if (selected) + Positioned( + top: 4, + right: 4, + child: Container( + padding: const EdgeInsets.all(2), + decoration: BoxDecoration( + color: colors.primary, + shape: BoxShape.circle, + ), + child: Icon( + Icons.check, + size: 16, + color: colors.onPrimary, + ), + ), + ), + ], + ), + const SizedBox(height: 4), + Text( + label, + style: theme.textTheme.bodySmall?.copyWith( + color: selected ? colors.onSurface : colors.onSurfaceVariant, + fontWeight: selected ? FontWeight.w600 : null, + ), + ), + if (packets != null) + Text( + l10n.imageSend_versionPackets(packets), + style: MeshTheme.mono( + fontSize: 12, + color: colors.onSurfaceVariant, + ), + ), + ], + ), + ), + ), + ); + } + Widget _sizeRow(ThemeData theme, ColorScheme colors, _RateEstimate estimate) { return Row( mainAxisAlignment: MainAxisAlignment.center, @@ -794,7 +1010,8 @@ class _ImageSendPreviewSheetState extends State { Widget _actions(ThemeData theme, bool ready) { final localizations = context.l10n; - final busy = _sending || _encoding; + final busy = _sending || (_current?.encoding ?? false); + final canSend = ready && !_sending && _current?.payload != null; return Padding( padding: const EdgeInsets.fromLTRB(16, 8, 16, 16), child: Row( @@ -810,7 +1027,7 @@ class _ImageSendPreviewSheetState extends State { const SizedBox(width: 12), Expanded( child: FilledButton.icon( - onPressed: (!ready || busy) ? null : _onSend, + onPressed: canSend ? _onSend : null, icon: busy ? const SizedBox( width: 16, diff --git a/lib/widgets/received_image_message.dart b/lib/widgets/received_image_message.dart index 6c4debdf..7fc4d55b 100644 --- a/lib/widgets/received_image_message.dart +++ b/lib/widgets/received_image_message.dart @@ -19,10 +19,13 @@ /// /// Both are private to this file, are not parameterised, and are reused /// verbatim by the full-screen viewer this widget opens on tap, so there is no -/// route in the app that can show the pixels without them. The only case in -/// which they are absent is [ReceivedImageEntry.isOutgoing], where the PNG is -/// the sender's own 512x512 crop and the label would be a falsehood — that is -/// read from the store's sidecar, never from a constructor argument. +/// route in the app that can show the pixels without them. They are absent +/// only when [ReceivedImageEntry.synthesized] is false: the sender's own image +/// on a device that could not decode, where the PNG is their stretched +/// original and the label would be a falsehood. The sender's own image is +/// otherwise the decoded version they picked, so it is labelled like any +/// other. That is read from the store's sidecar, never from a constructor +/// argument. /// /// The widget deliberately exposes no share/save action: an exported raster /// would have to have the caption composited into it first, and that belongs diff --git a/test/connector/image_echo_test.dart b/test/connector/image_echo_test.dart new file mode 100644 index 00000000..49531f88 --- /dev/null +++ b/test/connector/image_echo_test.dart @@ -0,0 +1,141 @@ +import 'dart:typed_data'; + +import 'package:crypto/crypto.dart' as crypto; +import 'package:flutter_test/flutter_test.dart'; +import 'package:meshcore_open/connector/meshcore_connector.dart'; +import 'package:meshcore_open/connector/meshcore_protocol.dart'; +import 'package:meshcore_open/models/channel.dart'; +import 'package:meshcore_open/review_mode/review_radio.dart'; +import 'package:meshcore_open/review_mode/review_script.dart'; +import 'package:meshcore_open/services/image_chunk_transport.dart'; +import 'package:meshcore_open/services/message_retry_service.dart'; +import 'package:meshcore_open/services/path_history_service.dart'; +import 'package:meshcore_open/services/storage_service.dart'; +import 'package:meshcore_open/storage/prefs_manager.dart'; +import 'package:meshcore_open/widgets/image_send_codec_binding.dart'; +import 'package:pointycastle/export.dart'; +import 'package:shared_preferences/shared_preferences.dart'; + +Future _waitFor(bool Function() condition, {int seconds = 8}) async { + final end = DateTime.now().add(Duration(seconds: seconds)); + while (!condition()) { + if (DateTime.now().isAfter(end)) fail('Timed out waiting for condition'); + await Future.delayed(const Duration(milliseconds: 20)); + } +} + +Uint8List _psk(int slot) => + Channel.parsePskHex(ReviewScript.channels[slot].pskHex); + +/// A raw-log frame for a repeater re-flooding [blob] as GRP_DATA on [psk]'s +/// channel, encrypted the way the firmware's createGroupDatagram does. +Uint8List _logRxGroupData(Uint8List psk, Uint8List blob, {int hops = 1}) { + final plain = [ + dataTypeAeicImage & 0xFF, + dataTypeAeicImage >> 8, + blob.length, + ...blob, + ]; + final padded = Uint8List(((plain.length + 15) ~/ 16) * 16) + ..setRange(0, plain.length, plain); + final cipher = ECBBlockCipher(AESEngine())..init(true, KeyParameter(psk)); + final encrypted = Uint8List(padded.length); + for (var i = 0; i < padded.length; i += 16) { + cipher.processBlock(padded, i, encrypted, i); + } + final key32 = Uint8List(32)..setRange(0, 16, psk); + final mac = crypto.Hmac(crypto.sha256, key32).convert(encrypted).bytes; + final channelHash = crypto.sha256.convert(psk).bytes[0]; + return Uint8List.fromList([ + pushCodeLogRxData, + 40, // snr + 0xA0, // rssi + 0x19, // flood route, GRP_DATA payload + hops, // path length: one 1-byte hash per hop + for (var i = 0; i < hops; i++) 0x42 + i, + channelHash, + mac[0], + mac[1], + ...encrypted, + ]); +} + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + + late MeshCoreConnector connector; + + // One connector for the file: the notification service it starts accepts a + // single listener per process. + setUpAll(() async { + SharedPreferences.setMockInitialValues({}); + PrefsManager.reset(); + await PrefsManager.initialize(); + connector = MeshCoreConnector(); + final retryService = MessageRetryService(); + connector.initialize( + retryService: retryService, + pathHistoryService: PathHistoryService(StorageService()), + ); + final radio = ReviewRadio( + scheduledMessageInterval: const Duration(hours: 1), + ackDelay: const Duration(milliseconds: 50), + replyDelay: const Duration(hours: 1), + ); + connector.reviewRadioFactoryForTest = () => radio; + await connector.connectReview(); + await _waitFor(() => connector.channelMapConfirmed); + }); + + tearDownAll(() async { + await connector.disconnect(); + connector.dispose(); + }); + + Future sendImage({ + required int slot, + required int imgId, + }) async { + final chunkSet = buildImageChunks( + payload: Uint8List.fromList(List.generate(200, (i) => i & 0xFF)), + metadata: const ImageStreamMetadata(rate: ImageCodecRatePoint.standard), + senderPrefix: senderPrefixFromKey(connector.selfPublicKey)!, + imgId: imgId, + ); + expect( + await connector.sendImageChunks( + chunkSet.blobs, + channelIndex: slot, + interChunkDelay: Duration.zero, + ), + isTrue, + ); + return chunkSet; + } + + test('a repeater echo of our own chunk is counted for that chunk', () async { + final chunkSet = await sendImage(slot: 0, imgId: 77); + expect(chunkSet.blobs.length, 3, reason: '2 data chunks + parity'); + expect(connector.ownImageEchoes(0, 77), [0, 0, 0]); + + connector.handleFrameForTest(_logRxGroupData(_psk(0), chunkSet.blobs[1])); + connector.handleFrameForTest(_logRxGroupData(_psk(0), chunkSet.blobs[1])); + connector.handleFrameForTest(_logRxGroupData(_psk(0), chunkSet.blobs[2])); + + expect(connector.ownImageEchoes(0, 77), [0, 2, 1]); + }); + + test('chunks from someone else, or on another channel key, are not ' + 'counted', () async { + final chunkSet = await sendImage(slot: 0, imgId: 78); + + // Same image id, different sender. + final foreign = Uint8List.fromList(chunkSet.blobs[0]); + foreign[0] ^= 0xFF; + connector.handleFrameForTest(_logRxGroupData(_psk(0), foreign)); + // Our chunk, but encrypted for a different channel than it was sent on. + connector.handleFrameForTest(_logRxGroupData(_psk(1), chunkSet.blobs[0])); + + expect(connector.ownImageEchoes(0, 78), [0, 0, 0]); + }); +} diff --git a/test/services/received_image_store_test.dart b/test/services/received_image_store_test.dart index 8dd9f0c8..6a72569e 100644 --- a/test/services/received_image_store_test.dart +++ b/test/services/received_image_store_test.dart @@ -826,6 +826,73 @@ void main() { expect(h.decoder.calls, 0); }); + test('a decoded preview is labelled synthesized and keeps its byte count ' + 'and echoes', () async { + final h = _build(); + final entry = await h.store.registerOutgoing( + channelIndex: 1, + senderPrefix: kSelf, + imgId: 4, + previewPng: Uint8List.fromList(List.filled(64, 1)), + rate: AeicRatePoint.ft32, + chunkCount: 2, + synthesized: true, + payloadBytes: 171, + chunkEchoes: const [1, 0, 2], + ); + expect(entry.synthesized, isTrue); + expect(entry.bitstreamByteCount, 171); + expect(entry.bitstreamStored, isFalse); + expect(entry.minDataChunkEchoes, 0); + + final restored = ReceivedImageEntry.fromJson(entry.toJson()); + expect(restored.synthesized, isTrue); + expect(restored.chunkEchoes, [1, 0, 2]); + }); + + test( + 'setOutgoingEchoes updates the newest matching outgoing image', + () async { + final h = _build(); + final older = await h.store.registerOutgoing( + channelIndex: 1, + senderPrefix: kSelf, + imgId: 9, + previewPng: Uint8List.fromList(List.filled(64, 1)), + rate: AeicRatePoint.ft32, + chunkCount: 1, + ); + h.advance(const Duration(seconds: 5)); + final newer = await h.store.registerOutgoing( + channelIndex: 1, + senderPrefix: kSelf, + imgId: 9, + previewPng: Uint8List.fromList(List.filled(64, 2)), + rate: AeicRatePoint.ft32, + chunkCount: 1, + ); + + await h.store.setOutgoingEchoes( + channelIndex: 1, + senderPrefix: kSelf, + imgId: 9, + echoes: const [3, 1], + ); + expect(h.store.entryFor(newer.streamId)!.chunkEchoes, [3, 1]); + expect(h.store.entryFor(newer.streamId)!.minDataChunkEchoes, 3); + expect(h.store.entryFor(older.streamId)!.chunkEchoes, isNull); + + // Another channel's image with the same id is left alone. + await h.store.setOutgoingEchoes( + channelIndex: 2, + senderPrefix: kSelf, + imgId: 9, + echoes: const [7, 7], + ); + expect(h.store.entryFor(newer.streamId)!.chunkEchoes, [3, 1]); + }, + ); + test('spent outgoing entries count toward maxEntries', () async { final h = _build(maxEntries: 2, maxAge: const Duration(minutes: 10)); final ids = []; diff --git a/test/widgets/image_send_codec_binding_test.dart b/test/widgets/image_send_codec_binding_test.dart new file mode 100644 index 00000000..da5aca7f --- /dev/null +++ b/test/widgets/image_send_codec_binding_test.dart @@ -0,0 +1,57 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:meshcore_open/widgets/image_send_codec_binding.dart'; + +void main() { + group('rerollSourceRect', () { + const sizes = [ + (4000, 3000), + (3000, 4000), + (1920, 1080), + (512, 512), + (7, 3), + ]; + + test('variant 0 is the whole photo', () { + for (final (w, h) in sizes) { + final r = rerollSourceRect(w, h, 0); + expect(r.left, 0); + expect(r.top, 0); + expect(r.width, w); + expect(r.height, h); + } + }); + + test('every variant stays inside the photo and keeps its shape', () { + for (final (w, h) in sizes) { + for (var v = 0; v < kImageSendMaxVersions; v++) { + final r = rerollSourceRect(w, h, v); + expect(r.left, greaterThanOrEqualTo(0), reason: '$w x $h v$v'); + expect(r.top, greaterThanOrEqualTo(0), reason: '$w x $h v$v'); + expect(r.left + r.width, lessThanOrEqualTo(w + 1e-9)); + expect(r.top + r.height, lessThanOrEqualTo(h + 1e-9)); + // Same aspect, so the version keeps the photo's aspect code. + expect(r.width / r.height, closeTo(w / h, (w / h) * 0.01)); + // "The whole photo": at most a few percent trimmed. + expect(r.width, greaterThanOrEqualTo(w * 0.95)); + } + } + }); + + test('variants differ from each other and are deterministic', () { + final rects = [ + for (var v = 0; v < kImageSendMaxVersions; v++) + rerollSourceRect(4000, 3000, v), + ]; + expect(rects.toSet().length, kImageSendMaxVersions); + expect(rerollSourceRect(4000, 3000, 3), rects[3]); + }); + + test('rejects a variant outside the table', () { + expect(() => rerollSourceRect(10, 10, -1), throwsRangeError); + expect( + () => rerollSourceRect(10, 10, kImageSendMaxVersions), + throwsRangeError, + ); + }); + }); +} diff --git a/test/widgets/image_send_preview_sheet_test.dart b/test/widgets/image_send_preview_sheet_test.dart index 749690b4..23271e3e 100644 --- a/test/widgets/image_send_preview_sheet_test.dart +++ b/test/widgets/image_send_preview_sheet_test.dart @@ -336,4 +336,145 @@ void main() { final l10n = await AppLocalizations.delegate.load(const Locale('en')); expect(_texts(tester), contains(l10n.imageSend_codecUnavailable)); }); + + group('versions', () { + final previewCodec = FakeImageSendCodec( + latency: Duration.zero, + previewPng: _onePixelPng, + ); + + Future tapText(WidgetTester tester, String text) async { + final finder = find.text(text); + await tester.tap(finder); + await tester.pumpAndSettle(); + } + + Future send( + WidgetTester tester, + ImageSendPreviewResult? Function() result, + ) async { + await tester.tap(find.byType(FilledButton)); + await tester.pumpAndSettle(); + return result(); + } + + testWidgets('opens on what recipients see, as Version 1', (tester) async { + _tallScreen(tester); + await _openSheet(tester, radio: _knownRadio, codec: previewCodec); + + final l10n = await AppLocalizations.delegate.load(const Locale('en')); + final texts = _texts(tester); + expect(texts, contains(l10n.imageSend_versionLabel(1))); + expect(texts, contains(l10n.imageSend_tryAnother)); + expect(texts, isNot(contains(l10n.imageSend_reconstructionUnavailable))); + final segments = tester.widget>( + find.byType(SegmentedButton), + ); + expect(segments.selected, { + false, + }, reason: 'the reconstruction, not the original, is the default view'); + }); + + testWidgets('trying another version selects it and sends its payload and ' + 'reconstruction', (tester) async { + _tallScreen(tester); + ImageSendPreviewResult? result; + await tester.pumpWidget(_host(previewCodec, (r) => result = r)); + await tester.tap(find.text('open')); + await tester.pumpAndSettle(); + final l10n = await AppLocalizations.delegate.load(const Locale('en')); + + await tapText(tester, l10n.imageSend_tryAnother); + expect(_texts(tester), contains(l10n.imageSend_versionLabel(2))); + + final sent = await send(tester, () => result); + expect( + sent!.payload.length, + FakeImageSendCodec.payloadBytesFor(kImageSendRatePoint, 1), + ); + expect(sent.previewPng, isNotNull); + }); + + testWidgets('going back to Version 1 sends Version 1', (tester) async { + _tallScreen(tester); + ImageSendPreviewResult? result; + await tester.pumpWidget(_host(previewCodec, (r) => result = r)); + await tester.tap(find.text('open')); + await tester.pumpAndSettle(); + final l10n = await AppLocalizations.delegate.load(const Locale('en')); + + await tapText(tester, l10n.imageSend_tryAnother); + await tapText(tester, l10n.imageSend_versionLabel(1)); + + final sent = await send(tester, () => result); + expect( + sent!.payload.length, + FakeImageSendCodec.payloadBytesFor(kImageSendRatePoint, 0), + ); + }); + + testWidgets('stops offering versions at the maximum', (tester) async { + _tallScreen(tester); + await _openSheet(tester, radio: _knownRadio, codec: previewCodec); + final l10n = await AppLocalizations.delegate.load(const Locale('en')); + + for (var i = 1; i < kImageSendMaxVersions; i++) { + await tapText(tester, l10n.imageSend_tryAnother); + } + final texts = _texts(tester); + expect( + texts, + contains(l10n.imageSend_versionLabel(kImageSendMaxVersions)), + ); + expect(texts, isNot(contains(l10n.imageSend_tryAnother))); + }); + + testWidgets('a device that cannot decode shows the original and no ' + 'versions', (tester) async { + _tallScreen(tester); + await _openSheet(tester, radio: _knownRadio); + + final l10n = await AppLocalizations.delegate.load(const Locale('en')); + final texts = _texts(tester); + expect(texts, contains(l10n.imageSend_reconstructionUnavailable)); + expect(texts, isNot(contains(l10n.imageSend_versionLabel(1)))); + expect(texts, isNot(contains(l10n.imageSend_tryAnother))); + expect(find.byType(SegmentedButton), findsNothing); + }); + }); } + +/// Room for the whole sheet, so version tiles and "Try another version" are +/// on screen without scrolling the draggable sheet. +void _tallScreen(WidgetTester tester) { + tester.view.physicalSize = const Size(1000, 2400); + tester.view.devicePixelRatio = 1.0; + addTearDown(tester.view.reset); +} + +/// A host app whose button opens the sheet and reports what it returned. +Widget _host( + ImageSendCodec codec, + void Function(ImageSendPreviewResult?) onResult, +) => MaterialApp( + localizationsDelegates: AppLocalizations.localizationsDelegates, + supportedLocales: AppLocalizations.supportedLocales, + home: Builder( + builder: (context) => Scaffold( + body: Center( + child: ElevatedButton( + onPressed: () async => onResult( + await showImageSendPreviewSheet( + context: context, + imageBytes: _onePixelPng, + originalFileBytes: 1024, + codec: codec, + radio: _knownRadio, + ), + ), + child: const Text('open'), + ), + ), + ), + ), +); diff --git a/tools/aeic/README.md b/tools/aeic/README.md index a246f15d..c3ce2c76 100644 --- a/tools/aeic/README.md +++ b/tools/aeic/README.md @@ -9,6 +9,10 @@ Dart side is checked against. They are committed here so `test/services/golden/` is reproducible. Nothing in the app runs them. +To measure reconstruction quality and byte cost instead, use the +[evaluation harness](eval/README.md), which runs on the downloaded model bundle +and needs no research checkout. + | script | produces | used by | |---|---|---| | `export_golden.py` | `golden/aeic_cdf_ft32.bin`, `golden/vectors/*.gv` | `rans_coder_test.dart`, `entropy_tables_test.dart` | diff --git a/tools/aeic/eval/README.md b/tools/aeic/eval/README.md new file mode 100644 index 00000000..effd2c10 --- /dev/null +++ b/tools/aeic/eval/README.md @@ -0,0 +1,115 @@ +# AEIC quality evaluation + +`eval.py` measures how good received images are and how many bytes they cost, +so codec changes can be compared with numbers rather than by eye. It runs the +same ONNX graphs the app downloads. Unlike the fixture scripts in the parent +directory, it needs no research checkout. + +For each photo and each preprocessing variant it: + +1. prepares the 512×512 codec input (the `stretch` variant mirrors the app's `_toSquareRgb`), +2. runs `aeic_entropy_side_fp32_op17.onnx` on CPU, +3. counts the exact bytes with `aeic_bytes.dart`, which calls the app's own entropy layer and rANS coder, +4. runs the decoder on `y_hat` (GPU when available), +5. restores the aspect ratio the way the receiver does and scores the result against the region that was sent. + +## Setup + +```bash +python3 -m venv ~/.venvs/aeic-eval # or: uv venv ~/.venvs/aeic-eval +~/.venvs/aeic-eval/bin/pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 +~/.venvs/aeic-eval/bin/pip install -r tools/aeic/eval/requirements.txt +``` + +You also need `dart` on PATH (`flutter pub get` already run), and the model +bundle. By default the bundle is read from `~/Documents/image_codec_models`; +use `--models` to point elsewhere. The first run downloads the LPIPS, DISTS and +CLIP weights into the torch and Hugging Face caches. + +Check that the byte counter matches the reference coder: + +```bash +dart run tools/aeic/eval/aeic_bytes.dart --selftest test/services/golden/e2e/*.aeicrec +``` + +## Running + +From the repository root: + +```bash +# Kodak (24 images, downloaded once) +~/.venvs/aeic-eval/bin/python -I tools/aeic/eval/eval.py \ + --download-kodak ~/aeic-data/kodak --out ~/aeic-runs/kodak-baseline \ + --variants stretch,crop,crop43,pad + +# Your own photos; Kodak alone is not representative of what people send +~/.venvs/aeic-eval/bin/python -I tools/aeic/eval/eval.py \ + --images ~/aeic-data/mesh-photos --out ~/aeic-runs/mesh-baseline +``` + +| option | meaning | +|---|---| +| `--variants` | comma list of variants to compare (below) | +| `--rois FILE` | JSON `{image stem: [l, t, r, b]}` as fractions; adds subject-region (`roi_*`) scores and limits the run to listed images. `rois/kodak.json` has 17 Kodak subjects | +| `--display N` | long side of the receiver view that is scored (default 512; use 1024 for two-part variants) | +| `--decoder` | score a different decoder export, e.g. fp16 instead of int8 | +| `--limit N` | first N images only | +| `--cpu` | run the decoder on CPU (much slower) | + +| variant | what is sent | +|---|---| +| `stretch` | the whole photo stretched to the square, as the app does | +| `crop`, `crop43`, `pad` | centre square crop; crop to at most 4:3 then stretch; letterbox on grey | +| `reroll1`..`reroll5` | the whole photo with the send sheet's "Try another version" tweaks (`kImageSendRerollTweaks`; keep the two tables in step) | +| `select`, `select_sq` | only the `--rois` box; the box grown to a square | +| `q` | quantisation step `` instead of 1 (e.g. `q0.7` spends more bytes) | +| `fit` | per image, the finest step from the 16-step ladder that fits `` (e.g. `fit157` = one packet) | +| `hd_tiles`, `hd_detail` | two encodes per photo: two overlapping halves; or the whole photo plus the `--rois` box, blended | + +`q`/`fit` re-run the four context stages with the decode-side entropy graph; +at step 1 they reproduce the app's bytes exactly. Two-part byte counts include +the extra stream header such a format would need. + +To add an experiment, write a function returning `Prepared` and register it in +`VARIANTS`. + +## Comparing with JPEG XL and AVIF + +`compare_codecs.py` sweeps AEIC over every quantisation step and fits JPEG XL +and AVIF to the same budgets (16–192 px thumbnails, best size per image, scored +after upscaling). AVIF is counted both as a real `.avif` file and as its AV1 +data alone. Needs `imagecodecs` (in `requirements.txt`). + +```bash +~/.venvs/aeic-eval/bin/python -I tools/aeic/eval/compare_codecs.py \ + --images ~/aeic-data/kodak --out ~/aeic-runs/rd +``` + +## Output + +| file | contents | +|---|---| +| `report.html` | source next to each variant's reconstruction, with bytes, packets, LPIPS and CLIP; a red border marks CLIP < 0.80 | +| `results.csv` | one row per image × variant | +| `summary.json` | per-variant mean/median/min/max, packet histogram, run metadata | +| `recon/`, `ref/` | the scored images, at the receiver's aspect ratio | +| `tensors/*.aeic` | the bitstreams, exactly as the app would send them | + +## Reading the metrics + +| metric | better | what it catches | +|---|---|---| +| `bytes`, `data_chunks` | lower | airtime; one chunk carries up to 157 image bytes | +| `lpips`, `dists` | lower | perceptual difference; the main quality score for a generative decoder | +| `clip` | higher | whether the reconstruction shows the same *things* as the sent region; low values are the "fox came back as a sheep" failures | +| `clip_full` | higher | the same, against the whole original photo, so a crop pays for what it dropped | +| `psnr`, `ms_ssim` | higher | pixel fidelity; included for reference, but weak signals for this codec | + +`1-pkt` in the console summary is the share of images that fit in a single data +chunk. Each image also gets one parity packet when recovery is on. + +## Caveats + +- Desktop ORT on CPU is not bit-identical to ORT on a phone, so an occasional rounding difference can change a byte count by one. The byte counter itself matches the reference coder exactly (see `--selftest`). +- The `stretch` resize approximates `dart:ui`'s `FilterQuality.medium`; it is not pixel-identical. +- CLIP and LPIPS are proxies. Look at the images flagged in `report.html` before drawing conclusions. diff --git a/tools/aeic/eval/aeic_bytes.dart b/tools/aeic/eval/aeic_bytes.dart new file mode 100644 index 00000000..be3423c9 --- /dev/null +++ b/tools/aeic/eval/aeic_bytes.dart @@ -0,0 +1,240 @@ +// Byte counter for the AEIC evaluation harness (`eval.py`). +// +// Python runs the encode-side ONNX graph; this turns its tensors into the exact +// bitstream the app would send, using the app's own entropy layer and rANS +// coder rather than a port. A port could drift; this cannot, because it is the +// shipping code. +// +// Usage (from the repository root): +// +// dart run tools/aeic/eval/aeic_bytes.dart ... +// dart run tools/aeic/eval/aeic_bytes.dart --selftest ... +// +// Each `.f32` file holds little-endian float32 tensors back to back, in this +// order: z_q [1,128,4,4], yq0..yq3 [1,256,16,16], sc0..sc3 [1,256,16,16]. The +// bitstream is written next to it as `.aeic`, and one JSON object per input is +// printed to stdout. +// +// `--selftest` replays the golden `.aeicrec` recordings in +// `test/services/golden/e2e/` and checks that the output is byte-identical to +// the reference C++ bitstream, which shows the harness counts what the app +// sends. + +import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:meshcore_open/services/entropy_tables.dart'; +import 'package:meshcore_open/services/image_chunk_transport.dart'; +import 'package:meshcore_open/services/image_codec_entropy.dart'; +import 'package:meshcore_open/services/rans_coder.dart'; + +const int _resolution = 512; + +Future main(List args) async { + if (args.isEmpty) { + stderr.writeln( + 'usage: aeic_bytes.dart ...\n' + ' aeic_bytes.dart --selftest ...', + ); + exitCode = 64; + return; + } + if (args.first == '--selftest') { + exitCode = await _selfTest(args.skip(1).toList()); + return; + } + + final tables = EntropyTables.parse(File(args.first).readAsBytesSync()); + final geometry = AeicEntropyGeometry.forResolution(_resolution); + for (final path in args.skip(1)) { + final tensors = _readTensorFile(path, geometry); + final stream = await _encode(tables, geometry, tensors); + final outPath = path.replaceFirst(RegExp(r'\.f32$'), '.aeic'); + File(outPath).writeAsBytesSync(stream); + stdout.writeln( + jsonEncode({ + 'file': path, + 'bitstream': outPath, + 'bytes': stream.length, + 'data_chunks': imageDataChunkCount(stream.length), + }), + ); + } +} + +Future _encode( + EntropyTables tables, + AeicEntropyGeometry geometry, + AeicEncodeSideTensors tensors, +) { + final codec = AeicEntropyCodec( + geometry: geometry, + network: _FixedNetwork(tensors), + coders: _Coders(tables), + ); + // The network ignores its input; encode() only needs a correctly sized RGB + // buffer to get past its own conversion step. + return codec.encode(Uint8List(_resolution * _resolution * 3)); +} + +AeicEncodeSideTensors _readTensorFile( + String path, + AeicEntropyGeometry geometry, +) { + final bytes = File(path).readAsBytesSync(); + final z = geometry.zElements; + final y = geometry.yElements; + final expected = (z + 8 * y) * 4; + if (bytes.length != expected) { + throw FormatException( + '$path is ${bytes.length} bytes, expected $expected ' + '(z_q + 4 yq + 4 sc as float32)', + ); + } + final all = Float32List.sublistView(bytes); + Float32List at(int offset, int length) => + Float32List.sublistView(all, offset, offset + length); + return AeicEncodeSideTensors( + zQ: at(0, z), + yQ: [for (var i = 0; i < 4; i++) at(z + i * y, y)], + scales: [for (var i = 0; i < 4; i++) at(z + (4 + i) * y, y)], + ); +} + +Future _selfTest(List recordings) async { + if (recordings.isEmpty) { + stderr.writeln('--selftest needs at least one .aeicrec file'); + return 64; + } + final geometry = AeicEntropyGeometry.forResolution(_resolution); + var failures = 0; + for (final path in recordings) { + final r = _Recording.load(path); + final tables = EntropyTables.parse( + File( + '${File(path).parent.parent.path}/${r.meta['cdf_table']}', + ).readAsBytesSync(), + ); + final tensors = AeicEncodeSideTensors( + zQ: r.f32('enc/z_q'), + yQ: [for (var i = 0; i < 4; i++) r.f32('enc/yq$i')], + scales: [for (var i = 0; i < 4; i++) r.f32('enc/sc$i')], + ); + final got = await _encode(tables, geometry, tensors); + final want = r.u8('enc/bitstream'); + final ok = + got.length == want.length && + Iterable.generate(got.length).every((i) => got[i] == want[i]); + if (!ok) failures++; + stdout.writeln( + '${ok ? 'ok ' : 'FAIL'} ${r.meta['stem']}: ${got.length} B ' + '(reference ${want.length} B)', + ); + } + return failures == 0 ? 0 : 1; +} + +class _FixedNetwork implements AeicEntropyNetwork { + _FixedNetwork(this.tensors); + + final AeicEncodeSideTensors tensors; + + @override + bool get supportsDecodeSide => false; + + @override + Future runEncodeSide(Float32List imageChw) async => + tensors; + + @override + Future runHyperSynthesis(Float32List zQ) => + throw UnsupportedError('encode only'); + + @override + Future runStage(int stage, Float32List base) => + throw UnsupportedError('encode only'); +} + +class _Coders implements AeicRansCoderFactory { + _Coders(this.tables); + + final EntropyTables tables; + + @override + AeicRansEncoder createEncoder() => _Encoder(RansEncoder(tables)); + + @override + AeicRansDecoder createDecoder(Uint8List bitstream) => + throw UnsupportedError('encode only'); +} + +class _Encoder implements AeicRansEncoder { + _Encoder(this._encoder); + + final RansEncoder _encoder; + + @override + void pushSymbols(Int16List symbols, Int16List indexes, int cdfGroup) => + _encoder.encodeWithIndexes(symbols, indexes, cdfGroup); + + @override + Uint8List finish() => _encoder.finish(); +} + +/// Minimal reader for the `.aeicrec` container (see +/// `test/services/image_codec_e2e_test.dart` for the full one). +class _Recording { + _Recording(this.index, this.bytes) + : _entries = >{ + for (final e + in (index['entries'] as List).cast>()) + e['name'] as String: e, + }; + + final Map index; + final Uint8List bytes; + final Map> _entries; + + static _Recording load(String path) { + final bytes = File(path).readAsBytesSync(); + final bd = ByteData.sublistView(bytes); + if (ascii.decode(bytes.sublist(0, 8)) != 'AEICREC1') { + throw FormatException('bad .aeicrec magic in $path'); + } + final indexOffset = bd.getUint64(16, Endian.little); + final indexLength = bd.getUint32(24, Endian.little); + final index = + jsonDecode( + utf8.decode( + bytes.sublist(indexOffset, indexOffset + indexLength), + ), + ) + as Map; + return _Recording(index, bytes); + } + + Map get meta => index['meta'] as Map; + + Map _entry(String name) { + final e = _entries[name]; + if (e == null) throw StateError('no entry "$name" in recording'); + return e; + } + + Float32List f32(String name) { + final e = _entry(name); + final offset = e['offset'] as int; + return Float32List.sublistView( + bytes, + offset, + offset + (e['length'] as int), + ); + } + + Uint8List u8(String name) { + final e = _entry(name); + final offset = e['offset'] as int; + return Uint8List.sublistView(bytes, offset, offset + (e['length'] as int)); + } +} diff --git a/tools/aeic/eval/compare_codecs.py b/tools/aeic/eval/compare_codecs.py new file mode 100644 index 00000000..8271c923 --- /dev/null +++ b/tools/aeic/eval/compare_codecs.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +"""Rate-distortion comparison: AEIC (as shipped and with step rate control) +against JPEG XL and AVIF at the same byte budgets. + + python -I tools/aeic/eval/compare_codecs.py --images ~/aeic-data/kodak --out ~/aeic-runs/rd + +AEIC is swept over every quantisation step in STEP_LADDER (step 1.0 is the +stream the app sends today) and also fitted per image to 1 and 2 packets. +JPEG XL and AVIF are fitted per image to each byte budget: for every candidate +size the quality setting is searched to fit, the result is decoded and +upscaled to the display size, and the size with the lowest LPIPS is kept. That +choice uses the original, which the sender has, so it is a fair best case for +the conventional codecs rather than an oracle. + +AVIF is counted twice. `avif` is the real .avif file, whose ftyp+meta boxes +cost about 275 bytes before any image data. `avif_av1` counts only the AV1 +payload (the mdat contents), as if the receiver rebuilt the fixed container +from a template; that is AVIF's best case for a custom wire format. + +Everything is scored against the source at the same display size (long side +512, the AEIC output size). Needs `imagecodecs` with JPEG XL support on top of +the eval.py requirements. +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import struct +import sys +from pathlib import Path + +import numpy as np +from PIL import Image + +HERE = Path(__file__).resolve().parent +_spec = importlib.util.spec_from_file_location("aeic_eval", HERE / "eval.py") +ev = importlib.util.module_from_spec(_spec) +sys.modules["aeic_eval"] = ev # dataclasses look their module up here +_spec.loader.exec_module(ev) # type: ignore[union-attr] + +import imagecodecs # noqa: E402 (after eval.py, which loads torch first) +from concurrent.futures import ThreadPoolExecutor # noqa: E402 + +POOL = ThreadPoolExecutor(max_workers=12) + +PACKET_BUDGETS = {"1 packet": ev.CHUNK_FIRST_CAPACITY, + "2 packets": ev.CHUNK_FIRST_CAPACITY + ev.CHUNK_CAPACITY} +BUDGETS = [100, 157, 200, 250, 315, 400, 500, 700] +SIZES = [16, 24, 32, 40, 48, 56, 64, 80, 96, 128, 160, 192] +JXL_EFFORT = 9 +AVIF_SPEED = 2 +DISPLAY = ev.SIZE + + +def display_size(img: Image.Image, long_side: int) -> tuple[int, int]: + s = long_side / max(img.size) + return (max(1, round(img.width * s)), max(1, round(img.height * s))) + + +def mdat_payload(avif: bytes) -> int: + """Size of the AV1 data in an .avif file (its mdat box contents).""" + i = 0 + while i + 8 <= len(avif): + size, kind = struct.unpack(">I4s", avif[i:i + 8]) + header = 8 + if size == 1: + size = struct.unpack(">Q", avif[i + 8:i + 16])[0] + header = 16 + elif size == 0: + size = len(avif) - i + if kind == b"mdat": + return size - header + i += size + raise ValueError("no mdat box") + + +def _jxl(arr: np.ndarray, q: float) -> bytes: + return imagecodecs.jpegxl_encode(arr, distance=q, effort=JXL_EFFORT) + + +def _avif(arr: np.ndarray, q: float) -> bytes: + return imagecodecs.avif_encode(arr, level=int(q), speed=AVIF_SPEED) + + +# name -> (encode(arr, q), decode, bytes counted, q for the smallest file, +# q for the largest, integer q?) +CODECS = { + "jxl": (_jxl, imagecodecs.jpegxl_decode, len, 25.0, 0.05, False), + "avif": (_avif, imagecodecs.avif_decode, len, 0, 100, True), + "avif_av1": (_avif, imagecodecs.avif_decode, mdat_payload, 0, 100, True), +} + + +def codec_fit(name: str, src: Image.Image, long_side: int, budget: int) -> tuple[int, Image.Image] | None: + """Best stream of codec `name` for `src` at `long_side` px within `budget`. + + Returns (counted bytes, decoded image), or None if even the lowest quality + does not fit. + """ + enc, dec, count, q_small, q_large, integer = CODECS[name] + small = np.asarray(src.resize(display_size(src, long_side), Image.LANCZOS)) + data = enc(small, q_small) + if count(data) > budget: + return None + best = data + lo, hi = q_small, q_large # lo always fits + for _ in range(12): + if integer: + if abs(hi - lo) <= 1: + break + mid = (lo + hi) // 2 + else: + mid = (lo * hi) ** 0.5 # JPEG XL distance acts roughly logarithmically + data = enc(small, mid) + if count(data) <= budget: + best, lo = data, mid + else: + hi = mid + decoded = Image.fromarray(np.asarray(dec(best))[..., :3], "RGB") + return count(best), decoded + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--images", type=Path, required=True) + ap.add_argument("--out", type=Path, required=True) + ap.add_argument("--models", type=Path, default=ev.DEFAULT_MODELS) + ap.add_argument("--limit", type=int) + args = ap.parse_args() + + images = ev.find_images(args.images, args.limit) + out: Path = args.out + work = out / "tensors" + (out / "img").mkdir(parents=True, exist_ok=True) + work.mkdir(parents=True, exist_ok=True) + + import torch + gpu = torch.cuda.is_available() + codec = ev.Codec(args.models, args.models / ev.DECODER_GRAPH, gpu) + metrics = ev.Metrics("cuda" if gpu else "cpu") + + # ---- AEIC: every step on the ladder, one Dart run for all byte counts. + srcs, aeic_y = {}, {} + files = [] + for path in images: + stem = path.stem + src = ev.load_source(path) + srcs[stem] = src + side = codec.encode_side(ev.variant_stretch(src).square) + for step in ev.STEP_LADDER: + t = codec.encode_with_step(side, step) + tf = work / f"{stem}.s{step:.2f}.f32" + ev.write_tensor_file(tf, t) + aeic_y[(stem, step)] = t["y_hat"] + files.append(tf) + print(f"encoded {stem}", file=sys.stderr) + nbytes = {k: v["bytes"] for k, v in ev.count_bytes(args.models / ev.CDF_TABLE, files).items()} + for tf in files: + tf.unlink() + + rows = [] + emb = {} + + def score(stem: str, shown: Image.Image) -> dict: + src = srcs[stem] + ref = src.resize(display_size(src, DISPLAY), Image.LANCZOS) + if stem not in emb: + emb[stem] = metrics.clip_embed(src) + s = metrics.score(ref, shown.resize(ref.size, Image.BICUBIC), emb[stem]) + s.pop("clip_full") + return s + + aeic_shown = {} + for path in images: + stem = path.stem + src = srcs[stem] + for step in ev.STEP_LADDER: + recon = codec.decode(aeic_y[(stem, step)]) + shown = recon.resize(display_size(src, DISPLAY), Image.LANCZOS) + aeic_shown[(stem, step)] = shown + b = nbytes[str(work / f"{stem}.s{step:.2f}.f32")] + rows.append({"image": stem, "codec": "aeic", "step": step, "bytes": b, **score(stem, shown)}) + print(f"scored AEIC {stem}", file=sys.stderr) + + # ---- Conventional codecs: per image and budget, the best size that fits. + for name in CODECS: + for path in images: + stem = path.stem + src = srcs[stem] + for budget in BUDGETS: + best = None + # imagecodecs releases the GIL, so the size search runs in threads. + fits = list(POOL.map(lambda ls: codec_fit(name, src, ls, budget), SIZES)) + for long_side, fit in zip(SIZES, fits): + if fit is None: + continue + nb, small = fit + shown = small.resize(display_size(src, DISPLAY), Image.BICUBIC) + ref = src.resize(shown.size, Image.LANCZOS) + lp = float(metrics.lpips(metrics._t(shown), metrics._t(ref))) + if best is None or lp < best[0]: + best = (lp, long_side, nb, shown) + if best is None: + continue + _, long_side, nb, shown = best + rows.append({"image": stem, "codec": name, "budget": budget, "px": long_side, + "bytes": nb, **score(stem, shown)}) + if budget in PACKET_BUDGETS.values(): + shown.save(out / "img" / f"{stem}.{name}{budget}.png") + print(f"scored {name} {stem}", file=sys.stderr) + + # ---- AEIC fitted to the packet budgets, picked from the sweep. + for stem in srcs: + for label, budget in PACKET_BUDGETS.items(): + cands = [r for r in rows if r["codec"] == "aeic" and r["image"] == stem and r["bytes"] <= budget] + if not cands: + cands = [max((r for r in rows if r["codec"] == "aeic" and r["image"] == stem), + key=lambda r: r["step"])] + pick = min(cands, key=lambda r: r["step"]) + rows.append({**pick, "codec": f"aeic_fit{budget}", "budget": budget}) + aeic_shown[(stem, pick["step"])].save(out / "img" / f"{stem}.aeic{budget}.png") + aeic_shown[(stem, 1.0)].save(out / "img" / f"{stem}.aeic_today.png") + srcs[stem].resize(display_size(srcs[stem], DISPLAY), Image.LANCZOS).save(out / "img" / f"{stem}.src.png") + + (out / "rows.json").write_text(json.dumps(rows, indent=0)) + + def mean(rs, k): + return float(np.mean([r[k] for r in rs])) + + def line(name, rs): + return (f"{name:<22}{mean(rs, 'bytes'):>7.0f}{mean(rs, 'lpips'):>8.3f}{mean(rs, 'dists'):>8.3f}" + f"{mean(rs, 'clip'):>7.3f}{mean(rs, 'ms_ssim'):>8.3f}{mean(rs, 'psnr'):>7.2f}") + + print(f"\n{'':<22}{'bytes':>7}{'lpips':>8}{'dists':>8}{'clip':>7}{'msssim':>8}{'psnr':>7}") + print(line("AEIC today (step 1)", [r for r in rows if r["codec"] == "aeic" and r["step"] == 1.0])) + for label, budget in PACKET_BUDGETS.items(): + print(f"-- {label} ({budget} B)") + print(line(" AEIC step-fit", [r for r in rows if r["codec"] == f"aeic_fit{budget}"])) + for name in CODECS: + rs = [r for r in rows if r["codec"] == name and r["budget"] == budget] + print(line(f" {name} ({len(rs)} img)", rs) if rs else f" {name}: does not fit") + print("\nAEIC curve") + for step in ev.STEP_LADDER: + print(line(f" step {step:.2f}", [r for r in rows if r["codec"] == "aeic" and r["step"] == step])) + for name in CODECS: + print(f"{name} curve") + for budget in BUDGETS: + rs = [r for r in rows if r["codec"] == name and r["budget"] == budget] + if rs: + print(line(f" <= {budget} B ({len(rs)} img)", rs)) + + +if __name__ == "__main__": + main() diff --git a/tools/aeic/eval/eval.py b/tools/aeic/eval/eval.py new file mode 100644 index 00000000..e5dd446c --- /dev/null +++ b/tools/aeic/eval/eval.py @@ -0,0 +1,831 @@ +#!/usr/bin/env python3 +"""Quality/rate evaluation harness for the AEIC image codec. + +Runs a folder of photos through the same three ONNX graphs the app ships, +counts the exact bytes with the app's own Dart rANS coder (aeic_bytes.dart), +and scores what the receiver would see. + + python -I tools/aeic/eval/eval.py --images ~/photos --out /tmp/aeic-eval + +See README.md in this directory for setup and how to read the output. +""" + +from __future__ import annotations + +import argparse +import csv +import datetime as dt +import hashlib +import html +import json +import math +import os +import shutil +import subprocess +import sys +import time +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Callable + +import numpy as np +from PIL import Image, ImageOps + +# torch must be imported before onnxruntime so the CUDA libraries that ship in +# the torch wheels are already loaded when ORT's CUDA provider looks for them. +import torch # noqa: E402 (import order is deliberate) +import onnxruntime as ort # noqa: E402 + +SIZE = 512 +REPO = Path(__file__).resolve().parents[3] +DART_TOOL = Path(__file__).resolve().parent / "aeic_bytes.dart" +DEFAULT_MODELS = Path.home() / "Documents" / "image_codec_models" +SIDE_GRAPH = "aeic_entropy_side_fp32_op17.onnx" +DECODER_GRAPH = "aeic_decoder_qdq_conv_pct.onnx" +CDF_TABLE = "aeic_cdf_ft32.bin" +CONTEXT_GRAPH = "aeic_entropy_decode_fp32_op17.onnx" + + +def _stage_masks() -> list[np.ndarray]: + """The four checkerboard masks, as AeicMaskSet.isLive defines them.""" + perm = [0, 3, 2, 1] + c = np.arange(256)[:, None, None] // 64 + yy = np.arange(16)[None, :, None] + xx = np.arange(16)[None, None, :] + micro = ((yy & 1) << 1) | (xx & 1) + return [((c ^ perm[s]) == micro)[None] for s in range(4)] + + +STAGE_MASKS = _stage_masks() + +# Quantisation steps a stream could signal in 4 bits (the metadata byte's rate +# and resolution fields). 1.0 is today's ft32 stream. +STEP_LADDER = [0.40, 0.45, 0.50, 0.55, 0.60, 0.66, 0.72, 0.79, + 0.86, 0.93, 1.00, 1.10, 1.20, 1.32, 1.45, 1.60] +IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".webp", ".bmp", ".tif", ".tiff"} + +# Mirrors lib/services/image_chunk_transport.dart. aeic_bytes.dart reports the +# authoritative count; these are only used for the summary's packet totals. +PARITY_CHUNKS = 1 + + +# -------------------------------------------------------------------------- +# Preprocessing variants +# -------------------------------------------------------------------------- +# +# A variant turns the (EXIF-rotated) source photo into the 512x512 codec input +# and says which part of the source it represents, so the reconstruction can +# be compared with that region at its true aspect ratio, which is what the +# receiver displays. + + +@dataclass +class Prepared: + square: Image.Image # 512x512 RGB, what the encoder sees + region: tuple[int, int, int, int] # source box (l, t, r, b) it represents + content_box: tuple[int, int, int, int] # box inside `square` holding it + # For multi-part variants: width in source pixels of the cross-fade along + # this part's edges that lie inside the picture (not on its border). + feather: int = 0 + + +def _resize(img: Image.Image, size: tuple[int, int]) -> Image.Image: + # The app draws with dart:ui FilterQuality.medium: bilinear sampling over + # mipmaps. A box pre-reduce followed by bilinear is the closest PIL match. + return img.resize(size, Image.BILINEAR, reducing_gap=2.0) + + +def _full(img: Image.Image) -> tuple[int, int, int, int]: + return (0, 0, img.width, img.height) + + +def variant_stretch(img: Image.Image) -> Prepared: + """What the app does today: the whole frame squeezed into the square.""" + return Prepared(_resize(img, (SIZE, SIZE)), _full(img), (0, 0, SIZE, SIZE)) + + +def _centre_crop_box(img: Image.Image, max_ratio: float) -> tuple[int, int, int, int]: + w, h = img.size + ratio = w / h + if ratio > max_ratio: + cw = round(h * max_ratio) + left = (w - cw) // 2 + return (left, 0, left + cw, h) + if ratio < 1 / max_ratio: + ch = round(w * max_ratio) + top = (h - ch) // 2 + return (0, top, w, top + ch) + return _full(img) + + +def variant_crop(img: Image.Image) -> Prepared: + """Centre square crop: no distortion, loses the edges of the frame.""" + box = _centre_crop_box(img, 1.0) + return Prepared(_resize(img.crop(box), (SIZE, SIZE)), box, (0, 0, SIZE, SIZE)) + + +def variant_crop43(img: Image.Image) -> Prepared: + """Crop to at most 4:3 (or 3:4), then stretch what is left.""" + box = _centre_crop_box(img, 4 / 3) + return Prepared(_resize(img.crop(box), (SIZE, SIZE)), box, (0, 0, SIZE, SIZE)) + + +def variant_pad(img: Image.Image) -> Prepared: + """Letterbox on mid-grey: no distortion, spends latent area on the bars.""" + w, h = img.size + scale = SIZE / max(w, h) + nw, nh = max(1, round(w * scale)), max(1, round(h * scale)) + canvas = Image.new("RGB", (SIZE, SIZE), (128, 128, 128)) + left, top = (SIZE - nw) // 2, (SIZE - nh) // 2 + canvas.paste(_resize(img, (nw, nh)), (left, top)) + return Prepared(canvas, _full(img), (left, top, left + nw, top + nh)) + + +# Re-roll tweaks: (zoom, dx, dy) as fractions of the source. Mirrors +# kImageSendRerollTweaks / rerollSourceRect in +# lib/widgets/image_send_codec_binding.dart; keep the two in step. +REROLL_TWEAKS = [ + (1.0, 0.0, 0.0), + (1.02, 0.005, 0.0), + (1.02, -0.005, 0.005), + (1.04, 0.0, -0.005), + (1.03, 0.01, 0.01), + (1.03, -0.01, -0.01), +] + + +def reroll_source_rect(w: int, h: int, variant: int) -> tuple[float, float, float, float]: + """Source rect for re-roll `variant`: a near-invisible zoom and shift.""" + zoom, dx, dy = REROLL_TWEAKS[variant] + cw, ch = w / zoom, h / zoom + left = min(max((w - cw) / 2 + dx * w, 0.0), w - cw) + top = min(max((h - ch) / 2 + dy * h, 0.0), h - ch) + return (left, top, left + cw, top + ch) + + +def variant_reroll(img: Image.Image, variant: int) -> Prepared: + """Whole photo with re-roll tweak `variant`, stretched like `stretch`. + + Scored against the full frame: the tweak only drops a percent or two at + the edges, which is what the sender sees as "the whole photo". + """ + l, t, r, b = reroll_source_rect(img.width, img.height, variant) + # PIL's resize box takes float source coordinates, like drawImageRect. + square = img.resize((SIZE, SIZE), Image.BILINEAR, box=(l, t, r, b), reducing_gap=2.0) + return Prepared(square, _full(img), (0, 0, SIZE, SIZE)) + + +VARIANTS: dict[str, Callable[[Image.Image], Prepared]] = { + "stretch": variant_stretch, + "crop": variant_crop, + "crop43": variant_crop43, + "pad": variant_pad, + **{f"reroll{k}": (lambda img, k=k: variant_reroll(img, k)) for k in range(1, len(REROLL_TWEAKS))}, +} + +# Need a box per image (--rois), so they are not in VARIANTS. +SELECT = "select" +SELECT_SQUARE = "select_sq" +SELECT_VARIANTS = (SELECT, SELECT_SQUARE) + +Box = tuple[int, int, int, int] + + +def roi_box(img: Image.Image, frac: list[float]) -> Box: + """Fractional [l, t, r, b] -> pixel box in the (EXIF-rotated) source.""" + l, t, r, b = frac + w, h = img.size + return (round(l * w), round(t * h), round(r * w), round(b * h)) + + +def variant_select(img: Image.Image, box: Box) -> Prepared: + """The user's selection, stretched into the square like `stretch`. + + The aspect nibble restores the box's shape on the receiver, so this is the + app's existing pipeline applied to a crop the user chose. + """ + return Prepared(_resize(img.crop(box), (SIZE, SIZE)), box, (0, 0, SIZE, SIZE)) + + +def square_box(img: Image.Image, box: Box) -> Box: + """Grow `box` to a square around its centre, shifted to stay in the image. + + Models a selection tool locked to 1:1, so the codec never sees a stretch. + Capped at the image's short side. + """ + l, t, r, b = box + side = min(max(r - l, b - t), img.width, img.height) + cx, cy = (l + r) / 2, (t + b) / 2 + nl = int(round(min(max(cx - side / 2, 0), img.width - side))) + nt = int(round(min(max(cy - side / 2, 0), img.height - side))) + return (nl, nt, nl + side, nt + side) + + +# -------------------------------------------------------------------------- +# HD: two encoded parts per photo, aimed at a two-packet budget +# -------------------------------------------------------------------------- +# +# The model is fixed at 512x512 / ft32, so "HD" spends the second packet on a +# second 512x512 encode. The parts travel as one stream; the receiver decodes +# both and blends them. Overheads below are what the stream would need on top +# of the bitstreams (a hypothetical layout, not implemented in the app). + +HD_TILES = "hd_tiles" +HD_DETAIL = "hd_detail" +HD_VARIANTS = (HD_TILES, HD_DETAIL) +HD_OVERHEAD = { + HD_TILES: 1, # length of part 0, so the receiver can split the stream + HD_DETAIL: 1 + 4, # that, plus the detail box quantised to 4 bytes +} +TILE_OVERLAP = 0.08 # of the long side, cross-faded at the seam + + +def variant_hd_tiles(img: Image.Image) -> list[Prepared]: + """Two overlapping halves along the long side, each encoded at 512x512. + + Doubles the pixel count. A 3:2 photo splits into two near-square halves, + so each half is also stretched far less than the whole frame would be. + """ + w, h = img.size + long_side = max(w, h) + ov = round(long_side * TILE_OVERLAP) + half = (long_side + ov) // 2 + if w >= h: + boxes = [(0, 0, half, h), (w - half, 0, w, h)] + else: + boxes = [(0, 0, w, half), (0, h - half, w, h)] + return [Prepared(_resize(img.crop(b), (SIZE, SIZE)), b, (0, 0, SIZE, SIZE), feather=ov) + for b in boxes] + + +def variant_hd_detail(img: Image.Image, box: Box) -> list[Prepared]: + """The whole photo, plus the selected subject blended in at higher detail.""" + detail = variant_select(img, box) + detail.feather = max(4, round(0.08 * min(box[2] - box[0], box[3] - box[1]))) + return [variant_stretch(img), detail] + + +def composite(src: Image.Image, parts: list[Prepared], recons: list[Image.Image], + long_side: int) -> tuple[Image.Image, Image.Image, Box]: + """Reference and receiver view for a variant made of one or more parts. + + The view covers the union of the parts' regions at `long_side` pixels on + its long edge. Parts are laid down in order; each later part fades in over + `feather` pixels along its edges that are inside the union, so seams + cross-fade instead of cutting hard. + """ + region = (min(p.region[0] for p in parts), min(p.region[1] for p in parts), + max(p.region[2] for p in parts), max(p.region[3] for p in parts)) + rw, rh = region[2] - region[0], region[3] - region[1] + s = long_side / max(rw, rh) + size = (max(1, round(rw * s)), max(1, round(rh * s))) + ref = src.crop(region).resize(size, Image.LANCZOS) + canvas = np.zeros((size[1], size[0], 3), dtype=np.float64) + for i, (p, recon) in enumerate(zip(parts, recons)): + l, t, r, b = p.region + x0, y0 = round((l - region[0]) * s), round((t - region[1]) * s) + x1, y1 = round((r - region[0]) * s), round((b - region[1]) * s) + tile = np.asarray(recon.crop(p.content_box).resize((x1 - x0, y1 - y0), Image.LANCZOS), + dtype=np.float64) + alpha = np.ones((y1 - y0, x1 - x0), dtype=np.float64) + f = min(round(p.feather * s), alpha.shape[0], alpha.shape[1]) + if i > 0 and f > 0: + ramp = np.arange(1, f + 1, dtype=np.float64) / f + if l > region[0]: + alpha[:, :f] *= ramp[None, :] + if r < region[2]: + alpha[:, -f:] *= ramp[::-1][None, :] + if t > region[1]: + alpha[:f, :] *= ramp[:, None] + if b < region[3]: + alpha[-f:, :] *= ramp[::-1][:, None] + a = alpha[..., None] + canvas[y0:y1, x0:x1] = canvas[y0:y1, x0:x1] * (1 - a) + tile * a + shown = Image.fromarray(np.clip(np.rint(canvas), 0, 255).astype(np.uint8), "RGB") + return ref, shown, region + + +# Mirrors imageDataChunkCount() in image_chunk_transport.dart. +CHUNK_FIRST_CAPACITY = 157 +CHUNK_CAPACITY = 158 + + +def parse_rate_variant(v: str) -> dict | None: + """`q0.7` -> fixed step 0.7; `fit315` -> per-image step to fit 315 bytes.""" + try: + if v.startswith("q"): + return {"steps": [float(v[1:])], "budget": None} + if v.startswith("fit"): + return {"steps": STEP_LADDER, "budget": int(v[3:])} + except ValueError: + return None + return None + + +def data_chunks(payload_bytes: int) -> int: + n = max(payload_bytes, 0) + if n <= CHUNK_FIRST_CAPACITY: + return 1 + return 1 + math.ceil((n - CHUNK_FIRST_CAPACITY) / CHUNK_CAPACITY) + + +# -------------------------------------------------------------------------- +# Codec +# -------------------------------------------------------------------------- + + +class Codec: + def __init__(self, models: Path, decoder: Path, gpu: bool) -> None: + cpu = ["CPUExecutionProvider"] + cuda = ["CUDAExecutionProvider", "CPUExecutionProvider"] if gpu else cpu + # The entropy graph stays on CPU: its rounded outputs decide the bytes, + # and CPU fp32 is the closest stand-in for the phone. It costs ~0.1 s. + self.side = ort.InferenceSession(str(models / SIDE_GRAPH), providers=cpu) + self.context = ort.InferenceSession(str(models / CONTEXT_GRAPH), providers=cpu) + self.decoder = ort.InferenceSession(str(decoder), providers=cuda) + self.decoder_provider = self.decoder.get_providers()[0] + + def encode_side(self, square: Image.Image) -> dict[str, np.ndarray]: + rgb = np.asarray(square, dtype=np.uint8) + # Same float32 steps as aeicRgbToChw in image_codec_entropy.dart. + chw = rgb.transpose(2, 0, 1).astype(np.float32) / np.float32(255.0) + chw = (chw - np.float32(0.5)) * np.float32(2.0) + names = (["z_q", "y_hat", "y", "base0"] + + [f"yq{i}" for i in range(4)] + [f"sc{i}" for i in range(4)]) + out = self.side.run(names, {"image": chw[None]}) + return dict(zip(names, out)) + + def encode_with_step(self, side: dict[str, np.ndarray], step: float) -> dict[str, np.ndarray]: + """Re-quantise y with step `step` instead of 1 (rate control). + + Replays the four-stage context loop the receiver runs + (AeicEntropyCodec.decodeToLatent) with the decode-side graph, coding + round((y - mean) / step) against scale / step. The receiver's + reconstruction becomes symbol * step + mean. step < 1 spends more + bytes, step > 1 fewer. At step == 1 this reproduces the side graph's + symbols, which the harness's q1 check relies on. + """ + st = np.float32(step) + y, z_q = side["y"], side["z_q"] + base = side["base0"].copy() + out: dict[str, np.ndarray] = {"z_q": z_q} + for stage in range(4): + means, scales = self.context.run( + ["means", "scales"], + {"z_q": z_q, "base": base, "stage": np.array([stage], dtype=np.int32)}) + m = STAGE_MASKS[stage] + sym = np.where(m, np.rint((y - means) / st), np.float32(0)).astype(np.float32) + base = np.where(m, sym * st + means, base).astype(np.float32) + out[f"yq{stage}"] = sym + out[f"sc{stage}"] = np.where(m, scales / st, np.float32(0)).astype(np.float32) + out["y_hat"] = base + return out + + def decode(self, y_hat: np.ndarray) -> Image.Image: + (out,) = self.decoder.run(["image"], {"y_hat": y_hat.astype(np.float32)}) + # Same mapping as _chwFloatsToRgb in image_codec_backend.dart: clamp, + # then round half away from zero (Dart's double.round()). + scaled = (out[0].astype(np.float64) + 1.0) * 127.5 + rgb = np.floor(np.clip(scaled, 0, 255) + 0.5).astype(np.uint8) + return Image.fromarray(rgb.transpose(1, 2, 0), "RGB") + + +def write_tensor_file(path: Path, t: dict[str, np.ndarray]) -> None: + parts = [t["z_q"]] + [t[f"yq{i}"] for i in range(4)] + [t[f"sc{i}"] for i in range(4)] + with open(path, "wb") as f: + for p in parts: + f.write(np.ascontiguousarray(p, dtype=" dict[str, dict]: + if not tensor_files: + return {} + dart = shutil.which("dart") + if dart is None: + sys.exit("dart is not on PATH; it is needed to run aeic_bytes.dart") + cmd = [dart, "run", str(DART_TOOL), str(cdf), *map(str, tensor_files)] + proc = subprocess.run(cmd, cwd=REPO, capture_output=True, text=True) + if proc.returncode != 0: + sys.exit(f"aeic_bytes.dart failed:\n{proc.stderr}") + results = {} + for line in proc.stdout.splitlines(): + line = line.strip() + if line.startswith("{"): + row = json.loads(line) + results[row["file"]] = row + return results + + +# -------------------------------------------------------------------------- +# Metrics +# -------------------------------------------------------------------------- + + +class Metrics: + """Full-reference scores. Higher is better except LPIPS and DISTS.""" + + def __init__(self, device: str) -> None: + import open_clip + import piq + + self.piq = piq + self.device = device + self.lpips = piq.LPIPS(reduction="none").to(device).eval() + self.dists = piq.DISTS(reduction="none").to(device).eval() + model, _, preprocess = open_clip.create_model_and_transforms( + "ViT-B-32", pretrained="laion2b_s34b_b79k" + ) + self.clip = model.to(device).eval() + self.clip_pre = preprocess + + def _t(self, img: Image.Image) -> torch.Tensor: + a = np.asarray(img, dtype=np.float32) / 255.0 + return torch.from_numpy(a).permute(2, 0, 1)[None].to(self.device) + + @torch.no_grad() + def clip_embed(self, img: Image.Image) -> torch.Tensor: + # CLIP's own preprocessing centre-crops to a square, which would hide + # the frame edges -- exactly what crop variants throw away. Letterbox + # first so the whole image is scored. + side = max(img.size) + boxed = Image.new("RGB", (side, side), (0, 0, 0)) + boxed.paste(img, ((side - img.width) // 2, (side - img.height) // 2)) + x = self.clip_pre(boxed)[None].to(self.device) + e = self.clip.encode_image(x) + return e / e.norm(dim=-1, keepdim=True) + + @torch.no_grad() + def score(self, ref: Image.Image, out: Image.Image, full_src_emb: torch.Tensor) -> dict: + r, o = self._t(ref), self._t(out) + out_emb = self.clip_embed(out) + return { + "psnr": float(self.piq.psnr(o, r, data_range=1.0)), + "ms_ssim": float(self.piq.multi_scale_ssim(o, r, data_range=1.0)), + "lpips": float(self.lpips(o, r)), + "dists": float(self.dists(o, r)), + # Semantic agreement with exactly what was sent ... + "clip": float((self.clip_embed(ref) * out_emb).sum()), + # ... and with the whole original photo, so cropping is charged + # for what it threw away. + "clip_full": float((full_src_emb * out_emb).sum()), + } + + +def roi_pair(src: Image.Image, region: Box, shown: Image.Image, box: Box) -> tuple[Image.Image, Image.Image] | None: + """The ROI as the receiver would see it after zooming in, plus its reference. + + `shown` covers `region` of the source; the ROI is mapped into it and + both crops are scaled to long side 512. For a full-frame variant that is an + upscale of a few dozen pixels, which is exactly what zooming in shows. + Returns None when the variant did not send the whole ROI. + """ + rl, rt, rr, rb = region + l, t, r, b = box + if l < rl or t < rt or r > rr or b > rb: + return None + sx = shown.width / (rr - rl) + sy = shown.height / (rb - rt) + out = shown.crop((round((l - rl) * sx), round((t - rt) * sy), + round((r - rl) * sx), round((b - rt) * sy))) + ref = src.crop(box) + scale = SIZE / max(ref.size) + size = (max(1, round(ref.width * scale)), max(1, round(ref.height * scale))) + return ref.resize(size, Image.LANCZOS), out.resize(size, Image.BICUBIC) + + +# -------------------------------------------------------------------------- +# Corpus +# -------------------------------------------------------------------------- + +KODAK_URL = "https://r0k.us/graphics/kodak/kodak/kodim{:02d}.png" + + +def download_kodak(dest: Path) -> None: + dest.mkdir(parents=True, exist_ok=True) + for i in range(1, 25): + target = dest / f"kodim{i:02d}.png" + if target.exists(): + continue + print(f"downloading {target.name}", file=sys.stderr) + tmp = target.with_suffix(".part") + with urllib.request.urlopen(KODAK_URL.format(i), timeout=60) as r, open(tmp, "wb") as f: + shutil.copyfileobj(r, f) + tmp.rename(target) + + +def load_source(path: Path) -> Image.Image: + img = Image.open(path) + # Picked photos are shown upright in the app, so honour EXIF orientation. + img = ImageOps.exif_transpose(img) + return img.convert("RGB") + + +def find_images(folder: Path, limit: int | None) -> list[Path]: + files = sorted(p for p in folder.rglob("*") if p.suffix.lower() in IMAGE_SUFFIXES) + return files[:limit] if limit else files + + +# -------------------------------------------------------------------------- +# Report +# -------------------------------------------------------------------------- + +METRIC_COLS = ["bytes", "data_chunks", "psnr", "ms_ssim", "lpips", "dists", "clip", "clip_full"] +ROI_COLS = ["roi_psnr", "roi_ms_ssim", "roi_lpips", "roi_dists", "roi_clip"] +LOWER_IS_BETTER = {"bytes", "data_chunks", "lpips", "dists", "roi_lpips", "roi_dists"} + + +def summarise(rows: list[dict], variants: list[str]) -> dict: + summary = {} + for v in variants: + vr = [r for r in rows if r["variant"] == v] + if not vr: + continue + s: dict = {"images": len(vr)} + for m in METRIC_COLS + ROI_COLS: + vals = np.array([r[m] for r in vr if r.get(m) not in (None, "")], dtype=np.float64) + if vals.size == 0: + continue + s[m] = {"mean": float(vals.mean()), "median": float(np.median(vals)), + "min": float(vals.min()), "max": float(vals.max())} + chunks = [r["data_chunks"] for r in vr] + s["chunk_histogram"] = {str(k): chunks.count(k) for k in sorted(set(chunks))} + s["packets_mean_with_parity"] = float(np.mean(chunks)) + PARITY_CHUNKS + # The share of images whose CLIP agreement with the source is low. + # This is the "fox came back as a sheep" bucket; worth eyeballing. + s["clip_below_0.80"] = sum(1 for r in vr if r["clip"] < 0.80) + summary[v] = s + return summary + + +def print_summary(summary: dict) -> None: + head = (f"{'variant':<10}" + "".join(f"{m:>12}" for m in METRIC_COLS) + + f"{'1-pkt':>8}{'<=2-pkt':>9}{'clip<.8':>9}") + print(head) + print("-" * len(head)) + for v, s in summary.items(): + line = f"{v:<10}" + "".join(f"{s[m]['mean']:>12.4g}" for m in METRIC_COLS) + one = s["chunk_histogram"].get("1", 0) + two = one + s["chunk_histogram"].get("2", 0) + line += f"{one / s['images']:>7.0%} {two / s['images']:>8.0%} {s['clip_below_0.80']:>8}" + print(line) + print("means; lower is better for bytes, data_chunks, lpips, dists") + if any("roi_lpips" in s for s in summary.values()): + print() + print(f"{'variant':<10}" + "".join(f"{m:>13}" for m in ROI_COLS)) + for v, s in summary.items(): + if "roi_lpips" in s: + print(f"{v:<10}" + "".join(f"{s[m]['mean']:>13.4g}" for m in ROI_COLS)) + print("scored on the selected region only, as the receiver sees it after zooming in") + + +def write_html(out: Path, rows: list[dict], summary: dict, meta: dict) -> None: + def fmt(v: float, m: str) -> str: + return f"{v:.1f}" if m in ("bytes", "data_chunks") else f"{v:.3f}" + + variants = list(summary) + stems = list(dict.fromkeys(r["image"] for r in rows)) + by = {(r["image"], r["variant"]): r for r in rows} + + sum_rows = "".join( + f"{html.escape(v)}" + + "".join(f"{fmt(summary[v][m]['mean'], m)}" for m in METRIC_COLS) + + "" + for v in variants + ) + cards = [] + for stem in stems: + cells = [] + first = by[(stem, variants[0])] + cells.append( + f'
' + f"
source
" + ) + for v in variants: + r = by.get((stem, v)) + if not r: + continue + warn = " warn" if r["clip"] < 0.80 else "" + cells.append( + f'
' + f"
{html.escape(v)} {r['bytes']} B / {r['data_chunks']} pkt
" + f"LPIPS {r['lpips']:.3f} · CLIP {r['clip']:.3f}
" + ) + cards.append(f"

{html.escape(stem)}

{''.join(cells)}
") + + doc = f""" + +AEIC eval + +

AEIC eval

+

{html.escape(json.dumps(meta))}

+

Means

+{''.join(f'' for m in METRIC_COLS)}{sum_rows}
{m}
+

Images are shown at the aspect ratio the receiver restores. A red border means CLIP similarity to the sent region is below 0.80, so check it by eye.

+{''.join(cards)} +""" + (out / "report.html").write_text(doc, encoding="utf-8") + + +# -------------------------------------------------------------------------- +# Main +# -------------------------------------------------------------------------- + + +def sha256(path: Path) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + for block in iter(lambda: f.read(1 << 20), b""): + h.update(block) + return h.hexdigest() + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--images", type=Path, help="folder of photos (searched recursively)") + ap.add_argument("--download-kodak", type=Path, metavar="DIR", + help="download the 24 Kodak images to DIR and use them") + ap.add_argument("--out", type=Path, required=True, help="output folder") + ap.add_argument("--models", type=Path, default=DEFAULT_MODELS, + help=f"model bundle folder (default {DEFAULT_MODELS})") + ap.add_argument("--decoder", type=Path, help="alternative decoder .onnx (e.g. an fp16 export)") + ap.add_argument("--variants", default="stretch", + help=f"comma list from: {', '.join(VARIANTS)} (default stretch)") + ap.add_argument("--limit", type=int, help="only the first N images") + ap.add_argument("--cpu", action="store_true", help="run the decoder on CPU too") + ap.add_argument("--rois", type=Path, + help="JSON {image stem: [l, t, r, b] fractions}; enables the select and " + "hd_detail variants and ROI scores, and limits the run to listed images") + ap.add_argument("--display", type=int, default=SIZE, + help="long side in pixels of the receiver view that is scored (default 512; " + "use 1024 when comparing HD variants)") + args = ap.parse_args() + + if args.download_kodak: + download_kodak(args.download_kodak) + args.images = args.images or args.download_kodak + if not args.images: + ap.error("give --images or --download-kodak") + variants = [v.strip() for v in args.variants.split(",") if v.strip()] + unknown = [v for v in variants + if v not in VARIANTS and v not in SELECT_VARIANTS and v not in HD_VARIANTS + and parse_rate_variant(v) is None] + if unknown: + ap.error(f"unknown variants: {unknown}") + rois: dict[str, list[float]] = {} + if args.rois: + rois = {k: v for k, v in json.loads(args.rois.read_text()).items() if not k.startswith("_")} + if any(v in SELECT_VARIANTS + (HD_DETAIL,) for v in variants) and not rois: + ap.error("the select and hd_detail variants need --rois") + + images = find_images(args.images, None) + if rois: + images = [p for p in images if p.relative_to(args.images).with_suffix("").as_posix() in rois] + images = images[: args.limit] if args.limit else images + if not images: + sys.exit(f"no images under {args.images}") + + out: Path = args.out + work = out / "tensors" + dirs = [work] + [out / k / v for k in ("recon", "ref", "roi") for v in variants] + for d in dirs: + d.mkdir(parents=True, exist_ok=True) + + decoder_path = args.decoder or (args.models / DECODER_GRAPH) + gpu = not args.cpu and torch.cuda.is_available() + codec = Codec(args.models, decoder_path, gpu) + metrics = Metrics("cuda" if gpu else "cpu") + print(f"{len(images)} images x {len(variants)} variants; decoder on {codec.decoder_provider}", + file=sys.stderr) + + # Pass 1: preprocess + encode side, so every byte count is one Dart run. + jobs = [] + for path in images: + src = load_source(path) + key = path.relative_to(args.images).with_suffix("").as_posix() + stem = key.replace("/", "__") + box = roi_box(src, rois[key]) if key in rois else None + for v in variants: + if (v in SELECT_VARIANTS or v == HD_DETAIL) and box is None: + continue + if v == SELECT: + parts = [variant_select(src, box)] + elif v == SELECT_SQUARE: + parts = [variant_select(src, square_box(src, box))] + elif v == HD_TILES: + parts = variant_hd_tiles(src) + elif v == HD_DETAIL: + parts = variant_hd_detail(src, box) + elif parse_rate_variant(v) is not None: + parts = [variant_stretch(src)] + else: + parts = [VARIANTS[v](src)] + rate = parse_rate_variant(v) + steps = rate["steps"] if rate else [None] + t0 = time.perf_counter() + encoded = [] # per part: [(step, y_hat, tensor file), ...] + for i, p in enumerate(parts): + side = codec.encode_side(p.square) + cands = [] + for step in steps: + tensors = side if step is None else codec.encode_with_step(side, step) + tag = "" if step is None else f".s{step:.2f}" + tf = work / f"{stem}.{v}.p{i}{tag}.f32" + write_tensor_file(tf, tensors) + cands.append((step, tensors["y_hat"], tf)) + encoded.append(cands) + enc_s = time.perf_counter() - t0 + jobs.append((path, stem, v, src, parts, box, encoded, enc_s)) + byte_rows = count_bytes(args.models / CDF_TABLE, + [tf for j in jobs for cands in j[6] for _, _, tf in cands]) + + # Pick one candidate per part. A fit variant takes the finest step whose + # stream fits N bytes, or the coarsest step if none does. + for j in jobs: + rate = parse_rate_variant(j[2]) + chosen = [] + for cands in j[6]: + if rate and rate["budget"] is not None: + fitting = [c for c in cands if byte_rows[str(c[2])]["bytes"] <= rate["budget"]] + chosen.append(min(fitting, key=lambda c: c[0]) if fitting + else max(cands, key=lambda c: c[0])) + else: + chosen.append(cands[0]) + j[6][:] = chosen + + # Pass 2: decode + score. + rows = [] + emb_cache: dict[str, torch.Tensor] = {} + for n, (path, stem, v, src, parts, box, encoded, enc_s) in enumerate(jobs, 1): + t0 = time.perf_counter() + recons = [codec.decode(y_hat) for _, y_hat, _ in encoded] + dec_s = time.perf_counter() - t0 + ref, shown, region = composite(src, parts, recons, args.display) + part_bytes = [byte_rows[str(tf)]["bytes"] for _, _, tf in encoded] + total_bytes = sum(part_bytes) + HD_OVERHEAD.get(v, 0) + chunks = data_chunks(total_bytes) + if len(parts) == 1: + assert chunks == byte_rows[str(encoded[0][2])]["data_chunks"], "chunk mirror drifted" + if stem not in emb_cache: + emb_cache[stem] = metrics.clip_embed(src) + scores = metrics.score(ref, shown, emb_cache[stem]) + roi = roi_pair(src, region, shown, box) if box else None + if roi: + roi_scores = metrics.score(roi[0], roi[1], emb_cache[stem]) + scores.update({f"roi_{k}": roi_scores[k] for k in ("psnr", "ms_ssim", "lpips", "dists", "clip")}) + roi[1].save(out / "roi" / v / f"{stem}.png") + roi[0].save(out / "roi" / v / f"{stem}.ref.png") + else: + scores.update({c: "" for c in ROI_COLS}) + out_png = Path("recon") / v / f"{stem}.png" + ref_png = Path("ref") / v / f"{stem}.png" + shown.save(out / out_png) + ref.save(out / ref_png) + row = { + "image": stem, "variant": v, "source": str(path), + "width": src.width, "height": src.height, + "bytes": total_bytes, "data_chunks": chunks, + "part_bytes": "+".join(map(str, part_bytes)), + "step": "+".join("1.00" if st is None else f"{st:.2f}" for st, _, _ in encoded), + **scores, + "encode_s": round(enc_s, 3), "decode_s": round(dec_s, 3), + "out_png": out_png.as_posix(), "ref_png": ref_png.as_posix(), + "bitstream": ";".join(os.path.relpath(byte_rows[str(tf)]["bitstream"], out) + for _, _, tf in encoded), + } + rows.append(row) + print(f"[{n}/{len(jobs)}] {stem} {v}: {row['bytes']} B, LPIPS {row['lpips']:.3f}, " + f"CLIP {row['clip']:.3f}, decode {dec_s:.2f}s", file=sys.stderr) + + summary = summarise(rows, variants) + meta = { + "date": dt.datetime.now().isoformat(timespec="seconds"), + "images": str(args.images), "count": len(images), "variants": variants, + "decoder": decoder_path.name, "decoder_sha256": sha256(decoder_path)[:16], + "decoder_provider": codec.decoder_provider, + "onnxruntime": ort.__version__, + } + with open(out / "results.csv", "w", newline="") as f: + w = csv.DictWriter(f, fieldnames=list(rows[0])) + w.writeheader() + w.writerows(rows) + (out / "summary.json").write_text(json.dumps({"meta": meta, "variants": summary}, indent=1)) + write_html(out, rows, summary, meta) + for tf in work.glob("*.f32"): # 2 MB each; the .aeic bitstreams are kept + tf.unlink() + print() + print_summary(summary) + print(f"\nreport: {out / 'report.html'}") + + +if __name__ == "__main__": + main() diff --git a/tools/aeic/eval/requirements.txt b/tools/aeic/eval/requirements.txt new file mode 100644 index 00000000..6abe164e --- /dev/null +++ b/tools/aeic/eval/requirements.txt @@ -0,0 +1,10 @@ +# For the CUDA build of torch, install it first from the PyTorch index, e.g. +# pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 +torch +torchvision +onnxruntime-gpu # or onnxruntime for CPU only +numpy +pillow +piq +open_clip_torch +imagecodecs # compare_codecs.py: JPEG XL and AVIF diff --git a/tools/aeic/eval/rois/kodak.json b/tools/aeic/eval/rois/kodak.json new file mode 100644 index 00000000..ce6727fc --- /dev/null +++ b/tools/aeic/eval/rois/kodak.json @@ -0,0 +1,20 @@ +{ + "_comment": "Subject boxes for the Kodak set, as fractions of width/height: [left, top, right, bottom]. Stand-ins for a user's 'select' crop.", + "kodim02": [0.15, 0.15, 0.70, 0.75], + "kodim03": [0.15, 0.05, 0.65, 0.65], + "kodim04": [0.20, 0.15, 0.80, 0.65], + "kodim05": [0.00, 0.00, 0.50, 1.00], + "kodim06": [0.15, 0.15, 0.80, 0.85], + "kodim09": [0.35, 0.10, 0.95, 0.75], + "kodim11": [0.35, 0.30, 0.85, 0.85], + "kodim12": [0.25, 0.20, 0.65, 0.85], + "kodim14": [0.00, 0.15, 0.90, 0.85], + "kodim15": [0.35, 0.00, 0.90, 0.75], + "kodim16": [0.40, 0.35, 0.90, 0.60], + "kodim17": [0.25, 0.00, 0.75, 0.45], + "kodim18": [0.25, 0.15, 0.85, 1.00], + "kodim19": [0.35, 0.00, 0.85, 0.75], + "kodim21": [0.30, 0.05, 0.70, 0.65], + "kodim22": [0.05, 0.10, 0.60, 0.70], + "kodim23": [0.50, 0.00, 1.00, 0.75] +} diff --git a/untranslated.json b/untranslated.json index 9e26dfee..3119859d 100644 --- a/untranslated.json +++ b/untranslated.json @@ -1 +1,191 @@ -{} \ No newline at end of file +{ + "bg": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "cs": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "de": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "es": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "fr": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "hu": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "it": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "ja": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "ko": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "nl": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "pl": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "pt": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "pt_PT": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "ru": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "sk": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "sl": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "sv": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "uk": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ], + + "zh": [ + "receivedImage_packetEchoes", + "receivedImage_recoveryPacketEchoes", + "imageSend_preparingPreview", + "imageSend_tryAnother", + "imageSend_versionsHint", + "imageSend_versionLabel", + "imageSend_versionPackets" + ] +}