From 2d56d5e349d8665602887f1780d9a5e169c419b2 Mon Sep 17 00:00:00 2001 From: zjs81 Date: Mon, 9 Nov 2026 22:32:40 -0700 Subject: [PATCH] Store histories and the tile index in SQLite Message, channel and route histories move from whole-blob preferences to indexed SQLite rows on native platforms, so chats page from storage instead of loading a full conversation. Web keeps the preferences store. Existing histories are imported once on the first launch. Linux and Windows keep the map tile index in SQLite and import the old JSON index on first use. Unreadable legacy data no longer blocks saves, migrations or startup: it is left in place and skipped. If the database cannot be opened, the run keeps histories in preferences and the next launch merges them in. Also adds a performance profiling reference and a script that clears the Dart HTTP profile. --- CLAUDE.md | 4 +- documentation/README.md | 1 + documentation/map-and-location.md | 9 + documentation/performance-profiling.md | 59 ++ lib/connector/meshcore_connector.dart | 442 ++++++++++----- lib/main.dart | 2 + lib/review_mode/review_mode_storage.dart | 12 +- lib/screens/map_screen.dart | 82 ++- lib/services/backup_service.dart | 91 ++-- lib/services/map_tile_cache_repository.dart | 2 + .../map_tile_cache_repository_io.dart | 312 +++++++++++ .../map_tile_cache_repository_stub.dart | 4 + lib/services/map_tile_cache_service.dart | 26 +- lib/services/path_history_service.dart | 163 ++++-- lib/services/storage_service.dart | 31 +- lib/storage/channel_message_store.dart | 201 +++---- lib/storage/channel_storage_migration.dart | 89 ++- lib/storage/channel_store.dart | 14 +- lib/storage/community_store.dart | 11 +- lib/storage/contact_discovery_store.dart | 6 +- lib/storage/contact_group_store.dart | 11 +- lib/storage/contact_settings_store.dart | 7 +- lib/storage/contact_store.dart | 11 +- lib/storage/history_archive.dart | 192 +++++++ lib/storage/history_store.dart | 149 +++++ lib/storage/history_store_io.dart | 356 ++++++++++++ lib/storage/history_store_stub.dart | 3 + lib/storage/legacy_key_migration.dart | 15 + lib/storage/message_store.dart | 259 +++++---- lib/storage/prefs_manager.dart | 3 + lib/storage/unread_store.dart | 11 +- lib/utils/coalesced_writes.dart | 9 +- pubspec.yaml | 3 + scripts/reset_http_profile.dart | 99 ++++ test/connector/history_pagination_test.dart | 372 +++++++++++++ test/connector/remove_contact_test.dart | 2 + test/screens/map_camera_persistence_test.dart | 160 ++++++ test/screens/map_rebuild_scope_test.dart | 188 +++++++ test/scripts/reset_http_profile_test.dart | 78 +++ .../map_tile_cache_repository_test.dart | 264 +++++++++ .../path_history_async_storage_test.dart | 230 ++++++++ test/storage/channel_storage_by_id_test.dart | 3 +- test/storage/history_store_test.dart | 511 ++++++++++++++++++ .../storage/legacy_migration_writes_test.dart | 218 ++++++++ test/storage/message_archive_memory_test.dart | 285 ++++++++++ test/storage/message_store_archive_test.dart | 37 +- test/utils/coalesced_writes_test.dart | 42 ++ 47 files changed, 4465 insertions(+), 614 deletions(-) create mode 100644 documentation/performance-profiling.md create mode 100644 lib/services/map_tile_cache_repository.dart create mode 100644 lib/services/map_tile_cache_repository_io.dart create mode 100644 lib/services/map_tile_cache_repository_stub.dart create mode 100644 lib/storage/history_archive.dart create mode 100644 lib/storage/history_store.dart create mode 100644 lib/storage/history_store_io.dart create mode 100644 lib/storage/history_store_stub.dart create mode 100644 scripts/reset_http_profile.dart create mode 100644 test/connector/history_pagination_test.dart create mode 100644 test/screens/map_camera_persistence_test.dart create mode 100644 test/screens/map_rebuild_scope_test.dart create mode 100644 test/scripts/reset_http_profile_test.dart create mode 100644 test/services/map_tile_cache_repository_test.dart create mode 100644 test/services/path_history_async_storage_test.dart create mode 100644 test/storage/history_store_test.dart create mode 100644 test/storage/legacy_migration_writes_test.dart create mode 100644 test/storage/message_archive_memory_test.dart diff --git a/CLAUDE.md b/CLAUDE.md index 5e096068..ef21d2be 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -39,7 +39,7 @@ For Linux, use `nix develop`; see [Linux with Nix](README.md#linux-with-nix) for | Serial marker/length framing | `lib/services/usb_serial_frame_codec.dart` | | ACK tracking and retry scheduling | `lib/services/message_retry_service.dart` | | Route scoring/history | `lib/services/path_history_service.dart` | -| Preferences and JSON stores | `lib/storage/` | +| Preferences, native history database, and JSON stores | `lib/storage/` | | Region list and channel/default assignments | `lib/storage/region_store.dart`, `lib/storage/channel_region_store.dart` | | Message filter (rules, duplicate/spam checks, on-device classifier) | `lib/services/message_filter/`, `lib/models/message_filter.dart`, `lib/storage/message_filter_store.dart`, `lib/screens/message_filter_settings_screen.dart`, `lib/screens/filtered_messages_screen.dart` | | Translation models/inference | `lib/services/translation_service.dart` | @@ -51,6 +51,8 @@ For Linux, use `nix develop`; see [Linux with Nix](README.md#linux-with-nix) for `Provider`/`ChangeNotifier` drives UI state. `main.dart` wires `MeshTheme.light()` and `MeshTheme.dark()` and uses the saved theme preference. Its `MultiProvider` exposes `MeshCoreConnector`, `MessageRetryService`, `PathHistoryService`, `AppSettingsService`, `BleDebugLogService`, `AppDebugLogService`, `ChatTextScaleService`, `TranslationService`, `UiViewStateService`, `StorageService` (plain `Provider`), `MapTileCacheService`, `TimeoutPredictionService`, `ImageCodecService`, `ReceivedImageStore`, and `MessageFilterService`; screens read them with `Consumer`, `context.watch()`, or `context.read()`. +Message/channel histories use indexed SQLite rows on native platforms, and route histories use the same database. Web builds use preferences. Startup imports old history preferences once; backup and review-mode cleanup include database histories. See [local history storage](documentation/performance-profiling.md#local-history-storage). + Most JSON stores scope keys by the first 10 hex chars of the connected device's public key, so per-radio data stays isolated; some preferences, including the saved region-name list, are global. Model bundles, received images, and caches use files. ## Localization diff --git a/documentation/README.md b/documentation/README.md index 267f6427..64533529 100644 --- a/documentation/README.md +++ b/documentation/README.md @@ -33,6 +33,7 @@ MeshCore Open is an open-source Flutter client for MeshCore LoRa mesh networking 20. [Message Filter](message-filter.md) - Hide spam and repeated channel messages with your own rules, automatic checks, and learning from Spam taps, all on the device 21. [UI Design Reference](design.md) - Contributor reference: theme, typography, spacing, shared widgets, sheets vs dialogs vs pages, states, and localization rules for UI changes 22. [Offline Mode](offline-mode.md) - Read-only view of the last radio's contacts, channels and messages while no radio is connected, and the Connect sheet +23. [Performance Profiling](performance-profiling.md) - Contributor reference: frame and memory baselines, HTTP capture retention, and resetting diagnostic recordings ## App Overview diff --git a/documentation/map-and-location.md b/documentation/map-and-location.md index 984246d1..de516375 100644 --- a/documentation/map-and-location.md +++ b/documentation/map-and-location.md @@ -184,6 +184,15 @@ A full-screen map with a draggable bottom sheet containing: ## Offline Map Cache +Linux and Windows keep the tile index in SQLite, so opening a large offline map +does not load every tile's metadata into memory. Existing JSON indexes migrate +automatically on first use; downloaded tile files and the cache limit are +preserved. The old index is removed only after its replacement is committed. +Unreadable records are skipped, and an index that contained any is kept beside +the database with an `.unreadable` suffix instead of being deleted. +The database and import marker are versioned, so a completed migration is not +repeated and an older app does not downgrade a newer database. + ### How to Access Settings → App Settings → Map Display → Offline Map Cache diff --git a/documentation/performance-profiling.md b/documentation/performance-profiling.md new file mode 100644 index 00000000..48657397 --- /dev/null +++ b/documentation/performance-profiling.md @@ -0,0 +1,59 @@ +# Performance and memory profiling + +Use `flutter run --profile` to measure frame times and memory after repeating +the same actions. Debug builds also retain Dart source, JIT code and debugger +objects, so their process memory is not a release-build baseline. + +## HTTP recording retains response bodies + +Dart's HTTP timeline logging defaults to off. The app does not enable it. +Developer tools can enable it through the VM service, independently of the +app's debug-log settings. When enabled, Dart retains HTTP request records and +response bodies until those captures are cleared. Map browsing and downloads +can therefore increase profiling memory even after the app stops displaying +the downloaded images. Stopping recording does not clear earlier captures. + +For a memory baseline, stop HTTP recording and clear its captures in DevTools' +Network view. Close or disconnect tools that turn recording back on. If the +Network controls are unavailable, run this command from the repository root +with the Dart VM service URI printed by `flutter run`: + +```sh +dart run scripts/reset_http_profile.dart http://127.0.0.1:PORT/TOKEN=/ +``` + +This explicitly stops HTTP recording and clears the retained Dart HTTP profile +in the running app's main isolate. Save any captures you need first. It does +not delete downloaded map tiles, change app settings, stop network requests or +change CPU/timeline profiling. You can enable HTTP recording again in DevTools +when investigating network requests. Other HTTP profiling providers may keep +their own records; the command reports any requests that remain. + +After clearing, use DevTools' memory view to collect garbage and compare live +heap snapshots at the same point in the workflow. Process RSS can remain high +because the runtime and native libraries can keep reusable allocations; a +larger RSS alone does not establish a leak. Keep frame measurements collected +before heap inspection separate from collection pauses caused by diagnostics. + +## Local history storage + +Native apps store direct messages, channel messages, and route histories in +`history_v1.db` in the application support directory. Small preferences remain +in SharedPreferences. Web builds continue to use browser preferences. + +The first launch after upgrading imports existing histories. The database has a +schema version and a separate import version, so a completed import is skipped +on later launches. Source preferences are removed only after their database +writes commit; an interrupted import retries safely. Unreadable history values +remain in preferences for recovery instead of being discarded; new messages for +that conversation are saved to the database beside them. A failed import step +does not stop the app: the database is used and the step repeats on the next +launch. If the database cannot be opened at all, that run keeps histories in +preferences, and the next launch that opens the database merges them in. + +Chat history is indexed by entry, allowing older pages to be read without loading +the whole conversation. Backups include native database histories and restore +them into the selected radio's scope. Review-mode cleanup also covers the database. +For a manual full application-data copy, keep the history database alongside the +application's other support files; copying the preferences file alone no longer +includes native message or route histories. diff --git a/lib/connector/meshcore_connector.dart b/lib/connector/meshcore_connector.dart index 138aa960..0c1e668f 100644 --- a/lib/connector/meshcore_connector.dart +++ b/lib/connector/meshcore_connector.dart @@ -255,6 +255,9 @@ class MeshCoreConnector extends ChangeNotifier { static const int _errCodeNotFound = 2; int _reactionSendQueueSequence = 0; final Set _loadedConversationKeys = {}; + int _messageHistoryGeneration = 0; + final Map>> _conversationPageLoads = {}; + final Map>> _channelPageLoads = {}; // Self public key of the radio whose messages are held in memory. String? _messageStateOwnerKey; // Self public key whose persisted stores were loaded this link session. @@ -985,63 +988,82 @@ class MeshCoreConnector extends ChangeNotifier { Future _loadMessagesForContact(String contactKeyHex) async { if (_loadedConversationKeys.contains(contactKeyHex)) return; _loadedConversationKeys.add(contactKeyHex); - - final allMessages = await _messageStore.loadMessages(contactKeyHex); - if (allMessages.isNotEmpty) { - // Keep only the most recent N messages in memory to bound memory usage - final windowedMessages = allMessages.length > _messageWindowSize - ? allMessages.sublist(allMessages.length - _messageWindowSize) - : allMessages; - final archivedMessages = allMessages.sublist( - 0, - allMessages.length - windowedMessages.length, + final generation = _messageHistoryGeneration; + final scope = _messageStore.publicKeyHex; + bool current() => + !_disposed && + generation == _messageHistoryGeneration && + scope == _messageStore.publicKeyHex; + final page = await _readSettledPage( + current: current, + window: () => _conversations[contactKeyHex] ?? const [], + archived: () => _messageStore.archivedCount(contactKeyHex), + read: (_, _) => _messageStore.loadMessagePage( + contactKeyHex, + limit: _messageWindowSize, + ), + ); + if (page == null) return; + final currentMessages = _conversations[contactKeyHex] ?? const []; + final mergedMessages = [...page.messages]; + final persistedKeyCounts = {}; + for (final message in page.messages) { + final key = _messageMergeKey(message); + persistedKeyCounts[key] = (persistedKeyCounts[key] ?? 0) + 1; + } + final currentKeyCounts = {}; + for (final message in currentMessages) { + final key = _messageMergeKey(message); + final count = (currentKeyCounts[key] ?? 0) + 1; + currentKeyCounts[key] = count; + if (count > (persistedKeyCounts[key] ?? 0)) mergedMessages.add(message); + } + mergedMessages.sort((a, b) => a.receivedAt.compareTo(b.receivedAt)); + final trimCount = (mergedMessages.length - _messageWindowSize).clamp( + 0, + mergedMessages.length, + ); + _messageStore.setArchivedCount(contactKeyHex, page.start); + if (trimCount > 0) { + _messageStore.archive( + contactKeyHex, + mergedMessages.sublist(0, trimCount), ); - - final currentMessages = - _conversations[contactKeyHex] ?? const []; - final mergedMessages = [...windowedMessages]; - final persistedKeyCounts = {}; - for (final message in windowedMessages) { - final key = _messageMergeKey(message); - persistedKeyCounts[key] = (persistedKeyCounts[key] ?? 0) + 1; - } - final currentKeyCounts = {}; - - for (final message in currentMessages) { - final key = _messageMergeKey(message); - final currentCount = (currentKeyCounts[key] ?? 0) + 1; - currentKeyCounts[key] = currentCount; - final persistedCount = persistedKeyCounts[key] ?? 0; - - // Preserve distinct duplicates without IDs (for example same text - // received multiple times in the same second) by only skipping the - // overlapping occurrences that already exist in persisted storage. - if (currentCount > persistedCount) { - mergedMessages.add(message); - } - } - - // Re-sort after merging persisted and in-memory messages so the - // conversation window remains stable after optimistic inserts. - mergedMessages.sort((a, b) => a.receivedAt.compareTo(b.receivedAt)); - final trimCount = mergedMessages.length > _messageWindowSize - ? mergedMessages.length - _messageWindowSize - : 0; - final windowedMergedMessages = mergedMessages.sublist(trimCount); - - // Everything stored ahead of the window stays in storage on the next - // save instead of being overwritten by the window. - _messageStore.setArchived(contactKeyHex, [ - ...archivedMessages, - ...mergedMessages.sublist(0, trimCount), - ]); - _conversations[contactKeyHex] = windowedMergedMessages; + } + if (mergedMessages.isNotEmpty) { + _conversations[contactKeyHex] = mergedMessages.sublist(trimCount); notifyListeners(); - } else { - _messageStore.setArchived(contactKeyHex, const []); } } + /// Reads a stored page once queued writes have landed, and again whenever + /// the in-memory [window] or the [archived] count moved during the read. + /// Returns null when [current] turns false, or when [older] history is + /// requested and none is archived. + Future _readSettledPage({ + required bool Function() current, + required List Function() window, + required int? Function() archived, + required Future

Function(int? archived, List window) read, + bool older = false, + }) async { + while (true) { + await _coalescedWrites.flush(); + if (!current()) return null; + final archivedBefore = archived(); + if (older && archivedBefore == 0) return null; + final snapshot = List.of(window()); + final page = await read(archivedBefore, snapshot); + if (!current()) return null; + if (listEquals(snapshot, window()) && archivedBefore == archived()) { + return page; + } + } + } + + static int _startOfWindow(int stored, List window) => + (stored - window.length).clamp(0, stored); + String _messageMergeKey(Message message) { final messageId = message.messageId; if (messageId.isNotEmpty) { @@ -1054,58 +1076,126 @@ class MeshCoreConnector extends ChangeNotifier { Future> loadOlderMessages( String contactKeyHex, { int count = 50, - }) async { - final allMessages = await _messageStore.loadMessages(contactKeyHex); - final currentMessages = _conversations[contactKeyHex] ?? []; + }) { + if (count <= 0) return Future.value([]); + final loadKey = '${_messageStore.publicKeyHex}:$contactKeyHex'; + final pending = _conversationPageLoads[loadKey]; + if (pending != null) return pending; + final started = _loadOlderMessages(contactKeyHex, count); + _conversationPageLoads[loadKey] = started; + return started.whenComplete(() { + if (identical(_conversationPageLoads[loadKey], started)) { + _conversationPageLoads.remove(loadKey); + } + }); + } - // Storage holds the archived entries followed by the in-memory window. - final currentOffset = - (_messageStore.archivedCount(contactKeyHex) ?? - allMessages.length - currentMessages.length) - .clamp(0, allMessages.length); - if (currentOffset == 0) { - return []; // No more messages to load - } - - final fetchCount = count.clamp(0, currentOffset); - final startIndex = currentOffset - fetchCount; - - final olderMessages = allMessages.sublist(startIndex, currentOffset); - _messageStore.unarchiveNewest(contactKeyHex, olderMessages.length); - - // Prepend to current conversation - _conversations[contactKeyHex] = [...olderMessages, ...currentMessages]; + Future> _loadOlderMessages( + String contactKeyHex, + int count, + ) async { + final generation = _messageHistoryGeneration; + final scope = _messageStore.publicKeyHex; + bool current() => + !_disposed && + generation == _messageHistoryGeneration && + scope == _messageStore.publicKeyHex; + final page = await _readSettledPage( + current: current, + window: () => _conversations[contactKeyHex] ?? const [], + archived: () => _messageStore.archivedCount(contactKeyHex), + older: true, + read: (archived, window) async => _messageStore.loadMessagePage( + contactKeyHex, + // Unreconciled storage is assumed to end with the loaded window. + before: + archived ?? + _startOfWindow( + (await _messageStore.loadMessagePage( + contactKeyHex, + limit: 0, + )).total, + window, + ), + limit: count, + ), + ); + if (page == null) return []; + _messageStore.setArchivedCount(contactKeyHex, page.start); + if (page.messages.isEmpty) return []; + final currentMessages = _conversations[contactKeyHex] ?? const []; + _conversations[contactKeyHex] = [...page.messages, ...currentMessages]; notifyListeners(); - - return olderMessages; + return page.messages; } /// Full stored history (archived prefix plus the in-memory window) for /// search. Read-only: nothing is unarchived and listeners are not notified. Future> allMessagesForSearch(String contactKeyHex) async { - final stored = await _messageStore.loadMessages(contactKeyHex); + final generation = _messageHistoryGeneration; + final scope = _messageStore.publicKeyHex; + await _coalescedWrites.flush(); + if (_disposed || + generation != _messageHistoryGeneration || + scope != _messageStore.publicKeyHex) { + return []; + } + final archived = _messageStore.archivedCount(contactKeyHex); + final stored = await _messageStore.loadMessages( + contactKeyHex, + limit: archived, + ); + if (_disposed || + generation != _messageHistoryGeneration || + scope != _messageStore.publicKeyHex) { + return []; + } final window = _conversations[contactKeyHex] ?? const []; - final archived = - (_messageStore.archivedCount(contactKeyHex) ?? - stored.length - window.length) - .clamp(0, stored.length); - return [...stored.sublist(0, archived), ...window]; + final prefix = archived == null + ? stored.sublist( + 0, + (stored.length - window.length).clamp(0, stored.length), + ) + : stored; + return [...prefix, ...window]; } /// Channel counterpart of [allMessagesForSearch]; read-only. Future> allChannelMessagesForSearch( int channelIndex, ) async { + final generation = _messageHistoryGeneration; + final scope = _channelMessageStore.publicKeyHex; final slot = _canonicalSlot(channelIndex); - final window = _channelMessages[slot] ?? const []; final channelId = _channelIdForSlot(slot); - if (channelId == null) return List.of(window); - final stored = await _channelMessageStore.loadChannelMessages(channelId); - final archived = - (_channelMessageStore.archivedCount(channelId) ?? - stored.length - window.length) - .clamp(0, stored.length); - return [..._bindToSlot(stored.sublist(0, archived), slot), ...window]; + if (channelId == null) return List.of(_channelMessages[slot] ?? []); + await _coalescedWrites.flush(); + if (_disposed || + generation != _messageHistoryGeneration || + scope != _channelMessageStore.publicKeyHex || + _slotChannelIds[slot] != channelId) { + return []; + } + final archived = _channelMessageStore.archivedCount(channelId); + final stored = await _channelMessageStore.loadChannelMessages( + channelId, + limit: archived, + ); + if (_disposed || + generation != _messageHistoryGeneration || + scope != _channelMessageStore.publicKeyHex || + _slotChannelIds[slot] != channelId) { + return []; + } + final window = _channelMessages[slot] ?? const []; + final rebound = _bindToSlot(stored, slot); + final prefix = archived == null + ? rebound.sublist( + 0, + (rebound.length - window.length).clamp(0, rebound.length), + ) + : rebound; + return [...prefix, ...window]; } List getChannelMessages(Channel channel) { @@ -1492,7 +1582,14 @@ class MeshCoreConnector extends ChangeNotifier { /// Runs queued persistence now. Called before the app is backgrounded and /// whenever the stores are about to switch to another radio. - Future flushPendingWrites() => _coalescedWrites.flush(); + Future flushPendingWrites() async { + await Future.wait([ + _coalescedWrites.flush(), + _unreadStore.flush(), + if (_pathHistoryService != null) + _pathHistoryService!.flushPendingWrites(), + ]); + } int getUnreadCountForContact(Contact contact) { if (contact.type == advTypeRepeater) return 0; @@ -2045,77 +2142,103 @@ class MeshCoreConnector extends ChangeNotifier { : message.copyWith(channelIndex: slot), ]; - /// Load persisted channel messages for a specific channel + /// Load only the newest persisted window of a channel. Future _loadChannelMessages(int channelIndex) async { - // The load replaces the in-memory history, so queued writes of it must - // reach storage (whose cache updates synchronously) first. - unawaited(_coalescedWrites.flush()); + final generation = _messageHistoryGeneration; + final scope = _channelMessageStore.publicKeyHex; final channelId = _slotChannelIds[channelIndex]; - final allMessages = channelId == null - ? [] - : _bindToSlot( - await _channelMessageStore.loadChannelMessages(channelId), - channelIndex, - ); - // Keep only the most recent N messages in memory to bound memory usage - final windowedMessages = allMessages.length > _messageWindowSize - ? allMessages.sublist(allMessages.length - _messageWindowSize) - : allMessages; - if (channelId != null) { - _channelMessageStore.setArchived( - channelId, - allMessages.sublist(0, allMessages.length - windowedMessages.length), + bool current() => + !_disposed && + generation == _messageHistoryGeneration && + scope == _channelMessageStore.publicKeyHex && + _slotChannelIds[channelIndex] == channelId; + var windowedMessages = []; + if (channelId == null) { + await _coalescedWrites.flush(); + if (!current()) return; + } else { + final page = await _readSettledPage( + current: current, + window: () => + _channelMessages[channelIndex] ?? const [], + archived: () => _channelMessageStore.archivedCount(channelId), + read: (_, _) => _channelMessageStore.loadMessagePage( + channelId, + limit: _messageWindowSize, + ), ); + if (page == null) return; + windowedMessages = _bindToSlot(page.messages, channelIndex); + _channelMessageStore.setArchivedCount(channelId, page.start); } - // Replace even when nothing is stored so another radio's messages never - // linger under this index. final hadMessages = _channelMessages[channelIndex]?.isNotEmpty ?? false; - // Rules may have changed since these were stored. - if (_applyFilterRules(channelIndex, windowedMessages)) { - _scheduleChannelSave(channelIndex); - } + final retagged = _applyFilterRules(channelIndex, windowedMessages); _channelMessages[channelIndex] = windowedMessages; + if (retagged) _scheduleChannelSave(channelIndex); if (hadMessages || windowedMessages.isNotEmpty) notifyListeners(); } - /// Load older channel messages (pagination) Future> loadOlderChannelMessages( int channelIndex, { int count = 50, - }) async { + }) { + if (count <= 0) return Future.value([]); final slot = _canonicalSlot(channelIndex); final channelId = _channelIdForSlot(slot); - if (channelId == null) return []; - final allMessages = await _channelMessageStore.loadChannelMessages( - channelId, + if (channelId == null) return Future.value([]); + final loadKey = '${_channelMessageStore.publicKeyHex}:$channelId'; + final pending = _channelPageLoads[loadKey]; + if (pending != null) return pending; + final started = _loadOlderChannelMessages(slot, channelId, count); + _channelPageLoads[loadKey] = started; + return started.whenComplete(() { + if (identical(_channelPageLoads[loadKey], started)) { + _channelPageLoads.remove(loadKey); + } + }); + } + + Future> _loadOlderChannelMessages( + int slot, + String channelId, + int count, + ) async { + final generation = _messageHistoryGeneration; + final scope = _channelMessageStore.publicKeyHex; + bool current() => + !_disposed && + generation == _messageHistoryGeneration && + scope == _channelMessageStore.publicKeyHex && + _slotChannelIds[slot] == channelId; + final page = await _readSettledPage( + current: current, + window: () => _channelMessages[slot] ?? const [], + archived: () => _channelMessageStore.archivedCount(channelId), + older: true, + read: (archived, window) async => _channelMessageStore.loadMessagePage( + channelId, + // Unreconciled storage is assumed to end with the loaded window. + before: + archived ?? + _startOfWindow( + (await _channelMessageStore.loadMessagePage( + channelId, + limit: 0, + )).total, + window, + ), + limit: count, + ), ); - final currentMessages = _channelMessages[slot] ?? []; - - // Storage holds the archived entries followed by the in-memory window. - final currentOffset = - (_channelMessageStore.archivedCount(channelId) ?? - allMessages.length - currentMessages.length) - .clamp(0, allMessages.length); - if (currentOffset == 0) { - return []; // No more messages to load - } - - final fetchCount = count.clamp(0, currentOffset); - final startIndex = currentOffset - fetchCount; - - final olderMessages = _bindToSlot( - allMessages.sublist(startIndex, currentOffset), - slot, - ); - _channelMessageStore.unarchiveNewest(channelId, olderMessages.length); - - // Rules may have changed since these were stored. + if (page == null) return []; + _channelMessageStore.setArchivedCount(channelId, page.start); + final olderMessages = _bindToSlot(page.messages, slot); + if (olderMessages.isEmpty) return []; final retagged = _applyFilterRules(slot, olderMessages); - // Prepend to current conversation + final currentMessages = _channelMessages[slot] ?? const []; _channelMessages[slot] = [...olderMessages, ...currentMessages]; if (retagged) _scheduleChannelSave(slot); notifyListeners(); - return olderMessages; } @@ -4305,6 +4428,7 @@ class MeshCoreConnector extends ChangeNotifier { _txCharacteristic = null; _deviceDisplayName = null; _deviceId = null; + await _coalescedWrites.flush(); _contacts.clear(); _discoveredContacts.clear(); _clearSessionMessageState(); @@ -6333,7 +6457,8 @@ class MeshCoreConnector extends ChangeNotifier { } // Queued channel writes find their channel by slot; land them before // the map moves, or a channel whose slot changed would never be saved. - unawaited(_coalescedWrites.flush()); + await _coalescedWrites.flush(); + if (generation != _channelSessionGeneration) return; _slotChannelIds ..clear() ..addAll(next); @@ -6455,18 +6580,32 @@ class MeshCoreConnector extends ChangeNotifier { /// Runs the one-shot storage migration, then preloads this radio's channel /// state from the saved slot map so it shows before the channel sync ends. Future _prepareChannelStorage() async { + final generation = _channelSessionGeneration; + final scope = _channelStore.publicKeyHex; + bool current() => + !_disposed && + generation == _channelSessionGeneration && + scope == _channelStore.publicKeyHex; try { // First: the channel sync is already running and carries unread counts // over from these. The migration never touches the channel cache. await loadCachedChannels(); - await ChannelStorageMigration.migrate( - _channelStore.publicKeyHex, - beforeDone: (slotIds) async => - _receivedImageStore?.stampLegacyChannelIds(slotIds), - ); + try { + await ChannelStorageMigration.migrate( + scope, + beforeDone: (slotIds) async => + _receivedImageStore?.stampLegacyChannelIds(slotIds), + ); + } catch (e) { + // It repeats on the next connect; what is already stored under + // channel IDs still preloads. + appLogger.warn('Channel storage migration failed: $e'); + } await _loadChannelOrder(); final saved = await _channelStore.loadSlotChannelIds(); - unawaited(_coalescedWrites.flush()); + await _coalescedWrites.flush(); + // The radio may have disconnected or changed while writes landed. + if (!current()) return; _slotChannelIds ..clear() ..addAll(saved); @@ -6541,7 +6680,7 @@ class MeshCoreConnector extends ChangeNotifier { await sendFrame(buildSetChannelFrame(index, '', Uint8List(16))); // A twin slot with the same PSK keeps this history, so queued writes of // it land before the list is dropped; a full delete clears them below. - unawaited(_coalescedWrites.flush()); + await _coalescedWrites.flush(); _channelMessages.remove(index); _processedChannelReactions.remove(index); try { @@ -9441,6 +9580,9 @@ class MeshCoreConnector extends ChangeNotifier { /// Drops the conversation and channel history held in memory, so a /// different radio never shows or persists another radio's messages. void _clearSessionMessageState() { + _messageHistoryGeneration++; + _conversationPageLoads.clear(); + _channelPageLoads.clear(); // Writes start synchronously, so they still read this radio's history and // store keys before both change below. unawaited(_coalescedWrites.flush()); @@ -9536,6 +9678,8 @@ class MeshCoreConnector extends ChangeNotifier { generation != _offlineLoadGeneration || _state != MeshCoreConnectionState.disconnected; + await _coalescedWrites.flush(); + if (superseded()) return; _clearSessionMessageState(); _bindStorageScope(keyHex); // A connect to this radio keeps what is loaded here; a different radio's diff --git a/lib/main.dart b/lib/main.dart index 959fc3c4..751e79d2 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -41,6 +41,7 @@ import 'services/ui_view_state_service.dart'; import 'services/chat_background_service.dart'; import 'services/timeout_prediction_service.dart'; import 'storage/prefs_manager.dart'; +import 'storage/history_store.dart'; import 'theme/mesh_theme.dart'; import 'utils/app_logger.dart'; import 'widgets/image_send_codec_binding.dart'; @@ -68,6 +69,7 @@ void main() async { // Initialize SharedPreferences cache await PrefsManager.initialize(); + await HistoryStore.initialize(); // Initialize services final storage = StorageService(); diff --git a/lib/review_mode/review_mode_storage.dart b/lib/review_mode/review_mode_storage.dart index de787edb..20f7840e 100644 --- a/lib/review_mode/review_mode_storage.dart +++ b/lib/review_mode/review_mode_storage.dart @@ -6,6 +6,7 @@ import '../services/storage_service.dart'; import '../services/timeout_prediction_service.dart'; import '../storage/legacy_key_migration.dart'; import '../storage/prefs_manager.dart'; +import '../storage/history_store.dart'; import 'review_radio.dart'; /// Channel notification modes (JSON map of channel name to mode name) as they @@ -26,9 +27,16 @@ const _savedGlobalKeyPrefix = 'review_mode_saved_global_'; Future clearReviewModeData() async { final prefs = PrefsManager.instance; final prefix = ReviewRadio.storagePrefix; - final keys = prefs.getKeys().where((key) => key.contains(prefix)).toList(); + final keys = { + ...prefs.getKeys(), + ...await HistoryStore.instance.getKeys(), + }.where((key) => key.contains(prefix)).toList(); for (final key in keys) { - await prefs.remove(key); + if (HistoryStore.isHistoryKey(key)) { + await HistoryStore.instance.remove(key); + } else { + await prefs.remove(key); + } } } diff --git a/lib/screens/map_screen.dart b/lib/screens/map_screen.dart index d948736a..018b7ec8 100644 --- a/lib/screens/map_screen.dart +++ b/lib/screens/map_screen.dart @@ -75,7 +75,7 @@ class MapScreen extends StatefulWidget { State createState() => _MapScreenState(); } -class _MapScreenState extends State { +class _MapScreenState extends State with WidgetsBindingObserver { // Zoom level at which node labels start to appear static const double _labelZoomThreshold = 14.0; // Below this zoom, nearby nodes collapse into clusters. @@ -135,6 +135,9 @@ class _MapScreenState extends State { ({LatLng center, double zoom})? _initialCamera; static const String _lastCameraKey = 'map_last_camera'; + static const Duration _cameraSaveDelay = Duration(milliseconds: 500); + Timer? _cameraSaveTimer; + List? _pendingCamera; ({LatLng center, double zoom}) _fallbackCamera(MeshCoreConnector connector) { final lat = connector.selfLatitude; @@ -152,23 +155,58 @@ class _MapScreenState extends State { return (center: const LatLng(20, 0), zoom: _mapMinZoom); } - void _saveLastCamera() { + void _scheduleCameraSave() { try { final camera = _mapController.camera; - PrefsManager.instance.setStringList(_lastCameraKey, [ + final value = [ camera.center.latitude.toString(), camera.center.longitude.toString(), camera.zoom.toString(), - ]); + ]; + _cameraSaveTimer?.cancel(); + _cameraSaveTimer = null; + _pendingCamera = + listEquals(value, PrefsManager.instance.getStringList(_lastCameraKey)) + ? null + : value; + if (_pendingCamera != null) { + _cameraSaveTimer = Timer(_cameraSaveDelay, _flushCameraSave); + } } catch (_) {} } + void _flushCameraSave() { + _cameraSaveTimer?.cancel(); + _cameraSaveTimer = null; + final value = _pendingCamera; + _pendingCamera = null; + if (value == null) return; + final prefs = PrefsManager.instance; + if (listEquals(value, prefs.getStringList(_lastCameraKey))) return; + unawaited( + prefs.setStringList(_lastCameraKey, value).catchError((Object error) { + debugPrint('Saving map camera failed: $error'); + return false; + }), + ); + } + + @override + void didChangeAppLifecycleState(AppLifecycleState state) { + if (state != AppLifecycleState.resumed) { + _scheduleCameraSave(); + _flushCameraSave(); + } + } + @override void dispose() { + WidgetsBinding.instance.removeObserver(this); _ageTimer?.cancel(); _online.removeListener(_onOnlineChanged); _online.dispose(); - _saveLastCamera(); + _scheduleCameraSave(); + _flushCameraSave(); _searchController.dispose(); _searchFocus.dispose(); _mapController.dispose(); @@ -278,6 +316,7 @@ class _MapScreenState extends State { @override void initState() { super.initState(); + WidgetsBinding.instance.addObserver(this); _loadRemovedMarkers(); _online.addListener(_onOnlineChanged); _ageTimer = Timer.periodic(const Duration(seconds: 60), (_) { @@ -531,7 +570,7 @@ class _MapScreenState extends State { final tileCache = context.read(); final scheme = Theme.of(context).colorScheme; final isDesktop = _isDesktopPlatform(defaultTargetPlatform); - final allContacts = connector.allContacts; + final allContacts = connectorSnapshot.allContacts; final contacts = settings.mapShowDiscoveryContacts ? allContacts @@ -912,11 +951,12 @@ class _MapScreenState extends State { event is MapEventFlingAnimationEnd || event is MapEventScrollWheelZoom || event is MapEventDoubleTapZoomEnd) { - _saveLastCamera(); + _scheduleCameraSave(); if (showOnline) _requestOnlineData(); } }, onPositionChanged: (camera, hasGesture) { + _scheduleCameraSave(); // Track zoom in half-step buckets so cluster/marker // detail levels update without rebuilding every frame. final bucket = (camera.zoom * 2).roundToDouble() / 2; @@ -1087,13 +1127,19 @@ class _MapScreenState extends State { ), bottomNavigationBar: SafeArea( top: false, - child: QuickSwitchBar( - selectedIndex: 2, - onDestinationSelected: (index) => - _handleQuickSwitch(index, context), - contactsUnreadCount: connector.getTotalContactsUnreadCount(), - channelsUnreadCount: connector.getTotalChannelsUnreadCount(), - highContrast: _usePaletteOverlay, + child: Selector( + selector: (_, connector) => ( + connector.getTotalContactsUnreadCount(), + connector.getTotalChannelsUnreadCount(), + ), + builder: (context, unread, child) => QuickSwitchBar( + selectedIndex: 2, + onDestinationSelected: (index) => + _handleQuickSwitch(index, context), + contactsUnreadCount: unread.$1, + channelsUnreadCount: unread.$2, + highContrast: _usePaletteOverlay, + ), ), ), floatingActionButton: @@ -1130,8 +1176,7 @@ class _MapScreenState extends State { }) { final visibleContactsSignature = Object.hashAll( contacts.map( - (contact) => - Object.hash(_mapContactSignature(contact), _ageOf(contact)), + (contact) => Object.hash(contact.publicKeyHex, _ageOf(contact)), ), ); final key = _NodeMarkersCacheKey( @@ -4226,6 +4271,7 @@ int _mapContactSignature(Contact contact) { class _MapConnectorSnapshot { final MeshCoreConnector connector; + final List allContacts; final int contactsSignature; final int markerSignature; final int batterySignature; @@ -4233,6 +4279,7 @@ class _MapConnectorSnapshot { const _MapConnectorSnapshot({ required this.connector, + required this.allContacts, required this.contactsSignature, required this.markerSignature, required this.batterySignature, @@ -4285,6 +4332,7 @@ class _MapConnectorSnapshot { return _MapConnectorSnapshot( connector: connector, + allContacts: allContacts, contactsSignature: contactsSignature, markerSignature: Object.hashAll(markerParts), batterySignature: batterySignature, @@ -4296,8 +4344,6 @@ class _MapConnectorSnapshot { connector.currentBwHz, connector.currentSf, connector.currentTxPower, - connector.getTotalContactsUnreadCount(), - connector.getTotalChannelsUnreadCount(), ), ); } diff --git a/lib/services/backup_service.dart b/lib/services/backup_service.dart index 13048808..ea0d5e36 100644 --- a/lib/services/backup_service.dart +++ b/lib/services/backup_service.dart @@ -13,6 +13,7 @@ import 'app_settings_service.dart'; import '../storage/channel_store.dart'; import '../storage/contact_store.dart'; import '../storage/prefs_manager.dart'; +import '../storage/history_store.dart'; const int _backupVersion = 1; const int _pbkdf2Iterations = 310000; @@ -134,6 +135,7 @@ class BackupService { required String password, }) async { if (items.isEmpty) throw StateError('Select at least one backup item'); + await connector.flushPendingWrites(); final includedItems = Set.of(items); final customVars = connector.currentCustomVars ?? const {}; if (includedItems.isEmpty) { @@ -175,7 +177,9 @@ class BackupService { }; } if (includedItems.contains(BackupItem.regions)) { - node['preferences'] = _snapshotPreferences((key) => key == 'regions'); + node['preferences'] = await _snapshotPreferences( + (key) => key == 'regions', + ); } sections[BackupSection.nodeSettings.name] = node; } @@ -239,8 +243,8 @@ class BackupService { } sections[section.name] = { 'preferences': section == BackupSection.appSettings - ? _snapshotAppSettingsPreferences(sectionItems) - : _snapshotPreferences( + ? await _snapshotAppSettingsPreferences(sectionItems) + : await _snapshotPreferences( (key) => _preferenceBelongsToItems(key, sectionItems, devicePrefix), ), @@ -266,6 +270,7 @@ class BackupService { required AppSettingsService appSettingsService, }) async { if (items.isEmpty) throw StateError('Select at least one item to restore'); + await connector.flushPendingWrites(); final payload = await _decodeImport(contents, password: password); final sectionData = payload['sections']; if (sectionData is! Map) throw const FormatException('Missing backup data'); @@ -956,14 +961,19 @@ class BackupService { static int _intOr(Object? value, int fallback) => value is int ? value : fallback; - static Map _snapshotPreferences( + static Future> _snapshotPreferences( bool Function(String) include, - ) { + ) async { final prefs = PrefsManager.instance; final values = {}; - for (final key in prefs.getKeys()) { + for (final key in { + ...prefs.getKeys(), + ...await HistoryStore.instance.getKeys(), + }) { if (!include(key)) continue; - final value = prefs.get(key); + final value = HistoryStore.isHistoryKey(key) + ? await HistoryStore.instance.getString(key) + : prefs.get(key); if (value is String) { values[key] = _expandJsonPreference(value); } else if (value is bool || value is int || value is double) { @@ -1003,10 +1013,10 @@ class BackupService { return value; } - static Map _snapshotAppSettingsPreferences( + static Future> _snapshotAppSettingsPreferences( Set items, - ) { - final values = _snapshotPreferences( + ) async { + final values = await _snapshotPreferences( (key) => _preferenceBelongsToItems(key, items, ''), ); final appSettings = _asMap(values['app_settings']); @@ -1224,30 +1234,37 @@ class BackupService { final existingAppSettings = _asJsonMapPreference( prefs.getString('app_settings'), ); - final matching = prefs.getKeys().where((key) { - if (section == BackupSection.appSettings && - key == 'app_settings' && - !items.contains(BackupItem.appSettings)) { - return false; - } - if (!_preferenceBelongsToSection( - key, - section, - devicePrefix: destinationPrefix, - ) || - !_preferenceBelongsToItems(key, items, destinationPrefix)) { - return false; - } - if (section == BackupSection.appSettings || - section == BackupSection.nodeSettings) { - return true; - } - return _hasDevicePrefix(key, section, destinationPrefix) || - (section == BackupSection.channels && - key.startsWith('message_filter_')); - }).toList(); + final matching = + {...prefs.getKeys(), ...await HistoryStore.instance.getKeys()}.where(( + key, + ) { + if (section == BackupSection.appSettings && + key == 'app_settings' && + !items.contains(BackupItem.appSettings)) { + return false; + } + if (!_preferenceBelongsToSection( + key, + section, + devicePrefix: destinationPrefix, + ) || + !_preferenceBelongsToItems(key, items, destinationPrefix)) { + return false; + } + if (section == BackupSection.appSettings || + section == BackupSection.nodeSettings) { + return true; + } + return _hasDevicePrefix(key, section, destinationPrefix) || + (section == BackupSection.channels && + key.startsWith('message_filter_')); + }).toList(); for (final key in matching) { - await prefs.remove(key); + if (HistoryStore.isHistoryKey(key)) { + await HistoryStore.instance.remove(key); + } else { + await prefs.remove(key); + } } for (final entry in values.entries) { if (!_preferenceBelongsToItems(entry.key, items, sourcePrefix)) { @@ -1260,7 +1277,13 @@ class BackupService { destinationPrefix, ); final value = entry.value; - if (section == BackupSection.appSettings && key == 'app_settings') { + if (HistoryStore.isHistoryKey(key)) { + await HistoryStore.instance.setString( + key, + value is String ? value : jsonEncode(value), + ); + } else if (section == BackupSection.appSettings && + key == 'app_settings') { final storedIncoming = _asJsonMapPreference(value); if (storedIncoming == null) { throw const FormatException('Invalid app settings in backup'); diff --git a/lib/services/map_tile_cache_repository.dart b/lib/services/map_tile_cache_repository.dart new file mode 100644 index 00000000..cb83bc26 --- /dev/null +++ b/lib/services/map_tile_cache_repository.dart @@ -0,0 +1,2 @@ +export 'map_tile_cache_repository_stub.dart' + if (dart.library.io) 'map_tile_cache_repository_io.dart'; diff --git a/lib/services/map_tile_cache_repository_io.dart b/lib/services/map_tile_cache_repository_io.dart new file mode 100644 index 00000000..13cac5f2 --- /dev/null +++ b/lib/services/map_tile_cache_repository_io.dart @@ -0,0 +1,312 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter/foundation.dart'; +import 'package:flutter_cache_manager/flutter_cache_manager.dart'; +import 'package:path_provider/path_provider.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +CacheInfoRepository createMapTileCacheRepository(String cacheKey) { + if (!Platform.isLinux && !Platform.isWindows) { + return CacheObjectProvider(databaseName: cacheKey); + } + sqfliteFfiInit(); + return SqliteMapTileCacheRepository(databaseName: cacheKey); +} + +/// Tile metadata stays in SQLite; only queried rows become Dart objects. +class SqliteMapTileCacheRepository implements CacheInfoRepository { + SqliteMapTileCacheRepository({ + required this.databaseName, + Future Function()? directory, + DatabaseFactory? factory, + }) : _directory = directory ?? getApplicationSupportDirectory, + _factory = factory ?? databaseFactoryFfi; + + static const int schemaVersion = 1; + static const int legacyImportVersion = 1; + static const String _table = 'cacheObject'; + static const String _migrationTable = 'cacheMigration'; + static const String _legacyMigration = 'legacyJson'; + static const String _unreadableSuffix = '.unreadable'; + + final String databaseName; + final Future Function() _directory; + final DatabaseFactory _factory; + Future? _resolvedDirectory; + Future? _opening; + int _connections = 0; + + Future _baseDirectory() => _resolvedDirectory ??= _directory(); + + Future _databasePath() async => + '${(await _baseDirectory()).path}${Platform.pathSeparator}$databaseName.db'; + + Future _legacyFile() async => File( + '${(await _baseDirectory()).path}${Platform.pathSeparator}$databaseName.json', + ); + + Future _database() async { + final pending = _opening; + if (pending != null) return pending; + final started = _openDatabase(); + _opening = started; + try { + return await started; + } catch (_) { + if (identical(_opening, started)) _opening = null; + rethrow; + } + } + + Future _openDatabase() async { + final path = await _databasePath(); + await File(path).parent.create(recursive: true); + final db = await _factory.openDatabase( + path, + options: OpenDatabaseOptions( + version: schemaVersion, + onCreate: (db, version) async { + await db.execute(''' + CREATE TABLE $_table ( + _id INTEGER PRIMARY KEY, + url TEXT NOT NULL, + key TEXT NOT NULL UNIQUE, + relativePath TEXT NOT NULL, + eTag TEXT, + validTill INTEGER NOT NULL, + touched INTEGER NOT NULL, + length INTEGER + ) + '''); + await db.execute('CREATE INDEX cacheTouched ON $_table (touched)'); + await db.execute(''' + CREATE TABLE $_migrationTable ( + name TEXT PRIMARY KEY, + version INTEGER NOT NULL + ) + '''); + }, + ), + ); + try { + await _importLegacyJson(db); + return db; + } catch (_) { + await db.close(); + rethrow; + } + } + + Future _importLegacyJson(Database db) async { + final versions = await db.query( + _migrationTable, + where: 'name = ?', + whereArgs: [_legacyMigration], + ); + final version = versions.isEmpty ? 0 : versions.single['version'] as int; + if (version > legacyImportVersion) return; + final legacy = await _legacyFile(); + var unreadable = false; + if (version < legacyImportVersion) { + final List> records; + (records, unreadable) = await compute(_readLegacyRecords, legacy.path); + await db.transaction((transaction) async { + final batch = transaction.batch(); + for (final record in records) { + batch.insert( + _table, + record, + conflictAlgorithm: ConflictAlgorithm.ignore, + ); + } + batch.insert(_migrationTable, { + 'name': _legacyMigration, + 'version': legacyImportVersion, + }, conflictAlgorithm: ConflictAlgorithm.replace); + await batch.commit(noResult: true); + }); + } + // A committed version marker makes a crash before cleanup safe to retry. + try { + if (unreadable && await legacy.exists()) { + // Kept for inspection rather than deleted; it is never imported again. + debugPrint('Map metadata migration skipped unreadable records'); + await legacy.rename('${legacy.path}$_unreadableSuffix'); + } + for (final file in [legacy, File('${legacy.path}.tmp')]) { + if (await file.exists()) await file.delete(); + } + } on FileSystemException catch (error) { + debugPrint('Map metadata migration cleanup: $error'); + } + } + + @override + Future open() async { + _connections++; + try { + await _database(); + return true; + } catch (_) { + _connections--; + rethrow; + } + } + + @override + Future close() async { + if (_connections == 0 || --_connections > 0) return false; + final pending = _opening; + _opening = null; + if (pending != null) await (await pending).close(); + return true; + } + + @override + Future exists() async => File(await _databasePath()).exists(); + + @override + Future deleteDataFile() async { + if (_connections != 0) { + throw StateError('Close map metadata connections before deleting it.'); + } + await _factory.deleteDatabase(await _databasePath()); + final legacy = await _legacyFile(); + for (final file in [ + legacy, + File('${legacy.path}.tmp'), + File('${legacy.path}$_unreadableSuffix'), + ]) { + if (await file.exists()) await file.delete(); + } + } + + @override + Future get(String key) async { + final rows = await (await _database()).query( + _table, + where: 'key = ?', + whereArgs: [key], + limit: 1, + ); + return rows.isEmpty ? null : CacheObject.fromMap(rows.single); + } + + @override + Future insert( + CacheObject cacheObject, { + bool setTouchedToNow = true, + }) async { + if (cacheObject.id != null) { + throw ArgumentError('Inserted cache objects must not have an ID.'); + } + final map = cacheObject.toMap(setTouchedToNow: setTouchedToNow); + final id = await (await _database()).insert(_table, map); + return CacheObject.fromMap({...map, CacheObject.columnId: id}); + } + + @override + Future update( + CacheObject cacheObject, { + bool setTouchedToNow = true, + }) async { + if (cacheObject.id == null) { + throw ArgumentError('Updated cache objects must have an ID.'); + } + return (await _database()).update( + _table, + cacheObject.toMap(setTouchedToNow: setTouchedToNow), + where: '_id = ?', + whereArgs: [cacheObject.id], + ); + } + + @override + Future updateOrInsert(CacheObject cacheObject) => + cacheObject.id == null ? insert(cacheObject) : update(cacheObject); + + @override + Future delete(int id) async => + (await _database()).delete(_table, where: '_id = ?', whereArgs: [id]); + + @override + Future deleteAll(Iterable ids) async { + final unique = ids.toSet().toList(); + if (unique.isEmpty) return 0; + final db = await _database(); + return db.transaction((transaction) async { + var deleted = 0; + for (var start = 0; start < unique.length; start += 500) { + final chunk = unique.skip(start).take(500).toList(); + deleted += await transaction.delete( + _table, + where: '_id IN (${List.filled(chunk.length, '?').join(',')})', + whereArgs: chunk, + ); + } + return deleted; + }); + } + + @override + Future> getAllObjects() async => + CacheObject.fromMapList(await (await _database()).query(_table)); + + @override + Future> getObjectsOverCapacity(int capacity) async => + CacheObject.fromMapList( + await (await _database()).query( + _table, + where: 'touched < ?', + whereArgs: [ + DateTime.now() + .subtract(const Duration(days: 1)) + .millisecondsSinceEpoch, + ], + orderBy: 'touched DESC, _id DESC', + offset: capacity, + limit: 100, + ), + ); + + @override + Future> getOldObjects(Duration maxAge) async => + CacheObject.fromMapList( + await (await _database()).query( + _table, + where: 'touched < ?', + whereArgs: [DateTime.now().subtract(maxAge).millisecondsSinceEpoch], + orderBy: 'touched ASC, _id ASC', + limit: 100, + ), + ); +} + +/// Returns the importable records and whether any part of the file was +/// unreadable. A broken file must not keep the tile cache from opening. +(List>, bool) _readLegacyRecords(String path) { + final file = File(path); + if (!file.existsSync()) return ([], false); + final Object? decoded; + try { + decoded = jsonDecode(file.readAsStringSync()); + } on Exception { + return ([], true); + } + if (decoded is! List) return ([], true); + final records = >[]; + var unreadable = false; + for (final value in decoded) { + try { + records.add( + CacheObject.fromMap( + (value as Map).cast(), + ).toMap(setTouchedToNow: false)..remove(CacheObject.columnId), + ); + } catch (_) { + unreadable = true; + } + } + return (records, unreadable); +} diff --git a/lib/services/map_tile_cache_repository_stub.dart b/lib/services/map_tile_cache_repository_stub.dart new file mode 100644 index 00000000..55acec5d --- /dev/null +++ b/lib/services/map_tile_cache_repository_stub.dart @@ -0,0 +1,4 @@ +import 'package:flutter_cache_manager/flutter_cache_manager.dart'; + +CacheInfoRepository createMapTileCacheRepository(String cacheKey) => + NonStoringObjectProvider(); diff --git a/lib/services/map_tile_cache_service.dart b/lib/services/map_tile_cache_service.dart index dbe2da8b..3a6d1de8 100644 --- a/lib/services/map_tile_cache_service.dart +++ b/lib/services/map_tile_cache_service.dart @@ -8,6 +8,7 @@ import 'package:flutter_map/flutter_map.dart'; import '../models/app_settings.dart'; import '../theme/mesh_theme.dart'; import 'app_settings_service.dart'; +import 'map_tile_cache_repository.dart'; enum MapRasterSourcePreset { osmAuto('osm_auto'), @@ -347,6 +348,7 @@ class MapTileCacheService extends ChangeNotifier { cacheKey, stalePeriod: const Duration(days: 365), maxNrOfCacheObjects: 200000, + repo: createMapTileCacheRepository(cacheKey), ), ) { tileProvider = CachedNetworkTileProvider(cacheManager: this.cacheManager); @@ -534,19 +536,23 @@ class MapTileCacheService extends ChangeNotifier { Future getCachedTileInventory() async { final repo = _concreteCacheManager.config.repo; await repo.open(); - final objects = await repo.getAllObjects(); - final tiles = []; - int totalBytes = 0; + try { + final objects = await repo.getAllObjects(); + final tiles = []; + int totalBytes = 0; - for (final object in objects) { - totalBytes += object.length ?? 0; - final tile = _parseCachedTile(object); - if (tile != null) { - tiles.add(tile); + for (final object in objects) { + totalBytes += object.length ?? 0; + final tile = _parseCachedTile(object); + if (tile != null) { + tiles.add(tile); + } } - } - return CachedTileInventory(tiles: tiles, totalBytes: totalBytes); + return CachedTileInventory(tiles: tiles, totalBytes: totalBytes); + } finally { + await repo.close(); + } } List filterTilesForActiveSource( diff --git a/lib/services/path_history_service.dart b/lib/services/path_history_service.dart index 4093300d..43cec96b 100644 --- a/lib/services/path_history_service.dart +++ b/lib/services/path_history_service.dart @@ -5,6 +5,7 @@ import 'package:flutter/foundation.dart'; import '../models/contact.dart'; import '../models/path_history.dart'; import '../models/path_selection.dart'; +import '../utils/app_logger.dart'; import '../utils/coalesced_writes.dart'; import 'route_chooser.dart'; import 'storage_service.dart'; @@ -13,7 +14,10 @@ class PathHistoryService extends ChangeNotifier { final StorageService _storage; final Map _cache = {}; final Map _floodStats = {}; - final Map> _pendingLoads = {}; + final Map> _pendingLoads = {}; + int _generation = 0; + bool _disposed = false; + Future _clearing = Future.value(); // A path record lands for most received messages, so saves are batched. final CoalescedWrites _writes = CoalescedWrites( delay: const Duration(seconds: 5), @@ -43,9 +47,12 @@ class PathHistoryService extends ChangeNotifier { _now = clock ?? DateTime.now; void setDevicePublicKey(String publicKeyHex) { + if (_disposed) return; // Queued saves use the storage key on entry, so they still land under // the previous radio's key. unawaited(_writes.flush()); + _generation++; + _pendingLoads.clear(); _storage.setPublicKeyHex = publicKeyHex; _cache.clear(); _cacheAccessOrder.clear(); @@ -59,6 +66,7 @@ class PathHistoryService extends ChangeNotifier { } void handlePathUpdated(Contact contact, {double initialWeight = 1.0}) { + if (_disposed) return; if (contact.pathLength < 0 && contact.path.isEmpty) return; final hopCount = contact.pathLength < 0 ? contact.path.length @@ -77,6 +85,7 @@ class PathHistoryService extends ChangeNotifier { } void recordPathAttempt(String contactPubKeyHex, PathSelection selection) { + if (_disposed) return; if (selection.useFlood) { _updateFloodStats(contactPubKeyHex); return; @@ -104,11 +113,18 @@ class PathHistoryService extends ChangeNotifier { double successIncrement = 0.5, double maxWeight = 5.0, }) { + if (_disposed) return; // A flood answered directly credits the direct (zero-hop) path. if (hopCount < 0 || (pathBytes.isEmpty && hopCount != 0)) return; if (!_cache.containsKey(contactPubKeyHex)) { - _ensureLoaded(contactPubKeyHex).then((_) { - if (!_cache.containsKey(contactPubKeyHex)) return; + final generation = _generation; + _ensureLoaded(contactPubKeyHex).then((loaded) { + if (!loaded || + _disposed || + generation != _generation || + !_cache.containsKey(contactPubKeyHex)) { + return; + } recordFloodPathAttribution( contactPubKeyHex: contactPubKeyHex, pathBytes: pathBytes, @@ -160,6 +176,7 @@ class PathHistoryService extends ChangeNotifier { double failureDecrement = 0.5, double maxWeight = 5.0, }) { + if (_disposed) return; if (selection.useFlood) { final stats = _floodStats.putIfAbsent( contactPubKeyHex, @@ -176,8 +193,14 @@ class PathHistoryService extends ChangeNotifier { } if (!_cache.containsKey(contactPubKeyHex)) { - _ensureLoaded(contactPubKeyHex).then((_) { - if (!_cache.containsKey(contactPubKeyHex)) return; + final generation = _generation; + _ensureLoaded(contactPubKeyHex).then((loaded) { + if (!loaded || + _disposed || + generation != _generation || + !_cache.containsKey(contactPubKeyHex)) { + return; + } recordPathResult( contactPubKeyHex, selection, @@ -294,6 +317,8 @@ class PathHistoryService extends ChangeNotifier { double? recentFailure, DateTime? statsAt, }) { + if (_disposed) return; + final generation = _generation; void add() => _addPathRecordInternal( contactPubKeyHex, hopCount, @@ -313,27 +338,43 @@ class PathHistoryService extends ChangeNotifier { if (_cache.containsKey(contactPubKeyHex)) { add(); } else { - _ensureLoaded(contactPubKeyHex).then((_) => add()); + _ensureLoaded(contactPubKeyHex).then((loaded) { + if (loaded && !_disposed && generation == _generation) add(); + }); } } - Future _ensureLoaded(String contactPubKeyHex) { - if (_cache.containsKey(contactPubKeyHex)) return Future.value(); - return _pendingLoads[contactPubKeyHex] ??= - _loadHistoryFromStorage(contactPubKeyHex) - .then((loaded) { - _cache[contactPubKeyHex] ??= - loaded ?? - ContactPathHistory( - contactPubKeyHex: contactPubKeyHex, - recentPaths: [], - ); - _trackAccess(contactPubKeyHex); - _evictIfNeeded(); - }) - .whenComplete(() { - _pendingLoads.remove(contactPubKeyHex); - }); + Future _ensureLoaded(String contactPubKeyHex) { + if (_disposed) return Future.value(false); + if (_cache.containsKey(contactPubKeyHex)) return Future.value(true); + final existing = _pendingLoads[contactPubKeyHex]; + if (existing != null) return existing; + final generation = _generation; + late final Future pending; + pending = _loadHistoryFromStorage(contactPubKeyHex) + .then((loaded) { + if (_disposed || + generation != _generation || + !identical(_pendingLoads[contactPubKeyHex], pending)) { + return false; + } + _cache[contactPubKeyHex] ??= + loaded ?? + ContactPathHistory( + contactPubKeyHex: contactPubKeyHex, + recentPaths: [], + ); + _trackAccess(contactPubKeyHex); + _evictIfNeeded(); + return true; + }) + .whenComplete(() { + if (identical(_pendingLoads[contactPubKeyHex], pending)) { + _pendingLoads.remove(contactPubKeyHex); + } + }); + _pendingLoads[contactPubKeyHex] = pending; + return pending; } void _addPathRecordInternal( @@ -351,6 +392,7 @@ class PathHistoryService extends ChangeNotifier { double? recentFailure, DateTime? statsAt, }) { + if (_disposed) return; var history = _cache[contactPubKeyHex]; if (history == null) return; _version++; @@ -429,6 +471,7 @@ class PathHistoryService extends ChangeNotifier { /// Reads saved paths straight from storage without adding them to the /// cache, evicting anything, or notifying listeners. Future> readStoredPaths(String contactPubKeyHex) async { + if (_disposed) return []; final cached = _cache[contactPubKeyHex]; if (cached != null) return cached.recentPaths; final stored = await _loadHistoryFromStorage(contactPubKeyHex); @@ -436,13 +479,16 @@ class PathHistoryService extends ChangeNotifier { } List getRecentPaths(String contactPubKeyHex) { + if (_disposed) return []; final history = _cache[contactPubKeyHex]; if (history != null) { _trackAccess(contactPubKeyHex); return history.recentPaths; } - _ensureLoaded(contactPubKeyHex).then((_) { + final generation = _generation; + _ensureLoaded(contactPubKeyHex).then((loaded) { + if (!loaded || _disposed || generation != _generation) return; if (_cache[contactPubKeyHex]?.recentPaths.isNotEmpty ?? false) { _version++; notifyListeners(); @@ -455,10 +501,12 @@ class PathHistoryService extends ChangeNotifier { Future _loadHistoryFromStorage( String contactPubKeyHex, ) async { - // An evicted contact may still have a queued save; it reaches the prefs - // cache synchronously, so the load below reads it. - unawaited(_writes.flush()); - return await _storage.loadPathHistory(contactPubKeyHex); + final generation = _generation; + await _clearing; + await _writes.flush(); + if (_disposed || generation != _generation) return null; + final history = await _storage.loadPathHistory(contactPubKeyHex); + return _disposed || generation != _generation ? null : history; } PathRecord? getFastestPath(String contactPubKeyHex) { @@ -494,12 +542,38 @@ class PathHistoryService extends ChangeNotifier { ); } + /// Runs [clear] after earlier clears and queued saves. [_clearing] itself + /// never fails, so one failed clear cannot break later loads and flushes; + /// the returned future still reports the failure to its caller. + Future _queueClear(int generation, Future Function() clear) { + final previous = _clearing; + final run = () async { + await previous; + await _writes.flush(); + if (_disposed || generation != _generation) return; + await clear(); + }(); + _clearing = run.then( + (_) {}, + onError: (Object error, StackTrace _) { + appLogger.warn('Clearing path history failed: $error', tag: 'Storage'); + }, + ); + return run; + } + Future clearPathHistory(String contactPubKeyHex) async { + if (_disposed) return; + final generation = _generation; + _pendingLoads.remove(contactPubKeyHex); _cache.remove(contactPubKeyHex); _cacheAccessOrder.remove(contactPubKeyHex); _floodStats.remove(contactPubKeyHex); _writes.cancel(contactPubKeyHex); - await _storage.clearPathHistory(contactPubKeyHex); + await _queueClear(generation, () { + return _storage.clearPathHistory(contactPubKeyHex); + }); + if (_disposed || generation != _generation) return; _version++; notifyListeners(); } @@ -508,6 +582,8 @@ class PathHistoryService extends ChangeNotifier { String contactPubKeyHex, List pathBytes, ) async { + if (_disposed) return; + final generation = _generation; final history = _cache[contactPubKeyHex]; if (history == null) return; @@ -519,8 +595,12 @@ class PathHistoryService extends ChangeNotifier { recentPaths: updatedPaths, ); + final updated = _cache[contactPubKeyHex]!; _writes.cancel(contactPubKeyHex); - await _storage.savePathHistory(contactPubKeyHex, _cache[contactPubKeyHex]!); + await _writes.flush(); + if (_disposed || generation != _generation) return; + await _storage.savePathHistory(contactPubKeyHex, updated); + if (_disposed || generation != _generation) return; _version++; notifyListeners(); } @@ -607,20 +687,39 @@ class PathHistoryService extends ChangeNotifier { } /// Runs queued saves now, e.g. before the app is backgrounded. - Future flushPendingWrites() => _writes.flush(); + Future flushPendingWrites() async { + await _clearing; + await _writes.flush(); + } @override void dispose() { + if (_disposed) return; unawaited(_writes.flush()); + _disposed = true; + _generation++; + _pendingLoads.clear(); + _cache.clear(); + _cacheAccessOrder.clear(); + _floodStats.clear(); super.dispose(); } void clearAllHistories() { + if (_disposed) return; + final generation = ++_generation; + _pendingLoads.clear(); _cache.clear(); _cacheAccessOrder.clear(); _floodStats.clear(); _writes.dispose(); - _storage.clearAllPathHistories(); + // Nothing awaits this one; _queueClear logs a failure. + unawaited( + _queueClear( + generation, + _storage.clearAllPathHistories, + ).catchError((Object _) {}), + ); _version = 0; notifyListeners(); } diff --git a/lib/services/storage_service.dart b/lib/services/storage_service.dart index 52045f0d..d4532b32 100644 --- a/lib/services/storage_service.dart +++ b/lib/services/storage_service.dart @@ -3,6 +3,7 @@ import '../models/delivery_observation.dart'; import '../models/path_history.dart'; import '../storage/legacy_key_migration.dart'; import '../storage/prefs_manager.dart'; +import '../storage/history_store.dart'; import '../utils/app_logger.dart'; class StorageService { @@ -64,30 +65,32 @@ class StorageService { String contactPubKeyHex, ContactPathHistory history, ) async { - final prefs = PrefsManager.instance; + final prefs = HistoryStore.instance; final key = _pathHistoryKey(contactPubKeyHex); final jsonStr = jsonEncode(history.toJson()); await prefs.setString(key, jsonStr); } Future loadPathHistory(String contactPubKeyHex) async { - final prefs = PrefsManager.instance; + final prefs = HistoryStore.instance; + final scope = _publicKeyHex; + final allowLegacyMigration = + scope.isNotEmpty && canMigrateLegacyKeys(scope); final key = _pathHistoryKey(contactPubKeyHex); - var jsonStr = prefs.getString(key); + var jsonStr = await prefs.getString(key); final legacyKey = '$_pathHistoryPrefix$contactPubKeyHex'; - if (jsonStr == null && - _publicKeyHex.isNotEmpty && - canMigrateLegacyKeys(_publicKeyHex)) { + if (jsonStr == null && allowLegacyMigration) { // Move the pre-scoping history into the first radio that loads it so // learned routes survive the upgrade without leaking into other radios. - final legacy = prefs.getString(legacyKey); + final legacy = await prefs.getString(legacyKey); if (legacy != null) { - await prefs.setString(key, legacy); - await prefs.remove(legacyKey); + if (await prefs.setString(key, legacy)) { + await prefs.remove(legacyKey); + } jsonStr = legacy; } - } else if (jsonStr == null && _publicKeyHex.isEmpty) { - jsonStr = prefs.getString(legacyKey); + } else if (jsonStr == null && scope.isEmpty) { + jsonStr = await prefs.getString(legacyKey); } if (jsonStr == null) return null; @@ -101,7 +104,7 @@ class StorageService { } Future clearPathHistory(String contactPubKeyHex) async { - final prefs = PrefsManager.instance; + final prefs = HistoryStore.instance; await prefs.remove(_pathHistoryKey(contactPubKeyHex)); await prefs.remove('$_pathHistoryPrefix$contactPubKeyHex'); } @@ -109,7 +112,7 @@ class StorageService { /// Clears the path histories of the connected radio only. Histories of other /// radios are left alone; with no radio scope set, everything is cleared. Future clearAllPathHistories() async { - final prefs = PrefsManager.instance; + final prefs = HistoryStore.instance; final scopedPrefix = _publicKeyHex.isEmpty ? _pathHistoryPrefix : '$_pathHistoryPrefix${_publicKeyHex}_'; @@ -117,7 +120,7 @@ class StorageService { // radio on the next load and bring the cleared routes back. final clearLegacy = _publicKeyHex.isNotEmpty && canMigrateLegacyKeys(_publicKeyHex); - final pathHistoryKeys = prefs.getKeys().where((key) { + final pathHistoryKeys = (await prefs.getKeys()).where((key) { if (key.startsWith(scopedPrefix)) return true; return clearLegacy && key.startsWith(_pathHistoryPrefix) && diff --git a/lib/storage/channel_message_store.dart b/lib/storage/channel_message_store.dart index 81294344..ca1efeb2 100644 --- a/lib/storage/channel_message_store.dart +++ b/lib/storage/channel_message_store.dart @@ -6,7 +6,8 @@ import '../models/channel_message.dart'; import '../models/message.dart' show legacyAwareReceivedAt; import '../models/translation_support.dart'; import '../helpers/smaz.dart'; -import 'prefs_manager.dart'; +import 'history_archive.dart'; +import 'history_store.dart'; class ChannelMessageStore { static const String _keyPrefix = 'channel_messages_'; @@ -21,153 +22,117 @@ class ChannelMessageStore { /// beyond it are dropped on save. static const int maxStoredMessages = 1000; - /// Encoded entries persisted ahead of the connector's in-memory window, - /// keyed by scoped storage key. A missing entry means the window has not - /// been reconciled with storage yet, so the next save keeps what is already - /// stored instead of replacing it. Shared across instances so a store - /// created by a screen to clear a key also resets the connector's view. - static final Map> _archived = {}; + static final HistoryArchive _archive = HistoryArchive( + mergeKey: _storedMergeKey, + maxEntries: maxStoredMessages, + ); String _keyForChannel(String channelId) => '${keyFor}_$channelId'; - /// Save messages for a specific channel: [messages] is the in-memory - /// window (oldest first) and is written after the archived entries that - /// sit ahead of it in storage. Future saveChannelMessages( String channelId, List messages, - ) async { - if (publicKeyHex.isEmpty) { - appLogger.warn( - 'Public key hex is not set. Cannot save channel messages.', - ); - return; - } - final prefs = PrefsManager.instance; - final key = _keyForChannel(channelId); - final visibleJson = messages.map(_messageToJson).toList(); - final archived = - _archived[key] ?? - _unreconciledArchive(prefs.getString(key), visibleJson); - final overflow = archived.length + visibleJson.length - maxStoredMessages; - if (overflow > 0) { - archived.removeRange(0, overflow.clamp(0, archived.length)); - } - _archived[key] = archived; - final entries = [...archived, ...visibleJson.map(jsonEncode)]; - await prefs.setString(key, '[${entries.join(',')}]'); - } - - /// Records [olderMessages] as the stored entries that precede the - /// in-memory window for [channelId]. - void setArchived(String channelId, List olderMessages) { - _archived[_keyForChannel(channelId)] = olderMessages - .map((message) => jsonEncode(_messageToJson(message))) - .toList(); - } - - /// Number of stored entries ahead of the in-memory window, or null when - /// the window has not been reconciled with storage. - int? archivedCount(String channelId) => - _archived[_keyForChannel(channelId)]?.length; - - /// Removes the newest [count] archived entries after they were paged back - /// into the in-memory window. - void unarchiveNewest(String channelId, int count) { - final archived = _archived[_keyForChannel(channelId)]; - if (archived == null || count <= 0) return; - archived.removeRange( - (archived.length - count).clamp(0, archived.length), - archived.length, - ); - } - - /// Drops the archive bookkeeping, for one channel or all of them, when the - /// matching in-memory window is discarded. - void forgetArchive([String? channelId]) { - if (channelId == null) { - _archived.clear(); - } else { - _archived.remove(_keyForChannel(channelId)); - } - } - - List _unreconciledArchive( - String? storedJson, - List> visibleJson, ) { - if (storedJson == null || storedJson.isEmpty) return []; - final List stored; - try { - stored = jsonDecode(storedJson) as List; - } catch (_) { - return []; + if (publicKeyHex.isEmpty) { + appLogger.warn('Public key hex is not set. Cannot save messages.'); + return Future.value(); } - final visibleKeys = visibleJson.map(_storedMergeKey).toSet(); - return [ - for (final entry in stored) - if (entry is Map && - !visibleKeys.contains(_storedMergeKey(entry))) - jsonEncode(entry), - ]; + return _archive.save(_keyForChannel(channelId), [ + for (final message in messages) jsonEncode(_messageToJson(message)), + ]); } + /// The stored prefix stays in the database, rather than another JSON cache. + void setArchivedCount(String channelId, int count) => + _archive.setArchivedCount(_keyForChannel(channelId), count); + + void archive(String channelId, List messages) => + _archive.archive( + _keyForChannel(channelId), + messages.map((message) => jsonEncode(_messageToJson(message))), + ); + + int? archivedCount(String channelId) => + _archive.archivedCount(_keyForChannel(channelId)); + + void forgetArchive([String? channelId]) => + _archive.forget(channelId == null ? null : _keyForChannel(channelId)); + static String _storedMergeKey(Map json) { final messageId = json['messageId']; if (messageId is String && messageId.isNotEmpty) return 'id:$messageId'; return 'fallback:${json['timestamp']}:${json['senderName']}:${json['text']}'; } - /// Load messages for a specific channel - Future> loadChannelMessages(String channelId) async { - if (publicKeyHex.isEmpty) { - appLogger.warn( - 'Public key hex is not set. Cannot load channel messages.', - ); - return []; - } - final jsonString = PrefsManager.instance.getString( - _keyForChannel(channelId), - ); - if (jsonString == null || jsonString.isEmpty) { - return []; - } - final List jsonList; + Future> loadChannelMessages( + String channelId, { + int offset = 0, + int? limit, + }) async { + if (publicKeyHex.isEmpty) return []; + final key = _keyForChannel(channelId); try { - jsonList = jsonDecode(jsonString) as List; - } catch (e) { + final entries = await HistoryStore.instance.readEntries( + key, + offset: offset, + limit: limit, + ); + return _decodeMessages(entries); + } catch (error) { appLogger.warn( - 'Stored messages for channel $channelId are unreadable: $e', + 'Stored messages for channel $channelId are unreadable: $error', ); return []; } + } + + Future<({List messages, int start, int end, int total})> + loadMessagePage(String channelId, {int? before, int limit = 50}) async { + RangeError.checkNotNegative(limit, 'limit'); + if (publicKeyHex.isEmpty) { + return (messages: [], start: 0, end: 0, total: 0); + } + final key = _keyForChannel(channelId); + try { + return await _archive.readPage( + key, + await HistoryStore.instance.entryCount(key), + _decodeMessages, + before: before, + limit: limit, + ); + } on FormatException catch (error) { + appLogger.warn( + 'Stored messages for channel $channelId are unreadable: $error', + ); + return (messages: [], start: 0, end: 0, total: 0); + } + } + + List _decodeMessages(List entries) { final messages = []; - for (final json in jsonList) { + for (final entry in entries) { try { - messages.add(_messageFromJson(json as Map)); - } catch (e) { - appLogger.warn('Skipping malformed stored channel message: $e'); + messages.add( + _messageFromJson(jsonDecode(entry) as Map), + ); + } catch (error) { + appLogger.warn('Skipping malformed stored channel message: $error'); } } return messages; } - /// Clear messages for a specific channel - Future clearChannelMessages(String channelId) async { - final prefs = PrefsManager.instance; - final key = _keyForChannel(channelId); - _archived[key] = []; - await prefs.remove(key); - } + Future clearChannelMessages(String channelId) => + _archive.clear(HistoryStore.instance, _keyForChannel(channelId)); - /// Clear all channel messages Future clearAllChannelMessages() async { - final prefs = PrefsManager.instance; - final keys = prefs.getKeys().where((k) => k.startsWith(keyFor)).toList(); - _archived.clear(); - for (var key in keys) { - await prefs.remove(key); - } + final store = HistoryStore.instance; + final prefix = keyFor; + final keys = (await store.getKeys()) + .where((key) => key.startsWith(prefix)) + .toList(); + await Future.wait(keys.map((key) => _archive.clear(store, key))); } /// Convert ChannelMessage to JSON map diff --git a/lib/storage/channel_storage_migration.dart b/lib/storage/channel_storage_migration.dart index ec4afbbb..ec0e8d66 100644 --- a/lib/storage/channel_storage_migration.dart +++ b/lib/storage/channel_storage_migration.dart @@ -5,13 +5,15 @@ import 'channel_message_store.dart'; import 'channel_store.dart'; import 'legacy_key_migration.dart'; import 'prefs_manager.dart'; +import 'history_store.dart'; /// One-shot move of a radio's per-channel data from slot-index keys /// (``) to channel-ID keys (`_`). /// /// The slot -> PSK mapping comes from the radio's cached channel list, which is /// the mapping the legacy data was written under. Legacy keys of slots with no -/// cached PSK are left alone and never read again. +/// cached PSK are left alone and never read again, and so is a legacy history +/// that cannot be parsed. class ChannelStorageMigration { static const String _doneKeyPrefix = 'channel_storage_v2_'; @@ -70,8 +72,8 @@ class ChannelStorageMigration { await _migrateOrder(scope, slotIds); if (slotIds.isNotEmpty) await beforeDone?.call(slotIds); - await prefs.setBool(doneKey, true); await channelStore.saveSlotChannelIds(slotIds); + await _requireWrite(prefs.setBool(doneKey, true)); if (slotIds.isNotEmpty) { appLogger.info( 'Moved channel data for ${slotIds.length} slots of $scope to channel IDs', @@ -81,38 +83,60 @@ class ChannelStorageMigration { } static Future _moveMessages(String scope, int slot, String id) async { - final prefs = PrefsManager.instance; + final prefs = HistoryStore.instance; final oldKey = '$_messagesPrefix$scope$slot'; final legacyKey = '$_messagesPrefix$slot'; final newKey = '$_messagesPrefix${scope}_$id'; - var json = prefs.getString(oldKey); - if (json == null || json.isEmpty) json = prefs.getString(legacyKey); + var json = await prefs.getString(oldKey); + if (json == null || json.isEmpty) json = await prefs.getString(legacyKey); if (json != null && json.isNotEmpty) { - final existing = prefs.getString(newKey); - // Two slots holding the same PSK now share one history. - await prefs.setString( - newKey, - existing == null || existing.isEmpty + final String moved; + try { + if (jsonDecode(json) is! List) { + throw const FormatException('Channel history is not an array'); + } + final existing = await prefs.getString(newKey); + // Two slots holding the same PSK now share one history. + moved = existing == null || existing.isEmpty ? json - : _mergeMessages(existing, json), - ); + : _mergeMessages(existing, json); + } on FormatException catch (error) { + // One unreadable history must not hold back the other slots. It + // stays under its old key, which nothing reads again. + appLogger.warn( + 'Leaving unreadable channel history of slot $slot in place: $error', + ); + return; + } + await _requireWrite(prefs.setString(newKey, moved)); } // Old keys go only after the new one is written, so an interrupted run // loses nothing; the next launch repeats it. - await prefs.remove(oldKey); - await prefs.remove(legacyKey); + await _requireWrite(prefs.remove(oldKey)); + await _requireWrite(prefs.remove(legacyKey)); } static String _mergeMessages(String a, String b) { try { // A rerun after an interrupted migration merges a history into // itself, so drop entries already present. + final existing = jsonDecode(a); + final incoming = jsonDecode(b); + if (existing is! List || + incoming is! List || + ![ + ...existing, + ...incoming, + ].every((entry) => entry is Map)) { + throw const FormatException( + 'Cannot merge unreadable channel histories', + ); + } final seen = {}; - final merged = - [...jsonDecode(a) as List, ...jsonDecode(b) as List] - .whereType>() - .where((m) => seen.add(_mergeKey(m))) - .toList(); + final merged = [...existing, ...incoming] + .cast>() + .where((m) => seen.add(_mergeKey(m))) + .toList(); merged.sort( (x, y) => ((x['timestamp'] as int?) ?? 0).compareTo( (y['timestamp'] as int?) ?? 0, @@ -122,7 +146,7 @@ class ChannelStorageMigration { if (overflow > 0) merged.removeRange(0, overflow); return jsonEncode(merged); } catch (_) { - return a; + throw const FormatException('Cannot merge unreadable channel histories'); } } @@ -145,19 +169,19 @@ class ChannelStorageMigration { var value = prefs.get(oldKey); if (value == null && legacyKey != null) value = prefs.get(legacyKey); if (value is bool && !prefs.containsKey(newKey)) { - await prefs.setBool(newKey, value); + await _requireWrite(prefs.setBool(newKey, value)); } - await prefs.remove(oldKey); - if (legacyKey != null) await prefs.remove(legacyKey); + await _removeIfPresent(oldKey); + if (legacyKey != null) await _removeIfPresent(legacyKey); } static Future _moveString(String oldKey, String newKey) async { final prefs = PrefsManager.instance; final value = prefs.get(oldKey); if (value is String && !prefs.containsKey(newKey)) { - await prefs.setString(newKey, value); + await _requireWrite(prefs.setString(newKey, value)); } - await prefs.remove(oldKey); + await _removeIfPresent(oldKey); } static Future _migrateOrder( @@ -188,7 +212,18 @@ class ChannelStorageMigration { final id = slotIds[slot]; if (id != null && !ids.contains(id)) ids.add(id); } - await prefs.setString(key, jsonEncode(ids)); - await prefs.remove(_orderPrefix); + await _requireWrite(prefs.setString(key, jsonEncode(ids))); + await _removeIfPresent(_orderPrefix); + } + + static Future _removeIfPresent(String key) async { + final prefs = PrefsManager.instance; + if (prefs.containsKey(key)) await prefs.remove(key); + } + + static Future _requireWrite(Future write) async { + if (!await write) { + throw StateError('Could not persist channel storage migration'); + } } } diff --git a/lib/storage/channel_store.dart b/lib/storage/channel_store.dart index 2de42fa5..49ec8c82 100644 --- a/lib/storage/channel_store.dart +++ b/lib/storage/channel_store.dart @@ -24,16 +24,7 @@ class ChannelStore { String? jsonString = prefs.getString(keyFor); if ((jsonString == null || jsonString.isEmpty) && canMigrateLegacyKeys(publicKeyHex)) { - // Attempt migration from legacy unscoped key on first load - final legacyJsonString = prefs.getString(_keyPrefix); - prefs.remove(_keyPrefix); - if (legacyJsonString != null && legacyJsonString.isNotEmpty) { - appLogger.info( - 'Migrating channel messages from legacy key $_keyPrefix to scoped key $keyFor', - ); - await prefs.setString(keyFor, legacyJsonString); - jsonString = legacyJsonString; - } + jsonString = await migrateLegacyStringPreference(_keyPrefix, keyFor); } if (jsonString == null || jsonString.isEmpty) { @@ -97,10 +88,11 @@ class ChannelStore { Future saveSlotChannelIds(Map ids) async { if (publicKeyHex.isEmpty) return; - await PrefsManager.instance.setString( + final saved = await PrefsManager.instance.setString( _slotIdsKey, jsonEncode({for (final e in ids.entries) '${e.key}': e.value}), ); + if (!saved) throw StateError('Could not persist channel slot map'); } Map _toJson(Channel channel) { diff --git a/lib/storage/community_store.dart b/lib/storage/community_store.dart index 3ceebf4b..719d0114 100644 --- a/lib/storage/community_store.dart +++ b/lib/storage/community_store.dart @@ -29,16 +29,7 @@ class CommunityStore { String? jsonString = prefs.getString(keyFor); if ((jsonString == null || jsonString.isEmpty) && canMigrateLegacyKeys(publicKeyHex)) { - // Attempt migration from legacy unscoped key on first load - final legacyJsonString = prefs.getString(_keyPrefix); - prefs.remove(_keyPrefix); - if (legacyJsonString != null && legacyJsonString.isNotEmpty) { - appLogger.info( - 'Migrating communities from legacy key $_keyPrefix to scoped key $keyFor', - ); - await prefs.setString(keyFor, legacyJsonString); - jsonString = legacyJsonString; - } + jsonString = await migrateLegacyStringPreference(_keyPrefix, keyFor); } if (jsonString == null || jsonString.isEmpty) { jsonString = prefs.getString(keyFor); diff --git a/lib/storage/contact_discovery_store.dart b/lib/storage/contact_discovery_store.dart index 39646794..b747f094 100644 --- a/lib/storage/contact_discovery_store.dart +++ b/lib/storage/contact_discovery_store.dart @@ -20,11 +20,7 @@ class ContactDiscoveryStore { final prefs = PrefsManager.instance; var jsonStr = prefs.getString(keyFor); if (jsonStr == null && canMigrateLegacyKeys(publicKeyHex)) { - jsonStr = prefs.getString(_keyPrefix); - if (jsonStr != null) { - await prefs.setString(keyFor, jsonStr); - await prefs.remove(_keyPrefix); - } + jsonStr = await migrateLegacyStringPreference(_keyPrefix, keyFor); } if (jsonStr == null) return []; diff --git a/lib/storage/contact_group_store.dart b/lib/storage/contact_group_store.dart index fb297005..281304dc 100644 --- a/lib/storage/contact_group_store.dart +++ b/lib/storage/contact_group_store.dart @@ -22,16 +22,7 @@ class ContactGroupStore { String? jsonString = prefs.getString(keyFor); if ((jsonString == null || jsonString.isEmpty) && canMigrateLegacyKeys(publicKeyHex)) { - // Attempt migration from legacy unscoped key on first load - final legacyJsonString = prefs.getString(_keyPrefix); - prefs.remove(_keyPrefix); - if (legacyJsonString != null && legacyJsonString.isNotEmpty) { - appLogger.info( - 'Migrating channel messages from legacy key $_keyPrefix to scoped key $keyFor', - ); - await prefs.setString(keyFor, legacyJsonString); - jsonString = legacyJsonString; - } + jsonString = await migrateLegacyStringPreference(_keyPrefix, keyFor); } if (jsonString == null || jsonString.isEmpty) { jsonString = prefs.getString(keyFor); diff --git a/lib/storage/contact_settings_store.dart b/lib/storage/contact_settings_store.dart index 49b1faa2..d6fe9596 100644 --- a/lib/storage/contact_settings_store.dart +++ b/lib/storage/contact_settings_store.dart @@ -29,15 +29,16 @@ class ContactSettingsStore { if (enabled == null && canMigrateLegacyKeys(publicKeyHex)) { // Attempt migration from legacy unscoped key on first load enabled = prefs.getBool(oldKey); - prefs.remove(oldKey); if (enabled != null) { appLogger.info( 'Migrating contact settings from legacy key $oldKey to scoped key $key', ); - await prefs.setBool(key, enabled); + if (await prefs.setBool(key, enabled)) { + await prefs.remove(oldKey); + } } } - return prefs.getBool(key) ?? false; + return enabled ?? false; } Future saveSmazEnabled(String contactKeyHex, bool enabled) async { diff --git a/lib/storage/contact_store.dart b/lib/storage/contact_store.dart index 3fc45187..5c9b2d2e 100644 --- a/lib/storage/contact_store.dart +++ b/lib/storage/contact_store.dart @@ -24,16 +24,7 @@ class ContactStore { String? jsonString = prefs.getString(keyFor); if ((jsonString == null || jsonString.isEmpty) && canMigrateLegacyKeys(publicKeyHex)) { - // Attempt migration from legacy unscoped key on first load - final legacyJsonString = prefs.getString(_keyPrefix); - prefs.remove(_keyPrefix); - if (legacyJsonString != null && legacyJsonString.isNotEmpty) { - appLogger.info( - 'Migrating contacts from legacy key $_keyPrefix to scoped key $keyFor', - ); - await prefs.setString(keyFor, legacyJsonString); - jsonString = legacyJsonString; - } + jsonString = await migrateLegacyStringPreference(_keyPrefix, keyFor); } if (jsonString == null || jsonString.isEmpty) { jsonString = prefs.getString(keyFor); diff --git a/lib/storage/history_archive.dart b/lib/storage/history_archive.dart new file mode 100644 index 00000000..46ae678b --- /dev/null +++ b/lib/storage/history_archive.dart @@ -0,0 +1,192 @@ +import 'dart:async'; +import 'dart:convert'; + +import '../utils/app_logger.dart'; +import 'history_store.dart'; + +typedef HistoryPage = ({List messages, int start, int end, int total}); + +/// Archive bookkeeping, ordered writes and paging shared by the direct and +/// channel message stores. Entries are JSON-encoded messages; the stored +/// prefix older than the in-memory window stays in the [HistoryStore]. +class HistoryArchive { + HistoryArchive({required this.mergeKey, required this.maxEntries}); + + /// Identifies a stored entry so an unreconciled save can drop overlaps. + final String Function(Map json) mergeKey; + final int maxEntries; + + final Expando> _archives = Expando(); + final Expando>> _writes = Expando(); + + _ArchiveState _stateFor(HistoryStore store, String key) => + (_archives[store] ??= {}).putIfAbsent(key, () => _ArchiveState()); + + Future _serialize( + HistoryStore store, + String key, + Future Function() operation, + ) { + final writes = _writes[store] ??= {}; + final previous = writes[key] ?? Future.value(); + final result = previous.then((_) => operation()); + final completion = result.then( + (_) {}, + onError: (Object _, StackTrace _) {}, + ); + writes[key] = completion; + unawaited( + completion.whenComplete(() { + if (identical(writes[key], completion)) writes.remove(key); + }), + ); + return result; + } + + /// Saves the [visible] window after the archived prefix. [readStored] + /// supplies the stored JSON when the prefix is not reconciled yet; it + /// defaults to the value under [key]. + Future save( + String key, + List visible, { + Future Function(HistoryStore store)? readStored, + }) { + final store = HistoryStore.instance; + final state = _stateFor(store, key); + final pendingBefore = state.nextId; + return _serialize(store, key, () async { + final pending = state.pending + .where((entry) => entry.$1 < pendingBefore) + .toList(); + final tail = [...pending.map((entry) => entry.$2), ...visible]; + final room = (maxEntries - visible.length).clamp(0, maxEntries); + var preserved = state.persistedCount; + if (preserved == null) { + final raw = await (readStored?.call(store) ?? store.getString(key)); + final prefix = _unreconciledPrefix(key, raw, tail); + await store.replaceTail(key, 0, [ + ...prefix, + ...tail, + ], maxEntries: maxEntries); + preserved = (prefix.length + pending.length).clamp(0, room); + } else { + final remaining = await store.replaceTail( + key, + preserved, + tail, + maxEntries: maxEntries, + ); + preserved = remaining + pending.length.clamp(0, room); + } + state.persistedCount = preserved; + state.pending.removeWhere((entry) => entry.$1 < pendingBefore); + }); + } + + /// Stored entries not repeated in [tail]. Unreadable stored history counts + /// as empty, so it cannot block every later save. + List _unreconciledPrefix(String key, String? raw, List tail) { + if (raw == null || raw.isEmpty) return []; + final Object? stored; + try { + stored = jsonDecode(raw); + } on FormatException catch (error) { + appLogger.warn('Saving over unreadable history $key: $error'); + return []; + } + if (stored is! List) { + appLogger.warn('Saving over history $key that is not a list'); + return []; + } + final overlaps = {}; + for (final entry in tail) { + final id = mergeKey(jsonDecode(entry) as Map); + overlaps[id] = (overlaps[id] ?? 0) + 1; + } + final kept = []; + for (final entry in stored.reversed) { + final id = entry is Map ? mergeKey(entry) : null; + final remaining = id == null ? 0 : overlaps[id] ?? 0; + if (remaining > 0) { + overlaps[id!] = remaining - 1; + } else { + kept.add(jsonEncode(entry)); + } + } + return kept.reversed.toList(); + } + + void setArchivedCount(String key, int count) { + RangeError.checkNotNegative(count, 'count'); + (_archives[HistoryStore.instance] ??= {})[key] = _ArchiveState(count); + } + + /// Only entries trimmed since the last successful save need buffering. + void archive(String key, Iterable entries) { + final state = _stateFor(HistoryStore.instance, key); + for (final entry in entries) { + state.pending.add((state.nextId++, entry)); + } + } + + int? archivedCount(String key) { + final state = _archives[HistoryStore.instance]?[key]; + final count = state?.persistedCount; + return count == null ? null : count + state!.pending.length; + } + + void forget([String? key]) { + final states = _archives[HistoryStore.instance]; + if (key == null) { + states?.clear(); + } else { + states?.remove(key); + } + } + + Future clear(HistoryStore store, String key) { + final state = _ArchiveState(0); + (_archives[store] ??= {})[key] = state; + return _serialize(store, key, () async { + if (!await store.remove(key)) { + state.persistedCount = null; + throw StateError('Could not clear message history'); + } + }); + } + + /// Reads a page backwards using physical stored-row offsets, so entries + /// [decode] skips do not make earlier valid history unreachable. + Future> readPage( + String key, + int total, + List Function(List entries) decode, { + int? before, + required int limit, + }) async { + final store = HistoryStore.instance; + final end = (before ?? total).clamp(0, total); + var start = end; + var messages = []; + while (start > 0 && messages.length < limit) { + final batchSize = (limit - messages.length).clamp(0, start); + final next = start - batchSize; + final entries = await store.readEntries( + key, + offset: next, + limit: batchSize, + ); + messages = [...decode(entries), ...messages]; + start = next; + } + return (messages: messages, start: start, end: end, total: total); + } +} + +class _ArchiveState { + _ArchiveState([this.persistedCount]); + + int? persistedCount; + int nextId = 0; + final List<(int, String)> pending = []; +} diff --git a/lib/storage/history_store.dart b/lib/storage/history_store.dart new file mode 100644 index 00000000..0bb25779 --- /dev/null +++ b/lib/storage/history_store.dart @@ -0,0 +1,149 @@ +import 'dart:convert'; + +import 'package:flutter/foundation.dart'; +import 'package:shared_preferences/shared_preferences.dart'; + +import '../utils/app_logger.dart'; +import 'history_store_stub.dart' + if (dart.library.io) 'history_store_io.dart' + as platform; +import 'prefs_manager.dart'; + +/// Histories use an indexed native store; preferences remain the web backend. +abstract class HistoryStore { + static HistoryStore _instance = PreferencesHistoryStore(); + static HistoryStore get instance => _instance; + + static bool isHistoryKey(String key) => + key.startsWith('messages_') || + key.startsWith('channel_messages_') || + key.startsWith('path_history_'); + + static bool isSequenceKey(String key) => + key.startsWith('messages_') || key.startsWith('channel_messages_'); + + /// Set while a run had to keep histories in preferences because the + /// database could not be opened; the next launch that opens it merges them. + static const String fallbackRunKey = 'history_fallback_run'; + + /// Never throws: a database that cannot be opened must not keep the app + /// from starting, so this run keeps histories in preferences instead. + static Future initialize() async { + try { + _instance = await platform.createHistoryStore(); + } catch (error) { + // The app logger is not set up this early. + debugPrint('History database unavailable: $error'); + _instance = PreferencesHistoryStore(); + await PrefsManager.instance.setBool(fallbackRunKey, true); + } + } + + static void use(HistoryStore store) => _instance = store; + static void reset() => _instance = PreferencesHistoryStore(); + + Future getString(String key); + Future setString(String key, String value); + Future remove(String key); + Future> getKeys(); + Future entryCount(String key); + Future> readEntries(String key, {int offset = 0, int? limit}); + + /// Replaces the visible tail, preserving the first [preservedCount] entries. + /// Returns the number of preserved entries remaining after the storage cap. + Future replaceTail( + String key, + int preservedCount, + List entries, { + int maxEntries = 1000, + }); + + Future close() async {} +} + +class PreferencesHistoryStore extends HistoryStore { + SharedPreferences? _preferences; + SharedPreferences get _prefs => _preferences ??= PrefsManager.instance; + @override + Future getString(String key) async => _prefs.getString(key); + + @override + Future setString(String key, String value) => + _prefs.setString(key, value); + + @override + Future remove(String key) async { + final prefs = _prefs; + return !prefs.containsKey(key) || await prefs.remove(key); + } + + @override + Future> getKeys() async => + _prefs.getKeys().where(HistoryStore.isHistoryKey).toSet(); + + /// Encoded entries of recently used histories, so paging and saves do not + /// re-parse the blob or re-encode the preserved prefix each time. An entry + /// is used only while the stored string is the one it was split from. + static const int _maxDecoded = 8; + final Map)> _decoded = {}; + + Future> _entries(String key) async { + final raw = await getString(key); + if (raw == null || raw.isEmpty) return const []; + final cached = _decoded[key]; + if (cached != null && cached.$1 == raw) return cached.$2; + final decoded = jsonDecode(raw); + if (decoded is! List) throw const FormatException('Expected message array'); + return _remember(key, raw, decoded.map(jsonEncode).toList()); + } + + List _remember(String key, String raw, List entries) { + _decoded.remove(key); + _decoded[key] = (raw, entries); + if (_decoded.length > _maxDecoded) _decoded.remove(_decoded.keys.first); + return entries; + } + + @override + Future> readEntries( + String key, { + int offset = 0, + int? limit, + }) async { + RangeError.checkNotNegative(offset, 'offset'); + if (limit != null) RangeError.checkNotNegative(limit, 'limit'); + final selected = (await _entries(key)).skip(offset); + return (limit == null ? selected : selected.take(limit)).toList(); + } + + @override + Future entryCount(String key) async => (await _entries(key)).length; + + @override + Future replaceTail( + String key, + int preservedCount, + List entries, { + int maxEntries = 1000, + }) async { + RangeError.checkNotNegative(preservedCount, 'preservedCount'); + RangeError.checkNotNegative(maxEntries, 'maxEntries'); + List previous; + try { + previous = (await _entries(key)).take(preservedCount).toList(); + } on FormatException catch (error) { + // Preferences hold one value per key, so the save replaces it. + appLogger.warn('Replacing unreadable history $key: $error'); + previous = []; + } + final combined = [...previous, ...entries]; + final overflow = (combined.length - maxEntries).clamp(0, combined.length); + final kept = combined.sublist(overflow); + final raw = '[${kept.join(',')}]'; + if (!await setString(key, raw)) { + throw StateError('Could not save message history'); + } + _remember(key, raw, kept); + return (previous.length - overflow).clamp(0, previous.length); + } +} diff --git a/lib/storage/history_store_io.dart b/lib/storage/history_store_io.dart new file mode 100644 index 00000000..669e2412 --- /dev/null +++ b/lib/storage/history_store_io.dart @@ -0,0 +1,356 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter/foundation.dart'; +import 'package:path_provider/path_provider.dart'; +import 'package:shared_preferences_platform_interface/shared_preferences_platform_interface.dart'; +import 'package:shared_preferences_platform_interface/types.dart'; +import 'package:sqflite/sqflite.dart' as sqlite; +import 'package:sqflite_common_ffi/sqflite_ffi.dart' as ffi; + +import '../utils/app_logger.dart'; +import 'history_store.dart'; +import 'prefs_manager.dart'; + +Future createHistoryStore() async { + final directory = await getApplicationSupportDirectory(); + if (Platform.isLinux || Platform.isWindows) ffi.sqfliteFfiInit(); + final store = await SqliteHistoryStore.open( + '${directory.path}${Platform.pathSeparator}history_v1.db', + factory: Platform.isLinux || Platform.isWindows + ? ffi.databaseFactoryFfi + : sqlite.databaseFactory, + ); + // The database is usable even when an import step fails: reads fall back + // to preferences for keys it does not hold, and the step repeats next launch. + try { + final prefs = PrefsManager.instance; + if (prefs.getBool(HistoryStore.fallbackRunKey) == true) { + await store.adoptFallbackHistories(); + await prefs.remove(HistoryStore.fallbackRunKey); + } + await store.migratePreferences(); + } catch (error) { + // The app logger is not set up this early. + debugPrint('History import incomplete: $error'); + } + return store; +} + +class SqliteHistoryStore extends HistoryStore { + SqliteHistoryStore._(this.database); + + static const schemaVersion = 1; + final sqlite.Database database; + + static Future open( + String path, { + required sqlite.DatabaseFactory factory, + }) async { + final database = await factory.openDatabase( + path, + options: sqlite.OpenDatabaseOptions( + version: schemaVersion, + onConfigure: (db) => db.execute('PRAGMA foreign_keys = ON'), + onCreate: (db, version) async { + await db.execute( + 'CREATE TABLE history_keys (key TEXT PRIMARY KEY, value TEXT)', + ); + await db.execute( + 'CREATE TABLE history_entries (' + 'key TEXT NOT NULL REFERENCES history_keys(key) ON DELETE CASCADE, ' + 'position INTEGER NOT NULL, value TEXT NOT NULL, ' + 'PRIMARY KEY (key, position))', + ); + await db.execute( + 'CREATE TABLE history_metadata (key TEXT PRIMARY KEY, value INTEGER NOT NULL)', + ); + }, + ), + ); + return SqliteHistoryStore._(database); + } + + Future migratePreferences() async { + final versions = await database.query( + 'history_metadata', + where: 'key = ?', + whereArgs: ['preferences_import_version'], + ); + final version = versions.isEmpty ? null : versions.first['value'] as int; + if (version != null && version >= schemaVersion) return; + + final prefs = PrefsManager.instance; + final imported = {}; + for (final key in prefs.getKeys().where(HistoryStore.isHistoryKey)) { + final value = prefs.get(key); + if (value is! String || value.isEmpty) continue; + try { + final decoded = jsonDecode(value); + if (HistoryStore.isSequenceKey(key) + ? decoded is! List + : decoded is! Map) { + continue; + } + } on FormatException { + appLogger.warn('Leaving unreadable history $key in preferences'); + continue; + } + // Rows from an interrupted earlier import may have been saved to since. + if ((await _header(key)).isEmpty) await setString(key, value); + imported.add(key); + } + if (imported.isNotEmpty) { + final removed = await SharedPreferencesStorePlatform.instance + .clearWithParameters( + ClearParameters( + filter: PreferencesFilter( + prefix: 'flutter.', + allowList: imported.map((key) => 'flutter.$key').toSet(), + ), + ), + ); + if (!removed) throw StateError('Could not finish history migration'); + await prefs.reload(); + } + await database.insert('history_metadata', { + 'key': 'preferences_import_version', + 'value': schemaVersion, + }, conflictAlgorithm: sqlite.ConflictAlgorithm.replace); + } + + /// Merges histories that a run without the database saved to preferences. + /// That run could not see the database rows, so its message lists hold + /// what it saved itself and are appended; a message it rewrote replaces the + /// stored copy. Route histories already in the database are kept. + Future adoptFallbackHistories() async { + final prefs = PrefsManager.instance; + final keys = prefs.getKeys().where(HistoryStore.isHistoryKey).toList(); + for (final key in keys) { + final value = prefs.get(key); + if (value is! String || value.isEmpty) continue; + final sequence = HistoryStore.isSequenceKey(key); + final Object? decoded; + try { + decoded = jsonDecode(value); + } on FormatException { + continue; + } + if (sequence ? decoded is! List : decoded is! Map) continue; + if ((await _header(key)).isEmpty) { + await setString(key, value); + } else if (sequence) { + final merged = await readEntries(key); + final positions = { + for (var i = 0; i < merged.length; i++) _entryId(merged[i]): i, + }; + for (final entry in (decoded as List).map(jsonEncode)) { + final id = _entryId(entry); + final position = positions[id]; + if (position == null) { + positions[id] = merged.length; + merged.add(entry); + } else { + merged[position] = entry; + } + } + await replaceTail(key, 0, merged); + } + if (!await prefs.remove(key)) { + throw StateError('Could not finish merging fallback histories'); + } + } + } + + static String _entryId(String entry) { + final decoded = jsonDecode(entry); + final id = decoded is Map ? decoded['messageId'] : null; + return id is String && id.isNotEmpty ? 'id:$id' : entry; + } + + Future>> _header(String key) => database.query( + 'history_keys', + columns: ['value'], + where: 'key = ?', + whereArgs: [key], + ); + + @override + Future getString(String key) async { + final header = await _header(key); + if (header.isEmpty) return PrefsManager.instance.getString(key); + final scalar = header.first['value']; + if (scalar is String) return scalar; + return '[${(await readEntries(key)).join(',')}]'; + } + + @override + Future setString(String key, String value) async { + final entries = HistoryStore.isSequenceKey(key) + ? (jsonDecode(value) as List).map(jsonEncode).toList() + : null; + await database.transaction((txn) async { + await txn.insert('history_keys', { + 'key': key, + 'value': entries == null ? value : null, + }, conflictAlgorithm: sqlite.ConflictAlgorithm.replace); + if (entries != null) { + final batch = txn.batch(); + for (var index = 0; index < entries.length; index++) { + batch.insert('history_entries', { + 'key': key, + 'position': index, + 'value': entries[index], + }); + } + await batch.commit(noResult: true); + } + }); + return true; + } + + @override + Future remove(String key) async { + await database.delete('history_keys', where: 'key = ?', whereArgs: [key]); + final prefs = PrefsManager.instance; + return !prefs.containsKey(key) || await prefs.remove(key); + } + + @override + Future> getKeys() async => { + for (final row in await database.query('history_keys', columns: ['key'])) + row['key'] as String, + ...PrefsManager.instance.getKeys().where(HistoryStore.isHistoryKey), + }; + + @override + Future entryCount(String key) async { + if ((await _header(key)).isEmpty) { + return PreferencesHistoryStore().entryCount(key); + } + return sqlite.Sqflite.firstIntValue( + await database.rawQuery( + 'SELECT COUNT(*) FROM history_entries WHERE key = ?', + [key], + ), + ) ?? + 0; + } + + @override + Future> readEntries( + String key, { + int offset = 0, + int? limit, + }) async { + RangeError.checkNotNegative(offset, 'offset'); + if (limit != null) RangeError.checkNotNegative(limit, 'limit'); + if ((await _header(key)).isEmpty) { + return PreferencesHistoryStore().readEntries( + key, + offset: offset, + limit: limit, + ); + } + final rows = await database.rawQuery( + 'SELECT value FROM history_entries WHERE key = ? ORDER BY position ' + 'LIMIT ? OFFSET ?', + [key, limit ?? -1, offset], + ); + return [for (final row in rows) row['value'] as String]; + } + + @override + Future replaceTail( + String key, + int preservedCount, + List entries, { + int maxEntries = 1000, + }) async { + RangeError.checkNotNegative(preservedCount, 'preservedCount'); + RangeError.checkNotNegative(maxEntries, 'maxEntries'); + // A readable preference left by the import is adopted first. An + // unreadable one stays in preferences, where a save never erases it; the + // database rows written below are what later reads return. + if ((await _header(key)).isEmpty) { + final legacy = PrefsManager.instance.getString(key); + if (legacy != null && legacy.isNotEmpty) { + if (_isJsonList(legacy)) { + await setString(key, legacy); + } else { + appLogger.warn('Saving $key beside its unreadable preference'); + } + } + } + return database.transaction((txn) async { + await txn.rawInsert( + 'INSERT OR IGNORE INTO history_keys (key, value) VALUES (?, NULL)', + [key], + ); + final prefix = preservedCount == 0 + ? >[] + : await txn.rawQuery( + 'SELECT position FROM history_entries WHERE key = ? ' + 'ORDER BY position LIMIT 1 OFFSET ?', + [key, preservedCount - 1], + ); + final count = + sqlite.Sqflite.firstIntValue( + await txn.rawQuery( + 'SELECT COUNT(*) FROM history_entries WHERE key = ?', + [key], + ), + ) ?? + 0; + final preserved = preservedCount.clamp(0, count); + final lastPosition = prefix.isNotEmpty + ? prefix.first['position'] as int + : preserved == 0 + ? -1 + : sqlite.Sqflite.firstIntValue( + await txn.rawQuery( + 'SELECT MAX(position) FROM history_entries WHERE key = ?', + [key], + ), + )!; + await txn.delete( + 'history_entries', + where: 'key = ? AND position > ?', + whereArgs: [key, lastPosition], + ); + final batch = txn.batch(); + for (var index = 0; index < entries.length; index++) { + batch.insert('history_entries', { + 'key': key, + 'position': lastPosition + index + 1, + 'value': entries[index], + }); + } + await batch.commit(noResult: true); + final overflow = (preserved + entries.length - maxEntries).clamp( + 0, + preserved + entries.length, + ); + if (overflow > 0) { + await txn.rawDelete( + 'DELETE FROM history_entries WHERE key = ? AND position IN ' + '(SELECT position FROM history_entries WHERE key = ? ' + 'ORDER BY position LIMIT ?)', + [key, key, overflow], + ); + } + return (preserved - overflow).clamp(0, preserved); + }); + } + + static bool _isJsonList(String value) { + try { + return jsonDecode(value) is List; + } on FormatException { + return false; + } + } + + @override + Future close() => database.close(); +} diff --git a/lib/storage/history_store_stub.dart b/lib/storage/history_store_stub.dart new file mode 100644 index 00000000..91008033 --- /dev/null +++ b/lib/storage/history_store_stub.dart @@ -0,0 +1,3 @@ +import 'history_store.dart'; + +Future createHistoryStore() async => PreferencesHistoryStore(); diff --git a/lib/storage/legacy_key_migration.dart b/lib/storage/legacy_key_migration.dart index e66af47f..ae157683 100644 --- a/lib/storage/legacy_key_migration.dart +++ b/lib/storage/legacy_key_migration.dart @@ -1,3 +1,5 @@ +import 'prefs_manager.dart'; + /// Storage scopes (first 10 hex chars of a radio's public key) that must /// never adopt data from the legacy unscoped keys. /// @@ -8,3 +10,16 @@ final Set scopesWithoutLegacyMigration = {}; bool canMigrateLegacyKeys(String scopeHex) => !scopesWithoutLegacyMigration.contains(scopeHex); + +Future migrateLegacyStringPreference( + String oldKey, + String newKey, +) async { + final prefs = PrefsManager.instance; + final value = prefs.getString(oldKey); + if (value == null || value.isEmpty) return value; + if (await prefs.setString(newKey, value)) { + await prefs.remove(oldKey); + } + return value; +} diff --git a/lib/storage/message_store.dart b/lib/storage/message_store.dart index b98142d3..ba2bb2c2 100644 --- a/lib/storage/message_store.dart +++ b/lib/storage/message_store.dart @@ -1,14 +1,13 @@ import 'dart:convert'; import 'dart:typed_data'; -import 'package:shared_preferences/shared_preferences.dart'; - import '../models/message.dart'; import '../models/translation_support.dart'; import '../helpers/smaz.dart'; import '../utils/app_logger.dart'; import 'legacy_key_migration.dart'; -import 'prefs_manager.dart'; +import 'history_archive.dart'; +import 'history_store.dart'; class MessageStore { static const String _keyPrefix = 'messages_'; @@ -23,102 +22,41 @@ class MessageStore { /// beyond it are dropped on save. static const int maxStoredMessages = 1000; - /// Encoded entries persisted ahead of the connector's in-memory window, - /// keyed by scoped storage key. A missing entry means the window has not - /// been reconciled with storage yet, so the next save keeps what is already - /// stored instead of replacing it. Shared across instances so a store - /// created by a screen to clear a key also resets the connector's view. - static final Map> _archived = {}; + static final HistoryArchive _archive = HistoryArchive( + mergeKey: _storedMergeKey, + maxEntries: maxStoredMessages, + ); - /// Persists [messages] (the in-memory window, oldest first) after the - /// archived entries that sit ahead of it in storage. - Future saveMessages( - String contactKeyHex, - List messages, - ) async { + Future saveMessages(String contactKeyHex, List messages) { if (publicKeyHex.isEmpty) { appLogger.warn('Public key hex is not set. Cannot save messages.'); - return; + return Future.value(); } - final prefs = PrefsManager.instance; final key = '$keyFor$contactKeyHex'; - final visibleJson = messages.map(_messageToJson).toList(); - final archived = - _archived[key] ?? - _unreconciledArchive(_readStored(prefs, contactKeyHex), visibleJson); - final overflow = archived.length + visibleJson.length - maxStoredMessages; - if (overflow > 0) { - archived.removeRange(0, overflow.clamp(0, archived.length)); - } - _archived[key] = archived; - final entries = [...archived, ...visibleJson.map(jsonEncode)]; - await prefs.setString(key, '[${entries.join(',')}]'); - } - - /// Records [olderMessages] as the stored entries that precede the - /// in-memory window for [contactKeyHex]. - void setArchived(String contactKeyHex, List olderMessages) { - _archived['$keyFor$contactKeyHex'] = olderMessages - .map((message) => jsonEncode(_messageToJson(message))) - .toList(); - } - - /// Moves [messages], just trimmed from the front of the in-memory window, - /// into the archive so the next save keeps them. - void archive(String contactKeyHex, List messages) { - final archived = _archived['$keyFor$contactKeyHex']; - // Unreconciled saves keep everything already stored anyway. - if (archived == null) return; - archived.addAll( - messages.map((message) => jsonEncode(_messageToJson(message))), + final scope = publicKeyHex; + return _archive.save( + key, + [for (final message in messages) jsonEncode(_messageToJson(message))], + readStored: (store) => + _readStored(store, key, '$_keyPrefix$contactKeyHex', scope), ); } - /// Number of stored entries ahead of the in-memory window, or null when - /// the window has not been reconciled with storage. + /// The stored prefix stays in the database, rather than another JSON cache. + void setArchivedCount(String contactKeyHex, int count) => + _archive.setArchivedCount('$keyFor$contactKeyHex', count); + + void archive(String contactKeyHex, List messages) => + _archive.archive( + '$keyFor$contactKeyHex', + messages.map((message) => jsonEncode(_messageToJson(message))), + ); + int? archivedCount(String contactKeyHex) => - _archived['$keyFor$contactKeyHex']?.length; + _archive.archivedCount('$keyFor$contactKeyHex'); - /// Removes the newest [count] archived entries after they were paged back - /// into the in-memory window. - void unarchiveNewest(String contactKeyHex, int count) { - final archived = _archived['$keyFor$contactKeyHex']; - if (archived == null || count <= 0) return; - archived.removeRange( - (archived.length - count).clamp(0, archived.length), - archived.length, - ); - } - - /// Drops the archive bookkeeping, for one conversation or all of them, - /// when the matching in-memory window is discarded. - void forgetArchive([String? contactKeyHex]) { - if (contactKeyHex == null) { - _archived.clear(); - } else { - _archived.remove('$keyFor$contactKeyHex'); - } - } - - List _unreconciledArchive( - String? storedJson, - List> visibleJson, - ) { - if (storedJson == null || storedJson.isEmpty) return []; - final List stored; - try { - stored = jsonDecode(storedJson) as List; - } catch (_) { - return []; - } - final visibleKeys = visibleJson.map(_storedMergeKey).toSet(); - return [ - for (final entry in stored) - if (entry is Map && - !visibleKeys.contains(_storedMergeKey(entry))) - jsonEncode(entry), - ]; - } + void forgetArchive([String? contactKeyHex]) => + _archive.forget(contactKeyHex == null ? null : '$keyFor$contactKeyHex'); static String _storedMergeKey(Map json) { final messageId = json['messageId']; @@ -126,69 +64,120 @@ class MessageStore { return 'fallback:${json['timestamp']}:${json['isOutgoing']}:${json['text']}'; } - /// Reads the stored JSON for [contactKeyHex], migrating the legacy unscoped - /// key on first access. - String? _readStored(SharedPreferences prefs, String contactKeyHex) { - final key = '$keyFor$contactKeyHex'; - final oldKey = '$_keyPrefix$contactKeyHex'; - final jsonString = prefs.getString(key); - if ((jsonString == null || jsonString.isEmpty) && - canMigrateLegacyKeys(publicKeyHex)) { - // Attempt migration from legacy unscoped key on first load - final legacyJsonString = prefs.getString(oldKey); - prefs.remove(oldKey); - if (legacyJsonString != null && legacyJsonString.isNotEmpty) { - appLogger.info( - 'Migrating messages from legacy key $oldKey to scoped key $key', - ); - prefs.setString(key, legacyJsonString); - return legacyJsonString; + Future _readStored( + HistoryStore store, + String key, + String oldKey, + String scope, + ) async { + final raw = await store.getString(key); + if ((raw == null || raw.isEmpty) && canMigrateLegacyKeys(scope)) { + final legacy = await store.getString(oldKey); + if (legacy != null && legacy.isNotEmpty) { + if (await store.setString(key, legacy)) await store.remove(oldKey); } + return legacy; } - return jsonString; + return raw; } - Future> loadMessages(String contactKeyHex) async { - if (publicKeyHex.isEmpty) { - appLogger.warn('Public key hex is not set. Cannot load messages.'); - return []; + Future<({String key, int count})> _storedHistory(String contactKeyHex) async { + final store = HistoryStore.instance; + final scope = publicKeyHex; + final key = '$keyFor$contactKeyHex'; + var count = await store.entryCount(key); + if (count > 0) return (key: key, count: count); + final raw = await store.getString(key); + if ((raw == null || raw.isEmpty) && canMigrateLegacyKeys(scope)) { + final oldKey = '$_keyPrefix$contactKeyHex'; + final legacy = await store.getString(oldKey); + if (legacy != null && legacy.isNotEmpty) { + if (await store.setString(key, legacy)) { + await store.remove(oldKey); + count = await store.entryCount(key); + } else { + throw StateError('Could not migrate stored message history'); + } + } } - final prefs = PrefsManager.instance; - String? jsonString = _readStored(prefs, contactKeyHex); - if (jsonString == null || jsonString.isEmpty) { - jsonString = prefs.getString(keyFor); - } - if (jsonString == null || jsonString.isEmpty) { - return []; + if (raw == null && count == 0) { + final fallback = '$_keyPrefix$scope'; + final fallbackCount = await store.entryCount(fallback); + if (fallbackCount > 0) { + final fallbackJson = await store.getString(fallback); + if (fallbackJson == null || !await store.setString(key, fallbackJson)) { + throw StateError('Could not migrate fallback message history'); + } + return (key: key, count: fallbackCount); + } } + return (key: key, count: count); + } - final List jsonList; + Future> loadMessages( + String contactKeyHex, { + int offset = 0, + int? limit, + }) async { + if (publicKeyHex.isEmpty) return []; try { - jsonList = jsonDecode(jsonString) as List; - } catch (e) { - appLogger.warn('Stored messages for $contactKeyHex are unreadable: $e'); + final history = await _storedHistory(contactKeyHex); + final entries = await HistoryStore.instance.readEntries( + history.key, + offset: offset, + limit: limit, + ); + return _decodeMessages(entries); + } catch (error) { + appLogger.warn( + 'Stored messages for $contactKeyHex are unreadable: $error', + ); return []; } + } + + /// Reads a page backwards using physical stored-row offsets. Malformed + /// records are skipped without making earlier valid history unreachable. + Future<({List messages, int start, int end, int total})> + loadMessagePage(String contactKeyHex, {int? before, int limit = 50}) async { + RangeError.checkNotNegative(limit, 'limit'); + if (publicKeyHex.isEmpty) { + return (messages: [], start: 0, end: 0, total: 0); + } + try { + final history = await _storedHistory(contactKeyHex); + return await _archive.readPage( + history.key, + history.count, + _decodeMessages, + before: before, + limit: limit, + ); + } on FormatException catch (error) { + appLogger.warn( + 'Stored messages for $contactKeyHex are unreadable: $error', + ); + return (messages: [], start: 0, end: 0, total: 0); + } + } + + List _decodeMessages(List entries) { final messages = []; - for (final json in jsonList) { + for (final entry in entries) { try { - messages.add(_messageFromJson(json as Map)); - } catch (e) { - appLogger.warn('Skipping malformed stored message: $e'); + messages.add( + _messageFromJson(jsonDecode(entry) as Map), + ); + } catch (error) { + appLogger.warn('Skipping malformed stored message: $error'); } } return messages; } - Future clearMessages(String contactKeyHex) async { - if (publicKeyHex.isEmpty) { - appLogger.warn('Public key hex is not set. Cannot clear messages.'); - return; - } - final prefs = PrefsManager.instance; - final key = '$keyFor$contactKeyHex'; - _archived[key] = []; - await prefs.remove(key); + Future clearMessages(String contactKeyHex) { + if (publicKeyHex.isEmpty) return Future.value(); + return _archive.clear(HistoryStore.instance, '$keyFor$contactKeyHex'); } Map _messageToJson(Message msg) { diff --git a/lib/storage/prefs_manager.dart b/lib/storage/prefs_manager.dart index 2bf6299c..78880018 100644 --- a/lib/storage/prefs_manager.dart +++ b/lib/storage/prefs_manager.dart @@ -1,5 +1,7 @@ import 'package:shared_preferences/shared_preferences.dart'; +import 'history_store.dart'; + /// Singleton wrapper for SharedPreferences to avoid redundant getInstance() calls. /// /// BEFORE: Every storage operation called SharedPreferences.getInstance() @@ -30,5 +32,6 @@ class PrefsManager { /// For testing: reset the instance static void reset() { _instance = null; + HistoryStore.reset(); } } diff --git a/lib/storage/unread_store.dart b/lib/storage/unread_store.dart index 59492790..c6cce28a 100644 --- a/lib/storage/unread_store.dart +++ b/lib/storage/unread_store.dart @@ -36,16 +36,7 @@ class UnreadStore { String? jsonString = prefs.getString(keyFor); if ((jsonString == null || jsonString.isEmpty) && canMigrateLegacyKeys(publicKeyHex)) { - // Attempt migration from legacy unscoped key on first load - final legacyJsonString = prefs.getString(_keyPrefix); - prefs.remove(_keyPrefix); - if (legacyJsonString != null && legacyJsonString.isNotEmpty) { - appLogger.info( - 'Migrating channel messages from legacy key $_keyPrefix to scoped key $keyFor', - ); - await prefs.setString(keyFor, legacyJsonString); - jsonString = legacyJsonString; - } + jsonString = await migrateLegacyStringPreference(_keyPrefix, keyFor); } if (jsonString == null || jsonString.isEmpty) { jsonString = prefs.getString(keyFor); diff --git a/lib/utils/coalesced_writes.dart b/lib/utils/coalesced_writes.dart index 5b9a4e68..d70e34c6 100644 --- a/lib/utils/coalesced_writes.dart +++ b/lib/utils/coalesced_writes.dart @@ -15,6 +15,7 @@ class CoalescedWrites { final Duration delay; final Map Function()> _pending = {}; + final Set> _inFlight = {}; Timer? _timer; bool get hasPending => _pending.isNotEmpty; @@ -38,20 +39,20 @@ class CoalescedWrites { Future flush() { _timer?.cancel(); _timer = null; - if (_pending.isEmpty) return Future.value(); final writes = _pending.values.toList(); _pending.clear(); // Callers fire and forget, so a failed write is logged rather than left // as an unhandled async error, and never stops the other writes. - final futures = >[]; for (final write in writes) { try { - futures.add(write().catchError(_logFailure)); + final future = write().catchError(_logFailure); + _inFlight.add(future); + unawaited(future.whenComplete(() => _inFlight.remove(future))); } catch (error) { _logFailure(error); } } - return Future.wait(futures).then((_) {}); + return Future.wait(_inFlight.toList()).then((_) {}); } static void _logFailure(Object error) { diff --git a/pubspec.yaml b/pubspec.yaml index 3e3a288c..bb7c3f11 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -50,6 +50,9 @@ dependencies: ffi: ^2.1.3 # calloc.free for flserial's native I/O buffers provider: ^6.1.5+1 shared_preferences: ^2.2.2 + shared_preferences_platform_interface: ^2.4.2 + sqflite: ^2.4.5 + sqflite_common_ffi: ^2.4.0 uuid: ^4.3.3 flutter_map: ^8.3.2 latlong2: ^0.9.1 diff --git a/scripts/reset_http_profile.dart b/scripts/reset_http_profile.dart new file mode 100644 index 00000000..c4b6d234 --- /dev/null +++ b/scripts/reset_http_profile.dart @@ -0,0 +1,99 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +Future main(List arguments) async { + if (arguments.length != 1) { + stderr.writeln( + 'Usage: dart run scripts/reset_http_profile.dart ', + ); + exitCode = 64; + return; + } + try { + final remaining = await resetHttpProfile(Uri.parse(arguments.single)); + stdout.writeln('HTTP recording stopped; retained HTTP profile cleared.'); + if (remaining > 0) { + stdout.writeln( + '$remaining requests remain in a separate HTTP profiling provider.', + ); + } + } catch (error) { + stderr.writeln('Could not reset HTTP profiling: $error'); + exitCode = 1; + } +} + +/// Stops recording before clearing captured bodies; normal networking continues. +/// Returns records still reported by other HTTP profiling providers. +Future resetHttpProfile(Uri serviceUri) async { + final scheme = switch (serviceUri.scheme) { + 'http' || 'ws' => 'ws', + 'https' || 'wss' => 'wss', + _ => throw const FormatException( + 'Expected an HTTP or WebSocket service URI', + ), + }; + final path = serviceUri.path.endsWith('/ws') + ? serviceUri.path + : '${serviceUri.path.endsWith('/') ? serviceUri.path : '${serviceUri.path}/'}ws'; + final socket = await WebSocket.connect( + serviceUri.replace(scheme: scheme, path: path).toString(), + ).timeout(const Duration(seconds: 10)); + final messages = StreamIterator(socket); + var nextId = 0; + + Future> call( + String method, [ + Map parameters = const {}, + ]) async { + final id = '${++nextId}'; + socket.add( + jsonEncode({ + 'jsonrpc': '2.0', + 'id': id, + 'method': method, + 'params': parameters, + }), + ); + while (await messages.moveNext().timeout(const Duration(seconds: 10))) { + if (messages.current is! String) continue; + final response = jsonDecode(messages.current as String) as Map; + if (response['id'] != id) continue; + if (response['error'] case final Map error) { + throw StateError('$method: ${error['message']}'); + } + return Map.from(response['result'] as Map); + } + throw StateError('The Dart VM service disconnected'); + } + + try { + final vm = await call('getVM'); + final isolates = (vm['isolates'] as List).cast(); + final mainIsolates = isolates.where((isolate) => isolate['name'] == 'main'); + if (mainIsolates.length != 1) { + throw StateError('Expected one running main isolate'); + } + final parameters = { + 'isolateId': mainIsolates.single['id'] as String, + }; + await call('ext.dart.io.httpEnableTimelineLogging', { + ...parameters, + 'enabled': 'false', + }); + await call('ext.dart.io.clearHttpProfile', parameters); + final state = await call( + 'ext.dart.io.httpEnableTimelineLogging', + parameters, + ); + if (state['enabled'] != false) { + throw StateError('HTTP recording was enabled again by another client'); + } + final profile = await call('ext.dart.io.getHttpProfile', parameters); + return (profile['requests'] as List).length; + } finally { + await messages.cancel(); + await socket.close(); + } +} diff --git a/test/connector/history_pagination_test.dart b/test/connector/history_pagination_test.dart new file mode 100644 index 00000000..0a9cc9a8 --- /dev/null +++ b/test/connector/history_pagination_test.dart @@ -0,0 +1,372 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:meshcore_open/connector/meshcore_connector.dart'; +import 'package:meshcore_open/connector/meshcore_protocol.dart'; +import 'package:meshcore_open/models/channel.dart'; +import 'package:meshcore_open/models/channel_message.dart'; +import 'package:meshcore_open/models/contact.dart'; +import 'package:meshcore_open/storage/channel_message_store.dart'; +import 'package:meshcore_open/storage/channel_store.dart'; +import 'package:meshcore_open/storage/contact_store.dart'; +import 'package:meshcore_open/storage/history_store.dart'; +import 'package:meshcore_open/storage/last_radio_store.dart'; +import 'package:meshcore_open/storage/message_store.dart'; +import 'package:meshcore_open/storage/prefs_manager.dart'; +import 'package:shared_preferences/shared_preferences.dart'; + +const _radioA = + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; +const _radioB = + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'; +final _contactKey = Uint8List.fromList(List.generate(32, (i) => i + 1)); +final _contactHex = pubKeyToHex(_contactKey); +final _channel = Channel( + index: 0, + name: 'Test', + psk: Uint8List.fromList(List.filled(16, 1)), +); + +String _dmKey(String radio) => 'messages_${radio.substring(0, 10)}$_contactHex'; +String _channelKey(String radio) => + 'channel_messages_${radio.substring(0, 10)}_${_channel.storageId}'; + +Map _dm(int i, {String prefix = 'dm'}) => { + 'senderKey': base64Encode(_contactKey), + 'text': '$prefix-$i', + 'timestamp': 1000 * (i + 1), + 'isOutgoing': false, + 'status': 2, + 'messageId': '$prefix-$i', + 'plainText': true, +}; +Map _ch(int i, {String prefix = 'ch'}) => { + 'senderName': 'Bob', + 'text': '$prefix-$i', + 'timestamp': 1000 * (i + 1), + 'isOutgoing': false, + 'status': 1, + 'channelIndex': 0, + 'messageId': '$prefix-$i', + 'plainText': true, +}; + +class _TrackingHistory extends HistoryStore { + final data = {}; + final reads = <({String key, int offset, int? limit})>[]; + final fullReads = []; + String? delayedKey; + Completer? gate; + Completer? started; + String? failedKey; + + @override + Future getString(String key) async { + fullReads.add(key); + return data[key]; + } + + @override + Future setString(String key, String value) async { + data[key] = value; + return true; + } + + @override + Future remove(String key) async { + data.remove(key); + return true; + } + + @override + Future> getKeys() async => data.keys.toSet(); + + List _entries(String key) => + ((jsonDecode(data[key] ?? '[]')) as List).map(jsonEncode).toList(); + + @override + Future entryCount(String key) async => _entries(key).length; + + @override + Future> readEntries( + String key, { + int offset = 0, + int? limit, + }) async { + if (key == failedKey) throw StateError('History storage unavailable'); + reads.add((key: key, offset: offset, limit: limit)); + final selected = _entries(key).skip(offset); + final snapshot = (limit == null ? selected : selected.take(limit)).toList(); + if (key == delayedKey && gate != null) { + final wait = gate!; + gate = null; + started?.complete(); + await wait.future; + } + return snapshot; + } + + @override + Future replaceTail( + String key, + int preservedCount, + List entries, { + int maxEntries = 1000, + }) async { + final prefix = _entries(key).take(preservedCount).toList(); + final combined = [...prefix, ...entries]; + final overflow = (combined.length - maxEntries).clamp(0, combined.length); + data[key] = '[${combined.skip(overflow).join(',')}]'; + return (prefix.length - overflow).clamp(0, prefix.length); + } + + Completer delayNextRead(String key) { + delayedKey = key; + gate = Completer(); + started = Completer(); + return gate!; + } +} + +Future _seedRadio(String radio) async { + final contact = Contact( + publicKey: _contactKey, + name: 'Alice', + type: advTypeChat, + pathLength: 0, + path: Uint8List(0), + lastSeen: DateTime.fromMillisecondsSinceEpoch(1000), + ); + await (ContactStore()..setPublicKeyHex = radio).saveContacts([contact]); + final channels = ChannelStore()..setPublicKeyHex = radio; + await channels.saveChannels([_channel]); + await channels.saveSlotChannelIds({0: _channel.storageId!}); + await PrefsManager.instance.setBool( + 'channel_storage_v2_${radio.substring(0, 10)}', + true, + ); + await LastRadioStore().save( + LastRadio( + publicKeyHex: radio, + name: 'Saved', + transport: LastRadio.transportBle, + bleRemoteId: 'AA:BB:CC:DD:EE:FF', + savedAt: DateTime.fromMillisecondsSinceEpoch(1000), + ), + ); +} + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + late _TrackingHistory history; + late MeshCoreConnector connector; + + setUp(() async { + SharedPreferences.setMockInitialValues({}); + PrefsManager.reset(); + await PrefsManager.initialize(); + await _seedRadio(_radioA); + history = _TrackingHistory(); + history.data[_dmKey(_radioA)] = jsonEncode([ + for (var i = 0; i < 620; i++) _dm(i), + ]); + history.data[_channelKey(_radioA)] = jsonEncode([ + for (var i = 0; i < 620; i++) _ch(i), + ]); + HistoryStore.use(history); + MessageStore().forgetArchive(); + ChannelMessageStore().forgetArchive(); + connector = MeshCoreConnector(); + addTearDown(connector.dispose); + }); + + test( + 'initial loads and older pages read only the requested stored rows', + () async { + await connector.restoreLastRadio(); + final contact = connector.contacts.single; + final channel = connector.channels.single; + expect(connector.getMessages(contact).map((m) => m.text), [ + for (var i = 420; i < 620; i++) 'dm-$i', + ]); + expect(connector.getChannelMessages(channel).map((m) => m.text), [ + for (var i = 420; i < 620; i++) 'ch-$i', + ]); + expect( + history.reads, + containsAll([ + (key: _dmKey(_radioA), offset: 420, limit: 200), + (key: _channelKey(_radioA), offset: 420, limit: 200), + ]), + ); + expect( + history.fullReads.where( + (key) => key == _dmKey(_radioA) || key == _channelKey(_radioA), + ), + isEmpty, + ); + history.reads.clear(); + final dms = await connector.loadOlderMessages(_contactHex); + final channels = await connector.loadOlderChannelMessages(0); + expect(dms.map((m) => m.text), [for (var i = 370; i < 420; i++) 'dm-$i']); + expect(channels.map((m) => m.text), [ + for (var i = 370; i < 420; i++) 'ch-$i', + ]); + expect(history.reads, [ + (key: _dmKey(_radioA), offset: 370, limit: 50), + (key: _channelKey(_radioA), offset: 370, limit: 50), + ]); + final next = await connector.loadOlderMessages(_contactHex); + expect(next.map((m) => m.text), [ + for (var i = 320; i < 370; i++) 'dm-$i', + ]); + expect( + connector.getMessages(contact).map((m) => m.messageId).toSet(), + hasLength(300), + ); + final search = await connector.allMessagesForSearch(_contactHex); + expect(search.map((m) => m.text), [ + for (var i = 0; i < 620; i++) 'dm-$i', + ]); + }, + ); + + test( + 'older pages start before the window when the archive is unreconciled', + () async { + await connector.restoreLastRadio(); + MessageStore().forgetArchive(); + ChannelMessageStore().forgetArchive(); + final dms = await connector.loadOlderMessages(_contactHex); + final channels = await connector.loadOlderChannelMessages(0); + expect(dms.map((m) => m.text), [for (var i = 370; i < 420; i++) 'dm-$i']); + expect(channels.map((m) => m.text), [ + for (var i = 370; i < 420; i++) 'ch-$i', + ]); + expect( + connector.getMessages(connector.contacts.single).map((m) => m.text), + [for (var i = 370; i < 620; i++) 'dm-$i'], + ); + }, + ); + + test( + 'malformed rows consume physical offsets and do not hide older pages', + () async { + history.data[_dmKey(_radioA)] = jsonEncode([ + for (var i = 0; i < 620; i++) + if (i >= 400 && i < 600) {'text': 1} else _dm(i), + ]); + await connector.restoreLastRadio(); + expect(connector.getMessages(connector.contacts.single), hasLength(200)); + final older = await connector.loadOlderMessages(_contactHex); + expect(older.map((m) => m.text), [ + for (var i = 170; i < 220; i++) 'dm-$i', + ]); + final stored = await connector.allMessagesForSearch(_contactHex); + expect(stored.map((m) => m.text), [ + for (var i = 0; i < 620; i++) + if (i < 400 || i >= 600) 'dm-$i', + ]); + expect(history.reads.last, ( + key: _dmKey(_radioA), + offset: 0, + limit: 170, + ), reason: 'Search reads the remaining prefix before the loaded window'); + }, + ); + + test( + 'a storage read failure keeps the loaded window and archive cursor', + () async { + await connector.restoreLastRadio(); + final before = List.of(connector.getMessages(connector.contacts.single)); + final store = MessageStore()..setPublicKeyHex = _radioA; + expect(store.archivedCount(_contactHex), 420); + history.failedKey = _dmKey(_radioA); + + await expectLater( + connector.loadOlderMessages(_contactHex), + throwsStateError, + ); + expect(connector.getMessages(connector.contacts.single), before); + expect(store.archivedCount(_contactHex), 420); + history.failedKey = null; + expect(await connector.loadOlderMessages(_contactHex), hasLength(50)); + }, + ); + + test( + 'overlapping requests share a page rather than prepending it twice', + () async { + await connector.restoreLastRadio(); + history.reads.clear(); + final gate = history.delayNextRead(_dmKey(_radioA)); + final first = connector.loadOlderMessages(_contactHex); + await history.started!.future; + final second = connector.loadOlderMessages(_contactHex); + gate.complete(); + await Future.wait([first, second]); + expect( + history.reads.where((r) => r.key == _dmKey(_radioA)), + hasLength(1), + ); + expect(connector.getMessages(connector.contacts.single), hasLength(250)); + }, + ); + + test( + 'a delayed old-radio page cannot enter a newly restored radio', + () async { + await connector.restoreLastRadio(); + final gate = history.delayNextRead(_dmKey(_radioA)); + final oldPage = connector.loadOlderMessages(_contactHex); + await history.started!.future; + await _seedRadio(_radioB); + history.data[_dmKey(_radioB)] = jsonEncode([ + for (var i = 0; i < 10; i++) _dm(i, prefix: 'new'), + ]); + history.data[_channelKey(_radioB)] = '[]'; + await connector.restoreLastRadio(); + gate.complete(); + expect(await oldPage, isEmpty); + expect(connector.storageKeyHex, _radioB); + expect( + connector.getMessages(connector.contacts.single).map((m) => m.text), + [for (var i = 0; i < 10; i++) 'new-$i'], + ); + }, + ); + + test( + 'channel edits during an asynchronous reload survive the stale read', + () async { + await connector.restoreLastRadio(); + final channel = connector.channels.single; + final gate = history.delayNextRead(_channelKey(_radioA)); + final reload = connector.loadAllChannelMessages(); + await history.started!.future; + final current = connector.getChannelMessages(channel); + final removed = current.first; + current.add( + ChannelMessage( + senderName: 'Live', + text: 'arrived during read', + timestamp: DateTime.fromMillisecondsSinceEpoch(1000000), + isOutgoing: false, + status: ChannelMessageStatus.sent, + channelIndex: 0, + messageId: 'live', + ), + ); + await connector.deleteChannelMessage(removed); + gate.complete(); + await reload; + final messages = connector.getChannelMessages(channel); + expect(messages.any((m) => m.messageId == 'live'), isTrue); + expect(messages.any((m) => m.messageId == removed.messageId), isFalse); + }, + ); +} diff --git a/test/connector/remove_contact_test.dart b/test/connector/remove_contact_test.dart index 9abb7eaf..f7a0388d 100644 --- a/test/connector/remove_contact_test.dart +++ b/test/connector/remove_contact_test.dart @@ -78,6 +78,8 @@ void main() { expect(connector.contacts, isNotEmpty); expect(listed(), isFalse); + // Contact saves are batched; land them before reading the store. + await connector.flushPendingWrites(); final store = ContactStore()..setPublicKeyHex = connector.selfPublicKeyHex; final saved = await store.loadContacts(); expect(saved.any((c) => c.publicKeyHex == contact.publicKeyHex), isFalse); diff --git a/test/screens/map_camera_persistence_test.dart b/test/screens/map_camera_persistence_test.dart new file mode 100644 index 00000000..cc9fbe64 --- /dev/null +++ b/test/screens/map_camera_persistence_test.dart @@ -0,0 +1,160 @@ +import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_map/flutter_map.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:flutter_cache_manager/flutter_cache_manager.dart'; +import 'package:latlong2/latlong.dart'; +import 'package:meshcore_open/connector/meshcore_connector.dart'; +import 'package:meshcore_open/l10n/app_localizations.dart'; +import 'package:meshcore_open/screens/map_screen.dart'; +import 'package:meshcore_open/services/app_settings_service.dart'; +import 'package:meshcore_open/services/map_tile_cache_service.dart'; +import 'package:meshcore_open/services/path_history_service.dart'; +import 'package:meshcore_open/services/storage_service.dart'; +import 'package:meshcore_open/services/ui_view_state_service.dart'; +import 'package:meshcore_open/storage/prefs_manager.dart'; +import 'package:meshcore_open/theme/mesh_theme.dart'; +import 'package:provider/provider.dart'; +import 'package:shared_preferences/shared_preferences.dart'; + +const _cameraKey = 'flutter.map_last_camera'; +const _preferencesChannel = MethodChannel( + 'plugins.flutter.io/shared_preferences', +); + +class _UnusedCacheManager implements BaseCacheManager { + @override + dynamic noSuchMethod(Invocation invocation) => super.noSuchMethod(invocation); +} + +class _MapWithoutTiles extends MapTileCacheService { + _MapWithoutTiles(AppSettingsService settings) + : super(appSettingsService: settings, cacheManager: _UnusedCacheManager()); + + @override + Widget buildTileLayer(BuildContext context, {double opacity = 1}) => + const SizedBox.shrink(); +} + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + late Map persisted; + late List> cameraWrites; + + setUp(() async { + persisted = {}; + cameraWrites = []; + SharedPreferences.resetStatic(); + PrefsManager.reset(); + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger + .setMockMethodCallHandler(_preferencesChannel, (call) async { + if (call.method == 'getAll') return persisted; + final args = call.arguments as Map; + final key = args['key'] as String; + if (call.method == 'remove') { + persisted.remove(key); + } else { + persisted[key] = args['value'] as Object; + if (key == _cameraKey) { + cameraWrites.add(List.from(args['value'] as List)); + } + } + return true; + }); + await PrefsManager.initialize(); + }); + + Future openMap(WidgetTester tester) async { + final settings = AppSettingsService(); + final connector = MeshCoreConnector(); + final paths = PathHistoryService(StorageService()); + final tiles = _MapWithoutTiles(settings); + addTearDown(() { + connector.dispose(); + paths.dispose(); + tiles.dispose(); + settings.dispose(); + }); + await tester.pumpWidget( + MultiProvider( + providers: [ + ChangeNotifierProvider.value(value: connector), + ChangeNotifierProvider.value(value: settings), + ChangeNotifierProvider.value(value: paths), + ChangeNotifierProvider.value(value: UiViewStateService()), + ChangeNotifierProvider.value(value: tiles), + ], + child: MaterialApp( + theme: MeshTheme.light(), + localizationsDelegates: AppLocalizations.localizationsDelegates, + supportedLocales: AppLocalizations.supportedLocales, + home: const MapScreen(), + ), + ), + ); + await tester.pump(); + return tester.widget(find.byType(FlutterMap)).mapController!; + } + + testWidgets('camera movements persist once after interaction settles', ( + tester, + ) async { + final controller = await openMap(tester); + controller.move(const LatLng(35, -112), 9); + await tester.pump(const Duration(milliseconds: 300)); + controller.move(const LatLng(36, -113), 10); + await tester.pump(const Duration(milliseconds: 300)); + expect(cameraWrites, isEmpty); + await tester.pump(const Duration(milliseconds: 201)); + expect(cameraWrites, [ + ['36.0', '-113.0', '10.0'], + ]); + + // Duplicate end events and disposal must not rewrite an unchanged camera. + final map = tester.widget(find.byType(FlutterMap)); + map.options.onMapEvent!( + MapEventMoveEnd( + source: MapEventSource.mapController, + camera: controller.camera, + ), + ); + await tester.pump(const Duration(milliseconds: 600)); + await tester.pumpWidget(const SizedBox.shrink()); + await tester.pump(); + expect(cameraWrites, hasLength(1)); + }); + + testWidgets( + 'leaving the map flushes the latest camera and cancels its timer', + (tester) async { + final controller = await openMap(tester); + controller.move(const LatLng(37, -114), 11); + await tester.pump(const Duration(milliseconds: 100)); + expect(cameraWrites, isEmpty); + await tester.pumpWidget(const SizedBox.shrink()); + await tester.pump(); + expect(cameraWrites, [ + ['37.0', '-114.0', '11.0'], + ]); + await tester.pump(const Duration(seconds: 1)); + expect(cameraWrites, hasLength(1)); + }, + ); + + testWidgets('backgrounding the app flushes without waiting for debounce', ( + tester, + ) async { + final controller = await openMap(tester); + controller.move(const LatLng(38, -115), 12); + await tester.pump(const Duration(milliseconds: 100)); + tester.binding.handleAppLifecycleStateChanged(AppLifecycleState.inactive); + await tester.pump(); + expect(cameraWrites, [ + ['38.0', '-115.0', '12.0'], + ]); + tester.binding.handleAppLifecycleStateChanged(AppLifecycleState.resumed); + await tester.pumpWidget(const SizedBox.shrink()); + await tester.pump(); + expect(cameraWrites, hasLength(1)); + }); +} diff --git a/test/screens/map_rebuild_scope_test.dart b/test/screens/map_rebuild_scope_test.dart new file mode 100644 index 00000000..03c94d08 --- /dev/null +++ b/test/screens/map_rebuild_scope_test.dart @@ -0,0 +1,188 @@ +import 'dart:typed_data'; + +import 'package:flutter/material.dart'; +import 'package:flutter_cache_manager/flutter_cache_manager.dart'; +import 'package:flutter_map/flutter_map.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:latlong2/latlong.dart'; +import 'package:meshcore_open/connector/meshcore_connector.dart'; +import 'package:meshcore_open/connector/meshcore_protocol.dart'; +import 'package:meshcore_open/l10n/app_localizations.dart'; +import 'package:meshcore_open/models/contact.dart'; +import 'package:meshcore_open/screens/map_screen.dart'; +import 'package:meshcore_open/services/app_settings_service.dart'; +import 'package:meshcore_open/services/map_tile_cache_service.dart'; +import 'package:meshcore_open/services/path_history_service.dart'; +import 'package:meshcore_open/services/storage_service.dart'; +import 'package:meshcore_open/services/ui_view_state_service.dart'; +import 'package:meshcore_open/storage/prefs_manager.dart'; +import 'package:meshcore_open/theme/mesh_theme.dart'; +import 'package:meshcore_open/widgets/quick_switch_bar.dart'; +import 'package:provider/provider.dart'; +import 'package:shared_preferences/shared_preferences.dart'; + +class _Connector extends MeshCoreConnector { + List nodes = []; + int contactUnread = 0; + int channelUnread = 0; + + @override + List get allContacts => List.unmodifiable(nodes); + + @override + List get contacts => List.unmodifiable(nodes); + + @override + int getTotalContactsUnreadCount() => contactUnread; + + @override + int getTotalChannelsUnreadCount() => channelUnread; +} + +class _UnusedCacheManager implements BaseCacheManager { + @override + dynamic noSuchMethod(Invocation invocation) => super.noSuchMethod(invocation); +} + +class _CountingTiles extends MapTileCacheService { + _CountingTiles(AppSettingsService settings) + : super(appSettingsService: settings, cacheManager: _UnusedCacheManager()); + + int builds = 0; + + @override + Widget buildTileLayer(BuildContext context, {double opacity = 1}) { + builds++; + return const SizedBox.shrink(); + } +} + +Contact _node({double latitude = 35, DateTime? lastSeen}) => Contact( + publicKey: Uint8List.fromList(List.filled(32, 1)), + name: 'Repeater', + type: advTypeRepeater, + pathLength: 0, + path: Uint8List(0), + latitude: latitude, + longitude: -112, + lastSeen: lastSeen ?? DateTime.now(), +); + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + late _Connector connector; + late AppSettingsService settings; + late _CountingTiles tiles; + late PathHistoryService paths; + late UiViewStateService viewState; + + setUp(() async { + SharedPreferences.setMockInitialValues({}); + PrefsManager.reset(); + await PrefsManager.initialize(); + connector = _Connector(); + settings = AppSettingsService(); + tiles = _CountingTiles(settings); + paths = PathHistoryService(StorageService()); + viewState = UiViewStateService(); + }); + + tearDown(() { + connector.dispose(); + settings.dispose(); + tiles.dispose(); + paths.dispose(); + viewState.dispose(); + }); + + Future openMap(WidgetTester tester) async { + await tester.pumpWidget( + MultiProvider( + providers: [ + ChangeNotifierProvider.value(value: connector), + ChangeNotifierProvider.value(value: settings), + ChangeNotifierProvider.value(value: paths), + ChangeNotifierProvider.value(value: viewState), + ChangeNotifierProvider.value(value: tiles), + ], + child: MaterialApp( + theme: MeshTheme.light(), + localizationsDelegates: AppLocalizations.localizationsDelegates, + supportedLocales: AppLocalizations.supportedLocales, + home: const MapScreen(), + ), + ), + ); + await tester.pump(); + await tester.pump(const Duration(milliseconds: 60)); + await tester.pump(); + } + + Iterable markerPoints(WidgetTester tester) => tester + .widgetList(find.byType(MarkerLayer)) + .expand((layer) => layer.markers) + .map((marker) => marker.point); + + testWidgets('unread badges update without rebuilding the map', ( + tester, + ) async { + connector.nodes = [_node()]; + await openMap(tester); + final originalBuilds = tiles.builds; + + connector.contactUnread = 3; + connector.channelUnread = 5; + connector.notifyListeners(); + await tester.pump(const Duration(milliseconds: 60)); + await tester.pump(); + + final navigation = tester.widget( + find.byType(QuickSwitchBar), + ); + expect(navigation.contactsUnreadCount, 3); + expect(navigation.channelsUnreadCount, 5); + expect(tiles.builds, originalBuilds); + + connector.notifyListeners(); + await tester.pump(const Duration(milliseconds: 60)); + await tester.pump(); + expect(tiles.builds, originalBuilds); + await tester.pumpWidget(const SizedBox.shrink()); + }); + + testWidgets('contact movement invalidates cached markers', (tester) async { + connector.nodes = [_node()]; + await openMap(tester); + expect(markerPoints(tester), contains(const LatLng(35, -112))); + final originalBuilds = tiles.builds; + + connector.nodes = [connector.nodes.single.copyWith(latitude: 36)]; + connector.notifyListeners(); + await tester.pump(const Duration(milliseconds: 60)); + await tester.pump(); + + expect(tiles.builds, greaterThan(originalBuilds)); + expect(markerPoints(tester), contains(const LatLng(36, -112))); + expect(markerPoints(tester), isNot(contains(const LatLng(35, -112)))); + await tester.pumpWidget(const SizedBox.shrink()); + }); + + testWidgets('time filters invalidate the visible marker cache', ( + tester, + ) async { + connector.nodes = [ + _node(lastSeen: DateTime.now().subtract(const Duration(hours: 3))), + ]; + await openMap(tester); + expect(markerPoints(tester), contains(const LatLng(35, -112))); + + await settings.setMapTimeFilterHours(1); + await tester.pump(); + expect(markerPoints(tester), isNot(contains(const LatLng(35, -112)))); + + await settings.setMapTimeFilterHours(0); + await tester.pump(); + expect(markerPoints(tester), contains(const LatLng(35, -112))); + await tester.pumpWidget(const SizedBox.shrink()); + }); +} diff --git a/test/scripts/reset_http_profile_test.dart b/test/scripts/reset_http_profile_test.dart new file mode 100644 index 00000000..9fac1035 --- /dev/null +++ b/test/scripts/reset_http_profile_test.dart @@ -0,0 +1,78 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; + +import '../../scripts/reset_http_profile.dart'; + +void main() { + test('stops recording before clearing and verifies the result', () async { + final server = await HttpServer.bind(InternetAddress.loopbackIPv4, 0); + final calls = >[]; + final paths = []; + var recording = true; + var requests = [ + {'id': 'captured-request'}, + ]; + final subscription = server.listen((request) async { + paths.add(request.uri.path); + final socket = await WebSocketTransformer.upgrade(request); + socket.listen((data) { + final call = jsonDecode(data as String) as Map; + calls.add(call); + final parameters = call['params'] as Map; + Map result; + switch (call['method']) { + case 'getVM': + result = { + 'isolates': [ + {'name': 'worker', 'id': 'worker'}, + {'name': 'main', 'id': 'main-isolate'}, + ], + }; + case 'ext.dart.io.httpEnableTimelineLogging': + if (parameters['enabled'] == 'false') recording = false; + result = {'enabled': recording}; + case 'ext.dart.io.clearHttpProfile': + expect(recording, isFalse); + requests = []; + result = {'type': 'Success'}; + case 'ext.dart.io.getHttpProfile': + result = {'requests': requests}; + default: + fail('Unexpected service method ${call['method']}'); + } + socket.add(jsonEncode({'id': call['id'], 'result': result})); + }); + }); + addTearDown(() async { + await subscription.cancel(); + await server.close(force: true); + }); + + expect( + await resetHttpProfile( + Uri.parse('http://127.0.0.1:${server.port}/service-token=/'), + ), + 0, + ); + expect(paths, ['/service-token=/ws']); + expect(calls.map((call) => call['method']), [ + 'getVM', + 'ext.dart.io.httpEnableTimelineLogging', + 'ext.dart.io.clearHttpProfile', + 'ext.dart.io.httpEnableTimelineLogging', + 'ext.dart.io.getHttpProfile', + ]); + for (final call in calls.skip(1)) { + expect((call['params'] as Map)['isolateId'], 'main-isolate'); + } + }); + + test('rejects unsupported URIs before connecting', () async { + await expectLater( + resetHttpProfile(Uri.parse('file:///tmp/service')), + throwsFormatException, + ); + }); +} diff --git a/test/services/map_tile_cache_repository_test.dart b/test/services/map_tile_cache_repository_test.dart new file mode 100644 index 00000000..24be14c7 --- /dev/null +++ b/test/services/map_tile_cache_repository_test.dart @@ -0,0 +1,264 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter_cache_manager/flutter_cache_manager.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:meshcore_open/services/map_tile_cache_repository_io.dart'; +import 'package:meshcore_open/services/map_tile_cache_repository_stub.dart' + as web; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +void main() { + late Directory directory; + late SqliteMapTileCacheRepository repository; + final factory = databaseFactoryFfiNoIsolate; + + SqliteMapTileCacheRepository createRepository() => + SqliteMapTileCacheRepository( + databaseName: 'map_tile_cache', + directory: () async => directory, + factory: factory, + ); + + CacheObject tile(String key, {int daysOld = 0, int? id}) => CacheObject( + 'https://tile.openstreetmap.org/$key.png', + key: key, + id: id, + relativePath: '$key.png', + validTill: DateTime(2030), + touched: DateTime.now().subtract(Duration(days: daysOld)), + length: 123, + eTag: 'etag-$key', + ); + + setUpAll(sqfliteFfiInit); + setUp(() async { + directory = await Directory.systemTemp.createTemp('meshcore-map-metadata-'); + repository = createRepository(); + }); + tearDown(() async { + await repository.close(); + await directory.delete(recursive: true); + }); + + test('desktop selects SQLite and web keeps its platform default', () { + expect( + createMapTileCacheRepository('map_tile_cache'), + Platform.isLinux || Platform.isWindows + ? isA() + : isNot(isA()), + ); + expect( + web.createMapTileCacheRepository('map_tile_cache'), + isA(), + ); + }); + + test('metadata survives reopen and supports updates and deletes', () async { + await repository.open(); + final inserted = await repository.insert(tile('a')); + expect(inserted.id, isNotNull); + expect(inserted.touched, isNotNull); + await repository.update(inserted.copyWith(length: 456)); + expect((await repository.get('a'))!.length, 456); + await repository.close(); + + repository = createRepository(); + await repository.open(); + final restored = (await repository.get('a'))!; + expect(restored.relativePath, 'a.png'); + expect(restored.eTag, 'etag-a'); + expect(restored.length, 456); + expect(await repository.deleteAll([]), 0); + expect(await repository.deleteAll([restored.id!, restored.id!]), 1); + expect(await repository.get('a'), isNull); + }); + + test('age and capacity queries select oldest metadata in SQLite', () async { + await repository.open(); + for (var days = 2; days <= 5; days++) { + await repository.insert( + tile('tile-$days', daysOld: days), + setTouchedToNow: false, + ); + } + expect( + (await repository.getObjectsOverCapacity(2)).map((entry) => entry.key), + ['tile-4', 'tile-5'], + ); + expect( + (await repository.getOldObjects( + const Duration(days: 3), + )).map((entry) => entry.key), + ['tile-5', 'tile-4', 'tile-3'], + ); + }); + + test('legacy JSON migrates without altering downloaded tile files', () async { + final legacy = File('${directory.path}/map_tile_cache.json'); + final oldTile = tile('10/1/2', daysOld: 2, id: 88); + final downloaded = File('${directory.path}/tile-bytes.png'); + await downloaded.writeAsBytes([1, 2, 3]); + await legacy.writeAsString( + jsonEncode([oldTile.toMap(setTouchedToNow: false)]), + ); + + await repository.open(); + final imported = (await repository.get(oldTile.key))!; + expect(imported.url, oldTile.url); + expect(imported.relativePath, oldTile.relativePath); + expect( + imported.touched!.millisecondsSinceEpoch, + oldTile.touched!.millisecondsSinceEpoch, + ); + expect(imported.length, oldTile.length); + expect(imported.eTag, oldTile.eTag); + expect(await downloaded.readAsBytes(), [1, 2, 3]); + expect(await legacy.exists(), isFalse); + await repository.close(); + + // A crash after commit but before cleanup must not replay stale metadata. + await legacy.writeAsString( + jsonEncode([tile('stale', id: 99).toMap(setTouchedToNow: false)]), + ); + repository = createRepository(); + await repository.open(); + expect(await repository.get('stale'), isNull); + expect((await repository.getAllObjects()).length, 1); + expect(await legacy.exists(), isFalse); + }); + + test( + 'malformed migration imports readable records and sets the file aside', + () async { + final legacy = File('${directory.path}/map_tile_cache.json'); + final aside = File('${legacy.path}.unreadable'); + final content = jsonEncode([ + {'invalid': true}, + tile('readable', id: 1).toMap(setTouchedToNow: false), + ]); + await legacy.writeAsString(content); + await repository.open(); + expect((await repository.getAllObjects()).single.key, 'readable'); + expect(await legacy.exists(), isFalse); + expect(await aside.readAsString(), content); + await repository.close(); + + // The file that was set aside is not imported again. + repository = createRepository(); + await repository.open(); + expect((await repository.getAllObjects()).length, 1); + await repository.close(); + await repository.deleteDataFile(); + expect(await aside.exists(), isFalse); + }, + ); + + test('a legacy file that is not JSON does not block opening', () async { + final legacy = File('${directory.path}/map_tile_cache.json'); + await legacy.writeAsString('{broken'); + await repository.open(); + expect(await repository.getAllObjects(), isEmpty); + expect(await legacy.exists(), isFalse); + expect(await File('${legacy.path}.unreadable').readAsString(), '{broken'); + }); + + test( + 'failed SQL import rolls back rows and preserves original JSON', + () async { + await repository.open(); + await repository.close(); + final db = await factory.openDatabase( + '${directory.path}/map_tile_cache.db', + ); + await db.delete('cacheMigration'); + await db.execute(''' + CREATE TRIGGER fail_import BEFORE INSERT ON cacheObject + WHEN NEW.key = 'bad' BEGIN SELECT RAISE(ABORT, 'test failure'); END + '''); + await db.close(); + final legacy = File('${directory.path}/map_tile_cache.json'); + await legacy.writeAsString( + jsonEncode([ + tile('good', id: 1).toMap(setTouchedToNow: false), + tile('bad', id: 2).toMap(setTouchedToNow: false), + ]), + ); + await expectLater(repository.open(), throwsA(isA())); + expect(await legacy.exists(), isTrue); + + final failed = await factory.openDatabase( + '${directory.path}/map_tile_cache.db', + ); + expect(await failed.query('cacheObject'), isEmpty); + expect(await failed.query('cacheMigration'), isEmpty); + await failed.execute('DROP TRIGGER fail_import'); + await failed.close(); + await repository.open(); + expect((await repository.getAllObjects()).length, 2); + expect(await legacy.exists(), isFalse); + }, + ); + + test( + 'newer database versions are rejected without deleting legacy data', + () async { + final db = await factory.openDatabase( + '${directory.path}/map_tile_cache.db', + options: OpenDatabaseOptions( + version: SqliteMapTileCacheRepository.schemaVersion + 1, + ), + ); + await db.close(); + final legacy = File('${directory.path}/map_tile_cache.json'); + await legacy.writeAsString('[]'); + await expectLater(repository.open(), throwsA(isA())); + expect(await legacy.exists(), isTrue); + }, + ); + + test('newer import markers are not overwritten or replayed', () async { + await repository.open(); + await repository.insert(tile('current')); + await repository.close(); + final db = await factory.openDatabase( + '${directory.path}/map_tile_cache.db', + ); + await db.update('cacheMigration', { + 'version': SqliteMapTileCacheRepository.legacyImportVersion + 1, + }); + await db.close(); + final legacy = File('${directory.path}/map_tile_cache.json'); + await legacy.writeAsString( + jsonEncode([tile('legacy', id: 1).toMap(setTouchedToNow: false)]), + ); + await repository.open(); + expect((await repository.getAllObjects()).map((entry) => entry.key), [ + 'current', + ]); + expect(await legacy.exists(), isTrue); + await repository.close(); + final reopened = await factory.openDatabase( + '${directory.path}/map_tile_cache.db', + ); + expect( + (await reopened.query('cacheMigration')).single['version'], + SqliteMapTileCacheRepository.legacyImportVersion + 1, + ); + await reopened.close(); + }); + + test( + 'inventory connections do not close the cache-manager connection', + () async { + await repository.open(); + await repository.open(); + expect(await repository.close(), isFalse); + final inserted = await repository.insert(tile('a')); + expect((await repository.get('a'))!.id, inserted.id); + expect(await repository.close(), isTrue); + await repository.deleteDataFile(); + expect(await repository.exists(), isFalse); + }, + ); +} diff --git a/test/services/path_history_async_storage_test.dart b/test/services/path_history_async_storage_test.dart new file mode 100644 index 00000000..94248c82 --- /dev/null +++ b/test/services/path_history_async_storage_test.dart @@ -0,0 +1,230 @@ +import 'dart:async'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:meshcore_open/models/contact.dart'; +import 'package:meshcore_open/models/path_history.dart'; +import 'package:meshcore_open/services/path_history_service.dart'; +import 'package:meshcore_open/services/storage_service.dart'; + +class _Load { + _Load(this.scope, this.contact); + final String scope; + final String contact; + final result = Completer(); +} + +class _Storage extends StorageService { + String scope = 'a'; + bool delayLoads = true; + Completer? saveGate; + bool failClears = false; + final loads = <_Load>[]; + final saved = {}; + + @override + set setPublicKeyHex(String value) => scope = value; + + @override + Future loadPathHistory(String contactPubKeyHex) async { + final load = _Load(scope, contactPubKeyHex); + loads.add(load); + if (delayLoads) return load.result.future; + return saved['${load.scope}:$contactPubKeyHex']; + } + + @override + Future savePathHistory( + String contactPubKeyHex, + ContactPathHistory history, + ) async { + final key = '$scope:$contactPubKeyHex'; + await saveGate?.future; + saved[key] = history; + } + + @override + Future clearPathHistory(String contactPubKeyHex) async { + if (failClears) throw StateError('clear failed'); + saved.remove('$scope:$contactPubKeyHex'); + } + + @override + Future clearAllPathHistories() async { + if (failClears) throw StateError('clear failed'); + saved.removeWhere((key, value) => key.startsWith('$scope:')); + } +} + +ContactPathHistory _history(String key, int hop) => ContactPathHistory( + contactPubKeyHex: key, + recentPaths: [ + PathRecord( + hopCount: 1, + tripTimeMs: 1, + timestamp: DateTime(2026), + wasFloodDiscovery: true, + pathBytes: [hop], + successCount: 1, + failureCount: 0, + ), + ], +); + +Contact _contact(int hop) => Contact( + publicKey: Uint8List(32), + name: 'Test', + type: 1, + pathLength: 1, + path: Uint8List.fromList([hop]), + lastSeen: DateTime(2026), +); + +Future _settle() => Future.delayed(Duration.zero); + +void main() { + late _Storage storage; + late PathHistoryService service; + setUp(() { + storage = _Storage(); + service = PathHistoryService(storage); + }); + tearDown(() => service.dispose()); + + test( + 'an old load cannot populate the next radio or remove its pending load', + () async { + service.getRecentPaths('contact'); + await _settle(); + service.setDevicePublicKey('b'); + service.getRecentPaths('contact'); + await _settle(); + expect(storage.loads.map((load) => load.scope), ['a', 'b']); + storage.loads[0].result.complete(_history('contact', 1)); + await _settle(); + expect(service.isCached('contact'), isFalse); + service.getRecentPaths('contact'); + await _settle(); + expect(storage.loads, hasLength(2)); + storage.loads[1].result.complete(_history('contact', 2)); + await _settle(); + expect(service.peekRecentPaths('contact').single.pathBytes, [2]); + }, + ); + + test('an old pending path event cannot be added to the next radio', () async { + final contact = _contact(1); + service.handlePathUpdated(contact); + await _settle(); + service.setDevicePublicKey('b'); + service.getRecentPaths(contact.publicKeyHex); + await _settle(); + storage.loads[1].result.complete(_history(contact.publicKeyHex, 2)); + await _settle(); + storage.loads[0].result.complete(null); + await _settle(); + expect(service.peekRecentPaths(contact.publicKeyHex).single.pathBytes, [2]); + }); + + test('clearing a contact invalidates an outstanding read', () async { + service.getRecentPaths('contact'); + await _settle(); + await service.clearPathHistory('contact'); + storage.loads.single.result.complete(_history('contact', 1)); + await _settle(); + expect(service.isCached('contact'), isFalse); + }); + + for (final dispose in [false, true]) { + test( + '${dispose ? 'disposing' : 'clearing all'} prevents delayed load notifications', + () async { + var notifications = 0; + service.addListener(() => notifications++); + service.getRecentPaths('contact'); + await _settle(); + if (dispose) { + service.dispose(); + } else { + service.clearAllHistories(); + } + final before = notifications; + storage.loads.single.result.complete(_history('contact', 1)); + await _settle(); + expect(service.isCached('contact'), isFalse); + expect(notifications, before); + }, + ); + } + + test('reads wait for an already-started history save', () async { + storage.delayLoads = false; + service.handlePathUpdated(_contact(1)); + await _settle(); + storage.saveGate = Completer(); + final save = service.flushPendingWrites(); + await _settle(); + final read = service.readStoredPaths('other'); + await _settle(); + expect(storage.loads.where((load) => load.contact == 'other'), isEmpty); + storage.saveGate!.complete(); + await save; + expect(await read, isEmpty); + expect( + storage.loads.where((load) => load.contact == 'other'), + hasLength(1), + ); + }); + + test( + 'clearing all waits for in-flight saves before allowing a new read', + () async { + storage.delayLoads = false; + final contact = _contact(1); + service.handlePathUpdated(contact); + await _settle(); + storage.saveGate = Completer(); + final save = service.flushPendingWrites(); + await _settle(); + service.clearAllHistories(); + final read = service.readStoredPaths(contact.publicKeyHex); + storage.saveGate!.complete(); + await save; + expect(await read, isEmpty); + expect(storage.saved, isEmpty); + }, + ); + + test('a scope switch while waiting for saves skips the old read', () async { + storage.delayLoads = false; + service.handlePathUpdated(_contact(1)); + await _settle(); + storage.saveGate = Completer(); + final save = service.flushPendingWrites(); + final read = service.readStoredPaths('other'); + service.setDevicePublicKey('b'); + storage.saveGate!.complete(); + await save; + expect(await read, isEmpty); + expect(storage.loads.where((load) => load.contact == 'other'), isEmpty); + expect(storage.saved.keys.single, startsWith('a:')); + }); + + test( + 'a failed clear does not break later loads, clears or flushes', + () async { + storage.delayLoads = false; + storage.failClears = true; + await expectLater(service.clearPathHistory('contact'), throwsStateError); + service.clearAllHistories(); + await _settle(); + + storage.failClears = false; + storage.saved['a:contact'] = _history('contact', 3); + await service.flushPendingWrites(); + expect((await service.readStoredPaths('contact')).single.pathBytes, [3]); + await service.clearPathHistory('contact'); + expect(storage.saved, isEmpty); + }, + ); +} diff --git a/test/storage/channel_storage_by_id_test.dart b/test/storage/channel_storage_by_id_test.dart index ec7a376c..064d56f3 100644 --- a/test/storage/channel_storage_by_id_test.dart +++ b/test/storage/channel_storage_by_id_test.dart @@ -119,7 +119,8 @@ void main() { test('the archive follows the channel ID', () async { await _init(); final store = ChannelMessageStore()..setPublicKeyHex = _deviceA; - store.setArchived(_idOne, [_ch('old', 1), _ch('older', 2)]); + await store.saveChannelMessages(_idOne, [_ch('old', 1), _ch('older', 2)]); + store.setArchivedCount(_idOne, 2); expect(store.archivedCount(_idOne), 2); expect(store.archivedCount(_idTwo), isNull); diff --git a/test/storage/history_store_test.dart b/test/storage/history_store_test.dart new file mode 100644 index 00000000..4f09017d --- /dev/null +++ b/test/storage/history_store_test.dart @@ -0,0 +1,511 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:meshcore_open/connector/meshcore_connector.dart'; +import 'package:meshcore_open/models/app_settings.dart'; +import 'dart:typed_data'; +import 'package:meshcore_open/models/channel.dart'; +import 'package:meshcore_open/storage/channel_storage_migration.dart'; +import 'package:meshcore_open/storage/channel_store.dart'; +import 'package:meshcore_open/services/app_settings_service.dart'; +import 'package:meshcore_open/services/backup_service.dart'; +import 'package:meshcore_open/review_mode/review_mode_storage.dart'; +import 'package:meshcore_open/review_mode/review_radio.dart'; +import 'package:meshcore_open/services/storage_service.dart'; +import 'package:meshcore_open/models/path_history.dart'; +import 'package:meshcore_open/storage/history_store.dart'; +import 'package:meshcore_open/storage/legacy_key_migration.dart'; +import 'package:meshcore_open/storage/history_store_io.dart'; +import 'package:meshcore_open/storage/prefs_manager.dart'; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:shared_preferences_platform_interface/shared_preferences_platform_interface.dart'; +import 'package:shared_preferences_platform_interface/types.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +class _DelayedHistoryStore extends PreferencesHistoryStore { + _DelayedHistoryStore(this.delayedKey, this.gate); + final String delayedKey; + final Completer gate; + @override + Future getString(String key) async { + if (key == delayedKey) await gate.future; + return super.getString(key); + } +} + +class _FailingCleanupPreferences extends InMemorySharedPreferencesStore { + _FailingCleanupPreferences() : super.withData({'flutter.messages_a': '[1]'}); + bool fail = true; + @override + Future clearWithParameters(ClearParameters parameters) async => + fail ? false : super.clearWithParameters(parameters); +} + +class _BackupConnector extends MeshCoreConnector { + _BackupConnector(this.radioKey); + final String radioKey; + @override + bool get isConnected => true; + @override + String get selfPublicKeyHex => radioKey; +} + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + late SqliteHistoryStore store; + + setUp(() async { + sqfliteFfiInit(); + SharedPreferences.setMockInitialValues({}); + PrefsManager.reset(); + await PrefsManager.initialize(); + store = await SqliteHistoryStore.open( + inMemoryDatabasePath, + factory: databaseFactoryFfiNoIsolate, + ); + HistoryStore.use(store); + }); + + tearDown(() async { + await store.close(); + PrefsManager.reset(); + }); + + test('native history writes leave small preferences untouched', () async { + await PrefsManager.instance.setString('theme', 'dark'); + await store.setString('messages_a', '[{"text":"hello"}]'); + await store.setString('path_history_a', '{"recentPaths":[]}'); + expect(PrefsManager.instance.getKeys(), {'theme'}); + expect(await store.getString('messages_a'), '[{"text":"hello"}]'); + expect(await store.getString('path_history_a'), '{"recentPaths":[]}'); + }); + + test('sequence reads return only the requested ordered page', () async { + await store.setString( + 'channel_messages_a', + jsonEncode([ + for (var i = 0; i < 1000; i++) {'id': i}, + ]), + ); + expect(await store.entryCount('channel_messages_a'), 1000); + expect( + await store.readEntries('channel_messages_a', offset: 950, limit: 3), + ['{"id":950}', '{"id":951}', '{"id":952}'], + ); + expect( + await store.readEntries('channel_messages_a', offset: 1000, limit: 3), + isEmpty, + ); + expect(await store.readEntries('channel_messages_a', limit: 0), isEmpty); + }); + + test( + 'tail replacement preserves history, deletes and storage limits', + () async { + await store.setString('messages_a', '[0,1,2,3,4]'); + expect( + await store.replaceTail('messages_a', 3, ['4', '5'], maxEntries: 4), + 2, + ); + expect(await store.readEntries('messages_a'), ['1', '2', '4', '5']); + expect(await store.replaceTail('messages_a', 2, ['6'], maxEntries: 4), 2); + expect(await store.readEntries('messages_a'), ['1', '2', '6']); + expect( + await store.replaceTail('messages_a', 50, ['7'], maxEntries: 4), + 3, + ); + expect(await store.readEntries('messages_a'), ['1', '2', '6', '7']); + expect(await store.replaceTail('messages_a', 0, [], maxEntries: 4), 0); + expect(await store.getString('messages_a'), '[]'); + }, + ); + + test('history data survives database reopen', () async { + final dir = await Directory.systemTemp.createTemp('meshcore-history-test-'); + addTearDown(() => dir.delete(recursive: true)); + final path = '${dir.path}/history_v1.db'; + final disk = await SqliteHistoryStore.open( + path, + factory: databaseFactoryFfiNoIsolate, + ); + await disk.setString('messages_a', '[1,2,3]'); + await disk.setString('path_history_a', '{"sample":true}'); + await disk.close(); + final reopened = await SqliteHistoryStore.open( + path, + factory: databaseFactoryFfiNoIsolate, + ); + addTearDown(reopened.close); + expect(await reopened.getString('messages_a'), '[1,2,3]'); + expect(await reopened.getString('path_history_a'), '{"sample":true}'); + }); + + test('versioned import removes only committed valid histories', () async { + final prefs = PrefsManager.instance; + await prefs.setString('messages_a', '[{"text":"hello"},42]'); + await prefs.setString('path_history_a', '{"recentPaths":[]}'); + await prefs.setString('messages_unreadable', '{broken'); + await prefs.setString('path_history_wrong_shape', '[]'); + await prefs.setBool('notification', true); + await store.migratePreferences(); + expect(prefs.getKeys(), { + 'messages_unreadable', + 'path_history_wrong_shape', + 'notification', + }); + expect(await store.readEntries('messages_a'), ['{"text":"hello"}', '42']); + expect(await store.getString('messages_unreadable'), '{broken'); + expect( + await store.getKeys(), + containsAll(['messages_a', 'path_history_a', 'messages_unreadable']), + ); + final metadata = await store.database.query('history_metadata'); + expect(metadata.single['value'], SqliteHistoryStore.schemaVersion); + // Current versions skip another scan/import. + await prefs.setString('messages_added_later', '[1]'); + await store.migratePreferences(); + expect(prefs.containsKey('messages_added_later'), isTrue); + expect( + await store.database.query( + 'history_keys', + where: 'key = ?', + whereArgs: ['messages_added_later'], + ), + isEmpty, + ); + }); + + test( + 'interrupted import does not replace committed newer database values', + () async { + await store.setString('messages_a', '[2]'); + await PrefsManager.instance.setString('messages_a', '[1]'); + await store.migratePreferences(); + expect(await store.getString('messages_a'), '[2]'); + expect(PrefsManager.instance.containsKey('messages_a'), isFalse); + }, + ); + + test('histories saved by a run without the database are merged', () async { + final prefs = PrefsManager.instance; + await store.migratePreferences(); + await store.setString( + 'messages_a', + '[{"messageId":"1","text":"old"},{"messageId":"2","text":"sent"}]', + ); + await store.setString('path_history_a', '{"stored":true}'); + // What the fallback run saved: it started from an empty history. + await prefs.setString( + 'messages_a', + '[{"messageId":"2","text":"delivered"},{"messageId":"3","text":"new"}]', + ); + await prefs.setString('messages_b', '[{"messageId":"4"}]'); + await prefs.setString('path_history_a', '{"stored":false}'); + await prefs.setString('messages_unreadable', '{broken'); + + await store.adoptFallbackHistories(); + + expect(await store.readEntries('messages_a'), [ + '{"messageId":"1","text":"old"}', + '{"messageId":"2","text":"delivered"}', + '{"messageId":"3","text":"new"}', + ]); + expect(await store.readEntries('messages_b'), ['{"messageId":"4"}']); + expect(await store.getString('path_history_a'), '{"stored":true}'); + expect(prefs.getKeys(), {'messages_unreadable'}); + }); + + test('initialize falls back to preferences and flags the run', () async { + // No path provider is registered here, so the database cannot be opened. + await HistoryStore.initialize(); + addTearDown(HistoryStore.reset); + expect(HistoryStore.instance, isA()); + expect(PrefsManager.instance.getBool(HistoryStore.fallbackRunKey), isTrue); + }); + + test('a save lands beside an unreadable preference', () async { + await PrefsManager.instance.setString('messages_a', '{broken'); + await store.migratePreferences(); + expect(await store.replaceTail('messages_a', 0, ['1', '2']), 0); + expect(await store.readEntries('messages_a'), ['1', '2']); + expect(await store.replaceTail('messages_a', 2, ['3']), 2); + expect(await store.getString('messages_a'), '[1,2,3]'); + expect(PrefsManager.instance.getString('messages_a'), '{broken'); + }); + + test('the preferences store replaces an unreadable history', () async { + final prefs = PreferencesHistoryStore(); + await PrefsManager.instance.setString('messages_a', '{broken'); + expect(await prefs.replaceTail('messages_a', 3, ['1', '2']), 0); + expect( + await prefs.replaceTail('messages_a', 2, ['3', '4'], maxEntries: 3), + 1, + ); + expect(await prefs.getString('messages_a'), '[2,3,4]'); + // A value written behind the store's back is read fresh. + await PrefsManager.instance.setString('messages_a', '[9]'); + expect(await prefs.readEntries('messages_a'), ['9']); + expect(await prefs.entryCount('messages_a'), 1); + }); + + test('future import versions are left untouched', () async { + await store.database.insert('history_metadata', { + 'key': 'preferences_import_version', + 'value': 99, + }); + await PrefsManager.instance.setString('messages_a', '[1]'); + await store.migratePreferences(); + expect(PrefsManager.instance.containsKey('messages_a'), isTrue); + expect( + (await store.database.query('history_metadata')).single['value'], + 99, + ); + }); + + test('review cleanup removes only its database histories', () async { + final prefix = ReviewRadio.storagePrefix; + await store.setString('messages_${prefix}c1', '[1]'); + await store.setString('channel_messages_${prefix}_id', '[2]'); + await store.setString('messages_real_c1', '[3]'); + await clearReviewModeData(); + expect(await store.getKeys(), {'messages_real_c1'}); + }); + + test( + 'path history scoping and clearing work through database storage', + () async { + final a = StorageService()..setPublicKeyHex = 'aaaaaaaaaaaa'; + final b = StorageService()..setPublicKeyHex = 'bbbbbbbbbbbb'; + await store.setString( + 'path_history_c1', + jsonEncode( + ContactPathHistory(contactPubKeyHex: 'c1', recentPaths: []).toJson(), + ), + ); + expect(await a.loadPathHistory('c1'), isNotNull); + expect(await b.loadPathHistory('c1'), isNull); + await b.savePathHistory( + 'c1', + ContactPathHistory(contactPubKeyHex: 'c1', recentPaths: []), + ); + await a.clearAllPathHistories(); + expect(await store.getKeys(), {'path_history_bbbbbbbbbb_c1'}); + }, + ); + test( + 'backup exports native histories and restores into the destination scope', + () async { + final source = _BackupConnector('a' * 64); + final destination = _BackupConnector('b' * 64); + final settings = AppSettingsService(); + addTearDown(source.dispose); + addTearDown(destination.dispose); + addTearDown(settings.dispose); + await store.setString('messages_aaaaaaaaaac1', '[{"text":"original"}]'); + await store.setString( + 'channel_messages_aaaaaaaaaa_0123456789abcdef', + '[{"text":"channel"}]', + ); + const service = BackupService(); + final backup = await service.createBackup( + items: {BackupItem.contactHistory, BackupItem.channelHistory}, + connector: source, + settings: AppSettings(), + password: '', + ); + final decoded = jsonDecode(backup) as Map; + final contacts = decoded['sections']['contacts']['preferences'] as Map; + expect(contacts['messages_aaaaaaaaaac1'], [ + {'text': 'original'}, + ]); + await store.setString('messages_bbbbbbbbbbc1', '[{"text":"old"}]'); + await service.restoreBackup( + contents: backup, + password: '', + items: {BackupItem.contactHistory, BackupItem.channelHistory}, + connector: destination, + appSettingsService: settings, + ); + expect( + await store.getString('messages_bbbbbbbbbbc1'), + '[{"text":"original"}]', + ); + expect( + await store.getString('channel_messages_bbbbbbbbbb_0123456789abcdef'), + '[{"text":"channel"}]', + ); + expect( + await store.getString('messages_aaaaaaaaaac1'), + '[{"text":"original"}]', + ); + expect( + PrefsManager.instance.getKeys().where(HistoryStore.isHistoryKey), + isEmpty, + ); + }, + ); + test( + 'failed preference cleanup leaves source and import version available for retry', + () async { + SharedPreferences.setMockInitialValues({}); + final platform = _FailingCleanupPreferences(); + SharedPreferencesStorePlatform.instance = platform; + PrefsManager.reset(); + await PrefsManager.initialize(); + HistoryStore.use(store); + await expectLater(store.migratePreferences(), throwsStateError); + expect(PrefsManager.instance.getString('messages_a'), '[1]'); + expect(await store.getString('messages_a'), '[1]'); + expect(await store.database.query('history_metadata'), isEmpty); + platform.fail = false; + await store.migratePreferences(); + expect(PrefsManager.instance.containsKey('messages_a'), isFalse); + expect( + (await store.database.query('history_metadata')).single['value'], + 1, + ); + }, + ); + + test( + 'failed database import preserves original preferences and can retry', + () async { + await PrefsManager.instance.setString('messages_a', '[1]'); + await PrefsManager.instance.setString('messages_b', '[2]'); + await store.database.execute( + "CREATE TRIGGER fail_history BEFORE INSERT ON history_entries WHEN NEW.key = 'messages_b' BEGIN SELECT RAISE(ABORT, 'test failure'); END", + ); + await expectLater( + store.migratePreferences(), + throwsA(isA()), + ); + expect(PrefsManager.instance.getString('messages_a'), '[1]'); + expect(PrefsManager.instance.getString('messages_b'), '[2]'); + expect(await store.database.query('history_metadata'), isEmpty); + await store.database.execute('DROP TRIGGER fail_history'); + await store.migratePreferences(); + expect(await store.readEntries('messages_a'), ['1']); + expect(await store.readEntries('messages_b'), ['2']); + expect(PrefsManager.instance.getKeys(), isEmpty); + }, + ); + test( + 'channel migration removes old database keys after committing the new ID', + () async { + const scope = 'aaaaaaaaaa'; + final channel = Channel( + index: 0, + name: 'Public', + psk: Uint8List.fromList(List.filled(16, 1)), + ); + await (ChannelStore()..setPublicKeyHex = '${scope}xx').saveChannels([ + channel, + ]); + final oldKey = 'channel_messages_${scope}0'; + final newKey = 'channel_messages_${scope}_${channel.storageId}'; + await store.setString(oldKey, '[{"messageId":"legacy","text":"keep"}]'); + + await ChannelStorageMigration.migrate(scope); + + expect(await store.getKeys(), {newKey}); + expect( + await store.getString(newKey), + '[{"messageId":"legacy","text":"keep"}]', + ); + expect( + PrefsManager.instance.getBool('channel_storage_v2_$scope'), + isTrue, + ); + }, + ); + + test( + 'channel migration leaves malformed sources in place and still completes', + () async { + const scope = 'aaaaaaaaaa'; + final psk = Uint8List.fromList(List.filled(16, 1)); + final id = Channel.storageIdForPsk(psk)!; + await PrefsManager.instance.setString( + 'channels$scope', + jsonEncode([ + { + 'index': 0, + 'name': 'Public', + 'psk': base64Encode(psk), + 'unreadCount': 0, + }, + ]), + ); + await PrefsManager.instance.setString( + 'channel_messages_${scope}0', + '{broken', + ); + await store.setString( + 'channel_messages_${scope}_$id', + '[{"messageId":"existing","text":"keep"}]', + ); + await ChannelStorageMigration.migrate(scope); + expect( + PrefsManager.instance.getString('channel_messages_${scope}0'), + '{broken', + ); + expect( + await store.getString('channel_messages_${scope}_$id'), + '[{"messageId":"existing","text":"keep"}]', + ); + expect( + PrefsManager.instance.getBool('channel_storage_v2_$scope'), + isTrue, + ); + // An array that cannot be merged safely is left alone as well. + await PrefsManager.instance.remove('channel_storage_v2_$scope'); + await PrefsManager.instance.setString( + 'channel_messages_${scope}0', + '[42]', + ); + await ChannelStorageMigration.migrate(scope); + expect( + PrefsManager.instance.getString('channel_messages_${scope}0'), + '[42]', + ); + expect( + await store.getString('channel_messages_${scope}_$id'), + '[{"messageId":"existing","text":"keep"}]', + ); + expect( + PrefsManager.instance.getBool('channel_storage_v2_$scope'), + isTrue, + ); + }, + ); + test( + 'path migration keeps the original scope policy while reading asynchronously', + () async { + final reviewScope = ReviewRadio.storagePrefix; + scopesWithoutLegacyMigration.add(reviewScope); + addTearDown(() => scopesWithoutLegacyMigration.remove(reviewScope)); + await PrefsManager.instance.setString( + 'path_history_c1', + jsonEncode( + ContactPathHistory(contactPubKeyHex: 'c1', recentPaths: []).toJson(), + ), + ); + final gate = Completer(); + HistoryStore.use( + _DelayedHistoryStore('path_history_${reviewScope}_c1', gate), + ); + final service = StorageService()..setPublicKeyHex = '${reviewScope}xx'; + final pending = service.loadPathHistory('c1'); + service.setPublicKeyHex = 'bbbbbbbbbbbb'; + gate.complete(); + expect(await pending, isNull); + expect(PrefsManager.instance.containsKey('path_history_c1'), isTrue); + expect( + PrefsManager.instance.containsKey('path_history_${reviewScope}_c1'), + isFalse, + ); + }, + ); +} diff --git a/test/storage/legacy_migration_writes_test.dart b/test/storage/legacy_migration_writes_test.dart new file mode 100644 index 00000000..4ce13b67 --- /dev/null +++ b/test/storage/legacy_migration_writes_test.dart @@ -0,0 +1,218 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:meshcore_open/models/channel.dart'; +import 'package:meshcore_open/storage/channel_storage_migration.dart'; +import 'package:meshcore_open/storage/channel_store.dart'; +import 'package:meshcore_open/storage/community_store.dart'; +import 'package:meshcore_open/storage/contact_discovery_store.dart'; +import 'package:meshcore_open/storage/contact_group_store.dart'; +import 'package:meshcore_open/storage/contact_settings_store.dart'; +import 'package:meshcore_open/storage/contact_store.dart'; +import 'package:meshcore_open/storage/legacy_key_migration.dart'; +import 'package:meshcore_open/storage/message_store.dart'; +import 'package:meshcore_open/storage/prefs_manager.dart'; +import 'package:meshcore_open/storage/unread_store.dart'; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:shared_preferences_platform_interface/shared_preferences_platform_interface.dart'; + +const _device = 'aaaaaaaaaa012345'; +const _scope = 'aaaaaaaaaa'; + +class _RecordingPreferences extends InMemorySharedPreferencesStore { + _RecordingPreferences(Map values) + : super.withData({ + for (final e in values.entries) 'flutter.${e.key}': e.value, + }); + + final calls = <(String, String)>[]; + final failedKeys = {}; + Completer? writeGate; + + @override + Future setValue(String valueType, String key, Object value) async { + calls.add(('set', key)); + if (failedKeys.contains(key)) return false; + await writeGate?.future; + return super.setValue(valueType, key, value); + } + + @override + Future remove(String key) async { + calls.add(('remove', key)); + return super.remove(key); + } +} + +Future<_RecordingPreferences> _init([ + Map values = const {}, +]) async { + SharedPreferences.setMockInitialValues({}); + final platform = _RecordingPreferences(values); + SharedPreferencesStorePlatform.instance = platform; + PrefsManager.reset(); + await PrefsManager.initialize(); + return platform; +} + +String _cachedChannel(Uint8List psk) => jsonEncode([ + {'index': 0, 'name': 'Public', 'psk': base64Encode(psk), 'unreadCount': 0}, +]); + +void main() { + tearDown(() { + scopesWithoutLegacyMigration.clear(); + PrefsManager.reset(); + SharedPreferences.setMockInitialValues({}); + }); + + test('loading absent legacy data performs no platform writes', () async { + final platform = await _init(); + await (MessageStore()..setPublicKeyHex = _device).loadMessages('c1'); + await (ContactSettingsStore()..setPublicKeyHex = _device).loadSmazEnabled( + 'c1', + ); + await (ContactStore()..setPublicKeyHex = _device).loadContacts(); + await (ChannelStore()..setPublicKeyHex = _device).loadChannels(); + await (CommunityStore()..setPublicKeyHex = _device).loadCommunities(); + await (ContactGroupStore()..setPublicKeyHex = _device).loadGroups(); + await (UnreadStore()..setPublicKeyHex = _device).loadContactUnreadCount(); + await (ContactDiscoveryStore()..setPublicKeyHex = _device).loadContacts(); + expect(platform.calls, isEmpty); + }); + + test( + 'string migration waits for persistence before deleting the source', + () async { + final platform = await _init({'messages_c1': '[]'}); + final gate = Completer(); + platform.writeGate = gate; + final pending = (MessageStore()..setPublicKeyHex = _device).loadMessages( + 'c1', + ); + await Future.delayed(Duration.zero); + expect(platform.calls, [('set', 'flutter.messages_${_scope}c1')]); + expect((await platform.getAll())['flutter.messages_c1'], '[]'); + gate.complete(); + await pending; + expect(platform.calls, [ + ('set', 'flutter.messages_${_scope}c1'), + ('remove', 'flutter.messages_c1'), + ]); + await (MessageStore()..setPublicKeyHex = _device).loadMessages('c1'); + expect(platform.calls, hasLength(2)); + }, + ); + + test('a failed string migration retains persisted legacy data', () async { + final platform = await _init({'messages_c1': '[]'}); + platform.failedKeys.add('flutter.messages_${_scope}c1'); + await (MessageStore()..setPublicKeyHex = _device).loadMessages('c1'); + expect(platform.calls, [('set', 'flutter.messages_${_scope}c1')]); + final persisted = await platform.getAll(); + expect(persisted['flutter.messages_c1'], '[]'); + expect(persisted, isNot(contains('flutter.messages_${_scope}c1'))); + }); + + test( + 'false legacy settings migrate, but stay persisted when writing fails', + () async { + final platform = await _init({'contact_smaz_c1': false}); + platform.failedKeys.add('flutter.contact_smaz_${_scope}c1'); + expect( + await (ContactSettingsStore()..setPublicKeyHex = _device) + .loadSmazEnabled('c1'), + isFalse, + ); + expect(platform.calls, [('set', 'flutter.contact_smaz_${_scope}c1')]); + expect((await platform.getAll())['flutter.contact_smaz_c1'], isFalse); + }, + ); + + test('review scopes never adopt legacy strings or settings', () async { + final platform = await _init({ + 'messages_c1': '[]', + 'contact_smaz_c1': true, + }); + scopesWithoutLegacyMigration.add(_scope); + expect( + await (MessageStore()..setPublicKeyHex = _device).loadMessages('c1'), + isEmpty, + ); + expect( + await (ContactSettingsStore()..setPublicKeyHex = _device).loadSmazEnabled( + 'c1', + ), + isFalse, + ); + expect(platform.calls, isEmpty); + }); + + test('channel migration skips removals of absent old keys', () async { + final psk = Uint8List.fromList(List.filled(16, 1)); + final platform = await _init({'channels$_scope': _cachedChannel(psk)}); + await ChannelStorageMigration.migrate(_scope); + expect(platform.calls.where((c) => c.$1 == 'remove'), isEmpty); + expect(PrefsManager.instance.getBool('channel_storage_v2_$_scope'), isTrue); + }); + + test( + 'channel migration keeps sources and stays unfinished when a write fails', + () async { + final psk = Uint8List.fromList(List.filled(16, 1)); + final id = Channel.storageIdForPsk(psk)!; + final sourceKey = 'channel_messages_${_scope}0'; + final platform = await _init({ + 'channels$_scope': _cachedChannel(psk), + sourceKey: '[]', + }); + platform.failedKeys.add('flutter.channel_messages_${_scope}_$id'); + await expectLater( + ChannelStorageMigration.migrate(_scope), + throwsStateError, + ); + expect((await platform.getAll())['flutter.$sourceKey'], '[]'); + expect( + PrefsManager.instance.getBool('channel_storage_v2_$_scope'), + isNull, + ); + expect(platform.calls.where((c) => c.$1 == 'remove'), isEmpty); + }, + ); + + test( + 'false legacy settings migrate successfully without extra writes', + () async { + final platform = await _init({'contact_smaz_c1': false}); + final store = ContactSettingsStore()..setPublicKeyHex = _device; + expect(await store.loadSmazEnabled('c1'), isFalse); + expect(await store.loadSmazEnabled('c1'), isFalse); + expect(platform.calls, [ + ('set', 'flutter.contact_smaz_${_scope}c1'), + ('remove', 'flutter.contact_smaz_c1'), + ]); + final persisted = await platform.getAll(); + expect(persisted['flutter.contact_smaz_${_scope}c1'], isFalse); + expect(persisted, isNot(contains('flutter.contact_smaz_c1'))); + }, + ); + + test( + 'channel migration is not marked done before its slot map is durable', + () async { + final psk = Uint8List.fromList(List.filled(16, 1)); + final platform = await _init({'channels$_scope': _cachedChannel(psk)}); + platform.failedKeys.add('flutter.channel_slot_ids_$_scope'); + await expectLater( + ChannelStorageMigration.migrate(_scope), + throwsStateError, + ); + expect( + PrefsManager.instance.getBool('channel_storage_v2_$_scope'), + isNull, + ); + }, + ); +} diff --git a/test/storage/message_archive_memory_test.dart b/test/storage/message_archive_memory_test.dart new file mode 100644 index 00000000..03d893b9 --- /dev/null +++ b/test/storage/message_archive_memory_test.dart @@ -0,0 +1,285 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:meshcore_open/models/channel_message.dart'; +import 'package:meshcore_open/models/message.dart'; +import 'package:meshcore_open/storage/channel_message_store.dart'; +import 'package:meshcore_open/storage/history_store.dart'; +import 'package:meshcore_open/storage/message_store.dart'; + +const _device = 'abcdef012345'; +const _contact = 'contact'; +const _channel = 'channel'; + +class _History extends HistoryStore { + final data = >{}; + int reads = 0; + bool failNextWrite = false; + bool failNextRemove = false; + Completer? nextWriteGate; + Completer? nextWriteStarted; + + @override + Future getString(String key) async { + reads++; + final entries = data[key]; + return entries == null ? null : '[${entries.join(',')}]'; + } + + @override + Future setString(String key, String value) async { + data[key] = (jsonDecode(value) as List).map(jsonEncode).toList(); + return true; + } + + @override + Future remove(String key) async { + if (failNextRemove) { + failNextRemove = false; + return false; + } + data.remove(key); + return true; + } + + @override + Future> getKeys() async => data.keys.toSet(); + + @override + Future entryCount(String key) async => data[key]?.length ?? 0; + + @override + Future> readEntries( + String key, { + int offset = 0, + int? limit, + }) async { + reads++; + final entries = (data[key] ?? []).skip(offset); + return (limit == null ? entries : entries.take(limit)).toList(); + } + + @override + Future replaceTail( + String key, + int preservedCount, + List entries, { + int maxEntries = 1000, + }) async { + final gate = nextWriteGate; + nextWriteGate = null; + nextWriteStarted?.complete(); + nextWriteStarted = null; + await gate?.future; + if (failNextWrite) { + failNextWrite = false; + throw StateError('Disk unavailable'); + } + final prefix = (data[key] ?? []).take(preservedCount).toList(); + final all = [...prefix, ...entries]; + final dropped = (all.length - maxEntries).clamp(0, all.length); + data[key] = all.skip(dropped).toList(); + return (prefix.length - dropped).clamp(0, prefix.length); + } + + List texts(String key) => [ + for (final entry in data[key] ?? []) + (jsonDecode(entry) as Map)['text'] as String, + ]; +} + +Message _dm(int i, {String? id, String? text}) => Message( + senderKey: Uint8List.fromList([1, 2, 3]), + text: text ?? 'm$i', + timestamp: DateTime.fromMillisecondsSinceEpoch(1000 * i), + isOutgoing: false, + messageId: id ?? 'id$i', +); + +ChannelMessage _ch(int i) => ChannelMessage( + senderKey: null, + senderName: 'Bob', + text: 'c$i', + timestamp: DateTime.fromMillisecondsSinceEpoch(1000 * i), + isOutgoing: false, + status: ChannelMessageStatus.sent, + channelIndex: 0, + messageId: 'cid$i', +); + +void main() { + late _History history; + late MessageStore direct; + late ChannelMessageStore channel; + late String directKey; + late String channelKey; + + setUp(() { + history = _History(); + HistoryStore.use(history); + direct = MessageStore()..setPublicKeyHex = _device; + channel = ChannelMessageStore()..setPublicKeyHex = _device; + directKey = '${direct.keyFor}$_contact'; + channelKey = '${channel.keyFor}_$_channel'; + }); + + tearDown(HistoryStore.reset); + + test( + 'reconciled saves preserve prefixes without reading archived JSON', + () async { + await direct.saveMessages(_contact, [for (var i = 0; i < 8; i++) _dm(i)]); + await channel.saveChannelMessages(_channel, [ + for (var i = 0; i < 8; i++) _ch(i), + ]); + direct.setArchivedCount(_contact, 6); + channel.setArchivedCount(_channel, 6); + history.reads = 0; + + await direct.saveMessages(_contact, [_dm(6), _dm(8)]); + await channel.saveChannelMessages(_channel, [_ch(6), _ch(8)]); + + expect(history.reads, 0); + expect(history.texts(directKey), [ + for (var i = 0; i <= 6; i++) 'm$i', + 'm8', + ]); + expect(history.texts(channelKey), [ + for (var i = 0; i <= 6; i++) 'c$i', + 'c8', + ]); + }, + ); + + test('overlapping saves consume each unsaved trim once', () async { + direct.setArchivedCount(_contact, 0); + await direct.saveMessages(_contact, [_dm(0), _dm(1), _dm(2)]); + direct.archive(_contact, [_dm(0)]); + final gate = Completer(); + final started = Completer(); + history.nextWriteGate = gate; + history.nextWriteStarted = started; + final first = direct.saveMessages(_contact, [_dm(1), _dm(2), _dm(3)]); + await started.future; + + direct.archive(_contact, [_dm(1)]); + final second = direct.saveMessages(_contact, [_dm(2), _dm(3), _dm(4)]); + gate.complete(); + await Future.wait([first, second]); + + expect(history.texts(directKey), [for (var i = 0; i <= 4; i++) 'm$i']); + expect(direct.archivedCount(_contact), 2); + }); + + test('failed writes retain unsaved trims for retry', () async { + direct.setArchivedCount(_contact, 0); + await direct.saveMessages(_contact, [_dm(0), _dm(1)]); + direct.archive(_contact, [_dm(0)]); + history.failNextWrite = true; + await expectLater( + direct.saveMessages(_contact, [_dm(1), _dm(2)]), + throwsStateError, + ); + expect(direct.archivedCount(_contact), 1); + expect(history.texts(directKey), ['m0', 'm1']); + + await direct.saveMessages(_contact, [_dm(1), _dm(2)]); + expect(history.texts(directKey), ['m0', 'm1', 'm2']); + expect(direct.archivedCount(_contact), 1); + }); + + test( + 'queued snapshots survive scope switches without restoring old bookkeeping', + () async { + direct.setArchivedCount(_contact, 0); + final messages = [_dm(0)]; + final gate = Completer(); + final started = Completer(); + history.nextWriteGate = gate; + history.nextWriteStarted = started; + final pending = direct.saveMessages(_contact, messages); + await started.future; + messages.clear(); + direct.forgetArchive(); + direct.setPublicKeyHex = '0123456789abcd'; + final newKey = '${direct.keyFor}$_contact'; + await direct.saveMessages(_contact, [_dm(1)]); + gate.complete(); + await pending; + + expect(history.texts(directKey), ['m0']); + expect(history.texts(newKey), ['m1']); + direct.setPublicKeyHex = _device; + expect(direct.archivedCount(_contact), isNull); + }, + ); + + test( + 'a clear through another instance is ordered after in-flight saves', + () async { + channel.setArchivedCount(_channel, 0); + final gate = Completer(); + final started = Completer(); + history.nextWriteGate = gate; + history.nextWriteStarted = started; + final save = channel.saveChannelMessages(_channel, [_ch(0)]); + await started.future; + final clear = (ChannelMessageStore()..setPublicKeyHex = _device) + .clearChannelMessages(_channel); + final laterSave = channel.saveChannelMessages(_channel, [_ch(1)]); + gate.complete(); + await Future.wait([save, clear, laterSave]); + + expect(history.texts(channelKey), ['c1']); + expect(channel.archivedCount(_channel), 0); + }, + ); + + test('cap reductions remain correct across queued saves', () async { + const cap = MessageStore.maxStoredMessages; + await direct.saveMessages(_contact, [for (var i = 0; i < cap; i++) _dm(i)]); + direct.setArchivedCount(_contact, cap - 2); + final first = direct.saveMessages(_contact, [ + _dm(cap - 2), + _dm(cap - 1), + _dm(cap), + ]); + final second = direct.saveMessages(_contact, [ + _dm(cap - 2), + _dm(cap - 1), + _dm(cap), + _dm(cap + 1), + ]); + await Future.wait([first, second]); + + expect(history.texts(directKey).first, 'm2'); + expect(history.texts(directKey).last, 'm${cap + 1}'); + expect(history.data[directKey], hasLength(cap)); + expect(direct.archivedCount(_contact), cap - 4); + }); + + test( + 'unreconciled merge preserves distinct duplicates without message IDs', + () async { + final duplicate = _dm(0, id: '', text: 'same'); + await direct.saveMessages(_contact, [duplicate, duplicate]); + direct.forgetArchive(); + await direct.saveMessages(_contact, [duplicate, _dm(1)]); + expect(history.texts(directKey), ['same', 'same', 'm1']); + }, + ); + + test( + 'backend replacement cannot reuse another backend archive counts', + () async { + direct.setArchivedCount(_contact, 100); + final replacement = _History(); + HistoryStore.use(replacement); + expect(direct.archivedCount(_contact), isNull); + await direct.saveMessages(_contact, [_dm(0)]); + expect(replacement.texts(directKey), ['m0']); + }, + ); +} diff --git a/test/storage/message_store_archive_test.dart b/test/storage/message_store_archive_test.dart index 51908258..f1026300 100644 --- a/test/storage/message_store_archive_test.dart +++ b/test/storage/message_store_archive_test.dart @@ -55,7 +55,7 @@ void main() { final loaded = await store.loadMessages(_contact); final window = loaded.sublist(6); - store.setArchived(_contact, loaded.sublist(0, 6)); + store.setArchivedCount(_contact, 6); await store.saveMessages(_contact, [...window, _dm(10)]); final reloaded = await store.loadMessages(_contact); @@ -67,7 +67,7 @@ void main() { test('trimmed messages are archived, deletes stay deleted', () async { await _init(); final store = _messageStore(); - store.setArchived(_contact, const []); + store.setArchivedCount(_contact, 0); final window = [for (var i = 0; i < 4; i++) _dm(i)]; await store.saveMessages(_contact, window); @@ -79,9 +79,23 @@ void main() { final reloaded = await store.loadMessages(_contact); expect(reloaded.map((m) => m.text), ['m0', 'm1', 'm3', 'm4']); expect(store.archivedCount(_contact), 2); + }); - store.unarchiveNewest(_contact, 1); - expect(store.archivedCount(_contact), 1); + test('an unreadable stored history does not block later saves', () async { + await _init(); + final store = _messageStore(); + final key = '${store.keyFor}$_contact'; + await PrefsManager.instance.setString(key, '{broken'); + + // Unreconciled: nothing was loaded before the first save. + await store.saveMessages(_contact, [_dm(0)]); + expect((await store.loadMessages(_contact)).map((m) => m.text), ['m0']); + + // Reconciled to zero by a load that found nothing readable. + await PrefsManager.instance.setString(key, '{broken'); + store.setArchivedCount(_contact, 0); + await store.saveMessages(_contact, [_dm(1)]); + expect((await store.loadMessages(_contact)).map((m) => m.text), ['m1']); }); test('an unreconciled save merges with what is stored', () async { @@ -100,8 +114,8 @@ void main() { test('clearMessages wipes archived history too', () async { await _init(); final store = _messageStore(); - store.setArchived(_contact, [_dm(0), _dm(1)]); - await store.saveMessages(_contact, [_dm(2)]); + await store.saveMessages(_contact, [_dm(0), _dm(1), _dm(2)]); + store.setArchivedCount(_contact, 2); await store.clearMessages(_contact); await store.saveMessages(_contact, [_dm(3)]); @@ -114,7 +128,10 @@ void main() { await _init(); final store = _messageStore(); const cap = MessageStore.maxStoredMessages; - store.setArchived(_contact, [for (var i = 0; i < cap; i++) _dm(i)]); + await store.saveMessages(_contact, [ + for (var i = 0; i < cap; i++) _dm(i), + ]); + store.setArchivedCount(_contact, cap); await store.saveMessages(_contact, [_dm(cap), _dm(cap + 1)]); final reloaded = await store.loadMessages(_contact); @@ -184,7 +201,7 @@ void main() { await store.saveChannelMessages(_channelId, all); final loaded = await store.loadChannelMessages(_channelId); - store.setArchived(_channelId, loaded.sublist(0, 5)); + store.setArchivedCount(_channelId, 5); await store.saveChannelMessages(_channelId, [ ...loaded.sublist(5), _ch(8), @@ -197,8 +214,8 @@ void main() { test('clearing through another instance resets the archive', () async { await _init(); final store = ChannelMessageStore()..setPublicKeyHex = _device; - store.setArchived(_channelId, [_ch(0), _ch(1)]); - await store.saveChannelMessages(_channelId, [_ch(2)]); + await store.saveChannelMessages(_channelId, [_ch(0), _ch(1), _ch(2)]); + store.setArchivedCount(_channelId, 2); await (ChannelMessageStore()..setPublicKeyHex = _device) .clearChannelMessages(_channelId); diff --git a/test/utils/coalesced_writes_test.dart b/test/utils/coalesced_writes_test.dart index 3e014311..e888414a 100644 --- a/test/utils/coalesced_writes_test.dart +++ b/test/utils/coalesced_writes_test.dart @@ -1,3 +1,5 @@ +import 'dart:async'; + import 'package:flutter_test/flutter_test.dart'; import 'package:meshcore_open/utils/coalesced_writes.dart'; @@ -63,4 +65,44 @@ void main() { await writes.flush(); expect(ran, isTrue); }); + + test('flush waits for writes already started by the timer', () async { + final writes = CoalescedWrites(delay: Duration.zero); + final started = Completer(); + final persisted = Completer(); + writes.schedule('history', () { + started.complete(); + return persisted.future; + }); + await started.future; + + var finished = false; + final flush = writes.flush().then((_) => finished = true); + await _wait(0); + expect(finished, isFalse); + persisted.complete(); + await flush; + expect(finished, isTrue); + }); + + test( + 'a later flush waits for both existing and newly queued writes', + () async { + final writes = CoalescedWrites(); + final first = Completer(); + final second = Completer(); + writes.schedule('first', () => first.future); + final firstFlush = writes.flush(); + writes.schedule('second', () => second.future); + var finished = false; + final secondFlush = writes.flush().then((_) => finished = true); + + second.complete(); + await _wait(0); + expect(finished, isFalse); + first.complete(); + await Future.wait([firstFlush, secondFlush]); + expect(finished, isTrue); + }, + ); }