From a9d369f42fa83fe9a2ed316a3887327f7eab4fc7 Mon Sep 17 00:00:00 2001 From: zjs81 Date: Fri, 6 Nov 2026 00:54:09 -0700 Subject: [PATCH] Address message filter review findings False positives - Same-sender repeats need at least 3 words and 10 characters, so short replies ("ok", "test") repeated within 10 minutes stay visible. - Repeated characters needs a run of 20+ of one non-digit character; "Noooooooo!", separators, a row of emoji and long numbers are chat. Correctness - Rules apply to stored messages: adding a rule hides matching messages already in the chat, and removing or disabling one shows again what only it was hiding. Other filter reasons are left alone. Also applied when history is loaded. - The filter runs after mesh dedup, inside _addChannelMessage, so each message is evaluated and counted by the duplicate tracker once. - Channels excluded from filtering no longer feed implicit clean examples to the classifier. - Unread counting and the unread anchor treat filtered messages as normal when filtering is off for the channel. Performance - Classifier training runs in a background isolate (injectable trainer for widget tests); app start-up no longer waits for it. - Reply previews look up the quoted message through an id index instead of scanning the channel per bubble. - The latest-messages channel sort computes each channel's latest visible message once. Cleanup - Use MessageFilterText.isStructuredPayload instead of a screen copy. Docs: message-filter.md explains how each check works, including the learned classifier, and drops the outdated "new messages only" limit. --- documentation/message-filter.md | 48 +++-- lib/connector/meshcore_connector.dart | 191 ++++++++++++++---- lib/l10n/app_bg.arb | 2 +- lib/l10n/app_de.arb | 2 +- lib/l10n/app_en.arb | 2 +- lib/l10n/app_es.arb | 2 +- lib/l10n/app_fr.arb | 2 +- lib/l10n/app_hu.arb | 2 +- lib/l10n/app_it.arb | 2 +- lib/l10n/app_ja.arb | 2 +- lib/l10n/app_ko.arb | 2 +- lib/l10n/app_localizations.dart | 2 +- lib/l10n/app_localizations_bg.dart | 2 +- lib/l10n/app_localizations_de.dart | 2 +- lib/l10n/app_localizations_en.dart | 2 +- lib/l10n/app_localizations_es.dart | 2 +- lib/l10n/app_localizations_fr.dart | 2 +- lib/l10n/app_localizations_hu.dart | 2 +- lib/l10n/app_localizations_it.dart | 2 +- lib/l10n/app_localizations_ja.dart | 2 +- lib/l10n/app_localizations_ko.dart | 2 +- lib/l10n/app_localizations_nl.dart | 2 +- lib/l10n/app_localizations_pl.dart | 2 +- lib/l10n/app_localizations_pt.dart | 2 +- lib/l10n/app_localizations_ru.dart | 2 +- lib/l10n/app_localizations_sk.dart | 2 +- lib/l10n/app_localizations_sl.dart | 2 +- lib/l10n/app_localizations_sv.dart | 2 +- lib/l10n/app_localizations_uk.dart | 2 +- lib/l10n/app_localizations_zh.dart | 2 +- lib/l10n/app_nl.arb | 2 +- lib/l10n/app_pl.arb | 2 +- lib/l10n/app_pt.arb | 2 +- lib/l10n/app_ru.arb | 2 +- lib/l10n/app_sk.arb | 2 +- lib/l10n/app_sl.arb | 2 +- lib/l10n/app_sv.arb | 2 +- lib/l10n/app_uk.arb | 2 +- lib/l10n/app_zh.arb | 2 +- lib/screens/channel_chat_screen.dart | 71 +++++-- lib/screens/channels_screen.dart | 19 +- .../message_filter/duplicate_tracker.dart | 15 +- .../message_filter_service.dart | 174 ++++++++++++---- .../message_filter/message_filter_text.dart | 19 +- .../message_filter/ngram_spam_classifier.dart | 83 +++++++- .../connector/message_filter_ingest_test.dart | 60 ++++++ .../message_filter_settings_screen_test.dart | 5 +- .../duplicate_tracker_test.dart | 33 ++- .../message_filter_service_test.dart | 72 ++++++- .../message_filter_text_test.dart | 20 +- 50 files changed, 703 insertions(+), 181 deletions(-) diff --git a/documentation/message-filter.md b/documentation/message-filter.md index 5de465df..072896eb 100644 --- a/documentation/message-filter.md +++ b/documentation/message-filter.md @@ -17,23 +17,47 @@ The message filter hides spam and repeated messages in channel chats. It runs en ## How it works -Each new incoming channel message is checked in this order. The first check that matches decides: +### When a message is checked + +A channel message is usually heard more than once: the same packet arrives over several repeater paths, and the radio may deliver it both as a channel message and in its RX log. The app first merges all of those copies into one message, and only then runs the filter, **once per message**. Repeats over the mesh are never mistaken for someone sending the same thing twice. + +The message is then checked in this order. The first check that matches decides, and the message is stored with that reason. It is never deleted. 1. **Sender rule**: one of your rules matches the sender's name. 2. **Text rule**: one of your rules matches the message text. -3. **Repeated characters** (automatic): the same character 8 or more times in a row, such as "aaaaaaaa" or "!!!!!!!!". +3. **Repeated characters** (automatic): one character repeated 20 or more times in a row, such as "aaaaaaaaaaaaaaaaaaaa". Digits and spaces don't count, so long numbers (frequencies, coordinates) are safe. Shorter runs, like "Noooooooo!", a `--------` separator or a row of emoji, are ordinary chat and are left alone. 4. **Repeated messages** (automatic): - - The **same sender** sends the same message again within 10 minutes. + - The **same sender** sends the same message again within 10 minutes. This only applies to messages of at least 3 words and 10 characters, because people naturally repeat short replies such as "ok", "yes" or "test". - The **same text from 3 or more different senders** within 30 minutes. This only applies to longer messages, at least 10 characters and at least 6 words, so short greetings such as "Happy New Year!" are never caught. -5. **Learned from your Spam taps** (automatic): the app compares the message with messages you marked as spam. This needs at least 2 messages marked as spam and 80% confidence. +5. **Learned from your Spam taps** (automatic): a small classifier scores how much the message looks like the messages you marked as spam (see below). It needs at least 2 messages marked as spam, and it hides a message only when it is at least 80% confident. -Details worth knowing: +### Matching text -- Time windows use the time the message was **sent**, not when it reached your phone. History that is synced after you reconnect is not mistaken for a flood. -- Rules ignore capital letters, accents, zero-width (invisible) characters and look-alike letters (for example Cyrillic or Greek letters that look like Latin ones, or fullwidth letters), so spammers cannot easily sneak around a rule. -- Repeated-message checks also ignore punctuation, emoji and `@[name]` mentions. +Before comparing, both your rule and the message are normalised: + +- Capital letters, accents and zero-width (invisible) characters are ignored. +- Look-alike letters are treated as the letter they imitate: Cyrillic or Greek letters that look Latin, fullwidth letters, and "styled" letters such as 𝐛𝐨𝐥𝐝 or 𝘪𝘵𝘢𝘭𝘪𝘤. Swapping in look-alike letters doesn't get a message past a rule. +- Repeated-message checks also ignore punctuation, emoji and `@[name]` mentions, so "Meet at the park!" and "meet at the park" count as the same text. - A reply's leading `@[name]` mention is not part of the text that is checked. +### Repeated-message memory + +The app keeps a short in-memory list of recent messages for each channel: up to 1,000 per channel, for 30 minutes. Each new message is compared against it. Every checked message is added, including ones a rule already hid, so a flood keeps being caught even after you add a rule for part of it. + +Time windows use the time the message was **sent**, not when it reached your phone. History synced in a burst after you reconnect was sent hours apart, so it is not mistaken for a flood. Old entries are forgotten by the time they *arrived*, so a sender with a wrong clock can't push everyone else out of the list. + +### Learning from your taps + +The learned check is a small statistical model, a class-balanced logistic regression. It runs entirely in the app; there is nothing to download. + +- **What it learns from:** + - Every message you mark as spam, up to 200. + - Every message you mark as not spam, up to 200. + - Up to 300 recent ordinary messages that weren't filtered. These implicit "clean" examples are taken only from channels where filtering is on. Channels you have switched filtering off for are ignored, because their unfiltered messages may well include spam. +- **What it looks at:** short letter sequences (3 to 5 characters within each word), whole words and word pairs, and the sender's name. All of these are normalised the same way as rules. So it recognises a promo that has been reworded or re-spaced, not just an exact copy. +- **When it trains:** whenever you tap Spam or Not spam, when you turn learning on, and when the app starts. Training runs in the background, so the app never freezes while it learns. The previous model keeps working until the new one is ready. +- **Reset learning** forgets every example; your rules are kept. + ## Marking messages yourself Long-press (or right-click) an incoming channel message: @@ -66,13 +90,13 @@ Under **Message filter → Filtered messages** choose one of two display modes: - **Show a small placeholder** (default): hidden messages are replaced by a small line, "Hidden message · Tap to view". Several hidden messages in a row collapse into one line, for example "3 hidden messages · Tap to view". - Tap the line to open it with a short animation. The header then reads "Showing hidden message · Tap to hide"; tap it again to close. - Each opened message shows why it was filtered (for example `Filtered: matches "meshos"`, `Filtered: repeated message` or `Filtered: you marked this as spam`) with **Not spam** and **Hide** buttons. -- **Hide completely**: nothing is shown in the chat. "Hidden messages stay in the Filtered messages list." +- **Hide completely**: nothing is shown in the chat. "Hidden messages stay in the Filtered messages list." If hidden messages leave too little in a chat to scroll, the app loads older history automatically, up to 20 pages, so earlier conversation stays reachable after a spam burst. If the chat is still empty after that, a **Load older messages** button appears. The **Filtered messages (n)** entry on the settings screen opens a list of everything that was filtered, newest first, across all channels. Tap a row to view it, or tap **Not spam** to restore it. Other effects of filtering: -- Filtered messages never count as **unread** and never send a **notification**. +- Filtered messages never count as **unread** and never send a **notification**. This includes a message you mark as spam while it is still being translated: it won't notify when translation finishes. In a channel where filtering is off, earlier-filtered messages show and count as ordinary messages. - They are not **auto-translated**. - They are not used as the **last-message preview** in the Channels list; the newest message that was not hidden is shown instead. - If someone replies to a hidden message, the quoted text in the reply is concealed ("Hidden message · Tap to view") until the original is opened. @@ -85,7 +109,7 @@ Other effects of filtering: | **Show a small placeholder** / **Hide completely** | Placeholder | How filtered messages appear in chat. | | **Your rules** | None | Your own rules (see above). | | **Repeated messages** | On | "The same message sent again, or by several senders". | -| **Repeated characters** | On | Messages like "aaaaaaaa" or "!!!!!!!!". | +| **Repeated characters** | On | Long runs of one character, like "aaaaaaaaaaaaaaaaaaaa". | | **Learn from Spam taps** | On | Shows how many "spam and not-spam examples" the app is using. | | **Reset learning** | | Asks for confirmation, then forgets everything learned from your Spam and Not spam taps. Your rules are kept. | @@ -100,7 +124,7 @@ Rules, settings and the examples used for learning are stored on your device in ## Limitations - **Channels only.** Direct messages and room chats are not filtered. -- **New messages only.** A new rule or a changed switch applies to messages received from then on, not to messages already in the chat. Use Mark as spam on an old message to hide it. +- **Rules apply to existing messages; automatic checks don't.** Adding, editing, turning off or deleting a rule updates messages already in your chats: "Filter this sender" also hides that sender's earlier messages, and deleting a rule shows again what only that rule was hiding. The automatic checks (repeats, repeated characters, learning) only judge messages as they arrive. Use Mark as spam on an old message to hide it. - **Reveal lasts while the chat is open.** Messages you open stay open until you tap the "Tap to hide" header (or Hide on one of them) or leave the chat; they collapse again next time. - **Learning is per device.** It is not shared between your phones or computers, and it needs at least 2 Spam taps before it does anything. - **Repeat tracking is not remembered.** The recent-message memory used for repeat checks is kept only while the app is running. diff --git a/lib/connector/meshcore_connector.dart b/lib/connector/meshcore_connector.dart index ae9aab36..0b8ff9f8 100644 --- a/lib/connector/meshcore_connector.dart +++ b/lib/connector/meshcore_connector.dart @@ -329,6 +329,10 @@ class MeshCoreConnector extends ChangeNotifier { TimeoutPredictionService? _timeoutPredictionService; TranslationService? _translationService; MessageFilterService? _messageFilterService; + + /// Rules revision last applied to stored messages (see + /// [_onMessageFilterChanged]). + int _appliedFilterRulesRevision = 0; ImageCodecService? _imageCodecService; // Holds the ImageReassembler, so partially received images survive across // frames (chunks of one image arrive in separate GRP_DATA frames, and via the @@ -1089,13 +1093,88 @@ class MeshCoreConnector extends ChangeNotifier { if (service == null || !isNew) return; if (message.isOutgoing || message.isFiltered) return; if (message.parseReaction() != null) return; - // The service ignores structured payloads (GIFs etc.) itself. + // The service ignores structured payloads (GIFs etc.) and channels + // excluded from filtering itself. service.recordClean( + channelId: _messageFilterChannelIdForSlot(message.channelIndex!), sender: message.senderName, text: _messageFilterText(message), ); } + /// [incoming] carrying the filter tag the stored copy got, if any. + /// + /// The filter runs inside [_addChannelMessage] on the stored copy (only for + /// new messages), while callers keep using the message as received (with + /// its reply mention) for translation and notifications. + static ChannelMessage _withFilterOf( + ChannelMessage incoming, + ChannelMessage? stored, + ) { + if (stored == null || !stored.isFiltered) return incoming; + return incoming.copyWith( + filterReason: stored.filterReason, + filterDetail: stored.filterDetail, + ); + } + + void _onMessageFilterChanged() { + final service = _messageFilterService; + if (service == null || + service.rulesRevision == _appliedFilterRulesRevision) { + return; + } + _appliedFilterRulesRevision = service.rulesRevision; + var changed = false; + _channelMessages.forEach((slot, messages) { + if (_applyFilterRules(slot, messages)) { + _scheduleChannelSave(slot); + changed = true; + } + }); + if (changed) notifyListeners(); + } + + /// Brings rule tags on stored [messages] of [slot] in line with the current + /// rules, in place: a new rule hides messages that already arrived, and a + /// deleted or disabled rule shows again what only it was hiding. Tags from + /// other reasons (duplicates, learned, manual spam) are left alone. Returns + /// whether anything changed. + bool _applyFilterRules(int slot, List messages) { + final service = _messageFilterService; + if (service == null || messages.isEmpty) return false; + final channelId = _messageFilterChannelIdForSlot(slot); + if (!service.isChannelFiltered(channelId)) return false; + var changed = false; + for (var i = 0; i < messages.length; i++) { + final message = messages[i]; + if (message.isOutgoing || message.parseReaction() != null) continue; + final reason = filterReasonFromJson(message.filterReason); + final ruleTagged = + reason == FilterReason.senderRule || reason == FilterReason.textRule; + if (message.isFiltered && !ruleTagged) continue; + final verdict = service.ruleVerdict( + channelId: channelId, + sender: message.senderName, + text: _messageFilterText(message), + ); + if (verdict == null) { + if (ruleTagged) { + messages[i] = message.copyWith(clearFilter: true); + changed = true; + } + } else if (reason != verdict.reason || + message.filterDetail != verdict.detail) { + messages[i] = applyFilterVerdict( + message.copyWith(clearFilter: true), + verdict, + ); + changed = true; + } + } + return changed; + } + /// Finds the stored message with [messageId] in [channel] and replaces it /// with [update]'s result, scheduling a save. Returns false when the /// message is not loaded. @@ -1845,6 +1924,10 @@ class MeshCoreConnector extends ChangeNotifier { // Replace even when nothing is stored so another radio's messages never // linger under this index. final hadMessages = _channelMessages[channelIndex]?.isNotEmpty ?? false; + // Rules may have changed since these were stored. + if (_applyFilterRules(channelIndex, windowedMessages)) { + _scheduleChannelSave(channelIndex); + } _channelMessages[channelIndex] = windowedMessages; if (hadMessages || windowedMessages.isNotEmpty) notifyListeners(); } @@ -1880,8 +1963,11 @@ class MeshCoreConnector extends ChangeNotifier { ); _channelMessageStore.unarchiveNewest(channelId, olderMessages.length); + // Rules may have changed since these were stored. + final retagged = _applyFilterRules(slot, olderMessages); // Prepend to current conversation _channelMessages[slot] = [...olderMessages, ...currentMessages]; + if (retagged) _scheduleChannelSave(slot); notifyListeners(); return olderMessages; @@ -1927,7 +2013,10 @@ class MeshCoreConnector extends ChangeNotifier { _lastShowLinkImagesByDefault = _showLinkImagesByDefault; appSettingsService?.addListener(_onAppSettingsChanged); _translationService = translationService; + _messageFilterService?.removeListener(_onMessageFilterChanged); _messageFilterService = messageFilterService; + _appliedFilterRulesRevision = messageFilterService?.rulesRevision ?? 0; + messageFilterService?.addListener(_onMessageFilterChanged); _imageCodecService = imageCodecService; _imageTransport = imageTransport; _receivedImageStore = receivedImageStore; @@ -7874,17 +7963,25 @@ class MeshCoreConnector extends ChangeNotifier { parsed.timestamp.millisecondsSinceEpoch ~/ 1000, '${parsed.senderName}: ${parsed.text}', ); - final message = _applyMessageFilter( - _messageFilterChannelIdForSlot(parsed.channelIndex!), - parsed.copyWith(packetHash: contentHash), + final incoming = parsed.copyWith(packetHash: contentHash); + final filterChannelId = _messageFilterChannelIdForSlot( + parsed.channelIndex!, ); _updateContactLastMessageAtByName( - message.senderName, - message.timestamp, - pathBytes: message.pathBytes, - pathHashWidth: message.pathHashWidth, + incoming.senderName, + incoming.timestamp, + pathBytes: incoming.pathBytes, + pathHashWidth: incoming.pathHashWidth, ); - final isNew = _addChannelMessage(message.channelIndex!, message); + // Filtered after dedup, so mesh repeats of one message are evaluated + // (and counted by the duplicate tracker) once. + final stored = _addChannelMessage( + incoming.channelIndex!, + incoming, + filterIfNew: (m) => _applyMessageFilter(filterChannelId, m), + ); + final isNew = stored != null; + final message = _withFilterOf(incoming, stored); _recordCleanChannelMessage(message, isNew: isNew); // Filtered messages are stored but never count as unread, notify or // get auto-translated. @@ -7964,34 +8061,38 @@ class MeshCoreConnector extends ChangeNotifier { final pktHash = computePacketHash(packet.payloadType, packet.payload); - final message = _applyMessageFilter( - messageFilterChannelId(channel), - ChannelMessage( - senderKey: null, - senderName: parsed.senderName, - text: decodedText, - timestamp: DateTime.fromMillisecondsSinceEpoch( - timestampRaw * 1000, - ), - receivedAt: DateTime.now(), - isOutgoing: false, - status: ChannelMessageStatus.sent, - pathLength: packet.isFlood ? packet.hopCount : 0, - pathHashWidth: packet.pathHashWidth, - pathBytes: packet.pathBytes, - channelIndex: slot, - region: _resolveTransportRegion(packet), - packetHash: pktHash, - ), - ); + final incoming = (ChannelMessage( + senderKey: null, + senderName: parsed.senderName, + text: decodedText, + timestamp: DateTime.fromMillisecondsSinceEpoch(timestampRaw * 1000), + receivedAt: DateTime.now(), + isOutgoing: false, + status: ChannelMessageStatus.sent, + pathLength: packet.isFlood ? packet.hopCount : 0, + pathHashWidth: packet.pathHashWidth, + pathBytes: packet.pathBytes, + channelIndex: slot, + region: _resolveTransportRegion(packet), + packetHash: pktHash, + )); _updateContactLastMessageAtByName( parsed.senderName, - message.timestamp, - pathBytes: message.pathBytes, - pathHashWidth: message.pathHashWidth, + incoming.timestamp, + pathBytes: incoming.pathBytes, + pathHashWidth: incoming.pathHashWidth, ); - final isNew = _addChannelMessage(slot, message); + final filterChannelId = messageFilterChannelId(channel); + // Filtered after dedup, so mesh repeats of one message are + // evaluated (and counted by the duplicate tracker) once. + final stored = _addChannelMessage( + slot, + incoming, + filterIfNew: (m) => _applyMessageFilter(filterChannelId, m), + ); + final isNew = stored != null; + final message = _withFilterOf(incoming, stored); _recordCleanChannelMessage(message, isNew: isNew); // Filtered messages are stored but never count as unread, notify // or get auto-translated. @@ -8708,7 +8809,15 @@ class MeshCoreConnector extends ChangeNotifier { return _ParsedText(senderName: 'Unknown', text: text); } - bool _addChannelMessage(int slot, ChannelMessage message) { + /// Stores [message] in [slot], merging it into an existing copy when it is a + /// mesh repeat. Returns the stored message when it was new, else null. + /// + /// [filterIfNew] runs on new messages only, just before they are stored. + ChannelMessage? _addChannelMessage( + int slot, + ChannelMessage message, { + ChannelMessage Function(ChannelMessage message)? filterIfNew, + }) { final channelIndex = _canonicalSlot(slot); _channelMessages.putIfAbsent(channelIndex, () => []); final messages = _channelMessages[channelIndex]!; @@ -8724,7 +8833,7 @@ class MeshCoreConnector extends ChangeNotifier { reactionIdentifier, ); - if (isDuplicate) return false; + if (isDuplicate) return null; // New reaction - process it final reactionApplied = _processReaction(messages, reactionInfo); @@ -8735,7 +8844,7 @@ class MeshCoreConnector extends ChangeNotifier { // Mark as processed _processedChannelReactions[channelIndex]!.add(reactionIdentifier); - return false; // Don't add reaction as a visible message + return null; // Don't add reaction as a visible message } // Looks like a reaction, but didn't match any message we have. // Old behavior is to silently drop these. We'll continue doing that @@ -8743,7 +8852,7 @@ class MeshCoreConnector extends ChangeNotifier { // informative on their own, but the MC1 format is useful even if // in raw form, so we'll let those through. appLogger.info('No match for reaction, id: $reactionIdentifier'); - if (reactionInfo.hashType == HashType.ours) return false; + if (reactionInfo.hashType == HashType.ours) return null; } // Parse reply info from message text @@ -8795,9 +8904,8 @@ class MeshCoreConnector extends ChangeNotifier { } final existingIndex = _findChannelRepeatIndex(messages, processedMessage); - var isNew = true; + ChannelMessage? added; if (existingIndex >= 0) { - isNew = false; final existing = messages[existingIndex]; final mergedPathBytes = _selectPreferredPathBytes( existing.pathBytes, @@ -8837,6 +8945,8 @@ class MeshCoreConnector extends ChangeNotifier { _pendingChannelSentQueue.remove(existing.messageId); } } else { + if (filterIfNew != null) processedMessage = filterIfNew(processedMessage); + added = processedMessage; messages.add(processedMessage); // Storage keeps only the newest maxStoredMessages per channel, so older // entries are gone after the next save anyway. Without this cap a long @@ -8846,7 +8956,7 @@ class MeshCoreConnector extends ChangeNotifier { } _scheduleChannelSave(channelIndex); - return isNew; + return added; } ChannelMessage? _findMessageBySender( @@ -9219,6 +9329,7 @@ class MeshCoreConnector extends ChangeNotifier { @override void dispose() { + _messageFilterService?.removeListener(_onMessageFilterChanged); _disposed = true; _appSettingsService?.removeListener(_onAppSettingsChanged); _scanSubscription?.cancel(); diff --git a/lib/l10n/app_bg.arb b/lib/l10n/app_bg.arb index c3ce2bc8..84e99a08 100644 --- a/lib/l10n/app_bg.arb +++ b/lib/l10n/app_bg.arb @@ -3559,7 +3559,7 @@ "messageFilter_duplicates": "Повтарящи се съобщения", "messageFilter_duplicatesSubtitle": "Същото съобщение, изпратено отново или от няколко подателя", "messageFilter_repeatedCharacters": "Повтарящи се символи", - "messageFilter_repeatedCharactersSubtitle": "Съобщения като „aaaaaaaa“ или „!!!!!!!!“", + "messageFilter_repeatedCharactersSubtitle": "Дълги поредици от един и същ знак, като „aaaaaaaaaaaaaaaaaaaa“", "messageFilter_learning": "Учи се от натискането на „Спам“", "messageFilter_learningSubtitle": "{spam} примера за спам и {notSpam} примера, които не са спам", "messageFilter_resetLearning": "Нулирай наученото", diff --git a/lib/l10n/app_de.arb b/lib/l10n/app_de.arb index fcf9345d..5034f071 100644 --- a/lib/l10n/app_de.arb +++ b/lib/l10n/app_de.arb @@ -3694,7 +3694,7 @@ "messageFilter_duplicates": "Wiederholte Nachrichten", "messageFilter_duplicatesSubtitle": "Dieselbe Nachricht erneut gesendet oder von mehreren Absendern", "messageFilter_repeatedCharacters": "Wiederholte Zeichen", - "messageFilter_repeatedCharactersSubtitle": "Nachrichten wie „aaaaaaaa“ oder „!!!!!!!!“", + "messageFilter_repeatedCharactersSubtitle": "Lange Folgen desselben Zeichens, z. B. „aaaaaaaaaaaaaaaaaaaa“", "messageFilter_learning": "Aus Spam-Markierungen lernen", "messageFilter_learningSubtitle": "{spam} Spam- und {notSpam} Kein-Spam-Beispiele", "messageFilter_resetLearning": "Gelerntes zurücksetzen", diff --git a/lib/l10n/app_en.arb b/lib/l10n/app_en.arb index 4fba0e56..6afcb015 100644 --- a/lib/l10n/app_en.arb +++ b/lib/l10n/app_en.arb @@ -3869,7 +3869,7 @@ "@messageFilter_repeatedCharacters": { "description": "Message filter: Repeated characters" }, - "messageFilter_repeatedCharactersSubtitle": "Messages like \"aaaaaaaa\" or \"!!!!!!!!\"", + "messageFilter_repeatedCharactersSubtitle": "Long runs of one character, like \"aaaaaaaaaaaaaaaaaaaa\"", "@messageFilter_repeatedCharactersSubtitle": { "description": "Message filter: Messages like \"aaaaaaaa\" or \"!!!!!!!!\"" }, diff --git a/lib/l10n/app_es.arb b/lib/l10n/app_es.arb index 459af4f5..750421ae 100644 --- a/lib/l10n/app_es.arb +++ b/lib/l10n/app_es.arb @@ -3693,7 +3693,7 @@ "messageFilter_duplicates": "Mensajes repetidos", "messageFilter_duplicatesSubtitle": "El mismo mensaje enviado de nuevo, o por varios remitentes", "messageFilter_repeatedCharacters": "Caracteres repetidos", - "messageFilter_repeatedCharactersSubtitle": "Mensajes como \"aaaaaaaa\" o \"!!!!!!!!\"", + "messageFilter_repeatedCharactersSubtitle": "Secuencias largas de un mismo carácter, como \"aaaaaaaaaaaaaaaaaaaa\"", "messageFilter_learning": "Aprender de los toques en Spam", "messageFilter_learningSubtitle": "{spam} ejemplos de spam y {notSpam} de no spam", "@messageFilter_learningSubtitle": { diff --git a/lib/l10n/app_fr.arb b/lib/l10n/app_fr.arb index 3b29c893..ecaae160 100644 --- a/lib/l10n/app_fr.arb +++ b/lib/l10n/app_fr.arb @@ -3693,7 +3693,7 @@ "messageFilter_duplicates": "Messages répétés", "messageFilter_duplicatesSubtitle": "Le même message envoyé de nouveau, ou par plusieurs expéditeurs", "messageFilter_repeatedCharacters": "Caractères répétés", - "messageFilter_repeatedCharactersSubtitle": "Messages comme « aaaaaaaa » ou « !!!!!!!! »", + "messageFilter_repeatedCharactersSubtitle": "Longues séquences d'un même caractère, comme « aaaaaaaaaaaaaaaaaaaa »", "messageFilter_learning": "Apprendre des appuis sur Spam", "messageFilter_learningSubtitle": "{spam} exemples de spam et {notSpam} de non-spam", "messageFilter_resetLearning": "Réinitialiser l'apprentissage", diff --git a/lib/l10n/app_hu.arb b/lib/l10n/app_hu.arb index bc83d91a..f186fd7e 100644 --- a/lib/l10n/app_hu.arb +++ b/lib/l10n/app_hu.arb @@ -3598,7 +3598,7 @@ "messageFilter_duplicates": "Ismétlődő üzenetek", "messageFilter_duplicatesSubtitle": "Ugyanaz az üzenet újra elküldve, vagy több feladótól", "messageFilter_repeatedCharacters": "Ismétlődő karakterek", - "messageFilter_repeatedCharactersSubtitle": "Olyan üzenetek, mint „aaaaaaaa” vagy „!!!!!!!!”", + "messageFilter_repeatedCharactersSubtitle": "Ugyanazon karakter hosszú sorozata, például „aaaaaaaaaaaaaaaaaaaa”", "messageFilter_learning": "Tanulás a Spam gombokból", "messageFilter_learningSubtitle": "{spam} spam és {notSpam} nem spam példa", "@messageFilter_learningSubtitle": { diff --git a/lib/l10n/app_it.arb b/lib/l10n/app_it.arb index 276a00f5..1daa666c 100644 --- a/lib/l10n/app_it.arb +++ b/lib/l10n/app_it.arb @@ -3693,7 +3693,7 @@ "messageFilter_duplicates": "Messaggi ripetuti", "messageFilter_duplicatesSubtitle": "Lo stesso messaggio inviato di nuovo, o da più mittenti", "messageFilter_repeatedCharacters": "Caratteri ripetuti", - "messageFilter_repeatedCharactersSubtitle": "Messaggi come «aaaaaaaa» o «!!!!!!!!»", + "messageFilter_repeatedCharactersSubtitle": "Lunghe sequenze dello stesso carattere, come «aaaaaaaaaaaaaaaaaaaa»", "messageFilter_learning": "Impara dai tocchi su Spam", "messageFilter_learningSubtitle": "{spam} esempi di spam e {notSpam} di non spam", "messageFilter_resetLearning": "Azzera apprendimento", diff --git a/lib/l10n/app_ja.arb b/lib/l10n/app_ja.arb index 943e2fed..a09658cc 100644 --- a/lib/l10n/app_ja.arb +++ b/lib/l10n/app_ja.arb @@ -3777,7 +3777,7 @@ "@messageFilter_repeatedCharacters": { "description": "Message filter: Repeated characters" }, - "messageFilter_repeatedCharactersSubtitle": "「aaaaaaaa」や「!!!!!!!!」のようなメッセージ", + "messageFilter_repeatedCharactersSubtitle": "同じ文字の長い連続(例:「aaaaaaaaaaaaaaaaaaaa」)", "@messageFilter_repeatedCharactersSubtitle": { "description": "Message filter: Messages like \"aaaaaaaa\" or \"!!!!!!!!\"" }, diff --git a/lib/l10n/app_ko.arb b/lib/l10n/app_ko.arb index f90faf16..d8cb0f6c 100644 --- a/lib/l10n/app_ko.arb +++ b/lib/l10n/app_ko.arb @@ -3777,7 +3777,7 @@ "@messageFilter_repeatedCharacters": { "description": "Message filter: Repeated characters" }, - "messageFilter_repeatedCharactersSubtitle": "\"aaaaaaaa\" 또는 \"!!!!!!!!\" 같은 메시지", + "messageFilter_repeatedCharactersSubtitle": "같은 문자가 길게 반복되는 경우, 예: \"aaaaaaaaaaaaaaaaaaaa\"", "@messageFilter_repeatedCharactersSubtitle": { "description": "Message filter: Messages like \"aaaaaaaa\" or \"!!!!!!!!\"" }, diff --git a/lib/l10n/app_localizations.dart b/lib/l10n/app_localizations.dart index 3e9d8589..1d6a9ca1 100644 --- a/lib/l10n/app_localizations.dart +++ b/lib/l10n/app_localizations.dart @@ -11668,7 +11668,7 @@ abstract class AppLocalizations { /// Message filter: Messages like "aaaaaaaa" or "!!!!!!!!" /// /// In en, this message translates to: - /// **'Messages like \"aaaaaaaa\" or \"!!!!!!!!\"'** + /// **'Long runs of one character, like \"aaaaaaaaaaaaaaaaaaaa\"'** String get messageFilter_repeatedCharactersSubtitle; /// Message filter: Learn from Spam taps diff --git a/lib/l10n/app_localizations_bg.dart b/lib/l10n/app_localizations_bg.dart index 0f7467ff..e0441251 100644 --- a/lib/l10n/app_localizations_bg.dart +++ b/lib/l10n/app_localizations_bg.dart @@ -6948,7 +6948,7 @@ class AppLocalizationsBg extends AppLocalizations { @override String get messageFilter_repeatedCharactersSubtitle => - 'Съобщения като „aaaaaaaa“ или „!!!!!!!!“'; + 'Дълги поредици от един и същ знак, като „aaaaaaaaaaaaaaaaaaaa“'; @override String get messageFilter_learning => 'Учи се от натискането на „Спам“'; diff --git a/lib/l10n/app_localizations_de.dart b/lib/l10n/app_localizations_de.dart index 33b2fbf6..bc2a2ae3 100644 --- a/lib/l10n/app_localizations_de.dart +++ b/lib/l10n/app_localizations_de.dart @@ -6959,7 +6959,7 @@ class AppLocalizationsDe extends AppLocalizations { @override String get messageFilter_repeatedCharactersSubtitle => - 'Nachrichten wie „aaaaaaaa“ oder „!!!!!!!!“'; + 'Lange Folgen desselben Zeichens, z. B. „aaaaaaaaaaaaaaaaaaaa“'; @override String get messageFilter_learning => 'Aus Spam-Markierungen lernen'; diff --git a/lib/l10n/app_localizations_en.dart b/lib/l10n/app_localizations_en.dart index bc3b0858..882ff97c 100644 --- a/lib/l10n/app_localizations_en.dart +++ b/lib/l10n/app_localizations_en.dart @@ -6844,7 +6844,7 @@ class AppLocalizationsEn extends AppLocalizations { @override String get messageFilter_repeatedCharactersSubtitle => - 'Messages like \"aaaaaaaa\" or \"!!!!!!!!\"'; + 'Long runs of one character, like \"aaaaaaaaaaaaaaaaaaaa\"'; @override String get messageFilter_learning => 'Learn from Spam taps'; diff --git a/lib/l10n/app_localizations_es.dart b/lib/l10n/app_localizations_es.dart index b73744e9..5edfb950 100644 --- a/lib/l10n/app_localizations_es.dart +++ b/lib/l10n/app_localizations_es.dart @@ -6945,7 +6945,7 @@ class AppLocalizationsEs extends AppLocalizations { @override String get messageFilter_repeatedCharactersSubtitle => - 'Mensajes como \"aaaaaaaa\" o \"!!!!!!!!\"'; + 'Secuencias largas de un mismo carácter, como \"aaaaaaaaaaaaaaaaaaaa\"'; @override String get messageFilter_learning => 'Aprender de los toques en Spam'; diff --git a/lib/l10n/app_localizations_fr.dart b/lib/l10n/app_localizations_fr.dart index 992211a1..94b63fe7 100644 --- a/lib/l10n/app_localizations_fr.dart +++ b/lib/l10n/app_localizations_fr.dart @@ -6971,7 +6971,7 @@ class AppLocalizationsFr extends AppLocalizations { @override String get messageFilter_repeatedCharactersSubtitle => - 'Messages comme « aaaaaaaa » ou « !!!!!!!! »'; + 'Longues séquences d\'un même caractère, comme « aaaaaaaaaaaaaaaaaaaa »'; @override String get messageFilter_learning => 'Apprendre des appuis sur Spam'; diff --git a/lib/l10n/app_localizations_hu.dart b/lib/l10n/app_localizations_hu.dart index 82f01c5a..9ddf8b69 100644 --- a/lib/l10n/app_localizations_hu.dart +++ b/lib/l10n/app_localizations_hu.dart @@ -6934,7 +6934,7 @@ class AppLocalizationsHu extends AppLocalizations { @override String get messageFilter_repeatedCharactersSubtitle => - 'Olyan üzenetek, mint „aaaaaaaa” vagy „!!!!!!!!”'; + 'Ugyanazon karakter hosszú sorozata, például „aaaaaaaaaaaaaaaaaaaa”'; @override String get messageFilter_learning => 'Tanulás a Spam gombokból'; diff --git a/lib/l10n/app_localizations_it.dart b/lib/l10n/app_localizations_it.dart index 118f6184..8c1e2cac 100644 --- a/lib/l10n/app_localizations_it.dart +++ b/lib/l10n/app_localizations_it.dart @@ -6955,7 +6955,7 @@ class AppLocalizationsIt extends AppLocalizations { @override String get messageFilter_repeatedCharactersSubtitle => - 'Messaggi come «aaaaaaaa» o «!!!!!!!!»'; + 'Lunghe sequenze dello stesso carattere, come «aaaaaaaaaaaaaaaaaaaa»'; @override String get messageFilter_learning => 'Impara dai tocchi su Spam'; diff --git a/lib/l10n/app_localizations_ja.dart b/lib/l10n/app_localizations_ja.dart index eb8c9c35..55683dc4 100644 --- a/lib/l10n/app_localizations_ja.dart +++ b/lib/l10n/app_localizations_ja.dart @@ -6625,7 +6625,7 @@ class AppLocalizationsJa extends AppLocalizations { @override String get messageFilter_repeatedCharactersSubtitle => - '「aaaaaaaa」や「!!!!!!!!」のようなメッセージ'; + '同じ文字の長い連続(例:「aaaaaaaaaaaaaaaaaaaa」)'; @override String get messageFilter_learning => 'スパムのタップから学習'; diff --git a/lib/l10n/app_localizations_ko.dart b/lib/l10n/app_localizations_ko.dart index 944040ae..28d6e638 100644 --- a/lib/l10n/app_localizations_ko.dart +++ b/lib/l10n/app_localizations_ko.dart @@ -6635,7 +6635,7 @@ class AppLocalizationsKo extends AppLocalizations { @override String get messageFilter_repeatedCharactersSubtitle => - '\"aaaaaaaa\" 또는 \"!!!!!!!!\" 같은 메시지'; + '같은 문자가 길게 반복되는 경우, 예: \"aaaaaaaaaaaaaaaaaaaa\"'; @override String get messageFilter_learning => '스팸 표시에서 학습'; diff --git a/lib/l10n/app_localizations_nl.dart b/lib/l10n/app_localizations_nl.dart index 196ff788..12c63d6a 100644 --- a/lib/l10n/app_localizations_nl.dart +++ b/lib/l10n/app_localizations_nl.dart @@ -6909,7 +6909,7 @@ class AppLocalizationsNl extends AppLocalizations { @override String get messageFilter_repeatedCharactersSubtitle => - 'Berichten zoals „aaaaaaaa” of „!!!!!!!!”'; + 'Lange reeksen van hetzelfde teken, zoals „aaaaaaaaaaaaaaaaaaaa”'; @override String get messageFilter_learning => 'Leren van Spam-tikken'; diff --git a/lib/l10n/app_localizations_pl.dart b/lib/l10n/app_localizations_pl.dart index 4f8bd0ba..2ce20088 100644 --- a/lib/l10n/app_localizations_pl.dart +++ b/lib/l10n/app_localizations_pl.dart @@ -6971,7 +6971,7 @@ class AppLocalizationsPl extends AppLocalizations { @override String get messageFilter_repeatedCharactersSubtitle => - 'Wiadomości typu „aaaaaaaa” lub „!!!!!!!!”'; + 'Długie ciągi jednego znaku, np. „aaaaaaaaaaaaaaaaaaaa”'; @override String get messageFilter_learning => 'Ucz się z oznaczeń Spam'; diff --git a/lib/l10n/app_localizations_pt.dart b/lib/l10n/app_localizations_pt.dart index 5e938d75..9af6fafe 100644 --- a/lib/l10n/app_localizations_pt.dart +++ b/lib/l10n/app_localizations_pt.dart @@ -6937,7 +6937,7 @@ class AppLocalizationsPt extends AppLocalizations { @override String get messageFilter_repeatedCharactersSubtitle => - 'Mensagens como \"aaaaaaaa\" ou \"!!!!!!!!\"'; + 'Sequências longas de um mesmo caractere, como \"aaaaaaaaaaaaaaaaaaaa\"'; @override String get messageFilter_learning => 'Aprender com os toques em Spam'; diff --git a/lib/l10n/app_localizations_ru.dart b/lib/l10n/app_localizations_ru.dart index f5e56e90..af2bf71c 100644 --- a/lib/l10n/app_localizations_ru.dart +++ b/lib/l10n/app_localizations_ru.dart @@ -6959,7 +6959,7 @@ class AppLocalizationsRu extends AppLocalizations { @override String get messageFilter_repeatedCharactersSubtitle => - 'Сообщения вроде «aaaaaaaa» или «!!!!!!!!»'; + 'Длинные последовательности одного символа, например «aaaaaaaaaaaaaaaaaaaa»'; @override String get messageFilter_learning => 'Учиться на нажатиях «Спам»'; diff --git a/lib/l10n/app_localizations_sk.dart b/lib/l10n/app_localizations_sk.dart index 4340e9e6..86ebeec3 100644 --- a/lib/l10n/app_localizations_sk.dart +++ b/lib/l10n/app_localizations_sk.dart @@ -6926,7 +6926,7 @@ class AppLocalizationsSk extends AppLocalizations { @override String get messageFilter_repeatedCharactersSubtitle => - 'Správy ako „aaaaaaaa“ alebo „!!!!!!!!“'; + 'Dlhé rady jedného znaku, napríklad „aaaaaaaaaaaaaaaaaaaa“'; @override String get messageFilter_learning => 'Učiť sa z klepnutí na Spam'; diff --git a/lib/l10n/app_localizations_sl.dart b/lib/l10n/app_localizations_sl.dart index 3a18c4bb..7904205e 100644 --- a/lib/l10n/app_localizations_sl.dart +++ b/lib/l10n/app_localizations_sl.dart @@ -6912,7 +6912,7 @@ class AppLocalizationsSl extends AppLocalizations { @override String get messageFilter_repeatedCharactersSubtitle => - 'Sporočila, kot sta „aaaaaaaa“ ali „!!!!!!!!“'; + 'Dolga zaporedja istega znaka, kot je „aaaaaaaaaaaaaaaaaaaa“'; @override String get messageFilter_learning => 'Uči se iz dotikov Neželeno'; diff --git a/lib/l10n/app_localizations_sv.dart b/lib/l10n/app_localizations_sv.dart index 284d8827..e556be9b 100644 --- a/lib/l10n/app_localizations_sv.dart +++ b/lib/l10n/app_localizations_sv.dart @@ -6885,7 +6885,7 @@ class AppLocalizationsSv extends AppLocalizations { @override String get messageFilter_repeatedCharactersSubtitle => - 'Meddelanden som ”aaaaaaaa” eller ”!!!!!!!!”'; + 'Långa följder av samma tecken, som ”aaaaaaaaaaaaaaaaaaaa”'; @override String get messageFilter_learning => 'Lär av tryck på Skräppost'; diff --git a/lib/l10n/app_localizations_uk.dart b/lib/l10n/app_localizations_uk.dart index 8a42fa68..cb38f11c 100644 --- a/lib/l10n/app_localizations_uk.dart +++ b/lib/l10n/app_localizations_uk.dart @@ -6966,7 +6966,7 @@ class AppLocalizationsUk extends AppLocalizations { @override String get messageFilter_repeatedCharactersSubtitle => - 'Повідомлення на кшталт «aaaaaaaa» або «!!!!!!!!»'; + 'Довгі послідовності одного символа, наприклад «aaaaaaaaaaaaaaaaaaaa»'; @override String get messageFilter_learning => 'Навчання на натисканнях «Спам»'; diff --git a/lib/l10n/app_localizations_zh.dart b/lib/l10n/app_localizations_zh.dart index 7f84774a..93877152 100644 --- a/lib/l10n/app_localizations_zh.dart +++ b/lib/l10n/app_localizations_zh.dart @@ -6472,7 +6472,7 @@ class AppLocalizationsZh extends AppLocalizations { @override String get messageFilter_repeatedCharactersSubtitle => - '类似“aaaaaaaa”或“!!!!!!!!”的消息'; + '同一字符的长串重复,例如“aaaaaaaaaaaaaaaaaaaa”'; @override String get messageFilter_learning => '从“垃圾消息”点击中学习'; diff --git a/lib/l10n/app_nl.arb b/lib/l10n/app_nl.arb index 7ecda781..6e1ce950 100644 --- a/lib/l10n/app_nl.arb +++ b/lib/l10n/app_nl.arb @@ -3460,7 +3460,7 @@ "messageFilter_duplicates": "Herhaalde berichten", "messageFilter_duplicatesSubtitle": "Hetzelfde bericht opnieuw verstuurd, of door meerdere afzenders", "messageFilter_repeatedCharacters": "Herhaalde tekens", - "messageFilter_repeatedCharactersSubtitle": "Berichten zoals „aaaaaaaa” of „!!!!!!!!”", + "messageFilter_repeatedCharactersSubtitle": "Lange reeksen van hetzelfde teken, zoals „aaaaaaaaaaaaaaaaaaaa”", "messageFilter_learning": "Leren van Spam-tikken", "messageFilter_learningSubtitle": "{spam} spam- en {notSpam} geen-spamvoorbeelden", "messageFilter_resetLearning": "Geleerde gegevens wissen", diff --git a/lib/l10n/app_pl.arb b/lib/l10n/app_pl.arb index 42c2bb2a..9d43a2ab 100644 --- a/lib/l10n/app_pl.arb +++ b/lib/l10n/app_pl.arb @@ -3676,7 +3676,7 @@ "@messageFilter_repeatedCharacters": { "description": "Message filter: Repeated characters" }, - "messageFilter_repeatedCharactersSubtitle": "Wiadomości typu „aaaaaaaa” lub „!!!!!!!!”", + "messageFilter_repeatedCharactersSubtitle": "Długie ciągi jednego znaku, np. „aaaaaaaaaaaaaaaaaaaa”", "@messageFilter_repeatedCharactersSubtitle": { "description": "Message filter: Messages like \"aaaaaaaa\" or \"!!!!!!!!\"" }, diff --git a/lib/l10n/app_pt.arb b/lib/l10n/app_pt.arb index c7fa4cff..c8e02f16 100644 --- a/lib/l10n/app_pt.arb +++ b/lib/l10n/app_pt.arb @@ -3559,7 +3559,7 @@ "messageFilter_duplicates": "Mensagens repetidas", "messageFilter_duplicatesSubtitle": "A mesma mensagem enviada de novo, ou por vários remetentes", "messageFilter_repeatedCharacters": "Caracteres repetidos", - "messageFilter_repeatedCharactersSubtitle": "Mensagens como \"aaaaaaaa\" ou \"!!!!!!!!\"", + "messageFilter_repeatedCharactersSubtitle": "Sequências longas de um mesmo caractere, como \"aaaaaaaaaaaaaaaaaaaa\"", "messageFilter_learning": "Aprender com os toques em Spam", "messageFilter_learningSubtitle": "{spam} exemplos de spam e {notSpam} de não spam", "@messageFilter_learningSubtitle": { diff --git a/lib/l10n/app_ru.arb b/lib/l10n/app_ru.arb index 9eb8f122..c9714458 100644 --- a/lib/l10n/app_ru.arb +++ b/lib/l10n/app_ru.arb @@ -2856,7 +2856,7 @@ "messageFilter_duplicates": "Повторяющиеся сообщения", "messageFilter_duplicatesSubtitle": "Одно и то же сообщение, отправленное снова или несколькими отправителями", "messageFilter_repeatedCharacters": "Повторяющиеся символы", - "messageFilter_repeatedCharactersSubtitle": "Сообщения вроде «aaaaaaaa» или «!!!!!!!!»", + "messageFilter_repeatedCharactersSubtitle": "Длинные последовательности одного символа, например «aaaaaaaaaaaaaaaaaaaa»", "messageFilter_learning": "Учиться на нажатиях «Спам»", "messageFilter_learningSubtitle": "Примеров спама: {spam}, не спама: {notSpam}", "messageFilter_resetLearning": "Сбросить обучение", diff --git a/lib/l10n/app_sk.arb b/lib/l10n/app_sk.arb index 53bd9fe3..cc25a9e5 100644 --- a/lib/l10n/app_sk.arb +++ b/lib/l10n/app_sk.arb @@ -3560,7 +3560,7 @@ "messageFilter_duplicates": "Opakované správy", "messageFilter_duplicatesSubtitle": "Rovnaká správa odoslaná znova alebo viacerými odosielateľmi", "messageFilter_repeatedCharacters": "Opakované znaky", - "messageFilter_repeatedCharactersSubtitle": "Správy ako „aaaaaaaa“ alebo „!!!!!!!!“", + "messageFilter_repeatedCharactersSubtitle": "Dlhé rady jedného znaku, napríklad „aaaaaaaaaaaaaaaaaaaa“", "messageFilter_learning": "Učiť sa z klepnutí na Spam", "messageFilter_learningSubtitle": "{spam} príkladov spamu a {notSpam} príkladov nie spamu", "@messageFilter_learningSubtitle": { diff --git a/lib/l10n/app_sl.arb b/lib/l10n/app_sl.arb index ac428e3a..7391ad2a 100644 --- a/lib/l10n/app_sl.arb +++ b/lib/l10n/app_sl.arb @@ -3644,7 +3644,7 @@ "@messageFilter_repeatedCharacters": { "description": "Message filter: Repeated characters" }, - "messageFilter_repeatedCharactersSubtitle": "Sporočila, kot sta „aaaaaaaa“ ali „!!!!!!!!“", + "messageFilter_repeatedCharactersSubtitle": "Dolga zaporedja istega znaka, kot je „aaaaaaaaaaaaaaaaaaaa“", "@messageFilter_repeatedCharactersSubtitle": { "description": "Message filter: Messages like \"aaaaaaaa\" or \"!!!!!!!!\"" }, diff --git a/lib/l10n/app_sv.arb b/lib/l10n/app_sv.arb index f7566da0..b65b5df4 100644 --- a/lib/l10n/app_sv.arb +++ b/lib/l10n/app_sv.arb @@ -3554,7 +3554,7 @@ "messageFilter_duplicates": "Upprepade meddelanden", "messageFilter_duplicatesSubtitle": "Samma meddelande skickas igen, eller av flera avsändare", "messageFilter_repeatedCharacters": "Upprepade tecken", - "messageFilter_repeatedCharactersSubtitle": "Meddelanden som ”aaaaaaaa” eller ”!!!!!!!!”", + "messageFilter_repeatedCharactersSubtitle": "Långa följder av samma tecken, som ”aaaaaaaaaaaaaaaaaaaa”", "messageFilter_learning": "Lär av tryck på Skräppost", "messageFilter_learningSubtitle": "{spam} skräppost- och {notSpam} inte-skräppost-exempel", "@messageFilter_learningSubtitle": { diff --git a/lib/l10n/app_uk.arb b/lib/l10n/app_uk.arb index 47b45b78..48f67f3f 100644 --- a/lib/l10n/app_uk.arb +++ b/lib/l10n/app_uk.arb @@ -3539,7 +3539,7 @@ "messageFilter_duplicates": "Повідомлення, що повторюються", "messageFilter_duplicatesSubtitle": "Те саме повідомлення, надіслане знову або кількома відправниками", "messageFilter_repeatedCharacters": "Повторювані символи", - "messageFilter_repeatedCharactersSubtitle": "Повідомлення на кшталт «aaaaaaaa» або «!!!!!!!!»", + "messageFilter_repeatedCharactersSubtitle": "Довгі послідовності одного символа, наприклад «aaaaaaaaaaaaaaaaaaaa»", "messageFilter_learning": "Навчання на натисканнях «Спам»", "messageFilter_learningSubtitle": "{spam} прикладів спаму та {notSpam} прикладів не спаму", "messageFilter_resetLearning": "Скинути навчання", diff --git a/lib/l10n/app_zh.arb b/lib/l10n/app_zh.arb index dc6f4392..00f88adf 100644 --- a/lib/l10n/app_zh.arb +++ b/lib/l10n/app_zh.arb @@ -3564,7 +3564,7 @@ "messageFilter_duplicates": "重复消息", "messageFilter_duplicatesSubtitle": "重复发送的相同消息,或多个发送者发送的相同消息", "messageFilter_repeatedCharacters": "重复字符", - "messageFilter_repeatedCharactersSubtitle": "类似“aaaaaaaa”或“!!!!!!!!”的消息", + "messageFilter_repeatedCharactersSubtitle": "同一字符的长串重复,例如“aaaaaaaaaaaaaaaaaaaa”", "messageFilter_learning": "从“垃圾消息”点击中学习", "messageFilter_learningSubtitle": "{spam} 个垃圾消息示例和 {notSpam} 个非垃圾消息示例", "@messageFilter_learningSubtitle": { diff --git a/lib/screens/channel_chat_screen.dart b/lib/screens/channel_chat_screen.dart index 060fb2b6..95f6651c 100644 --- a/lib/screens/channel_chat_screen.dart +++ b/lib/screens/channel_chat_screen.dart @@ -35,6 +35,7 @@ import '../services/chat_text_scale_service.dart'; import '../services/image_chunk_transport.dart'; import '../services/image_codec_service.dart'; import '../services/message_filter/message_filter_service.dart'; +import '../services/message_filter/message_filter_text.dart'; import '../services/received_image_store.dart'; import '../services/translation_service.dart'; import '../utils/image_save.dart'; @@ -215,8 +216,7 @@ class _ChannelChatScreenState extends State { var n = 0; ChannelMessage? oldest; for (final m in messages.reversed) { - // Filtered messages never count as unread (see the connector). - if (m.isOutgoing || m.isFiltered) continue; + if (!_countsAsUnread(m)) continue; n++; oldest = m; if (n >= unreadCount) break; @@ -771,6 +771,17 @@ class _ChannelChatScreenState extends State { ); } + /// Whether [m] counts toward the unread badge and divider. Filtered + /// messages don't while the filter hides them here (the connector never + /// counts them on arrival); with filtering off for this channel they show + /// as ordinary messages and count like ordinary messages. + bool _countsAsUnread(ChannelMessage m) { + if (m.isOutgoing) return false; + if (!m.isFiltered) return true; + return !(_messageFilter?.isChannelFiltered(_messageFilterChannelId) ?? + false); + } + void _markAsUnread(ChannelMessage message) { final connector = context.read(); final messages = connector.getChannelMessages(widget.channel); @@ -778,7 +789,7 @@ class _ChannelChatScreenState extends State { var found = false; for (final m in messages) { if (m.messageId == message.messageId) found = true; - if (found && !m.isOutgoing && !m.isFiltered) count++; + if (found && _countsAsUnread(m)) count++; } connector.setChannelUnreadCount(widget.channel.index, count); } @@ -1292,10 +1303,10 @@ class _ChannelChatScreenState extends State { final quotedId = message.replyToMessageId; final quoted = quotedId == null ? null - : connector - .getChannelMessages(widget.channel) - .where((m) => m.messageId == quotedId) - .lastOrNull; + : _channelMessageById( + connector.getChannelMessages(widget.channel), + quotedId, + ); final concealed = quoted != null && isReplyPreviewConcealed( @@ -1925,6 +1936,42 @@ class _ChannelChatScreenState extends State { void _onImageStoreChanged() => _imageStoreVersion++; + List? _indexSource; + int _indexSourceLength = -1; + Map _indexById = const {}; + + /// The newest message with [id] in [messages], via an id -> index map + /// rebuilt only when the list is replaced, changes length, or an entry + /// moved. Reply previews call this per bubble on every rebuild; a linear + /// scan each time made a busy transcript O(n^2). + ChannelMessage? _channelMessageById( + List messages, + String id, + ) { + if (!identical(messages, _indexSource) || + messages.length != _indexSourceLength) { + _rebuildIndex(messages); + } + var index = _indexById[id]; + if (index != null && + (index >= messages.length || messages[index].messageId != id)) { + _rebuildIndex(messages); + index = _indexById[id]; + } + return index == null ? null : messages[index]; + } + + void _rebuildIndex(List messages) { + final byId = {}; + // Later entries win, matching the previous `lastOrNull` lookup. + for (var i = 0; i < messages.length; i++) { + byId[messages[i].messageId] = i; + } + _indexById = byId; + _indexSource = messages; + _indexSourceLength = messages.length; + } + /// Cheap fingerprint of which messages are filtered, so the layout memo /// notices a message being marked spam or not spam in place. static int _filteredMessagesSignature(List messages) { @@ -2507,7 +2554,8 @@ class _ChannelChatScreenState extends State { !message.isOutgoing && filter != null && filter.isChannelFiltered(_messageFilterChannelId) && - (message.isFiltered || !_isStructuredPayload(message.text)); + (message.isFiltered || + !MessageFilterText.isStructuredPayload(message.text)); showMeshSheet( context, @@ -2674,13 +2722,6 @@ class _ChannelChatScreenState extends State { String get _messageFilterChannelId => MeshCoreConnector.messageFilterChannelId(widget.channel); - /// GIFs and other structured payloads (same test as the connector's - /// outbound check), which the filter never looks at. - static bool _isStructuredPayload(String text) { - final trimmed = text.trimLeft(); - return trimmed.startsWith('g:') || trimmed.startsWith('m:'); - } - void _setRevealed(Iterable ids, {required bool revealed}) { setState(() { if (revealed) { diff --git a/lib/screens/channels_screen.dart b/lib/screens/channels_screen.dart index ab768966..c47c604e 100644 --- a/lib/screens/channels_screen.dart +++ b/lib/screens/channels_screen.dart @@ -780,15 +780,18 @@ class _ChannelsScreenState extends State case ChannelSortOption.manual: break; case ChannelSortOption.latestMessages: + // Filtered messages don't bump a channel, matching the preview. + // Looked up once per channel: the lookup can scan back through + // filtered messages, and the comparator runs O(n log n) times. + final epoch = DateTime(1970); + final latest = { + for (final channel in filtered) + channel: + connector.latestVisibleChannelMessage(channel)?.timestamp ?? + epoch, + }; filtered.sort((a, b) { - // Filtered messages don't bump a channel, matching the preview. - final aLast = - connector.latestVisibleChannelMessage(a)?.timestamp ?? - DateTime(1970); - final bLast = - connector.latestVisibleChannelMessage(b)?.timestamp ?? - DateTime(1970); - final timeCompare = bLast.compareTo(aLast); + final timeCompare = latest[b]!.compareTo(latest[a]!); if (timeCompare != 0) return timeCompare; return compareByName(a, b); }); diff --git a/lib/services/message_filter/duplicate_tracker.dart b/lib/services/message_filter/duplicate_tracker.dart index 9a3d2cf2..7b8a66ce 100644 --- a/lib/services/message_filter/duplicate_tracker.dart +++ b/lib/services/message_filter/duplicate_tracker.dart @@ -28,6 +28,11 @@ class _SeenMessage { /// so a sender whose clock is far off cannot push everyone else's entries /// out early or keep its own around forever. /// +/// The same-sender rule only applies to texts of at least +/// [sameSenderMinWords] words and [sameSenderMinLength] characters: people +/// naturally repeat short replies ("ok", "yes", "test" when checking reach), +/// while spam that gets re-posted is a whole sentence. +/// /// The cross-sender rule only applies to sentence-length texts (at least /// [crossSenderMinWords] words): short greetings like "Happy New Year!" or /// "Good morning everyone" are naturally sent by many people at once, while @@ -35,6 +40,8 @@ class _SeenMessage { class DuplicateTracker { DuplicateTracker({ this.sameSenderWindow = const Duration(minutes: 10), + this.sameSenderMinLength = 10, + this.sameSenderMinWords = 3, this.crossSenderWindow = const Duration(minutes: 30), this.crossSenderMinSenders = 3, this.crossSenderMinLength = 10, @@ -43,6 +50,10 @@ class DuplicateTracker { }); final Duration sameSenderWindow; + + /// Minimum length / word count for the same-sender rule (see class docs). + final int sameSenderMinLength; + final int sameSenderMinWords; final Duration crossSenderWindow; final int crossSenderMinSenders; final int crossSenderMinLength; @@ -106,7 +117,9 @@ class DuplicateTracker { senders.add(entry.sender); } } - if (sameSender) { + if (sameSender && + key.length >= sameSenderMinLength && + _wordCount(key) >= sameSenderMinWords) { result = FilterReason.duplicate; } else if (key.length >= crossSenderMinLength && _wordCount(key) >= crossSenderMinWords && diff --git a/lib/services/message_filter/message_filter_service.dart b/lib/services/message_filter/message_filter_service.dart index 18545ba0..5f679c7d 100644 --- a/lib/services/message_filter/message_filter_service.dart +++ b/lib/services/message_filter/message_filter_service.dart @@ -8,6 +8,9 @@ import 'duplicate_tracker.dart'; import 'message_filter_text.dart'; import 'ngram_spam_classifier.dart'; +/// Trains a spam model from a job (see [MessageFilterService.new]). +typedef SpamTrainer = Future Function(SpamTrainingJob job); + /// Decides whether incoming channel messages should be filtered, and owns the /// filter's settings, user rules and the learned classifier. /// @@ -19,7 +22,9 @@ class MessageFilterService extends ChangeNotifier { DateTime Function()? clock, DuplicateTracker? duplicateTracker, NgramSpamClassifier? classifier, - }) : _store = store ?? MessageFilterStore(), + SpamTrainer? trainer, + }) : _trainer = trainer ?? _trainInBackground, + _store = store ?? MessageFilterStore(), _clock = clock ?? DateTime.now, _tracker = duplicateTracker ?? DuplicateTracker(), _classifier = classifier ?? NgramSpamClassifier(); @@ -40,6 +45,19 @@ class MessageFilterService extends ChangeNotifier { /// Recent-clean entries buffered before they are written to storage. static const int _recentCleanSaveEvery = 10; + /// Runs classifier training; a background isolate by default. Widget tests + /// pass [trainInline], since an isolate's result is never delivered under + /// the fake clock `testWidgets` runs on. + final SpamTrainer _trainer; + + static Future _trainInBackground(SpamTrainingJob job) => + compute(NgramSpamClassifier.fitInBackground, job); + + /// Trains on the calling isolate. For tests. + @visibleForTesting + static Future trainInline(SpamTrainingJob job) async => + NgramSpamClassifier.fitInBackground(job); + final MessageFilterStore _store; final DateTime Function() _clock; final DuplicateTracker _tracker; @@ -56,6 +74,13 @@ class MessageFilterService extends ChangeNotifier { /// A retrain is queued (see [_scheduleRetrainIfInactive]). bool _retrainPending = false; + /// Bumped on each retrain; a background result for an older generation is + /// discarded because newer examples have arrived since. + int _trainGeneration = 0; + Future _training = Future.value(); + + int _rulesRevision = 0; + static int _idCounter = 0; /// Unique-enough rule id: microsecond timestamp plus a process counter. @@ -66,6 +91,14 @@ class MessageFilterService extends ChangeNotifier { } MessageFilterSettings get settings => _settings; + + /// Changes whenever rules are added, edited or removed, so listeners can + /// re-apply rules to messages already stored. + int get rulesRevision => _rulesRevision; + + /// Completes when the most recently started classifier training has been + /// applied (or discarded). + Future get trainingSettled => _training; List get rules => List.unmodifiable(_rules); int get spamExampleCount => _spamExamples.length; int get notSpamExampleCount => _hamExamples.length; @@ -89,7 +122,8 @@ class MessageFilterService extends ChangeNotifier { _trimToCap(_hamExamples, maxNotSpamExamples); _trimToCap(_recentClean, maxRecentClean); _unsavedCleanCount = 0; - _retrain(); + // Training runs in the background so app start-up does not wait for it. + unawaited(_retrain()); _notify(); } @@ -127,24 +161,8 @@ class MessageFilterService extends ChangeNotifier { receivedAt: _clock(), ); - for (final rule in _rules) { - if (!rule.enabled || rule.field == FilterRuleField.text) continue; - if (MessageFilterText.matchesSender(rule, sender)) { - return FilterVerdict( - reason: FilterReason.senderRule, - detail: rule.pattern, - ); - } - } - for (final rule in _rules) { - if (!rule.enabled || rule.field == FilterRuleField.sender) continue; - if (MessageFilterText.matchesText(rule, text)) { - return FilterVerdict( - reason: FilterReason.textRule, - detail: rule.pattern, - ); - } - } + final byRule = _matchRules(sender, text); + if (byRule != null) return byRule; if (_settings.repeatedCharacters && MessageFilterText.hasRepeatedCharacters(text)) { @@ -167,10 +185,56 @@ class MessageFilterService extends ChangeNotifier { return null; } - /// Remembers an unfiltered incoming message as an implicit "not spam" - /// training example (rolling buffer of [maxRecentClean]). - void recordClean({required String sender, required String text}) { - if (!_settings.enabled || !_settings.learning) return; + /// The user-rule verdict for a message already stored, or null when no + /// enabled rule matches or [channelId] is not filtered. Unlike [evaluate] + /// it does not record anything, so it can be re-run over stored history + /// when the rules change. + FilterVerdict? ruleVerdict({ + required String channelId, + required String sender, + required String text, + }) { + if (!isChannelFiltered(channelId)) return null; + if (text.trim().isEmpty || MessageFilterText.isStructuredPayload(text)) { + return null; + } + return _matchRules(sender, text); + } + + FilterVerdict? _matchRules(String sender, String text) { + for (final rule in _rules) { + if (!rule.enabled || rule.field == FilterRuleField.text) continue; + if (MessageFilterText.matchesSender(rule, sender)) { + return FilterVerdict( + reason: FilterReason.senderRule, + detail: rule.pattern, + ); + } + } + for (final rule in _rules) { + if (!rule.enabled || rule.field == FilterRuleField.sender) continue; + if (MessageFilterText.matchesText(rule, text)) { + return FilterVerdict( + reason: FilterReason.textRule, + detail: rule.pattern, + ); + } + } + return null; + } + + /// Remembers an unfiltered incoming message in [channelId] as an implicit + /// "not spam" training example (rolling buffer of [maxRecentClean]). + /// + /// Channels excluded from filtering are skipped: the user has stopped + /// judging them, so their unfiltered messages (spam included) say nothing + /// about what is clean. + void recordClean({ + required String channelId, + required String sender, + required String text, + }) { + if (!isChannelFiltered(channelId) || !_settings.learning) return; if (text.trim().isEmpty || MessageFilterText.isStructuredPayload(text)) { return; } @@ -199,9 +263,11 @@ class MessageFilterService extends ChangeNotifier { if (_spamExamples.length < minSpamExamplesForLearning) return; _retrainPending = true; scheduleMicrotask(() { - _retrainPending = false; - if (_disposed || _classifier.isTrained) return; - _retrain(); + if (_disposed || _classifier.isTrained) { + _retrainPending = false; + return; + } + _retrain().whenComplete(() => _retrainPending = false); }); } @@ -212,13 +278,14 @@ class MessageFilterService extends ChangeNotifier { _recentClean.removeWhere((e) => MessageFilterText.normalize(e.text) == key); _hamExamples.removeWhere((e) => MessageFilterText.normalize(e.text) == key); _addExample(_spamExamples, _example(sender, text), maxSpamExamples); - _retrain(); + final training = _retrain(); _notify(); await _guardSave(() async { await _store.saveSpamExamples(_spamExamples); await _store.saveHamExamples(_hamExamples); await _flushRecentClean(); }); + await training; } /// User tapped "Not spam": store the example, exempt this text in this @@ -238,20 +305,22 @@ class MessageFilterService extends ChangeNotifier { (e) => MessageFilterText.normalize(e.text) == key, ); _addExample(_hamExamples, _example(sender, text), maxNotSpamExamples); - _retrain(); + final training = _retrain(); _notify(); await _guardSave(() async { await _store.saveSpamExamples(_spamExamples); await _store.saveHamExamples(_hamExamples); }); + await training; } Future updateSettings(MessageFilterSettings settings) async { final learningTurnedOn = settings.learning && !_settings.learning; _settings = settings; - if (learningTurnedOn) _retrain(); + final training = learningTurnedOn ? _retrain() : null; _notify(); await _guardSave(() => _store.saveSettings(_settings)); + if (training != null) await training; } Future setChannelExcluded(String channelId, bool excluded) async { @@ -276,6 +345,7 @@ class MessageFilterService extends ChangeNotifier { others.insert(index.clamp(0, others.length), rule); } _rules = others; + _rulesRevision++; _notify(); await _guardSave(() => _store.saveRules(_rules)); } @@ -287,6 +357,7 @@ class MessageFilterService extends ChangeNotifier { return; } _rules = List.of(_rules)..[index] = rule; + _rulesRevision++; _notify(); await _guardSave(() => _store.saveRules(_rules)); } @@ -295,6 +366,7 @@ class MessageFilterService extends ChangeNotifier { final before = _rules.length; _rules = _rules.where((r) => r.id != id).toList(); if (_rules.length == before) return; + _rulesRevision++; _notify(); await _guardSave(() => _store.saveRules(_rules)); } @@ -305,6 +377,7 @@ class MessageFilterService extends ChangeNotifier { _hamExamples = []; _recentClean = []; _unsavedCleanCount = 0; + _trainGeneration++; // drop any training still running _classifier.reset(); _notify(); await _guardSave(() async { @@ -355,20 +428,41 @@ class MessageFilterService extends ChangeNotifier { if (list.length > cap) list.removeRange(0, list.length - cap); } - void _retrain() { + /// Retrains from the current examples in a background isolate (training + /// is up to ~700 documents x 200 SGD epochs, too slow for the UI thread). + /// The previous model keeps scoring until the new one is ready. Completes + /// once the result is applied, or discarded because a newer retrain or a + /// reset started meanwhile. + Future _retrain() { + final generation = ++_trainGeneration; if (_spamExamples.isEmpty) { _classifier.reset(); - return; + return _training = Future.value(); } + final job = _classifier.trainingJob(List.of(_spamExamples), [ + ..._hamExamples, + ..._recentClean, + ]); final stopwatch = Stopwatch()..start(); - _classifier.train(_spamExamples, [..._hamExamples, ..._recentClean]); - stopwatch.stop(); - if (stopwatch.elapsedMilliseconds > 200) { - debugPrint( - 'MessageFilterService: classifier training took ' - '${stopwatch.elapsedMilliseconds} ms', - ); - } + return _training = _trainer(job) + .then((model) { + if (_disposed || generation != _trainGeneration) return; + if (stopwatch.elapsedMilliseconds > 1000) { + debugPrint( + 'MessageFilterService: classifier training took ' + '${stopwatch.elapsedMilliseconds} ms', + ); + } + if (model == null) { + _classifier.reset(); + } else { + _classifier.use(model); + } + _notify(); + }) + .catchError((Object e) { + debugPrint('MessageFilterService: training failed: $e'); + }); } Future _flushRecentClean() async { diff --git a/lib/services/message_filter/message_filter_text.dart b/lib/services/message_filter/message_filter_text.dart index e697d5f7..557eb903 100644 --- a/lib/services/message_filter/message_filter_text.dart +++ b/lib/services/message_filter/message_filter_text.dart @@ -13,7 +13,17 @@ class MessageFilterText { r'[^\p{L}\p{N}\s]+', unicode: true, ); - static final RegExp _repeatedChars = RegExp(r'(\S)\1{7,}', unicode: true); + + /// Shortest run of one character that counts as repeated characters. + static const int repeatedCharacterRun = 20; + + // Digits are left out: long numbers (frequencies, coordinates, ids) are + // legitimate. Shorter runs are too: "Noooooooo!", "--------" separators + // and a row of emoji are ordinary chat. + static final RegExp _repeatedChars = RegExp( + '([^\\s\\p{N}])\\1{${repeatedCharacterRun - 1},}', + unicode: true, + ); static final Map _foldMap = _buildFoldMap(); @@ -143,11 +153,12 @@ class MessageFilterText { .trim(); } - /// True when the same visible character appears 8 or more times in a row. + /// True when the same visible character appears [repeatedCharacterRun] or + /// more times in a row (keyboard-mash floods such as "aaaaaaaaaaaaaaaaaaaa"). /// - /// Whitespace runs are ignored (they are layout, not content). + /// Whitespace and digit runs are ignored. static bool hasRepeatedCharacters(String text) { - if (text.length < 8) return false; + if (text.length < repeatedCharacterRun) return false; return _repeatedChars.hasMatch(_fold(text)); } diff --git a/lib/services/message_filter/ngram_spam_classifier.dart b/lib/services/message_filter/ngram_spam_classifier.dart index 5e3a8dbf..34eb4866 100644 --- a/lib/services/message_filter/ngram_spam_classifier.dart +++ b/lib/services/message_filter/ngram_spam_classifier.dart @@ -4,6 +4,24 @@ import 'dart:typed_data'; import '../../models/message_filter.dart'; import 'message_filter_text.dart'; +/// Weights and bias of a trained [NgramSpamClassifier]. Plain data, so +/// training can run in a background isolate and hand the result back. +class SpamModel { + final Map weights; + final double bias; + const SpamModel(this.weights, this.bias); +} + +/// Inputs for [NgramSpamClassifier.fitInBackground]. +typedef SpamTrainingJob = ({ + int epochs, + double learningRate, + double lambda, + int seed, + List spam, + List ham, +}); + /// A sparse, L2-normalised feature vector (hashed bucket ids and values). class _SparseVector { final Int32List indices; @@ -143,8 +161,63 @@ class NgramSpamClassifier { /// Trains from scratch. Needs at least one example of each class; otherwise /// the classifier is left untrained (and [probability] returns 0). void train(List spam, List ham) { - reset(); - if (spam.isEmpty || ham.isEmpty) return; + final model = fit( + spam, + ham, + epochs: epochs, + learningRate: learningRate, + lambda: lambda, + seed: seed, + ); + if (model == null) { + reset(); + } else { + use(model); + } + } + + /// The job [fitInBackground] runs for this classifier's settings. + SpamTrainingJob trainingJob( + List spam, + List ham, + ) => ( + epochs: epochs, + learningRate: learningRate, + lambda: lambda, + seed: seed, + spam: spam, + ham: ham, + ); + + /// Entry point for `compute`: trains on [job] and returns the model, or + /// null when either class has no usable examples. + static SpamModel? fitInBackground(SpamTrainingJob job) => fit( + job.spam, + job.ham, + epochs: job.epochs, + learningRate: job.learningRate, + lambda: job.lambda, + seed: job.seed, + ); + + /// Switches to [model] (from [fit] or [fitInBackground]). + void use(SpamModel model) { + _weights = model.weights; + _bias = model.bias; + _trained = true; + } + + /// Pure training step: same inputs, same model. Null when either class has + /// no usable examples. + static SpamModel? fit( + List spam, + List ham, { + required int epochs, + required double learningRate, + required double lambda, + required int seed, + }) { + if (spam.isEmpty || ham.isEmpty) return null; // Vectorize and remap hashed buckets to a compact local index space. final localIndex = {}; @@ -174,7 +247,7 @@ class NgramSpamClassifier { final n = docs.length; final nPos = labels.where((y) => y == 1).length; final nNeg = n - nPos; - if (nPos == 0 || nNeg == 0) return; + if (nPos == 0 || nNeg == 0) return null; final wPos = n / (2 * nPos); final wNeg = n / (2 * nNeg); @@ -210,9 +283,7 @@ class NgramSpamClassifier { final weight = weights[local]; if (weight != 0) compact[bucket] = weight; }); - _weights = compact; - _bias = bias; - _trained = true; + return SpamModel(compact, bias); } /// Probability (0..1) that the message is spam; 0 when untrained. diff --git a/test/connector/message_filter_ingest_test.dart b/test/connector/message_filter_ingest_test.dart index acf28cb0..a9a36fbf 100644 --- a/test/connector/message_filter_ingest_test.dart +++ b/test/connector/message_filter_ingest_test.dart @@ -371,4 +371,64 @@ void main() { addTearDown(() => _filter.setChannelExcluded(channelId, false)); expect(_connector.latestVisibleChannelMessage(channel)?.text, text); }); + + test('rule changes apply to messages already in the chat', () async { + await _connect(); + const text = 'Cheap antennas at the swap meet, ask me'; + _connector.handleFrameForTest( + ReviewFrames.channelMessage( + channelIndex: 2, + sender: 'Vendor', + message: text, + timestampSecs: _nowSecs(), + ), + ); + await _waitFor(() => _find(2, text) != null); + expect(_find(2, text)!.isFiltered, isFalse); + + // A new rule hides the message that already arrived... + await _filter.addRule(const FilterRule(id: 'retro', pattern: 'antennas')); + expect( + _find(2, text)!.filterReason, + filterReasonToJson(FilterReason.textRule), + ); + expect(_find(2, text)!.filterDetail, 'antennas'); + + // ...and deleting it shows the message again. + await _filter.removeRule('retro'); + expect(_find(2, text)!.isFiltered, isFalse); + + // Other filter reasons are not touched by rule changes. + await _connector.markChannelMessageSpam(_channelAt(2), _find(2, text)!); + await _filter.addRule(const FilterRule(id: 'retro2', pattern: 'swap')); + await _filter.removeRule('retro2'); + expect(_find(2, text)!.filterReason, 'manual'); + }); + + test('mesh copies of one message are stored and filtered once', () async { + await _connect(); + const text = 'Weather net starts at seven tonight on the hill'; + final ts = _nowSecs(); + final cleanBefore = _filter.recentCleanCount; + _connector.handleFrameForTest( + ReviewFrames.channelMessage( + channelIndex: 2, + sender: 'Relay', + message: text, + timestampSecs: ts, + ), + ); + _connector.handleFrameForTest(_logRxGroupText(_psk(2), 'Relay: $text', ts)); + _connector.handleFrameForTest(_logRxGroupText(_psk(2), 'Relay: $text', ts)); + await _waitFor(() => _find(2, text) != null); + await Future.delayed(const Duration(milliseconds: 100)); + final copies = _connector + .getChannelMessages(_channelAt(2)) + .where((m) => m.text == text) + .toList(); + expect(copies, hasLength(1)); + // Not mistaken for the sender repeating themselves. + expect(copies.single.isFiltered, isFalse); + expect(_filter.recentCleanCount, cleanBefore + 1); + }); } diff --git a/test/screens/message_filter_settings_screen_test.dart b/test/screens/message_filter_settings_screen_test.dart index 8da769b1..df47b996 100644 --- a/test/screens/message_filter_settings_screen_test.dart +++ b/test/screens/message_filter_settings_screen_test.dart @@ -20,7 +20,10 @@ void main() { SharedPreferences.setMockInitialValues({}); PrefsManager.reset(); await PrefsManager.initialize(); - service = MessageFilterService(store: MessageFilterStore()); + service = MessageFilterService( + store: MessageFilterStore(), + trainer: MessageFilterService.trainInline, + ); await service.load(); connector = MeshCoreConnector(); }); diff --git a/test/services/message_filter/duplicate_tracker_test.dart b/test/services/message_filter/duplicate_tracker_test.dart index 9619a412..2a2e28c3 100644 --- a/test/services/message_filter/duplicate_tracker_test.dart +++ b/test/services/message_filter/duplicate_tracker_test.dart @@ -8,20 +8,44 @@ void main() { test('same sender repeat within 10 minutes is a duplicate', () { final tracker = DuplicateTracker(); expect( - tracker.check(channelId: 'c', sender: 'Bob', text: 'Hello', at: t0), + tracker.check( + channelId: 'c', + sender: 'Bob', + text: 'Anyone on the hill repeater?', + at: t0, + ), isNull, ); expect( tracker.check( channelId: 'c', sender: 'Bob', - text: 'hello!', + text: 'anyone on the hill repeater!', at: t0.add(const Duration(minutes: 9)), ), FilterReason.duplicate, ); }); + test('short replies repeated by the same sender are not duplicates', () { + final tracker = DuplicateTracker(); + for (final text in ['ok', 'test', 'yes sir', 'ok']) { + tracker.check(channelId: 'c', sender: 'Alice', text: text, at: t0); + } + for (final text in ['@[Bob] ok', 'test', 'yes sir']) { + expect( + tracker.check( + channelId: 'c', + sender: 'Alice', + text: text, + at: t0.add(const Duration(minutes: 5)), + ), + isNull, + reason: text, + ); + } + }); + test('same sender repeat after the window is not a duplicate', () { final tracker = DuplicateTracker(); tracker.check(channelId: 'c', sender: 'Bob', text: 'Hello', at: t0); @@ -141,13 +165,14 @@ void main() { test('filtered texts are still recorded so floods keep being caught', () { final tracker = DuplicateTracker(); - tracker.check(channelId: 'c', sender: 'Bob', text: 'buy now', at: t0); + const text = 'buy cheap radios now'; + tracker.check(channelId: 'c', sender: 'Bob', text: text, at: t0); for (var i = 1; i <= 5; i++) { expect( tracker.check( channelId: 'c', sender: 'Bob', - text: 'buy now', + text: text, at: t0.add(Duration(minutes: 8 * i)), ), FilterReason.duplicate, diff --git a/test/services/message_filter/message_filter_service_test.dart b/test/services/message_filter/message_filter_service_test.dart index 30ea3a50..865adff3 100644 --- a/test/services/message_filter/message_filter_service_test.dart +++ b/test/services/message_filter/message_filter_service_test.dart @@ -64,6 +64,48 @@ void main() { expect(eval('Bob', 'hello there'), isNull); }); + test('no implicit clean examples from excluded channels', () async { + await service.setChannelExcluded('ads', true); + final before = service.recentCleanCount; + service.recordClean( + channelId: 'ads', + sender: 'Bot', + text: 'Buy cheap radios today at the store', + ); + expect(service.recentCleanCount, before); + service.recordClean( + channelId: _channel, + sender: 'Sam', + text: 'Anyone on the hill repeater tonight?', + ); + expect(service.recentCleanCount, before + 1); + }); + + test('ruleVerdict matches rules without recording duplicates', () async { + await service.addRule(const FilterRule(id: 'r', pattern: 'meshos')); + expect( + service + .ruleVerdict(channelId: _channel, sender: 'a', text: 'get meshos') + ?.reason, + FilterReason.textRule, + ); + expect( + service.ruleVerdict(channelId: _channel, sender: 'a', text: 'hello'), + isNull, + ); + await service.setChannelExcluded('off', true); + expect( + service.ruleVerdict(channelId: 'off', sender: 'a', text: 'get meshos'), + isNull, + ); + // Nothing was recorded: the first real arrival is not a repeat. + const text = 'meet at the old fire tower at noon'; + for (var i = 0; i < 3; i++) { + service.ruleVerdict(channelId: _channel, sender: 'a', text: text); + } + expect(eval('a', text), isNull); + }); + test('skips outgoing, reactions, payloads, disabled and excluded', () async { await service.addRule( FilterRule(id: MessageFilterService.newRuleId(), pattern: 'bad'), @@ -135,11 +177,12 @@ void main() { ); test('repeated characters, togglable', () async { - expect(eval('a', 'nooooooooo')?.reason, FilterReason.repeatedCharacters); + expect(eval('a', 'n${'o' * 20}')?.reason, FilterReason.repeatedCharacters); + expect(eval('a', 'nooooooooo'), isNull); await service.updateSettings( service.settings.copyWith(repeatedCharacters: false), ); - expect(eval('b', 'yesssssssss'), isNull); + expect(eval('b', 'ye${'s' * 20}'), isNull); }); test('duplicates: same sender and cross sender', () { @@ -169,9 +212,15 @@ void main() { test('rule-filtered texts are still recorded for duplicates', () async { await service.addRule(const FilterRule(id: 'x', pattern: 'buy')); - expect(eval('a', 'buy stuff')?.reason, FilterReason.textRule); + expect( + eval('a', 'buy cheap radios here now')?.reason, + FilterReason.textRule, + ); await service.removeRule('x'); - expect(eval('a', 'buy stuff')?.reason, FilterReason.duplicate); + expect( + eval('a', 'buy cheap radios here now')?.reason, + FilterReason.duplicate, + ); }); test('not spam exempts duplicates in that channel', () async { @@ -187,7 +236,7 @@ void main() { test('learning needs two spam taps, then catches similar promos', () async { for (var i = 0; i < _chat.length; i++) { - service.recordClean(sender: 'User$i', text: _chat[i]); + service.recordClean(channelId: 'chat', sender: 'User$i', text: _chat[i]); } await service.markSpam( sender: 'PromoBot', @@ -259,7 +308,9 @@ void main() { await service.addRule(const FilterRule(id: 'n', pattern: 'x')); await service.markSpam(sender: 'a', text: 'b'); await service.setChannelExcluded('c', true); - expect(count, 3); + // addRule, markSpam (plus once more when its background training lands) + // and setChannelExcluded. + expect(count, 4); }); test('addRule with index restores a deleted rule in place', () async { @@ -297,9 +348,14 @@ void main() { // Ordinary chat then arrives; no further taps. for (var i = 0; i < _chat.length; i++) { - service.recordClean(sender: 'User$i', text: _chat[i]); + service.recordClean( + channelId: 'chat', + sender: 'User$i', + text: _chat[i], + ); } - await Future.delayed(Duration.zero); // let the queued retrain run + await Future.delayed(Duration.zero); // let the queued retrain start + await service.trainingSettled; expect(eval('B', probe)?.reason, FilterReason.learned); expect(eval('C', 'Anyone on the hill repeater tonight?'), isNull); }, diff --git a/test/services/message_filter/message_filter_text_test.dart b/test/services/message_filter/message_filter_text_test.dart index 74a4183e..5b72d3a4 100644 --- a/test/services/message_filter/message_filter_text_test.dart +++ b/test/services/message_filter/message_filter_text_test.dart @@ -136,11 +136,21 @@ void main() { }); test('repeated characters', () { - expect(MessageFilterText.hasRepeatedCharacters('aaaaaaaa'), isTrue); - expect(MessageFilterText.hasRepeatedCharacters('wow!!!!!!!!'), isTrue); - expect(MessageFilterText.hasRepeatedCharacters('AAAAaaaa'), isTrue); - expect(MessageFilterText.hasRepeatedCharacters('😀' * 8), isTrue); - expect(MessageFilterText.hasRepeatedCharacters('aaaaaaa'), isFalse); + expect(MessageFilterText.hasRepeatedCharacters('a' * 20), isTrue); + expect(MessageFilterText.hasRepeatedCharacters('wow${'!' * 20}'), isTrue); + expect( + MessageFilterText.hasRepeatedCharacters('AAAAAAAAAA${'a' * 10}'), + isTrue, + ); + expect(MessageFilterText.hasRepeatedCharacters('😀' * 20), isTrue); + expect(MessageFilterText.hasRepeatedCharacters('a' * 19), isFalse); + // Ordinary chat that the old 8-character threshold caught. + expect(MessageFilterText.hasRepeatedCharacters('Noooooooo!'), isFalse); + expect(MessageFilterText.hasRepeatedCharacters('-' * 12), isFalse); + expect(MessageFilterText.hasRepeatedCharacters('!!!!!!!!'), isFalse); + expect(MessageFilterText.hasRepeatedCharacters('😀' * 8), isFalse); + // Digit runs never count (frequencies, coordinates, ids). + expect(MessageFilterText.hasRepeatedCharacters('0' * 30), isFalse); expect( MessageFilterText.hasRepeatedCharacters('hello there'), isFalse,