diff options
| author | test@dangofactory.cos <you@example.com> | 2026-08-20 16:57:18 +0000 |
|---|---|---|
| committer | test@dangofactory.cos <you@example.com> | 2026-08-20 16:57:18 +0000 |
| commit | 50ce5d5e6cd0d720ff437d35eace545816848428 (patch) | |
| tree | b59af206a36265cf081945e434900b32b0c43edd /backend/imap/src/main/java/com | |
| download | monocles_mail-50ce5d5e6cd0d720ff437d35eace545816848428.tar.gz monocles_mail-50ce5d5e6cd0d720ff437d35eace545816848428.tar.bz2 monocles_mail-50ce5d5e6cd0d720ff437d35eace545816848428.zip | |
Inital
Diffstat (limited to 'backend/imap/src/main/java/com')
21 files changed, 1818 insertions, 0 deletions
diff --git a/backend/imap/src/main/java/com/fsck/k9/backend/imap/BackendIdleRefreshManager.kt b/backend/imap/src/main/java/com/fsck/k9/backend/imap/BackendIdleRefreshManager.kt new file mode 100644 index 0000000..b7996a1 --- /dev/null +++ b/backend/imap/src/main/java/com/fsck/k9/backend/imap/BackendIdleRefreshManager.kt @@ -0,0 +1,136 @@ +package com.fsck.k9.backend.imap + +import com.fsck.k9.mail.store.imap.IdleRefreshManager +import com.fsck.k9.mail.store.imap.IdleRefreshTimer + +private typealias Callback = () -> Unit + +private const val MIN_TIMER_DELTA = 1 * 60 * 1000L +private const val NO_TRIGGER_TIME = 0L + +/** + * Timer mechanism to refresh IMAP IDLE connections. + * + * Triggers timers early if necessary to reduce the number of times the device has to be woken up. + */ +class BackendIdleRefreshManager(private val alarmManager: SystemAlarmManager) : IdleRefreshManager { + private var timers = mutableSetOf<BackendIdleRefreshTimer>() + private var currentTriggerTime = NO_TRIGGER_TIME + private var minTimeout = Long.MAX_VALUE + private var minTimeoutTimestamp = 0L + + @Synchronized + override fun startTimer(timeout: Long, callback: Callback): IdleRefreshTimer { + require(timeout > MIN_TIMER_DELTA) { "Timeout needs to be greater than $MIN_TIMER_DELTA ms" } + + val now = alarmManager.now() + val triggerTime = now + timeout + + updateMinTimeout(timeout, now) + setOrUpdateAlarm(triggerTime) + + return BackendIdleRefreshTimer(triggerTime, callback).also { timer -> + timers.add(timer) + } + } + + override fun resetTimers() { + synchronized(this) { + cancelAlarm() + } + + onTimeout() + } + + private fun updateMinTimeout(timeout: Long, now: Long) { + if (minTimeoutTimestamp + minTimeout * 2 < now) { + minTimeout = Long.MAX_VALUE + } + + if (timeout <= minTimeout) { + minTimeout = timeout + minTimeoutTimestamp = now + } + } + + private fun setOrUpdateAlarm(triggerTime: Long) { + if (currentTriggerTime == NO_TRIGGER_TIME) { + setAlarm(triggerTime) + } else if (currentTriggerTime - triggerTime > MIN_TIMER_DELTA) { + adjustAlarm(triggerTime) + } + } + + private fun setAlarm(triggerTime: Long) { + currentTriggerTime = triggerTime + alarmManager.setAlarm(triggerTime, ::onTimeout) + } + + private fun adjustAlarm(triggerTime: Long) { + currentTriggerTime = triggerTime + alarmManager.cancelAlarm() + alarmManager.setAlarm(triggerTime, ::onTimeout) + } + + private fun cancelAlarm() { + currentTriggerTime = NO_TRIGGER_TIME + alarmManager.cancelAlarm() + } + + private fun onTimeout() { + val triggerTimers = synchronized(this) { + currentTriggerTime = NO_TRIGGER_TIME + + if (timers.isEmpty()) return + + val now = alarmManager.now() + val minNextTriggerTime = now + minTimeout + + val triggerTimers = timers.filter { it.triggerTime < minNextTriggerTime - MIN_TIMER_DELTA } + timers.removeAll(triggerTimers) + + timers.minOfOrNull { it.triggerTime }?.let { nextTriggerTime -> + setAlarm(nextTriggerTime) + } + + triggerTimers + } + + for (timer in triggerTimers) { + timer.onTimeout() + } + } + + @Synchronized + private fun removeTimer(timer: BackendIdleRefreshTimer) { + timers.remove(timer) + + if (timers.isEmpty()) { + cancelAlarm() + } + } + + internal inner class BackendIdleRefreshTimer( + val triggerTime: Long, + val callback: Callback + ) : IdleRefreshTimer { + override var isWaiting: Boolean = true + private set + + @Synchronized + override fun cancel() { + if (isWaiting) { + isWaiting = false + removeTimer(this) + } + } + + internal fun onTimeout() { + synchronized(this) { + isWaiting = false + } + + callback.invoke() + } + } +} diff --git a/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandDeleteAll.kt b/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandDeleteAll.kt new file mode 100644 index 0000000..fd6577b --- /dev/null +++ b/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandDeleteAll.kt @@ -0,0 +1,24 @@ +package com.fsck.k9.backend.imap + +import com.fsck.k9.mail.Flag +import com.fsck.k9.mail.MessagingException +import com.fsck.k9.mail.store.imap.ImapStore +import com.fsck.k9.mail.store.imap.OpenMode + +internal class CommandDeleteAll(private val imapStore: ImapStore) { + + @Throws(MessagingException::class) + fun deleteAll(folderServerId: String) { + val remoteFolder = imapStore.getFolder(folderServerId) + if (!remoteFolder.exists()) { + return + } + + try { + remoteFolder.open(OpenMode.READ_WRITE) + remoteFolder.setFlags(setOf(Flag.DELETED), true) + } finally { + remoteFolder.close() + } + } +} diff --git a/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandDownloadMessage.kt b/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandDownloadMessage.kt new file mode 100644 index 0000000..d925b3c --- /dev/null +++ b/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandDownloadMessage.kt @@ -0,0 +1,55 @@ +package com.fsck.k9.backend.imap + +import com.fsck.k9.backend.api.BackendStorage +import com.fsck.k9.mail.FetchProfile +import com.fsck.k9.mail.FetchProfile.Item.BODY +import com.fsck.k9.mail.FetchProfile.Item.ENVELOPE +import com.fsck.k9.mail.FetchProfile.Item.FLAGS +import com.fsck.k9.mail.FetchProfile.Item.STRUCTURE +import com.fsck.k9.mail.MessageDownloadState +import com.fsck.k9.mail.helper.fetchProfileOf +import com.fsck.k9.mail.store.imap.ImapFolder +import com.fsck.k9.mail.store.imap.ImapMessage +import com.fsck.k9.mail.store.imap.ImapStore +import com.fsck.k9.mail.store.imap.OpenMode + +internal class CommandDownloadMessage(private val backendStorage: BackendStorage, private val imapStore: ImapStore) { + + fun downloadMessageStructure(folderServerId: String, messageServerId: String) { + val folder = imapStore.getFolder(folderServerId) + try { + folder.open(OpenMode.READ_ONLY) + + val message = folder.getMessage(messageServerId) + + // fun fact: ImapFolder.fetch can't handle getting STRUCTURE at same time as headers + fetchMessage(folder, message, fetchProfileOf(FLAGS, ENVELOPE)) + fetchMessage(folder, message, fetchProfileOf(STRUCTURE)) + + val backendFolder = backendStorage.getFolder(folderServerId) + backendFolder.saveMessage(message, MessageDownloadState.ENVELOPE) + } finally { + folder.close() + } + } + + fun downloadCompleteMessage(folderServerId: String, messageServerId: String) { + val folder = imapStore.getFolder(folderServerId) + try { + folder.open(OpenMode.READ_ONLY) + + val message = folder.getMessage(messageServerId) + fetchMessage(folder, message, fetchProfileOf(FLAGS, BODY)) + + val backendFolder = backendStorage.getFolder(folderServerId) + backendFolder.saveMessage(message, MessageDownloadState.FULL) + } finally { + folder.close() + } + } + + private fun fetchMessage(remoteFolder: ImapFolder, message: ImapMessage, fetchProfile: FetchProfile) { + val maxDownloadSize = 0 + remoteFolder.fetch(listOf(message), fetchProfile, null, maxDownloadSize) + } +} diff --git a/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandExpunge.kt b/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandExpunge.kt new file mode 100644 index 0000000..1b9178c --- /dev/null +++ b/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandExpunge.kt @@ -0,0 +1,40 @@ +package com.fsck.k9.backend.imap + +import com.fsck.k9.logging.Timber +import com.fsck.k9.mail.store.imap.ImapStore +import com.fsck.k9.mail.store.imap.OpenMode + +internal class CommandExpunge(private val imapStore: ImapStore) { + + fun expunge(folderServerId: String) { + Timber.d("processPendingExpunge: folder = %s", folderServerId) + + val remoteFolder = imapStore.getFolder(folderServerId) + try { + if (!remoteFolder.exists()) return + + remoteFolder.open(OpenMode.READ_WRITE) + if (remoteFolder.mode != OpenMode.READ_WRITE) return + + remoteFolder.expunge() + + Timber.d("processPendingExpunge: complete for folder = %s", folderServerId) + } finally { + remoteFolder.close() + } + } + + fun expungeMessages(folderServerId: String, messageServerIds: List<String>) { + val remoteFolder = imapStore.getFolder(folderServerId) + try { + if (!remoteFolder.exists()) return + + remoteFolder.open(OpenMode.READ_WRITE) + if (remoteFolder.mode != OpenMode.READ_WRITE) return + + remoteFolder.expungeUids(messageServerIds) + } finally { + remoteFolder.close() + } + } +} diff --git a/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandFetchMessage.kt b/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandFetchMessage.kt new file mode 100644 index 0000000..9feb474 --- /dev/null +++ b/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandFetchMessage.kt @@ -0,0 +1,21 @@ +package com.fsck.k9.backend.imap + +import com.fsck.k9.mail.BodyFactory +import com.fsck.k9.mail.Part +import com.fsck.k9.mail.store.imap.ImapStore +import com.fsck.k9.mail.store.imap.OpenMode + +internal class CommandFetchMessage(private val imapStore: ImapStore) { + + fun fetchPart(folderServerId: String, messageServerId: String, part: Part, bodyFactory: BodyFactory) { + val folder = imapStore.getFolder(folderServerId) + try { + folder.open(OpenMode.READ_WRITE) + + val message = folder.getMessage(messageServerId) + folder.fetchPart(message, part, bodyFactory, -1) + } finally { + folder.close() + } + } +} diff --git a/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandFindByMessageId.kt b/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandFindByMessageId.kt new file mode 100644 index 0000000..12d1445 --- /dev/null +++ b/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandFindByMessageId.kt @@ -0,0 +1,17 @@ +package com.fsck.k9.backend.imap + +import com.fsck.k9.mail.store.imap.ImapStore +import com.fsck.k9.mail.store.imap.OpenMode + +internal class CommandFindByMessageId(private val imapStore: ImapStore) { + + fun findByMessageId(folderServerId: String, messageId: String): String? { + val folder = imapStore.getFolder(folderServerId) + try { + folder.open(OpenMode.READ_WRITE) + return folder.getUidFromMessageId(messageId) + } finally { + folder.close() + } + } +} diff --git a/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandMarkAllAsRead.kt b/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandMarkAllAsRead.kt new file mode 100644 index 0000000..c424280 --- /dev/null +++ b/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandMarkAllAsRead.kt @@ -0,0 +1,22 @@ +package com.fsck.k9.backend.imap + +import com.fsck.k9.mail.Flag +import com.fsck.k9.mail.store.imap.ImapStore +import com.fsck.k9.mail.store.imap.OpenMode + +internal class CommandMarkAllAsRead(private val imapStore: ImapStore) { + + fun markAllAsRead(folderServerId: String) { + val remoteFolder = imapStore.getFolder(folderServerId) + if (!remoteFolder.exists()) return + + try { + remoteFolder.open(OpenMode.READ_WRITE) + if (remoteFolder.mode != OpenMode.READ_WRITE) return + + remoteFolder.setFlags(setOf(Flag.SEEN), true) + } finally { + remoteFolder.close() + } + } +} diff --git a/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandMoveOrCopyMessages.kt b/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandMoveOrCopyMessages.kt new file mode 100644 index 0000000..b834f9a --- /dev/null +++ b/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandMoveOrCopyMessages.kt @@ -0,0 +1,77 @@ +package com.fsck.k9.backend.imap + +import com.fsck.k9.logging.Timber +import com.fsck.k9.mail.MessagingException +import com.fsck.k9.mail.store.imap.ImapFolder +import com.fsck.k9.mail.store.imap.ImapStore +import com.fsck.k9.mail.store.imap.OpenMode + +internal class CommandMoveOrCopyMessages(private val imapStore: ImapStore) { + + fun moveMessages( + sourceFolderServerId: String, + targetFolderServerId: String, + messageServerIds: List<String> + ): Map<String, String>? { + return moveOrCopyMessages(sourceFolderServerId, targetFolderServerId, messageServerIds, false) + } + + fun copyMessages( + sourceFolderServerId: String, + targetFolderServerId: String, + messageServerIds: List<String> + ): Map<String, String>? { + return moveOrCopyMessages(sourceFolderServerId, targetFolderServerId, messageServerIds, true) + } + + private fun moveOrCopyMessages( + srcFolder: String, + destFolder: String, + uids: Collection<String>, + isCopy: Boolean + ): Map<String, String>? { + var remoteSrcFolder: ImapFolder? = null + var remoteDestFolder: ImapFolder? = null + + return try { + remoteSrcFolder = imapStore.getFolder(srcFolder) + + if (uids.isEmpty()) { + Timber.i("moveOrCopyMessages: no remote messages to move, skipping") + return null + } + + if (!remoteSrcFolder.exists()) { + throw MessagingException("moveOrCopyMessages: remoteFolder $srcFolder does not exist", true) + } + + remoteSrcFolder.open(OpenMode.READ_WRITE) + if (remoteSrcFolder.mode != OpenMode.READ_WRITE) { + throw MessagingException( + "moveOrCopyMessages: could not open remoteSrcFolder $srcFolder read/write", + true + ) + } + + val messages = uids.map { uid -> remoteSrcFolder.getMessage(uid) } + + Timber.d( + "moveOrCopyMessages: source folder = %s, %d messages, destination folder = %s, isCopy = %s", + srcFolder, + messages.size, + destFolder, + isCopy + ) + + remoteDestFolder = imapStore.getFolder(destFolder) + if (isCopy) { + remoteSrcFolder.copyMessages(messages, remoteDestFolder) + } else { + remoteSrcFolder.moveMessages(messages, remoteDestFolder) + } + } finally { + remoteSrcFolder?.close() + remoteDestFolder?.close() + } + } +} diff --git a/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandRefreshFolderList.kt b/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandRefreshFolderList.kt new file mode 100644 index 0000000..cf71ed7 --- /dev/null +++ b/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandRefreshFolderList.kt @@ -0,0 +1,47 @@ +package com.fsck.k9.backend.imap + +import com.fsck.k9.backend.api.BackendStorage +import com.fsck.k9.backend.api.FolderInfo +import com.fsck.k9.backend.api.updateFolders +import com.fsck.k9.mail.FolderType +import com.fsck.k9.mail.store.imap.FolderListItem +import com.fsck.k9.mail.store.imap.ImapStore + +internal class CommandRefreshFolderList( + private val backendStorage: BackendStorage, + private val imapStore: ImapStore +) { + fun refreshFolderList() { + // TODO: Start using the proper server ID. + // For now we still use the old server ID format (decoded, with prefix removed). + val foldersOnServer = imapStore.getFolders().toLegacyFolderList() + val oldFolderServerIds = backendStorage.getFolderServerIds() + + backendStorage.updateFolders { + val foldersToCreate = mutableListOf<FolderInfo>() + for (folder in foldersOnServer) { + if (folder.serverId !in oldFolderServerIds) { + foldersToCreate.add(FolderInfo(folder.serverId, folder.name, folder.type)) + } else { + changeFolder(folder.serverId, folder.name, folder.type) + } + } + createFolders(foldersToCreate) + + val newFolderServerIds = foldersOnServer.map { it.serverId } + val removedFolderServerIds = oldFolderServerIds - newFolderServerIds + deleteFolders(removedFolderServerIds) + } + } +} + +private fun List<FolderListItem>.toLegacyFolderList(): List<LegacyFolderListItem> { + return this.filterNot { it.oldServerId == null } + .map { LegacyFolderListItem(it.oldServerId!!, it.name, it.type) } +} + +private data class LegacyFolderListItem( + val serverId: String, + val name: String, + val type: FolderType +) diff --git a/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandSearch.kt b/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandSearch.kt new file mode 100644 index 0000000..f052343 --- /dev/null +++ b/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandSearch.kt @@ -0,0 +1,24 @@ +package com.fsck.k9.backend.imap + +import com.fsck.k9.mail.Flag +import com.fsck.k9.mail.store.imap.ImapStore + +internal class CommandSearch(private val imapStore: ImapStore) { + + fun search( + folderServerId: String, + query: String?, + requiredFlags: Set<Flag>?, + forbiddenFlags: Set<Flag>?, + performFullTextSearch: Boolean + ): List<String> { + val folder = imapStore.getFolder(folderServerId) + try { + return folder.search(query, requiredFlags, forbiddenFlags, performFullTextSearch) + .sortedWith(UidReverseComparator()) + .map { it.uid } + } finally { + folder.close() + } + } +} diff --git a/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandSetFlag.kt b/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandSetFlag.kt new file mode 100644 index 0000000..90bfdf1 --- /dev/null +++ b/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandSetFlag.kt @@ -0,0 +1,26 @@ +package com.fsck.k9.backend.imap + +import com.fsck.k9.mail.Flag +import com.fsck.k9.mail.store.imap.ImapStore +import com.fsck.k9.mail.store.imap.OpenMode + +internal class CommandSetFlag(private val imapStore: ImapStore) { + + fun setFlag(folderServerId: String, messageServerIds: List<String>, flag: Flag, newState: Boolean) { + if (messageServerIds.isEmpty()) return + + val remoteFolder = imapStore.getFolder(folderServerId) + if (!remoteFolder.exists()) return + + try { + remoteFolder.open(OpenMode.READ_WRITE) + if (remoteFolder.mode != OpenMode.READ_WRITE) return + + val messages = messageServerIds.map { uid -> remoteFolder.getMessage(uid) } + + remoteFolder.setFlags(messages, setOf(flag), newState) + } finally { + remoteFolder.close() + } + } +} diff --git a/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandUploadMessage.kt b/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandUploadMessage.kt new file mode 100644 index 0000000..f7da483 --- /dev/null +++ b/backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandUploadMessage.kt @@ -0,0 +1,22 @@ +package com.fsck.k9.backend.imap + +import com.fsck.k9.mail.Message +import com.fsck.k9.mail.store.imap.ImapStore +import com.fsck.k9.mail.store.imap.OpenMode + +internal class CommandUploadMessage(private val imapStore: ImapStore) { + + fun uploadMessage(folderServerId: String, message: Message): String? { + val folder = imapStore.getFolder(folderServerId) + try { + folder.open(OpenMode.READ_WRITE) + + val localUid = message.uid + val uidMap = folder.appendMessages(listOf(message)) + + return uidMap?.get(localUid) + } finally { + folder.close() + } + } +} diff --git a/backend/imap/src/main/java/com/fsck/k9/backend/imap/ImapBackend.kt b/backend/imap/src/main/java/com/fsck/k9/backend/imap/ImapBackend.kt new file mode 100644 index 0000000..46c16cf --- /dev/null +++ b/backend/imap/src/main/java/com/fsck/k9/backend/imap/ImapBackend.kt @@ -0,0 +1,162 @@ +package com.fsck.k9.backend.imap + +import com.fsck.k9.backend.api.Backend +import com.fsck.k9.backend.api.BackendPusher +import com.fsck.k9.backend.api.BackendPusherCallback +import com.fsck.k9.backend.api.BackendStorage +import com.fsck.k9.backend.api.SyncConfig +import com.fsck.k9.backend.api.SyncListener +import com.fsck.k9.mail.BodyFactory +import com.fsck.k9.mail.Flag +import com.fsck.k9.mail.Message +import com.fsck.k9.mail.Part +import com.fsck.k9.mail.power.PowerManager +import com.fsck.k9.mail.store.imap.IdleRefreshManager +import com.fsck.k9.mail.store.imap.ImapStore +import com.fsck.k9.mail.transport.smtp.SmtpTransport + +class ImapBackend( + private val accountName: String, + backendStorage: BackendStorage, + private val imapStore: ImapStore, + private val powerManager: PowerManager, + private val idleRefreshManager: IdleRefreshManager, + private val pushConfigProvider: ImapPushConfigProvider, + private val smtpTransport: SmtpTransport +) : Backend { + private val imapSync = ImapSync(accountName, backendStorage, imapStore) + private val commandRefreshFolderList = CommandRefreshFolderList(backendStorage, imapStore) + private val commandSetFlag = CommandSetFlag(imapStore) + private val commandMarkAllAsRead = CommandMarkAllAsRead(imapStore) + private val commandExpunge = CommandExpunge(imapStore) + private val commandMoveOrCopyMessages = CommandMoveOrCopyMessages(imapStore) + private val commandDeleteAll = CommandDeleteAll(imapStore) + private val commandSearch = CommandSearch(imapStore) + private val commandDownloadMessage = CommandDownloadMessage(backendStorage, imapStore) + private val commandFetchMessage = CommandFetchMessage(imapStore) + private val commandFindByMessageId = CommandFindByMessageId(imapStore) + private val commandUploadMessage = CommandUploadMessage(imapStore) + + override val supportsFlags = true + override val supportsExpunge = true + override val supportsMove = true + override val supportsCopy = true + override val supportsUpload = true + override val supportsTrashFolder = true + override val supportsSearchByDate = true + override val isPushCapable = true + + override fun refreshFolderList() { + commandRefreshFolderList.refreshFolderList() + } + + override fun sync(folderServerId: String, syncConfig: SyncConfig, listener: SyncListener) { + imapSync.sync(folderServerId, syncConfig, listener) + } + + override fun downloadMessage(syncConfig: SyncConfig, folderServerId: String, messageServerId: String) { + imapSync.downloadMessage(syncConfig, folderServerId, messageServerId) + } + + override fun downloadMessageStructure(folderServerId: String, messageServerId: String) { + commandDownloadMessage.downloadMessageStructure(folderServerId, messageServerId) + } + + override fun downloadCompleteMessage(folderServerId: String, messageServerId: String) { + commandDownloadMessage.downloadCompleteMessage(folderServerId, messageServerId) + } + + override fun setFlag(folderServerId: String, messageServerIds: List<String>, flag: Flag, newState: Boolean) { + commandSetFlag.setFlag(folderServerId, messageServerIds, flag, newState) + } + + override fun markAllAsRead(folderServerId: String) { + commandMarkAllAsRead.markAllAsRead(folderServerId) + } + + override fun expunge(folderServerId: String) { + commandExpunge.expunge(folderServerId) + } + + override fun expungeMessages(folderServerId: String, messageServerIds: List<String>) { + commandExpunge.expungeMessages(folderServerId, messageServerIds) + } + + override fun deleteMessages(folderServerId: String, messageServerIds: List<String>) { + commandSetFlag.setFlag(folderServerId, messageServerIds, Flag.DELETED, true) + } + + override fun deleteAllMessages(folderServerId: String) { + commandDeleteAll.deleteAll(folderServerId) + } + + override fun moveMessages( + sourceFolderServerId: String, + targetFolderServerId: String, + messageServerIds: List<String> + ): Map<String, String>? { + return commandMoveOrCopyMessages.moveMessages(sourceFolderServerId, targetFolderServerId, messageServerIds) + } + + override fun moveMessagesAndMarkAsRead( + sourceFolderServerId: String, + targetFolderServerId: String, + messageServerIds: List<String> + ): Map<String, String>? { + val uidMapping = commandMoveOrCopyMessages.moveMessages( + sourceFolderServerId, + targetFolderServerId, + messageServerIds + ) + if (uidMapping != null) { + setFlag(targetFolderServerId, uidMapping.values.toList(), Flag.SEEN, true) + } + return uidMapping + } + + override fun copyMessages( + sourceFolderServerId: String, + targetFolderServerId: String, + messageServerIds: List<String> + ): Map<String, String>? { + return commandMoveOrCopyMessages.copyMessages(sourceFolderServerId, targetFolderServerId, messageServerIds) + } + + override fun search( + folderServerId: String, + query: String?, + requiredFlags: Set<Flag>?, + forbiddenFlags: Set<Flag>?, + performFullTextSearch: Boolean + ): List<String> { + return commandSearch.search(folderServerId, query, requiredFlags, forbiddenFlags, performFullTextSearch) + } + + override fun fetchPart(folderServerId: String, messageServerId: String, part: Part, bodyFactory: BodyFactory) { + commandFetchMessage.fetchPart(folderServerId, messageServerId, part, bodyFactory) + } + + override fun findByMessageId(folderServerId: String, messageId: String): String? { + return commandFindByMessageId.findByMessageId(folderServerId, messageId) + } + + override fun uploadMessage(folderServerId: String, message: Message): String? { + return commandUploadMessage.uploadMessage(folderServerId, message) + } + + override fun checkIncomingServerSettings() { + imapStore.checkSettings() + } + + override fun sendMessage(message: Message) { + smtpTransport.sendMessage(message) + } + + override fun checkOutgoingServerSettings() { + smtpTransport.checkSettings() + } + + override fun createPusher(callback: BackendPusherCallback): BackendPusher { + return ImapBackendPusher(imapStore, powerManager, idleRefreshManager, pushConfigProvider, callback, accountName) + } +} diff --git a/backend/imap/src/main/java/com/fsck/k9/backend/imap/ImapBackendPusher.kt b/backend/imap/src/main/java/com/fsck/k9/backend/imap/ImapBackendPusher.kt new file mode 100644 index 0000000..f08c4a8 --- /dev/null +++ b/backend/imap/src/main/java/com/fsck/k9/backend/imap/ImapBackendPusher.kt @@ -0,0 +1,251 @@ +package com.fsck.k9.backend.imap + +import com.fsck.k9.backend.api.BackendPusher +import com.fsck.k9.backend.api.BackendPusherCallback +import com.fsck.k9.logging.Timber +import com.fsck.k9.mail.AuthenticationFailedException +import com.fsck.k9.mail.MessagingException +import com.fsck.k9.mail.power.PowerManager +import com.fsck.k9.mail.store.imap.IdleRefreshManager +import com.fsck.k9.mail.store.imap.IdleRefreshTimeoutProvider +import com.fsck.k9.mail.store.imap.IdleRefreshTimer +import com.fsck.k9.mail.store.imap.ImapStore +import java.io.IOException +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.cancel +import kotlinx.coroutines.launch + +private const val IO_ERROR_TIMEOUT = 5 * 60 * 1000L +private const val UNEXPECTED_ERROR_TIMEOUT = 60 * 60 * 1000L + +/** + * Manages [ImapFolderPusher] instances that listen for changes to individual folders. + */ +internal class ImapBackendPusher( + private val imapStore: ImapStore, + private val powerManager: PowerManager, + private val idleRefreshManager: IdleRefreshManager, + private val pushConfigProvider: ImapPushConfigProvider, + private val callback: BackendPusherCallback, + private val accountName: String, + backgroundDispatcher: CoroutineDispatcher = Dispatchers.IO +) : BackendPusher, ImapPusherCallback { + private val coroutineScope = CoroutineScope(backgroundDispatcher) + private val lock = Any() + private val pushFolders = mutableMapOf<String, ImapFolderPusher>() + private var currentFolderServerIds: Collection<String> = emptySet() + private val pushFolderSleeping = mutableMapOf<String, IdleRefreshTimer>() + + private val idleRefreshTimeoutProvider = object : IdleRefreshTimeoutProvider { + override val idleRefreshTimeoutMs + get() = currentIdleRefreshMs + } + + @Volatile + private var currentMaxPushFolders = 0 + + @Volatile + private var currentIdleRefreshMs = 15 * 60 * 1000L + + override fun start() { + coroutineScope.launch { + pushConfigProvider.maxPushFoldersFlow.collect { maxPushFolders -> + currentMaxPushFolders = maxPushFolders + updateFolders() + } + } + + coroutineScope.launch { + pushConfigProvider.idleRefreshMinutesFlow.collect { idleRefreshMinutes -> + currentIdleRefreshMs = idleRefreshMinutes * 60 * 1000L + refreshFolderTimers() + } + } + } + + private fun refreshFolderTimers() { + synchronized(lock) { + for (pushFolder in pushFolders.values) { + pushFolder.refresh() + } + } + } + + override fun updateFolders(folderServerIds: Collection<String>) { + updateFolders(folderServerIds, currentMaxPushFolders) + } + + private fun updateFolders() { + val currentFolderServerIds = synchronized(lock) { currentFolderServerIds } + updateFolders(currentFolderServerIds, currentMaxPushFolders) + } + + private fun updateFolders(folderServerIds: Collection<String>, maxPushFolders: Int) { + Timber.v("ImapBackendPusher.updateFolders(): %s", folderServerIds) + + val pushFolderServerIds = if (folderServerIds.size > maxPushFolders) { + folderServerIds.take(maxPushFolders).also { pushFolderServerIds -> + Timber.v("..limiting Push to %d folders: %s", maxPushFolders, pushFolderServerIds) + } + } else { + folderServerIds + } + + val stopFolderPushers: List<ImapFolderPusher> + val startFolderPushers: List<ImapFolderPusher> + synchronized(lock) { + currentFolderServerIds = folderServerIds + + val oldRunningFolderServerIds = pushFolders.keys + val oldFolderServerIds = oldRunningFolderServerIds + pushFolderSleeping.keys + val removeFolderServerIds = oldFolderServerIds - pushFolderServerIds + stopFolderPushers = removeFolderServerIds + .asSequence() + .onEach { folderServerId -> cancelRetryTimer(folderServerId) } + .map { folderServerId -> pushFolders.remove(folderServerId) } + .filterNotNull() + .toList() + + val startFolderServerIds = pushFolderServerIds - oldRunningFolderServerIds + startFolderPushers = startFolderServerIds + .asSequence() + .filterNot { folderServerId -> isWaitingForRetry(folderServerId) } + .onEach { folderServerId -> pushFolderSleeping.remove(folderServerId) } + .map { folderServerId -> + createImapFolderPusher(folderServerId).also { folderPusher -> + pushFolders[folderServerId] = folderPusher + } + } + .toList() + } + + for (folderPusher in stopFolderPushers) { + folderPusher.stop() + } + + for (folderPusher in startFolderPushers) { + folderPusher.start() + } + } + + override fun stop() { + Timber.v("ImapBackendPusher.stop()") + + coroutineScope.cancel() + + synchronized(lock) { + for (pushFolder in pushFolders.values) { + pushFolder.stop() + } + pushFolders.clear() + + for (retryTimer in pushFolderSleeping.values) { + retryTimer.cancel() + } + pushFolderSleeping.clear() + + currentFolderServerIds = emptySet() + } + } + + override fun reconnect() { + Timber.v("ImapBackendPusher.reconnect()") + + synchronized(lock) { + for (pushFolder in pushFolders.values) { + pushFolder.stop() + } + pushFolders.clear() + + for (retryTimer in pushFolderSleeping.values) { + retryTimer.cancel() + } + pushFolderSleeping.clear() + } + + imapStore.closeAllConnections() + + updateFolders() + } + + private fun createImapFolderPusher(folderServerId: String): ImapFolderPusher { + return ImapFolderPusher( + imapStore, + powerManager, + idleRefreshManager, + this, + accountName, + folderServerId, + idleRefreshTimeoutProvider + ) + } + + override fun onPushEvent(folderServerId: String) { + callback.onPushEvent(folderServerId) + idleRefreshManager.resetTimers() + } + + override fun onPushError(folderServerId: String, exception: Exception) { + synchronized(lock) { + pushFolders.remove(folderServerId) + + when (exception) { + is AuthenticationFailedException -> { + Timber.v(exception, "Authentication failure when attempting to use IDLE") + // TODO: This could be happening because of too many connections to the host. Ideally we'd want to + // detect this case and use a lower timeout. + + startRetryTimer(folderServerId, UNEXPECTED_ERROR_TIMEOUT) + } + is IOException -> { + Timber.v(exception, "I/O error while trying to use IDLE") + + startRetryTimer(folderServerId, IO_ERROR_TIMEOUT) + } + is MessagingException -> { + Timber.v(exception, "MessagingException") + + if (exception.isPermanentFailure) { + startRetryTimer(folderServerId, UNEXPECTED_ERROR_TIMEOUT) + } else { + startRetryTimer(folderServerId, IO_ERROR_TIMEOUT) + } + } + else -> { + Timber.v(exception, "Unexpected error") + startRetryTimer(folderServerId, UNEXPECTED_ERROR_TIMEOUT) + } + } + + if (pushFolders.isEmpty()) { + callback.onPushError(exception) + } + } + } + + override fun onPushNotSupported() { + callback.onPushNotSupported() + } + + private fun startRetryTimer(folderServerId: String, timeout: Long) { + Timber.v("ImapBackendPusher for folder %s sleeping for %d ms", folderServerId, timeout) + pushFolderSleeping[folderServerId] = idleRefreshManager.startTimer(timeout, ::restartFolderPushers) + } + + private fun cancelRetryTimer(folderServerId: String) { + Timber.v("Canceling ImapBackendPusher retry timer for folder %s", folderServerId) + pushFolderSleeping.remove(folderServerId)?.cancel() + } + + private fun isWaitingForRetry(folderServerId: String): Boolean { + return pushFolderSleeping[folderServerId]?.isWaiting == true + } + + private fun restartFolderPushers() { + Timber.v("Refreshing ImapBackendPusher (at least one retry timer has expired)") + + updateFolders() + } +} diff --git a/backend/imap/src/main/java/com/fsck/k9/backend/imap/ImapFolderPusher.kt b/backend/imap/src/main/java/com/fsck/k9/backend/imap/ImapFolderPusher.kt new file mode 100644 index 0000000..ef3ffe0 --- /dev/null +++ b/backend/imap/src/main/java/com/fsck/k9/backend/imap/ImapFolderPusher.kt @@ -0,0 +1,101 @@ +package com.fsck.k9.backend.imap + +import com.fsck.k9.logging.Timber +import com.fsck.k9.mail.power.PowerManager +import com.fsck.k9.mail.store.imap.IdleRefreshManager +import com.fsck.k9.mail.store.imap.IdleRefreshTimeoutProvider +import com.fsck.k9.mail.store.imap.IdleResult +import com.fsck.k9.mail.store.imap.ImapFolderIdler +import com.fsck.k9.mail.store.imap.ImapStore +import kotlin.concurrent.thread + +/** + * Listens for changes to an IMAP folder in a dedicated thread. + */ +class ImapFolderPusher( + private val imapStore: ImapStore, + private val powerManager: PowerManager, + private val idleRefreshManager: IdleRefreshManager, + private val callback: ImapPusherCallback, + private val accountName: String, + private val folderServerId: String, + private val idleRefreshTimeoutProvider: IdleRefreshTimeoutProvider +) { + @Volatile + private var folderIdler: ImapFolderIdler? = null + + @Volatile + private var stopPushing = false + + fun start() { + Timber.v("Starting ImapFolderPusher for %s / %s", accountName, folderServerId) + + thread(name = "ImapFolderPusher-$accountName-$folderServerId") { + Timber.v("Starting ImapFolderPusher thread for %s / %s", accountName, folderServerId) + + runPushLoop() + + Timber.v("Exiting ImapFolderPusher thread for %s / %s", accountName, folderServerId) + } + } + + fun refresh() { + Timber.v("Refreshing ImapFolderPusher for %s / %s", accountName, folderServerId) + + folderIdler?.refresh() + } + + fun stop() { + Timber.v("Stopping ImapFolderPusher for %s / %s", accountName, folderServerId) + + stopPushing = true + folderIdler?.stop() + } + + private fun runPushLoop() { + val wakeLock = powerManager.newWakeLock("ImapFolderPusher-$accountName-$folderServerId") + wakeLock.acquire() + + performInitialSync() + + val folderIdler = ImapFolderIdler.create( + idleRefreshManager, + wakeLock, + imapStore, + folderServerId, + idleRefreshTimeoutProvider + ).also { + folderIdler = it + } + + try { + while (!stopPushing) { + when (folderIdler.idle()) { + IdleResult.SYNC -> { + callback.onPushEvent(folderServerId) + } + IdleResult.STOPPED -> { + // ImapFolderIdler only stops when we ask it to. + // But it can't hurt to make extra sure we exit the loop. + stopPushing = true + } + IdleResult.NOT_SUPPORTED -> { + stopPushing = true + callback.onPushNotSupported() + } + } + } + } catch (e: Exception) { + Timber.v(e, "Exception in ImapFolderPusher") + + this.folderIdler = null + callback.onPushError(folderServerId, e) + } + + wakeLock.release() + } + + private fun performInitialSync() { + callback.onPushEvent(folderServerId) + } +} diff --git a/backend/imap/src/main/java/com/fsck/k9/backend/imap/ImapPushConfigProvider.kt b/backend/imap/src/main/java/com/fsck/k9/backend/imap/ImapPushConfigProvider.kt new file mode 100644 index 0000000..74cb31e --- /dev/null +++ b/backend/imap/src/main/java/com/fsck/k9/backend/imap/ImapPushConfigProvider.kt @@ -0,0 +1,8 @@ +package com.fsck.k9.backend.imap + +import kotlinx.coroutines.flow.Flow + +interface ImapPushConfigProvider { + val maxPushFoldersFlow: Flow<Int> + val idleRefreshMinutesFlow: Flow<Int> +} diff --git a/backend/imap/src/main/java/com/fsck/k9/backend/imap/ImapPusherCallback.kt b/backend/imap/src/main/java/com/fsck/k9/backend/imap/ImapPusherCallback.kt new file mode 100644 index 0000000..642668e --- /dev/null +++ b/backend/imap/src/main/java/com/fsck/k9/backend/imap/ImapPusherCallback.kt @@ -0,0 +1,7 @@ +package com.fsck.k9.backend.imap + +interface ImapPusherCallback { + fun onPushEvent(folderServerId: String) + fun onPushError(folderServerId: String, exception: Exception) + fun onPushNotSupported() +} diff --git a/backend/imap/src/main/java/com/fsck/k9/backend/imap/ImapSync.kt b/backend/imap/src/main/java/com/fsck/k9/backend/imap/ImapSync.kt new file mode 100644 index 0000000..f5276a9 --- /dev/null +++ b/backend/imap/src/main/java/com/fsck/k9/backend/imap/ImapSync.kt @@ -0,0 +1,729 @@ +package com.fsck.k9.backend.imap + +import com.fsck.k9.backend.api.BackendFolder +import com.fsck.k9.backend.api.BackendFolder.MoreMessages +import com.fsck.k9.backend.api.BackendStorage +import com.fsck.k9.backend.api.SyncConfig +import com.fsck.k9.backend.api.SyncConfig.ExpungePolicy +import com.fsck.k9.backend.api.SyncListener +import com.fsck.k9.helper.ExceptionHelper +import com.fsck.k9.logging.Timber +import com.fsck.k9.mail.AuthenticationFailedException +import com.fsck.k9.mail.BodyFactory +import com.fsck.k9.mail.DefaultBodyFactory +import com.fsck.k9.mail.FetchProfile +import com.fsck.k9.mail.Flag +import com.fsck.k9.mail.MessageDownloadState +import com.fsck.k9.mail.internet.MessageExtractor +import com.fsck.k9.mail.store.imap.FetchListener +import com.fsck.k9.mail.store.imap.ImapFolder +import com.fsck.k9.mail.store.imap.ImapMessage +import com.fsck.k9.mail.store.imap.ImapStore +import com.fsck.k9.mail.store.imap.OpenMode +import java.util.Collections +import java.util.Date +import java.util.concurrent.atomic.AtomicInteger +import kotlin.math.max + +internal class ImapSync( + private val accountName: String, + private val backendStorage: BackendStorage, + private val imapStore: ImapStore +) { + fun sync(folder: String, syncConfig: SyncConfig, listener: SyncListener) { + synchronizeMailboxSynchronous(folder, syncConfig, listener) + } + + private fun synchronizeMailboxSynchronous(folder: String, syncConfig: SyncConfig, listener: SyncListener) { + Timber.i("Synchronizing folder %s:%s", accountName, folder) + + var remoteFolder: ImapFolder? = null + var backendFolder: BackendFolder? = null + var newHighestKnownUid: Long = 0 + try { + Timber.v("SYNC: About to get local folder %s", folder) + + backendFolder = backendStorage.getFolder(folder) + + listener.syncStarted(folder) + + Timber.v("SYNC: About to get remote folder %s", folder) + remoteFolder = imapStore.getFolder(folder) + + /* + * Synchronization process: + * + Open the folder + Upload any local messages that are marked as PENDING_UPLOAD (Drafts, Sent, Trash) + Get the message count + Get the list of the newest K9.DEFAULT_VISIBLE_LIMIT messages + getMessages(messageCount - K9.DEFAULT_VISIBLE_LIMIT, messageCount) + See if we have each message locally, if not fetch it's flags and envelope + Get and update the unread count for the folder + Update the remote flags of any messages we have locally with an internal date newer than the remote message. + Get the current flags for any messages we have locally but did not just download + Update local flags + For any message we have locally but not remotely, delete the local message to keep cache clean. + Download larger parts of any new messages. + (Optional) Download small attachments in the background. + */ + + /* + * Open the remote folder. This pre-loads certain metadata like message count. + */ + Timber.v("SYNC: About to open remote folder %s", folder) + + if (syncConfig.expungePolicy === ExpungePolicy.ON_POLL) { + Timber.d("SYNC: Expunging folder %s:%s", accountName, folder) + remoteFolder.expunge() + } + + remoteFolder.open(OpenMode.READ_ONLY) + + listener.syncAuthenticationSuccess() + + val uidValidity = remoteFolder.getUidValidity() + val oldUidValidity = backendFolder.getFolderExtraNumber(EXTRA_UID_VALIDITY) + if (oldUidValidity == null && uidValidity != null) { + Timber.d("SYNC: Saving UIDVALIDITY for %s", folder) + backendFolder.setFolderExtraNumber(EXTRA_UID_VALIDITY, uidValidity) + } else if (oldUidValidity != null && oldUidValidity != uidValidity) { + Timber.d("SYNC: UIDVALIDITY for %s changed; clearing local message cache", folder) + backendFolder.clearAllMessages() + backendFolder.setFolderExtraNumber(EXTRA_UID_VALIDITY, uidValidity!!) + backendFolder.setFolderExtraNumber(EXTRA_HIGHEST_KNOWN_UID, 0) + } + + /* + * Get the message list from the local store and create an index of + * the uids within the list. + */ + + val highestKnownUid = backendFolder.getFolderExtraNumber(EXTRA_HIGHEST_KNOWN_UID) ?: 0 + var localUidMap: Map<String, Long?>? = backendFolder.getAllMessagesAndEffectiveDates() + + /* + * Get the remote message count. + */ + val remoteMessageCount = remoteFolder.messageCount + + var visibleLimit = backendFolder.visibleLimit + if (visibleLimit < 0) { + visibleLimit = syncConfig.defaultVisibleLimit + } + + val remoteMessages = mutableListOf<ImapMessage>() + val remoteUidMap = mutableMapOf<String, ImapMessage>() + + Timber.v("SYNC: Remote message count for folder %s is %d", folder, remoteMessageCount) + + val earliestDate = syncConfig.earliestPollDate + val earliestTimestamp = earliestDate?.time ?: 0L + + var remoteStart = 1 + if (remoteMessageCount > 0) { + /* Message numbers start at 1. */ + remoteStart = if (visibleLimit > 0) { + max(0, remoteMessageCount - visibleLimit) + 1 + } else { + 1 + } + + Timber.v( + "SYNC: About to get messages %d through %d for folder %s", + remoteStart, + remoteMessageCount, + folder + ) + + val headerProgress = AtomicInteger(0) + listener.syncHeadersStarted(folder) + + val remoteMessageArray = remoteFolder.getMessages(remoteStart, remoteMessageCount, earliestDate, null) + + val messageCount = remoteMessageArray.size + + for (thisMess in remoteMessageArray) { + headerProgress.incrementAndGet() + listener.syncHeadersProgress(folder, headerProgress.get(), messageCount) + + val uid = thisMess.uid.toLong() + if (uid > highestKnownUid && uid > newHighestKnownUid) { + newHighestKnownUid = uid + } + + val localMessageTimestamp = localUidMap!![thisMess.uid] + if (localMessageTimestamp == null || localMessageTimestamp >= earliestTimestamp) { + remoteMessages.add(thisMess) + remoteUidMap[thisMess.uid] = thisMess + } + } + + Timber.v("SYNC: Got %d messages for folder %s", remoteUidMap.size, folder) + + listener.syncHeadersFinished(folder, headerProgress.get(), remoteUidMap.size) + } else if (remoteMessageCount < 0) { + throw Exception("Message count $remoteMessageCount for folder $folder") + } + + /* + * Remove any messages that are in the local store but no longer on the remote store or are too old + */ + var moreMessages = backendFolder.getMoreMessages() + if (syncConfig.syncRemoteDeletions) { + val destroyMessageUids = mutableListOf<String>() + for (localMessageUid in localUidMap!!.keys) { + if (remoteUidMap[localMessageUid] == null) { + destroyMessageUids.add(localMessageUid) + } + } + + if (destroyMessageUids.isNotEmpty()) { + moreMessages = MoreMessages.UNKNOWN + backendFolder.destroyMessages(destroyMessageUids) + for (uid in destroyMessageUids) { + listener.syncRemovedMessage(folder, uid) + } + } + } + + @Suppress("UNUSED_VALUE") // free memory early? (better break up the method!) + localUidMap = null + + if (moreMessages === MoreMessages.UNKNOWN) { + updateMoreMessages(remoteFolder, backendFolder, earliestDate, remoteStart) + } + + /* + * Now we download the actual content of messages. + */ + downloadMessages( + syncConfig, + remoteFolder, + backendFolder, + remoteMessages, + highestKnownUid, + listener + ) + + listener.folderStatusChanged(folder) + + /* Notify listeners that we're finally done. */ + + backendFolder.setLastChecked(System.currentTimeMillis()) + backendFolder.setStatus(null) + + Timber.d("Done synchronizing folder %s:%s @ %tc", accountName, folder, System.currentTimeMillis()) + + listener.syncFinished(folder) + + Timber.i("Done synchronizing folder %s:%s", accountName, folder) + } catch (e: AuthenticationFailedException) { + listener.syncFailed(folder, "Authentication failure", e) + } catch (e: Exception) { + Timber.e(e, "synchronizeMailbox") + // If we don't set the last checked, it can try too often during + // failure conditions + val rootMessage = ExceptionHelper.getRootCauseMessage(e) + if (backendFolder != null) { + try { + backendFolder.setStatus(rootMessage) + backendFolder.setLastChecked(System.currentTimeMillis()) + } catch (e: Exception) { + Timber.e(e, "Could not set last checked on folder %s:%s", accountName, folder) + } + } + + listener.syncFailed(folder, rootMessage, e) + + Timber.e( + "Failed synchronizing folder %s:%s @ %tc", + accountName, + folder, + System.currentTimeMillis() + ) + } finally { + if (newHighestKnownUid > 0 && backendFolder != null) { + Timber.v("Saving new highest known UID: %d", newHighestKnownUid) + backendFolder.setFolderExtraNumber(EXTRA_HIGHEST_KNOWN_UID, newHighestKnownUid) + } + remoteFolder?.close() + } + } + + fun downloadMessage(syncConfig: SyncConfig, folderServerId: String, messageServerId: String) { + val backendFolder = backendStorage.getFolder(folderServerId) + val remoteFolder = imapStore.getFolder(folderServerId) + try { + remoteFolder.open(OpenMode.READ_ONLY) + val remoteMessage = remoteFolder.getMessage(messageServerId) + + downloadMessages( + syncConfig, + remoteFolder, + backendFolder, + listOf(remoteMessage), + null, + SimpleSyncListener() + ) + } finally { + remoteFolder.close() + } + } + + /** + * Fetches the messages described by inputMessages from the remote store and writes them to local storage. + * + * @param remoteFolder + * The remote folder to download messages from. + * @param backendFolder + * The [BackendFolder] instance corresponding to the remote folder. + * @param inputMessages + * A list of messages objects that store the UIDs of which messages to download. + */ + private fun downloadMessages( + syncConfig: SyncConfig, + remoteFolder: ImapFolder, + backendFolder: BackendFolder, + inputMessages: List<ImapMessage>, + highestKnownUid: Long?, + listener: SyncListener + ) { + val folder = remoteFolder.serverId + + val syncFlagMessages = mutableListOf<ImapMessage>() + var unsyncedMessages = mutableListOf<ImapMessage>() + val downloadedMessageCount = AtomicInteger(0) + + val messages = inputMessages.toMutableList() + for (message in messages) { + evaluateMessageForDownload( + message, + backendFolder, + unsyncedMessages, + syncFlagMessages + ) + } + + val progress = AtomicInteger(0) + val todo = unsyncedMessages.size + syncFlagMessages.size + listener.syncProgress(folder, progress.get(), todo) + + Timber.d("SYNC: Have %d unsynced messages", unsyncedMessages.size) + + messages.clear() + val largeMessages = mutableListOf<ImapMessage>() + val smallMessages = mutableListOf<ImapMessage>() + if (unsyncedMessages.isNotEmpty()) { + Collections.sort(unsyncedMessages, UidReverseComparator()) + val visibleLimit = backendFolder.visibleLimit + val listSize = unsyncedMessages.size + + if (visibleLimit in 1 until listSize) { + unsyncedMessages = unsyncedMessages.subList(0, visibleLimit) + } + + Timber.d("SYNC: About to fetch %d unsynced messages for folder %s", unsyncedMessages.size, folder) + + fetchUnsyncedMessages( + syncConfig, + remoteFolder, + unsyncedMessages, + smallMessages, + largeMessages, + progress, + todo, + listener + ) + + Timber.d("SYNC: Synced unsynced messages for folder %s", folder) + } + + Timber.d( + "SYNC: Have %d large messages and %d small messages out of %d unsynced messages", + largeMessages.size, + smallMessages.size, + unsyncedMessages.size + ) + + unsyncedMessages.clear() + + /* + * Grab the content of the small messages first. This is going to + * be very fast and at very worst will be a single up of a few bytes and a single + * download of 625k. + */ + val maxDownloadSize = syncConfig.maximumAutoDownloadMessageSize + // TODO: Only fetch small and large messages if we have some + downloadSmallMessages( + remoteFolder, + backendFolder, + smallMessages, + progress, + downloadedMessageCount, + todo, + highestKnownUid, + listener + ) + smallMessages.clear() + + /* + * Now do the large messages that require more round trips. + */ + downloadLargeMessages( + remoteFolder, + backendFolder, + largeMessages, + progress, + downloadedMessageCount, + todo, + highestKnownUid, + listener, + maxDownloadSize + ) + largeMessages.clear() + + /* + * Refresh the flags for any messages in the local store that we didn't just + * download. + */ + refreshLocalMessageFlags(syncConfig, remoteFolder, backendFolder, syncFlagMessages, progress, todo, listener) + + Timber.d("SYNC: Synced remote messages for folder %s, %d new messages", folder, downloadedMessageCount.get()) + } + + private fun evaluateMessageForDownload( + message: ImapMessage, + backendFolder: BackendFolder, + unsyncedMessages: MutableList<ImapMessage>, + syncFlagMessages: MutableList<ImapMessage> + ) { + val messageServerId = message.uid + if (message.isSet(Flag.DELETED)) { + Timber.v("Message with uid %s is marked as deleted", messageServerId) + syncFlagMessages.add(message) + return + } + + val messagePresentLocally = backendFolder.isMessagePresent(messageServerId) + if (!messagePresentLocally) { + Timber.v("Message with uid %s has not yet been downloaded", messageServerId) + unsyncedMessages.add(message) + return + } + + val messageFlags = backendFolder.getMessageFlags(messageServerId) + if (!messageFlags.contains(Flag.DELETED)) { + Timber.v("Message with uid %s is present in the local store", messageServerId) + if (!messageFlags.contains(Flag.X_DOWNLOADED_FULL) && !messageFlags.contains(Flag.X_DOWNLOADED_PARTIAL)) { + Timber.v("Message with uid %s is not downloaded, even partially; trying again", messageServerId) + unsyncedMessages.add(message) + } else { + syncFlagMessages.add(message) + } + } else { + Timber.v("Local copy of message with uid %s is marked as deleted", messageServerId) + } + } + + private fun isOldMessage(messageServerId: String, highestKnownUid: Long?): Boolean { + if (highestKnownUid == null) return false + + try { + val messageUid = messageServerId.toLong() + return messageUid <= highestKnownUid + } catch (e: NumberFormatException) { + Timber.w(e, "Couldn't parse UID: %s", messageServerId) + } + + return false + } + + private fun fetchUnsyncedMessages( + syncConfig: SyncConfig, + remoteFolder: ImapFolder, + unsyncedMessages: List<ImapMessage>, + smallMessages: MutableList<ImapMessage>, + largeMessages: MutableList<ImapMessage>, + progress: AtomicInteger, + todo: Int, + listener: SyncListener + ) { + val folder = remoteFolder.serverId + val fetchProfile = FetchProfile().apply { + add(FetchProfile.Item.FLAGS) + add(FetchProfile.Item.ENVELOPE) + } + + remoteFolder.fetch( + unsyncedMessages, + fetchProfile, + object : FetchListener { + override fun onFetchResponse(message: ImapMessage, isFirstResponse: Boolean) { + try { + if (message.isSet(Flag.DELETED)) { + Timber.v( + "Newly downloaded message %s:%s:%s was marked deleted on server, skipping", + accountName, + folder, + message.uid + ) + + if (isFirstResponse) { + progress.incrementAndGet() + } + + // TODO: This might be the source of poll count errors in the UI. Is todo always the same as ofTotal + listener.syncProgress(folder, progress.get(), todo) + + return + } + + if (syncConfig.maximumAutoDownloadMessageSize > 0 && + message.size > syncConfig.maximumAutoDownloadMessageSize + ) { + largeMessages.add(message) + } else { + smallMessages.add(message) + } + } catch (e: Exception) { + Timber.e(e, "Error while storing downloaded message.") + } + } + }, + syncConfig.maximumAutoDownloadMessageSize + ) + } + + private fun downloadSmallMessages( + remoteFolder: ImapFolder, + backendFolder: BackendFolder, + smallMessages: List<ImapMessage>, + progress: AtomicInteger, + downloadedMessageCount: AtomicInteger, + todo: Int, + highestKnownUid: Long?, + listener: SyncListener + ) { + val folder = remoteFolder.serverId + val fetchProfile = FetchProfile().apply { + add(FetchProfile.Item.BODY) + } + + Timber.d("SYNC: Fetching %d small messages for folder %s", smallMessages.size, folder) + + remoteFolder.fetch( + smallMessages, + fetchProfile, + object : FetchListener { + override fun onFetchResponse(message: ImapMessage, isFirstResponse: Boolean) { + try { + // Store the updated message locally + backendFolder.saveMessage(message, MessageDownloadState.FULL) + + if (isFirstResponse) { + progress.incrementAndGet() + downloadedMessageCount.incrementAndGet() + } + + val messageServerId = message.uid + Timber.v( + "About to notify listeners that we got a new small message %s:%s:%s", + accountName, + folder, + messageServerId + ) + + // Update the listener with what we've found + listener.syncProgress(folder, progress.get(), todo) + + val isOldMessage = isOldMessage(messageServerId, highestKnownUid) + listener.syncNewMessage(folder, messageServerId, isOldMessage) + } catch (e: Exception) { + Timber.e(e, "SYNC: fetch small messages") + } + } + }, + -1 + ) + + Timber.d("SYNC: Done fetching small messages for folder %s", folder) + } + + private fun downloadLargeMessages( + remoteFolder: ImapFolder, + backendFolder: BackendFolder, + largeMessages: List<ImapMessage>, + progress: AtomicInteger, + downloadedMessageCount: AtomicInteger, + todo: Int, + highestKnownUid: Long?, + listener: SyncListener, + maxDownloadSize: Int + ) { + val folder = remoteFolder.serverId + val fetchProfile = FetchProfile().apply { + add(FetchProfile.Item.STRUCTURE) + } + + Timber.d("SYNC: Fetching large messages for folder %s", folder) + + remoteFolder.fetch(largeMessages, fetchProfile, null, maxDownloadSize) + for (message in largeMessages) { + if (message.body == null) { + downloadSaneBody(remoteFolder, backendFolder, message, maxDownloadSize) + } else { + downloadPartial(remoteFolder, backendFolder, message, maxDownloadSize) + } + + val messageServerId = message.uid + Timber.v( + "About to notify listeners that we got a new large message %s:%s:%s", + accountName, + folder, + messageServerId + ) + + // Update the listener with what we've found + progress.incrementAndGet() + downloadedMessageCount.incrementAndGet() + + listener.syncProgress(folder, progress.get(), todo) + + val isOldMessage = isOldMessage(messageServerId, highestKnownUid) + listener.syncNewMessage(folder, messageServerId, isOldMessage) + } + + Timber.d("SYNC: Done fetching large messages for folder %s", folder) + } + + private fun refreshLocalMessageFlags( + syncConfig: SyncConfig, + remoteFolder: ImapFolder, + backendFolder: BackendFolder, + syncFlagMessages: List<ImapMessage>, + progress: AtomicInteger, + todo: Int, + listener: SyncListener + ) { + val folder = remoteFolder.serverId + Timber.d("SYNC: About to sync flags for %d remote messages for folder %s", syncFlagMessages.size, folder) + + val fetchProfile = FetchProfile() + fetchProfile.add(FetchProfile.Item.FLAGS) + + val undeletedMessages = mutableListOf<ImapMessage>() + for (message in syncFlagMessages) { + if (!message.isSet(Flag.DELETED)) { + undeletedMessages.add(message) + } + } + + val maxDownloadSize = syncConfig.maximumAutoDownloadMessageSize + remoteFolder.fetch(undeletedMessages, fetchProfile, null, maxDownloadSize) + for (remoteMessage in syncFlagMessages) { + val messageChanged = syncFlags(syncConfig, backendFolder, remoteMessage) + if (messageChanged) { + listener.syncFlagChanged(folder, remoteMessage.uid) + } + progress.incrementAndGet() + listener.syncProgress(folder, progress.get(), todo) + } + } + + private fun downloadSaneBody( + remoteFolder: ImapFolder, + backendFolder: BackendFolder, + message: ImapMessage, + maxDownloadSize: Int + ) { + /* + * The provider was unable to get the structure of the message, so + * we'll download a reasonable portion of the message and mark it as + * incomplete so the entire thing can be downloaded later if the user + * wishes to download it. + */ + val fetchProfile = FetchProfile() + fetchProfile.add(FetchProfile.Item.BODY_SANE) + /* + * TODO a good optimization here would be to make sure that all Stores set + * the proper size after this fetch and compare the before and after size. If + * they equal we can mark this SYNCHRONIZED instead of PARTIALLY_SYNCHRONIZED + */ + remoteFolder.fetch(listOf(message), fetchProfile, null, maxDownloadSize) + + // Store the updated message locally + backendFolder.saveMessage(message, MessageDownloadState.PARTIAL) + } + + private fun downloadPartial( + remoteFolder: ImapFolder, + backendFolder: BackendFolder, + message: ImapMessage, + maxDownloadSize: Int + ) { + /* + * We have a structure to deal with, from which + * we can pull down the parts we want to actually store. + * Build a list of parts we are interested in. Text parts will be downloaded + * right now, attachments will be left for later. + */ + val viewables = MessageExtractor.collectTextParts(message) + + /* + * Now download the parts we're interested in storing. + */ + val bodyFactory: BodyFactory = DefaultBodyFactory() + for (part in viewables) { + remoteFolder.fetchPart(message, part, bodyFactory, maxDownloadSize) + } + + // Store the updated message locally + backendFolder.saveMessage(message, MessageDownloadState.PARTIAL) + } + + private fun syncFlags(syncConfig: SyncConfig, backendFolder: BackendFolder, remoteMessage: ImapMessage): Boolean { + val messageServerId = remoteMessage.uid + if (!backendFolder.isMessagePresent(messageServerId)) return false + + val localMessageFlags = backendFolder.getMessageFlags(messageServerId) + if (localMessageFlags.contains(Flag.DELETED)) return false + + var messageChanged = false + if (remoteMessage.isSet(Flag.DELETED)) { + if (syncConfig.syncRemoteDeletions) { + backendFolder.setMessageFlag(messageServerId, Flag.DELETED, true) + messageChanged = true + } + } else { + for (flag in syncConfig.syncFlags) { + if (remoteMessage.isSet(flag) != localMessageFlags.contains(flag)) { + backendFolder.setMessageFlag(messageServerId, flag, remoteMessage.isSet(flag)) + messageChanged = true + } + } + } + + return messageChanged + } + + private fun updateMoreMessages( + remoteFolder: ImapFolder, + backendFolder: BackendFolder, + earliestDate: Date?, + remoteStart: Int + ) { + if (remoteStart == 1) { + backendFolder.setMoreMessages(MoreMessages.FALSE) + } else { + val moreMessagesAvailable = remoteFolder.areMoreMessagesAvailable(remoteStart, earliestDate) + val newMoreMessages = if (moreMessagesAvailable) MoreMessages.TRUE else MoreMessages.FALSE + backendFolder.setMoreMessages(newMoreMessages) + } + } + + companion object { + private const val EXTRA_UID_VALIDITY = "imapUidValidity" + private const val EXTRA_HIGHEST_KNOWN_UID = "imapHighestKnownUid" + } +} diff --git a/backend/imap/src/main/java/com/fsck/k9/backend/imap/SimpleSyncListener.kt b/backend/imap/src/main/java/com/fsck/k9/backend/imap/SimpleSyncListener.kt new file mode 100644 index 0000000..0119761 --- /dev/null +++ b/backend/imap/src/main/java/com/fsck/k9/backend/imap/SimpleSyncListener.kt @@ -0,0 +1,18 @@ +package com.fsck.k9.backend.imap + +import com.fsck.k9.backend.api.SyncListener + +class SimpleSyncListener : SyncListener { + override fun syncStarted(folderServerId: String) = Unit + override fun syncAuthenticationSuccess() = Unit + override fun syncHeadersStarted(folderServerId: String) = Unit + override fun syncHeadersProgress(folderServerId: String, completed: Int, total: Int) = Unit + override fun syncHeadersFinished(folderServerId: String, totalMessagesInMailbox: Int, numNewMessages: Int) = Unit + override fun syncProgress(folderServerId: String, completed: Int, total: Int) = Unit + override fun syncNewMessage(folderServerId: String, messageServerId: String, isOldMessage: Boolean) = Unit + override fun syncRemovedMessage(folderServerId: String, messageServerId: String) = Unit + override fun syncFlagChanged(folderServerId: String, messageServerId: String) = Unit + override fun syncFinished(folderServerId: String) = Unit + override fun syncFailed(folderServerId: String, message: String, exception: Exception?) = Unit + override fun folderStatusChanged(folderServerId: String) = Unit +} diff --git a/backend/imap/src/main/java/com/fsck/k9/backend/imap/SystemAlarmManager.kt b/backend/imap/src/main/java/com/fsck/k9/backend/imap/SystemAlarmManager.kt new file mode 100644 index 0000000..e2133cf --- /dev/null +++ b/backend/imap/src/main/java/com/fsck/k9/backend/imap/SystemAlarmManager.kt @@ -0,0 +1,7 @@ +package com.fsck.k9.backend.imap + +interface SystemAlarmManager { + fun setAlarm(triggerTime: Long, callback: () -> Unit) + fun cancelAlarm() + fun now(): Long +} diff --git a/backend/imap/src/main/java/com/fsck/k9/backend/imap/UidReverseComparator.kt b/backend/imap/src/main/java/com/fsck/k9/backend/imap/UidReverseComparator.kt new file mode 100644 index 0000000..58d8ae5 --- /dev/null +++ b/backend/imap/src/main/java/com/fsck/k9/backend/imap/UidReverseComparator.kt @@ -0,0 +1,24 @@ +package com.fsck.k9.backend.imap + +import com.fsck.k9.mail.Message +import java.util.Comparator + +internal class UidReverseComparator : Comparator<Message> { + override fun compare(messageLeft: Message, messageRight: Message): Int { + val uidLeft = messageLeft.uidOrNull + val uidRight = messageRight.uidOrNull + if (uidLeft == null && uidRight == null) { + return 0 + } else if (uidLeft == null) { + return 1 + } else if (uidRight == null) { + return -1 + } + + // reverse order + return uidRight.compareTo(uidLeft) + } + + private val Message.uidOrNull + get() = uid?.toLongOrNull() +} |
