diff --git a/app/src/main/java/eu/kanade/domain/sync/SyncPreferences.kt b/app/src/main/java/eu/kanade/domain/sync/SyncPreferences.kt index d83a2b38ba..71aeb45d5f 100644 --- a/app/src/main/java/eu/kanade/domain/sync/SyncPreferences.kt +++ b/app/src/main/java/eu/kanade/domain/sync/SyncPreferences.kt @@ -13,6 +13,8 @@ class SyncPreferences( fun clientAPIKey() = preferenceStore.getString("sync_client_api_key", "") fun lastSyncTimestamp() = preferenceStore.getLong(Preference.appStateKey("last_sync_timestamp"), 0L) + fun lastSyncEtag() = preferenceStore.getString("sync_etag", "") + fun syncInterval() = preferenceStore.getInt("sync_interval", 0) fun syncService() = preferenceStore.getInt("sync_service", 0) diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/sync/SyncManager.kt b/app/src/main/java/eu/kanade/tachiyomi/data/sync/SyncManager.kt index 8ea29fc678..3ef70d6530 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/sync/SyncManager.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/sync/SyncManager.kt @@ -28,7 +28,6 @@ import tachiyomi.domain.manga.model.Manga import uy.kohesive.injekt.Injekt import uy.kohesive.injekt.api.get import java.io.File -import java.io.FileOutputStream import java.io.IOException import java.util.Date import kotlin.system.measureTimeMillis @@ -90,6 +89,8 @@ class SyncManager( sourceSettings = syncOptions.sourceSettings, privateSettings = syncOptions.privateSettings, ) + + logcat(LogPriority.DEBUG) { "Begin create backup" } val backup = Backup( backupManga = backupCreator.backupMangas(databaseManga, backupOptions), backupCategories = backupCreator.backupCategories(backupOptions), @@ -97,9 +98,11 @@ class SyncManager( backupPreferences = backupCreator.backupAppPreferences(backupOptions), backupSourcePreferences = backupCreator.backupSourcePreferences(backupOptions), ) + logcat(LogPriority.DEBUG) { "End create backup" } // Create the SyncData object val syncData = SyncData( + deviceId = syncPreferences.uniqueDeviceID(), backup = backup, ) @@ -126,8 +129,22 @@ class SyncManager( val remoteBackup = syncService?.doSync(syncData) + if (remoteBackup == null) { + logcat(LogPriority.DEBUG) { "Skip restore due to network issues" } + // should we call showSyncError? + return + } + + if (remoteBackup === syncData.backup){ + // nothing changed + logcat(LogPriority.DEBUG) { "Skip restore due to remote was overwrite from local" } + syncPreferences.lastSyncTimestamp().set(Date().time) + notifier.showSyncSuccess("Sync completed successfully") + return + } + // Stop the sync early if the remote backup is null or empty - if (remoteBackup?.backupManga?.size == 0) { + if (remoteBackup.backupManga?.size == 0) { notifier.showSyncError("No data found on remote server.") return } @@ -140,52 +157,51 @@ class SyncManager( return } - if (remoteBackup != null) { - val (filteredFavorites, nonFavorites) = filterFavoritesAndNonFavorites(remoteBackup) - updateNonFavorites(nonFavorites) + val (filteredFavorites, nonFavorites) = filterFavoritesAndNonFavorites(remoteBackup) + updateNonFavorites(nonFavorites) - val newSyncData = backup.copy( - backupManga = filteredFavorites, - backupCategories = remoteBackup.backupCategories, - backupSources = remoteBackup.backupSources, - backupPreferences = remoteBackup.backupPreferences, - backupSourcePreferences = remoteBackup.backupSourcePreferences, - ) + val newSyncData = backup.copy( + backupManga = filteredFavorites, + backupCategories = remoteBackup.backupCategories, + backupSources = remoteBackup.backupSources, + backupPreferences = remoteBackup.backupPreferences, + backupSourcePreferences = remoteBackup.backupSourcePreferences, - // It's local sync no need to restore data. (just update remote data) - if (filteredFavorites.isEmpty()) { - // update the sync timestamp - syncPreferences.lastSyncTimestamp().set(Date().time) - notifier.showSyncSuccess("Sync completed successfully") - return - } + ) - val backupUri = writeSyncDataToCache(context, newSyncData) - logcat(LogPriority.DEBUG) { "Got Backup Uri: $backupUri" } - if (backupUri != null) { - BackupRestoreJob.start( - context, - backupUri, - sync = true, - options = RestoreOptions( - appSettings = true, - sourceSettings = true, - library = true, - ), - ) + // It's local sync no need to restore data. (just update remote data) + if (filteredFavorites.isEmpty()) { + // update the sync timestamp + syncPreferences.lastSyncTimestamp().set(Date().time) + notifier.showSyncSuccess("Sync completed successfully") + return + } - // update the sync timestamp - syncPreferences.lastSyncTimestamp().set(Date().time) - } else { - logcat(LogPriority.ERROR) { "Failed to write sync data to file" } - } + val backupUri = writeSyncDataToCache(context, newSyncData) + logcat(LogPriority.DEBUG) { "Got Backup Uri: $backupUri" } + if (backupUri != null) { + BackupRestoreJob.start( + context, + backupUri, + sync = true, + options = RestoreOptions( + appSettings = true, + sourceSettings = true, + library = true, + ), + ) + + // update the sync timestamp + syncPreferences.lastSyncTimestamp().set(Date().time) + } else { + logcat(LogPriority.ERROR) { "Failed to write sync data to file" } } } private fun writeSyncDataToCache(context: Context, backup: Backup): Uri? { val cacheFile = File(context.cacheDir, "tachiyomi_sync_data.proto.gz") return try { - FileOutputStream(cacheFile).use { output -> + cacheFile.outputStream().use { output -> output.write(ProtoBuf.encodeToByteArray(BackupSerializer, backup)) Uri.fromFile(cacheFile) } diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt b/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt index bbfac66c5e..29c16e9ed6 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/GoogleDriveSyncService.kt @@ -11,6 +11,7 @@ import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets import com.google.api.client.googleapis.auth.oauth2.GoogleCredential import com.google.api.client.googleapis.auth.oauth2.GoogleTokenResponse import com.google.api.client.http.ByteArrayContent +import com.google.api.client.http.InputStreamContent import com.google.api.client.http.javanet.NetHttpTransport import com.google.api.client.json.JsonFactory import com.google.api.client.json.jackson2.JacksonFactory @@ -18,9 +19,12 @@ import com.google.api.services.drive.Drive import com.google.api.services.drive.DriveScopes import com.google.api.services.drive.model.File import eu.kanade.domain.sync.SyncPreferences +import eu.kanade.tachiyomi.data.backup.models.Backup import kotlinx.coroutines.delay -import kotlinx.serialization.encodeToString +import kotlinx.coroutines.launch import kotlinx.serialization.json.Json +import kotlinx.serialization.json.decodeFromStream +import kotlinx.serialization.json.encodeToStream import logcat.LogPriority import logcat.logcat import tachiyomi.core.common.i18n.stringResource @@ -29,8 +33,9 @@ import tachiyomi.core.common.util.system.logcat import tachiyomi.i18n.MR import uy.kohesive.injekt.Injekt import uy.kohesive.injekt.api.get -import java.io.ByteArrayOutputStream import java.io.IOException +import java.io.PipedInputStream +import java.io.PipedOutputStream import java.time.Instant import java.util.zip.GZIPInputStream import java.util.zip.GZIPOutputStream @@ -64,7 +69,43 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync private val googleDriveService = GoogleDriveService(context) - override suspend fun beforeSync() { + override suspend fun doSync(syncData: SyncData): Backup? { + beforeSync() + + try { + val remoteSData = pullSyncData() + + if (remoteSData != null ){ + // Get local unique device ID + val localDeviceId = syncPreferences.uniqueDeviceID() + val lastSyncDeviceId = remoteSData.deviceId + + // Log the device IDs + logcat(LogPriority.DEBUG, "SyncService") { + "Local device ID: $localDeviceId, Last sync device ID: $lastSyncDeviceId" + } + + // check if the last sync was done by the same device if so overwrite the remote data with the local data + return if (lastSyncDeviceId == localDeviceId) { + pushSyncData(syncData) + syncData.backup + }else{ + // Merge the local and remote sync data + val mergedSyncData = mergeSyncData(syncData, remoteSData) + pushSyncData(mergedSyncData) + mergedSyncData.backup + } + } + + pushSyncData(syncData) + return syncData.backup + } catch (e: Exception) { + logcat(LogPriority.ERROR, "SyncService") { "Error syncing: ${e.message}" } + return null + } + } + + private suspend fun beforeSync() { try { googleDriveService.refreshToken() val drive = googleDriveService.driveService @@ -120,13 +161,9 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync } } - override suspend fun pullSyncData(): SyncData? { - val drive = googleDriveService.driveService - - if (drive == null) { - logcat(LogPriority.DEBUG) { "Google Drive service not initialized" } - throw Exception(context.stringResource(MR.strings.google_drive_not_signed_in)) - } + private fun pullSyncData(): SyncData? { + val drive = googleDriveService.driveService ?: + throw Exception(context.stringResource(MR.strings.google_drive_not_signed_in)) val fileList = getAppDataFileList(drive) if (fileList.isEmpty()) { @@ -137,75 +174,53 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync val gdriveFileId = fileList[0].id logcat(LogPriority.DEBUG) { "Google Drive File ID: $gdriveFileId" } - val outputStream = ByteArrayOutputStream() try { - drive.files().get(gdriveFileId).executeMediaAndDownloadTo(outputStream) - logcat(LogPriority.DEBUG) { "File downloaded successfully" } + drive.files().get(gdriveFileId).executeMediaAsInputStream().use { inputStream -> + GZIPInputStream(inputStream).use { gzipInputStream -> + return Json.decodeFromStream(SyncData.serializer(), gzipInputStream) + } + } } catch (e: Exception) { logcat(LogPriority.ERROR, throwable = e) { "Error downloading file" } - return null - } - - return withIOContext { - try { - val gzipInputStream = GZIPInputStream(outputStream.toByteArray().inputStream()) - val jsonString = gzipInputStream.bufferedReader().use { it.readText() } - val syncData = json.decodeFromString(SyncData.serializer(), jsonString) - this@GoogleDriveSyncService.logcat(LogPriority.DEBUG) { "JSON deserialized successfully" } - syncData - } catch (e: Exception) { - this@GoogleDriveSyncService.logcat( - LogPriority.ERROR, - throwable = e, - ) { "Failed to convert json to sync data with kotlinx.serialization" } - throw Exception(e.message, e) - } + throw Exception("Failed to download sync data: ${e.message}", e) } } - override suspend fun pushSyncData(syncData: SyncData) { - val jsonData = json.encodeToString(syncData) + private suspend fun pushSyncData(syncData: SyncData) { val drive = googleDriveService.driveService ?: throw Exception(context.stringResource(MR.strings.google_drive_not_signed_in)) val fileList = getAppDataFileList(drive) - val byteArrayOutputStream = ByteArrayOutputStream() - withIOContext { - val gzipOutputStream = GZIPOutputStream(byteArrayOutputStream) - gzipOutputStream.write(jsonData.toByteArray(Charsets.UTF_8)) - gzipOutputStream.close() - this@GoogleDriveSyncService.logcat(LogPriority.DEBUG) { "JSON serialized successfully" } - } - val byteArrayContent = ByteArrayContent("application/octet-stream", byteArrayOutputStream.toByteArray()) + PipedOutputStream().use { pos -> + PipedInputStream(pos).use { pis -> + withIOContext { + // Start a coroutine or a background thread to write JSON to the PipedOutputStream + launch { + GZIPOutputStream(pos).use { gzipOutputStream -> + Json.encodeToStream(SyncData.serializer(), syncData, gzipOutputStream) + } + } - try { - if (fileList.isNotEmpty()) { - // File exists, so update it - val fileId = fileList[0].id - drive.files().update(fileId, null, byteArrayContent).execute() - logcat(LogPriority.DEBUG) { "Updated existing sync data file in Google Drive with file ID: $fileId" } - } else { - // File doesn't exist, so create it - val fileMetadata = File().apply { - name = remoteFileName - mimeType = "application/gzip" - parents = listOf("appDataFolder") + if (fileList.isNotEmpty()) { + val fileId = fileList[0].id + val mediaContent = InputStreamContent("application/gzip", pis) + drive.files().update(fileId, null, mediaContent).execute() + logcat(LogPriority.DEBUG) { "Updated existing sync data file in Google Drive with file ID: $fileId" } + } else { + val fileMetadata = File().apply { + name = remoteFileName + mimeType = "application/gzip" + parents = listOf("appDataFolder") + } + val mediaContent = InputStreamContent("application/gzip", pis) + val uploadedFile = drive.files().create(fileMetadata, mediaContent) + .setFields("id") + .execute() + logcat(LogPriority.DEBUG) { "Created new sync data file in Google Drive with file ID: ${uploadedFile.id}" } + } } - val uploadedFile = drive.files().create(fileMetadata, byteArrayContent) - .setFields("id") - .execute() - - logcat( - LogPriority.DEBUG, - ) { "Created new sync data file in Google Drive with file ID: ${uploadedFile.id}" } } - - // Data has been successfully pushed or updated, delete the lock file - deleteLockFile(drive) - } catch (e: Exception) { - logcat(LogPriority.ERROR, throwable = e) { "Failed to push or update sync data" } - throw Exception(context.stringResource(MR.strings.error_uploading_sync_data) + ": ${e.message}", e) } } @@ -392,7 +407,6 @@ class GoogleDriveService(private val context: Context) { } internal suspend fun refreshToken() = withIOContext { val refreshToken = syncPreferences.googleDriveRefreshToken().get() - val accessToken = syncPreferences.googleDriveAccessToken().get() val jsonFactory: JsonFactory = JacksonFactory.getDefaultInstance() val secrets = GoogleClientSecrets.load( @@ -412,16 +426,12 @@ class GoogleDriveService(private val context: Context) { credential.refreshToken = refreshToken - this@GoogleDriveService.logcat(LogPriority.DEBUG) { "Refreshing access token with: $refreshToken" } - try { credential.refreshToken() val newAccessToken = credential.accessToken // Save the new access token syncPreferences.googleDriveAccessToken().set(newAccessToken) setupGoogleDriveService(newAccessToken, credential.refreshToken) - this@GoogleDriveService - .logcat(LogPriority.DEBUG) { "Google Access token refreshed old: $accessToken new: $newAccessToken" } } catch (e: TokenResponseException) { if (e.details.error == "invalid_grant") { // The refresh token is invalid, prompt the user to sign in again @@ -473,9 +483,10 @@ class GoogleDriveService(private val context: Context) { } /** - * Handles the authorization code returned after the user has granted the application permission to access their Google Drive account. - * It obtains the access token and refresh token using the authorization code, saves the tokens to the SyncPreferences, - * sets up the Google Drive service using the obtained tokens, and initializes the service. + * Handles the authorization code returned after the user has granted the application permission to access their + * Google Drive account. + * It obtains the access token and refresh token using the authorization code, saves the tokens to the + * SyncPreferences, sets up the Google Drive service using the obtained tokens, and initializes the service. * @param authorizationCode The authorization code obtained from the OAuthCallbackServer. * @param activity The current activity. * @param onSuccess A callback function to be called on successful authorization. diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/SyncService.kt b/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/SyncService.kt index 0490e0f58b..ee8324c5e4 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/SyncService.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/SyncService.kt @@ -16,6 +16,7 @@ import logcat.logcat @Serializable data class SyncData( + val deviceId: String = "", val backup: Backup? = null, ) @@ -24,38 +25,7 @@ abstract class SyncService( val json: Json, val syncPreferences: SyncPreferences, ) { - open suspend fun doSync(syncData: SyncData): Backup? { - beforeSync() - - val remoteSData = pullSyncData() - - val finalSyncData = - if (remoteSData == null) { - pushSyncData(syncData) - syncData - } else { - val mergedSyncData = mergeSyncData(syncData, remoteSData) - pushSyncData(mergedSyncData) - mergedSyncData - } - - return finalSyncData.backup - } - - /** - * For refreshing tokens and other possible operations before connecting to the remote storage - */ - open suspend fun beforeSync() {} - - /** - * Download sync data from the remote storage - */ - abstract suspend fun pullSyncData(): SyncData? - - /** - * Upload sync data to the remote storage - */ - abstract suspend fun pushSyncData(syncData: SyncData) + abstract suspend fun doSync(syncData: SyncData): Backup?; /** * Merges the local and remote sync data into a single JSON string. @@ -64,11 +34,17 @@ abstract class SyncService( * @param remoteSyncData The SData containing the remote sync data. * @return The JSON string containing the merged sync data. */ - private fun mergeSyncData(localSyncData: SyncData, remoteSyncData: SyncData): SyncData { - val mergedMangaList = mergeMangaLists(localSyncData.backup?.backupManga, remoteSyncData.backup?.backupManga) + protected fun mergeSyncData(localSyncData: SyncData, remoteSyncData: SyncData): SyncData { val mergedCategoriesList = mergeCategoriesLists(localSyncData.backup?.backupCategories, remoteSyncData.backup?.backupCategories) + val mergedMangaList = mergeMangaLists( + localSyncData.backup?.backupManga, + remoteSyncData.backup?.backupManga, + localSyncData.backup?.backupCategories ?: emptyList(), + remoteSyncData.backup?.backupCategories ?: emptyList(), + mergedCategoriesList) + val mergedSourcesList = mergeSourcesLists(localSyncData.backup?.backupSources, remoteSyncData.backup?.backupSources) val mergedPreferencesList = @@ -78,6 +54,7 @@ abstract class SyncService( remoteSyncData.backup?.backupSourcePreferences, ) + // Create the merged Backup object val mergedBackup = Backup( backupManga = mergedMangaList, @@ -85,17 +62,31 @@ abstract class SyncService( backupSources = mergedSourcesList, backupPreferences = mergedPreferencesList, backupSourcePreferences = mergedSourcePreferencesList, + ) // Create the merged SData object return SyncData( + deviceId = syncPreferences.uniqueDeviceID(), backup = mergedBackup, ) } + /** + * Merges two lists of BackupManga objects, selecting the most recent manga based on the lastModifiedAt value. + * If lastModifiedAt is null for a manga, it treats that manga as the oldest possible for comparison purposes. + * This function is designed to reconcile local and remote manga lists, ensuring the most up-to-date manga is retained. + * + * @param localMangaList The list of local BackupManga objects or null. + * @param remoteMangaList The list of remote BackupManga objects or null. + * @return A list of BackupManga objects, each representing the most recent version of the manga from either local or remote sources. + */ private fun mergeMangaLists( localMangaList: List?, remoteMangaList: List?, + localCategories: List, + remoteCategories: List, + mergedCategories: List, ): List { val logTag = "MergeMangaLists" @@ -114,6 +105,18 @@ abstract class SyncService( val localMangaMap = localMangaListSafe.associateBy { mangaCompositeKey(it) } val remoteMangaMap = remoteMangaListSafe.associateBy { mangaCompositeKey(it) } + val localCategoriesMapByOrder = localCategories.associateBy { it.order } + val remoteCategoriesMapByOrder = remoteCategories.associateBy { it.order } + val mergedCategoriesMapByName = mergedCategories.associateBy { it.name } + + fun updateCategories(theManga: BackupManga, theMap: Map): BackupManga { + return theManga.copy(categories = theManga.categories.mapNotNull { + theMap[it]?.let { category -> + mergedCategoriesMapByName[category.name]?.order + } + }) + } + logcat(LogPriority.DEBUG, logTag) { "Starting merge. Local list size: ${localMangaListSafe.size}, Remote list size: ${remoteMangaListSafe.size}" } @@ -124,22 +127,26 @@ abstract class SyncService( // New version comparison logic when { - local != null && remote == null -> local - local == null && remote != null -> remote + local != null && remote == null -> updateCategories(local, localCategoriesMapByOrder) + local == null && remote != null -> updateCategories(remote, remoteCategoriesMapByOrder) local != null && remote != null -> { // Compare versions to decide which manga to keep if (local.version >= remote.version) { - logcat( - LogPriority.DEBUG, - logTag - ) { "Keeping local version of ${local.title} with merged chapters." } - local.copy(chapters = mergeChapters(local.chapters, remote.chapters)) + logcat(LogPriority.DEBUG, logTag) { + "Keeping local version of ${local.title} with merged chapters." + } + updateCategories( + local.copy(chapters = mergeChapters(local.chapters, remote.chapters)), + localCategoriesMapByOrder + ) } else { - logcat( - LogPriority.DEBUG, - logTag - ) { "Keeping remote version of ${remote.title} with merged chapters." } - remote.copy(chapters = mergeChapters(local.chapters, remote.chapters)) + logcat(LogPriority.DEBUG, logTag) { + "Keeping remote version of ${remote.title} with merged chapters." + } + updateCategories( + remote.copy(chapters = mergeChapters(local.chapters, remote.chapters)), + remoteCategoriesMapByOrder + ) } } else -> null // No manga found for key @@ -157,6 +164,23 @@ abstract class SyncService( return mergedList } +/** + * Merges two lists of BackupChapter objects, selecting the most recent chapter based on the lastModifiedAt value. + * If lastModifiedAt is null for a chapter, it treats that chapter as the oldest possible for comparison purposes. + * This function is designed to reconcile local and remote chapter lists, ensuring the most up-to-date chapter is retained. + * + * @param localChapters The list of local BackupChapter objects. + * @param remoteChapters The list of remote BackupChapter objects. + * @return A list of BackupChapter objects, each representing the most recent version of the chapter from either local or remote sources. + * + * - This function is used in scenarios where local and remote chapter lists need to be synchronized. + * - It iterates over the union of the URLs from both local and remote chapters. + * - For each URL, it compares the corresponding local and remote chapters based on the lastModifiedAt value. + * - If only one source (local or remote) has the chapter for a URL, that chapter is used. + * - If both sources have the chapter, the one with the more recent lastModifiedAt value is chosen. + * - If lastModifiedAt is null or missing, the chapter is considered the oldest for safety, ensuring that any chapter with a valid timestamp is preferred. + * - The resulting list contains the most recent chapters from the combined set of local and remote chapters. + */ private fun mergeChapters( localChapters: List, remoteChapters: List, @@ -195,7 +219,11 @@ abstract class SyncService( } localChapter != null && remoteChapter != null -> { // Use version number to decide which chapter to keep - val chosenChapter = if (localChapter.version >= remoteChapter.version) localChapter else remoteChapter + val chosenChapter = if (localChapter.version >= remoteChapter.version) { + localChapter + } else { + remoteChapter + } logcat(LogPriority.DEBUG, logTag) { "Merging chapter: ${chosenChapter.name}. Chosen version from: ${ if (localChapter.version >= remoteChapter.version) "Local" else "Remote" @@ -217,6 +245,13 @@ abstract class SyncService( return mergedChapters } + /** + * Merges two lists of SyncCategory objects, prioritizing the category with the most recent order value. + * + * @param localCategoriesList The list of local SyncCategory objects. + * @param remoteCategoriesList The list of remote SyncCategory objects. + * @return The merged list of SyncCategory objects. + */ private fun mergeCategoriesLists( localCategoriesList: List?, remoteCategoriesList: List?, @@ -256,7 +291,7 @@ abstract class SyncService( private fun mergeSourcesLists( localSources: List?, - remoteSources: List?, + remoteSources: List? ): List { val logTag = "MergeSources" @@ -301,7 +336,7 @@ abstract class SyncService( private fun mergePreferencesLists( localPreferences: List?, - remotePreferences: List?, + remotePreferences: List? ): List { val logTag = "MergePreferences" @@ -349,7 +384,7 @@ abstract class SyncService( private fun mergeSourcePreferencesLists( localPreferences: List?, - remotePreferences: List?, + remotePreferences: List? ): List { val logTag = "MergeSourcePreferences" @@ -363,8 +398,7 @@ abstract class SyncService( } // Merge both source preferences maps - val mergedSourcePreferences = (localPreferencesMap.keys + remotePreferencesMap.keys).distinct().mapNotNull { - sourceKey -> + val mergedSourcePreferences = (localPreferencesMap.keys + remotePreferencesMap.keys).distinct().mapNotNull { sourceKey -> val localSourcePreference = localPreferencesMap[sourceKey] val remoteSourcePreference = remotePreferencesMap[sourceKey] @@ -406,7 +440,7 @@ abstract class SyncService( private fun mergeIndividualPreferences( localPrefs: List, - remotePrefs: List, + remotePrefs: List ): List { val mergedPrefsMap = (localPrefs + remotePrefs).associateBy { it.key } return mergedPrefsMap.values.toList() diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/SyncYomiSyncService.kt b/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/SyncYomiSyncService.kt index b878d3941c..2af4d28ffe 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/SyncYomiSyncService.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/sync/service/SyncYomiSyncService.kt @@ -2,23 +2,26 @@ package eu.kanade.tachiyomi.data.sync.service import android.content.Context import eu.kanade.domain.sync.SyncPreferences +import eu.kanade.tachiyomi.data.backup.models.Backup +import eu.kanade.tachiyomi.data.backup.models.BackupSerializer import eu.kanade.tachiyomi.data.sync.SyncNotifier import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.network.PATCH -import eu.kanade.tachiyomi.network.POST +import eu.kanade.tachiyomi.network.PUT import eu.kanade.tachiyomi.network.await -import kotlinx.coroutines.delay -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable -import kotlinx.serialization.encodeToString +import kotlinx.serialization.SerializationException import kotlinx.serialization.json.Json +import kotlinx.serialization.protobuf.ProtoBuf import logcat.LogPriority +import logcat.logcat import okhttp3.Headers -import okhttp3.MediaType.Companion.toMediaTypeOrNull +import okhttp3.MediaType.Companion.toMediaType import okhttp3.OkHttpClient -import okhttp3.RequestBody.Companion.gzip import okhttp3.RequestBody.Companion.toRequestBody -import tachiyomi.core.common.util.system.logcat +import org.apache.http.HttpStatus +import tachiyomi.core.common.i18n.stringResource +import tachiyomi.i18n.MR +import uy.kohesive.injekt.Injekt +import uy.kohesive.injekt.api.get import java.util.concurrent.TimeUnit class SyncYomiSyncService( @@ -26,140 +29,117 @@ class SyncYomiSyncService( json: Json, syncPreferences: SyncPreferences, private val notifier: SyncNotifier, -) : SyncService(context, json, syncPreferences) { - - @Serializable - enum class SyncStatus { - @SerialName("pending") - Pending, - @SerialName("syncing") - Syncing, - - @SerialName("success") - Success, - } - - @Serializable - data class LockFile( - @SerialName("id") - val id: Int?, - @SerialName("user_api_key") - val userApiKey: String?, - @SerialName("acquired_by") - val acquiredBy: String?, - @SerialName("last_synced") - val lastSynced: String?, - @SerialName("status") - val status: SyncStatus, - @SerialName("acquired_at") - val acquiredAt: String?, - @SerialName("expires_at") - val expiresAt: String?, - ) - - @Serializable - data class LockfileCreateRequest( - @SerialName("acquired_by") - val acquiredBy: String, - ) - - @Serializable - data class LockfilePatchRequest( - @SerialName("user_api_key") - val userApiKey: String, - @SerialName("acquired_by") - val acquiredBy: String, - ) - - override suspend fun beforeSync() { - val host = syncPreferences.clientHost().get() - val apiKey = syncPreferences.clientAPIKey().get() - val lockFileApi = "$host/api/sync/lock" - val deviceId = syncPreferences.uniqueDeviceID() - val client = OkHttpClient() - val headers = Headers.Builder().add("X-API-Token", apiKey).build() - val json = Json { ignoreUnknownKeys = true } - - val createLockfileRequest = LockfileCreateRequest(deviceId) - val createLockfileJson = json.encodeToString(createLockfileRequest) - - val patchRequest = LockfilePatchRequest(apiKey, deviceId) - val patchJson = json.encodeToString(patchRequest) - - val lockFileRequest = GET( - url = lockFileApi, - headers = headers, - ) - - val lockFileCreate = POST( - url = lockFileApi, - headers = headers, - body = createLockfileJson.toRequestBody("application/json; charset=utf-8".toMediaTypeOrNull()), - ) - - val lockFileUpdate = PATCH( - url = lockFileApi, - headers = headers, - body = patchJson.toRequestBody("application/json; charset=utf-8".toMediaTypeOrNull()), - ) + private val protoBuf: ProtoBuf = Injekt.get(), +) : SyncService(context, json, syncPreferences) { - // create lock file first - client.newCall(lockFileCreate).await() - // update lock file acquired_by - client.newCall(lockFileUpdate).await() + private class SyncYomiException(message: String?) : Exception(message) - var backoff = 2000L // Start with 2 seconds - val maxBackoff = 32000L // Maximum backoff time e.g., 32 seconds - var lockFile: LockFile - do { - val response = client.newCall(lockFileRequest).await() - val responseBody = response.body.string() - lockFile = json.decodeFromString(responseBody) - logcat(LogPriority.DEBUG) { "SyncYomi lock file status: ${lockFile.status}" } + override suspend fun doSync(syncData: SyncData): Backup? { + try { + val (remoteData, etag) = pullSyncData() - if (lockFile.status != SyncStatus.Success) { - logcat(LogPriority.DEBUG) { "Lock file not ready, retrying in $backoff ms..." } - delay(backoff) - backoff = (backoff * 2).coerceAtMost(maxBackoff) + val finalSyncData = if (remoteData != null){ + assert(etag.isNotEmpty()) { "ETag should never be empty if remote data is not null" } + logcat(LogPriority.DEBUG, "SyncService") { + "Try update remote data with ETag($etag)" + } + mergeSyncData(syncData, remoteData) + } else { + // init or overwrite remote data + logcat(LogPriority.DEBUG) { + "Try overwrite remote data with ETag($etag)" + } + syncData } - } while (lockFile.status != SyncStatus.Success) - // update lock file acquired_by - client.newCall(lockFileUpdate).await() + pushSyncData(finalSyncData, etag) + return finalSyncData.backup + + } catch (e: Exception) { + logcat(LogPriority.ERROR) { "Error syncing: ${e.message}" } + notifier.showSyncError(e.message) + return null + } } - override suspend fun pullSyncData(): SyncData? { + private suspend fun pullSyncData(): Pair { val host = syncPreferences.clientHost().get() val apiKey = syncPreferences.clientAPIKey().get() - val downloadUrl = "$host/api/sync/download" + val downloadUrl = "$host/api/sync/content" - val client = OkHttpClient() - val headers = Headers.Builder().add("X-API-Token", apiKey).build() + val headersBuilder = Headers.Builder().add("X-API-Token", apiKey) + val lastETag = syncPreferences.lastSyncEtag().get() + if (lastETag != "") { + headersBuilder.add("If-None-Match", lastETag) + } + val headers = headersBuilder.build() val downloadRequest = GET( url = downloadUrl, headers = headers, ) + val client = OkHttpClient() val response = client.newCall(downloadRequest).await() - val responseBody = response.body.string() - return if (response.isSuccessful) { - json.decodeFromString(responseBody) + if (response.code == HttpStatus.SC_NOT_MODIFIED) { + // not modified + assert(lastETag.isNotEmpty()) + logcat(LogPriority.INFO) { + "Remote server not modified" + } + return Pair(null, lastETag) + } else if (response.code == HttpStatus.SC_NOT_FOUND) { + // maybe got deleted from remote + return Pair(null, "") + } + + if (response.isSuccessful) { + val newETag = response.headers["ETag"] + .takeIf { it?.isNotEmpty() == true } ?: throw SyncYomiException("Missing ETag") + + val byteArray = response.body.byteStream().use { + return@use it.readBytes() + } + + return try { + val backup = protoBuf.decodeFromByteArray(BackupSerializer, byteArray) + return Pair(SyncData(backup = backup), newETag) + } catch (_: SerializationException) { + logcat(LogPriority.INFO) { + "Bad content responsed from server" + } + // the body is invalid + // return default value so we can overwrite it + Pair(null, "") + } + } else { + val responseBody = response.body.string() notifier.showSyncError("Failed to download sync data: $responseBody") - responseBody.let { logcat(LogPriority.ERROR) { "SyncError:$it" } } - null + logcat(LogPriority.ERROR) { "SyncError: $responseBody" } + throw SyncYomiException("Failed to download sync data: $responseBody") } } - override suspend fun pushSyncData(syncData: SyncData) { + /** + * Return true if update success + */ + private suspend fun pushSyncData(syncData: SyncData, eTag: String) { + val backup = syncData.backup ?: return + val host = syncPreferences.clientHost().get() val apiKey = syncPreferences.clientAPIKey().get() - val uploadUrl = "$host/api/sync/upload" + val uploadUrl = "$host/api/sync/content" val timeout = 30L + val headersBuilder = Headers.Builder().add("X-API-Token", apiKey) + if (eTag.isNotEmpty()) { + headersBuilder.add("If-Match", eTag) + } + val headers = headersBuilder.build() + // Set timeout to 30 seconds val client = OkHttpClient.Builder() .connectTimeout(timeout, TimeUnit.SECONDS) @@ -167,32 +147,34 @@ class SyncYomiSyncService( .writeTimeout(timeout, TimeUnit.SECONDS) .build() - val headers = Headers.Builder().add( - "Content-Type", - "application/gzip", - ).add("Content-Encoding", "gzip").add("X-API-Token", apiKey).build() - - val mediaType = "application/gzip".toMediaTypeOrNull() - - val jsonData = json.encodeToString(syncData) - val body = jsonData.toRequestBody(mediaType).gzip() + val byteArray = protoBuf.encodeToByteArray(BackupSerializer, backup) + if (byteArray.isEmpty()) { + throw IllegalStateException(context.stringResource(MR.strings.empty_backup_error)) + } + val body = byteArray.toRequestBody("application/octet-stream".toMediaType()) - val uploadRequest = POST( + val uploadRequest = PUT( url = uploadUrl, headers = headers, body = body, ) - client.newCall(uploadRequest).await().use { - if (it.isSuccessful) { - logcat( - LogPriority.DEBUG, - ) { "SyncYomi sync completed!" } - } else { - val responseBody = it.body.string() - notifier.showSyncError("Failed to upload sync data: $responseBody") - responseBody.let { logcat(LogPriority.ERROR) { "SyncError:$it" } } - } + val response = client.newCall(uploadRequest).await() + + if (response.isSuccessful) { + val newETag = response.headers["ETag"] + .takeIf { it?.isNotEmpty() == true } ?: throw SyncYomiException("Missing ETag") + syncPreferences.lastSyncEtag().set(newETag) + logcat(LogPriority.DEBUG) { "SyncYomi sync completed" } + + } else if (response.code == HttpStatus.SC_PRECONDITION_FAILED) { + // other clients updated remote data, will try next time + logcat(LogPriority.DEBUG) { "SyncYomi sync failed with 412" } + + } else { + val responseBody = response.body.string() + notifier.showSyncError("Failed to upload sync data: $responseBody") + logcat(LogPriority.ERROR) { "SyncError: $responseBody" } } } } diff --git a/core/common/src/main/kotlin/eu/kanade/tachiyomi/network/Requests.kt b/core/common/src/main/kotlin/eu/kanade/tachiyomi/network/Requests.kt index 75886950af..6adb0de8ef 100644 --- a/core/common/src/main/kotlin/eu/kanade/tachiyomi/network/Requests.kt +++ b/core/common/src/main/kotlin/eu/kanade/tachiyomi/network/Requests.kt @@ -63,19 +63,6 @@ fun PUT( .cacheControl(cache) .build() } -fun PATCH( - url: String, - headers: Headers = DEFAULT_HEADERS, - body: RequestBody = DEFAULT_BODY, - cache: CacheControl = DEFAULT_CACHE_CONTROL, -): Request { - return Request.Builder() - .url(url) - .patch(body) - .headers(headers) - .cacheControl(cache) - .build() -} fun DELETE( url: String,