From 937549f53158e2f42375fd5ed6c9b68b5af105b0 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 7 May 2024 17:23:10 +0800 Subject: [PATCH] Save Android offline content to files instead of in DB This works around max size constraints in the Room DB, we will need to add something to clean up old files though. --- .../omnivore/core/data/LibrarySync.kt | 11 +++- .../repository/impl/LibraryRepositoryImpl.kt | 11 +++- .../core/database/OmnivoreDatabase.kt | 2 +- .../core/database/entities/SavedItem.kt | 2 - .../omnivore/core/network/SavedItemQuery.kt | 3 +- .../omnivore/core/network/SearchQuery.kt | 37 +++++++++++- .../feature/reader/PDFReaderViewModel.kt | 4 +- .../feature/reader/WebReaderContent.kt | 1 - .../feature/reader/WebReaderViewModel.kt | 60 ++++++++++--------- 9 files changed, 87 insertions(+), 44 deletions(-) diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/data/LibrarySync.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/data/LibrarySync.kt index 3843f3b2e..07a1f0843 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/data/LibrarySync.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/data/LibrarySync.kt @@ -6,6 +6,8 @@ import app.omnivore.omnivore.core.database.entities.Highlight import app.omnivore.omnivore.core.database.entities.SavedItem import app.omnivore.omnivore.core.database.entities.SavedItemLabel import app.omnivore.omnivore.core.database.entities.SavedItemWithLabelsAndHighlights +import app.omnivore.omnivore.core.network.loadLibraryItemContent +import app.omnivore.omnivore.core.network.saveLibraryItemContentToFile import app.omnivore.omnivore.core.network.savedItem import app.omnivore.omnivore.core.network.savedItemUpdates import app.omnivore.omnivore.core.network.search @@ -46,6 +48,7 @@ suspend fun DataService.sync(since: String, cursor: String?, limit: Int = 20): S } val savedItems = syncResult.items.map { + saveLibraryItemContentToFile(it.id, it.content) val savedItem = SavedItem( savedItemId = it.id, title = it.title, @@ -67,7 +70,6 @@ suspend fun DataService.sync(since: String, cursor: String?, limit: Int = 20): S isArchived = it.isArchived, contentReader = it.contentReader.rawValue, wordsCount = it.wordsCount, - content = it.content ) val labels = it.labels?.map { label -> SavedItemLabel( @@ -116,8 +118,11 @@ suspend fun DataService.sync(since: String, cursor: String?, limit: Int = 20): S suspend fun DataService.isSavedItemContentStoredInDB(slug: String): Boolean { val existingItem = db.savedItemDao().getSavedItemWithLabelsAndHighlights(slug) - val content = existingItem?.savedItem?.content ?: "" - return content.length > 10 + existingItem?.savedItem?.savedItemId?.let { savedItemId -> + val htmlContent = loadLibraryItemContent(savedItemId) + return (htmlContent ?: "").length > 10 + } + return false } suspend fun DataService.fetchSavedItemContent(slug: String) { diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/data/repository/impl/LibraryRepositoryImpl.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/data/repository/impl/LibraryRepositoryImpl.kt index d7b55cf76..6d3f709bd 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/data/repository/impl/LibraryRepositoryImpl.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/data/repository/impl/LibraryRepositoryImpl.kt @@ -27,7 +27,9 @@ import app.omnivore.omnivore.core.network.createHighlight import app.omnivore.omnivore.core.network.createNewLabel import app.omnivore.omnivore.core.network.deleteHighlights import app.omnivore.omnivore.core.network.deleteSavedItem +import app.omnivore.omnivore.core.network.loadLibraryItemContent import app.omnivore.omnivore.core.network.mergeHighlights +import app.omnivore.omnivore.core.network.saveLibraryItemContentToFile import app.omnivore.omnivore.core.network.savedItem import app.omnivore.omnivore.core.network.savedItemLabels import app.omnivore.omnivore.core.network.savedItemUpdates @@ -219,8 +221,11 @@ class LibraryRepositoryImpl @Inject constructor( override suspend fun isSavedItemContentStoredInDB(slug: String): Boolean { val existingItem = savedItemDao.getSavedItemWithLabelsAndHighlights(slug) - val content = existingItem?.savedItem?.content ?: "" - return content.length > 10 + existingItem?.savedItem?.savedItemId?.let { savedItemId -> + val htmlContent = loadLibraryItemContent(savedItemId) + return (htmlContent ?: "").length > 10 + } + return false } override suspend fun deleteSavedItem(itemID: String) { @@ -416,6 +421,7 @@ class LibraryRepositoryImpl @Inject constructor( } val savedItems = syncResult.items.map { + saveLibraryItemContentToFile(it.id, it.content) val savedItem = SavedItem( savedItemId = it.id, title = it.title, @@ -436,7 +442,6 @@ class LibraryRepositoryImpl @Inject constructor( slug = it.slug, isArchived = it.isArchived, contentReader = it.contentReader.rawValue, - content = it.content, wordsCount = it.wordsCount ) val labels = it.labels?.map { label -> diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/database/OmnivoreDatabase.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/database/OmnivoreDatabase.kt index 93ccfbbd1..8a82b336a 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/database/OmnivoreDatabase.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/database/OmnivoreDatabase.kt @@ -27,7 +27,7 @@ import app.omnivore.omnivore.core.database.entities.ViewerDao HighlightChange::class, SavedItemAndSavedItemLabelCrossRef::class, SavedItemAndHighlightCrossRef::class], - version = 27, + version = 28, exportSchema = true ) abstract class OmnivoreDatabase : RoomDatabase() { diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/database/entities/SavedItem.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/database/entities/SavedItem.kt index fd2796f0a..d35376fe7 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/database/entities/SavedItem.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/database/entities/SavedItem.kt @@ -26,9 +26,7 @@ data class SavedItem( val slug: String, var isArchived: Boolean, val contentReader: String? = null, - val content: String? = null, val createdId: String? = null, - val htmlContent: String? = null, val language: String? = null, val listenPositionIndex: Int? = null, val listenPositionOffset: Double? = null, diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/network/SavedItemQuery.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/network/SavedItemQuery.kt index f463890ca..b54fa2b6a 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/network/SavedItemQuery.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/network/SavedItemQuery.kt @@ -80,6 +80,8 @@ suspend fun Networker.savedItem(slug: String): SavedItemQueryResponse { localPDFPath = localFile.toPath().toString() } + saveLibraryItemContentToFile(article.articleFields.id, article.articleFields.content) + val savedItem = SavedItem( savedItemId = article.articleFields.id, title = article.articleFields.title, @@ -100,7 +102,6 @@ suspend fun Networker.savedItem(slug: String): SavedItemQueryResponse { slug = article.articleFields.slug, isArchived = article.articleFields.isArchived, contentReader = article.articleFields.contentReader.rawValue, - content = article.articleFields.content, wordsCount = article.articleFields.wordsCount, localPDFPath = localPDFPath ) diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/network/SearchQuery.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/network/SearchQuery.kt index 93307ddd0..2943e85cf 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/network/SearchQuery.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/network/SearchQuery.kt @@ -1,11 +1,16 @@ package app.omnivore.omnivore.core.network +import android.os.Environment import app.omnivore.omnivore.core.data.model.ServerSyncStatus import app.omnivore.omnivore.core.database.entities.Highlight import app.omnivore.omnivore.core.database.entities.SavedItem import app.omnivore.omnivore.core.database.entities.SavedItemLabel import app.omnivore.omnivore.graphql.generated.SearchQuery import com.apollographql.apollo3.api.Optional +import androidx.core.content.ContextCompat +import java.io.File +import java.io.FileInputStream +import java.io.FileOutputStream data class LibrarySearchQueryResponse( val cursor: String?, val items: List @@ -31,6 +36,7 @@ suspend fun Networker.search( val itemList = result.data?.search?.onSearchSuccess?.edges ?: listOf() val searchItems = itemList.map { + saveLibraryItemContentToFile(it.node.id, it.node.content) LibrarySearchItem(item = SavedItem( savedItemId = it.node.id, title = it.node.title, @@ -51,7 +57,6 @@ suspend fun Networker.search( slug = it.node.slug, isArchived = it.node.isArchived, contentReader = it.node.contentReader.rawValue, - content = it.node.content, wordsCount = it.node.wordsCount ), labels = (it.node.labels ?: listOf()).map { label -> SavedItemLabel( @@ -89,3 +94,33 @@ suspend fun Networker.search( return LibrarySearchQueryResponse(null, listOf()) } } + +fun saveLibraryItemContentToFile(libraryItemId: String, content: String?): Boolean { + return try { + content?.let { content -> + val directory = Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOCUMENTS) + val file = File(directory, "${libraryItemId}.html") + FileOutputStream(file).use { it.write(content.toByteArray()) } + return false + } + false + } catch (e: Exception) { + e.printStackTrace() + false + } +} + +fun loadLibraryItemContent(libraryItemId: String): String? { + return try { + val directory = Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOCUMENTS) + val file = File(directory, "${libraryItemId}.html") + if (file.exists()) { + return FileInputStream(file).bufferedReader().use { it.readText() } + } else { + null + } + } catch (e: Exception) { + e.printStackTrace() + null + } +} diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/feature/reader/PDFReaderViewModel.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/feature/reader/PDFReaderViewModel.kt index 90f0b1839..1d82f0aaf 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/feature/reader/PDFReaderViewModel.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/feature/reader/PDFReaderViewModel.kt @@ -73,7 +73,6 @@ class PDFReaderViewModel @Inject constructor( htmlContent = "", highlights = item.highlights, contentStatus = "SUCCEEDED", - objectID = "", labelsJSONString = Gson().toJson(item.labels) ) @@ -103,10 +102,9 @@ class PDFReaderViewModel @Inject constructor( override fun onComplete(output: File) { val articleContent = ArticleContent( title = article.title, - htmlContent = article.content ?: "", + htmlContent = "", highlights = articleQueryResult.highlights, contentStatus = "SUCCEEDED", - objectID = "", labelsJSONString = Gson().toJson(articleQueryResult.labels) ) diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/feature/reader/WebReaderContent.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/feature/reader/WebReaderContent.kt index 2616303ff..7c0fed158 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/feature/reader/WebReaderContent.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/feature/reader/WebReaderContent.kt @@ -37,7 +37,6 @@ data class ArticleContent( val htmlContent: String, val highlights: List, val contentStatus: String, // ArticleContentStatus, - val objectID: String?, // whatever the Room Equivalent of objectID is val labelsJSONString: String ) { fun highlightsJSONString(): String { diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/feature/reader/WebReaderViewModel.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/feature/reader/WebReaderViewModel.kt index ac70346d2..477def25d 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/feature/reader/WebReaderViewModel.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/feature/reader/WebReaderViewModel.kt @@ -39,6 +39,7 @@ import app.omnivore.omnivore.core.datastore.prefersWebHighContrastText import app.omnivore.omnivore.core.datastore.volumeForScroll import app.omnivore.omnivore.core.network.Networker import app.omnivore.omnivore.core.network.createNewLabel +import app.omnivore.omnivore.core.network.loadLibraryItemContent import app.omnivore.omnivore.core.network.saveUrl import app.omnivore.omnivore.core.network.savedItem import app.omnivore.omnivore.feature.components.HighlightColor @@ -263,35 +264,36 @@ class WebReaderViewModel @Inject constructor( private suspend fun loadItemFromDB(slug: String) { withContext(Dispatchers.IO) { - val persistedItem = - dataService.db.savedItemDao().getSavedItemWithLabelsAndHighlights(slug) + val persistedItem = dataService.db.savedItemDao().getSavedItemWithLabelsAndHighlights(slug) + val savedItemId = persistedItem?.savedItem?.savedItemId + if (savedItemId != null) { + val htmlContent = loadLibraryItemContent(savedItemId) + if (htmlContent != null) { + val articleContent = ArticleContent( + title = persistedItem.savedItem.title, + htmlContent = htmlContent, + highlights = persistedItem.highlights, + contentStatus = "SUCCEEDED", + labelsJSONString = Gson().toJson(persistedItem.labels) + ) - if (persistedItem?.savedItem?.content != null) { - val articleContent = ArticleContent( - title = persistedItem.savedItem.title, - htmlContent = persistedItem.savedItem.content, - highlights = persistedItem.highlights, - contentStatus = "SUCCEEDED", - objectID = "", - labelsJSONString = Gson().toJson(persistedItem.labels) - ) + val webReaderParams = WebReaderParams( + persistedItem.savedItem, + articleContent, + persistedItem.labels + ) - val webReaderParams = WebReaderParams( - persistedItem.savedItem, - articleContent, - persistedItem.labels - ) - - Log.d("sync", "data loaded from db") - eventTracker.track( - "link_read", - com.posthog.android.Properties() - .putValue("linkID", webReaderParams.item.savedItemId) - .putValue("slug", webReaderParams.item.slug) - .putValue("originalArticleURL", webReaderParams.item.pageURLString) - .putValue("loaded_from", "db") - ) - webReaderParamsLiveData.postValue(webReaderParams) + Log.d("sync", "data loaded from db") + eventTracker.track( + "link_read", + com.posthog.android.Properties() + .putValue("linkID", webReaderParams.item.savedItemId) + .putValue("slug", webReaderParams.item.slug) + .putValue("originalArticleURL", webReaderParams.item.pageURLString) + .putValue("loaded_from", "db") + ) + webReaderParamsLiveData.postValue(webReaderParams) + } } isLoading = false } @@ -301,13 +303,13 @@ class WebReaderViewModel @Inject constructor( val articleQueryResult = networker.savedItem(slug) val article = articleQueryResult.item ?: return null + val htmlContent = loadLibraryItemContent(article.savedItemId) val articleContent = ArticleContent( title = article.title, - htmlContent = article.content ?: "", + htmlContent = htmlContent ?: "", highlights = articleQueryResult.highlights, contentStatus = articleQueryResult.state, - objectID = "", labelsJSONString = Gson().toJson(articleQueryResult.labels) )