Benature/omnivore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Save Android offline content to files instead of in DB

Browse Source 
This works around max size constraints in the Room DB, we
will need to add something to clean up old files though.
This commit is contained in:
Jackson Harper
2024-05-07 17:23:10 +08:00
parent 137e3b2e94
commit 937549f531
9 changed files with 87 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/data/LibrarySync.kt
View File

@ -6,6 +6,8 @@ import app.omnivore.omnivore.core.database.entities.Highlight
import app.omnivore.omnivore.core.database.entities.SavedItem
import app.omnivore.omnivore.core.database.entities.SavedItemLabel
import app.omnivore.omnivore.core.database.entities.SavedItemWithLabelsAndHighlights
import app.omnivore.omnivore.core.network.loadLibraryItemContent
import app.omnivore.omnivore.core.network.saveLibraryItemContentToFile
import app.omnivore.omnivore.core.network.savedItem
import app.omnivore.omnivore.core.network.savedItemUpdates
import app.omnivore.omnivore.core.network.search
@ -46,6 +48,7 @@ suspend fun DataService.sync(since: String, cursor: String?, limit: Int = 20): S
}
val savedItems = syncResult.items.map {
saveLibraryItemContentToFile(it.id, it.content)
val savedItem = SavedItem(
savedItemId = it.id,
title = it.title,
@ -67,7 +70,6 @@ suspend fun DataService.sync(since: String, cursor: String?, limit: Int = 20): S
isArchived = it.isArchived,
contentReader = it.contentReader.rawValue,
wordsCount = it.wordsCount,
content = it.content
)
val labels = it.labels?.map { label ->
SavedItemLabel(
@ -116,8 +118,11 @@ suspend fun DataService.sync(since: String, cursor: String?, limit: Int = 20): S
suspend fun DataService.isSavedItemContentStoredInDB(slug: String): Boolean {
val existingItem = db.savedItemDao().getSavedItemWithLabelsAndHighlights(slug)
val content = existingItem?.savedItem?.content ?: ""
return content.length > 10
existingItem?.savedItem?.savedItemId?.let { savedItemId ->
val htmlContent = loadLibraryItemContent(savedItemId)
return (htmlContent ?: "").length > 10
}
return false
}
suspend fun DataService.fetchSavedItemContent(slug: String) {

11
android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/data/repository/impl/LibraryRepositoryImpl.kt
View File

@ -27,7 +27,9 @@ import app.omnivore.omnivore.core.network.createHighlight
import app.omnivore.omnivore.core.network.createNewLabel
import app.omnivore.omnivore.core.network.deleteHighlights
import app.omnivore.omnivore.core.network.deleteSavedItem
import app.omnivore.omnivore.core.network.loadLibraryItemContent
import app.omnivore.omnivore.core.network.mergeHighlights
import app.omnivore.omnivore.core.network.saveLibraryItemContentToFile
import app.omnivore.omnivore.core.network.savedItem
import app.omnivore.omnivore.core.network.savedItemLabels
import app.omnivore.omnivore.core.network.savedItemUpdates
@ -219,8 +221,11 @@ class LibraryRepositoryImpl @Inject constructor(
override suspend fun isSavedItemContentStoredInDB(slug: String): Boolean {
val existingItem = savedItemDao.getSavedItemWithLabelsAndHighlights(slug)
val content = existingItem?.savedItem?.content ?: ""
return content.length > 10
existingItem?.savedItem?.savedItemId?.let { savedItemId ->
val htmlContent = loadLibraryItemContent(savedItemId)
return (htmlContent ?: "").length > 10
}
return false
}
override suspend fun deleteSavedItem(itemID: String) {
@ -416,6 +421,7 @@ class LibraryRepositoryImpl @Inject constructor(
}
val savedItems = syncResult.items.map {
saveLibraryItemContentToFile(it.id, it.content)
val savedItem = SavedItem(
savedItemId = it.id,
title = it.title,
@ -436,7 +442,6 @@ class LibraryRepositoryImpl @Inject constructor(
slug = it.slug,
isArchived = it.isArchived,
contentReader = it.contentReader.rawValue,
content = it.content,
wordsCount = it.wordsCount
)
val labels = it.labels?.map { label ->

2
android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/database/OmnivoreDatabase.kt
View File

@ -27,7 +27,7 @@ import app.omnivore.omnivore.core.database.entities.ViewerDao
HighlightChange::class,
SavedItemAndSavedItemLabelCrossRef::class,
SavedItemAndHighlightCrossRef::class],
version = 27,
version = 28,
exportSchema = true
)
abstract class OmnivoreDatabase : RoomDatabase() {

2
android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/database/entities/SavedItem.kt
View File

@ -26,9 +26,7 @@ data class SavedItem(
val slug: String,
var isArchived: Boolean,
val contentReader: String? = null,
val content: String? = null,
val createdId: String? = null,
val htmlContent: String? = null,
val language: String? = null,
val listenPositionIndex: Int? = null,
val listenPositionOffset: Double? = null,

3
android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/network/SavedItemQuery.kt
View File

@ -80,6 +80,8 @@ suspend fun Networker.savedItem(slug: String): SavedItemQueryResponse {
localPDFPath = localFile.toPath().toString()
}
saveLibraryItemContentToFile(article.articleFields.id, article.articleFields.content)
val savedItem = SavedItem(
savedItemId = article.articleFields.id,
title = article.articleFields.title,
@ -100,7 +102,6 @@ suspend fun Networker.savedItem(slug: String): SavedItemQueryResponse {
slug = article.articleFields.slug,
isArchived = article.articleFields.isArchived,
contentReader = article.articleFields.contentReader.rawValue,
content = article.articleFields.content,
wordsCount = article.articleFields.wordsCount,
localPDFPath = localPDFPath
)

37
android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/network/SearchQuery.kt
View File

@ -1,11 +1,16 @@
package app.omnivore.omnivore.core.network
import android.os.Environment
import app.omnivore.omnivore.core.data.model.ServerSyncStatus
import app.omnivore.omnivore.core.database.entities.Highlight
import app.omnivore.omnivore.core.database.entities.SavedItem
import app.omnivore.omnivore.core.database.entities.SavedItemLabel
import app.omnivore.omnivore.graphql.generated.SearchQuery
import com.apollographql.apollo3.api.Optional
import androidx.core.content.ContextCompat
import java.io.File
import java.io.FileInputStream
import java.io.FileOutputStream
data class LibrarySearchQueryResponse(
val cursor: String?, val items: List<LibrarySearchItem>
@ -31,6 +36,7 @@ suspend fun Networker.search(
val itemList = result.data?.search?.onSearchSuccess?.edges ?: listOf()
val searchItems = itemList.map {
saveLibraryItemContentToFile(it.node.id, it.node.content)
LibrarySearchItem(item = SavedItem(
savedItemId = it.node.id,
title = it.node.title,
@ -51,7 +57,6 @@ suspend fun Networker.search(
slug = it.node.slug,
isArchived = it.node.isArchived,
contentReader = it.node.contentReader.rawValue,
content = it.node.content,
wordsCount = it.node.wordsCount
), labels = (it.node.labels ?: listOf()).map { label ->
SavedItemLabel(
@ -89,3 +94,33 @@ suspend fun Networker.search(
return LibrarySearchQueryResponse(null, listOf())
}
}
fun saveLibraryItemContentToFile(libraryItemId: String, content: String?): Boolean {
return try {
content?.let { content ->
val directory = Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOCUMENTS)
val file = File(directory, "${libraryItemId}.html")
FileOutputStream(file).use { it.write(content.toByteArray()) }
return false
}
false
} catch (e: Exception) {
e.printStackTrace()
false
}
}
fun loadLibraryItemContent(libraryItemId: String): String? {
return try {
val directory = Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOCUMENTS)
val file = File(directory, "${libraryItemId}.html")
if (file.exists()) {
return FileInputStream(file).bufferedReader().use { it.readText() }
} else {
null
}
} catch (e: Exception) {
e.printStackTrace()
null
}
}

4
android/Omnivore/app/src/main/java/app/omnivore/omnivore/feature/reader/PDFReaderViewModel.kt
View File

@ -73,7 +73,6 @@ class PDFReaderViewModel @Inject constructor(
htmlContent = "",
highlights = item.highlights,
contentStatus = "SUCCEEDED",
objectID = "",
labelsJSONString = Gson().toJson(item.labels)
)
@ -103,10 +102,9 @@ class PDFReaderViewModel @Inject constructor(
override fun onComplete(output: File) {
val articleContent = ArticleContent(
title = article.title,
htmlContent = article.content ?: "",
htmlContent = "",
highlights = articleQueryResult.highlights,
contentStatus = "SUCCEEDED",
objectID = "",
labelsJSONString = Gson().toJson(articleQueryResult.labels)
)

1
android/Omnivore/app/src/main/java/app/omnivore/omnivore/feature/reader/WebReaderContent.kt
View File

@ -37,7 +37,6 @@ data class ArticleContent(
val htmlContent: String,
val highlights: List<Highlight>,
val contentStatus: String, // ArticleContentStatus,
val objectID: String?, // whatever the Room Equivalent of objectID is
val labelsJSONString: String
) {
fun highlightsJSONString(): String {

60
android/Omnivore/app/src/main/java/app/omnivore/omnivore/feature/reader/WebReaderViewModel.kt
View File

@ -39,6 +39,7 @@ import app.omnivore.omnivore.core.datastore.prefersWebHighContrastText
import app.omnivore.omnivore.core.datastore.volumeForScroll
import app.omnivore.omnivore.core.network.Networker
import app.omnivore.omnivore.core.network.createNewLabel
import app.omnivore.omnivore.core.network.loadLibraryItemContent
import app.omnivore.omnivore.core.network.saveUrl
import app.omnivore.omnivore.core.network.savedItem
import app.omnivore.omnivore.feature.components.HighlightColor
@ -263,35 +264,36 @@ class WebReaderViewModel @Inject constructor(
private suspend fun loadItemFromDB(slug: String) {
withContext(Dispatchers.IO) {
val persistedItem =
dataService.db.savedItemDao().getSavedItemWithLabelsAndHighlights(slug)
val persistedItem = dataService.db.savedItemDao().getSavedItemWithLabelsAndHighlights(slug)
val savedItemId = persistedItem?.savedItem?.savedItemId
if (savedItemId != null) {
val htmlContent = loadLibraryItemContent(savedItemId)
if (htmlContent != null) {
val articleContent = ArticleContent(
title = persistedItem.savedItem.title,
htmlContent = htmlContent,
highlights = persistedItem.highlights,
contentStatus = "SUCCEEDED",
labelsJSONString = Gson().toJson(persistedItem.labels)
)
if (persistedItem?.savedItem?.content != null) {
val articleContent = ArticleContent(
title = persistedItem.savedItem.title,
htmlContent = persistedItem.savedItem.content,
highlights = persistedItem.highlights,
contentStatus = "SUCCEEDED",
objectID = "",
labelsJSONString = Gson().toJson(persistedItem.labels)
)
val webReaderParams = WebReaderParams(
persistedItem.savedItem,
articleContent,
persistedItem.labels
)
val webReaderParams = WebReaderParams(
persistedItem.savedItem,
articleContent,
persistedItem.labels
)
Log.d("sync", "data loaded from db")
eventTracker.track(
"link_read",
com.posthog.android.Properties()
.putValue("linkID", webReaderParams.item.savedItemId)
.putValue("slug", webReaderParams.item.slug)
.putValue("originalArticleURL", webReaderParams.item.pageURLString)
.putValue("loaded_from", "db")
)
webReaderParamsLiveData.postValue(webReaderParams)
Log.d("sync", "data loaded from db")
eventTracker.track(
"link_read",
com.posthog.android.Properties()
.putValue("linkID", webReaderParams.item.savedItemId)
.putValue("slug", webReaderParams.item.slug)
.putValue("originalArticleURL", webReaderParams.item.pageURLString)
.putValue("loaded_from", "db")
)
webReaderParamsLiveData.postValue(webReaderParams)
}
}
isLoading = false
}
@ -301,13 +303,13 @@ class WebReaderViewModel @Inject constructor(
val articleQueryResult = networker.savedItem(slug)
val article = articleQueryResult.item ?: return null
val htmlContent = loadLibraryItemContent(article.savedItemId)
val articleContent = ArticleContent(
title = article.title,
htmlContent = article.content ?: "",
htmlContent = htmlContent ?: "",
highlights = articleQueryResult.highlights,
contentStatus = articleQueryResult.state,
objectID = "",
labelsJSONString = Gson().toJson(articleQueryResult.labels)
)