Benature/omnivore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Sync more items in the background if user has a large changeset

Browse Source
This commit is contained in:
Jackson Harper
2022-12-27 20:15:54 +08:00
parent e23cb722fc
commit 10abb6582f
10 changed files with 118 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
apple/OmnivoreKit/Sources/App/Views/Home/Components/FeedCardNavigationLink.swift
View File

@ -26,7 +26,7 @@ struct MacFeedCardNavigationLink: View {
.opacity(0)
.buttonStyle(PlainButtonStyle())
.onAppear {
Task { await viewModel.itemAppeared(item: item, dataService: dataService, audioController: audioController) }
Task { await viewModel.itemAppeared(item: item, dataService: dataService) }
}
FeedCard(item: item, viewer: dataService.currentViewer) {
viewModel.selectedLinkItem = item.objectID
@ -56,7 +56,7 @@ struct FeedCardNavigationLink: View {
.opacity(0)
.buttonStyle(PlainButtonStyle())
.onAppear {
Task { await viewModel.itemAppeared(item: item, dataService: dataService, audioController: audioController) }
Task { await viewModel.itemAppeared(item: item, dataService: dataService) }
}
FeedCard(item: item, viewer: dataService.currentViewer)
}
@ -95,7 +95,7 @@ struct GridCardNavigationLink: View {
withAnimation { tapAction() }
})
.onAppear {
Task { await viewModel.itemAppeared(item: item, dataService: dataService, audioController: audioController) }
Task { await viewModel.itemAppeared(item: item, dataService: dataService) }
}
}
.aspectRatio(1.8, contentMode: .fill)

4
apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift
View File

@ -20,7 +20,7 @@ import Views
@ObservedObject var viewModel: HomeFeedViewModel
func loadItems(isRefresh: Bool) {
Task { await viewModel.loadItems(dataService: dataService, audioController: audioController, isRefresh: isRefresh) }
Task { await viewModel.loadItems(dataService: dataService, isRefresh: isRefresh) }
}
var body: some View {
@ -448,7 +448,7 @@ import Views
}
func loadItems(isRefresh: Bool) {
Task { await viewModel.loadItems(dataService: dataService, audioController: audioController, isRefresh: isRefresh) }
Task { await viewModel.loadItems(dataService: dataService, isRefresh: isRefresh) }
}
var body: some View {

65
apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewModel.swift
View File

@ -35,6 +35,15 @@ import Views
@Published var showLabelsSheet = false
@Published var showCommunityModal = false
var cursor: String?
// These are used to make sure we handle search result
// responses in the right order
var searchIdx = 0
var receivedIdx = 0
var syncCursor: String?
@AppStorage(UserDefaultKey.lastSelectedLinkedItemFilter.rawValue) var appliedFilter = LinkedItemFilter.inbox.rawValue
func handleReaderItemNotification(objectID: NSManagedObjectID, dataService: DataService) {
@ -66,14 +75,7 @@ import Views
}
}
var cursor: String?
// These are used to make sure we handle search result
// responses in the right order
var searchIdx = 0
var receivedIdx = 0
func itemAppeared(item: LinkedItem, dataService: DataService, audioController: AudioController) async {
func itemAppeared(item: LinkedItem, dataService: DataService) async {
if isLoading { return }
let itemIndex = items.firstIndex(where: { $0.id == item.id })
let thresholdIndex = items.index(items.endIndex, offsetBy: -5)
@ -81,8 +83,8 @@ import Views
// Check if user has scrolled to the last five items in the list
// Make sure we aren't currently loading though, as this would get triggered when the first set
// of items are presented to the user.
if let itemIndex = itemIndex, itemIndex > thresholdIndex, items.count < thresholdIndex + 10, !isLoading {
await loadItems(dataService: dataService, audioController: audioController, isRefresh: false)
if let itemIndex = itemIndex, itemIndex > thresholdIndex {
await loadMoreItems(dataService: dataService, isRefresh: false)
}
}
@ -106,15 +108,23 @@ import Views
}
}
func syncItems(dataService: DataService, syncStartTime _: Date) async {
func syncItems(dataService: DataService) async {
let syncStart = Date.now
let lastSyncDate = dateFormatter.date(from: dataService.lastItemSyncTime) ?? Date(timeIntervalSinceReferenceDate: 0)
let syncResult = try? await dataService.syncLinkedItems(since: lastSyncDate,
cursor: nil,
deferFetchingMore: true)
if syncResult?.hasMore {
// dataService.lastItemSyncTime = dateFormatter.string(from: syncStartTime)
self.isLoading = true
try? await dataService.syncOfflineItemsWithServerIfNeeded()
let syncResult = try? await dataService.syncLinkedItems(since: lastSyncDate,
cursor: nil)
syncCursor = syncResult?.cursor
if let syncResult = syncResult, syncResult.hasMore {
dataService.syncLinkedItemsInBackground(since: lastSyncDate) {
// Set isLoading to false here
self.isLoading = false
}
} else {
dataService.lastItemSyncTime = DateFormatter.formatterISO8601.string(from: syncStart)
}
// If possible start prefetching new pages in the background
@ -173,25 +183,20 @@ import Views
}
}
func loadItems(dataService: DataService, audioController _: AudioController, isRefresh: Bool) async {
let syncStartTime = Date()
func loadItems(dataService: DataService, isRefresh: Bool) async {
isLoading = true
showLoadingBar = true
await withTaskGroup(of: Void.self) { group in
group.addTask { await self.loadCurrentViewer(dataService: dataService) }
group.addTask { await self.loadLabels(dataService: dataService) }
group.addTask { await self.syncItems(dataService: dataService, syncStartTime: syncStartTime) }
group.addTask { await self.syncItems(dataService: dataService) }
await group.waitForAll()
}
if searchTerm.replacingOccurrences(of: " ", with: "").isEmpty {
updateFetchController(dataService: dataService)
// For now we are forcing the search because we are fetching items in reverse
// with the sync API, but search fetches in descending order
if appliedFilter != LinkedItemFilter.inbox.rawValue {
if appliedFilter != LinkedItemFilter.inbox.rawValue || !isRefresh {
await loadSearchQuery(dataService: dataService, isRefresh: isRefresh)
}
} else {
@ -202,6 +207,16 @@ import Views
showLoadingBar = false
}
func loadMoreItems(dataService: DataService, isRefresh _: Bool) async {
isLoading = true
showLoadingBar = true
await loadSearchQuery(dataService: dataService, isRefresh: isRefresh)
isLoading = false
showLoadingBar = false
}
private var fetchRequest: NSFetchRequest<Models.LinkedItem> {
let fetchRequest: NSFetchRequest<Models.LinkedItem> = LinkedItem.fetchRequest()

6
apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift
View File

@ -16,11 +16,15 @@ public struct LinkedItemSyncResult {
public let updatedItemIDs: [String]
public let cursor: String?
public let hasMore: Bool
public let mostRecentUpdatedAt: Date?
public let isEmpty: Bool
public init(updatedItemIDs: [String], cursor: String?, hasMore: Bool) {
public init(updatedItemIDs: [String], cursor: String?, hasMore: Bool, mostRecentUpdatedAt: Date?, isEmpty: Bool) {
self.updatedItemIDs = updatedItemIDs
self.cursor = cursor
self.hasMore = hasMore
self.mostRecentUpdatedAt = mostRecentUpdatedAt
self.isEmpty = isEmpty
}
}

2
apple/OmnivoreKit/Sources/Services/DataService/OfflineSync.swift
View File

@ -4,7 +4,7 @@ import Models
import Utils
public extension DataService {
internal func syncOfflineItemsWithServerIfNeeded() async throws {
func syncOfflineItemsWithServerIfNeeded() async throws {
var unsyncedLinkedItems = [LinkedItem]()
var unsyncedHighlights = [Highlight]()

102
apple/OmnivoreKit/Sources/Services/DataService/Public/LinkedItemLoading.swift
View File

@ -6,24 +6,12 @@ public extension DataService {
/// Requests `LinkedItem`s updates from the server since a certain datae
/// and stores it in CoreData while deleting all the items with ids the server says
/// have been deleted.
/// - Parameters:
/// - limit: max count of items
/// - searchQuery: search terms and filters
/// - cursor: cursor when loading batch for infinite list
/// - Returns: `LinkedItemQueryResult` (managed object IDs and an optional cursor)
func syncLinkedItems(
since date: Date,
since: Date,
cursor: String?,
previousQueryResult: LinkedItemSyncResult? = nil,
deferFetchingMore: Bool
) async throws -> LinkedItemSyncResult? {
if previousQueryResult == nil {
// Send offline changes to server before fetching items
// only on the first call of this function
try? await syncOfflineItemsWithServerIfNeeded()
}
let fetchResult = try await linkedItemUpdates(since: date, limit: 20, cursor: cursor)
descending: Bool = true
) async throws -> LinkedItemSyncResult {
let fetchResult = try await linkedItemUpdates(since: since, limit: 20, cursor: cursor, descending: descending)
LinkedItem.deleteItems(ids: fetchResult.deletedItemIDs, context: backgroundContext)
@ -31,39 +19,15 @@ public extension DataService {
throw BasicError.message(messageText: "CoreData error")
}
let prev = previousQueryResult?.updatedItemIDs ?? []
let newestChange = fetchResult.items.max { $0.updatedAt < $1.updatedAt }
let result = LinkedItemSyncResult(
updatedItemIDs: prev + fetchResult.items.map(\.id),
updatedItemIDs: fetchResult.items.map(\.id),
cursor: fetchResult.cursor,
hasMore: fetchResult.hasMoreItems
hasMore: fetchResult.hasMoreItems,
mostRecentUpdatedAt: newestChange?.updatedAt,
isEmpty: fetchResult.deletedItemIDs.isEmpty && fetchResult.items.isEmpty
)
print("synced since:", date, "total synced items:", result.updatedItemIDs.count, ", fetchResult.hasMoreItems", fetchResult.hasMoreItems)
if fetchResult.hasMoreItems {
if deferFetchingMore {
Task.detached(priority: .background) {
try await self.syncLinkedItems(
since: date,
cursor: fetchResult.cursor,
previousQueryResult: result,
deferFetchingMore: false
)
}
} else {
return try await syncLinkedItems(
since: date,
cursor: fetchResult.cursor,
previousQueryResult: result,
deferFetchingMore: deferFetchingMore
)
}
} else {
print("setting last synced time to: ", date)
DispatchQueue.main.async {
self.lastItemSyncTime = DateFormatter.formatterISO8601.string(from: date)
}
}
return result
}
@ -111,4 +75,52 @@ public extension DataService {
let articleContent = try await loadArticleContentWithRetries(itemID: requestID, username: username, requestCount: 0)
return articleContent.objectID
}
// This will iterate through a paginated list of sync updates, and upon completing each one,
// update the lastItemSyncTime to the pages most recent change. This allows us to paginate
// very large sets of changes that could fail due to rate limiting or network failures.
// Eventually we should be able to work through the list of changes and catch up.
func syncLinkedItemsInBackground(
since: Date,
onComplete: @escaping () -> Void
) {
Task.detached(priority: .background) {
var count = 0
for try await result in BackgroundSync(dataService: self, since: since, cursor: nil) {
count += result.updatedItemIDs.count
if count > 180 {
break
}
}
DispatchQueue.main.sync {
self.lastItemSyncTime = DateFormatter.formatterISO8601.string(from: Date.now)
onComplete()
}
}
}
}
struct BackgroundSync: AsyncSequence {
public typealias Element = LinkedItemSyncResult
public let dataService: DataService
public let since: Date
public let cursor: String?
public struct AsyncIterator: AsyncIteratorProtocol {
let dataService: DataService
public var since: Date
public var cursor: String?
public mutating func next() async throws -> LinkedItemSyncResult? {
let result = try await dataService.syncLinkedItems(since: since,
cursor: cursor,
descending: true)
cursor = result.cursor
return result.isEmpty ? nil : result
}
}
public func makeAsyncIterator() -> AsyncIterator {
AsyncIterator(dataService: dataService, since: since, cursor: cursor)
}
}

13
apple/OmnivoreKit/Sources/Services/DataService/Queries/LinkedItemNetworkQuery.swift
View File

@ -13,6 +13,7 @@ struct InternalLinkedItemUpdatesQueryResult {
let deletedItemIDs: [String]
let cursor: String?
let hasMoreItems: Bool
let totalCount: Int
}
private struct SyncItemEdge {
@ -26,12 +27,14 @@ extension DataService {
func linkedItemUpdates(
since: Date,
limit: Int,
cursor: String?
cursor: String?,
descending: Bool = true
) async throws -> InternalLinkedItemUpdatesQueryResult {
struct QuerySuccessResult {
let edges: [SyncItemEdge]
let cursor: String?
let hasMoreItems: Bool
let totalCount: Int
}
enum QueryResult {
case success(result: QuerySuccessResult)
@ -55,6 +58,9 @@ extension DataService {
}),
hasMoreItems: try $0.pageInfo(selection: Selection.PageInfo {
try $0.hasNextPage()
}),
totalCount: try $0.pageInfo(selection: Selection.PageInfo {
try $0.totalCount() ?? -1
})
)
)
@ -62,7 +68,7 @@ extension DataService {
)
}
let sort = InputObjects.SortParams(by: Enums.SortBy.savedAt, order: OptionalArgument(Enums.SortOrder.descending))
let sort = InputObjects.SortParams(by: Enums.SortBy.updatedTime, order: OptionalArgument(descending ? Enums.SortOrder.descending : Enums.SortOrder.ascending))
let query = Selection.Query {
try $0.updatesSince(
@ -99,7 +105,8 @@ extension DataService {
items: items,
deletedItemIDs: deletedItemIDs,
cursor: result.cursor,
hasMoreItems: result.hasMoreItems
hasMoreItems: result.hasMoreItems,
totalCount: result.totalCount
)
)
case let .error(error):

BIN
apple/OmnivoreKit/Sources/Views/Resources/Fonts/Georgia-Bold.ttf
View File

Binary file not shown.

BIN
apple/OmnivoreKit/Sources/Views/Resources/Fonts/Georgia-Italic.ttf
View File

Binary file not shown.

BIN
apple/OmnivoreKit/Sources/Views/Resources/Fonts/Georgia-Regular.ttf
View File

Binary file not shown.