diff --git a/apple/OmnivoreKit/Sources/App/Views/Home/Components/FeedCardNavigationLink.swift b/apple/OmnivoreKit/Sources/App/Views/Home/Components/FeedCardNavigationLink.swift index 3d90cbf61..c080ec145 100644 --- a/apple/OmnivoreKit/Sources/App/Views/Home/Components/FeedCardNavigationLink.swift +++ b/apple/OmnivoreKit/Sources/App/Views/Home/Components/FeedCardNavigationLink.swift @@ -26,7 +26,7 @@ struct MacFeedCardNavigationLink: View { .opacity(0) .buttonStyle(PlainButtonStyle()) .onAppear { - Task { await viewModel.itemAppeared(item: item, dataService: dataService, audioController: audioController) } + Task { await viewModel.itemAppeared(item: item, dataService: dataService) } } FeedCard(item: item, viewer: dataService.currentViewer) { viewModel.selectedLinkItem = item.objectID @@ -56,7 +56,7 @@ struct FeedCardNavigationLink: View { .opacity(0) .buttonStyle(PlainButtonStyle()) .onAppear { - Task { await viewModel.itemAppeared(item: item, dataService: dataService, audioController: audioController) } + Task { await viewModel.itemAppeared(item: item, dataService: dataService) } } FeedCard(item: item, viewer: dataService.currentViewer) } @@ -95,7 +95,7 @@ struct GridCardNavigationLink: View { withAnimation { tapAction() } }) .onAppear { - Task { await viewModel.itemAppeared(item: item, dataService: dataService, audioController: audioController) } + Task { await viewModel.itemAppeared(item: item, dataService: dataService) } } } .aspectRatio(1.8, contentMode: .fill) diff --git a/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift b/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift index d101bb750..3acbe96e3 100644 --- a/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift +++ b/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift @@ -20,7 +20,7 @@ import Views @ObservedObject var viewModel: HomeFeedViewModel func loadItems(isRefresh: Bool) { - Task { await viewModel.loadItems(dataService: dataService, audioController: audioController, isRefresh: isRefresh) } + Task { await viewModel.loadItems(dataService: dataService, isRefresh: isRefresh) } } var body: some View { @@ -448,7 +448,7 @@ import Views } func loadItems(isRefresh: Bool) { - Task { await viewModel.loadItems(dataService: dataService, audioController: audioController, isRefresh: isRefresh) } + Task { await viewModel.loadItems(dataService: dataService, isRefresh: isRefresh) } } var body: some View { diff --git a/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewModel.swift b/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewModel.swift index 515d047c1..18ba41f3b 100644 --- a/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewModel.swift +++ b/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewModel.swift @@ -35,6 +35,15 @@ import Views @Published var showLabelsSheet = false @Published var showCommunityModal = false + var cursor: String? + + // These are used to make sure we handle search result + // responses in the right order + var searchIdx = 0 + var receivedIdx = 0 + + var syncCursor: String? + @AppStorage(UserDefaultKey.lastSelectedLinkedItemFilter.rawValue) var appliedFilter = LinkedItemFilter.inbox.rawValue func handleReaderItemNotification(objectID: NSManagedObjectID, dataService: DataService) { @@ -66,14 +75,7 @@ import Views } } - var cursor: String? - - // These are used to make sure we handle search result - // responses in the right order - var searchIdx = 0 - var receivedIdx = 0 - - func itemAppeared(item: LinkedItem, dataService: DataService, audioController: AudioController) async { + func itemAppeared(item: LinkedItem, dataService: DataService) async { if isLoading { return } let itemIndex = items.firstIndex(where: { $0.id == item.id }) let thresholdIndex = items.index(items.endIndex, offsetBy: -5) @@ -81,8 +83,8 @@ import Views // Check if user has scrolled to the last five items in the list // Make sure we aren't currently loading though, as this would get triggered when the first set // of items are presented to the user. - if let itemIndex = itemIndex, itemIndex > thresholdIndex, items.count < thresholdIndex + 10, !isLoading { - await loadItems(dataService: dataService, audioController: audioController, isRefresh: false) + if let itemIndex = itemIndex, itemIndex > thresholdIndex { + await loadMoreItems(dataService: dataService, isRefresh: false) } } @@ -106,15 +108,23 @@ import Views } } - func syncItems(dataService: DataService, syncStartTime _: Date) async { + func syncItems(dataService: DataService) async { + let syncStart = Date.now let lastSyncDate = dateFormatter.date(from: dataService.lastItemSyncTime) ?? Date(timeIntervalSinceReferenceDate: 0) - let syncResult = try? await dataService.syncLinkedItems(since: lastSyncDate, - cursor: nil, - deferFetchingMore: true) - if syncResult?.hasMore { - // dataService.lastItemSyncTime = dateFormatter.string(from: syncStartTime) - self.isLoading = true + try? await dataService.syncOfflineItemsWithServerIfNeeded() + + let syncResult = try? await dataService.syncLinkedItems(since: lastSyncDate, + cursor: nil) + + syncCursor = syncResult?.cursor + if let syncResult = syncResult, syncResult.hasMore { + dataService.syncLinkedItemsInBackground(since: lastSyncDate) { + // Set isLoading to false here + self.isLoading = false + } + } else { + dataService.lastItemSyncTime = DateFormatter.formatterISO8601.string(from: syncStart) } // If possible start prefetching new pages in the background @@ -173,25 +183,20 @@ import Views } } - func loadItems(dataService: DataService, audioController _: AudioController, isRefresh: Bool) async { - let syncStartTime = Date() - + func loadItems(dataService: DataService, isRefresh: Bool) async { isLoading = true showLoadingBar = true await withTaskGroup(of: Void.self) { group in group.addTask { await self.loadCurrentViewer(dataService: dataService) } group.addTask { await self.loadLabels(dataService: dataService) } - group.addTask { await self.syncItems(dataService: dataService, syncStartTime: syncStartTime) } + group.addTask { await self.syncItems(dataService: dataService) } await group.waitForAll() } if searchTerm.replacingOccurrences(of: " ", with: "").isEmpty { updateFetchController(dataService: dataService) - // For now we are forcing the search because we are fetching items in reverse - // with the sync API, but search fetches in descending order - - if appliedFilter != LinkedItemFilter.inbox.rawValue { + if appliedFilter != LinkedItemFilter.inbox.rawValue || !isRefresh { await loadSearchQuery(dataService: dataService, isRefresh: isRefresh) } } else { @@ -202,6 +207,16 @@ import Views showLoadingBar = false } + func loadMoreItems(dataService: DataService, isRefresh _: Bool) async { + isLoading = true + showLoadingBar = true + + await loadSearchQuery(dataService: dataService, isRefresh: isRefresh) + + isLoading = false + showLoadingBar = false + } + private var fetchRequest: NSFetchRequest { let fetchRequest: NSFetchRequest = LinkedItem.fetchRequest() diff --git a/apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift b/apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift index eae67dbb7..e07f2edf9 100644 --- a/apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift +++ b/apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift @@ -16,11 +16,15 @@ public struct LinkedItemSyncResult { public let updatedItemIDs: [String] public let cursor: String? public let hasMore: Bool + public let mostRecentUpdatedAt: Date? + public let isEmpty: Bool - public init(updatedItemIDs: [String], cursor: String?, hasMore: Bool) { + public init(updatedItemIDs: [String], cursor: String?, hasMore: Bool, mostRecentUpdatedAt: Date?, isEmpty: Bool) { self.updatedItemIDs = updatedItemIDs self.cursor = cursor self.hasMore = hasMore + self.mostRecentUpdatedAt = mostRecentUpdatedAt + self.isEmpty = isEmpty } } diff --git a/apple/OmnivoreKit/Sources/Services/DataService/OfflineSync.swift b/apple/OmnivoreKit/Sources/Services/DataService/OfflineSync.swift index 60e85ac67..482b38c6e 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/OfflineSync.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/OfflineSync.swift @@ -4,7 +4,7 @@ import Models import Utils public extension DataService { - internal func syncOfflineItemsWithServerIfNeeded() async throws { + func syncOfflineItemsWithServerIfNeeded() async throws { var unsyncedLinkedItems = [LinkedItem]() var unsyncedHighlights = [Highlight]() diff --git a/apple/OmnivoreKit/Sources/Services/DataService/Public/LinkedItemLoading.swift b/apple/OmnivoreKit/Sources/Services/DataService/Public/LinkedItemLoading.swift index 6ef36ea88..ad776f9ce 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/Public/LinkedItemLoading.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/Public/LinkedItemLoading.swift @@ -6,24 +6,12 @@ public extension DataService { /// Requests `LinkedItem`s updates from the server since a certain datae /// and stores it in CoreData while deleting all the items with ids the server says /// have been deleted. - /// - Parameters: - /// - limit: max count of items - /// - searchQuery: search terms and filters - /// - cursor: cursor when loading batch for infinite list - /// - Returns: `LinkedItemQueryResult` (managed object IDs and an optional cursor) func syncLinkedItems( - since date: Date, + since: Date, cursor: String?, - previousQueryResult: LinkedItemSyncResult? = nil, - deferFetchingMore: Bool - ) async throws -> LinkedItemSyncResult? { - if previousQueryResult == nil { - // Send offline changes to server before fetching items - // only on the first call of this function - try? await syncOfflineItemsWithServerIfNeeded() - } - - let fetchResult = try await linkedItemUpdates(since: date, limit: 20, cursor: cursor) + descending: Bool = true + ) async throws -> LinkedItemSyncResult { + let fetchResult = try await linkedItemUpdates(since: since, limit: 20, cursor: cursor, descending: descending) LinkedItem.deleteItems(ids: fetchResult.deletedItemIDs, context: backgroundContext) @@ -31,39 +19,15 @@ public extension DataService { throw BasicError.message(messageText: "CoreData error") } - let prev = previousQueryResult?.updatedItemIDs ?? [] + let newestChange = fetchResult.items.max { $0.updatedAt < $1.updatedAt } let result = LinkedItemSyncResult( - updatedItemIDs: prev + fetchResult.items.map(\.id), + updatedItemIDs: fetchResult.items.map(\.id), cursor: fetchResult.cursor, - hasMore: fetchResult.hasMoreItems + hasMore: fetchResult.hasMoreItems, + mostRecentUpdatedAt: newestChange?.updatedAt, + isEmpty: fetchResult.deletedItemIDs.isEmpty && fetchResult.items.isEmpty ) - print("synced since:", date, "total synced items:", result.updatedItemIDs.count, ", fetchResult.hasMoreItems", fetchResult.hasMoreItems) - if fetchResult.hasMoreItems { - if deferFetchingMore { - Task.detached(priority: .background) { - try await self.syncLinkedItems( - since: date, - cursor: fetchResult.cursor, - previousQueryResult: result, - deferFetchingMore: false - ) - } - } else { - return try await syncLinkedItems( - since: date, - cursor: fetchResult.cursor, - previousQueryResult: result, - deferFetchingMore: deferFetchingMore - ) - } - } else { - print("setting last synced time to: ", date) - DispatchQueue.main.async { - self.lastItemSyncTime = DateFormatter.formatterISO8601.string(from: date) - } - } - return result } @@ -111,4 +75,52 @@ public extension DataService { let articleContent = try await loadArticleContentWithRetries(itemID: requestID, username: username, requestCount: 0) return articleContent.objectID } + + // This will iterate through a paginated list of sync updates, and upon completing each one, + // update the lastItemSyncTime to the pages most recent change. This allows us to paginate + // very large sets of changes that could fail due to rate limiting or network failures. + // Eventually we should be able to work through the list of changes and catch up. + func syncLinkedItemsInBackground( + since: Date, + onComplete: @escaping () -> Void + ) { + Task.detached(priority: .background) { + var count = 0 + for try await result in BackgroundSync(dataService: self, since: since, cursor: nil) { + count += result.updatedItemIDs.count + if count > 180 { + break + } + } + DispatchQueue.main.sync { + self.lastItemSyncTime = DateFormatter.formatterISO8601.string(from: Date.now) + onComplete() + } + } + } +} + +struct BackgroundSync: AsyncSequence { + public typealias Element = LinkedItemSyncResult + public let dataService: DataService + public let since: Date + public let cursor: String? + + public struct AsyncIterator: AsyncIteratorProtocol { + let dataService: DataService + public var since: Date + public var cursor: String? + + public mutating func next() async throws -> LinkedItemSyncResult? { + let result = try await dataService.syncLinkedItems(since: since, + cursor: cursor, + descending: true) + cursor = result.cursor + return result.isEmpty ? nil : result + } + } + + public func makeAsyncIterator() -> AsyncIterator { + AsyncIterator(dataService: dataService, since: since, cursor: cursor) + } } diff --git a/apple/OmnivoreKit/Sources/Services/DataService/Queries/LinkedItemNetworkQuery.swift b/apple/OmnivoreKit/Sources/Services/DataService/Queries/LinkedItemNetworkQuery.swift index e01a01bc5..eac3e5e26 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/Queries/LinkedItemNetworkQuery.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/Queries/LinkedItemNetworkQuery.swift @@ -13,6 +13,7 @@ struct InternalLinkedItemUpdatesQueryResult { let deletedItemIDs: [String] let cursor: String? let hasMoreItems: Bool + let totalCount: Int } private struct SyncItemEdge { @@ -26,12 +27,14 @@ extension DataService { func linkedItemUpdates( since: Date, limit: Int, - cursor: String? + cursor: String?, + descending: Bool = true ) async throws -> InternalLinkedItemUpdatesQueryResult { struct QuerySuccessResult { let edges: [SyncItemEdge] let cursor: String? let hasMoreItems: Bool + let totalCount: Int } enum QueryResult { case success(result: QuerySuccessResult) @@ -55,6 +58,9 @@ extension DataService { }), hasMoreItems: try $0.pageInfo(selection: Selection.PageInfo { try $0.hasNextPage() + }), + totalCount: try $0.pageInfo(selection: Selection.PageInfo { + try $0.totalCount() ?? -1 }) ) ) @@ -62,7 +68,7 @@ extension DataService { ) } - let sort = InputObjects.SortParams(by: Enums.SortBy.savedAt, order: OptionalArgument(Enums.SortOrder.descending)) + let sort = InputObjects.SortParams(by: Enums.SortBy.updatedTime, order: OptionalArgument(descending ? Enums.SortOrder.descending : Enums.SortOrder.ascending)) let query = Selection.Query { try $0.updatesSince( @@ -99,7 +105,8 @@ extension DataService { items: items, deletedItemIDs: deletedItemIDs, cursor: result.cursor, - hasMoreItems: result.hasMoreItems + hasMoreItems: result.hasMoreItems, + totalCount: result.totalCount ) ) case let .error(error): diff --git a/apple/OmnivoreKit/Sources/Views/Resources/Fonts/Georgia-Bold.ttf b/apple/OmnivoreKit/Sources/Views/Resources/Fonts/Georgia-Bold.ttf deleted file mode 100644 index a0c25f34a..000000000 Binary files a/apple/OmnivoreKit/Sources/Views/Resources/Fonts/Georgia-Bold.ttf and /dev/null differ diff --git a/apple/OmnivoreKit/Sources/Views/Resources/Fonts/Georgia-Italic.ttf b/apple/OmnivoreKit/Sources/Views/Resources/Fonts/Georgia-Italic.ttf deleted file mode 100644 index c1b59ff75..000000000 Binary files a/apple/OmnivoreKit/Sources/Views/Resources/Fonts/Georgia-Italic.ttf and /dev/null differ diff --git a/apple/OmnivoreKit/Sources/Views/Resources/Fonts/Georgia-Regular.ttf b/apple/OmnivoreKit/Sources/Views/Resources/Fonts/Georgia-Regular.ttf deleted file mode 100644 index 547802892..000000000 Binary files a/apple/OmnivoreKit/Sources/Views/Resources/Fonts/Georgia-Regular.ttf and /dev/null differ