Benature/omnivore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improvements to background syncing

Browse Source
This commit is contained in:
Jackson Harper
2022-06-01 17:24:42 -07:00
parent 3ab3fd6f77
commit b2cbcc23c3
8 changed files with 321 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files

160
apple/OmnivoreKit/Sources/App/AppExtensions/Share/ExtensionSaveService.swift Normal file
View File

@ -0,0 +1,160 @@
//
// File.swift
//
//
// Created by Jackson Harper on 6/1/22.
//
import Foundation
import Models
import Services
import Views
typealias UpdateStatusFunc = (ShareExtensionStatus) -> Void
class ExtensionSaveService {
let queue: OperationQueue
init() {
self.queue = OperationQueue()
}
private func queueSaveOperation(_ pageScrape: PageScrapePayload, updateStatusFunc: UpdateStatusFunc?) {
ProcessInfo().performExpiringActivity(withReason: "app.omnivore.SaveActivity") { [self] expiring in
guard !expiring else {
self.queue.cancelAllOperations()
self.queue.waitUntilAllOperationsAreFinished()
return
}
let operation = SaveOperation(pageScrapePayload: pageScrape, updateStatusFunc: updateStatusFunc)
self.queue.addOperation(operation)
self.queue.waitUntilAllOperationsAreFinished()
}
}
public func save(_ extensionContext: NSExtensionContext, updateStatusFunc: UpdateStatusFunc?) {
PageScraper.scrape(extensionContext: extensionContext) { [weak self] result in
guard let self = self else { return }
switch result {
case let .success(payload):
self.queueSaveOperation(payload, updateStatusFunc: updateStatusFunc)
case let .failure(error):
print("failed", error)
}
}
}
class SaveOperation: Operation, URLSessionDelegate {
let requestId: String
let services: Services
let pageScrapePayload: PageScrapePayload
let updateStatusFunc: UpdateStatusFunc?
var queue: OperationQueue?
var uploadTask: URLSessionTask?
enum State: Int {
case created
case started
case finished
}
init(pageScrapePayload: PageScrapePayload, updateStatusFunc: UpdateStatusFunc? = nil) {
self.pageScrapePayload = pageScrapePayload
self.updateStatusFunc = updateStatusFunc
self.state = .created
self.services = Services()
self.requestId = UUID().uuidString.lowercased()
}
open var state: State = .created {
willSet {
willChangeValue(forKey: "isReady")
willChangeValue(forKey: "isExecuting")
willChangeValue(forKey: "isFinished")
willChangeValue(forKey: "isCancelled")
}
didSet {
didChangeValue(forKey: "isCancelled")
didChangeValue(forKey: "isFinished")
didChangeValue(forKey: "isExecuting")
didChangeValue(forKey: "isReady")
}
}
override var isAsynchronous: Bool {
true
}
override var isReady: Bool {
true
}
override var isExecuting: Bool {
self.state == .started
}
override var isFinished: Bool {
self.state == .finished
}
override func start() {
guard !isCancelled else { return }
state = .started
queue = OperationQueue()
Task {
await persist(services: self.services, pageScrapePayload: self.pageScrapePayload, requestId: self.requestId)
}
}
override func cancel() {
// task?.cancel()
// finishOperation()
//
// storeUnresolvedSavedItem()
super.cancel()
}
private func updateStatus(newStatus: ShareExtensionStatus) {
DispatchQueue.main.async {
if let updateStatusFunc = self.updateStatusFunc {
updateStatusFunc(newStatus)
}
}
}
private func persist(services: Services, pageScrapePayload: PageScrapePayload, requestId: String) async {
do {
try await services.dataService.persistPageScrapePayload(pageScrapePayload, requestId: requestId)
} catch {
updateStatus(newStatus: .failed(error: SaveArticleError.unknown(description: "Unable to access content")))
return
}
do {
updateStatus(newStatus: .saved)
switch pageScrapePayload.contentType {
case .none:
try await services.dataService.syncUrl(id: requestId, url: pageScrapePayload.url)
case let .pdf(localUrl):
try await services.dataService.syncPdf(id: requestId, localPdfURL: localUrl, url: pageScrapePayload.url)
case let .html(html, title):
try await services.dataService.syncPage(id: requestId, originalHtml: html, title: title, url: pageScrapePayload.url)
}
} catch {
print("ERROR SYNCING", error)
updateStatus(newStatus: .syncFailed(error: SaveArticleError.unknown(description: "Unknown Error")))
}
state = .finished
updateStatus(newStatus: .synced)
}
}
}

127
apple/OmnivoreKit/Sources/App/AppExtensions/Share/ShareExtensionScene.swift
View File

@ -25,12 +25,10 @@ final class ShareExtensionViewModel: ObservableObject {
@Published var status: ShareExtensionStatus = .processing
@Published var debugText: String?
let services = Services()
var subscriptions = Set<AnyCancellable>()
var backgroundTask: UIBackgroundTaskIdentifier?
let requestID = UUID().uuidString.lowercased()
init() {}
let saveService = ExtensionSaveService()
func handleReadNowAction(extensionContext: NSExtensionContext?) {
#if os(iOS)
@ -43,135 +41,20 @@ final class ShareExtensionViewModel: ObservableObject {
}
func savePage(extensionContext: NSExtensionContext?) {
backgroundTask = UIApplication.shared.beginBackgroundTask(withName: requestID)
PageScraper.scrape(extensionContext: extensionContext) { [weak self] result in
guard let self = self else { return }
switch result {
case let .success(payload):
Task {
await self.persist(pageScrapePayload: payload, requestId: self.requestID)
self.endBackgroundTask()
}
case let .failure(error):
if let backgroundTask = self.backgroundTask {
UIApplication.shared.endBackgroundTask(backgroundTask)
self.backgroundTask = nil
}
self.debugText = error.message
self.endBackgroundTask()
}
}
}
private func endBackgroundTask() {
if let backgroundTask = self.backgroundTask {
UIApplication.shared.endBackgroundTask(backgroundTask)
}
}
private func persist(pageScrapePayload: PageScrapePayload, requestId: String) async {
// Save locally first
let linkedItem = try? await services.dataService.persistPageScrapePayload(pageScrapePayload, requestId: requestId)
if let linkedItem = linkedItem {
updateStatus(newStatus: .saved)
await services.dataService.syncLocalCreatedLinkedItem(item: linkedItem)
updateStatus(newStatus: .synced)
if let extensionContext = extensionContext {
saveService.save(extensionContext, updateStatusFunc: updateStatus)
} else {
updateStatus(newStatus: .failed(error: SaveArticleError.unknown(description: "Unable to save page")))
updateStatus(.failed(error: .unknown(description: "Internal Error")))
}
}
private func updateStatus(newStatus: ShareExtensionStatus) {
private func updateStatus(_ newStatus: ShareExtensionStatus) {
DispatchQueue.main.async {
self.status = newStatus
}
}
}
// Task {
// do {
// // Save locally, then attempt to sync to the server
// let item = try await services.dataService.persistPageScrapePayload(pageScrapePayload, requestId: requestId)
// // TODO: need to update this on the main thread and handle the result == false case here
// if item != nil {
// self.status = .saved
// } else {
// self.status = .failed(error: SaveArticleError.unknown(description: "Unable to save page"))
// return
// }
//
// // force a server sync
// if let item = item {
// let syncResult = services.dataService.syncLocalCreatedLinkedItem(item: item)
// print("RESULT", syncResult)
// }
//// self.status = .synced
//// } else {
//// self.status = .syncFailed(error: SaveArticleError.unknown(description: "Unable to sync page"))
//// }
//
// } catch {
// print("ERROR SAVING PAGE", error)
// }
// }
// // First persist to Core Data
// // services.dataService.persist(jsonArticle: article)
//
// guard services.authenticator.hasValidAuthToken else {
// status = .failed(error: .unauthorized)
// return
// }
//
// let saveLinkPublisher: AnyPublisher<Void, SaveArticleError> = {
// if case let .pdf(data) = pageScrapePayload.contentType {
// return services.dataService.uploadPDFPublisher(pageScrapePayload: pageScrapePayload,
// data: data,
// requestId: requestId)
// } else if case let .html(html, title) = pageScrapePayload.contentType {
// return services.dataService.savePagePublisher(pageScrapePayload: pageScrapePayload,
// html: html,
// title: title,
// requestId: requestId)
// } else {
// return services.dataService.saveUrlPublisher(pageScrapePayload: pageScrapePayload, requestId: requestId)
// }
// }()
//
// saveLinkPublisher
// .sink { [weak self] completion in
// guard case let .failure(error) = completion else { return }
// self?.debugText = "saveArticleError: \(error)"
// self?.status = .failed(error: error)
// if let backgroundTask = self?.backgroundTask {
// UIApplication.shared.endBackgroundTask(backgroundTask)
// }
// } receiveValue: { [weak self] _ in
// self?.status = .success
// if let backgroundTask = self?.backgroundTask {
// UIApplication.shared.endBackgroundTask(backgroundTask)
// }
// }
// .store(in: &subscriptions)
//
// // Check connection to get fast feedback for auth/network errors
// Task {
// let hasConnectionAndValidToken = await services.dataService.hasConnectionAndValidToken()
//
// if !hasConnectionAndValidToken {
// DispatchQueue.main.async {
// self.debugText = "saveArticleError: No connection or invalid token."
// self.status = .failed(error: .unknown(description: ""))
// }
// }
// }
// }
// }
struct ShareExtensionView: View {
let extensionContext: NSExtensionContext?
@StateObject private var viewModel = ShareExtensionViewModel()

18
apple/OmnivoreKit/Sources/Services/DataService/DataService.swift
View File

@ -15,10 +15,10 @@ public final class DataService: ObservableObject {
public static var showIntercomMessenger: (() -> Void)?
public let appEnvironment: AppEnvironment
let networker: Networker
public let networker: Networker
var persistentContainer: PersistentContainer
var backgroundContext: NSManagedObjectContext
public var backgroundContext: NSManagedObjectContext
var subscriptions = Set<AnyCancellable>()
public var viewContext: NSManagedObjectContext {
@ -41,6 +41,12 @@ public final class DataService: ObservableObject {
}
}
}
NotificationCenter.default
.addObserver(self,
selector: #selector(locallyCreatedItemSynced),
name: NSNotification.LocallyCreatedItemSynced,
object: nil)
}
public var currentViewer: Viewer? {
@ -101,9 +107,9 @@ public final class DataService: ObservableObject {
return isFirstRun
}
public func persistPageScrapePayload(_ pageScrape: PageScrapePayload, requestId: String) async throws -> LinkedItem? {
public func persistPageScrapePayload(_ pageScrape: PageScrapePayload, requestId: String) async throws {
try await backgroundContext.perform { [weak self] in
guard let self = self else { return nil }
guard let self = self else { return }
let fetchRequest: NSFetchRequest<Models.LinkedItem> = LinkedItem.fetchRequest()
fetchRequest.predicate = NSPredicate(format: "id == %@", requestId)
@ -135,8 +141,6 @@ public final class DataService: ObservableObject {
switch pageScrape.contentType {
case let .pdf(localUrl):
print("SAVING PDF", localUrl)
linkedItem.contentReader = "PDF"
linkedItem.localPdfURL = localUrl.absoluteString
linkedItem.title = self.titleFromPdfFile(pageScrape.url)
@ -147,7 +151,6 @@ public final class DataService: ObservableObject {
// linkedItem.imageURLString = thumbnailUrl.absoluteString
case let .html(html: html, title: title):
print("SAVING HTML", html, title ?? "no title")
linkedItem.contentReader = "WEB"
linkedItem.originalHtml = html
linkedItem.title = title ?? self.titleFromPdfFile(pageScrape.url)
@ -165,7 +168,6 @@ public final class DataService: ObservableObject {
print("Failed to save ArticleContent", error.localizedDescription, error)
throw error
}
return linkedItem
}
}

89
apple/OmnivoreKit/Sources/Services/DataService/Mutations/SavePDF.swift
View File

@ -3,14 +3,14 @@ import Foundation
import Models
import SwiftGraphQL
private struct UploadFileRequestPayload {
let uploadID: String?
let uploadFileID: String?
let urlString: String?
public struct UploadFileRequestPayload {
public let uploadID: String?
public let uploadFileID: String?
public let urlString: String?
}
private extension DataService {
public func uploadFileRequest(id: String, url: String) async throws -> URL {
public extension DataService {
func uploadFileRequest(id: String, url: String) async throws -> UploadFileRequestPayload {
enum MutationResult {
case success(payload: UploadFileRequestPayload)
case error(errorCode: Enums.UploadFileRequestErrorCode?)
@ -59,7 +59,7 @@ private extension DataService {
switch payload.data {
case let .success(payload):
if let urlString = payload.urlString, let url = URL(string: urlString) {
continuation.resume(returning: url)
continuation.resume(returning: payload)
} else {
continuation.resume(throwing: SaveArticleError.unknown(description: "No upload URL"))
}
@ -78,29 +78,49 @@ private extension DataService {
}
}
public func uploadFile(localPdfURL: String?, url: URL) -> URLSessionTask? {
func uploadFile(id _: String, localPdfURL: URL, url: URL) async throws {
var request = URLRequest(url: url)
request.httpMethod = "PUT"
request.addValue("application/pdf", forHTTPHeaderField: "content-type")
return try await withCheckedThrowingContinuation { continuation in
let task = networker.urlSession.uploadTask(with: request, fromFile: localPdfURL) { _, response, _ in
print("UPLOAD RESPONSE", response)
if let httpResponse = response as? HTTPURLResponse, 200 ... 299 ~= httpResponse.statusCode {
continuation.resume()
} else {
continuation.resume(throwing: SaveArticleError.unknown(description: "Invalid response"))
}
}
task.resume()
}
}
func uploadFileInBackground(id: String, localPdfURL: String?, url: URL, usingSession session: URLSession) -> URLSessionTask? {
if let localPdfURL = localPdfURL, let localUrl = URL(string: localPdfURL) {
var request = URLRequest(url: url)
request.httpMethod = "PUT"
request.setValue("application/pdf", forHTTPHeaderField: "content-type")
request.setValue(id, forHTTPHeaderField: "clientRequestId")
let task = networker.backgroundSession.uploadTask(with: request, fromFile: localUrl)
task.resume()
let task = session.uploadTask(with: request, fromFile: localUrl)
return task
} else {
// TODO: How should we handle this scenario?
print("NOT UPLOADING PDF DOCUMENT YET")
return nil
}
}
// swiftlint:disable:next line_length
func saveFilePublisher(pageScrapePayload: PageScrapePayload, uploadFileId: String, requestId: String) -> AnyPublisher<Void, SaveArticleError> {
func saveFilePublisher(requestId: String, uploadFileId: String, url: String) async throws {
enum MutationResult {
case saved(requestId: String, url: String)
case error(errorCode: Enums.SaveErrorCode)
}
let input = InputObjects.SaveFileInput(
url: pageScrapePayload.url,
url: url,
source: "ios-file",
clientRequestId: requestId,
uploadFileId: uploadFileId
@ -120,34 +140,31 @@ private extension DataService {
let path = appEnvironment.graphqlPath
let headers = networker.defaultHeaders
return Deferred {
Future { promise in
send(mutation, to: path, headers: headers) { result in
switch result {
case let .success(payload):
if let graphqlError = payload.errors {
promise(.failure(.unknown(description: graphqlError.first.debugDescription)))
}
switch payload.data {
case .saved:
promise(.success(()))
case let .error(errorCode: errorCode):
switch errorCode {
case .unauthorized:
promise(.failure(.unauthorized))
default:
promise(.failure(.unknown(description: errorCode.rawValue)))
}
}
case let .failure(error):
promise(.failure(SaveError.make(from: error)))
return try await withCheckedThrowingContinuation { continuation in
send(mutation, to: path, headers: headers) { result in
switch result {
case let .success(payload):
if let graphqlError = payload.errors {
continuation.resume(throwing: SaveArticleError.unknown(description: graphqlError.first.debugDescription))
return
}
switch payload.data {
case .saved:
continuation.resume()
case let .error(errorCode: errorCode):
switch errorCode {
case .unauthorized:
continuation.resume(throwing: SaveArticleError.unauthorized)
default:
continuation.resume(throwing: SaveArticleError.unknown(description: errorCode.rawValue))
}
}
case let .failure(error):
continuation.resume(throwing: SaveArticleError.make(from: error))
}
}
}
.receive(on: DispatchQueue.main)
.eraseToAnyPublisher()
}
}

33
apple/OmnivoreKit/Sources/Services/DataService/Networking/Networker.swift
View File

@ -4,6 +4,7 @@ import Models
public final class Networker: NSObject, URLSessionTaskDelegate {
let urlSession: URLSession
let appEnvironment: AppEnvironment
var uploadQueue: [String: URLSessionUploadTask] = [:]
var defaultHeaders: [String: String] {
var headers = URLRequest.defaultHeaders
@ -20,26 +21,26 @@ public final class Networker: NSObject, URLSessionTaskDelegate {
self.urlSession = .shared
}
lazy var backgroundSession: URLSession = {
let sessionConfig = URLSessionConfiguration.background(withIdentifier: "app.omnivoreapp.BackgroundSessionConfig")
public func createBackgroundSession() -> URLSession {
let sessionConfig = URLSessionConfiguration.background(withIdentifier: "app.omnivoreapp.BackgroundSessionConfig-")
sessionConfig.sharedContainerIdentifier = "group.app.omnivoreapp"
return URLSession(configuration: sessionConfig, delegate: self, delegateQueue: nil)
}()
public func urlSession(_: URLSession, task: URLSessionTask, didCompleteWithError: Error?) {
if let httpResponse = task.response as? HTTPURLResponse {
print("httpRespinse status code", httpResponse.statusCode)
}
print("finished upload of file:", task.taskIdentifier, task.currentRequest, task.response, "with error", didCompleteWithError)
}
public func urlSession(_: URLSession,
task: URLSessionTask,
didSendBodyData _: Int64,
totalBytesSent: Int64,
totalBytesExpectedToSend _: Int64)
{
print("sent background data:", task.taskIdentifier, totalBytesSent)
public func urlSession(_: URLSession, task: URLSessionTask, didCompleteWithError error: Error?) {
print("finished upload on original request", task.originalRequest, "error", error)
if let httpResponse = task.response as? HTTPURLResponse {
if 200 ... 299 ~= httpResponse.statusCode {
// success
if let requestId = task.originalRequest?.value(forHTTPHeaderField: "clientRequestId") {
print("COMPLETED UPLOADED REQUEST ID", requestId)
DispatchQueue.main.async {
NotificationCenter.default.post(name: NSNotification.LocallyCreatedItemSynced, object: nil, userInfo: ["objectID": requestId])
}
}
}
print("DONE")
}
}
}

92
apple/OmnivoreKit/Sources/Services/DataService/OfflineSync.swift
View File

@ -2,8 +2,8 @@ import CoreData
import Foundation
import Models
extension DataService {
func syncOfflineItemsWithServerIfNeeded() async throws {
public extension DataService {
internal func syncOfflineItemsWithServerIfNeeded() async throws {
// TODO: send a simple request to see if we're online?
var unsyncedLinkedItems = [LinkedItem]()
var unsyncedHighlights = [Highlight]()
@ -35,20 +35,61 @@ extension DataService {
}
}
public func syncLocalCreatedLinkedItem(item: LinkedItem) {
// func syncPdf(item: LinkedItem, usingSession session: URLSession) async throws -> Bool {
// try backgroundContext.performAndWait {
// item.serverSyncStatus = Int64(ServerSyncStatus.isSyncing.rawValue)
// try self.backgroundContext.save()
// }
//
// let id = item.unwrappedID
// let localPdfURL = item.localPdfURL
// let url = item.unwrappedPageURLString
// let uploadRequestUrl = try await uploadFileRequest(id: id, url: url)
// return await try uploadFile(id: id, localPdfURL: localPdfURL, url: uploadRequestUrl, usingSession: session)
// }
func syncPdf(id: String, localPdfURL: URL, url: String) async throws {
// try backgroundContext.performAndWait {
// item.serverSyncStatus = Int64(ServerSyncStatus.isSyncing.rawValue)
// try self.backgroundContext.save()
// }
let uploadRequest = try await uploadFileRequest(id: id, url: url)
if let urlString = uploadRequest.urlString, let uploadUrl = URL(string: urlString) {
try await uploadFile(id: id, localPdfURL: localPdfURL, url: uploadUrl)
// try await services.dataService.saveFilePublisher(requestId: requestId, uploadFileId: uploadFileID, url: url)
} else {
throw SaveArticleError.badData
}
}
func syncPage(id: String, originalHtml: String, title: String?, url: String) async throws {
// try backgroundContext.performAndWait {
// item.serverSyncStatus = Int64(ServerSyncStatus.isSyncing.rawValue)
// try self.backgroundContext.save()
// }
try await savePage(id: id, url: url, title: title ?? url, originalHtml: originalHtml)
}
func syncUrl(id: String, url: String) async throws {
try await saveURL(id: id, url: url)
}
func syncLocalCreatedLinkedItem(item: LinkedItem) {
switch item.contentReader {
case "PDF":
let id = item.unwrappedID
let localPdfURL = item.localPdfURL
let url = item.unwrappedPageURLString
Task {
let uploadRequestUrl = try await uploadFileRequest(id: id, url: url)
await uploadFile(localPdfURL: localPdfURL, url: uploadRequestUrl)
try await backgroundContext.perform {
item.serverSyncStatus = Int64(ServerSyncStatus.isNSync.rawValue)
try self.backgroundContext.save()
}
}
// let id = item.unwrappedID
// let localPdfURL = item.localPdfURL
// let url = item.unwrappedPageURLString
// Task {
// let uploadRequestUrl = try await uploadFileRequest(id: id, url: url)
// uploadFile(id: id, localPdfURL: localPdfURL, url: uploadRequestUrl)
// try await backgroundContext.perform {
// item.serverSyncStatus = Int64(ServerSyncStatus.isNSync.rawValue)
// try self.backgroundContext.save()
// }
// }
break
case "WEB":
let id = item.unwrappedID
let url = item.unwrappedPageURLString
@ -77,9 +118,7 @@ extension DataService {
switch syncStatus {
case .needsCreation:
// TODO: We will want to sync items that need creation in the background
// these items are forced to sync when saved, but should be re-tried in
// the background.
item.serverSyncStatus = Int64(ServerSyncStatus.isSyncing.rawValue)
syncLocalCreatedLinkedItem(item: item)
case .isNSync, .isSyncing:
break
@ -129,4 +168,23 @@ extension DataService {
}
}
}
@objc
func locallyCreatedItemSynced(notification: NSNotification) {
print("SYNCED LOCALLY CREATED ITEM", notification)
if let objectId = notification.userInfo?["objectID"] as? String {
do {
try backgroundContext.performAndWait {
let fetchRequest: NSFetchRequest<Models.LinkedItem> = LinkedItem.fetchRequest()
fetchRequest.predicate = NSPredicate(format: "id == %@", objectId)
if let existingItem = try? self.backgroundContext.fetch(fetchRequest).first {
existingItem.serverSyncStatus = Int64(ServerSyncStatus.isNSync.rawValue)
try self.backgroundContext.save()
}
}
} catch {
print("ERROR", error)
}
}
}
}

1
apple/OmnivoreKit/Sources/Services/DataService/SaveService.swift Normal file
View File

@ -0,0 +1 @@

8
apple/OmnivoreKit/Sources/Services/NSNotification+BackgroundSync.swift
View File

@ -1,8 +0,0 @@
//
// File.swift
//
//
// Created by Jackson Harper on 5/31/22.
//
import Foundation