diff --git a/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift b/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift index 834325291..9190160d5 100644 --- a/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift +++ b/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift @@ -27,196 +27,6 @@ case high } - // Somewhat based on: https://github.com/neekeetab/CachingPlayerItem/blob/master/CachingPlayerItem.swift - class SpeechPlayerItem: AVPlayerItem { - let resourceLoaderDelegate = ResourceLoaderDelegate() - let session: AudioController - let speechItem: SpeechItem - var speechMarks: [SpeechMark]? - - let completed: () -> Void - - var observer: Any? - - init(session: AudioController, speechItem: SpeechItem, completed: @escaping () -> Void) { - self.speechItem = speechItem - self.session = session - self.completed = completed - - guard let fakeUrl = URL(string: "app.omnivore.speech://\(speechItem.localAudioURL.path).mp3") else { - fatalError("internal inconsistency") - } - - let asset = AVURLAsset(url: fakeUrl) - asset.resourceLoader.setDelegate(resourceLoaderDelegate, queue: DispatchQueue.main) - - super.init(asset: asset, automaticallyLoadedAssetKeys: nil) - - resourceLoaderDelegate.owner = self - - self.observer = observe(\.status, options: [.new]) { item, _ in - if item.status == .readyToPlay { - let duration = CMTimeGetSeconds(item.duration) - item.session.updateDuration(forItem: item.speechItem, newDuration: duration) - } - if item.status == .failed { - item.session.stopWithError() - } - } - - NotificationCenter.default.addObserver( - forName: NSNotification.Name.AVPlayerItemDidPlayToEndTime, - object: self, queue: OperationQueue.main - ) { [weak self] _ in - guard let self = self else { return } - self.completed() - } - } - - deinit { - observer = nil - resourceLoaderDelegate.session?.invalidateAndCancel() - } - - open func download() { - if resourceLoaderDelegate.session == nil { - resourceLoaderDelegate.startDataRequest(with: speechItem.urlRequest) - } - } - - @objc func playbackStalledHandler() { - print("playback stalled...") - } - - class ResourceLoaderDelegate: NSObject, AVAssetResourceLoaderDelegate { - var session: URLSession? - var mediaData: Data? - var pendingRequests = Set() - weak var owner: SpeechPlayerItem? - - func resourceLoader(_: AVAssetResourceLoader, - shouldWaitForLoadingOfRequestedResource loadingRequest: AVAssetResourceLoadingRequest) -> Bool - { - if owner == nil { - return true - } - - if session == nil { - guard let initialUrl = owner?.speechItem.urlRequest else { - fatalError("internal inconsistency") - } - - startDataRequest(with: initialUrl) - } - - pendingRequests.insert(loadingRequest) - processPendingRequests() - return true - } - - func startDataRequest(with _: URLRequest) { - let configuration = URLSessionConfiguration.default - configuration.requestCachePolicy = .reloadIgnoringLocalAndRemoteCacheData - session = URLSession(configuration: configuration) - - Task { - guard let speechItem = self.owner?.speechItem else { - // This probably can't happen, but if it does, just returning should - DispatchQueue.main.async { - self.processPlaybackError(error: BasicError.message(messageText: "No speech item found.")) - } - return - } - - do { - let speechData = try await SpeechSynthesizer.download(speechItem: speechItem, session: self.session ?? URLSession.shared) - - DispatchQueue.main.async { - if speechData == nil { - self.session = nil - self.processPlaybackError(error: BasicError.message(messageText: "Unable to download speech data.")) - return - } - - if let owner = self.owner, let speechData = speechData { - owner.speechMarks = speechData.speechMarks - } - self.mediaData = speechData?.audioData - - self.processPendingRequests() - } - } catch URLError.cancelled { - print("cancelled request error being ignored") - } catch { - DispatchQueue.main.async { - self.processPlaybackError(error: error) - } - } - } - } - - func resourceLoader(_: AVAssetResourceLoader, didCancel loadingRequest: AVAssetResourceLoadingRequest) { - pendingRequests.remove(loadingRequest) - } - - func processPendingRequests() { - let requestsFulfilled = Set(pendingRequests.compactMap { - self.fillInContentInformationRequest($0.contentInformationRequest) - if self.haveEnoughDataToFulfillRequest($0.dataRequest!) { - $0.finishLoading() - return $0 - } - return nil - }) - - // remove fulfilled requests from pending requests - _ = requestsFulfilled.map { self.pendingRequests.remove($0) } - } - - func processPlaybackError(error: Error?) { - let requestsFulfilled = Set(pendingRequests.compactMap { - $0.finishLoading(with: error) - return nil - }) - - _ = requestsFulfilled.map { self.pendingRequests.remove($0) } - } - - func fillInContentInformationRequest(_ contentInformationRequest: AVAssetResourceLoadingContentInformationRequest?) { - contentInformationRequest?.contentType = UTType.mp3.identifier - - if let mediaData = mediaData { - contentInformationRequest?.isByteRangeAccessSupported = true - contentInformationRequest?.contentLength = Int64(mediaData.count) - } - } - - func haveEnoughDataToFulfillRequest(_ dataRequest: AVAssetResourceLoadingDataRequest) -> Bool { - let requestedOffset = Int(dataRequest.requestedOffset) - let requestedLength = dataRequest.requestedLength - let currentOffset = Int(dataRequest.currentOffset) - - guard let songDataUnwrapped = mediaData, - songDataUnwrapped.count > currentOffset - else { - // Don't have any data at all for this request. - return false - } - - let bytesToRespond = min(songDataUnwrapped.count - currentOffset, requestedLength) - let range = Range(uncheckedBounds: (currentOffset, currentOffset + bytesToRespond)) - let dataToRespond = songDataUnwrapped.subdata(in: range) - dataRequest.respond(with: dataToRespond) - - return songDataUnwrapped.count >= requestedLength + requestedOffset - } - - deinit { - session?.invalidateAndCancel() - } - } - } - // swiftlint:disable all public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate { @Published public var state: AudioControllerState = .stopped @@ -763,9 +573,12 @@ func synthesizeFrom(start: Int, playWhenReady: Bool, atOffset: Double = 0.0) { if let synthesizer = self.synthesizer, let items = self.synthesizer?.createPlayerItems(from: start) { + let prefetchQueue = OperationQueue() + prefetchQueue.maxConcurrentOperationCount = 5 + for speechItem in items { let isLast = speechItem.audioIdx == synthesizer.document.utterances.count - 1 - let playerItem = SpeechPlayerItem(session: self, speechItem: speechItem) { + let playerItem = SpeechPlayerItem(session: self, prefetchQueue: prefetchQueue, speechItem: speechItem) { if isLast { self.player?.pause() self.state = .reachedEnd diff --git a/apple/OmnivoreKit/Sources/Services/AudioSession/PrefetchSpeechItemOperation.swift b/apple/OmnivoreKit/Sources/Services/AudioSession/PrefetchSpeechItemOperation.swift new file mode 100644 index 000000000..85351c09a --- /dev/null +++ b/apple/OmnivoreKit/Sources/Services/AudioSession/PrefetchSpeechItemOperation.swift @@ -0,0 +1,77 @@ +// +// PrefetchSpeechItemOperation.swift +// +// +// Created by Jackson Harper on 11/9/22. +// + +import Foundation +import Models +import Utils +import Views + +final class PrefetchSpeechItemOperation: Operation, URLSessionDelegate { + let speechItem: SpeechItem + let session: URLSession + + enum State: Int { + case created + case started + case finished + } + + init(speechItem: SpeechItem) { + self.speechItem = speechItem + self.state = .created + + let configuration = URLSessionConfiguration.default + configuration.requestCachePolicy = .reloadIgnoringLocalAndRemoteCacheData + self.session = URLSession(configuration: configuration) + } + + public var state: State = .created { + willSet { + willChangeValue(forKey: "isReady") + willChangeValue(forKey: "isExecuting") + willChangeValue(forKey: "isFinished") + willChangeValue(forKey: "isCancelled") + } + didSet { + didChangeValue(forKey: "isCancelled") + didChangeValue(forKey: "isFinished") + didChangeValue(forKey: "isExecuting") + didChangeValue(forKey: "isReady") + } + } + + override var isAsynchronous: Bool { + true + } + + override var isReady: Bool { + true + } + + override var isExecuting: Bool { + self.state == .started + } + + override var isFinished: Bool { + self.state == .finished + } + + override func start() { + guard !isCancelled else { return } + state = .started + + Task { + _ = try await SpeechSynthesizer.download(speechItem: speechItem, session: session) + state = .finished + } + } + + override func cancel() { + session.invalidateAndCancel() + super.cancel() + } +} diff --git a/apple/OmnivoreKit/Sources/Services/AudioSession/SpeechPlayerItem.swift b/apple/OmnivoreKit/Sources/Services/AudioSession/SpeechPlayerItem.swift new file mode 100644 index 000000000..954fed78e --- /dev/null +++ b/apple/OmnivoreKit/Sources/Services/AudioSession/SpeechPlayerItem.swift @@ -0,0 +1,208 @@ +// +// SpeechPlayerItem.swift +// +// +// Created by Jackson Harper on 11/9/22. +// + +import AVFoundation +import Foundation + +import Models + +// Somewhat based on: https://github.com/neekeetab/CachingPlayerItem/blob/master/CachingPlayerItem.swift +class SpeechPlayerItem: AVPlayerItem { + let resourceLoaderDelegate = ResourceLoaderDelegate() + let session: AudioController + let speechItem: SpeechItem + var speechMarks: [SpeechMark]? + var prefetchOperation: PrefetchSpeechItemOperation? + + let completed: () -> Void + + var observer: Any? + + init(session: AudioController, prefetchQueue: OperationQueue, speechItem: SpeechItem, completed: @escaping () -> Void) { + self.speechItem = speechItem + self.session = session + self.completed = completed + + guard let fakeUrl = URL(string: "app.omnivore.speech://\(speechItem.localAudioURL.path).mp3") else { + fatalError("internal inconsistency") + } + + let asset = AVURLAsset(url: fakeUrl) + asset.resourceLoader.setDelegate(resourceLoaderDelegate, queue: DispatchQueue.main) + + super.init(asset: asset, automaticallyLoadedAssetKeys: nil) + + resourceLoaderDelegate.owner = self + + self.observer = observe(\.status, options: [.new]) { item, _ in + if item.status == .readyToPlay { + let duration = CMTimeGetSeconds(item.duration) + item.session.updateDuration(forItem: item.speechItem, newDuration: duration) + } + if item.status == .failed { + item.session.stopWithError() + } + } + + NotificationCenter.default.addObserver( + forName: NSNotification.Name.AVPlayerItemDidPlayToEndTime, + object: self, queue: OperationQueue.main + ) { [weak self] _ in + guard let self = self else { return } + self.completed() + } + + self.prefetchOperation = PrefetchSpeechItemOperation(speechItem: speechItem) + if let prefetchOperation = self.prefetchOperation { + prefetchQueue.addOperation(prefetchOperation) + } + } + + deinit { + observer = nil + prefetchOperation?.cancel() + resourceLoaderDelegate.session?.invalidateAndCancel() + } + + open func download() { + if resourceLoaderDelegate.session == nil { + resourceLoaderDelegate.startDataRequest(with: speechItem.urlRequest) + } + } + + @objc func playbackStalledHandler() { + print("playback stalled...") + } + + class ResourceLoaderDelegate: NSObject, AVAssetResourceLoaderDelegate { + var session: URLSession? + var mediaData: Data? + var pendingRequests = Set() + weak var owner: SpeechPlayerItem? + + func resourceLoader(_: AVAssetResourceLoader, + shouldWaitForLoadingOfRequestedResource loadingRequest: AVAssetResourceLoadingRequest) -> Bool + { + if owner == nil { + return true + } + + if session == nil { + guard let initialUrl = owner?.speechItem.urlRequest else { + fatalError("internal inconsistency") + } + + startDataRequest(with: initialUrl) + } + + pendingRequests.insert(loadingRequest) + processPendingRequests() + return true + } + + func startDataRequest(with _: URLRequest) { + let configuration = URLSessionConfiguration.default + configuration.requestCachePolicy = .reloadIgnoringLocalAndRemoteCacheData + session = URLSession(configuration: configuration) + + Task { + guard let speechItem = self.owner?.speechItem else { + // This probably can't happen, but if it does, just returning should + DispatchQueue.main.async { + self.processPlaybackError(error: BasicError.message(messageText: "No speech item found.")) + } + return + } + + do { + let speechData = try await SpeechSynthesizer.download(speechItem: speechItem, session: self.session ?? URLSession.shared) + + DispatchQueue.main.async { + if speechData == nil { + self.session = nil + self.processPlaybackError(error: BasicError.message(messageText: "Unable to download speech data.")) + return + } + + if let owner = self.owner, let speechData = speechData { + owner.speechMarks = speechData.speechMarks + } + self.mediaData = speechData?.audioData + + self.processPendingRequests() + } + } catch URLError.cancelled { + print("cancelled request error being ignored") + } catch { + DispatchQueue.main.async { + self.processPlaybackError(error: error) + } + } + } + } + + func resourceLoader(_: AVAssetResourceLoader, didCancel loadingRequest: AVAssetResourceLoadingRequest) { + pendingRequests.remove(loadingRequest) + } + + func processPendingRequests() { + let requestsFulfilled = Set(pendingRequests.compactMap { + self.fillInContentInformationRequest($0.contentInformationRequest) + if self.haveEnoughDataToFulfillRequest($0.dataRequest!) { + $0.finishLoading() + return $0 + } + return nil + }) + + // remove fulfilled requests from pending requests + _ = requestsFulfilled.map { self.pendingRequests.remove($0) } + } + + func processPlaybackError(error: Error?) { + let requestsFulfilled = Set(pendingRequests.compactMap { + $0.finishLoading(with: error) + return nil + }) + + _ = requestsFulfilled.map { self.pendingRequests.remove($0) } + } + + func fillInContentInformationRequest(_ contentInformationRequest: AVAssetResourceLoadingContentInformationRequest?) { + contentInformationRequest?.contentType = UTType.mp3.identifier + + if let mediaData = mediaData { + contentInformationRequest?.isByteRangeAccessSupported = true + contentInformationRequest?.contentLength = Int64(mediaData.count) + } + } + + func haveEnoughDataToFulfillRequest(_ dataRequest: AVAssetResourceLoadingDataRequest) -> Bool { + let requestedOffset = Int(dataRequest.requestedOffset) + let requestedLength = dataRequest.requestedLength + let currentOffset = Int(dataRequest.currentOffset) + + guard let songDataUnwrapped = mediaData, + songDataUnwrapped.count > currentOffset + else { + // Don't have any data at all for this request. + return false + } + + let bytesToRespond = min(songDataUnwrapped.count - currentOffset, requestedLength) + let range = Range(uncheckedBounds: (currentOffset, currentOffset + bytesToRespond)) + let dataToRespond = songDataUnwrapped.subdata(in: range) + dataRequest.respond(with: dataToRespond) + + return songDataUnwrapped.count >= requestedLength + requestedOffset + } + + deinit { + session?.invalidateAndCancel() + } + } +}