Separate out the speechplayeritem class, implement a queued prefetch
This commit is contained in:
Jackson Harper
@ -27,196 +27,6 @@
|
||||
case high
|
||||
}
|
||||
|
||||
// Somewhat based on: https://github.com/neekeetab/CachingPlayerItem/blob/master/CachingPlayerItem.swift
|
||||
class SpeechPlayerItem: AVPlayerItem {
|
||||
let resourceLoaderDelegate = ResourceLoaderDelegate()
|
||||
let session: AudioController
|
||||
let speechItem: SpeechItem
|
||||
var speechMarks: [SpeechMark]?
|
||||
|
||||
let completed: () -> Void
|
||||
|
||||
var observer: Any?
|
||||
|
||||
init(session: AudioController, speechItem: SpeechItem, completed: @escaping () -> Void) {
|
||||
self.speechItem = speechItem
|
||||
self.session = session
|
||||
self.completed = completed
|
||||
|
||||
guard let fakeUrl = URL(string: "app.omnivore.speech://\(speechItem.localAudioURL.path).mp3") else {
|
||||
fatalError("internal inconsistency")
|
||||
}
|
||||
|
||||
let asset = AVURLAsset(url: fakeUrl)
|
||||
asset.resourceLoader.setDelegate(resourceLoaderDelegate, queue: DispatchQueue.main)
|
||||
|
||||
super.init(asset: asset, automaticallyLoadedAssetKeys: nil)
|
||||
|
||||
resourceLoaderDelegate.owner = self
|
||||
|
||||
self.observer = observe(\.status, options: [.new]) { item, _ in
|
||||
if item.status == .readyToPlay {
|
||||
let duration = CMTimeGetSeconds(item.duration)
|
||||
item.session.updateDuration(forItem: item.speechItem, newDuration: duration)
|
||||
}
|
||||
if item.status == .failed {
|
||||
item.session.stopWithError()
|
||||
}
|
||||
}
|
||||
|
||||
NotificationCenter.default.addObserver(
|
||||
forName: NSNotification.Name.AVPlayerItemDidPlayToEndTime,
|
||||
object: self, queue: OperationQueue.main
|
||||
) { [weak self] _ in
|
||||
guard let self = self else { return }
|
||||
self.completed()
|
||||
}
|
||||
}
|
||||
|
||||
deinit {
|
||||
observer = nil
|
||||
resourceLoaderDelegate.session?.invalidateAndCancel()
|
||||
}
|
||||
|
||||
open func download() {
|
||||
if resourceLoaderDelegate.session == nil {
|
||||
resourceLoaderDelegate.startDataRequest(with: speechItem.urlRequest)
|
||||
}
|
||||
}
|
||||
|
||||
@objc func playbackStalledHandler() {
|
||||
print("playback stalled...")
|
||||
}
|
||||
|
||||
class ResourceLoaderDelegate: NSObject, AVAssetResourceLoaderDelegate {
|
||||
var session: URLSession?
|
||||
var mediaData: Data?
|
||||
var pendingRequests = Set<AVAssetResourceLoadingRequest>()
|
||||
weak var owner: SpeechPlayerItem?
|
||||
|
||||
func resourceLoader(_: AVAssetResourceLoader,
|
||||
shouldWaitForLoadingOfRequestedResource loadingRequest: AVAssetResourceLoadingRequest) -> Bool
|
||||
{
|
||||
if owner == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
if session == nil {
|
||||
guard let initialUrl = owner?.speechItem.urlRequest else {
|
||||
fatalError("internal inconsistency")
|
||||
}
|
||||
|
||||
startDataRequest(with: initialUrl)
|
||||
}
|
||||
|
||||
pendingRequests.insert(loadingRequest)
|
||||
processPendingRequests()
|
||||
return true
|
||||
}
|
||||
|
||||
func startDataRequest(with _: URLRequest) {
|
||||
let configuration = URLSessionConfiguration.default
|
||||
configuration.requestCachePolicy = .reloadIgnoringLocalAndRemoteCacheData
|
||||
session = URLSession(configuration: configuration)
|
||||
|
||||
Task {
|
||||
guard let speechItem = self.owner?.speechItem else {
|
||||
// This probably can't happen, but if it does, just returning should
|
||||
DispatchQueue.main.async {
|
||||
self.processPlaybackError(error: BasicError.message(messageText: "No speech item found."))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
do {
|
||||
let speechData = try await SpeechSynthesizer.download(speechItem: speechItem, session: self.session ?? URLSession.shared)
|
||||
|
||||
DispatchQueue.main.async {
|
||||
if speechData == nil {
|
||||
self.session = nil
|
||||
self.processPlaybackError(error: BasicError.message(messageText: "Unable to download speech data."))
|
||||
return
|
||||
}
|
||||
|
||||
if let owner = self.owner, let speechData = speechData {
|
||||
owner.speechMarks = speechData.speechMarks
|
||||
}
|
||||
self.mediaData = speechData?.audioData
|
||||
|
||||
self.processPendingRequests()
|
||||
}
|
||||
} catch URLError.cancelled {
|
||||
print("cancelled request error being ignored")
|
||||
} catch {
|
||||
DispatchQueue.main.async {
|
||||
self.processPlaybackError(error: error)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func resourceLoader(_: AVAssetResourceLoader, didCancel loadingRequest: AVAssetResourceLoadingRequest) {
|
||||
pendingRequests.remove(loadingRequest)
|
||||
}
|
||||
|
||||
func processPendingRequests() {
|
||||
let requestsFulfilled = Set<AVAssetResourceLoadingRequest>(pendingRequests.compactMap {
|
||||
self.fillInContentInformationRequest($0.contentInformationRequest)
|
||||
if self.haveEnoughDataToFulfillRequest($0.dataRequest!) {
|
||||
$0.finishLoading()
|
||||
return $0
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// remove fulfilled requests from pending requests
|
||||
_ = requestsFulfilled.map { self.pendingRequests.remove($0) }
|
||||
}
|
||||
|
||||
func processPlaybackError(error: Error?) {
|
||||
let requestsFulfilled = Set<AVAssetResourceLoadingRequest>(pendingRequests.compactMap {
|
||||
$0.finishLoading(with: error)
|
||||
return nil
|
||||
})
|
||||
|
||||
_ = requestsFulfilled.map { self.pendingRequests.remove($0) }
|
||||
}
|
||||
|
||||
func fillInContentInformationRequest(_ contentInformationRequest: AVAssetResourceLoadingContentInformationRequest?) {
|
||||
contentInformationRequest?.contentType = UTType.mp3.identifier
|
||||
|
||||
if let mediaData = mediaData {
|
||||
contentInformationRequest?.isByteRangeAccessSupported = true
|
||||
contentInformationRequest?.contentLength = Int64(mediaData.count)
|
||||
}
|
||||
}
|
||||
|
||||
func haveEnoughDataToFulfillRequest(_ dataRequest: AVAssetResourceLoadingDataRequest) -> Bool {
|
||||
let requestedOffset = Int(dataRequest.requestedOffset)
|
||||
let requestedLength = dataRequest.requestedLength
|
||||
let currentOffset = Int(dataRequest.currentOffset)
|
||||
|
||||
guard let songDataUnwrapped = mediaData,
|
||||
songDataUnwrapped.count > currentOffset
|
||||
else {
|
||||
// Don't have any data at all for this request.
|
||||
return false
|
||||
}
|
||||
|
||||
let bytesToRespond = min(songDataUnwrapped.count - currentOffset, requestedLength)
|
||||
let range = Range(uncheckedBounds: (currentOffset, currentOffset + bytesToRespond))
|
||||
let dataToRespond = songDataUnwrapped.subdata(in: range)
|
||||
dataRequest.respond(with: dataToRespond)
|
||||
|
||||
return songDataUnwrapped.count >= requestedLength + requestedOffset
|
||||
}
|
||||
|
||||
deinit {
|
||||
session?.invalidateAndCancel()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// swiftlint:disable all
|
||||
public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate {
|
||||
@Published public var state: AudioControllerState = .stopped
|
||||
@ -763,9 +573,12 @@
|
||||
|
||||
func synthesizeFrom(start: Int, playWhenReady: Bool, atOffset: Double = 0.0) {
|
||||
if let synthesizer = self.synthesizer, let items = self.synthesizer?.createPlayerItems(from: start) {
|
||||
let prefetchQueue = OperationQueue()
|
||||
prefetchQueue.maxConcurrentOperationCount = 5
|
||||
|
||||
for speechItem in items {
|
||||
let isLast = speechItem.audioIdx == synthesizer.document.utterances.count - 1
|
||||
let playerItem = SpeechPlayerItem(session: self, speechItem: speechItem) {
|
||||
let playerItem = SpeechPlayerItem(session: self, prefetchQueue: prefetchQueue, speechItem: speechItem) {
|
||||
if isLast {
|
||||
self.player?.pause()
|
||||
self.state = .reachedEnd
|
||||
|
||||
@ -0,0 +1,77 @@
|
||||
//
|
||||
// PrefetchSpeechItemOperation.swift
|
||||
//
|
||||
//
|
||||
// Created by Jackson Harper on 11/9/22.
|
||||
//
|
||||
|
||||
import Foundation
|
||||
import Models
|
||||
import Utils
|
||||
import Views
|
||||
|
||||
final class PrefetchSpeechItemOperation: Operation, URLSessionDelegate {
|
||||
let speechItem: SpeechItem
|
||||
let session: URLSession
|
||||
|
||||
enum State: Int {
|
||||
case created
|
||||
case started
|
||||
case finished
|
||||
}
|
||||
|
||||
init(speechItem: SpeechItem) {
|
||||
self.speechItem = speechItem
|
||||
self.state = .created
|
||||
|
||||
let configuration = URLSessionConfiguration.default
|
||||
configuration.requestCachePolicy = .reloadIgnoringLocalAndRemoteCacheData
|
||||
self.session = URLSession(configuration: configuration)
|
||||
}
|
||||
|
||||
public var state: State = .created {
|
||||
willSet {
|
||||
willChangeValue(forKey: "isReady")
|
||||
willChangeValue(forKey: "isExecuting")
|
||||
willChangeValue(forKey: "isFinished")
|
||||
willChangeValue(forKey: "isCancelled")
|
||||
}
|
||||
didSet {
|
||||
didChangeValue(forKey: "isCancelled")
|
||||
didChangeValue(forKey: "isFinished")
|
||||
didChangeValue(forKey: "isExecuting")
|
||||
didChangeValue(forKey: "isReady")
|
||||
}
|
||||
}
|
||||
|
||||
override var isAsynchronous: Bool {
|
||||
true
|
||||
}
|
||||
|
||||
override var isReady: Bool {
|
||||
true
|
||||
}
|
||||
|
||||
override var isExecuting: Bool {
|
||||
self.state == .started
|
||||
}
|
||||
|
||||
override var isFinished: Bool {
|
||||
self.state == .finished
|
||||
}
|
||||
|
||||
override func start() {
|
||||
guard !isCancelled else { return }
|
||||
state = .started
|
||||
|
||||
Task {
|
||||
_ = try await SpeechSynthesizer.download(speechItem: speechItem, session: session)
|
||||
state = .finished
|
||||
}
|
||||
}
|
||||
|
||||
override func cancel() {
|
||||
session.invalidateAndCancel()
|
||||
super.cancel()
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,208 @@
|
||||
//
|
||||
// SpeechPlayerItem.swift
|
||||
//
|
||||
//
|
||||
// Created by Jackson Harper on 11/9/22.
|
||||
//
|
||||
|
||||
import AVFoundation
|
||||
import Foundation
|
||||
|
||||
import Models
|
||||
|
||||
// Somewhat based on: https://github.com/neekeetab/CachingPlayerItem/blob/master/CachingPlayerItem.swift
|
||||
class SpeechPlayerItem: AVPlayerItem {
|
||||
let resourceLoaderDelegate = ResourceLoaderDelegate()
|
||||
let session: AudioController
|
||||
let speechItem: SpeechItem
|
||||
var speechMarks: [SpeechMark]?
|
||||
var prefetchOperation: PrefetchSpeechItemOperation?
|
||||
|
||||
let completed: () -> Void
|
||||
|
||||
var observer: Any?
|
||||
|
||||
init(session: AudioController, prefetchQueue: OperationQueue, speechItem: SpeechItem, completed: @escaping () -> Void) {
|
||||
self.speechItem = speechItem
|
||||
self.session = session
|
||||
self.completed = completed
|
||||
|
||||
guard let fakeUrl = URL(string: "app.omnivore.speech://\(speechItem.localAudioURL.path).mp3") else {
|
||||
fatalError("internal inconsistency")
|
||||
}
|
||||
|
||||
let asset = AVURLAsset(url: fakeUrl)
|
||||
asset.resourceLoader.setDelegate(resourceLoaderDelegate, queue: DispatchQueue.main)
|
||||
|
||||
super.init(asset: asset, automaticallyLoadedAssetKeys: nil)
|
||||
|
||||
resourceLoaderDelegate.owner = self
|
||||
|
||||
self.observer = observe(\.status, options: [.new]) { item, _ in
|
||||
if item.status == .readyToPlay {
|
||||
let duration = CMTimeGetSeconds(item.duration)
|
||||
item.session.updateDuration(forItem: item.speechItem, newDuration: duration)
|
||||
}
|
||||
if item.status == .failed {
|
||||
item.session.stopWithError()
|
||||
}
|
||||
}
|
||||
|
||||
NotificationCenter.default.addObserver(
|
||||
forName: NSNotification.Name.AVPlayerItemDidPlayToEndTime,
|
||||
object: self, queue: OperationQueue.main
|
||||
) { [weak self] _ in
|
||||
guard let self = self else { return }
|
||||
self.completed()
|
||||
}
|
||||
|
||||
self.prefetchOperation = PrefetchSpeechItemOperation(speechItem: speechItem)
|
||||
if let prefetchOperation = self.prefetchOperation {
|
||||
prefetchQueue.addOperation(prefetchOperation)
|
||||
}
|
||||
}
|
||||
|
||||
deinit {
|
||||
observer = nil
|
||||
prefetchOperation?.cancel()
|
||||
resourceLoaderDelegate.session?.invalidateAndCancel()
|
||||
}
|
||||
|
||||
open func download() {
|
||||
if resourceLoaderDelegate.session == nil {
|
||||
resourceLoaderDelegate.startDataRequest(with: speechItem.urlRequest)
|
||||
}
|
||||
}
|
||||
|
||||
@objc func playbackStalledHandler() {
|
||||
print("playback stalled...")
|
||||
}
|
||||
|
||||
class ResourceLoaderDelegate: NSObject, AVAssetResourceLoaderDelegate {
|
||||
var session: URLSession?
|
||||
var mediaData: Data?
|
||||
var pendingRequests = Set<AVAssetResourceLoadingRequest>()
|
||||
weak var owner: SpeechPlayerItem?
|
||||
|
||||
func resourceLoader(_: AVAssetResourceLoader,
|
||||
shouldWaitForLoadingOfRequestedResource loadingRequest: AVAssetResourceLoadingRequest) -> Bool
|
||||
{
|
||||
if owner == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
if session == nil {
|
||||
guard let initialUrl = owner?.speechItem.urlRequest else {
|
||||
fatalError("internal inconsistency")
|
||||
}
|
||||
|
||||
startDataRequest(with: initialUrl)
|
||||
}
|
||||
|
||||
pendingRequests.insert(loadingRequest)
|
||||
processPendingRequests()
|
||||
return true
|
||||
}
|
||||
|
||||
func startDataRequest(with _: URLRequest) {
|
||||
let configuration = URLSessionConfiguration.default
|
||||
configuration.requestCachePolicy = .reloadIgnoringLocalAndRemoteCacheData
|
||||
session = URLSession(configuration: configuration)
|
||||
|
||||
Task {
|
||||
guard let speechItem = self.owner?.speechItem else {
|
||||
// This probably can't happen, but if it does, just returning should
|
||||
DispatchQueue.main.async {
|
||||
self.processPlaybackError(error: BasicError.message(messageText: "No speech item found."))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
do {
|
||||
let speechData = try await SpeechSynthesizer.download(speechItem: speechItem, session: self.session ?? URLSession.shared)
|
||||
|
||||
DispatchQueue.main.async {
|
||||
if speechData == nil {
|
||||
self.session = nil
|
||||
self.processPlaybackError(error: BasicError.message(messageText: "Unable to download speech data."))
|
||||
return
|
||||
}
|
||||
|
||||
if let owner = self.owner, let speechData = speechData {
|
||||
owner.speechMarks = speechData.speechMarks
|
||||
}
|
||||
self.mediaData = speechData?.audioData
|
||||
|
||||
self.processPendingRequests()
|
||||
}
|
||||
} catch URLError.cancelled {
|
||||
print("cancelled request error being ignored")
|
||||
} catch {
|
||||
DispatchQueue.main.async {
|
||||
self.processPlaybackError(error: error)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func resourceLoader(_: AVAssetResourceLoader, didCancel loadingRequest: AVAssetResourceLoadingRequest) {
|
||||
pendingRequests.remove(loadingRequest)
|
||||
}
|
||||
|
||||
func processPendingRequests() {
|
||||
let requestsFulfilled = Set<AVAssetResourceLoadingRequest>(pendingRequests.compactMap {
|
||||
self.fillInContentInformationRequest($0.contentInformationRequest)
|
||||
if self.haveEnoughDataToFulfillRequest($0.dataRequest!) {
|
||||
$0.finishLoading()
|
||||
return $0
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// remove fulfilled requests from pending requests
|
||||
_ = requestsFulfilled.map { self.pendingRequests.remove($0) }
|
||||
}
|
||||
|
||||
func processPlaybackError(error: Error?) {
|
||||
let requestsFulfilled = Set<AVAssetResourceLoadingRequest>(pendingRequests.compactMap {
|
||||
$0.finishLoading(with: error)
|
||||
return nil
|
||||
})
|
||||
|
||||
_ = requestsFulfilled.map { self.pendingRequests.remove($0) }
|
||||
}
|
||||
|
||||
func fillInContentInformationRequest(_ contentInformationRequest: AVAssetResourceLoadingContentInformationRequest?) {
|
||||
contentInformationRequest?.contentType = UTType.mp3.identifier
|
||||
|
||||
if let mediaData = mediaData {
|
||||
contentInformationRequest?.isByteRangeAccessSupported = true
|
||||
contentInformationRequest?.contentLength = Int64(mediaData.count)
|
||||
}
|
||||
}
|
||||
|
||||
func haveEnoughDataToFulfillRequest(_ dataRequest: AVAssetResourceLoadingDataRequest) -> Bool {
|
||||
let requestedOffset = Int(dataRequest.requestedOffset)
|
||||
let requestedLength = dataRequest.requestedLength
|
||||
let currentOffset = Int(dataRequest.currentOffset)
|
||||
|
||||
guard let songDataUnwrapped = mediaData,
|
||||
songDataUnwrapped.count > currentOffset
|
||||
else {
|
||||
// Don't have any data at all for this request.
|
||||
return false
|
||||
}
|
||||
|
||||
let bytesToRespond = min(songDataUnwrapped.count - currentOffset, requestedLength)
|
||||
let range = Range(uncheckedBounds: (currentOffset, currentOffset + bytesToRespond))
|
||||
let dataToRespond = songDataUnwrapped.subdata(in: range)
|
||||
dataRequest.respond(with: dataToRespond)
|
||||
|
||||
return songDataUnwrapped.count >= requestedLength + requestedOffset
|
||||
}
|
||||
|
||||
deinit {
|
||||
session?.invalidateAndCancel()
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user