From 7f0a95b45428a884be7dc9953d6f9c0ad6adc242 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 22 Apr 2024 15:52:28 -0700 Subject: [PATCH] Add iOS digest views --- .../Sources/App/Views/AI/DigestView.swift | 290 ++++++++++++++ .../App/Views/AI/FullScreenDigestView.swift | 370 +++++++++++++----- .../Views/AudioPlayer/MiniPlayerViewer.swift | 23 +- .../App/Views/Home/HomeFeedViewIOS.swift | 26 +- .../App/Views/Home/HomeFeedViewModel.swift | 13 + .../Sources/App/Views/LibraryTabView.swift | 16 +- .../App/Views/RemoveLibraryItemAction.swift | 2 +- .../App/Views/WebReader/ExplainView.swift | 57 +++ .../Views/WebReader/WebReaderContainer.swift | 23 +- .../CoreDataModel.xcdatamodel/contents | 1 - .../AudioSession/AudioController.swift | 152 ++++--- .../AudioSession/SpeechSynthesizer.swift | 10 +- .../Services/DataService/AI/AITasks.swift | 212 +++++++--- .../Queries/LinkedItemNetworkQuery.swift | 1 + .../DataService/Queries/ViewerFetcher.swift | 10 +- .../Sources/Utils/UserDefaultKeys.swift | 1 + .../Views/Article/OmnivoreWebView.swift | 4 +- .../Sources/Views/Images/Images.swift | 1 + packages/api/src/routers/explain_router.ts | 58 +++ packages/api/src/server.ts | 2 + packages/api/src/services/explain.ts | 52 +++ packages/api/src/services/features.ts | 1 + 22 files changed, 1074 insertions(+), 251 deletions(-) create mode 100644 apple/OmnivoreKit/Sources/App/Views/AI/DigestView.swift create mode 100644 apple/OmnivoreKit/Sources/App/Views/WebReader/ExplainView.swift create mode 100644 packages/api/src/routers/explain_router.ts create mode 100644 packages/api/src/services/explain.ts diff --git a/apple/OmnivoreKit/Sources/App/Views/AI/DigestView.swift b/apple/OmnivoreKit/Sources/App/Views/AI/DigestView.swift new file mode 100644 index 000000000..437bccf77 --- /dev/null +++ b/apple/OmnivoreKit/Sources/App/Views/AI/DigestView.swift @@ -0,0 +1,290 @@ +import SwiftUI +import Models +import Services +import Views + +public class DigestViewModel: ObservableObject { + @Published var isLoading = false + @Published var digest: DigestResult? + + func load(dataService: DataService) async { + isLoading = true + + +// if digest == nil { +// do { +// digest = try await dataService.getLatestDigest(timeoutInterval: 10) +// } catch { +// print("ERROR WITH DIGEST: ", error) +// } +// } + isLoading = false + } +} + +@available(iOS 17.0, *) +@MainActor +struct DigestView: View { + let viewModel: DigestViewModel = DigestViewModel() + let dataService: DataService + + // @State private var currentIndex = 0 + @State private var items: [DigestItem] + // @State private var preloadedItems: [Int: String] = [:] + @Environment(\.dismiss) private var dismiss + + // let itemCount = 10 // Number of items to initially load + // let prefetchCount = 2 // Number of items to prefetch + + public init(dataService: DataService) { + self.dataService = dataService + self.items = [ + DigestItem( + id: "1468AFAA-88sdfsdfC-4546-BE02-EACF385288FC", + site: "CNBC.com", + siteIcon: URL(string: "https://www.cnbc.com/favicon.ico"), + author: "Kif Leswing", + title: "Apple shares just had their best day since last May", + summaryText: "In a significant political turn, the SOTU response faces unexpected collapse, marking a stark contrast to Trump's latest" + + " downturn, alongside an unprecedented surge in Biden's fundraising efforts as of 3/11/24, according to the TDPS Podcast. " + + "The analysis provides insights into the shifting dynamics of political support and the potential implications for future " + + "electoral strategies. ", + keyPointsText: "Key points from the article:", + highlightsText: "Highlights from the article:" + ), + DigestItem( + id: "1468AFAA-8sdfsdffsdf-4546-BE02-EACF385288FC", + site: "CNBC.com", + siteIcon: URL(string: "https://www.cnbc.com/favicon.ico"), + author: "Kif Leswing", + title: "Apple shares just had their best day since last May", + summaryText: "In a significant political turn, the SOTU response faces unexpected collapse, marking a stark contrast to Trump's latest" + + " downturn, alongside an unprecedented surge in Biden's fundraising efforts as of 3/11/24, according to the TDPS Podcast. " + + "The analysis provides insights into the shifting dynamics of political support and the potential implications for future " + + "electoral strategies. ", + keyPointsText: "Key points from the article:", + highlightsText: "Highlights from the article:" + ), + DigestItem( + id: "1468AFAA-882C-asdadfsa85288FC", + site: "CNBC.com", + siteIcon: URL(string: "https://www.cnbc.com/favicon.ico"), + author: "Kif Leswing", + title: "Apple shares just had their best day since last May", + summaryText: "In a significant political turn, the SOTU response faces unexpected collapse, marking a stark contrast to Trump's latest" + + " downturn, alongside an unprecedented surge in Biden's fundraising efforts as of 3/11/24, according to the TDPS Podcast. " + + "The analysis provides insights into the shifting dynamics of political support and the potential implications for future " + + "electoral strategies. ", + keyPointsText: "Key points from the article:", + highlightsText: "Highlights from the article:" + ) + ] + // currentIndex = 0 + // _preloadedItems = [Int:String] + } + + var body: some View { + if viewModel.isLoading { + ProgressView() + } else { + itemBody + .task { + await viewModel.load(dataService: dataService) + } + } + } + + @available(iOS 17.0, *) + var itemBody: some View { + ScrollView(.vertical) { + LazyVStack(spacing: 0) { + ForEach(Array(self.items.enumerated()), id: \.1.id) { idx, item in + PreviewItemView( + viewModel: PreviewItemViewModel(dataService: dataService, item: item, showSwipeHint: idx == 0) + ) + .containerRelativeFrame([.horizontal, .vertical]) + } + RatingView() + .containerRelativeFrame([.horizontal, .vertical]) + } + .scrollTargetLayout() + } + .scrollTargetBehavior(.paging) + .ignoresSafeArea() + } +} + +//@MainActor +//public class PreviewItemViewModel: ObservableObject { +// let dataService: DataService +// @Published var item: DigestItem +// let showSwipeHint: Bool +// +// @Published var isLoading = false +// @Published var resultText: String? +// @Published var promptDisplayText: String? +// +// init(dataService: DataService, item: DigestItem, showSwipeHint: Bool) { +// self.dataService = dataService +// self.item = item +// self.showSwipeHint = showSwipeHint +// } +// +// func loadResult() async { +//// isLoading = true +//// let taskId = try? await dataService.createAITask( +//// extraText: extraText, +//// libraryItemId: item?.id ?? "", +//// promptName: "summarize-001" +//// ) +//// +//// if let taskId = taskId { +//// do { +//// let fetchedText = try await dataService.pollAITask(jobId: taskId, timeoutInterval: 30) +//// resultText = fetchedText +//// } catch { +//// print("ERROR WITH RESULT TEXT: ", error) +//// } +//// } else { +//// print("NO TASK ID: ", taskId) +//// } +//// isLoading = false +// } +//} +// +//@MainActor +//struct PreviewItemView: View { +// @StateObject var viewModel: PreviewItemViewModel +// +// var body: some View { +// VStack(spacing: 10) { +// HStack { +// AsyncImage(url: viewModel.item.siteIcon) { phase in +// if let image = phase.image { +// image +// .resizable() +// .aspectRatio(contentMode: .fill) +// .frame(width: 20, height: 20, alignment: .center) +// } else { +// Color.appButtonBackground +// .frame(width: 20, height: 20, alignment: .center) +// } +// } +// Text(viewModel.item.site) +// .font(Font.system(size: 14)) +// .frame(maxWidth: .infinity, alignment: .topLeading) +// } +// .padding(.top, 10) +// Text(viewModel.item.title) +// // .font(.body) +// // .fontWeight(.semibold) +// .font(Font.system(size: 18, weight: .semibold)) +// .frame(maxWidth: .infinity, alignment: .topLeading) +// +// Text(viewModel.item.author) +// .font(Font.system(size: 14)) +// .foregroundColor(Color(hex: "898989")) +// .frame(maxWidth: .infinity, alignment: .topLeading) +// +// Color(hex: "2A2A2A") +// .frame(height: 1) +// .frame(maxWidth: .infinity, alignment: .center) +// .padding(.vertical, 20) +// +// if viewModel.isLoading { +// ProgressView() +// .task { +// await viewModel.loadResult() +// } +// .frame(maxWidth: .infinity, maxHeight: .infinity) +// } else { +// Text(viewModel.item.summaryText) +// .font(Font.system(size: 16)) +// // .font(.body) +// .lineSpacing(12.0) +// .frame(maxWidth: .infinity, alignment: .topLeading) +// HStack { +// Button(action: {}, label: { +// HStack(alignment: .center) { +// Text("Start listening") +// .font(Font.system(size: 14)) +// .frame(height: 42, alignment: .center) +// Image(systemName: "play.fill") +// .resizable() +// .frame(width: 10, height: 10) +// } +// .padding(.horizontal, 15) +// .background(Color.blue) +// .foregroundColor(.white) +// .cornerRadius(18) +// }) +// Spacer() +// } +// .padding(.top, 20) +// } +// Spacer() +// if viewModel.showSwipeHint { +// VStack { +// Image.doubleChevronUp +// Text("Swipe up for next article") +// .foregroundColor(Color(hex: "898989")) +// } +// .padding(.bottom, 50) +// } +// }.frame(maxWidth: .infinity, maxHeight: .infinity) +// .padding(.top, 100) +// .padding(.horizontal, 15) +// +// } +//} +// +//struct RatingView: View { +// @State private var rating: Int = 0 +// +// var body: some View { +// VStack(spacing: 30) { +// Text("Rate today's digest") +// .font(.title) +// .padding(.vertical, 40) +// Text("I liked the stories picked for today's digest") +// RatingWidget() +// +// Text("The stories were interesting") +// RatingWidget() +// +// Text("The voices sounded good") +// RatingWidget() +// +// Text("I liked the music") +// RatingWidget() +// Spacer() +// }.padding(.top, 60) +// } +//} +// +// +//struct StarView: View { +// var isFilled: Bool +// var body: some View { +// Image(systemName: isFilled ? "star.fill" : "star") +// .foregroundColor(isFilled ? Color.yellow : Color.gray) +// } +//} +// +//struct RatingWidget: View { +// @State private var rating: Int = 0 +// var body: some View { +// HStack { +// ForEach(1...5, id: \.self) { index in +// StarView(isFilled: index <= rating) +// .onTapGesture { +// rating = index +// } +// } +// } +// .padding() +// .background(Color(hex: "313131")) +// .cornerRadius(8) +// // .shadow(radius: 3) +// } +//} diff --git a/apple/OmnivoreKit/Sources/App/Views/AI/FullScreenDigestView.swift b/apple/OmnivoreKit/Sources/App/Views/AI/FullScreenDigestView.swift index d2472344f..7c4e6d929 100644 --- a/apple/OmnivoreKit/Sources/App/Views/AI/FullScreenDigestView.swift +++ b/apple/OmnivoreKit/Sources/App/Views/AI/FullScreenDigestView.swift @@ -1,123 +1,271 @@ import SwiftUI import Models import Services +import Views +import MarkdownUI +import Utils +@MainActor public class FullScreenDigestViewModel: ObservableObject { @Published var isLoading = false @Published var digest: DigestResult? + @AppStorage(UserDefaultKey.lastVisitedDigestId.rawValue) var lastVisitedDigestId = "" - func load(dataService: DataService) async { - isLoading = true - if digest == nil { - do { - digest = try await dataService.getLatestDigest(timeoutInterval: 10) - } catch { - print("ERROR WITH DIGEST: ", error) - } + func load(dataService: DataService, audioController: AudioController) async { + if let digest = dataService.loadStoredDigest() { + self.digest = digest + } else { + isLoading = true } + do { + if let digest = try await dataService.getLatestDigest(timeoutInterval: 10) { + self.digest = digest + lastVisitedDigestId = digest.id + + if let playingDigest = audioController.itemAudioProperties as? DigestAudioItem, playingDigest.digest.id == digest.id { + // Don't think we need to do anything here + } else { + audioController.play(itemAudioProperties: DigestAudioItem(digest: digest)) + audioController.pause() + } + } + } catch { + print("ERROR WITH DIGEST: ", error) + } + isLoading = false } -} -struct DigestAudioItem: AudioItemProperties { - let audioItemType = Models.AudioItemType.digest - - var itemID = "" - - var title = "TITLE" - - var byline: String? = "byline" - - var imageURL: URL? = nil - - var language: String? - - var startIndex: Int = 0 - var startOffset: Double = 0.0 + func refreshDigest(dataService: DataService) async { + do { + try await dataService.refreshDigest() + } catch { + print("ERROR WITH DIGEST: ", error) + } + } } @available(iOS 17.0, *) @MainActor struct FullScreenDigestView: View { - let viewModel: DigestViewModel = DigestViewModel() + @StateObject var viewModel = FullScreenDigestViewModel() let dataService: DataService let audioController: AudioController @Environment(\.dismiss) private var dismiss - let textBody = "In a significant political turn, the SOTU response faces unexpected collapse, " + - "marking a stark contrast to Trump's latest downturn, alongside an unprecedented " + - "surge in Biden's fundraising efforts as of 3/11/24, according to the TDPS Podcast. " + - "The analysis provides insights into the shifting dynamics of political support and " + - "the potential implications for future electoral strategies. Based on the information " + - "you provided, the video seems to discuss a recent event where former President " + - "Donald Trump made a controversial statement that shocked even his own audience. " + - "The video likely covers Trump's response to the State of the Union (SOTU) address " + - "and how it received negative feedback, possibly leading to a decline in his support " + - "or approval ratings. Additionally, it appears that the video touches upon a surge " + - "in fundraising for President Joe Biden's administration around March 11, 2024." - public init(dataService: DataService, audioController: AudioController) { self.dataService = dataService self.audioController = audioController } - var body: some View { - // ZStack(alignment: Alignment(horizontal: .trailing, vertical: .top)) { - Group { - if viewModel.isLoading { - ProgressView() - } else { - itemBody - .task { - await viewModel.load(dataService: dataService) - }.onAppear { - self.audioController.play(itemAudioProperties: DigestAudioItem()) - } - } - } .navigationTitle("Omnivore digest") - .navigationBarTitleDisplayMode(.inline) + var titleBlock: some View { + HStack { + Text("Omnivore Digest") + .font(Font.system(size: 18, weight: .semibold)) + Image.tabDigestSelected + Spacer() + closeButton + } + .padding(.top, 20) + .padding(.horizontal, 20) + } -// HStack(alignment: .top) { -// Spacer() -// closeButton -// } -// .padding(20) - // } + var createdString: String { + if let createdAt = viewModel.digest?.createdAt, + let date = DateFormatter.formatterISO8601.date(from: createdAt) { + let dateFormatter = DateFormatter() + dateFormatter.dateStyle = .medium + dateFormatter.timeStyle = .medium + dateFormatter.locale = Locale(identifier: "en_US") + + return "Created " + dateFormatter.string(from: date) + } + return "" + } + + var body: some View { + VStack { + titleBlock + + Group { + if viewModel.isLoading { + VStack { + Spacer() + ProgressView() + Spacer() + } + } else { + itemBody + } + } + .edgesIgnoringSafeArea(.bottom) + + }.task { + await viewModel.load(dataService: dataService, audioController: audioController) + } } var closeButton: some View { Button(action: { dismiss() }, label: { - ZStack { - Circle() - .foregroundColor(Color.appGrayText) - .frame(width: 36, height: 36) - .opacity(0.1) - - Image(systemName: "xmark") - .font(.appCallout) - .frame(width: 36, height: 36) - } + Text("Close") + .foregroundColor(Color.blue) }) .buttonStyle(.plain) } + func getChapterData(digest: DigestResult) -> [String:(time: String, start: Int, end: Int)] { + let speed = 1.0 + var chapterData: [String:(time: String, start: Int, end: Int)] = [:] + var currentAudioIndex = 0 + var currentWordCount = 0.0 + + for (index, speechFile) in digest.speechFiles.enumerated() { + let chapter = digest.chapters[index] + let duration = currentWordCount / SpeechDocument.averageWPM / speed * 60.0 + + chapterData[chapter.id] = ( + time: formatTimeInterval(duration) ?? "00:00", + start: Int(currentAudioIndex), + end: currentAudioIndex + Int(speechFile.utterances.count) + ) + currentAudioIndex += Int(speechFile.utterances.count) + currentWordCount += chapter.wordCount + } + return chapterData + } + + func formatTimeInterval(_ time: TimeInterval) -> String? { + let componentFormatter = DateComponentsFormatter() + componentFormatter.unitsStyle = .positional + componentFormatter.allowedUnits = time >= 3600 ? [.second, .minute, .hour] : [.second, .minute] + componentFormatter.zeroFormattingBehavior = .pad + return componentFormatter.string(from: time) + } + @available(iOS 17.0, *) var itemBody: some View { VStack { - ScrollView(.vertical) { - VStack(spacing: 20) { - Text("SOTU response collapses, Trump hits new low, Biden fundraising explodes 3/11/24 TDPS Podcast") - .font(.title) - Text(textBody) - .font(.body) + ScrollView { + VStack(alignment: .leading, spacing: 20) { + HStack { + Image.coloredSmallOmnivoreLogo + .resizable() + .frame(width: 20, height: 20) + Text("Omnivore.app") + .font(Font.system(size: 14)) + .foregroundColor(Color.themeLibraryItemSubtle) + Spacer() + } + if let digest = viewModel.digest { + Text(digest.title) + .font(Font.system(size: 17, weight: .semibold)) + .lineSpacing(5) + .lineLimit(3) + Text(createdString) + .font(Font.system(size: 12)) + .foregroundColor(Color(hex: "#898989")) + .lineLimit(1) + Text(digest.description) + .font(Font.system(size: 14)) + .lineSpacing(/*@START_MENU_TOKEN@*/10.0/*@END_MENU_TOKEN@*/) + .foregroundColor(Color.themeLibraryItemSubtle) + .lineLimit(6) + } else { + Text("We're building you a new digest") + .font(Font.system(size: 17, weight: .semibold)) + .lineLimit(3) + ProgressView() + } } - } - // .scrollTargetBehavior(.paging) - // .ignoresSafeArea() - MiniPlayerViewer() + .padding(15) + .background(Color.themeLabelBackground.opacity(0.6)) + .cornerRadius(5) + + if let digest = viewModel.digest { + VStack(alignment: .leading, spacing: 10) { + Text("Chapters") + .font(Font.system(size: 17, weight: .semibold)) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(0) + let chapterData = getChapterData(digest: digest) + ForEach(digest.chapters, id: \.id) { chapter in + if let startTime = chapterData[chapter.id]?.time, let skipIndex = chapterData[chapter.id]?.start { + let currentChapter = audioController.currentAudioIndex >= (chapterData[chapter.id]?.start ?? 0) && + audioController.currentAudioIndex < (chapterData[chapter.id]?.end ?? 0) + ChapterView( + startTime: startTime, + skipIndex: skipIndex, + chapter: chapter + ) + .onTapGesture { + audioController.seek(toIdx: skipIndex) + } + .background( + currentChapter ? Color.themeLabelBackground.opacity(0.6) : Color.clear + ) + .cornerRadius(5) + } + } + } + .padding(.top, 20) + } + + if let digest = viewModel.digest { + Text("Transcript") + .font(Font.system(size: 17, weight: .semibold)) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.top, 20) + + VStack { + Markdown(digest.content) + .foregroundColor(Color.appGrayTextContrast) + } + .padding(15) + .background(Color.themeLabelBackground.opacity(0.6)) + .cornerRadius(5) + } + + Spacer(minLength: 60) + + if viewModel.digest != nil { + Text("Rate today's digest") + .font(Font.system(size: 17, weight: .semibold)) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.bottom, 15) + .padding(.horizontal, 15) + + RatingWidget() + Spacer(minLength: 60) + } + + VStack(alignment: .leading, spacing: 20) { + Text("If you didn't like today's digest or would like another one you can create another one. The process takes a few minutes") + Button(action: { + Task { + await viewModel.refreshDigest(dataService: dataService) + } + }, label: { + Text("Create new digest") + .font(Font.system(size: 13, weight: .medium)) + .padding(.horizontal, 8) + .padding(.vertical, 5) + .tint(Color.blue) + .background(Color.themeLabelBackground) + .cornerRadius(5) + }) + } + .padding(15) + .background(Color.themeLabelBackground.opacity(0.6)) + .cornerRadius(5) + + }.contentMargins(10, for: .scrollContent) + + Spacer() + + MiniPlayerViewer(showStopButton: false) .padding(.top, 10) .padding(.bottom, 40) .background(Color.themeTabBarColor) @@ -128,6 +276,50 @@ struct FullScreenDigestView: View { } } +struct ChapterView: View { + let startTime: String + let skipIndex: Int + let chapter: DigestChapter + + var body: some View { + HStack(spacing: 15) { + if let thumbnail = chapter.thumbnail, let thumbnailURL = URL(string: thumbnail) { + AsyncImage(url: thumbnailURL) { image in + image + .resizable() + .aspectRatio(contentMode: .fill) + .frame(width: 90, height: 50) + .clipped() + } placeholder: { + Rectangle() + .foregroundColor(.gray) + .frame(width: 90, height: 50) + } + .cornerRadius(8) + } else { + Rectangle() + .foregroundColor(.gray) + .frame(width: 90, height: 50) + .cornerRadius(8) + } + VStack(alignment: .leading) { + (Text(startTime) + .foregroundColor(.blue) + .font(.caption) + + + + Text(" - " + chapter.title) + .foregroundColor(.primary) + .font(.caption)) + .lineLimit(2) + } + Spacer() + } + .padding(.leading, 4) + .padding(.vertical, 15) + } +} + @MainActor public class PreviewItemViewModel: ObservableObject { let dataService: DataService @@ -145,24 +337,6 @@ public class PreviewItemViewModel: ObservableObject { } func loadResult() async { -// isLoading = true -// let taskId = try? await dataService.createAITask( -// extraText: extraText, -// libraryItemId: item?.id ?? "", -// promptName: "summarize-001" -// ) -// -// if let taskId = taskId { -// do { -// let fetchedText = try await dataService.pollAITask(jobId: taskId, timeoutInterval: 30) -// resultText = fetchedText -// } catch { -// print("ERROR WITH RESULT TEXT: ", error) -// } -// } else { -// print("NO TASK ID: ", taskId) -// } -// isLoading = false } } @@ -200,7 +374,7 @@ struct PreviewItemView: View { .foregroundColor(Color(hex: "898989")) .frame(maxWidth: .infinity, alignment: .topLeading) - Color(hex: "2A2A2A") + Color.themeLabelBackground .frame(height: 1) .frame(maxWidth: .infinity, alignment: .center) .padding(.vertical, 20) @@ -296,7 +470,7 @@ struct RatingWidget: View { } } .padding() - .background(Color(hex: "313131")) + .background(Color.themeLabelBackground.opacity(0.6)) .cornerRadius(8) // .shadow(radius: 3) } diff --git a/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayerViewer.swift b/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayerViewer.swift index fb0c31b20..c3973c2ac 100644 --- a/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayerViewer.swift +++ b/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayerViewer.swift @@ -8,12 +8,16 @@ import Views public struct MiniPlayerViewer: View { + var showStopButton = true @EnvironmentObject var audioController: AudioController @Environment(\.colorScheme) private var colorScheme: ColorScheme @State var expanded = true var playPauseButtonImage: String { +#if targetEnvironment(simulator) + return "play.circle" +#endif switch audioController.state { case .playing: return "pause.circle" @@ -27,6 +31,17 @@ } var playPauseButtonItem: some View { +#if targetEnvironment(simulator) + return AnyView(Button( + action: {}, + label: { + Image(systemName: playPauseButtonImage) + .resizable(resizingMode: Image.ResizingMode.stretch) + .aspectRatio(contentMode: .fit) + .font(Font.title.weight(.light)) + } + ).buttonStyle(.plain)) +#endif if audioController.playbackError { return AnyView(Color.clear) } @@ -138,9 +153,11 @@ .frame(width: 40, height: 40) .foregroundColor(.themeAudioPlayerGray) } - stopButton - .frame(width: 40, height: 40) - .foregroundColor(.themeAudioPlayerGray) + if showStopButton { + stopButton + .frame(width: 40, height: 40) + .foregroundColor(.themeAudioPlayerGray) + } } .padding(.vertical, 5) .padding(.horizontal, 15) diff --git a/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift b/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift index 3b22ac24e..a9ce0c56f 100644 --- a/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift +++ b/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift @@ -205,7 +205,6 @@ struct AnimatingCellHeight: AnimatableModifier { @State private var selection = Set() @AppStorage("LibraryList::digestEnabled") var digestEnabled = false - @AppStorage("LibraryList::hasCheckedForDigestFeature") var hasCheckedForDigestFeature = false init(viewModel: HomeFeedViewModel, isEditMode: Binding) { _viewModel = ObservedObject(wrappedValue: viewModel) @@ -274,7 +273,11 @@ struct AnimatingCellHeight: AnimatableModifier { .padding(.bottom, 20) .background(Color.themeTabBarColor) .onTapGesture { - showExpandedAudioPlayer = true + if audioController.itemAudioProperties?.audioItemType == .digest { + showLibraryDigest = true + } else { + showExpandedAudioPlayer = true + } } } } @@ -353,11 +356,16 @@ struct AnimatingCellHeight: AnimatableModifier { } .task { do { - if let viewer = try await dataService.fetchViewer() { - digestEnabled = viewer.digestEnabled ?? false - if !hasCheckedForDigestFeature { - hasCheckedForDigestFeature = true - // selectedTab = "digest" + // If the user doesn't have digest enabled, try updating their features + // to see if they have it. + if !digestEnabled { + if let viewer = try await dataService.fetchViewer() { + digestEnabled = viewer.hasFeatureGranted("ai-digest") + } + } + if digestEnabled { + Task { + await viewModel.checkForDigestUpdate(dataService: dataService) } } } catch { @@ -409,10 +417,10 @@ struct AnimatingCellHeight: AnimatableModifier { if isEditMode == .active { Button(action: { isEditMode = .inactive }, label: { Text("Cancel") }) } else { - if #available(iOS 17.0, *) { + if #available(iOS 17.0, *), digestEnabled { Button( action: { showLibraryDigest = true }, - label: { Image.tabDigestSelected } + label: { viewModel.digestIsUnread ? Image.tabDigestSelected : Image.tabDigest } ) .buttonStyle(.plain) .padding(.trailing, 4) diff --git a/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewModel.swift b/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewModel.swift index 815e85a54..35aa4992f 100644 --- a/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewModel.swift +++ b/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewModel.swift @@ -38,6 +38,8 @@ enum LoadingBarStyle { @Published var selectedLabels = [LinkedItemLabel]() @Published var negatedLabels = [LinkedItemLabel]() @Published var appliedSort = LinkedItemSort.newest.rawValue + + @Published var digestIsUnread = false @State var lastMoreFetched: Date? @State var lastFiltersFetched: Date? @@ -47,6 +49,7 @@ enum LoadingBarStyle { @AppStorage(UserDefaultKey.hideFeatureSection.rawValue) var hideFeatureSection = false @AppStorage(UserDefaultKey.stopUsingFollowingPrimer.rawValue) var stopUsingFollowingPrimer = false @AppStorage("LibraryTabView::hideFollowingTab") var hideFollowingTab = false + @AppStorage(UserDefaultKey.lastVisitedDigestId.rawValue) var lastVisitedDigestId = "" @AppStorage(UserDefaultKey.lastSelectedFeaturedItemFilter.rawValue) var featureFilter = FeaturedItemFilter.continueReading.rawValue @@ -395,4 +398,14 @@ enum LoadingBarStyle { isEmptyingTrash = false } } + + func checkForDigestUpdate(dataService: DataService) async { + do { + if let result = try? await dataService.getLatestDigest(timeoutInterval: 2) { + if result.id != lastVisitedDigestId { + digestIsUnread = true + } + } + } + } } diff --git a/apple/OmnivoreKit/Sources/App/Views/LibraryTabView.swift b/apple/OmnivoreKit/Sources/App/Views/LibraryTabView.swift index e98786b5a..5040abcd1 100644 --- a/apple/OmnivoreKit/Sources/App/Views/LibraryTabView.swift +++ b/apple/OmnivoreKit/Sources/App/Views/LibraryTabView.swift @@ -15,6 +15,7 @@ struct LibraryTabView: View { @AppStorage(UserDefaultKey.lastSelectedTabItem.rawValue) var selectedTab = "inbox" @State var isEditMode: EditMode = .inactive + @State var showLibraryDigest = false @State var showExpandedAudioPlayer = false @State var presentPushContainer = true @State var pushLinkRequest: String? @@ -160,7 +161,11 @@ struct LibraryTabView: View { if audioController.itemAudioProperties != nil { MiniPlayerViewer() .onTapGesture { - showExpandedAudioPlayer = true + if audioController.itemAudioProperties?.audioItemType == .digest { + showLibraryDigest = true + } else { + showExpandedAudioPlayer = true + } } .padding(0) Color(hex: "#3D3D3D") @@ -193,6 +198,15 @@ struct LibraryTabView: View { } ) } + .fullScreenCover(isPresented: $showLibraryDigest) { + if #available(iOS 17.0, *) { + NavigationView { + FullScreenDigestView(dataService: dataService, audioController: audioController) + } + } else { + Text("Sorry digest is only available on iOS 17 and above") + } + } .navigationBarHidden(true) .onReceive(NSNotification.performSyncPublisher) { _ in Task { diff --git a/apple/OmnivoreKit/Sources/App/Views/RemoveLibraryItemAction.swift b/apple/OmnivoreKit/Sources/App/Views/RemoveLibraryItemAction.swift index ceaa39cd0..18cc21f3d 100644 --- a/apple/OmnivoreKit/Sources/App/Views/RemoveLibraryItemAction.swift +++ b/apple/OmnivoreKit/Sources/App/Views/RemoveLibraryItemAction.swift @@ -48,7 +48,7 @@ func removeLibraryItemAction(dataService: DataService, objectID: NSManagedObject } func archiveLibraryItemAction(dataService: DataService, objectID: NSManagedObjectID, archived: Bool) { - var localPdf: String? = nil + var localPdf: String? dataService.viewContext.performAndWait { if let item = dataService.viewContext.object(with: objectID) as? Models.LibraryItem { item.isArchived = archived diff --git a/apple/OmnivoreKit/Sources/App/Views/WebReader/ExplainView.swift b/apple/OmnivoreKit/Sources/App/Views/WebReader/ExplainView.swift new file mode 100644 index 000000000..12131ff3b --- /dev/null +++ b/apple/OmnivoreKit/Sources/App/Views/WebReader/ExplainView.swift @@ -0,0 +1,57 @@ +// swiftlint:disable line_length +import Foundation +import Models +import SwiftUI +import Views +import WebKit +import Services + +@MainActor public final class ExplainViewModel: ObservableObject { + @Published var isLoading = true + @Published var explanation = "" + + func load(dataService: DataService, text: String, libraryItemId: String) async { + isLoading = true + + do { + + explanation = try await dataService.explain(text: text, libraryItemId: libraryItemId) + } catch { + print("ERROR: ", error) + explanation = "There was an error generating your explanation" + } + + isLoading = false + } +} + +@MainActor +struct ExplainView: View { + let dataService: DataService + + let text: String + let item: Models.LibraryItem + + @StateObject var viewModel = ExplainViewModel() + + init(dataService: DataService, text: String, item: Models.LibraryItem) { + self.text = text + self.item = item + self.dataService = dataService + } + + var body: some View { + if viewModel.isLoading { + ProgressView() + .task { + await viewModel.load(dataService: dataService, text: text, libraryItemId: item.unwrappedID) + } + } else { + Text(viewModel.explanation) + .font(Font.system(size: 19)) + .lineSpacing(12) + .padding(20) + .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .topLeading) + } + } +} diff --git a/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContainer.swift b/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContainer.swift index 93491b275..9ea666087 100644 --- a/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContainer.swift +++ b/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContainer.swift @@ -27,6 +27,7 @@ struct WebReaderContainerView: View { @State var annotationSaveTransactionID: UUID? @State var showNavBarActionID: UUID? @State var showExpandedAudioPlayer = false + @State var showLibraryDigest = false @State var shareActionID: UUID? @State var annotation = String() @State private var bottomBarOpacity = 0.0 @@ -327,6 +328,9 @@ struct WebReaderContainerView: View { .formSheet(isPresented: $showPreferencesFormsheet, modalSize: CGSize(width: 400, height: 475)) { webPreferencesPopoverView } + .formSheet(isPresented: $showExplainSheet, modalSize: CGSize(width: 400, height: 475)) { + explainView + } #endif #if os(macOS) @@ -372,6 +376,10 @@ struct WebReaderContainerView: View { } #endif + var explainView: some View { + ExplainView(dataService: dataService, text: viewModel.explainText ?? "Nothing to explain", item: item) + } + var body: some View { ZStack { if let articleContent = viewModel.articleContent { @@ -476,6 +484,15 @@ struct WebReaderContainerView: View { showExpandedAudioPlayer = false }) } + .fullScreenCover(isPresented: $showLibraryDigest) { + if #available(iOS 17.0, *) { + NavigationView { + FullScreenDigestView(dataService: dataService, audioController: audioController) + } + } else { + Text("Sorry digest is only available on iOS 17 and above") + } + } #endif .alert(errorAlertMessage ?? LocalText.readerError, isPresented: $showErrorAlertMessage) { Button(LocalText.genericOk, role: .cancel, action: { @@ -621,7 +638,11 @@ struct WebReaderContainerView: View { .padding(.bottom, showBottomBar ? 10 : 40) .background(Color.themeTabBarColor) .onTapGesture { - showExpandedAudioPlayer = true + if audioController.itemAudioProperties?.audioItemType == .digest { + showLibraryDigest = true + } else { + showExpandedAudioPlayer = true + } } } if showBottomBar { diff --git a/apple/OmnivoreKit/Sources/Models/CoreData/CoreDataModel.xcdatamodeld/CoreDataModel.xcdatamodel/contents b/apple/OmnivoreKit/Sources/Models/CoreData/CoreDataModel.xcdatamodeld/CoreDataModel.xcdatamodel/contents index 57cf07b2e..e61ca93a9 100644 --- a/apple/OmnivoreKit/Sources/Models/CoreData/CoreDataModel.xcdatamodeld/CoreDataModel.xcdatamodel/contents +++ b/apple/OmnivoreKit/Sources/Models/CoreData/CoreDataModel.xcdatamodeld/CoreDataModel.xcdatamodel/contents @@ -137,7 +137,6 @@ - diff --git a/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift b/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift index 313265e66..b1c6a55f9 100644 --- a/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift +++ b/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift @@ -1,3 +1,5 @@ +// swiftlint:disable file_length type_body_length + #if os(iOS) import AVFoundation @@ -27,7 +29,33 @@ case high } - // swiftlint:disable all + public struct DigestAudioItem: AudioItemProperties { + public let audioItemType = Models.AudioItemType.digest + public let digest: DigestResult + public let itemID: String + public let title: String + public var byline: String? + public var imageURL: URL? + public var language: String? + public var startIndex: Int = 0 + public var startOffset: Double = 0.0 + + public init(digest: DigestResult) { + self.digest = digest + self.itemID = digest.id + self.title = digest.title + self.startIndex = 0 + self.startOffset = 0 + + self.imageURL = nil + + if let first = digest.speechFiles.first { + self.language = first.language + self.byline = digest.byline + } + } + } + public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate { @Published public var state: AudioControllerState = .stopped @Published public var currentAudioIndex: Int = 0 @@ -285,6 +313,30 @@ scrubState = .reset fireTimer() } + + public func seek(toIdx: Int) { + let before = durationBefore(playerIndex: toIdx) + let remainder = 0.0 + + // if the foundIdx happens to be the current item, we just set the position + if let playerItem = player?.currentItem as? SpeechPlayerItem { + if playerItem.speechItem.audioIdx == toIdx { + playerItem.seek(to: CMTimeMakeWithSeconds(remainder, preferredTimescale: 600), completionHandler: nil) + scrubState = .reset + fireTimer() + return + } + } + + // Move the playback to the found index, we also seek by the remainder amount + // before moving we pause the player so playback doesnt jump to a previous spot + player?.pause() + player?.removeAllItems() + synthesizeFrom(start: toIdx, playWhenReady: state == .playing, atOffset: remainder) + + scrubState = .reset + fireTimer() + } @AppStorage(UserDefaultKey.textToSpeechDefaultLanguage.rawValue) public var defaultLanguage = "en" { didSet { @@ -404,12 +456,12 @@ func updateReadText() { if let item = player?.currentItem as? SpeechPlayerItem, let speechMarks = item.speechMarks { var currentItemOffset = 0 - for i in 0 ..< speechMarks.count { - if speechMarks[i].time ?? 0 < 0 { + for idx in 0 ..< speechMarks.count { + if speechMarks[idx].time ?? 0 < 0 { continue } - if (speechMarks[i].time ?? 0.0) > CMTimeGetSeconds(item.currentTime()) * 1000 { - currentItemOffset = speechMarks[i].start ?? 0 + if (speechMarks[idx].time ?? 0.0) > CMTimeGetSeconds(item.currentTime()) * 1000 { + currentItemOffset = speechMarks[idx].start ?? 0 break } } @@ -621,7 +673,9 @@ durations = synthesizer.estimatedDurations(forSpeed: playbackRate) self.synthesizer = synthesizer +#if !targetEnvironment(simulator) synthesizeFrom(start: index, playWhenReady: true, atOffset: offset) +#endif } func synthesizeFrom(start: Int, playWhenReady: Bool, atOffset: Double = 0.0) { @@ -924,55 +978,39 @@ return nil } + func combineSpeechFiles(from digest: DigestResult) -> ([Utterance], Double) { + let allUtterances = digest.speechFiles.flatMap { $0.utterances } + var updatedUtterances: [Utterance] = [] + var currentWordOffset = 0.0 + + for (index, utterance) in allUtterances.enumerated() { + let newUtterance = Utterance( + idx: String(index + 1), + text: utterance.text, + voice: utterance.voice, + wordOffset: currentWordOffset, + wordCount: utterance.wordCount + ) + updatedUtterances.append(newUtterance) + currentWordOffset += utterance.wordCount + } + + return (updatedUtterances, currentWordOffset) + } + func downloadDigestItemSpeechFile(itemID: String, priority: DownloadPriority) async throws -> SpeechDocument? { - let decoder = JSONDecoder() - let speechFileUrl = URL.om_documentsDirectory.appendingPathComponent("digest").appendingPathComponent("speech-\(currentVoice).json") - - if FileManager.default.fileExists(atPath: speechFileUrl.path) { - let data = try Data(contentsOf: speechFileUrl) - document = try decoder.decode(SpeechDocument.self, from: data) - // If we can't load it from disk we make the API call - if let document = document { - return document - } - } - - let path = "/api/digest/v1/" - guard let url = URL(string: path, relativeTo: dataService.appEnvironment.serverBaseURL) else { - throw BasicError.message(messageText: "Invalid audio URL") - } - - var request = URLRequest(url: url) - request.httpMethod = "GET" - for (header, value) in dataService.networker.defaultHeaders { - request.setValue(value, forHTTPHeaderField: header) - } - - let result: (Data, URLResponse)? = try? await URLSession.shared.data(for: request) - guard let httpResponse = result?.1 as? HTTPURLResponse, 200 ..< 300 ~= httpResponse.statusCode else { - throw BasicError.message(messageText: "audioFetch failed. no response or bad status code.") - } - - guard let data = result?.0 else { - throw BasicError.message(messageText: "audioFetch failed. no data received.") - } - - let str = String(decoding: data, as: UTF8.self) - print("result digest file: ", str) - - do { - let digest = try JSONDecoder().decode(DigestResult.self, from: data) - let directory = URL.om_documentsDirectory.appendingPathComponent("digest") - // do { - try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) - try data.write(to: speechFileUrl) - return digest.speechFile -// } catch { -// print("error writing file", error) -// } - // } - } catch { - print("error with digest file", error) + if let digestItem = itemAudioProperties as? DigestAudioItem, let firstFile = digestItem.digest.speechFiles.first { + let (utterances, wordCount) = combineSpeechFiles(from: digestItem.digest) + + let document = SpeechDocument( + pageId: digestItem.itemID, + wordCount: wordCount, + language: firstFile.language, + defaultVoice: firstFile.defaultVoice, + utterances: utterances + ) + try? FileManager.default.createDirectory(at: document.audioDirectory, withIntermediateDirectories: true) + return document } return nil @@ -983,13 +1021,6 @@ return document } - public func audioPlayerDidFinishPlaying(_ player: AVAudioPlayer, successfully _: Bool) { - if player == self.player { - pause() - player.currentTime = 0 - } - } - func setupNotifications() { NotificationCenter.default.removeObserver(self, name: AVAudioSession.interruptionNotification, object: AVAudioSession.sharedInstance()) NotificationCenter.default.addObserver(self, @@ -1013,7 +1044,6 @@ pause() case .ended: // An interruption ended. Resume playback, if appropriate. - guard let optionsValue = userInfo[AVAudioSessionInterruptionOptionKey] as? UInt else { return } let options = AVAudioSession.InterruptionOptions(rawValue: optionsValue) if options.contains(.shouldResume) { diff --git a/apple/OmnivoreKit/Sources/Services/AudioSession/SpeechSynthesizer.swift b/apple/OmnivoreKit/Sources/Services/AudioSession/SpeechSynthesizer.swift index 2946c0fb0..a48342668 100644 --- a/apple/OmnivoreKit/Sources/Services/AudioSession/SpeechSynthesizer.swift +++ b/apple/OmnivoreKit/Sources/Services/AudioSession/SpeechSynthesizer.swift @@ -20,7 +20,7 @@ struct UtteranceRequest: Codable { let isOpenAIVoice: Bool } -public struct Utterance: Decodable { +public struct Utterance: Codable { public let idx: String public let text: String public let voice: String? @@ -39,10 +39,10 @@ public struct Utterance: Decodable { } } -public struct SpeechDocument: Decodable { - static let averageWPM: Double = 195 +public struct SpeechDocument: Codable { + public static let averageWPM: Double = 195 - public let pageId: String + public let pageId: String? public let wordCount: Double public let language: String public let defaultVoice: String @@ -54,7 +54,7 @@ public struct SpeechDocument: Decodable { } var audioDirectory: URL { - Self.audioDirectory(pageId: pageId) + Self.audioDirectory(pageId: pageId ?? "pageid") } static func audioDirectory(pageId: String) -> URL { diff --git a/apple/OmnivoreKit/Sources/Services/DataService/AI/AITasks.swift b/apple/OmnivoreKit/Sources/Services/DataService/AI/AITasks.swift index 7ea58022c..130aa538e 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/AI/AITasks.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/AI/AITasks.swift @@ -7,17 +7,40 @@ struct AITaskRequest: Decodable { public let requestId: String } -public struct DigestResult: Decodable { +public struct DigestResult: Codable { public let id: String public let title: String + public let byline: String public let content: String + public let description: String public let urlsToAudio: [String] - public let speechFile: SpeechDocument + public let chapters: [DigestChapter] + public let speechFiles: [SpeechDocument] public let jobState: String + public let createdAt: String } -public struct DigestItem: Decodable { +public struct DigestChapter: Codable { + public let title: String + public let id: String + public let url: String + public let wordCount: Double + public let thumbnail: String? + public init(title: String, id: String, url: String, wordCount: Double, thumbnail: String?) { + self.title = title + self.id = id + self.url = url + self.wordCount = wordCount + self.thumbnail = thumbnail + } +} + +public struct RefreshDigestResult: Codable { + public let jobId: String +} + +public struct DigestItem: Codable { public let id: String public let site: String public let siteIcon: URL? @@ -26,7 +49,7 @@ public struct DigestItem: Decodable { public let summaryText: String public let keyPointsText: String public let highlightsText: String - public init(id: String, site: String, siteIcon: URL?, + public init(id: String, site: String, siteIcon: URL?, author: String, title: String, summaryText: String, keyPointsText: String, highlightsText: String) { self.id = id @@ -40,32 +63,53 @@ public struct DigestItem: Decodable { } } +public struct DigestRequest: Codable { + public let schedule: String + public let voices: [String] + public init(schedule: String, voices: [String]) { + self.schedule = schedule + self.voices = voices + } +} + +public struct ExplainRequest: Codable { + public let text: String + public let libraryItemId: String + public init(text: String, libraryItemId: String) { + self.text = text + self.libraryItemId = libraryItemId + } +} + +public struct ExplainResult: Codable { + public let text: String +} + extension DataService { -// public func createAITask(extraText: String?, libraryItemId: String, promptName: String) async throws -> String? { -// let jsonData = try JSONSerialization.data(withJSONObject: [ -// "libraryItemId": libraryItemId, -// "promptName": promptName, -// "extraText": extraText -// ]) -// -// let urlRequest = URLRequest.create( -// baseURL: appEnvironment.serverBaseURL, -// urlPath: "/api/ai-task", -// requestMethod: .post(params: jsonData), -// includeAuthToken: true -// ) -// let resource = ServerResource( -// urlRequest: urlRequest, -// decode: AITaskRequest.decode -// ) -// -// do { -// let taskRequest = try await networker.urlSession.performRequest(resource: resource) -// return taskRequest.requestId -// } catch { -// return nil -// } -// } + public func refreshDigest() async throws { + let encoder = JSONEncoder() + let digestRequest = DigestRequest(schedule: "daily", voices: ["openai-nova"]) + let data = (try? encoder.encode(digestRequest)) ?? Data() + + let urlRequest = URLRequest.create( + baseURL: appEnvironment.serverBaseURL, + urlPath: "/api/digest/v1/", + requestMethod: .post(params: data), + includeAuthToken: true + ) + + let resource = ServerResource( + urlRequest: urlRequest, + decode: RefreshDigestResult.decode + ) + + do { + let digest = try await networker.urlSession.performRequest(resource: resource) + print("GOT RESPONSE: ", digest) + } catch { + print("ERROR FETCHING TASK: ", error) + } + } // Function to poll the status of the AI task with timeout public func getLatestDigest(timeoutInterval: TimeInterval) async throws -> DigestResult? { @@ -76,43 +120,81 @@ extension DataService { if count > 3 { return nil } - do { - // Check if timeout has occurred - if -startTime.timeIntervalSinceNow >= timeoutInterval { - throw NSError(domain: "Timeout Error", code: -1, userInfo: nil) - } - - let urlRequest = URLRequest.create( - baseURL: appEnvironment.serverBaseURL, - urlPath: "/api/digest/v1/", - requestMethod: .get, - includeAuthToken: true - ) - - let resource = ServerResource( - urlRequest: urlRequest, - decode: DigestResult.decode - ) - - do { - let digest = try await networker.urlSession.performRequest(resource: resource) - print("GOT RESPONSE: ", digest) - return digest - } catch { - print("ERROR FETCHING TASK: ", error) -// if let response = error as? ServerError { -// if response != .stillProcessing { -// return nil -// } -// } - } - // Wait for some time before polling again - try? await Task.sleep(nanoseconds: 3_000_000_000) - } catch let error { - throw error + do { + // Check if timeout has occurred + if -startTime.timeIntervalSinceNow >= timeoutInterval { + throw NSError(domain: "Timeout Error", code: -1, userInfo: nil) } + + let urlRequest = URLRequest.create( + baseURL: appEnvironment.serverBaseURL, + urlPath: "/api/digest/v1/", + requestMethod: .get, + includeAuthToken: true + ) + + let resource = ServerResource( + urlRequest: urlRequest, + decode: DigestResult.decode + ) + + do { + let digest = try await networker.urlSession.performRequest(resource: resource) + let oldDigest = loadStoredDigest() + + saveDigest(digest) + + return digest + } catch { + print("ERROR FETCHING TASK: ", error) + } + // Wait for some time before polling again + try? await Task.sleep(nanoseconds: 3_000_000_000) + } catch let error { + throw error + } } } + + public func loadStoredDigest() -> DigestResult? { + let decoder = JSONDecoder() + let localPath = URL.om_cachesDirectory.appendingPathComponent("digest.json") + if let data = try? Data(contentsOf: localPath), + let digest = try? decoder.decode(DigestResult.self, from: data) { + return digest + } + return nil + } + + func saveDigest(_ digest: DigestResult) { + let localPath = URL.om_cachesDirectory.appendingPathComponent("digest.json") + if let data = try? JSONEncoder().encode(digest) { + try? data.write(to: localPath) + } + } + + public func explain(text: String, libraryItemId: String) async throws -> String { + let encoder = JSONEncoder() + let explainRequest = ExplainRequest(text: text, libraryItemId: libraryItemId) + let data = (try? encoder.encode(explainRequest)) ?? Data() + + do { + let urlRequest = URLRequest.create( + baseURL: appEnvironment.serverBaseURL, + urlPath: "/api/explain/", + requestMethod: .post(params: data), + includeAuthToken: true + ) + + let resource = ServerResource( + urlRequest: urlRequest, + decode: ExplainResult.decode + ) + + let response = try await networker.urlSession.performRequest(resource: resource) + return response.text + } catch let error { + throw error + } + } } - - diff --git a/apple/OmnivoreKit/Sources/Services/DataService/Queries/LinkedItemNetworkQuery.swift b/apple/OmnivoreKit/Sources/Services/DataService/Queries/LinkedItemNetworkQuery.swift index 2ed12e99f..f5365ebc1 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/Queries/LinkedItemNetworkQuery.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/Queries/LinkedItemNetworkQuery.swift @@ -168,6 +168,7 @@ extension DataService { try $0.search( after: OptionalArgument(cursor), first: OptionalArgument(limit), + includeContent: OptionalArgument(true), query: OptionalArgument(searchQuery), selection: selection ) diff --git a/apple/OmnivoreKit/Sources/Services/DataService/Queries/ViewerFetcher.swift b/apple/OmnivoreKit/Sources/Services/DataService/Queries/ViewerFetcher.swift index f1251fdc0..c386b991d 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/Queries/ViewerFetcher.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/Queries/ViewerFetcher.swift @@ -18,8 +18,7 @@ public extension DataService { selection: .init { try $0.pictureUrl() } ), intercomHash: try $0.intercomHash(), - digestEnabled: true // (try $0.featureList(selection: featureSelection.list.nullable)? - // .filter { $0.enabled && $0.name == "digest" } ?? []).count > 0 + enabledFeatures: try $0.featureList(selection: featureSelection.list.nullable)?.filter { $0.enabled }.map { $0.name } ) } @@ -67,7 +66,11 @@ public struct ViewerInternal { public let name: String public let profileImageURL: String? public let intercomHash: String? - public let digestEnabled: Bool? + public let enabledFeatures: [String]? // We don't persist these as they can be dynamic + + public func hasFeatureGranted(_ name: String) -> Bool { + return enabledFeatures?.contains(name) ?? false + } func persist(context: NSManagedObjectContext) throws { try context.performAndWait { @@ -76,7 +79,6 @@ public struct ViewerInternal { viewer.username = username viewer.name = name viewer.profileImageURL = profileImageURL - viewer.digestEnabled = digestEnabled ?? false do { try context.save() diff --git a/apple/OmnivoreKit/Sources/Utils/UserDefaultKeys.swift b/apple/OmnivoreKit/Sources/Utils/UserDefaultKeys.swift index ee3758a02..04bfbd1e5 100644 --- a/apple/OmnivoreKit/Sources/Utils/UserDefaultKeys.swift +++ b/apple/OmnivoreKit/Sources/Utils/UserDefaultKeys.swift @@ -38,4 +38,5 @@ public enum UserDefaultKey: String { case openExternalLinksIn case prefersHideStatusBarInReader case visibleShareExtensionTab + case lastVisitedDigestId } diff --git a/apple/OmnivoreKit/Sources/Views/Article/OmnivoreWebView.swift b/apple/OmnivoreKit/Sources/Views/Article/OmnivoreWebView.swift index 05dcfb44d..b3f4efe85 100644 --- a/apple/OmnivoreKit/Sources/Views/Article/OmnivoreWebView.swift +++ b/apple/OmnivoreKit/Sources/Views/Article/OmnivoreWebView.swift @@ -400,8 +400,8 @@ public final class OmnivoreWebView: WKWebView { return } let highlight = UICommand(title: LocalText.genericHighlight, action: #selector(highlightSelection)) - // let explain = UICommand(title: "Explain", action: #selector(explainSelection)) - items = [highlight, /* explain, */ annotate] + let explain = UICommand(title: "Explain", action: #selector(explainSelection)) + items = [highlight, explain, annotate] } else { let remove = UICommand(title: "Remove", action: #selector(removeSelection)) let setLabels = UICommand(title: LocalText.labelsGeneric, action: #selector(setLabels)) diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.swift b/apple/OmnivoreKit/Sources/Views/Images/Images.swift index 9b684783e..ccfb85f88 100644 --- a/apple/OmnivoreKit/Sources/Views/Images/Images.swift +++ b/apple/OmnivoreKit/Sources/Views/Images/Images.swift @@ -2,6 +2,7 @@ import SwiftUI public extension Image { static var smallOmnivoreLogo: Image { Image("_smallOmnivoreLogo", bundle: .module) } + static var coloredSmallOmnivoreLogo: Image { Image("app-icon", bundle: .module) } static var omnivoreTitleLogo: Image { Image("_omnivoreTitleLogo", bundle: .module) } static var googleIcon: Image { Image("_googleIcon", bundle: .module) } diff --git a/packages/api/src/routers/explain_router.ts b/packages/api/src/routers/explain_router.ts new file mode 100644 index 000000000..9f2aebee1 --- /dev/null +++ b/packages/api/src/routers/explain_router.ts @@ -0,0 +1,58 @@ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +import { htmlToSpeechFile } from '@omnivore/text-to-speech-handler' +import cors from 'cors' +import express from 'express' +import { userRepository } from '../repository/user' +import { getClaimsByToken } from '../utils/auth' +import { corsConfig } from '../utils/corsConfig' +import { getAISummary } from '../services/ai-summaries' +import { explainText } from '../services/explain' +import { FeatureName, findGrantedFeatureByName } from '../services/features' + +export function explainRouter() { + const router = express.Router() + + // Get an indexed summary for an individual library item + router.post('/', cors(corsConfig), async (req, res) => { + const token = req?.cookies?.auth || req?.headers?.authorization + const claims = await getClaimsByToken(token) + if (!claims) { + return res.status(401).send('UNAUTHORIZED') + } + + const { uid } = claims + const user = await userRepository.findById(uid) + if (!user) { + return res.status(400).send('Bad Request') + } + + if (!(await findGrantedFeatureByName(FeatureName.Explain, user.id))) { + return res.status(403).send('Not granted') + } + + const libraryItemId = req.body.libraryItemId + if (!libraryItemId) { + return res.status(400).send('Bad request - no library item id provided') + } + + const text = req.body.text + if (!text) { + return res.status(400).send('Bad request - no idx provided') + } + + try { + const result = await explainText(uid, text, libraryItemId) + + return res.send({ + text: result, + }) + } catch (err) { + console.log('Error: ', err) + } + + return res.status(500).send('Error') + }) + + return router +} diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 58cdafbda..28e3d886e 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -44,6 +44,7 @@ import { analytics } from './utils/analytics' import { corsConfig } from './utils/corsConfig' import { buildLogger, buildLoggerTransport, logger } from './utils/logger' import { apiLimiter, authLimiter } from './utils/rate_limit' +import { explainRouter } from './routers/explain_router' const PORT = process.env.PORT || 4000 @@ -85,6 +86,7 @@ export const createApp = (): Express => { app.use('/api/user', userRouter()) app.use('/api/article', articleRouter()) app.use('/api/ai-summary', aiSummariesRouter()) + app.use('/api/explain', explainRouter()) app.use('/api/text-to-speech', textToSpeechRouter()) app.use('/api/notification', notificationRouter()) app.use('/api/integration', integrationRouter()) diff --git a/packages/api/src/services/explain.ts b/packages/api/src/services/explain.ts new file mode 100644 index 000000000..407f2b230 --- /dev/null +++ b/packages/api/src/services/explain.ts @@ -0,0 +1,52 @@ +import { OpenAI } from '@langchain/openai' +import { PromptTemplate } from '@langchain/core/prompts' +import { authTrx } from '../repository' +import { libraryItemRepository } from '../repository/library_item' +import { htmlToMarkdown } from '../utils/parser' + +export const explainText = async ( + userId: string, + text: string, + libraryItemId: string +): Promise => { + const llm = new OpenAI({ + modelName: 'gpt-4-0125-preview', + configuration: { + apiKey: process.env.OPENAI_API_KEY, + }, + }) + + const libraryItem = await authTrx( + async (tx) => + tx.withRepository(libraryItemRepository).findById(libraryItemId), + undefined, + userId + ) + + if (!libraryItem) { + throw 'No library item found' + } + + const content = htmlToMarkdown(libraryItem.readableContent) + + const contextualTemplate = PromptTemplate.fromTemplate( + `Create a brief, less than 300 character explanation of the provided + term. Use the article text for additional context. + + Term: {text} + + Article text: {content} + ` + ) + + console.log('template: ', contextualTemplate) + + const chain = contextualTemplate.pipe(llm) + const result = await chain.invoke({ + text: text, + content, + }) + console.log('result: ', result) + + return result +} diff --git a/packages/api/src/services/features.ts b/packages/api/src/services/features.ts index 6400c4779..c34a86322 100644 --- a/packages/api/src/services/features.ts +++ b/packages/api/src/services/features.ts @@ -17,6 +17,7 @@ export enum FeatureName { UltraRealisticVoice = 'ultra-realistic-voice', Notion = 'notion', AIDigest = 'ai-digest', + Explain = 'explain', } export const getFeatureName = (name: string): FeatureName | undefined => {