Merge pull request #1480 from omnivore-app/fix/android-webview-highlight-taps

Webview Highlight Taps - Android
This commit is contained in:
Satindar Dhillon
2022-12-05 20:16:43 -08:00
committed by GitHub
16 changed files with 323 additions and 77 deletions

File diff suppressed because one or more lines are too long

View File

@ -13,7 +13,7 @@ import com.pspdfkit.annotations.HighlightAnnotation
data class CreateHighlightParams(
val shortId: String?,
val highlightID: String?,
val id: String?,
val quote: String?,
val patch: String?,
val articleId: String?,
@ -22,13 +22,47 @@ data class CreateHighlightParams(
fun asCreateHighlightInput() = CreateHighlightInput(
annotation = Optional.presentIfNotNull(`annotation`),
articleId = articleId ?: "",
id = highlightID ?: "",
id = id ?: "",
patch = patch ?: "",
quote = quote ?: "",
shortId = shortId ?: ""
)
}
data class MergeHighlightsParams(
val shortId: String?,
val id: String?,
val quote: String?,
val patch: String?,
val articleId: String?,
val prefix: String?,
val suffix: String?,
val overlapHighlightIdList: List<String>?,
val `annotation`: String?
) {
fun asMergeHighlightInput() = MergeHighlightInput(
annotation = Optional.presentIfNotNull(`annotation`),
prefix = Optional.presentIfNotNull(prefix),
articleId = articleId ?: "",
id = id ?: "",
patch = patch ?: "",
quote = quote ?: "",
shortId = shortId ?: "",
overlapHighlightIdList = overlapHighlightIdList ?: listOf()
)
}
data class DeleteHighlightParams(
val highlightId: String?
) {
fun asIdList() = listOf(highlightId ?: "")
}
suspend fun Networker.deleteHighlight(jsonString: String): Boolean {
val input = Gson().fromJson(jsonString, DeleteHighlightParams::class.java).asIdList()
return deleteHighlights(input)
}
suspend fun Networker.deleteHighlights(highlightIDs: List<String>): Boolean {
val statuses: MutableList<Boolean> = mutableListOf()
for (highlightID in highlightIDs) {
@ -40,6 +74,11 @@ suspend fun Networker.deleteHighlights(highlightIDs: List<String>): Boolean {
return !hasFailure
}
suspend fun Networker.mergeWebHighlights(jsonString: String): Boolean {
val input = Gson().fromJson(jsonString, MergeHighlightsParams::class.java).asMergeHighlightInput()
return mergeHighlights(input)
}
suspend fun Networker.mergeHighlights(input: MergeHighlightInput): Boolean {
val result = authenticatedApolloClient().mutation(MergeHighlightMutation(input)).execute()
Log.d("Network", "highlight merge result: $result")

View File

@ -62,7 +62,7 @@ fun HomeViewContent(
LazyColumn(
state = listState,
verticalArrangement = Arrangement.Center,
verticalArrangement = Arrangement.Top,
horizontalAlignment = Alignment.CenterHorizontally,
modifier = modifier
.background(MaterialTheme.colorScheme.background)

View File

@ -9,11 +9,14 @@ import androidx.compose.foundation.layout.*
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material3.*
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.clip
import androidx.compose.ui.focus.FocusRequester
import androidx.compose.ui.focus.focusRequester
import androidx.compose.ui.platform.ComposeView
import androidx.compose.ui.platform.ViewCompositionStrategy
import androidx.compose.ui.unit.dp
@ -52,6 +55,7 @@ fun AnnotationEditView(
onCancel: () -> Unit,
) {
val annotation = remember { mutableStateOf(initialAnnotation) }
val focusRequester = FocusRequester()
Column(
modifier = Modifier
@ -61,39 +65,62 @@ fun AnnotationEditView(
) {
Column(
modifier = Modifier.padding(16.dp),
horizontalAlignment = Alignment.CenterHorizontally,
) {
Text(text = "Note")
Row {
TextButton(
onClick = {
onCancel()
}
) {
Text("Cancel")
}
Spacer(modifier = Modifier.weight(1.0F))
Text(text = "Note")
Spacer(modifier = Modifier.weight(1.0F))
TextButton(
onClick = {
onSave(annotation.value)
}
) {
Text("Save")
}
}
Spacer(modifier = Modifier.height(8.dp))
TextField(
value = annotation.value,
onValueChange = { annotation.value = it }
onValueChange = { annotation.value = it },
modifier = Modifier
.width(IntrinsicSize.Max)
.height(IntrinsicSize.Max)
.weight(1.0F)
.focusRequester(focusRequester)
)
}
Spacer(modifier = Modifier.height(8.dp))
Row(
modifier = Modifier.align(Alignment.End)
) {
Button(
onClick = {
onCancel()
}
) {
Text("Cancel")
LaunchedEffect(Unit) {
focusRequester.requestFocus()
}
Spacer(modifier = Modifier.width(8.dp))
Button(
onClick = {
onSave(annotation.value)
}
) {
Text("Save")
}
// Row {
// Spacer(modifier = Modifier.weight(0.1F))
// TextField(
// value = annotation.value,
// onValueChange = { annotation.value = it },
// modifier = Modifier
// .width(IntrinsicSize.Max)
// .height(IntrinsicSize.Max)
// .weight(1.0F)
// )
// Spacer(modifier = Modifier.weight(0.1F))
// }
}
Spacer(modifier = Modifier.height(16.dp))
}
}

View File

@ -269,7 +269,7 @@ class PDFReaderActivity: AppCompatActivity(), DocumentListener, TextSelectionMan
// TODO: anchor popover at exact position of tap (maybe add an empty view at tap loc and anchor to that?)
val popupMenu = PopupMenu(this, fragment.view, Gravity.CENTER, androidx.appcompat.R.attr.actionOverflowMenuStyle, 0)
popupMenu.menuInflater.inflate(R.menu.highlight_selection_menu, popupMenu.menu)
popupMenu.menuInflater.inflate(R.menu.pdf_highlight_selection_menu, popupMenu.menu)
popupMenu.setOnMenuItemClickListener { item ->
when(item.itemId) {

View File

@ -1,6 +1,8 @@
package app.omnivore.omnivore.ui.reader
import android.annotation.SuppressLint
import android.content.ClipData
import android.content.ClipboardManager
import android.content.Context
import android.graphics.Rect
import android.util.Log
@ -24,9 +26,13 @@ import androidx.compose.ui.input.nestedscroll.nestedScroll
import androidx.compose.ui.platform.LocalDensity
import androidx.compose.ui.unit.dp
import androidx.compose.ui.viewinterop.AndroidView
import androidx.lifecycle.viewmodel.compose.viewModel
import androidx.core.content.ContextCompat.getSystemService
import app.omnivore.omnivore.R
import com.google.gson.Gson
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import org.json.JSONObject
import java.util.*
import kotlin.math.roundToInt
@ -148,6 +154,8 @@ fun WebReader(
Box {
AndroidView(factory = {
OmnivoreWebView(it).apply {
viewModel = webReaderViewModel
layoutParams = ViewGroup.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.MATCH_PARENT
@ -162,12 +170,34 @@ fun WebReader(
}
val javascriptInterface = AndroidWebKitMessenger { actionID, json ->
webReaderViewModel.hasTappedExistingHighlight = false
when (actionID) {
"userTap" -> {
val tapCoordinates = Gson().fromJson(json, TapCoordinates::class.java)
Log.d("wvt", "received tap action: $tapCoordinates")
CoroutineScope(Dispatchers.Main).launch {
webReaderViewModel.lastTapCoordinates = tapCoordinates
actionMode?.finish()
actionMode = null
}
}
"existingHighlightTap" -> {
isExistingHighlightSelected = true
actionTapCoordinates = Gson().fromJson(json, ActionTapCoordinates::class.java)
Log.d("Loggo", "receive existing highlight tap action: $actionTapCoordinates")
startActionMode(null, ActionMode.TYPE_PRIMARY)
val tapCoordinates = Gson().fromJson(json, TapCoordinates::class.java)
Log.d("wv", "receive existing highlight tap action: $tapCoordinates")
CoroutineScope(Dispatchers.Main).launch {
webReaderViewModel.hasTappedExistingHighlight = true
webReaderViewModel.lastTapCoordinates = tapCoordinates
startActionMode(null, ActionMode.TYPE_FLOATING)
}
}
"writeToClipboard" -> {
val quote = Gson().fromJson(json, HighlightQuote::class.java).quote
quote.let { unwrappedQuote ->
val clipboard = context.getSystemService(Context.CLIPBOARD_SERVICE) as ClipboardManager
val clip = ClipData.newPlainText(unwrappedQuote, unwrappedQuote)
clipboard.setPrimaryClip(clip)
}
}
else -> {
webReaderViewModel.handleIncomingWebMessage(actionID, json)
@ -198,15 +228,16 @@ fun WebReader(
}
class OmnivoreWebView(context: Context) : WebView(context) {
var isExistingHighlightSelected = false
var actionTapCoordinates: ActionTapCoordinates? = null
var viewModel: WebReaderViewModel? = null
var actionMode: ActionMode? = null
private val actionModeCallback = object : ActionMode.Callback2() {
// Called when the action mode is created; startActionMode() was called
override fun onCreateActionMode(mode: ActionMode, menu: Menu): Boolean {
if (isExistingHighlightSelected) {
actionMode = mode
if (viewModel?.hasTappedExistingHighlight == true) {
Log.d("wv", "inflating existing highlight menu")
mode.menuInflater.inflate(R.menu.highlight_selection_menu, menu)
isExistingHighlightSelected = false
} else {
mode.menuInflater.inflate(R.menu.text_selection_menu, menu)
}
@ -222,24 +253,48 @@ class OmnivoreWebView(context: Context) : WebView(context) {
// Called when the user selects a contextual menu item
override fun onActionItemClicked(mode: ActionMode, item: MenuItem): Boolean {
return when (item.itemId) {
R.id.annotate -> {
R.id.annotateHighlight -> {
val script = "var event = new Event('annotate');document.dispatchEvent(event);"
evaluateJavascript(script, null)
mode.finish()
evaluateJavascript(script) {
mode.finish()
actionMode = null
}
true
}
R.id.highlight -> {
val script = "var event = new Event('highlight');document.dispatchEvent(event);"
evaluateJavascript(script, null)
clearFocus()
mode.finish()
evaluateJavascript(script) {
clearFocus()
mode.finish()
actionMode = null
}
true
}
R.id.delete -> {
R.id.copyHighlight -> {
val script = "var event = new Event('copyHighlight');document.dispatchEvent(event);"
evaluateJavascript(script) {
clearFocus()
mode.finish()
actionMode = null
}
true
}
R.id.copyTextSelection -> {
val script = "var event = new Event('copyTextSelection');document.dispatchEvent(event);"
evaluateJavascript(script) {
clearFocus()
mode.finish()
actionMode = null
}
true
}
R.id.removeHighlight -> {
val script = "var event = new Event('remove');document.dispatchEvent(event);"
evaluateJavascript(script, null)
clearFocus()
mode.finish()
evaluateJavascript(script) {
clearFocus()
mode.finish()
actionMode = null
}
true
}
else -> {
@ -251,18 +306,32 @@ class OmnivoreWebView(context: Context) : WebView(context) {
// Called when the user exits the action mode
override fun onDestroyActionMode(mode: ActionMode) {
Log.d("Loggo", "destroying menu: $mode")
isExistingHighlightSelected = false
actionTapCoordinates = null
Log.d("wv", "destroying menu: $mode")
viewModel?.hasTappedExistingHighlight = false
actionMode = null
}
override fun onGetContentRect(mode: ActionMode?, view: View?, outRect: Rect?) {
Log.d("Loggo", "outRect: $outRect, View: $view")
outRect?.set(left, top, right, bottom)
Log.d("wv", "outRect: $outRect, View: $view")
if (viewModel?.lastTapCoordinates != null) {
val scrollYOffset = viewModel?.scrollState?.value ?: 0
val xValue = viewModel!!.lastTapCoordinates!!.tapX.toInt()
val yValue = viewModel!!.lastTapCoordinates!!.tapY.toInt() + scrollYOffset
val rect = Rect(xValue, yValue, xValue, yValue)
Log.d("wv", "scrollState: $scrollYOffset")
Log.d("wv", "setting rect based on last tapped rect: ${viewModel?.lastTapCoordinates.toString()}")
Log.d("wv", "rect: $rect")
outRect?.set(rect)
} else {
outRect?.set(left, top, right, bottom)
}
}
}
override fun startActionMode(callback: ActionMode.Callback?): ActionMode {
Log.d("wv", "startActionMode:callback called")
return super.startActionMode(actionModeCallback)
}
@ -270,11 +339,12 @@ class OmnivoreWebView(context: Context) : WebView(context) {
originalView: View?,
callback: ActionMode.Callback?
): ActionMode {
Log.d("wv", "startActionMode:originalView:callback called")
return super.startActionModeForChild(originalView, actionModeCallback)
}
override fun startActionMode(callback: ActionMode.Callback?, type: Int): ActionMode {
Log.d("Loggo", "startActionMode:type called")
Log.d("wv", "startActionMode:type called")
return super.startActionMode(actionModeCallback, type)
}
}
@ -286,9 +356,9 @@ class AndroidWebKitMessenger(val messageHandler: (String, String) -> Unit) {
}
}
data class ActionTapCoordinates(
val rectX: Double,
val rectY: Double,
val rectWidth: Double,
val rectHeight: Double,
data class TapCoordinates(
val tapX: Double,
val tapY: Double
)
data class HighlightQuote(val quote: String?)

View File

@ -58,7 +58,7 @@ data class WebReaderContent(
<meta charset="utf-8" />
<meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1.0, minimum-scale=1.0, user-scalable=no' />
<style>
@import url("highlightCssFilePath");
@import url("$highlightCssFilePath");
</style>
</head>
<body>

View File

@ -13,7 +13,6 @@ import com.google.gson.Gson
import dagger.hilt.android.lifecycle.HiltViewModel
import kotlinx.coroutines.launch
import kotlinx.coroutines.runBlocking
import org.json.JSONObject
import java.util.*
import javax.inject.Inject
@ -39,6 +38,9 @@ class WebReaderViewModel @Inject constructor(
val annotationLiveData = MutableLiveData<String?>(null)
val javascriptActionLoopUUIDLiveData = MutableLiveData(lastJavascriptActionLoopUUID)
var hasTappedExistingHighlight = false
var lastTapCoordinates: TapCoordinates? = null
fun loadItem(slug: String) {
viewModelScope.launch {
val articleQueryResult = networker.linkedItem(slug)
@ -67,8 +69,11 @@ class WebReaderViewModel @Inject constructor(
}
}
"deleteHighlight" -> {
// { highlightId }
Log.d("Loggo", "receive delete highlight action: $jsonString")
viewModelScope.launch {
val isHighlightDeletionSynced = networker.deleteHighlight(jsonString)
Log.d("Network", "isHighlightDeletionSynced = $isHighlightDeletionSynced")
}
}
"updateHighlight" -> {
Log.d("Loggo", "receive update highlight action: $jsonString")
@ -90,6 +95,12 @@ class WebReaderViewModel @Inject constructor(
"shareHighlight" -> {
// unimplemented
}
"mergeHighlight" -> {
viewModelScope.launch {
val isHighlightSynced = networker.mergeWebHighlights(jsonString)
Log.d("Network", "isMergedHighlightSynced = $isHighlightSynced")
}
}
else -> {
Log.d("Loggo", "receive unrecognized action of $actionID with json: $jsonString")
}
@ -101,6 +112,8 @@ class WebReaderViewModel @Inject constructor(
annotationLiveData.value = null
scrollState = ScrollState(0)
javascriptDispatchQueue = mutableListOf()
hasTappedExistingHighlight = false
lastTapCoordinates = null
}
fun resetJavascriptDispatchQueue() {

View File

@ -2,14 +2,20 @@
<menu xmlns:app="http://schemas.android.com/apk/res-auto"
xmlns:android="http://schemas.android.com/apk/res/android">
<item
android:id="@+id/delete"
android:id="@+id/removeHighlight"
android:title="@string/pdf_remove_highlight"
app:showAsAction="always">
</item>
<item
android:id="@+id/copyPdfHighlight"
android:title="@string/pdf_highlight_copy"
android:id="@+id/annotateHighlight"
android:title="@string/highlight_note"
app:showAsAction="always">
</item>
<item
android:id="@+id/copyHighlight"
android:title="@string/pdf_highlight_copy"
app:showAsAction="ifRoom">
</item>
</menu>

View File

@ -0,0 +1,15 @@
<?xml version="1.0" encoding="utf-8"?>
<menu xmlns:app="http://schemas.android.com/apk/res-auto"
xmlns:android="http://schemas.android.com/apk/res/android">
<item
android:id="@+id/delete"
android:title="@string/pdf_remove_highlight"
app:showAsAction="always">
</item>
<item
android:id="@+id/copyPdfHighlight"
android:title="@string/pdf_highlight_copy"
app:showAsAction="always">
</item>
</menu>

View File

@ -1,6 +1,12 @@
<?xml version="1.0" encoding="utf-8"?>
<menu xmlns:app="http://schemas.android.com/apk/res-auto"
xmlns:android="http://schemas.android.com/apk/res/android">
<item
android:id="@+id/copyTextSelection"
android:title="@string/copyTextSelection"
app:showAsAction="always">
</item>
<item
android:id="@+id/highlight"
android:title="@string/highlight_menu_action"

View File

@ -5,8 +5,10 @@
<string name="welcome_subtitle">Save articles and read them later in our distraction-free reader.</string>
<string name="highlight_menu_action">Highlight</string>
<string name="copy_menu_action">Copy</string>
<string name="annotate_menu_action">Annotate</string>
<string name="annotate_menu_action">Note</string>
<string name="pdf_remove_highlight">Remove</string>
<string name="pdf_highlight_menu_action">Highlight</string>
<string name="pdf_highlight_copy">Copy</string>
<string name="highlight_note">Note</string>
<string name="copyTextSelection">Copy</string>
</resources>

File diff suppressed because one or more lines are too long

View File

@ -23,6 +23,29 @@ const mutation = async (name, input) => {
name,
JSON.stringify(input)
)
// TODO: handle errors
switch (name) {
case 'createHighlight':
return input
case 'deleteHighlight':
return true
case 'mergeHighlight':
return {
id: input['id'],
shortID: input['shortId'],
quote: input['quote'],
patch: input['patch'],
createdByMe: true,
labels: [],
}
case 'updateHighlight':
return true
case 'articleReadingProgress':
return true
default:
return true
}
}
}

View File

@ -106,7 +106,11 @@ export function HighlightsLayer(props: HighlightsLayerProps): JSX.Element {
const removeHighlightCallback = useCallback(
async (id?: string) => {
const highlightId = id || focusedHighlight?.id
if (!highlightId) return
if (!highlightId) {
console.error('Failed to identify highlight to be removed')
return
}
const didDeleteHighlight =
await props.articleMutations.deleteHighlightMutation(highlightId)
@ -270,6 +274,16 @@ export function HighlightsLayer(props: HighlightsLayerProps): JSX.Element {
return
}
const tapAttributes = {
tapX: event.clientX,
tapY: event.clientY,
}
window?.AndroidWebKitMessenger?.handleIdentifiableMessage(
'userTap',
JSON.stringify(tapAttributes)
)
focusedHighlightMousePos.current = { pageX, pageY }
if ((target as Element).hasAttribute(highlightIdAttribute)) {
@ -281,24 +295,24 @@ export function HighlightsLayer(props: HighlightsLayerProps): JSX.Element {
// FIXME: Apply note preview opening on the note icon click only
if (highlight) {
setFocusedHighlight(highlight)
// In the native app we post a message with the rect of the
// highlight, so the app can display a native menu
const rect = (target as Element).getBoundingClientRect()
const message = {
window?.webkit?.messageHandlers.viewerAction?.postMessage({
actionID: 'showMenu',
rectX: rect.x,
rectY: rect.y,
rectWidth: rect.width,
rectHeight: rect.height,
}
window?.webkit?.messageHandlers.viewerAction?.postMessage({
actionID: 'showMenu',
...message,
})
window?.AndroidWebKitMessenger?.handleIdentifiableMessage(
'existingHighlightTap',
JSON.stringify(message)
JSON.stringify({ ...tapAttributes })
)
setFocusedHighlight(highlight)
}
} else if ((target as Element).hasAttribute(highlightNoteIdAttribute)) {
const id = (target as HTMLSpanElement).getAttribute(
@ -309,7 +323,9 @@ export function HighlightsLayer(props: HighlightsLayerProps): JSX.Element {
highlight: highlight,
highlightModalAction: 'addComment',
})
} else setFocusedHighlight(undefined)
} else {
setFocusedHighlight(undefined)
}
},
[highlights, highlightLocations]
)
@ -454,7 +470,15 @@ export function HighlightsLayer(props: HighlightsLayerProps): JSX.Element {
const copy = async () => {
if (focusedHighlight) {
await navigator.clipboard.writeText(focusedHighlight.quote)
if (window.AndroidWebKitMessenger) {
window.AndroidWebKitMessenger.handleIdentifiableMessage(
'writeToClipboard',
JSON.stringify({ quote: focusedHighlight.quote })
)
} else {
await navigator.clipboard.writeText(focusedHighlight.quote)
}
setFocusedHighlight(undefined)
}
}

View File

@ -16,6 +16,16 @@ export function useSelection(
const handleFinishTouch = useCallback(
async (mouseEvent) => {
const tapAttributes = {
tapX: mouseEvent.clientX,
tapY: mouseEvent.clientY,
}
window?.AndroidWebKitMessenger?.handleIdentifiableMessage(
'userTap',
JSON.stringify(tapAttributes)
)
const result = await makeSelectionRange()
if (!result) {
@ -122,6 +132,15 @@ export function useSelection(
[highlightLocations]
)
const copyTextSelection = useCallback(async () => {
// Send message to Android to paste since we don't have the
// correct permissions to write to clipboard from WebView directly
window.AndroidWebKitMessenger?.handleIdentifiableMessage(
'writeToClipboard',
JSON.stringify({ quote: selectionAttributes?.selection.toString() })
)
}, [selectionAttributes?.selection])
useEffect(() => {
if (disabled) {
return
@ -130,13 +149,15 @@ export function useSelection(
document.addEventListener('mouseup', handleFinishTouch)
document.addEventListener('touchend', handleFinishTouch)
document.addEventListener('contextmenu', handleFinishTouch)
document.addEventListener('copyTextSelection', copyTextSelection)
return () => {
document.removeEventListener('mouseup', handleFinishTouch)
document.removeEventListener('touchend', handleFinishTouch)
document.removeEventListener('contextmenu', handleFinishTouch)
document.removeEventListener('copyTextSelection', copyTextSelection)
}
}, [highlightLocations, handleFinishTouch, disabled])
}, [highlightLocations, handleFinishTouch, disabled, copyTextSelection])
return [selectionAttributes, setSelectionAttributes]
}