commit 986cd86dfdfc329e514efc4b73f08a52c44c195b equwal <truex@equwal.com> 2026-09-21 22:11:35 -0700 Take live lines from a caption app A caption app makes captions from live audio, for example speech recognition of the sound of a video. It now sends its lines to the panel with the method `line` of the content provider, and `end` when it stops. The panel shows the newest line in place of the subtitle file. A partial line gets an ellipsis, and the next line replaces it. A final line stays as the line before, when the user shows three lines. The panel holds the line while a word is selected, so that the lookup has time. The answer in the key `live` tells the caption app why the panel cannot show a line: no notification access, no overlay permission, or the panel is hidden. A panel that the user closed does not come back for a line. Live lines end ten minutes after the last one, for a caption app that dies without `end`.
README.md | 35 ++++++++++ .../kotlin/space/subread/overlay/MediaListener.kt | 74 ++++++++++++++++++++++ .../kotlin/space/subread/overlay/OverlayView.kt | 3 + .../kotlin/space/subread/overlay/PlayerProvider.kt | 24 ++++++- .../kotlin/space/subread/overlay/core/LiveLines.kt | 38 +++++++++++ .../space/subread/overlay/core/LiveLinesTest.kt | 59 +++++++++++++++++ 6 files changed, 232 insertions(+), 1 deletion(-)
diff --git a/README.md b/README.md index 8146070..e7b8f46 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,41 @@ controls the player. The panel does not need to be on the screen. The [SubRead plugin for KOReader](https://github.com/equwal/subread.koplugin) uses this to turn the pages with the audiobook. +## For caption apps + +An app that makes captions from live audio, for example speech recognition of +the sound of a video, can show its lines on the panel. The user then taps the +words and looks them up, the same as with a subtitle file. The app calls the +same content provider: + +```kotlin +val panel = Uri.parse("content://space.subread.overlay.player") +contentResolver.call(panel, "line", "It was a dark", bundleOf("partial" to true)) +contentResolver.call(panel, "line", "It was a dark night.", null) +contentResolver.call(panel, "end", null, null) +``` + +`line` shows the text. The extra `partial` is true while the sentence goes on: +the panel adds `…` to the line, and the next line replaces it. A final line +(no `partial`) stays as the line before, when the user shows three lines. `end` +gives the panel back to the subtitle file. Without `end`, live lines end ten +minutes after the last one. + +The answer is in the bundle key `live`: `ok`, or the reason the panel cannot +show the line. `no_notification_access` and `no_overlay_permission`: the user +must allow steps 1 and 2. `panel_hidden`: the user must press "Show the +subtitles", or closed the panel with `✕`. The panel does not come back on its +own for a line, so that a close stays a close. + +While a word is selected, the panel holds the line, so that the lookup has +time. The newest line comes when the selection goes. + +From a shell, for a test: + +``` +adb shell content call --uri content://space.subread.overlay.player --method line --arg "It was a dark" --extra partial:b:true +``` + ## Build ``` diff --git a/app/src/main/kotlin/space/subread/overlay/MediaListener.kt b/app/src/main/kotlin/space/subread/overlay/MediaListener.kt index 42914e7..252eb80 100644 --- a/app/src/main/kotlin/space/subread/overlay/MediaListener.kt +++ b/app/src/main/kotlin/space/subread/overlay/MediaListener.kt @@ -12,6 +12,7 @@ import android.view.Gravity import android.view.WindowManager import android.widget.Toast import space.subread.overlay.core.CueIndex +import space.subread.overlay.core.LiveLines import space.subread.overlay.core.Srt import kotlin.concurrent.thread @@ -21,6 +22,9 @@ import kotlin.concurrent.thread * Android gives the media sessions of other apps only to a notification listener. This service * reads no notification: it has no `onNotificationPosted`. The system keeps a listener running, * so the panel needs no foreground service and no notification of its own. + * + * A caption app can send lines through [PlayerProvider]. While it does, the panel shows its + * newest line and not the subtitle file. */ class MediaListener : NotificationListenerService(), OverlayView.Events { @@ -28,6 +32,14 @@ class MediaListener : NotificationListenerService(), OverlayView.Events { private lateinit var store: Store private lateinit var follower: Follower private var panel: OverlayView? = null + private val live = LiveLines() + + /** True while a caption app sends lines. The follower then does not draw on the panel. */ + private var liveOn = false + + /** True when a live line came while a word was selected: the panel shows it after the selection. */ + private var livePending = false + private val liveTimeout = Runnable { endLive() } private val params = WindowManager.LayoutParams( WindowManager.LayoutParams.MATCH_PARENT, WindowManager.LayoutParams.WRAP_CONTENT, @@ -56,6 +68,7 @@ class MediaListener : NotificationListenerService(), OverlayView.Events { instance = null sessions.removeOnActiveSessionsChangedListener(onSessions) follower.stop() + handler.removeCallbacks(liveTimeout) removePanel() } @@ -106,8 +119,62 @@ class MediaListener : NotificationListenerService(), OverlayView.Events { panel = null } + /** + * A line from a caption app, from any thread. Returns [PlayerProvider.LIVE_OK] when the panel + * shows it, else the reason: the panel needs the overlay permission, and the user must have + * it on the screen. A panel that the user closed does not come back for a line. + */ + fun liveLine(text: String, partial: Boolean): String { + if (!Settings.canDrawOverlays(this)) return PlayerProvider.ERROR_NO_OVERLAY + if (!store.shown) return PlayerProvider.ERROR_PANEL_HIDDEN + handler.post { + if (!liveOn) { + liveOn = true + live.clear() + } + live.line(text, partial) + // A caption app that dies without `end` must not leave its last line for ever. + handler.removeCallbacks(liveTimeout) + handler.postDelayed(liveTimeout, LIVE_TIMEOUT_MS) + if (panel == null) showPanel() else showLive() + } + return PlayerProvider.LIVE_OK + } + + /** The caption app stopped: the panel goes back to the subtitle file. From any thread. */ + fun endLive(): String { + handler.post { + if (!liveOn) return@post + liveOn = false + livePending = false + live.clear() + handler.removeCallbacks(liveTimeout) + if (panel != null) reload() + } + return PlayerProvider.LIVE_OK + } + + /** + * Shows the newest live line. While a word is selected, the line waits: a line that changes + * under the finger would take the selection away before the lookup. + */ + private fun showLive() { + val view = panel ?: return + if (view.selection != null) { + livePending = true + return + } + livePending = false + val now = if (live.partial) live.now + " …" else live.now + if (store.linesAround) view.showLine(now, live.before, null) else view.showLine(now) + } + private fun show(at: Int, hasPlayer: Boolean, playing: Boolean) { val view = panel ?: return + if (liveOn) { + showLive() + return + } view.showPlaying(playing) val cues = follower.index.cues val line = cues.getOrNull(at)?.text @@ -134,6 +201,10 @@ class MediaListener : NotificationListenerService(), OverlayView.Events { override fun onClose() = hidePanel() + override fun onSelectionCleared() { + if (liveOn && livePending) showLive() + } + /** The player pauses for the lookup. The play button of the panel starts it again. */ override fun onLookUp(word: String) { follower.pause() @@ -167,6 +238,9 @@ class MediaListener : NotificationListenerService(), OverlayView.Events { } companion object { + /** Live lines end on their own this long after the last one. */ + const val LIVE_TIMEOUT_MS = 10 * 60_000L + /** The listener that the system runs now; null when the user did not allow it. */ @Volatile var instance: MediaListener? = null diff --git a/app/src/main/kotlin/space/subread/overlay/OverlayView.kt b/app/src/main/kotlin/space/subread/overlay/OverlayView.kt index 5c8ec7c..540d811 100644 --- a/app/src/main/kotlin/space/subread/overlay/OverlayView.kt +++ b/app/src/main/kotlin/space/subread/overlay/OverlayView.kt @@ -43,6 +43,8 @@ class OverlayView(context: Context, private val events: Events) : LinearLayout(c fun onTogglePlay() fun onShiftLines(steps: Int) fun onNudge(ms: Long) + /** The selection went away: a tap beside a word, or a new line. */ + fun onSelectionCleared() {} } /** @@ -172,6 +174,7 @@ class OverlayView(context: Context, private val events: Events) : LinearLayout(c } text.text = marked lookUpRow.visibility = if (range == null) View.GONE else View.VISIBLE + if (range == null) events.onSelectionCleared() } /** The index of the character under the point; null when the point is not on a character. */ diff --git a/app/src/main/kotlin/space/subread/overlay/PlayerProvider.kt b/app/src/main/kotlin/space/subread/overlay/PlayerProvider.kt index 5769cfd..bc173d4 100644 --- a/app/src/main/kotlin/space/subread/overlay/PlayerProvider.kt +++ b/app/src/main/kotlin/space/subread/overlay/PlayerProvider.kt @@ -29,8 +29,14 @@ import space.subread.overlay.core.PlayClock * `call` takes the method `play`, `pause` or `seek` (the argument is the position in * milliseconds), and returns the state line in the bundle key `state`. * + * A caption app, one that makes captions from live audio, sends its lines with the method `line` + * (the argument is the text, the extra `partial` is true while the sentence goes on) and `end` + * when it stops. The panel shows the newest line, in place of the subtitle file, until `end`. + * The bundle key `live` answers `ok`, or the reason the panel cannot show the line: + * `no_notification_access`, `no_overlay_permission` or `panel_hidden`. + * * The provider is open to each app. Each app can already send the media keys to the player, so - * this gives no new control over the device. + * this gives no new control over the device. A line is shown, not kept or sent. */ class PlayerProvider : ContentProvider() { @@ -48,6 +54,15 @@ class PlayerProvider : ContentProvider() { ): Cursor = MatrixCursor(arrayOf(COLUMN_STATE)).apply { addRow(arrayOf(stateLine())) } override fun call(method: String, arg: String?, extras: Bundle?): Bundle { + if (method == METHOD_LINE || method == METHOD_END) { + val listener = MediaListener.instance + val answer = when { + listener == null -> ERROR_NO_ACCESS + method == METHOD_END -> listener.endLive() + else -> listener.liveLine(arg.orEmpty(), extras?.getBoolean(EXTRA_PARTIAL) == true) + } + return Bundle().apply { putString(KEY_LIVE, answer) } + } val player = runCatching { player() }.getOrNull() val controls = player?.transportControls when (method) { @@ -89,8 +104,15 @@ class PlayerProvider : ContentProvider() { const val METHOD_PLAY = "play" const val METHOD_PAUSE = "pause" const val METHOD_SEEK = "seek" + const val METHOD_LINE = "line" + const val METHOD_END = "end" + const val EXTRA_PARTIAL = "partial" + const val KEY_LIVE = "live" + const val LIVE_OK = "ok" const val ERROR_NO_ACCESS = "no_notification_access" const val ERROR_NO_PLAYER = "no_player" + const val ERROR_NO_OVERLAY = "no_overlay_permission" + const val ERROR_PANEL_HIDDEN = "panel_hidden" /** The state line for a report of the player, at [nowMs] on the clock of the device. */ fun stateLine(state: PlaybackState, packageName: String, nowMs: Long): String { diff --git a/core/src/main/kotlin/space/subread/overlay/core/LiveLines.kt b/core/src/main/kotlin/space/subread/overlay/core/LiveLines.kt new file mode 100644 index 0000000..669c619 --- /dev/null +++ b/core/src/main/kotlin/space/subread/overlay/core/LiveLines.kt @@ -0,0 +1,38 @@ +package space.subread.overlay.core + +/** + * The lines that a caption app sends while it makes captions from live audio. + * + * A caption app sends a line many times while the speech goes on: a partial line, which grows. + * When the sentence is finished, it sends the line once more as a final line. The panel shows the + * newest line. Only a final line becomes the line before: a partial line goes away when the next + * line comes, because the final line of the same speech takes its place. + */ +class LiveLines { + + /** The newest line. Empty before the first line. */ + var now: String = "" + private set + + /** True when [now] is a partial line: the speech that it comes from goes on. */ + var partial: Boolean = false + private set + + /** The last final line before [now]; null when there is none. */ + var before: String? = null + private set + + /** A new line from the caption app. */ + fun line(text: String, partial: Boolean) { + if (!this.partial && now.isNotEmpty()) before = now + now = text + this.partial = partial + } + + /** Forgets each line: the caption app stopped. */ + fun clear() { + now = "" + partial = false + before = null + } +} diff --git a/core/src/test/kotlin/space/subread/overlay/core/LiveLinesTest.kt b/core/src/test/kotlin/space/subread/overlay/core/LiveLinesTest.kt new file mode 100644 index 0000000..f503506 --- /dev/null +++ b/core/src/test/kotlin/space/subread/overlay/core/LiveLinesTest.kt @@ -0,0 +1,59 @@ +package space.subread.overlay.core + +import io.kotest.property.Arb +import io.kotest.property.arbitrary.boolean +import io.kotest.property.arbitrary.list +import io.kotest.property.arbitrary.pair +import io.kotest.property.arbitrary.stringPattern +import io.kotest.property.checkAll +import kotlinx.coroutines.runBlocking +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Test + +class LiveLinesTest { + + /** Lines as a caption app sends them: the text, and true for a partial line. */ + private val feeds = Arb.list(Arb.pair(Arb.stringPattern("[a-zあ-ん ]{1,12}"), Arb.boolean()), 0..40) + + /** The line before is the last final line that came before the newest line. */ + @Test + fun theLineBeforeIsTheLastFinalLineBeforeTheNewestLine(): Unit = runBlocking { + checkAll(feeds) { feed -> + val lines = LiveLines() + feed.forEach { (text, partial) -> lines.line(text, partial) } + val last = feed.lastOrNull() + assertEquals(last?.first.orEmpty(), lines.now) + assertEquals(last?.second == true, lines.partial) + val finalsBefore = feed.dropLast(1).filter { !it.second } + assertEquals(finalsBefore.lastOrNull()?.first, lines.before) + } + } + + @Test + fun aPartialLineGrowsAndThenItsFinalLineTakesItsPlace() { + val lines = LiveLines() + lines.line("It was", partial = true) + lines.line("It was a dark", partial = true) + assertEquals("It was a dark", lines.now) + assertNull(lines.before) + lines.line("It was a dark night.", partial = false) + assertEquals("It was a dark night.", lines.now) + assertFalse(lines.partial) + assertNull("a partial line is not a line before", lines.before) + lines.line("The rain", partial = true) + assertEquals("It was a dark night.", lines.before) + } + + @Test + fun clearForgetsEachLine() { + val lines = LiveLines() + lines.line("one", partial = false) + lines.line("two", partial = true) + lines.clear() + assertEquals("", lines.now) + assertFalse(lines.partial) + assertNull(lines.before) + } +}