commit 854168767f52598d19a12d3a1f5536cb3f0db5fc equwal <truex@equwal.com> 2026-09-21 22:15:05 -0700 Stream the sound of an Android device to the Subrep engine The app captures the sound of the apps on the device (Android 10 or later) or the microphone, and streams it to subrep serve on the computer. The captions come back to the app, to the share relay of honjimaku.com under the room of this device, and to SubRead Overlay through its line method. The room and its secret are made once and kept, so the share link stays the same across restarts, as in the desktop app.
.gitignore | 7 + README.md | 84 ++++++ app/build.gradle.kts | 69 +++++ app/src/main/AndroidManifest.xml | 42 +++ .../main/kotlin/com/honjimaku/subrep/Capture.kt | 65 +++++ .../kotlin/com/honjimaku/subrep/CaptureService.kt | 218 +++++++++++++++ app/src/main/kotlin/com/honjimaku/subrep/Feed.kt | 130 +++++++++ .../kotlin/com/honjimaku/subrep/MainActivity.kt | 301 +++++++++++++++++++++ .../main/kotlin/com/honjimaku/subrep/Messages.kt | 43 +++ .../main/kotlin/com/honjimaku/subrep/Overlay.kt | 90 ++++++ app/src/main/kotlin/com/honjimaku/subrep/Pcm.kt | 18 ++ app/src/main/kotlin/com/honjimaku/subrep/Room.kt | 40 +++ app/src/main/kotlin/com/honjimaku/subrep/Socket.kt | 93 +++++++ app/src/main/kotlin/com/honjimaku/subrep/Store.kt | 50 ++++ app/src/main/res/drawable/icon.xml | 8 + app/src/main/res/values/strings.xml | 36 +++ .../kotlin/com/honjimaku/subrep/ProtocolTest.kt | 115 ++++++++ build.gradle.kts | 4 + gradle.properties | 4 + gradle/libs.versions.toml | 19 ++ gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 45457 bytes gradle/wrapper/gradle-wrapper.properties | 9 + gradlew | 251 +++++++++++++++++ gradlew.bat | 94 +++++++ settings.gradle.kts | 24 ++ 25 files changed, 1814 insertions(+)
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a4dddc0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.gradle/ +build/ +local.properties +*.iml +.idea/ +.kotlin/ +captures/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..5c537f3 --- /dev/null +++ b/README.md @@ -0,0 +1,84 @@ +# Subrep Streamer + +Live captions of the sound of an Android device, with the +[Subrep](https://honjimaku.com/subrep/) engine on your computer. + +The app captures the sound of the apps on the device (a video, a game, a +browser) or the microphone. It streams the sound to `subrep serve` on your +computer. The captions come back to the app, to your share link on +`honjimaku.com`, and to [SubRead Overlay](https://github.com/equwal/subread-overlay) +over the player, where a tap on a word opens the dictionary. + +Whisper does not run on the phone. The computer does the speech recognition +with the models and the languages of Subrep, and the phone only sends sound. +A phone from 2019 works the same as a new one. + +## Setup + +On the computer, once: + +1. Start the engine for the language of the video: + + ``` + subrep serve --host 0.0.0.0 --port 8794 --lang ja + ``` + +2. Let the phone reach port 8794. On Windows, as administrator: + + ``` + netsh advfirewall firewall add rule name="Subrep engine" dir=in action=allow protocol=TCP localport=8794 + ``` + +3. Find the address of the computer on the network: `ipconfig` on Windows, + `ip addr` on Linux. For example `192.168.0.9`. + +On the phone: + +1. Install the app. Install SubRead Overlay too, and allow its two steps. +2. Enter the engine as `address:port`, for example `192.168.0.9:8794`. +3. Enter the language. +4. Choose the sound: the apps on this device (Android 10 or later), or the + microphone. +5. Press "Start the captions". Android asks once for the capture of the + screen, and once for the microphone. The microphone permission also + covers the sound of the apps. +6. Start the video. The captions come to the app, to the link and to the + overlay. + +The notification has a "Stop" button. + +## What the device cannot capture + +- An app that refuses the capture of its sound: a video app with DRM, and + some music apps. Android then gives silence. Use the microphone. +- Sound from another device over Bluetooth. A phone is a Bluetooth source, + not a speaker, so it cannot receive sound this way without root. + +## The share link + +The link is `https://honjimaku.com/subrep/w/<room>`, the same relay as the +desktop app. It stays the same across restarts. "New link" makes another +one, and the old one stops. + +## Protocol + +The engine protocol is that of `subrep serve`: a JSON hello +(`{"type":"hello","sampleRate":16000,"lang":"ja","source":"android"}`), then +binary frames of 16-bit little-endian mono PCM, and captions back as +`{"type":"partial"|"final"|"clear","text":...}`. + +The relay protocol is that of `subrep.share`: a websocket to +`wss://honjimaku.com/subrep/pub/<room>`, a hello with the room secret, then +the captions as they came from the engine. + +The overlay protocol is the `line` method of the content provider +`content://space.subread.overlay.player`, with the extra `partial`. + +## Build + +``` +./gradlew :app:testDebugUnitTest :app:assembleDebug +``` + +For a release, set `SUBREP_KEYSTORE_FILE`, `SUBREP_KEYSTORE_PASSWORD` and +`SUBREP_KEY_ALIAS` (default `subrep`), then `./gradlew :app:assembleRelease`. diff --git a/app/build.gradle.kts b/app/build.gradle.kts new file mode 100644 index 0000000..114ea67 --- /dev/null +++ b/app/build.gradle.kts @@ -0,0 +1,69 @@ +plugins { + alias(libs.plugins.android.application) +} + +// Release signing comes from the environment, so the key never lives in the repository. +val keystorePath = providers.environmentVariable("SUBREP_KEYSTORE_FILE").orNull +val keystorePassword = providers.environmentVariable("SUBREP_KEYSTORE_PASSWORD").orNull +val keyAlias = providers.environmentVariable("SUBREP_KEY_ALIAS").orNull ?: "subrep" +val signingReady = !keystorePath.isNullOrBlank() && !keystorePassword.isNullOrBlank() && + file(keystorePath).isFile + +android { + namespace = "com.honjimaku.subrep" + compileSdk = 36 + + dependenciesInfo { + includeInApk = false + includeInBundle = false + } + + defaultConfig { + applicationId = "com.honjimaku.subrep" + // The sound of other apps can be captured from Android 10 (API 29). Older devices get the microphone. + minSdk = 26 + targetSdk = 36 + versionCode = 1 + versionName = "0.1.0" + } + + if (signingReady) { + signingConfigs { + create("release") { + storeFile = file(keystorePath!!) + storePassword = keystorePassword + this.keyAlias = keyAlias + keyPassword = keystorePassword + } + } + } + + buildTypes { + debug { applicationIdSuffix = ".debug" } + release { + isMinifyEnabled = false + if (signingReady) signingConfig = signingConfigs.getByName("release") + } + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 + } +} + +dependencies { + implementation(libs.okhttp) + + testImplementation(libs.junit) + testImplementation(libs.kotest.property) + testImplementation(libs.kotlinx.coroutines.core) + testImplementation(libs.json) +} + +tasks.withType<Test> { + testLogging { + events("failed") + exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL + } +} diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml new file mode 100644 index 0000000..713e04a --- /dev/null +++ b/app/src/main/AndroidManifest.xml @@ -0,0 +1,42 @@ +<?xml version="1.0" encoding="utf-8"?> +<manifest xmlns:android="http://schemas.android.com/apk/res/android"> + + <!-- The audio goes to the Subrep engine on the computer, and the captions go to the share relay. --> + <uses-permission android:name="android.permission.INTERNET" /> + <!-- Android asks for this permission for the sound of other apps too, not only for the microphone. --> + <uses-permission android:name="android.permission.RECORD_AUDIO" /> + <uses-permission android:name="android.permission.FOREGROUND_SERVICE" /> + <uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION" /> + <uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" /> + <uses-permission android:name="android.permission.POST_NOTIFICATIONS" /> + + <!-- SubRead Overlay shows the captions over the player. The app must be able to see it. --> + <queries> + <package android:name="space.subread.overlay" /> + </queries> + + <application + android:allowBackup="true" + android:icon="@drawable/icon" + android:label="@string/app_name" + android:supportsRtl="true" + android:theme="@android:style/Theme.DeviceDefault.Light.NoActionBar" + android:usesCleartextTraffic="true"> + + <activity + android:name=".MainActivity" + android:exported="true" + android:launchMode="singleTop"> + <intent-filter> + <action android:name="android.intent.action.MAIN" /> + <category android:name="android.intent.category.LAUNCHER" /> + </intent-filter> + </activity> + + <!-- Captures the sound and keeps the connections while the user is in the player. --> + <service + android:name=".CaptureService" + android:exported="false" + android:foregroundServiceType="mediaProjection|microphone" /> + </application> +</manifest> diff --git a/app/src/main/kotlin/com/honjimaku/subrep/Capture.kt b/app/src/main/kotlin/com/honjimaku/subrep/Capture.kt new file mode 100644 index 0000000..aadfb71 --- /dev/null +++ b/app/src/main/kotlin/com/honjimaku/subrep/Capture.kt @@ -0,0 +1,65 @@ +package com.honjimaku.subrep + +import android.annotation.SuppressLint +import android.annotation.TargetApi +import android.media.AudioAttributes +import android.media.AudioFormat +import android.media.AudioPlaybackCaptureConfiguration +import android.media.AudioRecord +import android.media.MediaRecorder +import android.media.projection.MediaProjection +import android.os.Build + +/** + * Where the sound comes from: the apps on this device, or the microphone. + * + * Both give 16 kHz mono 16-bit PCM, which is what speech recognition uses. The engine takes any + * rate, but a lower rate is less to send. + */ +object Capture { + + const val SAMPLE_RATE = 16_000 + + /** 100 ms of sound: one frame to the engine. */ + const val FRAME_BYTES = SAMPLE_RATE * 2 / 10 + + /** + * The sound of the apps on this device, with the consent of the user in [projection]. + * Android gives the media, game and unknown streams. An app can refuse the capture + * (a video app with DRM, some music apps): then there is silence. + */ + @TargetApi(Build.VERSION_CODES.Q) + @SuppressLint("MissingPermission") // The screen checks RECORD_AUDIO before the service starts. + fun ofApps(projection: MediaProjection): AudioRecord { + val config = AudioPlaybackCaptureConfiguration.Builder(projection) + .addMatchingUsage(AudioAttributes.USAGE_MEDIA) + .addMatchingUsage(AudioAttributes.USAGE_GAME) + .addMatchingUsage(AudioAttributes.USAGE_UNKNOWN) + .build() + return AudioRecord.Builder() + .setAudioFormat(format()) + .setBufferSizeInBytes(bufferBytes()) + .setAudioPlaybackCaptureConfig(config) + .build() + } + + /** The microphone, tuned for speech: for a television or a talk in the room. */ + @SuppressLint("MissingPermission") + fun ofMicrophone(): AudioRecord = AudioRecord.Builder() + .setAudioSource(MediaRecorder.AudioSource.VOICE_RECOGNITION) + .setAudioFormat(format()) + .setBufferSizeInBytes(bufferBytes()) + .build() + + private fun format() = AudioFormat.Builder() + .setEncoding(AudioFormat.ENCODING_PCM_16BIT) + .setSampleRate(SAMPLE_RATE) + .setChannelMask(AudioFormat.CHANNEL_IN_MONO) + .build() + + /** One second of sound, or more when the device needs more. */ + private fun bufferBytes(): Int { + val least = AudioRecord.getMinBufferSize(SAMPLE_RATE, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT) + return maxOf(least, SAMPLE_RATE * 2) + } +} diff --git a/app/src/main/kotlin/com/honjimaku/subrep/CaptureService.kt b/app/src/main/kotlin/com/honjimaku/subrep/CaptureService.kt new file mode 100644 index 0000000..0965502 --- /dev/null +++ b/app/src/main/kotlin/com/honjimaku/subrep/CaptureService.kt @@ -0,0 +1,218 @@ +package com.honjimaku.subrep + +import android.app.Notification +import android.app.NotificationChannel +import android.app.NotificationManager +import android.app.PendingIntent +import android.app.Service +import android.content.Context +import android.content.Intent +import android.content.pm.ServiceInfo +import android.graphics.drawable.Icon +import android.media.AudioRecord +import android.media.projection.MediaProjection +import android.media.projection.MediaProjectionManager +import android.os.Build +import android.os.Handler +import android.os.HandlerThread +import android.os.IBinder +import android.os.Looper +import okhttp3.OkHttpClient +import java.io.FileDescriptor +import java.io.PrintWriter +import java.util.concurrent.TimeUnit +import kotlin.concurrent.thread + +/** + * Captures the sound, streams it to the engine, and gives the captions to the screen, to the + * relay and to SubRead Overlay. A foreground service, so that it lives while the user is in + * the player. + */ +class CaptureService : Service() { + + private lateinit var store: Store + private val http = OkHttpClient.Builder().pingInterval(20, TimeUnit.SECONDS).build() + private val main = Handler(Looper.getMainLooper()) + private val captions = HandlerThread("captions").apply { start() } + private var engine: Socket? = null + private var relay: Socket? = null + private var overlay: OverlayFeed? = null + private var record: AudioRecord? = null + private var projection: MediaProjection? = null + private var pump: Thread? = null + + override fun onBind(intent: Intent?): IBinder? = null + + override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + when (intent?.action) { + ACTION_START -> if (!Feed.running) start(intent) + ACTION_STOP -> stopAll() + } + return START_NOT_STICKY + } + + private fun start(intent: Intent) { + store = Store(this) + val screen = intent.getBooleanExtra(EXTRA_SCREEN, false) + // Android 14 and later accept the projection only from a foreground service of this type. + foreground(screen) + Feed.start(store.share) + + val record = try { + if (screen) Capture.ofApps(projectionOf(intent)) else Capture.ofMicrophone() + } catch (e: Exception) { + Feed.fail("${e.javaClass.simpleName}: ${e.message}") + stopAll() + return + } + record.startRecording() + if (record.recordingState != AudioRecord.RECORDSTATE_RECORDING) { + Feed.fail("The recorder did not start.") + stopAll() + return + } + this.record = record + + val lang = store.lang + engine = Socket(http, "ws://${store.engine}", { Messages.engineHello(Capture.SAMPLE_RATE, lang, "android") }, ::onEngineText) { + Feed.engine(it) + }.also { it.start() } + if (store.share) { + val secret = store.secret + val title = store.title + relay = Socket(http, "${Room.RELAY}/pub/${store.room}", { Messages.relayHello(secret, lang, title) }, ::onRelayText) { + Feed.relay(it) + }.also { it.start() } + } + if (store.overlay && Overlay.installed(this)) overlay = OverlayFeed(this, Handler(captions.looper)) + pump = thread(name = "pcm") { pump(record) } + } + + private fun projectionOf(intent: Intent): MediaProjection { + val code = intent.getIntExtra(EXTRA_RESULT_CODE, 0) + @Suppress("DEPRECATION") + val data = if (Build.VERSION.SDK_INT >= 33) { + intent.getParcelableExtra(EXTRA_RESULT_DATA, Intent::class.java) + } else { + intent.getParcelableExtra(EXTRA_RESULT_DATA) + } ?: throw IllegalStateException("no consent") + val manager = getSystemService(MediaProjectionManager::class.java) + val projection = manager.getMediaProjection(code, data) ?: throw IllegalStateException("no projection") + // Android 14 and later need a callback before the capture starts. The user can stop the + // capture from the status bar; then this service stops too. + projection.registerCallback(object : MediaProjection.Callback() { + override fun onStop() = stopAll() + }, main) + this.projection = projection + return projection + } + + private fun pump(record: AudioRecord) { + val frame = ByteArray(Capture.FRAME_BYTES) + while (!Thread.currentThread().isInterrupted) { + val count = record.read(frame, 0, frame.size, AudioRecord.READ_BLOCKING) + if (count < 0) break + if (count == 0) continue + engine?.send(frame, count) + Feed.level(Pcm.peak(frame, count)) + } + } + + private fun onEngineText(text: String) { + Messages.error(text)?.let { Feed.fail("engine: $it") } + val caption = Messages.caption(text) ?: return + Feed.caption(caption) + relay?.send(text) + overlay?.caption(caption) + } + + private fun onRelayText(text: String) { + Messages.error(text)?.let { Feed.fail("relay: $it") } + } + + private fun stopAll() { + pump?.interrupt() + pump = null + record?.let { + runCatching { it.stop() } + it.release() + } + record = null + projection?.stop() + projection = null + engine?.stop() + engine = null + relay?.stop() + relay = null + overlay?.end() + overlay = null + Feed.stop() + stopForeground(STOP_FOREGROUND_REMOVE) + stopSelf() + } + + override fun onDestroy() { + captions.quitSafely() + super.onDestroy() + } + + /** For `adb shell dumpsys activity service com.honjimaku.subrep/.CaptureService`: the state, for a bug report. */ + override fun dump(fd: FileDescriptor, writer: PrintWriter, args: Array<out String>?) { + writer.println("running=${Feed.running} engine=${Feed.engineUp} relay=${Feed.relayOn}/${Feed.relayUp}") + writer.println("overlay=${Feed.overlayAnswer} peak=${Feed.peak} error=${Feed.error}") + writer.println(Feed.text()) + } + + private fun foreground(screen: Boolean) { + val manager = getSystemService(NotificationManager::class.java) + manager.createNotificationChannel( + NotificationChannel(CHANNEL, getString(R.string.notification_channel), NotificationManager.IMPORTANCE_LOW), + ) + val open = PendingIntent.getActivity( + this, 0, Intent(this, MainActivity::class.java), PendingIntent.FLAG_IMMUTABLE, + ) + val stop = PendingIntent.getService( + this, 1, Intent(this, CaptureService::class.java).setAction(ACTION_STOP), PendingIntent.FLAG_IMMUTABLE, + ) + val icon = Icon.createWithResource(this, R.drawable.icon) + val notification = Notification.Builder(this, CHANNEL) + .setSmallIcon(icon) + .setContentTitle(getString(R.string.notification_title)) + .setContentText(getString(R.string.notification_text)) + .setContentIntent(open) + .setOngoing(true) + .addAction(Notification.Action.Builder(icon, getString(R.string.stop), stop).build()) + .build() + when { + Build.VERSION.SDK_INT >= 30 -> startForeground( + NOTIFICATION_ID, notification, + if (screen) ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION else ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE, + ) + Build.VERSION.SDK_INT >= 29 -> startForeground(NOTIFICATION_ID, notification, ServiceInfo.FOREGROUND_SERVICE_TYPE_MANIFEST) + else -> startForeground(NOTIFICATION_ID, notification) + } + } + + companion object { + const val ACTION_START = "com.honjimaku.subrep.START" + const val ACTION_STOP = "com.honjimaku.subrep.STOP" + const val EXTRA_SCREEN = "screen" + const val EXTRA_RESULT_CODE = "result_code" + const val EXTRA_RESULT_DATA = "result_data" + private const val CHANNEL = "capture" + private const val NOTIFICATION_ID = 1 + + fun start(context: Context, screen: Boolean, resultCode: Int = 0, resultData: Intent? = null) { + val intent = Intent(context, CaptureService::class.java) + .setAction(ACTION_START) + .putExtra(EXTRA_SCREEN, screen) + .putExtra(EXTRA_RESULT_CODE, resultCode) + .putExtra(EXTRA_RESULT_DATA, resultData) + context.startForegroundService(intent) + } + + fun stop(context: Context) { + context.startService(Intent(context, CaptureService::class.java).setAction(ACTION_STOP)) + } + } +} diff --git a/app/src/main/kotlin/com/honjimaku/subrep/Feed.kt b/app/src/main/kotlin/com/honjimaku/subrep/Feed.kt new file mode 100644 index 0000000..39b3f31 --- /dev/null +++ b/app/src/main/kotlin/com/honjimaku/subrep/Feed.kt @@ -0,0 +1,130 @@ +package com.honjimaku.subrep + +import android.os.Handler +import android.os.Looper +import android.os.SystemClock +import java.util.concurrent.CopyOnWriteArraySet + +/** + * What the screen shows: the state of the service and the last captions. The service writes + * it from its threads; the screen reads it on the main thread after each change. + */ +object Feed { + + fun interface Listener { + fun onChange() + } + + private val main = Handler(Looper.getMainLooper()) + private val listeners = CopyOnWriteArraySet<Listener>() + private val lock = Any() + private val finals = ArrayDeque<String>() + private var lastLevelAt = 0L + + @Volatile var running = false + private set + @Volatile var engineUp = false + private set + @Volatile var relayUp = false + private set + @Volatile var relayOn = false + private set + @Volatile var overlayAnswer = "" + private set + @Volatile var error = "" + private set + + /** The loudest sample of the last frame, 0 to 32768. Shows that sound comes in at all. */ + @Volatile var peak = 0 + private set + @Volatile var partial = "" + private set + + fun add(listener: Listener) = listeners.add(listener) + + fun remove(listener: Listener) = listeners.remove(listener) + + fun start(relayOn: Boolean) { + synchronized(lock) { finals.clear() } + partial = "" + error = "" + overlayAnswer = "" + peak = 0 + engineUp = false + relayUp = false + this.relayOn = relayOn + running = true + changed() + } + + fun stop() { + running = false + engineUp = false + relayUp = false + peak = 0 + changed() + } + + fun fail(reason: String) { + error = reason + changed() + } + + fun engine(up: Boolean) { + engineUp = up + changed() + } + + fun relay(up: Boolean) { + relayUp = up + changed() + } + + fun overlay(answer: String) { + if (answer == overlayAnswer) return + overlayAnswer = answer + changed() + } + + /** The level meter of the screen, 0 to 10. */ + fun bars(): Int = (peak * 10 / 32768).coerceIn(0, 10) + + fun level(peak: Int) { + val before = bars() + this.peak = peak + // Frames come ten times a second. The screen redraws only when the meter changes, and + // not more than twice a second: an e-ink screen cannot draw more. + val now = SystemClock.elapsedRealtime() + if (bars() == before || now - lastLevelAt < 500) return + lastLevelAt = now + changed() + } + + fun caption(caption: Caption) { + synchronized(lock) { + when (caption.type) { + "final" -> { + finals.addLast(caption.text) + while (finals.size > LINES) finals.removeFirst() + partial = "" + } + "partial" -> partial = caption.text + "clear" -> { + finals.clear() + partial = "" + } + } + } + changed() + } + + /** The last lines, and the partial line after them. */ + fun text(): String { + val lines = synchronized(lock) { finals.toList() } + return (lines + listOf(partial).filter { it.isNotEmpty() }.map { "$it …" }).joinToString("\n") + } + + private fun changed() = main.post { listeners.forEach { it.onChange() } } + + private const val LINES = 8 +} diff --git a/app/src/main/kotlin/com/honjimaku/subrep/MainActivity.kt b/app/src/main/kotlin/com/honjimaku/subrep/MainActivity.kt new file mode 100644 index 0000000..868fac6 --- /dev/null +++ b/app/src/main/kotlin/com/honjimaku/subrep/MainActivity.kt @@ -0,0 +1,301 @@ +package com.honjimaku.subrep + +import android.Manifest +import android.annotation.SuppressLint +import android.app.Activity +import android.content.ClipData +import android.content.ClipboardManager +import android.content.Intent +import android.content.pm.PackageManager +import android.graphics.Color +import android.graphics.Typeface +import android.media.projection.MediaProjectionManager +import android.net.Uri +import android.os.Build +import android.os.Bundle +import android.text.InputType +import android.view.View +import android.widget.Button +import android.widget.CheckBox +import android.widget.EditText +import android.widget.LinearLayout +import android.widget.RadioButton +import android.widget.RadioGroup +import android.widget.ScrollView +import android.widget.TextView +import android.widget.Toast + +/** + * The one screen of the app: the engine, the language, the sound, the share link, the overlay, + * and the button. The state of the service and the last captions are under the button. + */ +class MainActivity : Activity(), Feed.Listener { + + private lateinit var store: Store + private lateinit var content: LinearLayout + private lateinit var engineField: EditText + private lateinit var langField: EditText + private lateinit var titleField: EditText + private lateinit var screenRadio: RadioButton + private lateinit var micRadio: RadioButton + private lateinit var shareBox: CheckBox + private lateinit var overlayBox: CheckBox + private lateinit var linkView: TextView + private lateinit var mainButton: Button + private lateinit var statusView: TextView + private lateinit var linesView: TextView + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + store = Store(this) + content = LinearLayout(this).apply { + orientation = LinearLayout.VERTICAL + setPadding(dp(20), dp(20), dp(20), dp(20)) + setBackgroundColor(Color.WHITE) + } + setContentView(ScrollView(this).apply { + fitsSystemWindows = true + setBackgroundColor(Color.WHITE) + addView(content) + }) + draw() + } + + override fun onResume() { + super.onResume() + Feed.add(this) + onChange() + } + + override fun onPause() { + Feed.remove(this) + save() + super.onPause() + } + + private fun draw() { + title(getString(R.string.app_name)) + note(getString(R.string.about)) + + step(R.string.step_engine, getString(R.string.step_engine_why)) + engineField = field(store.engine, "192.168.0.9:8794", InputType.TYPE_CLASS_TEXT or InputType.TYPE_TEXT_VARIATION_URI) + + step(R.string.step_lang, getString(R.string.step_lang_why)) + langField = field(store.lang, "ja", InputType.TYPE_CLASS_TEXT) + + step(R.string.step_source, "") + screenRadio = RadioButton(this).apply { + text = getString(R.string.source_screen) + setTextColor(Color.BLACK) + isChecked = store.screen + } + micRadio = RadioButton(this).apply { + text = getString(R.string.source_mic) + setTextColor(Color.BLACK) + isChecked = !store.screen + } + content.addView(RadioGroup(this).apply { + addView(screenRadio) + addView(micRadio) + }, wide()) + + step(R.string.step_share, "") + shareBox = CheckBox(this).apply { + text = getString(R.string.share_on) + setTextColor(Color.BLACK) + isChecked = store.share + } + content.addView(shareBox, wide()) + titleField = field(store.title, getString(R.string.share_title_hint), InputType.TYPE_CLASS_TEXT) + linkView = TextView(this).apply { + setTextColor(Color.BLACK) + textSize = 15f + setTextIsSelectable(true) + } + content.addView(linkView, wide()) + row( + button(getString(R.string.copy)) { copyLink() }, + button(getString(R.string.share)) { shareLink() }, + button(getString(R.string.new_link)) { + store.newRoom() + showLink() + }, + ) + showLink() + + step(R.string.step_overlay, "") + overlayBox = CheckBox(this).apply { + text = getString(R.string.overlay_on) + setTextColor(Color.BLACK) + isChecked = store.overlay + } + if (Overlay.installed(this)) { + content.addView(overlayBox, wide()) + content.addView(button(getString(R.string.open_overlay)) { + Overlay.launch(this)?.let { runCatching { startActivity(it) } } + }, LinearLayout.LayoutParams(-2, -2)) + } else { + note(getString(R.string.overlay_missing)) + content.addView(button(getString(R.string.install_overlay)) { open(Overlay.INSTALL) }, LinearLayout.LayoutParams(-2, -2)) + } + + mainButton = button(getString(R.string.start)) { toggle() }.apply { setTypeface(typeface, Typeface.BOLD) } + content.addView(mainButton, wide(top = 24)) + statusView = TextView(this).apply { + setTextColor(Color.BLACK) + textSize = 15f + } + content.addView(statusView, wide()) + linesView = TextView(this).apply { + setTextColor(Color.BLACK) + textSize = 18f + setTextIsSelectable(true) + } + content.addView(linesView, wide()) + } + + private fun save() { + store.engine = engineField.text.toString() + store.lang = langField.text.toString() + store.title = titleField.text.toString() + store.screen = screenRadio.isChecked + store.share = shareBox.isChecked + store.overlay = overlayBox.isChecked + } + + private fun toggle() { + if (Feed.running) CaptureService.stop(this) else startCapture() + } + + private fun startCapture() { + save() + if (store.engine.isBlank()) return toast(R.string.need_engine) + val missing = buildList { + if (checkSelfPermission(Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) add(Manifest.permission.RECORD_AUDIO) + if (Build.VERSION.SDK_INT >= 33 && checkSelfPermission(Manifest.permission.POST_NOTIFICATIONS) != PackageManager.PERMISSION_GRANTED) { + add(Manifest.permission.POST_NOTIFICATIONS) + } + } + if (missing.isNotEmpty()) return requestPermissions(missing.toTypedArray(), REQUEST_PERMISSIONS) + if (!store.screen) return CaptureService.start(this, screen = false) + if (Build.VERSION.SDK_INT < 29) return toast(R.string.need_android_10) + // Android asks the user for the consent. The answer comes to onActivityResult. + val manager = getSystemService(MediaProjectionManager::class.java) + startActivityForResult(manager.createScreenCaptureIntent(), REQUEST_PROJECTION) + } + + override fun onRequestPermissionsResult(requestCode: Int, permissions: Array<out String>, grantResults: IntArray) { + if (requestCode != REQUEST_PERMISSIONS) return + val audio = permissions.indexOf(Manifest.permission.RECORD_AUDIO) + if (audio >= 0 && grantResults[audio] != PackageManager.PERMISSION_GRANTED) return toast(R.string.need_audio_permission) + startCapture() + } + + @Deprecated("The platform Activity has no other result API, and this app has no AndroidX activity.") + override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) { + if (requestCode != REQUEST_PROJECTION) return + if (resultCode != RESULT_OK || data == null) return toast(R.string.capture_refused) + CaptureService.start(this, screen = true, resultCode = resultCode, resultData = data) + } + + override fun onChange() { + mainButton.text = getString(if (Feed.running) R.string.stop else R.string.start) + statusView.text = status() + linesView.text = Feed.text() + } + + private fun status(): String { + if (!Feed.running) return Feed.error + val engine = if (Feed.engineUp) "connected" else "connecting…" + val relay = when { + !Feed.relayOn -> "off" + Feed.relayUp -> "connected" + else -> "connecting…" + } + val overlay = when (Feed.overlayAnswer) { + "" -> if (store.overlay && Overlay.installed(this)) "waits for a line" else "off" + Overlay.OK -> "shows the line" + "panel_hidden" -> "press \"Show the subtitles\" in SubRead Overlay" + "no_overlay_permission", "no_notification_access" -> "allow steps 1 and 2 in SubRead Overlay" + Overlay.TOO_OLD -> "update SubRead Overlay: this version takes no lines" + else -> Feed.overlayAnswer + } + val bars = Feed.bars() + val level = "▮".repeat(bars) + "▯".repeat(10 - bars) + val error = if (Feed.error.isEmpty()) "" else "\n${Feed.error}" + return "Engine ${store.engine}: $engine\nShare link: $relay\nSubRead Overlay: $overlay\nSound: $level$error" + } + + private fun showLink() { + linkView.text = Room.watchUrl(Room.RELAY, store.room) + } + + private fun copyLink() { + getSystemService(ClipboardManager::class.java).setPrimaryClip(ClipData.newPlainText("link", linkView.text)) + toast(R.string.copied) + } + + private fun shareLink() { + val send = Intent(Intent.ACTION_SEND).setType("text/plain").putExtra(Intent.EXTRA_TEXT, linkView.text.toString()) + runCatching { startActivity(Intent.createChooser(send, null)) } + } + + private fun open(link: String) { + runCatching { startActivity(Intent(Intent.ACTION_VIEW, Uri.parse(link))) } + } + + private fun toast(message: Int) = Toast.makeText(this, message, Toast.LENGTH_LONG).show() + + // The screen is built in code: a few rows do not need a layout file each. + + private fun title(value: String) = content.addView(TextView(this).apply { + text = value + textSize = 26f + setTypeface(typeface, Typeface.BOLD) + setTextColor(Color.BLACK) + }) + + private fun note(value: String, top: Int = 8) = content.addView(TextView(this).apply { + text = value + textSize = 15f + setTextColor(Color.BLACK) + }, wide(top)) + + @SuppressLint("SetTextI18n") + private fun step(name: Int, detail: String) { + content.addView(View(this).apply { setBackgroundColor(Color.BLACK) }, LinearLayout.LayoutParams(-1, dp(1)).apply { topMargin = dp(16) }) + content.addView(TextView(this).apply { + text = getString(name) + textSize = 18f + setTypeface(typeface, Typeface.BOLD) + setTextColor(Color.BLACK) + }, wide(top = 12)) + if (detail.isNotEmpty()) note(detail, top = 4) + } + + private fun field(value: String, hint: String, type: Int) = EditText(this).apply { + setText(value) + this.hint = hint + inputType = type + setTextColor(Color.BLACK) + setHintTextColor(Color.GRAY) + isSingleLine = true + }.also { content.addView(it, wide()) } + + private fun row(vararg views: View) = content.addView(LinearLayout(this).apply { views.forEach { addView(it) } }) + + private fun button(label: String, onClick: () -> Unit) = Button(this).apply { + text = label + isAllCaps = false + setOnClickListener { onClick() } + } + + private fun wide(top: Int = 8) = LinearLayout.LayoutParams(-1, -2).apply { topMargin = dp(top) } + + private fun dp(value: Int) = (value * resources.displayMetrics.density).toInt() + + private companion object { + const val REQUEST_PERMISSIONS = 1 + const val REQUEST_PROJECTION = 2 + } +} diff --git a/app/src/main/kotlin/com/honjimaku/subrep/Messages.kt b/app/src/main/kotlin/com/honjimaku/subrep/Messages.kt new file mode 100644 index 0000000..060641e --- /dev/null +++ b/app/src/main/kotlin/com/honjimaku/subrep/Messages.kt @@ -0,0 +1,43 @@ +package com.honjimaku.subrep + +import org.json.JSONObject + +/** A caption from the engine: `partial` while the sentence goes on, `final` when it is done, or `clear`. */ +data class Caption(val type: String, val text: String) + +/** + * The messages of the two protocols. + * + * The engine (`subrep serve`) takes a hello, then frames of 16-bit PCM, and answers with + * captions as JSON text. The relay takes a hello with the room secret, then the captions as + * they came from the engine. + */ +object Messages { + + val CAPTIONS = setOf("partial", "final", "clear") + + fun engineHello(sampleRate: Int, lang: String, source: String): String = + JSONObject().put("type", "hello").put("sampleRate", sampleRate).put("lang", lang).put("source", source).toString() + + fun relayHello(secret: String, lang: String, title: String): String = + JSONObject().put("secret", secret).put("lang", lang).put("title", title).toString() + + /** The caption in a message of the engine; null for a message of another type or junk. */ + fun caption(json: String): Caption? { + val message = parse(json) ?: return null + val type = message.optString("type") + return if (type in CAPTIONS) Caption(type, message.optString("text")) else null + } + + /** The reason in an error message, from the engine or the relay; null for another message. */ + fun error(json: String): String? { + val message = parse(json) ?: return null + if (message.optString("type") != "error") return null + return message.optString("error").ifEmpty { "error" } + } + + /** True for the ready message that the engine and the relay send after the hello. */ + fun isReady(json: String): Boolean = parse(json)?.optString("type") == "ready" + + private fun parse(json: String): JSONObject? = runCatching { JSONObject(json) }.getOrNull() +} diff --git a/app/src/main/kotlin/com/honjimaku/subrep/Overlay.kt b/app/src/main/kotlin/com/honjimaku/subrep/Overlay.kt new file mode 100644 index 0000000..ef28a7f --- /dev/null +++ b/app/src/main/kotlin/com/honjimaku/subrep/Overlay.kt @@ -0,0 +1,90 @@ +package com.honjimaku.subrep + +import android.content.Context +import android.content.Intent +import android.net.Uri +import android.os.Bundle +import android.os.Handler +import android.os.SystemClock + +/** + * SubRead Overlay: the panel over the player. It takes a line with the method `line` of its + * content provider, and answers in the key `live` whether it shows it. + */ +object Overlay { + + const val PACKAGE = "space.subread.overlay" + const val INSTALL = "https://github.com/equwal/subread-overlay/releases/latest" + private const val AUTHORITY = "space.subread.overlay.player" + private val uri: Uri = Uri.parse("content://$AUTHORITY") + + const val OK = "ok" + const val NOT_INSTALLED = "not_installed" + + /** The overlay's answer for a version that has no `line` method. */ + const val TOO_OLD = "too_old" + + fun installed(context: Context): Boolean = context.packageManager.resolveContentProvider(AUTHORITY, 0) != null + + fun launch(context: Context): Intent? = context.packageManager.getLaunchIntentForPackage(PACKAGE) + + fun line(context: Context, text: String, partial: Boolean): String = + call(context, "line", text, Bundle().apply { putBoolean("partial", partial) }) + + fun end(context: Context): String = call(context, "end", null, null) + + private fun call(context: Context, method: String, arg: String?, extras: Bundle?): String { + val answer = try { + context.contentResolver.call(uri, method, arg, extras) + } catch (e: Exception) { + return NOT_INSTALLED + } ?: return NOT_INSTALLED + return answer.getString("live") ?: TOO_OLD + } +} + +/** + * Sends the captions to the overlay from its own thread, because a provider call waits for + * the other app. Partial lines go at most every [PARTIAL_GAP_MS]: the overlay draws each line, + * and an e-ink screen cannot draw five times a second. + */ +class OverlayFeed(private val context: Context, private val handler: Handler) { + + private var pendingPartial: String? = null + private var lastAt = 0L + private val flush = Runnable { + pendingPartial?.let { send(it, partial = true) } + pendingPartial = null + } + + fun caption(caption: Caption) { + handler.post { + when (caption.type) { + "final" -> { + handler.removeCallbacks(flush) + pendingPartial = null + send(caption.text, partial = false) + } + "partial" -> { + pendingPartial = caption.text + handler.removeCallbacks(flush) + handler.postDelayed(flush, (PARTIAL_GAP_MS - (SystemClock.elapsedRealtime() - lastAt)).coerceAtLeast(0)) + } + } + } + } + + fun end() { + handler.removeCallbacks(flush) + handler.post { Overlay.end(context) } + } + + private fun send(text: String, partial: Boolean) { + lastAt = SystemClock.elapsedRealtime() + Feed.overlay(Overlay.line(context, text, partial)) + } + + private companion object { + const val PARTIAL_GAP_MS = 700L + } +} diff --git a/app/src/main/kotlin/com/honjimaku/subrep/Pcm.kt b/app/src/main/kotlin/com/honjimaku/subrep/Pcm.kt new file mode 100644 index 0000000..b1c0b37 --- /dev/null +++ b/app/src/main/kotlin/com/honjimaku/subrep/Pcm.kt @@ -0,0 +1,18 @@ +package com.honjimaku.subrep + +/** Sums over 16-bit little-endian PCM, for the level meter. */ +object Pcm { + + /** The largest sample size in the first [count] bytes of [bytes]: 0 for silence, up to 32768. */ + fun peak(bytes: ByteArray, count: Int): Int { + var peak = 0 + var i = 0 + while (i + 1 < count) { + val sample = (bytes[i].toInt() and 0xFF) or (bytes[i + 1].toInt() shl 8) + val size = if (sample < 0) -sample else sample + if (size > peak) peak = size + i += 2 + } + return peak + } +} diff --git a/app/src/main/kotlin/com/honjimaku/subrep/Room.kt b/app/src/main/kotlin/com/honjimaku/subrep/Room.kt new file mode 100644 index 0000000..92baead --- /dev/null +++ b/app/src/main/kotlin/com/honjimaku/subrep/Room.kt @@ -0,0 +1,40 @@ +package com.honjimaku.subrep + +import java.security.SecureRandom +import java.util.Base64 + +/** + * The share room of this device on the relay, the same as the desktop app makes it. + * + * The room id is a capability: whoever has the link reads the feed. The secret lets only this + * device publish into the room. Both are made once and kept, so the link stays the same across + * restarts. The relay refuses a secret shorter than 16 characters. + */ +object Room { + + const val RELAY = "wss://honjimaku.com/subrep" + + private val random = SecureRandom() + + /** A room id: 12 characters of the URL alphabet, as the desktop app makes them. */ + fun newId(): String = token(9) + + /** A publisher secret: 32 characters. */ + fun newSecret(): String = token(24) + + /** The link that the viewers open, for the relay [relay] and the room [room]. */ + fun watchUrl(relay: String, room: String): String { + val base = relay.trimEnd('/') + val http = when { + base.startsWith("wss://") -> "https://" + base.removePrefix("wss://") + base.startsWith("ws://") -> "http://" + base.removePrefix("ws://") + else -> base + } + return "$http/w/$room" + } + + private fun token(bytes: Int): String { + val raw = ByteArray(bytes).also(random::nextBytes) + return Base64.getUrlEncoder().withoutPadding().encodeToString(raw) + } +} diff --git a/app/src/main/kotlin/com/honjimaku/subrep/Socket.kt b/app/src/main/kotlin/com/honjimaku/subrep/Socket.kt new file mode 100644 index 0000000..d6175aa --- /dev/null +++ b/app/src/main/kotlin/com/honjimaku/subrep/Socket.kt @@ -0,0 +1,93 @@ +package com.honjimaku.subrep + +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.Response +import okhttp3.WebSocket +import okhttp3.WebSocketListener +import okio.ByteString.Companion.toByteString +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit + +/** + * One websocket that comes back on its own after a loss. The first message after each connect + * is the hello of the protocol, so the other side knows what follows. + */ +class Socket( + private val client: OkHttpClient, + private val url: String, + private val hello: () -> String, + private val onText: (String) -> Unit, + /** True when the socket is open, false when it was lost. Not called for a failed connect. */ + private val onState: (Boolean) -> Unit, +) : WebSocketListener() { + + private val timer = Executors.newSingleThreadScheduledExecutor() + + @Volatile private var socket: WebSocket? = null + @Volatile private var wanted = false + @Volatile var open = false + private set + private var backoffMs = 1000L + + fun start() { + wanted = true + connect() + } + + fun stop() { + wanted = false + open = false + timer.shutdownNow() + socket?.close(1000, "stop") + socket = null + } + + fun send(text: String): Boolean = open && socket?.send(text) == true + + /** + * Sends the first [count] bytes of [bytes]. Dropped when the socket is not open, or when + * a slow connection holds more than a few seconds of sound: late captions help nobody. + */ + fun send(bytes: ByteArray, count: Int): Boolean { + val s = socket ?: return false + if (!open || s.queueSize() > QUEUE_LIMIT) return false + return s.send(bytes.toByteString(0, count)) + } + + private fun connect() { + if (!wanted) return + socket = client.newWebSocket(Request.Builder().url(url).build(), this) + } + + override fun onOpen(webSocket: WebSocket, response: Response) { + backoffMs = 1000 + open = true + webSocket.send(hello()) + onState(true) + } + + override fun onMessage(webSocket: WebSocket, text: String) = onText(text) + + override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) = lost() + + override fun onClosing(webSocket: WebSocket, code: Int, reason: String) { + webSocket.close(1000, null) + } + + override fun onClosed(webSocket: WebSocket, code: Int, reason: String) = lost() + + private fun lost() { + val was = open + open = false + if (was) onState(false) + if (!wanted) return + val wait = backoffMs + backoffMs = minOf(backoffMs * 2, 30_000) + runCatching { timer.schedule({ connect() }, wait, TimeUnit.MILLISECONDS) } + } + + private companion object { + const val QUEUE_LIMIT = 256 * 1024L + } +} diff --git a/app/src/main/kotlin/com/honjimaku/subrep/Store.kt b/app/src/main/kotlin/com/honjimaku/subrep/Store.kt new file mode 100644 index 0000000..0ff7f50 --- /dev/null +++ b/app/src/main/kotlin/com/honjimaku/subrep/Store.kt @@ -0,0 +1,50 @@ +package com.honjimaku.subrep + +import android.content.Context + +/** What the user chose. It stays when the app stops. */ +class Store(context: Context) { + + private val prefs = context.getSharedPreferences("subrep", Context.MODE_PRIVATE) + + /** The engine on the computer, as `address:port`. */ + var engine: String + get() = prefs.getString("engine", "").orEmpty() + set(value) = prefs.edit().putString("engine", value.trim()).apply() + + var lang: String + get() = prefs.getString("lang", "ja").orEmpty() + set(value) = prefs.edit().putString("lang", value.trim().ifEmpty { "auto" }).apply() + + /** True: the sound of the apps on this device. False: the microphone. */ + var screen: Boolean + get() = prefs.getBoolean("screen", true) + set(value) = prefs.edit().putBoolean("screen", value).apply() + + var share: Boolean + get() = prefs.getBoolean("share", true) + set(value) = prefs.edit().putBoolean("share", value).apply() + + /** What the viewers see as the name of the stream. */ + var title: String + get() = prefs.getString("title", "").orEmpty() + set(value) = prefs.edit().putString("title", value.trim().take(80)).apply() + + var overlay: Boolean + get() = prefs.getBoolean("overlay", true) + set(value) = prefs.edit().putBoolean("overlay", value).apply() + + /** The room on the relay. Made once, so the link stays the same. */ + val room: String + get() = prefs.getString("room", null) ?: newRoom() + + val secret: String + get() = prefs.getString("secret", null) ?: newRoom().let { prefs.getString("secret", "").orEmpty() } + + /** A new room: the old link stops. Returns the new room id. */ + fun newRoom(): String { + val room = Room.newId() + prefs.edit().putString("room", room).putString("secret", Room.newSecret()).apply() + return room + } +} diff --git a/app/src/main/res/drawable/icon.xml b/app/src/main/res/drawable/icon.xml new file mode 100644 index 0000000..b94a710 --- /dev/null +++ b/app/src/main/res/drawable/icon.xml @@ -0,0 +1,8 @@ +<?xml version="1.0" encoding="utf-8"?> +<vector xmlns:android="http://schemas.android.com/apk/res/android" + android:width="108dp" android:height="108dp" android:viewportWidth="108" android:viewportHeight="108"> + <path android:fillColor="#FFFFFF" android:pathData="M0,0h108v108h-108z" /> + <path android:fillColor="#111111" android:pathData="M18,64h72v22h-72z" /> + <path android:fillColor="#FFFFFF" android:pathData="M26,71h28v8h-28z M60,71h22v8h-22z" /> + <path android:fillColor="#111111" android:pathData="M30,44h8v12h-8z M42,32h8v24h-8z M54,24h8v32h-8z M66,36h8v20h-8z" /> +</vector> diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml new file mode 100644 index 0000000..94b18ea --- /dev/null +++ b/app/src/main/res/values/strings.xml @@ -0,0 +1,36 @@ +<?xml version="1.0" encoding="utf-8"?> +<resources> + <string name="app_name">Subrep Streamer</string> + <string name="about">Live captions of the sound of this device. The sound goes to the Subrep engine on your computer. The captions come back here, to your share link, and to SubRead Overlay over the player.</string> + + <string name="step_engine">1. Engine</string> + <string name="step_engine_why">The computer that runs \"subrep serve\", as address:port on the same network. For example 192.168.0.9:8794.</string> + <string name="step_lang">2. Language</string> + <string name="step_lang_why">ja, en, ru, fi, es, pt, or auto.</string> + <string name="step_source">3. Sound</string> + <string name="source_screen">The apps on this device (Android 10 or later)</string> + <string name="source_mic">The microphone</string> + <string name="step_share">4. Share</string> + <string name="share_on">Share the captions by link</string> + <string name="share_title_hint">Name of the stream, for the viewers</string> + <string name="copy">Copy</string> + <string name="share">Share</string> + <string name="new_link">New link</string> + <string name="copied">Link copied.</string> + <string name="step_overlay">5. SubRead Overlay</string> + <string name="overlay_on">Show the captions over the player, in SubRead Overlay</string> + <string name="overlay_missing">SubRead Overlay is not installed. With it, the captions are on top of the player, and a tap on a word opens the dictionary.</string> + <string name="install_overlay">Install SubRead Overlay</string> + <string name="open_overlay">Open SubRead Overlay</string> + + <string name="start">Start the captions</string> + <string name="stop">Stop the captions</string> + <string name="need_engine">Enter the address of the engine first.</string> + <string name="need_android_10">The sound of other apps needs Android 10. Use the microphone.</string> + <string name="need_audio_permission">Allow the microphone. Android asks for it for the sound of the apps too.</string> + <string name="capture_refused">Android did not allow the capture.</string> + + <string name="notification_channel">Live captions</string> + <string name="notification_title">Subrep: live captions</string> + <string name="notification_text">The sound goes to the engine.</string> +</resources> diff --git a/app/src/test/kotlin/com/honjimaku/subrep/ProtocolTest.kt b/app/src/test/kotlin/com/honjimaku/subrep/ProtocolTest.kt new file mode 100644 index 0000000..d310936 --- /dev/null +++ b/app/src/test/kotlin/com/honjimaku/subrep/ProtocolTest.kt @@ -0,0 +1,115 @@ +package com.honjimaku.subrep + +import io.kotest.property.Arb +import io.kotest.property.arbitrary.element +import io.kotest.property.arbitrary.int +import io.kotest.property.arbitrary.list +import io.kotest.property.arbitrary.string +import io.kotest.property.checkAll +import kotlinx.coroutines.runBlocking +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class RoomTest { + + /** What the relay accepts as a room id. */ + private val roomId = Regex("^[a-zA-Z0-9_-]{6,64}$") + + @Test + fun aRoomIdIsOneTheRelayAcceptsAndTwoAreNotTheSame() { + val ids = List(200) { Room.newId() } + ids.forEach { assertTrue(it, roomId.matches(it)) } + ids.forEach { assertEquals(12, it.length) } + assertEquals(200, ids.toSet().size) + } + + @Test + fun aSecretIsLongEnoughForTheRelay() { + repeat(50) { assertTrue(Room.newSecret().length >= 16) } + } + + @Test + fun theWatchLinkIsTheHttpsPageOfTheRoom() { + assertEquals("https://honjimaku.com/subrep/w/abc123def456", Room.watchUrl("wss://honjimaku.com/subrep", "abc123def456")) + assertEquals("https://honjimaku.com/subrep/w/abc123def456", Room.watchUrl("wss://honjimaku.com/subrep/", "abc123def456")) + assertEquals("http://127.0.0.1:8812/w/r", Room.watchUrl("ws://127.0.0.1:8812", "r")) + } +} + +class MessagesTest { + + private val texts = Arb.string(0..60) + private val types = Arb.element("partial", "final", "clear") + + /** The text of a caption of the engine comes out as it went in, in any script. */ + @Test + fun aCaptionIsReadBackUnchanged(): Unit = runBlocking { + checkAll(types, texts) { type, text -> + val json = JSONObject().put("type", type).put("text", text).put("ts", 1.5).toString() + assertEquals(Caption(type, text), Messages.caption(json)) + } + } + + @Test + fun theOtherMessagesAreNoCaption(): Unit = runBlocking { + checkAll(Arb.element("ready", "status", "hello", "error", "gone", ""), texts) { type, text -> + assertNull(Messages.caption(JSONObject().put("type", type).put("text", text).toString())) + } + assertNull(Messages.caption("not json")) + assertNull(Messages.caption("")) + } + + @Test + fun anErrorGivesItsReason() { + assertEquals("too many active tabs", Messages.error("""{"type":"error","error":"too many active tabs"}""")) + assertEquals("error", Messages.error("""{"type":"error"}""")) + assertNull(Messages.error("""{"type":"final","text":"x"}""")) + assertNull(Messages.error("junk")) + } + + @Test + fun theHellosHaveWhatTheServersRead(): Unit = runBlocking { + checkAll(Arb.int(8000..192000), texts, texts) { rate, lang, source -> + val hello = JSONObject(Messages.engineHello(rate, lang, source)) + assertEquals("hello", hello.getString("type")) + assertEquals(rate, hello.getInt("sampleRate")) + assertEquals(lang, hello.getString("lang")) + assertEquals(source, hello.getString("source")) + } + checkAll(texts, texts, texts) { secret, lang, title -> + val hello = JSONObject(Messages.relayHello(secret, lang, title)) + assertEquals(secret, hello.getString("secret")) + assertEquals(lang, hello.getString("lang")) + assertEquals(title, hello.getString("title")) + } + assertTrue(Messages.isReady("""{"type":"ready","room":"x"}""")) + } +} + +class PcmTest { + + /** The peak of a frame is the largest sample size in it, with the sign taken away. */ + @Test + fun thePeakIsTheLargestSampleSize(): Unit = runBlocking { + checkAll(Arb.list(Arb.int(-32768..32767), 0..400)) { samples -> + val bytes = ByteArray(samples.size * 2) + samples.forEachIndexed { i, s -> + bytes[2 * i] = (s and 0xFF).toByte() + bytes[2 * i + 1] = ((s shr 8) and 0xFF).toByte() + } + val expected = samples.maxOfOrNull { if (it < 0) -it else it } ?: 0 + assertEquals(expected, Pcm.peak(bytes, bytes.size)) + } + } + + @Test + fun onlyTheFirstCountBytesCount() { + val bytes = byteArrayOf(0, 0, 0, 0x7F.toByte()) + assertEquals(0, Pcm.peak(bytes, 2)) + assertEquals(32512, Pcm.peak(bytes, 4)) + assertEquals(0, Pcm.peak(bytes, 3)) + } +} diff --git a/build.gradle.kts b/build.gradle.kts new file mode 100644 index 0000000..2542156 --- /dev/null +++ b/build.gradle.kts @@ -0,0 +1,4 @@ +// Plugin versions live in gradle/libs.versions.toml; the app module applies its own. +plugins { + alias(libs.plugins.android.application) apply false +} diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 0000000..75718b5 --- /dev/null +++ b/gradle.properties @@ -0,0 +1,4 @@ +org.gradle.jvmargs=-Xmx1536m -Dfile.encoding=UTF-8 +org.gradle.caching=true +org.gradle.parallel=true +kotlin.code.style=official diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml new file mode 100644 index 0000000..7f7cd41 --- /dev/null +++ b/gradle/libs.versions.toml @@ -0,0 +1,19 @@ +[versions] +agp = "9.2.0" +okhttp = "4.12.0" +kotlinxCoroutines = "1.10.2" +kotestProperty = "5.9.1" +junit = "4.13.2" +json = "20250517" + +[libraries] +# The websocket client. The Java standard library of Android has none. +okhttp = { group = "com.squareup.okhttp3", name = "okhttp", version.ref = "okhttp" } +kotlinx-coroutines-core = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-core", version.ref = "kotlinxCoroutines" } +kotest-property = { group = "io.kotest", name = "kotest-property", version.ref = "kotestProperty" } +junit = { group = "junit", name = "junit", version.ref = "junit" } +# org.json for the unit tests: the android.jar of the JVM tests has only stubs of it. +json = { group = "org.json", name = "json", version.ref = "json" } + +[plugins] +android-application = { id = "com.android.application", version.ref = "agp" } diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..8bdaf60 Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..bb25ae9 --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,9 @@ +#Tue Apr 28 00:13:17 CST 2026 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionSha256Sum=2ab2958f2a1e51120c326cad6f385153bb11ee93b3c216c5fccebfdfbb7ec6cb +distributionUrl=https\://services.gradle.org/distributions/gradle-9.4.1-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew new file mode 100644 index 0000000..ef07e01 --- /dev/null +++ b/gradlew @@ -0,0 +1,251 @@ +#!/bin/sh + +# +# Copyright © 2015 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH="\\\"\\\"" + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000..db3a6ac --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,94 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH= + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/settings.gradle.kts b/settings.gradle.kts new file mode 100644 index 0000000..28772bc --- /dev/null +++ b/settings.gradle.kts @@ -0,0 +1,24 @@ +pluginManagement { + repositories { + google { + content { + includeGroupByRegex("com\\.android.*") + includeGroupByRegex("com\\.google.*") + includeGroupByRegex("androidx.*") + } + } + mavenCentral() + gradlePluginPortal() + } +} +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) + repositories { + google() + mavenCentral() + } +} + +rootProject.name = "SubrepStreamer" + +include(":app")