Recently Written · git

subrep-android

Subrep on Android: live captions of the sound of the phone, made on the phone by Whisper, shared by link.

git clone https://github.com/equwal/subrep-android

Log | Files | Refs

app/src/main/kotlin/com/honjimaku/subrep/CaptureService.kt (9393 bytes)

1 package com.honjimaku.subrep
2 
3 import android.app.Notification
4 import android.app.NotificationChannel
5 import android.app.NotificationManager
6 import android.app.PendingIntent
7 import android.app.Service
8 import android.content.Context
9 import android.content.Intent
10 import android.content.pm.ServiceInfo
11 import android.graphics.drawable.Icon
12 import android.media.AudioRecord
13 import android.media.projection.MediaProjection
14 import android.media.projection.MediaProjectionManager
15 import android.os.Build
16 import android.os.Handler
17 import android.os.HandlerThread
18 import android.os.IBinder
19 import android.os.Looper
20 import okhttp3.OkHttpClient
21 import java.io.FileDescriptor
22 import java.io.PrintWriter
23 import java.util.concurrent.TimeUnit
24 import kotlin.concurrent.thread
25 
26 /**
27  * Captures the sound, turns it into captions with Whisper on the phone, and gives the captions
28  * to the screen, to the relay and to SubRead Overlay. A foreground service, so that it lives
29  * while the user is in the player.
30  */
31 class CaptureService : Service() {
32 
33     private lateinit var store: Store
34     private val http = OkHttpClient.Builder().pingInterval(20, TimeUnit.SECONDS).build()
35     private val main = Handler(Looper.getMainLooper())
36     private val captions = HandlerThread("captions").apply { start() }
37     private var engine: Engine? = null
38     private var relay: Socket? = null
39     private var overlay: OverlayFeed? = null
40     private var record: AudioRecord? = null
41     private var projection: MediaProjection? = null
42     private var pump: Thread? = null
43 
44     override fun onBind(intent: Intent?): IBinder? = null
45 
46     override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
47         when (intent?.action) {
48             ACTION_START -> if (!Feed.running) start(intent)
49             ACTION_STOP -> stopAll()
50         }
51         return START_NOT_STICKY
52     }
53 
54     private fun start(intent: Intent) {
55         store = Store(this)
56         val screen = intent.getBooleanExtra(EXTRA_SCREEN, false)
57         // Android 14 and later accept the projection only from a foreground service of this type.
58         foreground(screen)
59         Feed.start(store.share)
60 
61         val cloud = store.cloud
62         val model = Model.byId(store.model)
63         if (!cloud && !model.downloaded(this)) {
64             Feed.fail("Download the ${model.label} model first.")
65             stopAll()
66             return
67         }
68 
69         val record = try {
70             if (screen) Capture.ofApps(projectionOf(intent)) else Capture.ofMicrophone()
71         } catch (e: Exception) {
72             Feed.fail("${e.javaClass.simpleName}: ${e.message}")
73             stopAll()
74             return
75         }
76         record.startRecording()
77         if (record.recordingState != AudioRecord.RECORDSTATE_RECORDING) {
78             Feed.fail("The recorder did not start.")
79             stopAll()
80             return
81         }
82         this.record = record
83 
84         val lang = store.lang
85         val title = store.title
86         if (store.share) {
87             val secret = store.secret
88             // With "auto", the viewers see the language once Whisper found it.
89             val hello = { Messages.relayHello(secret, if (lang == "auto") "" else lang, title) }
90             relay = Socket(http, "${Room.RELAY}/pub/${store.room}", hello, ::onRelayText) { Feed.relay(it) }
91                 .also { it.start() }
92         }
93         if (store.overlay && Overlay.installed(this)) overlay = OverlayFeed(this, Handler(captions.looper))
94         val open: () -> Recognizer = if (cloud) {
95             { CloudRecognizer(Cloud(http, store), Feed::cloudLeft) }
96         } else {
97             { Whisper.open(model.file(this)) }
98         }
99         engine = Engine(
100             open, lang,
101             onCaption = ::onCaption,
102             onLanguage = { found -> relay?.send(Messages.status(true, found, title)) },
103             onState = { Feed.engine(it) },
104         ).also { it.start() }
105         pump = thread(name = "pcm") { pump(record) }
106     }
107 
108     private fun projectionOf(intent: Intent): MediaProjection {
109         val code = intent.getIntExtra(EXTRA_RESULT_CODE, 0)
110         @Suppress("DEPRECATION")
111         val data = if (Build.VERSION.SDK_INT >= 33) {
112             intent.getParcelableExtra(EXTRA_RESULT_DATA, Intent::class.java)
113         } else {
114             intent.getParcelableExtra(EXTRA_RESULT_DATA)
115         } ?: throw IllegalStateException("no consent")
116         val manager = getSystemService(MediaProjectionManager::class.java)
117         val projection = manager.getMediaProjection(code, data) ?: throw IllegalStateException("no projection")
118         // Android 14 and later need a callback before the capture starts. The user can stop the
119         // capture from the status bar; then this service stops too.
120         projection.registerCallback(object : MediaProjection.Callback() {
121             override fun onStop() = stopAll()
122         }, main)
123         this.projection = projection
124         return projection
125     }
126 
127     private fun pump(record: AudioRecord) {
128         val frame = ByteArray(Capture.FRAME_BYTES)
129         while (!Thread.currentThread().isInterrupted) {
130             val count = record.read(frame, 0, frame.size, AudioRecord.READ_BLOCKING)
131             if (count < 0) break
132             if (count == 0) continue
133             engine?.feed(frame, count)
134             Feed.level(Pcm.peak(frame, count))
135         }
136     }
137 
138     private fun onCaption(caption: Caption) {
139         engine?.let { Feed.progress(it.speed, it.late) }
140         Feed.caption(caption)
141         relay?.send(Messages.caption(caption))
142         overlay?.caption(caption)
143     }
144 
145     private fun onRelayText(text: String) {
146         Messages.error(text)?.let { Feed.fail("relay: $it") }
147     }
148 
149     private fun stopAll() {
150         pump?.interrupt()
151         pump = null
152         record?.let {
153             runCatching { it.stop() }
154             it.release()
155         }
156         record = null
157         projection?.stop()
158         projection = null
159         // The last piece still goes out, so the relay and the overlay stop after it.
160         engine?.stop()
161         engine = null
162         relay?.stop()
163         relay = null
164         overlay?.end()
165         overlay = null
166         Feed.stop()
167         stopForeground(STOP_FOREGROUND_REMOVE)
168         stopSelf()
169     }
170 
171     override fun onDestroy() {
172         captions.quitSafely()
173         super.onDestroy()
174     }
175 
176     /** For `adb shell dumpsys activity service com.honjimaku.subrep/.CaptureService`: the state, for a bug report. */
177     override fun dump(fd: FileDescriptor, writer: PrintWriter, args: Array<out String>?) {
178         writer.println("running=${Feed.running} engine=${Feed.engine} speed=${Feed.speed} late=${Feed.late}")
179         writer.println("relay=${Feed.relayOn}/${Feed.relayUp} overlay=${Feed.overlayAnswer} peak=${Feed.peak} error=${Feed.error}")
180         writer.println(Feed.text())
181     }
182 
183     private fun foreground(screen: Boolean) {
184         val manager = getSystemService(NotificationManager::class.java)
185         manager.createNotificationChannel(
186             NotificationChannel(CHANNEL, getString(R.string.notification_channel), NotificationManager.IMPORTANCE_LOW),
187         )
188         val open = PendingIntent.getActivity(
189             this, 0, Intent(this, MainActivity::class.java), PendingIntent.FLAG_IMMUTABLE,
190         )
191         val stop = PendingIntent.getService(
192             this, 1, Intent(this, CaptureService::class.java).setAction(ACTION_STOP), PendingIntent.FLAG_IMMUTABLE,
193         )
194         val icon = Icon.createWithResource(this, R.drawable.icon)
195         val notification = Notification.Builder(this, CHANNEL)
196             .setSmallIcon(icon)
197             .setContentTitle(getString(R.string.notification_title))
198             .setContentText(getString(R.string.notification_text))
199             .setContentIntent(open)
200             .setOngoing(true)
201             .addAction(Notification.Action.Builder(icon, getString(R.string.stop), stop).build())
202             .build()
203         when {
204             Build.VERSION.SDK_INT >= 30 -> startForeground(
205                 NOTIFICATION_ID, notification,
206                 if (screen) ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION else ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE,
207             )
208             Build.VERSION.SDK_INT >= 29 -> startForeground(NOTIFICATION_ID, notification, ServiceInfo.FOREGROUND_SERVICE_TYPE_MANIFEST)
209             else -> startForeground(NOTIFICATION_ID, notification)
210         }
211     }
212 
213     companion object {
214         const val ACTION_START = "com.honjimaku.subrep.START"
215         const val ACTION_STOP = "com.honjimaku.subrep.STOP"
216         const val EXTRA_SCREEN = "screen"
217         const val EXTRA_RESULT_CODE = "result_code"
218         const val EXTRA_RESULT_DATA = "result_data"
219         private const val CHANNEL = "capture"
220         private const val NOTIFICATION_ID = 1
221 
222         fun start(context: Context, screen: Boolean, resultCode: Int = 0, resultData: Intent? = null) {
223             val intent = Intent(context, CaptureService::class.java)
224                 .setAction(ACTION_START)
225                 .putExtra(EXTRA_SCREEN, screen)
226                 .putExtra(EXTRA_RESULT_CODE, resultCode)
227                 .putExtra(EXTRA_RESULT_DATA, resultData)
228             context.startForegroundService(intent)
229         }
230 
231         fun stop(context: Context) {
232             context.startService(Intent(context, CaptureService::class.java).setAction(ACTION_STOP))
233         }
234     }
235 }