app/src/main/kotlin/space/subread/anki/CaptureService.kt (10996 bytes)
1 package space.subread.anki 2 3 import android.app.Notification 4 import android.app.NotificationChannel 5 import android.app.NotificationManager 6 import android.app.PendingIntent 7 import android.app.Service 8 import android.content.Context 9 import android.content.Intent 10 import android.content.pm.PackageManager 11 import android.content.pm.ServiceInfo 12 import android.graphics.Bitmap 13 import android.graphics.PixelFormat 14 import android.hardware.display.DisplayManager 15 import android.hardware.display.VirtualDisplay 16 import android.media.AudioAttributes 17 import android.media.AudioFormat 18 import android.media.AudioPlaybackCaptureConfiguration 19 import android.media.AudioRecord 20 import android.media.Image 21 import android.media.ImageReader 22 import android.media.projection.MediaProjection 23 import android.media.projection.MediaProjectionManager 24 import android.os.Build 25 import android.os.Handler 26 import android.os.IBinder 27 import android.os.Looper 28 import android.os.SystemClock 29 import android.util.DisplayMetrics 30 import android.view.Display 31 import androidx.core.content.IntentCompat 32 import androidx.core.graphics.createBitmap 33 import space.subread.anki.core.Loudness 34 import space.subread.anki.core.PcmRing 35 import java.io.File 36 import kotlin.concurrent.thread 37 38 /** 39 * Keeps the last minute and a half of the sound of the device and the last picture of the 40 * screen, while the user has the capture on. A card takes its clip and its picture from here. 41 * 42 * Android gives the sound of other apps to a media projection alone, and only since Android 43 * 10. The projection needs the consent of the user each time it starts, and Android shows it 44 * in the status bar. The service is a foreground service, as Android demands for a projection. 45 * 46 * The clock of the ring is `SystemClock.elapsedRealtimeNanos()`, the clock of the media 47 * sessions: a report of the player maps its media times straight onto the ring. 48 */ 49 class CaptureService : Service() { 50 51 private val handler = Handler(Looper.getMainLooper()) 52 private var projection: MediaProjection? = null 53 private var record: AudioRecord? = null 54 private var display: VirtualDisplay? = null 55 private var reader: ImageReader? = null 56 57 /** The newest picture of the screen. Kept open until a newer one comes. */ 58 private var latest: Image? = null 59 private val imageLock = Any() 60 61 private val ring = PcmRing(SAMPLE_RATE, 1, SAMPLE_RATE * SECONDS) 62 private val ringLock = Any() 63 64 @Volatile 65 private var running = false 66 67 /** True when the sound of the player is captured; false on Android 9 or without the permission. */ 68 var hasAudio = false 69 private set 70 71 override fun onBind(intent: Intent?): IBinder? = null 72 73 override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { 74 if (intent == null || intent.action == ACTION_STOP) { 75 stopSelf() 76 return START_NOT_STICKY 77 } 78 val code = intent.getIntExtra(EXTRA_CODE, 0) 79 val data = IntentCompat.getParcelableExtra(intent, EXTRA_DATA, Intent::class.java) 80 if (data == null || projection != null) { 81 stopSelf() 82 return START_NOT_STICKY 83 } 84 startInForeground() 85 val manager = getSystemService(MediaProjectionManager::class.java) 86 val projection = runCatching { manager.getMediaProjection(code, data) }.getOrNull() 87 if (projection == null) { 88 stopSelf() 89 return START_NOT_STICKY 90 } 91 this.projection = projection 92 // Android 14 demands the callback before the display, and stops the projection from 93 // the status bar or the lock screen: the service goes with it. 94 projection.registerCallback(object : MediaProjection.Callback() { 95 override fun onStop() = stopSelf() 96 }, handler) 97 running = true 98 instance = this 99 startAudio(projection) 100 startScreen(projection) 101 return START_NOT_STICKY 102 } 103 104 private fun startInForeground() { 105 val channel = NotificationChannel(CHANNEL, getString(R.string.capture_channel), NotificationManager.IMPORTANCE_LOW) 106 getSystemService(NotificationManager::class.java).createNotificationChannel(channel) 107 val stop = PendingIntent.getService( 108 this, 0, Intent(this, CaptureService::class.java).setAction(ACTION_STOP), 109 PendingIntent.FLAG_IMMUTABLE or PendingIntent.FLAG_UPDATE_CURRENT, 110 ) 111 val open = PendingIntent.getActivity(this, 0, Intent(this, MainActivity::class.java), PendingIntent.FLAG_IMMUTABLE) 112 val notification = Notification.Builder(this, CHANNEL) 113 .setSmallIcon(R.drawable.icon) 114 .setContentTitle(getString(R.string.app_name)) 115 .setContentText(getString(R.string.capture_notification)) 116 .setContentIntent(open) 117 .addAction(Notification.Action.Builder(null, getString(R.string.capture_stop), stop).build()) 118 .setOngoing(true) 119 .build() 120 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { 121 startForeground(NOTIFICATION_ID, notification, ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION) 122 } else { 123 startForeground(NOTIFICATION_ID, notification) 124 } 125 } 126 127 private fun startAudio(projection: MediaProjection) { 128 if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) return 129 if (checkSelfPermission(android.Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) return 130 val config = AudioPlaybackCaptureConfiguration.Builder(projection) 131 .addMatchingUsage(AudioAttributes.USAGE_MEDIA) 132 .addMatchingUsage(AudioAttributes.USAGE_GAME) 133 .addMatchingUsage(AudioAttributes.USAGE_UNKNOWN) 134 .build() 135 val minimum = AudioRecord.getMinBufferSize(SAMPLE_RATE, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT) 136 val record = runCatching { 137 AudioRecord.Builder() 138 .setAudioPlaybackCaptureConfig(config) 139 .setAudioFormat( 140 AudioFormat.Builder() 141 .setEncoding(AudioFormat.ENCODING_PCM_16BIT) 142 .setSampleRate(SAMPLE_RATE) 143 .setChannelMask(AudioFormat.CHANNEL_IN_MONO) 144 .build(), 145 ) 146 .setBufferSizeInBytes(maxOf(minimum, SAMPLE_RATE) * 2) 147 .build() 148 }.getOrNull() ?: return 149 if (record.state != AudioRecord.STATE_INITIALIZED) { 150 record.release() 151 return 152 } 153 this.record = record 154 hasAudio = true 155 record.startRecording() 156 thread(name = "capture-audio") { 157 val buffer = ShortArray(SAMPLE_RATE / 10) 158 while (running) { 159 val n = record.read(buffer, 0, buffer.size) 160 if (n <= 0) continue 161 val now = SystemClock.elapsedRealtimeNanos() 162 synchronized(ringLock) { ring.write(buffer, n, now) } 163 } 164 } 165 } 166 167 private fun startScreen(projection: MediaProjection) { 168 val metrics = DisplayMetrics() 169 val display = getSystemService(DisplayManager::class.java).getDisplay(Display.DEFAULT_DISPLAY) ?: return 170 @Suppress("DEPRECATION") // The size of the whole screen: a service has no window to measure. 171 display.getRealMetrics(metrics) 172 val reader = ImageReader.newInstance(metrics.widthPixels, metrics.heightPixels, PixelFormat.RGBA_8888, 3) 173 reader.setOnImageAvailableListener({ r -> 174 val image = runCatching { r.acquireLatestImage() }.getOrNull() ?: return@setOnImageAvailableListener 175 synchronized(imageLock) { 176 latest?.close() 177 latest = image 178 } 179 }, handler) 180 this.reader = reader 181 this.display = runCatching { 182 projection.createVirtualDisplay( 183 "SubReadAnki", metrics.widthPixels, metrics.heightPixels, metrics.densityDpi, 184 DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR, reader.surface, null, handler, 185 ) 186 }.getOrNull() 187 } 188 189 /** 190 * The sound between two moments of the ring clock, as AAC in [out]. Null when the ring 191 * has none of it, less than a fifth of a second, or only silence: a reader app plays no 192 * sound, and a card with a silent clip is worse than a card without one. 193 */ 194 fun clip(fromNanos: Long, toNanos: Long, out: File): File? { 195 val pcm = synchronized(ringLock) { ring.read(fromNanos, toNanos) } 196 if (pcm.size < SAMPLE_RATE / 5 || Loudness.isSilent(pcm)) return null 197 return runCatching { AacEncoder.encode(pcm, SAMPLE_RATE, 1, out); out }.getOrNull() 198 } 199 200 /** 201 * The newest picture of the screen. Called when a text comes in, before the pop-up draws, 202 * so the picture shows the app that sent the text. Null when there is none yet. 203 */ 204 fun snapshot(): Bitmap? = synchronized(imageLock) { 205 val image = latest ?: return null 206 val plane = image.planes[0] 207 val stride = plane.rowStride / plane.pixelStride 208 val full = createBitmap(stride, image.height) 209 full.copyPixelsFromBuffer(plane.buffer.also { it.rewind() }) 210 if (stride == image.width) full else Bitmap.createBitmap(full, 0, 0, image.width, image.height) 211 } 212 213 override fun onDestroy() { 214 running = false 215 instance = null 216 record?.let { runCatching { it.stop() }; it.release() } 217 record = null 218 display?.release() 219 display = null 220 reader?.close() 221 reader = null 222 synchronized(imageLock) { 223 latest?.close() 224 latest = null 225 } 226 projection?.stop() 227 projection = null 228 super.onDestroy() 229 } 230 231 companion object { 232 const val SAMPLE_RATE = 44_100 233 /** How much sound the ring keeps. */ 234 const val SECONDS = 90 235 const val CHANNEL = "capture" 236 const val NOTIFICATION_ID = 1 237 const val ACTION_STOP = "space.subread.anki.action.STOP_CAPTURE" 238 const val EXTRA_CODE = "code" 239 const val EXTRA_DATA = "data" 240 241 /** The service while it runs; null when the capture is off. */ 242 @Volatile 243 var instance: CaptureService? = null 244 private set 245 246 /** True when this Android can capture the sound of other apps. */ 247 val canCaptureAudio: Boolean get() = Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q 248 249 /** Starts the capture with the consent that Android gave [CaptureActivity]. */ 250 fun start(context: Context, code: Int, data: Intent) { 251 val intent = Intent(context, CaptureService::class.java).putExtra(EXTRA_CODE, code).putExtra(EXTRA_DATA, data) 252 context.startForegroundService(intent) 253 } 254 255 fun stop(context: Context) { 256 context.startService(Intent(context, CaptureService::class.java).setAction(ACTION_STOP)) 257 } 258 } 259 }