commit 9524f072333c9f8fda883416f6edb5e5f10e6d04 equwal <truex@equwal.com> 2026-09-21 01:15:59 -0700 Make a video + subtitles for any video player or YouTube After a job, the app writes name.mp4 and name.srt into a folder the user picks. The video is the cover of the book over the audio. The user picks the size (those the device has an encoder for) and 1 to 60 frames a second. The encoder makes one key frame. H264Still makes the other frames, so the frame rate costs about 20 bytes a frame. AAC audio is copied. Other audio is encoded to AAC; the decoder has its own thread, because on a slow device the cost of each codec call sets the speed (75 s of MP3: 49 s before, 10 s now). The device test decodes the file with the platform decoder. It checks the tracks, the frame count, the picture against the source, and the size of the video track.
README.md | 18 + .../kotlin/space/subread/app/StartButtonTest.kt | 2 + .../kotlin/space/subread/app/VideoMakerTest.kt | 171 ++++++++ .../main/kotlin/space/subread/app/MainActivity.kt | 78 +++- .../kotlin/space/subread/app/video/VideoExport.kt | 51 +++ .../kotlin/space/subread/app/video/VideoMaker.kt | 479 +++++++++++++++++++++ 6 files changed, 793 insertions(+), 6 deletions(-)
diff --git a/README.md b/README.md index 6629fc6..9fcf1a6 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,10 @@ Times an audiobook against its ebook, entirely on the phone, and writes the `.srt` that [Hoshi Reader](https://github.com/HuangAntimony/Hoshi-Reader-Android)'s read-along uses. Nothing is uploaded. +It also makes **videos + subtitles which can be played with any video player or +uploaded to YouTube**: an `.mp4` of the book's cover and the audio, with the +`.srt` beside it under the same name. + It is the same method as [subread.space](https://subread.space) and [SubPlz](https://github.com/kanjieater/SubPlz): a small speech model transcribes the audio, roughly; the transcript is aligned against the book; the subtitles @@ -36,6 +40,20 @@ narrated - front matter, notes - instead of matching chapters. ./gradlew :core:test ``` +## The video + +The picture of the video does not change, so almost none of it is encoded. +The device's H.264 encoder makes one key frame. `H264Still` writes each frame +after it by hand: a P slice in which each macroblock is skipped, about ten +bytes. One minute of these frames is written again and again with new time +stamps. AAC audio is copied; other audio is encoded to AAC once. + +The result for a 10-hour book is about 60 MB of video at 720p and one frame a +second. The frame rate adds about 20 bytes for each frame. The size of the +picture changes only the key frame, one each minute. The sizes on offer are +those the device has an encoder for (1080p in software, more with a hardware +encoder). No Android encoder makes 4K or 8K H.264 today. + ## Building the app CI builds it (`.github/workflows/build.yml`): the APK is an artifact of every diff --git a/app/src/androidTest/kotlin/space/subread/app/StartButtonTest.kt b/app/src/androidTest/kotlin/space/subread/app/StartButtonTest.kt index 8613872..9ba8a82 100644 --- a/app/src/androidTest/kotlin/space/subread/app/StartButtonTest.kt +++ b/app/src/androidTest/kotlin/space/subread/app/StartButtonTest.kt @@ -76,6 +76,8 @@ class StartButtonTest { assertEquals(Job.status.value.detail, Phase.DONE, Job.status.value.phase) compose.waitUntil(10_000) { !keepScreenOn() } compose.onNodeWithText("Save .srt").assertExists() + compose.onNodeWithText("Save video + .srt").assertExists() + compose.onNodeWithText("720p").assertExists() assertTrue((Job.status.value.matchRate ?: 0.0) > 0.8) } } diff --git a/app/src/androidTest/kotlin/space/subread/app/VideoMakerTest.kt b/app/src/androidTest/kotlin/space/subread/app/VideoMakerTest.kt new file mode 100644 index 0000000..3c7eae1 --- /dev/null +++ b/app/src/androidTest/kotlin/space/subread/app/VideoMakerTest.kt @@ -0,0 +1,171 @@ +package space.subread.app + +import android.media.MediaCodec +import android.media.MediaCodecInfo +import android.media.MediaExtractor +import android.media.MediaFormat +import android.net.Uri +import android.util.Log +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Assume.assumeTrue +import org.junit.Test +import org.junit.runner.RunWith +import space.subread.app.video.VideoMaker +import space.subread.app.video.VideoSize +import space.subread.core.BookText +import java.io.File +import kotlin.math.abs +import kotlin.math.log10 +import kotlin.math.roundToInt + +/** + * The video, made on a device and then read back with the platform's own + * demuxer: the tracks a player will find, and how long each is. + * Uses the local media of OnDeviceJobTest, and skips without it. + */ +@RunWith(AndroidJUnit4::class) +class VideoMakerTest { + + @Test + fun fromAnMp3() = check(".mp3") // encoded to AAC + + @Test + fun fromAnM4b() = check(".m4b") // AAC, copied + + @Test + fun fullHdAtSixtyFramesASecond() = check(".m4b", VideoSize(1920, 1080), 60) + + /** On most devices a hardware encoder makes this one. */ + @Test + fun theLargestSizeOfThisDevice() = check(".m4b", VideoMaker.sizes().last(), 30) + + private fun check(suffix: String, size: VideoSize = VideoMaker.sizes().first(), fps: Int = 1) { + val instrumentation = InstrumentationRegistry.getInstrumentation() + val names = instrumentation.context.assets.list("local").orEmpty() + val audioName = names.firstOrNull { it.endsWith(suffix) } + val bookName = names.firstOrNull { it.endsWith(".epub") } + assumeTrue("no local test media", audioName != null && bookName != null) + + val app = instrumentation.targetContext + fun stage(name: String): File = File(app.cacheDir, name).also { out -> + instrumentation.context.assets.open("local/$name").use { src -> out.outputStream().use { src.copyTo(it) } } + } + val audio = stage(audioName!!) + val cover = stage(bookName!!).inputStream().use { BookText.cover(it) } + assertTrue("the epub has a cover", cover != null && cover.size > 1024) + + val tag = "video$suffix ${size.label}$fps" + val out = File(app.cacheDir, "video$suffix.${size.label}$fps.mp4").apply { delete(); createNewFile() } + val started = System.nanoTime() + VideoMaker.make(app, Uri.fromFile(audio), cover, Uri.fromFile(out), size, fps) {} + val seconds = (System.nanoTime() - started) / 1e9 + + val ex = MediaExtractor() + ex.setDataSource(out.absolutePath) + val tracks = (0 until ex.trackCount).associate { + val f = ex.getTrackFormat(it) + f.getString(MediaFormat.KEY_MIME)!! to f.getLong(MediaFormat.KEY_DURATION) + } + ex.release() + Log.i("SubReadTest", "$tag: ${out.length()} B (audio ${audio.length()} B) in %.1fs, tracks=$tracks".format(seconds)) + + assertEquals(setOf("video/avc", "audio/mp4a-latm"), tracks.keys) + val video = tracks.getValue("video/avc") + val sound = tracks.getValue("audio/mp4a-latm") + assertTrue("sound is ${sound / 1e6}s", sound in 70_000_000..80_000_000) + // The picture runs for the length of the sound, to within a frame. + assertTrue("video $video vs audio $sound", abs(video - sound) <= 1_500_000 / fps + 100_000) + + // Each frame decodes, and the last frame, many "no change" frames after + // the second key frame, is still the first picture. + val luma = decodeLuma(out) + assertEquals("frames", (video * fps / 1_000_000.0).roundToInt(), luma.count) + assertEquals(size.width to size.height, luma.width to luma.height) + assertTrue("the last frame is not the first frame", luma.first.contentEquals(luma.last)) + val wanted = VideoMaker.Yuv(VideoMaker.picture(cover, size), size).y + val psnr = psnr(wanted, luma.last) + Log.i("SubReadTest", "$tag: picture PSNR %.1f dB".format(psnr)) + assertTrue("the picture is poor: PSNR $psnr dB", psnr > 38) + + // A still picture must cost next to nothing. Only the key frames cost: one each + // minute, in proportion to the picture size. The frame rate costs 20 bytes a frame. + val videoBytes = trackBytes(out, "video/avc") + Log.i("SubReadTest", "$tag: video track $videoBytes B, file ${out.length() - audio.length()} B over the audio") + val keyFrame = 150_000L * size.width * size.height / (1280 * 720) + assertTrue("picture costs $videoBytes B", videoBytes < 2 * keyFrame + luma.count * 20) + } + + private class Luma(val count: Int, val width: Int, val height: Int, val first: ByteArray, val last: ByteArray) + + private fun trackBytes(file: File, mime: String): Long { + val ex = MediaExtractor() + ex.setDataSource(file.absolutePath) + ex.selectTrack((0 until ex.trackCount).first { ex.getTrackFormat(it).getString(MediaFormat.KEY_MIME) == mime }) + var total = 0L + while (ex.sampleSize >= 0) { total += ex.sampleSize; ex.advance() } + ex.release() + return total + } + + private fun decodeLuma(file: File): Luma { + val ex = MediaExtractor() + ex.setDataSource(file.absolutePath) + val track = (0 until ex.trackCount).first { ex.getTrackFormat(it).getString(MediaFormat.KEY_MIME) == "video/avc" } + ex.selectTrack(track) + val format = ex.getTrackFormat(track) + val width = format.getInteger(MediaFormat.KEY_WIDTH) + val height = format.getInteger(MediaFormat.KEY_HEIGHT) + val duration = format.getLong(MediaFormat.KEY_DURATION) + // Needed to read frames as an Image. + format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible) + val codec = MediaCodec.createDecoderByType("video/avc") + codec.configure(format, null, null, 0) + codec.start() + val info = MediaCodec.BufferInfo() + var count = 0 + var first: ByteArray? = null + var last = ByteArray(0) + var fed = false + var done = false + while (!done) { + if (!fed) { + val i = codec.dequeueInputBuffer(10_000) + if (i >= 0) { + val n = ex.readSampleData(codec.getInputBuffer(i)!!, 0) + if (n < 0) { codec.queueInputBuffer(i, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM); fed = true } + else { codec.queueInputBuffer(i, 0, n, ex.sampleTime, 0); ex.advance() } + } + } + val o = try { codec.dequeueOutputBuffer(info, 10_000) } catch (e: IllegalStateException) { + throw AssertionError("the decoder failed after $count frames, fed up to ${ex.sampleTime / 1_000_000}s", e) + } + if (o < 0) continue + // Copy the first frame and the frames of the last second only: a copy of each frame takes minutes. + if (info.size > 0 && first != null && info.presentationTimeUs < duration - 1_000_000) count++ + else if (info.size > 0) { + val plane = codec.getOutputImage(o)!!.planes[0] + val y = ByteArray(width * height) + for (row in 0 until height) { + plane.buffer.position(row * plane.rowStride) + plane.buffer.get(y, row * width, width) + } + if (first == null) first = y + last = y + count++ + } + done = info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0 + codec.releaseOutputBuffer(o, false) + } + codec.stop(); codec.release(); ex.release() + return Luma(count, width, height, first!!, last) + } + + private fun psnr(a: ByteArray, b: ByteArray): Double { + var sum = 0.0 + for (i in a.indices) { val d = (a[i].toInt() and 0xff) - (b[i].toInt() and 0xff); sum += d * d } + return 10 * log10(255.0 * 255.0 / (sum / a.size).coerceAtLeast(1e-9)) + } +} diff --git a/app/src/main/kotlin/space/subread/app/MainActivity.kt b/app/src/main/kotlin/space/subread/app/MainActivity.kt index 01360aa..e8aa732 100644 --- a/app/src/main/kotlin/space/subread/app/MainActivity.kt +++ b/app/src/main/kotlin/space/subread/app/MainActivity.kt @@ -41,11 +41,15 @@ import androidx.compose.ui.platform.LocalView import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.unit.dp import androidx.core.content.FileProvider +import androidx.core.content.edit import androidx.lifecycle.compose.collectAsStateWithLifecycle import space.subread.app.job.Job import space.subread.app.job.JobStatus import space.subread.app.job.Phase import space.subread.app.job.TranscriptStore +import space.subread.app.video.VideoExport +import space.subread.app.video.VideoMaker +import space.subread.app.video.VideoStatus import java.io.File import kotlin.concurrent.thread @@ -110,12 +114,24 @@ private fun App() { context.contentResolver.openOutputStream(dest)?.use { out -> srt.inputStream().use { it.copyTo(out) } } } } + val video by VideoExport.status.collectAsStateWithLifecycle() + val sizes = remember { VideoMaker.sizes() } + var videoSize by remember { mutableStateOf(sizes.firstOrNull { it.label == prefs.getString("video_size", null) } ?: sizes.first()) } + var videoFps by remember { mutableStateOf(prefs.getInt("video_fps", 1).takeIf { it in VideoMaker.FRAME_RATES } ?: 1) } + val pickVideoFolder = rememberLauncherForActivityResult(ActivityResultContracts.OpenDocumentTree()) { folder -> + val a = audio + val srt = status.srt + if (folder != null && a != null && srt != null) { + val app = context.applicationContext + thread(name = "subread-video") { VideoExport.run(app, a, book, srt, folder, videoSize, videoFps) } + } + } // The job runs only while this screen is open. Keep the display on, or the // system sleeps and stops the work. No background service: the job saves // each finished chunk, so an interrupted run continues from that chunk. val view = LocalView.current - DisposableEffect(status.running) { - view.keepScreenOn = status.running + DisposableEffect(status.running || video.running) { + view.keepScreenOn = status.running || video.running onDispose { view.keepScreenOn = false } } @@ -128,7 +144,8 @@ private fun App() { Text("SubRead", style = MaterialTheme.typography.headlineMedium, fontWeight = FontWeight.Bold) Text( "Times an audiobook against its ebook, on this device. Nothing is uploaded. " + - "The result is an .srt for Hoshi Reader's read-along.", + "The result is an .srt for Hoshi Reader's read-along, or a video + subtitles " + + "that any video player plays.", style = MaterialTheme.typography.bodyMedium, ) HorizontalDivider(color = Color.Black) @@ -172,9 +189,20 @@ private fun App() { "screen stays on while it works. If it is interrupted, Start continues from there.", style = MaterialTheme.typography.bodySmall) } - Outcome(status, + Outcome(status, video, + videoOptions = { + Choice("Size", sizes.map { it to it.label }, videoSize) { + videoSize = it + prefs.edit { putString("video_size", it.label) } + } + Choice("Frames a second", VideoMaker.FRAME_RATES.map { it to "$it" }, videoFps) { + videoFps = it + prefs.edit { putInt("video_fps", it) } + } + }, onSave = { status.srt?.let { saveSrt.launch(it.name) } }, - onShare = { status.srt?.let { share(context, it) } }) + onShare = { status.srt?.let { share(context, it) } }, + onVideo = { pickVideoFolder.launch(null) }) } } @@ -212,6 +240,22 @@ private fun LanguagePick(code: String, enabled: Boolean, onPick: (String) -> Uni } } +@Composable +private fun <T> Choice(label: String, options: List<Pair<T, String>>, chosen: T, onPick: (T) -> Unit) { + var open by remember { mutableStateOf(false) } + Column(verticalArrangement = Arrangement.spacedBy(4.dp)) { + Text(label, style = MaterialTheme.typography.labelLarge, fontWeight = FontWeight.Bold) + Box { + OutlinedButton(onClick = { open = true }) { Text(options.first { it.first == chosen }.second) } + DropdownMenu(expanded = open, onDismissRequest = { open = false }) { + for ((value, name) in options) { + DropdownMenuItem(text = { Text(name) }, onClick = { onPick(value); open = false }) + } + } + } + } +} + @Composable private fun Progress(status: JobStatus) { Column(verticalArrangement = Arrangement.spacedBy(8.dp)) { @@ -231,7 +275,10 @@ private fun Progress(status: JobStatus) { } @Composable -private fun Outcome(status: JobStatus, onSave: () -> Unit, onShare: () -> Unit) { +private fun Outcome( + status: JobStatus, video: VideoStatus, videoOptions: @Composable () -> Unit, + onSave: () -> Unit, onShare: () -> Unit, onVideo: () -> Unit, +) { when (status.phase) { Phase.DONE -> Column(verticalArrangement = Arrangement.spacedBy(10.dp)) { val rate = ((status.matchRate ?: 0.0) * 100).toInt() @@ -250,6 +297,25 @@ private fun Outcome(status: JobStatus, onSave: () -> Unit, onShare: () -> Unit) } Text("In Hoshi Reader: long-press the book, Match, and choose the saved .srt.", style = MaterialTheme.typography.bodySmall) + + HorizontalDivider(color = Color.Black) + Text("Video + subtitles", fontWeight = FontWeight.Bold) + Text("An .mp4 of the cover and the audio, with the .srt beside it. It plays with subtitles " + + "in any video player, and you can upload it to YouTube.", style = MaterialTheme.typography.bodySmall) + if (video.running) { + LinearProgressIndicator( + progress = { video.fraction }, + modifier = Modifier.fillMaxWidth(), + color = Color.Black, trackColor = Color(0xFFCCCCCC), + ) + Text("Making the video: ${(video.fraction * 100).toInt()}%", style = MaterialTheme.typography.bodySmall) + } else { + Row(horizontalArrangement = Arrangement.spacedBy(12.dp)) { videoOptions() } + Text("The picture is still, so more frames a second add almost nothing to the file. " + + "A larger size adds a little.", style = MaterialTheme.typography.bodySmall) + OutlinedButton(onClick = onVideo) { Text("Save video + .srt") } + if (video.message.isNotEmpty()) Text(video.message, style = MaterialTheme.typography.bodySmall) + } } Phase.FAILED -> Text("Failed: ${status.detail}", fontWeight = FontWeight.Bold) Phase.CANCELLED -> Text(status.detail) diff --git a/app/src/main/kotlin/space/subread/app/video/VideoExport.kt b/app/src/main/kotlin/space/subread/app/video/VideoExport.kt new file mode 100644 index 0000000..f321714 --- /dev/null +++ b/app/src/main/kotlin/space/subread/app/video/VideoExport.kt @@ -0,0 +1,51 @@ +package space.subread.app.video + +import android.content.Context +import android.net.Uri +import android.provider.DocumentsContract +import android.util.Log +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import space.subread.app.job.TranscriptStore +import space.subread.core.BookText +import java.io.File + +data class VideoStatus(val running: Boolean = false, val fraction: Float = 0f, val message: String = "") + +/** + * Writes the video and its subtitles into a folder the user picked. The two + * files have the same name, which is how a video player finds the subtitles. + */ +object VideoExport { + private val state = MutableStateFlow(VideoStatus()) + val status: StateFlow<VideoStatus> = state + + /** Blocks until the files are written; call from a worker thread. */ + fun run(context: Context, audio: Uri, book: Uri?, srt: File, folder: Uri, size: VideoSize, fps: Int) { + state.value = VideoStatus(running = true) + val resolver = context.contentResolver + var video: Uri? = null + try { + val parent = DocumentsContract.buildDocumentUriUsingTree(folder, DocumentsContract.getTreeDocumentId(folder)) + val base = TranscriptStore.describe(context, audio).first.substringBeforeLast('.') + video = DocumentsContract.createDocument(resolver, parent, "video/mp4", "$base.mp4") + ?: throw IllegalStateException("The folder did not accept a new file.") + // The folder can change the name (a file with this name is there already). The subtitles follow it. + val written = TranscriptStore.describe(context, video).first.substringBeforeLast('.') + + val cover = book?.let { uri -> runCatching { resolver.openInputStream(uri)?.use(BookText::cover) }.getOrNull() } + VideoMaker.make(context, audio, cover, video, size, fps) { state.value = VideoStatus(true, it) } + + val subtitles = DocumentsContract.createDocument(resolver, parent, "application/x-subrip", "$written.srt") + ?: throw IllegalStateException("The folder did not accept the subtitles file.") + resolver.openOutputStream(subtitles)!!.use { out -> srt.inputStream().use { it.copyTo(out) } } + + state.value = VideoStatus(message = "Saved $written.mp4 and $written.srt.") + } catch (e: Throwable) { + Log.w("SubRead", "video export failed", e) + // Half a video is of no use to anyone. + video?.let { runCatching { DocumentsContract.deleteDocument(resolver, it) } } + state.value = VideoStatus(message = "The video failed: ${e.message ?: e.javaClass.simpleName}") + } + } +} diff --git a/app/src/main/kotlin/space/subread/app/video/VideoMaker.kt b/app/src/main/kotlin/space/subread/app/video/VideoMaker.kt new file mode 100644 index 0000000..6bf56d7 --- /dev/null +++ b/app/src/main/kotlin/space/subread/app/video/VideoMaker.kt @@ -0,0 +1,479 @@ +package space.subread.app.video + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.graphics.Color +import android.graphics.Paint +import android.graphics.RectF +import android.media.MediaCodec +import android.media.MediaCodecInfo +import android.media.MediaCodecList +import android.media.MediaExtractor +import android.media.MediaFormat +import android.media.MediaMuxer +import android.net.Uri +import android.util.Log +import androidx.core.graphics.createBitmap +import space.subread.core.H264Still +import java.nio.ByteBuffer + +/** + * A video of the book: a still picture over the audiobook's own audio, as an + * MP4 that any player opens and YouTube accepts. The subtitles go beside it as + * an .srt with the same name, which players load by themselves. + * + * A still picture is cheap if nothing is encoded twice. The encoder makes one + * key frame. The frames after it say "no change" in about ten bytes each; they + * are written by hand (see H264Still), because an encoder spends kilobytes on + * the same frame. That minute is written again and again with new time stamps. + * So the frame rate costs almost nothing, and the picture size costs only in + * the one key frame of each minute. + * AAC audio is copied as it is; other audio is encoded to AAC once, which the + * MP4 container needs. + */ +class VideoSize(val width: Int, val height: Int) { + val label: String get() = "${height}p" +} + +object VideoMaker { + private const val TAG = "SubRead" + + /** Sizes to offer. A device makes only those for which it has an encoder: see [sizes]. */ + private val SIZES = listOf(VideoSize(1280, 720), VideoSize(1920, 1080), VideoSize(2560, 1440), + VideoSize(3840, 2160), VideoSize(7680, 4320)) + val FRAME_RATES = listOf(1, 24, 30, 60) + private const val GOP_SECONDS = 60 + private const val KEY_FRAME_QP = 26 + + private class Frame(val bytes: ByteArray, val key: Boolean) + private class Sample(val bytes: ByteArray, val ptsUs: Long) + + /** The sizes this device can make, smallest first. The first one is the default. */ + fun sizes(): List<VideoSize> = SIZES.filter { encoderFor(it) != null }.ifEmpty { SIZES.take(1) } + + /** + * @param cover image bytes, or null for a plain card + * @param size one of [sizes] + * @param fps frames in a second, 1 to 60 + */ + fun make( + context: Context, audio: Uri, cover: ByteArray?, dest: Uri, + size: VideoSize = SIZES[0], fps: Int = 1, onProgress: (Float) -> Unit, + ) { + require(fps in 1..60) { "fps $fps" } + val yuv = Yuv(picture(cover, size), size) + var rate = fps + var (videoFormat, gop) = encodeStill(yuv, size, 1) + try { + val skips = H264Still.skipFrames(bytes(videoFormat, "csd-0"), bytes(videoFormat, "csd-1"), GOP_SECONDS * fps - 1) + gop = gop + skips.map { Frame(it, false) } + } catch (e: H264Still.Unsupported) { + // Bigger and at one frame a second, but correct: the encoder's own frames. + Log.w(TAG, "still: no hand-made frames for this stream (${e.message})") + rate = 1 + encodeStill(yuv, size, GOP_SECONDS).let { videoFormat = it.first; gop = it.second } + } + val level = H264Still.level(size.width, size.height, rate) + videoFormat.setByteBuffer("csd-0", ByteBuffer.wrap(H264Still.withLevel(bytes(videoFormat, "csd-0"), level))) + Log.i(TAG, "still: ${size.width}x${size.height} at $rate fps, level $level, ${gop.size} frames, " + + "key ${gop[0].bytes.size} B, the rest ${gop.drop(1).sumOf { it.bytes.size }} B") + + val extractor = MediaExtractor() + extractor.setDataSource(context, audio, null) + val track = (0 until extractor.trackCount).firstOrNull { + extractor.getTrackFormat(it).getString(MediaFormat.KEY_MIME).orEmpty().startsWith("audio/") + } ?: throw IllegalArgumentException("No audio track in this file.") + extractor.selectTrack(track) + val source = extractor.getTrackFormat(track) + val durationUs = if (source.containsKey(MediaFormat.KEY_DURATION)) source.getLong(MediaFormat.KEY_DURATION) else 0L + + val sound: AudioSource = + if (source.getString(MediaFormat.KEY_MIME) == MediaFormat.MIMETYPE_AUDIO_AAC) Copy(extractor, source) + else Transcode(extractor, source) + + context.contentResolver.openFileDescriptor(dest, "rw")!!.use { pfd -> + val muxer = MediaMuxer(pfd.fileDescriptor, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4) + try { + val v = muxer.addTrack(videoFormat) + val a = muxer.addTrack(sound.format()) + muxer.start() + + val info = MediaCodec.BufferInfo() + var frame = 0L + var frameUs = 0L + fun picture() { + val f = gop[(frame % gop.size).toInt()] + info.set(0, f.bytes.size, frameUs, if (f.key) MediaCodec.BUFFER_FLAG_KEY_FRAME else 0) + muxer.writeSampleData(v, ByteBuffer.wrap(f.bytes), info) + frame++ + frameUs = frame * 1_000_000 / rate + } + + var lastUs = 0L + while (true) { + val s = sound.next() ?: break + // Keep the two tracks side by side in the file, so a player + // does not have to jump about to find the next of either. + while (frameUs <= s.ptsUs) picture() + info.set(0, s.bytes.size, s.ptsUs, MediaCodec.BUFFER_FLAG_KEY_FRAME) + muxer.writeSampleData(a, ByteBuffer.wrap(s.bytes), info) + lastUs = s.ptsUs + if (durationUs > 0) onProgress((s.ptsUs.toFloat() / durationUs).coerceIn(0f, 1f)) + } + while (frameUs <= lastUs) picture() + muxer.stop() + } finally { + runCatching { muxer.release() } + sound.close() + extractor.release() + } + } + } + + private val AVC_LEVELS = mapOf( + 31 to MediaCodecInfo.CodecProfileLevel.AVCLevel31, 32 to MediaCodecInfo.CodecProfileLevel.AVCLevel32, + 40 to MediaCodecInfo.CodecProfileLevel.AVCLevel4, 42 to MediaCodecInfo.CodecProfileLevel.AVCLevel42, + 50 to MediaCodecInfo.CodecProfileLevel.AVCLevel5, 51 to MediaCodecInfo.CodecProfileLevel.AVCLevel51, + 52 to MediaCodecInfo.CodecProfileLevel.AVCLevel52, + // AVCLevel6, AVCLevel61 and AVCLevel62. The names are in API 29 and later; the values are the same on each version. + 60 to 0x20000, 61 to 0x40000, 62 to 0x80000, + ) + + private fun bytes(format: MediaFormat, key: String): ByteArray { + val buffer = format.getByteBuffer(key) ?: throw H264Still.Unsupported("no $key") + return ByteArray(buffer.remaining()).also { buffer.duplicate().get(it) } + } + + // ------------------------------------------------------------------ audio + + private interface AudioSource { + fun format(): MediaFormat + fun next(): Sample? + fun close() {} + } + + /** AAC goes into the MP4 untouched. */ + private class Copy(val extractor: MediaExtractor, val source: MediaFormat) : AudioSource { + private val buffer = ByteBuffer.allocate(1 shl 18) + override fun format() = source + override fun next(): Sample? { + val n = extractor.readSampleData(buffer, 0) + if (n < 0) return null + val bytes = ByteArray(n) + buffer.position(0) + buffer.get(bytes, 0, n) + val s = Sample(bytes, extractor.sampleTime) + extractor.advance() + return s + } + } + + /** + * MP3, Opus and the rest: decoded and encoded to AAC, once. + * + * Each call into a codec costs about a millisecond on a slow device, and + * that cost, not the arithmetic, sets the speed. So the decoder has its own + * thread and works while the encoder works, and the encoder gets the sound + * in large pieces: one call for many frames. + */ + private class Transcode(val extractor: MediaExtractor, source: MediaFormat) : AudioSource { + private val decoder = MediaCodec.createDecoderByType(source.getString(MediaFormat.KEY_MIME)!!).apply { + configure(source, null, null, 0); start() + } + @Volatile private var rate = source.getInteger(MediaFormat.KEY_SAMPLE_RATE) + @Volatile private var channels = source.getInteger(MediaFormat.KEY_CHANNEL_COUNT) + @Volatile private var failure: Throwable? = null + @Volatile private var closed = false + private val pieces = java.util.concurrent.ArrayBlockingQueue<ByteArray>(8) + private val decoding = kotlin.concurrent.thread(name = "subread-audio-decode") { + try { decode() } catch (e: Throwable) { if (!closed) failure = e } + runCatching { pieces.put(END) } + } + + private var encoder: MediaCodec? = null + private var outFormat: MediaFormat? = null + private val info = MediaCodec.BufferInfo() + private var piece: ByteArray? = null + private var pieceAt = 0 + private var pcmEnded = false + private var encoderEnded = false + private var framesIn = 0L + private val ready = ArrayDeque<Sample>() + private var finished = false + + override fun format(): MediaFormat { + while (outFormat == null && !finished) pump() + return outFormat ?: throw IllegalStateException("The audio could not be encoded.") + } + + override fun next(): Sample? { + while (ready.isEmpty() && !finished) pump() + return ready.removeFirstOrNull() + } + + /** The decoder's thread: compressed audio in, PCM out in pieces of about [PIECE] bytes. */ + private fun decode() { + val info = MediaCodec.BufferInfo() + val pcm = java.io.ByteArrayOutputStream(PIECE + (1 shl 14)) + var extractorDone = false + while (!closed) { + // Each free input buffer gets a frame, so the decoder never waits for this loop. + while (!extractorDone) { + val i = decoder.dequeueInputBuffer(0) + if (i < 0) break + val n = extractor.readSampleData(decoder.getInputBuffer(i)!!, 0) + if (n < 0) { + decoder.queueInputBuffer(i, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM) + extractorDone = true + } else { + decoder.queueInputBuffer(i, 0, n, 0, 0) + extractor.advance() + } + } + val o = decoder.dequeueOutputBuffer(info, 10_000) + if (o == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + // The container can be wrong about these; the decoder is not. + rate = decoder.outputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE) + channels = decoder.outputFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT) + } + if (o < 0) continue + if (info.size > 0) { + val bytes = ByteArray(info.size) + decoder.getOutputBuffer(o)!!.apply { position(info.offset); get(bytes) } + pcm.write(bytes) + } + val end = info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0 + decoder.releaseOutputBuffer(o, false) + if (pcm.size() >= PIECE || (end && pcm.size() > 0)) { + pieces.put(pcm.toByteArray()) + pcm.reset() + } + if (end) return + } + } + + private fun startEncoder(): MediaCodec { + val f = MediaFormat.createAudioFormat(MediaFormat.MIMETYPE_AUDIO_AAC, rate, channels).apply { + setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC) + setInteger(MediaFormat.KEY_BIT_RATE, if (channels == 1) 80_000 else 128_000) + setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, PIECE) + } + return MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC).apply { + configure(f, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); start() + }.also { encoder = it } + } + + /** One step of the encoder's side: take encoded audio out, or put PCM in. */ + private fun pump() { + encoder?.let { enc -> + // Wait only when there is nothing to put in. + val o = enc.dequeueOutputBuffer(info, if (piece != null || pieces.isNotEmpty()) 0 else 10_000) + if (o == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) outFormat = enc.outputFormat + else if (o >= 0) { + if (info.size > 0 && info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG == 0) { + val bytes = ByteArray(info.size) + enc.getOutputBuffer(o)!!.apply { position(info.offset); get(bytes) } + ready += Sample(bytes, info.presentationTimeUs) + } + val end = info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0 + enc.releaseOutputBuffer(o, false) + if (end) finished = true + return + } + } + + if (piece == null && !pcmEnded) { + val next = pieces.poll(10, java.util.concurrent.TimeUnit.MILLISECONDS) ?: return + if (next === END) { + pcmEnded = true + failure?.let { throw it } + } else { + piece = next + pieceAt = 0 + } + } + if (encoderEnded) return + + val enc = encoder ?: startEncoder() + val i = enc.dequeueInputBuffer(10_000) + if (i < 0) return + val ptsUs = framesIn * 1_000_000 / rate + val pcm = piece + if (pcm != null) { + val buf = enc.getInputBuffer(i)!! + val frame = 2 * channels + val n = minOf(buf.capacity() / frame * frame, pcm.size - pieceAt) + buf.clear() + buf.put(pcm, pieceAt, n) + enc.queueInputBuffer(i, 0, n, ptsUs, 0) + pieceAt += n + framesIn += n / frame + if (pieceAt >= pcm.size) piece = null + } else { + enc.queueInputBuffer(i, 0, 0, ptsUs, MediaCodec.BUFFER_FLAG_END_OF_STREAM) + encoderEnded = true + } + } + + override fun close() { + closed = true + pieces.clear() // so a decoder that waits to put a piece can go on, and stop + decoding.join(2_000) + runCatching { decoder.stop() }; decoder.release() + encoder?.let { runCatching { it.stop() }; it.release() } + } + + private companion object { + /** About a third of a second of stereo sound. */ + const val PIECE = 1 shl 16 + val END = ByteArray(0) + } + } + + // ---------------------------------------------------------------- picture + + /** The cover, letterboxed on a black card; a plain dark card when there is none. */ + internal fun picture(cover: ByteArray?, size: VideoSize): Bitmap { + val width = size.width + val height = size.height + val card = createBitmap(width, height) + val canvas = android.graphics.Canvas(card) + canvas.drawColor(Color.rgb(0x16, 0x16, 0x1a)) + val image = cover?.let { runCatching { BitmapFactory.decodeByteArray(it, 0, it.size) }.getOrNull() } + if (image != null) { + canvas.drawColor(Color.BLACK) + val scale = minOf(width.toFloat() / image.width, height.toFloat() / image.height) + val w = image.width * scale + val h = image.height * scale + canvas.drawBitmap(image, null, RectF((width - w) / 2, (height - h) / 2, (width + w) / 2, (height + h) / 2), + Paint(Paint.FILTER_BITMAP_FLAG)) + } + return card + } + + /** + * The encoder for a picture of this size. The software encoder is first: + * it is the same on each device. A hardware encoder makes the larger sizes. + */ + private fun encoderFor(size: VideoSize): String? = + MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos + .filter { c -> c.isEncoder && c.supportedTypes.any { it.equals(MediaFormat.MIMETYPE_VIDEO_AVC, true) } } + .filter { c -> + runCatching { + c.getCapabilitiesForType(MediaFormat.MIMETYPE_VIDEO_AVC).videoCapabilities.isSizeSupported(size.width, size.height) + }.getOrDefault(false) + } + .sortedBy { c -> if (c.name.startsWith("c2.android.") || c.name.startsWith("OMX.google.")) 0 else 1 } + .firstOrNull()?.name + + /** + * The first [count] seconds of the picture at one frame a second. The + * frames go in as YUV buffers: that is the input each encoder accepts + * (the MediaTek encoder fails on frames drawn with a Canvas). + */ + private fun encodeStill(yuv: Yuv, size: VideoSize, count: Int): Pair<MediaFormat, List<Frame>> { + val width = size.width + val height = size.height + val name = encoderFor(size) ?: throw IllegalStateException("This device has no H.264 encoder for ${size.label} video.") + + val format = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, width, height).apply { + setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible) + setInteger(MediaFormat.KEY_BIT_RATE, 600_000) + setInteger(MediaFormat.KEY_FRAME_RATE, 1) + setInteger("video-qp-i-min", KEY_FRAME_QP) + setInteger("video-qp-i-max", KEY_FRAME_QP) + // Far longer than the clip: only the first frame is a key frame. + setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 3600) + // Baseline: each player decodes it, and H264Still can write frames for it. + setInteger(MediaFormat.KEY_PROFILE, MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline) + // The level for one frame a second. make() writes the true level afterwards. + setInteger(MediaFormat.KEY_LEVEL, AVC_LEVELS.getValue(H264Still.level(width, height, 1))) + } + val codec = MediaCodec.createByCodecName(name) + codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) + codec.start() + val info = MediaCodec.BufferInfo() + val frames = ArrayList<Frame>() + var outFormat: MediaFormat? = null + var queued = 0 + var ended = false + try { + while (!ended) { + if (queued <= count) { + val i = codec.dequeueInputBuffer(10_000) + if (i >= 0) { + if (queued == count) { + codec.queueInputBuffer(i, 0, 0, queued * 1_000_000L, MediaCodec.BUFFER_FLAG_END_OF_STREAM) + } else { + yuv.fill(codec.getInputImage(i)!!) + codec.queueInputBuffer(i, 0, width * height * 3 / 2, queued * 1_000_000L, 0) + } + queued++ + } + } + val o = codec.dequeueOutputBuffer(info, 10_000) + if (o == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) outFormat = codec.outputFormat + if (o < 0) continue + if (info.size > 0 && info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG == 0) { + val bytes = ByteArray(info.size) + codec.getOutputBuffer(o)!!.apply { position(info.offset); get(bytes) } + frames += Frame(bytes, info.flags and MediaCodec.BUFFER_FLAG_KEY_FRAME != 0) + } + ended = info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0 + codec.releaseOutputBuffer(o, false) + } + } finally { + runCatching { codec.stop() } + codec.release() + } + check(frames.size == count && frames[0].key) { "The video encoder made ${frames.size} of $count frames." } + check(frames.drop(1).none { it.key }) { "The video encoder made more than one key frame." } + return (outFormat ?: throw IllegalStateException("The video encoder gave no format.")) to frames + } + + /** The picture as Y, U and V planes (BT.601, video range), made once and copied into each frame. */ + internal class Yuv(picture: Bitmap, size: VideoSize) { + private val width = size.width + private val height = size.height + val y = ByteArray(width * height) + private val u = ByteArray(width * height / 4) + private val v = ByteArray(width * height / 4) + + init { + val argb = IntArray(width * height) + picture.getPixels(argb, 0, width, 0, 0, width, height) + for (row in 0 until height) for (col in 0 until width) { + val c = argb[row * width + col] + val r = (c shr 16) and 0xff + val g = (c shr 8) and 0xff + val b = c and 0xff + y[row * width + col] = (((66 * r + 129 * g + 25 * b + 128) shr 8) + 16).toByte() + if (row % 2 == 0 && col % 2 == 0) { + val k = (row / 2) * (width / 2) + col / 2 + u[k] = (((-38 * r - 74 * g + 112 * b + 128) shr 8) + 128).toByte() + v[k] = (((112 * r - 94 * g - 18 * b + 128) shr 8) + 128).toByte() + } + } + } + + /** Planar or semi-planar, padded or not: the Image says, and this follows. */ + fun fill(image: android.media.Image) { + copy(y, width, height, image.planes[0]) + copy(u, width / 2, height / 2, image.planes[1]) + copy(v, width / 2, height / 2, image.planes[2]) + } + + private fun copy(src: ByteArray, w: Int, h: Int, plane: android.media.Image.Plane) { + val buffer = plane.buffer + if (plane.pixelStride == 1 && plane.rowStride == w) { + buffer.position(0) + buffer.put(src, 0, w * h) + return + } + for (row in 0 until h) for (col in 0 until w) { + buffer.put(row * plane.rowStride + col * plane.pixelStride, src[row * w + col]) + } + } + } +}