commit 4545f4ebea19ebc41bc6da5a6eaadd58038ac722 equwal <truex@equwal.com> 2026-09-21 01:15:58 -0700 Add H264Still: hand-made "no change" frames for a still video An encoder spends kilobytes on a frame that shows no change, because its rate control has bits to use. H264Still writes that frame as one P slice of skipped macroblocks, about ten bytes. It also gives the H.264 level for a picture size and frame rate, and raises the level of a parameter set. The test gives the frames to ffmpeg. Each frame must decode to the picture of the key frame.
.../main/kotlin/space/subread/core/H264Still.kt | 227 +++++++++++++++++++++ .../kotlin/space/subread/core/H264StillTest.kt | 142 +++++++++++++ 2 files changed, 369 insertions(+)
diff --git a/core/src/main/kotlin/space/subread/core/H264Still.kt b/core/src/main/kotlin/space/subread/core/H264Still.kt new file mode 100644 index 0000000..edb537f --- /dev/null +++ b/core/src/main/kotlin/space/subread/core/H264Still.kt @@ -0,0 +1,227 @@ +package space.subread.core + +import java.io.ByteArrayOutputStream + +/** + * "No change" frames for an H.264 video of a still picture. + * + * An encoder is given the picture once and makes one key frame. Each frame + * after it is written here by hand: a P slice in which each macroblock is + * skipped. A decoder shows the key frame again, bit for bit, and the frame + * costs about ten bytes. An encoder asked for the same frame spends kilobytes + * on it, because its rate control has bits to use. + * + * Only streams with CAVLC entropy coding, one slice group and frame + * macroblocks are handled (Baseline profile is always like that). For another + * stream, [skipFrames] throws [Unsupported] and the caller must use the + * encoder's own frames. + */ +object H264Still { + class Unsupported(message: String) : Exception(message) + + private val HIGH_PROFILES = setOf(100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134, 135) + + /** + * The frames that follow a key frame, in order. Frame `i` of the result is + * frame `i + 1` of the group of pictures. Each is one NAL unit with a + * four-byte start code, as MediaCodec gives and MediaMuxer takes. + * + * @param sps the sequence parameter set, with or without a start code + * @param pps the picture parameter set, with or without a start code + */ + fun skipFrames(sps: ByteArray, pps: ByteArray, count: Int): List<ByteArray> { + try { + val s = Sps(BitReader(unescape(payload(sps, 7)))) + val p = Pps(BitReader(unescape(payload(pps, 8)))) + return (1..count).map { slice(s, p, it) } + } catch (e: IndexOutOfBoundsException) { + throw Unsupported("The parameter sets are cut short.") + } + } + + /** level_idc, the largest frame in macroblocks, the most macroblocks in a second (H.264 table A-1). */ + private val LEVELS = listOf( + Triple(31, 3_600, 108_000), Triple(32, 5_120, 216_000), Triple(40, 8_192, 245_760), + Triple(42, 8_704, 522_240), Triple(50, 22_080, 589_824), Triple(51, 36_864, 983_040), + Triple(52, 36_864, 2_073_600), Triple(60, 139_264, 4_177_920), Triple(61, 139_264, 8_355_840), + Triple(62, 139_264, 16_711_680), + ) + + /** The lowest level that allows this picture size at this frame rate. */ + fun level(width: Int, height: Int, fps: Int): Int { + val macroblocks = ((width + 15) / 16) * ((height + 15) / 16) + return LEVELS.firstOrNull { (_, frame, second) -> macroblocks <= frame && macroblocks.toLong() * fps <= second }?.first + ?: throw Unsupported("no H.264 level allows ${width}x$height at $fps frames a second") + } + + /** + * The sequence parameter set with its level raised to [level]. The encoder + * sees one frame and declares a level for that; the file holds more frames + * in a second, and a decoder must be told the truth. Never lowers a level. + */ + fun withLevel(sps: ByteArray, level: Int): ByteArray { + val at = sps.size - payload(sps, 7).size + 2 // profile_idc, constraint flags, level_idc + val out = sps.copyOf() + if (out[at].toInt() and 0xff < level) out[at] = level.toByte() + return out + } + + private class Sps(r: BitReader) { + val log2MaxFrameNum: Int + val pocType: Int + var log2MaxPocLsb = 0 + var deltaPocAlwaysZero = false + val macroblocks: Int + + init { + val profile = r.bits(8) + r.bits(16) // constraint flags, level + r.ue() // seq_parameter_set_id + if (profile in HIGH_PROFILES) { + val chroma = r.ue() + if (chroma == 3 && r.bit() == 1) throw Unsupported("separate colour planes") + r.ue(); r.ue(); r.bit() // bit depths, transform bypass + if (r.bit() == 1) throw Unsupported("scaling matrix") + } + log2MaxFrameNum = r.ue() + 4 + pocType = r.ue() + when (pocType) { + 0 -> log2MaxPocLsb = r.ue() + 4 + 1 -> { + deltaPocAlwaysZero = r.bit() == 1 + r.se(); r.se() + repeat(r.ue()) { r.se() } + } + } + r.ue(); r.bit() // max_num_ref_frames, gaps allowed + val width = r.ue() + 1 + val height = r.ue() + 1 + if (r.bit() == 0) throw Unsupported("field macroblocks") + macroblocks = width * height + } + } + + private class Pps(r: BitReader) { + val id = r.ue() + val bottomFieldPoc: Boolean + val deblockingControl: Boolean + val redundantPicCnt: Boolean + + init { + r.ue() // seq_parameter_set_id + if (r.bit() == 1) throw Unsupported("CABAC") + bottomFieldPoc = r.bit() == 1 + if (r.ue() != 0) throw Unsupported("slice groups") + r.ue(); r.ue() // default reference counts + if (r.bit() == 1) throw Unsupported("weighted prediction") + r.bits(2) + r.se(); r.se(); r.se() // initial QP, QS, chroma QP offset + deblockingControl = r.bit() == 1 + r.bit() + redundantPicCnt = r.bit() == 1 + } + } + + /** Frame [n] after the key frame: slice header, one run of skipped macroblocks, end. */ + private fun slice(s: Sps, p: Pps, n: Int): ByteArray { + val w = BitWriter() + w.ue(0) // first_mb_in_slice + w.ue(5) // slice_type: P, and so is every slice of the picture + w.ue(p.id) + w.bits(n % (1 shl s.log2MaxFrameNum), s.log2MaxFrameNum) + if (s.pocType == 0) { + w.bits((2 * n) % (1 shl s.log2MaxPocLsb), s.log2MaxPocLsb) + if (p.bottomFieldPoc) w.se(0) + } + if (s.pocType == 1 && !s.deltaPocAlwaysZero) { + w.se(0) + if (p.bottomFieldPoc) w.se(0) + } + if (p.redundantPicCnt) w.ue(0) + w.bit(1); w.ue(0) // one reference picture: the frame before + w.bit(0) // no reference list modification + w.bit(0) // sliding-window reference marking + w.se(0) // slice_qp_delta + if (p.deblockingControl) w.ue(1) // no deblocking: there is nothing to smooth + w.ue(s.macroblocks) // mb_skip_run: all of them + val rbsp = w.trailing() + + val out = ByteArrayOutputStream() + out.write(byteArrayOf(0, 0, 0, 1, 0x41)) // start code; reference picture, non-IDR slice + out.write(escape(rbsp)) + return out.toByteArray() + } + + /** The bytes after the NAL header, which must be of [type]. */ + private fun payload(nal: ByteArray, type: Int): ByteArray { + var at = 0 + while (at < nal.size && nal[at].toInt() == 0) at++ + if (at > 0 && at < nal.size && nal[at].toInt() == 1) at++ else at = 0 + if (at >= nal.size || nal[at].toInt() and 0x1f != type) throw Unsupported("not a NAL unit of type $type") + return nal.copyOfRange(at + 1, nal.size) + } + + /** Puts in the 0x03 bytes that keep a start code from showing up inside a NAL unit. */ + internal fun escape(rbsp: ByteArray): ByteArray { + val out = ByteArrayOutputStream(rbsp.size + 4) + var zeros = 0 + for (b in rbsp) { + if (zeros >= 2 && b.toInt() and 0xff <= 3) { out.write(3); zeros = 0 } + out.write(b.toInt()) + zeros = if (b.toInt() == 0) zeros + 1 else 0 + } + return out.toByteArray() + } + + internal fun unescape(nal: ByteArray): ByteArray { + val out = ByteArrayOutputStream(nal.size) + var zeros = 0 + for (b in nal) { + if (zeros >= 2 && b.toInt() == 3) { zeros = 0; continue } + out.write(b.toInt()) + zeros = if (b.toInt() == 0) zeros + 1 else 0 + } + return out.toByteArray() + } + + internal class BitReader(private val data: ByteArray) { + private var at = 0 + fun bit(): Int { + if (at shr 3 >= data.size) throw IndexOutOfBoundsException() + val b = (data[at shr 3].toInt() shr (7 - (at and 7))) and 1 + at++ + return b + } + fun bits(n: Int): Int { var v = 0; repeat(n) { v = (v shl 1) or bit() }; return v } + fun ue(): Int { + var zeros = 0 + while (bit() == 0) if (++zeros > 31) throw Unsupported("bad Exp-Golomb code") + return (1 shl zeros) - 1 + bits(zeros) + } + fun se(): Int { val k = ue(); return if (k and 1 == 1) (k + 1) / 2 else -(k / 2) } + } + + internal class BitWriter { + private val out = ByteArrayOutputStream() + private var current = 0 + private var filled = 0 + fun bit(b: Int) { + current = (current shl 1) or (b and 1) + if (++filled == 8) { out.write(current); current = 0; filled = 0 } + } + fun bits(v: Int, count: Int) { for (i in count - 1 downTo 0) bit(v shr i) } + fun ue(v: Int) { + val x = v + 1 + val length = 32 - Integer.numberOfLeadingZeros(x) + bits(0, length - 1) + bits(x, length) + } + fun se(v: Int) = ue(if (v > 0) 2 * v - 1 else -2 * v) + /** The stop bit, then zeros to the end of the byte. */ + fun trailing(): ByteArray { + bit(1) + while (filled != 0) bit(0) + return out.toByteArray() + } + } +} diff --git a/core/src/test/kotlin/space/subread/core/H264StillTest.kt b/core/src/test/kotlin/space/subread/core/H264StillTest.kt new file mode 100644 index 0000000..265fa43 --- /dev/null +++ b/core/src/test/kotlin/space/subread/core/H264StillTest.kt @@ -0,0 +1,142 @@ +package space.subread.core + +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Assume.assumeTrue +import org.junit.Test +import java.io.File +import java.nio.file.Files +import kotlin.random.Random + +class H264StillTest { + + @Test + fun expGolombCodesRoundTrip() { + val random = Random(1) + repeat(2000) { + val unsigned = List(20) { if (it % 2 == 0) random.nextInt(0, 40) else random.nextInt(0, 1 shl 20) } + val signed = unsigned.map { if (random.nextBoolean()) it else -it } + val w = H264Still.BitWriter() + unsigned.forEach(w::ue) + signed.forEach(w::se) + val r = H264Still.BitReader(w.trailing()) + assertEquals(unsigned, List(unsigned.size) { r.ue() }) + assertEquals(signed, List(signed.size) { r.se() }) + } + } + + @Test + fun escapedBytesRoundTripAndHoldNoStartCode() { + val random = Random(2) + repeat(5000) { + // Mostly small values: runs of zeros are what the escape is for. + val rbsp = ByteArray(random.nextInt(0, 40)) { if (random.nextInt(4) == 0) random.nextInt(256).toByte() else random.nextInt(4).toByte() } + val nal = H264Still.escape(rbsp) + assertArrayEquals(rbsp, H264Still.unescape(nal)) + for (i in 0..nal.size - 3) { + assertFalse("start code in ${nal.toList()}", nal[i].toInt() == 0 && nal[i + 1].toInt() == 0 && nal[i + 2].toInt() in 0..2) + } + } + } + + /** A real decoder is the judge: each hand-made frame must decode to the key frame's exact picture. */ + @Test + fun aDecoderShowsTheKeyFrameAgainForEachSkipFrame() { + val dir = Files.createTempDirectory("h264still").toFile() + try { + val key = encodeKeyFrame(dir, "baseline") + val nals = split(key) + val sps = nals.first { it[0].toInt() and 0x1f == 7 } + val pps = nals.first { it[0].toInt() and 0x1f == 8 } + + // 70 frames: more than frame_num and the picture order count can hold, so both wrap. + val frames = H264Still.skipFrames(sps, pps, 70) + assertTrue("a skip frame is ${frames.maxOf { it.size }} bytes", frames.all { it.size <= 16 }) + val stream = File(dir, "still.h264") + stream.writeBytes(key + frames.reduce { a, b -> a + b }) + + val (code, out, err) = run(dir, "ffmpeg", "-v", "error", "-xerror", "-i", stream.name, "-f", "framemd5", "-") + assertEquals(err, 0, code) + assertEquals("decoder messages", "", err.trim()) + val hashes = out.lines().filter { it.isNotBlank() && !it.startsWith("#") }.map { it.substringAfterLast(',').trim() } + assertEquals(71, hashes.size) + assertEquals("each frame is the same picture", 1, hashes.toSet().size) + } finally { + dir.deleteRecursively() + } + } + + @Test + fun aStreamWithCabacIsRefused() { + val dir = Files.createTempDirectory("h264still").toFile() + try { + val nals = split(encodeKeyFrame(dir, "main")) + val sps = nals.first { it[0].toInt() and 0x1f == 7 } + val pps = nals.first { it[0].toInt() and 0x1f == 8 } + assertThrows(H264Still.Unsupported::class.java) { H264Still.skipFrames(sps, pps, 1) } + } finally { + dir.deleteRecursively() + } + } + + @Test + fun theLevelFollowsThePictureSizeAndTheFrameRate() { + assertEquals(31, H264Still.level(1280, 720, 1)) + assertEquals(31, H264Still.level(1280, 720, 30)) + assertEquals(32, H264Still.level(1280, 720, 60)) + assertEquals(40, H264Still.level(1920, 1080, 30)) + assertEquals(42, H264Still.level(1920, 1080, 60)) + assertEquals(50, H264Still.level(2560, 1440, 30)) + assertEquals(51, H264Still.level(2560, 1440, 60)) + assertEquals(52, H264Still.level(3840, 2160, 60)) + assertEquals(61, H264Still.level(7680, 4320, 60)) + assertThrows(H264Still.Unsupported::class.java) { H264Still.level(16384, 8640, 60) } + } + + @Test + fun theLevelOfAParameterSetIsRaisedAndNeverLowered() { + val sps = byteArrayOf(0, 0, 0, 1, 0x67, 0x42, 0xC0.toByte(), 31, 0x55) + assertArrayEquals(byteArrayOf(0, 0, 0, 1, 0x67, 0x42, 0xC0.toByte(), 42, 0x55), H264Still.withLevel(sps, 42)) + assertArrayEquals(sps, H264Still.withLevel(sps, 30)) + assertArrayEquals(byteArrayOf(0x67, 0x42, 0, 51), H264Still.withLevel(byteArrayOf(0x67, 0x42, 0, 40), 51)) + } + + @Test + fun cutParameterSetsAreRefused() { + assertThrows(H264Still.Unsupported::class.java) { H264Still.skipFrames(byteArrayOf(0x67, 0x42), byteArrayOf(0x68), 1) } + assertThrows(H264Still.Unsupported::class.java) { H264Still.skipFrames(byteArrayOf(0x68, 0x42), byteArrayOf(0x68), 1) } + } + + /** One key frame of a test card. Skips the test on a machine with no ffmpeg that can encode H.264. */ + private fun encodeKeyFrame(dir: File, profile: String): ByteArray { + val cabac = if (profile == "main") arrayOf("-coder", "cabac") else emptyArray() + val made = listOf("libx264" to profile, "libopenh264" to profile.replace("baseline", "constrained_baseline")).any { (encoder, name) -> + runCatching { + run(dir, "ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i", "testsrc=size=320x240:rate=1", "-frames:v", "1", + "-c:v", encoder, "-profile:v", name, *cabac, "-pix_fmt", "yuv420p", "-f", "h264", "key.h264").first == 0 + }.getOrDefault(false) + } + assumeTrue("no ffmpeg with an H.264 encoder on the PATH", made) + return File(dir, "key.h264").readBytes() + } + + private fun run(dir: File, vararg command: String): Triple<Int, String, String> { + val out = File(dir, "stdout.txt") + val err = File(dir, "stderr.txt") + val process = ProcessBuilder(*command).directory(dir).redirectOutput(out).redirectError(err).start() + return Triple(process.waitFor(), out.readText(), err.readText()) + } + + /** NAL units of an Annex B stream, without their start codes. */ + private fun split(stream: ByteArray): List<ByteArray> { + val starts = (0..stream.size - 3).filter { stream[it].toInt() == 0 && stream[it + 1].toInt() == 0 && stream[it + 2].toInt() == 1 } + return starts.mapIndexed { k, at -> + var end = if (k + 1 < starts.size) starts[k + 1] else stream.size + while (end > at + 3 && stream[end - 1].toInt() == 0) end-- + stream.copyOfRange(at + 3, end) + } + } +}