commit 2555c531690ddd578656af1da2d6d90d3c5f9dcb equwal <truex@equwal.com> 2026-09-20 17:04:09 -0700 Android app: on-device transcription, alignment and .srt export - whisper.cpp v1.9.4 as a pinned submodule, behind a minimal JNI bridge; built for ARMv8.2 (fp16 + dotprod), with a CPU check before loading so an older phone gets a message instead of a SIGILL - AudioDecoder streams any playable file as 16 kHz mono in two-minute chunks, cut at the quietest moment so no word straddles two chunks - TranscriptStore persists every finished chunk: a job killed hours in resumes at the last chunk, and re-aligning against another edition of the book reuses the transcript - foreground service (special use - the other types are stopped after six hours) with a wake lock and a progress notification - Compose UI in plain black on white with nothing animated, since the first device it has to work on is an e-ink reader - CI builds and signs the APK; the key lives in Actions secrets only
.github/workflows/build.yml | 108 +++++++++ .gitmodules | 4 + README.md | 54 +++++ app/build.gradle.kts | 85 +++++++ app/src/main/AndroidManifest.xml | 48 ++++ app/src/main/cpp/CMakeLists.txt | 34 +++ app/src/main/cpp/whisper_jni.c | 153 ++++++++++++ .../main/kotlin/space/subread/app/MainActivity.kt | 261 +++++++++++++++++++++ .../kotlin/space/subread/app/audio/AudioDecoder.kt | 218 +++++++++++++++++ .../kotlin/space/subread/app/job/AlignService.kt | 136 +++++++++++ app/src/main/kotlin/space/subread/app/job/Job.kt | 138 +++++++++++ .../space/subread/app/job/TranscriptStore.kt | 120 ++++++++++ .../kotlin/space/subread/app/whisper/Whisper.kt | 127 ++++++++++ .../main/res/drawable/ic_launcher_foreground.xml | 7 + app/src/main/res/drawable/ic_notification.xml | 6 + app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml | 6 + .../res/mipmap-anydpi-v26/ic_launcher_round.xml | 6 + app/src/main/res/values/colors.xml | 4 + app/src/main/res/values/strings.xml | 6 + app/src/main/res/values/themes.xml | 9 + app/src/main/res/xml/file_paths.xml | 5 + third_party/whisper.cpp | 1 + tools/write_res.py | 68 ++++++ 23 files changed, 1604 insertions(+)
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..17e51bc --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,108 @@ +name: Build + +"on": + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: build-${{ github.ref }} + cancel-in-progress: true + +jobs: + apk: + name: Test the engine, build the APK + runs-on: ubuntu-latest + timeout-minutes: 60 + + env: + CMAKE_VERSION: "3.31.6" + NDK_VERSION: "29.0.14206865" + # The speech model is not in git (43 MB). Pinned by hash, so the build + # fails loudly if what is served ever changes. + MODEL_URL: https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny-q8_0.bin + MODEL_SHA256: c2085835d3f50733e2ff6e4b41ae8a2b8d8110461e18821b09a15c40c42d1cca + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 1 + + - name: Set up JDK + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "21" + + - name: Set up Android SDK + uses: android-actions/setup-android@v3 + with: + packages: platform-tools + + - name: Install Android build tools + run: | + set -euo pipefail + sdkmanager "platforms;android-36" "build-tools;36.0.0" \ + "cmake;${CMAKE_VERSION}" "ndk;${NDK_VERSION}" + + - name: Set up Gradle + uses: gradle/actions/setup-gradle@v4 + + - name: Restore the speech model + id: model + uses: actions/cache@v4 + with: + path: app/src/main/assets/models + key: model-${{ env.MODEL_SHA256 }} + + - name: Download the speech model + if: steps.model.outputs.cache-hit != 'true' + run: | + set -euo pipefail + mkdir -p app/src/main/assets/models + curl -fsSL --retry 3 -o app/src/main/assets/models/ggml-tiny-q8_0.bin "${MODEL_URL}" + + - name: Verify the speech model + run: echo "${MODEL_SHA256} app/src/main/assets/models/ggml-tiny-q8_0.bin" | sha256sum -c - + + - name: Test the alignment engine + run: ./gradlew --no-daemon :core:test + + - name: Unpack the signing key + env: + KEYSTORE_B64: ${{ secrets.SUBREAD_KEYSTORE_B64 }} + run: | + set -euo pipefail + if [ -n "${KEYSTORE_B64}" ]; then + echo "${KEYSTORE_B64}" | base64 -d > "${RUNNER_TEMP}/subread.jks" + echo "SUBREAD_KEYSTORE_FILE=${RUNNER_TEMP}/subread.jks" >> "${GITHUB_ENV}" + else + echo "::warning::No signing key configured; the APK will be unsigned and will not install." + fi + + - name: Build the APK + env: + SUBREAD_KEYSTORE_PASSWORD: ${{ secrets.SUBREAD_KEYSTORE_PASSWORD }} + run: ./gradlew --no-daemon :app:assembleRelease -PversionCode=${{ github.run_number }} + + - name: Upload the APK + uses: actions/upload-artifact@v4 + with: + name: subread-apk + path: app/build/outputs/apk/release/*.apk + if-no-files-found: error + + - name: Upload test reports + if: failure() + uses: actions/upload-artifact@v4 + with: + name: reports + path: | + core/build/reports/tests + app/build/outputs/logs + if-no-files-found: ignore diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..ea2d8db --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "third_party/whisper.cpp"] + path = third_party/whisper.cpp + url = https://github.com/ggml-org/whisper.cpp + shallow = true diff --git a/README.md b/README.md new file mode 100644 index 0000000..c0cd615 --- /dev/null +++ b/README.md @@ -0,0 +1,54 @@ +# SubRead for Android + +Times an audiobook against its ebook, entirely on the phone, and writes the +`.srt` that [Hoshi Reader](https://github.com/HuangAntimony/Hoshi-Reader-Android)'s +read-along uses. Nothing is uploaded. + +It is the same method as [subread.space](https://subread.space) and +[SubPlz](https://github.com/kanjieater/SubPlz): a small speech model transcribes +the audio, roughly; the transcript is aligned against the book; the subtitles +take their *timing* from the transcript and their *words* from the book. + +## Layout + +| Path | What | +|---|---| +| `core/` | The alignment engine. Plain Kotlin, no Android - builds and tests with only a JDK | +| `app/` | The Android app: audio decoding, the whisper.cpp bridge, the long-running job, the UI | +| `third_party/whisper.cpp` | Pinned submodule | +| `tools/make_golden.py` | Generates test fixtures by running the reference Python implementation | + +## The engine + +`core/` is a port of SubPlz's aligner (`ats.align`, `subplz.align.shift_align`), +checked stage by stage against the original's real output on real Whisper-tiny +transcripts: every cue across the fixtures comes out identical. + +What is new is `AnchoredAligner`. The reference hands Biopython one chapter at a +time and needs gigabytes to do it, so it has to guess first which chapter of the +book each chapter of audio is. This aligns the whole book at once instead, by +anchoring on stretches unique to both texts and solving exactly only between +anchors: a 19-hour audiobook against its full text in a few seconds and a few +megabytes. `BookAligner` then uses the alignment itself to leave out text nobody +narrated - front matter, notes - instead of matching chapters. + +```bash +./gradlew :core:test +``` + +## Building the app + +CI builds it (`.github/workflows/build.yml`): the APK is an artifact of every +push to `main`. Locally it needs the Android SDK, NDK 29 and CMake 3.31, and the +speech model at `app/src/main/assets/models/ggml-tiny-q8_0.bin` (the workflow has +the URL and checksum). Without an SDK, Gradle leaves `:app` out and `:core` +still builds. + +The release key is not in the repository; CI reads it from Actions secrets. + +## Requirements + +Android 8+, a 64-bit ARM processor with ARMv8.2 half-precision and dot-product +instructions (anything from 2018 on). Transcription runs at a small multiple of +real time, so a long book takes hours; it carries on with the screen off and +resumes where it stopped if interrupted. diff --git a/app/build.gradle.kts b/app/build.gradle.kts new file mode 100644 index 0000000..fb0509e --- /dev/null +++ b/app/build.gradle.kts @@ -0,0 +1,85 @@ +plugins { + alias(libs.plugins.android.application) + alias(libs.plugins.kotlin.compose) +} + +// Release signing comes from the environment, so the key never lives in the +// repository. CI supplies it from repository secrets; without it the release +// build is simply left unsigned. +val keystorePath = providers.environmentVariable("SUBREAD_KEYSTORE_FILE").orNull +val keystorePassword = providers.environmentVariable("SUBREAD_KEYSTORE_PASSWORD").orNull +val signingReady = !keystorePath.isNullOrBlank() && !keystorePassword.isNullOrBlank() && + file(keystorePath).isFile + +android { + namespace = "space.subread.app" + compileSdk = 36 + ndkVersion = "29.0.14206865" + + defaultConfig { + applicationId = "space.subread.app" + minSdk = 26 + targetSdk = 36 + versionCode = (providers.gradleProperty("versionCode").orNull ?: "1").toInt() + versionName = providers.gradleProperty("versionName").orNull ?: "0.1.0" + + // Every phone worth transcribing on is 64-bit ARM; the speech library + // is built for ARMv8.2 specifically (see src/main/cpp/CMakeLists.txt). + ndk { abiFilters += "arm64-v8a" } + externalNativeBuild { + cmake { arguments += listOf("-DANDROID_STL=c++_static") } + } + } + + if (signingReady) { + signingConfigs { + create("release") { + storeFile = file(keystorePath!!) + storePassword = keystorePassword + keyAlias = "subread" + keyPassword = keystorePassword + } + } + } + + buildTypes { + debug { applicationIdSuffix = ".debug" } + release { + // Not shrunk yet: R8 rules want a device to be verified against. + isMinifyEnabled = false + if (signingReady) signingConfig = signingConfigs.getByName("release") + } + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 + } + buildFeatures { compose = true } + + externalNativeBuild { + cmake { + path = file("src/main/cpp/CMakeLists.txt") + version = "3.31.6" + } + } + + // The model is copied out of the APK to a real file on first run; storing + // it uncompressed makes that a straight copy. + androidResources { noCompress += "bin" } + + lint { abortOnError = false } +} + +dependencies { + implementation(project(":core")) + implementation(platform(libs.androidx.compose.bom)) + implementation(libs.androidx.activity.compose) + implementation(libs.androidx.compose.material3) + implementation(libs.androidx.compose.ui) + implementation(libs.androidx.compose.ui.graphics) + implementation(libs.androidx.core.ktx) + implementation(libs.androidx.lifecycle.runtime.ktx) + implementation(libs.androidx.lifecycle.runtime.compose) + implementation(libs.kotlinx.coroutines.android) +} diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml new file mode 100644 index 0000000..7dd66a8 --- /dev/null +++ b/app/src/main/AndroidManifest.xml @@ -0,0 +1,48 @@ +<?xml version="1.0" encoding="utf-8"?> +<manifest xmlns:android="http://schemas.android.com/apk/res/android"> + + <uses-permission android:name="android.permission.FOREGROUND_SERVICE" /> + <uses-permission android:name="android.permission.FOREGROUND_SERVICE_SPECIAL_USE" /> + <uses-permission android:name="android.permission.POST_NOTIFICATIONS" /> + <uses-permission android:name="android.permission.WAKE_LOCK" /> + + <application + android:allowBackup="false" + android:icon="@mipmap/ic_launcher" + android:label="@string/app_name" + android:largeHeap="true" + android:supportsRtl="true" + android:theme="@style/Theme.SubRead"> + + <activity + android:name=".MainActivity" + android:exported="true" + android:launchMode="singleTop"> + <intent-filter> + <action android:name="android.intent.action.MAIN" /> + <category android:name="android.intent.category.LAUNCHER" /> + </intent-filter> + </activity> + + <!-- Special use, not dataSync or mediaProcessing: those are stopped by + the system after six hours, and a long book takes longer. --> + <service + android:name=".job.AlignService" + android:exported="false" + android:foregroundServiceType="specialUse"> + <property + android:name="android.app.PROPERTY_SPECIAL_USE_FGS_SUBTYPE" + android:value="On-device speech recognition over a user-selected audiobook, to time it against the user's ebook. Runs for hours and must not be interrupted." /> + </service> + + <provider + android:name="androidx.core.content.FileProvider" + android:authorities="${applicationId}.files" + android:exported="false" + android:grantUriPermissions="true"> + <meta-data + android:name="android.support.FILE_PROVIDER_PATHS" + android:resource="@xml/file_paths" /> + </provider> + </application> +</manifest> diff --git a/app/src/main/cpp/CMakeLists.txt b/app/src/main/cpp/CMakeLists.txt new file mode 100644 index 0000000..e3f86f2 --- /dev/null +++ b/app/src/main/cpp/CMakeLists.txt @@ -0,0 +1,34 @@ +cmake_minimum_required(VERSION 3.22) +project(subread_whisper C CXX) + +set(CMAKE_C_STANDARD 11) +set(CMAKE_CXX_STANDARD 17) + +# Transcription is hours of matrix arithmetic. Unoptimised it is ten times +# slower, so optimise whatever Gradle thinks the build type is. +if (CMAKE_BUILD_TYPE STREQUAL "Debug") + add_compile_options(-O3) +endif () + +set(WHISPER_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../../../third_party/whisper.cpp) + +set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) +set(WHISPER_BUILD_TESTS OFF CACHE BOOL "" FORCE) +set(WHISPER_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) +set(WHISPER_BUILD_SERVER OFF CACHE BOOL "" FORCE) +set(GGML_NATIVE OFF CACHE BOOL "" FORCE) # cross-compiling: never probe the build host +set(GGML_OPENMP OFF CACHE BOOL "" FORCE) # ggml's own thread pool; no libomp to ship +set(GGML_LLAMAFILE OFF CACHE BOOL "" FORCE) + +# Half-precision and dot-product instructions are most of the speed on ARM. +# Everything since the Cortex-A55/A75 (2018) has them; MainActivity checks +# before loading this library, so an older CPU gets a message, not a SIGILL. +if (ANDROID_ABI STREQUAL "arm64-v8a") + set(GGML_CPU_ARM_ARCH "armv8.2-a+fp16+dotprod" CACHE STRING "" FORCE) +endif () + +add_subdirectory(${WHISPER_DIR} whisper EXCLUDE_FROM_ALL) + +add_library(subread_whisper SHARED whisper_jni.c) +target_link_libraries(subread_whisper PRIVATE whisper log) +target_link_options(subread_whisper PRIVATE -Wl,--gc-sections -Wl,--exclude-libs,ALL) diff --git a/app/src/main/cpp/whisper_jni.c b/app/src/main/cpp/whisper_jni.c new file mode 100644 index 0000000..7416e0c --- /dev/null +++ b/app/src/main/cpp/whisper_jni.c @@ -0,0 +1,153 @@ +// The narrowest possible bridge to whisper.cpp: load a model, transcribe one +// buffer of 16 kHz mono float PCM, read the segments back. +// +// One transcription runs at a time (the service guarantees it), which is what +// lets progress and cancellation be plain process-wide atomics rather than +// callbacks into the JVM from whisper's worker threads. + +#include <jni.h> +#include <stdatomic.h> +#include <stdbool.h> +#include <string.h> +#include <android/log.h> + +#include "whisper.h" + +#define TAG "subread-whisper" +#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__) +#define LOGW(...) __android_log_print(ANDROID_LOG_WARN, TAG, __VA_ARGS__) + +static atomic_int g_progress = 0; +static atomic_bool g_abort = false; + +static void on_progress(struct whisper_context *ctx, struct whisper_state *state, + int progress, void *user_data) { + (void) ctx; (void) state; (void) user_data; + atomic_store(&g_progress, progress); +} + +static bool should_abort(void *user_data) { + (void) user_data; + return atomic_load(&g_abort); +} + +#define JNI_FN(name) Java_space_subread_app_whisper_WhisperLib_##name + +JNIEXPORT jlong JNICALL +JNI_FN(initContext)(JNIEnv *env, jobject thiz, jstring model_path) { + (void) thiz; + const char *path = (*env)->GetStringUTFChars(env, model_path, NULL); + struct whisper_context_params cparams = whisper_context_default_params(); + cparams.use_gpu = false; + struct whisper_context *ctx = whisper_init_from_file_with_params(path, cparams); + if (ctx == NULL) LOGW("could not load model %s", path); + (*env)->ReleaseStringUTFChars(env, model_path, path); + return (jlong) ctx; +} + +JNIEXPORT void JNICALL +JNI_FN(freeContext)(JNIEnv *env, jobject thiz, jlong ptr) { + (void) env; (void) thiz; + if (ptr != 0) whisper_free((struct whisper_context *) ptr); +} + +// Returns 0 on success, -1 if cancelled, anything else is whisper's own error. +JNIEXPORT jint JNICALL +JNI_FN(transcribe)(JNIEnv *env, jobject thiz, jlong ptr, jfloatArray samples, + jint n_threads, jstring language) { + (void) thiz; + struct whisper_context *ctx = (struct whisper_context *) ptr; + if (ctx == NULL) return -2; + + const jsize n = (*env)->GetArrayLength(env, samples); + jfloat *pcm = (*env)->GetFloatArrayElements(env, samples, NULL); + const char *lang = (*env)->GetStringUTFChars(env, language, NULL); + + struct whisper_full_params p = whisper_full_default_params(WHISPER_SAMPLING_GREEDY); + p.print_realtime = false; + p.print_progress = false; + p.print_timestamps = false; + p.print_special = false; + p.translate = false; + p.language = lang; // "auto" detects; the caller pins it after the first chunk + p.n_threads = n_threads; + // Each chunk stands alone. Carrying text over as a prompt is how one + // hallucinated line turns into a page of the same line. + p.no_context = true; + p.single_segment = false; + p.suppress_blank = true; + p.suppress_nst = true; + p.progress_callback = on_progress; + p.abort_callback = should_abort; + + atomic_store(&g_progress, 0); + atomic_store(&g_abort, false); + + int rc = whisper_full(ctx, p, pcm, n); + + (*env)->ReleaseStringUTFChars(env, language, lang); + (*env)->ReleaseFloatArrayElements(env, samples, pcm, JNI_ABORT); + + if (atomic_load(&g_abort)) return -1; + if (rc != 0) LOGW("whisper_full failed: %d", rc); + return rc; +} + +JNIEXPORT void JNICALL +JNI_FN(cancel)(JNIEnv *env, jobject thiz) { + (void) env; (void) thiz; + atomic_store(&g_abort, true); +} + +JNIEXPORT jint JNICALL +JNI_FN(progress)(JNIEnv *env, jobject thiz) { + (void) env; (void) thiz; + return atomic_load(&g_progress); +} + +JNIEXPORT jint JNICALL +JNI_FN(segmentCount)(JNIEnv *env, jobject thiz, jlong ptr) { + (void) env; (void) thiz; + return whisper_full_n_segments((struct whisper_context *) ptr); +} + +// Text as raw UTF-8 bytes. NewStringUTF wants *modified* UTF-8 and mangles or +// aborts on anything outside the BMP, and the tiny model will happily emit a +// truncated multi-byte sequence at a segment edge. Let Kotlin decode leniently. +JNIEXPORT jbyteArray JNICALL +JNI_FN(segmentText)(JNIEnv *env, jobject thiz, jlong ptr, jint index) { + (void) thiz; + const char *text = whisper_full_get_segment_text((struct whisper_context *) ptr, index); + if (text == NULL) text = ""; + const jsize len = (jsize) strlen(text); + jbyteArray out = (*env)->NewByteArray(env, len); + if (out != NULL) (*env)->SetByteArrayRegion(env, out, 0, len, (const jbyte *) text); + return out; +} + +// Both in centiseconds, as whisper reports them. +JNIEXPORT jlong JNICALL +JNI_FN(segmentStart)(JNIEnv *env, jobject thiz, jlong ptr, jint index) { + (void) env; (void) thiz; + return whisper_full_get_segment_t0((struct whisper_context *) ptr, index); +} + +JNIEXPORT jlong JNICALL +JNI_FN(segmentEnd)(JNIEnv *env, jobject thiz, jlong ptr, jint index) { + (void) env; (void) thiz; + return whisper_full_get_segment_t1((struct whisper_context *) ptr, index); +} + +JNIEXPORT jstring JNICALL +JNI_FN(detectedLanguage)(JNIEnv *env, jobject thiz, jlong ptr) { + (void) thiz; + const int id = whisper_full_lang_id((struct whisper_context *) ptr); + const char *code = id >= 0 ? whisper_lang_str(id) : NULL; + return (*env)->NewStringUTF(env, code != NULL ? code : ""); +} + +JNIEXPORT jstring JNICALL +JNI_FN(systemInfo)(JNIEnv *env, jobject thiz) { + (void) thiz; + return (*env)->NewStringUTF(env, whisper_print_system_info()); +} diff --git a/app/src/main/kotlin/space/subread/app/MainActivity.kt b/app/src/main/kotlin/space/subread/app/MainActivity.kt new file mode 100644 index 0000000..d2bfbb3 --- /dev/null +++ b/app/src/main/kotlin/space/subread/app/MainActivity.kt @@ -0,0 +1,261 @@ +package space.subread.app + +import android.Manifest +import android.content.Context +import android.content.Intent +import android.net.Uri +import android.os.Build +import android.os.Bundle +import androidx.activity.ComponentActivity +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.compose.setContent +import androidx.activity.result.contract.ActivityResultContracts +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.safeDrawingPadding +import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.verticalScroll +import androidx.compose.material3.Button +import androidx.compose.material3.DropdownMenu +import androidx.compose.material3.DropdownMenuItem +import androidx.compose.material3.HorizontalDivider +import androidx.compose.material3.LinearProgressIndicator +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.OutlinedButton +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.material3.lightColorScheme +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.unit.dp +import androidx.core.content.FileProvider +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import space.subread.app.job.AlignService +import space.subread.app.job.Job +import space.subread.app.job.JobStatus +import space.subread.app.job.Phase +import space.subread.app.job.TranscriptStore +import java.io.File + +class MainActivity : ComponentActivity() { + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + setContent { App() } + } +} + +/** + * Black on white and nothing that moves: this has to be as usable on an e-ink + * reader - where an animation is a smear - as on a phone. + */ +private val Paper = lightColorScheme( + primary = Color.Black, onPrimary = Color.White, + secondary = Color.Black, onSecondary = Color.White, + background = Color.White, onBackground = Color.Black, + surface = Color.White, onSurface = Color.Black, + surfaceVariant = Color(0xFFEDEDED), onSurfaceVariant = Color(0xFF333333), + outline = Color.Black, +) + +/** Whisper's codes. "auto" listens to the first two minutes and decides. */ +private val LANGUAGES = listOf( + "auto" to "Detect automatically", + "ja" to "Japanese", "es" to "Spanish", "pt" to "Portuguese", "ru" to "Russian", + "fi" to "Finnish", "en" to "English", "de" to "German", "fr" to "French", + "it" to "Italian", "zh" to "Chinese", "ko" to "Korean", "nl" to "Dutch", + "sv" to "Swedish", "pl" to "Polish", "uk" to "Ukrainian", "tr" to "Turkish", +) + +@Composable +private fun App() { + val context = LocalContext.current + val prefs = remember { context.getSharedPreferences("picks", Context.MODE_PRIVATE) } + val status by Job.status.collectAsStateWithLifecycle() + + var audio by remember { mutableStateOf(prefs.getString("audio", null)?.let(Uri::parse)) } + var book by remember { mutableStateOf(prefs.getString("book", null)?.let(Uri::parse)) } + var language by remember { mutableStateOf(prefs.getString("language", "auto")!!) } + + fun keep(key: String, uri: Uri) { + // So the pick survives the process being killed mid-job and can resume. + runCatching { + context.contentResolver.takePersistableUriPermission(uri, Intent.FLAG_GRANT_READ_URI_PERMISSION) + } + prefs.edit().putString(key, uri.toString()).apply() + } + + val pickAudio = rememberLauncherForActivityResult(ActivityResultContracts.OpenDocument()) { uri -> + if (uri != null) { audio = uri; keep("audio", uri); Job.clear() } + } + val pickBook = rememberLauncherForActivityResult(ActivityResultContracts.OpenDocument()) { uri -> + if (uri != null) { book = uri; keep("book", uri); Job.clear() } + } + val saveSrt = rememberLauncherForActivityResult( + ActivityResultContracts.CreateDocument("application/x-subrip") + ) { dest -> + val srt = status.srt + if (dest != null && srt != null) { + context.contentResolver.openOutputStream(dest)?.use { out -> srt.inputStream().use { it.copyTo(out) } } + } + } + val askNotifications = rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) {} + + MaterialTheme(colorScheme = Paper) { + Surface(Modifier.fillMaxSize()) { + Column( + Modifier.safeDrawingPadding().padding(20.dp).verticalScroll(rememberScrollState()), + verticalArrangement = Arrangement.spacedBy(14.dp), + ) { + Text("SubRead", style = MaterialTheme.typography.headlineMedium, fontWeight = FontWeight.Bold) + Text( + "Times an audiobook against its ebook, on this device. Nothing is uploaded. " + + "The result is an .srt for Hoshi Reader's read-along.", + style = MaterialTheme.typography.bodyMedium, + ) + HorizontalDivider(color = Color.Black) + + Pick("Audiobook", audio?.let { TranscriptStore.describe(context, it).first }, !status.running) { + pickAudio.launch(arrayOf("audio/*", "video/mp4", "application/ogg", "application/octet-stream")) + } + Pick("Book", book?.let { TranscriptStore.describe(context, it).first }, !status.running) { + pickBook.launch(arrayOf("application/epub+zip", "text/plain", "application/zip", + "application/octet-stream")) + } + LanguagePick(language, !status.running) { + language = it + prefs.edit().putString("language", it).apply() + } + + HorizontalDivider(color = Color.Black) + + if (status.running) { + Progress(status) + OutlinedButton(onClick = { Job.cancel() }, modifier = Modifier.fillMaxWidth()) { + Text("Stop (progress is kept)") + } + } else { + val a = audio + val b = book + val resumable = remember(a, status.phase) { + a != null && TranscriptStore.forAudio(context, a).doneUntil > 0 + } + Button( + enabled = a != null && b != null, + modifier = Modifier.fillMaxWidth(), + onClick = { + if (Build.VERSION.SDK_INT >= 33) { + askNotifications.launch(Manifest.permission.POST_NOTIFICATIONS) + } + AlignService.start(context, a!!, b!!, language) + }, + ) { Text(if (resumable) "Continue" else "Start") } + + if (a == null || b == null) { + Text("Pick both files to begin. A long book takes hours; it carries on with the " + + "screen off, and picks up where it stopped if interrupted.", + style = MaterialTheme.typography.bodySmall) + } + Outcome(status, + onSave = { status.srt?.let { saveSrt.launch(it.name) } }, + onShare = { status.srt?.let { share(context, it) } }) + } + } + } + } +} + +@Composable +private fun Pick(label: String, chosen: String?, enabled: Boolean, onClick: () -> Unit) { + Column(verticalArrangement = Arrangement.spacedBy(4.dp)) { + Text(label, style = MaterialTheme.typography.labelLarge, fontWeight = FontWeight.Bold) + Row(horizontalArrangement = Arrangement.spacedBy(12.dp)) { + OutlinedButton(onClick = onClick, enabled = enabled) { Text(if (chosen == null) "Choose" else "Change") } + Text(chosen ?: "Nothing chosen", modifier = Modifier.padding(top = 12.dp), + style = MaterialTheme.typography.bodyMedium) + } + } +} + +@Composable +private fun LanguagePick(code: String, enabled: Boolean, onPick: (String) -> Unit) { + var open by remember { mutableStateOf(false) } + Column(verticalArrangement = Arrangement.spacedBy(4.dp)) { + Text("Language", style = MaterialTheme.typography.labelLarge, fontWeight = FontWeight.Bold) + Box { + OutlinedButton(onClick = { open = true }, enabled = enabled) { + Text(LANGUAGES.firstOrNull { it.first == code }?.second ?: code) + } + DropdownMenu(expanded = open, onDismissRequest = { open = false }) { + for ((c, name) in LANGUAGES) { + DropdownMenuItem(text = { Text(name) }, onClick = { onPick(c); open = false }) + } + } + } + } +} + +@Composable +private fun Progress(status: JobStatus) { + Column(verticalArrangement = Arrangement.spacedBy(8.dp)) { + Text(status.detail, fontWeight = FontWeight.Bold) + LinearProgressIndicator( + progress = { status.fraction }, + modifier = Modifier.fillMaxWidth(), + color = Color.Black, trackColor = Color(0xFFCCCCCC), + ) + val bits = listOfNotNull( + "${(status.fraction * 100).toInt()}%", + status.etaSeconds?.let { "about ${Job.clock(it.toDouble())} left" }, + status.speed?.let { "%.1f× real time".format(it) }, + ) + Text(bits.joinToString(" · "), style = MaterialTheme.typography.bodySmall) + } +} + +@Composable +private fun Outcome(status: JobStatus, onSave: () -> Unit, onShare: () -> Unit) { + when (status.phase) { + Phase.DONE -> Column(verticalArrangement = Arrangement.spacedBy(10.dp)) { + val rate = ((status.matchRate ?: 0.0) * 100).toInt() + Text("Done: ${status.cues} lines, $rate% found in the book.", fontWeight = FontWeight.Bold) + if (rate < 80) { + Text("That is low. Usually it means a different edition or translation of the book, " + + "or the wrong language.", style = MaterialTheme.typography.bodySmall) + } + if (status.paragraphsDropped > 0) { + Text("${status.paragraphsDropped} paragraphs of the book were never narrated " + + "(front matter, notes) and were left out.", style = MaterialTheme.typography.bodySmall) + } + Row(horizontalArrangement = Arrangement.spacedBy(12.dp)) { + Button(onClick = onSave) { Text("Save .srt") } + OutlinedButton(onClick = onShare) { Text("Share") } + } + Text("In Hoshi Reader: long-press the book, Match, and choose the saved .srt.", + style = MaterialTheme.typography.bodySmall) + } + Phase.FAILED -> Text("Failed: ${status.detail}", fontWeight = FontWeight.Bold) + Phase.CANCELLED -> Text(status.detail) + else -> {} + } +} + +private fun share(context: Context, srt: File) { + val uri = FileProvider.getUriForFile(context, "${context.packageName}.files", srt) + val send = Intent(Intent.ACTION_SEND) + .setType("application/x-subrip") + .putExtra(Intent.EXTRA_STREAM, uri) + .addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION) + context.startActivity(Intent.createChooser(send, srt.name)) +} diff --git a/app/src/main/kotlin/space/subread/app/audio/AudioDecoder.kt b/app/src/main/kotlin/space/subread/app/audio/AudioDecoder.kt new file mode 100644 index 0000000..0eaf5b4 --- /dev/null +++ b/app/src/main/kotlin/space/subread/app/audio/AudioDecoder.kt @@ -0,0 +1,218 @@ +package space.subread.app.audio + +import android.content.Context +import android.media.MediaCodec +import android.media.MediaExtractor +import android.media.MediaFormat +import android.net.Uri +import java.io.Closeable +import java.nio.ByteOrder +import kotlin.math.abs +import kotlin.math.max +import kotlin.math.min + +/** + * Streams any audio file the platform can play as 16 kHz mono float chunks, + * which is the only thing the speech model accepts. + * + * A ten-hour book is over two gigabytes as PCM, so it is never held whole: + * each call to [next] decodes a couple of minutes, cuts at the quietest moment + * near the end (so no word is split across two transcriptions) and carries the + * remainder into the next chunk. + */ +class AudioDecoder(context: Context, uri: Uri, startAtSeconds: Double = 0.0) : Closeable { + + class Chunk(val samples: FloatArray, val startSeconds: Double) { + val endSeconds: Double get() = startSeconds + samples.size / TARGET_RATE.toDouble() + } + + private val extractor = MediaExtractor() + private val codec: MediaCodec + private val info = MediaCodec.BufferInfo() + + val durationSeconds: Double + private var sourceRate: Int + private var channels: Int + + private var inputDone = false + private var outputDone = false + + /** Decoded, mono, still at the source rate; what did not fit the last chunk. */ + private var pending = FloatBuffer() + /** Time of pending[0], in seconds. Taken from the decoder, so seeking stays honest. */ + private var pendingStart = -1.0 + /** + * A seek lands on the frame *before* the one asked for. Audio ahead of this + * point was transcribed in an earlier session and is thrown away, or the + * resumed chunk would overlap the last one and repeat its final words. + */ + private val skipUntil = startAtSeconds + + init { + extractor.setDataSource(context, uri, null) + val track = (0 until extractor.trackCount).firstOrNull { + extractor.getTrackFormat(it).getString(MediaFormat.KEY_MIME).orEmpty().startsWith("audio/") + } ?: throw IllegalArgumentException("No audio track in this file.") + extractor.selectTrack(track) + + val format = extractor.getTrackFormat(track) + durationSeconds = if (format.containsKey(MediaFormat.KEY_DURATION)) + format.getLong(MediaFormat.KEY_DURATION) / 1e6 else 0.0 + sourceRate = format.getInteger(MediaFormat.KEY_SAMPLE_RATE) + channels = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT) + + if (startAtSeconds > 0) { + extractor.seekTo((startAtSeconds * 1e6).toLong(), MediaExtractor.SEEK_TO_PREVIOUS_SYNC) + } + + codec = MediaCodec.createDecoderByType(format.getString(MediaFormat.KEY_MIME)!!) + codec.configure(format, null, null, 0) + codec.start() + } + + /** The next chunk, or null at the end of the file. */ + fun next(targetSeconds: Int = CHUNK_SECONDS): Chunk? { + // Re-read the rate each time round: HE-AAC files announce half of what + // the decoder actually produces, and only say so once decoding starts. + while (pending.size < targetSeconds.toLong() * sourceRate && !outputDone) pump() + if (pending.size == 0) return null + val want = targetSeconds.toLong() * sourceRate + + val cut = if (outputDone && pending.size <= want) pending.size + else quietestPoint(pending, sourceRate, min(want, pending.size.toLong()).toInt()) + + val head = pending.take(cut) + val start = pendingStart + pending = pending.drop(cut) + pendingStart = start + cut / sourceRate.toDouble() + return Chunk(resample(head, sourceRate), start) + } + + private fun pump() { + if (!inputDone) { + val i = codec.dequeueInputBuffer(10_000) + if (i >= 0) { + val buffer = codec.getInputBuffer(i)!! + val n = extractor.readSampleData(buffer, 0) + if (n < 0) { + codec.queueInputBuffer(i, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM) + inputDone = true + } else { + codec.queueInputBuffer(i, 0, n, extractor.sampleTime, 0) + extractor.advance() + } + } + } + + when (val o = codec.dequeueOutputBuffer(info, 10_000)) { + MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> { + // The container can be wrong about these; the decoder is not. + val f = codec.outputFormat + sourceRate = f.getInteger(MediaFormat.KEY_SAMPLE_RATE) + channels = f.getInteger(MediaFormat.KEY_CHANNEL_COUNT) + } + in 0..Int.MAX_VALUE -> { + if (info.size > 0) { + val bufferStart = info.presentationTimeUs / 1e6 + val shorts = codec.getOutputBuffer(o)!!.order(ByteOrder.LITTLE_ENDIAN).asShortBuffer() + val frames = shorts.remaining() / channels + val skip = if (bufferStart >= skipUntil) 0 + else min(frames, ((skipUntil - bufferStart) * sourceRate).toInt()) + shorts.position(shorts.position() + skip * channels) + if (frames > skip && pendingStart < 0) { + pendingStart = bufferStart + skip / sourceRate.toDouble() + } + pending.ensure(frames - skip) + for (f in skip until frames) { + var sum = 0 + for (c in 0 until channels) sum += shorts.get() + pending.add(sum / (32768f * channels)) + } + } + codec.releaseOutputBuffer(o, false) + if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) outputDone = true + } + } + } + + override fun close() { + runCatching { codec.stop() } + runCatching { codec.release() } + runCatching { extractor.release() } + } + + companion object { + const val TARGET_RATE = 16_000 + const val CHUNK_SECONDS = 120 + + /** How far back from the end of a chunk to look for a pause. */ + private const val SEARCH_SECONDS = 12 + private const val WINDOW_SECONDS = 0.25 + + /** Index at which to cut: the middle of the quietest quarter second near [limit]. */ + internal fun quietestPoint(buf: FloatBuffer, rate: Int, limit: Int): Int { + val window = (WINDOW_SECONDS * rate).toInt() + val from = max(0, limit - SEARCH_SECONDS * rate) + if (limit - from <= window * 2) return limit + + var energy = 0.0 + for (i in from until from + window) energy += abs(buf[i]) + var best = energy + var bestAt = from + var i = from + while (i + window < limit) { + energy += abs(buf[i + window]) - abs(buf[i]) + i++ + if (energy < best) { best = energy; bestAt = i } + } + return bestAt + window / 2 + } + + /** + * To 16 kHz with a triangular kernel as wide as the rate ratio: a + * low-pass and an interpolator in one pass. Plenty for speech + * recognition, and it costs a handful of multiplies per sample. + */ + internal fun resample(src: FloatArray, rate: Int): FloatArray { + if (rate == TARGET_RATE) return src + val step = rate / TARGET_RATE.toDouble() + val half = max(step, 1.0) + val out = FloatArray((src.size / step).toInt()) + for (o in out.indices) { + val centre = o * step + val lo = max(0, (centre - half).toInt() + 1) + val hi = min(src.size - 1, (centre + half).toInt()) + var sum = 0.0 + var weight = 0.0 + for (k in lo..hi) { + val w = 1.0 - abs(k - centre) / half + if (w > 0) { sum += w * src[k]; weight += w } + } + out[o] = if (weight > 0) (sum / weight).toFloat() else 0f + } + return out + } + } +} + +/** A growable float array. Minutes of audio as boxed Floats would be absurd. */ +internal class FloatBuffer(capacity: Int = 1 shl 16) { + private var data = FloatArray(capacity) + var size = 0 + private set + + operator fun get(i: Int) = data[i] + + fun ensure(extra: Int) { + if (size + extra > data.size) data = data.copyOf(max(data.size * 2, size + extra)) + } + + fun add(v: Float) { data[size++] = v } + + fun take(n: Int): FloatArray = data.copyOf(n) + + fun drop(n: Int): FloatBuffer = FloatBuffer(max(size - n, 1 shl 16)).also { + System.arraycopy(data, n, it.data, 0, size - n) + it.size = size - n + } +} diff --git a/app/src/main/kotlin/space/subread/app/job/AlignService.kt b/app/src/main/kotlin/space/subread/app/job/AlignService.kt new file mode 100644 index 0000000..7dc24b1 --- /dev/null +++ b/app/src/main/kotlin/space/subread/app/job/AlignService.kt @@ -0,0 +1,136 @@ +package space.subread.app.job + +import android.app.Notification +import android.app.NotificationChannel +import android.app.NotificationManager +import android.app.PendingIntent +import android.app.Service +import android.content.Context +import android.content.Intent +import android.content.pm.ServiceInfo +import android.net.Uri +import android.os.Build +import android.os.IBinder +import android.os.PowerManager +import androidx.core.app.NotificationCompat +import androidx.core.app.ServiceCompat +import androidx.core.content.ContextCompat +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.flow.collectLatest +import kotlinx.coroutines.launch +import space.subread.app.MainActivity +import space.subread.app.R +import kotlin.concurrent.thread + +/** + * Keeps the job alive with the screen off and the app in the background. + * + * A foreground service of type "special use": transcription runs for hours, + * and the other candidate types (data sync, media processing) are cut off by + * the system after six. + */ +class AlignService : Service() { + + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Main) + private var wakeLock: PowerManager.WakeLock? = null + + override fun onBind(intent: Intent?): IBinder? = null + + override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + if (intent?.action == ACTION_CANCEL) { + Job.cancel() + return START_NOT_STICKY + } + val audio = intent?.getStringExtra(EXTRA_AUDIO) + val book = intent?.getStringExtra(EXTRA_BOOK) + if (audio == null || book == null || Job.status.value.running) { + if (!Job.status.value.running) stopSelf() + return START_NOT_STICKY + } + val language = intent.getStringExtra(EXTRA_LANGUAGE) ?: "auto" + + createChannel() + ServiceCompat.startForeground( + this, NOTIFICATION_ID, notification(Job.status.value), + if (Build.VERSION.SDK_INT >= 34) ServiceInfo.FOREGROUND_SERVICE_TYPE_SPECIAL_USE else 0, + ) + + // Without this the CPU sleeps a few minutes after the screen goes off, + // and a four-hour job becomes a four-day one. + wakeLock = getSystemService(PowerManager::class.java) + .newWakeLock(PowerManager.PARTIAL_WAKE_LOCK, "subread:align") + .apply { acquire(24 * 60 * 60 * 1000L) } + + scope.launch { + val manager = getSystemService(NotificationManager::class.java) + Job.status.collectLatest { manager.notify(NOTIFICATION_ID, notification(it)) } + } + + thread(name = "subread-job") { + Job.run(applicationContext, Uri.parse(audio), Uri.parse(book), language) + wakeLock?.let { if (it.isHeld) it.release() } + // Leave the final notification up: "done" is worth seeing hours later. + ServiceCompat.stopForeground(this, ServiceCompat.STOP_FOREGROUND_DETACH) + stopSelf() + } + return START_NOT_STICKY + } + + override fun onDestroy() { + scope.cancel() + wakeLock?.let { if (it.isHeld) it.release() } + super.onDestroy() + } + + private fun createChannel() { + val channel = NotificationChannel(CHANNEL, getString(R.string.channel_progress), + NotificationManager.IMPORTANCE_LOW) + getSystemService(NotificationManager::class.java).createNotificationChannel(channel) + } + + private fun notification(status: JobStatus): Notification { + val open = PendingIntent.getActivity( + this, 0, Intent(this, MainActivity::class.java), + PendingIntent.FLAG_IMMUTABLE or PendingIntent.FLAG_UPDATE_CURRENT, + ) + val cancel = PendingIntent.getService( + this, 1, Intent(this, AlignService::class.java).setAction(ACTION_CANCEL), + PendingIntent.FLAG_IMMUTABLE or PendingIntent.FLAG_UPDATE_CURRENT, + ) + val eta = status.etaSeconds?.let { " · about ${Job.clock(it.toDouble())} left" }.orEmpty() + return NotificationCompat.Builder(this, CHANNEL) + .setSmallIcon(R.drawable.ic_notification) + .setContentTitle(getString(R.string.app_name)) + .setContentText(status.detail + eta) + .setContentIntent(open) + .setOnlyAlertOnce(true) + .setOngoing(status.running) + .apply { + if (status.running) { + setProgress(1000, (status.fraction * 1000).toInt(), status.phase == Phase.PREPARING) + addAction(0, getString(R.string.stop), cancel) + } + } + .build() + } + + companion object { + private const val CHANNEL = "progress" + private const val NOTIFICATION_ID = 1 + private const val ACTION_CANCEL = "space.subread.app.CANCEL" + private const val EXTRA_AUDIO = "audio" + private const val EXTRA_BOOK = "book" + private const val EXTRA_LANGUAGE = "language" + + fun start(context: Context, audio: Uri, book: Uri, language: String) { + val intent = Intent(context, AlignService::class.java) + .putExtra(EXTRA_AUDIO, audio.toString()) + .putExtra(EXTRA_BOOK, book.toString()) + .putExtra(EXTRA_LANGUAGE, language) + ContextCompat.startForegroundService(context, intent) + } + } +} diff --git a/app/src/main/kotlin/space/subread/app/job/Job.kt b/app/src/main/kotlin/space/subread/app/job/Job.kt new file mode 100644 index 0000000..b088bfc --- /dev/null +++ b/app/src/main/kotlin/space/subread/app/job/Job.kt @@ -0,0 +1,138 @@ +package space.subread.app.job + +import android.content.Context +import android.net.Uri +import android.os.SystemClock +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import space.subread.app.audio.AudioDecoder +import space.subread.app.whisper.TranscriptionCancelled +import space.subread.app.whisper.Whisper +import space.subread.core.BookAligner +import space.subread.core.BookText +import space.subread.core.Language +import space.subread.core.Srt +import java.io.File + +enum class Phase { IDLE, PREPARING, TRANSCRIBING, ALIGNING, DONE, FAILED, CANCELLED } + +data class JobStatus( + val phase: Phase = Phase.IDLE, + val detail: String = "", + /** 0..1 over the whole job. */ + val fraction: Float = 0f, + val etaSeconds: Long? = null, + /** Audio seconds transcribed per wall-clock second. */ + val speed: Double? = null, + val srt: File? = null, + val cues: Int = 0, + val matchRate: Double? = null, + val paragraphsDropped: Int = 0, +) { + val running: Boolean get() = phase == Phase.PREPARING || phase == Phase.TRANSCRIBING || phase == Phase.ALIGNING +} + +/** The one job this app runs at a time, observable from the UI and the service alike. */ +object Job { + private val state = MutableStateFlow(JobStatus()) + val status: StateFlow<JobStatus> = state + + @Volatile private var cancelled = false + + fun cancel() { + cancelled = true + Whisper.cancel() + } + + fun clear() { if (!state.value.running) state.value = JobStatus() } + + /** + * Transcribe (or resume transcribing) [audio], align it to [book], write + * the subtitles. Blocks for as long as that takes; call from a worker thread. + * + * @param language a whisper language code, or "auto" + */ + fun run(context: Context, audio: Uri, book: Uri, language: String) { + cancelled = false + state.value = JobStatus(Phase.PREPARING, "Loading the speech model") + try { + val store = TranscriptStore.forAudio(context, audio) + var lang = if (language == "auto") store.language ?: "auto" else language + + if (!store.complete) transcribe(context, audio, store, lang) { lang = it } + if (lang == "auto") lang = store.language ?: "en" + + state.value = state.value.copy(phase = Phase.ALIGNING, detail = "Matching the book to the narration", + fraction = 0.97f, etaSeconds = null) + val (bookName, _) = TranscriptStore.describe(context, book) + val paragraphs = context.contentResolver.openInputStream(book)!!.use { BookText.read(it, bookName) } + if (paragraphs.isEmpty()) throw IllegalStateException("No text could be read from $bookName.") + + val result = BookAligner.align(store.segments(), paragraphs, Language.of(lang)) + + val (audioName, _) = TranscriptStore.describe(context, audio) + val out = File(context.filesDir, "subtitles").apply { mkdirs() } + .resolve(audioName.substringBeforeLast('.') + ".$lang.srt") + out.writeText(Srt.write(result.cues)) + + state.value = JobStatus( + Phase.DONE, "Done", 1f, srt = out, cues = result.cues.size, + matchRate = BookAligner.matchRate(result.cues), paragraphsDropped = result.paragraphsDropped, + ) + } catch (e: TranscriptionCancelled) { + state.value = JobStatus(Phase.CANCELLED, "Stopped. Progress is saved; start again to continue.") + } catch (e: Throwable) { + state.value = JobStatus(Phase.FAILED, e.message ?: e.javaClass.simpleName) + } + } + + private fun transcribe( + context: Context, audio: Uri, store: TranscriptStore, language: String, onLanguage: (String) -> Unit, + ) { + var lang = language + Whisper.open(context).use { whisper -> + AudioDecoder(context, audio, startAtSeconds = store.doneUntil).use { decoder -> + val total = decoder.durationSeconds + val resumedAt = store.doneUntil + val started = SystemClock.elapsedRealtime() + + while (true) { + if (cancelled) throw TranscriptionCancelled() + val chunk = decoder.next() ?: break + report(chunk.startSeconds, total, resumedAt, started) + + val segments = whisper.transcribe(chunk.samples, lang, chunk.startSeconds) + // Detect once, then hold it: a stretch of music must not be + // allowed to flip the rest of the book into another language. + if (lang == "auto") { + lang = whisper.detectedLanguage.ifEmpty { "en" } + onLanguage(lang) + } + store.append(segments, chunk.endSeconds, lang) + report(chunk.endSeconds, total, resumedAt, started) + } + store.finish() + } + } + } + + private fun report(at: Double, total: Double, resumedAt: Double, startedMs: Long) { + val elapsed = (SystemClock.elapsedRealtime() - startedMs) / 1000.0 + val done = at - resumedAt + val speed = if (elapsed > 5 && done > 0) done / elapsed else null + val eta = if (speed != null && total > 0) ((total - at) / speed).toLong().coerceAtLeast(0) else null + state.value = JobStatus( + Phase.TRANSCRIBING, + "Listening: ${clock(at)}" + if (total > 0) " of ${clock(total)}" else "", + // Alignment takes seconds; nearly all of the bar is transcription. + fraction = if (total > 0) (at / total * 0.96).toFloat().coerceIn(0f, 0.96f) else 0f, + etaSeconds = eta, speed = speed, + ) + } + + fun clock(seconds: Double): String { + val s = seconds.toLong() + return if (s >= 3600) "%d:%02d:%02d".format(s / 3600, s / 60 % 60, s % 60) + else "%d:%02d".format(s / 60, s % 60) + } +} diff --git a/app/src/main/kotlin/space/subread/app/job/TranscriptStore.kt b/app/src/main/kotlin/space/subread/app/job/TranscriptStore.kt new file mode 100644 index 0000000..7fbb9d2 --- /dev/null +++ b/app/src/main/kotlin/space/subread/app/job/TranscriptStore.kt @@ -0,0 +1,120 @@ +package space.subread.app.job + +import android.content.Context +import android.net.Uri +import android.provider.OpenableColumns +import org.json.JSONObject +import space.subread.core.TranscriptSegment +import java.io.File +import java.security.MessageDigest + +/** + * What has been transcribed so far, on disk, per audio file. + * + * Transcribing a book takes hours and Android is entitled to kill the process + * at any point in them. Every finished chunk is appended here, so a restart + * picks up at the last chunk rather than at the beginning - and aligning the + * same audio against a different edition of the book costs seconds, not hours, + * because the transcript depends on the audio alone. + */ +class TranscriptStore private constructor(private val dir: File) { + + private val segmentsFile = File(dir, "segments.jsonl") + private val stateFile = File(dir, "state.json") + + /** Audio before this point (seconds) is transcribed and saved. */ + var doneUntil: Double = 0.0 + private set + var language: String? = null + private set + var complete: Boolean = false + private set + + init { + dir.mkdirs() + if (stateFile.exists()) runCatching { + val o = JSONObject(stateFile.readText()) + doneUntil = o.optDouble("doneUntil", 0.0) + language = o.optString("language").ifEmpty { null } + complete = o.optBoolean("complete", false) + } + // A chunk's segments are written before its state. Killed in between, + // the file holds segments past doneUntil that are about to be + // transcribed a second time; remove them now or they end up doubled. + if (segmentsFile.exists()) { + val kept = segmentsFile.readLines().filter { line -> + runCatching { JSONObject(line).getDouble("s") < doneUntil }.getOrDefault(false) + } + segmentsFile.writeText(kept.joinToString("") { it + "\n" }) + } + } + + fun segments(): List<TranscriptSegment> { + if (!segmentsFile.exists()) return emptyList() + return segmentsFile.readLines().mapNotNull { line -> + runCatching { + val o = JSONObject(line) + TranscriptSegment(o.getString("t"), o.getDouble("s"), o.getDouble("e")) + }.getOrNull() + } + } + + fun append(chunk: List<TranscriptSegment>, until: Double, language: String?) { + segmentsFile.appendText(chunk.joinToString("") { s -> + JSONObject().put("s", s.start).put("e", s.end).put("t", s.text).toString() + "\n" + }) + doneUntil = until + if (language != null) this.language = language + save() + } + + fun finish() { + complete = true + save() + } + + fun reset() { + segmentsFile.delete() + stateFile.delete() + doneUntil = 0.0 + language = null + complete = false + } + + private fun save() { + val tmp = File(dir, "state.json.tmp") + tmp.writeText( + JSONObject().put("doneUntil", doneUntil).put("language", language ?: "") + .put("complete", complete).toString() + ) + tmp.renameTo(stateFile) + } + + companion object { + fun forAudio(context: Context, audio: Uri): TranscriptStore { + val (name, size) = describe(context, audio) + // Name and size, not the URI: the same file picked again through a + // different route gets a different URI and should still resume. + val key = MessageDigest.getInstance("SHA-1").digest("$name|$size".toByteArray()) + .joinToString("") { "%02x".format(it) }.take(20) + return TranscriptStore(File(context.filesDir, "transcripts/$key")) + } + + /** Display name and byte size of a picked document. */ + fun describe(context: Context, uri: Uri): Pair<String, Long> { + var name = uri.lastPathSegment ?: "file" + var size = -1L + runCatching { + context.contentResolver.query(uri, null, null, null, null)?.use { c -> + if (c.moveToFirst()) { + val n = c.getColumnIndex(OpenableColumns.DISPLAY_NAME) + val s = c.getColumnIndex(OpenableColumns.SIZE) + if (n >= 0 && !c.isNull(n)) name = c.getString(n) + if (s >= 0 && !c.isNull(s)) size = c.getLong(s) + } + } + } + return name to size + } + } +} diff --git a/app/src/main/kotlin/space/subread/app/whisper/Whisper.kt b/app/src/main/kotlin/space/subread/app/whisper/Whisper.kt new file mode 100644 index 0000000..19ea2d1 --- /dev/null +++ b/app/src/main/kotlin/space/subread/app/whisper/Whisper.kt @@ -0,0 +1,127 @@ +package space.subread.app.whisper + +import android.content.Context +import space.subread.core.TranscriptSegment +import java.io.File + +/** JNI surface. See src/main/cpp/whisper_jni.c. */ +internal object WhisperLib { + external fun initContext(modelPath: String): Long + external fun freeContext(ptr: Long) + external fun transcribe(ptr: Long, samples: FloatArray, threads: Int, language: String): Int + external fun cancel() + external fun progress(): Int + external fun segmentCount(ptr: Long): Int + external fun segmentText(ptr: Long, index: Int): ByteArray + external fun segmentStart(ptr: Long, index: Int): Long + external fun segmentEnd(ptr: Long, index: Int): Long + external fun detectedLanguage(ptr: Long): String + external fun systemInfo(): String +} + +class WhisperUnavailable(message: String) : Exception(message) +class TranscriptionCancelled : Exception("cancelled") + +/** A loaded model. Not thread-safe: one transcription at a time. */ +class Whisper private constructor(private var ptr: Long) : AutoCloseable { + + /** The language whisper settled on in the last call ("ja", "ru", ...). */ + val detectedLanguage: String get() = WhisperLib.detectedLanguage(ptr) + + /** + * @param samples 16 kHz mono, -1..1 + * @param offset where this buffer starts in the whole recording, seconds + */ + fun transcribe(samples: FloatArray, language: String, offset: Double): List<TranscriptSegment> { + when (val rc = WhisperLib.transcribe(ptr, samples, threads, language)) { + 0 -> {} + -1 -> throw TranscriptionCancelled() + else -> throw WhisperUnavailable("The speech model failed on this audio (code $rc).") + } + return (0 until WhisperLib.segmentCount(ptr)).mapNotNull { i -> + // Lenient decode: the tiny model can end a segment mid-character. + val text = String(WhisperLib.segmentText(ptr, i), Charsets.UTF_8).replace("�", "").trim() + if (text.isEmpty()) null + else TranscriptSegment( + text, + offset + WhisperLib.segmentStart(ptr, i) / 100.0, + offset + WhisperLib.segmentEnd(ptr, i) / 100.0, + ) + } + } + + override fun close() { + if (ptr != 0L) WhisperLib.freeContext(ptr) + ptr = 0 + } + + companion object { + private const val MODEL_ASSET = "models/ggml-tiny-q8_0.bin" + + /** 0..100 within the buffer currently being transcribed. */ + val progress: Int get() = if (loaded) WhisperLib.progress() else 0 + + fun cancel() { if (loaded) WhisperLib.cancel() } + + @Volatile private var loaded = false + + val threads: Int by lazy { fastCores() } + + fun open(context: Context): Whisper { + requireCpu() + if (!loaded) { + System.loadLibrary("subread_whisper") + loaded = true + } + val model = File(context.filesDir, "ggml-tiny-q8_0.bin") + val asset = context.assets.openFd(MODEL_ASSET) + // whisper.cpp wants a path, and an asset inside the APK does not have one. + if (!model.exists() || model.length() != asset.length) { + asset.createInputStream().use { src -> model.outputStream().use { src.copyTo(it) } } + } + asset.close() + val ptr = WhisperLib.initContext(model.absolutePath) + if (ptr == 0L) throw WhisperUnavailable("The speech model could not be loaded.") + return Whisper(ptr) + } + + /** + * The native library is built for ARMv8.2 with half-precision and + * dot-product instructions. Loading it without them is a crash with no + * message, so look first. + */ + private fun requireCpu() { + val features = runCatching { + File("/proc/cpuinfo").readLines() + .firstOrNull { it.startsWith("Features") }.orEmpty() + .substringAfter(':').trim().split(' ').toSet() + }.getOrDefault(emptySet()) + if (features.isEmpty()) return // cannot tell; let it try + val missing = listOf("asimdhp", "asimddp").filter { it !in features } + if (missing.isNotEmpty()) { + throw WhisperUnavailable( + "This phone's processor is too old for on-device transcription " + + "(needs ARMv8.2 half-precision and dot-product support)." + ) + } + } + + /** + * Threads to use: the fast cores only. whisper waits for its slowest + * thread at every layer, so adding little cores makes it slower, not + * faster. "Fast" is anything clocked above the slowest cluster. + */ + private fun fastCores(): Int { + val max = (0 until Runtime.getRuntime().availableProcessors()).mapNotNull { cpu -> + runCatching { + File("/sys/devices/system/cpu/cpu$cpu/cpufreq/cpuinfo_max_freq").readText().trim().toLong() + }.getOrNull() + } + if (max.isEmpty()) return 4 + val slowest = max.min() + val fast = max.count { it > slowest } + // All cores alike (or unreadable clusters): use half, at most four. + return if (fast == 0) (max.size / 2).coerceIn(2, 4) else fast.coerceIn(2, 6) + } + } +} diff --git a/app/src/main/res/drawable/ic_launcher_foreground.xml b/app/src/main/res/drawable/ic_launcher_foreground.xml new file mode 100644 index 0000000..1e6743f --- /dev/null +++ b/app/src/main/res/drawable/ic_launcher_foreground.xml @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="utf-8"?> +<vector xmlns:android="http://schemas.android.com/apk/res/android" + android:width="108dp" android:height="108dp" + android:viewportWidth="108" android:viewportHeight="108"> + <path android:fillColor="#000000" android:pathData="M30,34 C38,30 46,30 53,34 L53,76 C46,72 38,72 30,76 Z M78,34 C70,30 62,30 55,34 L55,76 C62,72 70,72 78,76 Z" /> + <path android:fillColor="#FFFFFF" android:pathData="M35,50 h2 v10 h-2 Z M40,45 h2 v20 h-2 Z M45,52 h2 v6 h-2 Z M61,52 h2 v6 h-2 Z M66,45 h2 v20 h-2 Z M71,50 h2 v10 h-2 Z" /> +</vector> diff --git a/app/src/main/res/drawable/ic_notification.xml b/app/src/main/res/drawable/ic_notification.xml new file mode 100644 index 0000000..c8b9f92 --- /dev/null +++ b/app/src/main/res/drawable/ic_notification.xml @@ -0,0 +1,6 @@ +<?xml version="1.0" encoding="utf-8"?> +<vector xmlns:android="http://schemas.android.com/apk/res/android" + android:width="24dp" android:height="24dp" + android:viewportWidth="108" android:viewportHeight="108"> + <path android:fillColor="#FFFFFF" android:pathData="M30,34 C38,30 46,30 53,34 L53,76 C46,72 38,72 30,76 Z M78,34 C70,30 62,30 55,34 L55,76 C62,72 70,72 78,76 Z" /> +</vector> diff --git a/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml b/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml new file mode 100644 index 0000000..5c84730 --- /dev/null +++ b/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml @@ -0,0 +1,6 @@ +<?xml version="1.0" encoding="utf-8"?> +<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android"> + <background android:drawable="@color/ic_launcher_background" /> + <foreground android:drawable="@drawable/ic_launcher_foreground" /> + <monochrome android:drawable="@drawable/ic_launcher_foreground" /> +</adaptive-icon> diff --git a/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml b/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml new file mode 100644 index 0000000..5c84730 --- /dev/null +++ b/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml @@ -0,0 +1,6 @@ +<?xml version="1.0" encoding="utf-8"?> +<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android"> + <background android:drawable="@color/ic_launcher_background" /> + <foreground android:drawable="@drawable/ic_launcher_foreground" /> + <monochrome android:drawable="@drawable/ic_launcher_foreground" /> +</adaptive-icon> diff --git a/app/src/main/res/values/colors.xml b/app/src/main/res/values/colors.xml new file mode 100644 index 0000000..f42ada6 --- /dev/null +++ b/app/src/main/res/values/colors.xml @@ -0,0 +1,4 @@ +<?xml version="1.0" encoding="utf-8"?> +<resources> + <color name="ic_launcher_background">#FFFFFF</color> +</resources> diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml new file mode 100644 index 0000000..5b7b417 --- /dev/null +++ b/app/src/main/res/values/strings.xml @@ -0,0 +1,6 @@ +<?xml version="1.0" encoding="utf-8"?> +<resources> + <string name="app_name">SubRead</string> + <string name="channel_progress">Progress</string> + <string name="stop">Stop</string> +</resources> diff --git a/app/src/main/res/values/themes.xml b/app/src/main/res/values/themes.xml new file mode 100644 index 0000000..a14ff15 --- /dev/null +++ b/app/src/main/res/values/themes.xml @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="utf-8"?> +<resources> + <!-- Compose draws everything; this only covers the moment before it does. --> + <style name="Theme.SubRead" parent="android:Theme.Material.Light.NoActionBar"> + <item name="android:windowBackground">@android:color/white</item> + <item name="android:statusBarColor">@android:color/white</item> + <item name="android:windowLightStatusBar">true</item> + </style> +</resources> diff --git a/app/src/main/res/xml/file_paths.xml b/app/src/main/res/xml/file_paths.xml new file mode 100644 index 0000000..a9f4ded --- /dev/null +++ b/app/src/main/res/xml/file_paths.xml @@ -0,0 +1,5 @@ +<?xml version="1.0" encoding="utf-8"?> +<paths> + <files-path name="subtitles" path="subtitles/" /> + <files-path name="video" path="video/" /> +</paths> diff --git a/third_party/whisper.cpp b/third_party/whisper.cpp new file mode 160000 index 0000000..927cfce --- /dev/null +++ b/third_party/whisper.cpp @@ -0,0 +1 @@ +Subproject commit 927cfce34f31707e17f2bff35c349632fb9e2c3a diff --git a/tools/write_res.py b/tools/write_res.py new file mode 100644 index 0000000..a60b764 --- /dev/null +++ b/tools/write_res.py @@ -0,0 +1,68 @@ +"""Writes the app's small static resources. Run once; the output is committed.""" +from pathlib import Path + +RES = Path(__file__).resolve().parent.parent / "app/src/main/res" + + +def w(rel: str, text: str) -> None: + p = RES / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(text, encoding="utf-8", newline="\n") + + +HEAD = '<?xml version="1.0" encoding="utf-8"?>\n' + +w("xml/file_paths.xml", HEAD + """<paths> + <files-path name="subtitles" path="subtitles/" /> + <files-path name="video" path="video/" /> +</paths> +""") + +w("values/strings.xml", HEAD + """<resources> + <string name="app_name">SubRead</string> + <string name="channel_progress">Progress</string> + <string name="stop">Stop</string> +</resources> +""") + +w("values/themes.xml", HEAD + """<resources> + <!-- Compose draws everything; this only covers the moment before it does. --> + <style name="Theme.SubRead" parent="android:Theme.Material.Light.NoActionBar"> + <item name="android:windowBackground">@android:color/white</item> + <item name="android:statusBarColor">@android:color/white</item> + <item name="android:windowLightStatusBar">true</item> + </style> +</resources> +""") + +w("values/colors.xml", HEAD + """<resources> + <color name="ic_launcher_background">#FFFFFF</color> +</resources> +""") + +# An open book with a sound wave across its pages, in one colour. +BOOK = ("M30,34 C38,30 46,30 53,34 L53,76 C46,72 38,72 30,76 Z " + "M78,34 C70,30 62,30 55,34 L55,76 C62,72 70,72 78,76 Z") +WAVE = ("M35,50 h2 v10 h-2 Z M40,45 h2 v20 h-2 Z M45,52 h2 v6 h-2 Z " + "M61,52 h2 v6 h-2 Z M66,45 h2 v20 h-2 Z M71,50 h2 v10 h-2 Z") + + +def vector(size: int, paths: list[tuple[str, str]]) -> str: + body = "".join(f' <path android:fillColor="{c}" android:pathData="{d}" />\n' for c, d in paths) + return (HEAD + '<vector xmlns:android="http://schemas.android.com/apk/res/android"\n' + f' android:width="{size}dp" android:height="{size}dp"\n' + ' android:viewportWidth="108" android:viewportHeight="108">\n' + body + "</vector>\n") + + +w("drawable/ic_launcher_foreground.xml", vector(108, [("#000000", BOOK), ("#FFFFFF", WAVE)])) +w("drawable/ic_notification.xml", vector(24, [("#FFFFFF", BOOK)])) + +for name in ("ic_launcher", "ic_launcher_round"): + w(f"mipmap-anydpi-v26/{name}.xml", HEAD + """<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android"> + <background android:drawable="@color/ic_launcher_background" /> + <foreground android:drawable="@drawable/ic_launcher_foreground" /> + <monochrome android:drawable="@drawable/ic_launcher_foreground" /> +</adaptive-icon> +""") + +print("resources written to", RES)