diff --git a/.github/actions/setup-custom-maven/action.yml b/.github/actions/setup-custom-maven/action.yml new file mode 100644 index 00000000..4f4f084d --- /dev/null +++ b/.github/actions/setup-custom-maven/action.yml @@ -0,0 +1,13 @@ +name: Setup Custom Maven Repo +description: Downloads custom-built onnxruntime/opencv maven artifacts from custom-lib-builds releases into maven-local/ + +runs: + using: composite + steps: + # Cache may not applicable for onnxruntime, as it may change its opset while + # not changing its version. + # Since GitHub Actions should have quite fast Internet access to its own + # release service, we don't set up caches here. + - name: Download and extract custom maven repo + run: bash .github/scripts/setup-custom-maven.sh + shell: bash diff --git a/.github/actions/setup-ocr-model/action.yml b/.github/actions/setup-ocr-model/action.yml index 40441e31..a112dcf0 100644 --- a/.github/actions/setup-ocr-model/action.yml +++ b/.github/actions/setup-ocr-model/action.yml @@ -9,7 +9,7 @@ runs: uses: actions/cache@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v6.0.0 with: path: app/src/main/assets/ocr - key: ocr-models_crnn-1.0.2 + key: ocr-models_crnn-1.0.3 - name: Ensure CRNN OCR model asset folder exists if: steps.cache-ocr.outputs.cache-hit != 'true' diff --git a/.github/scripts/setup-custom-maven.sh b/.github/scripts/setup-custom-maven.sh new file mode 100755 index 00000000..2624f467 --- /dev/null +++ b/.github/scripts/setup-custom-maven.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# setup-custom-maven.sh +# +# Downloads the custom-built onnxruntime/opencv maven artifacts from the +# custom-lib-builds stable release tags and extracts them into maven-local/. +# maven-local/ uses the same GAVs as the official artifacts, so Gradle +# resolves the custom builds first (settings.gradle.kts local repo block). +# +# Hard-fail: a download error aborts the build. Falling back to the official +# artifacts would ship an APK that differs from what was verified locally. +set -euo pipefail + +# .github/scripts/ -> repo root +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +MAVEN_LOCAL="${MAVEN_LOCAL:-$ROOT/maven-local}" + +# Stable tags, overwritten on publish (URL stays the same, content changes). +# No actions/cache: a version-based cache key would serve stale artifacts +# after a republish. GitHub Actions has fast access to its own releases, +# so we just re-download (~125MB) on every run. +ORT_TAG="${ORT_TAG:-onnxruntime-1.26.0}" +OPENCV_TAG="${OPENCV_TAG:-opencv-5.0.0}" +BASE_URL="https://github.com/ArcaeaOffline/custom-lib-builds/releases/download" + +mkdir -p "$MAVEN_LOCAL" + +download_and_extract() { # $1=tag, $2=local zip path + curl -fL -o "$2" "$BASE_URL/$1/maven_repo.zip" + unzip -q -o "$2" -d "$MAVEN_LOCAL" + rm -f "$2" +} + +download_and_extract "$ORT_TAG" /tmp/ort-maven.zip +download_and_extract "$OPENCV_TAG" /tmp/opencv-maven.zip + +# The two zips have com/ and org/ roots; they merge into maven-local/ +# without conflicts. Verify the expected AARs actually landed. +test -f "$MAVEN_LOCAL/com/microsoft/onnxruntime/onnxruntime-android/1.26.0/onnxruntime-android-1.26.0.aar" +test -f "$MAVEN_LOCAL/org/opencv/opencv/5.0.0/opencv-5.0.0.aar" diff --git a/.github/scripts/verify-apk-signing.sh b/.github/scripts/verify-apk-signing.sh new file mode 100755 index 00000000..bc36ae70 --- /dev/null +++ b/.github/scripts/verify-apk-signing.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# verify-apk-signing.sh +# +# Verifies signatures of all unstableRelease APKs. +# Hard-fail: unsigned or invalidly signed APKs must abort the build. +set -euo pipefail + +# Use the newest build-tools for this run +BUILD_TOOLS_VERSION="$(find "$ANDROID_HOME/build-tools" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | sort -V | tail -n 1)" +PATH="$ANDROID_HOME/build-tools/$BUILD_TOOLS_VERSION:$PATH" + +APK_DIR="${APK_DIR:-app/build/outputs/apk/unstable/release}" + +for apk in \ + "$APK_DIR/app-unstable-arm64-v8a-release.apk" \ + "$APK_DIR/app-unstable-armeabi-v7a-release.apk" \ + "$APK_DIR/app-unstable-x86_64-release.apk" \ + "$APK_DIR/app-unstable-x86-release.apk" \ + "$APK_DIR/app-unstable-universal-release.apk"; do + apksigner verify --verbose --min-sdk-version 24 "$apk" +done diff --git a/.github/scripts/verify-custom-libs.sh b/.github/scripts/verify-custom-libs.sh new file mode 100755 index 00000000..49bfbf98 --- /dev/null +++ b/.github/scripts/verify-custom-libs.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# Verifies the built APKs use the custom-built onnxruntime/opencv +# from maven-local/ instead of silently falling back to the +# official artifacts. +# +# Size is the primary signal: the custom .so files are ~50% the size +# of the official ones, far outside the +/-10% tolerance. +# sha256 hashes are reported for reference only: release build may +# strip on the .so files, so hashes legitimately differ there. +# +# Soft-fail: mismatches are reported as warnings in the job summary, +# but don't terminate the build. +set -euo pipefail + +# .github/scripts/ -> repo root +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +MAVEN_LOCAL="${MAVEN_LOCAL:-$ROOT/maven-local}" +APK_DIR="${APK_DIR:-$ROOT/app/build/outputs/apk/unstable/release}" + +# | +# Keep in sync with .github/actions/setup-custom-maven/action.yml +LIBS=( + "libonnxruntime.so|com/microsoft/onnxruntime/onnxruntime-android/1.26.0/onnxruntime-android-1.26.0.aar" + "libopencv_java5.so|org/opencv/opencv/5.0.0/opencv-5.0.0.aar" +) +# universal APK is the union of these four, so no special checks for it +ABIS=(arm64-v8a armeabi-v7a x86 x86_64) + +# sha256 of a file inside a zip; empty output if the entry is missing +# $1=zip file, $2=path inside zip +file_in_zip_sha256() { + if ! unzip -l "$1" "$2" >/dev/null 2>&1; then + return 1 + fi + unzip -p "$1" "$2" | sha256sum | cut -d' ' -f1 +} + +# size of a file inside a zip; empty output if the entry is missing +# $1=zip file, $2=path inside zip +file_in_zip_size() { + if ! unzip -l "$1" "$2" >/dev/null 2>&1; then + return 1 + fi + unzip -p "$1" "$2" | wc -c +} + +warnings=() + +{ + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + echo "## Custom build verification" + echo "" + fi + for entry in "${LIBS[@]}"; do + so="${entry%%|*}" + aar="$MAVEN_LOCAL/${entry#*|}" + echo "### $so" + echo "" + echo "| ABI | Size (maven-local, B) | Size (APK, B) | sha256 (maven-local) | sha256 (APK) | Result |" + echo "| --- | --- | --- | --- | --- | --- |" + for abi in "${ABIS[@]}"; do + apk="$APK_DIR/app-unstable-$abi-release.apk" + if [ ! -f "$apk" ]; then + warnings+=("missing APK: $apk") + echo "| $abi | - | - | - | - | APK missing |" + continue + fi + custom="$(file_in_zip_sha256 "$aar" "jni/$abi/$so" || true)" + packed="$(file_in_zip_sha256 "$apk" "lib/$abi/$so" || true)" + custom_size="$(file_in_zip_size "$aar" "jni/$abi/$so" || true)" + packed_size="$(file_in_zip_size "$apk" "lib/$abi/$so" || true)" + + result="OK" + if [ -z "$custom" ] || [ -z "$custom_size" ]; then + warnings+=("$abi/$so: custom .so not found in $aar") + result="NOT FOUND" + elif [ -z "$packed" ] || [ -z "$packed_size" ]; then + warnings+=("$abi/$so: .so not found in APK") + result="NOT FOUND" + elif [ "$packed_size" -lt $((custom_size * 90 / 100)) ] || \ + [ "$packed_size" -gt $((custom_size * 110 / 100)) ]; then + warnings+=("$abi/$so: size ${packed_size}B, expected ~${custom_size}B") + result="SIZE MISMATCH" + fi + echo "| $abi | ${custom_size:--} | ${packed_size:--} | ${custom:0:12}.. | ${packed:0:12}.. | $result |" + done + echo "" + done +} >>"${GITHUB_STEP_SUMMARY:-/dev/null}" + +if [ "${#warnings[@]}" -gt 0 ]; then + for w in "${warnings[@]}"; do + echo "::warning::$w" + done + # Non-zero exit marks the step yellow; the workflow step uses + # continue-on-error so this never blocks artifact upload/release. + exit 1 +fi +exit 0 diff --git a/.github/workflows/build_unstable.yml b/.github/workflows/build_unstable.yml index 3a12c0e4..41c6e211 100644 --- a/.github/workflows/build_unstable.yml +++ b/.github/workflows/build_unstable.yml @@ -26,6 +26,9 @@ jobs: - name: Setup Toolchain uses: ./.github/actions/setup-toolchain + - name: Setup Custom Maven + uses: ./.github/actions/setup-custom-maven + - name: Release keystore file run: base64 -d <<< "${{ secrets.KEYSTORE_FILE_BASE64 }}" > arcaea_offline.jks @@ -39,17 +42,11 @@ jobs: run: ./gradlew assembleUnstableRelease - name: Verify APK signing - run: | - # Locate the latest build tools - BUILD_TOOLS_VERSION=$(ls $ANDROID_HOME/build-tools/ | sort -V | tail -n 1) - # Adding build tools to PATH for this step only - PATH=$ANDROID_HOME/build-tools/$BUILD_TOOLS_VERSION:$PATH - - apksigner verify --verbose --min-sdk-version 24 app/build/outputs/apk/unstable/release/app-unstable-arm64-v8a-release.apk - apksigner verify --verbose --min-sdk-version 24 app/build/outputs/apk/unstable/release/app-unstable-armeabi-v7a-release.apk - apksigner verify --verbose --min-sdk-version 24 app/build/outputs/apk/unstable/release/app-unstable-x86_64-release.apk - apksigner verify --verbose --min-sdk-version 24 app/build/outputs/apk/unstable/release/app-unstable-x86-release.apk - apksigner verify --verbose --min-sdk-version 24 app/build/outputs/apk/unstable/release/app-unstable-universal-release.apk + run: bash .github/scripts/verify-apk-signing.sh + + - name: Verify custom-built libs in APK + continue-on-error: true + run: bash .github/scripts/verify-custom-libs.sh - name: Upload build result artifact (arm64-v8a) uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index eb2e7e04..71839b6e 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -29,6 +29,9 @@ jobs: - name: Setup Toolchain uses: ./.github/actions/setup-toolchain + - name: Setup Custom Maven + uses: ./.github/actions/setup-custom-maven + - name: Release local.properties file run: cp local.defaults.properties local.properties diff --git a/.github/workflows/connected-android-test.yml b/.github/workflows/connected-android-test.yml index ebed0e47..3d6c49aa 100644 --- a/.github/workflows/connected-android-test.yml +++ b/.github/workflows/connected-android-test.yml @@ -35,6 +35,9 @@ jobs: - name: Setup Toolchain uses: ./.github/actions/setup-toolchain + - name: Setup Custom Maven + uses: ./.github/actions/setup-custom-maven + - name: Release local.properties file run: cp local.defaults.properties local.properties diff --git a/.gitignore b/.gitignore index 15e5dd26..c4647eb7 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,9 @@ app/src/main/assets/ocr/model*.onnx app/src/main/assets/ocr/model*.ort app/src/main/assets/ocr/model_info.json +# Local maven repo (custom-built artifacts: onnxruntime, opencv) +/maven-local/ + # Android Studio added /.idea/caches /.idea/libraries diff --git a/README.md b/README.md new file mode 100644 index 00000000..1758c764 --- /dev/null +++ b/README.md @@ -0,0 +1,13 @@ +# Arcaea Offline Android Client + +## Custom Maven Repo (Optional) + +To reduce APK size, this project builds custom ONNX Runtime and OpenCV in [ArcaeaOffline/custom-lib-builds](https://github.com/ArcaeaOffline/custom-lib-builds), with unused features removed. + +`settings.gradle.kts` gives the local maven repo the highest priority, simply download the matching artifacts from the custom-lib-builds Releases and unpack them. + +If the local maven repo does not exist, Gradle falls back to Maven Central. This only affects APK size, not functionality. + +## License + +[GPL-3.0](LICENSE) diff --git a/app/schemas/xyz.sevive.arcaeaoffline.database.OcrQueueDatabase/3.json b/app/schemas/xyz.sevive.arcaeaoffline.database.OcrQueueDatabase/3.json deleted file mode 100644 index b9ab7565..00000000 --- a/app/schemas/xyz.sevive.arcaeaoffline.database.OcrQueueDatabase/3.json +++ /dev/null @@ -1,182 +0,0 @@ -{ - "formatVersion": 1, - "database": { - "version": 3, - "identityHash": "d2d3b16df860c5197c174a0d39691162", - "entities": [ - { - "tableName": "ocr_queue_tasks", - "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `inserted_at` INTEGER NOT NULL, `file_uri` TEXT NOT NULL, `status` INTEGER NOT NULL, `result` TEXT, `play_result` TEXT, `error_type` TEXT, `error_message` TEXT)", - "fields": [ - { - "fieldPath": "id", - "columnName": "id", - "affinity": "INTEGER", - "notNull": true - }, - { - "fieldPath": "insertedAt", - "columnName": "inserted_at", - "affinity": "INTEGER", - "notNull": true - }, - { - "fieldPath": "fileUri", - "columnName": "file_uri", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "status", - "columnName": "status", - "affinity": "INTEGER", - "notNull": true - }, - { - "fieldPath": "result", - "columnName": "result", - "affinity": "TEXT" - }, - { - "fieldPath": "playResult", - "columnName": "play_result", - "affinity": "TEXT" - }, - { - "fieldPath": "errorType", - "columnName": "error_type", - "affinity": "TEXT" - }, - { - "fieldPath": "errorMessage", - "columnName": "error_message", - "affinity": "TEXT" - } - ], - "primaryKey": { - "autoGenerate": true, - "columnNames": [ - "id" - ] - }, - "indices": [ - { - "name": "index_ocr_queue_tasks_file_uri", - "unique": true, - "columnNames": [ - "file_uri" - ], - "orders": [], - "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_ocr_queue_tasks_file_uri` ON `${TABLE_NAME}` (`file_uri`)" - } - ] - }, - { - "tableName": "ocr_queue_enqueue_buffer", - "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `batch_id` INTEGER NOT NULL, `uri` TEXT NOT NULL, `uri_type` INTEGER NOT NULL, `checked` INTEGER NOT NULL, `should_insert` INTEGER NOT NULL, FOREIGN KEY(`batch_id`) REFERENCES `ocr_queue_enqueue_batches`(`id`) ON UPDATE CASCADE ON DELETE CASCADE )", - "fields": [ - { - "fieldPath": "id", - "columnName": "id", - "affinity": "INTEGER", - "notNull": true - }, - { - "fieldPath": "batchId", - "columnName": "batch_id", - "affinity": "INTEGER", - "notNull": true - }, - { - "fieldPath": "uri", - "columnName": "uri", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "uriType", - "columnName": "uri_type", - "affinity": "INTEGER", - "notNull": true - }, - { - "fieldPath": "checked", - "columnName": "checked", - "affinity": "INTEGER", - "notNull": true - }, - { - "fieldPath": "shouldInsert", - "columnName": "should_insert", - "affinity": "INTEGER", - "notNull": true - } - ], - "primaryKey": { - "autoGenerate": true, - "columnNames": [ - "id" - ] - }, - "indices": [ - { - "name": "index_ocr_queue_enqueue_buffer_uri", - "unique": true, - "columnNames": [ - "uri" - ], - "orders": [], - "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_ocr_queue_enqueue_buffer_uri` ON `${TABLE_NAME}` (`uri`)" - } - ], - "foreignKeys": [ - { - "table": "ocr_queue_enqueue_batches", - "onDelete": "CASCADE", - "onUpdate": "CASCADE", - "columns": [ - "batch_id" - ], - "referencedColumns": [ - "id" - ] - } - ] - }, - { - "tableName": "ocr_queue_enqueue_batches", - "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `inserted_at` INTEGER NOT NULL, `options` TEXT NOT NULL)", - "fields": [ - { - "fieldPath": "id", - "columnName": "id", - "affinity": "INTEGER", - "notNull": true - }, - { - "fieldPath": "insertedAt", - "columnName": "inserted_at", - "affinity": "INTEGER", - "notNull": true - }, - { - "fieldPath": "options", - "columnName": "options", - "affinity": "TEXT", - "notNull": true - } - ], - "primaryKey": { - "autoGenerate": true, - "columnNames": [ - "id" - ] - } - } - ], - "setupQueries": [ - "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", - "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'd2d3b16df860c5197c174a0d39691162')" - ] - } -} \ No newline at end of file diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/data/Paths.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/data/Paths.kt index 7bbc80e2..7ac35daf 100644 --- a/app/src/main/java/xyz/sevive/arcaeaoffline/data/Paths.kt +++ b/app/src/main/java/xyz/sevive/arcaeaoffline/data/Paths.kt @@ -9,7 +9,6 @@ import kotlinx.io.files.Path class OcrDependencyPaths { val parentDir = Path(FileKit.filesDir.absolutePath()) / "ocr" / "dependencies" - val knnModelFile = parentDir / "digits.knn.dat" val phashDatabaseFile = parentDir / "image-phash.db" val imageHashesDatabaseFile = parentDir / "image-hashes.db" } diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/data/maintenance/AppDataMaintenanceManager.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/data/maintenance/AppDataMaintenanceManager.kt index 53c328f1..a2966ecc 100644 --- a/app/src/main/java/xyz/sevive/arcaeaoffline/data/maintenance/AppDataMaintenanceManager.kt +++ b/app/src/main/java/xyz/sevive/arcaeaoffline/data/maintenance/AppDataMaintenanceManager.kt @@ -3,6 +3,7 @@ package xyz.sevive.arcaeaoffline.data.maintenance import android.content.Context import co.touchlab.kermit.Logger import kotlinx.coroutines.CancellationException +import xyz.sevive.arcaeaoffline.data.maintenance.tasks.LegacyKnnModelCleanUpTask import xyz.sevive.arcaeaoffline.data.maintenance.tasks.ProtoDataStoreCleanUpTask class AppDataMaintenanceManager( @@ -13,6 +14,7 @@ class AppDataMaintenanceManager( private val tasks = listOf( ProtoDataStoreCleanUpTask(context), + LegacyKnnModelCleanUpTask(), ) suspend fun runAllTasks() = diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/data/maintenance/tasks/LegacyKnnModelCleanUpTask.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/data/maintenance/tasks/LegacyKnnModelCleanUpTask.kt new file mode 100644 index 00000000..2591b903 --- /dev/null +++ b/app/src/main/java/xyz/sevive/arcaeaoffline/data/maintenance/tasks/LegacyKnnModelCleanUpTask.kt @@ -0,0 +1,32 @@ +package xyz.sevive.arcaeaoffline.data.maintenance.tasks + +import co.touchlab.kermit.Logger +import io.github.vinceglb.filekit.FileKit +import io.github.vinceglb.filekit.absolutePath +import io.github.vinceglb.filekit.filesDir +import io.github.vinceglb.filekit.utils.div +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.io.files.Path +import kotlinx.io.files.SystemFileSystem +import xyz.sevive.arcaeaoffline.data.maintenance.AppDataMaintenanceTask + +class LegacyKnnModelCleanUpTask : AppDataMaintenanceTask { + override val id = "legacy_knn_model_cleanup" + override val version = 2 + + private val logger = Logger.withTag("LegacyKnnModelCleanUpTask") + + // Path is kept independent of OcrDependencyPaths.knnModelFile, which was removed + // together with the KNearest OCR pipeline. This task cleans up the leftover file + // on devices that installed the app before the removal. + private val legacyFile: Path = + Path(FileKit.filesDir.absolutePath()) / "ocr" / "dependencies" / "digits.knn.dat" + + override suspend fun execute() = + withContext(Dispatchers.IO) { + if (!SystemFileSystem.exists(legacyFile)) return@withContext + SystemFileSystem.delete(legacyFile) + logger.i { "Deleted legacy KNN model file: $legacyFile" } + } +} diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/database/OcrQueueDatabase.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/database/OcrQueueDatabase.kt index f2b6d643..9eaf799c 100644 --- a/app/src/main/java/xyz/sevive/arcaeaoffline/database/OcrQueueDatabase.kt +++ b/app/src/main/java/xyz/sevive/arcaeaoffline/database/OcrQueueDatabase.kt @@ -1,7 +1,6 @@ package xyz.sevive.arcaeaoffline.database import android.content.Context -import androidx.room.AutoMigration import androidx.room.Database import androidx.room.Room import androidx.room.RoomDatabase @@ -21,15 +20,11 @@ import xyz.sevive.arcaeaoffline.database.daos.OcrQueueTaskDao import xyz.sevive.arcaeaoffline.database.entities.OcrQueueStagingBatch import xyz.sevive.arcaeaoffline.database.entities.OcrQueueStagingItem import xyz.sevive.arcaeaoffline.database.entities.OcrQueueTask -import xyz.sevive.arcaeaoffline.database.migrations.OcrQueueMigration3To4 @Database( entities = [OcrQueueTask::class, OcrQueueStagingItem::class, OcrQueueStagingBatch::class], version = 4, exportSchema = true, - autoMigrations = [ - AutoMigration(from = 3, to = 4, spec = OcrQueueMigration3To4::class), - ], ) @TypeConverters( UriConverters::class, diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/database/migrations/OcrQueueMigration3To4.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/database/migrations/OcrQueueMigration3To4.kt deleted file mode 100644 index f7d1ddaf..00000000 --- a/app/src/main/java/xyz/sevive/arcaeaoffline/database/migrations/OcrQueueMigration3To4.kt +++ /dev/null @@ -1,8 +0,0 @@ -package xyz.sevive.arcaeaoffline.database.migrations - -import androidx.room.RenameTable -import androidx.room.migration.AutoMigrationSpec - -@RenameTable(fromTableName = "ocr_queue_enqueue_buffer", toTableName = "ocr_queue_staging_item") -@RenameTable(fromTableName = "ocr_queue_enqueue_batches", toTableName = "ocr_queue_staging_batch") -class OcrQueueMigration3To4 : AutoMigrationSpec diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/datastore/OcrQueueParallelism.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/datastore/OcrQueueParallelism.kt new file mode 100644 index 00000000..5c175f30 --- /dev/null +++ b/app/src/main/java/xyz/sevive/arcaeaoffline/datastore/OcrQueueParallelism.kt @@ -0,0 +1,12 @@ +package xyz.sevive.arcaeaoffline.datastore + +/** + * Device-dependent parallelism strategy for the OCR queue. Kept separate from + * [OcrQueuePreferences] (a serializable data class) so the "single source" for + * these runtime-derived values is not tied to the preferences schema itself. + */ +object OcrQueueParallelism { + fun defaultCount(): Int = (Runtime.getRuntime().availableProcessors() / 2).coerceAtLeast(1) + + fun countRange(): IntRange = 1..(Runtime.getRuntime().availableProcessors() * 2).coerceAtLeast(2) +} diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/datastore/OcrQueuePreferences.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/datastore/OcrQueuePreferences.kt index 1d60c31c..a9532333 100644 --- a/app/src/main/java/xyz/sevive/arcaeaoffline/datastore/OcrQueuePreferences.kt +++ b/app/src/main/java/xyz/sevive/arcaeaoffline/datastore/OcrQueuePreferences.kt @@ -21,7 +21,7 @@ data class OcrQueuePreferences( @SerialName("check_is_arcaea_image") val checkIsArcaeaImage: Boolean = OcrQueueStagingOptions.DEFAULTS.checkIsArcaeaImage, @SerialName("parallel_count") - val parallelCount: Int = (Runtime.getRuntime().availableProcessors() / 2).coerceAtLeast(1), + val parallelCount: Int = OcrQueueParallelism.defaultCount(), ) object OcrQueuePreferencesSerializer : OkioSerializer { diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/di/AppModule.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/di/AppModule.kt index 148e1ac0..f7576e33 100644 --- a/app/src/main/java/xyz/sevive/arcaeaoffline/di/AppModule.kt +++ b/app/src/main/java/xyz/sevive/arcaeaoffline/di/AppModule.kt @@ -40,6 +40,7 @@ import xyz.sevive.arcaeaoffline.ui.screens.database.manage.DatabaseManageViewMod import xyz.sevive.arcaeaoffline.ui.screens.database.playresultlist.DatabasePlayResultListViewModel import xyz.sevive.arcaeaoffline.ui.screens.database.r30list.DatabaseR30ListViewModel import xyz.sevive.arcaeaoffline.ui.screens.ocr.dependencies.OcrDependenciesScreenViewModel +import xyz.sevive.arcaeaoffline.ui.screens.ocr.performance.OcrPerformanceScreenViewModel import xyz.sevive.arcaeaoffline.ui.screens.ocr.queue.OcrQueueScreenViewModel import xyz.sevive.arcaeaoffline.ui.screens.ocr.queue.preferences.OcrQueuePreferencesViewModel import xyz.sevive.arcaeaoffline.ui.screens.ocr.queue.staging.OcrQueueStagingViewModel @@ -109,6 +110,7 @@ val appModule = viewModel() viewModel() viewModel() + viewModel() viewModel() viewModel() viewModel() diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/helpers/DeviceOcrHelper.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/helpers/DeviceOcrHelper.kt index dcd9ba76..279c90ba 100644 --- a/app/src/main/java/xyz/sevive/arcaeaoffline/helpers/DeviceOcrHelper.kt +++ b/app/src/main/java/xyz/sevive/arcaeaoffline/helpers/DeviceOcrHelper.kt @@ -21,7 +21,6 @@ import kotlinx.datetime.toInstant import kotlinx.io.buffered import org.opencv.core.MatOfByte import org.opencv.imgcodecs.Imgcodecs -import org.opencv.ml.KNearest import xyz.sevive.arcaeaoffline.core.database.entities.PlayResult import xyz.sevive.arcaeaoffline.core.ocr.ImageHashesDatabase import xyz.sevive.arcaeaoffline.core.ocr.device.CropBlackEdges @@ -35,6 +34,7 @@ import xyz.sevive.arcaeaoffline.core.ocr.device.rois.extractor.DeviceRoisExtract import xyz.sevive.arcaeaoffline.core.ocr.device.rois.masker.DeviceRoisMaskerAutoT1 import xyz.sevive.arcaeaoffline.core.ocr.device.rois.masker.DeviceRoisMaskerAutoT2 import xyz.sevive.arcaeaoffline.core.ocr.device.toPlayResult +import xyz.sevive.arcaeaoffline.core.ocr.opencv.use import xyz.sevive.arcaeaoffline.helpers.context.getFilename import kotlin.time.Instant @@ -56,45 +56,48 @@ object DeviceOcrHelper { suspend fun ocrImage( imageUri: Uri, - kNearestModel: KNearest, imageHashesDatabase: ImageHashesDatabase, ortSession: OrtSession, ): DeviceOcrResult { val byteArray = PlatformFile(imageUri).readBytes() - val img = Imgcodecs.imdecode(MatOfByte(*byteArray), Imgcodecs.IMREAD_COLOR) - val imgCropped = CropBlackEdges.crop(img) - val roisAutoType = DeviceRoisAutoSelector.select(img) - val rois = - when (roisAutoType) { - DeviceRoisAutoSelectorResult.T1 -> { - DeviceRoisAutoT1( - imgCropped.width(), - imgCropped.height(), - ) - } + return MatOfByte(*byteArray).use { matOfBytes -> + Imgcodecs.imdecode(matOfBytes, Imgcodecs.IMREAD_COLOR).use { img -> + val roisAutoType = DeviceRoisAutoSelector.select(img) + + CropBlackEdges.crop(img).use { imgCropped -> + val rois = + when (roisAutoType) { + DeviceRoisAutoSelectorResult.T1 -> { + DeviceRoisAutoT1( + imgCropped.width(), + imgCropped.height(), + ) + } + + else -> { + DeviceRoisAutoT2( + imgCropped.width(), + imgCropped.height(), + ) + } + } + val extractor = DeviceRoisExtractor(rois, imgCropped) + val masker = + when (roisAutoType) { + DeviceRoisAutoSelectorResult.T1 -> DeviceRoisMaskerAutoT1() + else -> DeviceRoisMaskerAutoT2() + } - else -> { - DeviceRoisAutoT2( - imgCropped.width(), - imgCropped.height(), - ) + DeviceOcr( + extractor = extractor, + masker = masker, + ortSession = ortSession, + hashesDb = imageHashesDatabase, + ).ocr() } } - val extractor = DeviceRoisExtractor(rois, imgCropped) - val masker = - when (roisAutoType) { - DeviceRoisAutoSelectorResult.T1 -> DeviceRoisMaskerAutoT1() - else -> DeviceRoisMaskerAutoT2() - } - - return DeviceOcr( - extractor = extractor, - masker = masker, - kNearestModel = kNearestModel, - ortSession = ortSession, - hashesDb = imageHashesDatabase, - ).ocr() + } } fun readImageDateFromExif( diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/helpers/OcrDependencyLoader.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/helpers/OcrDependencyLoader.kt index 7b7835c4..e242cb38 100644 --- a/app/src/main/java/xyz/sevive/arcaeaoffline/helpers/OcrDependencyLoader.kt +++ b/app/src/main/java/xyz/sevive/arcaeaoffline/helpers/OcrDependencyLoader.kt @@ -4,16 +4,9 @@ import androidx.sqlite.SQLiteConnection import androidx.sqlite.driver.bundled.BundledSQLiteDriver import androidx.sqlite.driver.bundled.SQLITE_OPEN_READONLY import kotlinx.io.files.Path -import org.opencv.ml.KNearest import xyz.sevive.arcaeaoffline.data.OcrDependencyPaths object OcrDependencyLoader { - fun kNearestModel(path: Path): KNearest = KNearest.load(path.toString()) - - fun kNearestModel(ocrDependencyPaths: OcrDependencyPaths) = kNearestModel(ocrDependencyPaths.knnModelFile) - - fun kNearestModel() = kNearestModel(OcrDependencyPaths()) - fun imageHashesSQLiteDatabase(path: Path): SQLiteConnection = BundledSQLiteDriver().open(path.toString(), SQLITE_OPEN_READONLY) fun imageHashesSQLiteDatabase(ocrDependencyPaths: OcrDependencyPaths) = diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/helpers/OcrDependencyStatus.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/helpers/OcrDependencyStatus.kt index d7b7ace6..1f73291a 100644 --- a/app/src/main/java/xyz/sevive/arcaeaoffline/helpers/OcrDependencyStatus.kt +++ b/app/src/main/java/xyz/sevive/arcaeaoffline/helpers/OcrDependencyStatus.kt @@ -1,7 +1,5 @@ package xyz.sevive.arcaeaoffline.helpers -import ai.onnxruntime.OnnxModelMetadata -import xyz.sevive.arcaeaoffline.core.ocr.device.DeviceOcrOnnxHelper import kotlin.time.Instant enum class OcrDependencyStatus { OK, ERROR, WARNING, ABSENCE, UNKNOWN } @@ -16,33 +14,15 @@ interface OcrDependencyStatusDetail { fun details(): String? { if (exception == null) return null - return exception!!.message ?: exception.toString() - } -} -data class KNearestModelStatusDetail( - override val absence: Boolean = false, - override val exception: Exception? = null, - val varCount: Int? = null, - val isTrained: Boolean = false, -) : OcrDependencyStatusDetail { - override fun status(): OcrDependencyStatus { - if (absence) return OcrDependencyStatus.ABSENCE - if (exception != null || !isTrained) return OcrDependencyStatus.ERROR - - return when (varCount) { - null -> OcrDependencyStatus.UNKNOWN - 81 -> OcrDependencyStatus.OK - else -> OcrDependencyStatus.WARNING - } + return exception?.let { + buildString { + append(it::class.simpleName ?: "Exception") + append(": ") + append(it.message) + } + } ?: exception.toString() } - - override fun summary(): String? = - when { - exception != null -> exception::class.simpleName ?: "Error" - varCount != null -> "varCount $varCount" - else -> null - } } data class ImageHashesDatabaseStatusDetail( @@ -79,34 +59,36 @@ data class ImageHashesDatabaseStatusDetail( data class CrnnModelStatusDetail( override val absence: Boolean = false, override val exception: Exception? = null, - val modelMetadata: OnnxModelMetadata? = null, + val modelVersion: List? = null, + val producerName: String? = null, + val producerVersion: String? = null, + val domain: String? = null, + val graphName: String? = null, val inputNames: Set? = null, val outputNames: Set? = null, + val builtTimestamp: Instant? = null, + val patchedTimestamp: Instant? = null, ) : OcrDependencyStatusDetail { override fun status(): OcrDependencyStatus { if (absence) return OcrDependencyStatus.ABSENCE if (exception != null) return OcrDependencyStatus.ERROR return when { - modelMetadata == null -> OcrDependencyStatus.UNKNOWN - modelMetadata.version == 0L -> OcrDependencyStatus.WARNING + modelVersion.isNullOrEmpty() -> OcrDependencyStatus.UNKNOWN else -> OcrDependencyStatus.OK } } - private val builtTimestampRaw get() = modelMetadata?.customMetadata?.get("built_timestamp") - private val builtTimestamp = builtTimestampRaw?.let { Instant.fromEpochSeconds(it.toLong()) } - private val builtTimestampReadable = builtTimestamp?.formatAsLocalizedDateTime() + private val modelVersionString = modelVersion?.takeIf { it.isNotEmpty() }?.let { "v" + it.joinToString(".") } override fun summary(): String? { if (absence) return null if (exception != null) return exception::class.simpleName ?: "Error" - if (modelMetadata == null) return null val parts = mutableListOf() - parts.add(DeviceOcrOnnxHelper.modelVersionString(modelMetadata.version)) - builtTimestampReadable?.let { parts.add(it) } + modelVersionString?.let { parts.add(it) } + builtTimestamp?.let { parts.add(it.formatAsLocalizedDateTime()) } return parts.filter { it.isNotEmpty() }.joinToString(", ") } @@ -117,17 +99,19 @@ data class CrnnModelStatusDetail( val parts = mutableListOf() - modelMetadata?.let { - parts.add("version: ${it.version} (${DeviceOcrOnnxHelper.modelVersion(it.version)})") - parts.add("producer: ${it.producerName}") - parts.add("domain: ${it.domain}") - parts.add("graph_name: ${it.graphName}") + modelVersion?.takeIf { it.isNotEmpty() }?.let { + parts.add("version: $it ($modelVersionString)") } + producerName?.let { parts.add("producer: $it") } + producerVersion?.let { parts.add("producer_version: $it") } + domain?.let { parts.add("domain: $it") } + graphName?.let { parts.add("graph_name: $it") } parts.add("inputs: $inputNames") parts.add("outputs: $outputNames") - builtTimestampRaw?.let { parts.add("built_timestamp: $it ($builtTimestamp)") } + builtTimestamp?.let { parts.add("built: $it") } + patchedTimestamp?.let { parts.add("patched: $it") } return when (val res = parts.joinToString("\n")) { "" -> null diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/helpers/OcrDependencyStatusBuilder.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/helpers/OcrDependencyStatusBuilder.kt index adf20250..bc8fc065 100644 --- a/app/src/main/java/xyz/sevive/arcaeaoffline/helpers/OcrDependencyStatusBuilder.kt +++ b/app/src/main/java/xyz/sevive/arcaeaoffline/helpers/OcrDependencyStatusBuilder.kt @@ -5,21 +5,10 @@ import kotlinx.io.files.SystemFileSystem import xyz.sevive.arcaeaoffline.core.ocr.ImageHashesDatabase import xyz.sevive.arcaeaoffline.core.ocr.device.DeviceOcrOnnxHelper import xyz.sevive.arcaeaoffline.data.OcrDependencyPaths +import kotlin.time.Instant import kotlin.use object OcrDependencyStatusBuilder { - fun kNearest(): KNearestModelStatusDetail { - try { - val paths = OcrDependencyPaths() - if (!SystemFileSystem.exists(paths.knnModelFile)) return KNearestModelStatusDetail(absence = true) - - val model = OcrDependencyLoader.kNearestModel() - return KNearestModelStatusDetail(varCount = model.varCount, isTrained = model.isTrained) - } catch (e: Exception) { - return KNearestModelStatusDetail(exception = e) - } - } - fun imageHashesDatabase(): ImageHashesDatabaseStatusDetail { try { val paths = OcrDependencyPaths() @@ -44,15 +33,25 @@ object OcrDependencyStatusBuilder { } fun crnnModel(context: Context): CrnnModelStatusDetail = - DeviceOcrOnnxHelper.createOrtSession(context).use { - try { + try { + // model_info.json can parse fine while the model asset itself is + // missing, empty or drifted from the json description (e.g. + // incomplete local debug assets, or a stale CI model cache) + val info = DeviceOcrOnnxHelper.verifyModelAsset(context) + with(info) { CrnnModelStatusDetail( - modelMetadata = it.metadata, - inputNames = it.inputNames.toSet(), // make a copy, same for below - outputNames = it.outputNames.toSet(), + modelVersion = patch?.version?.takeIf { it.isNotEmpty() }, + producerName = patch?.producerName, + producerVersion = patch?.producerVersion, + domain = patch?.domain, + graphName = patch?.graphName, + inputNames = patch?.inputNames?.toSet(), + outputNames = patch?.outputNames?.toSet(), + builtTimestamp = training.builtTimestamp.takeIf { it != 0L }?.let { Instant.fromEpochSeconds(it) }, + patchedTimestamp = patch?.patchedTimestamp?.takeIf { it != 0L }?.let { Instant.fromEpochSeconds(it) }, ) - } catch (e: Exception) { - CrnnModelStatusDetail(exception = e) } + } catch (e: Exception) { + CrnnModelStatusDetail(exception = e) } } diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/helpers/OcrQueueHelper.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/helpers/OcrQueueHelper.kt index d49add21..aaf34dc6 100644 --- a/app/src/main/java/xyz/sevive/arcaeaoffline/helpers/OcrQueueHelper.kt +++ b/app/src/main/java/xyz/sevive/arcaeaoffline/helpers/OcrQueueHelper.kt @@ -15,6 +15,7 @@ import org.opencv.core.MatOfByte import org.opencv.imgcodecs.Imgcodecs import org.opencv.imgproc.Imgproc import xyz.sevive.arcaeaoffline.core.ocr.device.ScreenshotDetect +import xyz.sevive.arcaeaoffline.core.ocr.opencv.use object OcrQueueHelper { suspend fun isUriImage(uri: Uri): Boolean = @@ -38,11 +39,14 @@ object OcrQueueHelper { async { val byteArray = PlatformFile(uri).readBytes() - val img = Imgcodecs.imdecode(MatOfByte(*byteArray), Imgcodecs.IMREAD_COLOR) - val imgHsv = Mat() - Imgproc.cvtColor(img, imgHsv, Imgproc.COLOR_BGR2HSV) - - ScreenshotDetect.isArcaeaScreenshot(imgHsv) + MatOfByte(*byteArray).use { matOfBytes -> + Imgcodecs.imdecode(matOfBytes, Imgcodecs.IMREAD_COLOR).use { img -> + Mat().use { imgHsv -> + Imgproc.cvtColor(img, imgHsv, Imgproc.COLOR_BGR2HSV) + ScreenshotDetect.isArcaeaScreenshot(imgHsv) + } + } + } }.await() } } diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/jobs/OcrQueueProcessingJob.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/jobs/OcrQueueProcessingJob.kt index 042a6de3..3bf7f28a 100644 --- a/app/src/main/java/xyz/sevive/arcaeaoffline/jobs/OcrQueueProcessingJob.kt +++ b/app/src/main/java/xyz/sevive/arcaeaoffline/jobs/OcrQueueProcessingJob.kt @@ -29,6 +29,7 @@ import xyz.sevive.arcaeaoffline.data.notification.Notifications import xyz.sevive.arcaeaoffline.database.entities.OcrQueueTask import xyz.sevive.arcaeaoffline.database.entities.OcrQueueTaskStatus import xyz.sevive.arcaeaoffline.database.repositories.OcrQueueTaskRepository +import xyz.sevive.arcaeaoffline.datastore.OcrQueueParallelism import xyz.sevive.arcaeaoffline.helpers.ArcaeaPlayResultValidator import xyz.sevive.arcaeaoffline.helpers.toWorkData import kotlin.time.Duration.Companion.milliseconds @@ -89,7 +90,7 @@ class OcrQueueProcessingJob( inputData .getInt( DATA_PARALLEL_COUNT, - Runtime.getRuntime().availableProcessors() / 2, + OcrQueueParallelism.defaultCount(), ).coerceAtLeast(1), ) diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/jobs/OcrQueueProcessingJobTaskExecutor.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/jobs/OcrQueueProcessingJobTaskExecutor.kt index 3f058d3b..0c46e59c 100644 --- a/app/src/main/java/xyz/sevive/arcaeaoffline/jobs/OcrQueueProcessingJobTaskExecutor.kt +++ b/app/src/main/java/xyz/sevive/arcaeaoffline/jobs/OcrQueueProcessingJobTaskExecutor.kt @@ -41,8 +41,6 @@ class OcrQueueOcrImageTaskExecutor( private val imageHashesDatabase = ImageHashesDatabase(imageHashesSQLiteDatabase) - private val kNearestModel = OcrDependencyLoader.kNearestModel() - override fun close() { ortSession.close() imageHashesSQLiteDatabase.close() @@ -60,7 +58,6 @@ class OcrQueueOcrImageTaskExecutor( val ocrResult = DeviceOcrHelper.ocrImage( uri, - kNearestModel, imageHashesDatabase, ortSession = ortSession, ) diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/ui/activities/EmergencyModeActivityViewModel.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/ui/activities/EmergencyModeActivityViewModel.kt index 81e16d6b..a45a7d2c 100644 --- a/app/src/main/java/xyz/sevive/arcaeaoffline/ui/activities/EmergencyModeActivityViewModel.kt +++ b/app/src/main/java/xyz/sevive/arcaeaoffline/ui/activities/EmergencyModeActivityViewModel.kt @@ -89,7 +89,6 @@ class EmergencyModeActivityViewModel( val paths = OcrDependencyPaths() viewModelScope.launch(Dispatchers.IO) { - if (SystemFileSystem.exists(paths.knnModelFile)) SystemFileSystem.delete(paths.knnModelFile) if (SystemFileSystem.exists(paths.phashDatabaseFile)) SystemFileSystem.delete(paths.phashDatabaseFile) if (SystemFileSystem.exists(paths.imageHashesDatabaseFile)) SystemFileSystem.delete(paths.imageHashesDatabaseFile) } diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/ui/activities/ocrfromshare/OcrFromShareOcrDependencyStatusCard.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/ui/activities/ocrfromshare/OcrFromShareOcrDependencyStatusCard.kt index d87c3b8e..23276b97 100644 --- a/app/src/main/java/xyz/sevive/arcaeaoffline/ui/activities/ocrfromshare/OcrFromShareOcrDependencyStatusCard.kt +++ b/app/src/main/java/xyz/sevive/arcaeaoffline/ui/activities/ocrfromshare/OcrFromShareOcrDependencyStatusCard.kt @@ -5,7 +5,6 @@ import androidx.compose.runtime.Composable import androidx.compose.ui.Modifier import xyz.sevive.arcaeaoffline.ui.components.ocr.OcrDependencyCrnnModelStatusViewer import xyz.sevive.arcaeaoffline.ui.components.ocr.OcrDependencyImageHashesDatabaseStatusViewer -import xyz.sevive.arcaeaoffline.ui.components.ocr.OcrDependencyKNearestModelStatusViewer @Composable internal fun OcrFromShareOcrDependencyStatusCard( @@ -13,7 +12,6 @@ internal fun OcrFromShareOcrDependencyStatusCard( modifier: Modifier = Modifier, ) { Card(modifier) { - OcrDependencyKNearestModelStatusViewer(uiState = uiState.kNearestModel) OcrDependencyImageHashesDatabaseStatusViewer(uiState = uiState.imageHashesDatabase) OcrDependencyCrnnModelStatusViewer(uiState = uiState.crnnModel) } diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/ui/activities/ocrfromshare/OcrFromShareViewModel.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/ui/activities/ocrfromshare/OcrFromShareViewModel.kt index 37f56569..c7a58b0f 100644 --- a/app/src/main/java/xyz/sevive/arcaeaoffline/ui/activities/ocrfromshare/OcrFromShareViewModel.kt +++ b/app/src/main/java/xyz/sevive/arcaeaoffline/ui/activities/ocrfromshare/OcrFromShareViewModel.kt @@ -38,7 +38,6 @@ import xyz.sevive.arcaeaoffline.helpers.OcrDependencyStatusBuilder import xyz.sevive.arcaeaoffline.permissions.storage.SaveBitmapToGallery import xyz.sevive.arcaeaoffline.ui.components.ocr.OcrDependencyCrnnModelStatusUiState import xyz.sevive.arcaeaoffline.ui.components.ocr.OcrDependencyImageHashesDatabaseStatusUiState -import xyz.sevive.arcaeaoffline.ui.components.ocr.OcrDependencyKNearestModelStatusUiState import kotlin.time.Clock class OcrFromShareViewModel( @@ -47,7 +46,6 @@ class OcrFromShareViewModel( private val ocrHistoryRepo: OcrHistoryRepository, ) : ViewModel() { class OcrDependencyViewersUiState( - val kNearestModel: OcrDependencyKNearestModelStatusUiState = OcrDependencyKNearestModelStatusUiState(), val imageHashesDatabase: OcrDependencyImageHashesDatabaseStatusUiState = OcrDependencyImageHashesDatabaseStatusUiState(), val crnnModel: OcrDependencyCrnnModelStatusUiState = OcrDependencyCrnnModelStatusUiState(), ) @@ -63,13 +61,11 @@ class OcrFromShareViewModel( fun reloadOcrDependencyViewersUiState(context: Context) { viewModelScope.launch(Dispatchers.IO) { - val kNearest = OcrDependencyStatusBuilder.kNearest() val imageHashesDatabase = OcrDependencyStatusBuilder.imageHashesDatabase() val crnnModel = OcrDependencyStatusBuilder.crnnModel(context) _ocrDependencyViewersUiState.value = OcrDependencyViewersUiState( - kNearestModel = OcrDependencyKNearestModelStatusUiState(kNearest), imageHashesDatabase = OcrDependencyImageHashesDatabaseStatusUiState(statusDetail = imageHashesDatabase), crnnModel = OcrDependencyCrnnModelStatusUiState(crnnModel), ) @@ -210,7 +206,6 @@ class OcrFromShareViewModel( withContext(Dispatchers.IO) { try { - val kNearestModel = OcrDependencyLoader.kNearestModel() val imageHashesSQLiteDatabase = OcrDependencyLoader.imageHashesSQLiteDatabase() @@ -219,7 +214,6 @@ class OcrFromShareViewModel( val imageHashesDatabase = ImageHashesDatabase(sqliteDb) DeviceOcrHelper.ocrImage( imageUri, - kNearestModel, imageHashesDatabase, ortSession = ortSession, ) diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/ui/components/ocr/OcrDependencyKNearestModelStatusViewer.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/ui/components/ocr/OcrDependencyKNearestModelStatusViewer.kt deleted file mode 100644 index be4aadb0..00000000 --- a/app/src/main/java/xyz/sevive/arcaeaoffline/ui/components/ocr/OcrDependencyKNearestModelStatusViewer.kt +++ /dev/null @@ -1,33 +0,0 @@ -package xyz.sevive.arcaeaoffline.ui.components.ocr - -import androidx.compose.runtime.Composable -import androidx.compose.runtime.remember -import androidx.compose.ui.Modifier -import androidx.compose.ui.graphics.vector.ImageVector -import androidx.compose.ui.res.stringResource -import androidx.compose.ui.res.vectorResource -import xyz.sevive.arcaeaoffline.R -import xyz.sevive.arcaeaoffline.helpers.KNearestModelStatusDetail - -data class OcrDependencyKNearestModelStatusUiState( - val statusDetail: KNearestModelStatusDetail = KNearestModelStatusDetail(), -) - -@Composable -fun OcrDependencyKNearestModelStatusViewer( - uiState: OcrDependencyKNearestModelStatusUiState, - modifier: Modifier = Modifier, -) { - val status = remember(uiState) { uiState.statusDetail.status() } - val summary = remember(uiState) { uiState.statusDetail.summary() } - val details = remember(uiState) { uiState.statusDetail.details() } - - OcrDependencyStatusViewer( - icon = ImageVector.vectorResource(R.drawable.ic_knearest_model), - title = stringResource(R.string.ocr_dependency_knn_model), - status = status, - summary = summary, - details = details, - modifier = modifier, - ) -} diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/ui/navigation/Ocr.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/ui/navigation/Ocr.kt index 266cdad6..d4bd4f1f 100644 --- a/app/src/main/java/xyz/sevive/arcaeaoffline/ui/navigation/Ocr.kt +++ b/app/src/main/java/xyz/sevive/arcaeaoffline/ui/navigation/Ocr.kt @@ -11,4 +11,5 @@ enum class OcrSubScreen( ) { Dependencies("$OCR_NAV_ROUTE_ROOT/dependencies", R.string.ocr_dependencies_title), Queue("$OCR_NAV_ROUTE_ROOT/queue", R.string.ocr_queue_title), + Performance("$OCR_NAV_ROUTE_ROOT/performance", R.string.ocr_performance_title), } diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/ui/screens/ocr/OcrEntryScreen.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/ui/screens/ocr/OcrEntryScreen.kt index b42604db..8800bc8c 100644 --- a/app/src/main/java/xyz/sevive/arcaeaoffline/ui/screens/ocr/OcrEntryScreen.kt +++ b/app/src/main/java/xyz/sevive/arcaeaoffline/ui/screens/ocr/OcrEntryScreen.kt @@ -4,6 +4,7 @@ import androidx.compose.runtime.Composable import xyz.sevive.arcaeaoffline.ui.AdaptiveEntryScreen import xyz.sevive.arcaeaoffline.ui.navigation.OcrSubScreen import xyz.sevive.arcaeaoffline.ui.screens.ocr.dependencies.OcrDependenciesScreen +import xyz.sevive.arcaeaoffline.ui.screens.ocr.performance.OcrPerformanceScreen import xyz.sevive.arcaeaoffline.ui.screens.ocr.queue.OcrQueueScreen @Composable @@ -14,6 +15,7 @@ fun OcrEntryScreen() = when (route) { OcrSubScreen.Dependencies.route -> OcrDependenciesScreen() OcrSubScreen.Queue.route -> OcrQueueScreen() + OcrSubScreen.Performance.route -> OcrPerformanceScreen() } }, ) diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/ui/screens/ocr/OcrNavEntry.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/ui/screens/ocr/OcrNavEntry.kt index 6d3bd25c..66f7baee 100644 --- a/app/src/main/java/xyz/sevive/arcaeaoffline/ui/screens/ocr/OcrNavEntry.kt +++ b/app/src/main/java/xyz/sevive/arcaeaoffline/ui/screens/ocr/OcrNavEntry.kt @@ -7,6 +7,7 @@ import androidx.compose.foundation.lazy.LazyColumn import androidx.compose.material.icons.Icons import androidx.compose.material.icons.filled.Api import androidx.compose.material.icons.filled.Queue +import androidx.compose.material.icons.filled.Speed import androidx.compose.material3.ExperimentalMaterial3Api import androidx.compose.material3.HorizontalDivider import androidx.compose.material3.Scaffold @@ -21,7 +22,6 @@ import androidx.lifecycle.compose.collectAsStateWithLifecycle import org.koin.compose.viewmodel.koinViewModel import xyz.sevive.arcaeaoffline.ui.components.ocr.OcrDependencyCrnnModelStatusViewer import xyz.sevive.arcaeaoffline.ui.components.ocr.OcrDependencyImageHashesDatabaseStatusViewer -import xyz.sevive.arcaeaoffline.ui.components.ocr.OcrDependencyKNearestModelStatusViewer import xyz.sevive.arcaeaoffline.ui.navigation.LocalListDetailNavigationContext import xyz.sevive.arcaeaoffline.ui.navigation.MainScreen import xyz.sevive.arcaeaoffline.ui.navigation.OcrSubScreen @@ -33,7 +33,6 @@ import xyz.sevive.arcaeaoffline.ui.screens.ocr.dependencies.OcrDependenciesScree fun OcrNavEntry(modifier: Modifier = Modifier) { val navContext = LocalListDetailNavigationContext.current val vm = koinViewModel() - val kNearestModelUiState by vm.kNearestModelUiState.collectAsStateWithLifecycle() val imageHashesDatabaseUiState by vm.imageHashesDatabaseUiState.collectAsStateWithLifecycle() val crnnModelUiState by vm.crnnModelUiState.collectAsStateWithLifecycle() @@ -52,7 +51,6 @@ fun OcrNavEntry(modifier: Modifier = Modifier) { ) { item { Column { - OcrDependencyKNearestModelStatusViewer(kNearestModelUiState) OcrDependencyImageHashesDatabaseStatusViewer(imageHashesDatabaseUiState) OcrDependencyCrnnModelStatusViewer(crnnModelUiState) } @@ -79,6 +77,15 @@ fun OcrNavEntry(modifier: Modifier = Modifier) { navContext.navigateToDetail(OcrSubScreen.Queue.route) } } + + item { + NavEntryNavigateButton( + titleResId = OcrSubScreen.Performance.title, + icon = Icons.Default.Speed, + ) { + navContext.navigateToDetail(OcrSubScreen.Performance.route) + } + } } } } diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/ui/screens/ocr/dependencies/OcrDependenciesScreen.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/ui/screens/ocr/dependencies/OcrDependenciesScreen.kt index bdae5af2..b2182861 100644 --- a/app/src/main/java/xyz/sevive/arcaeaoffline/ui/screens/ocr/dependencies/OcrDependenciesScreen.kt +++ b/app/src/main/java/xyz/sevive/arcaeaoffline/ui/screens/ocr/dependencies/OcrDependenciesScreen.kt @@ -21,7 +21,6 @@ import xyz.sevive.arcaeaoffline.ui.SubScreenContainer import xyz.sevive.arcaeaoffline.ui.components.ArcaeaAppIcon import xyz.sevive.arcaeaoffline.ui.components.ocr.OcrDependencyCrnnModelStatusViewer import xyz.sevive.arcaeaoffline.ui.components.ocr.OcrDependencyImageHashesDatabaseStatusViewer -import xyz.sevive.arcaeaoffline.ui.components.ocr.OcrDependencyKNearestModelStatusViewer import xyz.sevive.arcaeaoffline.ui.components.preferences.TextPreferencesWidget import xyz.sevive.arcaeaoffline.ui.navigation.OcrSubScreen @@ -32,17 +31,12 @@ fun OcrDependenciesScreen( ) { val context = LocalContext.current - val kNearestModelUiState by viewModel.kNearestModelUiState.collectAsStateWithLifecycle() val imageHashesDatabaseUiState by viewModel.imageHashesDatabaseUiState.collectAsStateWithLifecycle() val crnnModelUiState by viewModel.crnnModelUiState.collectAsStateWithLifecycle() val canBuildHashesDatabase by ArcaeaResourcesStateHolder.canBuildHashesDatabase.collectAsStateWithLifecycle() val buildHashesDatabaseButtonEnabled by viewModel.buildHashesDatabaseButtonEnabled.collectAsStateWithLifecycle() - val kNearestModelFileChooserLauncher = - rememberFileChooserLauncher { uri -> - uri?.let { viewModel.importKNearestModel(it, context) } - } val imageHashesDatabaseFileChooserLauncher = rememberFileChooserLauncher { uri -> uri?.let { viewModel.importImageHashesDatabase(it, context) } @@ -57,20 +51,6 @@ fun OcrDependenciesScreen( }, ) { LazyColumn(modifier) { - item { - OcrDependencyKNearestModelStatusViewer(kNearestModelUiState) - } - - item { - TextPreferencesWidget( - onClick = { kNearestModelFileChooserLauncher.launch("*/*") }, - title = stringResource(R.string.general_import), - leadingIcon = Icons.Default.FileOpen, - ) - } - - item { HorizontalDivider() } - item { OcrDependencyImageHashesDatabaseStatusViewer(imageHashesDatabaseUiState) } diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/ui/screens/ocr/dependencies/OcrDependenciesScreenViewModel.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/ui/screens/ocr/dependencies/OcrDependenciesScreenViewModel.kt index 78e0e95e..1ecb0e4d 100644 --- a/app/src/main/java/xyz/sevive/arcaeaoffline/ui/screens/ocr/dependencies/OcrDependenciesScreenViewModel.kt +++ b/app/src/main/java/xyz/sevive/arcaeaoffline/ui/screens/ocr/dependencies/OcrDependenciesScreenViewModel.kt @@ -21,7 +21,6 @@ import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.launch import kotlinx.io.buffered import kotlinx.io.files.SystemFileSystem -import org.opencv.ml.KNearest import xyz.sevive.arcaeaoffline.core.Progress import xyz.sevive.arcaeaoffline.core.ocr.ImageHashesDatabase import xyz.sevive.arcaeaoffline.data.OcrDependencyPaths @@ -35,7 +34,6 @@ import xyz.sevive.arcaeaoffline.helpers.fromWorkInfo import xyz.sevive.arcaeaoffline.jobs.ImageHashesDatabaseBuilderJob import xyz.sevive.arcaeaoffline.ui.components.ocr.OcrDependencyCrnnModelStatusUiState import xyz.sevive.arcaeaoffline.ui.components.ocr.OcrDependencyImageHashesDatabaseStatusUiState -import xyz.sevive.arcaeaoffline.ui.components.ocr.OcrDependencyKNearestModelStatusUiState import java.io.IOException class OcrDependenciesScreenViewModel( @@ -50,10 +48,6 @@ class OcrDependenciesScreenViewModel( private val logger = Logger.withTag(LOG_TAG) private val workManager = WorkManager.getInstance(context.applicationContext) - private val _kNearestModelUiState = - MutableStateFlow(OcrDependencyKNearestModelStatusUiState()) - val kNearestModelUiState = _kNearestModelUiState.asStateFlow() - private val imageHashesDatabaseBuilderJobInfo = workManager .getWorkInfosForUniqueWorkFlow(ImageHashesDatabaseBuilderJob.NAME) @@ -118,34 +112,6 @@ class OcrDependenciesScreenViewModel( return true } - fun importKNearestModel( - uri: Uri, - context: Context, - ) { - val paths = OcrDependencyPaths() - if (!mkOcrDependencyParentDirs(paths)) return - - viewModelScope.launch(Dispatchers.IO) { - if (isFileTooLarge(uri, context, logName = "KNearest")) return@launch - - val cacheFile = context.copyToCache(uri, "knearest_model_import_temp") ?: return@launch - try { - KNearest.load(cacheFile.toString()) - SystemFileSystem.source(cacheFile).buffered().use { src -> - SystemFileSystem.sink(paths.knnModelFile).buffered().use { dst -> - src.transferTo(dst) - } - } - } catch (e: Exception) { - logger.e(e) { "Error importing KNearest model" } - } finally { - SystemFileSystem.delete(cacheFile) - } - - reloadKNearestModelStatusDetailUiState() - } - } - fun importImageHashesDatabase( uri: Uri, context: Context, @@ -182,13 +148,6 @@ class OcrDependenciesScreenViewModel( } } - private fun reloadKNearestModelStatusDetailUiState() { - viewModelScope.launch(Dispatchers.IO) { - _kNearestModelUiState.value = - OcrDependencyKNearestModelStatusUiState(OcrDependencyStatusBuilder.kNearest()) - } - } - private fun reloadImageHashesDatabaseStatusDetailUiState() { viewModelScope.launch(Dispatchers.IO) { imagesHashesDatabaseStatusDetail.value = @@ -228,7 +187,6 @@ class OcrDependenciesScreenViewModel( } fun reloadAll(context: Context) { - reloadKNearestModelStatusDetailUiState() reloadImageHashesDatabaseStatusDetailUiState() reloadCrnnModelStatusDetailUiState(context) } diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/ui/screens/ocr/performance/OcrPerformanceScreen.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/ui/screens/ocr/performance/OcrPerformanceScreen.kt new file mode 100644 index 00000000..39e8d2ab --- /dev/null +++ b/app/src/main/java/xyz/sevive/arcaeaoffline/ui/screens/ocr/performance/OcrPerformanceScreen.kt @@ -0,0 +1,425 @@ +package xyz.sevive.arcaeaoffline.ui.screens.ocr.performance + +import android.content.ClipData +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.result.PickVisualMediaRequest +import androidx.activity.result.contract.ActivityResultContracts +import androidx.compose.animation.AnimatedContent +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.items +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.automirrored.filled.Sort +import androidx.compose.material.icons.filled.ContentCopy +import androidx.compose.material.icons.filled.Delete +import androidx.compose.material.icons.filled.Image +import androidx.compose.material.icons.filled.PlayArrow +import androidx.compose.material.icons.filled.Stop +import androidx.compose.material3.Card +import androidx.compose.material3.HorizontalDivider +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButton +import androidx.compose.material3.IconButtonDefaults +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.OutlinedButton +import androidx.compose.material3.SnackbarHost +import androidx.compose.material3.SnackbarHostState +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.platform.LocalClipboard +import androidx.compose.ui.platform.LocalResources +import androidx.compose.ui.platform.toClipEntry +import androidx.compose.ui.res.dimensionResource +import androidx.compose.ui.res.pluralStringResource +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.TextStyle +import androidx.compose.ui.text.style.TextAlign +import androidx.compose.ui.unit.dp +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import kotlinx.coroutines.launch +import org.koin.compose.viewmodel.koinViewModel +import xyz.sevive.arcaeaoffline.R +import xyz.sevive.arcaeaoffline.core.Progress +import xyz.sevive.arcaeaoffline.core.ocr.device.OcrPerformanceBenchmark +import xyz.sevive.arcaeaoffline.helpers.formatAsLocalizedDate +import xyz.sevive.arcaeaoffline.helpers.formatAsLocalizedTime +import xyz.sevive.arcaeaoffline.helpers.secondaryItemAlpha +import xyz.sevive.arcaeaoffline.ui.SubScreenContainer +import xyz.sevive.arcaeaoffline.ui.components.IconRow +import xyz.sevive.arcaeaoffline.ui.components.LinearProgressIndicatorWrapper +import xyz.sevive.arcaeaoffline.ui.components.ListGroupHeader +import xyz.sevive.arcaeaoffline.ui.components.preferences.BasePreferencesWidget +import xyz.sevive.arcaeaoffline.ui.components.preferences.SliderPreferencesWidget +import xyz.sevive.arcaeaoffline.ui.components.preferences.TextPreferencesWidget +import xyz.sevive.arcaeaoffline.ui.navigation.OcrSubScreen +import java.util.Locale + +@Composable +fun OcrPerformanceScreen( + modifier: Modifier = Modifier, + viewModel: OcrPerformanceScreenViewModel = koinViewModel(), +) { + val uiState by viewModel.uiState.collectAsStateWithLifecycle() + val snackbarHostState = remember { SnackbarHostState() } + + val imagePickerLauncher = + rememberLauncherForActivityResult( + ActivityResultContracts.PickMultipleVisualMedia(maxItems = 10), + ) { uris -> + if (uris.isNotEmpty()) { + viewModel.onImagesPicked(uris) + } + } + + SubScreenContainer( + title = stringResource(OcrSubScreen.Performance.title), + snackbarHost = { SnackbarHost(snackbarHostState) }, + ) { + LazyColumn(modifier) { + item { + ListGroupHeader(stringResource(R.string.ocr_performance_images_title)) + } + + item { + val selectedUris = uiState.selectedImageUris + val hasSelection = selectedUris.isNotEmpty() + TextPreferencesWidget( + title = stringResource(R.string.ocr_performance_pick_images_button), + content = + selectedUris.takeIf { hasSelection }?.let { + pluralStringResource( + R.plurals.ocr_performance_picked_images, + it.size, + it.size, + ) + }, + leadingSlot = { + Icon( + Icons.Default.Image, + contentDescription = null, + tint = MaterialTheme.colorScheme.primary, + ) + }, + enabled = !uiState.running, + onClick = { + imagePickerLauncher.launch( + PickVisualMediaRequest(ActivityResultContracts.PickVisualMedia.ImageOnly), + ) + }, + trailingSlot = + if (hasSelection) { + { + IconButton( + onClick = viewModel::clearImages, + enabled = !uiState.running, + colors = + IconButtonDefaults.iconButtonColors( + contentColor = MaterialTheme.colorScheme.error, + ), + ) { + Icon( + Icons.Default.Delete, + contentDescription = stringResource(R.string.ocr_performance_clear_selection), + ) + } + } + } else { + null + }, + ) + } + + if (uiState.imageLoadError) { + item { + Text( + stringResource(R.string.ocr_performance_image_load_failed), + color = MaterialTheme.colorScheme.error, + modifier = Modifier.padding(horizontal = 16.dp), + ) + } + } + + item { HorizontalDivider() } + + item { + ListGroupHeader(stringResource(R.string.ocr_performance_concurrency_title)) + } + + item { + AnimatedContent( + targetState = uiState.running, + label = "benchmarkRunningState", + ) { running -> + if (!running) { + SliderPreferencesWidget( + value = uiState.parallelCount.toFloat(), + onValueChange = viewModel::onParallelCountChange, + icon = Icons.AutoMirrored.Default.Sort, + title = stringResource(R.string.ocr_queue_queue_options_parallel_count), + description = uiState.parallelCount.toString(), + valueRange = OcrPerformanceScreenViewModel.parallelCountSliderRange, + steps = OcrPerformanceScreenViewModel.parallelCountSliderSteps, + trailingSlot = { + OutlinedButton( + onClick = { viewModel.runBenchmark() }, + enabled = uiState.selectedImageUris.isNotEmpty(), + ) { + IconRow { + Icon(Icons.Default.PlayArrow, contentDescription = null) + Text(stringResource(R.string.ocr_performance_run_button)) + } + } + }, + ) + } else { + BasePreferencesWidget( + title = { + uiState.runningParallel?.let { parallel -> + Text( + stringResource(R.string.ocr_performance_single_progress, parallel), + ) + } + }, + content = { + LinearProgressIndicatorWrapper( + progress = Progress(uiState.progress, uiState.progressTotal), + ) + }, + trailingSlot = { + OutlinedButton(onClick = { viewModel.cancelBenchmark() }) { + IconRow { + Icon(Icons.Default.Stop, contentDescription = null) + Text(stringResource(R.string.ocr_performance_cancel_button)) + } + } + }, + ) + } + } + } + + uiState.errorMessage?.let { message -> + item { + Text( + stringResource(R.string.ocr_performance_benchmark_error, message), + color = MaterialTheme.colorScheme.error, + modifier = Modifier.padding(horizontal = 16.dp), + ) + } + } + + uiState.result?.let { result -> + item { HorizontalDivider() } + + item { + ListGroupHeader(stringResource(R.string.ocr_performance_result_title)) + } + + item { + ResultCard( + parallel = uiState.resultParallel ?: uiState.parallelCount, + result = result, + snackbarHostState = snackbarHostState, + modifier = Modifier.padding(bottom = dimensionResource(R.dimen.list_padding)), + ) + } + } + + if (uiState.history.isNotEmpty()) { + item { HorizontalDivider() } + + item { + ListGroupHeader(stringResource(R.string.ocr_performance_history_title)) + } + + // Newest first, easier to compare recent runs + items(uiState.history.asReversed(), key = { it.uuid }) { entry -> + HistoryRow( + entry = entry, + modifier = Modifier.animateItem(), + ) + } + } + } + } +} + +@Composable +private fun ResultCard( + parallel: Int, + result: OcrPerformanceBenchmark.Result, + snackbarHostState: SnackbarHostState, + modifier: Modifier = Modifier, +) { + val clipboard = LocalClipboard.current + val coroutineScope = rememberCoroutineScope() + val resources = LocalResources.current + + Card(modifier = modifier.padding(horizontal = 16.dp)) { + Column( + verticalArrangement = Arrangement.spacedBy(dimensionResource(R.dimen.list_padding)), + modifier = Modifier.padding(dimensionResource(R.dimen.card_padding)), + ) { + Row( + horizontalArrangement = Arrangement.SpaceBetween, + verticalAlignment = Alignment.CenterVertically, + modifier = Modifier.fillMaxWidth(), + ) { + Text( + stringResource(R.string.ocr_performance_result_parallel, parallel), + style = MaterialTheme.typography.titleMedium, + ) + IconButton( + onClick = { + coroutineScope.launch { + val text = buildReportText(parallel, result) + val clipData = ClipData.newPlainText("OCR Performance", text) + clipboard.setClipEntry(clipData.toClipEntry()) + snackbarHostState.showSnackbar( + resources.getString(R.string.ocr_performance_report_copied), + ) + } + }, + ) { + Icon( + Icons.Default.ContentCopy, + contentDescription = stringResource(R.string.ocr_performance_copy_report), + ) + } + } + Spacer(Modifier.height(dimensionResource(R.dimen.list_padding))) + KeyValueRow( + label = stringResource(R.string.ocr_performance_result_median_label), + value = stringResource(R.string.ocr_performance_result_median_value, result.medianPerImageMs), + valueStyle = MaterialTheme.typography.titleMedium, + ) + KeyValueRow( + label = stringResource(R.string.ocr_performance_result_throughput_label), + value = stringResource(R.string.ocr_performance_result_throughput_value, result.throughputPerSecond), + valueStyle = MaterialTheme.typography.titleMedium, + ) + KeyValueRow( + label = stringResource(R.string.ocr_performance_result_batches_label), + value = stringResource(R.string.ocr_performance_result_batches_value, result.batchTimesMs.joinToString("/")), + valueStyle = MaterialTheme.typography.bodyMedium, + ) + KeyValueRow( + label = stringResource(R.string.ocr_performance_result_output_label), + value = + stringResource( + if (result.resultsConsistent) { + R.string.ocr_performance_result_output_consistent + } else { + R.string.ocr_performance_result_output_inconsistent + }, + ), + valueStyle = MaterialTheme.typography.bodyMedium, + valueColor = if (result.resultsConsistent) Color.Unspecified else MaterialTheme.colorScheme.error, + ) + } + } +} + +@Composable +private fun KeyValueRow( + label: String, + value: String, + valueStyle: TextStyle, + valueColor: Color = Color.Unspecified, +) { + Row( + horizontalArrangement = Arrangement.SpaceBetween, + verticalAlignment = Alignment.CenterVertically, + modifier = Modifier.fillMaxWidth(), + ) { + Text( + label, + style = MaterialTheme.typography.bodyMedium, + modifier = Modifier.secondaryItemAlpha(), + ) + Text(value, style = valueStyle, color = valueColor) + } +} + +/** + * Plain-text report for pasting into issue reports; deliberately not localized, + * and formatted with Locale.ROOT so numbers stay consistent across devices. + */ +private fun buildReportText( + parallel: Int, + result: OcrPerformanceBenchmark.Result, +): String = + buildString { + appendLine("OCR Performance (p$parallel)") + appendLine("median: %.0f ms/image".format(Locale.ROOT, result.medianPerImageMs)) + appendLine("throughput: %.1f it/s".format(Locale.ROOT, result.throughputPerSecond)) + appendLine("batches: ${result.batchTimesMs.joinToString("/")} ms") + append("consistent: ${result.resultsConsistent}") + } + +@Composable +private fun HistoryRow( + entry: OcrPerformanceScreenViewModel.HistoryEntry, + modifier: Modifier = Modifier, +) { + val timestampText = + remember(entry.timestamp) { + entry.timestamp.formatAsLocalizedDate() + "\n" + entry.timestamp.formatAsLocalizedTime() + } + + Row( + horizontalArrangement = Arrangement.spacedBy(dimensionResource(R.dimen.list_padding)), + verticalAlignment = Alignment.Top, + modifier = + modifier + .fillMaxWidth() + .padding(horizontal = 16.dp, vertical = dimensionResource(R.dimen.list_padding)), + ) { + Text( + timestampText, + textAlign = TextAlign.End, + style = MaterialTheme.typography.bodySmall, + ) + + Column( + verticalArrangement = Arrangement.spacedBy(2.dp), + modifier = Modifier.weight(1f), + ) { + Text( + "p%d, %.1f it/s".format( + Locale.ROOT, + entry.parallel, + entry.result.throughputPerSecond, + ), + style = MaterialTheme.typography.bodyMedium, + ) + Text( + stringResource( + R.string.ocr_performance_result_batches_value, + entry.result.batchTimesMs.joinToString("/"), + ), + style = MaterialTheme.typography.bodySmall, + modifier = Modifier.secondaryItemAlpha(), + ) + if (!entry.result.resultsConsistent) { + Text( + stringResource(R.string.ocr_performance_result_output_inconsistent), + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.error, + ) + } + } + } +} diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/ui/screens/ocr/performance/OcrPerformanceScreenViewModel.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/ui/screens/ocr/performance/OcrPerformanceScreenViewModel.kt new file mode 100644 index 00000000..09f66be0 --- /dev/null +++ b/app/src/main/java/xyz/sevive/arcaeaoffline/ui/screens/ocr/performance/OcrPerformanceScreenViewModel.kt @@ -0,0 +1,196 @@ +package xyz.sevive.arcaeaoffline.ui.screens.ocr.performance + +import android.content.Context +import android.net.Uri +import androidx.lifecycle.ViewModel +import androidx.lifecycle.viewModelScope +import co.touchlab.kermit.Logger +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.collectLatest +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import org.opencv.core.Mat +import org.opencv.core.MatOfByte +import org.opencv.imgcodecs.Imgcodecs +import xyz.sevive.arcaeaoffline.core.ocr.device.OcrPerformanceBenchmark +import xyz.sevive.arcaeaoffline.core.ocr.opencv.use +import xyz.sevive.arcaeaoffline.datastore.OcrQueueParallelism +import xyz.sevive.arcaeaoffline.datastore.OcrQueuePreferencesRepository +import java.io.IOException +import kotlin.math.round +import kotlin.time.Clock +import kotlin.uuid.Uuid + +class OcrPerformanceScreenViewModel( + context: Context, + private val preferencesRepository: OcrQueuePreferencesRepository, +) : ViewModel() { + companion object { + private const val LOG_TAG = "OcrPerfScreenVM" + + private val parallelCountIntRange = OcrQueueParallelism.countRange() + val parallelCountSliderRange = parallelCountIntRange.first.toFloat()..parallelCountIntRange.last.toFloat() + val parallelCountSliderSteps = parallelCountIntRange.count() - 1 + } + + data class HistoryEntry( + val uuid: Uuid = Uuid.generateV4(), + val timestamp: kotlin.time.Instant, + val parallel: Int, + val result: OcrPerformanceBenchmark.Result, + ) + + data class UiState( + val selectedImageUris: List = emptyList(), + val imageLoadError: Boolean = false, + val parallelCount: Int = OcrQueueParallelism.defaultCount(), + val parallelCountInitialized: Boolean = false, + val running: Boolean = false, + val runningParallel: Int? = null, + val progress: Int = 0, + val progressTotal: Int = 0, + val result: OcrPerformanceBenchmark.Result? = null, + val resultParallel: Int? = null, + val history: List = emptyList(), + val errorMessage: String? = null, + ) + + private val applicationContext = context.applicationContext + private val logger = Logger.withTag(LOG_TAG) + private val _uiState = MutableStateFlow(UiState()) + val uiState = _uiState.asStateFlow() + + private var benchmarkJob: Job? = null + + init { + viewModelScope.launch { + preferencesRepository.preferencesFlow.collectLatest { preferences -> + _uiState.update { + // Initial slider value follows the production config + if (it.parallelCountInitialized) { + it + } else { + it.copy(parallelCount = preferences.parallelCount, parallelCountInitialized = true) + } + } + } + } + } + + fun onImagesPicked(uris: List) { + _uiState.update { it.copy(selectedImageUris = uris, imageLoadError = false, errorMessage = null) } + } + + fun clearImages() { + _uiState.update { it.copy(selectedImageUris = emptyList(), imageLoadError = false) } + } + + fun onParallelCountChange(value: Float) { + _uiState.update { it.copy(parallelCount = round(value).toInt().coerceIn(parallelCountIntRange)) } + } + + fun runBenchmark() { + val state = _uiState.value + if (state.running) return + + benchmarkJob = + viewModelScope.launch { + val parallel = state.parallelCount + _uiState.update { + it.copy( + running = true, + runningParallel = parallel, + progress = 0, + progressTotal = 0, + result = null, + resultParallel = null, + errorMessage = null, + ) + } + try { + // no suspension point between decode and run, so ownership + // of the ROI Mats transfers to runBenchmark without a + // cancellation window + val result = + withContext(Dispatchers.Default) { + OcrPerformanceBenchmark.runBenchmark( + context = applicationContext, + roiSets = decodeAndExtractRois(state.selectedImageUris), + parallel = parallel, + ) { completed, total -> + _uiState.update { it.copy(progress = completed, progressTotal = total) } + } + } + _uiState.update { + it.copy( + result = result, + resultParallel = parallel, + history = + it.history + + HistoryEntry( + timestamp = Clock.System.now(), + parallel = parallel, + result = result, + ), + ) + } + } catch (e: CancellationException) { + throw e + } catch (e: ImageLoadException) { + logger.e(e) { "Failed to decode benchmark images" } + _uiState.update { + it.copy( + selectedImageUris = emptyList(), + imageLoadError = true, + ) + } + } catch (e: Exception) { + logger.e(e) { "Benchmark failed" } + _uiState.update { it.copy(errorMessage = e.message) } + } finally { + _uiState.update { it.copy(running = false, runningParallel = null) } + } + } + } + + fun cancelBenchmark() { + benchmarkJob?.cancel() + } + + private class ImageLoadException : IOException() + + /** + * Decodes each image and extracts its OCR ROIs; only the ROI clones + * survive this pass. Already-extracted ROIs are released on failure. + */ + private suspend fun decodeAndExtractRois(uris: List): List> = + withContext(Dispatchers.IO) { + val roiSets = mutableListOf>() + try { + uris.forEach { uri -> + // decoding below is all blocking calls, so cancellation + // must be polled manually between images + coroutineContext.ensureActive() + val bytes = + applicationContext.contentResolver.openInputStream(uri)?.use { it.readBytes() } + ?: throw ImageLoadException() + MatOfByte(*bytes).use { matOfBytes -> + Imgcodecs.imdecode(matOfBytes, Imgcodecs.IMREAD_COLOR).use { img -> + if (img.empty()) throw ImageLoadException() + roiSets.add(OcrPerformanceBenchmark.extractRois(img)) + } + } + } + roiSets + } catch (e: Exception) { + roiSets.flatten().forEach { it.release() } + throw e + } + } +} diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/ui/screens/ocr/queue/preferences/OcrQueuePreferencesViewModel.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/ui/screens/ocr/queue/preferences/OcrQueuePreferencesViewModel.kt index 36e3a8db..927b5f0d 100644 --- a/app/src/main/java/xyz/sevive/arcaeaoffline/ui/screens/ocr/queue/preferences/OcrQueuePreferencesViewModel.kt +++ b/app/src/main/java/xyz/sevive/arcaeaoffline/ui/screens/ocr/queue/preferences/OcrQueuePreferencesViewModel.kt @@ -6,6 +6,7 @@ import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.launch +import xyz.sevive.arcaeaoffline.datastore.OcrQueueParallelism import xyz.sevive.arcaeaoffline.datastore.OcrQueuePreferencesRepository import kotlin.time.Duration.Companion.seconds @@ -16,7 +17,7 @@ class OcrQueuePreferencesViewModel( val checkIsImage: Boolean = false, val checkIsArcaeaImage: Boolean = false, val parallelCount: Int = -1, - val parallelCountIntRange: IntRange = 1..Runtime.getRuntime().availableProcessors() * 2, + val parallelCountIntRange: IntRange = OcrQueueParallelism.countRange(), ) { val parallelCountSliderRange = parallelCountIntRange.first.toFloat()..parallelCountIntRange.last.toFloat() val parallelCountSliderSteps = parallelCountIntRange.count() - 1 diff --git a/app/src/main/res/drawable/ic_knearest_model.xml b/app/src/main/res/drawable/ic_knearest_model.xml deleted file mode 100644 index f489e38e..00000000 --- a/app/src/main/res/drawable/ic_knearest_model.xml +++ /dev/null @@ -1,77 +0,0 @@ - - - - - - - - - - - - - diff --git a/app/src/main/res/values-zh-rCN/strings.xml b/app/src/main/res/values-zh-rCN/strings.xml index 90eef879..5befb0b0 100644 --- a/app/src/main/res/values-zh-rCN/strings.xml +++ b/app/src/main/res/values-zh-rCN/strings.xml @@ -68,7 +68,6 @@ 保存图片 - KNearest 模型 图像哈希数据库 CRNN OCR 模型 @@ -259,6 +258,33 @@ OCR 依赖项 + OCR 性能 + 图片 + 参数 + 结果 + 复制报告 + 报告已复制 + 并发数 %d + 单任务中位 + %.0f ms + 吞吐 + %.1f it/s + 批耗时 + %s ms + 输出 + 一致 + 不一致 + 运行 + 正在测试并发数 %d + 清空选择 + 选择截图 + + 已选择 %d 张图片 + + 无法读取所选图片，请重新选择 + 取消 + 历史记录（本次会话） + 基准测试失败: %s 发送崩溃报告 diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 4922cf06..4e2c574a 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -80,7 +80,6 @@ CC BY-SA 3.0 Save image - KNearest model Image hashes database CRNN OCR model @@ -291,6 +290,34 @@ CC BY-SA 3.0 OCR Dependencies + OCR Performance + Images + Parameters + Result + Copy report + Report copied + Parallel Count %d + Median per image + %.0f ms + Throughput + %.1f it/s + Batch times + %s ms + Output + Consistent + Inconsistent + Run + Testing parallel count %d + Clear selection + Select screenshot(s) + + %d image selected + %d images selected + + Failed to read the selected image(s), please select again + Cancel + History (this session) + Benchmark failed: %s Send crash reports diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/ImageHashers.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/ImageHashers.kt index c8756298..01742f50 100644 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/ImageHashers.kt +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/ImageHashers.kt @@ -6,6 +6,7 @@ import org.opencv.core.Mat import org.opencv.core.Scalar import org.opencv.core.Size import org.opencv.imgproc.Imgproc +import xyz.sevive.arcaeaoffline.core.ocr.opencv.use @Suppress("FunctionName") private fun Mat._mean(): Scalar = Core.mean(this) @@ -55,14 +56,12 @@ object ImageHashers { private fun average( imgGray: Mat, hashSize: Double, - ): Mat { - val imgSize = Size(hashSize, hashSize) - val imgResized = resizeImage(imgGray, imgSize) - - val hashMat = Mat() - Core.compare(imgResized, imgResized._mean(), hashMat, Core.CMP_GT) - return hashMat - } + ): Mat = + resizeImage(imgGray, Size(hashSize, hashSize)).use { imgResized -> + val hashMat = Mat() + Core.compare(imgResized, imgResized._mean(), hashMat, Core.CMP_GT) + hashMat + } /** * Computes a simple hash comparing the intensity of each pixel in @@ -80,17 +79,20 @@ object ImageHashers { private fun difference( imgGray: Mat, hashSize: Double, - ): Mat { - val imgSize = Size(hashSize + 1.0, hashSize) - val imgResized = resizeImage(imgGray, imgSize) - - val previous = imgResized.submat(0, imgResized.rows(), 0, imgResized.cols() - 1) - val current = imgResized.submat(0, imgResized.rows(), 1, imgResized.cols()) - - val hashMat = Mat() - Core.compare(previous, current, hashMat, Core.CMP_GT) - return hashMat - } + ): Mat = + resizeImage(imgGray, Size(hashSize + 1.0, hashSize)).use { imgResized -> + val hashMat = Mat() + imgResized + .submat(0, imgResized.rows(), 0, imgResized.cols() - 1) + .use { previous -> + imgResized + .submat(0, imgResized.rows(), 1, imgResized.cols()) + .use { current -> + Core.compare(previous, current, hashMat, Core.CMP_GT) + } + } + hashMat + } /** * A hash based on the differences between adjacent pixels. @@ -112,13 +114,15 @@ object ImageHashers { val imgSizeBase = hashSize * highFreqFactor val imgSize = Size(imgSizeBase, imgSizeBase) - val imgResized = resizeImage(imgGray, imgSize) - imgResized.convertTo(imgResized, CvType.CV_32FC1) - val dctMat = Mat() - Core.dct(imgResized, dctMat) - val hashMat = dctMat.submat(0, hashSize.toInt(), 0, hashSize.toInt()).clone() - Core.compare(hashMat, hashMat._median(), hashMat, Core.CMP_GT) - return hashMat + return resizeImage(imgGray, imgSize).use { imgResized -> + imgResized.convertTo(imgResized, CvType.CV_32FC1) + Mat().use { dctMat -> + Core.dct(imgResized, dctMat) + val hashMat = dctMat.submat(0, hashSize.toInt(), 0, hashSize.toInt()).use { it.clone() } + Core.compare(hashMat, hashMat._median(), hashMat, Core.CMP_GT) + hashMat + } + } } /** @@ -138,6 +142,7 @@ object ImageHashers { /** * Return the hamming distance between [hash1] and [hash2]. */ + @Suppress("unused") fun compare( hash1: Mat, hash2: Mat, diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/ImageHashesDatabase.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/ImageHashesDatabase.kt index a57a58bd..710eee4b 100644 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/ImageHashesDatabase.kt +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/ImageHashesDatabase.kt @@ -11,7 +11,13 @@ private fun hammingDistance( byteArray2: ByteArray, ): Int { assert(byteArray1.size == byteArray2.size) { "hash size does not match!" } - return byteArray1.zip(byteArray2).count { (b1, b2) -> b1 != b2 } + // Must remain allocation-free: this method is called for EVERY entry in allHashes on EVERY lookup. + // Creating objects here will cause severe GC pressure. + var distance = 0 + for (i in byteArray1.indices) { + if (byteArray1[i] != byteArray2[i]) distance++ + } + return distance } class ImageHashesDatabase( @@ -31,6 +37,8 @@ class ImageHashesDatabase( private set var partnerIconHashesCount: Int by Delegates.notNull() private set + + @Suppress("unused") val hashesCount: Int get() = jacketHashesCount + partnerIconHashesCount private class HashEntry( @@ -178,9 +186,15 @@ class ImageHashesDatabase( val dHash = ImageHashers.difference(image, this.hashSize) val pHash = ImageHashers.dct(image, this.hashSize, this.highFreqFactor) - items.addAll(lookupAHash(type, aHash.toHashByteArray())) - items.addAll(lookupDHash(type, dHash.toHashByteArray())) - items.addAll(lookupPHash(type, pHash.toHashByteArray())) + try { + items.addAll(lookupAHash(type, aHash.toHashByteArray())) + items.addAll(lookupDHash(type, dHash.toHashByteArray())) + items.addAll(lookupPHash(type, pHash.toHashByteArray())) + } finally { + aHash.release() + dHash.release() + pHash.release() + } return items } diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/OcrDigits.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/OcrDigits.kt deleted file mode 100644 index e48c95c8..00000000 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/OcrDigits.kt +++ /dev/null @@ -1,108 +0,0 @@ -package xyz.sevive.arcaeaoffline.core.ocr - -import org.opencv.core.Core -import org.opencv.core.Mat -import org.opencv.core.MatOfFloat -import org.opencv.core.MatOfPoint -import org.opencv.core.Size -import org.opencv.imgproc.Imgproc -import org.opencv.ml.KNearest -import org.opencv.objdetect.HOGDescriptor -import kotlin.math.ceil -import kotlin.math.max -import kotlin.math.min - -fun resizeFillSquare( - img: Mat, - target: Int = 20, -): Mat { - val h = img.size().height - val w = img.size().width - - val newSize: Size = - if (h > w) { - Size(w * (target / h), target.toDouble()) - } else { - Size(target.toDouble(), h * (target / w)) - } - val resized = Mat() - Imgproc.resize(img, resized, newSize) - - val borderSize = - ceil( - (max(newSize.width, newSize.height) - min(newSize.width, newSize.height)) / 2, - ).toInt() - val bordered = Mat() - if (newSize.width < newSize.height) { - Core.copyMakeBorder(resized, bordered, 0, 0, borderSize, borderSize, Core.BORDER_CONSTANT) - } else { - Core.copyMakeBorder(resized, bordered, borderSize, borderSize, 0, 0, Core.BORDER_CONSTANT) - } - val final = Mat() - Imgproc.resize(bordered, final, Size(target.toDouble(), target.toDouble())) - return final -} - -fun preprocessHog(digitRois: List): Mat { - // https://learnopencv.com/handwritten-digits-classification-an-opencv-c-python-tutorial/ - val samples = mutableListOf() - for (digit in digitRois) { - val hog = - HOGDescriptor( - Size(20.0, 20.0), - Size(10.0, 10.0), - Size(5.0, 5.0), - Size(10.0, 10.0), - 9, - ) - val hist = MatOfFloat() - hog.compute(digit, hist) - samples.add(hist) - } - val mat = Mat() - Core.hconcat(samples.reversed(), mat) - Core.rotate(mat, mat, Core.ROTATE_90_COUNTERCLOCKWISE) - return mat -} - -fun ocrDigitSamplesKnn( - samples: Mat, - knnModel: KNearest, - k: Int = 4, -): Int { - val results = Mat() - knnModel.findNearest(samples, k, results) - var resultStr = "" - for (row in 0 until results.rows()) { - val data = results.get(row, 0) - if (data == null || data[0] < 0.0) continue - resultStr += data[0].toInt().toString() - } - return resultStr.toInt(10) -} - -fun ocrDigitsByContourGetSamples( - roiGray: Mat, - size: Int, -): Mat { - val roi = roiGray.clone() - val contours = ArrayList() - val hierarchy = Mat() - Imgproc.findContours(roi, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_NONE) - var rects = contours.map { Imgproc.boundingRect(it) } - rects = FixRects.connectBroken(rects, roi.width().toDouble(), roi.height().toDouble()) - rects = FixRects.splitConnected(roi, rects) - val sortedRects = rects.sortedBy { it.x } - val digitRois = sortedRects.map { resizeFillSquare(roi.submat(it), size) } - return preprocessHog(digitRois) -} - -fun ocrDigitsByContourKnn( - roiGray: Mat, - knnModel: KNearest, - k: Int = 4, - size: Int = 20, -): Int { - val samples = ocrDigitsByContourGetSamples(roiGray, size) - return ocrDigitSamplesKnn(samples, knnModel, k) -} diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/Utils.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/Utils.kt deleted file mode 100644 index feaedc19..00000000 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/Utils.kt +++ /dev/null @@ -1,217 +0,0 @@ -package xyz.sevive.arcaeaoffline.core.ocr - -import org.opencv.core.Core -import org.opencv.core.CvType -import org.opencv.core.Mat -import org.opencv.core.Rect -import org.opencv.core.Scalar -import kotlin.math.abs -import kotlin.math.floor -import kotlin.math.round -import kotlin.math.roundToInt -import kotlin.math.sqrt - -fun matMedian(mat: Mat): Double { - // Convert the matrix to a single row vector - val arr = mat.clone().reshape(1, mat.rows() * mat.cols()) - - val arrSorted = Mat() - Core.sort(arr, arrSorted, Core.SORT_EVERY_COLUMN + Core.SORT_ASCENDING) - - val rows = arr.rows() - return if (rows % 2 == 0) { - val midIndex1 = rows / 2 - val midIndex2 = midIndex1 - 1 - - val midValue1 = arrSorted.get(midIndex1, 0)[0] - val midValue2 = arrSorted.get(midIndex2, 0)[0] - (midValue1 + midValue2) / 2.0 - } else { - val middleIndex = rows / 2 - arrSorted.get(middleIndex, 0)[0] - } -} - -/** - * Port of numpy.bincount. - * - * [Original documentation](https://numpy.org/doc/stable/reference/generated/numpy.bincount.html#numpy-bincount) - * - * @param x array_like, 1 dimension, nonnegative ints. - * @param weights array_like, same shape as x. - * @param minLength int, minimum number of bins for the output array. - */ -@Suppress("unused") -fun binCount( - x: Mat, - weights: Mat? = null, - minLength: Int = 0, -): Mat { - assert(x.cols() == 1) { "binCount: `x` should be a one dimension array" } - if (weights != null) { - assert(weights.size() == x.size()) { "binCount: `weights` should have the same size as `x`" } - } - val checkMat = Mat() - Core.compare(x, Scalar(0.0), checkMat, Core.CMP_GE) - assert(Core.countNonZero(checkMat) == x.rows()) { "binCount: `x` should not have negative values" } - - // determine bin size - val binSizeFromMat = Core.minMaxLoc(x).maxVal.toInt() + 1 - val binSize = if (minLength > binSizeFromMat) minLength else binSizeFromMat - - // extract values - val matValues = mutableListOf() - for (i in 0 until x.rows()) { - matValues.add(x.get(i, 0)[0].toInt()) - } - - val binArr = (0 until binSize).map { index -> matValues.count { value -> index == value } } - var binResultArr = binArr.map { it.toDouble() } - - if (weights != null) { - val weightValues = mutableListOf() - for (i in 0 until weights.rows()) { - weightValues.add(weights.get(i, 0)[0]) - } - val binWeights = (0 until binSize).map { 0.0 }.toMutableList() - for ((index, weight) in matValues.zip(weightValues)) { - binWeights[index] = binWeights[index] + weight - } - binResultArr = binResultArr.zip(binWeights).map { (value, weight) -> value * weight } - } - - val resultMat = Mat.zeros(binSize, 1, CvType.CV_32F) - for (i in 0 until binSize) { - resultMat.put(i, 0, binResultArr[i]) - } - return resultMat -} - -fun collectionStandardDeviation(list: Collection): Double { - // https://www.programmingcube.com/write-a-kotlin-program-to-calculate-standard-deviation - val mean = list.average() - - val squaredDifferences = list.map { (it - mean) * (it - mean) } - val meanOfSquaredDifferences = squaredDifferences.average() - - return sqrt(meanOfSquaredDifferences) -} - -fun collectionMedian(list: List) = - list.sorted().let { - if (it.size % 2 == 0) { - (it[it.size / 2] + it[(it.size - 1) / 2]) / 2 - } else { - it[it.size / 2] - } - } - -class FixRects { - companion object { - fun connectBroken( - rects: List, - imgWidth: Double, - imgHeight: Double, - overrideTolerance: Int? = null, - ): List { - val tolerance: Int = overrideTolerance ?: floor(imgWidth * 0.08).toInt() - - val newRects = mutableListOf() - val consumedRects = mutableListOf() - - for (rect in rects) { - if (consumedRects.indexOf(rect) > -1) continue - - // filter out large rects - if (!(imgHeight * 0.1 <= rect.height && rect.height <= imgHeight * 0.6)) continue - - val group = mutableListOf() - // see if there's other rects that have near left & right borders - for (otherRect in rects) { - if (rect == otherRect) continue - - if (abs(rect.x - otherRect.x) < tolerance && abs((rect.x + rect.width) - (otherRect.x + otherRect.width)) < tolerance) { - group.add(otherRect) - } - } - - if (group.size > 0) { - group.add(rect) - consumedRects.addAll(group) - // calculate new rect - val newX = group.minBy { it.x }.x - val newY = group.minBy { it.y }.y - val newRightRect = group.maxBy { it.x + it.width } - val newRight = newRightRect.x + newRightRect.width - val newBottomRect = group.maxBy { it.y + it.height } - val newBottom = newBottomRect.y + newBottomRect.height - val newW = newRight - newX - val newH = newBottom - newY - newRects.add(Rect(newX, newY, newW, newH)) - } - } - - val returnRects = rects.filter { consumedRects.indexOf(it) == -1 }.toMutableList() - returnRects.addAll(newRects) - return returnRects.toList() - } - - fun splitConnected( - imgMasked: Mat, - rects: List, - rectWHRatio: Double = 1.05, - widthRangeRatio: Double = 0.1, - ): List { - val connectedRects = mutableListOf() - val newRects = mutableListOf() - - for (rect in rects) { - if ((rect.width.toDouble() / rect.height.toDouble()) <= rectWHRatio) continue - - connectedRects.add(rect) - - // find the thinnest part - val borderIgnore = round(rect.width * widthRangeRatio).toInt() - - val imgCropped = - imgMasked - .submat( - Rect( - borderIgnore, - rect.y, - rect.width - borderIgnore, - rect.height, - ), - ).clone() - val whitePixels = mutableMapOf() - for (i in 0 until imgCropped.rows()) { - val col = imgCropped.submat(i, i + 1, 0, imgCropped.cols()).clone() - whitePixels[rect.x + borderIgnore + i] = Core.countNonZero(col) - } - - if (whitePixels.values.all { it == 0 }) return rects - - val leastWhitePixels = whitePixels.values.minBy { it } - val xValuesMap = whitePixels.filter { it.value == leastWhitePixels } - val xValues = xValuesMap.keys.map { it.toDouble() } - - // select only middle values - val xMean = xValues.average() - val xStd = collectionStandardDeviation(xValues) - val xValuesInRange = - xValues.filter { - xMean - xStd * 1.5 <= it && it <= xMean + xStd * 1.5 - } - val xMid = collectionMedian(xValuesInRange).roundToInt() - - // split rect - newRects.add(Rect(rect.x, rect.y, xMid - rect.x, rect.height)) - newRects.add(Rect(xMid, rect.y, rect.x + rect.width - xMid, rect.height)) - } - - val returnRects = rects.filter { connectedRects.indexOf(it) == -1 }.toMutableList() - returnRects.addAll(newRects) - return returnRects.toList() - } - } -} diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/CropBlackEdges.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/CropBlackEdges.kt index 63bac669..cf04df84 100644 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/CropBlackEdges.kt +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/CropBlackEdges.kt @@ -7,6 +7,7 @@ import org.opencv.core.Rect import org.opencv.core.Scalar import org.opencv.imgproc.Imgproc import xyz.sevive.arcaeaoffline.core.ocr.device.CropBlackEdges.Companion.cropOrOriginal +import xyz.sevive.arcaeaoffline.core.ocr.opencv.use class CropBlackEdges { companion object { @@ -17,17 +18,17 @@ class CropBlackEdges { imgGraySlice: Mat, blackPixelThreshold: Int, ratio: Double = 0.6, - ): Boolean { - val pixelsCompared = Mat() - Core.compare( - imgGraySlice, - Scalar(blackPixelThreshold.toDouble()), - pixelsCompared, - Core.CMP_LT, - ) + ): Boolean = + Mat().use { pixelsCompared -> + Core.compare( + imgGraySlice, + Scalar(blackPixelThreshold.toDouble()), + pixelsCompared, + Core.CMP_LT, + ) - return Core.countNonZero(pixelsCompared) > (imgGraySlice.width() * imgGraySlice.height()) * ratio - } + Core.countNonZero(pixelsCompared) > (imgGraySlice.width() * imgGraySlice.height()) * ratio + } private fun getCropRect( imgGray: Mat, @@ -41,31 +42,29 @@ class CropBlackEdges { var top = 0 var bottom = height + // submat views are passed directly instead of pixel-copying clones; + // isBlackEdge only reads the slice for (i in 0..width) { - val rect = Rect(i, 0, 1, height) - val column = imgGray.submat(rect).clone() - if (!isBlackEdge(column, blackPixelThreshold)) break + val isBlack = imgGray.submat(Rect(i, 0, 1, height)).use { isBlackEdge(it, blackPixelThreshold) } + if (!isBlack) break left += 1 } for (i in width downTo 0) { - val rect = Rect(i - 1, 0, 1, height) - val column = imgGray.submat(rect).clone() - if (!isBlackEdge(column, blackPixelThreshold)) break + val isBlack = imgGray.submat(Rect(i - 1, 0, 1, height)).use { isBlackEdge(it, blackPixelThreshold) } + if (!isBlack) break right -= 1 } for (i in 0..height) { - val rect = Rect(0, i, width, 1) - val row = imgGray.submat(rect).clone() - if (!isBlackEdge(row, blackPixelThreshold)) break + val isBlack = imgGray.submat(Rect(0, i, width, 1)).use { isBlackEdge(it, blackPixelThreshold) } + if (!isBlack) break top += 1 } for (i in height downTo 0) { - val rect = Rect(0, i - 1, width, 1) - val row = imgGray.submat(rect).clone() - if (!isBlackEdge(row, blackPixelThreshold)) break + val isBlack = imgGray.submat(Rect(0, i - 1, width, 1)).use { isBlackEdge(it, blackPixelThreshold) } + if (!isBlack) break bottom -= 1 } @@ -84,12 +83,12 @@ class CropBlackEdges { img: Mat, convertFlag: Int = Imgproc.COLOR_BGR2GRAY, blackPixelThreshold: Int = 25, - ): Mat { - val imgGray = Mat() - Imgproc.cvtColor(img, imgGray, convertFlag) - val rect = getCropRect(imgGray, blackPixelThreshold) - return img.submat(rect).clone() - } + ): Mat = + Mat().use { imgGray -> + Imgproc.cvtColor(img, imgGray, convertFlag) + val rect = getCropRect(imgGray, blackPixelThreshold) + img.submat(rect).clone() + } /** * This function would try returning the cropped image. diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcr.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcr.kt index 63bc5535..caf2cd38 100644 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcr.kt +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcr.kt @@ -8,21 +8,16 @@ import org.opencv.core.MatOfPoint import org.opencv.core.Point import org.opencv.core.Scalar import org.opencv.imgproc.Imgproc -import org.opencv.ml.KNearest import xyz.sevive.arcaeaoffline.core.ArcaeaPartnerModifiers import xyz.sevive.arcaeaoffline.core.clearStatusToClearType import xyz.sevive.arcaeaoffline.core.constants.ArcaeaRatingClass import xyz.sevive.arcaeaoffline.core.database.entities.PlayResult -import xyz.sevive.arcaeaoffline.core.ocr.FixRects import xyz.sevive.arcaeaoffline.core.ocr.ImageHashItem import xyz.sevive.arcaeaoffline.core.ocr.ImageHashesDatabase import xyz.sevive.arcaeaoffline.core.ocr.device.rois.extractor.DeviceRoisExtractor import xyz.sevive.arcaeaoffline.core.ocr.device.rois.masker.DeviceRoisMasker import xyz.sevive.arcaeaoffline.core.ocr.getMostConfidentItem -import xyz.sevive.arcaeaoffline.core.ocr.ocrDigitSamplesKnn -import xyz.sevive.arcaeaoffline.core.ocr.ocrDigitsByContourKnn -import xyz.sevive.arcaeaoffline.core.ocr.preprocessHog -import xyz.sevive.arcaeaoffline.core.ocr.resizeFillSquare +import xyz.sevive.arcaeaoffline.core.ocr.opencv.use import kotlin.time.Instant import kotlin.uuid.Uuid @@ -80,7 +75,6 @@ fun DeviceOcrResult.toPlayResult( class DeviceOcr( private val extractor: DeviceRoisExtractor, private val masker: DeviceRoisMasker, - private val kNearestModel: KNearest, private val ortSession: OrtSession, private val hashesDb: ImageHashesDatabase, ) { @@ -103,129 +97,75 @@ class DeviceOcr( val w = iconSquared.width().toDouble() val h = iconSquared.height().toDouble() - Imgproc.fillPoly( - iconSquared, + val contours = listOf( MatOfPoint(Point(0.0, 0.0), Point(w / 2, 0.0), Point(0.0, h / 2)), MatOfPoint(Point(w, 0.0), Point(w / 2, 0.0), Point(w, h / 2)), MatOfPoint(Point(0.0, h), Point(w / 2, h), Point(0.0, h / 2)), MatOfPoint(Point(w, h), Point(w / 2, h), Point(w, h / 2)), - ), - Scalar(128.0), - ) + ) + try { + Imgproc.fillPoly(iconSquared, contours, Scalar(128.0)) + } finally { + contours.forEach { it.release() } + } return iconSquared } } - private fun pfl( - roiGray: Mat, - factor: Double = 1.0, - ): Int { - val contours = ArrayList() - Imgproc.findContours( - roiGray, - contours, - Mat(), - Imgproc.RETR_EXTERNAL, - Imgproc.CHAIN_APPROX_NONE, - ) - val filteredContours = contours.filter { Imgproc.contourArea(it) >= 5 * factor } - var rects = filteredContours.map { Imgproc.boundingRect(it) } - rects = - FixRects.connectBroken(rects, roiGray.width().toDouble(), roiGray.height().toDouble()) - - var filteredRects = rects.filter { it.width >= 5 * factor && it.height >= 6 * factor } - filteredRects = FixRects.splitConnected(roiGray, filteredRects) - filteredRects = filteredRects.sortedBy { it.x } - - val roiOcr = roiGray.clone() - for (contour in contours) { - if (filteredContours.indexOf(contour) > -1) continue - Imgproc.fillPoly(roiOcr, listOf(contour), Scalar(0.0)) + fun ratingClass(): ArcaeaRatingClass = + extractor.ratingClass.use { roi -> + val results = + listOf( + masker.ratingClassPst(roi), + masker.ratingClassPrs(roi), + masker.ratingClassFtr(roi), + masker.ratingClassByd(roi), + masker.ratingClassEtr(roi), + ) + try { + ArcaeaRatingClass.fromInt(results.indices.maxBy { Core.countNonZero(results[it]) }) + } finally { + results.forEach { it.release() } + } } - val digitRois = - filteredRects.map { rect -> resizeFillSquare(roiOcr.submat(rect).clone(), 20) } - val samples = preprocessHog(digitRois) - return ocrDigitSamplesKnn(samples, this.kNearestModel) - } - - fun pure() = pfl(masker.pure(extractor.pure)) - - fun far() = pfl(masker.far(extractor.far)) - - fun lost() = pfl(masker.lost(extractor.lost)) - - fun score(): Int { - val roi = masker.score(extractor.score) - val contours = ArrayList() - Imgproc.findContours( - roi, - contours, - Mat(), - Imgproc.RETR_EXTERNAL, - Imgproc.CHAIN_APPROX_NONE, - ) - for (contour in contours) { - if (Imgproc.boundingRect(contour).height < roi.height() * 0.6) { - Imgproc.fillPoly(roi, listOf(contour), Scalar(0.0)) + private fun clearStatus(): Int = + extractor.clearStatus.use { roi -> + val results = + listOf( + masker.clearStatusTrackLost(roi), + masker.clearStatusTrackComplete(roi), + masker.clearStatusFullRecall(roi), + masker.clearStatusPureMemory(roi), + ) + try { + results.indices.maxBy { Core.countNonZero(results[it]) } + } finally { + results.forEach { it.release() } } } - return ocrDigitsByContourKnn(roi, kNearestModel) - } - - fun ratingClass(): ArcaeaRatingClass { - val roi = extractor.ratingClass - val results = - listOf( - masker.ratingClassPst(roi), - masker.ratingClassPrs(roi), - masker.ratingClassFtr(roi), - masker.ratingClassByd(roi), - masker.ratingClassEtr(roi), - ) - return ArcaeaRatingClass.fromInt(results.indices.maxBy { Core.countNonZero(results[it]) }) - } - - fun maxRecall(): Int = ocrDigitsByContourKnn(masker.maxRecall(extractor.maxRecall), kNearestModel) - private fun clearStatus(): Int { - val roi = extractor.clearStatus - val results = - listOf( - masker.clearStatusTrackLost(roi), - masker.clearStatusTrackComplete(roi), - masker.clearStatusFullRecall(roi), - masker.clearStatusPureMemory(roi), - ) - return results.indices.maxBy { Core.countNonZero(results[it]) } - } - - private fun lookupSongId(): List { - val roiGray = Mat() - Imgproc.cvtColor(extractor.jacket, roiGray, Imgproc.COLOR_BGR2GRAY) - return hashesDb.lookupJacket(roiGray) - } + private fun lookupSongId(): List = + Mat().use { roiGray -> + Imgproc.cvtColor(extractor.jacket, roiGray, Imgproc.COLOR_BGR2GRAY) + hashesDb.lookupJacket(roiGray) + } - private fun lookupPartnerId(): List { - val roiGray = Mat() - Imgproc.cvtColor(extractor.partnerIcon, roiGray, Imgproc.COLOR_BGR2GRAY) - return hashesDb.lookupPartnerIcon(preprocessPartnerIcon(roiGray)) - } + private fun lookupPartnerId(): List = + Mat().use { roiGray -> + Imgproc.cvtColor(extractor.partnerIcon, roiGray, Imgproc.COLOR_BGR2GRAY) + preprocessPartnerIcon(roiGray).use { hashesDb.lookupPartnerIcon(it) } + } fun ocr(): DeviceOcrResult = DeviceOcrResult( ratingClass = ratingClass(), -// pure = pure(), -// far = far(), -// lost = lost(), -// score = score(), -// maxRecall = maxRecall(), - pure = DeviceOcrOnnxHelper.ocrBgrMat(extractor.pure, ortSession).toInt(), - far = DeviceOcrOnnxHelper.ocrBgrMat(extractor.far, ortSession).toInt(), - lost = DeviceOcrOnnxHelper.ocrBgrMat(extractor.lost, ortSession).toInt(), - score = DeviceOcrOnnxHelper.ocrBgrMat(extractor.score, ortSession).toInt(), - maxRecall = DeviceOcrOnnxHelper.ocrBgrMat(extractor.maxRecall, ortSession).toInt(), + pure = extractor.pure.use { DeviceOcrOnnxHelper.ocrBgrMat(it, ortSession).toInt() }, + far = extractor.far.use { DeviceOcrOnnxHelper.ocrBgrMat(it, ortSession).toInt() }, + lost = extractor.lost.use { DeviceOcrOnnxHelper.ocrBgrMat(it, ortSession).toInt() }, + score = extractor.score.use { DeviceOcrOnnxHelper.ocrBgrMat(it, ortSession).toInt() }, + maxRecall = extractor.maxRecall.use { DeviceOcrOnnxHelper.ocrBgrMat(it, ortSession).toInt() }, songIdResults = lookupSongId(), clearStatus = clearStatus(), partnerIdResults = lookupPartnerId(), diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcrOnnxHelper.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcrOnnxHelper.kt index 376e4734..838778df 100644 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcrOnnxHelper.kt +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcrOnnxHelper.kt @@ -13,12 +13,14 @@ import kotlinx.serialization.json.Json import org.opencv.core.Mat import org.opencv.core.Size import org.opencv.imgproc.Imgproc +import xyz.sevive.arcaeaoffline.core.ocr.opencv.use import java.nio.ByteBuffer import kotlin.jvm.optionals.getOrElse import kotlin.properties.Delegates object DeviceOcrOnnxHelper { private const val LOG_TAG = "OnnxHelper" + private const val MODEL_ASSET_PATH = "ocr/model_patched.onnx" private val logger = Logger.withTag(LOG_TAG) private var imageSize by Delegates.notNull() @@ -32,24 +34,47 @@ object DeviceOcrOnnxHelper { @SuppressLint("UnsafeOptInUsageError") @Serializable - private data class ModelInfo( + data class ModelInfo( @SerialName("image_height") val imageHeight: Long, @SerialName("image_width") val imageWidth: Long, @SerialName("labels") val labels: List, @SerialName("blank_token") val blankToken: String, @SerialName("pad_token") val padToken: String, + @SerialName("built_timestamp") val builtTimestamp: Long = 0, + ) + + @SuppressLint("UnsafeOptInUsageError") + @Serializable + data class ModelInfoFile( + val training: ModelInfo, + val patch: ModelPatchInfo? = null, + ) + + @SuppressLint("UnsafeOptInUsageError") + @Serializable + data class ModelPatchInfo( + @SerialName("version") val version: List = emptyList(), + @SerialName("producer_name") val producerName: String? = null, + @SerialName("producer_version") val producerVersion: String? = null, + @SerialName("domain") val domain: String? = null, + @SerialName("graph_name") val graphName: String? = null, + @SerialName("input_names") val inputNames: List = emptyList(), + @SerialName("output_names") val outputNames: List = emptyList(), + @SerialName("patched_timestamp") val patchedTimestamp: Long = 0, ) private fun getOrtEnvironment(): OrtEnvironment = OrtEnvironment.getEnvironment("ocr") + fun loadModelInfoFile(context: Context): ModelInfoFile = + jsonSerializer.decodeFromString( + context.assets + .open("ocr/model_info.json") + .bufferedReader() + .use { it.readText() }, + ) + fun loadModelInfo(context: Context) { - val modelInfo = - jsonSerializer.decodeFromString( - context.assets - .open("ocr/model_info.json") - .bufferedReader() - .use { it.readText() }, - ) + val modelInfo = loadModelInfoFile(context).training logger.d { "Loaded model info $modelInfo" } @@ -60,21 +85,41 @@ object DeviceOcrOnnxHelper { padToken = modelInfo.padToken } - private fun readOnnxModelBytes(context: Context): ByteArray = context.assets.open("ocr/model_patched.onnx").readBytes() + private fun readOnnxModelBytes(context: Context): ByteArray = context.assets.open(MODEL_ASSET_PATH).use { it.readBytes() } /** - * @see ONNX documentation - * - * @return arrayOf(major, minor, patch) + * Verifies the bundled model asset against model_info.json. Creating the + * ORT session validates the file itself and the custom ORT build's op + * coverage; the session metadata is then cross-checked against the json + * description. Returns the parsed info file on success; throws + * [IllegalStateException] listing all mismatches otherwise. */ - fun modelVersion(version: Long): List { - val major = ((version shr 48) and 0xFFFF).toInt() - val minor = ((version shr 32) and 0xFFFF).toInt() - val patch = (version and 0xFFFFFFFF).toInt() - return listOf(major, minor, patch) - } + fun verifyModelAsset(context: Context): ModelInfoFile { + val infoFile = loadModelInfoFile(context) - fun modelVersionString(version: Long): String = "v" + modelVersion(version).joinToString(".") + createOrtSession(context).use { session -> + val metadata = session.metadata + val mismatches = + collectMetadataMismatches( + modelVersion = metadata.version, + producerName = metadata.producerName, + domain = metadata.domain, + graphName = metadata.graphName, + customMetadata = metadata.customMetadata, + inputNames = session.inputNames, + outputNames = session.outputNames, + infoFile = infoFile, + ) + + if (mismatches.isNotEmpty()) { + throw IllegalStateException( + "OCR model asset does not match model_info.json:\n" + mismatches.joinToString("\n"), + ) + } + } + + return infoFile + } fun createOrtSession(context: Context): OrtSession { val ortEnvironment = getOrtEnvironment() @@ -82,32 +127,35 @@ object DeviceOcrOnnxHelper { return OrtSession.SessionOptions().use { it.setIntraOpNumThreads(Runtime.getRuntime().availableProcessors() / 2) + // Custom build onnxruntime cannot ensure optimized node exists, + // so the optimization must be disabled, otherwise ORT_NOT_IMPLEMENTED may occur. + it.setOptimizationLevel(OrtSession.SessionOptions.OptLevel.NO_OPT) ortEnvironment.createSession(onnxModelBytes, it) } } - private fun matToModelInput(rgbMat: Mat): OnnxTensor { - val ortMat = Mat() - Imgproc.resize(rgbMat, ortMat, imageSize) + private fun matToModelInput(rgbMat: Mat): OnnxTensor = + Mat().use { ortMat -> + Imgproc.resize(rgbMat, ortMat, imageSize) - // convert cv.Mat into ByteBuffer - val size = ortMat.total() * ortMat.elemSize() - val byteBuffer: ByteBuffer = ByteBuffer.allocate(size.toInt()) - ortMat.get(0, 0, byteBuffer.array()) + // convert cv.Mat into ByteBuffer + val size = ortMat.total() * ortMat.elemSize() + val byteBuffer: ByteBuffer = ByteBuffer.allocate(size.toInt()) + ortMat.get(0, 0, byteBuffer.array()) - return OnnxTensor.createTensor( - getOrtEnvironment(), - byteBuffer, - imageShape, - OnnxJavaType.UINT8, - ) - } + OnnxTensor.createTensor( + getOrtEnvironment(), + byteBuffer, + imageShape, + OnnxJavaType.UINT8, + ) + } private fun modelDecodedOutputToString(onnxTensor: OnnxTensor): String { val rawPredictions = mutableListOf() for (i in 0 until onnxTensor.info.shape[0]) { - rawPredictions.add(onnxTensor.longBuffer.get(i.toInt()).toInt()) + rawPredictions.add(onnxTensor.intBuffer.get(i.toInt())) } val predictions = rawPredictions.map { labels[it] } @@ -129,16 +177,25 @@ object DeviceOcrOnnxHelper { bgrMat: Mat, ortSession: OrtSession, ): String { - val rgbMat = Mat() - Imgproc.cvtColor(bgrMat, rgbMat, Imgproc.COLOR_BGR2RGB) - - val inputTensor = matToModelInput(rgbMat) - val result = ortSession.run(mapOf("raw_image" to inputTensor)) - val decodedOutput = - result - .get("decoded_output") - .getOrElse { throw NullPointerException("ONNX model output null!") } - val finalResult = modelDecodedOutputToString(decodedOutput as OnnxTensor) + // Ownership notes (per ORT Java API): + // - the input OnnxTensor is NOT owned by OrtSession.Result and must be + // closed by the caller; + // - Result.close() owns and closes the output tensors it contains, and + // the decoded output must be read before Result closes. + val finalResult = + Mat().use { rgbMat -> + Imgproc.cvtColor(bgrMat, rgbMat, Imgproc.COLOR_BGR2RGB) + + matToModelInput(rgbMat).use { inputTensor -> + ortSession.run(mapOf("raw_image" to inputTensor)).use { result -> + val decodedOutput = + result + .get("decoded_output") + .getOrElse { throw NullPointerException("ONNX model output null!") } + modelDecodedOutputToString(decodedOutput as OnnxTensor) + } + } + } var placeholderCount = 0 return buildString { for (char in finalResult) { @@ -155,3 +212,103 @@ object DeviceOcrOnnxHelper { } } } + +/** + * Cross-checks the parsed model_info.json against the model asset's + * metadata. Returns one message per mismatch; empty when they match. + * + * Pure function (no Android or ORT types in the signature) so mismatch + * scenarios can be unit-tested on the JVM. + */ +internal fun collectMetadataMismatches( + modelVersion: Long, + producerName: String?, + domain: String?, + graphName: String?, + customMetadata: Map, + inputNames: Set, + outputNames: Set, + infoFile: DeviceOcrOnnxHelper.ModelInfoFile, +): List { + val mismatches = mutableListOf() + + val patch = infoFile.patch + if (patch == null) { + mismatches += "model_info.json is missing the `patch` block" + return mismatches + } + + val modelVersionList = onnxModelVersion(modelVersion) + if (patch.version != modelVersionList) { + mismatches += "version: json ${patch.version}, model $modelVersionList" + } + + if (patch.producerName != producerName) { + mismatches += "producer_name: json ${patch.producerName}, model $producerName" + } + if (patch.domain != domain) { + mismatches += "domain: json ${patch.domain}, model $domain" + } + if (patch.graphName != graphName) { + mismatches += "graph_name: json ${patch.graphName}, model $graphName" + } + + if (patch.inputNames.toSet() != inputNames) { + mismatches += "input_names: json ${patch.inputNames}, model $inputNames" + } + if (patch.outputNames.toSet() != outputNames) { + mismatches += "output_names: json ${patch.outputNames}, model $outputNames" + } + + val training = infoFile.training + mismatches += + listOfNotNull( + customFieldMismatch(customMetadata, "image_width", training.imageWidth.toString()), + customFieldMismatch(customMetadata, "image_height", training.imageHeight.toString()), + customFieldMismatch(customMetadata, "blank_token", training.blankToken), + customFieldMismatch(customMetadata, "pad_token", training.padToken), + // 0 is the "not written" marker on the json side, matching an absent key + customTimestampMismatch(customMetadata, "built_timestamp", training.builtTimestamp), + customTimestampMismatch(customMetadata, "patched_timestamp", patch.patchedTimestamp), + ) + + return mismatches +} + +private fun customFieldMismatch( + customMetadata: Map, + key: String, + jsonValue: String, +): String? { + val modelValue = customMetadata[key] + return when { + modelValue == null -> "`$key`: json has \"$jsonValue\" but missing in model metadata" + modelValue != jsonValue -> "`$key`: json \"$jsonValue\", model \"$modelValue\"" + else -> null + } +} + +private fun customTimestampMismatch( + customMetadata: Map, + key: String, + jsonValue: Long, +): String? { + val modelValue = customMetadata[key] + return when { + jsonValue == 0L && modelValue == null -> null + jsonValue == 0L -> "`$key`: missing in json but model has \"$modelValue\"" + modelValue == null -> "`$key`: json has $jsonValue but missing in model metadata" + modelValue.toLongOrNull() != jsonValue -> "`$key`: json $jsonValue, model \"$modelValue\"" + else -> null + } +} + +/** + * @see ONNX documentation + */ +private fun onnxModelVersion(version: Long): List { + val major = ((version shr 48) and 0xFFFF).toInt() + val minor = ((version shr 32) and 0xFFFF).toInt() + val patch = (version and 0xFFFFFFFF).toInt() + return listOf(major, minor, patch) +} diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/OcrPerformanceBenchmark.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/OcrPerformanceBenchmark.kt new file mode 100644 index 00000000..075bbf41 --- /dev/null +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/OcrPerformanceBenchmark.kt @@ -0,0 +1,163 @@ +package xyz.sevive.arcaeaoffline.core.ocr.device + +import ai.onnxruntime.OrtSession +import android.content.Context +import co.touchlab.kermit.Logger +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.launch +import org.opencv.core.Mat +import xyz.sevive.arcaeaoffline.core.ocr.device.rois.DeviceRoisAutoSelector +import xyz.sevive.arcaeaoffline.core.ocr.device.rois.DeviceRoisAutoSelectorResult +import xyz.sevive.arcaeaoffline.core.ocr.device.rois.definition.DeviceRoisAutoT1 +import xyz.sevive.arcaeaoffline.core.ocr.device.rois.definition.DeviceRoisAutoT2 +import xyz.sevive.arcaeaoffline.core.ocr.device.rois.extractor.DeviceRoisExtractor +import xyz.sevive.arcaeaoffline.core.ocr.opencv.use +import java.util.Locale +import kotlin.system.measureTimeMillis + +/** + * OCR performance benchmark. Mirrors the production queue's Channel-based + * concurrency model (OcrQueueProcessingJob): runs fixed task batches at a + * given parallel count and reports timings/throughput, so users can pick a + * suitable OcrQueuePreferences.parallelCount. + * + * Inference path matches production (DeviceOcrOnnxHelper.createOrtSession + ocrBgrMat). + */ +object OcrPerformanceBenchmark { + private const val LOG_TAG = "OcrPerfBench" + private val logger = Logger.withTag(LOG_TAG) + + const val DEFAULT_TASKS_PER_BATCH: Int = 40 + const val DEFAULT_WARMUP_BATCHES: Int = 1 + const val DEFAULT_TIMED_BATCHES: Int = 5 + + data class Result( + val batchTimesMs: List, + val medianPerImageMs: Double, + val throughputPerSecond: Double, + val resultsConsistent: Boolean, + ) + + /** + * Returns the OCR ROIs for one image as cloned Mats that own their data; + * ownership transfers to the caller. + */ + fun extractRois(img: Mat): List = + CropBlackEdges.crop(img).use { imgCropped -> + val rois = + when (DeviceRoisAutoSelector.select(img)) { + DeviceRoisAutoSelectorResult.T1 -> DeviceRoisAutoT1(imgCropped.width(), imgCropped.height()) + else -> DeviceRoisAutoT2(imgCropped.width(), imgCropped.height()) + } + val extractor = DeviceRoisExtractor(rois, imgCropped) + + // extractor properties return submat views of imgCropped; clone them + // so the returned Mats stay valid after imgCropped is released. + listOf(extractor.pure, extractor.far, extractor.lost, extractor.score, extractor.maxRecall).map { view -> + view.use { it.clone() } + } + } + + /** + * Takes ownership of [roiSets]: the ROI Mats are released on every exit + * path (completion, failure, cancellation). Reuses the same ROI Mats for + * all batches and warmups; inference reads them without mutating, which + * is safe concurrently. + */ + suspend fun runBenchmark( + context: Context, + roiSets: List>, + parallel: Int, + tasksPerBatch: Int = DEFAULT_TASKS_PER_BATCH, + warmupBatches: Int = DEFAULT_WARMUP_BATCHES, + timedBatches: Int = DEFAULT_TIMED_BATCHES, + onProgress: (completed: Int, total: Int) -> Unit = { _, _ -> }, + ): Result { + require(roiSets.isNotEmpty()) { "At least one image is required" } + require(parallel >= 1) { "Parallel count must be >= 1" } + + val tasks = List(tasksPerBatch) { roiSets[it % roiSets.size] } + try { + DeviceOcrOnnxHelper.createOrtSession(context).use { session -> + val totalBatches = warmupBatches + timedBatches + repeat(warmupBatches) { i -> + processBatch(tasks, parallel, session) + onProgress(i + 1, totalBatches) + } + + val batchTimes = mutableListOf() + var reference: List>? = null + var consistent = true + repeat(timedBatches) { i -> + val batch = processBatch(tasks, parallel, session) + batchTimes += batch.elapsedMs + val ref = reference ?: batch.results.also { reference = it } + if (batch.results != ref) consistent = false + onProgress(warmupBatches + i + 1, totalBatches) + } + + // Median resists single-run spikes (DVFS/scheduling noise), used instead of the mean + val sortedBatchTimes = batchTimes.sorted() + val medianBatchMs = + if (sortedBatchTimes.size % 2 == 1) { + sortedBatchTimes[sortedBatchTimes.size / 2].toDouble() + } else { + (sortedBatchTimes[sortedBatchTimes.size / 2 - 1] + sortedBatchTimes[sortedBatchTimes.size / 2]) / 2.0 + } + logger.i { + "Benchmark done: parallel %d, taskPerBatch %d, batchTimes(ms) %s, throughput %.1f it/s, consistent %s".format( + Locale.ROOT, + parallel, + tasksPerBatch, + batchTimes.joinToString("/"), + tasksPerBatch * 1000.0 / medianBatchMs, + consistent, + ) + } + return Result( + batchTimesMs = batchTimes, + medianPerImageMs = medianBatchMs / tasksPerBatch, + throughputPerSecond = tasksPerBatch * 1000.0 / medianBatchMs, + resultsConsistent = consistent, + ) + } + } finally { + roiSets.flatten().forEach { it.release() } + } + } + + private data class BatchResult( + val elapsedMs: Long, + val results: List>, + ) + + private suspend fun processBatch( + tasks: List>, + parallel: Int, + session: OrtSession, + ): BatchResult { + val results = MutableList(tasks.size) { emptyList() } + val elapsedMs = + measureTimeMillis { + coroutineScope { + val channel = Channel(parallel) + + launch { + tasks.indices.forEach { channel.send(it) } + channel.close() + } + + repeat(parallel) { + launch(Dispatchers.IO) { + for (idx in channel) { + results[idx] = tasks[idx].map { DeviceOcrOnnxHelper.ocrBgrMat(it, session) } + } + } + } + } + } + return BatchResult(elapsedMs, results) + } +} diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/masker/DeviceRoisMasker.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/masker/DeviceRoisMasker.kt index 473e4f9e..1d09c728 100644 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/masker/DeviceRoisMasker.kt +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/masker/DeviceRoisMasker.kt @@ -2,6 +2,7 @@ package xyz.sevive.arcaeaoffline.core.ocr.device.rois.masker import org.opencv.core.Mat +/** Every method returns a newly allocated Mat; ownership passes to the caller. */ interface DeviceRoisMasker { fun pure(roiBgr: Mat): Mat diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/opencv/Utils.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/opencv/Utils.kt new file mode 100644 index 00000000..cb7ea058 --- /dev/null +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/opencv/Utils.kt @@ -0,0 +1,20 @@ +package xyz.sevive.arcaeaoffline.core.ocr.opencv + +import org.opencv.core.Mat +import kotlin.contracts.ExperimentalContracts +import kotlin.contracts.InvocationKind +import kotlin.contracts.contract + +/** + * OpenCV Mats are backed by native memory and only reclaimed by GC + * finalization otherwise, which delays reclamation unpredictably. + */ +@OptIn(ExperimentalContracts::class) +inline fun Mat.use(block: (Mat) -> T): T { + contract { callsInPlace(block, InvocationKind.EXACTLY_ONCE) } + return try { + block(this) + } finally { + release() + } +} diff --git a/core/src/test/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/ModelMetadataVerificationTest.kt b/core/src/test/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/ModelMetadataVerificationTest.kt new file mode 100644 index 00000000..9a34ec90 --- /dev/null +++ b/core/src/test/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/ModelMetadataVerificationTest.kt @@ -0,0 +1,122 @@ +package xyz.sevive.arcaeaoffline.core.ocr.device + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +/** + * JVM unit tests for [collectMetadataMismatches], using the field values of + * the real bundled model assets as the matching baseline. + */ +class ModelMetadataVerificationTest { + private val training = + DeviceOcrOnnxHelper.ModelInfo( + imageHeight = 50, + imageWidth = 220, + labels = listOf("∅", "-", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9"), + blankToken = "∅", + padToken = "-", + builtTimestamp = 1728488367L, + ) + + private val patch = + DeviceOcrOnnxHelper.ModelPatchInfo( + version = listOf(1, 0, 3), + producerName = "pytorch", + producerVersion = "2.4.1", + domain = "", + graphName = "crnn_patched", + inputNames = listOf("raw_image"), + outputNames = listOf("model_output", "decoded_output"), + patchedTimestamp = 1785689264L, + ) + + // encode_semver(1, 0, 3) + private val encodedVersion = (1L shl 48) or (0L shl 32) or 3L + + private fun customMetadata( + builtTimestamp: String? = "1728488367", + patchedTimestamp: String? = "1785689264", + ): Map = + buildMap { + put("image_width", "220") + put("image_height", "50") + put("blank_token", "∅") + put("pad_token", "-") + builtTimestamp?.let { put("built_timestamp", it) } + patchedTimestamp?.let { put("patched_timestamp", it) } + } + + private fun mismatches( + patchBlock: DeviceOcrOnnxHelper.ModelPatchInfo? = patch, + trainingBlock: DeviceOcrOnnxHelper.ModelInfo = training, + metadata: Map = customMetadata(), + version: Long = encodedVersion, + producerName: String? = "pytorch", + domain: String? = "", + graphName: String? = "crnn_patched", + inputNames: Set = setOf("raw_image"), + outputNames: Set = setOf("model_output", "decoded_output"), + ): List = + collectMetadataMismatches( + modelVersion = version, + producerName = producerName, + domain = domain, + graphName = graphName, + customMetadata = metadata, + inputNames = inputNames, + outputNames = outputNames, + infoFile = DeviceOcrOnnxHelper.ModelInfoFile(training = trainingBlock, patch = patchBlock), + ) + + @Test + fun matchingAssetProducesNoMismatches() { + assertEquals(emptyList(), mismatches()) + } + + @Test + fun missingPatchBlockIsReported() { + assertEquals( + listOf("model_info.json is missing the `patch` block"), + mismatches(patchBlock = null), + ) + } + + @Test + fun versionMismatchIsReported() { + val result = mismatches(version = (1L shl 48) or 4L) + assertTrue(result.any { it.startsWith("version:") }) + } + + @Test + fun producerNameMismatchIsReported() { + val result = mismatches(producerName = "other-producer") + assertTrue(result.any { it.startsWith("producer_name:") }) + } + + @Test + fun inputNamesMismatchIsReported() { + val result = mismatches(inputNames = setOf("image")) + assertTrue(result.any { it.startsWith("input_names:") }) + } + + @Test + fun missingCustomMetadataKeyIsReported() { + val result = mismatches(metadata = customMetadata(builtTimestamp = null)) + assertTrue(result.any { it.startsWith("`built_timestamp`:") }) + } + + @Test + fun nonNumericTimestampIsReported() { + val result = mismatches(metadata = customMetadata(patchedTimestamp = "not-a-number")) + assertTrue(result.any { it.startsWith("`patched_timestamp`:") }) + } + + @Test + fun bothSidesMissingTimestampIsNotReported() { + // json built_timestamp defaults to 0 (not written), matching an + // absent model metadata key + val result = mismatches(trainingBlock = training.copy(builtTimestamp = 0L), metadata = customMetadata(builtTimestamp = null)) + assertTrue(result.isEmpty()) + } +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index f10c89e8..a95e5686 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -14,7 +14,7 @@ compose-multiplatform = "1.11.1" compose-material3 = "1.9.0" compose-material-icons-extended = "1.7.3" -opencv = "4.13.0" +opencv = "5.0.0" apache-commons-compress = "1.28.0" io-sentry-sentryAndroid = "8.17.0" diff --git a/settings.gradle.kts b/settings.gradle.kts index d29b9587..d2e2fb04 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -13,6 +13,14 @@ dependencyResolutionManagement { repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) @Suppress("UnstableApiUsage") repositories { + maven { + setUrl( + providers + .gradleProperty("localMavenRepo") + .getOrElse(rootDir.resolve("maven-local").absolutePath), + ) + } + google() mavenCentral() diff --git a/shared/src/commonMain/composeResources/values-zh-rCN/strings.xml b/shared/src/commonMain/composeResources/values-zh-rCN/strings.xml index 6bfb2852..eea6fda0 100644 --- a/shared/src/commonMain/composeResources/values-zh-rCN/strings.xml +++ b/shared/src/commonMain/composeResources/values-zh-rCN/strings.xml @@ -67,7 +67,6 @@ 保存图片 - KNearest 模型 图像哈希数据库 CRNN OCR 模型 diff --git a/shared/src/commonMain/composeResources/values/strings.xml b/shared/src/commonMain/composeResources/values/strings.xml index 928fbe5e..051378c5 100644 --- a/shared/src/commonMain/composeResources/values/strings.xml +++ b/shared/src/commonMain/composeResources/values/strings.xml @@ -73,7 +73,6 @@ Save image - KNearest model Image hashes database CRNN OCR model