diff --git a/CHANGELOG.md b/CHANGELOG.md index 76e3f7fc..f2962d28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## 2.11.0 +- **FEAT**(android, iOS, macOS): Add `ChromaKey` โ€” green-screen removal with a soft edge and spill suppression. The keyed area becomes a `backgroundColor`, a `backgroundImage`, or (in a `VideoComposition`) the layer below. Settable on `VideoRenderData`, `VideoLayer` and `VideoSegment`, resolved per clip as segment โ†’ layer โ†’ global. H.264/HEVC carry no alpha, so on the single-track `videoSegments` path a key without a background is flattened to black. +- **FEAT**: `ChromaKey.autoDetect(video)` measures the key color and `similarity` off the footage, for any saturated screen hue. `ChromaKey.detect` returns the raw measurement. Costs one thumbnail decode, no render. +- **FEAT**: `ChromaKey.greenScreen()` and `ChromaKey.blueScreen()` presets. Blue separates skin better but crowds denim and blue eyes, so it keys tighter and despills more gently. +- **FIX**(android): A `VideoComposition` layer no longer squares its own alpha. `VideoCompositionTransformation` blended against an already-transparent framebuffer, thinning any partially transparent edge. +- **FIX**(android, iOS, macOS): Two sources resolved in the same millisecond no longer collide on one temp file in `EditorVideo.safeFilePath`. + ## 2.10.0 - **FEAT**(iOS, macOS): Add `VideoRenderData.trimToCommonTrackEnd` (default `false`). Ends each clip where both its video and audio track still have content, so an export no longer finishes on a stretch of missing audio โ€” the seam a looping player replays every cycle. Leave it off when a clip is meant to outlast its own audio. diff --git a/README.md b/README.md index 890a75fd..60d0f75c 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,7 @@ The ProVideoEditor is a Flutter widget designed for video editing within your ap - ๐ŸŽž๏ธ **Clip Transitions**: Add transitions between adjacent clips โ€” `dissolve`, `fadeToBlack`, `fadeToWhite`, `slide`, `push`, and `wipe` โ€” with configurable duration, easing curve, and direction. - ๐Ÿงฎ **Color Matrix**: Apply one or multiple 4x5 color matrices (e.g., for filters). - ๐Ÿ’ง **Blur**: Add a blur effect to the video. +- ๐ŸŸฉ **Chroma Key**: Remove a green (or blue, or any saturated hue) screen with a soft edge and spill suppression, and fill it with a color, an image, or โ€” in a `VideoComposition` โ€” the layer below. `ChromaKey.autoDetect` measures the key straight off the footage; `greenScreen()`/`blueScreen()` presets are there when you already know. Configurable globally, per `VideoLayer`, or per `VideoSegment`. - ๐Ÿ“ก **Bitrate**: Cap the video bitrate. Sources already below the cap are exported losslessly over the fast path; sources above it are re-encoded down to the cap. If constant bitrate (CBR) isn't supported, it will gracefully fall back to the next available mode. - ๐ŸŒ **Streaming Optimization**: Optimize video for progressive playback by placing metadata (moov atom) at the start of the file. @@ -160,6 +161,7 @@ The ProVideoEditor is a Flutter widget designed for video editing within your ap | `Multiple ColorMatrix 4x5` | โœ… | โœ… | โœ… | โŒ | โŒ | ๐Ÿšซ | | `Cancel export task` | โœ… | โœ… | โœ… | โŒ | โŒ | ๐Ÿšซ | | `Blur background` | ๐Ÿงช | ๐Ÿงช | ๐Ÿงช | โŒ | โŒ | ๐Ÿšซ | +| `Chroma Key (Greenscreen)` | โœ… | โœ… | โœ… | โŒ | โŒ | ๐Ÿšซ | | `Custom Audio Tracks` | โœ… | โœ… | โœ… | โŒ | โŒ | ๐Ÿšซ | | `Merge Videos` | โœ… | โœ… | โœ… | โŒ | โŒ | ๐Ÿšซ | | `Stop-Motion (Imagesโ†’Video)`| โœ… | โœ… | โœ… | โŒ | โŒ | ๐Ÿšซ | @@ -620,6 +622,15 @@ var task = VideoRenderData( ColorFilter(matrix: [ 1.0, 0.0, 0.0, 0.0, 50.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0 ]), ColorFilter(matrix: [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0 ]), ], + chromaKey: const ChromaKey( + // Defaults to SMPTE "chroma key green". Brightness matters as well as + // hue, so measure it with `ChromaKey.autoDetect` when you can. + color: Color(0xFF00B140), + similarity: 0.20, // raise it if the screen survives the key + smoothness: 0.08, // width of the soft edge + spill: 0.5, // pull the green cast off the subject's edges + backgroundColor: Color(0xFF000000), + ), ); Uint8List result = await ProVideoEditor.instance.renderVideo(task); @@ -627,6 +638,41 @@ Uint8List result = await ProVideoEditor.instance.renderVideo(task); /// Note: Blur is an experimental feature (๐Ÿงช in platform matrix) /// The blur effect may render differently than in Flutter's preview. +/// Chroma key: the most reliable key is measured, not guessed. Paint, lighting +/// and the camera all shift the recorded screen away from any constant, and +/// that shift has to be paid for with a wider `similarity` โ€” the very margin +/// that protects the subject. On a real studio clip the SMPTE constant sat +/// 0.10โ€“0.20 away from the screen and needed `similarity: 0.20`; the measured +/// color sat within 0.05 and 0.10 was plenty. +final key = await ChromaKey.autoDetect( + greenScreenVideo, + backgroundColor: Colors.black, +); + +/// When you already know the screen, the presets carry the measured values. +/// Note that blue is not green with a different hue: it separates skin better, +/// but denim (0.19), blue eyes (0.20) and light blue shirts (0.21) all crowd +/// it, so `blueScreen()` keys tighter and despills more gently. +const greenKey = ChromaKey.greenScreen(backgroundColor: Color(0xFF000000)); +const blueKey = ChromaKey.blueScreen(backgroundColor: Color(0xFF000000)); + +/// Chroma key: to put a *video* behind the green screen, leave the key +/// transparent and place the keyed clip on a layer above another one. In the +/// single-track `videoSegments` path there is nothing underneath and the codec +/// carries no alpha, so a key there needs a `backgroundColor` or +/// `backgroundImage`. +final keyedTask = VideoRenderData( + composition: VideoComposition( + layers: [ + VideoLayer(clips: [VideoSegment(video: backgroundVideo)]), + VideoLayer( + clips: [VideoSegment(video: greenScreenVideo)], + chromaKey: const ChromaKey(), + ), + ], + ), +); + /// Listen progress StreamBuilder( stream: ProVideoEditor.instance.progressStreamById(task.id), diff --git a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/RenderVideo.kt b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/RenderVideo.kt index 02f05aec..2389a628 100644 --- a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/RenderVideo.kt +++ b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/RenderVideo.kt @@ -26,6 +26,7 @@ import ch.waio.pro_video_editor.src.features.render.helpers.VideoReverser import ch.waio.pro_video_editor.src.features.render.helpers.ClipTransitionGeometry import ch.waio.pro_video_editor.src.features.render.helpers.ClipTransitionRenderer import ch.waio.pro_video_editor.src.features.render.helpers.MediaInfoExtractor +import ch.waio.pro_video_editor.src.features.render.models.ChromaKeyConfig import ch.waio.pro_video_editor.src.features.render.models.CodecResourceExhaustedException import ch.waio.pro_video_editor.src.features.render.models.RenderConfig import ch.waio.pro_video_editor.src.features.render.models.RenderJobHandle @@ -89,7 +90,17 @@ class RenderVideo(private val context: Context) { val hasBlur = config.blur != null && config.blur > 0.0 val hasColorFilters = config.colorFilters.isNotEmpty() - return hasImageLayers || hasBlur || hasColorFilters + // The chroma key is an ES 2.0 SDR shader, so a 10-bit HDR source has to + // be transcoded down first. Checked at every level it can be set: + // globally, per clip, and โ€” for the layered path โ€” per layer and per + // layer clip. + val hasChromaKey = config.chromaKey != null || + config.videoClips.any { it.chromaKey != null } || + config.composition?.layers?.any { layer -> + layer.chromaKey != null || layer.clips.any { it.chromaKey != null } + } == true + + return hasImageLayers || hasBlur || hasColorFilters || hasChromaKey } /** @@ -211,7 +222,16 @@ class RenderVideo(private val context: Context) { // 1) Pre-transcode HEVC 10-bit clips (if needed). if (needsPreTranscode) { Log.d(RENDER_TAG, "Pre-transcoding HEVC 10-bit videos...") - val inputPaths = workingConfig.videoClips.map { it.inputPath } + // Both paths, not just the single-track one: a GPU + // effect on a composition layer needs its clips + // transcoded too. ChromaKeyEffect is an ES 2.0 SDR + // shader and hard-fails on an HDR input, so leaving the + // layered clips untouched here aborted the export. + val inputPaths = ( + workingConfig.videoClips.map { it.inputPath } + + (workingConfig.composition?.layers ?: emptyList()) + .flatMap { layer -> layer.clips.map { it.inputPath } } + ).distinct() val transcodeMap = VideoTranscoder.transcodeClipsIfNeeded( context, inputPaths ) @@ -223,13 +243,22 @@ class RenderVideo(private val context: Context) { "Pre-transcoded ${transcodedFiles.size} HEVC 10-bit videos to H.264" ) } - val updatedClips = workingConfig.videoClips.map { clip -> + fun retarget(clip: VideoClip): VideoClip { val newPath = transcodeMap[clip.inputPath] ?: clip.inputPath - if (newPath != clip.inputPath) { + return if (newPath != clip.inputPath) { clip.copy(inputPath = newPath) } else clip } - workingConfig = workingConfig.copy(videoClips = updatedClips) + workingConfig = workingConfig.copy( + videoClips = workingConfig.videoClips.map(::retarget), + composition = workingConfig.composition?.let { composition -> + composition.copy( + layers = composition.layers.map { layer -> + layer.copy(clips = layer.clips.map(::retarget)) + } + ) + } + ) } // 2) Pre-render reversed clips into single forward MP4 temp files. @@ -262,6 +291,7 @@ class RenderVideo(private val context: Context) { val transitionedClips = preRenderTransitions( workingConfig.videoClips, enableAudio = workingConfig.enableAudio, + globalChromaKey = workingConfig.chromaKey, shouldStop = shouldStopPolling, collectPath = { paths.add(it) }, onProgress = { f -> transitionProgress(f.toDouble()) }, @@ -399,6 +429,7 @@ class RenderVideo(private val context: Context) { private fun preRenderTransitions( clips: List, enableAudio: Boolean, + globalChromaKey: ChromaKeyConfig?, shouldStop: AtomicBoolean, collectPath: (String) -> Unit, onProgress: (Float) -> Unit, @@ -497,11 +528,14 @@ class RenderVideo(private val context: Context) { // Keep the outgoing clip's speed; it now ends `tailSrc` of source // earlier (those frames moved into the speed-adjusted blend). addClip(current.copy(endUs = curEnd - tailSrc, transition = null)) + val blendKey = blendChromaKey(current, next, globalChromaKey, "$i") result.add( VideoClip( inputPath = rendered.outputPath, startUs = 0L, endUs = rendered.durationUs.takeIf { it > 0 }, + chromaKey = blendKey.config, + suppressChromaKey = blendKey.suppressed, ) ) // Trim the incoming head in place; it keeps its own speed/transition. @@ -587,11 +621,15 @@ class RenderVideo(private val context: Context) { result[0] = first.copy(startUs = firstStart + headSrc) result[lastIdx] = last.copy(endUs = lastEnd - tailSrc) } + val wrapKey = + blendChromaKey(last, first, globalChromaKey, "loop wrap") result.add( VideoClip( inputPath = rendered.outputPath, startUs = 0L, endUs = rendered.durationUs.takeIf { it > 0 }, + chromaKey = wrapKey.config, + suppressChromaKey = wrapKey.suppressed, ) ) } else { @@ -605,6 +643,47 @@ class RenderVideo(private val context: Context) { return result } + /** + * The chroma key to apply to a pre-rendered overlap blend. + * + * The blend is composed from the raw sources by [ClipTransitionRenderer], + * which knows nothing about keying, so the key has to be re-applied to its + * output. That only has a defined meaning when both sides key the same way: + * blending a keyed clip with an unkeyed one produces mixed colors that no + * single key can undo. Mismatched sides are therefore left unkeyed and + * reported, rather than silently keyed with one side's settings. + */ + private fun blendChromaKey( + outgoing: VideoClip, + incoming: VideoClip, + global: ChromaKeyConfig?, + boundary: String, + ): BlendChromaKey { + // Compare the *effective* keys. A clip that leaves its own key null + // still inherits the global one, so comparing the raw per-clip fields + // reported a mismatch for two clips that in fact key identically. + val outgoingKey = outgoing.chromaKey ?: global + val incomingKey = incoming.chromaKey ?: global + if (outgoingKey == incomingKey) return BlendChromaKey(outgoingKey, false) + + Log.w( + RENDER_TAG, + "Chroma key: the two clips at boundary $boundary use different keys; " + + "the pre-rendered transition blend is emitted unkeyed" + ) + // A null key alone would mean "inherit", and `VideoSequenceBuilder` + // would then fall back to the global key โ€” applying to the blend the + // very thing this warning promises it will not. The suppress flag is + // what makes "unkeyed" actually mean unkeyed. + return BlendChromaKey(null, true) + } + + /** The key for a pre-rendered blend, and whether it opts out of the global one. */ + private data class BlendChromaKey( + val config: ChromaKeyConfig?, + val suppressed: Boolean, + ) + /** * Internal render implementation after optional pre-transcoding. */ diff --git a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ApplyChromaKey.kt b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ApplyChromaKey.kt new file mode 100644 index 00000000..f00a9508 --- /dev/null +++ b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ApplyChromaKey.kt @@ -0,0 +1,56 @@ +import androidx.media3.common.Effect +import androidx.media3.common.util.UnstableApi +import ch.waio.pro_video_editor.src.features.render.helpers.ChromaKeyEffect +import ch.waio.pro_video_editor.src.features.render.models.ChromaKeyConfig +import ch.waio.pro_video_editor.src.shared.logging.PluginLog as Log + +/** + * Adds a chroma key ("green screen" removal) to a clip's effect chain. + * + * Must be added **before** rotation, flip, the color LUT and blur, so the key + * sees the original decoded colors. Media3's LUT shader preserves alpha + * (`gl_FragColor.a = inputColor.a`), so a color filter after the key is safe. + * + * @param videoEffects List to add the effect to + * @param config The resolved key (clip ?: layer ?: global), or null for none + * @param flattenTransparency Whether a key without a background should be + * filled with opaque black instead of left transparent. True on the + * single-track path โ€” see below. + */ +@UnstableApi +fun applyChromaKey( + videoEffects: MutableList, + config: ChromaKeyConfig?, + flattenTransparency: Boolean = false, +) { + if (config == null) return + + // H.264/HEVC carry no alpha channel, and the single-track path has no layer + // underneath to blend into, so "transparent" has no meaning there. Left + // alone the two platforms would disagree visibly: Apple's premultiplied + // color cube writes rgb*0 and the area comes out black, while this shader + // writes straight alpha and leaves the RGB untouched, so the screen would + // simply stay green โ€” "nothing happened". Substituting opaque black makes + // both platforms produce the documented result. + val effective = if (flattenTransparency && config.isTransparent) { + config.copy(backgroundColor = OPAQUE_BLACK) + } else { + config + } + + val background = when { + effective.backgroundImageData != null -> "image" + effective.backgroundColor != null -> "color" + else -> "transparent" + } + Log.d( + RENDER_TAG, + "Applying chroma key: similarity=${effective.similarity} " + + "smoothness=${effective.smoothness} spill=${effective.spill} " + + "background=$background" + ) + + videoEffects += ChromaKeyEffect(effective) +} + +private const val OPAQUE_BLACK = 0xFF000000.toInt() diff --git a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ApplyComposition.kt b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ApplyComposition.kt index 519d0cc8..25cf46b0 100644 --- a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ApplyComposition.kt +++ b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ApplyComposition.kt @@ -69,7 +69,8 @@ fun applyComposition( imageLayers = imageLayerConfigs, audioTracks = config.audioTracks, globalStartUs = config.startUs, - globalEndUs = config.endUs + globalEndUs = config.endUs, + globalChromaKey = config.chromaKey ) return CompositionResult( layeredBuilder.build(), diff --git a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ChromaKeyEffect.kt b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ChromaKeyEffect.kt new file mode 100644 index 00000000..3c24e3fb --- /dev/null +++ b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ChromaKeyEffect.kt @@ -0,0 +1,347 @@ +package ch.waio.pro_video_editor.src.features.render.helpers + +import RENDER_TAG +import android.content.Context +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.opengl.GLES20 +import androidx.media3.common.VideoFrameProcessingException +import androidx.media3.common.util.GlProgram +import androidx.media3.common.util.GlUtil +import androidx.media3.common.util.Size +import androidx.media3.common.util.UnstableApi +import androidx.media3.effect.BaseGlShaderProgram +import androidx.media3.effect.GlEffect +import androidx.media3.effect.GlShaderProgram +import ch.waio.pro_video_editor.src.features.render.models.ChromaKeyConfig +import ch.waio.pro_video_editor.src.shared.logging.PluginLog as Log + +/** + * Removes a solid-colored background ("green screen") from every frame. + * + * Pixels whose chroma sits within [ChromaKeyConfig.similarity] of the key color + * are removed, with a [ChromaKeyConfig.smoothness]-wide soft edge, and the key's + * color cast is pulled back out of the pixels that remain + * ([ChromaKeyConfig.spill]). + * + * The removed area becomes, in this order of precedence: the background image, + * the background color, or transparency. Transparency only means something in + * the layered path, where Media3's compositor blends the sequences โ€” see + * `applyChromaKey`, which substitutes opaque black on the single-track path so + * both platforms agree. + * + * The formula is specified once and implemented twice; see [ChromaKeyMath], + * which this shader mirrors and which the unit tests pin against the Swift + * implementation. + * + * Alpha is **straight**, not premultiplied โ€” that is Media3's convention + * throughout (`AlphaScale`, the LUT shader, and `DefaultCompositorGlProgram`'s + * `glBlendFuncSeparate(SRC_ALPHA, ONE_MINUS_SRC_ALPHA, ONE, ONE_MINUS_SRC_ALPHA)`). + * Apple's color cube stores premultiplied entries instead, because that is what + * `CIColorCube` requires. Both are correct for their platform; do not align them. + */ +@UnstableApi +class ChromaKeyEffect(private val config: ChromaKeyConfig) : GlEffect { + + override fun toGlShaderProgram(context: Context, useHdr: Boolean): GlShaderProgram { + if (useHdr) { + // The shader is an ES 2.0 SDR program. HEVC 10-bit/HDR sources are + // pre-transcoded to 8-bit SDR by VideoTranscoder before they reach + // here โ€” `RenderVideo.hasGpuEffects` detects a key at every level it + // can be set, and the transcode step rewrites both `videoClips` and + // every composition layer's clips. So this is a guard against that + // gate regressing rather than an expected path. + throw VideoFrameProcessingException( + "Chroma key does not support HDR input" + ) + } + return ChromaKeyShaderProgram(useHdr, config) + } + + @UnstableApi + private class ChromaKeyShaderProgram( + useHdr: Boolean, + private val config: ChromaKeyConfig + ) : BaseGlShaderProgram(useHdr, /* texturePoolCapacity= */ 1) { + + private val glProgram: GlProgram + + /** + * Bounds-only probe of the background image. + * + * Decoding just the header tells us whether the bytes are usable โ€” and + * how large they are โ€” without holding a full-size bitmap from + * construction until the first draw. The real decode happens in + * [drawFrame], where there is a GL context to size it against. + */ + private val backgroundBounds: BitmapFactory.Options? = + config.backgroundImageData?.let { bytes -> + BitmapFactory.Options().apply { + inJustDecodeBounds = true + BitmapFactory.decodeByteArray(bytes, 0, bytes.size, this) + }.takeIf { it.outWidth > 0 && it.outHeight > 0 } + } + + /** + * Background texture id. A 1x1 placeholder is uploaded when there is no + * background image, because Media3's [GlProgram] requires every sampler + * uniform to be bound before a draw. + */ + private var backgroundTexId: Int = -1 + + private val bgMode: Int = when { + backgroundBounds != null -> BG_IMAGE + config.backgroundColor != null -> BG_COLOR + // An image was asked for but its bytes will not decode. Falling + // through to BG_TRANSPARENT would quietly un-key the clip on the + // single-track path, where `applyChromaKey` has already ruled that + // the area must be filled โ€” the export would come out looking like + // the key never ran. Fill with opaque black instead, which is what + // Apple produces in the same situation. + config.backgroundImageData != null -> BG_COLOR + else -> BG_TRANSPARENT + } + + companion object { + private const val BG_TRANSPARENT = 0 + private const val BG_COLOR = 1 + private const val BG_IMAGE = 2 + + private const val VERTEX_SHADER_SOURCE = + "attribute vec4 aFramePosition;\n" + + "attribute vec4 aTexSamplingCoord;\n" + + "varying vec2 vTexSamplingCoord;\n" + + "void main() {\n" + + " gl_Position = aFramePosition;\n" + + " vTexSamplingCoord = aTexSamplingCoord.xy;\n" + + "}" + + // highp matters here: the Cb/Cr deltas the key discriminates are on + // the order of 1e-3, and mediump only guarantees ~10 mantissa bits. + private const val FRAGMENT_SHADER_SOURCE = + "#ifdef GL_FRAGMENT_PRECISION_HIGH\n" + + "precision highp float;\n" + + "#else\n" + + "precision mediump float;\n" + + "#endif\n" + + "uniform sampler2D uTexSampler;\n" + + "uniform sampler2D uBgSampler;\n" + + "uniform vec2 uKeyCbCr;\n" + + "uniform vec2 uKeyDir;\n" + + "uniform float uSimilarity;\n" + + "uniform float uSmoothness;\n" + + "uniform float uSpill;\n" + + "uniform vec3 uBgColor;\n" + + "uniform int uBgMode;\n" + + "varying vec2 vTexSamplingCoord;\n" + + "void main() {\n" + + " vec4 src = texture2D(uTexSampler, vTexSamplingCoord);\n" + + " vec3 rgb = src.rgb;\n" + + // BT.601 luma and chroma, matching ChromaKeyMath exactly. + " float y = dot(rgb, vec3(0.299, 0.587, 0.114));\n" + + " vec2 cbcr = vec2(\n" + + " dot(rgb, vec3(-0.168736, -0.331264, 0.5)),\n" + + " dot(rgb, vec3(0.5, -0.418688, -0.081312)));\n" + + // Matte: distance in the chroma plane, ramped by smoothstep. + " float d = distance(cbcr, uKeyCbCr);\n" + + " float a = smoothstep(uSimilarity," + + " uSimilarity + max(uSmoothness, 1e-4), d);\n" + + // Spill: remove the chroma component pointing at the key hue, + // keeping y so a despilled pixel never darkens. Only pixels + // leaning toward the key (projection > 0) are touched. + " float projection = dot(cbcr, uKeyDir);\n" + + " if (uSpill > 0.0 && projection > 0.0) {\n" + + " vec2 c = cbcr - uKeyDir * projection * uSpill;\n" + + " rgb = clamp(vec3(\n" + + " y + 1.402 * c.y,\n" + + " y - 0.344136 * c.x - 0.714136 * c.y,\n" + + " y + 1.772 * c.x), 0.0, 1.0);\n" + + " }\n" + + // Fold in the frame's own alpha, so an already-transparent + // pixel (a gap frame carrying AlphaScale(0)) stays transparent. + " float outA = src.a * a;\n" + + " if (uBgMode == 2) {\n" + + // The background comes from a Bitmap, and GLUtils.texImage2D + // uploads its first row at t=0, while a frame texture puts t=0 + // at the *bottom* of the frame. Sampling the bitmap with the + // frame's own coordinate would show it upside down, so flip t + // here โ€” the same correction Media3's BitmapOverlay applies via + // its flipVerticallyMatrix. + " vec2 bgCoord = vec2(vTexSamplingCoord.x," + + " 1.0 - vTexSamplingCoord.y);\n" + + " vec3 bg = texture2D(uBgSampler, bgCoord).rgb;\n" + + " gl_FragColor = vec4(mix(bg, rgb, outA), src.a);\n" + + " } else if (uBgMode == 1) {\n" + + " gl_FragColor = vec4(mix(uBgColor, rgb, outA), src.a);\n" + + " } else {\n" + + " gl_FragColor = vec4(rgb, outA);\n" + + " }\n" + + "}" + } + + init { + if (backgroundBounds == null && config.backgroundImageData != null) { + Log.w( + RENDER_TAG, + "Chroma key: the background image could not be decoded; " + + "filling the keyed area with opaque black instead" + ) + } + try { + glProgram = GlProgram(VERTEX_SHADER_SOURCE, FRAGMENT_SHADER_SOURCE) + } catch (e: Exception) { + throw VideoFrameProcessingException(e) + } + } + + /** Pass-through: keying does not change the frame geometry. */ + override fun configure(inputWidth: Int, inputHeight: Int): Size { + return Size(inputWidth, inputHeight) + } + + override fun drawFrame(inputTexId: Int, presentationTimeUs: Long) { + try { + glProgram.use() + + if (backgroundTexId == -1) { + // createTexture uploads the pixels, so nothing needs the + // bitmap afterwards โ€” including the placeholder, which used + // to be allocated and then dropped on the floor. + val bitmap = decodeBackgroundWithinTextureLimit() + ?: onePixelPlaceholder() + backgroundTexId = GlUtil.createTexture(bitmap) + bitmap.recycle() + } + + glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, 0) + glProgram.setSamplerTexIdUniform("uBgSampler", backgroundTexId, 1) + + glProgram.setFloatsUniform( + "uKeyCbCr", + floatArrayOf(config.keyCb.toFloat(), config.keyCr.toFloat()) + ) + glProgram.setFloatsUniform( + "uKeyDir", + floatArrayOf(config.keyDirCb.toFloat(), config.keyDirCr.toFloat()) + ) + glProgram.setFloatUniform("uSimilarity", config.similarity.toFloat()) + glProgram.setFloatUniform("uSmoothness", config.smoothness.toFloat()) + glProgram.setFloatUniform("uSpill", config.spill.toFloat()) + glProgram.setFloatsUniform("uBgColor", backgroundColorRgb()) + glProgram.setIntUniform("uBgMode", bgMode) + + val vertexData = GlUtil.getNormalizedCoordinateBounds() + glProgram.setBufferAttribute( + "aFramePosition", vertexData, if (vertexData.size == 8) 2 else 4 + ) + val texData = GlUtil.getTextureCoordinateBounds() + glProgram.setBufferAttribute( + "aTexSamplingCoord", texData, if (texData.size == 8) 2 else 4 + ) + + glProgram.bindAttributesAndUniforms() + + // No glClear and no blending: BaseGlShaderProgram already + // cleared the target to (0,0,0,0) and this draw covers it + // entirely, so the fragment output must simply replace it. + // Blending here would blend against transparent black and + // square the alpha. The one real blend is Media3's own + // compositor, which uses glBlendFuncSeparate. + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4) + GlUtil.checkGlError() + } catch (e: Exception) { + throw VideoFrameProcessingException(e, presentationTimeUs) + } + } + + override fun release() { + super.release() + try { + if (backgroundTexId != -1) { + GlUtil.deleteTexture(backgroundTexId) + backgroundTexId = -1 + } + } catch (e: Exception) { + throw VideoFrameProcessingException(e) + } finally { + // Deleting the texture can throw, and the program still has to + // go โ€” otherwise a failure on the way out leaks it. + try { + glProgram.delete() + } catch (e: Exception) { + throw VideoFrameProcessingException(e) + } + } + } + + /** + * Decodes the background image, downscaled to fit `GL_MAX_TEXTURE_SIZE`. + * + * A camera photo easily exceeds the 4096 (or 2048) px limit a mid-range + * GPU reports, and `GlUtil.createTexture` throws above it โ€” which inside + * [drawFrame] kills the whole export on its first frame. The shader + * stretches the texture to the frame anyway, so the extra resolution was + * never going to survive; dropping it here costs nothing. + * + * Must be called on the GL thread. + */ + private fun decodeBackgroundWithinTextureLimit(): Bitmap? { + val bytes = config.backgroundImageData ?: return null + val bounds = backgroundBounds ?: return null + + val limit = maxTextureSize() + var sampleSize = 1 + while (bounds.outWidth / sampleSize > limit || + bounds.outHeight / sampleSize > limit + ) { + sampleSize *= 2 + } + + val decoded = BitmapFactory.decodeByteArray( + bytes, 0, bytes.size, + BitmapFactory.Options().apply { inSampleSize = sampleSize } + ) ?: return null + + // inSampleSize only halves, so one more exact pass may be needed. + if (decoded.width <= limit && decoded.height <= limit) return decoded + val scale = minOf( + limit.toFloat() / decoded.width, + limit.toFloat() / decoded.height + ) + val scaled = Bitmap.createScaledBitmap( + decoded, + (decoded.width * scale).toInt().coerceAtLeast(1), + (decoded.height * scale).toInt().coerceAtLeast(1), + /* filter= */ true + ) + if (scaled !== decoded) decoded.recycle() + return scaled + } + + /** The GPU's maximum texture dimension, or a conservative fallback. */ + private fun maxTextureSize(): Int { + val value = IntArray(1) + GLES20.glGetIntegerv(GLES20.GL_MAX_TEXTURE_SIZE, value, 0) + return if (value[0] > 0) value[0] else 2048 + } + + /** The background color as linear-free RGB in 0..1, or black. */ + private fun backgroundColorRgb(): FloatArray { + val argb = config.backgroundColor ?: return floatArrayOf(0f, 0f, 0f) + return floatArrayOf( + ((argb shr 16) and 0xFF) / 255f, + ((argb shr 8) and 0xFF) / 255f, + (argb and 0xFF) / 255f + ) + } + + /** + * A 1x1 transparent texture, bound to `uBgSampler` when there is no + * background image so the sampler uniform is never left unbound. + */ + private fun onePixelPlaceholder(): Bitmap = + Bitmap.createBitmap(1, 1, Bitmap.Config.ARGB_8888).apply { + eraseColor(android.graphics.Color.TRANSPARENT) + } + } +} diff --git a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ChromaKeyMath.kt b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ChromaKeyMath.kt new file mode 100644 index 00000000..8695eba8 --- /dev/null +++ b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ChromaKeyMath.kt @@ -0,0 +1,90 @@ +package ch.waio.pro_video_editor.src.features.render.helpers + +import ch.waio.pro_video_editor.src.features.render.models.ChromaKeyConfig +import kotlin.math.max +import kotlin.math.min +import kotlin.math.sqrt + +/** + * The chroma-key formula, in plain Kotlin. + * + * This is the single spec both platforms implement. The GPU actually runs it + * inside [ChromaKeyEffect]'s fragment shader; Apple bakes the very same math + * into a Core Image color cube (`ApplyChromaKey.swift`). This class exists so + * the formula can be unit-tested without a GL context, and so the Kotlin and + * Swift tests can assert the *same* golden table โ€” a drift in either + * implementation then fails a fast test instead of an emulator run. + * + * Keep this file, the fragment shader and `chromaKeyed(r:g:b:_:)` in Swift in + * lockstep. All values are gamma-encoded, non-color-managed RGB in `0..1`. + */ +object ChromaKeyMath { + + /** BT.601 luma of a gamma-encoded RGB triple. */ + fun luma(r: Double, g: Double, b: Double): Double = + 0.299 * r + 0.587 * g + 0.114 * b + + /** + * BT.601 chroma projection of a gamma-encoded RGB triple, as `(Cb, Cr)`. + * + * The keyer measures distance in this plane. Note that this is a position, + * not a pure hue: Cb and Cr scale with brightness, so a dimly lit patch of + * the screen sits closer to neutral and further from the key point. The + * default `similarity` of 0.20 covers roughly 40%..100% of the screen's + * reference brightness. Same behaviour as FFmpeg's `chromakey` and OBS. + */ + fun chroma(r: Double, g: Double, b: Double): DoubleArray = doubleArrayOf( + -0.168736 * r - 0.331264 * g + 0.5 * b, + 0.5 * r - 0.418688 * g - 0.081312 * b, + ) + + /** Hermite smoothstep, matching GLSL's `smoothstep`. */ + fun smoothstep(edge0: Double, edge1: Double, x: Double): Double { + if (edge1 <= edge0) return if (x < edge0) 0.0 else 1.0 + val t = ((x - edge0) / (edge1 - edge0)).coerceIn(0.0, 1.0) + return t * t * (3 - 2 * t) + } + + /** + * Evaluates the key for one pixel. + * + * @return `[r, g, b, alpha]` with **straight** (not premultiplied) alpha, + * which is the convention Media3 uses end to end. + */ + fun evaluate(r: Double, g: Double, b: Double, config: ChromaKeyConfig): DoubleArray { + val c = chroma(r, g, b) + val dCb = c[0] - config.keyCb + val dCr = c[1] - config.keyCr + val distance = sqrt(dCb * dCb + dCr * dCr) + + // max() keeps a zero-width ramp from dividing by zero, matching the + // shader's `max(uSmoothness, 1e-4)`. + val alpha = smoothstep( + config.similarity, + config.similarity + max(config.smoothness, 1e-4), + distance, + ) + + // Spill suppression. `projection` is how far the pixel leans toward the + // key hue; only pixels leaning toward it (> 0) are touched, so a + // complementary color is never desaturated. Y stays untouched, so a + // despilled pixel never darkens. + val projection = c[0] * config.keyDirCb + c[1] * config.keyDirCr + if (config.spill <= 0.0 || projection <= 0.0) { + return doubleArrayOf(r, g, b, alpha) + } + + val y = luma(r, g, b) + val cb = c[0] - config.keyDirCb * projection * config.spill + val cr = c[1] - config.keyDirCr * projection * config.spill + + return doubleArrayOf( + clamp01(y + 1.402 * cr), + clamp01(y - 0.344136 * cb - 0.714136 * cr), + clamp01(y + 1.772 * cb), + alpha, + ) + } + + private fun clamp01(v: Double): Double = min(max(v, 0.0), 1.0) +} diff --git a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/CompositionBuilder.kt b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/CompositionBuilder.kt index 08d0b006..a563e8d4 100644 --- a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/CompositionBuilder.kt +++ b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/CompositionBuilder.kt @@ -77,6 +77,7 @@ class CompositionBuilder( .setRotation(rotationDegrees) .setFlip(config.flipX, config.flipY) .setScale(config.scaleX, config.scaleY) + .setChromaKey(config.chromaKey) .setOutputResolution(config.outputWidth, config.outputHeight) .setCrop(config.cropWidth, config.cropHeight, config.cropX, config.cropY) .setTimedImageLayers(config.imageLayers.map { imageLayer -> diff --git a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/LayeredCompositionBuilder.kt b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/LayeredCompositionBuilder.kt index 17e1670a..368f2963 100644 --- a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/LayeredCompositionBuilder.kt +++ b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/LayeredCompositionBuilder.kt @@ -1,6 +1,7 @@ package ch.waio.pro_video_editor.src.features.render.helpers import RENDER_TAG +import applyChromaKey import android.content.Context import android.graphics.Bitmap import android.graphics.Color @@ -17,6 +18,7 @@ import androidx.media3.transformer.EditedMediaItem import androidx.media3.transformer.EditedMediaItemSequence import androidx.media3.transformer.Effects import ch.waio.pro_video_editor.src.features.render.models.AudioTrackConfig +import ch.waio.pro_video_editor.src.features.render.models.ChromaKeyConfig import ch.waio.pro_video_editor.src.features.render.models.CompositionConfig import ch.waio.pro_video_editor.src.features.render.models.SegmentTransformConfig import ch.waio.pro_video_editor.src.features.render.models.VideoClip @@ -55,7 +57,12 @@ class LayeredCompositionBuilder( /** Global trim start across the whole composition, in microseconds. */ private val globalStartUs: Long? = null, /** Global trim end across the whole composition, in microseconds. */ - private val globalEndUs: Long? = null + private val globalEndUs: Long? = null, + /** + * Chroma key for clips that carry neither their own nor a layer key. + * Resolved per clip as `clip ?: layer ?: global`; never merged. + */ + private val globalChromaKey: ChromaKeyConfig? = null ) { /** Temp files (source duplicates) to delete after export. */ val temporaryFiles: MutableList = mutableListOf() @@ -196,7 +203,8 @@ class LayeredCompositionBuilder( seqBuilder.addItem( buildClipItem( clip, srcStartUs, srcEndUs, effectivePath, placement, - displayW, displayH, layer.opacity, canvasW, canvasH + displayW, displayH, layer.opacity, canvasW, canvasH, + layer.chromaKey ) ) outputCursorUs += outputDurationUs @@ -299,7 +307,8 @@ class LayeredCompositionBuilder( displayH: Int, opacity: Float, canvasW: Int, - canvasH: Int + canvasH: Int, + layerChromaKey: ChromaKeyConfig? ): EditedMediaItem { val mediaItemBuilder = MediaItem.Builder().setUri(Uri.fromFile(File(inputPath))) if (srcStartUs != null || srcEndUs != null) { @@ -310,6 +319,24 @@ class LayeredCompositionBuilder( } val effects = mutableListOf() + + // Chroma key first: each layer is keyed on its own source frame, before + // it is placed on the canvas. Keying the composed canvas afterwards + // would be meaningless, since it is already opaque. + // + // flattenTransparency stays off here โ€” this is the whole point of the + // layered path. Media3's compositor blends the sequences with + // glBlendFuncSeparate, so a transparent key really does let the layer + // below show through. + applyChromaKey( + effects, + if (clip.suppressChromaKey) { + null + } else { + clip.chromaKey ?: layerChromaKey ?: globalChromaKey + } + ) + val draw = placement.draw val clipBox = placement.clip effects += VideoCompositionTransformation( diff --git a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/VideoCompositionTransformation.kt b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/VideoCompositionTransformation.kt index 05385128..49b1536f 100644 --- a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/VideoCompositionTransformation.kt +++ b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/VideoCompositionTransformation.kt @@ -23,6 +23,10 @@ import kotlin.math.roundToInt * rest stays transparent, so the default Media3 compositor can alpha-blend the * layers on top of each other. * + * The draw itself does **not** blend โ€” see the note in `drawFrame`. Partial + * alpha (a chroma-keyed edge, an [androidx.media3.effect.AlphaScale]d layer) + * passes through unchanged and is blended once, by the Media3 compositor. + * * When a clip box ([clipX], [clipY], [clipWidth], [clipHeight]) is provided, the * draw is scissored to that box (canvas pixels, top-left origin). This clips * `cover` overflow so a scaled-up clip cannot bleed past its target rectangle @@ -99,9 +103,18 @@ class VideoCompositionTransformation( GLES20.glClearColor(0f, 0f, 0f, 0f) GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT) - // Enable alpha blending to support transparent layers/overlays. - GLES20.glEnable(GLES20.GL_BLEND) - GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA) + // No blending: this draw replaces. The target was just cleared + // to (0,0,0,0) and exactly one quad is drawn, so there is + // nothing to blend against โ€” and GL_SRC_ALPHA/ + // GL_ONE_MINUS_SRC_ALPHA against a transparent destination + // *squares* the alpha (dst.a = a*a + 0*(1-a)). That was + // harmless while this shader only ever saw opaque input, but a + // chroma-keyed layer arrives with a soft, partially transparent + // edge, which blending here would darken and thin out. + // + // The one real blend is Media3's own DefaultCompositorGlProgram, + // which uses glBlendFuncSeparate(SRC_ALPHA, ONE_MINUS_SRC_ALPHA, + // ONE, ONE_MINUS_SRC_ALPHA) โ€” correct straight-alpha source-over. val glMatrix = FloatArray(16) Matrix.setIdentityM(glMatrix, 0) @@ -170,7 +183,6 @@ class VideoCompositionTransformation( if (applyScissor) { GLES20.glDisable(GLES20.GL_SCISSOR_TEST) } - GLES20.glDisable(GLES20.GL_BLEND) GlUtil.checkGlError() } catch (e: Exception) { throw VideoFrameProcessingException(e, presentationTimeUs) diff --git a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/VideoSequenceBuilder.kt b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/VideoSequenceBuilder.kt index ba01cec2..a2dd8a6c 100644 --- a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/VideoSequenceBuilder.kt +++ b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/VideoSequenceBuilder.kt @@ -3,6 +3,7 @@ package ch.waio.pro_video_editor.src.features.render.helpers import RENDER_TAG import android.content.Context import android.net.Uri +import applyChromaKey import applyScale import androidx.media3.common.C import androidx.media3.common.Effect @@ -18,6 +19,7 @@ import androidx.media3.effect.SpeedChangeEffect import androidx.media3.transformer.EditedMediaItem import androidx.media3.transformer.EditedMediaItemSequence import androidx.media3.transformer.Effects +import ch.waio.pro_video_editor.src.features.render.models.ChromaKeyConfig import ch.waio.pro_video_editor.src.features.render.models.LayerAnimationConfig import ch.waio.pro_video_editor.src.features.render.models.VideoClip import ch.waio.pro_video_editor.src.features.render.utils.getRotatedVideoDimensions @@ -60,6 +62,7 @@ class VideoSequenceBuilder( private var scaleY: Float? = null private var outputWidth: Int? = null private var outputHeight: Int? = null + private var globalChromaKey: ChromaKeyConfig? = null private val rotatedDimensionsCache = mutableMapOf>() data class CropConfig( @@ -104,6 +107,16 @@ class VideoSequenceBuilder( return this } + /** + * Sets the chroma key applied to every clip that carries none of its own. + * + * A [VideoClip.chromaKey] overrides this per clip; the two are never merged. + */ + fun setChromaKey(chromaKey: ChromaKeyConfig?): VideoSequenceBuilder { + this.globalChromaKey = chromaKey + return this + } + /** * Sets the exact output canvas size. When set, each clip is scaled to fit * inside it (preserving aspect ratio), centered, and padded with black. @@ -595,6 +608,20 @@ class VideoSequenceBuilder( // Build video effects val clipVideoEffects = mutableListOf() + + // Chroma key first, so it sees the original decoded colors โ€” before + // rotation, flip, the color LUT and blur. A clip's own key wins over + // the global one; they are never merged. + // + // flattenTransparency is on because this is the single-track path: + // there is no layer underneath, so a key without a background is filled + // with opaque black instead (see applyChromaKey for why). + applyChromaKey( + clipVideoEffects, + if (clip.suppressChromaKey) null else clip.chromaKey ?: globalChromaKey, + flattenTransparency = true, + ) + clipVideoEffects.addAll(videoEffects) // Calculate video dimensions for image layer positioning diff --git a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/models/RenderConfig.kt b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/models/RenderConfig.kt index 9effc8d5..97cf3e35 100644 --- a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/models/RenderConfig.kt +++ b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/models/RenderConfig.kt @@ -61,7 +61,21 @@ data class VideoClip( /** Start position on the layer timeline in microseconds (composition only). */ val timelineStartUs: Long? = null, /** Placement within the composition canvas (composition only). */ - val transform: SegmentTransformConfig? = null + val transform: SegmentTransformConfig? = null, + /** + * Removes a solid-colored background from this clip. Overrides the layer's + * and the global key; null falls back to those. + */ + val chromaKey: ChromaKeyConfig? = null, + /** + * Opts this clip out of the layer/global key entirely. + * + * Internal, never parsed from the platform channel. `chromaKey = null` + * means "inherit", so it cannot express "deliberately unkeyed" โ€” which is + * exactly what a pre-rendered overlap blend needs when the two clips it was + * composed from carry different keys. See `RenderVideo.blendChromaKey`. + */ + val suppressChromaKey: Boolean = false ) { companion object { /** Parses a clip from a platform-channel map. */ @@ -70,6 +84,8 @@ data class VideoClip( val transitionRaw = clipMap["transition"] as? Map @Suppress("UNCHECKED_CAST") val transformRaw = clipMap["transform"] as? Map + @Suppress("UNCHECKED_CAST") + val chromaKeyRaw = clipMap["chromaKey"] as? Map return VideoClip( inputPath = clipMap["inputPath"] as String, startUs = (clipMap["startUs"] as? Number)?.toLong(), @@ -79,7 +95,8 @@ data class VideoClip( reverseVideo = clipMap["reverseVideo"] as? Boolean ?: false, transition = transitionRaw?.let { TransitionConfig.fromMap(it) }, timelineStartUs = (clipMap["timelineStartUs"] as? Number)?.toLong(), - transform = transformRaw?.let { SegmentTransformConfig.fromMap(it) } + transform = transformRaw?.let { SegmentTransformConfig.fromMap(it) }, + chromaKey = ChromaKeyConfig.fromMap(chromaKeyRaw) ) } } @@ -128,7 +145,12 @@ data class SegmentTransformConfig( data class LayerConfig( val clips: List, val opacity: Float, - val transform: SegmentTransformConfig? + val transform: SegmentTransformConfig?, + /** + * Default chroma key for the clips on this layer. A clip's own key wins; + * null falls back to the global key. + */ + val chromaKey: ChromaKeyConfig? = null ) { companion object { fun fromMap(map: Map): LayerConfig? { @@ -138,10 +160,13 @@ data class LayerConfig( if (clips.isEmpty()) return null @Suppress("UNCHECKED_CAST") val transformRaw = map["transform"] as? Map + @Suppress("UNCHECKED_CAST") + val chromaKeyRaw = map["chromaKey"] as? Map return LayerConfig( clips = clips, opacity = (map["opacity"] as? Number)?.toFloat() ?: 1.0f, - transform = transformRaw?.let { SegmentTransformConfig.fromMap(it) } + transform = transformRaw?.let { SegmentTransformConfig.fromMap(it) }, + chromaKey = ChromaKeyConfig.fromMap(chromaKeyRaw) ) } } @@ -205,6 +230,105 @@ data class ColorFilterConfig( } } +/** + * Removes a solid-colored background ("green screen"). + * + * Mirrors the Dart `ChromaKey` model and the Swift `ChromaKeyConfig`. The + * keying math lives in `ChromaKeyMath` and is the same formula Apple bakes into + * its color cube; the GPU runs it in `ChromaKeyEffect`'s fragment shader. + * + * @property keyR/keyG/keyB The screen color to remove, gamma-encoded, 0..1 + * @property similarity Chroma-plane radius within which a pixel is fully removed + * @property smoothness Width of the soft ramp just beyond [similarity] + * @property spill How strongly the key's color cast is pulled out of the rest + * @property backgroundColor Solid background ARGB, or null when none + * @property backgroundImageData Background image bytes, or null when none + */ +data class ChromaKeyConfig( + val keyR: Double, + val keyG: Double, + val keyB: Double, + val similarity: Double = 0.20, + val smoothness: Double = 0.08, + val spill: Double = 0.5, + val backgroundColor: Int? = null, + val backgroundImageData: ByteArray? = null, +) { + /** The key color projected onto the Cb/Cr chroma plane. */ + val keyCb: Double = -0.168736 * keyR - 0.331264 * keyG + 0.5 * keyB + val keyCr: Double = 0.5 * keyR - 0.418688 * keyG - 0.081312 * keyB + + /** + * Unit vector pointing from neutral toward the key hue, used to pull the + * key's cast back out during spill suppression. Zero for a neutral (gray) + * key color, which disables despill rather than dividing by zero. + */ + val keyDirCb: Double + val keyDirCr: Double + + init { + val length = kotlin.math.sqrt(keyCb * keyCb + keyCr * keyCr) + if (length > 1e-5) { + keyDirCb = keyCb / length + keyDirCr = keyCr / length + } else { + keyDirCb = 0.0 + keyDirCr = 0.0 + } + } + + /** Whether the keyed area is left transparent rather than filled. */ + val isTransparent: Boolean + get() = backgroundColor == null && backgroundImageData == null + + // Hand-written because of the ByteArray, like ImageLayer below. + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + other as ChromaKeyConfig + return keyR == other.keyR && + keyG == other.keyG && + keyB == other.keyB && + similarity == other.similarity && + smoothness == other.smoothness && + spill == other.spill && + backgroundColor == other.backgroundColor && + (backgroundImageData?.contentEquals(other.backgroundImageData) + ?: (other.backgroundImageData == null)) + } + + override fun hashCode(): Int { + var result = keyR.hashCode() + result = 31 * result + keyG.hashCode() + result = 31 * result + keyB.hashCode() + result = 31 * result + similarity.hashCode() + result = 31 * result + smoothness.hashCode() + result = 31 * result + spill.hashCode() + result = 31 * result + (backgroundColor?.hashCode() ?: 0) + result = 31 * result + (backgroundImageData?.contentHashCode() ?: 0) + return result + } + + companion object { + fun fromMap(map: Map?): ChromaKeyConfig? { + if (map == null) return null + val keyColor = (map["keyColor"] as? Number)?.toInt() ?: return null + val bgImage = (map["bgImageData"] as? ByteArray)?.takeIf { it.isNotEmpty() } + + return ChromaKeyConfig( + keyR = ((keyColor shr 16) and 0xFF) / 255.0, + keyG = ((keyColor shr 8) and 0xFF) / 255.0, + keyB = (keyColor and 0xFF) / 255.0, + similarity = (map["similarity"] as? Number)?.toDouble() ?: 0.20, + smoothness = (map["smoothness"] as? Number)?.toDouble() ?: 0.08, + spill = (map["spill"] as? Number)?.toDouble() ?: 0.5, + backgroundColor = (map["bgColor"] as? Number)?.toInt(), + backgroundImageData = bgImage, + ) + } + } +} + /** * Represents a custom audio track with timing and volume configuration. * @@ -360,6 +484,12 @@ data class RenderConfig( val colorFilters: List = emptyList(), val audioTracks: List = emptyList(), val blur: Double? = null, + /** + * Removes a solid-colored background ("green screen") from every clip that + * does not carry its own key. A [VideoClip.chromaKey] overrides this, and + * in the layered path a [LayerConfig.chromaKey] sits between the two. + */ + val chromaKey: ChromaKeyConfig? = null, /** Global start time in microseconds for trimming the final composition */ val startUs: Long? = null, /** Global end time in microseconds for trimming the final composition */ @@ -481,6 +611,9 @@ data class RenderConfig( colorFilters = colorFilters, audioTracks = audioTracks, blur = call.argument("blur")?.toDouble(), + chromaKey = ChromaKeyConfig.fromMap( + call.argument>("chromaKey") + ), startUs = call.argument("startUs")?.toLong(), endUs = call.argument("endUs")?.toLong(), shouldOptimizeForNetworkUse = call.argument("shouldOptimizeForNetworkUse") diff --git a/android/src/test/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ChromaKeyMathTest.kt b/android/src/test/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ChromaKeyMathTest.kt new file mode 100644 index 00000000..d8ae41a0 --- /dev/null +++ b/android/src/test/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ChromaKeyMathTest.kt @@ -0,0 +1,261 @@ +package ch.waio.pro_video_editor.src.features.render.helpers + +import ch.waio.pro_video_editor.src.features.render.models.ChromaKeyConfig +import kotlin.math.abs +import kotlin.math.hypot +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * Cross-platform parity guard for the chroma-key formula. + * + * The [golden] table below is duplicated verbatim in the Swift test + * (`example/macos/RunnerTests/RunnerTests.swift`, `ChromaKeyMathTests`). Both + * run it against their own implementation โ€” Kotlin's [ChromaKeyMath], which the + * GLSL shader mirrors, and Swift's `chromaKeyed(r:g:b:_:)`, which is baked into + * the Core Image color cube. If either platform drifts, one of these two tests + * fails immediately instead of the difference surfacing later as "the key looks + * slightly different on iOS". + * + * **When you change the formula, regenerate both tables.** + */ +internal class ChromaKeyMathTest { + + /** + * The config the golden table was computed for: SMPTE green with + * similarity 0.15. Pinned explicitly rather than taken from the defaults, + * so raising the library default never silently invalidates the table. + */ + private val config = ChromaKeyConfig( + keyR = 0x00 / 255.0, + keyG = 0xB1 / 255.0, + keyB = 0x40 / 255.0, + similarity = 0.15, + smoothness = 0.08, + spill = 0.5, + ) + + private val tolerance = 1e-4 + + private data class Golden( + val name: String, + val r: Double, + val g: Double, + val b: Double, + val outR: Double, + val outG: Double, + val outB: Double, + val alpha: Double, + ) + + private val golden = listOf( + // The key color itself and its neighbourhood: removed completely. + Golden("key color", 0.0, 0.694118, 0.25098, 0.218029, 0.565088, 0.343519, 0.0), + Golden("near key", 0.05, 0.72, 0.28, 0.261122, 0.595058, 0.369608, 0.0), + Golden("bright screen", 0.35, 0.9, 0.5, 0.525426, 0.796183, 0.574457, 0.0), + // Half-lit screen: past the default similarity, so only mostly removed. + // This is the documented brightness sensitivity, pinned on purpose. + Golden("dim screen 50%", 0.0, 0.347059, 0.12549, 0.109015, 0.282544, 0.17176, 0.081671), + // Neutrals sit at the chroma origin, far from any saturated key. + Golden("black", 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0), + Golden("mid gray", 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 1.0), + Golden("white", 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0), + // Skin tone โ€” the calibration anchor quoted in the Dart docs. + Golden("skin tone", 0.86, 0.65, 0.53, 0.86, 0.65, 0.53, 1.0), + Golden("pure red", 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0), + Golden("pure blue", 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0), + // Kept, but despilled: the green cast is pulled out at full alpha. + Golden( + "green-spilled gray", 0.55, 0.75, 0.55, + 0.616767, 0.710487, 0.578338, 1.0, + ), + // Leans away from the key hue, so despill leaves it alone. + Golden("magenta", 0.8, 0.2, 0.8, 0.8, 0.2, 0.8, 1.0), + ) + + @Test + fun goldenTable_matchesTheSharedFormula() { + for (row in golden) { + val out = ChromaKeyMath.evaluate(row.r, row.g, row.b, config) + val expected = doubleArrayOf(row.outR, row.outG, row.outB, row.alpha) + val labels = listOf("r", "g", "b", "alpha") + + for (i in 0..3) { + assertTrue( + abs(out[i] - expected[i]) < tolerance, + "${row.name}: ${labels[i]} was ${out[i]}, expected ${expected[i]}", + ) + } + } + } + + @Test + fun keyColor_isRemovedCompletely() { + val out = ChromaKeyMath.evaluate(config.keyR, config.keyG, config.keyB, config) + assertEquals(0.0, out[3], tolerance) + } + + @Test + fun softEdge_producesPartialAlphaBetweenTheThresholds() { + // Walk the key color toward neutral gray and collect the alpha ramp. + // Between "fully keyed" and "fully opaque" there must be intermediate + // values, or the edge would be a hard cutout. + val ramp = (0..60).map { step -> + val t = step / 60.0 + ChromaKeyMath.evaluate( + config.keyR + (0.5 - config.keyR) * t, + config.keyG + (0.5 - config.keyG) * t, + config.keyB + (0.5 - config.keyB) * t, + config, + )[3] + } + + assertTrue(ramp.any { it == 0.0 }, "expected a fully keyed sample") + assertTrue(ramp.any { it == 1.0 }, "expected a fully opaque sample") + assertTrue( + ramp.any { it > 0.01 && it < 0.99 }, + "expected a soft edge, but alpha jumped straight from 0 to 1", + ) + } + + @Test + fun alpha_isMonotonicInChromaDistance() { + // Alpha must never dip as a pixel moves further from the key, or the + // matte would show rings. + var previous = -1.0 + for (step in 0..100) { + val t = step / 100.0 + val alpha = ChromaKeyMath.evaluate( + config.keyR + (0.5 - config.keyR) * t, + config.keyG + (0.5 - config.keyG) * t, + config.keyB + (0.5 - config.keyB) * t, + config, + )[3] + assertTrue(alpha >= previous - tolerance, "alpha dipped at t=$t") + previous = alpha + } + } + + @Test + fun matte_isBrightnessSensitive_asDocumented() { + // Cb/Cr scale with brightness, so a dimly lit patch of the screen sits + // closer to the neutral origin and further from the key point. At the + // 0.15 used here that is roughly 55%..100% of the reference brightness; + // the library default of 0.20 reaches down to ~40%, which is what a + // real studio screen needs (its darkest corners measured 0.18). Pinned + // because the Dart docs promise exactly this behaviour. + fun screenAt(fraction: Double) = ChromaKeyMath.evaluate( + config.keyR * fraction, + config.keyG * fraction, + config.keyB * fraction, + config, + )[3] + + assertEquals(0.0, screenAt(1.0), tolerance) + assertEquals(0.0, screenAt(0.7), tolerance) + assertEquals(0.0, screenAt(0.55), tolerance) + assertTrue(screenAt(0.5) > 0.0, "a half-lit screen should start to survive") + + // Widening similarity brings the dim end back under the key. + val wide = config.copy(similarity = 0.35) + assertEquals( + 0.0, + ChromaKeyMath.evaluate( + config.keyR * 0.4, config.keyG * 0.4, config.keyB * 0.4, wide, + )[3], + tolerance, + ) + } + + @Test + fun spill_pullsTheKeyCastOutWithoutDarkening() { + // A green-tinted gray, the classic bounce off a screen. + val r = 0.55 + val g = 0.75 + val b = 0.55 + + val without = ChromaKeyMath.evaluate(r, g, b, config.copy(spill = 0.0)) + val with = ChromaKeyMath.evaluate(r, g, b, config.copy(spill = 1.0)) + + val castBefore = without[1] - maxOf(without[0], without[2]) + val castAfter = with[1] - maxOf(with[0], with[2]) + assertTrue( + castAfter < castBefore, + "despill did not reduce the green cast ($castBefore -> $castAfter)", + ) + + // Luma is preserved, so despill never darkens the subject. + assertEquals( + ChromaKeyMath.luma(without[0], without[1], without[2]), + ChromaKeyMath.luma(with[0], with[1], with[2]), + 1e-3, + ) + } + + @Test + fun spill_leavesTheComplementaryHueAlone() { + // Magenta leans away from green, so despill must not touch it. + val without = ChromaKeyMath.evaluate(0.8, 0.2, 0.8, config.copy(spill = 0.0)) + val with = ChromaKeyMath.evaluate(0.8, 0.2, 0.8, config.copy(spill = 1.0)) + + assertEquals(without[0], with[0], tolerance) + assertEquals(without[1], with[1], tolerance) + assertEquals(without[2], with[2], tolerance) + } + + @Test + fun spillOff_leavesColorsUntouched() { + val out = ChromaKeyMath.evaluate(0.55, 0.75, 0.55, config.copy(spill = 0.0)) + + assertEquals(0.55, out[0], tolerance) + assertEquals(0.75, out[1], tolerance) + assertEquals(0.55, out[2], tolerance) + } + + @Test + fun neutralKeyColor_disablesDespillInsteadOfDividingByZero() { + // A gray key has no hue to pull out; the direction vector collapses to + // zero rather than producing NaN. + val gray = ChromaKeyConfig(keyR = 0.5, keyG = 0.5, keyB = 0.5, spill = 1.0) + assertEquals(0.0, gray.keyDirCb, tolerance) + assertEquals(0.0, gray.keyDirCr, tolerance) + + val out = ChromaKeyMath.evaluate(0.8, 0.2, 0.4, gray) + assertTrue(out.none { it.isNaN() }, "despill produced NaN: ${out.toList()}") + } + + @Test + fun zeroSmoothness_doesNotDivideByZero() { + val hard = config.copy(smoothness = 0.0) + + assertEquals(0.0, ChromaKeyMath.evaluate(0.0, 0.694118, 0.25098, hard)[3], tolerance) + assertEquals(1.0, ChromaKeyMath.evaluate(1.0, 1.0, 1.0, hard)[3], tolerance) + } + + @Test + fun higherSimilarity_keysMore() { + // A color that survives the default key must fall to the wider one. + val narrow = ChromaKeyMath.evaluate(0.35, 0.6, 0.4, config)[3] + val wide = ChromaKeyMath.evaluate(0.35, 0.6, 0.4, config.copy(similarity = 0.5))[3] + + assertTrue(wide < narrow, "widening similarity did not key more ($narrow -> $wide)") + } + + @Test + fun skinTone_keepsAWideMarginFromTheKey() { + // The margin quoted in the Dart docs: skin sits ~0.43 from SMPTE green, + // nearly 3x the default similarity, which is why the default is safe. + val skin = ChromaKeyMath.chroma(0.86, 0.65, 0.53) + val distance = hypot(skin[0] - config.keyCb, skin[1] - config.keyCr) + + assertEquals(0.4259, distance, 1e-3) + assertTrue(distance > config.similarity * 2.5) + } + + @Test + fun derivedKeyChroma_isAUnitDirection() { + val length = hypot(config.keyDirCb, config.keyDirCr) + assertEquals(1.0, length, tolerance) + } +} diff --git a/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/RenderVideo.swift b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/RenderVideo.swift index 2b91f47a..5671a9ae 100644 --- a/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/RenderVideo.swift +++ b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/RenderVideo.swift @@ -118,7 +118,8 @@ class RenderVideo { volume: clip.volume, playbackSpeed: clip.playbackSpeed, reverseVideo: clip.reverseVideo, - transition: clip.transition + transition: clip.transition, + chromaKey: clip.chromaKey ) } return clip @@ -165,7 +166,8 @@ class RenderVideo { let (newClips, urls) = await preRenderTransitions( clips: workingConfig.videoClips, enableAudio: workingConfig.enableAudio, - outputFormat: workingConfig.outputFormat) + outputFormat: workingConfig.outputFormat, + globalChromaKey: workingConfig.chromaKey) workingConfig = workingConfig.copyWith(videoClips: newClips) transitionURLs = urls } @@ -193,12 +195,13 @@ class RenderVideo { let buildResult: ( AVMutableComposition, VideoCompositionData, CGSize, AVAudioMix?, CMPersistentTrackID, - [URL], [FadeWindow] + [URL], [FadeWindow], [ChromaKeyWindow] ) if let compositionConfig = workingConfig.composition { buildResult = try await LayeredCompositionBuilder(composition: compositionConfig) .setEnableAudio(workingConfig.enableAudio) .setAudioTracks(workingConfig.audioTracks) + .setChromaKey(workingConfig.chromaKey) .build() } else { buildResult = try await applyComposition( @@ -206,12 +209,13 @@ class RenderVideo { videoEffects: effectsConfig, enableAudio: workingConfig.enableAudio, audioTracks: workingConfig.audioTracks, - trimToCommonTrackEnd: workingConfig.trimToCommonTrackEnd + trimToCommonTrackEnd: workingConfig.trimToCommonTrackEnd, + chromaKey: workingConfig.chromaKey ) } let ( composition, videoCompData, renderSize, audioMix, sourceTrackID, audioTempURLs, - fadeWindows + fadeWindows, chromaKeyWindows ) = buildResult temporaryAudioURLs = audioTempURLs var videoCompConfig = videoCompData @@ -265,6 +269,11 @@ class RenderVideo { applyScale( config: &effectsConfig, scaleX: workingConfig.scaleX, scaleY: workingConfig.scaleY) + // Before applyColorMatrix, because the key must run on the original + // colors โ€” the compositor folds both into one color cube, since a + // second CIColorCube would take its alpha from its own cube and + // silently un-key the frame. + applyChromaKey(config: &effectsConfig, windows: chromaKeyWindows) applyColorMatrix( config: &effectsConfig, filters: workingConfig.colorFilters) @@ -430,13 +439,6 @@ class RenderVideo { // MARK: - Helper Methods - private static func hasGpuIntensiveEffects(_ config: RenderConfig) -> Bool { - let hasImageOverlay = !config.imageLayers.isEmpty - let hasBlur = config.blur != nil && config.blur! > 0 - let hasColorFilter = !config.colorFilters.isEmpty - return hasImageOverlay || hasBlur || hasColorFilter - } - private static func makeVideoCompositorSubclass(with config: VideoCompositorConfig) -> AVVideoCompositing.Type { @@ -585,7 +587,8 @@ class RenderVideo { /// (e.g. a neighbour is reversed or has too little content) โ€” those cases are /// handled live by the main pipeline. private static func preRenderTransitions( - clips: [VideoClip], enableAudio: Bool, outputFormat: String + clips: [VideoClip], enableAudio: Bool, outputFormat: String, + globalChromaKey: ChromaKeyConfig? ) async -> ([VideoClip], [URL]) { // An overlap transition on the last/only clip loops back into the first clip // (seamless loop). Captured before the between-clip pass clears it. @@ -682,15 +685,19 @@ class RenderVideo { VideoClip( inputPath: current.inputPath, startUs: curStart, endUs: curEnd - tailSrc, volume: current.volume, playbackSpeed: current.playbackSpeed, - reverseVideo: false, transition: nil)) + reverseVideo: false, transition: nil, chromaKey: current.chromaKey)) + let blend = blendChromaKey( + current, next!, global: globalChromaKey, boundary: "\(i)") result.append( VideoClip( - inputPath: rendered.outputURL.path, startUs: 0, endUs: rendered.durationUs)) + inputPath: rendered.outputURL.path, startUs: 0, endUs: rendered.durationUs, + chromaKey: blend.config, suppressChromaKey: blend.suppressed)) // Trim the incoming head in place; it keeps its own speed/transition. work[i + 1] = VideoClip( inputPath: next!.inputPath, startUs: nextStart + headSrc, endUs: nextEnd, volume: next!.volume, playbackSpeed: next!.playbackSpeed, - reverseVideo: next!.reverseVideo, transition: next!.transition) + reverseVideo: next!.reverseVideo, transition: next!.transition, + chromaKey: next!.chromaKey) } else { PluginLog.print("โš ๏ธ Transition render failed at boundary \(i); hard cut") appendClip(clearedOverlap(current)) @@ -763,21 +770,25 @@ class RenderVideo { result[0] = VideoClip( inputPath: first.inputPath, startUs: firstStart + headSrc, endUs: lastEnd - tailSrc, volume: first.volume, - playbackSpeed: first.playbackSpeed, reverseVideo: false, transition: nil) + playbackSpeed: first.playbackSpeed, reverseVideo: false, transition: nil, + chromaKey: first.chromaKey) } else { result[0] = VideoClip( inputPath: first.inputPath, startUs: firstStart + headSrc, endUs: first.endUs, volume: first.volume, playbackSpeed: first.playbackSpeed, reverseVideo: false, - transition: first.transition) + transition: first.transition, chromaKey: first.chromaKey) result[lastIdx] = VideoClip( inputPath: last.inputPath, startUs: last.startUs, endUs: lastEnd - tailSrc, volume: last.volume, playbackSpeed: last.playbackSpeed, - reverseVideo: false, transition: nil) + reverseVideo: false, transition: nil, chromaKey: last.chromaKey) } + let wrapBlend = blendChromaKey( + last, first, global: globalChromaKey, boundary: "loop wrap") result.append( VideoClip( - inputPath: rendered.outputURL.path, startUs: 0, endUs: rendered.durationUs)) + inputPath: rendered.outputURL.path, startUs: 0, endUs: rendered.durationUs, + chromaKey: wrapBlend.config, suppressChromaKey: wrapBlend.suppressed)) } else { PluginLog.print("โš ๏ธ Loop wrap render failed; seamless loop skipped") } @@ -796,7 +807,47 @@ class RenderVideo { return VideoClip( inputPath: clip.inputPath, startUs: clip.startUs, endUs: clip.endUs, volume: clip.volume, playbackSpeed: clip.playbackSpeed, - reverseVideo: clip.reverseVideo, transition: nil) + reverseVideo: clip.reverseVideo, transition: nil, chromaKey: clip.chromaKey) + } + + /// The chroma key to apply to a pre-rendered overlap blend. + /// + /// The blend is composed from the raw sources by `ClipTransitionRenderer`, + /// which knows nothing about keying, so the key has to be re-applied to its + /// output. That only has a defined meaning when both sides key the same way: + /// blending a keyed clip with an unkeyed one produces mixed colors that no + /// single key can undo. Mismatched sides are therefore left unkeyed and + /// reported, rather than silently keyed with one side's settings. + private static func blendChromaKey( + _ outgoing: VideoClip, _ incoming: VideoClip, global: ChromaKeyConfig?, + boundary: String + ) -> BlendChromaKey { + // Compare the *effective* keys. A clip that leaves its own key nil still + // inherits the global one, so comparing the raw per-clip fields reported a + // mismatch for two clips that in fact key identically. + let outgoingKey = outgoing.chromaKey ?? global + let incomingKey = incoming.chromaKey ?? global + if outgoingKey == incomingKey { return BlendChromaKey(outgoingKey, false) } + + PluginLog.print( + "โš ๏ธ Chroma key: the two clips at boundary \(boundary) use different keys; " + + "the pre-rendered transition blend is emitted unkeyed") + // A nil key alone would mean "inherit", and `computeChromaKeyWindows` would + // then fall back to the global key โ€” applying to the blend the very thing + // this warning promises it will not. The suppress flag is what makes + // "unkeyed" actually mean unkeyed. + return BlendChromaKey(nil, true) + } + + /// The key for a pre-rendered blend, and whether it opts out of the global one. + private struct BlendChromaKey { + let config: ChromaKeyConfig? + let suppressed: Bool + + init(_ config: ChromaKeyConfig?, _ suppressed: Bool) { + self.config = config + self.suppressed = suppressed + } } /// Loads a clip's total duration in microseconds. diff --git a/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/ApplyChromaKey.swift b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/ApplyChromaKey.swift new file mode 100644 index 00000000..9145d306 --- /dev/null +++ b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/ApplyChromaKey.swift @@ -0,0 +1,175 @@ +import CoreImage +import Foundation + +// MARK: - The shared keying formula +// +// This is the single spec both platforms implement. Android runs it per pixel +// in `ChromaKeyEffect.kt`'s fragment shader; Apple bakes it into a color cube +// (see `generateChromaLUTData`) because Core Image has no custom kernel here. +// The Kotlin and Swift unit tests assert the same golden table against both, so +// a drift in either implementation fails fast. +// +// All values are gamma-encoded, non-color-managed RGB in 0...1 โ€” the compositor +// runs with `workingColorSpace: NSNull()`, so these are exactly the numbers the +// decoder produced, which is what keeps the two platforms in step. +// +// Y = 0.299ยทr + 0.587ยทg + 0.114ยทb +// Cb = -0.168736ยทr - 0.331264ยทg + 0.5ยทb +// Cr = 0.5ยทr - 0.418688ยทg - 0.081312ยทb +// +// d = distance((Cb, Cr), (Cb_key, Cr_key)) +// alpha = smoothstep(similarity, similarity + smoothness, d) +// +// The distance lives in the Cb/Cr chroma plane, which is a position rather than +// a pure hue: Cb and Cr scale with brightness, so a dimly lit patch of the +// screen sits closer to neutral and further from the key point. The default +// `similarity` of 0.20 covers roughly 40%-100% of the screen's reference +// brightness. Same behaviour as FFmpeg's `chromakey` and OBS. +// +// Spill suppression removes the chroma component pointing toward the key hue, +// leaving Y untouched so a despilled pixel never darkens. + +/// BT.601 chroma projection of a gamma-encoded RGB triple. +func chromaOf(r: Double, g: Double, b: Double) -> (cb: Double, cr: Double) { + return ( + cb: -0.168736 * r - 0.331264 * g + 0.5 * b, + cr: 0.5 * r - 0.418688 * g - 0.081312 * b + ) +} + +/// BT.601 luma of a gamma-encoded RGB triple. +func lumaOf(r: Double, g: Double, b: Double) -> Double { + return 0.299 * r + 0.587 * g + 0.114 * b +} + +/// Hermite smoothstep, matching GLSL's `smoothstep`. +private func smoothstep(_ edge0: Double, _ edge1: Double, _ x: Double) -> Double { + guard edge1 > edge0 else { return x < edge0 ? 0 : 1 } + let t = min(max((x - edge0) / (edge1 - edge0), 0), 1) + return t * t * (3 - 2 * t) +} + +/// Evaluates the chroma key for one pixel. +/// +/// Returns the despilled color together with the matte alpha, both **straight** +/// (not premultiplied) โ€” the cube builder premultiplies afterwards, since that +/// is what `CIColorCube` requires. +func chromaKeyed(r: Double, g: Double, b: Double, _ config: ChromaKeyConfig) + -> (r: Double, g: Double, b: Double, a: Double) +{ + let chroma = chromaOf(r: r, g: g, b: b) + let key = config.keyChroma + + let dCb = chroma.cb - key.cb + let dCr = chroma.cr - key.cr + let distance = (dCb * dCb + dCr * dCr).squareRoot() + + // max() keeps a zero-width ramp from dividing by zero; smoothstep already + // guards it, but this also matches the shader's `max(uSmoothness, 1e-4)`. + let alpha = smoothstep( + config.similarity, + config.similarity + max(config.smoothness, 1e-4), + distance + ) + + // Spill suppression. `projection` is how far the pixel leans toward the key + // hue; only pixels leaning toward it (> 0) are touched, so a complementary + // color is never desaturated. + let direction = config.keyDirection + let projection = chroma.cb * direction.cb + chroma.cr * direction.cr + guard config.spill > 0, projection > 0 else { + return (r, g, b, alpha) + } + + let y = lumaOf(r: r, g: g, b: b) + let cb = chroma.cb - direction.cb * projection * config.spill + let cr = chroma.cr - direction.cr * projection * config.spill + + return ( + r: min(max(y + 1.402 * cr, 0), 1), + g: min(max(y - 0.344136 * cb - 0.714136 * cr, 0), 1), + b: min(max(y + 1.772 * cb, 0), 1), + a: alpha + ) +} + +// MARK: - Compositor wiring + +/// A chroma-key window in composition time. +/// +/// The single-track path resolves the key per clip, so each clip contributes +/// one window covering its own span of the timeline โ€” the same shape as the +/// existing `FadeWindow`. +public struct ChromaKeyWindow: Sendable { + /// Inclusive start of the window in composition microseconds. + let startUs: Int64 + /// Exclusive end of the window in composition microseconds. + let endUs: Int64 + let config: ChromaKeyConfig +} + +/// Hands the per-clip chroma-key windows to the custom video compositor. +/// +/// - Parameters: +/// - config: Video compositor configuration to modify. +/// - windows: One window per clip that carries a key; empty disables keying. +/// +/// - Note: The actual keying happens per frame in the compositor, which builds +/// a color cube from the key and composites its background. +func applyChromaKey( + config: inout VideoCompositorConfig, + windows: [ChromaKeyWindow] +) { + guard !windows.isEmpty else { return } + + config.chromaKeyWindows = windows + + PluginLog.print( + "[\(Tags.render)] ๐ŸŸฉ Applying chroma key: \(windows.count) window(s)" + ) +} + +// MARK: - Color cube + +/// Builds a premultiplied RGBA color cube for the chroma key. +/// +/// The whole key fits in a cube because both halves โ€” the matte alpha and the +/// despilled color โ€” are pure functions of the input RGB. This is what lets +/// Apple key without a custom `CIKernel` or any Metal. +/// +/// Entries are **premultiplied**, which `CIColorCube` requires: a fully removed +/// pixel is `(0, 0, 0, 0)`, not `(rgb, 0)`, or Core Image would composite the +/// screen color back in along the soft edge. Android's shader writes straight +/// alpha instead, per Media3's convention โ€” do not align the two. +/// +/// - Parameters: +/// - chroma: The key to bake in. +/// - size: Cube dimension per axis. `CIColorCube` supports up to 64. +func generateChromaLUTData( + chroma: ChromaKeyConfig, + size: Int +) -> Data? { + let floatCount = size * size * size * 4 + var cubeData = [Float](repeating: 0, count: floatCount) + + var offset = 0 + for b in 0...size) +} diff --git a/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/ApplyComposition.swift b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/ApplyComposition.swift index 064d5918..42183788 100644 --- a/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/ApplyComposition.swift +++ b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/ApplyComposition.swift @@ -12,6 +12,7 @@ import Foundation /// - videoEffects: Configuration for visual effects (rotation, scale, color, blur, etc.). /// - enableAudio: If true, includes original audio from video clips. /// - audioTracks: Array of audio track configurations to mix over the video. +/// - chromaKey: Global chroma key for clips that carry none of their own. /// /// - Returns: A tuple containing: /// - AVMutableComposition: The concatenated video/audio composition @@ -21,6 +22,7 @@ import Foundation /// - CMPersistentTrackID: The track ID of the video composition track (for fallback on older iOS) /// - [URL]: Temporary file URLs (e.g. pre-rendered audio WAVs) the caller MUST delete after export /// - [FadeWindow]: Dip-to-color windows for fadeToBlack / fadeToWhite transitions +/// - [ChromaKeyWindow]: Per-clip chroma-key windows for the single-track path /// /// - Throws: NSError if video clips are empty, files don't exist, or tracks can't be loaded. func applyComposition( @@ -28,14 +30,16 @@ func applyComposition( videoEffects: VideoCompositorConfig, enableAudio: Bool, audioTracks: [AudioTrackConfig], - trimToCommonTrackEnd: Bool = false + trimToCommonTrackEnd: Bool = false, + chromaKey: ChromaKeyConfig? = nil ) async throws -> ( AVMutableComposition, VideoCompositionData, CGSize, AVAudioMix?, CMPersistentTrackID, [URL], - [FadeWindow] + [FadeWindow], [ChromaKeyWindow] ) { return try await CompositionBuilder(videoClips: videoClips, videoEffects: videoEffects) .setEnableAudio(enableAudio) .setTrimToCommonTrackEnd(trimToCommonTrackEnd) .setAudioTracks(audioTracks) + .setChromaKey(chromaKey) .build() } diff --git a/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/BitrateCapPolicy.swift b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/BitrateCapPolicy.swift index 78822e05..6d8e29c1 100644 --- a/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/BitrateCapPolicy.swift +++ b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/BitrateCapPolicy.swift @@ -62,6 +62,7 @@ internal enum BitrateCapPolicy { config.maxFrameRate == nil, config.playbackSpeed == nil || config.playbackSpeed == 1.0, config.blur == nil || config.blur == 0, + config.chromaKey == nil, config.startUs == nil, config.endUs == nil else { return false } @@ -71,5 +72,6 @@ internal enum BitrateCapPolicy { && (clip.volume == nil || clip.volume == 1.0) && (clip.playbackSpeed == nil || clip.playbackSpeed == 1.0) && !clip.reverseVideo + && clip.chromaKey == nil } } diff --git a/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/CompositionBuilder.swift b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/CompositionBuilder.swift index f7cb897a..84e71b88 100644 --- a/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/CompositionBuilder.swift +++ b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/CompositionBuilder.swift @@ -14,6 +14,7 @@ internal class CompositionBuilder { private var enableAudio: Bool = true private var trimToCommonTrackEnd: Bool = false private var audioTracks: [AudioTrackConfig] = [] + private var globalChromaKey: ChromaKeyConfig? /// Initializes builder with configuration. /// @@ -53,13 +54,22 @@ internal class CompositionBuilder { return self } + /// Sets the chroma key applied to every clip that carries none of its own. + /// + /// - Parameter key: The global key, or nil for none + /// - Returns: Self for chaining + func setChromaKey(_ key: ChromaKeyConfig?) -> CompositionBuilder { + self.globalChromaKey = key + return self + } + /// Builds the complete composition. /// /// - Returns: Tuple containing composition, video composition, render size, audio mix, source track ID, and temporary file URLs to clean up after export /// - Throws: Error if composition creation fails func build() async throws -> ( AVMutableComposition, VideoCompositionData, CGSize, AVAudioMix?, CMPersistentTrackID, [URL], - [FadeWindow] + [FadeWindow], [ChromaKeyWindow] ) { guard !videoClips.isEmpty else { throw NSError( @@ -196,12 +206,49 @@ internal class CompositionBuilder { // Compute dip-to-color windows for fadeToBlack / fadeToWhite transitions. let fadeWindows = computeFadeWindows(clipInstructions: videoResult.clipInstructions) + // Resolve the chroma key per clip onto its span of the composition timeline. + let chromaKeyWindows = computeChromaKeyWindows( + clipInstructions: videoResult.clipInstructions) + return ( composition, videoCompositionData, videoResult.renderSize, audioMix, sourceTrackID, - temporaryAudioURLs, fadeWindows + temporaryAudioURLs, fadeWindows, chromaKeyWindows ) } + /// Builds the chroma-key windows from the per-clip instruction time ranges. + /// + /// Every composition frame belongs to exactly one clip instruction, so one + /// window per keyed clip covers the whole timeline โ€” the compositor needs no + /// separate global fallback. A clip's own key wins over the global one. + /// + /// H.264/HEVC carry no alpha and the single-track path has nothing underneath, + /// so a key without a background is substituted with opaque black here. The + /// Android shader does the same (`applyChromaKey(flattenTransparency:)`), which + /// is what keeps the two platforms from diverging into "black" versus + /// "unchanged green". + private func computeChromaKeyWindows(clipInstructions: [ClipInstruction]) + -> [ChromaKeyWindow] + { + var windows: [ChromaKeyWindow] = [] + for (index, clip) in videoClips.enumerated() { + guard index < clipInstructions.count, + !clip.suppressChromaKey, + let key = clip.chromaKey ?? globalChromaKey + else { continue } + + let effective = key.isTransparent ? key.withOpaqueBlackBackground() : key + let range = clipInstructions[index].timeRange + windows.append( + ChromaKeyWindow( + startUs: Int64(CMTimeGetSeconds(range.start) * 1_000_000), + endUs: Int64(CMTimeGetSeconds(CMTimeRangeGetEnd(range)) * 1_000_000), + config: effective + )) + } + return windows + } + /// Builds the dip-to-color windows for `fadeToBlack` / `fadeToWhite` /// transitions from the per-clip instruction time ranges. /// diff --git a/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/LayeredCompositionBuilder.swift b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/LayeredCompositionBuilder.swift index 1764fb62..7a302b0e 100644 --- a/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/LayeredCompositionBuilder.swift +++ b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/LayeredCompositionBuilder.swift @@ -16,6 +16,7 @@ internal class LayeredCompositionBuilder { private let config: CompositionConfig private var enableAudio: Bool = true private var audioTracks: [AudioTrackConfig] = [] + private var globalChromaKey: ChromaKeyConfig? init(composition: CompositionConfig) { self.config = composition @@ -31,6 +32,14 @@ internal class LayeredCompositionBuilder { return self } + /// Sets the chroma key for clips carrying neither their own nor a layer key. + /// + /// Resolved per clip as `clip ?? layer ?? global`; never merged. + func setChromaKey(_ key: ChromaKeyConfig?) -> LayeredCompositionBuilder { + self.globalChromaKey = key + return self + } + /// A clip after it has been inserted onto its layer's track, with the data /// needed to build instructions and the audio mix. private struct PlacedClip { @@ -42,6 +51,7 @@ internal class LayeredCompositionBuilder { let transformConfig: SegmentTransformConfig? let preferredTransform: CGAffineTransform let displaySize: CGSize + let chromaKey: ChromaKeyConfig? } private struct AudioWindow { @@ -52,7 +62,7 @@ internal class LayeredCompositionBuilder { func build() async throws -> ( AVMutableComposition, VideoCompositionData, CGSize, AVAudioMix?, CMPersistentTrackID, [URL], - [FadeWindow] + [FadeWindow], [ChromaKeyWindow] ) { guard !config.layers.isEmpty else { throw NSError( @@ -156,7 +166,8 @@ internal class LayeredCompositionBuilder { opacity: layer.opacity, transformConfig: clip.transform ?? layer.transform, preferredTransform: pt, - displaySize: displaySize)) + displaySize: displaySize, + chromaKey: clip.chromaKey ?? layer.chromaKey ?? globalChromaKey)) } } @@ -209,9 +220,12 @@ internal class LayeredCompositionBuilder { + "\(audioTracks.count) audio tracks, " + "canvas \(Int(canvasSize.width))x\(Int(canvasSize.height))") + // No chroma-key windows: the layered path keys each layer on its own + // source frame via LayerPlacement, before it reaches the canvas. Keying the + // composed (opaque) canvas afterwards would be meaningless. return ( composition, videoCompositionData, canvasSize, audioMix, firstVideoTrackID, - temporaryAudioURLs, [] + temporaryAudioURLs, [], [] ) } @@ -250,7 +264,8 @@ internal class LayeredCompositionBuilder { targetRect: resolveRect(clip.transformConfig, displaySize: clip.displaySize), fit: clip.transformConfig?.fit ?? "fill", preferredTransform: clip.preferredTransform, - displaySize: clip.displaySize) + displaySize: clip.displaySize, + chromaKey: clip.chromaKey) } let timeRange = CMTimeRange( diff --git a/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/VideoSequenceBuilder.swift b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/VideoSequenceBuilder.swift index b47124bc..66e2a312 100644 --- a/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/VideoSequenceBuilder.swift +++ b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/VideoSequenceBuilder.swift @@ -487,6 +487,9 @@ internal struct LayerPlacement: Sendable { let preferredTransform: CGAffineTransform /// The source display size after applying `preferredTransform`. let displaySize: CGSize + /// Chroma key for this layer, resolved as `clip ?? layer ?? global`. + /// Applied to the layer's own source frame, before it reaches the canvas. + let chromaKey: ChromaKeyConfig? } /// Custom video composition instruction that explicitly provides source track IDs. diff --git a/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/models/RenderConfig.swift b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/models/RenderConfig.swift index 6df6d979..e2a165bd 100644 --- a/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/models/RenderConfig.swift +++ b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/models/RenderConfig.swift @@ -162,6 +162,116 @@ public struct ColorFilterConfig: Sendable { } } +/// Configuration for removing a solid-colored background ("green screen"). +/// +/// Mirrors the Dart `ChromaKey` model and the Kotlin `ChromaKeyConfig`. The +/// keying math itself lives in `ApplyChromaKey.swift` and is byte-for-byte the +/// same formula the Android fragment shader runs. +public struct ChromaKeyConfig: Sendable, Equatable { + /// The screen color to remove, as gamma-encoded RGB in 0...1. + let keyR: Double + let keyG: Double + let keyB: Double + /// Chroma-plane radius within which a pixel is removed completely. + let similarity: Double + /// Width of the soft ramp just beyond `similarity`. + let smoothness: Double + /// How strongly the key's color cast is pulled out of the remaining pixels. + let spill: Double + /// Solid background ARGB, or -1 when none (the sentinel convention used by + /// `ColorFilterConfig.startUs`/`endUs`). + let backgroundColor: Int64 + /// Background image bytes, or nil when none. + let backgroundImageData: Data? + + /// Whether the keyed area is left transparent rather than filled. + var isTransparent: Bool { backgroundColor == -1 && backgroundImageData == nil } + + /// The same key, but filling the removed area with opaque black. + /// + /// Used on the single-track path, where nothing sits underneath and the codec + /// carries no alpha, so "transparent" has no meaning. Android does the same + /// substitution in `applyChromaKey(flattenTransparency:)`. + func withOpaqueBlackBackground() -> ChromaKeyConfig { + ChromaKeyConfig( + keyR: keyR, keyG: keyG, keyB: keyB, + similarity: similarity, smoothness: smoothness, spill: spill, + backgroundColor: Int64(0xFF00_0000), backgroundImageData: nil) + } + + /// The key color projected onto the Cb/Cr chroma plane. + var keyChroma: (cb: Double, cr: Double) { + chromaOf(r: keyR, g: keyG, b: keyB) + } + + /// The unit vector pointing from neutral toward the key hue, used to pull + /// the key's cast back out during spill suppression. Zero for a neutral + /// (gray) key color, which disables despill rather than dividing by zero. + var keyDirection: (cb: Double, cr: Double) { + let c = keyChroma + let length = (c.cb * c.cb + c.cr * c.cr).squareRoot() + guard length > 1e-5 else { return (0, 0) } + return (c.cb / length, c.cr / length) + } + + /// Stable identity for the compositor's color-cube cache. + /// + /// Deliberately not `hashValue`: Swift seeds its hashing per process, so a + /// hash is not a safe cache key across the lifetime of a render, and the + /// background image data is far too large to hash per frame. + var cacheKey: String { + return "\(keyR),\(keyG),\(keyB),\(similarity),\(smoothness),\(spill)," + + "\(backgroundColor),\(backgroundImageFingerprint)" + } + + /// Cheap content fingerprint of the background image. + /// + /// The byte count alone is not an identity โ€” two different backgrounds that + /// happen to be the same size would share a cache entry, and the second clip + /// would render the first one's image. Mixing in the head and tail bytes + /// separates them while staying O(1), which matters because `cacheKey` is + /// evaluated per frame. + private var backgroundImageFingerprint: String { + guard let data = backgroundImageData, !data.isEmpty else { return "0" } + let sampleSize = min(32, data.count) + let head = data.prefix(sampleSize).reduce(into: UInt64(1_469_598_103_934_665_603)) { + accumulator, byte in + accumulator = (accumulator ^ UInt64(byte)) &* 1_099_511_628_211 + } + let tail = data.suffix(sampleSize).reduce(into: UInt64(1_469_598_103_934_665_603)) { + accumulator, byte in + accumulator = (accumulator ^ UInt64(byte)) &* 1_099_511_628_211 + } + return "\(data.count)-\(head)-\(tail)" + } + + static func fromArguments(_ args: [String: Any]?) -> ChromaKeyConfig? { + guard let args = args, + let keyColor = (args["keyColor"] as? NSNumber)?.int64Value + else { return nil } + + let backgroundImageData: Data? + if let flutterData = args["bgImageData"] as? FlutterStandardTypedData { + backgroundImageData = flutterData.data.isEmpty ? nil : flutterData.data + } else if let data = args["bgImageData"] as? Data, !data.isEmpty { + backgroundImageData = data + } else { + backgroundImageData = nil + } + + return ChromaKeyConfig( + keyR: Double((keyColor >> 16) & 0xFF) / 255.0, + keyG: Double((keyColor >> 8) & 0xFF) / 255.0, + keyB: Double(keyColor & 0xFF) / 255.0, + similarity: (args["similarity"] as? NSNumber)?.doubleValue ?? 0.20, + smoothness: (args["smoothness"] as? NSNumber)?.doubleValue ?? 0.08, + spill: (args["spill"] as? NSNumber)?.doubleValue ?? 0.5, + backgroundColor: (args["bgColor"] as? NSNumber)?.int64Value ?? -1, + backgroundImageData: backgroundImageData + ) + } +} + /// Configuration for a custom audio track with timing and volume. struct AudioTrackConfig { let path: String @@ -227,6 +337,9 @@ struct LayerConfig: Sendable { let opacity: Float /// Default placement for clips without their own transform. let transform: SegmentTransformConfig? + /// Default chroma key for the clips on this layer. A clip's own key wins; + /// `nil` falls back to the global key. + let chromaKey: ChromaKeyConfig? static func fromArguments(_ args: [String: Any]?) -> LayerConfig? { guard let args = args, @@ -237,7 +350,8 @@ struct LayerConfig: Sendable { return LayerConfig( clips: clips, opacity: (args["opacity"] as? NSNumber)?.floatValue ?? 1.0, - transform: SegmentTransformConfig.fromArguments(args["transform"] as? [String: Any]) + transform: SegmentTransformConfig.fromArguments(args["transform"] as? [String: Any]), + chromaKey: ChromaKeyConfig.fromArguments(args["chromaKey"] as? [String: Any]) ) } } @@ -359,6 +473,11 @@ struct RenderConfig: Sendable { /// Blur radius (nil = no blur, experimental feature) let blur: Double? + /// Removes a solid-colored background ("green screen") from every clip that + /// does not carry its own key. A `VideoClip.chromaKey` overrides this, and in + /// the layered path a `LayerConfig.chromaKey` sits between the two. + let chromaKey: ChromaKeyConfig? + /// Global start time in microseconds for trimming the final composition let startUs: Int64? @@ -404,6 +523,7 @@ struct RenderConfig: Sendable { colorFilters: self.colorFilters, audioTracks: self.audioTracks, blur: self.blur, + chromaKey: self.chromaKey, startUs: self.startUs, endUs: self.endUs, shouldOptimizeForNetworkUse: self.shouldOptimizeForNetworkUse, @@ -474,6 +594,7 @@ struct RenderConfig: Sendable { colorFilters: colorFilters, audioTracks: audioTracks, blur: (args["blur"] as? NSNumber)?.doubleValue, + chromaKey: ChromaKeyConfig.fromArguments(args["chromaKey"] as? [String: Any]), startUs: (args["startUs"] as? NSNumber)?.int64Value, endUs: (args["endUs"] as? NSNumber)?.int64Value, shouldOptimizeForNetworkUse: args["shouldOptimizeForNetworkUse"] as? Bool ?? true, diff --git a/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/models/VideoClip.swift b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/models/VideoClip.swift index 3b23c1be..e62396dc 100644 --- a/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/models/VideoClip.swift +++ b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/models/VideoClip.swift @@ -18,6 +18,16 @@ internal struct VideoClip: Sendable { /// Placement of this clip within the composition canvas (composition only). /// Overrides the layer transform. `nil` = use the layer transform. let transform: SegmentTransformConfig? + /// Removes a solid-colored background from this clip. Overrides the layer's + /// and the global key. `nil` = fall back to those. + let chromaKey: ChromaKeyConfig? + /// Opts this clip out of the layer/global key entirely. + /// + /// Internal, never parsed from the platform channel. `chromaKey == nil` means + /// "inherit", so it cannot express "deliberately unkeyed" โ€” which is exactly + /// what a pre-rendered overlap blend needs when the two clips it was composed + /// from carry different keys. See `RenderVideo.blendChromaKey`. + let suppressChromaKey: Bool init( inputPath: String, @@ -28,7 +38,9 @@ internal struct VideoClip: Sendable { reverseVideo: Bool = false, transition: ClipTransitionConfig? = nil, timelineStartUs: Int64? = nil, - transform: SegmentTransformConfig? = nil + transform: SegmentTransformConfig? = nil, + chromaKey: ChromaKeyConfig? = nil, + suppressChromaKey: Bool = false ) { self.inputPath = inputPath self.startUs = startUs @@ -39,6 +51,8 @@ internal struct VideoClip: Sendable { self.transition = transition self.timelineStartUs = timelineStartUs self.transform = transform + self.chromaKey = chromaKey + self.suppressChromaKey = suppressChromaKey } /// Parses a clip from a platform-channel map. Used by both the single-track @@ -58,6 +72,9 @@ internal struct VideoClip: Sendable { timelineStartUs: (clipMap["timelineStartUs"] as? NSNumber)?.int64Value, transform: SegmentTransformConfig.fromArguments( clipMap["transform"] as? [String: Any] + ), + chromaKey: ChromaKeyConfig.fromArguments( + clipMap["chromaKey"] as? [String: Any] ) ) } diff --git a/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/models/VideoCompositorConfig.swift b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/models/VideoCompositorConfig.swift index 9390903f..ea168e94 100644 --- a/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/models/VideoCompositorConfig.swift +++ b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/models/VideoCompositorConfig.swift @@ -47,6 +47,15 @@ public struct VideoCompositorConfig { /// Dip-to-color windows for `fadeToBlack` / `fadeToWhite` clip transitions. var fadeWindows: [FadeWindow] = [] + /// Chroma-key windows for the **single-track** path, one per clip that + /// carries a key. + /// + /// Always empty on the layered path, where the key lives on each + /// `LayerPlacement` instead: there, every layer is keyed on its own source + /// frame before it reaches the canvas, and keying the composed (opaque) + /// canvas afterwards would be meaningless. + var chromaKeyWindows: [ChromaKeyWindow] = [] + var videoRotationDegrees: Double = 0.0 var shouldApplyOrientationCorrection: Bool = false diff --git a/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/utils/VideoCompositor.swift b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/utils/VideoCompositor.swift index 82c0348d..2d883967 100644 --- a/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/utils/VideoCompositor.swift +++ b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/utils/VideoCompositor.swift @@ -137,10 +137,22 @@ class VideoCompositor: NSObject, AVVideoCompositing { /// Dip-to-color windows for fadeToBlack / fadeToWhite clip transitions private var fadeWindows: [FadeWindow] = [] - /// Cache for computed LUTs keyed by active filter indices + /// Per-clip chroma-key windows (single-track path). + private var chromaKeyWindows: [ChromaKeyWindow] = [] + + /// Decoded chroma-key background images, keyed by config, so a background is + /// decoded once per render rather than once per frame. + private var chromaBackgroundCache: [String: CIImage] = [:] + + /// Cache for computed LUTs keyed by the active chroma key and filter indices private let lutCacheQueue = DispatchQueue(label: "lut.cache.queue") private var lutCache: [String: (data: Data, size: Int)] = [:] + /// Insertion order of `lutCache`, so the oldest entry can be evicted. + private var lutCacheOrder: [String] = [] private let defaultLutSize = 33 + /// A 33ยณ cube is 574 KB; a long timeline with many distinct keys and filters + /// would otherwise accumulate one per combination. + private let lutCacheLimit = 8 static var config = VideoCompositorConfig() @@ -176,8 +188,40 @@ class VideoCompositor: NSObject, AVVideoCompositing { self.setOverlayImageLayers(from: config.imageLayerConfigs) self.colorFilterConfigs = config.colorFilterConfigs self.fadeWindows = config.fadeWindows + self.chromaKeyWindows = config.chromaKeyWindows + } + + /// The chroma key active at the given composition time, if any. + /// + /// Windows are per clip and never overlap, so the first match wins. + private func activeChromaKey(at compositionTime: CMTime) -> ChromaKeyConfig? { + guard !chromaKeyWindows.isEmpty else { return nil } + let tUs = Int64(CMTimeGetSeconds(compositionTime) * 1_000_000) + for window in chromaKeyWindows + where window.endUs > window.startUs && tUs >= window.startUs && tUs < window.endUs { + return window.config + } + // The very last frame of the timeline can land exactly on the final + // window's end, so clamp to it rather than emitting one unkeyed frame. + // + // Only ever by a frame, and only for the window that actually ends the + // timeline. Windows are emitted per *keyed* clip, so the array is sparse: + // an unconditional clamp applied the last keyed clip's key to every clip + // after it, keying clips that carry no key at all. + if let last = chromaKeyWindows.last, tUs >= last.endUs, + tUs - last.endUs <= Self.chromaKeyEndClampToleranceUs + { + return last.config + } + return nil } + /// How far past a window's end still counts as its final frame. + /// + /// One frame at 24 fps, the slowest rate worth accommodating; anything beyond + /// that is a different clip, not a rounding edge. + private static let chromaKeyEndClampToleranceUs: Int64 = 41_667 + /// Applies a dip-to-color (fade-to-black / fade-to-white) at the given /// composition time, if any fade window is active. The video is mixed toward /// the dip color by `dipAmount` (0 = full video, 1 = full color). @@ -261,8 +305,9 @@ class VideoCompositor: NSObject, AVVideoCompositing { } } - /// Computes the LUT for a given set of active color filter indices. - /// Results are cached so that each unique combination is only computed once. + /// Computes the color-filter cube active at the given composition time. + /// + /// Results are cached so each unique combination is built only once. private func getLUTForActiveFilters(at compositionTime: CMTime) -> (data: Data, size: Int)? { guard !colorFilterConfigs.isEmpty else { return nil } @@ -279,7 +324,7 @@ class VideoCompositor: NSObject, AVVideoCompositing { } guard !activeIndices.isEmpty else { return nil } - let cacheKey = activeIndices.map { String($0) }.joined(separator: ",") + let cacheKey = "cf:" + activeIndices.map { String($0) }.joined(separator: ",") // Check cache var cached: (data: Data, size: Int)? @@ -293,16 +338,24 @@ class VideoCompositor: NSObject, AVVideoCompositing { let activeMatrices = activeIndices.map { colorFilterConfigs[$0].matrix } let combined = combineColorMatrices(activeMatrices) guard combined.count == 20 else { return nil } - guard let data = generateLUTData(from: combined, size: defaultLutSize) else { return nil } + guard let data = generateLUTData(from: combined, size: defaultLutSize) else { + return nil + } let result = (data: data, size: defaultLutSize) lutCacheQueue.sync { - lutCache[cacheKey] = result + if lutCache[cacheKey] == nil { + lutCache[cacheKey] = result + lutCacheOrder.append(cacheKey) + while lutCacheOrder.count > lutCacheLimit { + lutCache.removeValue(forKey: lutCacheOrder.removeFirst()) + } + } } return result } - /// Applies the LUT for active color filters at the given composition time. + /// Applies the active color cube at the given composition time. private func applyColorFilter(to image: CIImage, at compositionTime: CMTime) -> CIImage { guard let lut = getLUTForActiveFilters(at: compositionTime), let lutFilter = CIFilter(name: "CIColorCube") @@ -315,6 +368,146 @@ class VideoCompositor: NSObject, AVVideoCompositing { return lutFilter.outputImage ?? image } + /// Applies a chroma key as its own color cube. + /// + /// The cube encodes the whole key โ€” matte alpha *and* despilled color โ€” since + /// both are pure functions of the input RGB. Entries are premultiplied, as + /// `CIColorCube` requires. Cached like the color-filter cube. + private func applyChromaKeyCube(to image: CIImage, _ config: ChromaKeyConfig) -> CIImage { + let cacheKey = "ck:\(config.cacheKey)" + + var cached: (data: Data, size: Int)? + lutCacheQueue.sync { cached = lutCache[cacheKey] } + + let lut: (data: Data, size: Int) + if let cached = cached { + lut = cached + } else { + guard + let data = generateChromaLUTData(chroma: config, size: defaultLutSize) + else { return image } + lut = (data: data, size: defaultLutSize) + lutCacheQueue.sync { + if lutCache[cacheKey] == nil { + lutCache[cacheKey] = lut + lutCacheOrder.append(cacheKey) + while lutCacheOrder.count > lutCacheLimit { + lutCache.removeValue(forKey: lutCacheOrder.removeFirst()) + } + } + } + } + + guard let filter = CIFilter(name: "CIColorCube") else { return image } + filter.setValue(lut.size, forKey: "inputCubeDimension") + filter.setValue(lut.data, forKey: "inputCubeData") + filter.setValue(image, forKey: kCIInputImageKey) + return filter.outputImage ?? image + } + + /// Removes the chroma key from a source frame and fills the keyed area. + /// + /// Runs on the **raw source frame**, before crop, rotation, flip and scale. + /// Three reasons: + /// + /// 1. It is a single insertion point that covers both branches of the + /// `imageBytesWithCropping` split, where the color filter is duplicated. + /// Applying it inside `applyColorFilter` instead would place it before the + /// crop in one branch and after it in the other, so a background *image* + /// would be cropped along with the video in one case and not the other. + /// 2. It matches the Android ordering exactly: there the chroma shader is + /// first in the per-clip chain and `SingleColorLut` grades whatever it + /// produced โ€” so a color filter grades the substituted background too. + /// Folding the filter into the key's own cube instead would grade only the + /// video and leave the background at its raw color. + /// 3. It keys unresampled pixels, which keeps the soft edge crisp. + /// + /// Chaining the color filter's own `CIColorCube` afterwards is safe precisely + /// because the background has already been composited: the frame is opaque + /// again, so the second cube resetting alpha from its own data changes + /// nothing. Without a background it would un-key the frame โ€” which is why the + /// single-track path substitutes opaque black for a transparent key, and why + /// the layered path keys inside `composeLayered` instead. + private func applyChromaKeyStage(to image: CIImage, at compositionTime: CMTime) + -> CIImage + { + guard let chroma = activeChromaKey(at: compositionTime) else { return image } + return compositeChromaBackground(applyChromaKeyCube(to: image, chroma), chroma) + } + + /// Fills the keyed-out area with the key's background. + /// + /// A nil background leaves the image transparent, which only carries meaning + /// on the layered path โ€” the single-track path substitutes opaque black long + /// before this, in `CompositionBuilder.computeChromaKeyWindows`. + private func compositeChromaBackground(_ image: CIImage, _ config: ChromaKeyConfig) + -> CIImage + { + let extent = image.extent + guard extent.width > 0, extent.height > 0 else { return image } + + if let background = chromaBackgroundImage(for: config) { + return image.composited(over: scaleToFill(background, extent)) + } + + // A background image that will not decode must not quietly leave the keyed + // area transparent: on the layered path that shows the layer below instead + // of the requested backdrop, and Android would disagree. Fill with opaque + // black, matching `ChromaKeyEffect`'s fallback for the same case. + if config.backgroundImageData != nil { + return image.composited(over: CIImage(color: .black).cropped(to: extent)) + } + + guard config.backgroundColor != -1 else { return image } + let argb = config.backgroundColor + // Alpha is deliberately ignored: the fill replaces the removed screen + // rather than tinting it, and Android's shader has no alpha uniform for it. + // `ChromaKey.toAsyncMap` asserts the color is opaque, so this only bites in + // release. For a see-through result the key is left backgroundless and the + // backdrop goes on a lower layer. + let color = CIColor( + red: CGFloat((argb >> 16) & 0xFF) / 255.0, + green: CGFloat((argb >> 8) & 0xFF) / 255.0, + blue: CGFloat(argb & 0xFF) / 255.0, + alpha: 1.0) + return image.composited(over: CIImage(color: color).cropped(to: extent)) + } + + /// Stretches [image] to exactly cover [extent]. + /// + /// Matches `ImageLayer`'s "no offset means stretch to the frame" semantics and + /// the Android shader, which samples the background with the frame's own + /// texture coordinates. + private func scaleToFill(_ image: CIImage, _ extent: CGRect) -> CIImage { + let source = image.extent + guard source.width > 0, source.height > 0 else { return image } + let scaled = image.transformed( + by: CGAffineTransform( + scaleX: extent.width / source.width, y: extent.height / source.height)) + return scaled.transformed( + by: CGAffineTransform( + translationX: extent.minX - scaled.extent.minX, + y: extent.minY - scaled.extent.minY)) + } + + /// The decoded background image for [config], decoded once and cached. + private func chromaBackgroundImage(for config: ChromaKeyConfig) -> CIImage? { + guard let data = config.backgroundImageData else { return nil } + if let cached = chromaBackgroundCache[config.cacheKey] { return cached } + + #if os(iOS) + guard let image = UIImage(data: data), let cgImage = image.cgImage else { return nil } + #elseif os(macOS) + guard let image = NSImage(data: data), + let cgImage = image.cgImage(forProposedRect: nil, context: nil, hints: nil) + else { return nil } + #endif + + let ciImage = CIImage(cgImage: cgImage) + chromaBackgroundCache[config.cacheKey] = ciImage + return ciImage + } + private let context = CIContext(options: [ .workingColorSpace: NSNull(), .outputColorSpace: CGColorSpace(name: CGColorSpace.sRGB)!, @@ -354,6 +547,15 @@ class VideoCompositor: NSObject, AVVideoCompositing { guard let buffer = request.sourceFrame(byTrackID: placement.trackID) else { continue } var img = CIImage(cvPixelBuffer: buffer) + // 0. Remove this layer's chroma key on its raw source frame, before any + // orientation or scaling. A key without a background stays transparent + // here โ€” unlike the single-track path, there really is something + // underneath, and step 5's composite lets it show through. + if let key = placement.chromaKey { + img = applyChromaKeyCube(to: img, key) + img = compositeChromaBackground(img, key) + } + // 1. Orient using the source preferred transform (rotation + mirror // metadata), then normalize the extent back to the origin. let preferred = placement.preferredTransform @@ -496,6 +698,15 @@ class VideoCompositor: NSObject, AVVideoCompositing { outputImage = CIImage(cvPixelBuffer: sourceBuffer) + // Remove the chroma key on the raw decoded frame and fill the keyed area + // with its background โ€” before the layer-instruction transform below, + // which on a mixed-resolution timeline is a real bilinear resample. Keying + // interpolated pixels widens the matte edge by an invented pixel and runs + // the ramp over colors the decoder never produced. Android keys first for + // the same reason (`applyChromaKey` is added ahead of + // `VideoCompositionTransformation` in the effect chain). + outputImage = applyChromaKeyStage(to: outputImage, at: request.compositionTime) + // Apply layer instruction transform first (video scaling/centering/rotation) // This ensures all videos are properly sized and oriented before applying user effects. // The layerInstruction contains the preferredTransform which already handles video rotation diff --git a/example/assets/greenscreen.mp4 b/example/assets/greenscreen.mp4 new file mode 100644 index 00000000..2fd05c19 Binary files /dev/null and b/example/assets/greenscreen.mp4 differ diff --git a/example/integration_test/chroma_key_test.dart b/example/integration_test/chroma_key_test.dart new file mode 100644 index 00000000..31fe33ab --- /dev/null +++ b/example/integration_test/chroma_key_test.dart @@ -0,0 +1,614 @@ +import 'dart:typed_data'; +import 'dart:ui'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:integration_test/integration_test.dart'; +import 'package:pro_video_editor/pro_video_editor.dart'; +import 'package:pro_video_editor_example/core/constants/example_constants.dart'; + +/// End-to-end verification that the chroma key actually removes the screen and +/// fills it, on both the single-track and the layered path. +/// +/// The green-screen source is **synthesized in-test** rather than shipped as a +/// binary asset: a painted PNG is turned into a video with `renderStopMotion`. +/// That keeps the input fully deterministic (exact colors, exact geometry), so +/// the assertions can be about the key rather than about some clip's content. +/// +/// All assertions stay tolerant of YUV/codec rounding, consistent with the rest +/// of the suite. +void main() { + IntegrationTestWidgetsFlutterBinding.ensureInitialized(); + + final pve = ProVideoEditor.instance; + + // SMPTE "chroma key green" โ€” the ChromaKey default. + const screenGreen = Color(0xFF00B140); + // The subject: a saturated blue that is nowhere near the key hue. + const subjectBlue = Color(0xFF1040D0); + const backgroundRed = Color(0xFFE00000); + + const canvas = Size(640, 360); + const frameAt = Duration(milliseconds: 400); + + /// Renders [paint] into a PNG of [size]. + Future paintPng(Size size, void Function(Canvas) paint) async { + final recorder = PictureRecorder(); + final canvas = Canvas(recorder); + paint(canvas); + final picture = recorder.endRecording(); + final image = await picture.toImage( + size.width.toInt(), + size.height.toInt(), + ); + final data = await image.toByteData(format: ImageByteFormat.png); + return data!.buffer.asUint8List(); + } + + /// A PNG whose left half is [left] and right half is [right]. + Future splitPng(Color left, Color right) { + return paintPng(canvas, (c) { + final w = canvas.width; + final h = canvas.height; + c + ..drawRect(Rect.fromLTWH(0, 0, w / 2, h), Paint()..color = left) + ..drawRect(Rect.fromLTWH(w / 2, 0, w / 2, h), Paint()..color = right); + }); + } + + /// A PNG whose top half is [top] and bottom half is [bottom]. + /// + /// A solid background cannot tell a vertically flipped texture from a correct + /// one, and neither can a left/right split. This can. + Future stackedPng(Color top, Color bottom) { + return paintPng(canvas, (c) { + final w = canvas.width; + final h = canvas.height; + c + ..drawRect(Rect.fromLTWH(0, 0, w, h / 2), Paint()..color = top) + ..drawRect(Rect.fromLTWH(0, h / 2, w, h / 2), Paint()..color = bottom); + }); + } + + /// A solid-color PNG. + Future solidPng(Color color) { + return paintPng( + canvas, + (c) => c.drawRect( + Rect.fromLTWH(0, 0, canvas.width, canvas.height), + Paint()..color = color, + ), + ); + } + + /// Turns a still image into a video the key can work on. + /// + /// Built by painting [png] over a real clip rather than by + /// `renderStopMotion`: a stop-motion clip is a single held still, and two of + /// those do not concatenate (verified: 2s + 2s comes out as 2.03s), which the + /// multi-segment test below needs. Overlaying a real clip keeps a normal + /// frame/GOP structure while making every pixel an exactly known color. + Future videoFromImage( + Uint8List png, { + Duration duration = const Duration(seconds: 2), + }) async { + final bytes = await pve.renderVideo( + VideoRenderData( + videoSegments: [ + VideoSegment( + video: EditorVideo.asset('assets/tests/test_d.mp4'), + endTime: duration, + ), + ], + // No offset/size: the layer is stretched over the whole frame. + imageLayers: [ImageLayer(image: EditorLayerImage.memory(png))], + transform: ExportTransform( + width: canvas.width.toInt(), + height: canvas.height.toInt(), + ), + ), + ); + return EditorVideo.memory(bytes); + } + + /// Decodes a frame of [video] and returns the RGBA at the relative position + /// ([fx], [fy]) in `0..1`. + Future> samplePixel( + Uint8List video, + double fx, + double fy, { + Duration at = frameAt, + }) async { + final frames = await pve.getThumbnails( + ThumbnailConfigs( + video: EditorVideo.memory(video), + outputFormat: ThumbnailFormat.png, + timestamps: [at], + outputSize: canvas, + boxFit: ThumbnailBoxFit.cover, + ), + ); + expect(frames, isNotEmpty, reason: 'No frame extracted from the output'); + final codec = await instantiateImageCodec(frames.first); + final image = (await codec.getNextFrame()).image; + final data = await image.toByteData(); + final px = (fx * (image.width - 1)).round().clamp(0, image.width - 1); + final py = (fy * (image.height - 1)).round().clamp(0, image.height - 1); + final i = (py * image.width + px) * 4; + return [ + data!.getUint8(i), + data.getUint8(i + 1), + data.getUint8(i + 2), + data.getUint8(i + 3), + ]; + } + + /// Sum of absolute per-channel (RGB) differences between two pixels. + int colorDist(List a, List b) => + (a[0] - b[0]).abs() + (a[1] - b[1]).abs() + (a[2] - b[2]).abs(); + + bool isRed(List c) => c[0] > 150 && c[1] < 100 && c[2] < 100; + bool isBlue(List c) => c[2] > 130 && c[0] < 120 && c[1] < 130; + bool isGreen(List c) => c[1] > 110 && c[0] < 110; + + // Sample points well inside each half, away from the seam. + const leftPoints = [ + Offset(0.15, 0.3), + Offset(0.25, 0.5), + Offset(0.15, 0.7), + ]; + const rightPoints = [ + Offset(0.75, 0.3), + Offset(0.85, 0.5), + Offset(0.75, 0.7), + ]; + + /// The green/blue split source, built once and shared by the tests. + late EditorVideo greenScreen; + + setUpAll(() async { + greenScreen = await videoFromImage( + await splitPng(screenGreen, subjectBlue), + ); + + // Guard the fixture itself: if the synthesized source is not actually + // green on the left and blue on the right, every assertion below would be + // meaningless. + final source = await greenScreen.safeByteArray(); + for (final p in leftPoints) { + expect( + isGreen(await samplePixel(source, p.dx, p.dy)), + isTrue, + reason: 'Fixture is not green at $p', + ); + } + for (final p in rightPoints) { + expect( + isBlue(await samplePixel(source, p.dx, p.dy)), + isTrue, + reason: 'Fixture is not blue at $p', + ); + } + }); + + group('ChromaKey - single track', () { + testWidgets('a color background replaces the screen', (_) async { + final out = await pve.renderVideo( + VideoRenderData( + videoSegments: [VideoSegment(video: greenScreen)], + chromaKey: const ChromaKey(backgroundColor: backgroundRed), + ), + ); + + for (final p in leftPoints) { + final c = await samplePixel(out, p.dx, p.dy); + expect(isRed(c), isTrue, reason: 'Screen not replaced at $p, got $c'); + } + }); + + testWidgets('the subject is left alone', (_) async { + final out = await pve.renderVideo( + VideoRenderData( + videoSegments: [VideoSegment(video: greenScreen)], + chromaKey: const ChromaKey(backgroundColor: backgroundRed), + ), + ); + + for (final p in rightPoints) { + final c = await samplePixel(out, p.dx, p.dy); + expect(isBlue(c), isTrue, reason: 'Subject was keyed at $p, got $c'); + } + }); + + testWidgets('a tiny similarity keys nothing', (_) async { + // The regression guard: with the threshold below the encoder's own + // rounding of the key color, the output must still be the source. + final out = await pve.renderVideo( + VideoRenderData( + videoSegments: [VideoSegment(video: greenScreen)], + chromaKey: const ChromaKey( + similarity: 0.001, + smoothness: 0, + spill: 0, + backgroundColor: backgroundRed, + ), + ), + ); + + for (final p in leftPoints) { + final c = await samplePixel(out, p.dx, p.dy); + expect(isRed(c), isFalse, reason: 'Keyed despite similarityโ‰ˆ0 at $p'); + expect(isGreen(c), isTrue, reason: 'Screen not preserved at $p'); + } + }); + + testWidgets('an image background fills the screen', (_) async { + final out = await pve.renderVideo( + VideoRenderData( + videoSegments: [VideoSegment(video: greenScreen)], + chromaKey: ChromaKey( + backgroundImage: EditorLayerImage.memory( + await solidPng(backgroundRed), + ), + ), + ), + ); + + for (final p in leftPoints) { + final c = await samplePixel(out, p.dx, p.dy); + expect(isRed(c), isTrue, reason: 'Background image missing at $p'); + } + for (final p in rightPoints) { + final c = await samplePixel(out, p.dx, p.dy); + expect(isBlue(c), isTrue, reason: 'Subject was keyed at $p, got $c'); + } + }); + + testWidgets('the background image is not vertically flipped', (_) async { + // Android uploads the bitmap with GLUtils.texImage2D, which puts row 0 + // at t=0, but samples it with the frame's own coordinate, where t=0 is + // the *bottom*. Media3 flips bitmap textures for exactly this reason. + // Only an asymmetric background can see the difference. + final out = await pve.renderVideo( + VideoRenderData( + videoSegments: [VideoSegment(video: greenScreen)], + chromaKey: ChromaKey( + backgroundImage: EditorLayerImage.memory( + await stackedPng(backgroundRed, subjectBlue), + ), + ), + ), + ); + + // Sampled inside the keyed (left) half, so only the background shows. + final top = await samplePixel(out, 0.15, 0.2); + final bottom = await samplePixel(out, 0.15, 0.8); + + expect( + isRed(top), + isTrue, + reason: 'Background image top should be red, got $top โ€” flipped?', + ); + expect( + isBlue(bottom), + isTrue, + reason: + 'Background image bottom should be blue, got $bottom โ€” flipped?', + ); + }); + + testWidgets('a color filter does not un-key the frame', (_) async { + // Apple folds the key and the filter into one CIColorCube because a + // second cube would take its alpha from its own data and resurrect the + // screen. This is the regression guard for that. + final out = await pve.renderVideo( + VideoRenderData( + videoSegments: [VideoSegment(video: greenScreen)], + chromaKey: const ChromaKey(backgroundColor: backgroundRed), + colorFilters: const [ + ColorFilter( + matrix: [ + 0.299, 0.587, 0.114, 0, 0, // + 0.299, 0.587, 0.114, 0, 0, // + 0.299, 0.587, 0.114, 0, 0, // + 0, 0, 0, 1, 0, // + ], + ), + ], + ), + ); + + for (final p in leftPoints) { + final c = await samplePixel(out, p.dx, p.dy); + expect( + isGreen(c), + isFalse, + reason: 'The color filter resurrected the screen at $p, got $c', + ); + // Grayscaled red: R==G==B, and clearly not the original green. + expect( + (c[0] - c[1]).abs() < 30 && (c[1] - c[2]).abs() < 30, + isTrue, + reason: 'Output is not grayscale at $p, got $c', + ); + } + }); + + testWidgets('a bitrate cap does not skip the key via passthrough', ( + _, + ) async { + // A single untrimmed clip within the cap would otherwise take the + // lossless fast path (Apple passthrough / Android transmux) and the key + // would silently do nothing. + final out = await pve.renderVideo( + VideoRenderData( + videoSegments: [VideoSegment(video: greenScreen)], + chromaKey: const ChromaKey(backgroundColor: backgroundRed), + bitrate: 8000000, + ), + ); + + for (final p in leftPoints) { + final c = await samplePixel(out, p.dx, p.dy); + expect(isRed(c), isTrue, reason: 'Passthrough skipped the key at $p'); + } + }); + + testWidgets('imageBytesWithCropping keys in both branches', (_) async { + // The compositor duplicates its effect chain across this flag, so both + // sides have to be exercised or one can silently lose the key. + for (final withCropping in [false, true]) { + final out = await pve.renderVideo( + VideoRenderData( + videoSegments: [VideoSegment(video: greenScreen)], + chromaKey: const ChromaKey(backgroundColor: backgroundRed), + imageBytesWithCropping: withCropping, + ), + ); + + for (final p in leftPoints) { + final c = await samplePixel(out, p.dx, p.dy); + expect( + isRed(c), + isTrue, + reason: 'Key lost with imageBytesWithCropping=$withCropping at $p', + ); + } + } + }); + + testWidgets('a per-segment key overrides the global one', (_) async { + // Two sources, each re-encoded once through the normal render path: a + // stop-motion clip holds a single still for its whole duration, and two + // of those concatenate down to one segment. Re-encoding gives them an + // ordinary frame/GOP structure. + const half = Duration(seconds: 2); + Future concatenatable(Color right) async { + final still = await videoFromImage( + await splitPng(screenGreen, right), + duration: half, + ); + return EditorVideo.memory( + await pve.renderVideo( + VideoRenderData(videoSegments: [VideoSegment(video: still)]), + ), + ); + } + + final firstSource = await concatenatable(subjectBlue); + final secondSource = await concatenatable(const Color(0xFF804000)); + + final out = await pve.renderVideo( + VideoRenderData( + videoSegments: [ + VideoSegment(video: firstSource), + VideoSegment( + video: secondSource, + // Overrides the global red with a distinctly different fill. + chromaKey: const ChromaKey(backgroundColor: Color(0xFFFFFFFF)), + ), + ], + chromaKey: const ChromaKey(backgroundColor: backgroundRed), + ), + ); + + // Sample a quarter and three quarters into the real output rather than at + // fixed timestamps: the concatenated length depends on how the encoder + // rounds each segment, and a timestamp past the end yields no frame. + final duration = (await pve.getMetadata( + EditorVideo.memory(out), + )).duration; + expect( + duration.inMilliseconds, + closeTo(4000, 700), + reason: 'Both segments should be in the output, got $duration', + ); + final inFirst = duration * 0.25; + final inSecond = duration * 0.75; + + // First segment: the global key's red. + final first = await samplePixel(out, 0.2, 0.5, at: inFirst); + expect( + isRed(first), + isTrue, + reason: 'Global key not applied, got $first', + ); + + // Second segment: its own key's white. + final second = await samplePixel(out, 0.2, 0.5, at: inSecond); + expect( + second[0] > 180 && second[1] > 180 && second[2] > 180, + isTrue, + reason: 'Per-segment key did not override the global one, got $second', + ); + }); + }); + + group('ChromaKey - autoDetect', () { + testWidgets('measures the real studio screen', (_) async { + final detection = await ChromaKey.detect( + EditorVideo.asset('assets/greenscreen.mp4'), + ); + + // The recorded screen, not the paint it was mixed from: SMPTE green is + // 0xFF00B140, the camera saw roughly 0xFF2A9D37. + expect(detection.coverage, greaterThan(0.9)); + expect( + detection.color.g, + greaterThan(detection.color.r), + reason: 'a green screen should detect as green, got ${detection.color}', + ); + expect( + detection.color.g, + greaterThan(detection.color.b), + reason: 'a green screen should detect as green, got ${detection.color}', + ); + + // The whole point: a measured key sits much closer to the screen than + // the SMPTE constant, so it needs far less similarity to cover it. + expect( + detection.similarity, + lessThan(const ChromaKey().similarity), + reason: 'measured ${detection.similarity} should beat the constant', + ); + }); + + testWidgets('the detected key removes the screen', (_) async { + final gs = EditorVideo.asset('assets/greenscreen.mp4'); + final key = await ChromaKey.autoDetect( + gs, + backgroundColor: backgroundRed, + ); + + final out = await pve.renderVideo( + VideoRenderData( + videoSegments: [VideoSegment(video: gs)], + chromaKey: key, + ), + ); + + // Corners: screen everywhere, including the dimly lit bottom ones that + // survive a too-tight constant key. + for (final p in const [ + Offset(0.05, 0.05), + Offset(0.95, 0.05), + Offset(0.05, 0.95), + Offset(0.95, 0.95), + Offset(0.15, 0.5), + ]) { + final c = await samplePixel(out, p.dx, p.dy); + expect(isRed(c), isTrue, reason: 'Screen survived at $p, got $c'); + } + }); + + testWidgets('the detected key keeps the subject', (_) async { + final gs = EditorVideo.asset('assets/greenscreen.mp4'); + final key = await ChromaKey.autoDetect( + gs, + backgroundColor: backgroundRed, + ); + + final out = await pve.renderVideo( + VideoRenderData( + videoSegments: [VideoSegment(video: gs)], + chromaKey: key, + ), + ); + + // Center of the subject: pink outfit, must not be keyed. + final c = await samplePixel(out, 0.66, 0.45); + expect(isRed(c), isFalse, reason: 'The subject was keyed away, got $c'); + }); + + testWidgets('rejects a frame that is not a screen', (_) async { + // The plain demo video has no screen at all. + expect( + () => ChromaKey.detect(EditorVideo.asset(kVideoEditorExampleH264Path)), + throwsA(isA()), + ); + }); + }); + + group('ChromaKey - composition', () { + testWidgets('a transparent key lets the layer below show through', ( + _, + ) async { + // The one case that exercises real transparency end to end: it would + // fail on a premultiplied/straight alpha mix-up, on a Media3 blend + // regression, and on the VideoCompositionTransformation alpha-squaring. + final backdrop = await videoFromImage(await solidPng(backgroundRed)); + + final out = await pve.renderVideo( + VideoRenderData( + composition: VideoComposition( + canvasSize: canvas, + layers: [ + VideoLayer(clips: [VideoSegment(video: backdrop)]), + VideoLayer( + clips: [VideoSegment(video: greenScreen)], + chromaKey: const ChromaKey(), + ), + ], + ), + ), + ); + + for (final p in leftPoints) { + final c = await samplePixel(out, p.dx, p.dy); + expect( + isRed(c), + isTrue, + reason: 'The layer below does not show through at $p, got $c', + ); + // Specifically not black: that is what a flattened alpha looks like. + expect( + colorDist(c, [0, 0, 0, 255]) > 90, + isTrue, + reason: 'Keyed area came out black at $p, got $c', + ); + } + + for (final p in rightPoints) { + final c = await samplePixel(out, p.dx, p.dy); + expect(isBlue(c), isTrue, reason: 'Subject was keyed at $p, got $c'); + } + }); + + testWidgets('a per-clip key overrides the layer key', (_) async { + final backdrop = await videoFromImage(await solidPng(backgroundRed)); + + final out = await pve.renderVideo( + VideoRenderData( + composition: VideoComposition( + canvasSize: canvas, + layers: [ + VideoLayer(clips: [VideoSegment(video: backdrop)]), + VideoLayer( + clips: [ + VideoSegment( + video: greenScreen, + // Keys almost nothing, overriding the layer's default key. + chromaKey: const ChromaKey( + similarity: 0.001, + smoothness: 0, + spill: 0, + ), + ), + ], + chromaKey: const ChromaKey(), + ), + ], + ), + ), + ); + + for (final p in leftPoints) { + final c = await samplePixel(out, p.dx, p.dy); + expect( + isGreen(c), + isTrue, + reason: 'The layer key won over the clip key at $p, got $c', + ); + } + }); + }); +} diff --git a/example/lib/core/constants/example_constants.dart b/example/lib/core/constants/example_constants.dart index 9b7c2f88..b36b394c 100644 --- a/example/lib/core/constants/example_constants.dart +++ b/example/lib/core/constants/example_constants.dart @@ -21,6 +21,10 @@ const String kVideoEditorExampleDivinePath = /// Darwin AVAssetWriter path downmixes them to stereo). const String kVideoEditorExampleSurround51Path = 'assets/surround_5_1.mp4'; +/// A local path to a real green-screen clip: a person in front of a lit studio +/// screen. Cropped to the screen area, so nothing outside it survives the key. +const String kVideoEditorExampleGreenScreenPath = 'assets/greenscreen.mp4'; + /// A local path to the first example audio track used in video editor demos. const String kVideoEditorExampleAudio1Path = 'assets/audio1.mp3'; diff --git a/example/lib/features/chroma_key/chroma_key_example_page.dart b/example/lib/features/chroma_key/chroma_key_example_page.dart new file mode 100644 index 00000000..9d0adb2f --- /dev/null +++ b/example/lib/features/chroma_key/chroma_key_example_page.dart @@ -0,0 +1,392 @@ +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:chewie/chewie.dart'; +import 'package:flutter/material.dart'; +import 'package:path_provider/path_provider.dart'; +import 'package:pro_video_editor/pro_video_editor.dart'; +import 'package:pro_video_editor_example/shared/utils/render_cancel_capability.dart'; +import 'package:pro_video_editor_example/shared/widgets/video_renderer_progress.dart'; +import 'package:video_player/video_player.dart'; + +import '/core/constants/example_constants.dart'; +import '/shared/utils/bytes_formatter.dart'; +import '/shared/widgets/native_log_console.dart'; + +/// What the removed screen is filled with. +enum _Background { + /// A solid color. + color, + + /// A still image. + image, + + /// Nothing โ€” the layer below in a [VideoComposition] shows through. + video, +} + +/// A page demonstrating the chroma key: removing a green screen and putting +/// something else behind the subject. +class ChromaKeyExamplePage extends StatefulWidget { + /// Creates a [ChromaKeyExamplePage]. + const ChromaKeyExamplePage({super.key}); + + @override + State createState() => _ChromaKeyExamplePageState(); +} + +class _ChromaKeyExamplePageState extends State { + final _pve = ProVideoEditor.instance; + + VideoPlayerController? _controllerPreview; + ChewieController? _chewieControllerPreview; + bool _isPreviewInitialized = false; + + bool _isExporting = false; + Uint8List? _videoBytes; + Duration _generationTime = Duration.zero; + String _taskId = DateTime.now().microsecondsSinceEpoch.toString(); + + _Background _background = _Background.color; + Color _keyColor = const ChromaKey().color; + double _similarity = const ChromaKey().similarity; + double _smoothness = 0.08; + double _spill = 0.5; + + /// The last auto-detection, shown so the measurement is visible rather than + /// just silently applied. + ChromaKeyDetection? _detection; + bool _isDetecting = false; + + bool get _supportsCancel => canCancelOnCurrentPlatform(); + + static const _fillRed = Color(0xFFE00000); + + /// The green-screen asset's own size, used as the composition canvas so the + /// keyed clip and the backdrop line up without scaling. + static const _canvas = Size(756, 732); + + EditorVideo get _greenScreen => + EditorVideo.asset(kVideoEditorExampleGreenScreenPath); + + @override + void dispose() { + // Unconditional: the controller can be fully initialized and playing while + // `_isPreviewInitialized` is still false, when the page is popped between + // `initialize()` and the `setState` that flips the flag. + _chewieControllerPreview?.dispose(); + _controllerPreview?.dispose(); + super.dispose(); + } + + /// Builds the render for the selected background. + VideoRenderData _buildRenderData(EditorVideo source) { + switch (_background) { + case _Background.color: + return VideoRenderData( + id: _taskId, + videoSegments: [VideoSegment(video: source)], + chromaKey: ChromaKey( + color: _keyColor, + similarity: _similarity, + smoothness: _smoothness, + spill: _spill, + backgroundColor: _fillRed, + ), + ); + + case _Background.image: + return VideoRenderData( + id: _taskId, + videoSegments: [VideoSegment(video: source)], + chromaKey: ChromaKey( + color: _keyColor, + similarity: _similarity, + smoothness: _smoothness, + spill: _spill, + backgroundImage: EditorLayerImage.asset('assets/sticker.png'), + ), + ); + + case _Background.video: + // The key is left transparent, so the layer below โ€” a second video โ€” + // shows through. This is the only way to put a *video* behind the + // screen: the single-track path has nothing underneath, and neither + // H.264 nor HEVC carries an alpha channel. + return VideoRenderData( + id: _taskId, + composition: VideoComposition( + canvasSize: _canvas, + layers: [ + VideoLayer( + clips: [ + VideoSegment( + video: EditorVideo.asset(kVideoEditorExampleAssetWorldPath), + endTime: const Duration(seconds: 3), + ), + ], + ), + VideoLayer( + clips: [VideoSegment(video: source)], + chromaKey: ChromaKey( + color: _keyColor, + similarity: _similarity, + smoothness: _smoothness, + spill: _spill, + ), + ), + ], + ), + ); + } + } + + /// Measures the screen in the source and adopts the result, so the sliders + /// show what was found instead of a guess. + Future _autoDetect() async { + setState(() => _isDetecting = true); + try { + final detection = await ChromaKey.detect(_greenScreen); + if (!mounted) return; + setState(() { + _detection = detection; + _keyColor = detection.color; + _similarity = detection.similarity; + }); + } on ChromaKeyDetectionException catch (e) { + if (!mounted) return; + ScaffoldMessenger.of( + context, + ).showSnackBar(SnackBar(content: Text(e.message))); + } finally { + if (mounted) setState(() => _isDetecting = false); + } + } + + Future _render() async { + if (_isExporting) return; + + _taskId = DateTime.now().microsecondsSinceEpoch.toString(); + setState(() { + _isExporting = true; + _isPreviewInitialized = false; + }); + + await _disposePreview(); + + final data = _buildRenderData(_greenScreen); + + final directory = await getTemporaryDirectory(); + final now = DateTime.now().millisecondsSinceEpoch; + final outputPath = '${directory.path}/chroma_key_$now.mp4'; + + final sw = Stopwatch()..start(); + try { + await _pve.renderVideoToFile( + outputPath, + data, + nativeLogLevel: NativeLogLevel.debug, + ); + } on RenderCanceledException { + setState(() => _isExporting = false); + return; + } + _generationTime = sw.elapsed; + + _videoBytes = File(outputPath).readAsBytesSync(); + _isExporting = false; + + _controllerPreview = VideoPlayerController.file(File(outputPath)); + await _controllerPreview!.initialize(); + + // Checked before the Chewie controller exists, so a page popped during + // initialization never leaves an autoplaying player behind. + if (!mounted) { + await _disposePreview(); + return; + } + + _chewieControllerPreview = ChewieController( + videoPlayerController: _controllerPreview!, + autoPlay: true, + looping: true, + placeholder: Container(color: Colors.black), + ); + + setState(() => _isPreviewInitialized = true); + } + + Future _cancel() async { + if (!_supportsCancel) return; + await _pve.cancel(_taskId); + } + + Future _disposePreview() async { + await _chewieControllerPreview?.pause(); + _chewieControllerPreview?.dispose(); + _chewieControllerPreview = null; + final controller = _controllerPreview; + _controllerPreview = null; + await controller?.dispose(); + } + + @override + Widget build(BuildContext context) { + final bottom = MediaQuery.viewPaddingOf(context).bottom; + return Scaffold( + appBar: AppBar(title: const Text('Chroma Key')), + body: SingleChildScrollView( + padding: EdgeInsets.fromLTRB(16, 16, 16, 16 + bottom), + child: Column( + crossAxisAlignment: CrossAxisAlignment.stretch, + spacing: 20, + children: [ + if (_videoBytes != null) _buildPreview(), + if (_isExporting) + VideoRendererProgressPanel( + progressStream: _pve.progressStreamById(_taskId), + supportsCancel: _supportsCancel, + onCancel: _supportsCancel ? _cancel : null, + ) + else + _buildControls(), + NativeLogConsole(logStream: _pve.logStream), + ], + ), + ), + ); + } + + Widget _buildControls() { + return Column( + crossAxisAlignment: CrossAxisAlignment.stretch, + spacing: 12, + children: [ + const Text( + 'Source: a person in front of a lit studio green screen. The screen ' + 'is not evenly lit โ€” its darkest corners sit close to the default ' + 'similarity, so lowering that slider makes them survive the key.', + style: TextStyle(fontWeight: FontWeight.bold), + ), + OutlinedButton.icon( + onPressed: _isDetecting ? null : _autoDetect, + icon: _isDetecting + ? const SizedBox( + width: 16, + height: 16, + child: CircularProgressIndicator(strokeWidth: 2), + ) + : const Icon(Icons.colorize_outlined), + label: const Text('Measure the screen from the clip'), + ), + if (_detection != null) _buildDetectionSummary(_detection!), + const Text('Fill the removed screen with:'), + SegmentedButton<_Background>( + selected: {_background}, + onSelectionChanged: (v) => setState(() => _background = v.first), + segments: const [ + ButtonSegment( + value: _Background.color, + icon: Icon(Icons.format_color_fill_outlined), + label: Text('Color'), + ), + ButtonSegment( + value: _Background.image, + icon: Icon(Icons.image_outlined), + label: Text('Image'), + ), + ButtonSegment( + value: _Background.video, + icon: Icon(Icons.layers_outlined), + label: Text('Video'), + ), + ], + ), + if (_background == _Background.video) + const Text( + 'Transparent key on the upper layer of a VideoComposition, so the ' + 'video below shows through.', + style: TextStyle(fontSize: 12), + ), + _buildSlider( + label: 'similarity', + value: _similarity, + min: 0.01, + max: 0.6, + hint: 'Raise it if the screen survives, lower it if the subject goes', + onChanged: (v) => setState(() => _similarity = v), + ), + _buildSlider( + label: 'smoothness', + value: _smoothness, + min: 0, + max: 0.4, + hint: 'Width of the soft edge', + onChanged: (v) => setState(() => _smoothness = v), + ), + _buildSlider( + label: 'spill', + value: _spill, + min: 0, + max: 1, + hint: 'How much of the green cast is pulled off the subject', + onChanged: (v) => setState(() => _spill = v), + ), + FilledButton.icon( + onPressed: _render, + icon: const Icon(Icons.movie_creation_outlined), + label: const Text('Render with this key'), + ), + ], + ); + } + + Widget _buildDetectionSummary(ChromaKeyDetection d) { + final hex = d.color.toARGB32().toRadixString(16).toUpperCase(); + return Text( + 'Detected 0x$hex ยท spread ${d.spread.toStringAsFixed(3)} ยท ' + '${(d.coverage * 100).toStringAsFixed(0)}% of the border. ' + 'The SMPTE constant sits about 0.12 away from this โ€” margin the key ' + 'would have to spend instead of keeping the subject safe.', + style: const TextStyle(fontSize: 12), + ); + } + + Widget _buildSlider({ + required String label, + required double value, + required double min, + required double max, + required String hint, + required ValueChanged onChanged, + }) { + return Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text('$label: ${value.toStringAsFixed(2)}'), + Text(hint, style: const TextStyle(fontSize: 12)), + Slider(value: value, min: min, max: max, onChanged: onChanged), + ], + ); + } + + Widget _buildPreview() { + return Column( + crossAxisAlignment: CrossAxisAlignment.stretch, + spacing: 5, + children: [ + const Text('Output-Video'), + AspectRatio( + aspectRatio: _canvas.width / _canvas.height, + child: _isPreviewInitialized + ? Chewie(controller: _chewieControllerPreview!) + : const Center(child: CircularProgressIndicator()), + ), + Text( + 'Result: ${formatBytes(_videoBytes!.lengthInBytes)} bytes ' + 'in ${_generationTime.inMilliseconds}ms', + ), + ], + ); + } +} diff --git a/example/lib/main.dart b/example/lib/main.dart index bd5944c9..c652e085 100644 --- a/example/lib/main.dart +++ b/example/lib/main.dart @@ -7,6 +7,7 @@ import 'package:pro_video_editor_example/features/editor/pages/video_editor_grou import 'features/audio/audio_extract_example_page.dart'; import 'features/audio/audio_merge_example_page.dart'; +import 'features/chroma_key/chroma_key_example_page.dart'; import 'features/metadata/video_metadata_example_page.dart'; import 'features/render/video_renderer_page.dart'; import 'features/split/split_example_page.dart'; @@ -84,6 +85,11 @@ class _HomePageState extends State { title: 'Video-Renderer', pageBuilder: () => const VideoRendererPage(), ), + _ExampleListItem( + icon: Icons.filter_center_focus, + title: 'Chroma Key (Greenscreen)', + pageBuilder: () => const ChromaKeyExamplePage(), + ), _ExampleListItem( icon: Icons.burst_mode_outlined, title: 'Stop-Motion', diff --git a/example/macos/RunnerTests/RunnerTests.swift b/example/macos/RunnerTests/RunnerTests.swift index 2350453c..7f930767 100644 --- a/example/macos/RunnerTests/RunnerTests.swift +++ b/example/macos/RunnerTests/RunnerTests.swift @@ -542,3 +542,244 @@ enum ThumbnailTimestampFixture { return bestIndex } } + +// MARK: - Chroma key + +/// Cross-platform parity guard for the chroma-key formula. +/// +/// The golden table below is duplicated verbatim in the Kotlin test +/// (`android/src/test/.../ChromaKeyMathTest.kt`). Both run it against their own +/// implementation โ€” Swift's `chromaKeyed(r:g:b:_:)`, which is baked into the +/// Core Image color cube, and Kotlin's `ChromaKeyMath`, which the GLSL shader +/// mirrors. If either platform drifts, one of these two tests fails immediately +/// instead of the difference surfacing later as "the key looks slightly +/// different on Android". +/// +/// **When you change the formula, regenerate both tables.** +class ChromaKeyMathTests: XCTestCase { + + /// The config the golden table was computed for: SMPTE green, defaults. + private let config = ChromaKeyConfig( + keyR: Double(0x00) / 255.0, + keyG: Double(0xB1) / 255.0, + keyB: Double(0x40) / 255.0, + similarity: 0.15, + smoothness: 0.08, + spill: 0.5, + backgroundColor: -1, + backgroundImageData: nil + ) + + private let tolerance = 1e-4 + + /// The same key with a different despill strength. + private func withSpill(_ spill: Double) -> ChromaKeyConfig { + ChromaKeyConfig( + keyR: config.keyR, keyG: config.keyG, keyB: config.keyB, + similarity: config.similarity, smoothness: config.smoothness, + spill: spill, backgroundColor: -1, backgroundImageData: nil) + } + + private struct Golden { + let name: String + let r, g, b: Double + let outR, outG, outB, alpha: Double + } + + private let golden: [Golden] = [ + // The key color itself and its neighbourhood: removed completely. + Golden(name: "key color", r: 0.0, g: 0.694118, b: 0.25098, + outR: 0.218029, outG: 0.565088, outB: 0.343519, alpha: 0.0), + Golden(name: "near key", r: 0.05, g: 0.72, b: 0.28, + outR: 0.261122, outG: 0.595058, outB: 0.369608, alpha: 0.0), + Golden(name: "bright screen", r: 0.35, g: 0.9, b: 0.5, + outR: 0.525426, outG: 0.796183, outB: 0.574457, alpha: 0.0), + // Half-lit screen: past the default similarity, so only mostly removed. + // This is the documented brightness sensitivity, pinned on purpose. + Golden(name: "dim screen 50%", r: 0.0, g: 0.347059, b: 0.12549, + outR: 0.109015, outG: 0.282544, outB: 0.17176, alpha: 0.081671), + // Neutrals sit at the chroma origin, far from any saturated key. + Golden(name: "black", r: 0.0, g: 0.0, b: 0.0, + outR: 0.0, outG: 0.0, outB: 0.0, alpha: 1.0), + Golden(name: "mid gray", r: 0.5, g: 0.5, b: 0.5, + outR: 0.5, outG: 0.5, outB: 0.5, alpha: 1.0), + Golden(name: "white", r: 1.0, g: 1.0, b: 1.0, + outR: 1.0, outG: 1.0, outB: 1.0, alpha: 1.0), + // Skin tone โ€” the calibration anchor quoted in the Dart docs. + Golden(name: "skin tone", r: 0.86, g: 0.65, b: 0.53, + outR: 0.86, outG: 0.65, outB: 0.53, alpha: 1.0), + Golden(name: "pure red", r: 1.0, g: 0.0, b: 0.0, + outR: 1.0, outG: 0.0, outB: 0.0, alpha: 1.0), + Golden(name: "pure blue", r: 0.0, g: 0.0, b: 1.0, + outR: 0.0, outG: 0.0, outB: 1.0, alpha: 1.0), + // Kept, but despilled: the green cast is pulled out at full alpha. + Golden(name: "green-spilled gray", r: 0.55, g: 0.75, b: 0.55, + outR: 0.616767, outG: 0.710487, outB: 0.578338, alpha: 1.0), + // Leans away from the key hue, so despill leaves it alone. + Golden(name: "magenta", r: 0.8, g: 0.2, b: 0.8, + outR: 0.8, outG: 0.2, outB: 0.8, alpha: 1.0), + ] + + func testGoldenTableMatchesTheSharedFormula() { + for row in golden { + let out = chromaKeyed(r: row.r, g: row.g, b: row.b, config) + XCTAssertEqual(out.r, row.outR, accuracy: tolerance, "\(row.name): r") + XCTAssertEqual(out.g, row.outG, accuracy: tolerance, "\(row.name): g") + XCTAssertEqual(out.b, row.outB, accuracy: tolerance, "\(row.name): b") + XCTAssertEqual(out.a, row.alpha, accuracy: tolerance, "\(row.name): alpha") + } + } + + func testKeyColorIsRemovedCompletely() { + let out = chromaKeyed(r: config.keyR, g: config.keyG, b: config.keyB, config) + XCTAssertEqual(out.a, 0.0, accuracy: tolerance) + } + + func testSoftEdgeProducesPartialAlpha() { + // Walk the key color toward neutral gray and collect the alpha ramp. + let ramp: [Double] = (0...60).map { step in + let t = Double(step) / 60.0 + return chromaKeyed( + r: config.keyR + (0.5 - config.keyR) * t, + g: config.keyG + (0.5 - config.keyG) * t, + b: config.keyB + (0.5 - config.keyB) * t, + config + ).a + } + + XCTAssertTrue(ramp.contains(0.0), "expected a fully keyed sample") + XCTAssertTrue(ramp.contains(1.0), "expected a fully opaque sample") + XCTAssertTrue( + ramp.contains { $0 > 0.01 && $0 < 0.99 }, + "expected a soft edge, but alpha jumped straight from 0 to 1") + } + + func testSpillPullsTheKeyCastOutWithoutDarkening() { + let noSpill = withSpill(0.0) + let fullSpill = withSpill(1.0) + + let without = chromaKeyed(r: 0.55, g: 0.75, b: 0.55, noSpill) + let with = chromaKeyed(r: 0.55, g: 0.75, b: 0.55, fullSpill) + + let castBefore = without.g - max(without.r, without.b) + let castAfter = with.g - max(with.r, with.b) + XCTAssertLessThan(castAfter, castBefore, "despill did not reduce the green cast") + + // Luma is preserved, so despill never darkens the subject. + XCTAssertEqual( + lumaOf(r: without.r, g: without.g, b: without.b), + lumaOf(r: with.r, g: with.g, b: with.b), + accuracy: 1e-3) + } + + func testMatteIsBrightnessSensitiveAsDocumented() { + // Cb/Cr scale with brightness, so a dimly lit patch of the screen sits + // closer to neutral and further from the key point. The default similarity + // covers roughly 55%..100% of the reference brightness. Pinned because the + // Dart docs promise exactly this. + func screenAt(_ fraction: Double) -> Double { + chromaKeyed( + r: config.keyR * fraction, g: config.keyG * fraction, + b: config.keyB * fraction, config + ).a + } + + XCTAssertEqual(screenAt(1.0), 0.0, accuracy: tolerance) + XCTAssertEqual(screenAt(0.55), 0.0, accuracy: tolerance) + XCTAssertGreaterThan(screenAt(0.5), 0.0, "a half-lit screen should start to survive") + } + + func testNeutralKeyColorDisablesDespillInsteadOfDividingByZero() { + let gray = ChromaKeyConfig( + keyR: 0.5, keyG: 0.5, keyB: 0.5, similarity: 0.15, smoothness: 0.08, + spill: 1.0, backgroundColor: -1, backgroundImageData: nil) + + XCTAssertEqual(gray.keyDirection.cb, 0.0, accuracy: tolerance) + XCTAssertEqual(gray.keyDirection.cr, 0.0, accuracy: tolerance) + + let out = chromaKeyed(r: 0.8, g: 0.2, b: 0.4, gray) + XCTAssertFalse(out.r.isNaN || out.g.isNaN || out.b.isNaN || out.a.isNaN) + } + + /// The cube is what actually runs on Apple, so verify it carries the same + /// numbers **and** that its entries are premultiplied, which `CIColorCube` + /// requires and which is the one place Apple and Android must differ. + func testCubeIsPremultipliedAndMatchesTheFormula() throws { + let size = 33 + let data = try XCTUnwrap( + generateChromaLUTData(chroma: config, size: size)) + + XCTAssertEqual(data.count, size * size * size * 4 * MemoryLayout.size) + + let floats: [Float] = data.withUnsafeBytes { Array($0.bindMemory(to: Float.self)) } + + // Walk the cube's own sample grid, so no interpolation is involved. + for bi in stride(from: 0, to: size, by: 8) { + for gi in stride(from: 0, to: size, by: 8) { + for ri in stride(from: 0, to: size, by: 8) { + let rf = Double(ri) / Double(size - 1) + let gf = Double(gi) / Double(size - 1) + let bf = Double(bi) / Double(size - 1) + let expected = chromaKeyed(r: rf, g: gf, b: bf, config) + + let offset = (bi * size * size + gi * size + ri) * 4 + XCTAssertEqual( + Double(floats[offset]), expected.r * expected.a, accuracy: 1e-5) + XCTAssertEqual( + Double(floats[offset + 1]), expected.g * expected.a, accuracy: 1e-5) + XCTAssertEqual( + Double(floats[offset + 2]), expected.b * expected.a, accuracy: 1e-5) + XCTAssertEqual(Double(floats[offset + 3]), expected.a, accuracy: 1e-5) + } + } + } + } + + /// A fully keyed entry must be `(0, 0, 0, 0)`, not `(rgb, 0)` โ€” Core Image + /// would otherwise composite the key color back in at the edges. + func testFullyKeyedCubeEntriesAreZero() throws { + let size = 33 + let data = try XCTUnwrap( + generateChromaLUTData(chroma: config, size: size)) + let floats: [Float] = data.withUnsafeBytes { Array($0.bindMemory(to: Float.self)) } + + // Nearest grid point to the key color. + let ri = Int((config.keyR * Double(size - 1)).rounded()) + let gi = Int((config.keyG * Double(size - 1)).rounded()) + let bi = Int((config.keyB * Double(size - 1)).rounded()) + let offset = (bi * size * size + gi * size + ri) * 4 + + XCTAssertEqual(floats[offset + 3], 0.0, accuracy: 1e-5, "key should be fully removed") + XCTAssertEqual(floats[offset], 0.0, accuracy: 1e-5, "premultiplied red must be 0") + XCTAssertEqual(floats[offset + 1], 0.0, accuracy: 1e-5, "premultiplied green must be 0") + XCTAssertEqual(floats[offset + 2], 0.0, accuracy: 1e-5, "premultiplied blue must be 0") + } + + /// Two different background images of the same byte length must not share a + /// cache entry, or the second clip renders the first one's backdrop. + func testCacheKeySeparatesEqualLengthBackgrounds() { + func withBackground(_ bytes: [UInt8]) -> ChromaKeyConfig { + ChromaKeyConfig( + keyR: config.keyR, keyG: config.keyG, keyB: config.keyB, + similarity: config.similarity, smoothness: config.smoothness, + spill: config.spill, backgroundColor: -1, + backgroundImageData: Data(bytes)) + } + + let a = withBackground(Array(repeating: 0x11, count: 128)) + var bBytes = Array(repeating: UInt8(0x11), count: 128) + bBytes[0] = 0x22 + let b = withBackground(bBytes) + + XCTAssertEqual(a.backgroundImageData?.count, b.backgroundImageData?.count) + XCTAssertNotEqual(a.cacheKey, b.cacheKey) + XCTAssertEqual(a.cacheKey, withBackground(Array(repeating: 0x11, count: 128)).cacheKey) + } + + /// A key with no background image at all keeps a stable, distinct key. + func testCacheKeyIsStableWithoutABackgroundImage() { + XCTAssertEqual(config.cacheKey, config.cacheKey) + XCTAssertNotEqual(config.cacheKey, withSpill(0.9).cacheKey) + } + +} diff --git a/lib/core/models/video/chroma_key_model.dart b/lib/core/models/video/chroma_key_model.dart new file mode 100644 index 00000000..499a5ea3 --- /dev/null +++ b/lib/core/models/video/chroma_key_model.dart @@ -0,0 +1,517 @@ +// ignore_for_file: public_member_api_docs, sort_constructors_first +import 'dart:convert'; +import 'dart:typed_data'; +import 'dart:ui'; + +import 'package:pro_video_editor/pro_video_editor.dart'; +import 'package:pro_video_editor/shared/utils/parser/double_parser.dart'; +import 'package:pro_video_editor/shared/utils/parser/int_parser.dart'; + +/// Removes a solid-colored background (a "green screen") from the video. +/// +/// Pixels whose hue is close to [color] are made transparent, with a soft edge +/// so the matte does not alias, and the key's color cast is pulled back out of +/// the pixels that remain ([spill]). +/// +/// ## The algorithm +/// +/// Both platforms evaluate the exact same formula, so a key tuned on one +/// renders the same on the other. Every value is gamma-encoded RGB in `0..1`: +/// +/// ```text +/// Y = 0.299ยทr + 0.587ยทg + 0.114ยทb +/// Cb = -0.168736ยทr - 0.331264ยทg + 0.5ยทb +/// Cr = 0.5ยทr - 0.418688ยทg - 0.081312ยทb +/// +/// d = distance((Cb, Cr), (Cb_key, Cr_key)) +/// alpha = smoothstep(similarity, similarity + smoothness, d) +/// ``` +/// +/// The distance is measured in the **Cb/Cr chroma plane**, which is a position, +/// not a pure hue: Cb and Cr scale with brightness, so a dimly lit patch of the +/// screen sits closer to neutral and therefore further from the key point. The +/// default [similarity] covers roughly 40%โ€“100% of the screen's reference +/// brightness; a badly lit screen needs a wider one. This is the same behaviour +/// as FFmpeg's `chromakey` and OBS. +/// +/// ## What the keyed area becomes +/// +/// - [backgroundImage] โ€” the image fills it (stretched to the frame). +/// - [backgroundColor] โ€” that color fills it. +/// - neither โ€” the area is **transparent**, so a lower [VideoLayer] of a +/// [VideoComposition] shows through. This is how you put a video behind a +/// green screen. +/// +/// **Important:** H.264 and HEVC carry no alpha channel. In the single-track +/// [VideoRenderData.videoSegments] path there is no layer underneath, so a key +/// without a background is flattened to opaque **black**. Set a background, or +/// use [VideoRenderData.composition] and place the keyed clip on a layer above +/// another one. +/// +/// ## Where it can be set +/// +/// On [VideoRenderData], [VideoLayer] and [VideoSegment]. The most specific one +/// wins for a given clip โ€” `segment โ†’ layer โ†’ global` โ€” they are not merged. +/// +/// ## Limits +/// +/// - No time ranges (unlike [ColorFilter]). Split the clip into two +/// [VideoSegment]s when the key has to change mid-clip. +/// - Ignored inside an overlap [ClipTransition] (dissolve/slide/push/wipe) when +/// the two clips at that boundary do not carry the same key โ€” the blend is +/// pre-rendered from the raw sources and is emitted unkeyed with a warning. +/// - No matte choke/erode and no light wrap. +/// - Combining a transparent key with [VideoRenderData.blur] is not +/// recommended: the blur bleeds color out of the transparent pixels. +/// - Ignored on Web, Windows and Linux, which have no render pipeline. +class ChromaKey { + /// Creates a [ChromaKey]. + const ChromaKey({ + this.color = const Color(0xFF00B140), + this.similarity = 0.20, + this.smoothness = 0.08, + this.spill = 0.5, + this.backgroundColor, + this.backgroundImage, + }) : assert( + similarity > 0 && similarity <= 1, + '[similarity] must be greater than 0 and at most 1', + ), + assert( + smoothness >= 0 && smoothness <= 1, + '[smoothness] must be between 0 and 1', + ), + assert(spill >= 0 && spill <= 1, '[spill] must be between 0 and 1'), + assert( + backgroundColor == null || backgroundImage == null, + 'Provide at most one of [backgroundColor] or [backgroundImage]', + ); + + /// A key tuned for a **green** screen. + /// + /// Identical to the default constructor; it exists so the counterpart + /// [ChromaKey.blueScreen] reads as a deliberate choice rather than an + /// afterthought. + /// + /// Prefer [autoDetect] when you have the footage: measuring the screen beats + /// any constant, because it removes the offset between the paint and what the + /// camera recorded. + const ChromaKey.greenScreen({ + this.similarity = 0.20, + this.smoothness = 0.08, + this.spill = 0.5, + this.backgroundColor, + this.backgroundImage, + }) : color = const Color(0xFF00B140), + assert( + similarity > 0 && similarity <= 1, + '[similarity] must be greater than 0 and at most 1', + ), + assert( + smoothness >= 0 && smoothness <= 1, + '[smoothness] must be between 0 and 1', + ), + assert(spill >= 0 && spill <= 1, '[spill] must be between 0 and 1'), + assert( + backgroundColor == null || backgroundImage == null, + 'Provide at most one of [backgroundColor] or [backgroundImage]', + ); + + /// A key tuned for a **blue** screen. + /// + /// Blue is not green with a different hue โ€” the two fail on opposite things. + /// Blue separates skin better (`0.41`โ€“`0.46` versus green's `0.38`โ€“`0.42`), + /// which is why it was the film standard, but denim (`0.19`), blue eyes + /// (`0.20`) and a light blue shirt (`0.21`) all crowd it. So this preset + /// keys **tighter** than the green one and despills more gently, or a pair of + /// jeans disappears along with the background. + /// + /// The price of the tighter [similarity] is that the screen has to be lit + /// more evenly: `0.12` reaches down to about 64% of its brightest patch, + /// where green's `0.20` reaches 40%. If parts of the screen survive, raise + /// [similarity] โ€” but check the subject's wardrobe when you do. + const ChromaKey.blueScreen({ + this.similarity = 0.12, + this.smoothness = 0.08, + this.spill = 0.25, + this.backgroundColor, + this.backgroundImage, + }) : color = const Color(0xFF0047BB), + assert( + similarity > 0 && similarity <= 1, + '[similarity] must be greater than 0 and at most 1', + ), + assert( + smoothness >= 0 && smoothness <= 1, + '[smoothness] must be between 0 and 1', + ), + assert(spill >= 0 && spill <= 1, '[spill] must be between 0 and 1'), + assert( + backgroundColor == null || backgroundImage == null, + 'Provide at most one of [backgroundColor] or [backgroundImage]', + ); + + /// Measures the screen in [video] and returns a key tuned to it. + /// + /// This is the most reliable way to build a key, and it is hue-agnostic โ€” + /// green, blue or anything else, as long as it is saturated. + /// + /// A constant [color] is always a compromise. Paint, fabric, lighting and the + /// camera's color science all shift the recorded screen away from it, and + /// every bit of that shift has to be absorbed by a wider [similarity] โ€” which + /// is exactly the margin that protects the subject. Measured on a real studio + /// clip: against the SMPTE constant the screen sat `0.10`โ€“`0.20` away and + /// needed `similarity: 0.20`; against its own measured color it sat within + /// `0.05` and `0.10` was plenty. Same footage, twice the headroom. + /// + /// Frames are sampled through the existing thumbnail pipeline, so this costs + /// one decode and no render. + /// + /// **The screen has to reach the frame border**, because that is what lets + /// the subject be ignored โ€” only a ring around the edge is sampled. A frame + /// that also shows the studio around the screen will throw a + /// [ChromaKeyDetectionException]; crop it first, or set the key by hand. + /// + /// ```dart + /// final key = await ChromaKey.autoDetect( + /// myClip, + /// backgroundColor: Colors.black, + /// ); + /// ``` + /// + /// Throws [ChromaKeyDetectionException] when no usable screen is found. + static Future autoDetect( + EditorVideo video, { + List? timestamps, + double smoothness = 0.08, + double spill = 0.5, + Color? backgroundColor, + EditorLayerImage? backgroundImage, + }) async { + final detection = await detect(video, timestamps: timestamps); + return ChromaKey( + color: detection.color, + similarity: detection.similarity, + smoothness: smoothness, + spill: spill, + backgroundColor: backgroundColor, + backgroundImage: backgroundImage, + ); + } + + /// Measures the screen in [video] without building a key. + /// + /// Use this when you want to show the measurement to the user โ€” the detected + /// color, how evenly the screen is lit ([ChromaKeyDetection.spread]) and how + /// much of the border it covers โ€” before committing to it. + static Future detect( + EditorVideo video, { + List? timestamps, + }) async { + final editor = ProVideoEditor.instance; + final metadata = await editor.getMetadata(video); + + // Sample across the clip so a subject that briefly touches an edge cannot + // decide the result on its own. + final duration = metadata.duration; + final at = timestamps ?? [duration * 0.25, duration * 0.5, duration * 0.75]; + + // Small, and at the source aspect ratio: `cover` then neither crops nor + // pads, so the sampled ring really is the frame's own border. + const sampleHeight = 160.0; + final aspect = metadata.resolution.aspectRatio; + final size = Size((sampleHeight * aspect).roundToDouble(), sampleHeight); + + final frames = await editor.getThumbnails( + ThumbnailConfigs( + video: video, + outputFormat: ThumbnailFormat.png, + timestamps: at, + outputSize: size, + boxFit: ThumbnailBoxFit.cover, + ), + ); + + if (frames.isEmpty) { + throw const ChromaKeyDetectionException( + 'Could not extract a frame from the video', + ); + } + + final buffers = []; + int? width; + int? height; + for (final frame in frames) { + final codec = await instantiateImageCodec(frame); + try { + final image = (await codec.getNextFrame()).image; + try { + final data = await image.toByteData(); + if (data == null) continue; + width ??= image.width; + height ??= image.height; + if (image.width != width || image.height != height) continue; + buffers.add(data.buffer.asUint8List()); + } finally { + image.dispose(); + } + } finally { + codec.dispose(); + } + } + + if (buffers.isEmpty || width == null || height == null) { + throw const ChromaKeyDetectionException('Could not decode any frame'); + } + + return ChromaKeyDetector.fromFrames(buffers, width: width, height: height); + } + + /// The screen color to remove. + /// + /// **Default**: `0xFF00B140`, the SMPTE "chroma key green" most physical + /// screens are painted in. + /// + /// Both the hue **and** the brightness matter. The key point is this color's + /// un-normalized Cb/Cr projection, and Cb/Cr scale with brightness, so a key + /// color much darker than the recorded screen sits closer to neutral and can + /// leave the whole screen outside [similarity]. Prefer [autoDetect] over + /// picking a swatch by eye. + /// + /// ## Blue screens + /// + /// Any saturated hue works; nothing here is specific to green. A blue screen + /// (`0xFF0047BB`) keys just as cleanly and is in fact *safer for skin*, which + /// sits `0.41`โ€“`0.46` from it versus `0.38`โ€“`0.42` from green. + /// + /// The trade is what the subject wears. Measured against SMPTE blue: + /// + /// | | distance | + /// |---|---| + /// | denim / jeans | `0.19` | + /// | blue eyes | `0.20` | + /// | light blue shirt | `0.21` | + /// + /// All three sit at or below the default [similarity] of `0.20`, which is + /// tuned for green โ€” so on a blue screen denim and a light blue shirt are + /// keyed away along with the background. Drop [similarity] to about `0.12` + /// for blue, and lower [spill] as well when the subject wears blue, or the + /// despill pulls the blue out of the garment too (denim measured + /// `(59,91,140)` โ†’ `(92,79,98)` at the default `spill`). Skin is unaffected + /// by despill on a blue screen, since it leans away from that hue. + /// + /// A lower [similarity] needs a more evenly lit screen: `0.12` only covers + /// down to about 65% of the screen's reference brightness. + /// + /// Avoid the darker "digital blue" (`0xFF1A46A8`) โ€” its chroma magnitude is + /// only `0.25` versus `0.33`, so every subject color crowds it, a white shirt + /// included. + final Color color; + + /// How far from [color] a pixel may sit and still be removed completely. + /// + /// Measured as a distance in the Cb/Cr chroma plane. Some anchors for + /// calibration, all against the default SMPTE green (whose own chroma + /// magnitude is `0.33`): + /// + /// - skin tones sit `0.38`โ€“`0.42` away, so the default keeps faces safe by a + /// wide margin; + /// - the same screen lit at 40% of its brightest patch sits `0.20` away โ€” + /// exactly at the default threshold. A real, evenly lit studio screen + /// measured `0.18` in its darkest corners, which is why the default is not + /// tighter; + /// - a green-ish garment (an olive shirt) sits around `0.26`, so that is the + /// case where raising this too far starts eating the subject. + /// + /// So: raise it when shadowed or unevenly lit parts of the screen survive the + /// key, and lower it when the subject starts disappearing. + /// + /// **Default**: `0.20` + final double similarity; + + /// The width of the soft ramp just beyond [similarity]. + /// + /// Pixels between `similarity` and `similarity + smoothness` fade from fully + /// removed to fully opaque, which is what keeps hair and motion blur from + /// turning into a jagged cutout. + /// + /// `0` asks for a hard edge. Note that Apple evaluates the key through a + /// 33ยณ color cube with trilinear interpolation, so a perfectly hard edge is + /// not reachable there โ€” about one cube cell of ramp always remains, which in + /// practice just anti-aliases the matte. Values from `0.05` up span several + /// cells and are smooth on both platforms. + /// + /// **Default**: `0.08` + final double smoothness; + + /// How strongly the key's color cast is removed from the pixels that stay. + /// + /// A green screen bounces green light onto everything near it, leaving a tint + /// along the subject's edges. This pulls the chroma component that points + /// toward [color] back out, without changing the pixel's brightness. + /// + /// - `0.0`: off + /// - `1.0`: neutralize the key hue completely + /// + /// **Default**: `0.5` + final double spill; + + /// A solid color to put behind the subject. + /// + /// Mutually exclusive with [backgroundImage]. When both are `null` the keyed + /// area is transparent โ€” see the class docs for what that means per path. + /// + /// **Must be fully opaque.** Only the RGB channels are used: the fill is a + /// replacement for the removed screen, not a tint over it. A translucent + /// background is not expressible identically on both renderers, so it is + /// rejected rather than silently rendered two different ways. For a + /// see-through result leave this `null` and put the backdrop on a lower + /// [VideoLayer]. + final Color? backgroundColor; + + /// An image to put behind the subject, stretched to fill the frame. + /// + /// Mutually exclusive with [backgroundColor]. + final EditorLayerImage? backgroundImage; + + /// Whether the keyed area is left transparent rather than filled. + bool get isTransparent => backgroundColor == null && backgroundImage == null; + + /// Converts this key to a map for platform channel communication. + /// + /// Resolves [backgroundImage] to bytes, so this is asynchronous. + Future> toAsyncMap() async { + assert( + backgroundColor == null || backgroundColor!.a == 1.0, + '[backgroundColor] must be fully opaque โ€” only its RGB channels are ' + 'used, and both renderers fill the keyed area solidly. For a ' + 'see-through result leave it null and put the backdrop on a lower ' + 'VideoLayer of a VideoComposition.', + ); + return { + 'keyColor': color.toARGB32(), + 'similarity': similarity, + 'smoothness': smoothness, + 'spill': spill, + 'bgColor': backgroundColor?.toARGB32(), + 'bgImageData': await backgroundImage?.safeByteArray(), + }; + } + + /// Creates a copy with updated values. + /// + /// [backgroundColor] and [backgroundImage] are mutually exclusive, so passing + /// one **drops** the other โ€” that is what lets a key switch from an image + /// background to a color one. Passing neither keeps whichever is already set. + /// Pass [removeBackground] to clear both and leave the keyed area + /// transparent, which no combination of the other arguments can express. + ChromaKey copyWith({ + Color? color, + double? similarity, + double? smoothness, + double? spill, + Color? backgroundColor, + EditorLayerImage? backgroundImage, + bool removeBackground = false, + }) { + assert( + backgroundColor == null || backgroundImage == null, + 'Provide at most one of [backgroundColor] or [backgroundImage]', + ); + assert( + !removeBackground || (backgroundColor == null && backgroundImage == null), + '[removeBackground] clears the background, so it cannot be combined ' + 'with [backgroundColor] or [backgroundImage]', + ); + + final Color? nextColor; + final EditorLayerImage? nextImage; + if (removeBackground) { + nextColor = null; + nextImage = null; + } else if (backgroundColor != null) { + nextColor = backgroundColor; + nextImage = null; + } else if (backgroundImage != null) { + nextColor = null; + nextImage = backgroundImage; + } else { + nextColor = this.backgroundColor; + nextImage = this.backgroundImage; + } + + return ChromaKey( + color: color ?? this.color, + similarity: similarity ?? this.similarity, + smoothness: smoothness ?? this.smoothness, + spill: spill ?? this.spill, + backgroundColor: nextColor, + backgroundImage: nextImage, + ); + } + + Map toMap() { + return { + 'color': color.toARGB32(), + 'similarity': similarity, + 'smoothness': smoothness, + 'spill': spill, + 'backgroundColor': backgroundColor?.toARGB32(), + 'backgroundImage': backgroundImage?.toMap(), + }; + } + + factory ChromaKey.fromMap(Map map) { + return ChromaKey( + color: Color(safeParseInt(map['color'], fallback: 0xFF00B140)), + similarity: safeParseDouble(map['similarity'], fallback: 0.20), + smoothness: safeParseDouble(map['smoothness'], fallback: 0.08), + spill: safeParseDouble(map['spill'], fallback: 0.5), + backgroundColor: map['backgroundColor'] != null + ? Color(safeParseInt(map['backgroundColor'])) + : null, + backgroundImage: map['backgroundImage'] != null + ? EditorLayerImage.fromMap( + map['backgroundImage'] as Map, + ) + : null, + ); + } + + String toJson() => json.encode(toMap()); + + factory ChromaKey.fromJson(String source) => + ChromaKey.fromMap(json.decode(source) as Map); + + @override + String toString() { + return 'ChromaKey(color: $color, ' + 'similarity: $similarity, ' + 'smoothness: $smoothness, ' + 'spill: $spill, ' + 'backgroundColor: $backgroundColor, ' + 'backgroundImage: $backgroundImage)'; + } + + @override + bool operator ==(covariant ChromaKey other) { + if (identical(this, other)) return true; + + return other.color == color && + other.similarity == similarity && + other.smoothness == smoothness && + other.spill == spill && + other.backgroundColor == backgroundColor && + other.backgroundImage == backgroundImage; + } + + @override + int get hashCode { + return color.hashCode ^ + similarity.hashCode ^ + smoothness.hashCode ^ + spill.hashCode ^ + backgroundColor.hashCode ^ + backgroundImage.hashCode; + } +} diff --git a/lib/core/models/video/editor_video_model.dart b/lib/core/models/video/editor_video_model.dart index 4d59b561..2e413e69 100644 --- a/lib/core/models/video/editor_video_model.dart +++ b/lib/core/models/video/editor_video_model.dart @@ -5,6 +5,10 @@ import '/core/platform/path/path_provider_helper.dart'; import '/shared/utils/converters.dart'; import '/shared/utils/file_constructor_utils.dart'; +/// Distinguishes temp files written within the same millisecond, so several +/// sources resolved concurrently cannot collide on one path. +int _tempFileCounter = 0; + /// A model that encapsulates various ways to load and represent a video. /// /// This class supports videos from in-memory bytes, file system, network, @@ -165,9 +169,15 @@ class EditorVideo { final directory = await getTemporaryDirectory(); final now = DateTime.now().millisecondsSinceEpoch; + // A per-call counter on top of the timestamp. Several sources are + // routinely resolved concurrently (VideoRenderData.toAsyncMap awaits all + // of its segments together), and a millisecond is not fine enough to tell + // them apart โ€” two of them would pick the same path, write over each + // other and end up as one clip. + final unique = _tempFileCounter++; // Preserve original file extension for proper format detection final extension = _getFileExtension(); - filePath = '${directory.path}/media_$now.$extension'; + filePath = '${directory.path}/media_${now}_$unique.$extension'; } switch (typePreferredFile) { diff --git a/lib/core/models/video/video_layer_model.dart b/lib/core/models/video/video_layer_model.dart index 65e38f58..f688ca4b 100644 --- a/lib/core/models/video/video_layer_model.dart +++ b/lib/core/models/video/video_layer_model.dart @@ -18,9 +18,16 @@ import 'package:pro_video_editor/shared/utils/parser/double_parser.dart'; /// clip provides its own [VideoSegment.transform], placed using [transform]. class VideoLayer { /// Creates a [VideoLayer] from a list of [clips]. - const VideoLayer({required this.clips, this.opacity = 1.0, this.transform}) - : assert(clips.length > 0, 'A layer must contain at least one clip'), - assert(opacity >= 0 && opacity <= 1, '[opacity] must be between 0 and 1'); + const VideoLayer({ + required this.clips, + this.opacity = 1.0, + this.transform, + this.chromaKey, + }) : assert(clips.length > 0, 'A layer must contain at least one clip'), + assert( + opacity >= 0 && opacity <= 1, + '[opacity] must be between 0 and 1', + ); /// The time-ordered sequence of clips on this layer. final List clips; @@ -37,9 +44,21 @@ class VideoLayer { /// clips fill the entire canvas unless they define their own transform. final SegmentTransform? transform; + /// Default chroma key for the clips on this layer. + /// + /// This is a **default for the clips**, not a post-composite effect: each + /// clip is keyed on its own source frame, before it is placed on the canvas. + /// A clip with its own [VideoSegment.chromaKey] overrides this; when neither + /// is set, the clip falls back to [VideoRenderData.chromaKey]. + /// + /// Since the keyed area is transparent by default, the layer below shows + /// through โ€” which is how you put a video behind a green screen. + final ChromaKey? chromaKey; + /// Converts this layer to a map for platform channel communication. /// - /// Resolves each clip's input path, so this is asynchronous. + /// Resolves each clip's input path and any chroma-key background image, so + /// this is asynchronous. Future> toAsyncMap() async { assert( clips.every((c) => c.transition == null), @@ -57,6 +76,7 @@ class VideoLayer { 'clips': await Future.wait(clips.map((clip) => clip.toAsyncMap())), 'opacity': opacity, 'transform': transform?.toMap(), + 'chromaKey': await chromaKey?.toAsyncMap(), }; } @@ -65,11 +85,13 @@ class VideoLayer { List? clips, double? opacity, SegmentTransform? transform, + ChromaKey? chromaKey, }) { return VideoLayer( clips: clips ?? this.clips, opacity: opacity ?? this.opacity, transform: transform ?? this.transform, + chromaKey: chromaKey ?? this.chromaKey, ); } @@ -78,6 +100,7 @@ class VideoLayer { 'clips': clips.map((x) => x.toMap()).toList(), 'opacity': opacity, 'transform': transform?.toMap(), + 'chromaKey': chromaKey?.toMap(), }; } @@ -92,6 +115,9 @@ class VideoLayer { transform: map['transform'] != null ? SegmentTransform.fromMap(map['transform'] as Map) : null, + chromaKey: map['chromaKey'] != null + ? ChromaKey.fromMap(map['chromaKey'] as Map) + : null, ); } @@ -102,7 +128,8 @@ class VideoLayer { @override String toString() => - 'VideoLayer(clips: $clips, opacity: $opacity, transform: $transform)'; + 'VideoLayer(clips: $clips, opacity: $opacity, transform: $transform, ' + 'chromaKey: $chromaKey)'; @override bool operator ==(covariant VideoLayer other) { @@ -110,9 +137,14 @@ class VideoLayer { return listEquals(other.clips, clips) && other.opacity == opacity && - other.transform == transform; + other.transform == transform && + other.chromaKey == chromaKey; } @override - int get hashCode => clips.hashCode ^ opacity.hashCode ^ transform.hashCode; + int get hashCode => + clips.hashCode ^ + opacity.hashCode ^ + transform.hashCode ^ + chromaKey.hashCode; } diff --git a/lib/core/models/video/video_render_data_model.dart b/lib/core/models/video/video_render_data_model.dart index 4d174d33..d9c0163e 100644 --- a/lib/core/models/video/video_render_data_model.dart +++ b/lib/core/models/video/video_render_data_model.dart @@ -36,6 +36,7 @@ class VideoRenderData { this.colorFilters = const [], this.audioTracks = const [], this.blur, + this.chromaKey, this.bitrate, this.maxFrameRate, this.shouldOptimizeForNetworkUse = false, @@ -64,8 +65,38 @@ class VideoRenderData { assert( maxFrameRate == null || maxFrameRate > 0, '[maxFrameRate] must be greater than 0', + ), + assert( + _everyKeyHasABackground(composition, chromaKey, videoSegments), + 'A [chromaKey] without a background needs something underneath to ' + 'show through, and videoSegments is a single track with nothing ' + 'below it โ€” H.264/HEVC carry no alpha, so the keyed area would be ' + 'flattened to black. Set chromaKey.backgroundColor or ' + 'chromaKey.backgroundImage, or use composition and put the keyed ' + 'clip on a layer above another one. Pass ' + 'backgroundColor: Color(0xFF000000) if you really do want black.', ); + /// Whether every key that can reach the single-track path has a background. + /// + /// Checks the per-segment keys as well as the global one: they are resolved + /// as `segment โ†’ global`, so a transparent key set on a single [VideoSegment] + /// is flattened to black exactly like a transparent global one. The layered + /// path is exempt โ€” there a lower [VideoLayer] is what shows through. + static bool _everyKeyHasABackground( + VideoComposition? composition, + ChromaKey? chromaKey, + List? videoSegments, + ) { + if (composition != null) return true; + if (chromaKey != null && chromaKey.isTransparent) return false; + return videoSegments?.every((segment) { + final key = segment.chromaKey; + return key == null || !key.isTransparent; + }) ?? + true; + } + /// Creates a [VideoRenderData] with a predefined quality preset. /// /// This factory constructor simplifies video export by providing common @@ -96,6 +127,7 @@ class VideoRenderData { Duration? startTime, Duration? endTime, double? blur, + ChromaKey? chromaKey, int? bitrateOverride, int? maxFrameRate, List colorFilters = const [], @@ -118,6 +150,7 @@ class VideoRenderData { startTime: startTime, endTime: endTime, blur: blur, + chromaKey: chromaKey, bitrate: bitrateOverride ?? qualityConfig.bitrate, maxFrameRate: maxFrameRate, colorFilters: colorFilters, @@ -234,6 +267,22 @@ class VideoRenderData { /// Higher values result in a stronger blur effect. final double? blur; + /// Removes a solid-colored background ("green screen") from the video. + /// + /// Applies to every clip that does not carry its own key: a + /// [VideoSegment.chromaKey] overrides this, and inside a [composition] a + /// [VideoLayer.chromaKey] sits between the two. They are never merged. + /// + /// Keying runs on the original source colors, so it happens before + /// [colorFilters] and [blur]. + /// + /// See [ChromaKey] for how the key is tuned and what the removed area + /// becomes. Note that a key without a background is only meaningful with + /// [composition] โ€” see the assert on this constructor. + /// + /// **Note:** Ignored on Web, Windows and Linux. + final ChromaKey? chromaKey; + /// The maximum bitrate of the video in bits per second. /// /// This is an upper limit, not a target: when a simple export (a single @@ -420,6 +469,7 @@ class VideoRenderData { 'trimToCommonTrackEnd': trimToCommonTrackEnd, 'outputFormat': outputFormat.name, 'blur': blur, + 'chromaKey': await chromaKey?.toAsyncMap(), // Fall back to the quality config's bitrate when no explicit bitrate is // set, so a `qualityConfig` used on its own is still applied. 'bitrate': bitrate ?? qualityConfig?.bitrate, @@ -453,6 +503,7 @@ class VideoRenderData { List? colorFilters, List? audioTracks, double? blur, + ChromaKey? chromaKey, int? bitrate, int? maxFrameRate, bool? shouldOptimizeForNetworkUse, @@ -473,6 +524,7 @@ class VideoRenderData { colorFilters: colorFilters ?? this.colorFilters, audioTracks: audioTracks ?? this.audioTracks, blur: blur ?? this.blur, + chromaKey: chromaKey ?? this.chromaKey, bitrate: bitrate ?? this.bitrate, maxFrameRate: maxFrameRate ?? this.maxFrameRate, shouldOptimizeForNetworkUse: @@ -498,6 +550,7 @@ class VideoRenderData { 'colorFilters': colorFilters.map((x) => x.toMap()).toList(), 'audioTracks': audioTracks.map((x) => x.toMap()).toList(), 'blur': blur, + 'chromaKey': chromaKey?.toMap(), 'bitrate': bitrate, 'maxFrameRate': maxFrameRate, 'shouldOptimizeForNetworkUse': shouldOptimizeForNetworkUse, @@ -555,6 +608,9 @@ class VideoRenderData { ), ), blur: tryParseDouble(map['blur']), + chromaKey: map['chromaKey'] != null + ? ChromaKey.fromMap(map['chromaKey'] as Map) + : null, bitrate: map['bitrate'] != null ? safeParseInt(map['bitrate']) : null, maxFrameRate: map['maxFrameRate'] != null ? safeParseInt(map['maxFrameRate']) @@ -585,6 +641,7 @@ class VideoRenderData { 'colorFilters: $colorFilters, ' 'audioTracks: $audioTracks, ' 'blur: $blur, ' + 'chromaKey: $chromaKey, ' 'bitrate: $bitrate, ' 'maxFrameRate: $maxFrameRate, ' 'shouldOptimizeForNetworkUse: $shouldOptimizeForNetworkUse, ' @@ -609,6 +666,7 @@ class VideoRenderData { listEquals(other.colorFilters, colorFilters) && listEquals(other.audioTracks, audioTracks) && other.blur == blur && + other.chromaKey == chromaKey && other.bitrate == bitrate && other.maxFrameRate == maxFrameRate && other.shouldOptimizeForNetworkUse == shouldOptimizeForNetworkUse && @@ -631,6 +689,7 @@ class VideoRenderData { colorFilters.hashCode ^ audioTracks.hashCode ^ blur.hashCode ^ + chromaKey.hashCode ^ bitrate.hashCode ^ maxFrameRate.hashCode ^ shouldOptimizeForNetworkUse.hashCode ^ diff --git a/lib/core/models/video/video_segment_model.dart b/lib/core/models/video/video_segment_model.dart index 2af94aa6..135ef19f 100644 --- a/lib/core/models/video/video_segment_model.dart +++ b/lib/core/models/video/video_segment_model.dart @@ -21,6 +21,7 @@ class VideoSegment { this.transition, this.timelineStart, this.transform, + this.chromaKey, }) : assert( startTime == null || endTime == null || startTime < endTime, 'startTime must be before endTime', @@ -122,7 +123,21 @@ class VideoSegment { /// or fills the entire canvas if neither is set. final SegmentTransform? transform; + /// Removes a solid-colored background from this clip only. + /// + /// Overrides [VideoLayer.chromaKey] and [VideoRenderData.chromaKey] for this + /// clip; the three are never merged. When `null`, the clip falls back to its + /// layer's key, then to the global one. + /// + /// **Ignored inside an overlap [transition]** (dissolve/slide/push/wipe) when + /// the two clips at that boundary carry different keys โ€” that blend is + /// pre-rendered from the raw sources and is emitted unkeyed with a warning. + final ChromaKey? chromaKey; + /// Converts this clip to a map for platform channel communication. + /// + /// Resolves the input path and any chroma-key background image, so this is + /// asynchronous. Future> toAsyncMap() async { final inputPath = await video.safeFilePath(); @@ -136,6 +151,7 @@ class VideoSegment { 'transition': transition?.toMap(), 'timelineStartUs': timelineStart?.inMicroseconds, 'transform': transform?.toMap(), + 'chromaKey': await chromaKey?.toAsyncMap(), }; } @@ -150,6 +166,7 @@ class VideoSegment { ClipTransition? transition, Duration? timelineStart, SegmentTransform? transform, + ChromaKey? chromaKey, }) { return VideoSegment( video: video ?? this.video, @@ -161,6 +178,7 @@ class VideoSegment { transition: transition ?? this.transition, timelineStart: timelineStart ?? this.timelineStart, transform: transform ?? this.transform, + chromaKey: chromaKey ?? this.chromaKey, ); } @@ -176,7 +194,8 @@ class VideoSegment { other.reverseVideo == reverseVideo && other.transition == transition && other.timelineStart == timelineStart && - other.transform == transform; + other.transform == transform && + other.chromaKey == chromaKey; } @override @@ -189,7 +208,8 @@ class VideoSegment { reverseVideo.hashCode ^ transition.hashCode ^ timelineStart.hashCode ^ - transform.hashCode; + transform.hashCode ^ + chromaKey.hashCode; } @override @@ -202,7 +222,8 @@ class VideoSegment { 'reverseVideo: $reverseVideo, ' 'transition: $transition, ' 'timelineStart: $timelineStart, ' - 'transform: $transform)'; + 'transform: $transform, ' + 'chromaKey: $chromaKey)'; } Map toMap() { @@ -216,6 +237,7 @@ class VideoSegment { 'transition': transition?.toMap(), 'timelineStart': timelineStart?.inMicroseconds, 'transform': transform?.toMap(), + 'chromaKey': chromaKey?.toMap(), }; } @@ -240,6 +262,9 @@ class VideoSegment { transform: map['transform'] != null ? SegmentTransform.fromMap(map['transform'] as Map) : null, + chromaKey: map['chromaKey'] != null + ? ChromaKey.fromMap(map['chromaKey'] as Map) + : null, ); } diff --git a/lib/core/utils/chroma_key_detector.dart b/lib/core/utils/chroma_key_detector.dart new file mode 100644 index 00000000..557cc47f --- /dev/null +++ b/lib/core/utils/chroma_key_detector.dart @@ -0,0 +1,233 @@ +import 'dart:math'; +import 'dart:typed_data'; +import 'dart:ui'; + +/// What [ChromaKeyDetector] read out of a frame. +class ChromaKeyDetection { + /// Creates a [ChromaKeyDetection]. + const ChromaKeyDetection({ + required this.color, + required this.similarity, + required this.coverage, + required this.spread, + }); + + /// The screen color as it was actually recorded โ€” not the paint it was + /// mixed from. These differ by more than you would expect: a studio screen + /// painted SMPTE green (`0xFF00B140`) measured `0xFF2A9D37` through the + /// camera, which is a chroma distance of `0.12` โ€” most of the budget a + /// hand-set key has to spend before it touches the subject. + final Color color; + + /// A [similarity] that covers the measured spread of the screen with room to + /// spare, without reaching further than it has to. + final double similarity; + + /// The fraction of sampled border pixels that belong to the detected screen, + /// in `0..1`. + /// + /// Low coverage means the border was not one solid color โ€” the frame likely + /// shows more than just the screen (studio walls, lights, a subject standing + /// at the edge), and the result should not be trusted. + final double coverage; + + /// The 99th-percentile chroma distance of the screen from [color]. + /// + /// This is the raw measurement [similarity] is derived from; a large value + /// means an unevenly lit screen. + final double spread; + + @override + String toString() => + 'ChromaKeyDetection(color: $color, similarity: ' + '${similarity.toStringAsFixed(3)}, coverage: ' + '${(coverage * 100).toStringAsFixed(0)}%, spread: ' + '${spread.toStringAsFixed(3)})'; +} + +/// Thrown when a frame does not contain a usable screen. +class ChromaKeyDetectionException implements Exception { + /// Creates a [ChromaKeyDetectionException] with a [message] explaining what + /// the frame looked like instead. + const ChromaKeyDetectionException(this.message); + + /// What was measured, and why it is not a screen. + final String message; + + @override + String toString() => 'ChromaKeyDetectionException: $message'; +} + +/// Reads the screen color and its spread out of a frame. +/// +/// A fixed key color is always a compromise: paint, fabric, lighting and the +/// camera's own color science all move the recorded green away from whatever +/// constant the caller picked, and every bit of that offset has to be absorbed +/// by a wider [ChromaKey.similarity] โ€” which is budget taken away from the +/// margin that protects the subject. Measuring the screen instead removes the +/// offset entirely, and works the same for a green, blue or any other screen. +/// +/// The screen is assumed to reach the frame border, which is what lets the +/// subject be ignored: only a ring around the edge is sampled. +abstract final class ChromaKeyDetector { + /// Fraction of the frame, per side, sampled as the border ring. + static const _ringFraction = 0.12; + + /// A screen has to be at least this saturated. Below it, the border is grey + /// or black and there is nothing to key. + static const _minChromaMagnitude = 0.08; + + /// At least this share of the border must belong to one color. + static const _minCoverage = 0.6; + + /// How far from the median a border pixel may sit and still count as screen. + /// + /// Deliberately a **constant**, not a percentile of the measured + /// distribution: a band derived from the distances themselves would contain + /// a fixed share of them by construction, and the coverage test below would + /// pass for every possible frame. Sized off the default `similarity` โ€” wide + /// enough that an unevenly lit screen still counts (the falloff shows up as + /// spread, not as missing coverage), tight enough that skin (`0.38`โ€“`0.42` + /// from green) and an olive shirt (`0.26`) do not. + static const _coverageBand = 0.20; + + /// Multiplier on the measured spread, so the soft edge and a little + /// frame-to-frame noise stay inside the key. + static const _spreadMargin = 2.0; + + /// Never key tighter than this, or compression noise on a flat screen starts + /// poking through. + static const _minSimilarity = 0.08; + + /// Never key wider than this from a measurement alone โ€” beyond it the risk of + /// eating the subject outweighs whatever the border suggested. + static const _maxSimilarity = 0.35; + + /// BT.601 chroma of a gamma-encoded RGB triple in `0..1`, matching the + /// keying formula the renderers use. + static ({double cb, double cr}) chromaOf(double r, double g, double b) => ( + cb: -0.168736 * r - 0.331264 * g + 0.5 * b, + cr: 0.5 * r - 0.418688 * g - 0.081312 * b, + ); + + /// Detects the screen in one or more RGBA frames. + /// + /// [frames] are raw RGBA buffers, all [width] x [height]. Passing several + /// frames pools their border pixels, which averages out sensor noise and a + /// subject that briefly touches the edge. + /// + /// Throws a [ChromaKeyDetectionException] when the border is not one + /// saturated color. + static ChromaKeyDetection fromFrames( + List frames, { + required int width, + required int height, + }) { + if (frames.isEmpty) { + throw const ChromaKeyDetectionException('No frames to analyze'); + } + + final ringR = []; + final ringG = []; + final ringB = []; + + final marginX = max(1, (width * _ringFraction).round()); + final marginY = max(1, (height * _ringFraction).round()); + + for (final frame in frames) { + if (frame.length < width * height * 4) { + throw ChromaKeyDetectionException( + 'Frame buffer is ${frame.length} bytes, expected at least ' + '${width * height * 4} for ${width}x$height RGBA', + ); + } + for (var y = 0; y < height; y++) { + final onHorizontalEdge = y < marginY || y >= height - marginY; + for (var x = 0; x < width; x++) { + if (!onHorizontalEdge && x >= marginX && x < width - marginX) { + // Interior: skip ahead to the right-hand band in one step. + x = width - marginX - 1; + continue; + } + final i = (y * width + x) * 4; + ringR.add(frame[i] / 255); + ringG.add(frame[i + 1] / 255); + ringB.add(frame[i + 2] / 255); + } + } + } + + if (ringR.isEmpty) { + throw const ChromaKeyDetectionException('Frame is too small to sample'); + } + + // The median rejects the subject and any studio clutter that reaches the + // edge, as long as they are the minority there โ€” which an average would + // not, since a few bright pixels drag it a long way. + final r = _median(ringR); + final g = _median(ringG); + final b = _median(ringB); + final key = chromaOf(r, g, b); + final magnitude = sqrt(key.cb * key.cb + key.cr * key.cr); + + if (magnitude < _minChromaMagnitude) { + throw ChromaKeyDetectionException( + 'The frame border averages to a near-neutral color ' + '(chroma magnitude ${magnitude.toStringAsFixed(3)}), so there is no ' + 'screen to key. Is the screen actually reaching the frame edge?', + ); + } + + // How far each border pixel sits from that color. + final distances = List.generate(ringR.length, (i) { + final c = chromaOf(ringR[i], ringG[i], ringB[i]); + final dcb = c.cb - key.cb; + final dcr = c.cr - key.cr; + return sqrt(dcb * dcb + dcr * dcr); + })..sort(); + + // Which border pixels belong to the screen. The band is a constant, so + // this is a real measurement โ€” a wall or a sleeve falls outside it and + // drags coverage down, while an unevenly lit screen stays inside and shows + // up as spread instead. + final screen = distances.takeWhile((d) => d <= _coverageBand).toList(); + final coverage = screen.length / distances.length; + + if (coverage < _minCoverage) { + throw ChromaKeyDetectionException( + 'Only ${(coverage * 100).toStringAsFixed(0)}% of the frame border is ' + 'one color, so it is not a clean screen. Crop the frame to the screen ' + 'area, or set the key manually.', + ); + } + + // Measured over the screen pixels alone, so anything that reached the + // border without being part of the screen cannot widen the key. + final spread = + screen[(screen.length * 0.99).floor().clamp(0, screen.length - 1)]; + + final similarity = (spread * _spreadMargin).clamp( + _minSimilarity, + _maxSimilarity, + ); + + return ChromaKeyDetection( + color: Color.fromARGB( + 255, + (r * 255).round().clamp(0, 255), + (g * 255).round().clamp(0, 255), + (b * 255).round().clamp(0, 255), + ), + similarity: similarity, + coverage: coverage, + spread: spread, + ); + } + + static double _median(List values) { + final sorted = List.from(values)..sort(); + final mid = sorted.length ~/ 2; + if (sorted.length.isOdd) return sorted[mid]; + return (sorted[mid - 1] + sorted[mid]) / 2; + } +} diff --git a/lib/pro_video_editor.dart b/lib/pro_video_editor.dart index 045c309c..251f048b 100644 --- a/lib/pro_video_editor.dart +++ b/lib/pro_video_editor.dart @@ -15,6 +15,8 @@ export 'features/audio/models/waveform_style.dart'; export 'core/models/image/editor_layer_image_model.dart'; export 'core/models/image/image_layer_model.dart'; export 'core/models/image/layer_animation_model.dart'; +export 'core/models/video/chroma_key_model.dart'; +export 'core/utils/chroma_key_detector.dart'; export 'core/models/video/color_filter_model.dart'; export 'core/models/video/progress_model.dart'; export 'core/models/video/editor_video_model.dart'; diff --git a/pubspec.yaml b/pubspec.yaml index d0b18d01..9fa7e827 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,6 +1,6 @@ name: pro_video_editor description: "A Flutter video editor: Seamlessly enhance your videos with user-friendly editing features." -version: 2.10.0 +version: 2.11.0 homepage: https://github.com/hm21/pro_video_editor/ repository: https://github.com/hm21/pro_video_editor/ documentation: https://github.com/hm21/pro_video_editor/ diff --git a/test/core/models/video/chroma_key_model_test.dart b/test/core/models/video/chroma_key_model_test.dart new file mode 100644 index 00000000..ad4f6d94 --- /dev/null +++ b/test/core/models/video/chroma_key_model_test.dart @@ -0,0 +1,280 @@ +import 'dart:typed_data'; +import 'dart:ui'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:pro_video_editor/pro_video_editor.dart'; + +void main() { + group('ChromaKey', () { + const key = ChromaKey( + color: Color(0xFF00FF00), + similarity: 0.2, + smoothness: 0.1, + spill: 0.75, + backgroundColor: Color(0xFFFF0000), + ); + + group('defaults', () { + test('match the documented SMPTE green preset', () { + const defaults = ChromaKey(); + + expect(defaults.color, const Color(0xFF00B140)); + expect(defaults.similarity, 0.20); + expect(defaults.smoothness, 0.08); + expect(defaults.spill, 0.5); + expect(defaults.backgroundColor, isNull); + expect(defaults.backgroundImage, isNull); + }); + + test('are transparent, since no background is set', () { + expect(const ChromaKey().isTransparent, isTrue); + }); + }); + + group('assertions', () { + test('similarity must be greater than 0 and at most 1', () { + expect(() => ChromaKey(similarity: 0), throwsA(isA())); + expect( + () => ChromaKey(similarity: -0.1), + throwsA(isA()), + ); + expect( + () => ChromaKey(similarity: 1.1), + throwsA(isA()), + ); + expect(const ChromaKey(similarity: 1).similarity, 1); + }); + + test('smoothness must be between 0 and 1', () { + expect(() => ChromaKey(smoothness: -1), throwsA(isA())); + expect(() => ChromaKey(smoothness: 2), throwsA(isA())); + expect(const ChromaKey(smoothness: 0).smoothness, 0); + }); + + test('spill must be between 0 and 1', () { + expect(() => ChromaKey(spill: -1), throwsA(isA())); + expect(() => ChromaKey(spill: 2), throwsA(isA())); + expect(const ChromaKey(spill: 0).spill, 0); + }); + + test('rejects two backgrounds at once', () { + expect( + () => ChromaKey( + backgroundColor: const Color(0xFFFF0000), + backgroundImage: EditorLayerImage.asset('assets/bg.png'), + ), + throwsA(isA()), + ); + }); + }); + + group('isTransparent', () { + test('is false once a background is set', () { + expect( + const ChromaKey(backgroundColor: Color(0xFF123456)).isTransparent, + isFalse, + ); + expect( + ChromaKey( + backgroundImage: EditorLayerImage.asset('assets/bg.png'), + ).isTransparent, + isFalse, + ); + }); + }); + + group('toAsyncMap', () { + test('serializes colors as ARGB ints for the platform channel', () async { + final map = await key.toAsyncMap(); + + expect(map['keyColor'], 0xFF00FF00); + expect(map['similarity'], 0.2); + expect(map['smoothness'], 0.1); + expect(map['spill'], 0.75); + expect(map['bgColor'], 0xFFFF0000); + expect(map['bgImageData'], isNull); + }); + + test('sends null for both backgrounds when transparent', () async { + final map = await const ChromaKey().toAsyncMap(); + + expect(map['bgColor'], isNull); + expect(map['bgImageData'], isNull); + }); + + test('resolves the background image to bytes', () async { + final bytes = Uint8List.fromList([1, 2, 3, 4]); + final map = await ChromaKey( + backgroundImage: EditorLayerImage.memory(bytes), + ).toAsyncMap(); + + expect(map['bgImageData'], bytes); + }); + }); + + group('toMap / fromMap', () { + test('serializes all fields correctly', () { + final map = key.toMap(); + + expect(map['color'], 0xFF00FF00); + expect(map['similarity'], 0.2); + expect(map['smoothness'], 0.1); + expect(map['spill'], 0.75); + expect(map['backgroundColor'], 0xFFFF0000); + expect(map['backgroundImage'], isNull); + }); + + test('roundtrip preserves every field', () { + final restored = ChromaKey.fromMap(key.toMap()); + + expect(restored, key); + }); + + test('roundtrip preserves a transparent key', () { + const transparent = ChromaKey(); + final restored = ChromaKey.fromMap(transparent.toMap()); + + expect(restored, transparent); + expect(restored.isTransparent, isTrue); + }); + + test('roundtrip preserves an asset background image', () { + final withImage = ChromaKey( + backgroundImage: EditorLayerImage.asset('assets/bg.png'), + ); + final restored = ChromaKey.fromMap(withImage.toMap()); + + expect(restored.backgroundImage?.assetPath, 'assets/bg.png'); + }); + + test('falls back to the defaults for missing fields', () { + final restored = ChromaKey.fromMap(const {}); + + expect(restored, const ChromaKey()); + }); + + test('parses numeric strings safely', () { + final map = key.toMap() + ..['similarity'] = '0.3' + ..['spill'] = '1'; + + final restored = ChromaKey.fromMap(map); + expect(restored.similarity, 0.3); + expect(restored.spill, 1.0); + }); + + test('json roundtrip preserves every field', () { + expect(ChromaKey.fromJson(key.toJson()), key); + }); + }); + + group('copyWith', () { + test('overrides only what is given', () { + final copy = key.copyWith(similarity: 0.5); + + expect(copy.similarity, 0.5); + expect(copy.color, key.color); + expect(copy.smoothness, key.smoothness); + expect(copy.spill, key.spill); + expect(copy.backgroundColor, key.backgroundColor); + }); + + test('keeps every value when called empty', () { + expect(key.copyWith(), key); + }); + + test('switching from an image background to a color drops the image', () { + final withImage = key.copyWith( + backgroundImage: EditorLayerImage.asset('assets/bg.png'), + ); + expect(withImage.backgroundImage, isNotNull); + expect(withImage.backgroundColor, isNull); + + final withColor = withImage.copyWith( + backgroundColor: const Color(0xFF0000FF), + ); + + expect(withColor.backgroundColor, const Color(0xFF0000FF)); + expect( + withColor.backgroundImage, + isNull, + reason: + 'the two are mutually exclusive, so setting one clears the ' + 'other instead of tripping the assert', + ); + }); + + test('switching from a color background to an image drops the color', () { + final copy = key.copyWith( + backgroundImage: EditorLayerImage.asset('assets/bg.png'), + ); + + expect(copy.backgroundImage, isNotNull); + expect(copy.backgroundColor, isNull); + }); + + test('removeBackground clears both and leaves the key transparent', () { + final copy = key.copyWith(removeBackground: true); + + expect(copy.isTransparent, isTrue); + expect(copy.backgroundColor, isNull); + expect(copy.backgroundImage, isNull); + expect(copy.color, key.color, reason: 'only the background is cleared'); + expect(copy.similarity, key.similarity); + }); + + test('removeBackground cannot be combined with a background', () { + expect( + () => key.copyWith( + removeBackground: true, + backgroundColor: const Color(0xFF00FF00), + ), + throwsA(isA()), + ); + }); + }); + + group('backgroundColor opacity', () { + test('a translucent background is rejected when serialized', () { + // Only RGB reaches either renderer, so a translucent fill would render + // two different ways. Caught at the boundary rather than silently. + const translucent = ChromaKey(backgroundColor: Color(0x80FF0000)); + + expect(translucent.toAsyncMap(), throwsA(isA())); + }); + + test('an opaque background serializes fine', () async { + const opaque = ChromaKey(backgroundColor: Color(0xFFFF0000)); + + await expectLater(opaque.toAsyncMap(), completes); + }); + }); + + group('equality', () { + test('two identical keys are equal and share a hashCode', () { + const a = ChromaKey(similarity: 0.3); + const b = ChromaKey(similarity: 0.3); + + expect(a, b); + expect(a.hashCode, b.hashCode); + }); + + test('a differing parameter breaks equality', () { + expect(const ChromaKey(similarity: 0.3), isNot(const ChromaKey())); + expect(const ChromaKey(spill: 0.1), isNot(const ChromaKey())); + expect( + const ChromaKey(color: Color(0xFF0000FF)), + isNot(const ChromaKey()), + ); + }); + }); + + test('toString includes every field', () { + final text = key.toString(); + + expect(text, contains('similarity: 0.2')); + expect(text, contains('smoothness: 0.1')); + expect(text, contains('spill: 0.75')); + }); + }); +} diff --git a/test/core/models/video/video_composition_model_test.dart b/test/core/models/video/video_composition_model_test.dart index 066ca410..c0d4ade0 100644 --- a/test/core/models/video/video_composition_model_test.dart +++ b/test/core/models/video/video_composition_model_test.dart @@ -105,6 +105,42 @@ void main() { throwsAssertionError, ); }); + + group('chromaKey', () { + const key = ChromaKey(similarity: 0.25); + + test('defaults to null so clips inherit the global key', () { + expect(layer.chromaKey, isNull); + }); + + // toAsyncMap resolves each clip's source to a local path, so these use a + // file video โ€” an asset would need the platform channel. + final local = VideoLayer( + clips: [VideoSegment(video: EditorVideo.file('test.mp4'))], + ); + + test('toAsyncMap sends the key over the platform channel', () async { + final map = await local.copyWith(chromaKey: key).toAsyncMap(); + + expect(map['chromaKey'], isA>()); + expect((map['chromaKey'] as Map)['similarity'], 0.25); + }); + + test('toAsyncMap omits the key when unset', () async { + expect((await local.toAsyncMap())['chromaKey'], isNull); + }); + + test('toJson / fromJson roundtrip preserves the key', () { + final withKey = layer.copyWith(chromaKey: key); + + expect(VideoLayer.fromJson(withKey.toJson()), withKey); + expect(VideoLayer.fromJson(layer.toJson()).chromaKey, isNull); + }); + + test('a differing key breaks equality', () { + expect(layer.copyWith(chromaKey: key), isNot(layer)); + }); + }); }); group('VideoComposition', () { diff --git a/test/core/models/video/video_render_data_model_test.dart b/test/core/models/video/video_render_data_model_test.dart index 315d5ad8..f5a7fa07 100644 --- a/test/core/models/video/video_render_data_model_test.dart +++ b/test/core/models/video/video_render_data_model_test.dart @@ -1,7 +1,10 @@ import 'dart:ui'; import 'package:flutter_test/flutter_test.dart'; +import 'package:pro_video_editor/core/models/video/chroma_key_model.dart'; import 'package:pro_video_editor/core/models/video/editor_video_model.dart'; +import 'package:pro_video_editor/core/models/video/video_composition_model.dart'; +import 'package:pro_video_editor/core/models/video/video_layer_model.dart'; import 'package:pro_video_editor/core/models/video/video_quality_config.dart'; import 'package:pro_video_editor/core/models/video/video_quality_preset.dart'; import 'package:pro_video_editor/core/models/video/video_render_data_model.dart'; @@ -159,4 +162,144 @@ void main() { ); }); }); + + group('VideoRenderData chromaKey', () { + const key = ChromaKey(backgroundColor: Color(0xFFFF0000)); + + VideoRenderData buildData({ChromaKey? chromaKey}) { + return VideoRenderData( + id: 'test', + videoSegments: [VideoSegment(video: EditorVideo.file('test.mp4'))], + chromaKey: chromaKey, + ); + } + + test('defaults to null', () { + expect(buildData().chromaKey, isNull); + }); + + test('rejects a transparent key on the single-track path', () { + // videoSegments has nothing underneath and the codec carries no alpha, + // so a background-less key would silently flatten to black. + expect( + () => buildData(chromaKey: const ChromaKey()), + throwsA(isA()), + ); + }); + + test('rejects a transparent key set on a single segment', () { + // Keys resolve segment โ†’ global, so a transparent per-segment key is + // flattened to black exactly like a transparent global one. The guard + // used to look only at the global key and let this through silently. + expect( + () => VideoRenderData( + id: 'test', + videoSegments: [ + VideoSegment(video: EditorVideo.file('a.mp4')), + VideoSegment( + video: EditorVideo.file('b.mp4'), + chromaKey: const ChromaKey(), + ), + ], + ), + throwsA(isA()), + ); + }); + + test('allows a backed key on a segment', () { + expect( + VideoRenderData( + id: 'test', + videoSegments: [ + VideoSegment( + video: EditorVideo.file('a.mp4'), + chromaKey: const ChromaKey(backgroundColor: Color(0xFF000000)), + ), + ], + ).videoSegments!.first.chromaKey, + isNotNull, + ); + }); + + test('allows a transparent segment key inside a composition', () { + expect( + VideoRenderData( + id: 'test', + composition: VideoComposition( + layers: [ + VideoLayer( + clips: [ + VideoSegment( + video: EditorVideo.file('test.mp4'), + chromaKey: const ChromaKey(), + ), + ], + ), + ], + ), + ).composition, + isNotNull, + ); + }); + + test('allows a transparent key inside a composition', () { + expect( + VideoRenderData( + id: 'test', + composition: VideoComposition( + layers: [ + VideoLayer( + clips: [VideoSegment(video: EditorVideo.file('test.mp4'))], + ), + ], + ), + chromaKey: const ChromaKey(), + ).chromaKey, + const ChromaKey(), + ); + }); + + test('toAsyncMap sends the key over the platform channel', () async { + // toMap() is the Dart-side JSON form; toAsyncMap() is what actually + // reaches the native RenderConfig, so the wire keys are asserted here. + final map = (await buildData(chromaKey: key).toAsyncMap())['chromaKey']; + + expect(map, isA>()); + expect((map as Map)['keyColor'], 0xFF00B140); + expect(map['bgColor'], 0xFFFF0000); + }); + + test('toAsyncMap omits the key when unset', () async { + expect((await buildData().toAsyncMap())['chromaKey'], isNull); + }); + + test('toMap / fromMap roundtrip preserves the key', () { + final restored = VideoRenderData.fromMap( + buildData(chromaKey: key).toMap(), + ); + + expect(restored.chromaKey, key); + expect(VideoRenderData.fromMap(buildData().toMap()).chromaKey, isNull); + }); + + test('copyWith overrides and otherwise keeps the key', () { + const other = ChromaKey(backgroundColor: Color(0xFF0000FF)); + + expect( + buildData(chromaKey: key).copyWith(chromaKey: other).chromaKey, + other, + ); + expect(buildData(chromaKey: key).copyWith().chromaKey, key); + }); + + test('withQualityPreset forwards the key', () { + final data = VideoRenderData.withQualityPreset( + videoSegments: [VideoSegment(video: EditorVideo.file('test.mp4'))], + qualityPreset: VideoQualityPreset.p720, + chromaKey: key, + ); + + expect(data.chromaKey, key); + }); + }); } diff --git a/test/core/models/video/video_segment_model_test.dart b/test/core/models/video/video_segment_model_test.dart index 5954df36..57cf6e2f 100644 --- a/test/core/models/video/video_segment_model_test.dart +++ b/test/core/models/video/video_segment_model_test.dart @@ -1,3 +1,5 @@ +import 'dart:ui'; + import 'package:flutter_test/flutter_test.dart'; import 'package:pro_video_editor/pro_video_editor.dart'; @@ -125,5 +127,52 @@ void main() { test('toString contains class name', () { expect(segment.toString(), contains('VideoSegment')); }); + + group('chromaKey', () { + const key = ChromaKey( + similarity: 0.3, + backgroundColor: Color(0xFF0000FF), + ); + + test('defaults to null so the clip inherits the layer/global key', () { + expect(segment.chromaKey, isNull); + }); + + // toAsyncMap resolves the source to a local path, so these use a file + // video โ€” an asset would need the platform channel. + final local = VideoSegment(video: EditorVideo.file('test.mp4')); + + test('toAsyncMap sends the key over the platform channel', () async { + final map = await local.copyWith(chromaKey: key).toAsyncMap(); + + expect(map['chromaKey'], isA>()); + expect((map['chromaKey'] as Map)['similarity'], 0.3); + }); + + test('toAsyncMap omits the key when unset', () async { + expect((await local.toAsyncMap())['chromaKey'], isNull); + }); + + test('toMap / fromMap roundtrip preserves the key', () { + final withKey = segment.copyWith(chromaKey: key); + + expect(VideoSegment.fromMap(withKey.toMap()).chromaKey, key); + expect(VideoSegment.fromMap(segment.toMap()).chromaKey, isNull); + }); + + test('copyWith overrides and otherwise keeps the key', () { + final withKey = segment.copyWith(chromaKey: key); + + expect(withKey.copyWith().chromaKey, key); + expect( + withKey.copyWith(chromaKey: const ChromaKey()).chromaKey, + const ChromaKey(), + ); + }); + + test('a differing key breaks equality', () { + expect(segment.copyWith(chromaKey: key), isNot(segment)); + }); + }); }); } diff --git a/test/core/utils/chroma_key_detector_test.dart b/test/core/utils/chroma_key_detector_test.dart new file mode 100644 index 00000000..5e136dd3 --- /dev/null +++ b/test/core/utils/chroma_key_detector_test.dart @@ -0,0 +1,361 @@ +import 'dart:math'; +import 'dart:typed_data'; +import 'dart:ui'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:pro_video_editor/pro_video_editor.dart'; + +void main() { + const width = 80; + const height = 60; + + /// Builds an RGBA frame: [screen] everywhere, with a [subject]-colored block + /// covering the middle [subjectFraction] of the frame. + Uint8List frame({ + required Color screen, + Color? subject, + double subjectFraction = 0.5, + double lightingFalloff = 0, + }) { + final buffer = Uint8List(width * height * 4); + final inset = (1 - subjectFraction) / 2; + for (var y = 0; y < height; y++) { + // Vertical brightness gradient, like a screen lit from the top. + final dim = 1 - (y / height) * lightingFalloff; + for (var x = 0; x < width; x++) { + final inSubject = + subject != null && + x >= width * inset && + x < width * (1 - inset) && + y >= height * inset && + y < height * (1 - inset); + final c = inSubject ? subject : screen; + final scale = inSubject ? 1.0 : dim; + final i = (y * width + x) * 4; + buffer[i] = ((c.r * 255) * scale).round().clamp(0, 255); + buffer[i + 1] = ((c.g * 255) * scale).round().clamp(0, 255); + buffer[i + 2] = ((c.b * 255) * scale).round().clamp(0, 255); + buffer[i + 3] = 255; + } + } + return buffer; + } + + /// Builds an RGBA frame split vertically: [left] over the leftmost + /// [leftFraction] of the width, [right] over the rest. Both reach the frame + /// border, which is what the coverage test is there to catch. + Uint8List splitFrame({ + required Color left, + required Color right, + double leftFraction = 0.5, + }) { + final buffer = Uint8List(width * height * 4); + for (var y = 0; y < height; y++) { + for (var x = 0; x < width; x++) { + final c = x < width * leftFraction ? left : right; + final i = (y * width + x) * 4; + buffer[i] = (c.r * 255).round(); + buffer[i + 1] = (c.g * 255).round(); + buffer[i + 2] = (c.b * 255).round(); + buffer[i + 3] = 255; + } + } + return buffer; + } + + double chromaDistance(Color a, Color b) { + final ca = ChromaKeyDetector.chromaOf(a.r, a.g, a.b); + final cb = ChromaKeyDetector.chromaOf(b.r, b.g, b.b); + return sqrt(pow(ca.cb - cb.cb, 2) + pow(ca.cr - cb.cr, 2)); + } + + group('ChromaKeyDetector', () { + const green = Color(0xFF00B140); + const blue = Color(0xFF0047BB); + const skin = Color(0xFFC09070); + + test('finds a flat green screen behind a subject', () { + final result = ChromaKeyDetector.fromFrames( + [frame(screen: green, subject: skin)], + width: width, + height: height, + ); + + expect(chromaDistance(result.color, green), lessThan(0.01)); + expect(result.coverage, greaterThan(0.99)); + expect(result.spread, lessThan(0.01)); + }); + + test('is hue-agnostic โ€” a blue screen works the same', () { + final result = ChromaKeyDetector.fromFrames( + [frame(screen: blue, subject: skin)], + width: width, + height: height, + ); + + expect(chromaDistance(result.color, blue), lessThan(0.01)); + expect(result.coverage, greaterThan(0.99)); + }); + + test('a flat screen still gets a workable soft margin', () { + // A perfectly uniform synthetic screen has zero spread; the floor keeps + // the key from being so tight that codec noise pokes through. + final result = ChromaKeyDetector.fromFrames( + [frame(screen: green, subject: skin)], + width: width, + height: height, + ); + + expect(result.similarity, greaterThanOrEqualTo(0.08)); + }); + + test('the subject never decides the key', () { + // The block covers most of the frame, but not the border ring. + final result = ChromaKeyDetector.fromFrames( + [frame(screen: green, subject: skin, subjectFraction: 0.7)], + width: width, + height: height, + ); + + expect(chromaDistance(result.color, green), lessThan(0.01)); + expect(chromaDistance(result.color, skin), greaterThan(0.3)); + }); + + test('an unevenly lit screen widens similarity, not the color', () { + final even = ChromaKeyDetector.fromFrames( + [frame(screen: green, subject: skin)], + width: width, + height: height, + ); + final uneven = ChromaKeyDetector.fromFrames( + [frame(screen: green, subject: skin, lightingFalloff: 0.5)], + width: width, + height: height, + ); + + expect( + uneven.spread, + greaterThan(even.spread), + reason: 'a falloff should show up as spread', + ); + expect( + uneven.similarity, + greaterThan(even.similarity), + reason: 'and similarity should widen to cover it', + ); + // The hue did not change, only the brightness, so the detected color + // stays on the same chroma direction. + expect(chromaDistance(uneven.color, green), lessThan(0.08)); + }); + + test('similarity covers the measured spread with margin', () { + final result = ChromaKeyDetector.fromFrames( + [frame(screen: green, subject: skin, lightingFalloff: 0.4)], + width: width, + height: height, + ); + + expect(result.similarity, greaterThanOrEqualTo(result.spread)); + }); + + test('similarity is capped so a bad read cannot eat the subject', () { + // A border that is half green and half a very different green-ish tone + // produces a large spread; the cap keeps it from opening up wildly. + final result = ChromaKeyDetector.fromFrames( + [frame(screen: green, subject: skin, lightingFalloff: 0.9)], + width: width, + height: height, + ); + + expect(result.similarity, lessThanOrEqualTo(0.35)); + }); + + test('pools several frames', () { + final result = ChromaKeyDetector.fromFrames( + [ + frame(screen: green, subject: skin), + frame(screen: green, subject: skin), + frame(screen: green, subject: skin), + ], + width: width, + height: height, + ); + + expect(chromaDistance(result.color, green), lessThan(0.01)); + }); + + group('rejects what is not a screen', () { + test('a neutral border', () { + expect( + () => ChromaKeyDetector.fromFrames( + [frame(screen: const Color(0xFF808080))], + width: width, + height: height, + ), + throwsA( + isA().having( + (e) => e.message, + 'message', + contains('near-neutral'), + ), + ), + ); + }); + + test('a black border', () { + expect( + () => ChromaKeyDetector.fromFrames( + [frame(screen: const Color(0xFF000000))], + width: width, + height: height, + ), + throwsA(isA()), + ); + }); + + test('no frames at all', () { + expect( + () => ChromaKeyDetector.fromFrames([], width: width, height: height), + throwsA(isA()), + ); + }); + + test('a border that is half screen and half studio wall', () { + // The regression this guards: coverage used to be derived from a + // percentile of the very distances it was measuring, so it came out + // at ~99% for any frame at all and this never threw. + expect( + () => ChromaKeyDetector.fromFrames( + [splitFrame(left: green, right: const Color(0xFF8B5A2B))], + width: width, + height: height, + ), + throwsA( + isA().having( + (e) => e.message, + 'message', + contains('one color'), + ), + ), + ); + }); + + test('a border that is half screen and half red curtain', () { + expect( + () => ChromaKeyDetector.fromFrames( + [splitFrame(left: green, right: const Color(0xFFCC2222))], + width: width, + height: height, + ), + throwsA(isA()), + ); + }); + + test('coverage is a real measurement, not a constant', () { + // A minority of off-screen pixels is tolerated, but it has to show up + // in `coverage` โ€” the old implementation reported ~1.0 regardless. + final result = ChromaKeyDetector.fromFrames( + [splitFrame(left: green, right: skin, leftFraction: 0.8)], + width: width, + height: height, + ); + + expect(result.coverage, lessThan(0.95)); + expect(result.coverage, greaterThan(0.6)); + expect(chromaDistance(result.color, green), lessThan(0.01)); + }); + + test('a buffer that is too small for the stated size', () { + expect( + () => ChromaKeyDetector.fromFrames( + [Uint8List(10)], + width: width, + height: height, + ), + throwsA( + isA().having( + (e) => e.message, + 'message', + contains('expected at least'), + ), + ), + ); + }); + }); + + test('chromaOf matches the renderers\' BT.601 formula', () { + // Pinned against the same constants the shader and the color cube use. + final c = ChromaKeyDetector.chromaOf(0, 0xB1 / 255, 0x40 / 255); + expect(c.cb, closeTo(-0.1044, 1e-4)); + expect(c.cr, closeTo(-0.3110, 1e-4)); + }); + + test('a neutral color sits at the chroma origin', () { + final c = ChromaKeyDetector.chromaOf(0.5, 0.5, 0.5); + expect(c.cb, closeTo(0, 1e-9)); + expect(c.cr, closeTo(0, 1e-9)); + }); + }); + + group('ChromaKey presets', () { + test('greenScreen matches the default constructor', () { + const preset = ChromaKey.greenScreen(); + const plain = ChromaKey(); + + expect(preset.color, plain.color); + expect(preset.similarity, plain.similarity); + expect(preset.spill, plain.spill); + }); + + test('blueScreen keys tighter and despills more gently', () { + const green = ChromaKey.greenScreen(); + const blue = ChromaKey.blueScreen(); + + expect(blue.color, const Color(0xFF0047BB)); + expect( + blue.similarity, + lessThan(green.similarity), + reason: 'denim sits at 0.19 from blue, inside the green default', + ); + expect(blue.spill, lessThan(green.spill)); + }); + + test('the blue preset does not key denim', () { + const blue = ChromaKey.blueScreen(); + const denim = Color(0xFF3B5B8C); + + final d = chromaDistance(blue.color, denim); + expect( + d, + greaterThan(blue.similarity), + reason: 'denim ($d) must sit outside similarity (${blue.similarity})', + ); + }); + + test('the green default would key denim on a blue screen', () { + // The reason blueScreen exists as its own preset. + const denim = Color(0xFF3B5B8C); + final d = chromaDistance(const Color(0xFF0047BB), denim); + + expect(d, lessThan(const ChromaKey.greenScreen().similarity)); + }); + + test('presets carry a background through', () { + const key = ChromaKey.blueScreen(backgroundColor: Color(0xFFFF0000)); + + expect(key.isTransparent, isFalse); + expect(key.backgroundColor, const Color(0xFFFF0000)); + }); + + test('presets keep the shared assertions', () { + expect( + () => ChromaKey.blueScreen(similarity: 0), + throwsA(isA()), + ); + expect( + () => ChromaKey.greenScreen(spill: 2), + throwsA(isA()), + ); + }); + }); +} diff --git a/test/pro_video_editor_method_channel_test.mocks.dart b/test/pro_video_editor_method_channel_test.mocks.dart index 9a3ce0be..b51dc851 100644 --- a/test/pro_video_editor_method_channel_test.mocks.dart +++ b/test/pro_video_editor_method_channel_test.mocks.dart @@ -418,6 +418,7 @@ class MockVideoRenderData extends _i1.Mock implements _i4.VideoRenderData { List<_i4.ColorFilter>? colorFilters, List<_i4.VideoAudioTrack>? audioTracks, double? blur, + _i4.ChromaKey? chromaKey, int? bitrate, int? maxFrameRate, bool? shouldOptimizeForNetworkUse, @@ -439,6 +440,7 @@ class MockVideoRenderData extends _i1.Mock implements _i4.VideoRenderData { #colorFilters: colorFilters, #audioTracks: audioTracks, #blur: blur, + #chromaKey: chromaKey, #bitrate: bitrate, #maxFrameRate: maxFrameRate, #shouldOptimizeForNetworkUse: shouldOptimizeForNetworkUse, @@ -461,6 +463,7 @@ class MockVideoRenderData extends _i1.Mock implements _i4.VideoRenderData { #colorFilters: colorFilters, #audioTracks: audioTracks, #blur: blur, + #chromaKey: chromaKey, #bitrate: bitrate, #maxFrameRate: maxFrameRate, #shouldOptimizeForNetworkUse: shouldOptimizeForNetworkUse,