diff --git a/app/src/main/java/com/openterface/AOS/activity/MainActivity.java b/app/src/main/java/com/openterface/AOS/activity/MainActivity.java index ded66b4..f0ac2c4 100644 --- a/app/src/main/java/com/openterface/AOS/activity/MainActivity.java +++ b/app/src/main/java/com/openterface/AOS/activity/MainActivity.java @@ -227,6 +227,8 @@ public int getPreviewHeight() { // before adding a new one (e.g. in onSurfaceTextureSizeChanged). Without this, the // RendererHolder accumulates stale surfaces which can cause black screen in portrait. private android.view.Surface mCurrentPreviewSurface = null; + // Reference to touch/zoom controller in portrait mode, used to update Matrix on surface ready/size change + private CustomTouchListener mCustomTouchListener = null; private CameraControlsDialogFragment mControlsDialog; private DeviceListDialogFragment mDeviceListDialog; @@ -407,9 +409,9 @@ protected void onCreate(Bundle savedInstanceState) { getWindowManager().getDefaultDisplay().getMetrics(displayMetrics); //deal mouse click and button ,you can jump CustomTouchListener java - CustomTouchListener customTouchListener = new CustomTouchListener(this, usbDeviceManager); + mCustomTouchListener = new CustomTouchListener(this, usbDeviceManager); - mBinding.viewMainPreview.setOnTouchListener(customTouchListener); + mBinding.viewMainPreview.setOnTouchListener(mCustomTouchListener); // Setting up the gesture detector gestureDetector = new GestureDetector(this, new GestureListener()); @@ -924,8 +926,8 @@ private void reloadLayoutForOrientation() { Log.d(TAG, "reloadLayoutForOrientation: views initialized"); // Re-attach CustomTouchListener to the new preview view after layout re-inflation - CustomTouchListener customTouchListener = new CustomTouchListener(this, usbDeviceManager); - mBinding.viewMainPreview.setOnTouchListener(customTouchListener); + mCustomTouchListener = new CustomTouchListener(this, usbDeviceManager); + mBinding.viewMainPreview.setOnTouchListener(mCustomTouchListener); // Re-setup DrawerLayoutDeal for new layout (only in landscape) if (!nowPortrait) { @@ -1231,8 +1233,18 @@ private void initPreviewView() { Log.w(TAG, "initPreviewView: mBinding or viewMainPreview is null, skipping"); return; } - mBinding.viewMainPreview.setAspectRatio(mPreviewWidth, mPreviewHeight); - Log.d(TAG, "initPreviewView: mPreviewWidth=" + mPreviewWidth + " mPreviewHeight=" + mPreviewHeight); + // Portrait mode: disable aspect ratio constraint, TextureView fills container, Matrix handles display transform + // Landscape mode: keep original aspect ratio constraint behavior + if (!isPortraitMode) { + mBinding.viewMainPreview.setAspectRatio(mPreviewWidth, mPreviewHeight); + } else { + mBinding.viewMainPreview.setAspectRatioEnabled(false); + ViewGroup.LayoutParams lp = mBinding.viewMainPreview.getLayoutParams(); + lp.width = ViewGroup.LayoutParams.MATCH_PARENT; + lp.height = ViewGroup.LayoutParams.MATCH_PARENT; + mBinding.viewMainPreview.setLayoutParams(lp); + } + Log.d(TAG, "initPreviewView: mPreviewWidth=" + mPreviewWidth + " mPreviewHeight=" + mPreviewHeight + " isPortrait=" + isPortraitMode); mBinding.viewMainPreview.setSurfaceTextureListener(new TextureView.SurfaceTextureListener() { @Override public void onSurfaceTextureAvailable(@NonNull SurfaceTexture surface, int width, int height) { @@ -1260,7 +1272,10 @@ public void onSurfaceTextureAvailable(@NonNull SurfaceTexture surface, int width // fires AFTER this method returns. Setting the buffer here ensures the Surface // is created with the correct camera resolution. setSurfaceTextureBufferSize(surface, mPreviewWidth, mPreviewHeight); - mBinding.viewMainPreview.setAspectRatio(mPreviewWidth, mPreviewHeight); + // Portrait mode: skip setAspectRatio, Matrix handles display transform + if (!isPortraitMode) { + mBinding.viewMainPreview.setAspectRatio(mPreviewWidth, mPreviewHeight); + } // Create Surface from SurfaceTexture android.view.Surface previewSurface = new android.view.Surface(surface); @@ -1280,6 +1295,10 @@ public void onSurfaceTextureAvailable(@NonNull SurfaceTexture surface, int width mBinding.viewMainPreview.setKeepScreenOn(true); } }); + // Portrait mode: apply Matrix to correct stretch after surface is ready + if (isPortraitMode && mCustomTouchListener != null) { + mBinding.viewMainPreview.post(() -> mCustomTouchListener.applyPortraitZoomTransform()); + } } else if (mIsCameraConnected && surface == mCurrentPreviewSurfaceTexture) { Log.d(TAG, "onSurfaceTextureAvailable: surface already configured by onCameraOpen, skipping duplicate setup"); } else if (mIsCameraConnected && mMainSurfaceAdded && surface != mCurrentPreviewSurfaceTexture) { @@ -1295,7 +1314,10 @@ public void onSurfaceTextureAvailable(@NonNull SurfaceTexture surface, int width // Set buffer size BEFORE creating Surface - ensures Surface uses camera resolution setSurfaceTextureBufferSize(surface, mPreviewWidth, mPreviewHeight); - mBinding.viewMainPreview.setAspectRatio(mPreviewWidth, mPreviewHeight); + // Portrait mode: skip setAspectRatio, Matrix handles display transform + if (!isPortraitMode) { + mBinding.viewMainPreview.setAspectRatio(mPreviewWidth, mPreviewHeight); + } android.view.Surface previewSurface = new android.view.Surface(surface); // Use RendererHolder pipeline to enable frame distribution to multiple surfaces (including PIP) mCameraHelper.addSurface(previewSurface, false); @@ -1310,36 +1332,54 @@ public void onSurfaceTextureAvailable(@NonNull SurfaceTexture surface, int width mBinding.viewMainPreview.setKeepScreenOn(true); } }); + // Portrait mode: apply Matrix to correct stretch after surface reconnection + if (isPortraitMode && mCustomTouchListener != null) { + mBinding.viewMainPreview.post(() -> mCustomTouchListener.applyPortraitZoomTransform()); + } } } @Override public void onSurfaceTextureSizeChanged(@NonNull SurfaceTexture surface, int width, int height) { Log.d(TAG, "onSurfaceTextureSizeChanged: " + width + "x" + height + - " mMainSurfaceAdded=" + mMainSurfaceAdded + " mIsCameraConnected=" + mIsCameraConnected); + " mMainSurfaceAdded=" + mMainSurfaceAdded + " mIsCameraConnected=" + mIsCameraConnected + + " isPortrait=" + isPortraitMode); if (mMainSurfaceAdded && mCameraHelper != null && mIsCameraConnected) { - // Surface was already added by onCameraOpen, but the view size changed - // (e.g. portrait mode where AspectRatioTextureView's setAspectRatio - // triggers onMeasure which changes the view dimensions). - // CRITICAL: Remove the OLD Surface from RendererHolder BEFORE adding - // a new one. Without removing first, the RendererHolder accumulates - // stale Surface objects which causes the preview to render to the - // wrong surface (black screen). - // Both removeSurface and addSurface post to the same async handler, - // so they execute in order: remove first, then add. - Log.d(TAG, "onSurfaceTextureSizeChanged: removing old surface and re-adding with new buffer size"); - if (mCurrentPreviewSurface != null) { - mCameraHelper.removeSurface(mCurrentPreviewSurface); + // In portrait mode, TextureView is always MATCH_PARENT, size changes only occur during screen rotation. + // Skip setAspectRatio (portrait uses Matrix), avoid re-triggering onMeasure which causes flickering. + if (!isPortraitMode) { + // Surface was already added by onCameraOpen, but the view size changed + // (e.g. landscape mode where AspectRatioTextureView's setAspectRatio + // triggers onMeasure which changes the view dimensions). + // CRITICAL: Remove the OLD Surface from RendererHolder BEFORE adding + // a new one. Without removing first, the RendererHolder accumulates + // stale Surface objects which causes the preview to render to the + // wrong surface (black screen). + // Both removeSurface and addSurface post to the same async handler, + // so they execute in order: remove first, then add. + Log.d(TAG, "onSurfaceTextureSizeChanged: removing old surface and re-adding with new buffer size"); + if (mCurrentPreviewSurface != null) { + mCameraHelper.removeSurface(mCurrentPreviewSurface); + } + setSurfaceTextureBufferSize(surface, mPreviewWidth, mPreviewHeight); + mBinding.viewMainPreview.setAspectRatio(mPreviewWidth, mPreviewHeight); + android.view.Surface previewSurface = new android.view.Surface(surface); + mCameraHelper.addSurface(previewSurface, false); + mCurrentPreviewSurfaceTexture = surface; + mCurrentPreviewSurface = previewSurface; + mCameraHelper.startPreview(); + // Keep mMainSurfaceAdded = true — we just re-added it + } else { + // Portrait mode: surface size changes frequently during keyboard animation, skip surface deletion/recreation to avoid crash. + // Still need to set buffer size to camera native resolution to ensure clarity when zoomed. + Log.d(TAG, "onSurfaceTextureSizeChanged: portrait mode, updating buffer size without surface recreation"); + setSurfaceTextureBufferSize(surface, mPreviewWidth, mPreviewHeight); + // Update Matrix to recalculate stretch correction + if (mCustomTouchListener != null) { + mBinding.viewMainPreview.post(() -> mCustomTouchListener.applyPortraitZoomTransform()); + } } - setSurfaceTextureBufferSize(surface, mPreviewWidth, mPreviewHeight); - mBinding.viewMainPreview.setAspectRatio(mPreviewWidth, mPreviewHeight); - android.view.Surface previewSurface = new android.view.Surface(surface); - mCameraHelper.addSurface(previewSurface, false); - mCurrentPreviewSurfaceTexture = surface; - mCurrentPreviewSurface = previewSurface; - mCameraHelper.startPreview(); - // Keep mMainSurfaceAdded = true — we just re-added it } else { // First time or surface was destroyed — set buffer size for later if (!mMainSurfaceAdded) { @@ -1589,7 +1629,10 @@ public void onCameraOpen(UsbDevice device) { } // Step 2: Set aspect ratio on the view (controls how the view is measured) - mBinding.viewMainPreview.setAspectRatio(mPreviewWidth, mPreviewHeight); + // Portrait mode: skip setAspectRatio, Matrix handles display transform + if (!isPortraitMode) { + mBinding.viewMainPreview.setAspectRatio(mPreviewWidth, mPreviewHeight); + } // Step 3: Set SurfaceTexture buffer size to camera resolution BEFORE creating // the Surface. This is the critical step - the buffer size is read by @@ -1619,6 +1662,11 @@ public void onCameraOpen(UsbDevice device) { // Start preview after adding surfaces to ensure frames flow correctly mCameraHelper.startPreview(); Log.d(TAG, "onCameraOpen: startPreview called after adding surfaces, preview=" + mPreviewWidth + "x" + mPreviewHeight); + + // Portrait mode: apply Matrix to correct stretch after surface is ready + if (isPortraitMode && mCustomTouchListener != null) { + mBinding.viewMainPreview.post(() -> mCustomTouchListener.applyPortraitZoomTransform()); + } }); mIsCameraConnected = true; @@ -1729,9 +1777,12 @@ private void resizePreviewView(Size size) { mPreviewHeight = metrics.heightPixels; } } - Log.d(TAG, "resizePreviewView: " + mPreviewWidth + "x" + mPreviewHeight + " (mMainSurfaceAdded=" + mMainSurfaceAdded + ")"); + Log.d(TAG, "resizePreviewView: " + mPreviewWidth + "x" + mPreviewHeight + " (mMainSurfaceAdded=" + mMainSurfaceAdded + " isPortrait=" + isPortraitMode + ")"); // Set the aspect ratio of TextureView to match the aspect ratio of the camera - mBinding.viewMainPreview.setAspectRatio(mPreviewWidth, mPreviewHeight); + // Portrait mode: skip setAspectRatio, Matrix handles display transform + if (!isPortraitMode) { + mBinding.viewMainPreview.setAspectRatio(mPreviewWidth, mPreviewHeight); + } // Update SurfaceTexture buffer to chip resolution for better zoom quality. // IMPORTANT: Only update buffer size if no Surface has been created from this // SurfaceTexture yet. Once a Surface is created (mMainSurfaceAdded=true), the @@ -3417,7 +3468,7 @@ private void startSend(String text) { undoSnapshot = sendText; undoClearEligible = true; updateClearButtonIcon(); - // Do NOT clear editor after send (保留文本) + // Do NOT clear editor after send (keep text) } // Reset send button diff --git a/app/src/main/java/com/openterface/AOS/drawerLayout/ZoomLayoutDeal.java b/app/src/main/java/com/openterface/AOS/drawerLayout/ZoomLayoutDeal.java index 7812e5e..ad75559 100644 --- a/app/src/main/java/com/openterface/AOS/drawerLayout/ZoomLayoutDeal.java +++ b/app/src/main/java/com/openterface/AOS/drawerLayout/ZoomLayoutDeal.java @@ -547,11 +547,17 @@ public static void updateIndicatorFromMainView(float zoomScale, float translateX indicatorWidth = Math.max(20f, Math.min(indicatorWidth, pipWidth)); indicatorHeight = Math.max(20f, Math.min(indicatorHeight, pipHeight)); - // Update indicator size + // Update indicator size only when changed significantly (avoid layout thrash during pinch) + final int INDICATOR_SIZE_THRESHOLD = 3; ViewGroup.LayoutParams params = indicatorView.getLayoutParams(); - params.width = (int) indicatorWidth; - params.height = (int) indicatorHeight; - indicatorView.setLayoutParams(params); + int newWidth = (int) indicatorWidth; + int newHeight = (int) indicatorHeight; + if (Math.abs(params.width - newWidth) > INDICATOR_SIZE_THRESHOLD + || Math.abs(params.height - newHeight) > INDICATOR_SIZE_THRESHOLD) { + params.width = newWidth; + params.height = newHeight; + indicatorView.setLayoutParams(params); + } if (zoomScale <= 1.0f) { // Not zoomed, center the indicator @@ -691,10 +697,21 @@ private static void syncMainViewPosition(float thumbX, float thumbY) { normalizedX = Math.max(0, Math.min(normalizedX, 1)); normalizedY = Math.max(0, Math.min(normalizedY, 1)); + // Calculate actual content dimensions after stretch correction. + // Stretch correction fills viewWidth, maintains buffer aspect ratio, + // so contentHeight = viewWidth / bufferAspect (may be < viewHeight). + float bufferWidth = activity.getPreviewWidth(); + float bufferHeight = activity.getPreviewHeight(); + float contentWidth = viewWidth; + float contentHeight = viewWidth * bufferHeight / bufferWidth; + // Clamp: if content is larger than view, cap at view size + if (contentHeight > viewHeight) contentHeight = viewHeight; + if (contentWidth > viewWidth) contentWidth = viewWidth; + // Convert normalized position to main view translation - // Use view's display dimensions for max translation calculation - float maxTranslateX = (viewWidth * (currentZoomScale - 1f)) / 2f; - float maxTranslateY = (viewHeight * (currentZoomScale - 1f)) / 2f; + // Use corrected content dimensions for accurate edge-to-edge mapping + float maxTranslateX = (contentWidth * (currentZoomScale - 1f)) / 2f; + float maxTranslateY = (contentHeight * (currentZoomScale - 1f)) / 2f; // normalizedX=0 → translateX=+max (left edge visible) // normalizedX=1 → translateX=-max (right edge visible) diff --git a/app/src/main/java/com/openterface/AOS/serial/CustomTouchListener.java b/app/src/main/java/com/openterface/AOS/serial/CustomTouchListener.java index c7ffdcb..42af93f 100644 --- a/app/src/main/java/com/openterface/AOS/serial/CustomTouchListener.java +++ b/app/src/main/java/com/openterface/AOS/serial/CustomTouchListener.java @@ -159,9 +159,9 @@ public static boolean isVirtualTrackpadMode() { private boolean initialAbsDownSent = false; // Two-finger right-click button state - private boolean rightButtonPressed = false; // 右键当前是否按下 - private boolean isTwoFingerClick = false; // 是否为点击模式(未拖动) - private boolean twoFingerDragConfirmed = false; // 已确认为拖动(不再发送右键) + private boolean rightButtonPressed = false; // Whether right button is currently pressed + private boolean isTwoFingerClick = false; // Whether in click mode (not dragged) + private boolean twoFingerDragConfirmed = false; // Confirmed as drag (no longer send right click) // Two-finger pinch-to-zoom state private float twoFingerStartDist = 0f; // Initial distance between two fingers @@ -484,7 +484,8 @@ public static void setPortraitPan(float translateX, float translateY) { /** * Apply the current portrait zoom transform to the view - * Called by ZoomLayoutDeal after updating pan values + * Called by ZoomLayoutDeal after updating pan values (e.g., when PiP indicator is dragged) + * Includes stretch correction to prevent video distortion */ public static void applyCurrentPortraitTransform() { if (mPortraitZoomScale > 1.0f && activity != null && activity.mBinding != null @@ -495,12 +496,34 @@ public static void applyCurrentPortraitTransform() { if (viewWidth <= 0 || viewHeight <= 0) return; + // Get buffer dimensions for stretch correction + float bufferWidth = activity.getPreviewWidth(); + float bufferHeight = activity.getPreviewHeight(); + if (bufferWidth <= 0 || bufferHeight <= 0) return; + Matrix matrix = new Matrix(); + + // Check if stretch correction is needed + float surfaceAspect = viewWidth / viewHeight; + float bufferAspect = bufferWidth / bufferHeight; + boolean needsCorrection = Math.abs(surfaceAspect - bufferAspect) > 0.01f; + + if (needsCorrection) { + // Correct SurfaceTexture stretch: maintain aspect ratio, fill width, center vertically + float correctionScaleY = (bufferHeight * viewWidth) / (viewHeight * bufferWidth); + matrix.setScale(1.0f, correctionScaleY); + float correctedHeight = viewHeight * correctionScaleY; + float translateY = (viewHeight - correctedHeight) / 2f; + matrix.postTranslate(0, translateY); + } + + // Apply zoom and pan float pivotX = viewWidth / 2f; float pivotY = viewHeight / 2f; float scale = mPortraitZoomScale; - matrix.setScale(scale, scale, pivotX, pivotY); + matrix.postScale(scale, scale, pivotX, pivotY); matrix.postTranslate(mPortraitTranslateX, mPortraitTranslateY); + textureView.setTransform(matrix); textureView.invalidate(); } @@ -1261,105 +1284,133 @@ private float calculateDistance(float x1, float y1, float x2, float y2) { /** * Apply zoom using TextureView.setTransform(Matrix) for GPU-quality scaling. - * When zoomed in, the TextureView's physical height is expanded to show more vertical content. + * + * 竖屏模式下，TextureView 始终保持 MATCH_PARENT 填满 video_area_container。 + * SurfaceTexture 的实际尺寸 = screenWidth × containerHeight， + * 会把 1920×1080 的 buffer 拉伸到非 16:9 比例。 + * Matrix 负责：纠正拉伸 → 垂直居中 → 缩放 → 平移。 + * 这样不改变 TextureView 物理尺寸，避免触发 onSurfaceTextureSizeChanged 导致崩溃。 + */ + /** + * 应用竖屏 Matrix 变换（纠正拉伸 + 缩放 + 平移）。 + * 公开方法，供 MainActivity 在 surface 就绪 / 尺寸变化时调用。 */ - private void applyPortraitZoomTransform() { + public void applyPortraitZoomTransform() { if (activity == null || activity.mBinding == null || activity.mBinding.viewMainPreview == null) { return; } AspectRatioTextureView textureView = activity.mBinding.viewMainPreview; - float viewWidth = textureView.getWidth(); - float viewHeight = textureView.getHeight(); - if (viewWidth <= 0 || viewHeight <= 0) { + // Get actual SurfaceTexture dimensions (= TextureView visible area) + float surfaceWidth = textureView.getWidth(); + float surfaceHeight = textureView.getHeight(); + + if (surfaceWidth <= 0 || surfaceHeight <= 0) { return; } - // When zoomed in, expand the TextureView's physical height to show more content - expandTextureViewHeightIfNeeded(); + // Get camera buffer dimensions + float bufferWidth = activity.getPreviewWidth(); + float bufferHeight = activity.getPreviewHeight(); + if (bufferWidth <= 0 || bufferHeight <= 0) { + return; + } - // Create transform matrix + // Create Matrix Matrix matrix = new Matrix(); - // Apply scale around center pivot - float pivotX = viewWidth / 2f; - float pivotY = viewHeight / 2f; + // Check if stretch correction is needed: + // If SurfaceTexture aspect ratio ≠ buffer aspect ratio, it means the content is stretched + float surfaceAspect = surfaceWidth / surfaceHeight; + float bufferAspect = bufferWidth / bufferHeight; + boolean needsCorrection = Math.abs(surfaceAspect - bufferAspect) > 0.01f; + + if (needsCorrection) { + // SurfaceTexture stretches buffer (bufferWidth × bufferHeight) to surface (surfaceWidth × surfaceHeight) + // Stretch ratio: stretchX = surfaceWidth/bufferWidth, stretchY = surfaceHeight/bufferHeight + // + // Correction goal: Keep video at original 16:9 aspect ratio, fill surfaceWidth, calculate height proportionally, center vertically + // + // Step 1: Undo stretch → scaleX = bufferWidth/surfaceWidth, scaleY = bufferHeight/surfaceHeight + // Step 2: Uniform scale to fill width → scale = surfaceWidth/bufferWidth + // Combined: + // scaleX = (bufferWidth/surfaceWidth) * (surfaceWidth/bufferWidth) = 1.0 + // scaleY = (bufferHeight/surfaceHeight) * (surfaceWidth/bufferWidth) + // = (bufferHeight * surfaceWidth) / (surfaceHeight * bufferWidth) + float correctionScaleY = (bufferHeight * surfaceWidth) / (surfaceHeight * bufferWidth); + matrix.setScale(1.0f, correctionScaleY); + + // Corrected content height + float correctedHeight = surfaceHeight * correctionScaleY; + // Vertical centering + float translateY = (surfaceHeight - correctedHeight) / 2f; + matrix.postTranslate(0, translateY); + // Horizontal: corrected width = surfaceWidth, already filled, no extra translation needed + } + // If no correction needed (surfaceAspect ≈ bufferAspect), Matrix starts from identity - // Uniform scaling (scaleX == scaleY) - float scale = mPortraitZoomScale; + // Zoom (pivot at SurfaceTexture center) + float zoomScale = mPortraitZoomScale; + float pivotX = surfaceWidth / 2f; + float pivotY = surfaceHeight / 2f; + matrix.postScale(zoomScale, zoomScale, pivotX, pivotY); - matrix.setScale(scale, scale, pivotX, pivotY); + // Pan (user drag) matrix.postTranslate(mPortraitTranslateX, mPortraitTranslateY); - // Apply the matrix to TextureView (GPU-accelerated, bilinear filtering) + // Apply Matrix textureView.setTransform(matrix); textureView.invalidate(); - // Sync the PiP indicator with the main view's zoom/pan state + // Sync PiP indicator ZoomLayoutDeal.updateIndicatorFromMainView( - scale, + zoomScale, mPortraitTranslateX, mPortraitTranslateY ); - Log.d(TAG, "Portrait zoom applied: scale=" + scale + + Log.d(TAG, "Portrait zoom applied: scale=" + zoomScale + + " surface=" + surfaceWidth + "x" + surfaceHeight + + " needsCorrection=" + needsCorrection + " translateX=" + mPortraitTranslateX + " translateY=" + mPortraitTranslateY); } /** - * Expand TextureView's physical height when zoomed in, so more vertical content is visible. - * The height increases with zoom scale, capped at MAX_HEIGHT_RATIO of screen height. + * Ensure TextureView fills video_area_container (MATCH_PARENT) in portrait mode. + * No longer dynamically change TextureView size to avoid triggering onSurfaceTextureSizeChanged. + * All zoom/pan/stretch correction is handled by Matrix in applyPortraitZoomTransform(). + * + * @return null (no longer returns target size since TextureView size is not changed) */ - private void expandTextureViewHeightIfNeeded() { + private int[] expandTextureViewToTarget() { if (activity == null || activity.mBinding == null || activity.mBinding.viewMainPreview == null) { - return; + return null; } AspectRatioTextureView textureView = activity.mBinding.viewMainPreview; - View container = activity.mBinding.videoAreaContainer; - - if (container == null) return; - - // Get screen height - WindowManager wm = (WindowManager) activity.getSystemService("window"); - if (wm == null) return; - DisplayMetrics metrics = new DisplayMetrics(); - wm.getDefaultDisplay().getRealMetrics(metrics); - int screenHeight = metrics.heightPixels; - - // Get original container height (when not zoomed) - int containerHeight = container.getHeight(); - if (containerHeight <= 0) return; - - // Calculate expanded height based on zoom scale - // scale = 1.0 → height = containerHeight (original) - // scale = MAX_SCALE → height = MAX_HEIGHT_RATIO * screenHeight - // Linear interpolation between these two points - final float MAX_SCALE = 3.0f; - final float MAX_HEIGHT_RATIO = 0.85f; // Cap at 85% of screen height - - int maxHeight = (int) (screenHeight * MAX_HEIGHT_RATIO); - int minHeight = containerHeight; - - float scale = mPortraitZoomScale; - float t = (scale - 1.0f) / (MAX_SCALE - 1.0f); // 0 to 1 as scale goes from 1.0 to MAX_SCALE - t = Math.max(0f, Math.min(1f, t)); - - int targetHeight = minHeight + (int) ((maxHeight - minHeight) * t); - - // Update LayoutParams if height changed - ViewGroup.LayoutParams params = textureView.getLayoutParams(); - if (params.height != targetHeight) { - params.height = targetHeight; - textureView.setLayoutParams(params); - Log.d(TAG, "Expanded TextureView height: " + targetHeight + " (scale=" + scale + ")"); + + // Ensure TextureView is MATCH_PARENT in portrait mode, disable aspect ratio constraint + if (isPortraitMode && textureView.isAspectRatioEnabled()) { + textureView.setAspectRatioEnabled(false); + ViewGroup.LayoutParams params = textureView.getLayoutParams(); + if (params.width != ViewGroup.LayoutParams.MATCH_PARENT + || params.height != ViewGroup.LayoutParams.MATCH_PARENT) { + params.width = ViewGroup.LayoutParams.MATCH_PARENT; + params.height = ViewGroup.LayoutParams.MATCH_PARENT; + textureView.setLayoutParams(params); + Log.d(TAG, "Set TextureView to MATCH_PARENT for portrait Matrix mode"); + } } + + return null; // No longer change size, return null } /** - * Reset TextureView's height back to match_parent when zoomed out. + * Reset TextureView when zoomed out. + * In portrait mode, TextureView always stays MATCH_PARENT, no need to restore. + * Only ensure aspect ratio constraint is disabled (Matrix handles all display transforms). */ private void resetTextureViewHeight() { if (activity == null || activity.mBinding == null || activity.mBinding.viewMainPreview == null) { @@ -1367,11 +1418,17 @@ private void resetTextureViewHeight() { } AspectRatioTextureView textureView = activity.mBinding.viewMainPreview; - ViewGroup.LayoutParams params = textureView.getLayoutParams(); - if (params.height != ViewGroup.LayoutParams.MATCH_PARENT) { - params.height = ViewGroup.LayoutParams.MATCH_PARENT; - textureView.setLayoutParams(params); - Log.d(TAG, "Reset TextureView height to MATCH_PARENT"); + // Keep aspect ratio constraint disabled in portrait mode + if (isPortraitMode && textureView.isAspectRatioEnabled()) { + textureView.setAspectRatioEnabled(false); + ViewGroup.LayoutParams params = textureView.getLayoutParams(); + if (params.height != ViewGroup.LayoutParams.MATCH_PARENT + || params.width != ViewGroup.LayoutParams.MATCH_PARENT) { + params.height = ViewGroup.LayoutParams.MATCH_PARENT; + params.width = ViewGroup.LayoutParams.MATCH_PARENT; + textureView.setLayoutParams(params); + } + Log.d(TAG, "Reset TextureView to MATCH_PARENT (portrait Matrix mode)"); } } @@ -1385,7 +1442,22 @@ private void adjustZoom(float zoomFactor) { // Define scale limits float MIN_SCALE = 1f; - float MAX_SCALE = 3.0f; + // Dynamically calculate max scale: video display height should not exceed specified percentage of screen height + // Video natural height = screenWidth / (bufferWidth / bufferHeight) + // maxScale = (screenHeight × MAX_DISPLAY_HEIGHT_RATIO) / naturalHeight + final float MAX_DISPLAY_HEIGHT_RATIO = 0.5f; // Max video display height as ratio of screen height + float MAX_SCALE = 3.0f; // Default value, calculated dynamically below + if (activity.getPreviewWidth() > 0 && activity.getPreviewHeight() > 0) { + android.util.DisplayMetrics metrics = new android.util.DisplayMetrics(); + activity.getWindowManager().getDefaultDisplay().getRealMetrics(metrics); + float screenWidth = metrics.widthPixels; + float screenHeight = metrics.heightPixels; + float bufferAspect = (float) activity.getPreviewWidth() / activity.getPreviewHeight(); + float naturalHeight = screenWidth / bufferAspect; + float maxHeight = screenHeight * MAX_DISPLAY_HEIGHT_RATIO; + MAX_SCALE = maxHeight / naturalHeight; + if (MAX_SCALE < MIN_SCALE) MAX_SCALE = MIN_SCALE; + } // Calculate new scale float currentScale = mPortraitZoomScale; diff --git a/app/src/main/res/layout-port/activity_main.xml b/app/src/main/res/layout-port/activity_main.xml index ed2d726..76f8c5e 100644 --- a/app/src/main/res/layout-port/activity_main.xml +++ b/app/src/main/res/layout-port/activity_main.xml @@ -62,6 +62,8 @@ android:layout_width="match_parent" android:layout_height="0dp" android:layout_weight="1" + android:clipChildren="true" + android:clipToPadding="true" android:background="@color/black"> diff --git a/docs/video/VIDEO_STREAM_FRAMEWORK.md b/docs/video/VIDEO_STREAM_FRAMEWORK.md new file mode 100644 index 0000000..6dd5f46 --- /dev/null +++ b/docs/video/VIDEO_STREAM_FRAMEWORK.md @@ -0,0 +1,335 @@ +# Video Stream Display Framework - Portrait Mode Implementation + +## Overview + +This document describes the video stream display framework in the Openterface Android app, focusing on the portrait mode implementation that enables zoom/pan functionality while maintaining video aspect ratio and display quality. + +## Architecture + +### Core Components + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ MainActivity │ +│ - Surface lifecycle management │ +│ - Camera connection and preview control │ +│ - Portrait/Landscape mode detection │ +└─────────────────────────────────────────────────────────────────┘ + │ + ┌───────────────────┴───────────────────┐ + ▼ ▼ +┌─────────────────────────┐ ┌─────────────────────────┐ +│ AspectRatioTextureView │ │ CustomTouchListener │ +│ (viewMainPreview) │ │ - Pinch-to-zoom │ +│ - Main video display │ │ - Matrix transforms │ +│ - SurfaceTexture mgmt │ │ - Touch event handling │ +└─────────────────────────┘ └─────────────────────────┘ + │ │ + ▼ ▼ +┌─────────────────────────┐ ┌─────────────────────────┐ +│ RendererHolder │ │ ZoomLayoutDeal │ +│ - Frame distribution │ │ - PiP window control │ +│ - Primary/Slave │ │ - Indicator sync │ +│ surfaces │ │ - Pan position calc │ +└─────────────────────────┘ └─────────────────────────┘ + │ + ▼ +┌─────────────────────────┐ +│ AspectRatioSurfaceView│ +│ (cameraViewSecond) │ +│ - PiP video display │ +└─────────────────────────┘ +``` + +## Video Frame Flow + +``` +UVC Camera + │ + ▼ +CameraInternal.mRendererHolder (OpenGL ES context) + │ + ├──► Primary Surface (viewMainPreview) + │ │ + │ ├──► GPU Texture (1920x1080 buffer) + │ │ + │ └──► Matrix Transform (stretch correction + zoom + pan) + │ │ + │ └──► Display on screen + │ + └──► Slave Surface (cameraViewSecond - PiP) + │ + └──► Same texture rendered at smaller size +``` + +## Portrait Mode Implementation + +### Problem Statement + +In portrait mode, the video stream (typically 16:9 aspect ratio) needs to: +1. Display correctly without stretching +2. Support pinch-to-zoom with smooth scaling +3. Allow panning via PiP indicator +4. Maintain video quality when zoomed +5. Handle keyboard pop-up without crash or visual glitches + +### Solution: Matrix-Based Transform + +Instead of dynamically resizing the TextureView (which causes surface recreation and flickering), we use a fixed-size TextureView with Matrix transforms: + +```java +// TextureView is always MATCH_PARENT in portrait mode +// Matrix handles all visual transformations: +// 1. Stretch correction (SurfaceTexture aspect ratio mismatch) +// 2. Zoom scaling +// 3. Pan translation +``` + +### Key Files Modified + +#### 1. AspectRatioTextureView.java + +Added ability to toggle aspect ratio enforcement: + +```java +private boolean mAspectRatioEnabled = true; + +public void setAspectRatioEnabled(boolean enabled) { + mAspectRatioEnabled = enabled; // Does NOT call requestLayout() +} + +@Override +protected void onMeasure(int widthMeasureSpec, int heightMeasureSpec) { + if (mRequestedAspect > 0 && mAspectRatioEnabled) { + // Original aspect ratio enforcement logic + } + super.onMeasure(widthMeasureSpec, heightMeasureSpec); +} +``` + +#### 2. CustomTouchListener.java + +**Stretch Correction Logic:** + +When TextureView dimensions don't match camera buffer aspect ratio, SurfaceTexture stretches the content. Matrix corrects this: + +```java +public void applyPortraitZoomTransform() { + float surfaceAspect = surfaceWidth / surfaceHeight; + float bufferAspect = bufferWidth / bufferHeight; + boolean needsCorrection = Math.abs(surfaceAspect - bufferAspect) > 0.01f; + + if (needsCorrection) { + // Correct stretch: maintain aspect ratio, fill width, center vertically + float correctionScaleY = (bufferHeight * surfaceWidth) / (surfaceHeight * bufferWidth); + matrix.setScale(1.0f, correctionScaleY); + + float correctedHeight = surfaceHeight * correctionScaleY; + float translateY = (surfaceHeight - correctedHeight) / 2f; + matrix.postTranslate(0, translateY); + } + + // Apply zoom (pivot at center) + matrix.postScale(zoomScale, zoomScale, pivotX, pivotY); + + // Apply pan (user drag) + matrix.postTranslate(mPortraitTranslateX, mPortraitTranslateY); + + textureView.setTransform(matrix); +} +``` + +**Dynamic MAX_SCALE Calculation:** + +```java +final float MAX_DISPLAY_HEIGHT_RATIO = 0.5f; // Max 50% of screen height +float naturalHeight = screenWidth / bufferAspect; +float maxHeight = screenHeight * MAX_DISPLAY_HEIGHT_RATIO; +MAX_SCALE = maxHeight / naturalHeight; +``` + +#### 3. MainActivity.java + +**Surface Lifecycle in Portrait Mode:** + +```java +@Override +public void onSurfaceTextureSizeChanged(SurfaceTexture surface, int width, int height) { + if (isPortraitMode) { + // Skip surface recreation during keyboard animation + // Just update buffer size and Matrix + setSurfaceTextureBufferSize(surface, mPreviewWidth, mPreviewHeight); + mBinding.viewMainPreview.post(() -> + mCustomTouchListener.applyPortraitZoomTransform()); + } else { + // Landscape: full surface recreation with aspect ratio update + // ... (existing logic) + } +} +``` + +**Conditional Aspect Ratio Setting:** + +All `setAspectRatio()` calls are wrapped with portrait mode check: + +```java +if (!isPortraitMode) { + mBinding.viewMainPreview.setAspectRatio(mPreviewWidth, mPreviewHeight); +} +``` + +#### 4. ZoomLayoutDeal.java + +**PiP Indicator to Pan Translation:** + +```java +private static void syncMainViewPosition(float thumbX, float thumbY) { + if (drawerLayout == null) { + // Portrait mode: use Matrix-based panning + float currentZoomScale = CustomTouchListener.getPortraitZoomScale(); + + if (currentZoomScale <= 1.0f) return; + + // Calculate actual content dimensions after stretch correction + float contentHeight = viewWidth * bufferHeight / bufferWidth; + + // Use content dimensions (not view dimensions) for accurate edge mapping + float maxTranslateX = (contentWidth * (currentZoomScale - 1f)) / 2f; + float maxTranslateY = (contentHeight * (currentZoomScale - 1f)) / 2f; + + // Map indicator position to translation values + float normalizedX = indicatorX / maxIndicatorX; + float newTranslateX = (0.5f - normalizedX) * 2f * maxTranslateX; + + CustomTouchListener.setPortraitPan(newTranslateX, newTranslateY); + CustomTouchListener.applyCurrentPortraitTransform(); + } +} +``` + +## Surface Management + +### Buffer Size Configuration + +Critical for zoom quality - sets SurfaceTexture buffer to camera native resolution: + +```java +private void setSurfaceTextureBufferSize(SurfaceTexture surface, int bufferWidth, int bufferHeight) { + // Use camera resolution for buffer, not the small view size + surface.setDefaultBufferSize(bufferWidth, bufferHeight); +} +``` + +This ensures GPU has enough source pixels when zooming, preventing blur. + +### RendererHolder Frame Distribution + +``` +Primary Surface (viewMainPreview) + │ + ├──► handleDraw() - updateTexImage() + │ + └──► handleDrawSlaveSurfaces() + │ + └──► For each slave surface: + onDrawSlaveSurface() → surface.draw(mDrawer, texId, texMatrix, mvpMatrix) +``` + +Both surfaces receive identical frames from the same OpenGL texture. + +## PiP (Picture-in-Picture) System + +### Layout Structure + +```xml + + + + + + + + + + +``` + +### Drag Operations + +1. **Drag Button**: Moves PiP window position on screen (visual only) +2. **Thumbnail Drag**: Pans the zoomed main view via indicator position + +### Bidirectional Sync + +- **PiP → Main View**: `syncMainViewPosition()` → `setPortraitPan()` → `applyCurrentPortraitTransform()` +- **Main View → PiP**: `applyPortraitZoomTransform()` → `ZoomLayoutDeal.updateIndicatorFromMainView()` + +## Issues Fixed + +### 1. Video Stretching in Portrait Mode + +**Problem**: SurfaceTexture stretches content when view aspect ratio differs from buffer. + +**Solution**: Matrix stretch correction that scales Y-axis to maintain aspect ratio. + +### 2. Crash During Keyboard Pop-up + +**Problem**: `onSurfaceTextureSizeChanged` fired repeatedly during keyboard animation, each time removing/re-adding surfaces, causing crash. + +**Solution**: Skip surface recreation in portrait mode, only update buffer size and Matrix. + +### 3. Flickering During Pinch Zoom + +**Problem**: Multiple `requestLayout()` calls per touch event. + +**Solution**: +- Removed `requestLayout()` from `setAspectRatioEnabled()` +- Added thresholds to `setLayoutParams()` calls +- TextureView stays fixed size, Matrix handles all transforms + +### 4. Video Blur When Zoomed + +**Problem**: SurfaceTexture buffer defaulted to view size (smaller than camera resolution). + +**Solution**: Explicitly set buffer size to camera native resolution (1920x1080). + +### 5. PiP Drag Showing Black Bars + +**Problem**: `maxTranslateY` calculated using view height instead of corrected content height. + +**Solution**: Calculate max translation based on actual content dimensions after stretch correction. + +### 6. PiP Drag Causing Video Distortion + +**Problem**: `applyCurrentPortraitTransform()` lacked stretch correction. + +**Solution**: Added same stretch correction logic as `applyPortraitZoomTransform()`. + +## Code Statistics + +| File | Lines Changed | Key Additions | +|------|---------------|---------------| +| MainActivity.java | ~100 | Surface lifecycle, conditional aspect ratio | +| CustomTouchListener.java | ~200 | Matrix transforms, stretch correction | +| ZoomLayoutDeal.java | ~30 | Content-based translation calculation | +| AspectRatioTextureView.java | ~20 | Aspect ratio toggle | +| layout-port/activity_main.xml | 2 | clipChildren, clipToPadding | + +## Testing Checklist + +- [x] Initialization shows video correctly at natural aspect ratio +- [x] Pinch-to-zoom works smoothly without stretching +- [x] Zoom caps at 50% screen height +- [x] Opening keyboard doesn't compress video or crash +- [x] PiP indicator drag pans main view correctly +- [x] No black bars when dragging PiP to edges +- [x] Video remains clear when zoomed (no blur) +- [x] Screen rotation works correctly +- [x] PiP window can be dragged without affecting main video + +## Future Improvements + +1. **Smooth Animation**: Add interpolation for zoom/pan transitions +2. **Gesture Refinement**: Better double-tap to zoom to specific area +3. **Performance**: Optimize Matrix calculations for lower-end devices +4. **Accessibility**: Add keyboard shortcuts for zoom/pan control diff --git a/libuvccamera/src/main/java/com/serenegiant/widget/AspectRatioTextureView.java b/libuvccamera/src/main/java/com/serenegiant/widget/AspectRatioTextureView.java index c859d8d..5482fa6 100644 --- a/libuvccamera/src/main/java/com/serenegiant/widget/AspectRatioTextureView.java +++ b/libuvccamera/src/main/java/com/serenegiant/widget/AspectRatioTextureView.java @@ -46,6 +46,7 @@ public class AspectRatioTextureView extends TextureView // API >= 14 private static final String TAG = AspectRatioTextureView.class.getSimpleName(); private double mRequestedAspect = -1.0; + private boolean mAspectRatioEnabled = true; // 新增：控制是否强制宽高比 private CameraViewInterface.Callback mCallback; public AspectRatioTextureView(final Context context) { @@ -85,10 +86,26 @@ public double getAspectRatio() { return mRequestedAspect; } + /** + * 临时禁用/启用宽高比约束。 + * 竖屏放大时需要禁用，让外部设置的 height 生效； + * 缩回时恢复，让 onMeasure 重新按摄像头比例计算尺寸。 + * + * 注意：此方法不触发 requestLayout()，调用方需自行通过 setLayoutParams() 触发布局。 + */ + public void setAspectRatioEnabled(boolean enabled) { + mAspectRatioEnabled = enabled; + } + + public boolean isAspectRatioEnabled() { + return mAspectRatioEnabled; + } + @Override protected void onMeasure(int widthMeasureSpec, int heightMeasureSpec) { - if (mRequestedAspect > 0) { + // 竖屏放大时禁用宽高比约束，让外部设置的高度直接生效 + if (mRequestedAspect > 0 && mAspectRatioEnabled) { int initialWidth = MeasureSpec.getSize(widthMeasureSpec); int initialHeight = MeasureSpec.getSize(heightMeasureSpec);