From 29d6a92ee860d975fd24d7e3bdde302c19e0e3fb Mon Sep 17 00:00:00 2001 From: Yueyu Date: Fri, 18 Jun 2021 07:28:39 +0800 Subject: [PATCH] Android: upgrade to camera2 --- android/ScratchJr/app/build.gradle | 15 +- .../org/scratchjr/android/CameraxView.java | 300 ++++++++++++++++++ .../android/JavaScriptDirectInterface.java | 36 ++- .../scratchjr/android/ScratchJrActivity.java | 3 +- .../app/src/main/res/values/styles.xml | 2 +- 5 files changed, 341 insertions(+), 15 deletions(-) create mode 100644 android/ScratchJr/app/src/main/java/org/scratchjr/android/CameraxView.java diff --git a/android/ScratchJr/app/build.gradle b/android/ScratchJr/app/build.gradle index c0c064e..6f46d38 100644 --- a/android/ScratchJr/app/build.gradle +++ b/android/ScratchJr/app/build.gradle @@ -26,6 +26,11 @@ android { versionName "1.2.11" } } + + compileOptions { + sourceCompatibility JavaVersion.VERSION_1_8 + targetCompatibility JavaVersion.VERSION_1_8 + } } dependencies { @@ -33,7 +38,15 @@ dependencies { implementation 'com.google.firebase:firebase-core:17.2.0' implementation 'com.google.firebase:firebase-analytics:17.2.0' implementation 'com.google.android.gms:play-services-location:17.0.0' - implementation 'androidx.appcompat:appcompat:1.1.0' + + implementation 'androidx.appcompat:appcompat:1.3.0' + def camerax_version = "1.0.0-rc05" + // CameraX core library using camera2 implementation + implementation "androidx.camera:camera-camera2:$camerax_version" + // CameraX Lifecycle Library + implementation "androidx.camera:camera-lifecycle:$camerax_version" + // CameraX View class + implementation "androidx.camera:camera-view:1.0.0-alpha25" } def appModuleRootFolder = '.' diff --git a/android/ScratchJr/app/src/main/java/org/scratchjr/android/CameraxView.java b/android/ScratchJr/app/src/main/java/org/scratchjr/android/CameraxView.java new file mode 100644 index 0000000..ba55ba9 --- /dev/null +++ b/android/ScratchJr/app/src/main/java/org/scratchjr/android/CameraxView.java @@ -0,0 +1,300 @@ +package org.scratchjr.android; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import android.annotation.SuppressLint; +import android.content.Context; +import android.graphics.Bitmap; +import android.graphics.Bitmap.CompressFormat; +import android.graphics.ImageFormat; +import android.graphics.Matrix; +import android.graphics.Rect; +import android.graphics.RectF; +import android.graphics.YuvImage; +import android.hardware.display.DisplayManager; +import android.media.Image; +import android.util.Log; +import android.view.Display; +import android.view.Surface; +import android.view.WindowManager; +import android.widget.RelativeLayout; + +import androidx.appcompat.app.AppCompatActivity; +import androidx.camera.core.CameraInfoUnavailableException; +import androidx.camera.core.CameraSelector; +import androidx.camera.core.ImageCapture; +import androidx.camera.core.Preview; +import androidx.camera.lifecycle.ProcessCameraProvider; +import androidx.camera.view.PreviewView; +import androidx.core.content.ContextCompat; + +import com.google.common.util.concurrent.ListenableFuture; + +@SuppressLint("ViewConstructor") +public class CameraxView extends RelativeLayout { + private static final String LOG_TAG = "ScratchJr.CameraxView"; + + private final RectF _rect; + private boolean _currentFacingFront; + private final float _scale; + + private final AppCompatActivity _activity; + private ProcessCameraProvider _cameraProvider; + private final PreviewView _viewFinder; + private Preview _preview; + private ImageCapture _imageCapture; + private final ExecutorService _cameraExecutor; + private final DisplayManager _displayManager; + private int _displayId; + + public CameraxView(AppCompatActivity context, RectF rect, float scale, boolean facingFront) { + super(context); + _activity = context; + _currentFacingFront = facingFront; + _rect = rect; + _scale = scale; + + _viewFinder = new PreviewView(context); + + _displayManager = (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE); + + RelativeLayout.LayoutParams layoutParams = new RelativeLayout.LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT); + addView(_viewFinder, layoutParams); + post(() -> { + _displayId = _viewFinder.getDisplay().getDisplayId(); + setupCamera(); + }); + + _cameraExecutor = Executors.newSingleThreadExecutor(); + } + + @Override + protected void onAttachedToWindow() { + super.onAttachedToWindow(); + _displayManager.registerDisplayListener(displayListener, null); + } + + @Override + protected void onDetachedFromWindow() { + super.onDetachedFromWindow(); + _displayManager.unregisterDisplayListener(displayListener); + } + + private final DisplayManager.DisplayListener displayListener = new DisplayManager.DisplayListener() { + @Override + public void onDisplayAdded(int displayId) { + } + + @Override + public void onDisplayRemoved(int displayId) { + } + + @SuppressLint("UnsafeOptInUsageError") + @Override + public void onDisplayChanged(int displayId) { + if (displayId == _displayId) { + int rotation = _viewFinder.getDisplay().getRotation(); + if (_imageCapture != null) { + _imageCapture.setTargetRotation(rotation); + } + if (_preview != null) { + _preview.setTargetRotation(rotation); + } + } + } + }; + + private void setupCamera() { + ListenableFuture cameraProviderFuture = ProcessCameraProvider.getInstance(_activity); + cameraProviderFuture.addListener(() -> { + try { + _cameraProvider = cameraProviderFuture.get(); + + bindCameraUseCases(); + } catch (ExecutionException | InterruptedException e) { + e.printStackTrace(); + } + }, ContextCompat.getMainExecutor(_activity)); + } + + private void bindCameraUseCases() { + int rotation = _viewFinder.getDisplay().getRotation(); + Log.d(LOG_TAG, "rotation: " + rotation); + _imageCapture = new ImageCapture.Builder() + .setTargetRotation(rotation) + .build(); + + _preview = new Preview.Builder() + .setTargetRotation(rotation) + .build(); + + _cameraProvider.unbindAll(); + + int lensFacing = CameraSelector.LENS_FACING_FRONT; + if (!_currentFacingFront) { + lensFacing = CameraSelector.LENS_FACING_BACK; + } + + CameraSelector selector = new CameraSelector.Builder() + .requireLensFacing(lensFacing) + .build(); + + _cameraProvider.bindToLifecycle( + _activity, + selector, + _preview, + _imageCapture + ); + + _preview.setSurfaceProvider(_viewFinder.getSurfaceProvider()); + } + + public void captureStillImage(ImageCapture.OnImageCapturedCallback callback) { + _imageCapture.takePicture(_cameraExecutor, callback); + } + + public boolean setCameraFacing(boolean facingFront) { + if (_currentFacingFront != facingFront) { + if (facingFront && !hasFrontCamera()) { + return false; + } + if (!facingFront && !hasBackCamera()) { + return false; + } + _currentFacingFront = facingFront; + post(this::bindCameraUseCases); + } + return true; + } + + private boolean hasBackCamera() { + try { + return _cameraProvider.hasCamera(CameraSelector.DEFAULT_BACK_CAMERA); + } catch (CameraInfoUnavailableException e) { + e.printStackTrace(); + } + return false; + } + + private boolean hasFrontCamera() { + try { + return _cameraProvider.hasCamera(CameraSelector.DEFAULT_FRONT_CAMERA); + } catch (CameraInfoUnavailableException e) { + e.printStackTrace(); + } + return false; + } + + /** + * Take the given bitmap image from the camera and transform it to the correct + * aspect ratio, size, and rotation. + * + * @return jpeg-encoded data of the transformed image. + */ + public byte[] getTransformedImage(Bitmap originalImage, int exifRotation) { + Bitmap cropped = cropResizeAndRotate(originalImage, exifRotation); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + cropped.compress(CompressFormat.JPEG, 90, bos); + try { + bos.close(); + } catch (IOException e) { + // will not happen - this is a ByteArrayOutputStream + Log.e(LOG_TAG, "IOException while closing byte array stream", e); + } + return bos.toByteArray(); + } + + /** + * Crop and resize the given image to the dimensions of the rectangle for this camera view. + *

+ * If the image was front-facing, also mirror horizontally. + */ + private Bitmap cropResizeAndRotate(Bitmap image, int exifRotation) { + int imageWidth = image.getWidth(); + int imageHeight = image.getHeight(); + float rectWidth = _rect.width(); + float rectHeight = _rect.height(); + + float newHeight = rectWidth * imageHeight / imageWidth; + float scale = rectWidth / imageWidth; + int offsetX = 0; + int offsetY = (int) ((newHeight - rectHeight) / 2 * imageHeight / newHeight); + if (newHeight < rectHeight) { + float newWidth = rectHeight * imageWidth / imageHeight; + scale = rectHeight / imageHeight; + offsetY = 0; + offsetX = (int) ((newWidth - rectWidth) / 2 * imageWidth / newWidth); + } + + Matrix m = new Matrix(); + // Adjust the image to undo rotation done by JPEG generator + m.postRotate(-1.0f * exifRotation); + if (_currentFacingFront) { + // flip bitmap horizontally since front-facing camera is mirrored + m.preScale(-1.0f, 1.0f); + } + int rotation = CameraxView.findDisplayRotation(getContext(), _currentFacingFront); + if (rotation == 180) { + m.preScale(-1.0f, -1.0f); + } + m.postScale(scale / _scale, scale / _scale); + return Bitmap.createBitmap(image, offsetX, offsetY, imageWidth - offsetX * 2, imageHeight - offsetY * 2, m, true); + } + + // Image → JPEG + public byte[] imageToByteArray(Image image) { + byte[] data = null; + if (image.getFormat() == ImageFormat.JPEG) { + Image.Plane[] planes = image.getPlanes(); + ByteBuffer buffer = planes[0].getBuffer(); + data = new byte[buffer.capacity()]; + buffer.get(data); + return data; + } else if (image.getFormat() == ImageFormat.YUV_420_888) { + data = NV21toJPEG(YUV_420_888toNV21(image), + image.getWidth(), image.getHeight()); + } + return data; + } + + // YUV_420_888 → NV21 + private byte[] YUV_420_888toNV21(Image image) { + byte[] nv21; + ByteBuffer yBuffer = image.getPlanes()[0].getBuffer(); + ByteBuffer uBuffer = image.getPlanes()[1].getBuffer(); + ByteBuffer vBuffer = image.getPlanes()[2].getBuffer(); + int ySize = yBuffer.remaining(); + int uSize = uBuffer.remaining(); + int vSize = vBuffer.remaining(); + nv21 = new byte[ySize + uSize + vSize]; + yBuffer.get(nv21, 0, ySize); + vBuffer.get(nv21, ySize, vSize); + uBuffer.get(nv21, ySize + vSize, uSize); + return nv21; + } + + // NV21 → JPEG + private byte[] NV21toJPEG(byte[] nv21, int width, int height) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + YuvImage yuv = new YuvImage(nv21, ImageFormat.NV21, width, height, null); + yuv.compressToJpeg(new Rect(0, 0, width, height), 100, out); + return out.toByteArray(); + } + + private static int findDisplayRotation(Context context, boolean facingFront) { + WindowManager windowManager = (WindowManager) context.getSystemService(Context.WINDOW_SERVICE); + Display display = windowManager.getDefaultDisplay(); + int r = display.getRotation(); + int rotation = (r == Surface.ROTATION_180) ? 180 : 0; + if (facingFront) { + rotation = (rotation + 360) % 360; + } + return rotation; + } + +} diff --git a/android/ScratchJr/app/src/main/java/org/scratchjr/android/JavaScriptDirectInterface.java b/android/ScratchJr/app/src/main/java/org/scratchjr/android/JavaScriptDirectInterface.java index f1ec612..ca5de04 100644 --- a/android/ScratchJr/app/src/main/java/org/scratchjr/android/JavaScriptDirectInterface.java +++ b/android/ScratchJr/app/src/main/java/org/scratchjr/android/JavaScriptDirectInterface.java @@ -13,6 +13,7 @@ import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; +import android.annotation.SuppressLint; import android.content.Context; import android.content.Intent; import android.content.pm.PackageManager; @@ -20,6 +21,7 @@ import android.graphics.Bitmap; import android.graphics.BitmapFactory; import android.graphics.RectF; import android.hardware.Camera; +import android.media.Image; import android.net.Uri; import android.text.Html; import android.util.Base64; @@ -29,6 +31,11 @@ import android.webkit.JavascriptInterface; import android.widget.ImageView; import android.widget.RelativeLayout; +import androidx.annotation.NonNull; +import androidx.camera.core.ImageCapture; +import androidx.camera.core.ImageCaptureException; +import androidx.camera.core.ImageProxy; + /** * The methods in this inner class are exposed directly to JavaScript in the HTML5 pages * as AndroidInterface. @@ -42,7 +49,7 @@ public class JavaScriptDirectInterface { private final ScratchJrActivity _activity; /** Current camera view, if active */ - private CameraView _cameraView; + private CameraxView _cameraView; /** Current camera mask, if active */ private ImageView _cameraMask; @@ -410,19 +417,24 @@ public class JavaScriptDirectInterface { @JavascriptInterface public void scratchjr_captureimage(final String onCameraCaptureComplete) { - _cameraView.captureStillImage( - new Camera.PictureCallback() { - public void onPictureTaken(byte[] jpegData, Camera camera) { + _cameraView.captureStillImage(new ImageCapture.OnImageCapturedCallback() { + @Override + public void onCaptureSuccess(@NonNull ImageProxy imageProxy) { + @SuppressLint("UnsafeOptInUsageError") + Image image = imageProxy.getImage(); + if (image != null) { + byte[] jpegData = _cameraView.imageToByteArray(image); sendBase64Image(onCameraCaptureComplete, jpegData); } - }, - new Runnable() { - public void run() { - Log.e(LOG_TAG, "Could not capture picture"); - reportImageError(onCameraCaptureComplete); - } + imageProxy.close(); } - ); + + @Override + public void onError(@NonNull ImageCaptureException exception) { + Log.e(LOG_TAG, "Could not capture picture"); + reportImageError(onCameraCaptureComplete); + } + }); } @JavascriptInterface @@ -493,7 +505,7 @@ public class JavaScriptDirectInterface { scaleRectFromCenter(maskRect, scale); RelativeLayout container = _activity.getContainer(); - _cameraView = new CameraView(_activity, rect, scale * devicePixelRatio, true); // always start with front-facing camera + _cameraView = new CameraxView(_activity, rect, scale * devicePixelRatio, true); // always start with front-facing camera container.addView(_cameraView, new RelativeLayout.LayoutParams((int) (rect.width()), (int) (rect.height()))); _cameraView.setX(rect.left); _cameraView.setY(rect.top); diff --git a/android/ScratchJr/app/src/main/java/org/scratchjr/android/ScratchJrActivity.java b/android/ScratchJr/app/src/main/java/org/scratchjr/android/ScratchJrActivity.java index 5e1e623..d4023eb 100644 --- a/android/ScratchJr/app/src/main/java/org/scratchjr/android/ScratchJrActivity.java +++ b/android/ScratchJr/app/src/main/java/org/scratchjr/android/ScratchJrActivity.java @@ -15,6 +15,7 @@ import android.os.Build; import android.os.Bundle; import android.os.Handler; import androidx.annotation.NonNull; +import androidx.appcompat.app.AppCompatActivity; import androidx.core.app.ActivityCompat; import androidx.core.content.ContextCompat; import android.util.Log; @@ -49,7 +50,7 @@ import java.util.Vector; * @author markroth8 */ public class ScratchJrActivity - extends Activity + extends AppCompatActivity { /** Milliseconds to pan when showing the soft keyboard */ private static final int SOFT_KEYBOARD_PAN_MS = 250; diff --git a/android/ScratchJr/app/src/main/res/values/styles.xml b/android/ScratchJr/app/src/main/res/values/styles.xml index 8d8c40e..03d41e3 100644 --- a/android/ScratchJr/app/src/main/res/values/styles.xml +++ b/android/ScratchJr/app/src/main/res/values/styles.xml @@ -5,7 +5,7 @@ -