Merge pull request #451 from yueyuzhao/issue/27-camera2

Issue/27 camera2

android/ScratchJr/app/build.gradle

@@ -26,6 +26,14 @@ android {
             versionName "1.2.11"
         }
     }
+
+    // The camera-view dependency contains Java 8 bytecode,
+    // we need to support java 8 to dex.
+    // See https://developer.android.com/studio/write/java8-support.html for details.
+    compileOptions {
+        sourceCompatibility JavaVersion.VERSION_1_8
+        targetCompatibility JavaVersion.VERSION_1_8
+    }
 }
 
 dependencies {
@@ -33,7 +41,15 @@ dependencies {
     implementation 'com.google.firebase:firebase-core:17.2.0'
     implementation 'com.google.firebase:firebase-analytics:17.2.0'
     implementation 'com.google.android.gms:play-services-location:17.0.0'
-    implementation 'androidx.appcompat:appcompat:1.1.0'
+
+    implementation 'androidx.appcompat:appcompat:1.3.0'
+    def camerax_version = "1.0.0-rc05"
+    // CameraX core library using camera2 implementation
+    implementation "androidx.camera:camera-camera2:$camerax_version"
+    // CameraX Lifecycle Library
+    implementation "androidx.camera:camera-lifecycle:$camerax_version"
+    // CameraX View class
+    implementation "androidx.camera:camera-view:1.0.0-alpha25"
 }
 
 def appModuleRootFolder = '.'

android/ScratchJr/app/src/main/java/org/scratchjr/android/CameraView.java

@@ -2,179 +2,195 @@ package org.scratchjr.android;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.util.List;
+import java.nio.ByteBuffer;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
+import android.annotation.SuppressLint;
 import android.content.Context;
 import android.graphics.Bitmap;
 import android.graphics.Bitmap.CompressFormat;
+import android.graphics.ImageFormat;
 import android.graphics.Matrix;
+import android.graphics.Rect;
 import android.graphics.RectF;
-import android.hardware.Camera;
+import android.graphics.YuvImage;
-import android.hardware.Camera.CameraInfo;
+import android.hardware.display.DisplayManager;
-import android.hardware.Camera.Parameters;
+import android.media.Image;
-import android.hardware.Camera.PictureCallback;
-import android.hardware.Camera.Size;
-import android.hardware.SensorManager;
 import android.util.Log;
 import android.view.Display;
-import android.view.MotionEvent;
-import android.view.OrientationEventListener;
 import android.view.Surface;
-import android.view.SurfaceHolder;
-import android.view.SurfaceView;
-import android.view.ViewGroup;
 import android.view.WindowManager;
-import android.widget.LinearLayout;
+import android.widget.RelativeLayout;
-import android.widget.ScrollView;
 
-/**
+import androidx.appcompat.app.AppCompatActivity;
- * Creates a camera view that hovers at a particular location and has a mask.
+import androidx.camera.core.CameraInfoUnavailableException;
- * 
+import androidx.camera.core.CameraSelector;
- * We use a ScrollView because the camera will rescale (squish) the preview to whatever size the
+import androidx.camera.core.ExperimentalUseCaseGroup;
- * SurfaceView is and we want to keep the aspect ratio. So the ScrollView is of the desired
+import androidx.camera.core.ImageCapture;
- * size and then we add a SurfaceView to it with the camera preview.
+import androidx.camera.core.Preview;
- * 
+import androidx.camera.lifecycle.ProcessCameraProvider;
- * @author markroth8
+import androidx.camera.view.PreviewView;
- */
+import androidx.core.content.ContextCompat;
-public class CameraView
+
-    extends ScrollView
+import com.google.common.util.concurrent.ListenableFuture;
-{
+
-    private static final String LOG_TAG = "ScratchJr.CameraView";
+@SuppressLint("ViewConstructor")
+public class CameraView extends RelativeLayout {
+    private static final String LOG_TAG = "ScratchJr.CameraxView";
 
-    private CameraPreviewView _cameraPreview;
-    private Camera _camera;
     private final RectF _rect;
     private boolean _currentFacingFront;
-    private int _cameraId;
+    private final float _scale;
-    private CameraOrientationListener _orientationListener;
-    private float _scale;
 
-    public CameraView(Context context, RectF rect, float scale, boolean facingFront) {
+    private final AppCompatActivity _activity;
+    private ProcessCameraProvider _cameraProvider;
+    private final PreviewView _viewFinder;
+    private Preview _preview;
+    private ImageCapture _imageCapture;
+    private final ExecutorService _cameraExecutor;
+    private final DisplayManager _displayManager;
+    private int _displayId;
+
+    public CameraView(AppCompatActivity context, RectF rect, float scale, boolean facingFront) {
         super(context);
+        _activity = context;
         _currentFacingFront = facingFront;
         _rect = rect;
         _scale = scale;
 
-        _camera = safeOpenCamera(facingFront);
+        _viewFinder = new PreviewView(context);
-        if (_camera != null) {
-            _cameraPreview = new CameraPreviewView(context);
-            Size previewSize = _camera.getParameters().getPreviewSize();
 
-            float previewWidth = _rect.width();
+        _displayManager = (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE);
-            float previewHeight = previewSize.height * rect.width() / previewSize.width;
-            float centerScrollY = (previewHeight - rect.height()) / 2; 
-            float centerScrollX = 0.0f;
-            if (previewHeight < rect.height()) {
-                previewHeight = rect.height();
-                previewWidth = previewSize.width * rect.height() / previewSize.height;
-                centerScrollX = (previewWidth - rect.width()) / 2; 
-                centerScrollY = 0.0f;
-            }
-            LinearLayout linearLayout = new LinearLayout(context);
-            ViewGroup.LayoutParams layoutParams = new ViewGroup.LayoutParams((int) previewWidth, (int) previewHeight);
-            addView(linearLayout, layoutParams);
 
-            linearLayout.addView(_cameraPreview, layoutParams);
+        RelativeLayout.LayoutParams layoutParams = new RelativeLayout.LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT);
-            final float cx = centerScrollX;
+        addView(_viewFinder, layoutParams);
-            final float cy = centerScrollY;
+        post(() -> {
-            post(new Runnable() {
+            _displayId = _viewFinder.getDisplay().getDisplayId();
-                @Override
+            setupCamera();
-                public void run() {
-                    scrollTo((int) cx, (int) cy);
-                }
         });
-        }
 
-        _orientationListener = new CameraOrientationListener(context, SensorManager.SENSOR_DELAY_NORMAL);
+        _cameraExecutor = Executors.newSingleThreadExecutor();
-        enableOrientationListener();
     }
 
     @Override
-    public boolean onTouchEvent(MotionEvent ev) {
+    protected void onAttachedToWindow() {
-        // Disabling scrolling in this ScrollView
+        super.onAttachedToWindow();
-        return false;
+        _displayManager.registerDisplayListener(displayListener, null);
     }
 
-    public void captureStillImage(PictureCallback pictureCallback, Runnable failed) {
+    @Override
-        if (_camera != null) {
+    protected void onDetachedFromWindow() {
-            final Parameters params = _camera.getParameters();
+        super.onDetachedFromWindow();
-            params.setRotation(0);
+        _displayManager.unregisterDisplayListener(displayListener);
-
-            // Set picture size to the maximum supported resolution.
-            List<Size> supportedPictureSizes = params.getSupportedPictureSizes();
-            int maxHeight = 0;
-            for (Size size : supportedPictureSizes) {
-                if (size.height > maxHeight) {
-                    params.setPictureSize(size.width, size.height);
-                    maxHeight = size.height;
-                }
     }
 
-            _camera.setParameters(params);
+    private final DisplayManager.DisplayListener displayListener = new DisplayManager.DisplayListener() {
-            _camera.takePicture(null, null, pictureCallback);
+        @Override
-        } else {
+        public void onDisplayAdded(int displayId) {
-            failed.run();
-        }
         }
 
-    private Camera safeOpenCamera(boolean facingFront) {
+        @Override
-        Camera result = null;
+        public void onDisplayRemoved(int displayId) {
+        }
+
+        // The androidx.camera.core.Preview.setTargetRotation declaration is opt-in
+        // and its usage should be marked with @androidx.camera.core.ExperimentalUseCaseGroup
+        @ExperimentalUseCaseGroup
+        @Override
+        public void onDisplayChanged(int displayId) {
+            if (displayId == _displayId) {
+                int rotation = _viewFinder.getDisplay().getRotation();
+                if (_imageCapture != null) {
+                    _imageCapture.setTargetRotation(rotation);
+                }
+                if (_preview != null) {
+                    _preview.setTargetRotation(rotation);
+                }
+            }
+        }
+    };
+
+    private void setupCamera() {
+        ListenableFuture<ProcessCameraProvider> cameraProviderFuture = ProcessCameraProvider.getInstance(_activity);
+        cameraProviderFuture.addListener(() -> {
             try {
-            _cameraId = findFirstCameraId(facingFront);
+                _cameraProvider = cameraProviderFuture.get();
-            if (_cameraId != -1) {
+
-                result = Camera.open(_cameraId);
+                bindCameraUseCases();
+            } catch (ExecutionException | InterruptedException e) {
+                e.printStackTrace();
             }
-        } catch (RuntimeException e) {
+        }, ContextCompat.getMainExecutor(_activity));
-            Log.e(LOG_TAG, "Failed to open camera", e);
-        }
-        return result;
     }
 
-    private int findFirstCameraId(boolean facingFront) {
+    private void bindCameraUseCases() {
-        int result = -1;
+        int rotation = _viewFinder.getDisplay().getRotation();
-        int facingTarget = facingFront ? Camera.CameraInfo.CAMERA_FACING_FRONT : Camera.CameraInfo.CAMERA_FACING_BACK;
+        Log.d(LOG_TAG, "rotation: " + rotation);
-        int count = Camera.getNumberOfCameras();
+        _imageCapture = new ImageCapture.Builder()
-        CameraInfo cameraInfo = new CameraInfo();
+            .setTargetRotation(rotation)
-        for (int i = 0; i < count; i++) {
+            .build();
-            Camera.getCameraInfo(i, cameraInfo);
+
-            if (cameraInfo.facing == facingTarget) {
+        _preview = new Preview.Builder()
-                result = i;
+            .setTargetRotation(rotation)
-                break;
+            .build();
+
+        _cameraProvider.unbindAll();
+
+        int lensFacing = CameraSelector.LENS_FACING_FRONT;
+        if (!_currentFacingFront) {
+            lensFacing = CameraSelector.LENS_FACING_BACK;
         }
+
+        CameraSelector selector = new CameraSelector.Builder()
+            .requireLensFacing(lensFacing)
+            .build();
+
+        _cameraProvider.bindToLifecycle(
+            _activity,
+            selector,
+            _preview,
+            _imageCapture
+        );
+
+        _preview.setSurfaceProvider(_viewFinder.getSurfaceProvider());
     }
-        if (result == -1) {
+
-            Log.w(LOG_TAG, "No " + (facingFront ? "front" : "back") + " -facing camera detected on this device.");
+    public void captureStillImage(ImageCapture.OnImageCapturedCallback callback) {
-        }
+        _imageCapture.takePicture(_cameraExecutor, callback);
-        return result;
     }
 
     public boolean setCameraFacing(boolean facingFront) {
-        boolean result;
         if (_currentFacingFront != facingFront) {
-            // switch cameras
+            if (facingFront && !hasFrontCamera()) {
-            int id = findFirstCameraId(facingFront);
+                return false;
-            if (id == -1) {
+            }
-                result = false;
+            if (!facingFront && !hasBackCamera()) {
-            } else {
+                return false;
-                result = true;
+            }
             _currentFacingFront = facingFront;
-                if (_camera != null) {
+            post(this::bindCameraUseCases);
-                    disableOrientationListener();
-                    _camera.release();
-                    _camera = null;
         }
-                _camera = safeOpenCamera(facingFront);
+        return true;
-                _cameraPreview.startPreview();
-                enableOrientationListener();
-            }
-        } else {
-            result = true;
-        }
-        return result;
     }
 
-    public RectF getRect() {
+    private boolean hasBackCamera() {
-        return _rect;
+        try {
+            return _cameraProvider.hasCamera(CameraSelector.DEFAULT_BACK_CAMERA);
+        } catch (CameraInfoUnavailableException e) {
+            e.printStackTrace();
+        }
+        return false;
+    }
+
+    private boolean hasFrontCamera() {
+        try {
+            return _cameraProvider.hasCamera(CameraSelector.DEFAULT_FRONT_CAMERA);
+        } catch (CameraInfoUnavailableException e) {
+            e.printStackTrace();
+        }
+        return false;
     }
 
     /**
@@ -193,13 +209,12 @@ public class CameraView
             // will not happen - this is a ByteArrayOutputStream
             Log.e(LOG_TAG, "IOException while closing byte array stream", e);
         }
-        byte[] jpegData = bos.toByteArray();
+        return bos.toByteArray();
-        return jpegData;
     }
 
     /**
      * Crop and resize the given image to the dimensions of the rectangle for this camera view.
-     *
+     * <p>
      * If the image was front-facing, also mirror horizontally.
      */
     private Bitmap cropResizeAndRotate(Bitmap image, int exifRotation) {
@@ -226,167 +241,63 @@ public class CameraView
             // flip bitmap horizontally since front-facing camera is mirrored
             m.preScale(-1.0f, 1.0f);
         }
-        CameraInfo cameraInfo = new CameraInfo();
+        int rotation = CameraView.findDisplayRotation(getContext(), _currentFacingFront);
-        Camera.getCameraInfo(_cameraId, cameraInfo);
-        int rotation = findDisplayRotation(getContext(), cameraInfo.facing);
         if (rotation == 180) {
             m.preScale(-1.0f, -1.0f);
         }
         m.postScale(scale / _scale, scale / _scale);
-        Bitmap newBitmap = Bitmap.createBitmap(image, offsetX, offsetY, imageWidth - offsetX * 2, imageHeight - offsetY * 2, m, true);
+        return Bitmap.createBitmap(image, offsetX, offsetY, imageWidth - offsetX * 2, imageHeight - offsetY * 2, m, true);
-        return newBitmap;
     }
 
-    private void enableOrientationListener() {
+    // Image → JPEG
-        synchronized(_orientationListener) {
+    public byte[] imageToByteArray(Image image) {
-            if (_orientationListener.canDetectOrientation()) {
+        byte[] data = null;
-                _orientationListener.setCameraInfo(_camera, _cameraId);
+        if (image.getFormat() == ImageFormat.JPEG) {
-                _orientationListener.enable();
+            Image.Plane[] planes = image.getPlanes();
-            }
+            ByteBuffer buffer = planes[0].getBuffer();
+            data = new byte[buffer.capacity()];
+            buffer.get(data);
+            return data;
+        } else if (image.getFormat() == ImageFormat.YUV_420_888) {
+            data = NV21toJPEG(YUV_420_888toNV21(image),
+                image.getWidth(), image.getHeight());
         }
+        return data;
     }
 
-    private void disableOrientationListener() {
+    // YUV_420_888 → NV21
-        synchronized(_orientationListener) {
+    private byte[] YUV_420_888toNV21(Image image) {
-            if (_orientationListener != null) {
+        byte[] nv21;
-                _orientationListener.disable();
+        ByteBuffer yBuffer = image.getPlanes()[0].getBuffer();
-                _orientationListener.clearCameraInfo();
+        ByteBuffer uBuffer = image.getPlanes()[1].getBuffer();
-            }
+        ByteBuffer vBuffer = image.getPlanes()[2].getBuffer();
-        }
+        int ySize = yBuffer.remaining();
+        int uSize = uBuffer.remaining();
+        int vSize = vBuffer.remaining();
+        nv21 = new byte[ySize + uSize + vSize];
+        yBuffer.get(nv21, 0, ySize);
+        vBuffer.get(nv21, ySize, vSize);
+        uBuffer.get(nv21, ySize + vSize, uSize);
+        return nv21;
     }
 
-    private static Display findDisplay(Context context) {
+    // NV21 → JPEG
+    private byte[] NV21toJPEG(byte[] nv21, int width, int height) {
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        YuvImage yuv = new YuvImage(nv21, ImageFormat.NV21, width, height, null);
+        yuv.compressToJpeg(new Rect(0, 0, width, height), 100, out);
+        return out.toByteArray();
+    }
+
+    private static int findDisplayRotation(Context context, boolean facingFront) {
         WindowManager windowManager = (WindowManager) context.getSystemService(Context.WINDOW_SERVICE);
-        return windowManager.getDefaultDisplay();
+        Display display = windowManager.getDefaultDisplay();
-    }
-    
-    private static int findDisplayRotation(Context context, int facing) {
-        Display display = findDisplay(context);
         int r = display.getRotation();
         int rotation = (r == Surface.ROTATION_180) ? 180 : 0;
-        if (facing == CameraInfo.CAMERA_FACING_FRONT) {
+        if (facingFront) {
             rotation = (rotation + 360) % 360;
         }
         return rotation;
     }
 
-    private class CameraPreviewView
-        extends SurfaceView
-        implements SurfaceHolder.Callback
-    {
-        private final SurfaceHolder _holder;
-
-        public CameraPreviewView(Context context) {
-            super(context);
-            _holder = getHolder();
-            _holder.addCallback(CameraPreviewView.this);
-        }
-
-        @Override
-        public void surfaceCreated(SurfaceHolder holder) {
-            try {
-                if (_camera != null) {
-                    _camera.setPreviewDisplay(holder);
-                    _camera.startPreview();
-                }
-            } catch (IOException e) {
-                Log.e(LOG_TAG, "Error creating surface", e);
-            }
-        }
-
-        @Override
-        public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
-            if (_holder.getSurface() != null) {
-                if (_camera != null) {
-                    try {
-                        _camera.stopPreview();
-                    } catch (Exception e) {
-                        Log.e(LOG_TAG, "Error releasing camera", e);
-                    }
-                    
-                    startPreview();
-                }
-            }
-        }
-
-        @Override
-        public void surfaceDestroyed(SurfaceHolder holder) {
-            if (_camera != null) {
-                Log.i(LOG_TAG, "Releasing camera");
-                disableOrientationListener();
-                _camera.release();
-                _camera = null;
-            }
-        }
-
-        public void startPreview() {
-            if (_camera != null) {
-                try {
-                    _camera.setPreviewDisplay(_holder);
-                    Size previewSize = _camera.getParameters().getPreviewSize();
-                    Log.i(LOG_TAG, "Preview size: " + previewSize.width + " x " + previewSize.height);
-                    _camera.startPreview();
-                } catch (IOException e) {
-                    Log.e(LOG_TAG, "Error in starting preview", e);
-                } catch (RuntimeException e) {
-                    Log.e(LOG_TAG, "Error in starting preview", e);
-                }
-            }
-        }
-    }
-    
-    /**
-     * An {@link OrientationEventListener} which updates the camera preview
-     * based on the device's orientation.
-     * @author khu
-     *
-     */
-    private static class CameraOrientationListener extends OrientationEventListener {
-        private Camera _observedCamera;
-        private int _observedCameraId;
-        private Display _display;
-        private int _previousRotation = -1;
-        private Context _context;
-        
-        public CameraOrientationListener(Context context) {
-            super(context);
-            _context = context;
-            _display = findDisplay(context);
-        }
-
-        public CameraOrientationListener(Context context, int rate) {
-            super(context, rate);
-            _context = context;
-            _display = findDisplay(context);
-        }
-        
-        public synchronized void setCameraInfo(Camera camera, int cameraId) {
-            _observedCamera = camera;
-            _observedCameraId = cameraId;
-        }
-        
-        public synchronized void clearCameraInfo() {
-            _observedCamera = null;
-            _observedCameraId = -1;
-        }
-
-        @Override
-        public synchronized void onOrientationChanged(int orientation) {
-            if (orientation == ORIENTATION_UNKNOWN || _observedCamera == null 
-                    || _observedCameraId == -1) 
-            {
-                return;
-            }
-            
-            CameraInfo cameraInfo = new CameraInfo();
-            Camera.getCameraInfo(_observedCameraId, cameraInfo);
-            int rotation = findDisplayRotation(_context, cameraInfo.facing);
-            
-            if (rotation != _previousRotation) {
-                // Update the preview
-                _observedCamera.setDisplayOrientation(rotation);
-                _previousRotation = rotation;
-            }
-        }
-    }
 }

android/ScratchJr/app/src/main/java/org/scratchjr/android/JavaScriptDirectInterface.java

@@ -13,6 +13,7 @@ import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
 
+import android.annotation.SuppressLint;
 import android.content.Context;
 import android.content.Intent;
 import android.content.pm.PackageManager;
@@ -20,6 +21,7 @@ import android.graphics.Bitmap;
 import android.graphics.BitmapFactory;
 import android.graphics.RectF;
 import android.hardware.Camera;
+import android.media.Image;
 import android.net.Uri;
 import android.text.Html;
 import android.util.Base64;
@@ -29,6 +31,11 @@ import android.webkit.JavascriptInterface;
 import android.widget.ImageView;
 import android.widget.RelativeLayout;
 
+import androidx.annotation.NonNull;
+import androidx.camera.core.ImageCapture;
+import androidx.camera.core.ImageCaptureException;
+import androidx.camera.core.ImageProxy;
+
 /**
  * The methods in this inner class are exposed directly to JavaScript in the HTML5 pages
  * as AndroidInterface.
@@ -410,19 +417,24 @@ public class JavaScriptDirectInterface {
 
     @JavascriptInterface
     public void scratchjr_captureimage(final String onCameraCaptureComplete) {
-        _cameraView.captureStillImage(
+        _cameraView.captureStillImage(new ImageCapture.OnImageCapturedCallback() {
-            new Camera.PictureCallback() {
+            @Override
-                public void onPictureTaken(byte[] jpegData, Camera camera) {
+            public void onCaptureSuccess(@NonNull ImageProxy imageProxy) {
+                @SuppressLint("UnsafeOptInUsageError")
+                Image image = imageProxy.getImage();
+                if (image != null) {
+                    byte[] jpegData = _cameraView.imageToByteArray(image);
                     sendBase64Image(onCameraCaptureComplete, jpegData);
                 }
-            },
+                imageProxy.close();
-            new Runnable() {
+            }
-                public void run() {
+
+            @Override
+            public void onError(@NonNull ImageCaptureException exception) {
                 Log.e(LOG_TAG, "Could not capture picture");
                 reportImageError(onCameraCaptureComplete);
             }
-            }
+        });
-        );
     }
 
     @JavascriptInterface

android/ScratchJr/app/src/main/java/org/scratchjr/android/ScratchJrActivity.java

@@ -15,6 +15,7 @@ import android.os.Build;
 import android.os.Bundle;
 import android.os.Handler;
 import androidx.annotation.NonNull;
+import androidx.appcompat.app.AppCompatActivity;
 import androidx.core.app.ActivityCompat;
 import androidx.core.content.ContextCompat;
 import android.util.Log;
@@ -49,7 +50,7 @@ import java.util.Vector;
  * @author markroth8
  */
 public class ScratchJrActivity
-    extends Activity
+    extends AppCompatActivity
 {
     /** Milliseconds to pan when showing the soft keyboard */
     private static final int SOFT_KEYBOARD_PAN_MS = 250;

android/ScratchJr/app/src/main/res/values/styles.xml

@@ -5,7 +5,7 @@
         <!-- Customize your theme here. -->
     </style>
 
-    <style name="FullscreenTheme" parent="android:Theme.NoTitleBar">
+    <style name="FullscreenTheme" parent="Theme.AppCompat.Light.NoActionBar">
         <item name="android:windowContentOverlay">@null</item>
         <item name="android:windowBackground">@null</item>
         <item name="metaButtonBarStyle">@style/ButtonBar</item>