diff --git a/android/ScratchJr/app/build.gradle b/android/ScratchJr/app/build.gradle index c0c064e..afb92d7 100644 --- a/android/ScratchJr/app/build.gradle +++ b/android/ScratchJr/app/build.gradle @@ -26,6 +26,14 @@ android { versionName "1.2.11" } } + + // The camera-view dependency contains Java 8 bytecode, + // we need to support java 8 to dex. + // See https://developer.android.com/studio/write/java8-support.html for details. + compileOptions { + sourceCompatibility JavaVersion.VERSION_1_8 + targetCompatibility JavaVersion.VERSION_1_8 + } } dependencies { @@ -33,7 +41,15 @@ dependencies { implementation 'com.google.firebase:firebase-core:17.2.0' implementation 'com.google.firebase:firebase-analytics:17.2.0' implementation 'com.google.android.gms:play-services-location:17.0.0' - implementation 'androidx.appcompat:appcompat:1.1.0' + + implementation 'androidx.appcompat:appcompat:1.3.0' + def camerax_version = "1.0.0-rc05" + // CameraX core library using camera2 implementation + implementation "androidx.camera:camera-camera2:$camerax_version" + // CameraX Lifecycle Library + implementation "androidx.camera:camera-lifecycle:$camerax_version" + // CameraX View class + implementation "androidx.camera:camera-view:1.0.0-alpha25" } def appModuleRootFolder = '.' diff --git a/android/ScratchJr/app/src/main/java/org/scratchjr/android/CameraView.java b/android/ScratchJr/app/src/main/java/org/scratchjr/android/CameraView.java index 050c718..eb8cb96 100644 --- a/android/ScratchJr/app/src/main/java/org/scratchjr/android/CameraView.java +++ b/android/ScratchJr/app/src/main/java/org/scratchjr/android/CameraView.java @@ -2,179 +2,195 @@ package org.scratchjr.android; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.util.List; +import java.nio.ByteBuffer; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import android.annotation.SuppressLint; import android.content.Context; import android.graphics.Bitmap; import android.graphics.Bitmap.CompressFormat; +import android.graphics.ImageFormat; import android.graphics.Matrix; +import android.graphics.Rect; import android.graphics.RectF; -import android.hardware.Camera; -import android.hardware.Camera.CameraInfo; -import android.hardware.Camera.Parameters; -import android.hardware.Camera.PictureCallback; -import android.hardware.Camera.Size; -import android.hardware.SensorManager; +import android.graphics.YuvImage; +import android.hardware.display.DisplayManager; +import android.media.Image; import android.util.Log; import android.view.Display; -import android.view.MotionEvent; -import android.view.OrientationEventListener; import android.view.Surface; -import android.view.SurfaceHolder; -import android.view.SurfaceView; -import android.view.ViewGroup; import android.view.WindowManager; -import android.widget.LinearLayout; -import android.widget.ScrollView; +import android.widget.RelativeLayout; + +import androidx.appcompat.app.AppCompatActivity; +import androidx.camera.core.CameraInfoUnavailableException; +import androidx.camera.core.CameraSelector; +import androidx.camera.core.ExperimentalUseCaseGroup; +import androidx.camera.core.ImageCapture; +import androidx.camera.core.Preview; +import androidx.camera.lifecycle.ProcessCameraProvider; +import androidx.camera.view.PreviewView; +import androidx.core.content.ContextCompat; + +import com.google.common.util.concurrent.ListenableFuture; + +@SuppressLint("ViewConstructor") +public class CameraView extends RelativeLayout { + private static final String LOG_TAG = "ScratchJr.CameraxView"; -/** - * Creates a camera view that hovers at a particular location and has a mask. - * - * We use a ScrollView because the camera will rescale (squish) the preview to whatever size the - * SurfaceView is and we want to keep the aspect ratio. So the ScrollView is of the desired - * size and then we add a SurfaceView to it with the camera preview. - * - * @author markroth8 - */ -public class CameraView - extends ScrollView -{ - private static final String LOG_TAG = "ScratchJr.CameraView"; - - private CameraPreviewView _cameraPreview; - private Camera _camera; private final RectF _rect; private boolean _currentFacingFront; - private int _cameraId; - private CameraOrientationListener _orientationListener; - private float _scale; + private final float _scale; - public CameraView(Context context, RectF rect, float scale, boolean facingFront) { + private final AppCompatActivity _activity; + private ProcessCameraProvider _cameraProvider; + private final PreviewView _viewFinder; + private Preview _preview; + private ImageCapture _imageCapture; + private final ExecutorService _cameraExecutor; + private final DisplayManager _displayManager; + private int _displayId; + + public CameraView(AppCompatActivity context, RectF rect, float scale, boolean facingFront) { super(context); + _activity = context; _currentFacingFront = facingFront; _rect = rect; _scale = scale; - - _camera = safeOpenCamera(facingFront); - if (_camera != null) { - _cameraPreview = new CameraPreviewView(context); - Size previewSize = _camera.getParameters().getPreviewSize(); - - float previewWidth = _rect.width(); - float previewHeight = previewSize.height * rect.width() / previewSize.width; - float centerScrollY = (previewHeight - rect.height()) / 2; - float centerScrollX = 0.0f; - if (previewHeight < rect.height()) { - previewHeight = rect.height(); - previewWidth = previewSize.width * rect.height() / previewSize.height; - centerScrollX = (previewWidth - rect.width()) / 2; - centerScrollY = 0.0f; - } - LinearLayout linearLayout = new LinearLayout(context); - ViewGroup.LayoutParams layoutParams = new ViewGroup.LayoutParams((int) previewWidth, (int) previewHeight); - addView(linearLayout, layoutParams); - - linearLayout.addView(_cameraPreview, layoutParams); - final float cx = centerScrollX; - final float cy = centerScrollY; - post(new Runnable() { - @Override - public void run() { - scrollTo((int) cx, (int) cy); - } - }); - } - - _orientationListener = new CameraOrientationListener(context, SensorManager.SENSOR_DELAY_NORMAL); - enableOrientationListener(); + + _viewFinder = new PreviewView(context); + + _displayManager = (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE); + + RelativeLayout.LayoutParams layoutParams = new RelativeLayout.LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT); + addView(_viewFinder, layoutParams); + post(() -> { + _displayId = _viewFinder.getDisplay().getDisplayId(); + setupCamera(); + }); + + _cameraExecutor = Executors.newSingleThreadExecutor(); } - + @Override - public boolean onTouchEvent(MotionEvent ev) { - // Disabling scrolling in this ScrollView - return false; + protected void onAttachedToWindow() { + super.onAttachedToWindow(); + _displayManager.registerDisplayListener(displayListener, null); } - - public void captureStillImage(PictureCallback pictureCallback, Runnable failed) { - if (_camera != null) { - final Parameters params = _camera.getParameters(); - params.setRotation(0); - // Set picture size to the maximum supported resolution. - List supportedPictureSizes = params.getSupportedPictureSizes(); - int maxHeight = 0; - for (Size size : supportedPictureSizes) { - if (size.height > maxHeight) { - params.setPictureSize(size.width, size.height); - maxHeight = size.height; + @Override + protected void onDetachedFromWindow() { + super.onDetachedFromWindow(); + _displayManager.unregisterDisplayListener(displayListener); + } + + private final DisplayManager.DisplayListener displayListener = new DisplayManager.DisplayListener() { + @Override + public void onDisplayAdded(int displayId) { + } + + @Override + public void onDisplayRemoved(int displayId) { + } + + // The androidx.camera.core.Preview.setTargetRotation declaration is opt-in + // and its usage should be marked with @androidx.camera.core.ExperimentalUseCaseGroup + @ExperimentalUseCaseGroup + @Override + public void onDisplayChanged(int displayId) { + if (displayId == _displayId) { + int rotation = _viewFinder.getDisplay().getRotation(); + if (_imageCapture != null) { + _imageCapture.setTargetRotation(rotation); + } + if (_preview != null) { + _preview.setTargetRotation(rotation); } } - - _camera.setParameters(params); - _camera.takePicture(null, null, pictureCallback); - } else { - failed.run(); } + }; + + private void setupCamera() { + ListenableFuture cameraProviderFuture = ProcessCameraProvider.getInstance(_activity); + cameraProviderFuture.addListener(() -> { + try { + _cameraProvider = cameraProviderFuture.get(); + + bindCameraUseCases(); + } catch (ExecutionException | InterruptedException e) { + e.printStackTrace(); + } + }, ContextCompat.getMainExecutor(_activity)); } - private Camera safeOpenCamera(boolean facingFront) { - Camera result = null; - try { - _cameraId = findFirstCameraId(facingFront); - if (_cameraId != -1) { - result = Camera.open(_cameraId); - } - } catch (RuntimeException e) { - Log.e(LOG_TAG, "Failed to open camera", e); + private void bindCameraUseCases() { + int rotation = _viewFinder.getDisplay().getRotation(); + Log.d(LOG_TAG, "rotation: " + rotation); + _imageCapture = new ImageCapture.Builder() + .setTargetRotation(rotation) + .build(); + + _preview = new Preview.Builder() + .setTargetRotation(rotation) + .build(); + + _cameraProvider.unbindAll(); + + int lensFacing = CameraSelector.LENS_FACING_FRONT; + if (!_currentFacingFront) { + lensFacing = CameraSelector.LENS_FACING_BACK; } - return result; + + CameraSelector selector = new CameraSelector.Builder() + .requireLensFacing(lensFacing) + .build(); + + _cameraProvider.bindToLifecycle( + _activity, + selector, + _preview, + _imageCapture + ); + + _preview.setSurfaceProvider(_viewFinder.getSurfaceProvider()); } - private int findFirstCameraId(boolean facingFront) { - int result = -1; - int facingTarget = facingFront ? Camera.CameraInfo.CAMERA_FACING_FRONT : Camera.CameraInfo.CAMERA_FACING_BACK; - int count = Camera.getNumberOfCameras(); - CameraInfo cameraInfo = new CameraInfo(); - for (int i = 0; i < count; i++) { - Camera.getCameraInfo(i, cameraInfo); - if (cameraInfo.facing == facingTarget) { - result = i; - break; - } - } - if (result == -1) { - Log.w(LOG_TAG, "No " + (facingFront ? "front" : "back") + " -facing camera detected on this device."); - } - return result; + public void captureStillImage(ImageCapture.OnImageCapturedCallback callback) { + _imageCapture.takePicture(_cameraExecutor, callback); } public boolean setCameraFacing(boolean facingFront) { - boolean result; if (_currentFacingFront != facingFront) { - // switch cameras - int id = findFirstCameraId(facingFront); - if (id == -1) { - result = false; - } else { - result = true; - _currentFacingFront = facingFront; - if (_camera != null) { - disableOrientationListener(); - _camera.release(); - _camera = null; - } - _camera = safeOpenCamera(facingFront); - _cameraPreview.startPreview(); - enableOrientationListener(); + if (facingFront && !hasFrontCamera()) { + return false; } - } else { - result = true; + if (!facingFront && !hasBackCamera()) { + return false; + } + _currentFacingFront = facingFront; + post(this::bindCameraUseCases); } - return result; + return true; } - - public RectF getRect() { - return _rect; + + private boolean hasBackCamera() { + try { + return _cameraProvider.hasCamera(CameraSelector.DEFAULT_BACK_CAMERA); + } catch (CameraInfoUnavailableException e) { + e.printStackTrace(); + } + return false; + } + + private boolean hasFrontCamera() { + try { + return _cameraProvider.hasCamera(CameraSelector.DEFAULT_FRONT_CAMERA); + } catch (CameraInfoUnavailableException e) { + e.printStackTrace(); + } + return false; } /** @@ -193,13 +209,12 @@ public class CameraView // will not happen - this is a ByteArrayOutputStream Log.e(LOG_TAG, "IOException while closing byte array stream", e); } - byte[] jpegData = bos.toByteArray(); - return jpegData; + return bos.toByteArray(); } - + /** * Crop and resize the given image to the dimensions of the rectangle for this camera view. - * + *

* If the image was front-facing, also mirror horizontally. */ private Bitmap cropResizeAndRotate(Bitmap image, int exifRotation) { @@ -207,7 +222,7 @@ public class CameraView int imageHeight = image.getHeight(); float rectWidth = _rect.width(); float rectHeight = _rect.height(); - + float newHeight = rectWidth * imageHeight / imageWidth; float scale = rectWidth / imageWidth; int offsetX = 0; @@ -218,7 +233,7 @@ public class CameraView offsetY = 0; offsetX = (int) ((newWidth - rectWidth) / 2 * imageWidth / newWidth); } - + Matrix m = new Matrix(); // Adjust the image to undo rotation done by JPEG generator m.postRotate(-1.0f * exifRotation); @@ -226,167 +241,63 @@ public class CameraView // flip bitmap horizontally since front-facing camera is mirrored m.preScale(-1.0f, 1.0f); } - CameraInfo cameraInfo = new CameraInfo(); - Camera.getCameraInfo(_cameraId, cameraInfo); - int rotation = findDisplayRotation(getContext(), cameraInfo.facing); + int rotation = CameraView.findDisplayRotation(getContext(), _currentFacingFront); if (rotation == 180) { m.preScale(-1.0f, -1.0f); } m.postScale(scale / _scale, scale / _scale); - Bitmap newBitmap = Bitmap.createBitmap(image, offsetX, offsetY, imageWidth - offsetX * 2, imageHeight - offsetY * 2, m, true); - return newBitmap; + return Bitmap.createBitmap(image, offsetX, offsetY, imageWidth - offsetX * 2, imageHeight - offsetY * 2, m, true); } - private void enableOrientationListener() { - synchronized(_orientationListener) { - if (_orientationListener.canDetectOrientation()) { - _orientationListener.setCameraInfo(_camera, _cameraId); - _orientationListener.enable(); - } + // Image → JPEG + public byte[] imageToByteArray(Image image) { + byte[] data = null; + if (image.getFormat() == ImageFormat.JPEG) { + Image.Plane[] planes = image.getPlanes(); + ByteBuffer buffer = planes[0].getBuffer(); + data = new byte[buffer.capacity()]; + buffer.get(data); + return data; + } else if (image.getFormat() == ImageFormat.YUV_420_888) { + data = NV21toJPEG(YUV_420_888toNV21(image), + image.getWidth(), image.getHeight()); } + return data; } - - private void disableOrientationListener() { - synchronized(_orientationListener) { - if (_orientationListener != null) { - _orientationListener.disable(); - _orientationListener.clearCameraInfo(); - } - } + + // YUV_420_888 → NV21 + private byte[] YUV_420_888toNV21(Image image) { + byte[] nv21; + ByteBuffer yBuffer = image.getPlanes()[0].getBuffer(); + ByteBuffer uBuffer = image.getPlanes()[1].getBuffer(); + ByteBuffer vBuffer = image.getPlanes()[2].getBuffer(); + int ySize = yBuffer.remaining(); + int uSize = uBuffer.remaining(); + int vSize = vBuffer.remaining(); + nv21 = new byte[ySize + uSize + vSize]; + yBuffer.get(nv21, 0, ySize); + vBuffer.get(nv21, ySize, vSize); + uBuffer.get(nv21, ySize + vSize, uSize); + return nv21; } - - private static Display findDisplay(Context context) { + + // NV21 → JPEG + private byte[] NV21toJPEG(byte[] nv21, int width, int height) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + YuvImage yuv = new YuvImage(nv21, ImageFormat.NV21, width, height, null); + yuv.compressToJpeg(new Rect(0, 0, width, height), 100, out); + return out.toByteArray(); + } + + private static int findDisplayRotation(Context context, boolean facingFront) { WindowManager windowManager = (WindowManager) context.getSystemService(Context.WINDOW_SERVICE); - return windowManager.getDefaultDisplay(); - } - - private static int findDisplayRotation(Context context, int facing) { - Display display = findDisplay(context); + Display display = windowManager.getDefaultDisplay(); int r = display.getRotation(); int rotation = (r == Surface.ROTATION_180) ? 180 : 0; - if (facing == CameraInfo.CAMERA_FACING_FRONT) { + if (facingFront) { rotation = (rotation + 360) % 360; } return rotation; } - private class CameraPreviewView - extends SurfaceView - implements SurfaceHolder.Callback - { - private final SurfaceHolder _holder; - - public CameraPreviewView(Context context) { - super(context); - _holder = getHolder(); - _holder.addCallback(CameraPreviewView.this); - } - - @Override - public void surfaceCreated(SurfaceHolder holder) { - try { - if (_camera != null) { - _camera.setPreviewDisplay(holder); - _camera.startPreview(); - } - } catch (IOException e) { - Log.e(LOG_TAG, "Error creating surface", e); - } - } - - @Override - public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) { - if (_holder.getSurface() != null) { - if (_camera != null) { - try { - _camera.stopPreview(); - } catch (Exception e) { - Log.e(LOG_TAG, "Error releasing camera", e); - } - - startPreview(); - } - } - } - - @Override - public void surfaceDestroyed(SurfaceHolder holder) { - if (_camera != null) { - Log.i(LOG_TAG, "Releasing camera"); - disableOrientationListener(); - _camera.release(); - _camera = null; - } - } - - public void startPreview() { - if (_camera != null) { - try { - _camera.setPreviewDisplay(_holder); - Size previewSize = _camera.getParameters().getPreviewSize(); - Log.i(LOG_TAG, "Preview size: " + previewSize.width + " x " + previewSize.height); - _camera.startPreview(); - } catch (IOException e) { - Log.e(LOG_TAG, "Error in starting preview", e); - } catch (RuntimeException e) { - Log.e(LOG_TAG, "Error in starting preview", e); - } - } - } - } - - /** - * An {@link OrientationEventListener} which updates the camera preview - * based on the device's orientation. - * @author khu - * - */ - private static class CameraOrientationListener extends OrientationEventListener { - private Camera _observedCamera; - private int _observedCameraId; - private Display _display; - private int _previousRotation = -1; - private Context _context; - - public CameraOrientationListener(Context context) { - super(context); - _context = context; - _display = findDisplay(context); - } - - public CameraOrientationListener(Context context, int rate) { - super(context, rate); - _context = context; - _display = findDisplay(context); - } - - public synchronized void setCameraInfo(Camera camera, int cameraId) { - _observedCamera = camera; - _observedCameraId = cameraId; - } - - public synchronized void clearCameraInfo() { - _observedCamera = null; - _observedCameraId = -1; - } - - @Override - public synchronized void onOrientationChanged(int orientation) { - if (orientation == ORIENTATION_UNKNOWN || _observedCamera == null - || _observedCameraId == -1) - { - return; - } - - CameraInfo cameraInfo = new CameraInfo(); - Camera.getCameraInfo(_observedCameraId, cameraInfo); - int rotation = findDisplayRotation(_context, cameraInfo.facing); - - if (rotation != _previousRotation) { - // Update the preview - _observedCamera.setDisplayOrientation(rotation); - _previousRotation = rotation; - } - } - } } diff --git a/android/ScratchJr/app/src/main/java/org/scratchjr/android/JavaScriptDirectInterface.java b/android/ScratchJr/app/src/main/java/org/scratchjr/android/JavaScriptDirectInterface.java index f1ec612..188dccb 100644 --- a/android/ScratchJr/app/src/main/java/org/scratchjr/android/JavaScriptDirectInterface.java +++ b/android/ScratchJr/app/src/main/java/org/scratchjr/android/JavaScriptDirectInterface.java @@ -13,6 +13,7 @@ import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; +import android.annotation.SuppressLint; import android.content.Context; import android.content.Intent; import android.content.pm.PackageManager; @@ -20,6 +21,7 @@ import android.graphics.Bitmap; import android.graphics.BitmapFactory; import android.graphics.RectF; import android.hardware.Camera; +import android.media.Image; import android.net.Uri; import android.text.Html; import android.util.Base64; @@ -29,6 +31,11 @@ import android.webkit.JavascriptInterface; import android.widget.ImageView; import android.widget.RelativeLayout; +import androidx.annotation.NonNull; +import androidx.camera.core.ImageCapture; +import androidx.camera.core.ImageCaptureException; +import androidx.camera.core.ImageProxy; + /** * The methods in this inner class are exposed directly to JavaScript in the HTML5 pages * as AndroidInterface. @@ -410,19 +417,24 @@ public class JavaScriptDirectInterface { @JavascriptInterface public void scratchjr_captureimage(final String onCameraCaptureComplete) { - _cameraView.captureStillImage( - new Camera.PictureCallback() { - public void onPictureTaken(byte[] jpegData, Camera camera) { + _cameraView.captureStillImage(new ImageCapture.OnImageCapturedCallback() { + @Override + public void onCaptureSuccess(@NonNull ImageProxy imageProxy) { + @SuppressLint("UnsafeOptInUsageError") + Image image = imageProxy.getImage(); + if (image != null) { + byte[] jpegData = _cameraView.imageToByteArray(image); sendBase64Image(onCameraCaptureComplete, jpegData); } - }, - new Runnable() { - public void run() { - Log.e(LOG_TAG, "Could not capture picture"); - reportImageError(onCameraCaptureComplete); - } + imageProxy.close(); } - ); + + @Override + public void onError(@NonNull ImageCaptureException exception) { + Log.e(LOG_TAG, "Could not capture picture"); + reportImageError(onCameraCaptureComplete); + } + }); } @JavascriptInterface diff --git a/android/ScratchJr/app/src/main/java/org/scratchjr/android/ScratchJrActivity.java b/android/ScratchJr/app/src/main/java/org/scratchjr/android/ScratchJrActivity.java index 5e1e623..d4023eb 100644 --- a/android/ScratchJr/app/src/main/java/org/scratchjr/android/ScratchJrActivity.java +++ b/android/ScratchJr/app/src/main/java/org/scratchjr/android/ScratchJrActivity.java @@ -15,6 +15,7 @@ import android.os.Build; import android.os.Bundle; import android.os.Handler; import androidx.annotation.NonNull; +import androidx.appcompat.app.AppCompatActivity; import androidx.core.app.ActivityCompat; import androidx.core.content.ContextCompat; import android.util.Log; @@ -49,7 +50,7 @@ import java.util.Vector; * @author markroth8 */ public class ScratchJrActivity - extends Activity + extends AppCompatActivity { /** Milliseconds to pan when showing the soft keyboard */ private static final int SOFT_KEYBOARD_PAN_MS = 250; diff --git a/android/ScratchJr/app/src/main/res/values/styles.xml b/android/ScratchJr/app/src/main/res/values/styles.xml index 8d8c40e..03d41e3 100644 --- a/android/ScratchJr/app/src/main/res/values/styles.xml +++ b/android/ScratchJr/app/src/main/res/values/styles.xml @@ -5,7 +5,7 @@ -