scratchfoundation/scratchjr
1
0
Fork 0
mirror of https://github.com/scratchfoundation/scratchjr.git synced 2024-11-28 18:15:37 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #451 from yueyuzhao/issue/27-camera2

Browse source 
Issue/27 camera2
This commit is contained in:
chrisgarrity 2021-08-02 10:33:44 -04:00 committed by GitHub
commit 96ce9b0641
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 241 additions and 301 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
android/ScratchJr/app/build.gradle
View file

@ -26,6 +26,14 @@ android {
versionName "1.2.11"
}
}
// The camera-view dependency contains Java 8 bytecode,
// we need to support java 8 to dex.
// See https://developer.android.com/studio/write/java8-support.html for details.
compileOptions {
sourceCompatibility JavaVersion.VERSION_1_8
targetCompatibility JavaVersion.VERSION_1_8
}
}
dependencies {
@ -33,7 +41,15 @@ dependencies {
implementation 'com.google.firebase:firebase-core:17.2.0'
implementation 'com.google.firebase:firebase-analytics:17.2.0'
implementation 'com.google.android.gms:play-services-location:17.0.0'
implementation 'androidx.appcompat:appcompat:1.1.0'
implementation 'androidx.appcompat:appcompat:1.3.0'
def camerax_version = "1.0.0-rc05"
// CameraX core library using camera2 implementation
implementation "androidx.camera:camera-camera2:$camerax_version"
// CameraX Lifecycle Library
implementation "androidx.camera:camera-lifecycle:$camerax_version"
// CameraX View class
implementation "androidx.camera:camera-view:1.0.0-alpha25"
}
def appModuleRootFolder = '.'

487
android/ScratchJr/app/src/main/java/org/scratchjr/android/CameraView.java
View file

@ -2,179 +2,195 @@ package org.scratchjr.android;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.List;
import java.nio.ByteBuffer;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import android.annotation.SuppressLint;
import android.content.Context;
import android.graphics.Bitmap;
import android.graphics.Bitmap.CompressFormat;
import android.graphics.ImageFormat;
import android.graphics.Matrix;
import android.graphics.Rect;
import android.graphics.RectF;
import android.hardware.Camera;
import android.hardware.Camera.CameraInfo;
import android.hardware.Camera.Parameters;
import android.hardware.Camera.PictureCallback;
import android.hardware.Camera.Size;
import android.hardware.SensorManager;
import android.graphics.YuvImage;
import android.hardware.display.DisplayManager;
import android.media.Image;
import android.util.Log;
import android.view.Display;
import android.view.MotionEvent;
import android.view.OrientationEventListener;
import android.view.Surface;
import android.view.SurfaceHolder;
import android.view.SurfaceView;
import android.view.ViewGroup;
import android.view.WindowManager;
import android.widget.LinearLayout;
import android.widget.ScrollView;
import android.widget.RelativeLayout;
import androidx.appcompat.app.AppCompatActivity;
import androidx.camera.core.CameraInfoUnavailableException;
import androidx.camera.core.CameraSelector;
import androidx.camera.core.ExperimentalUseCaseGroup;
import androidx.camera.core.ImageCapture;
import androidx.camera.core.Preview;
import androidx.camera.lifecycle.ProcessCameraProvider;
import androidx.camera.view.PreviewView;
import androidx.core.content.ContextCompat;
import com.google.common.util.concurrent.ListenableFuture;
@SuppressLint("ViewConstructor")
public class CameraView extends RelativeLayout {
private static final String LOG_TAG = "ScratchJr.CameraxView";
/**
* Creates a camera view that hovers at a particular location and has a mask.
*
* We use a ScrollView because the camera will rescale (squish) the preview to whatever size the
* SurfaceView is and we want to keep the aspect ratio. So the ScrollView is of the desired
* size and then we add a SurfaceView to it with the camera preview.
*
* @author markroth8
*/
public class CameraView
extends ScrollView
{
private static final String LOG_TAG = "ScratchJr.CameraView";
private CameraPreviewView _cameraPreview;
private Camera _camera;
private final RectF _rect;
private boolean _currentFacingFront;
private int _cameraId;
private CameraOrientationListener _orientationListener;
private float _scale;
private final float _scale;
public CameraView(Context context, RectF rect, float scale, boolean facingFront) {
private final AppCompatActivity _activity;
private ProcessCameraProvider _cameraProvider;
private final PreviewView _viewFinder;
private Preview _preview;
private ImageCapture _imageCapture;
private final ExecutorService _cameraExecutor;
private final DisplayManager _displayManager;
private int _displayId;
public CameraView(AppCompatActivity context, RectF rect, float scale, boolean facingFront) {
super(context);
_activity = context;
_currentFacingFront = facingFront;
_rect = rect;
_scale = scale;
_camera = safeOpenCamera(facingFront);
if (_camera != null) {
_cameraPreview = new CameraPreviewView(context);
Size previewSize = _camera.getParameters().getPreviewSize();
float previewWidth = _rect.width();
float previewHeight = previewSize.height * rect.width() / previewSize.width;
float centerScrollY = (previewHeight - rect.height()) / 2;
float centerScrollX = 0.0f;
if (previewHeight < rect.height()) {
previewHeight = rect.height();
previewWidth = previewSize.width * rect.height() / previewSize.height;
centerScrollX = (previewWidth - rect.width()) / 2;
centerScrollY = 0.0f;
}
LinearLayout linearLayout = new LinearLayout(context);
ViewGroup.LayoutParams layoutParams = new ViewGroup.LayoutParams((int) previewWidth, (int) previewHeight);
addView(linearLayout, layoutParams);
linearLayout.addView(_cameraPreview, layoutParams);
final float cx = centerScrollX;
final float cy = centerScrollY;
post(new Runnable() {
@Override
public void run() {
scrollTo((int) cx, (int) cy);
}
});
}
_orientationListener = new CameraOrientationListener(context, SensorManager.SENSOR_DELAY_NORMAL);
enableOrientationListener();
_viewFinder = new PreviewView(context);
_displayManager = (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE);
RelativeLayout.LayoutParams layoutParams = new RelativeLayout.LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT);
addView(_viewFinder, layoutParams);
post(() -> {
_displayId = _viewFinder.getDisplay().getDisplayId();
setupCamera();
});
_cameraExecutor = Executors.newSingleThreadExecutor();
}
@Override
public boolean onTouchEvent(MotionEvent ev) {
// Disabling scrolling in this ScrollView
return false;
protected void onAttachedToWindow() {
super.onAttachedToWindow();
_displayManager.registerDisplayListener(displayListener, null);
}
public void captureStillImage(PictureCallback pictureCallback, Runnable failed) {
if (_camera != null) {
final Parameters params = _camera.getParameters();
params.setRotation(0);
// Set picture size to the maximum supported resolution.
List<Size> supportedPictureSizes = params.getSupportedPictureSizes();
int maxHeight = 0;
for (Size size : supportedPictureSizes) {
if (size.height > maxHeight) {
params.setPictureSize(size.width, size.height);
maxHeight = size.height;
@Override
protected void onDetachedFromWindow() {
super.onDetachedFromWindow();
_displayManager.unregisterDisplayListener(displayListener);
}
private final DisplayManager.DisplayListener displayListener = new DisplayManager.DisplayListener() {
@Override
public void onDisplayAdded(int displayId) {
}
@Override
public void onDisplayRemoved(int displayId) {
}
// The androidx.camera.core.Preview.setTargetRotation declaration is opt-in
// and its usage should be marked with @androidx.camera.core.ExperimentalUseCaseGroup
@ExperimentalUseCaseGroup
@Override
public void onDisplayChanged(int displayId) {
if (displayId == _displayId) {
int rotation = _viewFinder.getDisplay().getRotation();
if (_imageCapture != null) {
_imageCapture.setTargetRotation(rotation);
}
if (_preview != null) {
_preview.setTargetRotation(rotation);
}
}
_camera.setParameters(params);
_camera.takePicture(null, null, pictureCallback);
} else {
failed.run();
}
};
private void setupCamera() {
ListenableFuture<ProcessCameraProvider> cameraProviderFuture = ProcessCameraProvider.getInstance(_activity);
cameraProviderFuture.addListener(() -> {
try {
_cameraProvider = cameraProviderFuture.get();
bindCameraUseCases();
} catch (ExecutionException | InterruptedException e) {
e.printStackTrace();
}
}, ContextCompat.getMainExecutor(_activity));
}
private Camera safeOpenCamera(boolean facingFront) {
Camera result = null;
try {
_cameraId = findFirstCameraId(facingFront);
if (_cameraId != -1) {
result = Camera.open(_cameraId);
}
} catch (RuntimeException e) {
Log.e(LOG_TAG, "Failed to open camera", e);
private void bindCameraUseCases() {
int rotation = _viewFinder.getDisplay().getRotation();
Log.d(LOG_TAG, "rotation: " + rotation);
_imageCapture = new ImageCapture.Builder()
.setTargetRotation(rotation)
.build();
_preview = new Preview.Builder()
.setTargetRotation(rotation)
.build();
_cameraProvider.unbindAll();
int lensFacing = CameraSelector.LENS_FACING_FRONT;
if (!_currentFacingFront) {
lensFacing = CameraSelector.LENS_FACING_BACK;
}
return result;
CameraSelector selector = new CameraSelector.Builder()
.requireLensFacing(lensFacing)
.build();
_cameraProvider.bindToLifecycle(
_activity,
selector,
_preview,
_imageCapture
);
_preview.setSurfaceProvider(_viewFinder.getSurfaceProvider());
}
private int findFirstCameraId(boolean facingFront) {
int result = -1;
int facingTarget = facingFront ? Camera.CameraInfo.CAMERA_FACING_FRONT : Camera.CameraInfo.CAMERA_FACING_BACK;
int count = Camera.getNumberOfCameras();
CameraInfo cameraInfo = new CameraInfo();
for (int i = 0; i < count; i++) {
Camera.getCameraInfo(i, cameraInfo);
if (cameraInfo.facing == facingTarget) {
result = i;
break;
}
}
if (result == -1) {
Log.w(LOG_TAG, "No " + (facingFront ? "front" : "back") + " -facing camera detected on this device.");
}
return result;
public void captureStillImage(ImageCapture.OnImageCapturedCallback callback) {
_imageCapture.takePicture(_cameraExecutor, callback);
}
public boolean setCameraFacing(boolean facingFront) {
boolean result;
if (_currentFacingFront != facingFront) {
// switch cameras
int id = findFirstCameraId(facingFront);
if (id == -1) {
result = false;
} else {
result = true;
_currentFacingFront = facingFront;
if (_camera != null) {
disableOrientationListener();
_camera.release();
_camera = null;
}
_camera = safeOpenCamera(facingFront);
_cameraPreview.startPreview();
enableOrientationListener();
if (facingFront && !hasFrontCamera()) {
return false;
}
} else {
result = true;
if (!facingFront && !hasBackCamera()) {
return false;
}
_currentFacingFront = facingFront;
post(this::bindCameraUseCases);
}
return result;
return true;
}
public RectF getRect() {
return _rect;
private boolean hasBackCamera() {
try {
return _cameraProvider.hasCamera(CameraSelector.DEFAULT_BACK_CAMERA);
} catch (CameraInfoUnavailableException e) {
e.printStackTrace();
}
return false;
}
private boolean hasFrontCamera() {
try {
return _cameraProvider.hasCamera(CameraSelector.DEFAULT_FRONT_CAMERA);
} catch (CameraInfoUnavailableException e) {
e.printStackTrace();
}
return false;
}
/**
@ -193,13 +209,12 @@ public class CameraView
// will not happen - this is a ByteArrayOutputStream
Log.e(LOG_TAG, "IOException while closing byte array stream", e);
}
byte[] jpegData = bos.toByteArray();
return jpegData;
return bos.toByteArray();
}
/**
* Crop and resize the given image to the dimensions of the rectangle for this camera view.
*
* <p>
* If the image was front-facing, also mirror horizontally.
*/
private Bitmap cropResizeAndRotate(Bitmap image, int exifRotation) {
@ -207,7 +222,7 @@ public class CameraView
int imageHeight = image.getHeight();
float rectWidth = _rect.width();
float rectHeight = _rect.height();
float newHeight = rectWidth * imageHeight / imageWidth;
float scale = rectWidth / imageWidth;
int offsetX = 0;
@ -218,7 +233,7 @@ public class CameraView
offsetY = 0;
offsetX = (int) ((newWidth - rectWidth) / 2 * imageWidth / newWidth);
}
Matrix m = new Matrix();
// Adjust the image to undo rotation done by JPEG generator
m.postRotate(-1.0f * exifRotation);
@ -226,167 +241,63 @@ public class CameraView
// flip bitmap horizontally since front-facing camera is mirrored
m.preScale(-1.0f, 1.0f);
}
CameraInfo cameraInfo = new CameraInfo();
Camera.getCameraInfo(_cameraId, cameraInfo);
int rotation = findDisplayRotation(getContext(), cameraInfo.facing);
int rotation = CameraView.findDisplayRotation(getContext(), _currentFacingFront);
if (rotation == 180) {
m.preScale(-1.0f, -1.0f);
}
m.postScale(scale / _scale, scale / _scale);
Bitmap newBitmap = Bitmap.createBitmap(image, offsetX, offsetY, imageWidth - offsetX * 2, imageHeight - offsetY * 2, m, true);
return newBitmap;
return Bitmap.createBitmap(image, offsetX, offsetY, imageWidth - offsetX * 2, imageHeight - offsetY * 2, m, true);
}
private void enableOrientationListener() {
synchronized(_orientationListener) {
if (_orientationListener.canDetectOrientation()) {
_orientationListener.setCameraInfo(_camera, _cameraId);
_orientationListener.enable();
}
// Image → JPEG
public byte[] imageToByteArray(Image image) {
byte[] data = null;
if (image.getFormat() == ImageFormat.JPEG) {
Image.Plane[] planes = image.getPlanes();
ByteBuffer buffer = planes[0].getBuffer();
data = new byte[buffer.capacity()];
buffer.get(data);
return data;
} else if (image.getFormat() == ImageFormat.YUV_420_888) {
data = NV21toJPEG(YUV_420_888toNV21(image),
image.getWidth(), image.getHeight());
}
return data;
}
private void disableOrientationListener() {
synchronized(_orientationListener) {
if (_orientationListener != null) {
_orientationListener.disable();
_orientationListener.clearCameraInfo();
}
}
// YUV_420_888 → NV21
private byte[] YUV_420_888toNV21(Image image) {
byte[] nv21;
ByteBuffer yBuffer = image.getPlanes()[0].getBuffer();
ByteBuffer uBuffer = image.getPlanes()[1].getBuffer();
ByteBuffer vBuffer = image.getPlanes()[2].getBuffer();
int ySize = yBuffer.remaining();
int uSize = uBuffer.remaining();
int vSize = vBuffer.remaining();
nv21 = new byte[ySize + uSize + vSize];
yBuffer.get(nv21, 0, ySize);
vBuffer.get(nv21, ySize, vSize);
uBuffer.get(nv21, ySize + vSize, uSize);
return nv21;
}
private static Display findDisplay(Context context) {
// NV21 → JPEG
private byte[] NV21toJPEG(byte[] nv21, int width, int height) {
ByteArrayOutputStream out = new ByteArrayOutputStream();
YuvImage yuv = new YuvImage(nv21, ImageFormat.NV21, width, height, null);
yuv.compressToJpeg(new Rect(0, 0, width, height), 100, out);
return out.toByteArray();
}
private static int findDisplayRotation(Context context, boolean facingFront) {
WindowManager windowManager = (WindowManager) context.getSystemService(Context.WINDOW_SERVICE);
return windowManager.getDefaultDisplay();
}
private static int findDisplayRotation(Context context, int facing) {
Display display = findDisplay(context);
Display display = windowManager.getDefaultDisplay();
int r = display.getRotation();
int rotation = (r == Surface.ROTATION_180) ? 180 : 0;
if (facing == CameraInfo.CAMERA_FACING_FRONT) {
if (facingFront) {
rotation = (rotation + 360) % 360;
}
return rotation;
}
private class CameraPreviewView
extends SurfaceView
implements SurfaceHolder.Callback
{
private final SurfaceHolder _holder;
public CameraPreviewView(Context context) {
super(context);
_holder = getHolder();
_holder.addCallback(CameraPreviewView.this);
}
@Override
public void surfaceCreated(SurfaceHolder holder) {
try {
if (_camera != null) {
_camera.setPreviewDisplay(holder);
_camera.startPreview();
}
} catch (IOException e) {
Log.e(LOG_TAG, "Error creating surface", e);
}
}
@Override
public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
if (_holder.getSurface() != null) {
if (_camera != null) {
try {
_camera.stopPreview();
} catch (Exception e) {
Log.e(LOG_TAG, "Error releasing camera", e);
}
startPreview();
}
}
}
@Override
public void surfaceDestroyed(SurfaceHolder holder) {
if (_camera != null) {
Log.i(LOG_TAG, "Releasing camera");
disableOrientationListener();
_camera.release();
_camera = null;
}
}
public void startPreview() {
if (_camera != null) {
try {
_camera.setPreviewDisplay(_holder);
Size previewSize = _camera.getParameters().getPreviewSize();
Log.i(LOG_TAG, "Preview size: " + previewSize.width + " x " + previewSize.height);
_camera.startPreview();
} catch (IOException e) {
Log.e(LOG_TAG, "Error in starting preview", e);
} catch (RuntimeException e) {
Log.e(LOG_TAG, "Error in starting preview", e);
}
}
}
}
/**
* An {@link OrientationEventListener} which updates the camera preview
* based on the device's orientation.
* @author khu
*
*/
private static class CameraOrientationListener extends OrientationEventListener {
private Camera _observedCamera;
private int _observedCameraId;
private Display _display;
private int _previousRotation = -1;
private Context _context;
public CameraOrientationListener(Context context) {
super(context);
_context = context;
_display = findDisplay(context);
}
public CameraOrientationListener(Context context, int rate) {
super(context, rate);
_context = context;
_display = findDisplay(context);
}
public synchronized void setCameraInfo(Camera camera, int cameraId) {
_observedCamera = camera;
_observedCameraId = cameraId;
}
public synchronized void clearCameraInfo() {
_observedCamera = null;
_observedCameraId = -1;
}
@Override
public synchronized void onOrientationChanged(int orientation) {
if (orientation == ORIENTATION_UNKNOWN || _observedCamera == null
|| _observedCameraId == -1)
{
return;
}
CameraInfo cameraInfo = new CameraInfo();
Camera.getCameraInfo(_observedCameraId, cameraInfo);
int rotation = findDisplayRotation(_context, cameraInfo.facing);
if (rotation != _previousRotation) {
// Update the preview
_observedCamera.setDisplayOrientation(rotation);
_previousRotation = rotation;
}
}
}
}

32
android/ScratchJr/app/src/main/java/org/scratchjr/android/JavaScriptDirectInterface.java
View file

@ -13,6 +13,7 @@ import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import android.annotation.SuppressLint;
import android.content.Context;
import android.content.Intent;
import android.content.pm.PackageManager;
@ -20,6 +21,7 @@ import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.graphics.RectF;
import android.hardware.Camera;
import android.media.Image;
import android.net.Uri;
import android.text.Html;
import android.util.Base64;
@ -29,6 +31,11 @@ import android.webkit.JavascriptInterface;
import android.widget.ImageView;
import android.widget.RelativeLayout;
import androidx.annotation.NonNull;
import androidx.camera.core.ImageCapture;
import androidx.camera.core.ImageCaptureException;
import androidx.camera.core.ImageProxy;
/**
* The methods in this inner class are exposed directly to JavaScript in the HTML5 pages
* as AndroidInterface.
@ -410,19 +417,24 @@ public class JavaScriptDirectInterface {
@JavascriptInterface
public void scratchjr_captureimage(final String onCameraCaptureComplete) {
_cameraView.captureStillImage(
new Camera.PictureCallback() {
public void onPictureTaken(byte[] jpegData, Camera camera) {
_cameraView.captureStillImage(new ImageCapture.OnImageCapturedCallback() {
@Override
public void onCaptureSuccess(@NonNull ImageProxy imageProxy) {
@SuppressLint("UnsafeOptInUsageError")
Image image = imageProxy.getImage();
if (image != null) {
byte[] jpegData = _cameraView.imageToByteArray(image);
sendBase64Image(onCameraCaptureComplete, jpegData);
}
},
new Runnable() {
public void run() {
Log.e(LOG_TAG, "Could not capture picture");
reportImageError(onCameraCaptureComplete);
}
imageProxy.close();
}
);
@Override
public void onError(@NonNull ImageCaptureException exception) {
Log.e(LOG_TAG, "Could not capture picture");
reportImageError(onCameraCaptureComplete);
}
});
}
@JavascriptInterface

3
android/ScratchJr/app/src/main/java/org/scratchjr/android/ScratchJrActivity.java
View file

@ -15,6 +15,7 @@ import android.os.Build;
import android.os.Bundle;
import android.os.Handler;
import androidx.annotation.NonNull;
import androidx.appcompat.app.AppCompatActivity;
import androidx.core.app.ActivityCompat;
import androidx.core.content.ContextCompat;
import android.util.Log;
@ -49,7 +50,7 @@ import java.util.Vector;
* @author markroth8
*/
public class ScratchJrActivity
extends Activity
extends AppCompatActivity
{
/** Milliseconds to pan when showing the soft keyboard */
private static final int SOFT_KEYBOARD_PAN_MS = 250;

2
android/ScratchJr/app/src/main/res/values/styles.xml
View file

@ -5,7 +5,7 @@
<!-- Customize your theme here. -->
</style>
<style name="FullscreenTheme" parent="android:Theme.NoTitleBar">
<style name="FullscreenTheme" parent="Theme.AppCompat.Light.NoActionBar">
<item name="android:windowContentOverlay">@null</item>
<item name="android:windowBackground">@null</item>
<item name="metaButtonBarStyle">@style/ButtonBar</item>