scratch-vm/src/extensions/scratch3_video_sensing/library.js

/**
 * @file library.js
 *
 * Tony Hwang and John Maloney, January 2011
 * Michael "Z" Goddard, March 2018
 *
 * Video motion sensing primitives.
 */

const {motionVector, scratchAtan2} = require('./math');

/**
 * The width of the intended resolution to analyze for motion.
 * @type {number}
 */
const WIDTH = 480;

/**
 * The height of the intended resolution to analyze for motion.
 * @type {number}
 */
const HEIGHT = 360;

/**
 * A constant value to scale the magnitude of the x and y components called u
 * and v. This creates the motionAmount value.
 *
 * Old note: chosen empirically to give a range of roughly 0-100
 *
 * @type {number}
 */
const AMOUNT_SCALE = 100;

/**
 * A constant value to scale the magnitude of the x and y components called u
 * and v in the local motion derivative. This creates the motionAmount value on
 * a target's motion state.
 *
 * Old note: note 2e-4 * activePixelNum is an experimentally tuned threshold
 * for my logitech Pro 9000 webcam - TTH
 *
 * @type {number}
 */
const LOCAL_AMOUNT_SCALE = AMOUNT_SCALE * 2e-4;

/**
 * The motion amount must be higher than the THRESHOLD to calculate a new
 * direction value.
 * @type {number}
 */
const THRESHOLD = 10;

/**
 * The size of the radius of the window of summarized values when considering
 * the motion inside the full resolution of the sample.
 * @type {number}
 */
const WINSIZE = 8;

/**
 * A ceiling for the motionAmount stored to a local target's motion state. The
 * motionAmount is not allowed to be larger than LOCAL_MAX_AMOUNT.
 * @type {number}
 */
const LOCAL_MAX_AMOUNT = 100;

/**
 * The motion amount for a target's local motion must be higher than the
 * LOCAL_THRESHOLD to calculate a new direction value.
 * @type {number}
 */
const LOCAL_THRESHOLD = THRESHOLD / 3;

/**
 * Store the necessary image pixel data to compares frames of a video and
 * detect an amount and direction of motion in the full sample or in a
 * specified area.
 * @constructor
 */
class VideoMotion {
    constructor () {
        /**
         * The number of frames that have been added from a source.
         * @type {number}
         */
        this.frameNumber = 0;

        /**
         * The frameNumber last analyzed.
         * @type {number}
         */
        this.lastAnalyzedFrame = 0;

        /**
         * The amount of motion detected in the current frame.
         * @type {number}
         */
        this.motionAmount = 0;

        /**
         * The direction the motion detected in the frame is general moving in.
         * @type {number}
         */
        this.motionDirection = 0;

        /**
         * A copy of the current frame's pixel values. A index of the array is
         * represented in RGBA. The lowest byte is red. The next is green. The
         * next is blue. And the last is the alpha value of that pixel.
         * @type {Uint32Array}
         */
        this.curr = null;

        /**
         * A copy of the last frame's pixel values.
         * @type {Uint32Array}
         */
        this.prev = null;

        /**
         * A buffer for holding one component of a pixel's full value twice.
         * One for the current value. And one for the last value.
         * @type {number}
         */
        this._arrays = new ArrayBuffer(WIDTH * HEIGHT * 2 * 1);

        /**
         * A clamped uint8 view of _arrays. One component of each index of the
         * curr member is copied into this array.
         * @type {number}
         */
        this._curr = new Uint8ClampedArray(this._arrays, WIDTH * HEIGHT * 0 * 1, WIDTH * HEIGHT);

        /**
         * A clamped uint8 view of _arrays. One component of each index of the
         * prev member is copied into this array.
         * @type {number}
         */
        this._prev = new Uint8ClampedArray(this._arrays, WIDTH * HEIGHT * 1 * 1, WIDTH * HEIGHT);
    }

    /**
     * Reset internal state so future frame analysis does not consider values
     * from before this method was called.
     */
    reset () {
        this.frameNumber = 0;
        this.lastAnalyzedFrame = 0;
        this.motionAmount = this.motionDirection = 0;
        this.prev = this.curr = null;
    }

    /**
     * Add a frame to be next analyzed. The passed array represent a pixel with
     * each index in the RGBA format.
     * @param {Uint32Array} source - a source frame of pixels to copy
     */
    addFrame (source) {
        this.frameNumber++;

        // Swap curr to prev.
        this.prev = this.curr;
        // Create a clone of the array so any modifications made to the source
        // array do not affect the work done in here.
        this.curr = new Uint32Array(source.buffer.slice(0));

        // Swap _prev and _curr. Copy one of the color components of the new
        // array into _curr overwriting what was the old _prev data.
        const _tmp = this._prev;
        this._prev = this._curr;
        this._curr = _tmp;
        for (let i = 0; i < this.curr.length; i++) {
            this._curr[i] = this.curr[i] & 0xff;
        }
    }

    /**
     * Analyze the current frame against the previous frame determining the
     * amount of motion and direction of the motion.
     */
    analyzeFrame () {
        if (!this.curr || !this.prev) {
            this.motionAmount = this.motionDirection = -1;
            // Don't have two frames to analyze yet
            return;
        }

        // Return early if new data has not been received.
        if (this.lastAnalyzedFrame === this.frameNumber) {
            return;
        }
        this.lastAnalyzedFrame = this.frameNumber;

        const {
            _curr: curr,
            _prev: prev
        } = this;

        const winStep = (WINSIZE * 2) + 1;
        const wmax = WIDTH - WINSIZE - 1;
        const hmax = HEIGHT - WINSIZE - 1;

        // Accumulate 2d motion vectors from groups of pixels and average it
        // later.
        let uu = 0;
        let vv = 0;
        let n = 0;

        // Iterate over groups of cells building up the components to determine
        // a motion vector for each cell instead of the whole frame to avoid
        // integer overflows.
        for (let i = WINSIZE + 1; i < hmax; i += winStep) {
            for (let j = WINSIZE + 1; j < wmax; j += winStep) {
                let A2 = 0;
                let A1B2 = 0;
                let B1 = 0;
                let C1 = 0;
                let C2 = 0;

                // This is a performance critical math region.
                let address = ((i - WINSIZE) * WIDTH) + j - WINSIZE;
                let nextAddress = address + winStep;
                const maxAddress = ((i + WINSIZE) * WIDTH) + j + WINSIZE;
                for (; address <= maxAddress; address += WIDTH - winStep, nextAddress += WIDTH) {
                    for (; address <= nextAddress; address += 1) {
                        // The difference in color between the last frame and
                        // the current frame.
                        const gradT = ((prev[address]) - (curr[address]));
                        // The difference between the pixel to the left and the
                        // pixel to the right.
                        const gradX = ((curr[address - 1]) - (curr[address + 1]));
                        // The difference between the pixel above and the pixel
                        // below.
                        const gradY = ((curr[address - WIDTH]) - (curr[address + WIDTH]));

                        // Add the combined values of this pixel to previously
                        // considered pixels.
                        A2 += gradX * gradX;
                        A1B2 += gradX * gradY;
                        B1 += gradY * gradY;
                        C2 += gradX * gradT;
                        C1 += gradY * gradT;
                    }
                }

                // Use the accumalated values from the for loop to determine a
                // motion direction.
                const {u, v} = motionVector(A2, A1B2, B1, C2, C1);

                // If u and v are within negative winStep to positive winStep,
                // add them to a sum that will later be averaged.
                if (-winStep < u && u < winStep && -winStep < v && v < winStep) {
                    uu += u;
                    vv += v;
                    n++;
                }
            }
        }

        // Average the summed vector values of all of the motion groups.
        uu /= n;
        vv /= n;

        // Scale the magnitude of the averaged UV vector.
        this.motionAmount = Math.round(AMOUNT_SCALE * Math.hypot(uu, vv));
        if (this.motionAmount > THRESHOLD) {
            // Scratch direction
            this.motionDirection = scratchAtan2(vv, uu);
        }
    }

    /**
     * Build motion amount and direction values based on stored current and
     * previous frame that overlaps a given drawable.
     * @param {Drawable} drawable - touchable and bounded drawable to build motion for
     * @param {MotionState} state - state to store built values to
     */
    getLocalMotion (drawable, state) {
        if (!this.curr || !this.prev) {
            state.motionAmount = state.motionDirection = -1;
            // Don't have two frames to analyze yet
            return;
        }

        // Skip if the current frame has already been considered for this state.
        if (state.motionFrameNumber !== this.frameNumber) {
            const {
                _prev: prev,
                _curr: curr
            } = this;

            // Restrict the region the amount and direction are built from to
            // the area of the current frame overlapped by the given drawable's
            // bounding box.
            const boundingRect = drawable.getFastBounds();
            // Transform the bounding box from scratch space to a space from 0,
            // 0 to WIDTH, HEIGHT.
            const xmin = Math.max(Math.floor(boundingRect.left + (WIDTH / 2)), 1);
            const xmax = Math.min(Math.floor(boundingRect.right + (WIDTH / 2)), WIDTH - 1);
            const ymin = Math.max(Math.floor((HEIGHT / 2) - boundingRect.top), 1);
            const ymax = Math.min(Math.floor((HEIGHT / 2) - boundingRect.bottom), HEIGHT - 1);

            let A2 = 0;
            let A1B2 = 0;
            let B1 = 0;
            let C1 = 0;
            let C2 = 0;
            let scaleFactor = 0;

            const position = [0, 0, 0];

            // This is a performance critical math region.
            for (let i = ymin; i < ymax; i++) {
                for (let j = xmin; j < xmax; j++) {
                    // i and j are in a coordinate planning ranging from 0 to
                    // HEIGHT and 0 to WIDTH. Transform that into Scratch's
                    // range of HEIGHT / 2 to -HEIGHT / 2 and -WIDTH / 2 to
                    // WIDTH / 2;
                    position[0] = j - (WIDTH / 2);
                    position[1] = (HEIGHT / 2) - i;
                    // Consider only pixels in the drawable that can touch the
                    // edge or other drawables. Empty space in the current skin
                    // is skipped.
                    if (drawable.isTouching(position)) {
                        const address = (i * WIDTH) + j;
                        // The difference in color between the last frame and
                        // the current frame.
                        const gradT = ((prev[address]) - (curr[address]));
                        // The difference between the pixel to the left and the
                        // pixel to the right.
                        const gradX = ((curr[address - 1]) - (curr[address + 1]));
                        // The difference between the pixel above and the pixel
                        // below.
                        const gradY = ((curr[address - WIDTH]) - (curr[address + WIDTH]));

                        // Add the combined values of this pixel to previously
                        // considered pixels.
                        A2 += gradX * gradX;
                        A1B2 += gradX * gradY;
                        B1 += gradY * gradY;
                        C2 += gradX * gradT;
                        C1 += gradY * gradT;
                        scaleFactor++;
                    }
                }
            }

            // Use the accumalated values from the for loop to determine a
            // motion direction.
            let {u, v} = motionVector(A2, A1B2, B1, C2, C1);

            let activePixelNum = 0;
            if (scaleFactor) {
                // Store the area of the sprite in pixels
                activePixelNum = scaleFactor;

                scaleFactor /= (2 * WINSIZE * 2 * WINSIZE);
                u = u / scaleFactor;
                v = v / scaleFactor;
            }

            // Scale the magnitude of the averaged UV vector and the number of
            // overlapping drawable pixels.
            state.motionAmount = Math.round(LOCAL_AMOUNT_SCALE * activePixelNum * Math.hypot(u, v));
            if (state.motionAmount > LOCAL_MAX_AMOUNT) {
                // Clip all magnitudes greater than 100.
                state.motionAmount = LOCAL_MAX_AMOUNT;
            }
            if (state.motionAmount > LOCAL_THRESHOLD) {
                // Scratch direction.
                state.motionDirection = scratchAtan2(v, u);
            }

            // Skip future calls on this state until a new frame is added.
            state.motionFrameNumber = this.frameNumber;
        }
    }
}

module.exports = VideoMotion;