From a9f8858aee29175ba624786bd02c5202b208681d Mon Sep 17 00:00:00 2001 From: Jordan Santell Date: Wed, 10 Jan 2018 11:46:24 -0800 Subject: [PATCH] Use Sensor APIs when available instead of devicemotion. Fixes #10 --- src/cardboard-vr-display.js | 2 +- src/sensor-fusion/fusion-pose-sensor.js | 446 ++++++++++++++---------- 2 files changed, 261 insertions(+), 187 deletions(-) diff --git a/src/cardboard-vr-display.js b/src/cardboard-vr-display.js index aafef1e..bd1f972 100644 --- a/src/cardboard-vr-display.js +++ b/src/cardboard-vr-display.js @@ -85,7 +85,7 @@ CardboardVRDisplay.prototype = Object.create(VRDisplay.prototype); CardboardVRDisplay.prototype._getPose = function() { return { - position: this.poseSensor_.getPosition(), + position: null, orientation: this.poseSensor_.getOrientation(), linearVelocity: null, linearAcceleration: null, diff --git a/src/sensor-fusion/fusion-pose-sensor.js b/src/sensor-fusion/fusion-pose-sensor.js index 161f43e..c080621 100644 --- a/src/sensor-fusion/fusion-pose-sensor.js +++ b/src/sensor-fusion/fusion-pose-sensor.js @@ -17,6 +17,11 @@ import PosePredictor from './pose-predictor.js'; import * as MathUtil from '../math-util.js'; import * as Util from '../util.js'; +// Frequency which the Sensors will attempt to fire their +// `reading` functions at. Use 16.6ms to achieve 60FPS, since +// we generally can't get higher without native VR hardware +const SENSOR_FREQUENCY = 16.6; + /** * The pose sensor, implemented using DeviceMotion APIs. * @@ -25,211 +30,280 @@ import * as Util from '../util.js'; * @param {boolean} yawOnly * @param {boolean} isDebug */ -function FusionPoseSensor(kFilter, predictionTime, yawOnly, isDebug) { - this.yawOnly = yawOnly; - - this.accelerometer = new MathUtil.Vector3(); - this.gyroscope = new MathUtil.Vector3(); - - this.start(); - - this.filter = new ComplementaryFilter(kFilter, isDebug); - this.posePredictor = new PosePredictor(predictionTime, isDebug); - - this.filterToWorldQ = new MathUtil.Quaternion(); - - // Set the filter to world transform, depending on OS. - if (Util.isIOS()) { - this.filterToWorldQ.setFromAxisAngle(new MathUtil.Vector3(1, 0, 0), Math.PI / 2); - } else { - this.filterToWorldQ.setFromAxisAngle(new MathUtil.Vector3(1, 0, 0), -Math.PI / 2); +export default class FusionPoseSensor { + constructor(kFilter, predictionTime, yawOnly, isDebug) { + this.yawOnly = yawOnly; + + this.accelerometer = new MathUtil.Vector3(); + this.gyroscope = new MathUtil.Vector3(); + + this.filter = new ComplementaryFilter(kFilter, isDebug); + this.posePredictor = new PosePredictor(predictionTime, isDebug); + + this.filterToWorldQ = new MathUtil.Quaternion(); + + // Set the filter to world transform, depending on OS. + if (Util.isIOS()) { + this.filterToWorldQ.setFromAxisAngle(new MathUtil.Vector3(1, 0, 0), Math.PI / 2); + } else { + this.filterToWorldQ.setFromAxisAngle(new MathUtil.Vector3(1, 0, 0), -Math.PI / 2); + } + + this.inverseWorldToScreenQ = new MathUtil.Quaternion(); + this.worldToScreenQ = new MathUtil.Quaternion(); + this.originalPoseAdjustQ = new MathUtil.Quaternion(); + this.originalPoseAdjustQ.setFromAxisAngle(new MathUtil.Vector3(0, 0, 1), + -window.orientation * Math.PI / 180); + + this.setScreenTransform_(); + // Adjust this filter for being in landscape mode. + if (Util.isLandscapeMode()) { + this.filterToWorldQ.multiply(this.inverseWorldToScreenQ); + } + + // Keep track of a reset transform for resetSensor. + this.resetQ = new MathUtil.Quaternion(); + + this.isFirefoxAndroid = Util.isFirefoxAndroid(); + this.isIOS = Util.isIOS(); + + this.orientationOut_ = new Float32Array(4); + + // Store information on API in use (either 'devicemotion' or 'sensor'), + // as well as any errors observed from using sensors for debugging. + this.api = null; + this.error = null; + + this.onDeviceMotion_ = this.onDeviceMotion_.bind(this); + this.onOrientationChange_ = this.onOrientationChange_.bind(this); + this.onMessage_ = this.onMessage_.bind(this); + this.onSensorError_ = this.onSensorError_.bind(this); + this.onGyroscopeRead_ = this.onGyroscopeRead_.bind(this); + this.onAccelerometerRead_ = this.onAccelerometerRead_.bind(this); + + this.start(); } - this.inverseWorldToScreenQ = new MathUtil.Quaternion(); - this.worldToScreenQ = new MathUtil.Quaternion(); - this.originalPoseAdjustQ = new MathUtil.Quaternion(); - this.originalPoseAdjustQ.setFromAxisAngle(new MathUtil.Vector3(0, 0, 1), - -window.orientation * Math.PI / 180); - - this.setScreenTransform_(); - // Adjust this filter for being in landscape mode. - if (Util.isLandscapeMode()) { - this.filterToWorldQ.multiply(this.inverseWorldToScreenQ); + start() { + // Only listen for postMessages if we're in an iOS and embedded inside a cross + // origin IFrame. In this case, the polyfill can still work if the containing + // page sends synthetic devicemotion events. For an example of this, see + // the iframe example in the repo at `examples/iframe.html` + if (Util.isIOS() && Util.isInsideCrossOriginIFrame()) { + window.addEventListener('message', this.onMessage_); + } + + window.addEventListener('orientationchange', this.onOrientationChange_); + + // Attempt to use the Gyroscope and Accelerometer from Generic Sensor APIs. + // First available in Chrome M63, this can fail for several reasons, and attempt + // to fallback to devicemotion. Failure scenarios include: + // + // * Generic Sensor APIs do not exist, fallback to devicemotion + // * Underlying sensor does not exist, no fallback possible. + // * Feature policy failure (in an iframe); no fallback. + // * @TODO is it both possible, and respecting of intent, to fallback to + // devicemotion here the same way we handle iframes on iOS? + // @see `onMessage_()` + // * Permission to sensor data denied; respect user agent, no fallback to devicemotion + let accelerometer = null; + let gyroscope = null; + try { + accelerometer = new Accelerometer({ frequency: SENSOR_FREQUENCY }); + gyroscope = new Gyroscope({ frequency: SENSOR_FREQUENCY }); + accelerometer.addEventListener('error', this.onSensorError); + gyroscope.addEventListener('error', this.onSensorError); + } catch (error) { + this.error = error; + + if (error.name === 'SecurityError') { + console.error('Cannot construct sensors due to the feature policy'); + } else if (error.name === 'ReferenceError') { + // Fallback to devicemotion + this.api = 'devicemotion'; + window.addEventListener('devicemotion', this.onDeviceMotion_); + } + } + + if (accelerometer && gyroscope) { + this.sensors = { gyroscope, accelerometer }; + this.onAccelerometerRead_ = this.onAccelerometerRead_.bind(this); + accelerometer.addEventListener('reading', this.onAccelerometerRead_); + accelerometer.start(); + gyroscope.addEventListener('reading', this.onGyroscopeRead_); + gyroscope.start(); + this.api = 'sensor'; + } } - // Keep track of a reset transform for resetSensor. - this.resetQ = new MathUtil.Quaternion(); - - this.isFirefoxAndroid = Util.isFirefoxAndroid(); - this.isIOS = Util.isIOS(); - - this.orientationOut_ = new Float32Array(4); -} + stop() { + window.removeEventListener('message', this.onMessage_); + window.removeEventListener('orientationchange', this.onOrientationChange_); + window.removeEventListener('devicemotion', this.onDeviceMotion_); + if (this.sensors) { + this.sensors.gyroscope.removeEventListener('reading', this.onGyroscopeRead_); + this.sensors.accelerometer.removeEventListener('reading', this.onAccelerometerRead_); + } + } -FusionPoseSensor.prototype.getPosition = function() { - // This PoseSensor doesn't support position - return null; -}; - -FusionPoseSensor.prototype.getOrientation = function() { - // Convert from filter space to the the same system used by the - // deviceorientation event. - var orientation = this.filter.getOrientation(); - - // Predict orientation. - this.predictedQ = this.posePredictor.getPrediction(orientation, this.gyroscope, this.previousTimestampS); - - // Convert to THREE coordinate system: -Z forward, Y up, X right. - var out = new MathUtil.Quaternion(); - out.copy(this.filterToWorldQ); - out.multiply(this.resetQ); - out.multiply(this.predictedQ); - out.multiply(this.worldToScreenQ); - - // Handle the yaw-only case. - if (this.yawOnly) { - // Make a quaternion that only turns around the Y-axis. - out.x = 0; - out.z = 0; - out.normalize(); + getOrientation() { + // Convert from filter space to the the same system used by the + // deviceorientation event. + var orientation = this.filter.getOrientation(); + + // Predict orientation. + this.predictedQ = this.posePredictor.getPrediction(orientation, this.gyroscope, this.previousTimestampS); + + // Convert to THREE coordinate system: -Z forward, Y up, X right. + var out = new MathUtil.Quaternion(); + out.copy(this.filterToWorldQ); + out.multiply(this.resetQ); + out.multiply(this.predictedQ); + out.multiply(this.worldToScreenQ); + + // Handle the yaw-only case. + if (this.yawOnly) { + // Make a quaternion that only turns around the Y-axis. + out.x = 0; + out.z = 0; + out.normalize(); + } + + this.orientationOut_[0] = out.x; + this.orientationOut_[1] = out.y; + this.orientationOut_[2] = out.z; + this.orientationOut_[3] = out.w; + return this.orientationOut_; } - this.orientationOut_[0] = out.x; - this.orientationOut_[1] = out.y; - this.orientationOut_[2] = out.z; - this.orientationOut_[3] = out.w; - return this.orientationOut_; -}; - -FusionPoseSensor.prototype.resetPose = function() { - // Reduce to inverted yaw-only. - this.resetQ.copy(this.filter.getOrientation()); - this.resetQ.x = 0; - this.resetQ.y = 0; - this.resetQ.z *= -1; - this.resetQ.normalize(); - - // Take into account extra transformations in landscape mode. - if (Util.isLandscapeMode()) { - this.resetQ.multiply(this.inverseWorldToScreenQ); + resetPose() { + // Reduce to inverted yaw-only. + this.resetQ.copy(this.filter.getOrientation()); + this.resetQ.x = 0; + this.resetQ.y = 0; + this.resetQ.z *= -1; + this.resetQ.normalize(); + + // Take into account extra transformations in landscape mode. + if (Util.isLandscapeMode()) { + this.resetQ.multiply(this.inverseWorldToScreenQ); + } + + // Take into account original pose. + this.resetQ.multiply(this.originalPoseAdjustQ); } - // Take into account original pose. - this.resetQ.multiply(this.originalPoseAdjustQ); -}; - -FusionPoseSensor.prototype.onDeviceMotion_ = function(deviceMotion) { - this.updateDeviceMotion_(deviceMotion); -}; - -FusionPoseSensor.prototype.updateDeviceMotion_ = function(deviceMotion) { - var accGravity = deviceMotion.accelerationIncludingGravity; - var rotRate = deviceMotion.rotationRate; - var timestampS = deviceMotion.timeStamp / 1000; - - var deltaS = timestampS - this.previousTimestampS; - - // On Firefox/iOS the `timeStamp` properties can come in out of order. - // so emit a warning about it and then stop. The rotation still ends up - // working. - // @TODO is there a better way to handle this with the `interval` property - // from the device motion event? `timeStamp` seems to be non-standard. - if (deltaS < 0) { - Util.warnOnce('fusion-pose-sensor:invalid:non-monotonic', - 'Invalid timestamps detected: non-monotonic timestamp from devicemotion'); - this.previousTimestampS = timestampS; - return; - } else if (deltaS <= Util.MIN_TIMESTEP || deltaS > Util.MAX_TIMESTEP) { - Util.warnOnce('fusion-pose-sensor:invalid:outside-threshold', - 'Invalid timestamps detected: Timestamp from devicemotion outside expected range.'); + onDeviceMotion_(deviceMotion) { + var accGravity = deviceMotion.accelerationIncludingGravity; + var rotRate = deviceMotion.rotationRate; + var timestampS = deviceMotion.timeStamp / 1000; + + var deltaS = timestampS - this.previousTimestampS; + + // On Firefox/iOS the `timeStamp` properties can come in out of order. + // so emit a warning about it and then stop. The rotation still ends up + // working. + // @TODO is there a better way to handle this with the `interval` property + // from the device motion event? `timeStamp` seems to be non-standard. + if (deltaS < 0) { + Util.warnOnce('fusion-pose-sensor:invalid:non-monotonic', + 'Invalid timestamps detected: non-monotonic timestamp from devicemotion'); + this.previousTimestampS = timestampS; + return; + } else if (deltaS <= Util.MIN_TIMESTEP || deltaS > Util.MAX_TIMESTEP) { + Util.warnOnce('fusion-pose-sensor:invalid:outside-threshold', + 'Invalid timestamps detected: Timestamp from devicemotion outside expected range.'); + this.previousTimestampS = timestampS; + return; + } + + this.accelerometer.set(-accGravity.x, -accGravity.y, -accGravity.z); + if (Util.isR7()) { + this.gyroscope.set(-rotRate.beta, rotRate.alpha, rotRate.gamma); + } else { + this.gyroscope.set(rotRate.alpha, rotRate.beta, rotRate.gamma); + } + + // With iOS and Firefox Android, rotationRate is reported in degrees, + // so we first convert to radians. + if (this.isIOS || this.isFirefoxAndroid) { + this.gyroscope.multiplyScalar(Math.PI / 180); + } + + this.filter.addAccelMeasurement(this.accelerometer, timestampS); + this.filter.addGyroMeasurement(this.gyroscope, timestampS); + this.previousTimestampS = timestampS; - return; } - this.accelerometer.set(-accGravity.x, -accGravity.y, -accGravity.z); - if (Util.isR7()) { - this.gyroscope.set(-rotRate.beta, rotRate.alpha, rotRate.gamma); - } else { - this.gyroscope.set(rotRate.alpha, rotRate.beta, rotRate.gamma); + onOrientationChange_(screenOrientation) { + this.setScreenTransform_(); } - // With iOS and Firefox Android, rotationRate is reported in degrees, - // so we first convert to radians. - if (this.isIOS || this.isFirefoxAndroid) { - this.gyroscope.multiplyScalar(Math.PI / 180); + /** + * This is only needed if we are in an cross origin iframe on iOS to work around + * this issue: https://bugs.webkit.org/show_bug.cgi?id=152299. + */ + onMessage_(event) { + var message = event.data; + + // If there's no message type, ignore it. + if (!message || !message.type) { + return; + } + + // Ignore all messages that aren't devicemotion. + var type = message.type.toLowerCase(); + if (type !== 'devicemotion') { + return; + } + + // Update device motion. + this.onDeviceMotion_(message.deviceMotionEvent); } - this.filter.addAccelMeasurement(this.accelerometer, timestampS); - this.filter.addGyroMeasurement(this.gyroscope, timestampS); - - this.previousTimestampS = timestampS; -}; - -FusionPoseSensor.prototype.onOrientationChange_ = function(screenOrientation) { - this.setScreenTransform_(); -}; - -/** - * This is only needed if we are in an cross origin iframe on iOS to work around - * this issue: https://bugs.webkit.org/show_bug.cgi?id=152299. - */ -FusionPoseSensor.prototype.onMessage_ = function(event) { - var message = event.data; - - // If there's no message type, ignore it. - if (!message || !message.type) { - return; + onSensorError_(e) { + this.error = e.error; + if (e.error.name === 'NotAllowedError') { + console.error('Permission to access sensor was denied'); + } else if (e.error.name === 'NotReadableError' ) { + console.error('Sensor not present'); + } } - // Ignore all messages that aren't devicemotion. - var type = message.type.toLowerCase(); - if (type !== 'devicemotion') { - return; + onGyroscopeRead_() { + const timestampS = this.sensors.gyroscope.timestamp / 1000; + this.gyroscope.x = this.sensors.gyroscope.x; + this.gyroscope.y = this.sensors.gyroscope.y; + this.gyroscope.z = this.sensors.gyroscope.z; + this.filter.addGyroMeasurement(this.gyroscope, timestampS); + this.previousTimestampS = timestampS; } - // Update device motion. - this.updateDeviceMotion_(message.deviceMotionEvent); -}; - -FusionPoseSensor.prototype.setScreenTransform_ = function() { - this.worldToScreenQ.set(0, 0, 0, 1); - switch (window.orientation) { - case 0: - break; - case 90: - this.worldToScreenQ.setFromAxisAngle(new MathUtil.Vector3(0, 0, 1), -Math.PI / 2); - break; - case -90: - this.worldToScreenQ.setFromAxisAngle(new MathUtil.Vector3(0, 0, 1), Math.PI / 2); - break; - case 180: - // TODO. - break; - } - this.inverseWorldToScreenQ.copy(this.worldToScreenQ); - this.inverseWorldToScreenQ.inverse(); -}; - -FusionPoseSensor.prototype.start = function() { - this.onDeviceMotionCallback_ = this.onDeviceMotion_.bind(this); - this.onOrientationChangeCallback_ = this.onOrientationChange_.bind(this); - this.onMessageCallback_ = this.onMessage_.bind(this); - - // Only listen for postMessages if we're in an iOS and embedded inside a cross - // origin IFrame. In this case, the polyfill can still work if the containing - // page sends synthetic devicemotion events. For an example of this, see - // the iframe example in the repo at `examples/iframe.html` - if (Util.isIOS() && Util.isInsideCrossOriginIFrame()) { - window.addEventListener('message', this.onMessageCallback_); + onAccelerometerRead_() { + const timestampS = this.sensors.accelerometer.timestamp / 1000; + this.accelerometer.x = -this.sensors.accelerometer.x; + this.accelerometer.y = -this.sensors.accelerometer.y; + this.accelerometer.z = -this.sensors.accelerometer.z; + this.filter.addAccelMeasurement(this.accelerometer, timestampS); } - window.addEventListener('orientationchange', this.onOrientationChangeCallback_); - window.addEventListener('devicemotion', this.onDeviceMotionCallback_); -}; -FusionPoseSensor.prototype.stop = function() { - window.removeEventListener('devicemotion', this.onDeviceMotionCallback_); - window.removeEventListener('orientationchange', this.onOrientationChangeCallback_); - window.removeEventListener('message', this.onMessageCallback_); -}; - -export default FusionPoseSensor; + setScreenTransform_() { + this.worldToScreenQ.set(0, 0, 0, 1); + switch (window.orientation) { + case 0: + break; + case 90: + this.worldToScreenQ.setFromAxisAngle(new MathUtil.Vector3(0, 0, 1), -Math.PI / 2); + break; + case -90: + this.worldToScreenQ.setFromAxisAngle(new MathUtil.Vector3(0, 0, 1), Math.PI / 2); + break; + case 180: + // TODO. + break; + } + this.inverseWorldToScreenQ.copy(this.worldToScreenQ); + this.inverseWorldToScreenQ.inverse(); + } +}