Skip to content

Commit

Permalink
#32 Add hand tracking by using frame.fillPoses for devices that suppo…
Browse files Browse the repository at this point in the history 
…rt it
  • Loading branch information
@Rufus31415
Rufus31415 committed May 1, 2021
1 parent 069fade commit 7b37e75
Show file tree
Hide file tree
Showing 6 changed files with 155 additions and 41 deletions.
25 changes: 25 additions & 0 deletions ...eWebXR-Demo/Assets/SimpleWebXR/Scripts/MRTK/Profiles/SimpleWebXRHandTrackingProfile.asset
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
%YAML 1.1
%TAG !u! tag:unity3d.com,2011:
--- !u!114 &11400000
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 0}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 8275efdbe76bdff49a97a8e82fba118d, type: 3}
m_Name: SimpleWebXRHandTrackingProfile
m_EditorClassIdentifier:
isCustomProfile: 1
jointPrefab: {fileID: 1955475817299902, guid: 6a3f88d2571cd234a86d95ee5856b9ec,
type: 3}
palmPrefab: {fileID: 6797406804172968804, guid: 750bdc3344567a447960aae3eda2b462,
type: 3}
fingertipPrefab: {fileID: 7094064642998883381, guid: b37dde41a983d664c8a09a91313733e7,
type: 3}
handMeshPrefab: {fileID: 1887883006053652, guid: a86f479797fea8f4189f924b3b6ad979,
type: 3}
handMeshVisualizationModes: -1
handJointVisualizationModes: 0
8 changes: 8 additions & 0 deletions ...R-Demo/Assets/SimpleWebXR/Scripts/MRTK/Profiles/SimpleWebXRHandTrackingProfile.asset.meta
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion ...leWebXR-Demo/Assets/SimpleWebXR/Scripts/MRTK/Profiles/SimpleWebXRInputSystemProfile.asset
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,5 +74,5 @@ MonoBehaviour:
type: 2}
controllerVisualizationProfile: {fileID: 11400000, guid: 345c06fdf3732db46b96299bd3cba653,
type: 2}
handTrackingProfile: {fileID: 11400000, guid: 7f1e3cd673742f94ca860ac7ae733024,
handTrackingProfile: {fileID: 11400000, guid: 1ee5870617471ac4e82a5d1fe41dac1b,
type: 2}
30 changes: 24 additions & 6 deletions SimpleWebXR-Demo/Assets/SimpleWebXR/Scripts/MRTK/SimpleWebXRHand.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@ public override bool TryGetJoint(TrackedHandJoint joint, out MixedRealityPose po

#endregion IMixedRealityHand Implementation

private ArticulatedHandDefinition handDefinition;
private ArticulatedHandDefinition HandDefinition => handDefinition ?? (handDefinition = Definition as ArticulatedHandDefinition);



public override MixedRealityInteractionMapping[] DefaultInteractions => new[]
{
Expand All @@ -50,24 +54,35 @@ public override void SetupDefaultInteractions()

public override bool IsInPointingPose => true;

private MixedRealityPose GetJointMixedRealityPose(WebXRJoint joint)
{

var position = MixedRealityPlayspace.TransformPoint(joint.Position);
var rotation = MixedRealityPlayspace.Rotation * joint.Rotation;

return new MixedRealityPose(position, rotation);
}

public void UpdateController(WebXRInputSource controller)
{
if (!Enabled) return;

IsPositionAvailable = IsRotationAvailable = controller.Hand.Available;


for (int i = 0; i < WebXRHand.JOINT_COUNT; i++)
jointPoses[TrackedHandJoint.Wrist] = GetJointMixedRealityPose(controller.Hand.Joints[WebXRHand.WRIST]);


for (int i = WebXRHand.THUMB_METACARPAL; i < WebXRHand.JOINT_COUNT; i++)
{
var joint = controller.Hand.Joints[i];

var position = MixedRealityPlayspace.TransformPoint(joint.Position);
var rotation = MixedRealityPlayspace.Rotation * joint.Rotation;

jointPoses[(TrackedHandJoint)(i + 1)] = new MixedRealityPose(position, rotation);
jointPoses[(TrackedHandJoint)(i + 2)] = GetJointMixedRealityPose(joint);
}

var indexPose = jointPoses[(TrackedHandJoint)WebXRHand.INDEX_PHALANX_TIP + 1];
jointPoses[TrackedHandJoint.Palm] = new MixedRealityPose((jointPoses[TrackedHandJoint.MiddleMetacarpal].Position + jointPoses[TrackedHandJoint.MiddleMetacarpal].Position)/2, jointPoses[TrackedHandJoint.MiddleMetacarpal].Rotation) ;

var indexPose = jointPoses[TrackedHandJoint.IndexTip];

bool isSelecting;
MixedRealityPose spatialPointerPose;
Expand Down Expand Up @@ -152,6 +167,9 @@ public void UpdateController(WebXRInputSource controller)
CoreServices.InputSystem?.RaisePoseInputChanged(InputSource, ControllerHandedness, Interactions[i].MixedRealityInputAction, Interactions[i].PoseData);
}
break;
case DeviceInputType.ThumbStick:
HandDefinition?.UpdateCurrentTeleportPose(Interactions[i]);
break;
}
}
}
Expand Down
20 changes: 17 additions & 3 deletions SimpleWebXR-Demo/ProjectSettings/EditorSettings.asset
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,33 @@
--- !u!159 &1
EditorSettings:
m_ObjectHideFlags: 0
serializedVersion: 7
serializedVersion: 9
m_ExternalVersionControlSupport: Visible Meta Files
m_SerializationMode: 2
m_LineEndingsForNewScripts: 2
m_DefaultBehaviorMode: 0
m_PrefabRegularEnvironment: {fileID: 0}
m_PrefabUIEnvironment: {fileID: 0}
m_SpritePackerMode: 0
m_SpritePackerPaddingPower: 1
m_EtcTextureCompressorBehavior: 1
m_EtcTextureFastCompressor: 1
m_EtcTextureNormalCompressor: 2
m_EtcTextureBestCompressor: 4
m_ProjectGenerationIncludedExtensions: txt;xml;fnt;cd
m_ProjectGenerationIncludedExtensions: txt;xml;fnt;cd;asmref
m_ProjectGenerationRootNamespace:
m_UserGeneratedProjectSuffix:
m_CollabEditorSettings:
inProgressEnabled: 1
m_EnableTextureStreamingInEditMode: 1
m_EnableTextureStreamingInPlayMode: 1
m_AsyncShaderCompilation: 1
m_EnterPlayModeOptionsEnabled: 0
m_EnterPlayModeOptions: 3
m_ShowLightmapResolutionOverlay: 1
m_UseLegacyProbeSampleCount: 1
m_AssetPipelineMode: 0
m_CacheServerMode: 0
m_CacheServerEndpoint:
m_CacheServerNamespacePrefix: default
m_CacheServerEnableDownload: 1
m_CacheServerEnableUpload: 1
111 changes: 80 additions & 31 deletions com.rufus31415.simplewebxr/Runtime/Plugins/WebGL/SimpleWebXR.jslib
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,20 +85,20 @@ mergeInto(LibraryManager.library, {
_isArSupported = false;

if (!navigator.xr) {
document.dispatchEvent(new CustomEvent("SimpleWebXRInitialized", {success: false}));
document.dispatchEvent(new CustomEvent("SimpleWebXRInitialized", { success: false }));
return
};

// Check if WebXR immersive VR is supported (check immersive-vr before immersive-ar to make it work on Oculus Quest Browser)
navigator.xr.isSessionSupported('immersive-vr').then(function (supported) {
_isVrSupported = supported;
document.dispatchEvent(new CustomEvent("SimpleWebXRSessionSupported", {xrSessionMode:'immersive-vr', supported: supported}));
document.dispatchEvent(new CustomEvent("SimpleWebXRSessionSupported", { xrSessionMode: 'immersive-vr', supported: supported }));
});

// Check if WebXR immersive AR is supported
navigator.xr.isSessionSupported('immersive-ar').then(function (supported) {
_isArSupported = supported;
document.dispatchEvent(new CustomEvent("SimpleWebXRSessionSupported", {xrSessionMode:'immersive-ar', supported: supported}));
document.dispatchEvent(new CustomEvent("SimpleWebXRSessionSupported", { xrSessionMode: 'immersive-ar', supported: supported }));
});

// Initialize pointers to shared arrays that contains data (projection matrix, position, orientation, input sources)
Expand Down Expand Up @@ -126,7 +126,7 @@ mergeInto(LibraryManager.library, {
// Share position
var position = view.transform.position;

if(_firstFrame) _yOffset = position.y;
if (_firstFrame) _yOffset = position.y;

_dataArray[floatStartId + 16] = position.x;
_dataArray[floatStartId + 17] = position.y - _yOffset;
Expand Down Expand Up @@ -220,10 +220,10 @@ mergeInto(LibraryManager.library, {
_byteArray[byteStartId + 3] = 0
}

if(_dataArray[103+id]>0 && inputSource.gamepad.hapticActuators && inputSource.gamepad.hapticActuators.length>0){
if (_dataArray[103 + id] > 0 && inputSource.gamepad.hapticActuators && inputSource.gamepad.hapticActuators.length > 0) {
// Trigger of haptic vibration pulse(intensity [0..1], duration in ms)
inputSource.gamepad.hapticActuators[0].pulse(_dataArray[101+id], _dataArray[103+id]);
_dataArray[103+id] = 0; // reset flag once it's done
inputSource.gamepad.hapticActuators[0].pulse(_dataArray[101 + id], _dataArray[103 + id]);
_dataArray[103 + id] = 0; // reset flag once it's done
}
}
else {
Expand Down Expand Up @@ -253,31 +253,80 @@ mergeInto(LibraryManager.library, {

// Hand detection
// https://immersive-web.github.io/webxr-hand-input/#skeleton-joints-section
if (inputSource.hand && inputSource.hand.length == 25) {
_byteArray[46 + id] = 1; // hand supported
var delta = (_useLocalSpaceForInput ? 0 : _dataArray[100])
for (var j = 0; j < 25; j++) {
if (inputSource.hand[j] !== null) {
var joint = frame.getJointPose(inputSource.hand[j], _useLocalSpaceForInput ? _arSession.localSpace : _arSession.localFloorSpace);
if (joint !== null) {
var i = id * 200 + j * 8;
_handArray[i] = joint.transform.position.x;
_handArray[i + 1] = joint.transform.position.y - delta - _yOffset;
_handArray[i + 2] = joint.transform.position.z;
_handArray[i + 3] = joint.transform.orientation.x;
_handArray[i + 4] = joint.transform.orientation.y;
_handArray[i + 5] = joint.transform.orientation.z;
_handArray[i + 6] = joint.transform.orientation.w;
if (joint.radius !== null) {
_handArray[i + 7] = joint.radius;
if (inputSource.hand) {

// For browsers that support fillPoses
if (typeof frame.fillPoses === "function") {
_byteArray[46 + id] = 1; // hand supported

var delta = (_useLocalSpaceForInput ? 0 : _dataArray[100])

var refSpace = _useLocalSpaceForInput ? _arSession.localSpace : _arSession.localFloorSpace;

var radii = new Float32Array(25);
var poses = new Float32Array(16 * 25);

if (inputSource.hand.values) {
frame.fillPoses(inputSource.hand.values(), refSpace, poses);
frame.fillJointRadii(inputSource.hand.values(), radii);
} else {
frame.fillPoses(inputSource.hand, refSpace, poses);
frame.fillJointRadii(inputSource.hand, radii);
}

for (var j = 0; j < 25; j++) {
var jointIndex = j * 16;

var i = id * 200 + j * 8;
_handArray[i] = poses[jointIndex + 12];
_handArray[i + 1] = poses[jointIndex + 13] - delta - _yOffset;
_handArray[i + 2] = poses[jointIndex + 14];


_handArray[i + 7] = radii[j];

var quaternion = new Float32Array(4);

quaternion[3] = Math.sqrt(Math.max(0, 1 + poses[jointIndex + 0] + poses[jointIndex + 5] + poses[jointIndex + 10])) / 2;
quaternion[0] = Math.sqrt(Math.max(0, 1 + poses[jointIndex + 0] - poses[jointIndex + 5] - poses[jointIndex + 10])) / 2;
quaternion[1] = Math.sqrt(Math.max(0, 1 - poses[jointIndex + 0] + poses[jointIndex + 5] - poses[jointIndex + 10])) / 2;
quaternion[2] = Math.sqrt(Math.max(0, 1 - poses[jointIndex + 0] - poses[jointIndex + 5] + poses[jointIndex + 10])) / 2;
quaternion[0] *= Math.sign(quaternion[0] * (poses[jointIndex + 6] - poses[jointIndex + 9]));
quaternion[1] *= Math.sign(quaternion[1] * (poses[jointIndex + 8] - poses[jointIndex + 2]));
quaternion[2] *= Math.sign(quaternion[2] * (poses[jointIndex + 1] - poses[jointIndex + 4]));

_handArray[i + 3] = quaternion[0];
_handArray[i + 4] = quaternion[1];
_handArray[i + 5] = quaternion[2];
_handArray[i + 6] = quaternion[3];
}
}
else {
_byteArray[46 + id] = 1; // hand supported

for (var j = 0; j < 25; j++) {
if (inputSource.hand[j] !== null) {
var joint = frame.getJointPose(inputSource.hand[j], refSpace);
if (joint !== null) {
var i = id * 200 + j * 8;
_handArray[i] = joint.transform.position.x;
_handArray[i + 1] = joint.transform.position.y - delta - _yOffset;
_handArray[i + 2] = joint.transform.position.z;
_handArray[i + 3] = joint.transform.orientation.x;
_handArray[i + 4] = joint.transform.orientation.y;
_handArray[i + 5] = joint.transform.orientation.z;
_handArray[i + 6] = joint.transform.orientation.w;
if (joint.radius !== null) {
_handArray[i + 7] = joint.radius;
}
else {
_handArray[i + 7] = NaN;
}
}
else {
_handArray[i + 7] = NaN;
_byteArray[46 + id] = 0; // hand not fully supported
}
}
else {
_byteArray[46 + id] = 0; // hand not fully supported
}
}
}
}
Expand Down Expand Up @@ -434,7 +483,7 @@ mergeInto(LibraryManager.library, {
}
}(GLctx.bindFramebuffer);

document.dispatchEvent(new CustomEvent("SimpleWebXRInitialized", {success: true}));
document.dispatchEvent(new CustomEvent("SimpleWebXRInitialized", { success: true }));
},

/****************************************************************************/
Expand Down Expand Up @@ -468,7 +517,7 @@ mergeInto(LibraryManager.library, {
GLctx.ARSessionStarted = _isArSupported;
session.isInSession = true; // add field in session to indicate that a session in running

document.dispatchEvent(new CustomEvent("SimpleWebXRSessionStarted", {session: session, GLctx: GLctx}));
document.dispatchEvent(new CustomEvent("SimpleWebXRSessionStarted", { session: session, GLctx: GLctx }));

var glLayer = new XRWebGLLayer(session, GLctx);

Expand Down Expand Up @@ -531,7 +580,7 @@ mergeInto(LibraryManager.library, {
_orientationInfo[0] = 0;

_onDeviceOrientation = function (event) {
if(_orientationInfo[0] == 0) document.dispatchEvent(new CustomEvent("SimpleWebXRDeviceOrientationStarted"));
if (_orientationInfo[0] == 0) document.dispatchEvent(new CustomEvent("SimpleWebXRDeviceOrientationStarted"));

_orientationInfo[0] = 1;
_orientationArray[0] = event.alpha;
Expand Down

0 comments on commit 7b37e75

Please sign in to comment.