Skip to content

Commit

Permalink
Merge pull request #6289 from mozilla/object-menu-transform
Browse files Browse the repository at this point in the history
bitECS: Fix object menus transforms
  • Loading branch information
keianhzo authored Oct 12, 2023
2 parents d570fa0 + ead75ba commit 8246d3e
Show file tree
Hide file tree
Showing 24 changed files with 468 additions and 166 deletions.
16 changes: 13 additions & 3 deletions src/bit-components.js
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,8 @@ export const VideoMenu = defineComponent({
trackRef: Types.eid,
headRef: Types.eid,
playIndicatorRef: Types.eid,
pauseIndicatorRef: Types.eid
pauseIndicatorRef: Types.eid,
clearTargetTimer: Types.f64
});
export const AudioEmitter = defineComponent({
flags: Types.ui8
Expand Down Expand Up @@ -286,7 +287,9 @@ export const ObjectMenu = defineComponent({
rotateButtonRef: Types.eid,
mirrorButtonRef: Types.eid,
scaleButtonRef: Types.eid,
targetRef: Types.eid
targetRef: Types.eid,
handlingTargetRef: Types.eid,
flags: Types.ui8
});
// TODO: Store this data elsewhere, since only one or two will ever exist.
export const LinkHoverMenu = defineComponent({
Expand All @@ -309,7 +312,9 @@ export const PDFMenu = defineComponent({
targetRef: Types.eid,
clearTargetTimer: Types.f64
});
export const ObjectMenuTarget = defineComponent();
export const ObjectMenuTarget = defineComponent({
flags: Types.ui8
});
export const NetworkDebug = defineComponent();
export const NetworkDebugRef = defineComponent({
ref: Types.eid
Expand Down Expand Up @@ -393,3 +398,8 @@ export const MediaLink = defineComponent({
src: Types.ui32
});
MediaLink.src[$isStringType] = true;
export const ObjectMenuTransform = defineComponent({
targetObjectRef: Types.eid,
prevObjectRef: Types.eid,
flags: Types.ui8
});
55 changes: 13 additions & 42 deletions src/bit-systems/link-hover-menu.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import { Not, defineQuery, entityExists } from "bitecs";
import { Matrix4, Vector3 } from "three";
import { Not, addComponent, defineQuery, entityExists, removeComponent } from "bitecs";
import type { HubsWorld } from "../app";
import {
Link,
Expand All @@ -8,16 +7,17 @@ import {
TextTag,
Interacted,
LinkHoverMenuItem,
LinkInitializing
LinkInitializing,
ObjectMenuTransform
} from "../bit-components";
import { findAncestorWithComponent, findChildWithComponent } from "../utils/bit-utils";
import { hubIdFromUrl } from "../utils/media-url-utils";
import { Text as TroikaText } from "troika-three-text";
import { handleExitTo2DInterstitial } from "../utils/vr-interstitial";
import { changeHub } from "../change-hub";
import { EntityID } from "../utils/networking-types";
import { setMatrixWorld } from "../utils/three-utils";
import { LinkType } from "../inflators/link";
import { ObjectMenuTransformFlags } from "../inflators/object-menu-transform";

const menuQuery = defineQuery([LinkHoverMenu]);
const hoveredLinksQuery = defineQuery([HoveredRemoteRight, Link, Not(LinkInitializing)]);
Expand Down Expand Up @@ -89,43 +89,6 @@ async function handleLinkClick(world: HubsWorld, button: EntityID) {
}
}

const _moveTargetPos = new Vector3();
const _lookAtTargetPos = new Vector3();
const _objectPos = new Vector3();
const _mat4 = new Matrix4();

// Move the menu object to target object position but a little bit closer
// to the camera and make the menu object look at the camera.
// TODO: Similar code in object-menu system. Expose as util and reuse?
function moveToTarget(world: HubsWorld, menu: EntityID) {
const menuObj = world.eid2obj.get(menu)!;

const targetObj = world.eid2obj.get(LinkHoverMenu.targetObjectRef[menu])!;
targetObj.updateMatrices();

// TODO: Remove the dependency with AFRAME
const camera = AFRAME.scenes[0].systems["hubs-systems"].cameraSystem.viewingCamera;
camera.updateMatrices();

_moveTargetPos.setFromMatrixPosition(targetObj.matrixWorld);
_lookAtTargetPos.setFromMatrixPosition(camera.matrixWorld);

// Place the menu object a little bit closer to the camera in the scene
_objectPos
.copy(_lookAtTargetPos)
.sub(_moveTargetPos)
.normalize()
// TODO: 0.5 is an arbitrary number. 0.5 might be too small for
// huge target object. Using bounding box may be safer?
// TODO: What if camera is between the menu and the target object?
.multiplyScalar(0.5)
.add(_moveTargetPos);

_mat4.copy(camera.matrixWorld).setPosition(_objectPos);
setMatrixWorld(menuObj, _mat4);
menuObj.lookAt(_lookAtTargetPos);
}

function updateButtonText(world: HubsWorld, menu: EntityID, button: EntityID) {
const text = findChildWithComponent(world, TextTag, button)!;
const textObj = world.eid2obj.get(text)! as TroikaText;
Expand Down Expand Up @@ -157,6 +120,15 @@ function flushToObject3Ds(world: HubsWorld, menu: EntityID, frozen: boolean, for
const target = LinkHoverMenu.targetObjectRef[menu];
const visible = !!target && !frozen;

// TODO We are handling menus visibility in a similar way for all the object menus, we
// should probably refactor this to a common object-menu-visibility-system
if (visible) {
ObjectMenuTransform.targetObjectRef[menu] = target;
ObjectMenuTransform.flags[menu] |= ObjectMenuTransformFlags.Enabled;
} else {
ObjectMenuTransform.flags[menu] &= ~ObjectMenuTransformFlags.Enabled;
}

const obj = world.eid2obj.get(menu)!;
obj.visible = visible;

Expand All @@ -182,7 +154,6 @@ export function linkHoverMenuSystem(world: HubsWorld, sceneIsFrozen: boolean) {
updateLinkMenuTarget(world, menu, sceneIsFrozen);
const currTarget = LinkHoverMenu.targetObjectRef[menu];
if (currTarget) {
moveToTarget(world, menu);
clickedMenuItemQuery(world).forEach(eid => handleLinkClick(world, eid));
}
flushToObject3Ds(world, menu, sceneIsFrozen, prevTarget !== currTarget);
Expand Down
17 changes: 8 additions & 9 deletions src/bit-systems/media-loading.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,7 @@ import { inflateGrabbable } from "../inflators/grabbable";

const getBox = (() => {
const rotation = new Euler();
return (world: HubsWorld, eid: EntityID, rootEid: EntityID, worldSpace?: boolean) => {
const box = new Box3();
return (world: HubsWorld, eid: EntityID, rootEid: EntityID, box: Box3, worldSpace?: boolean) => {
const obj = world.eid2obj.get(eid)!;
const rootObj = world.eid2obj.get(rootEid)!;

Expand All @@ -61,8 +60,6 @@ const getBox = (() => {

rootObj.matrixWorldNeedsUpdate = true;
rootObj.updateMatrices();

return box;
};
})();

Expand Down Expand Up @@ -187,6 +184,7 @@ function* loadByMediaType(
case MediaType.IMAGE:
mediaEid = yield* loadImage(
world,
eid,
accessibleUrl,
contentType,
MediaImageLoaderData.has(eid) ? MediaImageLoaderData.get(eid)! : {}
Expand All @@ -195,6 +193,7 @@ function* loadByMediaType(
case MediaType.VIDEO:
mediaEid = yield* loadVideo(
world,
eid,
accessibleUrl,
contentType,
MediaVideoLoaderData.has(eid) ? MediaVideoLoaderData.get(eid)! : {}
Expand All @@ -204,18 +203,17 @@ function* loadByMediaType(
mediaEid = yield* loadModel(world, accessibleUrl, contentType, true);
break;
case MediaType.PDF:
mediaEid = yield* loadPDF(world, accessibleUrl);
break;
return yield* loadPDF(world, eid, accessibleUrl);
case MediaType.AUDIO:
mediaEid = yield* loadAudio(
world,
eid,
accessibleUrl,
MediaVideoLoaderData.has(eid) ? MediaVideoLoaderData.get(eid)! : {}
);
break;
case MediaType.HTML:
mediaEid = yield* loadHtml(world, canonicalUrl, thumbnail);
break;
return yield* loadHtml(world, eid, canonicalUrl, thumbnail);
default:
throw new UnsupportedMediaTypeError(eid, mediaType);
}
Expand Down Expand Up @@ -255,6 +253,7 @@ function* loadMedia(world: HubsWorld, eid: EntityID) {
}

const tmpVector = new Vector3();
const box = new Box3();
function* loadAndAnimateMedia(world: HubsWorld, eid: EntityID, clearRollbacks: ClearFunction) {
if (MediaLoader.flags[eid] & MEDIA_LOADER_FLAGS.IS_OBJECT_MENU_TARGET) {
addComponent(world, ObjectMenuTarget, eid);
Expand All @@ -272,7 +271,7 @@ function* loadAndAnimateMedia(world: HubsWorld, eid: EntityID, clearRollbacks: C

if (media) {
if (hasComponent(world, MediaLoaded, media)) {
const box = getBox(world, eid, media);
getBox(world, eid, media, box);
addComponent(world, MediaContentBounds, eid);
box.getSize(tmpVector);
MediaContentBounds.bounds[eid].set(tmpVector.toArray());
Expand Down
113 changes: 113 additions & 0 deletions src/bit-systems/object-menu-transform-system.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
import { defineQuery } from "bitecs";
import { HubsWorld } from "../app";
import { ObjectMenuTarget, ObjectMenuTransform } from "../bit-components";
import { EntityID } from "../utils/networking-types";
import { Box3, Matrix4, Object3D, Quaternion, Sphere, Vector3 } from "three";
import { isFacingCamera, setFromObject, setMatrixWorld } from "../utils/three-utils";
import { ObjectMenuTargetFlags } from "../inflators/object-menu-target";
import { ObjectMenuTransformFlags } from "../inflators/object-menu-transform";

const offset = new Vector3();
const tmpVec1 = new Vector3();
const tmpVec2 = new Vector3();
const tmpQuat1 = new Quaternion();
const tmpQuat2 = new Quaternion();
const tmpMat4 = new Matrix4();
const tmpMat42 = new Matrix4();
const aabb = new Box3();
const sphere = new Sphere();
const yVector = new Vector3(0, 1, 0);

// Calculate the AABB without accounting for the root object rotation
function getAABB(obj: Object3D, box: Box3, onlyVisible: boolean = false) {
const parent = obj.parent;
obj.removeFromParent();
obj.updateMatrices(true, true);
tmpMat4.copy(obj.matrixWorld);
tmpMat4.decompose(tmpVec1, tmpQuat1, tmpVec2);
tmpQuat2.copy(tmpQuat1);
obj.quaternion.identity();
obj.updateMatrix();
obj.updateMatrixWorld();
setFromObject(box, obj, onlyVisible);
parent?.add(obj);
obj.quaternion.copy(tmpQuat2);
obj.updateMatrix();
obj.updateMatrixWorld();
}

// Check https://github.com/mozilla/hubs/pull/6289#issuecomment-1739003555 for implementation details.
function transformMenu(world: HubsWorld, menu: EntityID) {
const targetEid = ObjectMenuTransform.targetObjectRef[menu];
const targetObj = world.eid2obj.get(targetEid);
const enabled = (ObjectMenuTransform.flags[menu] & ObjectMenuTransformFlags.Enabled) !== 0 ? true : false;
if (!targetObj || !enabled) return;

const menuObj = world.eid2obj.get(menu)!;

// Calculate the menu offset based on visible elements
const center = (ObjectMenuTransform.flags[menu] & ObjectMenuTransformFlags.Center) !== 0 ? true : false;
if (center && ObjectMenuTransform.targetObjectRef[menu] !== ObjectMenuTransform.prevObjectRef[menu]) {
getAABB(menuObj, aabb);
aabb.getCenter(tmpVec1);
getAABB(menuObj, aabb, true);
aabb.getCenter(tmpVec2);
offset.subVectors(tmpVec1, tmpVec2);
offset.z = 0;
}

const camera = APP.scene?.systems["hubs-systems"].cameraSystem.viewingCamera;
camera.updateMatrices();

const isFlat = (ObjectMenuTarget.flags[targetEid] & ObjectMenuTargetFlags.Flat) !== 0 ? true : false;
if (!isFlat) {
getAABB(targetObj, aabb);
aabb.getBoundingSphere(sphere);

tmpMat4.copy(targetObj.matrixWorld);
tmpMat4.decompose(tmpVec1, tmpQuat1, tmpVec2);
tmpVec2.set(1.0, 1.0, 1.0);
tmpMat4.compose(sphere.center, tmpQuat1, tmpVec2);

setMatrixWorld(menuObj, tmpMat4);

menuObj.lookAt(tmpVec2.setFromMatrixPosition(camera.matrixWorld));
menuObj.translateZ(sphere.radius);

if (center) {
menuObj.position.add(offset);
menuObj.matrixNeedsUpdate = true;
}

// TODO We need to handle the menu positioning when the player is inside the bounding sphere.
// For now we are defaulting to the current AFrame behavior.
} else {
targetObj.updateMatrices(true, true);
tmpMat4.copy(targetObj.matrixWorld);
tmpMat4.decompose(tmpVec1, tmpQuat1, tmpVec2);

const isFacing = isFacingCamera(targetObj);
if (!isFacing) {
tmpQuat1.setFromAxisAngle(yVector, Math.PI);
tmpMat42.makeRotationFromQuaternion(tmpQuat1);
tmpMat4.multiply(tmpMat42);
}

if (center) {
tmpMat42.makeTranslation(offset.x, offset.y, offset.z);
tmpMat4.multiply(tmpMat42);
}

setMatrixWorld(menuObj, tmpMat4);
}

ObjectMenuTransform.prevObjectRef[menu] = ObjectMenuTransform.targetObjectRef[menu];
}

const menuQuery = defineQuery([ObjectMenuTransform]);

export function objectMenuTransformSystem(world: HubsWorld) {
menuQuery(world).forEach(menu => {
transformMenu(world, menu);
});
}
Loading

0 comments on commit 8246d3e

Please sign in to comment.