diff --git a/Scripts/BaseListener.cs b/Scripts/BaseListener.cs new file mode 100644 index 0000000..08a8287 --- /dev/null +++ b/Scripts/BaseListener.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using UnityEngine; + +namespace PupilLabs +{ + public abstract class BaseListener + { + public bool IsListening { get; private set; } + + protected SubscriptionsController subsCtrl; + + public BaseListener(SubscriptionsController subsCtrl) + { + this.subsCtrl = subsCtrl; + } + + ~BaseListener() + { + if (subsCtrl.IsConnected) + { + Disable(); + } + } + + public void Enable() + { + if (!subsCtrl.IsConnected) + { + Debug.LogWarning($"{this.GetType().Name}: No connected. Waiting for connection."); + subsCtrl.requestCtrl.OnConnected += EnableOnConnect; + return; + } + + if (IsListening) + { + Debug.Log("Already listening."); + return; + } + + + CustomEnable(); + + IsListening = true; + } + + public void EnableOnConnect() + { + subsCtrl.requestCtrl.OnConnected -= EnableOnConnect; + Enable(); + } + + protected abstract void CustomEnable(); + + public void Disable() + { + if (!subsCtrl.IsConnected) + { + Debug.LogWarning("Not connected!"); + IsListening = false; + return; + } + + if (!IsListening) + { + Debug.Log("Not running."); + return; + } + + CustomDisable(); + IsListening = false; + } + + protected abstract void CustomDisable(); + } +} diff --git a/Scripts/Calibration.cs b/Scripts/Calibration.cs new file mode 100644 index 0000000..1309a2d --- /dev/null +++ b/Scripts/Calibration.cs @@ -0,0 +1,181 @@ +using System; +using System.Collections.Generic; +using UnityEngine; +using NetMQ.Sockets; +using NetMQ; +using MessagePack; + +namespace PupilLabs +{ + public class Calibration + { + //events + public event Action OnCalibrationStarted; + public event Action OnCalibrationSucceeded; + public event Action OnCalibrationFailed; + + //members + SubscriptionsController subsCtrl; + RequestController requestCtrl; + Publisher publisher; + CalibrationSettings settings; + + List> calibrationData = new List>(); + float[] rightEyeTranslation; + float[] leftEyeTranslation; + + public bool IsCalibrating { get; set; } + + public void StartCalibration(CalibrationSettings settings, SubscriptionsController subsCtrl) + { + this.settings = settings; + this.subsCtrl = subsCtrl; + this.requestCtrl = subsCtrl.requestCtrl; + + if (OnCalibrationStarted != null) + { + OnCalibrationStarted(); + } + + IsCalibrating = true; + + subsCtrl.SubscribeTo("notify.calibration.successful", ReceiveSuccess); + subsCtrl.SubscribeTo("notify.calibration.failed", ReceiveFailure); + + requestCtrl.StartPlugin(settings.PluginName); + publisher = new Publisher(requestCtrl); + + UpdateEyesTranslation(); + + requestCtrl.Send(new Dictionary { + { "subject","calibration.should_start" }, + { + "translation_eye0", + rightEyeTranslation + }, + { + "translation_eye1", + leftEyeTranslation + }, + { + "record", + true + } + }); + + Debug.Log("Calibration Started"); + + calibrationData.Clear(); + } + + public void AddCalibrationPointReferencePosition(float[] position, double timestamp) + { + calibrationData.Add(new Dictionary() { + { settings.PositionKey, position }, + { "timestamp", timestamp }, + }); + } + + public void SendCalibrationReferenceData() + { + Debug.Log("Send CalibrationReferenceData"); + + Send(new Dictionary { + { "subject","calibration.add_ref_data" }, + { + "ref_data", + calibrationData.ToArray () + }, + { + "record", + true + } + }); + + //Clear the current calibration data, so we can proceed to the next point if there is any. + calibrationData.Clear(); + } + + public void StopCalibration() + { + Debug.Log("Calibration should stop"); + + IsCalibrating = false; + + Send(new Dictionary { + { + "subject", + "calibration.should_stop" + }, + { + "record", + true + } + }); + } + + public void Destroy() + { + if (publisher != null) + { + publisher.Destroy(); + } + } + + private void Send(Dictionary data) + { + string topic = "notify." + data["subject"]; + publisher.Send(topic, data); + } + + private void UpdateEyesTranslation() + { + Vector3 leftEye = UnityEngine.XR.InputTracking.GetLocalPosition(UnityEngine.XR.XRNode.LeftEye); + Vector3 rightEye = UnityEngine.XR.InputTracking.GetLocalPosition(UnityEngine.XR.XRNode.RightEye); + Vector3 centerEye = UnityEngine.XR.InputTracking.GetLocalPosition(UnityEngine.XR.XRNode.CenterEye); + Quaternion centerRotation = UnityEngine.XR.InputTracking.GetLocalRotation(UnityEngine.XR.XRNode.CenterEye); + + //convert local coords into center eye coordinates + Vector3 globalCenterPos = Quaternion.Inverse(centerRotation) * centerEye; + Vector3 globalLeftEyePos = Quaternion.Inverse(centerRotation) * leftEye; + Vector3 globalRightEyePos = Quaternion.Inverse(centerRotation) * rightEye; + + //right + var relativeRightEyePosition = globalRightEyePos - globalCenterPos; + relativeRightEyePosition *= Helpers.PupilUnitScalingFactor; + rightEyeTranslation = new float[] { relativeRightEyePosition.x, relativeRightEyePosition.y, relativeRightEyePosition.z }; + + //left + var relativeLeftEyePosition = globalLeftEyePos - globalCenterPos; + relativeLeftEyePosition *= Helpers.PupilUnitScalingFactor; + leftEyeTranslation = new float[] { relativeLeftEyePosition.x, relativeLeftEyePosition.y, relativeLeftEyePosition.z }; + } + + private void ReceiveSuccess(string topic, Dictionary dictionary, byte[] thirdFrame) + { + if (OnCalibrationSucceeded != null) + { + OnCalibrationSucceeded(); + } + + CalibrationEnded(topic); + } + + private void ReceiveFailure(string topic, Dictionary dictionary, byte[] thirdFrame) + { + if (OnCalibrationFailed != null) + { + OnCalibrationFailed(); + } + + CalibrationEnded(topic); + } + + private void CalibrationEnded(string topic) + { + Debug.Log($"Calibration response: {topic}"); + subsCtrl.UnsubscribeFrom("notify.calibration.successful", ReceiveSuccess); + subsCtrl.UnsubscribeFrom("notify.calibration.failed", ReceiveFailure); + } + } +} \ No newline at end of file diff --git a/Scripts/CalibrationController.cs b/Scripts/CalibrationController.cs new file mode 100644 index 0000000..f9655bd --- /dev/null +++ b/Scripts/CalibrationController.cs @@ -0,0 +1,335 @@ +using System; +using System.Collections.Generic; +using UnityEngine; +using UnityEngine.UI; + +namespace PupilLabs +{ + public class CalibrationController : MonoBehaviour + { + [Header("Pupil Labs Connection")] + public SubscriptionsController subsCtrl; + public TimeSync timeSync; + + [Header("Scene References")] + public new Camera camera; + public Transform marker; + + [Header("Settings")] + public CalibrationSettings settings; + public CalibrationTargets targets; + public bool showPreview; + + public bool IsCalibrating { get { return calibration.IsCalibrating; } } + + //events + public event Action OnCalibrationStarted; + public event Action OnCalibrationRoutineDone; + public event Action OnCalibrationFailed; + public event Action OnCalibrationSucceeded; + + //members + Calibration calibration = new Calibration(); + + int targetIdx; + int targetSampleCount; + Vector3 currLocalTargetPos; + + float tLastSample = 0; + float tLastTarget = 0; + List previewMarkers = new List(); + + bool previewMarkersActive = false; + + void OnEnable() + { + calibration.OnCalibrationSucceeded += CalibrationSucceeded; + calibration.OnCalibrationFailed += CalibrationFailed; + + bool allReferencesValid = true; + if (subsCtrl == null) + { + Debug.LogError("SubscriptionsController reference missing!"); + allReferencesValid = false; + } + if (timeSync == null) + { + Debug.LogError("TimeSync reference missing!"); + allReferencesValid = false; + } + if (marker == null) + { + Debug.LogError("Marker reference missing!"); + allReferencesValid = false; + } + if (camera == null) + { + Debug.LogError("Camera reference missing!"); + allReferencesValid = false; + } + if (settings == null) + { + Debug.LogError("CalibrationSettings reference missing!"); + allReferencesValid = false; + } + if (targets == null) + { + Debug.LogError("CalibrationTargets reference missing!"); + allReferencesValid = false; + } + if (!allReferencesValid) + { + Debug.LogError("CalibrationController is missing required references to other components. Please connect the references, or the component won't work correctly."); + enabled = false; + return; + } + + InitPreviewMarker(); + } + + void OnDisable() + { + calibration.OnCalibrationSucceeded -= CalibrationSucceeded; + calibration.OnCalibrationFailed -= CalibrationFailed; + + if (calibration.IsCalibrating) + { + StopCalibration(); + } + } + + void Update() + { + if (showPreview != previewMarkersActive) + { + SetPreviewMarkers(showPreview); + } + + if (calibration.IsCalibrating) + { + UpdateCalibration(); + } + + if (Input.GetKeyUp(KeyCode.C)) + { + ToggleCalibration(); + } + else if (Input.GetKeyDown(KeyCode.P)) + { + showPreview = !showPreview; + } + } + + public void ToggleCalibration() + { + if (calibration.IsCalibrating) + { + StopCalibration(); + } + else + { + StartCalibration(); + } + } + + public void StartCalibration() + { + if (!enabled) + { + Debug.LogWarning("Component not enabled!"); + return; + } + + if (!subsCtrl.IsConnected) + { + Debug.LogWarning("Calibration not possible: not connected!"); + return; + } + + Debug.Log("Starting Calibration"); + + showPreview = false; + + targetIdx = 0; + targetSampleCount = 0; + + UpdatePosition(); + + marker.gameObject.SetActive(true); + + calibration.StartCalibration(settings, subsCtrl); + Debug.Log($"Sample Rate: {settings.SampleRate}"); + + if (OnCalibrationStarted != null) + { + OnCalibrationStarted(); + } + + //abort process on disconnecting + subsCtrl.OnDisconnecting += StopCalibration; + } + + public void StopCalibration() + { + if (!calibration.IsCalibrating) + { + Debug.Log("Nothing to stop."); + return; + } + + calibration.StopCalibration(); + + marker.gameObject.SetActive(false); + + if (OnCalibrationRoutineDone != null) + { + OnCalibrationRoutineDone(); + } + + subsCtrl.OnDisconnecting -= StopCalibration; + } + + void OnApplicationQuit() + { + calibration.Destroy(); + } + + private void UpdateCalibration() + { + UpdateMarker(); + + float tNow = Time.time; + if (tNow - tLastSample >= 1f / settings.SampleRate - Time.deltaTime / 2f) + { + + if (tNow - tLastTarget < settings.ignoreInitialSeconds - Time.deltaTime / 2f) + { + return; + } + + tLastSample = tNow; + + //Adding the calibration reference data to the list that will be passed on, once the required sample amount is met. + double sampleTimeStamp = timeSync.ConvertToPupilTime(Time.realtimeSinceStartup); + AddSample(sampleTimeStamp); + + targetSampleCount++;//Increment the current calibration sample. (Default sample amount per calibration point is 120) + + if (targetSampleCount >= settings.samplesPerTarget || tNow - tLastTarget >= settings.secondsPerTarget) + { + calibration.SendCalibrationReferenceData(); + + if (targetIdx < targets.GetTargetCount()) + { + targetSampleCount = 0; + + UpdatePosition(); + } + else + { + StopCalibration(); + } + } + } + } + + private void CalibrationSucceeded() + { + if (OnCalibrationSucceeded != null) + { + OnCalibrationSucceeded(); + } + } + + private void CalibrationFailed() + { + if (OnCalibrationFailed != null) + { + OnCalibrationFailed(); + } + } + + private void AddSample(double time) + { + float[] refData; + + refData = new float[] { currLocalTargetPos.x, currLocalTargetPos.y, currLocalTargetPos.z }; + refData[1] /= camera.aspect; + + for (int i = 0; i < refData.Length; i++) + { + refData[i] *= Helpers.PupilUnitScalingFactor; + } + + calibration.AddCalibrationPointReferencePosition(refData, time); + } + + private void UpdatePosition() + { + currLocalTargetPos = targets.GetLocalTargetPosAt(targetIdx); + + targetIdx++; + tLastTarget = Time.time; + } + + private void UpdateMarker() + { + marker.position = camera.transform.localToWorldMatrix.MultiplyPoint(currLocalTargetPos); + marker.LookAt(camera.transform.position); + } + + void OnDrawGizmos() + { + if (camera == null || targets == null) + { + return; + } + + if (Application.isPlaying) + { + return; + } + + Gizmos.matrix = camera.transform.localToWorldMatrix; + for (int i = 0; i < targets.GetTargetCount(); ++i) + { + var target = targets.GetLocalTargetPosAt(i); + Gizmos.DrawWireSphere(target, 0.035f); + } + } + + void InitPreviewMarker() + { + + var previewMarkerParent = new GameObject("Calibration Targets Preview"); + previewMarkerParent.transform.SetParent(camera.transform); + previewMarkerParent.transform.localPosition = Vector3.zero; + previewMarkerParent.transform.localRotation = Quaternion.identity; + + for (int i = 0; i < targets.GetTargetCount(); ++i) + { + var target = targets.GetLocalTargetPosAt(i); + var previewMarker = Instantiate(marker.gameObject); + previewMarker.transform.parent = previewMarkerParent.transform; + previewMarker.transform.localPosition = target; + previewMarker.transform.LookAt(camera.transform.position); + previewMarker.SetActive(true); + previewMarkers.Add(previewMarker); + } + + previewMarkersActive = true; + } + + void SetPreviewMarkers(bool value) + { + foreach (var marker in previewMarkers) + { + marker.SetActive(value); + } + + previewMarkersActive = value; + } + + + } +} diff --git a/Scripts/CalibrationSettings.cs b/Scripts/CalibrationSettings.cs new file mode 100644 index 0000000..94fa60d --- /dev/null +++ b/Scripts/CalibrationSettings.cs @@ -0,0 +1,30 @@ +using System.Collections; +using System.Collections.Generic; +using UnityEngine; + +namespace PupilLabs +{ + [CreateAssetMenu(fileName = "CalibrationSettings", menuName = "Pupil/CalibrationSettings", order = 1)] + public class CalibrationSettings : ScriptableObject + { + + [Header("Time and sample amount per target")] + public float secondsPerTarget = 1f; + public float ignoreInitialSeconds = 0.1f; + public int samplesPerTarget = 40; + + + public string PluginName { get { return "HMD3DChoreographyPlugin"; } } + public string PositionKey { get { return "mm_pos"; } } + public string DetectionMode { get { return "3d"; } } + + public float SampleRate + { + get + { + return (float)samplesPerTarget / (secondsPerTarget - ignoreInitialSeconds); + } + } + + } +} \ No newline at end of file diff --git a/Scripts/CalibrationStatusText.cs b/Scripts/CalibrationStatusText.cs new file mode 100644 index 0000000..d0bcf06 --- /dev/null +++ b/Scripts/CalibrationStatusText.cs @@ -0,0 +1,88 @@ +using System; +using System.Collections; +using UnityEngine; +using UnityEngine.UI; + +namespace PupilLabs +{ + [RequireComponent(typeof(CalibrationController))] + public class CalibrationStatusText : MonoBehaviour + { + public SubscriptionsController subsCtrl; + public Text statusText; + + private CalibrationController calibrationController; + + void Awake() + { + SetStatusText("Not connected"); + calibrationController = GetComponent(); + } + + void OnEnable() + { + subsCtrl.requestCtrl.OnConnected += OnConnected; + calibrationController.OnCalibrationStarted += OnCalibrationStarted; + calibrationController.OnCalibrationRoutineDone += OnCalibrationRoutineDone; + calibrationController.OnCalibrationSucceeded += CalibrationSucceeded; + calibrationController.OnCalibrationFailed += CalibrationFailed; + } + + void OnDisable() + { + subsCtrl.requestCtrl.OnConnected -= OnConnected; + calibrationController.OnCalibrationStarted -= OnCalibrationStarted; + calibrationController.OnCalibrationRoutineDone -= OnCalibrationRoutineDone; + calibrationController.OnCalibrationSucceeded -= CalibrationSucceeded; + calibrationController.OnCalibrationFailed -= CalibrationFailed; + } + + private void OnConnected() + { + string text = "Connected"; + text += "\n\nPlease warm up your eyes and press 'C' to start the calibration or 'P' to preview the calibration targets."; + SetStatusText(text); + } + + private void OnCalibrationStarted() + { + statusText.enabled = false; + } + + private void OnCalibrationRoutineDone() + { + statusText.enabled = true; + SetStatusText("Calibration routine is done. Waiting for results ..."); + } + + private void CalibrationSucceeded() + { + statusText.enabled = true; + SetStatusText("Calibration succeeded."); + + StartCoroutine(DisableTextAfter(1)); + } + + private void CalibrationFailed() + { + statusText.enabled = true; + SetStatusText("Calibration failed."); + + StartCoroutine(DisableTextAfter(1)); + } + + private void SetStatusText(string text) + { + if (statusText != null) + { + statusText.text = text; + } + } + + IEnumerator DisableTextAfter(float delay) + { + yield return new WaitForSeconds(delay); + statusText.enabled = false; + } + } +} diff --git a/Scripts/CalibrationTargets.cs b/Scripts/CalibrationTargets.cs new file mode 100644 index 0000000..5a5cdde --- /dev/null +++ b/Scripts/CalibrationTargets.cs @@ -0,0 +1,12 @@ +using System.Collections; +using System.Collections.Generic; +using UnityEngine; + +namespace PupilLabs +{ + public abstract class CalibrationTargets : ScriptableObject + { + public abstract int GetTargetCount(); + public abstract Vector3 GetLocalTargetPosAt(int idx); //unity camera space + } +} diff --git a/Scripts/CircleCalibrationTargets.cs b/Scripts/CircleCalibrationTargets.cs new file mode 100644 index 0000000..3213aa0 --- /dev/null +++ b/Scripts/CircleCalibrationTargets.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; +using UnityEngine; + +namespace PupilLabs +{ + + [CreateAssetMenu(fileName = "Circle Calibration Targets", menuName = "Pupil/CircleCalibrationTargets", order = 2)] + public class CircleCalibrationTargets : CalibrationTargets + { + [System.Serializable] + public struct Circle + { + public Vector3 center; + public float radius; + } + + public List circles = new List(); + [Tooltip("Points per circle")] public int points = 5; + + int pointIdx; + int circleIdx; + + public override int GetTargetCount() + { + return points * circles.Count; + } + + public override Vector3 GetLocalTargetPosAt(int idx) + { + pointIdx = (int)Mathf.Floor((float)idx / (float)circles.Count); + circleIdx = idx % circles.Count; + + return UpdateCalibrationPoint(); + } + + private Vector3 UpdateCalibrationPoint() + { + Circle circle = circles[circleIdx]; + Vector3 position = new Vector3(circle.center.x, circle.center.y, circle.center.z); + + if (pointIdx > 0 && pointIdx < points) + { + float angle = 360f * (float)(pointIdx - 1) / (points - 1f); + position.x += circle.radius * Mathf.Cos(Mathf.Deg2Rad * angle); + position.y += circle.radius * Mathf.Sin(Mathf.Deg2Rad * angle); + } + + return position; + } + } +} \ No newline at end of file diff --git a/Scripts/Editor/RequestEditor.cs b/Scripts/Editor/RequestEditor.cs new file mode 100644 index 0000000..5bd678a --- /dev/null +++ b/Scripts/Editor/RequestEditor.cs @@ -0,0 +1,63 @@ +using UnityEngine; +using UnityEditor; + +using PupilLabs; + +[CustomEditor(typeof(RequestController))] +public class RequestEditor : Editor +{ + private SerializedProperty ipProp; + private SerializedProperty portProp; + private SerializedProperty versionProp; + private SerializedProperty isConnectingProb; + + public void OnEnable() + { + SerializedProperty requestProp = serializedObject.FindProperty("request"); + ipProp = requestProp.FindPropertyRelative("IP"); + portProp = requestProp.FindPropertyRelative("PORT"); + + isConnectingProb = serializedObject.FindProperty("isConnecting"); + versionProp = serializedObject.FindProperty("PupilVersion"); + } + + public override void OnInspectorGUI() + { + serializedObject.Update(); + RequestController ctrl = serializedObject.targetObject as RequestController; + + DrawDefaultInspector(); + + // request + EditorGUILayout.Space(); + EditorGUILayout.LabelField("Connection", EditorStyles.boldLabel); + EditorGUILayout.PropertyField(ipProp,new GUIContent("IP")); + EditorGUILayout.PropertyField(portProp,new GUIContent("PORT")); + EditorGUILayout.LabelField("Pupil Version",versionProp.stringValue); + + GUILayout.BeginHorizontal(); + + string connectLabel = "Connect"; + GUI.enabled = !ctrl.IsConnected && Application.isPlaying; + if (isConnectingProb.boolValue) + { + connectLabel = "Connecting ..."; + GUI.enabled = false; + } + if (GUILayout.Button(connectLabel)) + { + ctrl.RunConnect(); + } + + GUI.enabled = ctrl.IsConnected; + if (GUILayout.Button("Disconnect")) + { + ctrl.Disconnect(); + } + + GUI.enabled = true; + GUILayout.EndHorizontal(); + + serializedObject.ApplyModifiedProperties(); + } +} diff --git a/Scripts/FrameListener.cs b/Scripts/FrameListener.cs new file mode 100644 index 0000000..ebb279b --- /dev/null +++ b/Scripts/FrameListener.cs @@ -0,0 +1,63 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using UnityEngine; + +namespace PupilLabs +{ + public class FrameListener : BaseListener + { + public event Action OnReceiveEyeFrame; + + public FrameListener(SubscriptionsController subsCtrl) : base(subsCtrl) { } + + protected override void CustomEnable() + { + Debug.Log("Enabling Frame Listener"); + + subsCtrl.SubscribeTo("frame.eye.", CustomReceiveData); + subsCtrl.requestCtrl.Send(new Dictionary { + {"subject", "frame_publishing.set_format"}, + {"format", "jpeg"} + }); + } + + protected override void CustomDisable() + { + Debug.Log("Disabling Frame Listener"); + + subsCtrl.UnsubscribeFrom("frame.eye.", CustomReceiveData); + } + + void CustomReceiveData(string topic, Dictionary dictionary, byte[] thirdFrame = null) + { + if (thirdFrame == null) + { + Debug.LogError("Received Frame Data without any image data"); + return; + } + + int eyeIdx; + if (topic == "frame.eye.0") + { + eyeIdx = 0; + } + else if (topic == "frame.eye.1") + { + eyeIdx = 1; + } + else + { + Debug.LogError($"{topic} isn't matching"); + return; + } + + if (OnReceiveEyeFrame != null) + { + OnReceiveEyeFrame(eyeIdx, thirdFrame); + } + } + } +} + + diff --git a/Scripts/FrameVisualizer.cs b/Scripts/FrameVisualizer.cs new file mode 100644 index 0000000..a8d32fb --- /dev/null +++ b/Scripts/FrameVisualizer.cs @@ -0,0 +1,147 @@ +using System.Collections; +using System.Collections.Generic; +using UnityEngine; + +namespace PupilLabs +{ + public class FrameVisualizer : MonoBehaviour + { + public SubscriptionsController subscriptionsController; + public Transform cameraAsParent; + public Material eyeFrameMaterial; + [Header("Settings")] + [Tooltip("Used for adjusting the Eye Frames. Changes require restarting play mode!")] + public bool model200Hz; + + public int targetFPS = 20; + + public FrameListener Listener { get; private set; } = null; + + Texture2D[] eyeTexture = new Texture2D[2]; + byte[][] eyeImageRaw = new byte[2][]; + MeshRenderer[] eyeRenderer = new MeshRenderer[2]; + bool[] eyePublishingInitialized = new bool[2]; + + void OnEnable() + { + bool allReferencesValid = true; + if (cameraAsParent == null) + { + Debug.LogError("Camera reference 'cameraAsParent' missing!"); + allReferencesValid = false; + } + if (subscriptionsController == null) + { + Debug.LogError("SubscriptionsController reference missing!"); + allReferencesValid = false; + } + if (eyeFrameMaterial == null) + { + Debug.LogError("EyeFrameMaterial reference missing!"); + allReferencesValid = false; + } + if (!allReferencesValid) + { + Debug.LogError("FrameVisualizer is missing required references to other components. Please connect the references, or the component won't work correctly."); + enabled = false; + return; + } + + if (Listener == null) + { + Listener = new FrameListener(subscriptionsController); + } + + Debug.Log("Enabling Frame Visualizer"); + + Listener.Enable(); + Listener.OnReceiveEyeFrame += ReceiveEyeFrame; + + eyePublishingInitialized = new bool[] { false, false }; + } + + void ReceiveEyeFrame(int eyeIdx, byte[] frameData) + { + if (!eyePublishingInitialized[eyeIdx]) + { + InitializeFramePublishing(eyeIdx); + } + eyeImageRaw[eyeIdx] = frameData; + } + + void InitializeFramePublishing(int eyeIndex) + { + Transform parent = cameraAsParent; + LayerMask layer = 21; + + eyeTexture[eyeIndex] = new Texture2D(100, 100); + eyeRenderer[eyeIndex] = InitializeEyeObject(eyeIndex, parent, layer); + eyeRenderer[eyeIndex].material = new Material(eyeFrameMaterial); + eyeRenderer[eyeIndex].material.mainTexture = eyeTexture[eyeIndex]; + Vector2 textureScale; + + if (eyeIndex == 0) //right by default + { + textureScale = model200Hz ? new Vector2(1, -1) : new Vector2(-1, 1); + } + else //index == 1 -> left by default + { + textureScale = model200Hz ? new Vector2(-1, 1) : new Vector2(1, -1); + } + + eyeRenderer[eyeIndex].material.mainTextureScale = textureScale; + + lastUpdate = Time.time; + + eyePublishingInitialized[eyeIndex] = true; + } + + MeshRenderer InitializeEyeObject(int eyeIndex, Transform parent, int LayerMask) + { + GameObject go = GameObject.CreatePrimitive(PrimitiveType.Plane); + go.name = "Eye " + eyeIndex.ToString(); + go.layer = LayerMask; + go.transform.parent = parent; + go.transform.localEulerAngles = Vector3.left * 90; + go.transform.localScale = Vector3.one * 0.05f; + go.transform.localPosition = new Vector3((eyeIndex == 1 ? -0.3f : 0.3f), -0.5f, 1.9999f); + + Destroy(go.GetComponent()); + + return go.GetComponent(); + } + + float lastUpdate; + void Update() + { + //Limiting the MainThread calls to framePublishFramePerSecondLimit to avoid issues. 20-30 ideal. + if ((Time.time - lastUpdate) >= (1f / targetFPS)) + { + for (int i = 0; i < 2; i++) + if (eyePublishingInitialized[i]) + eyeTexture[i].LoadImage(eyeImageRaw[i]); + lastUpdate = Time.time; + } + } + + void OnDisable() + { + Debug.Log("Disabling Frame Visualizer"); + + if (Listener != null) + { + Listener.OnReceiveEyeFrame -= ReceiveEyeFrame; + Listener.Disable(); + } + + for (int i = eyeRenderer.Length - 1; i >= 0; i--) + { + if (eyeRenderer[i] != null && eyeRenderer[i].gameObject != null) + { + Destroy(eyeRenderer[i].gameObject); + } + } + } + } +} + diff --git a/Scripts/GazeController.cs b/Scripts/GazeController.cs new file mode 100644 index 0000000..6db714d --- /dev/null +++ b/Scripts/GazeController.cs @@ -0,0 +1,48 @@ +using System; +using UnityEngine; + +namespace PupilLabs +{ + public class GazeController : MonoBehaviour + { + public SubscriptionsController subscriptionsController; + + public event Action OnReceive3dGaze; + + GazeListener listener; + + void OnEnable() + { + if (subscriptionsController == null) + { + Debug.LogError("GazeController is missing the required SubscriptionsController reference. Please connect the reference, or the component won't work correctly."); + enabled = false; + return; + } + + if (listener == null) + { + listener = new GazeListener(subscriptionsController); + listener.OnReceive3dGaze += Forward3dGaze; + } + + listener.Enable(); + } + + void OnDisable() + { + if (listener != null) + { + listener.Disable(); + } + } + + void Forward3dGaze(GazeData data) + { + if (OnReceive3dGaze != null) + { + OnReceive3dGaze(data); + } + } + } +} diff --git a/Scripts/GazeData.cs b/Scripts/GazeData.cs new file mode 100644 index 0000000..2c9e69e --- /dev/null +++ b/Scripts/GazeData.cs @@ -0,0 +1,241 @@ +using System.Collections; +using System.Collections.Generic; +using UnityEngine; + +namespace PupilLabs +{ + public class GazeData + { + + public enum GazeMappingContext + { + Monocular_0, + Monocular_1, + Binocular + } + + /// + /// MappingContext refers to GazeData being based on binocular or monocular mapping. + /// It also indicates the availability of EyeCenter/GazeNormal: + /// Binocular for both eyes or Monocular for the corresponding eye. + /// + public GazeMappingContext MappingContext { get; private set; } + + /// + /// Confidence of the pupil detection and 3d representation of the eye(s). + /// Used to filter data sets with low confidence (below ~0.6). + /// + public float Confidence { get; private set; } + /// + /// Pupil time in seconds. + /// + public double PupilTimestamp { get; private set; } + + /// + /// Gaze direction corresponding to the 3d gaze point. + /// Normalized vector in local camera space. + /// + public Vector3 GazeDirection { get; private set; } + /// + /// Distance in meters between VR camera and 3d gaze point. + /// + public float GazeDistance { get; private set; } + + /// + /// 3d gaze point in local camera space. + /// Recommended to use equivalent representation as GazeDirection plus GazeDistance, + /// as this clearly sperates the angular error from the depth error. + /// + [System.Obsolete("Using the data field GazePoint3d is not recommended. Use GazeDirection and GazeDistance instead.")] + public Vector3 GazePoint3d { get { return gazePoint3d; } } + + /// + /// Backprojection into viewport, based on camera intrinsics set in Pupil Capture. + /// Not available with Pupil Service. + /// + public Vector2 NormPos { get; private set; } + + /// + /// 3d coordinate of eye center 0 in local camera space. By default eye index 0 corresponds to the right eye. + /// + public Vector3 EyeCenter0 { get { return CheckAvailability(0) ? eyeCenter0 : Vector3.zero; } } + /// + /// 3d coordinate of eye center 1 in local camera space. By default eye index 1 corresponds to the left eye. + /// + public Vector3 EyeCenter1 { get { return CheckAvailability(1) ? eyeCenter1 : Vector3.zero; } } + + /// + /// Gaze vector of eye 0 in local camera space. By default eye index 0 corresponds to the right eye. + /// + public Vector3 GazeNormal0 { get { return CheckAvailability(0) ? gazeNormal0 : Vector3.zero; } } + /// + /// Gaze vector of eye 1 in local camera space. By default eye index 1 corresponds to the left eye. + /// + public Vector3 GazeNormal1 { get { return CheckAvailability(1) ? gazeNormal1 : Vector3.zero; } } + + private Vector3 gazePoint3d; + private Vector3 eyeCenter0, eyeCenter1; + private Vector3 gazeNormal0, gazeNormal1; + + public GazeData(string topic, Dictionary dictionary) + { + Parse(topic, dictionary); + } + + /// + /// Check availability of EyeCenter/GazeNormal for corresponding eye. + /// + public bool IsEyeDataAvailable(int eyeIdx) + { + return MappingContext == (GazeMappingContext)eyeIdx || MappingContext == GazeMappingContext.Binocular; + } + + /// + /// Parameterized version of EyeCenter0/1 + /// + public Vector3 GetEyeCenter(int eyeIdx) + { + if (eyeIdx == 0) + { + return EyeCenter0; + } + else if (eyeIdx == 1) + { + return EyeCenter1; + } + else + { + Debug.LogWarning($"EyeIdx of {eyeIdx} not supported. Valid options: 0 or 1"); + return Vector3.zero; + } + } + + /// + /// Parameterized version of GazeNormal0/1 + /// + public Vector3 GetGazeNormal(int eyeIdx) + { + if (eyeIdx == 0) + { + return GazeNormal0; + } + else if (eyeIdx == 1) + { + return GazeNormal1; + } + else + { + Debug.LogWarning($"EyeIdx of {eyeIdx} not supported. Valid options: 0 or 1"); + return Vector3.zero; + } + } + + private void Parse(string topic, Dictionary dictionary) + { + if (topic == "gaze.3d.01.") + { + MappingContext = GazeMappingContext.Binocular; + } + else if (topic == "gaze.3d.0.") + { + MappingContext = GazeMappingContext.Monocular_0; + } + else if (topic == "gaze.3d.1.") + { + MappingContext = GazeMappingContext.Monocular_1; + } + else + { + Debug.LogError("GazeData with no matching mode"); + return; + } + + Confidence = Helpers.FloatFromDictionary(dictionary, "confidence"); + PupilTimestamp = Helpers.DoubleFromDictionary(dictionary, "timestamp"); + + if (dictionary.ContainsKey("norm_pos")) + { + NormPos = Helpers.Position(dictionary["norm_pos"], false); + } + + gazePoint3d = ExtractAndParseGazePoint(dictionary); + GazeDirection = gazePoint3d.normalized; + GazeDistance = gazePoint3d.magnitude; + + if (MappingContext == GazeMappingContext.Binocular || MappingContext == GazeMappingContext.Monocular_0) + { + eyeCenter0 = ExtractEyeCenter(dictionary, MappingContext, 0); + gazeNormal0 = ExtractGazeNormal(dictionary, MappingContext, 0); + } + if (MappingContext == GazeMappingContext.Binocular || MappingContext == GazeMappingContext.Monocular_1) + { + eyeCenter1 = ExtractEyeCenter(dictionary, MappingContext, 1); + gazeNormal1 = ExtractGazeNormal(dictionary, MappingContext, 1); + } + } + + private Vector3 ExtractAndParseGazePoint(Dictionary dictionary) + { + Vector3 gazePos = Helpers.Position(dictionary["gaze_point_3d"], true); + gazePos.y *= -1f; // Pupil y axis is inverted + + // correct/flip pos if behind viewer + float angle = Vector3.Angle(Vector3.forward, gazePos); + if (angle >= 90f) + { + gazePos *= -1f; + } + + return gazePos; + } + + private Vector3 ExtractEyeCenter(Dictionary dictionary, GazeMappingContext context, byte eye) + { + + object vecObj; + if (context == GazeMappingContext.Binocular) + { + var binoDic = dictionary["eye_centers_3d"] as Dictionary; + // Starting with Pupil 3.0, all keys are strings + vecObj = binoDic.ContainsKey(eye) ? binoDic[eye] : binoDic[eye.ToString()]; + } + else + { + vecObj = dictionary["eye_center_3d"]; + } + Vector3 eyeCenter = Helpers.Position(vecObj, true); + eyeCenter.y *= -1; + return eyeCenter; + } + + private Vector3 ExtractGazeNormal(Dictionary dictionary, GazeMappingContext context, byte eye) + { + + object vecObj; + if (context == GazeMappingContext.Binocular) + { + var binoDic = dictionary["gaze_normals_3d"] as Dictionary; + // Starting with Pupil 3.0, all keys are strings + vecObj = binoDic.ContainsKey(eye) ? binoDic[eye] : binoDic[eye.ToString()]; + } + else + { + vecObj = dictionary["gaze_normal_3d"]; + } + Vector3 gazeNormal = Helpers.Position(vecObj, false); + gazeNormal.y *= -1f; + return gazeNormal; + } + + private bool CheckAvailability(int eyeIdx) + { + if (!IsEyeDataAvailable(eyeIdx)) + { + Debug.LogWarning("Data not available in this GazeData set. Please check GazeData.IsEyeDataAvailable or GazeData.MappingContext first."); + return false; + } + + return true; + } + } +} \ No newline at end of file diff --git a/Scripts/GazeListener.cs b/Scripts/GazeListener.cs new file mode 100644 index 0000000..ac30809 --- /dev/null +++ b/Scripts/GazeListener.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using UnityEngine; + +namespace PupilLabs +{ + public class GazeListener : BaseListener + { + public event Action OnReceive3dGaze; + + public GazeListener(SubscriptionsController subsCtrl) : base(subsCtrl) { } + + protected override void CustomEnable() + { + Debug.Log("Enabling Gaze Listener"); + subsCtrl.SubscribeTo("gaze.3d", Receive3DGaze); + } + + protected override void CustomDisable() + { + Debug.Log("Disabling Gaze Listener"); + subsCtrl.UnsubscribeFrom("gaze.3d", Receive3DGaze); + } + + void Receive3DGaze(string topic, Dictionary dictionary, byte[] thirdFrame = null) + { + GazeData gazeData = new GazeData(topic, dictionary); + + if (OnReceive3dGaze != null) + { + OnReceive3dGaze(gazeData); + } + } + } +} diff --git a/Scripts/GazeVisualizer.cs b/Scripts/GazeVisualizer.cs new file mode 100644 index 0000000..66421cc --- /dev/null +++ b/Scripts/GazeVisualizer.cs @@ -0,0 +1,313 @@ +using System.Collections; +using System.Collections.Generic; +using UnityEngine; +using ObjBehaviour; +using System.IO; +using Proyecto26; +using System; +using Firebase; +using Firebase.Database; + +namespace PupilLabs +{ // Create the class for recording the gazepoint in 2D plane. + [Serializable] + public class GazePoint2D + { + public float GazePoint2DX; + public float GazePoint2DY; + public float GazePoint2DZ; + + public GazePoint2D(float gazePoint2DX, float gazePoint2DY, float gazePoint2DZ) + { + this.GazePoint2DX = gazePoint2DX; + this.GazePoint2DY = gazePoint2DY; + this.GazePoint2DZ = gazePoint2DZ; + } + } + + public class GazeVisualizer : MonoBehaviour + { + public Transform gazeOrigin; + public GazeController gazeController; + + [Header("HeatMapOutputFilePath")] + public string FilePath; + + [Header("Settings")] + [Range(0f, 1f)] + public float confidenceThreshold = 0.6f; + public bool binocularOnly = true; + + [Header("Projected Visualization")] + public Transform projectionMarker; + public Transform gazeDirectionMarker; + [Range(0.01f, 0.1f)] + public float sphereCastRadius = 0.1f; + public bool isMarkerProjected = false; + + private int cubeLayerMask; + private int heatMapLayerMask; + + Vector3 localGazeDirection; + float gazeDistance; + bool isGazing = false; + + bool errorAngleBasedMarkerRadius = true; + float angleErrorEstimate = 2f; + + Vector3 origMarkerScale; + MeshRenderer targetRenderer; + GameObject[] allCubes; + float minAlpha = 0.2f; + float maxAlpha = 0.8f; + + float lastConfidence; + + private int fileUploadTime = 0; + private const string projectId = "unitytesting-5ab13-default-rtdb"; // Can find this in the Firebase project settings + private static readonly string databaseURL = $"https://unitytesting-5ab13-default-rtdb.asia-southeast1.firebasedatabase.app/"; + + + private void Start() + { + allCubes = GameObject.FindGameObjectsWithTag("Cube"); + + } + + void OnEnable() + { + bool allReferencesValid = true; + if (projectionMarker == null) + { + Debug.LogError("ProjectionMarker reference missing!"); + allReferencesValid = false; + } + if (gazeDirectionMarker == null) + { + Debug.LogError("GazeDirectionMarker reference missing!"); + allReferencesValid = false; + } + if (gazeOrigin == null) + { + Debug.LogError("GazeOrigin reference missing!"); + allReferencesValid = false; + } + if (gazeController == null) + { + Debug.LogError("GazeController reference missing!"); + allReferencesValid = false; + } + if (!allReferencesValid) + { + Debug.LogError("GazeVisualizer is missing required references to other components. Please connect the references, or the component won't work correctly."); + enabled = false; + return; + } + + origMarkerScale = gazeDirectionMarker.localScale; + targetRenderer = gazeDirectionMarker.GetComponent(); + + StartVisualizing(); + + // Hitpoint Output + TextWriter tw = new StreamWriter(FilePath, false); //write the cube position into .CSV file + tw.WriteLine("TimeStamp" + "," + "HitPointX" + "," + "HitPointY" + "," + "HitPointZ"); + tw.Close(); + + } + + void OnDisable() + { + if (gazeDirectionMarker != null) + { + gazeDirectionMarker.localScale = origMarkerScale; + } + + StopVisualizing(); + } + + void Update() + { + + if (!isGazing) + { + return; + } + + VisualizeConfidence(); + + ShowProjected(); + if (RecordingController.IsRecording) + { + HeatMapData(); + } + + } + + public void StartVisualizing() + { + if (!enabled) + { + Debug.LogWarning("Component not enabled."); + return; + } + + if (isGazing) + { + Debug.Log("Already gazing!"); + return; + } + + Debug.Log("Start Visualizing Gaze"); + + gazeController.OnReceive3dGaze += ReceiveGaze; + + projectionMarker.gameObject.SetActive(isMarkerProjected); + gazeDirectionMarker.gameObject.SetActive(isMarkerProjected); + isGazing = true; + } + + public void StopVisualizing() + { + if (!isGazing || !enabled) + { + Debug.Log("Nothing to stop."); + return; + } + + if (projectionMarker != null) + { + projectionMarker.gameObject.SetActive(false); + } + if (gazeDirectionMarker != null) + { + gazeDirectionMarker.gameObject.SetActive(false); + } + + isGazing = false; + + gazeController.OnReceive3dGaze -= ReceiveGaze; + } + + void ReceiveGaze(GazeData gazeData) + { + if (binocularOnly && gazeData.MappingContext != GazeData.GazeMappingContext.Binocular) + { + return; + } + + lastConfidence = gazeData.Confidence; + + if (gazeData.Confidence < confidenceThreshold) + { + return; + } + + localGazeDirection = gazeData.GazeDirection; // *Gaze direction corresponding to the 3d gaze point. Normalized vector in local camera space. + gazeDistance = gazeData.GazeDistance; // *Distance in meters between VR camera and the 3d gaze point. + } + + void VisualizeConfidence() + { + if (targetRenderer != null) + { + Color c = targetRenderer.material.color; + c.a = MapConfidence(lastConfidence); + targetRenderer.material.color = c; + } + } + + void ShowProjected() + { + gazeDirectionMarker.localScale = origMarkerScale; + + Vector3 origin = gazeOrigin.position; + Vector3 direction = gazeOrigin.TransformDirection(localGazeDirection); + + + cubeLayerMask = (1 << 8) | (1 << 9) | (1 << 10) | (1 << 11) | (1 << 12) | (1 << 13)| (1 << 14) | (1 << 15) | (1 << 16) | (1 << 17); + if (Physics.SphereCast(origin, sphereCastRadius, direction, out RaycastHit hit, Mathf.Infinity, cubeLayerMask)) + { + Debug.DrawRay(origin, direction * hit.distance, Color.yellow); + + projectionMarker.position = hit.point; + + gazeDirectionMarker.position = origin + direction * hit.distance; + gazeDirectionMarker.LookAt(origin); + + if (errorAngleBasedMarkerRadius) + { + gazeDirectionMarker.localScale = GetErrorAngleBasedScale(origMarkerScale, hit.distance, angleErrorEstimate); + } + // Enable the script when gazing the target cubes + TrackingTarget scriptTarget = hit.collider.gameObject.GetComponent(); + scriptTarget.enabled = true; + } + else + { + Debug.DrawRay(origin, direction * 10, Color.white); + + foreach (GameObject cube in allCubes) + { + TrackingTarget scriptTarget = cube.GetComponent(); + scriptTarget.enabled = false; + } + } + } + + // Record hitpoint of the raycast into .CSV for heatmap data + void HeatMapData() + { + Vector3 origin = gazeOrigin.position; + Vector3 direction = gazeOrigin.TransformDirection(localGazeDirection); + Vector3 hitPoint; + heatMapLayerMask = (1 << 22); + if (Physics.SphereCast(origin, sphereCastRadius, direction, out RaycastHit hit, Mathf.Infinity, heatMapLayerMask)) + { + hitPoint = origin + direction * hit.distance; + hitPoint = gazeOrigin.InverseTransformPoint(hitPoint); + + CSVWriter(hitPoint.x, hitPoint.y, hitPoint.z); + + GazePoint2D gazePoint2D = new GazePoint2D(hitPoint.x, hitPoint.y, hitPoint.z); + if ((int)Time.realtimeSinceStartup != fileUploadTime) + { + FireBaseUpload(gazePoint2D, ((int)Time.realtimeSinceStartup).ToString()); + fileUploadTime = (int)Time.realtimeSinceStartup; + } + + } + } + + public static void FireBaseUpload(GazePoint2D gazePoint2D, string timeStamp) + { + + RestClient.Put($"{databaseURL}GazePoints/{timeStamp}.json", gazePoint2D).Then(response => { + Debug.Log("The user was successfully uploaded to the database"); + }); + + } + + void CSVWriter(float x, float y, float z) + { + TextWriter tw = new StreamWriter(FilePath, true); + tw.WriteLine(Time.realtimeSinceStartup + "," + x + "," + y + "," + z); + tw.Close(); + } + + + Vector3 GetErrorAngleBasedScale(Vector3 origScale, float distance, float errorAngle) + { + Vector3 scale = origScale; + float scaleXY = distance * Mathf.Tan(Mathf.Deg2Rad * angleErrorEstimate) * 2; + scale.x = scaleXY; + scale.y = scaleXY; + return scale; + } + + float MapConfidence(float confidence) + { + return Mathf.Lerp(minAlpha, maxAlpha, confidence); + } + } +} \ No newline at end of file diff --git a/Scripts/Helpers.cs b/Scripts/Helpers.cs new file mode 100644 index 0000000..3136102 --- /dev/null +++ b/Scripts/Helpers.cs @@ -0,0 +1,108 @@ +using System; +using System.Collections.Generic; +using UnityEngine; + +namespace PupilLabs +{ + public class Helpers + { + public static float PupilUnitScalingFactor = 1000; // Pupil is currently operating in mm + public const string leftEyeID = "1"; + public const string rightEyeID = "0"; + + private static object[] position_o; + public static Vector3 ObjectToVector(object source) + { + position_o = source as object[]; + Vector3 result = Vector3.zero; + if (position_o.Length != 2 && position_o.Length != 3) + Debug.Log("Array length not supported"); + else + { + result.x = (float)(double)position_o[0]; + result.y = (float)(double)position_o[1]; + if (position_o.Length == 3) + result.z = (float)(double)position_o[2]; + } + return result; + } + public static Vector3 Position(object position, bool applyScaling) + { + Vector3 result = ObjectToVector(position); + if (applyScaling) + result /= PupilUnitScalingFactor; + return result; + } + + public static Vector3 VectorFromDictionary(Dictionary source, string key) + { + if (source.ContainsKey(key)) + return Position(source[key], false); + else + return Vector3.zero; + } + + public static int IntFromDictionary(Dictionary source, string key) + { + source.TryGetValue(key, out object value_o); + return (int)value_o; + } + + public static float FloatFromDictionary(Dictionary source, string key) + { + return (float)DoubleFromDictionary(source, key); + } + + public static double DoubleFromDictionary(Dictionary source, string key) + { + object value_o; + source.TryGetValue(key, out value_o); + return (double)value_o; + } + + public static double TryCastToDouble(object obj) + { + Double? d = obj as Double?; + if (d.HasValue) + { + return d.Value; + } + else + { + return 0f; + } + } + + private static object IDo; + public static string StringFromDictionary(Dictionary source, string key) + { + string result = ""; + if (source.TryGetValue(key, out IDo)) + result = IDo.ToString(); + return result; + } + + public static Dictionary DictionaryFromDictionary(Dictionary source, string key) + { + if (source.ContainsKey(key)) + return source[key] as Dictionary; + else + return null; + } + + public static string TopicsForDictionary(Dictionary dictionary) + { + string topics = ""; + foreach (var key in dictionary.Keys) + topics += key + ","; + return topics; + } + + public static Dictionary BaseData(Dictionary dictionary) + { + object o; + dictionary.TryGetValue("base_data", out o); + return o as Dictionary; + } + } +} \ No newline at end of file diff --git a/Scripts/NetMQCleanup.cs b/Scripts/NetMQCleanup.cs new file mode 100644 index 0000000..35e54fe --- /dev/null +++ b/Scripts/NetMQCleanup.cs @@ -0,0 +1,32 @@ +using NetMQ; +using System.Collections.Generic; +using UnityEngine; + +namespace PupilLabs +{ + public static class NetMQCleanup + { + private static List connections = new List(); + + public static void MonitorConnection(RequestController connection) + { + connections.Add(connection); + } + + public static void CleanupConnection(RequestController connection) + { + connections.Remove(connection); + + if (connections.Count == 0) + { + Debug.Log("Terminate Context."); + NetMQConfig.Cleanup(block: false); + } + } + + // static NetMQCleanup() + // { + // Application.quitting += () => {NetMQConfig.Cleanup(block: false);}; + // } + } +} \ No newline at end of file diff --git a/Scripts/Publisher.cs b/Scripts/Publisher.cs new file mode 100644 index 0000000..0a3a41a --- /dev/null +++ b/Scripts/Publisher.cs @@ -0,0 +1,74 @@ +using System.Collections.Generic; +using NetMQ; +using NetMQ.Sockets; +using MessagePack; + +namespace PupilLabs +{ + public class Publisher + { + private RequestController requestController; + private PublisherSocket publisherSocket; + private bool isSetup = false; + private bool waitingOnConnection = false; + + public Publisher(RequestController requestController) + { + this.requestController = requestController; + + if (requestController.IsConnected) + { + Setup(); + } + else + { + waitingOnConnection = true; + requestController.OnConnected += DelayedSetup; + } + } + + public void Destroy() + { + if (waitingOnConnection) + { + requestController.OnConnected -= DelayedSetup; + } + + if (isSetup) + { + if (publisherSocket != null) + { + publisherSocket.Close(); + } + } + } + + public void Send(string topic, Dictionary data, byte[] thirdFrame = null) + { + NetMQMessage m = new NetMQMessage(); + + m.Append(topic); + m.Append(MessagePackSerializer.Serialize>(data)); + + if (thirdFrame != null) + { + m.Append(thirdFrame); + } + + publisherSocket.SendMultipartMessage(m); + } + + private void DelayedSetup() + { + waitingOnConnection = false; + requestController.OnConnected -= DelayedSetup; + Setup(); + } + + private void Setup() + { + publisherSocket = new PublisherSocket(requestController.GetPubConnectionString()); + isSetup = true; + } + } +} diff --git a/Scripts/PupilData.cs b/Scripts/PupilData.cs new file mode 100644 index 0000000..8ccea92 --- /dev/null +++ b/Scripts/PupilData.cs @@ -0,0 +1,216 @@ +using System.Collections; +using System.Collections.Generic; +using UnityEngine; + +namespace PupilLabs +{ + public class PupilData + { + /// + /// Eye camera index (0/1 for right/left eye). + /// + public int EyeIdx { get; private set; } + /// + /// Confidence is an assessment by the pupil detector on how sure we can be on this measurement. + /// A value of 0 indicates no confidence. 1 indicates perfect confidence. + /// In our experience useful data carries a confidence value greater than ~0.6. + /// + public float Confidence { get; private set; } + /// + /// Indicates what detector was used to detect the pupil. + /// + public string Method { get; private set; } + + /// + /// Pupil time in seconds. + /// + public double PupilTimestamp { get; private set; } + + /// + /// Position in the eye image frame in normalized coordinates. + /// + public Vector2 NormPos { get; private set; } + + /// + /// Diameter of the pupil in image pixels as observed in the eye image frame (is not corrected for perspective) + /// + public float Diameter { get; private set; } + + /// + /// Pupil ellipse in eye camera image coordinate system. + /// + public class PupilEllipse + { + public Vector2 Center { get; set; } // Center of the pupil in image pixels. + public Vector2 Axis { get; set; } // First and second axis of the pupil ellipse in pixels. + public float Angle { get; set; } // Angle of the ellipse in degrees. + } + public PupilEllipse Ellipse { get; private set; } = new PupilEllipse(); + + /// + /// Confidence of the current eye model (0-1) + /// + public float ModelConfidence { get; private set; } + /// + /// Id of the current eye model. + /// When a slippage is detected the model is replaced and the id changes. + /// + public string ModelId { get; private set; } + /// + /// Model birth timestamp in pupil time in seconds. + /// + public double ModelBirthTimestamp { get; private set; } + + /// + /// Diameter of the pupil scaled to mm based on anthropomorphic avg eye ball diameter and corrected for perspective. + /// + public float Diameter3d { get; private set; } //- + + /// + /// Eyeball sphere in eye camera 3d space (units in mm). + /// + public class EyeSphere + { + public Vector3 Center { get; set; } // Pos of the eyeball sphere in eye camera 3d space in mm. + public float Radius { get; set; } // Radius of the eyeball. This is always 12mm (the anthropomorphic avg.) + } + public EyeSphere Sphere { get; private set; } = new EyeSphere(); + + /// + /// Pupil circle in eye camera 3d space (units in mm). + /// + public class PupilCircle + { + public Vector3 Center { get; set; } // Center of the pupil as 3d circle in eye pinhole camera 3d space units are mm. + public Vector3 Normal { get; set; } // Normals of the pupil as 3d circle. Indicates the direction that the pupil points at in 3d space. + public float Radius { get; set; } // Radius of the pupil as 3d circle. Same as diameter_3d. + public float Theta { get; set; } // Normal described in spherical coordinates in radians. + public float Phi { get; set; } // Normal described in spherical coordinates in radians. + } + public PupilCircle Circle { get; private set; } = new PupilCircle(); + + /// + /// Eyeball sphere projected back onto the eye camera image (units in pixel). + /// + public class ProjectedEyeSphere + { + public Vector2 Center { get; set; } // Center of the 3d sphere projected back onto the eye image frame. Units are in image pixels. + public Vector2 Axis { get; set; } // First and second axis of the 3d sphere projection. + public float Angle { get; set; } // Angle of the 3d sphere projection. Units are degrees. + } + public ProjectedEyeSphere ProjectedSphere { get; private set; } = new ProjectedEyeSphere(); + + public PupilData(Dictionary dictionary) + { + ParseDictionary(dictionary); + } + + void ParseDictionary(Dictionary dictionary) + { + EyeIdx = System.Int32.Parse(Helpers.StringFromDictionary(dictionary, "id")); + Confidence = Helpers.FloatFromDictionary(dictionary, "confidence"); + Method = Helpers.StringFromDictionary(dictionary, "method"); + + PupilTimestamp = Helpers.DoubleFromDictionary(dictionary, "timestamp"); + + NormPos = Helpers.ObjectToVector(dictionary["norm_pos"]); + Diameter = Helpers.FloatFromDictionary(dictionary, "diameter"); + + //+2d + if (Method.Contains("2d") || Method.Contains("3d")) + { + TryExtractEllipse(dictionary); + } + + //+3d + if (Method.Contains("3d")) + { + // Starting with Pupil 3.0, pye3d is used as 3d pupil detector. Its generated data does no + // longer include the keys model_id and model_birth_timestamp. We handle the missing data + // by filling a default value. + ModelId = dictionary.ContainsKey("model_id") ? + Helpers.StringFromDictionary(dictionary, "model_id") : "0"; + ModelBirthTimestamp = dictionary.ContainsKey("model_birth_timestamp") ? + Helpers.DoubleFromDictionary(dictionary, "model_birth_timestamp") : 0.0; + + ModelConfidence = Helpers.FloatFromDictionary(dictionary, "model_confidence"); + Diameter3d = Helpers.FloatFromDictionary(dictionary, "diameter_3d"); + + TryExtractCircle3d(dictionary); + ExtractSphericalCoordinates(dictionary); + + TryExtractSphere(dictionary); + TryExtractProjectedSphere(dictionary); + } + } + + bool TryExtractEllipse(Dictionary dictionary) + { + Dictionary subDic = Helpers.DictionaryFromDictionary(dictionary, "ellipse"); + if (subDic == null) + { + return false; + } + + Ellipse.Center = Helpers.ObjectToVector(subDic["center"]); + Ellipse.Axis = Helpers.ObjectToVector(subDic["axes"]); + Ellipse.Angle = (float)(double)subDic["angle"]; + + return true; + } + + bool TryExtractCircle3d(Dictionary dictionary) + { + Dictionary subDic = Helpers.DictionaryFromDictionary(dictionary, "circle_3d"); + + if (subDic == null) + { + return false; + } + + Circle.Center = Helpers.ObjectToVector(subDic["center"]); + Circle.Normal = Helpers.ObjectToVector(subDic["normal"]); + Circle.Radius = (float)(double)subDic["radius"]; + + return true; + } + + bool TryExtractSphere(Dictionary dictionary) + { + Dictionary subDic = Helpers.DictionaryFromDictionary(dictionary, "sphere"); + + if (subDic == null) + { + return false; + } + + Sphere.Center = Helpers.ObjectToVector(subDic["center"]); + Sphere.Radius = (float)(double)subDic["radius"]; + + return true; + } + + bool TryExtractProjectedSphere(Dictionary dictionary) + { + Dictionary subDic = Helpers.DictionaryFromDictionary(dictionary, "projected_sphere"); + + if (subDic == null) + { + return false; + } + + ProjectedSphere.Center = Helpers.ObjectToVector(subDic["center"]); + ProjectedSphere.Axis = Helpers.ObjectToVector(subDic["axes"]); + ProjectedSphere.Angle = (float)(double)subDic["angle"]; + + return true; + } + + void ExtractSphericalCoordinates(Dictionary dictionary) + { + // if circle normals are not available -> theta&phi are no doubles + Circle.Theta = (float)Helpers.TryCastToDouble(dictionary["theta"]); + Circle.Phi = (float)Helpers.TryCastToDouble(dictionary["phi"]); + } + } +} diff --git a/Scripts/PupilListener.cs b/Scripts/PupilListener.cs new file mode 100644 index 0000000..985d6c5 --- /dev/null +++ b/Scripts/PupilListener.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using UnityEngine; + +namespace PupilLabs +{ + public class PupilListener : BaseListener + { + public event Action OnReceivePupilData; + + public PupilListener(SubscriptionsController subsCtrl) : base(subsCtrl) { } + + protected override void CustomEnable() + { + Debug.Log("Enabling Pupil Listener"); + subsCtrl.SubscribeTo("pupil.", ReceivePupilData); + } + + protected override void CustomDisable() + { + Debug.Log("Disabling Pupil Listener"); + subsCtrl.UnsubscribeFrom("pupil.", ReceivePupilData); + } + + void ReceivePupilData(string topic, Dictionary dictionary, byte[] thirdFrame = null) + { + PupilData pupilData = new PupilData(dictionary); + + if (OnReceivePupilData != null) + { + OnReceivePupilData(pupilData); + } + } + } +} \ No newline at end of file diff --git a/Scripts/RecordingController.cs b/Scripts/RecordingController.cs new file mode 100644 index 0000000..fc0939b --- /dev/null +++ b/Scripts/RecordingController.cs @@ -0,0 +1,151 @@ +using System.Collections; +using System.Collections.Generic; +using UnityEngine; + +namespace PupilLabs +{ + [HelpURL("https://github.com/pupil-labs/hmd-eyes/blob/master/docs/Developer.md#recording-data")] + public class RecordingController : MonoBehaviour + { + public RequestController requestCtrl; + + [Header("Recording Path")] + public bool useCustomPath; + [SerializeField] private string customPath; + + [Header("Controls")] + [SerializeField] private bool recordEyeFrames = true; + [SerializeField] private bool startRecording; + [SerializeField] private bool stopRecording; + + public static bool IsRecording { get; private set; } + + void OnEnable() + { + if (requestCtrl == null) + { + Debug.LogError("RecordingController is missing the required RequestController reference. Please connect the reference, or the component won't work correctly."); + enabled = false; + return; + } + + } + + void OnDisable() + { + if (IsRecording) + { + StopRecording(); + } + } + + void Update() + { + if (Input.GetKeyDown(KeyCode.R)) + { + if (IsRecording) + { + stopRecording = true; + } + else + { + startRecording = true; + } + } + + if (startRecording) + { + startRecording = false; + StartRecording(); + } + + if (stopRecording) + { + stopRecording = false; + StopRecording(); + } + } + + public void StartRecording() + { + if (!enabled) + { + Debug.LogWarning("Component not enabled"); + return; + } + + if (!requestCtrl.IsConnected) + { + Debug.LogWarning("Recoder not connected"); + return; + } + + if (IsRecording) + { + Debug.Log("Recording is already running."); + return; + } + + + var path = GetRecordingPath(); + + requestCtrl.Send(new Dictionary + { + { "subject","recording.should_start" } + , { "session_name", path } + , { "record_eye",recordEyeFrames} + }); + IsRecording = true; + + //abort process on disconnecting + requestCtrl.OnDisconnecting += StopRecording; + } + + public void StopRecording() + { + if (!IsRecording) + { + Debug.Log("Recording is not running, nothing to stop."); + return; + } + + requestCtrl.Send(new Dictionary + { + { "subject", "recording.should_stop" } + }); + + IsRecording = false; + + requestCtrl.OnDisconnecting -= StopRecording; + } + + public void SetCustomPath(string path) + { + useCustomPath = true; + customPath = path; + } + + private string GetRecordingPath() + { + string path = ""; + + if (useCustomPath) + { + path = customPath; + } + else + { + string date = System.DateTime.Now.ToString("yyyy_MM_dd"); + path = $"{Application.dataPath}/{date}"; + path = path.Replace("Assets/", ""); //go one folder up + } + + if (!System.IO.Directory.Exists(path)) + { + System.IO.Directory.CreateDirectory(path); + } + + return path; + } + } +} diff --git a/Scripts/Request.cs b/Scripts/Request.cs new file mode 100644 index 0000000..36ab6a4 --- /dev/null +++ b/Scripts/Request.cs @@ -0,0 +1,141 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using UnityEngine; +using NetMQ; +using NetMQ.Sockets; +using MessagePack; + +namespace PupilLabs +{ + public partial class RequestController + { + [System.Serializable] + private class Request + { + public string IP = "127.0.0.1"; + public int PORT = 50020; + [SerializeField] + private string IPHeader; + private string subport; + private string pubport; + + private RequestSocket requestSocket = null; + private float timeout = 1f; + private TimeSpan requestTimeout = new System.TimeSpan(0, 0, 1); //= 1sec + + public bool IsConnected { get; set; } + + public string GetSubConnectionString() + { + return IPHeader + subport; + } + + public string GetPubConnectionString() + { + return IPHeader + pubport; + } + + public IEnumerator InitializeRequestSocketAsync(float timeout) + { + AsyncIO.ForceDotNet.Force(); + + IPHeader = $">tcp://{IP}:"; + Debug.Log("Attempting to connect to : " + IPHeader + PORT); + + requestSocket = new RequestSocket(IPHeader + PORT); + + yield return UpdatePorts(); + } + + public IEnumerator UpdatePorts() + { + yield return RequestReceiveAsync( + () => requestSocket.SendFrame("SUB_PORT"), + () => IsConnected = requestSocket.TryReceiveFrameString(out subport) + ); + + if (IsConnected) + { + yield return RequestReceiveAsync( + () => requestSocket.SendFrame("PUB_PORT"), + () => requestSocket.TryReceiveFrameString(out pubport) + ); + } + } + + private IEnumerator RequestReceiveAsync(Action request, Action receive) + { + float tStarted = Time.realtimeSinceStartup; + + request(); + + bool msgReceived = false; + while (!msgReceived) + { + if (Time.realtimeSinceStartup - tStarted > timeout) + { + yield break; + } + else + { + if (requestSocket.HasIn) + { + msgReceived = true; + receive(); + } + else + { + yield return new WaitForSeconds(0.1f); + } + } + } + } + + public void Close() + { + if (requestSocket != null) + { + requestSocket.Close(); + } + + IsConnected = false; + } + + public void SendRequestMessage(Dictionary data) + { + NetMQMessage m = new NetMQMessage(); + + m.Append("notify." + data["subject"]); + m.Append(MessagePackSerializer.Serialize>(data)); + + requestSocket.SendMultipartMessage(m); + ReceiveRequestResponse(); + } + + public bool SendCommand(string cmd, out string response) + { + if (requestSocket == null || !IsConnected) + { + response = null; + return false; + } + + requestSocket.SendFrame(cmd); + return requestSocket.TryReceiveFrameString(requestTimeout, out response); + } + + private void ReceiveRequestResponse() + { + NetMQMessage m = new NetMQMessage(); + requestSocket.TryReceiveMultipartMessage(requestTimeout, ref m); + } + + public void resetDefaultLocalConnection() + { + IP = "127.0.0.1"; + PORT = 50020; + } + } + } +} diff --git a/Scripts/RequestController.cs b/Scripts/RequestController.cs new file mode 100644 index 0000000..3e2e150 --- /dev/null +++ b/Scripts/RequestController.cs @@ -0,0 +1,257 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using UnityEngine; + +#if UNITY_EDITOR +using UnityEditor; +#endif + +namespace PupilLabs +{ + + public partial class RequestController : MonoBehaviour + { + [SerializeField][HideInInspector] + private Request request; + + [Header("Settings")] + public float retryConnectDelay = 5f; + public bool connectOnEnable = true; + + public event Action OnConnected = delegate { }; + public event Action OnDisconnecting = delegate { }; + + public bool IsConnected + { + get { return request.IsConnected && connectingDone; } + } + private bool connectingDone; + + [SerializeField][HideInInspector] + private bool isConnecting = false; + + public string IP + { + get { return request.IP; } + set { request.IP = value; } + } + + public int PORT + { + get { return request.PORT; } + set { request.PORT = value; } + } + + [SerializeField][HideInInspector] + private string PupilVersion; + + public string GetSubConnectionString() + { + return request.GetSubConnectionString(); + } + + public string GetPubConnectionString() + { + return request.GetPubConnectionString(); + } + + void Awake() + { + NetMQCleanup.MonitorConnection(this); + } + + void OnEnable() + { + if (request == null) + { + request = new Request(); + } + + PupilVersion = "not connected"; + if (!request.IsConnected && connectOnEnable) + { + RunConnect(3f); + } + } + + void OnDisable() + { + Disconnect(); + } + + void OnDestroy() + { + Disconnect(); + + NetMQCleanup.CleanupConnection(this); + } + + public void RunConnect(float delay = 0) + { + if (isConnecting) + { + Debug.LogWarning("Already trying to connect!"); + return; + } + + if (!enabled) + { + Debug.LogWarning("Component not enabled!"); + return; + } + + if (request.IsConnected) + { + Debug.LogWarning("Already connected!"); + return; + } + + StartCoroutine(Connect(retry: true, delay: delay)); + } + + private IEnumerator Connect(bool retry = false, float delay = 0) + { + isConnecting = true; + + yield return new WaitForSeconds(delay); + + connectingDone = false; + + while (!request.IsConnected) + { + yield return StartCoroutine(request.InitializeRequestSocketAsync(1f)); + + if (!request.IsConnected) + { + request.Close(); + + if (retry) + { + Debug.LogWarning("Could not connect, Re-trying in 5 seconds! "); + yield return new WaitForSeconds(retryConnectDelay); + } + else + { + Debug.LogWarning("Could not connect! "); + yield break; + } + } + } + + isConnecting = false; + Connected(); +#if UNITY_EDITOR + EditorUtility.SetDirty(this); +#endif + + yield break; + } + + private void Connected() + { + Debug.Log("Succesfully connected to Pupil! "); + + UpdatePupilVersion(); + + StartEyeProcesses(); + SetDetectionMode("3d"); + + connectingDone = true; + + OnConnected(); + } + + public void Disconnect() + { + if (!IsConnected) + { + return; + } + + OnDisconnecting(); + + request.Close(); + } + + public void Send(Dictionary dictionary) + { + if (!request.IsConnected) + { + Debug.LogWarning("Not connected!"); + return; + } + + request.SendRequestMessage(dictionary); + } + + public bool SendCommand(string command, out string response) + { + return request.SendCommand(command, out response); + } + + public void StartEyeProcesses() + { + var startLeftEye = new Dictionary { + {"subject", "eye_process.should_start.1"}, + {"eye_id", 1}, + {"delay", 0.1f} + }; + var startRightEye = new Dictionary { + {"subject", "eye_process.should_start.0"}, + { "eye_id", 0}, + { "delay", 0.2f} + }; + + Send(startLeftEye); + Send(startRightEye); + } + + public void StartPlugin(string name, Dictionary args = null) + { + Dictionary startPluginDic = new Dictionary { + { "subject", "start_plugin" }, + { "name", name } + }; + + if (args != null) + { + startPluginDic["args"] = args; + } + + Send(startPluginDic); + } + + public void StopPlugin(string name) + { + Send(new Dictionary { + { "subject","stop_plugin" }, + { "name", name } + }); + } + + public void SetDetectionMode(string mode) + { + Send(new Dictionary { { "subject", "set_detection_mapping_mode" }, { "mode", mode } }); + } + + public string GetPupilVersion() + { + string pupilVersion = null; + SendCommand("v", out pupilVersion); + return pupilVersion; + } + + private void UpdatePupilVersion() + { + PupilVersion = GetPupilVersion(); + Debug.Log($"Pupil Version: {PupilVersion}"); + } + + [ContextMenu("Reset To Default Connection")] + public void ResetDefaultLocalConnection() + { + request.resetDefaultLocalConnection(); + } + } +} \ No newline at end of file diff --git a/Scripts/ScreenCast.cs b/Scripts/ScreenCast.cs new file mode 100644 index 0000000..886cb68 --- /dev/null +++ b/Scripts/ScreenCast.cs @@ -0,0 +1,147 @@ +using System.Collections; +using System.Collections.Generic; +using UnityEngine; +using UnityEngine.Rendering; + +using NetMQ; +using NetMQ.Sockets; +using MessagePack; + +namespace PupilLabs +{ + public class ScreenCast : MonoBehaviour + { + public RequestController requestCtrl; + public TimeSync timeSync; + public Camera centeredCamera; + [Tooltip("Can't be changed at runtime")] + public int initialWidth = 640, initialHeight = 480; + [Range(1, 120)] + public int maxFrameRate = 90; + public bool pauseStreaming = false; + + public Texture2D StreamTexture { get; private set; } + + Publisher publisher; + RenderTexture renderTexture; + bool isSetup = false; + int index = 0; + float tLastFrame; + int width, height; + float[] intrinsics = {0,0,0,0,0,0,0,0,1}; + + const string topic = "hmd_streaming.world"; + + float deltaTimeAcc = 0f; + + void Awake() + { + width = initialWidth; + height = initialHeight; + + renderTexture = new RenderTexture(width, height, 24); + centeredCamera.targetTexture = renderTexture; + StreamTexture = new Texture2D(renderTexture.width, renderTexture.height, TextureFormat.RGB24, false); + } + + void OnEnable() + { + requestCtrl.OnConnected += Setup; + } + + void Setup() + { + requestCtrl.StartPlugin("HMD_Streaming_Source"); + + publisher = new Publisher(requestCtrl); + + isSetup = true; + } + + void Update() + { + if (!isSetup || pauseStreaming) + { + return; + } + + if (IgnoreFrameBasedOnFPS()) + { + return; + } + + UpdateIntrinsics(); + + AsyncGPUReadback.Request + ( + renderTexture, 0, TextureFormat.RGB24, + (AsyncGPUReadbackRequest r) => ReadbackDone(r, timeSync.ConvertToPupilTime(Time.realtimeSinceStartup)) + ); + } + + void OnApplicationQuit() + { + if (publisher != null) + { + publisher.Destroy(); + } + } + + void ReadbackDone(AsyncGPUReadbackRequest r, double timestamp) + { + if (StreamTexture == null) + { + return; + } + + StreamTexture.LoadRawTextureData(r.GetData()); + StreamTexture.Apply(); + + SendFrame(timestamp); + } + + void SendFrame(double timestamp) + { + Dictionary payload = new Dictionary { + {"topic", topic}, + {"width", width}, + {"height", height}, + {"index", index}, + {"timestamp", timestamp}, + {"format", "rgb"}, + {"projection_matrix", intrinsics} + }; + + publisher.Send(topic, payload, StreamTexture.GetRawTextureData()); + + index++; + } + + void UpdateIntrinsics() + { + float fov = centeredCamera.fieldOfView; + float focalLength = 1f / (Mathf.Tan(fov / (2 * Mathf.Rad2Deg)) / height) / 2; + + // f 0 width/2 + // 0 f height/2 + // 0 0 1 + + intrinsics[0] = focalLength; + intrinsics[2] = width/2; + intrinsics[4] = focalLength; + intrinsics[5] = height/2; + } + + bool IgnoreFrameBasedOnFPS() + { + deltaTimeAcc += Time.deltaTime; + if ( deltaTimeAcc < 1 / (float)maxFrameRate) + { + return true; + } + deltaTimeAcc %= 1 / (float)maxFrameRate; + + return false; + } + } +} \ No newline at end of file diff --git a/Scripts/Subscription.cs b/Scripts/Subscription.cs new file mode 100644 index 0000000..4cb40ae --- /dev/null +++ b/Scripts/Subscription.cs @@ -0,0 +1,82 @@ +using System.Collections.Generic; +using UnityEngine; +using NetMQ; +using NetMQ.Sockets; +using MessagePack; +using System.IO; + +namespace PupilLabs +{ + public partial class SubscriptionsController : MonoBehaviour + { + private class Subscription + { + public string topic; + private SubscriberSocket socket; + + public event ReceiveDataDelegate OnReceiveData; + + public bool HasSubscribers + { + get { return OnReceiveData != null; } + } + + public bool ShouldClose { get; set; } + + public Subscription(string connection, string topic) + { + Setup(connection, topic); + } + + public void Setup(string connection, string topic) + { + Close(); + + this.topic = topic; + + socket = new SubscriberSocket(connection); + socket.Subscribe(topic); + + socket.ReceiveReady += ParseData; + } + + public void ParseData(object s, NetMQSocketEventArgs eventArgs) + { + NetMQMessage m = new NetMQMessage(); + + while (eventArgs.Socket.TryReceiveMultipartMessage(ref m)) + { + string msgType = m[0].ConvertToString(); + MemoryStream mStream = new MemoryStream(m[1].ToByteArray()); + + byte[] thirdFrame = null; + if (m.FrameCount >= 3) + { + thirdFrame = m[2].ToByteArray(); + } + + if (OnReceiveData != null) + { + OnReceiveData(msgType, MessagePackSerializer.Deserialize>(mStream), thirdFrame); + } + } + } + + public void UpdateSocket() + { + if (socket.HasIn) + { + socket.Poll(); + } + } + + public void Close() + { + if (socket != null) + { + socket.Close(); + } + } + } + } +} diff --git a/Scripts/SubscriptionsController.cs b/Scripts/SubscriptionsController.cs new file mode 100644 index 0000000..ded94d4 --- /dev/null +++ b/Scripts/SubscriptionsController.cs @@ -0,0 +1,138 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using UnityEngine; + +namespace PupilLabs +{ + public partial class SubscriptionsController : MonoBehaviour + { + + public PupilLabs.RequestController requestCtrl; + + public bool IsConnected + { + get { return !(requestCtrl == null || !requestCtrl.IsConnected); } + } + + public event Action OnDisconnecting = delegate { }; + public delegate void ReceiveDataDelegate(string topic, Dictionary dictionary, byte[] thirdFrame = null); + + private Dictionary subscriptions = new Dictionary(); + + void OnEnable() + { + if (requestCtrl == null) + { + Debug.LogError("SubscriptionsController is missing the required RequestController reference. Please connect the reference, or the component won't work correctly."); + enabled = false; + return; + } + } + + void OnApplicationQuit() + { + if (IsConnected) + { + Disconnect(); + } + } + + void Update() + { + if (!IsConnected) + { + return; + } + + UpdateSubscriptionSockets(); + } + + public void Disconnect() + { + if (!IsConnected) + { + return; + } + + OnDisconnecting(); + + foreach (var socketKey in subscriptions.Keys) + { + CloseSubscriptionSocket(socketKey); + } + UpdateSubscriptionSockets(); + } + + public void SubscribeTo(string topic, ReceiveDataDelegate subscriberHandler) + { + if (!IsConnected) + { + Debug.LogWarning("Not connected!"); + return; + } + + if (!subscriptions.ContainsKey(topic)) + { + string connectionStr = requestCtrl.GetSubConnectionString(); + Subscription subscription = new Subscription(connectionStr, topic); + + subscriptions.Add(topic, subscription); + } + + subscriptions[topic].OnReceiveData += subscriberHandler; + } + + public void UnsubscribeFrom(string topic, ReceiveDataDelegate subscriberHandler) + { + if (!IsConnected) + { + return; + } + + if (subscriptions.ContainsKey(topic) && subscriberHandler != null) + { + subscriptions[topic].OnReceiveData -= subscriberHandler; + + if (!subscriptions[topic].HasSubscribers) + { + CloseSubscriptionSocket(topic); + } + } + } + + private void CloseSubscriptionSocket(string topic) + { + if (subscriptions.ContainsKey(topic)) + { + subscriptions[topic].ShouldClose = true; + } + } + + private void UpdateSubscriptionSockets() + { + + List toBeRemoved = new List(); + foreach (var subscription in subscriptions.Values.ToList()) + { + if (!subscription.ShouldClose) + { + subscription.UpdateSocket(); + } + else + { + subscription.Close(); + toBeRemoved.Add(subscription.topic); + } + } + + foreach (var removeTopic in toBeRemoved) + { + if (subscriptions.ContainsKey(removeTopic)) + { + subscriptions.Remove(removeTopic); + } + } + } + } +} \ No newline at end of file diff --git a/Scripts/TimeSync.cs b/Scripts/TimeSync.cs new file mode 100644 index 0000000..5bc0513 --- /dev/null +++ b/Scripts/TimeSync.cs @@ -0,0 +1,118 @@ +using System; +using System.Collections.Generic; +using UnityEngine; +using NetMQ; +using NetMQ.Sockets; +using MessagePack; + +namespace PupilLabs +{ + public class TimeSync : MonoBehaviour + { + [SerializeField] RequestController requestCtrl; + + public double UnityToPupilTimeOffset { get; private set; } + + void OnEnable() + { + requestCtrl.OnConnected += UpdateTimeSync; + } + + public double GetPupilTimestamp() + { + if (!requestCtrl.IsConnected) + { + Debug.LogWarning("Not connected"); + return 0; + } + + string response; + requestCtrl.SendCommand("t", out response); + + return double.Parse(response, System.Globalization.CultureInfo.InvariantCulture.NumberFormat); ; + } + + public double ConvertToUnityTime(double pupilTimestamp) + { + if (!requestCtrl.IsConnected) + { + Debug.LogWarning("Not connected"); + return 0; + } + + return pupilTimestamp - UnityToPupilTimeOffset; + } + + public double ConvertToPupilTime(double unityTime) + { + if (!requestCtrl.IsConnected) + { + Debug.LogWarning("Not connected"); + return 0; + } + + return unityTime + UnityToPupilTimeOffset; + } + + [ContextMenu("Update TimeSync")] + public void UpdateTimeSync() + { + if (!requestCtrl.IsConnected) + { + Debug.LogWarning("Not connected"); + return; + } + + double tBefore = Time.realtimeSinceStartup; + double pupilTime = GetPupilTimestamp(); + double tAfter = Time.realtimeSinceStartup; + + double unityTime = (tBefore + tAfter) / 2.0; + UnityToPupilTimeOffset = pupilTime - unityTime; + } + + [System.Obsolete("Setting the pupil timestamp might be in conflict with other plugins.")] + public void SetPupilTimestamp(double time) + { + if (!requestCtrl.IsConnected) + { + Debug.LogWarning("Not connected"); + return; + } + + string response; + string command = "T " + time.ToString("0.000000", System.Globalization.CultureInfo.InvariantCulture); + + float tBefore = Time.realtimeSinceStartup; + requestCtrl.SendCommand(command, out response); + float tAfter = Time.realtimeSinceStartup; + + UnityToPupilTimeOffset = -(tAfter - tBefore) / 2f; + } + + [ContextMenu("Check Time Sync")] + public void CheckTimeSync() + { + if (!requestCtrl.IsConnected) + { + Debug.LogWarning("Check Time Sync: not connected"); + return; + } + double pupilTime = GetPupilTimestamp(); + double unityTime = Time.realtimeSinceStartup; + Debug.Log($"Unity time: {unityTime}"); + Debug.Log($"Pupil Time: {pupilTime}"); + Debug.Log($"Unity to Pupil Offset {UnityToPupilTimeOffset}"); + Debug.Log($"out of sync by {unityTime + UnityToPupilTimeOffset - pupilTime}"); + } + + // [ContextMenu("Sync Pupil Time To Time.now")] + // void SyncPupilTimeToUnityTime() + // { + // if (requestCtrl.IsConnected) + // { + // SetPupilTimestamp(Time.realtimeSinceStartup); + // } + // } + } +} \ No newline at end of file diff --git a/Scripts/Utils/AttachToMainCamera.cs b/Scripts/Utils/AttachToMainCamera.cs new file mode 100644 index 0000000..6e81a92 --- /dev/null +++ b/Scripts/Utils/AttachToMainCamera.cs @@ -0,0 +1,9 @@ +using UnityEngine; + +public class AttachToMainCamera : MonoBehaviour +{ + void Start() + { + this.transform.SetParent(Camera.main.transform,false); + } +} diff --git a/Scripts/Utils/DisableDuringCalibration.cs b/Scripts/Utils/DisableDuringCalibration.cs new file mode 100644 index 0000000..1f4cfe8 --- /dev/null +++ b/Scripts/Utils/DisableDuringCalibration.cs @@ -0,0 +1,38 @@ +using System.Collections; +using System.Collections.Generic; +using UnityEngine; + +namespace PupilLabs +{ + public class DisableDuringCalibration : MonoBehaviour + { + + public CalibrationController controller; + public bool enableAfterCalibration; + + void Awake() + { + controller.OnCalibrationStarted += DisableMePls; + controller.OnCalibrationRoutineDone += EnableMePls; + } + + void OnDestroy() + { + controller.OnCalibrationStarted -= DisableMePls; + controller.OnCalibrationRoutineDone -= EnableMePls; + } + + void EnableMePls() + { + if (enableAfterCalibration) + { + gameObject.SetActive(true); + } + } + + void DisableMePls() + { + gameObject.SetActive(false); + } + } +} diff --git a/Scripts/Utils/DontDestroy.cs b/Scripts/Utils/DontDestroy.cs new file mode 100644 index 0000000..069c7c8 --- /dev/null +++ b/Scripts/Utils/DontDestroy.cs @@ -0,0 +1,12 @@ +using UnityEngine; + +namespace PupilLabs +{ + public class DontDestroy : MonoBehaviour + { + void Awake() + { + DontDestroyOnLoad(this.gameObject); + } + } +} \ No newline at end of file diff --git a/Scripts/Utils/LoadSceneAfterCalibration.cs b/Scripts/Utils/LoadSceneAfterCalibration.cs new file mode 100644 index 0000000..aa4aad3 --- /dev/null +++ b/Scripts/Utils/LoadSceneAfterCalibration.cs @@ -0,0 +1,28 @@ +using UnityEngine; +using UnityEngine.SceneManagement; + +namespace PupilLabs +{ + public class LoadSceneAfterCalibration : MonoBehaviour + { + public PupilLabs.CalibrationController calibrationController; + + [Tooltip("Specify scene by name (needs to be added to 'BuildSettings/Scenes In Build'")] + public string sceneToLoad; + + void OnEnable() + { + calibrationController.OnCalibrationSucceeded += LoadScene; + } + + void OnDisable() + { + calibrationController.OnCalibrationSucceeded -= LoadScene; + } + + void LoadScene() + { + SceneManager.LoadScene(sceneToLoad); + } + } +}