From fffb896a1b64341aa6038a83a4be81722d77aafe Mon Sep 17 00:00:00 2001
From: Bjoern Esswein <bjoern.esswein@gmail.com>
Date: Sat, 18 Nov 2023 14:55:57 +0100
Subject: [PATCH] Pointer down, move, up is transmitted to the browser

TODO: compensate image scaling
---
 Runtime/BrowserView.cs                        |  37 ++++-
 Runtime/ChromeDevtools/BrowserTab.cs          |  34 +++-
 Runtime/ChromeDevtools/Protocol/Input.meta    |   8 +
 .../ChromeDevtools/Protocol/Input/Input.cs    | 152 ++++++++++++++++++
 .../Protocol/Input/Input.cs.meta              |  11 ++
 5 files changed, 237 insertions(+), 5 deletions(-)
 create mode 100644 Runtime/ChromeDevtools/Protocol/Input.meta
 create mode 100644 Runtime/ChromeDevtools/Protocol/Input/Input.cs
 create mode 100644 Runtime/ChromeDevtools/Protocol/Input/Input.cs.meta

diff --git a/Runtime/BrowserView.cs b/Runtime/BrowserView.cs
index ce92496..8a91b2d 100644
--- a/Runtime/BrowserView.cs
+++ b/Runtime/BrowserView.cs
@@ -1,15 +1,17 @@
 using ChromeDevTools;
 using System.Collections;
 using UnityEngine;
+using UnityEngine.EventSystems;
 using UnityEngine.UI;
 
 [RequireComponent(typeof(RawImage))]
-public class BrowserView : MonoBehaviour
+public class BrowserView : MonoBehaviour, IPointerDownHandler, IPointerMoveHandler, IPointerUpHandler
 {
     private Browser browser;
     private BrowserTab tab;
 
     private RawImage rawImage;
+    private RectTransform rectTransform;
 
     public bool headlessBrowser = true;
     //TODO: handle changed targetUrl
@@ -19,6 +21,7 @@ public class BrowserView : MonoBehaviour
     private void Start()
     {
         rawImage = this.gameObject.GetComponent<RawImage>();
+        rectTransform = this.gameObject.GetComponent<RectTransform>();
     }
 
     private void OnEnable()
@@ -53,7 +56,37 @@ public class BrowserView : MonoBehaviour
     // Update is called once per frame
     private void Update()
     {
-        
+
+    }
+
+    public void OnPointerDown(PointerEventData eventData)
+    {
+        Vector2 pos;
+        if (RectTransformUtility.ScreenPointToLocalPointInRectangle(rectTransform, eventData.position, eventData.pressEventCamera, out pos)
+            && rectTransform.rect.Contains(pos))
+        {
+            tab.OnPointerDown(pos, eventData);
+        }
+    }
+
+    public void OnPointerMove(PointerEventData eventData)
+    {
+        Vector2 pos;
+        if (RectTransformUtility.ScreenPointToLocalPointInRectangle(rectTransform, eventData.position, eventData.enterEventCamera, out pos)
+            && rectTransform.rect.Contains(pos))
+        {
+            tab.OnPointerMove(pos, eventData);
+        }
+    }
+
+    public void OnPointerUp(PointerEventData eventData)
+    {
+        Vector2 pos;
+        if (RectTransformUtility.ScreenPointToLocalPointInRectangle(rectTransform, eventData.position, eventData.pressEventCamera, out pos)
+            && rectTransform.rect.Contains(pos))
+        {
+            tab.OnPointerUp(pos, eventData);
+        }
     }
 
     private void OnDisable()
diff --git a/Runtime/ChromeDevtools/BrowserTab.cs b/Runtime/ChromeDevtools/BrowserTab.cs
index c0c0389..589a535 100644
--- a/Runtime/ChromeDevtools/BrowserTab.cs
+++ b/Runtime/ChromeDevtools/BrowserTab.cs
@@ -1,9 +1,10 @@
+using ChromeDevTools.Protocol.Input;
 using ChromeDevTools.Protocol.Page;
 using ChromeDevTools.Protocol.Target;
 using System;
 using System.Collections;
-using System.Threading;
 using UnityEngine;
+using UnityEngine.EventSystems;
 
 namespace ChromeDevTools
 {
@@ -51,13 +52,13 @@ namespace ChromeDevTools
             // TODO: deregister on stop stream
             devtools.screencastFrameEventHandler += (screencastFrameEvent frameEvent) =>
             {
-                Debug.Log($"screencast frame, '{frameEvent.sessionId}'");
-                
                 // send an ack for this frame
                 var frameAck = new screencastFrameAck();
                 frameAck.sessionId = frameEvent.sessionId;
                 _ = devtools.SendCommandAsync(frameAck);
 
+                Debug.Log($"screencast frame, '{frameEvent.sessionId}'");
+
                 // parse the base64 encoded frame to a texture
                 var frameTexture = new Texture2D(1, 1); // new Texture2D only works on the main thread
                 frameTexture.LoadImage(Convert.FromBase64String(frameEvent.Data));
@@ -74,6 +75,33 @@ namespace ChromeDevTools
             return devtools.SendCommand(startScreencast);
         }
 
+        public void OnPointerDown(Vector2 position, PointerEventData eventData)
+        {
+            Debug.Log($"PointerDown {position}:\n{eventData}");
+            // TODO: compensate stream and texture scaling
+            var mousePressedEvent = new dispatchMouseEvent(MouseEventType.MousePressed, (int)position.x, (int)position.y, eventData);
+
+            _ = devtools.SendCommandAsync(mousePressedEvent);
+        }
+
+        internal void OnPointerMove(Vector2 position, PointerEventData eventData)
+        {
+            Debug.Log($"OnPointerMove {position}:\n{eventData}");
+            // TODO: compensate stream and texture scaling
+            var mousePressedEvent = new dispatchMouseEvent(MouseEventType.MouseMoved, (int)position.x, (int)position.y, eventData);
+
+            _ = devtools.SendCommandAsync(mousePressedEvent);
+        }
+
+        public void OnPointerUp(Vector2 position, PointerEventData eventData)
+        {
+            Debug.Log($"OnPointerUp {position}:\n{eventData}");
+            // TODO: compensate stream and texture scaling
+            var mouseReleasedEvent = new dispatchMouseEvent(MouseEventType.MouseReleased, (int)position.x, (int)position.y, eventData);
+
+            _ = devtools.SendCommandAsync(mouseReleasedEvent);
+        }
+
         /**
          * close this tab
          */
diff --git a/Runtime/ChromeDevtools/Protocol/Input.meta b/Runtime/ChromeDevtools/Protocol/Input.meta
new file mode 100644
index 0000000..7aee0bd
--- /dev/null
+++ b/Runtime/ChromeDevtools/Protocol/Input.meta
@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 23034037f1c0a824fb8eb0d89776c15f
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 
diff --git a/Runtime/ChromeDevtools/Protocol/Input/Input.cs b/Runtime/ChromeDevtools/Protocol/Input/Input.cs
new file mode 100644
index 0000000..f5bb061
--- /dev/null
+++ b/Runtime/ChromeDevtools/Protocol/Input/Input.cs
@@ -0,0 +1,152 @@
+using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
+using System;
+using Newtonsoft.Json.Serialization;
+using UnityEngine.EventSystems;
+using UnityEditor.UI;
+using static UnityEngine.EventSystems.PointerEventData;
+
+namespace ChromeDevTools
+{
+    namespace Protocol
+    {
+        namespace Input
+        {
+            /// <summary>
+            /// Dispatches a mouse event to the page.
+            /// </summary>
+            public class dispatchMouseEvent: IDevtoolsCommand
+            {
+                public dispatchMouseEvent(MouseEventType type, int x, int y)
+                {
+                    this.type = type;
+                    this.x = x;
+                    this.y = y;
+                }
+                public dispatchMouseEvent(MouseEventType type, int x, int y, PointerEventData eventData) : this(type, x, y)
+                {
+                    switch (eventData.button)
+                    {
+                        case InputButton.Left:
+                            button = MouseButton.Left; break;
+                        case InputButton.Right:
+                            button = MouseButton.Right; break;
+                        case InputButton.Middle:
+                            button = MouseButton.Middle; break;
+                    }
+                    clickCount = eventData.clickCount;
+                    // TODO: delta compensate stream and texture scaling
+                    deltaX = (int?)eventData.delta.x;
+                    deltaY = (int?)eventData.delta.y;
+                }
+
+                /// <summary>
+                /// Type of the mouse event.
+                /// Allowed Values: mousePressed, mouseReleased, mouseMoved, mouseWheel
+                /// </summary>
+                [JsonConverter(typeof(StringEnumConverter), typeof(CamelCaseNamingStrategy))]
+                public MouseEventType type { get; set; }
+                /// <summary>
+                /// X coordinate of the event relative to the main frame's viewport in CSS pixels.
+                /// </summary>
+                public int x { get; set; }
+                /// <summary>
+                /// Y coordinate of the event relative to the main frame's viewport in CSS pixels. 0 refers to the top of the viewport and Y increases as it proceeds towards the bottom of the viewport.
+                /// </summary>
+                public int y { get; set; }
+                /// <summary>
+                /// Bit field representing pressed modifier keys. Alt=1, Ctrl=2, Meta/Command=4, Shift=8 (default: 0).
+                /// </summary>
+                [JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
+                public int? modifiers { get; set; }
+                /// <summary>
+                /// Time at which the event occurred.
+                /// TimeSinceEpoch UTC time in seconds, counted from January 1, 1970.
+                /// </summary>
+                [JsonConverter(typeof(UnixDateTimeConverter))]
+                [JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
+                public DateTime? timestamp { get; set; }
+                /// <summary>
+                /// Allowed Values: none, left, middle, right, back, forward
+                /// Mouse button (default: "none").
+                /// </summary>
+                [JsonConverter(typeof(StringEnumConverter), typeof(CamelCaseNamingStrategy))]
+                [JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
+                public MouseButton? button { get; set; }
+                /// <summary>
+                /// A number indicating which buttons are pressed on the mouse when a mouse event is triggered. Left=1, Right=2, Middle=4, Back=8, Forward=16, None=0.
+                /// </summary>
+                [JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
+                public MouseButtonFlags? buttons { get; set; }
+                /// <summary>
+                /// Number of times the mouse button was clicked (default: 0).
+                /// </summary>
+                [JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
+                public int? clickCount { get; set; }
+                /// <summary>
+                /// The normalized pressure, which has a range of [0,1] (default: 0).
+                /// </summary>
+                [JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
+                public int? force { get; set; }
+                /// <summary>
+                /// The normalized tangential pressure, which has a range of [-1,1] (default: 0).
+                /// </summary>
+                [JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
+                public int? tangentialPressure { get; set; }
+                /// <summary>
+                /// The plane angle between the Y-Z plane and the plane containing both the stylus axis and the Y axis, in degrees of the range [-90,90], a positive tiltX is to the right (default: 0).
+                /// </summary>
+                [JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
+                public int? tiltX { get; set; }
+                /// <summary>
+                /// The plane angle between the X-Z plane and the plane containing both the stylus axis and the X axis, in degrees of the range [-90,90], a positive tiltY is towards the user (default: 0).
+                /// </summary>
+                [JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
+                public int? tiltY { get; set; }
+                /// <summary>
+                /// The clockwise rotation of a pen stylus around its own major axis, in degrees in the range [0,359] (default: 0).
+                /// </summary>
+                [JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
+                public int? twist { get; set; }
+                /// <summary>
+                /// X delta in CSS pixels for mouse wheel event (default: 0).
+                /// </summary>
+                [JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
+                public int? deltaX { get; set; }
+                /// <summary>
+                /// Y delta in CSS pixels for mouse wheel event (default: 0).
+                /// </summary>
+                [JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
+                public int? deltaY { get; set; }
+                /// <summary>
+                /// Pointer type (default: "mouse").
+                /// Allowed Values: mouse, pen
+                /// </summary>
+                [JsonConverter(typeof(StringEnumConverter), typeof(CamelCaseNamingStrategy))]
+                [JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
+                public PointerType? pointerType { get; set; }
+            }
+
+            public enum MouseEventType
+            {
+                MousePressed, MouseReleased, MouseMoved, MouseWheel
+            }
+
+            public enum MouseButton
+            {
+                None, Left, Middle, Right, Back, Forward
+            }
+
+            [Flags]
+            public enum MouseButtonFlags
+            {
+                None = 0, Left = 1, Right = 2, Middle = 4, Back = 8, Forward = 16
+            }
+
+            public enum PointerType
+            {
+                Mouse, Pen
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/Runtime/ChromeDevtools/Protocol/Input/Input.cs.meta b/Runtime/ChromeDevtools/Protocol/Input/Input.cs.meta
new file mode 100644
index 0000000..70ab290
--- /dev/null
+++ b/Runtime/ChromeDevtools/Protocol/Input/Input.cs.meta
@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 1a4364f07a30bb142ad2aae29c4e3fe1
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 
-- 
GitLab