Improved page scaling by using the device emulation api and made click events more reliable

Runtime/ChromeDevtools/BrowserTab.cs

 using bessw.Unity.WebView.ChromeDevTools.Protocol.DOM;
+using bessw.Unity.WebView.ChromeDevTools.Protocol.Emulation;
 using bessw.Unity.WebView.ChromeDevTools.Protocol.Input;
 using bessw.Unity.WebView.ChromeDevTools.Protocol.Page;
 using bessw.Unity.WebView.ChromeDevTools.Protocol.Runtime;
 using bessw.Unity.WebView.ChromeDevTools.Protocol.Target;
+using bessw.Unity.WebView.ChromeDevTools.Protocol.Types;
 using System;
 using System.Collections;
 using System.Collections.Generic;
@@ -19,9 +21,19 @@ namespace bessw.Unity.WebView.ChromeDevTools
         public DevtoolsProtocolHandler devtools;
 
         /// <summary>
-        /// width and height of the brower device
+        /// The size of the browser tab in pixels
         /// </summary>
-        public Vector2Int size { get; private set; }
+        public Vector2Int Size { get; private set; }
+
+        /// <summary>
+        /// Scales the page in the browser tab
+        /// </summary>
+        public int PageScaleFactor { get; private set; }
+
+        /// <summary>
+        /// width and height of the stream from the browser
+        /// </summary>
+        public Vector2Int StreamSize { get; private set; }
 
         public Dictionary<int, DomNodeWrapper> domNodes = new Dictionary<int, DomNodeWrapper>();
 
@@ -75,6 +87,27 @@ namespace bessw.Unity.WebView.ChromeDevTools
             yield return devtools.readLoop();
         }
 
+        public async Task SetSizeAndScale(Vector2Int size, int pageScaleFactor)
+        {
+            await devtools.SendCommandAsync(new setDeviceMetricsOverride
+            {
+                width = size.x,
+                height = size.y,
+                screenWidth = size.x,
+                screenHeight = size.y,
+                viewport = new()
+                {
+                    X = 0,
+                    Y = 0,
+                    Width = size.x,
+                    Height = size.y,
+                    Scale = pageScaleFactor
+                }
+            });
+            Size = size;
+            PageScaleFactor = pageScaleFactor;
+        }
+
         /// <summary>
         /// Evaluate a JavaScript expression in the browser
         /// </summary>
@@ -127,15 +160,11 @@ namespace bessw.Unity.WebView.ChromeDevTools
             }
         }
 
-        public IEnumerator CreateScreenshot(double width, double height, Action<Texture2D> callback)
+        public IEnumerator CreateScreenshot(Viewport viewport, Action<Texture2D> callback)
         {
             var screenshotCommand = new captureScreenshot
             {
-                Clip = new Protocol.Types.Viewport
-                {
-                    Width = width,
-                    Height = height
-                }
+                Clip = viewport
             };
 
             return devtools.SendCommand(screenshotCommand,
@@ -164,9 +193,9 @@ namespace bessw.Unity.WebView.ChromeDevTools
                 };
                 _ = devtools.SendCommandAsync(frameAck);
 
-                size = new Vector2Int(frameEvent.metadata.deviceWidth, frameEvent.metadata.deviceHeight);
+                StreamSize = new Vector2Int(frameEvent.metadata.deviceWidth, frameEvent.metadata.deviceHeight);
 
-                Debug.Log($"screencast frame, '{frameEvent.sessionId}'; size: {size}, pageScaleFactor: {frameEvent.metadata.pageScaleFactor}");
+                Debug.Log($"screencast frame, '{frameEvent.sessionId}'; size: {StreamSize}, pageScaleFactor: {frameEvent.metadata.pageScaleFactor}");
 
 
                 // parse the base64 encoded frame to a texture

Runtime/ChromeDevtools/DevtoolsProtocolHandler.cs

@@ -163,6 +163,9 @@ namespace bessw.Unity.WebView.ChromeDevTools
                 case "DOM.documentUpdated":
                     onDomDocumentUpdated?.Invoke( ev.Params.ToObject<documentUpdatedEvent>(Browser.devtoolsSerializer) );
                     break;
+                case "DOM.inlineStyleInvalidated":
+                    // TODO: BE: implement inlineStyleInvalidated event handler
+                    break;
                 case "DOM.setChildNodes":
                     onDomSetChildNodes?.Invoke( ev.Params.ToObject<setChildNodesEvent>(Browser.devtoolsSerializer) );
                     break;

Runtime/ChromeDevtools/Protocol/Emulation.meta

+fileFormatVersion: 2
+guid: 6347cc58456fb974c9d041c411bf1a7f
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Runtime/ChromeDevtools/Protocol/Emulation/Emulation.cs

+using bessw.Unity.WebView.ChromeDevTools.Protocol.Types;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Serialization;
+
+#nullable enable annotations
+namespace bessw.Unity.WebView.ChromeDevTools.Protocol.Emulation
+{
+    #region commands
+
+    /// <summary>
+    /// Overrides the values of device screen dimensions (window.screen.width, window.screen.height, window.innerWidth, window.innerHeight, and "device-width"/"device-height"-related CSS media query results).
+    /// </summary>
+    public class setDeviceMetricsOverride : IDevtoolsCommand
+    {
+        /// <summary>
+        /// Overriding width value in pixels (minimum 0, maximum 10000000). 0 disables the override.
+        /// </summary>
+        public int width = 0;
+        /// <summary>
+        /// Overriding height value in pixels (minimum 0, maximum 10000000). 0 disables the override.
+        /// </summary>
+        public int height = 0;
+        /// <summary>
+        /// Overriding device scale factor value. 0 disables the override.
+        /// </summary>
+        public int deviceScaleFactor = 0;
+        /// <summary>
+        /// Whether to emulate mobile device. This includes viewport meta tag, overlay scrollbars, text autosizing and more. (Default: false)
+        /// </summary>
+        public bool mobile = false;
+        /// <summary>
+        /// Scale to apply to resulting view image.
+        /// </summary>
+        /// <remarks>experimental</remarks>
+        [JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
+        public int? scale;
+        /// <summary>
+        /// Overriding screen width value in pixels (minimum 0, maximum 10000000).
+        /// </summary>
+        /// <remarks>experimental</remarks>
+        [JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
+        public int? screenWidth;
+        /// <summary>
+        /// Overriding screen height value in pixels (minimum 0, maximum 10000000).
+        /// </summary>
+        /// <remarks>experimental</remarks>
+        [JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
+        public int? screenHeight;
+        /// <summary>
+        /// If set, the visible area of the page will be overridden to this viewport. This viewport change is not observed by the page, e.g. viewport-relative elements do not change positions.
+        /// </summary>
+        /// <remarks>experimental</remarks>
+        [JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
+        public Viewport? viewport;
+    }
+
+    /// <summary>
+    /// Sets a specified page scale factor.
+    /// </summary>
+    public class setPageScaleFactor : IDevtoolsCommand
+    {
+        public double pageScaleFactor { get; set; }
+    }
+
+    #endregion commands
+}
\ No newline at end of file

Runtime/ChromeDevtools/Protocol/Emulation/Emulation.cs.meta

+fileFormatVersion: 2
+guid: f40f22b1bc5efa74d98902268e207549
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Runtime/ChromeDevtools/Protocol/Input/Input.cs

@@ -22,16 +22,31 @@ namespace bessw.Unity.WebView.ChromeDevTools.Protocol.Input
         }
         public dispatchMouseEvent(MouseEventType type, int x, int y, PointerEventData eventData) : this(type, x, y)
         {
+            // PointerEventData.clickCount is not always > 0 when the browser expects it to be, therefore we need to set dispatchMouseEvent.clickCount manually depending on other properties
+            if (eventData.dragging || eventData.eligibleForClick || type == MouseEventType.MousePressed || type == MouseEventType.MouseReleased)
+            {
+                clickCount = 1;
                 switch (eventData.button)
                 {
                     case InputButton.Left:
-                    button = MouseButton.Left; break;
+                        button = MouseButton.Left;
+                        buttons = MouseButtonFlags.Left;
+                        break;
                     case InputButton.Right:
-                    button = MouseButton.Right; break;
+                        button = MouseButton.Right;
+                        buttons = MouseButtonFlags.Right;
+                        break;
                     case InputButton.Middle:
-                    button = MouseButton.Middle; break;
+                        button = MouseButton.Middle;
+                        buttons = MouseButtonFlags.Middle;
+                        break;
+                }
+            }
+            else
+            {
+                clickCount = 0;
+                button = MouseButton.None;
             }
-            clickCount = eventData.clickCount;
             // TODO: delta compensate stream and texture scaling
             deltaX = (int?)eventData.delta.x;
             deltaY = (int?)eventData.delta.y;

Runtime/WebViewComponent.cs

 using bessw.Unity.WebView.ChromeDevTools;
 using bessw.Unity.WebView.ChromeDevTools.Protocol.DOM;
 using bessw.Unity.WebView.ChromeDevTools.Protocol.Input;
-using MoreLinq;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Serialization;
 using System;
 using System.Collections;
 using System.Collections.Generic;
-using System.IO;
-using System.Linq;
 using UnityEngine;
-using UnityEngine.Events;
 using UnityEngine.EventSystems;
 using UnityEngine.UI;
 
 namespace bessw.Unity.WebView
 {
     [RequireComponent(typeof(RawImage))]
+    [RequireComponent(typeof(RectTransform))]
     public class WebViewComponent : MonoBehaviour, IPointerDownHandler, IPointerMoveHandler, IPointerUpHandler, IPointerEnterHandler, IDropHandler, IPointerExitHandler
     {
         #region json serializer
@@ -66,6 +63,8 @@ namespace bessw.Unity.WebView
         //TODO: handle changed targetUrl
         public string targetUrl = "https://google.de";
 
+        public int PageScaleFactor = 3;
+
         // Start is called before the first frame update
         private void Start()
         {
@@ -80,21 +79,41 @@ namespace bessw.Unity.WebView
             var c = StartCoroutine(browser.OpenNewTab(targetUrl, (BrowserTab bt) =>
             {
                 tab = bt;
+                _ = tab.SetSizeAndScale(new Vector2Int((int)rectTransform.rect.width, (int)rectTransform.rect.height) / PageScaleFactor, PageScaleFactor);
+
                 StartCoroutine(tab.Update());
                 //StartCoroutine(createScreenshots());
-                StartCoroutine(tab.StartStream(900, 560, (frame) =>
+                StartCoroutine(tab.StartStream(tab.Size.x, tab.Size.y, (frame) =>
                 {
                     Debug.LogWarning("stream");
                     rawImage.texture = frame;
-                    rawImage.SetNativeSize();
+                    //rawImage.SetNativeSize();
                 }));
                 OnWebViewComponentReady?.Invoke();
             }));
         }
 
+        private void OnRectTransformDimensionsChange()
+        {
+            if (tab is not null)
+            {
+                var size = new Vector2Int((int)rectTransform.rect.width, (int)rectTransform.rect.height) / PageScaleFactor;
+                if (tab.Size != size)
+                {
+                    _ = tab.SetSizeAndScale(size, PageScaleFactor);
+                }
+            }
+        }
+
         public IEnumerator createScreenshots ()
         {
-            yield return tab.CreateScreenshot(900, 560, (screenshot) =>
+            yield return tab.CreateScreenshot(new()
+            {
+                Width = tab.Size.x,
+                Height = tab.Size.y,
+                Scale = PageScaleFactor
+                
+            }, (screenshot) =>
             {
                 rawImage.texture = screenshot;
                 StartCoroutine(createScreenshots());
@@ -195,12 +214,12 @@ namespace bessw.Unity.WebView
 
         private Vector2Int toBrowserCoordinates(Vector2 eventPos)
         {
-            Vector2 localPoint;
-            RectTransformUtility.ScreenPointToLocalPointInRectangle(rectTransform, eventPos, null, out localPoint);
+            RectTransformUtility.ScreenPointToLocalPointInRectangle(rectTransform, eventPos, null, out Vector2 localPoint);
 
             // invert y because the browser has y=0 on the top
             Vector2 invertedLocalPos = new Vector2(localPoint.x, rectTransform.rect.size.y - localPoint.y);
-            Vector2 browserCoorinate = tab.size / rectTransform.rect.size * invertedLocalPos;
+            Vector2 textureScale  = tab.StreamSize / rectTransform.rect.size;
+            Vector2 browserCoorinate = invertedLocalPos * textureScale / tab.PageScaleFactor;
             Debug.Log($"eventPos: {eventPos}, invertedLocalPos: {invertedLocalPos}, browserCoordinate: {browserCoorinate}");
             return new Vector2Int((int) browserCoorinate.x, (int) browserCoorinate.y);
         }

WebViewComponent.prefab

@@ -88,3 +88,4 @@ MonoBehaviour:
   m_EditorClassIdentifier: 
   headlessBrowser: 1
   targetUrl: https://google.de
+  PageScaleFactor: 3