Skip to content
Snippets Groups Projects
Verified Commit fffb896a authored by Björn Eßwein's avatar Björn Eßwein
Browse files

Pointer down, move, up is transmitted to the browser

TODO: compensate image scaling
parent 12915dd0
No related branches found
No related tags found
No related merge requests found
using ChromeDevTools; using ChromeDevTools;
using System.Collections; using System.Collections;
using UnityEngine; using UnityEngine;
using UnityEngine.EventSystems;
using UnityEngine.UI; using UnityEngine.UI;
[RequireComponent(typeof(RawImage))] [RequireComponent(typeof(RawImage))]
public class BrowserView : MonoBehaviour public class BrowserView : MonoBehaviour, IPointerDownHandler, IPointerMoveHandler, IPointerUpHandler
{ {
private Browser browser; private Browser browser;
private BrowserTab tab; private BrowserTab tab;
private RawImage rawImage; private RawImage rawImage;
private RectTransform rectTransform;
public bool headlessBrowser = true; public bool headlessBrowser = true;
//TODO: handle changed targetUrl //TODO: handle changed targetUrl
...@@ -19,6 +21,7 @@ public class BrowserView : MonoBehaviour ...@@ -19,6 +21,7 @@ public class BrowserView : MonoBehaviour
private void Start() private void Start()
{ {
rawImage = this.gameObject.GetComponent<RawImage>(); rawImage = this.gameObject.GetComponent<RawImage>();
rectTransform = this.gameObject.GetComponent<RectTransform>();
} }
private void OnEnable() private void OnEnable()
...@@ -56,6 +59,36 @@ public class BrowserView : MonoBehaviour ...@@ -56,6 +59,36 @@ public class BrowserView : MonoBehaviour
} }
public void OnPointerDown(PointerEventData eventData)
{
Vector2 pos;
if (RectTransformUtility.ScreenPointToLocalPointInRectangle(rectTransform, eventData.position, eventData.pressEventCamera, out pos)
&& rectTransform.rect.Contains(pos))
{
tab.OnPointerDown(pos, eventData);
}
}
public void OnPointerMove(PointerEventData eventData)
{
Vector2 pos;
if (RectTransformUtility.ScreenPointToLocalPointInRectangle(rectTransform, eventData.position, eventData.enterEventCamera, out pos)
&& rectTransform.rect.Contains(pos))
{
tab.OnPointerMove(pos, eventData);
}
}
public void OnPointerUp(PointerEventData eventData)
{
Vector2 pos;
if (RectTransformUtility.ScreenPointToLocalPointInRectangle(rectTransform, eventData.position, eventData.pressEventCamera, out pos)
&& rectTransform.rect.Contains(pos))
{
tab.OnPointerUp(pos, eventData);
}
}
private void OnDisable() private void OnDisable()
{ {
// TODO: do we want to close the browser when not in use? // TODO: do we want to close the browser when not in use?
......
using ChromeDevTools.Protocol.Input;
using ChromeDevTools.Protocol.Page; using ChromeDevTools.Protocol.Page;
using ChromeDevTools.Protocol.Target; using ChromeDevTools.Protocol.Target;
using System; using System;
using System.Collections; using System.Collections;
using System.Threading;
using UnityEngine; using UnityEngine;
using UnityEngine.EventSystems;
namespace ChromeDevTools namespace ChromeDevTools
{ {
...@@ -51,13 +52,13 @@ namespace ChromeDevTools ...@@ -51,13 +52,13 @@ namespace ChromeDevTools
// TODO: deregister on stop stream // TODO: deregister on stop stream
devtools.screencastFrameEventHandler += (screencastFrameEvent frameEvent) => devtools.screencastFrameEventHandler += (screencastFrameEvent frameEvent) =>
{ {
Debug.Log($"screencast frame, '{frameEvent.sessionId}'");
// send an ack for this frame // send an ack for this frame
var frameAck = new screencastFrameAck(); var frameAck = new screencastFrameAck();
frameAck.sessionId = frameEvent.sessionId; frameAck.sessionId = frameEvent.sessionId;
_ = devtools.SendCommandAsync(frameAck); _ = devtools.SendCommandAsync(frameAck);
Debug.Log($"screencast frame, '{frameEvent.sessionId}'");
// parse the base64 encoded frame to a texture // parse the base64 encoded frame to a texture
var frameTexture = new Texture2D(1, 1); // new Texture2D only works on the main thread var frameTexture = new Texture2D(1, 1); // new Texture2D only works on the main thread
frameTexture.LoadImage(Convert.FromBase64String(frameEvent.Data)); frameTexture.LoadImage(Convert.FromBase64String(frameEvent.Data));
...@@ -74,6 +75,33 @@ namespace ChromeDevTools ...@@ -74,6 +75,33 @@ namespace ChromeDevTools
return devtools.SendCommand(startScreencast); return devtools.SendCommand(startScreencast);
} }
public void OnPointerDown(Vector2 position, PointerEventData eventData)
{
Debug.Log($"PointerDown {position}:\n{eventData}");
// TODO: compensate stream and texture scaling
var mousePressedEvent = new dispatchMouseEvent(MouseEventType.MousePressed, (int)position.x, (int)position.y, eventData);
_ = devtools.SendCommandAsync(mousePressedEvent);
}
internal void OnPointerMove(Vector2 position, PointerEventData eventData)
{
Debug.Log($"OnPointerMove {position}:\n{eventData}");
// TODO: compensate stream and texture scaling
var mousePressedEvent = new dispatchMouseEvent(MouseEventType.MouseMoved, (int)position.x, (int)position.y, eventData);
_ = devtools.SendCommandAsync(mousePressedEvent);
}
public void OnPointerUp(Vector2 position, PointerEventData eventData)
{
Debug.Log($"OnPointerUp {position}:\n{eventData}");
// TODO: compensate stream and texture scaling
var mouseReleasedEvent = new dispatchMouseEvent(MouseEventType.MouseReleased, (int)position.x, (int)position.y, eventData);
_ = devtools.SendCommandAsync(mouseReleasedEvent);
}
/** /**
* close this tab * close this tab
*/ */
......
fileFormatVersion: 2
guid: 23034037f1c0a824fb8eb0d89776c15f
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using System;
using Newtonsoft.Json.Serialization;
using UnityEngine.EventSystems;
using UnityEditor.UI;
using static UnityEngine.EventSystems.PointerEventData;
namespace ChromeDevTools
{
namespace Protocol
{
namespace Input
{
/// <summary>
/// Dispatches a mouse event to the page.
/// </summary>
public class dispatchMouseEvent: IDevtoolsCommand
{
public dispatchMouseEvent(MouseEventType type, int x, int y)
{
this.type = type;
this.x = x;
this.y = y;
}
public dispatchMouseEvent(MouseEventType type, int x, int y, PointerEventData eventData) : this(type, x, y)
{
switch (eventData.button)
{
case InputButton.Left:
button = MouseButton.Left; break;
case InputButton.Right:
button = MouseButton.Right; break;
case InputButton.Middle:
button = MouseButton.Middle; break;
}
clickCount = eventData.clickCount;
// TODO: delta compensate stream and texture scaling
deltaX = (int?)eventData.delta.x;
deltaY = (int?)eventData.delta.y;
}
/// <summary>
/// Type of the mouse event.
/// Allowed Values: mousePressed, mouseReleased, mouseMoved, mouseWheel
/// </summary>
[JsonConverter(typeof(StringEnumConverter), typeof(CamelCaseNamingStrategy))]
public MouseEventType type { get; set; }
/// <summary>
/// X coordinate of the event relative to the main frame's viewport in CSS pixels.
/// </summary>
public int x { get; set; }
/// <summary>
/// Y coordinate of the event relative to the main frame's viewport in CSS pixels. 0 refers to the top of the viewport and Y increases as it proceeds towards the bottom of the viewport.
/// </summary>
public int y { get; set; }
/// <summary>
/// Bit field representing pressed modifier keys. Alt=1, Ctrl=2, Meta/Command=4, Shift=8 (default: 0).
/// </summary>
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public int? modifiers { get; set; }
/// <summary>
/// Time at which the event occurred.
/// TimeSinceEpoch UTC time in seconds, counted from January 1, 1970.
/// </summary>
[JsonConverter(typeof(UnixDateTimeConverter))]
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public DateTime? timestamp { get; set; }
/// <summary>
/// Allowed Values: none, left, middle, right, back, forward
/// Mouse button (default: "none").
/// </summary>
[JsonConverter(typeof(StringEnumConverter), typeof(CamelCaseNamingStrategy))]
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public MouseButton? button { get; set; }
/// <summary>
/// A number indicating which buttons are pressed on the mouse when a mouse event is triggered. Left=1, Right=2, Middle=4, Back=8, Forward=16, None=0.
/// </summary>
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public MouseButtonFlags? buttons { get; set; }
/// <summary>
/// Number of times the mouse button was clicked (default: 0).
/// </summary>
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public int? clickCount { get; set; }
/// <summary>
/// The normalized pressure, which has a range of [0,1] (default: 0).
/// </summary>
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public int? force { get; set; }
/// <summary>
/// The normalized tangential pressure, which has a range of [-1,1] (default: 0).
/// </summary>
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public int? tangentialPressure { get; set; }
/// <summary>
/// The plane angle between the Y-Z plane and the plane containing both the stylus axis and the Y axis, in degrees of the range [-90,90], a positive tiltX is to the right (default: 0).
/// </summary>
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public int? tiltX { get; set; }
/// <summary>
/// The plane angle between the X-Z plane and the plane containing both the stylus axis and the X axis, in degrees of the range [-90,90], a positive tiltY is towards the user (default: 0).
/// </summary>
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public int? tiltY { get; set; }
/// <summary>
/// The clockwise rotation of a pen stylus around its own major axis, in degrees in the range [0,359] (default: 0).
/// </summary>
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public int? twist { get; set; }
/// <summary>
/// X delta in CSS pixels for mouse wheel event (default: 0).
/// </summary>
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public int? deltaX { get; set; }
/// <summary>
/// Y delta in CSS pixels for mouse wheel event (default: 0).
/// </summary>
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public int? deltaY { get; set; }
/// <summary>
/// Pointer type (default: "mouse").
/// Allowed Values: mouse, pen
/// </summary>
[JsonConverter(typeof(StringEnumConverter), typeof(CamelCaseNamingStrategy))]
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public PointerType? pointerType { get; set; }
}
public enum MouseEventType
{
MousePressed, MouseReleased, MouseMoved, MouseWheel
}
public enum MouseButton
{
None, Left, Middle, Right, Back, Forward
}
[Flags]
public enum MouseButtonFlags
{
None = 0, Left = 1, Right = 2, Middle = 4, Back = 8, Forward = 16
}
public enum PointerType
{
Mouse, Pen
}
}
}
}
\ No newline at end of file
fileFormatVersion: 2
guid: 1a4364f07a30bb142ad2aae29c4e3fe1
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment