Skip to content
Snippets Groups Projects
Unverified Commit 322d9860 authored by Bjoern Esswein's avatar Bjoern Esswein
Browse files

added drag and drop capability

parent bb091e75
No related branches found
No related tags found
No related merge requests found
using ChromeDevTools;
using ChromeDevTools.Protocol.Input;
using Newtonsoft.Json;
using Newtonsoft.Json.Serialization;
using System;
using System.Collections;
using UnityEditor.Experimental.GraphView;
using UnityEngine;
using UnityEngine.EventSystems;
using UnityEngine.UI;
using UnityEngine.UIElements;
[RequireComponent(typeof(RawImage))]
public class BrowserView : MonoBehaviour, IPointerDownHandler, IPointerMoveHandler, IPointerUpHandler
{
public class BrowserView : MonoBehaviour, IPointerDownHandler, IPointerMoveHandler, IPointerUpHandler, IPointerEnterHandler, IDropHandler, IPointerExitHandler
{
#region json serializer
/// <summary>
/// JsonSerializer for the user space objects (e.g. transfering objects that have been droped on the BrowserView)
/// Users are allowed to change serializer settings to their liking.
/// </summary>
public static JsonSerializer serializer;
/// <summary>
/// Json serializer settings for the user space objects (e.g. transfering objects that have been droped on the BrowserView)
/// Users are allowed to change serializer settings to their liking.
/// </summary>
public static JsonSerializerSettings serializerSettings;
static BrowserView()
{
// initialize the JsonSerializer
serializerSettings = new JsonSerializerSettings
{
ContractResolver = new CamelCasePropertyNamesContractResolver(),
Converters = new JsonConverter[]
{
new ColorConverter(),
new Vector2Converter(),
new Vector3Converter(),
new Vector4Converter()
}
};
serializer = JsonSerializer.Create(serializerSettings);
}
#endregion json serializer
private Browser browser;
private BrowserTab tab;
......@@ -37,8 +72,8 @@ public class BrowserView : MonoBehaviour, IPointerDownHandler, IPointerMoveHandl
//StartCoroutine(createScreenshots());
StartCoroutine(tab.StartStream(900, 560, (frame) =>
{
Debug.Log("update texture");
rawImage.texture = frame;
rawImage.SetNativeSize();
}));
}));
......@@ -59,34 +94,102 @@ public class BrowserView : MonoBehaviour, IPointerDownHandler, IPointerMoveHandl
}
#region pointer event handlers
public void OnPointerDown(PointerEventData eventData)
{
Vector2 pos;
if (RectTransformUtility.ScreenPointToLocalPointInRectangle(rectTransform, eventData.position, eventData.pressEventCamera, out pos)
&& rectTransform.rect.Contains(pos))
{
Vector2Int pos = toBrowserCoordinates(eventData.position);
tab.OnPointerDown(pos, eventData);
}
}
public void OnPointerMove(PointerEventData eventData)
{
Vector2 pos;
if (RectTransformUtility.ScreenPointToLocalPointInRectangle(rectTransform, eventData.position, eventData.enterEventCamera, out pos)
&& rectTransform.rect.Contains(pos))
{
// TODO: transform eventData delta vector to browser coordinates
Vector2Int pos = toBrowserCoordinates(eventData.position);
tab.OnPointerMove(pos, eventData);
// On drag over
if (eventData.dragging)
{
Debug.LogWarning($"OnDragOver: {eventData.position}");
createDragEvent(DragEventType.dragOver, eventData);
}
}
public void OnPointerUp(PointerEventData eventData)
{
Vector2 pos;
if (RectTransformUtility.ScreenPointToLocalPointInRectangle(rectTransform, eventData.position, eventData.pressEventCamera, out pos)
&& rectTransform.rect.Contains(pos))
{
Vector2Int pos = toBrowserCoordinates(eventData.position);
tab.OnPointerUp(pos, eventData);
}
#endregion pointer event handlers
#region drag event handlers
public void OnPointerEnter(PointerEventData eventData)
{
if (eventData.dragging)
{
Debug.LogWarning($"OnDragEnter: {eventData.position}");
createDragEvent(DragEventType.dragEnter, eventData);
}
}
// TODO: OnDragMove -> PointerMove
public void OnDrop(PointerEventData eventData)
{
Debug.LogWarning($"OnDrop: {eventData.position}");
createDragEvent(DragEventType.drop, eventData);
}
public void OnPointerExit(PointerEventData eventData)
{
if (eventData.dragging)
{
Debug.LogWarning($"OnDragLeave: {eventData.position}");
//createDragEvent(DragEventType.dragCancel, eventData);
tab.CancelDragging();
// TODO: drag cancel seems to be ignored by the browser
}
}
private void createDragEvent(DragEventType dragEventType, PointerEventData eventData)
{
if (eventData.pointerDrag.TryGetComponent(out BrowserDropable dropable))
{
var position = toBrowserCoordinates(eventData.position);
var dragEvent = new dispatchDragEvent
{
type = dragEventType,
data = new DragData
{
items = new DragDataItem[]
{
new DragDataItem
{
mimeType = "application/json",
data = JsonConvert.SerializeObject(dropable, serializerSettings)
}
},
dragOperationsMask = DragOperationsMask.Copy
},
x = position.x,
y = position.y,
};
Debug.LogWarning($"DragEvent: {dragEvent.type}, {eventData.position}, '{dragEvent.data.items[0].data}'");
// send the DragEvent as drag event to the browser
tab.OnDragNDrop(dragEvent);
}
}
#endregion drag event handlers
private Vector2Int toBrowserCoordinates(Vector2 eventPos)
{
// invert y because the browser has y=0 on the top
Vector2 invertedPos = new Vector2(eventPos.x, rectTransform.rect.size.y - eventPos.y);
// TODO: fix coordinate transformation, maybe use image size instead of rectTransform, if possible
Vector2 browserCoorinate = tab.size / rectTransform.rect.size * invertedPos;
Debug.Log($"eventPos: {eventPos}, invertedPos: {invertedPos}, browserCoordinate: {browserCoorinate}");
return new Vector2Int((int) browserCoorinate.x, (int) browserCoorinate.y);
}
private void OnDisable()
......@@ -110,3 +213,5 @@ public class BrowserView : MonoBehaviour, IPointerDownHandler, IPointerMoveHandl
browser.Close();
}
}
public interface BrowserDropable { }
......@@ -21,9 +21,14 @@ namespace ChromeDevTools
public static bool headless = true;
private const int debugPort = 9222;
/* JsonSerializer */
public static JsonSerializer serializer;
public static JsonSerializerSettings serializerSettings;
/// <summary>
/// Json serializer for the devtools connection
/// </summary>
public static JsonSerializer devtoolsSerializer;
/// <summary>
/// Json serializer settings for the devtools connection
/// </summary>
public static JsonSerializerSettings devtoolsSerializerSettings;
public static CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
......@@ -46,11 +51,11 @@ namespace ChromeDevTools
static Browser()
{
// initialize the JsonSerializer
var camelCasePropertyNamesContractResolver = new CamelCasePropertyNamesContractResolver();
serializer = new JsonSerializer();
serializer.ContractResolver = camelCasePropertyNamesContractResolver;
serializerSettings = new JsonSerializerSettings();
serializerSettings.ContractResolver = camelCasePropertyNamesContractResolver;
devtoolsSerializerSettings = new JsonSerializerSettings
{
ContractResolver = new CamelCasePropertyNamesContractResolver(),
};
devtoolsSerializer = JsonSerializer.Create(devtoolsSerializerSettings);
}
/**
......@@ -113,7 +118,7 @@ namespace ChromeDevTools
{
yield return DevToolsApiRequest(true, $"/json/new?{targetUrl}", (response) =>
{
PageTargetInfo pageTarget = JsonConvert.DeserializeObject<PageTargetInfo>(response, serializerSettings);
PageTargetInfo pageTarget = JsonConvert.DeserializeObject<PageTargetInfo>(response, devtoolsSerializerSettings);
callback(new BrowserTab(pageTarget));
});
......
......@@ -13,6 +13,12 @@ namespace ChromeDevTools
{
private PageTargetInfo pageTarget;
private DevtoolsProtocolHandler devtools;
/// <summary>
/// width and height of the brower device
/// </summary>
public Vector2Int size { get; private set; }
public BrowserTab(PageTargetInfo pageTarget)
{
this.pageTarget = pageTarget;
......@@ -46,7 +52,7 @@ namespace ChromeDevTools
});
}
public IEnumerator StartStream(int width, int height, Action<Texture2D> callback)
public IEnumerator StartStream(int maxWidth, int maxHeight, Action<Texture2D> callback)
{
// register screencast frame event handler
// TODO: deregister on stop stream
......@@ -57,7 +63,10 @@ namespace ChromeDevTools
frameAck.sessionId = frameEvent.sessionId;
_ = devtools.SendCommandAsync(frameAck);
Debug.Log($"screencast frame, '{frameEvent.sessionId}'");
size = new Vector2Int(frameEvent.metadata.deviceWidth, frameEvent.metadata.deviceHeight);
Debug.Log($"screencast frame, '{frameEvent.sessionId}'; size: {size}, pageScaleFactor: {frameEvent.metadata.pageScaleFactor}");
// parse the base64 encoded frame to a texture
var frameTexture = new Texture2D(1, 1); // new Texture2D only works on the main thread
......@@ -68,40 +77,68 @@ namespace ChromeDevTools
};
var startScreencast = new startScreencast();
startScreencast.MaxWidth = width;
startScreencast.maxHeight = height;
startScreencast.MaxWidth = maxWidth;
startScreencast.maxHeight = maxHeight;
startScreencast.everyNthFrame = 1;
return devtools.SendCommand(startScreencast);
}
public void OnPointerDown(Vector2 position, PointerEventData eventData)
public void OnPointerDown(Vector2Int position, PointerEventData eventData)
{
Debug.Log($"PointerDown {position}:\n{eventData}");
//Debug.Log($"PointerDown {position}:\n{eventData}");
// TODO: compensate stream and texture scaling
var mousePressedEvent = new dispatchMouseEvent(MouseEventType.MousePressed, (int)position.x, (int)position.y, eventData);
var mousePressedEvent = new dispatchMouseEvent(MouseEventType.MousePressed, position.x, position.y, eventData);
//var mousePressedEvent = new dispatchMouseEvent(MouseEventType.MousePressed, (int)eventData.position.x, size.y - (int)eventData.position.y, eventData);
_ = devtools.SendCommandAsync(mousePressedEvent);
}
internal void OnPointerMove(Vector2 position, PointerEventData eventData)
public void OnPointerMove(Vector2Int position, PointerEventData eventData)
{
Debug.Log($"OnPointerMove {position}:\n{eventData}");
//Debug.Log($"OnPointerMove {position}:\n{eventData}");
// TODO: compensate stream and texture scaling
var mousePressedEvent = new dispatchMouseEvent(MouseEventType.MouseMoved, (int)position.x, (int)position.y, eventData);
var mousePressedEvent = new dispatchMouseEvent(MouseEventType.MouseMoved, position.x, position.y, eventData);
//var mousePressedEvent = new dispatchMouseEvent(MouseEventType.MouseMoved, (int)eventData.position.x, size.y - (int)eventData.position.y, eventData);
_ = devtools.SendCommandAsync(mousePressedEvent);
}
public void OnPointerUp(Vector2 position, PointerEventData eventData)
public void OnPointerUp(Vector2Int position, PointerEventData eventData)
{
Debug.Log($"OnPointerUp {position}:\n{eventData}");
//Debug.Log($"OnPointerUp {position}:\n{eventData}");
// TODO: compensate stream and texture scaling
var mouseReleasedEvent = new dispatchMouseEvent(MouseEventType.MouseReleased, (int)position.x, (int)position.y, eventData);
var mouseReleasedEvent = new dispatchMouseEvent(MouseEventType.MouseReleased, position.x, position.y, eventData);
//var mouseReleasedEvent = new dispatchMouseEvent(MouseEventType.MouseReleased, (int)eventData.position.x, size.y - (int)eventData.position.y, eventData);
_ = devtools.SendCommandAsync(mouseReleasedEvent);
}
public void OnDrop(Vector2Int position, DragData dragData)
{
// TODO: compensate stream and texture scaling
var dropEvent = new dispatchDragEvent
{
type = DragEventType.drop,
data = dragData,
x = position.x,
y = position.y,
};
_ = devtools.SendCommandAsync(dropEvent);
}
public void OnDragNDrop(dispatchDragEvent dragEvent)
{
// TODO: compensate stream and texture scaling
_ = devtools.SendCommandAsync(dragEvent);
}
public void CancelDragging()
{
_ = devtools.SendCommandAsync(new cancelDragging());
}
/**
* close this tab
*/
......
......@@ -95,7 +95,6 @@ namespace ChromeDevTools
{
throw new DevtoolsConnectionClosedException();
}
Debug.Log($"ws send: '{command}'");
return ws.SendAsync(Encoding.UTF8.GetBytes(command), WebSocketMessageType.Text, true, cancellationTokenSource.Token);
}
......
......@@ -7,8 +7,8 @@ namespace ChromeDevTools
///
public class DevtoolsCommandWrapper<T> where T: IDevtoolsCommand
{
private static long LAST_ID = 0;
public long Id { get; } = ++LAST_ID;
//private static long LAST_ID = 0;
public long Id { get; set; } // = ++LAST_ID;
public string Method
{
get
......
......@@ -15,6 +15,8 @@ namespace ChromeDevTools
/// </summary>
public class DevtoolsProtocolHandler
{
private static long LAST_COMMAND_ID = 0;
private IDevtoolsConnection devtools;
private ConcurrentDictionary<long, ResponseTypeAndCallback> commandResponseDict = new ConcurrentDictionary<long, ResponseTypeAndCallback>();
......@@ -92,7 +94,7 @@ namespace ChromeDevTools
}
// deserialize the result
IDevtoolsResponse commandResponse = (IDevtoolsResponse) response.Result.ToObject(responseTypeAndCallback.responseType, Browser.serializer);
IDevtoolsResponse commandResponse = (IDevtoolsResponse) response.Result.ToObject(responseTypeAndCallback.responseType, Browser.devtoolsSerializer);
// pass the response to the callback
responseTypeAndCallback.callback(commandResponse);
......@@ -106,10 +108,10 @@ namespace ChromeDevTools
switch (ev.Method)
{
case "Page.screencastFrame":
screencastFrameEventHandler?.Invoke( ev.Params.ToObject<screencastFrameEvent>(Browser.serializer) );
screencastFrameEventHandler?.Invoke( ev.Params.ToObject<screencastFrameEvent>(Browser.devtoolsSerializer) );
break;
case "Page.screencastVisibilityChanged":
screencastVisibilityChangedEventHandler?.Invoke( ev.Params.ToObject<screencastVisibilityChangedEvent>(Browser.serializer) );
screencastVisibilityChangedEventHandler?.Invoke( ev.Params.ToObject<screencastVisibilityChangedEvent>(Browser.devtoolsSerializer) );
break;
default:
throw new UnexpectedMessageException($"Event of type '{ev}' is not implemented");
......@@ -155,7 +157,10 @@ namespace ChromeDevTools
public async Task SendCommandAsync<T>(T command, Action<IDevtoolsResponse> callback) where T : IDevtoolsCommandWithResponse
{
// apply the message wrapper
var wrappedCommand = new DevtoolsCommandWrapper<T>(command);
var wrappedCommand = new DevtoolsCommandWrapper<T>(command)
{
Id = ++LAST_COMMAND_ID
};
// get the response type from the commands attribute
CommandResponseAttribute cra = (CommandResponseAttribute)Attribute.GetCustomAttribute(
......@@ -176,7 +181,7 @@ namespace ChromeDevTools
}
// json serialize the command and send it
var json = JsonConvert.SerializeObject(wrappedCommand, Browser.serializerSettings);
var json = JsonConvert.SerializeObject(wrappedCommand, Browser.devtoolsSerializerSettings);
Debug.Log($"ws send: '{json}'");
await devtools.SendCommandAsync(json);
}
......@@ -189,10 +194,13 @@ namespace ChromeDevTools
public async Task SendCommandAsync<T>(T command) where T : IDevtoolsCommand
{
// apply the message wrapper
var wrappedCommand = new DevtoolsCommandWrapper<T>(command);
var wrappedCommand = new DevtoolsCommandWrapper<T>(command)
{
Id = ++LAST_COMMAND_ID
};
// json serialize the command and send it
var json = JsonConvert.SerializeObject(wrappedCommand, Browser.serializerSettings);
var json = JsonConvert.SerializeObject(wrappedCommand, Browser.devtoolsSerializerSettings);
Debug.Log($"ws send: '{json}'");
await devtools.SendCommandAsync(json);
}
......
......@@ -3,7 +3,6 @@ using Newtonsoft.Json.Converters;
using System;
using Newtonsoft.Json.Serialization;
using UnityEngine.EventSystems;
using UnityEditor.UI;
using static UnityEngine.EventSystems.PointerEventData;
namespace ChromeDevTools
......@@ -12,6 +11,7 @@ namespace ChromeDevTools
{
namespace Input
{
#region dispatchMouseEvent
/// <summary>
/// Dispatches a mouse event to the page.
/// </summary>
......@@ -58,7 +58,7 @@ namespace ChromeDevTools
/// Bit field representing pressed modifier keys. Alt=1, Ctrl=2, Meta/Command=4, Shift=8 (default: 0).
/// </summary>
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public int? modifiers { get; set; }
public ModifierKeyFlags? modifiers { get; set; }
/// <summary>
/// Time at which the event occurred.
/// TimeSinceEpoch UTC time in seconds, counted from January 1, 1970.
......@@ -147,6 +147,90 @@ namespace ChromeDevTools
{
Mouse, Pen
}
#endregion dispatchMouseEvent
#region dispatchDragEvent
/// <summary>
/// Cancels any active dragging in the page.
/// </summary>
public class cancelDragging : IDevtoolsCommand
{
}
/// <summary>
/// Dispatches a drag event into the page.
/// </summary>
public class dispatchDragEvent : IDevtoolsCommand
{
/// <summary>
/// Type of the drag event.
/// Allowed Values: dragEnter, dragOver, drop, dragCancel
/// </summary>
[JsonConverter(typeof(StringEnumConverter), typeof(CamelCaseNamingStrategy))]
public DragEventType type { get; set; }
public int x { get; set; }
public int y { get; set; }
public DragData data { get; set; }
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public ModifierKeyFlags? modifiers { get; set; }
}
public enum DragEventType
{
dragEnter, dragOver, drop, dragCancel
}
public class DragData
{
public DragDataItem[] items { get; set; }
/// <summary>
/// List of filenames that should be included when dropping
/// </summary>
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public string[]? files { get; set; }
public DragOperationsMask dragOperationsMask { get; set; }
}
[Flags]
public enum DragOperationsMask
{
Copy = 1, Link = 2, Move = 16
}
public class DragDataItem
{
/// <summary>
/// Mime type of the dragged data.
/// </summary>
public string mimeType { get; set; } = "";
/// <summary>
/// Depending of the value of mimeType, it contains the dragged link, text, HTML markup or any other data.
/// </summary>
public string data { get; set; } = "";
/// <summary>
/// Title associated with a link. Only valid when mimeType == "text/uri-list".
/// </summary>
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public string? title { get; set; } = null;
/// <summary>
/// Stores the base URL for the contained markup. Only valid when mimeType == "text/html".
/// </summary>
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public string? baseURL { get; set; } = null;
}
#endregion dispatchDragEvent
/// <summary>
/// Bit field representing pressed modifier keys. Alt=1, Ctrl=2, Meta/Command=4, Shift=8 (default: 0).
/// </summary>
[Flags]
public enum ModifierKeyFlags
{
None = 0, Alt = 1, Ctrl = 2, Meta_Command = 4, Shift = 8
}
}
}
}
\ No newline at end of file
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using UnityEngine;
// Solutions to prevent serialization errors. Seen in https://forum.unity.com/threads/jsonserializationexception-self-referencing-loop-detected.1264253/
// Newtonsoft struggles serializing structs like Vector3 because it has a property .normalized
// that references Vector3, and thus entering a self-reference loop throwing circular reference error.
// Add the class to BootstrapJsonParser
public class ColorConverter : JsonConverter<Color>
{
public override void WriteJson(JsonWriter writer, Color value, JsonSerializer serializer)
{
JObject obj = new JObject() { ["r"] = value.r, ["g"] = value.g, ["b"] = value.b, ["a"] = value.a };
obj.WriteTo(writer);
}
public override Color ReadJson(JsonReader reader, Type objectType, Color existingValue, bool hasExistingValue, JsonSerializer serializer)
{
JObject obj = JObject.Load(reader);
return new Color((float)obj.GetValue("r"), (float)obj.GetValue("g"), (float)obj.GetValue("b"), (float)obj.GetValue("a"));
}
}
public class Vector2Converter : JsonConverter<Vector2>
{
public override void WriteJson(JsonWriter writer, Vector2 value, JsonSerializer serializer)
{
JObject obj = new JObject() { ["x"] = value.x, ["y"] = value.y };
obj.WriteTo(writer);
}
public override Vector2 ReadJson(JsonReader reader, Type objectType, Vector2 existingValue, bool hasExistingValue, JsonSerializer serializer)
{
JObject obj = JObject.Load(reader);
return new Vector2((float)obj.GetValue("x"), (float)obj.GetValue("y"));
}
}
public class Vector3Converter : JsonConverter<Vector3>
{
public override void WriteJson(JsonWriter writer, Vector3 value, JsonSerializer serializer)
{
JObject obj = new JObject() { ["x"] = value.x, ["y"] = value.y, ["z"] = value.z };
obj.WriteTo(writer);
}
public override Vector3 ReadJson(JsonReader reader, Type objectType, Vector3 existingValue, bool hasExistingValue, JsonSerializer serializer)
{
JObject obj = JObject.Load(reader);
return new Vector3((float)obj.GetValue("x"), (float)obj.GetValue("y"), (float)obj.GetValue("z"));
}
}
public class Vector4Converter : JsonConverter<Vector4>
{
public override void WriteJson(JsonWriter writer, Vector4 value, JsonSerializer serializer)
{
JObject obj = new JObject() { ["x"] = value.x, ["y"] = value.y, ["z"] = value.z, ["w"] = value.w };
obj.WriteTo(writer);
}
public override Vector4 ReadJson(JsonReader reader, Type objectType, Vector4 existingValue, bool hasExistingValue, JsonSerializer serializer)
{
JObject obj = JObject.Load(reader);
return new Vector4((float)obj.GetValue("x"), (float)obj.GetValue("y"), (float)obj.GetValue("z"), (float)obj.GetValue("w"));
}
}
fileFormatVersion: 2
guid: 212e3789f1271d042bc798f031d45f58
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment