Skip to content
Snippets Groups Projects
Verified Commit 9b78d579 authored by Björn Eßwein's avatar Björn Eßwein
Browse files

implemented command serialization and response deserialization

parent 704babea
No related branches found
No related tags found
No related merge requests found
Showing with 271 additions and 143 deletions
using ChromeDevTools;
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.UI;
//using Unity.Networking.Transport;
public class BrowserView : MonoBehaviour
[RequireComponent(typeof(RawImage))]
public class BrowserView : MonoBehaviour //TODO: Extends RawImage instead?
{
private Browser browser;
private BrowserTab tab;
......@@ -21,21 +24,26 @@ public class BrowserView : MonoBehaviour
void OnEnable()
{
browser = Browser.getInstance();
//BrowserView.LaunchBrowser();
//UnityEngine.Debug.Log($"ws State:\n{ws.State}");
//StartCoroutine(GetOpenTabs());
var c = StartCoroutine(browser.OpenNewTab(targetUrl, (BrowserTab bt) =>
{
tab = bt;
StartCoroutine(tab.CreateScreenshot(1920, 1080, (screenshot) =>
{
rawImage.texture = screenshot;
}));
StartCoroutine(tab.ReadWsMessage());
StartCoroutine(createScreenshots());
}));
}
public IEnumerator createScreenshots ()
{
yield return tab.CreateScreenshot(900, 560, (screenshot) =>
{
rawImage.texture = screenshot;
StartCoroutine(createScreenshots());
});
}
// Update is called once per frame
void Update()
{
......
......@@ -68,7 +68,7 @@ namespace ChromeDevTools
// set headlessBrowser to false to see the browser window
if (headlessBrowser)
{
Browser.browserProcess.StartInfo.Arguments = string.Concat(Browser.browserProcess.StartInfo.Arguments, "--headless");
Browser.browserProcess.StartInfo.Arguments = string.Concat(Browser.browserProcess.StartInfo.Arguments, " --headless=new");
}
else
{
......@@ -76,7 +76,7 @@ namespace ChromeDevTools
}
Browser.browserProcess.Start();
UnityEngine.Debug.Log("launched chrome");
UnityEngine.Debug.Log($"launched '{Browser.browserProcess.StartInfo.FileName} {Browser.browserProcess.StartInfo.Arguments}'");
}
}
......
using ChromeDevTools.Protocol;
using ChromeDevTools.Protocol.Page;
using ChromeDevTools.Protocol.Target;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.Collections;
using System.Collections.Concurrent;
using System.IO;
using System.Net.WebSockets;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using UnityEngine;
namespace ChromeDevTools
{
record ResponseTypeAndCallback
{
public ResponseTypeAndCallback(Type responseType, Action<IDevtoolsResponse> callback)
{
this.responseType = responseType;
this.callback = callback;
}
public Type responseType { get; }
public Action<IDevtoolsResponse> callback { get; }
}
public class BrowserTab
{
private ClientWebSocket ws = new ClientWebSocket();
private PageTargetInfo pageTarget;
private ConcurrentDictionary<long, ResponseTypeAndCallback> commandResponseDict = new ConcurrentDictionary<long, ResponseTypeAndCallback>();
public BrowserTab(PageTargetInfo pageTarget)
{
......@@ -26,91 +40,137 @@ namespace ChromeDevTools
/**
* Note: Forebg wl compatibility you need to use the JavaScript plugin, see also https://docs.unity3d.com/Manual/webgl-networking.html#UsingWebSockets
*/
var t = ws.ConnectAsync(new Uri(pageTarget.WebSocketDebuggerUrl), Browser.cancellationTokenSource.Token);
ws.ConnectAsync(new Uri(pageTarget.WebSocketDebuggerUrl), Browser.cancellationTokenSource.Token);
}
/// <summary>
/// send a json serializable message over the devtools websocket
/// json serializes and sends a command over the devtools websocket
/// </summary>
/// <typeparam name="T">IDevtoolsCommand</typeparam>
/// <param name="jsonSerializableMessage"></param>
/// <param name="command"></param>
/// <returns></returns>
private async Task SendWsMessage<T>(T jsonSerializableMessage)
private IEnumerator SendWsMessage<T>(T command, System.Action<IDevtoolsResponse> callback) where T: IDevtoolsCommand
{
// wait if the websocket is not yet open
if (ws.State != WebSocketState.Open)
{
if (ws.State != WebSocketState.Open) throw new InvalidOperationException($"WebSocket is not open: ws.State = {ws.State}");
yield return new WaitUntil(() => ws.State == WebSocketState.Open);
}
// apply the message wrapper
var wrappedCommand = new DevtoolsCommandWrapper<T>(command);
var json = JsonConvert.SerializeObject(jsonSerializableMessage, Browser.serializerSettings);
// get the response type from the commands attribute
CommandResponseAttribute cra = (CommandResponseAttribute) Attribute.GetCustomAttribute(
command.GetType(),
typeof(CommandResponseAttribute)
);
Type responseType = cra.responseType;
// register the response callback for this command id
if (!commandResponseDict.TryAdd(wrappedCommand.Id, new ResponseTypeAndCallback(responseType, callback)))
{
throw new InvalidOperationException($"could not add response callback for command '{wrappedCommand.Id}' to commandResponseDict");
}
// json serialize the command and send it
var json = JsonConvert.SerializeObject(wrappedCommand, Browser.serializerSettings);
UnityEngine.Debug.Log($"ws send: '{json}'");
//await ws.SendAsync(Encoding.UTF8.GetBytes(json), WebSocketMessageType.Text, true, Browser.cancellationTokenSource.Token);
var sendTask = ws.SendAsync(Encoding.UTF8.GetBytes(json), WebSocketMessageType.Text, true, Browser.cancellationTokenSource.Token);
// wait until the command has been send
yield return new WaitUntil(() => sendTask.IsCompleted);
}
/// <summary>
/// read and deserialize a json message from the devtools websocket
/// </summary>
/// <typeparam name="T">IDevtoolsCommandResponse</typeparam>
/// <returns></returns>
private async Task<JObject> ReadWsMessage<T>()
public IEnumerator ReadWsMessage()
{
// wait if the websocket is not yet open
if (ws.State != WebSocketState.Open)
{
yield return new WaitUntil(() => ws.State == WebSocketState.Open);
}
// start the message receive loop (it will exit when the coroutine is stopped)
while (true)
{
if (ws.State != WebSocketState.Open) throw new InvalidOperationException($"WebSocket is not open: ws.State = {ws.State}");
string msgString;
// create a MemoryStream to reconstruct the message
using (var ms = new MemoryStream())
{
WebSocketReceiveResult result;
/* A part of the message will be read into the message buffer and than be transfered
* to the MemoryStream. This will be repeated until the complete message has been received.
*/
do
{
var messageBuffer = WebSocket.CreateClientBuffer(1024, 16);
result = await ws.ReceiveAsync(messageBuffer, Browser.cancellationTokenSource.Token);
var reseiveTask = ws.ReceiveAsync(messageBuffer, Browser.cancellationTokenSource.Token);
// yield the coroutine until the async Task "reseiveTask" is completed
yield return new WaitUntil(() => reseiveTask.IsCompleted);
result = reseiveTask.Result;
// write the messageBuffer to the MemoryStream
ms.Write(messageBuffer.Array, messageBuffer.Offset, result.Count);
}
while (!result.EndOfMessage);
if (result.MessageType == WebSocketMessageType.Text)
// If the webSocket message type isn't text ignore this message
if (!(result.MessageType == WebSocketMessageType.Text))
{
var msgString = Encoding.UTF8.GetString(ms.ToArray());
UnityEngine.Debug.Log($"ws received: {msgString}");
var response = JObject.Parse(msgString);
UnityEngine.Debug.Log($"ws received: {response["method"]}");
UnityEngine.Debug.Log($"ws received: {response["result"]}");
//return response.ToObject<DevtoolsCommandResponse<CaptureScreenshotCommandResponse>>(serializer);
return response;
Debug.LogError($"Unexpected WebSocketMessageType: {result.MessageType}");
continue;
}
// convert the message stream to string
msgString = Encoding.UTF8.GetString(ms.ToArray());
Debug.Log($"ws reseived: '{msgString}'");
}
else
// deserialize the devtools response wrapper
var response = JsonConvert.DeserializeObject<DevtoolsResponseWrapper>(msgString, Browser.serializerSettings);
// get the callback and the type for this response
ResponseTypeAndCallback responseTypeAndCallback;
if (!commandResponseDict.TryRemove(response.Id, out responseTypeAndCallback))
{
throw new InvalidDataException($"Unexpected WebSocketMessageType: {result.MessageType}");
throw new InvalidOperationException($"There is no command waiting for the response '{response.Id}'");
}
//ms.Seek(0, SeekOrigin.Begin);
//ms.Position = 0;
// deserialize the result
IDevtoolsResponse commandResponse = (IDevtoolsResponse) response.Result.ToObject(responseTypeAndCallback.responseType, Browser.serializer);
// pass the response to the callback
responseTypeAndCallback.callback(commandResponse);
}
}
public IEnumerator CreateScreenshot(double width, double height, System.Action<Texture2D> callback)
public IEnumerator CreateScreenshot(double width, double height, Action<Texture2D> callback)
{
yield return new WaitUntil(() => ws.State == WebSocketState.Open);
var screenshotCommand = new CaptureScreenshotCommand();
var screenshotCommand = new captureScreenshot();
screenshotCommand.Clip = new Protocol.Types.Viewport();
screenshotCommand.Clip.Width = width;
screenshotCommand.Clip.Height = height;
//var sendTask =
SendWsMessage(screenshotCommand);
var screenshotCmd = "{\"method\":\"Page.captureScreenshot\",\"id\":1,\"params\": {\"clip\":{\"x\":0,\"y\":0,\"width\":900,\"height\":560,\"scale\":1}}}";
UnityEngine.Debug.Log($"SendAsy: '{screenshotCmd}'");
var sendTask = ws.SendAsync(Encoding.UTF8.GetBytes(screenshotCmd), WebSocketMessageType.Text, true, Browser.cancellationTokenSource.Token);
yield return new WaitUntil(() => sendTask.IsCompleted);
var receiveTask = ReadWsMessage<CaptureScreenshotCommandResponse>();
yield return new WaitUntil(() => receiveTask.IsCompleted);
Debug.Log($"screenshot result: '{receiveTask.Result["result"]["data"]}'");
//var screenshot = JsonConvert.DeserializeObject<CaptureScreenshotCommandResponse>(receiveTask.Result.Result, serializerSettings);
//UnityEngine.Debug.Log($"screenshot {receiveTask.Result.Id}: '{screenshot.Data}'");
byte[] imgBytes = Convert.FromBase64String(receiveTask.Result["result"]["data"].ToString());
return SendWsMessage(screenshotCommand,
(response) => {
CaptureScreenshotCommandResponse screenshotResponse = (CaptureScreenshotCommandResponse) response;
Debug.Log($"imgBytes.Length {imgBytes.Length}");
var myTexture = new Texture2D(1, 1);
myTexture.LoadImage(imgBytes);
// parse the base64 encoded screenshot to a texture
var screenshotTexture = new Texture2D(1, 1);
screenshotTexture.LoadImage( Convert.FromBase64String(screenshotResponse.Data) );
//myTexture.Apply();
callback(myTexture);
// return the texture via callback
callback(screenshotTexture);
});
}
/**
......@@ -119,16 +179,16 @@ namespace ChromeDevTools
public void Close()
{
Debug.Log($"BrowserTab close called for: '{pageTarget.Url}'");
SendWsMessage(new CloseTargetCommand(pageTarget.Id))
.ContinueWith(delegate
//TODO: fix SendWsMessage without coroutine
SendWsMessage(new closeTarget(pageTarget.Id), delegate
{
ws.CloseAsync(WebSocketCloseStatus.NormalClosure, "tab closed", CancellationToken.None);
})
ws.CloseAsync(WebSocketCloseStatus.NormalClosure, "tab closed", CancellationToken.None)
.ContinueWith(delegate
{
Debug.Log($"ws CloseAsync State:\n{ws.State}");
ws.Dispose();
});
});
}
}
}
using Newtonsoft.Json;
using ChromeDevTools.Protocol.Types;
namespace ChromeDevTools
{
namespace Protocol
{
/// <summary>
/// Capture page screenshot.
/// </summary>
public class CaptureScreenshotCommand : DevtoolsCommand
{
public string Method { get; } = "Page.captureScreenshot";
/// <summary>
/// Gets or sets Image compression format (defaults to png).
/// </summary>
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public string Format { get; set; }
/// <summary>
/// Gets or sets Compression quality from range [0..100] (jpeg only).
/// </summary>
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public long? Quality { get; set; }
/// <summary>
/// Gets or sets Capture the screenshot of a given region only.
/// </summary>
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public Viewport Clip { get; set; }
/// <summary>
/// Gets or sets Capture the screenshot from the surface, rather than the view. Defaults to true.
/// </summary>
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public bool? FromSurface { get; set; }
}
/// <summary>
/// Capture page screenshot response.
/// </summary>
public class CaptureScreenshotCommandResponse
{
/// <summary>
/// Gets or sets Base64-encoded image data.
/// </summary>
public string Data { get; }
}
}
}
\ No newline at end of file
using System;
namespace ChromeDevTools
{
namespace Protocol
{
[AttributeUsage(AttributeTargets.Class)]
public class CommandResponseAttribute : Attribute
{
public Type responseType { get; }
public CommandResponseAttribute(Type responseType) => this.responseType = responseType;
}
}
}
\ No newline at end of file
fileFormatVersion: 2
guid: 38276a38657a062489df0940470c1ec7
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:
using Newtonsoft.Json.Linq;
using System;
namespace ChromeDevTools
{
namespace Protocol
......@@ -5,21 +8,37 @@ namespace ChromeDevTools
///
/// Every devtools command has an id and a method
///
public class DevtoolsCommand
public class DevtoolsCommandWrapper<T> where T: IDevtoolsCommand
{
private static long LAST_ID = 0;
public long Id { get; } = ++LAST_ID;
public string Method { get; set; }
public string Method
{
get
{
// Remove the namespace prefix 'ChromeDevTools.Protocol.'
return Params.GetType().FullName.Substring("ChromeDevTools.Protocol.".Length);
}
}
public T Params { get; set; }
public DevtoolsCommandWrapper(T command) => Params = command;
}
public interface IDevtoolsCommand {}
public class DevtoolsResponseWrapper
{
public long Id { get; set; }
public JObject Result { get; set; }
}
///
/// Every devtools command response has the same id and a method as the corresponding command
///
public class DevtoolsCommandResponse<T>
public class DevtoolsResponseWrapper<T>: DevtoolsResponseWrapper where T: IDevtoolsResponse
{
public long Id { get; }
public string Method { get; }
public T Result { get; }
public new T Result { get; set; }
}
public interface IDevtoolsResponse {}
}
}
\ No newline at end of file
fileFormatVersion: 2
guid: a4d51b1ccebecb84194fe93377034264
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:
using Newtonsoft.Json;
using ChromeDevTools.Protocol.Types;
namespace ChromeDevTools
{
namespace Protocol
{
namespace Page
{
/// <summary>
/// Capture page screenshot.
/// </summary>
[CommandResponseAttribute(typeof(CaptureScreenshotCommandResponse))]
public class captureScreenshot : IDevtoolsCommand
{
/// <summary>
/// Gets or sets Image compression format (defaults to png).
/// </summary>
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public string Format { get; set; }
/// <summary>
/// Gets or sets Compression quality from range [0..100] (jpeg only).
/// </summary>
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public long? Quality { get; set; }
/// <summary>
/// Gets or sets Capture the screenshot of a given region only.
/// </summary>
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public Viewport Clip { get; set; }
/// <summary>
/// Gets or sets Capture the screenshot from the surface, rather than the view. Defaults to true.
/// </summary>
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public bool? FromSurface { get; set; }
}
/// <summary>
/// Capture page screenshot response.
/// </summary>
public class CaptureScreenshotCommandResponse : IDevtoolsResponse
{
/// <summary>
/// Gets or sets Base64-encoded image data.
/// </summary>
public string Data { get; set; }
}
}
}
}
\ No newline at end of file
using Newtonsoft.Json;
using ChromeDevTools.Protocol.Types;
namespace ChromeDevTools
{
namespace Protocol
{
namespace Target
{
/// <summary>
/// Closes the target. If the target is a page that gets closed too.
/// </summary>
public class CloseTargetCommand : DevtoolsCommand
[CommandResponseAttribute(typeof(CloseTargetCommandResponse))]
public class closeTarget : IDevtoolsCommand
{
[JsonIgnore]
public string Method { get; } = "Target.closeTarget";
public CloseTargetCommand(string targetId) => TargetId = targetId;
public closeTarget(string targetId) => TargetId = targetId;
/// <summary>
/// Gets or sets TargetId
......@@ -32,3 +35,4 @@ namespace ChromeDevTools
}
}
}
}
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment