Skip to content
Snippets Groups Projects
Verified Commit 12915dd0 authored by Björn Eßwein's avatar Björn Eßwein
Browse files

Restructured the devtools connection classes and implemented streaming via Page.startScreencast

parent b2a5aec0
No related branches found
No related tags found
No related merge requests found
Showing
with 485 additions and 172 deletions
......@@ -30,8 +30,13 @@ public class BrowserView : MonoBehaviour
var c = StartCoroutine(browser.OpenNewTab(targetUrl, (BrowserTab bt) =>
{
tab = bt;
StartCoroutine(tab.ReadWsMessage());
StartCoroutine(createScreenshots());
StartCoroutine(tab.Update());
//StartCoroutine(createScreenshots());
StartCoroutine(tab.StartStream(900, 560, (frame) =>
{
Debug.Log("update texture");
rawImage.texture = frame;
}));
}));
}
......
using ChromeDevTools.Protocol;
using ChromeDevTools.Protocol.Page;
using ChromeDevTools.Protocol.Target;
using Newtonsoft.Json;
using System;
using System.Collections;
using System.Collections.Concurrent;
using System.IO;
using System.Net.WebSockets;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using UnityEngine;
namespace ChromeDevTools
{
record ResponseTypeAndCallback
{
public ResponseTypeAndCallback(Type responseType, Action<IDevtoolsResponse> callback)
{
this.responseType = responseType;
this.callback = callback;
}
public Type responseType { get; }
public Action<IDevtoolsResponse> callback { get; }
}
public class BrowserTab
{
private ClientWebSocket ws = new ClientWebSocket();
private PageTargetInfo pageTarget;
private ConcurrentDictionary<long, ResponseTypeAndCallback> commandResponseDict = new ConcurrentDictionary<long, ResponseTypeAndCallback>();
private DevtoolsProtocolHandler devtools;
public BrowserTab(PageTargetInfo pageTarget)
{
this.pageTarget = pageTarget;
Debug.Log($"tab WebSocket: '{pageTarget.WebSocketDebuggerUrl}'");
// open remote devtools websocket connection
/**
* Note: For webgl compatibility you need to use the JavaScript plugin, see also https://docs.unity3d.com/Manual/webgl-networking.html#UsingWebSockets
*/
ws.ConnectAsync(new Uri(pageTarget.WebSocketDebuggerUrl), Browser.cancellationTokenSource.Token);
}
/// <summary>
/// Coroutine wrapper for _SendWsMessagel
/// json serializes and sends a command over the devtools websocket
/// </summary>
/// <typeparam name="T">IDevtoolsCommand</typeparam>
/// <param name="command"></param>
/// <returns></returns>
private IEnumerator SendWsMessage<T>(T command, System.Action<IDevtoolsResponse> callback) where T : IDevtoolsCommand
{
// wait if the websocket is not yet open
if (ws.State != WebSocketState.Open)
{
yield return new WaitUntil(() => ws.State == WebSocketState.Open);
}
var sendTask = _SendWsMessage(command, callback);
// wait until the command has been send
yield return new WaitUntil(() => sendTask.IsCompleted);
}
/// <summary>
/// json serializes and sends a command over the devtools websocket
/// </summary>
/// <typeparam name="T">IDevtoolsCommand</typeparam>
/// <param name="command"></param>
/// <returns>Task that resoves then the command is send</returns>
private async Task _SendWsMessage<T>(T command, System.Action<IDevtoolsResponse> callback) where T : IDevtoolsCommand
{
// wait if the websocket is not yet open
if (ws.State != WebSocketState.Open)
{
throw new InvalidOperationException("Websocket is not open");
}
// apply the message wrapper
var wrappedCommand = new DevtoolsCommandWrapper<T>(command);
// get the response type from the commands attribute
CommandResponseAttribute cra = (CommandResponseAttribute) Attribute.GetCustomAttribute(
command.GetType(),
typeof(CommandResponseAttribute)
);
Type responseType = cra.responseType;
// register the response callback for this command id
if (!commandResponseDict.TryAdd(wrappedCommand.Id, new ResponseTypeAndCallback(responseType, callback)))
{
throw new InvalidOperationException($"could not add response callback for command '{wrappedCommand.Id}' to commandResponseDict");
}
// json serialize the command and send it
var json = JsonConvert.SerializeObject(wrappedCommand, Browser.serializerSettings);
Debug.Log($"ws send: '{json}'");
await ws.SendAsync(Encoding.UTF8.GetBytes(json), WebSocketMessageType.Text, true, Browser.cancellationTokenSource.Token);
}
/// <summary>
/// read and deserialize a json message from the devtools websocket
/// </summary>
/// <returns></returns>
public IEnumerator ReadWsMessage()
{
// wait if the websocket is not yet open
if (ws.State != WebSocketState.Open)
{
yield return new WaitUntil(() => ws.State == WebSocketState.Open);
}
// start the message receive loop (it will exit when the coroutine is stopped)
while (true)
{
async Task<string> read()
{
// create a MemoryStream to reconstruct the message
using (var ms = new MemoryStream())
{
WebSocketReceiveResult result;
/* A part of the message will be read into the message buffer and than be transfered
* to the MemoryStream. This will be repeated until the complete message has been received.
*/
do
{
var messageBuffer = WebSocket.CreateClientBuffer(1024, 16);
result = await ws.ReceiveAsync(messageBuffer, Browser.cancellationTokenSource.Token);
// write the messageBuffer to the MemoryStream
ms.Write(messageBuffer.Array, messageBuffer.Offset, result.Count);
} while (!result.EndOfMessage);
// If the webSocket message type isn't text ignore this message
if (!(result.MessageType == WebSocketMessageType.Text))
{
Debug.LogError($"Unexpected WebSocketMessageType: {result.MessageType}");
return await read();
this.devtools = new DevtoolsProtocolHandler(new DevtoolsWebsocket(pageTarget.WebSocketDebuggerUrl));
}
// convert the message stream to string
return Encoding.UTF8.GetString(ms.ToArray());
}
}
var readTask = read();
// yield the coroutine until the async Task "reseiveTask" is completed
yield return new WaitUntil(() => readTask.IsCompleted);
string msgString = readTask.Result;
Debug.Log($"ws reseived: '{msgString}'");
// deserialize the devtools response wrapper
var response = JsonConvert.DeserializeObject<DevtoolsResponseWrapper>(msgString, Browser.serializerSettings);
// get the callback and the type for this response
ResponseTypeAndCallback responseTypeAndCallback;
if (!commandResponseDict.TryRemove(response.Id, out responseTypeAndCallback))
public IEnumerator Update()
{
throw new InvalidOperationException($"There is no command waiting for the response '{response.Id}'");
}
// deserialize the result
IDevtoolsResponse commandResponse = (IDevtoolsResponse) response.Result.ToObject(responseTypeAndCallback.responseType, Browser.serializer);
// pass the response to the callback
responseTypeAndCallback.callback(commandResponse);
}
yield return devtools.readLoop();
}
public IEnumerator CreateScreenshot(double width, double height, Action<Texture2D> callback)
......@@ -182,35 +32,57 @@ namespace ChromeDevTools
screenshotCommand.Clip.Width = width;
screenshotCommand.Clip.Height = height;
return SendWsMessage(screenshotCommand,
return devtools.SendCommand(screenshotCommand,
(response) => {
CaptureScreenshotCommandResponse screenshotResponse = (CaptureScreenshotCommandResponse) response;
// parse the base64 encoded screenshot to a texture
var screenshotTexture = new Texture2D(1, 1);
screenshotTexture.LoadImage( Convert.FromBase64String(screenshotResponse.Data) );
//myTexture.Apply();
// return the texture via callback
callback(screenshotTexture);
});
}
public IEnumerator StartStream(int width, int height, Action<Texture2D> callback)
{
// register screencast frame event handler
// TODO: deregister on stop stream
devtools.screencastFrameEventHandler += (screencastFrameEvent frameEvent) =>
{
Debug.Log($"screencast frame, '{frameEvent.sessionId}'");
// send an ack for this frame
var frameAck = new screencastFrameAck();
frameAck.sessionId = frameEvent.sessionId;
_ = devtools.SendCommandAsync(frameAck);
// parse the base64 encoded frame to a texture
var frameTexture = new Texture2D(1, 1); // new Texture2D only works on the main thread
frameTexture.LoadImage(Convert.FromBase64String(frameEvent.Data));
// return the texture via callback
callback(frameTexture);
};
var startScreencast = new startScreencast();
startScreencast.MaxWidth = width;
startScreencast.maxHeight = height;
startScreencast.everyNthFrame = 1;
return devtools.SendCommand(startScreencast);
}
/**
* close this tab
*/
public void Close()
{
Debug.Log($"BrowserTab close called for: '{pageTarget.Url}'");
//TODO: fix SendWsMessage without coroutine
_ = _SendWsMessage(new closeTarget(pageTarget.Id), delegate
_ = devtools.SendCommandAsync(new closeTarget(pageTarget.Id), delegate
{
ws.CloseAsync(WebSocketCloseStatus.NormalClosure, "tab closed", CancellationToken.None)
.ContinueWith(delegate
{
Debug.Log($"ws CloseAsync State:\n{ws.State}");
ws.Dispose();
});
devtools.Dispose();
});
}
}
......
using ChromeDevTools.Protocol;
using System;
using System.IO;
using System.Net.WebSockets;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using UnityEngine;
namespace ChromeDevTools
{
/// <summary>
/// Manages a chrome devtools websocket connection and parses the messages to C# objects
/// </summary>
public class DevtoolsWebsocket: IDevtoolsConnection
{
private Uri webSocketDebuggerUrl;
private ClientWebSocket ws = new ClientWebSocket();
private CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
public event Action<string> receivedMessage;
public DevtoolsWebsocket(string webSocketDebuggerUrl) : this(new Uri(webSocketDebuggerUrl)) { }
public DevtoolsWebsocket(Uri webSocketDebuggerUrl)
{
this.webSocketDebuggerUrl = webSocketDebuggerUrl;
}
public Task OpenAsync()
{
// open remote devtools websocket connection
/**
* Note: For webgl compatibility you need to use the JavaScript plugin, see also https://docs.unity3d.com/Manual/webgl-networking.html#UsingWebSockets
*/
return ws.ConnectAsync(this.webSocketDebuggerUrl, cancellationTokenSource.Token);
}
/// <summary>
/// Experimential! synchronisation with Unity coroutines is not jet implemented. Call ReadAsync from a coroutine instead.
/// </summary>
/// <returns></returns>
[Obsolete]
public async Task AsyncReadLoop()
{
var cancellationToken = cancellationTokenSource.Token;
do
{
var msg = await ReadAsync();
receivedMessage?.Invoke(msg);
await Task.Yield();
} while (!cancellationToken.IsCancellationRequested);
}
public async Task<string> ReadAsync()
{
// create a MemoryStream to reconstruct the message
using (var ms = new MemoryStream())
{
WebSocketReceiveResult result;
/* A part of the message will be read into the message buffer and than be transfered
* to the MemoryStream. This will be repeated until the complete message has been received.
*/
do
{
var messageBuffer = WebSocket.CreateClientBuffer(1024, 16);
result = await ws.ReceiveAsync(messageBuffer, cancellationTokenSource.Token).ConfigureAwait(false);
// write the messageBuffer to the MemoryStream
ms.Write(messageBuffer.Array, messageBuffer.Offset, result.Count);
} while (!result.EndOfMessage);
switch (result.MessageType)
{
case WebSocketMessageType.Text:
// convert the message stream to string and call the receivedMessage event
return Encoding.UTF8.GetString(ms.ToArray());
case WebSocketMessageType.Close:
throw new DevtoolsConnectionClosedException();
default:
throw new UnexpectedWebSocketMessageTypeException(result.MessageType);
}
}
}
/// <summary>
/// Sends a json serialized command over the devtools websocket
/// </summary>
/// <param name="command"></param>
/// <returns>Task that resoves then the command is send</returns>
public Task SendCommandAsync(string command)
{
// check that the websocket is open
if (ws.State != WebSocketState.Open)
{
throw new DevtoolsConnectionClosedException();
}
Debug.Log($"ws send: '{command}'");
return ws.SendAsync(Encoding.UTF8.GetBytes(command), WebSocketMessageType.Text, true, cancellationTokenSource.Token);
}
public void Dispose()
{
cancellationTokenSource.Cancel();
ws.CloseAsync(WebSocketCloseStatus.NormalClosure, "tab closed", CancellationToken.None)
.ContinueWith(delegate
{
Debug.Log($"ws CloseAsync State:\n{ws.State}");
ws.Dispose();
cancellationTokenSource.Dispose();
});
}
}
/////////////////////////////
// Utility classes:
public class UnexpectedWebSocketMessageTypeException : DevtoolsConnectionException
{
public readonly WebSocketMessageType messageType;
public UnexpectedWebSocketMessageTypeException(WebSocketMessageType messageType) : base($"Unexpected websocket message type: {messageType}")
{
this.messageType = messageType;
}
}
}
\ No newline at end of file
fileFormatVersion: 2
guid: c0bba16a21fb5e449aaf534ba8b217fa
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:
......@@ -25,5 +25,6 @@ namespace ChromeDevTools
public DevtoolsCommandWrapper(T command) => Params = command;
}
public interface IDevtoolsCommand { }
public interface IDevtoolsCommandWithResponse: IDevtoolsCommand { }
}
}
\ No newline at end of file
using ChromeDevTools.Protocol;
using ChromeDevTools.Protocol.Page;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.Collections;
using System.Collections.Concurrent;
using System.Threading;
using System.Threading.Tasks;
using UnityEngine;
namespace ChromeDevTools
{
/// <summary>
/// Manages a chrome devtools websocket connection and parses the messages to C# objects
/// </summary>
public class DevtoolsProtocolHandler
{
private IDevtoolsConnection devtools;
private CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
private ConcurrentDictionary<long, ResponseTypeAndCallback> commandResponseDict = new ConcurrentDictionary<long, ResponseTypeAndCallback>();
// devtools events
public event Action<screencastFrameEvent> screencastFrameEventHandler;
public event Action<screencastVisibilityChangedEvent> screencastVisibilityChangedEventHandler;
public DevtoolsProtocolHandler(IDevtoolsConnection devtools)
{
this.devtools = devtools;
devtools.receivedMessage += ParseMessage;
//TODO: let devtools open async
devtools.OpenAsync().Wait();
//devtools.AsyncReadLoop();
}
public IEnumerator readLoop()
{
while (true)
{
var task = devtools.ReadAsync();
yield return new WaitUntil(() => task.IsCompleted);
// exit the read loop if there was an unhandled exception (e.g. cancellation)
if (!task.IsCompletedSuccessfully)
{
break;
}
var mesgStr = task.GetAwaiter().GetResult();
Debug.Log($"ws reseived: '{mesgStr}'");
ParseMessage(mesgStr);
}
}
private void ParseMessage(string mesgStr)
{
// deserialize the devtools response wrapper
var message = JObject.Parse(mesgStr);
if (message.ContainsKey("id"))
{
handleResponse( message.ToObject<DevtoolsResponseWrapper>() );
}
else if (message.ContainsKey("method"))
{
handleEvent( message.ToObject<DevtoolsEventWrapper>() );
}
else
{
throw new UnexpectedMessageException(mesgStr);
}
}
/// <summary>
/// deserialize a json encoded command response from the devtools connection
/// </summary>
private void handleResponse(DevtoolsResponseWrapper response)
{
// get the callback and the type for this response
ResponseTypeAndCallback responseTypeAndCallback;
if (!commandResponseDict.TryRemove(response.Id, out responseTypeAndCallback))
{
Debug.Log($"There is no command waiting for the response '{response.Id}'");
return;
}
// deserialize the result
IDevtoolsResponse commandResponse = (IDevtoolsResponse) response.Result.ToObject(responseTypeAndCallback.responseType, Browser.serializer);
// pass the response to the callback
responseTypeAndCallback.callback(commandResponse);
}
/// <summary>
/// deserialize a json encoded event from the devtools connection and call its event handlers if at least one is registered
/// </summary>
private void handleEvent(DevtoolsEventWrapper ev)
{
switch (ev.Method)
{
case "Page.screencastFrame":
screencastFrameEventHandler?.Invoke( ev.Params.ToObject<screencastFrameEvent>(Browser.serializer) );
break;
case "Page.screencastVisibilityChanged":
screencastVisibilityChangedEventHandler?.Invoke( ev.Params.ToObject<screencastVisibilityChangedEvent>(Browser.serializer) );
break;
default:
throw new UnexpectedMessageException($"Event of type '{ev}' is not implemented");
}
}
/// <summary>
/// Coroutine wrapper for SendCommandAsync
/// json serializes and sends a command over the devtools websocket
/// </summary>
/// <typeparam name="T">IDevtoolsCommand</typeparam>
/// <param name="command"></param>
/// <returns></returns>
public IEnumerator SendCommand<T>(T command) where T : IDevtoolsCommand
{
var sendTask = SendCommandAsync(command);
// wait until the command has been send
yield return new WaitUntil(() => sendTask.IsCompleted);
}
/// <summary>
/// Coroutine wrapper for SendCommandAsync
/// json serializes and sends a command over the devtools websocket
/// </summary>
/// <typeparam name="T">IDevtoolsCommand</typeparam>
/// <param name="command"></param>
/// <returns></returns>
public IEnumerator SendCommand<T>(T command, Action<IDevtoolsResponse> callback) where T : IDevtoolsCommandWithResponse
{
var sendTask = SendCommandAsync(command, callback);
// wait until the command has been send
yield return new WaitUntil(() => sendTask.IsCompleted);
}
/// <summary>
/// json serializes and sends a command over the devtools websocket
/// </summary>
/// <typeparam name="T">IDevtoolsCommand</typeparam>
/// <param name="command"></param>
/// <returns>Task that resoves then the command is send</returns>
public async Task SendCommandAsync<T>(T command, Action<IDevtoolsResponse> callback) where T : IDevtoolsCommandWithResponse
{
// apply the message wrapper
var wrappedCommand = new DevtoolsCommandWrapper<T>(command);
// get the response type from the commands attribute
CommandResponseAttribute cra = (CommandResponseAttribute)Attribute.GetCustomAttribute(
command.GetType(),
typeof(CommandResponseAttribute)
);
if( cra == null )
{
throw new NotImplementedException("command has no respnse attribute");
}
// TODO: handle command without response
Type responseType = cra.responseType;
// register the response callback for this command id
if (!commandResponseDict.TryAdd(wrappedCommand.Id, new ResponseTypeAndCallback(responseType, callback)))
{
throw new InvalidOperationException($"could not add response callback for command '{wrappedCommand.Id}' to commandResponseDict");
}
// json serialize the command and send it
var json = JsonConvert.SerializeObject(wrappedCommand, Browser.serializerSettings);
Debug.Log($"ws send: '{json}'");
await devtools.SendCommandAsync(json);
}
/// <summary>
/// json serializes and sends a command over the devtools websocket
/// </summary>
/// <typeparam name="T">IDevtoolsCommand</typeparam>
/// <returns>Task that resoves then the command is send</returns>
public async Task SendCommandAsync<T>(T command) where T : IDevtoolsCommand
{
// apply the message wrapper
var wrappedCommand = new DevtoolsCommandWrapper<T>(command);
// json serialize the command and send it
var json = JsonConvert.SerializeObject(wrappedCommand, Browser.serializerSettings);
Debug.Log($"ws send: '{json}'");
await devtools.SendCommandAsync(json);
}
public void Dispose()
{
devtools.Dispose();
}
}
/////////////////////////////
// Utility classes:
/// <summary>
/// Container type to store a response type together with its callback.
/// This is used in DevtoolsProtocolHandler to parse the response to the correct C# data type and call the callback with it as parameter.
/// </summary>
record ResponseTypeAndCallback
{
public ResponseTypeAndCallback(Type responseType, Action<IDevtoolsResponse> callback)
{
this.responseType = responseType;
this.callback = callback;
}
public Type responseType { get; }
public Action<IDevtoolsResponse> callback { get; }
}
public class UnexpectedMessageException : DevtoolsConnectionException
{
public UnexpectedMessageException(string message) : base($"Unexpected message {message}") { }
}
}
\ No newline at end of file
fileFormatVersion: 2
guid: 3a6421e824ac19840bc5970b1a8f61b3
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:
using ChromeDevTools.Protocol;
using System;
using System.Collections;
using System.Threading.Tasks;
namespace ChromeDevTools
{
namespace Protocol
{
public interface IDevtoolsConnection
{
public event Action<string> receivedMessage;
/// <summary>
/// Opens the devtools connection.
/// </summary>
/// <returns>Returns a Task that is resolved when the connection is established.</returns>
public Task OpenAsync();
/// <summary>
/// Reads one message from the devtools connection.
/// </summary>
/// <returns>Returns a Task that is resolved with the message that has been read.</returns>
public Task<string> ReadAsync();
/// <summary>
/// experimential
/// </summary>
/// <returns></returns>
[Obsolete]
public Task AsyncReadLoop();
public Task SendCommandAsync(string command);
public void Dispose();
}
public class DevtoolsConnectionException : Exception
{
public DevtoolsConnectionException(string msg) : base(msg) { }
}
public class DevtoolsConnectionClosedException : DevtoolsConnectionException
{
public DevtoolsConnectionClosedException() : base("Devtools connection closed") { }
}
}
}
\ No newline at end of file
fileFormatVersion: 2
guid: 190b0eaaa999f85448d43dd42b02189f
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:
......@@ -11,7 +11,7 @@ namespace ChromeDevTools
/// Capture page screenshot.
/// </summary>
[CommandResponseAttribute(typeof(CaptureScreenshotCommandResponse))]
public class captureScreenshot : IDevtoolsCommand
public class captureScreenshot : IDevtoolsCommandWithResponse
{
/// <summary>
/// Gets or sets Image compression format (defaults to png).
......
......@@ -40,7 +40,7 @@ namespace ChromeDevTools
/// Send every n-th frame.
/// </summary>
[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public int everyNthFrame { get; set; }
public int? everyNthFrame { get; set; }
}
......@@ -85,7 +85,7 @@ namespace ChromeDevTools
/// <summary>
/// Fired when the page with currently enabled screencast was shown or hidden `.
/// </summary>
public class screencastVisibilityChanged : IDevtoolsEvent
public class screencastVisibilityChangedEvent : IDevtoolsEvent
{
/// <summary>
/// True if the page is visible.
......
......@@ -10,7 +10,7 @@ namespace ChromeDevTools
/// Closes the target. If the target is a page that gets closed too.
/// </summary>
[CommandResponseAttribute(typeof(CloseTargetCommandResponse))]
public class closeTarget : IDevtoolsCommand
public class closeTarget : IDevtoolsCommandWithResponse
{
[JsonIgnore]
public string Method { get; } = "Target.closeTarget";
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment