From 9b78d5792ecd12dd00f695fe5cf20eb6afe3da1b Mon Sep 17 00:00:00 2001 From: Bjoern Esswein <bjoern.esswein@gmail.com> Date: Wed, 19 Jul 2023 02:09:48 +0200 Subject: [PATCH] implemented command serialization and response deserialization --- Runtime/BrowserView.cs | 22 ++- Runtime/ChromeDevtools/Browser.cs | 4 +- Runtime/ChromeDevtools/BrowserTab.cs | 184 ++++++++++++------ .../Protocol/CaptureScreenshot.cs | 47 ----- .../Protocol/CommandResponseAttribute.cs | 15 ++ .../Protocol/CommandResponseAttribute.cs.meta | 11 ++ .../Protocol/DevtoolsCommand.cs | 31 ++- Runtime/ChromeDevtools/Protocol/Page.meta | 8 + .../Protocol/Page/CaptureScreenshot.cs | 50 +++++ .../{ => Page}/CaptureScreenshot.cs.meta | 0 .../Protocol/Target/CloseTarget.cs | 42 ++-- 11 files changed, 271 insertions(+), 143 deletions(-) delete mode 100644 Runtime/ChromeDevtools/Protocol/CaptureScreenshot.cs create mode 100644 Runtime/ChromeDevtools/Protocol/CommandResponseAttribute.cs create mode 100644 Runtime/ChromeDevtools/Protocol/CommandResponseAttribute.cs.meta create mode 100644 Runtime/ChromeDevtools/Protocol/Page.meta create mode 100644 Runtime/ChromeDevtools/Protocol/Page/CaptureScreenshot.cs rename Runtime/ChromeDevtools/Protocol/{ => Page}/CaptureScreenshot.cs.meta (100%) diff --git a/Runtime/BrowserView.cs b/Runtime/BrowserView.cs index bb78112..86ad1fb 100644 --- a/Runtime/BrowserView.cs +++ b/Runtime/BrowserView.cs @@ -1,9 +1,12 @@ using ChromeDevTools; +using System.Collections; +using System.Collections.Generic; using UnityEngine; using UnityEngine.UI; //using Unity.Networking.Transport; -public class BrowserView : MonoBehaviour +[RequireComponent(typeof(RawImage))] +public class BrowserView : MonoBehaviour //TODO: Extends RawImage instead? { private Browser browser; private BrowserTab tab; @@ -21,21 +24,26 @@ public class BrowserView : MonoBehaviour void OnEnable() { browser = Browser.getInstance(); - //BrowserView.LaunchBrowser(); - //UnityEngine.Debug.Log($"ws State:\n{ws.State}"); //StartCoroutine(GetOpenTabs()); var c = StartCoroutine(browser.OpenNewTab(targetUrl, (BrowserTab bt) => { tab = bt; - StartCoroutine(tab.CreateScreenshot(1920, 1080, (screenshot) => - { - rawImage.texture = screenshot; - })); + StartCoroutine(tab.ReadWsMessage()); + StartCoroutine(createScreenshots()); })); } + public IEnumerator createScreenshots () + { + yield return tab.CreateScreenshot(900, 560, (screenshot) => + { + rawImage.texture = screenshot; + StartCoroutine(createScreenshots()); + }); + } + // Update is called once per frame void Update() { diff --git a/Runtime/ChromeDevtools/Browser.cs b/Runtime/ChromeDevtools/Browser.cs index 2be57fc..49fad21 100644 --- a/Runtime/ChromeDevtools/Browser.cs +++ b/Runtime/ChromeDevtools/Browser.cs @@ -68,7 +68,7 @@ namespace ChromeDevTools // set headlessBrowser to false to see the browser window if (headlessBrowser) { - Browser.browserProcess.StartInfo.Arguments = string.Concat(Browser.browserProcess.StartInfo.Arguments, "--headless"); + Browser.browserProcess.StartInfo.Arguments = string.Concat(Browser.browserProcess.StartInfo.Arguments, " --headless=new"); } else { @@ -76,7 +76,7 @@ namespace ChromeDevTools } Browser.browserProcess.Start(); - UnityEngine.Debug.Log("launched chrome"); + UnityEngine.Debug.Log($"launched '{Browser.browserProcess.StartInfo.FileName} {Browser.browserProcess.StartInfo.Arguments}'"); } } diff --git a/Runtime/ChromeDevtools/BrowserTab.cs b/Runtime/ChromeDevtools/BrowserTab.cs index 380dbb5..f4f5ced 100644 --- a/Runtime/ChromeDevtools/BrowserTab.cs +++ b/Runtime/ChromeDevtools/BrowserTab.cs @@ -1,21 +1,35 @@ using ChromeDevTools.Protocol; +using ChromeDevTools.Protocol.Page; +using ChromeDevTools.Protocol.Target; using Newtonsoft.Json; -using Newtonsoft.Json.Linq; using System; using System.Collections; +using System.Collections.Concurrent; using System.IO; using System.Net.WebSockets; using System.Text; using System.Threading; -using System.Threading.Tasks; using UnityEngine; namespace ChromeDevTools { + record ResponseTypeAndCallback + { + public ResponseTypeAndCallback(Type responseType, Action<IDevtoolsResponse> callback) + { + this.responseType = responseType; + this.callback = callback; + } + + public Type responseType { get; } + public Action<IDevtoolsResponse> callback { get; } + } + public class BrowserTab { private ClientWebSocket ws = new ClientWebSocket(); private PageTargetInfo pageTarget; + private ConcurrentDictionary<long, ResponseTypeAndCallback> commandResponseDict = new ConcurrentDictionary<long, ResponseTypeAndCallback>(); public BrowserTab(PageTargetInfo pageTarget) { @@ -26,91 +40,137 @@ namespace ChromeDevTools /** * Note: Forebg wl compatibility you need to use the JavaScript plugin, see also https://docs.unity3d.com/Manual/webgl-networking.html#UsingWebSockets */ - var t = ws.ConnectAsync(new Uri(pageTarget.WebSocketDebuggerUrl), Browser.cancellationTokenSource.Token); + ws.ConnectAsync(new Uri(pageTarget.WebSocketDebuggerUrl), Browser.cancellationTokenSource.Token); } /// <summary> - /// send a json serializable message over the devtools websocket + /// json serializes and sends a command over the devtools websocket /// </summary> /// <typeparam name="T">IDevtoolsCommand</typeparam> - /// <param name="jsonSerializableMessage"></param> + /// <param name="command"></param> /// <returns></returns> - private async Task SendWsMessage<T>(T jsonSerializableMessage) + private IEnumerator SendWsMessage<T>(T command, System.Action<IDevtoolsResponse> callback) where T: IDevtoolsCommand { - if (ws.State != WebSocketState.Open) throw new InvalidOperationException($"WebSocket is not open: ws.State = {ws.State}"); + // wait if the websocket is not yet open + if (ws.State != WebSocketState.Open) + { + yield return new WaitUntil(() => ws.State == WebSocketState.Open); + } + + // apply the message wrapper + var wrappedCommand = new DevtoolsCommandWrapper<T>(command); - var json = JsonConvert.SerializeObject(jsonSerializableMessage, Browser.serializerSettings); + // get the response type from the commands attribute + CommandResponseAttribute cra = (CommandResponseAttribute) Attribute.GetCustomAttribute( + command.GetType(), + typeof(CommandResponseAttribute) + ); + Type responseType = cra.responseType; + + // register the response callback for this command id + if (!commandResponseDict.TryAdd(wrappedCommand.Id, new ResponseTypeAndCallback(responseType, callback))) + { + throw new InvalidOperationException($"could not add response callback for command '{wrappedCommand.Id}' to commandResponseDict"); + } + + // json serialize the command and send it + var json = JsonConvert.SerializeObject(wrappedCommand, Browser.serializerSettings); UnityEngine.Debug.Log($"ws send: '{json}'"); - //await ws.SendAsync(Encoding.UTF8.GetBytes(json), WebSocketMessageType.Text, true, Browser.cancellationTokenSource.Token); + var sendTask = ws.SendAsync(Encoding.UTF8.GetBytes(json), WebSocketMessageType.Text, true, Browser.cancellationTokenSource.Token); + + // wait until the command has been send + yield return new WaitUntil(() => sendTask.IsCompleted); } - /// <summary> - /// read and deserialize a json message from the devtools websocket - /// </summary> - /// <typeparam name="T">IDevtoolsCommandResponse</typeparam> - /// <returns></returns> - private async Task<JObject> ReadWsMessage<T>() + /// <summary> + /// read and deserialize a json message from the devtools websocket + /// </summary> + /// <returns></returns> + public IEnumerator ReadWsMessage() { - if (ws.State != WebSocketState.Open) throw new InvalidOperationException($"WebSocket is not open: ws.State = {ws.State}"); + // wait if the websocket is not yet open + if (ws.State != WebSocketState.Open) + { + yield return new WaitUntil(() => ws.State == WebSocketState.Open); + } - using (var ms = new MemoryStream()) + // start the message receive loop (it will exit when the coroutine is stopped) + while (true) { - WebSocketReceiveResult result; - do - { - var messageBuffer = WebSocket.CreateClientBuffer(1024, 16); - result = await ws.ReceiveAsync(messageBuffer, Browser.cancellationTokenSource.Token); - ms.Write(messageBuffer.Array, messageBuffer.Offset, result.Count); - } - while (!result.EndOfMessage); + string msgString; - if (result.MessageType == WebSocketMessageType.Text) + // create a MemoryStream to reconstruct the message + using (var ms = new MemoryStream()) { - var msgString = Encoding.UTF8.GetString(ms.ToArray()); - UnityEngine.Debug.Log($"ws received: {msgString}"); - var response = JObject.Parse(msgString); - UnityEngine.Debug.Log($"ws received: {response["method"]}"); - UnityEngine.Debug.Log($"ws received: {response["result"]}"); - //return response.ToObject<DevtoolsCommandResponse<CaptureScreenshotCommandResponse>>(serializer); - return response; + WebSocketReceiveResult result; + + /* A part of the message will be read into the message buffer and than be transfered + * to the MemoryStream. This will be repeated until the complete message has been received. + */ + do + { + var messageBuffer = WebSocket.CreateClientBuffer(1024, 16); + var reseiveTask = ws.ReceiveAsync(messageBuffer, Browser.cancellationTokenSource.Token); + + // yield the coroutine until the async Task "reseiveTask" is completed + yield return new WaitUntil(() => reseiveTask.IsCompleted); + result = reseiveTask.Result; + + // write the messageBuffer to the MemoryStream + ms.Write(messageBuffer.Array, messageBuffer.Offset, result.Count); + } + while (!result.EndOfMessage); + + // If the webSocket message type isn't text ignore this message + if (!(result.MessageType == WebSocketMessageType.Text)) + { + Debug.LogError($"Unexpected WebSocketMessageType: {result.MessageType}"); + continue; + } + + // convert the message stream to string + msgString = Encoding.UTF8.GetString(ms.ToArray()); + Debug.Log($"ws reseived: '{msgString}'"); } - else + + // deserialize the devtools response wrapper + var response = JsonConvert.DeserializeObject<DevtoolsResponseWrapper>(msgString, Browser.serializerSettings); + + // get the callback and the type for this response + ResponseTypeAndCallback responseTypeAndCallback; + if (!commandResponseDict.TryRemove(response.Id, out responseTypeAndCallback)) { - throw new InvalidDataException($"Unexpected WebSocketMessageType: {result.MessageType}"); + throw new InvalidOperationException($"There is no command waiting for the response '{response.Id}'"); } - //ms.Seek(0, SeekOrigin.Begin); - //ms.Position = 0; + + // deserialize the result + IDevtoolsResponse commandResponse = (IDevtoolsResponse) response.Result.ToObject(responseTypeAndCallback.responseType, Browser.serializer); + + // pass the response to the callback + responseTypeAndCallback.callback(commandResponse); } } - public IEnumerator CreateScreenshot(double width, double height, System.Action<Texture2D> callback) + public IEnumerator CreateScreenshot(double width, double height, Action<Texture2D> callback) { - yield return new WaitUntil(() => ws.State == WebSocketState.Open); - var screenshotCommand = new CaptureScreenshotCommand(); + var screenshotCommand = new captureScreenshot(); screenshotCommand.Clip = new Protocol.Types.Viewport(); screenshotCommand.Clip.Width = width; screenshotCommand.Clip.Height = height; - //var sendTask = - SendWsMessage(screenshotCommand); - var screenshotCmd = "{\"method\":\"Page.captureScreenshot\",\"id\":1,\"params\": {\"clip\":{\"x\":0,\"y\":0,\"width\":900,\"height\":560,\"scale\":1}}}"; - UnityEngine.Debug.Log($"SendAsy: '{screenshotCmd}'"); - var sendTask = ws.SendAsync(Encoding.UTF8.GetBytes(screenshotCmd), WebSocketMessageType.Text, true, Browser.cancellationTokenSource.Token); - yield return new WaitUntil(() => sendTask.IsCompleted); - var receiveTask = ReadWsMessage<CaptureScreenshotCommandResponse>(); - yield return new WaitUntil(() => receiveTask.IsCompleted); + return SendWsMessage(screenshotCommand, + (response) => { + CaptureScreenshotCommandResponse screenshotResponse = (CaptureScreenshotCommandResponse) response; - Debug.Log($"screenshot result: '{receiveTask.Result["result"]["data"]}'"); - //var screenshot = JsonConvert.DeserializeObject<CaptureScreenshotCommandResponse>(receiveTask.Result.Result, serializerSettings); - //UnityEngine.Debug.Log($"screenshot {receiveTask.Result.Id}: '{screenshot.Data}'"); - byte[] imgBytes = Convert.FromBase64String(receiveTask.Result["result"]["data"].ToString()); + // parse the base64 encoded screenshot to a texture + var screenshotTexture = new Texture2D(1, 1); + screenshotTexture.LoadImage( Convert.FromBase64String(screenshotResponse.Data) ); + //myTexture.Apply(); - Debug.Log($"imgBytes.Length {imgBytes.Length}"); - var myTexture = new Texture2D(1, 1); - myTexture.LoadImage(imgBytes); - //myTexture.Apply(); - callback(myTexture); + // return the texture via callback + callback(screenshotTexture); + }); } /** @@ -119,16 +179,16 @@ namespace ChromeDevTools public void Close() { Debug.Log($"BrowserTab close called for: '{pageTarget.Url}'"); - SendWsMessage(new CloseTargetCommand(pageTarget.Id)) - .ContinueWith(delegate - { - ws.CloseAsync(WebSocketCloseStatus.NormalClosure, "tab closed", CancellationToken.None); - }) + //TODO: fix SendWsMessage without coroutine + SendWsMessage(new closeTarget(pageTarget.Id), delegate + { + ws.CloseAsync(WebSocketCloseStatus.NormalClosure, "tab closed", CancellationToken.None) .ContinueWith(delegate { Debug.Log($"ws CloseAsync State:\n{ws.State}"); ws.Dispose(); }); + }); } } } diff --git a/Runtime/ChromeDevtools/Protocol/CaptureScreenshot.cs b/Runtime/ChromeDevtools/Protocol/CaptureScreenshot.cs deleted file mode 100644 index 88bcc2a..0000000 --- a/Runtime/ChromeDevtools/Protocol/CaptureScreenshot.cs +++ /dev/null @@ -1,47 +0,0 @@ -using Newtonsoft.Json; -using ChromeDevTools.Protocol.Types; - -namespace ChromeDevTools -{ - namespace Protocol - { - /// <summary> - /// Capture page screenshot. - /// </summary> - public class CaptureScreenshotCommand : DevtoolsCommand - { - public string Method { get; } = "Page.captureScreenshot"; - /// <summary> - /// Gets or sets Image compression format (defaults to png). - /// </summary> - [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] - public string Format { get; set; } - /// <summary> - /// Gets or sets Compression quality from range [0..100] (jpeg only). - /// </summary> - [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] - public long? Quality { get; set; } - /// <summary> - /// Gets or sets Capture the screenshot of a given region only. - /// </summary> - [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] - public Viewport Clip { get; set; } - /// <summary> - /// Gets or sets Capture the screenshot from the surface, rather than the view. Defaults to true. - /// </summary> - [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] - public bool? FromSurface { get; set; } - } - - /// <summary> - /// Capture page screenshot response. - /// </summary> - public class CaptureScreenshotCommandResponse - { - /// <summary> - /// Gets or sets Base64-encoded image data. - /// </summary> - public string Data { get; } - } - } -} \ No newline at end of file diff --git a/Runtime/ChromeDevtools/Protocol/CommandResponseAttribute.cs b/Runtime/ChromeDevtools/Protocol/CommandResponseAttribute.cs new file mode 100644 index 0000000..2e60417 --- /dev/null +++ b/Runtime/ChromeDevtools/Protocol/CommandResponseAttribute.cs @@ -0,0 +1,15 @@ +using System; + +namespace ChromeDevTools +{ + namespace Protocol + { + [AttributeUsage(AttributeTargets.Class)] + public class CommandResponseAttribute : Attribute + { + public Type responseType { get; } + public CommandResponseAttribute(Type responseType) => this.responseType = responseType; + + } + } +} \ No newline at end of file diff --git a/Runtime/ChromeDevtools/Protocol/CommandResponseAttribute.cs.meta b/Runtime/ChromeDevtools/Protocol/CommandResponseAttribute.cs.meta new file mode 100644 index 0000000..0f27df6 --- /dev/null +++ b/Runtime/ChromeDevtools/Protocol/CommandResponseAttribute.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 38276a38657a062489df0940470c1ec7 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/ChromeDevtools/Protocol/DevtoolsCommand.cs b/Runtime/ChromeDevtools/Protocol/DevtoolsCommand.cs index f1dbd18..0fd27d7 100644 --- a/Runtime/ChromeDevtools/Protocol/DevtoolsCommand.cs +++ b/Runtime/ChromeDevtools/Protocol/DevtoolsCommand.cs @@ -1,3 +1,6 @@ +using Newtonsoft.Json.Linq; +using System; + namespace ChromeDevTools { namespace Protocol @@ -5,21 +8,37 @@ namespace ChromeDevTools /// /// Every devtools command has an id and a method /// - public class DevtoolsCommand + public class DevtoolsCommandWrapper<T> where T: IDevtoolsCommand { private static long LAST_ID = 0; public long Id { get; } = ++LAST_ID; - public string Method { get; set; } + public string Method + { + get + { + // Remove the namespace prefix 'ChromeDevTools.Protocol.' + return Params.GetType().FullName.Substring("ChromeDevTools.Protocol.".Length); + } + } + + public T Params { get; set; } + public DevtoolsCommandWrapper(T command) => Params = command; } + public interface IDevtoolsCommand {} + public class DevtoolsResponseWrapper + { + public long Id { get; set; } + public JObject Result { get; set; } + } /// /// Every devtools command response has the same id and a method as the corresponding command /// - public class DevtoolsCommandResponse<T> + public class DevtoolsResponseWrapper<T>: DevtoolsResponseWrapper where T: IDevtoolsResponse { - public long Id { get; } - public string Method { get; } - public T Result { get; } + public new T Result { get; set; } } + + public interface IDevtoolsResponse {} } } \ No newline at end of file diff --git a/Runtime/ChromeDevtools/Protocol/Page.meta b/Runtime/ChromeDevtools/Protocol/Page.meta new file mode 100644 index 0000000..a475327 --- /dev/null +++ b/Runtime/ChromeDevtools/Protocol/Page.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: a4d51b1ccebecb84194fe93377034264 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/ChromeDevtools/Protocol/Page/CaptureScreenshot.cs b/Runtime/ChromeDevtools/Protocol/Page/CaptureScreenshot.cs new file mode 100644 index 0000000..84817ff --- /dev/null +++ b/Runtime/ChromeDevtools/Protocol/Page/CaptureScreenshot.cs @@ -0,0 +1,50 @@ +using Newtonsoft.Json; +using ChromeDevTools.Protocol.Types; + +namespace ChromeDevTools +{ + namespace Protocol + { + namespace Page + { + /// <summary> + /// Capture page screenshot. + /// </summary> + [CommandResponseAttribute(typeof(CaptureScreenshotCommandResponse))] + public class captureScreenshot : IDevtoolsCommand + { + /// <summary> + /// Gets or sets Image compression format (defaults to png). + /// </summary> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public string Format { get; set; } + /// <summary> + /// Gets or sets Compression quality from range [0..100] (jpeg only). + /// </summary> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public long? Quality { get; set; } + /// <summary> + /// Gets or sets Capture the screenshot of a given region only. + /// </summary> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public Viewport Clip { get; set; } + /// <summary> + /// Gets or sets Capture the screenshot from the surface, rather than the view. Defaults to true. + /// </summary> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public bool? FromSurface { get; set; } + } + + /// <summary> + /// Capture page screenshot response. + /// </summary> + public class CaptureScreenshotCommandResponse : IDevtoolsResponse + { + /// <summary> + /// Gets or sets Base64-encoded image data. + /// </summary> + public string Data { get; set; } + } + } + } +} \ No newline at end of file diff --git a/Runtime/ChromeDevtools/Protocol/CaptureScreenshot.cs.meta b/Runtime/ChromeDevtools/Protocol/Page/CaptureScreenshot.cs.meta similarity index 100% rename from Runtime/ChromeDevtools/Protocol/CaptureScreenshot.cs.meta rename to Runtime/ChromeDevtools/Protocol/Page/CaptureScreenshot.cs.meta diff --git a/Runtime/ChromeDevtools/Protocol/Target/CloseTarget.cs b/Runtime/ChromeDevtools/Protocol/Target/CloseTarget.cs index 79fb6a7..4d12a90 100644 --- a/Runtime/ChromeDevtools/Protocol/Target/CloseTarget.cs +++ b/Runtime/ChromeDevtools/Protocol/Target/CloseTarget.cs @@ -1,34 +1,38 @@ using Newtonsoft.Json; -using ChromeDevTools.Protocol.Types; namespace ChromeDevTools { namespace Protocol { - /// <summary> - /// Closes the target. If the target is a page that gets closed too. - /// </summary> - public class CloseTargetCommand : DevtoolsCommand + namespace Target { - public string Method { get; } = "Target.closeTarget"; - - public CloseTargetCommand(string targetId) => TargetId = targetId; - /// <summary> - /// Gets or sets TargetId + /// Closes the target. If the target is a page that gets closed too. /// </summary> - public string TargetId { get; set; } - } + [CommandResponseAttribute(typeof(CloseTargetCommandResponse))] + public class closeTarget : IDevtoolsCommand + { + [JsonIgnore] + public string Method { get; } = "Target.closeTarget"; + + public closeTarget(string targetId) => TargetId = targetId; + + /// <summary> + /// Gets or sets TargetId + /// </summary> + public string TargetId { get; set; } + } - /// <summary> - /// Closes the target. If the target is a page that gets closed too. - /// </summary> - public class CloseTargetCommandResponse - { /// <summary> - /// Always set to true. If an error occurs, the response indicates protocol error. + /// Closes the target. If the target is a page that gets closed too. /// </summary> - public bool Success { get; } + public class CloseTargetCommandResponse + { + /// <summary> + /// Always set to true. If an error occurs, the response indicates protocol error. + /// </summary> + public bool Success { get; } + } } } } \ No newline at end of file -- GitLab