From 32bbc9be2e099918d10d0e54e8993cae6e54799a Mon Sep 17 00:00:00 2001 From: Bjoern Esswein <692-bessw@users.noreply.gl.kwarc.info> Date: Thu, 11 Jul 2024 20:53:57 +0200 Subject: [PATCH] Implementet parts of the runtime protocol to evaluate js expresions and create js bindings for c# methods. --- Runtime/ChromeDevtools/BrowserTab.cs | 89 +++++++++++- .../ChromeDevtools/DevtoolsProtocolHandler.cs | 10 +- Runtime/ChromeDevtools/DomNodeWrapper.cs | 22 ++- Runtime/ChromeDevtools/Protocol/Runtime.meta | 8 ++ .../Protocol/Runtime/Runtime.cs | 135 ++++++++++++++++++ .../Protocol/Runtime/Runtime.cs.meta | 11 ++ .../Protocol/Types/ExceptionDetails.cs | 74 ++++++++++ .../Protocol/Types/ExceptionDetails.cs.meta | 11 ++ .../Protocol/Types/RemoteObject.cs | 55 +++++++ .../Protocol/Types/RemoteObject.cs.meta | 11 ++ Runtime/WebViewComponent.cs | 3 + 11 files changed, 418 insertions(+), 11 deletions(-) create mode 100644 Runtime/ChromeDevtools/Protocol/Runtime.meta create mode 100644 Runtime/ChromeDevtools/Protocol/Runtime/Runtime.cs create mode 100644 Runtime/ChromeDevtools/Protocol/Runtime/Runtime.cs.meta create mode 100644 Runtime/ChromeDevtools/Protocol/Types/ExceptionDetails.cs create mode 100644 Runtime/ChromeDevtools/Protocol/Types/ExceptionDetails.cs.meta create mode 100644 Runtime/ChromeDevtools/Protocol/Types/RemoteObject.cs create mode 100644 Runtime/ChromeDevtools/Protocol/Types/RemoteObject.cs.meta diff --git a/Runtime/ChromeDevtools/BrowserTab.cs b/Runtime/ChromeDevtools/BrowserTab.cs index 4daa6b1..d0c439b 100644 --- a/Runtime/ChromeDevtools/BrowserTab.cs +++ b/Runtime/ChromeDevtools/BrowserTab.cs @@ -1,9 +1,12 @@ +using bessw.Unity.WebView.ChromeDevTools.Protocol.DOM; using bessw.Unity.WebView.ChromeDevTools.Protocol.Input; using bessw.Unity.WebView.ChromeDevTools.Protocol.Page; +using bessw.Unity.WebView.ChromeDevTools.Protocol.Runtime; using bessw.Unity.WebView.ChromeDevTools.Protocol.Target; using System; using System.Collections; using System.Collections.Generic; +using System.Threading.Tasks; using UnityEngine; using UnityEngine.EventSystems; @@ -22,6 +25,9 @@ namespace bessw.Unity.WebView.ChromeDevTools public Dictionary<int, DomNodeWrapper> domNodes = new Dictionary<int, DomNodeWrapper>(); + private DomNodeWrapper document; + private Dictionary<string, Action<string>> runtimeBindings = new(); + public BrowserTab(PageTargetInfo pageTarget) { this.pageTarget = pageTarget; @@ -30,14 +36,23 @@ namespace bessw.Unity.WebView.ChromeDevTools this.devtools = new DevtoolsProtocolHandler(new DevtoolsWebsocket(pageTarget.WebSocketDebuggerUrl)); // register DOM event handlers + this.devtools.onDomDocumentUpdated += (documentUpdatedEvent) => DocumentUpdatedEventHandler(documentUpdatedEvent); this.devtools.onDomAttributeModified += (attributeModifiedEvent) => DomNodeWrapper.AttributeModifiedEventHandler(this, attributeModifiedEvent); this.devtools.onDomAttributeRemoved += (attributeRemovedEvent) => DomNodeWrapper.AttributeRemovedEventHandler(this, attributeRemovedEvent); this.devtools.onDomCharacterDataModified += (characterDataModifiedEvent) => DomNodeWrapper.CharacterDataModifiedEventHandler(this, characterDataModifiedEvent); this.devtools.onDomChildNodeCountUpdated += (childNodeCountUpdatedEvent) => DomNodeWrapper.ChildNodeCountUpdatedEventHandler(this, childNodeCountUpdatedEvent); this.devtools.onDomChildNodeInserted += (childNodeInsertedEvent) => DomNodeWrapper.ChildNodeInsertedEventHandler(this, childNodeInsertedEvent); this.devtools.onDomChildNodeRemoved += (childNodeRemovedEvent) => DomNodeWrapper.ChildNodeRemovedEventHandler(this, childNodeRemovedEvent); - this.devtools.onDomDocumentUpdated += (documentUpdatedEvent) => DomNodeWrapper.DocumentUpdatedEventHandler(this, documentUpdatedEvent); this.devtools.onDomSetChildNodes += (setChildNodesEvent) => DomNodeWrapper.SetChildNodesEventHandler(this, setChildNodesEvent); + + this.devtools.onRuntimeBindingCalled += (bindingCalledEvent) => OnBindingCalled(bindingCalledEvent); + } + + private void DocumentUpdatedEventHandler(documentUpdatedEvent eventData) + { + // clear the domNodes dictionary + domNodes.Clear(); + document = null; } public IEnumerator Update() @@ -45,6 +60,58 @@ namespace bessw.Unity.WebView.ChromeDevTools yield return devtools.readLoop(); } + /// <summary> + /// Evaluate a JavaScript expression in the browser + /// </summary> + /// <param name="jsExpression"></param> + /// <returns></returns> + public Task<evaluateResponse> Evaluate(string jsExpression) + { + var evaluateCommand = new evaluate + { + expression = jsExpression + }; + + return devtools.SendCommandAsync<evaluate, evaluateResponse>(evaluateCommand); + } + + /// <summary> + /// Adds a mathod with the given name that takes a string as argument to the browser's js runtime, and calls the given handler when the method is called. + /// </summary> + /// <param name="name"></param> + /// <param name="handler"></param> + /// <returns></returns> + public Task AddJSBinding(string name, Action<string> handler) + { + runtimeBindings.Add(name, handler); + return devtools.SendCommandAsync(new addBinding + { + name = name + }); + } + + /// <summary> + /// Removes the method with the given name from the browser's js runtime. + /// </summary> + /// <param name="name"></param> + /// <returns></returns> + public Task RemoveJSBinding(string name) + { + runtimeBindings.Remove(name); + return devtools.SendCommandAsync(new removeBinding + { + name = name + }); + } + + private void OnBindingCalled(bindingCalled binding) + { + if (runtimeBindings.TryGetValue(binding.name, out var handler)) + { + handler(binding.payload); + } + } + public IEnumerator CreateScreenshot(double width, double height, Action<Texture2D> callback) { var screenshotCommand = new captureScreenshot @@ -160,9 +227,25 @@ namespace bessw.Unity.WebView.ChromeDevTools _ = devtools.SendCommandAsync(new cancelDragging()); } - public IEnumerator GetDocument(Action<DomNodeWrapper> callback) + /// <summary> + /// returns the cached document DomNodeWrapper or fetches it from the browser + /// </summary> + /// <returns></returns> + public async Task<DomNodeWrapper> GetDocument() + { + if (document is not null) return document; + else return await DomNodeWrapper.getDocumentAsync(this); + } + + /// <summary> + /// returns the cached node if it exists, otherwise fetches it from the browser + /// </summary> + /// <param name="nodeId"></param> + /// <returns></returns> + public async Task<DomNodeWrapper> GetNode(int nodeId) { - return DomNodeWrapper.getDocument(this, callback); + if (domNodes.TryGetValue(nodeId, out DomNodeWrapper node)) return node; + else return await DomNodeWrapper.describeNodeAsync(this, nodeId); } /** diff --git a/Runtime/ChromeDevtools/DevtoolsProtocolHandler.cs b/Runtime/ChromeDevtools/DevtoolsProtocolHandler.cs index 97e6146..75bf081 100644 --- a/Runtime/ChromeDevtools/DevtoolsProtocolHandler.cs +++ b/Runtime/ChromeDevtools/DevtoolsProtocolHandler.cs @@ -1,6 +1,7 @@ using bessw.Unity.WebView.ChromeDevTools.Protocol; using bessw.Unity.WebView.ChromeDevTools.Protocol.DOM; using bessw.Unity.WebView.ChromeDevTools.Protocol.Page; +using bessw.Unity.WebView.ChromeDevTools.Protocol.Runtime; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using System; @@ -25,6 +26,7 @@ namespace bessw.Unity.WebView.ChromeDevTools // devtools events public event Action<screencastFrameEvent> onScreencastFrame; public event Action<screencastVisibilityChangedEvent> onScreencastVisibilityChanged; + public event Action<bindingCalled> onRuntimeBindingCalled; #region DOM events public event Action<attributeModifiedEvent> onDomAttributeModified; @@ -127,7 +129,10 @@ namespace bessw.Unity.WebView.ChromeDevTools case "Page.screencastVisibilityChanged": onScreencastVisibilityChanged?.Invoke( ev.Params.ToObject<screencastVisibilityChangedEvent>(Browser.devtoolsSerializer) ); break; - + case "Runtime.bindingCalled": + onRuntimeBindingCalled?.Invoke( ev.Params.ToObject<bindingCalled>(Browser.devtoolsSerializer) ); + break; + // switch cases that invoke the event handlers for the DOM events #region DOM events case "DOM.attributeModified": @@ -161,7 +166,8 @@ namespace bessw.Unity.WebView.ChromeDevTools unknownEventHandler.Invoke(ev); break; } else { - throw new UnexpectedMessageException($"Event of type '{ev}' is not implemented"); + Debug.LogError($"There is no handler implemented for the browser event of type '{ev.Method}'"); + break; } } } diff --git a/Runtime/ChromeDevtools/DomNodeWrapper.cs b/Runtime/ChromeDevtools/DomNodeWrapper.cs index 4d6c163..7f2eeae 100644 --- a/Runtime/ChromeDevtools/DomNodeWrapper.cs +++ b/Runtime/ChromeDevtools/DomNodeWrapper.cs @@ -159,12 +159,6 @@ namespace bessw.Unity.WebView.ChromeDevTools } } - public static void DocumentUpdatedEventHandler(BrowserTab tab, documentUpdatedEvent eventData) - { - // clear the domNodes dictionary - tab.domNodes.Clear(); - } - public static void SetChildNodesEventHandler(BrowserTab tab, setChildNodesEvent eventData) { var parentNode = createOrUpdateNode(tab, eventData.parentId); @@ -203,6 +197,22 @@ namespace bessw.Unity.WebView.ChromeDevTools return domNode; } + /// <summary> + /// get a node by its js object id + /// </summary> + /// <param name="tab"></param> + /// <param name="nodeId">Javascript html object id</param> + public static async Task<DomNodeWrapper> describeNodeAsync(BrowserTab tab, int nodeId, int? depth = null, bool? pierce = null) + { + var commandResponse = await tab.devtools.SendCommandAsync<describeNode, describeNodeCommandResponse>(new describeNode + { + nodeId = nodeId, + depth = depth, + pierce = pierce + }); + return createOrUpdateNode(tab, commandResponse.node.nodeId, null, null, commandResponse.node); + } + /// <summary> /// get a node by its js object id /// </summary> diff --git a/Runtime/ChromeDevtools/Protocol/Runtime.meta b/Runtime/ChromeDevtools/Protocol/Runtime.meta new file mode 100644 index 0000000..f12087b --- /dev/null +++ b/Runtime/ChromeDevtools/Protocol/Runtime.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: a77272afb3e6cdc46bfdc6c988abbedf +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/ChromeDevtools/Protocol/Runtime/Runtime.cs b/Runtime/ChromeDevtools/Protocol/Runtime/Runtime.cs new file mode 100644 index 0000000..3bc96b6 --- /dev/null +++ b/Runtime/ChromeDevtools/Protocol/Runtime/Runtime.cs @@ -0,0 +1,135 @@ +using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; + +#nullable enable annotations +namespace bessw.Unity.WebView.ChromeDevTools.Protocol.Runtime +{ + #region commands + + /// <summary> + /// If executionContextId is empty, adds binding with the given name on the global objects of all inspected contexts, including those created later, bindings survive reloads. Binding function takes exactly one argument, this argument should be string, in case of any other input, function throws an exception. Each binding function call produces Runtime.bindingCalled notification. + /// </summary> + public class addBinding : IDevtoolsCommand + { + public string name { get; set; } + + /// <summary> + /// If specified, the binding is exposed to the executionContext with matching name, even for contexts created after the binding is added. See also ExecutionContext.name and worldName parameter to Page.addScriptToEvaluateOnNewDocument. This parameter is mutually exclusive with executionContextId. + /// </summary> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public string? executionContextName { get; set; } + } + + /// <summary> + /// This method does not remove binding function from global object but unsubscribes current runtime agent from Runtime.bindingCalled notifications. + /// </summary> + public class removeBinding : IDevtoolsCommand + { + public string name { get; set; } + } + + /// <summary> + /// Evaluates expression on global object. + /// </summary> + [CommandResponse(typeof(evaluateResponse))] + public class evaluate : IDevtoolsCommandWithResponse + { + /// <summary> + /// Expression to evaluate. + /// </summary> + public string expression { get; set; } + + /// <summary> + /// Symbolic group name that can be used to release multiple objects. + /// </summary> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public string? objectGroup { get; set; } + + /// <summary> + /// Determines whether Command Line API should be available during the evaluation. + /// </summary> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public bool? includeCommandLineAPI { get; set; } + + /// <summary> + /// In silent mode exceptions thrown during evaluation are not reported and do not pause execution. Overrides setPauseOnException state. + /// </summary> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public bool? silent { get; set; } + + /// <summary> + /// Specifies in which execution context to perform evaluation. If the parameter is omitted the evaluation will be performed in the context of the inspected page. + /// This is mutually exclusive with uniqueContextId, which offers an alternative way to identify the execution context that is more reliable in a multi-process environment. + /// </summary> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public string? contextId { get; set; } + + /// <summary> + /// Whether the result is expected to be a JSON object that should be sent by value. + /// </summary> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public bool? returnByValue { get; set; } + + /// <summary> + /// Whether preview should be generated for the result. + /// </summary> + /// <remarks>experimental</remarks> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public bool? generatePreview { get; set; } + + /// <summary> + /// Whether execution should be treated as initiated by user in the UI. + /// </summary> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public bool? userGesture { get; set; } + + /// <summary> + /// Wetehr execution should wait for promise to be resolved. + /// </summary> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public bool? awaitPromise { get; set; } + + /// <summary> + /// Whether to throw an exception if side effect cannot be ruled out during evaluation. This implies disableBreaks below. + /// </summary> + /// <remarks>experimental</remarks> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public bool? throwOnSideEffect { get; set; } + + /// <summary> + /// Terminate execution after timing out (number of milliseconds). + /// </summary> + /// <remarks>experimental</remarks> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public bool? timeout { get; set; } + + /// <summary> + /// Setting this flag to true enables let re-declaration and top-level await. Note that let variables can only be re-declared if they originate from replMode themselves. + /// </summary> + /// <remarks>experimental</remarks> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public bool? replMode { get; set; } + } + + public class evaluateResponse : IDevtoolsResponse + { + public remoteObject result { get; set; } + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public exceptionDetails? exceptionDetails { get; set; } + } + + #endregion commands + + #region events + + /// <summary> + /// Notification is issued every time when binding is called. + /// </summary> + public class bindingCalled : IDevtoolsEvent + { + public string name { get; set; } + public string payload { get; set; } + } + + #endregion events +} \ No newline at end of file diff --git a/Runtime/ChromeDevtools/Protocol/Runtime/Runtime.cs.meta b/Runtime/ChromeDevtools/Protocol/Runtime/Runtime.cs.meta new file mode 100644 index 0000000..5315db7 --- /dev/null +++ b/Runtime/ChromeDevtools/Protocol/Runtime/Runtime.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: cc9def2f2bb2a7d46ae3c36bc5991769 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/ChromeDevtools/Protocol/Types/ExceptionDetails.cs b/Runtime/ChromeDevtools/Protocol/Types/ExceptionDetails.cs new file mode 100644 index 0000000..ee35784 --- /dev/null +++ b/Runtime/ChromeDevtools/Protocol/Types/ExceptionDetails.cs @@ -0,0 +1,74 @@ +using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; + +#nullable enable annotations +namespace bessw.Unity.WebView.ChromeDevTools.Protocol.Runtime +{ + /// <summary> + /// Detailed information about exception (or error) that was thrown during script compilation or execution. + /// </summary> + public class exceptionDetails : IDevtoolsResponse + { + /// <summary> + /// Exception id. + /// </summary> + public int exceptionId { get; set; } + + /// <summary> + /// Exception text, which should be used together with exception object when available. + /// </summary> + public string text { get; set; } + + /// <summary> + /// Line number of the exception location (0-based). + /// </summary> + public int lineNumber { get; set; } + + /// <summary> + /// Column number of the exception location (0-based). + /// </summary> + public int columnNumber { get; set; } + + /// <summary> + /// Script ID of the exception. + /// </summary> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public string? scriptId { get; set; } + + /// <summary> + /// URL of the exception location, to be used when the script was not reported. + /// </summary> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public string? url { get; set; } + + /// <summary> + /// JavaScript stack trace if available. + /// </summary> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public string? stackTrace { get; set; } + + /// <summary> + /// Exception object if available. + /// </summary> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public remoteObject? exception { get; set; } + + /// <summary> + /// Identifier of the context where exception happened. + /// </summary> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public int? executionContextId { get; set; } + + /// <summary> + /// Exception was thrown from the uncaught exception handler. + /// </summary> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public bool? uncaught { get; set; } + + /// <summary> + /// Exception was thrown from the unhandled promise rejection. + /// </summary> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public bool? unhandled { get; set; } + } +} diff --git a/Runtime/ChromeDevtools/Protocol/Types/ExceptionDetails.cs.meta b/Runtime/ChromeDevtools/Protocol/Types/ExceptionDetails.cs.meta new file mode 100644 index 0000000..66ce2fe --- /dev/null +++ b/Runtime/ChromeDevtools/Protocol/Types/ExceptionDetails.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 87f159856e46435458b3ee6141814904 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/ChromeDevtools/Protocol/Types/RemoteObject.cs b/Runtime/ChromeDevtools/Protocol/Types/RemoteObject.cs new file mode 100644 index 0000000..eec3b69 --- /dev/null +++ b/Runtime/ChromeDevtools/Protocol/Types/RemoteObject.cs @@ -0,0 +1,55 @@ +using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; + +#nullable enable annotations +namespace bessw.Unity.WebView.ChromeDevTools.Protocol.Runtime +{ + /// <summary> + /// Mirror object referencing original JavaScript object. + /// </summary> + public class remoteObject : IDevtoolsResponse + { + /// <summary> + /// Object type. + /// Allowed Values: object, function, undefined, string, number, boolean, symbol, bigint + /// </summary> + public string type { get; set; } + + /// <summary> + /// Object subtype hint. Specified for object type values only. NOTE: If you change anything here, make sure to also update subtype in ObjectPreview and PropertyPreview below. + /// Allowed Values: array, null, node, regexp, date, map, set, weakmap, weakset, iterator, generator, error, proxy, promise, typedarray, arraybuffer, dataview, webassemblymemory, wasmvalue + /// </summary> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public string? subtype { get; set; } + + /// <summary> + /// Object class (constructor) name. Specified for object type values only. + /// </summary> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public string? className { get; set; } + + /// <summary> + /// Remote object value in case of primitive values or JSON values (if it was requested). + /// </summary> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public string? value { get; set; } + + /// <summary> + /// Primitive value which can not be JSON-stringified does not have value, but gets this property. + /// </summary> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public string? unserializableValue { get; set; } + + /// <summary> + /// String representation of the object. + /// </summary> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public string? description { get; set; } + + /// <summary> + /// RemoteObjectId: Unique object identifier (for non-primitive values). + /// </summary> + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public string? objectId { get; set; } + } +} diff --git a/Runtime/ChromeDevtools/Protocol/Types/RemoteObject.cs.meta b/Runtime/ChromeDevtools/Protocol/Types/RemoteObject.cs.meta new file mode 100644 index 0000000..c67dad5 --- /dev/null +++ b/Runtime/ChromeDevtools/Protocol/Types/RemoteObject.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 6196d4604ccc0d3498d74e07318dfcdf +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/WebViewComponent.cs b/Runtime/WebViewComponent.cs index 1f69d85..b93a246 100644 --- a/Runtime/WebViewComponent.cs +++ b/Runtime/WebViewComponent.cs @@ -57,6 +57,8 @@ namespace bessw.Unity.WebView remove => tab.devtools.onDomDocumentUpdated -= value; } + public event Action OnWebViewComponentReady; + private RawImage rawImage; private RectTransform rectTransform; @@ -86,6 +88,7 @@ namespace bessw.Unity.WebView rawImage.texture = frame; rawImage.SetNativeSize(); })); + OnWebViewComponentReady?.Invoke(); })); } -- GitLab