create package for the BrowserView

BrowserView.prefab

+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!1 &8825460134736404412
+GameObject:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  serializedVersion: 6
+  m_Component:
+  - component: {fileID: 2974656142881083530}
+  - component: {fileID: 5559415116192402672}
+  - component: {fileID: 4531839590213585742}
+  - component: {fileID: 5511584970398147175}
+  m_Layer: 5
+  m_Name: BrowserView
+  m_TagString: Untagged
+  m_Icon: {fileID: 0}
+  m_NavMeshLayer: 0
+  m_StaticEditorFlags: 0
+  m_IsActive: 1
+--- !u!224 &2974656142881083530
+RectTransform:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 8825460134736404412}
+  m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
+  m_LocalPosition: {x: 0, y: 0, z: 0}
+  m_LocalScale: {x: 1, y: 1, z: 1}
+  m_ConstrainProportionsScale: 0
+  m_Children: []
+  m_Father: {fileID: 0}
+  m_RootOrder: 0
+  m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
+  m_AnchorMin: {x: 0.5, y: 0.5}
+  m_AnchorMax: {x: 0.5, y: 0.5}
+  m_AnchoredPosition: {x: -444.39624, y: -175.33963}
+  m_SizeDelta: {x: 100, y: 100}
+  m_Pivot: {x: 0.5, y: 0.5}
+--- !u!222 &5559415116192402672
+CanvasRenderer:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 8825460134736404412}
+  m_CullTransparentMesh: 1
+--- !u!114 &4531839590213585742
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 8825460134736404412}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 1344c3c82d62a2a41a3576d8abb8e3ea, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  m_Material: {fileID: 0}
+  m_Color: {r: 1, g: 1, b: 1, a: 1}
+  m_RaycastTarget: 1
+  m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0}
+  m_Maskable: 1
+  m_OnCullStateChanged:
+    m_PersistentCalls:
+      m_Calls: []
+  m_Texture: {fileID: 0}
+  m_UVRect:
+    serializedVersion: 2
+    x: 0
+    y: 0
+    width: 1
+    height: 1
+--- !u!114 &5511584970398147175
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 8825460134736404412}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 6893fa42f929f894286c6f00c40bcd06, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  targetUrl: https://google.de

BrowserView.prefab.meta

+fileFormatVersion: 2
+guid: 1e9d514b702f5784791a4df8d22e1866
+PrefabImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Runtime.meta

+fileFormatVersion: 2
+guid: 13d4f7b0c901773468a10206e6a88844
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Runtime/BrowserView.cs

+using ChromeDevToolsProtocol;
+using ChromeDevToolsProtocol.WebSocketProtocol;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using Newtonsoft.Json.Serialization;
+using System;
+using System.IO;
+using System.Text;
+using System.Net.WebSockets;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Collections;
+using System.Diagnostics;
+using UnityEngine;
+using UnityEngine.Networking;
+using UnityEngine.UI;
+//using Unity.Networking.Transport;
+
+public class BrowserView : MonoBehaviour
+{
+    // each instance of this class gets its own tab in this browser
+    private static Process browserProcess;
+    private const string browserExecutablePath = "chrome";
+    private const bool headlessBrowser = false;
+    private const int debugPort = 9222;
+
+    private ClientWebSocket ws = new ClientWebSocket();
+    private PageTargetInfo pageTarget;
+
+    private CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
+    private JsonSerializer serializer;
+    private JsonSerializerSettings serializerSettings;
+
+    private RawImage rawImage;
+
+    public string targetUrl = "https://google.de";
+
+    // Start is called before the first frame update
+    void Start()
+    {
+        rawImage = this.gameObject.GetComponent<RawImage>();
+    }
+
+    void OnEnable()
+    {
+        var camelCasePropertyNamesContractResolver = new CamelCasePropertyNamesContractResolver();
+        serializer = new JsonSerializer();
+        serializer.ContractResolver = camelCasePropertyNamesContractResolver;
+        serializerSettings = new JsonSerializerSettings();
+        serializerSettings.ContractResolver = camelCasePropertyNamesContractResolver;
+
+        BrowserView.LaunchBrowser();
+
+        UnityEngine.Debug.Log($"ws State:\n{ws.State}");
+        //StartCoroutine(GetOpenTabs());
+        var c = StartCoroutine(OpenNewTab(delegate
+        {
+            UnityEngine.Debug.Log($"ws ConnectAsync State:\n{ws.State}");
+            UnityEngine.Debug.Log("pre get tabs");
+            StartCoroutine(GetOpenTabs());
+            UnityEngine.Debug.Log("post  get tabs");
+        }));
+        StartCoroutine(CreateScreenshot());
+    }
+
+    // Update is called once per frame
+    void Update()
+    {
+
+    }
+    private IEnumerator GetOpenTabs()
+    {
+        yield return DevToolsApiRequest(false, "/json/list", (response) =>
+        {
+            UnityEngine.Debug.Log($"Currently open tabs:\n{response}");
+        });
+    }
+
+    private IEnumerator OpenNewTab(System.Action<Task> callback)
+    {
+        yield return DevToolsApiRequest(true, $"/json/new?{targetUrl}", (response) =>
+        {
+            pageTarget = JsonConvert.DeserializeObject<PageTargetInfo>(response, serializerSettings);
+            UnityEngine.Debug.Log($"tab WebSocket: '{pageTarget.WebSocketDebuggerUrl}'");
+
+            // open remote devtools websocket connection
+            /**
+             * Note: For webgl compatibility you need to use the JavaScript plugin, see also https://docs.unity3d.com/Manual/webgl-networking.html#UsingWebSockets
+             */
+            var t = ws.ConnectAsync(new System.Uri(pageTarget.WebSocketDebuggerUrl), cancellationTokenSource.Token);
+            t.ContinueWith(callback);
+        });
+        //yield return new WaitUntil(() => t.IsCompleted);
+    }
+    private IEnumerator CreateScreenshot()
+    {
+        yield return new WaitUntil(() => ws.State == WebSocketState.Open);
+        var screenshotCommand = new CaptureScreenshotCommand();
+        screenshotCommand.Clip = new ChromeDevToolsProtocol.WebSocketProtocol.Types.Viewport();
+        screenshotCommand.Clip.Width = 620;
+        screenshotCommand.Clip.Height = 480;
+        //var sendTask = SendWsMessage(screenshotCommand);
+        var rect = this.GetComponent<RectTransform>().rect;
+        UnityEngine.Debug.Log($"pre, w: {rect.width} h: {rect.height}");
+        var sendTask = ws.SendAsync(Encoding.UTF8.GetBytes("{\"method\":\"Page.captureScreenshot\",\"id\":1,\"params\": {\"clip\":{\"x\":0,\"y\":0,\"width\":900,\"height\":560,\"scale\":1}}}"), WebSocketMessageType.Text, true, cancellationTokenSource.Token);
+        UnityEngine.Debug.Log("post");
+        yield return new WaitUntil(() => sendTask.IsCompleted);
+
+        var receiveTask = ReadWsMessage<CaptureScreenshotCommandResponse>();
+        yield return new WaitUntil(() => receiveTask.IsCompleted);
+
+        UnityEngine.Debug.Log($"screenshot result: '{receiveTask.Result["result"]["data"]}'");
+        //var screenshot = JsonConvert.DeserializeObject<CaptureScreenshotCommandResponse>(receiveTask.Result.Result, serializerSettings);
+        //UnityEngine.Debug.Log($"screenshot {receiveTask.Result.Id}: '{screenshot.Data}'");
+        byte[] imgBytes = Convert.FromBase64String(receiveTask.Result["result"]["data"].ToString());
+
+        UnityEngine.Debug.Log($"imgBytes.Length {imgBytes.Length}");
+        var myTexture = new Texture2D(1, 1);
+        myTexture.LoadImage(imgBytes);
+        //myTexture.Apply();
+        rawImage.texture = myTexture;
+
+        yield return StartCoroutine(CreateScreenshot());
+    }
+
+    /// <summary>
+    /// read and deserialize a json message from the devtools websocket
+    /// </summary>
+    /// <typeparam name="T">IDevtoolsCommandResponse</typeparam>
+    /// <returns></returns>
+    private async Task<JObject> ReadWsMessage<T>()
+    {
+        if (ws.State != WebSocketState.Open) throw new InvalidOperationException($"WebSocket is not open: ws.State = {ws.State}");
+
+        using (var ms = new MemoryStream())
+        {
+            WebSocketReceiveResult result;
+            do
+            {
+                var messageBuffer = WebSocket.CreateClientBuffer(1024, 16);
+                result = await ws.ReceiveAsync(messageBuffer, cancellationTokenSource.Token);
+                ms.Write(messageBuffer.Array, messageBuffer.Offset, result.Count);
+            }
+            while (!result.EndOfMessage);
+
+            if (result.MessageType == WebSocketMessageType.Text)
+            {
+                var msgString = Encoding.UTF8.GetString(ms.ToArray());
+                UnityEngine.Debug.Log($"ws received: {msgString}");
+                var response = JObject.Parse(msgString);
+                UnityEngine.Debug.Log($"ws received: {response["method"]}");
+                UnityEngine.Debug.Log($"ws received: {response["result"]}");
+                //return response.ToObject<DevtoolsCommandResponse<CaptureScreenshotCommandResponse>>(serializer);
+                return response;
+            }
+            else
+            {
+                throw new InvalidDataException($"Unexpected WebSocketMessageType: {result.MessageType}");
+            }
+            //ms.Seek(0, SeekOrigin.Begin);
+            //ms.Position = 0;
+        }
+    }
+
+    /// <summary>
+    /// send a json serializable message over the devtools websocket
+    /// </summary>
+    /// <typeparam name="T">IDevtoolsCommand</typeparam>
+    /// <param name="jsonSerializableMessage"></param>
+    /// <returns></returns>
+    private async Task SendWsMessage<T>(T jsonSerializableMessage)
+    {
+        if (ws.State != WebSocketState.Open) throw new InvalidOperationException($"WebSocket is not open: ws.State = {ws.State}");
+
+        var json = JsonConvert.SerializeObject(jsonSerializableMessage, serializerSettings);
+        UnityEngine.Debug.Log($"ws send: {json}");
+        await ws.SendAsync(Encoding.UTF8.GetBytes(json), WebSocketMessageType.Text, true, cancellationTokenSource.Token);
+    }
+
+    /**
+     * Launch headless chrome browser with remote-debugging enabled, if not already running.
+     */
+    private static void LaunchBrowser()
+    {
+        // allow only one instance of chrome
+        if (BrowserView.browserProcess == null || BrowserView.browserProcess.HasExited)
+        {
+            BrowserView.browserProcess = new Process();
+            BrowserView.browserProcess.StartInfo.FileName = browserExecutablePath;
+            BrowserView.browserProcess.StartInfo.Arguments = $"--user-data-dir={Path.Join(Application.temporaryCachePath, "BrowserView")} --remote-debugging-port={debugPort} --remote-allow-origins=http://localhost:{debugPort}";
+
+            // set headlessBrowser to false to see the browser window
+            if (headlessBrowser)
+            {
+                BrowserView.browserProcess.StartInfo.Arguments = string.Concat(BrowserView.browserProcess.StartInfo.Arguments, "--headless");
+            }
+            else
+            {
+                BrowserView.browserProcess.StartInfo.WindowStyle = ProcessWindowStyle.Minimized;
+            }
+
+            BrowserView.browserProcess.Start();
+            UnityEngine.Debug.Log("launched chrome");
+        }
+    }
+
+    private IEnumerator DevToolsApiRequest(bool isPUT, string apiAddress, System.Action<string> callback)
+    {
+        UnityEngine.Debug.Log($"DevTools api Request: {apiAddress}");
+        UnityWebRequest webRequest;
+        if (isPUT)
+        {
+            webRequest = UnityWebRequest.Put($"http://localhost:{debugPort}{apiAddress}", "");
+        }
+        else
+        {
+            webRequest = UnityWebRequest.Get($"http://localhost:{debugPort}{apiAddress}");
+        }
+        yield return webRequest.SendWebRequest();
+
+        if (webRequest.result != UnityWebRequest.Result.Success)
+        {
+            UnityEngine.Debug.LogError(webRequest.error);
+            //TODO: handle error
+        }
+        else
+        {
+            UnityEngine.Debug.Log($"DevTools api response (for {apiAddress}):\n{webRequest.downloadHandler.text}");
+            callback(webRequest.downloadHandler.text);
+        }
+    }
+
+    private void OnDisable()
+    {
+        // TODO: do we want to close the browser when not in use?
+        // close browser when recompiling
+        if (browserProcess != null && !browserProcess.HasExited)
+        {
+            browserProcess.Kill();
+        }
+
+        UnityEngine.Debug.Log("BrowserView OnDestroy called");
+        ws.CloseAsync(WebSocketCloseStatus.NormalClosure, "BrowerView closed", CancellationToken.None)
+            .ContinueWith(delegate
+            {
+                UnityEngine.Debug.Log($"ws CloseAsync State:\n{ws.State}");
+            });
+        // TODO: fix close tab
+        // -> Coroutine couldn't be started because the the game object 'BrowserView' is inactive!
+        // consider using OnEnable and OnDisable
+        // see also https://stackoverflow.com/a/67699419
+        StartCoroutine(DevToolsApiRequest(false, $"/json/close/{pageTarget.Id}", (response) =>
+        {
+            UnityEngine.Debug.Log("browser tab has been closed");
+        }));
+    }
+    private void OnDestroy()
+    {
+        ws.Dispose();
+        cancellationTokenSource.Dispose();
+    }
+    /**
+     * Close all browser windows.
+     */
+    void OnApplicationQuit()
+    {
+        if (browserProcess != null && !browserProcess.HasExited)
+        {
+            browserProcess.Kill();
+        }
+    }
+}
+
+/**
+ * Json structs used in the chrome devtools protocol
+ */
+namespace ChromeDevToolsProtocol
+{
+    // suppress style warning, because names must correspond to json keys
+    //[System.Diagnostics.CodeAnalysis.SuppressMessage("Style", "IDE1006:Benennungsstile", Justification = "<Ausstehend>")]
+    public class PageTargetInfo
+    {
+        public string Description { get; set; }
+        public string DevtoolsFrontendUrl { get; set; }
+        public string Id { get; set; }
+        public string Title { get; set; }
+        public string Type { get; set; }
+        public string Url { get; set; }
+        public string WebSocketDebuggerUrl { get; set; }
+    }
+
+    namespace WebSocketProtocol
+    {
+        using Types;
+        public interface IDevtoolsCommand
+        {
+            long Id { get; }
+            string Method { get; }
+        }
+
+        ///
+        /// Every devtools command has an id and a method
+        ///
+        public class DevtoolsCommand : IDevtoolsCommand
+        {
+            private static long LAST_ID = 0;
+            public long Id { get; set; } = ++LAST_ID;
+            public string Method { get; set; }
+        }
+
+        ///
+        /// Every devtools command response has the same id and a method as the corresponding command
+        ///
+        public class DevtoolsCommandResponse<T>
+        {
+            public long Id { get; }
+            public string Method { get; }
+            public T Result { get; }
+        }
+
+        /// <summary>
+        /// Capture page screenshot.
+        /// </summary>
+        public class CaptureScreenshotCommand : DevtoolsCommand
+        {
+            public string Method { get; } = "Page.captureScreenshot";
+            /// <summary>
+            /// Gets or sets Image compression format (defaults to png).
+            /// </summary>
+            [JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
+            public string Format { get; set; }
+            /// <summary>
+            /// Gets or sets Compression quality from range [0..100] (jpeg only).
+            /// </summary>
+            [JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
+            public long? Quality { get; set; }
+            /// <summary>
+            /// Gets or sets Capture the screenshot of a given region only.
+            /// </summary>
+            [JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
+            public Viewport Clip { get; set; }
+            /// <summary>
+            /// Gets or sets Capture the screenshot from the surface, rather than the view. Defaults to true.
+            /// </summary>
+            [JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
+            public bool? FromSurface { get; set; }
+        }
+        /// <summary>
+        /// Capture page screenshot response.
+        /// </summary>
+        public class CaptureScreenshotCommandResponse
+        {
+            /// <summary>
+            /// Gets or sets Base64-encoded image data.
+            /// </summary>
+            public string Data { get; }
+        }
+
+        namespace Types
+        {
+            /// <summary>
+            /// Viewport for capturing screenshot.
+            /// </summary>
+            public class Viewport
+            {
+                /// <summary>
+                /// Gets or sets X offset in CSS pixels.
+                /// </summary>
+                public double X { get; set; } = 0;
+                /// <summary>
+                /// Gets or sets Y offset in CSS pixels
+                /// </summary>
+                public double Y { get; set; } = 0;
+                /// <summary>
+                /// Gets or sets Rectangle width in CSS pixels
+                /// </summary>
+                public double Width { get; set; } = 1920;
+                /// <summary>
+                /// Gets or sets Rectangle height in CSS pixels
+                /// </summary>
+                public double Height { get; set; } = 1080;
+                /// <summary>
+                /// Gets or sets Page scale factor.
+                /// </summary>
+                public double Scale { get; set; } = 1;
+            }
+        }
+    }
+}
\ No newline at end of file

Runtime/BrowserView.cs.meta

+fileFormatVersion: 2
+guid: 6893fa42f929f894286c6f00c40bcd06
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Runtime/in.esswe.browserview.asmdef

+{
+    "name": "in.esswe.browserview",
+    "rootNamespace": "",
+    "references": [],
+    "includePlatforms": [],
+    "excludePlatforms": [],
+    "allowUnsafeCode": false,
+    "overrideReferences": false,
+    "precompiledReferences": [],
+    "autoReferenced": true,
+    "defineConstraints": [],
+    "versionDefines": [],
+    "noEngineReferences": false
+}
\ No newline at end of file

Runtime/in.esswe.browserview.asmdef.meta

+fileFormatVersion: 2
+guid: e26a6836b550bcc4da6f0d171fe85881
+AssemblyDefinitionImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

package.json

+{
+    "name": "in.esswe.browserview",
+    "version": "0.0.1",
+    "description": "An interactive browser for unity running the systems browser in headless mode and rendering it to a texture instead.",
+    "displayName": "BrowserView",
+    "author": "Björn Eßwein",
+    "keywords": ["browser", "web", "webbrowser", "internet", "chrome", "chromium", "firefox"]
+}
\ No newline at end of file

package.json.meta

+fileFormatVersion: 2
+guid: 3e8c3b2599ea3824089453f18102d6da
+PackageManifestImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: