WebViewComponent.cs

using bessw.Unity.WebView.ChromeDevTools;
using bessw.Unity.WebView.ChromeDevTools.Protocol.DOM;
using bessw.Unity.WebView.ChromeDevTools.Protocol.Input;
using Newtonsoft.Json;
using Newtonsoft.Json.Serialization;
using System;
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.EventSystems;
using UnityEngine.UI;

namespace bessw.Unity.WebView
{
    [RequireComponent(typeof(RawImage))]
    [RequireComponent(typeof(RectTransform))]
    public class WebViewComponent : MonoBehaviour, IPointerDownHandler, IPointerMoveHandler, IPointerUpHandler, IPointerEnterHandler, IDropHandler, IPointerExitHandler
    {
        #region json serializer

        /// <summary>
        /// Json serializer settings for the user space objects (e.g. transferring objects that have been dropped on the BrowserView)
        /// Users are allowed to change serializer settings to their liking.
        /// </summary>
        public static JsonSerializerSettings serializerSettings = new JsonSerializerSettings
        {
            ContractResolver = new CamelCasePropertyNamesContractResolver(),
            Converters = new List<JsonConverter>()
                {
                    new ColorConverter(),
                    new Vector2Converter(),
                    new Vector3Converter(),
                    new Vector4Converter()
                }
        };
        /// <summary>
        /// JsonSerializer for the user space objects (e.g. transferring objects that have been dropped on the BrowserView)
        /// Users are allowed to change serializer settings to their liking.
        /// </summary>
        public static JsonSerializer serializer = JsonSerializer.Create(serializerSettings);

        #endregion json serializer

        private Browser browser;
        public BrowserTab tab;

        /// <summary>
        /// Fired when `Document` has been totally updated. Node ids are no longer valid.
        /// Other DOM events are fired directly on their `DomNodeWrapper` object.
        /// </summary>
        public event Action<documentUpdatedEvent> onDomDocumentUpdated
        {
            add => tab.devtools.onDomDocumentUpdated += value;
            remove => tab.devtools.onDomDocumentUpdated -= value;
        }

        public event Action OnWebViewComponentReady;

        private RawImage rawImage;
        private RectTransform rectTransform;

        public bool headlessBrowser = true;
        //TODO: handle changed targetUrl
        public string targetUrl = "https://google.de";

        public int PageScaleFactor = 3;

        // Start is called before the first frame update
        private void Start()
        {
            rawImage = this.gameObject.GetComponent<RawImage>();
            rectTransform = this.gameObject.GetComponent<RectTransform>();

            Browser.headless = headlessBrowser;
            browser = Browser.getInstance();

            //StartCoroutine(GetOpenTabs());
            var c = StartCoroutine(browser.OpenNewTab(targetUrl, (BrowserTab bt) =>
            {
                tab = bt;
                _ = tab.SetSizeAndScale(new Vector2Int((int)rectTransform.rect.width, (int)rectTransform.rect.height) / PageScaleFactor, PageScaleFactor);

                StartCoroutine(tab.Update());
                //StartCoroutine(createScreenshots());
                StartCoroutine(tab.StartStream(tab.Size.x, tab.Size.y, (frame) =>
                {
                    rawImage.texture = frame;
                    //rawImage.SetNativeSize();
                }));
                OnWebViewComponentReady?.Invoke();
            }));
        }

        private void OnRectTransformDimensionsChange()
        {
            if (tab is not null)
            {
                var size = new Vector2Int((int)rectTransform.rect.width, (int)rectTransform.rect.height) / PageScaleFactor;
                if (tab.Size != size)
                {
                    _ = tab.SetSizeAndScale(size, PageScaleFactor);
                }
            }
        }

        public IEnumerator createScreenshots ()
        {
            yield return tab.CreateScreenshot(new()
            {
                Width = tab.Size.x,
                Height = tab.Size.y,
                
            }, (screenshot) =>
            {
                rawImage.texture = screenshot;
                StartCoroutine(createScreenshots());
            });
        }

        // Update is called once per frame
        private void Update()
        {

        }

        #region pointer event handlers

        public void OnPointerDown(PointerEventData eventData)
        {
            Vector2Int pos = toBrowserCoordinates(eventData.position);
            tab.OnPointerDown(pos, eventData);
        }

        public void OnPointerMove(PointerEventData eventData)
        {
            // TODO: transform eventData delta vector to browser coordinates
            Vector2Int pos = toBrowserCoordinates(eventData.position);
            tab.OnPointerMove(pos, eventData);

            // On drag over
            if (eventData.dragging)
            {
                Debug.LogWarning($"OnDragOver: {eventData.position}");
                createDragEvent(DragEventType.dragOver, eventData);
            }
        }

        public void OnPointerUp(PointerEventData eventData)
        {
            Vector2Int pos = toBrowserCoordinates(eventData.position);
            tab.OnPointerUp(pos, eventData);
        }
        #endregion pointer event handlers

        #region drag event handlers
        public void OnPointerEnter(PointerEventData eventData)
        {
            if (eventData.dragging)
            {
                //Debug.LogWarning($"OnDragEnter: {eventData.position}");
                createDragEvent(DragEventType.dragEnter, eventData);
            }
        }

        public void OnDrop(PointerEventData eventData)
        {
            //Debug.LogWarning($"OnDrop: {eventData.position}");
            createDragEvent(DragEventType.drop, eventData);
        }

        public void OnPointerExit(PointerEventData eventData)
        {
            if (eventData.dragging)
            {
                //Debug.LogWarning($"OnDragLeave: {eventData.position}");
                //createDragEvent(DragEventType.dragCancel, eventData);
                tab.CancelDragging();
                // TODO: drag cancel seems to be ignored by the browser
            }
        }

        private void createDragEvent(DragEventType dragEventType, PointerEventData eventData)
        {
            if (eventData.pointerDrag.TryGetComponent(out IBrowserDroppable droppable))
            {
                var position = toBrowserCoordinates(eventData.position);
                var dragEvent = new dispatchDragEvent
                {
                    type = dragEventType,
                    data = new DragData
                    {
                        items = new DragDataItem[]
                        {
                            new DragDataItem
                            {
                                mimeType = "application/json",
                                data = JsonConvert.SerializeObject(droppable, serializerSettings)
                            }
                        },
                        dragOperationsMask = DragOperationsMask.Copy
                    },
                    x = position.x,
                    y = position.y,
                };
                Debug.LogWarning($"DragEvent: {dragEvent.type}, {eventData.position}, '{dragEvent.data.items[0].data}'");
                // send the DragEvent as drag event to the browser
                tab.OnDragNDrop(dragEvent);
            }
        }
        #endregion drag event handlers

        private Vector2Int toBrowserCoordinates(Vector2 eventPos)
        {
            RectTransformUtility.ScreenPointToLocalPointInRectangle(rectTransform, eventPos, null, out Vector2 localPoint);

            // invert y because the browser has y=0 on the top
            Vector2 invertedLocalPos = new Vector2(localPoint.x, rectTransform.rect.size.y - localPoint.y);
            Vector2 textureScale  = tab.StreamSize / rectTransform.rect.size;
            Vector2 browserCoordinate = invertedLocalPos * textureScale;// / tab.ViewportScaleFactor;
            //Debug.Log($"eventPos: {eventPos}, invertedLocalPos: {invertedLocalPos}, browserCoordinate: {browserCoordinate}");
            return new Vector2Int((int) browserCoordinate.x, (int) browserCoordinate.y);
        }

        private void OnDisable()
        {
            // TODO: do we want to close the browser when not in use?
            // close browser when recompiling
            tab.Close();
            browser.Close();
        }
        private void OnDestroy()
        {
            tab.Close();
            browser.Close();
        }
        /**
         * Close all browser windows.
         */
        private void OnApplicationQuit()
        {
            tab.Close();
            browser.Close();
        }
    }

    public interface IBrowserDroppable { }

}