/* * Copyright (C) 2021 because-why-not.com Limited * * Please refer to the license.txt for license information */ using System; using Unity.Collections; using Unity.Collections.LowLevel.Unsafe; using UnityEngine; using UnityEngine.UI; namespace Byn.Awrtc.Unity { public class UnityMediaHelper { /// /// Material used to show i420 image via Unitys RawImage /// public static readonly string I420_SINGLE_MAT_NAME = "I420_one_buffer"; /// /// Updates a texture with a new IFrame. /// Only ABGR (RGBA32 in unity) frames are properly supported at the moment. /// /// /// /// public static bool UpdateTexture(IFrame frame, ref Texture2D tex) { var format = frame.Format; if (frame.Format == FramePixelFormat.ABGR) { bool newTextureCreated = false; //texture exists but has the wrong height /width? -> destroy it and set the value to null if (tex != null && (tex.width != frame.Width || tex.height != frame.Height)) { Texture2D.Destroy(tex); tex = null; } //no texture? create a new one first if (tex == null) { newTextureCreated = true; Debug.Log("Creating new texture with resolution " + frame.Width + "x" + frame.Height + " Format:" + format); //so far only ABGR is really supported. this will change later if (format == FramePixelFormat.ABGR) { tex = new Texture2D(frame.Width, frame.Height, TextureFormat.RGBA32, false); } else { Debug.LogWarning("YUY2 texture is set. This is only for testing"); tex = new Texture2D(frame.Width, frame.Height, TextureFormat.YUY2, false); } tex.wrapMode = TextureWrapMode.Clamp; } //copy image data into the texture and apply //Watch out the RawImage has the top pixels in the top row but //unity has the top pixels in the bottom row. Result is an image that is //flipped. Fixing this here would waste a lot of CPU power thus //the UI will simply set scale.Y of the UI element to -1 to reverse this. tex.LoadRawTextureData(frame.Buffer); tex.Apply(); return newTextureCreated; } else if (frame.Format == FramePixelFormat.I420p && frame is IDirectMemoryFrame) { //Watch out this conversion is only possible on native platforms //and some might still crash later if their internal doesn't use the correct internal format var dframe = frame as IDirectMemoryFrame; bool newTextureCreated = EnsureTex(frame, TextureFormat.R8, ref tex); NativeArray data = tex.GetRawTextureData(); if(data == null || data.IsCreated == false) { Debug.LogWarning("GetRawTextureData failed"); return false; } unsafe { int ystride = tex.width; int ustride = tex.width; int vstride = tex.width; int uoffset = tex.width * dframe.Height; //rounding down here otherwise we shoot past the image and corrupt memory int voffset = uoffset + dframe.Width / 2; byte* startPtr = (byte*)data.GetUnsafePtr(); IntPtr y = (IntPtr)(startPtr); IntPtr u = (IntPtr)(startPtr + uoffset); IntPtr v = (IntPtr)(startPtr + voffset); //WARNING: VERY HIGH RISK ACCESS HERE //this directly accesses the image within WebRTC's memory //if the structure of the memory is different on untested platforms //or after updates this will cause a crash or memory corruption //Copy & convert into a format usable by the shaders dframe.ToBufferI420p(y, ystride, u, ustride, v, vstride); } dframe.Dispose(); tex.Apply(); return newTextureCreated; } else if (frame.Format == FramePixelFormat.Native) { var dframe = frame as TextureFrame; var newTexture = dframe.TakeOwnership(); //if the system created a new texture destroy the old if (tex != null && tex != newTexture) { //Debug.Log("Destroying old texture " + tex.width + "x" + tex.height + ". New tex " + newTexture.width + "x" + newTexture.height); Texture2D.Destroy(tex); tex = null; } tex = newTexture; //Debug.Log("Texture id out: " + (int)tex.GetNativeTexturePtr()); dframe.Dispose(); //tex.Apply(); return true; } else { Debug.LogError("Format not supported"); return false; } } public static void CalcTextureResolution(IFrame frame, out int width, out int height) { width = frame.Width; height = frame.Height; if (frame is IDirectMemoryFrame) { height = frame.Height + ((frame.Height + 1) / 2); } } public static bool NeedsRecreate(IFrame frame, Texture2D tex) { if (tex == null) return true; int width; int height; CalcTextureResolution(frame, out width, out height); if ((tex.width != width || tex.height != height)) return true; return false; } /// /// Helper to ensure the texture has a fixed size and format. /// If not it will be created / destroyed and recreated /// /// /// /// /// /// True if a new texture was created. /// Meaning external references need to be refreshed! /// private static bool EnsureTex(IFrame frame, TextureFormat texFormat, ref Texture2D tex) { if (NeedsRecreate(frame, tex)) { Texture2D.Destroy(tex); tex = null; } if (tex == null) { int width; int height; CalcTextureResolution(frame, out width, out height); tex = new Texture2D(width, height, texFormat, false); return true; } else { return false; } } /// /// Helper to update a RawImage based on IFrame. /// /// Watch out this method might change the material and textures /// used for the RawImage. /// /// This method can be used in the future to support i420p using /// a specific shader / material / texture combination. /// /// /// /// /// public static bool UpdateRawImage(RawImage target, IFrame frame) { if (target == null || frame == null) return false; bool textureCreated; if (frame.Format == FramePixelFormat.I420p && frame is IDirectMemoryFrame) { string matname = I420_SINGLE_MAT_NAME; if (matname == null) { if (target.material != target.defaultMaterial) { target.material = target.defaultMaterial; } } else if (target.material.name != matname) { Material mat = Resources.Load(matname); target.material = new Material(mat); } Texture2D mainTex = target.texture as Texture2D; //remove for now to avoid destroying a used texture if (NeedsRecreate(frame, mainTex)) { target.texture = null; } //update existing textures or create new ones textureCreated = UnityMediaHelper.UpdateTexture(frame, ref mainTex); if (textureCreated) { //new textures where creates (e.g. due to resolution change) //update the UI target.texture = mainTex; } frame.Dispose(); frame = null; } else { if (target.material != target.defaultMaterial) { target.material = target.defaultMaterial; } Texture2D mainTex = target.texture as Texture2D; textureCreated = UnityMediaHelper.UpdateTexture(frame, ref mainTex); target.texture = mainTex; } return textureCreated; } public static bool UpdateRawImageTransform(RawImage target, IFrame frame, bool mirror) { bool textureCreated = UpdateRawImage(target, frame); float mirrorVal; float rotFactor; if (mirror) { mirrorVal = -1; rotFactor = 1; } else { mirrorVal = 1; rotFactor = -1; } float upSideDown = 1; if (frame.IsTopRowFirst) { upSideDown = -1; } target.transform.localScale = new Vector3(mirrorVal, upSideDown, 1); target.transform.localRotation = Quaternion.Euler(0, 0, frame.Rotation * rotFactor); return textureCreated; } //do not use. Note all platforms return an image in the expected format anymore /* public static bool UpdateTextureThreeBuffers(IDirectMemoryFrame frame, ref Texture2D yplane, ref Texture2D uplane, ref Texture2D vplane) { if (frame.Format == FramePixelFormat.I420p) { var dframe = frame as IDirectMemoryFrame; int width = frame.Width; int height = frame.Height; int hwidth = frame.Width / 2; int hheight = (frame.Height + 1) / 2; TextureFormat texFormat = TextureFormat.R8; bool newTextureCreated = EnsureTexRaw(width, height, texFormat, ref yplane); newTextureCreated |= EnsureTexRaw(hwidth, hheight, texFormat, ref uplane); newTextureCreated |= EnsureTexRaw(hwidth, hheight, texFormat, ref vplane); IntPtr ystart = dframe.GetIntPtr(); long ylength = width * height; IntPtr ustart = new IntPtr(ystart.ToInt64() + ylength); long ulength = (hwidth * hheight); IntPtr vstart = new IntPtr(ustart.ToInt64() + ulength); yplane.LoadRawTextureData(ystart, (int)ylength); uplane.LoadRawTextureData(ustart, (int)ulength); vplane.LoadRawTextureData(vstart, (int)ulength); yplane.Apply(); uplane.Apply(); vplane.Apply(); return newTextureCreated; } else { Debug.LogError("Format not supported"); return false; } } private static bool EnsureTexRaw(int width, int height, TextureFormat texFormat, ref Texture2D tex) { if (tex != null) { Texture2D.Destroy(tex); tex = null; } if (tex == null) { tex = new Texture2D(width, height, texFormat, false); return true; } else { return false; } } */ } }