Porting C# script to UdonSharp results in 'Type" archetype error

Error in Question
“Assets/Scripts/PixelScreen/UdonPixelSc.cs(41,12): System.Exception: Capture scope must have the ‘Type’ archetype to convert to an array”

Full Code Dump

using UdonSharp;
using System.Collections;
using UnityEngine;
using VRC.SDKBase;
using VRC.Udon;





public class UdonPixelScreen : UdonSharpBehaviour
{
    public bool debug;

    #region InternalRefs
    [Header("Internal Refs")]
    [SerializeField] private Camera RenderingCamera;
    [SerializeField] private GameObject PixelHub;
    [SerializeField] private GameObject pixel;
    #endregion

    #region PixelGrid

    [System.Serializable]
    public struct PixelData {
        public bool mutex;
        public SpriteRenderer renderer;


        public PixelData(bool _mutex, SpriteRenderer _renderer) {
            mutex = _mutex;
            renderer = _renderer;
        }


    }

    //public static System.Type PixelArray = typeof(PixelData).MakeArrayType(2);

    private PixelData[][] TheGrid;

    #endregion

    #region EditorVariables
    [Header("Grid Settings")]
    [Tooltip("Controls the size of the pixel grid scaled by 4x3")]
    [Range(1, 50, order = 1)]
    [SerializeField] private int Scale = 1;

    [Tooltip("Please set the Ratio values above 1. Y Should be a multiple of Scale.")]
    [SerializeField] private Vector2Int Ratio = new Vector2Int(1, 1);

    /* Storing of Pixel Jobs to run asynchronously */
    [System.Serializable]
    public struct PixelJob {
        public ScreenEffect effect;
        [ColorUsage(true)]
        public Color color;
        public Vector2Int position;
        [Range(0.1f, 10f)]
        public float rate;
        [Range(0.1f, 5f)]
        public float flow;

        public PixelJob(ScreenEffect _effect, Color _color, Vector2Int _position = new Vector2Int(), float _rate = 1f, float _flow = 0.1f) {
            effect = _effect;
            color = _color;
            position = _position;
            rate = _rate;
            flow = _flow;
        }
    }

    [Header("Job List")]
    [SerializeField] private PixelJob[] jobs;

    [System.Serializable]
    public struct PixelGlitch {
        public GlitchEffect effect;
        public Vector2Int position;
        [Range(0.1f, 10f)]
        public float rate;
    }

    [Header("Glitch List")]
    [SerializeField] private PixelGlitch[] glitches;
    #endregion

    #region UnityFunctions
    // Start is called before the first frame update
    void Start() {
        Random.InitState((int)System.DateTime.Now.Ticks);
        InitPixels();
        foreach (PixelJob _job in jobs) {
            RunJob(_job);
        }
        foreach (PixelGlitch _glitch in glitches) {
            RunGlitch(_glitch);
        }
    }
    #endregion

    #region EditorFunctions

    public void InitCamera() {
        if (!RenderingCamera) {
            _error("No rendering camera! Please attach the camera before invoking InitCamera");
            return;
        }

        RenderingCamera.orthographicSize = Scale;
        RenderingCamera.transform.localPosition = new Vector3(Scale * Ratio.y / Ratio.x, Scale, -1);
    }

    public void InitPixels() {
        if (!pixel) {
            _error("No pixel prefab to build! Please attach a pixel prefab!");
        }
        if (!PixelHub) {
            _error("No PixelHub gameobject to build pixels inside! Please attach a PixelHub gameobject!");
        }
        if (Scale % Ratio.y != 0) {
            _error("Ratio y value is not a multiple of Scale. Please adjust values to satisfy this.");
        }

        TransformUtil.KillChildren(PixelHub.transform);

        int GridY = Scale * 2;
        int GridX = GridY / Ratio.y * Ratio.x;

        TheGrid = new PixelData[GridX][];

        float pixelOffset = 0.5f;

        for (int i = 0; i < GridX; i++) {

            TheGrid[i] = new PixelData[GridY];

            for (int k = 0; k < GridY; k++) {
                GameObject newPixel = Instantiate(pixel, PixelHub.transform);
                newPixel.transform.localPosition = new Vector3(i + pixelOffset, k + pixelOffset, 0);
                newPixel.gameObject.name = "Pixel " + (i * GridY + k).ToString();
                TheGrid[i][k] = new PixelData(false, newPixel.GetComponent<SpriteRenderer>());
            }
        }



    }

    #endregion

    #region PublicFunctions

    public void ClearGrid() {
        PixelJob job = new PixelJob(ScreenEffect.SET_ALL, Color.black);
        RunJob(job);
    }
    #endregion

    #region PrivateFunctions
    private bool InBounds(Vector2Int _pos) {
        return 0 <= _pos.y && _pos.y < TheGrid[0].Length && 0 <= _pos.x && _pos.x < TheGrid.Length;
    }

    private void RunSetAll(Color _color) {

        for (int i = 0; i < TheGrid.Length; i++) {
            for (int k = 0; k < TheGrid[0].Length; k++) {
                TheGrid[i][k].renderer.color = _color;
            }
        }
    }

    private void RunGlitch(PixelGlitch _glitch) {
        switch (_glitch.effect) {
            case GlitchEffect.NONE:
                _log("No glitch effect given! Please make sure that this is intended behavior.");
                break;
            case GlitchEffect.SHIFT_COL:
                StartCoroutine(ColShift(_glitch.position, _glitch.rate));
                break;
            case GlitchEffect.SHIFT_ROW:
                StartCoroutine(RowShift(_glitch.position, _glitch.rate));
                break;
            case GlitchEffect.SHIFT_RAND:
                StartCoroutine(RandShift(_glitch.rate));
                break;
            case GlitchEffect.DISTORT_ALL:
                break;
        }
    }

    private void RunJob(PixelJob _job) {
        if (!InBounds(_job.position)) {
            _error("Job has invalid coordinate values.");
            return;
        }
        _log("Running Job with id: " + _job.effect.ToString());

        switch (_job.effect) {
            case ScreenEffect.NONE:
                _log("Job given without an effect! Double check that this is correct.");
                break;
            case ScreenEffect.PULSE_COL:
                StartCoroutine(ColumnPulseStart(_job.color, _job.position, _job.rate, _job.flow));
                break;
            case ScreenEffect.PULSE_FULL:
                break;
            case ScreenEffect.PULSE_ROW:
                StartCoroutine(RowPulseStart(_job.color, _job.position, _job.rate, _job.flow));
                break;
            case ScreenEffect.VOLCANO:
                StartCoroutine(Volcano(_job.color, _job.position, _job.rate, _job.flow));
                break;
            case ScreenEffect.VOLCANO_COL:
                for (int k = 0; k < TheGrid[0].Length; k++) {
                    Vector2Int newPos = new Vector2Int(_job.position.x, k);
                    StartCoroutine(Volcano(_job.color, newPos, _job.rate, _job.flow));
                }
                break;
            case ScreenEffect.VOLCANO_ROW:
                for (int i = 0; i < TheGrid.Length; i++) {
                    Vector2Int newPos = new Vector2Int(i, _job.position.y);
                    StartCoroutine(Volcano(_job.color, newPos, _job.rate, _job.flow));
                }
                break;
            case ScreenEffect.VOLCANO_RAND:
                StartCoroutine(RandomVolcanoes(_job.color, _job.rate, _job.flow));
                break;
            case ScreenEffect.SET_ALL:
                RunSetAll(_job.color);
                break;
        }
    }
    #endregion

    #region GlitchIEnumerators

    private IEnumerator RandShift(float _rate) {
        while (true) {
            int roll = Random.Range(0, 2);
            switch (roll) {
                case 0:
                    int y = Random.Range(0, TheGrid[0].Length);
                    RShift(new Vector2Int(0, y));
                    break;
                case 1:
                    int x = Random.Range(0, TheGrid.Length);
                    CShift(new Vector2Int(x, 0));
                    break;
            }

            float variance = 0.5f + Random.Range(0f, 1f);
            yield return new WaitForSeconds(_rate * variance);
        }
    }


    private void RShift(Vector2Int _pos) {
        Color passing;
        Color apply = Color.white;
        for (int i = 0; i < TheGrid.Length; i++) {
            passing = TheGrid[i][_pos.y].renderer.color;
            TheGrid[i][_pos.y].renderer.color = apply;
            apply = passing;
        }
        TheGrid[0][_pos.y].renderer.color = apply;
    }

    private void CShift(Vector2Int _pos) {
        Color passing;
        Color apply = Color.white;
        for (int k = 0; k < TheGrid[0].Length; k++) {
            passing = TheGrid[_pos.x][k].renderer.color;
            TheGrid[_pos.x][k].renderer.color = apply;
            apply = passing;
        }
        TheGrid[_pos.x][0].renderer.color = apply;
    }

    private IEnumerator RowShift(Vector2Int _pos, float _rate) {
        while (true) {
            RShift(_pos);
            yield return new WaitForSeconds(_rate);
        }
    }

    private IEnumerator ColShift(Vector2Int _pos, float _rate) {
        while (true) {
            CShift(_pos);
            yield return new WaitForSeconds(_rate);
        }
    }

    #endregion

    #region JobIEnumerators

    //PULSE ENUMS
    private IEnumerator ColumnPulseStart(Color _c, Vector2Int _pos, float _rate, float _flow) {
        while (true) {

            StartCoroutine(ColumnPulseShot(_c, new Vector2Int(_pos.x, 0), _flow));

            yield return new WaitForSeconds(_rate);
        }
    }

    private IEnumerator ColumnPulseShot(Color _c, Vector2Int _pos, float _flow) {
        if (InBounds(_pos)) {
            for (int k = -1; k <= 1; k++) {
                Vector2Int pos = new Vector2Int(_pos.x, _pos.y + k);
                if (InBounds(pos)) {
                    StartCoroutine(ColorLinger(_c, pos, _flow * 3f));
                }
            }
            yield return new WaitForSeconds(_flow);
            StartCoroutine(ColumnPulseShot(_c, new Vector2Int(_pos.x, _pos.y + 1), _flow));
        }
    }

    private IEnumerator RowPulseStart(Color _c, Vector2Int _pos, float _rate, float _flow) {
        while (true) {

            StartCoroutine(RowPulseShot(_c, new Vector2Int(0, _pos.y), _flow));

            yield return new WaitForSeconds(_rate);
        }
    }

    private IEnumerator RowPulseShot(Color _c, Vector2Int _pos, float _flow) {
        if (InBounds(_pos)) {
            for (int i = -1; i <= 1; i++) {
                Vector2Int pos = new Vector2Int(_pos.x + i, _pos.y);
                if (InBounds(pos)) {
                    StartCoroutine(ColorLinger(_c, pos, _flow * 3f));
                }
            }
            yield return new WaitForSeconds(_flow);
            StartCoroutine(RowPulseShot(_c, new Vector2Int(_pos.x + 1, _pos.y), _flow));
        }
    }

    private IEnumerator RandomVolcanoes(Color _c, float _rate, float _flow) {
        while (true) {
            int x = Random.Range(0, TheGrid.Length);
            int y = Random.Range(0, TheGrid[0].Length);
            Vector2Int pos = new Vector2Int(x, y);

            Coroutine volcano = StartCoroutine(Volcano(_c, pos, _rate / 2f, _flow));

            yield return new WaitForSeconds(_rate);

            StopCoroutine(volcano);
            StartCoroutine(SubColor(_c * 5f, pos));
        }
    }

    //A type of effect where a central pixel creates a color and spreads out from it, using randomization to limit the range of spread.
    private IEnumerator Volcano(Color _c, Vector2Int _pos, float rate, float _flow) {
        StartCoroutine(AddColor(_c * 5f, _pos));
        while (true) {
            //Create VolcanoChildren adjacent
            for (int i = -1; i <= 1; i++) {
                for (int k = -1; k <= 1; k++) {
                    if (Mathf.Abs(i) != Mathf.Abs(k)) {
                        Vector2Int childPos = new Vector2Int(_pos.x + i, _pos.y + k);
                        if (InBounds(childPos)) {
                            StartCoroutine(VolcanoChild(_c, childPos, _flow, 0));
                        }
                    }
                }
            }
            yield return new WaitForSeconds(rate);
        }
    }
    private int LifeSpan = 1;
    //Child of the Volcano effect, representing the flow of color from the center point.
    private IEnumerator VolcanoChild(Color _c, Vector2Int _pos, float _flow, int _lifespan) {
        if (_lifespan > LifeSpan) { /*Debug.Log("KILLING VOLCANO");*/ yield break; }
        StartCoroutine(ColorLinger(_c, _pos, _flow * 1.5f));
        float variance = 0.5f + Random.Range(0f, 1f);
        yield return new WaitForSeconds(_flow * variance);
        for (int i = -1; i <= 1; i++) {
            for (int k = -1; k <= 1; k++) {
                if (Mathf.Abs(i) != Mathf.Abs(k)) {
                    Vector2Int childPos = new Vector2Int(_pos.x + i, _pos.y + k);
                    if (InBounds(childPos)) {
                        int CoinFlip = Random.Range(0, 3);
                        if (CoinFlip == 0) {
                            StartCoroutine(VolcanoChild(_c, childPos, _flow, _lifespan++));
                        }
                    }
                }
            }
        }

    }

    private IEnumerator AddColor(Color _c, Vector2Int _pos) {
        yield return new WaitUntil(() => !TheGrid[_pos.x][_pos.y].mutex);
        TheGrid[_pos.x][_pos.y].mutex = true;
        TheGrid[_pos.x][_pos.y].renderer.color += _c;
        TheGrid[_pos.x][_pos.y].mutex = false;
    }

    private IEnumerator SubColor(Color _c, Vector2Int _pos) {
        yield return new WaitUntil(() => !TheGrid[_pos.x][_pos.y].mutex);
        TheGrid[_pos.x][_pos.y].mutex = true;
        TheGrid[_pos.x][_pos.y].renderer.color -= _c;
        TheGrid[_pos.x][_pos.y].mutex = false;
    }

    private IEnumerator ColorLinger(Color _c, Vector2Int _pos, float _length) {
        yield return new WaitUntil(() => !TheGrid[_pos.x][_pos.y].mutex);
        TheGrid[_pos.x][_pos.y].mutex = true;
        TheGrid[_pos.x][_pos.y].renderer.color += _c;
        TheGrid[_pos.x][_pos.y].mutex = false;
        yield return new WaitForSeconds(_length);
        yield return new WaitUntil(() => !TheGrid[_pos.x][_pos.y].mutex);
        TheGrid[_pos.x][_pos.y].mutex = true;
        TheGrid[_pos.x][_pos.y].renderer.color -= _c;
        TheGrid[_pos.x][_pos.y].mutex = false;
    }

    #endregion

    #region Debug/Util

    private void _log(string _message) {
        if (debug)
            Debug.Log("[Pixel Screen Manager] " + _message);
    }

    private void _error(string _message) {
        if (debug)
            Debug.LogError("[Pixel Screen Manager] " + _message);
    }
    #endregion
}

Problem in Question

The purpose of this script is to simulate pixels moving around in a fashion similar to conway’s game of life. This is done by a two-dimensional array of data defined by the PixelData type.

The PixelData I created is an approach I enjoy using in C# to have a data type I can expand on, but upon switching to UdonSharp it does not seem to comprehend this type.

My Main Questions Are

Is there a way to satisfy the error in question by defining a type archetype for PixelData?

If not, is there an approach to the PixelData structure that fits within UdonSharp’s system?

Demo of the Script Running in C#
skybox example

Thank you! <3

U# does not support structs (yet) But you can use object jagged array and cast it to the correct one when you need it. Object[][] will work and will be pretty much what U# will be doing behind the scene when generating the UASM

StartCoroutine is also not supported, together with IEnumerators. You’ll have to use something like SendCustomEventDelayedFrames or SendCustomEventDelayedSeconds to simulate the same thing. Or your own way.

1 Like

As Phaxenor mentions, coroutines are not supported. If all you want is to have something happen after a fixed period of time, then the typical way around this is to just use SendCustomEventDelayed.

However your use case is to run it asynchronously for performance reasons, not just to delay it. Udon does not have any way of multi-threading, so you will not be able to make use of asynchronous events whatsoever. In addition to that, Udon is a virtual machine running inside of C#, so it is quite a bit slower than C# itself. Because of that, I worry that running all of this on the main thread every single frame will have major performance problems.

If this is a project you’re using just to get familiar with udonsharp, I would simply recommend trying something else that is less performance-intensive. Perhaps building in U# from the beginning will help to see where the limits are before building your entire structure on something that is not supported.

However if you actually need this to work, I would recommend doing this in a shader. It’s a completely different programming style, but GPUs are much much more capable of handling lots of pixels like this.

2 Likes

Thank you very much for this insight! I hadn’t known of all of Udonsharp’s limitations and this will save me a lot of time going forward.

On one followup to what was mentioned, I’m curious which type of shader you were recommending. I had looked into shaders that can support this type of simulation…but the main result, Compute Shaders, are not supported by Udon yet…

I’m no expert on shaders so I can’t help with the specifics much, but the general idea is that you use a camera to create a feedback loop. The camera renders to a texture which is then put on a flat plane directly in front of the camera. Once you have that set up, every pixel is a piece of memory and you can perform operations on it. That is technically all you need to do anything from the game of life to a face recognition neural network.

1 Like