2

How do I convert a structure that contains an array to a byte array in C#?

There was a question here about a struct without array.

But if the struct contains an array like this:

public struct DiObject
{
    public byte Command; 
    public byte ErrorClass; 
    public byte Reserved; 
    public byte Flags; 
}

public struct MyPacket
{
    public uint ProtocolIdentifier; 
    public uint NumDi;    
    public DiObject[] Di; 
}

It results with an access violation exception when converting the struct in a byte:

private static byte[] GetBytes(MyPacket packet, int packetSize)
{
    var data = new byte[packetSize];
    var ptr = Marshal.AllocHGlobal(packetSize);

    // ==== Access violation exception occurs here ====
    Marshal.StructureToPtr(packet, ptr, true);

    Marshal.Copy(ptr, data, 0, packetSize);
    Marshal.FreeHGlobal(ptr);
    return data;
}

My goal is to send a message in bytes in a message queue with MSMQ.

Here the complete code that compiles and reproduce the problem.

using System;
//using System.IO;
//using System.Messaging;
using System.Runtime.InteropServices;

namespace StructToBytes
{
    // 4 bytes
    [Serializable]
    public struct DiObject
    {
        public byte Command; 
        public byte ErrorClass; 
        public byte Reserved; 
        public byte Flags; 
    }

    // 8 + (numDi*4) bytes
    [Serializable]
    public struct MyPacket
    {
        public uint ProtocolIdentifier; 
        public uint NumDi;    
        public DiObject[] Di; 
    }

    internal class Program
    {
        private static byte[] GetBytes(MyPacket packet, int packetSize)
        {
            var data = new byte[packetSize];
            var ptr = Marshal.AllocHGlobal(packetSize);

            // ==== Access violation exception occurs here ====
            Marshal.StructureToPtr(packet, ptr, true);

            Marshal.Copy(ptr, data, 0, packetSize);
            Marshal.FreeHGlobal(ptr);
            return data;
        }

        private static MyPacket FromBytes(byte[] data)
        {
            var packet = new MyPacket();
            var dataSize = Marshal.SizeOf(packet);
            var ptr = Marshal.AllocHGlobal(dataSize);
            Marshal.Copy(data, 0, ptr, dataSize);
            packet = (MyPacket) Marshal.PtrToStructure(ptr, packet.GetType());
            Marshal.FreeHGlobal(ptr);
            return packet;
        }

        private static void Main(string[] args)
        {
            const string queuePath = @".\private$\test_msmq";

            // Create the packet
            var packet = new MyPacket();

            // 8 bytes
            packet.ProtocolIdentifier = 1;
            packet.NumDi = 2;

            // 8 bytes
            packet.Di = new DiObject[packet.NumDi];
            packet.Di[0].Command = 2;
            packet.Di[0].ErrorClass = 3;
            packet.Di[0].Flags = 4;
            packet.Di[0].Reserved = 5;
            packet.Di[1].Command = 6;
            packet.Di[1].ErrorClass = 7;
            packet.Di[1].Flags = 8;
            packet.Di[1].Reserved = 9;

            // Convert the struct in bytes
            const int packetSize = 16;
            var packetBytes = GetBytes(packet, packetSize);

            // Create the message
            /*
            var msg = new Message();
            msg.BodyStream = new MemoryStream(packetBytes);

            // Open or create the message queue
            if (!MessageQueue.Exists(queuePath))
                MessageQueue.Create(queuePath);

            // Open the queue
            var q = new MessageQueue(queuePath); // {Formatter = new BinaryMessageFormatter()};

            // Send the message to the queue
            q.Send(msg);
            */
        }
    }
}
skink
  • 5,133
  • 6
  • 37
  • 58
Less White
  • 561
  • 4
  • 13
  • Just curious - why not just let the `BinaryMessageFormatter` serialize this? Is it about trying to make the message smaller? The overhead of the MSMQ message is already going to be larger than the message content itself, so with something this small it's not likely to make much of a difference. – Scott Hannen Oct 24 '17 at 14:39
  • If I use the BinaryMessageFormatter, it stores the data as 340 bytes instead of 16 bytes. I would like the smallest packet possible. Here the code that shows this: // Open or create the message queue if (!MessageQueue.Exists(queuePath)) MessageQueue.Create(queuePath); // Open the queue var q = new MessageQueue(queuePath); // {Formatter = new BinaryMessageFormatter()}; var message = new Message(packet, new BinaryMessageFormatter()); q.Send(message); – Less White Oct 24 '17 at 14:42
  • In short, I would like the smallest packet possible since I would like to store a lot of digital and analog points. This example is small just for having it simple. – Less White Oct 24 '17 at 14:44
  • 1
    Not answers the question but I'd suggest to just use more compact serialization protocol (but not `Marshal` stuff). For example you can use protobuf which will serialize your packet with one `DiObject` to 14 bytes which is not bad. – Evk Oct 24 '17 at 14:51
  • I think if you look into those 340 bytes you're going to find that a tiny part of that is your serialized data and the rest is overhead. Even an empty message is around 272 bytes. If you want to make your messages smaller you'll get more mileage out of putting more data into each message. Fewer messages = less overhead = less storage. – Scott Hannen Oct 24 '17 at 14:53
  • 1
    Let Google `Protocol Buffers` do the heavy lifting for you. It's standard, widely adopted, and is fully capable of solving your problem with minimal code requirements on your part (as in, you just have to define a .proto contract file). – code4life Oct 24 '17 at 14:57

1 Answers1

2

The problem lies with wrong assumption about how structure is represented in C#

// 8 + (numDi*4) bytes
[Serializable]
public struct MyPacket
{
    public uint ProtocolIdentifier;
    public uint NumDi;
    public DiObject[] Di;
}

The assumption that size of public DiObject[] Di member is numDi * 4 is not true. In place of this field there is a pointer to the array of structures. Array is a class in .NET and is not included in place in structure declaration.

To solve this problem one can use fixed arrays. I understand that the idea behind the design is to get variable length array and it is presented in next code listing.

This code does not raise AccessViolationException during executin:

using System;
using System.IO;
using System.Messaging;
using System.Runtime.InteropServices;

namespace StructToBytes
{
    // 4 bytes
    [Serializable]
    [StructLayout(LayoutKind.Explicit)]
    public unsafe struct DiObject
    {
        [FieldOffset(0)]
        public byte Command;

        [FieldOffset(1)]
        public byte ErrorClass;

        [FieldOffset(2)]
        public byte Reserved;

        [FieldOffset(3)]
        public byte Flags;
    }

    // 8 + (numDi*4) bytes
    [Serializable]
    public unsafe struct MyPacket
    {
        public uint ProtocolIdentifier;
        public uint NumDi;
        public fixed byte Di[2 * 4];
    }

    internal unsafe class Program
    {
        private static byte[] GetBytes(MyPacket packet, int packetSize)
        {
            var data = new byte[packetSize];
            var ptr = Marshal.AllocHGlobal(packetSize);

            // ==== Access violation exception occurs here ====
            Marshal.StructureToPtr(packet, ptr, true);

            Marshal.Copy(ptr, data, 0, packetSize);
            Marshal.FreeHGlobal(ptr);
            return data;
        }

        private static MyPacket FromBytes(byte[] data)
        {
            var packet = new MyPacket();
            var dataSize = Marshal.SizeOf(packet);
            var ptr = Marshal.AllocHGlobal(dataSize);
            Marshal.Copy(data, 0, ptr, dataSize);
            packet = (MyPacket)Marshal.PtrToStructure(ptr, packet.GetType());
            Marshal.FreeHGlobal(ptr);
            return packet;
        }

        private static void Main(string[] args)
        {
            const string queuePath = @".\private$\test_msmq";

            // Create the packet
            var packet = new MyPacket();

            // 8 bytes
            packet.ProtocolIdentifier = 1;
            packet.NumDi = 2;

            // 8 bytes
            // packet.Di = new DiObject[packet.NumDi];
            packet.Di[0] = 2;
            packet.Di[1] = 3;
            packet.Di[2] = 4;
            packet.Di[3] = 5;
            packet.Di[4] = 6;
            packet.Di[5] = 7;
            packet.Di[6] = 8;
            packet.Di[7] = 9;

            // Convert the struct in bytes
            int packetSize = Marshal.SizeOf<MyPacket>();
            var packetBytes = GetBytes(packet, packetSize);

            // Create the message

            var msg = new Message();
            msg.BodyStream = new MemoryStream(packetBytes);

            // Open or create the message queue
            if (!MessageQueue.Exists(queuePath))
                MessageQueue.Create(queuePath);

            // Open the queue
            var q = new MessageQueue(queuePath); // {Formatter = new BinaryMessageFormatter()};

            // Send the message to the queue
            q.Send(msg);

        }
    }
}

Code below provides efficient conversion to byte array and from byte array for MyPacket struct with variable internal array size. Implementation avoids casts and bounds checks by using unsafe pointer arithmetic.

using System;
using System.IO;
using System.Messaging;
using System.Runtime.InteropServices;

namespace StructToBytes
{
    // 4 bytes
    [Serializable]
    [StructLayout(LayoutKind.Explicit)]
    public unsafe struct DiObject
    {
        [FieldOffset(0)]
        public byte Command;

        [FieldOffset(1)]
        public byte ErrorClass;

        [FieldOffset(2)]
        public byte Reserved;

        [FieldOffset(3)]
        public byte Flags;
    }

    [Serializable]
    public unsafe struct MyPacket
    {
        public uint ProtocolIdentifier;
        public uint NumDi;
        public DiObject[] Di;

        public byte[] ToBytes()
        {
            byte[] buffer = new byte[NumDi];

            fixed(DiObject* pDi = Di)
            fixed(byte* pBuff = buffer)
            {
                var pBuffDi = (DiObject*)pBuff;
                var pDiPtr = pDi;
                for (int i = 0; i < NumDi; i++)
                    *pBuffDi++ = *pDiPtr++;
            }
            return buffer;
        }

        public static MyPacket Create(byte[] buffer)
        {
            // argument checking code here

            var packet = new MyPacket();
            packet.ProtocolIdentifier = buffer[0];
            packet.NumDi = buffer[1];
            packet.Di = new DiObject[packet.NumDi];

            fixed (byte* pBuf = buffer)
            fixed (DiObject* pDi = packet.Di)
            {
                byte* pBufPtr = pBuf;
                pBufPtr += 2;
                var pBufDi = (DiObject*)pBufPtr;
                var pDiPtr = pDi;

                for (int i = 0; i < packet.NumDi; i++)
                    *pDiPtr++ = *pBufDi++;
            }

            return packet;
        }
    }

    internal unsafe class Program
    {

        private static void Main(string[] args)
        {
            const string queuePath = @".\private$\test_msmq";

            // Create the packet
            var packet = new MyPacket();

            // 8 bytes
            packet.ProtocolIdentifier = 1;
            packet.NumDi = 5;

            // 8 bytes
            packet.Di = new DiObject[packet.NumDi];
            packet.Di[0].Command = 2;
            packet.Di[0].ErrorClass = 3;
            packet.Di[0].Flags = 4;
            packet.Di[0].Reserved = 5;
            packet.Di[1].Command = 6;
            packet.Di[1].ErrorClass = 7;
            packet.Di[1].Flags = 8;
            packet.Di[1].Reserved = 9;
            packet.Di[2].Command = 6;
            packet.Di[2].ErrorClass = 7;
            packet.Di[2].Flags = 8;
            packet.Di[2].Reserved = 9;
            packet.Di[3].Command = 6;
            packet.Di[3].ErrorClass = 7;
            packet.Di[3].Flags = 8;
            packet.Di[3].Reserved = 9;

            // Create the message

            var msg = new Message();
            msg.BodyStream = new MemoryStream(packet.ToBytes());

            // Open or create the message queue
            if (!MessageQueue.Exists(queuePath))
                MessageQueue.Create(queuePath);

            // Open the queue
            var q = new MessageQueue(queuePath);

            // Send the message to the queue
            q.Send(msg);

        }
    }
}
Jacek Blaszczynski
  • 3,183
  • 14
  • 25
  • Thanks for your clear explaination about the array. Your solution works perfectly and you quite understood my need to connect to a C++ program to the queue. – Less White Oct 24 '17 at 15:37