How Do I Deserialize Into Local Collections in .NET?

Question

I am trying to build a data pipeline in .NET. I have been given an xsd and have used the XML Schema Definition Tool to generate C# classes that represent the object model. In order to load this data into my data store I need to transform the data coming out of the XML into structure that matches my application schema and collect/dedupe elements. To do this I have a parser class that will read a file and load the contents into local collections which will then be loaded into my database. I see two options to do this -

1. Manually loop through the XML with an XmlReader and pull out the data I need, loading it into the local collections in stream. This is not desirable because it does not take advantage of the strongly typed/strict xsd that I was given and requires a lot of hard coding things like while (reader.Read()), check for specific XML nodes, and then `reader.GetAttribute("HardCodedString").
2. Use XmlSerializer to deserialize the whole file at once and then loop through the deserialized collections and insert into my local collections. This is not desirable because the files could be very large and this method forces me to loop through all of the data twice (once to deserialize and once to extract the data to my local collections).

Ideally I would like some way to register a delegate to be executed as each object is deserialized to insert into my local collections. Is there something in the framework that allows me to do this? Requirements are as follows:

1. Performant - Only loop through the data once.
2. Functional - Data is inserted into the local collections during deserialization.
3. Maintainable - Utilize strongly typed classes that were generated via the xsd.

I have created a minimal example to illustrate my point.

Example XML File:

<Hierarchy xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://www.example.com/example">
    <Children>
        <Child ChildId="1" ChildName="First">
            <Parents>
                <Parent ParentId="1" ParentName="First" RelationshipStart="1900-01-01T00:00:00"/>
                <Parent ParentId="2" ParentName="Second" RelationshipStart="2000-01-01T00:00:00"/>
            </Parents>
        </Child>
        <Child ChildId="2" ChildName="Second">
            <Parents>
                <Parent ParentId="2" ParentName="Second" RelationshipStart="1900-01-01T00:00:00"/>
                <Parent ParentId="3" ParentName="Third" RelationshipStart="2000-01-01T00:00:00"/>
            </Parents>
        </Child>
    </Children>
</Hierarchy>

Local collections I am trying to load:

public Dictionary<int, string> Parents { get; }
public Dictionary<int, string> Children { get; }
public List<Relationship> Relationships { get; }

Manual version (not maintainable and doesn't use xsd):

public void ParseFileManually(string fileName)
{
    using (var reader = XmlReader.Create(fileName))
    {
        while (reader.Read())
        {
            if (reader.NodeType == XmlNodeType.Element && reader.Name == "Hierarchy")
            {
                while (reader.Read())
                {
                    if (reader.NodeType == XmlNodeType.Element && reader.Name == "Child")
                    {
                        int childId = int.Parse(reader.GetAttribute("ChildId"));
                        string childName = reader.GetAttribute("ChildName");
                        Children[childId] = childName;

                        while (reader.Read())
                        {
                            if (reader.NodeType == XmlNodeType.Element && reader.Name == "Parent")
                            {
                                int parentId = int.Parse(reader.GetAttribute("ParentId"));
                                string parentName = reader.GetAttribute("ParentName");
                                DateTime relationshipStart = DateTime.Parse(reader.GetAttribute("RelationshipStart"));
                                
                                Parents[parentId] = parentName;
                                Relationships.Add(
                                    new Relationship{
                                        ParentId = parentId,
                                        ChildId = childId,
                                        Start = relationshipStart
                                    });
                            }
                            else if (reader.NodeType == XmlNodeType.EndElement && reader.Name == "Child")
                            {
                                break;
                            }
                        }
                    }
                }
            }
        }
    }
}

Deserialize version (loops through the data twice):

public void ParseFileWithDeserialize(string fileName)
{
    var serializer = new XmlSerializer(typeof(Hierarchy));
    using (var fileStream = new FileStream(fileName, FileMode.Open))
    {
        var fileData = (Hierarchy) serializer.Deserialize(fileStream);
        foreach (var child in fileData.Children)
        {
            Children[child.ChildId] = child.ChildName;

            foreach (var parent in child.Parents)
            {
                Parents[parent.ParentId] = parent.ParentName;
                Relationships.Add(
                    new Relationship
                    {
                        ParentId = parent.ParentId,
                        ChildId = child.ChildId,
                        Start = parent.RelationshipStart
                    });
            }
        }
    }
}

Answer 1

You should use some annotations to get the data from the correct field in the XML, if you use these definitions;

public class Hierarchy
{ 
    public Hierarchy()
    {
        Children = new List<Child>();
    }
    public List<Child> Children { get; set; }
}
public class Child
{
    public Child()
    {
        Parents = new List<Parent>();
    }
    [XmlAttribute("ChildId")]
    public int ChildId { get; set; }
    [XmlAttribute("ChildName")]
    public string ChildName { get; set; }
    public List<Parent> Parents { get; set; }
}
public class Parent
{
    [XmlAttribute("ParentId")]
    public int ParentId { get; set; }
    [XmlAttribute("ParentName")]
    public string ParentName { get; set; }
    [XmlAttribute("RelationshipStart")]
    public DateTime RelationshipStart { get; set; }
}

Then you should be able to simplify your code to;

    public static Hierarchy Deserialize(string fileName)
    {
        using (var fileStream = new StreamReader(fileName, Encoding.UTF8))
        {
            XmlSerializer ser = new XmlSerializer(typeof(Hierarchy));
            return (Hierarchy)ser.Deserialize(fileStream);
        }
    }

To test it out you can create a sample data set and serialize it to a file, then use the above code to read it back

    public static void Serialize(Hierarchy h, string fileName)
    {
        System.Xml.Serialization.XmlSerializer ser = new System.Xml.Serialization.XmlSerializer(typeof(Hierarchy));
        StreamWriter sw = new StreamWriter(fileName, false, Encoding.UTF8);
        ser.Serialize(sw, h);
    }

Test Code

    static void Test()
    {
        Hierarchy h = new Hierarchy();
        Parent p1 = new Parent() { ParentId = 1, ParentName = "First", RelationshipStart = DateTime.Now };
        Parent p2 = new Parent() { ParentId = 2, ParentName = "Second", RelationshipStart = DateTime.Now };
        Parent p3 = new Parent() { ParentId = 3, ParentName = "Third", RelationshipStart = DateTime.Now };
        Child c1 = new Child() { ChildId = 1, ChildName = "First" };
        c1.Parents.Add(p1);
        c1.Parents.Add(p2);
        Child c2 = new Child() { ChildId = 2, ChildName = "Second" };
        c2.Parents.Add(p2);
        c2.Parents.Add(p3);
        h.Children.Add(c1);
        h.Children.Add(c2);
        Serialize(h, AppContext.BaseDirectory + "Text.xml");
        Hierarchy hReadBack = Deserialize(AppContext.BaseDirectory + "Text.xml");
    }

Edit : To answer your question

Use these classes

public class Hierarchy
{ 
    public Hierarchy()
    {
        Children = new List<Child>();
    }
    public List<Child> Children { get; set; }

    private Dictionary<int, string> _parents;
    private Dictionary<int, string> _childrenList;
    private List<Relationship> _relationships;
    private void CalcuateLists()
    {
        _parents = new Dictionary<int, string>();
        _childrenList = new Dictionary<int, string>();
        _relationships = new List<Relationship>();
        foreach (Child c in this.Children)
        {
            if (!_childrenList.ContainsKey(c.ChildId))
            {
                _childrenList.Add(c.ChildId, c.ChildName);
            }
            foreach (Parent p in c.Parents)
            {
                if (!_parents.ContainsKey(p.ParentId))
                {
                    _parents.Add(p.ParentId, p.ParentName);
                }
                if (_relationships.FirstOrDefault(dat => dat.ParentId == p.ParentId && dat.ChildId == c.ChildId) == null)
                {
                    _relationships.Add(new Relationship() { ChildId = c.ChildId, ParentId = p.ParentId, Start = p.RelationshipStart });
                }
            }
        }
    }
    public Dictionary<int, string> Parents { 
        get
        {
            if (_parents == null)
                CalcuateLists();
            return _parents;
        }
    }
    public Dictionary<int, string> ChildrenList {
        get
        {
            if (_childrenList == null)
                CalcuateLists();
            return _childrenList;
        }
    }
    public List<Relationship> Relationships { 
        get
        {
            if (_relationships == null)
                CalcuateLists();
            return _relationships;
        }
    }
}
public class Child
{
    public Child()
    {
        Parents = new List<Parent>();
    }
    [XmlAttribute("ChildId")]
    public int ChildId { get; set; }
    [XmlAttribute("ChildName")]
    public string ChildName { get; set; }
    public List<Parent> Parents { get; set; }
}
public class Parent
{
    [XmlAttribute("ParentId")]
    public int ParentId { get; set; }
    [XmlAttribute("ParentName")]
    public string ParentName { get; set; }
    [XmlAttribute("RelationshipStart")]
    public DateTime RelationshipStart { get; set; }
}

Then your test code becomes

public static void Test()
{
    Hierarchy h = new Hierarchy();
    Parent p1 = new Parent() { ParentId = 1, ParentName = "First", RelationshipStart = DateTime.Now };
    Parent p2 = new Parent() { ParentId = 2, ParentName = "Second", RelationshipStart = DateTime.Now };
    Parent p3 = new Parent() { ParentId = 3, ParentName = "Third", RelationshipStart = DateTime.Now };
    Child c1 = new Child() { ChildId = 1, ChildName = "First" };
    c1.Parents.Add(p1);
    c1.Parents.Add(p2);
    Child c2 = new Child() { ChildId = 2, ChildName = "Second" };
    c2.Parents.Add(p2);
    c2.Parents.Add(p3);
    h.Children.Add(c1);
    h.Children.Add(c2);
    Serialize(h, AppContext.BaseDirectory + "Text.xml");
    Hierarchy hReadBack = Deserialize(AppContext.BaseDirectory + "Text.xml");

    Dictionary<int, string> Parents = hReadBack.Parents;
    Dictionary<int, string> Children = hReadBack.ChildrenList;
    List<Relationship> Relationships = hReadBack.Relationships;
}

EDIT

To get the results directly without looping

You will need this class

public class Relationship
{
    public int ParentId { get; set; }
    public int ChildId { get; set; }
    public DateTime Start { get; set; }
}  

And this selection

// Get a list of child ids and names
Dictionary<int, string> Children = (from c in hReadBack.Children select new { ChildId = c.ChildId, Name = c.ChildName}).ToDictionary(dat => dat.ChildId, dat => dat.Name);
// Get a parent ids and names
Dictionary<int, string> Parents = (from p in hReadBack.Children.SelectMany(i => i.Parents) select new { ParentId = p.ParentId, Name = p.ParentName }).Distinct().ToDictionary(dat => dat.ParentId, dat => dat.Name);
// Get the relationships
List<Relationship> Relationship = (from Child c in hReadBack.Children from Parent p in c.Parents select new Relationship() { ChildId = c.ChildId, ParentId = p.ParentId, Start = p.RelationshipStart }).ToList();