JSON .NET Not Deserializing Refs

Question

I have the following minimal example:

class Program
    {
        static void Main()
        {
            var plan = new Plan
            {
                Steps =
                {
                    new Step
                    {
                        Contexts =
                        {
                            new Context
                            {
                                Name = "1"
                            },
                            new Context
                            {
                                Name = "2"
                            }
                        }
                    }
                }
            };

            var settings = new JsonSerializerSettings
            { PreserveReferencesHandling = PreserveReferencesHandling.Objects, Formatting = Formatting.Indented };
            var json = JsonConvert.SerializeObject(plan, settings);
            var deserialized = JsonConvert.DeserializeObject<Plan>(json, settings);

        }
    }

    class Plan
    {
        public IEnumerable AllContexts => Steps.SelectMany(i => i.Contexts);
        [JsonProperty(Order = int.MaxValue)]
        public ICollection<Step> Steps { get; set; } = new List<Step>();
    }

    class Step
    {
        public ICollection<Context> Contexts { get; set; } = new List<Context>();
    }

    class Context
    {
        public string Name { get; set; }
    }

In this example deserialized has lost its references upon deserialization and deserialized.AllContexts is a collection of 2 null values.

I can get this working by changing [JsonProperty(Order = int.MaxValue)] to [JsonProperty(Order = int.MinValue)] so the Steps are serialized first - but in my scenario I want the actual JSON to have all its properties on the flat AllContexts array and for the Steps to only have $refs like this:

{
  "$id": "1",
  "AllContexts": [
    {
      "$id": "2",
      "Name": "1"
    },
    {
      "$id": "3",
      "Name": "2"
    }
  ],
  "Steps": [
    {
      "$id": "4",
      "Contexts": [
        {
          "$ref": "2"
        },
        {
          "$ref": "3"
        }
      ]
    }
  ]
}

This seems like a bug in JSON .NET - is there a way to work around it?