Why is this Linq Aggregate on IEnumerable & IEnumerable much slower than IEnumerable

Question

I have a program that calls a method with the linq aggregate a lot. The purpose is to find the closest value in the list. It currently does it on a List but I need to move the data the double holds into a struct or class. But when I do the program takes almost twice as long to run.

I wrote a sample program below to help illustrate what is going on.

I haven't been programming long so I am sure there is something I am not doing right. Can someone help me understand why its so much slower and is there a way to speed it up? Maybe I shouldn't use linq for this?

    static class Program
    {
        private static readonly List<double> _list = new List<double>();
        private static readonly List<ClassFoo> _classFooList = new List<ClassFoo>();
        private static readonly List<StructFoo> _structFooList = new List<StructFoo>();

        static void Main()
        {
            double _searchVal = 1000.5;
            double _searchCount = 300000;
            
            ClassFoo searchClassFoo = new ClassFoo(_searchVal);
            StructFoo searchStructFoo = new StructFoo(_searchVal);

            for (int i = 0; i < 2000; i++)
            {
                _list.Add(i);
                _classFooList.Add(new ClassFoo(i));
                _structFooList.Add(new StructFoo(i));
            }

            Stopwatch watch;
            Console.WriteLine("double...");
            watch = Stopwatch.StartNew();
            for (int i = 0; i < _searchCount; i++)
            {
                double closest = _list.Aggregate((x, y) => Math.Abs(x - _searchVal) < Math.Abs(y - _searchVal) ? x : y);
            }
            watch.Stop();
            Console.WriteLine(watch.ElapsedMilliseconds);

            Console.WriteLine();
            Console.WriteLine("Struct...");
            watch = Stopwatch.StartNew();
            for (int i = 0; i < _searchCount; i++)
            {
                StructFoo closest = _structFooList.Aggregate((x, y) => Math.Abs(x.Value - searchStructFoo.Value) < Math.Abs(y.Value - searchStructFoo.Value) ? x : y);
            }
            watch.Stop();
            Console.WriteLine(watch.ElapsedMilliseconds);

            Console.WriteLine();
            Console.WriteLine("Class...");
            watch = Stopwatch.StartNew();
            for (int i = 0; i < _searchCount; i++)
            {
                ClassFoo closest = _classFooList.Aggregate((x, y) => Math.Abs(x.Value - searchClassFoo.Value) < Math.Abs(y.Value - searchClassFoo.Value) ? x : y);
            }
            watch.Stop();
            Console.WriteLine(watch.ElapsedMilliseconds);
        }
    }

    readonly struct StructFoo
    {
        public StructFoo(double value) => Value = value;

        public double Value { get; }

        //There will be more properties but not needed for this example.
    }

    class ClassFoo
    {
        public ClassFoo(double value) => Value = value;

        public double Value { get; }

        //There will be more properties but not needed for this example.
    }

Here are my results:

double...
6340

Struct...
11249

Class...
11806
Press any key to continue . . .