4

I want to calculate the mean absolute deviation and I'm currently using the following class from Stack Overflow (link here), posted by Alex:

public class MovingAverageCalculator
{
    private readonly int _period;
    private readonly double[] _window;
    private int _numAdded;
    private double _varianceSum;

    public MovingAverageCalculator(int period)
    {
        _period = period;
        _window = new double[period];
    }

    public double Average { get; private set; }

    public double StandardDeviation
    {
        get
        {
            var variance = Variance;
            if (variance >= double.Epsilon)
            {
                var sd = Math.Sqrt(variance);
                return double.IsNaN(sd) ? 0.0 : sd;
            }
            return 0.0;
        }
    }

    public double Variance
    {
        get
        {
            var n = N;
            return n > 1 ? _varianceSum / (n - 1) : 0.0;
        }
    }

    public bool HasFullPeriod
    {
        get { return _numAdded >= _period; }
    }

    public IEnumerable<double> Observations
    {
        get { return _window.Take(N); }
    }

    public int N
    {
        get { return Math.Min(_numAdded, _period); }
    }

    public void AddObservation(double observation)
    {
        // Window is treated as a circular buffer.
        var ndx = _numAdded % _period;
        var old = _window[ndx];     // get value to remove from window
        _window[ndx] = observation; // add new observation in its place.
        _numAdded++;

        // Update average and standard deviation using deltas
        var old_avg = Average;
        if (_numAdded <= _period)
        {
            var delta = observation - old_avg;
            Average += delta / _numAdded;
            _varianceSum += (delta * (observation - Average));
        }
        else // use delta vs removed observation.
        {
            var delta = observation - old;
            Average += delta / _period;
            _varianceSum += (delta * ((observation - Average) + (old - old_avg)));
        }
    }
}

The person who made that class, did it somehow weirdly and I don't understand the calculations, because he's using different formulas. For example, people normally calculate the variance just like this.

Can someone explain me how to calculate the mean absolution deviation in that class?

Formulas:

enter image description here

Edit:

This one is a bit more accurate but not enough. Any ideas?

Output (should be):

CCI = -29.189669
CCI = -57.578105
CCI = 1.537557
CCI = 46.973803
CCI = 68.662979
CCI = 78.647204
CCI = 52.798310
CCI = 84.266845
CCI = 104.694912
CCI = 99.048428
CCI = 58.068118
CCI = 57.575758
CCI = 68.387309
CCI = 127.625967
CCI = 128.826508
CCI = 124.751608
CCI = 112.929293
CCI = 165.170449
CCI = 141.586505
CCI = 114.463325
CCI = 155.766418

Output (what it is):

CCI = -26.630104
CCI = -53.295597
CCI = 1.476909
CCI = 44.829571
CCI = 67.857143
CCI = 80.059829
CCI = 55.447471
CCI = 90.681818
CCI = 116.030534
CCI = 106.314948
CCI = 61.242833
CCI = 61.664226
CCI = 74.962064
CCI = 150.864780
CCI = 163.034547
CCI = 162.636347
CCI = 153.194865
CCI = 197.583882
CCI = 159.622130
CCI = 122.744143
CCI = 163.325826

Output from another CCI (should be):

Typical Price: 0.010153 | SMA: 0.009989 | Mean Deviation: 0.000139
Typical Price: 0.010100 | SMA: 0.009988 | Mean Deviation: 0.000142
Typical Price: 0.010180 | SMA: 0.009991 | Mean Deviation: 0.000150
Typical Price: 0.010230 | SMA: 0.009990 | Mean Deviation: 0.000153
Typical Price: 0.010233 | SMA: 0.010000 | Mean Deviation: 0.000157
Typical Price: 0.010147 | SMA: 0.010008 | Mean Deviation: 0.000159
Typical Price: 0.010160 | SMA: 0.010027 | Mean Deviation: 0.000154
Typical Price: 0.010200 | SMA: 0.010044 | Mean Deviation: 0.000152
Typical Price: 0.010380 | SMA: 0.010077 | Mean Deviation: 0.000158
Typical Price: 0.010413 | SMA: 0.010107 | Mean Deviation: 0.000159
Typical Price: 0.010447 | SMA: 0.010138 | Mean Deviation: 0.000165
Typical Price: 0.010450 | SMA: 0.010171 | Mean Deviation: 0.000165
Typical Price: 0.010657 | SMA: 0.010199 | Mean Deviation: 0.000185
Typical Price: 0.010647 | SMA: 0.010224 | Mean Deviation: 0.000199
Typical Price: 0.010623 | SMA: 0.010252 | Mean Deviation: 0.000216
Typical Price: 0.010880 | SMA: 0.010308 | Mean Deviation: 0.000245
Typical Price: 0.010863 | SMA: 0.010354 | Mean Deviation: 0.000263
Typical Price: 0.010853 | SMA: 0.010397 | Mean Deviation: 0.000285
Typical Price: 0.010967 | SMA: 0.010442 | Mean Deviation: 0.000307
Typical Price: 0.011480 | SMA: 0.010517 | Mean Deviation: 0.000356
Typical Price: 0.011750 | SMA: 0.010600 | Mean Deviation: 0.000408
Typical Price: 0.011653 | SMA: 0.010674 | Mean Deviation: 0.000448

Output from CommodityChannelIndex.cs (what it is):

Typical Price: 0.010153 | SMA: 0.009989 | Mean Deviation: 0.000137
Typical Price: 0.010100 | SMA: 0.009988 | Mean Deviation: 0.000135
Typical Price: 0.010180 | SMA: 0.009991 | Mean Deviation: 0.000139
Typical Price: 0.010230 | SMA: 0.009990 | Mean Deviation: 0.000138
Typical Price: 0.010233 | SMA: 0.010000 | Mean Deviation: 0.000146
Typical Price: 0.010147 | SMA: 0.010008 | Mean Deviation: 0.000151
Typical Price: 0.010160 | SMA: 0.010027 | Mean Deviation: 0.000144
Typical Price: 0.010200 | SMA: 0.010044 | Mean Deviation: 0.000139
Typical Price: 0.010380 | SMA: 0.010077 | Mean Deviation: 0.000134
Typical Price: 0.010413 | SMA: 0.010107 | Mean Deviation: 0.000125
Typical Price: 0.010447 | SMA: 0.010138 | Mean Deviation: 0.000127
Typical Price: 0.010450 | SMA: 0.010171 | Mean Deviation: 0.000122
Typical Price: 0.010657 | SMA: 0.010199 | Mean Deviation: 0.000154
Typical Price: 0.010647 | SMA: 0.010224 | Mean Deviation: 0.000177
Typical Price: 0.010623 | SMA: 0.010252 | Mean Deviation: 0.000202
Typical Price: 0.010880 | SMA: 0.010308 | Mean Deviation: 0.000234
Typical Price: 0.010863 | SMA: 0.010354 | Mean Deviation: 0.000244
Typical Price: 0.010853 | SMA: 0.010397 | Mean Deviation: 0.000255
Typical Price: 0.010967 | SMA: 0.010442 | Mean Deviation: 0.000273
Typical Price: 0.011480 | SMA: 0.010517 | Mean Deviation: 0.000329
Typical Price: 0.011750 | SMA: 0.010600 | Mean Deviation: 0.000389
Typical Price: 0.011653 | SMA: 0.010674 | Mean Deviation: 0.000423

A working example code:

public decimal[] Calculate(IReadOnlyList<(decimal High, decimal Low, decimal Close)> candles, int period)
{
    var ccis = new decimal[candles.Count];

    SMA sma = new SMA(period);
    var smas = sma.Calculate(candles.Select(e => e.Close).ToArray());

    for (int i = 0; i < candles.Count; i++)
    {
        var typicalPrice = (candles[i].High + candles[i].Low + candles[i].Close) / 3m;

        decimal total = 0m;

        for (int j = i; j >= Math.Max(i - period + 1, 0); j--)
        {
            total += Math.Abs(smas[j] - candles[j].Close);

            Console.WriteLine("Sum = " + total.ToString("f6"));
        }

        decimal meanDeviation = total / period;
        decimal cci = meanDeviation != 0 ? (typicalPrice - smas[i]) / meanDeviation / 0.015m : 0;

        //Console.WriteLine($"Typical Price: {typicalPrice.ToString("f6")} | SMA: {smas[i].ToString("f6")} | Mean Deviation: {meanDeviation.ToString("f6")}");

        ccis[i] = cci;
    }

    return ccis;
}

The actual (broken) code that I want to fix:

public class MovingAverageCalculator
{
    private readonly int _period;
    private readonly double[] _window;
    private int _numAdded;
    private double _varianceSum;

    public MovingAverageCalculator(int period)
    {
        _period = period;
        _window = new double[period];
    }

    public double Average { get; private set; }

    public double StandardDeviation
    {
        get
        {
            var variance = Variance;
            if (variance >= double.Epsilon)
            {
                var sd = Math.Sqrt(variance);
                return double.IsNaN(sd) ? 0.0 : sd;
            }
            return 0.0;
        }
    }

    public double Variance
    {
        get
        {
            var n = N;
            return n > 1 ? _varianceSum / (n - 1) : 0.0;
        }
    }

    public double MeanAbsoluteDeviation
    {
        get
        {
            //return _window.Average(e => Math.Abs(e - Average));

            // https://stackoverflow.com/questions/5336457/how-to-calculate-a-standard-deviation-array
            var n = N;
            var sumOfDifferences = _window.Sum(e => Math.Abs(e - Average));
            return n > 1 ? sumOfDifferences / (n - 1) : 0.0;
        }
    }

    public bool HasFullPeriod
    {
        get { return _numAdded >= _period; }
    }

    public IEnumerable<double> Observations
    {
        get { return _window.Take(N); }
    }

    public int N
    {
        get { return Math.Min(_numAdded, _period); }
    }

    public void AddObservation(double observation)
    {
        // Window is treated as a circular buffer.
        var ndx = _numAdded % _period;
        var old = _window[ndx];     // get value to remove from window
        _window[ndx] = observation; // add new observation in its place.
        _numAdded++;

        // Update average and standard deviation using deltas
        var oldAvg = Average;
        if (_numAdded <= _period)
        {
            var delta = observation - oldAvg;
            Average += delta / _numAdded;
            _varianceSum += (delta * (observation - Average));
        }
        else // use delta vs removed observation.
        {
            var delta = observation - old;
            Average += delta / _period;
            _varianceSum += (delta * ((observation - Average) + (old - oldAvg)));
        }
    }

    public void Reset()
    {
        _numAdded = 0;
        _varianceSum = 0;
    }
}

public class CommodityChannelIndex : Indicator<(decimal High, decimal Low, decimal Close), decimal>
{
    private readonly int _period;
    private readonly MovingAverageCalculator _movingAvg;

    public CommodityChannelIndex(int period)
    {
        _period = period;

        _movingAvg = new MovingAverageCalculator(_period);
    }

    public override decimal ComputeNextValue((decimal High, decimal Low, decimal Close) input)
    {
        decimal typicalPrice = (input.High + input.Low + input.Close) / 3m;

        _movingAvg.AddObservation((double)input.Close);

        if (_movingAvg.HasFullPeriod)
        {
            var average = (decimal)_movingAvg.Average;
            var meanDeviation = (decimal)_movingAvg.MeanAbsoluteDeviation;

            return meanDeviation != 0m ? (typicalPrice - average) / meanDeviation / 0.015m : 0m;
        }

        return 0;
    }

    public override void Reset()
    {
        throw new System.NotImplementedException();
    }
}

// USAGE
CommodityChannelIndex rsi = new CommodityChannelIndex(20);
for (int i = 0; i < candles.Count - 1; i++)
{
    var result = rsi.ComputeNextValue((candles[i].High, candles[i].Low, candles[i].Close));
    Console.WriteLine($"CCI = {result.ToString("f6")}");
}

The sum for the mean absolute deviation is broken. The question is how to fix it?

Edit:

After Alireza's correction. Still inaccurate.

Correct (how it should be):
CCI = -11.554556
CCI = 21.045918
CCI = 38.828097
CCI = 22.566381
CCI = 59.149184
CCI = 77.075455
CCI = 38.104311
CCI = 13.746847
CCI = -41.996578
CCI = -89.997229
CCI = -77.630112
CCI = 18.273976
CCI = 9.525936
CCI = -11.306480
CCI = 74.880871
CCI = 186.070619
CCI = 19.839042
CCI = -159.106198

Incorrect (n - 1) - before Alireza's answer:
CCI = -29.587542
CCI = 45.010768
CCI = 69.666667
CCI = 34.518799
CCI = 75.449922
CCI = 89.486260
CCI = 43.181818
CCI = 15.962060
CCI = -47.174211
CCI = -99.664083
CCI = -80.542391
CCI = 17.952962
CCI = 8.888889
CCI = -10.138104
CCI = 66.560510
CCI = 169.550087
CCI = 18.679280
CCI = -154.360812

Incorrect (n) - after Alireza's answer
CCI = -31.144781
CCI = 47.379756
CCI = 73.333333
CCI = 36.335578
CCI = 79.420970
CCI = 94.196064
CCI = 45.454545
CCI = 16.802168
CCI = -49.657064
CCI = -104.909561
CCI = -84.781464
CCI = 18.897855
CCI = 9.356725
CCI = -10.671689
CCI = 70.063694
CCI = 178.473776
CCI = 19.662400
CCI = -162.485066

Project repo:

https://github.com/Warrolen/test-project

nop
  • 4,711
  • 6
  • 32
  • 93
  • 3
    See Wiki : https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance – jdweng Mar 17 '20 at 23:36
  • @jdweng, both formulas can be found here: https://statistics.laerd.com/statistical-guides/measures-of-spread-absolute-deviation-variance.php. The different isn't that much, but I don't know why it doesn't work. – nop Mar 18 '20 at 13:38
  • 1
    Another implementation: https://www.geeksforgeeks.org/program-mean-absolute-deviation/ – nop Mar 18 '20 at 13:52
  • Do you know the difference between "Standard Deviation" and Variance? One use the square and the other doesn't. – jdweng Mar 18 '20 at 13:57
  • 1
    @jdweng, yes. Here is standard deviation vs mean absolute deviation: https://i.imgur.com/liSFe4h.png. First is the mean absolute deviation. The second formula is standard deviation. – nop Mar 18 '20 at 14:01
  • 1
    `_varianceSum += (delta * (observation - Average));` is supposed to be `sum(xi - x)^2` while it is not `^2` in that MovingAverageCalculator class. That's the problem. – nop Mar 18 '20 at 14:07
  • 1
    Btw MovingAverageCalculator's standard deviation and variance are working well, even with that weird formula. Only the mean absolute deviation is not working. – nop Mar 18 '20 at 14:08
  • 1
    [SUM[ABS(X-u(old)] + N * [u(new) - u(old)] + ABS(X+1 - u(new)]^2/ (N + 1) – jdweng Mar 18 '20 at 14:16
  • @HereticMonkey, added the person who posted it and the link. – nop Mar 18 '20 at 14:24
  • 1
    @jdweng, I don't understand that. – nop Mar 18 '20 at 14:25
  • I assume you to not want to do the calculation from scratch but use previous results. So the average changes when you add new value. So how much does Sum(X - u) change without doing the sum? It is the difference of the old average and the new average times number of values. Then you have to add the new value minus the new average and divided by N + 1. – jdweng Mar 18 '20 at 14:34
  • 1
    @jdweng, would you like to type that with my variables, because I don't understand u(new)/u(old) in your formula above, actually all abbreviations? `_meanSum = Math.Abs(observation - oldAvg) + _period * (Average - oldAvg) + Math.Pow(Math.Abs(observation + 1 - Average), 2);` so far. And yes, if it's possible I want to use previous result. – nop Mar 18 '20 at 14:58
  • Why not `for (int i = 0; i < candles.Count; i++) { }` and you have the terminate when `i < candles.Count - 1` ? – John Alexiou Mar 26 '20 at 00:08
  • Also, note that `return n > 1 ? sumOfDifferences / (n - 1) : 0.0;` differs from the formula in your posting. The formula is `return sumOfDifferences/n;` simply. – John Alexiou Mar 26 '20 at 00:17
  • @ja72, the reason I wanted it way is because on each new candle, instead of recalculating everything, I could save the data till now and just calculate it for the new candle which will reduce the amount of time. – nop Mar 26 '20 at 00:41
  • @nop - I think deep down you are asking if you know that `MAD` for `n-1` values, and the _n_-th value `x`, what is the new `MAD` value. Similar to how you would calculate averages by `(AVE*(n-1)+x)/n`. All the rest of the post is just noise. You already have calculated what you want in the `CCI` class, you just want to do a different way for some reason. – John Alexiou Mar 26 '20 at 00:48

2 Answers2

1

Why don't you roll your own statistics class that processes data? Here is a start:

class Program
{
    static void Main(string[] args)
    {
        double[] observations = new double[]
        {
            61.27657038 ,
            6.738725617 ,
            4.888532706 ,
            68.5439831  ,
            7.979694724 ,
            46.29444503 ,
            55.91040488 ,
            2.120589448 ,
            18.75847801 ,
            1.340159128 ,
            1.188675161 ,
            0.444025252 ,
            0.126010202 ,
            55.90778938 ,
            55.76919429 ,
            4.976797265 ,
            56.27591183 ,
            34.25639959 ,
            1.045892651 ,
            15.92770207 ,
        };

        // take the last 10 data for statistics
        var data = observations.Reverse().Take(10).ToArray();
        var stats = new Statistics(data);

        Debug.WriteLine($"Count     ={stats.Count}");
        Debug.WriteLine($"Min       ={stats.MinValue}");
        Debug.WriteLine($"Max       ={stats.MaxValue}");
        Debug.WriteLine($"Median    ={stats.MedianValue}");
        Debug.WriteLine($"Mean      ={stats.MeanValue}");
        Debug.WriteLine($"Var       ={stats.Variance}");
        Debug.WriteLine($"Mad       ={stats.MeanAbsoluteDeviation}");
        Debug.WriteLine($"SDev      ={stats.StandardDeviation}");
    }
}

public class Statistics
{
    public Statistics(params double[] data)
    {
        this.Count = data.Length;
        // first pass to get mean;
        double sum = 0, min = 0, max = 0;
        for (int i = 0; i < data.Length; i++)
        {
            double x = data[i];
            if (i==0) // use first data for min/max
            {
                min = x;
                max = x;
            }
            else
            {
                min = Math.Min(min, x);
                max = Math.Max(max, x);
            }
            sum += x;
        }
        this.MinValue = min;
        this.MaxValue = max;
        this.MeanValue = sum/Count;

        // second pass for variance
        double var = 0, dev = 0, dx;
        for (int i = 0; i < data.Length; i++)
        {
            dx = data[i]-MeanValue;
            dev += Math.Abs(dx);
            var += dx*dx;
        }
        this.MeanAbsoluteDeviation = dev/Count;
        this.Variance = var/Count;
    }
    public int Count { get; }
    public double MaxValue { get; }
    public double MinValue { get; }
    public double MedianValue => (MinValue+MaxValue)/2;
    public double MeanValue { get; }
    public double MeanAbsoluteDeviation { get; }
    public double Variance { get; }
    public double StandardDeviation => Sqrt(Variance);
}

with example results:

Count     =10
Min       =0.126010202
Max       =56.27591183
Median    =28.200961016
Mean      =22.5918397691
Var       =575.363095007382
Mad       =22.36838720272
SDev      =23.9867274759893
John Alexiou
  • 28,472
  • 11
  • 77
  • 133
  • I have a working code to calculate it for all elements at once (@`params double[] data`). The point is to make it calculate element by element, look at `class CommodityChannelIndex` and its ComputeNextValue method. It should also be calculating it for a period, not just for all elements. For example, its constructor should be accepting `public Statistics(int period)` and there should be a method `AddValue(double value)`. You can also check MovingAverageCalculator class, it is made like that. That's what I'm struggling with. – nop Mar 20 '20 at 01:09
  • Just use a slice of the data to calculate the statistics if you want moving average etc, BTW where in the question does it state the form the data is presented and that it needs to work continuously with a stream of data? – John Alexiou Mar 25 '20 at 14:22
  • because I was asking for that particular class. The class is made like that. The stream is not important, hard-coded data is fine. In my case, I'm pulling data on each hour, because it's static for the next hour. I just want calculate it for each close value, not for all at once. You can check https://github.com/Warrolen/test-project. The input will be same for the next hour, so it will not change meanwhile. – nop Mar 25 '20 at 14:31
  • @nop - I have edited the question to show how to slice the data to take the last 10 values for example. Notice that the order of the values does not affect the results. – John Alexiou Mar 25 '20 at 17:23
1

I think this correction is what you want. You should replace the following incorrect code:

public double MeanAbsoluteDeviation
{
    get
    {
        //return _window.Average(e => Math.Abs(e - Average));

        // https://stackoverflow.com/questions/5336457/how-to-calculate-a-standard-deviation-array
        var n = N;
        var sumOfDifferences = _window.Sum(e => Math.Abs(e - Average));
        return n > 1 ? sumOfDifferences / (n - 1) : 0.0;
    }
}

with this correct one:

 public double MeanAbsoluteDeviation
{
    get
    {
        //return _window.Average(e => Math.Abs(e - Average));

        // https://stackoverflow.com/questions/5336457/how-to-calculate-a-standard-deviation-array
        var n = N;
        var sumOfDifferences = _window.Sum(e => Math.Abs(e - Average));
        return n > 1 ? sumOfDifferences / n : 0.0;
    }
}

Edit

Also note that your calculation of MeanAbsoluteDeviation is wrong! You are considering the current average for all values, while you should compare each value with it's average.

Here is the complete solution:

public class Candle
    {
        public double average { get; set; } = 0;
        public double close { get; set; } = 0;
    }
    public class MovingAverageCalculator
    {
        private readonly int _period;
        private readonly Candle[] _window;
        private int _numAdded;
        private double _varianceSum;

        public MovingAverageCalculator(int period)
        {
            _period = period;
            _window = new Candle[period];
            for (int i = 0; i < period; i++)
                _window[i] = new Candle();

        }

        public double Average { get; private set; }

        public double StandardDeviation
        {
            get
            {
                var variance = Variance;
                if (variance >= double.Epsilon)
                {
                    var sd = Math.Sqrt(variance);
                    return double.IsNaN(sd) ? 0.0 : sd;
                }
                return 0.0;
            }
        }

        public double Variance
        {
            get
            {
                var n = N;
                return n > 1 ? _varianceSum / (n - 1) : 0.0;
            }
        }

        public double MeanAbsoluteDeviation
        {
            get
            {
                //return _window.Average(e => Math.Abs(e - Average));

                // https://stackoverflow.com/questions/5336457/how-to-calculate-a-standard-deviation-array
                var n = N;
                var sumOfDifferences = _window.Sum(e => Math.Abs(e.close - e.average));

                return n > 1 ? sumOfDifferences / n : 0.0;
            }
        }

        public bool HasFullPeriod
        {
            get { return _numAdded > _period; }
        }

        public IEnumerable<double> Observations
        {
            get { return _window.Take(N).Select(a=>a.close); }
        }

        public int N
        {
            get { return Math.Min(_numAdded, _period); }
        }

        public void AddObservation(double observation)
        {
            // Window is treated as a circular buffer.
            var ndx = _numAdded % _period;
            var old = _window[ndx].close;     // get value to remove from window
            _window[ndx] = new Candle() { close=observation,average=0 }; // add new observation in its place.
            _numAdded++;

            // Update average and standard deviation using deltas
            var oldAvg = Average;
            if (_numAdded <= _period)
            {
                var delta = observation - oldAvg;
                Average += delta / _numAdded;
                _window[ndx].average = Average;
                _varianceSum += (delta * (observation - Average));
            }
            else // use delta vs removed observation.
            {
                var delta = observation - old;
                Average += delta / _period;
                _window[ndx].average = Average;
                _varianceSum += (delta * ((observation - Average) + (old - oldAvg)));
            }
        }

        public void Reset()
        {
            _numAdded = 0;
            _varianceSum = 0;
        }
    }
Alireza Mahmoudi
  • 964
  • 8
  • 35
  • Bessel's correction didn't make it that accurate. You can check my edit. – nop Mar 25 '20 at 12:17
  • I'm sure that inputs are different. This is pure math and nothing is different from working examaple to your code. In your working example we have a parameter called "period". It affect the average calculations. May be your are considering all values to calculate the average, but it needs to consider period (mean only a subset of values should be consider to compute average). – Alireza Mahmoudi Mar 25 '20 at 12:32
  • uploaded the project here: https://github.com/Warrolen/test-project. Test it, so you can see that the inputs are same. – nop Mar 25 '20 at 13:36
  • @nop your calculation was wrong for MeanAbsoluteDeviation. The last value of Average is consider, while each value should be compared with it's corresponding average. check my edit. – Alireza Mahmoudi Mar 25 '20 at 14:34