Remove steps in time series

Question

I am analyzing lizards increase of temperature. Problem is that the animals sometime move moving the thermometer we are using for the measurements. This results in spikes (few measurements extremely high or low) and steps (sudden offsets of the curve). I wanted to clean my curves from this noise, removing the spikes and aligning the two parts of the curve separated by each step. I guess the spikes can be easily smoothed but what about the steps? Examples of time series with steps marked in yellow and spikes in red

This is a sample from my dataframe

structure(list(ID = structure(c(183L, 183L, 183L, 183L, 183L, 
183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 
183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 
183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 
183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 
183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 
183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 
183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 
183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 
183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 
183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 
183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 
183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 
183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 
183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 
183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 183L, 
183L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 
194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 
194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 
194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 
194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 
194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 
194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 
194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 
194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 
194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 
194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 
194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 
194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 
194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 
194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 
194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 
194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 
194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 
194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 194L, 
194L, 194L, 194L), .Label = c("101", "102", "104", "106", "107", 
"109", "110", "111", "112", "113", "114", "115", "116", "118", 
"119", "121", "122", "123", "124", "125", "126", "128", "129", 
"130", "132", "133", "134", "157", "158", "161", "163", "164", 
"165", "166", "167", "168", "169", "170", "171", "172", "173", 
"174", "175", "177", "178", "179", "180", "181", "182", "183", 
"186", "187", "188", "189", "191", "192", "193", "194", "195", 
"196", "198", "199", "201", "202", "203", "204", "205", "207", 
"208", "209", "210", "211", "213", "214", "215", "218", "219", 
"220", "221", "222", "223", "224", "225", "226", "227", "228", 
"229", "230", "233", "235", "307", "308", "310", "311", "312", 
"313", "314", "315", "316", "317", "318", "319", "320", "321", 
"322", "323", "324", "325", "326", "327", "328", "329", "330", 
"331", "332", "333", "334", "335", "336", "339", "340", "341", 
"343", "344", "346", "347", "348", "349", "350", "351", "352", 
"353", "354", "355", "356", "357", "358", "58", "59", "60", "61", 
"62", "63", "64", "65", "66", "67", "68", "69", "70", "71", "72", 
"73", "74", "75", "76", "78", "79", "80", "81", "82", "83", "84", 
"85", "86", "87", "88", "89", "90", "91", "92", "93", "94", "95", 
"96", "97", "98", "99", "F04000017", "F04001300", "F04060000", 
"F04070000", "F04080000", "F05000017", "F05000020", "F05060000", 
"F05070000", "FSUM", "M03100000", "M04000016", "M04090000", "M04100000", 
"M05000016", "M05000018", "M05080000", "M05090000", "M05100000", 
"MSUM"), class = "factor"), TIME = c(1600, 0, 180, 280, 1270, 
190, 1570, 100, 630, 110, 1530, 790, 90, 650, 910, 1520, 460, 
270, 710, 1240, 1610, 550, 470, 1320, 360, 1220, 860, 540, 290, 
1330, 1180, 170, 1700, 990, 1060, 420, 10, 330, 1110, 1160, 890, 
260, 620, 1140, 1420, 380, 300, 1650, 1430, 200, 490, 1590, 150, 
430, 720, 950, 730, 1660, 1260, 740, 810, 780, 1370, 960, 1130, 
1210, 1670, 1010, 760, 1280, 1450, 770, 1070, 1000, 1030, 690, 
80, 1040, 1050, 160, 1350, 1230, 1630, 640, 1390, 1460, 1100, 
850, 1120, 390, 1170, 980, 320, 590, 520, 1090, 560, 870, 1020, 
750, 140, 700, 70, 1500, 340, 1290, 530, 840, 230, 370, 1440, 
1200, 1480, 30, 1400, 210, 1300, 480, 450, 1580, 1560, 410, 1360, 
900, 670, 1540, 50, 500, 400, 1380, 930, 580, 1680, 600, 1340, 
240, 1190, 570, 1640, 940, 350, 1550, 830, 310, 880, 1080, 920, 
1490, 610, 220, 1470, 800, 820, 40, 1250, 130, 60, 680, 20, 970, 
1620, 440, 1690, 120, 510, 1410, 660, 250, 1310, 1150, 1510, 
1230, 1540, 320, 1380, 680, 410, 480, 580, 130, 670, 1200, 1900, 
1410, 1470, 1950, 400, 870, 970, 1740, 500, 1300, 1220, 240, 
660, 1100, 20, 490, 590, 900, 820, 1910, 920, 690, 1290, 1710, 
750, 1860, 1840, 880, 1250, 2070, 640, 1580, 1310, 800, 1000, 
2090, 1760, 330, 760, 40, 600, 1800, 200, 1730, 2080, 470, 30, 
1490, 2030, 860, 1620, 250, 1880, 790, 90, 360, 1520, 460, 180, 
1770, 1450, 1940, 1370, 1180, 290, 2060, 1240, 1500, 1480, 510, 
980, 1610, 1630, 950, 1980, 990, 140, 60, 110, 1680, 10, 550, 
1700, 1750, 1270, 1690, 230, 2010, 1070, 910, 300, 1850, 1140, 
1350, 930, 170, 1650, 1170, 1820, 1040, 1420, 120, 1150, 50, 
960, 1560, 1790, 630, 280, 740, 1890, 2020, 370, 2040, 2050, 
310, 380, 1920, 340, 1030, 430, 1330, 1670, 1960, 830, 1590, 
420, 1400, 560, 1360, 1460, 1440, 1120, 1870, 1720, 1130, 730, 
390, 850, 520, 1090, 1550, 100, 1930, 1010, 440, 210, 1390, 720, 
1190, 450, 770, 1320, 220, 1640, 570, 2100, 1210, 650, 2000, 
840, 1340, 1830, 530, 1110, 1260, 890, 700, 80, 810, 1060, 260, 
1510, 1600, 70, 1430, 1280, 1530, 1020, 540, 940, 1050, 710, 
1660, 1570, 270, 610, 620, 780, 1780, 350, 1970, 0, 1160, 1990, 
190, 150, 1080, 160, 1810), LIZ = c(33.88628, 16.01848, 19.07537, 
20.7521, 31.20864, 19.28933, 33.81645, 17.74926, 25.17476, 17.96775, 
35.05947, 27.00651, 17.59508, 25.4069, 28.208, 34.99299, 22.88728, 
20.59275, 26.18355, 31.03587, 33.947, 24.10997, 23.02703, 33.7095, 
22.17608, 30.91595, 27.75973, 23.97134, 20.93754, 33.80886, 30.68742, 
18.91375, 33.92175, 29.09865, 29.63962, 22.70535, 16.21696, 21.63548, 
30.08059, 30.53127, 27.98175, 20.43409, 25.05877, 30.36036, 34.46184, 
22.44901, 21.10243, 33.94136, 34.52333, 19.46398, 23.28305, 33.80731, 
18.61317, 22.47734, 26.25653, 28.72091, 26.38854, 33.93099, 31.15667, 
26.51461, 27.30298, 26.90585, 34.10386, 28.83213, 30.1994, 30.83049, 
33.89071, 29.26265, 26.67329, 32.40561, 34.64661, 26.80018, 29.67405, 
29.16813, 29.42029, 25.87777, 17.42933, 29.50874, 29.60253, 18.75925, 
33.97794, 30.99272, 33.94607, 25.2893, 34.24567, 34.68945, 30.0688, 
27.65221, 30.13411, 22.59461, 30.63778, 28.96221, 21.41397, 24.58403, 
23.68083, 29.85481, 24.22983, 27.8342, 29.36583, 26.60143, 18.46704, 
26.03891, 17.26645, 34.9014, 21.84469, 33.39574, 23.82, 27.57049, 
19.93821, 22.30472, 34.60674, 30.77051, 34.79948, 16.55761, 34.33158, 
19.62092, 33.46234, 23.15645, 22.68871, 33.79304, 33.88377, 22.99619, 
34.04042, 28.1129, 25.6251, 34.21122, 16.89074, 23.43516, 22.78885, 
34.17764, 28.47507, 24.45472, 33.8742, 24.72735, 33.90453, 20.10577, 
30.73417, 24.35771, 33.93048, 28.62741, 21.99781, 33.97049, 27.48719, 
21.25535, 27.92438, 29.80396, 28.35731, 34.8615, 24.93956, 19.783, 
34.76926, 27.19699, 27.40897, 16.72252, 31.08047, 18.32881, 17.0416, 
25.72955, 16.38935, 28.8926, 33.97636, 22.53829, 33.86432, 18.22302, 
23.55972, 34.38583, 25.50873, 20.27311, 33.6045, 30.43855, 34.93784, 
35.288345, 36.257075, 25.391195, 36.09727, 30.813695, 27.044815, 
28.04567, 29.548165, 20.606055, 30.69551, 35.129335, 37.32639, 
36.37374, 36.308585, 37.41784, 26.86707, 32.548155, 33.36402, 
37.37834, 28.385865, 35.88449, 35.231535, 23.53661, 30.580155, 
34.58997, 16.9712, 28.21824, 29.69837, 32.803, 32.16565, 37.3345, 
32.96087, 30.917135, 35.835545, 37.40881, 31.57274, 37.28267, 
37.3545, 32.635805, 35.639945, 37.55586, 30.322245, 36.361875, 
35.93363, 32.083515, 33.9122, 37.57389, 37.38447, 25.60093, 31.70867, 
17.74926, 29.83628, 37.32308, 22.486975, 37.37677, 37.54575, 
27.8517, 17.37036, 36.13542, 37.58466, 32.4728, 36.84912, 23.79395, 
37.2999, 32.03574, 19.402135, 27.835535, 36.185425, 27.68629, 
21.91643, 37.37574, 36.612895, 37.38406, 36.11211, 35.03594, 
24.74032, 37.55308, 35.4775, 36.152685, 36.125095, 28.51215, 
33.614975, 36.71473, 36.859965, 33.190425, 37.57786, 33.82105, 
20.911715, 18.444165, 19.9942, 36.89602, 16.551, 29.161815, 37.4295, 
37.38779, 35.75298, 36.91141, 23.29514, 37.59916, 34.393665, 
32.88079, 24.885135, 37.30579, 34.812115, 36.05718, 33.03272, 
21.730805, 36.877595, 34.980885, 37.34123, 34.19463, 36.436875, 
20.27451, 34.880975, 18.10635, 33.268755, 36.24434, 37.33012, 
30.18474, 24.530635, 31.445215, 37.32058, 37.6003, 36.417575, 
37.57818, 37.56227, 25.15269, 26.548085, 37.34238, 25.799625, 
34.12894, 27.14156, 36.0024, 36.89349, 37.48383, 32.23698, 36.386895, 
27.013425, 36.295295, 29.29934, 36.091595, 36.642735, 36.5601, 
34.706135, 37.27491, 37.38146, 34.758915, 31.35675, 26.709435, 
32.374665, 28.69313, 34.52595, 36.22561, 19.717425, 37.35049, 
33.991875, 27.288855, 22.76699, 36.179625, 31.25419, 35.08752, 
27.507965, 31.821405, 35.97357, 23.037415, 36.86924, 29.408195, 
37.60048, 35.18184, 30.45653, 37.58567, 32.27861, 36.04241, 37.34733, 
28.86293, 34.64831, 35.684655, 32.71743, 31.01652, 19.07221, 
32.1161, 34.32663, 24.059185, 36.16283, 36.413085, 18.77961, 
36.49902, 35.7932, 36.214135, 34.05778, 29.01832, 33.111965, 
34.2606, 31.14351, 36.88914, 36.326515, 24.30303, 29.97399, 30.103995, 
31.92968, 37.36615, 26.01068, 37.52906, 16.12443, 34.929045, 
37.59149, 22.175545, 21.19651, 34.4613, 21.473975, 37.33004), 
    COP = c(22.39478, 22.68049, 22.27224, 22.19685, 22.16573, 
    22.26254, 22.34891, 22.41931, 22.15782, 22.39973, 22.31492, 
    22.13082, 22.44822, 22.16036, 22.13963, 22.3111, 22.15529, 
    22.20868, 22.14849, 22.16546, 22.40751, 22.15873, 22.15807, 
    22.15667, 22.17006, 22.16576, 22.13619, 22.15633, 22.18792, 
    22.15968, 22.15784, 22.28282, 22.39207, 22.15544, 22.15307, 
    22.15378, 22.65061, 22.18484, 22.14693, 22.1517, 22.14945, 
    22.22223, 22.15782, 22.15069, 22.18498, 22.17095, 22.18882, 
    22.44182, 22.18867, 22.25752, 22.16363, 22.37156, 22.33576, 
    22.14907, 22.15658, 22.15133, 22.1547, 22.43926, 22.16738, 
    22.15422, 22.13264, 22.14114, 22.17317, 22.15817, 22.14417, 
    22.16643, 22.43323, 22.14401, 22.14921, 22.15957, 22.22265, 
    22.14089, 22.14463, 22.14593, 22.14691, 22.14661, 22.47322, 
    22.15314, 22.15694, 22.3083, 22.16482, 22.16823, 22.42285, 
    22.16284, 22.17225, 22.2391, 22.1465, 22.12234, 22.14539, 
    22.15492, 22.14823, 22.15392, 22.18663, 22.14773, 22.16047, 
    22.14056, 22.15791, 22.14152, 22.14251, 22.1509, 22.35001, 
    22.14744, 22.49713, 22.28185, 22.18089, 22.15846, 22.15929, 
    22.12327, 22.23335, 22.16951, 22.20456, 22.16193, 22.25876, 
    22.60373, 22.17513, 22.25065, 22.15608, 22.16255, 22.14552, 
    22.36192, 22.3395, 22.15159, 22.16513, 22.14421, 22.15804, 
    22.31474, 22.5485, 22.16658, 22.15816, 22.16993, 22.14259, 
    22.14321, 22.4235, 22.16097, 22.16565, 22.22893, 22.16082, 
    22.14911, 22.43463, 22.14959, 22.16988, 22.32926, 22.12439, 
    22.18906, 22.14787, 22.143, 22.14448, 22.26607, 22.16002, 
    22.24653, 22.25201, 22.13589, 22.13117, 22.578, 22.16947, 
    22.36273, 22.51722, 22.15239, 22.62479, 22.15932, 22.41646, 
    22.14778, 22.41367, 22.38414, 22.16356, 22.18521, 22.16367, 
    22.22729, 22.16007, 22.15065, 22.29892, 21.72296, 21.62249, 
    22.017515, 21.693785, 21.70153, 21.87866, 21.81236, 21.732425, 
    22.36322, 21.70375, 21.72846, 21.91553, 21.68792, 21.660505, 
    22.00908, 21.89052, 21.763825, 21.760305, 21.83179, 21.79684, 
    21.71059, 21.726415, 22.149185, 21.70785, 21.75106, 22.571055, 
    21.804205, 21.7291, 21.761135, 21.747215, 21.92869, 21.76272, 
    21.705115, 21.712995, 21.83027, 21.708885, 21.83844, 21.84054, 
    21.768415, 21.71803, 22.11175, 21.70465, 21.60251, 21.70955, 
    21.742635, 21.753505, 22.13507, 21.82251, 21.999255, 21.717505, 
    22.53302, 21.72247, 21.82229, 22.22044, 21.829, 22.12267, 
    21.82473, 22.55087, 21.6558, 22.09163, 21.76516, 21.593285, 
    22.13412, 21.86871, 21.73596, 22.43231, 21.943965, 21.636055, 
    21.836155, 22.26584, 21.82428, 21.672995, 21.98196, 21.694165, 
    21.73181, 22.066775, 22.10273, 21.71976, 21.64993, 21.65895, 
    21.788935, 21.759365, 21.59079, 21.59272, 21.762905, 22.04811, 
    21.755645, 22.34494, 22.4932, 22.397445, 21.59399, 22.582895, 
    21.750065, 21.84207, 21.82297, 21.712635, 21.593685, 22.164485, 
    22.07698, 21.7517, 21.760385, 22.04753, 21.8382, 21.741645, 
    21.69029, 21.759415, 22.28582, 21.590015, 21.73355, 21.83193, 
    21.75102, 21.68036, 22.378585, 21.73784, 22.51333, 21.75792, 
    21.609515, 21.82392, 21.70592, 22.084645, 21.708, 21.89372, 
    22.08329, 21.93241, 22.10217, 22.10085, 22.030495, 21.91551, 
    21.94469, 21.979055, 21.75095, 21.858405, 21.698245, 21.59609, 
    22.02914, 21.755635, 21.593795, 21.86841, 21.689295, 21.739825, 
    21.691495, 21.66596, 21.674615, 21.74691, 21.84458, 21.82984, 
    21.741525, 21.710355, 21.90157, 21.762305, 21.77891, 21.750345, 
    21.61608, 22.41471, 21.95989, 21.7551, 21.84604, 22.198325, 
    21.691115, 21.709805, 21.732425, 21.84097, 21.723055, 21.70937, 
    22.179165, 21.58997, 21.732705, 22.13691, 21.725925, 21.70574, 
    22.06154, 21.755495, 21.692555, 21.83686, 21.77328, 21.74777, 
    21.716135, 21.761175, 21.709325, 22.451175, 21.749225, 21.750545, 
    22.117925, 21.636325, 21.59097, 22.47463, 21.67836, 21.715015, 
    21.63479, 21.75122, 21.762415, 21.764115, 21.751305, 21.70731, 
    21.591635, 21.606505, 22.09898, 21.71826, 21.71403, 21.727555, 
    21.81842, 21.963065, 22.03673, 22.597065, 21.73344, 22.05635, 
    22.243885, 22.325185, 21.75266, 22.3091, 21.82619), BLK = c(28.64989, 
    15.80412, 20.22131, 22.01231, 28.0591, 20.41488, 28.57393, 
    18.42224, 25.81396, 18.66597, 28.51571, 26.66224, 18.17439, 
    25.95201, 27.20529, 28.51703, 24.38048, 21.85311, 26.30937, 
    28.02257, 28.6773, 25.22704, 24.48749, 28.1236, 23.18583, 
    27.98485, 27.00689, 25.14024, 22.16927, 28.1335, 27.89662, 
    20.00687, 28.67155, 27.44175, 27.62627, 23.92489, 16.09932, 
    22.78544, 27.75227, 27.83675, 27.13084, 21.69284, 25.74085, 
    27.80579, 28.28594, 23.44893, 22.33534, 28.7166, 28.29154, 
    20.61311, 24.68904, 28.60215, 19.59335, 24.03528, 26.36296, 
    27.34839, 26.40381, 28.7205, 28.04637, 26.45016, 26.76595, 
    26.62852, 28.18898, 27.38003, 27.79384, 27.96629, 28.71652, 
    27.47686, 26.54292, 28.06953, 28.37088, 26.5854, 27.64778, 
    27.45691, 27.54817, 26.17697, 17.91728, 27.57453, 27.59667, 
    19.79719, 28.15391, 28.01052, 28.70145, 25.8918, 28.19863, 
    28.40318, 27.73245, 26.95575, 27.77656, 23.56018, 27.86381, 
    27.42115, 22.64232, 25.52273, 24.96468, 27.69186, 25.29434, 
    27.05567, 27.51177, 26.4991, 19.37665, 26.24547, 17.66023, 
    28.45211, 22.92771, 28.08868, 25.05819, 26.9147, 21.16986, 
    23.31623, 28.31888, 27.94109, 28.41742, 16.63277, 28.21869, 
    20.81001, 28.10906, 24.58388, 24.26295, 28.59208, 28.56833, 
    23.80437, 28.16685, 27.16407, 26.06582, 28.53314, 17.15336, 
    24.78023, 23.69012, 28.19161, 27.27431, 25.4426, 28.70709, 
    25.60821, 28.15239, 21.35028, 27.92147, 25.36057, 28.71294, 
    27.31714, 23.05455, 28.55633, 26.8609, 22.49237, 27.09901, 
    27.67287, 27.23569, 28.43207, 25.6761, 20.99499, 28.41302, 
    26.72662, 26.81219, 16.89905, 28.04126, 19.14027, 17.39888, 
    26.11992, 16.37336, 27.39998, 28.69902, 24.15099, 28.69344, 
    18.90929, 24.86793, 28.25387, 26.01419, 21.52158, 28.12068, 
    27.81882, 28.48342, 27.96163, 28.197645, 22.70333, 28.117945, 
    26.073955, 23.85303, 24.593665, 25.413525, 19.11083, 26.00622, 
    27.90615, 27.98023, 28.14475, 28.161525, 28.08333, 23.7368, 
    27.01432, 27.341995, 27.83233, 24.77127, 28.08601, 27.96112, 
    21.39113, 25.949465, 27.68429, 15.858655, 24.68182, 25.48676, 
    27.12372, 26.823595, 27.98649, 27.17823, 26.13994, 28.074105, 
    27.80747, 26.44518, 27.88772, 27.88865, 27.054655, 27.983055, 
    28.14709, 25.80761, 28.22729, 28.087765, 26.741425, 27.415515, 
    28.16615, 27.82628, 22.852945, 26.515965, 16.55507, 25.558165, 
    27.86497, 20.59904, 27.82338, 28.15537, 24.503635, 16.208215, 
    28.18434, 28.12301, 26.98152, 28.238175, 21.57631, 27.93972, 
    26.69019, 18.11697, 23.249475, 28.194915, 24.400325, 20.18436, 
    27.83363, 28.152575, 28.05914, 28.1046, 27.87944, 22.246695, 
    28.13895, 27.97113, 28.17759, 28.17134, 24.852555, 27.36403, 
    28.231645, 28.240475, 27.28467, 28.09842, 27.38207, 19.33762, 
    17.208895, 18.63795, 28.32022, 15.478795, 25.178505, 27.81531, 
    27.83335, 28.025865, 28.32901, 21.198015, 28.12642, 27.594495, 
    27.149095, 22.40141, 27.87721, 27.77537, 28.07577, 27.214415, 
    19.976595, 28.24768, 27.87019, 27.87417, 27.512335, 28.1487, 
    18.87582, 27.808485, 16.88221, 27.304755, 28.22269, 27.85485, 
    25.751105, 22.085525, 26.402235, 27.96385, 28.12362, 23.3511, 
    28.13134, 28.13854, 22.55042, 23.483435, 27.99603, 22.988955, 
    27.479825, 24.074565, 28.06897, 28.300515, 28.08899, 26.850145, 
    28.21175, 23.961985, 28.148655, 25.25804, 28.084975, 28.15017, 
    28.142015, 27.74106, 27.89856, 27.81585, 27.762185, 26.34704, 
    23.61241, 26.9402, 24.945815, 27.660965, 28.20952, 18.388425, 
    28.03487, 27.44823, 24.182, 20.80328, 28.138395, 26.294595, 
    27.8952, 24.294155, 26.569235, 28.076645, 21.003065, 28.221055, 
    25.336215, 28.17753, 27.934355, 25.886285, 28.10892, 26.890965, 
    28.06394, 27.89246, 25.030615, 27.71521, 27.994955, 27.091235, 
    26.189505, 17.829845, 26.78325, 27.566845, 21.749355, 28.181235, 
    28.216485, 17.523135, 28.145175, 28.05206, 28.198345, 27.46945, 
    25.099975, 27.266245, 27.54595, 26.236675, 28.27499, 28.230255, 
    21.91532, 25.631615, 25.69132, 26.62857, 27.83423, 23.119855, 
    28.08352, 15.10413, 27.84174, 28.10417, 20.39175, 19.55529, 
    27.62871, 19.76947, 27.87254)), row.names = c(39410L, 39411L, 
39412L, 39413L, 39414L, 39415L, 39416L, 39417L, 39418L, 39419L, 
39420L, 39421L, 39422L, 39423L, 39424L, 39425L, 39426L, 39427L, 
39428L, 39429L, 39430L, 39431L, 39432L, 39433L, 39434L, 39435L, 
39436L, 39437L, 39438L, 39439L, 39440L, 39441L, 39442L, 39443L, 
39444L, 39445L, 39446L, 39447L, 39448L, 39449L, 39450L, 39451L, 
39452L, 39453L, 39454L, 39455L, 39456L, 39457L, 39458L, 39459L, 
39460L, 39461L, 39462L, 39463L, 39464L, 39465L, 39466L, 39467L, 
39468L, 39469L, 39470L, 39471L, 39472L, 39473L, 39474L, 39475L, 
39476L, 39477L, 39478L, 39479L, 39480L, 39481L, 39482L, 39483L, 
39484L, 39485L, 39486L, 39487L, 39488L, 39489L, 39490L, 39491L, 
39492L, 39493L, 39494L, 39495L, 39496L, 39497L, 39498L, 39499L, 
39500L, 39501L, 39502L, 39503L, 39504L, 39505L, 39506L, 39507L, 
39508L, 39509L, 39510L, 39511L, 39512L, 39513L, 39514L, 39515L, 
39516L, 39517L, 39518L, 39519L, 39520L, 39521L, 39522L, 39523L, 
39524L, 39525L, 39526L, 39527L, 39528L, 39529L, 39530L, 39531L, 
39532L, 39533L, 39534L, 39535L, 39536L, 39537L, 39538L, 39539L, 
39540L, 39541L, 39542L, 39543L, 39544L, 39545L, 39546L, 39547L, 
39548L, 39549L, 39550L, 39551L, 39552L, 39553L, 39554L, 39555L, 
39556L, 39557L, 39558L, 39559L, 39560L, 39561L, 39562L, 39563L, 
39564L, 39565L, 39566L, 39567L, 39568L, 39569L, 39570L, 39571L, 
39572L, 39573L, 39574L, 39575L, 39576L, 39577L, 39578L, 39579L, 
39580L, 41926L, 41927L, 41928L, 41929L, 41930L, 41931L, 41932L, 
41933L, 41934L, 41935L, 41936L, 41937L, 41938L, 41939L, 41940L, 
41941L, 41942L, 41943L, 41944L, 41945L, 41946L, 41947L, 41948L, 
41949L, 41950L, 41951L, 41952L, 41953L, 41954L, 41955L, 41956L, 
41957L, 41958L, 41959L, 41960L, 41961L, 41962L, 41963L, 41964L, 
41965L, 41966L, 41967L, 41968L, 41969L, 41970L, 41971L, 41972L, 
41973L, 41974L, 41975L, 41976L, 41977L, 41978L, 41979L, 41980L, 
41981L, 41982L, 41983L, 41984L, 41985L, 41986L, 41987L, 41988L, 
41989L, 41990L, 41991L, 41992L, 41993L, 41994L, 41995L, 41996L, 
41997L, 41998L, 41999L, 42000L, 42001L, 42002L, 42003L, 42004L, 
42005L, 42006L, 42007L, 42008L, 42009L, 42010L, 42011L, 42012L, 
42013L, 42014L, 42015L, 42016L, 42017L, 42018L, 42019L, 42020L, 
42021L, 42022L, 42023L, 42024L, 42025L, 42026L, 42027L, 42028L, 
42029L, 42030L, 42031L, 42032L, 42033L, 42034L, 42035L, 42036L, 
42037L, 42038L, 42039L, 42040L, 42041L, 42042L, 42043L, 42044L, 
42045L, 42046L, 42047L, 42048L, 42049L, 42050L, 42051L, 42052L, 
42053L, 42054L, 42055L, 42056L, 42057L, 42058L, 42059L, 42060L, 
42061L, 42062L, 42063L, 42064L, 42065L, 42066L, 42067L, 42068L, 
42069L, 42070L, 42071L, 42072L, 42073L, 42074L, 42075L, 42076L, 
42077L, 42078L, 42079L, 42080L, 42081L, 42082L, 42083L, 42084L, 
42085L, 42086L, 42087L, 42088L, 42089L, 42090L, 42091L, 42092L, 
42093L, 42094L, 42095L, 42096L, 42097L, 42098L, 42099L, 42100L, 
42101L, 42102L, 42103L, 42104L, 42105L, 42106L, 42107L, 42108L, 
42109L, 42110L, 42111L, 42112L, 42113L, 42114L, 42115L, 42116L, 
42117L, 42118L, 42119L, 42120L, 42121L, 42122L, 42123L, 42124L, 
42125L, 42126L, 42127L, 42128L, 42129L, 42130L, 42131L, 42132L, 
42133L, 42134L, 42135L, 42136L), class = "data.frame")

My problem is actually very similar to the one that this person had in python Remove jumps like peaks and steps in timeseries but I haven't been able to find something similar for R

EDIT: I actually found something similar R-related in this question https://stats.stackexchange.com/questions/139660/detecting-changes-in-time-series-r-example?newreg=f119230044de4802a9f0f6f4e4637d8f The solution using tsoutliers looks applicable to my problem but so far it didn't work very well. I am not sure about what I am doing wrong.

tso(dat.ts, types = c("LS","TC"))

flattens completely my curve

tso(dat.ts, types = c("LS","TC"), discard.method = "bottom-up")

Moves my curve in the right direction but not enough to fix the problem.

Any approach using the bottom-up method to discard outliers gives this result, any approac using the en-masse approach flattens the curve.

Hi,you can try ```mean(data) + 2*SD(data)``` for outlier removal and for continues data take a lag of data and then take difference if difference zero then same value you can remove those. this is waht i understand.Hope this will help — Tushar Lad, Feb 11 '20 at 13:37

score 1 · Accepted Answer · answered Feb 14 '20 at 16:03

There are a couple methods attempted in the code below attempting to capture the changes: by percent change and by using a rolling median method.

First import a couple useful libraries. Also changing your data into a tibble called 'lizard_data'

library(tidyverse)
library(RcppRoll)
lizard_data <- tibble(your_data)

Then using ggplot2, we can visualize all the time series.

lizard_data %>% 
  pivot_longer(names_to = 'key', values_to = 'value', cols = c(LIZ, COP, BLK)) %>% 
  arrange(TIME) %>% 
  ggplot(aes(x = TIME, y = value, color = key))+
    geom_line()+
  facet_wrap(~ID, nrow = 2)

Then we remove the adjustments by finding the delta, and we can use two different methods to smooth the time series. First, with a percentile method (changing the top 5% and bottom 5% of values to the median) and second, with the rolling median (if the max change is ten percent higher or lower than the median, replacing the value with the median).

ld_w_change <- lizard_data %>% 
  pivot_longer(names_to = 'key', values_to = 'value', cols = c(LIZ, COP, BLK)) %>% 
  group_by(ID, key) %>% 
  arrange(TIME) %>% 
  mutate(lag = lag(value),
    raw_change = (value-lag),
    #using the percentile change method
    med_raw_change = median(raw_change, na.rm = T),
    q_05 = quantile(raw_change,.05, na.rm = T),
    q_95 = quantile(raw_change,.95, na.rm = T),
    adj_raw_change = if_else(raw_change > q_95 | raw_change < q_05, med_raw_change, raw_change),
    normalized_change = if_else(is.na(adj_raw_change), 0,adj_raw_change),
    initial_value = first(value),
    roll_raw_change = cumsum(normalized_change),
    new_value_pct = initial_value + roll_raw_change,
    # using the rolling median method
    rolling_median_change = roll_median(raw_change, n = 5, align = "right", na.rm = T, fill = NA),
    adj_median_change = case_when(raw_change > rolling_median_change*1.1 | raw_change < rolling_median_change*.9 ~ rolling_median_change, 
                                  is.na(rolling_median_change) & is.na(raw_change) ~ 0,
                                  T ~ raw_change),
    normalized_med_change = cumsum(adj_median_change),
    new_value_roll = initial_value + normalized_med_change
    )

I personally prefer the median adjustment, it seems to preserve the shape of the data better than the percentile method. To compare these methods visually, we can plot them side-by-side:

ld_w_change %>% 
  pivot_longer(names_to = 'method', values_to = 'adjusted_temp', cols = c(new_value_pct, new_value_roll)) %>% 
  ggplot(aes(x = TIME, y =adjusted_temp, color = key))+
  geom_line()+
  facet_wrap(ID ~ method, nrow = 2)

Thank you so much for your help. This works amazingly! I agree that the rolling median is maintaining the shape of the curves way better. Some of the rougher curves are still creating problems but it's probably just a matter of playing a bit with the code. Just one thing: for some reason tibble() didn't work for me but as_tibble() did the job. — Federico Massetti, Feb 17 '20 at 17:12
happy this could help. one thing to keep in mind - when smoothing, there often isn't one best answer, so the recommended approach is to explore multiple kinds of smoothing to see which result is preferable for your problem. — DanWaters, Feb 19 '20 at 18:22

Remove steps in time series

1 Answers1