The problem I have is that my Cloudflare indices report the following ILM errors:
- on index with alias:
illegal_argument_exception: rollover target [cloudflare] does not point to a write index - on index without alias:
illegal_argument_exception: index.lifecycle.rollover_alias [cloudflare] does not point to index [cloudflare-2022.08.13-000001]
Basically what I was able to find out is that when a new index is created, it doesn't receive the alias from rollover_alias:
{
"settings": {
"index": {
"lifecycle": {
"name": "cloudflare",
"rollover_alias": "cloudflare"
},
option which makes the rollover fail. When I assign the alias manually to all indices affected, rollover and ILM starts to work again but I want to understand why does it happen and find out a permanent solution to this problem. Otherwise I will have to check this cluster manually and force moving the data from HOT to WARM nodes when the HOT storage fills up.
The setup on Cloudflare is based on this guide, in other words Cloudflare pushes the logs to S3 bucket, then AWS Lambda pushes them to ELK (elastic.co).
Cloudflare index template in question:
"cloudflare": {
"index_patterns": [
"cloudflare-*"
],
"mappings": {
"properties": {
"observer.ip": {
"type": "ip"
},
"cloudflare.parent.ray_id": {
"type": "keyword"
},
"cloudflare.worker.subrequest_count": {
"type": "long"
},
"cloudflare.origin.ip": {
"type": "ip"
},
"cloudflare.edge.rate.limit.id": {
"type": "long"
},
"user_agent.version": {
"type": "keyword"
},
"cloudflare.device.type": {
"type": "keyword"
},
"cloudflare.edge.pathing.op": {
"type": "keyword"
},
"user_agent.os.version": {
"type": "keyword"
},
"source.port": {
"type": "long"
},
"cloudflare.edge.server.ip": {
"type": "ip"
},
"cloudflare.security_level": {
"type": "keyword"
},
"observer.vendor": {
"type": "keyword"
},
"event.dataset": {
"type": "keyword"
},
"cloudflare.worker.cpu_time": {
"type": "long"
},
"http.response.status_code": {
"type": "long"
},
"user_agent.minor": {
"type": "keyword"
},
"cloudflare.cache.response.status": {
"type": "long"
},
"user_agent.patch": {
"type": "keyword"
},
"@timestamp": {
"type": "date"
},
"cloudflare.edge.colo.id": {
"type": "integer"
},
"user_agent.os.full": {
"type": "keyword"
},
"source.address": {
"type": "keyword"
},
"user_agent.build": {
"type": "keyword"
},
"source.as.number": {
"type": "long"
},
"cloudflare.edge.start.timestamp": {
"type": "date"
},
"cloudflare.waf.rule.id": {
"type": "keyword"
},
"cloudflare.origin.ssl.protocol": {
"type": "keyword"
},
"http.request.bytes": {
"type": "long"
},
"source.geo.country_iso_code": {
"type": "keyword"
},
"cloudflare.edge.pathing.src": {
"type": "keyword"
},
"cloudflare.edge.response.bytes": {
"type": "long"
},
"cloudflare.edge.response.status": {
"type": "long"
},
"cloudflare.waf.rule.message": {
"type": "keyword"
},
"cloudflare.origin.response.time": {
"type": "long"
},
"url.path": {
"fields": {
"path": {
"index": true,
"eager_global_ordinals": false,
"fielddata": false,
"index_options": "positions",
"index_phrases": false,
"norms": true,
"type": "text",
"store": false
}
},
"type": "keyword"
},
"cloudflare.edge.response.compression_ratio": {
"type": "float"
},
"cloudflare.worker.subrequest": {
"type": "boolean"
},
"cloudflare.cache.response.bytes": {
"type": "long"
},
"cloudflare.waf.profile": {
"type": "keyword"
},
"cloudflare.waf.flags": {
"type": "keyword"
},
"cloudflare.firewall.matches.actions": {
"type": "keyword"
},
"cloudflare.http.response.status_code": {
"type": "long"
},
"user_agent.os.platform": {
"type": "keyword"
},
"cloudflare.waf.matched_var": {
"type": "keyword"
},
"user_agent.os_minor": {
"type": "keyword"
},
"cloudflare.worker.status": {
"type": "keyword"
},
"@version": {
"type": "keyword"
},
"cloudflare.firewall.matches.rule_ids": {
"type": "keyword"
},
"user_agent.os_major": {
"type": "keyword"
},
"cloudflare.origin.response.bytes": {
"type": "long"
},
"source.ip": {
"type": "ip"
},
"http.response.bytes": {
"type": "long"
},
"cloudflare.client.ssl.protocol": {
"type": "keyword"
},
"url.full": {
"type": "keyword"
},
"client.address": {
"type": "keyword"
},
"user_agent.os_name": {
"type": "keyword"
},
"cloudflare.edge.end.timestamp": {
"type": "date"
},
"cloudflare.origin.response.http.last_modified": {
"ignore_malformed": true,
"type": "date"
},
"user_agent.original": {
"type": "keyword"
},
"cloudflare.cache.tiered.fill": {
"type": "boolean"
},
"cloudflare.origin.response.http.expires": {
"type": "date",
"format": "E, d MMM uuuu HH:mm:ss 'UTC'"
},
"user_agent.name": {
"type": "keyword"
},
"cloudflare.waf.action": {
"type": "keyword"
},
"cloudflare.cache.status": {
"type": "keyword"
},
"cloudflare.edge.request.host": {
"type": "keyword"
},
"source.geo": {
"type": "object",
"properties": {
"region_code": {
"type": "keyword"
},
"longitude": {
"type": "float"
},
"region_iso_code": {
"type": "keyword"
},
"region_name": {
"type": "keyword"
},
"country_code2": {
"type": "keyword"
},
"ip": {
"type": "ip"
},
"continent_code": {
"type": "keyword"
},
"postal_code": {
"type": "keyword"
},
"country_code3": {
"type": "keyword"
},
"latitude": {
"type": "float"
},
"city_name": {
"type": "keyword"
},
"dma_code": {
"type": "long"
},
"country_name": {
"type": "keyword"
},
"continent_name": {
"type": "keyword"
},
"timezone": {
"type": "keyword"
},
"location": {
"type": "geo_point"
}
}
},
"cloudflare.edge.rate.limit.action": {
"type": "keyword"
},
"cloudflare.client.ssl.cipher": {
"type": "keyword"
},
"user_agent.os.name": {
"type": "keyword"
},
"cloudflare.edge.pathing.status": {
"type": "keyword"
},
"cloudflare.zone_id": {
"type": "integer"
},
"client.port": {
"type": "long"
},
"observer.type": {
"type": "keyword"
},
"http.request.referrer": {
"type": "keyword"
},
"user_agent.major": {
"type": "keyword"
},
"event.end": {
"type": "date"
},
"cloudflare.client.request.protocol": {
"type": "keyword"
},
"user_agent.device.name": {
"type": "keyword"
},
"destination.ip": {
"type": "ip"
},
"url.domain": {
"type": "keyword"
},
"http.request.method": {
"type": "keyword"
},
"cloudflare.firewall.matches.sources": {
"type": "keyword"
},
"cloudflare.edge.response.content_type": {
"type": "keyword"
},
"cloudflare.ray_id": {
"type": "keyword"
},
"event.start": {
"type": "date"
},
"ecs.version": {
"type": "keyword"
},
"client.ip": {
"type": "ip"
},
"cloudflare.edge.colo.code": {
"type": "keyword"
},
"http.version": {
"type": "keyword"
},
"cloudflare.client.ip.class": {
"type": "keyword"
},
"server.ip": {
"type": "ip"
},
"user_agent.os.kernel": {
"type": "keyword"
}
}
},
"aliases": {},
"order": 0,
"settings": {
"index": {
"number_of_replicas": "1",
"mapping": {
"ignore_malformed": "true"
},
"number_of_shards": "1",
"lifecycle": {
"rollover_alias": "cloudflare",
"name": "cloudflare"
},
"routing": {
"allocation": {
"include": {
"_tier_preference": null
}
}
}
}
}
}
}
ILM Policy in quesion:
{
"cloudflare": {
"policy": {
"phases": {
"cold": {
"actions": {
"set_priority": {
"priority": 0
}
},
"min_age": "30d"
},
"warm": {
"actions": {
"set_priority": {
"priority": 50
}
},
"min_age": "0ms"
},
"hot": {
"actions": {
"rollover": {
"max_age": "1d"
},
"set_priority": {
"priority": 100
}
},
"min_age": "0ms"
},
"delete": {
"actions": {
"delete": {
"delete_searchable_snapshot": true
}
},
"min_age": "60d"
}
}
},
"modified_date": "2021-11-02T17:18:34.417Z",
"in_use_by": {
"indices": [
"cloudflare-2022.07.09-000001",
"cloudflare-2022.07.08-000001",
"cloudflare-2022.07.04-000001",
"cloudflare-2022.07.06-000001",
"cloudflare-2022.07.07-000001",
"cloudflare-2022.07.05-000001",
"cloudflare-2022.06.10-000001",
"cloudflare-2022.06.12-000001",
"cloudflare-2022.06.11-000001",
"cloudflare-2022.06.13-000001",
"cloudflare-2022.08.02-000001",
"cloudflare-2022.08.03-000001",
"cloudflare-2022.08.01-000001",
"cloudflare-2022.08.04-000001",
"cloudflare-2022.08.08-000001",
"cloudflare-2022.06.18-000001",
"cloudflare-2022.08.06-000001",
"cloudflare-2022.06.07-000001",
"cloudflare-2022.06.16-000001",
"cloudflare-2022.06.14-000001",
"cloudflare-2022.06.09-000001",
"cloudflare-2022.06.05-000001",
"cloudflare-2022.06.03-000001",
"cloudflare-2022.05.23-000001",
"cloudflare-2022.05.21-000001",
"cloudflare-2022.07.02-000001",
"cloudflare-2022.07.11-000001",
"cloudflare-2022.07.13-000001",
"cloudflare-2022.08.01-000017",
"cloudflare-2022.07.17-000001",
"cloudflare-2022.07.18-000001",
"cloudflare-2022.05.28-000001",
"cloudflare-2022.05.27-000001",
"cloudflare-2022.05.24-000001",
"cloudflare-2022.06.01-000001",
"cloudflare-2022.06.22-000001",
"cloudflare-2022.08.02-000023",
"cloudflare-2022.08.03-000024",
"cloudflare-2022.08.02-000021",
"cloudflare-2022.06.23-000001",
"cloudflare-2022.08.02-000022",
"cloudflare-2022.08.12-000001",
"cloudflare-2022.08.06-000027",
"cloudflare-2022.08.13-000001",
"cloudflare-2022.08.07-000028",
"cloudflare-2022.06.19-000001",
"cloudflare-2022.08.16-000001",
"cloudflare-2022.06.26-000001",
"cloudflare-2022.08.09-000001",
"cloudflare-2022.08.05-000001",
"cloudflare-2022.08.02-000020",
"cloudflare-2022.06.15-000001",
"cloudflare-2022.05.20-000001",
"cloudflare-2022.06.08-000001",
"cloudflare-2022.07.10-000001",
"cloudflare-2022.06.04-000001",
"cloudflare-2022.07.03-000001",
"cloudflare-2022.05.31-000001",
"cloudflare-2022.07.14-000001",
"cloudflare-2022.07.25-000004",
"cloudflare-2022.07.21-000001",
"cloudflare-2022.07.25-000001",
"cloudflare-2022.08.02-000018",
"cloudflare-2022.08.02-000019",
"cloudflare-2022.07.29-000001",
"cloudflare-2022.07.26-000001",
"cloudflare-2022.07.27-000009",
"cloudflare-2022.07.30-000015",
"cloudflare-2022.07.30-000014",
"cloudflare-2022.07.31-000016",
"cloudflare-2022.07.30-000013",
"cloudflare-2022.07.27-000010",
"cloudflare-2022.06.30-000001",
"cloudflare-2022.07.28-000011",
"cloudflare-2022.08.17-000001",
"cloudflare-2022.07.29-000012",
"cloudflare-2022.06.27-000001",
"cloudflare-2022.06.29-000001",
"cloudflare-2022.06.25-000001",
"cloudflare-2022.05.30-000001",
"cloudflare-2022.07.26-000008",
"cloudflare-2022.07.22-000001",
"cloudflare-2022.07.26-000007",
"cloudflare-2022.07.31-000001",
"cloudflare-2022.07.26-000006",
"cloudflare-2022.07.24-000001",
"cloudflare-2022.07.26-000005",
"cloudflare-2022.07.20-000001",
"cloudflare-2022.07.24-000003",
"cloudflare-2022.07.28-000001",
"cloudflare-2022.05.29-000001",
"cloudflare-2022.07.16-000001",
"cloudflare-2022.07.19-000001",
"cloudflare-2022.07.15-000001",
"cloudflare-2022.08.09-000030",
"cloudflare-2022.05.25-000001",
"cloudflare-2022.05.26-000001",
"cloudflare-2022.06.02-000001",
"cloudflare-2022.06.21-000001",
"cloudflare-2022.06.20-000001",
"cloudflare-2022.06.24-000001",
"cloudflare-2022.08.05-000026",
"cloudflare-2022.08.04-000025",
"cloudflare-2022.08.14-000001",
"cloudflare-2022.08.10-000001",
"cloudflare-2022.08.15-000001",
"cloudflare-2022.08.11-000001",
"cloudflare-2022.08.08-000029",
"cloudflare-2022.08.07-000001",
"cloudflare-2022.06.28-000001",
"cloudflare-2022.06.17-000001",
"cloudflare-2022.06.06-000001",
"cloudflare-2022.05.22-000001",
"cloudflare-2022.07.01-000001",
"cloudflare-2022.07.12-000001",
"cloudflare-2022.07.30-000001",
"cloudflare-2022.07.27-000001",
"cloudflare-2022.07.23-000001",
"cloudflare-2022.07.23-000002"
],
"data_streams": [],
"composable_templates": []
},
"version": 12
}
}
Elastic version: v7.16.2 provided by elastic.co on AWS
