0

I just stood up a 3 node elastic search cluster (3 nodes are masters and data nodes) and a 4th node that has elasticsearch/kibana on it but the elasticsearch instance on that node has node.master: false, node.data: false, node.ingest: false so that it acts as the load balancer. When I launch kibana, I get this in the console:

  log   [19:04:18.112] [info][status][plugin:kibana@5.2.1] Status changed from uninitialized to green - Ready
  log   [19:04:18.175] [info][status][plugin:elasticsearch@5.2.1] Status changed from uninitialized to yellow - Waiting for Elasticsearch
  log   [19:04:18.203] [info][status][plugin:console@5.2.1] Status changed from uninitialized to green - Ready
  log   [19:04:18.224] [error][status][plugin:elasticsearch@5.2.1] Status changed from yellow to red - Elasticsearch is still initializing the kibana index.
  log   [19:04:18.360] [info][status][plugin:timelion@5.2.1] Status changed from uninitialized to green - Ready
  log   [19:04:18.364] [info][listening] Server running at http://ipaddy:9051
  log   [19:04:18.366] [error][status][ui settings] Status changed from uninitialized to red - Elasticsearch plugin is red

here is my cluster statecurl -XGET 'http://node4:9050/_cluster/state?pretty':

{
  "cluster_name" : "cluster_elastic",
  "version" : 10,
  "state_uuid" : "LaBEHRpOS6Wya5oBrEeK_g",
  "master_node" : "ogQmsoyDSRG_jZ25XHsrvQ",
  "blocks" : { },
  "nodes" : {
    "fR8QvhMHTLCsF5r-2EwKIg" : {
      "name" : "node1",
      "ephemeral_id" : "PX8xN6kVSwmnApQcUdXJAA",
      "transport_address" : "node1:9300",
      "attributes" : { }
    },
    "kqIzY8oIQuuy88WgsjjpYg" : {
      "name" : "node2",
      "ephemeral_id" : "hy2j_0omTRqZyhwsSjBd0Q",
      "transport_address" : "node2:9300",
      "attributes" : { }
    },
    "VxLbvwf1SBK1S_IaO9qgWQ" : {
      "name" : "node3",
      "ephemeral_id" : "1yzEIm5xRUeJLZDGh2PkCw",
      "transport_address" : "node3:9300",
      "attributes" : { }
    },
    "ogQmsoyDSRG_jZ25XHsrvQ" : {
      "name" : "node4",
      "ephemeral_id" : "9DUQ2jy5TpOQT1BShVuvjQ",
      "transport_address" : "node4:9300",
      "attributes" : { }
    }
  },
  "metadata" : {
    "cluster_uuid" : "yIcz4aKiTyKfl49CXagevw",
    "templates" : { },
    "indices" : {
      ".kibana" : {
        "state" : "open",
        "settings" : {
          "index" : {
            "creation_date" : "1487085174946",
            "number_of_shards" : "1",
            "number_of_replicas" : "1",
            "uuid" : "lDnD6bTESpCr6Wd5SxGIFg",
            "version" : {
              "created" : "5020099",
              "upgraded" : "5020199"
            },
            "provided_name" : ".kibana"
          }
        },
        "mappings" : {
          "index-pattern" : {
            "properties" : {
              "notExpandable" : {
                "type" : "boolean"
              },
              "fieldFormatMap" : {
                "type" : "text"
              },
              "sourceFilters" : {
                "type" : "text"
              },
              "timeFieldName" : {
                "type" : "text"
              },
              "intervalName" : {
                "type" : "text"
              },
              "fields" : {
                "type" : "text"
              },
              "title" : {
                "type" : "text"
              }
            }
          },
          "config" : {
            "properties" : {
              "buildNum" : {
                "type" : "keyword"
              }
            }
          },
          "server" : {
            "properties" : {
              "uuid" : {
                "type" : "keyword"
              }
            }
          }
        },
        "aliases" : [ ],
        "primary_terms" : {
          "0" : 3
        },
        "in_sync_allocations" : {
          "0" : [
            "baJSWiYsSXiFiNsN27Q-Qw"
          ]
        }
      }
    },
    "index-graveyard" : {
      "tombstones" : [
        {
          "index" : {
            "index_name" : "kafkajmxclient",
            "index_uuid" : "Uv-hsbg4QXuQGqdO-Al8LA"
          },
          "delete_date_in_millis" : 1487704508081
        }
      ]
    }
  },
  "routing_table" : {
    "indices" : {
      ".kibana" : {
        "shards" : {
          "0" : [
            {
              "state" : "UNASSIGNED",
              "primary" : true,
              "node" : null,
              "relocating_node" : null,
              "shard" : 0,
              "index" : ".kibana",
              "recovery_source" : {
                "type" : "EXISTING_STORE"
              },
              "unassigned_info" : {
                "reason" : "DANGLING_INDEX_IMPORTED",
                "at" : "2017-02-21T18:38:00.405Z",
                "delayed" : false,
                "allocation_status" : "no_valid_shard_copy"
              }
            },
            {
              "state" : "UNASSIGNED",
              "primary" : false,
              "node" : null,
              "relocating_node" : null,
              "shard" : 0,
              "index" : ".kibana",
              "recovery_source" : {
                "type" : "PEER"
              },
              "unassigned_info" : {
                "reason" : "DANGLING_INDEX_IMPORTED",
                "at" : "2017-02-21T18:38:00.405Z",
                "delayed" : false,
                "allocation_status" : "no_attempt"
              }
            }
          ]
        }
      }
    }
  },
  "routing_nodes" : {
    "unassigned" : [
      {
        "state" : "UNASSIGNED",
        "primary" : true,
        "node" : null,
        "relocating_node" : null,
        "shard" : 0,
        "index" : ".kibana",
        "recovery_source" : {
          "type" : "EXISTING_STORE"
        },
        "unassigned_info" : {
          "reason" : "DANGLING_INDEX_IMPORTED",
          "at" : "2017-02-21T18:38:00.405Z",
          "delayed" : false,
          "allocation_status" : "no_valid_shard_copy"
        }
      },
      {
        "state" : "UNASSIGNED",
        "primary" : false,
        "node" : null,
        "relocating_node" : null,
        "shard" : 0,
        "index" : ".kibana",
        "recovery_source" : {
          "type" : "PEER"
        },
        "unassigned_info" : {
          "reason" : "DANGLING_INDEX_IMPORTED",
          "at" : "2017-02-21T18:38:00.405Z",
          "delayed" : false,
          "allocation_status" : "no_attempt"
        }
      }
    ],
    "nodes" : {
      "kqIzY8oIQuuy88WgsjjpYg" : [ ],
      "ogQmsoyDSRG_jZ25XHsrvQ" : [ ],
      "fR8QvhMHTLCsF5r-2EwKIg" : [ ]
    }
  }
}

here is my cluster health curl -XGET 'http://node4:9050/_cluster/health?pretty':

{
  "cluster_name" : "sdp_elastic",
  "status" : "red",
  "timed_out" : false,
  "number_of_nodes" : 4,
  "number_of_data_nodes" : 3,
  "active_primary_shards" : 0,
  "active_shards" : 0,
  "relocating_shards" : 0,
  "initializing_shards" : 0,
  "unassigned_shards" : 2,
  "delayed_unassigned_shards" : 0,
  "number_of_pending_tasks" : 0,
  "number_of_in_flight_fetch" : 0,
  "task_max_waiting_in_queue_millis" : 0,
  "active_shards_percent_as_number" : 0.0
}
lightweight
  • 3,227
  • 14
  • 79
  • 142
  • 1
    Your ES cluster state is `RED` due to dangling index. Did you perform any delete operation on `.kibana` index? – avr Feb 21 '17 at 19:52
  • no, although I did delete an index called `kafkajmxclient`...how do I fix this? I don't need any of the data, I actually want to start fresh...should I delete the `.kibana` index? Seems like its important so wasn't sure if I should mess with it... – lightweight Feb 21 '17 at 20:21
  • 1
    If you really want to start fresh then 1. Stop ES on all 4 nodes 2. Change/delete data directory on all 4 nodes 3. Start ES on all nodes. It should resolve the issue. – avr Feb 21 '17 at 20:27
  • didn't try it that way but I did find this post and it fixed the issue...http://stackoverflow.com/questions/31201051/elasticsearch-is-still-initializing-the-kibana-index – lightweight Feb 21 '17 at 20:31
  • Thats Nice and much easy solution. – avr Feb 21 '17 at 20:35

0 Answers0