6

We are having an issue on one of our production systems and for the life of us can not find a solution to it

2017-08-22 19:15:52.337 [Supervisor-1/Connection-2] Got an error, no more attempts to do. Completing with a failure... reactivemongo.core.actors.Exceptions$ChannelNotFound: MongoError['Channel not found from the primary node: '{{cluster-shard-0}}.mongodb.net:27017' { authenticated:0, connected:10, channels:10 } (Supervisor-1/Connection-2)'] Caused by: reactivemongo.core.actors.Exceptions$InternalState: null at reactivemongo.ConnectAll$IsMaster(785, {{NodeSet None Node[{{cluster-shard-0}}.mongodb.net:27017: Primary (10/10 available connections), latency=5], auth=Set() | Node[{{cluster-shard-1}}.mongodb.net:27017: Secondary (10/10 available connections), latency=5], auth=Set(Authenticated(admin,{{collection}})) | Node[{{cluster-shard-2}}.mongodb.net:27017: Secondary (10/10 available connections), latency=5], auth=Set(Authenticated(admin,{{collection}})) }})() at reactivemongo.ConnectAll$IsMaster(783, {{NodeSet None Node[{{cluster-shard-0}}.mongodb.net:27017: Primary (10/10 available connections), latency=5], auth=Set() | Node[{{cluster-shard-1}}.mongodb.net:27017: Secondary (10/10 available connections), latency=5], auth=Set(Authenticated(admin,{{collection}})) | Node[{{cluster-shard-2}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) }})() at reactivemongo.ConnectAll$IsMaster(784, {{NodeSet None Node[{{cluster-shard-0}}.mongodb.net:27017: Primary (10/10 available connections), latency=2], auth=Set() | Node[{{cluster-shard-1}}.mongodb.net:27017: Secondary (10/10 available connections), latency=5], auth=Set(Authenticated(admin,{{collection}})) | Node[{{cluster-shard-2}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) }})() at reactivemongo.IsMaster(785, {{NodeSet None Node[{{cluster-shard-0}}.mongodb.net:27017: Primary (10/10 available connections), latency=5], auth=Set() | Node[{{cluster-shard-1}}.mongodb.net:27017: Secondary (10/10 available connections), latency=5], auth=Set(Authenticated(admin,{{collection}})) | Node[{{cluster-shard-2}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) }}() at reactivemongo.IsMaster(783, {{NodeSet None Node[{{cluster-shard-0}}.mongodb.net:27017: Primary (10/10 available connections), latency=2], auth=Set() | Node[{{cluster-shard-1}}.mongodb.net:27017: Secondary (10/10 available connections), latency=5], auth=Set(Authenticated(admin,{{collection}})) | Node[{{cluster-shard-2}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) }}() at reactivemongo.IsMaster(784, {{NodeSet None Node[{{cluster-shard-0}}.mongodb.net:27017: Primary (10/10 available connections), latency=2], auth=Set() | Node[{{cluster-shard-1}}.mongodb.net:27017: Secondary (10/10 available connections), latency=1], auth=Set(Authenticated(admin,{{collection}})) | Node[{{cluster-shard-2}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) }}() at reactivemongo.RefreshAll({{NodeSet None Node[{{cluster-shard-0}}.mongodb.net:27017: Primary (10/10 available connections), latency=2], auth=Set() | Node[{{cluster-shard-1}}.mongodb.net:27017: Secondary (10/10 available connections), latency=1], auth=Set(Authenticated(admin,{{collection}})) | Node[{{cluster-shard-2}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) }})() at reactivemongo.ConnectAll({{NodeSet None Node[{{cluster-shard-0}}.mongodb.net:27017: Primary (10/10 available connections), latency=2], auth=Set() | Node[{{cluster-shard-1}}.mongodb.net:27017: Secondary (10/10 available connections), latency=1], auth=Set(Authenticated(admin,{{collection}})) | Node[{{cluster-shard-2}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) }})() at reactivemongo.ConnectAll$IsMaster(782, {{NodeSet None Node[{{cluster-shard-0}}.mongodb.net:27017: Primary (10/10 available connections), latency=2], auth=Set() | Node[{{cluster-shard-1}}.mongodb.net:27017: Secondary (10/10 available connections), latency=1], auth=Set(Authenticated(admin,{{collection}})) | Node[{{cluster-shard-2}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) }})() at reactivemongo.IsMaster(782, {{NodeSet None Node[{{cluster-shard-0}}.mongodb.net:27017: Primary (10/10 available connections), latency=2], auth=Set() | Node[{{cluster-shard-1}}.mongodb.net:27017: Secondary (10/10 available connections), latency=1], auth=Set(Authenticated(admin,{{collection}})) | Node[{{cluster-shard-2}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) }}() at reactivemongo.ConnectAll$IsMaster(780, {{NodeSet None Node[{{cluster-shard-0}}.mongodb.net:27017: Primary (10/10 available connections), latency=2], auth=Set() | Node[{{cluster-shard-1}}.mongodb.net:27017: Secondary (10/10 available connections), latency=1], auth=Set(Authenticated(admin,{{collection}})) | Node[{{cluster-shard-2}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) }})() at reactivemongo.IsMaster(780, {{NodeSet None Node[{{cluster-shard-0}}.mongodb.net:27017: Primary (10/10 available connections), latency=2], auth=Set() | Node[{{cluster-shard-1}}.mongodb.net:27017: Secondary (10/10 available connections), latency=1], auth=Set(Authenticated(admin,{{collection}})) | Node[{{cluster-shard-2}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) }}() at reactivemongo.ConnectAll$IsMaster(781, {{NodeSet None Node[{{cluster-shard-0}}.mongodb.net:27017: Primary (10/10 available connections), latency=2], auth=Set() | Node[{{cluster-shard-1}}.mongodb.net:27017: Secondary (10/10 available connections), latency=1], auth=Set(Authenticated(admin,{{collection}})) | Node[{{cluster-shard-2}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) }})() at reactivemongo.IsMaster(781, {{NodeSet None Node[{{cluster-shard-0}}.mongodb.net:27017: Primary (10/10 available connections), latency=2], auth=Set() | Node[{{cluster-shard-1}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) | Node[{{cluster-shard-2}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) }}() at reactivemongo.RefreshAll({{NodeSet None Node[{{cluster-shard-0}}.mongodb.net:27017: Primary (10/10 available connections), latency=2], auth=Set() | Node[{{cluster-shard-1}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) | Node[{{cluster-shard-2}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) }})() at reactivemongo.ConnectAll({{NodeSet None Node[{{cluster-shard-0}}.mongodb.net:27017: Primary (10/10 available connections), latency=2], auth=Set() | Node[{{cluster-shard-1}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) | Node[{{cluster-shard-2}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) }})() at reactivemongo.ConnectAll$IsMaster(779, {{NodeSet None Node[{{cluster-shard-0}}.mongodb.net:27017: Primary (10/10 available connections), latency=2], auth=Set() | Node[{{cluster-shard-1}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) | Node[{{cluster-shard-2}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) }})() at reactivemongo.IsMaster(779, {{NodeSet None Node[{{cluster-shard-0}}.mongodb.net:27017: Primary (10/10 available connections), latency=2], auth=Set() | Node[{{cluster-shard-1}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) | Node[{{cluster-shard-2}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) }}() at reactivemongo.ConnectAll$IsMaster(777, {{NodeSet None Node[{{cluster-shard-0}}.mongodb.net:27017: Primary (10/10 available connections), latency=2], auth=Set() | Node[{{cluster-shard-1}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) | Node[{{cluster-shard-2}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) }})() at reactivemongo.IsMaster(777, {{NodeSet None Node[{{cluster-shard-0}}.mongodb.net:27017: Primary (10/10 available connections), latency=5], auth=Set() | Node[{{cluster-shard-1}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) | Node[{{cluster-shard-2}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) }}() at reactivemongo.ConnectAll$IsMaster(778, {{NodeSet None Node[{{cluster-shard-0}}.mongodb.net:27017: Primary (10/10 available connections), latency=5], auth=Set() | Node[{{cluster-shard-1}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) | Node[{{cluster-shard-2}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) }})() at reactivemongo.IsMaster(778, {{NodeSet None Node[{{cluster-shard-0}}.mongodb.net:27017: Primary (10/10 available connections), latency=5], auth=Set() | Node[{{cluster-shard-1}}.mongodb.net:27017: Secondary (10/10 available connections), latency=1], auth=Set(Authenticated(admin,{{collection}})) | Node[{{cluster-shard-2}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) }}() at reactivemongo.RefreshAll({{NodeSet None Node[{{cluster-shard-0}}.mongodb.net:27017: Primary (10/10 available connections), latency=5], auth=Set() | Node[{{cluster-shard-1}}.mongodb.net:27017: Secondary (10/10 available connections), latency=1], auth=Set(Authenticated(admin,{{collection}})) | Node[{{cluster-shard-2}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) }})() at reactivemongo.ConnectAll({{NodeSet None Node[{{cluster-shard-0}}.mongodb.net:27017: Primary (10/10 available connections), latency=5], auth=Set() | Node[{{cluster-shard-1}}.mongodb.net:27017: Secondary (10/10 available connections), latency=1], auth=Set(Authenticated(admin,{{collection}})) | Node[{{cluster-shard-2}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) }})() at reactivemongo.ConnectAll$IsMaster(776, {{NodeSet None Node[{{cluster-shard-0}}.mongodb.net:27017: Primary (10/10 available connections), latency=5], auth=Set() | Node[{{cluster-shard-1}}.mongodb.net:27017: Secondary (10/10 available connections), latency=1], auth=Set(Authenticated(admin,{{collection}})) | Node[{{cluster-shard-2}}.mongodb.net:27017: Secondary (10/10 available connections), latency=2], auth=Set(Authenticated(admin,{{collection}})) }})()

It seems to only happen on:

  • One availability zone in AWS
  • When the primary is re-elected

Any ideas?

cchantep
  • 9,118
  • 3
  • 30
  • 41
Deadcore
  • 61
  • 2
  • Would need much more system and network debugging: many things could cause the primary to no longer be available in such environment – cchantep Aug 24 '17 at 08:15

0 Answers0