When stracing nagstamon to attempt to work out why I'm seeing the "ERROR:
please check settings/network" message (there were no issues with the
servers or network at the time), I noticed a broken attempt at DNS
resolution:
write(6, "A", 1) = 1
futex(0x23aaa18, FUTEX_WAIT_PRIVATE, 2, NULL) = 0
futex(0x23aaa18, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x291e9d0, FUTEX_WAKE_PRIVATE, 1) = 1
select(0, NULL, NULL, NULL, {5, 0}) = 0 (Timeout)
select(0, NULL, NULL, NULL, {5, 0}) = 0 (Timeout)
select(0, NULL, NULL, NULL, {10, 0}) = 0 (Timeout)
socket(PF_NETLINK, SOCK_RAW, 0) = 11
bind(11, {sa_family=AF_NETLINK, pid=0, groups=00000000}, 12) = 0
getsockname(11, {sa_family=AF_NETLINK, pid=3056, groups=00000000}, [12]) = 0
sendto(11, "\24\0\0\0\26\0\1\3\3756\254L\0\0\0\0\0\0\0\0", 20, 0, {sa_family=AF_NETLINK, pid=0, groups=00000000}, 12) = 20
recvmsg(11, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"0\0\0\0\24\0\2\0\3756\254L\360\v\0\0\2\10\200\376\1\0\0\0\10\0\1\0\177\0\0\1"..., 4096}], msg_controllen=0, msg_flags=0}, 0) = 108
recvmsg(11, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"@\0\0\0\24\0\2\0\3756\254L\360\v\0\0\n\200\200\376\1\0\0\0\24\0\1\0\0\0\0\0"..., 4096}], msg_controllen=0, msg_flags=0}, 0) = 128
recvmsg(11, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"\24\0\0\0\3\0\2\0\3756\254L\360\v\0\0\0\0\0\0\1\0\0\0\24\0\1\0\0\0\0\0"..., 4096}], msg_controllen=0, msg_flags=0}, 0) = 20
close(11) = 0
stat("/etc/resolv.conf", {st_mode=S_IFREG|0644, st_size=231, ...}) = 0
open("/etc/hosts", O_RDONLY|O_CLOEXEC) = 11
fstat(11, {st_mode=S_IFREG|0644, st_size=295, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3c2e107000
read(11, "127.0.0.1\tlocalhost\n127.0.1.1\tcr"..., 4096) = 295
read(11, "", 4096) = 0
close(11) = 0
munmap(0x7f3c2e107000, 4096) = 0
stat("/etc/resolv.conf", {st_mode=S_IFREG|0644, st_size=231, ...}) = 0
socket(PF_INET, SOCK_DGRAM|SOCK_NONBLOCK, IPPROTO_IP) = 11
connect(11, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("0.0.0.0")}, 16) = 0
poll([{fd=11, events=POLLOUT}], 1, 0) = 1 ([{fd=11, revents=POLLOUT}])
sendto(11, "\6g\1\0\0\1\0\0\0\0\0\0\6nagios\010xxxxxxxx\3org"..., 37, MSG_NOSIGNAL, NULL, 0) = 37
poll([{fd=11, events=POLLIN|POLLOUT}], 1, 5000) = 1 ([{fd=11, revents=POLLOUT|POLLERR}])
sendto(11, "\305\21\1\0\0\1\0\0\0\0\0\0\6nagios\010xxxxxxxx\3org"..., 37, MSG_NOSIGNAL, NULL, 0) = -1 ECONNREFUSED (Connection refused)
close(11) = 0
socket(PF_INET, SOCK_DGRAM|SOCK_NONBLOCK, IPPROTO_IP) = 11
connect(11, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("0.0.0.0")}, 16) = 0
poll([{fd=11, events=POLLOUT}], 1, 0) = 1 ([{fd=11, revents=POLLOUT}])
sendto(11, "\6g\1\0\0\1\0\0\0\0\0\0\6nagios\010xxxxxxxx\3org"..., 37, MSG_NOSIGNAL, NULL, 0) = 37
poll([{fd=11, events=POLLIN|POLLOUT}], 1, 5000) = 1 ([{fd=11, revents=POLLOUT|POLLERR}])
sendto(11, "\305\21\1\0\0\1\0\0\0\0\0\0\6nagios\010xxxxxxxx\3org"..., 37, MSG_NOSIGNAL, NULL, 0) = -1 ECONNREFUSED (Connection refused)
close(11) = 0
socket(PF_INET, SOCK_DGRAM|SOCK_NONBLOCK, IPPROTO_IP) = 11
connect(11, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("0.0.0.0")}, 16) = 0
poll([{fd=11, events=POLLOUT}], 1, 0) = 1 ([{fd=11, revents=POLLOUT}])
sendto(11, "\320\252\1\0\0\1\0\0\0\0\0\0\6nagios\010xxxxxxxx\3org"..., 63, MSG_NOSIGNAL, NULL, 0) = 63
poll([{fd=11, events=POLLIN|POLLOUT}], 1, 5000) = 1 ([{fd=11, revents=POLLOUT|POLLERR}])
sendto(11, "tM\1\0\0\1\0\0\0\0\0\0\6nagios\010xxxxxxxx\3org"..., 63, MSG_NOSIGNAL, NULL, 0) = -1 ECONNREFUSED (Connection refused)
close(11) = 0
socket(PF_INET, SOCK_DGRAM|SOCK_NONBLOCK, IPPROTO_IP) = 11
connect(11, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("0.0.0.0")}, 16) = 0
poll([{fd=11, events=POLLOUT}], 1, 0) = 1 ([{fd=11, revents=POLLOUT}])
sendto(11, "\320\252\1\0\0\1\0\0\0\0\0\0\6nagios\010xxxxxxxx\3org"..., 63, MSG_NOSIGNAL, NULL, 0) = 63
poll([{fd=11, events=POLLIN|POLLOUT}], 1, 5000) = 1 ([{fd=11, revents=POLLOUT|POLLERR}])
sendto(11, "tM\1\0\0\1\0\0\0\0\0\0\6nagios\010xxxxxxxx\3org"..., 63, MSG_NOSIGNAL, NULL, 0) = -1 ECONNREFUSED (Connection refused)
close(11) = 0
(I've redacted the domain name)
As you can see from the "connect" call above, nagstamon is attempting to
perform DNS resolution by connecting to the IP address "0.0.0.0", rather
than my actual DNS server (192.168.5.1).
Interestingly, the other threads which were dealing with my other two
servers had no problems resolving names:
munmap(0x7f3c2e107000, 4096) = 0
stat("/etc/resolv.conf", {st_mode=S_IFREG|0644, st_size=231, ...}) = 0
socket(PF_INET, SOCK_DGRAM|SOCK_NONBLOCK, IPPROTO_IP) = 11
connect(11, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("192.168.5.1")}, 16) = 0
poll([{fd=11, events=POLLOUT}], 1, 0) = 1 ([{fd=11, revents=POLLOUT}])
sendto(11, "i\230\1\0\0\1\0\0\0\0\0\0\6secure\tcrustynet\3or"..., 41, MSG_NOSIGNAL, NULL, 0) = 41
poll([{fd=11, events=POLLIN|POLLOUT}], 1, 5000) = 1 ([{fd=11, revents=POLLOUT}])
sendto(11, "\334\367\1\0\0\1\0\0\0\0\0\0\6secure\tcrustynet\3or"..., 41, MSG_NOSIGNAL, NULL, 0) = 41
poll([{fd=11, events=POLLIN}], 1, 4999) = 1 ([{fd=11, revents=POLLIN}])
ioctl(11, FIONREAD, [173]) = 0
recvfrom(11, "i\230\201\200\0\1\0\2\0\4\0\0\6secure\tcrustynet\3or"..., 2048, 0, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("192.168.5.1")}, [16]) = 173
poll([{fd=11, events=POLLIN}], 1, 4997) = 1 ([{fd=11, revents=POLLIN}])
ioctl(11, FIONREAD, [102]) = 0
recvfrom(11, "\334\367\201\200\0\1\0\1\0\1\0\0\6secure\tcrustynet\3or"..., 1875, 0, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("192.168.5.1")}, [16]) = 102
close(11) = 0
I was able to fix the problem by going into the settings, editing the server
in question (making no changes) and then saving it. This restarted the
thread that was checking that server, and all seems fine for now. I expect
it will happen again though.
I've not been able to find what triggers this.. it just seems to happen
after nagstamon has been running for a while.