Add metrics from SNTPv4 packet to ntp collector & add ntpd sanity check (prometheus#655)

* Add metrics from SNTPv4 packet to ntp collector & add ntpd sanity check

1. Checking local clock against remote NTP daemon is bad idea, local
ntpd acting as a  client should do it better and avoid excessive load on
remote NTP server so the collector is refactored to query local NTP
server.

2. Checking local clock against remote one does not check local ntpd
itself. Local ntpd may be down or out of sync due to network issues, but
clock will be OK.

3. Checking NTP server using sanity of it's response is tricky and
depends on ntpd implementation, that's why common `node_ntp_sanity`
variable is exported.

* `govendor add golang.org/x/net/ipv4`, it is dependency of github.com/beevik/ntp

* Update github.com/beevik/ntp to include boring SNTP fix

* Use variable name from RFC5905

* ntp: move code to make export of raw metrics more explicit

* Move NTP math to `github.com/beevik/ntp`

* Make `golint` happy

* Add some brief docs explaining `ntp` prometheus#655 and `timex` prometheus#664 modules

* ntp: drop XXX comment that got its decision

* ntp: add `_seconds` suffix to relevant metrics

* Better `node_ntp_leap` comment

* s/node_ntp_reftime/node_ntp_reference_timestamp_seconds/ as requested by @discordianfish

* Extract subsystem name to const as suggested by @SuperQ

Author: darkk

README.md

@@ -64,6 +64,7 @@ logind | Exposes session counts from [logind](http://www.freedesktop.org/wiki/So
 meminfo\_numa | Exposes memory statistics from `/proc/meminfo_numa`. | Linux
 mountstats | Exposes filesystem statistics from `/proc/self/mountstats`. Exposes detailed NFS client statistics. | Linux
 nfs | Exposes NFS client statistics from `/proc/net/rpc/nfs`. This is the same information as `nfsstat -c`. | Linux
+ntp | Exposes local NTP daemon health to check [time](./docs/TIME.md) | _any_
 qdisc | Exposes [queuing discipline](https://en.wikipedia.org/wiki/Network_scheduler#Linux_kernel) statistics | Linux
 runit | Exposes service status from [runit](http://smarden.org/runit/). | _any_
 supervisord | Exposes service status from [supervisord](http://supervisord.org/). | _any_
@@ -78,7 +79,6 @@ Name     | Description | OS
 ---------|-------------|----
 gmond | Exposes statistics from Ganglia. | _any_
 megacli | Exposes RAID statistics from MegaCLI. | Linux
-ntp | Exposes time drift from an NTP server. | _any_
 
 ### Textfile Collector
 

collector/ntp.go

@@ -17,62 +17,146 @@ package collector
 
 import (
 	"fmt"
+	"net"
+	"time"
 
 	"github.com/beevik/ntp"
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/common/log"
 	"gopkg.in/alecthomas/kingpin.v2"
 )
 
+const (
+	hour24 = 24 * time.Hour // `time` does not export `Day` as Day != 24h because of DST
+	ntpSubsystem = "ntp"
+)
+
 var (
-	ntpServer          = kingpin.Flag("collector.ntp.server", "NTP server to use for ntp collector.").Default("").String()
+	ntpServer          = kingpin.Flag("collector.ntp.server", "NTP server to use for ntp collector").Default("127.0.0.1").String()
 	ntpProtocolVersion = kingpin.Flag("collector.ntp.protocol-version", "NTP protocol version").Default("4").Int()
+	ntpServerIsLocal   = kingpin.Flag("collector.ntp.server-is-local", "Certify that collector.ntp.server address is the same local host as this collector.").Default("false").Bool()
+	ntpIPTTL           = kingpin.Flag("collector.ntp.ip-ttl", "IP TTL to use while sending NTP query").Default("1").Int()
+	// 3.46608s ~ 1.5s + PHI * (1 << maxPoll), where 1.5s is MAXDIST from ntp.org, it is 1.0 in RFC5905
+	// max-distance option is used as-is without phi*(1<<poll)
+	ntpMaxDistance     = kingpin.Flag("collector.ntp.max-distance", "Max accumulated distance to the root").Default("3.46608s").Duration()
+	ntpOffsetTolerance = kingpin.Flag("collector.ntp.local-offset-tolerance", "Offset between local clock and local ntpd time to tolerate").Default("1ms").Duration()
+
+	leapMidnight time.Time
 )
 
 type ntpCollector struct {
-	drift, stratum typedDesc
+	stratum, leap, rtt, offset, reftime, rootDelay, rootDispersion, sanity typedDesc
 }
 
 func init() {
 	Factories["ntp"] = NewNtpCollector
 }
 
-// NewNtpCollector returns a new Collector exposing the offset between ntp and
-// the current system time.
+// NewNtpCollector returns a new Collector exposing sanity of local NTP server.
+// Default definition of "local" is:
+// - collector.ntp.server address is a loopback address (or collector.ntp.server-is-mine flag is turned on)
+// - the server is reachable with outgoin IP_TTL = 1
 func NewNtpCollector() (Collector, error) {
-	warnDeprecated("ntp")
-	if *ntpServer == "" {
-		return nil, fmt.Errorf("no NTP server specified, see -collector.ntp.server")
+	ipaddr := net.ParseIP(*ntpServer)
+	if !*ntpServerIsLocal && (ipaddr == nil || !ipaddr.IsLoopback()) {
+		return nil, fmt.Errorf("only IP address of local NTP server is valid for -collector.ntp.server")
 	}
+
 	if *ntpProtocolVersion < 2 || *ntpProtocolVersion > 4 {
 		return nil, fmt.Errorf("invalid NTP protocol version %d; must be 2, 3, or 4", *ntpProtocolVersion)
 	}
 
+	if *ntpOffsetTolerance < 0 {
+		return nil, fmt.Errorf("Offset tolerance must be non-negative")
+	}
+
 	return &ntpCollector{
-		drift: typedDesc{prometheus.NewDesc(
-			prometheus.BuildFQName(Namespace, "ntp", "drift_seconds"),
-			"Time between system time and ntp time.",
+		stratum: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, ntpSubsystem, "stratum"),
+			"NTPD stratum.",
 			nil, nil,
 		), prometheus.GaugeValue},
-		stratum: typedDesc{prometheus.NewDesc(
-			prometheus.BuildFQName(Namespace, "ntp", "stratum"),
-			"NTP server stratum.",
+		leap: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, ntpSubsystem, "leap"),
+			"NTPD leap second indicator, 2 bits.",
+			nil, nil,
+		), prometheus.GaugeValue},
+		rtt: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, ntpSubsystem, "rtt_seconds"),
+			"RTT to NTPD.",
+			nil, nil,
+		), prometheus.GaugeValue},
+		offset: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, ntpSubsystem, "offset_seconds"),
+			"ClockOffset between NTP and local clock.",
+			nil, nil,
+		), prometheus.GaugeValue},
+		reftime: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, ntpSubsystem, "reference_timestamp_seconds"),
+			"NTPD ReferenceTime, UNIX timestamp.",
+			nil, nil,
+		), prometheus.GaugeValue},
+		rootDelay: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, ntpSubsystem, "root_delay_seconds"),
+			"NTPD RootDelay.",
+			nil, nil,
+		), prometheus.GaugeValue},
+		rootDispersion: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, ntpSubsystem, "root_dispersion_seconds"),
+			"NTPD RootDispersion.",
+			nil, nil,
+		), prometheus.GaugeValue},
+		sanity: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, ntpSubsystem, "sanity"),
+			"NTPD sanity according to RFC5905 heuristics and configured limits.",
 			nil, nil,
 		), prometheus.GaugeValue},
 	}, nil
 }
 
 func (c *ntpCollector) Update(ch chan<- prometheus.Metric) error {
-	resp, err := ntp.Query(*ntpServer, *ntpProtocolVersion)
+	resp, err := ntp.QueryWithOptions(*ntpServer, ntp.QueryOptions{
+		Version: *ntpProtocolVersion,
+		TTL:     *ntpIPTTL,
+		Timeout: time.Second, // default `ntpdate` timeout
+	})
 	if err != nil {
-		return fmt.Errorf("couldn't get NTP drift: %s", err)
+		return fmt.Errorf("couldn't get SNTP reply: %s", err)
+	}
+
+	ch <- c.stratum.mustNewConstMetric(float64(resp.Stratum))
+	ch <- c.leap.mustNewConstMetric(float64(resp.Leap))
+	ch <- c.rtt.mustNewConstMetric(resp.RTT.Seconds())
+	ch <- c.offset.mustNewConstMetric(resp.ClockOffset.Seconds())
+	if resp.ReferenceTime.Unix() > 0 {
+		// Go Zero is   0001-01-01 00:00:00 UTC
+		// NTP Zero is  1900-01-01 00:00:00 UTC
+		// UNIX Zero is 1970-01-01 00:00:00 UTC
+		// so let's keep ALL ancient `reftime` values as zero
+		ch <- c.reftime.mustNewConstMetric(float64(resp.ReferenceTime.UnixNano()) / 1e9)
+	} else {
+		ch <- c.reftime.mustNewConstMetric(0)
+	}
+	ch <- c.rootDelay.mustNewConstMetric(resp.RootDelay.Seconds())
+	ch <- c.rootDispersion.mustNewConstMetric(resp.RootDispersion.Seconds())
+
+	// Here is SNTP packet sanity check that is exposed to move burden of
+	// configuration from node_exporter user to the developer.
+
+	maxerr := *ntpOffsetTolerance
+	if resp.Leap == ntp.LeapAddSecond || resp.Leap == ntp.LeapDelSecond {
+		// state of leapMidnight is cached as leap flag is dropped right after midnight
+		leapMidnight = resp.Time.Truncate(hour24).Add(hour24)
+	}
+	if leapMidnight.Add(-hour24).Before(resp.Time) && resp.Time.Before(leapMidnight.Add(hour24)) {
+		// tolerate leap smearing
+		maxerr += time.Second
+	}
+
+	if resp.Validate() && resp.RootDistance <= *ntpMaxDistance && resp.CausalityViolation <= maxerr {
+		ch <- c.sanity.mustNewConstMetric(1)
+	} else {
+		ch <- c.sanity.mustNewConstMetric(0)
 	}
-	driftSeconds := resp.ClockOffset.Seconds()
-	log.Debugf("Set ntp_drift_seconds: %f", driftSeconds)
-	ch <- c.drift.mustNewConstMetric(driftSeconds)
 
-	stratum := float64(resp.Stratum)
-	log.Debugf("Set ntp_stratum: %f", stratum)
-	ch <- c.stratum.mustNewConstMetric(stratum)
 	return nil
 }