lib/relay, lib/svcutil: Improve service logging (fixes #7580) (#7647)

lib/svcutil/svcutil.go

@@ -160,7 +160,7 @@ func SpecWithDebugLogger(l logger.Logger) suture.Spec {
 }
 
 func SpecWithInfoLogger(l logger.Logger) suture.Spec {
-	return spec(func(e suture.Event) { l.Infoln(e) })
+	return spec(infoEventHook(l))
 }
 
 func spec(eventHook suture.EventHook) suture.Spec {
@@ -171,3 +171,35 @@ func spec(eventHook suture.EventHook) suture.Spec {
 		DontPropagateTermination: false,
 	}
 }
+
+// infoEventHook prints service failures and failures to stop services at level
+// info. All other events and identical, consecutive failures are logged at
+// debug only.
+func infoEventHook(l logger.Logger) suture.EventHook {
+	var prevTerminate suture.EventServiceTerminate
+	return func(ei suture.Event) {
+		switch e := ei.(type) {
+		case suture.EventStopTimeout:
+			l.Infof("%s: Service %s failed to terminate in a timely manner", e.SupervisorName, e.ServiceName)
+		case suture.EventServicePanic:
+			l.Warnln("Caught a service panic, which shouldn't happen")
+			l.Infoln(e)
+		case suture.EventServiceTerminate:
+			msg := fmt.Sprintf("%s: service %s failed: %s", e.SupervisorName, e.ServiceName, e.Err)
+			if e.ServiceName == prevTerminate.ServiceName && e.Err == prevTerminate.Err {
+				l.Debugln(msg)
+			} else {
+				l.Infoln(msg)
+			}
+			prevTerminate = e
+			l.Debugln(e) // Contains some backoff statistics
+		case suture.EventBackoff:
+			l.Debugf("%s: exiting the backoff state.", e.SupervisorName)
+		case suture.EventResume:
+			l.Debugf("%s: too many service failures - entering the backoff state.", e.SupervisorName)
+		default:
+			l.Warnln("Unknown suture supervisor event type", e.Type())
+			l.Infoln(e)
+		}
+	}
+}