Improve readiness probe output (grafana#556)

Signed-off-by: Steven Sheehy <ssheehy@firescope.com>

pkg/ingester/ingester.go

@@ -7,6 +7,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/go-kit/kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/weaveworks/common/user"
@@ -18,6 +19,8 @@ import (
 	"github.com/grafana/loki/pkg/logproto"
 )
 
+var readinessProbeSuccess = []byte("Ready")
+
 var flushQueueLength = promauto.NewGauge(prometheus.GaugeOpts{
 	Name: "cortex_ingester_flush_queue_length",
 	Help: "The total number of series pending in the flush queue.",
@@ -195,13 +198,17 @@ func (*Ingester) Watch(*grpc_health_v1.HealthCheckRequest, grpc_health_v1.Health
 }
 
 // ReadinessHandler is used to indicate to k8s when the ingesters are ready for
-// the addition removal of another ingester. Returns 204 when the ingester is
+// the addition removal of another ingester. Returns 200 when the ingester is
 // ready, 500 otherwise.
 func (i *Ingester) ReadinessHandler(w http.ResponseWriter, r *http.Request) {
-	if err := i.lifecycler.CheckReady(r.Context()); err == nil {
-		w.WriteHeader(http.StatusNoContent)
-	} else {
-		w.WriteHeader(http.StatusInternalServerError)
+	if err := i.lifecycler.CheckReady(r.Context()); err != nil {
+		http.Error(w, "Not ready: "+err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	w.WriteHeader(http.StatusOK)
+	if _, err := w.Write(readinessProbeSuccess); err != nil {
+		level.Error(util.Logger).Log("msg", "error writing success message", "error", err)
 	}
 }
 

pkg/promtail/server/server.go

@@ -10,6 +10,8 @@ import (
 	"strings"
 	"text/template"
 
+	logutil "github.com/cortexproject/cortex/pkg/util"
+	"github.com/go-kit/kit/log/level"
 	"github.com/pkg/errors"
 	"github.com/prometheus/common/version"
 	serverww "github.com/weaveworks/common/server"
@@ -18,6 +20,11 @@ import (
 	"github.com/grafana/loki/pkg/promtail/targets"
 )
 
+var (
+	readinessProbeFailure = "Not ready: Unable to find any logs to tail. Please verify permissions, volumes, scrape_config, etc."
+	readinessProbeSuccess = []byte("Ready")
+)
+
 // Server embed weaveworks server with static file and templating capability
 type Server struct {
 	*serverww.Server
@@ -155,10 +162,14 @@ func (s *Server) targets(rw http.ResponseWriter, _ *http.Request) {
 
 // ready serves the ready endpoint
 func (s *Server) ready(rw http.ResponseWriter, _ *http.Request) {
-	if s.tms.Ready() {
-		rw.WriteHeader(http.StatusNoContent)
-	} else {
-		rw.WriteHeader(http.StatusInternalServerError)
+	if !s.tms.Ready() {
+		http.Error(rw, readinessProbeFailure, http.StatusInternalServerError)
+		return
+	}
+
+	rw.WriteHeader(http.StatusOK)
+	if _, err := rw.Write(readinessProbeSuccess); err != nil {
+		level.Error(logutil.Logger).Log("msg", "error writing success message", "error", err)
 	}
 }