Add service discovery

README.md

@@ -56,6 +56,8 @@ docker run -p 9340:9340 -e AWS_SDK_LOAD_CONFIG=true -e HOME=/ -v $HOME/.aws:/.aw
       --web.metrics-path="/metrics"
                                  Path under which to expose metrics
       --web.probe-path="/probe"  Path under which to expose the probe endpoint
+      --web.discovery-path="/discovery"
+                                 Path under which to expose service discovery
       --s3.endpoint-url=""       Custom endpoint URL
       --s3.disable-ssl           Custom disable SSL
       --s3.force-path-style      Custom force path style
@@ -83,7 +85,7 @@ Flags can also be set as environment variables, prefixed by `S3_EXPORTER_`. For
 
 ### Configuration
 
-You should pass the params to a single instance of the exporter using relabelling, like so:
+You can pass the params to a single instance of the exporter using relabelling, like so:
 
 ```yml
 scrape_configs:
@@ -106,6 +108,47 @@ scrape_configs:
         replacement: 127.0.0.1:9340 # S3 exporter.
 ```
 
+### Service Discovery
+
+Rather than defining a static list of buckets you can use the `/discovery` endpoint
+in conjunction with HTTP service discovery to discover all the buckets the
+exporter has access to.
+
+This should be all the config required to successfully scrape every bucket:
+
+```
+scrape_configs:
+  - job_name: 's3'
+    metrics_path: /probe
+    http_sd_configs:
+      - url: http://127.0.0.1:9340/discovery
+```
+
+You can limit the buckets returned with the `bucket_pattern` parameter. Refer to
+the documentation for [`path.Match`](https://golang.org/pkg/path/#Match) for the
+pattern syntax.
+
+```
+scrape_configs:
+  - job_name: 's3'
+    metrics_path: /probe
+    http_sd_configs:
+      # This will only discover buckets with a name that starts with example-
+      - url: http://127.0.0.1:9340/discovery?bucket_pattern=example-*
+```
+
+The prefix can be set too, but be mindful that this will apply to all buckets:
+
+```
+scrape_configs:
+  - job_name: 's3'
+    metrics_path: /probe
+    http_sd_configs:
+      - url: http://127.0.0.1:9340/discovery?bucket_pattern=example-*
+    params:
+      prefix: ["thing.txt"]
+```
+
 ### Example Queries
 
 Return series where the last modified object date is more than 24 hours ago:

go.sum

@@ -13,6 +13,7 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=

s3_exporter.go

@@ -1,8 +1,10 @@
 package main
 
 import (
+	"encoding/json"
 	"net/http"
 	"os"
+	"path"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -165,6 +167,52 @@ func probeHandler(w http.ResponseWriter, r *http.Request, svc s3iface.S3API) {
 	h.ServeHTTP(w, r)
 }
 
+type discoveryTarget struct {
+	Targets []string          `json:"targets"`
+	Labels  map[string]string `json:"labels"`
+}
+
+func discoveryHandler(w http.ResponseWriter, r *http.Request, svc s3iface.S3API) {
+	// If a bucket pattern isn't specified, then return every bucket
+	bucketPattern := r.URL.Query().Get("bucket_pattern")
+	if bucketPattern == "" {
+		bucketPattern = "*"
+	}
+
+	result, err := svc.ListBuckets(&s3.ListBucketsInput{})
+	if err != nil {
+		http.Error(w, "error listing buckets", http.StatusInternalServerError)
+		return
+	}
+
+	targets := []discoveryTarget{}
+	for _, b := range result.Buckets {
+		name := aws.StringValue(b.Name)
+
+		matched, err := path.Match(bucketPattern, name)
+		if err != nil {
+			http.Error(w, "bad pattern provided for 'bucket_pattern'", http.StatusBadRequest)
+		}
+		if matched {
+			t := discoveryTarget{
+				Targets: []string{r.Host},
+				Labels: map[string]string{
+					"__param_bucket": name,
+				},
+			}
+			targets = append(targets, t)
+		}
+	}
+
+	data, err := json.Marshal(targets)
+	if err != nil {
+		http.Error(w, "error marshalling json", http.StatusInternalServerError)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(data)
+}
+
 func init() {
 	prometheus.MustRegister(version.NewCollector(namespace + "_exporter"))
 }
@@ -175,6 +223,7 @@ func main() {
 		listenAddress  = app.Flag("web.listen-address", "Address to listen on for web interface and telemetry.").Default(":9340").String()
 		metricsPath    = app.Flag("web.metrics-path", "Path under which to expose metrics").Default("/metrics").String()
 		probePath      = app.Flag("web.probe-path", "Path under which to expose the probe endpoint").Default("/probe").String()
+		discoveryPath  = app.Flag("web.discovery-path", "Path under which to expose service discovery").Default("/discovery").String()
 		endpointURL    = app.Flag("s3.endpoint-url", "Custom endpoint URL").Default("").String()
 		disableSSL     = app.Flag("s3.disable-ssl", "Custom disable SSL").Bool()
 		forcePathStyle = app.Flag("s3.force-path-style", "Custom force path style").Bool()
@@ -210,13 +259,17 @@ func main() {
 	http.HandleFunc(*probePath, func(w http.ResponseWriter, r *http.Request) {
 		probeHandler(w, r, svc)
 	})
+	http.HandleFunc(*discoveryPath, func(w http.ResponseWriter, r *http.Request) {
+		discoveryHandler(w, r, svc)
+	})
 	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		w.Write([]byte(`<html>
 						 <head><title>AWS S3 Exporter</title></head>
 						 <body>
 						 <h1>AWS S3 Exporter</h1>
 						 <p><a href="` + *probePath + `?bucket=BUCKET&prefix=PREFIX">Query metrics for objects in BUCKET that match PREFIX</a></p>
 						 <p><a href='` + *metricsPath + `'>Metrics</a></p>
+						 <p><a href='` + *discoveryPath + `'>Service Discovery</a></p>
 						 </body>
 						 </html>`))
 	})