diff --git a/README.md b/README.md
index 5149756..70349a3 100644
--- a/README.md
+++ b/README.md
@@ -56,6 +56,8 @@ docker run -p 9340:9340 -e AWS_SDK_LOAD_CONFIG=true -e HOME=/ -v $HOME/.aws:/.aw
--web.metrics-path="/metrics"
Path under which to expose metrics
--web.probe-path="/probe" Path under which to expose the probe endpoint
+ --web.discovery-path="/discovery"
+ Path under which to expose service discovery
--s3.endpoint-url="" Custom endpoint URL
--s3.disable-ssl Custom disable SSL
--s3.force-path-style Custom force path style
@@ -83,7 +85,7 @@ Flags can also be set as environment variables, prefixed by `S3_EXPORTER_`. For
### Configuration
-You should pass the params to a single instance of the exporter using relabelling, like so:
+You can pass the params to a single instance of the exporter using relabelling, like so:
```yml
scrape_configs:
@@ -106,6 +108,47 @@ scrape_configs:
replacement: 127.0.0.1:9340 # S3 exporter.
```
+### Service Discovery
+
+Rather than defining a static list of buckets you can use the `/discovery` endpoint
+in conjunction with HTTP service discovery to discover all the buckets the
+exporter has access to.
+
+This should be all the config required to successfully scrape every bucket:
+
+```
+scrape_configs:
+ - job_name: 's3'
+ metrics_path: /probe
+ http_sd_configs:
+ - url: http://127.0.0.1:9340/discovery
+```
+
+You can limit the buckets returned with the `bucket_pattern` parameter. Refer to
+the documentation for [`path.Match`](https://golang.org/pkg/path/#Match) for the
+pattern syntax.
+
+```
+scrape_configs:
+ - job_name: 's3'
+ metrics_path: /probe
+ http_sd_configs:
+ # This will only discover buckets with a name that starts with example-
+ - url: http://127.0.0.1:9340/discovery?bucket_pattern=example-*
+```
+
+The prefix can be set too, but be mindful that this will apply to all buckets:
+
+```
+scrape_configs:
+ - job_name: 's3'
+ metrics_path: /probe
+ http_sd_configs:
+ - url: http://127.0.0.1:9340/discovery?bucket_pattern=example-*
+ params:
+ prefix: ["thing.txt"]
+```
+
### Example Queries
Return series where the last modified object date is more than 24 hours ago:
diff --git a/go.sum b/go.sum
index b03252a..91a56f6 100644
--- a/go.sum
+++ b/go.sum
@@ -13,6 +13,7 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
diff --git a/s3_exporter.go b/s3_exporter.go
index caac9d7..3eef667 100644
--- a/s3_exporter.go
+++ b/s3_exporter.go
@@ -1,8 +1,10 @@
package main
import (
+ "encoding/json"
"net/http"
"os"
+ "path"
"time"
"github.com/prometheus/client_golang/prometheus"
@@ -165,6 +167,52 @@ func probeHandler(w http.ResponseWriter, r *http.Request, svc s3iface.S3API) {
h.ServeHTTP(w, r)
}
+type discoveryTarget struct {
+ Targets []string `json:"targets"`
+ Labels map[string]string `json:"labels"`
+}
+
+func discoveryHandler(w http.ResponseWriter, r *http.Request, svc s3iface.S3API) {
+ // If a bucket pattern isn't specified, then return every bucket
+ bucketPattern := r.URL.Query().Get("bucket_pattern")
+ if bucketPattern == "" {
+ bucketPattern = "*"
+ }
+
+ result, err := svc.ListBuckets(&s3.ListBucketsInput{})
+ if err != nil {
+ http.Error(w, "error listing buckets", http.StatusInternalServerError)
+ return
+ }
+
+ targets := []discoveryTarget{}
+ for _, b := range result.Buckets {
+ name := aws.StringValue(b.Name)
+
+ matched, err := path.Match(bucketPattern, name)
+ if err != nil {
+ http.Error(w, "bad pattern provided for 'bucket_pattern'", http.StatusBadRequest)
+ }
+ if matched {
+ t := discoveryTarget{
+ Targets: []string{r.Host},
+ Labels: map[string]string{
+ "__param_bucket": name,
+ },
+ }
+ targets = append(targets, t)
+ }
+ }
+
+ data, err := json.Marshal(targets)
+ if err != nil {
+ http.Error(w, "error marshalling json", http.StatusInternalServerError)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ w.Write(data)
+}
+
func init() {
prometheus.MustRegister(version.NewCollector(namespace + "_exporter"))
}
@@ -175,6 +223,7 @@ func main() {
listenAddress = app.Flag("web.listen-address", "Address to listen on for web interface and telemetry.").Default(":9340").String()
metricsPath = app.Flag("web.metrics-path", "Path under which to expose metrics").Default("/metrics").String()
probePath = app.Flag("web.probe-path", "Path under which to expose the probe endpoint").Default("/probe").String()
+ discoveryPath = app.Flag("web.discovery-path", "Path under which to expose service discovery").Default("/discovery").String()
endpointURL = app.Flag("s3.endpoint-url", "Custom endpoint URL").Default("").String()
disableSSL = app.Flag("s3.disable-ssl", "Custom disable SSL").Bool()
forcePathStyle = app.Flag("s3.force-path-style", "Custom force path style").Bool()
@@ -210,6 +259,9 @@ func main() {
http.HandleFunc(*probePath, func(w http.ResponseWriter, r *http.Request) {
probeHandler(w, r, svc)
})
+ http.HandleFunc(*discoveryPath, func(w http.ResponseWriter, r *http.Request) {
+ discoveryHandler(w, r, svc)
+ })
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`
AWS S3 Exporter
@@ -217,6 +269,7 @@ func main() {
AWS S3 Exporter
Query metrics for objects in BUCKET that match PREFIX
Metrics
+ Service Discovery