Consolidate AWS credentials, Cloudwatch dimension wilcards

README.md

@@ -20,12 +20,12 @@ new plugins.
 ### Linux deb and rpm Packages:
 
 Latest:
-* https://dl.influxdata.com/telegraf/releases/telegraf_0.13.0-1_amd64.deb
-* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.0-1.x86_64.rpm
+* https://dl.influxdata.com/telegraf/releases/telegraf_0.13.0_amd64.deb
+* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.0.x86_64.rpm
 
 Latest (arm):
-* https://dl.influxdata.com/telegraf/releases/telegraf_0.13.0-1_armhf.deb
-* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.0-1.armhf.rpm
+* https://dl.influxdata.com/telegraf/releases/telegraf_0.13.0_armhf.deb
+* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.0.armhf.rpm
 
 ##### Package Instructions:
 
@@ -46,28 +46,28 @@ to use this repo to install & update telegraf.
 ### Linux tarballs:
 
 Latest:
-* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.0-1_linux_amd64.tar.gz
-* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.0-1_linux_i386.tar.gz
-* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.0-1_linux_armhf.tar.gz
+* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.0_linux_amd64.tar.gz
+* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.0_linux_i386.tar.gz
+* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.0_linux_armhf.tar.gz
 
 ##### tarball Instructions:
 
 To install the full directory structure with config file, run:
 
 ```
-sudo tar -C / -zxvf ./telegraf-0.13.0-1_linux_amd64.tar.gz
+sudo tar -C / -zxvf ./telegraf-0.13.0_linux_amd64.tar.gz
 ```
 
 To extract only the binary, run:
 
 ```
-tar -zxvf telegraf-0.13.0-1_linux_amd64.tar.gz --strip-components=3 ./usr/bin/telegraf
+tar -zxvf telegraf-0.13.0_linux_amd64.tar.gz --strip-components=3 ./usr/bin/telegraf
 ```
 
 ### FreeBSD tarball:
 
 Latest:
-* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.0-1_freebsd_amd64.tar.gz
+* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.0_freebsd_amd64.tar.gz
 
 ##### tarball Instructions:
 
@@ -87,8 +87,8 @@ brew install telegraf
 ### Windows Binaries (EXPERIMENTAL)
 
 Latest:
-* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.0-1_windows_amd64.zip
-* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.0-1_windows_i386.zip
+* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.0_windows_amd64.zip
+* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.0_windows_i386.zip
 
 ### From Source:
 

internal/config/aws/credentials.go

@@ -0,0 +1,49 @@
+package aws
+
+import (
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/client"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
+	"github.com/aws/aws-sdk-go/aws/session"
+)
+
+type CredentialConfig struct {
+	Region    string
+	AccessKey string
+	SecretKey string
+	RoleARN   string
+	Profile   string
+	Filename  string
+	Token     string
+}
+
+func (c *CredentialConfig) Credentials() client.ConfigProvider {
+	if c.RoleARN != "" {
+		return c.assumeCredentials()
+	} else {
+		return c.rootCredentials()
+	}
+}
+
+func (c *CredentialConfig) rootCredentials() client.ConfigProvider {
+	config := &aws.Config{
+		Region: aws.String(c.Region),
+	}
+	if c.AccessKey != "" || c.SecretKey != "" {
+		config.Credentials = credentials.NewStaticCredentials(c.AccessKey, c.SecretKey, c.Token)
+	} else if c.Profile != "" || c.Filename != "" {
+		config.Credentials = credentials.NewSharedCredentials(c.Filename, c.Profile)
+	}
+
+	return session.New(config)
+}
+
+func (c *CredentialConfig) assumeCredentials() client.ConfigProvider {
+	rootCredentials := c.rootCredentials()
+	config := &aws.Config{
+		Region: aws.String(c.Region),
+	}
+	config.Credentials = stscreds.NewCredentials(rootCredentials, c.RoleARN)
+	return session.New(config)
+}

plugins/inputs/cloudwatch/cloudwatch.go

@@ -6,25 +6,30 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
 
 	"github.com/aws/aws-sdk-go/service/cloudwatch"
 
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/internal"
+	internalaws "github.com/influxdata/telegraf/internal/config/aws"
 	"github.com/influxdata/telegraf/plugins/inputs"
 )
 
 type (
 	CloudWatch struct {
-		Region      string            `toml:"region"`
-		AccessKey   string            `toml:"access_key"`
-		SecretKey   string            `toml:"secret_key"`
+		Region    string `toml:"region"`
+		AccessKey string `toml:"access_key"`
+		SecretKey string `toml:"secret_key"`
+		RoleARN   string `toml:"role_arn"`
+		Profile   string `toml:"profile"`
+		Filename  string `toml:"shared_credential_file"`
+		Token     string `toml:"token"`
+
 		Period      internal.Duration `toml:"period"`
 		Delay       internal.Duration `toml:"delay"`
 		Namespace   string            `toml:"namespace"`
 		Metrics     []*Metric         `toml:"metrics"`
+		CacheTTL    internal.Duration `toml:"cache_ttl"`
 		client      cloudwatchClient
 		metricCache *MetricCache
 	}
@@ -58,12 +63,18 @@ func (c *CloudWatch) SampleConfig() string {
 
   ## Amazon Credentials
   ## Credentials are loaded in the following order
-  ## 1) explicit credentials from 'access_key' and 'secret_key'
-  ## 2) environment variables
-  ## 3) shared credentials file
-  ## 4) EC2 Instance Profile
+  ## 1) Assumed credentials via STS if role_arn is specified
+  ## 2) explicit credentials from 'access_key' and 'secret_key'
+  ## 3) shared profile from 'profile'
+  ## 4) environment variables
+  ## 5) shared credentials file
+  ## 6) EC2 Instance Profile
   #access_key = ""
   #secret_key = ""
+  #token = ""
+  #role_arn = ""
+  #profile = ""
+  #shared_credential_file = ""
 
   ## Requested CloudWatch aggregation Period (required - must be a multiple of 60s)
   period = '1m'
@@ -75,6 +86,10 @@ func (c *CloudWatch) SampleConfig() string {
   ## gaps or overlap in pulled data
   interval = '1m'
 
+  ## Configure the TTL for the internal cache of metrics.
+  ## Defaults to 1 hr if not specified
+  #cache_ttl = '10m'
+
   ## Metric Statistic Namespace (required)
   namespace = 'AWS/ELB'
 
@@ -106,20 +121,40 @@ func (c *CloudWatch) Gather(acc telegraf.Accumulator) error {
 	if c.Metrics != nil {
 		metrics = []*cloudwatch.Metric{}
 		for _, m := range c.Metrics {
-			dimensions := make([]*cloudwatch.Dimension, len(m.Dimensions))
-			for k, d := range m.Dimensions {
-				dimensions[k] = &cloudwatch.Dimension{
-					Name:  aws.String(d.Name),
-					Value: aws.String(d.Value),
+			if !hasWilcard(m.Dimensions) {
+				dimensions := make([]*cloudwatch.Dimension, len(m.Dimensions))
+				for k, d := range m.Dimensions {
+					fmt.Printf("Dimension [%s]:[%s]\n", d.Name, d.Value)
+					dimensions[k] = &cloudwatch.Dimension{
+						Name:  aws.String(d.Name),
+						Value: aws.String(d.Value),
+					}
+				}
+				for _, name := range m.MetricNames {
+					metrics = append(metrics, &cloudwatch.Metric{
+						Namespace:  aws.String(c.Namespace),
+						MetricName: aws.String(name),
+						Dimensions: dimensions,
+					})
+				}
+			} else {
+				allMetrics, err := c.fetchNamespaceMetrics()
+				if err != nil {
+					return err
+				}
+				for _, name := range m.MetricNames {
+					for _, metric := range allMetrics {
+						if isSelected(metric, m.Dimensions) {
+							metrics = append(metrics, &cloudwatch.Metric{
+								Namespace:  aws.String(c.Namespace),
+								MetricName: aws.String(name),
+								Dimensions: metric.Dimensions,
+							})
+						}
+					}
 				}
 			}
-			for _, name := range m.MetricNames {
-				metrics = append(metrics, &cloudwatch.Metric{
-					Namespace:  aws.String(c.Namespace),
-					MetricName: aws.String(name),
-					Dimensions: dimensions,
-				})
-			}
+
 		}
 	} else {
 		var err error
@@ -153,22 +188,29 @@ func (c *CloudWatch) Gather(acc telegraf.Accumulator) error {
 
 func init() {
 	inputs.Add("cloudwatch", func() telegraf.Input {
-		return &CloudWatch{}
+		ttl, _ := time.ParseDuration("1hr")
+		return &CloudWatch{
+			CacheTTL: internal.Duration{Duration: ttl},
+		}
 	})
 }
 
 /*
  * Initialize CloudWatch client
  */
 func (c *CloudWatch) initializeCloudWatch() error {
-	config := &aws.Config{
-		Region: aws.String(c.Region),
-	}
-	if c.AccessKey != "" || c.SecretKey != "" {
-		config.Credentials = credentials.NewStaticCredentials(c.AccessKey, c.SecretKey, "")
+	credentialConfig := &internalaws.CredentialConfig{
+		Region:    c.Region,
+		AccessKey: c.AccessKey,
+		SecretKey: c.SecretKey,
+		RoleARN:   c.RoleARN,
+		Profile:   c.Profile,
+		Filename:  c.Filename,
+		Token:     c.Token,
 	}
+	configProvider := credentialConfig.Credentials()
 
-	c.client = cloudwatch.New(session.New(config))
+	c.client = cloudwatch.New(configProvider)
 	return nil
 }
 
@@ -203,11 +245,10 @@ func (c *CloudWatch) fetchNamespaceMetrics() (metrics []*cloudwatch.Metric, err
 		more = token != nil
 	}
 
-	cacheTTL, _ := time.ParseDuration("1hr")
 	c.metricCache = &MetricCache{
 		Metrics: metrics,
 		Fetched: time.Now(),
-		TTL:     cacheTTL,
+		TTL:     c.CacheTTL.Duration,
 	}
 
 	return
@@ -309,3 +350,32 @@ func (c *CloudWatch) getStatisticsInput(metric *cloudwatch.Metric, now time.Time
 func (c *MetricCache) IsValid() bool {
 	return c.Metrics != nil && time.Since(c.Fetched) < c.TTL
 }
+
+func hasWilcard(dimensions []*Dimension) bool {
+	for _, d := range dimensions {
+		if d.Value == "" || d.Value == "*" {
+			return true
+		}
+	}
+	return false
+}
+
+func isSelected(metric *cloudwatch.Metric, dimensions []*Dimension) bool {
+	if len(metric.Dimensions) != len(dimensions) {
+		return false
+	}
+	for _, d := range dimensions {
+		selected := false
+		for _, d2 := range metric.Dimensions {
+			if d.Name == *d2.Name {
+				if d.Value == "" || d.Value == "*" || d.Value == *d2.Value {
+					selected = true
+				}
+			}
+		}
+		if !selected {
+			return false
+		}
+	}
+	return true
+}

plugins/outputs/cloudwatch/cloudwatch.go

@@ -8,19 +8,23 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/cloudwatch"
 
 	"github.com/influxdata/telegraf"
+	internalaws "github.com/influxdata/telegraf/internal/config/aws"
 	"github.com/influxdata/telegraf/plugins/outputs"
 )
 
 type CloudWatch struct {
-	Region    string `toml:"region"`     // AWS Region
-	AccessKey string `toml:"access_key"` // Explicit AWS Access Key ID
-	SecretKey string `toml:"secret_key"` // Explicit AWS Secret Access Key
-	Namespace string `toml:"namespace"`  // CloudWatch Metrics Namespace
+	Region    string `toml:"region"`
+	AccessKey string `toml:"access_key"`
+	SecretKey string `toml:"secret_key"`
+	RoleARN   string `toml:"role_arn"`
+	Profile   string `toml:"profile"`
+	Filename  string `toml:"shared_credential_file"`
+	Token     string `toml:"token"`
+
+	Namespace string `toml:"namespace"` // CloudWatch Metrics Namespace
 	svc       *cloudwatch.CloudWatch
 }
 
@@ -30,12 +34,18 @@ var sampleConfig = `
 
   ## Amazon Credentials
   ## Credentials are loaded in the following order
-  ## 1) explicit credentials from 'access_key' and 'secret_key'
-  ## 2) environment variables
-  ## 3) shared credentials file
-  ## 4) EC2 Instance Profile
+  ## 1) Assumed credentials via STS if role_arn is specified
+  ## 2) explicit credentials from 'access_key' and 'secret_key'
+  ## 3) shared profile from 'profile'
+  ## 4) environment variables
+  ## 5) shared credentials file
+  ## 6) EC2 Instance Profile
   #access_key = ""
   #secret_key = ""
+  #token = ""
+  #role_arn = ""
+  #profile = ""
+  #shared_credential_file = ""
 
   ## Namespace for the CloudWatch MetricDatums
   namespace = 'InfluxData/Telegraf'
@@ -50,14 +60,18 @@ func (c *CloudWatch) Description() string {
 }
 
 func (c *CloudWatch) Connect() error {
-	Config := &aws.Config{
-		Region: aws.String(c.Region),
-	}
-	if c.AccessKey != "" || c.SecretKey != "" {
-		Config.Credentials = credentials.NewStaticCredentials(c.AccessKey, c.SecretKey, "")
+	credentialConfig := &internalaws.CredentialConfig{
+		Region:    c.Region,
+		AccessKey: c.AccessKey,
+		SecretKey: c.SecretKey,
+		RoleARN:   c.RoleARN,
+		Profile:   c.Profile,
+		Filename:  c.Filename,
+		Token:     c.Token,
 	}
+	configProvider := credentialConfig.Credentials()
 
-	svc := cloudwatch.New(session.New(Config))
+	svc := cloudwatch.New(configProvider)
 
 	params := &cloudwatch.ListMetricsInput{
 		Namespace: aws.String(c.Namespace),