diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ec655970977..777282222730 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ * [9463](https://github.com/grafana/loki/pull/9463) **Totalus**: Fix OpenStack Swift client object listing to fetch all the objects properly. * [9471](https://github.com/grafana/loki/pull/9471) **sandeepsukhani**: query-scheduler: fix query distribution in SSD mode. * [9495](https://github.com/grafana/loki/pull/9495) **thampiotr**: Promtail: Fix potential goroutine leak in file tailer. +* [9650](https://github.com/grafana/loki/pull/9650) **ashwanthgoli**: Config: ensure storage config defaults apply to named stores. * [9629](https://github.com/grafana/loki/pull/9629) **periklis**: Fix duplicate label values from ingester streams. ##### Changes diff --git a/pkg/loki/config_wrapper_test.go b/pkg/loki/config_wrapper_test.go index 1bfde9eb72e0..20d5801d60b5 100644 --- a/pkg/loki/config_wrapper_test.go +++ b/pkg/loki/config_wrapper_test.go @@ -16,10 +16,14 @@ import ( "github.com/grafana/loki/pkg/distributor" "github.com/grafana/loki/pkg/loki/common" "github.com/grafana/loki/pkg/storage/bucket/swift" + "github.com/grafana/loki/pkg/storage/chunk/client/alibaba" "github.com/grafana/loki/pkg/storage/chunk/client/aws" "github.com/grafana/loki/pkg/storage/chunk/client/azure" "github.com/grafana/loki/pkg/storage/chunk/client/baidubce" "github.com/grafana/loki/pkg/storage/chunk/client/gcp" + "github.com/grafana/loki/pkg/storage/chunk/client/ibmcloud" + "github.com/grafana/loki/pkg/storage/chunk/client/local" + "github.com/grafana/loki/pkg/storage/chunk/client/openstack" "github.com/grafana/loki/pkg/storage/config" "github.com/grafana/loki/pkg/util" "github.com/grafana/loki/pkg/util/cfg" @@ -1702,3 +1706,136 @@ common: assert.Equal(t, []string{"ringsshouldusethis"}, config.CompactorConfig.CompactorRing.InstanceInterfaceNames) }) } + +func TestNamedStores_applyDefaults(t *testing.T) { + namedStoresConfig := `storage_config: + named_stores: + aws: + store-1: + s3: "s3.test" + storage_class: GLACIER + dynamodb: + dynamodb_url: "dynamo.test" + azure: + store-2: + environment: AzureGermanCloud + account_name: foo + container_name: bar + bos: + store-3: + bucket_name: foobar + gcs: + store-4: + bucket_name: foobar + enable_http2: false + cos: + store-5: + endpoint: cos.test + http_config: + idle_conn_timeout: 30s + filesystem: + store-6: + directory: foobar + swift: + store-7: + container_name: foobar + request_timeout: 30s + alibabacloud: + store-8: + bucket: foobar + endpoint: oss.test +` + // make goconst happy + bucketName := "foobar" + + config, defaults, err := configWrapperFromYAML(t, namedStoresConfig, nil) + require.NoError(t, err) + + nsCfg := config.StorageConfig.NamedStores + + t.Run("aws", func(t *testing.T) { + assert.Len(t, config.StorageConfig.NamedStores.AWS, 1) + + // expect the defaults to be set on named store config + expected := defaults.StorageConfig.AWSStorageConfig + assert.NoError(t, expected.DynamoDB.Set("dynamo.test")) + assert.NoError(t, expected.S3.Set("s3.test")) + // override defaults + expected.StorageClass = "GLACIER" + + assert.Equal(t, expected, (aws.StorageConfig)(nsCfg.AWS["store-1"])) + }) + + t.Run("azure", func(t *testing.T) { + assert.Len(t, config.StorageConfig.NamedStores.Azure, 1) + + expected := defaults.StorageConfig.AzureStorageConfig + expected.StorageAccountName = "foo" + expected.ContainerName = "bar" + // override defaults + expected.Environment = "AzureGermanCloud" + + assert.Equal(t, expected, (azure.BlobStorageConfig)(nsCfg.Azure["store-2"])) + }) + + t.Run("bos", func(t *testing.T) { + assert.Len(t, config.StorageConfig.NamedStores.BOS, 1) + + expected := defaults.StorageConfig.BOSStorageConfig + expected.BucketName = bucketName + + assert.Equal(t, expected, (baidubce.BOSStorageConfig)(nsCfg.BOS["store-3"])) + }) + + t.Run("gcs", func(t *testing.T) { + assert.Len(t, config.StorageConfig.NamedStores.GCS, 1) + + expected := defaults.StorageConfig.GCSConfig + expected.BucketName = bucketName + // override defaults + expected.EnableHTTP2 = false + + assert.Equal(t, expected, (gcp.GCSConfig)(nsCfg.GCS["store-4"])) + }) + + t.Run("cos", func(t *testing.T) { + assert.Len(t, config.StorageConfig.NamedStores.COS, 1) + + expected := defaults.StorageConfig.COSConfig + expected.Endpoint = "cos.test" + // override defaults + expected.HTTPConfig.IdleConnTimeout = 30 * time.Second + + assert.Equal(t, expected, (ibmcloud.COSConfig)(nsCfg.COS["store-5"])) + }) + + t.Run("filesystem", func(t *testing.T) { + assert.Len(t, config.StorageConfig.NamedStores.Filesystem, 1) + + expected := defaults.StorageConfig.FSConfig + expected.Directory = bucketName + + assert.Equal(t, expected, (local.FSConfig)(nsCfg.Filesystem["store-6"])) + }) + + t.Run("swift", func(t *testing.T) { + assert.Len(t, config.StorageConfig.NamedStores.Swift, 1) + + expected := defaults.StorageConfig.Swift + expected.ContainerName = bucketName + // override defaults + expected.RequestTimeout = 30 * time.Second + + assert.Equal(t, expected, (openstack.SwiftConfig)(nsCfg.Swift["store-7"])) + }) + + t.Run("alibabacloud", func(t *testing.T) { + assert.Len(t, config.StorageConfig.NamedStores.AlibabaCloud, 1) + + expected := defaults.StorageConfig.AlibabaStorageConfig + expected.Bucket = bucketName + expected.Endpoint = "oss.test" + + assert.Equal(t, expected, (alibaba.OssConfig)(nsCfg.AlibabaCloud["store-8"])) + }) +} diff --git a/pkg/storage/factory.go b/pkg/storage/factory.go index 803169cdbc1f..963df4dc7fe5 100644 --- a/pkg/storage/factory.go +++ b/pkg/storage/factory.go @@ -12,6 +12,8 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "github.com/grafana/dskit/flagext" + "github.com/grafana/loki/pkg/storage/chunk/cache" "github.com/grafana/loki/pkg/storage/chunk/client" "github.com/grafana/loki/pkg/storage/chunk/client/alibaba" @@ -64,16 +66,103 @@ type StoreLimits interface { CardinalityLimit(string) int } +// Storage configs defined as Named stores don't get any defaults as they do not +// register flags. To get around this we implement Unmarshaler interface that +// assigns the defaults before calling unmarshal. + +// We cannot implement Unmarshaler directly on aws.StorageConfig or other stores +// as it would end up overriding values set as part of ApplyDynamicConfig(). +// Note: we unmarshal a second time after applying dynamic configs +// +// Implementing the Unmarshaler for Named*StorageConfig types is fine as +// we do not apply any dynamic config on them. + +type NamedAWSStorageConfig aws.StorageConfig + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (cfg *NamedAWSStorageConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + flagext.DefaultValues((*aws.StorageConfig)(cfg)) + return unmarshal((*aws.StorageConfig)(cfg)) +} + +func (cfg *NamedAWSStorageConfig) Validate() error { + return (*aws.StorageConfig)(cfg).Validate() +} + +type NamedBlobStorageConfig azure.BlobStorageConfig + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (cfg *NamedBlobStorageConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + flagext.DefaultValues((*azure.BlobStorageConfig)(cfg)) + return unmarshal((*azure.BlobStorageConfig)(cfg)) +} + +func (cfg *NamedBlobStorageConfig) Validate() error { + return (*azure.BlobStorageConfig)(cfg).Validate() +} + +type NamedBOSStorageConfig baidubce.BOSStorageConfig + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (cfg *NamedBOSStorageConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + flagext.DefaultValues((*baidubce.BOSStorageConfig)(cfg)) + return unmarshal((*baidubce.BOSStorageConfig)(cfg)) +} + +type NamedFSConfig local.FSConfig + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (cfg *NamedFSConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + flagext.DefaultValues((*local.FSConfig)(cfg)) + return unmarshal((*local.FSConfig)(cfg)) +} + +type NamedGCSConfig gcp.GCSConfig + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (cfg *NamedGCSConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + flagext.DefaultValues((*gcp.GCSConfig)(cfg)) + return unmarshal((*gcp.GCSConfig)(cfg)) +} + +type NamedOssConfig alibaba.OssConfig + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (cfg *NamedOssConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + flagext.DefaultValues((*alibaba.OssConfig)(cfg)) + return unmarshal((*alibaba.OssConfig)(cfg)) +} + +type NamedSwiftConfig openstack.SwiftConfig + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (cfg *NamedSwiftConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + flagext.DefaultValues((*openstack.SwiftConfig)(cfg)) + return unmarshal((*openstack.SwiftConfig)(cfg)) +} + +func (cfg *NamedSwiftConfig) Validate() error { + return (*openstack.SwiftConfig)(cfg).Validate() +} + +type NamedCOSConfig ibmcloud.COSConfig + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (cfg *NamedCOSConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + flagext.DefaultValues((*ibmcloud.COSConfig)(cfg)) + return unmarshal((*ibmcloud.COSConfig)(cfg)) +} + // NamedStores helps configure additional object stores from a given storage provider type NamedStores struct { - AWS map[string]aws.StorageConfig `yaml:"aws"` - Azure map[string]azure.BlobStorageConfig `yaml:"azure"` - BOS map[string]baidubce.BOSStorageConfig `yaml:"bos"` - Filesystem map[string]local.FSConfig `yaml:"filesystem"` - GCS map[string]gcp.GCSConfig `yaml:"gcs"` - AlibabaCloud map[string]alibaba.OssConfig `yaml:"alibabacloud"` - Swift map[string]openstack.SwiftConfig `yaml:"swift"` - COS map[string]ibmcloud.COSConfig `yaml:"cos"` + AWS map[string]NamedAWSStorageConfig `yaml:"aws"` + Azure map[string]NamedBlobStorageConfig `yaml:"azure"` + BOS map[string]NamedBOSStorageConfig `yaml:"bos"` + Filesystem map[string]NamedFSConfig `yaml:"filesystem"` + GCS map[string]NamedGCSConfig `yaml:"gcs"` + AlibabaCloud map[string]NamedOssConfig `yaml:"alibabacloud"` + Swift map[string]NamedSwiftConfig `yaml:"swift"` + COS map[string]NamedCOSConfig `yaml:"cos"` // contains mapping from named store reference name to store type storeType map[string]string `yaml:"-"` @@ -508,43 +597,47 @@ func NewObjectClient(name string, cfg Config, clientMetrics ClientMetrics) (clie case config.StorageTypeAlibabaCloud: ossCfg := cfg.AlibabaStorageConfig if namedStore != "" { - var ok bool - ossCfg, ok = cfg.NamedStores.AlibabaCloud[namedStore] + nsCfg, ok := cfg.NamedStores.AlibabaCloud[namedStore] if !ok { return nil, fmt.Errorf("Unrecognized named alibabacloud oss storage config %s", name) } + + ossCfg = (alibaba.OssConfig)(nsCfg) } return alibaba.NewOssObjectClient(context.Background(), ossCfg) case config.StorageTypeGCS: gcsCfg := cfg.GCSConfig if namedStore != "" { - var ok bool - gcsCfg, ok = cfg.NamedStores.GCS[namedStore] + nsCfg, ok := cfg.NamedStores.GCS[namedStore] if !ok { return nil, fmt.Errorf("Unrecognized named gcs storage config %s", name) } + + gcsCfg = (gcp.GCSConfig)(nsCfg) } return gcp.NewGCSObjectClient(context.Background(), gcsCfg, cfg.Hedging) case config.StorageTypeAzure: azureCfg := cfg.AzureStorageConfig if namedStore != "" { - var ok bool - azureCfg, ok = cfg.NamedStores.Azure[namedStore] + nsCfg, ok := cfg.NamedStores.Azure[namedStore] if !ok { return nil, fmt.Errorf("Unrecognized named azure storage config %s", name) } + + azureCfg = (azure.BlobStorageConfig)(nsCfg) } return azure.NewBlobStorage(&azureCfg, clientMetrics.AzureMetrics, cfg.Hedging) case config.StorageTypeSwift: swiftCfg := cfg.Swift if namedStore != "" { - var ok bool - swiftCfg, ok = cfg.NamedStores.Swift[namedStore] + nsCfg, ok := cfg.NamedStores.Swift[namedStore] if !ok { return nil, fmt.Errorf("Unrecognized named swift storage config %s", name) } + + swiftCfg = (openstack.SwiftConfig)(nsCfg) } return openstack.NewSwiftObjectClient(swiftCfg, cfg.Hedging) @@ -553,22 +646,24 @@ func NewObjectClient(name string, cfg Config, clientMetrics ClientMetrics) (clie case config.StorageTypeFileSystem: fsCfg := cfg.FSConfig if namedStore != "" { - var ok bool - fsCfg, ok = cfg.NamedStores.Filesystem[namedStore] + nsCfg, ok := cfg.NamedStores.Filesystem[namedStore] if !ok { return nil, fmt.Errorf("Unrecognized named filesystem storage config %s", name) } + + fsCfg = (local.FSConfig)(nsCfg) } return local.NewFSObjectClient(fsCfg) case config.StorageTypeBOS: bosCfg := cfg.BOSStorageConfig if namedStore != "" { - var ok bool - bosCfg, ok = cfg.NamedStores.BOS[namedStore] + nsCfg, ok := cfg.NamedStores.BOS[namedStore] if !ok { return nil, fmt.Errorf("Unrecognized named bos storage config %s", name) } + + bosCfg = (baidubce.BOSStorageConfig)(nsCfg) } return baidubce.NewBOSObjectStorage(&bosCfg) @@ -576,11 +671,12 @@ func NewObjectClient(name string, cfg Config, clientMetrics ClientMetrics) (clie case config.StorageTypeCOS: cosCfg := cfg.COSConfig if namedStore != "" { - var ok bool - cosCfg, ok = cfg.NamedStores.COS[namedStore] + nsCfg, ok := cfg.NamedStores.COS[namedStore] if !ok { return nil, fmt.Errorf("Unrecognized named cos storage config %s", name) } + + cosCfg = (ibmcloud.COSConfig)(nsCfg) } return ibmcloud.NewCOSObjectClient(cosCfg, cfg.Hedging) default: diff --git a/pkg/storage/factory_test.go b/pkg/storage/factory_test.go index 87b585eda44c..ac71fb1c15af 100644 --- a/pkg/storage/factory_test.go +++ b/pkg/storage/factory_test.go @@ -12,9 +12,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/grafana/loki/pkg/storage/chunk/client/aws" "github.com/grafana/loki/pkg/storage/chunk/client/cassandra" - "github.com/grafana/loki/pkg/storage/chunk/client/gcp" "github.com/grafana/loki/pkg/storage/chunk/client/local" "github.com/grafana/loki/pkg/storage/config" "github.com/grafana/loki/pkg/storage/stores/indexshipper" @@ -105,7 +103,7 @@ func TestNamedStores(t *testing.T) { cfg := Config{ NamedStores: NamedStores{ - Filesystem: map[string]local.FSConfig{ + Filesystem: map[string]NamedFSConfig{ "named-store": {Directory: path.Join(tempDir, "named-store")}, }, }, @@ -171,11 +169,11 @@ func TestNamedStores(t *testing.T) { func TestNamedStores_populateStoreType(t *testing.T) { t.Run("found duplicates", func(t *testing.T) { ns := NamedStores{ - AWS: map[string]aws.StorageConfig{ + AWS: map[string]NamedAWSStorageConfig{ "store-1": {}, "store-2": {}, }, - GCS: map[string]gcp.GCSConfig{ + GCS: map[string]NamedGCSConfig{ "store-1": {}, }, } @@ -187,7 +185,7 @@ func TestNamedStores_populateStoreType(t *testing.T) { t.Run("illegal store name", func(t *testing.T) { ns := NamedStores{ - GCS: map[string]gcp.GCSConfig{ + GCS: map[string]NamedGCSConfig{ "aws": {}, }, } @@ -199,11 +197,11 @@ func TestNamedStores_populateStoreType(t *testing.T) { t.Run("lookup populated entries", func(t *testing.T) { ns := NamedStores{ - AWS: map[string]aws.StorageConfig{ + AWS: map[string]NamedAWSStorageConfig{ "store-1": {}, "store-2": {}, }, - GCS: map[string]gcp.GCSConfig{ + GCS: map[string]NamedGCSConfig{ "store-3": {}, }, } diff --git a/pkg/storage/store_test.go b/pkg/storage/store_test.go index 29c7e617eb24..59b4a79548e5 100644 --- a/pkg/storage/store_test.go +++ b/pkg/storage/store_test.go @@ -1022,7 +1022,7 @@ func TestStore_MultiPeriod(t *testing.T) { }, TSDBShipperConfig: shipperConfig, NamedStores: NamedStores{ - Filesystem: map[string]local.FSConfig{ + Filesystem: map[string]NamedFSConfig{ "named-store": {Directory: path.Join(tempDir, "named-store")}, }, }, diff --git a/tools/doc-generator/parse/parser.go b/tools/doc-generator/parse/parser.go index 09d1fa42e728..192ce1ea2f02 100644 --- a/tools/doc-generator/parse/parser.go +++ b/tools/doc-generator/parse/parser.go @@ -97,9 +97,11 @@ func (e ConfigEntry) Description() string { } type RootBlock struct { - Name string - Desc string - StructType reflect.Type + Name string + Desc string + // multiple entries are useful if the root blocks share the same + // underlying type + StructType []reflect.Type } func Flags(cfg flagext.Registerer) map[uintptr]*flag.Flag { @@ -629,8 +631,10 @@ func getFieldDescription(cfg interface{}, field reflect.StructField, fallback st func isRootBlock(t reflect.Type, rootBlocks []RootBlock) (string, string, bool) { for _, rootBlock := range rootBlocks { - if t == rootBlock.StructType { - return rootBlock.Name, rootBlock.Desc, true + for _, structType := range rootBlock.StructType { + if t == structType { + return rootBlock.Name, rootBlock.Desc, true + } } } diff --git a/tools/doc-generator/parse/root_blocks.go b/tools/doc-generator/parse/root_blocks.go index d892e1f8d64c..61e227c89d07 100644 --- a/tools/doc-generator/parse/root_blocks.go +++ b/tools/doc-generator/parse/root_blocks.go @@ -49,109 +49,109 @@ var ( RootBlocks = []RootBlock{ { Name: "server", - StructType: reflect.TypeOf(server.Config{}), + StructType: []reflect.Type{reflect.TypeOf(server.Config{})}, Desc: "Configures the server of the launched module(s).", }, { Name: "distributor", - StructType: reflect.TypeOf(distributor.Config{}), + StructType: []reflect.Type{reflect.TypeOf(distributor.Config{})}, Desc: "Configures the distributor.", }, { Name: "querier", - StructType: reflect.TypeOf(querier.Config{}), + StructType: []reflect.Type{reflect.TypeOf(querier.Config{})}, Desc: "Configures the querier. Only appropriate when running all modules or just the querier.", }, { Name: "query_scheduler", - StructType: reflect.TypeOf(scheduler.Config{}), + StructType: []reflect.Type{reflect.TypeOf(scheduler.Config{})}, Desc: "The query_scheduler block configures the Loki query scheduler. When configured it separates the tenant query queues from the query-frontend.", }, { Name: "frontend", - StructType: reflect.TypeOf(frontend.Config{}), + StructType: []reflect.Type{reflect.TypeOf(frontend.Config{})}, Desc: "The frontend block configures the Loki query-frontend.", }, { Name: "query_range", - StructType: reflect.TypeOf(queryrange.Config{}), + StructType: []reflect.Type{reflect.TypeOf(queryrange.Config{})}, Desc: "The query_range block configures the query splitting and caching in the Loki query-frontend.", }, { Name: "ruler", - StructType: reflect.TypeOf(ruler.Config{}), + StructType: []reflect.Type{reflect.TypeOf(ruler.Config{})}, Desc: "The ruler block configures the Loki ruler.", }, { Name: "ingester_client", - StructType: reflect.TypeOf(ingester_client.Config{}), + StructType: []reflect.Type{reflect.TypeOf(ingester_client.Config{})}, Desc: "The ingester_client block configures how the distributor will connect to ingesters. Only appropriate when running all components, the distributor, or the querier.", }, { Name: "ingester", - StructType: reflect.TypeOf(ingester.Config{}), + StructType: []reflect.Type{reflect.TypeOf(ingester.Config{})}, Desc: "The ingester block configures the ingester and how the ingester will register itself to a key value store.", }, { Name: "index_gateway", - StructType: reflect.TypeOf(indexgateway.Config{}), + StructType: []reflect.Type{reflect.TypeOf(indexgateway.Config{})}, Desc: "The index_gateway block configures the Loki index gateway server, responsible for serving index queries without the need to constantly interact with the object store.", }, { Name: "storage_config", - StructType: reflect.TypeOf(storage.Config{}), + StructType: []reflect.Type{reflect.TypeOf(storage.Config{})}, Desc: "The storage_config block configures one of many possible stores for both the index and chunks. Which configuration to be picked should be defined in schema_config block.", }, { Name: "chunk_store_config", - StructType: reflect.TypeOf(storage_config.ChunkStoreConfig{}), + StructType: []reflect.Type{reflect.TypeOf(storage_config.ChunkStoreConfig{})}, Desc: "The chunk_store_config block configures how chunks will be cached and how long to wait before saving them to the backing store.", }, { Name: "schema_config", - StructType: reflect.TypeOf(storage_config.SchemaConfig{}), + StructType: []reflect.Type{reflect.TypeOf(storage_config.SchemaConfig{})}, Desc: "Configures the chunk index schema and where it is stored.", }, { Name: "compactor", - StructType: reflect.TypeOf(compactor.Config{}), + StructType: []reflect.Type{reflect.TypeOf(compactor.Config{})}, Desc: "The compactor block configures the compactor component, which compacts index shards for performance.", }, { Name: "limits_config", - StructType: reflect.TypeOf(validation.Limits{}), + StructType: []reflect.Type{reflect.TypeOf(validation.Limits{})}, Desc: "The limits_config block configures global and per-tenant limits in Loki.", }, { Name: "frontend_worker", - StructType: reflect.TypeOf(querier_worker.Config{}), + StructType: []reflect.Type{reflect.TypeOf(querier_worker.Config{})}, Desc: "The frontend_worker configures the worker - running within the Loki querier - picking up and executing queries enqueued by the query-frontend.", }, { Name: "table_manager", - StructType: reflect.TypeOf(index.TableManagerConfig{}), + StructType: []reflect.Type{reflect.TypeOf(index.TableManagerConfig{})}, Desc: "The table_manager block configures the table manager for retention.", }, { Name: "runtime_config", - StructType: reflect.TypeOf(runtimeconfig.Config{}), + StructType: []reflect.Type{reflect.TypeOf(runtimeconfig.Config{})}, Desc: "Configuration for 'runtime config' module, responsible for reloading runtime configuration file.", }, { Name: "tracing", - StructType: reflect.TypeOf(tracing.Config{}), + StructType: []reflect.Type{reflect.TypeOf(tracing.Config{})}, Desc: "Configuration for tracing.", }, { Name: "analytics", - StructType: reflect.TypeOf(analytics.Config{}), + StructType: []reflect.Type{reflect.TypeOf(analytics.Config{})}, Desc: "Configuration for analytics.", }, { Name: "common", - StructType: reflect.TypeOf(common.Config{}), + StructType: []reflect.Type{reflect.TypeOf(common.Config{})}, Desc: "Common configuration to be shared between multiple modules. If a more specific configuration is given in other sections, the related configuration within this section will be ignored.", }, @@ -159,17 +159,17 @@ var ( // StoreConfig dskit type: https://github.com/grafana/dskit/blob/main/kv/client.go#L44-L52 { Name: "consul", - StructType: reflect.TypeOf(consul.Config{}), + StructType: []reflect.Type{reflect.TypeOf(consul.Config{})}, Desc: "Configuration for a Consul client. Only applies if the selected kvstore is consul.", }, { Name: "etcd", - StructType: reflect.TypeOf(etcd.Config{}), + StructType: []reflect.Type{reflect.TypeOf(etcd.Config{})}, Desc: "Configuration for an ETCD v3 client. Only applies if the selected kvstore is etcd.", }, { Name: "memberlist", - StructType: reflect.TypeOf(memberlist.KVConfig{}), + StructType: []reflect.Type{reflect.TypeOf(memberlist.KVConfig{})}, Desc: `Configuration for memberlist client. Only applies if the selected kvstore is memberlist. When a memberlist config with atleast 1 join_members is defined, kvstore of type memberlist is automatically selected for all the components that require a ring unless otherwise specified in the component's configuration section.`, @@ -177,77 +177,80 @@ When a memberlist config with atleast 1 join_members is defined, kvstore of type // GRPC client { Name: "grpc_client", - StructType: reflect.TypeOf(grpcclient.Config{}), + StructType: []reflect.Type{reflect.TypeOf(grpcclient.Config{})}, Desc: "The grpc_client block configures the gRPC client used to communicate between two Loki components.", }, // TLS config { Name: "tls_config", - StructType: reflect.TypeOf(tls.ClientConfig{}), + StructType: []reflect.Type{reflect.TypeOf(tls.ClientConfig{})}, Desc: "The TLS configuration.", }, // Cache config { Name: "cache_config", - StructType: reflect.TypeOf(cache.Config{}), + StructType: []reflect.Type{reflect.TypeOf(cache.Config{})}, Desc: "The cache block configures the cache backend.", }, // Schema periodic config { Name: "period_config", - StructType: reflect.TypeOf(storage_config.PeriodConfig{}), + StructType: []reflect.Type{reflect.TypeOf(storage_config.PeriodConfig{})}, Desc: "The period_config block configures what index schemas should be used for from specific time periods.", }, // Storage config { - Name: "aws_storage_config", - StructType: reflect.TypeOf(aws.StorageConfig{}), + Name: "aws_storage_config", + // aws.StorageConfig is the underlying type for storage.NamedAWSStorageConfig + // having these as separate block entries would result in duplicate storage configs + // similar reasoning applies to other storage configs listed below + StructType: []reflect.Type{reflect.TypeOf(aws.StorageConfig{}), reflect.TypeOf(storage.NamedAWSStorageConfig{})}, Desc: "The aws_storage_config block configures the connection to dynamoDB and S3 object storage. Either one of them or both can be configured.", }, { Name: "azure_storage_config", - StructType: reflect.TypeOf(azure.BlobStorageConfig{}), + StructType: []reflect.Type{reflect.TypeOf(azure.BlobStorageConfig{}), reflect.TypeOf(storage.NamedBlobStorageConfig{})}, Desc: "The azure_storage_config block configures the connection to Azure object storage backend.", }, { Name: "alibabacloud_storage_config", - StructType: reflect.TypeOf(alibaba.OssConfig{}), + StructType: []reflect.Type{reflect.TypeOf(alibaba.OssConfig{}), reflect.TypeOf(storage.NamedOssConfig{})}, Desc: "The alibabacloud_storage_config block configures the connection to Alibaba Cloud Storage object storage backend.", }, { Name: "gcs_storage_config", - StructType: reflect.TypeOf(gcp.GCSConfig{}), + StructType: []reflect.Type{reflect.TypeOf(gcp.GCSConfig{}), reflect.TypeOf(storage.NamedGCSConfig{})}, Desc: "The gcs_storage_config block configures the connection to Google Cloud Storage object storage backend.", }, { Name: "s3_storage_config", - StructType: reflect.TypeOf(aws.S3Config{}), + StructType: []reflect.Type{reflect.TypeOf(aws.S3Config{})}, Desc: "The s3_storage_config block configures the connection to Amazon S3 object storage backend.", }, { Name: "bos_storage_config", - StructType: reflect.TypeOf(baidubce.BOSStorageConfig{}), + StructType: []reflect.Type{reflect.TypeOf(baidubce.BOSStorageConfig{}), reflect.TypeOf(storage.NamedBOSStorageConfig{})}, Desc: "The bos_storage_config block configures the connection to Baidu Object Storage (BOS) object storage backend.", }, { Name: "swift_storage_config", - StructType: reflect.TypeOf(openstack.SwiftConfig{}), + StructType: []reflect.Type{reflect.TypeOf(openstack.SwiftConfig{}), reflect.TypeOf(storage.NamedSwiftConfig{})}, Desc: "The swift_storage_config block configures the connection to OpenStack Object Storage (Swift) object storage backend.", }, { Name: "cos_storage_config", - StructType: reflect.TypeOf(ibmcloud.COSConfig{}), + StructType: []reflect.Type{reflect.TypeOf(ibmcloud.COSConfig{}), reflect.TypeOf(storage.NamedCOSConfig{})}, Desc: "The cos_storage_config block configures the connection to IBM Cloud Object Storage (COS) backend.", }, { Name: "local_storage_config", - StructType: reflect.TypeOf(local.FSConfig{}), + StructType: []reflect.Type{reflect.TypeOf(local.FSConfig{}), reflect.TypeOf(storage.NamedFSConfig{})}, Desc: "The local_storage_config block configures the usage of local file system as object storage backend.", }, { Name: "named_stores_config", - StructType: reflect.TypeOf(storage.NamedStores{}), + StructType: []reflect.Type{reflect.TypeOf(storage.NamedStores{})}, Desc: `Configures additional object stores for a given storage provider. Supported stores: aws, azure, bos, filesystem, gcs, swift. Example: