Elasticsearch indices data source (#686)

* Elasticsearch indices data source

* Elasticseach indices data source test fix

* Refactoring from new index resource with terraform plugin framework

* Keeping documentation name and description

---------

Co-authored-by: Toby Brain <toby.brain@elastic.co>

docs/data-sources/elasticsearch_indices.md

@@ -0,0 +1,135 @@
+---
+subcategory: "Index"
+layout: ""
+page_title: "Elasticstack: elasticstack_elasticsearch_indices Data Source"
+description: |-
+  Retrieves indices.
+---
+
+# Data Source: elasticstack_elasticsearch_indices
+
+Use this data source to retrieve and get information about existing Elasticsearch indices. See, https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-get-index.html
+
+## Example Usage
+
+```terraform
+provider "elasticstack" {
+  elasticsearch {}
+  kibana {}
+}
+
+data "elasticstack_elasticsearch_indices" "logs" {
+  search = "log*"
+}
+```
+
+<!-- schema generated by tfplugindocs -->
+## Schema
+
+### Optional
+
+- `target` (String) Comma-separated list of data streams, indices, and aliases used to limit the request. Supports wildcards (*). To target all data streams and indices, omit this attribute or use * or _all.
+
+### Read-Only
+
+- `id` (String) Generated ID for the indices.
+- `indices` (Attributes List) The list of indices. (see [below for nested schema](#nestedatt--indices))
+
+<a id="nestedatt--indices"></a>
+### Nested Schema for `indices`
+
+Required:
+
+- `name` (String) Name of the index.
+
+Optional:
+
+- `alias` (Attributes Set) Aliases for the index. (see [below for nested schema](#nestedatt--indices--alias))
+- `analysis_analyzer` (String) A JSON string describing the analyzers applied to the index.
+- `analysis_char_filter` (String) A JSON string describing the char_filters applied to the index.
+- `analysis_filter` (String) A JSON string describing the filters applied to the index.
+- `analysis_normalizer` (String) A JSON string describing the normalizers applied to the index.
+- `analysis_tokenizer` (String) A JSON string describing the tokenizers applied to the index.
+- `analyze_max_token_count` (Number) The maximum number of tokens that can be produced using _analyze API.
+- `auto_expand_replicas` (String) Set the number of replicas to the node count in the cluster. Set to a dash delimited lower and upper bound (e.g. 0-5) or use all for the upper bound (e.g. 0-all)
+- `blocks_metadata` (Boolean) Set to `true` to disable index metadata reads and writes.
+- `blocks_read` (Boolean) Set to `true` to disable read operations against the index.
+- `blocks_read_only` (Boolean) Set to `true` to make the index and index metadata read only, `false` to allow writes and metadata changes.
+- `blocks_read_only_allow_delete` (Boolean) Identical to `index.blocks.read_only` but allows deleting the index to free up resources.
+- `blocks_write` (Boolean) Set to `true` to disable data write operations against the index. This setting does not affect metadata.
+- `codec` (String) The `default` value compresses stored data with LZ4 compression, but this can be set to `best_compression` which uses DEFLATE for a higher compression ratio. This can be set only on creation.
+- `default_pipeline` (String) The default ingest node pipeline for this index. Index requests will fail if the default pipeline is set and the pipeline does not exist.
+- `deletion_protection` (Boolean) Whether to allow Terraform to destroy the index. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply command that deletes the instance will fail.
+- `final_pipeline` (String) Final ingest pipeline for the index. Indexing requests will fail if the final pipeline is set and the pipeline does not exist. The final pipeline always runs after the request pipeline (if specified) and the default pipeline (if it exists). The special pipeline name _none indicates no ingest pipeline will run.
+- `gc_deletes` (String) The length of time that a deleted document's version number remains available for further versioned operations.
+- `highlight_max_analyzed_offset` (Number) The maximum number of characters that will be analyzed for a highlight request.
+- `include_type_name` (Boolean) If true, a mapping type is expected in the body of mappings. Defaults to false. Supported for Elasticsearch 7.x.
+- `indexing_slowlog_level` (String) Set which logging level to use for the search slow log, can be: `warn`, `info`, `debug`, `trace`
+- `indexing_slowlog_source` (String) Set the number of characters of the `_source` to include in the slowlog lines, `false` or `0` will skip logging the source entirely and setting it to `true` will log the entire source regardless of size. The original `_source` is reformatted by default to make sure that it fits on a single log line.
+- `indexing_slowlog_threshold_index_debug` (String) Set the cutoff for shard level slow search logging of slow searches for indexing queries, in time units, e.g. `2s`
+- `indexing_slowlog_threshold_index_info` (String) Set the cutoff for shard level slow search logging of slow searches for indexing queries, in time units, e.g. `5s`
+- `indexing_slowlog_threshold_index_trace` (String) Set the cutoff for shard level slow search logging of slow searches for indexing queries, in time units, e.g. `500ms`
+- `indexing_slowlog_threshold_index_warn` (String) Set the cutoff for shard level slow search logging of slow searches for indexing queries, in time units, e.g. `10s`
+- `load_fixed_bitset_filters_eagerly` (Boolean) Indicates whether cached filters are pre-loaded for nested queries. This can be set only on creation.
+- `mapping_coerce` (Boolean) Set index level coercion setting that is applied to all mapping types.
+- `mappings` (String) Mapping for fields in the index.
+							If specified, this mapping can include: field names, [field data types](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html), [mapping parameters](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-params.html).
+							**NOTE:**
+							- Changing datatypes in the existing _mappings_ will force index to be re-created.
+							- Removing field will be ignored by default same as elasticsearch. You need to recreate the index to remove field completely.
+- `master_timeout` (String) Period to wait for a connection to the master node. If no response is received before the timeout expires, the request fails and returns an error. Defaults to `30s`. This value is ignored when running against Serverless projects.
+- `max_docvalue_fields_search` (Number) The maximum number of `docvalue_fields` that are allowed in a query.
+- `max_inner_result_window` (Number) The maximum value of `from + size` for inner hits definition and top hits aggregations to this index.
+- `max_ngram_diff` (Number) The maximum allowed difference between min_gram and max_gram for NGramTokenizer and NGramTokenFilter.
+- `max_refresh_listeners` (Number) Maximum number of refresh listeners available on each shard of the index.
+- `max_regex_length` (Number) The maximum length of regex that can be used in Regexp Query.
+- `max_rescore_window` (Number) The maximum value of `window_size` for `rescore` requests in searches of this index.
+- `max_result_window` (Number) The maximum value of `from + size` for searches to this index.
+- `max_script_fields` (Number) The maximum number of `script_fields` that are allowed in a query.
+- `max_shingle_diff` (Number) The maximum allowed difference between max_shingle_size and min_shingle_size for ShingleTokenFilter.
+- `max_terms_count` (Number) The maximum number of terms that can be used in Terms Query.
+- `number_of_replicas` (Number) Number of shard replicas.
+- `number_of_routing_shards` (Number) Value used with number_of_shards to route documents to a primary shard. This can be set only on creation.
+- `number_of_shards` (Number) Number of shards for the index. This can be set only on creation.
+- `query_default_field` (Set of String) Wildcard (*) patterns matching one or more fields. Defaults to '*', which matches all fields eligible for term-level queries, excluding metadata fields.
+- `refresh_interval` (String) How often to perform a refresh operation, which makes recent changes to the index visible to search. Can be set to `-1` to disable refresh.
+- `routing_allocation_enable` (String) Controls shard allocation for this index. It can be set to: `all` , `primaries` , `new_primaries` , `none`.
+- `routing_partition_size` (Number) The number of shards a custom routing value can go to. This can be set only on creation.
+- `routing_rebalance_enable` (String) Enables shard rebalancing for this index. It can be set to: `all`, `primaries` , `replicas` , `none`.
+- `search_idle_after` (String) How long a shard can not receive a search or get request until it’s considered search idle.
+- `search_slowlog_level` (String) Set which logging level to use for the search slow log, can be: `warn`, `info`, `debug`, `trace`
+- `search_slowlog_threshold_fetch_debug` (String) Set the cutoff for shard level slow search logging of slow searches in the fetch phase, in time units, e.g. `2s`
+- `search_slowlog_threshold_fetch_info` (String) Set the cutoff for shard level slow search logging of slow searches in the fetch phase, in time units, e.g. `5s`
+- `search_slowlog_threshold_fetch_trace` (String) Set the cutoff for shard level slow search logging of slow searches in the fetch phase, in time units, e.g. `500ms`
+- `search_slowlog_threshold_fetch_warn` (String) Set the cutoff for shard level slow search logging of slow searches in the fetch phase, in time units, e.g. `10s`
+- `search_slowlog_threshold_query_debug` (String) Set the cutoff for shard level slow search logging of slow searches in the query phase, in time units, e.g. `2s`
+- `search_slowlog_threshold_query_info` (String) Set the cutoff for shard level slow search logging of slow searches in the query phase, in time units, e.g. `5s`
+- `search_slowlog_threshold_query_trace` (String) Set the cutoff for shard level slow search logging of slow searches in the query phase, in time units, e.g. `500ms`
+- `search_slowlog_threshold_query_warn` (String) Set the cutoff for shard level slow search logging of slow searches in the query phase, in time units, e.g. `10s`
+- `shard_check_on_startup` (String) Whether or not shards should be checked for corruption before opening. When corruption is detected, it will prevent the shard from being opened. Accepts `false`, `true`, `checksum`.
+- `sort_field` (Set of String) The field to sort shards in this index by.
+- `sort_order` (List of String) The direction to sort shards in. Accepts `asc`, `desc`.
+- `timeout` (String) Period to wait for a response. If no response is received before the timeout expires, the request fails and returns an error. Defaults to `30s`.
+- `unassigned_node_left_delayed_timeout` (String) Time to delay the allocation of replica shards which become unassigned because a node has left, in time units, e.g. `10s`
+- `wait_for_active_shards` (String) The number of shard copies that must be active before proceeding with the operation. Set to `all` or any positive integer up to the total number of shards in the index (number_of_replicas+1). Default: `1`, the primary shard. This value is ignored when running against Serverless projects.
+
+Read-Only:
+
+- `id` (String) Internal identifier of the resource.
+- `settings_raw` (String) All raw settings fetched from the cluster.
+
+<a id="nestedatt--indices--alias"></a>
+### Nested Schema for `indices.alias`
+
+Required:
+
+- `name` (String) Index alias name.
+
+Optional:
+
+- `filter` (String) Query used to limit documents the alias can access.
+- `index_routing` (String) Value used to route indexing operations to a specific shard. If specified, this overwrites the `routing` value for indexing operations.
+- `is_hidden` (Boolean) If true, the alias is hidden.
+- `is_write_index` (Boolean) If true, the index is the write index for the alias.
+- `routing` (String) Value used to route indexing and search operations to a specific shard.
+- `search_routing` (String) Value used to route search operations to a specific shard. If specified, this overwrites the routing value for search operations.

examples/data-sources/elasticstack_elasticsearch_indices/data-source.tf

@@ -0,0 +1,8 @@
+provider "elasticstack" {
+  elasticsearch {}
+  kibana {}
+}
+
+data "elasticstack_elasticsearch_indices" "logs" {
+  search = "log*"
+}

internal/clients/elasticsearch/index.go

@@ -304,6 +304,16 @@ func DeleteIndex(ctx context.Context, apiClient *clients.ApiClient, name string)
 }
 
 func GetIndex(ctx context.Context, apiClient *clients.ApiClient, name string) (*models.Index, fwdiags.Diagnostics) {
+	indices, diags := GetIndices(ctx, apiClient, name)
+	if diags.HasError() {
+		return nil, diags
+	}
+
+	index := indices[name]
+	return &index, diags
+}
+
+func GetIndices(ctx context.Context, apiClient *clients.ApiClient, name string) (map[string]models.Index, fwdiags.Diagnostics) {
 	esClient, err := apiClient.GetESClient()
 	if err != nil {
 		return nil, fwdiags.Diagnostics{
@@ -333,8 +343,8 @@ func GetIndex(ctx context.Context, apiClient *clients.ApiClient, name string) (*
 			fwdiags.NewErrorDiagnostic(err.Error(), err.Error()),
 		}
 	}
-	index := indices[name]
-	return &index, nil
+
+	return indices, nil
 }
 
 func DeleteIndexAlias(ctx context.Context, apiClient *clients.ApiClient, index string, aliases []string) fwdiags.Diagnostics {

internal/elasticsearch/index/indices/data_source.go

@@ -0,0 +1,46 @@
+package indices
+
+import (
+	"context"
+
+	"github.com/elastic/terraform-provider-elasticstack/internal/clients"
+	"github.com/hashicorp/terraform-plugin-framework/datasource"
+)
+
+// Ensure the implementation satisfies the expected interfaces.
+var (
+	_ datasource.DataSource              = &dataSource{}
+	_ datasource.DataSourceWithConfigure = &dataSource{}
+)
+
+// NewDataSource is a helper function to simplify the provider implementation.
+func NewDataSource() datasource.DataSource {
+	return &dataSource{}
+}
+
+// dataSource is the data source implementation.
+type dataSource struct {
+	client clients.ApiClient
+}
+
+// Metadata returns the data source type name.
+func (d *dataSource) Metadata(_ context.Context, req datasource.MetadataRequest, resp *datasource.MetadataResponse) {
+	resp.TypeName = req.ProviderTypeName + "_elasticsearch_indices"
+}
+
+// Configure adds the provider configured client to the data source.
+func (d *dataSource) Configure(_ context.Context, req datasource.ConfigureRequest, resp *datasource.ConfigureResponse) {
+	// Add a nil check when handling ProviderData because Terraform
+	// sets that data after it calls the ConfigureProvider RPC.
+	if req.ProviderData == nil {
+		return
+	}
+
+	apiClient, diags := clients.ConvertProviderData(req.ProviderData)
+	resp.Diagnostics.Append(diags...)
+	if resp.Diagnostics.HasError() {
+		return
+	}
+
+	d.client = *apiClient
+}

internal/elasticsearch/index/indices/data_source_test.go

@@ -0,0 +1,36 @@
+package indices_test
+
+import (
+	"testing"
+
+	"github.com/elastic/terraform-provider-elasticstack/internal/acctest"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
+)
+
+func TestAccIndicesDataSource(t *testing.T) {
+	resource.Test(t, resource.TestCase{
+		PreCheck:                 func() { acctest.PreCheck(t) },
+		ProtoV6ProviderFactories: acctest.Providers,
+		Steps: []resource.TestStep{
+			{
+				Config: testAccIndicesDataSourceConfig,
+				Check: resource.ComposeAggregateTestCheckFunc(
+					resource.TestCheckResourceAttr("data.elasticstack_elasticsearch_indices.security_indices", "indices.0.name", ".security-7"),
+					resource.TestCheckResourceAttr("data.elasticstack_elasticsearch_indices.security_indices", "indices.0.number_of_shards", "1"),
+					resource.TestCheckResourceAttr("data.elasticstack_elasticsearch_indices.security_indices", "indices.0.alias.0.name", ".security"),
+				),
+			},
+		},
+	})
+}
+
+const testAccIndicesDataSourceConfig = `
+provider "elasticstack" {
+	elasticsearch {}
+	kibana {}
+}
+
+data "elasticstack_elasticsearch_indices" "security_indices" {
+	target = ".security-*"
+}
+`