Update to v8.0.0 (#544)

List of changes impacting docker-elk:

- [logstash]: The output to Elasticsearch is handled as a data stream.

  Starting with v8.0.0, the `elasticsearch` output for Logstash sends
  log data to a data stream instead of `logstash-*` indices by default.
  The name of the default data stream is `logs-generic-default`.
  docker-elk remains unopinionated and simply uses Elastic's defaults
  like it always has, so users who prefer to retain the old behaviour
  need to explicitly opt-out of data streams in their Logstash
  pipelines.

  Refs:
  - https://www.elastic.co/guide/en/elasticsearch/reference/current/data-streams.html
  - https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams

- [logstash]: The (legacy) monitoring data collection is now disabled.

  This feature was deprecated since v7.9.0, and removed in v8.0.0.

  Ref: https://www.elastic.co/guide/en/logstash/current/monitoring-internal-collection-legacy.html

- [kibana]: An index pattern for `logs-*` indices is automatically
  created.

  It used to be required to manually create an index pattern for indices
  managed by Logstash, even when using the default Logstash indices.
  This is no longer the case since the output data is now being handled
  as a data stream, and Kibana automatically creates index patterns for
  these.

- [elasticsearch]: The command line tool `elasticsearch-setup-passwords`
  was deprecated in favour of a new `elasticsearch-reset-password` tool.

  Passwords for built-in users must now be generated one by one.

  Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-passwords.html

- [enterprise-search]: Kibana is now the new management interface, and
  the only one available moving forward.

  The old standalone Enterprise Search interface was removed in v8.0.0.

  Ref: https://www.elastic.co/guide/en/enterprise-search/current/user-interfaces.html

.env

@@ -1 +1 @@
-ELK_VERSION=7.17.0
+ELK_VERSION=8.0.0

.github/workflows/scripts/elasticsearch-setup-passwords.exp

@@ -1,7 +1,7 @@
 #!/usr/bin/expect -f
 
 # List of expected users with dummy password
-set user "(elastic|apm_system|kibana_system|logstash_system|beats_system|remote_monitoring_user)"
+set users {"elastic" "kibana_system" "logstash_system" "beats_system" "apm_system" "remote_monitoring_user"}
 set password "testpasswd"
 
 # Find elasticsearch container id
@@ -12,17 +12,27 @@ if { [string match "swarm" $MODE] } {
 	set cid [exec docker ps -q -f label=com.docker.compose.service=elasticsearch]
 }
 
-set cmd "docker exec -it $cid bin/elasticsearch-setup-passwords interactive -s -b -u http://localhost:9200"
+foreach user $users {
+	set cmd "docker exec -it $cid bin/elasticsearch-reset-password --batch --user $user -i"
 
-spawn {*}$cmd
+	spawn {*}$cmd
 
-expect {
-	-re "(E|Ree)nter password for \\\[$user\\\]: " {
-		send "$password\r"
-		exp_continue
+	expect {
+		-re "(E|Re-e)nter password for \\\[$user\\\]: " {
+			send "$password\r"
+			exp_continue
+		}
+		timeout {
+			puts "\ntimed out waiting for input"
+			exit 4
+		}
+		eof
 	}
-	eof
-}
 
-lassign [wait] pid spawnid os_error_flag value
-exit $value
+	lassign [wait] pid spawnid os_error_flag value
+
+	if {$value != 0} {
+		if {$os_error_flag == 0} { puts "exit status: $value" } else { puts "errno: $value" }
+		exit $value
+	}
+}

.github/workflows/scripts/run-tests-core.sh

@@ -24,25 +24,6 @@ poll_ready "$cid_ls" "http://${ip_ls}:9600/_node/pipelines/main?pretty"
 log 'Waiting for readiness of Kibana'
 poll_ready "$cid_kb" "http://${ip_kb}:5601/api/status" -u 'kibana_system:testpasswd'
 
-log 'Creating Logstash index pattern in Kibana'
-source .env
-curl -X POST -D- "http://${ip_kb}:5601/api/saved_objects/index-pattern" \
-	-s -w '\n' \
-	-H 'Content-Type: application/json' \
-	-H "kbn-version: ${ELK_VERSION}" \
-	-u elastic:testpasswd \
-	-d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}'
-
-log 'Searching index pattern via Kibana API'
-response="$(curl "http://${ip_kb}:5601/api/saved_objects/_find?type=index-pattern" -s -u elastic:testpasswd)"
-echo "$response"
-declare -i count
-count="$(jq -rn --argjson data "${response}" '$data.total')"
-if (( count != 1 )); then
-	echo "Expected 1 index pattern, got ${count}"
-	exit 1
-fi
-
 log 'Sending message to Logstash TCP input'
 
 declare -i was_retried=0
@@ -62,13 +43,14 @@ if ((was_retried)); then
 	echo >&2
 fi
 
-sleep 3
-curl -X POST "http://${ip_es}:9200/_refresh" -u elastic:testpasswd \
+sleep 5
+curl -X POST "http://${ip_es}:9200/logs-generic-default/_refresh" -u elastic:testpasswd \
 	-s -w '\n'
 
 log 'Searching message in Elasticsearch'
-response="$(curl "http://${ip_es}:9200/logstash-*/_search?q=message:dockerelk&pretty" -s -u elastic:testpasswd)"
+response="$(curl "http://${ip_es}:9200/logs-generic-default/_search?q=message:dockerelk&pretty" -s -u elastic:testpasswd)"
 echo "$response"
+declare -i count
 count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')"
 if (( count != 1 )); then
 	echo "Expected 1 document, got ${count}"

.github/workflows/scripts/run-tests-logspout.sh

@@ -39,7 +39,7 @@ declare -i was_retried=0
 
 # retry for max 60s (30*2s)
 for _ in $(seq 1 30); do
-	response="$(curl "http://${ip_es}:9200/logstash-*/_search?q=docker.image:%22docker-elk_logspout%22%20AND%20message:%22logspout%20gliderlabs%22~3&pretty" -s -u elastic:testpasswd)"
+	response="$(curl "http://${ip_es}:9200/logs-generic-default/_search?q=docker.image:%22docker-elk_logspout%22%20AND%20message:%22logspout%20gliderlabs%22~3&pretty" -s -u elastic:testpasswd)"
 
 	set +u  # prevent "unbound variable" if assigned value is not an integer
 	count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')"

.github/workflows/update.yml

@@ -12,13 +12,13 @@ jobs:
     strategy:
       matrix:
         release:
+          - 8.x
           - 7.x
-          - 6.x
         include:
-          - release: 7.x
+          - release: 8.x
             branch: main
-          - release: 6.x
-            branch: release-6.x
+          - release: 7.x
+            branch: release-7.x
 
     steps:
       - uses: actions/setup-node@v2

README.md

@@ -1,6 +1,6 @@
 # Elastic stack (ELK) on Docker
 
-[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-7.17.0-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases)
+[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-8.0.0-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases)
 [![Build Status](https://github.com/deviantony/docker-elk/workflows/CI/badge.svg?branch=main)](https://github.com/deviantony/docker-elk/actions?query=workflow%3ACI+branch%3Amain)
 [![Join the chat at https://gitter.im/deviantony/docker-elk](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/deviantony/docker-elk?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
@@ -24,7 +24,7 @@ Based on the official Docker images from Elastic:
 
 Other available stack variants:
 
-* [`tls`](https://github.com/deviantony/docker-elk/tree/tls): TLS encryption enabled in Elasticsearch.
+* [`tls`](https://github.com/deviantony/docker-elk/tree/tls): TLS encryption enabled in Elasticsearch
 * [`searchguard`](https://github.com/deviantony/docker-elk/tree/searchguard): Search Guard support
 
 ---
@@ -54,7 +54,6 @@ own_. [sherifabdlnaby/elastdocker][elastdocker] is one example among others of p
    * [Initial setup](#initial-setup)
      * [Setting up user authentication](#setting-up-user-authentication)
      * [Injecting data](#injecting-data)
-     * [Default Kibana index pattern creation](#default-kibana-index-pattern-creation)
 1. [Configuration](#configuration)
    * [How to configure Elasticsearch](#how-to-configure-elasticsearch)
    * [How to configure Kibana](#how-to-configure-kibana)
@@ -114,7 +113,7 @@ instructions from the [documentation][mac-filesharing] to add more locations.
 ### Version selection
 
 This repository tries to stay aligned with the latest version of the Elastic stack. The `main` branch tracks the current
-major version (7.x).
+major version (8.x).
 
 To use a different version of the core Elastic components, simply change the version number inside the `.env` file. If
 you are upgrading an existing stack, please carefully read the note in the next section.
@@ -124,8 +123,9 @@ performing a stack upgrade.**
 
 Older major versions are also supported on separate branches:
 
-* [`release-6.x`](https://github.com/deviantony/docker-elk/tree/release-6.x): 6.x series
-* [`release-5.x`](https://github.com/deviantony/docker-elk/tree/release-5.x): 5.x series (End-Of-Life)
+* [`release-7.x`](https://github.com/deviantony/docker-elk/tree/release-7.x): 7.x series
+* [`release-6.x`](https://github.com/deviantony/docker-elk/tree/release-6.x): 6.x series (End-of-life)
+* [`release-5.x`](https://github.com/deviantony/docker-elk/tree/release-5.x): 5.x series (End-of-life)
 
 ### Bringing up the stack
 
@@ -168,11 +168,31 @@ users][builtin-users] instead for increased security.
 
 1. Initialize passwords for built-in users
 
+    The commands below generate random passwords for all 6 built-in users. Take note of them.
+
+    ```console
+    $ docker-compose exec -T elasticsearch bin/elasticsearch-reset-password --batch --user elastic
+    ```
+
+    ```console
+    $ docker-compose exec -T elasticsearch bin/elasticsearch-reset-password --batch --user kibana_system
+    ```
+
     ```console
-    $ docker-compose exec -T elasticsearch bin/elasticsearch-setup-passwords auto --batch
+    $ docker-compose exec -T elasticsearch bin/elasticsearch-reset-password --batch --user logstash_system
     ```
 
-    Passwords for all 6 built-in users will be randomly generated. Take note of them.
+    ```console
+    $ docker-compose exec -T elasticsearch bin/elasticsearch-reset-password --batch --user beats_system
+    ```
+
+    ```console
+    $ docker-compose exec -T elasticsearch bin/elasticsearch-reset-password --batch --user apm_system
+    ```
+
+    ```console
+    $ docker-compose exec -T elasticsearch bin/elasticsearch-reset-password --batch --user remote_monitoring_user
+    ```
 
 1. Unset the bootstrap password (_optional_)
 
@@ -181,9 +201,8 @@ users][builtin-users] instead for increased security.
 
 1. Replace usernames and passwords in configuration files
 
-    Use the `kibana_system` user (`kibana` for releases <7.8.0) inside the Kibana configuration file
-    (`kibana/config/kibana.yml`) and the `logstash_system` user inside the Logstash configuration file
-    (`logstash/config/logstash.yml`) in place of the existing `elastic` user.
+    Use the `kibana_system` user inside the Kibana configuration file (`kibana/config/kibana.yml`) in place of the
+    existing `elastic` user.
 
     Replace the password for the `elastic` user inside the Logstash pipeline file (`logstash/pipeline/logstash.conf`).
 
@@ -225,37 +244,6 @@ $ cat /path/to/logfile.log | nc -c localhost 5000
 
 You can also load the sample data provided by your Kibana installation.
 
-### Default Kibana index pattern creation
-
-When Kibana launches for the first time, it is not configured with any index pattern.
-
-#### Via the Kibana web UI
-
-*:information_source: You need to inject data into Logstash before being able to configure a Logstash index pattern via
-the Kibana web UI.*
-
-Navigate to the _Discover_ view of Kibana from the left sidebar. You will be prompted to create an index pattern. Enter
-`logstash-*` to match Logstash indices then, on the next page, select `@timestamp` as the time filter field. Finally,
-click _Create index pattern_ and return to the _Discover_ view to inspect your log entries.
-
-Refer to [Connect Kibana with Elasticsearch][connect-kibana] and [Creating an index pattern][index-pattern] for detailed
-instructions about the index pattern configuration.
-
-#### On the command line
-
-Create an index pattern via the Kibana API:
-
-```console
-$ curl -XPOST -D- 'http://localhost:5601/api/saved_objects/index-pattern' \
-    -H 'Content-Type: application/json' \
-    -H 'kbn-version: 7.17.0' \
-    -u elastic:<your generated elastic password> \
-    -d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}'
-```
-
-The created pattern will automatically be marked as the default index pattern as soon as the Kibana UI is opened for the
-first time.
-
 ## Configuration
 
 *:information_source: Configuration is not dynamically reloaded, you will need to restart individual components after

docker-stack.yml

@@ -3,7 +3,7 @@ version: '3.3'
 services:
 
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.0.0
     ports:
       - "9200:9200"
       - "9300:9300"
@@ -25,7 +25,7 @@ services:
       replicas: 1
 
   logstash:
-    image: docker.elastic.co/logstash/logstash:7.17.0
+    image: docker.elastic.co/logstash/logstash:8.0.0
     ports:
       - "5044:5044"
       - "5000:5000"
@@ -44,7 +44,7 @@ services:
       replicas: 1
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:7.17.0
+    image: docker.elastic.co/kibana/kibana:8.0.0
     ports:
       - "5601:5601"
     configs:

extensions/enterprise-search/README.md

@@ -57,6 +57,17 @@ add the following setting:
 xpack.security.authc.api_key.enabled: true
 ```
 
+### Configure the Enterprise Search host in Kibana
+
+Kibana acts as the [management interface][enterprisesearch-ui] to Enterprise Search.
+
+To enable the management experience for Enterprise Search, modify the Kibana configuration file in
+[`kibana/config/kibana.yml`][config-kbn] and add the following setting:
+
+```yaml
+enterpriseSearch.host: http://enterprise-search:3002
+```
+
 ### Start the server
 
 To include Enterprise Search in the stack, run Docker Compose from the root of the repository with an additional command
@@ -129,6 +140,8 @@ Docker container: [Running Enterprise Search Using Docker][enterprisesearch-dock
 [enterprisesearch-config]: https://www.elastic.co/guide/en/enterprise-search/current/configuration.html
 [enterprisesearch-docker]: https://www.elastic.co/guide/en/enterprise-search/current/docker.html
 [enterprisesearch-docs]: https://www.elastic.co/guide/en/enterprise-search/current/index.html
+[enterprisesearch-ui]: https://www.elastic.co/guide/en/enterprise-search/current/user-interfaces.html
 
 [es-security]: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html#api-key-service-settings
 [config-es]: ../../elasticsearch/config/elasticsearch.yml
+[config-kbn]: ../../kibana/config/kibana.yml

extensions/enterprise-search/config/enterprise-search.yml

@@ -15,8 +15,9 @@ secret_management.encryption_keys:
 # IP address Enterprise Search listens on
 ent_search.listen_host: 0.0.0.0
 
-# URL at which users reach Enterprise Search
+# URL at which users reach Enterprise Search / Kibana
 ent_search.external_url: http://localhost:3002
+kibana.host: http://localhost:5601
 
 # Elasticsearch URL and credentials
 elasticsearch.host: http://elasticsearch:9200

logstash/config/logstash.yml

@@ -3,10 +3,3 @@
 ## https://github.com/elastic/logstash/blob/master/docker/data/logstash/config/logstash-full.yml
 #
 http.host: "0.0.0.0"
-xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ]
-
-## X-Pack security credentials
-#
-xpack.monitoring.enabled: true
-xpack.monitoring.elasticsearch.username: elastic
-xpack.monitoring.elasticsearch.password: changeme

logstash/pipeline/logstash.conf

@@ -15,6 +15,5 @@ output {
 		hosts => "elasticsearch:9200"
 		user => "elastic"
 		password => "changeme"
-		ecs_compatibility => disabled
 	}
 }