adding fluent-plugin-detect-exceptions plugin to fluentd-elasticsearch image…

[shivpathak]

Signed-off-by: Shiv Pathak shivpathak23@gmail.com

[repeatedly]

Thanks!

[NareshAnaparthi]

Could one of you share config map to enable detect-exceptions plugin. Using image 'fluentd-kubernetes-daemonset'. Thanks

[shivpathak]

Could one of you share config map to enable detect-exceptions plugin. Using image 'fluentd-kubernetes-daemonset'. Thanks

@NareshAnaparthi here is sample config for fluentd.

you can mount this configmap in fluentd daemonset and ensure all environment variable defined this configmap are available to daemonset pods e.g. (FLUENT_ELASTICSEARCH_HOST and FLUENT_ELASTICSEARCH_PORT) etc

apiVersion: v1
data:
  fluent.conf: |-
    #Just an example of what kind of variables can come inside. This part does not apply as config. Do your config with ENV vars
    <match **>
      @type elasticsearch
      @id out_es
      @log_level info
      include_tag_key true
      host "#{ENV['FLUENT_ELASTICSEARCH_HOST']}"
      port "#{ENV['FLUENT_ELASTICSEARCH_PORT']}"
      scheme "#{ENV['FLUENT_ELASTICSEARCH_SCHEME'] || 'http'}"
      ssl_verify "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERIFY'] || 'true'}"
      user "#{ENV['FLUENT_ELASTICSEARCH_USER']}"
      password "#{ENV['FLUENT_ELASTICSEARCH_PASSWORD']}"
      reload_connections "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_CONNECTIONS'] || 'true'}"
      logstash_prefix "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_PREFIX'] || 'logstash'}"
      logstash_format true
      buffer_chunk_limit "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_CHUNK_LIMIT_SIZE'] || '2M'}"
      buffer_queue_limit "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_QUEUE_LIMIT_LENGTH'] || '32'}"
      flush_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_INTERVAL'] || '5s'}"
      max_retry_wait "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_RETRY_MAX_INTERVAL'] || '30'}"
      disable_retry_limit num_threads "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_THREAD_COUNT'] || '8'}"
    </match>
  kubernetes.conf: |-
    <match fluent.**>
      @type null
    </match>

    <source>
      @type tail
      @id in_tail_containers_logs
      path /var/log/containers/*.log
      pos_file /var/log/fluentd-containers.log.pos
      tag kubernetes.*
      read_from_head true
      format json
      time_format %Y-%m-%dT%H:%M:%S.%NZ
    </source>

    <filter kubernetes.var.log.containers.nginx-**.log>
      @type parser
      key_name log
      reserve_data yes
      format /^(?<remote_ip>[^-]*) - (?<user_name>[^ ]*) \[(?<datetime>[^\]]*)\] \"(?<method>[^ ]*) (?<uri>[^ ]*) (?<http_version>[^"]*)\" (?<status>[^ ]*) (?<body_bytes_sent>[^ ]*) \"(?<referer>[^ ]*)\" \"(?<user_agent>[^"]*)\" (?<request_length>[^ ]*) (?<request_time>[^ ]*) \[(?<proxy_upstream_name>[^\]]*)\] (?<proxy_alternative_upstream_name>[^ ]*) (?<upstream_addr>[^ ]*) (?<upstream_response_length>[^ ]*) (?<upstream_response_time>[^ ]*) (?<upstream_response_status>[^ ]*) (?<request_id>[a-zA-Z0-9]+)/
      types status:integer,body_bytes_sent:integer,request_length:integer,request_time:float,upstream_response_length:integer,upstream_response_time:float,upstream_response_status:integer
    </filter>

    <filter kubernetes.**>
      @type kubernetes_metadata
      @id filter_kube_metadata
    </filter>

    <match kubernetes.**>
      @type detect_exceptions
      remove_tag_prefix kubernetes
      message log
      languages java
      multiline_flush_interval 0.1
      stream real
    </match>

    <match var.**>
       @type elasticsearch
       @log_level info
       include_tag_key true
       host "#{ENV['FLUENT_ELASTICSEARCH_HOST']}"
       port "#{ENV['FLUENT_ELASTICSEARCH_PORT']}"
       scheme "#{ENV['FLUENT_ELASTICSEARCH_SCHEME'] || 'http'}"
       ssl_verify "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERIFY'] || 'true'}"
       reload_connections "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_CONNECTIONS'] || 'true'}"
       logstash_prefix "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_PREFIX'] || 'logstash'}"
       logstash_format true
       type_name fluentd
       reload_connections false
       reconnect_on_error true
       reload_on_failure true
       buffer_chunk_limit "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_CHUNK_LIMIT_SIZE'] || '8M'}"
       buffer_queue_limit "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_QUEUE_LIMIT_LENGTH'] || '256'}"
       flush_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_INTERVAL'] || '10s'}"
       max_retry_wait "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_RETRY_MAX_INTERVAL'] || '30'}"
       disable_retry_limit
       num_threads "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_THREAD_COUNT'] || '32'}"
    </match>

kind: ConfigMap
metadata:
  labels:
    stack: efk
  name: fluentd-config
  namespace: kube-system

[NareshAnaparthi]

Thanks Shiv for sharing. Finally i made it to work using concat plugin.