Hey Everyone,
I am facing 400 error while trying to send logs from EKS to opensearch. Below is the Error and fluentd.yaml that am trying out. Please help me to resolve this issue. I understood that type is not supported. But am not sure how to modify config to make it work.
opensearch version:
helm search repo opensearch
NAME CHART VERSION APP VERSION DESCRIPTION
opensearch/opensearch 2.6.0 2.3.0 A Helm chart for OpenSearch
opensearch/opensearch-dashboards 2.5.1 2.3.0 A Helm chart for OpenSearch Dashboards
Error:
2022-09-16 12:11:00 +0000 [info]: #0 fluentd worker is now running worker=0
2022-09-16 12:11:01 +0000 [warn]: #0 failed to flush the buffer. retry_time=1 next_retry_seconds=2022-09-16 12:11:02 +0000 chunk="5e8c999454d850ebb12f622e956b91bd" error_class=Fluent::Plugin::ElasticsearchOutput::RecoverableRequestFailure error="could not push logs to Elasticsearch cluster ({:host=>\"opensearch.master.acceldata.dev\", :port=>9200, :scheme=>\"https\", :user=>\"admin\", :password=>\"obfuscated\"}): [400] {\"error\":{\"root_cause\":[{\"type\":\"illegal_argument_exception\",\"reason\":\"Action/metadata line [1] contains an unknown parameter [_type]\"}],\"type\":\"illegal_argument_exception\",\"reason\":\"Action/metadata line [1] contains an unknown parameter [_type]\"},\"status\":400}"
fluentd.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: fluentd
namespace: logging
labels:
app: fluentd
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: fluentd
labels:
app: fluentd
rules:
- apiGroups:
- ""
resources:
- nodes
- pods
- namespaces
verbs:
- get
- list
- watch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: fluentd
roleRef:
kind: ClusterRole
name: fluentd
apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
name: fluentd
namespace: logging
---
apiVersion: v1
kind: ConfigMap
metadata:
name: fluentd-configmap
namespace: logging
labels:
app: fluentd
kubernetes.io/cluster-service: "true"
data:
fluent.conf: |
<match fluent.**>
# this tells fluentd to not output its log on stdout
@type null
</match>
# here we read the logs from Docker's containers and parse them
<source>
@type tail
path /var/log/containers/*.log
pos_file /var/log/app.log.pos
tag kubernetes.*
read_from_head true
<parse>
@type json
time_format %Y-%m-%dT%H:%M:%S.%NZ
</parse>
</source>
# we use kubernetes metadata plugin to add metadatas to the log
<filter kubernetes.**>
@type kubernetes_metadata
</filter>
# we send the logs to Elasticsearch
<match **>
@type elasticsearch_dynamic
@log_level info
include_tag_key true
host "#{ENV['FLUENT_ELASTICSEARCH_HOST']}"
port "#{ENV['FLUENT_ELASTICSEARCH_PORT']}"
user "#{ENV['FLUENT_ELASTICSEARCH_USER']}"
password "#{ENV['FLUENT_ELASTICSEARCH_PASSWORD']}"
scheme "#{ENV['FLUENT_ELASTICSEARCH_SCHEME']}"
ssl_verify "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERIFY']}"
ssl_version "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERSION']}"
reload_connections true
logstash_format true
logstash_prefix logstash
<buffer>
@type file
path /var/log/fluentd-buffers/kubernetes.system.buffer
flush_mode interval
retry_type exponential_backoff
flush_thread_count 2
flush_interval 5s
retry_forever true
retry_max_interval 30
chunk_limit_size 2M
queue_limit_length 32
overflow_action block
</buffer>
</match>
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: fluentd
namespace: logging
labels:
app: fluentd
spec:
selector:
matchLabels:
app: fluentd
template:
metadata:
labels:
app: fluentd
spec:
serviceAccount: fluentd
serviceAccountName: fluentd
tolerations:
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
- operator: "Exists"
effect: "NoExecute"
- operator: "Exists"
effect: "NoSchedule"
containers:
- name: fluentd
image: fluent/fluentd-kubernetes-daemonset:v1.4.2-debian-elasticsearch-1.1
env:
- name: FLUENT_ELASTICSEARCH_HOST
value: "opensearch.master"
- name: FLUENT_ELASTICSEARCH_PORT
value: "9200"
- name: FLUENT_ELASTICSEARCH_SCHEME
value: "https"
- name: FLUENTD_SYSTEMD_CONF
value: "disable"
- name: FLUENT_ELASTICSEARCH_USER
value: "admin"
- name: FLUENT_ELASTICSEARCH_PASSWORD
value: "admin"
- name: FLUENT_ELASTICSEARCH_SSL_VERIFY
value: "false"
- name: FLUENT_ELASTICSEARCH_SSL_VERSION
value: "TLSv1_2"
- name: FLUENT_CONTAINER_TAIL_EXCLUDE_PATH
value: /var/log/containers/fluent*
- name: FLUENT_CONTAINER_TAIL_PARSER_TYPE
value: /^(?<time>.+) (?<stream>stdout|stderr)( (?<logtag>.))? (?<log>.*)$/
resources:
limits:
memory: 512Mi
requests:
cpu: 100m
memory: 512Mi
volumeMounts:
- name: varlog
mountPath: /var/log
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: fluentd-configmap
mountPath: /fluentd/etc/fluent.conf
subPath: fluent.conf
# - name: certs
# mountPath: /fluentd/etc/certs
terminationGracePeriodSeconds: 30
volumes:
- name: fluentd-configmap
configMap:
name: fluentd-configmap
- name: varlog
hostPath:
path: /var/log
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
# - name: certs
# secret:
# secretName: opensearch-fluentd-cert