Metadata line [1] contains an unknown parameter [_type]

OpenSearch
1

Hey Everyone,

I am facing 400 error while trying to send logs from EKS to opensearch. Below is the Error and fluentd.yaml that am trying out. Please help me to resolve this issue. I understood that type is not supported. But am not sure how to modify config to make it work.

opensearch version:

helm search repo opensearch
NAME                            	CHART VERSION	APP VERSION	DESCRIPTION                           
opensearch/opensearch           	2.6.0        	2.3.0      	A Helm chart for OpenSearch           
opensearch/opensearch-dashboards	2.5.1        	2.3.0      	A Helm chart for OpenSearch Dashboards

Error:

2022-09-16 12:11:00 +0000 [info]: #0 fluentd worker is now running worker=0
2022-09-16 12:11:01 +0000 [warn]: #0 failed to flush the buffer. retry_time=1 next_retry_seconds=2022-09-16 12:11:02 +0000 chunk="5e8c999454d850ebb12f622e956b91bd" error_class=Fluent::Plugin::ElasticsearchOutput::RecoverableRequestFailure error="could not push logs to Elasticsearch cluster ({:host=>\"opensearch.master.acceldata.dev\", :port=>9200, :scheme=>\"https\", :user=>\"admin\", :password=>\"obfuscated\"}): [400] {\"error\":{\"root_cause\":[{\"type\":\"illegal_argument_exception\",\"reason\":\"Action/metadata line [1] contains an unknown parameter [_type]\"}],\"type\":\"illegal_argument_exception\",\"reason\":\"Action/metadata line [1] contains an unknown parameter [_type]\"},\"status\":400}"

fluentd.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: fluentd
  namespace: logging
  labels:
    app: fluentd
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: fluentd
  labels:
    app: fluentd
rules:
- apiGroups:
  - ""
  resources:
  - nodes
  - pods
  - namespaces
  verbs:
  - get
  - list
  - watch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: fluentd
roleRef:
  kind: ClusterRole
  name: fluentd
  apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
  name: fluentd
  namespace: logging
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: fluentd-configmap
  namespace: logging
  labels:
    app: fluentd
    kubernetes.io/cluster-service: "true"
data:
  fluent.conf: |
    <match fluent.**>
        # this tells fluentd to not output its log on stdout
        @type null
    </match>
    # here we read the logs from Docker's containers and parse them
    <source>
      @type tail
      path /var/log/containers/*.log
      pos_file /var/log/app.log.pos
      tag kubernetes.*
      read_from_head true
      <parse>
        @type json
        time_format %Y-%m-%dT%H:%M:%S.%NZ
      </parse>
    </source>
    # we use kubernetes metadata plugin to add metadatas to the log
    <filter kubernetes.**>
        @type kubernetes_metadata
    </filter>
     # we send the logs to Elasticsearch
    <match **>
       @type elasticsearch_dynamic
       @log_level info
       include_tag_key true
       host "#{ENV['FLUENT_ELASTICSEARCH_HOST']}"
       port "#{ENV['FLUENT_ELASTICSEARCH_PORT']}"
       user "#{ENV['FLUENT_ELASTICSEARCH_USER']}"
       password "#{ENV['FLUENT_ELASTICSEARCH_PASSWORD']}"
       scheme "#{ENV['FLUENT_ELASTICSEARCH_SCHEME']}"
       ssl_verify "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERIFY']}"
       ssl_version "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERSION']}"
       reload_connections true
       logstash_format true
       logstash_prefix logstash
       <buffer>
           @type file
           path /var/log/fluentd-buffers/kubernetes.system.buffer
           flush_mode interval
           retry_type exponential_backoff
           flush_thread_count 2
           flush_interval 5s
           retry_forever true
           retry_max_interval 30
           chunk_limit_size 2M
           queue_limit_length 32
           overflow_action block
       </buffer>
    </match>
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: fluentd
  namespace: logging
  labels:
    app: fluentd
spec:
  selector:
    matchLabels:
      app: fluentd
  template:
    metadata:
      labels:
        app: fluentd
    spec:
      serviceAccount: fluentd
      serviceAccountName: fluentd
      tolerations:
      - key: node-role.kubernetes.io/master
        operator: Exists
        effect: NoSchedule
      - operator: "Exists"
        effect: "NoExecute"
      - operator: "Exists"
        effect: "NoSchedule"
      containers:
      - name: fluentd
        image: fluent/fluentd-kubernetes-daemonset:v1.4.2-debian-elasticsearch-1.1
        env:
          - name:  FLUENT_ELASTICSEARCH_HOST
            value: "opensearch.master"
          - name:  FLUENT_ELASTICSEARCH_PORT
            value: "9200"
          - name: FLUENT_ELASTICSEARCH_SCHEME
            value: "https"
          - name: FLUENTD_SYSTEMD_CONF
            value: "disable"
          - name: FLUENT_ELASTICSEARCH_USER
            value: "admin"
          - name: FLUENT_ELASTICSEARCH_PASSWORD
            value: "admin"
          - name: FLUENT_ELASTICSEARCH_SSL_VERIFY
            value: "false"
          - name:  FLUENT_ELASTICSEARCH_SSL_VERSION
            value: "TLSv1_2"
          - name: FLUENT_CONTAINER_TAIL_EXCLUDE_PATH
            value: /var/log/containers/fluent*
          - name: FLUENT_CONTAINER_TAIL_PARSER_TYPE
            value: /^(?<time>.+) (?<stream>stdout|stderr)( (?<logtag>.))? (?<log>.*)$/
        resources:
          limits:
            memory: 512Mi
          requests:
            cpu: 100m
            memory: 512Mi
        volumeMounts:
        - name: varlog
          mountPath: /var/log
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
        - name: fluentd-configmap
          mountPath: /fluentd/etc/fluent.conf
          subPath: fluent.conf
#        - name: certs
#          mountPath: /fluentd/etc/certs
      terminationGracePeriodSeconds: 30
      volumes:
      - name: fluentd-configmap
        configMap:
          name: fluentd-configmap
      - name: varlog
        hostPath:
          path: /var/log
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers
#      - name: certs
#        secret:
#          secretName: opensearch-fluentd-cert
2

Hi @sharath_90!

Don’t despair - I actually went through something similar. OpenSearch has done away with the _type field, so I’m pretty sure you can solve your problem by defining the variable ‘suppress_type_name’.

Add that to your fluentd configuration, and I’m pretty sure that should be all you need.

Let me know if that works!

Nate

3

@nateynate Thanks for reply. I have placed the said parameter on all the blocks containing @type. But it doesn’t solve my problem. Can you please help me further?

apiVersion: v1
kind: ServiceAccount
metadata:
  name: fluentd
  namespace: logging
  labels:
    app: fluentd
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: fluentd
  labels:
    app: fluentd
rules:
- apiGroups:
  - ""
  resources:
  - nodes
  - pods
  - namespaces
  verbs:
  - get
  - list
  - watch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: fluentd
roleRef:
  kind: ClusterRole
  name: fluentd
  apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
  name: fluentd
  namespace: logging
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: fluentd-configmap
  namespace: logging
  labels:
    app: fluentd
    kubernetes.io/cluster-service: "true"
data:
  fluent.conf: |
    <match fluent.**>
        # this tells fluentd to not output its log on stdout
        @type null
        suppress_type_name true
    </match>
    # here we read the logs from Docker's containers and parse them
    <source>
      @type tail
      path /var/log/containers/*.log
      pos_file /var/log/app.log.pos
      tag kubernetes.*
      read_from_head true
      suppress_type_name true
      <parse>
        @type json
        time_format %Y-%m-%dT%H:%M:%S.%NZ
        suppress_type_name true
      </parse>
    </source>
    # we use kubernetes metadata plugin to add metadatas to the log
    <filter kubernetes.**>
        @type kubernetes_metadata
        suppress_type_name true
    </filter>
     # we send the logs to Elasticsearch
    <match **>
       @type elasticsearch_dynamic
       @log_level info
       include_tag_key true
       host "#{ENV['FLUENT_ELASTICSEARCH_HOST']}"
       port "#{ENV['FLUENT_ELASTICSEARCH_PORT']}"
       user "#{ENV['FLUENT_ELASTICSEARCH_USER']}"
       password "#{ENV['FLUENT_ELASTICSEARCH_PASSWORD']}"
       scheme "#{ENV['FLUENT_ELASTICSEARCH_SCHEME']}"
       ssl_verify "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERIFY']}"
       ssl_version "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERSION']}"
       reload_connections true
       logstash_format true
       logstash_prefix logstash
       suppress_type_name true
       <buffer>
           @type file
           path /var/log/fluentd-buffers/kubernetes.system.buffer
           flush_mode interval
           retry_type exponential_backoff
           flush_thread_count 2
           flush_interval 5s
           retry_forever true
           retry_max_interval 30
           chunk_limit_size 2M
           queue_limit_length 32
           overflow_action block
           suppress_type_name true
       </buffer>
    </match>
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: fluentd
  namespace: logging
  labels:
    app: fluentd
spec:
  selector:
    matchLabels:
      app: fluentd
  template:
    metadata:
      labels:
        app: fluentd
    spec:
      serviceAccount: fluentd
      serviceAccountName: fluentd
      tolerations:
      - key: node-role.kubernetes.io/master
        operator: Exists
        effect: NoSchedule
      - operator: "Exists"
        effect: "NoExecute"
      - operator: "Exists"
        effect: "NoSchedule"
      containers:
      - name: fluentd
        image: fluent/fluentd-kubernetes-daemonset:v1.4.2-debian-elasticsearch-1.1
        env:
          - name:  FLUENT_ELASTICSEARCH_HOST
            value: "opensearch.master"
          - name:  FLUENT_ELASTICSEARCH_PORT
            value: "9200"
          - name: FLUENT_ELASTICSEARCH_SCHEME
            value: "https"
          - name: FLUENTD_SYSTEMD_CONF
            value: "disable"
          - name: FLUENT_ELASTICSEARCH_USER
            value: "admin"
          - name: FLUENT_ELASTICSEARCH_PASSWORD
            value: "admin"
          - name: FLUENT_ELASTICSEARCH_SSL_VERIFY
            value: "false"
          - name:  FLUENT_ELASTICSEARCH_SSL_VERSION
            value: "TLSv1_2"
          - name: FLUENT_CONTAINER_TAIL_EXCLUDE_PATH
            value: /var/log/containers/fluent*
          - name: FLUENT_CONTAINER_TAIL_PARSER_TYPE
            value: /^(?<time>.+) (?<stream>stdout|stderr)( (?<logtag>.))? (?<log>.*)$/
        resources:
          limits:
            memory: 512Mi
          requests:
            cpu: 100m
            memory: 512Mi
        volumeMounts:
        - name: varlog
          mountPath: /var/log
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
        - name: fluentd-configmap
          mountPath: /fluentd/etc/fluent.conf
          subPath: fluent.conf
#        - name: certs
#          mountPath: /fluentd/etc/certs
      terminationGracePeriodSeconds: 30
      volumes:
      - name: fluentd-configmap
        configMap:
          name: fluentd-configmap
      - name: varlog
        hostPath:
          path: /var/log
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers
#      - name: certs
#        secret:
#          secretName: opensearch-fluentd-cert