Versions (relevant - OpenSearch/Dashboard/Server OS/Browser):
opensearch 2.16.0
opensearch-dashboards 2.16.0
Describe the issue:
i am using opentelemetry to get log and metrics in opensearch using data-prepper
there are create index in metrics-index(ss4o_metrics-otel-2024.08.21) and log-index(ss4o_log-otel-myopensearch)
there are show in DQL query in data but logs in not show (ss4o_log-otel-myopensearch) log data
metrics in not show metrics data (ss4o_metrics-otel-2024.08.21)
Configuration:
opentelemetry config
config:
exporters:
opensearch/log:
logs_index: ss4o_log-otel-myopensearch
http:
endpoint: https://opensearch-cluster-master.myopensearch.svc.cluster.local:9200
auth:
authenticator: basicauth/client
tls:
insecure: false
insecure_skip_verify: true
timeout: 2m
retry_on_failure:
enabled: true
otlp/traces:
endpoint: mydataprepper-data-prepper.default.svc.cluster.local:21890
tls:
insecure: true
otlp/data-prepper:
endpoint: mydataprepper-data-prepper.default.svc.cluster.local:21891
tls:
insecure: true
otlp/logs:
endpoint: mydataprepper-data-prepper.default.svc.cluster.local:21892
tls:
insecure: true
# opensearch/logs:
# http:
# endpoint: https://opensearch-cluster-master-headless.myopensearch.svc.cluster.local:9200
# logs_index: my_log_index
debug: {}
extensions:
basicauth/client:
client_auth:
username: admin
password: TadhakDev01
# The health_check extension is mandatory for this chart.
# Without the health_check extension the collector will fail the readiness and liveliness probes.
# The health_check extension can be modified, but should never be removed.
health_check:
endpoint: ${env:MY_POD_IP}:13133
processors:
batch: {}
# Default memory limiter configuration for the collector based on k8s resource limits.
memory_limiter:
# check_interval is the time between measurements of memory usage.
check_interval: 5s
# By default limit_mib is set to 80% of ".Values.resources.limits.memory"
limit_percentage: 80
# By default spike_limit_mib is set to 25% of ".Values.resources.limits.memory"
spike_limit_percentage: 5
receivers:
filelog:
include:
- /app/log/naman/namanlog.log
start_at: end
operators:
- type: json_parser
severity:
parse_from: attributes.Level
id_field: SpanId
trace_id_field: TraceId
include:
- Timestamp
- Level
- MessageTemplate
- TraceId
- SpanId
- Properties
kubeletstats:
insecure_skip_verify: true
metric_groups: [ container, pod, node ]
metrics:
k8s.node.cpu.usage:
enabled: true
k8s.node.memory.available:
enabled: true
k8s.node.memory.usage:
enabled: true
k8s.pod.cpu.usage:
enabled: true
k8s.pod.memory.usage:
enabled: true
otlp:
protocols:
grpc:
endpoint: ${env:MY_POD_IP}:4317
http:
endpoint: ${env:MY_POD_IP}:4318
cors:
allowed_origins:
- "http://*"
- "https://*"
service:
telemetry:
metrics:
address: ${env:MY_POD_IP}:8888
extensions:
- basicauth/client
- health_check
pipelines:
logs:
receivers: [filelog]
exporters: [opensearch/log]
metrics:
receivers: [kubeletstats]
exporters: [otlp/data-prepper]
data prepper config
config:
## Simple Example
otel-logs-pipeline:
workers: 5
delay: 10
source:
otel_logs_source:
ssl: false
buffer:
bounded_blocking:
sink:
- opensearch:
hosts: ["https://otel-demo-opensearch:9200"]
username: "admin"
password: "TadhakDev01"
insecure: true
index_type: custom
index: otel-events-%{yyyy.MM.dd}
bulk_size: 4
entry-pipeline:
delay: "100"
source:
otel_trace_source:
ssl: false
sink:
- pipeline:
name: "raw-pipeline"
- pipeline:
name: "service-map-pipeline"
raw-pipeline:
source:
pipeline:
name: "entry-pipeline"
processor:
- otel_trace_raw:
sink:
- opensearch:
hosts: [ "https://otel-demo-opensearch:9200" ]
insecure: true
username: "admin"
password: "TadhakDev01"
index_type: trace-analytics-raw
service-map-pipeline:
delay: "100"
source:
pipeline:
name: "entry-pipeline"
processor:
- service_map_stateful:
sink:
- opensearch:
hosts: ["https://otel-demo-opensearch:9200"]
insecure: true
username: "admin"
password: "TadhakDev01"
index_type: trace-analytics-service-map
otel-metrics-pipeline:
workers: 8
delay: 3000
source:
otel_metrics_source:
health_check_service: true
ssl: false
buffer:
bounded_blocking:
buffer_size: 1024 # max number of records the buffer accepts
batch_size: 1024 # max number of records the buffer drains after each read
processor:
- otel_metrics:
calculate_histogram_buckets: true
calculate_exponential_histogram_buckets: true
exponential_histogram_max_allowed_scale: 10
flatten_attributes: false
sink:
- opensearch:
hosts: ["https://otel-demo-opensearch:9200"]
username: "admin"
password: "TadhakDev01"
insecure: true
index_type: custom
index: ss4o_metrics-otel-%{yyyy.MM.dd}
bulk_size: 4
template_type: index-template
template_content: >
{
"index_patterns": [
"ss4o_metrics-*-*"
],
"template": {
"mappings": {
"_meta": {
"version": "1.0.0",
"catalog": "observability",
"type": "metrics",
"component": "metrics",
"correlations" : [
{
"field": "spanId",
"foreign-schema" : "traces",
"foreign-field" : "spanId"
},
{
"field": "traceId",
"foreign-schema" : "traces",
"foreign-field" : "traceId"
}
]
},
"_source": {
"enabled": true
},
"dynamic_templates": [
{
"exemplar_attributes_map": {
"mapping": {
"type": "keyword"
},
"path_match": "exemplar.attributes.*"
}
},
{
"instrumentation_scope_attributes_map": {
"mapping": {
"type": "keyword"
},
"path_match": "instrumentationScope.attributes.*"
}
}
],
"properties": {
"serviceName": {
"type": "keyword"
},
"name": {
"type": "keyword",
"ignore_above": 256
},
"attributes": {
"type": "object",
"properties": {
"data_stream": {
"properties": {
"dataset": {
"ignore_above": 128,
"type": "keyword"
},
"namespace": {
"ignore_above": 128,
"type": "keyword"
},
"type": {
"ignore_above": 56,
"type": "keyword"
}
}
}
}
},
"description": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"unit": {
"type": "keyword",
"ignore_above": 128
},
"kind": {
"type": "keyword",
"ignore_above": 128
},
"aggregationTemporality": {
"type": "keyword",
"ignore_above": 128
},
"monotonic": {
"type": "boolean"
},
"startTime": {
"type": "date"
},
"@timestamp": {
"type": "date"
},
"observedTimestamp": {
"type": "date_nanos"
},
"value@int": {
"type": "integer"
},
"value@double": {
"type": "double"
},
"buckets": {
"type" : "nested",
"properties": {
"count": {
"type": "long"
},
"sum": {
"type": "double"
},
"max": {
"type": "float"
},
"min": {
"type": "float"
}
}
},
"bucketCount": {
"type": "long"
},
"bucketCountsList": {
"type": "long"
},
"explicitBoundsList": {
"type": "float"
},
"explicitBoundsCount": {
"type": "float"
},
"quantiles": {
"properties": {
"quantile": {
"type": "double"
},
"value": {
"type": "double"
}
}
},
"quantileValuesCount": {
"type": "long"
},
"positiveBuckets": {
"type" : "nested",
"properties": {
"count": {
"type": "long"
},
"max": {
"type": "float"
},
"min": {
"type": "float"
}
}
},
"negativeBuckets": {
"type" : "nested",
"properties": {
"count": {
"type": "long"
},
"max": {
"type": "float"
},
"min": {
"type": "float"
}
}
},
"negativeOffset": {
"type": "integer"
},
"positiveOffset": {
"type": "integer"
},
"zeroCount": {
"type": "long"
},
"scale": {
"type": "long"
},
"max": {
"type": "float"
},
"min": {
"type": "float"
},
"sum": {
"type": "float"
},
"count": {
"type": "long"
},
"exemplar": {
"properties": {
"time": {
"type": "date"
},
"traceId": {
"ignore_above": 256,
"type": "keyword"
},
"serviceName": {
"ignore_above": 256,
"type": "keyword"
},
"spanId": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"instrumentationScope": {
"properties": {
"name": {
"type": "keyword",
"ignore_above": 256
},
"version": {
"type": "keyword",
"ignore_above": 256
},
"droppedAttributesCount": {
"type": "integer"
},
"schemaUrl": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"schemaUrl": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"aliases" : {
"otel-metrics-" : {}
},
"settings": {
"index": {
"mapping": {
"total_fields": {
"limit": 10000
}
},
"refresh_interval": "5s"
}
}
},
"composed_of": [],
"version": 1,
"_meta": {
"description": "Observability Metrics Mapping Template",
"catalog": "observability",
"type": "metrics",
"correlations" : [
{
"field": "spanId",
"foreign-schema" : "traces",
"foreign-field" : "spanId"
},
{
"field": "traceId",
"foreign-schema" : "traces",
"foreign-field" : "traceId"
}
]
}
}
Relevant Logs or Screenshots:
metrics data in opensearch dashboard
logs in opensearch dashboard
but this log and metrics get in DQL query show data
metrics
logs
