Reducing latency in aws osis pipeline

Data Prepper
1

using aws osis pipeline (whatever version is currently on aws) with Elasticsearch 7.10

My pipeline takes about 35 seconds to stream data from dynamodb to AWS Elasticsearch
Sink latency metrics:

image
image1530×606 42.2 KB

pipeline config:

{
  "TABLENAME": {
    "processor": [
      {
        "date": {
          "destination": "log_ingestion_time",
          "from_time_received": true
        }
      },
      {
        "date": {
          "destination": "@timestamp",
          "from_time_received": true
        }
      }
    ],
    "sink": [
      {
        "opensearch": {
          "aws": {
            "region": "us-east-1",
            "serverless": false,
            "sts_role_arn": "arn:aws:iam::XXXXX:role/osis-pipeline-role-TABLENAME"
          },
          "dlq": {
            "s3": {
              "bucket": "XXX-osis-dlq",
              "key_path_prefix": "TABLENAME/failed-records/",
              "region": "us-east-1",
              "sts_role_arn": "arn:aws:iam::XXXXX:role/osis-pipeline-role-TABLENAME"
            }
          },
          "max_retries": "5",
          "document_id": "${getMetadata(\"primary_key\")}",
          "action": "${getMetadata(\"opensearch_action\")}",
          "hosts": [
            "https://ENDPOINT.us-east-1.es.amazonaws.com"
          ],
          "index": "INDEXNAME"
        }
      }
    ],
    "source": {
      "dynamodb": {
        "acknowledgments": true,
        "aws": {
          "region": "us-east-1",
          "sts_role_arn": "arn:aws:iam::XXXXX:role/osis-pipeline-role-TABLE"
        },
        "tables": [
          {
            "stream": {
              "start_position": "LATEST"
            },
            "table_arn": "arn:aws:dynamodb:us-east-1:XXXXX:table/TABLE"
          }
        ]
      }
    }
  },
  "version": "2"
}

is there any way I can improve on this?
the lag is huge

2

This topic was automatically closed 60 days after the last reply. New replies are no longer allowed.