Impossible upgrade Opensearch 2.19.3 to 3.5.0 - index version incompatible

OpenSearch
1

Versions (relevant - OpenSearch/Dashboard/Server OS/Browser):

  • Update Opensearch from 2.19.3 to 3.5.0
  • The environment was originally migrated from version 1.3.2

Description:

java.lang.IllegalStateException: The index [[.opendistro-alerting-alerts/LyOaYfJ7RZm9rkKTAaS21w]] was created with version [1.3.2] but the minimum compatible version is [2.0.0]. It should be re-indexed in OpenSearch 2.x before upgrading to 3.5.0.

As the index contains no documents I removed the index. But the error persists.

DELETE /.opendistro-alerting-alerts/
{
  "acknowledged": true
}


GET /.opendistro-alerting-alerts/

{
  ".opendistro-alerting-alerts": {
    "aliases": {},
    "mappings": {
      "dynamic": "strict",
      "_meta": {
        "schema_version": 5
      },
      "_routing": {
        "required": true
      },
      "properties": {
        "acknowledged_time": {
          "type": "date"
        },
        "action_execution_results": {
          "type": "nested",
          "properties": {
            "action_id": {
              "type": "keyword"
            },
            "last_execution_time": {
              "type": "date"
            },
            "throttled_count": {
              "type": "integer"
            }
          }
        },
        "agg_alert_content": {
          "dynamic": "true",
          "properties": {
            "bucket_key": {
              "type": "text"
            },
            "parent_bucket_path": {
              "type": "text"
            }
          }
        },
        "alert_history": {
          "type": "nested",
          "properties": {
            "message": {
              "type": "text"
            },
            "timestamp": {
              "type": "date"
            }
          }
        },
        "associated_alert_ids": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword"
            }
          }
        },
        "clusters": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword"
            }
          }
        },
        "end_time": {
          "type": "date"
        },
        "error_message": {
          "type": "text"
        },
        "execution_id": {
          "type": "keyword"
        },
        "finding_ids": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword"
            }
          }
        },
        "id": {
          "type": "keyword"
        },
        "last_notification_time": {
          "type": "date"
        },
        "monitor_id": {
          "type": "keyword"
        },
        "monitor_name": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "monitor_user": {
          "properties": {
            "backend_roles": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword"
                }
              }
            },
            "custom_attribute_names": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword"
                }
              }
            },
            "name": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "roles": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword"
                }
              }
            }
          }
        },
        "monitor_version": {
          "type": "long"
        },
        "related_doc_ids": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword"
            }
          }
        },
        "schema_version": {
          "type": "integer"
        },
        "severity": {
          "type": "keyword"
        },
        "start_time": {
          "type": "date"
        },
        "state": {
          "type": "keyword"
        },
        "trigger_id": {
          "type": "keyword"
        },
        "trigger_name": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "version": {
          "type": "long"
        },
        "workflow_id": {
          "type": "keyword"
        },
        "workflow_name": {
          "type": "keyword"
        }
      }
    },
    "settings": {
      "index": {
        "replication": {
          "type": "DOCUMENT"
        },
        "hidden": "true",
        "number_of_shards": "1",
        "provided_name": ".opendistro-alerting-alerts",
        "creation_date": "1774856944543",
        "number_of_replicas": "1",
        "uuid": "aPIm8wb7QB-kCHJEBwZJzA",
        "version": {
          "created": "136408127"
        }
      }
    }
  }
}

After deletion the uuid changed - but in logfile the old uuid is printed.

I also tried to reindex but no success.

2

@rofr_dd But your error regards .opendistro-alerting-alerts index.
Youve deleted .opendistro-reports-instances

3

Hi, thank you for the hint. I just copied the wrong snippets.
I updated the description.

4

@rofr_dd According to your index settings, the current index version is 136408127. So, you state that after deleting the index, you’re still seeing the exact error below?

Did you restart the OpenSearch service?

Could you share the exact steps for deleting that index?
Did you run your cluster with version 2.x, delete the reported index and upgrade the cluster to 3.5.0?

5

@pablo: Restarting every node after reindexing solved the problem with most of the indices now. Thank you. Just wondering why this step is neccessary in the cluster.