Versions (relevant - OpenSearch/Dashboard/Server OS/Browser):
elasticsearch (8.5.0)
elasticsearch-hadoop-8.5.0.jar
Describe the issue:
Hi all,im trying to write data to opensearch using spark df ,wheb i run the code im getting the error:
Configuration:
import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.Row
import org.apache.spark.sql.Dataset
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.types.StringType
import org.apache.spark.sql.types.StructField
import org.apache.spark.sql.types.DateType
import org.apache.spark.sql.types.MapType
import org.apache.spark.sql.types.ArrayType
import org.apache.log4j.Level
import org.apache.log4j.Logger
import org.elasticsearch.spark.sql._
import spark.sqlContext.implicits._
import org.apache.spark.sql.functions._
import spark.implicits._
object Customer extends App {
Logger.getLogger(“org”).setLevel(Level.ERROR)
val sparkConf = new SparkConf()
sparkConf.set(“spark.app.name”, “my first application”)
val spark = SparkSession.builder()
.config(sparkConf)
.config(“es.nodes”,“vp.amazonaws.com”)
.config(“es.net.ssl”, “false”)
.config(“spark.es.port”,“443”)
.config(“es.resource”, “name/name”)
.config(“es.http.retries”, “3”)
.config(“es.nodes.wan.only”,“true”)
.getOrCreate()
}
val data = sc.textFile(“s3://bucket/”).toDF
data.saveToEs(“name/name”)
Relevant Logs or Screenshots:
failed (java.net.ConnectException: Connection refused (Connection refused)); no other nodes left - aborting…
org.elasticsearch.hadoop.EsHadoopIllegalArgumentException: Cannot detect ES version - typically this happens if the network/Elasticsearch cluster is not accessible or when targeting a WAN/Cloud instance without the proper setting ‘es.nodes.wan.only’