Nagendra Darla
04/06/2023, 5:52 AMSparkSession spark = SparkSession.builder()
.config("fs.s3.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem")
.config("fs.AbstractFileSystem.s3.impl", "org.apache.hadoop.fs.s3a.S3A")
.config("fs.s3a.aws.credentials.provider", "com.amazonaws.auth.DefaultAWSCredentialsProviderChain")
.config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
.config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog")
.config("spark.delta.logStore.s3.impl", "io.delta.storage.S3DynamoDBLogStore")
.config("spark.io.delta.storage.S3DynamoDBLogStore.ddb.tableName", "delta_log")
.config("spark.io.delta.storage.S3DynamoDBLogStore.ddb.region", "us-east-1")
.config("spark.io.delta.storage.S3DynamoDBLogStore.credentials.provider",
"com.amazonaws.auth.DefaultAWSCredentialsProviderChain")
.getOrCreate();