https://delta.io logo
d

Dhruvil Shah

02/08/2023, 1:19 AM
Copy code
import sys
from awsglue.transforms import *
from awsglue.utils import getResolvedOptions
from pyspark.context import SparkContext
from awsglue.context import GlueContext
from awsglue.job import Job

## @params: [JOB_NAME]
args = getResolvedOptions(sys.argv, ['JOB_NAME'])
a
sc = SparkContext()
glueContext = GlueContext(sc)
spark = glueContext.spark_session
job = Job(glueContext)
job.init(args['JOB_NAME'], args)


df = spark.read.format("delta") \
  .option("readChangeFeed", "true") \
  .load("<s3://lakeformation-dhruv/cdctest/>")


print ( df )
print( df.show() )


job.commit()