{ "eventTime":"2023-11-02T18:42:00.619Z", "producer":"https://github.com/OpenLineage/OpenLineage/tree/1.5.0/integration/spark", "schemaURL":"https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent", "eventType":"START", "run":{ "runId":"957e5191-a1bf-4c65-b08a-b7d125da2ff3", "facets":{ "spark.logicalPlan":{ "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.5.0/integration/spark", "_schemaURL":"https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet", "plan":[ { "class":"org.apache.spark.sql.catalyst.plans.logical.ReplaceTableAsSelect", "num-children":0, "name":[ { "class":"org.apache.spark.sql.catalyst.analysis.ResolvedIdentifier", "num-children":0, "catalog":null, "identifier":null } ], "partitioning":[ ], "query":[ { "class":"org.apache.spark.sql.execution.datasources.LogicalRelation", "num-children":0, "relation":null, "output":[ [ { "class":"org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children":0, "name":"household_id", "dataType":"integer", "nullable":true, "metadata":{ }, "exprId":{ "product-class":"org.apache.spark.sql.catalyst.expressions.ExprId", "id":131, "jvmId":"ae8ac2ed-0395-48ec-8418-2fc62a0fd2b4" }, "qualifier":[ ] } ], [ { "class":"org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children":0, "name":"basket_id", "dataType":"long", "nullable":true, "metadata":{ }, "exprId":{ "product-class":"org.apache.spark.sql.catalyst.expressions.ExprId", "id":132, "jvmId":"ae8ac2ed-0395-48ec-8418-2fc62a0fd2b4" }, "qualifier":[ ] } ], [ { "class":"org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children":0, "name":"day", "dataType":"integer", "nullable":true, "metadata":{ }, "exprId":{ "product-class":"org.apache.spark.sql.catalyst.expressions.ExprId", "id":133, "jvmId":"ae8ac2ed-0395-48ec-8418-2fc62a0fd2b4" }, "qualifier":[ ] } ], [ { "class":"org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children":0, "name":"product_id", "dataType":"integer", "nullable":true, "metadata":{ }, "exprId":{ "product-class":"org.apache.spark.sql.catalyst.expressions.ExprId", "id":134, "jvmId":"ae8ac2ed-0395-48ec-8418-2fc62a0fd2b4" }, "qualifier":[ ] } ], [ { "class":"org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children":0, "name":"quantity", "dataType":"integer", "nullable":true, "metadata":{ }, "exprId":{ "product-class":"org.apache.spark.sql.catalyst.expressions.ExprId", "id":135, "jvmId":"ae8ac2ed-0395-48ec-8418-2fc62a0fd2b4" }, "qualifier":[ ] } ], [ { "class":"org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children":0, "name":"sales_amount", "dataType":"float", "nullable":true, "metadata":{ }, "exprId":{ "product-class":"org.apache.spark.sql.catalyst.expressions.ExprId", "id":136, "jvmId":"ae8ac2ed-0395-48ec-8418-2fc62a0fd2b4" }, "qualifier":[ ] } ], [ { "class":"org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children":0, "name":"store_id", "dataType":"integer", "nullable":true, "metadata":{ }, "exprId":{ "product-class":"org.apache.spark.sql.catalyst.expressions.ExprId", "id":137, "jvmId":"ae8ac2ed-0395-48ec-8418-2fc62a0fd2b4" }, "qualifier":[ ] } ], [ { "class":"org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children":0, "name":"discount_amount", "dataType":"float", "nullable":true, "metadata":{ }, "exprId":{ "product-class":"org.apache.spark.sql.catalyst.expressions.ExprId", "id":138, "jvmId":"ae8ac2ed-0395-48ec-8418-2fc62a0fd2b4" }, "qualifier":[ ] } ], [ { "class":"org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children":0, "name":"transaction_time", "dataType":"integer", "nullable":true, "metadata":{ }, "exprId":{ "product-class":"org.apache.spark.sql.catalyst.expressions.ExprId", "id":139, "jvmId":"ae8ac2ed-0395-48ec-8418-2fc62a0fd2b4" }, "qualifier":[ ] } ], [ { "class":"org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children":0, "name":"week_no", "dataType":"integer", "nullable":true, "metadata":{ }, "exprId":{ "product-class":"org.apache.spark.sql.catalyst.expressions.ExprId", "id":140, "jvmId":"ae8ac2ed-0395-48ec-8418-2fc62a0fd2b4" }, "qualifier":[ ] } ], [ { "class":"org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children":0, "name":"coupon_discount", "dataType":"float", "nullable":true, "metadata":{ }, "exprId":{ "product-class":"org.apache.spark.sql.catalyst.expressions.ExprId", "id":141, "jvmId":"ae8ac2ed-0395-48ec-8418-2fc62a0fd2b4" }, "qualifier":[ ] } ], [ { "class":"org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children":0, "name":"coupon_discount_match", "dataType":"float", "nullable":true, "metadata":{ }, "exprId":{ "product-class":"org.apache.spark.sql.catalyst.expressions.ExprId", "id":142, "jvmId":"ae8ac2ed-0395-48ec-8418-2fc62a0fd2b4" }, "qualifier":[ ] } ] ], "isStreaming":false } ], "tableSpec":null, "writeOptions":null, "orCreate":true, "isAnalyzed":true } ] }, "debug":{ "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.5.0/integration/spark", "_schemaURL":"https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet", "classpath":{ "openLineageVersion":"1.5.0", "sparkVersion":"3.4.1", "scalaVersion":"2.12.15", "jars":[ ], "classDetails":[ { "className":"org.apache.spark.sql.delta.catalog.DeltaCatalog", "onClasspath":true }, { "className":"org.apache.iceberg.catalog.Catalog", "onClasspath":false }, { "className":"com.google.cloud.spark.bigquery.BigQueryRelation", "packageVersion":"0.22.2-SNAPSHOT", "onClasspath":true } ] }, "system":{ "sparkDeployMode":"client", "javaVersion":"1.8.0_372", "javaVendor":"Azul Systems, Inc.", "osArch":"amd64", "osName":"Linux", "osVersion":"5.15.0-1049-azure", "userLanguage":"en", "userTimezone":"Etc/UTC" }, "config":{ "extraListeners":"io.openlineage.spark.agent.OpenLineageSparkListener", "openLineageConfig":{ "endpoint":"api/v1/lineage", "debugFacet":"enabled", "url.param.code":"8kZl0bo2TJfnbpFxBv-R2v7xBDj-PgWMol3yUm5iP1vaAzFu9kIZGg==", "facets.disabled":"[spark_unknown;]", "namespace":"adb-4679476628690204.4#default", "transport.type":"http", "transport.url":"https://490f-2607-fb90-c13e-509-2854-a7e-2d53-a767.ngrok-free.app" }, "catalogClass":"org.apache.spark.sql.internal.CatalogImpl" }, "logicalPlan":{ "nodes":[ { "id":"ReplaceTableAsSelect@1968008258", "desc":"ReplaceTableAsSelect TableSpec(Map(),Some(delta),Map(),Some(wasbs://studio@clororetaildevadls.blob.core.windows.net/examples/data/csv/completejourney/silver/transactions),None,None,false,Set(),None,None,None), [overwriteSchema=true, path=wasbs://studio@clororetaildevadls.blob.core.windows.net/examples/data/csv/completejourney/silver/transactions], true, true\n :- ResolvedIdentifier com.databricks.sql.managedcatalog.UnityCatalogV2Proxy@698ca05e, journey.transactions\n +- Relation [household_id#131,basket_id#132L,day#133,product_id#134,quantity#135,sales_amount#136,store_id#137,discount_amount#138,transaction_time#139,week_no#140,coupon_discount#141,coupon_discount_match#142] csv\n", "children":[ ] } ] } }, "spark_version":{ "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.5.0/integration/spark", "_schemaURL":"https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet", "spark-version":"3.4.1", "openlineage-spark-version":"1.5.0" }, "processing_engine":{ "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.5.0/integration/spark", "_schemaURL":"https://openlineage.io/spec/facets/1-1-0/ProcessingEngineRunFacet.json#/$defs/ProcessingEngineRunFacet", "version":"3.4.1", "name":"spark", "openlineageAdapterVersion":"1.5.0" } } }, "job":{ "namespace":"adb-4679476628690204.4#default", "name":"adb-4679476628690204.4.azuredatabricks.net.atomic_replace_table_as_select.journey_db_transactions", "facets":{ } }, "inputs":[ { "namespace":"wasbs://studio@clororetaildevadls.blob.core.windows.net", "name":"/examples/data/csv/completejourney/transaction_data.csv", "facets":{ "dataSource":{ "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.5.0/integration/spark", "_schemaURL":"https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet", "name":"wasbs://studio@clororetaildevadls.blob.core.windows.net", "uri":"wasbs://studio@clororetaildevadls.blob.core.windows.net" }, "schema":{ "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.5.0/integration/spark", "_schemaURL":"https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet", "fields":[ { "name":"household_id", "type":"integer" }, { "name":"basket_id", "type":"long" }, { "name":"day", "type":"integer" }, { "name":"product_id", "type":"integer" }, { "name":"quantity", "type":"integer" }, { "name":"sales_amount", "type":"float" }, { "name":"store_id", "type":"integer" }, { "name":"discount_amount", "type":"float" }, { "name":"transaction_time", "type":"integer" }, { "name":"week_no", "type":"integer" }, { "name":"coupon_discount", "type":"float" }, { "name":"coupon_discount_match", "type":"float" } ] } }, "inputFacets":{ } } ], "outputs":[ { "namespace":"dbfs", "name":"/user/hive/warehouse/journey.db/transactions", "facets":{ "dataSource":{ "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.5.0/integration/spark", "_schemaURL":"https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet", "name":"dbfs", "uri":"dbfs" }, "schema":{ "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.5.0/integration/spark", "_schemaURL":"https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet", "fields":[ { "name":"household_id", "type":"integer" }, { "name":"basket_id", "type":"long" }, { "name":"day", "type":"integer" }, { "name":"product_id", "type":"integer" }, { "name":"quantity", "type":"integer" }, { "name":"sales_amount", "type":"float" }, { "name":"store_id", "type":"integer" }, { "name":"discount_amount", "type":"float" }, { "name":"transaction_time", "type":"integer" }, { "name":"week_no", "type":"integer" }, { "name":"coupon_discount", "type":"float" }, { "name":"coupon_discount_match", "type":"float" } ] }, "storage":{ "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.5.0/integration/spark", "_schemaURL":"https://openlineage.io/spec/facets/1-0-0/StorageDatasetFacet.json#/$defs/StorageDatasetFacet", "storageLayer":"unity", "fileFormat":"parquet" }, "symlinks":{ "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.5.0/integration/spark", "_schemaURL":"https://openlineage.io/spec/facets/1-0-0/SymlinksDatasetFacet.json#/$defs/SymlinksDatasetFacet", "identifiers":[ { "namespace":"/user/hive/warehouse/journey.db", "name":"journey.transactions", "type":"TABLE" } ] }, "lifecycleStateChange":{ "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.5.0/integration/spark", "_schemaURL":"https://openlineage.io/spec/facets/1-0-0/LifecycleStateChangeDatasetFacet.json#/$defs/LifecycleStateChangeDatasetFacet", "lifecycleStateChange":"OVERWRITE" } }, "outputFacets":{ } } ] }