{'eventTime': '2023-12-11T06:59:11.602Z', 'producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', 'schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent', 'eventType': 'START', 'run': {'runId': '78faad9a-148b-47b5-8a1f-b8762b576c6e', 'facets': {'spark.logicalPlan': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet', 'plan': [{'class': 'org.apache.spark.sql.catalyst.plans.logical.AppendData', 'num-children': 1, 'table': [{'class': 'org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation', 'num-children': 0, 'table': None, 'output': [[{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'ID', 'dataType': 'long', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2553, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'FIRST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2554, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'LAST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2555, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}]], 'catalog': None, 'identifier': None, 'options': None}], 'query': 0, 'writeOptions': None, 'isByName': False, 'requireImplicitCasting': True}, {'class': 'org.apache.spark.sql.execution.datasources.LogicalRelation', 'num-children': 0, 'relation': None, 'output': [[{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'ID', 'dataType': 'long', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2550, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'FIRST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2551, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'LAST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2552, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}]], 'catalogTable': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogTable', 'identifier': {'product-class': 'org.apache.spark.sql.catalyst.TableIdentifier', 'table': 'jaffle_shop_customers', 'database': 'default', 'catalog': 'hive_metastore'}, 'tableType': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogTableType', 'name': 'MANAGED'}, 'storage': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogStorageFormat', 'locationUri': None, 'inputFormat': 'org.apache.hadoop.mapred.SequenceFileInputFormat', 'outputFormat': 'org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat', 'serde': 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe', 'compressed': False, 'properties': None}, 'schema': {'type': 'struct', 'fields': [{'name': 'ID', 'type': 'long', 'nullable': True, 'metadata': {}}, {'name': 'FIRST_NAME', 'type': 'string', 'nullable': True, 'metadata': {}}, {'name': 'LAST_NAME', 'type': 'string', 'nullable': True, 'metadata': {}}]}, 'provider': 'delta', 'partitionColumnNames': [], 'owner': 'root', 'createTime': 1696916365000, 'lastAccessTime': 0, 'createVersion': '3.3.2', 'properties': None, 'comment': 'Created by the file upload UI', 'unsupportedFeatures': [], 'tracksPartitionsInCatalog': True, 'schemaPreservesCase': True, 'ignoredProperties': None}, 'isStreaming': False}]}, 'spark_version': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet', 'spark-version': '3.1.2', 'openlineage-spark-version': '1.1.0'}}}, 'job': {'namespace': 'default', 'name': 'adb-4598673832939986.6.azuredatabricks.net.append_data_exec_v1.dbt_simran_db_jaffle_shop_customers_trg', 'facets': {}}, 'inputs': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'facets': {'dataSource': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet', 'name': 'dbfs', 'uri': 'dbfs'}, 'schema': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet', 'fields': [{'name': 'ID', 'type': 'long'}, {'name': 'FIRST_NAME', 'type': 'string'}, {'name': 'LAST_NAME', 'type': 'string'}]}, 'symlinks': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SymlinksDatasetFacet.json#/$defs/SymlinksDatasetFacet', 'identifiers': [{'namespace': '/user/hive/warehouse', 'name': 'default.jaffle_shop_customers', 'type': 'TABLE'}]}}, 'inputFacets': {}}], 'outputs': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/dbt_simran.db/jaffle_shop_customers_trg', 'facets': {'dataSource': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet', 'name': 'dbfs', 'uri': 'dbfs'}, 'schema': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet', 'fields': [{'name': 'ID', 'type': 'long'}, {'name': 'FIRST_NAME', 'type': 'string'}, {'name': 'LAST_NAME', 'type': 'string'}]}, 'storage': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/StorageDatasetFacet.json#/$defs/StorageDatasetFacet', 'storageLayer': 'delta', 'fileFormat': 'parquet'}, 'columnLineage': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-1/ColumnLineageDatasetFacet.json#/$defs/ColumnLineageDatasetFacet', 'fields': {'ID': {'inputFields': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'field': 'ID'}]}, 'FIRST_NAME': {'inputFields': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'field': 'FIRST_NAME'}]}, 'LAST_NAME': {'inputFields': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'field': 'LAST_NAME'}]}}}, 'symlinks': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SymlinksDatasetFacet.json#/$defs/SymlinksDatasetFacet', 'identifiers': [{'namespace': '/user/hive/warehouse/dbt_simran.db', 'name': 'hive_metastore.dbt_simran.jaffle_shop_customers_trg', 'type': 'TABLE'}]}}, 'outputFacets': {}}]}

{'eventTime': '2023-12-11T06:59:13.418Z', 'producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', 'schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent', 'eventType': 'START', 'run': {'runId': '78faad9a-148b-47b5-8a1f-b8762b576c6e', 'facets': {'spark.logicalPlan': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet', 'plan': [{'class': 'org.apache.spark.sql.catalyst.plans.logical.AppendData', 'num-children': 1, 'table': [{'class': 'org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation', 'num-children': 0, 'table': None, 'output': [[{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'ID', 'dataType': 'long', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2553, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'FIRST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2554, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'LAST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2555, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}]], 'catalog': None, 'identifier': None, 'options': None}], 'query': 0, 'writeOptions': None, 'isByName': False, 'requireImplicitCasting': True}, {'class': 'org.apache.spark.sql.execution.datasources.LogicalRelation', 'num-children': 0, 'relation': None, 'output': [[{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'ID', 'dataType': 'long', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2550, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'FIRST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2551, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'LAST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2552, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}]], 'catalogTable': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogTable', 'identifier': {'product-class': 'org.apache.spark.sql.catalyst.TableIdentifier', 'table': 'jaffle_shop_customers', 'database': 'default', 'catalog': 'hive_metastore'}, 'tableType': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogTableType', 'name': 'MANAGED'}, 'storage': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogStorageFormat', 'locationUri': None, 'inputFormat': 'org.apache.hadoop.mapred.SequenceFileInputFormat', 'outputFormat': 'org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat', 'serde': 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe', 'compressed': False, 'properties': None}, 'schema': {'type': 'struct', 'fields': [{'name': 'ID', 'type': 'long', 'nullable': True, 'metadata': {}}, {'name': 'FIRST_NAME', 'type': 'string', 'nullable': True, 'metadata': {}}, {'name': 'LAST_NAME', 'type': 'string', 'nullable': True, 'metadata': {}}]}, 'provider': 'delta', 'partitionColumnNames': [], 'owner': 'root', 'createTime': 1696916365000, 'lastAccessTime': 0, 'createVersion': '3.3.2', 'properties': None, 'comment': 'Created by the file upload UI', 'unsupportedFeatures': [], 'tracksPartitionsInCatalog': True, 'schemaPreservesCase': True, 'ignoredProperties': None}, 'isStreaming': False}]}, 'spark_version': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet', 'spark-version': '3.1.2', 'openlineage-spark-version': '1.1.0'}, 'spark_properties': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet', 'properties': {'spark.master': 'spark://10.139.64.100:7077', 'spark.app.name': 'Databricks Shell'}}, 'environment-properties': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet', 'environment-properties': {'spark.databricks.clusterUsageTags.clusterName': 'Databook', 'spark.databricks.clusterUsageTags.azureSubscriptionId': '6b3f47d8-dd33-461f-bfbb-8c6c7b87fdfe', 'spark.databricks.notebook.path': '/Users/leela.kannan@maersk.com/2) SLA Notebooks - Leela/OpenLineage - Facet Test', 'mountPoints': [], 'spark.databricks.clusterUsageTags.clusterOwnerOrgId': '4598673832939986', 'user': 'simran.suri@maersk.com', 'userId': '5799628310045251', 'orgId': '4598673832939986'}}}}, 'job': {'namespace': 'default', 'name': 'adb-4598673832939986.6.azuredatabricks.net.append_data_exec_v1.dbt_simran_db_jaffle_shop_customers_trg', 'facets': {}}, 'inputs': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'facets': {'dataSource': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet', 'name': 'dbfs', 'uri': 'dbfs'}, 'schema': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet', 'fields': [{'name': 'ID', 'type': 'long'}, {'name': 'FIRST_NAME', 'type': 'string'}, {'name': 'LAST_NAME', 'type': 'string'}]}, 'symlinks': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SymlinksDatasetFacet.json#/$defs/SymlinksDatasetFacet', 'identifiers': [{'namespace': '/user/hive/warehouse', 'name': 'default.jaffle_shop_customers', 'type': 'TABLE'}]}}, 'inputFacets': {}}], 'outputs': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/dbt_simran.db/jaffle_shop_customers_trg', 'facets': {'dataSource': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet', 'name': 'dbfs', 'uri': 'dbfs'}, 'schema': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet', 'fields': [{'name': 'ID', 'type': 'long'}, {'name': 'FIRST_NAME', 'type': 'string'}, {'name': 'LAST_NAME', 'type': 'string'}]}, 'storage': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/StorageDatasetFacet.json#/$defs/StorageDatasetFacet', 'storageLayer': 'delta', 'fileFormat': 'parquet'}, 'columnLineage': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-1/ColumnLineageDatasetFacet.json#/$defs/ColumnLineageDatasetFacet', 'fields': {'ID': {'inputFields': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'field': 'ID'}]}, 'FIRST_NAME': {'inputFields': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'field': 'FIRST_NAME'}]}, 'LAST_NAME': {'inputFields': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'field': 'LAST_NAME'}]}}}, 'symlinks': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SymlinksDatasetFacet.json#/$defs/SymlinksDatasetFacet', 'identifiers': [{'namespace': '/user/hive/warehouse/dbt_simran.db', 'name': 'hive_metastore.dbt_simran.jaffle_shop_customers_trg', 'type': 'TABLE'}]}}, 'outputFacets': {}}]}

{'eventTime': '2023-12-11T06:59:17.044Z', 'producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', 'schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent', 'eventType': 'COMPLETE', 'run': {'runId': '78faad9a-148b-47b5-8a1f-b8762b576c6e', 'facets': {'spark.logicalPlan': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet', 'plan': [{'class': 'org.apache.spark.sql.catalyst.plans.logical.AppendData', 'num-children': 1, 'table': [{'class': 'org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation', 'num-children': 0, 'table': None, 'output': [[{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'ID', 'dataType': 'long', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2553, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'FIRST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2554, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'LAST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2555, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}]], 'catalog': None, 'identifier': None, 'options': None}], 'query': 0, 'writeOptions': None, 'isByName': False, 'requireImplicitCasting': True}, {'class': 'org.apache.spark.sql.execution.datasources.LogicalRelation', 'num-children': 0, 'relation': None, 'output': [[{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'ID', 'dataType': 'long', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2550, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'FIRST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2551, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'LAST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2552, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}]], 'catalogTable': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogTable', 'identifier': {'product-class': 'org.apache.spark.sql.catalyst.TableIdentifier', 'table': 'jaffle_shop_customers', 'database': 'default', 'catalog': 'hive_metastore'}, 'tableType': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogTableType', 'name': 'MANAGED'}, 'storage': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogStorageFormat', 'locationUri': None, 'inputFormat': 'org.apache.hadoop.mapred.SequenceFileInputFormat', 'outputFormat': 'org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat', 'serde': 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe', 'compressed': False, 'properties': None}, 'schema': {'type': 'struct', 'fields': [{'name': 'ID', 'type': 'long', 'nullable': True, 'metadata': {}}, {'name': 'FIRST_NAME', 'type': 'string', 'nullable': True, 'metadata': {}}, {'name': 'LAST_NAME', 'type': 'string', 'nullable': True, 'metadata': {}}]}, 'provider': 'delta', 'partitionColumnNames': [], 'owner': 'root', 'createTime': 1696916365000, 'lastAccessTime': 0, 'createVersion': '3.3.2', 'properties': None, 'comment': 'Created by the file upload UI', 'unsupportedFeatures': [], 'tracksPartitionsInCatalog': True, 'schemaPreservesCase': True, 'ignoredProperties': None}, 'isStreaming': False}]}, 'spark_version': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet', 'spark-version': '3.1.2', 'openlineage-spark-version': '1.1.0'}}}, 'job': {'namespace': 'default', 'name': 'adb-4598673832939986.6.azuredatabricks.net.append_data_exec_v1.dbt_simran_db_jaffle_shop_customers_trg', 'facets': {}}, 'inputs': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'facets': {'dataSource': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet', 'name': 'dbfs', 'uri': 'dbfs'}, 'schema': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet', 'fields': [{'name': 'ID', 'type': 'long'}, {'name': 'FIRST_NAME', 'type': 'string'}, {'name': 'LAST_NAME', 'type': 'string'}]}, 'symlinks': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SymlinksDatasetFacet.json#/$defs/SymlinksDatasetFacet', 'identifiers': [{'namespace': '/user/hive/warehouse', 'name': 'default.jaffle_shop_customers', 'type': 'TABLE'}]}}, 'inputFacets': {}}], 'outputs': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/dbt_simran.db/jaffle_shop_customers_trg', 'facets': {'dataSource': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet', 'name': 'dbfs', 'uri': 'dbfs'}, 'schema': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet', 'fields': [{'name': 'ID', 'type': 'long'}, {'name': 'FIRST_NAME', 'type': 'string'}, {'name': 'LAST_NAME', 'type': 'string'}]}, 'storage': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/StorageDatasetFacet.json#/$defs/StorageDatasetFacet', 'storageLayer': 'delta', 'fileFormat': 'parquet'}, 'columnLineage': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-1/ColumnLineageDatasetFacet.json#/$defs/ColumnLineageDatasetFacet', 'fields': {'ID': {'inputFields': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'field': 'ID'}]}, 'FIRST_NAME': {'inputFields': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'field': 'FIRST_NAME'}]}, 'LAST_NAME': {'inputFields': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'field': 'LAST_NAME'}]}}}, 'symlinks': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SymlinksDatasetFacet.json#/$defs/SymlinksDatasetFacet', 'identifiers': [{'namespace': '/user/hive/warehouse/dbt_simran.db', 'name': 'hive_metastore.dbt_simran.jaffle_shop_customers_trg', 'type': 'TABLE'}]}}, 'outputFacets': {'outputStatistics': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/OutputStatisticsOutputDatasetFacet.json#/$defs/OutputStatisticsOutputDatasetFacet', 'rowCount': 0, 'size': 0}}}]}

{'eventTime': '2023-12-11T06:59:20.121Z', 'producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', 'schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent', 'eventType': 'START', 'run': {'runId': 'b6c918de-8662-49ad-ab8c-52179c77866a', 'facets': {'spark.logicalPlan': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet', 'plan': [{'class': 'org.apache.spark.sql.execution.datasources.LogicalRelation', 'num-children': 0, 'relation': None, 'output': [[{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'ID', 'dataType': 'long', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2550, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'FIRST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2551, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'LAST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2552, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}]], 'catalogTable': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogTable', 'identifier': {'product-class': 'org.apache.spark.sql.catalyst.TableIdentifier', 'table': 'jaffle_shop_customers', 'database': 'default', 'catalog': 'hive_metastore'}, 'tableType': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogTableType', 'name': 'MANAGED'}, 'storage': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogStorageFormat', 'locationUri': None, 'inputFormat': 'org.apache.hadoop.mapred.SequenceFileInputFormat', 'outputFormat': 'org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat', 'serde': 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe', 'compressed': False, 'properties': None}, 'schema': {'type': 'struct', 'fields': [{'name': 'ID', 'type': 'long', 'nullable': True, 'metadata': {}}, {'name': 'FIRST_NAME', 'type': 'string', 'nullable': True, 'metadata': {}}, {'name': 'LAST_NAME', 'type': 'string', 'nullable': True, 'metadata': {}}]}, 'provider': 'delta', 'partitionColumnNames': [], 'owner': 'root', 'createTime': 1696916365000, 'lastAccessTime': 0, 'createVersion': '3.3.2', 'properties': None, 'comment': 'Created by the file upload UI', 'unsupportedFeatures': [], 'tracksPartitionsInCatalog': True, 'schemaPreservesCase': True, 'ignoredProperties': None}, 'isStreaming': False}]}, 'spark_version': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet', 'spark-version': '3.1.2', 'openlineage-spark-version': '1.1.0'}}}, 'job': {'namespace': 'default', 'name': 'adb-4598673832939986.6.azuredatabricks.net.columnar_to_row', 'facets': {}}, 'inputs': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'facets': {'dataSource': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet', 'name': 'dbfs', 'uri': 'dbfs'}, 'schema': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet', 'fields': [{'name': 'ID', 'type': 'long'}, {'name': 'FIRST_NAME', 'type': 'string'}, {'name': 'LAST_NAME', 'type': 'string'}]}, 'symlinks': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SymlinksDatasetFacet.json#/$defs/SymlinksDatasetFacet', 'identifiers': [{'namespace': '/user/hive/warehouse', 'name': 'default.jaffle_shop_customers', 'type': 'TABLE'}]}}, 'inputFacets': {}}], 'outputs': []}

{'eventTime': '2023-12-11T06:59:20.313Z', 'producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', 'schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent', 'eventType': 'START', 'run': {'runId': 'b6c918de-8662-49ad-ab8c-52179c77866a', 'facets': {'spark.logicalPlan': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet', 'plan': [{'class': 'org.apache.spark.sql.execution.datasources.LogicalRelation', 'num-children': 0, 'relation': None, 'output': [[{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'ID', 'dataType': 'long', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2550, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'FIRST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2551, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'LAST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2552, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}]], 'catalogTable': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogTable', 'identifier': {'product-class': 'org.apache.spark.sql.catalyst.TableIdentifier', 'table': 'jaffle_shop_customers', 'database': 'default', 'catalog': 'hive_metastore'}, 'tableType': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogTableType', 'name': 'MANAGED'}, 'storage': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogStorageFormat', 'locationUri': None, 'inputFormat': 'org.apache.hadoop.mapred.SequenceFileInputFormat', 'outputFormat': 'org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat', 'serde': 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe', 'compressed': False, 'properties': None}, 'schema': {'type': 'struct', 'fields': [{'name': 'ID', 'type': 'long', 'nullable': True, 'metadata': {}}, {'name': 'FIRST_NAME', 'type': 'string', 'nullable': True, 'metadata': {}}, {'name': 'LAST_NAME', 'type': 'string', 'nullable': True, 'metadata': {}}]}, 'provider': 'delta', 'partitionColumnNames': [], 'owner': 'root', 'createTime': 1696916365000, 'lastAccessTime': 0, 'createVersion': '3.3.2', 'properties': None, 'comment': 'Created by the file upload UI', 'unsupportedFeatures': [], 'tracksPartitionsInCatalog': True, 'schemaPreservesCase': True, 'ignoredProperties': None}, 'isStreaming': False}]}, 'spark_version': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet', 'spark-version': '3.1.2', 'openlineage-spark-version': '1.1.0'}, 'spark_properties': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet', 'properties': {'spark.master': 'spark://10.139.64.100:7077', 'spark.app.name': 'Databricks Shell'}}, 'environment-properties': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet', 'environment-properties': {'spark.databricks.clusterUsageTags.clusterName': 'Databook', 'spark.databricks.clusterUsageTags.azureSubscriptionId': '6b3f47d8-dd33-461f-bfbb-8c6c7b87fdfe', 'spark.databricks.notebook.path': '/Users/leela.kannan@maersk.com/2) SLA Notebooks - Leela/OpenLineage - Facet Test', 'mountPoints': [], 'spark.databricks.clusterUsageTags.clusterOwnerOrgId': '4598673832939986', 'user': 'simran.suri@maersk.com', 'userId': '5799628310045251', 'orgId': '4598673832939986'}}}}, 'job': {'namespace': 'default', 'name': 'adb-4598673832939986.6.azuredatabricks.net.columnar_to_row', 'facets': {}}, 'inputs': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'facets': {'dataSource': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet', 'name': 'dbfs', 'uri': 'dbfs'}, 'schema': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet', 'fields': [{'name': 'ID', 'type': 'long'}, {'name': 'FIRST_NAME', 'type': 'string'}, {'name': 'LAST_NAME', 'type': 'string'}]}, 'symlinks': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SymlinksDatasetFacet.json#/$defs/SymlinksDatasetFacet', 'identifiers': [{'namespace': '/user/hive/warehouse', 'name': 'default.jaffle_shop_customers', 'type': 'TABLE'}]}}, 'inputFacets': {}}], 'outputs': []}

{'eventTime': '2023-12-11T06:59:21.182Z', 'producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', 'schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent', 'eventType': 'COMPLETE', 'run': {'runId': 'b6c918de-8662-49ad-ab8c-52179c77866a', 'facets': {'spark.logicalPlan': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet', 'plan': [{'class': 'org.apache.spark.sql.execution.datasources.LogicalRelation', 'num-children': 0, 'relation': None, 'output': [[{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'ID', 'dataType': 'long', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2550, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'FIRST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2551, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'LAST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2552, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}]], 'catalogTable': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogTable', 'identifier': {'product-class': 'org.apache.spark.sql.catalyst.TableIdentifier', 'table': 'jaffle_shop_customers', 'database': 'default', 'catalog': 'hive_metastore'}, 'tableType': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogTableType', 'name': 'MANAGED'}, 'storage': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogStorageFormat', 'locationUri': None, 'inputFormat': 'org.apache.hadoop.mapred.SequenceFileInputFormat', 'outputFormat': 'org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat', 'serde': 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe', 'compressed': False, 'properties': None}, 'schema': {'type': 'struct', 'fields': [{'name': 'ID', 'type': 'long', 'nullable': True, 'metadata': {}}, {'name': 'FIRST_NAME', 'type': 'string', 'nullable': True, 'metadata': {}}, {'name': 'LAST_NAME', 'type': 'string', 'nullable': True, 'metadata': {}}]}, 'provider': 'delta', 'partitionColumnNames': [], 'owner': 'root', 'createTime': 1696916365000, 'lastAccessTime': 0, 'createVersion': '3.3.2', 'properties': None, 'comment': 'Created by the file upload UI', 'unsupportedFeatures': [], 'tracksPartitionsInCatalog': True, 'schemaPreservesCase': True, 'ignoredProperties': None}, 'isStreaming': False}]}, 'spark_version': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet', 'spark-version': '3.1.2', 'openlineage-spark-version': '1.1.0'}}}, 'job': {'namespace': 'default', 'name': 'adb-4598673832939986.6.azuredatabricks.net.columnar_to_row', 'facets': {}}, 'inputs': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'facets': {'dataSource': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet', 'name': 'dbfs', 'uri': 'dbfs'}, 'schema': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet', 'fields': [{'name': 'ID', 'type': 'long'}, {'name': 'FIRST_NAME', 'type': 'string'}, {'name': 'LAST_NAME', 'type': 'string'}]}, 'symlinks': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SymlinksDatasetFacet.json#/$defs/SymlinksDatasetFacet', 'identifiers': [{'namespace': '/user/hive/warehouse', 'name': 'default.jaffle_shop_customers', 'type': 'TABLE'}]}}, 'inputFacets': {}}], 'outputs': []}

{'eventTime': '2023-12-11T06:59:21.303Z', 'producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', 'schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent', 'eventType': 'COMPLETE', 'run': {'runId': 'b6c918de-8662-49ad-ab8c-52179c77866a', 'facets': {'spark.logicalPlan': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet', 'plan': [{'class': 'org.apache.spark.sql.execution.datasources.LogicalRelation', 'num-children': 0, 'relation': None, 'output': [[{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'ID', 'dataType': 'long', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2550, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'FIRST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2551, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'LAST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2552, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}]], 'catalogTable': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogTable', 'identifier': {'product-class': 'org.apache.spark.sql.catalyst.TableIdentifier', 'table': 'jaffle_shop_customers', 'database': 'default', 'catalog': 'hive_metastore'}, 'tableType': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogTableType', 'name': 'MANAGED'}, 'storage': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogStorageFormat', 'locationUri': None, 'inputFormat': 'org.apache.hadoop.mapred.SequenceFileInputFormat', 'outputFormat': 'org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat', 'serde': 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe', 'compressed': False, 'properties': None}, 'schema': {'type': 'struct', 'fields': [{'name': 'ID', 'type': 'long', 'nullable': True, 'metadata': {}}, {'name': 'FIRST_NAME', 'type': 'string', 'nullable': True, 'metadata': {}}, {'name': 'LAST_NAME', 'type': 'string', 'nullable': True, 'metadata': {}}]}, 'provider': 'delta', 'partitionColumnNames': [], 'owner': 'root', 'createTime': 1696916365000, 'lastAccessTime': 0, 'createVersion': '3.3.2', 'properties': None, 'comment': 'Created by the file upload UI', 'unsupportedFeatures': [], 'tracksPartitionsInCatalog': True, 'schemaPreservesCase': True, 'ignoredProperties': None}, 'isStreaming': False}]}, 'spark_version': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet', 'spark-version': '3.1.2', 'openlineage-spark-version': '1.1.0'}}}, 'job': {'namespace': 'default', 'name': 'adb-4598673832939986.6.azuredatabricks.net.columnar_to_row', 'facets': {}}, 'inputs': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'facets': {'dataSource': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet', 'name': 'dbfs', 'uri': 'dbfs'}, 'schema': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet', 'fields': [{'name': 'ID', 'type': 'long'}, {'name': 'FIRST_NAME', 'type': 'string'}, {'name': 'LAST_NAME', 'type': 'string'}]}, 'symlinks': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SymlinksDatasetFacet.json#/$defs/SymlinksDatasetFacet', 'identifiers': [{'namespace': '/user/hive/warehouse', 'name': 'default.jaffle_shop_customers', 'type': 'TABLE'}]}}, 'inputFacets': {}}], 'outputs': []}

{'eventTime': '2023-12-11T06:59:22.337Z', 'producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', 'schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent', 'eventType': 'START', 'run': {'runId': '78faad9a-148b-47b5-8a1f-b8762b576c6e', 'facets': {'spark.logicalPlan': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet', 'plan': [{'class': 'org.apache.spark.sql.catalyst.plans.logical.AppendData', 'num-children': 1, 'table': [{'class': 'org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation', 'num-children': 0, 'table': None, 'output': [[{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'ID', 'dataType': 'long', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2553, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'FIRST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2554, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'LAST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2555, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}]], 'catalog': None, 'identifier': None, 'options': None}], 'query': 0, 'writeOptions': None, 'isByName': False, 'requireImplicitCasting': True}, {'class': 'org.apache.spark.sql.execution.datasources.LogicalRelation', 'num-children': 0, 'relation': None, 'output': [[{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'ID', 'dataType': 'long', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2550, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'FIRST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2551, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'LAST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2552, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}]], 'catalogTable': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogTable', 'identifier': {'product-class': 'org.apache.spark.sql.catalyst.TableIdentifier', 'table': 'jaffle_shop_customers', 'database': 'default', 'catalog': 'hive_metastore'}, 'tableType': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogTableType', 'name': 'MANAGED'}, 'storage': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogStorageFormat', 'locationUri': None, 'inputFormat': 'org.apache.hadoop.mapred.SequenceFileInputFormat', 'outputFormat': 'org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat', 'serde': 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe', 'compressed': False, 'properties': None}, 'schema': {'type': 'struct', 'fields': [{'name': 'ID', 'type': 'long', 'nullable': True, 'metadata': {}}, {'name': 'FIRST_NAME', 'type': 'string', 'nullable': True, 'metadata': {}}, {'name': 'LAST_NAME', 'type': 'string', 'nullable': True, 'metadata': {}}]}, 'provider': 'delta', 'partitionColumnNames': [], 'owner': 'root', 'createTime': 1696916365000, 'lastAccessTime': 0, 'createVersion': '3.3.2', 'properties': None, 'comment': 'Created by the file upload UI', 'unsupportedFeatures': [], 'tracksPartitionsInCatalog': True, 'schemaPreservesCase': True, 'ignoredProperties': None}, 'isStreaming': False}]}, 'spark_version': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet', 'spark-version': '3.1.2', 'openlineage-spark-version': '1.1.0'}, 'spark_properties': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet', 'properties': {'spark.master': 'spark://10.139.64.100:7077', 'spark.app.name': 'Databricks Shell'}}, 'environment-properties': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet', 'environment-properties': {'spark.databricks.clusterUsageTags.clusterName': 'Databook', 'spark.databricks.clusterUsageTags.azureSubscriptionId': '6b3f47d8-dd33-461f-bfbb-8c6c7b87fdfe', 'spark.databricks.notebook.path': '/Users/leela.kannan@maersk.com/2) SLA Notebooks - Leela/OpenLineage - Facet Test', 'mountPoints': [], 'spark.databricks.clusterUsageTags.clusterOwnerOrgId': '4598673832939986', 'user': 'simran.suri@maersk.com', 'userId': '5799628310045251', 'orgId': '4598673832939986'}}}}, 'job': {'namespace': 'default', 'name': 'adb-4598673832939986.6.azuredatabricks.net.append_data_exec_v1.dbt_simran_db_jaffle_shop_customers_trg', 'facets': {}}, 'inputs': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'facets': {'dataSource': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet', 'name': 'dbfs', 'uri': 'dbfs'}, 'schema': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet', 'fields': [{'name': 'ID', 'type': 'long'}, {'name': 'FIRST_NAME', 'type': 'string'}, {'name': 'LAST_NAME', 'type': 'string'}]}, 'symlinks': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SymlinksDatasetFacet.json#/$defs/SymlinksDatasetFacet', 'identifiers': [{'namespace': '/user/hive/warehouse', 'name': 'default.jaffle_shop_customers', 'type': 'TABLE'}]}}, 'inputFacets': {}}], 'outputs': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/dbt_simran.db/jaffle_shop_customers_trg', 'facets': {'dataSource': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet', 'name': 'dbfs', 'uri': 'dbfs'}, 'schema': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet', 'fields': [{'name': 'ID', 'type': 'long'}, {'name': 'FIRST_NAME', 'type': 'string'}, {'name': 'LAST_NAME', 'type': 'string'}]}, 'storage': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/StorageDatasetFacet.json#/$defs/StorageDatasetFacet', 'storageLayer': 'delta', 'fileFormat': 'parquet'}, 'columnLineage': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-1/ColumnLineageDatasetFacet.json#/$defs/ColumnLineageDatasetFacet', 'fields': {'ID': {'inputFields': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'field': 'ID'}]}, 'FIRST_NAME': {'inputFields': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'field': 'FIRST_NAME'}]}, 'LAST_NAME': {'inputFields': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'field': 'LAST_NAME'}]}}}, 'symlinks': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SymlinksDatasetFacet.json#/$defs/SymlinksDatasetFacet', 'identifiers': [{'namespace': '/user/hive/warehouse/dbt_simran.db', 'name': 'hive_metastore.dbt_simran.jaffle_shop_customers_trg', 'type': 'TABLE'}]}}, 'outputFacets': {}}]}

{'eventTime': '2023-12-11T06:59:23.576Z', 'producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', 'schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent', 'eventType': 'COMPLETE', 'run': {'runId': '78faad9a-148b-47b5-8a1f-b8762b576c6e', 'facets': {'spark.logicalPlan': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet', 'plan': [{'class': 'org.apache.spark.sql.catalyst.plans.logical.AppendData', 'num-children': 1, 'table': [{'class': 'org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation', 'num-children': 0, 'table': None, 'output': [[{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'ID', 'dataType': 'long', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2553, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'FIRST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2554, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'LAST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2555, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}]], 'catalog': None, 'identifier': None, 'options': None}], 'query': 0, 'writeOptions': None, 'isByName': False, 'requireImplicitCasting': True}, {'class': 'org.apache.spark.sql.execution.datasources.LogicalRelation', 'num-children': 0, 'relation': None, 'output': [[{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'ID', 'dataType': 'long', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2550, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'FIRST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2551, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'LAST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2552, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}]], 'catalogTable': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogTable', 'identifier': {'product-class': 'org.apache.spark.sql.catalyst.TableIdentifier', 'table': 'jaffle_shop_customers', 'database': 'default', 'catalog': 'hive_metastore'}, 'tableType': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogTableType', 'name': 'MANAGED'}, 'storage': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogStorageFormat', 'locationUri': None, 'inputFormat': 'org.apache.hadoop.mapred.SequenceFileInputFormat', 'outputFormat': 'org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat', 'serde': 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe', 'compressed': False, 'properties': None}, 'schema': {'type': 'struct', 'fields': [{'name': 'ID', 'type': 'long', 'nullable': True, 'metadata': {}}, {'name': 'FIRST_NAME', 'type': 'string', 'nullable': True, 'metadata': {}}, {'name': 'LAST_NAME', 'type': 'string', 'nullable': True, 'metadata': {}}]}, 'provider': 'delta', 'partitionColumnNames': [], 'owner': 'root', 'createTime': 1696916365000, 'lastAccessTime': 0, 'createVersion': '3.3.2', 'properties': None, 'comment': 'Created by the file upload UI', 'unsupportedFeatures': [], 'tracksPartitionsInCatalog': True, 'schemaPreservesCase': True, 'ignoredProperties': None}, 'isStreaming': False}]}, 'spark_version': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet', 'spark-version': '3.1.2', 'openlineage-spark-version': '1.1.0'}}}, 'job': {'namespace': 'default', 'name': 'adb-4598673832939986.6.azuredatabricks.net.append_data_exec_v1.dbt_simran_db_jaffle_shop_customers_trg', 'facets': {}}, 'inputs': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'facets': {'dataSource': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet', 'name': 'dbfs', 'uri': 'dbfs'}, 'schema': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet', 'fields': [{'name': 'ID', 'type': 'long'}, {'name': 'FIRST_NAME', 'type': 'string'}, {'name': 'LAST_NAME', 'type': 'string'}]}, 'symlinks': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SymlinksDatasetFacet.json#/$defs/SymlinksDatasetFacet', 'identifiers': [{'namespace': '/user/hive/warehouse', 'name': 'default.jaffle_shop_customers', 'type': 'TABLE'}]}}, 'inputFacets': {}}], 'outputs': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/dbt_simran.db/jaffle_shop_customers_trg', 'facets': {'dataSource': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet', 'name': 'dbfs', 'uri': 'dbfs'}, 'schema': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet', 'fields': [{'name': 'ID', 'type': 'long'}, {'name': 'FIRST_NAME', 'type': 'string'}, {'name': 'LAST_NAME', 'type': 'string'}]}, 'storage': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/StorageDatasetFacet.json#/$defs/StorageDatasetFacet', 'storageLayer': 'delta', 'fileFormat': 'parquet'}, 'columnLineage': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-1/ColumnLineageDatasetFacet.json#/$defs/ColumnLineageDatasetFacet', 'fields': {'ID': {'inputFields': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'field': 'ID'}]}, 'FIRST_NAME': {'inputFields': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'field': 'FIRST_NAME'}]}, 'LAST_NAME': {'inputFields': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'field': 'LAST_NAME'}]}}}, 'symlinks': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SymlinksDatasetFacet.json#/$defs/SymlinksDatasetFacet', 'identifiers': [{'namespace': '/user/hive/warehouse/dbt_simran.db', 'name': 'hive_metastore.dbt_simran.jaffle_shop_customers_trg', 'type': 'TABLE'}]}}, 'outputFacets': {}}]}

{'eventTime': '2023-12-11T06:59:23.586Z', 'producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', 'schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent', 'eventType': 'COMPLETE', 'run': {'runId': 'fd182451-49c4-4bca-88ed-205a6b7033d5', 'facets': {'spark.logicalPlan': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet', 'plan': [{'class': 'org.apache.spark.sql.catalyst.plans.logical.AppendData', 'num-children': 1, 'table': [{'class': 'org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation', 'num-children': 0, 'table': None, 'output': [[{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'ID', 'dataType': 'long', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2553, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'FIRST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2554, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'LAST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2555, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}]], 'catalog': None, 'identifier': None, 'options': None}], 'query': 0, 'writeOptions': None, 'isByName': False, 'requireImplicitCasting': True}, {'class': 'org.apache.spark.sql.execution.datasources.LogicalRelation', 'num-children': 0, 'relation': None, 'output': [[{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'ID', 'dataType': 'long', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2550, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'FIRST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2551, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}], [{'class': 'org.apache.spark.sql.catalyst.expressions.AttributeReference', 'num-children': 0, 'name': 'LAST_NAME', 'dataType': 'string', 'nullable': True, 'metadata': {}, 'exprId': {'product-class': 'org.apache.spark.sql.catalyst.expressions.ExprId', 'id': 2552, 'jvmId': 'd73313fa-d9d8-4245-84c7-406d999e6432'}, 'qualifier': []}]], 'catalogTable': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogTable', 'identifier': {'product-class': 'org.apache.spark.sql.catalyst.TableIdentifier', 'table': 'jaffle_shop_customers', 'database': 'default', 'catalog': 'hive_metastore'}, 'tableType': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogTableType', 'name': 'MANAGED'}, 'storage': {'product-class': 'org.apache.spark.sql.catalyst.catalog.CatalogStorageFormat', 'locationUri': None, 'inputFormat': 'org.apache.hadoop.mapred.SequenceFileInputFormat', 'outputFormat': 'org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat', 'serde': 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe', 'compressed': False, 'properties': None}, 'schema': {'type': 'struct', 'fields': [{'name': 'ID', 'type': 'long', 'nullable': True, 'metadata': {}}, {'name': 'FIRST_NAME', 'type': 'string', 'nullable': True, 'metadata': {}}, {'name': 'LAST_NAME', 'type': 'string', 'nullable': True, 'metadata': {}}]}, 'provider': 'delta', 'partitionColumnNames': [], 'owner': 'root', 'createTime': 1696916365000, 'lastAccessTime': 0, 'createVersion': '3.3.2', 'properties': None, 'comment': 'Created by the file upload UI', 'unsupportedFeatures': [], 'tracksPartitionsInCatalog': True, 'schemaPreservesCase': True, 'ignoredProperties': None}, 'isStreaming': False}]}, 'spark_version': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet', 'spark-version': '3.1.2', 'openlineage-spark-version': '1.1.0'}}}, 'job': {'namespace': 'default', 'name': 'adb-4598673832939986.6.azuredatabricks.net.append_data_exec_v1.dbt_simran_db_jaffle_shop_customers_trg', 'facets': {}}, 'inputs': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'facets': {'dataSource': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet', 'name': 'dbfs', 'uri': 'dbfs'}, 'schema': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet', 'fields': [{'name': 'ID', 'type': 'long'}, {'name': 'FIRST_NAME', 'type': 'string'}, {'name': 'LAST_NAME', 'type': 'string'}]}, 'symlinks': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SymlinksDatasetFacet.json#/$defs/SymlinksDatasetFacet', 'identifiers': [{'namespace': '/user/hive/warehouse', 'name': 'default.jaffle_shop_customers', 'type': 'TABLE'}]}}, 'inputFacets': {}}], 'outputs': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/dbt_simran.db/jaffle_shop_customers_trg', 'facets': {'dataSource': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet', 'name': 'dbfs', 'uri': 'dbfs'}, 'schema': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet', 'fields': [{'name': 'ID', 'type': 'long'}, {'name': 'FIRST_NAME', 'type': 'string'}, {'name': 'LAST_NAME', 'type': 'string'}]}, 'storage': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/StorageDatasetFacet.json#/$defs/StorageDatasetFacet', 'storageLayer': 'delta', 'fileFormat': 'parquet'}, 'columnLineage': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-1/ColumnLineageDatasetFacet.json#/$defs/ColumnLineageDatasetFacet', 'fields': {'ID': {'inputFields': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'field': 'ID'}]}, 'FIRST_NAME': {'inputFields': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'field': 'FIRST_NAME'}]}, 'LAST_NAME': {'inputFields': [{'namespace': 'dbfs', 'name': '/user/hive/warehouse/jaffle_shop_customers', 'field': 'LAST_NAME'}]}}}, 'symlinks': {'_producer': 'https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark', '_schemaURL': 'https://openlineage.io/spec/facets/1-0-0/SymlinksDatasetFacet.json#/$defs/SymlinksDatasetFacet', 'identifiers': [{'namespace': '/user/hive/warehouse/dbt_simran.db', 'name': 'hive_metastore.dbt_simran.jaffle_shop_customers_trg', 'type': 'TABLE'}]}}, 'outputFacets': {}}]}