{"eventTime": "2023-09-12T20:44:10.764Z", "producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent", "eventType": "START", "run": {"runId": "9293fb3d-bbe9-4237-b518-719a7c0f149d", "facets": {"spark.logicalPlan": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet", "plan": [{"class": "org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand", "num-children": 0, "query": [{"class": "org.apache.spark.sql.catalyst.plans.logical.Project", "num-children": 1, "projectList": [[{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "customer_id", "dataType": "integer", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 497, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.Alias", "num-children": 1, "child": 0, "name": "order_year", "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 507, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": [], "nonInheritableMetadataKeys": "[__dataset_id, __col_position]"}, {"class": "org.apache.spark.sql.catalyst.expressions.Year", "num-children": 1, "child": 0}, {"class": "org.apache.spark.sql.catalyst.expressions.Cast", "num-children": 1, "child": 0, "dataType": "date", "timeZoneId": "Etc/UTC", "evalMode": null}, {"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_date", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 499, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.Alias", "num-children": 1, "child": 0, "name": "order_month", "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 508, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": [], "nonInheritableMetadataKeys": "[__dataset_id, __col_position]"}, {"class": "org.apache.spark.sql.catalyst.expressions.Month", "num-children": 1, "child": 0}, {"class": "org.apache.spark.sql.catalyst.expressions.Cast", "num-children": 1, "child": 0, "dataType": "date", "timeZoneId": "Etc/UTC", "evalMode": null}, {"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_date", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 499, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.Alias", "num-children": 1, "child": 0, "name": "order_day", "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 509, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": [], "nonInheritableMetadataKeys": "[__dataset_id, __col_position]"}, {"class": "org.apache.spark.sql.catalyst.expressions.DayOfMonth", "num-children": 1, "child": 0}, {"class": "org.apache.spark.sql.catalyst.expressions.Cast", "num-children": 1, "child": 0, "dataType": "date", "timeZoneId": "Etc/UTC", "evalMode": null}, {"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_date", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 499, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.Alias", "num-children": 1, "child": 0, "name": "order_amount", "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 510, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": [], "nonInheritableMetadataKeys": "[__dataset_id, __col_position]"}, {"class": "org.apache.spark.sql.catalyst.expressions.Cast", "num-children": 1, "child": 0, "dataType": "float", "timeZoneId": "Etc/UTC", "evalMode": null}, {"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_amount", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 500, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.Alias", "num-children": 1, "child": 0, "name": "order_key", "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 511, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": [], "nonInheritableMetadataKeys": "[__dataset_id, __col_position]"}, {"class": "org.apache.spark.sql.catalyst.expressions.ConcatWs", "num-children": 3, "children": [0, 1, 2]}, {"class": "org.apache.spark.sql.catalyst.expressions.Literal", "num-children": 0, "value": "-", "dataType": "string"}, {"class": "org.apache.spark.sql.catalyst.expressions.Cast", "num-children": 1, "child": 0, "dataType": "string", "timeZoneId": "Etc/UTC", "evalMode": null}, {"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "customer_id", "dataType": "integer", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 497, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}, {"class": "org.apache.spark.sql.catalyst.expressions.Cast", "num-children": 1, "child": 0, "dataType": "string", "timeZoneId": "Etc/UTC", "evalMode": null}, {"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_id", "dataType": "integer", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 498, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "product_name", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 501, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}]], "child": 0}, {"class": "org.apache.spark.sql.execution.datasources.LogicalRelation", "num-children": 0, "relation": null, "output": [[{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "customer_id", "dataType": "integer", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 497, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_id", "dataType": "integer", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 498, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_date", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 499, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_amount", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 500, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "product_name", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 501, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}]], "isStreaming": false}], "dataSource": null, "options": null, "mode": null}]}, "spark_version": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet", "spark-version": "3.4.1", "openlineage-spark-version": "1.1.0"}, "environment-properties": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet", "environment-properties": {}}}}, "job": {"namespace": "staging", "name": "etl_test.execute_save_into_data_source_command.atlan_orders", "facets": {}}, "inputs": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "facets": {"dataSource": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet", "name": "file", "uri": "file"}, "schema": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet", "fields": [{"name": "customer_id", "type": "integer"}, {"name": "order_id", "type": "integer"}, {"name": "order_date", "type": "string"}, {"name": "order_amount", "type": "string"}, {"name": "product_name", "type": "string"}]}}, "inputFacets": {}}], "outputs": [{"namespace": "mysql://atlan-mysql.crmgvlgwn1cx.ap-south-1.rds.amazonaws.com:3306", "name": "atlan.orders", "facets": {"dataSource": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet", "name": "mysql://atlan-mysql.crmgvlgwn1cx.ap-south-1.rds.amazonaws.com:3306", "uri": "mysql://atlan-mysql.crmgvlgwn1cx.ap-south-1.rds.amazonaws.com:3306"}, "schema": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet", "fields": [{"name": "customer_id", "type": "integer"}, {"name": "order_year", "type": "integer"}, {"name": "order_month", "type": "integer"}, {"name": "order_day", "type": "integer"}, {"name": "order_amount", "type": "integer"}, {"name": "order_key", "type": "string"}, {"name": "product_name", "type": "string"}]}, "columnLineage": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/ColumnLineageDatasetFacet.json#/$defs/ColumnLineageDatasetFacet", "fields": {"customer_id": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "customer_id"}]}, "order_year": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "order_date"}]}, "order_month": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "order_date"}]}, "order_day": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "order_date"}]}, "order_amount": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "order_amount"}]}, "order_key": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "customer_id"}, {"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "order_id"}]}, "product_name": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "product_name"}]}}}, "lifecycleStateChange": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/LifecycleStateChangeDatasetFacet.json#/$defs/LifecycleStateChangeDatasetFacet", "lifecycleStateChange": "OVERWRITE"}}, "outputFacets": {}}]}
{"eventTime": "2023-09-12T20:44:11.462Z", "producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent", "eventType": "START", "run": {"runId": "9293fb3d-bbe9-4237-b518-719a7c0f149d", "facets": {"spark.logicalPlan": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet", "plan": [{"class": "org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand", "num-children": 0, "query": [{"class": "org.apache.spark.sql.catalyst.plans.logical.Project", "num-children": 1, "projectList": [[{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "customer_id", "dataType": "integer", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 497, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.Alias", "num-children": 1, "child": 0, "name": "order_year", "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 507, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": [], "nonInheritableMetadataKeys": "[__dataset_id, __col_position]"}, {"class": "org.apache.spark.sql.catalyst.expressions.Year", "num-children": 1, "child": 0}, {"class": "org.apache.spark.sql.catalyst.expressions.Cast", "num-children": 1, "child": 0, "dataType": "date", "timeZoneId": "Etc/UTC", "evalMode": null}, {"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_date", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 499, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.Alias", "num-children": 1, "child": 0, "name": "order_month", "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 508, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": [], "nonInheritableMetadataKeys": "[__dataset_id, __col_position]"}, {"class": "org.apache.spark.sql.catalyst.expressions.Month", "num-children": 1, "child": 0}, {"class": "org.apache.spark.sql.catalyst.expressions.Cast", "num-children": 1, "child": 0, "dataType": "date", "timeZoneId": "Etc/UTC", "evalMode": null}, {"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_date", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 499, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.Alias", "num-children": 1, "child": 0, "name": "order_day", "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 509, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": [], "nonInheritableMetadataKeys": "[__dataset_id, __col_position]"}, {"class": "org.apache.spark.sql.catalyst.expressions.DayOfMonth", "num-children": 1, "child": 0}, {"class": "org.apache.spark.sql.catalyst.expressions.Cast", "num-children": 1, "child": 0, "dataType": "date", "timeZoneId": "Etc/UTC", "evalMode": null}, {"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_date", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 499, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.Alias", "num-children": 1, "child": 0, "name": "order_amount", "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 510, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": [], "nonInheritableMetadataKeys": "[__dataset_id, __col_position]"}, {"class": "org.apache.spark.sql.catalyst.expressions.Cast", "num-children": 1, "child": 0, "dataType": "float", "timeZoneId": "Etc/UTC", "evalMode": null}, {"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_amount", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 500, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.Alias", "num-children": 1, "child": 0, "name": "order_key", "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 511, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": [], "nonInheritableMetadataKeys": "[__dataset_id, __col_position]"}, {"class": "org.apache.spark.sql.catalyst.expressions.ConcatWs", "num-children": 3, "children": [0, 1, 2]}, {"class": "org.apache.spark.sql.catalyst.expressions.Literal", "num-children": 0, "value": "-", "dataType": "string"}, {"class": "org.apache.spark.sql.catalyst.expressions.Cast", "num-children": 1, "child": 0, "dataType": "string", "timeZoneId": "Etc/UTC", "evalMode": null}, {"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "customer_id", "dataType": "integer", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 497, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}, {"class": "org.apache.spark.sql.catalyst.expressions.Cast", "num-children": 1, "child": 0, "dataType": "string", "timeZoneId": "Etc/UTC", "evalMode": null}, {"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_id", "dataType": "integer", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 498, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "product_name", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 501, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}]], "child": 0}, {"class": "org.apache.spark.sql.execution.datasources.LogicalRelation", "num-children": 0, "relation": null, "output": [[{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "customer_id", "dataType": "integer", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 497, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_id", "dataType": "integer", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 498, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_date", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 499, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_amount", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 500, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "product_name", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 501, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}]], "isStreaming": false}], "dataSource": null, "options": null, "mode": null}]}, "spark_version": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet", "spark-version": "3.4.1", "openlineage-spark-version": "1.1.0"}, "spark_properties": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet", "properties": {"spark.master": "local", "spark.app.name": "etl-test"}}, "environment-properties": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet", "environment-properties": {}}}}, "job": {"namespace": "staging", "name": "etl_test.execute_save_into_data_source_command.atlan_orders", "facets": {}}, "inputs": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "facets": {"dataSource": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet", "name": "file", "uri": "file"}, "schema": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet", "fields": [{"name": "customer_id", "type": "integer"}, {"name": "order_id", "type": "integer"}, {"name": "order_date", "type": "string"}, {"name": "order_amount", "type": "string"}, {"name": "product_name", "type": "string"}]}}, "inputFacets": {}}], "outputs": [{"namespace": "mysql://atlan-mysql.crmgvlgwn1cx.ap-south-1.rds.amazonaws.com:3306", "name": "atlan.orders", "facets": {"dataSource": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet", "name": "mysql://atlan-mysql.crmgvlgwn1cx.ap-south-1.rds.amazonaws.com:3306", "uri": "mysql://atlan-mysql.crmgvlgwn1cx.ap-south-1.rds.amazonaws.com:3306"}, "schema": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet", "fields": [{"name": "customer_id", "type": "integer"}, {"name": "order_year", "type": "integer"}, {"name": "order_month", "type": "integer"}, {"name": "order_day", "type": "integer"}, {"name": "order_amount", "type": "integer"}, {"name": "order_key", "type": "string"}, {"name": "product_name", "type": "string"}]}, "columnLineage": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/ColumnLineageDatasetFacet.json#/$defs/ColumnLineageDatasetFacet", "fields": {"customer_id": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "customer_id"}]}, "order_year": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "order_date"}]}, "order_month": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "order_date"}]}, "order_day": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "order_date"}]}, "order_amount": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "order_amount"}]}, "order_key": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "customer_id"}, {"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "order_id"}]}, "product_name": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "product_name"}]}}}, "lifecycleStateChange": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/LifecycleStateChangeDatasetFacet.json#/$defs/LifecycleStateChangeDatasetFacet", "lifecycleStateChange": "OVERWRITE"}}, "outputFacets": {}}]}
{"eventTime": "2023-09-12T20:44:12.106Z", "producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent", "eventType": "COMPLETE", "run": {"runId": "9293fb3d-bbe9-4237-b518-719a7c0f149d", "facets": {"spark.logicalPlan": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet", "plan": [{"class": "org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand", "num-children": 0, "query": [{"class": "org.apache.spark.sql.catalyst.plans.logical.Project", "num-children": 1, "projectList": [[{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "customer_id", "dataType": "integer", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 497, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.Alias", "num-children": 1, "child": 0, "name": "order_year", "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 507, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": [], "nonInheritableMetadataKeys": "[__dataset_id, __col_position]"}, {"class": "org.apache.spark.sql.catalyst.expressions.Year", "num-children": 1, "child": 0}, {"class": "org.apache.spark.sql.catalyst.expressions.Cast", "num-children": 1, "child": 0, "dataType": "date", "timeZoneId": "Etc/UTC", "evalMode": null}, {"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_date", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 499, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.Alias", "num-children": 1, "child": 0, "name": "order_month", "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 508, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": [], "nonInheritableMetadataKeys": "[__dataset_id, __col_position]"}, {"class": "org.apache.spark.sql.catalyst.expressions.Month", "num-children": 1, "child": 0}, {"class": "org.apache.spark.sql.catalyst.expressions.Cast", "num-children": 1, "child": 0, "dataType": "date", "timeZoneId": "Etc/UTC", "evalMode": null}, {"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_date", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 499, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.Alias", "num-children": 1, "child": 0, "name": "order_day", "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 509, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": [], "nonInheritableMetadataKeys": "[__dataset_id, __col_position]"}, {"class": "org.apache.spark.sql.catalyst.expressions.DayOfMonth", "num-children": 1, "child": 0}, {"class": "org.apache.spark.sql.catalyst.expressions.Cast", "num-children": 1, "child": 0, "dataType": "date", "timeZoneId": "Etc/UTC", "evalMode": null}, {"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_date", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 499, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.Alias", "num-children": 1, "child": 0, "name": "order_amount", "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 510, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": [], "nonInheritableMetadataKeys": "[__dataset_id, __col_position]"}, {"class": "org.apache.spark.sql.catalyst.expressions.Cast", "num-children": 1, "child": 0, "dataType": "float", "timeZoneId": "Etc/UTC", "evalMode": null}, {"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_amount", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 500, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.Alias", "num-children": 1, "child": 0, "name": "order_key", "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 511, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": [], "nonInheritableMetadataKeys": "[__dataset_id, __col_position]"}, {"class": "org.apache.spark.sql.catalyst.expressions.ConcatWs", "num-children": 3, "children": [0, 1, 2]}, {"class": "org.apache.spark.sql.catalyst.expressions.Literal", "num-children": 0, "value": "-", "dataType": "string"}, {"class": "org.apache.spark.sql.catalyst.expressions.Cast", "num-children": 1, "child": 0, "dataType": "string", "timeZoneId": "Etc/UTC", "evalMode": null}, {"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "customer_id", "dataType": "integer", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 497, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}, {"class": "org.apache.spark.sql.catalyst.expressions.Cast", "num-children": 1, "child": 0, "dataType": "string", "timeZoneId": "Etc/UTC", "evalMode": null}, {"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_id", "dataType": "integer", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 498, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "product_name", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 501, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}]], "child": 0}, {"class": "org.apache.spark.sql.execution.datasources.LogicalRelation", "num-children": 0, "relation": null, "output": [[{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "customer_id", "dataType": "integer", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 497, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_id", "dataType": "integer", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 498, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_date", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 499, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_amount", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 500, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "product_name", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 501, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}]], "isStreaming": false}], "dataSource": null, "options": null, "mode": null}]}, "spark_version": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet", "spark-version": "3.4.1", "openlineage-spark-version": "1.1.0"}, "environment-properties": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet", "environment-properties": {}}}}, "job": {"namespace": "staging", "name": "etl_test.execute_save_into_data_source_command.atlan_orders", "facets": {}}, "inputs": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "facets": {"dataSource": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet", "name": "file", "uri": "file"}, "schema": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet", "fields": [{"name": "customer_id", "type": "integer"}, {"name": "order_id", "type": "integer"}, {"name": "order_date", "type": "string"}, {"name": "order_amount", "type": "string"}, {"name": "product_name", "type": "string"}]}}, "inputFacets": {}}], "outputs": [{"namespace": "mysql://atlan-mysql.crmgvlgwn1cx.ap-south-1.rds.amazonaws.com:3306", "name": "atlan.orders", "facets": {"dataSource": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet", "name": "mysql://atlan-mysql.crmgvlgwn1cx.ap-south-1.rds.amazonaws.com:3306", "uri": "mysql://atlan-mysql.crmgvlgwn1cx.ap-south-1.rds.amazonaws.com:3306"}, "schema": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet", "fields": [{"name": "customer_id", "type": "integer"}, {"name": "order_year", "type": "integer"}, {"name": "order_month", "type": "integer"}, {"name": "order_day", "type": "integer"}, {"name": "order_amount", "type": "integer"}, {"name": "order_key", "type": "string"}, {"name": "product_name", "type": "string"}]}, "columnLineage": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/ColumnLineageDatasetFacet.json#/$defs/ColumnLineageDatasetFacet", "fields": {"customer_id": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "customer_id"}]}, "order_year": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "order_date"}]}, "order_month": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "order_date"}]}, "order_day": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "order_date"}]}, "order_amount": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "order_amount"}]}, "order_key": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "customer_id"}, {"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "order_id"}]}, "product_name": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "product_name"}]}}}, "lifecycleStateChange": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/LifecycleStateChangeDatasetFacet.json#/$defs/LifecycleStateChangeDatasetFacet", "lifecycleStateChange": "OVERWRITE"}}, "outputFacets": {"outputStatistics": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/OutputStatisticsOutputDatasetFacet.json#/$defs/OutputStatisticsOutputDatasetFacet", "rowCount": 1, "size": 0}}}]}
{"eventTime": "2023-09-12T20:44:12.518Z", "producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent", "eventType": "COMPLETE", "run": {"runId": "9293fb3d-bbe9-4237-b518-719a7c0f149d", "facets": {"spark.logicalPlan": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet", "plan": [{"class": "org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand", "num-children": 0, "query": [{"class": "org.apache.spark.sql.catalyst.plans.logical.Project", "num-children": 1, "projectList": [[{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "customer_id", "dataType": "integer", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 497, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.Alias", "num-children": 1, "child": 0, "name": "order_year", "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 507, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": [], "nonInheritableMetadataKeys": "[__dataset_id, __col_position]"}, {"class": "org.apache.spark.sql.catalyst.expressions.Year", "num-children": 1, "child": 0}, {"class": "org.apache.spark.sql.catalyst.expressions.Cast", "num-children": 1, "child": 0, "dataType": "date", "timeZoneId": "Etc/UTC", "evalMode": null}, {"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_date", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 499, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.Alias", "num-children": 1, "child": 0, "name": "order_month", "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 508, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": [], "nonInheritableMetadataKeys": "[__dataset_id, __col_position]"}, {"class": "org.apache.spark.sql.catalyst.expressions.Month", "num-children": 1, "child": 0}, {"class": "org.apache.spark.sql.catalyst.expressions.Cast", "num-children": 1, "child": 0, "dataType": "date", "timeZoneId": "Etc/UTC", "evalMode": null}, {"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_date", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 499, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.Alias", "num-children": 1, "child": 0, "name": "order_day", "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 509, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": [], "nonInheritableMetadataKeys": "[__dataset_id, __col_position]"}, {"class": "org.apache.spark.sql.catalyst.expressions.DayOfMonth", "num-children": 1, "child": 0}, {"class": "org.apache.spark.sql.catalyst.expressions.Cast", "num-children": 1, "child": 0, "dataType": "date", "timeZoneId": "Etc/UTC", "evalMode": null}, {"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_date", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 499, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.Alias", "num-children": 1, "child": 0, "name": "order_amount", "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 510, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": [], "nonInheritableMetadataKeys": "[__dataset_id, __col_position]"}, {"class": "org.apache.spark.sql.catalyst.expressions.Cast", "num-children": 1, "child": 0, "dataType": "float", "timeZoneId": "Etc/UTC", "evalMode": null}, {"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_amount", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 500, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.Alias", "num-children": 1, "child": 0, "name": "order_key", "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 511, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": [], "nonInheritableMetadataKeys": "[__dataset_id, __col_position]"}, {"class": "org.apache.spark.sql.catalyst.expressions.ConcatWs", "num-children": 3, "children": [0, 1, 2]}, {"class": "org.apache.spark.sql.catalyst.expressions.Literal", "num-children": 0, "value": "-", "dataType": "string"}, {"class": "org.apache.spark.sql.catalyst.expressions.Cast", "num-children": 1, "child": 0, "dataType": "string", "timeZoneId": "Etc/UTC", "evalMode": null}, {"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "customer_id", "dataType": "integer", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 497, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}, {"class": "org.apache.spark.sql.catalyst.expressions.Cast", "num-children": 1, "child": 0, "dataType": "string", "timeZoneId": "Etc/UTC", "evalMode": null}, {"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_id", "dataType": "integer", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 498, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "product_name", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 501, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}]], "child": 0}, {"class": "org.apache.spark.sql.execution.datasources.LogicalRelation", "num-children": 0, "relation": null, "output": [[{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "customer_id", "dataType": "integer", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 497, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_id", "dataType": "integer", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 498, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_date", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 499, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "order_amount", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 500, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}], [{"class": "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children": 0, "name": "product_name", "dataType": "string", "nullable": true, "metadata": {}, "exprId": {"product-class": "org.apache.spark.sql.catalyst.expressions.ExprId", "id": 501, "jvmId": "bca387c7-9171-4d47-8061-7031cec5e75f"}, "qualifier": []}]], "isStreaming": false}], "dataSource": null, "options": null, "mode": null}]}, "spark_version": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet", "spark-version": "3.4.1", "openlineage-spark-version": "1.1.0"}, "environment-properties": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet", "environment-properties": {}}}}, "job": {"namespace": "staging", "name": "etl_test.execute_save_into_data_source_command.atlan_orders", "facets": {}}, "inputs": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "facets": {"dataSource": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet", "name": "file", "uri": "file"}, "schema": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet", "fields": [{"name": "customer_id", "type": "integer"}, {"name": "order_id", "type": "integer"}, {"name": "order_date", "type": "string"}, {"name": "order_amount", "type": "string"}, {"name": "product_name", "type": "string"}]}}, "inputFacets": {}}], "outputs": [{"namespace": "mysql://atlan-mysql.crmgvlgwn1cx.ap-south-1.rds.amazonaws.com:3306", "name": "atlan.orders", "facets": {"dataSource": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet", "name": "mysql://atlan-mysql.crmgvlgwn1cx.ap-south-1.rds.amazonaws.com:3306", "uri": "mysql://atlan-mysql.crmgvlgwn1cx.ap-south-1.rds.amazonaws.com:3306"}, "schema": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet", "fields": [{"name": "customer_id", "type": "integer"}, {"name": "order_year", "type": "integer"}, {"name": "order_month", "type": "integer"}, {"name": "order_day", "type": "integer"}, {"name": "order_amount", "type": "integer"}, {"name": "order_key", "type": "string"}, {"name": "product_name", "type": "string"}]}, "columnLineage": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/ColumnLineageDatasetFacet.json#/$defs/ColumnLineageDatasetFacet", "fields": {"customer_id": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "customer_id"}]}, "order_year": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "order_date"}]}, "order_month": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "order_date"}]}, "order_day": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "order_date"}]}, "order_amount": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "order_amount"}]}, "order_key": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "customer_id"}, {"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "order_id"}]}, "product_name": {"inputFields": [{"namespace": "file", "name": "/home/jovyan/notebooks/input.json", "field": "product_name"}]}}}, "lifecycleStateChange": {"_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.1.0/integration/spark", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/LifecycleStateChangeDatasetFacet.json#/$defs/LifecycleStateChangeDatasetFacet", "lifecycleStateChange": "OVERWRITE"}}, "outputFacets": {}}]}