-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathingest_new_flight_data.json
1 lines (1 loc) · 42.5 KB
/
ingest_new_flight_data.json
1
{"flowContents":{"identifier":"9ea1816a-c813-3787-b2b1-c19069522334","instanceIdentifier":"ed44f063-018b-1000-0000-0000462dd380","name":"New_Flight_Data_to_Flights_Iceberg_Table","comments":"","position":{"x":544.0,"y":56.0},"processGroups":[],"remoteProcessGroups":[],"processors":[{"identifier":"05b86a8e-3730-39d9-b0a5-06004c90858b","instanceIdentifier":"0f9a51dd-5f54-36f8-b1a3-4082c6c20de0","name":"ConvertRecord","comments":"","position":{"x":1336.0,"y":40.0},"type":"org.apache.nifi.processors.standard.ConvertRecord","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-nar","version":"1.21.0.2.2.7.100-3"},"properties":{"record-writer":"93b5ac02-9a21-351c-a290-c346b242899a","record-reader":"07053613-68bb-319a-a2d2-2a87d91bd95b","include-zero-record-flowfiles":"true"},"propertyDescriptors":{"record-writer":{"name":"record-writer","displayName":"Record Writer","identifiesControllerService":true,"sensitive":false},"record-reader":{"name":"record-reader","displayName":"Record Reader","identifiesControllerService":true,"sensitive":false},"include-zero-record-flowfiles":{"name":"include-zero-record-flowfiles","displayName":"Include Zero Record FlowFiles","identifiesControllerService":false,"sensitive":false}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["failure"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"9ea1816a-c813-3787-b2b1-c19069522334"},{"identifier":"9fa7f16b-d3fe-35c8-8ae1-99841d0ea494","instanceIdentifier":"dfe6840e-a526-3870-86de-e21b72a97cae","name":"Only Files RouteOnAttribute","comments":"","position":{"x":440.0,"y":296.0},"type":"org.apache.nifi.processors.standard.RouteOnAttribute","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-nar","version":"1.21.0.2.2.7.100-3"},"properties":{"Routing Strategy":"Route to Property name","FilesOnly":"${s3.length:gt(0)}"},"propertyDescriptors":{"Routing Strategy":{"name":"Routing Strategy","displayName":"Routing Strategy","identifiesControllerService":false,"sensitive":false},"FilesOnly":{"name":"FilesOnly","displayName":"FilesOnly","identifiesControllerService":false,"sensitive":false}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":25,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["unmatched"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"9ea1816a-c813-3787-b2b1-c19069522334"},{"identifier":"1f00d975-886d-302c-87ec-6841ba3b9102","instanceIdentifier":"cc24cf05-5686-360b-8d74-244183d625d1","name":"List New Flight Data Files ListS3","comments":"","position":{"x":424.0,"y":80.0},"type":"org.apache.nifi.processors.aws.s3.ListS3","bundle":{"group":"org.apache.nifi","artifact":"nifi-aws-nar","version":"1.21.0.2.2.7.100-3"},"properties":{"Proxy Host":null,"list-type":"1","et-initial-listing-target":"all","Access Key":"#{Access Key ID}","prefix":"#{Prefix}","record-writer":null,"proxy-configuration-service":null,"Endpoint Override URL":null,"et-time-window":"3 hours","custom-signer-class-name":null,"requester-pays":"false","custom-signer-module-location":null,"listing-strategy":"timestamps","Signer Override":"Default Signature","delimiter":null,"Secret Key":"#{Secret Access Key}","min-age":"0 sec","write-s3-user-metadata":"false","max-age":null,"AWS Credentials Provider service":null,"Listing Batch Size":"100","SSL Context Service":null,"write-s3-object-tags":"false","Bucket":"#{Bucket}","Communications Timeout":"30 secs","Region":"us-east-2","Credentials File":null,"et-state-cache":null,"proxy-user-name":null,"Proxy Host Port":null,"use-versions":"false"},"propertyDescriptors":{"Proxy Host":{"name":"Proxy Host","displayName":"Proxy Host","identifiesControllerService":false,"sensitive":false},"list-type":{"name":"list-type","displayName":"List Type","identifiesControllerService":false,"sensitive":false},"et-initial-listing-target":{"name":"et-initial-listing-target","displayName":"Entity Tracking Initial Listing Target","identifiesControllerService":false,"sensitive":false},"Access Key":{"name":"Access Key","displayName":"Access Key ID","identifiesControllerService":false,"sensitive":true},"prefix":{"name":"prefix","displayName":"Prefix","identifiesControllerService":false,"sensitive":false},"record-writer":{"name":"record-writer","displayName":"Record Writer","identifiesControllerService":true,"sensitive":false},"proxy-configuration-service":{"name":"proxy-configuration-service","displayName":"Proxy Configuration Service","identifiesControllerService":true,"sensitive":false},"Endpoint Override URL":{"name":"Endpoint Override URL","displayName":"Endpoint Override URL","identifiesControllerService":false,"sensitive":false},"et-time-window":{"name":"et-time-window","displayName":"Entity Tracking Time Window","identifiesControllerService":false,"sensitive":false},"custom-signer-class-name":{"name":"custom-signer-class-name","displayName":"Custom Signer Class Name","identifiesControllerService":false,"sensitive":false},"requester-pays":{"name":"requester-pays","displayName":"Requester Pays","identifiesControllerService":false,"sensitive":false},"custom-signer-module-location":{"name":"custom-signer-module-location","displayName":"Custom Signer Module Location","identifiesControllerService":false,"sensitive":false,"resourceDefinition":{"cardinality":"MULTIPLE","resourceTypes":["DIRECTORY","FILE"]}},"listing-strategy":{"name":"listing-strategy","displayName":"Listing Strategy","identifiesControllerService":false,"sensitive":false},"Signer Override":{"name":"Signer Override","displayName":"Signer Override","identifiesControllerService":false,"sensitive":false},"delimiter":{"name":"delimiter","displayName":"Delimiter","identifiesControllerService":false,"sensitive":false},"proxy-user-password":{"name":"proxy-user-password","displayName":"Proxy Password","identifiesControllerService":false,"sensitive":true},"Secret Key":{"name":"Secret Key","displayName":"Secret Access Key","identifiesControllerService":false,"sensitive":true},"min-age":{"name":"min-age","displayName":"Minimum Object Age","identifiesControllerService":false,"sensitive":false},"write-s3-user-metadata":{"name":"write-s3-user-metadata","displayName":"Write User Metadata","identifiesControllerService":false,"sensitive":false},"max-age":{"name":"max-age","displayName":"Maximum Object Age","identifiesControllerService":false,"sensitive":false},"AWS Credentials Provider service":{"name":"AWS Credentials Provider service","displayName":"AWS Credentials Provider Service","identifiesControllerService":true,"sensitive":false},"Listing Batch Size":{"name":"Listing Batch Size","displayName":"Listing Batch Size","identifiesControllerService":false,"sensitive":false},"SSL Context Service":{"name":"SSL Context Service","displayName":"SSL Context Service","identifiesControllerService":true,"sensitive":false},"write-s3-object-tags":{"name":"write-s3-object-tags","displayName":"Write Object Tags","identifiesControllerService":false,"sensitive":false},"Bucket":{"name":"Bucket","displayName":"Bucket","identifiesControllerService":false,"sensitive":false},"Communications Timeout":{"name":"Communications Timeout","displayName":"Communications Timeout","identifiesControllerService":false,"sensitive":false},"Region":{"name":"Region","displayName":"Region","identifiesControllerService":false,"sensitive":false},"Credentials File":{"name":"Credentials File","displayName":"Credentials File","identifiesControllerService":false,"sensitive":false,"resourceDefinition":{"cardinality":"SINGLE","resourceTypes":["FILE"]}},"et-state-cache":{"name":"et-state-cache","displayName":"Entity Tracking State Cache","identifiesControllerService":true,"sensitive":false},"proxy-user-name":{"name":"proxy-user-name","displayName":"Proxy Username","identifiesControllerService":false,"sensitive":false},"Proxy Host Port":{"name":"Proxy Host Port","displayName":"Proxy Host Port","identifiesControllerService":false,"sensitive":false},"use-versions":{"name":"use-versions","displayName":"Use Versions","identifiesControllerService":false,"sensitive":false}},"style":{},"schedulingPeriod":"1 min","schedulingStrategy":"TIMER_DRIVEN","executionNode":"PRIMARY","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":[],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"9ea1816a-c813-3787-b2b1-c19069522334"},{"identifier":"bb0abc17-6d89-3ee2-97db-a08c441e134a","instanceIdentifier":"f7b2e2a9-8463-33c0-8a35-45fe6647d108","name":"UnpackContent","comments":"","position":{"x":952.0,"y":296.0},"type":"org.apache.nifi.processors.standard.UnpackContent","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-nar","version":"1.21.0.2.2.7.100-3"},"properties":{"Packaging Format":"use mime.type attribute","File Filter":".*"},"propertyDescriptors":{"Packaging Format":{"name":"Packaging Format","displayName":"Packaging Format","identifiesControllerService":false,"sensitive":false},"File Filter":{"name":"File Filter","displayName":"File Filter","identifiesControllerService":false,"sensitive":false},"Password":{"name":"Password","displayName":"Password","identifiesControllerService":false,"sensitive":true}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["original","failure"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"9ea1816a-c813-3787-b2b1-c19069522334"},{"identifier":"0646be18-05d2-372f-87f6-944a7da91bde","instanceIdentifier":"83967228-c828-3c74-ae68-dd6bf592314d","name":"Get New Flight Data Files FetchS3","comments":"","position":{"x":440.0,"y":536.0},"type":"org.apache.nifi.processors.aws.s3.FetchS3Object","bundle":{"group":"org.apache.nifi","artifact":"nifi-aws-nar","version":"1.21.0.2.2.7.100-3"},"properties":{"Proxy Host":null,"range-length":null,"Access Key":"#{Access Key ID}","range-start":null,"proxy-configuration-service":null,"Endpoint Override URL":null,"custom-signer-class-name":null,"requester-pays":"false","Object Key":"${filename}","AWS Credentials Provider service":null,"custom-signer-module-location":null,"SSL Context Service":null,"Signer Override":"Default Signature","Bucket":"#{Bucket}","Version":null,"Communications Timeout":"30 secs","Region":"us-east-2","Secret Key":"#{Secret Access Key}","Credentials File":null,"proxy-user-name":null,"Proxy Host Port":null,"encryption-service":null},"propertyDescriptors":{"Proxy Host":{"name":"Proxy Host","displayName":"Proxy Host","identifiesControllerService":false,"sensitive":false},"range-length":{"name":"range-length","displayName":"Range Length","identifiesControllerService":false,"sensitive":false},"Access Key":{"name":"Access Key","displayName":"Access Key ID","identifiesControllerService":false,"sensitive":true},"range-start":{"name":"range-start","displayName":"Range Start","identifiesControllerService":false,"sensitive":false},"proxy-configuration-service":{"name":"proxy-configuration-service","displayName":"Proxy Configuration Service","identifiesControllerService":true,"sensitive":false},"Endpoint Override URL":{"name":"Endpoint Override URL","displayName":"Endpoint Override URL","identifiesControllerService":false,"sensitive":false},"custom-signer-class-name":{"name":"custom-signer-class-name","displayName":"Custom Signer Class Name","identifiesControllerService":false,"sensitive":false},"requester-pays":{"name":"requester-pays","displayName":"Requester Pays","identifiesControllerService":false,"sensitive":false},"Object Key":{"name":"Object Key","displayName":"Object Key","identifiesControllerService":false,"sensitive":false},"AWS Credentials Provider service":{"name":"AWS Credentials Provider service","displayName":"AWS Credentials Provider Service","identifiesControllerService":true,"sensitive":false},"custom-signer-module-location":{"name":"custom-signer-module-location","displayName":"Custom Signer Module Location","identifiesControllerService":false,"sensitive":false,"resourceDefinition":{"cardinality":"MULTIPLE","resourceTypes":["DIRECTORY","FILE"]}},"SSL Context Service":{"name":"SSL Context Service","displayName":"SSL Context Service","identifiesControllerService":true,"sensitive":false},"Signer Override":{"name":"Signer Override","displayName":"Signer Override","identifiesControllerService":false,"sensitive":false},"Bucket":{"name":"Bucket","displayName":"Bucket","identifiesControllerService":false,"sensitive":false},"Version":{"name":"Version","displayName":"Version","identifiesControllerService":false,"sensitive":false},"proxy-user-password":{"name":"proxy-user-password","displayName":"Proxy Password","identifiesControllerService":false,"sensitive":true},"Communications Timeout":{"name":"Communications Timeout","displayName":"Communications Timeout","identifiesControllerService":false,"sensitive":false},"Region":{"name":"Region","displayName":"Region","identifiesControllerService":false,"sensitive":false},"Secret Key":{"name":"Secret Key","displayName":"Secret Access Key","identifiesControllerService":false,"sensitive":true},"Credentials File":{"name":"Credentials File","displayName":"Credentials File","identifiesControllerService":false,"sensitive":false,"resourceDefinition":{"cardinality":"SINGLE","resourceTypes":["FILE"]}},"proxy-user-name":{"name":"proxy-user-name","displayName":"Proxy Username","identifiesControllerService":false,"sensitive":false},"Proxy Host Port":{"name":"Proxy Host Port","displayName":"Proxy Host Port","identifiesControllerService":false,"sensitive":false},"encryption-service":{"name":"encryption-service","displayName":"Encryption Service","identifiesControllerService":true,"sensitive":false}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["failure"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"9ea1816a-c813-3787-b2b1-c19069522334"},{"identifier":"4f1fda20-02dd-3924-adc9-2f1c585d765c","instanceIdentifier":"43a14ad6-5d8a-3b53-8080-122bf0b66570","name":"PutIceberg","comments":"","position":{"x":1648.0,"y":384.0},"type":"org.apache.nifi.processors.iceberg.PutIceberg","bundle":{"group":"org.apache.nifi","artifact":"nifi-iceberg-processors-nar","version":"1.21.0.2.2.7.100-3"},"properties":{"maximum-file-size":null,"number-of-commit-retries":"10","maximum-commit-duration":"30 sec","maximum-commit-wait-time":"2 sec","file-format":"PARQUET","catalog-service":"3e066fdf-31b9-33f6-991f-5e286bba8c5b","record-reader":"34bcbbd8-e06c-303b-8695-dc59d0c5506e","table-name":"#{Table Name}","kerberos-user-service":"ddaa59b4-5d10-340c-8c2f-4df67ab119e4","minimum-commit-wait-time":"100 ms","catalog-namespace":"#{Database}"},"propertyDescriptors":{"maximum-file-size":{"name":"maximum-file-size","displayName":"Maximum File Size","identifiesControllerService":false,"sensitive":false},"number-of-commit-retries":{"name":"number-of-commit-retries","displayName":"Number of Commit Retries","identifiesControllerService":false,"sensitive":false},"maximum-commit-duration":{"name":"maximum-commit-duration","displayName":"Maximum Commit Duration","identifiesControllerService":false,"sensitive":false},"maximum-commit-wait-time":{"name":"maximum-commit-wait-time","displayName":"Maximum Commit Wait Time","identifiesControllerService":false,"sensitive":false},"file-format":{"name":"file-format","displayName":"File Format","identifiesControllerService":false,"sensitive":false},"catalog-service":{"name":"catalog-service","displayName":"Catalog Service","identifiesControllerService":true,"sensitive":false},"record-reader":{"name":"record-reader","displayName":"Record Reader","identifiesControllerService":true,"sensitive":false},"table-name":{"name":"table-name","displayName":"Table Name","identifiesControllerService":false,"sensitive":false},"kerberos-user-service":{"name":"kerberos-user-service","displayName":"Kerberos User Service","identifiesControllerService":true,"sensitive":false},"minimum-commit-wait-time":{"name":"minimum-commit-wait-time","displayName":"Minimum Commit Wait Time","identifiesControllerService":false,"sensitive":false},"catalog-namespace":{"name":"catalog-namespace","displayName":"Catalog Namespace","identifiesControllerService":false,"sensitive":false}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["success"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"9ea1816a-c813-3787-b2b1-c19069522334"}],"inputPorts":[],"outputPorts":[],"connections":[{"identifier":"fbf30162-a546-341a-af7a-69347b49fff2","instanceIdentifier":"4f76b268-c1f3-3727-a111-220e7cd9952e","name":"","source":{"id":"bb0abc17-6d89-3ee2-97db-a08c441e134a","type":"PROCESSOR","groupId":"9ea1816a-c813-3787-b2b1-c19069522334","name":"UnpackContent","comments":"","instanceIdentifier":"f7b2e2a9-8463-33c0-8a35-45fe6647d108"},"destination":{"id":"05b86a8e-3730-39d9-b0a5-06004c90858b","type":"PROCESSOR","groupId":"9ea1816a-c813-3787-b2b1-c19069522334","name":"ConvertRecord","comments":"","instanceIdentifier":"0f9a51dd-5f54-36f8-b1a3-4082c6c20de0"},"labelIndex":1,"zIndex":0,"selectedRelationships":["success"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"9ea1816a-c813-3787-b2b1-c19069522334"},{"identifier":"0e43c614-e10e-349e-bb01-275de4447305","instanceIdentifier":"a9e55055-9e09-3676-9dbf-d163479c8e1e","name":"","source":{"id":"4f1fda20-02dd-3924-adc9-2f1c585d765c","type":"PROCESSOR","groupId":"9ea1816a-c813-3787-b2b1-c19069522334","name":"PutIceberg","comments":"","instanceIdentifier":"43a14ad6-5d8a-3b53-8080-122bf0b66570"},"destination":{"id":"22c007c5-4bf2-37da-9ba8-47cbd333fc5f","type":"FUNNEL","groupId":"9ea1816a-c813-3787-b2b1-c19069522334","name":"Funnel","comments":"","instanceIdentifier":"73f79213-1d21-333e-a8ed-bfe161ee4669"},"labelIndex":1,"zIndex":0,"selectedRelationships":["failure"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"9ea1816a-c813-3787-b2b1-c19069522334"},{"identifier":"b8de065d-985f-347a-a9dd-dc7fc78c1635","instanceIdentifier":"27cfabf2-9c1b-3658-8ef8-ea79f83c7d45","name":"","source":{"id":"9fa7f16b-d3fe-35c8-8ae1-99841d0ea494","type":"PROCESSOR","groupId":"9ea1816a-c813-3787-b2b1-c19069522334","name":"Only Files RouteOnAttribute","comments":"","instanceIdentifier":"dfe6840e-a526-3870-86de-e21b72a97cae"},"destination":{"id":"0646be18-05d2-372f-87f6-944a7da91bde","type":"PROCESSOR","groupId":"9ea1816a-c813-3787-b2b1-c19069522334","name":"Get New Flight Data Files FetchS3","comments":"","instanceIdentifier":"83967228-c828-3c74-ae68-dd6bf592314d"},"labelIndex":1,"zIndex":0,"selectedRelationships":["FilesOnly"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"9ea1816a-c813-3787-b2b1-c19069522334"},{"identifier":"c83670de-8740-3cce-92aa-a6e6ab74214c","instanceIdentifier":"b5a770f0-93f2-3ab8-a312-b59f0891c5ca","name":"","source":{"id":"1f00d975-886d-302c-87ec-6841ba3b9102","type":"PROCESSOR","groupId":"9ea1816a-c813-3787-b2b1-c19069522334","name":"List New Flight Data Files ListS3","comments":"","instanceIdentifier":"cc24cf05-5686-360b-8d74-244183d625d1"},"destination":{"id":"9fa7f16b-d3fe-35c8-8ae1-99841d0ea494","type":"PROCESSOR","groupId":"9ea1816a-c813-3787-b2b1-c19069522334","name":"Only Files RouteOnAttribute","comments":"","instanceIdentifier":"dfe6840e-a526-3870-86de-e21b72a97cae"},"labelIndex":1,"zIndex":0,"selectedRelationships":["success"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"9ea1816a-c813-3787-b2b1-c19069522334"},{"identifier":"88db7dd8-ee3b-3298-a64a-69d23fbfa03a","instanceIdentifier":"73eaf0f0-abc5-3f4b-b616-d3ab0e6f4751","name":"","source":{"id":"05b86a8e-3730-39d9-b0a5-06004c90858b","type":"PROCESSOR","groupId":"9ea1816a-c813-3787-b2b1-c19069522334","name":"ConvertRecord","comments":"","instanceIdentifier":"0f9a51dd-5f54-36f8-b1a3-4082c6c20de0"},"destination":{"id":"4f1fda20-02dd-3924-adc9-2f1c585d765c","type":"PROCESSOR","groupId":"9ea1816a-c813-3787-b2b1-c19069522334","name":"PutIceberg","comments":"","instanceIdentifier":"43a14ad6-5d8a-3b53-8080-122bf0b66570"},"labelIndex":1,"zIndex":0,"selectedRelationships":["success"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"9ea1816a-c813-3787-b2b1-c19069522334"},{"identifier":"416d60ff-e23d-33e1-a7de-b7411a36d0e4","instanceIdentifier":"2d4b87eb-107e-3e1f-a79e-81c5e4c7cffb","name":"","source":{"id":"0646be18-05d2-372f-87f6-944a7da91bde","type":"PROCESSOR","groupId":"9ea1816a-c813-3787-b2b1-c19069522334","name":"Get New Flight Data Files FetchS3","comments":"","instanceIdentifier":"83967228-c828-3c74-ae68-dd6bf592314d"},"destination":{"id":"bb0abc17-6d89-3ee2-97db-a08c441e134a","type":"PROCESSOR","groupId":"9ea1816a-c813-3787-b2b1-c19069522334","name":"UnpackContent","comments":"","instanceIdentifier":"f7b2e2a9-8463-33c0-8a35-45fe6647d108"},"labelIndex":1,"zIndex":0,"selectedRelationships":["success"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"9ea1816a-c813-3787-b2b1-c19069522334"}],"labels":[{"identifier":"13925b99-1237-3da0-a036-06179d2e6e6f","instanceIdentifier":"f3adbb51-018b-1000-0000-000007f375aa","position":{"x":1336.0,"y":408.0},"label":"Is Hive Metastore URI able to be dynamically generated\nor is there an easy way to allow a user to configure\nthis as the parameter value","zIndex":0,"width":312.0,"height":80.0,"style":{"background-color":"#ffd6e0","font-size":"12px"},"componentType":"LABEL","groupIdentifier":"9ea1816a-c813-3787-b2b1-c19069522334"},{"identifier":"ce15fddf-a762-3d31-92f8-78ebe1e2b135","instanceIdentifier":"f3aa5c4a-018b-1000-ffff-ffffc738f4dd","position":{"x":1704.0,"y":32.0},"label":"Need automation to get DL config to NiFi Hosts\nhive-site.xml, core-site.xml, and hdfs-site.xml\nand the directory should be owned by nifi:nifi\nuser, group","zIndex":0,"width":272.0,"height":96.0,"style":{"background-color":"#ffd6e3","font-size":"12px"},"componentType":"LABEL","groupIdentifier":"9ea1816a-c813-3787-b2b1-c19069522334"},{"identifier":"4271f119-b610-378e-8333-8788e5f61183","instanceIdentifier":"57707c72-6782-3c60-bc87-a3f649791f5c","position":{"x":312.0,"y":16.0},"label":"Should get the data directly form GitHub?\n \nCould write custom Python to download file(s) and then process them by passing along the .zip file","zIndex":0,"width":544.0,"height":664.0,"style":{"font-size":"12px"},"componentType":"LABEL","groupIdentifier":"9ea1816a-c813-3787-b2b1-c19069522334"}],"funnels":[{"identifier":"22c007c5-4bf2-37da-9ba8-47cbd333fc5f","instanceIdentifier":"73f79213-1d21-333e-a8ed-bfe161ee4669","position":{"x":1741.5,"y":694.5},"componentType":"FUNNEL","groupIdentifier":"9ea1816a-c813-3787-b2b1-c19069522334"}],"controllerServices":[{"identifier":"07053613-68bb-319a-a2d2-2a87d91bd95b","instanceIdentifier":"2e627cca-0ce1-3d21-9699-bccf8e930027","name":"New Flight Data CSVReader","comments":"","type":"org.apache.nifi.csv.CSVReader","bundle":{"group":"org.apache.nifi","artifact":"nifi-record-serialization-services-nar","version":"1.21.0.2.2.7.100-3"},"properties":{"ignore-csv-header":"true","schema-branch":null,"Trim double quote":"true","CSV Format":"custom","Quote Character":"\"","csvutils-allow-duplicate-header-names":"true","Value Separator":",","Record Separator":"\\n","Timestamp Format":null,"Escape Character":"\\","Date Format":null,"Null String":null,"Skip Header Line":"true","Trim Fields":"true","schema-name":"${schema.name}","schema-registry":null,"csv-reader-csv-parser":"commons-csv","Time Format":null,"Comment Marker":null,"schema-access-strategy":"schema-text-property","schema-version":null,"csvutils-character-set":"UTF-8","schema-text":"{\n \"type\": \"record\",\n \"namespace\": \"com.cloudera\",\n \"name\": \"flights_new_data\",\n \"fields\": [\n {\n \"name\": \"year\",\n \"type\": \"int\"\n },\n {\n \"name\": \"month\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dayofmonth\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dayofweek\",\n \"type\": \"int\"\n },\n {\n \"name\": \"uniquecarrier\",\n \"type\": \"string\"\n },\n {\n \"name\": \"tailnum\",\n \"type\": [\"null\",\"string\"]\n },\n {\n \"name\": \"flightnum\",\n \"type\": [\"null\",\"int\"]\n },\n {\n \"name\": \"origin\",\n \"type\": \"string\"\n },\n {\n \"name\": \"dest\",\n \"type\": \"string\"\n },\n {\n \"name\": \"crsdeptime\",\n \"type\": [\"null\",\"int\"]\n },\n {\n \"name\": \"deptime\",\n \"type\": [\"null\",\"int\"]\n },\n {\n \"name\": \"depdelay\",\n \"type\": [\"null\",\"double\"]\n },\n {\n \"name\": \"taxiout\",\n \"type\": [\"null\",\"double\"]\n },\n {\n \"name\": \"taxiin\",\n \"type\": [\"null\",\"double\"]\n },\n {\n \"name\": \"crsarrtime\",\n \"type\": [\"null\",\"int\"]\n },\n {\n \"name\": \"arrtime\",\n \"type\": [\"null\",\"int\"]\n },\n {\n \"name\": \"arrdelay\",\n \"type\": [\"null\",\"double\"]\n },\n {\n \"name\": \"cancelled\",\n \"type\": \"double\"\n },\n {\n \"name\": \"cancellationcode\",\n \"type\": [\"null\",\"string\"]\n },\n {\n \"name\": \"diverted\",\n \"type\": \"string\"\n },\n {\n \"name\": \"crselapsedtime\",\n \"type\": [\"null\",\"double\"]\n },\n {\n \"name\": \"actualelapsedtime\",\n \"type\": [\"null\",\"double\"]\n },\n {\n \"name\": \"airtime\",\n \"type\": [\"null\",\"double\"]\n },\n {\n \"name\": \"distance\",\n \"type\": [\"null\",\"double\"]\n },\n {\n \"name\": \"carrierdelay\",\n \"type\": [\"null\",\"double\"]\n },\n {\n \"name\": \"weatherdelay\",\n \"type\": [\"null\",\"double\"]\n },\n {\n \"name\": \"nasdelay\",\n \"type\": [\"null\",\"double\"]\n },\n {\n \"name\": \"securitydelay\",\n \"type\": [\"null\",\"double\"]\n },\n {\n \"name\": \"lateaircraftdelay\",\n \"type\": [\"null\",\"double\"]\n }\n ]\n}"},"propertyDescriptors":{"ignore-csv-header":{"name":"ignore-csv-header","displayName":"Ignore CSV Header Column Names","identifiesControllerService":false,"sensitive":false},"schema-branch":{"name":"schema-branch","displayName":"Schema Branch","identifiesControllerService":false,"sensitive":false},"Trim double quote":{"name":"Trim double quote","displayName":"Trim double quote","identifiesControllerService":false,"sensitive":false},"CSV Format":{"name":"CSV Format","displayName":"CSV Format","identifiesControllerService":false,"sensitive":false},"Quote Character":{"name":"Quote Character","displayName":"Quote Character","identifiesControllerService":false,"sensitive":false},"csvutils-allow-duplicate-header-names":{"name":"csvutils-allow-duplicate-header-names","displayName":"Allow Duplicate Header Names","identifiesControllerService":false,"sensitive":false},"Value Separator":{"name":"Value Separator","displayName":"Value Separator","identifiesControllerService":false,"sensitive":false},"Record Separator":{"name":"Record Separator","displayName":"Record Separator","identifiesControllerService":false,"sensitive":false},"Timestamp Format":{"name":"Timestamp Format","displayName":"Timestamp Format","identifiesControllerService":false,"sensitive":false},"Escape Character":{"name":"Escape Character","displayName":"Escape Character","identifiesControllerService":false,"sensitive":false},"Date Format":{"name":"Date Format","displayName":"Date Format","identifiesControllerService":false,"sensitive":false},"Null String":{"name":"Null String","displayName":"Null String","identifiesControllerService":false,"sensitive":false},"Skip Header Line":{"name":"Skip Header Line","displayName":"Treat First Line as Header","identifiesControllerService":false,"sensitive":false},"Trim Fields":{"name":"Trim Fields","displayName":"Trim Fields","identifiesControllerService":false,"sensitive":false},"schema-name":{"name":"schema-name","displayName":"Schema Name","identifiesControllerService":false,"sensitive":false},"schema-registry":{"name":"schema-registry","displayName":"Schema Registry","identifiesControllerService":true,"sensitive":false},"csv-reader-csv-parser":{"name":"csv-reader-csv-parser","displayName":"CSV Parser","identifiesControllerService":false,"sensitive":false},"Time Format":{"name":"Time Format","displayName":"Time Format","identifiesControllerService":false,"sensitive":false},"Comment Marker":{"name":"Comment Marker","displayName":"Comment Marker","identifiesControllerService":false,"sensitive":false},"schema-access-strategy":{"name":"schema-access-strategy","displayName":"Schema Access Strategy","identifiesControllerService":false,"sensitive":false},"schema-version":{"name":"schema-version","displayName":"Schema Version","identifiesControllerService":false,"sensitive":false},"csvutils-character-set":{"name":"csvutils-character-set","displayName":"Character Set","identifiesControllerService":false,"sensitive":false},"schema-text":{"name":"schema-text","displayName":"Schema Text","identifiesControllerService":false,"sensitive":false}},"controllerServiceApis":[{"type":"org.apache.nifi.serialization.RecordReaderFactory","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"1.21.0.2.2.7.100-3"}}],"scheduledState":"DISABLED","bulletinLevel":"WARN","componentType":"CONTROLLER_SERVICE","groupIdentifier":"9ea1816a-c813-3787-b2b1-c19069522334"},{"identifier":"93b5ac02-9a21-351c-a290-c346b242899a","instanceIdentifier":"f23e791e-b547-377c-89bd-adab70043028","name":"New Flights AvroRecordSetWriter","comments":"","type":"org.apache.nifi.avro.AvroRecordSetWriter","bundle":{"group":"org.apache.nifi","artifact":"nifi-record-serialization-services-nar","version":"1.21.0.2.2.7.100-3"},"properties":{"compression-format":"NONE","Schema Write Strategy":"avro-embedded","schema-branch":null,"schema-name":"${schema.name}","cache-size":"1000","schema-registry":null,"schema-access-strategy":"schema-text-property","schema-protocol-version":"1","schema-version":null,"encoder-pool-size":"32","schema-cache":null,"schema-text":"{\n \"type\": \"record\",\n \"namespace\": \"com.cloudera\",\n \"name\": \"flights_new_data\",\n \"fields\": [\n {\n \"name\": \"month\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dayofmonth\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dayofweek\",\n \"type\": \"int\"\n },\n {\n \"name\": \"deptime\",\n \"type\": [\"null\",\"int\"]\n },\n {\n \"name\": \"crsdeptime\",\n \"type\": [\"null\",\"int\"]\n },\n {\n \"name\": \"arrtime\",\n \"type\": [\"null\",\"int\"]\n },\n {\n \"name\": \"crsarrtime\",\n \"type\": [\"null\",\"int\"]\n },\n {\n \"name\": \"uniquecarrier\",\n \"type\": \"string\"\n },\n {\n \"name\": \"flightnum\",\n \"type\": [\"null\",\"int\"]\n },\n {\n \"name\": \"tailnum\",\n \"type\": [\"null\",\"string\"]\n },\n {\n \"name\": \"actualelapsedtime\",\n \"type\": [\"null\",\"int\"]\n },\n {\n \"name\": \"crselapsedtime\",\n \"type\": [\"null\",\"int\"]\n },\n {\n \"name\": \"airtime\",\n \"type\": [\"null\",\"int\"]\n },\n {\n \"name\": \"arrdelay\",\n \"type\": [\"null\",\"int\"]\n },\n {\n \"name\": \"depdelay\",\n \"type\": [\"null\",\"int\"]\n },\n {\n \"name\": \"origin\",\n \"type\": \"string\"\n },\n {\n \"name\": \"dest\",\n \"type\": \"string\"\n },\n {\n \"name\": \"distance\",\n \"type\": [\"null\",\"int\"]\n },\n {\n \"name\": \"taxiin\",\n \"type\": [\"null\",\"int\"]\n },\n {\n \"name\": \"taxiout\",\n \"type\": [\"null\",\"int\"]\n },\n {\n \"name\": \"cancelled\",\n \"type\": \"int\"\n },\n {\n \"name\": \"cancellationcode\",\n \"type\": [\"null\",\"string\"]\n },\n {\n \"name\": \"diverted\",\n \"type\": \"string\"\n },\n {\n \"name\": \"carrierdelay\",\n \"type\": [\"null\",\"int\"]\n },\n {\n \"name\": \"weatherdelay\",\n \"type\": [\"null\",\"int\"]\n },\n {\n \"name\": \"nasdelay\",\n \"type\": [\"null\",\"int\"]\n },\n {\n \"name\": \"securitydelay\",\n \"type\": [\"null\",\"int\"]\n },\n {\n \"name\": \"lateaircraftdelay\",\n \"type\": [\"null\",\"int\"]\n },\n {\n \"name\": \"year\",\n \"type\": \"int\"\n }\n ]\n}"},"propertyDescriptors":{"compression-format":{"name":"compression-format","displayName":"Compression Format","identifiesControllerService":false,"sensitive":false},"Schema Write Strategy":{"name":"Schema Write Strategy","displayName":"Schema Write Strategy","identifiesControllerService":false,"sensitive":false},"schema-branch":{"name":"schema-branch","displayName":"Schema Branch","identifiesControllerService":false,"sensitive":false},"schema-name":{"name":"schema-name","displayName":"Schema Name","identifiesControllerService":false,"sensitive":false},"cache-size":{"name":"cache-size","displayName":"Cache Size","identifiesControllerService":false,"sensitive":false},"schema-registry":{"name":"schema-registry","displayName":"Schema Registry","identifiesControllerService":true,"sensitive":false},"schema-access-strategy":{"name":"schema-access-strategy","displayName":"Schema Access Strategy","identifiesControllerService":false,"sensitive":false},"schema-protocol-version":{"name":"schema-protocol-version","displayName":"Schema Protocol Version","identifiesControllerService":false,"sensitive":false},"schema-version":{"name":"schema-version","displayName":"Schema Version","identifiesControllerService":false,"sensitive":false},"encoder-pool-size":{"name":"encoder-pool-size","displayName":"Encoder Pool Size","identifiesControllerService":false,"sensitive":false},"schema-cache":{"name":"schema-cache","displayName":"Schema Cache","identifiesControllerService":true,"sensitive":false},"schema-text":{"name":"schema-text","displayName":"Schema Text","identifiesControllerService":false,"sensitive":false}},"controllerServiceApis":[{"type":"org.apache.nifi.serialization.RecordSetWriterFactory","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"1.21.0.2.2.7.100-3"}}],"scheduledState":"DISABLED","bulletinLevel":"WARN","componentType":"CONTROLLER_SERVICE","groupIdentifier":"9ea1816a-c813-3787-b2b1-c19069522334"},{"identifier":"ddaa59b4-5d10-340c-8c2f-4df67ab119e4","instanceIdentifier":"1577d441-505b-35c1-af5f-3e593a2ef191","name":"KerberosPasswordUserService","comments":"","type":"org.apache.nifi.kerberos.KerberosPasswordUserService","bundle":{"group":"org.apache.nifi","artifact":"nifi-kerberos-user-service-nar","version":"1.21.0.2.2.7.100-3"},"properties":{"Kerberos Password":"#{Workload Password}","Kerberos Principal":"#{Workload User}"},"propertyDescriptors":{"Kerberos Password":{"name":"Kerberos Password","displayName":"Kerberos Password","identifiesControllerService":false,"sensitive":true},"Kerberos Principal":{"name":"Kerberos Principal","displayName":"Kerberos Principal","identifiesControllerService":false,"sensitive":false}},"controllerServiceApis":[{"type":"org.apache.nifi.kerberos.KerberosUserService","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"1.21.0.2.2.7.100-3"}}],"scheduledState":"DISABLED","bulletinLevel":"WARN","componentType":"CONTROLLER_SERVICE","groupIdentifier":"9ea1816a-c813-3787-b2b1-c19069522334"},{"identifier":"3e066fdf-31b9-33f6-991f-5e286bba8c5b","instanceIdentifier":"00d82c08-2032-3968-aacf-8aeeea72b56e","name":"HiveCatalogService","comments":"","type":"org.apache.nifi.services.iceberg.HiveCatalogService","bundle":{"group":"org.apache.nifi","artifact":"nifi-iceberg-services-nar","version":"1.21.0.2.2.7.100-3"},"properties":{"hadoop-config-resources":"#{Hadoop Configuration Resources}","warehouse-location":null,"hive-metastore-uri":"#{Hive Metastore URI}"},"propertyDescriptors":{"hadoop-config-resources":{"name":"hadoop-config-resources","displayName":"Hadoop Configuration Resources","identifiesControllerService":false,"sensitive":false,"resourceDefinition":{"cardinality":"MULTIPLE","resourceTypes":["FILE"]}},"warehouse-location":{"name":"warehouse-location","displayName":"Default Warehouse Location","identifiesControllerService":false,"sensitive":false},"hive-metastore-uri":{"name":"hive-metastore-uri","displayName":"Hive Metastore URI","identifiesControllerService":false,"sensitive":false}},"controllerServiceApis":[{"type":"org.apache.nifi.services.iceberg.IcebergCatalogService","bundle":{"group":"org.apache.nifi","artifact":"nifi-iceberg-services-api-nar","version":"1.21.0.2.2.7.100-3"}}],"scheduledState":"DISABLED","bulletinLevel":"WARN","componentType":"CONTROLLER_SERVICE","groupIdentifier":"9ea1816a-c813-3787-b2b1-c19069522334"},{"identifier":"34bcbbd8-e06c-303b-8695-dc59d0c5506e","instanceIdentifier":"f46215e3-22ed-322f-a64e-92e9cbe7ee70","name":"Formatted New Flight Data AvroReader","comments":"","type":"org.apache.nifi.avro.AvroReader","bundle":{"group":"org.apache.nifi","artifact":"nifi-record-serialization-services-nar","version":"1.21.0.2.2.7.100-3"},"properties":{"schema-branch":null,"schema-name":"${schema.name}","cache-size":"1000","schema-registry":null,"schema-access-strategy":"embedded-avro-schema","schema-version":null,"schema-text":"${avro.schema}"},"propertyDescriptors":{"schema-branch":{"name":"schema-branch","displayName":"Schema Branch","identifiesControllerService":false,"sensitive":false},"schema-name":{"name":"schema-name","displayName":"Schema Name","identifiesControllerService":false,"sensitive":false},"cache-size":{"name":"cache-size","displayName":"Cache Size","identifiesControllerService":false,"sensitive":false},"schema-registry":{"name":"schema-registry","displayName":"Schema Registry","identifiesControllerService":true,"sensitive":false},"schema-access-strategy":{"name":"schema-access-strategy","displayName":"Schema Access Strategy","identifiesControllerService":false,"sensitive":false},"schema-version":{"name":"schema-version","displayName":"Schema Version","identifiesControllerService":false,"sensitive":false},"schema-text":{"name":"schema-text","displayName":"Schema Text","identifiesControllerService":false,"sensitive":false}},"controllerServiceApis":[{"type":"org.apache.nifi.serialization.RecordReaderFactory","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"1.21.0.2.2.7.100-3"}}],"scheduledState":"DISABLED","bulletinLevel":"WARN","componentType":"CONTROLLER_SERVICE","groupIdentifier":"9ea1816a-c813-3787-b2b1-c19069522334"}],"variables":{},"parameterContextName":"New_Flights_Parameters","defaultFlowFileExpiration":"0 sec","defaultBackPressureObjectThreshold":10000,"defaultBackPressureDataSizeThreshold":"1 GB","flowFileConcurrency":"UNBOUNDED","flowFileOutboundPolicy":"STREAM_WHEN_AVAILABLE","componentType":"PROCESS_GROUP"},"externalControllerServices":{},"parameterContexts":{"New_Flights_Parameters":{"name":"New_Flights_Parameters","parameters":[{"name":"Table Name","description":"","sensitive":false,"provided":false,"value":"flights"},{"name":"Workload User","description":"","sensitive":false,"provided":false,"value":"jingalls"},{"name":"Bucket","description":"","sensitive":false,"provided":false,"value":"jingalls-demo-data"},{"name":"Access Key ID","description":"","sensitive":true,"provided":false},{"name":"Hive Metastore URI","description":"","sensitive":false,"provided":false,"value":"thrift://se-aw-mdl-master0.se-sandb.a465-9q4k.cloudera.site:9083,thrift://se-aw-mdl-master1.se-sandb.a465-9q4k.cloudera.site:9083"},{"name":"Prefix","description":"","sensitive":false,"provided":false,"value":"airlines-csv/flights_new"},{"name":"Database","description":"","sensitive":false,"provided":false,"value":"jing_airlines"},{"name":"Workload Password","description":"","sensitive":true,"provided":false},{"name":"Hadoop Configuration Resources","description":"","sensitive":false,"provided":false,"value":"/tmp/jingalls/hive-site.xml,/tmp/jingalls/core-site.xml,/tmp/jingalls/hdfs-site.xml"},{"name":"Secret Access Key","description":"","sensitive":true,"provided":false}],"inheritedParameterContexts":[],"componentType":"PARAMETER_CONTEXT"}},"flowEncodingVersion":"1.0","parameterProviders":{},"latest":false}