Updating pipeline: dataset_processing

Adding dataset: workspace Updating dataset: preprocessed_dataset_sink
mahdikhashan · Dec 29, 2024 · ea2423a · ea2423a
1 parent a46ca8d
commit ea2423a
Show file tree

Hide file tree

Showing 3 changed files with 140 additions and 1 deletion.
diff --git a/dataset/preprocessed_dataset_sink.json b/dataset/preprocessed_dataset_sink.json
@@ -11,6 +11,9 @@
 			"location": {
 				"type": "AzureBlobStorageLocation",
 				"container": "sink"
+			},
+			"compression": {
+				"type": "TarGZip"
 			}
 		}
 	}

diff --git a/dataset/workspace.json b/dataset/workspace.json
@@ -0,0 +1,17 @@
+{
+	"name": "workspace",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "Dataset",
+			"type": "LinkedServiceReference"
+		},
+		"annotations": [],
+		"type": "Binary",
+		"typeProperties": {
+			"location": {
+				"type": "AzureBlobStorageLocation",
+				"container": "workspace"
+			}
+		}
+	}
+}
diff --git a/pipeline/dataset_processing.json b/pipeline/dataset_processing.json
@@ -29,7 +29,8 @@
 					"sink": {
 						"type": "BinarySink",
 						"storeSettings": {
-							"type": "AzureBlobStorageWriteSettings"
+							"type": "AzureBlobStorageWriteSettings",
+							"copyBehavior": "FlattenHierarchy"
 						}
 					},
 					"enableStaging": false
@@ -40,6 +41,119 @@
 						"type": "DatasetReference"
 					}
 				],
+				"outputs": [
+					{
+						"referenceName": "workspace",
+						"type": "DatasetReference"
+					}
+				]
+			},
+			{
+				"name": "preprocess_dataset",
+				"type": "AzureFunctionActivity",
+				"dependsOn": [
+					{
+						"activity": "unzip",
+						"dependencyConditions": [
+							"Succeeded"
+						]
+					}
+				],
+				"policy": {
+					"timeout": "0.12:00:00",
+					"retry": 0,
+					"retryIntervalInSeconds": 30,
+					"secureOutput": false,
+					"secureInput": false
+				},
+				"userProperties": []
+			},
+			{
+				"name": "unzip",
+				"type": "AzureFunctionActivity",
+				"dependsOn": [
+					{
+						"activity": "duplicate_data",
+						"dependencyConditions": [
+							"Succeeded"
+						]
+					}
+				],
+				"policy": {
+					"timeout": "0.12:00:00",
+					"retry": 0,
+					"retryIntervalInSeconds": 30,
+					"secureOutput": false,
+					"secureInput": false
+				},
+				"userProperties": []
+			},
+			{
+				"name": "compress",
+				"type": "AzureFunctionActivity",
+				"dependsOn": [
+					{
+						"activity": "preprocess_dataset",
+						"dependencyConditions": [
+							"Succeeded"
+						]
+					}
+				],
+				"policy": {
+					"timeout": "0.12:00:00",
+					"retry": 0,
+					"retryIntervalInSeconds": 30,
+					"secureOutput": false,
+					"secureInput": false
+				},
+				"userProperties": []
+			},
+			{
+				"name": "Copy data1",
+				"type": "Copy",
+				"dependsOn": [
+					{
+						"activity": "compress",
+						"dependencyConditions": [
+							"Succeeded"
+						]
+					}
+				],
+				"policy": {
+					"timeout": "0.12:00:00",
+					"retry": 0,
+					"retryIntervalInSeconds": 30,
+					"secureOutput": false,
+					"secureInput": false
+				},
+				"userProperties": [],
+				"typeProperties": {
+					"source": {
+						"type": "BinarySource",
+						"storeSettings": {
+							"type": "AzureBlobStorageReadSettings",
+							"recursive": true,
+							"deleteFilesAfterCompletion": false
+						},
+						"formatSettings": {
+							"type": "BinaryReadSettings"
+						}
+					},
+					"sink": {
+						"type": "BinarySink",
+						"storeSettings": {
+							"type": "AzureBlobStorageWriteSettings",
+							"copyBehavior": "FlattenHierarchy"
+						}
+					},
+					"enableStaging": false
+				},
+				"inputs": [
+					{
+						"referenceName": "workspace",
+						"type": "DatasetReference"
+					}
+				],
 				"outputs": [
 					{
 						"referenceName": "preprocessed_dataset_sink",
@@ -48,6 +162,11 @@
 				]
 			}
 		],
+		"variables": {
+			"file": {
+				"type": "Array"
+			}
+		},
 		"annotations": []
 	}
 }