设计 ADF 数据管道的环境设置

1az group create --name $ResourceGroupName --location $Region

1az storage account create --resource-group $ResourceGroupName \
2  --name $StorageAccountName --location $Region 

1az storage container create --resource-group $ResourceGroupName \
2  --name $StorageContainerName \
3  --account-name $StorageAccountName --auth-mode key

1export AZ_CONNECTION_STRING=$(az storage account show-connection-string --name $StorageAccountName --resource-group $ResourceGroupName --output tsv)

1az datafactory create --resource-group $ResourceGroupName \
2    --factory-name $DataFactoryName

1az datafactory show --resource-group $ResourceGroupName \
2    --factory-name $DataFactoryName

1az datafactory linked-service create \
2    --factory-name $DataFactoryName \
3    --properties "{\"type\":\"AzureStorage\",\"typeProperties\":{\"connectionString\":{\"type\":\"SecureString\",\"value\":\"$AZ_CONNECTION_STRING\"}}}" \
4    --name $LinkedServiceName \
5    --resource-group $ResourceGroupName

1This is a text file to perform a sample copy pipeline in ADF.

1{
2    "linkedServiceName": {
3        "referenceName": "$LinkedServiceName",
4        "type": "LinkedServiceReference"
5    },
6    "annotations": [],
7    "type": "Binary",
8    "typeProperties": {
9        "location": {
10            "type": "AzureBlobStorageLocation",
11            "fileName": "sampletxt.txt",
12            "folderPath": "input",
13            "container": "$StorageContainerName"
14        }
15    }
16}

1{
2    "linkedServiceName": {
3        "referenceName": "$LinkedServiceName",
4        "type": "LinkedServiceReference"
5    },
6    "annotations": [],
7    "type": "Binary",
8    "typeProperties": {
9        "location": {
10            "type": "AzureBlobStorageLocation",
11            "folderPath": "output",
12            "container": "$StorageContainerName"
13        }
14    }
15}

1# Create the input dataset
2az datafactory dataset create --resource-group $ResourceGroupName \
3  --dataset-name InputDataset --factory-name $DataFactoryName \
4  --properties "{\"linkedServiceName\":{\"referenceName\":\"$LinkedServiceName\",\"type\":\"LinkedServiceReference\"},\"annotations\":[],\"type\":\"Binary\",\"typeProperties\":{\"location\":{\"type\":\"AzureBlobStorageLocation\",\"fileName\":\"sampletxt.txt\",\"folderPath\":\"input\",\"container\":\"$StorageContainerName\"}}}"
5
6# Create the output dataset
7  az datafactory dataset create --resource-group $ResourceGroupName \
8  --dataset-name OutputDataset --factory-name $DataFactoryName \
9  --properties "{\"linkedServiceName\":{\"referenceName\":\"$LinkedServiceName\",\"type\":\"LinkedServiceReference\"},\"annotations\":[],\"type\":\"Binary\",\"typeProperties\":{\"location\":{\"type\":\"AzureBlobStorageLocation\",\"folderPath\":\"output\",\"container\":\"$StorageContainerName\"}}}"

运行以下命令，逐条在下方终端中执行：

1# 创建新的资源组
2az group create --name $ResourceGroupName --location $Region
3# 创建新的存储账户
4az storage account create --resource-group $ResourceGroupName \
5  --name $StorageAccountName --location $Region 
6# 在存储账户中创建存储容器
7az storage container create --resource-group $ResourceGroupName \
8  --name $StorageContainerName \
9  --account-name $StorageAccountName --auth-mode key
10# 导出存储容器的连接字符串
11export AZ_CONNECTION_STRING=$(az storage account show-connection-string --name $StorageAccountName --resource-group $ResourceGroupName --output tsv)
12# 创建新的数据工厂
13az datafactory create --resource-group $ResourceGroupName \
14    --factory-name $DataFactoryName
15# 检查数据工厂是否成功创建
16az datafactory show --resource-group $ResourceGroupName \
17    --factory-name $DataFactoryName
18# 为存储账户创建新的链接服务
19az datafactory linked-service create \
20    --factory-name $DataFactoryName \
21    --properties "{\"type\":\"AzureStorage\",\"typeProperties\":{\"connectionString\":{\"type\":\"SecureString\",\"value\":\"$AZ_CONNECTION_STRING\"}}}" \
22    --name $LinkedServiceName \
23    --resource-group $ResourceGroupName
24# 创建输入数据集
25az datafactory dataset create --resource-group $ResourceGroupName \
26  --dataset-name InputDataset --factory-name $DataFactoryName \
27  --properties "{\"linkedServiceName\":{\"referenceName\":\"$LinkedServiceName\",\"type\":\"LinkedServiceReference\"},\"annotations\":[],\"type\":\"Binary\",\"typeProperties\":{\"location\":{\"type\":\"AzureBlobStorageLocation\",\"fileName\":\"sampletxt.txt\",\"folderPath\":\"input\",\"container\":\"$StorageContainerName\"}}}"
28# 创建输出数据集
29  az datafactory dataset create --resource-group $ResourceGroupName \
30  --dataset-name OutputDataset --factory-name $DataFactoryName \
31  --properties "{\"linkedServiceName\":{\"referenceName\":\"$LinkedServiceName\",\"type\":\"LinkedServiceReference\"},\"annotations\":[],\"type\":\"Binary\",\"typeProperties\":{\"location\":{\"type\":\"AzureBlobStorageLocation\",\"folderPath\":\"output\",\"container\":\"$StorageContainerName\"}}}"

请在下方终端中逐条运行以下命令：

1# 创建新的资源组
2az group create --name $ResourceGroupName --location $Region
3# 创建新的存储账户
4az storage account create --resource-group $ResourceGroupName \
5  --name $StorageAccountName --location $Region 
6# 在存储账户中创建存储容器
7az storage container create --resource-group $ResourceGroupName \
8  --name $StorageContainerName \
9  --account-name $StorageAccountName --auth-mode key
10# 导出存储容器的连接字符串
11export AZ_CONNECTION_STRING=$(az storage account show-connection-string --name $StorageAccountName --resource-group $ResourceGroupName --output tsv)
12# 创建新的数据工厂
13az datafactory create --resource-group $ResourceGroupName \
14    --factory-name $DataFactoryName
15# 检查数据工厂是否成功创建
16az datafactory show --resource-group $ResourceGroupName \
17    --factory-name $DataFactoryName
18# 为存储账户创建新的链接服务
19az datafactory linked-service create \
20    --factory-name $DataFactoryName \
21    --properties "{\"type\":\"AzureStorage\",\"typeProperties\":{\"connectionString\":{\"type\":\"SecureString\",\"value\":\"$AZ_CONNECTION_STRING\"}}}" \
22    --name $LinkedServiceName \
23    --resource-group $ResourceGroupName
24# 创建输入数据集
25az datafactory dataset create --resource-group $ResourceGroupName \
26  --dataset-name InputDataset --factory-name $DataFactoryName \
27  --properties "{\"linkedServiceName\":{\"referenceName\":\"$LinkedServiceName\",\"type\":\"LinkedServiceReference\"},\"annotations\":[],\"type\":\"Binary\",\"typeProperties\":{\"location\":{\"type\":\"AzureBlobStorageLocation\",\"fileName\":\"sampletxt.txt\",\"folderPath\":\"input\",\"container\":\"$StorageContainerName\"}}}"
28# 创建输出数据集
29  az datafactory dataset create --resource-group $ResourceGroupName \
30  --dataset-name OutputDataset --factory-name $DataFactoryName \
31  --properties "{\"linkedServiceName\":{\"referenceName\":\"$LinkedServiceName\",\"type\":\"LinkedServiceReference\"},\"annotations\":[],\"type\":\"Binary\",\"typeProperties\":{\"location\":{\"type\":\"AzureBlobStorageLocation\",\"folderPath\":\"output\",\"container\":\"$StorageContainerName\"}}}"