I am trying to get all the information about the activities in a pipeline and log it into some SQL Database.
I did some R&D and I found that I can do it with RESP API.
I read and performed it with the help of below link:-
URL for the API
Request Body for API
CodePudding user response：
How are you getting the Pipeline run ID, make sure that is right one and runs exist between the interval specified (UTC). Format the JSON body in the request as I've shown. Validate it using some tool or editor. And use the right authentication.
Are you sure you are not seeing any error at all ?
Output
{
"value": [
{
"activityRunEnd": "2021-10-12T13:03:17.1874105Z",
"activityName": "Data flow1",
"activityRunStart": "2021-10-12T13:00:14.7523815Z",
"activityType": "ExecuteDataFlow",
"durationInMs": 182435,
"retryAttempt": null,
"error": {
"errorCode": "",
"message": "",
"failureType": "",
"target": "Data flow1",
"details": ""
},
"activityRunId": "5e7aece7-5fb7-4ffd-939e-d01d3fcb494a",
"iterationHash": "",
"input": {
"dataflow": {
"referenceName": "generateParquet",
"type": "DataFlowReference",
"parameters": {},
"datasetParameters": {
"source1": {},
"sink1": {}
}
},
"staging": {},
"compute": {
"coreCount": 8,
"computeType": "General"
},
"traceLevel": "Fine",
"dataFlowETag": "48019f9b-0000-0300-0000-616475e20000"
},
"linkedServiceName": "",
"output": {
"runStatus": {
"computeAcquisitionDuration": 107335,
"version": "20211011.2",
"profile": {
"source1": {
"computed": [],
"lineage": {},
"dropped": 0,
"drifted": 0,
"newer": 16,
"total": 16,
"updated": 0
},
"sink1": {
"computed": [],
"lineage": {
"CompanyName": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"CompanyName"
]
}
]
},
"LastName": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"LastName"
]
}
]
},
"MiddleName": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"MiddleName"
]
}
]
},
"PasswordSalt": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"PasswordSalt"
]
}
]
},
"ModifiedDate": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"ModifiedDate"
]
}
]
},
"path": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"path"
]
}
]
},
"rowguid": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"rowguid"
]
}
]
},
"Title": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"Title"
]
}
]
},
"NameStyle": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"NameStyle"
]
}
]
},
"CustomerID": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"CustomerID"
]
}
]
},
"FirstName": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"FirstName"
]
}
]
},
"Suffix": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"Suffix"
]
}
]
},
"Phone": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"Phone"
]
}
]
},
"PasswordHash": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"PasswordHash"
]
}
]
},
"SalesPerson": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"SalesPerson"
]
}
]
},
"EmailAddress": {
"mapped": true,
"from": [
{
"source": "source1",
"columns": [
"EmailAddress"
]
}
]
}
},
"dropped": 0,
"drifted": 0,
"newer": 0,
"total": 16,
"updated": 16
}
},
"metrics": {
"sink1": {
"format": "parquet",
"stages": [
{
"stage": 1,
"partitionTimes": [
525
],
"lastUpdateTime": "2021-10-12 13:02:50.453",
"bytesWritten": 0,
"bytesRead": 199491,
"partitionStatus": "Success",
"streams": {
"source1": {
"count": 847,
"cached": false,
"totalPartitions": 1,
"partitionStatus": "Success",
"partitionCounts": [
847
],
"type": "source"
}
},
"target": "sink1",
"time": 640,
"progressState": "Completed"
},
{
"stage": 2,
"partitionTimes": [
3436,
3373,
3441,
3475,
2304,
2321,
2364,
2291,
1989,
2435
],
"lastUpdateTime": "2021-10-12 13:02:58.897",
"bytesWritten": 177728,
"bytesRead": 0,
"partitionStatus": "Success",
"streams": {
"sink1": {
"count": 847,
"cached": false,
"totalPartitions": 10,
"partitionStatus": "Success",
"partitionCounts": [
84,
85,
85,
85,
85,
85,
85,
85,
84,
84
],
"type": "sink"
}
},
"target": "sink1",
"time": 8452,
"progressState": "Completed"
}
],
"sinkPostProcessingTime": 0,
"data": [],
"store": "blob",
"rowsWritten": 847,
"details": {
"postCommandsDuration": [
3
],
"preCommandsDuration": [
1
]
},
"sources": {
"source1": {
"rowsRead": 847,
"store": "blob",
"details": {
"fileSystemInitDuration": [
7751
]
},
"format": "delimited"
}
},
"sinkProcessingTime": 12380
}
},
"dsl": "\nsource() ~> source1\n\nsource1 sink() ~> sink1"
},
"effectiveIntegrationRuntime": "DefaultIntegrationRuntime (Central US)",
"billingReference": {
"activityType": "executedataflow",
"billableDuration": [
{
"meterType": "General",
"duration": 0.3691165737777778,
"unit": "coreHour",
"sessionType": "JobCluster"
}
]
},
"reportLineageToPurview": {
"status": "NotReported"
}
},
"userProperties": {},
"pipelineName": "partiion_parquets",
"pipelineRunId": "c619e850-c7de-4d1f-8d0c-4b73dcacb9d8",
"status": "Succeeded",
"recoveryStatus": "None",
"integrationRuntimeNames": [
"defaultintegrationruntime"
],
"executionDetails": {
"integrationRuntime": [
{
"name": "AutoResolveIntegrationRuntime",
"type": "Managed",
"computeType": "General",
"coreCount": "8"
}
]
},
"id": "/SUBSCRIPTIONS/B83C1EE3-xxxx-xxxx-B5BA-2E83A074C23K/RESOURCEGROUPS/myrg/PROVIDERS/MICROSOFT.DATAFACTORY/FACTORIES/KTESTADF/pipelineruns/c619e850-xxxx-xxxx-xxxx-4b73dcacb9d8/activityruns/5e7aece7-xxxx-4ffd-939e-d01d3fcb494a"
}
],
"ADFWebActivityResponseHeaders": {
"Pragma": "no-cache",
"Strict-Transport-Security": "max-age=31536000; includeSubDomains",
"X-Content-Type-Options": "nosniff",
"x-ms-ratelimit-remaining-subscription-reads": "11999",
"x-ms-request-id": "e8fa08f2-594a-4464-b43e-d71104c1950c",
"x-ms-correlation-request-id": "e8fa08f2-594a-4464-b43e-d71104c1950c",
"x-ms-routing-request-id": "CENTRALUS:20211012T131050Z:e8fa08f2-594a-4464-b43e-d71104c1950c",
"Cache-Control": "no-cache",
"Date": "Tue, 12 Oct 2021 13:10:49 GMT",
"Server": "Microsoft-IIS/10.0",
"X-Powered-By": "ASP.NET",
"Content-Length": "4307",
"Content-Type": "application/json; charset=utf-8",
"Expires": "-1"
},
"effectiveIntegrationRuntime": "DefaultIntegrationRuntime (Central US)",
"executionDuration": 0,
"durationInQueue": {
"integrationRuntimeQueue": 0
},
"billingReference": {
"activityType": "ExternalActivity",
"billableDuration": [
{
"meterType": "AzureIR",
"duration": 0.016666666666666666,
"unit": "Hours"
}
]
}
}
From the above you can filter for "activityType": "ExecuteDataFlow" , durationInMs, activityRunStart, activityRunEnd, invokedByType, input, output and other metrics.
Just follow Azure Data Factory version 2 (V2) REST API operations, parameterize selected outputs and store in variables if necessary. Using Add dynamic content build URL and JSON request body appropriately.