I have a api in which I need to post a request and write the response to individual txt files.

Each post request needs to go with a MaterialID which I am reading from a list. The list can contain anything from 1000 to 5000 MaterialIDs

I need a way to do multiple parallel requests.

The below code is what I currently have but is built more for synchronous request.

How do i go about doing parallel requests ?

 //reads MaterialID from file
      System.IO.StreamReader fileTXT =
                new System.IO.StreamReader(@"C:\\Temp\Test.txt");

            while ((line = fileTXT.ReadLine()) != null)
            {
                Material_Count.Add(line.Trim());
                UpdateLogTxt("Material_Count = "   line.ToString()   "  ");

            }

            fileTXT.Close();
       

//loops through MaterialID list and starts post request 
 foreach (var item in Material_Count)
                {

                try
                {
                    UpdateLogTxt(" Submitting   = "   item.ToString());


                    var Set = Properties.Settings.Default;

                    using (var httpClient = new HttpClient())
                    {

                        string FCSEndPoint = @"https://www.api.co.za";

                        string FCSToken = "";

                        FCSToken = textBoxDescription.Text.Trim();


                        string uri = $"{FCSEndPoint}/material/pageIndex="   item.ToString()   "";

                        HttpResponseMessage response = null;

                        httpClient.BaseAddress = new Uri(uri);
                        httpClient.DefaultRequestHeaders.Accept.Clear();
                        httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
                        httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("SessionToken", FCSToken);

                        AuthenticationHeaderValue headerObj = new AuthenticationHeaderValue("SessionToken", FCSToken);

                        httpClient.DefaultRequestHeaders.Authorization = headerObj;

                        var TaskObj = httpClient.PostAsJsonAsync(uri, String.Empty);

                        TaskObj.Wait();

                        HttpResponseMessage messageOut = TaskObj.Result;

                        response = TaskObj.Result;


                        if (response.IsSuccessStatusCode)
                        {
                            var TaskObj2 = messageOut.Content.ReadAsStringAsync();
                            TaskObj2.Wait();

                            string ResponseStr = TaskObj2.Result;
    
//writes response to a txt file. 
                                string fileName = @"C:\Temp\material\Material_ "   item.ToString()   " "   DateTime.Now.ToString("yyyyMMddmmss")   ".txt";
    
                                try
                                {
                                    // Check if file already exists. If yes, delete it.     
                                    if (File.Exists(fileName))
                                    {
                                        File.Delete(fileName);
                                    }
    
                                    // Create a new file     
                                    using (StreamWriter sw = File.CreateText(fileName))
                                    {
                                        sw.WriteLine(ResponseStr.ToString());
                                    }
    
    
                                }
                                catch (Exception ex)
                                {
                                    UpdateLogTxt("Exception Write to file  Failed =  ---  "   ex.ToString());
                                }
    
                            }
                            else if (response.StatusCode == System.Net.HttpStatusCode.Forbidden)
                            { 
                                UpdateLogTxt("Response Failed (Forbidden) =  ---  "   response.StatusCode.ToString());
    
                            }
                            else
                            {
                            }
    
                        }
                    }
                    catch (HttpRequestException h)
                {
                    UpdateLogTxt("HttpRequestException Send Failed = ---  "   h.ToString());
                }
                catch (Exception ex)
                {
                    UpdateLogTxt("Exception Send Failed = ---  "   ex.ToString());
                }
    
            }

CodePudding user response：

Replace the foreach by the Parallel.ForEach in the System.Threading.Tasks namespace

You can find handy sample codes here => Parallel.ForEach method

CodePudding user response：

Here's an alternative using Microsoft's Reactive Framework - NuGet System.Reactive and then you can do this:

Start by collecting a couple of constants used in your code to make sure they happen on the UI thread and only once:

string fcsToken = textBoxDescription.Text.Trim();
string now = DateTime.Now.ToString("yyyyMMddmmss");

Now lets create a helper method to handle the HTTP call:

Task<HttpResponseMessage> PostAsJsonAsync(HttpClient hc, string item)
{
    string fcsEndPoint = @"https://www.api.co.za";

    string uri = $"{fcsEndPoint}/material/pageIndex={item}";

    hc.BaseAddress = new Uri(uri);
    hc.DefaultRequestHeaders.Accept.Clear();
    hc.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
    hc.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("SessionToken", fcsToken);

    AuthenticationHeaderValue authorization = new AuthenticationHeaderValue("SessionToken", fcsToken);

    hc.DefaultRequestHeaders.Authorization = authorization;

    return hc.PostAsJsonAsync(uri, String.Empty);
}

Now we can write the observable query:

IObservable<(string fileName, string responseText)> query =
    from item in
        File
            .ReadLines(@"C:\\Temp\Test.txt")
            .Select(x => x.Trim())
            .ToObservable()
    from output in
        Observable
            .Using(
                () => new HttpClient(),
                hc =>
                    from response in Observable.FromAsync(() => PostAsJsonAsync(hc, item))
                    where response.IsSuccessStatusCode
                    from responseText in Observable.FromAsync(() => response.Content.ReadAsStringAsync())
                    select
                    (
                        fileName: $@"C:\Temp\material\Material_ {item} {now}.txt",
                        responseText: responseText
                    ))
    select output;

And finally we have the subscription:

IDisposable subscription =
    query
        .Subscribe(x => File.WriteAllText(x.fileName, x.responseText));

That's it. It's all asynchronous and concurrent.