How to use HTTPS GET command with TIdTCPClient?-CodePudding

I'm using Indy TIdTCPClient to call HTTPS but the server is timing out waiting for my HTTP command. In Wireshark I can see the TLS handshake is done, but none of the WriteLn() commands are responded by the server.

I know using TIdHTTP will be easier, but I have another purpose. Below is the attached code:

procedure TForm1.Button1Click(Sender: TObject);
var
  sid, result: string;
  lParam: TStringList;
  TCPC: TIdTCPClient;
  OnSSL: TIdSSLIOHandlerSocketOpenSSL;
begin
  begin
    lParam := TStringList.Create;
    TCPC := TIdTCPClient.Create();
    OnSSL := TIdSSLIOHandlerSocketOpenSSL.Create();
    try
      OnSSL.SSLOptions.Method := sslvSSLv23;
      OnSSL.SSLOptions.SSLVersions := [sslvSSLv2, sslvSSLv3, sslvTLSv1, sslvTLSv1_1, sslvTLSv1_2];
      OnSSL.UseNagle := True;
      OnSSL.PassThrough := False;

      TCPC.Host := 'xxxxxxx.edu';
      TCPC.Port := 443;
      TCPC.ConnectTimeout := 100000;
      TCPC.ReadTimeout := 500000;
      TCPC.IOHandler := OnSSL;
      TCPC.IOHandler.DefStringEncoding := IndyTextEncoding_UTF8;
      TCPC.Connect;

      try
        logging(TCPC.IOHandler.ReadLn);
        if TCPC.IOHandler.Connected then
        begin
          TCPC.IOHandler.WriteLn('GET / HTTP/1.1', IndyTextEncoding_UTF8);
          TCPC.IOHandler.WriteLn('Host: xxxxxxx.edu', IndyTextEncoding_UTF8);
          TCPC.IOHandler.WriteLn('Connection: keep-alive', IndyTextEncoding_UTF8);
          TCPC.IOHandler.WriteLn('Pragma: no-cache', IndyTextEncoding_UTF8);
          TCPC.IOHandler.WriteLn('Cache-Control: no-cache', IndyTextEncoding_UTF8);
          TCPC.IOHandler.Write('sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="96", "Google Chrome";v="96"', IndyTextEncoding_UTF8);
          TCPC.IOHandler.WriteLn('sec-ch-ua-mobile: ?0', IndyTextEncoding_UTF8);
          TCPC.IOHandler.WriteLn('sec-ch-ua-platform: "Windows"', IndyTextEncoding_UTF8);
          TCPC.IOHandler.WriteLn('Upgrade-Insecure-Requests: 1', IndyTextEncoding_UTF8);
          TCPC.IOHandler.WriteLn('User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36', IndyTextEncoding_UTF8);
          TCPC.IOHandler.Writeln('Accept: text/html,application/xhtml xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', IndyTextEncoding_UTF8);
          TCPC.IOHandler.WriteLn('Sec-Fetch-Site: none', IndyTextEncoding_UTF8);
          TCPC.IOHandler.WriteLn('Sec-Fetch-Mode: navigate', IndyTextEncoding_UTF8);
          TCPC.IOHandler.WriteLn('Sec-Fetch-User: ?1', IndyTextEncoding_UTF8);
          TCPC.IOHandler.WriteLn('Sec-Fetch-Dest: document', IndyTextEncoding_UTF8);
          TCPC.IOHandler.WriteLn('Accept-Encoding: gzip, deflate, br', IndyTextEncoding_UTF8);
          TCPC.IOHandler.WriteLn('Accept-Language: en-US,en;q=0.9', IndyTextEncoding_UTF8);
        end;

        result := TCPC.IOHandler.AllData(IndyTextEncoding_UTF8);

        logging(result);
      finally
        TCPC.Disconnect;
      end;
    finally
      lParam.Free;
      OnSSL.Free;
      TCPC.Free;
    end;
  end;
end;

end.

CodePudding user response：

To start with you need to send a blank line to mark the end of the HTTP metadata/headers.

TCPC.IOHandler.Writeln('',IndyTextEncoding_UTF8);

CodePudding user response：

There are many problems with your code:

wasting memory to create an unused TStringList object.
enabling obsolete SSL v2 and v3, which are no longer secure.
calling IOHandler.ReadLn() before sending the HTTP request. The HTTP/S protocol does not involve an initial greeting.
making an erroneous IOHandler.Write() call for the sec-ch-ua header. It should be using IOHandler.WriteLn() instead.
requesting that compression of the HTTP body is allowed, when you are not actually prepared to decompress it.
not terminating the HTTP request correctly. You need to send a blank line after the headers. This is the reason for the server timeout error.
using IOHandler.AllData() to read the HTTP response, which will read until the server disconnects, but you are also enabling HTTP keepalives, which asks the server not to disconnect. Even without HTTP keepalives (since you are destroying the connection after reading the HTTP response, using a keepalive makes no sense anyway), AllData() is still the wrong way to read an HTTP response. (I've posted many answers in the past regarding the proper way to read an HTTP response).
redundant use of the IOHandler.DefStringEncoding property and AByteEncoding parameters. You don't need to use the latter when using the former, and vice versa.

Try this instead:

procedure TForm1.Button1Click(Sender: TObject);
var
  Result: string;
  TCPC: TIdTCPClient;
  OnSSL: TIdSSLIOHandlerSocketOpenSSL;
begin
  TCPC := TIdTCPClient.Create();
  try
    OnSSL := TIdSSLIOHandlerSocketOpenSSL.Create(TCPC);
    OnSSL.SSLOptions.SSLVersions := [sslvTLSv1, sslvTLSv1_1, sslvTLSv1_2];
    OnSSL.PassThrough := False;
    OnSSL.DefStringEncoding := IndyTextEncoding_UTF8;
    
    TCPC.Host := 'xxxxxxx.edu';
    TCPC.Port := 443;
    TCPC.ConnectTimeout := 100000;
    TCPC.ReadTimeout := 500000;
    TCPC.IOHandler := OnSSL;

    TCPC.Connect;
    try
      TCPC.IOHandler.WriteLn('GET / HTTP/1.1');
      TCPC.IOHandler.WriteLn('Host: xxxxxxx.edu');
      TCPC.IOHandler.WriteLn('Connection: close');
      TCPC.IOHandler.WriteLn('Pragma: no-cache');
      TCPC.IOHandler.WriteLn('Cache-Control: no-cache');
      TCPC.IOHandler.WriteLn('sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="96", "Google Chrome";v="96"');
      TCPC.IOHandler.WriteLn('sec-ch-ua-mobile: ?0');
      TCPC.IOHandler.WriteLn('sec-ch-ua-platform: "Windows"');
      TCPC.IOHandler.WriteLn('Upgrade-Insecure-Requests: 1');
      TCPC.IOHandler.WriteLn('User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36');
      TCPC.IOHandler.Writeln('Accept: text/html,application/xhtml xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9');
      TCPC.IOHandler.WriteLn('Sec-Fetch-Site: none');
      TCPC.IOHandler.WriteLn('Sec-Fetch-Mode: navigate');
      TCPC.IOHandler.WriteLn('Sec-Fetch-User: ?1');
      TCPC.IOHandler.WriteLn('Sec-Fetch-Dest: document');
      //TCPC.IOHandler.WriteLn('Accept-Encoding: gzip, deflate, br');
      TCPC.IOHandler.WriteLn('Accept-Language: en-US,en;q=0.9');
      TCPC.IOHandler.WriteLn();

      // I'm leaving this here only because I don't have the time right now
      // to rewrite it properly. Go read my previous answers on this topic...
      Result := TCPC.IOHandler.AllData();
    
      logging(Result);
    finally
      TCPC.Disconnect;
    end;
  finally
    TCPC.Free;
  end;
end;

That being said, you are correct that using TIdHTTP would be much simpler, eg:

procedure TForm1.Button1Click(Sender: TObject);
var
  Result: string;
  HTTP: TIdHTTP;
  OnSSL: TIdSSLIOHandlerSocketOpenSSL;
begin
  HTTP := TIdHTTP.Create();
  try
    OnSSL := TIdSSLIOHandlerSocketOpenSSL.Create(TCPC);
    OnSSL.SSLOptions.SSLVersions := [sslvTLSv1, sslvTLSv1_1, sslvTLSv1_2];
    
    HTTP.ConnectTimeout := 100000;
    HTTP.ReadTimeout := 500000;
    HTTP.IOHandler := OnSSL;

    HTTP.Request.Connection := 'close';
    HTTP.Request.Pragma := 'no-cache';
    HTTP.Request.CacheControl := 'no-cache';
    HTTP.Request.UserAgent := 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36';
    HTTP.Request.Accept := 'text/html,application/xhtml xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9';
    HTTP.Request.AcceptLanguage := 'en-US,en;q=0.9';
    HTTP.Request.CustomHeaders.Values['sec-ch-ua'] :='" Not A;Brand";v="99", "Chromium";v="96", "Google Chrome";v="96"';
    HTTP.Request.CustomHeaders.Values['sec-ch-ua-mobile'] := '?0';
    HTTP.Request.CustomHeaders.Values['sec-ch-ua-platform'] := '"Windows"';
    HTTP.Request.CustomHeaders.Values['Upgrade-Insecure-Requests'] := '1';
    HTTP.Request.CustomHeaders.Values['Sec-Fetch-Site'] := 'none';
    HTTP.Request.CustomHeaders.Values['Sec-Fetch-Mode'] := 'navigate';
    HTTP.Request.CustomHeaders.Values['Sec-Fetch-User'] := '?1';
    HTTP.Request.CustomHeaders.Values['Sec-Fetch-Dest'] := 'document';

    // if you want to support compressed responses, assign a
    // TIdZLibCompressorBase-derived component to the
    // TIdHTTP.Compressor property...

    Result := HTTP.Get('https://xxxxxxx.edu/');
    
    logging(Result);
  finally
    HTTP.Free;
  end;
end;

There are very few reasons to use TIdTCPClient directly rather than TIdHTTP.