Why does this code result in 301 error when trying to access sites that has a .net suffix

void test(const char * host, const char *index)
{
    BIO *bio, *out;
    SSL_CTX * ctx;
    SSL * ssl;
    int len;
    char tmpbuf[1024];
    ERR_load_crypto_strings();
    char ready[1204];
    char format[] = "%s:http";
    sprintf(ready, format , host);
    char req_template[] =  "GET %s HTTP/1.1\r\nHost: %s\r\nConnection: close\r\n\r\n";
    char ready_request[1024];
    sprintf(ready_request , req_template , index, host);
    const SSL_METHOD * method = SSLv23_client_method(); 
    if (!method)
        exit(-1);
    ctx = SSL_CTX_new(method);
    if (!ctx)
        exit(-1);
    if (!SSL_CTX_load_verify_locations(ctx,"/etc/ssl/certs/ca-certificates.crt","/etc/ssl/certs/"))
    {
        SSL_CTX_free(ctx);
        exit(-1);
    }
    bio = BIO_new_ssl_connect(ctx); 
    BIO_get_ssl(bio, &ssl);
    SSL_set_mode(ssl, SSL_MODE_AUTO_RETRY);
    char temp[1024];
    sprintf(temp, "%s:https",host);
    if (BIO_set_conn_hostname(bio, temp) < 0)
    {
        memset(temp, 0, sizeof(temp));
        sprintf(temp, "%s:http", host);
        bio = BIO_new_connect(temp);
        if (BIO_do_connect(bio) < 0 )
        {
            BIO_free_all(bio);
            SSL_CTX_free(ctx);
            exit(-1);
        }       
    }
    printf("###\n%s\n###\n",ready_request);
    out = BIO_new_fp(stdout, BIO_NOCLOSE);
    if(BIO_do_connect(bio) <= 0) 
        exit(-1);

    BIO_puts(bio,ready_request);
    for(;;) 
    {
        len = BIO_read(bio, tmpbuf, 1024);
        if(len <= 0) break;
        BIO_write(out, tmpbuf, len);
    }
    BIO_free(bio);
    BIO_free(out);
}
int main()
{

    test("openssl.org", "/docs/manpages.html");
    test("pastebin.com", "/raw/j0BnRwBw");
    test("pastebin.com", "/j0BnRwBw");

}

for some reason that i can't figure out the first time test is called it returns a 301 status code but the two time test is called it returns the html code or the paste code with out any problems

Does this have anything to do with the fact that the websites use different technologies or if they have some sort of firewall, I believe pastebin uses cloudflare to protect it self, I also tried using User-Agent but still got the same result

CodePudding user response：

The first request https://openssl.org/docs/manpages.html returns:

HTTP/1.1 301 Moved Permanently
...
Location: https://www.openssl.org/docs/manpages.html
...

You make another request to that url. To demonstrate it working, I changed your first test case to read:

test("www.openssl.org", "/docs/manpages.html");
//    ^^^^

and the server now returns the response you were expecting:

HTTP/1.1 200 OK
...                                                                                                        

CodePudding user response：

Just add www. as a prefix to the host header and for BIO_set_conn_hostname you should use the format www.<hostname>.com:https or www.<host>.org:http for BIO_new_connect

for some reason, the docs do not mention this

void test(const char * host, const char *index)
{
    BIO *bio, *out;
    SSL_CTX * ctx;
    SSL * ssl;
    int len;
    char tmpbuf[1024];
    ERR_load_crypto_strings();
    char req_template[] =  "GET %s HTTP/1.1\r\nHost: www.%s\r\nConnection: close\r\n\r\n";
    char ready_request[1024];
    sprintf(ready_request , req_template , index, host);
    const SSL_METHOD * method = SSLv23_client_method(); 
    if (!method)
        exit(-1);
    ctx = SSL_CTX_new(method);
    if (!ctx)
        exit(-1);
    if (!SSL_CTX_load_verify_locations(ctx,"/etc/ssl/certs/ca-certificates.crt","/etc/ssl/certs/"))
    {
        SSL_CTX_free(ctx);
        exit(-1);
    }
    bio = BIO_new_ssl_connect(ctx); 
    BIO_get_ssl(bio, &ssl);
    SSL_set_mode(ssl, SSL_MODE_AUTO_RETRY);
    char temp[1024];
    sprintf(temp, "www.%s:https",host);
    if (BIO_set_conn_hostname(bio, temp) < 0)
    {
        memset(temp, 0, sizeof(temp));
        sprintf(temp, "www.%s:http", host);
        bio = BIO_new_connect(temp);
        if (BIO_do_connect(bio) < 0 )
        {
            BIO_free_all(bio);
            SSL_CTX_free(ctx);
            exit(-1);
        }       
    }
    printf("###\n%s\n###\n",ready_request);
    out = BIO_new_fp(stdout, BIO_NOCLOSE);
    if(BIO_do_connect(bio) <= 0) 
        exit(-1);

    BIO_puts(bio,ready_request);
    for(;;) 
    {
        len = BIO_read(bio, tmpbuf, 1024);
        if(len <= 0) break;
        BIO_write(out, tmpbuf, len);
    }
    BIO_free(bio);
    BIO_free(out);
}