How is data being fetched-CodePudding

While working on finding how data is being processed on the webpage. I was figuring out this site investorscout.co/investors.

I tried looking at the Network tab to see how they are rendering the data from backend onto the page. I have also looked into WS but no luck.

I am confused as to how the site is able to display the data while none of the requests in the Network tab shows that. I aim to fetch the data using requests and bs4.

CodePudding user response：

Sending a GET request to the page https://investorscout.co/investors returns a response with multiple references to external JavaScript code in it. This is what is being loaded on the page - dynamic content based on JavaScript functions.

I would suggest an implementation involving selenium instead as you would not be able to scrape content on the page otherwise.

HTML code of page for reference:

<!DOCTYPE html>
<html lang="en">
  <head>
    <meta charset="utf-8" />
    <meta
      name="viewport"
      content="width=device-width,initial-scale=1,shrink-to-fit=no"
    />
    <meta name="theme-color" content="#000000" />
    <link rel="manifest" href="/manifest.json" />
    <link rel="”shortcut" icon” href="”/favicon.ico”" />
    <title>Investor Scout</title>
    <script>
      !function(n,u){n._rwq=u,n[u]=n[u]||function(){(n[u].q=n[u].q||[]).push(arguments)}}(window,"rewardful")
    </script>
    <script
      async
      src="https://r.wdfl.co/rw.js"
      data-rewardful="76b542"
    ></script>
    <script type="text/javascript">
      var _iub=_iub||[];_iub.csConfiguration={consentOnContinuedBrowsing:!1,ccpaAcknowledgeOnDisplay:!0,whitelabel:!1,lang:"en",siteId:2020596,enableCcpa:!0,countryDetection:!0,cookiePolicyId:26558236,banner:{acceptButtonDisplay:!0,customizeButtonDisplay:!0,acceptButtonColor:"#0073CE",acceptButtonCaptionColor:"white",customizeButtonColor:"#DADADA",customizeButtonCaptionColor:"#4D4D4D",rejectButtonColor:"#0073CE",rejectButtonCaptionColor:"white",position:"float-top-center",textColor:"black",backgroundColor:"white"}}
    </script>
    <script
      type="text/javascript"
      src="//cdn.iubenda.com/cs/ccpa/stub.js"
    ></script>
    <script
      type="text/javascript"
      src="//cdn.iubenda.com/cs/iubenda_cs.js"
      charset="UTF-8"
      async
    ></script>
    <script
      defer="defer"
      src="https://use.fontawesome.com/releases/v5.3.1/js/all.js"
    ></script>
    <script type="text/javascript">
      window.__lo_site_id=176375,function(){var t=document.createElement("script");t.type="text/javascript",t.async=!0,t.src="https://d10lpsik1i8c69.cloudfront.net/w.js";var e=document.getElementsByTagName("script")[0];e.parentNode.insertBefore(t,e)}()
    </script>
    <script type="text/javascript">
      window.$crisp=[],window.CRISP_WEBSITE_ID="95efad36-fefd-4cf1-ae4b-a3bb5a61360c",d=document,s=d.createElement("script"),s.src="https://client.crisp.chat/l.js",s.async=1,d.getElementsByTagName("head")[0].appendChild(s)
    </script>
    <link href="/static/css/main.fc05b0f9.chunk.css" rel="stylesheet" />
  </head>
  <body>
    <noscript>You need to enable JavaScript to run this app.</noscript>
    <div id="root"></div>
    <script>
      !function(e){function t(t){for(var n,i,l=t[0],f=t[1],a=t[2],p=0,s=[];p<l.length;p  )i=l[p],Object.prototype.hasOwnProperty.call(o,i)&&o[i]&&s.push(o[i][0]),o[i]=0;for(n in f)Object.prototype.hasOwnProperty.call(f,n)&&(e[n]=f[n]);for(c&&c(t);s.length;)s.shift()();return u.push.apply(u,a||[]),r()}function r(){for(var e,t=0;t<u.length;t  ){for(var r=u[t],n=!0,l=1;l<r.length;l  ){var f=r[l];0!==o[f]&&(n=!1)}n&&(u.splice(t--,1),e=i(i.s=r[0]))}return e}var n={},o={1:0},u=[];function i(t){if(n[t])return n[t].exports;var r=n[t]={i:t,l:!1,exports:{}};return e[t].call(r.exports,r,r.exports,i),r.l=!0,r.exports}i.m=e,i.c=n,i.d=function(e,t,r){i.o(e,t)||Object.defineProperty(e,t,{enumerable:!0,get:r})},i.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},i.t=function(e,t){if(1&t&&(e=i(e)),8&t)return e;if(4&t&&"object"==typeof e&&e&&e.__esModule)return e;var r=Object.create(null);if(i.r(r),Object.defineProperty(r,"default",{enumerable:!0,value:e}),2&t&&"string"!=typeof e)for(var n in e)i.d(r,n,function(t){return e[t]}.bind(null,n));return r},i.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return i.d(t,"a",t),t},i.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},i.p="/";var l=this["webpackJsonpinvestor-scout"]=this["webpackJsonpinvestor-scout"]||[],f=l.push.bind(l);l.push=t,l=l.slice();for(var a=0;a<l.length;a  )t(l[a]);var c=f;r()}([])
    </script>
    <script src="/static/js/2.540fc93a.chunk.js"></script>
    <script src="/static/js/main.9ac00620.chunk.js"></script>
  </body>
</html>