I try to get HTML code of the page, but driver.page_source always return HTML JavaScript.
I know it is many ways described earlier, but in my case they don't work.
The page is - https://qe.com.qa/en/companymoreinformationsearch?CompanyCode=QNBK
CodePudding user response：
innerHTML = driver.execute_script("return document.documentElement.outerHTML")
CodePudding user response：
To extract the HTML along with the JavaScript you need to induce WebDriverWait for the visibility_of_all_elements_located() with id as moreinfo-heading and you can use the following Locator Strategy:
Code Block:
Using
driver.page_source:driver.get("https://qe.com.qa/en/companymoreinformationsearch?CompanyCode=QNBK") WebDriverWait(driver, 20).until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, "#moreinfo-heading"))) print(driver.page_source)
As an alternative, you can also use document.documentElement.outerHTML and document.documentElement.innerHTML as follows:
Using
document.documentElement.outerHTML:driver.get("https://qe.com.qa/en/companymoreinformationsearch?CompanyCode=QNBK") print(driver.execute_script("return document.documentElement.outerHTML"))Using
document.documentElement.innerHTML:driver.get("https://qe.com.qa/en/companymoreinformationsearch?CompanyCode=QNBK") print(driver.execute_script("return document.documentElement.innerHTML"))
Console Output:
getSessionId: function() {
return '';
},
getSiteAdminURL: function() {
return 'https://qe.com.qa/group/guest/~/control_panel/manage?p_p_lifecycle=0&p_p_state=maximized&p_p_mode=view';
},
getSiteGroupId: function() {
return '20181';
},
getURLControlPanel: function() {
return '/en/group/control_panel?refererPlid=1002057';
},
getURLHome: function() {
return 'https\x3a\x2f\x2fqe\x2ecom\x2eqa\x2fweb\x2fguest';
},
getUserEmailAddress: function() {
return '';
},
getUserId: function() {
return '20158';
},
getUserName: function() {
return '';
},
isAddSessionIdToURL: function() {
return false;
},
isImpersonated: function() {
return false;
},
isSignedIn: function() {
return false;
},
isStateExclusive: function() {
return false;
},
isStateMaximized: function() {
return false;
},
isStatePopUp: function() {
return false;
}
};
var themeDisplay = Liferay.ThemeDisplay;
Liferay.AUI = {
getAvailableLangPath: function() {
return 'available_languages.jsp?browserId=other&themeId=mainsitetheme_WAR_mainsitetheme&colorSchemeId=01&minifierType=js&languageId=en_US&b=7210&t=1635431572526';
},
getCombine: function() {
return true;
},
getComboPath: function() {
return '/combo/?browserId=other&minifierType=&languageId=en_US&b=7210&t=1620065756342&';
},
getDateFormat: function() {
return '%m/%d/%Y';
},
getEditorCKEditorPath: function() {
return '/o/frontend-editor-ckeditor-web';
},
getFilter: function() {
var filter = 'raw';
filter = 'min';
return filter;
},
getFilterConfig: function() {
var instance = this;
var filterConfig = null;
if (!instance.getCombine()) {
filterConfig = {
replaceStr: '.js' instance.getStaticResourceURLParams(),
searchExp: '\\.js$'
};
}
return filterConfig;
},
getJavaScriptRootPath: function() {
return '/o/frontend-js-web';
},
getLangPath: function() {
return 'aui_lang.jsp?browserId=other&themeId=mainsitetheme_WAR_mainsitetheme&colorSchemeId=01&minifierType=js&languageId=en_US&b=7210&t=1620065756342';
},
getPortletRootPath: function() {
return '/html/portlet';
},
getStaticResourceURLParams: function() {
return '?browserId=other&minifierType=&languageId=en_US&b=7210&t=1620065756342';
}
};
Liferay.authToken = '8LGXsu8b';
Liferay.currentURL = '\x2fen\x2fcompanymoreinformationsearch\x3fCompanyCode\x3dQNBK';
Liferay.currentURLEncoded = '\x252Fen\x252Fcompanymoreinformationsearch\x253FCompanyCode\x253DQNBK';
// ]]>
</script>
<script src="/o/js_loader_config?t=1635431593465" type="text/javascript"></script>
<script data-senna-track="permanent" src="/combo?browserId=other&minifierType=js&languageId=en_US&b=7210&t=1620065756342&/o/frontend-js-web/loader/config.js&/o/frontend-js-web/loader/loader.js&/o/frontend-js-web/aui/aui/aui.js&/o/frontend-js-web/aui/aui-base-html5-shiv/aui-base-html5-shiv.js&/o/frontend-js-web/liferay/browser_selectors.js&/o/frontend-js-web/liferay/modules.js&/o/frontend-js-web/liferay/aui_sandbox.js&/o/frontend-js-web/misc/svg4everybody.js&/o/frontend-js-web/aui/arraylist-add/arraylist-add.js&/o/frontend-js-web/aui/arraylist-filter/arraylist-filter.js&/o/frontend-js-web/aui/arraylist/arraylist.js&/o/frontend-js-web/aui/array-extras/array-extras.js&/o/frontend-js-web/aui/array-invoke/array-invoke.js&/o/frontend-js-web/aui/attribute-base/attribute-base.js&/o/frontend-js-web/aui/attribute-complex/attribute-complex.js&/o/frontend-js-web/aui/attribute-core/attribute-core.js&/o/frontend-js-web/aui/attribute-observable/attribute-observable.js&/o/frontend-js-web/aui/attribute-extras/attribute-extras.js&/o/frontend-js-web/aui/base-base/base-base.js&/o/frontend-js-web/aui/base-pluginhost/base-pluginhost.js&/o/frontend-js-web/aui/classnamemanager/classnamemanager.js&/o/frontend-js-web/aui/datatype-xml-format/datatype-xml-format.js&/o/frontend-js-web/aui/datatype-xml-parse/datatype-xml-parse.js&/o/frontend-js-web/aui/dom-base/dom-base.js&/o/frontend-js-web/aui/dom-core/dom-core.js&/o/frontend-js-web/aui/dom-screen/dom-screen.js&/o/frontend-js-web/aui/dom-style/dom-style.js&/o/frontend-js-web/aui/event-base/event-base.js&/o/frontend-js-web/aui/event-custom-base/event-custom-base.js&/o/frontend-js-web/aui/event-custom-complex/event-custom-complex.js&/o/frontend-js-web/aui/event-delegate/event-delegate.js&/o/frontend-js-web/aui/event-focus/event-focus.js&/o/frontend-js-web/aui/event-hover/event-hover.js&/o/frontend-js-web/aui/event-key/event-key.js&/o/frontend-js-web/aui/event-mouseenter/event-mouseenter.js&/o/frontend-js-web/aui/event-mousewheel/event-mousewheel.js" type="text/javascript"></script>
<script data-senna-track="permanent" src="/combo?browserId=other&minifierType=js&languageId=en_US&b=7210&t=1620065756342&/o/frontend-js-web/aui/event-outside/event-outside.js&/o/frontend-js-web/aui/event-resize/event-resize.js&/o/frontend-js-web/aui/event-simulate/event-simulate.js&/o/frontend-js-web/aui/event-synthetic/event-synthetic.js&/o/frontend-js-web/aui/intl/intl.js&/o/frontend-js-web/aui/io-base/io-base.js&/o/frontend-js-web/aui/io-form/io-form.js&/o/frontend-js-web/aui/io-queue/io-queue.js&/o/frontend-js-web/aui/io-upload-iframe/io-upload-iframe.js&/o/frontend-js-web/aui/io-xdr/io-xdr.js&/o/frontend-js-web/aui/json-parse/json-parse.js&/o/frontend-js-web/aui/json-stringify/json-stringify.js&/o/frontend-js-web/aui/node-base/node-base.js&/o/frontend-js-web/aui/node-core/node-core.js&/o/frontend-js-web/aui/node-event-delegate/node-event-delegate.js&/o/frontend-js-web/aui/node-event-simulate/node-event-simulate.js&/o/frontend-js-web/aui/node-focusmanager/node-focusmanager.js&/o/frontend-js-web/aui/node-pluginhost/node-pluginhost.js&/o/frontend-js-web/aui/node-screen/node-screen.js&/o/frontend-js-web/aui/node-style/node-style.js&/o/frontend-js-web/aui/oop/oop.js&/o/frontend-js-web/aui/plugin/plugin.js&/o/frontend-js-web/aui/pluginhost-base/pluginhost-base.js&/o/frontend-js-web/aui/pluginhost-config/pluginhost-config.js&/o/frontend-js-web/aui/querystring-stringify-simple/querystring-stringify-simple.js&/o/frontend-js-web/aui/queue-promote/queue-promote.js&/o/frontend-js-web/aui/selector-css2/selector-css2.js&/o/frontend-js-web/aui/selector-css3/selector-css3.js&/o/frontend-js-web/aui/selector-native/selector-native.js&/o/frontend-js-web/aui/selector/selector.js&/o/frontend-js-web/aui/widget-base/widget-base.js&/o/frontend-js-web/aui/widget-htmlparser/widget-htmlparser.js&/o/frontend-js-web/aui/widget-skin/widget-skin.js&/o/frontend-js-web/aui/widget-uievents/widget-uievents.js&/o/frontend-js-web/aui/yui-throttle/yui-throttle.js&/o/frontend-js-web/aui/aui-base-core/aui-base-core.js" type="text/javascript"></script>
<script data-senna-track="permanent" src="/combo?browserId=other&minifierType=js&languageId=en_US&b=7210&t=1620065756342&/o/frontend-js-web/aui/aui-base-lang/aui-base-lang.js&/o/frontend-js-web/aui/aui-classnamemanager/aui-classnamemanager.js&/o/frontend-js-web/aui/aui-component/aui-component.js&/o/frontend-js-web/aui/aui-debounce/aui-debounce.js&/o/frontend-js-web/aui/aui-delayed-task-deprecated/aui-delayed-task-deprecated.js&/o/frontend-js-web/aui/aui-event-base/aui-event-base.js&/o/frontend-js-web/aui/aui-event-input/aui-event-input.js&/o/frontend-js-web/aui/aui-form-validator/aui-form-validator.js&/o/frontend-js-web/aui/aui-node-base/aui-node-base.js&/o/frontend-js-web/aui/aui-node-html5/aui-node-html5.js&/o/frontend-js-web/aui/aui-selector/aui-selector.js&/o/frontend-js-web/aui/aui-timer/aui-timer.js&/o/frontend-js-web/liferay/dependency.js&/o/frontend-js-web/liferay/dom_task_runner.js&/o/frontend-js-web/liferay/events.js&/o/frontend-js-web/liferay/language.js&/o/frontend-js-web/liferay/lazy_load.js&/o/frontend-js-web/liferay/liferay.js&/o/frontend-js-web/liferay/util.js&/o/frontend-js-web/liferay/global.bundle.js&/o/frontend-js-web/liferay/portal.js&/o/frontend-js-web/liferay/portlet.js&/o/frontend-js-web/liferay/workflow.js&/o/frontend-js-web/liferay/form.js&/o/frontend-js-web/liferay/form_placeholders.js&/o/frontend-js-web/liferay/icon.js&/o/frontend-js-web/liferay/menu.js&/o/frontend-js-web/liferay/notice.js&/o/frontend-js-web/liferay/poller.js" type="text/javascript"></script>
<script data-senna-track="temporary" src="/o/js_bundle_config?t=1635431612919" type="text/javascript"></script>
<script data-senna-track="temporary" type="text/javascript">
// <![CDATA[
// ]]>
</script>
...