I have found so many questions about this on here, but not sure why they are not answered.
I am trying to crawl a web page after logging in with this code : source
var steps=[]; var testindex = 0; var loadInProgress = false;//This is set to true when a page is still loading /*********SETTINGS*********************/ var webPage = require('webpage'); var page = webPage.create(); page.settings.userAgent = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36'; page.settings.javascriptEnabled = true; page.settings.loadImages = false;//Script is much faster with this field set to false phantom.cookiesEnabled = true; phantom.javascriptEnabled = true; /*********SETTINGS END*****************/ console.log('All settings loaded, start with execution'); page.onConsoleMessage = function(msg) { console.log(msg); }; /**********DEFINE STEPS THAT FANTOM SHOULD DO***********************/ steps = [ //Step 1 - Open Amazon home page function(){ console.log('Step 1 - Abrindo página de login'); page.open("http://parceriascury.housecrm.com.br", function(status){ }); }, //Step 3 - Populate and submit the login form function(){ console.log('Step 3 - Preenchendo o form'); page.evaluate(function(){ document.getElementById("login").value="xxxxx"; document.getElementById("senha").value="xxxxx"; document.getElementById("frmlandingpage").submit(); }); }, //Step 4 - Wait Amazon to login user. After user is successfully logged in, user is redirected to home page. Content of the home page is saved to AmazonLoggedIn.html. You can find this file where phantomjs.exe file is. You can open this file using Chrome to ensure that you are logged in. function(){ console.log("Step 4 - Wait Amazon to login user. After user is successfully logged in, user is redirected to home page. Content of the home page is saved to AmazonLoggedIn.html. You can find this file where phantomjs.exe file is. You can open this file using Chrome to ensure that you are logged in."); var fs = require('fs'); var result = page.evaluate(function() { return document.documentElement.outerHTML; }); fs.write('C:\\phantomjs\\logado_cury_10.html',result,'w'); }, ]; /**********END STEPS THAT FANTOM SHOULD DO***********************/ //Execute steps one by one interval = setInterval(executeRequestsStepByStep,5000); function executeRequestsStepByStep(){ if (loadInProgress == false && typeof steps[testindex] == "function") { //console.log("step " + (testindex + 1)); steps[testindex](); testindex++; } if (typeof steps[testindex] != "function") { console.log("test complete!"); phantom.exit(); } } /** * These listeners are very important in order to phantom work properly. Using these listeners, we control loadInProgress marker which controls, weather a page is fully loaded. * Without this, we will get content of the page, even a page is not fully loaded. */ page.onLoadStarted = function() { loadInProgress = true; console.log('Loading started'); }; page.onLoadFinished = function() { loadInProgress = false; console.log('Loading finished'); }; page.onConsoleMessage = function(msg) { console.log(msg); }; But the response only this:
<html><head></head><body>ok</body></html> I need to get the content of next page with URL:
http://parceriascury.housecrm.com.br/parceiro_busca I can access this page directly, but not with all complements, because it needs to be logged in.
No errors and I don't know where I am making a mistake.
Edit Other solutions are welcome, i think maybe curl...But after js loading...
Sorry for my bad english.
1 Answers
Answers 1
This code could be better:
var loadInProgress = false;//This is set to true when a page is still loading /*********SETTINGS*********************/ var page = require('webpage').create({viewportSize:{width: 1600,height: 900}, settings:{userAgent:'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36', javascriptEnabled:'true', loadImages:'false' }}); var fs = require('fs'); /*********SETTINGS END*****************/ console.log('All settings loaded, start with execution'); /** * These listeners are very important in order to phantom work properly. Using these listeners, we control loadInProgress marker which controls, weather a page is fully loaded. * Without this, we will get content of the page, even a page is not fully loaded. */ page.onLoadStarted = function() { loadInProgress = true; console.log('Loading started'); }; page.onLoadFinished = function() { loadInProgress = false; console.log('Loading finished'); }; page.onConsoleMessage = function(msg) { console.log(msg); }; //Log in to your account, then view the cookie you got, now you can use these cookies to login // the site will recognize you with your cookies. //for freebitco.in auth phantom.cookies = [{// an array of objects 'name' : 'btc_address', 'value' : '1AuMxR6sPtB2Z6TkahSnpmm1H4KpYPBKqe', 'domain' : 'freebitco.in', 'path' : '/', 'httponly' : false, 'secure' : true, 'expires' : (new Date()).getTime() + (1000 * 60 * 60 * 43800) //5 years },{ 'name' : 'password', 'value' : 'f574ca68a8650d1264d38da4b7687ca3bf631e6dfc59a98c89dd2564c7601f84', 'domain' : 'freebitco.in', 'path' : '/', 'httponly' : false, 'secure' : true, 'expires' : (new Date()).getTime() + (1000 * 60 * 60 * 43800) }] //Execute steps one by one page.open("http://parceriascury.housecrm.com.br/parceiro_busca", function(status){ console.log('Step 1 has been completed - we are on the target page!'); setTimeout(step2,5000);// Maybe we don't need to wait here, we can execute step2 immediately. function step2(){ console.log("Step 2 - Content of the home page is saved to AmazonLoggedIn.html. You can find this file where phantomjs.exe file is. You can open this file using Chrome to ensure that you are logged in."); var result = page.evaluate(function(){ return document.documentElement.outerHTML; }); fs.write('C:\\phantomjs\\logado_cury_10.html',result,'w'); phantom.exit(); } }); 
0 comments:
Post a Comment