How can I scrape Twitter user followers using PhantomJS?
-2
I am using PhantomJS and I'm trying to scrape Twitter followers for a specific user but I can't find any result.
I used some code to log in to Twitter and scrape followers, but it doesn't work, and I've searched on websites but I can't solve this problem.
I think I need to first log in to Twitter and redirect the browser to the follower link and scrape it, but it doesn't work for me. Can anyone edit this code and give me the right code?
var steps=;
var testindex = 0;
var loadInProgress = false;//This is set to true when a page is still loading
/*********SETTINGS*********************/
var webPage = require('webpage');
var page = webPage.create();
page.settings.userAgent = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36';
page.settings.javascriptEnabled = true;
page.settings.loadImages = false;//Script is much faster with this field set to false
phantom.cookiesEnabled = true;
phantom.javascriptEnabled = true;
/*********SETTINGS END*****************/
console.log('All settings loaded, start with execution');
page.onConsoleMessage = function(msg) {
console.log(msg);
};
/**********DEFINE STEPS THAT FANTOM SHOULD DO***********************/
steps = [
//Step 1 - Open Twitter login page
function(){
console.log('Step 1 - Open Twitter Login page');
page.open("https://twitter.com", function(status){
});
},
//Step 2 - Populate and submit the login form
function(){
console.log('Step 2 - Populate and submit the login form');
page.evaluate(function(){
document.getElementsByName("session[username_or_email]")[1].value="YOUR USERNAME";
document.getElementsByName("session[password]")[1].value="YOUR PASSWORD";
document.getElementsByClassName('flex-table-btn')[0].click();
});
},
//Step 3 - Wait for loading Twitter home page and extract the content
function(){
console.log("Step 3 - Wait for loading Twitter home page and extract the content");
var fs = require('fs');
var result = page.evaluate(function() {
var pageTweets = document.getElementsByClassName('js-tweet-text');
var result=new Array();
for(var i=0; i < pageTweets.length; i++){
result.push(pageTweets[i].innerHTML);
}
return JSON.stringify(result);
});
fs.write('tweets.txt',result,'w');
},
];
/**********END STEPS THAT FANTOM SHOULD DO***********************/
//Execute steps one by one
interval = setInterval(executeRequestsStepByStep,50);
function executeRequestsStepByStep(){
if (loadInProgress == false && typeof steps[testindex] == "function") {
//console.log("step " + (testindex + 1));
steps[testindex]();
testindex++;
}
if (typeof steps[testindex] != "function") {
console.log("test complete!");
phantom.exit();
}
}
/**
* These listeners are very important in order to phantom work properly. Using these listeners, we control loadInProgress marker which controls, weather a page is fully loaded.
* Without this, we will get content of the page, even a page is not fully loaded.
*/
page.onLoadStarted = function() {
loadInProgress = true;
console.log('Loading started');
};
page.onLoadFinished = function() {
loadInProgress = false;
console.log('Loading finished');
};
page.onConsoleMessage = function(msg) {
console.log(msg);
};javascript phantomjs
edited Jan 20 at 16:57
Mihai Chelaru
2,210101122
asked Jan 20 at 12:21
zakaria elaghazakaria elagha
1
add a comment |
-2
I am using PhantomJS and I'm trying to scrape Twitter followers for a specific user but I can't find any result.
I used some code to log in to Twitter and scrape followers, but it doesn't work, and I've searched on websites but I can't solve this problem.
I think I need to first log in to Twitter and redirect the browser to the follower link and scrape it, but it doesn't work for me. Can anyone edit this code and give me the right code?
var steps=;
var testindex = 0;
var loadInProgress = false;//This is set to true when a page is still loading
/*********SETTINGS*********************/
var webPage = require('webpage');
var page = webPage.create();
page.settings.userAgent = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36';
page.settings.javascriptEnabled = true;
page.settings.loadImages = false;//Script is much faster with this field set to false
phantom.cookiesEnabled = true;
phantom.javascriptEnabled = true;
/*********SETTINGS END*****************/
console.log('All settings loaded, start with execution');
page.onConsoleMessage = function(msg) {
console.log(msg);
};
/**********DEFINE STEPS THAT FANTOM SHOULD DO***********************/
steps = [
//Step 1 - Open Twitter login page
function(){
console.log('Step 1 - Open Twitter Login page');
page.open("https://twitter.com", function(status){
});
},
//Step 2 - Populate and submit the login form
function(){
console.log('Step 2 - Populate and submit the login form');
page.evaluate(function(){
document.getElementsByName("session[username_or_email]")[1].value="YOUR USERNAME";
document.getElementsByName("session[password]")[1].value="YOUR PASSWORD";
document.getElementsByClassName('flex-table-btn')[0].click();
});
},
//Step 3 - Wait for loading Twitter home page and extract the content
function(){
console.log("Step 3 - Wait for loading Twitter home page and extract the content");
var fs = require('fs');
var result = page.evaluate(function() {
var pageTweets = document.getElementsByClassName('js-tweet-text');
var result=new Array();
for(var i=0; i < pageTweets.length; i++){
result.push(pageTweets[i].innerHTML);
}
return JSON.stringify(result);
});
fs.write('tweets.txt',result,'w');
},
];
/**********END STEPS THAT FANTOM SHOULD DO***********************/
//Execute steps one by one
interval = setInterval(executeRequestsStepByStep,50);
function executeRequestsStepByStep(){
if (loadInProgress == false && typeof steps[testindex] == "function") {
//console.log("step " + (testindex + 1));
steps[testindex]();
testindex++;
}
if (typeof steps[testindex] != "function") {
console.log("test complete!");
phantom.exit();
}
}
/**
* These listeners are very important in order to phantom work properly. Using these listeners, we control loadInProgress marker which controls, weather a page is fully loaded.
* Without this, we will get content of the page, even a page is not fully loaded.
*/
page.onLoadStarted = function() {
loadInProgress = true;
console.log('Loading started');
};
page.onLoadFinished = function() {
loadInProgress = false;
console.log('Loading finished');
};
page.onConsoleMessage = function(msg) {
console.log(msg);
};javascript phantomjs
edited Jan 20 at 16:57
Mihai Chelaru
2,210101122
asked Jan 20 at 12:21
zakaria elaghazakaria elagha
1
1
Instead of saying it doesn't work, explain precisely what you're trying to achieve and what error you're facing
– Nino Filiu
Jan 20 at 12:36
i want to scrap follower from user but i cant find the method to do it
– zakaria elagha
Jan 20 at 13:05
add a comment |
-2
-2
-2
I am using PhantomJS and I'm trying to scrape Twitter followers for a specific user but I can't find any result.
I used some code to log in to Twitter and scrape followers, but it doesn't work, and I've searched on websites but I can't solve this problem.
I think I need to first log in to Twitter and redirect the browser to the follower link and scrape it, but it doesn't work for me. Can anyone edit this code and give me the right code?
var steps=;
var testindex = 0;
var loadInProgress = false;//This is set to true when a page is still loading
/*********SETTINGS*********************/
var webPage = require('webpage');
var page = webPage.create();
page.settings.userAgent = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36';
page.settings.javascriptEnabled = true;
page.settings.loadImages = false;//Script is much faster with this field set to false
phantom.cookiesEnabled = true;
phantom.javascriptEnabled = true;
/*********SETTINGS END*****************/
console.log('All settings loaded, start with execution');
page.onConsoleMessage = function(msg) {
console.log(msg);
};
/**********DEFINE STEPS THAT FANTOM SHOULD DO***********************/
steps = [
//Step 1 - Open Twitter login page
function(){
console.log('Step 1 - Open Twitter Login page');
page.open("https://twitter.com", function(status){
});
},
//Step 2 - Populate and submit the login form
function(){
console.log('Step 2 - Populate and submit the login form');
page.evaluate(function(){
document.getElementsByName("session[username_or_email]")[1].value="YOUR USERNAME";
document.getElementsByName("session[password]")[1].value="YOUR PASSWORD";
document.getElementsByClassName('flex-table-btn')[0].click();
});
},
//Step 3 - Wait for loading Twitter home page and extract the content
function(){
console.log("Step 3 - Wait for loading Twitter home page and extract the content");
var fs = require('fs');
var result = page.evaluate(function() {
var pageTweets = document.getElementsByClassName('js-tweet-text');
var result=new Array();
for(var i=0; i < pageTweets.length; i++){
result.push(pageTweets[i].innerHTML);
}
return JSON.stringify(result);
});
fs.write('tweets.txt',result,'w');
},
];
/**********END STEPS THAT FANTOM SHOULD DO***********************/
//Execute steps one by one
interval = setInterval(executeRequestsStepByStep,50);
function executeRequestsStepByStep(){
if (loadInProgress == false && typeof steps[testindex] == "function") {
//console.log("step " + (testindex + 1));
steps[testindex]();
testindex++;
}
if (typeof steps[testindex] != "function") {
console.log("test complete!");
phantom.exit();
}
}
/**
* These listeners are very important in order to phantom work properly. Using these listeners, we control loadInProgress marker which controls, weather a page is fully loaded.
* Without this, we will get content of the page, even a page is not fully loaded.
*/
page.onLoadStarted = function() {
loadInProgress = true;
console.log('Loading started');
};
page.onLoadFinished = function() {
loadInProgress = false;
console.log('Loading finished');
};
page.onConsoleMessage = function(msg) {
console.log(msg);
};javascript phantomjs
edited Jan 20 at 16:57
Mihai Chelaru
2,210101122
asked Jan 20 at 12:21
zakaria elaghazakaria elagha
1
I am using PhantomJS and I'm trying to scrape Twitter followers for a specific user but I can't find any result.
I used some code to log in to Twitter and scrape followers, but it doesn't work, and I've searched on websites but I can't solve this problem.
I think I need to first log in to Twitter and redirect the browser to the follower link and scrape it, but it doesn't work for me. Can anyone edit this code and give me the right code?
var steps=;
var testindex = 0;
var loadInProgress = false;//This is set to true when a page is still loading
/*********SETTINGS*********************/
var webPage = require('webpage');
var page = webPage.create();
page.settings.userAgent = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36';
page.settings.javascriptEnabled = true;
page.settings.loadImages = false;//Script is much faster with this field set to false
phantom.cookiesEnabled = true;
phantom.javascriptEnabled = true;
/*********SETTINGS END*****************/
console.log('All settings loaded, start with execution');
page.onConsoleMessage = function(msg) {
console.log(msg);
};
/**********DEFINE STEPS THAT FANTOM SHOULD DO***********************/
steps = [
//Step 1 - Open Twitter login page
function(){
console.log('Step 1 - Open Twitter Login page');
page.open("https://twitter.com", function(status){
});
},
//Step 2 - Populate and submit the login form
function(){
console.log('Step 2 - Populate and submit the login form');
page.evaluate(function(){
document.getElementsByName("session[username_or_email]")[1].value="YOUR USERNAME";
document.getElementsByName("session[password]")[1].value="YOUR PASSWORD";
document.getElementsByClassName('flex-table-btn')[0].click();
});
},
//Step 3 - Wait for loading Twitter home page and extract the content
function(){
console.log("Step 3 - Wait for loading Twitter home page and extract the content");
var fs = require('fs');
var result = page.evaluate(function() {
var pageTweets = document.getElementsByClassName('js-tweet-text');
var result=new Array();
for(var i=0; i < pageTweets.length; i++){
result.push(pageTweets[i].innerHTML);
}
return JSON.stringify(result);
});
fs.write('tweets.txt',result,'w');
},
];
/**********END STEPS THAT FANTOM SHOULD DO***********************/
//Execute steps one by one
interval = setInterval(executeRequestsStepByStep,50);
function executeRequestsStepByStep(){
if (loadInProgress == false && typeof steps[testindex] == "function") {
//console.log("step " + (testindex + 1));
steps[testindex]();
testindex++;
}
if (typeof steps[testindex] != "function") {
console.log("test complete!");
phantom.exit();
}
}
/**
* These listeners are very important in order to phantom work properly. Using these listeners, we control loadInProgress marker which controls, weather a page is fully loaded.
* Without this, we will get content of the page, even a page is not fully loaded.
*/
page.onLoadStarted = function() {
loadInProgress = true;
console.log('Loading started');
};
page.onLoadFinished = function() {
loadInProgress = false;
console.log('Loading finished');
};
page.onConsoleMessage = function(msg) {
console.log(msg);
};var steps=;
var testindex = 0;
var loadInProgress = false;//This is set to true when a page is still loading
/*********SETTINGS*********************/
var webPage = require('webpage');
var page = webPage.create();
page.settings.userAgent = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36';
page.settings.javascriptEnabled = true;
page.settings.loadImages = false;//Script is much faster with this field set to false
phantom.cookiesEnabled = true;
phantom.javascriptEnabled = true;
/*********SETTINGS END*****************/
console.log('All settings loaded, start with execution');
page.onConsoleMessage = function(msg) {
console.log(msg);
};
/**********DEFINE STEPS THAT FANTOM SHOULD DO***********************/
steps = [
//Step 1 - Open Twitter login page
function(){
console.log('Step 1 - Open Twitter Login page');
page.open("https://twitter.com", function(status){
});
},
//Step 2 - Populate and submit the login form
function(){
console.log('Step 2 - Populate and submit the login form');
page.evaluate(function(){
document.getElementsByName("session[username_or_email]")[1].value="YOUR USERNAME";
document.getElementsByName("session[password]")[1].value="YOUR PASSWORD";
document.getElementsByClassName('flex-table-btn')[0].click();
});
},
//Step 3 - Wait for loading Twitter home page and extract the content
function(){
console.log("Step 3 - Wait for loading Twitter home page and extract the content");
var fs = require('fs');
var result = page.evaluate(function() {
var pageTweets = document.getElementsByClassName('js-tweet-text');
var result=new Array();
for(var i=0; i < pageTweets.length; i++){
result.push(pageTweets[i].innerHTML);
}
return JSON.stringify(result);
});
fs.write('tweets.txt',result,'w');
},
];
/**********END STEPS THAT FANTOM SHOULD DO***********************/
//Execute steps one by one
interval = setInterval(executeRequestsStepByStep,50);
function executeRequestsStepByStep(){
if (loadInProgress == false && typeof steps[testindex] == "function") {
//console.log("step " + (testindex + 1));
steps[testindex]();
testindex++;
}
if (typeof steps[testindex] != "function") {
console.log("test complete!");
phantom.exit();
}
}
/**
* These listeners are very important in order to phantom work properly. Using these listeners, we control loadInProgress marker which controls, weather a page is fully loaded.
* Without this, we will get content of the page, even a page is not fully loaded.
*/
page.onLoadStarted = function() {
loadInProgress = true;
console.log('Loading started');
};
page.onLoadFinished = function() {
loadInProgress = false;
console.log('Loading finished');
};
page.onConsoleMessage = function(msg) {
console.log(msg);
};var steps=;
var testindex = 0;
var loadInProgress = false;//This is set to true when a page is still loading
/*********SETTINGS*********************/
var webPage = require('webpage');
var page = webPage.create();
page.settings.userAgent = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36';
page.settings.javascriptEnabled = true;
page.settings.loadImages = false;//Script is much faster with this field set to false
phantom.cookiesEnabled = true;
phantom.javascriptEnabled = true;
/*********SETTINGS END*****************/
console.log('All settings loaded, start with execution');
page.onConsoleMessage = function(msg) {
console.log(msg);
};
/**********DEFINE STEPS THAT FANTOM SHOULD DO***********************/
steps = [
//Step 1 - Open Twitter login page
function(){
console.log('Step 1 - Open Twitter Login page');
page.open("https://twitter.com", function(status){
});
},
//Step 2 - Populate and submit the login form
function(){
console.log('Step 2 - Populate and submit the login form');
page.evaluate(function(){
document.getElementsByName("session[username_or_email]")[1].value="YOUR USERNAME";
document.getElementsByName("session[password]")[1].value="YOUR PASSWORD";
document.getElementsByClassName('flex-table-btn')[0].click();
});
},
//Step 3 - Wait for loading Twitter home page and extract the content
function(){
console.log("Step 3 - Wait for loading Twitter home page and extract the content");
var fs = require('fs');
var result = page.evaluate(function() {
var pageTweets = document.getElementsByClassName('js-tweet-text');
var result=new Array();
for(var i=0; i < pageTweets.length; i++){
result.push(pageTweets[i].innerHTML);
}
return JSON.stringify(result);
});
fs.write('tweets.txt',result,'w');
},
];
/**********END STEPS THAT FANTOM SHOULD DO***********************/
//Execute steps one by one
interval = setInterval(executeRequestsStepByStep,50);
function executeRequestsStepByStep(){
if (loadInProgress == false && typeof steps[testindex] == "function") {
//console.log("step " + (testindex + 1));
steps[testindex]();
testindex++;
}
if (typeof steps[testindex] != "function") {
console.log("test complete!");
phantom.exit();
}
}
/**
* These listeners are very important in order to phantom work properly. Using these listeners, we control loadInProgress marker which controls, weather a page is fully loaded.
* Without this, we will get content of the page, even a page is not fully loaded.
*/
page.onLoadStarted = function() {
loadInProgress = true;
console.log('Loading started');
};
page.onLoadFinished = function() {
loadInProgress = false;
console.log('Loading finished');
};
page.onConsoleMessage = function(msg) {
console.log(msg);
};javascript phantomjs
javascript phantomjs
edited Jan 20 at 16:57
Mihai Chelaru
2,210101122
asked Jan 20 at 12:21
zakaria elaghazakaria elagha
1
edited Jan 20 at 16:57
Mihai Chelaru
2,210101122
asked Jan 20 at 12:21
zakaria elaghazakaria elagha
1
edited Jan 20 at 16:57
Mihai Chelaru
2,210101122
edited Jan 20 at 16:57
Mihai Chelaru
2,210101122
edited Jan 20 at 16:57
Mihai Chelaru
2,210101122
2,210101122
asked Jan 20 at 12:21
zakaria elaghazakaria elagha
1
asked Jan 20 at 12:21
zakaria elaghazakaria elagha
1
asked Jan 20 at 12:21
zakaria elaghazakaria elagha
1
1
1
Instead of saying it doesn't work, explain precisely what you're trying to achieve and what error you're facing
– Nino Filiu
Jan 20 at 12:36
i want to scrap follower from user but i cant find the method to do it
– zakaria elagha
Jan 20 at 13:05
add a comment |
1
Instead of saying it doesn't work, explain precisely what you're trying to achieve and what error you're facing
– Nino Filiu
Jan 20 at 12:36
i want to scrap follower from user but i cant find the method to do it
– zakaria elagha
Jan 20 at 13:05
1
1
Instead of saying it doesn't work, explain precisely what you're trying to achieve and what error you're facing
– Nino Filiu
Jan 20 at 12:36
Instead of saying it doesn't work, explain precisely what you're trying to achieve and what error you're facing
– Nino Filiu
Jan 20 at 12:36
i want to scrap follower from user but i cant find the method to do it
– zakaria elagha
Jan 20 at 13:05
i want to scrap follower from user but i cant find the method to do it
– zakaria elagha
Jan 20 at 13:05
add a comment |
0
active
oldest
votes
Your Answer
StackExchange.ifUsing("editor", function () {
StackExchange.using("externalEditor", function () {
StackExchange.using("snippets", function () {
StackExchange.snippets.init();
});
});
}, "code-snippets");
StackExchange.ready(function() {
var channelOptions = {
tags: "".split(" "),
id: "1"
};
initTagRenderer("".split(" "), "".split(" "), channelOptions);
StackExchange.using("externalEditor", function() {
// Have to fire editor after snippets, if snippets enabled
if (StackExchange.settings.snippets.snippetsEnabled) {
StackExchange.using("snippets", function() {
createEditor();
});
}
else {
createEditor();
}
});
function createEditor() {
StackExchange.prepareEditor({
heartbeatType: 'answer',
autoActivateHeartbeat: false,
convertImagesToLinks: true,
noModals: true,
showLowRepImageUploadWarning: true,
reputationToPostImages: 10,
bindNavPrevention: true,
postfix: "",
imageUploader: {
brandingHtml: "Powered by u003ca class="icon-imgur-white" href="https://imgur.com/"u003eu003c/au003e",
contentPolicyHtml: "User contributions licensed under u003ca href="https://creativecommons.org/licenses/by-sa/3.0/"u003ecc by-sa 3.0 with attribution requiredu003c/au003e u003ca href="https://stackoverflow.com/legal/content-policy"u003e(content policy)u003c/au003e",
allowUrls: true
},
onDemand: true,
discardSelector: ".discard-answer"
,immediatelyShowMarkdownHelp:true
});
}
});
draft saved
draft discarded
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f54276377%2fhow-can-i-scrape-twitter-user-followers-using-phantomjs%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
0
active
oldest
votes
0
active
oldest
votes
active
oldest
votes
active
oldest
votes
draft saved
draft discarded
Thanks for contributing an answer to Stack Overflow!
- Please be sure to answer the question. Provide details and share your research!
But avoid …
- Asking for help, clarification, or responding to other answers.
- Making statements based on opinion; back them up with references or personal experience.
To learn more, see our tips on writing great answers.
draft saved
draft discarded
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f54276377%2fhow-can-i-scrape-twitter-user-followers-using-phantomjs%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
1
Instead of saying it doesn't work, explain precisely what you're trying to achieve and what error you're facing
– Nino Filiu
Jan 20 at 12:36
i want to scrap follower from user but i cant find the method to do it
– zakaria elagha
Jan 20 at 13:05