I am new to javascript and I am trying to capture an html file mostly generated by javascript code.
The site I want to scrape is this link:
http://www.modcloth.com/shop/dresses/luck-be-a-lady-dress-in-black-and-red
For this specific webpage, I would like to capture all the output after clicking the button named viewall
.
The code below is able to capture the current display of the webpage, but it does not seem to capture the display after clicking the viewall
button. I have been banging my head against this problem for a couple of days now, and was hoping someone might have a better idea.
// scrape_techstars.js
var webPage = require('webpage');
var page = webPage.create();
var fs = require('fs');
var path = 'modcloth.html'
page.open('http://www.modcloth.com/shop/dresses/luck-be-a-lady-dress-in-black-and-red', function() {
page.includeJs("http://ajax.googleapis.com/ajax/libs/jquery/1.6.1/jquery.min.js", function() {
page.evaluate(function() {
$("#viewall").click();
});
var content = page.content;
fs.write(path,content,'w')
phantom.exit()
});
});