javascript - Use PhantomJS to extract html and text

Question

I try to extract all the text content of a page (because it doesn't work with Simpledomparser)

I try to modify this simple example from the manual

var page = require('webpage').create();
console.log('The default user agent is ' + page.settings.userAgent);
page.settings.userAgent = 'SpecialAgent';
page.open('http://www.httpuseragent.org', function (status) {
    if (status !== 'success') {
        console.log('Unable to access network');
    } else {
        var ua = page.evaluate(function () {
            return document.getElementById('myagent').textContent;
        });
        console.log(ua);
    }
    phantom.exit();
});

I try to change

return document.getElementById('myagent').textContent;

to

return document.textContent;

This doesn't work.

What's the right way to do this simple thing?

Answer 1

This version of your script should return the entire contents of the page:

var page = require('webpage').create();
page.settings.userAgent = 'SpecialAgent';
page.open('http://www.httpuseragent.org', function (status) {
    if (status !== 'success') {
        console.log('Unable to access network');
    } else {
        var ua = page.evaluate(function () {
            return document.getElementsByTagName('html')[0].outerHTML;
        });
        console.log(ua);
    }
    phantom.exit();
});