nodejs downloads web pages through phantomjs
- 2020-06-03 05:51:01
- OfStack
In fact, the function is very simple, phantomjs.exe collects the resources loaded by url, and starts nodejs to load all the resources through the subprocess. For the resources of css, match the contents of css and download the url resources
Of course, the function is very simple, in the case of responsive design and asynchronous loading, there are still many resources can not be downloaded, need to be handled according to the actual situation
The first, of course, is to download nodejs and phantomjs
The following is down. js executed by ES16en. exe
var page = require('webpage').create(),
system = require('system');
var spawn = require("child_process").spawn
if (system.args.length === 1) {
console.log('Usage: netsniff.js <some URL>');
phantom.exit(1);
} else {
var urls = [];
page.address = system.args[1];
page.onResourceReceived = function (res) {
if (res.stage === 'start') {
urls.push(res.url);
}
};
page.open(page.address, function (status) {
var har;
if (status !== 'success') {
console.log('FAIL to load the address');
phantom.exit(1);
} else {
console.log('down resource ' + urls.length + ' urls.');
var child = spawn("node", ["--harmony", "downHtml.js", urls.join(',')])
child.stdout.on("data", function (data) {
console.log(data);
})
child.stderr.on("data", function (data) {
console.log(data);
})
child.on("exit", function (code) {
phantom.exit();
})
}
});
}
Below is downHtml.js run for the corresponding node
"use strict";
var fs = require('fs');
var http = require('http');
var path = require('path');
var r_url = require('url');
var dirCache = {};// Cache reduction judgment
function makedir (pathStr, callback) {
if (dirCache[pathStr] == 1) {
callback();
} else {
fs.exists(pathStr, function (exists) {
if (exists == true) {
dirCache[pathStr] == 1;
callback();
} else {
makedir(path.dirname(pathStr), function () {
fs.mkdir(pathStr, function () {
dirCache[pathStr] == 1;
callback();
})
});
}
})
}
};
var reg = /[:,]\s*url\(['"]?.*?(\1)\)/g
var reg2 = /\((['"]?)(.*?)(\1)\)/
var isDownMap = {};
var downImgFromCss = function (URL) {
http.get(URL, function(res) {
//console.log(path.resolve(process.cwd(), 'index.min.css'))
//res.pipe(fs.createWriteStream(path.resolve(process.cwd(), 'index.min.css')));
var body = "";
res.setEncoding('utf8');
res.on('data', function (chunk) {
body += chunk;
});
res.on('end', function () {
var match = body.match(reg);
for (var i = 0, len = match.length; i < len; i++){
var m = match[i].match(reg2);
if (m && m[2]) {
var url = m[2];
let imgUrl = r_url.resolve(URL, url);
if (!isDownMap[imgUrl]) {
var uo = r_url.parse(imgUrl);
let filepath = CWD + '/' + uo.hostname + uo.pathname;
makedir(path.dirname(filepath), function () {
http.get(imgUrl, function (res) {
res.pipe(fs.createWriteStream(filepath));
})
})
isDownMap[imgUrl] = 1;
}
}
}
});
});
}
var URLS = process.argv[2].split(',');
var CWD = process.cwd();
// Download resources
URLS.forEach(function (URL) {
var uo = r_url.parse(URL);
var filepath;
if (uo.pathname == '/' || uo.pathname == '') {
filepath = CWD + '/' + uo.hostname + '/index.html';
} else {
filepath = CWD + '/' + uo.hostname + uo.pathname;
}
makedir(path.dirname(filepath), function () {
http.get(URL, function (res) {
if (URL.indexOf('.css') != -1 || (res.headers["content-type"] && res.headers["content-type"].indexOf('text/css')!= -1)) {
console.log('down images form css file:' + URL + '.');
downImgFromCss(URL);
}
res.pipe(fs.createWriteStream(filepath));
})
});
});
down. js downHtml. js run in the same folder as cmd below
D:\phantomjs-2.0.0-windows\bin\phantomjs.exe down.js http://www.youku.com/
This is the end of this article, I hope you enjoy it.