64 lines
2.4 KiB
JavaScript
64 lines
2.4 KiB
JavaScript
// 抓取内容分析页：作品列表 + 单篇数据
|
|
import { connect, attachPage, evaluate, saveJson, listPages } from './lib.mjs';
|
|
|
|
const cdp = await connect();
|
|
|
|
// 直接在百家号标签页内导航到内容分析
|
|
const hit = await attachPage(cdp, 'baijiahao.baidu.com');
|
|
if (!hit) { console.log('NO_TAB'); process.exit(1); }
|
|
const { sessionId } = hit;
|
|
|
|
// 先看内容分析页有哪些数据接口，装 XHR 钩子
|
|
await cdp.send('Page.enable', {}, sessionId).catch(() => {});
|
|
await cdp.send('Runtime.enable', {}, sessionId).catch(() => {});
|
|
|
|
await evaluate(cdp, sessionId, `(() => {
|
|
window.__CAP = window.__CAP || [];
|
|
if (!window.__HOOKED) {
|
|
window.__HOOKED = true;
|
|
const origOpen = XMLHttpRequest.prototype.open;
|
|
const origSend = XMLHttpRequest.prototype.send;
|
|
XMLHttpRequest.prototype.open = function(m, u) { this.__u = u; this.__m = m; return origOpen.apply(this, arguments); };
|
|
XMLHttpRequest.prototype.send = function() {
|
|
this.addEventListener('load', () => {
|
|
try {
|
|
const ct = this.getResponseHeader('content-type') || '';
|
|
if (ct.includes('json')) {
|
|
window.__CAP.push({ url: this.__u, body: this.responseText.slice(0, 200000) });
|
|
}
|
|
} catch (e) {}
|
|
});
|
|
return origSend.apply(this, arguments);
|
|
};
|
|
const origFetch = window.fetch;
|
|
window.fetch = function(...args) {
|
|
return origFetch.apply(this, args).then(res => {
|
|
try {
|
|
const u = (typeof args[0] === 'string') ? args[0] : args[0].url;
|
|
res.clone().text().then(t => {
|
|
if (t && (t[0] === '{' || t[0] === '[')) window.__CAP.push({ url: u, body: t.slice(0, 200000) });
|
|
}).catch(() => {});
|
|
} catch (e) {}
|
|
return res;
|
|
});
|
|
};
|
|
}
|
|
return 'hooked';
|
|
})()`);
|
|
|
|
// 导航到内容分析页
|
|
await evaluate(cdp, sessionId, `location.href = 'https://baijiahao.baidu.com/builder/rc/analysiscontent'`);
|
|
await new Promise(r => setTimeout(r, 9000));
|
|
|
|
const state = await evaluate(cdp, sessionId, `(() => ({
|
|
url: location.href,
|
|
title: document.title,
|
|
text: (document.body ? document.body.innerText : '').slice(0, 6000),
|
|
captured: (window.__CAP || []).map(x => x.url)
|
|
}))()`);
|
|
|
|
saveJson('out/analysis-content.json', state);
|
|
console.log('URL=' + String(state.url).replace(/[^\x20-\x7E]/g, '?'));
|
|
console.log('CAPTURED=' + (state.captured || []).length);
|
|
process.exit(0);
|