67 lines
2.6 KiB
JavaScript
67 lines
2.6 KiB
JavaScript
// 打开百家号内容分析页，挂 XHR 钩子，抓取作品列表数据
|
|
import { connect, evaluate, saveJson } from './lib.mjs';
|
|
|
|
const cdp = await connect();
|
|
console.log('port=' + cdp.port);
|
|
|
|
// 复用已有标签或新建
|
|
const { targetInfos } = await cdp.send('Target.getTargets');
|
|
let page = targetInfos.find(t => t.type === 'page' && t.url.includes('baijiahao.baidu.com'));
|
|
if (!page) {
|
|
const { targetId } = await cdp.send('Target.createTarget', {
|
|
url: 'https://baijiahao.baidu.com/builder/rc/analysiscontent',
|
|
background: false,
|
|
});
|
|
await new Promise(r => setTimeout(r, 1000));
|
|
const t2 = await cdp.send('Target.getTargets');
|
|
page = t2.targetInfos.find(t => t.targetId === targetId);
|
|
}
|
|
if (!page) { console.log('NO_PAGE'); process.exit(1); }
|
|
|
|
const { sessionId } = await cdp.send('Target.attachToTarget', { targetId: page.targetId, flatten: true });
|
|
await cdp.send('Page.enable', {}, sessionId).catch(() => {});
|
|
await cdp.send('Runtime.enable', {}, sessionId).catch(() => {});
|
|
|
|
// 装钩子（在页面加载前装不了，页面内导航可以）
|
|
await evaluate(cdp, sessionId, `(() => {
|
|
window.__CAP = [];
|
|
if (!window.__HOOKED) {
|
|
window.__HOOKED = true;
|
|
const oOpen = XMLHttpRequest.prototype.open;
|
|
const oSend = XMLHttpRequest.prototype.send;
|
|
XMLHttpRequest.prototype.open = function(m,u){ this.__u=u; return oOpen.apply(this,arguments); };
|
|
XMLHttpRequest.prototype.send = function(){
|
|
this.addEventListener('load', () => {
|
|
try {
|
|
const ct = this.getResponseHeader('content-type') || '';
|
|
if (ct.includes('json') && this.responseText) {
|
|
window.__CAP.push({ url: this.__u, body: this.responseText.slice(0,300000) });
|
|
}
|
|
} catch(e){}
|
|
});
|
|
return oSend.apply(this,arguments);
|
|
};
|
|
}
|
|
return 'hooked';
|
|
})()`).catch(e => console.log('hook skipped: ' + e.message));
|
|
|
|
await new Promise(r => setTimeout(r, 4000));
|
|
|
|
// 页面内导航，触发数据加载
|
|
await evaluate(cdp, sessionId, `location.href='https://baijiahao.baidu.com/builder/rc/analysiscontent'`).catch(()=>{});
|
|
await new Promise(r => setTimeout(r, 12000));
|
|
|
|
const state = await evaluate(cdp, sessionId, `(() => ({
|
|
url: location.href,
|
|
title: document.title,
|
|
text: (document.body?document.body.innerText:'').slice(0,8000),
|
|
capUrls: (window.__CAP||[]).map(x=>x.url),
|
|
capBodies: (window.__CAP||[]).map(x=>x.body)
|
|
}))()`);
|
|
|
|
saveJson('out/analysis-raw.json', state);
|
|
console.log('URL=' + String(state.url).replace(/[^\x20-\x7E]/g,'?'));
|
|
console.log('CAP=' + (state.capUrls||[]).length);
|
|
console.log('TEXT_LEN=' + String(state.text||'').length);
|
|
process.exit(0);
|