43 lines
2.1 KiB
JavaScript
43 lines
2.1 KiB
JavaScript
// 抓数据：装持久钩子(Page.addScriptToEvaluateOnNewDocument)，读取整体数据接口
|
|
import { connect, evaluate, saveJson } from './lib.mjs';
|
|
|
|
const cdp = await connect();
|
|
const { targetInfos } = await cdp.send('Target.getTargets');
|
|
const page = targetInfos.find(t => t.type === 'page' && t.url.includes('baijiahao.baidu.com'));
|
|
if (!page) { console.log('NO_PAGE'); process.exit(1); }
|
|
|
|
const { sessionId } = await cdp.send('Target.attachToTarget', { targetId: page.targetId, flatten: true });
|
|
await cdp.send('Page.enable', {}, sessionId);
|
|
await cdp.send('Runtime.enable', {}, sessionId);
|
|
|
|
// 关键：在每次新文档创建前注入钩子，导航后依然有效
|
|
const HOOK = `(() => {
|
|
window.__CAP = [];
|
|
const push = (u, t) => { try { if (t && (t[0]==='{'||t[0]==='[')) window.__CAP.push({url:u, body:t.slice(0,400000)}); } catch(e){} };
|
|
const oOpen = XMLHttpRequest.prototype.open;
|
|
const oSend = XMLHttpRequest.prototype.send;
|
|
XMLHttpRequest.prototype.open = function(m,u){ this.__u=u; return oOpen.apply(this,arguments); };
|
|
XMLHttpRequest.prototype.send = function(){
|
|
this.addEventListener('load', () => {
|
|
try { const ct=this.getResponseHeader('content-type')||''; if(ct.includes('json')) push(this.__u, this.responseText); } catch(e){}
|
|
});
|
|
return oSend.apply(this,arguments);
|
|
};
|
|
const oFetch = window.fetch;
|
|
window.fetch = function(...a){
|
|
return oFetch.apply(this,a).then(r=>{ try{ const u=typeof a[0]==='string'?a[0]:a[0].url; r.clone().text().then(t=>push(u,t)).catch(()=>{}); }catch(e){} return r; });
|
|
};
|
|
})();`;
|
|
|
|
await cdp.send('Page.addScriptToEvaluateOnNewDocument', { source: HOOK }, sessionId);
|
|
|
|
// 导航到内容分析
|
|
await cdp.send('Page.navigate', { url: 'https://baijiahao.baidu.com/builder/rc/analysiscontent' }, sessionId);
|
|
await new Promise(r => setTimeout(r, 15000));
|
|
|
|
const cap = await evaluate(cdp, sessionId, `(window.__CAP||[])`);
|
|
saveJson('out/analysis-capture.json', cap);
|
|
console.log('CAP_COUNT=' + (cap||[]).length);
|
|
if (cap) for (const c of cap) console.log('API ' + String(c.url).slice(0,150));
|
|
process.exit(0);
|