124 lines
5.8 KiB
JavaScript
124 lines
5.8 KiB
JavaScript
// 只读：后台开一个隐藏标签页抓取指定页面的 mtop 响应与 DOM，抓完关闭
|
|
// 用法: node cdp-cap.mjs <url> <outTag>
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
|
|
const URL_ = process.argv[2];
|
|
const TAG = process.argv[3] || 'cap';
|
|
const OUT = path.resolve('.cdp/out');
|
|
fs.mkdirSync(path.join(OUT, 'cap'), { recursive: true });
|
|
|
|
const UDD = 'C:\\Users\\Administrator\\AppData\\Local\\Google\\Chrome\\User Data';
|
|
const lines = fs.readFileSync(path.join(UDD, 'DevToolsActivePort'), 'utf8')
|
|
.split('\n').map(s => s.trim()).filter(Boolean);
|
|
const say = (s) => console.log(String(s));
|
|
|
|
class Cdp {
|
|
constructor(ws) { this.ws = ws; this.seq = 0; this.pending = new Map(); }
|
|
static async attach(url) {
|
|
const ws = new WebSocket(url);
|
|
await new Promise((res, rej) => {
|
|
const t = setTimeout(() => rej(new Error('ws timeout')), 15000);
|
|
ws.addEventListener('open', () => { clearTimeout(t); res(); }, { once: true });
|
|
ws.addEventListener('error', () => { clearTimeout(t); rej(new Error('ws error')); }, { once: true });
|
|
});
|
|
const c = new Cdp(ws);
|
|
ws.addEventListener('message', (ev) => {
|
|
let m; try { m = JSON.parse(ev.data); } catch { return; }
|
|
if (m.id && c.pending.has(m.id)) {
|
|
const p = c.pending.get(m.id); c.pending.delete(m.id); clearTimeout(p.timer);
|
|
m.error ? p.reject(new Error(m.error.message)) : p.resolve(m.result);
|
|
} else if (m.method) c.onEvent?.(m.method, m.params, m.sessionId);
|
|
});
|
|
return c;
|
|
}
|
|
send(method, params = {}, sessionId, t = 60000) {
|
|
const id = ++this.seq;
|
|
return new Promise((res, rej) => {
|
|
const timer = setTimeout(() => { this.pending.delete(id); rej(new Error('timeout ' + method)); }, t);
|
|
this.pending.set(id, { resolve: res, reject: rej, timer });
|
|
const msg = { id, method, params }; if (sessionId) msg.sessionId = sessionId;
|
|
this.ws.send(JSON.stringify(msg));
|
|
});
|
|
}
|
|
async ev(expr, sid) {
|
|
const r = await this.send('Runtime.evaluate', { expression: expr, returnByValue: true, awaitPromise: true }, sid);
|
|
if (r.exceptionDetails) return { __error: r.exceptionDetails.text };
|
|
return r.result?.value;
|
|
}
|
|
close() { try { this.ws.close(); } catch {} }
|
|
}
|
|
|
|
const cdp = await Cdp.attach(`ws://127.0.0.1:${lines[0]}${lines[1]}`);
|
|
const { targetId } = await cdp.send('Target.createTarget', { url: URL_, background: true });
|
|
say('created target ' + targetId + ' -> ' + URL_);
|
|
const { sessionId: S } = await cdp.send('Target.attachToTarget', { targetId, flatten: true });
|
|
|
|
const hits = []; const reqs = new Map(); let last = Date.now();
|
|
cdp.onEvent = (m, p, sid) => {
|
|
if (sid !== S) return;
|
|
last = Date.now();
|
|
if (m === 'Network.requestWillBeSent') {
|
|
reqs.set(p.requestId, { url: p.request.url, postData: p.request.postData || null });
|
|
} else if (m === 'Network.responseReceived') {
|
|
if (/mtop\./i.test(p.response.url)) hits.push({ requestId: p.requestId, url: p.response.url });
|
|
}
|
|
};
|
|
await cdp.send('Network.enable', {}, S);
|
|
await cdp.send('Page.enable', {}, S);
|
|
|
|
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
|
|
const quiet = async (q = 3000, max = 40000) => {
|
|
const t0 = Date.now();
|
|
while (Date.now() - t0 < max) { await sleep(300); if (Date.now() - last > q) return; }
|
|
};
|
|
await quiet(4000, 30000);
|
|
say('after load: mtop hits = ' + hits.length);
|
|
|
|
// 探测页面结构
|
|
const probe = await cdp.ev(`(() => {
|
|
const q = s => Array.from(document.querySelectorAll(s));
|
|
const main = document.querySelector('.ant-layout-content') || document.body;
|
|
return {
|
|
href: location.href, title: document.title,
|
|
radioGroups: q('.ant-radio-group').map(g => {
|
|
const o = Array.from(g.querySelectorAll('.ant-radio-wrapper, .ant-radio-button-wrapper'));
|
|
const c = o.find(x => /checked/.test(x.className) || x.querySelector('.ant-radio-checked'));
|
|
return { opts: o.map(x => (x.innerText||'').trim()), checked: c ? (c.innerText||'').trim() : null };
|
|
}),
|
|
headers: q('.ant-table-thead th').map(t => t.innerText.replace(/\\s+/g,' ').trim()).filter(Boolean),
|
|
headerRows: q('.ant-table-thead tr').map(tr => Array.from(tr.querySelectorAll('th')).map(t => t.innerText.replace(/\\s+/g,' ').trim())),
|
|
rowCount: q('.ant-table-tbody tr').length,
|
|
rows: q('.ant-table-tbody tr').slice(0, 25).map(tr => Array.from(tr.querySelectorAll('td')).map(td => (td.innerText||'').replace(/\\s+/g,' ').trim())),
|
|
pagination: q('.ant-pagination-item').map(e => e.innerText.trim()),
|
|
total: (document.querySelector('.ant-pagination-total-text')||{}).innerText || null,
|
|
buttons: q('button').map(b => b.innerText.trim()).filter(Boolean).slice(0,30),
|
|
mainTextHead: (main.innerText||'').slice(0, 2500),
|
|
};
|
|
})()`, S);
|
|
fs.writeFileSync(path.join(OUT, 'cap', TAG + '-probe.json'), JSON.stringify(probe, null, 2), 'utf8');
|
|
say('headers = ' + JSON.stringify(probe.headers));
|
|
say('radioGroups = ' + JSON.stringify(probe.radioGroups.map(g => g.checked)));
|
|
say('rows = ' + probe.rowCount + ' total=' + probe.total + ' pagination=' + JSON.stringify(probe.pagination));
|
|
|
|
// 存接口响应
|
|
let n = 0;
|
|
for (const h of hits) {
|
|
try {
|
|
const b = await cdp.send('Network.getResponseBody', { requestId: h.requestId }, S, 20000);
|
|
let body = b.base64Encoded ? Buffer.from(b.body, 'base64').toString('utf8') : b.body;
|
|
const req = [...reqs.entries()].find(([id]) => id === h.requestId)?.[1];
|
|
const api = (h.url.match(/mtop\.[A-Za-z0-9._]+/) || [''])[0];
|
|
fs.writeFileSync(path.join(OUT, 'cap', `${TAG}-${String(n).padStart(2,'0')}.txt`),
|
|
'URL: ' + h.url + '\nPOST: ' + (req?.postData || '-') + '\n\n' + body.slice(0, 600000), 'utf8');
|
|
say(` [${n}] ${api.replace('mtop.alibaba.idle.seller.pc.datacompass.','')} ${body.length}B`);
|
|
n++;
|
|
} catch { }
|
|
}
|
|
say('saved ' + n + ' api bodies');
|
|
|
|
await cdp.send('Target.closeTarget', { targetId });
|
|
say('closed tab');
|
|
cdp.close();
|
|
process.exit(0);
|