190 lines
8.0 KiB
JavaScript
190 lines
8.0 KiB
JavaScript
// 只读采集：数据总览 近1天/近7天/近30天 + 捕获 mtop 接口响应
|
|
// 仅切换时间范围单选按钮（看板筛选），不点击任何操作/下载/导出按钮。
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
|
|
const OUT = path.resolve(process.argv[2] || '.cdp/out');
|
|
fs.mkdirSync(path.join(OUT, 'ranges'), { recursive: true });
|
|
fs.mkdirSync(path.join(OUT, 'api'), { recursive: true });
|
|
|
|
const USER_DATA_DIR =
|
|
process.env.CHROME_USER_DATA_DIR ||
|
|
'C:\\Users\\Administrator\\AppData\\Local\\Google\\Chrome\\User Data';
|
|
|
|
const say = (s) => console.log(String(s));
|
|
const wj = (p, o) => fs.writeFileSync(p, JSON.stringify(o, null, 2), 'utf8');
|
|
const wt = (p, s) => fs.writeFileSync(p, String(s ?? ''), 'utf8');
|
|
|
|
class Cdp {
|
|
constructor(ws) { this.ws = ws; this.seq = 0; this.pending = new Map(); }
|
|
static async attach(url) {
|
|
const ws = new WebSocket(url);
|
|
await new Promise((res, rej) => {
|
|
const t = setTimeout(() => rej(new Error('ws open timeout')), 15000);
|
|
ws.addEventListener('open', () => { clearTimeout(t); res(); }, { once: true });
|
|
ws.addEventListener('error', () => { clearTimeout(t); rej(new Error('ws error')); }, { once: true });
|
|
});
|
|
const c = new Cdp(ws);
|
|
ws.addEventListener('message', (ev) => {
|
|
let m; try { m = JSON.parse(ev.data); } catch { return; }
|
|
if (m.id && c.pending.has(m.id)) {
|
|
const p = c.pending.get(m.id); c.pending.delete(m.id); clearTimeout(p.timer);
|
|
m.error ? p.reject(new Error(m.error.message)) : p.resolve(m.result);
|
|
} else if (m.method) {
|
|
c.emit(m.method, m.params, m.sessionId);
|
|
}
|
|
});
|
|
return c;
|
|
}
|
|
emit(method, params, sessionId) { if (this.onEvent) this.onEvent(method, params, sessionId); }
|
|
send(method, params = {}, sessionId, timeoutMs = 60000) {
|
|
const id = ++this.seq;
|
|
return new Promise((resolve, reject) => {
|
|
const timer = setTimeout(() => { this.pending.delete(id); reject(new Error('timeout: ' + method)); }, timeoutMs);
|
|
this.pending.set(id, { resolve, reject, timer });
|
|
const msg = { id, method, params };
|
|
if (sessionId) msg.sessionId = sessionId;
|
|
this.ws.send(JSON.stringify(msg));
|
|
});
|
|
}
|
|
async ev(expression, sessionId) {
|
|
const r = await this.send('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true }, sessionId);
|
|
if (r.exceptionDetails) return { __error: r.exceptionDetails.text + ' :: ' + (r.exceptionDetails.exception?.description || '') };
|
|
return r.result?.value;
|
|
}
|
|
close() { try { this.ws.close(); } catch {} }
|
|
}
|
|
|
|
// ---------- endpoint ----------
|
|
const lines = fs.readFileSync(path.join(USER_DATA_DIR, 'DevToolsActivePort'), 'utf8')
|
|
.split('\n').map(s => s.trim()).filter(Boolean);
|
|
const wsUrl = `ws://127.0.0.1:${lines[0]}${lines[1]}`;
|
|
const cdp = await Cdp.attach(wsUrl);
|
|
const { targetInfos } = await cdp.send('Target.getTargets');
|
|
const target = targetInfos.find(t => t.type === 'page' && /seller\.goofish\.com/.test(t.url));
|
|
if (!target) { say('no goofish tab'); process.exit(2); }
|
|
const { sessionId: S } = await cdp.send('Target.attachToTarget', { targetId: target.targetId, flatten: true });
|
|
say('attached: ' + target.title);
|
|
|
|
// ---------- network capture ----------
|
|
const reqs = new Map();
|
|
const apiHits = [];
|
|
let lastEvent = Date.now();
|
|
cdp.onEvent = (method, p, sid) => {
|
|
if (sid !== S) return;
|
|
lastEvent = Date.now();
|
|
if (method === 'Network.requestWillBeSent') {
|
|
reqs.set(p.requestId, { url: p.request.url, method: p.request.method, postData: p.request.postData || null });
|
|
} else if (method === 'Network.responseReceived') {
|
|
const r = reqs.get(p.requestId);
|
|
if (r) { r.status = p.response.status; r.mime = p.response.mimeType; }
|
|
if (/mtop|api|\.json/i.test(p.response.url)) apiHits.push({ requestId: p.requestId, url: p.response.url, ts: Date.now() });
|
|
}
|
|
};
|
|
await cdp.send('Network.enable', {}, S);
|
|
|
|
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
|
|
async function waitQuiet(quietMs = 2500, maxMs = 25000) {
|
|
const t0 = Date.now();
|
|
while (Date.now() - t0 < maxMs) {
|
|
await sleep(250);
|
|
if (Date.now() - lastEvent > quietMs) return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
const SNAP = String.raw`(() => {
|
|
const q = (s) => Array.from(document.querySelectorAll(s));
|
|
const main = document.querySelector('.ant-layout-content') || document.body;
|
|
const groups = q('.ant-radio-group').map(g => {
|
|
const opts = Array.from(g.querySelectorAll('.ant-radio-wrapper, .ant-radio-button-wrapper'));
|
|
const checked = opts.find(o => /checked/.test(o.className) || o.querySelector('.ant-radio-checked'));
|
|
return { opts: opts.map(o => (o.innerText || '').trim()), checked: checked ? (checked.innerText || '').trim() : null };
|
|
});
|
|
return {
|
|
href: location.href,
|
|
mainText: (main.innerText || '').slice(0, 30000),
|
|
groups,
|
|
tables: q('.ant-table').length,
|
|
tableHeaders: q('.ant-table-thead th').map(th => th.innerText.replace(/\s+/g, ' ').trim()),
|
|
renderedRows: q('.ant-table-tbody tr').length,
|
|
navLinks: q('a[href]').map(a => ({ t: (a.innerText || '').trim(), h: a.getAttribute('href') })).filter(x => x.t).slice(0, 60),
|
|
menuText: q('[class*="menu"] li, [class*="Menu"] li').map(li => (li.innerText || '').replace(/\s+/g, ' ').trim()).filter(Boolean).slice(0, 60),
|
|
};
|
|
})()`;
|
|
|
|
async function snapshot(tag) {
|
|
const s = await cdp.ev(SNAP, S);
|
|
if (!s || s.__error) { say('snapshot error ' + tag + ': ' + JSON.stringify(s)); return null; }
|
|
wt(path.join(OUT, 'ranges', tag + '-maintext.txt'), s.mainText);
|
|
wj(path.join(OUT, 'ranges', tag + '-state.json'), { href: s.href, groups: s.groups, tables: s.tables, tableHeaders: s.tableHeaders, renderedRows: s.renderedRows, navLinks: s.navLinks, menuText: s.menuText });
|
|
say('[' + tag + '] groups=' + JSON.stringify(s.groups.map(g => g.checked)) +
|
|
' tables=' + s.tables + ' rows=' + s.renderedRows +
|
|
' headers=' + JSON.stringify(s.tableHeaders));
|
|
return s;
|
|
}
|
|
|
|
async function clickRange(label) {
|
|
const expr = `(() => {
|
|
const label = ${JSON.stringify(label)};
|
|
const out = { clicked: 0, skipped: 0 };
|
|
for (const g of document.querySelectorAll('.ant-radio-group')) {
|
|
const opts = Array.from(g.querySelectorAll('.ant-radio-wrapper, .ant-radio-button-wrapper'));
|
|
const t = opts.find(o => (o.innerText || '').trim() === label);
|
|
if (!t) continue;
|
|
if (/checked/.test(t.className) || t.querySelector('.ant-radio-checked')) { out.skipped++; continue; }
|
|
t.click(); out.clicked++;
|
|
}
|
|
return out;
|
|
})()`;
|
|
return await cdp.ev(expr, S);
|
|
}
|
|
|
|
async function dumpApi(tag) {
|
|
const seen = new Set();
|
|
let n = 0;
|
|
for (const hit of apiHits) {
|
|
if (seen.has(hit.requestId)) continue;
|
|
seen.add(hit.requestId);
|
|
try {
|
|
const b = await cdp.send('Network.getResponseBody', { requestId: hit.requestId }, S, 20000);
|
|
let body = b.base64Encoded ? Buffer.from(b.body, 'base64').toString('utf8') : b.body;
|
|
if (body.length > 400000) body = body.slice(0, 400000) + '\n...[truncated]';
|
|
const name = `${tag}-${String(n).padStart(2, '0')}.txt`;
|
|
wt(path.join(OUT, 'api', name), 'URL: ' + hit.url + '\n\n' + body);
|
|
n++;
|
|
} catch (e) { /* body unavailable */ }
|
|
}
|
|
say('[' + tag + '] api bodies saved = ' + n + ' / hits=' + apiHits.length);
|
|
return n;
|
|
}
|
|
|
|
// ---------- run ----------
|
|
say('--- initial state ---');
|
|
const s0 = await snapshot('1d-before');
|
|
await dumpApi('1d-before');
|
|
|
|
say('--- switch to 近7天 ---');
|
|
say('click result: ' + JSON.stringify(await clickRange('近7天')));
|
|
await waitQuiet();
|
|
await snapshot('7d');
|
|
await dumpApi('7d');
|
|
|
|
say('--- switch to 近30天 ---');
|
|
say('click result: ' + JSON.stringify(await clickRange('近30天')));
|
|
await waitQuiet();
|
|
await snapshot('30d');
|
|
await dumpApi('30d');
|
|
|
|
say('--- restore 近1天 ---');
|
|
say('click result: ' + JSON.stringify(await clickRange('近1天')));
|
|
await waitQuiet();
|
|
await snapshot('1d-after');
|
|
|
|
// 请求清单
|
|
wj(path.join(OUT, 'requests.json'), Array.from(reqs.entries()).map(([id, r]) => ({ id, ...r }))
|
|
.filter(r => /mtop|api|summary|item/i.test(r.url)));
|
|
say('done.');
|
|
cdp.close();
|
|
process.exit(0);
|