// 只读采集:数据总览 近1天/近7天/近30天 + 捕获 mtop 接口响应 // 仅切换时间范围单选按钮(看板筛选),不点击任何操作/下载/导出按钮。 import fs from 'node:fs'; import path from 'node:path'; const OUT = path.resolve(process.argv[2] || '.cdp/out'); fs.mkdirSync(path.join(OUT, 'ranges'), { recursive: true }); fs.mkdirSync(path.join(OUT, 'api'), { recursive: true }); const USER_DATA_DIR = process.env.CHROME_USER_DATA_DIR || 'C:\\Users\\Administrator\\AppData\\Local\\Google\\Chrome\\User Data'; const say = (s) => console.log(String(s)); const wj = (p, o) => fs.writeFileSync(p, JSON.stringify(o, null, 2), 'utf8'); const wt = (p, s) => fs.writeFileSync(p, String(s ?? ''), 'utf8'); class Cdp { constructor(ws) { this.ws = ws; this.seq = 0; this.pending = new Map(); } static async attach(url) { const ws = new WebSocket(url); await new Promise((res, rej) => { const t = setTimeout(() => rej(new Error('ws open timeout')), 15000); ws.addEventListener('open', () => { clearTimeout(t); res(); }, { once: true }); ws.addEventListener('error', () => { clearTimeout(t); rej(new Error('ws error')); }, { once: true }); }); const c = new Cdp(ws); ws.addEventListener('message', (ev) => { let m; try { m = JSON.parse(ev.data); } catch { return; } if (m.id && c.pending.has(m.id)) { const p = c.pending.get(m.id); c.pending.delete(m.id); clearTimeout(p.timer); m.error ? p.reject(new Error(m.error.message)) : p.resolve(m.result); } else if (m.method) { c.emit(m.method, m.params, m.sessionId); } }); return c; } emit(method, params, sessionId) { if (this.onEvent) this.onEvent(method, params, sessionId); } send(method, params = {}, sessionId, timeoutMs = 60000) { const id = ++this.seq; return new Promise((resolve, reject) => { const timer = setTimeout(() => { this.pending.delete(id); reject(new Error('timeout: ' + method)); }, timeoutMs); this.pending.set(id, { resolve, reject, timer }); const msg = { id, method, params }; if (sessionId) msg.sessionId = sessionId; this.ws.send(JSON.stringify(msg)); }); } async ev(expression, sessionId) { const r = await this.send('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true }, sessionId); if (r.exceptionDetails) return { __error: r.exceptionDetails.text + ' :: ' + (r.exceptionDetails.exception?.description || '') }; return r.result?.value; } close() { try { this.ws.close(); } catch {} } } // ---------- endpoint ---------- const lines = fs.readFileSync(path.join(USER_DATA_DIR, 'DevToolsActivePort'), 'utf8') .split('\n').map(s => s.trim()).filter(Boolean); const wsUrl = `ws://127.0.0.1:${lines[0]}${lines[1]}`; const cdp = await Cdp.attach(wsUrl); const { targetInfos } = await cdp.send('Target.getTargets'); const target = targetInfos.find(t => t.type === 'page' && /seller\.goofish\.com/.test(t.url)); if (!target) { say('no goofish tab'); process.exit(2); } const { sessionId: S } = await cdp.send('Target.attachToTarget', { targetId: target.targetId, flatten: true }); say('attached: ' + target.title); // ---------- network capture ---------- const reqs = new Map(); const apiHits = []; let lastEvent = Date.now(); cdp.onEvent = (method, p, sid) => { if (sid !== S) return; lastEvent = Date.now(); if (method === 'Network.requestWillBeSent') { reqs.set(p.requestId, { url: p.request.url, method: p.request.method, postData: p.request.postData || null }); } else if (method === 'Network.responseReceived') { const r = reqs.get(p.requestId); if (r) { r.status = p.response.status; r.mime = p.response.mimeType; } if (/mtop|api|\.json/i.test(p.response.url)) apiHits.push({ requestId: p.requestId, url: p.response.url, ts: Date.now() }); } }; await cdp.send('Network.enable', {}, S); const sleep = (ms) => new Promise(r => setTimeout(r, ms)); async function waitQuiet(quietMs = 2500, maxMs = 25000) { const t0 = Date.now(); while (Date.now() - t0 < maxMs) { await sleep(250); if (Date.now() - lastEvent > quietMs) return true; } return false; } const SNAP = String.raw`(() => { const q = (s) => Array.from(document.querySelectorAll(s)); const main = document.querySelector('.ant-layout-content') || document.body; const groups = q('.ant-radio-group').map(g => { const opts = Array.from(g.querySelectorAll('.ant-radio-wrapper, .ant-radio-button-wrapper')); const checked = opts.find(o => /checked/.test(o.className) || o.querySelector('.ant-radio-checked')); return { opts: opts.map(o => (o.innerText || '').trim()), checked: checked ? (checked.innerText || '').trim() : null }; }); return { href: location.href, mainText: (main.innerText || '').slice(0, 30000), groups, tables: q('.ant-table').length, tableHeaders: q('.ant-table-thead th').map(th => th.innerText.replace(/\s+/g, ' ').trim()), renderedRows: q('.ant-table-tbody tr').length, navLinks: q('a[href]').map(a => ({ t: (a.innerText || '').trim(), h: a.getAttribute('href') })).filter(x => x.t).slice(0, 60), menuText: q('[class*="menu"] li, [class*="Menu"] li').map(li => (li.innerText || '').replace(/\s+/g, ' ').trim()).filter(Boolean).slice(0, 60), }; })()`; async function snapshot(tag) { const s = await cdp.ev(SNAP, S); if (!s || s.__error) { say('snapshot error ' + tag + ': ' + JSON.stringify(s)); return null; } wt(path.join(OUT, 'ranges', tag + '-maintext.txt'), s.mainText); wj(path.join(OUT, 'ranges', tag + '-state.json'), { href: s.href, groups: s.groups, tables: s.tables, tableHeaders: s.tableHeaders, renderedRows: s.renderedRows, navLinks: s.navLinks, menuText: s.menuText }); say('[' + tag + '] groups=' + JSON.stringify(s.groups.map(g => g.checked)) + ' tables=' + s.tables + ' rows=' + s.renderedRows + ' headers=' + JSON.stringify(s.tableHeaders)); return s; } async function clickRange(label) { const expr = `(() => { const label = ${JSON.stringify(label)}; const out = { clicked: 0, skipped: 0 }; for (const g of document.querySelectorAll('.ant-radio-group')) { const opts = Array.from(g.querySelectorAll('.ant-radio-wrapper, .ant-radio-button-wrapper')); const t = opts.find(o => (o.innerText || '').trim() === label); if (!t) continue; if (/checked/.test(t.className) || t.querySelector('.ant-radio-checked')) { out.skipped++; continue; } t.click(); out.clicked++; } return out; })()`; return await cdp.ev(expr, S); } async function dumpApi(tag) { const seen = new Set(); let n = 0; for (const hit of apiHits) { if (seen.has(hit.requestId)) continue; seen.add(hit.requestId); try { const b = await cdp.send('Network.getResponseBody', { requestId: hit.requestId }, S, 20000); let body = b.base64Encoded ? Buffer.from(b.body, 'base64').toString('utf8') : b.body; if (body.length > 400000) body = body.slice(0, 400000) + '\n...[truncated]'; const name = `${tag}-${String(n).padStart(2, '0')}.txt`; wt(path.join(OUT, 'api', name), 'URL: ' + hit.url + '\n\n' + body); n++; } catch (e) { /* body unavailable */ } } say('[' + tag + '] api bodies saved = ' + n + ' / hits=' + apiHits.length); return n; } // ---------- run ---------- say('--- initial state ---'); const s0 = await snapshot('1d-before'); await dumpApi('1d-before'); say('--- switch to 近7天 ---'); say('click result: ' + JSON.stringify(await clickRange('近7天'))); await waitQuiet(); await snapshot('7d'); await dumpApi('7d'); say('--- switch to 近30天 ---'); say('click result: ' + JSON.stringify(await clickRange('近30天'))); await waitQuiet(); await snapshot('30d'); await dumpApi('30d'); say('--- restore 近1天 ---'); say('click result: ' + JSON.stringify(await clickRange('近1天'))); await waitQuiet(); await snapshot('1d-after'); // 请求清单 wj(path.join(OUT, 'requests.json'), Array.from(reqs.entries()).map(([id, r]) => ({ id, ...r })) .filter(r => /mtop|api|summary|item/i.test(r.url))); say('done.'); cdp.close(); process.exit(0);