first commit

小鱼/.cdp/cdp-cap.mjs

@@ -0,0 +1,123 @@
+// 只读：后台开一个隐藏标签页抓取指定页面的 mtop 响应与 DOM，抓完关闭
+// 用法: node cdp-cap.mjs <url> <outTag>
+import fs from 'node:fs';
+import path from 'node:path';
+
+const URL_ = process.argv[2];
+const TAG = process.argv[3] || 'cap';
+const OUT = path.resolve('.cdp/out');
+fs.mkdirSync(path.join(OUT, 'cap'), { recursive: true });
+
+const UDD = 'C:\\Users\\Administrator\\AppData\\Local\\Google\\Chrome\\User Data';
+const lines = fs.readFileSync(path.join(UDD, 'DevToolsActivePort'), 'utf8')
+  .split('\n').map(s => s.trim()).filter(Boolean);
+const say = (s) => console.log(String(s));
+
+class Cdp {
+  constructor(ws) { this.ws = ws; this.seq = 0; this.pending = new Map(); }
+  static async attach(url) {
+    const ws = new WebSocket(url);
+    await new Promise((res, rej) => {
+      const t = setTimeout(() => rej(new Error('ws timeout')), 15000);
+      ws.addEventListener('open', () => { clearTimeout(t); res(); }, { once: true });
+      ws.addEventListener('error', () => { clearTimeout(t); rej(new Error('ws error')); }, { once: true });
+    });
+    const c = new Cdp(ws);
+    ws.addEventListener('message', (ev) => {
+      let m; try { m = JSON.parse(ev.data); } catch { return; }
+      if (m.id && c.pending.has(m.id)) {
+        const p = c.pending.get(m.id); c.pending.delete(m.id); clearTimeout(p.timer);
+        m.error ? p.reject(new Error(m.error.message)) : p.resolve(m.result);
+      } else if (m.method) c.onEvent?.(m.method, m.params, m.sessionId);
+    });
+    return c;
+  }
+  send(method, params = {}, sessionId, t = 60000) {
+    const id = ++this.seq;
+    return new Promise((res, rej) => {
+      const timer = setTimeout(() => { this.pending.delete(id); rej(new Error('timeout ' + method)); }, t);
+      this.pending.set(id, { resolve: res, reject: rej, timer });
+      const msg = { id, method, params }; if (sessionId) msg.sessionId = sessionId;
+      this.ws.send(JSON.stringify(msg));
+    });
+  }
+  async ev(expr, sid) {
+    const r = await this.send('Runtime.evaluate', { expression: expr, returnByValue: true, awaitPromise: true }, sid);
+    if (r.exceptionDetails) return { __error: r.exceptionDetails.text };
+    return r.result?.value;
+  }
+  close() { try { this.ws.close(); } catch {} }
+}
+
+const cdp = await Cdp.attach(`ws://127.0.0.1:${lines[0]}${lines[1]}`);
+const { targetId } = await cdp.send('Target.createTarget', { url: URL_, background: true });
+say('created target ' + targetId + ' -> ' + URL_);
+const { sessionId: S } = await cdp.send('Target.attachToTarget', { targetId, flatten: true });
+
+const hits = []; const reqs = new Map(); let last = Date.now();
+cdp.onEvent = (m, p, sid) => {
+  if (sid !== S) return;
+  last = Date.now();
+  if (m === 'Network.requestWillBeSent') {
+    reqs.set(p.requestId, { url: p.request.url, postData: p.request.postData || null });
+  } else if (m === 'Network.responseReceived') {
+    if (/mtop\./i.test(p.response.url)) hits.push({ requestId: p.requestId, url: p.response.url });
+  }
+};
+await cdp.send('Network.enable', {}, S);
+await cdp.send('Page.enable', {}, S);
+
+const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+const quiet = async (q = 3000, max = 40000) => {
+  const t0 = Date.now();
+  while (Date.now() - t0 < max) { await sleep(300); if (Date.now() - last > q) return; }
+};
+await quiet(4000, 30000);
+say('after load: mtop hits = ' + hits.length);
+
+// 探测页面结构
+const probe = await cdp.ev(`(() => {
+  const q = s => Array.from(document.querySelectorAll(s));
+  const main = document.querySelector('.ant-layout-content') || document.body;
+  return {
+    href: location.href, title: document.title,
+    radioGroups: q('.ant-radio-group').map(g => {
+      const o = Array.from(g.querySelectorAll('.ant-radio-wrapper, .ant-radio-button-wrapper'));
+      const c = o.find(x => /checked/.test(x.className) || x.querySelector('.ant-radio-checked'));
+      return { opts: o.map(x => (x.innerText||'').trim()), checked: c ? (c.innerText||'').trim() : null };
+    }),
+    headers: q('.ant-table-thead th').map(t => t.innerText.replace(/\\s+/g,' ').trim()).filter(Boolean),
+    headerRows: q('.ant-table-thead tr').map(tr => Array.from(tr.querySelectorAll('th')).map(t => t.innerText.replace(/\\s+/g,' ').trim())),
+    rowCount: q('.ant-table-tbody tr').length,
+    rows: q('.ant-table-tbody tr').slice(0, 25).map(tr => Array.from(tr.querySelectorAll('td')).map(td => (td.innerText||'').replace(/\\s+/g,' ').trim())),
+    pagination: q('.ant-pagination-item').map(e => e.innerText.trim()),
+    total: (document.querySelector('.ant-pagination-total-text')||{}).innerText || null,
+    buttons: q('button').map(b => b.innerText.trim()).filter(Boolean).slice(0,30),
+    mainTextHead: (main.innerText||'').slice(0, 2500),
+  };
+})()`, S);
+fs.writeFileSync(path.join(OUT, 'cap', TAG + '-probe.json'), JSON.stringify(probe, null, 2), 'utf8');
+say('headers = ' + JSON.stringify(probe.headers));
+say('radioGroups = ' + JSON.stringify(probe.radioGroups.map(g => g.checked)));
+say('rows = ' + probe.rowCount + ' total=' + probe.total + ' pagination=' + JSON.stringify(probe.pagination));
+
+// 存接口响应
+let n = 0;
+for (const h of hits) {
+  try {
+    const b = await cdp.send('Network.getResponseBody', { requestId: h.requestId }, S, 20000);
+    let body = b.base64Encoded ? Buffer.from(b.body, 'base64').toString('utf8') : b.body;
+    const req = [...reqs.entries()].find(([id]) => id === h.requestId)?.[1];
+    const api = (h.url.match(/mtop\.[A-Za-z0-9._]+/) || [''])[0];
+    fs.writeFileSync(path.join(OUT, 'cap', `${TAG}-${String(n).padStart(2,'0')}.txt`),
+      'URL: ' + h.url + '\nPOST: ' + (req?.postData || '-') + '\n\n' + body.slice(0, 600000), 'utf8');
+    say(`  [${n}] ${api.replace('mtop.alibaba.idle.seller.pc.datacompass.','')}  ${body.length}B`);
+    n++;
+  } catch { }
+}
+say('saved ' + n + ' api bodies');
+
+await cdp.send('Target.closeTarget', { targetId });
+say('closed tab');
+cdp.close();
+process.exit(0);