josefandersson/teams-export.mjs

teams-export.mjs · 21 KiB · JavaScript Raw

// Export a Teams 1:1 chat in frenfoil's chat import format (chats.json + photos/, source "Teams").
// See docs/CHAT-IMPORT-FORMAT.md in frenfoil.
//
// Restarts the new Teams desktop app with its WebView2 debug port open, borrows the auth headers Teams sends
// while it starts up, then fetches the chat and its images directly.
//
// Run again into a folder that holds an export to extend it: the chat is picked from its chats.json, only
// the last --backtrack-days (default 7) before its newest message onward are fetched (--full for everything), and what Teams no
// longer has is kept. The new messages and their photos also go to a folder beside it,
// <outDir>-new-<UTC timestamp>, as an export of their own.
//
//   node teams-export.mjs <outDir> [--with <name>] [--since YYYY-MM-DD] [--backtrack-days <n>] [--full] [--yes] [--no-restart] [--port 9222]

import { spawn } from 'node:child_process';
import { createInterface } from 'node:readline/promises';
import * as fs from 'node:fs';
import * as path from 'node:path';

const FORMAT = 'frenfoil-chat-import';
// Frenfoil recognises a re-imported message by source, chat id and message id, so this never changes.
const SOURCE = 'Teams';

const outPath = path.join(outDir, 'chats.json');
const previous = fs.existsSync(outPath) ? JSON.parse(fs.readFileSync(outPath, 'utf8')) : null;
const previousChat = previous?.chats?.[0];
if (previous && (previous.format !== FORMAT || previous.source !== SOURCE || !Array.isArray(previousChat?.messages)))
  throw new Error(`${outPath} is not an export this tool wrote.`);

// Edits and reactions land on recent messages, so an extension re-reads the days before its newest one.
if (previous && !sinceArg && !full) {
  const newest = previousChat.messages.reduce((n, m) => Math.max(n, +m.time || 0), 0);
  since = Math.max(0, newest - backtrackDays * 86_400_000);
}

const sleep = ms => new Promise(r => setTimeout(r, ms));
const rl = createInterface({ input: process.stdin, output: process.stdout });
const ask = async q => (yes ? '' : (await rl.question(q)).trim());

// --- borrowing Teams' auth --------------------------------------------------------------------------

// Attach to every Teams page and worker as it is created, and keep the headers of its first chat-service
// and image requests.
async function captureAuth() {
  const got = {};
  const deadline = Date.now() + 180_000;
  const done = () => got.chatsvc && (got.objects || Date.now() > got.chatsvcAt + 20_000);

while (!done()) {
    if (Date.now() > deadline) throw new Error('Did not see Teams sign in within 3 minutes.');
    let ws;
    try {
      const ver = await (await fetch(`http://localhost:${port}/json/version`)).json();
      ws = new WebSocket(ver.webSocketDebuggerUrl);
      await new Promise((res, rej) => { ws.onopen = res; ws.onerror = rej; });
    } catch {
      await sleep(300);
      continue;
    }

ws.onmessage = e => {
      const m = JSON.parse(e.data);
      if (m.method === 'Target.attachedToTarget') {
        const s = m.params.sessionId;
        send('Network.enable', {}, s);
        send('Target.setAutoAttach', { autoAttach: true, waitForDebuggerOnStart: true, flatten: true }, s);
        send('Runtime.runIfWaitingForDebugger', {}, s);
      } else if (m.method === 'Network.requestWillBeSent') {
        urls.set(m.params.requestId, m.params.request.url);
        see(m.params.request.url, m.params.request.headers);
      } else if (m.method === 'Network.requestWillBeSentExtraInfo') {
        see(urls.get(m.params.requestId), m.params.headers);
      }
    };
    send('Target.setAutoAttach', { autoAttach: true, waitForDebuggerOnStart: true, flatten: true });

await new Promise(res => {
      ws.onclose = res;
      const t = setInterval(() => { if (done() || Date.now() > deadline) { clearInterval(t); res(); } }, 500);
    });
    try { ws.close(); } catch {}
  }
  return got;
}

// --- HTTP ------------------------------------------------------------------------------------------

async function get(url, headers) {
  for (let i = 0; ; i++) {
    let r;
    try {
      r = await fetch(url, { headers });
    } catch (e) {
      if (i >= 5) throw e;
      await sleep(2 ** i * 1000);
      continue;
    }
    if ((r.status === 429 || r.status >= 500) && i < 5) {
      await sleep((+r.headers.get('retry-after') || 2 ** i) * 1000);
      continue;
    }
    return r;
  }
}

async function getJson(url, headers) {
  const r = await get(url, headers);
  if (r.status === 401) throw new Error('Teams login expired. Run again (it restarts Teams for a fresh one).');
  if (!r.ok) throw new Error(`HTTP ${r.status} for ${url.split('?')[0]}`);
  return r.json();
}

// --- Teams message → import message ---------------------------------------------------------------

const decode = s => s
  .replace(/&nbsp;/g, ' ')
  .replace(/&lt;/g, '<')
  .replace(/&gt;/g, '>')
  .replace(/&quot;/g, '"')
  .replace(/&#39;|&apos;/g, "'")
  .replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(+n))
  .replace(/&amp;/g, '&');

const toText = html => decode((html ?? '')
  .replace(/<br\s*\/?>|<\/(p|div|li)>/gi, '\n')
  .replace(/<[^>]+>/g, ''))
  .replace(/\n{3,}/g, '\n\n')
  .trim();

const esc = s => s.replace(/[&<>"]/g, c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;' }[c]));

// Teams' formatting in the message-styling syntax frenfoil renders. A span only styles when it hugs its
// words on one line; anything else stays plain rather than wrong.
const span = marker => (all, tag, inner) => {
  const t = decode(inner.replace(/<[^>]+>/g, ''));
  return t && t === t.trim() && !/<br|\n/i.test(inner + t) && !t.includes(marker) ? esc(`${marker}${t}${marker}`) : inner;
};
const styled = html => html
  .replace(/<pre\b[^>]*>([\s\S]*?)<\/pre>/gi, (_, inner) => `<br>${esc(`\`\`\`\n${toText(inner)}\n\`\`\``)}<br>`)
  .replace(/<blockquote\b[^>]*>([\s\S]*?)<\/blockquote>/gi, (_, inner) =>
    `<br>${esc(toText(inner).split('\n').map(line => `> ${line}`).join('\n'))}<br>`)
  .replace(/<(b|strong)\b[^>]*>([\s\S]*?)<\/\1>/gi, span('*'))
  .replace(/<(i|em)\b[^>]*>([\s\S]*?)<\/\1>/gi, span('_'))
  .replace(/<(s|strike|del)\b[^>]*>([\s\S]*?)<\/\1>/gi, span('~'))
  .replace(/<(code)\b[^>]*>([\s\S]*?)<\/\1>/gi, span('`'));

const reactions = (jsonProp(m.properties?.emotions) ?? []).flatMap(e => {
    const emoji = emojiOf(e.key);
    return emoji ? (e.users ?? []).map(u => guidOf(u.mri)).filter(Boolean).map(user => ({ emoji, user })) : [];
  });

return {
    id: String(m.id),
    time: timeOf(m),
    edited: +m.properties?.edittime > 0,
    from: guidOf(m.from),
    name: m.imdisplayname || '',
    text: [toText(styled(content)), ...files].filter(Boolean).join('\n'),
    srcs,
    replyTo,
    reactions,
  };
}

// The signed-in user's AAD object id, read from whichever JWT the borrowed headers carry.
function selfGuid(...headerSets) {
  for (const headers of headerSets) {
    for (const v of Object.values(headers ?? {})) {
      for (const t of String(v).match(/[\w-]{10,}\.[\w-]{10,}\.[\w-]+/g) ?? []) {
        try {
          const c = JSON.parse(Buffer.from(t.split('.')[1], 'base64url').toString());
          const id = c.oid || (typeof c.skypeid === 'string' && c.skypeid.split(':').pop());
          if (id) return String(id).toLowerCase();
        } catch {}
      }
    }
  }
}

// --- main ------------------------------------------------------------------------------------------

if (!previous && fs.existsSync(path.join(outDir, 'result.json')))
  console.log('That folder holds an older Telegram-format export (result.json). Writing a full chats.json beside it.');

if (!noRestart) restartTeams();
else console.log('Waiting for Teams to sign in… (restart it with the debug port if nothing happens)');
const auth = await captureAuth();
console.log('Got Teams login.');

const H = auth.chatsvc.headers;
const base = /^(.*\/api\/chatsvc\/[^/]+\/v1)\//.exec(auth.chatsvc.url)[1];
const self = selfGuid(H, auth.objects?.headers, auth.skype && { t: auth.skype });
if (!self) throw new Error('Could not tell which account is signed in.');

async function oneOnOnes() {
  const out = [];
  let next = `${base}/users/ME/conversations?view=${encodeURIComponent(VIEW)}&pageSize=200`;
  for (let page = 0; next && page < 10; page++) {
    const j = await getJson(next, H);
    const list = j.conversations ?? [];
    out.push(...list.filter(c => ONE_ON_ONE_RE.test(c.id)));
    const link = j._metadata?.backwardLink;
    next = list.length && link !== next ? link : null;
  }
  return out.sort((a, b) => (timeOf(b.lastMessage ?? {}) || 0) - (timeOf(a.lastMessage ?? {}) || 0));
}

async function fetchAll() {
  const all = new Map();
  let next = `${messagesUrl(chat.id)}?view=${encodeURIComponent(VIEW)}&pageSize=200&startTime=1`;
  while (next) {
    const j = await getJson(next, H);
    const page = j.messages ?? [];
    page.forEach(m => all.set(m.id, m));
    process.stdout.write(`\r${all.size} messages…`);
    // Pages come newest-changed first, so an old message with a fresh reaction can sit among new ones:
    // only a page changed entirely before `since` means everything after it was already seen.
    const oldest = Math.min(...page.map(m => +m.version || timeOf(m) || Infinity));
    const link = j._metadata?.backwardLink;
    next = page.length && !(since && oldest < since) && link !== next ? link : null;
  }
  process.stdout.write('\n');
  return [...all.values()];
}

const fetched = await fetchAll();
const kept = new Set(fetched.filter(m => /^(Text|RichText)/.test(m.messagetype) && !m.properties?.deletetime));
const leftOut = new Map();
for (const m of fetched) {
  if (kept.has(m)) continue;
  const why = m.properties?.deletetime ? 'deleted' : m.messagetype;
  leftOut.set(why, (leftOut.get(why) ?? 0) + 1);
}

const prepared = [...kept]
  .map(prepare)
  .filter(p => p.time > 0 && p.time >= since)
  .sort((a, b) => a.time - b.time || Number(a.id) - Number(b.id));

const photosDir = path.join(outDir, 'photos');
fs.mkdirSync(photosDir, { recursive: true });
const onDisk = new Map(fs.readdirSync(photosDir).map(n => [n.replace(/\.[^.]+$/, ''), n]));

const objectsHost = auth.objects && new URL(auth.objects.url).host;
const headerSets = [
  auth.objects?.headers,
  auth.skype && { authorization: `skype_token ${auth.skype}` },
  H,
].filter(Boolean);

// The view a message links (imgo) is an 800px preview; the full-size views are tried first.
const FULL_VIEWS = ['imgpsh_fullsize_anim', 'imgpsh_fullsize'];

async function downloadImage(src) {
  const hosts = [src];
  if (objectsHost && /\/v1\/objects\//.test(src) && new URL(src).host !== objectsHost) {
    const u = new URL(src);
    u.host = objectsHost;
    hosts.push(u.href);
  }
  const urls = [...new Set(hosts.flatMap(h => [
    ...FULL_VIEWS.map(v => h.replace(/\/views\/[^/?#]+/, `/views/${v}`)),
    h,
  ]))];
  let status = 0;
  for (const url of urls) {
    for (const [i, headers] of headerSets.entries()) {
      try {
        const r = await get(url, headers);
        status = r.status;
        const type = (r.headers.get('content-type') || '').split(';')[0];
        if (!r.ok || !type.startsWith('image/')) continue;
        if (i > 0) headerSets.unshift(...headerSets.splice(i, 1));
        return { type, bytes: Buffer.from(await r.arrayBuffer()) };
      } catch {}
    }
  }
  return { status };
}

const side = guid => (guid === peer ? 'them' : 'me');

// Merged over what an earlier run saved: nothing is dropped, and an image saved then is not lost to a
// download that failed now.
const pathed = m => (m.files ?? []).filter(f => f.path).length;
const merged = new Map((previousChat?.messages ?? []).map(m => [String(m.id), m]));
const added = [];
let updated = 0;
for (const m of messages) {
  const old = merged.get(m.id);
  if (old && pathed(old) > pathed(m)) continue;
  if (!old) added.push(m);
  else if (JSON.stringify(old) !== JSON.stringify(m)) updated++;
  merged.set(m.id, m);
}
const all = [...merged.values()].sort((a, b) => a.time - b.time || Number(a.id) - Number(b.id));

const exportJson = msgs => JSON.stringify({
  format: FORMAT,
  version: 1,
  source: SOURCE,
  chats: [{ id: chat.id, kind: 'direct', name: peerDisplay, messages: msgs }],
}, null, 1);

if (previous) fs.copyFileSync(outPath, `${outPath}.bak`);
fs.writeFileSync(`${outPath}.tmp`, exportJson(all));
fs.renameSync(`${outPath}.tmp`, outPath);

const failed = unique.length - saved.size;
console.log(`Done: ${added.length} new, ${updated} updated, ${all.length} in total; ${saved.size}/${unique.length} images → ${path.resolve(outPath)}`);

// An extension also gets a folder beside the export with only its new messages and their photos.
if (previous && added.length) {
  const stamp = new Date().toISOString().slice(0, 16).replace('T', '_').replace(':', '');
  const deltaDir = `${path.resolve(outDir)}-new-${stamp}`;
  fs.mkdirSync(path.join(deltaDir, 'photos'), { recursive: true });
  const photos = new Set(added.flatMap(m => (m.files ?? []).map(f => f.path).filter(Boolean)));
  for (const p of photos) fs.copyFileSync(path.join(outDir, p), path.join(deltaDir, p));
  fs.writeFileSync(path.join(deltaDir, 'chats.json'), exportJson(added));
  console.log(`New only: ${added.length} messages, ${photos.size} images → ${deltaDir}`);
}
if (failed)
  console.log(`${failed} images failed (HTTP ${[...failures].map(([s, n]) => `${s || 'error'}×${n}`).join(', ')}). Run again with --full into the same folder to retry them.`);

rl.close();
process.exit(0);

1	// Export a Teams 1:1 chat in frenfoil's chat import format (chats.json + photos/, source "Teams").
2	// See docs/CHAT-IMPORT-FORMAT.md in frenfoil.
3	//
4	// Restarts the new Teams desktop app with its WebView2 debug port open, borrows the auth headers Teams sends
5	// while it starts up, then fetches the chat and its images directly.
6	//
7	// Run again into a folder that holds an export to extend it: the chat is picked from its chats.json, only
8	// the last --backtrack-days (default 7) before its newest message onward are fetched (--full for everything), and what Teams no
9	// longer has is kept. The new messages and their photos also go to a folder beside it,
10	// <outDir>-new-<UTC timestamp>, as an export of their own.
11	//
12	// node teams-export.mjs <outDir> [--with <name>] [--since YYYY-MM-DD] [--backtrack-days <n>] [--full] [--yes] [--no-restart] [--port 9222]
13
14	import { spawn } from 'node:child_process';
15	import { createInterface } from 'node:readline/promises';
16	import * as fs from 'node:fs';
17	import * as path from 'node:path';
18
19	const args = process.argv.slice(2);
20	const flag = name => { const i = args.indexOf(`--${name}`); return i >= 0 ? (args.splice(i, 1), true) : false; };
21	const opt = name => { const i = args.indexOf(`--${name}`); return i >= 0 ? args.splice(i, 2)[1] : undefined; };
22	const port = opt('port') ?? '9222';
23	const sinceArg = opt('since');
24	const backtrackArg = opt('backtrack-days');
25	const withName = opt('with');
26	const yes = flag('yes');
27	const full = flag('full');
28	const noRestart = flag('no-restart');
29	const outDir = args[0];
30	if (!outDir) {
31	console.log('usage: node teams-export.mjs <outDir> [--with <name>] [--since YYYY-MM-DD] [--backtrack-days <n>] [--full] [--yes] [--no-restart] [--port 9222]');
32	process.exit(1);
33	}
34	let since = sinceArg ? Date.parse(sinceArg) : 0;
35	if (isNaN(since)) throw new Error(`Invalid --since date: ${sinceArg}`);
36	const backtrackDays = +(backtrackArg ?? 7);
37	if (!(backtrackDays >= 0)) throw new Error(`Invalid --backtrack-days: ${backtrackArg}`);
38
39	const FORMAT = 'frenfoil-chat-import';
40	// Frenfoil recognises a re-imported message by source, chat id and message id, so this never changes.
41	const SOURCE = 'Teams';
42
43	const outPath = path.join(outDir, 'chats.json');
44	const previous = fs.existsSync(outPath) ? JSON.parse(fs.readFileSync(outPath, 'utf8')) : null;
45	const previousChat = previous?.chats?.[0];
46	if (previous && (previous.format !== FORMAT \|\| previous.source !== SOURCE \|\| !Array.isArray(previousChat?.messages)))
47	throw new Error(`${outPath} is not an export this tool wrote.`);
48
49	// Edits and reactions land on recent messages, so an extension re-reads the days before its newest one.
50	if (previous && !sinceArg && !full) {
51	const newest = previousChat.messages.reduce((n, m) => Math.max(n, +m.time \|\| 0), 0);
52	since = Math.max(0, newest - backtrackDays * 86_400_000);
53	}
54
55	const PARALLEL = 4;
56	const ONE_ON_ONE_RE = /^19:([0-9a-f-]{36})_([0-9a-f-]{36})@unq\.gbl\.spaces$/i;
57	const EXT = { 'image/png': 'png', 'image/jpeg': 'jpg', 'image/gif': 'gif', 'image/webp': 'webp', 'image/bmp': 'bmp' };
58	const VIEW = 'msnp24Equivalent\|supportsMessageProperties';
59
60	const sleep = ms => new Promise(r => setTimeout(r, ms));
61	const rl = createInterface({ input: process.stdin, output: process.stdout });
62	const ask = async q => (yes ? '' : (await rl.question(q)).trim());
63
64	// --- borrowing Teams' auth --------------------------------------------------------------------------
65
66	function restartTeams() {
67	console.log('Restarting Teams to borrow its login…');
68	const env = { ...process.env, WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS: `--remote-debugging-port=${port}` };
69	spawn('powershell.exe', ['-NoProfile', '-Command',
70	'Get-Process ms-teams -ErrorAction SilentlyContinue \| Stop-Process -Force; Start-Sleep 3; Start-Process ms-teams.exe'],
71	{ env, stdio: 'ignore' });
72	}
73
74	const cleanHeaders = h => Object.fromEntries(Object.entries(h)
75	.map(([k, v]) => [k.toLowerCase(), v])
76	.filter(([k]) => !/^(:\|host$\|cookie$\|content-length$\|content-type$\|connection$\|accept-encoding$)/.test(k)));
77
78	// Attach to every Teams page and worker as it is created, and keep the headers of its first chat-service
79	// and image requests.
80	async function captureAuth() {
81	const got = {};
82	const deadline = Date.now() + 180_000;
83	const done = () => got.chatsvc && (got.objects \|\| Date.now() > got.chatsvcAt + 20_000);
84
85	while (!done()) {
86	if (Date.now() > deadline) throw new Error('Did not see Teams sign in within 3 minutes.');
87	let ws;
88	try {
89	const ver = await (await fetch(`http://localhost:${port}/json/version`)).json();
90	ws = new WebSocket(ver.webSocketDebuggerUrl);
91	await new Promise((res, rej) => { ws.onopen = res; ws.onerror = rej; });
92	} catch {
93	await sleep(300);
94	continue;
95	}
96
97	let n = 0;
98	const urls = new Map();
99	const send = (method, params, sessionId) =>
100	ws.send(JSON.stringify({ id: ++n, method, params: params ?? {}, ...sessionId ? { sessionId } : {} }));
101	const see = (url, headers) => {
102	if (!url) return;
103	const h = cleanHeaders(headers);
104	if (h['x-skypetoken']) got.skype ??= h['x-skypetoken'];
105	if (!h.authorization) return;
106	if (!got.chatsvc && /\/api\/chatsvc\/[^/]+\/v1\//.test(url)) {
107	got.chatsvc = { url, headers: h };
108	got.chatsvcAt = Date.now();
109	} else if (!got.objects && /\/v1\/objects\//.test(url)) {
110	got.objects = { url, headers: h };
111	}
112	};
113
114	ws.onmessage = e => {
115	const m = JSON.parse(e.data);
116	if (m.method === 'Target.attachedToTarget') {
117	const s = m.params.sessionId;
118	send('Network.enable', {}, s);
119	send('Target.setAutoAttach', { autoAttach: true, waitForDebuggerOnStart: true, flatten: true }, s);
120	send('Runtime.runIfWaitingForDebugger', {}, s);
121	} else if (m.method === 'Network.requestWillBeSent') {
122	urls.set(m.params.requestId, m.params.request.url);
123	see(m.params.request.url, m.params.request.headers);
124	} else if (m.method === 'Network.requestWillBeSentExtraInfo') {
125	see(urls.get(m.params.requestId), m.params.headers);
126	}
127	};
128	send('Target.setAutoAttach', { autoAttach: true, waitForDebuggerOnStart: true, flatten: true });
129
130	await new Promise(res => {
131	ws.onclose = res;
132	const t = setInterval(() => { if (done() \|\| Date.now() > deadline) { clearInterval(t); res(); } }, 500);
133	});
134	try { ws.close(); } catch {}
135	}
136	return got;
137	}
138
139	// --- HTTP ------------------------------------------------------------------------------------------
140
141	async function get(url, headers) {
142	for (let i = 0; ; i++) {
143	let r;
144	try {
145	r = await fetch(url, { headers });
146	} catch (e) {
147	if (i >= 5) throw e;
148	await sleep(2 ** i * 1000);
149	continue;
150	}
151	if ((r.status === 429 \|\| r.status >= 500) && i < 5) {
152	await sleep((+r.headers.get('retry-after') \|\| 2 ** i) * 1000);
153	continue;
154	}
155	return r;
156	}
157	}
158
159	async function getJson(url, headers) {
160	const r = await get(url, headers);
161	if (r.status === 401) throw new Error('Teams login expired. Run again (it restarts Teams for a fresh one).');
162	if (!r.ok) throw new Error(`HTTP ${r.status} for ${url.split('?')[0]}`);
163	return r.json();
164	}
165
166	// --- Teams message → import message ---------------------------------------------------------------
167
168	const decode = s => s
169	.replace(/ /g, ' ')
170	.replace(/</g, '<')
171	.replace(/>/g, '>')
172	.replace(/"/g, '"')
173	.replace(/'\|'/g, "'")
174	.replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(+n))
175	.replace(/&/g, '&');
176
177	const toText = html => decode((html ?? '')
178	.replace(/<br\s*\/?>\|<\/(p\|div\|li)>/gi, '\n')
179	.replace(/<[^>]+>/g, ''))
180	.replace(/\n{3,}/g, '\n\n')
181	.trim();
182
183	const esc = s => s.replace(/[&<>"]/g, c => ({ '&': '&', '<': '<', '>': '>', '"': '"' }[c]));
184
185	// Teams' formatting in the message-styling syntax frenfoil renders. A span only styles when it hugs its
186	// words on one line; anything else stays plain rather than wrong.
187	const span = marker => (all, tag, inner) => {
188	const t = decode(inner.replace(/<[^>]+>/g, ''));
189	return t && t === t.trim() && !/<br\|\n/i.test(inner + t) && !t.includes(marker) ? esc(`${marker}${t}${marker}`) : inner;
190	};
191	const styled = html => html
192	.replace(/<pre\b[^>]>([\s\S]?)<\/pre>/gi, (_, inner) => `<br>${esc(`\`\`\`\n${toText(inner)}\n\`\`\``)}<br>`)
193	.replace(/<blockquote\b[^>]>([\s\S]?)<\/blockquote>/gi, (_, inner) =>
194	`<br>${esc(toText(inner).split('\n').map(line => `> ${line}`).join('\n'))}<br>`)
195	.replace(/<(b\|strong)\b[^>]>([\s\S]?)<\/\1>/gi, span('*'))
196	.replace(/<(i\|em)\b[^>]>([\s\S]?)<\/\1>/gi, span('_'))
197	.replace(/<(s\|strike\|del)\b[^>]>([\s\S]?)<\/\1>/gi, span('~'))
198	.replace(/<(code)\b[^>]>([\s\S]?)<\/\1>/gi, span('`'));
199
200	const fnv = s => { let h = 0x811c9dc5; for (const c of s) h = Math.imul(h ^ c.codePointAt(0), 0x01000193) >>> 0; return h; };
201	const imageId = src => (/\/objects\/([^/?#]+)/.exec(src)?.[1] \|\| 'x' + fnv(src).toString(36)).replace(/[^\w.-]/g, '_');
202	const guidOf = mri => /8:orgid:([0-9a-f-]{36})/i.exec(mri ?? '')?.[1]?.toLowerCase();
203	const jsonProp = v => { try { return typeof v === 'string' ? JSON.parse(v) : v; } catch { return null; } };
204	const timeOf = m => Date.parse(m.composetime \|\| m.originalarrivaltime);
205
206	const EMOTION = { like: '👍', heart: '❤️', laugh: '😆', surprised: '😮', sad: '😢', angry: '😡' };
207	function emojiOf(key) {
208	if (EMOTION[key]) return EMOTION[key];
209	const hex = /^([0-9a-f]{4,6}(?:-[0-9a-f]{4,6})*)_/i.exec(key)?.[1];
210	return hex ? String.fromCodePoint(...hex.split('-').map(h => parseInt(h, 16))) : null;
211	}
212
213	function prepare(m) {
214	let content = m.content ?? '';
215	let replyTo;
216	content = content.replace(/<blockquote\b([^>])>[\s\S]?<\/blockquote>/gi, (all, attrs) => {
217	if (!/schema\.skype\.com\/Reply/i.test(attrs)) return all;
218	replyTo = /itemid="(\d+)"/i.exec(attrs)?.[1];
219	return '';
220	});
221	const srcs = [];
222	content = content.replace(/<img\b[^>]*>/gi, tag => {
223	const attr = k => (new RegExp(`\\s${k}="([^"]*)"`, 'i').exec(tag) \|\| [])[1] \|\| '';
224	if (/emoji/i.test(attr('itemtype'))) return attr('alt');
225	const src = decode(attr('src'));
226	if (/^https:/.test(src)) srcs.push(src);
227	return '';
228	});
229	content = content.replace(/<a\b[^>]?href="([^"])"[^>]>([\s\S]?)<\/a>/gi, (_, href, label) => {
230	const h = decode(href);
231	const l = toText(label);
232	return esc(!l \|\| l === h ? h : `${l} (${h})`);
233	});
234
235	const files = (jsonProp(m.properties?.files) ?? []).map(f => {
236	const name = f.fileName \|\| f.title \|\| 'file';
237	const url = f.objectUrl \|\| f.fileInfo?.shareUrl \|\| '';
238	return `📎 ${name}${url ? ` (${url})` : ''}`;
239	});
240
241	const reactions = (jsonProp(m.properties?.emotions) ?? []).flatMap(e => {
242	const emoji = emojiOf(e.key);
243	return emoji ? (e.users ?? []).map(u => guidOf(u.mri)).filter(Boolean).map(user => ({ emoji, user })) : [];
244	});
245
246	return {
247	id: String(m.id),
248	time: timeOf(m),
249	edited: +m.properties?.edittime > 0,
250	from: guidOf(m.from),
251	name: m.imdisplayname \|\| '',
252	text: [toText(styled(content)), ...files].filter(Boolean).join('\n'),
253	srcs,
254	replyTo,
255	reactions,
256	};
257	}
258
259	// The signed-in user's AAD object id, read from whichever JWT the borrowed headers carry.
260	function selfGuid(...headerSets) {
261	for (const headers of headerSets) {
262	for (const v of Object.values(headers ?? {})) {
263	for (const t of String(v).match(/[\w-]{10,}\.[\w-]{10,}\.[\w-]+/g) ?? []) {
264	try {
265	const c = JSON.parse(Buffer.from(t.split('.')[1], 'base64url').toString());
266	const id = c.oid \|\| (typeof c.skypeid === 'string' && c.skypeid.split(':').pop());
267	if (id) return String(id).toLowerCase();
268	} catch {}
269	}
270	}
271	}
272	}
273
274	// --- main ------------------------------------------------------------------------------------------
275
276	if (!previous && fs.existsSync(path.join(outDir, 'result.json')))
277	console.log('That folder holds an older Telegram-format export (result.json). Writing a full chats.json beside it.');
278
279	if (!noRestart) restartTeams();
280	else console.log('Waiting for Teams to sign in… (restart it with the debug port if nothing happens)');
281	const auth = await captureAuth();
282	console.log('Got Teams login.');
283
284	const H = auth.chatsvc.headers;
285	const base = /^(.*\/api\/chatsvc\/[^/]+\/v1)\//.exec(auth.chatsvc.url)[1];
286	const self = selfGuid(H, auth.objects?.headers, auth.skype && { t: auth.skype });
287	if (!self) throw new Error('Could not tell which account is signed in.');
288
289	const messagesUrl = conv => `${base}/users/ME/conversations/${encodeURIComponent(conv)}/messages`;
290	const peerOf = conv => { const [, a, b] = ONE_ON_ONE_RE.exec(conv); return a.toLowerCase() === self ? b.toLowerCase() : a.toLowerCase(); };
291
292	async function oneOnOnes() {
293	const out = [];
294	let next = `${base}/users/ME/conversations?view=${encodeURIComponent(VIEW)}&pageSize=200`;
295	for (let page = 0; next && page < 10; page++) {
296	const j = await getJson(next, H);
297	const list = j.conversations ?? [];
298	out.push(...list.filter(c => ONE_ON_ONE_RE.test(c.id)));
299	const link = j._metadata?.backwardLink;
300	next = list.length && link !== next ? link : null;
301	}
302	return out.sort((a, b) => (timeOf(b.lastMessage ?? {}) \|\| 0) - (timeOf(a.lastMessage ?? {}) \|\| 0));
303	}
304
305	const peerNames = new Map();
306	async function peerName(c) {
307	const peer = peerOf(c.id);
308	if (peerNames.has(peer)) return peerNames.get(peer);
309	let name = guidOf(c.lastMessage?.from) === peer ? c.lastMessage.imdisplayname : '';
310	if (!name) {
311	const j = await getJson(`${messagesUrl(c.id)}?view=${encodeURIComponent(VIEW)}&pageSize=100&startTime=1`, H);
312	name = (j.messages ?? []).find(m => guidOf(m.from) === peer && m.imdisplayname)?.imdisplayname ?? '';
313	}
314	peerNames.set(peer, name);
315	return name;
316	}
317
318	const chats = await oneOnOnes();
319	let chat;
320	if (previous) {
321	chat = chats.find(c => c.id === previousChat.id);
322	if (!chat) throw new Error(`The chat in ${outPath} is no longer in Teams. The export there is left as it is.`);
323	if (withName && !(await peerName(chat)).toLowerCase().includes(withName.toLowerCase()))
324	throw new Error(`${outPath} holds the chat with ${await peerName(chat)}, not "${withName}". Use another folder.`);
325	} else if (withName) {
326	for (const c of chats) {
327	if ((await peerName(c)).toLowerCase().includes(withName.toLowerCase())) { chat = c; break; }
328	}
329	if (!chat) throw new Error(`No 1:1 chat with "${withName}" among ${chats.length} chats.`);
330	} else {
331	const top = chats.slice(0, 20);
332	for (const [i, c] of top.entries()) console.log(`${String(i + 1).padStart(2)}. ${await peerName(c) \|\| '(unknown)'}`);
333	if (yes) throw new Error('Pass --with <name> together with --yes.');
334	chat = top[+(await ask('Which chat? ')) - 1];
335	if (!chat) throw new Error('No such chat.');
336	}
337	const peer = peerOf(chat.id);
338	const peerDisplay = await peerName(chat) \|\| previousChat?.name \|\| 'Teams chat';
339	console.log(`Chat with ${peerDisplay}`);
340	if (previous && since)
341	console.log(`Extending the export in ${path.resolve(outDir)} (${previousChat.messages.length} messages), from ${new Date(since).toISOString().slice(0, 10)}.`);
342
343	async function fetchAll() {
344	const all = new Map();
345	let next = `${messagesUrl(chat.id)}?view=${encodeURIComponent(VIEW)}&pageSize=200&startTime=1`;
346	while (next) {
347	const j = await getJson(next, H);
348	const page = j.messages ?? [];
349	page.forEach(m => all.set(m.id, m));
350	process.stdout.write(`\r${all.size} messages…`);
351	// Pages come newest-changed first, so an old message with a fresh reaction can sit among new ones:
352	// only a page changed entirely before `since` means everything after it was already seen.
353	const oldest = Math.min(...page.map(m => +m.version \|\| timeOf(m) \|\| Infinity));
354	const link = j._metadata?.backwardLink;
355	next = page.length && !(since && oldest < since) && link !== next ? link : null;
356	}
357	process.stdout.write('\n');
358	return [...all.values()];
359	}
360
361	const fetched = await fetchAll();
362	const kept = new Set(fetched.filter(m => /^(Text\|RichText)/.test(m.messagetype) && !m.properties?.deletetime));
363	const leftOut = new Map();
364	for (const m of fetched) {
365	if (kept.has(m)) continue;
366	const why = m.properties?.deletetime ? 'deleted' : m.messagetype;
367	leftOut.set(why, (leftOut.get(why) ?? 0) + 1);
368	}
369
370	const prepared = [...kept]
371	.map(prepare)
372	.filter(p => p.time > 0 && p.time >= since)
373	.sort((a, b) => a.time - b.time \|\| Number(a.id) - Number(b.id));
374
375	const unique = [...new Set(prepared.flatMap(p => p.srcs))];
376	const day = ms => new Date(ms).toISOString().slice(0, 10);
377	const span2 = prepared.length ? ` (${day(prepared[0].time)} → ${day(prepared.at(-1).time)})` : '';
378	console.log(`${prepared.length} messages${span2}, ${unique.length} images.`);
379	if (leftOut.size) console.log(`Left out: ${[...leftOut].map(([k, n]) => `${n} ${k}`).join(', ')}.`);
380	if (/^n/i.test(await ask(`Export to ${path.resolve(outDir)}? [Y/n] `))) process.exit(0);
381
382	const photosDir = path.join(outDir, 'photos');
383	fs.mkdirSync(photosDir, { recursive: true });
384	const onDisk = new Map(fs.readdirSync(photosDir).map(n => [n.replace(/\.[^.]+$/, ''), n]));
385
386	const objectsHost = auth.objects && new URL(auth.objects.url).host;
387	const headerSets = [
388	auth.objects?.headers,
389	auth.skype && { authorization: `skype_token ${auth.skype}` },
390	H,
391	].filter(Boolean);
392
393	// The view a message links (imgo) is an 800px preview; the full-size views are tried first.
394	const FULL_VIEWS = ['imgpsh_fullsize_anim', 'imgpsh_fullsize'];
395
396	async function downloadImage(src) {
397	const hosts = [src];
398	if (objectsHost && /\/v1\/objects\//.test(src) && new URL(src).host !== objectsHost) {
399	const u = new URL(src);
400	u.host = objectsHost;
401	hosts.push(u.href);
402	}
403	const urls = [...new Set(hosts.flatMap(h => [
404	...FULL_VIEWS.map(v => h.replace(/\/views\/[^/?#]+/, `/views/${v}`)),
405	h,
406	]))];
407	let status = 0;
408	for (const url of urls) {
409	for (const [i, headers] of headerSets.entries()) {
410	try {
411	const r = await get(url, headers);
412	status = r.status;
413	const type = (r.headers.get('content-type') \|\| '').split(';')[0];
414	if (!r.ok \|\| !type.startsWith('image/')) continue;
415	if (i > 0) headerSets.unshift(...headerSets.splice(i, 1));
416	return { type, bytes: Buffer.from(await r.arrayBuffer()) };
417	} catch {}
418	}
419	}
420	return { status };
421	}
422
423	const saved = new Map();
424	const failures = new Map();
425	let done = 0;
426	let next = 0;
427	await Promise.all(Array.from({ length: PARALLEL }, async () => {
428	while (next < unique.length) {
429	const src = unique[next++];
430	const id = imageId(src);
431	let name = onDisk.get(id);
432	if (!name) {
433	const r = await downloadImage(src);
434	if (r.bytes) {
435	name = `${id}.${EXT[r.type] \|\| 'jpg'}`;
436	fs.writeFileSync(path.join(photosDir, name), r.bytes);
437	onDisk.set(id, name);
438	} else {
439	failures.set(r.status, (failures.get(r.status) ?? 0) + 1);
440	}
441	}
442	if (name) saved.set(src, `photos/${name}`);
443	process.stdout.write(`\rImages ${++done}/${unique.length}…`);
444	}
445	}));
446	if (unique.length) process.stdout.write('\n');
447
448	const side = guid => (guid === peer ? 'them' : 'me');
449
450	const messages = prepared.map(p => {
451	// An image without a path is one the export could not include; frenfoil says so in the message.
452	const files = p.srcs.map(src => (saved.has(src) ? { path: saved.get(src) } : { name: 'Image' }));
453	return {
454	id: p.id,
455	time: p.time,
456	from: side(p.from),
457	...p.text ? { text: p.text } : {},
458	...files.length ? { files } : {},
459	...p.replyTo ? { replyTo: p.replyTo } : {},
460	...p.edited ? { edited: true } : {},
461	...p.reactions.length ? { reactions: p.reactions.map(r => ({ emoji: r.emoji, from: side(r.user) })) } : {},
462	};
463	});
464
465	// Merged over what an earlier run saved: nothing is dropped, and an image saved then is not lost to a
466	// download that failed now.
467	const pathed = m => (m.files ?? []).filter(f => f.path).length;
468	const merged = new Map((previousChat?.messages ?? []).map(m => [String(m.id), m]));
469	const added = [];
470	let updated = 0;
471	for (const m of messages) {
472	const old = merged.get(m.id);
473	if (old && pathed(old) > pathed(m)) continue;
474	if (!old) added.push(m);
475	else if (JSON.stringify(old) !== JSON.stringify(m)) updated++;
476	merged.set(m.id, m);
477	}
478	const all = [...merged.values()].sort((a, b) => a.time - b.time \|\| Number(a.id) - Number(b.id));
479
480	const exportJson = msgs => JSON.stringify({
481	format: FORMAT,
482	version: 1,
483	source: SOURCE,
484	chats: [{ id: chat.id, kind: 'direct', name: peerDisplay, messages: msgs }],
485	}, null, 1);
486
487	if (previous) fs.copyFileSync(outPath, `${outPath}.bak`);
488	fs.writeFileSync(`${outPath}.tmp`, exportJson(all));
489	fs.renameSync(`${outPath}.tmp`, outPath);
490
491	const failed = unique.length - saved.size;
492	console.log(`Done: ${added.length} new, ${updated} updated, ${all.length} in total; ${saved.size}/${unique.length} images → ${path.resolve(outPath)}`);
493
494	// An extension also gets a folder beside the export with only its new messages and their photos.
495	if (previous && added.length) {
496	const stamp = new Date().toISOString().slice(0, 16).replace('T', '_').replace(':', '');
497	const deltaDir = `${path.resolve(outDir)}-new-${stamp}`;
498	fs.mkdirSync(path.join(deltaDir, 'photos'), { recursive: true });
499	const photos = new Set(added.flatMap(m => (m.files ?? []).map(f => f.path).filter(Boolean)));
500	for (const p of photos) fs.copyFileSync(path.join(outDir, p), path.join(deltaDir, p));
501	fs.writeFileSync(path.join(deltaDir, 'chats.json'), exportJson(added));
502	console.log(`New only: ${added.length} messages, ${photos.size} images → ${deltaDir}`);
503	}
504	if (failed)
505	console.log(`${failed} images failed (HTTP ${[...failures].map(([s, n]) => `${s \|\| 'error'}×${n}`).join(', ')}). Run again with --full into the same folder to retry them.`);
506
507	rl.close();
508	process.exit(0);
509