josefandersson/teams-export.mjs

teams-export.mjs · 20 KiB · JavaScript Raw

// Export a Teams 1:1 chat as a Telegram Desktop JSON export (result.json + photos/), for frenfoil's chat import.
//
// Restarts the new Teams desktop app with its WebView2 debug port open, borrows the auth headers Teams sends
// while it starts up, then fetches the chat and its images directly.
//
// Run again into a folder that holds an export to extend it: the chat is picked from its result.json, only
// the last week before its newest message onward is fetched (--full for everything), and what Teams no
// longer has is kept.
//
//   node teams-export.mjs <outDir> [--with <name>] [--since YYYY-MM-DD] [--full] [--yes] [--no-restart] [--port 9222]

import { spawn } from 'node:child_process';
import { createInterface } from 'node:readline/promises';
import * as fs from 'node:fs';
import * as path from 'node:path';

const resultPath = path.join(outDir, 'result.json');
const previous = fs.existsSync(resultPath) ? JSON.parse(fs.readFileSync(resultPath, 'utf8')) : null;
if (previous && (previous.type !== 'personal_chat' || !Array.isArray(previous.messages)))
  throw new Error(`${resultPath} is not an export this tool wrote.`);

// Edits and reactions land on recent messages, so an extension re-reads the week before its newest one.
const OVERLAP_DAYS = 7;
if (previous && !sinceArg && !full) {
  const newest = previous.messages.reduce((n, m) => Math.max(n, +m.date_unixtime || 0), 0);
  since = Math.max(0, newest - OVERLAP_DAYS * 86400) * 1000;
}

const sleep = ms => new Promise(r => setTimeout(r, ms));
const rl = createInterface({ input: process.stdin, output: process.stdout });
const ask = async q => (yes ? '' : (await rl.question(q)).trim());

// --- borrowing Teams' auth --------------------------------------------------------------------------

// Attach to every Teams page and worker as it is created, and keep the headers of its first chat-service
// and image requests.
async function captureAuth() {
  const got = {};
  const deadline = Date.now() + 180_000;
  const done = () => got.chatsvc && (got.objects || Date.now() > got.chatsvcAt + 20_000);

while (!done()) {
    if (Date.now() > deadline) throw new Error('Did not see Teams sign in within 3 minutes.');
    let ws;
    try {
      const ver = await (await fetch(`http://localhost:${port}/json/version`)).json();
      ws = new WebSocket(ver.webSocketDebuggerUrl);
      await new Promise((res, rej) => { ws.onopen = res; ws.onerror = rej; });
    } catch {
      await sleep(300);
      continue;
    }

ws.onmessage = e => {
      const m = JSON.parse(e.data);
      if (m.method === 'Target.attachedToTarget') {
        const s = m.params.sessionId;
        send('Network.enable', {}, s);
        send('Target.setAutoAttach', { autoAttach: true, waitForDebuggerOnStart: true, flatten: true }, s);
        send('Runtime.runIfWaitingForDebugger', {}, s);
      } else if (m.method === 'Network.requestWillBeSent') {
        urls.set(m.params.requestId, m.params.request.url);
        see(m.params.request.url, m.params.request.headers);
      } else if (m.method === 'Network.requestWillBeSentExtraInfo') {
        see(urls.get(m.params.requestId), m.params.headers);
      }
    };
    send('Target.setAutoAttach', { autoAttach: true, waitForDebuggerOnStart: true, flatten: true });

await new Promise(res => {
      ws.onclose = res;
      const t = setInterval(() => { if (done() || Date.now() > deadline) { clearInterval(t); res(); } }, 500);
    });
    try { ws.close(); } catch {}
  }
  return got;
}

// --- HTTP ------------------------------------------------------------------------------------------

async function get(url, headers) {
  for (let i = 0; ; i++) {
    let r;
    try {
      r = await fetch(url, { headers });
    } catch (e) {
      if (i >= 5) throw e;
      await sleep(2 ** i * 1000);
      continue;
    }
    if ((r.status === 429 || r.status >= 500) && i < 5) {
      await sleep((+r.headers.get('retry-after') || 2 ** i) * 1000);
      continue;
    }
    return r;
  }
}

async function getJson(url, headers) {
  const r = await get(url, headers);
  if (r.status === 401) throw new Error('Teams login expired. Run again (it restarts Teams for a fresh one).');
  if (!r.ok) throw new Error(`HTTP ${r.status} for ${url.split('?')[0]}`);
  return r.json();
}

// --- Teams message → Telegram message -------------------------------------------------------------

const decode = s => s
  .replace(/&nbsp;/g, ' ')
  .replace(/&lt;/g, '<')
  .replace(/&gt;/g, '>')
  .replace(/&quot;/g, '"')
  .replace(/&#39;|&apos;/g, "'")
  .replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(+n))
  .replace(/&amp;/g, '&');

const toText = html => decode((html ?? '')
  .replace(/<br\s*\/?>|<\/(p|div|li)>/gi, '\n')
  .replace(/<[^>]+>/g, ''))
  .replace(/\n{3,}/g, '\n\n')
  .trim();

const esc = s => s.replace(/[&<>"]/g, c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;' }[c]));

const reactions = (jsonProp(m.properties?.emotions) ?? []).flatMap(e => {
    const emoji = emojiOf(e.key);
    const users = (e.users ?? []).map(u => guidOf(u.mri)).filter(Boolean);
    return emoji && users.length ? [{ emoji, users }] : [];
  });

const id = /^\d{1,17}$/.test(m.id) ? m.id : String(fnv(m.id));
  const edit = m.properties?.edittime && Math.floor(+m.properties.edittime / 1000);

return {
    id,
    at: Math.floor(timeOf(m) / 1000),
    edited: edit > 0 ? edit : undefined,
    from: guidOf(m.from),
    name: m.imdisplayname || '',
    text: [toText(content), ...files].filter(Boolean).join('\n'),
    srcs,
    replyTo,
    reactions,
  };
}

// The signed-in user's AAD object id, read from whichever JWT the borrowed headers carry.
function selfGuid(...headerSets) {
  for (const headers of headerSets) {
    for (const v of Object.values(headers ?? {})) {
      for (const t of String(v).match(/[\w-]{10,}\.[\w-]{10,}\.[\w-]+/g) ?? []) {
        try {
          const c = JSON.parse(Buffer.from(t.split('.')[1], 'base64url').toString());
          const id = c.oid || (typeof c.skypeid === 'string' && c.skypeid.split(':').pop());
          if (id) return String(id).toLowerCase();
        } catch {}
      }
    }
  }
}

// --- main ------------------------------------------------------------------------------------------

if (!noRestart) restartTeams();
else console.log('Waiting for Teams to sign in… (restart it with the debug port if nothing happens)');
const auth = await captureAuth();
console.log('Got Teams login.');

const H = auth.chatsvc.headers;
const base = /^(.*\/api\/chatsvc\/[^/]+\/v1)\//.exec(auth.chatsvc.url)[1];
const self = selfGuid(H, auth.objects?.headers, auth.skype && { t: auth.skype });
if (!self) throw new Error('Could not tell which account is signed in.');

async function oneOnOnes() {
  const out = [];
  let next = `${base}/users/ME/conversations?view=${encodeURIComponent(VIEW)}&pageSize=200`;
  for (let page = 0; next && page < 10; page++) {
    const j = await getJson(next, H);
    const list = j.conversations ?? [];
    out.push(...list.filter(c => ONE_ON_ONE_RE.test(c.id)));
    const link = j._metadata?.backwardLink;
    next = list.length && link !== next ? link : null;
  }
  return out.sort((a, b) => (timeOf(b.lastMessage ?? {}) || 0) - (timeOf(a.lastMessage ?? {}) || 0));
}

async function fetchAll() {
  const all = new Map();
  let next = `${messagesUrl(chat.id)}?view=${encodeURIComponent(VIEW)}&pageSize=200&startTime=1`;
  while (next) {
    const j = await getJson(next, H);
    const page = j.messages ?? [];
    page.forEach(m => all.set(m.id, m));
    process.stdout.write(`\r${all.size} messages…`);
    // Pages come newest-changed first, so an old message with a fresh reaction can sit among new ones:
    // only a page changed entirely before `since` means everything after it was already seen.
    const oldest = Math.min(...page.map(m => +m.version || timeOf(m) || Infinity));
    const link = j._metadata?.backwardLink;
    next = page.length && !(since && oldest < since) && link !== next ? link : null;
  }
  process.stdout.write('\n');
  return [...all.values()];
}

const prepared = (await fetchAll())
  .filter(m => /^(Text|RichText)/.test(m.messagetype) && !m.properties?.deletetime)
  .map(prepare)
  .filter(p => p.at > 0 && p.at * 1000 >= since)
  .sort((a, b) => a.at - b.at || Number(a.id) - Number(b.id));

const names = new Map([[peer, peerDisplay]]);
for (const p of prepared) if (p.from && p.name) names.set(p.from, p.name);

const photosDir = path.join(outDir, 'photos');
fs.mkdirSync(photosDir, { recursive: true });
const existing = new Map(fs.readdirSync(photosDir).map(n => [n.replace(/\.[^.]+$/, ''), n]));

const objectsHost = auth.objects && new URL(auth.objects.url).host;
const headerSets = [
  auth.objects?.headers,
  auth.skype && { authorization: `skype_token ${auth.skype}` },
  H,
].filter(Boolean);

// The view a message links (imgo) is an 800px preview; the full-size views are tried first.
const FULL_VIEWS = ['imgpsh_fullsize_anim', 'imgpsh_fullsize'];

async function downloadImage(src) {
  const hosts = [src];
  if (objectsHost && /\/v1\/objects\//.test(src) && new URL(src).host !== objectsHost) {
    const u = new URL(src);
    u.host = objectsHost;
    hosts.push(u.href);
  }
  const urls = [...new Set(hosts.flatMap(h => [
    ...FULL_VIEWS.map(v => h.replace(/\/views\/[^/?#]+/, `/views/${v}`)),
    h,
  ]))];
  let status = 0;
  for (const url of urls) {
    for (const [i, headers] of headerSets.entries()) {
      try {
        const r = await get(url, headers);
        status = r.status;
        const type = (r.headers.get('content-type') || '').split(';')[0];
        if (!r.ok || !type.startsWith('image/')) continue;
        if (i > 0) headerSets.unshift(...headerSets.splice(i, 1));
        return { type, bytes: Buffer.from(await r.arrayBuffer()) };
      } catch {}
    }
  }
  return { status };
}

// Telegram's own ids for the two sides: frenfoil files everything not from the peer as ours.
// Ids derive only from Teams' own ids and an image's position in its message, so every run of the
// same chat yields the same ids and frenfoil skips what it already imported.
const chatId = String(fnv(chat.id));
const fromId = guid => (guid === peer ? `user${chatId}` : 'user0');
const fromName = guid => names.get(guid) || (guid === peer ? peerDisplay : 'Me');
const dated = at => ({ date: new Date(at * 1000).toISOString().slice(0, 19), date_unixtime: String(at) });

// Merged over what an earlier run saved: nothing is dropped, and a photo saved then is not lost to a
// download that failed now.
const merged = new Map((previous?.messages ?? []).map(m => [String(m.id), m]));
let added = 0;
let updated = 0;
for (const m of messages) {
  const old = merged.get(m.id);
  if (old && old.photo && !m.photo) continue;
  if (!old) added++;
  else if (JSON.stringify(old) !== JSON.stringify(m)) updated++;
  merged.set(m.id, m);
}
const all = [...merged.values()]
  .sort((a, b) => +a.date_unixtime - +b.date_unixtime || Number(a.id) - Number(b.id));

if (previous) fs.copyFileSync(resultPath, `${resultPath}.bak`);
fs.writeFileSync(`${resultPath}.tmp`, JSON.stringify({
  name: peerDisplay,
  type: 'personal_chat',
  id: chatId,
  teams_conversation_id: chat.id,
  messages: all,
}, null, 1));
fs.renameSync(`${resultPath}.tmp`, resultPath);

rl.close();
process.exit(0);

1	// Export a Teams 1:1 chat as a Telegram Desktop JSON export (result.json + photos/), for frenfoil's chat import.
2	//
3	// Restarts the new Teams desktop app with its WebView2 debug port open, borrows the auth headers Teams sends
4	// while it starts up, then fetches the chat and its images directly.
5	//
6	// Run again into a folder that holds an export to extend it: the chat is picked from its result.json, only
7	// the last week before its newest message onward is fetched (--full for everything), and what Teams no
8	// longer has is kept.
9	//
10	// node teams-export.mjs <outDir> [--with <name>] [--since YYYY-MM-DD] [--full] [--yes] [--no-restart] [--port 9222]
11
12	import { spawn } from 'node:child_process';
13	import { createInterface } from 'node:readline/promises';
14	import * as fs from 'node:fs';
15	import * as path from 'node:path';
16
17	const args = process.argv.slice(2);
18	const flag = name => { const i = args.indexOf(`--${name}`); return i >= 0 ? (args.splice(i, 1), true) : false; };
19	const opt = name => { const i = args.indexOf(`--${name}`); return i >= 0 ? args.splice(i, 2)[1] : undefined; };
20	const port = opt('port') ?? '9222';
21	const sinceArg = opt('since');
22	const withName = opt('with');
23	const yes = flag('yes');
24	const full = flag('full');
25	const noRestart = flag('no-restart');
26	const outDir = args[0];
27	if (!outDir) {
28	console.log('usage: node teams-export.mjs <outDir> [--with <name>] [--since YYYY-MM-DD] [--full] [--yes] [--no-restart] [--port 9222]');
29	process.exit(1);
30	}
31	let since = sinceArg ? Date.parse(sinceArg) : 0;
32	if (isNaN(since)) throw new Error(`Invalid --since date: ${sinceArg}`);
33
34	const resultPath = path.join(outDir, 'result.json');
35	const previous = fs.existsSync(resultPath) ? JSON.parse(fs.readFileSync(resultPath, 'utf8')) : null;
36	if (previous && (previous.type !== 'personal_chat' \|\| !Array.isArray(previous.messages)))
37	throw new Error(`${resultPath} is not an export this tool wrote.`);
38
39	// Edits and reactions land on recent messages, so an extension re-reads the week before its newest one.
40	const OVERLAP_DAYS = 7;
41	if (previous && !sinceArg && !full) {
42	const newest = previous.messages.reduce((n, m) => Math.max(n, +m.date_unixtime \|\| 0), 0);
43	since = Math.max(0, newest - OVERLAP_DAYS * 86400) * 1000;
44	}
45
46	const PARALLEL = 4;
47	const ONE_ON_ONE_RE = /^19:([0-9a-f-]{36})_([0-9a-f-]{36})@unq\.gbl\.spaces$/i;
48	const EXT = { 'image/png': 'png', 'image/jpeg': 'jpg', 'image/gif': 'gif', 'image/webp': 'webp', 'image/bmp': 'bmp' };
49	const VIEW = 'msnp24Equivalent\|supportsMessageProperties';
50
51	const sleep = ms => new Promise(r => setTimeout(r, ms));
52	const rl = createInterface({ input: process.stdin, output: process.stdout });
53	const ask = async q => (yes ? '' : (await rl.question(q)).trim());
54
55	// --- borrowing Teams' auth --------------------------------------------------------------------------
56
57	function restartTeams() {
58	console.log('Restarting Teams to borrow its login…');
59	const env = { ...process.env, WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS: `--remote-debugging-port=${port}` };
60	spawn('powershell.exe', ['-NoProfile', '-Command',
61	'Get-Process ms-teams -ErrorAction SilentlyContinue \| Stop-Process -Force; Start-Sleep 3; Start-Process ms-teams.exe'],
62	{ env, stdio: 'ignore' });
63	}
64
65	const cleanHeaders = h => Object.fromEntries(Object.entries(h)
66	.map(([k, v]) => [k.toLowerCase(), v])
67	.filter(([k]) => !/^(:\|host$\|cookie$\|content-length$\|content-type$\|connection$\|accept-encoding$)/.test(k)));
68
69	// Attach to every Teams page and worker as it is created, and keep the headers of its first chat-service
70	// and image requests.
71	async function captureAuth() {
72	const got = {};
73	const deadline = Date.now() + 180_000;
74	const done = () => got.chatsvc && (got.objects \|\| Date.now() > got.chatsvcAt + 20_000);
75
76	while (!done()) {
77	if (Date.now() > deadline) throw new Error('Did not see Teams sign in within 3 minutes.');
78	let ws;
79	try {
80	const ver = await (await fetch(`http://localhost:${port}/json/version`)).json();
81	ws = new WebSocket(ver.webSocketDebuggerUrl);
82	await new Promise((res, rej) => { ws.onopen = res; ws.onerror = rej; });
83	} catch {
84	await sleep(300);
85	continue;
86	}
87
88	let n = 0;
89	const urls = new Map();
90	const send = (method, params, sessionId) =>
91	ws.send(JSON.stringify({ id: ++n, method, params: params ?? {}, ...sessionId ? { sessionId } : {} }));
92	const see = (url, headers) => {
93	if (!url) return;
94	const h = cleanHeaders(headers);
95	if (h['x-skypetoken']) got.skype ??= h['x-skypetoken'];
96	if (!h.authorization) return;
97	if (!got.chatsvc && /\/api\/chatsvc\/[^/]+\/v1\//.test(url)) {
98	got.chatsvc = { url, headers: h };
99	got.chatsvcAt = Date.now();
100	} else if (!got.objects && /\/v1\/objects\//.test(url)) {
101	got.objects = { url, headers: h };
102	}
103	};
104
105	ws.onmessage = e => {
106	const m = JSON.parse(e.data);
107	if (m.method === 'Target.attachedToTarget') {
108	const s = m.params.sessionId;
109	send('Network.enable', {}, s);
110	send('Target.setAutoAttach', { autoAttach: true, waitForDebuggerOnStart: true, flatten: true }, s);
111	send('Runtime.runIfWaitingForDebugger', {}, s);
112	} else if (m.method === 'Network.requestWillBeSent') {
113	urls.set(m.params.requestId, m.params.request.url);
114	see(m.params.request.url, m.params.request.headers);
115	} else if (m.method === 'Network.requestWillBeSentExtraInfo') {
116	see(urls.get(m.params.requestId), m.params.headers);
117	}
118	};
119	send('Target.setAutoAttach', { autoAttach: true, waitForDebuggerOnStart: true, flatten: true });
120
121	await new Promise(res => {
122	ws.onclose = res;
123	const t = setInterval(() => { if (done() \|\| Date.now() > deadline) { clearInterval(t); res(); } }, 500);
124	});
125	try { ws.close(); } catch {}
126	}
127	return got;
128	}
129
130	// --- HTTP ------------------------------------------------------------------------------------------
131
132	async function get(url, headers) {
133	for (let i = 0; ; i++) {
134	let r;
135	try {
136	r = await fetch(url, { headers });
137	} catch (e) {
138	if (i >= 5) throw e;
139	await sleep(2 ** i * 1000);
140	continue;
141	}
142	if ((r.status === 429 \|\| r.status >= 500) && i < 5) {
143	await sleep((+r.headers.get('retry-after') \|\| 2 ** i) * 1000);
144	continue;
145	}
146	return r;
147	}
148	}
149
150	async function getJson(url, headers) {
151	const r = await get(url, headers);
152	if (r.status === 401) throw new Error('Teams login expired. Run again (it restarts Teams for a fresh one).');
153	if (!r.ok) throw new Error(`HTTP ${r.status} for ${url.split('?')[0]}`);
154	return r.json();
155	}
156
157	// --- Teams message → Telegram message -------------------------------------------------------------
158
159	const decode = s => s
160	.replace(/ /g, ' ')
161	.replace(/</g, '<')
162	.replace(/>/g, '>')
163	.replace(/"/g, '"')
164	.replace(/'\|'/g, "'")
165	.replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(+n))
166	.replace(/&/g, '&');
167
168	const toText = html => decode((html ?? '')
169	.replace(/<br\s*\/?>\|<\/(p\|div\|li)>/gi, '\n')
170	.replace(/<[^>]+>/g, ''))
171	.replace(/\n{3,}/g, '\n\n')
172	.trim();
173
174	const esc = s => s.replace(/[&<>"]/g, c => ({ '&': '&', '<': '<', '>': '>', '"': '"' }[c]));
175
176	const fnv = s => { let h = 0x811c9dc5; for (const c of s) h = Math.imul(h ^ c.codePointAt(0), 0x01000193) >>> 0; return h; };
177	const imageId = src => (/\/objects\/([^/?#]+)/.exec(src)?.[1] \|\| 'x' + fnv(src).toString(36)).replace(/[^\w.-]/g, '_');
178	const guidOf = mri => /8:orgid:([0-9a-f-]{36})/i.exec(mri ?? '')?.[1]?.toLowerCase();
179	const jsonProp = v => { try { return typeof v === 'string' ? JSON.parse(v) : v; } catch { return null; } };
180	const timeOf = m => Date.parse(m.composetime \|\| m.originalarrivaltime);
181
182	const EMOTION = { like: '👍', heart: '❤️', laugh: '😆', surprised: '😮', sad: '😢', angry: '😡' };
183	function emojiOf(key) {
184	if (EMOTION[key]) return EMOTION[key];
185	const hex = /^([0-9a-f]{4,6}(?:-[0-9a-f]{4,6})*)_/i.exec(key)?.[1];
186	return hex ? String.fromCodePoint(...hex.split('-').map(h => parseInt(h, 16))) : null;
187	}
188
189	function prepare(m) {
190	let content = m.content ?? '';
191	let replyTo;
192	content = content.replace(/<blockquote\b([^>])>[\s\S]?<\/blockquote>/gi, (all, attrs) => {
193	if (!/schema\.skype\.com\/Reply/i.test(attrs)) return all;
194	replyTo = /itemid="(\d+)"/i.exec(attrs)?.[1];
195	return '';
196	});
197	const srcs = [];
198	content = content.replace(/<img\b[^>]*>/gi, tag => {
199	const attr = k => (new RegExp(`\\s${k}="([^"]*)"`, 'i').exec(tag) \|\| [])[1] \|\| '';
200	if (/emoji/i.test(attr('itemtype'))) return attr('alt');
201	const src = decode(attr('src'));
202	if (/^https:/.test(src)) srcs.push(src);
203	return '';
204	});
205	content = content.replace(/<a\b[^>]?href="([^"])"[^>]>([\s\S]?)<\/a>/gi, (_, href, label) => {
206	const h = decode(href);
207	const l = toText(label);
208	return esc(!l \|\| l === h ? h : `${l} (${h})`);
209	});
210
211	const files = (jsonProp(m.properties?.files) ?? []).map(f => {
212	const name = f.fileName \|\| f.title \|\| 'file';
213	const url = f.objectUrl \|\| f.fileInfo?.shareUrl \|\| '';
214	return `📎 ${name}${url ? ` (${url})` : ''}`;
215	});
216
217	const reactions = (jsonProp(m.properties?.emotions) ?? []).flatMap(e => {
218	const emoji = emojiOf(e.key);
219	const users = (e.users ?? []).map(u => guidOf(u.mri)).filter(Boolean);
220	return emoji && users.length ? [{ emoji, users }] : [];
221	});
222
223	const id = /^\d{1,17}$/.test(m.id) ? m.id : String(fnv(m.id));
224	const edit = m.properties?.edittime && Math.floor(+m.properties.edittime / 1000);
225
226	return {
227	id,
228	at: Math.floor(timeOf(m) / 1000),
229	edited: edit > 0 ? edit : undefined,
230	from: guidOf(m.from),
231	name: m.imdisplayname \|\| '',
232	text: [toText(content), ...files].filter(Boolean).join('\n'),
233	srcs,
234	replyTo,
235	reactions,
236	};
237	}
238
239	// The signed-in user's AAD object id, read from whichever JWT the borrowed headers carry.
240	function selfGuid(...headerSets) {
241	for (const headers of headerSets) {
242	for (const v of Object.values(headers ?? {})) {
243	for (const t of String(v).match(/[\w-]{10,}\.[\w-]{10,}\.[\w-]+/g) ?? []) {
244	try {
245	const c = JSON.parse(Buffer.from(t.split('.')[1], 'base64url').toString());
246	const id = c.oid \|\| (typeof c.skypeid === 'string' && c.skypeid.split(':').pop());
247	if (id) return String(id).toLowerCase();
248	} catch {}
249	}
250	}
251	}
252	}
253
254	// --- main ------------------------------------------------------------------------------------------
255
256	if (!noRestart) restartTeams();
257	else console.log('Waiting for Teams to sign in… (restart it with the debug port if nothing happens)');
258	const auth = await captureAuth();
259	console.log('Got Teams login.');
260
261	const H = auth.chatsvc.headers;
262	const base = /^(.*\/api\/chatsvc\/[^/]+\/v1)\//.exec(auth.chatsvc.url)[1];
263	const self = selfGuid(H, auth.objects?.headers, auth.skype && { t: auth.skype });
264	if (!self) throw new Error('Could not tell which account is signed in.');
265
266	const messagesUrl = conv => `${base}/users/ME/conversations/${encodeURIComponent(conv)}/messages`;
267	const peerOf = conv => { const [, a, b] = ONE_ON_ONE_RE.exec(conv); return a.toLowerCase() === self ? b.toLowerCase() : a.toLowerCase(); };
268
269	async function oneOnOnes() {
270	const out = [];
271	let next = `${base}/users/ME/conversations?view=${encodeURIComponent(VIEW)}&pageSize=200`;
272	for (let page = 0; next && page < 10; page++) {
273	const j = await getJson(next, H);
274	const list = j.conversations ?? [];
275	out.push(...list.filter(c => ONE_ON_ONE_RE.test(c.id)));
276	const link = j._metadata?.backwardLink;
277	next = list.length && link !== next ? link : null;
278	}
279	return out.sort((a, b) => (timeOf(b.lastMessage ?? {}) \|\| 0) - (timeOf(a.lastMessage ?? {}) \|\| 0));
280	}
281
282	const peerNames = new Map();
283	async function peerName(c) {
284	const peer = peerOf(c.id);
285	if (peerNames.has(peer)) return peerNames.get(peer);
286	let name = guidOf(c.lastMessage?.from) === peer ? c.lastMessage.imdisplayname : '';
287	if (!name) {
288	const j = await getJson(`${messagesUrl(c.id)}?view=${encodeURIComponent(VIEW)}&pageSize=100&startTime=1`, H);
289	name = (j.messages ?? []).find(m => guidOf(m.from) === peer && m.imdisplayname)?.imdisplayname ?? '';
290	}
291	peerNames.set(peer, name);
292	return name;
293	}
294
295	const chats = await oneOnOnes();
296	let chat;
297	if (previous) {
298	chat = chats.find(c => c.id === previous.teams_conversation_id \|\| String(fnv(c.id)) === String(previous.id));
299	if (!chat) throw new Error(`The chat in ${resultPath} is no longer in Teams. The export there is left as it is.`);
300	if (withName && !(await peerName(chat)).toLowerCase().includes(withName.toLowerCase()))
301	throw new Error(`${resultPath} holds the chat with ${await peerName(chat)}, not "${withName}". Use another folder.`);
302	} else if (withName) {
303	for (const c of chats) {
304	if ((await peerName(c)).toLowerCase().includes(withName.toLowerCase())) { chat = c; break; }
305	}
306	if (!chat) throw new Error(`No 1:1 chat with "${withName}" among ${chats.length} chats.`);
307	} else {
308	const top = chats.slice(0, 20);
309	for (const [i, c] of top.entries()) console.log(`${String(i + 1).padStart(2)}. ${await peerName(c) \|\| '(unknown)'}`);
310	if (yes) throw new Error('Pass --with <name> together with --yes.');
311	chat = top[+(await ask('Which chat? ')) - 1];
312	if (!chat) throw new Error('No such chat.');
313	}
314	const peer = peerOf(chat.id);
315	const peerDisplay = await peerName(chat) \|\| 'Teams chat';
316	console.log(`Chat with ${peerDisplay}`);
317	if (previous && since)
318	console.log(`Extending the export in ${path.resolve(outDir)} (${previous.messages.length} messages), from ${new Date(since).toISOString().slice(0, 10)}.`);
319
320	async function fetchAll() {
321	const all = new Map();
322	let next = `${messagesUrl(chat.id)}?view=${encodeURIComponent(VIEW)}&pageSize=200&startTime=1`;
323	while (next) {
324	const j = await getJson(next, H);
325	const page = j.messages ?? [];
326	page.forEach(m => all.set(m.id, m));
327	process.stdout.write(`\r${all.size} messages…`);
328	// Pages come newest-changed first, so an old message with a fresh reaction can sit among new ones:
329	// only a page changed entirely before `since` means everything after it was already seen.
330	const oldest = Math.min(...page.map(m => +m.version \|\| timeOf(m) \|\| Infinity));
331	const link = j._metadata?.backwardLink;
332	next = page.length && !(since && oldest < since) && link !== next ? link : null;
333	}
334	process.stdout.write('\n');
335	return [...all.values()];
336	}
337
338	const prepared = (await fetchAll())
339	.filter(m => /^(Text\|RichText)/.test(m.messagetype) && !m.properties?.deletetime)
340	.map(prepare)
341	.filter(p => p.at > 0 && p.at * 1000 >= since)
342	.sort((a, b) => a.at - b.at \|\| Number(a.id) - Number(b.id));
343
344	const names = new Map([[peer, peerDisplay]]);
345	for (const p of prepared) if (p.from && p.name) names.set(p.from, p.name);
346
347	const unique = [...new Set(prepared.flatMap(p => p.srcs))];
348	const span = prepared.length ? ` (${new Date(prepared[0].at * 1000).toISOString().slice(0, 10)} → ${new Date(prepared.at(-1).at * 1000).toISOString().slice(0, 10)})` : '';
349	console.log(`${prepared.length} messages${span}, ${unique.length} images.`);
350	if (/^n/i.test(await ask(`Export to ${path.resolve(outDir)}? [Y/n] `))) process.exit(0);
351
352	const photosDir = path.join(outDir, 'photos');
353	fs.mkdirSync(photosDir, { recursive: true });
354	const existing = new Map(fs.readdirSync(photosDir).map(n => [n.replace(/\.[^.]+$/, ''), n]));
355
356	const objectsHost = auth.objects && new URL(auth.objects.url).host;
357	const headerSets = [
358	auth.objects?.headers,
359	auth.skype && { authorization: `skype_token ${auth.skype}` },
360	H,
361	].filter(Boolean);
362
363	// The view a message links (imgo) is an 800px preview; the full-size views are tried first.
364	const FULL_VIEWS = ['imgpsh_fullsize_anim', 'imgpsh_fullsize'];
365
366	async function downloadImage(src) {
367	const hosts = [src];
368	if (objectsHost && /\/v1\/objects\//.test(src) && new URL(src).host !== objectsHost) {
369	const u = new URL(src);
370	u.host = objectsHost;
371	hosts.push(u.href);
372	}
373	const urls = [...new Set(hosts.flatMap(h => [
374	...FULL_VIEWS.map(v => h.replace(/\/views\/[^/?#]+/, `/views/${v}`)),
375	h,
376	]))];
377	let status = 0;
378	for (const url of urls) {
379	for (const [i, headers] of headerSets.entries()) {
380	try {
381	const r = await get(url, headers);
382	status = r.status;
383	const type = (r.headers.get('content-type') \|\| '').split(';')[0];
384	if (!r.ok \|\| !type.startsWith('image/')) continue;
385	if (i > 0) headerSets.unshift(...headerSets.splice(i, 1));
386	return { type, bytes: Buffer.from(await r.arrayBuffer()) };
387	} catch {}
388	}
389	}
390	return { status };
391	}
392
393	const saved = new Map();
394	const failures = new Map();
395	let done = 0;
396	let next = 0;
397	await Promise.all(Array.from({ length: PARALLEL }, async () => {
398	while (next < unique.length) {
399	const src = unique[next++];
400	const id = imageId(src);
401	let name = existing.get(id);
402	if (!name) {
403	const r = await downloadImage(src);
404	if (r.bytes) {
405	name = `${id}.${EXT[r.type] \|\| 'jpg'}`;
406	fs.writeFileSync(path.join(photosDir, name), r.bytes);
407	existing.set(id, name);
408	} else {
409	failures.set(r.status, (failures.get(r.status) ?? 0) + 1);
410	}
411	}
412	if (name) saved.set(src, `photos/${name}`);
413	process.stdout.write(`\rImages ${++done}/${unique.length}…`);
414	}
415	}));
416	if (unique.length) process.stdout.write('\n');
417
418	// Telegram's own ids for the two sides: frenfoil files everything not from the peer as ours.
419	// Ids derive only from Teams' own ids and an image's position in its message, so every run of the
420	// same chat yields the same ids and frenfoil skips what it already imported.
421	const chatId = String(fnv(chat.id));
422	const fromId = guid => (guid === peer ? `user${chatId}` : 'user0');
423	const fromName = guid => names.get(guid) \|\| (guid === peer ? peerDisplay : 'Me');
424	const dated = at => ({ date: new Date(at * 1000).toISOString().slice(0, 19), date_unixtime: String(at) });
425
426	const messages = prepared.flatMap(p => {
427	const photos = p.srcs.map(src => saved.get(src));
428	const missing = photos.filter(x => !x).length;
429	const sender = { from: fromName(p.from), from_id: fromId(p.from) };
430	const text = [p.text, missing ? `[${missing} image${missing > 1 ? 's' : ''} could not be downloaded]` : ''].filter(Boolean).join('\n');
431	const reactions = p.reactions.map(r => ({
432	type: 'emoji', count: r.users.length, emoji: r.emoji, recent: r.users.map(u => ({ from_id: fromId(u) })),
433	}));
434
435	const first = {
436	id: p.id,
437	type: 'message',
438	...dated(p.at),
439	...p.edited ? { edited_unixtime: String(p.edited) } : {},
440	...sender,
441	...p.replyTo ? { reply_to_message_id: p.replyTo } : {},
442	...reactions.length ? { reactions } : {},
443	text,
444	...photos[0] ? { photo: photos[0] } : {},
445	};
446	// Telegram carries one photo per message, so further images follow as their own messages.
447	const rest = photos.flatMap((photo, i) => (i > 0 && photo ? [{
448	id: `${p.id}${String(i).padStart(2, '0')}`, type: 'message', ...dated(p.at), ...sender, text: '', photo,
449	}] : []));
450	return [first, ...rest];
451	});
452
453	// Merged over what an earlier run saved: nothing is dropped, and a photo saved then is not lost to a
454	// download that failed now.
455	const merged = new Map((previous?.messages ?? []).map(m => [String(m.id), m]));
456	let added = 0;
457	let updated = 0;
458	for (const m of messages) {
459	const old = merged.get(m.id);
460	if (old && old.photo && !m.photo) continue;
461	if (!old) added++;
462	else if (JSON.stringify(old) !== JSON.stringify(m)) updated++;
463	merged.set(m.id, m);
464	}
465	const all = [...merged.values()]
466	.sort((a, b) => +a.date_unixtime - +b.date_unixtime \|\| Number(a.id) - Number(b.id));
467
468	if (previous) fs.copyFileSync(resultPath, `${resultPath}.bak`);
469	fs.writeFileSync(`${resultPath}.tmp`, JSON.stringify({
470	name: peerDisplay,
471	type: 'personal_chat',
472	id: chatId,
473	teams_conversation_id: chat.id,
474	messages: all,
475	}, null, 1));
476	fs.renameSync(`${resultPath}.tmp`, resultPath);
477
478	const failed = unique.length - saved.size;
479	console.log(`Done: ${added} new, ${updated} updated, ${all.length} in total; ${saved.size}/${unique.length} images → ${path.resolve(resultPath)}`);
480	if (failed)
481	console.log(`${failed} images failed (HTTP ${[...failures].map(([s, n]) => `${s \|\| 'error'}×${n}`).join(', ')}). Run again into the same folder to retry just those.`);
482
483	rl.close();
484	process.exit(0);
485