Revision of teams-export.mjs

8

+

// the last --backtrack-days (default 7) before its newest message onward are fetched (--full for everything), and what Teams no

9

+

// longer has is kept. The new messages and their photos also go to a folder beside it,

10

+

// <outDir>-new-<UTC timestamp>, as an export of their own.

12

+

// node teams-export.mjs <outDir> [--with <name>] [--since YYYY-MM-DD] [--backtrack-days <n>] [--full] [--yes] [--no-restart] [--port 9222]

24

+

const backtrackArg = opt('backtrack-days');

31

+

console.log('usage: node teams-export.mjs <outDir> [--with <name>] [--since YYYY-MM-DD] [--backtrack-days <n>] [--full] [--yes] [--no-restart] [--port 9222]');

36

+

const backtrackDays = +(backtrackArg ?? 7);

37

+

if (!(backtrackDays >= 0)) throw new Error(`Invalid --backtrack-days: ${backtrackArg}`);

49

+

// Edits and reactions land on recent messages, so an extension re-reads the days before its newest one.

52

+

since = Math.max(0, newest - backtrackDays * 86_400_000);

469

+

const added = [];

474

+

if (!old) added.push(m);

480

+

const exportJson = msgs => JSON.stringify({

484

+

chats: [{ id: chat.id, kind: 'direct', name: peerDisplay, messages: msgs }],

485

+

}, null, 1);

486

+

487

+

if (previous) fs.copyFileSync(outPath, `${outPath}.bak`);

488

+

fs.writeFileSync(`${outPath}.tmp`, exportJson(all));

492

+

console.log(`Done: ${added.length} new, ${updated} updated, ${all.length} in total; ${saved.size}/${unique.length} images → ${path.resolve(outPath)}`);

493

+

494

+

// An extension also gets a folder beside the export with only its new messages and their photos.

495

+

if (previous && added.length) {

496

+

const stamp = new Date().toISOString().slice(0, 16).replace('T', '_').replace(':', '');

497

+

const deltaDir = `${path.resolve(outDir)}-new-${stamp}`;

498

+

fs.mkdirSync(path.join(deltaDir, 'photos'), { recursive: true });

499

+

const photos = new Set(added.flatMap(m => (m.files ?? []).map(f => f.path).filter(Boolean)));

500

+

for (const p of photos) fs.copyFileSync(path.join(outDir, p), path.join(deltaDir, p));

501

+

fs.writeFileSync(path.join(deltaDir, 'chats.json'), exportJson(added));

502

+

console.log(`New only: ${added.length} messages, ${photos.size} images → ${deltaDir}`);

503

+

}

			@@ -5,10 +5,11 @@
5	5		// while it starts up, then fetches the chat and its images directly.
6	6		//
7	7		// Run again into a folder that holds an export to extend it: the chat is picked from its chats.json, only
8		-	// the last week before its newest message onward is fetched (--full for everything), and what Teams no
9		-	// longer has is kept.
	8	+	// the last --backtrack-days (default 7) before its newest message onward are fetched (--full for everything), and what Teams no
	9	+	// longer has is kept. The new messages and their photos also go to a folder beside it,
	10	+	// <outDir>-new-<UTC timestamp>, as an export of their own.
10	11		//
11		-	// node teams-export.mjs <outDir> [--with <name>] [--since YYYY-MM-DD] [--full] [--yes] [--no-restart] [--port 9222]
	12	+	// node teams-export.mjs <outDir> [--with <name>] [--since YYYY-MM-DD] [--backtrack-days <n>] [--full] [--yes] [--no-restart] [--port 9222]
12	13
13	14		import { spawn } from 'node:child_process';
14	15		import { createInterface } from 'node:readline/promises';
			@@ -20,17 +21,20 @@ const flag = name => { const i = args.indexOf(`--${name}`); return i >= 0 ? (arg
20	21		const opt = name => { const i = args.indexOf(`--${name}`); return i >= 0 ? args.splice(i, 2)[1] : undefined; };
21	22		const port = opt('port') ?? '9222';
22	23		const sinceArg = opt('since');
	24	+	const backtrackArg = opt('backtrack-days');
23	25		const withName = opt('with');
24	26		const yes = flag('yes');
25	27		const full = flag('full');
26	28		const noRestart = flag('no-restart');
27	29		const outDir = args[0];
28	30		if (!outDir) {
29		-	console.log('usage: node teams-export.mjs <outDir> [--with <name>] [--since YYYY-MM-DD] [--full] [--yes] [--no-restart] [--port 9222]');
	31	+	console.log('usage: node teams-export.mjs <outDir> [--with <name>] [--since YYYY-MM-DD] [--backtrack-days <n>] [--full] [--yes] [--no-restart] [--port 9222]');
30	32		process.exit(1);
31	33		}
32	34		let since = sinceArg ? Date.parse(sinceArg) : 0;
33	35		if (isNaN(since)) throw new Error(`Invalid --since date: ${sinceArg}`);
	36	+	const backtrackDays = +(backtrackArg ?? 7);
	37	+	if (!(backtrackDays >= 0)) throw new Error(`Invalid --backtrack-days: ${backtrackArg}`);
34	38
35	39		const FORMAT = 'frenfoil-chat-import';
36	40		// Frenfoil recognises a re-imported message by source, chat id and message id, so this never changes.
			@@ -42,11 +46,10 @@ const previousChat = previous?.chats?.[0];
42	46		if (previous && (previous.format !== FORMAT \|\| previous.source !== SOURCE \|\| !Array.isArray(previousChat?.messages)))
43	47		throw new Error(`${outPath} is not an export this tool wrote.`);
44	48
45		-	// Edits and reactions land on recent messages, so an extension re-reads the week before its newest one.
46		-	const OVERLAP_DAYS = 7;
	49	+	// Edits and reactions land on recent messages, so an extension re-reads the days before its newest one.
47	50		if (previous && !sinceArg && !full) {
48	51		const newest = previousChat.messages.reduce((n, m) => Math.max(n, +m.time \|\| 0), 0);
49		-	since = Math.max(0, newest - OVERLAP_DAYS * 86_400_000);
	52	+	since = Math.max(0, newest - backtrackDays * 86_400_000);
50	53		}
51	54
52	55		const PARALLEL = 4;
			@@ -463,28 +466,41 @@ const messages = prepared.map(p => {
463	466		// download that failed now.
464	467		const pathed = m => (m.files ?? []).filter(f => f.path).length;
465	468		const merged = new Map((previousChat?.messages ?? []).map(m => [String(m.id), m]));
466		-	let added = 0;
	469	+	const added = [];
467	470		let updated = 0;
468	471		for (const m of messages) {
469	472		const old = merged.get(m.id);
470	473		if (old && pathed(old) > pathed(m)) continue;
471		-	if (!old) added++;
	474	+	if (!old) added.push(m);
472	475		else if (JSON.stringify(old) !== JSON.stringify(m)) updated++;
473	476		merged.set(m.id, m);
474	477		}
475	478		const all = [...merged.values()].sort((a, b) => a.time - b.time \|\| Number(a.id) - Number(b.id));
476	479
477		-	if (previous) fs.copyFileSync(outPath, `${outPath}.bak`);
478		-	fs.writeFileSync(`${outPath}.tmp`, JSON.stringify({
	480	+	const exportJson = msgs => JSON.stringify({
479	481		format: FORMAT,
480	482		version: 1,
481	483		source: SOURCE,
482		-	chats: [{ id: chat.id, kind: 'direct', name: peerDisplay, messages: all }],
483		-	}, null, 1));
	484	+	chats: [{ id: chat.id, kind: 'direct', name: peerDisplay, messages: msgs }],
	485	+	}, null, 1);
	486	+
	487	+	if (previous) fs.copyFileSync(outPath, `${outPath}.bak`);
	488	+	fs.writeFileSync(`${outPath}.tmp`, exportJson(all));
484	489		fs.renameSync(`${outPath}.tmp`, outPath);
485	490
486	491		const failed = unique.length - saved.size;
487		-	console.log(`Done: ${added} new, ${updated} updated, ${all.length} in total; ${saved.size}/${unique.length} images → ${path.resolve(outPath)}`);
	492	+	console.log(`Done: ${added.length} new, ${updated} updated, ${all.length} in total; ${saved.size}/${unique.length} images → ${path.resolve(outPath)}`);
	493	+
	494	+	// An extension also gets a folder beside the export with only its new messages and their photos.
	495	+	if (previous && added.length) {
	496	+	const stamp = new Date().toISOString().slice(0, 16).replace('T', '_').replace(':', '');
	497	+	const deltaDir = `${path.resolve(outDir)}-new-${stamp}`;
	498	+	fs.mkdirSync(path.join(deltaDir, 'photos'), { recursive: true });
	499	+	const photos = new Set(added.flatMap(m => (m.files ?? []).map(f => f.path).filter(Boolean)));
	500	+	for (const p of photos) fs.copyFileSync(path.join(outDir, p), path.join(deltaDir, p));
	501	+	fs.writeFileSync(path.join(deltaDir, 'chats.json'), exportJson(added));
	502	+	console.log(`New only: ${added.length} messages, ${photos.size} images → ${deltaDir}`);
	503	+	}
488	504		if (failed)
489	505		console.log(`${failed} images failed (HTTP ${[...failures].map(([s, n]) => `${s \|\| 'error'}×${n}`).join(', ')}). Run again with --full into the same folder to retry them.`);
490	506

josefandersson / teams-export.mjs

josefandersson revised this gist 2 weeks ago · f639d5d

josefandersson revised this gist 3 weeks ago · 3a898ee

josefandersson revised this gist 3 weeks ago · 6e989e8