iwa-inspector/src/iwa.ts

import * as CFB from 'cfb';

import Messages, { $_TSP_MessageInfo, MessageTypes } from "./messages/";

/* see https://bugs.webkit.org/show_bug.cgi?id=243148 -- affects iOS Safari */
declare var Buffer: any; // Buffer is typeof-guarded but TS still needs this :(
var subarray: "subarray" | "slice" = (() => {
	try {
		if(typeof Uint8Array == "undefined") return "slice";
		if(typeof Uint8Array.prototype.subarray == "undefined") return "slice";
		// NOTE: feature tests are for node < 6.x
		if(typeof Buffer !== "undefined") {
			if(typeof Buffer.prototype.subarray == "undefined") return "slice";
			if((typeof Buffer.from == "function" ? Buffer.from([72,62]) : new Buffer([72,62])) instanceof Uint8Array) return "subarray";
			return "slice";
		}
		return "subarray";
	} catch(e) { return "slice"; }
})();

/** Concatenate Uint8Arrays */
function u8concat(u8a: Uint8Array[]): Uint8Array {
	var len = 0;
	for(var i = 0; i < u8a.length; ++i) len += u8a[i].length;
	var out = new Uint8Array(len);
	var off = 0;
	for(i = 0; i < u8a.length; ++i) {
		var u8 = u8a[i], L = u8.length;
		if(L < 250) { for(var j = 0; j < L; ++j) out[off++] = u8[j]; }
		else { out.set(u8, off); off += L; }
	}
	return out;
}

interface Ptr { l: number; }

/** Parse an integer from the varint that can be exactly stored in a double */
function parse_varint49(buf: Uint8Array, ptr: Ptr): number {
	var l = ptr.l;
	var usz = buf[l] & 0x7F;
	varint: if(buf[l++] >= 0x80) {
		usz |= (buf[l] & 0x7F) <<  7; if(buf[l++] < 0x80) break varint;
		usz |= (buf[l] & 0x7F) << 14; if(buf[l++] < 0x80) break varint;
		usz |= (buf[l] & 0x7F) << 21; if(buf[l++] < 0x80) break varint;
		usz += (buf[l] & 0x7F) * Math.pow(2, 28); ++l; if(buf[l++] < 0x80) break varint;
		usz += (buf[l] & 0x7F) * Math.pow(2, 35); ++l; if(buf[l++] < 0x80) break varint;
		usz += (buf[l] & 0x7F) * Math.pow(2, 42); ++l; if(buf[l++] < 0x80) break varint;
	}
	ptr.l = l;
	return usz;
}
/** Parse a repeated varint [packed = true] field */
function parse_packed_varints(buf: Uint8Array): number[] {
	var ptr: Ptr = {l: 0};
	var out: number[] = [];
	while(ptr.l < buf.length) out.push(parse_varint49(buf, ptr));
	return out;
}
/** Parse a BigInt from the varint */
function parse_varint64(buf: Uint8Array, ptr: Ptr): BigInt {
	var l = ptr.l;
	var usz = BigInt(buf[l] & 0x7F);
	varint: if(buf[l++] >= 0x80) {
		usz += BigInt(buf[l] & 0x7F) << 7n; if(buf[l++] < 0x80) break varint;
		usz += BigInt(buf[l] & 0x7F) << 14n; if(buf[l++] < 0x80) break varint;
		usz += BigInt(buf[l] & 0x7F) << 21n; if(buf[l++] < 0x80) break varint;
		usz += BigInt(buf[l] & 0x7F) << 28n; ++l; if(buf[l++] < 0x80) break varint;
		usz += BigInt(buf[l] & 0x7F) << 35n; ++l; if(buf[l++] < 0x80) break varint;
		usz += BigInt(buf[l] & 0x7F) << 42n; ++l; if(buf[l++] < 0x80) break varint;
		usz += BigInt(buf[l] & 0x7F) << 49n; ++l; if(buf[l++] < 0x80) break varint;
		usz += BigInt(buf[l] & 0x7F) << 56n; ++l; if(buf[l++] < 0x80) break varint;
		usz += BigInt(buf[l] & 0x7F) << 63n; ++l; if(buf[l++] < 0x80) break varint;
	}
	ptr.l = l;
	return usz;
}
/** Parse a repeated varint [packed = true] field */
function parse_packed_varint64(buf: Uint8Array): BigInt[] {
	var ptr: Ptr = {l: 0};
	var out: BigInt[] = [];
	while(ptr.l < buf.length) out.push(parse_varint64(buf, ptr));
	return out;
}
/** Parse a 32-bit signed integer from the raw varint */
function varint_to_i32(buf: Uint8Array): number {
	var l = 0,
	i32  = (buf[l] & 0x7F)      ; if(buf[l++] < 0x80) return i32;
	i32 |= (buf[l] & 0x7F) <<  7; if(buf[l++] < 0x80) return i32;
	i32 |= (buf[l] & 0x7F) << 14; if(buf[l++] < 0x80) return i32;
	i32 |= (buf[l] & 0x7F) << 21; if(buf[l++] < 0x80) return i32;
	i32 |= (buf[l] & 0x0F) << 28;                     return i32;
}
/** Parse a 64-bit unsigned integer as a pair */
function varint_to_u64(buf: Uint8Array): [number, number] {
	var l = 0, lo = buf[l] & 0x7F, hi = 0;
	varint: if(buf[l++] >= 0x80) {
		lo |= (buf[l] & 0x7F) <<  7; if(buf[l++] < 0x80) break varint;
		lo |= (buf[l] & 0x7F) << 14; if(buf[l++] < 0x80) break varint;
		lo |= (buf[l] & 0x7F) << 21; if(buf[l++] < 0x80) break varint;
		lo |= (buf[l] & 0x7F) << 28; hi = (buf[l] >> 4) & 0x07; if(buf[l++] < 0x80) break varint;
		hi |= (buf[l] & 0x7F) <<  3; if(buf[l++] < 0x80) break varint;
		hi |= (buf[l] & 0x7F) << 10; if(buf[l++] < 0x80) break varint;
		hi |= (buf[l] & 0x7F) << 17; if(buf[l++] < 0x80) break varint;
		hi |= (buf[l] & 0x7F) << 24; if(buf[l++] < 0x80) break varint;
		hi |= (buf[l] & 0x7F) << 31;
	}
	return [lo >>> 0, hi >>> 0];
}
export { varint_to_i32, varint_to_u64 };

interface ProtoItem {
	data: Uint8Array;
	type: number;
}
type ProtoField = Array<ProtoItem>;
type ProtoMessage = Array<ProtoField>;
interface IWAMessage {
	/** Metadata in .TSP.MessageInfo */
	meta: ProtoMessage;
	rawmeta: Uint8Array;
	data: Uint8Array;
	pre?: string;
	parsed?: any;
	parsedmeta?: $_TSP_MessageInfo;
}
interface IWAArchiveInfo {
	id: number;
	merge?: boolean;
	messages: IWAMessage[];
}
/** Shallow parse of a Protobuf message */
function parse_shallow(buf: Uint8Array): ProtoMessage {
	var out: ProtoMessage = [], ptr: Ptr = {l: 0};
	while(ptr.l < buf.length) {
		var off = ptr.l;
		var num = parse_varint49(buf, ptr);
		var type = num & 0x07; num = (num / 8)|0;
		var data: Uint8Array;
		var l = ptr.l;
		switch(type) {
			case 0: {
				while(buf[l++] >= 0x80);
				data = buf[subarray](ptr.l, l);
				ptr.l = l;
			} break;
			case 1: { data = buf[subarray](l, l + 8); ptr.l = l + 8; } break;
			case 2: {
				var len = parse_varint49(buf, ptr);
				data = buf[subarray](ptr.l, ptr.l + len);
				ptr.l += len;
			} break;
			case 5: { data = buf[subarray](l, l + 4); ptr.l = l + 4; } break;
			default: throw new Error(`PB Type ${type} for Field ${num} at offset ${off}`);
		}
		var v: ProtoItem = { data, type };
		if(out[num] == null) out[num] = [];
		out[num].push(v);
	}
	return out;
}

/** Extract all messages from a IWA file */
function parse_iwa_file(buf: Uint8Array): IWAArchiveInfo[] {
	var out: IWAArchiveInfo[] = [], ptr: Ptr = {l: 0};
	while(ptr.l < buf.length) {
		/* .TSP.ArchiveInfo */
		var len = parse_varint49(buf, ptr);
		var ai = parse_shallow(buf[subarray](ptr.l, ptr.l + len));
		ptr.l += len;

		var res: IWAArchiveInfo = {
			/* TODO: technically ID is optional */
			id: varint_to_i32(ai[1][0].data),
			messages: []
		};
		ai[2].forEach(b => {
			var mi = parse_shallow(b.data);
			var fl = varint_to_i32(mi[3][0].data);
			res.messages.push({
				meta: mi,
				rawmeta: b.data,
				data: buf[subarray](ptr.l, ptr.l + fl)
			});
			ptr.l += fl;
		});
		if(ai[3]?.[0]) res.merge = (varint_to_i32(ai[3][0].data) >>> 0) > 0;
		out.push(res);
	}
	return out;
}

/** Decompress a snappy chunk */
function parse_snappy_chunk(type: number, buf: Uint8Array): Uint8Array[] {
	if(type != 0) throw new Error(`Unexpected Snappy chunk type ${type}`);
	var ptr: Ptr = {l: 0};

	var usz = parse_varint49(buf, ptr);
	var chunks: Uint8Array[] = [];
	var l = ptr.l;
	while(l < buf.length) {
		var tag = buf[l] & 0x3;
		if(tag == 0) {
			var len = buf[l++] >> 2;
			if(len < 60) ++len;
			else {
				var c = len - 59;
				len = buf[l];
				if(c > 1) len |= (buf[l+1]<<8);
				if(c > 2) len |= (buf[l+2]<<16);
				if(c > 3) len |= (buf[l+3]<<24);
				len >>>=0; len++;
				l += c;
			}
			chunks.push(buf[subarray](l, l + len)); l += len; continue;
		} else {
			var offset = 0, length = 0;
			if(tag == 1) {
				length = ((buf[l] >> 2) & 0x7) + 4;
				offset = (buf[l++] & 0xE0) << 3;
				offset |= buf[l++];
			} else {
				length = (buf[l++] >> 2) + 1;
				if(tag == 2) { offset = buf[l] | (buf[l+1]<<8); l += 2; }
				else { offset = (buf[l] | (buf[l+1]<<8) | (buf[l+2]<<16) | (buf[l+3]<<24))>>>0; l += 4; }
			}
			if(offset == 0) throw new Error("Invalid offset 0");
			var j = chunks.length - 1, off = offset;
			while(j >=0 && off >= chunks[j].length) { off -= chunks[j].length; --j; }
			if(j < 0) {
				if(off == 0) off = chunks[(j = 0)].length;
				else throw new Error("Invalid offset beyond length");
			}
			// Node 0.8 Buffer slice does not support negative indices
			if(length < off) chunks.push(chunks[j][subarray](chunks[j].length-off, chunks[j].length-off + length));
			else {
				if(off > 0) { chunks.push(chunks[j][subarray](chunks[j].length-off)); length -= off; } ++j;
				while(length >= chunks[j].length) { chunks.push(chunks[j]); length -= chunks[j].length; ++j; }
				if(length) chunks.push(chunks[j][subarray](0, length));
			}
			if(chunks.length > 25) chunks = [u8concat(chunks)];
		}
	}
	var clen = 0; for(var u8i = 0; u8i < chunks.length; ++u8i) clen += chunks[u8i].length;
	if(clen != usz) throw new Error(`Unexpected length: ${clen} != ${usz}`);
	return chunks;
}

/** Decompress IWA file */
function decompress_iwa_file(buf: Uint8Array): Uint8Array {
	if(Array.isArray(buf)) buf = new Uint8Array(buf);
	var out: Uint8Array[] = [];
	var l = 0;
	while(l < buf.length) {
		var t = buf[l++];
		var len = buf[l] | (buf[l+1]<<8) | (buf[l+2] << 16); l += 3;
		out.push.apply(out, parse_snappy_chunk(t, buf[subarray](l, l + len)));
		l += len;
	}
	if(l !== buf.length) throw new Error("data is not a valid framed stream!");
	return out.length == 1 ? out[0] : u8concat(out);
}

type MessageSpace = {[id: number]: IWAMessage[]};
type TableItem = { id: number; path: string; type: number; message: string; };

interface ParsedFile {
	space: MessageSpace;
	tbl: TableItem[];
	type: MessageTypes;
}

function parse_iwa(cfb: CFB.CFB$Container): ParsedFile {
	var M: MessageSpace = {}, indices: number[] = [], tbl: TableItem[] = [];
	cfb.FullPaths.forEach(p => { if(p.match(/\.iwpv2/)) throw new Error(`Unsupported password protection`); });

	var root!: TableItem, rootmsg!: IWAMessage;
	/* collect entire message space */
	cfb.FileIndex.forEach((s, idx) => {
		if(!s.name.match(/\.iwa$/)) return;
		if(s.content[0] != 0) return; // TODO: this should test if the iwa follows the framing format
		var o: Uint8Array;
		try { o = decompress_iwa_file(s.content as Uint8Array); } catch(e: any) { return console.log("?? " + s.content.length + " " + ((e as Error).message || e)); }
		var packets: IWAArchiveInfo[];
		try { packets = parse_iwa_file(o); } catch(e: any) { return console.log("## " + ((e as Error).message || e)); }
		packets.forEach(packet => { M[packet.id] = packet.messages; indices.push(packet.id);
			var type = varint_to_i32(packet.messages[0].meta[1][0].data);
			var item = {
				id: packet.id,
				type,
				path:cfb.FullPaths[idx].replace(/Root Entry/, ""),
				message: "??"
			};
			if(item.id == 1) { root = item; rootmsg = packet.messages[0]; }
			tbl.push(item);
		});
	});
	if(!indices.length) throw new Error("File has no messages");
	if(!root) throw new Error(`Root element (id 1) missing!`);
	var type: MessageTypes = "N";
	if(root.type == 10000) type = "P";
	else if(parse_shallow(rootmsg.data)?.[2]?.[0]) type = "K";

	tbl.forEach(item => item.message = Messages[type][item.type] || "??");

	return { space: M, tbl, type };
}

function process_item(item: {data: Uint8Array; type: number}, type: string, protos: any) {
	switch(item.type) {
		case 0:
			var varint = parse_varint49(item.data, {l:0});
			switch(type) {
				case "bool": return !!+varint;
				case "uint32": return varint;
				case "int32": return varint | 0;
				case "sint32": return (-(varint&1))^(varint>>1);
				case "uint64": var u64 = varint_to_u64(item.data); return (BigInt(u64[1])<<32n) + BigInt(u64[0]);
				case "int64": var u64 = varint_to_u64(item.data); return new BigInt64Array([(BigInt(u64[1])<<32n) + BigInt(u64[0])])[0];
				case "sint64": var u64 = varint_to_u64(item.data); var bi = (BigInt(u64[1])<<32n) + BigInt(u64[0]); return (-(bi&1n))^(bi>>1n);
			} break;
		case 1: switch(type) {
			case "fixed64": return new BigUint64Array(new Uint8Array([...item.data]).buffer)[0];
			case "sfixed64": return new BigInt64Array(new Uint8Array([...item.data]).buffer)[0];
			case "double": return new Float64Array(new Uint8Array([...item.data]).buffer)[0];
		} break;
		case 2: switch(type) {
			case "string": return new TextDecoder().decode(item.data);
			case "bytes": return item.data;
		} break;
		case 5: switch(type) {
			case "float": return new Float32Array(new Uint8Array([...item.data]).buffer)[0];
			case "fixed32": return new Uint32Array(new Uint8Array([...item.data]).buffer)[0];
			case "sfixed32": return new Int32Array(new Uint8Array([...item.data]).buffer)[0];
		}
	}
	if(protos[type]) return process(item.data, type, protos);
	console.error(item);
	return item;
}

function process_enum(data: Uint8Array, message: string, protos: any) {
	var val = parse_varint49(data, {l: 0});
	if(val >= 4294967296) val |= 0;
	var msg = protos[message].split("\n");
	for(let m of msg) {
		if(m.startsWith("enum")) continue;
		if(m.indexOf("=")> -1) {
			let [field, , value] = m.trim().split(" ");
			if(val == parseInt(value, 10)) {
				var res = {}
				Object.defineProperty(res, "value", { get: () => val});
				Object.defineProperty(res, "PB_ENUM", { value: field, enumerable: false})
				return res;
			}
		}
	}
	throw [val, message, protos[message]];
}

function process(data: Uint8Array, message: string, protos: any) {
	if(!protos[message]) return parse_shallow(data);
	var shallow = parse_shallow(data);
	var proto: string[] = protos[message].split("\n");
	var out: any = {};
	if(proto[0].startsWith("enum")) return process_enum(data, message, protos);
	proto.forEach(line => {
		if(!line.startsWith("  ") || line.indexOf("=") == -1 || line.startsWith("   ")) return;
		const [freq, type, name, , idx] = line.trim().split(/\s+/);
		var i = parseInt(idx, 10);
		if(isNaN(i)) return;
		if(!shallow[i]?.length) return;
		switch(freq) {
			case "repeated": {
				if(/packed\s*=\s*true/.test(line)) {
					/* these are the known types as of iwa 13.0 */
					switch(type) {
						case "uint32": out[name] = parse_packed_varints(shallow[i][0].data); break;
						case "uint64": out[name] = parse_packed_varint64(shallow[i][0].data); break;
						case "fixed64": out[name] = new BigUint64Array(new Uint8Array(shallow[i][0].data).buffer); break;
						default:
							console.log(shallow[i][0], line);
							throw new Error(`unsupported packed field of type ${type}`);
					}
				} else out[name] = shallow[i].map(item => protos[type]?.startsWith("enum") ? process_enum(item.data, type, protos) : process_item(item, type, protos));
			} break;
			case "required": case "optional": out[name] = protos[type]?.startsWith("enum") ? process_enum(shallow[i][0].data, type, protos) : process_item(shallow[i][0], type, protos); break
			default: throw `unsupported frequency ${freq}`;
		}
		if(type.startsWith(".")) {
			if(freq == "repeated") out[name].forEach((n: any) => {try { Object.defineProperty(n, "PB_TYPE", {value: type, enumerable: false}); } catch(e){}});
			else try { Object.defineProperty(out[name], "PB_TYPE", {value: type, enumerable: false}); } catch(e) {}
		}
		try {
			if(freq == "repeated") out[name].forEach((n: any, idx:number) => {try { Object.defineProperty(n, "PB_RAW", {value: shallow[i][idx], enumerable: false}); } catch(e){}});
			else try { Object.defineProperty(out[name], "PB_RAW", {value: shallow[i][0], enumerable: false}); } catch(e) {}
		} catch(e){console.log(e);}
		try {
			if(freq == "repeated") out[name].forEach((n: any, idx:number) => {try { Object.defineProperty(n, "PB_FIELD", {value: `${name}[${idx}]` , enumerable: false}); } catch(e){}});
			else try { Object.defineProperty(out[name], "PB_FIELD", {value: name, enumerable: false}); } catch(e) {}
		} catch(e){console.log(e);}
	})
	return out;
}

function read(ab: ArrayBuffer): ParsedFile {
	let cfb: CFB.CFB$Container;
	try {
		cfb = CFB.read(new Uint8Array(ab), {type: "buffer"});
	} catch(e) { throw new Error(`Failed to read archive: |${e&&(e as any).message||e}|`); }
	return parse_iwa(cfb);
}

export { parse_iwa, read, process }
export type { MessageSpace, TableItem, ParsedFile };