import * as CFB from 'cfb'; import Messages, { $_TSP_MessageInfo, MessageTypes } from "./messages/"; /* see https://bugs.webkit.org/show_bug.cgi?id=243148 -- affects iOS Safari */ declare var Buffer: any; // Buffer is typeof-guarded but TS still needs this :( var subarray: "subarray" | "slice" = (() => { try { if(typeof Uint8Array == "undefined") return "slice"; if(typeof Uint8Array.prototype.subarray == "undefined") return "slice"; // NOTE: feature tests are for node < 6.x if(typeof Buffer !== "undefined") { if(typeof Buffer.prototype.subarray == "undefined") return "slice"; if((typeof Buffer.from == "function" ? Buffer.from([72,62]) : new Buffer([72,62])) instanceof Uint8Array) return "subarray"; return "slice"; } return "subarray"; } catch(e) { return "slice"; } })(); /** Concatenate Uint8Arrays */ function u8concat(u8a: Uint8Array[]): Uint8Array { var len = 0; for(var i = 0; i < u8a.length; ++i) len += u8a[i].length; var out = new Uint8Array(len); var off = 0; for(i = 0; i < u8a.length; ++i) { var u8 = u8a[i], L = u8.length; if(L < 250) { for(var j = 0; j < L; ++j) out[off++] = u8[j]; } else { out.set(u8, off); off += L; } } return out; } interface Ptr { l: number; } /** Parse an integer from the varint that can be exactly stored in a double */ function parse_varint49(buf: Uint8Array, ptr: Ptr): number { var l = ptr.l; var usz = buf[l] & 0x7F; varint: if(buf[l++] >= 0x80) { usz |= (buf[l] & 0x7F) << 7; if(buf[l++] < 0x80) break varint; usz |= (buf[l] & 0x7F) << 14; if(buf[l++] < 0x80) break varint; usz |= (buf[l] & 0x7F) << 21; if(buf[l++] < 0x80) break varint; usz += (buf[l] & 0x7F) * Math.pow(2, 28); ++l; if(buf[l++] < 0x80) break varint; usz += (buf[l] & 0x7F) * Math.pow(2, 35); ++l; if(buf[l++] < 0x80) break varint; usz += (buf[l] & 0x7F) * Math.pow(2, 42); ++l; if(buf[l++] < 0x80) break varint; } ptr.l = l; return usz; } /** Parse a repeated varint [packed = true] field */ function parse_packed_varints(buf: Uint8Array): number[] { var ptr: Ptr = {l: 0}; var out: number[] = []; while(ptr.l < buf.length) out.push(parse_varint49(buf, ptr)); return out; } /** Parse a BigInt from the varint */ function parse_varint64(buf: Uint8Array, ptr: Ptr): BigInt { var l = ptr.l; var usz = BigInt(buf[l] & 0x7F); varint: if(buf[l++] >= 0x80) { usz += BigInt(buf[l] & 0x7F) << 7n; if(buf[l++] < 0x80) break varint; usz += BigInt(buf[l] & 0x7F) << 14n; if(buf[l++] < 0x80) break varint; usz += BigInt(buf[l] & 0x7F) << 21n; if(buf[l++] < 0x80) break varint; usz += BigInt(buf[l] & 0x7F) << 28n; ++l; if(buf[l++] < 0x80) break varint; usz += BigInt(buf[l] & 0x7F) << 35n; ++l; if(buf[l++] < 0x80) break varint; usz += BigInt(buf[l] & 0x7F) << 42n; ++l; if(buf[l++] < 0x80) break varint; usz += BigInt(buf[l] & 0x7F) << 49n; ++l; if(buf[l++] < 0x80) break varint; usz += BigInt(buf[l] & 0x7F) << 56n; ++l; if(buf[l++] < 0x80) break varint; usz += BigInt(buf[l] & 0x7F) << 63n; ++l; if(buf[l++] < 0x80) break varint; } ptr.l = l; return usz; } /** Parse a repeated varint [packed = true] field */ function parse_packed_varint64(buf: Uint8Array): BigInt[] { var ptr: Ptr = {l: 0}; var out: BigInt[] = []; while(ptr.l < buf.length) out.push(parse_varint64(buf, ptr)); return out; } /** Parse a 32-bit signed integer from the raw varint */ function varint_to_i32(buf: Uint8Array): number { var l = 0, i32 = (buf[l] & 0x7F) ; if(buf[l++] < 0x80) return i32; i32 |= (buf[l] & 0x7F) << 7; if(buf[l++] < 0x80) return i32; i32 |= (buf[l] & 0x7F) << 14; if(buf[l++] < 0x80) return i32; i32 |= (buf[l] & 0x7F) << 21; if(buf[l++] < 0x80) return i32; i32 |= (buf[l] & 0x0F) << 28; return i32; } /** Parse a 64-bit unsigned integer as a pair */ function varint_to_u64(buf: Uint8Array): [number, number] { var l = 0, lo = buf[l] & 0x7F, hi = 0; varint: if(buf[l++] >= 0x80) { lo |= (buf[l] & 0x7F) << 7; if(buf[l++] < 0x80) break varint; lo |= (buf[l] & 0x7F) << 14; if(buf[l++] < 0x80) break varint; lo |= (buf[l] & 0x7F) << 21; if(buf[l++] < 0x80) break varint; lo |= (buf[l] & 0x7F) << 28; hi = (buf[l] >> 4) & 0x07; if(buf[l++] < 0x80) break varint; hi |= (buf[l] & 0x7F) << 3; if(buf[l++] < 0x80) break varint; hi |= (buf[l] & 0x7F) << 10; if(buf[l++] < 0x80) break varint; hi |= (buf[l] & 0x7F) << 17; if(buf[l++] < 0x80) break varint; hi |= (buf[l] & 0x7F) << 24; if(buf[l++] < 0x80) break varint; hi |= (buf[l] & 0x7F) << 31; } return [lo >>> 0, hi >>> 0]; } export { varint_to_i32, varint_to_u64 }; interface ProtoItem { data: Uint8Array; type: number; } type ProtoField = Array; type ProtoMessage = Array; interface IWAMessage { /** Metadata in .TSP.MessageInfo */ meta: ProtoMessage; rawmeta: Uint8Array; data: Uint8Array; pre?: string; parsed?: any; parsedmeta?: $_TSP_MessageInfo; } interface IWAArchiveInfo { id: number; merge?: boolean; messages: IWAMessage[]; } /** Shallow parse of a Protobuf message */ function parse_shallow(buf: Uint8Array): ProtoMessage { var out: ProtoMessage = [], ptr: Ptr = {l: 0}; while(ptr.l < buf.length) { var off = ptr.l; var num = parse_varint49(buf, ptr); var type = num & 0x07; num = (num / 8)|0; var data: Uint8Array; var l = ptr.l; switch(type) { case 0: { while(buf[l++] >= 0x80); data = buf[subarray](ptr.l, l); ptr.l = l; } break; case 1: { data = buf[subarray](l, l + 8); ptr.l = l + 8; } break; case 2: { var len = parse_varint49(buf, ptr); data = buf[subarray](ptr.l, ptr.l + len); ptr.l += len; } break; case 5: { data = buf[subarray](l, l + 4); ptr.l = l + 4; } break; default: throw new Error(`PB Type ${type} for Field ${num} at offset ${off}`); } var v: ProtoItem = { data, type }; if(out[num] == null) out[num] = []; out[num].push(v); } return out; } /** Extract all messages from a IWA file */ function parse_iwa_file(buf: Uint8Array): IWAArchiveInfo[] { var out: IWAArchiveInfo[] = [], ptr: Ptr = {l: 0}; while(ptr.l < buf.length) { /* .TSP.ArchiveInfo */ var len = parse_varint49(buf, ptr); var ai = parse_shallow(buf[subarray](ptr.l, ptr.l + len)); ptr.l += len; var res: IWAArchiveInfo = { /* TODO: technically ID is optional */ id: varint_to_i32(ai[1][0].data), messages: [] }; ai[2].forEach(b => { var mi = parse_shallow(b.data); var fl = varint_to_i32(mi[3][0].data); res.messages.push({ meta: mi, rawmeta: b.data, data: buf[subarray](ptr.l, ptr.l + fl) }); ptr.l += fl; }); if(ai[3]?.[0]) res.merge = (varint_to_i32(ai[3][0].data) >>> 0) > 0; out.push(res); } return out; } /** Decompress a snappy chunk */ function parse_snappy_chunk(type: number, buf: Uint8Array): Uint8Array[] { if(type != 0) throw new Error(`Unexpected Snappy chunk type ${type}`); var ptr: Ptr = {l: 0}; var usz = parse_varint49(buf, ptr); var chunks: Uint8Array[] = []; var l = ptr.l; while(l < buf.length) { var tag = buf[l] & 0x3; if(tag == 0) { var len = buf[l++] >> 2; if(len < 60) ++len; else { var c = len - 59; len = buf[l]; if(c > 1) len |= (buf[l+1]<<8); if(c > 2) len |= (buf[l+2]<<16); if(c > 3) len |= (buf[l+3]<<24); len >>>=0; len++; l += c; } chunks.push(buf[subarray](l, l + len)); l += len; continue; } else { var offset = 0, length = 0; if(tag == 1) { length = ((buf[l] >> 2) & 0x7) + 4; offset = (buf[l++] & 0xE0) << 3; offset |= buf[l++]; } else { length = (buf[l++] >> 2) + 1; if(tag == 2) { offset = buf[l] | (buf[l+1]<<8); l += 2; } else { offset = (buf[l] | (buf[l+1]<<8) | (buf[l+2]<<16) | (buf[l+3]<<24))>>>0; l += 4; } } if(offset == 0) throw new Error("Invalid offset 0"); var j = chunks.length - 1, off = offset; while(j >=0 && off >= chunks[j].length) { off -= chunks[j].length; --j; } if(j < 0) { if(off == 0) off = chunks[(j = 0)].length; else throw new Error("Invalid offset beyond length"); } // Node 0.8 Buffer slice does not support negative indices if(length < off) chunks.push(chunks[j][subarray](chunks[j].length-off, chunks[j].length-off + length)); else { if(off > 0) { chunks.push(chunks[j][subarray](chunks[j].length-off)); length -= off; } ++j; while(length >= chunks[j].length) { chunks.push(chunks[j]); length -= chunks[j].length; ++j; } if(length) chunks.push(chunks[j][subarray](0, length)); } if(chunks.length > 25) chunks = [u8concat(chunks)]; } } var clen = 0; for(var u8i = 0; u8i < chunks.length; ++u8i) clen += chunks[u8i].length; if(clen != usz) throw new Error(`Unexpected length: ${clen} != ${usz}`); return chunks; } /** Decompress IWA file */ function decompress_iwa_file(buf: Uint8Array): Uint8Array { if(Array.isArray(buf)) buf = new Uint8Array(buf); var out: Uint8Array[] = []; var l = 0; while(l < buf.length) { var t = buf[l++]; var len = buf[l] | (buf[l+1]<<8) | (buf[l+2] << 16); l += 3; out.push.apply(out, parse_snappy_chunk(t, buf[subarray](l, l + len))); l += len; } if(l !== buf.length) throw new Error("data is not a valid framed stream!"); return out.length == 1 ? out[0] : u8concat(out); } type MessageSpace = {[id: number]: IWAMessage[]}; type TableItem = { id: number; path: string; type: number; message: string; }; interface ParsedFile { space: MessageSpace; tbl: TableItem[]; type: MessageTypes; } function parse_iwa(cfb: CFB.CFB$Container): ParsedFile { var M: MessageSpace = {}, indices: number[] = [], tbl: TableItem[] = []; cfb.FullPaths.forEach(p => { if(p.match(/\.iwpv2/)) throw new Error(`Unsupported password protection`); }); var root!: TableItem, rootmsg!: IWAMessage; /* collect entire message space */ cfb.FileIndex.forEach((s, idx) => { if(!s.name.match(/\.iwa$/)) return; if(s.content[0] != 0) return; // TODO: this should test if the iwa follows the framing format var o: Uint8Array; try { o = decompress_iwa_file(s.content as Uint8Array); } catch(e: any) { return console.log("?? " + s.content.length + " " + ((e as Error).message || e)); } var packets: IWAArchiveInfo[]; try { packets = parse_iwa_file(o); } catch(e: any) { return console.log("## " + ((e as Error).message || e)); } packets.forEach(packet => { M[packet.id] = packet.messages; indices.push(packet.id); var type = varint_to_i32(packet.messages[0].meta[1][0].data); var item = { id: packet.id, type, path:cfb.FullPaths[idx].replace(/Root Entry/, ""), message: "??" }; if(item.id == 1) { root = item; rootmsg = packet.messages[0]; } tbl.push(item); }); }); if(!indices.length) throw new Error("File has no messages"); if(!root) throw new Error(`Root element (id 1) missing!`); var type: MessageTypes = "N"; if(root.type == 10000) type = "P"; else if(parse_shallow(rootmsg.data)?.[2]?.[0]) type = "K"; tbl.forEach(item => item.message = Messages[type][item.type] || "??"); return { space: M, tbl, type }; } function process_item(item: {data: Uint8Array; type: number}, type: string, protos: any) { switch(item.type) { case 0: var varint = parse_varint49(item.data, {l:0}); switch(type) { case "bool": return !!+varint; case "uint32": return varint; case "int32": return varint | 0; case "sint32": return (-(varint&1))^(varint>>1); case "uint64": var u64 = varint_to_u64(item.data); return (BigInt(u64[1])<<32n) + BigInt(u64[0]); case "int64": var u64 = varint_to_u64(item.data); return new BigInt64Array([(BigInt(u64[1])<<32n) + BigInt(u64[0])])[0]; case "sint64": var u64 = varint_to_u64(item.data); var bi = (BigInt(u64[1])<<32n) + BigInt(u64[0]); return (-(bi&1n))^(bi>>1n); } break; case 1: switch(type) { case "fixed64": return new BigUint64Array(new Uint8Array([...item.data]).buffer)[0]; case "sfixed64": return new BigInt64Array(new Uint8Array([...item.data]).buffer)[0]; case "double": return new Float64Array(new Uint8Array([...item.data]).buffer)[0]; } break; case 2: switch(type) { case "string": return new TextDecoder().decode(item.data); case "bytes": return item.data; } break; case 5: switch(type) { case "float": return new Float32Array(new Uint8Array([...item.data]).buffer)[0]; case "fixed32": return new Uint32Array(new Uint8Array([...item.data]).buffer)[0]; case "sfixed32": return new Int32Array(new Uint8Array([...item.data]).buffer)[0]; } } if(protos[type]) return process(item.data, type, protos); console.error(item); return item; } function process_enum(data: Uint8Array, message: string, protos: any) { var val = parse_varint49(data, {l: 0}); if(val >= 4294967296) val |= 0; var msg = protos[message].split("\n"); for(let m of msg) { if(m.startsWith("enum")) continue; if(m.indexOf("=")> -1) { let [field, , value] = m.trim().split(" "); if(val == parseInt(value, 10)) { var res = {} Object.defineProperty(res, "value", { get: () => val}); Object.defineProperty(res, "PB_ENUM", { value: field, enumerable: false}) return res; } } } throw [val, message, protos[message]]; } function process(data: Uint8Array, message: string, protos: any) { if(!protos[message]) return parse_shallow(data); var shallow = parse_shallow(data); var proto: string[] = protos[message].split("\n"); var out: any = {}; if(proto[0].startsWith("enum")) return process_enum(data, message, protos); proto.forEach(line => { if(!line.startsWith(" ") || line.indexOf("=") == -1 || line.startsWith(" ")) return; const [freq, type, name, , idx] = line.trim().split(/\s+/); var i = parseInt(idx, 10); if(isNaN(i)) return; if(!shallow[i]?.length) return; switch(freq) { case "repeated": { if(/packed\s*=\s*true/.test(line)) { /* these are the known types as of iwa 13.0 */ switch(type) { case "uint32": out[name] = parse_packed_varints(shallow[i][0].data); break; case "uint64": out[name] = parse_packed_varint64(shallow[i][0].data); break; case "fixed64": out[name] = new BigUint64Array(new Uint8Array(shallow[i][0].data).buffer); break; default: console.log(shallow[i][0], line); throw new Error(`unsupported packed field of type ${type}`); } } else out[name] = shallow[i].map(item => protos[type]?.startsWith("enum") ? process_enum(item.data, type, protos) : process_item(item, type, protos)); } break; case "required": case "optional": out[name] = protos[type]?.startsWith("enum") ? process_enum(shallow[i][0].data, type, protos) : process_item(shallow[i][0], type, protos); break default: throw `unsupported frequency ${freq}`; } if(type.startsWith(".")) { if(freq == "repeated") out[name].forEach((n: any) => {try { Object.defineProperty(n, "PB_TYPE", {value: type, enumerable: false}); } catch(e){}}); else try { Object.defineProperty(out[name], "PB_TYPE", {value: type, enumerable: false}); } catch(e) {} } try { if(freq == "repeated") out[name].forEach((n: any, idx:number) => {try { Object.defineProperty(n, "PB_RAW", {value: shallow[i][idx], enumerable: false}); } catch(e){}}); else try { Object.defineProperty(out[name], "PB_RAW", {value: shallow[i][0], enumerable: false}); } catch(e) {} } catch(e){console.log(e);} try { if(freq == "repeated") out[name].forEach((n: any, idx:number) => {try { Object.defineProperty(n, "PB_FIELD", {value: `${name}[${idx}]` , enumerable: false}); } catch(e){}}); else try { Object.defineProperty(out[name], "PB_FIELD", {value: name, enumerable: false}); } catch(e) {} } catch(e){console.log(e);} }) return out; } function read(ab: ArrayBuffer): ParsedFile { let cfb: CFB.CFB$Container; try { cfb = CFB.read(new Uint8Array(ab), {type: "buffer"}); } catch(e) { throw new Error(`Failed to read archive: |${e&&(e as any).message||e}|`); } return parse_iwa(cfb); } export { parse_iwa, read, process } export type { MessageSpace, TableItem, ParsedFile };