412 lines
16 KiB
TypeScript
412 lines
16 KiB
TypeScript
import * as CFB from 'cfb';
|
|
|
|
import Messages, { MessageTypes } from "./messages/";
|
|
|
|
/* see https://bugs.webkit.org/show_bug.cgi?id=243148 -- affects iOS Safari */
|
|
declare var Buffer: any; // Buffer is typeof-guarded but TS still needs this :(
|
|
var subarray: "subarray" | "slice" = (() => {
|
|
try {
|
|
if(typeof Uint8Array == "undefined") return "slice";
|
|
if(typeof Uint8Array.prototype.subarray == "undefined") return "slice";
|
|
// NOTE: feature tests are for node < 6.x
|
|
if(typeof Buffer !== "undefined") {
|
|
if(typeof Buffer.prototype.subarray == "undefined") return "slice";
|
|
if((typeof Buffer.from == "function" ? Buffer.from([72,62]) : new Buffer([72,62])) instanceof Uint8Array) return "subarray";
|
|
return "slice";
|
|
}
|
|
return "subarray";
|
|
} catch(e) { return "slice"; }
|
|
})();
|
|
|
|
/** Concatenate Uint8Arrays */
|
|
function u8concat(u8a: Uint8Array[]): Uint8Array {
|
|
var len = 0;
|
|
for(var i = 0; i < u8a.length; ++i) len += u8a[i].length;
|
|
var out = new Uint8Array(len);
|
|
var off = 0;
|
|
for(i = 0; i < u8a.length; ++i) {
|
|
var u8 = u8a[i], L = u8.length;
|
|
if(L < 250) { for(var j = 0; j < L; ++j) out[off++] = u8[j]; }
|
|
else { out.set(u8, off); off += L; }
|
|
}
|
|
return out;
|
|
}
|
|
|
|
interface Ptr { l: number; }
|
|
|
|
/** Parse an integer from the varint that can be exactly stored in a double */
|
|
function parse_varint49(buf: Uint8Array, ptr: Ptr): number {
|
|
var l = ptr.l;
|
|
var usz = buf[l] & 0x7F;
|
|
varint: if(buf[l++] >= 0x80) {
|
|
usz |= (buf[l] & 0x7F) << 7; if(buf[l++] < 0x80) break varint;
|
|
usz |= (buf[l] & 0x7F) << 14; if(buf[l++] < 0x80) break varint;
|
|
usz |= (buf[l] & 0x7F) << 21; if(buf[l++] < 0x80) break varint;
|
|
usz += (buf[l] & 0x7F) * Math.pow(2, 28); ++l; if(buf[l++] < 0x80) break varint;
|
|
usz += (buf[l] & 0x7F) * Math.pow(2, 35); ++l; if(buf[l++] < 0x80) break varint;
|
|
usz += (buf[l] & 0x7F) * Math.pow(2, 42); ++l; if(buf[l++] < 0x80) break varint;
|
|
}
|
|
ptr.l = l;
|
|
return usz;
|
|
}
|
|
/** Parse a repeated varint [packed = true] field */
|
|
function parse_packed_varints(buf: Uint8Array): number[] {
|
|
var ptr: Ptr = {l: 0};
|
|
var out: number[] = [];
|
|
while(ptr.l < buf.length) out.push(parse_varint49(buf, ptr));
|
|
return out;
|
|
}
|
|
/** Parse a BigInt from the varint */
|
|
function parse_varint64(buf: Uint8Array, ptr: Ptr): BigInt {
|
|
var l = ptr.l;
|
|
var usz = BigInt(buf[l] & 0x7F);
|
|
varint: if(buf[l++] >= 0x80) {
|
|
usz += BigInt(buf[l] & 0x7F) << 7n; if(buf[l++] < 0x80) break varint;
|
|
usz += BigInt(buf[l] & 0x7F) << 14n; if(buf[l++] < 0x80) break varint;
|
|
usz += BigInt(buf[l] & 0x7F) << 21n; if(buf[l++] < 0x80) break varint;
|
|
usz += BigInt(buf[l] & 0x7F) << 28n; ++l; if(buf[l++] < 0x80) break varint;
|
|
usz += BigInt(buf[l] & 0x7F) << 35n; ++l; if(buf[l++] < 0x80) break varint;
|
|
usz += BigInt(buf[l] & 0x7F) << 42n; ++l; if(buf[l++] < 0x80) break varint;
|
|
usz += BigInt(buf[l] & 0x7F) << 49n; ++l; if(buf[l++] < 0x80) break varint;
|
|
usz += BigInt(buf[l] & 0x7F) << 56n; ++l; if(buf[l++] < 0x80) break varint;
|
|
usz += BigInt(buf[l] & 0x7F) << 63n; ++l; if(buf[l++] < 0x80) break varint;
|
|
}
|
|
ptr.l = l;
|
|
return usz;
|
|
}
|
|
/** Parse a repeated varint [packed = true] field */
|
|
function parse_packed_varint64(buf: Uint8Array): BigInt[] {
|
|
var ptr: Ptr = {l: 0};
|
|
var out: BigInt[] = [];
|
|
while(ptr.l < buf.length) out.push(parse_varint64(buf, ptr));
|
|
return out;
|
|
}
|
|
/** Parse a 32-bit signed integer from the raw varint */
|
|
function varint_to_i32(buf: Uint8Array): number {
|
|
var l = 0,
|
|
i32 = (buf[l] & 0x7F) ; if(buf[l++] < 0x80) return i32;
|
|
i32 |= (buf[l] & 0x7F) << 7; if(buf[l++] < 0x80) return i32;
|
|
i32 |= (buf[l] & 0x7F) << 14; if(buf[l++] < 0x80) return i32;
|
|
i32 |= (buf[l] & 0x7F) << 21; if(buf[l++] < 0x80) return i32;
|
|
i32 |= (buf[l] & 0x0F) << 28; return i32;
|
|
}
|
|
/** Parse a 64-bit unsigned integer as a pair */
|
|
function varint_to_u64(buf: Uint8Array): [number, number] {
|
|
var l = 0, lo = buf[l] & 0x7F, hi = 0;
|
|
varint: if(buf[l++] >= 0x80) {
|
|
lo |= (buf[l] & 0x7F) << 7; if(buf[l++] < 0x80) break varint;
|
|
lo |= (buf[l] & 0x7F) << 14; if(buf[l++] < 0x80) break varint;
|
|
lo |= (buf[l] & 0x7F) << 21; if(buf[l++] < 0x80) break varint;
|
|
lo |= (buf[l] & 0x7F) << 28; hi = (buf[l] >> 4) & 0x07; if(buf[l++] < 0x80) break varint;
|
|
hi |= (buf[l] & 0x7F) << 3; if(buf[l++] < 0x80) break varint;
|
|
hi |= (buf[l] & 0x7F) << 10; if(buf[l++] < 0x80) break varint;
|
|
hi |= (buf[l] & 0x7F) << 17; if(buf[l++] < 0x80) break varint;
|
|
hi |= (buf[l] & 0x7F) << 24; if(buf[l++] < 0x80) break varint;
|
|
hi |= (buf[l] & 0x7F) << 31;
|
|
}
|
|
return [lo >>> 0, hi >>> 0];
|
|
}
|
|
export { varint_to_i32, varint_to_u64 };
|
|
|
|
interface ProtoItem {
|
|
data: Uint8Array;
|
|
type: number;
|
|
}
|
|
type ProtoField = Array<ProtoItem>;
|
|
type ProtoMessage = Array<ProtoField>;
|
|
interface IWAMessage {
|
|
/** Metadata in .TSP.MessageInfo */
|
|
meta: ProtoMessage;
|
|
rawmeta: Uint8Array;
|
|
data: Uint8Array;
|
|
}
|
|
interface IWAArchiveInfo {
|
|
id: number;
|
|
merge?: boolean;
|
|
messages: IWAMessage[];
|
|
}
|
|
/** Shallow parse of a Protobuf message */
|
|
function parse_shallow(buf: Uint8Array): ProtoMessage {
|
|
var out: ProtoMessage = [], ptr: Ptr = {l: 0};
|
|
while(ptr.l < buf.length) {
|
|
var off = ptr.l;
|
|
var num = parse_varint49(buf, ptr);
|
|
var type = num & 0x07; num = (num / 8)|0;
|
|
var data: Uint8Array;
|
|
var l = ptr.l;
|
|
switch(type) {
|
|
case 0: {
|
|
while(buf[l++] >= 0x80);
|
|
data = buf[subarray](ptr.l, l);
|
|
ptr.l = l;
|
|
} break;
|
|
case 1: { data = buf[subarray](l, l + 8); ptr.l = l + 8; } break;
|
|
case 2: {
|
|
var len = parse_varint49(buf, ptr);
|
|
data = buf[subarray](ptr.l, ptr.l + len);
|
|
ptr.l += len;
|
|
} break;
|
|
case 5: { data = buf[subarray](l, l + 4); ptr.l = l + 4; } break;
|
|
default: throw new Error(`PB Type ${type} for Field ${num} at offset ${off}`);
|
|
}
|
|
var v: ProtoItem = { data, type };
|
|
if(out[num] == null) out[num] = [];
|
|
out[num].push(v);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
/** Extract all messages from a IWA file */
|
|
function parse_iwa_file(buf: Uint8Array): IWAArchiveInfo[] {
|
|
var out: IWAArchiveInfo[] = [], ptr: Ptr = {l: 0};
|
|
while(ptr.l < buf.length) {
|
|
/* .TSP.ArchiveInfo */
|
|
var len = parse_varint49(buf, ptr);
|
|
var ai = parse_shallow(buf[subarray](ptr.l, ptr.l + len));
|
|
ptr.l += len;
|
|
|
|
var res: IWAArchiveInfo = {
|
|
/* TODO: technically ID is optional */
|
|
id: varint_to_i32(ai[1][0].data),
|
|
messages: []
|
|
};
|
|
ai[2].forEach(b => {
|
|
var mi = parse_shallow(b.data);
|
|
var fl = varint_to_i32(mi[3][0].data);
|
|
res.messages.push({
|
|
meta: mi,
|
|
rawmeta: b.data,
|
|
data: buf[subarray](ptr.l, ptr.l + fl)
|
|
});
|
|
ptr.l += fl;
|
|
});
|
|
if(ai[3]?.[0]) res.merge = (varint_to_i32(ai[3][0].data) >>> 0) > 0;
|
|
out.push(res);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
/** Decompress a snappy chunk */
|
|
function parse_snappy_chunk(type: number, buf: Uint8Array): Uint8Array[] {
|
|
if(type != 0) throw new Error(`Unexpected Snappy chunk type ${type}`);
|
|
var ptr: Ptr = {l: 0};
|
|
|
|
var usz = parse_varint49(buf, ptr);
|
|
var chunks: Uint8Array[] = [];
|
|
var l = ptr.l;
|
|
while(l < buf.length) {
|
|
var tag = buf[l] & 0x3;
|
|
if(tag == 0) {
|
|
var len = buf[l++] >> 2;
|
|
if(len < 60) ++len;
|
|
else {
|
|
var c = len - 59;
|
|
len = buf[l];
|
|
if(c > 1) len |= (buf[l+1]<<8);
|
|
if(c > 2) len |= (buf[l+2]<<16);
|
|
if(c > 3) len |= (buf[l+3]<<24);
|
|
len >>>=0; len++;
|
|
l += c;
|
|
}
|
|
chunks.push(buf[subarray](l, l + len)); l += len; continue;
|
|
} else {
|
|
var offset = 0, length = 0;
|
|
if(tag == 1) {
|
|
length = ((buf[l] >> 2) & 0x7) + 4;
|
|
offset = (buf[l++] & 0xE0) << 3;
|
|
offset |= buf[l++];
|
|
} else {
|
|
length = (buf[l++] >> 2) + 1;
|
|
if(tag == 2) { offset = buf[l] | (buf[l+1]<<8); l += 2; }
|
|
else { offset = (buf[l] | (buf[l+1]<<8) | (buf[l+2]<<16) | (buf[l+3]<<24))>>>0; l += 4; }
|
|
}
|
|
if(offset == 0) throw new Error("Invalid offset 0");
|
|
var j = chunks.length - 1, off = offset;
|
|
while(j >=0 && off >= chunks[j].length) { off -= chunks[j].length; --j; }
|
|
if(j < 0) {
|
|
if(off == 0) off = chunks[(j = 0)].length;
|
|
else throw new Error("Invalid offset beyond length");
|
|
}
|
|
// Node 0.8 Buffer slice does not support negative indices
|
|
if(length < off) chunks.push(chunks[j][subarray](chunks[j].length-off, chunks[j].length-off + length));
|
|
else {
|
|
if(off > 0) { chunks.push(chunks[j][subarray](chunks[j].length-off)); length -= off; } ++j;
|
|
while(length >= chunks[j].length) { chunks.push(chunks[j]); length -= chunks[j].length; ++j; }
|
|
if(length) chunks.push(chunks[j][subarray](0, length));
|
|
}
|
|
if(chunks.length > 25) chunks = [u8concat(chunks)];
|
|
}
|
|
}
|
|
var clen = 0; for(var u8i = 0; u8i < chunks.length; ++u8i) clen += chunks[u8i].length;
|
|
if(clen != usz) throw new Error(`Unexpected length: ${clen} != ${usz}`);
|
|
return chunks;
|
|
}
|
|
|
|
/** Decompress IWA file */
|
|
function decompress_iwa_file(buf: Uint8Array): Uint8Array {
|
|
if(Array.isArray(buf)) buf = new Uint8Array(buf);
|
|
var out: Uint8Array[] = [];
|
|
var l = 0;
|
|
while(l < buf.length) {
|
|
var t = buf[l++];
|
|
var len = buf[l] | (buf[l+1]<<8) | (buf[l+2] << 16); l += 3;
|
|
out.push.apply(out, parse_snappy_chunk(t, buf[subarray](l, l + len)));
|
|
l += len;
|
|
}
|
|
if(l !== buf.length) throw new Error("data is not a valid framed stream!");
|
|
return out.length == 1 ? out[0] : u8concat(out);
|
|
}
|
|
|
|
type MessageSpace = {[id: number]: IWAMessage[]};
|
|
type TableItem = { id: number; path: string; type: number; message: string; };
|
|
|
|
interface ParsedFile {
|
|
space: MessageSpace;
|
|
tbl: TableItem[];
|
|
type: MessageTypes;
|
|
}
|
|
|
|
function parse_iwa(cfb: CFB.CFB$Container): ParsedFile {
|
|
var M: MessageSpace = {}, indices: number[] = [], tbl: TableItem[] = [];
|
|
cfb.FullPaths.forEach(p => { if(p.match(/\.iwpv2/)) throw new Error(`Unsupported password protection`); });
|
|
|
|
var root!: TableItem, rootmsg!: IWAMessage;
|
|
/* collect entire message space */
|
|
cfb.FileIndex.forEach((s, idx) => {
|
|
if(!s.name.match(/\.iwa$/)) return;
|
|
if(s.content[0] != 0) return; // TODO: this should test if the iwa follows the framing format
|
|
var o: Uint8Array;
|
|
try { o = decompress_iwa_file(s.content as Uint8Array); } catch(e: any) { return console.log("?? " + s.content.length + " " + ((e as Error).message || e)); }
|
|
var packets: IWAArchiveInfo[];
|
|
try { packets = parse_iwa_file(o); } catch(e: any) { return console.log("## " + ((e as Error).message || e)); }
|
|
packets.forEach(packet => { M[packet.id] = packet.messages; indices.push(packet.id);
|
|
var type = varint_to_i32(packet.messages[0].meta[1][0].data);
|
|
var item = {
|
|
id: packet.id,
|
|
type,
|
|
path:cfb.FullPaths[idx].replace(/Root Entry/, ""),
|
|
message: "??"
|
|
};
|
|
if(item.id == 1) { root = item; rootmsg = packet.messages[0]; }
|
|
tbl.push(item);
|
|
});
|
|
});
|
|
if(!indices.length) throw new Error("File has no messages");
|
|
if(!root) throw new Error(`Root element (id 1) missing!`);
|
|
var type: MessageTypes = "N";
|
|
if(root.type == 10000) type = "P";
|
|
else if(parse_shallow(rootmsg.data)?.[2]?.[0]) type = "K";
|
|
|
|
tbl.forEach(item => item.message = Messages[type][item.type] || "??");
|
|
|
|
return { space: M, tbl, type };
|
|
}
|
|
|
|
function process_item(item: {data: Uint8Array; type: number}, type: string, protos: any) {
|
|
switch(item.type) {
|
|
case 0:
|
|
var varint = parse_varint49(item.data, {l:0});
|
|
switch(type) {
|
|
case "bool": return !!+varint;
|
|
case "uint32": return varint;
|
|
case "int32": return varint | 0;
|
|
case "sint32": return (-(varint&1))^(varint>>1);
|
|
case "uint64": var u64 = varint_to_u64(item.data); return (BigInt(u64[1])<<32n) + BigInt(u64[0]);
|
|
case "int64": var u64 = varint_to_u64(item.data); return new BigInt64Array([(BigInt(u64[1])<<32n) + BigInt(u64[0])])[0];
|
|
case "sint64": var u64 = varint_to_u64(item.data); var bi = (BigInt(u64[1])<<32n) + BigInt(u64[0]); return (-(bi&1n))^(bi>>1n);
|
|
} break;
|
|
case 1: switch(type) {
|
|
case "fixed64": return new BigUint64Array(new Uint8Array([...item.data]).buffer)[0];
|
|
case "sfixed64": return new BigInt64Array(new Uint8Array([...item.data]).buffer)[0];
|
|
case "double": return new Float64Array(new Uint8Array([...item.data]).buffer)[0];
|
|
} break;
|
|
case 2: switch(type) {
|
|
case "string": return new TextDecoder().decode(item.data);
|
|
case "bytes": return item.data;
|
|
} break;
|
|
case 5: switch(type) {
|
|
case "float": return new Float32Array(new Uint8Array([...item.data]).buffer)[0];
|
|
case "fixed32": return new Uint32Array(new Uint8Array([...item.data]).buffer)[0];
|
|
case "sfixed32": return new Int32Array(new Uint8Array([...item.data]).buffer)[0];
|
|
}
|
|
}
|
|
if(protos[type]) return process(item.data, type, protos);
|
|
console.error(item);
|
|
return item;
|
|
}
|
|
|
|
function process_enum(data: Uint8Array, message: string, protos: any) {
|
|
var val = parse_varint49(data, {l: 0});
|
|
if(val >= 4294967296) val |= 0;
|
|
var msg = protos[message].split("\n");
|
|
for(let m of msg) {
|
|
if(m.startsWith("enum")) continue;
|
|
if(m.indexOf("=")> -1) {
|
|
let [field, , value] = m.trim().split(" ");
|
|
if(val == parseInt(value, 10)) {
|
|
var res = {}
|
|
Object.defineProperty(res, "value", { get: () => val});
|
|
Object.defineProperty(res, "PB_ENUM", { value: field, enumerable: false})
|
|
return res;
|
|
}
|
|
}
|
|
}
|
|
throw [val, message, protos[message]];
|
|
}
|
|
|
|
function process(data: Uint8Array, message: string, protos: any) {
|
|
if(!protos[message]) return parse_shallow(data);
|
|
var shallow = parse_shallow(data);
|
|
var proto: string[] = protos[message].split("\n");
|
|
var out: any = {};
|
|
if(proto[0].startsWith("enum")) return process_enum(data, message, protos);
|
|
proto.forEach(line => {
|
|
if(!line.startsWith(" ") || line.indexOf("=") == -1 || line.startsWith(" ")) return;
|
|
const [freq, type, name, , idx] = line.trim().split(/\s+/);
|
|
var i = parseInt(idx, 10);
|
|
if(isNaN(i)) return;
|
|
if(!shallow[i]?.length) return;
|
|
switch(freq) {
|
|
case "repeated": {
|
|
if(/packed\s*=\s*true/.test(line)) {
|
|
/* these are the known types as of iwa 13.0 */
|
|
switch(type) {
|
|
case "uint32": out[name] = parse_packed_varints(shallow[i][0].data); break;
|
|
case "uint64": out[name] = parse_packed_varint64(shallow[i][0].data); break;
|
|
case "fixed64": out[name] = new BigUint64Array(new Uint8Array(shallow[i][0].data).buffer); break;
|
|
default:
|
|
console.log(shallow[i][0], line);
|
|
throw new Error(`unsupported packed field of type ${type}`);
|
|
}
|
|
} else out[name] = shallow[i].map(item => protos[type]?.startsWith("enum") ? process_enum(item.data, type, protos) : process_item(item, type, protos));
|
|
} break;
|
|
case "required": case "optional": out[name] = protos[type]?.startsWith("enum") ? process_enum(shallow[i][0].data, type, protos) : process_item(shallow[i][0], type, protos); break
|
|
default: throw `unsupported frequency ${freq}`;
|
|
}
|
|
if(type.startsWith(".")) {
|
|
if(freq == "repeated") out[name].forEach((n: any) => {try { Object.defineProperty(n, "PB_TYPE", {value: type, enumerable: false}); } catch(e){}});
|
|
else try { Object.defineProperty(out[name], "PB_TYPE", {value: type, enumerable: false}); } catch(e) {}
|
|
}
|
|
try {
|
|
if(freq == "repeated") out[name].forEach((n: any, idx:number) => {try { Object.defineProperty(n, "PB_RAW", {value: shallow[i][idx], enumerable: false}); } catch(e){}});
|
|
else try { Object.defineProperty(out[name], "PB_RAW", {value: shallow[i][0], enumerable: false}); } catch(e) {}
|
|
} catch(e){console.log(e);}
|
|
try {
|
|
if(freq == "repeated") out[name].forEach((n: any, idx:number) => {try { Object.defineProperty(n, "PB_FIELD", {value: `${name}[${idx}]` , enumerable: false}); } catch(e){}});
|
|
else try { Object.defineProperty(out[name], "PB_FIELD", {value: name, enumerable: false}); } catch(e) {}
|
|
} catch(e){console.log(e);}
|
|
})
|
|
return out;
|
|
}
|
|
|
|
function read(ab: ArrayBuffer): ParsedFile {
|
|
let cfb: CFB.CFB$Container;
|
|
try {
|
|
cfb = CFB.read(new Uint8Array(ab), {type: "buffer"});
|
|
} catch(e) { throw new Error(`Failed to read archive: |${e&&(e as any).message||e}|`); }
|
|
return parse_iwa(cfb);
|
|
}
|
|
|
|
export { parse_iwa, read, process }
|
|
export type { MessageSpace, TableItem, ParsedFile };
|