0.17.5 to 0.18.2 invalid array length issue during writeFile #2539

New Issue

2022-03-02T12:09:33Z

RScherzer commented

2022-03-02 12:09:33 +00:00

(Migrated from github.com)

Hi,
just had the time to test 0.18.2 and in runs into a "Range Error: Invalid array length" issue when doing a "XLSX.writeFile(wb, args.fileName)" on a workbook which perfectly works fine with 0.17.5 (we're talking about one sheet with 25000 rows * 186 columns). Replacing xlsx.full.min.js with the 0.17.5 version and it is running and saving fine. 0.18.2 creates the problem.

Only had a quick look at the minimized code (sorry): The error is happening in xlsx.full.min.js at pos 9:22755 (corresponds to function z(e){var r=new Array(e.length);for(var t=0;t<e.length;++t)r[t]=String.fromCharCode(e[t]);return r.join("")} This is most likely the "a2s" function
....so maybe another unicode issue? Would be great if you can check it. In the meantime I try to do more tests on my side checking which data is actually used there.

Small update: Use xlsx.js and it is indeed the a2s(o) function (line 2124). The o.length is in my crashing case 181579121 but when reaching already i=11184812 and setting out[i] to whatever leads to the invalid array length error. Call stack comes from write, write_zip_denouement, write_zip_type....

Using Win11 with Edge 98.0.1108.62 by the way

Hi, just had the time to test 0.18.2 and in runs into a "Range Error: Invalid array length" issue when doing a "XLSX.writeFile(wb, args.fileName)" on a workbook which perfectly works fine with 0.17.5 (we're talking about one sheet with 25000 rows * 186 columns). Replacing xlsx.full.min.js with the 0.17.5 version and it is running and saving fine. 0.18.2 creates the problem. Only had a quick look at the minimized code (sorry): The error is happening in xlsx.full.min.js at pos 9:22755 (corresponds to function z(e){var r=new Array(e.length);for(var t=0;t<e.length;++t)r[t]=String.fromCharCode(e[t]);return r.join("")} This is most likely the "a2s" function ....so maybe another unicode issue? Would be great if you can check it. In the meantime I try to do more tests on my side checking which data is actually used there. Small update: Use xlsx.js and it is indeed the a2s(o) function (line 2124). The o.length is in my crashing case 181579121 but when reaching already i=11184812 and setting out[i] to whatever leads to the invalid array length error. Call stack comes from write, write_zip_denouement, write_zip_type.... Using Win11 with Edge 98.0.1108.62 by the way

SheetJSDev commented

2022-03-02 19:32:18 +00:00

(Migrated from github.com)

The function that actually performs the download mantras is expecting a binary string. That doesn't make sense now that the zip writer builds a Uint8Array by default (the writeFile path and the array / buffer output types now convert from Uint8Array to binary string and back). This also applies to Deno, since Deno.writeFileSync only operates on Uint8Array

The following patch should work, feel free to submit a PR:

diff --git a/bits/88_write.js b/bits/88_write.js
@@ -45,14 +45,15 @@ function write_zip_typeXLSX(wb/*:Workbook*/, opts/*:?WriteOpts*/)/*:any*/ {
 }
 function write_zip_denouement(z/*:any*/, o/*:?WriteOpts*/)/*:any*/ {
        var oopts = {};
+       var ftype = has_buf ? "nodebuffer" : (typeof Uint8Array !== "undefined" ? "array" : "string");
        if(o.compression) oopts.compression = 'DEFLATE';
-       if(o.password) oopts.type = has_buf ? "nodebuffer" : "string";
+       if(o.password) oopts.type = ftype;
        else switch(o.type) {
                case "base64": oopts.type = "base64"; break;
                case "binary": oopts.type = "string"; break;
                case "string": throw new Error("'string' output type invalid for '" + o.bookType + "' files");
                case "buffer":
-               case "file": oopts.type = has_buf ? "nodebuffer" : "string"; break;
+               case "file": oopts.type = ftype; break;
                default: throw new Error("Unrecognized type " + o.type);
        }
        var out = z.FullPaths ? CFB.write(z, {fileType:"zip", type: /*::(*/{"nodebuffer": "buffer", "string": "binary"}/*:: :any)*/[oopts.type] || oopts.type, compression: !!o.compression}) : z.generate(oopts);
diff --git a/bits/19_fsutils.js b/bits/19_fsutils.js
@@ -13,7 +13,7 @@ function write_dl(fname/*:string*/, payload/*:any*/, enc/*:?string*/) {
        if(typeof _fs !== 'undefined' && _fs.writeFileSync) return enc ? _fs.writeFileSync(fname, payload, enc) : _fs.writeFileSync(fname, payload);
        if(typeof Deno !== 'undefined') {
                /* in this spot, it's safe to assume typed arrays and TextEncoder/TextDecoder exist */
-               if(enc) switch(enc) {
+               if(enc && typeof payload == "string") switch(enc) {
                        case "utf8": payload = new TextEncoder(enc).encode(payload); break;
                        case "binary": payload = s2ab(payload); break;
                        /* TODO: binary equivalent */
diff --git a/misc/19_mjsfs.js b/misc/19_mjsfs.js
@@ -14,7 +14,7 @@ function write_dl(fname/*:string*/, payload/*:any*/, enc/*:?string*/) {
        if(typeof _fs !== 'undefined' && _fs.writeFileSync) return enc ? _fs.writeFileSync(fname, payload, enc) : _fs.writeFileSync(fname, payload);
        if(typeof Deno !== 'undefined') {
                /* in this spot, it's safe to assume typed arrays and TextEncoder/TextDecoder exist */
-               if(enc) switch(enc) {
+               if(enc && typeof payload == "string") switch(enc) {
                        case "utf8": payload = new TextEncoder(enc).encode(payload); break;
                        case "binary": payload = s2ab(payload); break;
                        /* TODO: binary equivalent */

Actually generating the final zip is about twice as fast now in the realm of 25K rows x 200 columns. In local tests we were able to push ~80K rows x 200 columns of numbers and small text strings before hitting the invalid string length issue in the XML generation.

The function that actually performs the download mantras is expecting a binary string. That doesn't make sense now that the zip writer builds a `Uint8Array` by default (the `writeFile` path and the `array` / `buffer` output types now convert from `Uint8Array` to binary string and back). This also applies to Deno, since `Deno.writeFileSync` only operates on `Uint8Array` The following patch should work, feel free to submit a PR: ```diff diff --git a/bits/88_write.js b/bits/88_write.js @@ -45,14 +45,15 @@ function write_zip_typeXLSX(wb/*:Workbook*/, opts/*:?WriteOpts*/)/*:any*/ { } function write_zip_denouement(z/*:any*/, o/*:?WriteOpts*/)/*:any*/ { var oopts = {}; + var ftype = has_buf ? "nodebuffer" : (typeof Uint8Array !== "undefined" ? "array" : "string"); if(o.compression) oopts.compression = 'DEFLATE'; - if(o.password) oopts.type = has_buf ? "nodebuffer" : "string"; + if(o.password) oopts.type = ftype; else switch(o.type) { case "base64": oopts.type = "base64"; break; case "binary": oopts.type = "string"; break; case "string": throw new Error("'string' output type invalid for '" + o.bookType + "' files"); case "buffer": - case "file": oopts.type = has_buf ? "nodebuffer" : "string"; break; + case "file": oopts.type = ftype; break; default: throw new Error("Unrecognized type " + o.type); } var out = z.FullPaths ? CFB.write(z, {fileType:"zip", type: /*::(*/{"nodebuffer": "buffer", "string": "binary"}/*:: :any)*/[oopts.type] || oopts.type, compression: !!o.compression}) : z.generate(oopts); diff --git a/bits/19_fsutils.js b/bits/19_fsutils.js @@ -13,7 +13,7 @@ function write_dl(fname/*:string*/, payload/*:any*/, enc/*:?string*/) { if(typeof _fs !== 'undefined' && _fs.writeFileSync) return enc ? _fs.writeFileSync(fname, payload, enc) : _fs.writeFileSync(fname, payload); if(typeof Deno !== 'undefined') { /* in this spot, it's safe to assume typed arrays and TextEncoder/TextDecoder exist */ - if(enc) switch(enc) { + if(enc && typeof payload == "string") switch(enc) { case "utf8": payload = new TextEncoder(enc).encode(payload); break; case "binary": payload = s2ab(payload); break; /* TODO: binary equivalent */ diff --git a/misc/19_mjsfs.js b/misc/19_mjsfs.js @@ -14,7 +14,7 @@ function write_dl(fname/*:string*/, payload/*:any*/, enc/*:?string*/) { if(typeof _fs !== 'undefined' && _fs.writeFileSync) return enc ? _fs.writeFileSync(fname, payload, enc) : _fs.writeFileSync(fname, payload); if(typeof Deno !== 'undefined') { /* in this spot, it's safe to assume typed arrays and TextEncoder/TextDecoder exist */ - if(enc) switch(enc) { + if(enc && typeof payload == "string") switch(enc) { case "utf8": payload = new TextEncoder(enc).encode(payload); break; case "binary": payload = s2ab(payload); break; /* TODO: binary equivalent */ ``` Actually generating the final zip is about twice as fast now in the realm of 25K rows x 200 columns. In local tests we were able to push ~80K rows x 200 columns of numbers and small text strings before hitting the invalid string length issue in the XML generation.

Sign in to join this conversation.