diff --git a/.spelling b/.spelling index 7753f25..19efca3 100644 --- a/.spelling +++ b/.spelling @@ -37,13 +37,19 @@ CommonJS Ethercalc ExtendScript FileSaver +IndexedDB JavaScriptCore +LocalStorage NPM Nuxt.js +Redis RequireJS Rollup +SessionStorage +SQLite SystemJS VueJS +WebSQL iOS nodejs npm @@ -57,10 +63,12 @@ ArrayBuffer Base64 Booleans JS +NoSQL README UTF-16 XHR XMLHttpRequest +bundler bundlers cleanroom config @@ -95,6 +103,15 @@ ui-grid - demos/angular2/README.md angular-cli + - demos/database/README.md +LowDB +MariaDB +MySQL +PostgreSQL +schemaless +schemas +storages + - demos/extendscript/README.md Photoshop minifier diff --git a/README.md b/README.md index 8916180..e24b4f8 100644 --- a/README.md +++ b/README.md @@ -196,9 +196,12 @@ The [`demos` directory](demos/) includes sample projects for: - [`vue 2.x and weex`](demos/vue/) - [`XMLHttpRequest and fetch`](demos/xhr/) - [`nodejs server`](demos/server/) +- [`databases and key/value stores`](demos/database/) **Bundlers and Tooling** - [`browserify`](demos/browserify/) +- [`fusebox`](demos/fusebox/) +- [`parcel`](demos/parcel/) - [`requirejs`](demos/requirejs/) - [`rollup`](demos/rollup/) - [`systemjs`](demos/systemjs/) diff --git a/demos/README.md b/demos/README.md index 88b9c4e..56437d1 100644 --- a/demos/README.md +++ b/demos/README.md @@ -25,9 +25,12 @@ can be installed with Bash on Windows or with `cygwin`. - [`vue 2.x and weex`](vue/) - [`XMLHttpRequest and fetch`](xhr/) - [`nodejs server`](server/) +- [`databases and key/value stores`](database/) **Bundlers and Tooling** - [`browserify`](browserify/) +- [`fusebox`](fusebox/) +- [`parcel`](parcel/) - [`requirejs`](requirejs/) - [`rollup`](rollup/) - [`systemjs`](systemjs/) diff --git a/demos/database/.eslintrc b/demos/database/.eslintrc new file mode 100644 index 0000000..d059de3 --- /dev/null +++ b/demos/database/.eslintrc @@ -0,0 +1,7 @@ +{ + "env": { "shared-node-browser":true }, + "parserOptions": { + "ecmaVersion": 2017 + }, + "plugins": [ "html", "json" ] +} diff --git a/demos/database/.gitignore b/demos/database/.gitignore new file mode 100644 index 0000000..98e6ef6 --- /dev/null +++ b/demos/database/.gitignore @@ -0,0 +1 @@ +*.db diff --git a/demos/database/LocalForage.html b/demos/database/LocalForage.html new file mode 100644 index 0000000..6534702 --- /dev/null +++ b/demos/database/LocalForage.html @@ -0,0 +1,59 @@ + + + + + + +SheetJS Live Demo + + + +
+SheetJS LocalStorage Demo
+
+Original Data:
+
+
+Output:
+
+
+ + + + + + + diff --git a/demos/database/LocalStorage.html b/demos/database/LocalStorage.html new file mode 100644 index 0000000..780184d --- /dev/null +++ b/demos/database/LocalStorage.html @@ -0,0 +1,57 @@ + + + + + + +SheetJS Live Demo + + + +
+SheetJS LocalStorage Demo
+
+Original Data:
+
+
+Output:
+
+
+ + + + + + diff --git a/demos/database/LowDBTest.js b/demos/database/LowDBTest.js new file mode 100644 index 0000000..baa8c4f --- /dev/null +++ b/demos/database/LowDBTest.js @@ -0,0 +1,23 @@ +/* xlsx.js (C) 2013-present SheetJS -- http://sheetjs.com */ +/* eslint-env node */ +var low = require('lowdb'); +var SheetJSAdapter = require('./SheetJSLowDB'); +var adapter = new SheetJSAdapter(); +var db = low(adapter); + +db.defaults({ posts: [], user: {}, count: 0 }).write(); +db.get('posts').push({ id: 1, title: 'lowdb is awesome'}).write(); +db.set('user.name', 'typicode').write(); +db.update('count', function(n) { return n + 1; }).write(); + +adapter.dumpFile('ldb1.xlsx'); + +var adapter2 = new SheetJSAdapter(); +adapter2.loadFile('ldb1.xlsx'); +var db2 = low(adapter2); + +db2.get('posts').push({ id: 2, title: 'mongodb is not'}).write(); +db2.set('user.name', 'sheetjs').write(); +db2.update('count', function(n) { return n + 1; }).write(); + +adapter2.dumpFile('ldb2.xlsx'); diff --git a/demos/database/Makefile b/demos/database/Makefile new file mode 100644 index 0000000..31433d8 --- /dev/null +++ b/demos/database/Makefile @@ -0,0 +1,16 @@ +.PHONY: init +init: + rm -f node_modules/xlsx + mkdir -p node_modules + cd node_modules; ln -s ../../../ xlsx; cd - + rm -f xlsx.full.min.js + ln -s ../../dist/xlsx.full.min.js + +FILES=$(filter-out xlsx.full.min.js,$(wildcard *.js)) $(wildcard *.html) +.PHONY: lint +lint: $(FILES) + eslint $(FILES) + +.PHONY: clean +clean: + rm -f *.db *.xlsx diff --git a/demos/database/MySQLTest.js b/demos/database/MySQLTest.js new file mode 100644 index 0000000..40d1bb0 --- /dev/null +++ b/demos/database/MySQLTest.js @@ -0,0 +1,70 @@ +/* xlsx.js (C) 2013-present SheetJS -- http://sheetjs.com */ +/* eslint-env node */ +var XLSX = require('xlsx'); +var assert = require('assert'); +var SheetJSSQL = require('./SheetJSSQL'); +var mysql = require('mysql2/promise'); + +/* Connection options (requires two databases sheetjs and sheetj5) */ +var opts = { + host : 'localhost', + user : 'SheetJS', + password : 'SheetJS', +}; + +/* Sample data table */ +var init = [ + "DROP TABLE IF EXISTS pres", + "CREATE TABLE pres (name TEXT, idx TINYINT)", + "INSERT INTO pres VALUES ('Barack Obama', 44)", + "INSERT INTO pres VALUES ('Donald Trump', 45)", + "DROP TABLE IF EXISTS fmts", + "CREATE TABLE fmts (ext TEXT, ctr TEXT, multi TINYINT)", + "INSERT INTO fmts VALUES ('XLSB', 'ZIP', 1)", + "INSERT INTO fmts VALUES ('XLS', 'CFB', 1)", + "INSERT INTO fmts VALUES ('XLML', '', 1)", + "INSERT INTO fmts VALUES ('CSV', '', 0)", +]; + +(async () => { + const conn1 = await mysql.createConnection({...opts, database: "sheetjs"}); + for(var i = 0; i < init.length; ++i) await conn1.query(init[i]); + + /* Export table to XLSX */ + var wb = XLSX.utils.book_new(); + + async function book_append_table(wb, name) { + var r_f = await conn1.query('SELECT * FROM ' + name); + var r = r_f[0]; + var ws = XLSX.utils.json_to_sheet(r); + XLSX.utils.book_append_sheet(wb, ws, name); + } + + await book_append_table(wb, "pres"); + await book_append_table(wb, "fmts"); + XLSX.writeFile(wb, "mysql.xlsx"); + + /* Capture first database info and close */ + var P1 = (await conn1.query("SELECT * FROM pres"))[0]; + var F1 = (await conn1.query("SELECT * FROM fmts"))[0]; + await conn1.close(); + + /* Import XLSX to table */ + const conn2 = await mysql.createConnection({...opts, database: "sheetj5"}); + var wb2 = XLSX.readFile("mysql.xlsx"); + var queries = SheetJSSQL.book_to_sql(wb2, "MYSQL"); + for(i = 0; i < queries.length; ++i) await conn2.query(queries[i]); + + /* Capture first database info and close */ + var P2 = (await conn2.query("SELECT * FROM pres"))[0]; + var F2 = (await conn2.query("SELECT * FROM fmts"))[0]; + await conn2.close(); + + /* Compare results */ + assert.deepEqual(P1, P2); + assert.deepEqual(F1, F2); + + /* Display results */ + console.log(P2); + console.log(F2); +})(); diff --git a/demos/database/ObjUtils.js b/demos/database/ObjUtils.js new file mode 100644 index 0000000..cb3a0c5 --- /dev/null +++ b/demos/database/ObjUtils.js @@ -0,0 +1,59 @@ +/* xlsx.js (C) 2013-present SheetJS -- http://sheetjs.com */ +/*global XLSX, module, require */ +var ObjUtils = (function() { + +var X; +if(typeof XLSX !== "undefined") X = XLSX; +else if(typeof require !== 'undefined') X = require('xlsx'); +else throw new Error("Could not find XLSX"); + +function walk(obj, key, arr) { + if(Array.isArray(obj)) return; + if(typeof obj != "object" || obj instanceof Date) { arr.push({path:key, value:obj}); return; } + Object.keys(obj).forEach(function(k) { + walk(obj[k], key ? key + "." + k : k, arr); + }); +} + +function object_to_workbook(obj) { + var wb = X.utils.book_new(); + + var base = []; walk(obj, "", base); + var ws = X.utils.json_to_sheet(base, {header:["path", "value"]}); + X.utils.book_append_sheet(wb, ws, "_keys"); + + Object.keys(obj).forEach(function(k) { + if(!Array.isArray(obj[k])) return; + X.utils.book_append_sheet(wb, X.utils.json_to_sheet(obj[k]), k); + }); + + return wb; +} + +function deepset(obj, path, value) { + if(path.indexOf(".") == -1) return obj[path] = value; + var parts = path.split("."); + if(!obj[parts[0]]) obj[parts[0]] = {}; + return deepset(obj[parts[0]], parts.slice(1).join("."), value); +} +function workbook_set_object(obj, wb) { + var ws = wb.Sheets["_keys"]; if(ws) { + var data = X.utils.sheet_to_json(ws, {raw:true}); + data.forEach(function(r) { deepset(obj, r.path, r.value); }); + } + wb.SheetNames.forEach(function(n) { + if(n == "_keys") return; + obj[n] = X.utils.sheet_to_json(wb.Sheets[n], {raw:true}); + }); +} + +function workbook_to_object(wb) { var obj = {}; workbook_set_object(obj, wb); return obj; } + +return { + workbook_to_object: workbook_to_object, + object_to_workbook: object_to_workbook, + workbook_set_object: workbook_set_object +}; +})(); + +if(typeof module !== 'undefined') module.exports = ObjUtils; diff --git a/demos/database/PgSQLTest.js b/demos/database/PgSQLTest.js new file mode 100644 index 0000000..a021ff5 --- /dev/null +++ b/demos/database/PgSQLTest.js @@ -0,0 +1,72 @@ +/* xlsx.js (C) 2013-present SheetJS -- http://sheetjs.com */ +/* eslint-env node */ +var XLSX = require('xlsx'); +var assert = require('assert'); +var SheetJSSQL = require('./SheetJSSQL'); +var Client = require('pg').Client; + +/* Connection options (requires two databases sheetjs and sheetj5) */ +var opts = { + host : 'localhost', + user : 'SheetJS', + password : 'SheetJS', +}; + +/* Sample data table */ +var init = [ + "DROP TABLE IF EXISTS pres", + "CREATE TABLE pres (name text, idx smallint)", + "INSERT INTO pres VALUES ('Barack Obama', 44)", + "INSERT INTO pres VALUES ('Donald Trump', 45)", + "DROP TABLE IF EXISTS fmts", + "CREATE TABLE fmts (ext text, ctr text, multi smallint)", + "INSERT INTO fmts VALUES ('XLSB', 'ZIP', 1)", + "INSERT INTO fmts VALUES ('XLS', 'CFB', 1)", + "INSERT INTO fmts VALUES ('XLML', '', 1)", + "INSERT INTO fmts VALUES ('CSV', '', 0)", +]; + +var conn1 = new Client({...opts, database: "sheetjs"}); +var conn2 = new Client({...opts, database: "sheetj5"}); +(async () => { + await conn1.connect(); + for(var i = 0; i < init.length; ++i) await conn1.query(init[i]); + + /* Export table to XLSX */ + var wb = XLSX.utils.book_new(); + + async function book_append_table(wb, name) { + var r_f = await conn1.query('SELECT * FROM ' + name); + var r = r_f.rows; + var ws = XLSX.utils.json_to_sheet(r); + XLSX.utils.book_append_sheet(wb, ws, name); + } + + await book_append_table(wb, "pres"); + await book_append_table(wb, "fmts"); + XLSX.writeFile(wb, "pgsql.xlsx"); + + /* Capture first database info and close */ + var P1 = (await conn1.query("SELECT * FROM pres")).rows; + var F1 = (await conn1.query("SELECT * FROM fmts")).rows; + await conn1.end(); + + /* Import XLSX to table */ + await conn2.connect(); + var wb2 = XLSX.readFile("pgsql.xlsx"); + var queries = SheetJSSQL.book_to_sql(wb2, "PGSQL"); + for(i = 0; i < queries.length; ++i) { console.log(queries[i]); await conn2.query(queries[i]); } + + /* Capture first database info and close */ + var P2 = (await conn2.query("SELECT * FROM pres")).rows; + var F2 = (await conn2.query("SELECT * FROM fmts")).rows; + await conn2.end(); + + /* Compare results */ + assert.deepEqual(P1, P2); + assert.deepEqual(F1, F2); + + /* Display results */ + console.log(P2); + console.log(F2); +})(); diff --git a/demos/database/README.md b/demos/database/README.md new file mode 100644 index 0000000..4010ee9 --- /dev/null +++ b/demos/database/README.md @@ -0,0 +1,279 @@ +# Databases + +"Database" is a catch-all term referring to traditional RDBMS as well as K/V +stores, document databases, and other "NoSQL" storages. There are many external +database systems as well as browser APIs like WebSQL and `localStorage` + +This demo discusses general strategies and provides examples for a variety of +database systems. The examples are merely intended to demonstrate very basic +functionality. + + +## Structured Tables + +Database tables are a common import and export target for spreadsheets. One +common representation of a database table is an array of JS objects whose keys +are column headers and whose values are the underlying data values. For example, + +| Name | Index | +| :----------- | ----: | +| Barack Obama | 44 | +| Donald Trump | 45 | + +is naturally represented as an array of objects + +```js +[ + { Name: "Barack Obama", Index: 44 }, + { Name: "Donald Trump", Index: 45 } +] +``` + +The `sheet_to_json` and `json_to_sheet` helper functions work with objects of +similar shape, converting to and from worksheet objects. The corresponding +worksheet would include a header row for the labels: + +``` +XXX| A | B | +---+--------------+-------+ + 1 | Name | Index | + 2 | Barack Obama | 44 | + 3 | Donald Trump | 45 | +``` + + +## Building Schemas from Worksheets + +The `sheet_to_json` helper function generates arrays of JS objects that can be +scanned to determine the column "types", and there are third-party connectors +that can push arrays of JS objects to database tables. + +The [`sexql`](http://sheetjs.com/sexql) browser demo uses WebSQL, which is +limited to the SQLite fundamental types. Its schema builder scans the first row +to find headers: + +```js + if(!ws || !ws['!ref']) return; + var range = XLSX.utils.decode_range(ws['!ref']); + if(!range || !range.s || !range.e || range.s > range.e) return; + var R = range.s.r, C = range.s.c; + + var names = new Array(range.e.c-range.s.c+1); + for(C = range.s.c; C<= range.e.c; ++C){ + var addr = XLSX.utils.encode_cell({c:C,r:R}); + names[C-range.s.c] = ws[addr] ? ws[addr].v : XLSX.utils.encode_col(C); + } +``` + +After finding the headers, a deduplication step ensures that data is not lost. +Duplicate headers will be suffixed with `_1`, `_2`, etc. + +```js + for(var i = 0; i < names.length; ++i) if(names.indexOf(names[i]) < i) + for(var j = 0; j < names.length; ++j) { + var _name = names[i] + "_" + (j+1); + if(names.indexOf(_name) > -1) continue; + names[i] = _name; + } +``` + +A column-major walk helps determine the data type. For SQLite the only relevant +data types are `REAL` and `TEXT`. If a string or date or error is seen in any +value of a column, the column is marked as `TEXT`: + +```js + var types = new Array(range.e.c-range.s.c+1); + for(C = range.s.c; C<= range.e.c; ++C) { + var seen = {}, _type = ""; + for(R = range.s.r+1; R<= range.e.r; ++R) + seen[(ws[XLSX.utils.encode_cell({c:C,r:R})]||{t:"z"}).t] = true; + if(seen.s || seen.str) _type = "TEXT"; + else if(seen.n + seen.b + seen.d + seen.e > 1) _type = "TEXT"; + else switch(true) { + case seen.b: + case seen.n: _type = "REAL"; break; + case seen.e: _type = "TEXT"; break; + case seen.d: _type = "TEXT"; break; + } + types[C-range.s.c] = _type || "TEXT"; + } +``` + +The included `SheetJSSQL.js` script demonstrates SQL statement generation. + +## Objects, K/V and "Schema-less" Databases + +So-called "Schema-less" databases allow for arbitrary keys and values within the +entries in the database. K/V stores and Objects add additional restrictions. + +There is no natural way to translate arbitrarily shaped schemas to worksheets +in a workbook. One common trick is to dedicate one worksheet to holding named +keys. For example, considering the JS object: + +```json +{ + "title": "SheetDB", + "metadata": { + "author": "SheetJS", + "code": 7262 + }, + "data": [ + { "Name": "Barack Obama", "Index": 44 }, + { "Name": "Donald Trump", "Index": 45 }, + ] +} +``` + +A dedicated worksheet should store the one-off named values: + +``` +XXX| A | B | +---+-----------------+---------+ + 1 | Path | Value | + 2 | title | SheetDB | + 3 | metadata.author | SheetJS | + 4 | metadata.code | 7262 | +``` + +The included `ObjUtils.js` script demonstrates object-workbook conversion: + +```js +function deepset(obj, path, value) { + if(path.indexOf(".") == -1) return obj[path] = value; + var parts = path.split("."); + if(!obj[parts[0]]) obj[parts[0]] = {}; + return deepset(obj[parts[0]], parts.slice(1).join("."), value); +} +function workbook_to_object(wb) { + var out = {}; + + /* assign one-off keys */ + var ws = wb.Sheets["_keys"]; if(ws) { + var data = XLSX.utils.sheet_to_json(ws, {raw:true}); + data.forEach(function(r) { deepset(out, r.path, r.value); }); + } + + /* assign arrays from worksheet tables */ + wb.SheetNames.forEach(function(n) { + if(n == "_keys") return; + out[n] = XLSX.utils.sheet_to_json(wb.Sheets[n], {raw:true}); + }); + + return out; +} + +function walk(obj, key, arr) { + if(Array.isArray(obj)) return; + if(typeof obj != "object") { arr.push({path:key, value:obj}); return; } + Object.keys(obj).forEach(function(k) { walk(obj[k], key?key+"."+k:k, arr); }); +} +function object_to_workbook(obj) { + var wb = XLSX.utils.book_new(); + + /* keyed entries */ + var base = []; walk(obj, "", base); + var ws = XLSX.utils.json_to_sheet(base, {header:["path", "value"]}); + XLSX.utils.book_append_sheet(wb, ws, "_keys"); + + /* arrays */ + Object.keys(obj).forEach(function(k) { + if(!Array.isArray(obj[k])) return; + XLSX.utils.book_append_sheet(wb, XLSX.utils.json_to_sheet(obj[k]), k); + }); + + return wb; +} +``` + + +## Browser APIs + +#### WebSQL + +WebSQL is a popular SQL-based in-browser database available on Chrome / Safari. +In practice, it is powered by SQLite, and most simple SQLite-compatible queries +work as-is in WebSQL. + +The public demo generates a database from workbook. + +#### LocalStorage and SessionStorage + +The Storage API, encompassing `localStorage` and `sessionStorage`, describes +simple key-value stores that only support string values and keys. Objects can be +stored as JSON using `JSON.stringify` and `JSON.parse` to set and get keys. + +`SheetJSStorage.js` extends the `Storage` prototype with a `load` function to +populate the db based on an object and a `dump` function to generate a workbook +from the data in the storage. `LocalStorage.html` tests `localStorage`. + +#### IndexedDB + +IndexedDB is a more complex storage solution, but the `localForage` wrapper +supplies a Promise-based interface mimicking the `Storage` API. + +`SheetJSForage.js` extends the `localforage` object with a `load` function to +populate the db based on an object and a `dump` function to generate a workbook +from the data in the storage. `LocalForage.html` forces IndexedDB mode. + + +## External Database Demos + +### SQL Databases + +There are nodejs connector libraries for all of the popular RDBMS systems. They +have facilities for connecting to a database, executing queries, and obtaining +results as arrays of JS objects that can be passed to `json_to_sheet`. The main +differences surround API shape and supported data types. + +#### SQLite + +[The `better-sqlite3` module](https://www.npmjs.com/package/better-sqlite3) +provides a very simple API for working with SQLite databases. `Statement#all` +runs a prepared statement and returns an array of JS objects + +`SQLiteTest.js` generates a simple two-table SQLite database (`SheetJS1.db`), +exports to XLSX (`sqlite.xlsx`), imports the new XLSX file to a new database +(`SheetJS2.db`) and verifies the tables are preserved. + +#### MySQL / MariaDB + +[The `mysql2` module](https://www.npmjs.com/package/mysql2) supplies a callback +API as well as a Promise wrapper. `Connection#query` runs a statement and +returns an array whose first element is an array of JS objects. + +`MySQLTest.js` connects to the MySQL instance running on `localhost`, builds two +tables in the `sheetjs` database, exports to XLSX, imports the new XLSX file to +the `sheetj5` database and verifies the tables are preserved. + +#### PostgreSQL + +[The `pg` module](https://www.npmjs.com/package/pg) supplies a Promise wrapper. +Like with `mysql2`, `Client#query` runs a statement and returns a result object. +The `rows` key of the object is an array of JS objects. + +`PgSQLTest.js` connects to the PostgreSQL server on `localhost`, builds two +tables in the `sheetjs` database, exports to XLSX, imports the new XLSX file to +the `sheetj5` database and verifies the tables are preserved. + +### Key/Value Stores + +#### Redis + +Redis is a powerful data structure server that can store simple strings, sets, +sorted sets, hashes and lists. One simple database representation stores the +strings in a special worksheet (`_strs`), the manifest in another worksheet +(`_manifest`), and each object in its own worksheet (`obj##`). + +`RedisTest.js` connects to a local Redis server, populates data based on the +official Redis tutorial, exports to XLSX, flushes the server, imports the new +XLSX file and verifies the data round-tripped correctly. `SheetJSRedis.js` +includes the implementation details + +#### LowDB + +LowDB is a small schemaless database powered by `lodash`. `_.get` and `_.set` +helper functions make storing metadata a breeze. The included `SheetJSLowDB.js` +script demonstrates a simple adapter that can load and dump data. + + +[![Analytics](https://ga-beacon.appspot.com/UA-36810333-1/SheetJS/js-xlsx?pixel)](https://github.com/SheetJS/js-xlsx) diff --git a/demos/database/RedisTest.js b/demos/database/RedisTest.js new file mode 100644 index 0000000..e0307f6 --- /dev/null +++ b/demos/database/RedisTest.js @@ -0,0 +1,50 @@ +/* xlsx.js (C) 2013-present SheetJS -- http://sheetjs.com */ +/* eslint-env node */ +var XLSX = require("xlsx"); +var SheetJSRedis = require("./SheetJSRedis"); +var assert = require('assert'); +var redis = require("redis"), util = require("util"); +var client = redis.createClient(); + + +/* Sample data */ +var init = [ + ["FLUSHALL", []], + ["SADD", ["birdpowers", "flight", "pecking"]], + ["SET", ["foo", "bar"]], + ["SET", ["baz", 0]], + ["RPUSH", ["friends", "sam", "alice", "bob"]], + ["ZADD", ["hackers", 1906, 'Grace Hopper', 1912, 'Alan Turing', 1916, 'Claude Shannon', 1940, 'Alan Kay', 1953, 'Richard Stallman', 1957, 'Sophie Wilson', 1965, 'Yukihiro Matsumoto', 1969, 'Linus Torvalds']], + ["SADD", ["superpowers", "flight", 'x-ray vision']], + ["HMSET", ["user:1000", "name", 'John Smith', "email", 'john.smith@example.com', "password", "s3cret", "visits", 1]], + ["HMSET", ["user:1001", "name", 'Mary Jones', "email", 'mjones@example.com', "password", "hidden"]] +]; + +const R = (()=>{ + const Rcache = {}; + const R_ = (n) => Rcache[n] || (Rcache[n] = util.promisify(client[n]).bind(client)); + return (n) => R_(n.toLowerCase()); +})(); + +(async () => { + for(var i = 0; i < init.length; ++i) await R(init[i][0])(init[i][1]); + + /* Export database to XLSX */ + var wb = await SheetJSRedis.redis_to_wb(R); + XLSX.writeFile(wb, "redis.xlsx"); + + /* Import XLSX to database */ + await R("flushall")(); + var wb2 = XLSX.readFile("redis.xlsx"); + await SheetJSRedis.wb_to_redis(wb2, R); + + /* Verify */ + assert.equal(await R("get")("foo"), "bar"); + assert.equal(await R("lindex")("friends", 1), "alice"); + assert.equal(await R("zscore")("hackers", "Claude Shannon"), 1916); + assert.equal(await R("hget")("user:1000", "name"), "John Smith"); + assert.equal(await R("sismember")("superpowers", "flight"), "1"); + assert.equal(await R("sismember")("birdpowers", "pecking"), "1"); + + client.quit(); +})(); diff --git a/demos/database/SQLiteTest.js b/demos/database/SQLiteTest.js new file mode 100644 index 0000000..5b2500a --- /dev/null +++ b/demos/database/SQLiteTest.js @@ -0,0 +1,52 @@ +/* xlsx.js (C) 2013-present SheetJS -- http://sheetjs.com */ +/* eslint-env node */ +var XLSX = require('xlsx'); +var assert = require('assert'); +var SheetJSSQL = require('./SheetJSSQL'); +var Database = require('better-sqlite3'); +var db1 = new Database('SheetJS1.db'); + +/* Sample data table */ +var init = [ + "DROP TABLE IF EXISTS pres", + "CREATE TABLE pres (name TEXT, idx INTEGER)", + "INSERT INTO pres VALUES ('Barack Obama', 44)", + "INSERT INTO pres VALUES ('Donald Trump', 45)", + "DROP TABLE IF EXISTS fmts", + "CREATE TABLE fmts (ext TEXT, ctr TEXT, multi INTEGER)", + "INSERT INTO fmts VALUES ('XLSB', 'ZIP', 1)", + "INSERT INTO fmts VALUES ('XLS', 'CFB', 1)", + "INSERT INTO fmts VALUES ('XLML', '', 1)", + "INSERT INTO fmts VALUES ('CSV', '', 0)", +]; +db1.exec(init.join(";")); + +/* Export table to XLSX */ +var wb = XLSX.utils.book_new(); +function book_append_table(wb, db, name) { + var r = db.prepare('SELECT * FROM ' + name).all(); + var ws = XLSX.utils.json_to_sheet(r); + XLSX.utils.book_append_sheet(wb, ws, name); +} +book_append_table(wb, db1, "pres"); +book_append_table(wb, db1, "fmts"); +XLSX.writeFile(wb, "sqlite.xlsx"); + +/* Import XLSX to table */ +var db2 = new Database('SheetJS2.db'); +var wb2 = XLSX.readFile("sqlite.xlsx"); +var queries = SheetJSSQL.book_to_sql(wb2, "SQLITE"); +queries.forEach(function(q) { db2.exec(q); }); + +/* Compare databases */ +var P1 = db1.prepare("SELECT * FROM pres").all(); +var P2 = db2.prepare("SELECT * FROM pres").all(); +var F1 = db1.prepare("SELECT * FROM fmts").all(); +var F2 = db2.prepare("SELECT * FROM fmts").all(); +assert.deepEqual(P1, P2); +assert.deepEqual(F1, F2); + +/* Display results */ +console.log(P2); +console.log(F2); + diff --git a/demos/database/SheetJSForage.js b/demos/database/SheetJSForage.js new file mode 100644 index 0000000..39b0a65 --- /dev/null +++ b/demos/database/SheetJSForage.js @@ -0,0 +1,20 @@ +/* xlsx.js (C) 2013-present SheetJS -- http://sheetjs.com */ +/*global ObjUtils, localforage */ +localforage.load = async function foo(data) { + var keys = Object.keys(data); + for(var i = 0; i < keys.length; ++i) { + var key = keys[i], val = JSON.stringify(data[keys[i]]) + await localforage.setItem(key, val); + } +}; + +localforage.dump = async function() { + var obj = {}; + var length = await localforage.length(); + for(var i = 0; i < length; ++i) { + var key = await this.key(i); + var val = await this.getItem(key); + obj[key] = JSON.parse(val); + } + return ObjUtils.object_to_workbook(obj); +}; diff --git a/demos/database/SheetJSLowDB.js b/demos/database/SheetJSLowDB.js new file mode 100644 index 0000000..0c7906f --- /dev/null +++ b/demos/database/SheetJSLowDB.js @@ -0,0 +1,20 @@ +/* xlsx.js (C) 2013-present SheetJS -- http://sheetjs.com */ +/* eslint-env node */ + +var XLSX = require('xlsx'); +var ObjUtils = require('./ObjUtils'); + +function SheetJSAdapter() { this.defaultValue = {}; }; + +SheetJSAdapter.prototype.read = function() { return this.defaultValue; }; +SheetJSAdapter.prototype.write = function(/*data*/) {}; + +SheetJSAdapter.prototype.dumpRaw = function() { return ObjUtils.object_to_workbook(this.defaultValue); }; +SheetJSAdapter.prototype.dump = function(options) { XLSX.write(this.dumpRaw(), options); }; +SheetJSAdapter.prototype.dumpFile = function(path, options) { XLSX.writeFile(this.dumpRaw(), path, options); }; + +SheetJSAdapter.prototype.loadRaw = function(wb) { ObjUtils.workbook_set_object(this.defaultValue, wb); }; +SheetJSAdapter.prototype.load = function(data, options) { this.loadRaw(XLSX.read(data, options)); }; +SheetJSAdapter.prototype.loadFile = function(path, options) { this.loadRaw(XLSX.readFile(path, options)); }; + +if(typeof module !== 'undefined') module.exports = SheetJSAdapter; diff --git a/demos/database/SheetJSRedis.js b/demos/database/SheetJSRedis.js new file mode 100644 index 0000000..8f5b3f0 --- /dev/null +++ b/demos/database/SheetJSRedis.js @@ -0,0 +1,73 @@ +/* xlsx.js (C) 2013-present SheetJS -- http://sheetjs.com */ +/* eslint-env node */ +var XLSX = require("xlsx"); + +const pair = (arr) => arr.map((x,i)=>!(i%2)&&[x,+arr[i+1]]).filter(x=>x); +const keyify = (obj) => Object.keys(obj).map(x => [x, obj[x]]); + +async function redis_to_wb(R) { + var wb = XLSX.utils.book_new(); + var manifest = [], strs = []; + + /* store strings in strs and keep note of other objects in manifest */ + var keys = await R("keys")("*"), type = ""; + for(var i = 0; i < keys.length; ++i) { + type = await R("type")(keys[i]); + switch(type) { + case "string": strs.push({key:keys[i], value: await R("get")(keys[i])}); break; + case "list": case "zset": case "set": case "hash": manifest.push({key:keys[i], type:type}); break; + default: throw new Error("bad type " + type); + } + } + + /* add worksheets if relevant */ + if(strs.length > 0) { + var wss = XLSX.utils.json_to_sheet(strs, {header: ["key", "value"], skipHeader:1}); + XLSX.utils.book_append_sheet(wb, wss, "_strs"); + } + if(manifest.length > 0) { + var wsm = XLSX.utils.json_to_sheet(manifest, {header: ["key", "type"]}); + XLSX.utils.book_append_sheet(wb, wsm, "_manifest"); + } + for(i = 0; i < manifest.length; ++i) { + var sn = "obj" + i; + var aoa, key = manifest[i].key; + switch((type=manifest[i].type)) { + case "list": + aoa = (await R("lrange")(key, 0, -1)).map(x => [x]); break; + case "set": + aoa = (await R("smembers")(key)).map(x => [x]); break; + case "zset": + aoa = pair(await R("zrange")(key, 0, -1, "withscores")); break; + case "hash": + aoa = keyify(await R("hgetall")(key)); break; + default: throw new Error("bad type " + type); + } + XLSX.utils.book_append_sheet(wb, XLSX.utils.aoa_to_sheet(aoa), sn); + } + return wb; +} + +/* convert worksheet aoa to specific redis type */ +const aoa_to_redis = { + list: async (aoa, R, key) => await R("RPUSH")([key].concat(aoa.map(x=>x[0]))), + zset: async (aoa, R, key) => await R("ZADD" )([key].concat(aoa.reduce((acc,x)=>acc.concat([+x[1], x[0]]), []))), + hash: async (aoa, R, key) => await R("HMSET")([key].concat(aoa.reduce((acc,x)=>acc.concat(x), []))), + set: async (aoa, R, key) => await R("SADD" )([key].concat(aoa.map(x=>x[0]))) +}; +async function wb_to_redis(wb, R) { + if(wb.Sheets._strs) { + var strs = XLSX.utils.sheet_to_json(wb.Sheets._strs, {header:1}); + for(var i = 0; i < strs.length; ++i) await R("SET")(strs[i]); + } + if(!wb.Sheets._manifest) return; + var M = XLSX.utils.sheet_to_json(wb.Sheets._manifest); + for(i = 0; i < M.length; ++i) { + var aoa = XLSX.utils.sheet_to_json(wb.Sheets["obj" + i], {header:1}); + await aoa_to_redis[M[i].type](aoa, R, M[i].key); + } +} +module.exports = { + redis_to_wb, + wb_to_redis +}; diff --git a/demos/database/SheetJSSQL.js b/demos/database/SheetJSSQL.js new file mode 100644 index 0000000..02eb9d4 --- /dev/null +++ b/demos/database/SheetJSSQL.js @@ -0,0 +1,88 @@ +/* xlsx.js (C) 2013-present SheetJS -- http://sheetjs.com */ +var SheetJSSQL = (function() { + +var X; +if(typeof XLSX !== "undefined") X = XLSX; +else if(typeof require !== 'undefined') X = require('xlsx'); +else throw new Error("Could not find XLSX"); + +var _TYPES = { + "PGSQL": { t:"text", n:"float8", d:"timestamp", b:"boolean" }, + "MYSQL": { t:"TEXT", n:"REAL", d:"DATETIME", b:"TINYINT" }, + "SQLITE": { t:"TEXT", n:"REAL", d:"TEXT", b:"REAL" } +} +function sheet_to_sql(ws, sname, mode) { + var TYPES = _TYPES[mode || "SQLITE"] + if(!ws || !ws['!ref']) return; + var range = X.utils.decode_range(ws['!ref']); + if(!range || !range.s || !range.e || range.s > range.e) return; + var R = range.s.r, C = range.s.c; + + var names = new Array(range.e.c-range.s.c+1); + for(C = range.s.c; C<= range.e.c; ++C){ + var addr = X.utils.encode_cell({c:C,r:R}); + names[C-range.s.c] = ws[addr] ? ws[addr].v : X.utils.encode_col(C); + } + + for(var i = 0; i < names.length; ++i) if(names.indexOf(names[i]) < i) + for(var j = 0; j < names.length; ++j) { + var _name = names[i] + "_" + (j+1); + if(names.indexOf(_name) > -1) continue; + names[i] = _name; + } + + var types = new Array(range.e.c-range.s.c+1); + for(C = range.s.c; C<= range.e.c; ++C) { + var seen = {}, _type = ""; + for(R = range.s.r+1; R<= range.e.r; ++R) + seen[(ws[X.utils.encode_cell({c:C,r:R})]||{t:"z"}).t] = true; + if(seen.s || seen.str) _type = TYPES.t; + else if(seen.n + seen.b + seen.d + seen.e > 1) _type = TYPES.t; + else switch(true) { + case seen.b: _type = TYPES.b; break; + case seen.n: _type = TYPES.n; break; + case seen.e: _type = TYPES.t; break; + case seen.d: _type = TYPES.d; break; + } + types[C-range.s.c] = _type || TYPES.t; + } + + var out = []; + + var BT = mode == "PGSQL" ? "" : "`"; + var Q = mode == "PGSQL" ? "'" : '"'; + var R = mode == "PGSQL" ? /'/g : /"/g; + out.push("DROP TABLE IF EXISTS " + BT + sname + BT ); + out.push("CREATE TABLE " + BT + sname + BT + " (" + names.map(function(n, i) { return BT + n + BT + " " + (types[i]||"TEXT"); }).join(", ") + ");" ); + + for(R = range.s.r+1; R<= range.e.r; ++R) { + var fields = [], values = []; + for(C = range.s.c; C<= range.e.c; ++C) { + var cell = ws[X.utils.encode_cell({c:C,r:R})]; + if(!cell) continue; + fields.push(BT + names[C-range.s.c] + BT); + var val = cell.v; + switch(types[C-range.s.c]) { + case TYPES.n: if(cell.t == 'b' || typeof val == 'boolean' ) val = +val; break; + default: val = Q + val.toString().replace(R, Q + Q) + Q; + } + values.push(val); + } + out.push("INSERT INTO " + BT +sname+ BT + " (" + fields.join(", ") + ") VALUES (" + values.join(",") + ");"); + } + + return out; +} + +function book_to_sql(wb, mode) { + return wb.SheetNames.reduce(function(acc, n) { + return acc.concat(sheet_to_sql(wb.Sheets[n], n, mode)); + }, []); +} + +return { + book_to_sql: book_to_sql, + sheet_to_sql: sheet_to_sql +}; +})(); +if(typeof module !== 'undefined') module.exports = SheetJSSQL; diff --git a/demos/database/SheetJSStorage.js b/demos/database/SheetJSStorage.js new file mode 100644 index 0000000..3e5eac4 --- /dev/null +++ b/demos/database/SheetJSStorage.js @@ -0,0 +1,18 @@ +/* xlsx.js (C) 2013-present SheetJS -- http://sheetjs.com */ +/* eslint-env browser */ +/*global ObjUtils */ +Storage.prototype.load = function(data) { + var self = this; + Object.keys(data).forEach(function(k) { + self.setItem(k, JSON.stringify(data[k])); + }); +}; + +Storage.prototype.dump = function() { + var obj = {}; + for(var i = 0; i < this.length; ++i) { + var key = this.key(i); + obj[key] = JSON.parse(this.getItem(key)); + } + return ObjUtils.object_to_workbook(obj); +}; diff --git a/demos/database/xlsx.full.min.js b/demos/database/xlsx.full.min.js new file mode 120000 index 0000000..dbca48d --- /dev/null +++ b/demos/database/xlsx.full.min.js @@ -0,0 +1 @@ +../../dist/xlsx.full.min.js \ No newline at end of file diff --git a/demos/fusebox/README.md b/demos/fusebox/README.md index 6cbeb57..424be6c 100644 --- a/demos/fusebox/README.md +++ b/demos/fusebox/README.md @@ -48,7 +48,7 @@ The native shims must be suppressed for browser usage: ```js const fuse = FuseBox.init({ homeDir: ".", - target: "node", + target: "browser", natives: { Buffer: false, stream: false, diff --git a/demos/fusebox/index.html b/demos/fusebox/index.html new file mode 100644 index 0000000..d657400 --- /dev/null +++ b/demos/fusebox/index.html @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/demos/parcel/.gitignore b/demos/parcel/.gitignore new file mode 100644 index 0000000..ba70f38 --- /dev/null +++ b/demos/parcel/.gitignore @@ -0,0 +1,2 @@ +.cache +dist diff --git a/demos/parcel/Makefile b/demos/parcel/Makefile new file mode 100644 index 0000000..b6678ae --- /dev/null +++ b/demos/parcel/Makefile @@ -0,0 +1,7 @@ +.PHONY: app +app: + parcel build index.html --public-url ./ + +.PHONY: ctest +ctest: + parcel index.html diff --git a/demos/parcel/README.md b/demos/parcel/README.md new file mode 100644 index 0000000..9966785 --- /dev/null +++ b/demos/parcel/README.md @@ -0,0 +1,14 @@ +# Parcel + +Parcel Bundler starting from version 1.5.0 should play nice with this library +out of the box. The standard import form can be used in JS files: + +```js +import XLSX from 'xlsx' +``` + +Errors of the form `Could not statically evaluate fs call` stem from a parcel +[bug](https://github.com/parcel-bundler/parcel/pull/523#issuecomment-357486164). +Upgrade to version 1.5.0 or later. + +[![Analytics](https://ga-beacon.appspot.com/UA-36810333-1/SheetJS/js-xlsx?pixel)](https://github.com/SheetJS/js-xlsx) diff --git a/demos/parcel/index.html b/demos/parcel/index.html new file mode 100644 index 0000000..50ee3d0 --- /dev/null +++ b/demos/parcel/index.html @@ -0,0 +1,50 @@ + + + + + + +SheetJS Live Demo + + + +
+SheetJS Data Preview Live Demo
+(Base64 text works back to IE6; drag and drop works back to IE10)
+
+Source Code Repo
+Issues?  Something look weird?  Click here and report an issue
+Output Format: 
+
Drop a spreadsheet file here to see sheet data
+ ... or click here to select a file + + +
+Advanced Demo Options: +Use readAsBinaryString: (when available) +
+

+
+
+ + + diff --git a/demos/parcel/index.js b/demos/parcel/index.js new file mode 100644 index 0000000..2486570 --- /dev/null +++ b/demos/parcel/index.js @@ -0,0 +1,136 @@ +/* xlsx.js (C) 2013-present SheetJS -- http://sheetjs.com */ +/*jshint browser:true */ +import X from '../../' + +console.log(X.version); + +var global_wb; + +var process_wb = (function() { + var OUT = document.getElementById('out'); + var HTMLOUT = document.getElementById('htmlout'); + + var get_format = (function() { + var radios = document.getElementsByName( "format" ); + return function() { + for(var i = 0; i < radios.length; ++i) if(radios[i].checked || radios.length === 1) return radios[i].value; + }; + })(); + + var to_json = function to_json(workbook) { + var result = {}; + workbook.SheetNames.forEach(function(sheetName) { + var roa = X.utils.sheet_to_json(workbook.Sheets[sheetName]); + if(roa.length) result[sheetName] = roa; + }); + return JSON.stringify(result, 2, 2); + }; + + var to_csv = function to_csv(workbook) { + var result = []; + workbook.SheetNames.forEach(function(sheetName) { + var csv = X.utils.sheet_to_csv(workbook.Sheets[sheetName]); + if(csv.length){ + result.push("SHEET: " + sheetName); + result.push(""); + result.push(csv); + } + }); + return result.join("\n"); + }; + + var to_fmla = function to_fmla(workbook) { + var result = []; + workbook.SheetNames.forEach(function(sheetName) { + var formulae = X.utils.get_formulae(workbook.Sheets[sheetName]); + if(formulae.length){ + result.push("SHEET: " + sheetName); + result.push(""); + result.push(formulae.join("\n")); + } + }); + return result.join("\n"); + }; + + var to_html = function to_html(workbook) { + HTMLOUT.innerHTML = ""; + workbook.SheetNames.forEach(function(sheetName) { + var htmlstr = X.write(workbook, {sheet:sheetName, type:'binary', bookType:'html'}); + HTMLOUT.innerHTML += htmlstr; + }); + return ""; + }; + + return function process_wb(wb) { + global_wb = wb; + var output = ""; + switch(get_format()) { + case "form": output = to_fmla(wb); break; + case "html": output = to_html(wb); break; + case "json": output = to_json(wb); break; + default: output = to_csv(wb); + } + if(OUT.innerText === undefined) OUT.textContent = output; + else OUT.innerText = output; + if(typeof console !== 'undefined') console.log("output", new Date()); + }; +})(); + +var setfmt = window.setfmt = function setfmt() { if(global_wb) process_wb(global_wb); }; + +var b64it = window.b64it = (function() { + var tarea = document.getElementById('b64data'); + return function b64it() { + if(typeof console !== 'undefined') console.log("onload", new Date()); + var wb = X.read(tarea.value, {type:'base64', WTF:false}); + process_wb(wb); + }; +})(); + +var do_file = (function() { + var rABS = typeof FileReader !== "undefined" && (FileReader.prototype||{}).readAsBinaryString; + var domrabs = document.getElementsByName("userabs")[0]; + if(!rABS) domrabs.disabled = !(domrabs.checked = false); + + return function do_file(files) { + rABS = domrabs.checked; + var f = files[0]; + var reader = new FileReader(); + reader.onload = function(e) { + if(typeof console !== 'undefined') console.log("onload", new Date(), rABS); + var data = e.target.result; + if(!rABS) data = new Uint8Array(data); + process_wb(X.read(data, {type: rABS ? 'binary' : 'array'})); + }; + if(rABS) reader.readAsBinaryString(f); + else reader.readAsArrayBuffer(f); + }; +})(); + +(function() { + var drop = document.getElementById('drop'); + if(!drop.addEventListener) return; + + function handleDrop(e) { + e.stopPropagation(); + e.preventDefault(); + do_file(e.dataTransfer.files); + } + + function handleDragover(e) { + e.stopPropagation(); + e.preventDefault(); + e.dataTransfer.dropEffect = 'copy'; + } + + drop.addEventListener('dragenter', handleDragover, false); + drop.addEventListener('dragover', handleDragover, false); + drop.addEventListener('drop', handleDrop, false); +})(); + +(function() { + var xlf = document.getElementById('xlf'); + if(!xlf.addEventListener) return; + function handleFile(e) { do_file(e.target.files); } + xlf.addEventListener('change', handleFile, false); +})(); diff --git a/docbits/11_demos.md b/docbits/11_demos.md index 5d3d2d2..c0f728e 100644 --- a/docbits/11_demos.md +++ b/docbits/11_demos.md @@ -10,9 +10,12 @@ The [`demos` directory](demos/) includes sample projects for: - [`vue 2.x and weex`](demos/vue/) - [`XMLHttpRequest and fetch`](demos/xhr/) - [`nodejs server`](demos/server/) +- [`databases and key/value stores`](demos/database/) **Bundlers and Tooling** - [`browserify`](demos/browserify/) +- [`fusebox`](demos/fusebox/) +- [`parcel`](demos/parcel/) - [`requirejs`](demos/requirejs/) - [`rollup`](demos/rollup/) - [`systemjs`](demos/systemjs/) diff --git a/misc/docs/README.md b/misc/docs/README.md index 2553a21..8008c0e 100644 --- a/misc/docs/README.md +++ b/misc/docs/README.md @@ -187,9 +187,12 @@ The [`demos` directory](demos/) includes sample projects for: - [`vue 2.x and weex`](demos/vue/) - [`XMLHttpRequest and fetch`](demos/xhr/) - [`nodejs server`](demos/server/) +- [`databases and key/value stores`](demos/database/) **Bundlers and Tooling** - [`browserify`](demos/browserify/) +- [`fusebox`](demos/fusebox/) +- [`parcel`](demos/parcel/) - [`requirejs`](demos/requirejs/) - [`rollup`](demos/rollup/) - [`systemjs`](demos/systemjs/)