This commit is contained in:
SheetJS 2022-08-15 05:30:09 -04:00
parent c2433e1c83
commit d5b838993d
4 changed files with 268 additions and 12 deletions

@ -3,6 +3,9 @@ sidebar_position: 11
title: NoSQL Data Stores
---
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
So-called "Schema-less" databases allow for arbitrary keys and values within the
entries in the database. K/V stores and Objects add additional restrictions.
@ -52,10 +55,15 @@ Redis has 5 core data types: "String", List", "Set", "Sorted Set", and "Hash".
Since the keys and values are limited to simple strings (and numbers), it is
possible to store complete databases in a single worksheet.
<details open><summary><b>Sample Mapping</b> (click to hide)</summary>
![SheetJSRedis.xlsx](pathname:///nosql/sheetjsredis.png)
#### Mapping
The first row holds the data type and the second row holds the property name.
<Tabs>
<TabItem value="strings" label="Strings">
Strings can be stored in a unified String table. The first column holds keys
and the second column holds values:
@ -68,19 +76,68 @@ XXX| A | B |
4 | Sheet | JS |
```
Lists and Sets are unidimensional and can be stored in their own columns. The
second row holds the list name:
The SheetJS array-of-arrays representation of the string table is an array of
key/value pairs:
```
XXX| C | D |
---+---------+-------+
1 | List | Set |
2 | List1 | Set1 |
3 | List1V1 | Set1A |
4 | List1V2 | Set1B |
```js
let aoa = ["Strings"]; aoa.length = 2; // [ "Strings", empty ]
const keys = await client.KEYS("*");
for(let key of keys) {
const type = await client.TYPE(key);
if(type == "string") aoa.push([key, await client.GET(key)]);
}
```
Sorted Sets have an associated score which can be stored in the second column:
</TabItem>
<TabItem value="list" label="Lists">
Lists are unidimensional and can be stored in their own columns.
```
XXX| C |
---+---------+
1 | List |
2 | List1 |
3 | List1V1 |
4 | List1V2 |
```
The SheetJS array-of-arrays representation of lists is a column of values.
```js
if(type == "list") {
let values = await client.LRANGE(key, 0, -1);
aoa = [ ["List"], [key] ].concat(values.map(v => [v]));
}
```
</TabItem>
<TabItem value="set" label="Sets">
Sets are unidimensional and can be stored in their own columns.
```
XXX| D |
---+-------+
1 | Set |
2 | Set1 |
3 | Set1A |
4 | Set1B |
```
The SheetJS array-of-arrays representation of sets is a column of values.
```js
if(type == "set") {
let values = await client.SMEMBERS(key);
aoa = [ ["Set"], [key] ].concat(values.map(v => [v]));
}
```
</TabItem>
<TabItem value="zset" label="Sorted Sets">
Sorted Sets have an associated score which can be stored in the second column.
```
XXX| E | F |
@ -91,7 +148,19 @@ XXX| E | F |
4 | Key2 | 2 |
```
Hashes are stored like the string table, with key and value columns in order:
The SheetJS array-of-arrays representation is an array of key/score pairs.
```js
if(type == "zset") {
let values = await client.ZRANGE_WITHSCORES(key, 0, -1);
aoa = [ ["Sorted"], [key] ].concat(values.map(v => [v.value, v.score]));
}
```
</TabItem>
<TabItem value="hashes" label="Hashes">
Hashes are stored like the string table, with key and value columns in order.
```
XXX| G | H |
@ -102,4 +171,38 @@ XXX| G | H |
4 | Key2 | Val2 |
```
The SheetJS array-of-arrays representation is an array of key/value pairs.
```js
if(type == "hash") {
let values = await client.HGETALL(key);
aoa = [ ["Hash"], [key] ].concat(Object.entries(values));
}
```
</TabItem>
</Tabs>
#### Example
<details><summary><b>Complete Example</b> (click to show)</summary>
0) Set up and start a local Redis server
1) Download the following scripts:
- [`SheetJSRedis.mjs`](pathname:///nosql/SheetJSRedis.mjs)
- [`SheetJSRedisTest.mjs`](pathname:///nosql/SheetJSRedisTest.mjs)
2) Install dependencies and run:
```bash
npm i --save https://cdn.sheetjs.com/xlsx-latest/xlsx-latest.tgz redis
node SheetJSRedisTest.mjs
```
Inspect the output and compare with the data in `SheetJSRedisTest.mjs`.
Open `SheetJSRedis.xlsx` and verify the columns have the correct data
</details>

@ -0,0 +1,98 @@
/* sheetjs (C) 2013-present SheetJS -- https://sheetjs.com */
import { utils } from "xlsx";
/* Generate worksheet from database */
async function redis_to_ws(client) {
/* Get a list of every item in the database */
const keys = await client.KEYS("*");
/* Collect the full contents */
const data = [];
for(let key of keys) {
const type = await client.TYPE(key);
let value;
switch(type) {
case "string": value = await client.GET(key); break;
case "list": value = await client.LRANGE(key, 0, -1); break;
case "set": value = await client.SMEMBERS(key); break;
case "zset": value = await client.ZRANGE_WITHSCORES(key, 0, -1); break;
case "hash": value = await client.HGETALL(key); break;
default: console.warn(`unsupported type ${type}`); break
}
data.push({key, type, value});
}
/* Create a new worksheet and add the string table */
const ws = utils.aoa_to_sheet([["Strings"]]);
utils.sheet_add_aoa(ws, data.filter(r => r.type == "string").map(r => [r.key, r.value]), {origin: "A3"});
/* Add the other types */
let C = 2;
data.forEach(row => {
switch(row.type) {
case "set":
case "list": {
/* `value` is an array. aoa prepends type and key, then transposes to make a column */
var aoa = [row.type == "set" ? "Set" : "List", row.key].concat(row.value).map(x => ([x]));
utils.sheet_add_aoa(ws, aoa, {origin: utils.encode_col(C) + "1"});
++C;
} break;
case "zset": {
/* `value` is an object with value/score keys. generate array with map and prepend metadata */
var aoa = [["Sorted"], [row.key]].concat(row.value.map(r => ([r.value, r.score])));
utils.sheet_add_aoa(ws, aoa, {origin: utils.encode_col(C) + "1"});
C += 2;
} break;
case "hash": {
/* `value` is an object. Object.entries returns an array of arrays */
var aoa = [["Hash"], [row.key]].concat(Object.entries(row.value));
utils.sheet_add_aoa(ws, aoa, {origin: utils.encode_col(C) + "1"});
C += 2;
} break;
case "string": break;
}
});
return ws;
}
/* Generate array of Redis commands */
async function ws_to_redis(ws) {
const cmds = [];
/* Extract data from the worksheet */
const aoa = utils.sheet_to_json(ws, { header: 1 });
/* Iterate over the values in the first row */
aoa[0].forEach((type, C) => {
/* The name is exepcted in the second row, same column, unless type is Strings */
if(type != "Strings" && !aoa[1][C]) throw new Error(`Column ${utils.encode_col(C)} missing name!`)
switch(type) {
case "Strings": {
if(aoa[0][C+1]) throw new Error(`${type} requires 2 columns!`);
/* For each row starting with SheetJS 2, check if key and value are present */
for(let R = 2; R < aoa.length; ++R)
if(aoa[R]?.[C] != null && aoa[R]?.[C+1] != null)
/* When key and value are present, emit a SET command */
cmds.push(["SET", [aoa[R][C], aoa[R][C+1]]]);
} break;
case "Set":
case "List": {
/* SADD (Set) / RPUSH (List) second argument is an array of values to add to the set */
cmds.push([ type == "Set" ? "SADD" : "RPUSH", [ aoa[1][C], aoa.slice(2).map(r => r[C]).filter(x => x != null) ]]);
} break;
case "Sorted": {
/* ZADD second argument is an array of objects with `value` and `score` */
if(aoa[0][C+1]) throw new Error(`${type} requires 2 columns!`);
cmds.push([ "ZADD", [ aoa[1][C], aoa.slice(2).map(r => ({value: r[C], score:r[C+1]})).filter(x => x.value != null && x.score != null) ]]);
} break;
case "Hash": {
/* HSET second argument is an object. `Object.fromEntries` generates object from an array of K/V pairs */
if(aoa[0][C+1]) throw new Error(`${type} requires 2 columns!`);
cmds.push([ "HSET", [ aoa[1][C], Object.fromEntries(aoa.slice(2).map(r => [r[C], r[C+1]]).filter(x => x[0] != null && x[1] != null)) ]]);
} break;
default: console.error(`Unrecognized column type ${type}`); break;
}
})
return cmds;
}
export { ws_to_redis, redis_to_ws };

@ -0,0 +1,55 @@
/* sheetjs (C) 2013-present SheetJS -- https://sheetjs.com */
import { utils, writeFile, set_fs } from "xlsx";
import { createClient } from "redis";
import { ws_to_redis, redis_to_ws } from "./SheetJSRedis.mjs";
import * as fs from 'fs';
set_fs(fs);
const client = createClient();
client.on("error", err => console.error("REDIS", err));
await client.connect();
/* This data is based on the Try Redis tutorial */
var init = [
["FLUSHALL", []],
["SADD", ["birdpowers", ["flight", "pecking"]]],
["SET", ["foo", "bar"]],
["SET", ["baz", 0]],
["RPUSH", ["friends", ["sam", "alice", "bob"]]],
["ZADD", ["hackers", [
{ score: 1906, value: 'Grace Hopper' },
{ score: 1912, value: 'Alan Turing' },
{ score: 1916, value: 'Claude Shannon'},
{ score: 1940, value: 'Alan Kay'},
{ score: 1953, value: 'Richard Stallman'},
{ score: 1957, value: 'Sophie Wilson'},
{ score: 1965, value: 'Yukihiro Matsumoto'},
{ score: 1969, value: 'Linus Torvalds'}
] ] ],
["SADD", ["superpowers", ["flight", 'x-ray vision']]],
["HSET", ["user:1000", {
"name": 'John Smith',
"email": 'john.smith@example.com',
"password": "s3cret",
"visits": 1}]],
["HSET", ["user:1001", {
"name": 'Mary Jones',
"email": 'mjones@example.com',
"password": "hunter2"}]]
];
/* Execute each command in order */
for(var i = 0; i < init.length; ++i) await client[init[i][0]](...init[i][1]);
/* Generate worksheet and disconnect */
const ws = await redis_to_ws(client);
await client.disconnect();
/* Create a workbook, add worksheet, and write to SheetJSRedis.xlsx */
const wb = utils.book_new();
utils.book_append_sheet(wb, ws, "database");
writeFile(wb, "SheetJSRedis.xlsx");
/* Generate and show the equivalent Redis commands from the worksheet */
const cmds = await ws_to_redis(ws);
cmds.forEach(x => console.log(x[0], x[1]));

Binary file not shown.

After

Width:  |  Height:  |  Size: 106 KiB