diff --git a/docz/docs/03-demos/11-nosql.md b/docz/docs/03-demos/11-nosql.md index 9f5d000..56419ab 100644 --- a/docz/docs/03-demos/11-nosql.md +++ b/docz/docs/03-demos/11-nosql.md @@ -3,6 +3,9 @@ sidebar_position: 11 title: NoSQL Data Stores --- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + So-called "Schema-less" databases allow for arbitrary keys and values within the entries in the database. K/V stores and Objects add additional restrictions. @@ -52,10 +55,15 @@ Redis has 5 core data types: "String", List", "Set", "Sorted Set", and "Hash". Since the keys and values are limited to simple strings (and numbers), it is possible to store complete databases in a single worksheet. -
Sample Mapping (click to hide) +![SheetJSRedis.xlsx](pathname:///nosql/sheetjsredis.png) + +#### Mapping The first row holds the data type and the second row holds the property name. + + + Strings can be stored in a unified String table. The first column holds keys and the second column holds values: @@ -68,19 +76,68 @@ XXX| A | B | 4 | Sheet | JS | ``` -Lists and Sets are unidimensional and can be stored in their own columns. The -second row holds the list name: +The SheetJS array-of-arrays representation of the string table is an array of +key/value pairs: -``` -XXX| C | D | ----+---------+-------+ - 1 | List | Set | - 2 | List1 | Set1 | - 3 | List1V1 | Set1A | - 4 | List1V2 | Set1B | +```js +let aoa = ["Strings"]; aoa.length = 2; // [ "Strings", empty ] +const keys = await client.KEYS("*"); +for(let key of keys) { + const type = await client.TYPE(key); + if(type == "string") aoa.push([key, await client.GET(key)]); +} ``` -Sorted Sets have an associated score which can be stored in the second column: + + + +Lists are unidimensional and can be stored in their own columns. + +``` +XXX| C | +---+---------+ + 1 | List | + 2 | List1 | + 3 | List1V1 | + 4 | List1V2 | +``` + +The SheetJS array-of-arrays representation of lists is a column of values. + +```js +if(type == "list") { + let values = await client.LRANGE(key, 0, -1); + aoa = [ ["List"], [key] ].concat(values.map(v => [v])); +} +``` + + + + +Sets are unidimensional and can be stored in their own columns. + +``` +XXX| D | +---+-------+ + 1 | Set | + 2 | Set1 | + 3 | Set1A | + 4 | Set1B | +``` + +The SheetJS array-of-arrays representation of sets is a column of values. + +```js +if(type == "set") { + let values = await client.SMEMBERS(key); + aoa = [ ["Set"], [key] ].concat(values.map(v => [v])); +} +``` + + + + +Sorted Sets have an associated score which can be stored in the second column. ``` XXX| E | F | @@ -91,7 +148,19 @@ XXX| E | F | 4 | Key2 | 2 | ``` -Hashes are stored like the string table, with key and value columns in order: +The SheetJS array-of-arrays representation is an array of key/score pairs. + +```js +if(type == "zset") { + let values = await client.ZRANGE_WITHSCORES(key, 0, -1); + aoa = [ ["Sorted"], [key] ].concat(values.map(v => [v.value, v.score])); +} +``` + + + + +Hashes are stored like the string table, with key and value columns in order. ``` XXX| G | H | @@ -102,4 +171,38 @@ XXX| G | H | 4 | Key2 | Val2 | ``` +The SheetJS array-of-arrays representation is an array of key/value pairs. + +```js +if(type == "hash") { + let values = await client.HGETALL(key); + aoa = [ ["Hash"], [key] ].concat(Object.entries(values)); +} +``` + + + + +#### Example + +
Complete Example (click to show) + +0) Set up and start a local Redis server + +1) Download the following scripts: + +- [`SheetJSRedis.mjs`](pathname:///nosql/SheetJSRedis.mjs) +- [`SheetJSRedisTest.mjs`](pathname:///nosql/SheetJSRedisTest.mjs) + +2) Install dependencies and run: + +```bash +npm i --save https://cdn.sheetjs.com/xlsx-latest/xlsx-latest.tgz redis +node SheetJSRedisTest.mjs +``` + +Inspect the output and compare with the data in `SheetJSRedisTest.mjs`. + +Open `SheetJSRedis.xlsx` and verify the columns have the correct data +
diff --git a/docz/static/nosql/SheetJSRedis.mjs b/docz/static/nosql/SheetJSRedis.mjs new file mode 100644 index 0000000..110e93b --- /dev/null +++ b/docz/static/nosql/SheetJSRedis.mjs @@ -0,0 +1,98 @@ +/* sheetjs (C) 2013-present SheetJS -- https://sheetjs.com */ +import { utils } from "xlsx"; + +/* Generate worksheet from database */ +async function redis_to_ws(client) { + /* Get a list of every item in the database */ + const keys = await client.KEYS("*"); + + /* Collect the full contents */ + const data = []; + for(let key of keys) { + const type = await client.TYPE(key); + let value; + switch(type) { + case "string": value = await client.GET(key); break; + case "list": value = await client.LRANGE(key, 0, -1); break; + case "set": value = await client.SMEMBERS(key); break; + case "zset": value = await client.ZRANGE_WITHSCORES(key, 0, -1); break; + case "hash": value = await client.HGETALL(key); break; + default: console.warn(`unsupported type ${type}`); break + } + data.push({key, type, value}); + } + + /* Create a new worksheet and add the string table */ + const ws = utils.aoa_to_sheet([["Strings"]]); + utils.sheet_add_aoa(ws, data.filter(r => r.type == "string").map(r => [r.key, r.value]), {origin: "A3"}); + + /* Add the other types */ + let C = 2; + data.forEach(row => { + switch(row.type) { + case "set": + case "list": { + /* `value` is an array. aoa prepends type and key, then transposes to make a column */ + var aoa = [row.type == "set" ? "Set" : "List", row.key].concat(row.value).map(x => ([x])); + utils.sheet_add_aoa(ws, aoa, {origin: utils.encode_col(C) + "1"}); + ++C; + } break; + case "zset": { + /* `value` is an object with value/score keys. generate array with map and prepend metadata */ + var aoa = [["Sorted"], [row.key]].concat(row.value.map(r => ([r.value, r.score]))); + utils.sheet_add_aoa(ws, aoa, {origin: utils.encode_col(C) + "1"}); + C += 2; + } break; + case "hash": { + /* `value` is an object. Object.entries returns an array of arrays */ + var aoa = [["Hash"], [row.key]].concat(Object.entries(row.value)); + utils.sheet_add_aoa(ws, aoa, {origin: utils.encode_col(C) + "1"}); + C += 2; + } break; + case "string": break; + } + }); + + return ws; +} + +/* Generate array of Redis commands */ +async function ws_to_redis(ws) { + const cmds = []; + /* Extract data from the worksheet */ + const aoa = utils.sheet_to_json(ws, { header: 1 }); + /* Iterate over the values in the first row */ + aoa[0].forEach((type, C) => { + /* The name is exepcted in the second row, same column, unless type is Strings */ + if(type != "Strings" && !aoa[1][C]) throw new Error(`Column ${utils.encode_col(C)} missing name!`) + switch(type) { + case "Strings": { + if(aoa[0][C+1]) throw new Error(`${type} requires 2 columns!`); + /* For each row starting with SheetJS 2, check if key and value are present */ + for(let R = 2; R < aoa.length; ++R) + if(aoa[R]?.[C] != null && aoa[R]?.[C+1] != null) + /* When key and value are present, emit a SET command */ + cmds.push(["SET", [aoa[R][C], aoa[R][C+1]]]); + } break; + case "Set": + case "List": { + /* SADD (Set) / RPUSH (List) second argument is an array of values to add to the set */ + cmds.push([ type == "Set" ? "SADD" : "RPUSH", [ aoa[1][C], aoa.slice(2).map(r => r[C]).filter(x => x != null) ]]); + } break; + case "Sorted": { + /* ZADD second argument is an array of objects with `value` and `score` */ + if(aoa[0][C+1]) throw new Error(`${type} requires 2 columns!`); + cmds.push([ "ZADD", [ aoa[1][C], aoa.slice(2).map(r => ({value: r[C], score:r[C+1]})).filter(x => x.value != null && x.score != null) ]]); + } break; + case "Hash": { + /* HSET second argument is an object. `Object.fromEntries` generates object from an array of K/V pairs */ + if(aoa[0][C+1]) throw new Error(`${type} requires 2 columns!`); + cmds.push([ "HSET", [ aoa[1][C], Object.fromEntries(aoa.slice(2).map(r => [r[C], r[C+1]]).filter(x => x[0] != null && x[1] != null)) ]]); + } break; + default: console.error(`Unrecognized column type ${type}`); break; + } + }) + return cmds; +} + +export { ws_to_redis, redis_to_ws }; \ No newline at end of file diff --git a/docz/static/nosql/SheetJSRedisTest.mjs b/docz/static/nosql/SheetJSRedisTest.mjs new file mode 100644 index 0000000..19e83ea --- /dev/null +++ b/docz/static/nosql/SheetJSRedisTest.mjs @@ -0,0 +1,55 @@ +/* sheetjs (C) 2013-present SheetJS -- https://sheetjs.com */ +import { utils, writeFile, set_fs } from "xlsx"; +import { createClient } from "redis"; +import { ws_to_redis, redis_to_ws } from "./SheetJSRedis.mjs"; +import * as fs from 'fs'; +set_fs(fs); + +const client = createClient(); +client.on("error", err => console.error("REDIS", err)); +await client.connect(); + +/* This data is based on the Try Redis tutorial */ +var init = [ + ["FLUSHALL", []], + ["SADD", ["birdpowers", ["flight", "pecking"]]], + ["SET", ["foo", "bar"]], + ["SET", ["baz", 0]], + ["RPUSH", ["friends", ["sam", "alice", "bob"]]], + ["ZADD", ["hackers", [ + { score: 1906, value: 'Grace Hopper' }, + { score: 1912, value: 'Alan Turing' }, + { score: 1916, value: 'Claude Shannon'}, + { score: 1940, value: 'Alan Kay'}, + { score: 1953, value: 'Richard Stallman'}, + { score: 1957, value: 'Sophie Wilson'}, + { score: 1965, value: 'Yukihiro Matsumoto'}, + { score: 1969, value: 'Linus Torvalds'} + ] ] ], + ["SADD", ["superpowers", ["flight", 'x-ray vision']]], + ["HSET", ["user:1000", { + "name": 'John Smith', + "email": 'john.smith@example.com', + "password": "s3cret", + "visits": 1}]], + ["HSET", ["user:1001", { + "name": 'Mary Jones', + "email": 'mjones@example.com', + "password": "hunter2"}]] +]; + +/* Execute each command in order */ +for(var i = 0; i < init.length; ++i) await client[init[i][0]](...init[i][1]); + +/* Generate worksheet and disconnect */ +const ws = await redis_to_ws(client); +await client.disconnect(); + +/* Create a workbook, add worksheet, and write to SheetJSRedis.xlsx */ +const wb = utils.book_new(); +utils.book_append_sheet(wb, ws, "database"); +writeFile(wb, "SheetJSRedis.xlsx"); + +/* Generate and show the equivalent Redis commands from the worksheet */ +const cmds = await ws_to_redis(ws); +cmds.forEach(x => console.log(x[0], x[1])); \ No newline at end of file diff --git a/docz/static/nosql/sheetjsredis.png b/docz/static/nosql/sheetjsredis.png new file mode 100644 index 0000000..0860d14 Binary files /dev/null and b/docz/static/nosql/sheetjsredis.png differ