redis

docz/docs/03-demos/11-nosql.md

@@ -3,6 +3,9 @@ sidebar_position: 11
 title: NoSQL Data Stores
 ---
 
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
 So-called "Schema-less" databases allow for arbitrary keys and values within the
 entries in the database.  K/V stores and Objects add additional restrictions.
 
@@ -52,10 +55,15 @@ Redis has 5 core data types: "String", List", "Set", "Sorted Set", and "Hash".
 Since the keys and values are limited to simple strings (and numbers), it is
 possible to store complete databases in a single worksheet.
 
-<details open><summary><b>Sample Mapping</b> (click to hide)</summary>
+![SheetJSRedis.xlsx](pathname:///nosql/sheetjsredis.png)
+
+#### Mapping
 
 The first row holds the data type and the second row holds the property name.
 
+<Tabs>
+  <TabItem value="strings" label="Strings">
+
 Strings can be stored in a unified String table. The first column holds keys
 and the second column holds values:
 
@@ -68,19 +76,68 @@ XXX|    A    |   B   |
  4 | Sheet   | JS    |
 ```
 
-Lists and Sets are unidimensional and can be stored in their own columns.  The
-second row holds the list name:
+The SheetJS array-of-arrays representation of the string table is an array of
+key/value pairs:
 
-```
-XXX|    C    |   D   |
----+---------+-------+
- 1 | List    | Set   |
- 2 | List1   | Set1  |
- 3 | List1V1 | Set1A |
- 4 | List1V2 | Set1B |
+```js
+let aoa = ["Strings"]; aoa.length = 2; // [ "Strings", empty ]
+const keys = await client.KEYS("*");
+for(let key of keys) {
+  const type = await client.TYPE(key);
+  if(type == "string") aoa.push([key, await client.GET(key)]);
+}
 ```
 
-Sorted Sets have an associated score which can be stored in the second column:
+  </TabItem>
+  <TabItem value="list" label="Lists">
+
+Lists are unidimensional and can be stored in their own columns.
+
+```
+XXX|    C    |
+---+---------+
+ 1 | List    |
+ 2 | List1   |
+ 3 | List1V1 |
+ 4 | List1V2 |
+```
+
+The SheetJS array-of-arrays representation of lists is a column of values.
+
+```js
+if(type == "list") {
+  let values = await client.LRANGE(key, 0, -1);
+  aoa = [ ["List"], [key] ].concat(values.map(v => [v]));
+}
+```
+
+  </TabItem>
+  <TabItem value="set" label="Sets">
+
+Sets are unidimensional and can be stored in their own columns.
+
+```
+XXX|   D   |
+---+-------+
+ 1 | Set   |
+ 2 | Set1  |
+ 3 | Set1A |
+ 4 | Set1B |
+```
+
+The SheetJS array-of-arrays representation of sets is a column of values.
+
+```js
+if(type == "set") {
+  let values = await client.SMEMBERS(key);
+  aoa = [ ["Set"], [key] ].concat(values.map(v => [v]));
+}
+```
+
+  </TabItem>
+  <TabItem value="zset" label="Sorted Sets">
+
+Sorted Sets have an associated score which can be stored in the second column.
 
 ```
 XXX|    E    | F |
@@ -91,7 +148,19 @@ XXX|    E    | F |
  4 | Key2    | 2 |
 ```
 
-Hashes are stored like the string table, with key and value columns in order:
+The SheetJS array-of-arrays representation is an array of key/score pairs.
+
+```js
+if(type == "zset") {
+  let values = await client.ZRANGE_WITHSCORES(key, 0, -1);
+  aoa = [ ["Sorted"], [key] ].concat(values.map(v => [v.value, v.score]));
+}
+```
+
+  </TabItem>
+  <TabItem value="hashes" label="Hashes">
+
+Hashes are stored like the string table, with key and value columns in order.
 
 ```
 XXX|   G   |   H   |
@@ -102,4 +171,38 @@ XXX|   G   |   H   |
  4 | Key2  | Val2  |
 ```
 
+The SheetJS array-of-arrays representation is an array of key/value pairs.
+
+```js
+if(type == "hash") {
+  let values = await client.HGETALL(key);
+  aoa = [ ["Hash"], [key] ].concat(Object.entries(values));
+}
+```
+
+  </TabItem>
+</Tabs>
+
+#### Example
+
+<details><summary><b>Complete Example</b> (click to show)</summary>
+
+0) Set up and start a local Redis server
+
+1) Download the following scripts:
+
+- [`SheetJSRedis.mjs`](pathname:///nosql/SheetJSRedis.mjs)
+- [`SheetJSRedisTest.mjs`](pathname:///nosql/SheetJSRedisTest.mjs)
+
+2) Install dependencies and run:
+
+```bash
+npm i --save https://cdn.sheetjs.com/xlsx-latest/xlsx-latest.tgz redis
+node SheetJSRedisTest.mjs
+```
+
+Inspect the output and compare with the data in `SheetJSRedisTest.mjs`.
+
+Open `SheetJSRedis.xlsx` and verify the columns have the correct data
+
 </details>

docz/static/nosql/SheetJSRedis.mjs

@@ -0,0 +1,98 @@
+/* sheetjs (C) 2013-present SheetJS -- https://sheetjs.com */
+import { utils } from "xlsx";
+
+/* Generate worksheet from database */
+async function redis_to_ws(client) {
+  /* Get a list of every item in the database */
+  const keys = await client.KEYS("*");
+
+  /* Collect the full contents */
+  const data = [];
+  for(let key of keys) {
+    const type = await client.TYPE(key);
+    let value;
+    switch(type) {
+      case "string": value = await client.GET(key); break;
+      case "list": value = await client.LRANGE(key, 0, -1); break;
+      case "set": value = await client.SMEMBERS(key); break;
+      case "zset": value = await client.ZRANGE_WITHSCORES(key, 0, -1); break;
+      case "hash": value = await client.HGETALL(key); break;
+      default: console.warn(`unsupported type ${type}`); break
+    }
+    data.push({key, type, value});
+  }
+
+  /* Create a new worksheet and add the string table */
+  const ws = utils.aoa_to_sheet([["Strings"]]);
+  utils.sheet_add_aoa(ws, data.filter(r => r.type == "string").map(r => [r.key, r.value]), {origin: "A3"});
+
+  /* Add the other types */
+  let C = 2;
+  data.forEach(row => {
+    switch(row.type) {
+      case "set":
+      case "list": {
+        /* `value` is an array.  aoa prepends type and key, then transposes to make a column */
+        var aoa = [row.type == "set" ? "Set" : "List", row.key].concat(row.value).map(x => ([x]));
+        utils.sheet_add_aoa(ws, aoa, {origin: utils.encode_col(C) + "1"});
+        ++C;
+      } break;
+      case "zset": {
+        /* `value` is an object with value/score keys. generate array with map and prepend metadata */
+        var aoa = [["Sorted"], [row.key]].concat(row.value.map(r => ([r.value, r.score])));
+        utils.sheet_add_aoa(ws, aoa, {origin: utils.encode_col(C) + "1"});
+        C += 2;
+      } break;
+      case "hash": {
+        /* `value` is an object.  Object.entries returns an array of arrays */
+        var aoa = [["Hash"], [row.key]].concat(Object.entries(row.value));
+        utils.sheet_add_aoa(ws, aoa, {origin: utils.encode_col(C) + "1"});
+        C += 2;
+      } break;
+      case "string": break;
+    }
+  });
+
+  return ws;
+}
+
+/* Generate array of Redis commands */
+async function ws_to_redis(ws) {
+  const cmds = [];
+  /* Extract data from the worksheet */
+  const aoa = utils.sheet_to_json(ws, { header: 1 });
+  /* Iterate over the values in the first row */
+  aoa[0].forEach((type, C) => {
+    /* The name is exepcted in the second row, same column, unless type is Strings */
+    if(type != "Strings" && !aoa[1][C]) throw new Error(`Column ${utils.encode_col(C)} missing name!`)
+    switch(type) {
+      case "Strings": {
+        if(aoa[0][C+1]) throw new Error(`${type} requires 2 columns!`);
+        /* For each row starting with SheetJS 2, check if key and value are present */
+        for(let R = 2; R < aoa.length; ++R)
+          if(aoa[R]?.[C] != null && aoa[R]?.[C+1] != null)
+            /* When key and value are present, emit a SET command */
+            cmds.push(["SET", [aoa[R][C], aoa[R][C+1]]]);
+      } break;
+      case "Set":
+      case "List": {
+        /* SADD (Set) / RPUSH (List) second argument is an array of values to add to the set */
+        cmds.push([ type == "Set" ? "SADD" : "RPUSH",  [ aoa[1][C], aoa.slice(2).map(r => r[C]).filter(x => x != null) ]]);
+      } break;
+      case "Sorted": {
+        /* ZADD second argument is an array of objects with `value` and `score` */
+        if(aoa[0][C+1]) throw new Error(`${type} requires 2 columns!`);
+        cmds.push([ "ZADD", [ aoa[1][C], aoa.slice(2).map(r => ({value: r[C], score:r[C+1]})).filter(x => x.value != null && x.score != null) ]]);
+      } break;
+      case "Hash": {
+        /* HSET second argument is an object. `Object.fromEntries` generates object from an array of K/V pairs */
+        if(aoa[0][C+1]) throw new Error(`${type} requires 2 columns!`);
+        cmds.push([ "HSET", [ aoa[1][C], Object.fromEntries(aoa.slice(2).map(r => [r[C], r[C+1]]).filter(x => x[0] != null && x[1] != null)) ]]);
+      } break;
+      default: console.error(`Unrecognized column type ${type}`); break;
+    }
+  })
+  return cmds;
+}
+
+export { ws_to_redis, redis_to_ws };

docz/static/nosql/SheetJSRedisTest.mjs

@@ -0,0 +1,55 @@
+/* sheetjs (C) 2013-present SheetJS -- https://sheetjs.com */
+import { utils, writeFile, set_fs } from "xlsx";
+import { createClient } from "redis";
+import { ws_to_redis, redis_to_ws } from "./SheetJSRedis.mjs";
+import * as fs from 'fs';
+set_fs(fs);
+
+const client = createClient();
+client.on("error", err => console.error("REDIS", err));
+await client.connect();
+
+/* This data is based on the Try Redis tutorial */
+var init = [
+  ["FLUSHALL", []],
+  ["SADD",  ["birdpowers", ["flight", "pecking"]]],
+  ["SET",   ["foo", "bar"]],
+  ["SET",   ["baz", 0]],
+  ["RPUSH", ["friends", ["sam", "alice", "bob"]]],
+  ["ZADD",  ["hackers", [
+    { score: 1906, value: 'Grace Hopper' },
+    { score: 1912, value: 'Alan Turing' },
+    { score: 1916, value: 'Claude Shannon'},
+    { score: 1940, value: 'Alan Kay'},
+    { score: 1953, value: 'Richard Stallman'},
+    { score: 1957, value: 'Sophie Wilson'},
+    { score: 1965, value: 'Yukihiro Matsumoto'},
+    { score: 1969, value: 'Linus Torvalds'}
+  ] ] ],
+  ["SADD",  ["superpowers", ["flight", 'x-ray vision']]],
+  ["HSET", ["user:1000", {
+    "name": 'John Smith',
+    "email": 'john.smith@example.com',
+    "password": "s3cret",
+    "visits": 1}]],
+  ["HSET", ["user:1001", {
+    "name": 'Mary Jones',
+    "email": 'mjones@example.com',
+    "password": "hunter2"}]]
+];
+
+/* Execute each command in order */
+for(var i = 0; i < init.length; ++i) await client[init[i][0]](...init[i][1]);
+
+/* Generate worksheet and disconnect */
+const ws = await redis_to_ws(client);
+await client.disconnect();
+
+/* Create a workbook, add worksheet, and write to SheetJSRedis.xlsx */
+const wb = utils.book_new();
+utils.book_append_sheet(wb, ws, "database");
+writeFile(wb, "SheetJSRedis.xlsx");
+
+/* Generate and show the equivalent Redis commands from the worksheet */
+const cmds = await ws_to_redis(ws);
+cmds.forEach(x => console.log(x[0], x[1]));