From 60118bad51bbaeeb4160d704acbbdb39e8612aa7 Mon Sep 17 00:00:00 2001 From: SheetJS Date: Mon, 16 Oct 2023 05:12:56 -0400 Subject: [PATCH] server --- .../01-installation/07-bun.md | 5 +- docz/docs/03-demos/03-net/01-network.mdx | 1 + docz/docs/03-demos/03-net/02-server.md | 683 ------------------ .../03-demos/03-net/02-server/01-express.md | 174 +++++ .../03-demos/03-net/02-server/04-drash.md | 143 ++++ .../03-demos/03-net/02-server/11-nestjs.md | 233 ++++++ .../03-demos/03-net/02-server/19-fastify.md | 187 +++++ .../03-demos/03-net/02-server/_category_.json | 5 + docz/docs/03-demos/03-net/02-server/index.md | 246 +++++++ docz/docs/03-demos/03-net/03-email.md | 1 + docz/docs/03-demos/08-local/09-indexeddb.md | 4 +- docz/docs/03-demos/09-cloud/11-aws.md | 4 +- docz/static/chakra/Makefile | 4 +- docz/static/drash/SheetJSDrash.ts | 78 ++ docz/static/server/SheetJSDrash.ts | 2 +- 15 files changed, 1077 insertions(+), 693 deletions(-) delete mode 100644 docz/docs/03-demos/03-net/02-server.md create mode 100644 docz/docs/03-demos/03-net/02-server/01-express.md create mode 100644 docz/docs/03-demos/03-net/02-server/04-drash.md create mode 100644 docz/docs/03-demos/03-net/02-server/11-nestjs.md create mode 100644 docz/docs/03-demos/03-net/02-server/19-fastify.md create mode 100644 docz/docs/03-demos/03-net/02-server/_category_.json create mode 100644 docz/docs/03-demos/03-net/02-server/index.md create mode 100644 docz/static/drash/SheetJSDrash.ts diff --git a/docz/docs/02-getting-started/01-installation/07-bun.md b/docz/docs/02-getting-started/01-installation/07-bun.md index b5dbb87..5ffb3ad 100644 --- a/docz/docs/02-getting-started/01-installation/07-bun.md +++ b/docz/docs/02-getting-started/01-installation/07-bun.md @@ -1,4 +1,5 @@ --- +title: Bun pagination_prev: getting-started/index pagination_next: getting-started/examples/index sidebar_position: 7 @@ -9,8 +10,6 @@ sidebar_custom_props: import current from '/version.js'; import CodeBlock from '@theme/CodeBlock'; -# Bun - Tarballs are available on .

https://cdn.sheetjs.com/xlsx-{current}/xlsx-{current}.tgz is the URL for version {current}

@@ -85,5 +84,5 @@ import * as cpexcel from 'xlsx/dist/cpexcel.full.mjs'; XLSX.set_cptable(cpexcel); ``` -[^1]: Bun releases before the official 1.0.0 release did not support tarball dependencies. If a pre-1.0.0 release must be used, the module can be installed with a package manager like Yarn or the ESM script can be vendored. +[^1]: Bun releases before the official 1.0.0 release did not support tarball dependencies. If a pre-1.0.0 release must be used, the [ES Module script can be vendored](/docs/getting-started/installation/standalone#ecmascript-module-imports) or the [NodeJS module can be installed with a NodeJS-compatible package manager](/docs/getting-started/installation/nodejs). [^2]: See [the relevant issue in the Bun issue tracker](https://github.com/oven-sh/bun/issues/101) \ No newline at end of file diff --git a/docz/docs/03-demos/03-net/01-network.mdx b/docz/docs/03-demos/03-net/01-network.mdx index b6f28c7..caf9f7b 100644 --- a/docz/docs/03-demos/03-net/01-network.mdx +++ b/docz/docs/03-demos/03-net/01-network.mdx @@ -1,5 +1,6 @@ --- title: HTTP Network Requests +pagination_next: demos/net/server/index --- diff --git a/docz/docs/03-demos/03-net/02-server.md b/docz/docs/03-demos/03-net/02-server.md deleted file mode 100644 index edd8600..0000000 --- a/docz/docs/03-demos/03-net/02-server.md +++ /dev/null @@ -1,683 +0,0 @@ ---- -title: HTTP Server Processing ---- - -import current from '/version.js'; -import CodeBlock from '@theme/CodeBlock'; - -Server-Side JS platforms like NodeJS and Deno have built-in APIs for listening -on network interfaces. They provide wrappers for requests and responses. - -## Overview - -#### Reading Data - -Typically servers receive form data with content type `multipart/form-data` or -`application/x-www-form-urlencoded`. The platforms themselves typically do not -provide "body parsing" functions, instead leaning on the community to supply -modules to take the encoded data and split into form fields and files. - -NodeJS servers typically use a parser like `formidable`. In the example below, -`formidable` will write to file and `XLSX.readFile` will read the file: - -```js -var XLSX = require("xlsx"); // This is using the CommonJS build -var formidable = require("formidable"); - -require("http").createServer(function(req, res) { - if(req.method !== "POST") return res.end(""); - - /* parse body and implement logic in callback */ - // highlight-next-line - (new formidable.IncomingForm()).parse(req, function(err, fields, files) { - /* if successful, files is an object whose keys are param names */ - // highlight-next-line - var file = files["upload"]; // - /* file.path is a location in the filesystem, usually in a temp folder */ - // highlight-next-line - var wb = XLSX.readFile(file.filepath); - // print the first worksheet back as a CSV - res.end(XLSX.utils.sheet_to_csv(wb.Sheets[wb.SheetNames[0]])); - }); -}).listen(process.env.PORT || 3000); -``` - -`XLSX.read` will accept NodeJS buffers as well as `Uint8Array`, Base64 strings, -binary strings, and plain Arrays of bytes. This covers the interface types of -a wide variety of frameworks. - -#### Writing Data - -Typically server libraries use a response API that accepts `Uint8Array` data. -`XLSX.write` with the option `type: "buffer"` will generate data. To force the -response to be treated as an attachment, set the `Content-Disposition` header: - -```js -var XLSX = require("xlsx"); // This is using the CommonJS build - -require("http").createServer(function(req, res) { - if(req.method !== "GET") return res.end(""); - var wb = XLSX.read("S,h,e,e,t,J,S\n5,4,3,3,7,9,5", {type: "binary"}); - // highlight-start - res.setHeader('Content-Disposition', 'attachment; filename="SheetJS.xlsx"'); - res.end(XLSX.write(wb, {type:"buffer", bookType: "xlsx"})); - // highlight-end -}).listen(process.env.PORT || 3000); -``` - -## NodeJS - -When processing small files, the work is best handled in the server response -handler function. This approach is used in the "Framework Demos" section. - -When processing large files, the direct approach will freeze the server. NodeJS -provides ["Worker Threads"](#worker-threads) for this exact use case. - -### Framework Demos - -#### Express - -:::note - -This demo was verified on 2023 August 27 using `express-formidable@1.2.0` and -ExpressJS `4.18.2` - -::: - -The `express-formidable` middleware is powered by the `formidable` parser. It -adds a `files` property to the request. - -When downloading binary data, Express handles `Buffer` data in `res.end`. The -convenience `attachment` method adds the required header: - -```js -// Header 'Content-Disposition: attachment; filename="SheetJS.xlsx"' -res.attachment("SheetJS.xlsx"); -``` - -The following demo Express server will respond to POST requests to `/upload` -with a CSV output of the first sheet. It will also respond to GET requests to -`/download`, responding with a fixed XLSX worksheet: - -```js title="SheetJSExpressCSV.js" -var XLSX = require('xlsx'), express = require('express'); - -/* create app */ -var app = express(); -/* add express-formidable middleware */ -// highlight-next-line -app.use(require('express-formidable')()); -/* route for handling uploaded data */ -app.post('/upload', function(req, res) { - // highlight-start - var f = req.files["upload"]; // - var wb = XLSX.readFile(f.path); - // highlight-end - /* respond with CSV data from the first sheet */ - res.status(200).end(XLSX.utils.sheet_to_csv(wb.Sheets[wb.SheetNames[0]])); -}); -app.get('/download', function(req, res) { - /* generate workbook object */ - var ws = XLSX.utils.aoa_to_sheet(["SheetJS".split(""), [5,4,3,3,7,9,5]]); - var wb = XLSX.utils.book_new(); XLSX.utils.book_append_sheet(wb, ws, "Data"); - // highlight-start - /* generate buffer */ - var buf = XLSX.write(wb, {type: "buffer", bookType: "xlsx"}); - /* set headers */ - res.attachment("SheetJSExpress.xlsx"); - /* respond with file data */ - res.status(200).end(buf); - // highlight-end -}); -app.listen(+process.env.PORT||3000); -``` - -
Testing (click to show) - -0) Save the code sample to `SheetJSExpressCSV.js` - -1) Install dependencies: - -{`\ -npm i --save https://cdn.sheetjs.com/xlsx-${current}/xlsx-${current}.tgz express@4.18.2 express-formidable@1.2.0`} - - -2) Start server (note: it will not print anything to console when running) - -```bash -node SheetJSExpressCSV.js -``` - -3) Test POST requests using : - -```bash -curl -LO https://sheetjs.com/pres.numbers -curl -X POST -F upload=@pres.numbers http://localhost:3000/upload -``` - -The response should show the data in CSV rows. - -4) Test GET requests by opening `http://localhost:3000/download` in your browser. - -It should prompt to download `SheetJSExpress.xlsx` - -
- -#### NestJS - -:::note - -This demo was verified on 2023 August 27 using NestJS `10.2.0` and Multer. - -::: - -[The NestJS docs](https://docs.nestjs.com/techniques/file-upload) have detailed -instructions for file upload support. In the controller, the `path` property -works with `XLSX.readFile`. - -When downloading binary data, NestJS expects `StreamableFile`-wrapped Buffers. - -The following demo NestJS Controller will respond to POST requests to `/upload` -with a CSV output of the first sheet. It will also respond to GET requests to -`/download`, responding with a fixed export: - -```ts title="src/sheetjs/sheetjs.controller.js" -import { Controller, Get, Header, Post, StreamableFile, UploadedFile, UseInterceptors } from '@nestjs/common'; -import { FileInterceptor } from '@nestjs/platform-express'; -import { readFile, utils, write } from 'xlsx'; - -@Controller('sheetjs') -export class SheetjsController { - @Post('upload') // - @UseInterceptors(FileInterceptor('upload')) - async uploadXlsxFile(@UploadedFile() file: Express.Multer.File) { - /* file.path is a path to the workbook */ - // highlight-next-line - const wb = readFile(file.path); - /* generate CSV of first worksheet */ - return utils.sheet_to_csv(wb.Sheets[wb.SheetNames[0]]); - } - - @Get('download') - @Header('Content-Disposition', 'attachment; filename="SheetJSNest.xlsx"') - async downloadXlsxFile(): Promise { - var ws = utils.aoa_to_sheet(["SheetJS".split(""), [5,4,3,3,7,9,5]]); - var wb = utils.book_new(); utils.book_append_sheet(wb, ws, "Data"); - // highlight-start - /* generate buffer */ - var buf = write(wb, {type: "buffer", bookType: "xlsx"}); - /* Return a streamable file */ - return new StreamableFile(buf); - // highlight-end - } -} -``` - -
Testing (click to show) - -1) Create a new project: - -{`\ -npx @nestjs/cli new -p npm sheetjs-nest -cd sheetjs-nest -npm i --save https://cdn.sheetjs.com/xlsx-${current}/xlsx-${current}.tgz -npm i --save-dev @types/multer -mkdir -p upload`} - - -2) Create a new controller and a new module: - -```bash -npx @nestjs/cli generate module sheetjs -npx @nestjs/cli generate controller sheetjs -``` - -3) Add `multer` to the new module by editing `src/sheetjs/sheetjs.module.ts`. -The highlighted lines should be added to the file: - -```ts title="src/sheetjs/sheetjs.module.ts" -import { Module } from '@nestjs/common'; -import { SheetjsController } from './sheetjs.controller'; -// highlight-next-line -import { MulterModule } from '@nestjs/platform-express'; - -@Module({ -// highlight-start - imports: [ - MulterModule.register({ - dest: './upload', - }), - ], -// highlight-end - controllers: [SheetjsController] -}) -export class SheetjsModule {} -``` - -4) Copy the `src/sheetjs/sheetjs.controller.ts` example from earlier, replacing -the contents of the existing file. - -5) Start the server with - -```bash -npx @nestjs/cli start -``` - -3) Test POST requests using : - -```bash -curl -LO https://sheetjs.com/pres.numbers -curl -X POST -F upload=@pres.numbers http://localhost:3000/sheetjs/upload -``` - -The response should show the data in CSV rows. - -4) Test GET requests by opening `http://localhost:3000/sheetjs/download` in your browser. - -It should prompt to download `SheetJSNest.xlsx` - -
- -#### Fastify - -:::note - -This demo was verified on 2023 August 27 using `fastify@4.22.0` - -::: - -_Reading Data_ - -`@fastify/multipart`, which uses `busbuy` under the hood, can be registered: - -```js -/* load SheetJS Library */ -const XLSX = require("xlsx"); -/* load fastify and enable body parsing */ -const fastify = require('fastify')({logger: true}); -// highlight-next-line -fastify.register(require('@fastify/multipart'), { attachFieldsToBody: true }); -``` - -Once registered with the option `attachFieldsToBody`, route handlers can use -`req.body` directly: - -```js -/* POST / reads submitted file and exports to requested format */ -fastify.post('/', async(req, reply) => { - /* "file" is the name of the field in the HTML form*/ - const file = req.body.upload; - /* toBuffer returns a promise that resolves to a Buffer */ - // highlight-next-line - const buf = await file.toBuffer(); - /* `XLSX.read` can read the Buffer */ - const wb = XLSX.read(buf); - /* reply with a CSV */ - reply.send(XLSX.utils.sheet_to_csv(wb.Sheets[wb.SheetNames[0]])); -}); -``` - -:::caution pass - -Out of the box, Fastify will return an error `FST_ERR_CTP_BODY_TOO_LARGE` when -processing large spreadsheets (`statusCode 413`). This is a Fastify issue. - -The default body size limit (including all uploaded files and fields) is 1 MB. -It can be increased by setting the `bodyLimit` option during server creation: - -```js -/* increase request body size limit to 5MB = 5 * 1024 * 1024 bytes */ -const fastify = require('fastify')({bodyLimit: 5 * 1024 * 1024}); -``` - -::: - -_Writing Data_ - -The `Content-Disposition` header must be set manually: - -```js -/* GET / returns a workbook */ -fastify.get('/', (req, reply) => { - /* make a workbook */ - var wb = XLSX.read("S,h,e,e,t,J,S\n5,4,3,3,7,9,5", {type: "binary"}); - - /* write to Buffer */ - const buf = XLSX.write(wb, {type:"buffer", bookType: "xlsx"}); - - /* set Content-Disposition header and send data */ - // highlight-next-line - reply.header('Content-Disposition', 'attachment; filename="SheetJSFastify.xlsx"').send(buf); -}); -``` - -
Testing (click to show) - -0) Save the following snippet to `SheetJSFastify.js`: - -```js title="SheetJSFastify.js" -/* load SheetJS Library */ -const XLSX = require("xlsx"); -/* load fastify and enable body parsing */ -const fastify = require('fastify')({logger: true}); -fastify.register(require('@fastify/multipart'), { attachFieldsToBody: true }); - -/* GET / returns a workbook */ -fastify.get('/', (req, reply) => { - /* make a workbook */ - var wb = XLSX.read("S,h,e,e,t,J,S\n5,4,3,3,7,9,5", {type: "binary"}); - - /* write to Buffer */ - const buf = XLSX.write(wb, {type:"buffer", bookType: "xlsx"}); - - /* set Content-Disposition header and send data */ - reply.header('Content-Disposition', 'attachment; filename="SheetJSFastify.xlsx"').send(buf); -}); - -/* POST / reads submitted file and exports to requested format */ -fastify.post('/', async(req, reply) => { - - /* "file" is the name of the field in the HTML form*/ - const file = req.body.upload; - - /* toBuffer returns a promise that resolves to a Buffer */ - const wb = XLSX.read(await file.toBuffer()); - - /* send back a CSV */ - reply.send(XLSX.utils.sheet_to_csv(wb.Sheets[wb.SheetNames[0]])); -}); - -/* start */ -fastify.listen({port: process.env.PORT || 3000}, (err, addr) => { if(err) throw err; }); -``` - -1) Install dependencies: - -{`\ -npm i --save https://cdn.sheetjs.com/xlsx-${current}/xlsx-${current}.tgz fastify @fastify/multipart`} - - -2) Start server - -```bash -node SheetJSFastify.js -``` - -3) Test POST requests using : - -```bash -curl -LO https://sheetjs.com/pres.numbers -curl -X POST -F upload=@pres.numbers http://localhost:3000/ -``` - -The response should show the data in CSV rows. - -4) Test GET requests by opening `http://localhost:3000/` in your browser. - -It should prompt to download `SheetJSFastify.xlsx` - -
- -### Worker Threads - -NodeJS "Worker Threads" were introduced in v14 and eventually marked as stable -in v16. Coupled with `AsyncResource`, a simple thread pool enables processing -without blocking the server! The official NodeJS docs include a sample worker -pool implementation. - -This example uses ExpressJS to create a general XLSX conversion service, but -the same approach applies to any NodeJS server side framework. - -When reading large files, it is strongly recommended to run the body parser in -the main server process. Body parsers like `formidable` will write uploaded -files to the filesystem, and the file path should be passed to the worker (and -the worker would be responsible for reading and cleaning up the files). - -:::note pass - -The `child_process` module can also spawn [command-line tools](/docs/demos/cli). -That approach is not explored in this demo. - -::: - -
Complete Example (click to show) - -:::note - -This demo was last tested on 2023 August 27 with NodeJS 20.5.1 + ExpressJS -4.18.2 + Formidable 2.1.1 - -::: - -0) Create a simple ECMAScript-Module-enabled `package.json`: - -```json title="package.json" -{ "type": "module" } -``` - -1) Install the dependencies: - -{`\ -npm i --save https://cdn.sheetjs.com/xlsx-${current}/xlsx-${current}.tgz express@4.18.2 formidable@2.1.1`} - - -2) Create a worker script `worker.js` that listens for messages. When a message -is received, it will read the file from the filesystem, generate and pass back a -new XLSX file, and delete the original file: - -```js title="worker.js" -/* load the worker_threads module */ -import { parentPort } from 'node:worker_threads'; - -/* load the SheetJS module and hook to FS */ -import { set_fs, readFile, write } from 'xlsx'; -import * as fs from 'fs'; -set_fs(fs); - -/* the server will send a message with the `path` field */ -parentPort.on('message', (task) => { - /* highlight-start */ - // read file - const wb = readFile(task.path, { dense: true }); - // send back XLSX - parentPort.postMessage(write(wb, { type: "buffer", bookType: "xlsx" })); - /* highlight-end */ - // remove file - fs.unlink(task.path, ()=>{}); -}); -``` - -3) Download [`worker_pool.js`](pathname:///server/worker_pool.js): - -```bash -curl -LO https://docs.sheetjs.com/server/worker_pool.js -``` - -(this is a slightly modified version of the example in the NodeJS docs) - -4) Save the following server code to `main.mjs`: - -```js title="main.mjs" -/* load dependencies */ -import os from 'node:os'; -import process from 'node:process' -import express from 'express'; -import formidable from 'formidable'; - -/* load worker pool */ -import WorkerPool from './worker_pool.js'; - -const pool = new WorkerPool(os.cpus().length); -process.on("beforeExit", () => { pool.close(); }) - -/* create server */ -const app = express(); -app.post('/', (req, res, next) => { - // parse body - const form = formidable({}); - form.parse(req, (err, fields, files) => { - // look for "upload" field - if(err) return next(err); - if(!files["upload"]) return next(new Error("missing `upload` file")); - - // send a message to the worker with the path to the uploaded file - // highlight-next-line - pool.runTask({ path: files["upload"].filepath }, (err, result) => { - if(err) return next(err); - // send the file back as an attachment - res.attachment("SheetJSPool.xlsx"); - res.status(200).end(result); - }); - }); -}); - -// start server -app.listen(7262, () => { console.log(`Example app listening on port 7262`); }); -``` - -5) Run the server: - -```bash -node main.mjs -``` - -Test with the [`pres.numbers` sample file](https://sheetjs.com/pres.numbers): - -```bash -curl -LO https://sheetjs.com/pres.numbers -curl -X POST -F upload=@pres.numbers http://localhost:7262/ -J -O -``` - -This will generate `SheetJSPool.xlsx`. - -
- -## Deno - -:::caution pass - -Many hosted services like Deno Deploy do not offer filesystem access. - -This breaks web frameworks that use the filesystem in body parsing. - -::: - -Deno provides the basic elements to implement a web server. It does not provide -a body parser out of the box. - -#### Drash - -In testing, [Drash](https://drash.land/drash/) had an in-memory body parser -which could handle file uploads on hosted services like Deno Deploy. - -_Reading Data_ - -`Request#bodyParam` reads body parameters. For uploaded files, the `content` -property is a `Uint8Array` which can be parsed with the SheetJS `read` method[^1]. - -This example server responds to POST requests. The server will look for a file -in the request body under the `"upload"` key. If a file is present, the server -will parse the file, generate an HTML table using the `sheet_to_html` method[^2] -and respond with the HTML code: - -{`\ -// @deno-types="https://cdn.sheetjs.com/xlsx-${current}/package/types/index.d.ts" -import { read, utils } from 'https://cdn.sheetjs.com/xlsx-${current}/package/xlsx.mjs'; -\n\ -import * as Drash from "https://cdn.jsdelivr.net/gh/drashland/drash@v2.8.0/mod.ts"; -\n\ -class ParseResource extends Drash.Resource { - public paths = ["/"]; -\n\ - public POST(request: Drash.Request, response: Drash.Response) { - // assume a form upload like - // highlight-next-line - const file = request.bodyParam("upload"); - if (!file) throw new Error("File is required!"); - // highlight-next-line - var wb = read(file.content); - return response.html(utils.sheet_to_html(wb.Sheets[wb.SheetNames[0]])); - } -}`} - - -_Writing Data_ - -Headers are manually set with `Response#headers.set` while the raw body is set -with `Response#send`. The raw body can be `Uint8Array` or `ArrayBuffer` objects. - -Given a SheetJS workbook object, the `write` method using `type: "buffer"`[^3] -generates data objects compatible with Drash. - -This example server responds to GET requests. The server will generate a SheetJS -worksheet object from an array of arrays[^4], build up a new workbook using the -`book_new`[^5] and `book_append_sheet`[^6] utility methods, generate a XLSX file -using `write`, and send the file with appropriate headers to initiate a download -with file name `"SheetJSDrash.xlsx"`: - -{`\ -// @deno-types="https://cdn.sheetjs.com/xlsx-${current}/package/types/index.d.ts" -import { utils, write } from 'https://cdn.sheetjs.com/xlsx-${current}/package/xlsx.mjs'; -\n\ -import * as Drash from "https://cdn.jsdelivr.net/gh/drashland/drash@v2.8.0/mod.ts"; -\n\ -class WriteResource extends Drash.Resource { - public paths = ["/export"]; -\n\ - public GET(request: Drash.Request, response: Drash.Response): void { - // create some fixed workbook - const data = ["SheetJS".split(""), [5,4,3,3,7,9,5]]; - const ws = utils.aoa_to_sheet(data); - const wb = utils.book_new(); utils.book_append_sheet(wb, ws, "data"); - // write the workbook to XLSX as a Uint8Array - // highlight-next-line - const file = write(wb, { bookType: "xlsx", type: "buffer"}); - // set headers - response.headers.set("Content-Disposition", 'attachment; filename="SheetJSDrash.xlsx"'); - // send data - // highlight-next-line - return response.send("application/vnd.ms-excel", file); - } -}`} - - -
Complete Example (click to show) - -:::note - -This demo was last tested on 2023 August 18 against Drash 2.8.0 and Deno 1.36.1. - -::: - -1) Download [`SheetJSDrash.ts`](pathname:///server/SheetJSDrash.ts): - -```bash -curl -LO https://docs.sheetjs.com/server/SheetJSDrash.ts -``` - -2) Run the server: - -```bash -deno run --allow-net SheetJSDrash.ts -``` - -3) Download the test file - -4) Open `http://localhost:7262/` in your browser. - -Click "Choose File" and select `pres.numbers`. Then click "Submit" - -The page should show the contents of the file as an HTML table. - -5) Open `http://localhost:7262/export` in your browser. - -The page should attempt to download `SheetJSDrash.xlsx` . Open the new file. - -
- -[^1]: See [`read` in "Reading Files"](/docs/api/parse-options) -[^2]: See [`sheet_to_html` in "Utilities"](/docs/api/utilities/html#html-table-output) -[^3]: See [`write` in "Writing Files"](/docs/api/write-options) -[^4]: See [`aoa_to_sheet` in "Utilities"](/docs/api/utilities/array#array-of-arrays-input) -[^5]: See [`book_new` in "Utilities"](/docs/api/utilities/wb) -[^6]: See [`book_append_sheet` in "Utilities"](/docs/api/utilities/wb) diff --git a/docz/docs/03-demos/03-net/02-server/01-express.md b/docz/docs/03-demos/03-net/02-server/01-express.md new file mode 100644 index 0000000..c7680ac --- /dev/null +++ b/docz/docs/03-demos/03-net/02-server/01-express.md @@ -0,0 +1,174 @@ +--- +title: Sheets in ExpressJS +sidebar_label: ExpressJS +pagination_prev: demos/net/network +pagination_next: demos/net/email +--- + +import current from '/version.js'; +import CodeBlock from '@theme/CodeBlock'; + +[ExpressJS](https://expressjs.com/) is a lightweight NodeJS framework for +building server-side applications. + +[SheetJS](https://sheetjs.com) is a JavaScript library for reading and writing +data from spreadsheets. + +This demo uses ExpressJS and SheetJS to read and write data. We'll explore how +to parse uploaded files in a POST request handler and respond to GET requests +with downloadable spreadsheets. + +The ["Complete Example"](#complete-example) section includes a complete server. + +:::note + +This demo was tested on 2023 October 16 using `express-formidable@1.2.0` and +ExpressJS `4.18.2` + +::: + +## Integration Details + +The [SheetJS NodeJS module](/docs/getting-started/installation/nodejs) can be +imported from scripts that use ExpressJS. + +### Exporting Data to Workbooks (GET) + +The SheetJS `write` method[^1] with the option `type: "buffer"` generates NodeJS +Buffer objects containing the raw file data. + +ExpressJS can directly handle `Buffer` data in `Response#end`[^2]. + +The exported filename can be specified using `Response#attachment`[^3]. + +The following demo ExpressJS server will respond to GET requests to `/download` +with a XLSX spreadsheet. In this example, the SheetJS `aoa_to_sheet` method[^4] +generates a sheet object and the `book_new` and `book_append_sheet` helpers[^5] +build the workbook object. + +```js +var XLSX = require('xlsx'), express = require('express'); + +/* create app */ +var app = express(); + +app.get('/download', function(req, res) { + /* generate workbook object */ + var ws = XLSX.utils.aoa_to_sheet(["SheetJS".split(""), [5,4,3,3,7,9,5]]); + var wb = XLSX.utils.book_new(); XLSX.utils.book_append_sheet(wb, ws, "Data"); + // highlight-start + /* generate buffer */ + var buf = XLSX.write(wb, {type: "buffer", bookType: "xlsx"}); + /* set headers */ + res.attachment("SheetJSExpress.xlsx"); + /* respond with file data */ + res.status(200).end(buf); + // highlight-end +}); +app.listen(+process.env.PORT||3000); +``` + +### Parsing Uploaded Files (POST) + +The `express-formidable` middleware is powered by the `formidable` parser. It +adds a `files` object to the request. + +Each value in the `files` object has a `path` property which represents the path +to the file in the filesystem. The SheetJS `readFile` method[^6] can read the +file and generate a workbook object[^7]. + +The following demo ExpressJS server will respond to POST requests to `/upload`. +Assuming the `upload` field of the form data is the file, the SheetJS `read` +method will parse the file. CSV rows are generated from the first worksheet +using the SheetJS `sheet_to_csv` method[^8]. + +```js +var XLSX = require('xlsx'), express = require('express'); + +/* create app */ +var app = express(); +/* add express-formidable middleware */ +// highlight-next-line +app.use(require('express-formidable')()); +/* route for handling uploaded data */ +app.post('/upload', function(req, res) { + // highlight-start + var f = req.files["upload"]; // + var wb = XLSX.readFile(f.path); + // highlight-end + /* respond with CSV data from the first sheet */ + res.status(200).end(XLSX.utils.sheet_to_csv(wb.Sheets[wb.SheetNames[0]])); +}); +app.listen(+process.env.PORT||3000); +``` + +## Complete Example + +1) Save the code sample to `SheetJSExpressCSV.js`: + +```js title="SheetJSExpressCSV.js" +var XLSX = require('xlsx'), express = require('express'); + +/* create app */ +var app = express(); +/* add express-formidable middleware */ +// highlight-next-line +app.use(require('express-formidable')()); +/* route for handling uploaded data */ +app.post('/upload', function(req, res) { + // highlight-start + var f = req.files["upload"]; // + var wb = XLSX.readFile(f.path); + // highlight-end + /* respond with CSV data from the first sheet */ + res.status(200).end(XLSX.utils.sheet_to_csv(wb.Sheets[wb.SheetNames[0]])); +}); +app.get('/download', function(req, res) { + /* generate workbook object */ + var ws = XLSX.utils.aoa_to_sheet(["SheetJS".split(""), [5,4,3,3,7,9,5]]); + var wb = XLSX.utils.book_new(); XLSX.utils.book_append_sheet(wb, ws, "Data"); + // highlight-start + /* generate buffer */ + var buf = XLSX.write(wb, {type: "buffer", bookType: "xlsx"}); + /* set headers */ + res.attachment("SheetJSExpress.xlsx"); + /* respond with file data */ + res.status(200).end(buf); + // highlight-end +}); +app.listen(+process.env.PORT||3000); +``` + +2) Install dependencies: + +{`\ +npm i --save https://cdn.sheetjs.com/xlsx-${current}/xlsx-${current}.tgz express@4.18.2 express-formidable@1.2.0`} + + +3) Start server (note: it will not print anything to console when running) + +```bash +node SheetJSExpressCSV.js +``` + +4) Test POST requests using : + +```bash +curl -LO https://sheetjs.com/pres.numbers +curl -X POST -F upload=@pres.numbers http://localhost:3000/upload +``` + +The response should show the data in CSV rows. + +5) Test GET requests by opening `http://localhost:3000/download` in your browser. + +It should prompt to download `SheetJSExpress.xlsx` + +[^1]: See [`write` in "Writing Files"](/docs/api/write-options) +[^2]: See [`res.end`](https://expressjs.com/en/4x/api.html#res.end) in the ExpressJS documentation. +[^3]: See [`res.attachment`](https://expressjs.com/en/4x/api.html#res.attachment) in the ExpressJS documentation. +[^4]: See [`aoa_to_sheet` in "Utilities"](/docs/api/utilities/array#array-of-arrays-input) +[^5]: See ["Workbook Helpers" in "Utilities"](/docs/api/utilities/wb) for details on `book_new` and `book_append_sheet`. +[^6]: See [`readFile` in "Reading Files"](/docs/api/parse-options) +[^7]: See ["Workbook Object"](/docs/csf/book) +[^8]: See [`sheet_to_csv` in "CSV and Text"](/docs/api/utilities/csv#delimiter-separated-output) \ No newline at end of file diff --git a/docz/docs/03-demos/03-net/02-server/04-drash.md b/docz/docs/03-demos/03-net/02-server/04-drash.md new file mode 100644 index 0000000..0f4881c --- /dev/null +++ b/docz/docs/03-demos/03-net/02-server/04-drash.md @@ -0,0 +1,143 @@ +--- +title: Sheets in Drash +sidebar_label: Drash +pagination_prev: demos/net/network +pagination_next: demos/net/email +--- + +import current from '/version.js'; +import CodeBlock from '@theme/CodeBlock'; + +[Drash](https://drash.land/drash/) is a Deno server-side framework. The body +parser works in memory and can handle file uploads on hosted services. + +[SheetJS](https://sheetjs.com) is a JavaScript library for reading and writing +data from spreadsheets. + +This demo uses Drash and SheetJS to read and write data. We'll explore how to +parse uploaded files in a POST request handler and respond to GET requests with +downloadable spreadsheets. + +The ["Complete Example"](#complete-example) section includes a complete server. + +:::note + +This demo was last tested on 2023 October 16 against Drash 2.8.1 and Deno 1.37.2. + +::: + +## Integration Details + +The [SheetJS Deno module](/docs/getting-started/installation/nodejs) can be +imported from Drash server scripts. + +### Reading Data + +`Request#bodyParam` reads body parameters. For uploaded files, the `content` +property is a `Uint8Array` which can be parsed with the SheetJS `read` method[^1]. + +This example server responds to POST requests. The server will look for a file +in the request body under the `"upload"` key. If a file is present, the server +will parse the file, generate an HTML table using the `sheet_to_html` method[^2] +and respond with the HTML code: + +{`\ +// @deno-types="https://cdn.sheetjs.com/xlsx-${current}/package/types/index.d.ts" +import { read, utils } from 'https://cdn.sheetjs.com/xlsx-${current}/package/xlsx.mjs'; +\n\ +import * as Drash from "https://cdn.jsdelivr.net/gh/drashland/drash@v2.8.1/mod.ts"; +\n\ +class ParseResource extends Drash.Resource { + public paths = ["/"]; +\n\ + public POST(request: Drash.Request, response: Drash.Response) { + // assume a form upload like + // highlight-next-line + const file = request.bodyParam("upload"); + if (!file) throw new Error("File is required!"); +\n\ + // read file + // highlight-next-line + var wb = read(file.content); +\n\ + // respond with HTML from first worksheet + var ws = wb.Sheets[wb.SheetNames[0]]; + var html = utils.sheet_to_html(html) + return response.html(html); + } +}`} + + +### Writing Data + +Headers are manually set with `Response#headers.set` while the raw body is set +with `Response#send`. The raw body can be `Uint8Array` or `ArrayBuffer` objects. + +Given a SheetJS workbook object, the `write` method using `type: "buffer"`[^3] +generates data objects compatible with Drash. + +This example server responds to GET requests. The server will generate a SheetJS +worksheet object from an array of arrays[^4], build up a new workbook using the +`book_new`[^5] and `book_append_sheet`[^6] utility methods, generate a XLSX file +using `write`, and send the file with appropriate headers to initiate a download +with file name `"SheetJSDrash.xlsx"`: + +{`\ +// @deno-types="https://cdn.sheetjs.com/xlsx-${current}/package/types/index.d.ts" +import { utils, write } from 'https://cdn.sheetjs.com/xlsx-${current}/package/xlsx.mjs'; +\n\ +import * as Drash from "https://cdn.jsdelivr.net/gh/drashland/drash@v2.8.1/mod.ts"; +\n\ +class WriteResource extends Drash.Resource { + public paths = ["/export"]; +\n\ + public GET(request: Drash.Request, response: Drash.Response): void { + // create some fixed workbook + const data = ["SheetJS".split(""), [5,4,3,3,7,9,5]]; + const ws = utils.aoa_to_sheet(data); + const wb = utils.book_new(); utils.book_append_sheet(wb, ws, "data"); + \n\ + // write the workbook to XLSX as a Uint8Array + // highlight-next-line + const file = write(wb, { bookType: "xlsx", type: "buffer"}); + // set headers + response.headers.set("Content-Disposition", 'attachment; filename="SheetJSDrash.xlsx"'); + // send data + // highlight-next-line + return response.send("application/vnd.ms-excel", file); + } +}`} + + +## Complete Example + +1) Download [`SheetJSDrash.ts`](pathname:///drash/SheetJSDrash.ts): + +```bash +curl -LO https://docs.sheetjs.com/drash/SheetJSDrash.ts +``` + +2) Run the server: + +```bash +deno run --allow-net SheetJSDrash.ts +``` + +3) Download the test file + +4) Open `http://localhost:7262/` in your browser. + +Click "Choose File" and select `pres.numbers`. Then click "Submit" + +The page should show the contents of the file as an HTML table. + +5) Open `http://localhost:7262/export` in your browser. + +The page should attempt to download `SheetJSDrash.xlsx` . Open the new file. + +[^1]: See [`read` in "Reading Files"](/docs/api/parse-options) +[^2]: See [`sheet_to_html` in "Utilities"](/docs/api/utilities/html#html-table-output) +[^3]: See [`write` in "Writing Files"](/docs/api/write-options) +[^4]: See [`aoa_to_sheet` in "Utilities"](/docs/api/utilities/array#array-of-arrays-input) +[^5]: See [`book_new` in "Utilities"](/docs/api/utilities/wb) +[^6]: See [`book_append_sheet` in "Utilities"](/docs/api/utilities/wb) diff --git a/docz/docs/03-demos/03-net/02-server/11-nestjs.md b/docz/docs/03-demos/03-net/02-server/11-nestjs.md new file mode 100644 index 0000000..88a2df8 --- /dev/null +++ b/docz/docs/03-demos/03-net/02-server/11-nestjs.md @@ -0,0 +1,233 @@ +--- +title: Sheets in NestJS +sidebar_label: NestJS +pagination_prev: demos/net/network +pagination_next: demos/net/email +--- + +import current from '/version.js'; +import CodeBlock from '@theme/CodeBlock'; + +[NestJS](https://nestjs.com/) is a NodeJS framework for building server-side +applications. + +[SheetJS](https://sheetjs.com) is a JavaScript library for reading and writing +data from spreadsheets. + +This demo uses NestJS and SheetJS to read and write data. We'll explore how to +parse uploaded files in a POST request handler and respond to GET requests with +downloadable spreadsheets. + +The ["Complete Example"](#complete-example) section includes a complete server. + +:::note + +This demo was tested on 2023 October 15 using NestJS `10.2.7`. + +::: + +## Integration Details + +The [SheetJS NodeJS module](/docs/getting-started/installation/nodejs) can be +imported from NestJS controller scripts. + +:::caution pass + +NestJS does not follow the conventional NodeJS server code structure. + +It is strongly recommended to review the official documentation. The official +documentation covers Controller structure and various JS Decorators. + +::: + +### Exporting Data to Workbooks (GET) + +The SheetJS `write` method[^1] with the option `type: "buffer"` generates NodeJS +Buffer objects containing the raw file data. + +NestJS strongly recommends wrapping the Buffer in a `StreamableFile` object[^2]. + +The exported filename can be specified in a `@Header` decorator[^3] + +The following demo NestJS Controller will respond to GET requests to `/download` +with a XLSX spreadsheet. In this example, the SheetJS `aoa_to_sheet` method[^4] +generates a sheet object and the `book_new` and `book_append_sheet` helpers[^5] +build the workbook object. + +```ts title="src/sheetjs/sheetjs.controller.js" +import { Controller, Get, Header, StreamableFile } from '@nestjs/common'; +import { utils, write } from 'xlsx'; + +@Controller('sheetjs') +export class SheetjsController { + @Get('download') + @Header('Content-Disposition', 'attachment; filename="SheetJSNest.xlsx"') + async downloadXlsxFile(): Promise { + var ws = utils.aoa_to_sheet(["SheetJS".split(""), [5,4,3,3,7,9,5]]); + var wb = utils.book_new(); utils.book_append_sheet(wb, ws, "Data"); + // highlight-start + /* generate buffer */ + var buf = write(wb, {type: "buffer", bookType: "xlsx"}); + /* Return a streamable file */ + return new StreamableFile(buf); + // highlight-end + } +} +``` + +### Parsing Uploaded Files (POST) + +:::note pass + +[The NestJS documentation](https://docs.nestjs.com/techniques/file-upload) has +detailed instructions for file upload support. + +::: + +As explained in the NestJS documentation, in a POST request handler, the +`FileInterceptor` interceptor and `UploadedFile` decorator are used in tandem to +expose a file uploaded with a specified key. + +The file object has a `buffer` property which represents the raw bytes. The +SheetJS `read` method[^6] can parse the Buffer into a workbook object[^7]. + +The following demo NestJS Controller will respond to POST requests to `/upload`. +Assuming the `upload` field of the form data is the file, the SheetJS `read` +method will parse the file. CSV rows are generated from the first worksheet +using the SheetJS `sheet_to_csv` method[^8]. + +```ts title="src/sheetjs/sheetjs.controller.js" +import { Controller, Post, UploadedFile, UseInterceptors } from '@nestjs/common'; +import { FileInterceptor } from '@nestjs/platform-express'; +import { read, utils } from 'xlsx'; + +@Controller('sheetjs') +export class SheetjsController { + @Post('upload') // + @UseInterceptors(FileInterceptor('upload')) + async uploadXlsxFile(@UploadedFile() file: Express.Multer.File) { + /* file.buffer is a Buffer */ + // highlight-next-line + const wb = read(file.buffer); + /* generate CSV of first worksheet */ + return utils.sheet_to_csv(wb.Sheets[wb.SheetNames[0]]); + } +} +``` + +## Complete Example + +1) Create a new NestJS project: + +```bash +npx @nestjs/cli@latest new -p npm sheetjs-nest +cd sheetjs-nest +``` + +2) Install the `@types/multer` package as a development dependency: + +```bash +npm i --save-dev @types/multer +``` + +3) Install the SheetJS library: + +{`\ +npm i --save https://cdn.sheetjs.com/xlsx-${current}/xlsx-${current}.tgz`} + + +4) Make a folder for uploaded files: + +```bash +mkdir -p upload +``` + +5) Create a new controller: + +```bash +npx @nestjs/cli generate controller sheetjs +``` + +6) Replace `src/sheetjs/sheetjs.controller.ts` with the following code block: + +```ts title="src/sheetjs/sheetjs.controller.js" +import { Controller, Get, Header, Post, StreamableFile, UploadedFile, UseInterceptors } from '@nestjs/common'; +import { FileInterceptor } from '@nestjs/platform-express'; +import { read, utils, write } from 'xlsx'; + +@Controller('sheetjs') +export class SheetjsController { + @Post('upload') // + @UseInterceptors(FileInterceptor('upload')) + async uploadXlsxFile(@UploadedFile() file: Express.Multer.File) { + /* file.path is a path to the workbook */ + const wb = read(file.buffer); + /* generate CSV of first worksheet */ + return utils.sheet_to_csv(wb.Sheets[wb.SheetNames[0]]); + } + + @Get('download') + @Header('Content-Disposition', 'attachment; filename="SheetJSNest.xlsx"') + async downloadXlsxFile(): Promise { + var ws = utils.aoa_to_sheet(["SheetJS".split(""), [5,4,3,3,7,9,5]]); + var wb = utils.book_new(); utils.book_append_sheet(wb, ws, "Data"); + /* generate buffer */ + var buf = write(wb, {type: "buffer", bookType: "xlsx"}); + /* Return a streamable file */ + return new StreamableFile(buf); + } +} +``` + +7) Start the server with + +```bash +npx @nestjs/cli start +``` + +:::note + +In the most recent test, the process failed with a message referencing Multer: + +``` +src/sheetjs/sheetjs.controller.ts:9:54 - error TS2694: Namespace 'global.Express' has no exported member 'Multer'. + +9 async uploadXlsxFile(@UploadedFile() file: Express.Multer.File) { + ~~~~~~ +``` + +This error indicates that `@types/multer` is not available. + +**This is a bug in the `npm` client** + +The recommended fix is to install `@types/multer` again: + +```bash +npm i --save-dev @types/multer +``` + +::: + +8) Test POST requests in the terminal with : + +```bash +curl -LO https://sheetjs.com/pres.numbers +curl -X POST -F upload=@pres.numbers http://localhost:3000/sheetjs/upload +``` + +The response should show the data in CSV rows. + +9) Test GET requests by opening `http://localhost:3000/sheetjs/download` in a +web browser. + +The browser should attempt to download `SheetJSNest.xlsx`. Save the file and +open in a spreadsheet editor. + +[^1]: See [`write` in "Writing Files"](/docs/api/write-options) +[^2]: See ["Streaming files"](https://docs.nestjs.com/techniques/streaming-files) in the NestJS documentation. +[^3]: See ["Headers"](https://docs.nestjs.com/controllers#headers) in the NestJS documentation. +[^4]: See [`aoa_to_sheet` in "Utilities"](/docs/api/utilities/array#array-of-arrays-input) +[^5]: See ["Workbook Helpers" in "Utilities"](/docs/api/utilities/wb) for details on `book_new` and `book_append_sheet`. +[^6]: See [`read` in "Reading Files"](/docs/api/parse-options) +[^7]: See ["Workbook Object"](/docs/csf/book) +[^8]: See [`sheet_to_csv` in "CSV and Text"](/docs/api/utilities/csv#delimiter-separated-output) \ No newline at end of file diff --git a/docz/docs/03-demos/03-net/02-server/19-fastify.md b/docz/docs/03-demos/03-net/02-server/19-fastify.md new file mode 100644 index 0000000..00fda2f --- /dev/null +++ b/docz/docs/03-demos/03-net/02-server/19-fastify.md @@ -0,0 +1,187 @@ +--- +title: Sheets in FastifyJS +sidebar_label: FastifyJS +pagination_prev: demos/net/network +pagination_next: demos/net/email +--- + +import current from '/version.js'; +import CodeBlock from '@theme/CodeBlock'; + +[FastifyJS](https://openjsf.org/projects/) is a NodeJS web framework. + +[SheetJS](https://sheetjs.com) is a JavaScript library for reading and writing +data from spreadsheets. + +This demo uses FastifyJS and SheetJS to read and write data. We'll explore how +to parse uploaded files in a POST request handler and respond to GET requests +with downloadable spreadsheets. + +The ["Complete Example"](#complete-example) section includes a complete server. + +:::note + +This demo was verified on 2023 October 16 using `fastify@4.24.2` + +::: + +## Integration Details + +The [SheetJS NodeJS module](/docs/getting-started/installation/nodejs) can be +imported from scripts that use FastifyJS. + +### Exporting Data to Workbooks (GET) + +The SheetJS `write` method[^1] with the option `type: "buffer"` generates NodeJS +Buffer objects containing the raw file data. + +FastifyJS can directly handle `Buffer` data in `Response#end` + +The exported filename can be specified using the `Content-Disposition` header. + +The following demo FastifyJS server will respond to GET requests to `/download` +with a XLSX spreadsheet. In this example, the SheetJS `aoa_to_sheet` method[^2] +generates a sheet object and the `book_new` and `book_append_sheet` helpers[^3] +build the workbook object. + +```js +/* GET / returns a workbook */ +fastify.get('/', (req, reply) => { + /* make a workbook */ + var wb = XLSX.read("S,h,e,e,t,J,S\n5,4,3,3,7,9,5", {type: "binary"}); + + /* write to Buffer */ + const buf = XLSX.write(wb, {type:"buffer", bookType: "xlsx"}); + + /* set Content-Disposition header and send data */ + // highlight-next-line + reply.header('Content-Disposition', 'attachment; filename="SheetJSFastify.xlsx"').send(buf); +}); +``` + +### Parsing Uploaded Files (POST) + +`@fastify/multipart`, which uses `busbuy` under the hood, must be registered: + +```js +/* load SheetJS Library */ +const XLSX = require("xlsx"); +/* load fastify and enable body parsing */ +const fastify = require('fastify')({logger: true}); +// highlight-next-line +fastify.register(require('@fastify/multipart'), { attachFieldsToBody: true }); +``` + +Once registered with the option `attachFieldsToBody`, route handlers can use +`req.body` directly. + +Each file object in the body has a `toBuffer` method that resolves to a Buffer +object. The SheetJS `read` method[^4] can read the Buffer and generate a +workbook object[^5]. + +The following demo FastifyJS server will respond to POST requests to `/upload`. +Assuming the `upload` field of the form data is the file, the SheetJS `read` +method will parse the file. CSV rows are generated from the first worksheet +using the SheetJS `sheet_to_csv` method[^6]. + +```js +/* POST / reads submitted file and exports to requested format */ +fastify.post('/', async(req, reply) => { + /* "file" is the name of the field in the HTML form*/ + const file = req.body.upload; + /* toBuffer returns a promise that resolves to a Buffer */ + // highlight-next-line + const buf = await file.toBuffer(); + /* `XLSX.read` can read the Buffer */ + const wb = XLSX.read(buf); + /* reply with a CSV */ + reply.send(XLSX.utils.sheet_to_csv(wb.Sheets[wb.SheetNames[0]])); +}); +``` + +:::caution pass + +Out of the box, Fastify will return an error `FST_ERR_CTP_BODY_TOO_LARGE` when +processing large spreadsheets (`statusCode 413`). This is a Fastify issue. + +The default body size limit (including all uploaded files and fields) is 1 MB. +It can be increased by setting the `bodyLimit` option during server creation: + +```js +/* increase request body size limit to 5MB = 5 * 1024 * 1024 bytes */ +const fastify = require('fastify')({bodyLimit: 5 * 1024 * 1024}); +``` + +::: + +## Complete Example + +0) Save the following snippet to `SheetJSFastify.js`: + +```js title="SheetJSFastify.js" +/* load SheetJS Library */ +const XLSX = require("xlsx"); +/* load fastify and enable body parsing */ +const fastify = require('fastify')({logger: true}); +fastify.register(require('@fastify/multipart'), { attachFieldsToBody: true }); + +/* GET / returns a workbook */ +fastify.get('/', (req, reply) => { + /* make a workbook */ + var wb = XLSX.read("S,h,e,e,t,J,S\n5,4,3,3,7,9,5", {type: "binary"}); + + /* write to Buffer */ + const buf = XLSX.write(wb, {type:"buffer", bookType: "xlsx"}); + + /* set Content-Disposition header and send data */ + reply.header('Content-Disposition', 'attachment; filename="SheetJSFastify.xlsx"').send(buf); +}); + +/* POST / reads submitted file and exports to requested format */ +fastify.post('/', async(req, reply) => { + + /* "file" is the name of the field in the HTML form*/ + const file = req.body.upload; + + /* toBuffer returns a promise that resolves to a Buffer */ + const wb = XLSX.read(await file.toBuffer()); + + /* send back a CSV */ + reply.send(XLSX.utils.sheet_to_csv(wb.Sheets[wb.SheetNames[0]])); +}); + +/* start */ +fastify.listen({port: process.env.PORT || 3000}, (err, addr) => { if(err) throw err; }); +``` + +1) Install dependencies: + +{`\ +npm i --save https://cdn.sheetjs.com/xlsx-${current}/xlsx-${current}.tgz fastify@4.24.2 @fastify/multipart@8.0.0`} + + +2) Start server + +```bash +node SheetJSFastify.js +``` + +3) Test POST requests using : + +```bash +curl -LO https://sheetjs.com/pres.numbers +curl -X POST -F upload=@pres.numbers http://localhost:3000/ +``` + +The response should show the data in CSV rows. + +4) Test GET requests by opening `http://localhost:3000/` in your browser. + +It should prompt to download `SheetJSFastify.xlsx` + +[^1]: See [`write` in "Writing Files"](/docs/api/write-options) +[^2]: See [`aoa_to_sheet` in "Utilities"](/docs/api/utilities/array#array-of-arrays-input) +[^3]: See ["Workbook Helpers" in "Utilities"](/docs/api/utilities/wb) for details on `book_new` and `book_append_sheet`. +[^4]: See [`readFile` in "Reading Files"](/docs/api/parse-options) +[^5]: See ["Workbook Object"](/docs/csf/book) +[^6]: See [`sheet_to_csv` in "CSV and Text"](/docs/api/utilities/csv#delimiter-separated-output) \ No newline at end of file diff --git a/docz/docs/03-demos/03-net/02-server/_category_.json b/docz/docs/03-demos/03-net/02-server/_category_.json new file mode 100644 index 0000000..ed2370a --- /dev/null +++ b/docz/docs/03-demos/03-net/02-server/_category_.json @@ -0,0 +1,5 @@ +{ + "label": "HTTP Server Processing", + "collapsed": false, + "position": 2 +} \ No newline at end of file diff --git a/docz/docs/03-demos/03-net/02-server/index.md b/docz/docs/03-demos/03-net/02-server/index.md new file mode 100644 index 0000000..4f27f50 --- /dev/null +++ b/docz/docs/03-demos/03-net/02-server/index.md @@ -0,0 +1,246 @@ +--- +title: HTTP Server Processing +pagination_prev: demos/net/network +pagination_next: demos/net/email +--- + +import current from '/version.js'; +import CodeBlock from '@theme/CodeBlock'; + +Server-Side JS platforms like NodeJS and Deno have built-in APIs for listening +on network interfaces. They provide wrappers for requests and responses. + +## Overview + +#### Parsing Files in POST Requests + +Typically servers receive form data with content type `multipart/form-data` or +`application/x-www-form-urlencoded`. The platforms themselves typically do not +provide "body parsing" functions, instead leaning on the community to supply +modules to take the encoded data and split into form fields and files. + +NodeJS servers typically use a parser like `formidable`. In the example below, +`formidable` will write to file and `XLSX.readFile` will read the file: + +```js +var XLSX = require("xlsx"); // This is using the CommonJS build +var formidable = require("formidable"); + +require("http").createServer(function(req, res) { + if(req.method !== "POST") return res.end(""); + + /* parse body and implement logic in callback */ + // highlight-next-line + (new formidable.IncomingForm()).parse(req, function(err, fields, files) { + /* if successful, files is an object whose keys are param names */ + // highlight-next-line + var file = files["upload"]; // + /* file.path is a location in the filesystem, usually in a temp folder */ + // highlight-next-line + var wb = XLSX.readFile(file.filepath); + // print the first worksheet back as a CSV + res.end(XLSX.utils.sheet_to_csv(wb.Sheets[wb.SheetNames[0]])); + }); +}).listen(process.env.PORT || 3000); +``` + +`XLSX.read` will accept NodeJS buffers as well as `Uint8Array`, Base64 strings, +binary strings, and plain Arrays of bytes. This covers the interface types of +a wide variety of frameworks. + +#### Writing Files in GET Requests + +Typically server libraries use a response API that accepts `Uint8Array` data. +`XLSX.write` with the option `type: "buffer"` will generate data. To force the +response to be treated as an attachment, set the `Content-Disposition` header: + +```js +var XLSX = require("xlsx"); // This is using the CommonJS build + +require("http").createServer(function(req, res) { + if(req.method !== "GET") return res.end(""); + var wb = XLSX.read("S,h,e,e,t,J,S\n5,4,3,3,7,9,5", {type: "binary"}); + // highlight-start + res.setHeader('Content-Disposition', 'attachment; filename="SheetJS.xlsx"'); + res.end(XLSX.write(wb, {type:"buffer", bookType: "xlsx"})); + // highlight-end +}).listen(process.env.PORT || 3000); +``` + +## NodeJS + +When processing small files, the work is best handled in the server response +handler function. This approach is used in the "Framework Demos" section. + +When processing large files, the direct approach will freeze the server. NodeJS +provides ["Worker Threads"](#worker-threads) for this exact use case. + +### Framework Demos + +#### Express + +**[The exposition has been moved to a separate page.](/docs/demos/net/server/express)** + +#### NestJS + +**[The exposition has been moved to a separate page.](/docs/demos/net/server/nestjs)** + +#### Fastify + +**[The exposition has been moved to a separate page.](/docs/demos/net/server/fastify)** + +### Worker Threads + +NodeJS "Worker Threads" were introduced in v14 and eventually marked as stable +in v16. Coupled with `AsyncResource`, a simple thread pool enables processing +without blocking the server! The official NodeJS docs include a sample worker +pool implementation. + +This example uses ExpressJS to create a general XLSX conversion service, but +the same approach applies to any NodeJS server side framework. + +When reading large files, it is strongly recommended to run the body parser in +the main server process. Body parsers like `formidable` will write uploaded +files to the filesystem, and the file path should be passed to the worker (and +the worker would be responsible for reading and cleaning up the files). + +:::note pass + +The `child_process` module can also spawn [command-line tools](/docs/demos/cli). +That approach is not explored in this demo. + +::: + +
Complete Example (click to show) + +:::note + +This demo was last tested on 2023 August 27 with NodeJS 20.5.1 + ExpressJS +4.18.2 + Formidable 2.1.1 + +::: + +0) Create a simple ECMAScript-Module-enabled `package.json`: + +```json title="package.json" +{ "type": "module" } +``` + +1) Install the dependencies: + +{`\ +npm i --save https://cdn.sheetjs.com/xlsx-${current}/xlsx-${current}.tgz express@4.18.2 formidable@2.1.1`} + + +2) Create a worker script `worker.js` that listens for messages. When a message +is received, it will read the file from the filesystem, generate and pass back a +new XLSX file, and delete the original file: + +```js title="worker.js" +/* load the worker_threads module */ +import { parentPort } from 'node:worker_threads'; + +/* load the SheetJS module and hook to FS */ +import { set_fs, readFile, write } from 'xlsx'; +import * as fs from 'fs'; +set_fs(fs); + +/* the server will send a message with the `path` field */ +parentPort.on('message', (task) => { + /* highlight-start */ + // read file + const wb = readFile(task.path, { dense: true }); + // send back XLSX + parentPort.postMessage(write(wb, { type: "buffer", bookType: "xlsx" })); + /* highlight-end */ + // remove file + fs.unlink(task.path, ()=>{}); +}); +``` + +3) Download [`worker_pool.js`](pathname:///server/worker_pool.js): + +```bash +curl -LO https://docs.sheetjs.com/server/worker_pool.js +``` + +(this is a slightly modified version of the example in the NodeJS docs) + +4) Save the following server code to `main.mjs`: + +```js title="main.mjs" +/* load dependencies */ +import os from 'node:os'; +import process from 'node:process' +import express from 'express'; +import formidable from 'formidable'; + +/* load worker pool */ +import WorkerPool from './worker_pool.js'; + +const pool = new WorkerPool(os.cpus().length); +process.on("beforeExit", () => { pool.close(); }) + +/* create server */ +const app = express(); +app.post('/', (req, res, next) => { + // parse body + const form = formidable({}); + form.parse(req, (err, fields, files) => { + // look for "upload" field + if(err) return next(err); + if(!files["upload"]) return next(new Error("missing `upload` file")); + + // send a message to the worker with the path to the uploaded file + // highlight-next-line + pool.runTask({ path: files["upload"].filepath }, (err, result) => { + if(err) return next(err); + // send the file back as an attachment + res.attachment("SheetJSPool.xlsx"); + res.status(200).end(result); + }); + }); +}); + +// start server +app.listen(7262, () => { console.log(`Example app listening on port 7262`); }); +``` + +5) Run the server: + +```bash +node main.mjs +``` + +Test with the [`pres.numbers` sample file](https://sheetjs.com/pres.numbers): + +```bash +curl -LO https://sheetjs.com/pres.numbers +curl -X POST -F upload=@pres.numbers http://localhost:7262/ -J -O +``` + +This will generate `SheetJSPool.xlsx`. + +
+ +## Other Platforms + +### Deno + +:::caution pass + +Many hosted services like Deno Deploy do not offer filesystem access. + +This breaks web frameworks that use the filesystem in body parsing. + +::: + +Deno provides the basic elements to implement a web server. It does not provide +a body parser out of the box. + +#### Drash + +In testing, [Drash](https://drash.land/drash/) had an in-memory body parser +which could handle file uploads on hosted services like Deno Deploy. + +**[The exposition has been moved to a separate page.](/docs/demos/net/server/drash)** diff --git a/docz/docs/03-demos/03-net/03-email.md b/docz/docs/03-demos/03-net/03-email.md index c0750ba..e536b14 100644 --- a/docz/docs/03-demos/03-net/03-email.md +++ b/docz/docs/03-demos/03-net/03-email.md @@ -1,5 +1,6 @@ --- title: Electronic Mail +pagination_prev: demos/net/server/index --- import current from '/version.js'; diff --git a/docz/docs/03-demos/08-local/09-indexeddb.md b/docz/docs/03-demos/08-local/09-indexeddb.md index 2bc0b4b..5d1169e 100644 --- a/docz/docs/03-demos/08-local/09-indexeddb.md +++ b/docz/docs/03-demos/08-local/09-indexeddb.md @@ -15,8 +15,8 @@ sidebar_custom_props: IndexedDB is a very low-level API. -Browser vendors recommend using libraries or [WebSQL](/docs/demos/data/websql) -in production applications. +Browser vendors recommend using [WebSQL](/docs/demos/local/websql) or wrapper +libraries in production applications. ::: diff --git a/docz/docs/03-demos/09-cloud/11-aws.md b/docz/docs/03-demos/09-cloud/11-aws.md index 83158fb..bf39824 100644 --- a/docz/docs/03-demos/09-cloud/11-aws.md +++ b/docz/docs/03-demos/09-cloud/11-aws.md @@ -354,7 +354,8 @@ required in NodeJS scripts. ### Connecting to S3 -The `aws-sdk` module includes a function `S3` that performs the connection. +The `aws-sdk` module exports a function `S3` that performs the connection. The +function expects an options object that includes an API version and credentials. Access keys for an IAM user[^9] must be used: ```js @@ -370,7 +371,6 @@ var s3 = new AWS.S3({ apiVersion: '2006-03-01', credentials: { accessKeyId, secretAccessKey } }); - ``` ### Downloading Data diff --git a/docz/static/chakra/Makefile b/docz/static/chakra/Makefile index 7736782..25f15c0 100644 --- a/docz/static/chakra/Makefile +++ b/docz/static/chakra/Makefile @@ -9,7 +9,7 @@ ICU4C=ChakraCore/deps/thirdparty/icu/lib ICULIB=$(ICU4C)/libicui18n.a $(ICU4C)/libicuuc.a $(ICU4C)/libicudata.a sheetjs.ch: sheetjs.ch.cpp - clang++ $< $(CFLAGS) $(CHAKRALIB) $(ICULIB) -lm -ldl -Wno-c++11-compat-deprecated-writable-strings -Wno-deprecated-declarations -Wno-unknown-warning-option -o $@ + clang++ $< $(CFLAGS) $(CHAKRALIB) $(ICULIB) -lm -ldl -Wno-c++11-compat-deprecated-writable-strings -Wno-deprecated-declarations -Wno-unknown-warning-option -o $@ endif ifeq ($(UNAME__S),Darwin) @@ -17,7 +17,7 @@ ICU4C=/usr/local/opt/icu4c/lib ICULIB=$(ICU4C)/libicudata.a $(ICU4C)/libicuuc.a $(ICU4C)/libicui18n.a sheetjs.ch: sheetjs.ch.cpp - g++ $< $(CFLAGS) -Wl,-force_load $(CHAKRALIB) $(ICULIB) -framework CoreFoundation -framework Security -lm -ldl -Wno-c++11-compat-deprecated-writable-strings -Wno-deprecated-declarations -Wno-unknown-warning-option -o $@ + g++ $< $(CFLAGS) -Wl,-force_load $(CHAKRALIB) $(ICULIB) -framework CoreFoundation -framework Security -lm -ldl -Wno-c++11-compat-deprecated-writable-strings -Wno-deprecated-declarations -Wno-unknown-warning-option -o $@ endif diff --git a/docz/static/drash/SheetJSDrash.ts b/docz/static/drash/SheetJSDrash.ts new file mode 100644 index 0000000..2c1b8f9 --- /dev/null +++ b/docz/static/drash/SheetJSDrash.ts @@ -0,0 +1,78 @@ +/*! sheetjs (C) SheetJS -- https://sheetjs.com */ +// @deno-types="https://cdn.sheetjs.com/xlsx-latest/package/types/index.d.ts" +import { read, utils, write, set_cptable } from 'https://cdn.sheetjs.com/xlsx-latest/package/xlsx.mjs'; +import * as cptable from 'https://cdn.sheetjs.com/xlsx-latest/package/dist/cpexcel.full.mjs'; +set_cptable(cptable); + +import * as Drash from "https://cdn.jsdelivr.net/gh/drashland/drash@v2.8.1/mod.ts"; + +class ParseResource extends Drash.Resource { + public paths = ["/"]; + + public POST(request: Drash.Request, response: Drash.Response) { + const file = request.bodyParam("file"); + if (!file) throw new Error("File is required!"); + var wb = read(file.content); + return response.html(utils.sheet_to_html(wb.Sheets[wb.SheetNames[0]])); + } + + public GET(request: Drash.Request, response: Drash.Response): void { + return response.html(`\ + + + + SheetJS Spreadsheet to HTML Conversion Service + + + +
SheetJS Spreadsheet Conversion Service

+API
+
+Send a POST request to http://localhost:7262/ with the file in the "file" body parameter:
+
+$ curl -X POST -F"file=@test.xlsx" http://localhost:7262/
+
+The response will be an HTML TABLE generated from the first worksheet.
+
+Try it out!

+
+
+
+Use the file input element to select a file, then click "Submit"
+
+
+

+
+ +`, + ); + } +} + +class WriteResource extends Drash.Resource { + public paths = ["/export"]; + + public GET(request: Drash.Request, response: Drash.Response): void { + // create some fixed workbook + const data = ["SheetJS".split(""), [5,4,3,3,7,9,5]]; + const ws = utils.aoa_to_sheet(data); + const wb = utils.book_new(); utils.book_append_sheet(wb, ws, "data"); + // write the workbook to XLSX as a Uint8Array + const file = write(wb, { bookType: "xlsx", type: "buffer"}); + // set headers + response.headers.set("Content-Disposition", 'attachment; filename="SheetJSDrash.xlsx"'); + // send data + return response.send("application/vnd.ms-excel", file); + } +} + +const server = new Drash.Server({ + hostname: "", + port: 7262, + protocol: "http", + resources: [ ParseResource, WriteResource ], +}); + +server.run(); + +console.log(`Server running at ${server.address}.`); diff --git a/docz/static/server/SheetJSDrash.ts b/docz/static/server/SheetJSDrash.ts index dfc4f3d..2c1b8f9 100644 --- a/docz/static/server/SheetJSDrash.ts +++ b/docz/static/server/SheetJSDrash.ts @@ -4,7 +4,7 @@ import { read, utils, write, set_cptable } from 'https://cdn.sheetjs.com/xlsx-la import * as cptable from 'https://cdn.sheetjs.com/xlsx-latest/package/dist/cpexcel.full.mjs'; set_cptable(cptable); -import * as Drash from "https://cdn.jsdelivr.net/gh/drashland/drash@v2.8.0/mod.ts"; +import * as Drash from "https://cdn.jsdelivr.net/gh/drashland/drash@v2.8.1/mod.ts"; class ParseResource extends Drash.Resource { public paths = ["/"];