diff --git a/docz/docs/02-getting-started/02-example.mdx b/docz/docs/02-getting-started/02-example.mdx index 7e84ca9..a610b1c 100644 --- a/docz/docs/02-getting-started/02-example.mdx +++ b/docz/docs/02-getting-started/02-example.mdx @@ -18,7 +18,7 @@ and birthdays. [Click here](#live-demo) to jump to the live demo ### Raw Data -[The raw data is available in JSON form](https://theunitedstates.io/congress-legislators/data/executive.json). +[The raw data is available in JSON form](https://theunitedstates.io/congress-legislators/executive.json). For convenience, it has been [mirrored here](https://sheetjs.com/data/executive.json) The data result is an Array of objects. This is the data for John Adams: diff --git a/docz/docs/03-demos/03-database.md b/docz/docs/03-demos/03-database.md index 9dee237..3eea2d9 100644 --- a/docz/docs/03-demos/03-database.md +++ b/docz/docs/03-demos/03-database.md @@ -545,8 +545,7 @@ function SheetJStorage() { ### IndexedDB -[`localForage`](https://localforage.github.io/localForage/) is a lightweight -IndexedDB wrapper that presents an async Storage interface. +`localForage` is a IndexedDB wrapper that presents an async Storage interface. Arrays of objects can be stored using `JSON.stringify` using row index as key: diff --git a/docz/docs/03-demos/25-azure.md b/docz/docs/03-demos/25-azure.md index 0379b02..12a2128 100644 --- a/docz/docs/03-demos/25-azure.md +++ b/docz/docs/03-demos/25-azure.md @@ -16,6 +16,12 @@ will be available in the future. This demo focuses on two key offerings: cloud storage ("Azure Blob Storage") and the "Serverless Function" platform ("Azure Functions"). +:::note + +This was tested on 2022 August 21. + +::: + ## Azure Functions This discussion focuses on the "HTTP Trigger" function type. @@ -210,12 +216,6 @@ Get the function url and test using the same sequence as in step 5. ## Azure Blob Storage -:::note - -This was tested on 2022 August 21. - -::: - The main module for Azure Blob Storage is `@azure/storage-blob`. This example was tested using the "Connection String" authentication method. The strings are found in the Azure Portal under "Access Keys" for the storage account. diff --git a/docz/docs/03-demos/26-aws.md b/docz/docs/03-demos/26-aws.md new file mode 100644 index 0000000..51842a8 --- /dev/null +++ b/docz/docs/03-demos/26-aws.md @@ -0,0 +1,266 @@ +--- +sidebar_position: 26 +title: Amazon Web Services +--- + +AWS is a Cloud Services platform which includes traditional virtual machine +support, "Serverless Functions", cloud storage and much more. + +:::caution + +AWS iterates quickly and there is no guarantee that the referenced services +will be available in the future. + +::: + +This demo focuses on two key offerings: cloud storage ("S3") and the +"Serverless Function" platform ("Lambda"). + +:::note + +This was tested on 2022 August 21. + +::: + +## AWS Lambda Functions + +In this demo, the "Function URL" (automatic API Gateway management) features +are used. Older deployments required special "Binary Media Types" to handle +formats like XLSX. At the time of testing, the configuration was not required. + +### Reading Data + +In the Lambda handler method, the `event.body` attribute is a Base64-encoded +string. The `busboy` body parser can accept a decoded body. + +
Code Sample (click to show) + +```js +const XLSX = require('xlsx'); +var Busboy = require('busboy'); + +exports.handler = function(event, context, callback) { + /* set up busboy */ + var ctype = event.headers['Content-Type']||event.headers['content-type']; + var bb = Busboy({headers:{'content-type':ctype}}); + + /* busboy is evented; accumulate the fields and files manually */ + var fields = {}, files = {}; + bb.on('error', function(err) { callback(null, { body: err.message }); }); + bb.on('field', function(fieldname, val) {fields[fieldname] = val }); + // highlight-start + bb.on('file', function(fieldname, file, filename) { + /* concatenate the individual data buffers */ + var buffers = []; + file.on('data', function(data) { buffers.push(data); }); + file.on('end', function() { files[fieldname] = [Buffer.concat(buffers), filename]; }); + }); + // highlight-end + + /* on the finish event, all of the fields and files are ready */ + bb.on('finish', function() { + /* grab the first file */ + var f = files["upload"]; + if(!f) callback(new Error("Must submit a file for processing!")); + + /* f[0] is a buffer */ + // highlight-next-line + var wb = XLSX.read(f[0]); + + /* grab first worksheet and convert to CSV */ + var ws = wb.Sheets[wb.SheetNames[0]]; + callback(null, { statusCode: 200, body: XLSX.utils.sheet_to_csv(ws) }); + }); + + /* start the processing */ + // highlight-next-line + bb.end(Buffer.from(event.body, "base64")); +}; +``` + +
+ +### Writing Data + +For safely transmitting binary data, the `base64` type should be used. Lambda +callback response `isBase64Encoded` property forces a binary download: + +
Code Sample (click to show) + +```js +var XLSX = require('xlsx'); + +exports.handler = function(event, context, callback) { + /* make workbook */ + var wb = XLSX.read("S,h,e,e,t,J,S\n5,4,3,3,7,9,5", {type: "binary"}); + /* write to XLSX file in base64 encoding */ + // highlight-next-line + var body = XLSX.write(wb, {type:"base64", bookType: "xlsx"}); + /* mark as attached file */ + var headers = { "Content-Disposition": 'attachment; filename="SheetJSLambda.xlsx"'}; + /* Send back data */ + callback(null, { + statusCode: 200, + // highlight-next-line + isBase64Encoded: true, + body: body, + headers: headers + }); +}; +``` + +
+ +### Demo + +
Complete Example (click to show) + +0) Review the quick start for JavaScript on AWS + +1) Create a new folder and download [`index.js`](pathname:///aws/index.js): + +```bash +mkdir SheetJSLambda +cd SheetJSLambda +curl -LO https://docs.sheetjs.com/aws/index.js +``` + +2) Install dependencies to the current directory; + +```bash +mkdir node_modules +npm install https://cdn.sheetjs.com/xlsx-latest/xlsx-latest.tgz busboy +``` + +3) Create a .zip package of the contents of the folder: + +```bash +yes | zip -c ../SheetJSLambda.zip -r . +``` + +4) In the web interface for AWS Lambda, create a new Function with the options: + +- Select "Author from scratch" (default choice when last verified) +- "Function Name": SheetJSLambda +- "Runtime": "Node.js" (select the version in the "Latest supported" block) +- Advanced Settings: + + check "Enable function URL" + + Auth type: NONE + + Check "Configure CORS" + +5) In the Interface, click "Upload from" and select ".zip file". Click the +"Upload" button in the modal, select SheetJSLambda.zip, and click "Save". + +At the time of writing, the ZIP is small enough that the Lambda code editor +will load the package. + +6) Enable external access to the function. + +Under Configuration > Function URL, click "Edit" and ensure that Auth type is +set to NONE. If it is not, select NONE and hit Save. + +Under Configuration > Permissions, scroll down to "Resource-based policy". +If no policy statements are defined, select "Add Permission" with the options: + +- Select "Function URL" at the top +- Auth type: NONE +- Ensure that Statement ID is set to `FunctionURLAllowPublicAccess` +- Ensure that Principal is set to `*` +- Ensure that Action is set to `lambda:InvokeFunctionUrl` + +Click "Save" and a new Policy statement should be created. + +7) Find the Function URL (It is in the "Function Overview" section). + +Try to access that URL in a web browser and the site will try to download +`SheetJSLambda.xlsx`. Save and open the file to confirm it is valid. + +To test parsing, download and run + +```bash +curl -X POST -F "upload=@pres.numbers" FUNCTION_URL +``` + +The result should be a CSV output of the first sheet. + +
+ +## S3 Storage + +The main module for S3 and all AWS services is `aws-sdk`. + +### Reading Data + +The `s3#getObject` method returns an object with a `createReadStream` method. +Buffers can be concatenated and passed to `XLSX.read`: + +
Code Sample (click to show) + +```js title="SheetJSReadFromS3.mjs" +var XLSX = require("xlsx"); +var AWS = require('aws-sdk'); + +/* replace these constants */ +var accessKeyId = ""; +var secretAccessKey = ""; +var Bucket = ""; +var Key = ""; + +/* Get stream */ +var s3 = new AWS.S3({ + apiVersion: '2006-03-01', + credentials: { + accessKeyId: accessKeyId, + secretAccessKey: secretAccessKey + } +}); +var f = s3.getObject({ Bucket: Bucket, Key: Key }).createReadStream(); + +/* collect data */ +var bufs = []; +f.on('data', function(data) { bufs.push(data); }); +f.on('end', function() { + /* concatenate and parse */ + var wb = XLSX.read(Buffer.concat(bufs)); + console.log(XLSX.utils.sheet_to_csv(wb.Sheets[wb.SheetNames[0]])); +}); +``` + +
+ +### Writing Data + +`S3#upload` directly accepts a Buffer: + +
Code Sample (click to show) + +```js title="SheetJSWriteToS3.js" +var XLSX = require("xlsx"); +var AWS = require('aws-sdk'); + +/* replace these constants */ +var accessKeyId = ""; +var secretAccessKey = ""; +var Bucket = ""; +var Key = ""; + +/* Create a simple workbook and write XLSX to buffer */ +var ws = XLSX.utils.aoa_to_sheet(["SheetJS".split(""), [5,4,3,3,7,9,5]]); +var wb = XLSX.utils.book_new(); XLSX.utils.book_append_sheet(wb, ws, "Sheet1"); +var Body = XLSX.write(wb, {type: "buffer", bookType: "xlsx"}); + +/* upload buffer */ +var s3 = new AWS.S3({ + apiVersion: '2006-03-01', + credentials: { + accessKeyId: accessKeyId, + secretAccessKey: secretAccessKey + } +}); +s3.upload({ Bucket: Bucket, Key: Key, Body: Body }, function(err, data) { + if(err) throw err; + console.log("Uploaded to " + data.Location); +}); +``` + +
\ No newline at end of file diff --git a/docz/docs/03-demos/index.md b/docz/docs/03-demos/index.md index de9cf88..f55712d 100644 --- a/docz/docs/03-demos/index.md +++ b/docz/docs/03-demos/index.md @@ -51,6 +51,7 @@ The demo projects include small runnable examples and short explainers. - [`Headless Automation`](./headless) - [`Other JavaScript Engines`](./engines) - [`Azure Functions and Storage`](./azure) +- [`Amazon Web Services`](./aws) - [`Databases and Structured Data Stores`](./database) - [`NoSQL and Unstructured Data Stores`](./nosql) - [`Legacy Internet Explorer`](./legacy#internet-explorer) diff --git a/docz/docs/06-solutions/01-input.md b/docz/docs/06-solutions/01-input.md index a47ac2f..9357af9 100644 --- a/docz/docs/06-solutions/01-input.md +++ b/docz/docs/06-solutions/01-input.md @@ -321,7 +321,7 @@ const server = http.createServer((req, res) => { }).listen(process.env.PORT || 7262); ``` -The [`server` demo](https://github.com/SheetJS/SheetJS/tree/master/demos/server) has more advanced examples. +The [`server` demo](../demos/server) has more advanced examples. diff --git a/docz/static/angularjs/index.html b/docz/static/angularjs/index.html index 71ab290..af21087 100644 --- a/docz/static/angularjs/index.html +++ b/docz/static/angularjs/index.html @@ -17,7 +17,7 @@ The core library can be used as-is in AngularJS applications. The Community Edition README details some common use cases. -We also have some more public demos +We also have some more public demos This demo shows: - $http request for XLSX file and scope update with data diff --git a/docz/static/angularjs/ui-grid.html b/docz/static/angularjs/ui-grid.html index 0a4f6b7..17a466f 100644 --- a/docz/static/angularjs/ui-grid.html +++ b/docz/static/angularjs/ui-grid.html @@ -29,7 +29,7 @@ The core library can be used as-is in AngularJS applications. The Community Edition README details some common use cases. -We also have some more public demos +We also have some more public demos This demo shows: - SheetJSExportService: a service for exporting data from a ui-grid diff --git a/docz/static/aws/index.js b/docz/static/aws/index.js new file mode 100644 index 0000000..ad84b20 --- /dev/null +++ b/docz/static/aws/index.js @@ -0,0 +1,54 @@ +/* sheetjs (C) 2013-present SheetJS -- http://sheetjs.com */ +'use strict'; +var XLSX = require('xlsx'); +var Busboy = require('busboy'); + +exports.handler = function(event, context, callback) { + if(event.requestContext.http.method == "GET") { + /* make workbook */ + var wb = XLSX.read("S,h,e,e,t,J,S\n5,4,3,3,7,9,5", {type: "binary"}); + /* write to XLSX file in base64 encoding */ + var body = XLSX.write(wb, {type:"base64", bookType: "xlsx"}); + /* mark as attached file */ + var headers = { "Content-Disposition": 'attachment; filename="SheetJSLambda.xlsx"'}; + /* Send back data */ + callback(null, { + statusCode: 200, + isBase64Encoded: true, + body: body, + headers: headers + }); + return; + } + + /* set up busboy */ + var ctype = event.headers['Content-Type']||event.headers['content-type']; + var bb = Busboy({headers:{'content-type':ctype}}); + + /* busboy is evented; accumulate the fields and files manually */ + var fields = {}, files = {}; + bb.on('error', function(err) { callback(null, { body: err.message }); }); + bb.on('field', function(fieldname, val) {fields[fieldname] = val }); + bb.on('file', function(fieldname, file, filename) { + /* concatenate the individual data buffers */ + var buffers = []; + file.on('data', function(data) { buffers.push(data); }); + file.on('end', function() { files[fieldname] = [Buffer.concat(buffers), filename]; }); + }); + + /* on the finish event, all of the fields and files are ready */ + bb.on('finish', function() { + /* grab the first file */ + var f = files["upload"]; + if(!f) callback(new Error("Must submit a file for processing!")); + + /* f[0] is a buffer */ + var wb = XLSX.read(f[0]); + + /* grab first worksheet and convert to CSV */ + var ws = wb.Sheets[wb.SheetNames[0]]; + callback(null, { statusCode: 200, body: XLSX.utils.sheet_to_csv(ws) }); + }); + + bb.end(Buffer.from(event.body, "base64")); +};