From 1aa1226316a116626409f27f81049de03b075c3b Mon Sep 17 00:00:00 2001 From: TomDo1234 Date: Wed, 3 Apr 2024 11:12:07 +1100 Subject: [PATCH 01/10] Working on CF page --- docz/docs/03-demos/30-cloud/14-cloudflare.md | 686 +++++++++++++++++++ 1 file changed, 686 insertions(+) create mode 100644 docz/docs/03-demos/30-cloud/14-cloudflare.md diff --git a/docz/docs/03-demos/30-cloud/14-cloudflare.md b/docz/docs/03-demos/30-cloud/14-cloudflare.md new file mode 100644 index 0000000..a240563 --- /dev/null +++ b/docz/docs/03-demos/30-cloud/14-cloudflare.md @@ -0,0 +1,686 @@ +--- +title: Cloudflare +pagination_prev: demos/local/index +pagination_next: demos/extensions/index +--- + +import current from '/version.js'; +import CodeBlock from '@theme/CodeBlock'; + +[Cloudflare](https://www.cloudflare.com/) is a cloud services +platform which includes "Serverless +Functions" and cloud storage. + +[SheetJS](https://sheetjs.com) is a JavaScript library for reading and writing +data from spreadsheets. + +This demo explores two key Cloudflare offerings: + +- ["Cloudflare Workers"](#cloudflare-workers) explores the serverless + computing offering. The demo creates a JavaScript function that can process + user-submitted files and generate spreadsheets. + +- ["Cloudflare R2"](#cloudflare-r2) explores the cloud storage ("R2") offering. The + demo uses the NodeJS connection library to read spreadsheets from R2 and write + spreadsheets to an R2 bucket. + +:::note Tested Deployments + +This demo was last tested on 2024 April 02. + +::: + +## Cloudflare Workers + +Cloudflare offers NodeJS runtimes for running JavaScript serverless functions.[^1] + +The [SheetJS NodeJS module](/docs/getting-started/installation/nodejs) can be +required in Lambda functions. When deploying, the entire `node_modules` folder +can be added to the ZIP package. + +:::note pass + +In this demo, the "Function URL" (automatic API Gateway management) features +are used. Older deployments required special "Binary Media Types" to handle +formats like XLSX. At the time of testing, the configuration was not required. + +::: + +:::info pass + +Node.js runtime can use `x86_64` or `arm64` CPU architectures. SheetJS libraries +work on both platforms in Linux, Windows, and macOS operating systems. + +::: + +### Reading Data + +In the Lambda handler, the `event.body` attribute is a Base64-encoded string +representing the HTTP request form data. This body must be parsed. + +#### Processing Form Bodies + +The `busboy` body parser[^2] is battle-tested in NodeJS deployments. + +`busboy` fires a `'file'` event for every file in the form body. The callback +receives a NodeJS stream that should be collected into a Buffer: + +```js +/* accumulate the files manually */ +var files = {}; +bb.on('file', function(fieldname, file, filename) { + /* concatenate the individual data buffers */ + var buffers = []; + file.on('data', function(data) { buffers.push(data); }); + file.on('end', function() { files[fieldname] = Buffer.concat(buffers); }); +}); +``` + +`busboy` fires a `'finish'` event when the body parsing is finished. Callbacks +can assume every file in the form body has been stored in NodeJS Buffer objects. + +#### Processing NodeJS Buffers + +The SheetJS `read` method[^3] can read the Buffer objects and generate SheetJS +workbook objects[^4] which can be processed with other API functions. + +For example, a handler can use `sheet_to_csv`[^5] to generate CSV text: + +```js +/* on the finish event, all of the fields and files are ready */ +bb.on('finish', function() { + /* grab the first file */ + var f = files["upload"]; + if(!f) callback(new Error("Must submit a file for processing!")); + + /* f[0] is a buffer */ + // highlight-next-line + var wb = XLSX.read(f[0]); + + /* grab first worksheet and convert to CSV */ + var ws = wb.Sheets[wb.SheetNames[0]]; + callback(null, { statusCode: 200, body: XLSX.utils.sheet_to_csv(ws) }); +}); +``` + +
Complete Code Sample (click to show) + +This example takes the first uploaded file submitted with the key `upload`, +parses the file and returns the CSV content of the first worksheet. + +```js +const XLSX = require('xlsx'); +var Busboy = require('busboy'); + +exports.handler = function(event, context, callback) { + /* set up busboy */ + var ctype = event.headers['Content-Type']||event.headers['content-type']; + var bb = Busboy({headers:{'content-type':ctype}}); + + /* busboy is evented; accumulate the fields and files manually */ + var fields = {}, files = {}; + bb.on('error', function(err) { callback(null, { body: err.message }); }); + bb.on('field', function(fieldname, val) {fields[fieldname] = val }); + // highlight-start + bb.on('file', function(fieldname, file, filename) { + /* concatenate the individual data buffers */ + var buffers = []; + file.on('data', function(data) { buffers.push(data); }); + file.on('end', function() { files[fieldname] = [Buffer.concat(buffers), filename]; }); + }); + // highlight-end + + /* on the finish event, all of the fields and files are ready */ + bb.on('finish', function() { + /* grab the first file */ + var f = files["upload"]; + if(!f) callback(new Error("Must submit a file for processing!")); + + /* f[0] is a buffer */ + // highlight-next-line + var wb = XLSX.read(f[0]); + + /* grab first worksheet and convert to CSV */ + var ws = wb.Sheets[wb.SheetNames[0]]; + callback(null, { statusCode: 200, body: XLSX.utils.sheet_to_csv(ws) }); + }); + + /* start the processing */ + // highlight-next-line + bb.end(Buffer.from(event.body, "base64")); +}; +``` + +
+ +### Writing Data + +For safely transmitting binary data, Base64 strings should be used. + +The SheetJS `write` method[^6] with the option `type: "base64"` will generate +Base64-encoded strings. + +```js +/* sample SheetJS workbook object */ +var wb = XLSX.read("S,h,e,e,t,J,S\n5,4,3,3,7,9,5", {type: "binary"}); +/* write to XLSX file in Base64 encoding */ +var b64 = XLSX.write(wb, { type: "base64", bookType: "xlsx" }); +``` + +The Lambda callback response function accepts options. Setting `isBase64Encoded` +to `true` will ensure the callback handler decodes the data. To ensure browsers +will try to download the response, the `Content-Disposition` header must be set: + +```js +callback(null, { + statusCode: 200, + /* Base64-encoded file */ + isBase64Encoded: true, + body: b64, + headers: { + /* Browsers will treat the response as the file SheetJSLambda.xlsx */ + "Content-Disposition": 'attachment; filename="SheetJSLambda.xlsx"' + } +}); +``` + +
Complete Code Sample (click to show) + +This example creates a sample workbook object and sends the file in the response: + +```js +var XLSX = require('xlsx'); + +exports.handler = function(event, context, callback) { + /* make workbook */ + var wb = XLSX.read("S,h,e,e,t,J,S\n5,4,3,3,7,9,5", {type: "binary"}); + /* write to XLSX file in Base64 encoding */ + // highlight-next-line + var body = XLSX.write(wb, { type: "base64", bookType: "xlsx" }); + /* mark as attached file */ + var headers = { "Content-Disposition": 'attachment; filename="SheetJSLambda.xlsx"'}; + /* Send back data */ + callback(null, { + statusCode: 200, + // highlight-next-line + isBase64Encoded: true, + body: body, + headers: headers + }); +}; +``` + +
+ +### Lambda Demo + +:::note pass + +At the time of writing, the Cloudflare Free Tier included an allowance of 1 million +free requests per month and 400 thousand GB-seconds of compute resources. + +::: + +0) If you do not have an account, create a new Cloudflare free tier account[^7]. + +#### Create Project ZIP + +1) Create a new folder and download [`index.js`](pathname:///Cloudflare/index.js): + +```bash +mkdir -p SheetJSLambda +cd SheetJSLambda +curl -LO https://docs.sheetjs.com/Cloudflare/index.js +``` + +2) Install dependencies in the project directory; + +{`\ +mkdir -p node_modules +npm i https://cdn.sheetjs.com/xlsx-${current}/xlsx-${current}.tgz busboy`} + + +3) Create a .zip package of the contents of the folder: + +```bash +yes | zip -c ../SheetJSLambda.zip -r . +``` + +#### Lambda Setup + +4) Sign into the [Cloudflare Management Console](https://Cloudflare.amazon.com/console/) with +a root user account. + +5) Type "Lambda" in the top search box and click Lambda (under Services). + +6) Open "Functions" in the left sidebar. + +If the left sidebar is not open, click the `≡` icon in the left edge of the page. + +7) Click the "Create function" button in the main panel. + +8) Select the following options: + +- In the top list, select "Author from scratch" (default choice) + +- Type a memorable "Function Name" ("SheetJSLambda" when last tested) + +- In the "Runtime" dropdown, look for the "Latest supported" section and select + "Node.js" ("Node.js 18.x" when last tested) + +- Expand "Advanced Settings" and check "Enable function URL". This will display + a few sub-options: + + "Auth type" select "NONE" (disable IAM authentication) + + Check "Configure cross-origin resource sharing (CORS)" + +9) Click "Create function" to create the function. + +#### Upload Code + +10) In the Interface, scroll down and select the "Code" tab. + +11) Click the "Upload from" dropdown and select ".zip file". + +12) Click the "Upload" button in the modal. With the file picker, select the +`SheetJSLambda.zip` file created in step 3. Click "Save". + +:::note pass +When the demo was last tested, the ZIP was small enough that the Lambda code +editor will load the package. + +::: + +13) In the code editor, double-click `index.js` and confirm the code editor +displays JavaScript code. + +#### External Access + +14) Click "Configuration" in the tab list. + +15) In the sidebar below the tab list, select "Function URL" and click "Edit". + +16) Set the "Auth type" to "NONE" and click Save. The page will redirect to the +Function properties. + +17) Select the "Configuration" tab and select "Permissions" in the left sidebar. + +18) Scroll down to "Resource-based policy statements" and ensure that +`FunctionURLAllowPublicAccess` is listed. + +If no policy statements are defined, select "Add Permission" with the options: + +- Select "Function URL" at the top +- Auth type: NONE +- Ensure that Statement ID is set to `FunctionURLAllowPublicAccess` +- Ensure that Principal is set to `*` +- Ensure that Action is set to `lambda:InvokeFunctionUrl` + +Click "Save" and a new Policy statement should be created. + +#### Lambda Testing + +19) Find the Function URL (It is in the "Function Overview" section). + +20) Try to access the function URL in a web browser. + +The site will attempt to download `SheetJSLambda.xlsx`. Save and open the file +to confirm it is valid. + +21) Download and make a POST request to the +public function URL. + +This can be tested on the command line. Change `FUNCTION_URL` in the commands: + +```bash +curl -LO https://sheetjs.com/pres.numbers +curl -X POST -F "upload=@pres.numbers" FUNCTION_URL +``` + +The terminal will display CSV output of the first sheet. + +## Cloudflare R2 + +The main NodeJS module for S3 and all Cloudflare services is `Cloudflare-sdk`[^8]. + +The [SheetJS NodeJS module](/docs/getting-started/installation/nodejs) can be +required in NodeJS scripts. + +### Connecting to R2 + +The `Cloudflare-sdk` module exports a function `S3` that performs the connection. The +function expects an options object that includes an API version and credentials. +Access keys for an IAM user[^9] must be used: + +```js +/* credentials */ +var accessKeyId = "...", secretAccessKey = "...""; + +/* file location */ +var Bucket = "...", Key = "pres.numbers"; + +/* connect to s3 account */ +var Cloudflare = require('Cloudflare-sdk'); +var s3 = new Cloudflare.S3({ + apiVersion: '2006-03-01', + credentials: { accessKeyId, secretAccessKey } +}); +``` + +### Downloading Data + +#### Fetching Files from S3 + +The `s3#getObject` method returns an object with a `createReadStream` method. +`createReadStream` returns a NodeJS stream: + +```js +/* open stream to the file */ +var stream = s3.getObject({ Bucket: Bucket, Key: Key }).createReadStream(); +``` + +#### Concatenating NodeJS Streams + +Buffers can be concatenated from the stream into one unified Buffer object: + +```js +/* array of buffers */ +var bufs = []; +/* add each data chunk to the array */ +stream.on('data', function(data) { bufs.push(data); }); +/* the callback will be called after all of the data is collected */ +stream.on('end', function() { + /* concatenate */ + var buf = Buffer.concat(bufs); + + /* AT THIS POINT, `buf` is a NodeJS Buffer */ +}); +``` + +#### Parsing NodeJS Buffers + +The SheetJS `read` method[^10] can read the final object and generate SheetJS +workbook objects[^11] which can be processed with other API functions. + +For example, a callback can use `sheet_to_csv`[^12] to generate CSV text: + +```js +stream.on('end', function() { + /* concatenate */ + var buf = Buffer.concat(bufs); + + /* parse */ + var wb = XLSX.read(Buffer.concat(bufs)); + + /* generate CSV from first worksheet */ + var first_ws = wb.Sheets[wb.SheetNames[0]]; + var csv = XLSX.utils.sheet_to_csv(first_ws); + console.log(csv); +}); +``` + +### Uploading Data + +The SheetJS `write` method[^13] with the option `type: "buffer"` will generate +NodeJS Buffers. `S3#upload` directly accepts these Buffer objects. + +This example creates a sample workbook object, generates XLSX file data in a +NodeJS Buffer, and uploads the data to S3: + +```js +/* generate sample workbook */ +var wb = XLSX.read("S,h,e,e,t,J,S\n5,4,3,3,7,9,5", {type: "binary"}); + +/* write to XLSX file in a NodeJS Buffer */ +var Body = XLSX.write(wb, {type: "buffer", bookType: "xlsx"}); + +/* upload buffer */ +s3.upload({ Bucket, Key, Body }, function(err, data) { + if(err) throw err; + console.log("Uploaded to " + data.Location); +}); +``` + +### S3 Demo + +:::note pass + +At the time of writing, the Cloudflare Free Tier included 5GB of Cloudflare R2 with 20,000 +Get requests and 2000 Put requests per month. + +::: + +This sample fetches a buffer from S3 and parses the workbook. + +0) If you do not have an account, create a new Cloudflare free tier account[^14]. + +#### Create S3 Bucket + +1) Sign into the [Cloudflare Management Console](https://Cloudflare.amazon.com/console/) with +a root user account. + +2) Type "S3" in the top search box and click S3 (under Services). + +3) Open "Buckets" in the left sidebar. + +If the left sidebar is not open, click the `≡` icon in the left edge of the page. + +4) Click the "Create bucket" button in the main panel. + +5) Select the following options: + +- Type a memorable "Bucket Name" ("sheetjsbouquet" when last tested) + +- In the "Object Ownership" section, select "ACLs disabled" + +- Check "Block *all* public access" + +- Look for the "Bucket Versioning" section and select "Disable" + +6) Click "Create bucket" to create the bucket. + +#### Create IAM User + +7) Type "IAM" in the top search box and click IAM (under Services). + +8) Open "Users" in the left sidebar. + +If the left sidebar is not open, click the `≡` icon in the left edge of the page. + +9) Click the "Create user" button in the main panel. + +10) In step 1, type a memorable "Bucket Name" ("sheetjs-user" when last tested). +Click "Next". + +11) In step 2, click "Next" + +12) In step 3, click "Create user" to create the user. + +#### Add Permissions + +13) Click the new user name in the Users table. + +14) Select the "Permissions" tab + +15) Click the "Add permissions" dropdown and select "Add permissions". + +16) Select "Attach policies directly". + +17) In the "Permissions policies" section, search for "AmazonS3FullAccess". +There should be one entry. + +18) Check the checkbox next to "AmazonS3FullAccess" and click the "Next" button. + +19) In the "Review" screen, click "Add permissions" + +#### Generate Keys + +20) Click "Security credentials", then click "Create access key". + +21) Select the "Local code" option. Check "I understand the above recommendation +and want to proceed to create an access key." and click "Next" + +22) Click "Create Access Key" and click "Download .csv file" in the next screen. + +In the generated CSV: + +- Cell A2 is the "Access key ID" (`accessKeyId` in the Cloudflare API) +- Cell B2 is the "Secret access key" (`secretAccessKey` in the Cloudflare API) + +#### Set up Project + +23) Create a new NodeJS project: + +```bash +mkdir SheetJSS3 +cd SheetJSS3 +npm init -y +``` + +24) Install dependencies: + +{`\ +mkdir -p node_modules +npm i --save https://cdn.sheetjs.com/xlsx-${current}/xlsx-${current}.tgz Cloudflare-sdk@2.1467.0`} + + +#### Write Test + +:::note pass + +This sample creates a simple workbook, generates a NodeJS buffer, and uploads +the buffer to S3. + +``` + | A | B | C | D | E | F | G | +---+---|---|---|---|---|---|---| + 1 | S | h | e | e | t | J | S | + 2 | 5 | 4 | 3 | 3 | 7 | 9 | 5 | +``` + +::: + +25) Save the following script to `SheetJSWriteToS3.js`: + +```js title="SheetJSWriteToS3.js" +var XLSX = require("xlsx"); +var Cloudflare = require('Cloudflare-sdk'); + +/* replace these constants */ +// highlight-start +var accessKeyId = ""; +var secretAccessKey = ""; +var Bucket = ""; +// highlight-end + +var Key = "test.xlsx"; + +/* Create a simple workbook and write XLSX to buffer */ +var ws = XLSX.utils.aoa_to_sheet(["SheetJS".split(""), [5,4,3,3,7,9,5]]); +var wb = XLSX.utils.book_new(); XLSX.utils.book_append_sheet(wb, ws, "Sheet1"); +var Body = XLSX.write(wb, {type: "buffer", bookType: "xlsx"}); + +/* upload buffer */ +var s3 = new Cloudflare.S3({ + apiVersion: '2006-03-01', + credentials: { + accessKeyId: accessKeyId, + secretAccessKey: secretAccessKey + } +}); +s3.upload({ Bucket: Bucket, Key: Key, Body: Body }, function(err, data) { + if(err) throw err; + console.log("Uploaded to " + data.Location); +}); +``` + +26) Edit `SheetJSWriteToS3.js` and replace the highlighted lines: + +- `accessKeyId`: access key for the Cloudflare account +- `secretAccessKey`: secret access key for the Cloudflare account +- `Bucket`: name of the bucket + +The keys are found in the CSV from step 22. The Bucket is the name from step 5. + +27) Run the script: + +```bash +node SheetJSWriteToS3.js +``` + +This file will be stored with the object name `test.xlsx`. It can be manually +downloaded from the S3 web interface. + +#### Read Test + +This sample will download and process the test file from "Write Test". + +28) Save the following script to `SheetJSReadFromS3.js`: + +```js title="SheetJSReadFromS3.js" +var XLSX = require("xlsx"); +var Cloudflare = require('Cloudflare-sdk'); + +/* replace these constants */ +// highlight-start +var accessKeyId = ""; +var secretAccessKey = ""; +var Bucket = ""; +// highlight-end + +var Key = "test.xlsx"; + +/* Get stream */ +var s3 = new Cloudflare.S3({ + apiVersion: '2006-03-01', + credentials: { + accessKeyId: accessKeyId, + secretAccessKey: secretAccessKey + } +}); +var f = s3.getObject({ Bucket: Bucket, Key: Key }).createReadStream(); + +/* collect data */ +var bufs = []; +f.on('data', function(data) { bufs.push(data); }); +f.on('end', function() { + /* concatenate and parse */ + var wb = XLSX.read(Buffer.concat(bufs)); + console.log(XLSX.utils.sheet_to_csv(wb.Sheets[wb.SheetNames[0]])); +}); +``` + +29) Edit `SheetJSReadFromS3.js` and replace the highlighted lines: + +- `accessKeyId`: access key for the Cloudflare account +- `secretAccessKey`: secret access key for the Cloudflare account +- `Bucket`: name of the bucket + +The keys are found in the CSV from Step 22. The Bucket is the name from Step 5. + +30) Run the script: + +```bash +node SheetJSReadFromS3.js +``` + +The program will display the data in CSV format. + +``` +S,h,e,e,t,J,S +5,4,3,3,7,9,5 +``` + +[^1]: See ["Node.js compatibility"](https://developers.cloudflare.com/workers/runtime-apis/nodejs/) in the Cloudflare documentation +[^2]: The `busboy` module is distributed [on the public NPM registry](https://npm.im/busboy) +[^3]: See [`read` in "Reading Files"](/docs/api/parse-options) +[^4]: See ["Workbook Object" in "SheetJS Data Model"](/docs/csf/book) for more details. +[^5]: See [`sheet_to_csv` in "CSV and Text"](/docs/api/utilities/csv#delimiter-separated-output) +[^6]: See [`write` in "Writing Files"](/docs/api/write-options) +[^7]: Registering for a free account [on the Cloudflare Free Tier](https://Cloudflare.amazon.com/free/) requires a valid phone number and a valid credit card. +[^8]: The `Cloudflare-sdk` module is distributed [on the public NPM registry](https://npm.im/Cloudflare-sdk) +[^9]: See ["Managing access keys for IAM users"](https://docs.Cloudflare.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html) in the Cloudflare documentation +[^10]: See [`read` in "Reading Files"](/docs/api/parse-options) +[^11]: See ["Workbook Object" in "SheetJS Data Model"](/docs/csf/book) for more details. +[^12]: See [`sheet_to_csv` in "CSV and Text"](/docs/api/utilities/csv#delimiter-separated-output) +[^13]: See [`write` in "Writing Files"](/docs/api/write-options) +[^14]: Registering for a free account [on the Cloudflare Free Tier](https://Cloudflare.amazon.com/free/) requires a valid phone number and a valid credit card. \ No newline at end of file -- 2.34.1 From 65ec4fb2e6f39bfa25dffd2c2838168dd421bb59 Mon Sep 17 00:00:00 2001 From: TomDo1234 Date: Wed, 3 Apr 2024 11:16:16 +1100 Subject: [PATCH 02/10] Working on CF page 2 --- docz/docs/03-demos/30-cloud/14-cloudflare.md | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/docz/docs/03-demos/30-cloud/14-cloudflare.md b/docz/docs/03-demos/30-cloud/14-cloudflare.md index a240563..b035b58 100644 --- a/docz/docs/03-demos/30-cloud/14-cloudflare.md +++ b/docz/docs/03-demos/30-cloud/14-cloudflare.md @@ -35,21 +35,13 @@ This demo was last tested on 2024 April 02. Cloudflare offers NodeJS runtimes for running JavaScript serverless functions.[^1] The [SheetJS NodeJS module](/docs/getting-started/installation/nodejs) can be -required in Lambda functions. When deploying, the entire `node_modules` folder -can be added to the ZIP package. +required in Lambda functions. Cloudflare provides a convenient cli called Wrangler [^2] +which can streamline development and deployment :::note pass -In this demo, the "Function URL" (automatic API Gateway management) features -are used. Older deployments required special "Binary Media Types" to handle -formats like XLSX. At the time of testing, the configuration was not required. - -::: - -:::info pass - -Node.js runtime can use `x86_64` or `arm64` CPU architectures. SheetJS libraries -work on both platforms in Linux, Windows, and macOS operating systems. +In this demo, Wrangler is used as that is Cloudflare's recommended way of developing and deploying to +Workers ::: @@ -671,7 +663,7 @@ S,h,e,e,t,J,S ``` [^1]: See ["Node.js compatibility"](https://developers.cloudflare.com/workers/runtime-apis/nodejs/) in the Cloudflare documentation -[^2]: The `busboy` module is distributed [on the public NPM registry](https://npm.im/busboy) +[^2]: [Wrangler documentation](https://developers.cloudflare.com/workers/wrangler/) [^3]: See [`read` in "Reading Files"](/docs/api/parse-options) [^4]: See ["Workbook Object" in "SheetJS Data Model"](/docs/csf/book) for more details. [^5]: See [`sheet_to_csv` in "CSV and Text"](/docs/api/utilities/csv#delimiter-separated-output) -- 2.34.1 From 5a8d89bfbafd6fd45d28737b2508f7bca7715164 Mon Sep 17 00:00:00 2001 From: TomDo1234 Date: Wed, 3 Apr 2024 13:05:56 +1100 Subject: [PATCH 03/10] mention c3 --- docz/docs/03-demos/30-cloud/14-cloudflare.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docz/docs/03-demos/30-cloud/14-cloudflare.md b/docz/docs/03-demos/30-cloud/14-cloudflare.md index b035b58..77df9b1 100644 --- a/docz/docs/03-demos/30-cloud/14-cloudflare.md +++ b/docz/docs/03-demos/30-cloud/14-cloudflare.md @@ -35,8 +35,8 @@ This demo was last tested on 2024 April 02. Cloudflare offers NodeJS runtimes for running JavaScript serverless functions.[^1] The [SheetJS NodeJS module](/docs/getting-started/installation/nodejs) can be -required in Lambda functions. Cloudflare provides a convenient cli called Wrangler [^2] -which can streamline development and deployment +required in Cloudflare Workers. Cloudflare provides a cli called C3 [^2] in order to setup your worker project and another cli called Wrangler [^3] +which can streamline development and deployment, :::note pass @@ -663,8 +663,8 @@ S,h,e,e,t,J,S ``` [^1]: See ["Node.js compatibility"](https://developers.cloudflare.com/workers/runtime-apis/nodejs/) in the Cloudflare documentation -[^2]: [Wrangler documentation](https://developers.cloudflare.com/workers/wrangler/) -[^3]: See [`read` in "Reading Files"](/docs/api/parse-options) +[^2]: See ["Get started guide"](https://developers.cloudflare.com/workers/get-started/guide/#1-create-a-new-worker-project) +[^3]: [Wrangler documentation](https://developers.cloudflare.com/workers/wrangler/) [^4]: See ["Workbook Object" in "SheetJS Data Model"](/docs/csf/book) for more details. [^5]: See [`sheet_to_csv` in "CSV and Text"](/docs/api/utilities/csv#delimiter-separated-output) [^6]: See [`write` in "Writing Files"](/docs/api/write-options) -- 2.34.1 From 615365588885dbf1740129435734be95c35e3bb5 Mon Sep 17 00:00:00 2001 From: TomDo1234 Date: Wed, 3 Apr 2024 15:42:25 +1100 Subject: [PATCH 04/10] Done with the basics --- docz/docs/03-demos/30-cloud/14-cloudflare.md | 212 +++++-------------- 1 file changed, 56 insertions(+), 156 deletions(-) diff --git a/docz/docs/03-demos/30-cloud/14-cloudflare.md b/docz/docs/03-demos/30-cloud/14-cloudflare.md index 77df9b1..e15003a 100644 --- a/docz/docs/03-demos/30-cloud/14-cloudflare.md +++ b/docz/docs/03-demos/30-cloud/14-cloudflare.md @@ -45,175 +45,75 @@ Workers ::: -### Reading Data +### Overview -In the Lambda handler, the `event.body` attribute is a Base64-encoded string -representing the HTTP request form data. This body must be parsed. +Cloudflare Workers exposes a handler in src/index.(js|ts) when you create a worker project using C3 as per above, +this handler exposes a Request object which can be used to expose the formdata or json of the request to the worker -#### Processing Form Bodies - -The `busboy` body parser[^2] is battle-tested in NodeJS deployments. - -`busboy` fires a `'file'` event for every file in the form body. The callback -receives a NodeJS stream that should be collected into a Buffer: - -```js -/* accumulate the files manually */ -var files = {}; -bb.on('file', function(fieldname, file, filename) { - /* concatenate the individual data buffers */ - var buffers = []; - file.on('data', function(data) { buffers.push(data); }); - file.on('end', function() { files[fieldname] = Buffer.concat(buffers); }); -}); -``` - -`busboy` fires a `'finish'` event when the body parsing is finished. Callbacks -can assume every file in the form body has been stored in NodeJS Buffer objects. - -#### Processing NodeJS Buffers - -The SheetJS `read` method[^3] can read the Buffer objects and generate SheetJS -workbook objects[^4] which can be processed with other API functions. - -For example, a handler can use `sheet_to_csv`[^5] to generate CSV text: - -```js -/* on the finish event, all of the fields and files are ready */ -bb.on('finish', function() { - /* grab the first file */ - var f = files["upload"]; - if(!f) callback(new Error("Must submit a file for processing!")); - - /* f[0] is a buffer */ - // highlight-next-line - var wb = XLSX.read(f[0]); - - /* grab first worksheet and convert to CSV */ - var ws = wb.Sheets[wb.SheetNames[0]]; - callback(null, { statusCode: 200, body: XLSX.utils.sheet_to_csv(ws) }); -}); -``` - -
Complete Code Sample (click to show) - -This example takes the first uploaded file submitted with the key `upload`, -parses the file and returns the CSV content of the first worksheet. - -```js -const XLSX = require('xlsx'); -var Busboy = require('busboy'); - -exports.handler = function(event, context, callback) { - /* set up busboy */ - var ctype = event.headers['Content-Type']||event.headers['content-type']; - var bb = Busboy({headers:{'content-type':ctype}}); - - /* busboy is evented; accumulate the fields and files manually */ - var fields = {}, files = {}; - bb.on('error', function(err) { callback(null, { body: err.message }); }); - bb.on('field', function(fieldname, val) {fields[fieldname] = val }); - // highlight-start - bb.on('file', function(fieldname, file, filename) { - /* concatenate the individual data buffers */ - var buffers = []; - file.on('data', function(data) { buffers.push(data); }); - file.on('end', function() { files[fieldname] = [Buffer.concat(buffers), filename]; }); - }); - // highlight-end - - /* on the finish event, all of the fields and files are ready */ - bb.on('finish', function() { - /* grab the first file */ - var f = files["upload"]; - if(!f) callback(new Error("Must submit a file for processing!")); - - /* f[0] is a buffer */ - // highlight-next-line - var wb = XLSX.read(f[0]); - - /* grab first worksheet and convert to CSV */ - var ws = wb.Sheets[wb.SheetNames[0]]; - callback(null, { statusCode: 200, body: XLSX.utils.sheet_to_csv(ws) }); - }); - - /* start the processing */ - // highlight-next-line - bb.end(Buffer.from(event.body, "base64")); +```ts +export default { + async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { + const data = await request.formData() + // const data = await request.json() if you are receiving a json request + //... + }, }; ``` -
+### Reading Data + +The SheetJS `read` method[^4] can read ArrayBuffer objects and generate SheetJS +workbook objects[^5] which can be processed with other API functions. + +For example, a handler can use `sheet_to_csv`[^6] to generate CSV text: + +```ts +import XLSX from "xlsx"; +export default { + async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { + if (request.method === 'POST') { + const data = await request.formData() + // const data = await request.json() if you are doing a json request + const file = data.get('file') as File; + const buffer = await file.arrayBuffer(); + const wb = XLSX.read(buffer); + const ws = wb.Sheets[wb.SheetNames[0]]; + var b64 = XLSX.write(wb, { type: "base64", bookType: "xlsx" }); + return new Response(XLSX.utils.sheet_to_csv(ws)) + } + return new Response('Hello World!'); + }, +}; +``` ### Writing Data -For safely transmitting binary data, Base64 strings should be used. - -The SheetJS `write` method[^6] with the option `type: "base64"` will generate -Base64-encoded strings. - -```js -/* sample SheetJS workbook object */ -var wb = XLSX.read("S,h,e,e,t,J,S\n5,4,3,3,7,9,5", {type: "binary"}); -/* write to XLSX file in Base64 encoding */ -var b64 = XLSX.write(wb, { type: "base64", bookType: "xlsx" }); -``` - -The Lambda callback response function accepts options. Setting `isBase64Encoded` -to `true` will ensure the callback handler decodes the data. To ensure browsers -will try to download the response, the `Content-Disposition` header must be set: - -```js -callback(null, { - statusCode: 200, - /* Base64-encoded file */ - isBase64Encoded: true, - body: b64, - headers: { - /* Browsers will treat the response as the file SheetJSLambda.xlsx */ - "Content-Disposition": 'attachment; filename="SheetJSLambda.xlsx"' - } -}); -``` - -
Complete Code Sample (click to show) - -This example creates a sample workbook object and sends the file in the response: - -```js -var XLSX = require('xlsx'); - -exports.handler = function(event, context, callback) { - /* make workbook */ - var wb = XLSX.read("S,h,e,e,t,J,S\n5,4,3,3,7,9,5", {type: "binary"}); - /* write to XLSX file in Base64 encoding */ - // highlight-next-line - var body = XLSX.write(wb, { type: "base64", bookType: "xlsx" }); - /* mark as attached file */ - var headers = { "Content-Disposition": 'attachment; filename="SheetJSLambda.xlsx"'}; - /* Send back data */ - callback(null, { - statusCode: 200, - // highlight-next-line - isBase64Encoded: true, - body: body, - headers: headers - }); +```ts +import XLSX from "xlsx"; +export default { + async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { + var wb = XLSX.read("S,h,e,e,t,J,S\n5,4,3,3,7,9,5", {type: "binary"}); + var data = XLSX.write(wb, { type: "buffer", bookType: "xlsx" }); + return new Response(data,{ + status: 200, + headers: { + "Content-Disposition": 'attachment; filename="SheetJSCFWorker.xlsx"' //Content disposition will make the browser download the file + } + }) + }, }; ``` -
- -### Lambda Demo +### Cloudflare Workers Demo :::note pass -At the time of writing, the Cloudflare Free Tier included an allowance of 1 million -free requests per month and 400 thousand GB-seconds of compute resources. +At the time of writing, the Cloudflare Workers Free Tier included an allowance of 100,000 +free requests per day and 10 milliseconds of CPU time per invocation. ::: -0) If you do not have an account, create a new Cloudflare free tier account[^7]. +1) If you do not have an account, create a new account[^7]. #### Create Project ZIP @@ -665,10 +565,10 @@ S,h,e,e,t,J,S [^1]: See ["Node.js compatibility"](https://developers.cloudflare.com/workers/runtime-apis/nodejs/) in the Cloudflare documentation [^2]: See ["Get started guide"](https://developers.cloudflare.com/workers/get-started/guide/#1-create-a-new-worker-project) [^3]: [Wrangler documentation](https://developers.cloudflare.com/workers/wrangler/) -[^4]: See ["Workbook Object" in "SheetJS Data Model"](/docs/csf/book) for more details. -[^5]: See [`sheet_to_csv` in "CSV and Text"](/docs/api/utilities/csv#delimiter-separated-output) -[^6]: See [`write` in "Writing Files"](/docs/api/write-options) -[^7]: Registering for a free account [on the Cloudflare Free Tier](https://Cloudflare.amazon.com/free/) requires a valid phone number and a valid credit card. +[^4]: See [`read` in "Reading Files"](/docs/api/parse-options) +[^5]: See ["Workbook Object" in "SheetJS Data Model"](/docs/csf/book) for more details. +[^6]: See [`sheet_to_csv` in "CSV and Text"](/docs/api/utilities/csv#delimiter-separated-output) +[^7]: Registering for a free account [on the Cloudflare Free Tier](https://dash.cloudflare.com/sign-up). [^8]: The `Cloudflare-sdk` module is distributed [on the public NPM registry](https://npm.im/Cloudflare-sdk) [^9]: See ["Managing access keys for IAM users"](https://docs.Cloudflare.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html) in the Cloudflare documentation [^10]: See [`read` in "Reading Files"](/docs/api/parse-options) -- 2.34.1 From 0821d57f5a1502c6ff22ce74543c738df7360687 Mon Sep 17 00:00:00 2001 From: TomDo1234 Date: Wed, 3 Apr 2024 15:56:14 +1100 Subject: [PATCH 05/10] Finished with Workers, now on to R2 --- docz/docs/03-demos/30-cloud/14-cloudflare.md | 115 +------------------ 1 file changed, 2 insertions(+), 113 deletions(-) diff --git a/docz/docs/03-demos/30-cloud/14-cloudflare.md b/docz/docs/03-demos/30-cloud/14-cloudflare.md index e15003a..0fed0cc 100644 --- a/docz/docs/03-demos/30-cloud/14-cloudflare.md +++ b/docz/docs/03-demos/30-cloud/14-cloudflare.md @@ -115,120 +115,9 @@ free requests per day and 10 milliseconds of CPU time per invocation. 1) If you do not have an account, create a new account[^7]. -#### Create Project ZIP +### Setting up the Project and Testing -1) Create a new folder and download [`index.js`](pathname:///Cloudflare/index.js): - -```bash -mkdir -p SheetJSLambda -cd SheetJSLambda -curl -LO https://docs.sheetjs.com/Cloudflare/index.js -``` - -2) Install dependencies in the project directory; - -{`\ -mkdir -p node_modules -npm i https://cdn.sheetjs.com/xlsx-${current}/xlsx-${current}.tgz busboy`} - - -3) Create a .zip package of the contents of the folder: - -```bash -yes | zip -c ../SheetJSLambda.zip -r . -``` - -#### Lambda Setup - -4) Sign into the [Cloudflare Management Console](https://Cloudflare.amazon.com/console/) with -a root user account. - -5) Type "Lambda" in the top search box and click Lambda (under Services). - -6) Open "Functions" in the left sidebar. - -If the left sidebar is not open, click the `≡` icon in the left edge of the page. - -7) Click the "Create function" button in the main panel. - -8) Select the following options: - -- In the top list, select "Author from scratch" (default choice) - -- Type a memorable "Function Name" ("SheetJSLambda" when last tested) - -- In the "Runtime" dropdown, look for the "Latest supported" section and select - "Node.js" ("Node.js 18.x" when last tested) - -- Expand "Advanced Settings" and check "Enable function URL". This will display - a few sub-options: - + "Auth type" select "NONE" (disable IAM authentication) - + Check "Configure cross-origin resource sharing (CORS)" - -9) Click "Create function" to create the function. - -#### Upload Code - -10) In the Interface, scroll down and select the "Code" tab. - -11) Click the "Upload from" dropdown and select ".zip file". - -12) Click the "Upload" button in the modal. With the file picker, select the -`SheetJSLambda.zip` file created in step 3. Click "Save". - -:::note pass -When the demo was last tested, the ZIP was small enough that the Lambda code -editor will load the package. - -::: - -13) In the code editor, double-click `index.js` and confirm the code editor -displays JavaScript code. - -#### External Access - -14) Click "Configuration" in the tab list. - -15) In the sidebar below the tab list, select "Function URL" and click "Edit". - -16) Set the "Auth type" to "NONE" and click Save. The page will redirect to the -Function properties. - -17) Select the "Configuration" tab and select "Permissions" in the left sidebar. - -18) Scroll down to "Resource-based policy statements" and ensure that -`FunctionURLAllowPublicAccess` is listed. - -If no policy statements are defined, select "Add Permission" with the options: - -- Select "Function URL" at the top -- Auth type: NONE -- Ensure that Statement ID is set to `FunctionURLAllowPublicAccess` -- Ensure that Principal is set to `*` -- Ensure that Action is set to `lambda:InvokeFunctionUrl` - -Click "Save" and a new Policy statement should be created. - -#### Lambda Testing - -19) Find the Function URL (It is in the "Function Overview" section). - -20) Try to access the function URL in a web browser. - -The site will attempt to download `SheetJSLambda.xlsx`. Save and open the file -to confirm it is valid. - -21) Download and make a POST request to the -public function URL. - -This can be tested on the command line. Change `FUNCTION_URL` in the commands: - -```bash -curl -LO https://sheetjs.com/pres.numbers -curl -X POST -F "upload=@pres.numbers" FUNCTION_URL -``` - -The terminal will display CSV output of the first sheet. +[The Cloudflare Docs provide a simple step by step guide to setup, develop, test, build and deploy your project](https://developers.cloudflare.com/workers/get-started/guide/) ## Cloudflare R2 -- 2.34.1 From 065a7d8cac38d04432ab595ce1b0f2c45515e03e Mon Sep 17 00:00:00 2001 From: TomDo1234 Date: Wed, 3 Apr 2024 16:57:36 +1100 Subject: [PATCH 06/10] Working on it --- docz/docs/03-demos/30-cloud/14-cloudflare.md | 363 ++++--------------- 1 file changed, 67 insertions(+), 296 deletions(-) diff --git a/docz/docs/03-demos/30-cloud/14-cloudflare.md b/docz/docs/03-demos/30-cloud/14-cloudflare.md index 0fed0cc..8f2a3e9 100644 --- a/docz/docs/03-demos/30-cloud/14-cloudflare.md +++ b/docz/docs/03-demos/30-cloud/14-cloudflare.md @@ -121,336 +121,111 @@ free requests per day and 10 milliseconds of CPU time per invocation. ## Cloudflare R2 -The main NodeJS module for S3 and all Cloudflare services is `Cloudflare-sdk`[^8]. +Due to R2's S3 compatability, the main NodeJS module for Cloudflare R2 is actually the AWS S3 SDK `@aws-sdk/client-s3`[^8]. The [SheetJS NodeJS module](/docs/getting-started/installation/nodejs) can be required in NodeJS scripts. +:::note pass + +As the time of writing, Cloudflare R2 is not 100% compatible with the S3 API, please refer to the [S3 API Compatibility section of the official Cloudflare R2 Docs](https://developers.cloudflare.com/r2/api/s3/api/) to know more about what S3 Features are and are not supported. + +::: + ### Connecting to R2 -The `Cloudflare-sdk` module exports a function `S3` that performs the connection. The -function expects an options object that includes an API version and credentials. -Access keys for an IAM user[^9] must be used: +The `@aws-sdk/client-s3` module exports a class `S3Client` that performs the connection. The +function expects an options object that includes a region, credentials and in the case of R2 an endpoint. +this client is used to interact with R2. -```js +```ts +import { S3Client } from "@aws-sdk/client-s3"; /* credentials */ -var accessKeyId = "...", secretAccessKey = "...""; +const accessKeyId = "...", secretAccessKey = "..."; /* file location */ -var Bucket = "...", Key = "pres.numbers"; +const Bucket = "...", Key = "pres.numbers"; /* connect to s3 account */ -var Cloudflare = require('Cloudflare-sdk'); -var s3 = new Cloudflare.S3({ - apiVersion: '2006-03-01', - credentials: { accessKeyId, secretAccessKey } +const Cloudflare = require('Cloudflare-sdk'); +const s3 = new S3Client({ + region: "REGION", //For cloudflare, if the cloudflare region is Auto then you put "auto" + credentials: { + accessKeyId, + secretAccessKey + }, + endpoint: 'R2-ENDPOINT' //Note that the cloudflare dashboard will include the bucket prefix to the endpoint + //Make sure the endpoint ends with .r2.cloudflarestorage.com not .r2.cloudflarestorage.com/bucket-name }); ``` ### Downloading Data -#### Fetching Files from S3 +#### Fetching Files from R2 -The `s3#getObject` method returns an object with a `createReadStream` method. -`createReadStream` returns a NodeJS stream: +The S3 SDK's client mentioned above when instantiated does needs to be fed commands through its ```.send()``` method +to interact with R2. -```js -/* open stream to the file */ -var stream = s3.getObject({ Bucket: Bucket, Key: Key }).createReadStream(); -``` - -#### Concatenating NodeJS Streams - -Buffers can be concatenated from the stream into one unified Buffer object: - -```js -/* array of buffers */ -var bufs = []; -/* add each data chunk to the array */ -stream.on('data', function(data) { bufs.push(data); }); -/* the callback will be called after all of the data is collected */ -stream.on('end', function() { - /* concatenate */ - var buf = Buffer.concat(bufs); - - /* AT THIS POINT, `buf` is a NodeJS Buffer */ -}); -``` - -#### Parsing NodeJS Buffers - -The SheetJS `read` method[^10] can read the final object and generate SheetJS -workbook objects[^11] which can be processed with other API functions. - -For example, a callback can use `sheet_to_csv`[^12] to generate CSV text: - -```js -stream.on('end', function() { - /* concatenate */ - var buf = Buffer.concat(bufs); - - /* parse */ - var wb = XLSX.read(Buffer.concat(bufs)); - - /* generate CSV from first worksheet */ - var first_ws = wb.Sheets[wb.SheetNames[0]]; - var csv = XLSX.utils.sheet_to_csv(first_ws); - console.log(csv); -}); +```ts +import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3"; +/* + ... Client instantiation code from before + ... + ... +*/ +const command = new GetObjectCommand({ + Bucket: 'bucket-name', + Key: 'key' +}) //Create the command object, we are using GetObjectCommand to fetch an item from R2 +const data = await client.send(command); //Send the command +const bytes = await data.Body?.transformToByteArray(); //After receiving the item, transform it to a byte array +const wb = XLSX.read(bytes); //Use SheetJS as normal +const first_ws = wb.Sheets[wb.SheetNames[0]]; +const csv = XLSX.utils.sheet_to_csv(first_ws); ``` ### Uploading Data -The SheetJS `write` method[^13] with the option `type: "buffer"` will generate -NodeJS Buffers. `S3#upload` directly accepts these Buffer objects. +The SheetJS `write` method[^9] with the option `type: "buffer"` will generate +NodeJS Buffers. `R2` directly accepts these Buffer objects. This example creates a sample workbook object, generates XLSX file data in a -NodeJS Buffer, and uploads the data to S3: +NodeJS Buffer, and uploads the data to R2: + +Here we are using the PutObjectCommand object to upload the data written by +`write` into the R2 bucket ```js +import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3"; +/* + ... Client instantiation code from before + ... + ... +*/ /* generate sample workbook */ var wb = XLSX.read("S,h,e,e,t,J,S\n5,4,3,3,7,9,5", {type: "binary"}); /* write to XLSX file in a NodeJS Buffer */ var Body = XLSX.write(wb, {type: "buffer", bookType: "xlsx"}); - -/* upload buffer */ -s3.upload({ Bucket, Key, Body }, function(err, data) { - if(err) throw err; - console.log("Uploaded to " + data.Location); -}); +const command = new PutObjectCommand({ + Bucket: 'bucket-name', + Key: 'key', + Body +}) +await client.send(command); ``` -### S3 Demo +### R2 Demo :::note pass -At the time of writing, the Cloudflare Free Tier included 5GB of Cloudflare R2 with 20,000 -Get requests and 2000 Put requests per month. +At the time of writing, the Cloudflare Free Tier included 10GB of Cloudflare R2 with 1 million Class A operations +and 10 million Class B operations per month + +Please visit the [official pricing model for R2 for more info](https://developers.cloudflare.com/r2/pricing/) ::: -This sample fetches a buffer from S3 and parses the workbook. - -0) If you do not have an account, create a new Cloudflare free tier account[^14]. - -#### Create S3 Bucket - -1) Sign into the [Cloudflare Management Console](https://Cloudflare.amazon.com/console/) with -a root user account. - -2) Type "S3" in the top search box and click S3 (under Services). - -3) Open "Buckets" in the left sidebar. - -If the left sidebar is not open, click the `≡` icon in the left edge of the page. - -4) Click the "Create bucket" button in the main panel. - -5) Select the following options: - -- Type a memorable "Bucket Name" ("sheetjsbouquet" when last tested) - -- In the "Object Ownership" section, select "ACLs disabled" - -- Check "Block *all* public access" - -- Look for the "Bucket Versioning" section and select "Disable" - -6) Click "Create bucket" to create the bucket. - -#### Create IAM User - -7) Type "IAM" in the top search box and click IAM (under Services). - -8) Open "Users" in the left sidebar. - -If the left sidebar is not open, click the `≡` icon in the left edge of the page. - -9) Click the "Create user" button in the main panel. - -10) In step 1, type a memorable "Bucket Name" ("sheetjs-user" when last tested). -Click "Next". - -11) In step 2, click "Next" - -12) In step 3, click "Create user" to create the user. - -#### Add Permissions - -13) Click the new user name in the Users table. - -14) Select the "Permissions" tab - -15) Click the "Add permissions" dropdown and select "Add permissions". - -16) Select "Attach policies directly". - -17) In the "Permissions policies" section, search for "AmazonS3FullAccess". -There should be one entry. - -18) Check the checkbox next to "AmazonS3FullAccess" and click the "Next" button. - -19) In the "Review" screen, click "Add permissions" - -#### Generate Keys - -20) Click "Security credentials", then click "Create access key". - -21) Select the "Local code" option. Check "I understand the above recommendation -and want to proceed to create an access key." and click "Next" - -22) Click "Create Access Key" and click "Download .csv file" in the next screen. - -In the generated CSV: - -- Cell A2 is the "Access key ID" (`accessKeyId` in the Cloudflare API) -- Cell B2 is the "Secret access key" (`secretAccessKey` in the Cloudflare API) - -#### Set up Project - -23) Create a new NodeJS project: - -```bash -mkdir SheetJSS3 -cd SheetJSS3 -npm init -y -``` - -24) Install dependencies: - -{`\ -mkdir -p node_modules -npm i --save https://cdn.sheetjs.com/xlsx-${current}/xlsx-${current}.tgz Cloudflare-sdk@2.1467.0`} - - -#### Write Test - -:::note pass - -This sample creates a simple workbook, generates a NodeJS buffer, and uploads -the buffer to S3. - -``` - | A | B | C | D | E | F | G | ----+---|---|---|---|---|---|---| - 1 | S | h | e | e | t | J | S | - 2 | 5 | 4 | 3 | 3 | 7 | 9 | 5 | -``` - -::: - -25) Save the following script to `SheetJSWriteToS3.js`: - -```js title="SheetJSWriteToS3.js" -var XLSX = require("xlsx"); -var Cloudflare = require('Cloudflare-sdk'); - -/* replace these constants */ -// highlight-start -var accessKeyId = ""; -var secretAccessKey = ""; -var Bucket = ""; -// highlight-end - -var Key = "test.xlsx"; - -/* Create a simple workbook and write XLSX to buffer */ -var ws = XLSX.utils.aoa_to_sheet(["SheetJS".split(""), [5,4,3,3,7,9,5]]); -var wb = XLSX.utils.book_new(); XLSX.utils.book_append_sheet(wb, ws, "Sheet1"); -var Body = XLSX.write(wb, {type: "buffer", bookType: "xlsx"}); - -/* upload buffer */ -var s3 = new Cloudflare.S3({ - apiVersion: '2006-03-01', - credentials: { - accessKeyId: accessKeyId, - secretAccessKey: secretAccessKey - } -}); -s3.upload({ Bucket: Bucket, Key: Key, Body: Body }, function(err, data) { - if(err) throw err; - console.log("Uploaded to " + data.Location); -}); -``` - -26) Edit `SheetJSWriteToS3.js` and replace the highlighted lines: - -- `accessKeyId`: access key for the Cloudflare account -- `secretAccessKey`: secret access key for the Cloudflare account -- `Bucket`: name of the bucket - -The keys are found in the CSV from step 22. The Bucket is the name from step 5. - -27) Run the script: - -```bash -node SheetJSWriteToS3.js -``` - -This file will be stored with the object name `test.xlsx`. It can be manually -downloaded from the S3 web interface. - -#### Read Test - -This sample will download and process the test file from "Write Test". - -28) Save the following script to `SheetJSReadFromS3.js`: - -```js title="SheetJSReadFromS3.js" -var XLSX = require("xlsx"); -var Cloudflare = require('Cloudflare-sdk'); - -/* replace these constants */ -// highlight-start -var accessKeyId = ""; -var secretAccessKey = ""; -var Bucket = ""; -// highlight-end - -var Key = "test.xlsx"; - -/* Get stream */ -var s3 = new Cloudflare.S3({ - apiVersion: '2006-03-01', - credentials: { - accessKeyId: accessKeyId, - secretAccessKey: secretAccessKey - } -}); -var f = s3.getObject({ Bucket: Bucket, Key: Key }).createReadStream(); - -/* collect data */ -var bufs = []; -f.on('data', function(data) { bufs.push(data); }); -f.on('end', function() { - /* concatenate and parse */ - var wb = XLSX.read(Buffer.concat(bufs)); - console.log(XLSX.utils.sheet_to_csv(wb.Sheets[wb.SheetNames[0]])); -}); -``` - -29) Edit `SheetJSReadFromS3.js` and replace the highlighted lines: - -- `accessKeyId`: access key for the Cloudflare account -- `secretAccessKey`: secret access key for the Cloudflare account -- `Bucket`: name of the bucket - -The keys are found in the CSV from Step 22. The Bucket is the name from Step 5. - -30) Run the script: - -```bash -node SheetJSReadFromS3.js -``` - -The program will display the data in CSV format. - -``` -S,h,e,e,t,J,S -5,4,3,3,7,9,5 -``` - [^1]: See ["Node.js compatibility"](https://developers.cloudflare.com/workers/runtime-apis/nodejs/) in the Cloudflare documentation [^2]: See ["Get started guide"](https://developers.cloudflare.com/workers/get-started/guide/#1-create-a-new-worker-project) [^3]: [Wrangler documentation](https://developers.cloudflare.com/workers/wrangler/) @@ -458,10 +233,6 @@ S,h,e,e,t,J,S [^5]: See ["Workbook Object" in "SheetJS Data Model"](/docs/csf/book) for more details. [^6]: See [`sheet_to_csv` in "CSV and Text"](/docs/api/utilities/csv#delimiter-separated-output) [^7]: Registering for a free account [on the Cloudflare Free Tier](https://dash.cloudflare.com/sign-up). -[^8]: The `Cloudflare-sdk` module is distributed [on the public NPM registry](https://npm.im/Cloudflare-sdk) -[^9]: See ["Managing access keys for IAM users"](https://docs.Cloudflare.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html) in the Cloudflare documentation -[^10]: See [`read` in "Reading Files"](/docs/api/parse-options) -[^11]: See ["Workbook Object" in "SheetJS Data Model"](/docs/csf/book) for more details. -[^12]: See [`sheet_to_csv` in "CSV and Text"](/docs/api/utilities/csv#delimiter-separated-output) -[^13]: See [`write` in "Writing Files"](/docs/api/write-options) +[^8]: The `@aws-sdk/client-s3` module is distributed [on the public NPM registry](https://www.npmjs.com/package/@aws-sdk/client-s3) +[^9]: See [`write` in "Writing Files"](/docs/api/write-options) [^14]: Registering for a free account [on the Cloudflare Free Tier](https://Cloudflare.amazon.com/free/) requires a valid phone number and a valid credit card. \ No newline at end of file -- 2.34.1 From 7f64cfa3c41bbf8c438c20e33a0f122ea1f7cd49 Mon Sep 17 00:00:00 2001 From: TomDo1234 Date: Wed, 3 Apr 2024 17:46:37 +1100 Subject: [PATCH 07/10] Done --- docz/docs/03-demos/30-cloud/14-cloudflare.md | 181 +++++++++++++++++++ 1 file changed, 181 insertions(+) diff --git a/docz/docs/03-demos/30-cloud/14-cloudflare.md b/docz/docs/03-demos/30-cloud/14-cloudflare.md index 8f2a3e9..4d6011e 100644 --- a/docz/docs/03-demos/30-cloud/14-cloudflare.md +++ b/docz/docs/03-demos/30-cloud/14-cloudflare.md @@ -226,6 +226,187 @@ Please visit the [official pricing model for R2 for more info](https://developer ::: +This sample fetches a buffer from R2 and parses the workbook. + +1) If you do not have an account, create a new Cloudflare account [^7] + +2) Once in the dashboard, click on R2 , you will need a valid credit card to get access to R2 + +3) Click on Create Bucket. Select your bucket name and region. Then click "Create Bucket" at the bottom + +4) Click on R2 again on the dashboard, then click Manage R2 API Tokens + +5) Click on Create API Token, then make sure to set the correct permissions for what you want to do, then click "Create API Token" at the bottom + +#### Set up Project + +6) Create a new NodeJS project and install dependencies: + +```bash +mkdir SheetJSR2 +cd SheetJSR2 +npm init -y +# Install Dependencies +mkdir -p node_modules +npm i --save https://cdn.sheetjs.com/xlsx-${current}/xlsx-${current}.tgz @aws-sdk/client-s3@3.540.0 +``` + +#### Write Test + +:::note pass + +This sample creates a simple workbook, generates a NodeJS buffer, and uploads +the buffer to S3. + +``` + | A | B | C | D | E | F | G | +---+---|---|---|---|---|---|---| + 1 | S | h | e | e | t | J | S | + 2 | 5 | 4 | 3 | 3 | 7 | 9 | 5 | +``` + +::: + +7) Save the following script to `SheetJSWriteToR2.js`: + +```js title="SheetJSWriteToR2.js" +const { PutObjectCommand, S3Client } = require("@aws-sdk/client-s3"); +const XLSX = require("xlsx"); + +/* replace these constants */ +// highlight-start +var accessKeyId = "accessKeyId"; +var secretAccessKey = "secretAccessKey"; +var Bucket = "Bucket"; +// highlight-end + +var region = "REGION"; +var Key = "sheetjstest.xlsx"; +var endpoint = "ENDPOINT"; + +/* Create a simple workbook and write XLSX to buffer */ +var ws = XLSX.utils.aoa_to_sheet(["SheetJS".split(""), [5,4,3,3,7,9,5]]); +var wb = XLSX.utils.book_new(); XLSX.utils.book_append_sheet(wb, ws, "Sheet1"); +var Body = XLSX.write(wb, {type: "buffer", bookType: "xlsx"}); + +/* upload to R2 */ +var s3 = new S3Client({ + credentials: { + accessKeyId, + secretAccessKey + }, + region, + endpoint +}); + +const command = new PutObjectCommand({ + Body, + Bucket, + Key +}) + +s3.send(command).then(data => { + console.log("Entity Tag: " + data.ETag); +}).catch(e => { + throw e +}); +``` + +:::note pass + +Although this minimal example is in regular js and therefore cannot use async await syntax at top level, be aware that the S3 SDK is async +and thus you can use async await if you wish + +::: + +1) Edit `SheetJSWriteToR2.js` and replace the highlighted lines: + +- `accessKeyId`: access key for the AWS account +- `secretAccessKey`: secret access key for the AWS account +- `Bucket`: name of the bucket +- `REGION`: region of the bucket +- `ENDPOINT`: endpoint of your cloudflare R2, should end with .r2.cloudflarestorage.com , + To get this endpoint go to the settings of your bucket and copy the "S3 API" in Bucket details, remove the /bucket-name at the end of the url + +The keys are found in the CSV from step 22. The Bucket is the name from step 5. + +1) Run the script: + +```bash +node SheetJSWriteToR2.js +``` + +This file will be stored with the object name `test.xlsx`. It can be manually +downloaded from the S3 web interface. + +#### Read Test + +This sample will download and process the test file from "Write Test". + +10) Save the following script to `SheetJSReadFromR2.js`: + +```js title="SheetJSReadFromR2.js" +const { GetObjectCommand, S3Client } = require("@aws-sdk/client-s3"); +const XLSX = require("xlsx"); + +/* replace these constants */ +// highlight-start +var accessKeyId = "accessKeyId"; +var secretAccessKey = "secretAccessKey"; +var Bucket = "Bucket"; +// highlight-end + +var region = "REGION"; +var Key = "sheetjstest.xlsx"; +var endpoint = "ENDPOINT"; + +/* upload to R2 */ +var s3 = new S3Client({ + credentials: { + accessKeyId, + secretAccessKey + }, + region, + endpoint +}); + +const command = new GetObjectCommand({ + Bucket, + Key +}) + +s3.send(command).then(async (data) => { + var wb = XLSX.read(await data.Body?.transformToByteArray()); + console.log(XLSX.utils.sheet_to_csv(wb.Sheets[wb.SheetNames[0]])); +}).catch(e => { + throw e +}); +``` + +11) Edit `SheetJSReadFromR2.js` and replace the highlighted lines: + +- `accessKeyId`: access key for the AWS account +- `secretAccessKey`: secret access key for the AWS account +- `Bucket`: name of the bucket +- `REGION`: region of the bucket +- `ENDPOINT`: endpoint of your cloudflare R2, should end with .r2.cloudflarestorage.com , + To get this endpoint go to the settings of your bucket and copy the "S3 API" in Bucket details, remove the /bucket-name at the end of the url + +The keys are found in the CSV from Step 22. The Bucket is the name from Step 5. + +12) Run the script: + +```bash +node SheetJSReadFromR2.js +``` + +The program will display the data in CSV format. + +``` +S,h,e,e,t,J,S +5,4,3,3,7,9,5 +``` + [^1]: See ["Node.js compatibility"](https://developers.cloudflare.com/workers/runtime-apis/nodejs/) in the Cloudflare documentation [^2]: See ["Get started guide"](https://developers.cloudflare.com/workers/get-started/guide/#1-create-a-new-worker-project) [^3]: [Wrangler documentation](https://developers.cloudflare.com/workers/wrangler/) -- 2.34.1 From 46523eb75a0a635698e404e66c2d1f9aae407595 Mon Sep 17 00:00:00 2001 From: TomDo1234 Date: Wed, 3 Apr 2024 17:47:53 +1100 Subject: [PATCH 08/10] Done 2 --- docz/docs/03-demos/30-cloud/14-cloudflare.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docz/docs/03-demos/30-cloud/14-cloudflare.md b/docz/docs/03-demos/30-cloud/14-cloudflare.md index 4d6011e..c75e295 100644 --- a/docz/docs/03-demos/30-cloud/14-cloudflare.md +++ b/docz/docs/03-demos/30-cloud/14-cloudflare.md @@ -278,11 +278,11 @@ const XLSX = require("xlsx"); var accessKeyId = "accessKeyId"; var secretAccessKey = "secretAccessKey"; var Bucket = "Bucket"; +var region = "REGION"; +var endpoint = "ENDPOINT"; // highlight-end -var region = "REGION"; var Key = "sheetjstest.xlsx"; -var endpoint = "ENDPOINT"; /* Create a simple workbook and write XLSX to buffer */ var ws = XLSX.utils.aoa_to_sheet(["SheetJS".split(""), [5,4,3,3,7,9,5]]); @@ -354,11 +354,11 @@ const XLSX = require("xlsx"); var accessKeyId = "accessKeyId"; var secretAccessKey = "secretAccessKey"; var Bucket = "Bucket"; +var region = "REGION"; +var endpoint = "ENDPOINT"; // highlight-end -var region = "REGION"; var Key = "sheetjstest.xlsx"; -var endpoint = "ENDPOINT"; /* upload to R2 */ var s3 = new S3Client({ @@ -383,7 +383,7 @@ s3.send(command).then(async (data) => { }); ``` -11) Edit `SheetJSReadFromR2.js` and replace the highlighted lines: +1) Edit `SheetJSReadFromR2.js` and replace the highlighted lines: - `accessKeyId`: access key for the AWS account - `secretAccessKey`: secret access key for the AWS account -- 2.34.1 From be1192f0d325d6557095ecbcd1fc9a02bcb828cf Mon Sep 17 00:00:00 2001 From: TomDo1234 Date: Wed, 3 Apr 2024 18:22:39 +1100 Subject: [PATCH 09/10] Done with worker doc --- docz/docs/03-demos/30-cloud/14-cloudflare.md | 53 ++++++++++++++++++-- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/docz/docs/03-demos/30-cloud/14-cloudflare.md b/docz/docs/03-demos/30-cloud/14-cloudflare.md index c75e295..292cbe0 100644 --- a/docz/docs/03-demos/30-cloud/14-cloudflare.md +++ b/docz/docs/03-demos/30-cloud/14-cloudflare.md @@ -113,11 +113,58 @@ free requests per day and 10 milliseconds of CPU time per invocation. ::: -1) If you do not have an account, create a new account[^7]. +1) If you do not have an account, create a new Cloudflare account[^7]. -### Setting up the Project and Testing +#### Setting up the Project and Testing -[The Cloudflare Docs provide a simple step by step guide to setup, develop, test, build and deploy your project](https://developers.cloudflare.com/workers/get-started/guide/) +We will be using the C3 CLI which means we do not have to interact with the Cloudflare dashboard as much. + +2) Create a Worker project with C3 CLI. It will ask you to authenticate into cloudflare. It will also give you some options but for this example we will be using JS, + +```bash +npm create cloudflare@latest +``` + +3) Install Required Dependencies + +```bash +npm i --save https://cdn.sheetjs.com/xlsx-${current}/xlsx-${current}.tgz @aws-sdk/client-s3@3.540.0 +``` + + +4) Navigate to src/index.js, then replace the default `async fetch` function there with this + +```js +const XLSX = require("xlsx"); +export default { + async fetch(request, env, ctx) { + const wb = XLSX.read("S,h,e,e,t,J,S\n5,4,3,3,7,9,5", {type: "binary"}); + /* write to XLSX file in a NodeJS Buffer */ + const data = XLSX.write(wb, {type: "buffer", bookType: "xlsx"}); + return new Response(data,{ + headers: { + "Content-Disposition": 'attachment; filename="SheetJSCFWorker.xlsx"' + } + }) + } +} +``` + +5) Start the development server + +```bash +npx wrangler dev +``` +Then open the browser at the port wrangler opened, it should download a file called SheetJSCFWorker.xlsx with some test data in it + +6) Deploy the worker and configure live enviromnent secrets + +```bash +npx wrangler deploy +``` + +7) Go to your Cloudflare dashboard, go to Workers & Pages, click on your worker, then go to Settings -> Triggers and then click on your worker route, +it should download SheetJSCFWorker.xlsx with the same test data ## Cloudflare R2 -- 2.34.1 From 85b6afe90196981ce40712c9db8738391a3306cb Mon Sep 17 00:00:00 2001 From: TomDo1234 Date: Wed, 3 Apr 2024 18:38:06 +1100 Subject: [PATCH 10/10] Removed 14 footnote --- docz/docs/03-demos/30-cloud/14-cloudflare.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docz/docs/03-demos/30-cloud/14-cloudflare.md b/docz/docs/03-demos/30-cloud/14-cloudflare.md index 292cbe0..ce0d5a4 100644 --- a/docz/docs/03-demos/30-cloud/14-cloudflare.md +++ b/docz/docs/03-demos/30-cloud/14-cloudflare.md @@ -462,5 +462,4 @@ S,h,e,e,t,J,S [^6]: See [`sheet_to_csv` in "CSV and Text"](/docs/api/utilities/csv#delimiter-separated-output) [^7]: Registering for a free account [on the Cloudflare Free Tier](https://dash.cloudflare.com/sign-up). [^8]: The `@aws-sdk/client-s3` module is distributed [on the public NPM registry](https://www.npmjs.com/package/@aws-sdk/client-s3) -[^9]: See [`write` in "Writing Files"](/docs/api/write-options) -[^14]: Registering for a free account [on the Cloudflare Free Tier](https://Cloudflare.amazon.com/free/) requires a valid phone number and a valid credit card. \ No newline at end of file +[^9]: See [`write` in "Writing Files"](/docs/api/write-options) \ No newline at end of file -- 2.34.1