diff --git a/Makefile b/Makefile index 884f9e3..f7b1b5c 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ SHELL := bash ts_files := $(wildcard src/*.ts src/test/integration/*.ts types/*.ts) -fmt_files := $(shell echo examples/{worker,stream-detection}/*.{js,md} .github/workflows/*.yml *.js{,on} *.md src/*.js) +fmt_files := $(shell echo examples/{worker/*.{mjs,md},stream-detection/*.{js,md}} .github/workflows/*.yml *.js{,on} *.md src/*.js) num_processors := $(shell nproc || printf "1") export EMCC_CFLAGS = -msimd128 -O2 diff --git a/README.md b/README.md index 2c174d3..51ac206 100644 --- a/README.md +++ b/README.md @@ -17,21 +17,31 @@ provides accurate filetype detection with zero prod dependencies. npm install wasmagic ``` -Detect the mime and encoding of something: +Detect the mime of something: ```javascript -const { WASMagic, WASMagicFlags } = require("wasmagic"); +import { WASMagic } from "wasmagic"; + +const magic = await WASMagic.create(); + +const pngFile = Buffer.from("89504E470D0A1A0A0000000D49484452", "hex"); +console.log(magic.getMime(pngFile)); +// outputs: image/png +``` + +CommonJS version: + +```javascript +const { WASMagic } = require("wasmagic"); async function main() { - const magic = await WASMagic.create({ - flags: WASMagicFlags.MIME_TYPE | WASMagicFlags.MIME_ENCODING, - }); + const magic = await WASMagic.create(); const pngFile = Buffer.from("89504E470D0A1A0A0000000D49484452", "hex"); console.log(magic.getMime(pngFile)); } main().catch((err) => console.error(err)); -// outputs: image/png; charset=binary +// outputs: image/png ``` ### Options @@ -68,7 +78,19 @@ Default options: ##### `flags` `flags` control `libmagic` behavior. To use the flags, import the `enum` from -the module, [per the example above](#usage). +the module, and pass the desired combination of flags: + +```javascript +import { WASMagic, WASMagicFlags } from "wasmagic"; + +const magic = await WASMagic.create({ + flags: WASMagicFlags.MIME_TYPE | WASMagicFlags.MIME_ENCODING, +}); + +const pngFile = Buffer.from("89504E470D0A1A0A0000000D49484452", "hex"); +console.log(magic.getMime(pngFile)); +// outputs: image/png; charset=binary +``` [Please refer to the code for the flag definitions](src/index.ts#L5) @@ -94,7 +116,7 @@ As these are passed as `Uint8Array`s, custom definitions can be loaded from a file, or embedded directly in your code. For example: ```javascript -const { WASMagic } = require("wasmagic"); +import { WASMagic } from "wasmagic"; const foobarMagic = Buffer.from( ` @@ -103,24 +125,21 @@ const foobarMagic = Buffer.from( `, ); -async function main() { - const magic = await WASMagic.create({ - magicFiles: [foobarMagic], - }); +const magic = await WASMagic.create({ + magicFiles: [foobarMagic], +}); - console.log( - magic.getMime( - Buffer.from( - `FOOBARFILETYPE +console.log( + magic.getMime( + Buffer.from( + `FOOBARFILETYPE Some made up stuff `, - ), ), - ); -} + ), +); -main().catch((err) => console.error(err)); // outputs: foobarfiletype ``` @@ -154,8 +173,11 @@ If you want to offload processing to another thread (and in production workloads you probably should be), take a look at the [Async / Worker threads](examples/worker/) example. +--- + If you aren't passing your instantiated dependencies down in your application, -and are trying to use this as a global, try something like the following: +and you aren't using Javascript modules (or Typescript), and are trying to use +this as a global, try something like the following for a CommonJS style module: ```javascript const { WASMagic } = require("wasmagic"); @@ -185,21 +207,20 @@ To ensure that your application remains performant, either load only the head of the file you want to detect, or slice the head off of a file buffer: ```javascript -const { WASMagic } = require("wasmagic"); -const fs = require("fs/promises"); +import { WASMagic } from "wasmagic"; +import * as fs from "fs/promises"; -async function main() { - const magic = await WASMagic.create(); - const file = await fs.open("largeFile.mp4"); - // Only read the first 1024 bytes of our large file - const { bytesRead, buffer } = await file.read({ buffer: Buffer.alloc(1024) }); - // We're assuming that largeFile.mp4 is >= 1024 bytes in size and our buffer - // will only have the first 1024 bytes of largeFile.mp4 in it - console.log(magic.getMime(buffer)); - await file.close(); -} +const magic = await WASMagic.create(); +const file = await fs.open("largeFile.mp4"); + +// Only read the first 1024 bytes of our large file +const { bytesRead, buffer } = await file.read({ buffer: Buffer.alloc(1024) }); + +// We're assuming that largeFile.mp4 is >= 1024 bytes in size and our buffer +// will only have the first 1024 bytes of largeFile.mp4 in it +console.log(magic.getMime(buffer)); +await file.close(); -main().catch((err) => console.error(err)); // outputs: video/mp4 ``` diff --git a/examples/worker/README.md b/examples/worker/README.md index 7af8e04..b4e830d 100644 --- a/examples/worker/README.md +++ b/examples/worker/README.md @@ -24,5 +24,5 @@ npm install Run: ```bash -node index.js +node index.mjs ``` diff --git a/examples/worker/index.js b/examples/worker/index.js deleted file mode 100644 index 4093a7f..0000000 --- a/examples/worker/index.js +++ /dev/null @@ -1,49 +0,0 @@ -const fs = require("fs/promises"); -const assert = require("assert"); -const path = require("path"); -const os = require("os"); -const Piscina = require("piscina"); - -const testCases = require("../../dist/test/integration/data.js").cases; - -const numCpus = os.cpus().length - 1; - -const wasmagicWorkerPool = new Piscina({ - filename: "./worker.js", - minThreads: numCpus, - maxThreads: numCpus, - idleTimeout: 10000, -}); - -async function main() { - // Fill Buffers - for (const testCase of testCases) { - const file = await fs.open(path.join("..", "..", testCase[0])); - const stats = await file.stat(); - // Copying directly into a SharedArrayBuffer is faster than copying - // the buffer to the worker. - // If you're loading from a file, you could just pass the file path - // instead of the Buffer, and load the file from the worker thread. - const sharedBuf = Buffer.from(new SharedArrayBuffer(stats.size)); - const { bytesRead } = await file.read({ buffer: sharedBuf }); - await file.close(); - assert.equal(stats.size, bytesRead); - testCase.push(sharedBuf); - } - - // Kick off the workers - for (const testCase of testCases) { - testCase.push(wasmagicWorkerPool.run(testCase[2])); - } - - // Check results - for (const testCase of testCases) { - const result = await testCase[3]; - assert.equal(result, testCase[1]); - } -} - -main().catch((err) => { - console.error(err); - process.exit(1); -}); diff --git a/examples/worker/index.mjs b/examples/worker/index.mjs new file mode 100644 index 0000000..093e488 --- /dev/null +++ b/examples/worker/index.mjs @@ -0,0 +1,42 @@ +import * as fs from "fs/promises"; +import * as assert from "assert"; +import * as path from "path"; +import * as os from "os"; +import { Piscina } from "piscina"; + +import { cases as testCases } from "../../dist/test/integration/data.js"; + +const numCpus = os.cpus().length - 1; + +const wasmagicWorkerPool = new Piscina({ + filename: "./worker.mjs", + minThreads: numCpus, + maxThreads: numCpus, + idleTimeout: 10000, +}); + +// Fill Buffers +for (const testCase of testCases) { + const file = await fs.open(path.join("..", "..", testCase[0])); + const stats = await file.stat(); + // Copying directly into a SharedArrayBuffer is faster than copying + // the buffer to the worker. + // If you're loading from a file, you could just pass the file path + // instead of the Buffer, and load the file from the worker thread. + const sharedBuf = Buffer.from(new SharedArrayBuffer(stats.size)); + const { bytesRead } = await file.read({ buffer: sharedBuf }); + await file.close(); + assert.equal(stats.size, bytesRead); + testCase.push(sharedBuf); +} + +// Kick off the workers +for (const testCase of testCases) { + testCase.push(wasmagicWorkerPool.run(testCase[3])); +} + +// Check results +for (const testCase of testCases) { + const result = await testCase[4]; + assert.equal(result, testCase[1]); +} diff --git a/examples/worker/package.json b/examples/worker/package.json index 891b057..c632853 100644 --- a/examples/worker/package.json +++ b/examples/worker/package.json @@ -2,7 +2,7 @@ "name": "wasmagic-worker-example", "version": "0.0.0-dev", "description": "Example using worker threads with WASMagic", - "main": "index.js", + "main": "index.mjs", "author": "Colin Kennedy ", "private": true, "license": "BSD-2-Clause", diff --git a/examples/worker/worker.js b/examples/worker/worker.js deleted file mode 100644 index 8f7574d..0000000 --- a/examples/worker/worker.js +++ /dev/null @@ -1,6 +0,0 @@ -const { WASMagic } = require("../../"); - -module.exports = (async function init() { - const magic = await WASMagic.create(); - return (buf) => magic.getMime(buf); -})(); diff --git a/examples/worker/worker.mjs b/examples/worker/worker.mjs new file mode 100644 index 0000000..13f86a4 --- /dev/null +++ b/examples/worker/worker.mjs @@ -0,0 +1,4 @@ +import { WASMagic } from "../../dist/index.js"; +const magic = await WASMagic.create(); + +export default (buf) => magic.getMime(buf);