Improvements

Author: xenova

.prettierrc

@@ -0,0 +1,10 @@
+{
+  "overrides": [
+    {
+      "files": ["tests/**/*.ts"],
+      "options": {
+        "printWidth": 10000000
+      }
+    }
+  ]
+}

package.json

@@ -1,22 +1,23 @@
 {
   "name": "@huggingface/tokenizers",
   "version": "0.0.1",
-  "description": "",
-  "main": "dist/tokenizers.cjs",
+  "description": "🤗 Tokenizers.js: A pure JS/TS implementation of today's most used tokenizers",
   "type": "module",
-  "module": "dist/tokenizers.mjs",
+  "main": "dist/tokenizers.min.mjs",
+  "browser": "dist/tokenizers.min.mjs",
+  "module": "dist/tokenizers.min.mjs",
   "types": "types/index.d.ts",
   "exports": {
     ".": {
       "types": "./types/index.d.ts",
       "node": {
-        "require": "./dist/tokenizers.cjs",
-        "import": "./dist/tokenizers.mjs"
+        "require": "./dist/tokenizers.min.cjs",
+        "import": "./dist/tokenizers.min.mjs"
       },
       "browser": {
-        "import": "./dist/tokenizers.mjs"
+        "import": "./dist/tokenizers.min.mjs"
       },
-      "default": "./dist/tokenizers.mjs"
+      "default": "./dist/tokenizers.min.mjs"
     }
   },
   "files": [

scripts/build.mjs

@@ -1,22 +1,11 @@
-import { build } from "esbuild";
+import { build as esbuild } from "esbuild";
 import { execSync } from "node:child_process";
 import { readFileSync } from "node:fs";
 import { gzipSync } from "node:zlib";
 
 console.log("Generating TypeScript declarations...");
 execSync("tsc -p tsconfig.build.json", { stdio: "inherit" });
 
-const config = {
-  bundle: true,
-  minify: true,
-  minifySyntax: true,
-  treeShaking: true,
-  logLevel: "silent",
-  entryPoints: ["src/index.ts"],
-  platform: "neutral",
-  metafile: true,
-};
-
 const formatSize = (bytes) => {
   if (bytes < 1024) return `${bytes}b`;
   if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}kb`;
@@ -34,16 +23,27 @@ const reportSize = (outfile) => {
   console.log(`⚡ Done\n`);
 };
 
-await build({
-  ...config,
-  format: "esm",
-  outfile: "dist/tokenizers.mjs",
-});
-reportSize("dist/tokenizers.mjs");
+const build = async (outfile) => {
+  const format = outfile.endsWith(".mjs") ? "esm" : "cjs";
+  const minifyOptions = /\.min\.[cm]js$/.test(outfile)
+    ? { minify: true, minifySyntax: true }
+    : {};
+
+  await esbuild({
+    bundle: true,
+    treeShaking: true,
+    logLevel: "silent",
+    entryPoints: ["src/index.ts"],
+    platform: "neutral",
+    metafile: true,
+    format,
+    outfile,
+    ...minifyOptions,
+  });
+  reportSize(outfile);
+}
 
-await build({
-  ...config,
-  format: "cjs",
-  outfile: "dist/tokenizers.cjs",
-});
-reportSize("dist/tokenizers.cjs");
+await build("dist/tokenizers.mjs");
+await build("dist/tokenizers.cjs");
+await build("dist/tokenizers.min.mjs");
+await build("dist/tokenizers.min.cjs");