2.0.0

Author: jonathantneal

.gitignore

@@ -2,3 +2,4 @@ coverage
 dist
 node_modules
 package-lock.json
+yarn.lock

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changes to CSS Tokenizer
 
+### 2.0.0 (May 11, 2021)
+
+- Supports function token (previously an identifier token followed by a parenthesis delimiter token).
+- Changed from default export to named export (`tokenizer`).
+
 ### 1.0.0 (September 26, 2020)
 
 Initial version

README.md

@@ -1,7 +1,7 @@
 # CSS Tokenizer
 
 [<img alt="npm version" src="https://img.shields.io/npm/v/@csstools/tokenizer.svg" height="20">](https://www.npmjs.com/package/@csstools/tokenizer)
-[<img alt="build status" src="https://img.shields.io/travis/csstools/tokenizer/master.svg" height="20">](https://travis-ci.org/github/csstools/tokenizer)
+[<img alt="build status" src="https://img.shields.io/travis/csstools/tokenizer/main.svg" height="20">](https://travis-ci.org/github/csstools/tokenizer)
 [<img alt="code coverage" src="https://img.shields.io/codecov/c/github/csstools/tokenizer" height="20">](https://codecov.io/gh/csstools/tokenizer)
 [<img alt="issue tracker" src="https://img.shields.io/github/issues/csstools/tokenizer.svg" height="20">](https://github.com/csstools/tokenizer/issues)
 [<img alt="pull requests" src="https://img.shields.io/github/issues-pr/csstools/tokenizer.svg" height="20">](https://github.com/csstools/tokenizer/pulls)
@@ -29,17 +29,17 @@ npm install @csstools/tokenizer
 Tokenize CSS in JavaScript:
 
 ```js
-import cssTokenizer from '@csstools/tokenizer'
+import { tokenizer } from '@csstools/tokenizer'
 
-const tokens = Array.from(cssTokenizer(cssText)) // an array of css tokens
+const tokens = Array.from(tokenizer(cssText)) // an array of css tokens
 ```
 
 Tokenize CSS in _classical_ NodeJS:
 
 ```js
-const cssTokenizer = require('@csstools/tokenizer')
+const { tokenizer } = require('@csstools/tokenizer')
 
-let iterator = cssTokenizer(cssText), iteration
+let iterator = tokenizer(cssText), iteration
 
 while (!(iteration = iterator()).done) {
   console.log(iteration.value) // logs an individual css token
@@ -51,9 +51,9 @@ Tokenize CSS in client-side scripts:
 ```html
 <script type="module">
 
-import cssTokenizer from 'https://unpkg.com/@csstools/tokenizer?module'
+import { tokenizer } from 'https://unpkg.com/@csstools/tokenizer?module'
 
-const tokens = Array.from(cssTokenizer(cssText)) // an array of css tokens
+const tokens = Array.from(tokenizer(cssText)) // an array of css tokens
 
 </script>
 ```
@@ -78,16 +78,25 @@ The CSS tokenizer separates a string of CSS into tokens represented as an array.
   /** Position in the string at which the token was retrieved. */
   number,
 
-  /** Negative number that identifies the kind of token. */
-  number,
-
-  /** Opening token content, like the opening of a comment or the quotation mark of a string. */
+  /** Number identifying the kind of token. */
+  | -2 // Comment
+  | -3 // Space
+  | -4 // Identifier
+  | -5 // Function
+  | -6 // At-Identifier
+  | -7 // Hash
+  | -8 // String
+  | -9 // Numeric
+  | number // Delimiter (character code)
+  ,
+
+  /** Lead content, like the opening of a comment or the quotation mark of a string. */
   string,
 
-  /** Main token content, like the numbers before a unit, or the letters after an at-sign. */
+  /** Main content, like the numbers before a unit, or the letters after an at-sign. */
   string,
 
-  /** Closing token content, like the unit of a number, or the closing of a comment. */
+  /** Tail content, like the unit of a number, or the closing of a comment. */
   string,
 ]
 ```
@@ -104,22 +113,23 @@ The **first** number represents the position at which the token was read. The **
 
 ## Benchmarks
 
-As of September 26, 2020, these benchmarks were averaged from my local machine:
+As of May 11, 2021, these benchmarks were averaged from my local machine:
 
 ```
 Benchmark: Tailwind CSS
   ┌──────────────────────────────────────────────────┬───────┬────────┬────────┐
   │                     (index)                      │  ms   │ ms/50k │ tokens │
   ├──────────────────────────────────────────────────┼───────┼────────┼────────┤
-  │ PostCSS x 12.38 ops/sec ±3.33% (35 runs sampled) │ 80.79 │  6.97  │ 579896 │
-  │ Develop x 12.07 ops/sec ±1.54% (34 runs sampled) │ 82.86 │  6.17  │ 670972 │
+  │ PostCSS x 11.86 ops/sec ±2.64% (35 runs sampled) │ 84.31 │  4.52  │ 933299 │
+  │ Develop x 14.98 ops/sec ±1.04% (42 runs sampled) │ 66.75 │  3.53  │ 946324 │
   └──────────────────────────────────────────────────┴───────┴────────┴────────┘
+
 Benchmark: Bootstrap
   ┌────────────────────────────────────────────────┬──────┬────────┬────────┐
   │                    (index)                     │  ms  │ ms/50k │ tokens │
   ├────────────────────────────────────────────────┼──────┼────────┼────────┤
-  │ PostCSS x 230 ops/sec ±0.50% (89 runs sampled) │ 4.35 │  4.37  │ 49807  │
-  │ Develop x 142 ops/sec ±0.76% (80 runs sampled) │ 7.04 │  5.92  │ 59502  │
+  │ PostCSS x 369 ops/sec ±0.12% (95 runs sampled) │ 2.71 │  2.77  │ 49006  │
+  │ Develop x 272 ops/sec ±0.10% (93 runs sampled) │ 3.68 │  3.22  │ 57141  │
   └────────────────────────────────────────────────┴──────┴────────┴────────┘
 ```
 

cmd/_.mjs

@@ -16,7 +16,7 @@ export const spawnNode     = (cmd, args, opts) => spawn('node', [path.resolve(cm
 export const spawnTsc      = (args, opts) => spawnNode('node_modules/typescript/lib/tsc.js', args, opts)
 export const spawnTscWatch = (args, opts) => spawnNode('node_modules/tsc-watch/lib/tsc-watch.js', args, opts)
 
-export const rmdir = (...paths) => fs.rmdirSync(resolve(...paths), { recursive: true })
+export const rmdir = (...paths) => (fs.rmSync || fs.rmdirSync)(resolve(...paths), { force: true, recursive: true })
 
 export const question = query => new Promise(resolve => {
 	const rl = readline.createInterface(question.options)

cmd/test.mjs

@@ -8,6 +8,7 @@ console.log(`Starting testing...`)
 console.log()
 
 _.rmdir('coverage')
+
 _.spawnNode(jestBin, [
 	'--colors',
 	'--passWithNoTests',

jest.config.mjs

@@ -11,6 +11,12 @@ export default /** @type {import('@jest/types').Config.InitialOptions} */ ({
 		'js',
 		'ts'
 	],
+	moduleNameMapper: {
+		"^\\./(.*)\\.js$": [
+			"./$1.js",
+			"./$1.ts"
+		]
+	},
 	roots: [
 		'<rootDir>/src'
 	],

package.json

@@ -2,17 +2,24 @@
   "$schema": "https://json.schemastore.org/package",
   "name": "@csstools/tokenizer",
   "description": "Tokenize CSS according to the CSS Syntax",
-  "version": "1.0.0",
+  "version": "2.0.0",
   "type": "module",
   "main": "dist/index.cjs",
   "module": "dist/index.mjs",
   "types": "dist/index.d.ts",
   "unpkg": "dist/index.js",
+  "exports": {
+    ".": {
+      "require": "./dist/index.cjs",
+      "import": "./dist/index.mjs",
+      "types": "./dist/index.d.ts"
+    }
+  },
   "files": [
     "dist"
   ],
   "engines": {
-    "node": "^10 || ^12 || ^14"
+    "node": ">= 10"
   },
   "scripts": {
     "build": "node cmd/build.mjs",
@@ -29,24 +36,25 @@
     "access": "public"
   },
   "devDependencies": {
-    "@babel/preset-env": "^7.11.5",
-    "@babel/preset-typescript": "^7.10.4",
-    "@rollup/plugin-babel": "^5.2.1",
-    "@types/benchmark": "^1.0.33",
-    "@types/jest": "^26.0.14",
-    "@types/node": "^14.11.2",
+    "@babel/core": "^7.14.0",
+    "@babel/preset-env": "^7.14.1",
+    "@babel/preset-typescript": "^7.13.0",
+    "@rollup/plugin-babel": "^5.3.0",
+    "@types/benchmark": "^2.1.0",
+    "@types/jest": "^26.0.23",
+    "@types/node": "^15.0.2",
     "benchmark": "^2.1.4",
-    "bootstrap": "^4.5.2",
-    "codecov": "^3.7.2",
-    "jest": "^26.4.2",
+    "bootstrap": "^5.0.0",
+    "codecov": "^3.8.2",
+    "jest": "^26.6.3",
     "magic-string": "^0.25.7",
-    "postcss": "^8.0.9",
-    "rollup": "^2.28.2",
+    "postcss": "^8.2.15",
+    "rollup": "^2.47.0",
     "rollup-plugin-bundle-size": "^1.0.3",
     "rollup-plugin-terser": "^7.0.2",
-    "tailwindcss": "^1.8.10",
+    "tailwindcss": "^2.1.2",
     "tsc-watch": "^4.2.9",
-    "tslib": "^2.0.1",
-    "typescript": "^4.0.3"
+    "tslib": "^2.2.0",
+    "typescript": "^4.2.4"
   }
 }

rollup.config.js

@@ -24,7 +24,7 @@ const terserOptions = {
 const resolveJsTsExtension = {
 	name: 'resolve-js-ts-extension',
 	resolveId(/** @type {string} */ importee, /** @type {string} */ importer) {
-		if (importer?.endsWith('.ts') && importee) return path.join(path.dirname(importer), importee + '.ts')
+		if (importer?.endsWith('.ts') && importee) return path.join(path.dirname(importer), importee).replace(/(\.js)?$/, '.ts')
 		return null
 	}
 }
@@ -45,7 +45,8 @@ const config = /** @type {import('rollup').NormalizedInputOptions[]} */ ([
 	{
 		input: './src/index.ts',
 		output: {
-			exports: 'default',
+			esModule: false,
+			exports: 'named',
 			file: './dist/index.mjs',
 			format: 'esm',
 			strict: true,
@@ -60,7 +61,8 @@ const config = /** @type {import('rollup').NormalizedInputOptions[]} */ ([
 	{
 		input: './src/index.ts',
 		output: {
-			exports: 'default',
+			esModule: false,
+			exports: 'named',
 			file: './dist/index.cjs',
 			format: 'cjs',
 			strict: true,
@@ -73,8 +75,9 @@ const config = /** @type {import('rollup').NormalizedInputOptions[]} */ ([
 		],
 	},
 	{
-		input: './src/index.ts',
+		input: './src/index.iife.ts',
 		output: {
+			esModule: false,
 			exports: 'default',
 			file: './dist/index.js',
 			format: 'iife',

src/index.benchmark.ts

@@ -1,7 +1,7 @@
 import * as fs from 'fs'
 import Benchmark from 'benchmark'
-import tokenizerPostCSS from 'postcss/lib/tokenize'
-import tokenizerDevelop from './index'
+import tokenizerPostCSS from 'postcss/lib/tokenize.js'
+import tokenizerDevelop from './index.js'
 
 const counter = { value: 0 }
 

src/index.iife.ts

@@ -0,0 +1 @@
+export { tokenizer as default } from './index.js'