Merge pull request #149 from remusao/next

* Performance improvements in all methods of public API (up to x2 faster)
  * `extractHostname`: will now avoid lower-casing the result in some cases
  * `extractHostname`: handles single or triple '/' after protocol
  * `extractHostname`: has fast-path for validation of common protocols (e.g. https)
  * `isProbablyIpv4`: performs first quick check on length of hostname
  * `isProbablyIpv6`: performs first quick check on length of hostname
  * `isValidHostname`: make use of `charCodeAt` instead of `codePointAt`
  * `lookupInTrie`: makes use of Trie with more stable structure (faster)
  * `lookupInTrie`: lazily allocate memory for result
  * `suffixLookup`: uses fast-path for most common suffixes (massive speed-up)
  * `suffixLookup`: does not allocate memory for result anymore
  * `setDefaults`: fast-path in case no argument was provided
  * `getSubdomain`: fast-path if subdomain is empty
* Add more options to fine-tune behavior and performance
  * `detectIp` allows to disable IP check
  * `mixedInput` allows to specify if we expect a mix of URLs and hostnames as
    input. If only hostnames are expected then `extractHostname` can be set to
    `false` to speed-up parsing. If only URLs are expected then `mixedInputs`
    can be set to `false`. The `mixedInputs` is only a hint and will not
    change the behavior of the library.
  * `validateHostname` can be set to `false` to disable validation and
    speed-up processing further.
* Check that input is string before parsing
* Fix support for reserved keywords in hostnames
* Add tests and bring back coverage to 100%
* Minified bundle is now also tested with the same suite
* Migrate utils scripts from `bin/` folder to TypeScript
* Add small `tldts` cli which can be used to parse URLs
* Update README with more accurate information

Author: remusao

.editorconfig

@@ -7,6 +7,6 @@ insert_final_newline = true
 charset = utf-8
 trim_trailing_whitespace = true
 
-[*.{es,js,json,jsm,jsx}]
+[*.{es,js,json,jsm,jsx,ts}]
 indent_style = space
 indent_size = 2

.eslintrc

@@ -1,29 +1,18 @@
-sudo: false
-dist: trusty
+dist: xenial
 language: node_js
 
 node_js:
-  - 6
-  - 8
-  - 9
+  - lts/*
   - node
 
-cache:
-  directories:
-  - node_modules
+cache: npm
 
 notifications:
   email:
     on_failure: change
 
 before_install:
-  - npm install -g npm@latest
-
-install:
-  - npm ci
-
-script:
-  - npm run test
+  - npm install --global npm@latest
 
 after_script:
   - cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js && rm -rf ./coverage

.travis.yml

@@ -2,6 +2,41 @@
 
 ### Not Released
 
+### 5.0.0
+
+*2019-05-23*
+
+- Improvements in various areas [#149](https://github.com/remusao/tldts/pull/149)
+  * Performance improvements in all methods of public API (up to x2 faster)
+    * `extractHostname`: will now avoid lower-casing the result in some cases
+    * `extractHostname`: handles single or triple '/' after protocol
+    * `extractHostname`: has fast-path for validation of common protocols (e.g. https)
+    * `isProbablyIpv4`: performs first quick check on length of hostname
+    * `isProbablyIpv6`: performs first quick check on length of hostname
+    * `isValidHostname`: make use of `charCodeAt` instead of `codePointAt`
+    * `lookupInTrie`: makes use of Trie with more stable structure (faster)
+    * `lookupInTrie`: lazily allocate memory for result
+    * `suffixLookup`: uses fast-path for most common suffixes (massive speed-up)
+    * `suffixLookup`: does not allocate memory for result anymore
+    * `setDefaults`: fast-path in case no argument was provided
+    * `getSubdomain`: fast-path if subdomain is empty
+  * Add more options to fine-tune behavior and performance
+    * `detectIp` allows to disable IP check
+    * `mixedInput` allows to specify if we expect a mix of URLs and hostnames as
+      input. If only hostnames are expected then `extractHostname` can be set to
+      `false` to speed-up parsing. If only URLs are expected then `mixedInputs`
+      can be set to `false`. The `mixedInputs` is only a hint and will not
+      change the behavior of the library.
+    * `validateHostname` can be set to `false` to disable validation and
+      speed-up processing further.
+  * Check that input is string before parsing
+  * Fix support for reserved keywords in hostnames
+  * Add tests and bring back coverage to 100%
+  * Minified bundle is now also tested with the same suite
+  * Migrate utils scripts from `bin/` folder to TypeScript
+  * Add small `tldts` cli which can be used to parse URLs
+  * Update README with more accurate information
+
 ### 4.0.6
 
 *2019-04-15*

CHANGELOG.md

@@ -1,24 +1,23 @@
-# tldts - Hostname and Domain Parsing using Public Suffix Lists
+# tldts - Blazing Fast URL Parsing
 
 [![NPM](https://nodei.co/npm/tldts.png?downloads=true&downloadRank=true)](https://nodei.co/npm/tldts/)
 
 [![Build Status][badge-ci]](http://travis-ci.org/remusao/tldts) ![][badge-downloads]
 ![Coverage Status](https://coveralls.io/repos/github/remusao/tldts/badge.svg?branch=master)
 [![Known Vulnerabilities](https://snyk.io/test/github/remusao/tldts/badge.svg?targetFile=package.json)](https://snyk.io/test/github/remusao/tldts?targetFile=package.json)
 
-
-> `tldts` is a Typescript library to parse hostnames, domains, public suffixes, top-level domains and subdomains from URLs.
-
+`tldts` is a JavaScript library to extract hostnames, domains, public suffixes, top-level domains and subdomains from URLs.
 
 **Features**:
-1. **Fastest library** around (up to 2M operations per second, that's 3 orders of
-   magnitude faster than the most popular library out there)
-2. Written in **TypeScript**, ships with `umd`, `esm`, `cjs` bundles and *type definitions*
+1. Tuned for **performance** (order of 0.1 to 1 μs per input)
+2. Handles both URLs and hostnames
 3. Full Unicode/IDNA support
-4. Support both ICANN and Private suffixes
-5. Ships with continuously updated version of the list: it works *out of the box*!
-6. Support parsing full URLs or hostnames
-7. Small bundles and small memory footprint
+4. Support parsing email addresses
+5. Detect IPv4 and IPv6 addresses
+6. Continuously updated version of the public suffix list
+7. **TypeScript**, ships with `umd`, `esm`, `cjs` bundles and *type definitions*
+8. Small bundles and small memory footprint
+9. Battle tested: full test coverage and production use
 
 # Install
 
@@ -29,7 +28,7 @@ npm install --save tldts
 # Usage
 
 ```js
-const tldts = require('tldts');
+const { parse } = require('tldts');
 
 // Retrieving hostname related informations of a given URL
 parse('http://www.writethedocs.org/conf/eu/2017/');
@@ -42,6 +41,12 @@ parse('http://www.writethedocs.org/conf/eu/2017/');
 //   subdomain: 'www' }
 ```
 
+Modern *ES6 modules import* is also supported:
+
+```js
+import { parse } from 'tldts';
+```
+
 # API
 
 * `tldts.parse(url | hostname, options)`
@@ -51,7 +56,9 @@ parse('http://www.writethedocs.org/conf/eu/2017/');
 * `tldts.getSubdomain(url, | hostname, options)`
 
 The behavior of `tldts` can be customized using an `options` argument for all
-the functions exposed as part of the public API.
+the functions exposed as part of the public API. This is useful to both change
+the behavior of the library as well as fine-tune the performance depending on
+your inputs.
 
 ```js
 {
@@ -60,7 +67,19 @@ the functions exposed as part of the public API.
   // Use suffixes from Private section (default: false)
   allowPrivateDomains: boolean;
   // Extract and validate hostname (default: true)
+  // When set to `false`, inputs will be considered valid hostnames.
   extractHostname: boolean;
+  // Validate hostnames after parsing (default: true)
+  // If a hostname is not valid, not further processing is performed. When set
+  // to `false`, inputs to the library will be considered valid and parsing will
+  // proceed regardless.
+  validateHostname: boolean;
+  // Perform IP address detection (default: true).
+  detectIp: boolean;
+  // Assume that both URLs and hostnames can be given as input (default: true)
+  // If set to `false` we assume only URLs will be given as input, which
+  // speed-ups processing.
+  mixedInputs: boolean;
   // Specifies extra valid suffixes (default: null)
   validHosts: string[] | null;
 }

README.md

@@ -1,39 +1,13 @@
-#!/usr/bin/env node
-
+const Benchmark = require('benchmark');
+const chalk = require('chalk');
 const { URL } = require('url');
 const fs = require('fs');
 const path = require('path');
-
-const tldtsExperimental = require(path.resolve(
-  __dirname,
-  '../dist/tldts-experimental.umd.min.js',
-));
-const tldtsDefault = require(path.resolve(
-  __dirname,
-  '../dist/tldts.umd.min.js',
-));
-
-function bench(title, tldts, inputs) {
-  console.log(`* Start: ${title}`);
-  const t0 = Date.now();
-  for (let i = 0; i < inputs.length; i += 1) {
-    tldts.parse(inputs[i]);
-    tldts.parse(inputs[i]);
-    tldts.parse(inputs[i]);
-    tldts.parse(inputs[i]);
-    tldts.parse(inputs[i]);
-  }
-  const total = Date.now() - t0;
-  console.log(`  - ${total / 5} ms`);
-  console.log(`  - ${total / inputs.length / 5} ms per input`);
-  console.log(
-    `  - ${Math.floor(1000 / (total / inputs.length / 5))} calls per second`,
-  );
-}
+const tldts = require('..');
 
 function main() {
-  const urls = [
-    ...new Set(
+  const urls = Array.from(
+    new Set(
       fs
         .readFileSync(path.resolve(__dirname, './requests.json'), {
           encoding: 'utf-8',
@@ -42,16 +16,67 @@ function main() {
         .map(JSON.parse)
         .map(({ url }) => url),
     ),
-  ];
-  console.log('urls', urls.length);
+  );
+  const hostnames = Array.from(new Set(urls.map(url => new URL(url).hostname)));
+
+  function bench(name, args, fn) {
+    const suite = new Benchmark.Suite();
+    suite
+      .add(name, () => fn(args))
+      .on('cycle', event => {
+        console.log(
+          `  + ${name} ${Math.floor(event.target.hz * args.length)} ops/second`,
+        );
+      })
+      .run({ async: false });
+  }
 
-  const hostnames = [...new Set(urls.map(url => new URL(url).hostname))];
-  console.log('Hosts', hostnames.length);
+  for (const method of [
+    'parse',
+    'getHostname',
+    'getPublicSuffix',
+    'getDomain',
+    'getSubdomain',
+  ]) {
+    console.log(`= ${chalk.bold(method)}`);
+    const fn = tldts[method];
 
-  bench('tldts URLs', tldtsDefault, urls);
-  bench('tldts-experimental URLs', tldtsExperimental, urls);
-  bench('tldts hostnames', tldtsDefault, hostnames);
-  bench('tldts-experimental hostnames', tldtsExperimental, hostnames);
+    for (const options of [
+      undefined, // defaults
+      { validateHostname: false },
+      { validateHostname: false, detectIp: false, mixedInputs: false },
+    ]) {
+      bench(
+        `#${chalk.bold(method)}(url, ${chalk.underline(
+          JSON.stringify(options),
+        )})`,
+        urls,
+        urls => {
+          for (let i = 0; i < urls.length; i += 1) {
+            fn(urls[i], options);
+          }
+        },
+      );
+    }
+
+    for (const options of [
+      undefined, // defaults
+      { validateHostname: false },
+      { validateHostname: false, detectIp: false, extractHostname: false },
+    ]) {
+      bench(
+        `#${chalk.bold(method)}(hostname, ${chalk.underline(
+          JSON.stringify(options),
+        )})`,
+        hostnames,
+        hostnames => {
+          for (let i = 0; i < hostnames.length; i += 1) {
+            fn(hostnames[i], options);
+          }
+        },
+      );
+    }
+  }
 }
 
 main();

bench/benchmark.js

@@ -84,32 +84,37 @@
  * 4. Hash can be computed on-the-fly from end to start without any string copy
  */
 
-const parse = require('../parser');
+import parse from '../parser';
 
 /**
  * Compute 32 bits hash of `str` backward.
  */
-function fastHash(str) {
-  let hash = 5381;
+function fastHash(str: string): number {
+  let hash: number = 5381;
   for (let j = str.length - 1; j >= 0; j -= 1) {
     hash = (hash * 33) ^ str.charCodeAt(j);
   }
   return hash >>> 0;
 }
 
+interface IRules {
+  icann: any[];
+  priv: any[];
+}
+
 /**
  * Build packed typed array given the raw public list as a string.
  */
-module.exports = (body) => {
-  const rules = {
+export default (body: string) => {
+  const rules: IRules = {
     icann: [],
     priv: [],
   };
-  const wildcards = {
+  const wildcards: IRules = {
     icann: [],
     priv: [],
   };
-  const exceptions = {
+  const exceptions: IRules = {
     icann: [],
     priv: [],
   };
@@ -121,13 +126,7 @@ module.exports = (body) => {
   let maximumNumberOfLabels = 0;
 
   // Iterate on public suffix rules
-  parse(body, ({
-    rule,
-    isIcann,
-    isException,
-    isWildcard,
-    isNormal,
-  }) => {
+  parse(body, ({ rule, isIcann, isException, isWildcard, isNormal }) => {
     // Select correct section to insert the rule
     let hashesPerLabels = null;
     if (isException) {
@@ -140,6 +139,10 @@ module.exports = (body) => {
       hashesPerLabels = isIcann ? rules.icann : rules.priv;
     }
 
+    if (hashesPerLabels === null) {
+      return;
+    }
+
     // Count number of labels in this suffix
     const numberOfLabels = rule.split('.').length;
 
@@ -164,7 +167,7 @@ module.exports = (body) => {
   });
 
   // Pack everything together
-  const chunks = [];
+  const chunks: number[][] = [];
   const pushHashes = (hashes = []) => {
     chunks.push([
       hashes.length,
@@ -189,5 +192,6 @@ module.exports = (body) => {
     pushHashes(rules.priv[label]);
   }
 
+  // @ts-ignore
   return new Uint32Array([maximumNumberOfLabels, ...[].concat(...chunks)]);
 };