From f1a5d6f3b400c1b576ab4bcf6db97a6330d1d529 Mon Sep 17 00:00:00 2001 From: Nicolas Gaborit Date: Tue, 25 Oct 2016 00:06:17 +0200 Subject: [PATCH] Add benchmarks (#368) * Add script for benchmark * Add error handling * Rename folder to perf/benchmarks * Add README * Avoid memoization between benchmark runs * Handle multiple benchmark. Add setup to benchmarks * Run benchmarks through Travis * Add command line options for JSON output * Add export to JSON, and comparison with reference * Improve serialize and fix results display * Add perf/ to .npmignore * Print error message * Create normal example for normalize * Add normalize-document wide and deep * Add split-block normal, deep and wide * Add delete-backward benchmarks * Fix too much newlines * Use microtime for better results maybe? * Print number of runs * Add minSamples options for better accuracy * Use babel-node to launch benchmarks * Use jsdom-global instead of mocha-jsdom (deprecated) * Add rendering benchmark example * Fix jsdom usage. * Use JSX because we can * Only use on('cycle') that is called even on error * Example of successive rendering benchmark * Rename README, and explain how to add a benchmark * Add C++11 to Travis to install microtime * Update Readme.md # Understanding the results * Try to fix Travis build with microtime * Travis: use before_install Instead of overwriting install * Forgot to remove mocha-jsdom import Thanks node_modules... * Add jsdom as devDependency (required as peer dependency by jsdom-global) * Add --only option to run only a specific benchmark * Print name onStart rather than at end --- .gitignore | 1 + .npmignore | 1 + .travis.yml | 12 +- package.json | 9 +- perf/Readme.md | 63 ++++ perf/benchmarks/delete-backward-deep/index.js | 14 + .../delete-backward-deep/input.yaml | 71 ++++ .../delete-backward-normal/index.js | 14 + .../delete-backward-normal/input.yaml | 41 +++ perf/benchmarks/delete-backward-wide/index.js | 14 + .../delete-backward-wide/input.yaml | 120 ++++++ perf/benchmarks/first-render-normal/index.js | 12 + .../benchmarks/first-render-normal/input.yaml | 40 ++ .../normalize-document-deep/index.js | 6 + .../normalize-document-deep/input.yaml | 70 ++++ .../normalize-document-normal/index.js | 6 + .../normalize-document-normal/input.yaml | 40 ++ .../normalize-document-wide/index.js | 6 + .../normalize-document-wide/input.yaml | 119 ++++++ perf/benchmarks/render-split-block/index.js | 24 ++ perf/benchmarks/render-split-block/input.yaml | 41 +++ perf/benchmarks/split-block-deep/index.js | 14 + perf/benchmarks/split-block-deep/input.yaml | 71 ++++ perf/benchmarks/split-block-normal/index.js | 14 + perf/benchmarks/split-block-normal/input.yaml | 41 +++ perf/benchmarks/split-block-wide/index.js | 14 + perf/benchmarks/split-block-wide/input.yaml | 120 ++++++ perf/index.js | 347 ++++++++++++++++++ test/schema/index.js | 4 +- 29 files changed, 1344 insertions(+), 5 deletions(-) create mode 100644 perf/Readme.md create mode 100644 perf/benchmarks/delete-backward-deep/index.js create mode 100644 perf/benchmarks/delete-backward-deep/input.yaml create mode 100644 perf/benchmarks/delete-backward-normal/index.js create mode 100644 perf/benchmarks/delete-backward-normal/input.yaml create mode 100644 perf/benchmarks/delete-backward-wide/index.js create mode 100644 perf/benchmarks/delete-backward-wide/input.yaml create mode 100644 perf/benchmarks/first-render-normal/index.js create mode 100644 perf/benchmarks/first-render-normal/input.yaml create mode 100644 perf/benchmarks/normalize-document-deep/index.js create mode 100644 perf/benchmarks/normalize-document-deep/input.yaml create mode 100644 perf/benchmarks/normalize-document-normal/index.js create mode 100644 perf/benchmarks/normalize-document-normal/input.yaml create mode 100644 perf/benchmarks/normalize-document-wide/index.js create mode 100644 perf/benchmarks/normalize-document-wide/input.yaml create mode 100644 perf/benchmarks/render-split-block/index.js create mode 100644 perf/benchmarks/render-split-block/input.yaml create mode 100644 perf/benchmarks/split-block-deep/index.js create mode 100644 perf/benchmarks/split-block-deep/input.yaml create mode 100644 perf/benchmarks/split-block-normal/index.js create mode 100644 perf/benchmarks/split-block-normal/input.yaml create mode 100644 perf/benchmarks/split-block-wide/index.js create mode 100644 perf/benchmarks/split-block-wide/input.yaml create mode 100644 perf/index.js diff --git a/.gitignore b/.gitignore index cad6bef59..a96ab7317 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ examples/build.prod.js node_modules tmp _book +perf/reference.json \ No newline at end of file diff --git a/.npmignore b/.npmignore index e507adc88..5cfd1f77a 100644 --- a/.npmignore +++ b/.npmignore @@ -3,3 +3,4 @@ examples src test tmp +perf \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 4321f95c4..bb91ca280 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,16 @@ language: node_js -script: npm run lint && npm test +script: npm run lint && npm test && npm run benchmarks node_js: - "node" - "6" - "4" +# Compiling package microtime requires C++11 compiler. Benchmarks will +# automatically use microtime if installed. +addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-4.8 +before_install: + - if [[ $TRAVIS_OS_NAME == "linux" ]]; then export CXX=g++-4.8; fi diff --git a/package.json b/package.json index 26e22b13f..1938cdedd 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "babel-preset-react": "^6.5.0", "babel-preset-stage-0": "^6.5.0", "babelify": "^7.3.0", + "benchmark": "^2.1.1", "browserify": "^13.0.1", "browserify-global-shim": "^1.0.3", "browserify-shim": "^3.8.12", @@ -46,8 +47,10 @@ "http-server": "^0.9.0", "is-image": "^1.0.1", "is-url": "^1.2.2", + "jsdom": "9.6.0", + "jsdom-global": "2.1.0", + "microtime": "2.1.1", "mocha": "^2.5.3", - "mocha-jsdom": "^1.1.0", "np": "^2.9.0", "npm-run-all": "^2.3.0", "prismjs": "^1.5.1", @@ -86,6 +89,10 @@ "gh-pages": "npm run build && npm run examples && gh-pages --dist ./examples", "lint": "eslint --ignore-pattern 'build.dev.js' --ignore-pattern 'build.prod.js' '{examples,src}/**/*.js'", "open": "open http://localhost:8080/dev.html", + "perf": "npm-run-all lint build:npm benchmarks", + "perf:save": "npm-run-all lint build:npm benchmarks:save", + "benchmarks": "babel-node ./perf/index.js --compare ./perf/reference.json", + "benchmarks:save": "babel-node ./perf/index.js --output ./perf/reference.json", "prepublish": "npm run build", "postpublish": "npm run gh-pages", "release": "np", diff --git a/perf/Readme.md b/perf/Readme.md new file mode 100644 index 000000000..1de903638 --- /dev/null +++ b/perf/Readme.md @@ -0,0 +1,63 @@ +# Benchmarks + +This folder contains a set of benchmarks used to compare performances between Slate's version. We use [BenchmarkJS](https://benchmarkjs.com/) to measure performances. + +## Running the benchmark + +The following command will make sure to compile Slate before running the benchmarks. + +``` +npm run perf +``` + +You can skip Slate's compilation by running directly + +``` +npm run benchmarks +``` + +### Comparing results + +You can save the results of the benchmarks with: + +```shell +npm run perf:save +``` + +The results are saved as JSON in `./perf/reference.json`. You can then checkout a different implementation, and run a comparison benchmark with the usual command: + +``` +npm run perf +``` + +`perf` and `benchmarks` automatically look for an existing `reference.json` to use for comparison. + +### Understanding the results + +Each benchmark prints its results, showing: +- The number of **operation per second**. This is the relevant value, that must be compared with values from different implementation. +- The **number of samples** run. BenchmarkJS has a special heuristic to choose how many samples must be made. The results are more accurate with a high number of samples. Low samples count is often tied with high relative margin of error +- The **relative margin** of error for the measure. The lower the value, the more accurate the results are. When compared with previous results, we display the average relative margin of error. +- (comparison only) A **comparison** of the two implementation, according to BenchmarkJS. It can be Slower, Faster, or Indeterminate. +- (comparison only) The **difference** in operations per second. Expressed as a percentage of the reference. + +## Writing a benchmark + +To add a benchmark, create a new folder in the `perf/benchmarks/` directory. It must contain two files: + +1. `input.yaml` to provide an initial State +2. `index.js` to tell what to run + +`index.js` must export a `run(state)` function. This whole function will be benchmarked. It will be run several times, with the parsed state from `input.yaml` as parameter. You can optionally export a `setup(state) -> state` function, to modify the state parsed from `input.yaml`. + +Note 1: Everything must be sync. + +Note 2: To avoid unwanted memoization, a different instance of `state` will be passed for every `run` call. + +## Detailed options for the benchmark script + +You can also launch the benchmark script directly. See usage: + +``` shell +babel-node ./perf/index.js -h +``` diff --git a/perf/benchmarks/delete-backward-deep/index.js b/perf/benchmarks/delete-backward-deep/index.js new file mode 100644 index 000000000..7dfab6fc7 --- /dev/null +++ b/perf/benchmarks/delete-backward-deep/index.js @@ -0,0 +1,14 @@ + +module.exports = { + setup(state) { + // Move cursor + return state.transform() + .collapseToStartOf({ key: '_cursor_' }) + .moveForward(10) // Move inside the text + .apply() + }, + + run(state) { + return state.transform().deleteBackward().apply() + } +} diff --git a/perf/benchmarks/delete-backward-deep/input.yaml b/perf/benchmarks/delete-backward-deep/input.yaml new file mode 100644 index 000000000..663398117 --- /dev/null +++ b/perf/benchmarks/delete-backward-deep/input.yaml @@ -0,0 +1,71 @@ +nodes: + # Same than normalize-document-normal, but nested in 3 levels of paragraphs +- kind: block + type: paragraph + nodes: + - kind: block + type: paragraph + nodes: + - kind: block + type: paragraph + nodes: + - kind: text + ranges: + - text: 'This is editable ' + - text: 'rich' + marks: + - type: bold + - text: ' text, ' + - kind: block + type: paragraph + nodes: + - kind: text + ranges: + - text: 'much' + marks: + - type: italic + - text: ' better than a ' + - text: '