commit b0ddb606b5d494477a42a83ae8950dc0e75458aa Author: Thomas Wilkerling Date: Sat Mar 17 19:49:01 2018 +0100 -- INITIAL COMMIT -- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..55df214 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +node_modules +.db +!*.keep diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..a874c55 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,3 @@ +language: node_js +node_js: stable +after_success: npm run coverage diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..94134e3 --- /dev/null +++ b/README.md @@ -0,0 +1,793 @@ +

+
+ Search Library +

+ + + + + + + +

+ +

+

Worlds fastest and most memory efficient full text search library with zero dependencies.

+ +When it comes to raw search speed FlexSearch outperforms every single searching library out there and also provides flexible search capabilities like multi-word matching, phonetic transformations or partial matching. It also has the __most memory-efficient index__. Keep in mind that updating existing items from the index has a significant cost. When your index needs to be updated continuously then BulkSearch may be a better choice. FlexSearch also provides you a non-blocking asynchronous processing model as well as web workers to perform any updates on the index as well as queries through dedicated threads. + +Benchmark: +- Library Comparison: https://jsperf.com/compare-search-libraries +- BulkSearch vs. FlexSearch: https://jsperf.com/flexsearch + +Supported Platforms: +- Browser +- Node.js + +Supported Module Definitions: +- AMD (RequireJS) +- CommonJS (Node.js) +- Closure (Xone) +- Global (Browser) + +All Features: + + + +#### Contextual Search + +FlexSearch introduce a new scoring mechanism called __Contextual Search__ which was invented by Thomas Wilkerling, the author of this library. A Contextual Search __incredibly boost up queries to a new level__. +The basic idea of this concept is to limit relevance by context instead of calculating relevance through the whole (unlimited) distance. +Imagine you add a text block of some sentences to an index ID. Assuming the query includes a combination of first and last word from this text block, are they really relevant to each other? +In this way contextual search also improves the results of relevance-based queries on large amount of text data. + +
+

+ +

+
+ +__Note:__ This feature is actually not enabled by default. + + +#### Web-Worker Support + +Workers get its own dedicated memory. Especially for larger indexes, web worker improves speed and available memory a lot. FlexSearch index was tested with a 250 Mb text file including 10 Million words. The indexing was done silently in background by multiple parallel running workers in about 7 minutes. The final index reserves ~ 8.2 Mb memory/space. The search result took ~ 0.25 ms. + +__Note:__ It is slightly faster to use no web worker when the index or query isn't too big (index < 500,000 words, query < 25 words). + +#### Compare BulkSearch vs. FlexSearch + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Description + BulkSearch + FlexSearch +
AccessRead-Write optimized indexRead-Memory optimized index
MemoryLarge: ~ 5 Mb per 100,000 wordsTiny: ~ 100 Kb per 100,000 words
Usecase
  • Limited content
  • Use when existing items of the index needs to be updated continously (update, remove)
  • Supports pagination
  • Fastest possible search
  • Existing items of the index does not need to be updated continously (update, remove)
  • Max out memory capabilities
PaginationYesNo
Ranked SearchingNoYes
Contextual IndexNoYes
WebWorkerNoYes
+ +## Installation + +##### HTML / Javascript + +```html + + + + +... +``` +__Note:__ Use _flexsearch.min.js_ for production and _flexsearch.js_ for development. + +Use latest from CDN: +```html + +``` + +##### Node.js + +```npm +npm install flexsearch +``` + +In your code include as follows: + +```javascript +var FlexSearch = require("flexsearch"); +``` + +Or pass in options when requiring: + +```javascript +var index = require("flexsearch").create({/* options */}); +``` + +__AMD__ + +```javascript +var FlexSearch = require("./flexsearch.js"); +``` + +## API Overview + +Global methods: +- FlexSearch.__create__(\) +- FlexSearch.__addMatcher__({_KEY: VALUE_}) +- FlexSearch.__register__(name, encoder) +- FlexSearch.__encode__(name, string) + +Index methods: +- Index.__add__(id, string) +- Index.__search__(string, \, \) +- Index.__update__(id, string) +- Index.__remove__(id) +- Index.__reset__() +- Index.__destroy__() +- Index.__init__(\) +- Index.__info__() +- Index.__addMatcher__({_KEY: VALUE_}) +- Index.__encode__(string) + +## Usage + +#### Create a new index + +> FlexSearch.__create(\)__ + +```js +var index = new FlexSearch(); +``` + +alternatively you can also use: + +```js +var index = FlexSearch.create(); +``` + +##### Create a new index with custom options + +```js +var index = new FlexSearch({ + + // default values: + + encode: "icase", + mode: "forward", + multi: false, + async: false, + cache: false +}); +``` + +__Read more:__ Phonetic Search, Phonetic Comparison, Improve Memory Usage + +#### Add items to an index + +> Index.__add___(id, string) + +```js +index.add(10025, "John Doe"); +``` + +#### Search items + +> Index.__search(string|options, \, \)__ + +```js +index.search("John"); +``` + +Limit the result: + +```js +index.search("John", 10); +``` + +Perform queries asynchronously: + +```js +index.search("John", function(result){ + + // array of results +}); +``` + +#### Update item to the index + +> Index.__update__(id, string) + +```js +index.update(10025, "Road Runner"); +``` + +#### Remove item to the index + +> Index.__remove__(id) + +```js +index.remove(10025); +``` + +#### Reset index + +```js +index.reset(); +``` + +#### Destroy the index + +```js +index.destroy(); +``` + +#### Re-Initialize index + +> Index.__init(\)__ + +__Note:__ Re-initialization will also destroy the old index! + +Initialize (with same options): +```js +index.init(); +``` + +Initialize with new options: +```js +index.init({ + + /* options */ +}); +``` + +#### Add custom matcher + +> FlexSearch.__addMatcher({_REGEX: REPLACE_})__ + +Add global matchers for all instances: +```js +FlexSearch.addMatcher({ + + 'ä': 'a', // replaces all 'ä' to 'a' + 'ó': 'o', + '[ûúù]': 'u' // replaces multiple +}); +``` + +Add private matchers for a specific instance: +```js +index.addMatcher({ + + 'ä': 'a', // replaces all 'ä' to 'a' + 'ó': 'o', + '[ûúù]': 'u' // replaces multiple +}); +``` + +#### Add custom encoder + +Define a private custom encoder during creation/initialization: +```js +var index = new FlexSearch({ + + encode: function(str){ + + // do something with str ... + + return str; + } +}); +``` + +##### Register a global encoder to be used by all instances + +> FlexSearch.__register(name, encoder)__ + +```js +FlexSearch.register('whitespace', function(str){ + + return str.replace(/ /g, ''); +}); +``` + +Use global encoders: +```js +var index = new FlexSearch({ encode: 'whitespace' }); +``` + +##### Call encoders directly + +Private encoder: +```js +var encoded = index.encode("sample text"); +``` + +Global encoder: +```js +var encoded = FlexSearch.encode("whitespace", "sample text"); +``` + +##### Mixup/Extend multiple encoders + +```js +FlexSearch.register('mixed', function(str){ + + str = this.encode("icase", str); // built-in + str = this.encode("whitespace", str); // custom + + return str; +}); +``` +```js +FlexSearch.register('extended', function(str){ + + str = this.encode("custom", str); + + // do something additional with str ... + + return str; +}); +``` + + +#### Get info + +```js +index.info(); +``` + +Returns information about the index, e.g.: + +```json +{ + "bytes": 3600356288, + "id": 0, + "matchers": 0, + "size": 10000, + "status": false +} +``` + +#### Chaining + +Simply chain methods like: + +```js +var index = FlexSearch.create() + .addMatcher({'â': 'a'}) + .add(0, 'foo') + .add(1, 'bar'); +``` + +```js +index.remove(0).update(1, 'foo').add(2, 'foobar'); +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
OptionValuesDescription
mode




 + "strict"
+ "foward"
+ "reverse"
+ "ngram"
+ "full" + 		+ The indexing mode (tokenizer).




+
encode





 + false
+ "icase"
+ "simple"
+ "advanced"
+ "extra"
+ function() + 		The encoding type.

Choose one of the built-ins or pass a custom encoding function.



cache

 + true
+ false + 		Enable/Disable caching.

async

 + true
+ false + 		Enable/Disable asynchronous processing.

worker

 + false
+ {number} + 		Enable/Disable and set count of running worker threads.

depth

 + false
+ {number} + 		Enable/Disable contextual indexing and also sets relevance depth (experimental).

+ + +## Tokenizer + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
OptionDescriptionExampleMemory Factor (n = length of word)
"strict"index whole wordsfoobar* 1
"foward"incrementally index words in forward directionfoobar
foobar
* n
"reverse"incrementally index words in both directionsfoobar
foobar		* 2n
"ngram" (default)index words partially through phonetic n-gramsfoobar
foobar		* n/4
"full"index every possible combinationfoobar
foobar		* n*(n-1)
+ + +## Phonetic Encoding + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
OptionDescriptionFalse-PositivesCompression
falseTurn off encodingnono
"icase" (default)Case in-sensitive encodingnono
"simple"Phonetic normalizationsno~ 7%
"advanced"Phonetic normalizations + Literal transformationsno~ 35%
"extra"Phonetic normalizations + Soundex transformationsyes~ 60%
function()Pass custom encoding: function(string):string
+ + +#### Comparison + +Reference String: __"Björn-Phillipp Mayer"__ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
QueryiCaseSimpleAdvancedExtra
björnyesyesyesyes
björyesyesyesyes
bjornnoyesyesyes
bjoernnonoyesyes
philippnonoyesyes
filipnonoyesyes
björnphillipnoyesyesyes
meiernonoyesyes
björn meiernonoyesyes
meier fhilipnonoyesyes
byorn mairnononoyes
(false positives)nononoyes
+ + +## Memory Usage + +__Note:__ The required memory depends on several options. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
EncodingMemory usage of every ~ 100,000 indexed words
false260 kb
"icase" (default)210 kb
"simple"190 kb
"advanced"150 kb
"extra"90 kb
ModeMultiplied with: (n = average length of all words)
"strict"* 1
"forward"* n
"reverse"* 2n
"ngram" (default)* n/4
"full"* n*(n-1)
+ +--- + +Author FlexSearch: Thomas Wilkerling
+License: Apache 2.0 License
diff --git a/contextual_index.svg b/contextual_index.svg new file mode 100644 index 0000000..04baad8 --- /dev/null +++ b/contextual_index.svg @@ -0,0 +1,2 @@ + +
Word 2
Word 2
Word 3
Word 3
Word 4
Word 4
Word 5
Word 5
Word 6
Word 6
Word 1
Word 1
Word 7
Word 7
Contextual Index
[Not supported by viewer]
...
...
...
...
Context of Relevance
Context of Relevance
Query
Query
scored by
distance
scored by<br>distance \ No newline at end of file diff --git a/flexsearch.js b/flexsearch.js new file mode 100644 index 0000000..63de198 --- /dev/null +++ b/flexsearch.js @@ -0,0 +1,2126 @@ +;/**! + * FlexSearch - Superfast lightweight full text search engine + * ---------------------------------------------------------- + * @author: Thomas Wilkerling + * @preserve https://github.com/nextapps-de/flexsearch + * @version: 0.2.0 + * @license: Apache 2.0 Licence + */ + +(function(){ + + provide('FlexSearch', (function factory(register_worker){ + + "use strict"; + + /** + * @struct + * @private + * @const + * @final + */ + + var defaults = { + + // bitsize of assigned IDs (data type) + type: 'integer', + + // type of information + mode: 'forward', + + // use flexible cache (scales automatically) + cache: false, + + // use flexible cache (scales automatically) + async: false, + + // use flexible cache (scales automatically) + worker: false, + + // minimum scoring (0 - 9) + threshold: 0, + + // contextual depth + depth: 0, + + // use on of built-in functions + // or pass custom encoding algorithm + encode: false + }; + + /** + * @type {Array} + * @private + */ + + var global_matcher = []; + + /** + * @type {number} + * @private + */ + + var id_counter = 0; + + /** + * @enum {number} + */ + + var enum_task = { + + add: 0, + update: 1, + remove: 2 + }; + + /** @const {RegExp} */ + var regex_split = regex("[ -\/]"); + + /** + * @param {Object=} options + * @constructor + * @private + */ + + function FlexSearch(options){ + + options || (options = defaults); + + // generate UID + + /** @export */ + this.id = options['id'] || id_counter++; + + // initialize index + + this.init(options); + + // define functional properties + + Object.defineProperty(this, 'index', { + + /** + * @this {FlexSearch} + */ + + get: function(){ + + return this._ids; + } + }); + + Object.defineProperty(this, 'length', { + + /** + * @this {FlexSearch} + */ + + get: function(){ + + return Object.keys(this._ids).length; + } + }); + } + + /** + * @param {Object=} options + * @export + */ + + FlexSearch.new = function(options){ + + return new this(options); + }; + + /** + * @param {Object=} options + * @export + */ + + FlexSearch.create = function(options){ + + return FlexSearch.new(options); + }; + + /** + * @param {Object} matcher + * @export + */ + + FlexSearch.addMatcher = function(matcher){ + + for(var key in matcher){ + + if(matcher.hasOwnProperty(key)){ + + global_matcher[global_matcher.length] = regex(key); + global_matcher[global_matcher.length] = matcher[key]; + } + } + + return this; + }; + + /** + * @param {string} name + * @param {function(string):string} encoder + * @export + */ + + FlexSearch.register = function(name, encoder){ + + global_encoder[name] = encoder; + + return this; + }; + + /** + * @param {!string} name + * @param {?string} value + * @returns {?string} + * @export + */ + + FlexSearch.encode = function(name, value){ + + return global_encoder[name](value); + }; + + /** + * @param {Object=} options + * @export + */ + + FlexSearch.prototype.init = function(options){ + + /** @type {Array} */ + this._matcher = []; + + if(options){ + + // initialize worker + + if(options['worker']){ + + if(typeof Worker === 'undefined'){ + + options['worker'] = false; + options['async'] = true; + + this._worker = null; + } + else{ + + var self = this; + var threads = parseInt(options['worker'], 10) || 4; + + self._current_task = -1; + self._task_completed = 0; + self._task_result = []; + self._current_callback = null; + self._worker = new Array(threads); + + for(var i = 0; i < threads; i++){ + + self._worker[i] = add_worker(self.id, i, options || defaults, function(id, query, result, limit){ + + if(self._task_completed === self.worker){ + + return; + } + + self._task_result = self._task_result.concat(result); + self._task_completed++; + + if(limit && (self._task_result.length >= limit)){ + + self._task_completed = self.worker; + } + + if(self._current_callback && (self._task_completed === self.worker)){ + + if(self._task_result.length){ + + self._last_empty_query = ""; + } + else{ + + self._last_empty_query || (self._last_empty_query = query); + } + + // store result to cache + + if(self.cache){ + + self._cache.set(query, self._task_result); + } + + self._current_callback(self._task_result); + self._task_result = []; + } + }); + } + } + } + + // apply options + + this.mode = ( + + options['mode'] || + this.mode || + defaults.mode + ); + + this.cache = ( + + options['cache'] || + this.cache || + defaults.cache + ); + + this.async = ( + + options['async'] || + this.async || + defaults.async + ); + + this.worker = ( + + options['worker'] || + this.worker || + defaults.worker + ); + + this.threshold = ( + + options['threshold'] || + this.threshold || + defaults.threshold + ); + + this.depth = ( + + options['depth'] || + this.depth || + defaults.depth + ); + + /** @export */ + this.encoder = ( + + (options['encode'] && global_encoder[options['encode']]) || + this.encoder || + //(defaults.encode && global_encoder[defaults.encode]) || + options['encode'] + ); + + //if(DEBUG){ + + this.debug = ( + + options['debug'] || + this.debug + ); + //} + + if(options['matcher']) { + + this.addMatcher(/** @type {Object} */ (options['matcher'])); + } + } + + // initialize index + + this._map = [ + + {/* 0 */}, + {/* 1 */}, + {/* 2 */}, + {/* 3 */}, + {/* 4 */}, + {/* 5 */}, + {/* 6 */}, + {/* 7 */}, + {/* 8 */}, + {/* 9 */}, + {/* ctx */} + ]; + + this._ids = {}; + + /** + * @type {Object} + */ + + this._stack = {}; + + /** + * @type {Array} + */ + + this._stack_keys = []; + + /** + * @type {number|null} + */ + + this._timer = null; + this._last_empty_query = ""; + this._status = true; + this._cache = this.cache ? + + (new cache(30 * 1000, 50, true)) + : + false; + + return this; + }; + + /** + * @param {?string} value + * @returns {?string} + * @export + */ + + FlexSearch.prototype.encode = function(value){ + + if(value && global_matcher.length){ + + value = replace(value, global_matcher); + } + + if(value && this._matcher.length){ + + value = replace(value, this._matcher); + } + + if(value && this.encoder){ + + value = this.encoder.call(global_encoder, value); + } + + return value; + }; + + /** + * @param {Object} matcher + * @export + */ + + FlexSearch.prototype.addMatcher = function(matcher){ + + for(var key in matcher){ + + if(matcher.hasOwnProperty(key)){ + + this._matcher[this._matcher.length] = regex(key); + this._matcher[this._matcher.length] = matcher[key]; + } + } + + return this; + }; + + /** + * @param {?number|string} id + * @param {?string} content + * @this {FlexSearch} + * @export + */ + + FlexSearch.prototype.add = function(id, content){ + + if((typeof content === 'string') && content && (id || (id === 0))){ + + // check if index ID already exist + + if(this._ids[id]){ + + this.update(id, content); + } + else{ + + if(this.worker){ + + if(++this._current_task >= this._worker.length) this._current_task = 0; + + this._worker[this._current_task].postMessage(this._current_task, { + + 'add': true, + 'id': id, + 'content': content + }); + + this._ids[id] = "" + this._current_task; + + return this; + } + + if(this.async){ + + this._stack[id] || ( + + this._stack_keys[this._stack_keys.length] = id + ); + + this._stack[id] = [ + + enum_task.add, + id, + content + ]; + + register_task(this); + + return this; + } + + content = this.encode(content); + + if(!content.length){ + + return this; + } + + var words = ( + + content.constructor === Array ? + + /** @type {!Array} */ (content) + : + /** @type {string} */ (content).split(regex_split) + ); + + var dupes = { + + '_ctx': {} + }; + + var threshold = this.threshold; + var map = this._map; + var word_length = words.length; + + // tokenize + + for(var i = 0; i < word_length; i++){ + + /** @type {string} */ + var value = words[i]; + + if(value){ + + var length = value.length; + + switch(this.mode){ + + case 'reverse': + case 'both': + + var tmp = ""; + + for(var a = length - 1; a >= 1; a--){ + + tmp = value[a] + tmp; + + addIndex( + + map, + dupes, + tmp, + id, + /** @type {string} */ (content), + threshold + ); + } + + // Note: no break here, fallthrough to next case + + case 'forward': + + var tmp = ""; + + for(var a = 0; a < length; a++){ + + tmp += value[a]; + + addIndex( + + map, + dupes, + tmp, + id, + /** @type {string} */ (content), + threshold + ); + } + + break; + + case 'full': + + var tmp = ""; + + for(var x = 0; x < length; x++){ + + for(var y = length; y > x; y--){ + + tmp = value.substring(x, y); + + addIndex( + + map, + dupes, + tmp, + id, + /** @type {string} */ (content), + threshold + ); + } + } + + break; + + case 'ngram': + + // TODO + + break; + + case 'strict': + default: + + var score = addIndex( + + map, + dupes, + value, + id, + /** @type {string} */ (content), + threshold + ); + + if((word_length > 1) && this.depth && (score > threshold)){ + + var ctx_map = map[10]; + var ctx_dupes = dupes['_ctx'][value] || (dupes['_ctx'][value] = {}); + var ctx_tmp = ctx_map[value] || (ctx_map[value] = [ + + {/* 0 */}, + {/* 1 */}, + {/* 2 */}, + {/* 3 */}, + {/* 4 */}, + {/* 5 */}, + {/* 6 */}, + {/* 7 */}, + {/* 8 */}, + {/* 9 */} + ]); + + if(i) { + + if(i > 1) { + + addIndex( + + ctx_tmp, + ctx_dupes, + words[i - 2], + id, + /** @type {string} */ (content), + threshold + ); + } + + addIndex( + + ctx_tmp, + ctx_dupes, + words[i - 1], + id, + /** @type {string} */ (content), + threshold + ); + } + + if(i < (word_length - 1)) { + + addIndex( + + ctx_tmp, + ctx_dupes, + words[i + 1], + id, + /** @type {string} */ (content), + threshold + ); + + if(i < (word_length - 2)) { + + addIndex( + + ctx_tmp, + ctx_dupes, + words[i + 2], + id, + /** @type {string} */ (content), + threshold + ); + } + } + } + + break; + } + } + } + + // update status + + this._ids[id] = "1"; + this._status = false; + } + } + + return this; + }; + + /** + * @param id + * @param content + * @export + */ + + FlexSearch.prototype.update = function(id, content){ + + if((typeof content === 'string') && (id || (id === 0))){ + + if(this._ids[id]){ + + if(this.worker){ + + var int = parseInt(this._ids[id], 10); + + this._worker[int].postMessage(int, { + + 'update': true, + 'id': id, + 'content': content + }); + + return this; + } + + if(this.async){ + + this._stack[id] || ( + + this._stack_keys[this._stack_keys.length] = id + ); + + this._stack[id] = [ + + enum_task.update, + id, + content + ]; + + register_task(this); + + return this; + } + + this.remove(id); + + if(content){ + + this.add(id, content); + } + } + } + + return this; + }; + + /** + * @param id + * @export + */ + + FlexSearch.prototype.remove = function(id){ + + if(this._ids[id]){ + + if(this.worker){ + + var int = parseInt(this._ids[id], 10); + + this._worker[int].postMessage(int, { + + 'remove': true, + 'id': id + }); + + delete this._ids[id]; + + return this; + } + + if(this.async){ + + this._stack[id] || ( + + this._stack_keys[this._stack_keys.length] = id + ); + + this._stack[id] = [ + + enum_task.remove, + id + ]; + + register_task(this); + + return this; + } + + for(var z = 0; z < 10; z++){ + + var keys = Object.keys(this._map[z]); + + for(var i = 0; i < keys.length; i++){ + + var key = keys[i]; + var tmp = this._map[z]; + tmp = tmp && tmp[key]; + + if(tmp && tmp.length){ + + for(var a = 0; a < tmp.length; a++){ + + if(tmp[a] === id){ + + tmp.splice(a, 1); + + break; + } + } + } + + if(!tmp.length){ + + delete this._map[z][key]; + } + } + } + + delete this._ids[id]; + + this._status = false; + } + + return this; + }; + + /** + * @param {!string} query + * @param {number|Function=} limit + * @param {Function=} callback + * @returns {Array} + * @export + */ + + FlexSearch.prototype.search = function(query, limit, callback){ + + var threshold; + var result = []; + + if(query && (typeof query === 'object')){ + + // re-assign properties + + callback = query['callback'] || /** @type {?Function} */ (limit); + limit = query['limit']; + threshold = query['threshold']; + query = query['query']; + } + + threshold || (threshold = 0); + + if(typeof limit === 'function'){ + + callback = limit; + limit = 1000; + } + else{ + + limit || (limit = 1000); + } + + if(this.worker){ + + this._current_callback = callback; + this._task_completed = 0; + this._task_result = []; + + for(var i = 0; i < this.worker; i++){ + + this._worker[i].postMessage(i, { + + 'search': true, + 'limit': limit, + 'threshold': threshold, + 'content': query + }); + } + + return null; + } + + if(callback){ + + /** @type {FlexSearch} */ + var self = this; + + queue(function(){ + + callback(self.search(query, limit)); + self = null; + + }, 1, 'search-' + this.id); + + return null; + } + + if(!query || (typeof query !== 'string')){ + + return result; + } + + /** @type {!string|Array} */ + var _query = query; + + // invalidate cache + + if(!this._status){ + + if(this.cache){ + + this._last_empty_query = ""; + this._cache.reset(); + } + + this._status = true; + } + + // validate cache + + else if(this.cache){ + + var cache = this._cache.get(query); + + if(cache){ + + return cache; + } + } + + // validate last query + + else if(this._last_empty_query && (query.indexOf(this._last_empty_query) === 0)){ + + return result; + } + + // encode string + + _query = this.encode(/** @type {string} */ (_query)); + + if(!_query.length){ + + return result; + } + + // convert words into single components + + var words = ( + + _query.constructor === Array ? + + /** @type {!Array} */ (_query) + : + /** @type {string} */ (_query).split(regex_split) + ); + + var length = words.length; + var found = true; + var check = []; + var check_words = {}; + + if(length > 1){ + + if(this.depth){ + + var use_contextual = true; + var key = words[0]; + + check_words[key] = "1"; + } + else{ + + // Note: sort words by length only in non-contextual mode + + words.sort(sort_by_length_down); + } + } + + var ctx_map; + + if(!use_contextual || (ctx_map = this._map[10])[key]){ + + for(var a = use_contextual ? 1 : 0; a < length; a++){ + + var value = words[a]; + + if(value && !check_words[value]){ + + var map; + var map_found = false; + var map_check = []; + var count = 0; + + for(var z = 9; z >= threshold; z--){ + + map = ( + + use_contextual ? + + ctx_map[key][z][value] + : + this._map[z][value] + ); + + if(map){ + + map_check[count++] = map; + map_found = true; + } + } + + if(!map_found){ + + found = false; + break; + } + else{ + + // TODO: handle by intersection + + check[check.length] = ( + + count > 1 ? + + check.concat.apply([], map_check) + : + map_check[0] + ); + } + + check_words[value] = "1"; + } + + key = value; + } + } + else{ + + found = false; + } + + if(found /*&& check.length*/){ + + result = intersect(check, limit); + } + + if(result.length){ + + this._last_empty_query = ""; + } + else{ + + this._last_empty_query || (this._last_empty_query = query); + } + + // store result to cache + + if(this.cache){ + + this._cache.set(query, result); + } + + return result; + }; + + /** + * @export + */ + + FlexSearch.prototype.info = function(){ + + if(this.worker){ + + for(var i = 0; i < this.worker; i++) this._worker[i].postMessage(i, { + + 'info': true, + 'id': this.id + }); + + return; + } + + var keys; + var length; + + var bytes = 0, + words = 0, + chars = 0; + + for(var z = 0; z < 10; z++){ + + keys = Object.keys(this._map[z]); + + for(var i = 0; i < keys.length; i++){ + + length = this._map[z][keys[i]].length; + + // Note: 1 char values allocates 1 byte "Map (OneByteInternalizedString)" + bytes += length * 1 + keys[i].length * 2 + 4; + words += length; + chars += keys[i].length * 2; + } + } + + keys = Object.keys(this._ids); + + var items = keys.length; + + for(var i = 0; i < items; i++){ + + bytes += keys[i].length * 2 + 2; + } + + return { + + 'id': this.id, + 'memory': bytes, + 'items': items, + 'sequences': words, + 'chars': chars, + 'status': this._status, + 'cache': this._stack_keys.length, + 'matcher': global_matcher.length, + 'worker': this.worker + }; + }; + + /** + * @export + */ + + FlexSearch.prototype.reset = function(){ + + // destroy index + + this.destroy(); + + // initialize index + + return this.init(); + }; + + /** + * @export + */ + + FlexSearch.prototype.destroy = function(){ + + // cleanup cache + + if(this.cache){ + + this._cache.reset(); + } + + // release references + + this._scores = + this._map = + this._ids = + this._cache = null; + + return this; + }; + + /** + * Phonetic Encoders + * @dict + * @enum {Function} + * @private + * @const + * @final + */ + + var global_encoder = { + + // case insensitive search + + 'icase': function(value){ + + return value.toLowerCase(); + }, + + // simple phonetic normalization (latin) + + 'simple': (function(){ + + var regex_strip = regex('[^a-z0-9 ]'), + regex_split = regex('[-\/]'), + regex_a = regex('[àáâãäå]'), + regex_e = regex('[èéêë]'), + regex_i = regex('[ìíîï]'), + regex_o = regex('[òóôõöő]'), + regex_u = regex('[ùúûüű]'), + regex_y = regex('[ýŷÿ]'), + regex_n = regex('ñ'), + regex_c = regex('ç'), + regex_s = regex('ß'); + + /** @const {Array} */ + var regex_pairs = [ + + regex_a, 'a', + regex_e, 'e', + regex_i, 'i', + regex_o, 'o', + regex_u, 'u', + regex_y, 'y', + regex_n, 'n', + regex_c, 'c', + regex_s, 's', + regex_split, ' ', + regex_strip, '' + ]; + + return function(str){ + + return ( + + replace(str.toLowerCase(), regex_pairs) + ); + }; + }()), + + // advanced phonetic transformation (latin) + + 'advanced': (function(){ + + var regex_space = regex(' '), + regex_ae = regex('ae'), + regex_ai = regex('ai'), + regex_ay = regex('ay'), + regex_ey = regex('ey'), + regex_oe = regex('oe'), + regex_ue = regex('ue'), + regex_ie = regex('ie'), + regex_sz = regex('sz'), + regex_zs = regex('zs'), + regex_ck = regex('ck'), + regex_cc = regex('cc'), + regex_sh = regex('sh'), + //regex_th = regex('th'), + regex_dt = regex('dt'), + regex_ph = regex('ph'), + regex_pf = regex('pf'), + regex_ou = regex('ou'), + regex_uo = regex('uo'); + + /** @const {Array} */ + var regex_pairs = [ + + regex_ae, 'a', + regex_ai, 'ei', + regex_ay, 'ei', + regex_ey, 'ei', + regex_oe, 'o', + regex_ue, 'u', + regex_ie, 'i', + regex_sz, 's', + regex_zs, 's', + regex_sh, 's', + regex_ck, 'k', + regex_cc, 'k', + //regex_th, 't', + regex_dt, 't', + regex_ph, 'f', + regex_pf, 'f', + regex_ou, 'o', + regex_uo, 'u' + ]; + + return /** @this {global_encoder} */ function(string, _skip_post_processing){ + + if(!string){ + + return string; + } + + // perform simple encoding + string = this['simple'](string); + + // normalize special pairs + if(string.length > 2){ + + string = replace(string, regex_pairs) + } + + if(!_skip_post_processing){ + + // remove white spaces + //string = string.replace(regex_space, ''); + + // delete all repeating chars + if(string.length > 1){ + + string = collapseRepeatingChars(string); + } + } + + return string; + }; + + })(), + + 'extra': (function(){ + + var soundex_b = regex('p'), + //soundex_c = regex('[sz]'), + soundex_s = regex('z'), + soundex_k = regex('[cgq]'), + //soundex_i = regex('[jy]'), + soundex_m = regex('n'), + soundex_t = regex('d'), + soundex_f = regex('[vw]'); + + /** @const {RegExp} */ + var regex_vowel = regex('[aeiouy]'); // [aeiouy] + + /** @const {Array} */ + var regex_pairs = [ + + soundex_b, 'b', + soundex_s, 's', + soundex_k, 'k', + //soundex_i, 'i', + soundex_m, 'm', + soundex_t, 't', + soundex_f, 'f', + regex_vowel, '' + ]; + + return /** @this {global_encoder} */ function(str){ + + if(!str){ + + return str; + } + + // perform advanced encoding + str = this['advanced'](str, /* skip post processing? */ true); + + if(str.length > 1){ + + str = str.split(" "); + + for(var i = 0; i < str.length; i++){ + + var current = str[i]; + + if(current.length > 1){ + + // remove all vowels after 2nd char + str[i] = current[0] + replace(current.substring(1), regex_pairs) + } + } + + str = str.join(""); + str = collapseRepeatingChars(str); + } + + return str; + }; + })(), + + /** + * @param {!string} value + * @this {global_encoder} + * @returns {Array} + */ + + 'ngram': function(value){ + + var parts = []; + + if(!value){ + + return parts; + } + + // perform advanced encoding + value = this['advanced'](value); + + if(!value){ + + return parts; + } + + var count_vowels = 0, + count_literal = 0, + count_parts = -1; + + var tmp = ""; + var length = value.length; + + for(var i = 0; i < length; i++){ + + var char = value[i]; + var char_is_vowel = ( + + (char === 'a') || + (char === 'e') || + (char === 'i') || + (char === 'o') || + (char === 'u') || + (char === 'y') + ); + + if(char_is_vowel){ + + count_vowels++; + } + else{ + + count_literal++; + } + + if(char !== ' ') { + + tmp += char; + } + + // dynamic n-gram sequences + + if((char === ' ') || ((count_vowels >= 2) && (count_literal >= 2)) || (i === length - 1)){ + + if(tmp){ + + var tmp_length = tmp.length; + + if((tmp_length > 2) || (char === ' ') || (i === length - 1)){ + + var char_code = tmp.charCodeAt(0); + + if((tmp_length > 1) || (char_code >= 48) || (char_code <= 57)){ + + parts[++count_parts] = tmp; + } + } + else if(parts[count_parts]){ + + parts[count_parts] += tmp; + } + + tmp = ""; + } + + count_vowels = 0; + count_literal = 0; + } + } + + return parts; + } + + // TODO: provide some common encoder plugins + // soundex + // cologne + // metaphone + // caverphone + // levinshtein + // hamming + // matchrating + }; + + // Xone Async Handler Fallback + + var queue = (function(){ + + var stack = {}; + + return function(fn, delay, id){ + + var timer = stack[id]; + + if(timer){ + + clearTimeout(timer); + } + + return ( + + stack[id] = setTimeout(fn, delay) + ); + }; + })(); + + // Xone Flexi-Cache Handler Fallback + + var cache = (function(){ + + /** @this {Cache} */ + function Cache(){ + + this.cache = {}; + } + + /** @this {Cache} */ + Cache.prototype.reset = function(){ + + this.cache = {}; + }; + + /** @this {Cache} */ + Cache.prototype.set = function(id, value){ + + this.cache[id] = value; + }; + + /** @this {Cache} */ + Cache.prototype.get = function(id){ + + return this.cache[id]; + }; + + return Cache; + })(); + + return FlexSearch; + + // --------------------------------------------------------- + // Helpers + + /** + * @param {!string} str + * @returns {RegExp} + */ + + function regex(str){ + + return new RegExp(str, 'g'); + } + + /** + * @param {!string} str + * @param {RegExp|Array} regex + * @param {string=} replacement + * @returns {string} + */ + + function replace(str, regex, replacement){ + + if(typeof replacement === 'undefined'){ + + for(var i = 0; i < /** @type {Array} */ (regex).length; i += 2){ + + str = str.replace(regex[i], regex[i + 1]); + } + + return str; + } + else{ + + return str.replace(/** @type {!RegExp} */ (regex), replacement); + } + } + + /** + * @param {Array} map + * @param {Object} dupes + * @param {string} tmp + * @param {string|number} id + * @param {string} content + * @param {number} threshold + */ + + function addIndex(map, dupes, tmp, id, content, threshold){ + + if(typeof dupes[tmp] === 'undefined'){ + + var score = calcScore(tmp, content); + + dupes[tmp] = score; + + if(score > threshold){ + + var arr = map[score]; + arr = arr[tmp] || (arr[tmp] = []); + arr[arr.length] = id; + } + } + + return score || dupes[tmp]; + } + + /** + * @param {!string} part + * @param {!string} ref + * @returns {number} + */ + + function calcScore(part, ref){ + + var context_index = ref.indexOf(part); + var partial_index = context_index - ref.lastIndexOf(" ", context_index); + + return ( + + ((3 / ref.length * (ref.length - context_index)) + (6 / partial_index) + 0.5) | 0 + ); + } + + /** + * @param {!string} string + * @returns {string} + */ + + function collapseRepeatingChars(string){ + + var collapsed_string = '', + char_prev = '', + char_next = ''; + + for(var i = 0; i < string.length; i++){ + + var char = string[i]; + + if(char !== char_prev){ + + if((i > 0) && (char === 'h')){ + + var char_prev_is_vowel = ( + + (char_prev === 'a') || + (char_prev === 'e') || + (char_prev === 'i') || + (char_prev === 'o') || + (char_prev === 'u') || + (char_prev === 'y') + ); + + var char_next_is_vowel = ( + + (char_next === 'a') || + (char_next === 'e') || + (char_next === 'i') || + (char_next === 'o') || + (char_next === 'u') || + (char_next === 'y') + ); + + if(char_prev_is_vowel && char_next_is_vowel){ + + collapsed_string += char; + } + } + else{ + + collapsed_string += char; + } + } + + char_next = ( + + (i === (string.length - 1)) ? + + '' + : + string[i + 1] + ); + + char_prev = char; + } + + return collapsed_string; + } + + /** + * @param {string} a + * @param {string} b + * @returns {number} + */ + + function sort_by_length_down(a, b){ + + var diff = a.length - b.length; + + return ( + + diff < 0 ? + + 1 + :( + diff > 0 ? + + -1 + : + 0 + ) + ); + } + + /** + * @param {Array} a + * @param {Array} b + * @returns {number} + */ + + function sort_by_length_up(a, b){ + + var diff = a.length - b.length; + + return ( + + diff < 0 ? + + -1 + :( + diff > 0 ? + + 1 + : + 0 + ) + ); + } + + /** + * @param {!Array>} arr + * @param {number=} limit + * @returns {Array} + */ + + function intersect(arr, limit) { + + var result = []; + var length_z = arr.length; + + if(length_z > 1){ + + arr.sort(sort_by_length_up); + + var map = {}; + var a = arr[0]; + + for(var i = 0, length = a.length; i < length; ++i) { + + map[a[i]] = 1; + } + + var tmp, count = 0; + + for(var z = 1; z < length_z; ++z){ + + var b = arr[z]; + var found = false; + + for(var i = 0, length = b.length; i < length; ++i){ + + if((map[tmp = b[i]]) === z){ + + if(z === (length_z - 1)){ + + result[count++] = tmp; + + if(limit && (count === limit)){ + + return result; + } + } + + found = true; + map[tmp] = z + 1; + + break; + } + } + + if(!found){ + + return []; + } + } + + return result; + } + else if(length_z){ + + result = arr[0]; + + if(limit && result && (result.length > limit)){ + + // Note: do not touch original array + + result = result.slice(0, limit); + } + } + + return result; + } + + /** + * @param {FlexSearch} ref + */ + + function runner(ref){ + + var async = ref.async; + var current; + + if(async){ + + ref.async = false; + } + + if(ref._stack_keys.length){ + + var start = time(); + var key; + + while((key = ref._stack_keys.shift()) || (key === 0)){ + + current = ref._stack[key]; + + switch(current[0]){ + + case enum_task.add: + + ref.add(current[1], current[2]); + break; + + case enum_task.update: + + ref.update(current[1], current[2]); + break; + + case enum_task.remove: + + ref.remove(current[1]); + break; + } + + ref._stack[key] = null; + delete ref._stack[key]; + + if((time() - start) > 100){ + + break; + } + } + + if(ref._stack_keys.length){ + + register_task(ref); + } + } + + if(async){ + + ref.async = async; + } + } + + /** + * @param {FlexSearch} ref + */ + + function register_task(ref){ + + ref._timer || ( + + ref._timer = queue(function(){ + + ref._timer = null; + + runner(ref); + + }, 1, 'search-async-' + ref.id) + ); + } + + /** + * @returns {number} + */ + + function time(){ + + return ( + + typeof performance !== 'undefined' ? + + performance.now() + : + (new Date()).getTime() + ); + } + + function add_worker(id, core, options, callback){ + + var thread = register_worker( + + // name: + 'flexsearch', + + // id: + 'id' + id, + + // worker: + function(){ + + var id; + + /** @type {FlexSearch} */ + var flexsearch; + + /** @lends {Worker} */ + self.onmessage = function(event){ + + var data = event['data']; + + if(data){ + + // if(flexsearch.debug){ + // + // console.log("Worker Job Started: " + data['id']); + // } + + if(data['search']){ + + /** @lends {Worker} */ + self.postMessage({ + + 'result': flexsearch['search'](data['content'], data['threshold'] ? {'limit': data['limit'], 'threshold': data['threshold']} : data['limit']), + 'id': id, + 'content': data['content'], + 'limit': data['limit'] + }); + } + else if(data['add']){ + + flexsearch['add'](data['id'], data['content']); + } + else if(data['update']){ + + flexsearch['update'](data['id'], data['content']); + } + else if(data['remove']){ + + flexsearch['remove'](data['id']); + } + else if(data['reset']){ + + flexsearch['reset'](); + } + else if(data['info']){ + + var info = flexsearch['info'](); + + info['worker'] = id; + + if(flexsearch.debug){ + + console.log(info); + } + + /** @lends {Worker} */ + //self.postMessage(info); + } + else if(data['register']){ + + id = data['id']; + + data['options']['cache'] = false; + data['options']['async'] = true; + data['options']['worker'] = false; + + flexsearch = new Function( + + data['register'].substring( + + data['register'].indexOf('{') + 1, + data['register'].lastIndexOf('}') + ) + )(); + + flexsearch = new flexsearch(data['options']); + } + } + }; + }, + + // callback: + function(event){ + + var data = event['data']; + + if(data && data['result']){ + + callback(data['id'], data['content'], data['result'], data['limit']); + } + //else{ + + //if(DEBUG && options['debug']){ + + //console.log(data); + //} + //} + }, + + // cores: + core + ); + + var fn_str = factory.toString(); + + options['id'] = core; + + thread.postMessage(core, { + + 'register': fn_str, + 'options': options, + 'id': core + }); + + return thread; + } + })( + // Xone Worker Handler Fallback + + (function register_worker(){ + + var worker_stack = {}; + var inline_is_supported = !!((typeof Blob !== 'undefined') && (typeof URL !== 'undefined') && URL.createObjectURL); + + return ( + + /** + * @param {!string} _name + * @param {!number|string} _id + * @param {!Function} _worker + * @param {!Function} _callback + * @param {number=} _core + */ + + function(_name, _id, _worker, _callback, _core){ + + var name = _name; + var worker_payload = ( + + inline_is_supported ? + + /* Load Inline Worker */ + + URL.createObjectURL( + + new Blob( + + ['(' + _worker.toString() + ')()'], { + + 'type': 'text/javascript' + } + ) + ) + : + + /* Load Extern Worker (but also requires CORS) */ + + '../' + name + '.js' + ); + + name += '-' + _id; + + worker_stack[name] || (worker_stack[name] = []); + + worker_stack[name][_core] = new Worker(worker_payload); + worker_stack[name][_core]['onmessage'] = _callback; + + //if(DEBUG){ + + //console.log('Register Worker: ' + name + '@' + _core); + //} + + return { + + 'postMessage': function(id, data){ + + worker_stack[name][id]['postMessage'](data); + } + }; + } + ); + })() + + ), this); + + /** -------------------------------------------------------------------------------------- + * UMD Wrapper for Browser and Node.js + * @param {!string} name + * @param {!Function|Object} factory + * @param {!Function|Object=} root + * @suppress {checkVars} + * @const + */ + + function provide(name, factory, root){ + + var prop; + + // AMD (RequireJS) + if((prop = root['define']) && prop['amd']){ + + prop([], function(){ + + return factory; + }); + } + // Closure (Xone) + else if((prop = root['modules'])){ + + prop[name.toLowerCase()] = factory; + } + // CommonJS (Node.js) + else if(typeof module !== 'undefined'){ + + /** @export */ + module.exports = factory; + } + // Global (window) + else{ + + root[name] = factory; + } + } + +}).call(this); + +// --define='DEBUG=false' diff --git a/flexsearch.min.js b/flexsearch.min.js new file mode 100644 index 0000000..93536eb --- /dev/null +++ b/flexsearch.min.js @@ -0,0 +1,25 @@ +/* + https://github.com/nextapps-de/flexsearch + @version: 0.2.0 + @license: Apache 2.0 Licence +*/ +'use strict';(function(l,C,e){var d;(d=e.define)&&d.amd?d([],function(){return C}):(d=e.modules)?d[l.toLowerCase()]=C:"undefined"!==typeof module?module.exports=C:e[l]=C})("FlexSearch",function G(l){function e(a){a||(a=u);this.id=a.id||H++;this.init(a);Object.defineProperty(this,"index",{get:function(){return this.a}});Object.defineProperty(this,"length",{get:function(){return Object.keys(this.a).length}})}function d(a){return new RegExp(a,"g")}function q(a,b,c){if("undefined"===typeof c){for(c=0;c< +b.length;c+=2)a=a.replace(b[c],b[c+1]);return a}return a.replace(b,c)}function r(a,b,c,f,h,t){if("undefined"===typeof b[c]){var g=h.indexOf(c);g=3/h.length*(h.length-g)+6/(g-h.lastIndexOf(" ",g))+.5|0;b[c]=g;g>t&&(a=a[g],a=a[c]||(a[c]=[]),a[a.length]=f)}return g||b[c]}function w(a){for(var b="",c="",f="",h=0;ha?1:0a?-1:0b&&(c=c.slice(0,b)));return c}function B(a){a.w||(a.w=D(function(){a.w= +null;var b=a.async;b&&(a.async=!1);if(a.c.length){for(var c=E(),f;(f=a.c.shift())||0===f;){var h=a.h[f];switch(h[0]){case z.add:a.add(h[1],h[2]);break;case z.update:a.update(h[1],h[2]);break;case z.remove:a.remove(h[1])}a.h[f]=null;delete a.h[f];if(100=d&&(b.o=b.b),b.v&&b.o===b.b&&(b.i.length?b.f="":b.f||(b.f=c),b.cache&&b.l.set(c,b.i),b.v(b.i),b.i=[]))})}this.mode=a.mode|| +this.mode||u.mode;this.cache=a.cache||this.cache||u.cache;this.async=a.async||this.async||u.async;this.b=a.worker||this.b||u.b;this.threshold=a.threshold||this.threshold||u.threshold;this.depth=a.depth||this.depth||u.depth;this.encoder=a.encode&&A[a.encode]||this.encoder||a.encode;this.A=a.debug||this.A;a.matcher&&this.addMatcher(a.matcher)}this.g=[{},{},{},{},{},{},{},{},{},{},{}];this.a={};this.h={};this.c=[];this.w=null;this.f="";this.u=!0;this.l=this.cache?new L(3E4,50,!0):!1;return this};e.prototype.encode= +function(a){a&&v.length&&(a=q(a,v));a&&this.m.length&&(a=q(a,this.m));a&&this.encoder&&(a=this.encoder.call(A,a));return a};e.prototype.addMatcher=function(a){for(var b in a)a.hasOwnProperty(b)&&(this.m[this.m.length]=d(b),this.m[this.m.length]=a[b]);return this};e.prototype.add=function(a,b){if("string"===typeof b&&b&&(a||0===a))if(this.a[a])this.update(a,b);else{if(this.b)return++this.s>=this.j.length&&(this.s=0),this.j[this.s].postMessage(this.s,{add:!0,id:a,content:b}),this.a[a]=""+this.s,this; +if(this.async)return this.h[a]||(this.c[this.c.length]=a),this.h[a]=[z.add,a,b],B(this),this;b=this.encode(b);if(!b.length)return this;for(var c=b.constructor===Array?b:b.split(F),f={_ctx:{}},h=this.threshold,d=this.g,g=c.length,e=0;em;l--)n=k.substring(m,l), +r(d,f,n,a,b,h);break;case "ngram":break;default:p=r(d,f,k,a,b,h),1h&&(n=d[10],p=f._ctx[k]||(f._ctx[k]={}),k=n[k]||(n[k]=[{},{},{},{},{},{},{},{},{},{}]),e&&(1b;b++)for(var c=Object.keys(this.g[b]),f=0;f=d;A--)if(v=m?r[l][A][q]: +this.g[A][q])w[B++]=v,z=!0;if(z)p[p.length]=1g;g++)for(b=Object.keys(this.g[g]),a=0;a=k)b[++d]=e}else b[d]&&(b[d]+=e);e=""}f=c=0}}return b}},D=function(){var a={};return function(b,c,d){var e=a[d];e&&clearTimeout(e);return a[d]=setTimeout(b,c)}}(),L=function(){function a(){this.cache={}}a.prototype.reset=function(){this.cache={}};a.prototype.set=function(a, +c){this.cache[a]=c};a.prototype.get=function(a){return this.cache[a]};return a}();return e}(function(){var l={},C=!("undefined"===typeof Blob||"undefined"===typeof URL||!URL.createObjectURL);return function(e,d,q,r,w){var x=e;e=C?URL.createObjectURL(new Blob(["("+q.toString()+")()"],{type:"text/javascript"})):"../"+x+".js";x+="-"+d;l[x]||(l[x]=[]);l[x][w]=new Worker(e);l[x][w].onmessage=r;return{postMessage:function(d,e){l[x][d].postMessage(e)}}}}()),this); diff --git a/flexsearch.svg b/flexsearch.svg new file mode 100644 index 0000000..732e57d --- /dev/null +++ b/flexsearch.svg @@ -0,0 +1,98 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/package.json b/package.json new file mode 100644 index 0000000..679d5fe --- /dev/null +++ b/package.json @@ -0,0 +1,46 @@ +{ + "name": "flexsearch", + "version": "0.2.0", + "description": "Superfast, lightweight and memory efficient full text search library.", + "keywords": [], + "bugs": { + "url": "https://github.com/nextapps-de/flexsearch/issues", + "email": "info@nextapps.de" + }, + "main": "flexsearch.js", + "preferGlobal": true, + "bin": {}, + "repository": { + "type": "git", + "url": "https://github.com/nextapps-de/flexsearch.git" + }, + "scripts": { + "build": "java -jar -Xms256m -Xmx4096m node_modules/google-closure-compiler/compiler.jar --compilation_level=ADVANCED_OPTIMIZATIONS --use_types_for_optimization=true --new_type_inf=true --jscomp_warning=newCheckTypes --generate_exports=true --export_local_property_definitions=true --language_in=ECMASCRIPT5_STRICT --language_out=ECMASCRIPT5_STRICT --process_closure_primitives=true --summary_detail_level=3 --warning_level=VERBOSE --emit_use_strict=true --output_manifest=log/manifest.log --output_module_dependencies=log/module_dependencies.log --property_renaming_report=log/renaming_report.log --js='flexsearch.js' --js_output_file='flexsearch.min.js' && echo Build Complete. && exit 0", + "test-production": "nyc --reporter=html --reporter=text mocha --timeout=3000 test --exit", + "test-develop": "nyc --reporter=html --reporter=text mocha --timeout=3000 --exit", + "test-browser": "mocha-phantomjs test/index.html", + "test": "npm run test-develop && npm run test-production && npm run test-browser", + "update": "node_modules/.bin/updtr --to non-breaking", + "coverage": "nyc report --reporter=lcov --reporter=text-lcov | coveralls" + }, + "files": [ + "flexsearch.js", + "flexsearch.min.js", + "test/" + ], + "homepage": "https://nextapps-de.github.io/xone/", + "author": "Thomas Wilkerling", + "license": "Apache-2.0", + "dependencies": {}, + "devDependencies": { + "chai": "^4.1.2", + "coveralls": "^3.0.0", + "google-closure-compiler": "^20180204.0.0", + "mocha": "^5.0.4", + "mocha-lcov-reporter": "^1.3.0", + "mocha-phantomjs": "^4.1.0", + "nyc": "^11.6.0", + "phantomjs-prebuilt": "^2.1.16", + "updtr": "^2.0.0" + } +} diff --git a/test/index.html b/test/index.html new file mode 100644 index 0000000..477f4ec --- /dev/null +++ b/test/index.html @@ -0,0 +1,35 @@ + + + + + Tests + + + + +
+
+ + + + + + + + + diff --git a/test/test.js b/test/test.js new file mode 100644 index 0000000..96f1500 --- /dev/null +++ b/test/test.js @@ -0,0 +1,1065 @@ +if(typeof module !== 'undefined'){ + + // Node.js Stub + + URL = function(string){}; + URL.createObjectURL = function(val){}; + Blob = function(string){}; + + var env = process.argv[3] === 'test' ? '.min' : ''; + var expect = require('chai').expect; + var FlexSearch = require("../flexsearch" + env + ".js"); +} + +var flexsearch_default; +var flexsearch_sync; +var flexsearch_async; +var flexsearch_worker; +var flexsearch_cache; + +var flexsearch_icase; +var flexsearch_simple; +var flexsearch_advanced; +var flexsearch_extra; +var flexsearch_custom; + +var flexsearch_strict; +var flexsearch_forward; +var flexsearch_reverse; +var flexsearch_full; +var flexsearch_ngram; + +// ------------------------------------------------------------------------ +// Acceptance Tests +// ------------------------------------------------------------------------ + +describe('Initialize', function(){ + + it('Should have been initialized successfully', function(){ + + flexsearch_default = new FlexSearch(); + + flexsearch_sync = new FlexSearch({ + + encode: false, + async: false, + worker: false + }); + + flexsearch_async = FlexSearch.create({ + + encode: false, + async: true, + worker: false + }); + + flexsearch_icase = new FlexSearch({ + + encode: 'icase', + async: false, + worker: false + }); + + flexsearch_simple = FlexSearch.create({ + + encode: 'simple', + async: false, + worker: false + }); + + flexsearch_advanced = new FlexSearch({ + + encode: 'advanced', + async: false, + worker: false + }); + + flexsearch_extra = FlexSearch.create({ + + encode: 'extra', + async: false, + worker: false + }); + + flexsearch_custom = new FlexSearch({ + + encode: test_encoder, + async: false, + worker: false + }); + + flexsearch_strict = new FlexSearch({ + + encode: 'icase', + mode: 'strict', + async: false, + worker: false + }); + + flexsearch_forward = new FlexSearch({ + + encode: 'icase', + mode: 'forward', + async: false, + worker: false + }); + + flexsearch_reverse = new FlexSearch({ + + encode: 'icase', + mode: 'reverse', + async: false, + worker: false + }); + + flexsearch_full = new FlexSearch({ + + encode: 'icase', + mode: 'full', + async: false, + worker: false + }); + + flexsearch_ngram = new FlexSearch({ + + encode: 'ngram', + mode: 'strict', + async: false, + worker: false + }); + + flexsearch_cache = new FlexSearch({ + + encode: 'icase', + mode: 'reverse', + cache: true + }); + + expect(flexsearch_default).to.be.an.instanceOf(FlexSearch); + expect(flexsearch_sync).to.be.an.instanceOf(FlexSearch); + expect(flexsearch_async).to.be.an.instanceOf(FlexSearch); + }); + + it('Should have all provided methods', function(){ + + expect(flexsearch_default).to.respondTo("search"); + expect(flexsearch_default).to.respondTo("add"); + expect(flexsearch_default).to.respondTo("update"); + expect(flexsearch_default).to.respondTo("remove"); + expect(flexsearch_default).to.respondTo("reset"); + expect(flexsearch_default).to.respondTo("init"); + expect(flexsearch_default).to.respondTo("info"); + }); + + it('Should have correct uuids', function(){ + + expect(flexsearch_default.id).to.equal(0); + expect(flexsearch_sync.id).to.equal(1); + expect(flexsearch_async.id).to.equal(2); + expect(flexsearch_icase.id).to.equal(3); + expect(flexsearch_simple.id).to.equal(4); + expect(flexsearch_advanced.id).to.equal(5); + expect(flexsearch_extra.id).to.equal(6); + }); + + it('Should have the correct options', function(){ + + expect(flexsearch_default.async).to.equal(false); + expect(flexsearch_default.mode).to.equal("forward"); + expect(flexsearch_sync.async).to.equal(false); + expect(flexsearch_async.async).to.equal(true); + expect(flexsearch_custom.encoder).to.equal(test_encoder); + expect(flexsearch_strict.mode).to.equal("strict"); + expect(flexsearch_forward.mode).to.equal("forward"); + expect(flexsearch_reverse.mode).to.equal("reverse"); + expect(flexsearch_full.mode).to.equal("full"); + expect(flexsearch_ngram.mode).to.equal("strict"); + }); +}); + +describe('Add (Sync)', function(){ + + it('Should have been added to the index', function(){ + + flexsearch_sync.add(0, "foo"); + flexsearch_sync.add(2, "bar"); + flexsearch_sync.add(1, "foobar"); + + expect(flexsearch_sync.index).to.have.keys([0, 1, 2]); + expect(flexsearch_sync.length).to.equal(3); + }); + + it('Should not have been added to the index', function(){ + + flexsearch_sync.add("foo"); + flexsearch_sync.add(3); + flexsearch_sync.add(null, "foobar"); + flexsearch_sync.add(void 0, "foobar"); + flexsearch_sync.add(3, null); + flexsearch_sync.add(3, false); + flexsearch_sync.add(3, []); + flexsearch_sync.add(3, {}); + flexsearch_sync.add(3, function(){}); + + expect(flexsearch_sync.length).to.equal(3); + + flexsearch_extra.add(3, ""); + flexsearch_extra.add(3, " "); + flexsearch_extra.add(3, " "); + flexsearch_extra.add(3, " - "); + + expect(flexsearch_extra.length).to.equal(0); + + flexsearch_extra.add(4, "Thomas"); + flexsearch_extra.add(5, "Arithmetic"); + flexsearch_extra.add(6, "Mahagoni"); + + expect(flexsearch_extra.search("tomass")).to.include(4); + expect(flexsearch_extra.search("arytmetik")).to.include(5); + expect(flexsearch_extra.search("mahagony")).to.include(6); + }); +}); + +describe('Search (Sync)', function(){ + + it('Should have been matched from index', function(){ + + expect(flexsearch_sync.search("foo")).to.have.members([0, 1]); + expect(flexsearch_sync.search("bar")).to.include(2); + expect(flexsearch_sync.search("foobar")).to.include(1); + + expect(flexsearch_sync.search("foo foo")).to.have.members([0, 1]); + expect(flexsearch_sync.search("foo foo")).to.have.members([0, 1]); + }); + + it('Should have been limited', function(){ + + expect(flexsearch_sync.search("foo", 1)).to.include(0); + expect(flexsearch_sync.search({query: "foo", limit: 1})).to.include(0); + expect(flexsearch_sync.search("foo", 1)).to.not.include(1); + }); + + it('Should not have been matched from index', function(){ + + expect(flexsearch_sync.search("barfoo")).to.have.lengthOf(0); + expect(flexsearch_sync.search("")).to.have.lengthOf(0); + expect(flexsearch_sync.search(" ")).to.have.lengthOf(0); + expect(flexsearch_sync.search(" - ")).to.have.lengthOf(0); + expect(flexsearch_sync.search(" o ")).to.have.lengthOf(0); + }); +}); + +describe('Update (Sync)', function(){ + + it('Should have been updated to the index', function(){ + + flexsearch_sync.update(0, "bar"); + flexsearch_sync.update(2, "foobar"); + flexsearch_sync.update(1, "foo"); + + expect(flexsearch_sync.length).to.equal(3); + expect(flexsearch_sync.search("foo")).to.have.members([2, 1]); + expect(flexsearch_sync.search("bar")).to.include(0); + expect(flexsearch_sync.search("bar")).to.not.include(2); + expect(flexsearch_sync.search("foobar")).to.include(2); + + // bypass update: + + flexsearch_sync.add(2, "bar"); + flexsearch_sync.add(0, "foo"); + flexsearch_sync.add(1, "foobar"); + + expect(flexsearch_sync.length).to.equal(3); + expect(flexsearch_sync.search("foo")).to.have.members([0, 1]); + expect(flexsearch_sync.search("bar")).to.include(2); + expect(flexsearch_sync.search("foobar")).to.include(1); + }); + + it('Should not have been updated to the index', function(){ + + flexsearch_sync.update("foo"); + flexsearch_sync.update(0); + flexsearch_sync.update(null, "foobar"); + flexsearch_sync.update(void 0, "foobar"); + flexsearch_sync.update(1, null); + flexsearch_sync.update(2, false); + flexsearch_sync.update(0, []); + flexsearch_sync.update(1, {}); + flexsearch_sync.update(2, function(){}); + + expect(flexsearch_sync.length).to.equal(3); + expect(flexsearch_sync.search("foo")).to.have.members([0, 1]); + expect(flexsearch_sync.search("bar")).to.include(2); + expect(flexsearch_sync.search("foobar")).to.include(1); + }); +}); + +describe('Remove (Sync)', function(){ + + it('Should have been removed from the index', function(){ + + flexsearch_sync.remove(0); + flexsearch_sync.remove(2); + flexsearch_sync.remove(1); + + expect(flexsearch_sync.length).to.equal(0); + expect(flexsearch_sync.search("foo")).to.have.lengthOf(0); + expect(flexsearch_sync.search("bar")).to.have.lengthOf(0); + expect(flexsearch_sync.search("foobar")).to.have.lengthOf(0); + }); +}); + +// ------------------------------------------------------------------------ +// Scoring +// ------------------------------------------------------------------------ + +describe('Apply Sort by Scoring', function(){ + + it('Should have been sorted properly by scoring', function(){ + + flexsearch_sync.add(0, "foo bar foobar"); + flexsearch_sync.add(2, "bar foo foobar"); + flexsearch_sync.add(1, "foobar foo bar"); + + expect(flexsearch_sync.length).to.equal(3); + expect(flexsearch_sync.search("foo")[0]).to.equal(0); + expect(flexsearch_sync.search("foo")[1]).to.equal(1); + expect(flexsearch_sync.search("foo")[2]).to.equal(2); + + expect(flexsearch_sync.search("bar")[0]).to.equal(2); + expect(flexsearch_sync.search("bar")[1]).to.equal(0); // partial scoring! + expect(flexsearch_sync.search("bar")[2]).to.equal(1); + + expect(flexsearch_sync.search("foobar")[0]).to.equal(1); + expect(flexsearch_sync.search("foobar")[1]).to.equal(0); + expect(flexsearch_sync.search("foobar")[2]).to.equal(2); + }); + + it('Should have been sorted properly by threshold', function(){ + + flexsearch_reverse.add(0, "foobarxxx foobarfoobarfoobarxxx foobarfoobarfoobaryyy foobarfoobarfoobarzzz"); + + expect(flexsearch_reverse.search("xxx").length).to.equal(1); + expect(flexsearch_reverse.search("yyy").length).to.equal(1); + expect(flexsearch_reverse.search("zzz").length).to.equal(0); + expect(flexsearch_reverse.search({query: "xxx", threshold: 2}).length).to.equal(1); + expect(flexsearch_reverse.search({query: "xxx", threshold: 5}).length).to.equal(0); + expect(flexsearch_reverse.search({query: "yyy", threshold: 2}).length).to.equal(0); + expect(flexsearch_reverse.search({query: "zzz", threshold: 0}).length).to.equal(0); + }); +}); + +// ------------------------------------------------------------------------ +// Async Tests +// ------------------------------------------------------------------------ + +describe('Add (Async)', function(){ + + it('Should have been added to the index', function(done){ + + flexsearch_async.add(0, "foo"); + flexsearch_async.add(2, "bar"); + flexsearch_async.add(1, "foobar"); + + expect(flexsearch_async.length).to.equal(0); + + setTimeout(function(){ + + expect(flexsearch_async.length).to.equal(3); + expect(flexsearch_async.index).to.have.keys([0, 1, 2]); + + done(); + + }, 25); + }); + + it('Should not have been added to the index', function(done){ + + flexsearch_async.add("foo"); + flexsearch_async.add(3); + flexsearch_async.add(null, "foobar"); + flexsearch_async.add(void 0, "foobar"); + flexsearch_async.add(3, null); + flexsearch_async.add(3, false); + flexsearch_async.add(3, []); + flexsearch_async.add(3, {}); + flexsearch_async.add(3, function(){}); + + setTimeout(function(){ + + expect(flexsearch_async.length).to.equal(3); + expect(flexsearch_async.index).to.have.keys([0, 1, 2]); + + done(); + + }, 25); + }); +}); + +describe('Search (Async)', function(){ + + it('Should have been matched from index', function(done){ + + flexsearch_async.search("foo", function(result){ + + expect(result).to.have.members([0, 1]); + }); + + flexsearch_async.search("bar", function(result){ + + expect(result).to.include(2); + }); + + flexsearch_async.search("foobar", function(result){ + + expect(result).to.include(1); + }); + + setTimeout(function(){ + + done(); + + }, 25); + }); + + it('Should have been limited', function(done){ + + flexsearch_async.search("foo", 1, function(result){ + + expect(result).to.include(0); + expect(result).to.not.include(1); + }); + + setTimeout(function(){ + + done(); + + }, 25); + }); + + it('Should not have been matched from index', function(done){ + + flexsearch_async.search("barfoo", function(result){ + + expect(result).to.have.lengthOf(0); + }); + + flexsearch_async.search("", function(result){ + + expect(result).to.have.lengthOf(0); + }); + + flexsearch_async.search(" ", function(result){ + + expect(result).to.have.lengthOf(0); + }); + + flexsearch_async.search(" o ", function(result){ + + expect(result).to.have.lengthOf(0); + }); + + setTimeout(function(){ + + done(); + + }, 25); + }); +}); + +describe('Update (Async)', function(){ + + it('Should have been updated to the index', function(done){ + + flexsearch_async.update(0, "bar"); + flexsearch_async.update(2, "foobar"); + flexsearch_async.update(1, "foo"); + + expect(flexsearch_async.length).to.equal(3); + expect(flexsearch_async.search("foo")).to.not.have.members([2, 1]); + expect(flexsearch_async.search("bar")).to.not.include(0); + expect(flexsearch_async.search("bar")).to.include(2); + expect(flexsearch_async.search("foobar")).to.not.include(2); + + setTimeout(function(){ + + expect(flexsearch_async.length).to.equal(3); + expect(flexsearch_async.search("foo")).to.have.members([2, 1]); + expect(flexsearch_async.search("bar")).to.include(0); + expect(flexsearch_async.search("bar")).to.not.include(2); + expect(flexsearch_async.search("foobar")).to.include(2); + + done(); + + }, 25); + }); + + it('Should not have been updated to the index', function(done){ + + flexsearch_async.update("foo"); + flexsearch_async.update(0); + flexsearch_async.update(null, "foobar"); + flexsearch_async.update(void 0, "foobar"); + flexsearch_async.update(1, null); + flexsearch_async.update(2, false); + flexsearch_async.update(0, []); + flexsearch_async.update(1, {}); + flexsearch_async.update(2, function(){}); + + setTimeout(function(){ + + expect(flexsearch_async.length).to.equal(3); + expect(flexsearch_async.search("foo")).to.have.members([2, 1]); + expect(flexsearch_async.search("bar")).to.include(0); + expect(flexsearch_async.search("bar")).to.not.include(2); + expect(flexsearch_async.search("foobar")).to.include(2); + + done(); + + }, 25); + }); +}); + +describe('Remove (Async)', function(){ + + it('Should have been removed from the index', function(done){ + + flexsearch_async.remove(0); + flexsearch_async.remove(2); + flexsearch_async.remove(1); + + expect(flexsearch_async.length).to.equal(3); + + setTimeout(function(){ + + expect(flexsearch_async.length).to.equal(0); + expect(flexsearch_async.search("foo")).to.have.lengthOf(0); + expect(flexsearch_async.search("bar")).to.have.lengthOf(0); + expect(flexsearch_async.search("foobar")).to.have.lengthOf(0); + + done(); + + }, 25); + }); +}); + +// ------------------------------------------------------------------------ +// Worker Tests +// ------------------------------------------------------------------------ + +describe('Add (Worker)', function(){ + + it('Should support worker', function(done){ + + if(typeof Worker === 'undefined'){ + + Worker = function(string){}; + Worker.prototype.postMessage = function(val){ + this.onmessage(val); + }; + Worker.prototype.onmessage = function(val){ + return val; + }; + } + + flexsearch_worker = new FlexSearch({ + + encode: false, + mode: 'strict', + async: true, + worker: 4 + }); + + done(); + }); + + it('Should have been added to the index', function(done){ + + flexsearch_worker.add(0, "foo"); + flexsearch_worker.add(2, "bar"); + flexsearch_worker.add(1, "foobar"); + + expect(flexsearch_worker.length).to.equal(3); + expect(flexsearch_worker.index).to.have.keys([0, 1, 2]); + + setTimeout(function(){ + + expect(flexsearch_worker.length).to.equal(3); + expect(flexsearch_worker.index).to.have.keys([0, 1, 2]); + + done(); + + }, 25); + }); + + it('Should not have been added to the index', function(done){ + + flexsearch_worker.add("foo"); + flexsearch_worker.add(3); + flexsearch_worker.add(null, "foobar"); + flexsearch_worker.add(void 0, "foobar"); + flexsearch_worker.add(3, null); + flexsearch_worker.add(3, false); + flexsearch_worker.add(3, []); + flexsearch_worker.add(3, {}); + flexsearch_worker.add(3, function(){}); + + setTimeout(function(){ + + expect(flexsearch_worker.length).to.equal(3); + expect(flexsearch_worker.index).to.have.keys([0, 1, 2]); + + done(); + + }, 25); + }); +}); + +describe('Search (Worker)', function(){ + + it('Should have been matched from index', function(done){ + + flexsearch_worker.search("foo", function(result){ + + expect(result).to.have.members([0, 1]); + }); + + flexsearch_worker.search("bar", function(result){ + + expect(result).to.have.members([2, 1]); + }); + + flexsearch_worker.search("foobar", function(result){ + + expect(result).to.include(1); + }); + + setTimeout(function(){ + + done(); + + }, 25); + }); + + it('Should have been limited', function(done){ + + flexsearch_worker.search("foo", 1, function(result){ + + expect(result).to.include(0); + expect(result).to.not.include(1); + }); + + setTimeout(function(){ + + done(); + + }, 25); + }); + + it('Should not have been matched from index', function(done){ + + flexsearch_worker.search("barfoo", function(result){ + + expect(result).to.have.lengthOf(0); + }); + + flexsearch_worker.search("", function(result){ + + expect(result).to.have.lengthOf(0); + }); + + flexsearch_worker.search(" ", function(result){ + + expect(result).to.have.lengthOf(0); + }); + + flexsearch_worker.search(" o ", function(result){ + + expect(result).to.have.lengthOf(0); + }); + + setTimeout(function(){ + + done(); + + }, 25); + }); +}); + +describe('Update (Worker)', function(){ + + it('Should have been updated to the index', function(done){ + + flexsearch_worker.update(0, "bar"); + flexsearch_worker.update(2, "foobar"); + flexsearch_worker.update(1, "foo"); + + expect(flexsearch_worker.length).to.equal(3); + + flexsearch_worker.search("foo", function(results){ + + expect(results).to.have.members([2, 1]); + }); + + flexsearch_worker.search("bar", function(results){ + + expect(results).to.have.members([0, 2]); + }); + + flexsearch_worker.search("foobar", function(results){ + + expect(results).to.include(2); + }); + + setTimeout(function(){ + + done(); + + }, 25); + }); +}); + +describe('Remove (Worker)', function(){ + + it('Should have been removed from the index', function(done){ + + flexsearch_worker.remove(0); + flexsearch_worker.remove(2); + flexsearch_worker.remove(1); + + expect(flexsearch_worker.length).to.equal(0); + + flexsearch_worker.search("foo", function(results){ + + expect(results).to.not.include(1); + expect(results).to.not.include(2); + }); + + flexsearch_worker.search("bar", function(results){ + + expect(results).to.not.include(0); + expect(results).to.not.include(2); + }); + + flexsearch_worker.search("foobar", function(results){ + + expect(results).to.not.include(2); + }); + + setTimeout(function(){ + + done(); + + }, 25); + }); + + it('Should have been debug mode activated', function(){ + + flexsearch_worker.info(); + }); +}); + +describe('Worker Not Supported', function(){ + + it('Should not support worker', function(){ + + if(typeof Worker !== 'undefined'){ + + Worker = void 0; + } + + flexsearch_worker = new FlexSearch({ + + encode: false, + async: true, + worker: 4 + }); + + expect(flexsearch_worker.info().worker).to.equal(false); + }); +}); + +// ------------------------------------------------------------------------ +// Phonetic Tests +// ------------------------------------------------------------------------ + +describe('Encoding', function(){ + + it('Should have been encoded properly: iCase', function(){ + + expect(flexsearch_icase.encode("Björn-Phillipp Mayer")).to.equal(flexsearch_icase.encode("björn-phillipp mayer")); + }); + + it('Should have been encoded properly: Simple', function(){ + + expect(flexsearch_simple.encode("Björn-Phillipp Mayer")).to.equal(flexsearch_simple.encode("bjorn/phillipp mayer")); + }); + + it('Should have been encoded properly: Advanced', function(){ + + expect(flexsearch_advanced.encode("Björn-Phillipp Mayer")).to.equal(flexsearch_advanced.encode("bjoern filip mair")); + }); + + it('Should have been encoded properly: Extra', function(){ + + expect(flexsearch_extra.encode("Björn-Phillipp Mayer")).to.equal(flexsearch_extra.encode("bjorm filib mayr")); + }); + + it('Should have been encoded properly: Custom Encoder', function(){ + + expect(flexsearch_custom.encode("Björn-Phillipp Mayer")).to.equal("-[BJÖRN-PHILLIPP MAYER]-"); + }); + + it('Should have been encoded properly: Custom Encoder', function(){ + + FlexSearch.register('custom', test_encoder); + + expect(FlexSearch.encode('custom', "Björn-Phillipp Mayer")).to.equal(flexsearch_custom.encode("Björn-Phillipp Mayer")); + }); +}); + +// ------------------------------------------------------------------------ +// Contextual Indexing +// ------------------------------------------------------------------------ + +describe('Context', function(){ + + it('Should have been added properly to the context', function(){ + + var flexsearch_depth = new FlexSearch({ + + encode: 'icase', + mode: 'strict', + depth: 2, + async: false, + worker: false + }); + + flexsearch_depth.add(0, "zero one two three four five six seven eight nine ten"); + + expect(flexsearch_depth.length).to.equal(1); + expect(flexsearch_depth.search("zero one")).to.include(0); + expect(flexsearch_depth.search("zero two")).to.include(0); + expect(flexsearch_depth.search("zero three").length).to.equal(0); + expect(flexsearch_depth.search("three seven").length).to.equal(0); + expect(flexsearch_depth.search("three five seven")).to.include(0); + // TODO + // expect(flexsearch_depth.search("three seven five")).to.include(0); + expect(flexsearch_depth.search("three foobar seven").length).to.equal(0); + expect(flexsearch_depth.search("eight ten")).to.include(0); + expect(flexsearch_depth.search("seven ten").length).to.equal(0); + + flexsearch_depth.add(1, "1 2 3 1 4 2 5 1"); + + expect(flexsearch_depth.search("1")).to.include(1); + expect(flexsearch_depth.search("1 5")).to.include(1); + expect(flexsearch_depth.search("2 4 1")).to.include(1); + }); +}); + +// ------------------------------------------------------------------------ +// Encoding Tests +// ------------------------------------------------------------------------ + +describe('Options', function(){ + + it('Should have been added properly to the index: Strict', function(){ + + flexsearch_strict.add(0, "björn phillipp mayer"); + + expect(flexsearch_strict.length).to.equal(1); + expect(flexsearch_strict.search("björn phillipp")).to.include(0); + expect(flexsearch_strict.search("björn mayer")).to.include(0); + }); + + it('Should have been added properly to the index: Forward', function(){ + + flexsearch_forward.add(0, "björn phillipp mayer"); + + expect(flexsearch_forward.length).to.equal(1); + expect(flexsearch_forward.search("bjö phil may")).to.have.lengthOf(1); + expect(flexsearch_forward.search("bjö phil may")).to.include(0); + }); + + it('Should have been added properly to the index: Inverse', function(){ + + flexsearch_reverse.add(0, "björn phillipp mayer"); + + expect(flexsearch_reverse.length).to.equal(1); + expect(flexsearch_reverse.search("jörn phil yer")).to.have.lengthOf(1); + expect(flexsearch_reverse.search("jörn phil yer")).to.include(0); + }); + + it('Should have been added properly to the index: Full', function(){ + + flexsearch_full.add(0, "björn phillipp mayer"); + + expect(flexsearch_full.length).to.equal(1); + expect(flexsearch_full.search("jör illi may")).to.have.lengthOf(1); + expect(flexsearch_full.search("jör illi may")).to.include(0); + }); + + it('Should have been added properly to the index: Full', function(){ + + flexsearch_ngram.add(0, "björn-phillipp mayer"); + + expect(flexsearch_ngram.length).to.equal(1); + expect(flexsearch_ngram.search("mayer")).to.have.lengthOf(1); + expect(flexsearch_ngram.search("philip meier")).to.have.lengthOf(1); + expect(flexsearch_ngram.search("philip meier")).to.include(0); + expect(flexsearch_ngram.search("björn meier")).to.have.lengthOf(1); + expect(flexsearch_ngram.search("björn meier")).to.include(0); + expect(flexsearch_ngram.search("björn-peter mayer")).to.have.lengthOf(0); + }); +}); + +// ------------------------------------------------------------------------ +// Feature Tests +// ------------------------------------------------------------------------ + +describe('Add Matchers', function(){ + + it('Should have been added properly', function(){ + + FlexSearch.addMatcher({ + + '1': 'a', + '2': 'b', + '3': 'c', + '[456]': 'd' + }); + + flexsearch_forward.init({ + + encode: false + + }).init({ + + encode: 'not-found', + matcher: { + + '7': 'e' + } + + }).addMatcher({ + + '8': 'f' + }); + + flexsearch_forward.add(0, "12345678"); + + expect(flexsearch_forward.search("12345678")).to.include(0); + expect(flexsearch_forward.search("abcd")).to.include(0); + expect(flexsearch_forward.encode("12345678")).to.equal("abcdddef"); + }); +}); + +// ------------------------------------------------------------------------ +// Caching +// ------------------------------------------------------------------------ + +describe('Caching', function(){ + + it('Should have been cached properly', function(){ + + flexsearch_cache.add(0, 'foo') + .add(1, 'bar') + .add(2, 'foobar'); + // fetch: + + expect(flexsearch_cache.search("foo")).to.have.members([0, 2]); + expect(flexsearch_cache.search("bar")).to.have.members([1, 2]); + expect(flexsearch_cache.search("foobar")).to.include(2); + + // cache: + + expect(flexsearch_cache.search("foo")).to.have.members([0, 2]); + expect(flexsearch_cache.search("bar")).to.have.members([1, 2]); + expect(flexsearch_cache.search("foobar")).to.include(2); + + // update: + + flexsearch_cache.remove(2).update(1, 'foo').add(3, 'foobar'); + + // fetch: + + expect(flexsearch_cache.search("foo")).to.have.members([0, 1, 3]); + expect(flexsearch_cache.search("bar")).to.include(3); + expect(flexsearch_cache.search("foobar")).to.include(3); + + // cache: + + expect(flexsearch_cache.search("foo")).to.have.members([0, 1, 3]); + expect(flexsearch_cache.search("bar")).to.include(3); + expect(flexsearch_cache.search("foobar")).to.include(3); + }); +}); + +// ------------------------------------------------------------------------ +// Debug Information +// ------------------------------------------------------------------------ + +describe('Debug', function(){ + + it('Should have been debug mode activated', function(){ + + var info = flexsearch_cache.info(); + + expect(info).to.have.keys([ + + 'id', + 'chars', + 'status', + 'cache', + 'items', + 'matcher', + 'memory', + 'sequences', + 'worker' + ]); + }); +}); + +// ------------------------------------------------------------------------ +// Chaining +// ------------------------------------------------------------------------ + +describe('Chaining', function(){ + + it('Should have been chained properly', function(){ + + var index = FlexSearch.create({mode: 'forward', encode: 'icase'}) + .addMatcher({'â': 'a'}) + .add(0, 'foo') + .add(1, 'bar'); + + expect(index.search("foo")).to.include(0); + expect(index.search("bar")).to.include(1); + expect(index.encode("bâr")).to.equal("bar"); + + index.remove(0).update(1, 'foo').add(2, 'foobâr'); + + expect(index.search("foo")).to.have.members([1, 2]); + expect(index.search("bar")).to.have.lengthOf(0); + expect(index.search("foobar")).to.include(2); + + index.reset().add(0, 'foo').add(1, 'bar'); + + expect(index.search("foo")).to.include(0); + expect(index.search("bar")).to.include(1); + expect(index.search("foobar")).to.have.lengthOf(0); + + flexsearch_cache.destroy().init().add(0, 'foo').add(1, 'bar'); + + expect(flexsearch_cache.search("foo")).to.include(0); + expect(flexsearch_cache.search("bar")).to.include(1); + expect(flexsearch_cache.search("foobar")).to.have.lengthOf(0); + }); +}); + +/* Test Helpers */ + +function test_encoder(str){ + + return str = '-[' + str.toUpperCase() + ']-'; +}