diff --git a/README.md b/README.md index 0fb3b95..34d77d6 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@


- Search Library + Search Library

@@ -18,6 +18,8 @@ When it comes to raw search speed BulkSearch may be a better choice. FlexSearch also provides you a non-blocking asynchronous processing model as well as web workers to perform any updates on the index as well as queries through dedicated threads. +Installation Guide  •  API Reference  •  Example Options  •  Custom Builds + Comparison: - Library Benchmarks - BulkSearch vs. FlexSearch Benchmark @@ -40,14 +42,27 @@ All Features:

  • Multiple Words
  • Phonetic Search
  • Relevance-based Scoring
    • -
  • Contextual Indexes
    • +
  • Contextual Indexes
  • Limit Results
    • -
  • Caching
    • -
  • Asynchronous Mode
    • -
  • Custom Matchers
    • -
  • Custom Encoders
    • +
  • Supports Caching
    • +
  • Asynchronous Processing
    • +
  • Customizable: Matcher, Encoder, Tokenizer, Stemmer, Stopword-Filter
    • +This features are not available in the 50% smaller light version: + +- WebWorker +- Async handler +- Cache handler +- Built-in encoders except 'balanced' (you can still pass in customs) +- Built-in stemmers and stopword filter (you can still pass in customs) +- Debug logging +- _index.info()_ method + +The light version is just available as compiled version (flexsearch.light.js). + +> You can also make Custom Builds pretty simple + #### Contextual Search @@ -57,7 +72,7 @@ Imagine you add a text block of some sentences to an index ID. Assuming the quer In this way contextual search also improves the results of relevance-based queries on large amount of text data.

    - +

    __Note:__ This feature is actually not enabled by default. @@ -93,7 +108,7 @@ __Note:__ It is slightly faster to use no web worker when the index or query isn Usecase - + @@ -121,6 +136,7 @@ __Note:__ It is slightly faster to use no web worker when the index or query isn + ## Installation ##### HTML / Javascript @@ -163,6 +179,7 @@ __AMD__ var FlexSearch = require("./flexsearch.js"); ``` + ## API Overview Global methods: @@ -381,7 +398,21 @@ FlexSearch.register('extended', function(str){ return str; }); ``` + +#### Add custom tokenizer +Define a private custom tokenizer during creation/initialization: +```js +var index = new FlexSearch({ + + mode: function(str){ + + // split string into components, e.g.: + + return str.split(/ -\//g); + } +}); +``` #### Get info @@ -473,21 +504,23 @@ FlexSearch ist highly customizable. Make use of the the right options can really "foward"
    "reverse"
    "ngram"
    - "full" + "full"
    + function() - The indexing mode (tokenizer).
    + The indexing mode (tokenizer).

    Choose one of the built-ins or pass a custom tokenizer function.
    - encode





    + encode






    false
    "icase"
    "simple"
    "advanced"
    "extra"
    + "balanced"
    function() The encoding type.

    Choose one of the built-ins or pass a custom encoding function. @@ -574,7 +607,7 @@ Tokenizer effects the required memory also as query time and flexibility of part "ngram" (default) index words partially through phonetic n-grams foobar
    foobar - * n / 3.5 + * n / 3 @@ -766,7 +799,7 @@ Encoding effects the required memory also as query time and phonetic matches. Tr - + ## Memory Usage The required memory for the index depends on several options: @@ -812,7 +845,7 @@ The required memory for the index depends on several options: "ngram" (default) - * n / 3.5 + * n / 3 @@ -839,6 +872,7 @@ The required memory for the index depends on several options: + ## Example Options Memory-optimized: @@ -871,18 +905,51 @@ Matching-tolerant: } ``` -Balanced: +Well-balanced: ```js { - encode: "simple", + encode: "extra", mode: "ngram", - threshold: 3, + threshold: 4, depth: 3 } ``` + +## Custom Builds + +Default Build: +```bash +npm run build +``` + +Light Build: +```bash +npm run build-light +``` + +Custom Build: +```bash +npm run build-custom SUPPORT_WORKER=true SUPPORT_ASYNC=true +``` + +Supported flags: + +- SUPPORT_DEBUG +- SUPPORT_WORKER +- SUPPORT_CACHE +- SUPPORT_ASYNC +- SUPPORT_BUILTINS (english stemmer and filter) + +Alternatively you can also use: +```bash +node compile SUPPORT_WORKER=true +``` + +The custom build was saved to flexsearch.custom.js + --- -Author FlexSearch: Thomas Wilkerling
    -License: Apache 2.0 License
    +Copyright 2017-2018 Thomas Wilkerling
    +Released under the Apache 2.0 License
    diff --git a/compile.js b/compile.js new file mode 100644 index 0000000..3d4c2c3 --- /dev/null +++ b/compile.js @@ -0,0 +1,88 @@ +var child_process = require('child_process'); + +console.log("Start build ....."); + +var parameter = (function(opt){ + + var parameter = ''; + + for(var index in opt){ + + if(opt.hasOwnProperty(index)){ + + parameter += ' --' + index + '=' + opt[index]; + } + } + + console.log(parameter); + + return parameter; +})({ + + compilation_level: "ADVANCED_OPTIMIZATIONS", + use_types_for_optimization: true, + new_type_inf: true, + jscomp_warning: "newCheckTypes", + generate_exports: true, + export_local_property_definitions: true, + language_in: "ECMASCRIPT5_STRICT", + language_out: "ECMASCRIPT5_STRICT", + process_closure_primitives: true, + summary_detail_level: 3, + warning_level: "VERBOSE", + emit_use_strict: true, + output_manifest: "log/manifest.log", + output_module_dependencies: "log/module_dependencies.log", + property_renaming_report: "log/renaming_report.log" +}); + +var options = (function(argv){ + + var arr = {}; + var count = 0; + + argv.forEach(function(val, index) { + + if(++count > 2){ + + index = val.split('='); + val = index[1]; + index = index[0]; + + arr[index] = val; + + if(count > 3) console.log(index + ': ' + val); + } + }); + + console.log('RELEASE: ' + (arr['RELEASE'] || 'custom')); + + return arr; + +})(process.argv); + +exec("java -jar node_modules/google-closure-compiler/compiler.jar" + parameter + " --define='SUPPORT_DEBUG=" + (options['SUPPORT_DEBUG'] || 'false') + "' --define='SUPPORT_WORKER=" + (options['SUPPORT_WORKER'] || 'false') + "' --define='SUPPORT_BUILTINS=" + (options['SUPPORT_BUILTINS'] || 'false') + "' --define='SUPPORT_CACHE=" + (options['SUPPORT_CACHE'] || 'false') + "' --define='SUPPORT_ASYNC=" + (options['SUPPORT_ASYNC'] || 'false') + "' --js='flexsearch.js' --js_output_file='flexsearch." + (options['RELEASE'] || 'custom') + ".js' && exit 0", function(){ + + console.log("Build Complete: flexsearch." + (options['RELEASE'] || 'custom') + ".js"); +}); + +function exec(prompt, callback){ + + var child = child_process.exec(prompt, function(err, stdout, stderr){ + + if(err){ + + console.log(err); + } + else{ + + if(callback){ + + callback(); + } + } + }); + + child.stdout.pipe(process.stdout); + child.stderr.pipe(process.stderr); +} diff --git a/contextual_index.svg b/doc/contextual_index.svg similarity index 100% rename from contextual_index.svg rename to doc/contextual_index.svg diff --git a/flexsearch.svg b/doc/flexsearch.svg similarity index 100% rename from flexsearch.svg rename to doc/flexsearch.svg diff --git a/flexsearch.js b/flexsearch.js index 40ed1e8..f58bac2 100644 --- a/flexsearch.js +++ b/flexsearch.js @@ -1,12 +1,25 @@ ;/**! - * FlexSearch - Superfast lightweight full text search engine - * ---------------------------------------------------------- - * @author: Thomas Wilkerling - * @preserve https://github.com/nextapps-de/flexsearch - * @version: 0.2.2 - * @license: Apache 2.0 Licence + * @preserve FlexSearch v0.2.3 + * Copyright 2017-2018 Thomas Wilkerling + * Released under the Apache 2.0 Licence + * https://github.com/nextapps-de/flexsearch */ +/** @define {boolean} */ +var SUPPORT_WORKER = true; + +/** @define {boolean} */ +var SUPPORT_BUILTINS = true; + +/** @define {boolean} */ +var SUPPORT_DEBUG = true; + +/** @define {boolean} */ +var SUPPORT_CACHE = true; + +/** @define {boolean} */ +var SUPPORT_ASYNC = true; + (function(){ provide('FlexSearch', (function factory(register_worker){ @@ -76,6 +89,276 @@ /** @const {RegExp} */ var regex_split = regex("[ -\/]"); + /** + * http://www.ranks.nl/stopwords + * @const {Array} + */ + + var filter = SUPPORT_BUILTINS ? [ + + "a", + "about", + "above", + "after", + "again", + "against", + "all", + "also", + "am", + "an", + "and", + "any", + "are", + "aren't", + "as", + "at", + //"back", + "be", + "because", + "been", + "before", + "being", + "below", + //"between", + "both", + "but", + "by", + "can", + "cannot", + "can't", + "come", + "could", + "couldn't", + //"day", + "did", + "didn't", + "do", + "does", + "doesn't", + "doing", + "dont", + "down", + "during", + "each", + "even", + "few", + "first", + "for", + "from", + "further", + "get", + //"give", + "go", + //"good", + "had", + "hadn't", + "has", + "hasn't", + "have", + "haven't", + "having", + "he", + "hed", + //"hell", + "her", + "here", + "here's", + "hers", + "herself", + "hes", + "him", + "himself", + "his", + "how", + "how's", + "i", + "id", + "if", + "ill", + "im", + "in", + "into", + "is", + "isn't", + "it", + "it's", + "itself", + "i've", + "just", + "know", + "let's", + "like", + //"look", + "make", + "me", + "more", + "most", + "mustn't", + "my", + "myself", + "new", + "no", + "nor", + "not", + "now", + "of", + "off", + "on", + "once", + //"one", + "only", + "or", + "other", + "ought", + "our", + "our's", + "ourselves", + "out", + "over", + "own", + //"people", + "same", + "say", + "see", + "shan't", + "she", + "she'd", + "shell", + "shes", + "should", + "shouldn't", + "so", + "some", + "such", + //"take", + "than", + "that", + "that's", + "the", + "their", + "theirs", + "them", + "themselves", + "then", + "there", + "there's", + "these", + "they", + "they'd", + "they'll", + "they're", + "they've", + //"think", + "this", + "those", + "through", + "time", + "to", + "too", + //"two", + //"under", + "until", + "up", + "us", + //"use", + "very", + "want", + "was", + "wasn't", + "way", + "we", + "wed", + "well", + "were", + "weren't", + "we've", + "what", + "what's", + "when", + "when's", + "where", + "where's", + "which", + "while", + "who", + "whom", + "who's", + "why", + "why's", + "will", + "with", + "won't", + //"work", + "would", + "wouldn't", + //"year", + "you", + "you'd", + "you'll", + "your", + "you're", + "your's", + "yourself", + "yourselves", + "you've" + + ] : null; + + /** + * @const {Object} + */ + + var stemmer = SUPPORT_BUILTINS ? { + + "ational": "ate", + "tional": "tion", + "enci": "ence", + "anci": "ance", + "izer": "ize", + "bli": "ble", + "alli": "al", + "entli": "ent", + "eli": "e", + "ousli": "ous", + "ization": "ize", + "ation": "ate", + "ator": "ate", + "alism": "al", + "iveness": "ive", + "fulness": "ful", + "ousness": "ous", + "aliti": "al", + "iviti": "ive", + "biliti": "ble", + "logi": "log", + "icate": "ic", + "ative": "", + "alize": "al", + "iciti": "ic", + "ical": "ic", + "ful": "", + "ness": "", + "al": "", + "ance": "", + "ence": "", + "er": "", + "ic": "", + "able": "", + "ible": "", + "ant": "", + "ement": "", + "ment": "", + "ent": "", + "ou": "", + "ism": "", + "ate": "", + "iti": "", + "ous": "", + "ive": "", + "ize": "" + + } : null; + /** * @param {Object=} options * @constructor @@ -97,28 +380,14 @@ // define functional properties - Object.defineProperty(this, 'index', { + registerProperty(this, 'index', /** @this {FlexSearch} */ function(){ - /** - * @this {FlexSearch} - */ - - get: function(){ - - return this._ids; - } + return this._ids; }); - Object.defineProperty(this, 'length', { + registerProperty(this, 'length', /** @this {FlexSearch} */ function(){ - /** - * @this {FlexSearch} - */ - - get: function(){ - - return Object.keys(this._ids).length; - } + return Object.keys(this._ids).length; }); } @@ -200,12 +469,16 @@ // initialize worker - if(options['worker']){ + if(SUPPORT_WORKER && options['worker']){ if(typeof Worker === 'undefined'){ options['worker'] = false; - options['async'] = true; + + if(SUPPORT_ASYNC){ + + options['async'] = true; + } this._worker = null; } @@ -272,21 +545,21 @@ defaults.mode ); - this.cache = ( + if(SUPPORT_CACHE) this.cache = ( options['cache'] || this.cache || defaults.cache ); - this.async = ( + if(SUPPORT_ASYNC) this.async = ( options['async'] || this.async || defaults.async ); - this.worker = ( + if(SUPPORT_WORKER) this.worker = ( options['worker'] || this.worker || @@ -307,27 +580,51 @@ defaults.depth ); - /** @export */ this.encoder = ( (options['encode'] && global_encoder[options['encode']]) || (typeof options['encode'] === 'function' ? options['encode'] : this.encoder || false) - //(defaults.encode && global_encoder[defaults.encode]) || ); - //if(DEBUG){ + if(SUPPORT_DEBUG){ this.debug = ( options['debug'] || this.debug ); - //} + } if(options['matcher']) { this.addMatcher(/** @type {Object} */ (options['matcher'])); } + + if(options['filter']) { + + this.filter = initFilter( + + (options['filter'] === true ? + + filter + : + /** @type {Array} */ (options['filter']) + + ), this.encoder); + } + + if(options['stemmer']) { + + this.stemmer = initStemmer( + + (options['stemmer'] === true ? + + stemmer + : + /** @type {Object} */ (options['stemmer']) + + ), this.encoder); + } } // initialize index @@ -368,7 +665,8 @@ this._timer = null; this._last_empty_query = ""; this._status = true; - this._cache = this.cache ? + + if(SUPPORT_CACHE) this._cache = this.cache ? (new cache(30 * 1000, 50, true)) : @@ -400,6 +698,37 @@ value = this.encoder.call(global_encoder, value); } + if(value && this.filter){ + + var words = value.split(' '); + var final = ""; + + for(var i = 0; i < words.length; i++){ + + var word = words[i]; + + if(this.filter[word]){ + + //var length = word.length - 1; + + // TODO completely filter out words actually breaks the context chain + words[i] = this.filter[word]; + //words[i] = word[0] + (length ? word[1] : ''); + //words[i] = '~' + word[0]; + //words.splice(i, 1); + //i--; + //final += (final ? ' ' : '') + word; + } + } + + value = words.join(' '); // final; + } + + if(value && this.stemmer){ + + value = replace(value, this.stemmer); + } + return value; }; @@ -441,7 +770,7 @@ } else{ - if(this.worker){ + if(SUPPORT_WORKER && this.worker){ if(++this._current_task >= this._worker.length) this._current_task = 0; @@ -457,7 +786,7 @@ return this; } - if(this.async){ + if(SUPPORT_ASYNC && this.async){ this._stack[id] || ( @@ -483,13 +812,20 @@ return this; } + var tokenizer = this.mode; + var words = ( - this.mode === 'ngram' ? + typeof tokenizer === 'function' ? - /** @type {!Array} */ (ngram(content)) - : - /** @type {string} */ (content).split(regex_split) + tokenizer(content) + :( + tokenizer === 'ngram' ? + + /** @type {!Array} */ (ngram(content)) + : + /** @type {string} */ (content).split(regex_split) + ) ); var dupes = { @@ -513,7 +849,7 @@ var length = value.length; - switch(this.mode){ + switch(tokenizer){ case 'reverse': case 'both': @@ -663,7 +999,7 @@ if(this._ids[id]){ - if(this.worker){ + if(SUPPORT_WORKER && this.worker){ var int = parseInt(this._ids[id], 10); @@ -677,7 +1013,7 @@ return this; } - if(this.async){ + if(SUPPORT_ASYNC && this.async){ this._stack[id] || ( @@ -717,7 +1053,7 @@ if(this._ids[id]){ - if(this.worker){ + if(SUPPORT_WORKER && this.worker){ var int = parseInt(this._ids[id], 10); @@ -732,7 +1068,7 @@ return this; } - if(this.async){ + if(SUPPORT_ASYNC && this.async){ this._stack[id] || ( @@ -823,7 +1159,7 @@ limit || (limit = 1000); } - if(this.worker){ + if(SUPPORT_WORKER && this.worker){ this._current_callback = callback; this._task_completed = 0; @@ -870,7 +1206,7 @@ if(!this._status){ - if(this.cache){ + if(SUPPORT_CACHE && this.cache){ this._last_empty_query = ""; this._cache.reset(); @@ -881,7 +1217,7 @@ // validate cache - else if(this.cache){ + else if(SUPPORT_CACHE && this.cache){ var cache = this._cache.get(query); @@ -909,13 +1245,20 @@ // convert words into single components + var tokenizer = this.mode; + var words = ( - _query.constructor === Array ? + typeof tokenizer === 'function' ? - /** @type {!Array} */ (_query) - : - /** @type {string} */ (_query).split(regex_split) + tokenizer(_query) + :( + tokenizer === 'ngram' ? + + /** @type {!Array} */ (ngram(_query)) + : + /** @type {string} */ (_query).split(regex_split) + ) ); var length = words.length; @@ -936,7 +1279,7 @@ // Note: sort words by length only in non-contextual mode - words.sort(sort_by_length_down); + words.sort(sortByLengthDown); } } @@ -1020,7 +1363,7 @@ // store result to cache - if(this.cache){ + if(SUPPORT_CACHE && this.cache){ this._cache.set(query, result); } @@ -1028,67 +1371,70 @@ return result; }; - /** - * @export - */ + if(SUPPORT_DEBUG){ - FlexSearch.prototype.info = function(){ + /** + * @export + */ - if(this.worker){ + FlexSearch.prototype.info = function(){ - for(var i = 0; i < this.worker; i++) this._worker[i].postMessage(i, { + if(SUPPORT_WORKER && this.worker){ - 'info': true, - 'id': this.id - }); + for(var i = 0; i < this.worker; i++) this._worker[i].postMessage(i, { - return; - } + 'info': true, + 'id': this.id + }); - var keys; - var length; - - var bytes = 0, - words = 0, - chars = 0; - - for(var z = 0; z < 10; z++){ - - keys = Object.keys(this._map[z]); - - for(var i = 0; i < keys.length; i++){ - - length = this._map[z][keys[i]].length; - - // Note: 1 char values allocates 1 byte "Map (OneByteInternalizedString)" - bytes += length * 1 + keys[i].length * 2 + 4; - words += length; - chars += keys[i].length * 2; + return; } - } - keys = Object.keys(this._ids); + var keys; + var length; - var items = keys.length; + var bytes = 0, + words = 0, + chars = 0; - for(var i = 0; i < items; i++){ + for(var z = 0; z < 10; z++){ - bytes += keys[i].length * 2 + 2; - } + keys = Object.keys(this._map[z]); - return { + for(var i = 0; i < keys.length; i++){ - 'id': this.id, - 'memory': bytes, - 'items': items, - 'sequences': words, - 'chars': chars, - 'status': this._status, - 'cache': this._stack_keys.length, - 'matcher': global_matcher.length, - 'worker': this.worker + length = this._map[z][keys[i]].length; + + // Note: 1 char values allocates 1 byte "Map (OneByteInternalizedString)" + bytes += length * 1 + keys[i].length * 2 + 4; + words += length; + chars += keys[i].length * 2; + } + } + + keys = Object.keys(this._ids); + + var items = keys.length; + + for(var i = 0; i < items; i++){ + + bytes += keys[i].length * 2 + 2; + } + + return { + + 'id': this.id, + 'memory': bytes, + 'items': items, + 'sequences': words, + 'chars': chars, + 'status': this._status, + 'cache': this._stack_keys.length, + 'matcher': global_matcher.length, + 'worker': this.worker + }; }; - }; + } /** * @export @@ -1113,7 +1459,7 @@ // cleanup cache - if(this.cache){ + if(SUPPORT_CACHE && this.cache){ this._cache.reset(); } @@ -1128,16 +1474,39 @@ return this; }; + /** @const */ + + var global_encoder_balanced = (function(){ + + var regex_whitespace = regex('\\s\\s+'), + regex_strip = regex('[^a-z0-9 ]'), + regex_space = regex('[-\/]'), + regex_vowel = regex('[aeiouy]'); + + /** @const {Array} */ + var regex_pairs = [ + + regex_space, ' ', + regex_strip, '', + regex_whitespace, ' ', + regex_vowel, '' + ]; + + return function(value){ + + return collapseRepeatingChars(replace(value.toLowerCase(), regex_pairs)); + } + })(); + /** * Phonetic Encoders - * @dict - * @enum {Function} + * @dict {Function} * @private * @const * @final */ - var global_encoder = { + var global_encoder = SUPPORT_BUILTINS ? { // case insensitive search @@ -1152,7 +1521,7 @@ var regex_whitespace = regex('\\s\\s+'), regex_strip = regex('[^a-z0-9 ]'), - regex_split = regex('[-\/]'), + regex_space = regex('[-\/]'), regex_a = regex('[àáâãäå]'), regex_e = regex('[èéêë]'), regex_i = regex('[ìíîï]'), @@ -1161,7 +1530,8 @@ regex_y = regex('[ýŷÿ]'), regex_n = regex('ñ'), regex_c = regex('ç'), - regex_s = regex('ß'); + regex_s = regex('ß'), + regex_and = regex(' & '); /** @const {Array} */ var regex_pairs = [ @@ -1175,7 +1545,8 @@ regex_n, 'n', regex_c, 'c', regex_s, 's', - regex_split, ' ', + regex_and, ' and ', + regex_space, ' ', regex_strip, '', regex_whitespace, ' ' ]; @@ -1283,7 +1654,7 @@ soundex_f = regex('[vw]'); /** @const {RegExp} */ - var regex_vowel = regex('[aeiouy]'); // [aeiouy] + var regex_vowel = regex('[aeiouy]'); /** @const {Array} */ var regex_pairs = [ @@ -1319,7 +1690,7 @@ if(current.length > 1){ // remove all vowels after 2nd char - str[i] = current[0] + replace(current.substring(1), regex_pairs) + str[i] = current[0] + replace(current.substring(1), regex_pairs); } } @@ -1329,7 +1700,9 @@ return str; }; - })() + })(), + + 'balanced': global_encoder_balanced // TODO: provide some common encoder plugins // soundex @@ -1339,11 +1712,22 @@ // levinshtein // hamming // matchrating + + } : { + + // case insensitive search + + 'icase': function(value){ + + return value.toLowerCase(); + }, + + 'balanced': global_encoder_balanced }; // Xone Async Handler Fallback - var queue = (function(){ + var queue = SUPPORT_ASYNC ? (function(){ var stack = {}; @@ -1361,11 +1745,12 @@ stack[id] = setTimeout(fn, delay) ); }; - })(); + + })() : null; // Xone Flexi-Cache Handler Fallback - var cache = (function(){ + var cache = SUPPORT_CACHE ? (function(){ /** @this {Cache} */ function Cache(){ @@ -1392,13 +1777,24 @@ }; return Cache; - })(); + + })() : null; return FlexSearch; // --------------------------------------------------------- // Helpers + function registerProperty(obj, key, fn){ + + // define functional properties + + Object.defineProperty(obj, key, { + + get: fn + }); + } + /** * @param {!string} str * @returns {RegExp} @@ -1628,13 +2024,65 @@ return collapsed_string; } + /** + * @param {Array} words + * @param encoder + * @returns {Object} + */ + + function initFilter(words, encoder){ + + var final = {}; + + if(stemmer){ + + for(var i = 0; i < words.length; i++){ + + var word = encoder ? encoder.call(global_encoder, words[i]) : words[i]; + + final[word] = String.fromCharCode((65000 - words.length) + i); + } + } + + return final; + } + + /** + * @param {Object} stemmer + * @param encoder + * @returns {Array} + */ + + function initStemmer(stemmer, encoder){ + + var final = []; + + if(stemmer){ + + var count = 0; + + for(var key in stemmer){ + + if(stemmer.hasOwnProperty(key)){ + + var tmp = encoder ? encoder.call(global_encoder, key) : key; + + final[count++] = regex('(?=.{' + (tmp.length + 3) + ',})' + tmp + '$'); + final[count++] = encoder ? encoder.call(global_encoder, stemmer[key]) : stemmer[key]; + } + } + } + + return final; + } + /** * @param {string} a * @param {string} b * @returns {number} */ - function sort_by_length_down(a, b){ + function sortByLengthDown(a, b){ var diff = a.length - b.length; @@ -1659,7 +2107,7 @@ * @returns {number} */ - function sort_by_length_up(a, b){ + function sortByLengthUp(a, b){ var diff = a.length - b.length; @@ -1679,7 +2127,6 @@ } /** - * Fastest intersect method for a set of unsorted arrays so far * @param {!Array>} arrays * @param {number=} limit * @returns {Array} @@ -1694,7 +2141,7 @@ // pre-sort arrays by length up - arrays.sort(sort_by_length_up); + arrays.sort(sortByLengthUp); // fill initial map @@ -2023,13 +2470,13 @@ callback(data['id'], data['content'], data['result'], data['limit']); } - //else{ + else{ - //if(DEBUG && options['debug']){ + if(SUPPORT_DEBUG && options['debug']){ - //console.log(data); - //} - //} + console.log(data); + } + } }, // cores: @@ -2052,7 +2499,7 @@ })( // Xone Worker Handler Fallback - (function register_worker(){ + SUPPORT_WORKER ? (function register_worker(){ var worker_stack = {}; var inline_is_supported = !!((typeof Blob !== 'undefined') && (typeof URL !== 'undefined') && URL.createObjectURL); @@ -2100,10 +2547,10 @@ worker_stack[name][_core] = new Worker(worker_payload); worker_stack[name][_core]['onmessage'] = _callback; - //if(DEBUG){ + if(SUPPORT_DEBUG){ - //console.log('Register Worker: ' + name + '@' + _core); - //} + console.log('Register Worker: ' + name + '@' + _core); + } return { @@ -2114,7 +2561,7 @@ }; } ); - })() + })() : false ), this); @@ -2158,5 +2605,3 @@ } }).call(this); - -// --define='DEBUG=false' diff --git a/flexsearch.light.js b/flexsearch.light.js new file mode 100644 index 0000000..df6b35d --- /dev/null +++ b/flexsearch.light.js @@ -0,0 +1,18 @@ +/* + FlexSearch v0.2.3 + Copyright 2017-2018 Thomas Wilkerling + Released under the Apache 2.0 Licence + https://github.com/nextapps-de/flexsearch +*/ +'use strict';(function(h,t,l){var q;(q=l.define)&&q.amd?q([],function(){return t}):(q=l.modules)?q[h.toLowerCase()]=t:"undefined"!==typeof module?module.exports=t:l[h]=t})("FlexSearch",function(){function h(a){a||(a=y);this.id=a.id||C++;this.init(a);t(this,"index",function(){return this.a});t(this,"length",function(){return Object.keys(this.a).length})}function t(a,b,c){Object.defineProperty(a,b,{get:c})}function l(a){return new RegExp(a,"g")}function q(a,b,c){if("undefined"===typeof c){for(c=0;c< +b.length;c+=2)a=a.replace(b[c],b[c+1]);return a}return a.replace(b,c)}function w(a,b,c,g,d,e){if("undefined"===typeof b[c]){var f=d.indexOf(c);f=3/d.length*(d.length-f)+6/(f-d.lastIndexOf(" ",f))+.5|0;b[c]=f;f>e&&(a=a[f],a=a[c]||(a[c]=[]),a[a.length]=g)}return f||b[c]}function A(a){var b=[];if(!a)return b;for(var c=0,g=0,d=0,e="",f=a.length,p=0;pa?1:0a?-1:0b&&(c=c.slice(0,b)));return c}var y= +{type:"integer",mode:"forward",cache:!1,async:!1,l:!1,threshold:0,depth:0,encode:"icase"},x=[],C=0,B=l("[ -/]");h.new=function(a){return new this(a)};h.create=function(a){return h.new(a)};h.addMatcher=function(a){for(var b in a)a.hasOwnProperty(b)&&(x[x.length]=l(b),x[x.length]=a[b]);return this};h.register=function(a,b){v[a]=b;return this};h.encode=function(a,b){return v[a].call(v,b)};h.prototype.init=function(a){this.c=[];if(a&&(this.mode=a.mode||this.mode||y.mode,this.threshold=a.threshold||this.threshold|| +y.threshold,this.depth=a.depth||this.depth||y.depth,this.h=a.encode&&v[a.encode]||("function"===typeof a.encode?a.encode:this.h||!1),a.matcher&&this.addMatcher(a.matcher),a.filter&&(this.i={}),a.stemmer)){a=!0===a.stemmer?null:a.stemmer;var b=this.h,c=[];if(a){var g=0,d;for(d in a)if(a.hasOwnProperty(d)){var e=b?b.call(v,d):d;c[g++]=l("(?=.{"+(e.length+3)+",})"+e+"$");c[g++]=b?b.call(v,a[d]):a[d]}}this.j=c}this.b=[{},{},{},{},{},{},{},{},{},{},{}];this.a={};this.f="";this.g=!0;return this};h.prototype.encode= +function(a){a&&x.length&&(a=q(a,x));a&&this.c.length&&(a=q(a,this.c));a&&this.h&&(a=this.h.call(v,a));if(a&&this.i){a=a.split(" ");for(var b=0;bk;l--)u=n.substring(k,l),w(p,d,u,a,b,e);break;default:if(k=w(p,d,n,a,b,e),f&&1e)for(k=p[10], +r=d._ctx[n]||(d._ctx[n]={}),n=k[n]||(k[n]=[{},{},{},{},{},{},{},{},{},{}]),k=h-f,l=h+f,0>k&&(k=0),l>m-1&&(l=m-1);k<=l;k++)k!==h&&w(n,r,g[k],a,b,e)}}}this.a[a]="1";this.g=!1}return this};h.prototype.update=function(a,b){"string"===typeof b&&(a||0===a)&&this.a[a]&&(this.remove(a),b&&this.add(a,b));return this};h.prototype.remove=function(a){if(this.a[a]){for(var b=0;10>b;b++)for(var c=Object.keys(this.b[b]),g=0;g=d;z--)if(v=(r?k[u]:this.b)[z][t])w[y++]=v,x=!0;if(x)l[l.length]=1k&&(a=a[f],a=a[c]||(a[c]=[]),a[a.length]=h)}return f||b[c]}function x(a){for(var b="",c="",h="",n=0;na?1:0a?-1:0b&&(c=c.slice(0,b)));return c}function A(a){a.w|| -(a.w=E(function(){a.w=null;var b=a.async;b&&(a.async=!1);if(a.c.length){for(var c=F(),h;(h=a.c.shift())||0===h;){var d=a.h[h];switch(d[0]){case z.add:a.add(d[1],d[2]);break;case z.update:a.update(d[1],d[2]);break;case z.remove:a.remove(d[1])}a.h[h]=null;delete a.h[h];if(100=d&&(b.o=b.b),b.v&&b.o===b.b&&(b.i.length?b.f="":b.f||(b.f=c),b.cache&&b.l.set(c,b.i),b.v(b.i), -b.i=[]))})}this.mode=a.mode||this.mode||u.mode;this.cache=a.cache||this.cache||u.cache;this.async=a.async||this.async||u.async;this.b=a.worker||this.b||u.b;this.threshold=a.threshold||this.threshold||u.threshold;this.depth=a.depth||this.depth||u.depth;this.encoder=a.encode&&B[a.encode]||("function"===typeof a.encode?a.encode:this.encoder||!1);this.A=a.debug||this.A;a.matcher&&this.addMatcher(a.matcher)}this.g=[{},{},{},{},{},{},{},{},{},{},{}];this.a={};this.h={};this.c=[];this.w=null;this.f="";this.u= -!0;this.l=this.cache?new M(3E4,50,!0):!1;return this};e.prototype.encode=function(a){a&&w.length&&(a=v(a,w));a&&this.m.length&&(a=v(a,this.m));a&&this.encoder&&(a=this.encoder.call(B,a));return a};e.prototype.addMatcher=function(a){for(var b in a)a.hasOwnProperty(b)&&(this.m[this.m.length]=d(b),this.m[this.m.length]=a[b]);return this};e.prototype.add=function(a,b){if("string"===typeof b&&b&&(a||0===a))if(this.a[a])this.update(a,b);else{if(this.b)return++this.s>=this.j.length&&(this.s=0),this.j[this.s].postMessage(this.s, -{add:!0,id:a,content:b}),this.a[a]=""+this.s,this;if(this.async)return this.h[a]||(this.c[this.c.length]=a),this.h[a]=[z.add,a,b],A(this),this;b=this.encode(b);if(!b.length)return this;if("ngram"===this.mode){var c=b;var h=[];if(c)for(var d=0,k=0,f=0,e="",r=c.length,m=0;mg;l--)t=m.substring(g,l),q(f,h,t,a,b,d);break;default:if(g=q(f,h,m,a,b,d),k&&1d)for(g=f[10],p=h._ctx[m]||(h._ctx[m]={}),m=g[m]||(g[m]=[{},{},{},{},{},{},{},{},{},{}]),g=r-k,l=r+k,0>g&&(g=0), -l>e-1&&(l=e-1);g<=l;g++)g!==r&&q(m,p,c[g],a,b,d)}}this.a[a]="1";this.u=!1}return this};e.prototype.update=function(a,b){if("string"===typeof b&&(a||0===a)&&this.a[a]){if(this.b){var c=parseInt(this.a[a],10);this.j[c].postMessage(c,{update:!0,id:a,content:b});return this}if(this.async)return this.h[a]||(this.c[this.c.length]=a),this.h[a]=[z.update,a,b],A(this),this;this.remove(a);b&&this.add(a,b)}return this};e.prototype.remove=function(a){if(this.a[a]){if(this.b){var b=parseInt(this.a[a],10);this.j[b].postMessage(b, -{remove:!0,id:a});delete this.a[a];return this}if(this.async)return this.h[a]||(this.c[this.c.length]=a),this.h[a]=[z.remove,a],A(this),this;for(b=0;10>b;b++)for(var c=Object.keys(this.g[b]),d=0;d=e;A--)if(w=(p?u[t]:this.g)[A][q])x[B++]=w,z=!0;if(z)m[m.length]=1f;f++)for(b=Object.keys(this.g[f]),a=0;ag&&(a=a[k],a=a[c]||(a[c]=[]),a[a.length]=e)}return k||b[c]}function x(a){var b=[];if(!a)return b;for(var c=0,e=0,h=0,g="",k=a.length,B=0;Ba?1:0a?-1:0b&&(c=c.slice(0,b)));return c}function E(a){a.B||(a.B=H(function(){a.B=null;var b=a.async;b&&(a.async=!1);if(a.c.length){for(var c=I(),e;(e=a.c.shift())||0===e;){var d=a.h[e];switch(d[0]){case D.add:a.add(d[1],d[2]);break;case D.update:a.update(d[1],d[2]);break;case D.remove:a.remove(d[1])}a.h[e]=null;delete a.h[e];if(100=d&&(b.o=b.b),b.w&&b.o===b.b&&(b.i.length?b.f="":b.f||(b.f=c),b.cache&&b.l.set(c,b.i),b.w(b.i),b.i=[]))})}this.mode=a.mode||this.mode||u.mode;this.cache=a.cache||this.cache||u.cache;this.async=a.async||this.async||u.async;this.b=a.worker||this.b||u.b;this.threshold=a.threshold||this.threshold||u.threshold;this.depth=a.depth||this.depth||u.depth;this.v=a.encode&&y[a.encode]||("function"===typeof a.encode?a.encode:this.v||!1);this.C=a.debug||this.C;a.matcher&&this.addMatcher(a.matcher); +a.filter&&(this.A=M(!0===a.filter?S:a.filter,this.v));a.stemmer&&(this.D=N(!0===a.stemmer?G:a.stemmer,this.v))}this.g=[{},{},{},{},{},{},{},{},{},{},{}];this.a={};this.h={};this.c=[];this.B=null;this.f="";this.u=!0;this.l=this.cache?new T(3E4,50,!0):!1;return this};f.prototype.encode=function(a){a&&z.length&&(a=v(a,z));a&&this.m.length&&(a=v(a,this.m));a&&this.v&&(a=this.v.call(y,a));if(a&&this.A){a=a.split(" ");for(var b=0;b=this.j.length&&(this.s=0),this.j[this.s].postMessage(this.s,{add:!0,id:a,content:b}),this.a[a]=""+this.s,this;if(this.async)return this.h[a]||(this.c[this.c.length]=a),this.h[a]=[D.add,a,b],E(this),this;b=this.encode(b); +if(!b.length)return this;for(var c=this.mode,e="function"===typeof c?c(b):"ngram"===c?x(b):b.split(J),d={_ctx:{}},g=this.threshold,k=this.depth,f=this.g,n=e.length,q=0;ql;A--)t=m.substring(l,A),w(f,d,t,a,b,g);break;default:if(l=w(f,d,m,a,b,g),k&&1g)for(l=f[10], +r=d._ctx[m]||(d._ctx[m]={}),m=l[m]||(l[m]=[{},{},{},{},{},{},{},{},{},{}]),l=q-k,A=q+k,0>l&&(l=0),A>n-1&&(A=n-1);l<=A;l++)l!==q&&w(m,r,e[l],a,b,g)}}}this.a[a]="1";this.u=!1}return this};f.prototype.update=function(a,b){if("string"===typeof b&&(a||0===a)&&this.a[a]){if(this.b){var c=parseInt(this.a[a],10);this.j[c].postMessage(c,{update:!0,id:a,content:b});return this}if(this.async)return this.h[a]||(this.c[this.c.length]=a),this.h[a]=[D.update,a,b],E(this),this;this.remove(a);b&&this.add(a,b)}return this}; +f.prototype.remove=function(a){if(this.a[a]){if(this.b){var b=parseInt(this.a[a],10);this.j[b].postMessage(b,{remove:!0,id:a});delete this.a[a];return this}if(this.async)return this.h[a]||(this.c[this.c.length]=a),this.h[a]=[D.remove,a],E(this),this;for(b=0;10>b;b++)for(var c=Object.keys(this.g[b]),d=0;d=h;z--)if(v=(r?l[t]:this.g)[z][p])u[y++]=v,w=!0;if(w)q[q.length]=1k;k++)for(b=Object.keys(this.g[k]),a=0;a Matching Test - + @@ -68,7 +68,7 @@ var text_data = "LIBRARY OF THE FUTURE (R) First Edition Ver. 4.02 Gulliver's Tr wait ... - "i already observed" + "princes of the ambition" wait ... wait ... wait ... @@ -80,7 +80,7 @@ var text_data = "LIBRARY OF THE FUTURE (R) First Edition Ver. 4.02 Gulliver's Tr wait ... - "let a of his" + "five-thousand leagues" wait ... wait ... wait ... @@ -92,7 +92,7 @@ var text_data = "LIBRARY OF THE FUTURE (R) First Edition Ver. 4.02 Gulliver's Tr wait ... - "take that to the rocks" + "i already observed" wait ... wait ... wait ... @@ -104,7 +104,7 @@ var text_data = "LIBRARY OF THE FUTURE (R) First Edition Ver. 4.02 Gulliver's Tr wait ... - "bignes of splaknuk" + "let a of his" wait ... wait ... wait ... @@ -116,7 +116,7 @@ var text_data = "LIBRARY OF THE FUTURE (R) First Edition Ver. 4.02 Gulliver's Tr wait ... - "matematikal musikal instruments" + "take that to the rocks" wait ... wait ... wait ... @@ -128,6 +128,30 @@ var text_data = "LIBRARY OF THE FUTURE (R) First Edition Ver. 4.02 Gulliver's Tr wait ... + "bignes of splaknuk" + wait ... + wait ... + wait ... + wait ... + wait ... + wait ... + wait ... + wait ... + wait ... + + + "matematikal musikal instruments" + wait ... + wait ... + wait ... + wait ... + wait ... + wait ... + wait ... + wait ... + wait ... + + "lalkon the camberlayhn" wait ... wait ... @@ -197,6 +221,8 @@ var text_data = "LIBRARY OF THE FUTURE (R) First Edition Ver. 4.02 Gulliver's Tr mode: 'strict', threshold: 4, depth: 3, + filter: true, + stemmer: true, async: false, cache: false, worker: false @@ -297,12 +323,14 @@ var text_data = "LIBRARY OF THE FUTURE (R) First Edition Ver. 4.02 Gulliver's Tr do_test('test-1', 'without breach of modesty', '493'); do_test('test-2', 'went softly stream', '446'); - do_test('test-3', 'i already observed', '458'); - do_test('test-4', 'let a of his', '50'); - do_test('test-5', 'take that to the rocks', '175'); - do_test('test-6', 'bignes of splaknuk', '146'); - do_test('test-7', 'matematikal musikal instruments', '267'); - do_test('test-8', 'lalkon the camberlayhn', '99'); + do_test('test-3', 'princes of the ambition', '72'); + do_test('test-4', 'five-thousand leagues', '2'); + do_test('test-5', 'i already observed', '458'); + do_test('test-6', 'let a of his', '50'); + do_test('test-7', 'take that to the rocks', '175'); + do_test('test-8', 'bignes of splaknuk', '146'); + do_test('test-9', 'matematikal musikal instruments', '267'); + do_test('test-10', 'lalkon the camberlayhn', '99'); // --------------------------------------- diff --git a/test/test.js b/test/test.js index ac45656..3e0ada6 100644 --- a/test/test.js +++ b/test/test.js @@ -6,9 +6,9 @@ if(typeof module !== 'undefined'){ URL.createObjectURL = function(val){}; Blob = function(string){}; - var env = process.argv[3] === 'test' ? '.min' : ''; + var env = process.argv[3] === 'test' ? 'min' : process.argv[3] === 'test/' ? 'light' : ''; var expect = require('chai').expect; - var FlexSearch = require("../flexsearch" + env + ".js"); + var FlexSearch = require("../flexsearch" + (env ? '.' + env : '') + ".js"); } var flexsearch_default; @@ -148,7 +148,11 @@ describe('Initialize', function(){ expect(flexsearch_default).to.respondTo("remove"); expect(flexsearch_default).to.respondTo("reset"); expect(flexsearch_default).to.respondTo("init"); - expect(flexsearch_default).to.respondTo("info"); + + if(env !== 'light'){ + + expect(flexsearch_default).to.respondTo("info"); + } }); it('Should have correct uuids', function(){ @@ -168,12 +172,17 @@ describe('Initialize', function(){ expect(flexsearch_default.mode).to.equal("forward"); expect(flexsearch_sync.async).to.equal(false); expect(flexsearch_async.async).to.equal(true); - expect(flexsearch_custom.encoder).to.equal(test_encoder); expect(flexsearch_strict.mode).to.equal("strict"); expect(flexsearch_forward.mode).to.equal("forward"); expect(flexsearch_reverse.mode).to.equal("reverse"); expect(flexsearch_full.mode).to.equal("full"); expect(flexsearch_ngram.mode).to.equal("ngram"); + + // not available in compiled version: + if(typeof flexsearch_custom.encoder !== 'undefined'){ + + expect(flexsearch_custom.encoder).to.equal(test_encoder); + } }); }); @@ -223,13 +232,16 @@ describe('Search (Sync)', function(){ expect(flexsearch_sync.search("foo foo")).to.have.members([0, 1]); expect(flexsearch_sync.search("foo foo")).to.have.members([0, 1]); - flexsearch_extra.add(4, "Thomas"); - flexsearch_extra.add(5, "Arithmetic"); - flexsearch_extra.add(6, "Mahagoni"); + if(env !== 'light'){ - expect(flexsearch_extra.search("tomass")).to.include(4); - expect(flexsearch_extra.search("arytmetik")).to.include(5); - expect(flexsearch_extra.search("mahagony")).to.include(6); + flexsearch_extra.add(4, "Thomas"); + flexsearch_extra.add(5, "Arithmetic"); + flexsearch_extra.add(6, "Mahagoni"); + + expect(flexsearch_extra.search("tomass")).to.include(4); + expect(flexsearch_extra.search("arytmetik")).to.include(5); + expect(flexsearch_extra.search("mahagony")).to.include(6); + } }); it('Should have been limited', function(){ @@ -843,6 +855,7 @@ describe('Context', function(){ expect(flexsearch_depth.search("zero three").length).to.equal(0); expect(flexsearch_depth.search("three seven").length).to.equal(0); expect(flexsearch_depth.search("three five seven")).to.include(0); + expect(flexsearch_depth.search("eight six four")).to.include(0); // TODO // expect(flexsearch_depth.search("three seven five")).to.include(0); expect(flexsearch_depth.search("three foobar seven").length).to.equal(0); @@ -961,13 +974,14 @@ describe('Relevance', function(){ index.add(0, "1 2 3 2 4 1 5 3"); index.add(1, "zero one two three four five six seven eight nine ten"); - index.add(2, "four two zero one three ten five seven eight six nine"); + index.add(2, "five two zero one three four ten seven eight six nine"); expect(index.search("1 3 4")).to.have.members([0]); expect(index.search("1 5 3 4")).to.have.members([0]); expect(index.search("one")).to.have.members([1, 2]); - expect(index.search("one two")).to.have.members([1, 2]); - expect(index.search("four one")).to.have.members([1, 2]); + expect(index.search("one three")).to.have.members([1, 2]); + expect(index.search("three one")).to.have.members([1, 2]); + expect(index.search("zero five one ten")).to.have.members([2]); }); }); @@ -1016,67 +1030,73 @@ describe('Add Matchers', function(){ // Caching // ------------------------------------------------------------------------ -describe('Caching', function(){ +if(env !== 'light'){ - it('Should have been cached properly', function(){ + describe('Caching', function(){ - flexsearch_cache.add(0, 'foo') - .add(1, 'bar') - .add(2, 'foobar'); - // fetch: + it('Should have been cached properly', function(){ - expect(flexsearch_cache.search("foo")).to.have.members([0, 2]); - expect(flexsearch_cache.search("bar")).to.have.members([1, 2]); - expect(flexsearch_cache.search("foobar")).to.include(2); + flexsearch_cache.add(0, 'foo') + .add(1, 'bar') + .add(2, 'foobar'); + // fetch: - // cache: + expect(flexsearch_cache.search("foo")).to.have.members([0, 2]); + expect(flexsearch_cache.search("bar")).to.have.members([1, 2]); + expect(flexsearch_cache.search("foobar")).to.include(2); - expect(flexsearch_cache.search("foo")).to.have.members([0, 2]); - expect(flexsearch_cache.search("bar")).to.have.members([1, 2]); - expect(flexsearch_cache.search("foobar")).to.include(2); + // cache: - // update: + expect(flexsearch_cache.search("foo")).to.have.members([0, 2]); + expect(flexsearch_cache.search("bar")).to.have.members([1, 2]); + expect(flexsearch_cache.search("foobar")).to.include(2); - flexsearch_cache.remove(2).update(1, 'foo').add(3, 'foobar'); + // update: - // fetch: + flexsearch_cache.remove(2).update(1, 'foo').add(3, 'foobar'); - expect(flexsearch_cache.search("foo")).to.have.members([0, 1, 3]); - expect(flexsearch_cache.search("bar")).to.include(3); - expect(flexsearch_cache.search("foobar")).to.include(3); + // fetch: - // cache: + expect(flexsearch_cache.search("foo")).to.have.members([0, 1, 3]); + expect(flexsearch_cache.search("bar")).to.include(3); + expect(flexsearch_cache.search("foobar")).to.include(3); - expect(flexsearch_cache.search("foo")).to.have.members([0, 1, 3]); - expect(flexsearch_cache.search("bar")).to.include(3); - expect(flexsearch_cache.search("foobar")).to.include(3); + // cache: + + expect(flexsearch_cache.search("foo")).to.have.members([0, 1, 3]); + expect(flexsearch_cache.search("bar")).to.include(3); + expect(flexsearch_cache.search("foobar")).to.include(3); + }); }); -}); +} // ------------------------------------------------------------------------ // Debug Information // ------------------------------------------------------------------------ -describe('Debug', function(){ +if(env !== 'light'){ - it('Should have been debug mode activated', function(){ + describe('Debug', function(){ - var info = flexsearch_cache.info(); + it('Should have been debug mode activated', function(){ - expect(info).to.have.keys([ + var info = flexsearch_cache.info(); - 'id', - 'chars', - 'status', - 'cache', - 'items', - 'matcher', - 'memory', - 'sequences', - 'worker' - ]); + expect(info).to.have.keys([ + + 'id', + 'chars', + 'status', + 'cache', + 'items', + 'matcher', + 'memory', + 'sequences', + 'worker' + ]); + }); }); -}); +} // ------------------------------------------------------------------------ // Chaining