From aae24b5e3bed00ea9cd605c91c9250980aa5e88f Mon Sep 17 00:00:00 2001 From: Thomas Wilkerling Date: Tue, 27 Mar 2018 19:16:15 +0200 Subject: [PATCH] ADD suggestions --- README.md | 41 ++++++++---- doc/contextual-index.svg | 2 +- flexsearch.js | 130 +++++++++++++++++++++++++++++---------- flexsearch.light.js | 29 ++++----- flexsearch.min.js | 51 +++++++-------- package.json | 2 +- test/matching.html | 87 +++++++++++++++++++------- test/test.js | 35 +++++++++-- 8 files changed, 267 insertions(+), 110 deletions(-) diff --git a/README.md b/README.md index 495569a..53176a1 100644 --- a/README.md +++ b/README.md @@ -7,17 +7,14 @@ - - +

World's fastest and most memory efficient full text search library with zero dependencies.

+ When it comes to raw search speed FlexSearch outperforms every single searching library out there and also provides flexible search capabilities like multi-word matching, phonetic transformations or partial matching. -It also has the most memory-efficient index. - +It also has the most memory-efficient index. FlexSearch also provides you a non-blocking asynchronous processing model as well as web workers to perform any updates or queries on the index in parallel through dedicated balanced threads. Installation Guide  •  API Reference  •  Example Options  •  Custom Builds @@ -201,13 +198,15 @@ var FlexSearch = require("./flexsearch.js"); Global methods: - FlexSearch.__create__(\) -- FlexSearch.__registerMatcher__({_KEY: VALUE_}) -- FlexSearch.__registerEncoder__(name, encoder) +- FlexSearch.__registerMatcher__({_KEY: VALUE_}) +- FlexSearch.__registerEncoder__(name, encoder) +- FlexSearch.__registerLanguage__(lang, {stemmer:{}, filter:[]}) - FlexSearch.__encode__(name, string) Index methods: - Index.__add__(id, string) - Index.__search__(string, \, \) +- Index.__search__(\) - Index.__update__(id, string) - Index.__remove__(id) - Index.__reset__() @@ -296,20 +295,38 @@ index.search({ query: "John", limit: 1000, threshold: 5, // >= initial threshold - depth: 3 // <= initial depth + depth: 3, // <= initial depth + callback: function(results){/* ... */} }); ``` -Get suggestions of a query: +The same from above could also be written as: + +```js +index.search("John", { + + limit: 1000, + threshold: 5, + depth: 3 + +}, function(results){ + + // .... +}); +``` + +Get also suggestions for a query: ```js index.search({ - query: "John", + query: "John Doe", suggest: true }); ``` +When suggestion is enabled all results will be filled up (until limit, default 1000) with similar matches ordered by relevance. + #### Update item to the index @@ -461,7 +478,7 @@ var index = new FlexSearch({ } }); ``` - + #### Add language-specific stemmer and/or filter > __Stemmer:__ several linguistic mutations of the same word (e.g. "run" and "running") diff --git a/doc/contextual-index.svg b/doc/contextual-index.svg index 5cd4406..cae6e16 100644 --- a/doc/contextual-index.svg +++ b/doc/contextual-index.svg @@ -1,2 +1,2 @@ -
Word 4
Word 4
Contextual Index
[Not supported by viewer]
Context of Relevance
Context of Relevance
Query
Query
2. contextual score
[Not supported by viewer]
1. partial score
1. partial score
context limit (depth)
[Not supported by viewer]
3. document score
[Not supported by viewer]
"Model of Contextual-based Scoring"
Copyright 2018 Thomas Wilkerling
Nextapps GmbH
[Not supported by viewer]
Word 2
Word 2
Word 3
Word 3
Word 5
Word 5
Word 6
Word 6
Word 1
Word 1
Word 7
Word 7
...
...
...
... \ No newline at end of file +
Word 4
Word 4
Contextual Index
[Not supported by viewer]
Context of Relevance
Context of Relevance
Query
Query
2. contextual score
[Not supported by viewer]
1. partial score
1. partial score
context limit (depth)
[Not supported by viewer]
3. document score
[Not supported by viewer]
"Model of Contextual-based Scoring"
Copyright 2018 Thomas Wilkerling
Nextapps GmbH
[Not supported by viewer]
Word 2
Word 2
Word 3
Word 3
Word 5
Word 5
Word 6
Word 6
Word 1
Word 1
Word 7
Word 7
...
...
...
... diff --git a/flexsearch.js b/flexsearch.js index 6f22804..9ba89bc 100644 --- a/flexsearch.js +++ b/flexsearch.js @@ -1,5 +1,5 @@ ;/**! - * @preserve FlexSearch v0.2.46 + * @preserve FlexSearch v0.2.48 * Copyright 2018 Thomas Wilkerling * Released under the Apache 2.0 Licence * https://github.com/nextapps-de/flexsearch @@ -37,6 +37,7 @@ var SUPPORT_ASYNC = true; encode: 'icase', mode: 'ngram', + suggest: false, cache: false, async: false, worker: false, @@ -315,13 +316,16 @@ var SUPPORT_ASYNC = true; if(self._current_callback && (self._task_completed === self.worker)){ - if(self._task_result.length){ + if(typeof self._last_empty_query !== 'undefined'){ - self._last_empty_query = ""; - } - else{ + if(self._task_result.length){ - self._last_empty_query || (self._last_empty_query = query); + self._last_empty_query = ""; + } + else{ + + self._last_empty_query || (self._last_empty_query = query); + } } // store result to cache @@ -379,6 +383,13 @@ var SUPPORT_ASYNC = true; defaults.depth ); + this.suggest = ( + + options['suggest'] || + this.suggest || + defaults.suggest + ); + custom = options['encode'] || profile.encode; this.encoder = ( @@ -434,9 +445,13 @@ var SUPPORT_ASYNC = true; */ this._timer = null; - this._last_empty_query = ""; this._status = true; + if(this.mode === 'forward' || this.mode === 'reverse' || this.mode === 'both'){ + + this._last_empty_query = ""; + } + if(SUPPORT_CACHE) { this.cache = custom = ( @@ -479,6 +494,8 @@ var SUPPORT_ASYNC = true; value = this.encoder.call(global_encoder, value); } + // TODO completely filter out words actually can break the context chain + /* if(value && this.filter){ var words = value.split(' '); @@ -493,7 +510,6 @@ var SUPPORT_ASYNC = true; //var length = word.length - 1; - // TODO completely filter out words actually breaks the context chain words[i] = filter; //words[i] = word[0] + (length ? word[1] : ''); //words[i] = '~' + word[0]; @@ -505,6 +521,7 @@ var SUPPORT_ASYNC = true; value = words.join(' '); // final; } + */ if(value && this.stemmer){ @@ -934,7 +951,11 @@ var SUPPORT_ASYNC = true; if(SUPPORT_CACHE && this.cache){ - this._last_empty_query = ""; + if(typeof this._last_empty_query !== 'undefined'){ + + this._last_empty_query = ""; + } + this._cache.reset(); } @@ -955,7 +976,7 @@ var SUPPORT_ASYNC = true; // validate last query - else if(this._last_empty_query && (query.indexOf(this._last_empty_query) === 0)){ + else if((typeof this._last_empty_query !== 'undefined') && this._last_empty_query && (query.indexOf(this._last_empty_query) === 0)){ return result; } @@ -1047,10 +1068,13 @@ var SUPPORT_ASYNC = true; if(!map_found){ - found = false; - break; + if(!this.suggest){ + + found = false; + break; + } } - else{ + else { // Not handled by intersection: @@ -1083,20 +1107,23 @@ var SUPPORT_ASYNC = true; // Not handled by intersection: - result = intersect(check, limit); + result = intersect(check, limit, this.suggest); // Handled by intersection: - //result = intersect_3d(check, limit); + //result = intersect_3d(check, limit, this.suggest); } - if(result.length){ + if(typeof this._last_empty_query !== 'undefined'){ - this._last_empty_query = ""; - } - else{ + if(result.length){ - this._last_empty_query || (this._last_empty_query = query); + this._last_empty_query = ""; + } + else{ + + this._last_empty_query || (this._last_empty_query = query); + } } // store result to cache @@ -2007,12 +2034,14 @@ var SUPPORT_ASYNC = true; /** * @param {!Array>} arrays * @param {number=} limit + * @param {boolean=} suggest * @returns {Array} */ - function intersect(arrays, limit) { + function intersect(arrays, limit, suggest) { var result = []; + var suggestions = []; var length_z = arrays.length; if(length_z > 1){ @@ -2043,25 +2072,28 @@ var SUPPORT_ASYNC = true; // get each array one by one var found = false; + var is_final_loop = (z === (length_z - 1)); + suggestions = []; arr = arrays[z]; length = arr.length; - i = 0; + i = -1; while(i < length){ - if((check[tmp = arr[i++]]) === z){ + var check_val = check[tmp = arr[++i]]; + + if(check_val === z){ // fill in during last round - if(z === (length_z - 1)){ + if(is_final_loop){ result[count++] = tmp; if(limit && (count === limit)){ - found = false; - break; + return result; } } @@ -2070,15 +2102,49 @@ var SUPPORT_ASYNC = true; found = true; check[tmp] = z + 1; } + else if(suggest){ + + var current_suggestion = suggestions[check_val] || (suggestions[check_val] = []); + + current_suggestion[current_suggestion.length] = tmp; + } } - if(!found){ + if(!found && !suggest){ break; } z++; } + + if(suggest){ + + limit || (limit = 1000); + count = result.length; + length = suggestions.length; + + if((count < limit) && length){ + + for(z = length - 1; z >= 0; z--){ + + tmp = suggestions[z]; + + if(tmp){ + + for(i = 0; i < tmp.length; i++){ + + result[count++] = tmp[i]; + + if(limit && (count === limit)){ + + return result; + } + } + } + } + } + } } else if(length_z){ @@ -2088,7 +2154,7 @@ var SUPPORT_ASYNC = true; // Note: do not modify the original index array! - return result.slice(0, limit); + result = result.slice(0, limit); } // Note: handle references to the original index array @@ -2285,10 +2351,10 @@ var SUPPORT_ASYNC = true; ref.add(current[1], current[2]); break; - case enum_task.update: - - ref.update(current[1], current[2]); - break; + // case enum_task.update: + // + // ref.update(current[1], current[2]); + // break; case enum_task.remove: diff --git a/flexsearch.light.js b/flexsearch.light.js index 1290389..ad0dee2 100644 --- a/flexsearch.light.js +++ b/flexsearch.light.js @@ -1,19 +1,20 @@ /* - FlexSearch v0.2.46 + FlexSearch v0.2.48 Copyright 2018 Thomas Wilkerling Released under the Apache 2.0 Licence https://github.com/nextapps-de/flexsearch */ -'use strict';(function(d,w,p){var q;(q=p.define)&&q.amd?q([],function(){return w}):(q=p.modules)?q[d.toLowerCase()]=w:"undefined"!==typeof module?module.exports=w:p[d]=w})("FlexSearch",function(){function d(a){"string"===typeof a&&(a=B[a]);a||(a=z);this.id=a.id||G++;this.init(a);w(this,"index",function(){return this.a});w(this,"length",function(){return Object.keys(this.a).length})}function w(a,b,c){Object.defineProperty(a,b,{get:c})}function p(a){return new RegExp(a,"g")}function q(a,b,c){if("undefined"=== -typeof c){for(c=0;c=e&&(a=a[h+.5|0],a=a[c]||(a[c]=[]),a[a.length]=f)}return h||b[c]}function A(a,b){if(a)for(var c=Object.keys(a),f=0,g=c.length;f=(8=(8a?1:0a?-1:0b))return c.slice(0,b);return c}var z={encode:"icase",mode:"ngram",cache:!1,async:!1,j:!1,threshold:0,depth:0},B={memory:{encode:"extra",mode:"strict",threshold:7},speed:{encode:"icase",mode:"strict",threshold:7,depth:2},match:{encode:"extra",mode:"full"},score:{encode:"extra",mode:"strict",threshold:5,depth:4},balance:{encode:"balance", -mode:"ngram",threshold:6,depth:3},fastest:{encode:"icase",mode:"strict",threshold:9,depth:1}},x=[],G=0,D=p("[ -/]"),E={},F={};d.new=function(a){return new this(a)};d.create=function(a){return d.new(a)};d.registerMatcher=function(a){for(var b in a)a.hasOwnProperty(b)&&(x[x.length]=p(b),x[x.length]=a[b]);return this};d.registerEncoder=function(a,b){v[a]=b;return this};d.registerLanguage=function(a,b){E[a]=b.filter;F[a]=b.stemmer;return this};d.encode=function(a,b){return v[a].call(v,b)};d.prototype.init= -function(a){this.i=[];a||(a=z);var b=a.profile;b=b?B[b]:{};this.mode=a.mode||b.mode||this.mode||z.mode;this.threshold=a.threshold||b.threshold||this.threshold||z.threshold;this.depth=a.depth||b.depth||this.depth||z.depth;this.f=(b=a.encode||b.encode)&&v[b]||("function"===typeof b?b:this.f||!1);(b=a.matcher)&&this.addMatcher(b);if(b=a.filter){b=E[b]||b;var c=this.f,f={};if(b)for(var g=0;gl;t--)r=d.substring(l,t),y(e,f,r,a,p,u,g);break;default:if(l=y(e,f,d,a,1,u,g),m&&1=g)for(n=f._ctx[d]||(f._ctx[d]={}),d=this.b[d]||(this.b[d]=[{},{},{},{},{},{},{},{},{},{}]),l=k-m,t=k+m+1,0>l&&(l=0),t>h&&(t=h);lb;b++)A(this.g[b],a);this.depth&&A(this.b,a);delete this.a[a];this.h=!1}return this};d.prototype.search=function(a,b,c){var f=[];if(a&&"object"===typeof a){c=a.callback||b;b=a.limit;var g=a.threshold;a=a.query}g=(g||this.threshold||0)|0;"function"===typeof b?(c=b,b=1E3):b||(b=1E3);if(c){var d=this; -K(function(){c(d.search(a,b));d=null},1,"search-"+this.id);return null}if(!a||"string"!==typeof a)return f;if(!this.h)this.h=!0;else if(this.c&&0===a.indexOf(this.c))return f;var e=this.encode(a);if(!e.length)return f;var h=this.mode;e="function"===typeof h?h(e):"ngram"===h?C(e):e.split(D);h=e.length;var k=!0,p=[],n={};if(1=g;y--)if(v= -(u?l[r]:this.g)[y][t])x[z++]=v,w=!0;if(w)p[p.length]=1=g&&(a=a[h+.5|0],a=a[c]||(a[c]=[]),a[a.length]=d)}return h||b[c]}function A(a,b){if(a)for(var c=Object.keys(a),d=0,f=c.length;d=(8=(8a?1:0a?-1:0b&&(d=d.slice(0,b)));return d}var x={encode:"icase",mode:"ngram",g:!1,cache:!1,async:!1,l:!1,threshold:0,depth:0},B={memory:{encode:"extra", +mode:"strict",threshold:7},speed:{encode:"icase",mode:"strict",threshold:7,depth:2},match:{encode:"extra",mode:"full"},score:{encode:"extra",mode:"strict",threshold:5,depth:4},balance:{encode:"balance",mode:"ngram",threshold:6,depth:3},fastest:{encode:"icase",mode:"strict",threshold:9,depth:1}},w=[],G=0,D=n("[ -/]"),E={},F={};e.new=function(a){return new this(a)};e.create=function(a){return e.new(a)};e.registerMatcher=function(a){for(var b in a)a.hasOwnProperty(b)&&(w[w.length]=n(b),w[w.length]=a[b]); +return this};e.registerEncoder=function(a,b){u[a]=b;return this};e.registerLanguage=function(a,b){E[a]=b.filter;F[a]=b.stemmer;return this};e.encode=function(a,b){return u[a].call(u,b)};e.prototype.init=function(a){this.j=[];a||(a=x);var b=a.profile;b=b?B[b]:{};this.mode=a.mode||b.mode||this.mode||x.mode;this.threshold=a.threshold||b.threshold||this.threshold||x.threshold;this.depth=a.depth||b.depth||this.depth||x.depth;this.g=a.suggest||this.g||x.g;this.f=(b=a.encode||b.encode)&&u[b]||("function"=== +typeof b?b:this.f||!1);(b=a.matcher)&&this.addMatcher(b);if(b=a.filter){b=E[b]||b;var c=this.f,d={};if(b)for(var f=0;fl;r--)m=e.substring(l,r),z(g,d,m,a,t,n,f);break;default:if(l=z(g,d,e,a,1,n,f),p&&1=f)for(q= +d._ctx[e]||(d._ctx[e]={}),e=this.c[e]||(this.c[e]=[{},{},{},{},{},{},{},{},{},{}]),l=k-p,r=k+p+1,0>l&&(l=0),r>h&&(r=h);lb;b++)A(this.h[b],a);this.depth&&A(this.c,a);delete this.a[a];this.i=!1}return this};e.prototype.search=function(a,b,c){var d= +[];if(a&&"object"===typeof a){c=a.callback||b;b=a.limit;var f=a.threshold;a=a.query}f=(f||this.threshold||0)|0;"function"===typeof b?(c=b,b=1E3):b||(b=1E3);if(c){var e=this;K(function(){c(e.search(a,b));e=null},1,"search-"+this.id);return null}if(!a||"string"!==typeof a)return d;if(!this.i)this.i=!0;else if("undefined"!==typeof this.b&&this.b&&0===a.indexOf(this.b))return d;var g=this.encode(a);if(!g.length)return d;var h=this.mode;g="function"===typeof h?h(g):"ngram"===h?C(g):g.split(D);h=g.length; +var k=!0,n=[],q={};if(1=f;A--)if(v=(t?l[m]:this.h)[A][r])w[z++]=v,x=!0;if(x)n[n.length]=1=k&&(a=a[h+.5|0],a=a[d]||(a[d]=[]),a[a.length]=c)}return h||b[d]}function n(a,b){if(a)for(var d=Object.keys(a),c=0,f=d.length;c=(8=(8a?1:0a?-1:0b))return d.slice(0,b);return d}function E(a){a.C||(a.C=H(function(){a.C=null;var b=a.async;b&&(a.async=!1);if(a.f.length){for(var d=I(),c;(c=a.f.shift())||0===c;){var f=a.h[c];switch(f[0]){case C.add:a.add(f[1],f[2]);break;case C.update:a.update(f[1], -f[2]);break;case C.remove:a.remove(f[1])}a.h[c]=null;delete a.h[c];if(100=f&&(c.m=c.b),c.w&&c.m===c.b&&(c.g.length?c.c="":c.c||(c.c=b),c.cache&&c.j.set(b,c.g),c.w(c.g),c.g=[]))})}this.mode=a.mode||d.mode||this.mode||x.mode;this.async=a.async||this.async||x.async;this.b=a.worker||this.b||x.b;this.threshold=a.threshold||d.threshold||this.threshold||x.threshold;this.depth=a.depth||d.depth||this.depth||x.depth;this.v= -(b=a.encode||d.encode)&&z[b]||("function"===typeof b?b:this.v||!1);this.D=a.debug||this.D;(b=a.matcher)&&this.addMatcher(b);if(b=a.filter)this.filter=O(K[b]||b,this.v);if(b=a.stemmer)this.stemmer=P(L[b]||b,this.v);this.l=[{},{},{},{},{},{},{},{},{},{}];this.o={};this.a={};this.h={};this.f=[];this.C=null;this.c="";this.u=!0;this.j=(this.cache=b=a.cache||this.cache||x.cache)?new U(b):!1;return this};g.prototype.encode=function(a){a&&A.length&&(a=y(a,A));a&&this.B.length&&(a=y(a,this.B));a&&this.v&& -(a=this.v.call(z,a));if(a&&this.filter){a=a.split(" ");for(var b=0;b=this.i.length&&(this.s=0),this.i[this.s].postMessage(this.s, -{add:!0,id:a,content:b}),this.a[a]=""+this.s,this;if(this.async)return this.h[a]||(this.f[this.f.length]=a),this.h[a]=[C.add,a,b],E(this),this;b=this.encode(b);if(!b.length)return this;d=this.mode;b="function"===typeof d?d(b):"ngram"===d?G(b):b.split(J);for(var c={_ctx:{}},f=this.threshold,e=this.depth,k=this.l,h=b.length,g=0;gl;w--)v=r.substring(l,w),u(k,c,v,a,q,n,f);break;default:if(l=u(k,c,r,a,1,n,f),e&&1=f)for(p=c._ctx[r]||(c._ctx[r]={}),r=this.o[r]||(this.o[r]=[{},{},{},{},{},{},{},{},{},{}]),l=g-e,w=g+e+1,0>l&&(l=0),w>h&&(w=h);lb;b++)n(this.l[b],a);this.depth&&n(this.o,a);delete this.a[a];this.u=!1}return this};g.prototype.search=function(a,b,d){var c=[];if(a&&"object"===typeof a){d=a.callback||b;b=a.limit;var f=a.threshold;a=a.query}f=(f||this.threshold||0)|0;"function"=== -typeof b?(d=b,b=1E3):b||(b=1E3);if(this.b){this.w=d;this.m=0;this.g=[];for(c=0;c=f;A--)if(x=(q?l[v]:this.l)[A][w])u[z++]=x,y=!0;if(y)n[n.length]=1g;g++)for(b=Object.keys(this.l[g]),a=0;ag;c--)h=k[c-1],k[c]=h,e[h]=c;k[g]=a;e[a]=g}}}return b};return a}();return g}(function(){var q={},B=!("undefined"===typeof Blob||"undefined"===typeof URL||!URL.createObjectURL);return function(g,t,e,y,u){var n=g;g=B?URL.createObjectURL(new Blob(["var SUPPORT_WORKER = true;var SUPPORT_BUILTINS = true;var SUPPORT_DEBUG = true;var SUPPORT_CACHE = true;var SUPPORT_ASYNC = true;("+ -e.toString()+")()"],{type:"text/javascript"})):"../"+n+".js";n+="-"+t;q[n]||(q[n]=[]);q[n][u]=new Worker(g);q[n][u].onmessage=y;console.log("Register Worker: "+n+"@"+u);return{postMessage:function(e,g){q[n][e].postMessage(g)}}}}()),this); +'use strict';(function(v,C,f){var w;(w=f.define)&&w.amd?w([],function(){return C}):(w=f.modules)?w[v.toLowerCase()]=C:"undefined"!==typeof module?module.exports=C:f[v]=C})("FlexSearch",function N(v){function f(a){"string"===typeof a&&(a=G[a]);a||(a=x);this.id=a.id||O++;this.init(a);w(this,"index",function(){return this.b});w(this,"length",function(){return Object.keys(this.b).length})}function w(a,b,e){Object.defineProperty(a,b,{get:e})}function d(a){return new RegExp(a,"g")}function y(a,b,e){if("undefined"=== +typeof e){for(e=0;e=g&&(a=a[h+.5|0],a=a[e]||(a[e]=[]),a[a.length]=c)}return h||b[e]}function q(a,b){if(a)for(var e=Object.keys(a),c=0,k=e.length;c=(8=(8a?1:0a?-1:0b&&(c=c.slice(0,b)));return c}function F(a){a.D|| +(a.D=I(function(){a.D=null;var b=a.async;b&&(a.async=!1);if(a.f.length){for(var e=J(),c;(c=a.f.shift())||0===c;){var d=a.h[c];switch(d[0]){case D.add:a.add(d[1],d[2]);break;case D.remove:a.remove(d[1])}a.h[c]=null;delete a.h[c];if(100=d&&(c.m=c.c),c.A&&c.m===c.c&&("undefined"!==typeof c.a&&(c.g.length?c.a="":c.a||(c.a=b)),c.cache&&c.j.set(b,c.g),c.A(c.g),c.g=[]))})}this.mode=a.mode||e.mode||this.mode||x.mode; +this.async=a.async||this.async||x.async;this.c=a.worker||this.c||x.c;this.threshold=a.threshold||e.threshold||this.threshold||x.threshold;this.depth=a.depth||e.depth||this.depth||x.depth;this.w=a.suggest||this.w||x.w;this.v=(b=a.encode||e.encode)&&A[b]||("function"===typeof b?b:this.v||!1);this.F=a.debug||this.F;(b=a.matcher)&&this.addMatcher(b);if(b=a.filter)this.filter=P(L[b]||b,this.v);if(b=a.stemmer)this.stemmer=Q(M[b]||b,this.v);this.l=[{},{},{},{},{},{},{},{},{},{}];this.o={};this.b={};this.h= +{};this.f=[];this.D=null;this.u=!0;if("forward"===this.mode||"reverse"===this.mode||"both"===this.mode)this.a="";this.j=(this.cache=b=a.cache||this.cache||x.cache)?new V(b):!1;return this};f.prototype.encode=function(a){a&&B.length&&(a=y(a,B));a&&this.C.length&&(a=y(a,this.C));a&&this.v&&(a=this.v.call(A,a));a&&this.stemmer&&(a=y(a,this.stemmer));return a};f.prototype.addMatcher=function(a){var b=this.C,e;for(e in a)a.hasOwnProperty(e)&&(b[b.length]=d(e),b[b.length]=a[e]);return this};f.prototype.add= +function(a,b,e){if("string"===typeof b&&b&&(a||0===a))if(this.b[a]&&!e)this.update(a,b);else{if(this.c)return++this.s>=this.i.length&&(this.s=0),this.i[this.s].postMessage(this.s,{add:!0,id:a,content:b}),this.b[a]=""+this.s,this;if(this.async)return this.h[a]||(this.f[this.f.length]=a),this.h[a]=[D.add,a,b],F(this),this;b=this.encode(b);if(!b.length)return this;e=this.mode;b="function"===typeof e?e(b):"ngram"===e?H(b):b.split(K);for(var c={_ctx:{}},d=this.threshold,t=this.depth,g=this.l,h=b.length, +f=0;fl;r--)n=m.substring(l,r),z(g,c,n,a,q,u,d);break;default:if(l=z(g,c,m,a,1,u,d),t&&1=d)for(p=c._ctx[m]||(c._ctx[m]={}),m=this.o[m]||(this.o[m]=[{},{},{},{},{},{},{},{},{},{}]),l=f-t,r=f+t+1,0>l&&(l=0),r>h&&(r=h);lb;b++)q(this.l[b],a);this.depth&&q(this.o,a);delete this.b[a]; +this.u=!1}return this};f.prototype.search=function(a,b,e){var c=[];if(a&&"object"===typeof a){e=a.callback||b;b=a.limit;var d=a.threshold;a=a.query}d=(d||this.threshold||0)|0;"function"===typeof b?(e=b,b=1E3):b||(b=1E3);if(this.c){this.A=e;this.m=0;this.g=[];for(c=0;c= +d;A--)if(w=(u?l[n]:this.l)[A][r])y[z++]=w,x=!0;if(x)m[m.length]=1g;g++)for(b=Object.keys(this.l[g]),a=0;af;c--)h=g[c-1],g[c]=h,d[h]=c;g[f]=a;d[a]=f}}}return b};return a}(); +return f}(function(){var v={},C=!("undefined"===typeof Blob||"undefined"===typeof URL||!URL.createObjectURL);return function(f,w,d,y,z){var q=f;f=C?URL.createObjectURL(new Blob(["var SUPPORT_WORKER = true;var SUPPORT_BUILTINS = true;var SUPPORT_DEBUG = true;var SUPPORT_CACHE = true;var SUPPORT_ASYNC = true;("+d.toString()+")()"],{type:"text/javascript"})):"../"+q+".js";q+="-"+w;v[q]||(v[q]=[]);v[q][z]=new Worker(f);v[q][z].onmessage=y;console.log("Register Worker: "+q+"@"+z);return{postMessage:function(d, +f){v[q][d].postMessage(f)}}}}()),this); diff --git a/package.json b/package.json index 4e40670..6de57e3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "flexsearch", - "version": "0.2.461", + "version": "0.2.48", "description": "Next-Generation full text search library with zero dependencies.", "homepage": "https://nextapps-de.github.io/xone/", "author": "Thomas Wilkerling", diff --git a/test/matching.html b/test/matching.html index 5b842a6..84c9c4d 100644 --- a/test/matching.html +++ b/test/matching.html @@ -3,8 +3,8 @@ Matching Test - - + + @@ -44,7 +44,7 @@ var text_data = "LIBRARY OF THE FUTURE (R) First Edition Ver. 4.02 Gulliver's Tr bm25 - "without breach of modesty" + wait ... wait ... wait ... @@ -56,7 +56,7 @@ var text_data = "LIBRARY OF THE FUTURE (R) First Edition Ver. 4.02 Gulliver's Tr wait ... - "went softly stream" + wait ... wait ... wait ... @@ -68,7 +68,7 @@ var text_data = "LIBRARY OF THE FUTURE (R) First Edition Ver. 4.02 Gulliver's Tr wait ... - "princes of the ambition" + wait ... wait ... wait ... @@ -80,7 +80,7 @@ var text_data = "LIBRARY OF THE FUTURE (R) First Edition Ver. 4.02 Gulliver's Tr wait ... - "five-thousand leagues" + wait ... wait ... wait ... @@ -92,7 +92,7 @@ var text_data = "LIBRARY OF THE FUTURE (R) First Edition Ver. 4.02 Gulliver's Tr wait ... - "i already observed" + wait ... wait ... wait ... @@ -104,7 +104,7 @@ var text_data = "LIBRARY OF THE FUTURE (R) First Edition Ver. 4.02 Gulliver's Tr wait ... - "let a of his" + wait ... wait ... wait ... @@ -116,7 +116,7 @@ var text_data = "LIBRARY OF THE FUTURE (R) First Edition Ver. 4.02 Gulliver's Tr wait ... - "take that to the rocks" + wait ... wait ... wait ... @@ -128,7 +128,7 @@ var text_data = "LIBRARY OF THE FUTURE (R) First Edition Ver. 4.02 Gulliver's Tr wait ... - "bignes of splaknuk" + wait ... wait ... wait ... @@ -140,7 +140,7 @@ var text_data = "LIBRARY OF THE FUTURE (R) First Edition Ver. 4.02 Gulliver's Tr wait ... - "matematikal musikal instruments" + wait ... wait ... wait ... @@ -152,7 +152,7 @@ var text_data = "LIBRARY OF THE FUTURE (R) First Edition Ver. 4.02 Gulliver's Tr wait ... - "matical sical strument" + wait ... wait ... wait ... @@ -164,7 +164,37 @@ var text_data = "LIBRARY OF THE FUTURE (R) First Edition Ver. 4.02 Gulliver's Tr wait ... - "lalkon the camberlayhn" + + wait ... + wait ... + wait ... + wait ... + wait ... + wait ... + wait ... + wait ... + wait ... + + + + wait ... + wait ... + wait ... + wait ... + wait ... + wait ... + wait ... + wait ... + wait ... + + + +
+ Contextual Search Test: + + + + wait ... wait ... wait ... @@ -207,6 +237,12 @@ var text_data = "LIBRARY OF THE FUTURE (R) First Edition Ver. 4.02 Gulliver's Tr } } + data.push('zero one two three four five six seven eight nine ten.'); + data.push('zero one two three four five six seven eight nine ten.'); + data.push('four two zero one three ten five seven eight six nine.'); + data.push('zero one two three four five six seven eight nine ten.'); + data.push('zero one two three four five six seven eight nine ten.'); + tmp = null; new_data = null; text_data = null; @@ -225,10 +261,14 @@ var text_data = "LIBRARY OF THE FUTURE (R) First Edition Ver. 4.02 Gulliver's Tr bulksearch = new BulkSearch({ type: 'short', // this type specifies the maximum bitlength of assigned IDs! - encode: 'advanced', + encode: 'extra', multi: true, + suggest:true, + filter: true, + stemmer: true, async: false, - cache: false + cache: false, + worker: false }); console.time('bulksearch'); @@ -246,6 +286,7 @@ var text_data = "LIBRARY OF THE FUTURE (R) First Edition Ver. 4.02 Gulliver's Tr mode: 'strict', threshold: 5, depth: 3, + suggest:true, filter: true, stemmer: true, async: false, @@ -376,12 +417,14 @@ var text_data = "LIBRARY OF THE FUTURE (R) First Edition Ver. 4.02 Gulliver's Tr do_test('test-3', 'princes of the ambition', [72, 408]); do_test('test-4', 'five-thousand leagues', [2]); do_test('test-5', 'i already observed', [458, 346]); - do_test('test-6', 'let a of his', [50]); - do_test('test-7', 'take that to the rocks', [175]); - do_test('test-8', 'bignes of splaknuk', [146]); - do_test('test-9', 'matematikal musikal instruments', [267]); - do_test('test-10', 'matical sical strument', [267]); - do_test('test-11', 'lalkon the camberlayhn', [99]); + do_test('test-6', 'disgust the bigness', [175]); + do_test('test-7', 'bignes of splaknuk', [146]); + do_test('test-8', 'matematikal musikal instruments', [267]); + do_test('test-9', 'composition of minerals gums juices vegetables', [303]); + do_test('test-10', 'lalkon the camberlayhn', [99]); + do_test('test-11', 'to be at all this is', [184]); + do_test('test-12', 'matical sical strument', [267]); + do_test('test-13', 'zero one three ten', [504]); // --------------------------------------- @@ -389,6 +432,8 @@ var text_data = "LIBRARY OF THE FUTURE (R) First Edition Ver. 4.02 Gulliver's Tr var nodes = document.getElementById(id).getElementsByTagName('td'); + nodes[0].innerHTML = query; + for(var i = 1; i < nodes.length; i++){ var results; diff --git a/test/test.js b/test/test.js index a85b1ed..9a54a07 100644 --- a/test/test.js +++ b/test/test.js @@ -884,7 +884,7 @@ describe('Context', function(){ }); // ------------------------------------------------------------------------ -// Encoding Tests +// Tokenizer Tests // ------------------------------------------------------------------------ describe('Options', function(){ @@ -991,6 +991,7 @@ describe('Relevance', function(){ expect(index.search("1 3 4")).to.have.members([0]); expect(index.search("1 5 3 4")).to.have.members([0]); + expect(index.search("1 3 4 7")).to.have.lengthOf(0); expect(index.search("one")).to.have.members([1, 2]); expect(index.search("one three")).to.have.members([1, 2]); expect(index.search("three one")).to.have.members([1, 2]); @@ -998,6 +999,33 @@ describe('Relevance', function(){ }); }); +// ------------------------------------------------------------------------ +// Suggestion Tests +// ------------------------------------------------------------------------ + +describe('Suggestion', function(){ + + it('Should have been suggested properly by relevance', function(){ + + var index = new FlexSearch({ + encode: 'advanced', + mode: 'strict', + suggest: true + }); + + index.add(0, "1 2 3 2 4 1 5 3"); + index.add(1, "zero one two three four five six seven eight nine ten"); + index.add(2, "four two zero one three ten five seven eight six nine"); + + expect(index.search("1 3 4 7")).to.have.members([0]); + expect(index.search("1 3 9 7")).to.have.members([0]); + expect(index.search("one foobar two")).to.have.members([1, 2]); + expect(index.search("zero one foobar two foobar")).to.have.members([1, 2]); + //TODO + //expect(index.search("zero one foobar two foobar")[0]).to.equal(1); + }); +}); + // ------------------------------------------------------------------------ // Feature Tests // ------------------------------------------------------------------------ @@ -1029,9 +1057,8 @@ describe('Add Matchers', function(){ }).addMatcher({ '8': 'f' - }); - flexsearch_forward.add(0, "12345678"); + }).add(0, "12345678"); expect(flexsearch_forward.search("12345678")).to.include(0); expect(flexsearch_forward.search("abcd")).to.include(0); @@ -1152,5 +1179,5 @@ describe('Chaining', function(){ function test_encoder(str){ - return str = '-[' + str.toUpperCase() + ']-'; + return '-[' + str.toUpperCase() + ']-'; }