From c00e62b2210755e335e5c95d92dfa8865717e52d Mon Sep 17 00:00:00 2001 From: Thomas Wilkerling Date: Tue, 5 Feb 2019 18:17:29 +0100 Subject: [PATCH] v0.4.0 --- CHANGELOG.md | 22 ++ README.md | 214 ++++++++++++++++- demo/autocomplete.html | 73 ++++-- dist/flexsearch.compact.js | 38 ++-- dist/flexsearch.es5.js | 66 +++--- dist/flexsearch.light.js | 24 +- dist/flexsearch.min.js | 53 +++-- dist/flexsearch.node.js | 46 ++-- flexsearch.js | 455 +++++++++++++++++++++++++++++++------ package.json | 15 +- test/test.es6.js | 353 ++++++++++++++++++++++++++++ test/test.js | 112 +++++++-- 12 files changed, 1243 insertions(+), 228 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 test/test.es6.js diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..f64b8d8 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,22 @@ +# Changelog + +#### v0.4.0 + +- Index Documents (Field-Search) + +#### v0.3.6 + +- Right-To-Left Support +- CJK Word Splitting Support + +#### v0.3.5 + +- Promise Support + +#### v0.3.4 + +- Export / Import Indexes (Serialize) + +#### v0.3.0 + +- Profiler Support diff --git a/README.md b/README.md index a08900f..eea32e0 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ FlexSearch also provides you a non-blocking asynchronous processing model as wel FlexSearch Server is also available here: https://github.com/nextapps-de/flexsearch-server. -Installation Guide  •  API Reference  •  Example Options  •  Custom Builds  •  Flexsearch Server +Installation Guide  •  API Reference  •  Example Options  •  Custom Builds  •  Flexsearch Server  •  Changelog Supported Platforms: - Browser @@ -114,6 +114,15 @@ All Features: ✔ + + + Index Documents (Field-Search) + + ✔ + ✔ + - + + Partial Matching @@ -994,6 +1003,201 @@ Returns information e.g.: "contextual": true } ``` + + +## Index Documents (Field-Search) + +#### The Document Descriptor + +Assume the document is an array of data like: + +```js +var docs = [{ + id: 0, + title: "Title", + cat: "Category", + content: "Body" +},{ + id: 1, + title: "Title", + cat: "Category", + content: "Body" +}]; +``` + +Provide a document descriptor ___doc___ when initializing a new index, e.g. related to the example above: + +```js +var index = new FlexSearch({ + tokenize: "strict", + depth: 3, + doc: { + id: "id", + field: "content" + } +}); +``` + +The above example will index one field ("content"), to index multiple fields just pass an array: + +```js +var index = new FlexSearch({ + doc: { + id: "id", + field: [ + "title", + "cat", + "content" + ] + } +}); +``` + +#### Complex Objects + +Assume the document array looks more complex (has nested branches etc.), e.g.: + +```js +var docs = [{ + data:{ + id: 0, + title: "Title", + body: { + content: "Body" + } + } +},{ + data:{ + id: 1, + title: "Title", + body: { + content: "Body" + } + } +}]; +``` + +Then use the colon separated notation ___"root:child"___ to define hierarchy within the document descriptor: + +```js +var index = new FlexSearch({ + doc: { + id: "data:id", + field: [ + "data:title", + "data:content:body" + ] + } +}); +``` + +#### Add/Update/Remove Documents to the Index + +Just pass the document array (or one single object) to the index: +```js +index.add(docs); +``` + +Update (single object or array of objects): +```js +index.update({ + data:{ + id: 0, + title: "Foo", + body: { + content: "Bar" + } + } +}); +``` + +Remove (single object or array of objects): +```js +index.remove(docs); +``` + +When you know the id, you can also simply remove by: +```js +index.remove(id); +``` + +#### Field-Search + +When searching you have several options when using documents. + +This will search through all indexed fields: +```js +var results = index.search("body"); +``` + +This will search on a specific field): + +```js +var results = index.search({ + field: "title", + query: "title" +}); +``` + +The colon notation also has to be applied for the searching respectively, e.g.: + +```js +var results = index.search({ + field: "data:body", + query: "body" +}); +``` + +This could also be written as: +```js +var results = index.search("body", { + field: "data:body", +}); +``` + +Search the same query on multiple fields: + +```js +var results = index.search({ + query: "title", + field: ["title", "body"] +}); +``` + +Could also be written as: + +```js +var results = index.search("title", { + field: ["title", "body"] +}); +``` + +Search different queries on multiple fields: + +```js +var results = index.search([{ + field: "title", + query: "title" +},{ + field: "body", + query: "body" +}]); +``` + +Boost scoring on specific fields: + +```js +var results = index.search([{ + field: "title", + query: "title", + boost: 2 +},{ + field: "body", + query: "body", + boost: 0.5 +}]); +``` + ### Chaining @@ -1011,7 +1215,7 @@ index.remove(0).update(1, 'foo').add(2, 'foobar'); ``` -### Enable Contextual Scoring +## Enable Contextual Scoring Create an index and just set the limit of relevance as "depth": ```js @@ -1031,7 +1235,7 @@ var index = new FlexSearch({ > Try to use the __lowest depth__ and __highest threshold__ which fits your needs. -### Enable Auto-Balanced Cache +## Enable Auto-Balanced Cache Create index and just set a limit of cache entries: ```js @@ -1047,7 +1251,7 @@ var index = new FlexSearch({ > When just using "true" the cache is unbounded and perform actually 2-3 times faster (because the balancer do not have to run). -### WebWorker Sharding (Browser only) +## WebWorker Sharding (Browser only) Worker get its own dedicated memory and also run in their own dedicated thread without blocking the UI while processing. Especially for larger indexes, web worker improves speed and available memory a lot. FlexSearch index was tested with a 250 Mb text file including 10 Million words. @@ -1901,6 +2105,8 @@ node compile SUPPORT_WORKER=true +Changelog + --- Copyright 2019 Nextapps GmbH
diff --git a/demo/autocomplete.html b/demo/autocomplete.html index 756d312..e658ea9 100644 --- a/demo/autocomplete.html +++ b/demo/autocomplete.html @@ -4,61 +4,75 @@ Demo: Auto-Complete - + - - - -
+
-
-
+
diff --git a/dist/flexsearch.compact.js b/dist/flexsearch.compact.js index 80db403..aa1a7e1 100644 --- a/dist/flexsearch.compact.js +++ b/dist/flexsearch.compact.js @@ -1,23 +1,27 @@ /* - FlexSearch v0.3.62 + FlexSearch v0.4.0 Copyright 2019 Nextapps GmbH Author: Thomas Wilkerling Released under the Apache 2.0 Licence https://github.com/nextapps-de/flexsearch */ -'use strict';(function(f,p,d){let k;(k=d.define)&&k.amd?k([],function(){return p}):(k=d.modules)?k[f.toLowerCase()]=p:"object"===typeof exports?module.exports=p:d[f]=p})("FlexSearch",function(){function f(b){this.id=b&&!y(b.id)?b.id:L++;this.init(b);p(this,"index",function(){return this.b});p(this,"length",function(){return Object.keys(this.b).length})}function p(b,a,c){Object.defineProperty(b,a,{get:c})}function d(b){return new RegExp(b,"g")}function k(b,a){for(let c=0;c=m&&(b=b[9-(e+.5>>0)],b=b[c]||(b[c]=[]),b[b.length]=g);return e}function C(b,a){if(b){const c=Object.keys(b);for(let g=0,d=c.length;gb?1:b?-1:0}function N(b,a){b=b.length-a.length;return 0>b?-1:b?1:0}function O(b,a,c){let d=[],e;const t=b.length;if(1a&&(d=d.slice(0,a)));return d}function A(b){return"string"===typeof b}function B(b){return"function"===typeof b}function y(b){return"undefined"===typeof b}function G(b){const a=Array(b);for(let c= -0;ch;c--)n=m.substring(h,c),z(t,k,n,b,a,l,q)}break;default:if(f=z(t,k,m,b,1,l,q),r&&1=q)for(f=k._ctx[m]||(k._ctx[m]=u()),m=this.c[m]||(this.c[m]=G(10-(q||0))),l=a-r,n=a+r+1,0>l&&(l=0),n>v&&(n=v);l=e&&(a=a[9-(g+.5>>0)],a=a[c]||(a[c]=[]),a[a.length]=h);return g}function E(a,b){if(a){const c=Object.keys(a);for(let h=0,g=c.length;ha?1:a?-1:0}function O(a,b){a=a.length-b.length;return 0>a?-1:a?1:0}function P(a,b,c,h){let g=[],l;const e=a.length;if(1b&&(g=g.slice(0,b));return g}function B(a){return"string"===typeof a}function C(a){return"function"=== +typeof a}function y(a){return"object"===typeof a}function D(a){return"undefined"===typeof a}function H(a){const b=Array(a);for(let c=0;ck;c--)f=e.substring(k,c),A(t,q,f,a,b,n,m)}break;default:if(d=A(t,q,e,a,1,n,m),p&&1=m)for(d=q._ctx[e]||(q._ctx[e]=v()),e=this.g[e]||(this.g[e]=H(10-(m||0))),n=b-p,f=b+p+1,0>n&&(n=0),f>w&&(f=w);nn&&(b=h));for(n=p?1:0;n=c&&(this.v=this.h),this.C&&this.v===this.h&&(this.cache&&this.o.set(a, -this.l),this.C(this.l),this.l=[]));return this}function l(b,a,f){Object.defineProperty(b,a,{get:f})}function c(b){return new RegExp(b,"g")}function d(b,a){for(var f=0;f=g&&(b=b[9-(d+.5>>0)],b=b[f]||(b[f]=[]),b[b.length]=c);return d}function D(b,a){if(b)for(var f=Object.keys(b),c=0,d=f.length;cb?1:b?-1:0}function W(b,a){b=b.length-a.length;return 0>b?-1:b?1:0}function V(b,a,f){var c=[],d=b.length;if(1a&&(c=c.slice(0,a)));return c}function C(b){return"string"=== -typeof b}function H(b){return"function"===typeof b}function t(b){return"undefined"===typeof b}function M(b){for(var a=Array(b),f=0;f=this.f.length&&(this.u=0),this.f[this.u].postMessage({add:!0,id:b,content:a}),this.a[e]=""+this.u,f&&f(),this;if(!d){if(this.async&& -"function"!==typeof importScripts){var g=this;e=new Promise(function(f){setTimeout(function(){g.add(b,a,null,c,!0);g=null;f()})});if(f)e.then(f);else return e;return this}if(f)return this.add(b,a,null,c,!0),f(),this}a=this.encode(a);if(!a.length)return this;f=this.g;d=H(f)?f(a):a.split(O);var h=z();h._ctx=z();for(var B=this.threshold,l=this.depth,I=this.b,q=d.length,x=this.B,u=0;ur;v--)w=p.substring(r,v),A(I,h,w,b,t,n,B);break;default:if(m=A(I,h,p,b,1,n,B),l&&1=B)for(m=h._ctx[p]||(h._ctx[p]=z()),p=this.c[p]||(this.c[p]=M(10-(B||0))),n=u-l,w=u+l+1,0>n&&(n=0),w>q&&(w=q);nd;c--)h=g[c-1],g[c]=h,e[h]= -c;g[d]=a;e[a]=d}}}return b};return b}();return e}(function(){var g={},k="undefined"!==typeof Blob&&"undefined"!==typeof URL&&URL.createObjectURL;return function(e,h,l,c,d){l=k?URL.createObjectURL(new Blob(["("+l.toString()+")()"],{type:"text/javascript"})):e+".es5.js";e+="-"+h;g[e]||(g[e]=[]);g[e][d]=new Worker(l);g[e][d].onmessage=c;console.log("Register Worker: "+e+"@"+d);return g[e][d]}}()),this); +'use strict';function L(g){var l=0;return function(){return l=d&&(this.B=this.h),this.G&&this.B===this.h&&(this.cache&& +this.w.set(b,this.u),this.G(this.u),this.u=[]));return this}function k(a,b,c){Object.defineProperty(a,b,{get:c})}function f(a){return new RegExp(a,"g")}function e(a,b){for(var c=0;c=e&&(a=a[9-(h+.5>>0)],a=a[c]||(a[c]=[]),a[a.length]=d);return h}function C(a,b){if(a)for(var c=Object.keys(a),d=0,h=c.length;da?1:a?-1:0}function X(a,b){a=a.length-b.length;return 0>a?-1:a?1:0}function W(a,b,c,d){var h=[],u=a.length;if(1b&&(h=h.slice(0,b));return h}function F(a){return"string"===typeof a}function I(a){return"function"===typeof a}function E(a){return"object"===typeof a}function z(a){return"undefined"===typeof a}function N(a){for(var b=Array(a),c=0;c=this.a.length&&(this.v=0),this.a[this.v].postMessage({add:!0,id:a,content:b}),this.b[e]= +""+this.v,c&&c(),this;if(!h){if(this.async&&"function"!==typeof importScripts){var f=this;e=new Promise(function(c){setTimeout(function(){f.add(a,b,null,d,!0);f=null;c()})});if(c)e.then(c);else return e;return this}if(c)return this.add(a,b,null,d,!0),c(),this}b=this.encode(b);if(!b.length)return this;c=this.g;h=I(c)?c(b):b.split(P);var g=B();g._ctx=B();for(var r=this.threshold,m=this.depth,p=this.m,k=h.length,q=this.F,A=0;Ay;z--)w=x.substring(y,z),D(p,g,w,a,t,n,r);break;default:if(v=D(p,g,x,a,1,n,r),m&&1=r)for(v=g._ctx[x]||(g._ctx[x]=B()),x=this.s[x]||(this.s[x]=N(10-(r||0))),n=A-m,w=A+m+1,0>n&&(n=0),w>k&&(w=k);nm&&(v=r));for(m=k?1:0;mf;d--)k=g[d- +1],g[d]=k,e[k]=d;g[f]=a;e[a]=f}}}return b};return a}();return d}(function(){var g={},l="undefined"!==typeof Blob&&"undefined"!==typeof URL&&URL.createObjectURL;return function(d,m,k,f,e){k=l?URL.createObjectURL(new Blob(["("+k.toString()+")()"],{type:"text/javascript"})):d+".es5.js";d+="-"+m;g[d]||(g[d]=[]);g[d][e]=new Worker(k);g[d][e].onmessage=f;console.log("Register Worker: "+d+"@"+e);return g[d][e]}}()),this); diff --git a/dist/flexsearch.light.js b/dist/flexsearch.light.js index 168fa63..3c74d9d 100644 --- a/dist/flexsearch.light.js +++ b/dist/flexsearch.light.js @@ -1,18 +1,18 @@ /* - FlexSearch v0.3.62 + FlexSearch v0.4.0 Copyright 2019 Nextapps GmbH Author: Thomas Wilkerling Released under the Apache 2.0 Licence https://github.com/nextapps-de/flexsearch */ -'use strict';(function(e,l,u){let n;(n=u.define)&&n.amd?n([],function(){return l}):(n=u.modules)?n[e.toLowerCase()]=l:"object"===typeof exports?module.exports=l:u[e]=l})("FlexSearch",function(){function e(a){this.id=a&&!x(a.id)?a.id:G++;this.init(a);l(this,"index",function(){return this.a});l(this,"length",function(){return Object.keys(this.a).length})}function l(a,b,c){Object.defineProperty(a,b,{get:c})}function u(a,b){for(let c=0;c=g&&(a=a[9-(d+.5>>0)],a=a[c]||(a[c]=[]),a[a.length]=f);return d}function z(a,b){if(a){const c=Object.keys(a);for(let f=0,d=c.length;fa?1:a?-1:0}function I(a,b){a=a.length-b.length;return 0>a?-1:a?1:0}function y(a){return"function"=== -typeof a}function x(a){return"undefined"===typeof a}function C(a){const b=Array(a);for(let c=0;ch;c--)m=p.substring(h,c),n(r,f,m,a,b,k,d)}break;default:if(g=n(r,f,p,a,1,k,d),B&&1=d)for(g=f._ctx[p]||(f._ctx[p]=t()),p=this.c[p]||(this.c[p]=C(10-(d||0))),k=e-B,m=e+B+1,0>k&&(k=0),m>q&&(m=q);kb&&(g=g.slice(0,b)));e=g}return e};e.prototype.clear=function(){return this.destroy().init()};e.prototype.destroy=function(){this.g=this.c= -this.a=null;return this};const v={icase:function(a){return a.toLowerCase()}};return e}(!1),this); +'use strict';(function(e,l,u){let n;(n=u.define)&&n.amd?n([],function(){return l}):(n=u.modules)?n[e.toLowerCase()]=l:"object"===typeof exports?module.exports=l:u[e]=l})("FlexSearch",function(){function e(a,c){const b=c?c.id:a&&a.id;this.id=b||0===b?b:G++;this.init(a,c);l(this,"index",function(){return this.a});l(this,"length",function(){return Object.keys(this.a).length})}function l(a,c,b){Object.defineProperty(a,c,{get:b})}function u(a,c){for(let b=0;b=g&&(a=a[9-(d+.5>>0)],a=a[b]||(a[b]=[]),a[a.length]=f);return d}function y(a,c){if(a){const b=Object.keys(a);for(let f=0,d=b.length;fa?1:a?-1:0}function I(a,c){a=a.length-c.length;return 0>a?-1:a?1:0}function x(a){return"function"=== +typeof a}function z(a){return"undefined"===typeof a}function C(a){const c=Array(a);for(let b=0;bh;b--)m=p.substring(h,b),n(r,f,m,a,c,k,d)}break;default:if(g=n(r,f,p,a,1,k,d),B&&1=d)for(g=f._ctx[p]||(f._ctx[p]=t()),p=this.c[p]||(this.c[p]= +C(10-(d||0))),k=e-B,m=e+B+1,0>k&&(k=0),m>q&&(m=q);kc&&(g=g.slice(0,c)));e=g}return e};e.prototype.clear=function(){return this.destroy().init()}; +e.prototype.destroy=function(){this.g=this.c=this.a=null;return this};const w={icase:function(a){return a.toLowerCase()}};return e}(!1),this); diff --git a/dist/flexsearch.min.js b/dist/flexsearch.min.js index f2c4716..7805714 100644 --- a/dist/flexsearch.min.js +++ b/dist/flexsearch.min.js @@ -1,30 +1,35 @@ /* - FlexSearch v0.3.62 + FlexSearch v0.4.0 Copyright 2019 Nextapps GmbH Author: Thomas Wilkerling Released under the Apache 2.0 Licence https://github.com/nextapps-de/flexsearch */ -'use strict';(function(u,A,h){let v;(v=h.define)&&v.amd?v([],function(){return A}):(v=h.modules)?v[u.toLowerCase()]=A:"object"===typeof exports?module.exports=A:h[u]=A})("FlexSearch",function P(u){function h(b){this.id=b&&!w(b.id)?b.id:Q++;this.init(b);B(this,"index",function(){return this.a});B(this,"length",function(){return Object.keys(this.a).length})}function v(b,a,c,d){this.m!==this.c&&(this.g=this.g.concat(c),this.m++,d&&this.g.length>=d&&(this.m=this.c),this.B&&this.m===this.c&&(this.cache&& -this.j.set(a,this.g),this.B(this.g),this.g=[]));return this}function B(b,a,c){Object.defineProperty(b,a,{get:c})}function f(b){return new RegExp(b,"g")}function m(b,a){for(let c=0;c=g&&(b=b[9-(e+.5>>0)],b=b[c]||(b[c]=[]),b[b.length]=d);return e}function I(b,a){if(b){const c=Object.keys(b);for(let d=0,e=c.length;db?1:b?-1:0}function S(b,a){b=b.length-a.length;return 0>b?-1:b?1:0}function T(b, -a,c){let d=[],e;const l=b.length;if(1a&&(d=d.slice(0,a)));return d}function F(b){return"string"===typeof b}function G(b){return"function"===typeof b}function w(b){return"undefined"===typeof b}function K(b){const a=Array(b);for(let c=0;c=this.i.length&&(this.w=0),this.i[this.w].postMessage({add:!0,id:b,content:a}),this.a[f]=""+this.w,c&&c(),this;if(!e){if(this.async&& -"function"!==typeof importScripts){let e=this;f=new Promise(function(c){setTimeout(function(){e.add(b,a,null,d,!0);e=null;c()})});if(c)f.then(c);else return f;return this}if(c)return this.add(b,a,null,d,!0),c(),this}a=this.encode(a);if(!a.length)return this;c=this.b;e=G(c)?c(a):a.split(M);const l=r();l._ctx=r();const q=this.threshold,t=this.depth,E=this.f,m=e.length,y=this.A;for(let a=0;ak;c--)n=g.substring(k,c),D(E,l,n,b,a,p,q)}break;default:if(h=D(E,l,g,b,1,p,q),t&&1=q)for(h=l._ctx[g]||(l._ctx[g]=r()),g=this.h[g]||(this.h[g]=K(10-(q||0))),p=a-t,n=a+t+1,0>p&&(p=0),n>m&&(n=m);pc;d--)e=f[d-1],f[d]=e,b[e]=d;f[c]=a;b[a]=c}}}return b};return b}();return h}(function(){const u={},A="undefined"!==typeof Blob&&"undefined"!==typeof URL&&URL.createObjectURL;return function(h,v,B,f,m){B=A?URL.createObjectURL(new Blob(["("+B.toString()+")()"],{type:"text/javascript"})):h+".min.js";h+="-"+v;u[h]||(u[h]=[]);u[h][m]=new Worker(B);u[h][m].onmessage=f;return u[h][m]}}()),this); +'use strict';(function(t,A,g){let w;(w=g.define)&&w.amd?w([],function(){return A}):(w=g.modules)?w[t.toLowerCase()]=A:"object"===typeof exports?module.exports=A:g[t]=A})("FlexSearch",function Q(t){function g(a,b){const c=b?b.id:a&&a.id;this.id=c||0===c?c:R++;this.init(a,b);B(this,"index",function(){return this.a});B(this,"length",function(){return Object.keys(this.a).length})}function w(a,b,c,e){this.s!==this.f&&(this.j=this.j.concat(c),this.s++,e&&this.j.length>=e&&(this.s=this.f),this.F&&this.s=== +this.f&&(this.cache&&this.m.set(b,this.j),this.F(this.j),this.j=[]));return this}function B(a,b,c){Object.defineProperty(a,b,{get:c})}function f(a){return new RegExp(a,"g")}function n(a,b){for(let c=0;c=h&&(a=a[9-(d+.5>>0)],a=a[c]||(a[c]=[]),a[a.length]=e);return d}function J(a,b){if(a){const c=Object.keys(a);for(let e=0,d=c.length;ea?1:a?-1:0}function T(a,b){a=a.length-b.length;return 0>a?-1:a?1:0}function U(a, +b,c,e){let d=[],l;const h=a.length;if(1b&&(d=d.slice(0,b));return d}function G(a){return"string"===typeof a}function H(a){return"function"===typeof a}function z(a){return"object"===typeof a}function y(a){return"undefined"===typeof a}function L(a){const b=Array(a);for(let c=0;c=this.l.length&&(this.C=0),this.l[this.C].postMessage({add:!0,id:a,content:b}), +this.a[l]=""+this.C,c&&c(),this;if(!d){if(this.async&&"function"!==typeof importScripts){let d=this;l=new Promise(function(c){setTimeout(function(){d.add(a,b,null,e,!0);d=null;c()})});if(c)l.then(c);else return l;return this}if(c)return this.add(a,b,null,e,!0),c(),this}b=this.encode(b);if(!b.length)return this;c=this.c;d=H(c)?c(b):b.split(N);const r=u();r._ctx=u();const m=this.threshold,x=this.depth,v=this.h,D=d.length,n=this.D;for(let b=0;bp;c--)k=h.substring(p,c),F(v,r,k,a,b,g,m)}break;default:if(f=F(v,r,h,a,1,g,m),x&&1=m)for(f=r._ctx[h]||(r._ctx[h]=u()),h=this.i[h]||(this.i[h]=L(10-(m||0))),g=b-x,k=b+x+1,0>g&&(g=0),k>D&&(k=D);gn&&(a=g));for(n=k?1:0;nc;e--)d=f[e-1],f[e]=d,b[d]=e;f[c]=a;b[a]=c}}}return b};return a}();return g}(function(){const t={},A="undefined"!==typeof Blob&&"undefined"!==typeof URL&&URL.createObjectURL;return function(g,w,B,f,n){B=A?URL.createObjectURL(new Blob(["("+B.toString()+")()"],{type:"text/javascript"})):g+".min.js";g+="-"+w;t[g]||(t[g]=[]);t[g][n]=new Worker(B);t[g][n].onmessage=f;return t[g][n]}}()), +this); diff --git a/dist/flexsearch.node.js b/dist/flexsearch.node.js index f345ea0..21ba0a9 100644 --- a/dist/flexsearch.node.js +++ b/dist/flexsearch.node.js @@ -1,27 +1,31 @@ /* - FlexSearch v0.3.62 + FlexSearch v0.4.0 Copyright 2019 Nextapps GmbH Author: Thomas Wilkerling Released under the Apache 2.0 Licence https://github.com/nextapps-de/flexsearch */ -'use strict';(function(g,y,d){let m;(m=d.define)&&m.amd?m([],function(){return y}):(m=d.modules)?m[g.toLowerCase()]=y:"object"===typeof exports?module.exports=y:d[g]=y})("FlexSearch",function(){function g(b){this.id=b&&!t(b.id)?b.id:L++;this.init(b);y(this,"index",function(){return this.b});y(this,"length",function(){return Object.keys(this.b).length})}function y(b,a,c){Object.defineProperty(b,a,{get:c})}function d(b){return new RegExp(b,"g")}function m(b,a){for(let c=0;c=h&&(b=b[9-(e+.5>>0)],b=b[c]||(b[c]=[]),b[b.length]=f);return e}function E(b,a){if(b){const c=Object.keys(b);for(let f=0,e=c.length;fb?1:b?-1:0}function N(b,a){b=b.length-a.length;return 0>b?-1:b?1:0}function O(b,a,c){let f=[],e;const d=b.length;if(1a&&(f=f.slice(0,a)));return f}function B(b){return"string"===typeof b}function C(b){return"function"===typeof b}function t(b){return"undefined"===typeof b}function G(b){const a=Array(b);for(let c= -0;ck;c--)q=h.substring(k,c),A(m,n,q,b,a,l,p)}break;default:if(g=A(m,n,h,b,1,l,p),v&&1=p)for(g=n._ctx[h]||(n._ctx[h]=u()),h=this.f[h]||(this.f[h]=G(10- -(p||0))),l=a-v,q=a+v+1,0>l&&(l=0),q>w&&(q=w);lc;d--)e=f[d-1],f[d]=e,b[e]=d;f[c]=a;b[a]=c}}}return b}; -return b}();return g}(!1),this); +'use strict';(function(h,w,d){let p;(p=d.define)&&p.amd?p([],function(){return w}):(p=d.modules)?p[h.toLowerCase()]=w:"object"===typeof exports?module.exports=w:d[h]=w})("FlexSearch",function(){function h(a,b){const c=b?b.id:a&&a.id;this.id=c||0===c?c:N++;this.init(a,b);w(this,"index",function(){return this.b});w(this,"length",function(){return Object.keys(this.b).length})}function w(a,b,c){Object.defineProperty(a,b,{get:c})}function d(a){return new RegExp(a,"g")}function p(a,b){for(let c=0;c=g&&(a=a[9-(e+.5>>0)],a=a[c]||(a[c]=[]),a[a.length]=f);return e}function G(a,b){if(a){const c=Object.keys(a);for(let f=0,e=c.length;fa?1:a?-1:0}function P(a,b){a=a.length-b.length;return 0>a?-1:a?1:0}function Q(a,b,c,f){let e=[],l;const g=a.length;if(1b&&(e=e.slice(0,b));return e}function D(a){return"string"===typeof a}function E(a){return"function"=== +typeof a}function x(a){return"object"===typeof a}function z(a){return"undefined"===typeof a}function I(a){const b=Array(a);for(let c=0;ch;c--)k=g.substring(h,c),C(v,r,k,a,b,n,m)}break;default:if(d=C(v,r,g,a,1,n,m),p&&1=m)for(d=r._ctx[g]||(r._ctx[g]=u()),g= +this.h[g]||(this.h[g]=I(10-(m||0))),n=b-p,k=b+p+1,0>n&&(n=0),k>A&&(k=A);nn&&(a=h));for(n=k?1:0;nc;d--)e=f[d-1],f[d]=e,b[e]=d;f[c]=a;b[a]=c}}}return b}; +return a}();return h}(!1),this); diff --git a/flexsearch.js b/flexsearch.js index fd3e13d..68d6f16 100644 --- a/flexsearch.js +++ b/flexsearch.js @@ -1,5 +1,5 @@ /**! - * @preserve FlexSearch v0.3.62 + * @preserve FlexSearch v0.4.0 * Copyright 2019 Nextapps GmbH * Author: Thomas Wilkerling * Released under the Apache 2.0 Licence @@ -17,6 +17,7 @@ /** @define {boolean} */ const SUPPORT_SUGGESTIONS = true; /** @define {boolean} */ const SUPPORT_SERIALIZE = true; /** @define {boolean} */ const SUPPORT_INFO = true; +/** @define {boolean} */ const SUPPORT_DOCUMENTS = true; // noinspection ThisExpressionReferencesGlobalObjectJS (function(){ @@ -44,7 +45,10 @@ threshold: 0, // contextual depth - depth: 0 + depth: 0, + + // multi-field documents + doc: false }; /** @@ -152,10 +156,11 @@ /** * @param {string|Object=} options + * @param {Object=} settings * @constructor */ - function FlexSearch(options){ + function FlexSearch(options, settings){ if(PROFILER){ @@ -165,28 +170,11 @@ this.stats = profile; } - /* - if(SUPPORT_PRESETS && is_string(options)){ - - options = presets[options]; - - if(DEBUG && !options){ - - console.warn("Preset not found: " + options); - } - } - */ - - //options || (options = defaults); - - // generate UID + const id = settings ? settings["id"] : options && options["id"]; /** @export */ - this.id = options && !is_undefined(options["id"]) ? options["id"] : id_counter++; - - // initialize index - - this.init(options); + this.id = id || (id === 0) ? id : id_counter++; + this.init(options, settings); // define functional properties @@ -274,14 +262,7 @@ FlexSearch.encode = function(name, value){ - // if(index_blacklist[name]){ - // - // return value; - // } - // else{ - - return global_encoder[name](value); - // } + return global_encoder[name](value); }; function worker_handler(id, query, result, limit){ @@ -315,17 +296,28 @@ /** * @param {Object|string=} options + * @param {Object=} settings * @export */ - FlexSearch.prototype.init = function(options){ + FlexSearch.prototype.init = function(options, settings){ /** @type {Array} @private */ this._matcher = []; - options || (options = defaults); + let custom; + + if(settings){ + + custom = /** @type {?string} */ (settings["preset"]); + options = settings; + } + else{ + + options || (options = defaults); + custom = /** @type {?string} */ (options["preset"]); + } - let custom = /** @type {?string} */ (options["preset"]); let preset = {}; if(SUPPORT_PRESETS){ @@ -467,7 +459,8 @@ custom = is_undefined(custom = options["encode"]) ? - preset.encode + preset.encode || + defaults.encode : custom; @@ -499,6 +492,17 @@ this.stemmer = init_stemmer(stemmer[custom] || custom, this.encoder); } + let doc; + + if(SUPPORT_DOCUMENTS) /** @private */ this.doc = doc = ( + + (custom = options["doc"]) ? + + custom + : + this.doc || defaults.doc + ); + // initialize primary index /** @private */ @@ -507,11 +511,47 @@ this._ctx = create_object(); /** @private */ this._ids = create_object(); + + if(SUPPORT_DOCUMENTS){ + + /** @private */ + this._doc = doc && create_object(); + } + /** @private */ //this._stack = create_object(); /** @private */ //this._stack_keys = []; + if(doc){ + + options["doc"] = null; + + const field = doc["field"]; + const index = doc["index"] = []; + const ref = doc["ref"] = {}; + + doc["id"] = doc["id"].split(":"); + + if(is_array(field)){ + + for(let i = 0; i < field.length; i++){ + + ref[field[i]] = i; + field[i] = field[i].split(":"); + index[i] = new FlexSearch(options); + index[i]._doc = this._doc; + } + } + else{ + + ref[field] = 0; + doc["field"] = [field.split(":")]; + index[0] = new FlexSearch(options); + index[0]._doc = this._doc; + } + } + /** * @type {number|null} * @private @@ -638,7 +678,7 @@ /** * @param {number|string} id - * @param {string} content + * @param {string|Function} content * @param {Function=} callback * @param {boolean=} _skip_update * @param {boolean=} _recall @@ -648,6 +688,11 @@ FlexSearch.prototype.add = function(id, content, callback, _skip_update, _recall){ + if(SUPPORT_DOCUMENTS && this.doc && is_object(id)){ + + return this.handle_docs("add", id, /** @type {Function} */ (content)); + } + if(content && is_string(content) && ((id /*&& !index_blacklist[id]*/) || (id === 0))){ // TODO: do not mix ids as string "1" and as number 1 @@ -733,7 +778,7 @@ profile_start("add"); } - content = this.encode(content); + content = this.encode(/** @type {string} */ (content)); if(!content.length){ @@ -919,15 +964,110 @@ return this; }; + if(SUPPORT_DOCUMENTS){ + + /** + * @param {!string} job + * @param docs + * @param {Function=} callback + * @returns {*} + */ + + FlexSearch.prototype.handle_docs = function(job, docs, callback){ + + if(is_array(docs)){ + + for(let i = 0, len = docs.length; i < len; i++){ + + if(i === len - 1){ + + return this.handle_docs(job, docs[i], callback); + } + else{ + + this.handle_docs(job, docs[i]); + } + } + } + else{ + + const index = this.doc["index"]; + let tree = this.doc["id"]; + let id; + + for(let i = 0; i < tree.length; i++){ + + id = (id || docs)[tree[i]]; + } + + if(job === "remove"){ + + delete this._doc["@" + id]; + + for(let z = 0, length = index.length; z < length; z++){ + + if(z === length - 1){ + + return index[z].remove(id, callback); + } + else{ + + index[z].remove(id); + } + } + } + + tree = this.doc["field"]; + + for(let i = 0, len = tree.length; i < len; i++){ + + const branch = tree[i]; + let content; + + for(let x = 0; x < branch.length; x++){ + + content = (content || docs)[branch[x]]; + } + + this._doc["@" + id] = docs; + + const self = index[i]; + const fn = ( + + job === "add" ? + + self.add + : + self.update + ); + + if(i === len - 1){ + + return fn.call(self, id, content, callback); + } + else{ + + fn.call(self, id, content); + } + } + } + }; + } + /** * @param {number|string} id - * @param {string} content + * @param {string|Function} content * @param {Function=} callback * @export */ FlexSearch.prototype.update = function(id, content, callback){ + if(SUPPORT_DOCUMENTS && this.doc && is_object(id)){ + + return this.handle_docs("update", id, /** @type {Function} */ (content)); + } + const index = "@" + id; if(this._ids[index] && is_string(content)){ @@ -958,6 +1098,11 @@ FlexSearch.prototype.remove = function(id, callback, _recall){ + if(SUPPORT_DOCUMENTS && this.doc && is_object(id)){ + + return this.handle_docs("remove", id, callback); + } + const index = "@" + id; if(this._ids[index]){ @@ -1053,7 +1198,7 @@ }; /** - * @param {!string} query + * @param {!string|Object|Array} query * @param {number|Function=} limit * @param {Function=} callback * @param {boolean=} _recall @@ -1063,11 +1208,30 @@ FlexSearch.prototype.search = function(query, limit, callback, _recall){ + if(SUPPORT_DOCUMENTS && is_object(limit)){ + + if(is_array(limit)){ + + for(let i = 0; i < limit.length; i++){ + + limit[i]["query"] = query; + } + } + else{ + + limit["query"] = query; + } + + query = /** @type {Object} */ (limit); + limit = 0; + } + let _query = query; let threshold; + let boost; let result = []; - if(is_object(query)){ + if(is_object(query) && (!SUPPORT_DOCUMENTS || !is_array(query))){ // re-assign properties @@ -1083,15 +1247,86 @@ limit = query["limit"]; threshold = query["threshold"]; + boost = query["boost"]; query = query["query"]; } - /* - if(index_blacklist[query]){ + if(SUPPORT_DOCUMENTS && this.doc){ - return result; + const doc_ref = this.doc["ref"]; + const doc_idx = this.doc["index"]; + + let queries; + let field = _query["field"]; + + if(field){ + + _query["field"] = null; + } + else if(is_array(_query)){ + + queries = _query; + field = []; + + for(let i = 0; i < _query.length; i++){ + + field[i] = _query[i]["field"]; + } + } + else{ + + field = get_keys(doc_ref); + } + + if(is_object(field)){ + + if(!is_array(field)){ + + field = [field]; + } + + const len = field.length; + + for(let i = 0; i < len; i++){ + + if(queries){ + + _query = queries[i]; + } + + const ref = doc_ref[field[i]]; + + // TODO: Support Field-Merging (return array before intersection and apply here) + + result[i] = doc_idx[ref].search(_query); + } + + if(SUPPORT_ASYNC && callback){ + + return callback(result.concat.apply([], result)); + } + else if(SUPPORT_ASYNC && this.async){ + + return new Promise(function(resolve){ + + Promise.all(/** @type {!Iterable} */ (result)).then(function(values){ + + resolve(values.concat.apply([], values)); + }); + }); + } + else{ + + return result.concat.apply([], result); + } + } + else{ + + const ref = doc_ref[field]; + + return doc_idx[ref].search(_query, callback); + } } - */ threshold || (threshold = this.threshold || 0); @@ -1215,6 +1450,7 @@ tokenizer(_query) :( + // NOTE: ngram matches inconsistently, research or remove //SUPPORT_ENCODER && (tokenizer === "ngram") ? /** @type {!Array} */ @@ -1238,7 +1474,7 @@ if(this.depth){ use_contextual = true; - ctx_root = words[0]; // TODO: iterate roots? + ctx_root = words[0]; // TODO: iterate roots check_words[ctx_root] = 1; } else{ @@ -1252,6 +1488,18 @@ if(!use_contextual || (ctx_map = this._ctx)[ctx_root]){ + let initial_z = 0; + + if(SUPPORT_DOCUMENTS && boost){ + + threshold = (threshold || 1) / boost; + + if(boost < 0){ + + initial_z = threshold; + } + } + for(let a = (use_contextual ? 1 : 0); a < length; a++){ const value = words[a]; @@ -1274,7 +1522,7 @@ let map_value; - for(let z = 0; z < (10 - threshold); z++){ + for(let z = initial_z; z < (10 - threshold); z++){ if((map_value = map[z][value])){ @@ -1324,7 +1572,7 @@ // Not handled by intersection: - result = intersect(check, limit, SUPPORT_SUGGESTIONS && this.suggest); + result = intersect(check, limit, SUPPORT_DOCUMENTS && this._doc, SUPPORT_SUGGESTIONS && this.suggest); // Handled by intersection: @@ -1380,7 +1628,7 @@ length = this._map[z][keys[i]].length; - // Note: 1 char values allocates 1 byte "Map (OneByteInternalizedString)" + // TODO: Proof if 1 char values allocates 1 byte "Map (OneByteInternalizedString)" bytes += length * 1 + keys[i].length * 2 + 4; words += length; chars += keys[i].length * 2; @@ -1419,8 +1667,6 @@ FlexSearch.prototype.clear = function(){ - // destroy + initialize index - return this.destroy().init(); }; @@ -1430,24 +1676,27 @@ FlexSearch.prototype.destroy = function(){ - // cleanup cache - if(SUPPORT_CACHE && this.cache){ this._cache.clear(); this._cache = null; } - // release references - - //this.filter = - //this.stemmer = - //this._scores = this._map = this._ctx = - this._ids = - /*this._stack = - this._stack_keys =*/ null; + this._ids = null; + + if(SUPPORT_DOCUMENTS && this.doc){ + + const index = this._doc["index"]; + + for(let i = 0; i < index.length; i++){ + + index.destroy(); + } + + this._doc = null; + } return this; }; @@ -1460,6 +1709,29 @@ FlexSearch.prototype.export = function(){ + if(SUPPORT_DOCUMENTS && this.doc){ + + const index = this.doc["index"]; + const length = index.length; + const payload = new Array(length + 1); + + let i = 0; + + for(; i < index.length; i++){ + + payload[i] = [ + + index[i]._map, + index[i]._ctx, + index[i]._ids + ]; + } + + payload[i] = this._doc; + + return JSON.stringify(payload); + } + return JSON.stringify([ this._map, @@ -1476,9 +1748,30 @@ payload = JSON.parse(payload); - this._map = payload[0]; - this._ctx = payload[1]; - this._ids = payload[2]; + if(SUPPORT_DOCUMENTS && this.doc){ + + const index = this.doc["index"]; + const length = index.length; + + for(let i = 0; i < length; i++){ + + const idx = index[i]; + + idx._map = payload[i][0]; + idx._ctx = payload[i][1]; + idx._ids = payload[i][2]; + idx._doc = payload[length]; + } + + this._doc = payload[length]; + } + else{ + + this._map = payload[0]; + this._ctx = payload[1]; + this._ids = payload[2]; + this._doc = payload[3]; + } }; } @@ -2302,11 +2595,12 @@ /** * @param {!Array>} arrays * @param {number=} limit + * @param {Object=} docs * @param {boolean=} suggest * @returns {Array} */ - function intersect(arrays, limit, suggest) { + function intersect(arrays, limit, docs, suggest) { let result = []; let suggestions; @@ -2363,7 +2657,7 @@ if(is_final_loop){ - result[count++] = tmp; + result[count++] = docs ? docs[index] : tmp; if(limit && (count === limit)){ @@ -2415,7 +2709,7 @@ for(i = 0, length = tmp.length; i < length; i++){ - result[count++] = tmp[i]; + result[count++] = docs ? docs["@" + tmp[i]] : tmp[i]; if(limit && (count === limit)){ @@ -2429,13 +2723,33 @@ } else if(length_z){ - result = arrays[0]; + if(docs){ - if(limit && /*result &&*/ (result.length > limit)){ + const arr = arrays[0]; + let length = arr.length; - // Note: do not modify the original index array! + if(limit && (limit < length)){ - result = result.slice(0, limit); + length = limit; + } + + result = new Array(length); + + for(let i = 0; i < length; i++){ + + result[i] = docs["@" + arr[i]]; + } + } + else{ + + result = arrays[0]; + + if(limit && /*result &&*/ (result.length > limit)){ + + // Note: do not modify the original index array! + + result = result.slice(0, limit); + } } // Note: handle references to the original index array @@ -2954,6 +3268,7 @@ "var SUPPORT_ASYNC = " + (SUPPORT_ASYNC ? "true" : "false") + ";" + "var SUPPORT_SERIALIZE = " + (SUPPORT_SERIALIZE ? "true" : "false") + ";" + "var SUPPORT_INFO = " + (SUPPORT_INFO ? "true" : "false") + ";" + + "var SUPPORT_DOCUMENTS = " + (SUPPORT_DOCUMENTS ? "true" : "false") + ";" + "var SUPPORT_WORKER = true;" ) + "(" + _worker.toString() + ")()" diff --git a/package.json b/package.json index 08bf044..9ff173d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "flexsearch", - "version": "0.3.62", + "version": "0.4.0", "description": "Next-Generation full text search library with zero dependencies.", "homepage": "https://github.com/nextapps-de/flexsearch/", "author": "Thomas Wilkerling", @@ -26,12 +26,12 @@ "url": "https://github.com/nextapps-de/flexsearch.git" }, "scripts": { - "build": "node compile RELEASE=min DEBUG=false PROFILER=false SUPPORT_WORKER=true SUPPORT_ENCODER=true SUPPORT_CACHE=true SUPPORT_ASYNC=true SUPPORT_PRESETS=true SUPPORT_SUGGESTIONS=true SUPPORT_SERIALIZE=true SUPPORT_INFO=true SUPPORT_LANG_DE=false SUPPORT_LANG_EN=false", - "build-light": "node compile RELEASE=light DEBUG=false PROFILER=false SUPPORT_WORKER=false SUPPORT_ENCODER=false SUPPORT_CACHE=false SUPPORT_ASYNC=false SUPPORT_PRESETS=false SUPPORT_SUGGESTIONS=false SUPPORT_SERIALIZE=false SUPPORT_INFO=false SUPPORT_LANG_DE=false SUPPORT_LANG_EN=false", - "build-compact": "node compile RELEASE=compact DEBUG=false PROFILER=false SUPPORT_WORKER=false SUPPORT_ENCODER=true SUPPORT_CACHE=false SUPPORT_ASYNC=true SUPPORT_PRESETS=true SUPPORT_SUGGESTIONS=false SUPPORT_SERIALIZE=false SUPPORT_INFO=false SUPPORT_LANG_DE=false SUPPORT_LANG_EN=false", - "build-custom": "node compile RELEASE=custom DEBUG=false PROFILER=false SUPPORT_WORKER=false SUPPORT_ENCODER=false SUPPORT_CACHE=false SUPPORT_ASYNC=false SUPPORT_PRESETS=false SUPPORT_SUGGESTIONS=false SUPPORT_SERIALIZE=false SUPPORT_INFO=false SUPPORT_LANG_DE=false SUPPORT_LANG_EN=false", - "build-es5": "node compile RELEASE=es5 DEBUG=true PROFILER=false SUPPORT_WORKER=true SUPPORT_ENCODER=true SUPPORT_CACHE=true SUPPORT_ASYNC=true SUPPORT_PRESETS=true SUPPORT_SUGGESTIONS=true SUPPORT_SERIALIZE=true SUPPORT_INFO=true SUPPORT_LANG_DE=false SUPPORT_LANG_EN=false LANGUAGE_OUT=ECMASCRIPT5_STRICT", - "build-node": "node compile RELEASE=node DEBUG=false PROFILER=false SUPPORT_WORKER=false SUPPORT_ENCODER=true SUPPORT_CACHE=true SUPPORT_ASYNC=true SUPPORT_PRESETS=true SUPPORT_SUGGESTIONS=true SUPPORT_SERIALIZE=true SUPPORT_INFO=true SUPPORT_LANG_DE=false SUPPORT_LANG_EN=false", + "build": "node compile RELEASE=min DEBUG=false PROFILER=false SUPPORT_WORKER=true SUPPORT_ENCODER=true SUPPORT_CACHE=true SUPPORT_ASYNC=true SUPPORT_PRESETS=true SUPPORT_SUGGESTIONS=true SUPPORT_SERIALIZE=true SUPPORT_INFO=true SUPPORT_DOCUMENTS=true SUPPORT_LANG_DE=false SUPPORT_LANG_EN=false", + "build-light": "node compile RELEASE=light DEBUG=false PROFILER=false SUPPORT_WORKER=false SUPPORT_ENCODER=false SUPPORT_CACHE=false SUPPORT_ASYNC=false SUPPORT_PRESETS=false SUPPORT_SUGGESTIONS=false SUPPORT_SERIALIZE=false SUPPORT_INFO=false SUPPORT_DOCUMENTS=false SUPPORT_LANG_DE=false SUPPORT_LANG_EN=false", + "build-compact": "node compile RELEASE=compact DEBUG=false PROFILER=false SUPPORT_WORKER=false SUPPORT_ENCODER=true SUPPORT_CACHE=false SUPPORT_ASYNC=true SUPPORT_PRESETS=true SUPPORT_SUGGESTIONS=false SUPPORT_SERIALIZE=false SUPPORT_INFO=false SUPPORT_DOCUMENTS=true SUPPORT_LANG_DE=false SUPPORT_LANG_EN=false", + "build-custom": "node compile RELEASE=custom DEBUG=false PROFILER=false SUPPORT_WORKER=false SUPPORT_ENCODER=false SUPPORT_CACHE=false SUPPORT_ASYNC=false SUPPORT_PRESETS=false SUPPORT_SUGGESTIONS=false SUPPORT_SERIALIZE=false SUPPORT_INFO=false SUPPORT_DOCUMENTS=false SUPPORT_LANG_DE=false SUPPORT_LANG_EN=false", + "build-es5": "node compile RELEASE=es5 DEBUG=true PROFILER=false SUPPORT_WORKER=true SUPPORT_ENCODER=true SUPPORT_CACHE=true SUPPORT_ASYNC=true SUPPORT_PRESETS=true SUPPORT_SUGGESTIONS=true SUPPORT_SERIALIZE=true SUPPORT_INFO=true SUPPORT_DOCUMENTS=true SUPPORT_LANG_DE=false SUPPORT_LANG_EN=false LANGUAGE_OUT=ECMASCRIPT5_STRICT", + "build-node": "node compile RELEASE=node DEBUG=false PROFILER=false SUPPORT_WORKER=false SUPPORT_ENCODER=true SUPPORT_CACHE=true SUPPORT_ASYNC=true SUPPORT_PRESETS=true SUPPORT_SUGGESTIONS=true SUPPORT_SERIALIZE=true SUPPORT_INFO=true SUPPORT_DOCUMENTS=true SUPPORT_LANG_DE=false SUPPORT_LANG_EN=false", "build-lang": "node compile RELEASE=lang", "build-all": "npm run build && npm run build-light && npm run build-compact && npm run build-es5 && npm run build-node && npm run build-lang", "test-production": "nyc --reporter=html --reporter=text mocha --timeout=3000 test --exit", @@ -49,6 +49,7 @@ "test/", "compile.js", "README.md", + "CHANGELOG.md", "LICENSE" ], "readme": "README.md", diff --git a/test/test.es6.js b/test/test.es6.js new file mode 100644 index 0000000..b909d40 --- /dev/null +++ b/test/test.es6.js @@ -0,0 +1,353 @@ +if(typeof module !== "undefined"){ + + var expect = require("chai").expect; +} + +module.exports = function(FlexSearch, env){ + + // ------------------------------------------------------------------------ + // Multi-Field Documents + // ------------------------------------------------------------------------ + + if(env === "") describe("Index Multi-Field Documents", function(){ + + var data = [{ + + data:{ + id: 0, + title: "Title 1", + body: "Body 1" + } + },{ + data:{ + id: 1, + title: "Title 2", + body: "Body 2" + } + },{ + data:{ + id: 2, + title: "Title 3", + body: "Body 3" + } + }]; + + var update = [{ + + data:{ + id: 0, + title: "Foo 1", + body: "Bar 1" + } + },{ + data:{ + id: 1, + title: "Foo 2", + body: "Bar 2" + } + },{ + data:{ + id: 2, + title: "Foo 3", + body: "Bar 3" + } + }]; + + it("Should have been indexed properly", function(){ + + var index = new FlexSearch({ + + doc: { + + id: "data:id", + field: [ + "data:title", + "data:body" + ] + } + }); + + index.add(data); + + expect(index.doc.index[0].length).to.equal(3); + expect(index.doc.index[1].length).to.equal(3); + + expect(index.search({field: "data:body", query: "body"})).to.have.members(data); + expect(index.search({field: "data:title", query: "title"})).to.have.members(data); + + expect(index.search({field: "data:body", query: "title"})).to.have.lengthOf(0); + expect(index.search({field: "data:title", query: "body"})).to.have.lengthOf(0); + + expect(index.search({field: ["data:title", "data:body"], query: "body"})).to.have.members(data); + expect(index.search({field: ["data:body", "data:title"], query: "title"})).to.have.members(data); + + expect(index.search({query: "body"})).to.have.members(data); + expect(index.search("title")).to.have.members(data); + + expect(index.search({ + + field: "data:title", + query: "title", + boost: 2 + + })).to.have.members(data); + + expect(index.search([{ + + field: "data:title", + query: "body", + boost: 2 + },{ + field: "data:body", + query: "body", + boost: 2 + + }])).to.have.members(data); + + expect(index.search("title", { + + field: "data:title", + boost: 2 + + })).to.have.members(data); + + expect(index.search("title", { + + field: "data:body", + boost: 2 + + })).to.have.lengthOf(0); + + expect(index.search("body", [{ + + field: "data:title", + boost: 2 + },{ + field: "data:body", + boost: 2 + + }])).to.have.members(data); + + index.update(update); + + expect(index.search("foo")).not.to.have.members(data); + expect(index.search("bar")).not.to.have.members(data); + expect(index.search("foo")).to.have.members(update); + expect(index.search("bar")).to.have.members(update); + + index.remove(update); + + expect(index.doc.index[0].length).to.equal(0); + expect(index.doc.index[1].length).to.equal(0); + }); + + it("Should have been boosted properly", function(){ + + var index = new FlexSearch({ + + tokenize: "strict", + depth: 3, + doc: { + id: "id", + field: ["title", "body"] + } + }); + + index.add([{ + + id: 0, + title: "1 2 3 4 5", + body: "1 2 3 4 5" + },{ + id: 1, + title: "1 2 3 4 5", + body: "1 2 5 4 3" // <-- body + },{ + id: 2, + title: "1 2 5 4 3", // <-- title + body: "1 2 3 4 5" + }]); + + expect(index.search([{ + + field: "title", + query: "5", + boost: 0.1 + },{ + field: "body", + query: "5", + boost: 9 + + }])[0].id).to.equal(1); + + expect(index.search([{ + + field: "title", + query: "5", + boost: 9 + },{ + field: "body", + query: "5", + boost: 0.1 + + }])[0].id).to.equal(2); + }); + + it("Should have been indexed properly (Async)", async function(){ + + var index = new FlexSearch({ + + async: true, + doc: { + + id: "data:id", + field: [ + "data:title", + "data:body" + ] + } + }); + + await index.add(data); + + expect(index.doc.index[0].length).to.equal(3); + expect(index.doc.index[1].length).to.equal(3); + + expect(await index.search({field: "data:body", query: "body"})).to.have.members(data); + expect(await index.search({field: "data:title", query: "title"})).to.have.members(data); + + expect(await index.search({field: "data:body", query: "title"})).to.have.lengthOf(0); + expect(await index.search({field: "data:title", query: "body"})).to.have.lengthOf(0); + + expect(await index.search({field: ["data:title", "data:body"], query: "body"})).to.have.members(data); + expect(await index.search({field: ["data:body", "data:title"], query: "title"})).to.have.members(data); + + expect(await index.search({query: "body"})).to.have.members(data); + expect(await index.search("title")).to.have.members(data); + + await index.update(update); + + expect(await index.search("foo")).not.to.have.members(data); + expect(await index.search("bar")).not.to.have.members(data); + expect(await index.search("foo")).to.have.members(update); + expect(await index.search("bar")).to.have.members(update); + + await index.remove(update); + + expect(await index.doc.index[0].length).to.equal(0); + expect(await index.doc.index[1].length).to.equal(0); + }); + + it("Should have been indexed properly (Worker)", async function(){ + + var index = new FlexSearch({ + + worker: 4, + async: true, + doc: { + + id: "data:id", + field: [ + "data:title", + "data:body" + ] + } + }); + + await index.add(data); + + expect(index.doc.index[0].length).to.equal(3); + expect(index.doc.index[1].length).to.equal(3); + + expect(await index.search({field: "data:body", query: "body"})).to.have.members(data); + expect(await index.search({field: "data:title", query: "title"})).to.have.members(data); + + expect(await index.search({field: "data:body", query: "title"})).to.have.lengthOf(0); + expect(await index.search({field: "data:title", query: "body"})).to.have.lengthOf(0); + + expect(await index.search({field: ["data:title", "data:body"], query: "body"})).to.have.members(data); + expect(await index.search({field: ["data:body", "data:title"], query: "title"})).to.have.members(data); + + expect(await index.search({query: "body"})).to.have.members(data); + expect(await index.search("title")).to.have.members(data); + + await index.update(update); + + expect(await index.search("foo")).not.to.have.members(data); + expect(await index.search("bar")).not.to.have.members(data); + expect(await index.search("foo")).to.have.members(update); + expect(await index.search("bar")).to.have.members(update); + + await index.remove(update); + + expect(await index.doc.index[0].length).to.equal(0); + expect(await index.doc.index[1].length).to.equal(0); + }); + }); + + // ------------------------------------------------------------------------ + // Export / Import + // ------------------------------------------------------------------------ + + if(env !== "light") describe("Export / Import", function(){ + + var data; + + it("Should have been exported properly", function(){ + + var index = new FlexSearch({ + tokenize: "reverse", + doc: { + id: "id", + field: "title" + } + }); + + index.add({id: 0, title: "foo"}); + index.add({id: 1, title: "bar"}); + index.add({id: 2, title: "foobar"}); + + if(env === ""){ + + expect(index.doc.index[0].length).to.equal(3); + + data = index.export(); + + expect(data).to.equal(JSON.stringify([ + [ + index.doc.index[0]._map, + index.doc.index[0]._ctx, + index.doc.index[0]._ids + ], + index._doc + ])); + } + else{ + + data = index.export(); + } + }); + + it("Should have been imported properly", function(){ + + var index = new FlexSearch("score", { + doc: { + id: "id", + field: "title" + } + }); + + index.import(data); + + if(env === ""){ + + expect(index.doc.index[0].length).to.equal(3); + } + + expect(index.search("foo")).to.have.lengthOf(2); + expect(index.search("bar")).to.have.lengthOf(2); + expect(index.search("foobar")).to.have.lengthOf(1); + expect(index.search("foobar")[0].id).to.equal(2); + }); + }); +}; \ No newline at end of file diff --git a/test/test.js b/test/test.js index 642b504..bfe87cc 100644 --- a/test/test.js +++ b/test/test.js @@ -1077,32 +1077,104 @@ describe("Relevance", function(){ // Suggestion Tests // ------------------------------------------------------------------------ -if(env !== "light"){ +if(env !== "light") describe("Suggestion", function(){ - describe("Suggestion", function(){ + it("Should have been suggested properly by relevance", function(){ - it("Should have been suggested properly by relevance", function(){ - - var index = new FlexSearch({ - encode: "advanced", - tokenize: "strict", - suggest: true - }); - - index.add(0, "1 2 3 2 4 1 5 3"); - index.add(1, "zero one two three four five six seven eight nine ten"); - index.add(2, "four two zero one three ten five seven eight six nine"); - - expect(index.search("1 3 4 7")).to.have.members([0]); - expect(index.search("1 3 9 7")).to.have.members([0]); - expect(index.search("one foobar two")).to.have.members([1, 2]); - expect(index.search("zero one foobar two foobar")).to.have.members([1, 2]); - //TODO - //expect(index.search("zero one foobar two foobar")[0]).to.equal(1); + var index = new FlexSearch({ + encode: "advanced", + tokenize: "strict", + suggest: true }); + + index.add(0, "1 2 3 2 4 1 5 3"); + index.add(1, "zero one two three four five six seven eight nine ten"); + index.add(2, "four two zero one three ten five seven eight six nine"); + + expect(index.search("1 3 4 7")).to.have.members([0]); + expect(index.search("1 3 9 7")).to.have.members([0]); + expect(index.search("one foobar two")).to.have.members([1, 2]); + expect(index.search("zero one foobar two foobar")).to.have.members([1, 2]); + //TODO + //expect(index.search("zero one foobar two foobar")[0]).to.equal(1); }); +}); + +// ------------------------------------------------------------------------ +// Multi-Field Documents +// ------------------------------------------------------------------------ + +if(!this._phantom){ + + require("./test.es6.js")(FlexSearch, env); } +// ------------------------------------------------------------------------ +// Export / Import +// ------------------------------------------------------------------------ + +if(env !== "light") describe("Export / Import", function(){ + + var data; + + it("Should have been exported properly", function(){ + + var index = new FlexSearch({ + tokenize: "reverse", + doc: { + id: "id", + field: "title" + } + }); + + index.add({id: 0, title: "foo"}); + index.add({id: 1, title: "bar"}); + index.add({id: 2, title: "foobar"}); + + if(env === ""){ + + expect(index.doc.index[0].length).to.equal(3); + + data = index.export(); + + expect(data).to.equal(JSON.stringify([ + [ + index.doc.index[0]._map, + index.doc.index[0]._ctx, + index.doc.index[0]._ids + ], + index._doc + ])); + } + else{ + + data = index.export(); + } + }); + + it("Should have been imported properly", function(){ + + var index = new FlexSearch("score", { + doc: { + id: "id", + field: "title" + } + }); + + index.import(data); + + if(env === ""){ + + expect(index.doc.index[0].length).to.equal(3); + } + + expect(index.search("foo")).to.have.lengthOf(2); + expect(index.search("bar")).to.have.lengthOf(2); + expect(index.search("foobar")).to.have.lengthOf(1); + expect(index.search("foobar")[0].id).to.equal(2); + }); +}); + // ------------------------------------------------------------------------ // Feature Tests // ------------------------------------------------------------------------