From 0386c3fc94b37c1020d85a105a08573ec41074fd Mon Sep 17 00:00:00 2001 From: Thomas Wilkerling Date: Mon, 24 May 2021 22:10:27 +0200 Subject: [PATCH] improve language handler --- dist/flexsearch.bundle.js | 53 ++-- dist/flexsearch.compact.js | 41 +-- dist/flexsearch.debug.js | 423 +++++++++++++++++------------- dist/flexsearch.es5.js | 77 +++--- dist/flexsearch.light.js | 23 +- dist/lang/arabic/default.min.js | 1 + dist/lang/at.min.js | 2 + dist/lang/cjk/default.min.js | 1 + dist/lang/cyrillic/default.min.js | 1 + dist/lang/de.min.js | 2 + dist/lang/en.min.js | 2 + dist/lang/latin/advanced.min.js | 2 + dist/lang/latin/balance.min.js | 1 + dist/lang/latin/default.min.js | 1 + dist/lang/latin/extra.min.js | 2 + dist/lang/latin/simple.min.js | 1 + dist/lang/us.min.js | 2 + package.json | 23 +- src/config.js | 5 +- src/index.js | 5 +- src/lang.js | 31 ++- src/lang/arabic/default.js | 6 + src/lang/cjk/default.js | 6 + src/lang/cyrillic/default.js | 6 + src/lang/latin/advanced.js | 64 +++-- src/lang/latin/balance.js | 111 +++++++- src/lang/latin/default.js | 10 +- src/lang/latin/extra.js | 37 +-- src/lang/latin/simple.js | 16 +- src/lang/latin/soundex.js | 135 ---------- src/type.js | 13 + src/webpack.js | 14 +- task/build.js | 24 +- 33 files changed, 627 insertions(+), 514 deletions(-) create mode 100644 dist/lang/arabic/default.min.js create mode 100644 dist/lang/at.min.js create mode 100644 dist/lang/cjk/default.min.js create mode 100644 dist/lang/cyrillic/default.min.js create mode 100644 dist/lang/de.min.js create mode 100644 dist/lang/en.min.js create mode 100644 dist/lang/latin/advanced.min.js create mode 100644 dist/lang/latin/balance.min.js create mode 100644 dist/lang/latin/default.min.js create mode 100644 dist/lang/latin/extra.min.js create mode 100644 dist/lang/latin/simple.min.js create mode 100644 dist/lang/us.min.js delete mode 100644 src/lang/latin/soundex.js diff --git a/dist/flexsearch.bundle.js b/dist/flexsearch.bundle.js index be4ba98..d3b51ce 100644 --- a/dist/flexsearch.bundle.js +++ b/dist/flexsearch.bundle.js @@ -5,29 +5,32 @@ * Licence: Apache-2.0 * https://github.com/nextapps-de/flexsearch */ -(function _f(self){'use strict';try{if(module)self=module}catch(e){}self._factory=_f;var u;const w=/[\W_]+/;function aa(a){if(a=a.toLowerCase())if(a&&this.C&&(a=y(a,this.C)),this.G&&1=b)return f.concat(l.slice(c,b-h+c));f=f.concat(c? -l.slice(c):l);h+=n;c=0}return f}function ha(a,b){const c=B(),d=B(),e=[];for(let f=0;f=e&&(f=e-1);this.s=e;this.threshold=f;this.u=b=c&&c.I||a.tokenize||"strict";this.depth="strict"===b&&h.depth;this.v=z(h.bidirectional,!0);this.m=g="memory"=== -a.optimize;this.j=z(a.fastupdate,!0);this.o=a.minlength||1;this.map=g?A(e-f):B();e=h.resolution||e;f=h.threshold||f;f>=e&&(f=e-1);this.h=e;this.l=f;this.g=g?A(e-f):B();this.F=c&&c.F||a.rtl;this.C=(b=a.matcher||d&&d.C)&&H(b,!1);this.G=(b=a.stemmer||d&&d.G)&&H(b,!0);if(c=b=a.filter||d&&d.filter){c=b;d=B();for(let k=0,l=c.length;k=this.o&&(n||!r[m])){var e=Math.min(this.s/l*p|0,p);if(eg;k--)f=m.substring(g,k),f.length>=this.o&&O(this,r,f,h,a,c)}break}case "reverse":if(2< -d){for(g=d-1;0=this.o&&O(this,r,f,e,a,c);f=""}case "forward":if(1=this.o&&O(this,r,f,e,a,c);break;default:if(O(this,r,m,e,a,c),n&&1=this.o&&!e[m]){if(e[m]=1,k=Math.min((this.h-g)/l*p+h|0,p+(h-1)),kf;O(this,v,t?f:m,k,a,c,t?m:f)}}else g=Math.min(g+1,l-p)}}}}this.j||(this.register[a]=1)}}return this}; -function O(a,b,c,d,e,f,g){let h=g?a.g:a.map;if(!b[c]||g&&!b[c][g])a.m&&(h=h[d]),g?(b=b[c]||(b[c]=B()),b[g]=1,h=h[g]||(h[g]=B())):b[c]=1,h=h[c]||(h[c]=[]),a.m||(h=h[d]||(h[d]=[])),f&&-1!==h.indexOf(e)||(h[h.length]=e,a.j&&(a=a.register[e]||(a.register[e]=[]),a[a.length]=h))} -u.search=function(a,b,c){D(a)?(c=a,a=c.query):D(b)&&(c=b);let d=[],e;var f=this.threshold;let g,h=0;if(c){b=c.limit;h=c.offset||0;f=z(c.threshold,f);var k=c.context;g=c.suggest}if(a&&(a=this.encode(a),e=a.length,1=this.o&&!c[v])if(this.m||g||this.map[v])l[r++]=v,c[v]=1;else return d;a=l;e=a.length}if(!e)return d;b||(b=100);c=this.s-f;f=this.h-f;k=this.depth&&1=e)));r++);if(q){if(g)return oa(l,e,0);b[b.length]=l;return}}return!c&&l}function oa(a,b,c){a=1===a.length?a[0]:[].concat.apply([],a);return c||a.length>b?a.slice(c,c+b):a} -function pa(a,b,c,d){c?(d=d&&b>c,a=(a=a[d?b:c])&&a[d?c:b]):a=a[b];return a}u.contain=function(a){return!!this.register[a]};u.update=function(a,b){return this.remove(a).add(a,b)}; -u.remove=function(a,b){const c=this.register[a];if(c){if(this.j)for(let d=0,e;d=b)return f.concat(l.slice(c,b-h+c));f=f.concat(c? +l.slice(c):l);h+=m;c=0}return f}function ma(a,b){const c=z(),d=z(),e=[];for(let f=0;f=e&&(f=e-1);this.C=e;this.threshold=f;this.D=b=c&&c.I||a.tokenize||"strict";this.depth="strict"===b&&h.depth;this.G=w(h.bidirectional,!0);this.A=g="memory"=== +a.optimize;this.o=w(a.fastupdate,!0);this.B=a.minlength||1;this.map=g?x(e-f):z();e=h.resolution||e;f=h.threshold||f;f>=e&&(f=e-1);this.l=e;this.s=f;this.h=g?x(e-f):z();this.F=c&&c.F||a.rtl;this.L=(b=a.matcher||d&&d.L)&&ca(b,!1);this.M=(b=a.stemmer||d&&d.M)&&ca(b,!0);if(c=b=a.filter||d&&d.filter){c=b;d=z();for(let k=0,l=c.length;k=this.B&&(m||!r[n])){var e=Math.min(this.C/l*p|0,p);if(eg;k--)f=n.substring(g,k),f.length>=this.B&&M(this,r,f,h,a,c)}break}case "reverse":if(2< +d){for(g=d-1;0=this.B&&M(this,r,f,e,a,c);f=""}case "forward":if(1=this.B&&M(this,r,f,e,a,c);break;default:if(M(this,r,n,e,a,c),m&&1=this.B&&!e[n]){if(e[n]=1,k=Math.min((this.l-g)/l*p+h|0,p+(h-1)),kf;M(this,v,t?f:n,k,a,c,t?n:f)}}else g=Math.min(g+1,l-p)}}}}this.o||(this.register[a]=1)}}return this}; +function M(a,b,c,d,e,f,g){let h=g?a.h:a.map;if(!b[c]||g&&!b[c][g])a.A&&(h=h[d]),g?(b=b[c]||(b[c]=z()),b[g]=1,h=h[g]||(h[g]=z())):b[c]=1,h=h[c]||(h[c]=[]),a.A||(h=h[d]||(h[d]=[])),f&&-1!==h.indexOf(e)||(h[h.length]=e,a.o&&(a=a.register[e]||(a.register[e]=[]),a[a.length]=h))} +u.search=function(a,b,c){B(a)?(c=a,a=c.query):B(b)&&(c=b);let d=[],e;var f=this.threshold;let g,h=0;if(c){b=c.limit;h=c.offset||0;f=w(c.threshold,f);var k=c.context;g=c.suggest}if(a&&(a=this.encode(a),e=a.length,1=this.B&&!c[v])if(this.A||g||this.map[v])l[r++]=v,c[v]=1;else return d;a=l;e=a.length}if(!e)return d;b||(b=100);c=this.C-f;f=this.l-f;k=this.depth&&1=e)));r++);if(q){if(g)return ta(l,e,0);b[b.length]=l;return}}return!c&&l}function ta(a,b,c){a=1===a.length?a[0]:[].concat.apply([],a);return c||a.length>b?a.slice(c,c+b):a} +function ua(a,b,c,d){c?(d=d&&b>c,a=(a=a[d?b:c])&&a[d?c:b]):a=a[b];return a}u.contain=function(a){return!!this.register[a]};u.update=function(a,b){return this.remove(a).add(a,b)}; +u.remove=function(a,b){const c=this.register[a];if(c){if(this.o)for(let d=0,e;db||c)e=e.slice(c,c+b);d&&(e=ua.call(this,e));return{tag:a,result:e}}}function ua(a){const b=Array(a.length);for(let c=0,d;cb||c)e=e.slice(c,c+b);d&&(e=za.call(this,e));return{tag:a,result:e}}}function za(a){const b=Array(a.length);for(let c=0,d;c=b)return f.concat(l.slice(c,b-h+c));f=f.concat(c? -l.slice(c):l);h+=n;c=0}return f}function fa(a,b){const c=B(),e=B(),d=[];for(let f=0;f=d&&(f=d-1);this.m=d;this.threshold=f;this.u=b=c&&c.H||a.tokenize||"strict";this.depth="strict"===b&&h.depth;this.v=z(h.bidirectional,!0);this.l=g="memory"=== -a.optimize;this.s=z(a.fastupdate,!0);this.j=a.minlength||1;this.g=g?A(d-f):B();d=h.resolution||d;f=h.threshold||f;f>=d&&(f=d-1);this.h=d;this.o=f;this.i=g?A(d-f):B();this.B=c&&c.B||a.rtl;this.A=(b=a.matcher||e&&e.A)&&G(b,!1);this.C=(b=a.stemmer||e&&e.C)&&G(b,!0);if(a=b=a.filter||e&&e.filter){a=b;c=B();for(let k=0,l=a.length;k=this.j&&(n||!r[m])){var d=Math.min(this.m/l*p|0,p);if(dg;k--)f=m.substring(g,k),f.length>=this.j&&Q(this,r,f,h,a,c)}break}case "reverse":if(2< -e){for(g=e-1;0=this.j&&Q(this,r,f,d,a,c);f=""}case "forward":if(1=this.j&&Q(this,r,f,d,a,c);break;default:if(Q(this,r,m,d,a,c),n&&1=this.j&&!d[m]){if(d[m]=1,k=Math.min((this.h-g)/l*p+h|0,p+(h-1)),kf;Q(this,u,t?f:m,k,a,c,t?m:f)}}else g=Math.min(g+1,l-p)}}}}this.s||(this.register[a]=1)}}return this}; -function Q(a,b,c,e,d,f,g){let h=g?a.i:a.g;if(!b[c]||g&&!b[c][g])a.l&&(h=h[e]),g?(b=b[c]||(b[c]=B()),b[g]=1,h=h[g]||(h[g]=B())):b[c]=1,h=h[c]||(h[c]=[]),a.l||(h=h[e]||(h[e]=[])),f&&-1!==h.indexOf(d)||(h[h.length]=d,a.s&&(a=a.register[d]||(a.register[d]=[]),a[a.length]=h))} -v.search=function(a,b,c){D(a)?(c=a,a=c.query):D(b)&&(c=b);let e=[],d;var f=this.threshold;let g,h=0;if(c){b=c.limit;h=c.offset||0;f=z(c.threshold,f);var k=c.context;g=c.suggest}if(a&&(a=this.encode(a),d=a.length,1=this.j&&!c[u])if(this.l||g||this.g[u])l[r++]=u,c[u]=1;else return e;a=l;d=a.length}if(!d)return e;b||(b=100);c=this.m-f;f=this.h-f;k=this.depth&&1=d)));r++);if(q){if(g)return ia(l,d,0);b[b.length]=l;return}}return!c&&l}function ia(a,b,c){a=1===a.length?a[0]:[].concat.apply([],a);return c||a.length>b?a.slice(c,c+b):a} -function ja(a,b,c,e){c?(e=e&&b>c,a=(a=a[e?b:c])&&a[e?c:b]):a=a[b];return a}v.contain=function(a){return!!this.register[a]};v.update=function(a,b){return this.remove(a).add(a,b)};v.remove=function(a,b){const c=this.register[a];if(c){if(this.s)for(let e=0,d;eb||c)d=d.slice(c,c+b);e&&(d=la.call(this,d));return{tag:a,result:d}}}function la(a){const b=Array(a.length);for(let c=0,e;c=b)return f.concat(l.slice(c,b-h+c));f=f.concat(c? +l.slice(c):l);h+=m;c=0}return f}function ka(a,b){const c=z(),e=z(),d=[];for(let f=0;f=d&&(f=d-1);this.A=d;this.threshold=f;this.F=b=c&&c.H||a.tokenize||"strict";this.depth="strict"===b&&h.depth;this.G=w(h.bidirectional,!0);this.s=g="memory"=== +a.optimize;this.D=w(a.fastupdate,!0);this.o=a.minlength||1;this.h=g?y(d-f):z();d=h.resolution||d;f=h.threshold||f;f>=d&&(f=d-1);this.l=d;this.B=f;this.m=g?y(d-f):z();this.C=c&&c.C||a.rtl;this.I=(b=a.matcher||e&&e.I)&&D(b,!1);this.J=(b=a.stemmer||e&&e.J)&&D(b,!0);if(a=b=a.filter||e&&e.filter){a=b;c=z();for(let k=0,l=a.length;k=this.o&&(m||!r[n])){var d=Math.min(this.A/l*p|0,p);if(dg;k--)f=n.substring(g,k),f.length>=this.o&&Q(this,r,f,h,a,c)}break}case "reverse":if(2< +e){for(g=e-1;0=this.o&&Q(this,r,f,d,a,c);f=""}case "forward":if(1=this.o&&Q(this,r,f,d,a,c);break;default:if(Q(this,r,n,d,a,c),m&&1=this.o&&!d[n]){if(d[n]=1,k=Math.min((this.l-g)/l*p+h|0,p+(h-1)),kf;Q(this,u,t?f:n,k,a,c,t?n:f)}}else g=Math.min(g+1,l-p)}}}}this.D||(this.register[a]=1)}}return this}; +function Q(a,b,c,e,d,f,g){let h=g?a.m:a.h;if(!b[c]||g&&!b[c][g])a.s&&(h=h[e]),g?(b=b[c]||(b[c]=z()),b[g]=1,h=h[g]||(h[g]=z())):b[c]=1,h=h[c]||(h[c]=[]),a.s||(h=h[e]||(h[e]=[])),f&&-1!==h.indexOf(d)||(h[h.length]=d,a.D&&(a=a.register[d]||(a.register[d]=[]),a[a.length]=h))} +v.search=function(a,b,c){B(a)?(c=a,a=c.query):B(b)&&(c=b);let e=[],d;var f=this.threshold;let g,h=0;if(c){b=c.limit;h=c.offset||0;f=w(c.threshold,f);var k=c.context;g=c.suggest}if(a&&(a=this.encode(a),d=a.length,1=this.o&&!c[u])if(this.s||g||this.h[u])l[r++]=u,c[u]=1;else return e;a=l;d=a.length}if(!d)return e;b||(b=100);c=this.A-f;f=this.l-f;k=this.depth&&1=d)));r++);if(q){if(g)return na(l,d,0);b[b.length]=l;return}}return!c&&l}function na(a,b,c){a=1===a.length?a[0]:[].concat.apply([],a);return c||a.length>b?a.slice(c,c+b):a} +function oa(a,b,c,e){c?(e=e&&b>c,a=(a=a[e?b:c])&&a[e?c:b]):a=a[b];return a}v.contain=function(a){return!!this.register[a]};v.update=function(a,b){return this.remove(a).add(a,b)};v.remove=function(a,b){const c=this.register[a];if(c){if(this.D)for(let e=0,d;eb||c)d=d.slice(c,c+b);e&&(d=qa.call(this,d));return{tag:a,result:d}}}function qa(a){const b=Array(a.length);for(let c=0,e;c= b) { + for (let h = d.length - 1, l, m; 0 <= h; h--) { + if ((m = (l = d[h]) && l.length) && c && (m <= c ? (c -= m, m = 0) : m -= c), m) { + if (k + m >= b) { return f.concat(l.slice(c, b - k + c)); } f = f.concat(c ? l.slice(c) : l); - k += n; + k += m; c = 0; } } } return f; } -function ea(a, b) { +function ja(a, b) { const c = y(), d = y(), e = []; for (let f = 0; f < a.length; f++) { c[a[f]] = 1; @@ -138,18 +153,18 @@ function ea(a, b) { } return e; } -;function N(a) { +;function L(a) { this.limit = !0 !== a && a; this.cache = y(); this.queue = []; } -function fa(a, b, c) { +function ka(a, b, c) { A(a) && (a = a.query); let d = this.cache.get(a); d || (d = this.search(a, b, c), this.cache.set(a, d)); return d; } -N.prototype.set = function(a, b) { +L.prototype.set = function(a, b) { if (!this.cache[a]) { var c = this.queue.length; c === this.limit ? delete this.cache[this.queue[c - 1]] : c++; @@ -160,7 +175,7 @@ N.prototype.set = function(a, b) { } this.cache[a] = b; }; -N.prototype.get = function(a) { +L.prototype.get = function(a) { const b = this.cache[a]; if (this.limit && b && (a = this.queue.indexOf(a))) { const c = this.queue[a - 1]; @@ -169,13 +184,13 @@ N.prototype.get = function(a) { } return b; }; -N.prototype.del = function(a) { +L.prototype.del = function(a) { for (let b = 0, c, d; b < this.queue.length; b++) { d = this.queue[b], c = this.cache[d], -1 !== c.indexOf(a) && (this.queue.splice(b--, 1), delete this.cache[d]); } }; -const ha = {memory:{charset:"latin:extra", resolution:3, minlength:3, fastupdate:!1, optimize:"memory"}, performance:{threshold:8, minlength:3, context:{depth:1, bidirectional:!0}}, match:{charset:"latin:extra", tokenize:"full", resolution:3, }, score:{charset:"latin:advanced", threshold:1, context:{depth:3, bidirectional:!0}}, "default":{resolution:3, threshold:0, depth:3}, }; -function ka(a, b, c, d, e, f) { +const la = {memory:{charset:"latin:extra", resolution:3, minlength:3, fastupdate:!1, optimize:"memory"}, performance:{threshold:8, minlength:3, context:{depth:1, bidirectional:!0}}, match:{charset:"latin:extra", tokenize:"full", resolution:3, }, score:{charset:"latin:advanced", threshold:1, context:{depth:3, bidirectional:!0}}, "default":{resolution:3, threshold:0, depth:3}, }; +function oa(a, b, c, d, e, f) { setTimeout(function() { const g = a(c, JSON.stringify(f)); g && g.then ? g.then(function() { @@ -183,14 +198,15 @@ function ka(a, b, c, d, e, f) { }) : b.export(a, b, c, d, e + 1); }); } -;function O(a, b) { - if (!(this instanceof O)) { - return new O(a); +;J["latin:default"] = da; +function M(a, b) { + if (!(this instanceof M)) { + return new M(a); } var c; if (a) { if (z(a)) { - ha[a] || console.warn("Preset not found: " + a), a = ha[a]; + la[a] || console.warn("Preset not found: " + a), a = la[a]; } else { if (c = a.preset) { c[c] || console.warn("Preset not found: " + c), a = Object.assign({}, c[c], a); @@ -198,13 +214,13 @@ function ka(a, b, c, d, e, f) { } c = a.charset; var d = a.lang; - z(c) && (-1 === c.indexOf(":") && (c += ":default"), c = H[c]); - z(d) && (d = G[d]); + z(c) && (-1 === c.indexOf(":") && (c += ":default"), c = J[c]); + z(d) && (d = ea[d]); } else { a = {}; } let e, f, g, k = a.context || {}; - this.encode = a.encode || c && c.encode || ba; + this.encode = a.encode || c && c.encode || da; this.register = b || y(); e = a.resolution || 9; f = a.threshold || 0; @@ -217,16 +233,16 @@ function ka(a, b, c, d, e, f) { this.optimize = g = "memory" === a.optimize; this.fastupdate = v(a.fastupdate, !0); this.minlength = a.minlength || 1; - this.map = g ? x(e - f) : y(); + this.map = g ? w(e - f) : y(); e = k.resolution || e; f = k.threshold || f; f >= e && (f = e - 1); this.resolution_ctx = e; this.threshold_ctx = f; - this.ctx = g ? x(e - f) : y(); + this.ctx = g ? w(e - f) : y(); this.rtl = c && c.rtl || a.rtl; - this.matcher = (b = a.matcher || d && d.matcher) && C(b, !1); - this.stemmer = (b = a.stemmer || d && d.stemmer) && C(b, !0); + this.matcher = (b = a.matcher || d && d.matcher) && D(b, !1); + this.stemmer = (b = a.stemmer || d && d.stemmer) && D(b, !0); if (c = b = a.filter || d && d.filter) { c = b; d = y(); @@ -236,41 +252,29 @@ function ka(a, b, c, d, e, f) { c = d; } this.filter = c; - this.cache = (b = a.cache) && new N(b); + this.cache = (b = a.cache) && new L(b); } -O.prototype.pipeline = function(a, b, c, d) { - if (a) { - b && a && (a = D(a, b)); - a && this.matcher && (a = D(a, this.matcher)); - this.stemmer && 1 < a.length && (a = D(a, this.stemmer)); - if (d && 1 < a.length) { - d = b = ""; - for (let e = 0, f = a.length, g; e < f; e++) { - (g = a[e]) !== d && (b += d = g); +M.prototype.pipeline = function(a, b, c, d) { + if (a && (b && (a = F(a, b)), this.matcher && (a = F(a, this.matcher)), this.stemmer && 1 < a.length && (a = F(a, this.stemmer)), d && 1 < a.length && (a = I(a)), c || "" === c)) { + a = a.split(c); + if (this.filter) { + b = this.filter; + c = a.length; + d = []; + for (let e = 0, f = 0; e < c; e++) { + const g = a[e]; + g && !b[g] && (d[f++] = g); } - a = b; - } - if (a && (c || "" === c)) { - a = a.split(c); - if (this.filter) { - c = this.filter; - b = a.length; - d = []; - for (let e = 0, f = 0; e < b; e++) { - const g = a[e]; - g && !c[g] && (d[f++] = g); - } - a = d; - } - return a; + a = d; } + return a; } return a; }; -O.prototype.append = function(a, b) { +M.prototype.append = function(a, b) { return this.add(a, b, !0); }; -O.prototype.add = function(a, b, c) { +M.prototype.add = function(a, b, c) { if (this.register[a] && !c) { return this.update(a, b); } @@ -278,11 +282,11 @@ O.prototype.add = function(a, b, c) { b = this.encode(b); const l = b.length; if (l) { - const n = this.depth, q = this.resolution - this.threshold, r = y(), p = y(); + const m = this.depth, q = this.resolution - this.threshold, r = y(), p = y(); for (let t = 0; t < l; t++) { - let m = b[this.rtl ? l - 1 - t : t]; - var d = m.length; - if (m && d >= this.minlength && (n || !r[m])) { + let n = b[this.rtl ? l - 1 - t : t]; + var d = n.length; + if (n && d >= this.minlength && (m || !r[n])) { var e = Math.min(this.resolution / l * t | 0, t); if (e < q) { var f = ""; @@ -293,7 +297,7 @@ O.prototype.add = function(a, b, c) { var k = g ? Math.min(e / 2 + this.resolution / d * g / 2 | 0, e + g) : e; if (k < q) { for (var h = d; h > g; h--) { - f = m.substring(g, h), f.length >= this.minlength && this.push_index(r, f, k, a, c); + f = n.substring(g, h), f.length >= this.minlength && this.push_index(r, f, k, a, c); } } } @@ -302,24 +306,24 @@ O.prototype.add = function(a, b, c) { case "reverse": if (2 < d) { for (g = d - 1; 0 < g; g--) { - f = m[g] + f, f.length >= this.minlength && this.push_index(r, f, e, a, c); + f = n[g] + f, f.length >= this.minlength && this.push_index(r, f, e, a, c); } f = ""; } case "forward": if (1 < d) { for (g = 0; g < d; g++) { - f += m[g], f.length >= this.minlength && this.push_index(r, f, e, a, c); + f += n[g], f.length >= this.minlength && this.push_index(r, f, e, a, c); } } break; default: - if (this.push_index(r, m, e, a, c), n && 1 < l && t < l - 1) { - for (d = this.resolution_ctx - this.threshold_ctx, e = y(), f = m, g = Math.min(n + 1, l - t), e[f] = 1, k = 1; k < g; k++) { - if ((m = b[this.rtl ? l - 1 - t - k : t + k]) && m.length >= this.minlength && !e[m]) { - if (e[m] = 1, h = Math.min((this.resolution_ctx - g) / l * t + k | 0, t + (k - 1)), h < d) { - const u = this.bidirectional && m > f; - this.push_index(p, u ? f : m, h, a, c, u ? m : f); + if (this.push_index(r, n, e, a, c), m && 1 < l && t < l - 1) { + for (d = this.resolution_ctx - this.threshold_ctx, e = y(), f = n, g = Math.min(m + 1, l - t), e[f] = 1, k = 1; k < g; k++) { + if ((n = b[this.rtl ? l - 1 - t - k : t + k]) && n.length >= this.minlength && !e[n]) { + if (e[n] = 1, h = Math.min((this.resolution_ctx - g) / l * t + k | 0, t + (k - 1)), h < d) { + const u = this.bidirectional && n > f; + this.push_index(p, u ? f : n, h, a, c, u ? n : f); } } else { g = Math.min(g + 1, l - t); @@ -335,13 +339,13 @@ O.prototype.add = function(a, b, c) { } return this; }; -O.prototype.push_index = function(a, b, c, d, e, f) { +M.prototype.push_index = function(a, b, c, d, e, f) { let g = f ? this.ctx : this.map; if (!a[b] || f && !a[b][f]) { this.optimize && (g = g[c]), f ? (a = a[b] || (a[b] = y()), a[f] = 1, g = g[f] || (g[f] = y())) : a[b] = 1, g = g[b] || (g[b] = []), this.optimize || (g = g[c] || (g[c] = [])), e && -1 !== g.indexOf(d) || (g[g.length] = d, this.fastupdate && (a = this.register[d] || (this.register[d] = []), a[a.length] = g)); } }; -O.prototype.search = function(a, b, c) { +M.prototype.search = function(a, b, c) { A(a) ? (c = a, a = c.query) : A(b) && (c = b); let d = [], e; var f = this.threshold; @@ -376,11 +380,11 @@ O.prototype.search = function(a, b, c) { f = this.resolution_ctx - f; h = this.depth && 1 < e && !1 !== h; l = 0; - let n; - h ? (n = a[0], l = 1) : 1 < e && a.sort(ca); + let m; + h ? (m = a[0], l = 1) : 1 < e && a.sort(aa); for (let q, r; l < e; l++) { r = a[l]; - h ? (q = this.add_result(d, g, f, b, k, 2 === e, r, n), g && !1 === q && d.length || (n = r)) : q = this.add_result(d, g, c, b, k, 1 === e, r); + h ? (q = this.add_result(d, g, f, b, k, 2 === e, r, m), g && !1 === q && d.length || (m = r)) : q = this.add_result(d, g, c, b, k, 1 === e, r); if (q) { return q; } @@ -395,23 +399,23 @@ O.prototype.search = function(a, b, c) { return d; } if (1 === p) { - return la(d[0], b, k); + return pa(d[0], b, k); } } } - return da(d, b, k, g); + return ia(d, b, k, g); }; -O.prototype.add_result = function(a, b, c, d, e, f, g, k) { +M.prototype.add_result = function(a, b, c, d, e, f, g, k) { let h = [], l = k ? this.ctx : this.map; - this.optimize || (l = ma(l, g, k, this.bidirectional)); + this.optimize || (l = qa(l, g, k, this.bidirectional)); if (l) { - let n = 0; + let m = 0; c = Math.min(l.length, c); - for (let q = 0, r = 0, p, t; q < c && !(p = l[q], this.optimize && (p = ma(p, g, k, this.bidirectional)), p && f && (t = p.length, t <= e ? (e -= t, p = null) : e && (p = p.slice(e), e = 0)), p && (h[n++] = p, f && (r += p.length, r >= d))); q++) { + for (let q = 0, r = 0, p, t; q < c && !(p = l[q], this.optimize && (p = qa(p, g, k, this.bidirectional)), p && f && (t = p.length, t <= e ? (e -= t, p = null) : e && (p = p.slice(e), e = 0)), p && (h[m++] = p, f && (r += p.length, r >= d))); q++) { } - if (n) { + if (m) { if (f) { - return la(h, d, 0); + return pa(h, d, 0); } a[a.length] = h; return; @@ -419,21 +423,21 @@ O.prototype.add_result = function(a, b, c, d, e, f, g, k) { } return !b && h; }; -function la(a, b, c) { +function pa(a, b, c) { a = 1 === a.length ? a[0] : [].concat.apply([], a); return c || a.length > b ? a.slice(c, c + b) : a; } -function ma(a, b, c, d) { +function qa(a, b, c, d) { c ? (d = d && b > c, a = (a = a[d ? b : c]) && a[d ? c : b]) : a = a[b]; return a; } -O.prototype.contain = function(a) { +M.prototype.contain = function(a) { return !!this.register[a]; }; -O.prototype.update = function(a, b) { +M.prototype.update = function(a, b) { return this.remove(a).add(a, b); }; -O.prototype.remove = function(a, b) { +M.prototype.remove = function(a, b) { const c = this.register[a]; if (c) { if (this.fastupdate) { @@ -468,8 +472,8 @@ function P(a, b, c, d, e) { } return f; } -O.prototype.searchCache = fa; -O.prototype.export = function(a, b, c, d, e) { +M.prototype.searchCache = ka; +M.prototype.export = function(a, b, c, d, e) { let f, g; switch(e || (e = 0)) { case 0: @@ -498,10 +502,10 @@ O.prototype.export = function(a, b, c, d, e) { default: return; } - ka(a, b || this, c ? c + "." + f : f, d, e, g); + oa(a, b || this, c ? c + "." + f : f, d, e, g); return !0; }; -O.prototype.import = function(a, b) { +M.prototype.import = function(a, b) { if (b) { switch(z(b) && (b = JSON.parse(b)), a) { case "cfg": @@ -519,8 +523,8 @@ O.prototype.import = function(a, b) { } } }; -J(O.prototype); -function na(a) { +ha(M.prototype); +function ra(a) { var b = self._index; a = a.data; var c = a.args; @@ -532,7 +536,7 @@ function na(a) { c = b.encode; b.cache = !1; c && 0 === c.indexOf("function") && (b.encode = Function("return " + c)()); - a ? (Function("return " + a)()(self), self._index = new self.FlexSearch.Index(b), delete self.FlexSearch) : self._index = new O(b); + a ? (Function("return " + a)()(self), self._index = new self.FlexSearch.Index(b), delete self.FlexSearch) : self._index = new M(b); break; case "search": b = b.search.apply(b, c); @@ -547,7 +551,7 @@ function na(a) { A(a) ? b = a : b ? B(c = b.encode) && (b.encode = c.toString()) : b = {}; (a = (self || window)._factory) && (a = a.toString()); c = self.exports; - if (this.worker = oa(a, c)) { + if (this.worker = sa(a, c)) { this.resolver = null; var d = this; if (c) { @@ -562,18 +566,18 @@ function na(a) { this.worker.postMessage({task:"init", factory:a, options:b}); } } -S("add"); -S("append"); -S("search"); -S("update"); -S("remove"); -function S(a) { +R("add"); +R("append"); +R("search"); +R("update"); +R("remove"); +function R(a) { Q.prototype[a] = Q.prototype[a + "Async"] = function() { const b = this, c = [].slice.call(arguments); var d = c[c.length - 1]; let e; B(d) && (e = d, c.splice(c.length - 1, 1)); - d = new I(function(f) { + d = new fa(function(f) { setTimeout(function() { b.worker.postMessage({task:a, args:c}); "search" === a ? b.resolver = f : f(); @@ -582,17 +586,17 @@ function S(a) { return e ? (d.then(e), this) : d; }; } -function oa(a, b) { +function sa(a, b) { let c; try { - c = b ? eval('new (require("worker_threads")["Worker"])("../dist/node/node.js")') : a ? new Worker(URL.createObjectURL(new Blob(["onmessage=" + na.toString()], {type:"text/javascript"}))) : new Worker("worker.js", {type:"module"}); + c = b ? eval('new (require("worker_threads")["Worker"])("../dist/node/node.js")') : a ? new Worker(URL.createObjectURL(new Blob(["onmessage=" + ra.toString()], {type:"text/javascript"}))) : new Worker("worker.js", {type:"module"}); } catch (d) { } return c; } -;function T(a) { - if (!(this instanceof T)) { - return new T(a); +;function S(a) { + if (!(this instanceof S)) { + return new S(a); } var b; a || (a = {}); @@ -600,14 +604,14 @@ function oa(a, b) { this.field = []; this.marker = []; this.register = y(); - this.key = (b = a.key) && U(b, this.marker) || "id"; + this.key = (b = a.key) && T(b, this.marker) || "id"; this.fastupdate = v(a.fastupdate, !0); this.extern = !!(b = a.extern); this.storetree = !this.extern && (b = a.store) && !0 !== b && []; this.store = b && (this.extern ? b : y()); - this.tag = (b = a.tag) && U(b, this.marker); + this.tag = (b = a.tag) && T(b, this.marker); this.tagindex = b && y(); - this.cache = (b = a.cache) && new N(b); + this.cache = (b = a.cache) && new L(b); a.cache = !1; this.worker = a.worker; this.async = !1; @@ -623,16 +627,16 @@ function oa(a, b) { } } for (let e = 0, f, g; e < c.length; e++) { - f = c[e], z(f) ? d && (g = d[f]) : (g = f, f = f.field), g = A(g) ? Object.assign({}, b, g) : b, this.worker && (a[f] = new Q(g), a[f].worker || (this.worker = !1)), this.worker || (a[f] = new O(g, this.register)), this.tree[e] = U(f, this.marker), this.field[e] = f; + f = c[e], z(f) ? d && (g = d[f]) : (g = f, f = f.field), g = A(g) ? Object.assign({}, b, g) : b, this.worker && (a[f] = new Q(g), a[f].worker || (this.worker = !1)), this.worker || (a[f] = new M(g, this.register)), this.tree[e] = T(f, this.marker), this.field[e] = f; } if (this.storetree) { for (d = b.store, z(d) && (d = [d]), b = 0; b < d.length; b++) { - this.storetree[b] = U(d[b], this.marker); + this.storetree[b] = T(d[b], this.marker); } } this.index = a; } -function U(a, b) { +function T(a, b) { const c = a.split(":"); let d = 0; for (let e = 0; e < c.length; e++) { @@ -641,7 +645,7 @@ function U(a, b) { d < c.length && (c.length = d); return 1 < d ? c : c[0]; } -function pa(a, b) { +function ta(a, b) { if (z(b)) { a = a[b]; } else { @@ -692,8 +696,8 @@ function W(a, b, c, d, e, f, g, k) { } } } -T.prototype.add = function(a, b, c) { - A(a) && (b = a, a = pa(b, this.key)); +S.prototype.add = function(a, b, c) { + A(a) && (b = a, a = ta(b, this.key)); if (b && (a || 0 === a)) { if (this.register[a]) { return this.update(a, b); @@ -702,7 +706,7 @@ T.prototype.add = function(a, b, c) { f = this.field[d], e = this.tree[d], z(e) && (e = [e]), W(b, e, this.marker, 0, this.index[f], a, e[0], c); } if (this.tag) { - let d = pa(b, this.tag), e = y(); + let d = ta(b, this.tag), e = y(); z(d) && (d = [d]); for (let f = 0, g, k; f < d.length; f++) { if (g = d[f], !e[g] && (e[g] = 1, k = this.tagindex[g] || (this.tagindex[g] = []), !c || -1 === k.indexOf(a))) { @@ -726,13 +730,13 @@ T.prototype.add = function(a, b, c) { } return this; }; -T.prototype.append = function(a, b) { +S.prototype.append = function(a, b) { return this.add(a, b, !0); }; -T.prototype.update = function(a, b) { +S.prototype.update = function(a, b) { return this.remove(a).add(a, b); }; -T.prototype.remove = function(a) { +S.prototype.remove = function(a) { A(a) && (a = a[this.key]); if (this.register[a]) { for (var b = 0; b < this.field.length && (this.index[this.field[b]].remove(a, !0), !this.fastupdate || this.worker); b++) { @@ -749,16 +753,16 @@ T.prototype.remove = function(a) { } return this; }; -T.prototype.search = function(a, b, c, d) { +S.prototype.search = function(a, b, c, d) { A(a) ? (c = a, a = c.query) : A(b) && (c = b, b = 0); - let e = [], f = [], g, k, h, l, n, q, r, p = 0; + let e = [], f = [], g, k, h, l, m, q, r, p = 0; if (c) { if (c.constructor === Array) { h = c, c = null; } else { - if (h = (g = c.pluck) || c.field || c.doc, n = c.tag, k = this.store && c.enrich, q = "and" === c.bool, b = c.limit || 100, r = c.offset || 0, h && (z(h) ? h = [h] : h.constructor !== Array && (l = h, h = Object.keys(h))), n && (z(n) && (n = [n]), !a)) { - for (let m = 0, u; m < n.length; m++) { - if (u = qa.call(this, n[m], b, r, k)) { + if (h = (g = c.pluck) || c.field || c.doc, m = c.tag, k = this.store && c.enrich, q = "and" === c.bool, b = c.limit || 100, r = c.offset || 0, h && (z(h) ? h = [h] : h.constructor !== Array && (l = h, h = Object.keys(h))), m && (z(m) && (m = [m]), !a)) { + for (let n = 0, u; n < m.length; n++) { + if (u = ua.call(this, m[n], b, r, k)) { e[e.length] = u, p++; } } @@ -767,30 +771,30 @@ T.prototype.search = function(a, b, c, d) { } } h || (h = this.field); - q = q && (1 < h.length || n && 1 < n.length); + q = q && (1 < h.length || m && 1 < m.length); const t = !d && (this.worker || this.async) && []; - for (let m = 0, u, w, E; m < h.length; m++) { - let K; - w = h[m]; - z(w) ? l && (K = l[w]) : (K = w, w = w.field); + for (let n = 0, u, x, G; n < h.length; n++) { + let N; + x = h[n]; + z(x) ? l && (N = l[x]) : (N = x, x = x.field); if (t) { - t[m] = this.index[w].searchAsync(a, b, K || c); + t[n] = this.index[x].searchAsync(a, b, N || c); } else { - u = d ? d[m] : this.index[w].search(a, b, K || c); - E = u.length; - if (n && E) { - const F = []; - let ia = 0; - q && (F[0] = [u]); - for (let R = 0, ja, L; R < n.length; R++) { - if (ja = n[R], E = (L = this.tagindex[ja]) && L.length) { - ia++, F[F.length] = q ? [L] : L; + u = d ? d[n] : this.index[x].search(a, b, N || c); + G = u.length; + if (m && G) { + const H = []; + let ma = 0; + q && (H[0] = [u]); + for (let U = 0, na, O; U < m.length; U++) { + if (na = m[U], G = (O = this.tagindex[na]) && O.length) { + ma++, H[H.length] = q ? [O] : O; } } - ia && (u = q ? da(F, b || 100, r || 0) : ea(u, F), E = u.length); + ma && (u = q ? ia(H, b || 100, r || 0) : ja(u, H), G = u.length); } - if (E) { - f[p] = w, e[p++] = u; + if (G) { + f[p] = x, e[p++] = u; } else { if (q) { return []; @@ -799,10 +803,10 @@ T.prototype.search = function(a, b, c, d) { } } if (t) { - const m = this; + const n = this; return new Promise(function(u) { - Promise.all(t).then(function(w) { - u(m.search(a, b, c, w)); + Promise.all(t).then(function(x) { + u(n.search(a, b, c, x)); }); }); } @@ -812,45 +816,45 @@ T.prototype.search = function(a, b, c, d) { if (g && (!k || !this.store)) { return e[0]; } - for (let m = 0, u; m < f.length; m++) { - u = e[m]; - u.length && k && (u = ra.call(this, u)); + for (let n = 0, u; n < f.length; n++) { + u = e[n]; + u.length && k && (u = va.call(this, u)); if (g) { return u; } - e[m] = {field:f[m], result:u}; + e[n] = {field:f[n], result:u}; } return e; }; -function qa(a, b, c, d) { +function ua(a, b, c, d) { let e = this.tagindex[a], f = e && e.length - c; if (f && 0 < f) { if (f > b || c) { e = e.slice(c, c + b); } - d && (e = ra.call(this, e)); + d && (e = va.call(this, e)); return {tag:a, result:e}; } } -function ra(a) { +function va(a) { const b = Array(a.length); for (let c = 0, d; c < a.length; c++) { d = a[c], b[c] = {key:d, doc:this.store[d]}; } return b; } -T.prototype.contain = function(a) { +S.prototype.contain = function(a) { return !!this.register[a]; }; -T.prototype.get = function(a) { +S.prototype.get = function(a) { return this.store[a]; }; -T.prototype.set = function(a, b) { +S.prototype.set = function(a, b) { this.store[a] = b; return this; }; -T.prototype.searchCache = fa; -T.prototype.export = function(a, b, c, d, e) { +S.prototype.searchCache = ka; +S.prototype.export = function(a, b, c, d, e) { e || (e = 0); d || (d = 0); if (d < this.field.length) { @@ -873,10 +877,10 @@ T.prototype.export = function(a, b, c, d, e) { default: return; } - ka(a, this, c, d, e, f); + oa(a, this, c, d, e, f); } }; -T.prototype.import = function(a, b) { +S.prototype.import = function(a, b) { if (b) { switch(z(b) && (b = JSON.parse(b)), a) { case "tag": @@ -900,13 +904,56 @@ T.prototype.import = function(a, b) { } } }; -J(T.prototype); +ha(S.prototype); +var xa = {encode:wa, rtl:!1, tokenize:""}; +const ya = /[\W_]+/, za = E("[\u00e0\u00e1\u00e2\u00e3\u00e4\u00e5]"), Aa = E("[\u00e8\u00e9\u00ea\u00eb]"), Ba = E("[\u00ec\u00ed\u00ee\u00ef]"), Ca = E("[\u00f2\u00f3\u00f4\u00f5\u00f6\u0151]"), Da = E("[\u00f9\u00fa\u00fb\u00fc\u0171]"), Ea = E("[\u00fd\u0177\u00ff]"), Fa = E("\u00f1"), Ga = E("[\u00e7c]"), Ha = E("\u00df"), Ia = E(" & "), Ja = [za, "a", Aa, "e", Ba, "i", Ca, "o", Da, "u", Ea, "y", Fa, "n", Ga, "k", Ha, "s", Ia, " and "]; +function wa(a) { + return this.pipeline(C(a).toLowerCase(), !a.normalize && Ja, ya, !1); +} +;var La = {encode:Ka, rtl:!1, tokenize:"strict"}; +const Ma = /[^a-z0-9]+/, Na = {b:"p", v:"f", w:"f", z:"s", x:"s", "\u00df":"s", d:"t", n:"m", c:"k", g:"k", j:"k", q:"k", i:"e", y:"e", u:"o"}; +function Ka(a) { + a = wa.call(this, a).join(" "); + const b = []; + if (a) { + const c = a.split(Ma), d = c.length; + for (let e = 0, f, g = 0; e < d; e++) { + if ((a = c[e]) && (!this.filter || !this.filter[a])) { + f = a[0]; + let k = Na[f] || f, h = k; + for (let l = 1; l < a.length; l++) { + f = a[l]; + const m = Na[f] || f; + m && m !== h && (k += m, h = m); + } + b[g++] = k; + } + } + } + return b; +} +;var Pa = {encode:Oa, rtl:!1, tokenize:""}; +const Qa = E("ae"), Ra = E("oe"), Sa = E("sh"), Ta = E("th"), Ua = E("ph"), Va = E("pf"), Wa = [Qa, "a", Ra, "o", Sa, "s", Ta, "t", Ua, "f", Va, "f", ]; +function Oa(a, b) { + a && (a = Ka.call(this, a).join(" "), 2 < a.length && (a = F(a, Wa)), b || (1 < a.length && (a = I(a)), a && (a = a.split(" ")))); + return a; +} +;var Ya = {encode:Xa, rtl:!1, tokenize:""}; +const Za = E("(?!\\b)[aeiouy]"); +function Xa(a) { + a && (a = Oa.call(this, a, !0), 1 < a.length && (a = a.replace(Za, "")), 1 < a.length && (a = I(a)), a && (a = a.split(" "))); + return a; +} +;J["latin:simple"] = xa; +J["latin:balance"] = La; +J["latin:advanced"] = Pa; +J["latin:extra"] = Ya; const X = self; let Y; -const Z = {Index:O, Document:T, Worker:Q, registerCharset:function(a, b) { - H[a] = b; +const Z = {Index:M, Document:S, Worker:Q, registerCharset:function(a, b) { + J[a] = b; }, registerLanguage:function(a, b) { - G[a] = b; + ea[a] = b; }}; (Y = X.define) && Y.amd ? Y([], function() { return Z; diff --git a/dist/flexsearch.es5.js b/dist/flexsearch.es5.js index f537b05..e6bc8b0 100644 --- a/dist/flexsearch.es5.js +++ b/dist/flexsearch.es5.js @@ -5,40 +5,43 @@ * Licence: Apache-2.0 * https://github.com/nextapps-de/flexsearch */ -(function(self){'use strict';var u;function v(a){var b=0;return function(){return b>>0)+"_",f=0;return b}); -B("Symbol.iterator",function(a){if(a)return a;a=Symbol("Symbol.iterator");for(var b="Array Int8Array Uint8Array Uint8ClampedArray Int16Array Uint16Array Int32Array Uint32Array Float32Array Float64Array".split(" "),c=0;c=b)return g.concat(w.slice(c,b-e+c));g=g.concat(c?w.slice(c):w);e+=f;c= -0}return g}function qa(a,b){for(var c=G(),d=G(),f=[],g=0;g=e&&(h=e-1);this.v=e;this.threshold=h;this.j=b=c&&c.O||a.tokenize||"strict";this.depth="strict"===b&&g.depth;this.C=F(g.bidirectional,!0);this.s=f= -"memory"===a.optimize;this.m=F(a.fastupdate,!0);this.u=a.minlength||1;this.map=f?ha(e-h):G();e=g.resolution||e;h=g.threshold||h;h>=e&&(h=e-1);this.h=e;this.i=h;this.g=f?ha(e-h):G();this.H=c&&c.H||a.rtl;this.G=(b=a.matcher||d&&d.G)&&ja(b,!1);this.I=(b=a.stemmer||d&&d.I)&&ja(b,!0);if(c=b=a.filter||d&&d.filter){c=b;d=G();g=0;for(e=c.length;g=this.u&&(f||!e[l])){var q=Math.min(this.v/d*k|0,k);if(qp;t--)m=l.substring(p,t),m.length>=this.u&&O(this,e,m,r,a,c)}break}case "reverse":if(2< -n){for(p=n-1;0=this.u&&O(this,e,m,q,a,c);m=""}case "forward":if(1=this.u&&O(this,e,m,q,a,c);break;default:if(O(this,e,l,q,a,c),f&&1=this.u&&!q[l]){if(q[l]=1,t=Math.min((this.h-p)/d*k+r|0,k+(r-1)),tm;O(this,h,x?m:l,t,a,c,x?l:m)}}else p=Math.min(p+1,d-k)}}}}this.m||(this.register[a]=1)}}return this}; -function O(a,b,c,d,f,g,e){var h=e?a.g:a.map;if(!b[c]||e&&!b[c][e])a.s&&(h=h[d]),e?(b=b[c]||(b[c]=G()),b[e]=1,h=h[e]||(h[e]=G())):b[c]=1,h=h[c]||(h[c]=[]),a.s||(h=h[d]||(h[d]=[])),g&&-1!==h.indexOf(f)||(h[h.length]=f,a.m&&(a=a.register[f]||(a.register[f]=[]),a[a.length]=h))} -u.search=function(a,b,c){I(a)?(c=a,a=c.query):I(b)&&(c=b);var d=[],f=this.threshold,g=0;if(c){b=c.limit;g=c.offset||0;f=F(c.threshold,f);var e=c.context;var h=c.suggest}if(a){a=this.encode(a);var k=a.length;if(1=this.u&&!c[m])if(this.s||h||this.map[m])l[q++]=m,c[m]=1;else return d;a=l;k=a.length}}if(!k)return d;b||(b=100);c=this.v-f;f=this.h-f;e=this.depth&&1=f)));m++);if(q){if(e)return va(l,f,0);b[b.length]=l;return}}return!c&&l}function va(a,b,c){a=1===a.length?a[0]:[].concat.apply([],a);return c||a.length>b?a.slice(c,c+b):a} -function wa(a,b,c,d){c?(d=d&&b>c,a=(a=a[d?b:c])&&a[d?c:b]):a=a[b];return a}u.contain=function(a){return!!this.register[a]};u.update=function(a,b){return this.remove(a).add(a,b)}; -u.remove=function(a,b){var c=this.register[a];if(c){if(this.m)for(var d=0,f;db||c)f=f.slice(c,c+b);d&&(f=Ba.call(this,f));return{tag:a,result:f}}}function Ba(a){for(var b=Array(a.length),c=0,d;c>>0)+"_",f=0;return b}); +z("Symbol.iterator",function(a){if(a)return a;a=Symbol("Symbol.iterator");for(var b="Array Int8Array Uint8Array Uint8ClampedArray Int16Array Uint16Array Int32Array Uint32Array Float32Array Float64Array".split(" "),c=0;c=b)return g.concat(x.slice(c,b-e+c));g=g.concat(c?x.slice(c):x);e+=f;c= +0}return g}function va(a,b){for(var c=C(),d=C(),f=[],g=0;g=e&&(h=e-1);this.F=e;this.threshold=h;this.o=b=c&&c.L||a.tokenize||"strict";this.depth="strict"===b&&g.depth;this.K=B(g.bidirectional,!0);this.C=f="memory"=== +a.optimize;this.A=B(a.fastupdate,!0);this.D=a.minlength||1;this.map=f?ha(e-h):C();e=g.resolution||e;h=g.threshold||h;h>=e&&(h=e-1);this.l=e;this.m=h;this.h=f?ha(e-h):C();this.H=c&&c.H||a.rtl;this.N=(b=a.matcher||d&&d.N)&&la(b,!1);this.O=(b=a.stemmer||d&&d.O)&&la(b,!0);if(c=b=a.filter||d&&d.filter){c=b;d=C();g=0;for(e=c.length;g=this.D&&(f||!e[l])){var q=Math.min(this.F/d*k|0,k);if(qp;t--)m=l.substring(p,t),m.length>=this.D&&O(this,e,m,r,a,c)}break}case "reverse":if(2< +n){for(p=n-1;0=this.D&&O(this,e,m,q,a,c);m=""}case "forward":if(1=this.D&&O(this,e,m,q,a,c);break;default:if(O(this,e,l,q,a,c),f&&1=this.D&&!q[l]){if(q[l]=1,t=Math.min((this.l-p)/d*k+r|0,k+(r-1)),tm;O(this,h,y?m:l,t,a,c,y?l:m)}}else p=Math.min(p+1,d-k)}}}}this.A||(this.register[a]=1)}}return this}; +function O(a,b,c,d,f,g,e){var h=e?a.h:a.map;if(!b[c]||e&&!b[c][e])a.C&&(h=h[d]),e?(b=b[c]||(b[c]=C()),b[e]=1,h=h[e]||(h[e]=C())):b[c]=1,h=h[c]||(h[c]=[]),a.C||(h=h[d]||(h[d]=[])),g&&-1!==h.indexOf(f)||(h[h.length]=f,a.A&&(a=a.register[f]||(a.register[f]=[]),a[a.length]=h))} +u.search=function(a,b,c){E(a)?(c=a,a=c.query):E(b)&&(c=b);var d=[],f=this.threshold,g=0;if(c){b=c.limit;g=c.offset||0;f=B(c.threshold,f);var e=c.context;var h=c.suggest}if(a){a=this.encode(a);var k=a.length;if(1=this.D&&!c[m])if(this.C||h||this.map[m])l[q++]=m,c[m]=1;else return d;a=l;k=a.length}}if(!k)return d;b||(b=100);c=this.F-f;f=this.l-f;e=this.depth&&1=f)));m++);if(q){if(e)return Aa(l,f,0);b[b.length]=l;return}}return!c&&l}function Aa(a,b,c){a=1===a.length?a[0]:[].concat.apply([],a);return c||a.length>b?a.slice(c,c+b):a} +function Ba(a,b,c,d){c?(d=d&&b>c,a=(a=a[d?b:c])&&a[d?c:b]):a=a[b];return a}u.contain=function(a){return!!this.register[a]};u.update=function(a,b){return this.remove(a).add(a,b)}; +u.remove=function(a,b){var c=this.register[a];if(c){if(this.A)for(var d=0,f;db||c)f=f.slice(c,c+b);d&&(f=Ga.call(this,f));return{tag:a,result:f}}}function Ga(a){for(var b=Array(a.length),c=0,d;c=b)return g.concat(k.slice(c,b-h+c));g=g.concat(c? -k.slice(c):k);h+=n;c=0}return g};function H(a,b){if(!(this instanceof H))return new H(a);let c;if(a){var d=a.charset;c=a.lang;"string"===typeof d&&(-1===d.indexOf(":")&&(d+=":default"),d=F[d]);"string"===typeof c&&(c=E[c])}else a={};let e,g,f,h=a.context||{};this.encode=a.encode||d&&d.encode||v;this.register=b||B();e=a.resolution||9;g=a.threshold||0;g>=e&&(g=e-1);this.i=e;this.s=g;this.F=b=d&&d.G||a.tokenize||"strict";this.m="strict"===b&&h.depth;this.v=z(h.bidirectional,!0);this.g=f="memory"===a.optimize;this.C=z(a.fastupdate,!0); -this.h=a.minlength||1;this.j=f?A(e-g):B();e=h.resolution||e;g=h.threshold||g;g>=e&&(g=e-1);this.l=e;this.D=g;this.B=f?A(e-g):B();this.u=d&&d.u||a.rtl;this.o=(b=a.matcher||c&&c.o)&&D(b,!1);this.A=(b=a.stemmer||c&&c.A)&&D(b,!0);if(a=b=a.filter||c&&c.filter){a=b;d=B();for(let l=0,k=a.length;l=this.h&&(n||!q[p])){var e=Math.min(this.i/k*m|0,m);if(ef;l--)g=p.substring(f,l),g.length>=this.h&&I(this,q,g,h,a,c)}break}case "reverse":if(2< -d){for(f=d-1;0=this.h&&I(this,q,g,e,a,c);g=""}case "forward":if(1=this.h&&I(this,q,g,e,a,c);break;default:if(I(this,q,p,e,a,c),n&&1=this.h&&!e[p]){if(e[p]=1,l=Math.min((this.l-f)/k*m+h|0,m+(h-1)),lg;I(this,t,x?g:p,l,a,c,x?p:g)}}else f=Math.min(f+1,k-m)}}}}this.C||(this.register[a]=1)}}return this}; -function I(a,b,c,d,e,g,f){let h=f?a.B:a.j;if(!b[c]||f&&!b[c][f])a.g&&(h=h[d]),f?(b=b[c]||(b[c]=B()),b[f]=1,h=h[f]||(h[f]=B())):b[c]=1,h=h[c]||(h[c]=[]),a.g||(h=h[d]||(h[d]=[])),g&&-1!==h.indexOf(e)||(h[h.length]=e,a.C&&(a=a.register[e]||(a.register[e]=[]),a[a.length]=h))} -H.prototype.search=function(a,b,c){"object"===typeof a?(c=a,a=c.query):"object"===typeof b&&(c=b);let d=[],e;var g=this.s;let f,h=0;if(c){b=c.limit;h=c.offset||0;g=z(c.threshold,g);var l=c.context;f=!1}if(a&&(a=this.encode(a),e=a.length,1=this.h&&!c[t])if(this.g||f||this.j[t])k[q++]=t,c[t]=1;else return d;a=k;e=a.length}if(!e)return d;b||(b=100);c=this.i-g;g=this.l-g;l=this.m&&1=e)));q++);if(r){if(f)return K(k,e,0);b[b.length]=k;return}}return!c&&k}function K(a,b,c){a=1===a.length?a[0]:[].concat.apply([],a);return c||a.length>b?a.slice(c,c+b):a} -function L(a,b,c,d){c?(d=d&&b>c,a=(a=a[d?b:c])&&a[d?c:b]):a=a[b];return a}H.prototype.contain=function(a){return!!this.register[a]};H.prototype.update=function(a,b){return this.remove(a).add(a,b)};H.prototype.remove=function(a,b){const c=this.register[a];if(c){if(this.C)for(let d=0,e;d=b)return g.concat(k.slice(c,b-h+c));g=g.concat(c? +k.slice(c):k);h+=n;c=0}return g};G["latin:default"]=E; +function I(a,b){if(!(this instanceof I))return new I(a);let c;if(a){var d=a.charset;c=a.lang;"string"===typeof d&&(-1===d.indexOf(":")&&(d+=":default"),d=G[d]);"string"===typeof c&&(c=F[c])}else a={};let e,g,f,h=a.context||{};this.encode=a.encode||d&&d.encode||E;this.register=b||y();e=a.resolution||9;g=a.threshold||0;g>=e&&(g=e-1);this.i=e;this.s=g;this.F=b=d&&d.G||a.tokenize||"strict";this.m="strict"===b&&h.depth;this.v=u(h.bidirectional,!0);this.g=f="memory"===a.optimize;this.C=u(a.fastupdate,!0); +this.h=a.minlength||1;this.j=f?v(e-g):y();e=h.resolution||e;g=h.threshold||g;g>=e&&(g=e-1);this.l=e;this.D=g;this.B=f?v(e-g):y();this.u=d&&d.u||a.rtl;this.o=(b=a.matcher||c&&c.o)&&B(b,!1);this.A=(b=a.stemmer||c&&c.A)&&B(b,!0);if(a=b=a.filter||c&&c.filter){a=b;d=y();for(let l=0,k=a.length;l=this.h&&(n||!q[p])){var e=Math.min(this.i/k*m|0,m);if(ef;l--)g=p.substring(f,l),g.length>=this.h&&J(this,q,g,h,a,c)}break}case "reverse":if(2< +d){for(f=d-1;0=this.h&&J(this,q,g,e,a,c);g=""}case "forward":if(1=this.h&&J(this,q,g,e,a,c);break;default:if(J(this,q,p,e,a,c),n&&1=this.h&&!e[p]){if(e[p]=1,l=Math.min((this.l-f)/k*m+h|0,m+(h-1)),lg;J(this,t,x?g:p,l,a,c,x?p:g)}}else f=Math.min(f+1,k-m)}}}}this.C||(this.register[a]=1)}}return this}; +function J(a,b,c,d,e,g,f){let h=f?a.B:a.j;if(!b[c]||f&&!b[c][f])a.g&&(h=h[d]),f?(b=b[c]||(b[c]=y()),b[f]=1,h=h[f]||(h[f]=y())):b[c]=1,h=h[c]||(h[c]=[]),a.g||(h=h[d]||(h[d]=[])),g&&-1!==h.indexOf(e)||(h[h.length]=e,a.C&&(a=a.register[e]||(a.register[e]=[]),a[a.length]=h))} +I.prototype.search=function(a,b,c){"object"===typeof a?(c=a,a=c.query):"object"===typeof b&&(c=b);let d=[],e;var g=this.s;let f,h=0;if(c){b=c.limit;h=c.offset||0;g=u(c.threshold,g);var l=c.context;f=!1}if(a&&(a=this.encode(a),e=a.length,1=this.h&&!c[t])if(this.g||f||this.j[t])k[q++]=t,c[t]=1;else return d;a=k;e=a.length}if(!e)return d;b||(b=100);c=this.i-g;g=this.l-g;l=this.m&&1=e)));q++);if(r){if(f)return L(k,e,0);b[b.length]=k;return}}return!c&&k}function L(a,b,c){a=1===a.length?a[0]:[].concat.apply([],a);return c||a.length>b?a.slice(c,c+b):a} +function M(a,b,c,d){c?(d=d&&b>c,a=(a=a[d?b:c])&&a[d?c:b]):a=a[b];return a}I.prototype.contain=function(a){return!!this.register[a]};I.prototype.update=function(a,b){return this.remove(a).add(a,b)};I.prototype.remove=function(a,b){const c=this.register[a];if(c){if(this.C)for(let d=0,e;d} */ (normalize)); } - if(str && this.matcher){ + if(this.matcher){ str = replace(str, this.matcher); } - if(this.stemmer && str.length > 1){ + if(this.stemmer && (str.length > 1)){ str = replace(str, this.stemmer); } - if(_collapse && str.length > 1){ + if(_collapse && (str.length > 1)){ str = collapse(str); } - if(str){ + if(split || (split === "")){ - if(split || (split === "")){ + const words = str.split(/** @type {string|RegExp} */ (split)); - const words = str.split(/** @type {string|RegExp} */ (split)); - - return this.filter ? filter(words, this.filter) : words; - } + return this.filter ? filter(words, this.filter) : words; } } return str; } +const regex_normalize = /[\u0300-\u036f]/g; + +export function normalize(str){ + + if(str.normalize){ + + str = str.normalize("NFD").replace(regex_normalize, ""); + } + + return str; +} + /** * @param {!string} str * @param {boolean|Array=} normalize diff --git a/src/lang/arabic/default.js b/src/lang/arabic/default.js index 1ed5765..5915a16 100644 --- a/src/lang/arabic/default.js +++ b/src/lang/arabic/default.js @@ -1,3 +1,5 @@ +import { IndexInterface } from "../../type.js"; + export const rtl = true; export const tokenize = ""; export default { @@ -7,6 +9,10 @@ export default { const regex = /[\x00-\x7F]+/g; +/** + * @this IndexInterface + */ + export function encode(str){ return this.pipeline( diff --git a/src/lang/cjk/default.js b/src/lang/cjk/default.js index b5f3076..9a3827b 100644 --- a/src/lang/cjk/default.js +++ b/src/lang/cjk/default.js @@ -1,3 +1,5 @@ +import { IndexInterface } from "../../type.js"; + export const rtl = false; export const tokenize = "strict"; export default { @@ -8,6 +10,10 @@ export default { const regex = /[\x00-\x7F]+/g; +/** + * @this IndexInterface + */ + export function encode(str){ return this.pipeline( diff --git a/src/lang/cyrillic/default.js b/src/lang/cyrillic/default.js index b86b6a6..3adabbe 100644 --- a/src/lang/cyrillic/default.js +++ b/src/lang/cyrillic/default.js @@ -1,3 +1,5 @@ +import { IndexInterface } from "../../type.js"; + export const rtl = false; export const tokenize = ""; export default { @@ -7,6 +9,10 @@ export default { const regex = /[\x00-\x7F]+/g; +/** + * @this IndexInterface + */ + export function encode(str){ return this.pipeline( diff --git a/src/lang/latin/advanced.js b/src/lang/latin/advanced.js index d336ac6..222dfa6 100644 --- a/src/lang/latin/advanced.js +++ b/src/lang/latin/advanced.js @@ -1,60 +1,66 @@ +import { IndexInterface } from "../../type.js"; import { regex, replace, collapse } from "../../lang.js"; -import { encode as encode_simple } from "./simple.js"; +import { encode as encode_balance } from "./balance.js"; export const rtl = false; export const tokenize = ""; export default { encode: encode, - rtl: rtl + rtl: rtl, + tokenize: tokenize } // Phonetic Normalization const regex_ae = regex("ae"), - regex_ai = regex("ai"), - regex_ay = regex("ay"), - regex_ey = regex("ey"), + //regex_ai = regex("ai"), + //regex_ay = regex("ay"), + //regex_ey = regex("ey"), regex_oe = regex("oe"), - regex_ue = regex("ue"), - regex_ie = regex("ie"), - regex_sz = regex("sz"), - regex_zs = regex("zs"), - regex_ck = regex("ck"), - regex_cc = regex("cc"), + //regex_ue = regex("ue"), + //regex_ie = regex("ie"), + //regex_sz = regex("sz"), + //regex_zs = regex("zs"), + //regex_ck = regex("ck"), + //regex_cc = regex("cc"), regex_sh = regex("sh"), regex_th = regex("th"), - regex_dt = regex("dt"), + //regex_dt = regex("dt"), regex_ph = regex("ph"), - regex_pf = regex("pf"), - regex_ou = regex("ou"), - regex_uo = regex("uo"); + regex_pf = regex("pf"); + //regex_ou = regex("ou"), + //regex_uo = regex("uo"); const pairs = [ regex_ae, "a", - regex_ai, "ei", - regex_ay, "ei", - regex_ey, "ei", + // regex_ai, "ei", + // regex_ay, "ei", + // regex_ey, "ei", regex_oe, "o", - regex_ue, "u", - regex_ie, "i", - regex_sz, "s", - regex_zs, "s", + // regex_ue, "u", + // regex_ie, "i", + // regex_sz, "s", + // regex_zs, "s", regex_sh, "s", - regex_ck, "k", - regex_cc, "k", + // regex_ck, "k", + // regex_cc, "k", regex_th, "t", - regex_dt, "t", + // regex_dt, "t", regex_ph, "f", regex_pf, "f", - regex_ou, "o", - regex_uo, "u" + // regex_ou, "o", + // regex_uo, "u" ]; -export function encode(str, self, _skip_postprocessing){ +/** + * @this IndexInterface + */ + +export function encode(str, _skip_postprocessing){ if(str){ - str = encode_simple(str, self || this).join(" "); + str = encode_balance.call(this, str).join(" "); if(str.length > 2){ diff --git a/src/lang/latin/balance.js b/src/lang/latin/balance.js index 452bd37..2000bca 100644 --- a/src/lang/latin/balance.js +++ b/src/lang/latin/balance.js @@ -1,31 +1,118 @@ -import Index from "../../index.js"; +import { IndexInterface } from "../../type.js"; +import { encode as encode_simple } from "./simple.js"; + +// custom soundex implementation export const rtl = false; -export const tokenize = ""; +export const tokenize = "strict"; export default { encode: encode, - rtl: rtl + rtl: rtl, + tokenize: tokenize } -const regex_whitespace = /[\W_]+/; -//const regex_strip = /[^a-z0-9 ]/g; +//const regex_whitespace = /[\W_]+/g; +const regex_strip = /[^a-z0-9]+/; // const pairs = [ // regex_whitespace, " ", // regex_strip, "" // ]; +// modified + +const soundex = { + + "b": "p", + //"p": "p", + + //"f": "f", + "v": "f", + "w": "f", + + //"s": "s", + "z": "s", + "x": "s", + "ß": "s", + + "d": "t", + //"t": "t", + + //"l": "l", + + //"m": "m", + "n": "m", + + "c": "k", + "g": "k", + "j": "k", + //"k": "k", + "q": "k", + + //"r": "r", + //"h": "h", + //"a": "a", + + //"e": "e", + "i": "e", + "y": "e", + + //"o": "o", + "u": "o" +}; + /** - * @this Index + * @this IndexInterface */ export function encode(str){ - return this.pipeline( + str = encode_simple.call(this, str).join(" "); - /* string: */ str.toLowerCase(), - /* normalize: */ false, - /* split: */ regex_whitespace, - /* collapse: */ false - ); + // str = this.pipeline( + // + // /* string: */ normalize(str).toLowerCase(), + // /* normalize: */ false, + // /* split: */ false, + // /* collapse: */ false + // ); + + const result = []; + + if(str){ + + const words = str.split(regex_strip); + const length = words.length; + + for(let x = 0, tmp, count = 0; x < length; x++){ + + if((str = words[x]) /*&& (str.length > 2)*/ && (!this.filter || !this.filter[str])){ + + tmp = str[0]; + let code = soundex[tmp] || tmp; //str[0]; + let previous = code; //soundex[code] || code; + + for(let i = 1; i < str.length; i++){ + + tmp = str[i]; + const current = soundex[tmp] || tmp; + + if(current && (current !== previous)){ + + code += current; + previous = current; + + // if(code.length === 7){ + // + // break; + // } + } + } + + result[count++] = code; //(code + "0000").substring(0, 4); + } + } + } + + return result; } diff --git a/src/lang/latin/default.js b/src/lang/latin/default.js index d66d801..ba213b9 100644 --- a/src/lang/latin/default.js +++ b/src/lang/latin/default.js @@ -1,23 +1,25 @@ -import Index from "../../index.js"; +import { IndexInterface } from "../../type.js"; +import { normalize } from "../../lang.js"; export const rtl = false; export const tokenize = ""; export default { encode: encode, - rtl: rtl + rtl: rtl, + tokenize: tokenize } const regex_whitespace = /[\W_]+/; /** - * @this Index + * @this IndexInterface */ export function encode(str){ return this.pipeline( - /* string: */ str.toLowerCase(), + /* string: */ normalize(str).toLowerCase(), /* normalize: */ false, /* split: */ regex_whitespace, /* collapse: */ false diff --git a/src/lang/latin/extra.js b/src/lang/latin/extra.js index 93fa286..31d1747 100644 --- a/src/lang/latin/extra.js +++ b/src/lang/latin/extra.js @@ -1,3 +1,4 @@ +import { IndexInterface } from "../../type.js"; import { regex, replace, collapse } from "../../lang.js"; import { encode as encode_advanced } from "./advanced.js"; @@ -5,40 +6,46 @@ export const rtl = false; export const tokenize = ""; export default { encode: encode, - rtl: rtl + rtl: rtl, + tokenize: tokenize } // Soundex Normalization const prefix = "(?!\\b)"; -const soundex_b = regex(prefix + "p"), - soundex_s = regex(prefix + "z"), - soundex_k = regex(prefix + "[cgq]"), - soundex_m = regex(prefix + "n"), - soundex_t = regex(prefix + "d"), - soundex_f = regex(prefix + "[vw]"), +const //soundex_b = regex(prefix + "p"), + // soundex_s = regex(prefix + "z"), + // soundex_k = regex(prefix + "[cgq]"), + // soundex_m = regex(prefix + "n"), + // soundex_t = regex(prefix + "d"), + // soundex_f = regex(prefix + "[vw]"), regex_vowel = regex(prefix + "[aeiouy]"); const pairs = [ - soundex_b, "b", - soundex_s, "s", - soundex_k, "k", - soundex_m, "m", - soundex_t, "t", - soundex_f, "f", + // soundex_b, "b", + // soundex_s, "s", + // soundex_k, "k", + // soundex_m, "m", + // soundex_t, "t", + // soundex_f, "f", regex_vowel, "" ]; +/** + * @this IndexInterface + */ + export function encode(str){ if(str){ - str = encode_advanced(str, this, /* skip post-processing: */ true); + str = encode_advanced.call(this, str, /* skip post-processing: */ true); if(str.length > 1){ - str = replace(str, pairs); + //str = replace(str, pairs); + str = str.replace(regex_vowel, ""); } if(str.length > 1){ diff --git a/src/lang/latin/simple.js b/src/lang/latin/simple.js index 7584e56..55d32d6 100644 --- a/src/lang/latin/simple.js +++ b/src/lang/latin/simple.js @@ -1,11 +1,12 @@ -import Index from "../../index.js"; -import { regex } from "../../lang.js"; +import { IndexInterface } from "../../type.js"; +import { normalize, regex } from "../../lang.js"; export const rtl = false; export const tokenize = ""; export default { encode: encode, - rtl: rtl + rtl: rtl, + tokenize: tokenize } // Charset Normalization @@ -41,14 +42,15 @@ const pairs = [ /** * @param {string} str + * @this IndexInterface */ -export function encode(str, self){ +export function encode(str){ - return /** @type {Index} */ (self || this).pipeline( + return this.pipeline( - /* string: */ str.toLowerCase(), - /* normalize: */ pairs, + /* string: */ normalize(str).toLowerCase(), + /* normalize: */ !str.normalize && pairs, /* split: */ regex_whitespace, /* collapse: */ false ); diff --git a/src/lang/latin/soundex.js b/src/lang/latin/soundex.js deleted file mode 100644 index 223c055..0000000 --- a/src/lang/latin/soundex.js +++ /dev/null @@ -1,135 +0,0 @@ -import Index from "../../index.js"; - -export const rtl = false; -export const tokenize = "strict"; -export default { - encode: encode, - rtl: rtl, - tokenize: tokenize -} - -//const regex_whitespace = /[\W_]+/g; -const regex_strip = /[^a-z]+/; - -// const pairs = [ -// regex_whitespace, " ", -// regex_strip, "" -// ]; - -// modified - -const soundex = { - - "b": 1, - "f": 1, - "p": 1, - "v": 1, - - "c": 7, - "g": 7, - "j": 7, - "k": 7, - "q": 7, - - "s": 2, - "x": 2, - "z": 2, - - "d": 3, - "t": 3, - - "l": 4, - - "m": 5, - "n": 5, - - "r": 6 -}; - -/** - * @this Index - */ - -export function encode(str){ - - str = this.pipeline( - - /* string: */ str.toLowerCase(), - /* normalize: */ false, - /* split: */ false, - /* collapse: */ false - ); - - const result = []; - - if(str){ - - const words = str.split(regex_strip); - const length = words.length; - - for(let x = 0, count = 0; x < length; x++){ - - if((str = words[x]) && (str.length > 2) && (!this.filter || !this.filter[str])){ - - let code = str[0]; - let previous = getCode(code); - - for(let i = 1; i < str.length; i++){ - - const current = soundex[str[i]]; - - if(current && (current !== previous)){ - - code += current; - previous = current; - - if(code.length === 4){ - - break; - } - } - } - - result[count++] = (code + "0000").substring(0, 4); - } - } - } - - return result; -} - - -// https://www.rosettacode.org/wiki/Soundex - -// function getCode(char){ -// -// switch(char){ -// -// case 'b': -// case 'f': -// case 'p': -// case 'v': -// return 1; -// case 'c': -// case 'g': -// case 'j': -// case 'k': -// case 'q': -// case 's': -// case 'x': -// case 'z': -// return 2; -// case 'd': -// case 't': -// return 3; -// case 'l': -// return 4; -// case 'm': -// case 'n': -// return 5; -// case 'r': -// return 6; -// } -// -// return ""; -// } diff --git a/src/type.js b/src/type.js index 0ca900e..82ccde6 100644 --- a/src/type.js +++ b/src/type.js @@ -5,8 +5,21 @@ export function IndexInterface(){ this.cache = null; + this.matcher = null; + this.stemmer = null; + this.filter = null; } +/** + * @param {!string} str + * @param {boolean|Array=} normalize + * @param {boolean|string|RegExp=} split + * @param {boolean=} collapse + * @returns {string|Array} + */ + +IndexInterface.prototype.pipeline; + /** * @param {!number|string} id * @param {!string} content diff --git a/src/webpack.js b/src/webpack.js index 30c3bd8..3e0b21a 100644 --- a/src/webpack.js +++ b/src/webpack.js @@ -1,8 +1,12 @@ -import { SUPPORT_ASYNC, SUPPORT_DOCUMENT, SUPPORT_CACHE, SUPPORT_SERIALIZE, SUPPORT_WORKER } from "./config.js"; +import { SUPPORT_ASYNC, SUPPORT_DOCUMENT, SUPPORT_CACHE, SUPPORT_SERIALIZE, SUPPORT_WORKER, SUPPORT_ENCODER } from "./config.js"; import Document from "./document.js"; import Index from "./index.js"; import WorkerIndex from "./worker/index.js"; import { registerCharset, registerLanguage } from "./global.js"; +import charset_simple from "./lang/latin/simple.js" +import charset_balance from "./lang/latin/balance.js" +import charset_advanced from "./lang/latin/advanced.js" +import charset_extra from "./lang/latin/extra.js" /** @export */ Document.prototype.add; /** @export */ Document.prototype.append; @@ -49,6 +53,14 @@ if(SUPPORT_SERIALIZE){ /** @export */ Document.prototype.import; } +if(SUPPORT_ENCODER){ + + registerCharset("latin:simple", charset_simple); + registerCharset("latin:balance", charset_balance); + registerCharset("latin:advanced", charset_advanced); + registerCharset("latin:extra", charset_extra); +} + const root = self; let tmp; diff --git a/task/build.js b/task/build.js index f0969e8..a047711 100644 --- a/task/build.js +++ b/task/build.js @@ -18,7 +18,7 @@ var supported_lang = [ var supported_charset = { - 'latin': ["default", "advanced", "balance", "extra", "simple", "soundex"], + 'latin': ["default", "advanced", "balance", "extra", "simple"], 'cjk': ["default"], 'cyrillic': ["default"], 'arabic': ["default"], @@ -85,6 +85,18 @@ var release = options["RELEASE"]; // const es5_version = (release === "es5") || (process.argv[2] === "--es5"); // const module_version = (release === "module") || (process.argv[2] === "--module"); +if(release){ + + let filename + + if(!fs.existsSync(filename = "src/config/" + release + "/config.js")){ + + filename = "src/config/bundle/config.js"; + } + + fs.writeFileSync("tmp/config.js", fs.readFileSync(filename)); +} + if(release === "es5"){ release = "ES5"; @@ -126,7 +138,7 @@ let parameter = (function(opt){ process_closure_primitives: true, summary_detail_level: 3, warning_level: "VERBOSE", - emit_use_strict: true, + emit_use_strict: true, // release !== "lang", output_manifest: "log/manifest.log", //output_module_dependencies: "log/module_dependencies.log", @@ -147,7 +159,7 @@ let parameter = (function(opt){ rewrite_polyfills: use_polyfill || false, // isolation_mode: "IIFE", - output_wrapper: "(function(self){%output%}(this));" + output_wrapper: /*release === "lang" ? "%output%" :*/ "(function(self){%output%}(this));" //formatting: "PRETTY_PRINT" }); @@ -181,7 +193,8 @@ if(release === "lang"){ fs.writeFileSync("tmp/" + lang + ".js", ` import lang from "../src/lang/${lang}.js"; - window["FlexSearch"]["registerLanguage"]("${lang}", lang); + /*try{if(module)self=module}catch(e){}*/ + self["FlexSearch"]["registerLanguage"]("${lang}", lang); `); exec("java -jar node_modules/google-closure-compiler-java/compiler.jar" + parameter + " --entry_point='tmp/" + lang + ".js' --js='tmp/" + lang + ".js' --js='src/**.js'" + flag_str + " --js_output_file='dist/lang/" + lang + ".min.js' && exit 0", function(){ @@ -203,7 +216,8 @@ if(release === "lang"){ fs.writeFileSync("tmp/" + charset + "_" + variant + ".js", ` import charset from "../src/lang/${charset}/${variant}.js"; - window["FlexSearch"]["registerCharset"]("${charset}:${variant}", charset); + /*try{if(module)self=module}catch(e){}*/ + self["FlexSearch"]["registerCharset"]("${charset}:${variant}", charset); `); exec("java -jar node_modules/google-closure-compiler-java/compiler.jar" + parameter + " --entry_point='tmp/" + charset + "_" + variant + ".js' --js='tmp/" + charset + "_" + variant + ".js' --js='src/**.js'" + flag_str + " --js_output_file='dist/lang/" + charset + "/" + variant + ".min.js' && exit 0", function(){