From 53f6de26b07df504b6b9fe284179d7b109dd74c8 Mon Sep 17 00:00:00 2001 From: Thomas Wilkerling Date: Mon, 24 May 2021 14:41:49 +0200 Subject: [PATCH] support for node.js worker threads --- dist/flexsearch.min.js | 32 ++++++++++----------- dist/node/node.js | 39 +++++++++++++++++++++++++ doc/0.7.0.md | 9 +++--- src/async.js | 11 +++---- src/document.js | 2 +- src/worker/handler.js | 37 +++++++----------------- src/worker/index.js | 65 ++++++++++++++++++++++++++---------------- src/worker/node.js | 39 +++++++++++++++++++++++++ task/build.js | 4 +-- 9 files changed, 158 insertions(+), 80 deletions(-) create mode 100644 dist/node/node.js create mode 100644 src/worker/node.js diff --git a/dist/flexsearch.min.js b/dist/flexsearch.min.js index 3fb4649..01f08e4 100644 --- a/dist/flexsearch.min.js +++ b/dist/flexsearch.min.js @@ -5,29 +5,29 @@ * Licence: Apache-2.0 * https://github.com/nextapps-de/flexsearch */ -(function _f(self){'use strict';try{if(module)self=module}catch(e){}self._factory=_f;var u;const w=/[\W_]+/;function aa(a){if(a=a.toLowerCase())if(a&&this.C&&(a=y(a,this.C)),this.G&&1=b)return f.concat(l.slice(c,b-h+c));f=f.concat(c? -l.slice(c):l);h+=n;c=0}return f}function ha(a,b){const c=B(),d=B(),e=[];for(let f=0;f=e&&(f=e-1);this.s=e;this.threshold=f;this.u=b=c&&c.I||a.tokenize||"strict";this.depth="strict"===b&&h.depth;this.v=z(h.bidirectional,!0);this.m=g="memory"=== -a.optimize;this.i=z(a.fastupdate,!0);this.o=a.minlength||1;this.map=g?A(e-f):B();e=h.resolution||e;f=h.threshold||f;f>=e&&(f=e-1);this.h=e;this.l=f;this.g=g?A(e-f):B();this.F=c&&c.F||a.rtl;this.C=(b=a.matcher||d&&d.C)&&H(b,!1);this.G=(b=a.stemmer||d&&d.G)&&H(b,!0);if(c=b=a.filter||d&&d.filter){c=b;d=B();for(let k=0,l=c.length;k=b)return f.concat(l.slice(c,b-h+c));f=f.concat(c? +l.slice(c):l);h+=n;c=0}return f}function ha(a,b){const c=B(),d=B(),e=[];for(let f=0;f=e&&(f=e-1);this.s=e;this.threshold=f;this.u=b=c&&c.I||a.tokenize||"strict";this.depth="strict"===b&&h.depth;this.v=z(h.bidirectional,!0);this.m=g="memory"=== +a.optimize;this.j=z(a.fastupdate,!0);this.o=a.minlength||1;this.map=g?A(e-f):B();e=h.resolution||e;f=h.threshold||f;f>=e&&(f=e-1);this.h=e;this.l=f;this.g=g?A(e-f):B();this.F=c&&c.F||a.rtl;this.C=(b=a.matcher||d&&d.C)&&H(b,!1);this.G=(b=a.stemmer||d&&d.G)&&H(b,!0);if(c=b=a.filter||d&&d.filter){c=b;d=B();for(let k=0,l=c.length;k=this.o&&(n||!r[m])){var e=Math.min(this.s/l*p|0,p);if(eg;k--)f=m.substring(g,k),f.length>=this.o&&M(this,r,f,h,a,c)}break}case "reverse":if(2< -d){for(g=d-1;0=this.o&&M(this,r,f,e,a,c);f=""}case "forward":if(1=this.o&&M(this,r,f,e,a,c);break;default:if(M(this,r,m,e,a,c),n&&1=this.o&&!e[m]){if(e[m]=1,k=Math.min((this.h-g)/l*p+h|0,p+(h-1)),kf;M(this,v,t?f:m,k,a,c,t?m:f)}}else g=Math.min(g+1,l-p)}}}}this.i||(this.register[a]=1)}}return this}; -function M(a,b,c,d,e,f,g){let h=g?a.g:a.map;if(!b[c]||g&&!b[c][g])a.m&&(h=h[d]),g?(b=b[c]||(b[c]=B()),b[g]=1,h=h[g]||(h[g]=B())):b[c]=1,h=h[c]||(h[c]=[]),a.m||(h=h[d]||(h[d]=[])),f&&-1!==h.indexOf(e)||(h[h.length]=e,a.i&&(a=a.register[e]||(a.register[e]=[]),a[a.length]=h))} +d){for(g=d-1;0=this.o&&M(this,r,f,e,a,c);f=""}case "forward":if(1=this.o&&M(this,r,f,e,a,c);break;default:if(M(this,r,m,e,a,c),n&&1=this.o&&!e[m]){if(e[m]=1,k=Math.min((this.h-g)/l*p+h|0,p+(h-1)),kf;M(this,v,t?f:m,k,a,c,t?m:f)}}else g=Math.min(g+1,l-p)}}}}this.j||(this.register[a]=1)}}return this}; +function M(a,b,c,d,e,f,g){let h=g?a.g:a.map;if(!b[c]||g&&!b[c][g])a.m&&(h=h[d]),g?(b=b[c]||(b[c]=B()),b[g]=1,h=h[g]||(h[g]=B())):b[c]=1,h=h[c]||(h[c]=[]),a.m||(h=h[d]||(h[d]=[])),f&&-1!==h.indexOf(e)||(h[h.length]=e,a.j&&(a=a.register[e]||(a.register[e]=[]),a[a.length]=h))} u.search=function(a,b,c){D(a)?(c=a,a=c.query):D(b)&&(c=b);let d=[],e;var f=this.threshold;let g,h=0;if(c){b=c.limit;h=c.offset||0;f=z(c.threshold,f);var k=c.context;g=c.suggest}if(a&&(a=this.encode(a),e=a.length,1=this.o&&!c[v])if(this.m||g||this.map[v])l[r++]=v,c[v]=1;else return d;a=l;e=a.length}if(!e)return d;b||(b=100);c=this.s-f;f=this.h-f;k=this.depth&&1=e)));r++);if(q){if(g)return oa(l,e,0);b[b.length]=l;return}}return!c&&l}function oa(a,b,c){a=1===a.length?a[0]:[].concat.apply([],a);return c||a.length>b?a.slice(c,c+b):a} function pa(a,b,c,d){c?(d=d&&b>c,a=(a=a[d?b:c])&&a[d?c:b]):a=a[b];return a}u.contain=function(a){return!!this.register[a]};u.update=function(a,b){return this.remove(a).add(a,b)}; -u.remove=function(a,b){const c=this.register[a];if(c){if(this.i)for(let d=0,e;db||c)e=e.slice(c,c+b);d&&(e=va.call(this,e));return{tag:a,result:e}}}function va(a){const b=Array(a.length);for(let c=0,d;cb||c)e=e.slice(c,c+b);d&&(e=ua.call(this,e));return{tag:a,result:e}}}function ua(a){const b=Array(a.length);for(let c=0,d;c What is not included yet but comes soon? - ~~WebWorker~~ -- Worker for Node.js (almost done) +- ~~Worker for Node.js~~ - ~~Inline Worker (optionally)~~ - ~~Offset-Pagination~~ -- ~~Export/Import (not implemented yet)~~ -- ~~Tags (almost done)~~ +- ~~Export/Import~~ +- ~~Tags~~ - ~~Bundles: Light, Compact, Full~~ - Test Automation (needs to be migrated) - Benchmark Suite (almost done) @@ -107,8 +107,7 @@ What will be dropped? - Where-Clause - Index Information `index.info()` -- Paging Cursor (gets replaced by offset) -- ~~Inline Worker (was replaced by native workers)~~ +- Paging Cursor (was replaced by offset) ## Builtin Profiles diff --git a/src/async.js b/src/async.js index f022a60..eef5795 100644 --- a/src/async.js +++ b/src/async.js @@ -1,5 +1,6 @@ +import { IndexInterface, DocumentInterface } from "./type.js"; import { promise as Promise } from "./polyfill.js"; -import { is_function, is_object } from "./common.js"; +import { is_function, is_object, is_string } from "./common.js"; export default function(prototype){ @@ -14,6 +15,7 @@ function register(prototype, key){ prototype[key + "Async"] = function(){ + /** @type {IndexInterface|DocumentInterface} */ const self = this; const args = /*[].slice.call*/(arguments); const arg = args[args.length - 1]; @@ -29,10 +31,9 @@ function register(prototype, key){ setTimeout(function(){ - const fn = self[key]; - fn.async = true; - resolve(fn.apply(self, args)); - fn.async = false; + self.async = true; + resolve(self[key].apply(self, args)); + self.async = false; }); }); diff --git a/src/document.js b/src/document.js index dc735c3..0872d45 100644 --- a/src/document.js +++ b/src/document.js @@ -298,7 +298,7 @@ function add_index(obj, tree, marker, pos, index, id, key, _append){ * @param id * @param content * @param {boolean=} _append - * @returns {Document} + * @returns {Document|Promise} */ Document.prototype.add = function(id, content, _append){ diff --git a/src/worker/handler.js b/src/worker/handler.js index 3e22434..fd164a7 100644 --- a/src/worker/handler.js +++ b/src/worker/handler.js @@ -6,8 +6,9 @@ export default function handler(event) { const index = self["_index"]; const data = event["data"]; const args = data["args"]; + const task = data["task"]; - switch(data["task"]){ + switch(task){ case "init": @@ -16,20 +17,19 @@ export default function handler(event) { const encode = options["encode"]; options["cache"] = false; - //root["_id"] = data["id"]; - if(typeof encode === "string"){ + if(encode && (encode.indexOf("function") === 0)){ - options["encode"] = new Function("return " + encode)(); + options["encode"] = Function("return " + encode)(); } if(factory){ // export the FlexSearch global payload to "self" - new Function("return " + factory)()(self); + Function("return " + factory)()(self); /** @type Index */ - self["_index"] = self["FlexSearch"]["Index"](options); + self["_index"] = new self["FlexSearch"]["Index"](options); // destroy the exported payload delete self["FlexSearch"]; @@ -41,31 +41,14 @@ export default function handler(event) { break; - case "add": - - index.add.apply(index, args); - break; - - case "append": - - index.append.apply(index, args); - break; - case "search": - const results = index.search.apply(index, args); - //postMessage({ id: id, results: results }); - postMessage(results); + const message = index.search.apply(index, args); + postMessage(message); break; - case "update": + default: - index.update.apply(index, args); - break; - - case "remove": - - index.remove.apply(index, args); - break; + index[task].apply(index, args); } }; diff --git a/src/worker/index.js b/src/worker/index.js index 0488b5d..c1287b8 100644 --- a/src/worker/index.js +++ b/src/worker/index.js @@ -17,7 +17,7 @@ function WorkerIndex(id, options){ if(is_object(id)){ options = /** @type {Object} */ (id); - id = 0; + //id = 0; } else{ @@ -44,18 +44,28 @@ function WorkerIndex(id, options){ factory = factory.toString(); } - const _self = this; + const is_node_js = self["exports"]; - this.id = id || counter++; - this.resolver = null; - this.worker = create(factory); + this.worker = create(factory, is_node_js); if(!this.worker){ return; } - this.worker.onmessage = function(e){ _self.resolver(e["data"]/*["results"]*/) }; + //this.id = id || counter++; + this.resolver = null; + const _self = this; + + if(is_node_js){ + + this.worker["on"]("message", function(msg){ _self.resolver(msg/*["results"]*/) }); + } + else{ + + this.worker.onmessage = function(e){ _self.resolver(e["data"]/*["results"]*/) }; + } + this.worker.postMessage({ "task": "init", "factory": factory, /*id: this.id,*/ "options": options }); } @@ -85,16 +95,19 @@ function register(key){ const promise = new Promise(function(resolve){ - self.worker.postMessage({ "task": key, /*id: this.id,*/ "args": args }); + setTimeout(function(){ - if(key === "search"){ + self.worker.postMessage({ "task": key, /*id: this.id,*/ "args": args }); - self.resolver = resolve; - } - else{ + if(key === "search"){ - resolve(); - } + self.resolver = resolve; + } + else{ + + resolve(); + } + }); }); if(callback){ @@ -109,24 +122,28 @@ function register(key){ }; } -function create(factory){ +function create(factory, is_node_js){ let worker try{ - worker = factory ? + worker = is_node_js ? - new Worker(URL.createObjectURL( - - new Blob([ - - "onmessage=" + handler.toString() - - ], { "type": "text/javascript" }) - )) + eval('new (require("worker_threads")["Worker"])("../dist/node/node.js")') : - new Worker("worker.js", { type: "module" }); + factory ? + + new Worker(URL.createObjectURL( + + new Blob([ + + "onmessage=" + handler.toString() + + ], { "type": "text/javascript" }) + )) + : + new Worker("worker.js", { type: "module" }); } catch(e){} diff --git a/src/worker/node.js b/src/worker/node.js new file mode 100644 index 0000000..725667e --- /dev/null +++ b/src/worker/node.js @@ -0,0 +1,39 @@ +const { parentPort } = require("worker_threads"); +const { Index } = require("../flexsearch.min.js"); + +let index; + +parentPort.on("message", function(data){ + + /** @type Index */ + const args = data["args"]; + const task = data["task"]; + + switch(task){ + + case "init": + + const options = data["options"] || {}; + const encode = options["encode"]; + + options["cache"] = false; + + if(encode && (encode.indexOf("function") === 0)){ + + options["encode"] = new Function("return " + encode)(); + } + + index = new Index(options); + break; + + case "search": + + const message = index.search.apply(index, args); + parentPort.postMessage(message); + break; + + default: + + index[task].apply(index, args); + } +}); diff --git a/task/build.js b/task/build.js index ad0637d..db2e807 100644 --- a/task/build.js +++ b/task/build.js @@ -127,7 +127,7 @@ let parameter = (function(opt){ assume_function_wrapper: true, //transform_amd_modules: true, - process_common_js_modules: true, + process_common_js_modules: false, module_resolution: "BROWSER", //dependency_mode: "SORT_ONLY", //js_module_root: "./", @@ -222,7 +222,7 @@ else{ : "java -jar node_modules/google-closure-compiler-java/compiler.jar" - ) + parameter + " --js='src/**.js'" + flag_str + " --js_output_file='" + filename + "' && exit 0", function(){ + ) + parameter + " --js='src/**.js' --js='!src/**/node.js'" + flag_str + " --js_output_file='" + filename + "' && exit 0", function(){ let build = fs.readFileSync(filename); let preserve = fs.readFileSync("src/index.js", "utf8");