mirror_js/flexsearch
1
0
Fork 0
mirror of https://github.com/nextapps-de/flexsearch.git synced 2025-09-24 20:41:28 +02:00
Code Issues Releases Wiki Activity

more tests, more fixes

Browse Source
This commit is contained in:
Thomas Wilkerling
2025-03-24 18:33:20 +01:00
parent 63531ef4f1
commit e878ce5f99
113 changed files with 6722 additions and 5621 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
dist/module-debug/bundle.js vendored
View File

@@ -8,6 +8,7 @@ import Resolver from "./resolver.js";
import Encoder from "./encoder.js";
import IdxDB from "./db/indexeddb/index.js";
import Charset from "./charset.js";
import { KeystoreMap, KeystoreArray, KeystoreSet } from "./keystore.js";
/** @export */Index.prototype.add;
/** @export */Index.prototype.append;
@@ -111,6 +112,10 @@ import Charset from "./charset.js";
/** @export */StorageInterface.prototype.commit;
/** @export */StorageInterface.prototype.remove;
/** @export */KeystoreArray.length;
/** @export */KeystoreMap.size;
/** @export */KeystoreSet.size;
/** @export */Charset.LatinExact;
/** @export */Charset.LatinDefault;
/** @export */Charset.LatinSimple;

94
dist/module-debug/common.js vendored
View File

@@ -61,58 +61,58 @@ export function merge_option(value, default_value, merge_value) {
return "undefined" == type_value ? default_value : value;
}
//
// /**
// * @param {!number} count
// * @returns {Array<Object>}
// */
/**
* @param {!number} count
* @returns {Array<Object>}
*/
// export function create_object_array(count){
//
// const array = new Array(count);
//
// for(let i = 0; i < count; i++){
// array[i] = create_object();
// }
//
// return array;
// }
export function create_object_array(count) {
// /**
// * @param {!number} count
// * @returns {Array<Object>}
// */
const array = Array(count);
// export function create_map_array(count){
//
// const array = new Array(count);
//
// for(let i = 0; i < count; i++){
// array[i] = new Map();
// }
//
// return array;
// }
for (let i = 0; i < count; i++) {
array[i] = create_object();
}
// export function create_arrays(count){
//
// const array = new Array(count);
//
// for(let i = 0; i < count; i++){
// array[i] = [];
// }
//
// return array;
// }
//
// /**
// * @param {!Object} obj
// * @returns {Array<string>}
// */
return array;
}
/**
* @param {!number} count
* @returns {Array<Object>}
*/
export function create_map_array(count) {
const array = Array(count);
for (let i = 0; i < count; i++) {
array[i] = new Map();
}
return array;
}
export function create_arrays(count) {
const array = Array(count);
for (let i = 0; i < count; i++) {
array[i] = [];
}
return array;
}
/**
* @param {!Object} obj
* @returns {Array<string>}
*/
export function get_keys(obj) {
return Object.keys(obj);
}
// export function get_keys(obj){
// return Object.keys(obj);
// }
export function create_object() {
return Object.create(null);

26
dist/module-debug/db/clickhouse/index.js vendored
View File

@@ -1,4 +1,3 @@
import { ClickHouse } from "clickhouse";
import StorageInterface from "../interface.js";
import { concat, toArray } from "../../common.js";
@@ -174,8 +173,8 @@ ClickhouseDB.prototype.open = async function () {
};
ClickhouseDB.prototype.close = function () {
this.db.close();
this.db = null;
//DB && DB.close();
this.db = DB = null;
return this;
};
@@ -306,17 +305,16 @@ ClickhouseDB.prototype.enrich = async function (ids) {
return 1 === result.length ? result[0] : 1 < result.length ? concat(result) : result;
};
ClickhouseDB.prototype.has = function (id) {
return this.db.query(`
SELECT EXISTS(
SELECT 1
FROM ${this.id}.reg
WHERE id = {id:${this.type /*=== "number" ? "Int32" : "String"*/}}
LIMIT 1
)`, { params: { id } }).toPromise();
ClickhouseDB.prototype.has = async function (id) {
const result = await this.db.query(`
SELECT 1 as exist
FROM ${this.id}.reg
WHERE id = {id:${this.type /*=== "number" ? "Int32" : "String"*/}}
LIMIT 1`, { params: { id } }).toPromise();
return !!(result && result[0] && result[0].exist);
};
ClickhouseDB.prototype.search = function (flexsearch, query, limit = 100, offset = 0, suggest = !1, resolve = !0, enrich = !0, tags) {
ClickhouseDB.prototype.search = function (flexsearch, query, limit = 100, offset = 0, suggest = !1, resolve = !0, enrich = !1, tags) {
let rows;
if (1 < query.length && flexsearch.depth) {
let where = "",
@@ -349,7 +347,7 @@ ClickhouseDB.prototype.search = function (flexsearch, query, limit = 100, offset
${enrich ? ", doc" : ""}
FROM (
SELECT id, count(*) as count,
${suggest ? "SUM" : "MIN"}(res) as res
${suggest ? "SUM" : "SUM" /*"MIN"*/}(res) as res
FROM ${this.id}.ctx${this.field}
WHERE ${where}
GROUP BY id
@@ -414,7 +412,7 @@ ClickhouseDB.prototype.search = function (flexsearch, query, limit = 100, offset
${enrich ? ", doc" : ""}
FROM (
SELECT id, count(*) as count,
${suggest ? "SUM" : "MIN"}(res) as res
${suggest ? "SUM" : "SUM" /*"MIN"*/}(res) as res
FROM ${this.id}.map${this.field}
WHERE ${where}
GROUP BY id

6
dist/module-debug/db/indexeddb/index.js vendored
View File

@@ -114,7 +114,7 @@ IdxDB.prototype.open = function () {
};
IdxDB.prototype.close = function () {
this.db.close();
this.db && this.db.close();
this.db = null;
};
@@ -253,7 +253,9 @@ IdxDB.prototype.has = function (id) {
map = transaction.objectStore("reg"),
req = map.getKey(id);
return promisfy(req);
return promisfy(req).then(function (result) {
return !!result;
});
};
IdxDB.prototype.search = null;

61
dist/module-debug/db/mongodb/index.js vendored
View File

@@ -124,8 +124,9 @@ MongoDB.prototype.open = async function () {
};
MongoDB.prototype.close = function () {
this.db.close();
this.db = null;
//CLIENT && CLIENT.close();
this.db = CLIENT = null;
DB[this.id] = null;
return this;
};
@@ -257,7 +258,9 @@ MongoDB.prototype.enrich = function (ids) {
};
MongoDB.prototype.has = function (id) {
return this.db.collection("reg").countDocuments({ id }, { limit: 1 });
return this.db.collection("reg").countDocuments({ id }, { limit: 1 }).then(function (result) {
return !!result;
});
};
MongoDB.prototype.search = async function (flexsearch, query, limit = 100, offset = 0, suggest = !1, resolve = !0, enrich = !1, tags) {
@@ -281,12 +284,14 @@ MongoDB.prototype.search = async function (flexsearch, query, limit = 100, offse
keyword = term;
}
let project = resolve ? { _id: 1 } : { _id: 1, res: 1 };
const project = { _id: 1 };
if (!resolve) project.res = 1;
if (enrich) project.doc = 1;
const stmt = [{ $match: { $or: params } }, { $group: {
_id: "$id",
res: suggest ? { $sum: 1 } : { $min: 1 },
count: { $sum: 1 }
count: { $sum: 1 },
res: suggest ? { $sum: "$res" } : { $sum /*$min*/: "$res" }
} }];
suggest || stmt.push({ $match: { count: query.length - 1 } });
@@ -319,32 +324,34 @@ MongoDB.prototype.search = async function (flexsearch, query, limit = 100, offse
count++;
}
stmt.push({ $project: project }, { $match: match });
} else {
stmt.push({ $project: project });
stmt.push(
//{ $project: project },
{ $match: match });
}
stmt.push({ $sort: suggest ? { count: -1, res: 1 } : { res: 1 } }, { $skip: offset }, { $limit: limit });
if (tags) {
project = { _id: 1 };
if (!resolve) project.res = 1;
if (enrich) project.doc = 1;
// if(tags){
// project = { _id: 1 };
// if(!resolve) project["res"] = 1;
// if(enrich) project["doc"] = 1;
// }
stmt.push({ $project: project });
}
stmt.push({ $project: project });
rows = await this.db.collection("ctx" + this.field).aggregate(stmt);
} else {
let project = resolve ? { _id: 1 } : { _id: 1, res: 1 };
const project = { _id: 1 };
if (!resolve) project.res = 1;
if (enrich) project.doc = 1;
const stmt = [{ $match: {
key: { $in: query }
} }, { $group: {
_id: "$id",
res: suggest ? { $sum: 1 } : { $min: 1 },
count: { $sum: 1 }
count: { $sum: 1 },
res: suggest ? { $sum: "$res" } : { $sum /*$min*/: "$res" }
} }];
suggest || stmt.push({ $match: { count: query.length } });
@@ -377,20 +384,20 @@ MongoDB.prototype.search = async function (flexsearch, query, limit = 100, offse
count++;
}
stmt.push({ $project: project }, { $match: match });
} else {
stmt.push({ $project: project });
stmt.push(
//{ $project: project },
{ $match: match });
}
stmt.push({ $sort: suggest ? { count: -1, res: 1 } : { res: 1 } }, { $skip: offset }, { $limit: limit });
if (tags) {
project = { _id: 1 };
if (!resolve) project.res = 1;
if (enrich) project.doc = 1;
// if(tags){
// project = { _id: 1 };
// if(!resolve) project["res"] = 1;
// if(enrich) project["doc"] = 1;
// }
stmt.push({ $project: project });
}
stmt.push({ $project: project });
rows = await this.db.collection("map" + this.field).aggregate(stmt);
}

44
dist/module-debug/db/postgres/index.js vendored
View File

@@ -1,5 +1,3 @@
import pg_promise from "pg-promise";
import StorageInterface from "../interface.js";
import { concat, toArray } from "../../common.js";
@@ -11,7 +9,7 @@ const defaults = {
host: "localhost",
port: "5432"
},
pgp = pg_promise({ noWarnings: ! /* tag? */!0 /*await rows.hasNext()*/ /*await rows.hasNext()*/ /*await rows.hasNext()*/ }),
pgp = pg_promise(),
VERSION = 1,
MAXIMUM_QUERY_VARS = 16000,
fields = ["map", "ctx", "reg", "tag", "cfg"],
@@ -55,7 +53,7 @@ export default function PostgresDB(name, config = {}) {
}
if ("object" == typeof name) {
config = name;
name = name.name;
name = config.name;
}
if (!name) {
console.info("Default storage space was used, because a name was not passed.");
@@ -63,7 +61,8 @@ export default function PostgresDB(name, config = {}) {
this.id = (config.schema ? sanitize(config.schema) : defaults.schema) + (name ? "_" + sanitize(name) : "");
this.field = config.field ? "_" + sanitize(config.field) : "";
this.type = config.type ? types[config.type.toLowerCase()] : "text";
this.support_tag_search = !0;
this.support_tag_search = /* tag? */!0 /*await rows.hasNext()*/ /*await rows.hasNext()*/
/*await rows.hasNext()*/;
if (!this.type) throw new Error("Unknown type of ID '" + config.type + "'");
this.db = DB || (DB = config.db || null);
Object.assign(defaults, config);
@@ -178,8 +177,8 @@ PostgresDB.prototype.open = async function () {
};
PostgresDB.prototype.close = function () {
this.db.close && this.db.close();
this.db = DB = null;
//DB && DB.close && DB.close();
this.db = /*DB =*/null;
return this;
};
@@ -318,7 +317,9 @@ PostgresDB.prototype.enrich = async function (ids) {
};
PostgresDB.prototype.has = function (id) {
return this.db.oneOrNone("SELECT EXISTS(SELECT 1 FROM " + this.id + ".reg WHERE id = $1)", [id]);
return this.db.oneOrNone("SELECT EXISTS(SELECT 1 FROM " + this.id + ".reg WHERE id = $1)", [id]).then(function (result) {
return !!(result && result.exists);
});
};
PostgresDB.prototype.search = function (flexsearch, query, limit = 100, offset = 0, suggest = !1, resolve = !0, enrich = !1, tags) {
@@ -356,7 +357,7 @@ PostgresDB.prototype.search = function (flexsearch, query, limit = 100, offset =
${enrich ? ", doc" : ""}
FROM (
SELECT id, count(*) as count,
${suggest ? "SUM" : "MIN"}(res) as res
${suggest ? "SUM" : "SUM" /*"MIN"*/}(res) as res
FROM ${this.id}.ctx${this.field}
WHERE ${where}
GROUP BY id
@@ -406,8 +407,7 @@ PostgresDB.prototype.search = function (flexsearch, query, limit = 100, offset =
for (let i = 0; i < query_length; i++) {
where += (where ? "," : "") + "$" + count++;
}
where = "key " + (1 < query_length ? "IN (" + where + ")" : "= " + where);
}where = "key " + (1 < query_length ? "IN (" + where + ")" : "= " + where);
if (tags) {
where = "(" + where + ")";
@@ -423,7 +423,7 @@ PostgresDB.prototype.search = function (flexsearch, query, limit = 100, offset =
${enrich ? ", doc" : ""}
FROM (
SELECT id, count(*) as count,
${suggest ? "SUM" : "MIN"}(res) as res
${suggest ? "SUM" : "SUM" /*"MIN"*/}(res) as res
FROM ${this.id}.map${this.field}
WHERE ${where}
GROUP BY id
@@ -523,17 +523,23 @@ PostgresDB.prototype.info = function () {
// };
PostgresDB.prototype.transaction = function (task) {
if (TRX) {
return task.call(this, TRX);
}
const self = this;
return (TRX || this.db).tx(function (trx) {
return task.call(self, TRX = trx);
}).finally(function () {
TRX = null;
return this.db.tx(function (trx) {
return task.call(self, trx);
});
};
// PostgresDB.prototype.transaction = async function(task){
// if(TRX){
// return await task.call(this, TRX);
// }
// const self = this;
// return this.db.tx(async function(trx){
// await task.call(self, TRX = trx);
// TRX = null;
// });
// };
PostgresDB.prototype.commit = async function (flexsearch, _replace, _append) {
// process cleanup tasks

30
dist/module-debug/db/redis/index.js vendored
View File

@@ -1,5 +1,3 @@
import { createClient } from "redis";
const defaults = {
host: "localhost",
@@ -74,7 +72,7 @@ RedisDB.prototype.open = async function () {
};
RedisDB.prototype.close = async function () {
await this.db.disconnect(); // this.db.client.disconnect();
// this.db.client.disconnect();
this.db = null;
return this;
};
@@ -167,14 +165,14 @@ RedisDB.prototype.has = function (id) {
};
RedisDB.prototype.search = function (flexsearch, query, limit = 100, offset = 0, suggest = !1, resolve = !0, enrich = !1, tags) {
let result,
params = [];
let result;
if (1 < query.length && flexsearch.depth) {
const key = this.id + "ctx" + this.field + ":";
let params = [],
keyword = query[0],
let keyword = query[0],
term;
@@ -184,29 +182,37 @@ RedisDB.prototype.search = function (flexsearch, query, limit = 100, offset = 0,
params.push(key + (swap ? term : keyword) + ":" + (swap ? keyword : term));
keyword = term;
}
query = params;
} else {
const key = this.id + "map" + this.field + ":";
for (let i = 0; i < query.length; i++) {
query[i] = key + query[i];
params.push(key + query[i]);
}
}
query = params;
const type = this.type;
let key = this.id + "tmp:" + Math.random();
if (suggest) {
if (tags) for (let i = 0; i < tags.length; i += 2) {
query.push(this.id + "tag-" + sanitize(tags[i]) + ":" + tags[i + 1]);
}
const multi = this.db.multi().zUnionStore(key, query, { AGGREGATE: "SUM" });
// Strict Tag Intersection: it does not put tags into union, instead it calculates the
// intersection against the term match union. This was the default behavior
// of Tag-Search. But putting everything into union will also provide suggestions derived
// from tags when no term was matched.
if (tags) {
// copy over zInterStore into the same destination surprisingly works fine
// const key2 = key + ":2";
query = [key];
for (let i = 0; i < tags.length; i += 2) {
query.push(this.id + "tag-" + sanitize(tags[i]) + ":" + tags[i + 1]);
}
multi.zInterStore(key, query, { AGGREGATE: "SUM" });
// .unlink(key)
// key = key2;
}
result = multi[resolve ? "zRange" : "zRangeWithScores"](key, "" + offset, "" + (offset + limit - 1), { REV: !0 }).unlink(key).exec();
} else {

24
dist/module-debug/db/sqlite/index.js vendored
View File

@@ -1,5 +1,3 @@
//const sqlite3 = require("sqlite3").verbose();
import sqlite3 from "sqlite3";
import path from "path";
@@ -197,7 +195,7 @@ SqliteDB.prototype.open = async function () {
};
SqliteDB.prototype.close = function () {
this.db.close();
this.db && this.db.close();
this.db = null;
DB[this.id] = null;
TRX[this.id] = null;
@@ -392,7 +390,7 @@ SqliteDB.prototype.has = function (id) {
stmt: `SELECT EXISTS(SELECT 1 FROM main.reg WHERE id = ?) as exist`,
params: [id]
}).then(function (result) {
return result && result.exist;
return !!(result && result.exist);
});
};
@@ -431,7 +429,7 @@ SqliteDB.prototype.search = function (flexsearch, query, limit = 100, offset = 0
${enrich ? ", doc" : ""}
FROM (
SELECT id, count(*) as count,
${suggest ? "SUM" : "MIN"}(res) as res
${suggest ? "SUM" : "SUM" /*"MIN"*/}(res) as res
FROM main.ctx${this.field}
WHERE ${stmt}
GROUP BY id
@@ -500,7 +498,7 @@ SqliteDB.prototype.search = function (flexsearch, query, limit = 100, offset = 0
${enrich ? ", doc" : ""}
FROM (
SELECT id, count(*) as count,
${suggest ? "SUM" : "MIN"}(res) as res
${suggest ? "SUM" : "SUM" /*"MIN"*/}(res) as res
FROM main.map${this.field}
WHERE ${stmt}
GROUP BY id
@@ -554,15 +552,13 @@ SqliteDB.prototype.info = function () {
SqliteDB.prototype.transaction = function (task, callback) {
const self = this;
if (TRX[this.id]) {
return TRX[this.id].then(function () {
return self.transaction(task, callback);
});
return task.call(this);
}
const db = this.db;
const db = this.db,
self = this;
return TRX[this.id] = new Promise(function (resolve, reject) {
db.exec("PRAGMA optimize");
@@ -780,8 +776,8 @@ SqliteDB.prototype.remove = function (ids) {
this.db.run("DELETE FROM main.ctx" + self.field + " WHERE id IN (" + stmt + ")", ids);
this.db.run("DELETE FROM main.tag" + self.field + " WHERE id IN (" + stmt + ")", ids);
this.db.run("DELETE FROM main.reg WHERE id IN (" + stmt + ")", ids);
}).then(function (res) {
return next ? self.remove(next) : res;
}).then(function (result) {
return next ? self.remove(next) : result;
});
};

31
dist/module-debug/document.js vendored
View File

@@ -43,9 +43,7 @@ export default function Document(options) {
keystore && (this.keystore = keystore);
this.fastupdate = !!options.fastupdate;
// Shared Registry
this.reg = this.fastupdate && !options.worker && !options.db ? keystore && /* tag? */ /* stringify */ /* stringify */ /* single param */ /* skip update: */ /* append: */ /* skip update: */ /* skip_update: */ /* skip deletion */!0 /*await rows.hasNext()*/ /*await rows.hasNext()*/ /*await rows.hasNext()*/
? new KeystoreMap(keystore) : new Map() : keystore && !0 ? new KeystoreSet(keystore) : new Set();
this.reg = this.fastupdate && !options.worker && !options.db ? keystore && /* tag? */ /* stringify */ /* stringify */ /* single param */ /* skip update: */ /* append: */ /* skip update: */ /* skip_update: */ /* skip deletion */!0 /*await rows.hasNext()*/ /*await rows.hasNext()*/ /*await rows.hasNext()*/ ? new KeystoreMap(keystore) : new Map() : keystore && !0 ? new KeystoreSet(keystore) : new Set();
// todo support custom filter function
this.storetree = (tmp = document.store || null) && tmp && !0 !== tmp && [];
@@ -54,9 +52,7 @@ export default function Document(options) {
this.cache = (tmp = options.cache || null) && new Cache(tmp);
// do not apply cache again for the indexes since .searchCache()
// is just a wrapper over .search()
options.cache = /* suggest */ /* append: */ /* enrich */
!1;
options.cache = /* suggest */ /* append: */ /* enrich */!1;
this.worker = options.worker;
@@ -208,7 +204,7 @@ Document.prototype.commit = async function (replace, append) {
// parallel:
const promises = [];
for (const index of this.index.values()) {
promises.push(index.db.commit(index, replace, append));
promises.push(index.commit(index, replace, append));
}
await Promise.all(promises);
this.reg.clear();
@@ -404,15 +400,15 @@ Document.prototype.remove = function (id) {
Document.prototype.clear = function () {
//const promises = [];
const promises = [];
for (const index of this.index.values()) {
// db index will add clear task
index.clear();
// const promise = index.clear();
// if(promise instanceof Promise){
// promises.push(promise);
// }
const promise = index.clear();
// worker indexes will return promises
if (promise.then) {
promises.push(promise);
}
}
if (this.tag) {
@@ -425,9 +421,11 @@ Document.prototype.clear = function () {
this.store.clear();
}
return this; /*promises.length
? Promise.all(promises)
:*/
if (this.cache) {
this.cache.clear();
}
return promises.length ? Promise.all(promises) : this;
};
/**
@@ -435,6 +433,7 @@ Document.prototype.clear = function () {
* @return {boolean|Promise<boolean>}
*/
Document.prototype.contain = function (id) {
if (this.db) {
return this.index.get(this.field[0]).db.has(id);
}

10
dist/module-debug/document/search.js vendored
View File

@@ -1,5 +1,5 @@
import { DocumentSearchOptions, DocumentSearchResults, EnrichedDocumentSearchResults, MergedDocumentSearchResults, EnrichedSearchResults, SearchResults, IntermediateSearchResults } from "../type.js";
import { DocumentSearchOptions, DocumentSearchResults, EnrichedDocumentSearchResults, MergedDocumentSearchResults, MergedDocumentSearchEntry, EnrichedSearchResults, SearchResults, IntermediateSearchResults } from "../type.js";
import { create_object, is_array, is_object, is_string, parse_simple } from "../common.js";
import { intersect_union } from "../intersect.js";
import Document from "../document.js";
@@ -105,7 +105,7 @@ Document.prototype.search = function (query, limit, options, _promises) {
enrich = this.store && options.enrich && resolve;
highlight = options.highlight && enrich;
highlight = enrich && options.highlight;
limit = options.limit || limit;
offset = options.offset || 0;
limit || (limit = 100);
@@ -438,7 +438,7 @@ function highlight_fields(result, query, index, field, tree, template) {
// if(typeof template === "string"){
// template = new RegExp(template, "g");
// }
console.log("template", template);
let encoder, query_enc, tokenize;
@@ -518,7 +518,7 @@ function highlight_fields(result, query, index, field, tree, template) {
* @return {MergedDocumentSearchResults}
*/
function merge_fields(fields, limit) {
/** @type {MergedDocumentSearchResults} */
/** @type {Array<MergedDocumentSearchEntry>} */
const final = [],
set = create_object();
@@ -547,7 +547,7 @@ function merge_fields(fields, limit) {
return final;
}
entry.field = set[id] = [field.field];
final.push(entry);
final.push( /** @type {MergedDocumentSearchEntry} */entry);
} else {
tmp.push(field.field);
}

40
dist/module-debug/encoder.js vendored
View File

@@ -91,9 +91,7 @@ Encoder.prototype.assign = function (options) {
* pre-processing string input
* @type {Function|boolean}
*/
this.normalize = /** @type {Function|boolean} */merge_option(options.normalize, /* tag? */ /* stringify */ /* stringify */ /* single param */ /* skip update: */ /* append: */ /* skip update: */ /* skip_update: */ /* skip deletion */!0 /*await rows.hasNext()*/ /*await rows.hasNext()*/
/*await rows.hasNext()*/, this.normalize);
this.normalize = /** @type {Function|boolean} */merge_option(options.normalize, /* tag? */ /* stringify */ /* stringify */ /* single param */ /* skip update: */ /* append: */ /* skip update: */ /* skip_update: */ /* skip deletion */!0 /*await rows.hasNext()*/ /*await rows.hasNext()*/ /*await rows.hasNext()*/, this.normalize);
// {
// letter: true,
@@ -185,20 +183,16 @@ Encoder.prototype.assign = function (options) {
/** @type {Array<Array<string, string>>} */normalize_polyfill);
}
// options
this.rtl = merge_option(options.rtl, !1, this.rtl);
tmp = options.filter;
this.filter = "function" == typeof tmp ? tmp : merge_option(tmp && new Set(tmp), null, this.filter);
this.dedupe = merge_option(options.dedupe, !1, this.dedupe);
this.filter = merge_option((tmp = options.filter) && new Set(tmp), null, this.filter);
this.matcher = merge_option((tmp = options.matcher) && new Map(tmp), null, this.matcher);
this.mapper = merge_option((tmp = options.mapper) && new Map(tmp), null, this.mapper);
this.stemmer = merge_option((tmp = options.stemmer) && new Map(tmp), null, this.stemmer);
this.replacer = merge_option(options.replacer, null, this.replacer);
this.minlength = merge_option(options.minlength, 1, this.minlength);
this.maxlength = merge_option(options.maxlength, 0, this.maxlength);
// minimum required tokenizer by this encoder
//this.tokenize = options["tokenize"] || "";
this.rtl = merge_option(options.rtl, !1, this.rtl);
// auto-balanced cache
this.cache = tmp = merge_option(options.cache, !0, this.cache);
@@ -260,8 +254,13 @@ Encoder.prototype.addStemmer = function (match, replace) {
};
Encoder.prototype.addFilter = function (term) {
this.filter || (this.filter = new Set());
this.filter.add(term);
if ("function" == typeof term) {
// does not support merge yet
this.filter = term; //merge_option(term, term, this.filter);
} else {
this.filter || (this.filter = new Set());
this.filter.add(term);
}
this.cache && clear(this);
return this;
};
@@ -401,7 +400,7 @@ Encoder.prototype.encode = function (str) {
}
// pre-filter before cache
if (this.filter && this.filter.has(word)) {
if (this.filter && ("function" == typeof this.filter ? !this.filter(word) : this.filter.has(word))) {
continue;
}
@@ -416,16 +415,17 @@ Encoder.prototype.encode = function (str) {
this.timer = setTimeout(clear, 50, this);
}
}
// todo compare advantages when filter/stemmer are also encoded
// apply stemmer before bigger transformations
if (this.stemmer && 2 < word.length) {
// for(const item of this.stemmer){
// const key = item[0];
// const value = item[1];
//
// if(word.length > key.length && word.endsWith(key)){
// word = word.substring(0, word.length - key.length) + value;
// break;
// }
//
// // const position = word.length - key.length;
// // if(position > 0 && word.substring(position) === key){
// // word = word.substring(0, position) + value;
@@ -433,11 +433,15 @@ Encoder.prototype.encode = function (str) {
// // }
// }
this.stemmer_test || (this.stemmer_test = new RegExp("(?!^)(" + this.stemmer_str + ")$"));
const old = word;
word = word.replace(this.stemmer_test, match => this.stemmer.get(match));
// 4. post-filter after matcher and stemmer was applied
if (word.length < this.minlength || this.filter && this.filter.has(word)) {
word = "";
// 4. post-filter after stemmer was applied
if (old !== word && this.filter && word.length >= this.minlength) {
if ("function" == typeof this.filter ? !this.filter(word) : this.filter.has(word)) {
word = "";
}
}
}

12
dist/module-debug/index.js vendored
View File

@@ -35,9 +35,10 @@ export default function Index(options, _register) {
options = /** @type IndexOptions */options ? apply_preset(options) : {};
/** @type {*} */
let tmp = options.context;
/** @type ContextOptions */
const context = /* tag? */ /* stringify */ /* stringify */ /* single param */ /* skip update: */ /* append: */ /* skip update: */ /* skip_update: */ /* skip deletion */!0 /*await rows.hasNext()*/ /*await rows.hasNext()*/ /*await rows.hasNext()*/ === tmp ? { depth: 1 } : tmp || {},
const context = /** @type ContextOptions */ /* tag? */ /* stringify */ /* stringify */ /* single param */ /* skip update: */ /* append: */ /* skip update: */ /* skip_update: */ /* skip deletion */!0 /*await rows.hasNext()*/ /*await rows.hasNext()*/ /*await rows.hasNext()*/ === tmp ? { depth: 1 } : tmp || {},
encoder = is_string(options.encoder) ? Charset[options.encoder] : options.encode || options.encoder || default_encoder;
/** @type Encoder */
@@ -47,12 +48,17 @@ export default function Index(options, _register) {
this.resolution = options.resolution || 9;
this.tokenize = (tmp = options.tokenize) && "default" !== tmp && tmp || "strict";
this.tokenize = tmp = (tmp = options.tokenize) && "default" !== tmp && tmp || "strict";
this.depth = "strict" === tmp && context.depth || 0;
this.bidirectional = !1 !== context.bidirectional;
this.fastupdate = !!options.fastupdate;
this.score = options.score || null;
if (context && "strict" !== this.tokenize) {
console.warn("Context-Search could not applied, because it is just supported when using the tokenizer \"strict\".");
}
tmp = options.keystore || 0;
tmp && (this.keystore = tmp);
@@ -167,7 +173,7 @@ function cleanup_index(map) {
for (let i = 0, arr; i < map.length; i++) {
(arr = map[i]) && (count += arr.length);
}
} else for (const item of map) {
} else for (const item of map.entries()) {
const key = item[0],
value = item[1],
tmp = cleanup_index(value);

3
dist/module-debug/index/search.js vendored
View File

@@ -57,7 +57,8 @@ Index.prototype.search = function (query, limit, options) {
offset = options.offset || 0;
context = options.context;
suggest = options.suggest;
resolve = /*global_resolve &&*/ /* suggest */ /* append: */ /* enrich */!1 !== options.resolve;
resolve = /*global_resolve &&*/ /* suggest */
/* append: */ /* enrich */!1 !== options.resolve;
//resolve || (global_resolve = 0);
enrich = resolve && options.enrich;
boost = options.boost;

15
dist/module-debug/intersect.js vendored
View File

@@ -77,6 +77,19 @@ export function intersect(arrays, resolution, limit, offset, suggest, boost, res
}
tmp.push(id);
// fast path early result when limit was set
if (resolve) {
if (limit && count === length - 1) {
// if(tmp.length - offset > limit){
// return tmp.slice(offset, limit + offset);
// }
if (tmp.length - offset === limit) {
return tmp;
}
}
}
// todo break early on suggest: true
}
}
}
@@ -104,6 +117,8 @@ export function intersect(arrays, resolution, limit, offset, suggest, boost, res
result = result.slice(offset, limit + offset);
}
} else {
// todo this is doing the same as Resolver.resolve({limit}) ?
// todo check limit + offset when resolve = false
const final = [];
for (let i = 0, arr; i < result.length; i++) {
arr = result[i];

153
dist/module-debug/keystore.js vendored
View File

@@ -1,63 +1,57 @@
import { create_object } from "./common.js";
/**
* @param bitlength
* @constructor
*/
export function KeystoreObj(bitlength = 8) {
if (!this) {
return new KeystoreObj(bitlength);
}
this.index = create_object();
this.keys = [];
if (32 < bitlength) {
this.crc = lcg64;
this.bit = BigInt(bitlength);
} else {
this.crc = lcg;
this.bit = bitlength;
}
return (/*this.proxy =*/new Proxy(this, {
get(target, key) {
const address = target.crc(key),
obj = target.index[address];
return obj && obj[key];
},
set(target, key, value) {
const address = target.crc(key);
let obj = target.index[address];
if (!obj) {
target.index[address] = obj = create_object();
target.keys.push(address);
}
obj[key] = value;
return (/* tag? */ /* stringify */ /* stringify */ /* single param */ /* skip update: */ /* append: */ /* skip update: */ /* skip_update: */ /* skip deletion */
// splice:
!0 /*await rows.hasNext()*/
/*await rows.hasNext()*/ /*await rows.hasNext()*/
);
},
delete(target, key) {
const address = target.crc(key),
obj = target.index[address];
obj && delete obj[key];
return !0;
}
})
);
}
KeystoreObj.prototype.clear = function () {
this.index = create_object();
this.keys = [];
};
// /**
// * @param bitlength
// * @constructor
// */
//
// export function KeystoreObj(bitlength = 8){
//
// if(!this || this.constructor !== KeystoreObj){
// return new KeystoreObj(bitlength);
// }
//
// this.index = create_object();
// this.keys = [];
//
// if(bitlength > 32){
// this.crc = lcg64;
// this.bit = BigInt(bitlength);
// }
// else {
// this.crc = lcg;
// this.bit = bitlength;
// }
//
// return /*this.proxy =*/ new Proxy(this, {
// get(target, key) {
// const address = target.crc(key);
// const obj = target.index[address];
// return obj && obj[key];
// },
// set(target, key, value){
// const address = target.crc(key);
// let obj = target.index[address];
// if(!obj){
// target.index[address] = obj = create_object();
// target.keys.push(address);
// }
// obj[key] = value;
// return true;
// },
// delete(target, key){
// const address = target.crc(key);
// const obj = target.index[address];
// obj && delete obj[key];
// return true;
// }
// });
// }
//
// KeystoreObj.prototype.clear = function(){
// this.index = create_object();
// this.keys = [];
// };
// KeystoreObj.prototype.destroy = function(){
// this.index = null;
@@ -94,7 +88,7 @@ function _slice(self, start, end, splice) {
export function KeystoreArray(arr) {
if (!this) {
if (!this || this.constructor !== KeystoreArray) {
return new KeystoreArray(arr);
}
@@ -138,7 +132,11 @@ export function KeystoreArray(arr) {
return function (key) {
for (let i = 0; i < self.index.length; i++) {
if (self.index[i].includes(key)) {
return !0;
return (/* tag? */ /* stringify */ /* stringify */ /* single param */ /* skip update: */ /* append: */
/* skip update: */ /* skip_update: */ /* skip deletion */
// splice:
!0 /*await rows.hasNext()*/ /*await rows.hasNext()*/ /*await rows.hasNext()*/
);
}
}
return (/* suggest */ /* append: */ /* enrich */!1
@@ -189,22 +187,43 @@ KeystoreArray.prototype.destroy = function () {
KeystoreArray.prototype.push = function () {};
/**
* @interface
*/
function Keystore() {
/** @type {Object<number, Map|Set>} */
this.index;
/** @type {Array<Map|Set>} */
this.refs;
/** @type {number} */
this.size;
/** @type {function((string|number)):number} */
this.crc;
/** @type {bigint|number} */
this.bit;
}
/**
* @param bitlength
* @constructor
* @implements {Keystore}
*/
export function KeystoreMap(bitlength = 8) {
if (!this) {
if (!this || this.constructor !== KeystoreMap) {
return new KeystoreMap(bitlength);
}
/** @type {Object<number, Map>} */
this.index = create_object();
/** @type {Array<Map>} */
this.refs = [];
/** @type {number} */
this.size = 0;
if (32 < bitlength) {
/** @type {function((string|number)):number} */
this.crc = lcg64;
this.bit = BigInt(bitlength);
} else {
@@ -231,26 +250,31 @@ KeystoreMap.prototype.set = function (key, value) {
} else {
this.index[address] = map = new Map([[key, value]]);
this.refs.push(map);
this.size++;
}
};
/**
* @param bitlength
* @constructor
* @implements {Keystore}
*/
export function KeystoreSet(bitlength = 8) {
if (!this) {
if (!this || this.constructor !== KeystoreSet) {
return new KeystoreSet(bitlength);
}
// using plain Object with numeric key access
// just for max performance
/** @type {Object<number, Set>} */
this.index = create_object();
/** @type {Array<Set>} */
this.refs = [];
/** @type {number} */
this.size = 0;
if (32 < bitlength) {
/** @type {function((string|number)):number} */
this.crc = lcg64;
this.bit = BigInt(bitlength);
} else {
@@ -270,6 +294,7 @@ KeystoreSet.prototype.add = function (key) {
} else {
this.index[address] = set = new Set([key]);
this.refs.push(set);
this.size++;
}
};

77
dist/module-debug/serialize.js vendored
View File

@@ -412,20 +412,41 @@ ctx: "gulliver+travel:1,2,3|4,5,6|7,8,9;"
export function serialize(withFunctionWrapper = /* tag? */ /* stringify */ /* stringify */ /* single param */ /* skip update: */ /* append: */ /* skip update: */ /* skip_update: */ /* skip deletion */ // splice:
!0 /*await rows.hasNext()*/ /*await rows.hasNext()*/ /*await rows.hasNext()*/) {
if (!this.reg.size) return "";
let reg = '',
type = "";
map = '',
ctx = '';
for (const key of this.reg.keys()) {
type || (type = typeof key);
reg += (reg ? ',' : '') + ("string" == type ? '"' + key + '"' : key);
if (this.reg.size) {
let type;
for (const key of this.reg.keys()) {
type || (type = typeof key);
reg += (reg ? ',' : '') + ("string" === type ? '"' + key + '"' : key);
}
reg = 'index.reg=new Set([' + reg + ']);';
map = parse_map(this.map, type);
map = "index.map=new Map([" + map + "]);";
for (const context of this.ctx.entries()) {
const key_ctx = context[0],
value_ctx = context[1];
let ctx_map = parse_map(value_ctx, type);
ctx_map = "new Map([" + ctx_map + "])";
ctx_map = '["' + key_ctx + '",' + ctx_map + ']';
ctx += (ctx ? ',' : '') + ctx_map;
}
ctx = "index.ctx=new Map([" + ctx + "]);";
}
reg = 'index.reg=new Set([' + reg + ']);';
let map = '';
for (const item of this.map.entries()) {
return withFunctionWrapper ? "function inject(index){" + reg + map + ctx + "}" : reg + map + ctx;
}
function parse_map(map, type) {
let result = '';
for (const item of map.entries()) {
const key = item[0],
value = item[1];
@@ -434,43 +455,13 @@ export function serialize(withFunctionWrapper = /* tag? */ /* stringify */ /* st
ids = value[i] || [''];
let str = '';
for (let j = 0; j < ids.length; j++) {
str += (str ? ',' : '') + ("string" == type ? '"' + ids[j] + '"' : ids[j]);
str += (str ? ',' : '') + ("string" === type ? '"' + ids[j] + '"' : ids[j]);
}
str = '[' + str + ']';
res += (res ? ',' : '') + str;
}
res = '["' + key + '",[' + res + ']]';
map += (map ? ',' : '') + res;
result += (result ? ',' : '') + res;
}
map = "index.map=new Map([" + map + "]);";
let ctx = '';
for (const context of this.ctx.entries()) {
const key_ctx = context[0],
value_ctx = context[1];
for (const item of value_ctx.entries()) {
const key = item[0],
value = item[1];
let res = '';
for (let i = 0, ids; i < value.length; i++) {
ids = value[i] || [''];
let str = '';
for (let j = 0; j < ids.length; j++) {
str += (str ? ',' : '') + ("string" == type ? '"' + ids[j] + '"' : ids[j]);
}
str = '[' + str + ']';
res += (res ? ',' : '') + str;
}
res = 'new Map([["' + key + '",[' + res + ']]])';
res = '["' + key_ctx + '",' + res + ']';
ctx += (ctx ? ',' : '') + res;
}
}
ctx = "index.ctx=new Map([" + ctx + "]);";
return withFunctionWrapper ? "function inject(index){" + reg + map + ctx + "}" : reg + map + ctx;
return result;
}

11
dist/module-debug/type.js vendored
View File

@@ -186,12 +186,17 @@ export let DocumentSearchResults = [];
export let EnrichedDocumentSearchResults = [];
/**
* @typedef Array<{
* @typedef {{
* id: (number|string),
* doc: (Object|null),
* field: (Array<string>|undefined),
* tag: (Array<string>|undefined)
* }>
* }}
*/
export let MergedDocumentSearchEntry = {};
/**
* @typedef Array<MergedDocumentSearchEntry>
*/
export let MergedDocumentSearchResults = [];
@@ -218,7 +223,7 @@ export let EncoderSplitOptions = {};
* normalize: (boolean|(function(string):string)|undefined),
* prepare: ((function(string):string)|undefined),
* finalize: ((function(Array<string>):(Array<string>|void))|undefined),
* filter: (Set<string>|undefined),
* filter: (Set<string>|function(string):boolean|undefined),
* matcher: (Map<string, string>|undefined),
* mapper: (Map<string, string>|undefined),
* stemmer: (Map<string, string>|undefined),