mirror_js/flexsearch
1
0
Fork 0
mirror of https://github.com/nextapps-de/flexsearch.git synced 2025-08-27 07:44:35 +02:00
Code Issues Releases Wiki Activity

improve redis implementation

Browse Source
This commit is contained in:
Thomas Wilkerling
2025-04-16 09:03:16 +02:00
parent ac4e0cd8d1
commit 77af8c4941
1 changed files with 50 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
src/db/redis/index.js
View File

@@ -38,6 +38,8 @@ export default function RedisDB(name, config = {}){
this.fastupdate = true; this.fastupdate = true;
this.db = config.db || DB || null; this.db = config.db || DB || null;
this.support_tag_search = true; this.support_tag_search = true;
this.resolution = 9;
this.resolution_ctx = 9;
//this.trx = false; //this.trx = false;
Object.assign(defaults, config); Object.assign(defaults, config);
this.db && delete defaults.db; this.db && delete defaults.db;
@@ -53,6 +55,8 @@ RedisDB.prototype.mount = function(flexsearch){
return flexsearch.mount(this); return flexsearch.mount(this);
} }
flexsearch.db = this; flexsearch.db = this;
this.resolution = flexsearch.resolution;
this.resolution_ctx = flexsearch.resolution_ctx;
// todo support // todo support
//this.fastupdate = flexsearch.fastupdate; //this.fastupdate = flexsearch.fastupdate;
return this.open(); return this.open();
@@ -98,7 +102,7 @@ RedisDB.prototype.clear = function(){
]); ]);
}; };
function create_result(range, type, resolve, enrich){ function create_result(range, type, resolve, enrich, resolution){
if(resolve){ if(resolve){
if(type === "number"){ if(type === "number"){
for(let i = 0, tmp, id; i < range.length; i++){ for(let i = 0, tmp, id; i < range.length; i++){
@@ -120,7 +124,7 @@ function create_result(range, type, resolve, enrich){
id = type === "number" id = type === "number"
? parseInt(tmp.id || tmp, 10) ? parseInt(tmp.id || tmp, 10)
: tmp.id || tmp; : tmp.id || tmp;
score = tmp.score; score = resolution - tmp.score;
result[score] || (result[score] = []); result[score] || (result[score] = []);
result[score].push(id); result[score].push(id);
} }
@@ -161,7 +165,7 @@ RedisDB.prototype.get = function(key, ctx, limit = 0, offset = 0, resolve = true
return result.then(async function(range){ return result.then(async function(range){
if(!range.length) return range; if(!range.length) return range;
if(enrich) range = await self.enrich(range); if(enrich) range = await self.enrich(range);
return create_result(range, type, resolve, enrich); return create_result(range, type, resolve, enrich, ctx ? self.resolution_ctx : self.resolution);
}); });
}; };
@@ -198,10 +202,12 @@ RedisDB.prototype.has = function(id){
RedisDB.prototype.search = function(flexsearch, query, limit = 100, offset = 0, suggest = false, resolve = true, enrich = false, tags){ RedisDB.prototype.search = function(flexsearch, query, limit = 100, offset = 0, suggest = false, resolve = true, enrich = false, tags){
const ctx = query.length > 1 && flexsearch.depth;
let result; let result;
let params = []; let params = [];
let weights = [];
if(query.length > 1 && flexsearch.depth){ if(ctx){
const key = this.id + "ctx" + this.field + ":"; const key = this.id + "ctx" + this.field + ":";
let keyword = query[0]; let keyword = query[0];
@@ -211,14 +217,16 @@ RedisDB.prototype.search = function(flexsearch, query, limit = 100, offset = 0,
term = query[i]; term = query[i];
swap = flexsearch.bidirectional && (term > keyword); swap = flexsearch.bidirectional && (term > keyword);
params.push(key + (swap ? term : keyword) + ":" + (swap ? keyword : term)); params.push(key + (swap ? term : keyword) + ":" + (swap ? keyword : term));
weights.push(1);
keyword = term; keyword = term;
} }
} }
else { else{
const key = this.id + "map" + this.field + ":"; const key = this.id + "map" + this.field + ":";
for(let i = 0; i < query.length; i++){ for(let i = 0; i < query.length; i++){
params.push(key + query[i]); params.push(key + query[i]);
weights.push(1);
} }
} }
@@ -232,9 +240,18 @@ RedisDB.prototype.search = function(flexsearch, query, limit = 100, offset = 0,
if(!strict_tag_intersection){ if(!strict_tag_intersection){
if(tags) for(let i = 0; i < tags.length; i += 2){ if(tags) for(let i = 0; i < tags.length; i += 2){
query.push(this.id + "tag-" + sanitize(tags[i]) + ":" + tags[i + 1]); query.push(this.id + "tag-" + sanitize(tags[i]) + ":" + tags[i + 1]);
weights.push(1);
} }
} }
const multi = this.db.multi().zUnionStore(key, query, { AGGREGATE: "SUM" }); const multi = this.db.multi();
// The zStore implementation lacks of ordering by match count (occurrences).
// Unfortunately, I couldn't find an elegant way to overcome this issue completely.
// For this reason it needs additionally a zInterStore to boost at least matches
// when all terms were found
multi.zInterStore(key, query, { AGGREGATE: "SUM" });
query.push(key);
weights.push(query.length);
multi.zUnionStore(key, query, { WEIGHTS: weights, AGGREGATE: "SUM" });
// Strict Tag Intersection: it does not put tags into union, instead it calculates the // Strict Tag Intersection: it does not put tags into union, instead it calculates the
// intersection against the term match union. This was the default behavior // intersection against the term match union. This was the default behavior
// of Tag-Search. But putting everything into union will also provide suggestions derived // of Tag-Search. But putting everything into union will also provide suggestions derived
@@ -247,23 +264,33 @@ RedisDB.prototype.search = function(flexsearch, query, limit = 100, offset = 0,
for(let i = 0; i < tags.length; i += 2){ for(let i = 0; i < tags.length; i += 2){
query.push(this.id + "tag-" + sanitize(tags[i]) + ":" + tags[i + 1]); query.push(this.id + "tag-" + sanitize(tags[i]) + ":" + tags[i + 1]);
} }
multi.zInterStore(key, query, { AGGREGATE: "SUM" }); multi.zInterStore(key, query, { AGGREGATE: "MAX" });
// .unlink(key) // .unlink(key)
// key = key2; // key = key2;
} }
} }
result = multi result = multi
[(resolve ? "zRange" : "zRangeWithScores")](key, "" + offset, "" + (offset + limit - 1), { REV: true }) [(resolve ? "zRange" : "zRangeWithScores")](
key,
"" + offset,
"" + (offset + limit - 1),
{ REV: true }
)
.unlink(key) .unlink(key)
.exec(); .exec();
} }
else{ else{
if(tags) for(let i = 0; i < tags.length; i+=2){ if(tags) for(let i = 0; i < tags.length; i += 2){
query.push(this.id + "tag-" + sanitize(tags[i]) + ":" + tags[i + 1]); query.push(this.id + "tag-" + sanitize(tags[i]) + ":" + tags[i + 1]);
} }
result = this.db.multi() result = this.db.multi()
.zInterStore(key, query, { AGGREGATE: "MIN" }) .zInterStore(key, query, { AGGREGATE: "MAX" })
[(resolve ? "zRange" : "zRangeWithScores")](key, "" + offset, "" + (offset + limit - 1), { REV: true }) [(resolve ? "zRange" : "zRangeWithScores")](
key,
"" + offset,
"" + (offset + limit - 1),
{ REV: true }
)
.unlink(key) .unlink(key)
.exec(); .exec();
} }
@@ -272,12 +299,12 @@ RedisDB.prototype.search = function(flexsearch, query, limit = 100, offset = 0,
return result.then(async function(range){ return result.then(async function(range){
range = suggest && tags range = suggest && tags
// take the 3rd result from batch return // take the 3rd result from batch return
? range[2] ? range[3]
// take the 2nd result from batch return // take the 2nd result from batch return
: range[1]; : range[suggest ? 2 : 1];
if(!range.length) return range; if(!range.length) return range;
if(enrich) range = await self.enrich(range); if(enrich) range = await self.enrich(range);
return create_result(range, type, resolve, enrich); return create_result(range, type, resolve, enrich, ctx ? self.resolution_ctx : self.resolution);
}); });
}; };
@@ -285,7 +312,7 @@ RedisDB.prototype.info = function(){
// todo // todo
}; };
RedisDB.prototype.transaction = async function(task, callback){ RedisDB.prototype.transaction = function(task, callback){
if(TRX){ if(TRX){
return task.call(this, TRX); return task.call(this, TRX);
@@ -295,8 +322,9 @@ RedisDB.prototype.transaction = async function(task, callback){
let promise1 = /*await*/ task.call(this, TRX) let promise1 = /*await*/ task.call(this, TRX)
let promise2 = TRX.exec(); let promise2 = TRX.exec();
TRX = null; TRX = null;
callback && promise.then(callback); return Promise.all([promise1, promise2]).then(function(){
await Promise.all([promise1, promise2]); callback && callback();
});
}; };
RedisDB.prototype.commit = async function(flexsearch, _replace, _append){ RedisDB.prototype.commit = async function(flexsearch, _replace, _append){
@@ -346,7 +374,7 @@ RedisDB.prototype.commit = async function(flexsearch, _replace, _append){
let result = []; let result = [];
for(let j = 0; j < ids.length; j++){ for(let j = 0; j < ids.length; j++){
result.push({ result.push({
score: i, score: this.resolution - i,
value: "" + ids[j] value: "" + ids[j]
}); });
} }
@@ -395,7 +423,10 @@ RedisDB.prototype.commit = async function(flexsearch, _replace, _append){
if((ids = arr[i]) && ids.length){ if((ids = arr[i]) && ids.length){
let result = []; let result = [];
for(let j = 0; j < ids.length; j++){ for(let j = 0; j < ids.length; j++){
result.push({ score: i, value: "" + ids[j] }); result.push({
score: this.resolution_ctx - i,
value: "" + ids[j]
});
} }
if(typeof ids[0] === "number"){ if(typeof ids[0] === "number"){
this.type = "number"; this.type = "number";