mirror of
https://github.com/nextapps-de/flexsearch.git
synced 2025-08-12 00:54:49 +02:00
export/import document worker
This commit is contained in:
Thomas Wilkerling
@@ -0,0 +1,7 @@
|
||||
```bash
|
||||
npm install
|
||||
```
|
||||
|
||||
```bash
|
||||
node index.js
|
||||
```
|
||||
@@ -0,0 +1,30 @@
|
||||
const { Encoder, Charset } = require("flexsearch");
|
||||
const EnglishPreset = require("flexsearch/lang/en");
|
||||
const fs = require("fs").promises;
|
||||
|
||||
(async function(){
|
||||
await fs.mkdir("./export/").catch(e => {});
|
||||
}());
|
||||
|
||||
module.exports = {
|
||||
tokenize: "forward",
|
||||
encoder: new Encoder(
|
||||
Charset.LatinBalance,
|
||||
EnglishPreset,
|
||||
{
|
||||
normalize: function(str){
|
||||
return str.toLowerCase();
|
||||
},
|
||||
filter: false,
|
||||
minlength: 3
|
||||
}
|
||||
),
|
||||
export: async function(key, data){
|
||||
|
||||
await fs.writeFile("./export/" + key, data, "utf8");
|
||||
},
|
||||
import: async function(file){
|
||||
|
||||
return await fs.readFile("./export/" + file, "utf8");
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,30 @@
|
||||
const { Encoder, Charset } = require("flexsearch");
|
||||
const EnglishPreset = require("flexsearch/lang/en");
|
||||
const fs = require("fs").promises;
|
||||
|
||||
(async function(){
|
||||
await fs.mkdir("./export/").catch(e => {});
|
||||
}());
|
||||
|
||||
module.exports = {
|
||||
tokenize: "forward",
|
||||
encoder: new Encoder(
|
||||
Charset.LatinBalance,
|
||||
EnglishPreset,
|
||||
{
|
||||
normalize: function(str){
|
||||
return str.toLowerCase();
|
||||
},
|
||||
filter: false,
|
||||
minlength: 3
|
||||
}
|
||||
),
|
||||
export: async function(key, data){
|
||||
|
||||
await fs.writeFile("./export/" + key, data, "utf8");
|
||||
},
|
||||
import: async function(file){
|
||||
|
||||
return await fs.readFile("./export/" + file, "utf8");
|
||||
}
|
||||
};
|
||||
141
example/nodejs-commonjs/document-worker-export-import/data.json
Normal file
141
example/nodejs-commonjs/document-worker-export-import/data.json
Normal file
@@ -0,0 +1,141 @@
|
||||
[
|
||||
{
|
||||
"tconst": "tt0000001",
|
||||
"titleType": "short",
|
||||
"primaryTitle": "Carmencita",
|
||||
"originalTitle": "Carmencita",
|
||||
"isAdult": 0,
|
||||
"startYear": "1894",
|
||||
"endYear": "",
|
||||
"runtimeMinutes": "1",
|
||||
"genres": [
|
||||
"Documentary",
|
||||
"Short"
|
||||
]
|
||||
},
|
||||
{
|
||||
"tconst": "tt0000002",
|
||||
"titleType": "short",
|
||||
"primaryTitle": "Le clown et ses chiens",
|
||||
"originalTitle": "Le clown et ses chiens",
|
||||
"isAdult": 0,
|
||||
"startYear": "1892",
|
||||
"endYear": "",
|
||||
"runtimeMinutes": "5",
|
||||
"genres": [
|
||||
"Animation",
|
||||
"Short"
|
||||
]
|
||||
},
|
||||
{
|
||||
"tconst": "tt0000003",
|
||||
"titleType": "short",
|
||||
"primaryTitle": "Pauvre Pierrot",
|
||||
"originalTitle": "Pauvre Pierrot",
|
||||
"isAdult": 0,
|
||||
"startYear": "1892",
|
||||
"endYear": "",
|
||||
"runtimeMinutes": "4",
|
||||
"genres": [
|
||||
"Animation",
|
||||
"Comedy",
|
||||
"Romance"
|
||||
]
|
||||
},
|
||||
{
|
||||
"tconst": "tt0000004",
|
||||
"titleType": "short",
|
||||
"primaryTitle": "Un bon bock",
|
||||
"originalTitle": "Un bon bock",
|
||||
"isAdult": 0,
|
||||
"startYear": "1892",
|
||||
"endYear": "",
|
||||
"runtimeMinutes": "12",
|
||||
"genres": [
|
||||
"Animation",
|
||||
"Short"
|
||||
]
|
||||
},
|
||||
{
|
||||
"tconst": "tt0000005",
|
||||
"titleType": "short",
|
||||
"primaryTitle": "Blacksmith Scene",
|
||||
"originalTitle": "Blacksmith Scene",
|
||||
"isAdult": 0,
|
||||
"startYear": "1893",
|
||||
"endYear": "",
|
||||
"runtimeMinutes": "1",
|
||||
"genres": [
|
||||
"Comedy",
|
||||
"Short"
|
||||
]
|
||||
},
|
||||
{
|
||||
"tconst": "tt0000006",
|
||||
"titleType": "short",
|
||||
"primaryTitle": "Chinese Opium Den",
|
||||
"originalTitle": "Chinese Opium Den",
|
||||
"isAdult": 0,
|
||||
"startYear": "1894",
|
||||
"endYear": "",
|
||||
"runtimeMinutes": "1",
|
||||
"genres": [
|
||||
"Short"
|
||||
]
|
||||
},
|
||||
{
|
||||
"tconst": "tt0000007",
|
||||
"titleType": "short",
|
||||
"primaryTitle": "Corbett and Courtney Before the Kinetograph",
|
||||
"originalTitle": "Corbett and Courtney Before the Kinetograph",
|
||||
"isAdult": 0,
|
||||
"startYear": "1894",
|
||||
"endYear": "",
|
||||
"runtimeMinutes": "1",
|
||||
"genres": [
|
||||
"Short",
|
||||
"Sport"
|
||||
]
|
||||
},
|
||||
{
|
||||
"tconst": "tt0000008",
|
||||
"titleType": "short",
|
||||
"primaryTitle": "Edison Kinetoscopic Record of a Sneeze",
|
||||
"originalTitle": "Edison Kinetoscopic Record of a Sneeze",
|
||||
"isAdult": 0,
|
||||
"startYear": "1894",
|
||||
"endYear": "",
|
||||
"runtimeMinutes": "1",
|
||||
"genres": [
|
||||
"Documentary",
|
||||
"Short"
|
||||
]
|
||||
},
|
||||
{
|
||||
"tconst": "tt0000009",
|
||||
"titleType": "movie",
|
||||
"primaryTitle": "Miss Jerry",
|
||||
"originalTitle": "Miss Jerry",
|
||||
"isAdult": 0,
|
||||
"startYear": "1894",
|
||||
"endYear": "",
|
||||
"runtimeMinutes": "45",
|
||||
"genres": [
|
||||
"Romance"
|
||||
]
|
||||
},
|
||||
{
|
||||
"tconst": "tt0000010",
|
||||
"titleType": "short",
|
||||
"primaryTitle": "Leaving the Factory",
|
||||
"originalTitle": "La sortie de l'usine Lumière à Lyon",
|
||||
"isAdult": 0,
|
||||
"startYear": "1895",
|
||||
"endYear": "",
|
||||
"runtimeMinutes": "1",
|
||||
"genres": [
|
||||
"Documentary",
|
||||
"Short"
|
||||
]
|
||||
}
|
||||
]
|
||||
102
example/nodejs-commonjs/document-worker-export-import/index.js
Normal file
102
example/nodejs-commonjs/document-worker-export-import/index.js
Normal file
@@ -0,0 +1,102 @@
|
||||
const { Document } = require("flexsearch");
|
||||
const { promises: fs } = require("fs");
|
||||
|
||||
// loading test data
|
||||
const data = require(__dirname + "/data.json");
|
||||
|
||||
// you will need to keep the index configuration
|
||||
// they will not export, also every change to the
|
||||
// configuration requires a full re-index
|
||||
const config = {
|
||||
worker: true,
|
||||
document: {
|
||||
id: "tconst",
|
||||
store: true,
|
||||
index: [{
|
||||
field: "primaryTitle",
|
||||
config: __dirname + "/config.primaryTitle.js"
|
||||
},{
|
||||
field: "originalTitle",
|
||||
config: __dirname + "/config.originalTitle.js"
|
||||
}],
|
||||
tag: [{
|
||||
field: "startYear"
|
||||
},{
|
||||
field: "genres"
|
||||
}]
|
||||
}
|
||||
};
|
||||
|
||||
(async function(){
|
||||
|
||||
// create the document and await (!) for the instance response
|
||||
let document = await new Document(config);
|
||||
|
||||
// add test data
|
||||
for(let i = 0; i < data.length; i++){
|
||||
await document.add(data[i]);
|
||||
}
|
||||
|
||||
// perform a query
|
||||
let result = await document.search({
|
||||
query: "karmen",
|
||||
tag: {
|
||||
"startYear": "1894",
|
||||
"genres": [
|
||||
"Documentary",
|
||||
"Short"
|
||||
]
|
||||
},
|
||||
suggest: true,
|
||||
enrich: true,
|
||||
merge: true
|
||||
});
|
||||
|
||||
console.log(result);
|
||||
|
||||
// -----------------------
|
||||
// EXPORT
|
||||
// -----------------------
|
||||
|
||||
// create folders for the export
|
||||
// it should be empty before export
|
||||
await fs.mkdir("./export/").catch(e => {});
|
||||
|
||||
await document.export(async function(key, data){
|
||||
await fs.writeFile("./export/" + key, data, "utf8");
|
||||
});
|
||||
|
||||
// -----------------------
|
||||
// IMPORT
|
||||
// -----------------------
|
||||
|
||||
// create the same type of index you have used by .export()
|
||||
// along with the same configuration
|
||||
document = await new Document(config);
|
||||
|
||||
// load them in parallel
|
||||
const files = await fs.readdir("./export/");
|
||||
await Promise.all(files.map(async file => {
|
||||
const data = await fs.readFile("./export/" + file, "utf8");
|
||||
await document.import(file, data);
|
||||
}));
|
||||
|
||||
// perform a query
|
||||
result = await document.search({
|
||||
query: "karmen",
|
||||
tag: {
|
||||
"startYear": "1894",
|
||||
"genres": [
|
||||
"Documentary",
|
||||
"Short"
|
||||
]
|
||||
},
|
||||
suggest: true,
|
||||
enrich: true,
|
||||
merge: true
|
||||
});
|
||||
|
||||
console.log("-------------------------------------");
|
||||
console.log(result);
|
||||
|
||||
}());
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"name": "nodejs-commonjs-document-worker-export-import",
|
||||
"dependencies": {
|
||||
"flexsearch": "github:nextapps-de/flexsearch"
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user