Mercurial Hosting > luan

annotate src/luan/modules/lucene/Lucene.luan @ 1802:ca98dee04e08 default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
add Parsers.json_null
author Franklin Schmidt <fschmidt@gmail.com>
date Sun, 21 Apr 2024 21:25:15 -0600
parents c62324841dfb
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1353
8d95711f6615 replace java() with require "java"
Franklin Schmidt <fschmidt@gmail.com>
parents: 1348
diff changeset
1 require "java"
693
ca169567ce07 module URIs must now include ".luan"
Franklin Schmidt <fschmidt@gmail.com>
parents: 625
diff changeset
2 local Luan = require "luan:Luan.luan"
503
92c3d22745b8 make _ENV optional
Franklin Schmidt <fschmidt@gmail.com>
parents: 435
diff changeset
3 local error = Luan.error
625
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
4 local ipairs = Luan.ipairs or error()
1429
82415c9c0015 move versioning into Lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1427
diff changeset
5 local pairs = Luan.pairs or error()
625
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
6 local type = Luan.type or error()
796
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
7 local set_metatable = Luan.set_metatable or error()
1578
c922446f53aa immutable threading
Franklin Schmidt <fschmidt@gmail.com>
parents: 1567
diff changeset
8 local get_local_cloned = Luan.get_local_cloned
c922446f53aa immutable threading
Franklin Schmidt <fschmidt@gmail.com>
parents: 1567
diff changeset
9 local set_local_cloned = Luan.set_local_cloned
1763
164c6ea53147 fix change_password
Franklin Schmidt <fschmidt@gmail.com>
parents: 1749
diff changeset
10 local pairs_local_cloned = Luan.pairs_local_cloned
1771
c62324841dfb improve NumberFieldParser and add key map lucene query
Franklin Schmidt <fschmidt@gmail.com>
parents: 1763
diff changeset
11 local stringify = Luan.stringify or error()
1365
6617763dfd76 fix logging and lucene backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1353
diff changeset
12 local Boot = require "luan:Boot.luan"
693
ca169567ce07 module URIs must now include ".luan"
Franklin Schmidt <fschmidt@gmail.com>
parents: 625
diff changeset
13 local Html = require "luan:Html.luan"
1429
82415c9c0015 move versioning into Lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1427
diff changeset
14 local Number = require "luan:Number.luan"
82415c9c0015 move versioning into Lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1427
diff changeset
15 local integer = Number.integer or error()
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
16 local Time = require "luan:Time.luan"
707
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 693
diff changeset
17 local Io = require "luan:Io.luan"
746
293c397e8dee improve zip
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
18 local uri = Io.uri or error()
707
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 693
diff changeset
19 local String = require "luan:String.luan"
1716
b82767112d8e add String.regex
Franklin Schmidt <fschmidt@gmail.com>
parents: 1696
diff changeset
20 local starts_with = String.starts_with or error()
749
85f5444fb7d4 add rcp lucene backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 748
diff changeset
21 local Rpc = require "luan:Rpc.luan"
321
7f7708e8fdd4 remove import statement
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 320
diff changeset
22 local LuceneIndex = require "java:luan.modules.lucene.LuceneIndex"
1458
6b6c11c9164e goodjava.lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1457
diff changeset
23 local NumberFieldParser = require "java:goodjava.lucene.queryparser.NumberFieldParser"
1459
b04b8fc5f4f4 GoodQueryParser
Franklin Schmidt <fschmidt@gmail.com>
parents: 1458
diff changeset
24 local GoodQueryParser = require "java:goodjava.lucene.queryparser.GoodQueryParser"
1368
5225cd6ed478 add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1367
diff changeset
25 local Logging = require "luan:logging/Logging.luan"
5225cd6ed478 add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1367
diff changeset
26 local logger = Logging.logger "Lucene"
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 542
diff changeset
27
320
fed1893821bf remove global namespace
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 312
diff changeset
28
1088
bae2d0c2576c change module naming convention
Franklin Schmidt <fschmidt@gmail.com>
parents: 796
diff changeset
29 local Lucene = {}
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
30
1567
349eef23a13c lucene named backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1565
diff changeset
31 local indexes = {}
1578
c922446f53aa immutable threading
Franklin Schmidt <fschmidt@gmail.com>
parents: 1567
diff changeset
32 local indexes_mt = {
c922446f53aa immutable threading
Franklin Schmidt <fschmidt@gmail.com>
parents: 1567
diff changeset
33 __index = get_local_cloned
c922446f53aa immutable threading
Franklin Schmidt <fschmidt@gmail.com>
parents: 1567
diff changeset
34 __new_index = set_local_cloned
1763
164c6ea53147 fix change_password
Franklin Schmidt <fschmidt@gmail.com>
parents: 1749
diff changeset
35 __pairs = pairs_local_cloned
1578
c922446f53aa immutable threading
Franklin Schmidt <fschmidt@gmail.com>
parents: 1567
diff changeset
36 }
c922446f53aa immutable threading
Franklin Schmidt <fschmidt@gmail.com>
parents: 1567
diff changeset
37 set_metatable(indexes,indexes_mt)
1567
349eef23a13c lucene named backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1565
diff changeset
38
349eef23a13c lucene named backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1565
diff changeset
39 function Rpc.functions.lucene_backup(password,name)
349eef23a13c lucene named backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1565
diff changeset
40 Io.password == password or error "wrong password"
349eef23a13c lucene named backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1565
diff changeset
41 local index = indexes[name] or error("index not found: "..name)
349eef23a13c lucene named backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1565
diff changeset
42 local zip_file = index.dir.parent().child("backup.zip")
349eef23a13c lucene named backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1565
diff changeset
43 index.zip(zip_file)
349eef23a13c lucene named backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1565
diff changeset
44 return zip_file
349eef23a13c lucene named backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1565
diff changeset
45 end
349eef23a13c lucene named backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1565
diff changeset
46
349eef23a13c lucene named backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1565
diff changeset
47 function Rpc.functions.lucene_restore(password,name,zip_file)
349eef23a13c lucene named backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1565
diff changeset
48 Io.password == password or error "wrong password"
349eef23a13c lucene named backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1565
diff changeset
49 local index = indexes[name] or error "index not found"
349eef23a13c lucene named backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1565
diff changeset
50 local backup_zip = index.dir.parent().child("backup.zip")
349eef23a13c lucene named backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1565
diff changeset
51 backup_zip.write(zip_file)
349eef23a13c lucene named backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1565
diff changeset
52 index.restore(backup_zip)
349eef23a13c lucene named backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1565
diff changeset
53 end
349eef23a13c lucene named backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1565
diff changeset
54
349eef23a13c lucene named backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1565
diff changeset
55
1088
bae2d0c2576c change module naming convention
Franklin Schmidt <fschmidt@gmail.com>
parents: 796
diff changeset
56 Lucene.type = {
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
57 english = LuceneIndex.ENGLISH_FIELD_PARSER
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
58 string = LuceneIndex.STRING_FIELD_PARSER
1532
060ff7695317 Lucene.type.lowercase
Franklin Schmidt <fschmidt@gmail.com>
parents: 1531
diff changeset
59 lowercase = LuceneIndex.LOWERCASE_FIELD_PARSER
1749
d1e7564a9ce5 improve query parser
Franklin Schmidt <fschmidt@gmail.com>
parents: 1745
diff changeset
60 simple = LuceneIndex.SIMPLE_FIELD_PARSER
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
61 integer = NumberFieldParser.INT
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
62 long = NumberFieldParser.LONG
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
63 double = NumberFieldParser.DOUBLE
1745
f8950d99f47d add Lucene.type.float
Franklin Schmidt <fschmidt@gmail.com>
parents: 1716
diff changeset
64 float = NumberFieldParser.FLOAT
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 542
diff changeset
65 }
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 542
diff changeset
66
1622
Franklin Schmidt <fschmidt@gmail.com>
parents: 1619
diff changeset
67 Lucene.quote = GoodQueryParser.quote
Franklin Schmidt <fschmidt@gmail.com>
parents: 1619
diff changeset
68 Lucene.escape = GoodQueryParser.escape
547
0be287ab0309 add lucene/Versioning and simplify Lucene fn names
Franklin Schmidt <fschmidt@gmail.com>
parents: 546
diff changeset
69
1685
46cf5137cb6b misc fixes
Franklin Schmidt <fschmidt@gmail.com>
parents: 1672
diff changeset
70 local function get_file(f)
46cf5137cb6b misc fixes
Franklin Schmidt <fschmidt@gmail.com>
parents: 1672
diff changeset
71 type(f)=="table" or error "index_dir must be table"
1716
b82767112d8e add String.regex
Franklin Schmidt <fschmidt@gmail.com>
parents: 1696
diff changeset
72 f.to_uri_string and starts_with(f.to_uri_string(),"file:") or error "index_dir must be file"
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
73 return f.java.file or error()
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
74 end
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
75
1395
9dfff82dfc59 finish postgres work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1392
diff changeset
76 function Lucene.index(index_dir,options)
1685
46cf5137cb6b misc fixes
Franklin Schmidt <fschmidt@gmail.com>
parents: 1672
diff changeset
77 options = options or {}
303
fdb4bd391c28 add lucene close();
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 300
diff changeset
78 local index = {}
1578
c922446f53aa immutable threading
Franklin Schmidt <fschmidt@gmail.com>
parents: 1567
diff changeset
79 local options_name = options.name
591
790d5de23042 add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents: 547
diff changeset
80 index.dir = index_dir
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
81 index_dir = get_file(index_dir)
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
82 options = options or {}
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
83 options.log_dir = options.log_dir and get_file(options.log_dir)
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
84 options.log_time = options.log_time or Time.period{days=30}
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
85 local java_index = LuceneIndex.getLuceneIndex(index_dir,options)
1129
3234a14bb1f8 minor lucene changes
Franklin Schmidt <fschmidt@gmail.com>
parents: 1088
diff changeset
86 index.java = java_index
796
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
87
1578
c922446f53aa immutable threading
Franklin Schmidt <fschmidt@gmail.com>
parents: 1567
diff changeset
88 if options_name ~= nil then
c922446f53aa immutable threading
Franklin Schmidt <fschmidt@gmail.com>
parents: 1567
diff changeset
89 indexes[options_name] = index
c922446f53aa immutable threading
Franklin Schmidt <fschmidt@gmail.com>
parents: 1567
diff changeset
90 end
c922446f53aa immutable threading
Franklin Schmidt <fschmidt@gmail.com>
parents: 1567
diff changeset
91
796
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
92 index.indexed_fields = {}
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
93 local mt = {}
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
94 set_metatable(index.indexed_fields,mt)
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
95 function mt.__index(_,key)
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
96 return java_index.getIndexedFieldParser(key)
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
97 end
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
98 function mt.__new_index(_,key,value)
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
99 return java_index.setIndexedFieldParser(key,value)
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
100 end
618
5e495e4e560b add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents: 617
diff changeset
101
303
fdb4bd391c28 add lucene close();
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 300
diff changeset
102 index.to_string = java_index.to_string
707
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 693
diff changeset
103 index.snapshot = java_index.snapshot
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
104 index.advanced_search = java_index.advanced_search
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
105 index.search_in_transaction = java_index.search_in_transaction
1534
Franklin Schmidt <fschmidt@gmail.com>
parents: 1533
diff changeset
106 index.delete_all = java_index.delete_all
Franklin Schmidt <fschmidt@gmail.com>
parents: 1533
diff changeset
107 index.delete = java_index.delete
1535
e73b72a510b4 Lucene reindex
Franklin Schmidt <fschmidt@gmail.com>
parents: 1534
diff changeset
108 index.reindex = java_index.reindex
1533
28f06ac6ce39 add not_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1532
diff changeset
109 --index.save = java_index.save
28f06ac6ce39 add not_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1532
diff changeset
110 index.is_in_transaction = java_index.is_in_transaction
1379
87a3738d7cc5 run_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1377
diff changeset
111 index.run_in_transaction = java_index.run_in_transaction
591
790d5de23042 add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents: 547
diff changeset
112 index.ensure_open = java_index.ensure_open
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
113 index.highlighter = java_index.highlighter
1341
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1330
diff changeset
114 index.count_tokens = java_index.count_tokens
1544
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1541
diff changeset
115 index.tag = java_index.tag
709
96a280ca32a2 add Lucene.instances
Franklin Schmidt <fschmidt@gmail.com>
parents: 707
diff changeset
116
1556
52241b69c339 lucene logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
117 index.relog = java_index.relog
1544
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1541
diff changeset
118 index.restore_from_log = java_index.restore_from_log
1556
52241b69c339 lucene logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
119 index.force_restore_from_log = java_index.force_restore_from_log
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
120
1453
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1431
diff changeset
121 index.check = java_index.check
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
122
1749
d1e7564a9ce5 improve query parser
Franklin Schmidt <fschmidt@gmail.com>
parents: 1745
diff changeset
123 index.explain_query = java_index.explain_query
d1e7564a9ce5 improve query parser
Franklin Schmidt <fschmidt@gmail.com>
parents: 1745
diff changeset
124
1533
28f06ac6ce39 add not_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1532
diff changeset
125 function index.not_in_transaction() end
28f06ac6ce39 add not_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1532
diff changeset
126
28f06ac6ce39 add not_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1532
diff changeset
127 function index.check_in_transaction()
28f06ac6ce39 add not_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1532
diff changeset
128 if not index.is_in_transaction() then
28f06ac6ce39 add not_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1532
diff changeset
129 index.not_in_transaction()
28f06ac6ce39 add not_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1532
diff changeset
130 end
28f06ac6ce39 add not_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1532
diff changeset
131 end
28f06ac6ce39 add not_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1532
diff changeset
132
1687
f48db13ae2d9 unlogged lucene support
Franklin Schmidt <fschmidt@gmail.com>
parents: 1685
diff changeset
133 function index.save(doc,unstored,boosts)
1533
28f06ac6ce39 add not_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1532
diff changeset
134 index.check_in_transaction()
1687
f48db13ae2d9 unlogged lucene support
Franklin Schmidt <fschmidt@gmail.com>
parents: 1685
diff changeset
135 java_index.save(doc,unstored,boosts)
1533
28f06ac6ce39 add not_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1532
diff changeset
136 end
28f06ac6ce39 add not_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1532
diff changeset
137
1342
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
138 function index.search( query, from, to, options )
770
Franklin Schmidt <fschmidt@gmail.com>
parents: 761
diff changeset
139 from or error "missing 'from' parameter"
Franklin Schmidt <fschmidt@gmail.com>
parents: 761
diff changeset
140 to or error "missing 'to' parameter"
1342
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
141 options = options or {}
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
142 local explain_fld = options.explain
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
143 local score_fld = options.score
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
144 local results = {}
1342
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
145 local function fn(i,doc_fn,score)
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
146 if i >= from then
1342
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
147 local doc
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
148 if explain_fld == nil then
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
149 doc = doc_fn()
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
150 else
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
151 local explanation
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
152 doc, explanation = doc_fn("explain")
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
153 doc[explain_fld] = explanation.toString()
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
154 end
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
155 if score_fld ~= nil then
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
156 doc[score_fld] = score
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
157 end
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
158 results[#results+1] = doc
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
159 end
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
160 end
1342
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
161 local total_hits = index.advanced_search(query,fn,to,options.sort)
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
162 return results, total_hits
257
c5c60eca33dd allow Lucene search for 0 rows
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
163 end
c5c60eca33dd allow Lucene search for 0 rows
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
164
c5c60eca33dd allow Lucene search for 0 rows
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
165 function index.get_document(query)
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
166 local doc
1348
af478a1bd23d bug fix
Franklin Schmidt <fschmidt@gmail.com>
parents: 1345
diff changeset
167 local function fn(_,doc_fn,_)
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
168 doc = doc_fn()
257
c5c60eca33dd allow Lucene search for 0 rows
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
169 end
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
170 local total_hits = index.advanced_search(query,fn,1)
1771
c62324841dfb improve NumberFieldParser and add key map lucene query
Franklin Schmidt <fschmidt@gmail.com>
parents: 1763
diff changeset
171 total_hits <= 1 or error("found "..total_hits.." documents for query: "..stringify(query))
257
c5c60eca33dd allow Lucene search for 0 rows
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
172 return doc
c5c60eca33dd allow Lucene search for 0 rows
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
173 end
c5c60eca33dd allow Lucene search for 0 rows
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
174
c5c60eca33dd allow Lucene search for 0 rows
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
175 function index.count(query)
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
176 return index.advanced_search(query)
232
9ce18106f95a more lucene work
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 230
diff changeset
177 end
9ce18106f95a more lucene work
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 230
diff changeset
178
625
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
179 function index.html_highlighter(query,formatter,container_tags)
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
180 local highlighter = index.highlighter(query,formatter)
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
181 return function(html)
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
182 local list = Html.parse(html,container_tags)
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
183 local result = {}
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
184 for _, obj in ipairs(list) do
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
185 if type(obj) == "string" then
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
186 obj = highlighter(obj)
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
187 end
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
188 result[#result+1] = obj
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
189 end
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
190 return Html.to_string(result)
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
191 end
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
192 end
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
193
707
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 693
diff changeset
194 function index.zip(zip_file)
761
99356cfde2f0 remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents: 757
diff changeset
195 index.snapshot( function(dir_path,file_names)
99356cfde2f0 remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents: 757
diff changeset
196 zip_file.delete()
99356cfde2f0 remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents: 757
diff changeset
197 local zip_path = zip_file.canonical().to_string()
99356cfde2f0 remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents: 757
diff changeset
198 local dir = uri("file:"..dir_path)
99356cfde2f0 remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents: 757
diff changeset
199 local dir_name = dir.name()
99356cfde2f0 remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents: 757
diff changeset
200 local options = {dir=dir.parent()}
707
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 693
diff changeset
201 for _, file_name in ipairs(file_names) do
761
99356cfde2f0 remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents: 757
diff changeset
202 local cmd = "zip "..zip_path.." "..dir_name.."/"..file_name
1431
Franklin Schmidt <fschmidt@gmail.com>
parents: 1429
diff changeset
203 uri("os:"..cmd,options).read_text()
707
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 693
diff changeset
204 end
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 693
diff changeset
205 end )
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 693
diff changeset
206 end
1365
6617763dfd76 fix logging and lucene backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1353
diff changeset
207 index.zip = Boot.no_security(index.zip)
707
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 693
diff changeset
208
754
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 753
diff changeset
209 function index.restore(zip_file)
756
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
210 java_index.run_in_lock( function()
1369
709f7498a363 change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1368
diff changeset
211 local lucene_dir = index.dir
756
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
212 local before_restore = lucene_dir.parent().child("before_restore.zip")
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
213 index.zip(before_restore)
1398
67c0e47b5be3 more lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1397
diff changeset
214 java_index.close()
756
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
215 lucene_dir.delete()
1431
Franklin Schmidt <fschmidt@gmail.com>
parents: 1429
diff changeset
216 uri("os:unzip "..zip_file.canonical().to_string(),{dir=lucene_dir.parent()}).read_text()
756
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
217 java_index.reopen()
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
218 end )
754
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 753
diff changeset
219 end
1365
6617763dfd76 fix logging and lucene backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1353
diff changeset
220 index.restore = Boot.no_security(index.restore)
754
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 753
diff changeset
221
1429
82415c9c0015 move versioning into Lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1427
diff changeset
222 function index.update(steps)
82415c9c0015 move versioning into Lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1427
diff changeset
223 local version = 0
82415c9c0015 move versioning into Lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1427
diff changeset
224 for i in pairs(steps) do
82415c9c0015 move versioning into Lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1427
diff changeset
225 if version < i then
82415c9c0015 move versioning into Lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1427
diff changeset
226 version = i
82415c9c0015 move versioning into Lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1427
diff changeset
227 end
82415c9c0015 move versioning into Lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1427
diff changeset
228 end
82415c9c0015 move versioning into Lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1427
diff changeset
229 local doc = index.get_document"type:version" or { type="version", version=integer(0) }
82415c9c0015 move versioning into Lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1427
diff changeset
230 while doc.version < version do
82415c9c0015 move versioning into Lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1427
diff changeset
231 doc.version = integer(doc.version + 1)
82415c9c0015 move versioning into Lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1427
diff changeset
232 logger.error("step "..doc.version)
82415c9c0015 move versioning into Lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1427
diff changeset
233 index.run_in_transaction( function()
82415c9c0015 move versioning into Lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1427
diff changeset
234 local step = steps[doc.version] or error("missing step "..doc.version)
82415c9c0015 move versioning into Lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1427
diff changeset
235 step()
82415c9c0015 move versioning into Lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1427
diff changeset
236 index.save(doc)
82415c9c0015 move versioning into Lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1427
diff changeset
237 end )
82415c9c0015 move versioning into Lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1427
diff changeset
238 end
82415c9c0015 move versioning into Lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1427
diff changeset
239 end
82415c9c0015 move versioning into Lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1427
diff changeset
240
1763
164c6ea53147 fix change_password
Franklin Schmidt <fschmidt@gmail.com>
parents: 1749
diff changeset
241 index.close_down = java_index.close_down
164c6ea53147 fix change_password
Franklin Schmidt <fschmidt@gmail.com>
parents: 1749
diff changeset
242
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
243 return index
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
244 end
503
92c3d22745b8 make _ENV optional
Franklin Schmidt <fschmidt@gmail.com>
parents: 435
diff changeset
245
1763
164c6ea53147 fix change_password
Franklin Schmidt <fschmidt@gmail.com>
parents: 1749
diff changeset
246 function Rpc.functions.close_lucene(password)
164c6ea53147 fix change_password
Franklin Schmidt <fschmidt@gmail.com>
parents: 1749
diff changeset
247 Io.password == password or error "wrong password"
164c6ea53147 fix change_password
Franklin Schmidt <fschmidt@gmail.com>
parents: 1749
diff changeset
248 logger.info("close_all")
164c6ea53147 fix change_password
Franklin Schmidt <fschmidt@gmail.com>
parents: 1749
diff changeset
249 for name, index in pairs(indexes) do
164c6ea53147 fix change_password
Franklin Schmidt <fschmidt@gmail.com>
parents: 1749
diff changeset
250 logger.info("closing "..name)
164c6ea53147 fix change_password
Franklin Schmidt <fschmidt@gmail.com>
parents: 1749
diff changeset
251 index.close_down()
164c6ea53147 fix change_password
Franklin Schmidt <fschmidt@gmail.com>
parents: 1749
diff changeset
252 end
164c6ea53147 fix change_password
Franklin Schmidt <fschmidt@gmail.com>
parents: 1749
diff changeset
253 end
164c6ea53147 fix change_password
Franklin Schmidt <fschmidt@gmail.com>
parents: 1749
diff changeset
254
1088
bae2d0c2576c change module naming convention
Franklin Schmidt <fschmidt@gmail.com>
parents: 796
diff changeset
255 return Lucene