|
1
|
1 local Luan = require "luan:Luan.luan"
|
|
|
2 local error = Luan.error
|
|
|
3 local ipairs = Luan.ipairs or error()
|
|
|
4 local type = Luan.type or error()
|
|
|
5 local stringify = Luan.stringify or error()
|
|
|
6 local Io = require "luan:Io.luan"
|
|
|
7 local uri = Io.uri or error()
|
|
|
8 local print = Io.print or error()
|
|
|
9 local String = require "luan:String.luan"
|
|
|
10 local starts_with = String.starts_with or error()
|
|
|
11 local Html = require "luan:Html.luan"
|
|
|
12 local html_parse = Html.parse or error()
|
|
|
13 local Table = require "luan:Table.luan"
|
|
|
14 local copy = Table.copy or error()
|
|
|
15
|
|
|
16 function Io.schemes.site(path)
|
|
|
17 return uri( "file:../src"..path )
|
|
|
18 end
|
|
|
19
|
|
|
20 local Db = require "site:/lib/Db.luan"
|
|
|
21
|
|
|
22 Db.delete_all()
|
|
|
23
|
|
|
24
|
|
|
25 local function get_user_htmls(html)
|
|
|
26 local rtn = {}
|
|
|
27 local n = #html
|
|
|
28 local i = 1
|
|
|
29 local el
|
|
|
30 while true do
|
|
|
31 while true do
|
|
|
32 if i > n then
|
|
|
33 return rtn
|
|
|
34 end
|
|
|
35 el = html[i]
|
|
|
36 if type(el)=="table" and el.type=="tag" and el.name=="div" and el.attributes.userid~=nil then
|
|
|
37 break
|
|
|
38 end
|
|
|
39 i = i + 1
|
|
|
40 end
|
|
|
41 local start = i
|
|
|
42 local nesting = 0
|
|
|
43 repeat
|
|
|
44 if type(el)=="table" and el.type=="tag" then
|
|
|
45 if el.name=="div" then
|
|
|
46 nesting = nesting + 1
|
|
|
47 elseif el.name=="/div" then
|
|
|
48 nesting = nesting - 1
|
|
|
49 end
|
|
|
50 end
|
|
|
51 i = i + 1
|
|
|
52 el = html[i]
|
|
|
53 until nesting==0
|
|
|
54 rtn[#rtn+1] = copy(html,start,i-1)
|
|
|
55 end
|
|
|
56 end
|
|
|
57
|
|
|
58 local function process_user_html(html)
|
|
|
59 --print(stringify(html[1]))
|
|
|
60 local user_id = html[1].attributes.userid or error()
|
|
|
61 if Db.count("user_id:"..user_id) > 0 then
|
|
|
62 return
|
|
|
63 end
|
|
|
64 local doc = {}
|
|
|
65 doc.user_id = user_id
|
|
|
66 for i, el in ipairs(html) do
|
|
|
67 if not (type(el)=="table" and el.type=="tag") then
|
|
|
68 continue
|
|
|
69 end
|
|
|
70 local name = el.name
|
|
|
71 local attributes = el.attributes
|
|
|
72 local class = attributes.class
|
|
|
73 if name=="span" and class=="organization" then
|
|
|
74 local user_name = html[i+1]
|
|
|
75 if type(user_name)=="table" and user_name.type=="tag" and user_name.name=="/span" then
|
|
|
76 user_name = ""
|
|
|
77 end
|
|
|
78 doc.user_name = user_name
|
|
|
79 elseif name=="a" and class~=nil and starts_with(class,"afftag ") then
|
|
|
80 doc.category = doc.category or {}
|
|
|
81 doc.category[#doc.category+1] = html[i+1]
|
|
|
82 elseif name=="section" and class=="description" then
|
|
|
83 doc.description = html[i+1]
|
|
|
84 elseif name=="a" and class=="aff-website" then
|
|
|
85 doc.websites = doc.websites or {}
|
|
|
86 doc.websites[#doc.websites+1] = attributes.href
|
|
|
87 end
|
|
|
88 end
|
|
|
89 Db.save(doc)
|
|
|
90 end
|
|
|
91
|
|
|
92 local pages_dir = uri "file:pages"
|
|
|
93
|
|
|
94 for _, dir in ipairs(pages_dir.children()) do
|
|
|
95 if starts_with( dir.name(), "." ) then
|
|
|
96 continue
|
|
|
97 end
|
|
|
98 --print(dir.name())
|
|
|
99 for _, file in ipairs(dir.children()) do
|
|
|
100 local text = file.read_text()
|
|
|
101 local html = html_parse(text)
|
|
|
102 local user_htmls = get_user_htmls(html)
|
|
|
103 print(file.name().." "..#user_htmls)
|
|
|
104 for _, user_html in ipairs(user_htmls) do
|
|
|
105 process_user_html(user_html)
|
|
|
106 end
|
|
|
107 --break
|
|
|
108 end
|
|
|
109 end
|
