Mercurial Hosting > luan

annotate host/admin/src/private/lib/monitor.luan @ 2023:544ccce564f6 default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
minor
author Franklin Schmidt <fschmidt@gmail.com>
date Mon, 20 Oct 2025 17:34:14 -0600
parents 969291201e12
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1995
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
1 local Luan = require "luan:Luan.luan"
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
2 local error = Luan.error
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
3 local Io = require "luan:Io.luan"
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
4 local uri = Io.uri or error()
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
5 local String = require "luan:String.luan"
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
6 local trim = String.trim or error()
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
7 local Time = require "luan:Time.luan"
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
8 local Thread = require "luan:Thread.luan"
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
9 local Http = require "luan:http/Http.luan"
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
10 local Config = require "site:/private/Config.luan"
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
11 local Utils = require "site:/private/lib/Utils.luan"
2022
969291201e12 ping lucene backups
Franklin Schmidt <fschmidt@gmail.com>
parents: 2021
diff changeset
12 local send_mail = Utils.send_mail or error()
1995
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
13 local Logging = require "luan:logging/Logging.luan"
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
14 local logger = Logging.logger "monitor"
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
15
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
16
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
17 local who_monitors_who = {
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
18 ["admin.s1.luan.software"] = "admin.s2.luan.software"
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
19 ["admin.s2.luan.software"] = "admin.s1.luan.software"
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
20 }
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
21 local frequency = Time.period{minutes=2}
2021
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
22 local url
1995
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
23
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
24 --[[ -- for development
2021
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
25 who_monitors_who["admin.me.luan.software"] = "admin.me.luan.software"
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
26 url = "http://admin.me.luan.software:8080/hi.txt"
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
27 frequency = Time.period{seconds=30}
1995
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
28 Config.email_to = "fschmidt@gmail.com"
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
29 --]]
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
30
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
31
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
32
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
33 local domain = who_monitors_who[Utils.domain]
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
34
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
35 if domain == nil then
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
36 logger.info "nothing to monitor"
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
37 return true
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
38 end
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
39
2021
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
40 url = url or "https://"..domain.."/hi.txt"
1995
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
41 local options = { time_out = Time.period{seconds=20} }
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
42
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
43 local function init_check()
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
44 local fails = 0
2021
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
45 local last_page
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
46
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
47 local function fail()
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
48 fails = fails + 1
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
49 if fails < 2 then return end
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
50 try
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
51 local s = Utils.ssh(domain,"/Users/administrator/luan/host/restart.sh monitoring")
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
52 if trim(s) == "stopped with stop script" then
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
53 logger.info("stopped with stop script")
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
54 else
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
55 logger.error("restart successful\n"..s)
2022
969291201e12 ping lucene backups
Franklin Schmidt <fschmidt@gmail.com>
parents: 2021
diff changeset
56 send_mail {
2021
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
57 Subject = domain.." restarted"
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
58 body = s
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
59 }
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
60 end
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
61 fails = 0
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
62 catch e
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
63 logger.error("restart failed: "..e.get_message())
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
64 if fails < 5 then
2022
969291201e12 ping lucene backups
Franklin Schmidt <fschmidt@gmail.com>
parents: 2021
diff changeset
65 send_mail {
2021
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
66 Subject = domain.." restart failed"
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
67 body = e.get_message()
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
68 }
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
69 end
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
70 end
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
71 end
1995
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
72
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
73 return function()
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
74 try
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
75 local page = uri(url,options).read_text()
2021
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
76 if page == last_page then
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
77 logger.error("Scheduler not working on "..domain)
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
78 fail()
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
79 return
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
80 end
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
81 last_page = page
1995
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
82 fails = 0
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
83 logger.info(domain.." is okay")
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
84 catch e
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
85 logger.error("Error connecting to "..domain..": "..e.get_message())
2021
b8e5d53c4fc9 admin monitors scheduler
Franklin Schmidt <fschmidt@gmail.com>
parents: 1995
diff changeset
86 fail()
1995
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
87 end
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
88 end
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
89 end
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
90
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
91 Thread.schedule_closure(init_check,{repeating_delay=frequency})
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
92
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
93 logger.info("monitoring "..domain)
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
94
301a6561fb6b add host/admin
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
95 return true