Page MenuHomePhabricator
Files F5007591

No OneTemporary
Actions

View Options

diff --git a/AutoCoverTool/online/monitor_worker.py b/AutoCoverTool/online/monitor_worker.py
index 7635d26..1fbd853 100644
--- a/AutoCoverTool/online/monitor_worker.py
+++ b/AutoCoverTool/online/monitor_worker.py
@@ -1,64 +1,64 @@
"""
监控程序
通过查看最近一个state设置成1的时间距离现在超过10分钟，则认为消费程序出错，需要报警
"""
import json
import requests
from online.common import *
def feishu_send(feishu_url, msg, users):
data = {
"mobiles": users,
"msg": msg
}
ret = requests.post(feishu_url, json.dumps(data))
def alarm(msg):
tm = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
msg = "tm:{}|msg:{}".format(tm, msg)
feishu_url = "http://sg-prod-songbook-webmp-1:8000/api/feishu/people"
users = ["18810833785"]
feishu_send(feishu_url, msg, users)
def monitor_check():
sql = "select update_time, song_id from av_db.svc_queue_table where state=1 order by update_time desc limit 1"
data = get_data_by_mysql(sql, banned_user_map)
if len(data) == 0:
return 1
up_time = int(data[0][0])
cur_time = int(time.time())
- # 发现被取走的数据距离当前时间超过5分钟
- if cur_time - up_time > 5 * 60:
+ # 发现被取走的数据距离当前时间超过15分钟
+ if cur_time - up_time > 15 * 60:
return 2
return 0
def monitor_process():
err_msg = {
1: "异常！变调队列中没有正常被处理的数据请及时查看！",
2: "异常！变调队列中长时间没有任务被消费请及时查看！"
}
while True:
print("\n\n-----------start------------------->>>>>")
time.sleep(5)
err = monitor_check()
if err != 0:
for i in range(0, 3):
time.sleep(5)
monitor_check()
err = monitor_check()
if err != 0:
alarm(err_msg[err])
print("-------alarm--err={}-------------->>>>>".format(err))
# 报警后等待冷却5分钟
time.sleep(5 * 60)
print("-------success---------------->>>>>")
if __name__ == '__main__':
monitor_process()

File Metadata

Mime Type
text/x-diff
Expires
Tue, Apr 29, 01:29 (1 d, 2 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1426879
Default Alt Text
(2 KB)

Event Timeline