diff --git a/AutoCoverTool/online/monitor_worker.py b/AutoCoverTool/online/monitor_worker.py index 7635d26..1fbd853 100644 --- a/AutoCoverTool/online/monitor_worker.py +++ b/AutoCoverTool/online/monitor_worker.py @@ -1,64 +1,64 @@ """ 监控程序 通过查看最近一个state设置成1的时间距离现在超过10分钟,则认为消费程序出错,需要报警 """ import json import requests from online.common import * def feishu_send(feishu_url, msg, users): data = { "mobiles": users, "msg": msg } ret = requests.post(feishu_url, json.dumps(data)) def alarm(msg): tm = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) msg = "tm:{}|msg:{}".format(tm, msg) feishu_url = "http://sg-prod-songbook-webmp-1:8000/api/feishu/people" users = ["18810833785"] feishu_send(feishu_url, msg, users) def monitor_check(): sql = "select update_time, song_id from av_db.svc_queue_table where state=1 order by update_time desc limit 1" data = get_data_by_mysql(sql, banned_user_map) if len(data) == 0: return 1 up_time = int(data[0][0]) cur_time = int(time.time()) - # 发现被取走的数据距离当前时间超过5分钟 - if cur_time - up_time > 5 * 60: + # 发现被取走的数据距离当前时间超过15分钟 + if cur_time - up_time > 15 * 60: return 2 return 0 def monitor_process(): err_msg = { 1: "异常!变调队列中没有正常被处理的数据请及时查看!", 2: "异常!变调队列中长时间没有任务被消费请及时查看!" } while True: print("\n\n-----------start------------------->>>>>") time.sleep(5) err = monitor_check() if err != 0: for i in range(0, 3): time.sleep(5) monitor_check() err = monitor_check() if err != 0: alarm(err_msg[err]) print("-------alarm--err={}-------------->>>>>".format(err)) # 报警后等待冷却5分钟 time.sleep(5 * 60) print("-------success---------------->>>>>") if __name__ == '__main__': monitor_process()