forked from GithubBackups/healthchecks
112 lines
3.3 KiB
Python
112 lines
3.3 KiB
Python
import time
|
|
from threading import Thread
|
|
|
|
from django.core.management.base import BaseCommand
|
|
from django.utils import timezone
|
|
from hc.api.models import Check
|
|
|
|
|
|
def notify(check_id, stdout):
|
|
check = Check.objects.get(id=check_id)
|
|
tmpl = "Sending alert, status=%s, code=%s\n"
|
|
stdout.write(tmpl % (check.status, check.code))
|
|
|
|
# Set dates for followup nags
|
|
if check.status == "down" and check.user.profile:
|
|
check.user.profile.set_next_nag_date()
|
|
|
|
# Send notifications
|
|
errors = check.send_alert()
|
|
for ch, error in errors:
|
|
stdout.write("ERROR: %s %s %s\n" % (ch.kind, ch.value, error))
|
|
|
|
|
|
def notify_on_thread(check_id, stdout):
|
|
t = Thread(target=notify, args=(check_id, stdout))
|
|
t.start()
|
|
|
|
|
|
class Command(BaseCommand):
|
|
help = 'Sends UP/DOWN email alerts'
|
|
owned = Check.objects.filter(user__isnull=False).order_by("alert_after")
|
|
|
|
def add_arguments(self, parser):
|
|
parser.add_argument(
|
|
'--no-loop',
|
|
action='store_false',
|
|
dest='loop',
|
|
default=True,
|
|
help='Do not keep running indefinitely in a 2 second wait loop',
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--no-threads',
|
|
action='store_false',
|
|
dest='use_threads',
|
|
default=False,
|
|
help='Send alerts synchronously, without using threads',
|
|
)
|
|
|
|
def handle_one(self, use_threads=True):
|
|
""" Process a single check. """
|
|
|
|
now = timezone.now()
|
|
|
|
# Look for checks that are going down
|
|
q = self.owned.filter(alert_after__lt=now, status="up")
|
|
check = q.first()
|
|
|
|
# If none found, look for checks that are going up
|
|
if not check:
|
|
q = self.owned.filter(alert_after__gt=now, status="down")
|
|
check = q.first()
|
|
|
|
if check is None:
|
|
return False
|
|
|
|
q = Check.objects.filter(id=check.id, status=check.status)
|
|
current_status = check.get_status()
|
|
|
|
# During the grace period sendalerts considers the check as "up":
|
|
if current_status == "grace":
|
|
current_status = "up"
|
|
|
|
if check.status == current_status:
|
|
# Stored status is already up-to-date. Update alert_after
|
|
# as needed but don't send notifications
|
|
q.update(alert_after=check.get_alert_after())
|
|
return True
|
|
else:
|
|
# Atomically update status to the opposite
|
|
num_updated = q.update(status=current_status)
|
|
if num_updated == 1:
|
|
# Send notifications only if status update succeeded
|
|
# (no other sendalerts process got there first)
|
|
if use_threads:
|
|
notify_on_thread(check.id, self.stdout)
|
|
else:
|
|
notify(check.id, self.stdout)
|
|
|
|
return True
|
|
|
|
return False
|
|
|
|
def handle(self, use_threads=True, loop=True, *args, **options):
|
|
self.stdout.write("sendalerts is now running\n")
|
|
|
|
i, sent = 0, 0
|
|
while True:
|
|
while self.handle_one(use_threads):
|
|
sent += 1
|
|
|
|
if not loop:
|
|
break
|
|
|
|
time.sleep(2)
|
|
i += 1
|
|
if i % 60 == 0:
|
|
timestamp = timezone.now().isoformat()
|
|
self.stdout.write("-- MARK %s --\n" % timestamp)
|
|
|
|
return "Sent %d alert(s)" % sent
|