#!/usr/bin/python
|
#
|
# Copyright 2010 Google Inc. All Rights Reserved.
|
|
"""Tool to check the data consistency between master autotest db and replica.
|
|
This tool will issue 'show master status;' and 'show slave status;' commands to
|
two replicated databases to compare its log position.
|
|
It will also take a delta command line argument to allow certain time delay
|
between master and slave. If the delta of two log positions falls into the
|
defined range, it will be treated as synced.
|
|
It will send out an email notification upon any problem if specified an --to
|
argument.
|
"""
|
|
import getpass
|
import MySQLdb
|
import optparse
|
import os
|
import socket
|
import sys
|
|
import common
|
from autotest_lib.client.common_lib import global_config
|
|
|
c = global_config.global_config
|
_section = 'AUTOTEST_WEB'
|
DATABASE_HOST = c.get_config_value(_section, "host")
|
REPLICA_DATABASE_HOST = c.get_config_value(_section, "readonly_host")
|
DATABASE_NAME = c.get_config_value(_section, "database")
|
DATABASE_USER = c.get_config_value(_section, "user")
|
DATABASE_PASSWORD = c.get_config_value(_section, "password")
|
SYSTEM_USER = 'chromeos-test'
|
|
|
def ParseOptions():
|
parser = optparse.OptionParser()
|
parser.add_option('-d', '--delta', help='Difference between master and '
|
'replica db', type='int', dest='delta', default=0)
|
parser.add_option('--to', help='Comma separated Email notification TO '
|
'recipients.', dest='to', type='string', default='')
|
parser.add_option('--cc', help='Comma separated Email notification CC '
|
'recipients.', dest='cc', type='string', default='')
|
parser.add_option('-t', '--test-mode', help='skip common group email',
|
dest='testmode', action='store_true', default=False)
|
options, _ = parser.parse_args()
|
return options
|
|
|
def FetchMasterResult():
|
master_conn = MySQLdb.connect(host=DATABASE_HOST,
|
user=DATABASE_USER,
|
passwd=DATABASE_PASSWORD,
|
db=DATABASE_NAME )
|
cursor = master_conn.cursor(MySQLdb.cursors.DictCursor)
|
cursor.execute ("show master status;")
|
master_result = cursor.fetchone()
|
master_conn.close()
|
return master_result
|
|
|
def FetchSlaveResult():
|
replica_conn = MySQLdb.connect(host=REPLICA_DATABASE_HOST,
|
user=DATABASE_USER,
|
passwd=DATABASE_PASSWORD,
|
db=DATABASE_NAME )
|
cursor = replica_conn.cursor(MySQLdb.cursors.DictCursor)
|
cursor.execute ("show slave status;")
|
slave_result = cursor.fetchone()
|
replica_conn.close()
|
return slave_result
|
|
|
def RunChecks(options, master_result, slave_result):
|
master_pos = master_result['Position']
|
slave_pos = slave_result['Read_Master_Log_Pos']
|
if (master_pos - slave_pos) > options.delta:
|
return 'DELTA EXCEEDED: master=%s, slave=%s' % (master_pos, slave_pos)
|
if slave_result['Last_SQL_Error'] != '':
|
return 'SLAVE Last_SQL_Error'
|
if slave_result['Slave_IO_State'] != 'Waiting for master to send event':
|
return 'SLAVE Slave_IO_State'
|
if slave_result['Last_IO_Error'] != '':
|
return 'SLAVE Last_IO_Error'
|
if slave_result['Slave_SQL_Running'] != 'Yes':
|
return 'SLAVE Slave_SQL_Running'
|
if slave_result['Slave_IO_Running'] != 'Yes':
|
return 'SLAVE Slave_IO_Running'
|
return None
|
|
|
def ShowStatus(options, master_result, slave_result, msg):
|
summary = 'Master (%s) and slave (%s) databases are out of sync.' % (
|
DATABASE_HOST, REPLICA_DATABASE_HOST) + msg
|
if not options.to:
|
print summary
|
print 'Master status:'
|
print str(master_result)
|
print 'Slave status:'
|
print str(slave_result)
|
else:
|
email_to = ['%s@google.com' % to.strip() for to in options.to.split(',')]
|
email_cc = []
|
if options.cc:
|
email_cc.extend(
|
'%s@google.com' % cc.strip() for cc in options.cc.split(','))
|
if getpass.getuser() == SYSTEM_USER and not options.testmode:
|
email_cc.append('chromeos-build-alerts+db-replica-checker@google.com')
|
body = ('%s\n\n'
|
'Master (%s) status:\n%s\n\n'
|
'Slave (%s) status:\n%s' % (summary, DATABASE_HOST, master_result,
|
REPLICA_DATABASE_HOST, slave_result))
|
p = os.popen('/usr/sbin/sendmail -t', 'w')
|
p.write('To: %s\n' % ','.join(email_to))
|
if email_cc:
|
p.write('Cc: %s\n' % ','.join(email_cc))
|
|
p.write('Subject: Inconsistency detected in cautotest DB replica on %s.\n'
|
% socket.gethostname())
|
p.write('Content-Type: text/plain')
|
p.write('\n') # blank line separating headers from body
|
p.write(body)
|
p.write('\n')
|
return_code = p.close()
|
if return_code is not None:
|
print 'Sendmail exit status %s' % return_code
|
|
|
def main():
|
options = ParseOptions()
|
master_result = FetchMasterResult()
|
slave_result = FetchSlaveResult()
|
problem_msg = RunChecks(options, master_result, slave_result)
|
if problem_msg:
|
ShowStatus(options, master_result, slave_result, problem_msg)
|
sys.exit(-1)
|
|
if __name__ == '__main__':
|
main()
|
