# $Id: mirrorstats.py,v 1.1 2003/02/28 19:21:53 preisl Exp $
# Module for analysing and making available mirror statistics
import os
import time
import re
import log4py
import constants
TRUE = 1
FALSE = 0
# ----------------------------------------------------------------------
# Holds information about a specific mirror
class MirrorInfo:
def __init__(self):
self.path = None # Path of logfile containing this data
self.title = None # Title of the Mirror
self.category = None # Category of the mirror (for logfiles)
self.description = None # Description of the mirror
self.status = None # Status of mirror
self.exitstatus = None # Exit status of mirror
self.nrbytesretr = 0L # Number of bytes retrieved
self.nrfilesretr = 0 # Number of files retrieved
self.nrdirscreated = 0 # Number of directories created
self.nrlinkscreated = 0 # Number of links created
self.nrbytesdel = 0L # Number of bytes deleted
self.nrfilesdel = 0 # Number of files deleted
self.nrdirsdel = 0 # Number of directories deleted
self.nrlinksdel = 0 # Number of links deleted
self.starttime = 0 # Start time of the mirroring
self.endtime = 0 # End time of the mirroring
# ----------------------------------------------------------------------
# Provides statistics for each mirror and its history
class MirrorStats:
# Stats id constants
STARTHOUR = 0
ENDHOUR = 1
DURATION = 2
TXRATE = 3
SIZE = 4
FILES = 5
DIRS = 6
LINKS = 7
SIZEDEL = 8
FILESDEL = 9
DIRSDEL = 10
LINKSDEL = 11
def __init__(self):
self.log4py = log4py.Logger("FALSE").get_instance(self)
self.mirrorhash = {} # Table of title: Table of end time: MirrorInfo objects
self.lastupdhash = {} # Table of title: last updated end time
self.lastchkhash = {} # Table of title: last check end time
self.categoryhash = {} # Table of category: title
self.activemirrors = [] # List of mirrors that appear to be active
def update(self, logdir, lockdir):
self.log4py.debug("Updating from logsslocksslogdirlockdir import
self._parsealllogs(logdir, lockdir)
def purgelogs(self, history = 28):
self._deleteoldlogs(history)
def _parsealllogs(self, mainlogdir, lockdir):
"Parse all the log files in the main log directory"
if (mainlogdir[-1:] != "/"):
mainlogdir += "/"
if not os.path.exists(mainlogdir):
self.log4py.error("[ Main ] Error: Directory \"%s\" doesn't exist !" % mainlogdir)
return
logdirs = os.listdir(mainlogdir)
pat = re.compile(".*_logs");
for file in logdirs[:]:
if (not pat.match(file)):
logdirs.remove(file)
self.log4py.debug("Found %s log directories" % len(logdirs))
for dir in logdirs:
self._parselogs(os.path.normpath('/'.join((mainlogdir, dir))))
self.log4py.debug("Found %s categories and %s mirrors" % (len(self.categoryhash), len(self.mirrorhash)))
for mirror in self.mirrorhash.keys():
lockname = mirror.strip().replace(" ", "_").lower() + ".lock"
if os.path.exists(lockdir + "/" + lockname):
self.activemirrors.append(mirror)
def _parselogs(self, logdir):
"Parse the logs for an individual mirror"
if (logdir[-1:] != "/"):
logdir = logdir + "/"
if not os.path.exists(logdir):
self.log4py.error("[ Main ] Error: Directory \"%s\" doesn't exist !" % logdir)
return
logfiles = os.listdir(logdir)
pat = re.compile("log-.*");
for file in logfiles[:]:
if (not pat.match(file)):
logfiles.remove(file)
self.log4py.debug("Found %s log files in directory %s" % (len(logfiles), logdir))
loghash = {}
for file in logfiles:
logpath = logdir + file
log = open(logpath, "r")
mi = MirrorInfo()
mi.path = logpath
founddata = FALSE
while 1:
line = log.readline()
if not line: break
line = line.strip()
if not founddata:
# Skip until data start found
if line.startswith("<!-- DATA START"): founddata = TRUE
continue
if line.startswith("<!-- DATA END"):
# Stop when end of data found
break
elif line.startswith("<!-- TITLE"):
mi.title = line[12:-4].strip()
elif line.startswith("<!-- STATUS"):
mi.status = line[13:-4].strip()
elif line.startswith("<!-- EXITSTATUS"):
mi.exitstatus = int(line[17:-4].strip())
elif line.startswith("<!-- CATEGORY"):
mi.category = line[15:-4].strip()
elif line.startswith("<!-- DESCRIPTION"):
mi.description = line[18:-4].strip()
elif line.startswith("<!-- STATISTICS"):
stats = line[17:-4].split()
mi.nrbytesretr = long(stats[0])
mi.nrfilesretr = long(stats[1])
mi.nrlinkscreated = long(stats[2])
mi.nrdirscreated = long(stats[3])
mi.nrbytesdel = long(stats[4])
mi.nrfilesdel = long(stats[5])
mi.nrlinksdel = long(stats[6])
mi.nrdirsdel = long(stats[7])
mi.starttime = long(stats[8])
mi.endtime = long(stats[9])
log.close()
if (mi.title == None):
self.log4py.warn("[ Main ] Warning: Missing info in log file %s" % logpath)
else:
if loghash.has_key(mi.endtime): mi.endtime += 1
loghash[mi.endtime] = mi
if (len(loghash) == 0):
self.log4py.warn("[ Main ] Warning: Log dir %s contains no logs" % logdir)
return
# Get most recent title and category for entry into hash tables
endtimes = loghash.keys()
endtimes.sort()
endtimes.reverse()
title = loghash[endtimes[0]].title
category = loghash[endtimes[0]].category
if self.mirrorhash.has_key(title):
# Just add in to other hash, but will overwrite any duplicates
self.mirrorhash[title].update(loghash)
else:
self.mirrorhash[title] = loghash
if self.categoryhash.has_key(category):
self.categoryhash[category].append(title)
else:
self.categoryhash[category] = [title]
def _deleteoldlogs(self, history):
"Delete logs past time limit but keep those for the last updates"
range = self.gettimetup_range(time.localtime(time.time()), history)
reaptime = time.mktime(range[0])
for title in self.mirrorhash.keys():
loghash = self.mirrorhash[title]
lastupdate = self.getlastupdate(title)
for endtime in loghash.keys():
if endtime < reaptime and endtime != lastupdate:
path = loghash[endtime].path
try:
os.unlink(path)
except:
self.log4py.error("[ Main ] Error: Failed to delete log file: %s" % path)
def getmirrorsforcategory(self, category):
return self.categoryhash[category]
def getcategories(self):
return self.categoryhash.keys()
def isactive(self, title):
return title in self.activemirrors
def _toendofday(self, tt):
return (tt[0], tt[1], tt[2], 23, 59, 59, tt[6], tt[7], tt[8])
def _tomidday(self, tt):
return (tt[0], tt[1], tt[2], 12, 0, 0, tt[6], tt[7], tt[8])
def _tostartofday(self, tt):
return (tt[0], tt[1], tt[2], 0, 0, 0, tt[6], tt[7], tt[8])
def widen(self, timetup_range):
"Returns range extended to start and end of first and last days"
(s, e) = timetup_range
return (self._tostartofday(s), self._toendofday(e))
def _getdayrange(self, timetup):
"Return tuple containing start and end time of day for given time"
return (self._tostartofday(timetup), self._toendofday(timetup))
def getinfolist(self, title, timetup_range):
"Return list of mirror infos for mirror and time range"
stime, etime = timetup_range
st = time.mktime(stime)
et = time.mktime(etime)
list = []
loghash = self.mirrorhash[title]
if loghash:
keys = loghash.keys()
keys.sort()
for endtime in keys:
if endtime >= st and endtime <= et:
list.append(loghash[endtime])
return list
def getupdates(self, title, timetup_range):
"Return list (can be 0 len) of update end times for the mirror and time range specified"
stime, etime = timetup_range
st = time.mktime(stime)
et = time.mktime(etime)
loghash = self.mirrorhash[title]
list = []
if loghash:
for endtime in loghash.keys():
if endtime >= st and endtime <= et:
if loghash[endtime].status[0].isdigit():
list.append(endtime)
return list
def getlastupdate(self, title):
"Return end time of the last update of given mirror, or None"
if self.lastupdhash.has_key(title):
# In cache
return self.lastupdhash[title]
else:
updates = self.getupdates(title, (time.localtime(0), time.localtime(time.time())))
if len(updates) == 0:
return None
else:
updates.sort()
# Add to cache
self.lastupdhash[title] = updates[-1]
return updates[-1]
def getlastcheck(self, title):
"Return end time of the last check on the mirror, or None"
if self.lastchkhash.has_key(title):
# In cache
return self.lastchkhash[title]
else:
loghash = self.mirrorhash[title]
if not loghash:
return None
else:
times = loghash.keys()
times.sort()
# Add to cache
self.lastchkhash[title] = times[-1]
return times[-1]
def getinfo(self, title, endtime):
"Return mirror info object for given title and endtime"
loghash = self.mirrorhash[title]
if not loghash: return None
if not endtime: return None
return loghash[endtime]
def _sum(self, list):
sum = 0.0
for item in list:
sum += item
return sum
def gettimetup_range(self, timetup, daysback):
"Return whole day time range for given time and #days back in time"
s = time.localtime(time.mktime(self._tomidday(timetup)) - daysback * 24 * 60 * 60)
e = timetup
return self.widen((s, e))
def getdaytimes(self, timetup_range):
"Return list of midday timetups for each day in the time range"
stime = time.mktime(self._tomidday(timetup_range[0]))
etime = time.mktime(self._toendofday(timetup_range[1]))
daytimes = []
day = 24 * 60 * 60
for t in range(stime, etime, day):
daytimes.append(self._tomidday(time.localtime(t)))
return daytimes
def getmirrorcalendar(self, title, timetups):
"Return mapping of 'timetup:list of mirror infos' for mirror and list of timetups"
cal = {}
for t in timetups:
cal[t] = self.getinfolist(title, self._getdayrange(t))
return cal
def getstatistic(self, title, timetup_range, statistic):
"Return sum, min, mean, max of the requested statistic from mirror and time range. Min and max include paths for logs"
updates = self.getupdates(title, timetup_range)
updates.sort()
updates.reverse()
list = []
paths = []
for t in updates:
mi = self.getinfo(title, t)
if not mi: continue
paths.append(mi.path)
if statistic == MirrorStats.STARTHOUR:
list.append(time.localtime(mi.starttime)[3])
elif statistic == MirrorStats.ENDHOUR:
list.append(time.localtime(mi.endtime)[3])
elif statistic == MirrorStats.DURATION:
list.append(mi.endtime - mi.starttime)
elif statistic == MirrorStats.TXRATE:
if (mi.endtime - mi.starttime > 0):
list.append(float(mi.nrbytesretr / 1024)/(mi.endtime - mi.starttime))
else:
list.append(0.0)
elif statistic == MirrorStats.SIZE:
list.append(mi.nrbytesretr)
elif statistic == MirrorStats.FILES:
list.append(mi.nrfilesretr)
elif statistic == MirrorStats.DIRS:
list.append(mi.nrdirscreated)
elif statistic == MirrorStats.LINKS:
list.append(mi.nrlinkscreated)
elif statistic == MirrorStats.SIZEDEL:
list.append(mi.nrbytesdel)
elif statistic == MirrorStats.FILESDEL:
list.append(mi.nrfilesdel)
elif statistic == MirrorStats.DIRSDEL:
list.append(mi.nrdirsdel)
elif statistic == MirrorStats.LINKSDEL:
list.append(mi.nrlinksdel)
if len(list) > 0:
sum = self._sum(list)
minimum = min(list)
minimumpath = paths[list.index(minimum)]
maximum = max(list)
maximumpath = paths[list.index(maximum)]
mean = sum / len(list)
return (sum, (minimum, minimumpath), mean, (maximum, maximumpath))
else:
return (0, (0,None), 0, (0,None))
# Some utility functions for displaying stats
def get_status_gif(mirrorinfo):
if mirrorinfo.status[0].isdigit():
gif = "updated"
elif mirrorinfo.status == constants.MSneverloggedin:
gif = "nologin"
elif mirrorinfo.status == constants.MSunchanged:
gif = "unchanged"
elif mirrorinfo.status == constants.MSactive:
gif = "active"
else:
gif = "unknown"
if mirrorinfo.exitstatus == 1: gif += "-aborted"
return gif
def fmttime(timesecs):
return time.strftime("%d/%m/%y %H:%M %Z", time.localtime(timesecs))
def fmtdur(timesecs):
minutes, seconds = divmod(timesecs, 60)
hours, minutes = divmod(minutes, 60)
return "%02u:%02u:%02u" % (hours, minutes, seconds)
def fmtnum(number):
num = []
number = str(long(number))
for i in range(len(number)): num.append(number[i])
if num[-1] == "L":
num = num[:-1]
num.reverse()
groups = divmod(len(num) - 1, 3)[0]
for i in range(groups):
num.insert(3 + (i * 3) + i, " ")
num.reverse()
return "".join(num)
|