"""
The main program of EMirror
"""
# ----------------------------------------------------------------------
# Import functions I need
from string import strip,find,split,lower,atoi,atof,atol,digits,rfind,capitalize,zfill,join
from sys import exit,argv,hexversion
from os import chdir,mkdir,utime,unlink,getcwd,chmod,symlink,lstat,readlink
from os import environ,rename,system,getuid,chown,remove,kill,getpid,umask
from time import time,mktime,sleep,ctime,localtime,asctime,strftime
if (hexversion >= 0x20000f0):
# Use ftplib if later than version 2.0
from ftplib import FTP,all_errors,error_reply,error_temp,error_perm,error_proto
else:
from ftplibbeta import FTP,all_errors,error_reply,error_temp,error_perm,error_proto
from re import match,sub,compile
from types import TupleType,InstanceType,ListType,StringType
from urlparse import urlunparse
from tempfile import mktemp
from ConfigParser import ConfigParser
from error import errormsgs
from tools import *
from constants import *
from traceback import print_exception,format_exception
import os
import getopt
import copy
import sys
import signal
import shutil
import socket
import popen2
import log4py
import process
import mirrorstats
import traceback
import rsync
import wget
# ----------------------------------------------------------------------
#LOG_FMT = "%T %L %M"
LOG_FMT = log4py.FMT_DEBUG
# ----------------------------------------------------------------------
# Definition of the File-Class:
class File:
def __init__(self):
self.log4py = log4py.Logger().get_instance(self)
self.log4py.set_loglevel(log4py.LOGLEVEL_ERROR)
self.name = "" # Name of the file
self.path = "" # Directory path of file
self.filetype = None # FileType (File, Directory, Link)
self.connection = None # Connection necessary for this file
self.permissions = 0 # File Permissions
self.size = 0L # Size of the file
self.path = "" # Path to the file
self.linktarget = ""
self.modtime = 0
# split a file entry so that blanks are not lost, e.g. "a b c.txt" -> ("a ", "b", "c.txt")
def filesplit(self, string, character):
list = []
element = ""
while (len(string) > 0):
current = string[0]
if (current == character):
while (len(string) > 1) and (string[1] == character):
element = "%s%s" % (element, character)
string = string[1:]
current = string[0]
list.append(element)
element = ""
else:
element = "%s%s" % (element, current)
string = string[1:]
if (element != ""):
list.append(element)
return list
def GetFilePerms(self, permstring):
tempperms = 0
# File permissions for owner
if permstring[0] == "r":
tempperms = tempperms | 256
if permstring[1] == "w":
tempperms = tempperms | 128
if permstring[2] == "x":
tempperms = tempperms | 64
elif permstring[2] == "S":
tempperms = tempperms | 2048
elif permstring[2] == "s":
tempperms = tempperms | 2112
# File permissions for group
if permstring[3] == "r":
tempperms = tempperms | 32
if permstring[4] == "w":
tempperms = tempperms | 16
if permstring[5] == "x":
tempperms = tempperms | 8
elif permstring[5] == "S":
tempperms = tempperms | 1024
elif permstring[5] == "s":
tempperms = tempperms | 1032
# File permissions for other
if permstring[6] == "r":
tempperms = tempperms | 4
if permstring[7] == "w":
tempperms = tempperms | 2
if permstring[8] == "x":
tempperms = tempperms | 1
elif permstring[8] == "T":
tempperms = tempperms | 512
elif permstring[8] == "t":
tempperms = tempperms | 513
self.permissions = tempperms
def SplitNonEPLF(self, filestruct):
if (filestruct[0] == "-"):
self.filetype = FTRegularFile
elif (filestruct[0] == "d"):
self.filetype = FTDirectory
elif (filestruct[0] == "l"):
self.filetype = FTLink
permissionstring = filestruct[1:10]
self.GetFilePerms(permissionstring)
splitted = self.filesplit(filestruct, " ")
self.name = ""
if (lower(strip(splitted[5])) in MSMonthStrings): # WU-FTPD < 2.6.*, ProFTPD and similar FTPD format
self.size = atol(strip(splitted[4]))
for i in range(8, len(splitted)):
self.name = self.name + " " + splitted[i]
else: # WU-FTPD 2.6.* (some versions don't include the file owner)
self.size = atol(strip(splitted[3]))
for i in range(7, len(splitted)):
self.name = self.name + " " + splitted[i]
self.name = lstrip(self.name)
if (self.filetype == FTLink):
splitted = split(self.name, "->")
self.name = strip(splitted[0])
self.linktarget = strip(splitted[1])
else:
self.linktarget = ""
def SplitEPLF(self, filestruct):
splitted = split(filestruct[1:], ",")
self.name = strip(splitted[len(splitted) - 1])
self.linktarget = ""
for i in range(len(splitted) - 1):
parameter = strip(splitted[i])
if (parameter == "/"):
self.filetype = FTDirectory
self.permissions = 493 # rwxr-xr-x
elif (parameter == "r"):
self.filetype = FTRegularFile
self.permissions = 420 # rw-r--r--
elif (parameter[0] == "s"):
self.size = atol(parameter[1:])
elif (parameter[0] == "m"):
self.modtime = atoi(parameter[1:])
def SplitMSFormat(self, filestruct):
if (filestruct[24:29] == "<DIR>"):
self.filetype = FTDirectory
self.permissions = 493 # rwxr-xr-x
else:
self.filetype = FTRegularFile
self.permissions = 420 # rw-r--r--
self.size = atol(strip(filestruct[29:38]))
self.name = filestruct[39:]
def SplitFileInfo(self, filestruct, title):
if (filestruct[0] == "+"):
self.SplitEPLF(filestruct)
self.log4py.debug("[ %s (debug) ] SplitFileInfo (%s): EPLF detected" % (str(title), self.name))
elif (MSDate.match(filestruct)):
self.SplitMSFormat(filestruct)
self.log4py.debug("[ %s (debug) ] SplitFileInfo (%s): Microsoft (NT5 Beta ?) format detected" % (str(title), self.name))
else:
self.SplitNonEPLF(filestruct)
self.log4py.debug("[ %s (debug) ] SplitFileInfo (%s): Standard unix format detected" % (str(title), self.name))
# ----------------------------------------------------------------------
class Installer:
# Initialize the installer
def __init__(self):
self.index = 0 # The index of the source file (are ordered)
self.title = None # Mirror Title (= Installer Title)
self.log4py = log4py.Logger().get_instance(self)
self.log4py.set_loglevel(log4py.LOGLEVEL_ERROR)
self.sources = {}
self.sourceregexps = {}
self.srpms = []
self.srpmregexp = None
self.rpms = []
self.rpmregexp = None
self.rpmopts = INSRPMOpts
self.rpmbuildopts = INSRPMBuildOpts
self.builddir = INSBuildDir
self.logfile = None # Logfile of the build output
# Rebuild a SRPM file
def rebuildSRPM(self, filename):
self.log4py.info("[ %s Installer ] Rebuilding SRPM %s" % (str(self.title), filename))
rpms = ls(INSRPMDir, recursive = TRUE, includepath = TRUE)
commandline = "rpm %s %s 2>&1" % (self.rpmbuildopts, filename)
doit = popen2.popen2(commandline)
output = doit[0].readlines()
if (self.logfile != None):
file = open(self.logfile, "a")
file.write("\n[ %s Installer ] Rebuilding SRPM %s\n\n" % (str(self.title), filename))
for i in range(len(output)):
file.write(output[i])
file.write("\n")
file.close()
newrpms = listdiff(rpms, ls(INSRPMDir, recursive = TRUE, includepath = TRUE))
if (len(newrpms) > 0):
self.rpms = self.rpms + newrpms
else:
self.log4py.error("[ %s Installer ] Rebuild of %s failed or RPM already exists. Check logfile for details." % (str(self.title), filename))
# Install RPMs
def installRPMs(self):
filelist = ""
for i in range(len(self.rpms)):
filelist = "%s %s" % (filelist, self.rpms[i])
self.log4py.info("[ %s Installer ] Installing RPMs: %s" % (str(self.title), strip(filelist)))
commandline = "rpm %s %s 2>&1" % (self.rpmopts, strip(filelist))
doit = popen2.popen2(commandline)
output = doit[0].readlines()
if (self.logfile != None):
file = open(self.logfile, "a")
file.write("\n[ %s Installer ] Installing RPMs: %s\n\n" % (str(self.title), strip(filelist)))
for i in range(len(output)):
file.write(output[i])
file.write("\n")
file.close()
for i in range(len(output)):
if find(output[i], "already installed") or find(output[i], "failed"):
self.log4py.error("[ %s Installer ] Installation of RPMs failed. Check logfile for details." % str(self.title))
# Install Source
def installSource(self, file, commands):
self.log4py.info("[ %s Installer ] Installing Source: %s [%s]" % (str(self.title), file, commands))
if (find(commands, ";") != -1):
splitted = split(commands, ";")
else:
splitted = [commands]
currentdir = getcwd()
chdir(self.builddir)
dirlist = ls(self.builddir, includepath = TRUE, onlydirectories = TRUE)
newdir = []
filelog = open(self.logfile, "a")
for i in range(len(splitted)):
command = splitted[i]
command = sub("%f", file, command)
if (len(newdir) == 1):
command = sub("%d", newdir[0], command)
self.log4py.debug("[ %s Installer (debug) ] Executing: %s" % (str(self.title), command))
filelog.write("\n[ %s Installer ] Source: Executing %s\n\n" % (str(self.title), command))
if (command[:2] == "cd"):
chdir(command[3:])
else:
doit = popen2.popen2("%s 2>&1" % command)
output = doit[0].readlines()
for i in range(len(output)):
filelog.write(output[i])
filelog.write("\n")
if (len(newdir) == 0):
newdir = listdiff(dirlist, ls(self.builddir, includepath = TRUE, onlydirectories = TRUE))
filelog.close()
if (len(newdir) == 1): # Cleanup (remove build directory)
rm(newdir[0])
chdir(currentdir)
# Do the real installation
def finish(self):
if (self.logfile != None) and (os.path.exists(self.logfile)): unlink(self.logfile)
for i in range(len(self.srpms)):
self.rebuildSRPM(self.srpms[i])
if (len(self.rpms) > 0):
self.installRPMs()
keys = self.sources.keys()
keys.sort()
for i in range(len(keys)):
key = keys[i]
self.installSource(self.sources[key][0], self.sources[key][1])
# Check wether a file has to be installed or not
def checkfile(self, path, filename, localname):
if (self.rpmregexp != None) and ((self.rpmregexp.match(filename)) or (self.rpmregexp.match(path + filename))):
self.rpms.append(localname)
if (self.srpmregexp != None) and ((self.srpmregexp.match(filename)) or (self.srpmregexp.match(path + filename))):
self.srpms.append(localname)
keys = self.sourceregexps.keys()
keys.sort()
for i in range(len(keys)):
key = keys[i]
if (self.sourceregexps[key][0].match(filename)) or (self.sourceregexps[key][0].match(path + filename)):
self.sources[self.index] = [localname, self.sourceregexps[key][1]]
self.index = self.index + 1
# Add regular expressions to the installer (from the config file)
def add(self, option, value):
if (lower(option) == "rpmoptions"):
self.rpmopts = value
elif (lower(option) == "rpmbuildoptions"):
self.rpmbuildopts = value
elif (lower(option) == "logfile"):
self.logfile = value
elif (lower(option)[:4] == "rpms"):
self.rpmregexp = compile(unfold(value))
elif (lower(option)[:5] == "srpms"):
self.srpmregexp = compile(unfold(value))
elif (lower(option)[:5] == "other"):
index = atoi(option[5:])
filename = value[:find(value, "[") - 1]
command = value[find(value, "[") + 1:-1]
self.sourceregexps[index] = [compile(unfold(filename)), command]
elif (lower(option) == "builddir"):
self.builddir = value
# ----------------------------------------------------------------------
class Connection:
def __init__(self):
self.protocol = None # Protocol to connect
self.hostname = None # Hostname of the remote host
self.realhostname = "" # real Hostname / IP (for hidden sites)
self.port = None # Port to connect to
self.username = None # Username
self.password = None # Password used ...
self.status = None # Connection status
# ----------------------------------------------------------------------
# Definition of the Mirror-Class:
class Mirror:
# Initialisation of some variables
def __init__(self):
self.url = "" # Full URL of the mirror
self.protocol = "" # Protocol to start with
self.username = "" # Username
self.password = "" # Password
self.hostname = "" # Hostname / IP of the remote site
self.realhostname = "" # real Hostname / IP (for hidden sites)
self.path = "" # Remote directory to mirror
self.directory = getcwd() + "/" # Local directory
self.noexception = FALSE # Do NOT use exceptions ?
self.log4py = log4py.Logger().get_instance(self)
self.log4py.set_loglevel(log4py.LOGLEVEL_ERROR)
self.test = FALSE # Test mode
self.port = 0 # Port Number
self.delay = DVDelay # Delay time between retries
self.lockdir = "/var/tmp/emirror-locks/" # Directory containing mirror locks
self.mlock = None # Mirror lock
self.locked = FALSE # Flag indicating lock status
self.alarmedoperation = None # Name of operation with alarm set
self.locktimeout = 43200 # Time in seconds before locked mirror is considered dead (12h)
self.operationtimeout = 10800 # Time in seconds before mirror op is aborted (3h)
self.completed_file = None # File to create on completion in parent dir
self.firstchange = TRUE # TRUE until first mirror change
self.followsymlinks = FALSE # Do not follow symbolic links
self.parentformat = PFFull # Full parent directory format
self.history_logging = FALSE # TRUE to keep all log files
self.logdirectory = "" # Directory where the logfiles reside
self.logfilename = None # Name of the logfile
self.logfileformat = LFPlain # Format of the logfiles (Plain, HTML)
self.logext = "" # Filename extension for log file
self.logdata = [] # The Stuff which should appear in the log
self.retries = DVRetries # Number of retries
self.logfile = None # Filehandler of the logfile
self.recursive = FALSE # Recursive download
self.starttime = 0 # Start time of the mirroring
self.endtime = 0 # End time of the mirroring
self.category = "Assorted" # Category of the mirror (for logfiles)
self.description = None # Description of the mirror
self.adminemail = None # EMail-adress of the ftp-maintainer
self.errorcolor = "#CC0033" # Default color for error messages
self.warningcolor = "#191970" # Default color for warning messages
self.logtemplate = None # Template file for logs
self.template = None # Template data
self.templateloop = None # Template data for
self.maxdays = 0 # maximum age of files to download (days)
self.maxage = 0 # maximum age of local files (days)
self.maxdelete = None # maximum size or percantage of files to delete
self.onlylatest = [] # Only get the latest version
self.title = None # Title of the Mirror
self.localurl = None # Local download URL
self.exclude = None # Exclude pattern
self.include = None # Regular expression for file selection
self.ignore = None # Regular expression to ignore local files
self.deletelocal = FALSE # Always delete local files, which don't exist on the remote side
self.errormailcmd = None # Command to execute on error
self.infomailcmd = None # Command to execute when a mirror has changed
self.ignoremdtm = FALSE # Do not ignore the MDTM when downloading files
self.nomodcheck = FALSE # Don't abort if remote file is modified during transfer
self.passivemode = FALSE # Use passive mode ?
self.continueftp = FALSE # Continue FTP downloads ?
self.umask = None
self.deletefirst = TRUE # Delete before downloading replacement (include del in stats)
self.neverloggedin = TRUE # Indicates login never occurred
self.filelist = [] # List of files for iterative checking
self.connection = None # Current connection
self.ftp = FTP() # The FTP-Connection
self.downloadfile = "" # Name of the file which is being downloaded
self.currentfile = "" # the current file being processed
self.remotefiles = [] # Remote file listing for a new directory
self.localfiles = [] # Local file list for a new directory
self.statfilelist = [] # File-list for the MyStat-Function
self.nrbytesretr = 0L # Number of bytes retrieved
self.nrfilesretr = 0 # Number of files retrieved
self.nrdirscreated = 0 # Number of directories created
self.nrlinkscreated = 0 # Number of links created
self.nrbytesdel = 0L # Number of bytes deleted
self.nrfilesdel = 0 # Number of files deleted
self.nrdirsdel = 0 # Number of directories deleted
self.nrlinksdel = 0 # Number of links deleted
self.owner = None # Set owner of downloaded files to this UID
self.group = None # Set group of downloaded files to this GID
self.filelistcache = {} # Simple File Listing cache
self.installer = Installer() # File Installer
self.proxy = None # Proxy host:port
def lock(self):
if not os.path.exists(self.lockdir):
try:
mkdirtree(self.lockdir)
self.log4py.info("[ %s ] Lock directory \"%s\" created" % (str(self.title), self.lockdir))
except IOError, reason:
self.log4py.error("[ %s ] Failed to create lock directory \"%s\": %s" % (str(self.title), self.lockdir, reason))
name = self.title.strip().replace(" ", "_").lower() + ".lock"
self.mlock = FileLock(self.lockdir + name, self.locktimeout)
self.mlock.log4py.set_loglevel(self.log4py.get_loglevel())
status = self.mlock.status()
if (status == FileLock.LOCKED_OTHER) or (status == FileLock.LOCKED_SELF):
# Already locked
return FALSE
if (status == FileLock.STALE) or (status == FileLock.ORPHAN):
# Locked - but lock should be removed
self.log4py.info("[ %s ] Killing stale/orphan lock" % self.title)
if not self.mlock.killStaleLock():
self.log4py.error("[ %s ] Failed to kill stale lock. Mirror marked as 'active'" % str(self.title))
return FALSE
# Should now be unlocked so acquire the lock
if (self.mlock.acquire() == FALSE):
self.log4py.error("[ %s ] Failed to acquire free lock. Mirror marked as 'active'" % str(self.title))
return FALSE
return TRUE
def unlock(self):
if (self.mlock.status() == FileLock.LOCKED_SELF):
return self.mlock.release()
# Don't own the lock!
return FALSE
def alarmOn(self, alarmedoperation):
self.alarmedoperation = alarmedoperation
signal.alarm(self.operationtimeout)
def alarmOff(self):
signal.alarm(0)
self.alarmedoperation = None
def dofirstchangeaction(self):
"Called just before the first change of a mirror"
if not self.firstchange:
return
self.firstchange = FALSE
if self.completed_file != None:
self.removeCompleted()
def dolastchangeaction(self):
"Called just after the last change of a complete mirror"
if self.firstchange:
return # there were no changes
if self.completed_file != None:
self.createCompleted()
def removeCompleted(self):
fn = os.path.normpath(self.directory + "/" + self.completed_file)
self.log4py.info("[ %s ] Removing Completed file" % self.title)
try:
unlink(fn)
except:
self.log4py.warn("[ %s ] Failed to remove Completed file: %s" % (self.title, sys.exc_info()[1]))
def createCompleted(self):
fn = os.path.normpath(self.directory + "/" + self.completed_file)
self.log4py.info("[ %s ] Creating Completed file" % self.title)
try:
f = open(fn, "w")
f.write(str(time()) + "\n")
for line in self.logdata:
action = line[0].strip()
file = line[1].strip()
if file.startswith("<"):
file = file[file.find(">") + 1 : file.rfind("<")].strip()
f.write("%s: %s\n" % (action, file))
f.close()
except:
self.log4py.error("[ %s ] Failed to write Completed file: %s" % (self.title, sys.exc_info()[1]))
# Open the logfile ... that is notice the start time ;-)
def StartLog(self):
self.starttime = time()
# Append a line to the logfile
def AppendLog(self, line):
if (self.logfilename != None):
if (line[0] == LMRetrieve) and (self.localurl != None):
downloadurl = self.GetLCwd(parentdirectory = self.localurl, hostname = self.hostname) + self.currentfile.name
downloadname = self.currentfile.path + self.currentfile.name
if (self.logfileformat == LFPlain):
logrow = [ line[0], downloadname ]
else:
logrow = [ line[0], "<A HREF=\"%s\">%s</A>" % (downloadurl, downloadname) ]
else:
logrow = [ line[0], line[1] ]
self.logdata.append(logrow)
# Read the logtemplate file
def ReadLogTemplate(self):
self.log4py.debug("[ %s (debug) ] Logfile: reading template file" % str(self.title))
file = open(self.logtemplate, "r")
line = "<empty>"
self.template = []
self.templateloop = []
target = self.template
if (self.logfileformat == LFHTML):
dateiname = "index" + LEHTML
fileupdatedname = "updated" + LEHTML
fileerrorsname = "errors" + LEHTML
elif (self.logfileformat == LFPHP):
dateiname = "index" + LEPHP
fileupdatedname = "updated" + LEPHP
fileerrorsname = "errors" + LEPHP
elif (self.logfileformat == LFPlain):
dateiname = "index" + LEPlain
fileupdatedname = "updated" + LEPlain
fileerrorsname = "errors" + LEPlain
while (line != ""):
line = file.readline()
line = sub("\$indexfilename\$", dateiname, line)
line = sub("\$updatedfilename\$", fileupdatedname, line)
line = sub("\$errorfilename\$", fileerrorsname, line)
if (lower(strip(line)) == "$logstart$"):
target.append("$logdata$")
target = self.templateloop
elif (lower(strip(line)) == "$logend$"):
target = self.template
elif (find(strip(lower(line)), "<!-- errorcolor:") != -1):
color = strip(lower(line))
position = find(color, "<!-- errorcolor:")
color = color[position + len("<!-- errorcolor:"):]
position = find(color, "-->")
color = strip(color[:position])
self.errorcolor = color
elif (find(strip(lower(line)), "<!-- warningcolor:") != -1):
color = strip(lower(line))
position = find(color, "<!-- warningcolor:")
color = color[position + len("<!-- warningcolor:"):]
position = find(color, "-->")
color = strip(color[:position])
self.warningcolor = color
else:
target.append(line)
file.close()
# Check the logfilename and adopt its extension
def CheckLogfilename(self, filename):
if (rfind(filename, ".") != -1):
extension = lower(filename[rfind(filename, "."):])
basicname = filename[:rfind(filename, ".")]
else:
extension = ""
basicname = filename
if (extension == "") or (extension in [".html", ".asc", ".txt", ".php", ".php3", ".php4", ".htm"]):
if (self.logfileformat == LFHTML):
return basicname + LEHTML
elif (self.logfileformat == LFPHP):
return basicname + LEPHP
elif (self.logfileformat == LFPlain):
return basicname + LEPlain
else:
return filename
def CheckLogDirectory(self, dir):
if (os.path.exists(dir) and (dir != "")):
return TRUE
try:
mkdirtree(dir)
self.log4py.info("[ %s ] Logfile: Directory \"%s\" created" % (str(self.title), dir))
except:
self.log4py.error("[ %s ] Logfile: Error: Couldn't create directory \"%s\" !" % (str(self.title), dir))
return FALSE
return TRUE
# Write the whole log file at once
def WriteLog(self, exitstatus = 0):
if (self.logdirectory != None):
self.log4py.info("[ %s ] Logfile: writing" % str(self.title))
if (not self.CheckLogDirectory(self.logdirectory)): return
self.endtime = time()
if (self.history_logging):
self.logfilename = "log-" + str(self.endtime) + self.logext
try:
newFilename = self.CheckLogfilename(self.logfilename)
self.logfile = open(self.logdirectory + newFilename, "w")
except:
self.log4py.error("[ %s ] Logfile: Error: Couldn't write logfile \"%s\" !" % (str(self.title), self.logfilename))
return
difference = long(self.endtime) - long(self.starttime)
if (self.logtemplate == None):
self.log4py.error("[ %s ] Logfile: Error: No template file specified !" % str(self.title))
return
# Read the template file
self.ReadLogTemplate()
# Unparse the URL for a nice outfit :)
url = urlunparse((self.protocol, self.hostname, self.path, "", "", ""))
# Check wether a local download-URL is set
if (self.localurl != None):
downloadurl = self.GetLCwd(hostname = self.hostname, path = self.path, parentdirectory = self.localurl)
else:
downloadurl = ""
# Check the username and password
if (lower(self.username) == "ftp") or (lower(self.username) == "anonymous"):
username = self.username
password = self.password
else:
username = "-- Hidden --"
password = "-- Hidden --"
# Check the administrator EMail address
if self.adminemail != None:
email = self.adminemail
else:
email = ""
# Calculate the time
totalseconds = difference
minutes, seconds = divmod(totalseconds, 60)
hours, minutes = divmod(minutes, 60)
# Create Format specific text
if (self.logfileformat == LFPlain):
duration = "%2d Hour(s), %2d Minute(s) and %2d Second(s)" % (hours, minutes, seconds)
downloaded = "%s Byte(s) in %.0f File(s), %.0f Link(s) and %.0f Directories ..." % (lng2str(self.nrbytesretr), self.nrfilesretr, self.nrlinkscreated, self.nrdirscreated)
deleted = "%s Byte(s) in %.0f File(s), %.0f Link(s) and %.0f Directories ..." % (lng2str(self.nrbytesdel), self.nrfilesdel, self.nrlinksdel, self.nrdirsdel)
else:
duration = "<b>%2d</b> Hour(s), <b>%2d</b> Minute(s) and <b>%2d</b> Second(s)" % (hours, minutes, seconds)
downloaded = "<b>%s</b> Byte(s) in <b>%.0f</b> File(s), <b>%.0f</b> Link(s) and <b>%.0f</b> Directories ..." % (lng2str(self.nrbytesretr), self.nrfilesretr, self.nrlinkscreated, self.nrdirscreated)
deleted = "<b>%s</b> Byte(s) in <b>%.0f</b> File(s), <b>%.0f</b> Link(s) and <b>%.0f</b> Directories ..." % (lng2str(self.nrbytesdel), self.nrfilesdel, self.nrlinksdel, self.nrdirsdel)
# Calculate download rate
if (self.nrbytesretr > 0):
if (totalseconds > 0):
rate = self.nrbytesretr / totalseconds
else:
rate = 0
kbytepersec = int(rate / 1024)
if (kbytepersec > 0):
if (self.logfileformat == LFPlain):
downloadrate = "%.0f KByte/sec" % int(kbytepersec)
else:
downloadrate = "<b>%.0f</b> KByte/sec" % int(kbytepersec)
else:
if (self.logfileformat == LFPlain):
downloadrate = "%.0f Byte/sec" % int(rate)
else:
downloadrate = "<b>%.0f</b> Byte/sec" % int(rate)
else:
downloadrate = "Nothing downloaded"
# Calculate the entry for STATUS
if (self.locked == FALSE):
status = MSactive
self.logdata.append(["Information", "This mirror is currently being updated"])
elif (self.nrfilesretr == 0) and (self.nrfilesdel == 0) and (self.nrlinksdel == 0) and (self.nrdirscreated == 0) and (self.nrdirsdel == 0) and (self.nrlinkscreated == 0):
if self.neverloggedin:
status = MSneverloggedin
else:
status = MSunchanged
self.logdata.append(["Information", "No changes were found on the remote site"])
else:
status = str(self.endtime)
statsstring = "%.0f %.0f %.0f %.0f %.0f %.0f %.0f %.0f %.0f %.0f" % (self.nrbytesretr, self.nrfilesretr, self.nrlinkscreated, self.nrdirscreated, self.nrbytesdel, self.nrfilesdel, self.nrlinksdel, self.nrdirsdel, self.starttime, self.endtime)
# Process the template
for i in range(len(self.template)):
line = self.template[i]
if (line == "$logdata$"):
for j in range(len(self.logdata)):
loginfo = self.logdata[j]
for k in range(len(self.templateloop)):
line = self.templateloop[k]
if (self.logfileformat != LFPlain) and (loginfo[0] == LMError):
line = sub("\$type\$", "<font color=\"%s\">%s</font>" % (self.errorcolor, loginfo[0]), line)
line = sub("\$message\$", "<font color=\"%s\">%s</font>" % (self.errorcolor, loginfo[1]), line)
elif (self.logfileformat != LFPlain) and (loginfo[0] == LMWarning):
line = sub("\$type\$", "<font color=\"%s\">%s</font>" % (self.warningcolor, loginfo[0]), line)
line = sub("\$message\$", "<font color=\"%s\">%s</font>" % (self.warningcolor, loginfo[1]), line)
else:
line = sub("\$type\$", loginfo[0], line)
line = sub("\$message\$", loginfo[1], line)
self.logfile.write(line)
else:
line = sub("\$title\$", str(self.title), line)
line = sub("\$url\$", url, line)
line = sub("\$category\$", str(self.category), line)
line = sub("\$description\$", str(self.description), line)
line = sub("\$localurl\$", downloadurl, line)
line = sub("\$email\$", email, line)
line = sub("\$hostname\$", self.hostname, line)
line = sub("\$port\$", str(self.port), line)
line = sub("\$username\$", username, line)
line = sub("\$password\$", password, line)
line = sub("\$remotedirectory\$", self.path, line)
line = sub("\$localdirectory\$", self.directory, line)
line = sub("\$tries\$", str(self.retries), line)
line = sub("\$delay\$", str(self.delay), line)
line = sub("\$downloadrate\$", downloadrate, line)
line = sub("\$starttime\$", strftime("%d/%m/%y %H:%M %Z", localtime(self.starttime)), line)
line = sub("\$endtime\$", strftime("%d/%m/%y %H:%M %Z", localtime(self.endtime)), line)
line = sub("\$duration\$", duration, line)
line = sub("\$downloadstats\$", downloaded, line)
line = sub("\$deletestats\$", deleted, line)
line = sub("\$mdtm\$", status, line)
line = sub("\$exitstatus\$", str(exitstatus), line)
line = sub("\$statistics\$", statsstring, line)
line = sub("\$lastupdate\$", strftime("%d/%m/%y %H:%M %Z", localtime(time())), line)
line = sub("\$lastupdate\$", ctime(time()), line)
line = sub("\$timestamp\$", str(time()), line)
self.logfile.write(line)
# If mirror changed send email to interested users
if (status[0] in digits) and (self.infomailcmd != None):
if find(self.infomailcmd, "%s") != -1:
cmd = self.infomailcmd % ("Mirror changed: %s" % self.title)
else:
cmd = self.infomailcmd
self.log4py.info("[ %s ] Executing: %s" % (str(self.title), cmd))
filename = mktemp()
file = open(filename, "w")
if (exitstatus == 1):
file.write("\nECLiPt-Mirror: Package \"%s\" updated, but mirror aborted!\n\n" % str(self.title))
else:
file.write("\nECLiPt-Mirror: Package \"%s\" updated\n\n" % str(self.title))
file.write("Changes:\n")
for i in range(len(self.logdata)):
type = self.logdata[i][0].strip()
msg = self.logdata[i][1].strip()
if msg.startswith("<"):
# Remove HTML tags
msg = msg[msg.find(">") + 1 : msg.rfind("<")].strip()
file.write("%s: %s\n" % (type, msg))
file.write("\n")
file.write("Generated at %s" % asctime(localtime(time())))
file.close()
system("cat %s | %s" % (filename, cmd))
unlink(filename)
self.logfile.close()
# Check whether a local directory exists & create, if not
def CheckDirectory(self, directory):
if (directory[-1:] == "/"):
directory = directory[:-1]
if (os.path.exists(directory)) or (os.path.islink(directory)):
if (os.path.islink(directory)) or (os.path.isfile(directory)):
if self.firstchange: self.dofirstchangeaction()
unlink(directory)
if not os.path.exists(directory):
pwd = getcwd()
splitted = split(directory, "/")
path = "/"
for i in range(len(splitted)):
if (os.path.exists(path + splitted[i])) and (os.path.isfile(path + splitted[i])):
if self.firstchange: self.dofirstchangeaction()
unlink(path + splitted[i])
if (not os.path.exists(path + splitted[i])) and (not os.path.islink(path + splitted[i])):
try:
chdir(path)
except:
self.log4py.error("[ %s ] Error: couldn't change to directory %s" % (str(self.title), directory))
exit(1)
try:
if self.firstchange: self.dofirstchangeaction()
mkdir(splitted[i])
if (self.owner != None):
chown(splitted[i], self.owner, self.group)
except:
self.log4py.error("[ %s ] Error: Couldn't create directory %s" % (str(self.title), directory))
exit(1)
self.nrdirscreated = self.nrdirscreated + 1
path = path + splitted[i] + "/"
chdir(pwd)
# Create the correct linktargetname and path from (name, path, target)
def AdoptLinkTarget(self, name, path, target):
if (target[0] != "/"): # it's a relative link -> adopt the linktarget
newpath = path
newname = target
if newpath[-1:] != "/":
newpath = newpath + "/"
if (newname[0:3] == "../"):
newpath = newpath[:-1]
while (newname[0:3] == "../"):
newname = newname[3:]
newpath = newpath[0:rfind(newpath, "/")]
newpath = newpath + "/"
if (rfind(newname, "/") != -1):
newpath = newpath + newname[:rfind(newname, "/") + 1]
newname = newname[rfind(newname, "/") + 1:]
while (find(newname, "/") != -1):
newpath = newpath + newname[:find(newname, "/") + 1]
newname = newname[find(newname, "/") + 1:]
else: # the linktarget is a absolut link
newpath = target
newname = ""
return (newname, newpath)
# Create a symbolic link
def CreateLink(self):
self.currentfile.linktarget = sub("//", "/", self.currentfile.linktarget)
createlink = TRUE # Default: create link
append = TRUE # Append file to the list (only if followsymlink is true)
newFile = File()
newFile.log4py.set_loglevel(self.log4py.get_loglevel())
newFile.filetype = FTLinkTarget
# Check the Linktarget and modify name & path if necessary
newFile.name, newFile.path = self.AdoptLinkTarget(self.currentfile.name, self.currentfile.path, self.currentfile.linktarget)
newFile.connection = self.currentfile.connection
# Check Include / Exclude rules
if (self.parentformat == PFNone) and ((len(newFile.path) < len(self.path)) or (newFile.path[:len(self.path)] != self.path)):
self.log4py.warn("[ %s ] Warning: Link %s%s is outside the parent directory !" % (str(self.title), newFile.path, newFile.name))
createlink = FALSE
else:
# Include rules
if (self.include != None) and ((self.include.match(newFile.name) == None) and (self.include.match(newFile.path + newFile.name) == None)):
createlink = FALSE
# Exclude rules
if ((self.exclude != None) and ((self.exclude.match(newFile.name) != None) or self.exclude.match(newFile.path + newFile.name) != None)):
createlink = FALSE
# Ignore rules
if ((self.ignore != None) and ((self.ignore.match(newFile.name) != None) or self.ignore.match(newFile.path + newFile.name) != None)):
createlink = FALSE
# Ignore 'completed' files if using them
if ((self.completed_file != None) and (self.completed_file == newFile.name)):
createlink = FALSE
if ((self.currentfile.path + self.currentfile.name) == (newFile.path + newFile.name)): # Check wether the link points to itself (endless loop)
self.log4py.warn("[ %s ] Warning: Link %s%s points to itself !" % (str(self.title), self.currentfile.path, self.currentfile.name))
append = FALSE
if (newFile.name == "."): # Check wether the link points to the current directory (endless loop)
self.log4py.warn("[ %s ] Warning: Link %s%s points to . (current directory) !" % (str(self.title), self.currentfile.path, self.currentfile.name))
createlink = FALSE
if (self.followsymlinks == TRUE) and (createlink == TRUE) and (append == TRUE): # Don't append the file if you don't want to follow links
self.filelist.append(newFile)
pwd = getcwd()
directory = self.GetLCwd()
chdir(directory)
deletelink = FALSE
if os.path.islink(self.currentfile.name): # check wether the link already exists
linktarget = readlink(self.currentfile.name)
if (linktarget != self.currentfile.linktarget): # if the linktarget has changed -> delete the link
deletelink = TRUE
if (createlink == FALSE): # if the link exists, but shouldn't -> delete it
deletelink = TRUE # This makes the mirror clean (hopefully without dead links)
elif os.path.exists(self.currentfile.name): # this is, if a directory or file has become a link ... delete the old stuff then
deletelink = TRUE
if (os.path.islink(self.currentfile.name) and (createlink == FALSE)) or (deletelink == TRUE): # The link exists, but it shouldn't
self.log4py.info("[ %s ] Deleting Link / File / Directory (FTP): %s%s " % (str(self.title), self.currentfile.path, self.currentfile.name))
if self.firstchange: self.dofirstchangeaction()
stats = rm(self.currentfile.name)
self.AppendLog([LMDelete, self.currentfile.path + self.currentfile.name])
self.nrdirsdel = self.nrdirsdel + stats[0]
self.nrfilesdel = self.nrfilesdel + stats[1]
self.nrbytesdel = self.nrbytesdel + stats[2]
self.nrlinksdel = self.nrlinksdel + stats[3]
if (createlink == TRUE) and (not os.path.islink(self.currentfile.name)):
self.log4py.info("[ %s ] Creating Link (FTP): %s%s -> %s" % (str(self.title), self.currentfile.path, self.currentfile.name, self.currentfile.linktarget))
if self.firstchange: self.dofirstchangeaction()
try:
symlink(self.currentfile.linktarget, self.currentfile.name)
except os.error, detail:
self.log4py.warn("[ %s ] Warning: couldn't create link %s%s -> %s: %s" % (
str(self.title), self.currentfile.path, self.currentfile.name, self.currentfile.linktarget, detail))
self.AppendLog([LMLink, self.currentfile.path + self.currentfile.name + " -> " + self.currentfile.linktarget])
self.nrlinkscreated = self.nrlinkscreated + 1
chdir(pwd)
# Get Remote Files
def GetRemoteFiles(self):
self.log4py.debug("[ %s (debug) ] Getting list of remote files" % str(self.title))
files2download = self.remotefiles
for i in range(len(files2download)):
remotefile = files2download[i]
download = FALSE
if (remotefile.filetype == FTDirectory):
if (self.recursive == TRUE):
download = TRUE
elif (remotefile.filetype == FTRegularFile) or (remotefile.filetype == FTLink):
download = TRUE
if (download == TRUE):
# Insert file at the beginning to reduce memory usage
if (remotefile.filetype == FTLink) or (remotefile.filetype == FTRegularFile):
self.filelist.insert(0, remotefile)
else:
self.filelist.append(remotefile)
# Delete a file or subtree
def DeleteFile(self, path, directory, filename, reason = ""):
# Do not delete which are being ignored !
if (self.ignore != None) and ((self.ignore.match(filename) != None) or (self.ignore.match(directory + filename) != None)):
return
# Ignore 'completed' files if using them
elif ((self.completed_file != None) and (self.completed_file == filename)):
return
else:
if (self.maxdelete > -1):
if (self.nrbytesdel < self.maxdelete):
if (self.test == FALSE):
self.log4py.info("[ %s ] Deleting: %s%s (max %s Byte) %s" % (str(self.title), path, filename, lng2str(self.maxdelete), reason))
else:
self.log4py.info("[ %s (Test) ] Deleting: %s%s (max %s Byte) %s" % (str(self.title), path, filename, lng2str(self.maxdelete), reason))
else:
if (self.test == FALSE):
self.log4py.info("[ %s ] Deleting: %s%s %s" % (str(self.title), path, filename, reason))
else:
self.log4py.info("[ %s (Test) ] Deleting: %s%s %s" % (str(self.title), path, filename, reason))
try:
if (self.nrbytesdel < self.maxdelete) or (self.maxdelete <= -1):
if (self.test == FALSE):
if self.firstchange: self.dofirstchangeaction()
stats = rm(directory + filename, self.maxdelete, self.nrbytesdel)
else:
stats = [0, 0, 0, 0]
else:
stats = [0, 0, 0, 0]
except OSError, detail:
self.HandleError(detail, ESDeleteFile, extra = "filename / directory: \"%s/%s\")" % (directory, filename))
return FALSE
self.AppendLog([LMDelete + " " + reason, path + filename])
self.nrdirsdel = self.nrdirsdel + stats[0]
self.nrfilesdel = self.nrfilesdel + stats[1]
self.nrbytesdel = self.nrbytesdel + stats[2]
self.nrlinksdel = self.nrlinksdel + stats[3]
# Delete local files, which don't exist on the remote site any more
def DelOldLocalFiles(self):
self.log4py.debug("[ %s (debug) ] Deleting old local files" % str(self.title))
directory = self.GetLCwd()
if (self.maxage != 0):
maxmdtm = long(time() - (self.maxage * 60 * 60 * 24))
localfilenames = [] # Remove files which are older than maxage days
for i in range(len(self.localfiles)):
if (self.maxage != 0):
filename = directory + "/" + self.localfiles[i]
if (os.path.isfile(filename) and (not os.path.islink(filename))):
if (lstat(filename)[8] < maxmdtm):
self.DeleteFile(self.currentfile.path, directory, self.localfiles[i], "(file older than " + str(self.maxage) + " day(s))")
else:
localfilenames.append(self.localfiles[i])
else:
localfilenames.append(self.localfiles[i])
else:
localfilenames.append(self.localfiles[i])
remotefilenames = []
for i in range(len(self.remotefiles)):
if (self.remotefiles[i].filetype == FTDirectory):
remotefilenames.append(self.remotefiles[i].path[rfind(self.remotefiles[i].path, "/", 0, -2) + 1:][:-1])
else:
remotefilenames.append(self.remotefiles[i].name)
remotefilenames.sort()
for i in range(len(localfilenames)):
filename = localfilenames[i]
if (not filename in remotefilenames):
if (not (self.IsTempFilename(filename) and (self.RealFilename(filename) in remotefilenames))) or (self.continueftp == FALSE):
if (self.include == None) or (self.deletelocal == TRUE) or ((self.include != None) and ((self.include.match(filename) != None) or (self.include.match(self.currentfile.path + filename) != None))):
self.DeleteFile(self.currentfile.path, directory, filename)
# Get the local filelist
def GetLocalFileList(self):
self.log4py.debug("[ %s (debug) ] Getting list of local files" % (str(self.title)))
directory = self.GetLCwd()
try:
self.localfiles = listdir(directory)
except OSError, detail:
self.localfiles = []
self.HandleError(detail, ESGetLocalFileList, extra = "listdir(\"%s\")" % directory)
# Callback for listing files
def LsCallBack(self, line):
if ((line != "") and (lower(line) != "ls: .: permission denied") and (line[0] in [ "-", "d", "l", "+" ]) or (MSDate.match(line))):
newfile = File()
newfile.log4py.set_loglevel(self.log4py.get_loglevel())
newfile.SplitFileInfo(line, self.title)
if (newfile.name != ".") and (newfile.name != ".."):
self.remotefiles.append(newfile)
# Get the remote filelist
def GetRemoteFileList(self, directory = "", cache = TRUE, exclude = TRUE):
if (directory == ""):
directory = self.currentfile.path
if self.filelistcache.has_key(directory) and (cache == TRUE):
self.log4py.info("[ %s ] Get Filelist: %s (cached)" %(str(self.title), directory))
self.remotefiles = copy.copy(self.filelistcache[directory])
else:
self.log4py.info("[ %s ] Get Filelist: %s" % (str(self.title), directory))
if (exclude == FALSE):
tempinclude = self.include
tempexclude = self.exclude
self.include = self.exclude = None
self.remotefiles = []
# Change to the directory and get listing
if (self.noexception == TRUE):
self.alarmOn(ESGetRemoteFileListCwd)
self.ftp.cwd(directory)
self.ftp.retrlines("LIST -la", self.LsCallBack)
self.alarmOff()
else:
try:
self.alarmOn(ESGetRemoteFileListCwd)
self.ftp.cwd(directory)
self.ftp.retrlines("LIST -la", self.LsCallBack)
self.alarmOff()
except (error_reply, error_temp, error_perm, error_proto, socket.error, IOError), detail:
self.alarmOff()
self.HandleError(detail, ESGetRemoteFileListLS, extra = "ftp.dir(\"%s\")" % directory)
return "Exception"
except EOFError, detail:
self.alarmOff()
self.HandleError("901 ftplib internal: connection lost", ESGetRemoteFileListLS, extra = "ftp.dir(\"%s\")" % directory)
return "Exception"
for i in range(len(self.remotefiles)): # Fixup name and path of all files
if (self.remotefiles[i].filetype == FTDirectory):
self.remotefiles[i].path = directory + self.remotefiles[i].name + "/"
self.remotefiles[i].name = ""
else:
self.remotefiles[i].path = directory
self.remotefiles[i].connection = self.currentfile.connection
newremotefiles = [] # Remove files, which don't match include or exclude
for i in range(len(self.remotefiles)):
remove = FALSE
if (self.remotefiles[i].filetype == FTDirectory):
fullname = self.remotefiles[i].path[:-1]
filename = fullname[rfind(fullname, "/") + 1:]
else:
filename = self.remotefiles[i].name
fullname = directory + filename
# Rules for including files
if (self.include != None) and ((self.include.match(filename) == None) and (self.include.match(fullname) == None)):
remove = TRUE
# Rules for excluding files
if (self.exclude != None) and ((self.exclude.match(filename) != None) or (self.exclude.match(fullname) != None)):
remove = TRUE
# Rules for ignoreing files
if (self.ignore != None) and ((self.ignore.match(filename) != None) or (self.ignore.match(fullname) != None)):
remove = TRUE
# Ignore 'completed' file if using them
if ((self.completed_file != None) and (self.completed_file == filename)):
remove = TRUE
if (remove == FALSE):
newremotefiles.append(self.remotefiles[i])
self.remotefiles = newremotefiles
if (exclude == FALSE):
self.include = tempinclude
self.exclude = tempexclude
if (cache == TRUE):
self.filelistcache[directory] = copy.copy(self.remotefiles)
def ConnectFTP(self):
""" Connect to a FTP site. """
if self.connection.realhostname != "":
hostname = self.connection.realhostname
else:
hostname = self.connection.hostname
self.log4py.info("[ %s ] Connecting (FTP): %s port %d" % (str(self.title), hostname, self.connection.port))
nrretries = 1
errormessage = None
successful = FALSE
if (self.retries == 0):
unlimited = TRUE
else:
unlimited = FALSE
while (((nrretries <= self.retries) or (unlimited == TRUE)) and (successful == FALSE)):
if (self.proxy == None):
if (nrretries == 1):
self.log4py.info("[ %s ] FTP login: first try (out of %d)" % (str(self.title), self.retries))
else:
self.log4py.info("[ %s ] FTP login: retry #%d (out of %d) [%s]" % (str(self.title), nrretries - 1, self.retries, str(errormessage)))
elif (self.proxy != None):
if (nrretries == 1):
self.log4py.info("[ %s ] FTP login (proxy: %s): first try (out of %d)" % (str(self.title), self.proxy, self.retries))
else:
self.log4py.info("[ %s ] FTP login (proxy: %s): retry #%d (out of %d) [%s]" % (str(self.title), self.proxy, nrretries - 1, self.retries, str(errormessage)))
try:
# Connect via a Proxy host
if (self.proxy != None):
proxyhost, proxyport = split(self.proxy, ":")
self.alarmOn(ESConnect)
self.ftp.connect(proxyhost, atoi(proxyport))
self.ftp.login("%s@%s:%d" % (self.connection.username, hostname, self.connection.port), self.connection.password)
self.alarmOff()
successful = TRUE
# Connect directly
else:
self.alarmOn(ESConnect)
self.ftp.connect(hostname, self.connection.port)
self.ftp.login(self.connection.username, self.connection.password)
self.alarmOff()
successful = TRUE
except all_errors, detail:
if (type(detail) == ListType):
self.log4py.debug("[ %s (Debug) ] FTP login failed: %s - %s" % (str(self.title), detail[0], detail[1]))
else:
self.log4py.debug("[ %s (Debug) ] FTP login failed: %s" % (str(self.title), detail))
nrretries = nrretries + 1
sleep(self.delay)
errormessage = detail
if (successful != TRUE):
self.log4py.warn("[ %s ] FTP login failed after %d tries (%s)" % (str(self.title), self.retries, str(errormessage)))
self.AppendLog([LMError, "Couldn't login to \"%s\" after %d tries (%s)" % (self.connection.hostname, self.retries, str(errormessage))])
else:
self.alarmOn(ESConnect)
if (self.passivemode == TRUE):
self.log4py.info("[ %s ] FTP: Setting passive mode" % str(self.title))
self.ftp.set_pasv(1)
else:
self.ftp.set_pasv(0)
self.alarmOff()
errormessage = None
self.log4py.info("[ %s ] FTP login succeeded" % str(self.title))
self.neverloggedin = FALSE # Have now successfully logged in at least once
return errormessage
# Disconnect from a FTP-site
def DisconnectFTP(self):
try:
self.alarmOn(ESDisconnect)
self.ftp.quit()
self.alarmOff()
except:
self.alarmOff()
pass # no need to worry when disconnection failed
# Reconnect from/to a FTP-site
def ReconnectFTP(self):
try:
self.alarmOn(ESDisconnect)
self.ftp.quit()
self.alarmOff()
self.log4py.info("[ %s ] FTP Reconnect: disconnect succeeded" % str(self.title))
except:
self.log4py.info("[ %s ] FTP Reconnect: disconnect failed" % str(self.title))
err = self.ConnectFTP()
if (err != None):
self.log4py.error("[ %s ] FTP Reconnect: failed" % str(self.title))
return FALSE
# Retry the current file
self.filelist.insert(0, self.currentfile)
return TRUE
def CloseConnection(self):
""" Close a connection (ftp, http, rsync). """
if (self.connection != None) and (self.connection.status == CSConnected):
if (self.connection.protocol == CTFTP):
self.DisconnectFTP()
elif (self.connection.protocol == CTHTTP) or (self.connection.protocol == CTRSYNC):
# Nothing to do with external tools
pass
self.connection.status = CSDisconnected
def OpenConnection(self):
""" Open a connection (ftp, http, rsync). """
if (self.connection.protocol == CTFTP):
err = self.ConnectFTP()
elif (self.connection.protocol == CTHTTP) or (self.connection.protocol == CTRSYNC):
# Nothing to do with external tools
err = None
if (err == None):
self.connection.status = CSConnected
return TRUE
else:
return FALSE
# Callback for downloading a file
def RetrCallBack(self, line):
self.downloadfile.write(line)
# Get the modification time of a remote file
def GetRemoteMDTM(self, file):
try:
self.alarmOn(ESDownloadFile)
modtime = split(self.ftp.voidcmd("MDTM " + file.path + file.name), " ")[1]
self.alarmOff()
if (modtime[0:3] == "191"): # Fix for old WU-FTP Daemons (y2k bug)
modtime = "2000%s" % (modtime[5:])
year = atoi(modtime[0:4])
month = atoi(modtime[4:6])
day = atoi(modtime[6:8])
hours = atoi(modtime[8:10])
minutes = atoi(modtime[10:12])
seconds = atoi(modtime[12:14])
file.modtime = int(mktime((year, month, day, hours, minutes, seconds, 0, 0, -1)))
except:
file.modtime = -1
# Returns the download filename for a given filename
def TempFilename(self, filename):
return "%s/.%s.tmp" % (os.path.dirname(filename), os.path.basename(filename))
# Checks wether a filename is a tempfile or not
def IsTempFilename(self, filename):
tmpfilename = os.path.basename(filename)
if len(tmpfilename) >= 5:
if (tmpfilename[0] == ".") and (tmpfilename[-4:] == ".tmp"):
return TRUE
else:
return FALSE
else:
return FALSE
# Returns the real filename of a tempfile
def RealFilename(self, filename):
if len(filename) >= 5:
return filename[1:-4]
else:
return filename
# The real file-downloading function (FTP)
def DownloadFileFTP(self, localfilepath, filename):
rest = None
tmpfilename = self.TempFilename(filename)
if os.path.exists(tmpfilename):
if (self.continueftp == TRUE):
size = lstat(tmpfilename)[6] # determine file size
if (self.test == FALSE):
self.log4py.info("[ %s ] Restarting at position %.0f (FTP): %s%s" % (str(self.title), size, self.currentfile.path, self.currentfile.name))
self.downloadfile = open(tmpfilename, "a")
rest = size
else:
self.log4py.info("[ %s (Test) ] Restarting at position %.0f (FTP): %s%s" % (str(self.title), size, self.currentfile.path, self.currentfile.name))
else:
if (self.test == FALSE):
unlink(tmpfilename)
self.downloadfile = open(tmpfilename, "w")
else:
if (self.test == FALSE):
self.downloadfile = open(tmpfilename, "w")
if (self.test == FALSE):
# Timeout alarm set in DownloadFile
self.ftp.retrbinary("RETR %s%s" % (self.currentfile.path, self.currentfile.name), self.RetrCallBack, 1024, rest)
self.downloadfile.close()
if (self.nomodcheck == FALSE):
if (self.currentfile.modtime != -1):
modtime = self.currentfile.modtime
self.GetRemoteMDTM(self.currentfile)
if (modtime != self.currentfile.modtime):
self.log4py.warn("[ %s ] MDTM changed during transfer (%s -> %s) for :%s%s" % (self.title, modtime, self.currentfile.modtime, self.currentfile.path, self.currentfile.name))
# Set it back incase we continue
self.currentfile.modtime = modtime
raise IOError, "955 remote file modified during transfer"
else:
self.log4py.warn("[ %s ] MDTM not supported. Can't do file modification check on: %s%s" % (self.title, self.currentfile.path, self.currentfile.name))
rename(tmpfilename, filename)
if (self.owner != None):
chown(filename, self.owner, self.group)
if (self.currentfile.modtime != -1):
utime(localfilepath + self.currentfile.name, (self.currentfile.modtime, self.currentfile.modtime))
if (self.umask == None):
chmod(localfilepath + self.currentfile.name, self.currentfile.permissions)
self.AppendLog([LMRetrieve, "%s%s" % (self.currentfile.path, self.currentfile.name)])
self.nrbytesretr = self.nrbytesretr + self.currentfile.size
self.nrfilesretr = self.nrfilesretr + 1
self.installer.checkfile(self.currentfile.path, self.currentfile.name, filename)
# Download a file
def DownloadFile(self):
localfilepath = self.GetLCwd()
downloadfile = FALSE
setmdtm = FALSE
if (self.currentfile.modtime == 0):
self.GetRemoteMDTM(self.currentfile)
if (not (os.path.exists(localfilepath + self.currentfile.name))):
downloadfile = TRUE
else:
localinfo = lstat(localfilepath + self.currentfile.name)
if ((self.currentfile.modtime != int(localinfo[8]) and (self.currentfile.modtime != -1) and (self.ignoremdtm == FALSE))) or (self.currentfile.size != localinfo[6]):
downloadfile = TRUE
if self.deletefirst:
if self.firstchange: self.dofirstchangeaction()
rm(localfilepath + self.currentfile.name)
elif ((self.ignoremdtm == TRUE) and (self.currentfile.modtime != -1) and (self.currentfile.modtime != int(localinfo[8]))):
setmdtm = TRUE
# If local files older than maxage are been deleted then there's no need to download those
if (downloadfile == TRUE) and (self.currentfile.modtime != -1) and (self.maxage != 0):
maxmdtm = long(time() - (self.maxage * 60 * 60 * 24))
if (self.currentfile.modtime < maxmdtm):
downloadfile = FALSE
# Don't download files newer than maxdays
if (downloadfile == TRUE) and (self.currentfile.modtime != -1) and (self.maxdays != 0):
newmdtm = long(time() - (self.maxdays * 60 * 60 * 24))
if (self.currentfile.modtime < newmdtm):
downloadfile = FALSE
if (downloadfile == TRUE):
if self.firstchange: self.dofirstchangeaction()
if (self.test == FALSE):
self.log4py.info("[ %s ] Downloading (FTP): %s%s" % (str(self.title), self.currentfile.path, self.currentfile.name))
else:
self.log4py.info("[ %s (Test) ] Downloading (FTP): %s%s" % (str(self.title), self.currentfile.path, self.currentfile.name))
filename = "%s%s" % (self.GetLCwd(), self.currentfile.name)
if (self.noexception == TRUE):
self.alarmOn(ESDownloadFile)
self.DownloadFileFTP(localfilepath, filename)
self.alarmOff()
else:
try:
self.alarmOn(ESDownloadFile)
self.DownloadFileFTP(localfilepath, filename)
self.alarmOff()
except (error_reply, error_temp, error_perm, error_proto, socket.error, IOError), detail:
self.alarmOff()
self.HandleError(detail, ESDownloadFile)
except EOFError, detail:
self.alarmOff()
self.HandleError("901 ftplib internal: connection lost", ESDownloadFile)
elif ((setmdtm == TRUE) and (self.currentfile.modtime != -1)):
if (self.test == FALSE):
self.log4py.info("[ %s ] Setting MDTM (FTP): %s%s" % (str(self.title), self.currentfile.path, self.currentfile.name))
else:
self.log4py.info("[ %s (Test)] Setting MDTM (FTP): %s%s" % (str(self.title), self.currentfile.path, self.currentfile.name))
try:
if (self.test == FALSE):
if self.firstchange: self.dofirstchangeaction()
utime(localfilepath + self.currentfile.name, (self.currentfile.modtime, self.currentfile.modtime))
except all_errors, detail:
self.HandleError(detail, ESDownloadFile)
# Optimizes the remote filelist, so that only the latest version will be retrieved
def ReduceRemoteFileList(self):
allremotefiles = []
# Check the remote filelist for files matching each only-latest regexp
for i in range(len(self.onlylatest)):
regexpremotefiles = []
for j in range(len(self.remotefiles)):
if ((self.onlylatest[i].match(self.remotefiles[j].name) != None) or (self.onlylatest[i].match(self.remotefiles[j].path + self.remotefiles[j].name))) and (self.remotefiles[j].filetype != FTDirectory):
regexpremotefiles.append(self.remotefiles[j])
if (len(regexpremotefiles) > 1):
mostrecentfile = File()
for j in range(len(regexpremotefiles)):
if (regexpremotefiles[j].modtime == 0):
self.GetRemoteMDTM(regexpremotefiles[j])
if (regexpremotefiles[j].modtime > mostrecentfile.modtime):
mostrecentfile = regexpremotefiles[j]
allremotefiles.append(mostrecentfile)
elif (len(regexpremotefiles) == 1):
allremotefiles.append(regexpremotefiles[0])
# Append all directories and files which do not match _any_ only-latest regexp
for i in range(len(self.remotefiles)):
if (self.remotefiles[i].filetype == FTDirectory):
allremotefiles.append(self.remotefiles[i])
else:
append = TRUE
for j in range(len(self.onlylatest)):
if ((self.onlylatest[j].match(self.remotefiles[i].name)) != None) or (self.onlylatest[j].match(self.remotefiles[i].path + self.remotefiles[i].name) != None):
append = FALSE
if (append == TRUE):
allremotefiles.append(self.remotefiles[i])
self.remotefiles = allremotefiles
# Get the filetype of the first file (the one specified on the command line)
def GetFileType(self, cache = TRUE, exclude = TRUE, firstURL = FALSE):
self.currentfile.filetype = None
if (self.currentfile.name == "") and (self.currentfile.path == "/"):
self.currentfile.filetype = FTDirectory
else:
if (self.currentfile.name == ""):
if (self.currentfile.path[-1] == "/") and (firstURL == TRUE):
try:
self.alarmOn(ESGetRemoteFileListCwd)
self.ftp.cwd(self.currentfile.path)
self.alarmOff()
self.currentfile.filetype = FTDirectory
except:
directory = self.currentfile.path[:rfind(self.currentfile.path, "/", 0, -2)] + "/"
else:
directory = self.currentfile.path[:rfind(self.currentfile.path, "/", 0, -2)] + "/"
else:
directory = self.currentfile.path
if (self.currentfile.filetype == None):
err = self.GetRemoteFileList(directory, cache, exclude)
if (err == None):
for i in range(len(self.remotefiles)):
if ((self.remotefiles[i].path + self.remotefiles[i].name) == (self.currentfile.path + self.currentfile.name)):
self.currentfile = copy.copy(self.remotefiles[i])
# This second check is necessary, because if the path = /pub2/ and pub2 is a link, it's not found
# with the first check !
if (self.currentfile.filetype == None) or (self.currentfile.filetype == FTLinkTarget):
for i in range(len(self.remotefiles)):
if ((self.remotefiles[i].path + self.remotefiles[i].name) + "/" == (self.currentfile.path + self.currentfile.name)):
self.currentfile = copy.copy(self.remotefiles[i])
# The third check is, if /dir/dir is a directory (no trailing /)
if (self.currentfile.filetype == None) or (self.currentfile.filetype == FTLinkTarget):
for i in range(len(self.remotefiles)):
if ((self.remotefiles[i].path + self.remotefiles[i].name) == (self.currentfile.path + self.currentfile.name + "/")):
self.currentfile = copy.copy(self.remotefiles[i])
else:
return err
# Split a URL
def SplitURL(self):
file = File()
file.log4py.set_loglevel(self.log4py.get_loglevel())
self.protocol, username, password, self.hostname, self.port, self.path, params, query, fragment = urlparse(self.url)
if (self.username == ""):
self.username = username
if (self.password == ""):
self.password = password
if (lower(self.protocol) == "ftp") and (self.username == ""):
self.username = DVUsername
self.password = DVPassword
file.path, file.name = os.path.split(self.path)
if (file.path != "/"):
file.path = file.path + "/"
return file
# Check max delete value
def ConvertMaxDelete(self, path):
""" Set self.maxdelete to the maximum size of files, which may be deleted. """
if (self.maxdelete != None):
if (self.maxdelete[-1:] == "%"):
self.log4py.info("[ %s ] Calculating disk usage ... " % str(self.title))
if atoi(self.maxdelete[:-1]) != 0:
diskusage = du(path)
self.maxdelete = divmod(diskusage, atoi(self.maxdelete[:-1]))[0]
self.log4py.info("[ %s ] Disk Usage: %.0f Byte ..." % (str(self.title), diskusage))
else:
self.maxdelete = 0
elif (lower(self.maxdelete[-1:]) == "k"):
self.maxdelete = atoi(self.maxdelete[:-1]) * 1024
elif (lower(self.maxdelete[-1:]) == "m"):
self.maxdelete = atoi(self.maxdelete[:-1]) * 1024 * 1024
else:
self.maxdelete = -1
def GetLCwd(self, hostname = "", path = "", parentdirectory = ""):
""" Returns the local directory corresponding to FULL, PATH or NONE. """
if (hostname == ""):
hostname = self.currentfile.connection.hostname
if (path == ""):
path = self.currentfile.path
if (parentdirectory == ""):
parentdirectory = self.directory
if (self.parentformat == PFFull):
directory = parentdirectory + hostname + path
elif (self.parentformat == PFPath):
directory = parentdirectory + path[1:]
elif (self.parentformat == PFNone):
shortpath = path[len(self.path):]
if (len(shortpath) > 0) and (shortpath[0] == "/"):
shortpath = shortpath[1:]
directory = parentdirectory + shortpath
return directory
def StartMirrorHTTP(self):
""" Start HTTP mirror by invoking wget. """
wgetMirror = wget.wget()
wgetMirror.timestamping = TRUE
wgetMirror.level = 0
wgetMirror.recursive = self.recursive
wgetMirror.tries = self.retries
wgetMirror.username = self.username
wgetMirror.password = self.password
wgetMirror.relative = FALSE
wgetMirror.set_source("http://%s%s" % (self.hostname, self.path))
wgetMirror.set_target(self.GetLCwd(hostname = self.hostname, path = self.path))
self.log4py.info("[ %s ] Executing %s %s" % (str(self.title), wgetMirror.executable(), join(wgetMirror.arguments())))
result = wgetMirror.execute(FALSE)
if (result != 0):
self.log4py.error("[ %s ] WGet failed." % (str(self.title)))
def StartMirrorRSYNC(self):
""" Start RSYNC mirror by invoking rsync. """
rsyncMirror = rsync.rsync()
rsyncMirror.compress = TRUE
rsyncMirror.recursive = self.recursive
rsyncMirror.delete = TRUE
rsyncMirror.links = TRUE
rsyncMirror.times = TRUE
rsyncMirror.archive = TRUE
rsyncMirror.username = self.username
rsyncMirror.password = self.password
rsyncMirror.set_source(self.hostname, self.path)
rsyncMirror.set_target(self.GetLCwd(hostname = self.hostname, path = self.path))
self.log4py.info("[ %s ] Executing %s %s" % (str(self.title), rsyncMirror.executable(), join(rsyncMirror.arguments())))
result = rsyncMirror.execute(FALSE)
if (result != 0):
self.log4py.error("[ %s ] RSync error: %d - %s" % (str(self.title), result, rsync.RERR[result]))
def AnalyzeDifference(self, added, deleted, modified):
""" Analylze difference of two directories. """
for filename in added.keys():
statinfo = added[filename]
self.AppendLog([LMRetrieve, "%s" % filename])
self.nrbytesretr = self.nrbytesretr + statinfo[6]
self.nrfilesretr = self.nrfilesretr + 1
for filename in deleted.keys():
statinfo = deleted[filename]
self.AppendLog([LMDelete, filename])
self.nrfilesdel = self.nrfilesdel + 1
self.nrbytesdel = self.nrbytesdel + statinfo[6]
def MainLoopFTP(self):
""" main loop for handling and downloading FTP data. """
connected = FALSE
pt = time()
while len(self.filelist) > 0:
self.currentfile = self.filelist[0]
self.filelist.remove(self.currentfile)
# Host has changed - open a new connection
if (self.currentfile.connection != self.connection):
self.CloseConnection()
self.connection = self.currentfile.connection
connected = self.OpenConnection()
if (connected):
self.log4py.debug("[ %s (debug) ] Getting file type of %s%s" % (str(self.title), self.currentfile.path, self.currentfile.name))
# Get the filetype of the first URL
self.GetFileType(cache = FALSE, exclude = FALSE, firstURL = TRUE)
while (self.currentfile.filetype == FTLink):
self.log4py.info("[ %s ] Following initial link %s%s" % (str(self.title), self.currentfile.path, self.currentfile.name))
self.currentfile.name, self.currentfile.path = self.AdoptLinkTarget(self.currentfile.name, self.currentfile.path, self.currentfile.linktarget)
self.path = self.currentfile.path + self.currentfile.name
self.remotefiles = []
self.log4py.debug("[ %s (debug) ] Getting file type of %s%s" % (str(self.title), self.currentfile.path, self.currentfile.name))
self.GetFileType(cache = FALSE, exclude = FALSE)
self.CheckDirectory(self.GetLCwd())
else:
if (self.currentfile.connection.protocol == CTFTP):
self.HandleError("953 emirror FTP connect failed", ESConnect)
if (connected):
if (self.currentfile.connection.protocol == CTFTP):
if (self.currentfile.filetype == FTRegularFile):
self.DownloadFile()
elif (self.currentfile.filetype == FTDirectory):
err = self.GetRemoteFileList()
if (err == None):
if (len(self.onlylatest) > 0):
self.ReduceRemoteFileList()
self.CheckDirectory(self.GetLCwd())
self.GetLocalFileList()
self.DelOldLocalFiles()
self.GetRemoteFiles()
elif (self.currentfile.filetype == FTLinkTarget):
self.GetFileType()
if self.currentfile.filetype == None:
self.GetFileType()
self.CheckDirectory(self.GetLCwd())
self.filelist.insert(0, self.currentfile)
elif (self.currentfile.filetype == FTLink):
self.CreateLink()
else:
self.log4py.info("[ %s ] Warning: %s%s not found" % (str(self.title), self.currentfile.path, self.currentfile.name))
self.AppendLog([LMWarning, self.currentfile.path + self.currentfile.name + " not found"])
elif ((self.currentfile.connection.protocol == CTHTTP) or (self.currentfile.connection.protocol == CTRSYNC)):
self.log4py.warn("[ %s ] Sorry, but ECLiPt-Mirror doesn't support HTTP / rsync" % str(self.title))
# Update lock timestamp periodically
ct = time()
if (ct - pt > 600):
self.mlock.refresh()
pt = ct
self.CloseConnection()
# Do some error handling
def HandleError(self, error, source, extra = None):
if (error == None):
if (str(sys.exc_info()[0].__name__) == "EOFError"):
exargs = (902, "EOFError")
else:
exargs = (999, format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))
elif (type(error) == TupleType) or (type(error) == InstanceType):
if (type(error) == InstanceType):
args = error.args
else:
args = error
if (len(args) > 1):
exargs = (int(args[0]), args[1])
elif (len(args) == 1):
s = str(args[0]);
space = s.find(" ");
try:
exargs = (int(s[:space].strip()), s[space:].strip())
except:
exargs = (999, "Unrecognised error: " + str(error));
else:
exargs = (999, "Unrecognised error: " + str(error));
else:
# Assume single string
s = str(error)
space = s.find(" ");
try:
exargs = (int(s[:space].strip()), s[space:].strip())
except:
exargs = (999, "Unrecognised error: " + str(error));
# Process the error
enum = exargs[0]
emsg = exargs[1]
self.log4py.error("[ %s ] Error %d: %s" % (str(self.title), enum, emsg))
for e in errormsgs:
if (enum == e[0]) and (find(lower(emsg), lower(e[1])) != -1):
if (e[2] == EAContinue):
self.AppendLog([LMError, self.currentfile.path + self.currentfile.name +
" (" + source + "): " + str(enum) + "-" + emsg])
return
elif (e[2] == EARetry):
self.log4py.info("[ %s ] Attempting FTP reconnect" % str(self.title))
if self.ReconnectFTP():
return
else:
self.HandleError("954 emirror FTP reconnect failed", ESConnect)
elif (e[2] == EAAbort):
break
else: # EAIgnore
return
#############################################################
##### Abort mirror if EAAbort or unknown internal error #####
#############################################################
self.AppendLog([LMError, self.currentfile.path + self.currentfile.name +
" (" + source + "): " + str(enum) + "-" + emsg])
# Don't do installer stage if internal error or asked to terminate
if (enum != 952) and (enum != 999):
self.installer.finish()
self.WriteLog(1)
# Send error to log file
self.log4py.error("[ %s ] Mirror aborted with the following error:" % str(self.title))
self.log4py.error(" Path: %s" % self.currentfile.path)
self.log4py.error(" Name: %s" % self.currentfile.name)
self.log4py.error(" Source: %s" % str(source))
self.log4py.error(" Error: %s" % str(error))
self.log4py.error(" Type(Error): %s" % str(type(error)))
if (type(error) == InstanceType):
self.log4py.error(" Dictionary: %s" % str(error.__dict__))
self.log4py.error(" Errornumber: %d" % enum)
self.log4py.error(" Errormessage: %s" % emsg)
self.log4py.error(" Extra: %s\n" % str(extra))
if (enum == 999):
# Assume it was an exception
(t, v, tb) = sys.exc_info()
text = traceback.format_exception(t, v, tb)
self.log4py.error("[ %s ] Details of last exception:" % self.title)
for line in text:
self.log4py.error(line.rstrip())
# Send error email if requested
if (self.errormailcmd != None):
filename = mktemp()
file = open(filename, "w")
file.write("%s" % PIHeader)
file.write("\nError Report (%s):\n\n" % str(self.title))
file.write(" Admin EMail: %s\n" % str(self.adminemail))
file.write(" Mirror URL: %s\n" % str(self.url))
file.write(" Path: %s\n" % self.currentfile.path)
file.write(" Name: %s\n" % self.currentfile.name)
file.write(" Source: %s\n" % str(source))
file.write(" Error: %s\n" % str(error))
file.write(" Type(Error): %s\n" % str(type(error)))
if (type(error) == InstanceType):
file.write(" Dictionary: %s\n" % str(error.__dict__))
file.write(" Errornumber: %d\n" % enum)
file.write(" Errormessage: %s\n" % emsg)
file.write(" Extra: %s\n\n" % str(extra))
if (enum == 999):
# Assume it was an exception
file.write("Details of last exception:")
(t, v, tb) = sys.exc_info()
text = traceback.format_exception(t, v, tb)
for ln in text:
file.write(ln)
file.close()
if find(self.errormailcmd, "%s") != -1:
cmd = self.errormailcmd % ("Mirror failed: %s" % self.title)
else:
cmd = self.errormailcmd
self.log4py.info("[ %s ] Executing: %s" % (str(self.title), cmd))
system("cat %s | %s" % (filename, cmd))
unlink(filename)
else:
self.log4py.error("Please report this error to Martin.Preishuber@eclipt.at")
self.log4py.error("or check error.py and adapt it (it's not really difficult ;-))")
if self.locked:
if not self.unlock():
self.log4py.error("[ %s ] Unable to unlock mirror!!" % str(self.title))
else:
self.log4py.info("[ %s ] Released mirror lock" % str(self.title))
sys.exit(1)
#################################
##### Abnormal program exit #####
#################################
# Handler for termination of mirror child process
def SigTermHandler(self, signal_number, stack_frame):
self.log4py.info("[ %s ] Received terminate signal %s. Exiting gracefully..." % (str(self.title), signal_number))
if (self.alarmedoperation != None):
self.HandleError("952 emirror received termination signal", self.alarmedoperation)
else:
self.HandleError("952 emirror received termination signal", "Unknown")
# Handler for alarm signals during potential hanging operations
def SigAlrmHandler(self, signal_number, stack_frame):
self.log4py.info("[ %s ] Received alarm signal %s" % (str(self.title), signal_number))
self.HandleError("951 operation aborted by emirror", self.alarmedoperation)
# Start the mirror
def Start(self):
self.log4py.info("[ %s ] Starting mirror (PID:%s)" % (str(self.title), getpid()))
file = self.SplitURL()
if self.history_logging:
# Change log filename into a directory where logs for this mirror are kept
self.logfilename = self.CheckLogfilename(self.logfilename) # Gets correct extension
self.logext = self.logfilename[self.logfilename.rfind("."):]
if not self.logdirectory.endswith("/"):
self.logdirectory = self.logdirectory + "/"
self.logdirectory = self.logdirectory + self.logfilename[:self.logfilename.rfind(".")] + "_logs/"
self.CheckLogDirectory(self.logdirectory)
self.logfilename = ""
self.StartLog()
if (self.hostname == ""):
self.log4py.error("[ %s ] Invalid URL \"%s\" specified, exiting." % (str(self.title), self.url))
exit(1)
connection = Connection()
connection.protocol = self.protocol
connection.hostname = self.hostname
connection.realhostname = self.realhostname
connection.port = self.port
connection.username = self.username
connection.password = self.password
connection.status = CSDisconnected
signal.signal(signal.SIGALRM, self.SigAlrmHandler)
signal.signal(signal.SIGTERM, self.SigTermHandler)
if (self.lock()):
self.locked = TRUE
if (self.umask != None):
umask(self.umask)
self.log4py.info("[ %s ] Setting umask to %s" % (str(self.title), zfill(oct(self.umask), 3)))
if (connection.protocol == CTFTP):
file.connection = connection
self.filelist.append(file)
self.ConvertMaxDelete(self.GetLCwd(file.connection.hostname, self.path))
self.MainLoopFTP()
elif (connection.protocol == CTHTTP) or (connection.protocol == CTRSYNC):
localDirectory = self.GetLCwd(hostname = self.hostname, path = self.path)
self.CheckDirectory(localDirectory)
oldFileList = lstatfiles(ls(localDirectory, TRUE))
if (connection.protocol == CTHTTP):
self.StartMirrorHTTP()
elif (connection.protocol == CTRSYNC):
self.StartMirrorRSYNC()
newFileList = lstatfiles(ls(localDirectory, TRUE))
added, deleted, modified = cmpdirs(oldFileList, newFileList)
self.log4py.debug("External file modification stats: %s %s %s" % (str(added), str(deleted), str(modified)))
self.AnalyzeDifference(added, deleted, modified)
else:
# Skip the actual mirroring since this mirror is locked
self.locked = FALSE
self.log4py.info("[ %s ] Mirror locked by another process. Marked as 'active'" % str(self.title))
self.dolastchangeaction()
self.installer.finish()
self.WriteLog()
if self.locked:
if not self.unlock():
self.log4py.error("[ %s ] Unable to unlock mirror!!" % str(self.title))
else:
self.log4py.info("[ %s ] Released mirror lock" % str(self.title))
self.log4py.info("[ %s ] Finishing Mirror" % str(self.title))
sys.exit(0)
class MirrorHistory:
def __init__(self, template_file, logdir, lockdir, history = 28, row = 28, purge = TRUE):
self.log4py = log4py.Logger().get_instance(self)
self.log4py.set_loglevel(log4py.LOGLEVEL_ERROR)
self.history = history
self.row = row
self.numcols = row + 6
self.template = []
self.tableloop = []
self.categoryloop = []
self.mirrorloop = []
self.dateloop = []
self.statusloop = []
self.stats = None
self.purge = purge
self.init_template(template_file)
self.output = None
self.logdir = logdir
self.lockdir = lockdir
def init_template(self, template_file):
try:
f = open(template_file)
except:
self.log4py.error("Template file not found: %s" % template_file)
return FALSE
target = self.template
while 1:
line = f.readline()
if not line: break
if (line.strip().lower() == "$tables_start$"):
target.append("$tableloop$")
target = self.tableloop
elif (line.strip().lower() == "$tables_end$"):
target = self.template
elif (line.strip().lower() == "$dates_start$"):
target.append("$dateloop$")
target = self.dateloop
elif (line.strip().lower() == "$dates_end$"):
target = self.tableloop
elif (line.strip().lower() == "$categories_start$"):
target.append("$categoryloop$")
target = self.categoryloop
elif (line.strip().lower() == "$categories_end$"):
target = self.tableloop
elif (line.strip().lower() == "$mirrors_start$"):
target.append("$mirrorloop$")
target = self.mirrorloop
elif (line.strip().lower() == "$mirrors_end$"):
target = self.categoryloop
elif (line.strip().lower() == "$status_start$"):
target.append("$statusloop$")
target = self.statusloop
elif (line.strip().lower() == "$status_end$"):
target = self.mirrorloop
else:
target.append(line)
f.close()
def write_statusrow(self, mirror, times):
cal = self.stats.getmirrorcalendar(mirror, times)
for t in times:
for line in self.statusloop:
if line.find("$status$") != -1:
milist = cal[t]
if len(milist) > 0:
milist.reverse()
slist = []
for mi in milist:
gif = mirrorstats.get_status_gif(mi)
tmp = mi.path.split("/")
url = "./" + "/".join((tmp[-2], tmp[-1]))
slist.append("<A HREF=\"%s\"><img class=\"status\" src=\"%s.gif\" alt=\"%s\"></A>" % (url, gif, gif))
self.output.write(line.replace("$status$", "<BR>".join(slist)))
else:
self.output.write(line.replace("$status$", " "))
else:
self.output.write(line)
def fmtstat(self, s):
sum, (min, minpath), mean, (max, maxpath) = s
if minpath:
tmp = minpath.split("/")
minurl = "./" + "/".join((tmp[-2], tmp[-1]))
minstr = "<A HREF=\"%s\">%s</A>" % (minurl, min)
else:
minstr = str(min)
if maxpath:
tmp = maxpath.split("/")
maxurl = "./" + "/".join((tmp[-2], tmp[-1]))
maxstr = "<A HREF=\"%s\">%s</A>" % (maxurl, max)
else:
maxstr = str(max)
return "%s<%s>%s" % (minstr, str(mean), maxstr)
def write_mirrors(self, mirrors, timetup_range, times):
for mirror in mirrors:
for line in self.mirrorloop:
if line.find("$statusloop$") != -1:
self.write_statusrow(mirror, times)
elif line.find("$title$") != -1:
self.output.write(line.replace("$title$", mirror))
elif line.find("$start$") != -1:
s = self.stats.getstatistic(mirror, timetup_range, self.stats.STARTHOUR)
str = self.fmtstat((s[0], s[1], int(s[2]), s[3]))
self.output.write(line.replace("$start$", str))
elif line.find("$end$") != -1:
s = self.stats.getstatistic(mirror, timetup_range, self.stats.ENDHOUR)
str = self.fmtstat((s[0], s[1], int(s[2]), s[3]))
self.output.write(line.replace("$end$", str))
elif line.find("$duration$") != -1:
s = self.stats.getstatistic(mirror, timetup_range, self.stats.DURATION)
str = self.fmtstat((s[0],
(mirrorstats.fmtdur(s[1][0]), s[1][1]),
mirrorstats.fmtdur(s[2]),
(mirrorstats.fmtdur(s[3][0]), s[3][1])))
self.output.write(line.replace("$duration$", str))
elif line.find("$size$") != -1:
s = self.stats.getstatistic(mirror, timetup_range, self.stats.SIZE)
str = self.fmtstat((s[0],
(mirrorstats.fmtnum(s[1][0]), s[1][1]),
mirrorstats.fmtnum(s[2]),
(mirrorstats.fmtnum(s[3][0]), s[3][1])))
self.output.write(line.replace("$size$", str))
elif line.find("$rate$") != -1:
s = self.stats.getstatistic(mirror, timetup_range, self.stats.TXRATE)
str = self.fmtstat((s[0],
(mirrorstats.fmtnum(s[1][0]), s[1][1]),
mirrorstats.fmtnum(s[2]),
(mirrorstats.fmtnum(s[3][0]), s[3][1])))
self.output.write(line.replace("$rate$", str))
else:
self.output.write(line)
def write_categories(self, categories, timetup_range, times):
for cat in categories:
for line in self.categoryloop:
if line.find("$mirrorloop$") != -1:
mirrors = self.stats.getmirrorsforcategory(cat)
if not mirrors:
self.log4py.warn("No mirrors in category %s!" % cat)
continue
mirrors.sort()
self.write_mirrors(mirrors, timetup_range, times)
elif line.find("$category$") != -1:
if line.find("$colspan$") != -1:
line = line.replace("$colspan$", str(self.numcols))
self.output.write(line.replace("$category$", cat))
else:
self.output.write(line)
def write_datesrow(self, times):
for t in times:
date = strftime("%a", t)[0] + strftime("<BR>%d<BR>%m<BR>%y", t)
for line in self.dateloop:
if line.find("$date$") != -1:
self.output.write(line.replace("$date$", date))
else:
self.output.write(line)
def write_table(self, timetup_range):
times = self.stats.getdaytimes(timetup_range)
times.reverse()
for line in self.tableloop:
if line.find("$dateloop$") != -1:
self.write_datesrow(times)
elif line.find("$categoryloop$") != -1:
categories = self.stats.getcategories()
if not categories:
self.log4py.warn("No categories found!")
continue
categories.sort()
self.write_categories(categories, timetup_range, times)
else:
self.output.write(line)
def write(self, output):
self.output = output
if self.stats == None:
self.stats = mirrorstats.MirrorStats()
self.stats.log4py.set_loglevel(self.log4py.get_loglevel())
self.stats.update(self.logdir, self.lockdir)
for line in self.template:
if line.find("$tableloop$") != -1:
start = localtime(time())
for daysback in range(0, self.history, self.row):
ttr = self.stats.gettimetup_range(start, self.row - 1)
self.write_table(ttr)
start = self.stats.gettimetup_range(ttr[0], 1)[0]
elif line.find("$date$") != -1:
self.output.write(line.replace("$date$", mirrorstats.fmttime(time())))
else:
self.output.write(line)
if self.purge: self.stats.purgelogs(self.history)
# ----------------------------------------------------------------------
# Definition of the Indexfile-class
class IndexFile:
def __init__(self):
self.directory = None # Directory where to find the logs
self.mirrorlogs = [] # List of Logs-files
self.mirrorstatus = {} # Status of mirrors
self.exitstatus = {} # Exit status of mirrors
self.mirrorcategory = {} # Category of the mirror
self.categorymirror = {} # Inverse of mirrorcategory
self.description = {} # Description of mirrors
self.timestamp = {} # List of time-stamps for mirrors
self.logfilename = {} # Name of the mirror logfile
self.lastchanged = {} # Last change of the mirror
self.log4py = log4py.Logger().get_instance(self)
self.log4py.set_loglevel(log4py.LOGLEVEL_ERROR)
self.errorcolor = "#CC0033" # Default error color
self.warningcolor = "#009933" # Default color for warnings
self.logtemplate = None # Filename of the log template-file
self.summarytemplate = None # Filename of the summary template-file
self.template = []
self.templatecategory = []
self.templateloop = []
self.templatesummary = []
self.templatesummaryloop = []
self.logfileformat = LFPlain # Format of the indexfile
self.mailusers = "" # Mail info about to changes to some users
self.mailusercmd = "" # Command to send mail to users
self.nrbytesretr = 0L
self.nrfilesretr = 0L
self.nrlinkscreated = 0L
self.nrdirscreated = 0L
self.nrbytesdel = 0L
self.nrfilesdel = 0L
self.nrlinksdel = 0L
self.nrdirsdel = 0L
self.totalseconds = 0L
self.firststarttime = None
self.lastendtime = None
self.proxy = None # Proxy host and port
# Analyze logfiles to get Title, Status & Category of mirrors
def AnalyzeLogs(self):
currentdir = getcwd()
chdir(self.directory)
for i in range(len(self.mirrorlogs)):
datei = open(self.mirrorlogs[i], "r")
title = ""
line = "<empty>"
copyfile = FALSE
while line != "":
line = datei.readline()
stripped = strip(line)
if (stripped[0:4] == "<!--"):
restofline = strip(sub("-->", "", stripped[5:]))
if restofline[0:5] == "TITLE":
title = restofline[7:]
self.logfilename[title] = self.mirrorlogs[i]
self.mirrorstatus[title] = MSunknown
self.description[title] = "None"
self.timestamp[title] = "0"
elif restofline[0:6] == "STATUS":
self.mirrorstatus[title] = restofline[8:]
if restofline[8:][0] in digits:
self.lastchanged[title] = restofline[8:]
copyfile = TRUE
elif restofline[0:10] == "EXITSTATUS":
self.exitstatus[title] = atoi(restofline[12:])
elif restofline[0:8] == "CATEGORY":
category = restofline[10:]
self.categorymirror[title] = category
if self.mirrorcategory.has_key(category):
self.mirrorcategory[category].append(title)
else:
self.mirrorcategory[category] = [title]
elif restofline[0:9] == "TIMESTAMP":
self.timestamp[title] = restofline[11:]
elif restofline[0:11] == "DESCRIPTION":
self.description[title] = restofline[13:]
elif restofline[0:10] == "STATISTICS":
statsline = strip(restofline[12:])
splitted = split(statsline)
for j in range(len(splitted)):
splitted[j] = atol(splitted[j])
currentday = localtime(time())
midnight = mktime((currentday[0], currentday[1], currentday[2], 0, 0, 0, 0, 0, currentday[8]))
if (splitted[8] >= midnight):
self.nrbytesretr = self.nrbytesretr + splitted[0]
self.nrfilesretr = self.nrfilesretr + splitted[1]
self.nrlinkscreated = self.nrlinkscreated + splitted[2]
self.nrdirscreated = self.nrdirscreated + splitted[3]
self.nrbytesdel = self.nrbytesdel + splitted[4]
self.nrfilesdel = self.nrfilesdel + splitted[5]
self.nrlinksdel = self.nrlinksdel + splitted[6]
self.nrdirsdel = self.nrdirsdel + splitted[7]
self.totalseconds = self.totalseconds + (splitted[9] - splitted[8])
if (self.firststarttime == None) or (self.firststarttime > splitted[8]):
self.firststarttime = splitted[8]
if (self.lastendtime == None) or (self.lastendtime < splitted[9]):
self.lastendtime = splitted[9]
datei.close()
if (copyfile == TRUE): # save last logfile, where changes have occured
allfiles = listdir(".")
for j in range(len(allfiles)):
if (match(".*" + self.mirrorlogs[i], allfiles[j]) != None):
if (allfiles[j][0] in digits):
try:
unlink(allfiles[j])
except:
self.log4py.error("[ main ] Index-file: Error while unlinking \"%s\" - Please check the file / directory permissions !" % allfiles[j])
try:
shutil.copy(self.mirrorlogs[i], self.mirrorstatus[title] + "-" + self.mirrorlogs[i])
except:
self.log4py.error("[ main ] Index-file: Error while copying \"%s -> %s\" - Please check the file / directory permissions !" % (self.mirrorlogs[i], self.mirrorstatus[title] + "-" + self.mirrorlogs[i]))
chdir(currentdir)
# Read Template file
def ReadTemplateFile(self):
file = open(self.logtemplate)
line = "<empty>"
self.template = []
self.templatecategory = []
self.templateloop = []
target = self.template
while (line != ""):
line = file.readline()
if (lower(strip(line)) == "$categorystart$"):
target.append("$categoryloop$")
target = self.templatecategory
elif (lower(strip(line)) == "$categoryend$"):
target = self.template
elif (lower(strip(line)) == "$mirrorstart$"):
target.append("$mirrorloop$")
target = self.templateloop
elif (lower(strip(line)) == "$mirrorend$"):
target = self.templatecategory
elif (find(strip(lower(line)), "<!-- errorcolor:") != -1):
color = strip(lower(line))
position = find(color, "<!-- errorcolor:")
color = color[position + len("<!-- errorcolor:"):]
position = find(color, "-->")
color = strip(color[:position])
self.errorcolor = color
elif (find(strip(lower(line)), "<!-- warningcolor:") != -1):
color = strip(lower(line))
position = find(color, "<!-- warningcolor:")
color = color[position + len("<!-- warningcolor:"):]
position = find(color, "-->")
color = strip(color[:position])
self.warningcolor = color
else:
target.append(line)
file.close()
# Read Template file
def ReadSummaryTemplateFile(self, datei, fileupdated, fileerrors):
file = open(self.summarytemplate)
line = "<empty>"
self.summarytemplate = []
self.summarytemplateloop = []
target = self.summarytemplate
while (line != ""):
line = file.readline()
line = sub("\$indexfilename\$", datei.name, line)
line = sub("\$updatedfilename\$", fileupdated.name, line)
line = sub("\$errorfilename\$", fileerrors.name, line)
if (lower(strip(line)) == "$mirrorstart$"):
target.append("$mirrorloop$")
target = self.summarytemplateloop
elif (lower(strip(line)) == "$mirrorend$"):
target = self.summarytemplate
else:
target.append(line)
file.close()
# Create the master index-file
def CreateMasterIndexFile(self, categorylist, duration, downloaded, deleted, downloadrate, datei, categorykeys, fileupdated, fileerrors):
for i in range(len(self.template)):
line = self.template[i]
if (line == "$categoryloop$"): # Add lines for each category
for j in range(len(categorykeys)):
category = categorykeys[j]
mirrorlist = self.mirrorcategory[categorykeys[j]]
mirrorlist.sort()
for k in range(len(self.templatecategory)):
line = self.templatecategory[k]
if (line == "$mirrorloop$"): # Add lines for each mirror
for l in range(len(mirrorlist)):
title = mirrorlist[l]
update = "Unknown"
if (self.logfileformat == LFPlain):
mirrortitle = title
exitstatus = self.exitstatus[title]
status = self.mirrorstatus[title]
if status == MSunknown: status = "in progress"
elif status == MSactive: status = "in progress"
elif status == MSneverloggedin: status = "no login"
elif status == MSunchanged: status = "unchanged"
else: status = "updated"
if (exitstatus == 1):
status = status + " (Aborted!)"
if self.lastchanged.has_key(title):
update = ctime(atof(self.lastchanged[title]))
else:
mirrortitle = "<a href=\"%s\">%s</a>" % (self.logfilename[title], title)
exitstatus = self.exitstatus[title]
status = self.mirrorstatus[title]
c = None
if (status == MSunknown):
s = "in progress"
c = self.warningcolor
elif (status == MSactive):
s = "in progress"
c = self.warningcolor
elif (status == MSneverloggedin):
s = "no login"
c = self.errorcolor
elif (status == MSunchanged):
s = "unchanged"
else:
s = "updated"
c = self.warningcolor
if (exitstatus == 1):
s = s + " (Aborted!)"
c = self.errorcolor
if (c == None):
status = s
else:
status = "<font color=\"%s\">%s</font>" % (c, s)
if self.lastchanged.has_key(title):
update = "<a href=\"%s\">%s</a>" % (self.lastchanged[title] + "-" + self.logfilename[title], ctime(atof(self.lastchanged[title])))
for m in range(len(self.templateloop)):
line = self.templateloop[m]
line = sub("\$status\$", status, line)
line = sub("\$title\$", mirrortitle, line)
line = sub("\$update\$", update, line)
datei.write(line)
else:
line = sub("\$category\$", category, line)
datei.write(line)
else:
line = sub("\$categorylist\$", categorylist, line)
line = sub("\$lastupdate\$", ctime(time()), line)
if (self.firststarttime != None):
line = sub("\$starttime\$", ctime(self.firststarttime), line)
else:
line = sub("\$starttime\$", "Unknown", line)
if (self.lastendtime != None):
line = sub("\$endtime\$", ctime(self.lastendtime), line)
else:
line = sub("\$endtime\$", "Unknown", line)
line = sub("\$duration\$", duration, line)
line = sub("\$downloadstats\$", downloaded, line)
line = sub("\$deletestats\$", deleted, line)
line = sub("\$downloadrate\$", downloadrate, line)
line = sub("\$proxy\$", str(self.proxy), line)
line = sub("\$indexfilename\$", datei.name, line)
line = sub("\$updatedfilename\$", fileupdated.name, line)
line = sub("\$errorfilename\$", fileerrors.name, line)
datei.write(line)
# Create errors & updated file
def CreateSummaryFile(self, fileupdated, fileerrors):
updatedlist = {}
errorlist = {}
for i in range(len(self.mirrorcategory.keys())):
category = self.mirrorcategory.keys()[i]
for j in range(len(self.mirrorcategory[category])):
title = self.mirrorcategory[category][j]
timestamp = self.timestamp[title]
description = self.description[title]
if self.mirrorstatus[title] == MSneverloggedin:
errorlist[timestamp] = title
elif (self.mirrorstatus[title] != MSactive) and (self.mirrorstatus[title] != MSunchanged) and (self.mirrorstatus[title] != MSunknown):
updatedlist[timestamp] = title
for i in range(len(self.summarytemplate)):
line = self.summarytemplate[i]
if (line == "$mirrorloop$"):
# Write updated file information
if len(updatedlist.keys()) == 0:
updatedlist["0"] = "None"
self.categorymirror["None"] = "None"
self.logfilename["None"] = ""
self.description["None"] = "No packages have been updated"
timelist = updatedlist.keys()
timelist.sort()
for j in range(len(timelist)):
timestamp = timelist[j]
title = updatedlist[timestamp]
mirrortitle = title
categorylink = self.categorymirror[title]
if (self.logfileformat == LFHTML) or (self.logfileformat == LFPHP):
mirrortitle = "<a href=\"%s\">%s</a>" % (self.logfilename[title], title)
if (self.logfileformat == LFHTML):
categorylink = "<a href=\"%s\">%s</a>" % ("index" + LEHTML, self.categorymirror[title])
elif (self.logfileformat == LFPHP):
categorylink = "<a href=\"%s#%s\">%s</a>" % ("index" + LEPHP, self.categorymirror[title], self.categorymirror[title])
for k in range(len(self.summarytemplateloop)):
line = self.summarytemplateloop[k]
line = sub("\$update\$", asctime(localtime(atof(timestamp))), line)
line = sub("\$title\$", mirrortitle, line)
line = sub("\$category\$", categorylink, line)
line = sub("\$description\$", self.description[title], line)
fileupdated.write(line)
# Write error file information
if len(errorlist.keys()) == 0:
errorlist["0"] = "None"
self.categorymirror["None"] = "None"
self.logfilename["None"] = ""
self.description["None"] = "No errors have occured"
timelist = errorlist.keys()
timelist.sort()
for j in range(len(timelist)):
timestamp = timelist[j]
title = errorlist[timestamp]
mirrortitle = title
categorylink = self.categorymirror[title]
if (self.logfileformat == LFHTML) or (self.logfileformat == LFPHP):
mirrortitle = "<a href=\"%s\">%s</a>" % (self.logfilename[title], title)
if (self.logfileformat == LFHTML):
categorylink = "<a href=\"%s\">%s</a>" % ("index" + LEHTML, self.categorymirror[title])
elif (self.logfileformat == LFPHP):
categorylink = "<a href=\"%s#%s\">%s</a>" % ("index" + LEPHP, self.categorymirror[title], self.categorymirror[title])
for k in range(len(self.summarytemplateloop)):
line = self.summarytemplateloop[k]
line = sub("\$update\$", asctime(localtime(atof(timestamp))), line)
line = sub("\$title\$", mirrortitle, line)
line = sub("\$category\$", categorylink, line)
line = sub("\$description\$", self.description[title], line)
fileerrors.write(line)
else:
line = sub("\$lastupdate\$", ctime(time()), line)
if find(line, "$summary$") != -1:
lineupdated = sub("\$summary\$", "Updated mirrors", line)
lineerrors = sub("\$summary\$", "Failed mirrors", line)
fileupdated.write(lineupdated)
fileerrors.write(lineerrors)
else:
fileupdated.write(line)
fileerrors.write(line)
# Create the new Index-file
def CreateFile(self):
currentdir = getcwd()
chdir(self.directory)
if (self.logtemplate == None) or (self.summarytemplate == None):
self.log4py.error("[ Index ] Logfile: Error: No template file specified !")
return
try:
if (self.logfileformat == LFHTML):
datei = open("index" + LEHTML, "w")
fileupdated = open("updated" + LEHTML, "w")
fileerrors = open("errors" + LEHTML, "w")
elif (self.logfileformat == LFPHP):
datei = open("index" + LEPHP, "w")
fileupdated = open("updated" + LEPHP, "w")
fileerrors = open("errors" + LEPHP, "w")
elif (self.logfileformat == LFPlain):
datei = open("index" + LEPlain, "w")
fileupdated = open("updated" + LEPlain, "w")
fileerrors = open("errors" + LEPlain, "w")
except:
self.log4py.error("[ Index ] Logfile: Couldn't open index file !")
return
self.ReadTemplateFile()
self.ReadSummaryTemplateFile(datei, fileupdated, fileerrors)
# Calculate the time
minutes, seconds = divmod(self.totalseconds, 60)
hours, minutes = divmod(minutes, 60)
# Create Format specific text
if (self.logfileformat == LFPlain):
duration = "%2d Hour(s), %2d Minute(s) and %2d Second(s)" % (hours, minutes, seconds)
downloaded = "%s Byte(s) in %.0f File(s), %.0f Link(s) and %.0f Directories ..." % (lng2str(self.nrbytesretr), self.nrfilesretr, self.nrlinkscreated, self.nrdirscreated)
deleted = "%s Byte(s) in %.0f File(s), %.0f Link(s) and %.0f Directories ..." % (lng2str(self.nrbytesdel), self.nrfilesdel, self.nrlinksdel, self.nrdirsdel)
else:
duration = "<b>%2d</b> Hour(s), <b>%2d</b> Minute(s) and <b>%2d</b> Second(s)" % (hours, minutes, seconds)
downloaded = "<b>%s</b> Byte(s) in <b>%.0f</b> File(s), <b>%.0f</b> Link(s) and <b>%.0f</b> Directories ..." % (lng2str(self.nrbytesretr), self.nrfilesretr, self.nrlinkscreated, self.nrdirscreated)
deleted = "<b>%s</b> Byte(s) in <b>%.0f</b> File(s), <b>%.0f</b> Link(s) and <b>%.0f</b> Directories ..." % (lng2str(self.nrbytesdel), self.nrfilesdel, self.nrlinksdel, self.nrdirsdel)
# Calculate download rate
if (self.nrbytesretr > 0):
if (self.totalseconds > 0):
rate = self.nrbytesretr / self.totalseconds
else:
rate = 0
kbytepersec = int(rate / 1024)
if (kbytepersec > 0):
if (self.logfileformat == LFPlain):
downloadrate = "%.0f KByte/sec" % int(kbytepersec)
else:
downloadrate = "<b>%.0f</b> KByte/sec" % int(kbytepersec)
else:
if (self.logfileformat == LFPlain):
downloadrate = "%.0f Byte/sec" % int(rate)
else:
downloadrate = "<b>%.0f</b> Byte/sec" % int(rate)
else:
downloadrate = "Nothing downloaded"
categorykeys = self.mirrorcategory.keys()
categorykeys.sort()
categorylist = ""
for i in range(len(categorykeys)):
if (self.logfileformat == LFPlain):
categorylist = "%s%s" % (categorylist, categorykeys[i])
else:
categorylist = "%s<a href=\"#%s\">%s</a>" % (categorylist, categorykeys[i], categorykeys[i])
if (i < (len(categorykeys) - 1)):
categorylist = "%s | " % categorylist
self.CreateMasterIndexFile(categorylist, duration, downloaded, deleted, downloadrate, datei, categorykeys, fileupdated, fileerrors)
self.CreateSummaryFile(fileupdated, fileerrors)
datei.close()
fileupdated.close()
fileerrors.close()
chdir(currentdir)
def GetLastlog(self):
currentdir = getcwd()
chdir(self.directory)
if os.path.exists(LastlogFilename):
datei = open(LastlogFilename, "r")
line = "<empty>"
while line != "":
line = datei.readline()
if strip(line) != "":
splitted = split(strip(line), ";")
if not self.lastchanged.has_key(splitted[0]):
self.lastchanged[splitted[0]] = splitted[1]
datei.close()
chdir(currentdir)
def WriteLastlog(self):
currentdir = getcwd()
chdir(self.directory)
try:
datei = open(LastlogFilename, "w")
except:
self.log4py.error("[ Index ] Logfile: Couldn't write lastlog file !")
return
alltitles = self.lastchanged.keys()
for i in range(len(alltitles)):
datei.write(alltitles[i] + ";" + self.lastchanged[alltitles[i]] + "\n")
datei.close()
chdir(currentdir)
# Send mail to users, if things have changed or if emirror couldn't log in
def MailUsers(self):
changed = []
neverloggedin = []
active = []
for i in range(len(self.mirrorstatus.keys())):
key = self.mirrorstatus.keys()[i]
if (self.mirrorstatus[key] == MSactive):
active.append(key)
elif (self.mirrorstatus[key] == MSneverloggedin):
neverloggedin.append(key)
elif ((self.mirrorstatus[key] != MSunknown) and (self.mirrorstatus[key] != MSunchanged)):
changed.append(key)
changed.sort()
neverloggedin.sort()
active.sort()
if (len(changed) > 0) or (len(neverloggedin) > 0) or (len(active) > 0):
self.log4py.info("[ Main ] Mirrors have changed / updated - sending mail.")
parser = ConfigParser()
parser.read(self.mailusers)
emailaddresses = split(parser.get("email", "list"), ",")
filename = mktemp()
file = open(filename, "w")
file.write("\nWelcome to the EMirror EMail-Update Service !\n\n")
# Calculate the time
minutes, seconds = divmod(self.totalseconds, 60)
hours, minutes = divmod(minutes, 60)
# Create Format specific text
duration = "%2d Hour(s), %2d Minute(s) and %2d Second(s)" % (hours, minutes, seconds)
downloaded = "%s Byte(s) in %.0f File(s), %.0f Link(s) and %.0f Directories ..." % (lng2str(self.nrbytesretr), self.nrfilesretr, self.nrlinkscreated, self.nrdirscreated)
deleted = "%s Byte(s) in %.0f File(s), %.0f Link(s) and %.0f Directories ..." % (lng2str(self.nrbytesdel), self.nrfilesdel, self.nrlinksdel, self.nrdirsdel)
if (self.nrbytesretr > 0):
if (self.totalseconds > 0):
rate = self.nrbytesretr / self.totalseconds
else:
rate = 0
kbytepersec = int(rate / 1024)
if (kbytepersec > 0):
downloadrate = "%.0f KByte/sec" % int(kbytepersec)
else:
downloadrate = "%.0f Byte/sec" % int(rate)
else:
downloadrate = "Nothing downloaded"
if (self.firststarttime != None):
file.write("Start of first mirror: %s\n" % ctime(self.firststarttime))
else:
file.write("Start of first mirror: <not set>\n")
if (self.lastendtime != None):
file.write(" End of last mirror: %s\n" % ctime(self.lastendtime))
else:
file.write(" End of last mirror: <not set>\n")
file.write(" Total duration: %s\n" % duration)
file.write(" Downloaded: %s\n" % downloaded)
file.write(" Avg. download rate: %s\n" % downloadrate)
file.write(" Deleted: %s\n" % deleted)
file.write(" Proxy host/port: %s\n\n" % str(self.proxy))
file.write("Changed mirrors:\n\n")
if (len(changed) > 0):
for i in range(len(changed)):
file.write(" %s [%s]\n" % (changed[i], self.categorymirror[changed[i]]))
else:
file.write(" [None]\n")
if (len(neverloggedin) > 0):
file.write("\nFailed logins:\n\n")
for i in range(len(neverloggedin)):
file.write(" %s [%s]\n" % (neverloggedin[i], self.categorymirror[neverloggedin[i]]))
if (len(active) > 0):
file.write("\nIn progress:\n\n")
for i in range(len(active)):
file.write(" %s [%s]\n" % (active[i], self.categorymirror[active[i]]))
file.close()
for i in range(len(emailaddresses)):
emailaddress = emailaddresses[i]
self.log4py.debug("[ Main (debug) ] Executing mail-command: cat %s | %s %s " % (filename, self.mailusercmd, emailaddress))
system("cat %s | %s %s" % (filename, self.mailusercmd, emailaddress))
unlink(filename)
def Start(self):
if (self.directory[-1:] != "/"):
self.directory = self.directory + "/"
if not os.path.exists(self.directory):
self.log4py.error("[ Main ] Error: Directory \"%s\" doesn't exist !" % self.directory)
exit(1)
allfiles = listdir(self.directory)
excludefiles = compile("index.*|.*jpg$|.*gif$")
for i in range(len(allfiles)):
if (allfiles[i][0] not in digits) and (not excludefiles.match(allfiles[i])):
self.mirrorlogs.append(allfiles[i])
self.AnalyzeLogs()
self.GetLastlog()
self.CreateFile()
self.WriteLastlog()
if (self.mailusers != "") and (self.mailusercmd != ""):
self.MailUsers()
else:
if (self.mailusers != "") or (self.mailusercmd != ""):
self.log4py.warn("[ Main ] Warning: you have to set mail-users AND mail-user-cmd for sending mail !")
# ----------------------------------------------------------------------
# Definition of the Program-Class:
class Program:
# Initialisation of variables
def __init__(self):
self.mirror = Mirror() # The mirror to set parameters
self.mirrorlist = [] # List of mirrors
self.indexfile = IndexFile() # IndexFile to create
self.historytemplate = None # Template for history log
self.historydays = 28 # total # days in history log
self.historyrowdays = 7 # # days per row in history log
self.historypurge = TRUE # TRUE to purge logs older than historydays
self.configfile = None # Master configuration file
self.configdirectory = None # Directory containing configuration files
self.maxfork = 1 # How many parallel mirrors shall I run
self.uid = None # The UID of the user who executes the program
self.log4py = log4py.Logger().get_instance(self)
self.log4py.set_loglevel(log4py.LOGLEVEL_ERROR)
self.children = [] # Array of child process PIDs
# Print some information about the program & syntax
def PrintInfo(self, message, errormsg = None):
sys.stdout.write("%s\n" % PIHeader)
if (errormsg != None):
sys.stdout.write("%s. Try emirror --help for a list of possible parameters.\n" % capitalize(str(errormsg)))
else:
sys.stdout.write("%s\n" % message)
sys.stdout.write("%s\n" % PIOptions)
exit(0)
# checks wether a file exists and reports an error if not
def FileExists(self, file):
if (not os.path.exists(file)):
self.log4py.warn("[ Main ] Warning: file \"%s\" does not exist." % file)
return ""
else:
if (find(file, "/") != -1):
return file
else:
return getcwd() + "/" + file
# Check command line / configuration file options
def CheckOptions(self, parameter, value, filename = "", option = None):
if (parameter[0:2] == "--"):
parameter = parameter[2:]
# Startup
if (parameter == "-V" or parameter == "version"):
sys.stdout.write("%s\n" % PIHeader)
sys.stdout.write("%s\n" % PIRevLog)
exit(0)
elif (parameter == "-h" or parameter == "help"):
self.PrintInfo(PIUsage)
elif (parameter == "-v" or parameter == "verbose"):
self.log4py.set_loglevel(log4py.LOGLEVEL_VERBOSE)
self.mirror.log4py.set_loglevel(log4py.LOGLEVEL_VERBOSE)
self.mirror.installer.log4py.set_loglevel(log4py.LOGLEVEL_VERBOSE)
self.indexfile.log4py.set_loglevel(log4py.LOGLEVEL_VERBOSE)
elif (parameter == "quiet"):
self.log4py.set_loglevel(log4py.LOGLEVEL_NONE)
self.mirror.log4py.set_loglevel(log4py.LOGLEVEL_NONE)
self.mirror.installer.log4py.set_loglevel(log4py.LOGLEVEL_NONE)
self.indexfile.log4py.set_loglevel(log4py.LOGLEVEL_NONE)
# Logging and input files
elif (parameter == "-o" or parameter == "output-file"):
self.mirror.logfilename = value
elif (parameter == "-O" or parameter == "output-directory"):
if value[-1:] != "/":
value = value + "/"
self.mirror.logdirectory = value
elif (parameter == "-f" or parameter == "output-format"):
format = lower(value)
if (format != LFHTML) and (format != LFPlain) and (format != LFPHP):
self.log4py.warn("[ Main ] Warning: invalid logfile format, choose HTML or Plain.")
format = LFHTML
self.mirror.logfileformat = format
self.indexfile.logfileformat = format
elif (parameter == "-c" or parameter == "config-file"):
self.configfile = value
elif (parameter == "-d" or parameter == "config-directory"):
self.configdirectory = value
elif (parameter == "-n" or parameter == "index-directory"):
if value[-1:] != "/":
value = value + "/"
self.indexfile.directory = value
elif (parameter == "category"):
self.mirror.category = value
elif (parameter == "description"):
self.mirror.description = value
elif (parameter == "title"):
self.mirror.title = value
self.mirror.installer.title = value
elif (parameter == "email"):
self.mirror.adminemail = value
elif (parameter == "mail-users"):
self.indexfile.mailusers = self.FileExists(value)
elif (parameter == "mail-user-cmd"):
self.indexfile.mailusercmd = value
elif (parameter == "log-template"):
self.mirror.logtemplate = self.FileExists(value)
elif (parameter == "index-template"):
self.indexfile.logtemplate = self.FileExists(value)
elif (parameter == "summary-template"):
self.indexfile.summarytemplate = self.FileExists(value)
elif (parameter == "history-template"):
self.historytemplate = self.FileExists(value)
self.mirror.history_logging = TRUE
elif (parameter == "history-days"):
self.historydays = int(value)
elif (parameter == "history-row-days"):
self.historyrowdays = int(value)
elif (parameter == "no-history-purge"):
self.historypurge = FALSE
# Mirror locking and related options
elif (parameter == "lock-dir"):
if value[-1:] != "/":
value = value + "/"
self.mirror.lockdir = value
elif (parameter == "lock-timeout"):
self.mirror.locktimeout = int(value)
elif (parameter == "operation-timeout"):
self.mirror.operationtimeout = int(value)
elif (parameter == "completed-file"):
self.mirror.completed_file = value
# Download and FTP options
elif (parameter == "-P" or parameter == "parent"):
if value[-1:] != "/":
value = value + "/"
if value[0] != "/":
self.log4py.warn("[ Main ] Warning: use absolut values for the \"parent\" parameter !.")
value = "%s/%s" % (os.getcwd(), value)
self.mirror.directory = value
elif (parameter == "-D" or parameter == "download-url"):
if value[-1:] != "/":
value = value + "/"
self.mirror.localurl = value
elif (parameter == "-t" or parameter == "tries"):
self.mirror.retries = atoi(value)
elif (parameter == "-w" or parameter == "wait"):
self.mirror.delay = atoi(value)
elif (parameter == "-U" or parameter == "umask"):
self.mirror.umask = atoi(value, 8)
elif (parameter == "-r" or parameter == "recursive"):
if (str2bool(value) == TRUE) or (value == ""):
self.mirror.recursive = TRUE
elif (str2bool(value) == FALSE):
self.mirror.recursive = FALSE
else:
self.log4py.warn("[ Main ] Warning: invalid value for -r/--recursive. Use Yes/No or TRUE/False.")
elif (parameter == "-e" or parameter == "exclude"):
try:
self.mirror.exclude = compile(unfold(value))
except:
self.log4py.error("[ Main ] Error in exclude-regexp: %s" % value)
exit(1)
elif (parameter == "-i" or parameter == "include"):
try:
self.mirror.include = compile(unfold(value))
except:
self.log4py.error("[ Main ] Error in include-regexp: %s" % value)
exit(1)
elif (parameter == "ignore"):
try:
self.mirror.ignore = compile(unfold(value))
except:
self.log4py.error("[ Main ] Error in ignore-regexp: %s" % value)
exit(1)
elif (parameter == "proxy"):
splitted = split(value, ":")
if (len(splitted) != 2):
self.log4py.warn("[ Main ] Invalid value for \"Proxy\", use <hostname:port> !")
else:
self.mirror.proxy = value
self.indexfile.proxy = value
if (self.mirror.passivemode == FALSE):
self.mirror.passivemode = TRUE
elif (parameter == "test"):
if (str2bool(value) == TRUE) or (value == ""):
self.mirror.test = TRUE
elif (str2bool(value) == FALSE):
self.mirror.test = FALSE
else:
self.log4py.warn("[ Main ] Warning: invalid value for --test. Use Yes/No or True/False.")
# Set of only-latest values
elif (parameter == "-l" or parameter == "only-latest"):
try:
regexp = compile(unfold(value))
self.mirror.onlylatest.append(regexp)
except:
self.log4py.error("[ Main ] Error in only-latest-regexp: %s" % value)
exit(1)
# Set of auto install values
elif (parameter == "auto-install"):
self.mirror.installer.add(option, value)
elif (parameter == "-u" or parameter == "username"):
self.mirror.username = value
elif (parameter == "-p" or parameter == "password"):
self.mirror.password = value
elif (parameter == "-m" or parameter == "max-parallel"):
self.maxfork = atoi(value)
elif (parameter == "-R" or parameter == "real-hostname"):
self.mirror.realhostname = value
elif (parameter == "-M" or parameter == "max-days"):
self.mirror.maxdays = atoi(value)
elif (parameter == "-A" or parameter == "max-age"):
self.mirror.maxage = atoi(value)
elif (parameter == "-E" or parameter == "max-delete"):
if (value[-1:] != "%") and (lower(value[-1:]) != "k") and (lower(value[-1:]) != "m"):
self.log4py.warn("[ Main ] Warning: invalid value for --max-delete (has to end in M, K or %).")
else:
self.mirror.maxdelete = value
elif (parameter == "-F" or parameter == "follow-symlinks"):
if (str2bool(value) == TRUE) or (value == ""):
self.mirror.followsymlinks = TRUE
elif (str2bool(value) == FALSE):
self.mirror.followsymlinks = FALSE
else:
self.log4py.warn("[ Main ] Warning: invalid value for -F/--follow-symlinks. Use Yes/No or True/False.")
elif (parameter == "-s" or parameter == "parent-format"):
if (lower(value) == "full") or (lower(value) == "path") or (lower(value) == "none"):
self.mirror.parentformat = lower(value)
else:
self.log4py.warn("[ Main ] Warning: invalid value for -s/--parent-format: use FULL, PATH or NONE.")
elif (parameter == "ignore-mdtm"):
if (str2bool(value) == TRUE) or (value == ""):
self.mirror.ignoremdtm = TRUE
elif (str2bool(value) == FALSE):
self.mirror.ignoremdtm = FALSE
else:
self.log4py.warn("[ Main ] Warning: invalid value for --ignore-mdtm: use Yes/No or True/False.")
elif (parameter == "no-mod-check"):
if (str2bool(value) == TRUE) or (value == ""):
self.mirror.nomodcheck = TRUE
elif (str2bool(value) == FALSE):
self.mirror.nomodcheck = FALSE
else:
self.log4py.warn("[ Main ] Warning: invalid value for --no-mod-check: use Yes/No or True/False.")
elif (parameter == "owner"):
if (self.uid != 0):
self.log4py.warn("[ Main ] Warning: only user \"root\" can set the owner of files.")
else:
if (find(value, ":") == -1):
self.log4py.warn("[ Main ] Warning: wrong format for owner: use UID:GID")
else:
splitted = split(value, ":")
self.mirror.owner = atoi(splitted[0])
self.mirror.group = atoi(splitted[1])
elif (parameter == "delete-first"):
if (str2bool(value) == TRUE) or (value == ""):
self.mirror.deletefirst = TRUE
elif (str2bool(value) == FALSE):
self.mirror.deletefirst = FALSE
else:
self.log4py.warn("[ Main ] Warning: invalid value for --delete-first: use Yes/No or True/False.")
elif (parameter == "delete-local"):
if (str2bool(value) == TRUE) or (value == ""):
self.mirror.deletelocal = TRUE
elif (str2bool(value) == FALSE):
self.mirror.deletelocal = FALSE
else:
self.log4py.warn("[ Main ] Warning: invalid value for --delete-local: use Yes/No or True/False.")
elif (parameter == "passive"):
if (str2bool(value) == TRUE) or (value == ""):
self.mirror.passivemode = TRUE
elif (str2bool(value) == FALSE):
self.mirror.passivemode = FALSE
else:
self.log4py.warn("[ Main ] Warning: invalid value for --passive: use Yes/No or True/False.")
elif (parameter == "continue"):
if (str2bool(value) == TRUE) or (value == ""):
self.mirror.continueftp = TRUE
elif (str2bool(value) == FALSE):
self.mirror.continueftp = FALSE
else:
self.log4py.warn("[ Main ] Warning: invalid value for --continue: use Yes/No or True/False.")
# Error handling options
elif (parameter == "error-mailcmd"):
self.mirror.errormailcmd = value
# Info handling
elif (parameter == "info-mailcmd"):
self.mirror.infomailcmd = value
# Debug options
elif (parameter == "debug"):
self.log4py.set_loglevel(log4py.LOGLEVEL_DEBUG)
self.mirror.log4py.set_loglevel(log4py.LOGLEVEL_DEBUG)
self.mirror.installer.log4py.set_loglevel(log4py.LOGLEVEL_DEBUG)
self.indexfile.log4py.set_loglevel(log4py.LOGLEVEL_DEBUG)
elif (parameter == "noexception"):
self.mirror.noexception = TRUE
# Other options
elif (parameter == "url"):
self.mirror.url = value
else:
if (filename != ""):
self.log4py.error("[ Main ] Error in config-file %s - invalid parameter \"%s\"\n" % (filename, parameter))
else:
self.log4py.error("[ Main ] Error - invalid parameter \"%s\"\n\n" % parameter)
exit(1)
# Parse the command line, if a url has been specified
def ParseCmdLine(self):
if environ.has_key(EMirrorEnvOptions):
envargv = split(environ[EMirrorEnvOptions])
fullargv = envargv + argv[1:]
else:
fullargv = argv[1:]
try:
optlist, args = getopt.getopt(fullargv,
'Vhvo:O:f:c:d:n:P:D:t:w:re:i:lu:p:m:R:M:A:E:Fs:U:',
['version', 'help', 'verbose', 'quiet', 'output-file=', 'output-directory=', 'output-format=',
'config-file=', 'config-directory=', 'index-directory=', 'category=',
'description=', 'title=', 'email=', 'parent=', 'download-url=', 'tries=', 'wait=',
'recursive', 'exclude=', 'include=', 'only-latest', 'username=', 'password=',
'max-parallel=', 'real-hostname=',
'max-days=', 'max-age=', 'passive', 'umask=',
'max-delete=', 'log-template=', 'index-template=',
'follow-symlinks=', 'parent-format=', 'error-mailcmd=', 'ignore-mdtm', 'no-mod-check', 'owner=',
'delete-local', 'delete-first'
'mail-users=', 'mail-user-cmd=', 'debug', 'noexception', 'proxy=', 'continue',
'info-mailcmd=', 'test', 'ignore=',
'history-template=', 'history-days=', 'history-row-days=', 'no-history-purge',
'lock-dir=', 'lock-timeout=', 'operation-timeout=', 'completed-file='])
except getopt.error, detail:
self.PrintInfo(PIUsage, detail)
for flag, arg in optlist:
self.CheckOptions(flag, arg)
if (self.configfile == None) and (self.indexfile.directory == None):
if (len(args) != 1):
self.PrintInfo(PIUsage)
if (len(args) > 0):
self.mirror.url = args[0]
if (lower(self.mirror.url[0:6]) != "ftp://") and (lower(self.mirror.url[0:7]) != "http://") and (lower(self.mirror.url[0:8]) != "rsync://"):
self.log4py.error("%s" % PIHeader)
self.log4py.error("Error: specified protocol not supported ...\n")
exit(1)
self.mirrorlist.append(self.mirror)
# Read a configuration file
def ReadConfigFile(self, configfilename, general = None):
found = FALSE
if (os.path.exists(configfilename)):
found = TRUE
if (found == FALSE):
if (environ.has_key(EMirrorEnvironment)):
emirrorcfgpath = environ[EMirrorEnvironment]
if emirrorcfgpath[-1:] != "/":
emirrorcfgpath = emirrorcfgpath + "/"
configfilename = emirrorcfgpath + configfilename
if (not os.path.exists(configfilename)):
self.log4py.error("Configuration file \"%s\" does't exist ..." % configfilename)
exit(1)
if (general == None):
parser = ConfigParser()
else:
parser = general
try:
parser.read(configfilename)
except:
self.log4py.error("[ Main ] Error: invalid config-file: %s" % configfilename)
exit(1)
for i in range(len(parser.sections())):
section = parser.sections()[i]
for j in range(len(parser.options(section))):
option = parser.options(section)[j]
if (option != "name") and (option != "__name__"): # name is the name of the section
if (lower(section) == "only-latest"):
self.CheckOptions("only-latest", parser.get(section, option), configfilename)
elif (lower(section) == "auto-install"):
self.CheckOptions("auto-install", parser.get(section, option), configfilename, option)
else:
self.CheckOptions(option, parser.get(section, option), configfilename)
# Get a list of configuration files ...
def GetConfigFiles(self, defaults):
if (not os.path.exists(self.configdirectory)):
self.log4py.error("\n[ Main ] Error: Directory / File %s doesn't exist !\n" % self.configdirectory)
exit(1)
if (os.path.isdir(self.configdirectory)):
if (self.configdirectory[-1:] != "/"):
self.configdirectory = "%s/" % self.configdirectory
files = listdir(self.configdirectory, incdirs = FALSE, inclinks = TRUE)
else:
files = [os.path.basename(self.configdirectory)]
self.configdirectory = "%s/" % os.path.dirname(self.configdirectory)
defaultmirror = copy.deepcopy(self.mirror)
count = 0
for i in range(len(files)):
filename = files[i]
if (filename[-1:] != "~"):
self.mirror = copy.deepcopy(defaultmirror)
self.ReadConfigFile(self.configdirectory + filename, defaults)
self.mirrorlist.append(self.mirror)
count = count + 1
self.log4py.debug("[ Main (debug) ] %.0f configuration file(s) read" % count)
# Signal Handler for SIGCHLD
def SigChldHandler(self, signal_number, stack_frame):
""" Signal Handler for SIGCHLD. """
# Reinstall the sigchild handler, because it gets deleted on libc5/sysv machines
signal.signal(signal.SIGCHLD, self.SigChldHandler)
# Check specifically if a child mirror process terminated. Otherwise
# waitpid reaps status from non-mirror children (e.g. from getoutput())
for childpid in self.children[:]:
try:
(pid, status) = os.waitpid(childpid, os.WNOHANG)
except:
break
if (pid == childpid):
self.children.remove(pid)
if (os.WIFSIGNALED(status)):
sig = os.WTERMSIG(status)
self.log4py.warn("[ Main ] Warning: child (PID %s) terminated with signal %s" % (pid, sig))
elif (os.WIFEXITED(status)):
ev = os.WEXITSTATUS(status)
if (ev == 0):
self.log4py.debug("[ Main (debug) ] child (PID %s) finished normally" % pid)
else:
self.log4py.warn("[ Main ] Warning: child (PID %s) exited with status %s" % (pid, ev))
else:
self.log4py.warn("[ Main ] Warning: child (PID %s) terminated" % pid)
# Define a Handler for SigTerm for the parent. Kill children and exit ASAP.
def SigTermHandler(self, signal_number, stack_frame):
self.log4py.info("[ Main ] main program aborted or killed - terminating children")
# Stop Program forking new children
for childpid in self.children[:]:
try:
# Kill children immediately - some mirror locks may be orphaned but
# will be cleaned up next time the mirrors run. Using SIGKILL means that
# the children will not have the chance to terminate gracefully.
kill(childpid, signal.SIGKILL)
self.log4py.debug("[ Main (debug) ]: killed child PID %s" % childpid)
except:
self.log4py.debug("[ Main (debug) ]: kill -9 %s (child) failed" % childpid)
exit(1)
# Start mirroring ;-)
def Start(self):
sys.stderr = sys.stdout
self.uid = getuid()
self.ParseCmdLine()
if (self.configfile != None):
self.log4py.debug("[ Main (debug) ] reading main configuration file")
defaults = self.ReadConfigFile(self.configfile)
if (self.configdirectory != None):
self.log4py.debug("[ Main (debug) ] reading other configuration files")
self.GetConfigFiles(defaults)
if (self.maxfork == 1):
self.log4py.info("[ Main ] ECLiPt Mirroring Tool Version %s - starting 1 copy" % PIVersion)
else:
self.log4py.info("[ Main ] ECLiPt Mirroring Tool Version %s - starting max. %d copies" % (PIVersion, self.maxfork))
signal.signal(signal.SIGINT, self.SigTermHandler)
signal.signal(signal.SIGTERM, self.SigTermHandler)
signal.signal(signal.SIGCHLD, self.SigChldHandler)
while (len(self.mirrorlist) > 0):
if (len(self.children) < self.maxfork):
self.log4py.debug("[ Main (debug) ] starting new child")
mirror = self.mirrorlist[0]
self.mirrorlist = self.mirrorlist[1:]
self.log4py.debug("[ Main (debug) ] %d mirrors remaining" % len(self.mirrorlist))
pid = os.fork()
if (pid == 0):
mirror.Start()
elif (pid == -1):
self.log4py.error("[ Main ] Error: Failed to fork child process")
exit(1)
self.children.append(pid)
else:
self.log4py.debug("[ Main (debug) ] Waiting for children to finish")
signal.pause()
while (len(self.children) > 0):
self.log4py.debug("[ Main (debug) ] Waiting for children to finish")
signal.pause()
self.log4py.debug("[ Main (debug) ] All mirrors finished")
if (self.historytemplate != None):
self.log4py.info("[ Main ] Writing history file in directory %s" % self.mirror.logdirectory)
mh = MirrorHistory(self.historytemplate,
self.mirror.logdirectory,
self.mirror.lockdir,
self.historydays,
self.historyrowdays,
self.historypurge)
mh.log4py.set_loglevel(self.log4py.get_loglevel())
fn = self.mirror.logdirectory + "/" + "history.html"
tmpfn = self.mirror.logdirectory + "/" + str(getpid()) + "-history.html"
try:
out = open(tmpfn, "w")
except IOError, reason:
self.log4py.error("Unable to open output file %s: %s" % (tmpfn, reason))
return
mh.write(out)
rename(tmpfn, fn)
elif (self.indexfile.directory != None):
# Note that IndexFile won't work unless indexfile.directory == mirror.logdirectory
self.log4py.info("[ Main ] Writing Index-file in directory %s" % self.indexfile.directory)
if self.configfile != None:
self.ReadConfigFile(self.configfile)
self.indexfile.Start()
|