Daylight savings and getmtime

Discussion in 'Python' started by Qvx, Jan 28, 2005.

  1. Qvx

    Qvx Guest

    Hello,

    I'we written a simple web deployment program which scans for the
    changes made to local copy of web site. Changed files are than
    packaged into a zip file and deployed to web server.

    Now here's the catch. Changes are computed using (1) log file from the
    last deployment and (2) local file system. Log file contains
    datestamps (integers) returned from os.path.getmtime(f) function at
    the time of last deployment. So i'm comparing two getmtime() values.
    The problem is when Daylight saving kicks in: suddenly all local files
    are reported as older than they were at the time of deployment.

    How do I compensate for this?

    Thanks,
    Tvrtko



    For those curious, here is the script. I apologize for Croatian
    comments and literals and missing private libraries, but I think the
    code is self-explanatory.
    ______________

    # -*- coding: windows-1250 -*-

    from os.path import getmtime, join
    from os import walk, rename
    from zipfile import ZipFile, ZIP_DEFLATED
    from sets import Set
    from StringIO import StringIO
    from ftplib import FTP
    from qvx.io import adapt_stdout, unadapt_stdout
    from qvx.composite import Dot
    from qvx.compositeutil import read_composite
    import sys
    import time

    class DeploymentError(Exception):
    pass

    class Deployer:
    def __init__ (self, cfg_file):
    self.reset(cfg_file)

    def reset (self, cfg_file):
    self.read_cfg(cfg_file)
    self.local_files = []
    self.remote_files = []
    self.new_files = []
    self.deleted_files = []
    self.newer_files = []
    self.older_files = []
    self.www_all_time = None
    self.old_deployed = False

    def read_cfg (self, cfg_file):
    tree = read_composite(cfg_file)
    self.cfg = Dot(tree).DEPLOYMENT

    def prepare_file_lists (self):
    # Sastavi popis _datoteka_ u DIR direktoriju.
    # Izostavi datoteke iz _notes direktorija
    self.local_files = []
    for root, dirs, files in walk(self.cfg.DIR):
    filtered = [join(root, f).replace('\\', '/') for f in
    files if f not in self.cfg.SKIP_FILES]
    self.local_files.extend(filtered)
    for skip_dir in self.cfg.SKIP_DIRS.split(','):
    if skip_dir.strip() in dirs:
    dirs.remove(skip_dir)

    # Sastavi popis datoteka na serveru
    # Koristi se sa informacijama od zadnjeg deploymenta
    # Popis se nalazi u www_all.txt datoteci
    self.remote_files = []
    remote_stamps = {}
    zip = ZipFile(self.cfg.FILE, 'r')
    for line in zip.read('www_all.txt').split('\n'):
    name, stamp = line.split('\t')
    remote_stamps[name] = int(stamp)
    self.remote_files.append(name)
    self.www_all_time = zip.getinfo('www_all.txt').date_time

    # Deployment nije obavljen ako nije zapisan log
    self.old_deployed = 'deployment.log' in zip.namelist()
    zip.close()

    # Rastavi datoteke u tri kategorije: nove, obrisane i iste
    lset = Set(self.local_files)
    rset = Set(self.remote_files)
    self.new_files = list(lset - rset)
    self.deleted_files = list(rset - lset)
    common_files = list(lset & rset)

    # Pogledaj što se promijenilo u zajedničkim datotekama
    self.newer_files = []
    self.older_files = []
    for name in common_files:
    remotetime = remote_stamps[name]
    localtime = getmtime(name) #+ 3600
    # Ako je razlika unutar sekunde, zanemari
    if abs(remotetime-localtime) > int(self.cfg.IGNORE_SEC):
    if remotetime > localtime:
    self.older_files.append(name)
    elif localtime > remotetime:
    self.newer_files.append(name)

    def need_redeployment (self):
    return not self.old_deployed

    def check_changes (self):
    # Ne bi trebalo biti starijih
    if self.older_files:
    raise DeploymentError('Ne smije biti starijih datoteka!')
    if not (self.new_files or self.deleted_files or
    self.newer_files):
    raise DeploymentError('Nema promjena!')

    def make_deployment_file (self):
    # Uključi potrebne datoteke
    deployment = ZipFile('new_'+self.cfg.FILE, 'w', ZIP_DEFLATED)
    for name in self.new_files + self.newer_files:
    deployment.write(name)

    # Uključi popis svih datoteka
    all_files = '\n'.join([f+'\t'+str(getmtime(f)) for f in
    self.local_files])
    deployment.writestr('www_all.txt', all_files)

    # Uključi popis datoteka za obrisati
    for_delete = '\n'.join(self.deleted_files)
    if for_delete:
    deployment.writestr('www_delete.txt', for_delete)
    deployment.close()
    print '\nNapravljena je nova deployment datoteka.'

    # Preimenuj deployment datoteke
    timestr = '%04d-%02d-%02d_%02d-%02d-%02d' % self.www_all_time
    old_deployment = self.cfg.FILE.replace('.zip',
    '_'+timestr+'.zip')
    rename(self.cfg.FILE, old_deployment)
    rename('new_'+self.cfg.FILE, self.cfg.FILE)
    print 'Stara deployment datoteka se sada zove', old_deployment

    def exec_ftp (self, silent, logtext, func, *arg):
    try:
    self.ftp_log = self.ftp_log + '\n\n' + logtext
    self.ftp_log = self.ftp_log + '\n' + func(*arg)
    return 'OK'
    except Exception, e:
    self.ftp_log = self.ftp_log + '\n' + str(e)
    if not silent:
    raise DeploymentError(str(e))
    return 'ERROR'

    def deploy_ftp (self):
    self.ftp_log = ''

    # Spoji se na FTP server
    print '\nSpajam se na ftp server %s ...' % (self.cfg.SERVER,),
    try:
    ftp = FTP(self.cfg.SERVER)
    print 'OK'
    except Exception, e:
    print 'ERROR'
    raise DeploymentError('Ne mogu se spojiti na FTP server:
    '+str(e))

    # Logiraj se
    print 'Logiram se kao %s ... ' % (self.cfg.USER,),
    print self.exec_ftp(False, 'LOGIN', ftp.login, self.cfg.USER,
    self.cfg.PASSWORD)

    # Kopiraj datoteke
    deployment = ZipFile(self.cfg.FILE, 'r')
    deployment_files = [n for n in deployment.namelist() if n not
    in ['www_all.txt', 'www_delete.txt', 'deployment.log']]
    if deployment_files:
    print 'Šaljem datoteke:'
    for name in deployment_files:
    bytes = deployment.read(name)
    fp = StringIO(bytes)
    print ' ', name, len(bytes), ' bytes ...',
    print self.exec_ftp(True, 'STORBIN '+name, ftp.storbinary,
    'STOR '+name, fp)
    fp.close()

    # Obriši datoteke
    if 'www_delete.txt' in deployment.namelist():
    deleted_files =
    deployment.read('www_delete.txt').split('\n')
    print 'Brišem datoteke:'
    for name in deleted_files:
    print ' ', name, '...',
    print self.exec_ftp(True, 'DEL '+name, ftp.delete,
    name)
    deployment.close()

    # Bye bye
    print 'Završavam s radom ...',
    print self.exec_ftp(True, 'BYE', ftp.quit)

    # Ispiši FTP log
    print '\nFTP log:'
    print '-'*20,
    print self.ftp_log

    def write_log (self, text):
    # Zapiši deployment log
    deployment = ZipFile(self.cfg.FILE, 'a', ZIP_DEFLATED)
    deployment.writestr('deployment.log', text)
    deployment.close()

    def run_interactively (self):
    # Adaptiraj stdout: mijenja kodnu stranicu u cp852 radi ispisa
    # hrvatskih grafema, također logira sve poruke
    ofa = adapt_stdout('cp852', True)

    try:
    try:
    # Analiziraj datoteke
    self.prepare_file_lists()
    print '*'*5, 'Obrisani', '*'*5, self.deleted_files
    print '*'*5, 'Novi ', '*'*5, self.new_files
    print '*'*5, 'Noviji ', '*'*5, self.newer_files
    print '*'*5, 'Stariji ', '*'*5, self.older_files

    if self.need_redeployment():
    # Pitaj korisnika da li želi poslati od prije
    pripremljeni deployment na server
    yn = raw_input('\nOd prije pripremljeni deployment
    nije obavljen.\n>>> Da li želite taj deployment poslati na server?
    [y/N]: ')
    if yn.lower().strip() == 'y':
    self.deploy_ftp()
    self.write_log(''.join(ofa._log))
    else:
    self.check_changes()

    # Pitaj korisnika da li želi napraviti deployment
    datoteku
    yn = raw_input('\n>>> Da li želite pripremiti
    deployment? [y/N]: ')
    if yn.lower().strip() == 'y':
    desc = raw_input('\n>>> Upišite kratki opis:
    ')
    print '\nOpis:', desc

    self.make_deployment_file()

    # Pitaj korisnika da li želi poslati
    deployment na server
    yn = raw_input('\n>>> Da li želite poslati
    deployment na server? [y/N]: ')
    if yn.lower().strip() == 'y':
    self.deploy_ftp()
    self.write_log(''.join(ofa._log))
    except DeploymentError, e:
    print str(e)
    finally:
    # Vrati stari stdout
    unadapt_stdout(ofa)

    if __name__ == '__main__':
    deployer = Deployer('hpk.ini')
    deployer.run_interactively()
    x = raw_input('\n[Pritisnite Enter ...] ')
    sys.exit(0)
     
    Qvx, Jan 28, 2005
    #1
    1. Advertising

  2. Qvx wrote:

    > Hello,
    >
    > I'we written a simple web deployment program which scans for the
    > changes made to local copy of web site. Changed files are than
    > packaged into a zip file and deployed to web server.
    >
    > Now here's the catch. Changes are computed using (1) log file from the
    > last deployment and (2) local file system. Log file contains
    > datestamps (integers) returned from os.path.getmtime(f) function at
    > the time of last deployment. So i'm comparing two getmtime() values.
    > The problem is when Daylight saving kicks in: suddenly all local files
    > are reported as older than they were at the time of deployment.
    >
    > How do I compensate for this?
    >
    > Thanks,
    > Tvrtko


    Never use the local time, always use GMT ( UTC ) time instead.

    Since it seems os.path.getmtime already gives UTC time, onemust wonder if
    your OS isn't broken in some way ;) If you can't solve that problem, then
    use a simple md5sum of the files instead. md5sum isn't that slow to compute
    and it gives better results than timestanps.

    Or use a specialised tool like rsync which does a very good job for that.
     
    Christophe Cavalaria, Jan 28, 2005
    #2
    1. Advertising

  3. Qvx

    Guest

    on my windows xp box os.path.getmtime gives back local time (I just
    saved the file):
    >>> os.path.getmtime('c:\\temp\\testset.py')

    1106955016
    >>> print time.mktime(time.localtime())

    1106955034.0

    You can try to figure out if DST is on by comparing time.localtime()
    versus time.gmtime(). In the Western European Timezone there's one hour
    difference in winter and two hours in summer.

    >>> os.path.getmtime('c:\\temp\\testset.py')

    1106954702
    >>> g = time.mktime(time.gmtime())
    >>> l = time.mktime(time.localtime())
    >>> print g, l

    1106951381.0 1106954987.0
    >>> os.path.getmtime('c:\\temp\\testset.py')

    1106955016
    >>> print time.mktime(time.localtime())

    1106955034.0
    >>> print l - g

    3606.0
    >>>
     
    , Jan 28, 2005
    #3
  4. Qvx

    Nick Coghlan Guest

    wrote:
    > on my windows xp box os.path.getmtime gives back local time (I just
    > saved the file):
    >
    >>>>os.path.getmtime('c:\\temp\\testset.py')

    >
    > 1106955016
    >
    >>>>print time.mktime(time.localtime())

    >
    > 1106955034.0


    No. mktime is converting the local time to UTC. Try this instead:

    Py> time.gmtime(os.path.getmtime("c:/devel/mtime_test.txt"))
    (2005, 1, 29, 3, 35, 37, 5, 29, 0)

    Py> time.localtime(os.path.getmtime("c:/devel/mtime_test.txt"))
    (2005, 1, 29, 15, 35, 37, 5, 29, 1)

    (This is with my machine set to Sydney time, so that I'm at UTC + 10, with
    daylight savings currently active)

    > You can try to figure out if DST is on by comparing time.localtime()
    > versus time.gmtime(). In the Western European Timezone there's one hour
    > difference in winter and two hours in summer.


    To figure out if DST is currently active, you can use:

    Py> import time
    Py> time.localtime()
    (2005, 1, 29, 14, 29, 26, 5, 29, 0)
    Py> time.localtime().tm_isdst
    0

    Switching to Sydney time (which actually uses DST, unlike Brisbane) gives:

    Py> import time
    Py> time.localtime().tm_isdst
    1

    The same trick will work on historical dates, too. For instance, consider a file
    created last winter, when Sydney was NOT on daylight savings:

    Py> time.localtime(os.path.getmtime("c:/log.txt"))
    (2004, 7, 20, 19, 46, 35, 1, 202, 0)

    Cheers,
    Nick.

    --
    Nick Coghlan | | Brisbane, Australia
    ---------------------------------------------------------------
    http://boredomandlaziness.skystorm.net
     
    Nick Coghlan, Jan 29, 2005
    #4
    1. Advertising

Want to reply to this thread or ask your own question?

It takes just 2 minutes to sign up (and it's free!). Just click the sign up button to choose a username and then you can ask your own questions on the forum.
Similar Threads
  1. Ryan Ternier
    Replies:
    1
    Views:
    1,484
    Ryan Ternier
    Oct 14, 2005
  2. John Taylor
    Replies:
    2
    Views:
    505
    Mark Tolonen
    Apr 12, 2004
  3. James
    Replies:
    1
    Views:
    415
  4. Polaris431
    Replies:
    4
    Views:
    495
    =?ISO-8859-1?Q?G=F6ran_Andersson?=
    May 28, 2007
  5. Dean
    Replies:
    14
    Views:
    1,239
    Nigel Wade
    Jun 25, 2007
Loading...

Share This Page