How To Force A Rotating Name With Python's Timedrotatingfilehandler?

I am trying to use TimedRotatingFileHandler to keep daily logs in separate log files. The rotation works perfectly as intended, but what I don't like how it does is the naming of t

Solution 1:

I have created a class ParallelTimedRotatingFileHandler mainly aimed at allowing multiple processes writing in parallel to a log file. The problems with parallel processes solved by this class, are:

• The rollover moment when all processes are trying to copy or rename the same file at the same time, gives errors.
• The solution for this problem was exactly the naming convention you suggest. So, for a file name Service that you supply in the handler, logging does not go to e.g. Service.log but today to Service.2014-08-18.log and tomorrow Service.2014-08-19.log.
• Another solution is to open the files in a (append) mode instead of w to allow parallel writes.
• Deleting the backup files also needs to be done with caution as multiple parallel processes are deleting the same files at the same time.
• This implementation does not take into account leap seconds (which is not a problem for Unix). In other OS, it might still be 30/6/2008 23:59:60 at the rollover moment, so the date has not changed, so, we take the same file name as yesterday.
• I know that the standard Python recommendation is that the logging module is not foreseen for parallel processes, and I should use SocketHandler, but at least in my environment, this works.

The code is just a slight variation of the code in the standard Python handlers.py module. Of course copyright to the copyright holders.

Here is the code:

import logging
import logging.handlers
import os
import time
import re

classParallelTimedRotatingFileHandler(logging.handlers.TimedRotatingFileHandler):
    def__init__(self, filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False, postfix = ".log"):

        self.origFileName = filename
        self.when = when.upper()
        self.interval = interval
        self.backupCount = backupCount
        self.utc = utc
        self.postfix = postfix

        if self.when == 'S':
            self.interval = 1# one second
            self.suffix = "%Y-%m-%d_%H-%M-%S"
            self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}$"elif self.when == 'M':
            self.interval = 60# one minute
            self.suffix = "%Y-%m-%d_%H-%M"
            self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}$"elif self.when == 'H':
            self.interval = 60 * 60# one hour
            self.suffix = "%Y-%m-%d_%H"
            self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}$"elif self.when == 'D'or self.when == 'MIDNIGHT':
            self.interval = 60 * 60 * 24# one day
            self.suffix = "%Y-%m-%d"
            self.extMatch = r"^\d{4}-\d{2}-\d{2}$"elif self.when.startswith('W'):
            self.interval = 60 * 60 * 24 * 7# one weekiflen(self.when) != 2:
                raise ValueError("You must specify a day for weekly rollover from 0 to 6 (0 is Monday): %s" % self.when)
            if self.when[1] < '0'or self.when[1] > '6':
                 raise ValueError("Invalid day specified for weekly rollover: %s" % self.when)
            self.dayOfWeek = int(self.when[1])
            self.suffix = "%Y-%m-%d"
            self.extMatch = r"^\d{4}-\d{2}-\d{2}$"else:
            raise ValueError("Invalid rollover interval specified: %s" % self.when)

        currenttime = int(time.time())
        logging.handlers.BaseRotatingHandler.__init__(self, self.calculateFileName(currenttime), 'a', encoding, delay)

        self.extMatch = re.compile(self.extMatch)
        self.interval = self.interval * interval # multiply by units requested

        self.rolloverAt = self.computeRollover(currenttime)

    defcalculateFileName(self, currenttime):
        if self.utc:
             timeTuple = time.gmtime(currenttime)
        else:
             timeTuple = time.localtime(currenttime)

        return self.origFileName + "." + time.strftime(self.suffix, timeTuple) + self.postfix

    defgetFilesToDelete(self, newFileName):
        dirName, fName = os.path.split(self.origFileName)
        dName, newFileName = os.path.split(newFileName)

        fileNames = os.listdir(dirName)
        result = []
        prefix = fName + "."
        postfix = self.postfix
        prelen = len(prefix)
        postlen = len(postfix)
        for fileName in fileNames:
            if fileName[:prelen] == prefix and fileName[-postlen:] == postfix andlen(fileName)-postlen > prelen and fileName != newFileName:
                 suffix = fileName[prelen:len(fileName)-postlen]
                 if self.extMatch.match(suffix):
                     result.append(os.path.join(dirName, fileName))
        result.sort()
        iflen(result) < self.backupCount:
            result = []
        else:
            result = result[:len(result) - self.backupCount]
        return result

     defdoRollover(self):
         if self.stream:
            self.stream.close()
            self.stream = None

         currentTime = self.rolloverAt
         newFileName = self.calculateFileName(currentTime)
         newBaseFileName = os.path.abspath(newFileName)
         self.baseFilename = newBaseFileName
         self.mode = 'a'
         self.stream = self._open()

         if self.backupCount > 0:
             for s in self.getFilesToDelete(newFileName):
                 try:
                     os.remove(s)
                 except:
                     pass

         newRolloverAt = self.computeRollover(currentTime)
         while newRolloverAt <= currentTime:
             newRolloverAt = newRolloverAt + self.interval

         #If DST changes and midnight or weekly rollover, adjust for this.if (self.when == 'MIDNIGHT'or self.when.startswith('W')) andnot self.utc:
             dstNow = time.localtime(currentTime)[-1]
             dstAtRollover = time.localtime(newRolloverAt)[-1]
             if dstNow != dstAtRollover:
                 ifnot dstNow:  # DST kicks in before next rollover, so we need to deduct an hour
                     newRolloverAt = newRolloverAt - 3600else:           # DST bows out before next rollover, so we need to add an hour
                     newRolloverAt = newRolloverAt + 3600
         self.rolloverAt = newRolloverAt

Solution 2:

As far as I know there is no way to directly achieve this.

One solution you could try is to override the default behavior.

• Create your own TimedRotatingFileHandler class and override the doRollover() function.
• Check the source in your python installation <PythonInstallDir>/Lib/logging/handlers.py

Something like this:

classMyTimedRotatingFileHandler(TimedRotatingFileHandler):def__init__(self, **kwargs):
        TimedRotatingFileHandler.__init__(self, **kwargs)

    defdoRollover(self):
        # Do your stuff, rename the file as you want 

Solution 3:

Here's a simple solution: add a custom namer function to the handler. The logging utility will call your namer function to create the name for the rolled-over file as Jester originally noted like 6 years (!) ago with filename.log.YYYYMMDD, so we need to "move" the .log part to the end:

defnamer(name):
    return name.replace(".log", "") + ".log"

Then after you've set up your handler just assign your function to its namer attribute:

handler.namer = namer

Here's my full logging init script, I'm new to python, criticism/advice is welcome:

import os
import logging
from logging.handlers import TimedRotatingFileHandler
from config import constants

defnamer(name):
    return name.replace(".log", "") + ".log"definit(baseFilename):
    logPath = constants.LOGGING_DIR
    envSuffix = '-prod'if constants.ENV == 'prod'else'-dev'
    logFilename = os.path.join(logPath, baseFilename + envSuffix + '.log')
    print(f"Logging to {logFilename}")

    handler = TimedRotatingFileHandler(logFilename,
    when = "midnight", 
    backupCount = 30,
    encoding = 'utf8')
    handler.setLevel(logging.DEBUG)
    handler.suffix = "%Y%m%d"
    handler.namer = namer # <-- Here's where I assign the custom namer.

    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s [%(module)s:%(lineno)d]')
    handler.setFormatter(formatter)

    logging.basicConfig(
        handlers = [handler],
        format = '%(asctime)s %(levelname)s %(message)s [%(module)s:%(lineno)d]',
        level = logging.DEBUG,
        datefmt = '%Y-%m-%d %H:%M:%S')


if __name__ == '__main__':
    init('testing')
    logging.error("ohai")
    logging.debug("ohai debug")
    logging.getLogger().handlers[0].doRollover()
    logging.error("ohai next day")
    logging.debug("ohai debug next day")

Solution 4:

I used solution https://stackoverflow.com/a/25387192/6619512 with Python 3.7 and it is a great solution.

But for 'midnight' when parameter and for when parameter starting with 'W' it did not work since the atTime parameter was introduced and used in the TimedRotatingFileHandler class.

To make use of this solution use the following __init__ line:

def__init__(self, filename, when='h', interval=1, backupCount=0,
             encoding=None, delay=False, utc=False, atTime=None, postfix = ".log"):

Also add the following to the content of the __init__ declarations:

self.postfix = postfix