This example shows how to use the python logging package so that you can log results to a log file as well as to the console. Though it may look to be extraneous, this will be very useful when troubleshooting code that ran in the past for example.
from datetime import datetime
import logging
import pandas as pd
import warnings
warnings.filterwarnings('ignore')
input_file = "somefile.csv"
applicationName = "SomeApp"
# Logging
def log(applicationName):
# define logging to file
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s, %(name)s, %(levelname)s, %(message)s',
datefmt='%m-%d-%Y %H:%M:%S',
filename='log_' + applicationName + '.log',
filemode='a')
# define logging to console
logger = logging.getLogger(applicationName)
# create formatter and add it to the handlers
formatter = logging.Formatter('%(levelname)s - %(message)s')
# define a Handler which writes INFO messages or higher to the sys.stderr/console
consolelog = logging.StreamHandler()
consolelog.setLevel(logging.INFO)
consolelog.setFormatter(formatter)
# add the handlers to logger
logger.addHandler(consolelog)
return logger
logger = log(applicationName)
logger.info("Start logging for application {0} at {1}.".format(applicationName, datetime.now()))
try:
logger.info('Attempting to read the input file {0} for parsing.'.format(input_file))
# Read the csv to a pandas parser. Due to file structure, skip the top 3 and bottom rows
csv = pd.read_csv(input_file, skiprows=3, skip_footer=1)
except:
logger.warning('Could not find the input file {0} for parsing.'.format(input_file))
The output below shows the information that was logged to the file. Note that this is the same as the console output.
log_file = pd.read_csv('log_' + applicationName + '.log')
print(log_file)