Continue running code if one aspect of a try block fails, but still raise the exception-CodePudding

As part of a much larger script to process excel files into different formats with multiple Python functions, I have this function to actually run all the other functions:

print = logging.info
class MyException(Exception):
    pass
@catch_exception_and_send_email
def validation_and_processing(file_list):
    '''
    This function the file list and runs each file in the directory through the validation and processing functions.
    
    args:
        file_list : output file path     
    '''
    exceptions = []
    for file in files:
        try:
            if file.startswith('Nomura'):
                check_filename_format(input_file = file)
                nomura = nomura_validation(input_dir = input_dir, file = file)
                nomura_df = nomura_processing(nomura)
                write_output(path=output_dir, filename=file, dataframe=nomura_df)
                shutil.move(input_dir file, archive_dir file)
                print("Congratulations! {} dealer file processed successfully on {} at {}".format(file, date, time))

            elif file.startswith('MSTN'):
                check_filename_format(input_file = file)
                morgan = morgan_validation(input_dir = input_dir, file = file) 
                morgan_df = morgan_processing(morgan)
                write_output(path=output_dir, filename=file, dataframe=morgan_df)
                shutil.move(input_dir file, archive_dir file)
                print("Congratulations! {} dealer file processed successfully on {} at {}".format(file, date, time))

            elif file.startswith('JPM'):
                check_filename_format(input_file = file)
                jpm, input_file = jpm_validation(input_dir = input_dir, file = file)
                jpm = jpm_processing(jpm, input_file)
                write_output(path=output_dir, filename=file, dataframe=jpm)
                shutil.move(input_dir file, archive_dir file)
                print("Congratulations! {} dealer file processed successfully on {} at {}".format(file, date, time))

            elif file.startswith('BofA'):
                check_filename_format(input_file = file)
                bofa_not_ginnie, bofa_ginnie = bofa_validation(input_dir=input_dir, file=file)
                bofa_final = bofa_processing(df1=bofa_not_ginnie, df2=bofa_ginnie, input_file=file)       
                write_output(path=output_dir, filename=file, dataframe=bofa_final)
                shutil.move(input_dir file, archive_dir file)
                print("Congratulations! {} dealer file processed successfully on {} at {}".format(file, date, time))

            elif file.startswith('CITI'):
                check_filename_format(input_file = file)
                citi = citi_validation(input_dir=input_dir, file=file)
                citi_final = citi_processing(df=citi, input_file=file)    
                write_output(path=output_dir, filename=file, dataframe=citi_final)
                shutil.move(input_dir file, archive_dir file)
                print("Congratulations! {} dealer file processed successfully on {} at {}".format(file, date, time))
                
        except (OSError, IOError, MyException):
            # This should cast a wide net since:
            # In 3.3, IOError became an alias for OSError, 
            # and FileNotFoundError is a subclass of OSError
            # also catches PermissionError
            # Source: https://stackoverflow.com/questions/15032108/pythons-open-throws-different-errors-for-file-not-found-how-to-handle-b/15032444#15032444
            # Logs the error appropriately.
            logging.error("Error with file:"   file)
            continue
        except Exception as e:
            # Reraise exception adding info about which file
            pass
            raise Exception("Error with file:", file) from e
    return exceptions

The idea is if one of the file's function fails, it should raise an exception, but should still make sure to try to run all the other files. When I run a test case with one of the files being incorrectly formatted, it raises the error for that file and successfully completes the files that occur in the directory before that file, but not the ones after. How can I modify my code to accomplish both the error being raised if it occurs (I need the code to ultimately fail if one file is incorrect because an email is sent for exceptions and a log file is updated with the errors), but still run the other files if they are correct.

Here's my code for building the log file. I use catch_exception_and_send_email as a decorator function for other functions in the script in case they raise an exception.

now = datetime.now()
date = now.strftime("%m_%d_%Y") 
time = now.strftime("%H_%M_%S")
log_filename = "log_"   date   "_"   time   ".txt"

#Initialize log file
logging.basicConfig(filename=log_filename, level=logging.DEBUG, format='%(asctime)s %(levelname)s %(name)s %(message)s', force = True)
logger=logging.getLogger(__name__)
logging.getLogger('PIL').setLevel(logging.WARNING)

'''
The below function is a decorator function that checks every function in the file at runtime for exceptions,
and updates the log file and sends an email with details of exception
'''

def catch_exception_and_send_email(func):
    def exception_catcher(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except Exception as e:
            print(e)
            send_error_email(exception=e, message="An exception {} occurred in {}".format(e, func.__name__))
            logging.exception('Oh no! {} On {} at {} in: {} function.'.format(e, date, time, func.__name__), exc_info = False)
            sys.exit(1)
    return exception_catcher

CodePudding user response：

You need to be able to just retain the exception:

def validation_and_processing(file_list):
    '''
    This function the file list and runs each file in the directory through the validation and processing functions.
    
    args:
        file_list : output file path     
    '''
    exceptions = []
    for file in files:
        try:
            if file.startswith('Nomura'):
                ...
                ...

        except Exception as e:      
            exceptions.append(e)

    return exceptions

CodePudding user response：

Goal is to:

log expected exceptions for files and continue processing
raise exceptions and stop for other exceptions

Code

import traceback
import logging

def validation_and_processing(file_list):
    '''
    This function the file list and runs each file in the directory through the validation and processing functions.
    
    args:
        file_list : output file path     
    '''
    for file in files:
        try:
            if file.startswith('Nomura'):
                ...
                ...
        except (OSError, IOError, CustomExceptionName):
            # This should cast a wide net since:
            # In 3.3, IOError became an alias for OSError, 
            # and FileNotFoundError is a subclass of OSError
            # also catches PermissionError
            # Source: https://stackoverflow.com/questions/15032108/pythons-open-throws-different-errors-for-file-not-found-how-to-handle-b/15032444#15032444
            # CustomExceptionName class similar to: https://stackoverflow.com/questions/1319615/proper-way-to-declare-custom-exceptions-in-modern-python/53469898#53469898 
            # Logs the error appropriately.
            logging.exception("Error with file:"   file)
            continue     # continue processing

       except Exception as e:
            # Reraise exception adding info about which file
            raise Exception("Error with file:", file) from e