diff --git a/src/logwatcher/classifier.py b/src/logwatcher/classifier.py index 69d911f..c1d9fe2 100644 --- a/src/logwatcher/classifier.py +++ b/src/logwatcher/classifier.py @@ -1,23 +1,24 @@ import re +from logwatcher import logger from logwatcher.models import LogEntry # ============================================================ # Atomic patterns — reusable building blocks (no named groups) # ============================================================ -_GRP = r"GRP_\d+" # store group -_STORE = r"\d{5}/\d{3}" # store code -_SAP = r"\d{4}" # SAP code (4 digits) -_ORD = r"\w+" # order number (alphanumeric) -_BL = rf"{_SAP}_{_ORD}_DESADV" # BL number -_SIZE = r"SizeRange \d+\.\d+" # size range -_BRACKETS = r"(?:\[[^\]]+\])+" # one or more [Tag] blocks +_GRP = r"GRP_\d+" # store group +_STORE = r"\d{5}/\d{3}" # store code +_SAP = r"\d{4}" # SAP code (4 digits) +_ORD = r"\w+" # order number (alphanumeric) +_BL = rf"{_SAP}_{_ORD}_DESADV" # BL number +_SIZE = r"SizeRange \d+\.\d+" # size range +_BRACKETS = r"(?:\[[^\]]+\])+" # one or more [Tag] blocks # Composite prefixes — reused across multiple patterns _STORE_PREFIX_ERR = rf"(?P{_GRP}) / (?P{_STORE}) (?P{_SAP})_(?P{_ORD}) ERREUR" _STORE_PREFIX_WRN = rf"(?P{_GRP}) / (?P{_STORE}) (?P{_SAP})_(?P{_ORD}) WARNING" -_BL_PREFIX = rf"(?P{_BL}) : / (?P{_STORE}) {_BL}" +_BL_PREFIX = rf"(?P{_BL}) : / (?P{_STORE}) {_BL}" # error message patterns N2_PATTERNS = { @@ -35,37 +36,50 @@ N2_PATTERNS = { "PARTIAL_INTEGRATION": r"Erreur : (?P(?P\d{4})_(?P\w+)_DESADV) : ISC ERROR CODE:(?P\d+) \(attention intégration partielle du BL ou d'une commande\)", "SAME_LINE_EXCEPT_DISCOUNT": r"Erreur : (?PGRP_\d+) / (?P\d{5}/\d{3}) (?P\d{4})_(?P\w+) ERREUR (?P(?P\w+ (?P\w+))(?P\w{3})) / SetCOMBCDE -> OrderNumber : \d+ / Une commande ne peut pas contenir plusieurs lignes avec le même modèle, la même collection, la même date de livraison et avoir des remises différentes", "MULTIPLE_ROWS_IN_SINGLETON_SELECT": r"Erreur : (?PGRP_\d+) / (?P\d{5}/\d{3}) (?P\d{4})_(?P\w+) ERREUR (?P(?P\w+ (?P\w+))(?P\w{3})) / SizeRange (?P\d+\.\d+) / SetMrkFourn -> (?: \[[^\]]+\])+multiple rows in singleton select", - # low priority errors "SIZE_NOT_FOUND": r"Erreur : (?PGRP_\d+) / (?P\d{5}/\d{3}) (?P\d{4})_(?P\w+) WARNING (\d+ \w+) / SizeRange (?P\d+\.\d+) - Columnx \d+ / Taille Inexistante", "UNSYNCED_ORD_SD_FILES": r"Erreur : (?PGRP_\d+) / (?P\d{5}/\d{3}) (?P\d{4})_(?P\w+) WARNING (?P(?P\w+ (?P\w+)(?P\w{3}))) / OrderNumber : \d+ / Le modèle n'existe pas dans le fichier SD correspondant", "CODE_MODEL_SIZE_OUT_OF_BOUND": r"Erreur : (?PGRP_\d+) / (?P\d{5}/\d{3}) (?P\d{4})_(?P\w+) WARNING [\w ]+ \w+ / SetArticle -> SizeRange \d+\.\d+ / (?: \[[^\]]+\])+-\w+\. Données trop volumineuses pour la variable (?: \[[^\]]+\]). Long. max. = \[\d+\], long. réelle = \[\d+\] Conseil : attribuez une valeur supérieure au paramètre TFDParam.Size", "USER_NOT_CREATED_ON_NOSYMAG": r"Erreur : (?PGRP_\d+) / (?P\d{5}/\d{3}) (?P\d{4})_(?P\w+) ERREUR / OrderNumber : \d+ / Utilisateur : configuration incorrecte \(valeur 0\) - vérifier dans IF_Config", - "INCORRECT_MDC_XML_FILES": r"Exception : Impossible de convertir le variant de type \(Null\) en type \(OleStr\)" + "INCORRECT_MDC_XML_FILES": r"Exception : Impossible de convertir le variant de type \(Null\) en type \(OleStr\)", } # compiled all patterns -_COMPILED_N2_PATTERNS = {error_name: re.compile(pattern) for error_name, pattern in N2_PATTERNS.items()} +_COMPILED_N2_PATTERNS = { + error_name: re.compile(pattern) for error_name, pattern in N2_PATTERNS.items() +} -def _match_n2_pattern(log_entry: LogEntry) -> str|None: + +def _match_n2_pattern(log_entry: LogEntry) -> str | None: for error_code, pattern in _COMPILED_N2_PATTERNS.items(): if pattern.search(log_entry.error_message): return error_code return None -def classify_log_entries(log_entries: list[LogEntry]) -> tuple[list[LogEntry], list[LogEntry]]: + +def classify_log_entries( + log_entries: list[LogEntry], +) -> tuple[list[LogEntry], list[LogEntry]]: """ - Separate logs that require N2 intervention from those that don't. + Separate logs that require N2 intervention from those that don't. """ relevant_log_entries = [] irrelevant_log_entries = [] + logger.info("\tclassification job started.") for log_entry in log_entries: error_name = _match_n2_pattern(log_entry) if error_name: + logger.info( + f"\t\tError '{error_name}' associated with log entry '{log_entry.error_message}'. This is a n2 log entry" + ) log_entry.error_name = error_name relevant_log_entries.append(log_entry) else: irrelevant_log_entries.append(log_entry) - - return relevant_log_entries, irrelevant_log_entries \ No newline at end of file + + logger.info( + f"\t\tn2 logs: {len(relevant_log_entries)}, other logs: {len(irrelevant_log_entries)}" + ) + logger.info("\tclassification job completed.") + return relevant_log_entries, irrelevant_log_entries diff --git a/src/logwatcher/cli.py b/src/logwatcher/cli.py index cea6754..e2305e0 100644 --- a/src/logwatcher/cli.py +++ b/src/logwatcher/cli.py @@ -97,25 +97,27 @@ def cli( Returns: None """ - logger.info("logwatcher starts its work.") + logger.info("============ logwatcher started ============") if input_files and input_dir: raise typer.BadParameter( "Only one of input_files or input_dir must be provided." ) if input_files: - logger.info(f"input_files option selected.\n input_files: '{input_files}'") + logger.info(f"input_files option selected. input_files: {input_files}") log_files = input_files elif input_dir: - logger.info(f"input_dir option selected with value '{input_dir}'") + logger.info(f"input_dir option selected with value {input_dir}") log_files = _get_files_from_dir(input_dir) else: raise typer.BadParameter("Either input_files or input_dir must be provided") + logger.info(f"reports will be located at '{output_dir.cwd()}'") + log_entries = _files_to_log_entries(log_files) relevant, irrelevant = classify_log_entries(log_entries) write_log_report(relevant, irrelevant, len(log_files), output_dir) - logger.info("logwatcher has finished its task.") + logger.info("============ logwatcher ended ============") if __name__ == "__main__": diff --git a/src/logwatcher/parser.py b/src/logwatcher/parser.py index 5754720..a3450a8 100644 --- a/src/logwatcher/parser.py +++ b/src/logwatcher/parser.py @@ -36,53 +36,58 @@ effective_file_path = Path.cwd() / "output/effective.log" def parse_log_line(log_line: str) -> LogEntry | None: """Parse a raw log line into a LogEntry object. - Args: - log_line: a line in a log file received by N2 technicians + Args: + log_line: a line in a log file received by N2 technicians - Returns: - LogEntry if the line is a valid line, None if - it is empty or not a valid line. + Returns: + LogEntry if the line is a valid line, None if + it is empty or not a valid line. """ match = LOG_PATTERN.search(log_line) if not match: - logger.debug(f"Unrecognize format. Ignoring line {log_line}.") return None - + data = match.groupdict() return LogEntry( server_ip=data["server_ip"], mdc_server_name=data["mdc_server_name"], - start_time=datetime.strptime(data["start_time"], DATETIME_FORMAT).astimezone(FRENCH_TIMEZONE), + start_time=datetime.strptime(data["start_time"], DATETIME_FORMAT).astimezone( + FRENCH_TIMEZONE + ), store_name=data["store_name"], - error_time=datetime.strptime(data["error_time"], DATETIME_FORMAT).astimezone(FRENCH_TIMEZONE), + error_time=datetime.strptime(data["error_time"], DATETIME_FORMAT).astimezone( + FRENCH_TIMEZONE + ), error_message=data["error_message"], - raw_line=log_line + raw_line=log_line, ) + def parse_log_file(log_file_path: Path) -> list[LogEntry]: """Transforms the content of a log file into a list of LogEntry. - Uses `parse_log_line` to parse each line of the choosen log file. - Raise exception if a log line encounters an error. + Uses `parse_log_line` to parse each line of the choosen log file. + Raise exception if a log line encounters an error. - Args: - log_file_path: Path of a log file - - Returns: - A list of LogEntry + Args: + log_file_path: Path of a log file + + Returns: + A list of LogEntry """ log_entry_list: list[LogEntry] = [] - with open(log_file_path, 'r', encoding="windows-1252") as log_file: + with open(log_file_path, "r", encoding="windows-1252") as log_file: + logger.info(f"\tparsing log file '{log_file_path.name}' started.") for index, log_line in enumerate(log_file): try: log_entry = parse_log_line(log_line) except ValueError as error: - logger.error(f"Error while parsing the log file {log_file.name} at {log_line}: {error}") + logger.error( + f"\t\tline {index}: error while parsing the log file '{log_file.name}': {error}" + ) else: if log_entry: log_entry_list.append(log_entry) - logger.debug(f"Line {index} was successfully parsed.") - else: - logger.debug(f"Line {index} is empty. Skip.") - - return log_entry_list \ No newline at end of file + logger.info(f"\t\t{len(log_entry_list)} lines were parsed.") + logger.info("\tparsing completed.") + return log_entry_list diff --git a/src/logwatcher/reporter.py b/src/logwatcher/reporter.py index 9cc065d..0bc71a0 100644 --- a/src/logwatcher/reporter.py +++ b/src/logwatcher/reporter.py @@ -2,6 +2,7 @@ from datetime import datetime from pathlib import Path from string import Template +from logwatcher import logger from logwatcher.config import DATETIME_FORMAT, RESULT_PATH from logwatcher.models import LogEntry @@ -59,15 +60,30 @@ def write_log_report( file. Use the range date of relevant and irrelevant lists to get the period of time the logs were generated. + + Args: + relevant: List of n2 log entry + irrelevant: List of not n2 log entry + nb_files: Number of files scanned + output_dir: Location where all reports will be written + Returns: + None """ + logger.info("\twriting reports job started.") start_date, end_date = _get_period(relevant + irrelevant) + logger.info(f"\t\tperiod found: {start_date} -> {end_date}") reports_dict = build_reports(relevant, irrelevant, start_date, end_date, nb_files) for target in reports_dict: + logger.info(f"\t\tgenerating report '{target}'...") output_dir.mkdir(parents=True, exist_ok=True) + logger.debug(f"\t\t\t{output_dir} exists.") target_file = output_dir / str(target + ".log") target_file.write_text(reports_dict[target], encoding="windows-1252") + logger.debug(f"\t\t\treport was written in {target_file.name}") + logger.info(f"\t\treport '{target}' was generated.") + logger.info("\twriting reports job completed.") def build_reports( @@ -77,7 +93,17 @@ def build_reports( Build the three output reports: n2, other, and all. "all" report contains both "n2" and "other" sections. "n2" and "other" are two unique sections. + + Args: + relevant: List of n2 log entry + irrelevant: List of not n2 log entry + start_date: Date of the oldest log entry in relevant + irrelevant list + end_date: Date of the newest log entry in relevant + irrelevant list + nb_files: number of files scanned + Returns: + Dictionary of reports in string format """ + logger.info("\t\tbuilding reports job started.") relevant_report = _render_target_report( log_entries=relevant, target_template=N2_SUPPORT_TEMPLATE ) @@ -85,6 +111,8 @@ def build_reports( irrelevant_report = _render_target_report( log_entries=irrelevant, target_template=OTHER_TEMPLATE ) + logger.info("\t\t\tn2 and other reports were generated.") + logger.info("\t\tbuilding reports job completed.") return { "n2": BASE_TEMPLATE.substitute( @@ -112,11 +140,19 @@ def build_reports( # build templates -def _render_target_report(log_entries: list[LogEntry], target_template) -> str: +def _render_target_report( + log_entries: list[LogEntry], target_template: Template +) -> str: """ Render report using either N2_SUPPORT_TEMPLATE or OTHER_TEMPLATE. Call _render_entries() and include rendered log entries in the new report. + + Args: + log_entries: List of log entries + target_template: Template to use for N2_SUPPORT_TEMPLATE + Returns: + content of report in string format """ error_list = _render_entries(log_entries) return target_template.substitute(nb_errors=len(log_entries), error_list=error_list) @@ -126,6 +162,11 @@ def _render_entries(log_entries: list[LogEntry]) -> str: """ Render a list of log entries using the `ERROR_TEMPLATE`. Used in generated report files in the `output` directory + + Args: + log_entries: List of log entries + Returns: + error report in string format """ error_list = [] for index, log_entry in enumerate(log_entries, start=1):