[Logging] Switch from Standard Logging to Loguru

As loguru is used in some places, we update other logging instances
for consistency with its behavior

* metrics/logger.py: Change `level: Optional[int]` to
  `level: Optional[Union[int, str]]` to better support loguru's logging
  level behavior
* Update `PythonLogger._create_default_logger` to use loguru.logger
* PythonLogger: Add class-level variable `_global_file_sink_id` to prevent
  duplicate file sink creations and remove manual stream handler
  setup since loguru adds a default `sys.stderr` sink
* Simplify `PythonLogger` by no longer accepting an external logger instance,
  because the main concept of loguru is that there is one and only one
  `logger`
* Simplify `SparsificationGroupLogger` by replacing the `python` union
  type with `bool`, aligning it with `log_python` in `LoggerManager` and
  removing ambiguity

Author: dichn

src/llmcompressor/metrics/logger.py

@@ -2,14 +2,13 @@
 Contains code for loggers that help visualize the information from each modifier
 """
 
-import logging
 import os
 import time
 import warnings
 from abc import ABC
+from loguru import logger
 from contextlib import contextmanager
 from datetime import datetime
-from logging import CRITICAL, DEBUG, ERROR, INFO, WARN, Logger
 from pathlib import Path
 from types import ModuleType
 from typing import Any, Callable, Dict, List, Optional, Union
@@ -52,16 +51,8 @@
     "WANDBLogger",
     "SparsificationGroupLogger",
     "LoggerManager",
-    "LOGGING_LEVELS",
 ]
 ALL_TOKEN = "__ALL__"
-LOGGING_LEVELS = {
-    "debug": DEBUG,
-    "info": INFO,
-    "warn": WARN,
-    "error": ERROR,
-    "critical": CRITICAL,
-}
 DEFAULT_TAG = "defaul_tag"
 
 
@@ -231,7 +222,7 @@ def lambda_func(
     def log_hyperparams(
         self,
         params: Dict,
-        level: Optional[int] = None,
+        level: Optional[Union[int, str]] = None,
     ) -> bool:
         """
         :param params: Each key-value pair in the dictionary is the name of the
@@ -256,7 +247,7 @@ def log_scalar(
         value: float,
         step: Optional[int] = None,
         wall_time: Optional[float] = None,
-        level: Optional[int] = None,
+        level: Optional[Union[int, str]] = None,
     ) -> bool:
         """
         :param tag: identifying tag to log the value with
@@ -285,7 +276,7 @@ def log_scalars(
         values: Dict[str, float],
         step: Optional[int] = None,
         wall_time: Optional[float] = None,
-        level: Optional[int] = None,
+        level: Optional[Union[int, str]] = None,
     ) -> bool:
         """
         :param tag: identifying tag to log the values with
@@ -313,40 +304,28 @@ class PythonLogger(LambdaLogger):
     """
     Modifier metrics that handles printing values into a python metrics instance.
 
-    :param logger: a metrics instance to log to, if None then will create it's own
-    :param log_level: default level to log any incoming data at on the logging.Logger
-        instance when an explicit log level isn't provided
     :param name: name given to the metrics, used for identification;
         defaults to python
     :param enabled: True to log, False otherwise
     """
 
+    # Class-level variable to track if file sink is created
+    _global_file_sink_id = None
+
     def __init__(
         self,
-        logger: Logger = None,
-        log_level: int = None,
         name: str = "python",
         enabled: bool = True,
     ):
-        self._logger = logger or self._create_default_logger(log_level=log_level)
+        self._create_default_logger()
 
         super().__init__(
             lambda_func=self._log_lambda,
             name=name,
             enabled=enabled,
         )
 
-    def __getattr__(self, item):
-        return getattr(self._logger, item)
-
-    @property
-    def logger(self) -> Logger:
-        """
-        :return: a metrics instance to log to, if None then will create it's own
-        """
-        return self._logger
-
-    def _create_default_logger(self, log_level: Optional[int] = None) -> logging.Logger:
+    def _create_default_logger(self) -> None:
         """
         Create a default modifier metrics,
         with a file handler logging at the debug level
@@ -355,24 +334,9 @@ def _create_default_logger(self, log_level: Optional[int] = None) -> logging.Log
         :param log_level: logging level for the console metrics
         :return: metrics
         """
-        logger = logging.getLogger(__name__)
-
-        # Console handler, for logging high level modifier logs
-        # must be created before the file handler
-        # as file handler is also a stream handler
-        if not any(
-            isinstance(handler, logging.StreamHandler) for handler in logger.handlers
-        ):
-            stream_handler = logging.StreamHandler()
-            stream_handler.setLevel(
-                log_level or logging.getLogger("llmcompressor").level
-            )
-            logger.addHandler(stream_handler)
 
         # File handler setup, for logging modifier debug statements
-        if not any(
-            isinstance(handler, logging.FileHandler) for handler in logger.handlers
-        ):
+        if PythonLogger._global_file_sink_id is None:
             base_log_path = (
                 os.environ.get("NM_TEST_LOG_DIR")
                 if os.environ.get("NM_TEST_MODE")
@@ -382,27 +346,19 @@ def _create_default_logger(self, log_level: Optional[int] = None) -> logging.Log
             dt_string = now.strftime("%d-%m-%Y_%H.%M.%S")
             log_path = os.path.join(base_log_path, f"{dt_string}.log")
             os.makedirs(base_log_path, exist_ok=True)
-            file_handler = logging.FileHandler(
-                log_path,
-                delay=True,
+            PythonLogger._global_file_sink_id = logger.add(
+                log_path, level="DEBUG", delay=True
             )
-            file_handler.setLevel(LOGGING_LEVELS["debug"])
-            logger.addHandler(file_handler)
             logger.info(f"Logging all LLM Compressor modifier-level logs to {log_path}")
 
-        logger.setLevel(LOGGING_LEVELS["debug"])
-        logger.propagate = False
-
-        return logger
-
     def _log_lambda(
         self,
         tag: Optional[str],
         value: Optional[Union[float, str]],
         values: Optional[Dict[str, float]],
         step: Optional[int],
         wall_time: Optional[float],
-        level: Optional[int] = None,
+        level: Optional[Union[int, str]] = None,
     ) -> bool:
         """
         :param tag: identifying tag to log the values with
@@ -415,9 +371,11 @@ def _log_lambda(
         :return: True if logged, False otherwise.
         """
         if not level:
-            level = LOGGING_LEVELS["debug"]
+            level = "DEBUG"
 
-        if level > LOGGING_LEVELS["debug"]:
+        if (isinstance(level, int) and level > logger.level("DEBUG").no) or (
+            isinstance(level, str) and logger.level(level).no > logger.level("DEBUG").no
+        ):
             if step is not None:
                 format = "%s %s step %s: %s"
                 log_args = [
@@ -433,7 +391,7 @@ def _log_lambda(
             format = "%s %s [%s - %s]: %s"
             log_args = [self.name, tag, step, wall_time, values or value]
 
-        self._logger.log(level, format, *log_args)
+        logger.log(level, format, *log_args)
 
         return True
 
@@ -443,7 +401,7 @@ def log_string(
         string: Optional[str],
         step: Optional[int],
         wall_time: Optional[float] = None,
-        level: Optional[int] = None,
+        level: Optional[Union[int, str]] = None,
     ) -> bool:
         """
         :param tag: identifying tag to log the values with
@@ -540,7 +498,7 @@ def _log_lambda(
         values: Optional[Dict[str, float]],
         step: Optional[int],
         wall_time: Optional[float],
-        level: Optional[int] = None,
+        level: Optional[Union[int, str]] = None,
     ) -> bool:
         if value is not None:
             self._writer.add_scalar(tag, value, step, wall_time)
@@ -614,7 +572,7 @@ def _log_lambda(
         values: Optional[Dict[str, float]],
         step: Optional[int],
         wall_time: Optional[float],
-        level: Optional[int] = None,
+        level: Optional[Union[int, str]] = None,
     ) -> bool:
         params = {}
 
@@ -656,11 +614,10 @@ class SparsificationGroupLogger(BaseLogger):
     :param lambda_func: an optional lambda function to call back into with any logs.
         The expected call sequence is (tag, value, values, step, wall_time) -> bool
         The return type is True if logged and False otherwise.
-    :param python: an optional argument for logging to a python metrics.
-        May be a logging.Logger instance to log to, True to create a metrics instance,
-        or non truthy to not log anything (False, None)
+    :param python: an bool argument for logging to a python metrics.
+        True to create a metrics instance, or False to not log anything
     :param python_log_level: if python,
-        the level to log any incoming data at on the logging.Logger instance
+        the level to log any incoming data at on the loguru.logger instance
     :param tensorboard: an optional argument for logging to a tensorboard writer.
         May be a SummaryWriter instance to log to, a string representing the directory
         to create a new SummaryWriter to log to, True to create a new SummaryWriter,
@@ -688,8 +645,8 @@ def __init__(
                 bool,
             ]
         ] = None,
-        python: Optional[Union[bool, Logger]] = None,
-        python_log_level: int = logging.INFO,
+        python: bool = False,
+        python_log_level: Optional[Union[int, str]] = "INFO",
         tensorboard: Optional[Union[bool, str, SummaryWriter]] = None,
         wandb_: Optional[Union[bool, Dict]] = None,
         name: str = "sparsification",
@@ -706,8 +663,6 @@ def __init__(
         if python:
             self._loggers.append(
                 PythonLogger(
-                    logger=python if isinstance(python, Logger) else None,
-                    log_level=python_log_level,
                     name=name,
                     enabled=enabled,
                 )
@@ -751,7 +706,7 @@ def loggers(self) -> List[BaseLogger]:
         """
         return self._loggers
 
-    def log_hyperparams(self, params: Dict, level: Optional[int] = None):
+    def log_hyperparams(self, params: Dict, level: Optional[Union[int, str]] = None):
         """
         :param params: Each key-value pair in the dictionary is the name of the
             hyper parameter and it's corresponding value.
@@ -765,7 +720,7 @@ def log_scalar(
         value: float,
         step: Optional[int] = None,
         wall_time: Optional[float] = None,
-        level: Optional[int] = None,
+        level: Optional[Union[int, str]] = None,
     ):
         """
         :param tag: identifying tag to log the value with
@@ -783,7 +738,7 @@ def log_scalars(
         values: Dict[str, float],
         step: Optional[int] = None,
         wall_time: Optional[float] = None,
-        level: Optional[int] = None,
+        level: Optional[Union[int, str]] = None,
     ):
         """
         :param tag: identifying tag to log the values with
@@ -956,7 +911,7 @@ def log_scalar(
         step: Optional[int] = None,
         wall_time: Optional[float] = None,
         log_types: Union[str, List[str]] = ALL_TOKEN,
-        level: Optional[int] = None,
+        level: Optional[Union[int, str]] = None,
     ):
         """
         (Note: this method is deprecated and will be removed in a future version,
@@ -986,7 +941,7 @@ def log_scalars(
         step: Optional[int] = None,
         wall_time: Optional[float] = None,
         log_types: Union[str, List[str]] = ALL_TOKEN,
-        level: Optional[int] = None,
+        level: Optional[Union[int, str]] = None,
     ):
         """
         (Note: this method is deprecated and will be removed in a future version,
@@ -1013,7 +968,7 @@ def log_hyperparams(
         self,
         params: Dict,
         log_types: Union[str, List[str]] = ALL_TOKEN,
-        level: Optional[int] = None,
+        level: Optional[Union[int, str]] = None,
     ):
         """
         (Note: this method is deprecated and will be removed in a future version,
@@ -1036,7 +991,7 @@ def log_string(
         step: Optional[int] = None,
         wall_time: Optional[float] = None,
         log_types: Union[str, List[str]] = ALL_TOKEN,
-        level: Optional[int] = None,
+        level: Optional[Union[int, str]] = None,
     ):
         """
         (Note: this method is deprecated and will be removed in a future version,
@@ -1119,7 +1074,7 @@ def log_string(
         step: Optional[int] = None,
         wall_time: Optional[float] = None,
         log_types: Union[str, List[str]] = ALL_TOKEN,
-        level: Optional[int] = None,
+        level: Optional[Union[int, str]] = None,
     ):
         """
         :param tag: identifying tag to log the values with
@@ -1151,7 +1106,7 @@ def debug(self, tag, string, *args, **kwargs):
         :param kwargs: additional arguments to pass to the metrics,
             see `log_string` for more details
         """
-        kwargs["level"] = logging.DEBUG
+        kwargs["level"] = "DEBUG"
         self.log_string(tag=tag, string=string, *args, **kwargs)
 
     def info(self, tag, string, *args, **kwargs):
@@ -1166,7 +1121,7 @@ def info(self, tag, string, *args, **kwargs):
         :param kwargs: additional arguments to pass to the metrics,
             see `log_string` for more details
         """
-        kwargs["level"] = logging.INFO
+        kwargs["level"] = "INFO"
         self.log_string(tag=tag, string=string, *args, **kwargs)
 
     def warning(self, tag, string, *args, **kwargs):
@@ -1181,7 +1136,7 @@ def warning(self, tag, string, *args, **kwargs):
         :param kwargs: additional arguments to pass to the metrics,
             see `log_string` for more details
         """
-        kwargs["level"] = logging.WARNING
+        kwargs["level"] = "WARNING"
         self.log_string(tag=tag, string=string, *args, **kwargs)
 
     def warn(self, tag, string, *args, **kwargs):
@@ -1204,7 +1159,7 @@ def error(self, tag, string, *args, **kwargs):
         :param kwargs: additional arguments to pass to the metrics,
             see `log_string` for more details
         """
-        kwargs["level"] = logging.ERROR
+        kwargs["level"] = "ERROR"
         self.log_string(tag=tag, string=string, *args, **kwargs)
 
     def critical(self, tag, string, *args, **kwargs):
@@ -1219,7 +1174,7 @@ def critical(self, tag, string, *args, **kwargs):
         :param kwargs: additional arguments to pass to the metrics,
             see `log_string` for more details
         """
-        kwargs["level"] = logging.CRITICAL
+        kwargs["level"] = "CRITICAL"
         self.log_string(tag=tag, string=string, *args, **kwargs)
 
 
@@ -1232,7 +1187,7 @@ def log_hyperparams(
         self,
         params: Dict,
         log_types: Union[str, List[str]] = ALL_TOKEN,
-        level: Optional[int] = None,
+        level: Optional[Union[int, str]] = None,
     ):
         """
         :param params: Each key-value pair in the dictionary is the name of the
@@ -1249,7 +1204,7 @@ def log_scalar(
         step: Optional[int] = None,
         wall_time: Optional[float] = None,
         log_types: Union[str, List[str]] = ALL_TOKEN,
-        level: Optional[int] = None,
+        level: Optional[Union[int, str]] = None,
     ):
         """
         :param tag: identifying tag to log the value with
@@ -1276,7 +1231,7 @@ def log_scalars(
         step: Optional[int] = None,
         wall_time: Optional[float] = None,
         log_types: Union[str, List[str]] = ALL_TOKEN,
-        level: Optional[int] = None,
+        level: Optional[Union[int, str]] = None,
     ):
         """
         :param tag: identifying tag to log the values with