Skip to content

Commit

Permalink
detail log for llm (#2325)
Browse files Browse the repository at this point in the history
* add inference load balancer for fastdeploy llm

* add inference load balance controller for llm

* add ic for llm

* add ic for llm

* add fastdeploy ic for llm

* add fastdeploy ic to llm

* Fix asyncio.CancelError exception

* Improve robust for llm service

* Improve robust for llm service

* Add detailed log for llm service

* Add detailed log for llm service

* Add detailed log for llm service

* Add detailed log for llm service

* Add detailed log for llm service

* add detailed log

* add detailed log
  • Loading branch information
rainyfly authored Dec 14, 2023
1 parent fda8c37 commit cc89731
Show file tree
Hide file tree
Showing 3 changed files with 23 additions and 18 deletions.
15 changes: 1 addition & 14 deletions llm/fastdeploy_llm/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import logging
import fastdeploy_llm
from fastdeploy_llm.utils.utils import check_model
from fastdeploy_llm.utils.logging_util import logger, Logger
from fastdeploy_llm.utils.logging_util import logger


class Config:
Expand All @@ -26,27 +26,14 @@ def __init__(self, model_dir, decode_strategy="sampling", mp_num=None):
is_static, rank = check_model(model_dir)

self.log_home = os.getenv("LOG_HOME", ".")
fastdeploy_llm.utils.logging_util.warning_logger = Logger(
name="fastDeploy_llm_serving_warning",
log_file=os.path.join(self.log_home, "fastdeploy_llm_serving_warning.log"),
time_rotation=7,
level=logging.DEBUG)
if os.getenv("ENABLE_DEBUG_LOG", "0") == "1":
logger.info(
"Detect enviroment variable `ENABLE_DEBUG_LOG`, all the debug log information will output to fastdeploy_llm_serving.log."
)
fastdeploy_llm.utils.logging_util.logger = Logger(
log_file=os.path.join(self.log_home, "fastdeploy_llm_serving.log"),
time_rotation=7,
level=logging.DEBUG)
else:
logger.info(
"The logging level is set as INFO, if more information needed, please execute `export ENABLE_DEBUG_LOG=1` before launching service."
)
fastdeploy_llm.utils.logging_util.logger = Logger(
log_file=os.path.join(self.log_home, "fastdeploy_llm_serving.log"),
time_rotation=7,
level=logging.INFO)

assert decode_strategy in [
"sampling", "greedy_search"
Expand Down
4 changes: 2 additions & 2 deletions llm/fastdeploy_llm/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,10 +165,10 @@ def _init_engine(self, configs):
ret = self.engine_proc.poll()
if ret is not None:
logger.error(
"The engine launch failed, check log/workerlog for more details."
"The engine launch failed, check log/infer.log for more details."
)
raise Exception(
"The engine launch failed, check log/workerlog for more details."
"The engine launch failed, check log/infer.log for more details."
)
logger.info("Paddle Inference Engine intialized successed!")

Expand Down
22 changes: 20 additions & 2 deletions llm/fastdeploy_llm/utils/logging_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import contextlib
import logging
import threading
Expand Down Expand Up @@ -172,5 +173,22 @@ def use_terminator(self, terminator: str) -> Generator[None, None, None]:
self.handler.terminator = old_terminator


logger = Logger()
warning_logger = Logger(name="fastDeploy_llm_serving_warning")
log_home = os.getenv("LOG_HOME", ".")

if os.getenv("ENABLE_DEBUG_LOG", "0") == "1":
logger = Logger(
name="fastdeploy_llm_serving",
log_file=os.path.join(log_home, "fastdeploy_llm_serving.log"),
time_rotation=7,
level=logging.DEBUG)
else:
logger = Logger(
log_file=os.path.join(log_home, "fastdeploy_llm_serving.log"),
time_rotation=7,
level=logging.INFO)

warning_logger = Logger(
name="fastDeploy_llm_serving_warning",
log_file=os.path.join(log_home, "fastdeploy_llm_serving_warning.log"),
time_rotation=7,
level=logging.DEBUG)

0 comments on commit cc89731

Please sign in to comment.