From cb5257ea64aaedce3bd196262acaa27991553caf Mon Sep 17 00:00:00 2001 From: sufubao Date: Sat, 17 Oct 2026 19:54:41 +0800 Subject: [PATCH 01/13] refactor(logging): clarify lifecycle and bound runtime diagnostics --- docs/CN/source/index.rst | 1 + docs/CN/source/tutorial/logging.rst | 209 +++++ docs/EN/source/index.rst | 1 + docs/EN/source/tutorial/logging.rst | 198 +++++ .../basemodel/attention/create_utils.py | 10 +- .../common/basemodel/attention/fa3/neo.py | 23 +- lightllm/common/basemodel/basemodel.py | 52 +- lightllm/common/basemodel/cuda_graph.py | 20 +- .../layer_infer/cache_tensor_manager.py | 5 +- .../basemodel/layer_weights/hf_load_utils.py | 22 +- .../common/basemodel/prefill_cuda_graph.py | 30 +- .../fused_moe/grouped_fused_moe_ep.py | 2 +- .../common/kv_cache_mem_manager/allocator.py | 2 - .../kv_cache_mem_manager/mem_manager.py | 7 +- .../common/kv_cache_mem_manager/mem_utils.py | 10 +- lightllm/common/quantization/__init__.py | 11 +- lightllm/common/req_manager/base.py | 4 - lightllm/common/triton_utils/autotuner.py | 24 +- lightllm/distributed/communication_op.py | 2 +- lightllm/distributed/flashinfer_all_reduce.py | 9 +- lightllm/models/qwen3next/model.py | 3 +- lightllm/server/api_anthropic.py | 4 +- lightllm/server/api_cli.py | 49 ++ lightllm/server/api_http.py | 65 +- lightllm/server/api_http_pd.py | 25 +- lightllm/server/api_openai.py | 4 +- lightllm/server/api_responses.py | 4 +- lightllm/server/api_server.py | 13 +- lightllm/server/api_start.py | 40 +- .../audioserver/model_infer/model_rpc.py | 8 +- lightllm/server/build_prompt.py | 8 +- lightllm/server/config_server/api_http.py | 25 +- lightllm/server/core/objs/shm_req_manager.py | 2 - lightllm/server/core/objs/start_args_type.py | 7 + lightllm/server/detokenization/manager.py | 15 +- lightllm/server/embed_cache/utils.py | 6 +- lightllm/server/health_monitor/manager.py | 14 +- lightllm/server/httpserver/manager.py | 148 ++-- lightllm/server/httpserver/pd_loop.py | 35 +- .../httpserver_for_pd_master/manager.py | 103 ++- .../pd_selector/cache_aware.py | 40 +- .../pd_selector/pd_selector.py | 12 +- .../httpserver_for_pd_master/register_loop.py | 7 +- lightllm/server/metrics/manager.py | 2 +- lightllm/server/router/batch.py | 11 +- lightllm/server/router/engine_stats.py | 70 ++ lightllm/server/router/manager.py | 113 ++- .../server/router/model_infer/infer_batch.py | 25 +- .../model_infer/mode_backend/base_backend.py | 39 +- .../chunked_prefill/impl_for_reward_model.py | 3 +- .../mode_backend/diverse_backend/impl.py | 10 +- .../dp_backend/dp_shared_kv_trans.py | 8 +- .../decode_kv_move_manager.py | 8 +- .../decode_node_impl/decode_trans_process.py | 54 +- .../pd/decode_node_impl/up_status.py | 20 +- .../mode_backend/pd/nccl_kv_transporter.py | 13 +- .../mode_backend/pd/nixl_kv_transporter.py | 33 +- .../prefill_kv_move_manager.py | 9 +- .../prefill_trans_process.py | 85 +- .../server/router/model_infer/model_rpc.py | 16 +- .../chunked_prefill/impl_for_pd_prefill.py | 3 +- .../visualserver/model_infer/model_rpc.py | 8 +- .../visualserver/visual_only_manager.py | 8 +- lightllm/utils/config_utils.py | 20 +- lightllm/utils/device_utils.py | 6 +- lightllm/utils/dist_check_utils.py | 11 +- lightllm/utils/dist_utils.py | 4 + lightllm/utils/embed_utils.py | 2 +- lightllm/utils/graceful_utils.py | 3 + lightllm/utils/log_utils.py | 746 +++++++++++++++--- lightllm/utils/multimodal_utils.py | 6 +- lightllm/utils/net_utils.py | 4 +- lightllm/utils/rpyc_fix_utils.py | 2 +- lightllm/utils/sgl_utils.py | 4 +- lightllm/utils/shm_port_args.py | 4 +- lightllm/utils/start_utils.py | 102 ++- lightllm/utils/statics_utils.py | 6 +- lightllm/utils/vllm_utils.py | 2 +- .../basemodel/attention/fa3/test_neo.py | 54 +- unit_tests/common/test_autotuner.py | 18 + .../httpserver/test_pd_generate_error.py | 37 + .../test_pd_master_cached_tokens.py | 29 +- .../mode_backend/test_pd_prefill_abort.py | 71 ++ unit_tests/server/test_engine_throughput.py | 195 +++++ unit_tests/server/test_logging.py | 146 ++++ unit_tests/utils/test_debug_logging.py | 173 ++++ unit_tests/utils/test_log_utils.py | 530 +++++++++++++ unit_tests/utils/test_start_utils.py | 27 + 88 files changed, 3328 insertions(+), 691 deletions(-) create mode 100644 docs/CN/source/tutorial/logging.rst create mode 100644 docs/EN/source/tutorial/logging.rst create mode 100644 lightllm/server/router/engine_stats.py create mode 100644 unit_tests/server/test_engine_throughput.py create mode 100644 unit_tests/server/test_logging.py create mode 100644 unit_tests/utils/test_debug_logging.py diff --git a/docs/CN/source/index.rst b/docs/CN/source/index.rst index a594e31066..fe04dcdfb1 100755 --- a/docs/CN/source/index.rst +++ b/docs/CN/source/index.rst @@ -48,6 +48,7 @@ Lightllm 整合了众多的开源方案的优点，包括但不限于 FasterTran :maxdepth: 1 :caption: 部署教程 + 日志配置与分级 DeepSeek R1 部署 FP8 KV 量化与校准 多级缓存部署 diff --git a/docs/CN/source/tutorial/logging.rst b/docs/CN/source/tutorial/logging.rst new file mode 100644 index 0000000000..29526f118a --- /dev/null +++ b/docs/CN/source/tutorial/logging.rst @@ -0,0 +1,209 @@ +日志配置与分级规范 +================== + +默认行为 +-------- + +LightLLM 使用 Python 标准库 logging。默认服务级别为 INFO，输出到 stdout。 +模块 logger 继承 ``lightllm`` 的配置；模块本身不添加 handler。 +默认文本日志使用 ``HH:mm:ss.SSS LEVEL [组件] 消息``，例如 ``14:51:26.838 INFO [infer] event=backend.selected ...``。 +P/D 角色合并到组件中，例如 ``[P/infer]`` 和 ``[D/http]``。 +多卡、多 DP、多节点时才显示对应的已知 rank，例如 ``[P/infer tp=1]``；未知 rank 不输出。 +主机名、PID、完整模块名在默认文本中省略；DEBUG、ERROR/CRITICAL 和附带 traceback 的日志保留文件行号。 +JSON 保留带日期/时区的毫秒时间、主机、PID、完整模块/组件、并行拓扑和来源文件行号。 +进程上下文不向 fork/spawn 子进程传递，子进程自行初始化；节点和 P/D 角色通过环境变量传递。 + +HTTP access 默认关闭；成功请求完成摘要默认采样 1%，失败摘要不采样。 +请求正文、采样参数、token ID 和张量不会因为开启请求摘要而输出。 +现有 ``--detail_log`` 保留为显式内容诊断开关，其 DEBUG 输出仍需要对应模块启用 DEBUG。 + +配置 +---- + +显式 CLI 参数优先于环境变量；未设置 CLI 时继承环境变量，最后使用默认值。 +CLI 的连字符和下划线形式均可使用，例如 ``--log-level`` 和 ``--log_level``。 +启动器将配置写入环境变量，使 spawn 和 HTTP 子进程使用同一配置。 + +.. list-table:: + :header-rows: 1 + :widths: 27 38 35 + + * - CLI + - 环境变量 + - 默认值 / 含义 + * - ``--log-level`` + - ``LIGHTLLM_LOG_LEVEL`` + - ``info``；服务日志级别 + * - ``--log-format`` + - ``LIGHTLLM_LOG_FORMAT`` + - ``text``；也支持 ``json`` + * - ``--log-module-levels`` + - ``LIGHTLLM_LOG_MODULE_LEVELS`` + - ``{}``；模块前缀到级别的 JSON 映射 + * - ``--log-request-sample-rate`` + - ``LIGHTLLM_LOG_REQUEST_SAMPLE_RATE`` + - ``0.01``；成功请求采样 1%，0 关闭，1 全量；失败不采样 + * - ``--log-stats-interval`` + - ``LIGHTLLM_LOG_STATS_INTERVAL`` + - ``10``；运行摘要间隔，单位为秒，必须大于 0 + * - ``--log-http-level`` + - ``LIGHTLLM_LOG_HTTP_LEVEL`` + - ``warning``；Hypercorn 错误日志级别 + * - ``--enable-http-access-log`` + - ``LIGHTLLM_LOG_HTTP_ACCESS`` + - ``0``；设为 1 开启 HTTP access 日志 + * - 无 + - ``LIGHTLLM_LOG_HTTP_EXCLUDE_PATHS`` + - ``/health,/healthz,/readiness,/liveness,/metrics``；应用 access 日志排除的路径及其子路径，逗号分隔；空字符串取消排除 + * - 无 + - ``LIGHTLLM_LOG_DIR`` + - 未设置；设置后同时写入按 PID 隔离的日志文件 + * - 无 + - ``LIGHTLLM_LOG_MAX_BYTES`` + - ``104857600``；单文件轮转阈值，单位为字节 + * - 无 + - ``LIGHTLLM_LOG_BACKUP_COUNT`` + - ``3``；每进程保留的轮转备份数，必须大于 0 + +只排查调度模块，不开启其他模块 DEBUG： + +.. code-block:: bash + + export LIGHTLLM_LOG_LEVEL=INFO + export LIGHTLLM_LOG_MODULE_LEVELS='{"lightllm.server.router":"DEBUG"}' + +结构化输出，并抽样记录 1% 的请求完成摘要： + +.. code-block:: bash + + export LIGHTLLM_LOG_FORMAT=json + export LIGHTLLM_LOG_REQUEST_SAMPLE_RATE=0.01 + +``lightllm.requests`` 和 ``lightllm.access`` 在各自开启时使用 INFO，不受全局服务级别提高的影响； +可通过模块覆盖将它们调为 WARNING 等级以进一步过滤。 +应用 access 日志由 LightLLM 中间件输出，避免再开启一份 Hypercorn access 日志。 +配置服务使用 Hypercorn access；应用的路径过滤仅适用于 LightLLM 中间件。 +用户提供的 Hypercorn 配置文件也可以自行定义 access 输出，需要避免重复配置。 +Python warnings 通过统一日志输出；原生库 stdout/stderr 和 Hypercorn 自身日志不强制转换为 LightLLM JSON 格式。 + +输出类别与范围 +-------------- + +* 服务日志：生命周期、最终配置、重要状态变化、降级与失败。 +* ``lightllm.stats``：节点内 router 的 running、waiting、paused，以及各本地 DP 的 KV 使用率快照。 + 所有摘要严格遵守配置间隔；持续空闲满一个间隔才输出 idle，恢复工作后按间隔输出；持续空闲不重复输出。 + 多节点纯 TP 由主节点输出，避免镜像 router 重复记录。 + 队列与 KV 是瞬时快照，吞吐使用引擎计数的窗口增量；长期吞吐和延迟分析使用 Prometheus 指标。 +* ``lightllm.requests``：请求完成摘要，包含内部请求 ID、输入/输出 token 数、TTFT、总耗时和缓存/MTP 信息。 + normal/P/D 与 PD master 采用同一基础字段；P/D 摘要表示该角色内的处理结果。 + PD 分段请求的摘要保留对应分段的请求 ID，不代表完整客户端请求的汇总。 +* ``lightllm.access``：HTTP 方法、路径、状态码和耗时；不记录 query、请求正文或认证头。 + +JSON 每行一个对象，额外字段可用于检索；异常 traceback 位于 ``exception`` 字段。 +文本模式给多行消息和 traceback 的每一行补齐前缀。 +各进程共享的模型配置、后端选择和 CUDA graph 初始化信息，在 TP 组代表 rank 上使用 INFO， +其他 rank 使用 DEBUG。各 worker 的 WARNING/ERROR 不应用这种过滤。 +节点硬件、实际容量和 worker 故障仍需保留发生位置。 + +写日志的规则 +------------ + +.. list-table:: + :header-rows: 1 + + * - 级别 + - 使用条件 + * - DEBUG + - 请求流转、批次选择、缓存分配/释放、KV 传输步骤、IPC 和配置推导细节。 + * - INFO + - 启动阶段、最终配置、后端选择、模型加载完成、容量、ready、周期汇总和重要状态变化。 + * - WARNING + - 非预期但能够恢复、重试或降级的情况；避免把正常取消、客户端断连和幂等清理当作警告。 + * - ERROR + - 一次应完成的操作最终失败，例如推理失败或 KV 传输失败。 + * - CRITICAL + - 服务或进程无法继续可靠工作，例如健康监控决定终止服务。 + +``logger.exception`` 是附带 traceback 的 ERROR，不是独立日志级别。 +同一个捕获边界中不要连续使用 ``error`` 和 ``exception`` 记录同一个失败。 +中间层继续抛出的异常应避免层层重复记录完整 traceback。 +未知异常的 traceback 应由能明确处理失败的请求、后台任务或进程边界记录。 + +消息使用稳定的英文描述，新增事件使用 ``event`` 字段；请求/批次/传输日志携带相应 ID。 +不要直接在日志中输出完整启动参数或对象 repr；启动摘要仅列出允许的运行参数，避免泄露密钥。 +昂贵的调试参数需要在 ``isEnabledFor(logging.DEBUG)`` 判断之后构建， +使用 ``logger.debug("Batch: %s", batch.simple_log())`` 仍会提前执行 ``simple_log``。 + +升级说明 +-------- + +原来的默认 DEBUG 改为 INFO。需要原有详细输出时显式设置 DEBUG，推荐只打开相关模块。 +成功请求完成摘要默认采样 1%；HTTP access 按需显式开启。 +设置日志目录后统一写入 ``process..log``，不再额外创建 ``default.log``； +轮转备份为 ``process..log.1`` 等。fork 子进程会切换到自己的 PID 文件。 +备份数限制每个 PID 的文件数量，不会自动删除历史 PID 文件；部署系统需要配置目录保留周期。 +容器部署建议由日志采集系统收集 stdout 并负责归档。 + +请求关联与非交互输出 +-------------------- + +HTTP 入口保留客户端的 ``X-Request-ID``，未提供时生成 ID，并通过响应头返回。 +ID 必须为不超过 128 字符的可见 ASCII 字符串，否则替换为生成的 ID。 +``http_request_id`` 贯穿 access 与请求完成摘要；摘要中的 ``request_id`` 仍表示内部推理请求 ID。 +关联 ID 即使在 access 关闭时也生效，并为并发和流式请求保持独立上下文。 +日志来源文件和行号指向业务调用处。 + +权重加载和预热在非终端环境关闭 tqdm 进度条，改为阶段开始/完成事件。 +JSON 输出和高于 INFO 的日志级别同样关闭进度条。 + +简洁前缀通过 ``LIGHTLLM_LOG_TP_SIZE``、``LIGHTLLM_LOG_DP_SIZE``、``LIGHTLLM_LOG_NNODES`` +识别拓扑；这些内部变量由启动器自动设置，无需手动配置。 + +P/D 转发通过可选元数据传递 ``http_request_id``；旧消息仍可接收。 +请求摘要省略为零的缓存字段，仅在启用 MTP 时输出 MTP 字段。 +``timing_scope`` 为 ``end_to_end``、``prefill`` 或 ``decode``，标明 TTFT 和耗时的计时边界； +P/D 节点耗时不可直接当作客户端端到端耗时。 +通信后端探测失败在 WARNING 保留简短原因和禁用动作，完整诊断在 DEBUG。 + +DEBUG 体积约束 +-------------- + +常规 DEBUG 输出请求阶段、调度决定及原因、缓存状态变化；不输出 prompt、生成文本、 +token 数组、grammar 全文或内存描述符。诊断事件最多 16 个字段；字符串最多 256 字符， +序列仅保留数量与前 8 项，嵌套对象仅保留类型或数量。 + +KV 成功传输不逐页记录握手和完成。发送/接收进程按 ``--log-stats-interval`` +（默认 10 秒）聚合页数、物理传输字节、KV token 数和任务耗时合计，并保留最多 8 个请求 ID 样本。 +首次完成立即输出；之后限频，空闲时补齐最后一个窗口，不输出空窗口。 +``task_duration_ms_total`` 是各页面任务从创建到完成的耗时之和，不是请求端到端耗时。 +接收端另保留 GPU 拷贝耗时合计；显式启用 NIXL 遥测时输出聚合遥测。 +进程被强制终止时，尚未输出的最后一个统计窗口可能丢失。 + +TP 公共缓存状态和暂停/恢复事件只由组 leader 输出；各 rank 的独有异常仍保留。 +反复发生的 P/D 连接故障保留首个完整 traceback；相同调用点、异常类型和简短原因的 +后续故障每 10 秒输出重复计数。不同原因保留独立 traceback；去重状态最多 128 项。 +请求和传输失败未套用此连接故障去重策略。 + +引擎窗口吞吐 +------------ + +``engine.stats`` / ``engine.idle`` 展示 ``prefill_tok_s``、``output_tok_s``、 +``completed_req_s``；JSON 另外保留 ``window_s`` 以及窗口原始计数 +``prefill_tokens``、``output_tokens``、``completed_reqs``，便于核对。使用单调时钟的实际窗口长度计算速率， +默认仍按 10 秒输出；长请求未完成时也会报告已处理的 token。 + +每个 TP 组由负责发布进度的 leader 累加 CPU 共享计数，router 汇总本节点 DP 行。 +纯跨节点 TP 仍仅主 router 输出。prefill 统计实际计算的输入 token，排除 GPU/CPU/磁盘 +缓存恢复和 P/D 传入 KV；暂停后实际重算的输入会再次计入。生成统计只包含提交的输出 +（含接受的 MTP token），不包含越过停止条件的 token 或模拟终止 token。 +多候选、分段模式的完成计数按引擎内部请求计算，仅包含正常 stop/length 结束。 + +P 节点计算首 token，D 节点只计后续本地生成 token，不重复统计 P 传来的首 token。 +因此两节点的生成计数相加才是分离推理的总生成量；不要将 master 的端到端请求摘要 +再次相加。短请求即使在两个窗口之间完成，其计数仍会在下一窗口输出；稳定空闲后只输出 +一次空闲转换。所有 token 计数在现有推理后处理同步之后更新，不额外同步 GPU。 + +INFO 启动日志以 HTTP /readiness 成功后的 service.ready 为准，包含地址和总启动时间。 +运行时每个节点仅打印一份吞吐与队列摘要；KV 活跃占用与可复用缓存分开显示。 +文本省略原始窗口计数，JSON 保留全部计数与时间窗口。P/D 内部请求摘要只在 DEBUG 输出。 +失败摘要不采样，取消请求只在 DEBUG 输出；终止原因包含 stop、length、abort 或 error。 diff --git a/docs/EN/source/index.rst b/docs/EN/source/index.rst index d4a36385cf..b6934c0a2f 100755 --- a/docs/EN/source/index.rst +++ b/docs/EN/source/index.rst @@ -47,6 +47,7 @@ Documentation List :maxdepth: 1 :caption: Deployment Tutorials + Logging DeepSeek R1 Deployment FP8 KV Quantization and Calibration Multi-Level Cache Deployment diff --git a/docs/EN/source/tutorial/logging.rst b/docs/EN/source/tutorial/logging.rst new file mode 100644 index 0000000000..228f6027a3 --- /dev/null +++ b/docs/EN/source/tutorial/logging.rst @@ -0,0 +1,198 @@ +Logging guide +============= + +Defaults and configuration +-------------------------- + +LightLLM uses Python logging with INFO service logs on stdout. Module loggers inherit +from the central ``lightllm`` logger. Text records use ``HH:mm:ss.SSS LEVEL [component] message``. P/D roles are included +in the component, such as ``[P/infer]`` or ``[D/http]``. Applicable known ranks are +shown only for multi-card, multi-DP or multi-node topologies, e.g. ``[P/infer tp=1]``. +Unknown ranks, hostname, PID and full module names are omitted from default text. +DEBUG, ERROR/CRITICAL and traceback-bearing records retain source file/line. +JSON retains full date/timezone, host, PID, module/component, topology and source. Each worker initializes its own process context. + +Explicit CLI values override environment variables. Unspecified values inherit the +environment. Both hyphenated and underscored CLI names are accepted. Configuration +is propagated to spawned workers and the HTTP subprocess through environment variables. + +.. list-table:: + :header-rows: 1 + + * - CLI + - Environment variable + - Default + * - ``--log-level`` + - ``LIGHTLLM_LOG_LEVEL`` + - ``info`` + * - ``--log-format`` + - ``LIGHTLLM_LOG_FORMAT`` + - ``text``; alternatively ``json`` + * - ``--log-module-levels`` + - ``LIGHTLLM_LOG_MODULE_LEVELS`` + - ``{}``; JSON mapping of module prefixes to levels + * - ``--log-request-sample-rate`` + - ``LIGHTLLM_LOG_REQUEST_SAMPLE_RATE`` + - ``0.01``; 1% of successful completions; failures are never sampled + * - ``--log-stats-interval`` + - ``LIGHTLLM_LOG_STATS_INTERVAL`` + - ``10`` seconds; must be positive + * - ``--log-http-level`` + - ``LIGHTLLM_LOG_HTTP_LEVEL`` + - ``warning``; Hypercorn error logger + * - ``--enable-http-access-log`` + - ``LIGHTLLM_LOG_HTTP_ACCESS`` + - ``0``; set to ``1`` to enable + * - Environment only + - ``LIGHTLLM_LOG_HTTP_EXCLUDE_PATHS`` + - ``/health,/healthz,/readiness,/liveness,/metrics``; excludes these application paths and descendants + * - Environment only + - ``LIGHTLLM_LOG_DIR`` + - Unset; optional per-PID file output in addition to stdout + * - Environment only + - ``LIGHTLLM_LOG_MAX_BYTES`` + - ``104857600``; file rotation threshold + * - Environment only + - ``LIGHTLLM_LOG_BACKUP_COUNT`` + - ``3``; positive number of rotation backups per PID + +Enable DEBUG only for scheduling: + +.. code-block:: bash + + export LIGHTLLM_LOG_LEVEL=INFO + export LIGHTLLM_LOG_MODULE_LEVELS='{"lightllm.server.router":"DEBUG"}' + +Use JSON and sample 1% of request completion summaries: + +.. code-block:: bash + + export LIGHTLLM_LOG_FORMAT=json + export LIGHTLLM_LOG_REQUEST_SAMPLE_RATE=0.01 + +Sampling is deterministic by request ID and does not alter application RNG state. +Request summaries never enable prompt, output text, sampling parameter, or tensor dumps. +The existing explicit ``--detail_log`` diagnostic option remains available; its DEBUG +messages also require enabling DEBUG for the relevant module. + +Categories and severity +----------------------- + +* Service logs: lifecycle, selected configuration/backends, state changes and failures. +* ``lightllm.stats``: node-local router snapshots of running/waiting/paused requests + and KV utilization per local DP. All snapshots respect the configured interval; + idle is reported only after a full interval without work and is not repeatedly logged. Mirrored multi-node TP slave routers do + not duplicate snapshots. Queue and KV values are instantaneous; throughput uses deltas of engine progress + counters over the reporting window. Use Prometheus for historical throughput and latency analysis. +* ``lightllm.requests``: completion metadata, including request ID, token counts, + TTFT, elapsed time and cache/MTP statistics. Normal/P/D and PD master use the same + base fields. P/D summaries describe processing within that role; segmented PD + requests retain segment IDs and are not whole-client-request aggregates. +* ``lightllm.access``: HTTP method, path, status and duration, without queries, + authentication headers or bodies. Application health/metrics paths are excluded + by default; set ``LIGHTLLM_LOG_HTTP_EXCLUDE_PATHS`` to an empty string to include them. + +Enabled request/access categories use INFO independently of service verbosity; explicit +module-level overrides can further filter them. Application access records come from +LightLLM's middleware rather than a second Hypercorn access logger. The configuration +server uses Hypercorn access; application path exclusions do not apply there. Custom +Hypercorn configuration can add its own outputs, so avoid duplicate configuration. +Python warnings use the central logger. Native library stdout/stderr and Hypercorn logs are not forcibly converted to LightLLM JSON. + +Use DEBUG for request/batch/transfer internals and allocation details; INFO for startup, +final choices, readiness, periodic summaries and meaningful state transitions; WARNING +for unexpected recovery, retries or fallback; ERROR for an operation's final failure; +and CRITICAL when a process or service cannot continue reliably. Normal client +cancellation and idempotent cleanup belong at DEBUG. ``logger.exception`` emits ERROR +with a traceback, not a separate severity. Record a failure's traceback at its handling +boundary, avoiding adjacent error/exception messages and repeated stack traces on rethrow. + +Shared model, backend and CUDA graph configuration is INFO on the TP group's leader +and DEBUG on peers. Never apply this filtering to worker-specific warnings/errors. +New events should use stable English messages and an ``event`` field, plus applicable +request/batch/transfer IDs. Guard expensive DEBUG arguments with ``isEnabledFor``; +parameterized formatting alone does not defer argument evaluation. Startup summaries +allowlist operational settings instead of dumping potentially sensitive arguments. + +JSON contains one object per line with structured context and an ``exception`` field +for tracebacks. Text mode prefixes every physical message/traceback line. + +Migration +--------- + +The former DEBUG default is now INFO. Completion summaries are enabled by default; HTTP access is opt-in. +File logging writes only ``process..log`` and numbered rotation backups, without +creating ``default.log``. Forked workers switch to their own PID files. Backup limits +apply per PID; deployment tooling must expire historical PID files. Container deployments +should normally collect and archive stdout externally. + +Correlation and non-interactive output +-------------------------------------- + +The HTTP entrypoint preserves ``X-Request-ID`` or generates one, and returns it in +the response header. Client IDs must contain at most 128 visible ASCII characters; +invalid IDs are replaced. ``http_request_id`` connects access and completion logs, +while ``request_id`` remains the internal inference ID. Correlation works even with +access logging disabled and is isolated across concurrent and streaming requests. +Source files and line numbers identify business callsites rather than logging helpers. + +Weight loading and warmup replace progress bars with start/completion events on +non-interactive streams. JSON output and levels above INFO also disable progress bars. + +P/D forwarding carries ``http_request_id`` in optional metadata; legacy messages +remain accepted. Summaries omit zero cache fields and include MTP fields only when +MTP is enabled. ``timing_scope`` identifies ``end_to_end``, ``prefill`` or ``decode`` +measurement boundaries; node timings are not client end-to-end timings. Backend +probe warnings contain a short reason and disable action; full diagnostics use DEBUG. + +DEBUG volume bounds +------------------- + +Normal DEBUG logs cover request phases, scheduling decisions/reasons and cache changes. +They do not dump prompts, generated text, token arrays, grammars or memory descriptors. +Diagnostic events allow at most 16 fields, strings up to 256 characters and sequences +summarized by count and 8 items; nested objects retain only type/count information. + +Successful KV transfers aggregate page counts, physical bytes, KV tokens and summed +page-task durations per sender/receiver using ``--log-stats-interval`` (default 10 s). +The first completion logs immediately; subsequent updates are rate limited and the +final partial window flushes on idle. Empty windows do not log. Request ID samples +are capped at 8. ``task_duration_ms_total`` sums page-task creation-to-completion +times, not request end-to-end latency. Receiver GPU copy times and explicitly enabled +NIXL telemetry remain aggregated. Forced termination may lose the pending window. + +Shared TP cache and pause/resume events use the group leader. Rank-specific errors +remain visible. Recurring PD connection failures keep the first traceback, followed +by counts every 10 s for the same callsite, exception type and short reason. Different +reasons retain independent tracebacks; deduplication state is capped at 128 entries. +Request and transfer errors do not use this connection-failure deduplication policy. + +Engine throughput windows +------------------------- + +``engine.stats`` / ``engine.idle`` display ``prefill_tok_s``, ``output_tok_s`` and +``completed_req_s``. JSON also retains ``window_s`` and raw window counts +``prefill_tokens``, ``output_tokens`` and ``completed_reqs``. Rates use actual +monotonic elapsed time; the default interval remains 10 seconds. In-flight +requests contribute tokens before they finish. + +One progress-publishing TP leader writes each CPU shared counter row; the router +sums local DP rows. Pure multi-node TP still logs only on the primary router. +Prefill counts computed input tokens, excluding GPU/CPU/disk cache restoration +and received PD KV. Actual prompt recomputation after a pause counts again. +Generation counts committed outputs, including accepted MTP tokens, excluding +outputs after a stop condition and simulated termination tokens. Completions +count internal engine requests (choices/segments), with normal stop/length only. + +Prefill nodes count the first generated token; decode nodes exclude that received +token and count subsequent local generation. Add P and D generation counts for +PD totals; do not add the master's end-to-end request summaries again. Short +requests completed between snapshots appear in the next window. Stable idle emits +one transition. Counters update after existing inference synchronization without +introducing GPU synchronization. + +Service readiness is announced only after the bound HTTP /readiness endpoint succeeds. +INFO reports one throughput, queue and KV usage snapshot per node. Active KV and reusable +cache occupancy are separate. JSON retains raw window counts omitted from compact text. +Successful request summaries default to 1% sampling; P/D stage summaries are DEBUG only. +Failures bypass sampling; cancellations are DEBUG. Terminal reasons are preserved. diff --git a/lightllm/common/basemodel/attention/create_utils.py b/lightllm/common/basemodel/attention/create_utils.py index 344ba08a1c..e168fddbcf 100644 --- a/lightllm/common/basemodel/attention/create_utils.py +++ b/lightllm/common/basemodel/attention/create_utils.py @@ -1,7 +1,7 @@ """Attention backend selection utilities.""" from lightllm.utils.envs_utils import get_env_start_args -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import init_logger, log_info_on_leader from lightllm.utils.backend_validator import validate from typing import Dict from .base_att import BaseAttBackend @@ -13,7 +13,7 @@ from .fa3.fp import Fa3AttBackend from .fa3.fp8 import Fp8Fa3AttBackend from .fa3.mla import MlaFa3AttBackend -from .fa3.neo import NeoFa3AttBackend, HAS_FLASH_ATTN_INTERFACE +from .fa3.neo import NeoFa3AttBackend, HAS_FLASH_ATTN_INTERFACE, require_neo_fa3 from .flashinfer.fp8 import Fp8FlashInferAttBackend from .flashinfer.fp import FlashInferAttBackend from .flashinfer.mla import MlaFlashInferAttBackend @@ -183,6 +183,12 @@ def get_neo_prefill_att_backend_class(index=0, priority_list: list = ["fa3", "tr llm_dtype = args.llm_kv_type backend_str = args.llm_prefill_att_backend[index] if backend_str != "auto": + if backend_str == "fa3": + require_neo_fa3() return neo_data_type_to_backend[llm_dtype][backend_str] else: + if not HAS_FLASH_ATTN_INTERFACE and "fa3" in priority_list: + log_info_on_leader( + logger, "event=backend.fallback family=neo requested=auto skipped=fa3 reason=extension_unavailable" + ) return _auto_select_backend(llm_dtype, kv_type_to_backend=neo_data_type_to_backend, priority_list=priority_list) diff --git a/lightllm/common/basemodel/attention/fa3/neo.py b/lightllm/common/basemodel/attention/fa3/neo.py index 6af1f17a2b..60b166a773 100644 --- a/lightllm/common/basemodel/attention/fa3/neo.py +++ b/lightllm/common/basemodel/attention/fa3/neo.py @@ -1,6 +1,5 @@ import dataclasses import inspect -import warnings import torch @@ -19,6 +18,8 @@ ) +_NEO_FA3_IMPORT_ERROR = None + try: from flash_attn_interface import flash_attn_with_kvcache as flash_attn_with_kvcache_neo @@ -29,15 +30,22 @@ HAS_FLASH_ATTN_INTERFACE = True except ImportError as exc: - warnings.warn( - f"Neo FA3 is unavailable: {exc}. " - "Automatic Neo prefill selection will fall back to Triton. " - "To select Triton explicitly, use --llm_prefill_att_backend triton.\n" + _NEO_FA3_INSTALL_HINT - ) + # Optional backend discovery must stay silent in every importing worker. + _NEO_FA3_IMPORT_ERROR = str(exc) flash_attn_with_kvcache_neo = None HAS_FLASH_ATTN_INTERFACE = False +def require_neo_fa3(): + """Explain installation only when the unavailable backend is actually requested.""" + if not HAS_FLASH_ATTN_INTERFACE: + reason = _NEO_FA3_IMPORT_ERROR or "extension unavailable" + raise ImportError( + f"Neo prefill requires FA3 with image_token_end support: {reason}.\n" + "Use --llm_prefill_att_backend triton to select Triton instead.\n" + _NEO_FA3_INSTALL_HINT + ) + + class NeoFa3AttBackend(Fa3AttBackend): def create_att_prefill_state(self, infer_state) -> "NeoFa3PrefillAttState": return NeoFa3PrefillAttState(backend=self, infer_state=infer_state) @@ -55,8 +63,7 @@ def prefill_att( ) -> torch.Tensor: # neo_chat*: image-token bidirectional attention requires flash_attn_interface # (sgl_kernel's flash_attn_with_kvcache does not support image_token_end). - if not HAS_FLASH_ATTN_INTERFACE: - raise ImportError("Neo prefill requires FA3 with image_token_end support.\n" + _NEO_FA3_INSTALL_HINT) + require_neo_fa3() if self.infer_state.b_image_token_end is None: raise ValueError("Neo prefill requires b_image_token_end to describe image attention spans") o = flash_attn_with_kvcache_neo( diff --git a/lightllm/common/basemodel/basemodel.py b/lightllm/common/basemodel/basemodel.py index 5a2d5b77d8..7937d32a52 100755 --- a/lightllm/common/basemodel/basemodel.py +++ b/lightllm/common/basemodel/basemodel.py @@ -2,6 +2,7 @@ # os.environ["CUDA_LAUNCH_BLOCKING"] = "1" import gc +import time import copy import math import json @@ -26,7 +27,7 @@ from lightllm.common.basemodel.prefill_cuda_graph import PrefillCudaGraph from lightllm.common.quantization import Quantcfg from lightllm.common.basemodel.triton_kernel.gather_token_id import gather_token, gather_token_prefill_decode_mixed -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import init_logger, log_info_on_leader, progress_enabled from lightllm.utils.dist_utils import get_dp_world_size from lightllm.utils.profile_max_tokens import profile_mtp_weight_memory from lightllm.utils.envs_utils import ( @@ -129,11 +130,18 @@ def __init__(self, kvargs): self._init_att_backend() self._init_att_backend1() - logger.info(f"use prefill att backend: {self.prefill_att_backend.__class__.__name__}") - logger.info(f"use decode att backend: {self.decode_att_backend.__class__.__name__}") - if self.prefill_att_backend1 is not None: - logger.info(f"use prefill att backend1: {self.prefill_att_backend1.__class__.__name__}") - logger.info(f"use decode att backend1: {self.decode_att_backend1.__class__.__name__}") + backends = { + "prefill": self.prefill_att_backend.__class__.__name__, + "decode": self.decode_att_backend.__class__.__name__, + "linear_prefill": self.prefill_att_backend1.__class__.__name__ if self.prefill_att_backend1 else None, + "linear_decode": self.decode_att_backend1.__class__.__name__ if self.decode_att_backend1 else None, + } + log_info_on_leader( + logger, + "event=backend.selected %s", + " ".join(f"{k}={v}" for k, v in backends.items() if v is not None), + extra={"event": "backend.selected", **backends}, + ) self._init_hidden_collector() self._autotune_warmup() @@ -168,7 +176,9 @@ def _verify_params(self): def _init_quant(self): self.quant_cfg = Quantcfg(self.config, self.quant_type, self.quant_cfg_path, self.expert_dtype) - logger.info(f"Initial quantization. " f"The default quantization method is {self.quant_cfg.quant_type}") + log_info_on_leader( + logger, f"Initial quantization. " f"The default quantization method is {self.quant_cfg.quant_type}" + ) def _init_weights(self, start_layer_index=0): self.pre_post_weight = self.pre_and_post_weight_class(self.data_type, network_config=self.config) @@ -630,7 +640,6 @@ def _decode( @final def _context_forward(self, infer_state: InferStateInfo): - input_embs = self.pre_infer.context_forward(infer_state.input_ids, infer_state, self.pre_post_weight) if self.args.enable_dp_prefill_balance: assert not self.args.enable_prefill_cudagraph, "not support now" @@ -974,15 +983,15 @@ def _overlap_tpsp_token_forward(self, infer_state: InferStateInfo, infer_state1: def _check_max_len_infer(self): disable_check_max_len_infer = os.getenv("DISABLE_CHECK_MAX_LEN_INFER", None) is not None if disable_check_max_len_infer: - logger.info("disable_check_max_len_infer is true") + log_info_on_leader(logger, "disable_check_max_len_infer is true") return # 做一次 同步 - torch.distributed.barrier() + torch.distributed.barrier(device_ids=[torch.cuda.current_device()]) # 模拟最大长度进行 prefill，观察是否出现 OOM try: - logger.info("begin check max_len infer") + logger.debug("Checking maximum batch token capacity") dummy_input_ids = torch.ones(self.batch_max_tokens, dtype=torch.int64, device="cuda") b_req_idx = torch.tensor([self.req_manager.alloc()], dtype=torch.int32, device="cuda") page_size = self.mem_manager.page_size @@ -1022,7 +1031,7 @@ def _check_max_len_infer(self): prob_out = None self.req_manager.free_all() self.mem_manager.free_all() - logger.info(f"check max_len {self.batch_max_tokens} infer ok") + logger.debug("Batch token capacity check passed: batch_max_tokens=%s", self.batch_max_tokens) except (RuntimeError, torch.OutOfMemoryError) as e: logger.exception(str(e)) exception_str = ( @@ -1045,7 +1054,7 @@ def autotune_layers(self): @post_empty_cache def _autotune_warmup(self): Autotuner.start_autotune_warmup(AutotuneKernelType.GENERAL) - torch.distributed.barrier() + torch.distributed.barrier(device_ids=[torch.cuda.current_device()]) warmup_lengths = [1, 4, 8, 16, 32, 64, 128, 256, 1024, 2048, 4096] @@ -1058,7 +1067,11 @@ def _autotune_warmup(self): layer_num_bak = self.layers_num self.layers_num = self.autotune_layers() - for input_len in tqdm(warmup_lengths, desc="warming up"): + started = time.monotonic() + log_info_on_leader( + logger, "event=model.warmup_started shapes=%s", len(warmup_lengths), extra={"event": "model.warmup_started"} + ) + for input_len in tqdm(warmup_lengths, desc="warming up", disable=not progress_enabled()): try: rand_gen = torch.Generator(device="cuda") rand_gen.manual_seed(input_len) @@ -1113,8 +1126,15 @@ def _autotune_warmup(self): gc.collect() torch.cuda.empty_cache() self.layers_num = layer_num_bak - torch.distributed.barrier() + torch.distributed.barrier(device_ids=[torch.cuda.current_device()]) Autotuner.end_autotune_warmup() + duration_s = round(time.monotonic() - started, 3) + log_info_on_leader( + logger, + "event=model.warmup_completed duration_s=%.3f", + duration_s, + extra={"event": "model.warmup_completed", "duration_s": duration_s}, + ) @final @torch.no_grad() @@ -1123,7 +1143,7 @@ def _init_padded_req(self): 对 padded 所使用的req 进行初始化， 目前有非常多的地方需要使用，所以将其初始化固定为固定流程 """ # 做一次 同步 - torch.distributed.barrier() + torch.distributed.barrier(device_ids=[torch.cuda.current_device()]) # prefill init padding req. prefill_input_len = 1 diff --git a/lightllm/common/basemodel/cuda_graph.py b/lightllm/common/basemodel/cuda_graph.py index ee888c1a00..bd2a4c7608 100644 --- a/lightllm/common/basemodel/cuda_graph.py +++ b/lightllm/common/basemodel/cuda_graph.py @@ -6,7 +6,7 @@ import math import triton from typing import Optional -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import init_logger, log_info_on_leader from lightllm.utils.envs_utils import get_env_start_args from lightllm.distributed import dist_group_manager from lightllm.common.basemodel.batch_objs import ModelInput, ModelOutput @@ -82,7 +82,7 @@ def __init__( max_batch_size=self.max_batch_size, tp_world_size=self.tp_world_size, ) - logger.info(f"cuda graph batch_sizes: {self.cuda_graph_batch_sizes}") + logger.debug("CUDA graph batch_sizes=%s", self.cuda_graph_batch_sizes) def can_run(self, batch_size, max_len_in_batch): return batch_size <= self.max_batch_size and max_len_in_batch <= self.graph_max_len_in_batch @@ -182,7 +182,7 @@ def _measure_replay_cost(self, graph_obj: torch.cuda.CUDAGraph, batch_size: int) if not self.capture_infer_cost: return - dist.barrier(group=dist.group.WORLD) + dist.barrier(group=dist.group.WORLD, device_ids=[torch.cuda.current_device()]) start_event = torch.cuda.Event(enable_timing=True) end_event = torch.cuda.Event(enable_timing=True) graph_obj.replay() @@ -251,7 +251,7 @@ def replay(self, infer_state, infer_state1=None): @torch.no_grad() def warmup(self, model): - logger.info("Begin capture cudagraph, use the --disable_cudagraph to disable it.") + log_info_on_leader(logger, "Begin capture cudagraph, use the --disable_cudagraph to disable it.") # for typing easy from .basemodel import TpPartBaseModel @@ -300,14 +300,15 @@ def warmup(self, model): del locals()[var_name] torch.cuda.empty_cache() - logger.info( + log_info_on_leader( + logger, f"Capture cudagraph success, batch_size <={self.max_batch_size} " - f"and max_len_in_batch <= {self.graph_max_len_in_batch} will infer with cudagraph." + f"and max_len_in_batch <= {self.graph_max_len_in_batch} will infer with cudagraph.", ) @torch.no_grad() def warmup_overlap(self, model): - logger.info("Begin capture overlap cudagraph, use the --disable_cudagraph to disable it.") + log_info_on_leader(logger, "Begin capture overlap cudagraph, use the --disable_cudagraph to disable it.") # for typing easy from .basemodel import TpPartBaseModel @@ -365,7 +366,8 @@ def warmup_overlap(self, model): del locals()[var_name] torch.cuda.empty_cache() - logger.info( + log_info_on_leader( + logger, f"Capture overlap cudagraph success, batch_size <={self.max_batch_size} " - f"and max_len_in_batch <= {self.graph_max_len_in_batch} will infer with cudagraph." + f"and max_len_in_batch <= {self.graph_max_len_in_batch} will infer with cudagraph.", ) diff --git a/lightllm/common/basemodel/layer_infer/cache_tensor_manager.py b/lightllm/common/basemodel/layer_infer/cache_tensor_manager.py index 8bcf99b992..2b64d7f915 100644 --- a/lightllm/common/basemodel/layer_infer/cache_tensor_manager.py +++ b/lightllm/common/basemodel/layer_infer/cache_tensor_manager.py @@ -14,11 +14,10 @@ _disable_gpu_tensor_cache = os.getenv("DISABLE_GPU_TENSOR_CACHE", None) is not None if torch.__version__ >= "2.1.0" and (not _disable_gpu_tensor_cache): - logger.info("USE_GPU_TENSOR_CACHE is On") + logger.debug("USE_GPU_TENSOR_CACHE is On") # 用于进行引用计数调整和判断 def custom_del(self: torch.Tensor): - global g_cache_manager if hasattr(self, "storage_weak_ptr"): storage_weak_ptr = self.storage_weak_ptr else: @@ -140,7 +139,7 @@ def alloc_tensor( return ans else: - logger.info("USE_GPU_TENSOR_CACHE is OFF") + logger.debug("USE_GPU_TENSOR_CACHE is OFF") class CacheTensorManager: def __init__(self): diff --git a/lightllm/common/basemodel/layer_weights/hf_load_utils.py b/lightllm/common/basemodel/layer_weights/hf_load_utils.py index 8cf66a5ad6..dbfa73a07d 100755 --- a/lightllm/common/basemodel/layer_weights/hf_load_utils.py +++ b/lightllm/common/basemodel/layer_weights/hf_load_utils.py @@ -1,6 +1,10 @@ import torch import os import gc +import time +from lightllm.utils.log_utils import init_logger, log_info_on_leader, progress_enabled + +logger = init_logger(__name__) from safetensors import safe_open from tqdm import tqdm import lightllm.utils.petrel_helper as utils @@ -61,12 +65,28 @@ def load_hf_weights(data_type, weight_dir, pre_post_layer=None, transformer_laye weight_dir=weight_dir, ) # noqa worker = int(os.environ.get("LOADWORKER", 1)) + started = time.monotonic() + log_info_on_leader( + logger, + "event=model.weights_loading shards=%s workers=%s", + len(candidate_files), + worker, + extra={"event": "model.weights_loading"}, + ) with Pool(worker) as p: iterator = p.imap_unordered(partial_func, candidate_files, chunksize=1) desc_str = f"pid {os.getpid()} Loading model weights with {worker} workers" - iterator = tqdm(iterator, total=len(candidate_files), desc=desc_str) + iterator = tqdm(iterator, total=len(candidate_files), desc=desc_str, disable=not progress_enabled()) for _ in iterator: pass + duration_s = round(time.monotonic() - started, 3) + log_info_on_leader( + logger, + "event=model.weights_loaded shards=%s duration_s=%.3f", + len(candidate_files), + duration_s, + extra={"event": "model.weights_loaded", "duration_s": duration_s}, + ) return diff --git a/lightllm/common/basemodel/prefill_cuda_graph.py b/lightllm/common/basemodel/prefill_cuda_graph.py index 28d44fefbf..959c7b37f4 100644 --- a/lightllm/common/basemodel/prefill_cuda_graph.py +++ b/lightllm/common/basemodel/prefill_cuda_graph.py @@ -5,7 +5,7 @@ import triton from typing import List, Tuple from typing import Optional -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import init_logger, log_info_on_leader from lightllm.utils.envs_utils import get_env_start_args from lightllm.utils.tensor_utils import tensor_to_no_ref_tensor from lightllm.distributed import dist_group_manager @@ -54,7 +54,7 @@ def __init__(self, decode_cuda_graph: CudaGraph, tp_world_size: int): graph_handle_token_nums.sort() self.graph_handle_token_nums = graph_handle_token_nums - logger.info(f"prefill cuda graph graph_handle_token_nums: {self.graph_handle_token_nums}") + log_info_on_leader(logger, f"prefill cuda graph graph_handle_token_nums: {self.graph_handle_token_nums}") def can_run(self, handle_token_num: int): return handle_token_num <= self.max_handle_token_num @@ -91,13 +91,14 @@ def _capture_prefill( graph_hidden_collector = infer_state.hidden_collector hidden_tensor_count, hidden_tensor_nbytes = graph_hidden_collector.release_graph_tensor_ownership() - logger.info( + log_info_on_leader( + logger, f"Prefill CUDA Graph hidden collector 已完成 capture 状态托管：" f"handle_token_num={handle_token_num}, " f"collector={graph_hidden_collector.__class__.__name__}, " f"no_ref_tensor_count={hidden_tensor_count}, " f"no_ref_tensor_nbytes={hidden_tensor_nbytes}。" - f"这些 tensor 的固定地址由 graph memory pool 管理，collector 仅保留无所有权引用供 replay 使用。" + f"这些 tensor 的固定地址由 graph memory pool 管理，collector 仅保留无所有权引用供 replay 使用。", ) self.graph[handle_token_num] = ( infer_state, @@ -183,7 +184,9 @@ def replay(self, input_tensors, infer_state, input_tensor1=None, infer_state1=No @torch.no_grad() def warmup(self, model): - logger.info("Begin capture prefill cudagraph, remove the --enable_prefill_cudagraph to disable it.") + log_info_on_leader( + logger, "Begin capture prefill cudagraph, remove the --enable_prefill_cudagraph to disable it." + ) # for typing easy from .basemodel import TpPartBaseModel @@ -191,7 +194,7 @@ def warmup(self, model): # prefill cuda graph init for handle_token_num in self.graph_handle_token_nums[::-1]: - logger.info(f"Capture prefill cudagraph, handle_token_num: {handle_token_num}") + log_info_on_leader(logger, f"Capture prefill cudagraph, handle_token_num: {handle_token_num}") total_token_num = handle_token_num input_ids = torch.tensor([1 for _ in range(total_token_num)], dtype=torch.int64, device="cuda") b_req_idx = torch.tensor([model.req_manager.HOLD_REQUEST_ID], dtype=torch.int32, device="cuda") @@ -234,13 +237,17 @@ def warmup(self, model): del locals()[var_name] torch.cuda.empty_cache() - logger.info( - f"Capture repfill cudagraph success, token_num <={self.max_handle_token_num} " f"will infer with cudagraph." + log_info_on_leader( + logger, + f"Capture repfill cudagraph success, token_num <={self.max_handle_token_num} " + f"will infer with cudagraph.", ) @torch.no_grad() def warmup_overlap(self, model): - logger.info("Begin capture prefill overlap cudagraph, remove the --enable_prefill_cudagraph to disable it.") + log_info_on_leader( + logger, "Begin capture prefill overlap cudagraph, remove the --enable_prefill_cudagraph to disable it." + ) # for typing easy from .basemodel import TpPartBaseModel @@ -300,7 +307,8 @@ def warmup_overlap(self, model): del locals()[var_name] torch.cuda.empty_cache() - logger.info( + log_info_on_leader( + logger, f"Capture overlap cudagraph success, handle_token_num <={self.max_handle_token_num} " - f" will infer with cudagraph." + f" will infer with cudagraph.", ) diff --git a/lightllm/common/basemodel/triton_kernel/fused_moe/grouped_fused_moe_ep.py b/lightllm/common/basemodel/triton_kernel/fused_moe/grouped_fused_moe_ep.py index 712860bd8c..734dec9a6e 100644 --- a/lightllm/common/basemodel/triton_kernel/fused_moe/grouped_fused_moe_ep.py +++ b/lightllm/common/basemodel/triton_kernel/fused_moe/grouped_fused_moe_ep.py @@ -39,7 +39,7 @@ HAS_DEEPGEMM = True except: - logger.warning("no deepep or deep_gemm") + logger.debug("no deepep or deep_gemm") HAS_DEEPGEMM = False diff --git a/lightllm/common/kv_cache_mem_manager/allocator.py b/lightllm/common/kv_cache_mem_manager/allocator.py index 1331943b1a..96bbcf170a 100644 --- a/lightllm/common/kv_cache_mem_manager/allocator.py +++ b/lightllm/common/kv_cache_mem_manager/allocator.py @@ -71,8 +71,6 @@ def free(self, free_index: Union[torch.Tensor, List[int]]): self.can_use_mem_size += len(free_index) self.shared_can_use_token_num.set_value(self.can_use_mem_size) - if self.can_use_mem_size == len(self.mem_state): - logger.debug(f"freed all gpu mem size {self.can_use_mem_size}") return def free_all(self): diff --git a/lightllm/common/kv_cache_mem_manager/mem_manager.py b/lightllm/common/kv_cache_mem_manager/mem_manager.py index 38c870ce4e..5384de4097 100755 --- a/lightllm/common/kv_cache_mem_manager/mem_manager.py +++ b/lightllm/common/kv_cache_mem_manager/mem_manager.py @@ -28,7 +28,6 @@ class MemoryManager: - operator_class = NormalMemOperator def __init__(self, size, dtype, head_num, head_dim, layer_num, always_copy=False, mem_fraction=0.9): @@ -98,7 +97,7 @@ def profile_size(self, mem_fraction): available_memory = get_available_gpu_memory(world_size) - get_total_gpu_memory() * (1 - mem_fraction) cell_size = self.get_cell_size() pd_kv_move_buffer_size = self.get_pd_kv_move_buffer_size() - available_memory_bytes = available_memory * 1024 ** 3 - pd_kv_move_buffer_size + available_memory_bytes = available_memory * 1024**3 - pd_kv_move_buffer_size self.size = int(available_memory_bytes / cell_size) if world_size > 1: tensor = torch.tensor(self.size, dtype=torch.int64, device=f"cuda:{get_current_device_id()}") @@ -258,7 +257,7 @@ def write_to_shm(self, req_manager): shm = create_or_link_shm( name=shm_name, expected_size=obj_size * (node_world_size * 2) + 4 + 4, force_mode="create" ) - logger.info(f"create shm {shm.name} size {shm.size} for mem manger shared buffer") + logger.debug(f"create shm {shm.name} size {shm.size} for mem manger shared buffer") shm.buf[0:4] = (node_world_size * 2).to_bytes(4, "little") shm.buf[4:8] = obj_size.to_bytes(4, "little") start_index = 8 @@ -270,7 +269,7 @@ def write_to_shm(self, req_manager): def loads_from_shm(rank_in_node: int) -> "MemoryManager": shm_name = f"mem_manager_{rank_in_node}" lock = FileLock(f"/tmp/{get_service_shm_name('mem_manager_lock')}") - logger.info(f"get memmanager from shm {shm_name}") + logger.debug(f"get memmanager from shm {shm_name}") with lock: shm = create_or_link_shm(name=shm_name, expected_size=-1, force_mode="link") left_num = int.from_bytes(shm.buf[0:4], "little") diff --git a/lightllm/common/kv_cache_mem_manager/mem_utils.py b/lightllm/common/kv_cache_mem_manager/mem_utils.py index 79ea448794..ffa65581d1 100644 --- a/lightllm/common/kv_cache_mem_manager/mem_utils.py +++ b/lightllm/common/kv_cache_mem_manager/mem_utils.py @@ -7,7 +7,7 @@ FP8StaticPerHeadQuantMemManager, FP8StaticPerTensorQuantMemManager, ) -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import init_logger, log_info_on_leader from lightllm.utils.envs_utils import get_env_start_args from lightllm.utils.llm_utils import get_llm_model_class from functools import lru_cache @@ -30,18 +30,18 @@ def select_mem_manager_class(): mem_class = FP8PerTokenGroupQuantDeepseek3_2MemoryManager else: mem_class = Deepseek3_2MemoryManager - logger.info(f"Model kv cache using default, mem_manager class: {mem_class}") + log_info_on_leader(logger, f"Model kv cache using default, mem_manager class: {mem_class}") return mem_class from lightllm.models import Deepseek2TpPartModel if issubclass(model_class, Deepseek2TpPartModel): mem_class = Deepseek2MemoryManager - logger.info(f"Model kv cache using default, mem_manager class: {mem_class}") + log_info_on_leader(logger, f"Model kv cache using default, mem_manager class: {mem_class}") return mem_class # case normal - logger.info(f"mode setting params: {get_env_start_args().llm_kv_type}") + log_info_on_leader(logger, f"mode setting params: {get_env_start_args().llm_kv_type}") if get_env_start_args().llm_kv_type == "int8kv": memory_manager_class = PPLINT8KVMemoryManager elif get_env_start_args().llm_kv_type == "int4kv": @@ -53,7 +53,7 @@ def select_mem_manager_class(): elif get_env_start_args().llm_kv_type == "None": memory_manager_class = MemoryManager - logger.info(f"Model kv cache using mem_manager class: {memory_manager_class}") + log_info_on_leader(logger, f"Model kv cache using mem_manager class: {memory_manager_class}") return memory_manager_class diff --git a/lightllm/common/quantization/__init__.py b/lightllm/common/quantization/__init__.py index 6ce4509776..15f99ae60d 100644 --- a/lightllm/common/quantization/__init__.py +++ b/lightllm/common/quantization/__init__.py @@ -6,7 +6,7 @@ from .deepgemm import * from .awq import * from .no_quant import * -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import init_logger, log_info_on_leader from lightllm.utils.device_utils import is_sm100_gpu logger = init_logger(__name__) @@ -73,9 +73,10 @@ def _mapping_expert_quant_method(self): layer_quant_cfg["fused_moe"] = target source = "startup arguments" if self.start_args_expert_dtype is not None else "model config" - logger.info( + log_info_on_leader( + logger, f"select fused_moe quant way from {source} expert_dtype=`{expert_dtype}`: {target}; " - "priority is startup arguments > quant_cfg > model config." + "priority is startup arguments > quant_cfg > model config.", ) def _mapping_quant_method(self): @@ -88,14 +89,14 @@ def _mapping_quant_method(self): self.quant_type = "fp8w8a8-b128-deepgemm" else: self.quant_type = "fp8w8a8-b128-vllm" - logger.info(f"select fp8w8a8-b128 quant way: {self.quant_type}") + log_info_on_leader(logger, f"select fp8w8a8-b128 quant way: {self.quant_type}") self._mapping_expert_quant_method() elif self.hf_quantization_method == "awq": self.quant_type = "awq" if is_awq_marlin_compatible(self.hf_quantization_config): self.quant_type = "awq_marlin" - logger.info(f"select awq quant way: {self.quant_type}") + log_info_on_leader(logger, f"select awq quant way: {self.quant_type}") else: # TODO: more quant method pass diff --git a/lightllm/common/req_manager/base.py b/lightllm/common/req_manager/base.py index cfe5d716ee..d7c6138948 100644 --- a/lightllm/common/req_manager/base.py +++ b/lightllm/common/req_manager/base.py @@ -102,14 +102,10 @@ def free(self, free_req_indexes: List[int], free_token_index): for req_index in free_req_indexes: self.req_list.free(req_index) - if self.req_list.is_all_free(): - logger.debug(f"freed all request size {self.req_list.can_alloc_size}") self.mem_manager.free(free_token_index) def free_req(self, free_req_index: int): self.req_list.free(free_req_index) - if self.req_list.is_all_free(): - logger.debug(f"freed all request size {self.req_list.can_alloc_size}") return def free_token(self, free_token_index): diff --git a/lightllm/common/triton_utils/autotuner.py b/lightllm/common/triton_utils/autotuner.py index 6752e73e5e..15c40a68b7 100644 --- a/lightllm/common/triton_utils/autotuner.py +++ b/lightllm/common/triton_utils/autotuner.py @@ -12,7 +12,7 @@ from tqdm import tqdm from frozendict import frozendict from lightllm.utils.device_utils import get_current_device_name -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import init_logger, progress_enabled from typing import Callable, List, Optional from lightllm.utils.envs_utils import get_triton_autotune_level from lightllm.common.kernel_config import KernelConfigs @@ -109,6 +109,7 @@ def decorator(fn: Callable) -> Callable: class Autotuner: + _defaults_warning_emitted = False _autotune_warmup_kernel_type: Optional[AutotuneKernelType] = None @staticmethod @@ -156,7 +157,6 @@ def __init__( rebuild_input_func: Optional[Callable] = None, warmup_all_exist_config: bool = True, ): - self.configs_gen_func = configs_gen_func self.kernel_name = kernel_name self.kernel_type = AutotuneKernelType(kernel_type) @@ -227,10 +227,17 @@ def __call__(self, *args, **kwargs): if static_key not in self.cached_configs and autotune_level == AutotuneLevel.USE_AUTOTUNE_HIS_CONFIG: if (dist.is_initialized() and get_current_rank_in_node() == 0) or not dist.is_initialized(): - logger.warning( - f"No kernel config for {self.kernel_name} in {KernelConfigs.get_config_file_name(static_key)}," - f"the performance may be suboptimal!" - f"You can use LIGHTLLM_TRITON_AUTOTUNE_LEVEL=1 to enable autotune.", + if not Autotuner._defaults_warning_emitted: + logger.warning( + "event=kernel.defaults_used reason=tuned_configs_missing action=use_default_configs " + "performance=may_be_suboptimal hint=LIGHTLLM_TRITON_AUTOTUNE_LEVEL=1 " + "details=DEBUG" + ) + Autotuner._defaults_warning_emitted = True + logger.debug( + "event=kernel.config_missing kernel=%s config=%s", + self.kernel_name, + KernelConfigs.get_config_file_name(static_key), ) self.cached_configs[static_key] = {} @@ -295,7 +302,7 @@ def _try_load_cache(self, static_key): cache_file = os.path.join(self.cache_dir, KernelConfigs.get_config_file_name(static_key)) if os.path.exists(cache_file): - logger.info(f"Loading cached configs for {self.kernel_name} - {static_key}") + logger.debug(f"Loading cached configs for {self.kernel_name} - {static_key}") with open(cache_file, "rb") as f: self.cached_configs[static_key] = orjson.loads(f.read()) return True @@ -385,6 +392,7 @@ def _autotune(self, args, kwargs, static_key, run_key, rank_id, world_size): desc=f"Autotuning {self.kernel_name} for {run_key}", position=rank_id, dynamic_ncols=True, + disable=not progress_enabled(), ) enum_configs = enumerate(bar) for i, config in enum_configs: @@ -440,7 +448,7 @@ def _autotune(self, args, kwargs, static_key, run_key, rank_id, world_size): option=orjson.OPT_INDENT_2 | orjson.OPT_SORT_KEYS | orjson.OPT_NON_STR_KEYS, ) ) - logger.info(f"Saved configs for {self.kernel_name} - {_static_key}") + logger.debug(f"Saved configs for {self.kernel_name} - {_static_key}") logger.info(f"rank {rank_id} tuning {self.kernel_name} _static_key {static_key} finished") diff --git a/lightllm/distributed/communication_op.py b/lightllm/distributed/communication_op.py index 93c603212d..e272395716 100644 --- a/lightllm/distributed/communication_op.py +++ b/lightllm/distributed/communication_op.py @@ -50,7 +50,7 @@ HAS_DEEPEP = True except: HAS_DEEPEP = False - logger.info("deep_ep is not installed, you can't use the api of it.") + logger.debug("deep_ep is not installed, you can't use the api of it.") class CustomProcessGroup: diff --git a/lightllm/distributed/flashinfer_all_reduce.py b/lightllm/distributed/flashinfer_all_reduce.py index 35cb0aaedf..02a1c2fb65 100644 --- a/lightllm/distributed/flashinfer_all_reduce.py +++ b/lightllm/distributed/flashinfer_all_reduce.py @@ -6,7 +6,7 @@ import torch.distributed as dist from torch.distributed import ProcessGroup -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import init_logger, summarize_exception logger = init_logger(__name__) @@ -104,7 +104,12 @@ def _ensure_workspace(self, hidden_dim: int, dtype: torch.dtype) -> bool: comm_backend=TorchDistBackend(group=self.group), ) except Exception as e: - logger.warning("FlashInferAllReduce workspace init failed: %s. Disabling.", e) + if self.rank == 0: + logger.warning( + "event=allreduce.unavailable backend=flashinfer reason=%r action=disable_backend", + summarize_exception(e), + ) + logger.debug("FlashInfer workspace initialization diagnostics", exc_info=True) self.disabled = True self._workspace = None return False diff --git a/lightllm/models/qwen3next/model.py b/lightllm/models/qwen3next/model.py index 08233444eb..6704e6bf0b 100644 --- a/lightllm/models/qwen3next/model.py +++ b/lightllm/models/qwen3next/model.py @@ -25,7 +25,6 @@ class Qwen3NextTpPartModel(Qwen3MOEModel): - # weight class pre_and_post_weight_class = Qwen3NextPreAndPostLayerWeight transformer_weight_class = Qwen3NextTransformerLayerWeight @@ -47,7 +46,7 @@ def _triton_allocator(size: int, alignment: int, stream: Optional[int]) -> torch # Set Triton allocator for TMA descriptors # This is required for kernels in common/.../linear_att/fla/ops/solve_tril.py triton.set_allocator(_triton_allocator) - logger.info("Triton allocator set for Qwen3Next model") + logger.debug("Triton allocator set for Qwen3Next model") return def autotune_layers(self): diff --git a/lightllm/server/api_anthropic.py b/lightllm/server/api_anthropic.py index 299fd524de..3f2608c05a 100644 --- a/lightllm/server/api_anthropic.py +++ b/lightllm/server/api_anthropic.py @@ -32,7 +32,7 @@ from fastapi.responses import JSONResponse, Response from lightllm.utils.envs_utils import get_env_start_args -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import init_logger, log_debug_event from .api_errors import is_rate_limit_error @@ -968,7 +968,7 @@ async def _openai_sse_to_anthropic_events( try: chunk = json.loads(payload) except Exception: - logger.debug("Skipping non-JSON SSE payload: %r", payload) + log_debug_event(logger, "sse.payload_skipped", payload_chars=len(payload), reason="invalid_json") continue if "error" in chunk and "choices" not in chunk: diff --git a/lightllm/server/api_cli.py b/lightllm/server/api_cli.py index 2ecb26c607..444e155e65 100644 --- a/lightllm/server/api_cli.py +++ b/lightllm/server/api_cli.py @@ -3,6 +3,55 @@ def add_cli_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: parser.formatter_class = argparse.RawTextHelpFormatter + logging_group = parser.add_argument_group("Logging") + logging_group.add_argument( + "--log-level", + "--log_level", + choices=["debug", "info", "warning", "error", "critical"], + default=None, + help="Service log level (LIGHTLLM_LOG_LEVEL; default: info).", + ) + logging_group.add_argument( + "--log-format", + "--log_format", + choices=["text", "json"], + default=None, + help="Log output format (LIGHTLLM_LOG_FORMAT; default: text).", + ) + logging_group.add_argument( + "--log-module-levels", + "--log_module_levels", + default=None, + help='JSON mapping, e.g. {"lightllm.server.router": "DEBUG"}. Overrides inherited levels.', + ) + logging_group.add_argument( + "--log-request-sample-rate", + "--log_request_sample_rate", + type=float, + default=None, + help="Successful request summary sampling rate, 0..1 (default: 0.01, 1%). Failures are not sampled.", + ) + logging_group.add_argument( + "--log-stats-interval", + "--log_stats_interval", + type=float, + default=None, + help="Engine summary interval in seconds (default: 10).", + ) + logging_group.add_argument( + "--log-http-level", + "--log_http_level", + default=None, + choices=["debug", "info", "warning", "error", "critical"], + help="HTTP server error logger level (default: warning).", + ) + logging_group.add_argument( + "--enable-http-access-log", + "--enable_http_access_log", + action="store_true", + default=None, + help="Enable HTTP access logs independently of service/request logs (default: disabled).", + ) parser.add_argument( "--run_mode", diff --git a/lightllm/server/api_http.py b/lightllm/server/api_http.py index cc33846023..6b22fbd987 100755 --- a/lightllm/server/api_http.py +++ b/lightllm/server/api_http.py @@ -45,7 +45,14 @@ from .httpserver_for_pd_master.manager import HttpServerManagerForPDMaster from .api_lightllm import lightllm_get_score from lightllm.utils.envs_utils import get_env_start_args -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import ( + init_logger, + set_log_context, + log_http_access, + new_http_request_id, + set_http_request_id, + reset_http_request_id, +) from lightllm.utils.error_utils import ClientDisconnected, InvalidRequestError, SERVER_BUSY_MESSAGE, ServerBusyError from lightllm.server.metrics.manager import MetricClient from lightllm.utils.envs_utils import get_unique_server_name @@ -82,6 +89,7 @@ class G_Objs: def set_args(self, args: StartArgs): self.args = args + set_log_context(component="http", node_rank=args.node_rank) from .api_lightllm import lightllm_generate, lightllm_generate_stream from .api_tgi import tgi_generate_impl, tgi_generate_stream_impl @@ -119,36 +127,45 @@ def set_args(self, args: StartArgs): app = FastAPI() g_objs.app = app -_ACCESS_LOG_STATUS_COLORS = {2: "\033[32m", 3: "\033[36m", 4: "\033[33m", 5: "\033[31m"} -_ACCESS_LOG_RESET = "\033[0m" - class _AccessLogMiddleware: def __init__(self, app): self.app = app async def __call__(self, scope, receive, send): - if scope["type"] not in ("http", "websocket"): + if scope["type"] != "http": await self.app(scope, receive, send) return - status_holder = {"status": 0} + request_id = new_http_request_id(scope.get("headers", [])) + scope = dict(scope) + scope["headers"] = [(k, v) for k, v in scope.get("headers", []) if k.lower() != b"x-request-id"] + scope["headers"].append((b"x-request-id", request_id.encode("ascii"))) + token = set_http_request_id(request_id) + started = time.monotonic() + status_holder = {"status": 500} async def send_wrapper(message): if message["type"] == "http.response.start": status_holder["status"] = message["status"] + message = dict(message) + message["headers"] = [(k, v) for k, v in message.get("headers", []) if k.lower() != b"x-request-id"] + message["headers"].append((b"x-request-id", request_id.encode("ascii"))) await send(message) try: await self.app(scope, receive, send_wrapper) finally: - if scope["type"] == "http": - status = status_holder["status"] - msg = f"{scope['method']} {scope['path']} {status}" - color = _ACCESS_LOG_STATUS_COLORS.get(status // 100, "") - if color: - msg = color + msg + _ACCESS_LOG_RESET - logger.info(msg) + try: + log_http_access( + scope["method"], + scope["path"], + status_holder["status"], + (time.monotonic() - started) * 1000, + http_request_id=request_id, + ) + finally: + reset_http_request_id(token) app.add_middleware(_AccessLogMiddleware) @@ -309,7 +326,7 @@ async def generate(request: Request) -> Response: except ValueError as e: return create_error_response(HTTPStatus.BAD_REQUEST, str(e)) except ClientDisconnected as e: - logger.warning(str(e)) + logger.debug(str(e)) return Response(status_code=499) except Exception as e: logger.error("An error occurred: %s", str(e), exc_info=True) @@ -331,7 +348,7 @@ async def generate_stream(request: Request) -> Response: except ValueError as e: return create_error_response(HTTPStatus.BAD_REQUEST, str(e)) except ClientDisconnected as e: - logger.warning(str(e)) + logger.debug(str(e)) return Response(status_code=499) except Exception as e: logger.error("An error occurred: %s", str(e), exc_info=True) @@ -351,7 +368,7 @@ async def get_score(request: Request) -> Response: logger.warning("Server busy detail: %s", e.message) return create_server_busy_response(e) except ClientDisconnected as e: - logger.warning(str(e)) + logger.debug(str(e)) return Response(status_code=499) except Exception as e: return create_error_response(HTTPStatus.EXPECTATION_FAILED, str(e)) @@ -387,7 +404,7 @@ async def chat_completions(request: ChatCompletionRequest, raw_request: Request) logger.warning("Server busy detail: %s", e.message) return create_server_busy_response(e) except ClientDisconnected as e: - logger.warning(str(e)) + logger.debug(str(e)) return Response(status_code=499) return resp @@ -407,7 +424,7 @@ async def completions(request: CompletionRequest, raw_request: Request) -> Respo logger.warning("Server busy detail: %s", e.message) return create_server_busy_response(e) except ClientDisconnected as e: - logger.warning(str(e)) + logger.debug(str(e)) return Response(status_code=499) return resp @@ -429,7 +446,7 @@ async def anthropic_messages(raw_request: Request) -> Response: g_objs.metric_client.counter_inc("lightllm_request_failure") return _anthropic_error_response(HTTPStatus(e.status_code), SERVER_BUSY_MESSAGE) except ClientDisconnected as e: - logger.warning(str(e)) + logger.debug(str(e)) return Response(status_code=499) @@ -440,7 +457,7 @@ async def anthropic_count_tokens(raw_request: Request) -> Response: try: return await anthropic_count_tokens_impl(raw_request) except ClientDisconnected as e: - logger.warning(str(e)) + logger.debug(str(e)) return Response(status_code=499) except Exception as e: logger.error("An error occurred: %s", str(e), exc_info=True) @@ -463,7 +480,7 @@ async def openai_responses(raw_request: Request) -> Response: except ValueError as e: return create_error_response(HTTPStatus.BAD_REQUEST, str(e)) except ClientDisconnected as e: - logger.warning(str(e)) + logger.debug(str(e)) return Response(status_code=499) except Exception as e: logger.error("An error occurred: %s", str(e), exc_info=True) @@ -515,7 +532,7 @@ async def tokens(request: Request): except ValueError as e: return create_error_response(HTTPStatus.BAD_REQUEST, str(e)) except ClientDisconnected as e: - logger.warning(str(e)) + logger.debug(str(e)) return Response(status_code=499) except Exception as e: return create_error_response(HTTPStatus.EXPECTATION_FAILED, f"error: {str(e)}") @@ -577,9 +594,9 @@ async def shutdown(): @app.on_event("startup") async def startup_event(): - logger.info("server start up") + set_log_context(component="http") loop = asyncio.get_event_loop() g_objs.set_args(get_env_start_args()) loop.create_task(g_objs.httpserver_manager.handle_loop()) - logger.info(f"server start up ok, loop use is {asyncio.get_event_loop()}") + logger.info("event=http.initialized", extra={"event": "http.initialized"}) return diff --git a/lightllm/server/api_http_pd.py b/lightllm/server/api_http_pd.py index 0081d83028..7696086b4d 100644 --- a/lightllm/server/api_http_pd.py +++ b/lightllm/server/api_http_pd.py @@ -16,7 +16,7 @@ from lightllm.server.pd_io_struct import ObjType from lightllm.utils.envs_utils import get_lightllm_websocket_max_message_size -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import init_logger, log_debug_event logger = init_logger(__name__) @@ -32,7 +32,8 @@ async def register_and_keep_alive(websocket: WebSocket): client_ip, client_port = websocket.client logger.info(f"Client connected from IP: {client_ip}, Port: {client_port}") regist_json = json.loads(await websocket.receive_text()) - logger.info(f"received regist_json {regist_json}") + peer = f"{regist_json.get('mode')}@{regist_json.get('client_ip_port')}" + logger.debug("PD registration received: peer=%s", peer) await g_objs.httpserver_manager.register_pd(regist_json, websocket) try: @@ -45,18 +46,17 @@ async def register_and_keep_alive(websocket: WebSocket): await g_objs.httpserver_manager.put_to_handle_queue(obj) except asyncio.TimeoutError: - logger.warning(f"client {regist_json} heartbeat timed out after {heartbeat_timeout_seconds} seconds") + logger.warning(f"client {peer} heartbeat timed out after {heartbeat_timeout_seconds} seconds") try: await websocket.close(code=1011, reason="PD heartbeat timed out") except BaseException: - logger.debug(f"failed to close timed-out client {regist_json}", exc_info=True) + logger.debug(f"failed to close timed-out client {peer}", exc_info=True) except WebSocketDisconnect as e: - logger.info(f"client {regist_json} disconnected: {str(e)}") + logger.info(f"client {peer} disconnected: {str(e)}") except BaseException as e: - logger.error(f"client {regist_json} has error {str(e)}") - logger.exception(str(e)) + logger.exception(f"client {peer} has error {str(e)}") finally: - logger.error(f"client {regist_json} removed") + logger.info(f"client {peer} removed") await g_objs.httpserver_manager.remove_pd(regist_json) return @@ -72,9 +72,10 @@ async def kv_move_status(websocket: WebSocket): while True: data = await websocket.receive_bytes() upkv_status = pickle.loads(data) - logger.debug(f"received upkv_status {upkv_status} from {(client_ip, client_port)}") + log_debug_event(logger, "pd.kv_ready", request_id=upkv_status.group_request_id, peer=client_ip) await g_objs.httpserver_manager.update_req_status(upkv_status) - except (WebSocketDisconnect, Exception, RuntimeError) as e: - logger.error(f"kv_move_status client {(client_ip, client_port)} has error {str(e)}") - logger.exception(str(e)) + except WebSocketDisconnect: + logger.info("KV status connection closed: peer=%s:%s", client_ip, client_port) + except Exception as e: + logger.exception(f"kv_move_status client {(client_ip, client_port)} has error {str(e)}") return diff --git a/lightllm/server/api_openai.py b/lightllm/server/api_openai.py index bb675c7810..2cc6c490b2 100644 --- a/lightllm/server/api_openai.py +++ b/lightllm/server/api_openai.py @@ -37,7 +37,7 @@ ) from lightllm.utils.error_utils import ClientDisconnected, InvalidRequestError, SERVER_BUSY_MESSAGE, ServerBusyError -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import init_logger, log_debug_event from lightllm.server.metrics.manager import MetricClient from lightllm.utils.envs_utils import get_unique_server_name from dataclasses import dataclass @@ -102,7 +102,7 @@ async def _safe_stream_wrapper(stream_generator): yield f"data: {error_data}\n\n" yield "data: [DONE]\n\n" except ClientDisconnected as e: - logger.warning(str(e)) + logger.debug(str(e)) # Client is gone — there's no point yielding more SSE chunks. Stop quietly. return diff --git a/lightllm/server/api_responses.py b/lightllm/server/api_responses.py index e1b79acc78..01df0973a3 100644 --- a/lightllm/server/api_responses.py +++ b/lightllm/server/api_responses.py @@ -9,7 +9,7 @@ from fastapi import Request from fastapi.responses import JSONResponse, Response -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import init_logger, log_debug_event from .api_errors import create_error_response, is_rate_limit_error from .api_stream_obj import CustomStreamingResponse @@ -441,7 +441,7 @@ def open_item(kind: str, item: Dict[str, Any]): try: chunk = json.loads(payload) except Exception: - logger.debug("Skipping non-JSON SSE payload: %r", payload) + log_debug_event(logger, "sse.payload_skipped", payload_chars=len(payload), reason="invalid_json") continue if "error" in chunk and "choices" not in chunk: diff --git a/lightllm/server/api_server.py b/lightllm/server/api_server.py index 501c13239d..bf2e2bafa3 100755 --- a/lightllm/server/api_server.py +++ b/lightllm/server/api_server.py @@ -1,7 +1,9 @@ +import os +import time import torch from .api_cli import add_cli_args from lightllm.server.core.objs.start_args_type import StartArgs -from lightllm.utils.log_utils import init_logger, set_log_node_role +from lightllm.utils.log_utils import init_logger, configure_logging_from_args logger = init_logger(__name__) @@ -9,7 +11,14 @@ def launch_server(args: StartArgs): if args.use_dp_split_mode_connect_pd_master and args.run_mode != "prefill": raise ValueError("--use_dp_split_mode_connect_pd_master only supports prefill") - set_log_node_role(args.run_mode) + configure_logging_from_args(args) + os.environ["LIGHTLLM_SERVICE_START_TIME"] = str(time.monotonic()) + logger.info( + "event=service.starting role=%s model=%s", + args.run_mode, + args.model_name, + extra={"event": "service.starting", "run_mode": args.run_mode, "model": args.model_name}, + ) from .api_start import pd_master_start, normal_or_p_d_start, config_server_start, visual_only_start try: diff --git a/lightllm/server/api_start.py b/lightllm/server/api_start.py index ca680f9f0f..5901f2a8e8 100644 --- a/lightllm/server/api_start.py +++ b/lightllm/server/api_start.py @@ -6,7 +6,7 @@ from lightllm.utils.start_utils import process_manager from .metrics.manager import start_metric_manager from .embed_cache.manager import start_cache_manager -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import init_logger, log_startup_config, get_http_logging_args from lightllm.utils.envs_utils import set_env_start_args, set_unique_server_name, get_unique_server_name from lightllm.utils.shm_port_args import get_shm_port_args from lightllm.utils.net_utils import validate_ports @@ -124,7 +124,7 @@ def _launch_subprocesses(args: StartArgs): assert args.zmq_mode in ["tcp://", "ipc:///tmp/"] - logger.info(f"use tgi api: {args.use_tgi_api}") + logger.debug(f"use tgi api: {args.use_tgi_api}") # 当使用config_server来初始化nccl时，nccl_host和config_server_host必须一致 if args.use_config_server_to_init_nccl: @@ -336,7 +336,7 @@ def _launch_subprocesses(args: StartArgs): zmq_mode = f"{args.zmq_mode}_{get_unique_server_name()}_" args.zmq_mode = None # args 的参数不能直接设置，只能先设置None，再设置才能成功 args.zmq_mode = zmq_mode - logger.info(f"zmq mode head: {args.zmq_mode}") + logger.debug(f"zmq mode head: {args.zmq_mode}") # p d 分离模式下用于标识节点的id args.pd_node_id = uuid.uuid4().int @@ -377,7 +377,7 @@ def _launch_subprocesses(args: StartArgs): # 多机用于收发node ip, 这个地方修改了args env,所以需要重新设置一下。 send_and_receive_node_ip(args) set_env_start_args(args) - logger.info(f"all start args:{args}") + log_startup_config(logger, args) if args.enable_multimodal: process_manager.start_submodule_processes( @@ -469,12 +469,7 @@ def normal_or_p_d_start(args: StartArgs): f"{args.httpserver_workers}", "--bind", f"{args.host}:{get_shm_port_args().port}", - "--log-level", - "info", - "--access-logfile", - "-", - "--error-logfile", - "-", + *get_http_logging_args(application_access=True), "lightllm.server.api_http:app", ] @@ -490,6 +485,7 @@ def normal_or_p_d_start(args: StartArgs): from lightllm.server.health_monitor.manager import start_health_check_process process_manager.start_submodule_processes(start_funcs=[start_health_check_process], start_args=[(args,)]) + process_manager.start_readiness_monitor(http_server_process, args) process_manager.setup_signal_handlers(http_server_process) process_manager.supervise_processes(http_server_process) @@ -512,13 +508,13 @@ def pd_master_start(args: StartArgs): else: args.pd_node_id = 0 - logger.info(f"use tgi api: {args.use_tgi_api}") + logger.debug(f"use tgi api: {args.use_tgi_api}") validate_ports([args.port]) set_env_start_args(args) process_manager.setup_exit_controller() get_shm_port_args(create=True) - logger.info(f"all start args:{args}") + log_startup_config(logger, args) process_manager.start_submodule_processes( start_funcs=[ @@ -534,12 +530,7 @@ def pd_master_start(args: StartArgs): "1", "--bind", f"{args.host}:{get_shm_port_args().port}", - "--log-level", - "info", - "--access-logfile", - "-", - "--error-logfile", - "-", + *get_http_logging_args(application_access=True), "lightllm.server.api_http:app", ] @@ -550,6 +541,7 @@ def pd_master_start(args: StartArgs): process_manager.start_submodule_processes(start_funcs=[start_health_check_process], start_args=[(args,)]) + process_manager.start_readiness_monitor(http_server_process, args) process_manager.setup_signal_handlers(http_server_process) process_manager.supervise_processes(http_server_process) @@ -584,7 +576,7 @@ def visual_only_start(args): set_env_start_args(args) process_manager.setup_exit_controller() get_shm_port_args(create=True) - logger.info(f"all start args:{args}") + log_startup_config(logger, args) from .visualserver.visual_only_manager import start_visual_process @@ -612,7 +604,7 @@ def config_server_start(args): set_env_start_args(args) process_manager.setup_exit_controller() get_shm_port_args(create=True) - logger.info(f"all start args:{args}") + log_startup_config(logger, args) if args.config_server_visual_redis_port is not None: start_redis_service(args) @@ -624,16 +616,12 @@ def config_server_start(args): "1", "--bind", f"{args.config_server_host}:{get_shm_port_args().config_server_port}", - "--log-level", - "info", - "--access-logfile", - "-", - "--error-logfile", - "-", + *get_http_logging_args(), "lightllm.server.config_server.api_http:app", ] http_server_process = subprocess.Popen(command) + process_manager.start_readiness_monitor(http_server_process, args) process_manager.setup_signal_handlers(http_server_process) process_manager.supervise_processes(http_server_process) diff --git a/lightllm/server/audioserver/model_infer/model_rpc.py b/lightllm/server/audioserver/model_infer/model_rpc.py index 82919856d9..6db627e40a 100644 --- a/lightllm/server/audioserver/model_infer/model_rpc.py +++ b/lightllm/server/audioserver/model_infer/model_rpc.py @@ -61,11 +61,7 @@ def exposed_init_model(self, kvargs): ) self._init_taskes() except Exception as e: - print("#" * 16) - print("load model error:", str(e), e, type(e)) - import traceback - - traceback.print_exc() + logger.exception("Model loading failed") raise e set_random_seed(2147483647) @@ -88,7 +84,7 @@ def _log_latency(self, audio: AudioItem, stage: str): return latency = time.time() - audio.start_time if latency > 0.02: - logger.info(f"{stage} latency {latency:.4f} seconds for audio with md5 {audio.md5}") + logger.debug(f"{stage} latency {latency:.4f} seconds for audio with md5 {audio.md5}") audio.start_time = time.time() def _init_taskes(self): diff --git a/lightllm/server/build_prompt.py b/lightllm/server/build_prompt.py index 6e55ebd68f..e95fc5cd21 100644 --- a/lightllm/server/build_prompt.py +++ b/lightllm/server/build_prompt.py @@ -55,16 +55,16 @@ def tokenizer_supports_force_thinking() -> bool: try: ans = "thinking" in tokenizer.chat_template or "enable_thinking" in tokenizer.chat_template - logger.debug(f"chat_template: {tokenizer.chat_template}") - logger.info(f"tokenizer_supports_force_thinking : {ans}") + logger.debug("chat_template chars=%s", len(tokenizer.chat_template)) + logger.debug(f"tokenizer_supports_force_thinking : {ans}") return ans except: pass try: ans = "thinking" in tokenizer.tokenizer.chat_template or "enable_thinking" in tokenizer.tokenizer.chat_template - logger.debug(f"tokenizer.tokenizer.chat_template: {tokenizer.tokenizer.chat_template}") - logger.info(f"tokenizer_supports_force_thinking : {ans}") + logger.debug("chat_template chars=%s", len(tokenizer.tokenizer.chat_template)) + logger.debug(f"tokenizer_supports_force_thinking : {ans}") return ans except: pass diff --git a/lightllm/server/config_server/api_http.py b/lightllm/server/config_server/api_http.py index 3ce39bb6e6..61a5f39700 100644 --- a/lightllm/server/config_server/api_http.py +++ b/lightllm/server/config_server/api_http.py @@ -65,11 +65,12 @@ async def websocket_endpoint(websocket: WebSocket): while True: data = await websocket.receive_text() assert data == "heartbeat" - except (WebSocketDisconnect, Exception, RuntimeError) as e: - logger.error(f"registered_pd_master_obj {registered_pd_master_obj} has error {str(e)}") - logger.exception(str(e)) + except WebSocketDisconnect: + logger.info("Registration connection closed: peer=%s:%s", client_ip, client_port) + except Exception as e: + logger.exception(f"registered_pd_master_obj {registered_pd_master_obj} has error {str(e)}") finally: - logger.error(f"registered_pd_master_obj {registered_pd_master_obj} removed") + logger.info(f"registered_pd_master_obj {registered_pd_master_obj} removed") with registered_pd_master_obj_lock: registered_pd_master_objs.pop(registered_pd_master_obj.node_id, None) return @@ -89,11 +90,12 @@ async def visual_websocket_endpoint(websocket: WebSocket): while True: data = await websocket.receive_text() assert data == "heartbeat" - except (WebSocketDisconnect, Exception, RuntimeError) as e: - logger.error(f"registered_visual_server_obj {registered_visual_server_obj} has error {str(e)}") - logger.exception(str(e)) + except WebSocketDisconnect: + logger.info("Registration connection closed: peer=%s:%s", client_ip, client_port) + except Exception as e: + logger.exception(f"registered_visual_server_obj {registered_visual_server_obj} has error {str(e)}") finally: - logger.error(f"registered_visual_server_obj {registered_visual_server_obj} removed") + logger.info(f"registered_visual_server_obj {registered_visual_server_obj} removed") with registered_visual_server_obj_lock: registered_visual_server_objs.pop(registered_visual_server_obj.node_id, None) return @@ -132,7 +134,7 @@ async def allocate_global_id_range(): global global_req_id range_size = 800000 with global_req_id_lock: - if global_req_id + range_size > 2 ** 63 - 1: + if global_req_id + range_size > 2**63 - 1: global_req_id = 0 start_id = global_req_id global_req_id += range_size @@ -146,7 +148,7 @@ async def allocate_global_unique_multimodal_id_range(): global global_multimodal_embedding_id range_size = 8000000 with global_multimodal_embedding_id_lock: - if global_multimodal_embedding_id + range_size > 2 ** 63 - 1: + if global_multimodal_embedding_id + range_size > 2**63 - 1: global_multimodal_embedding_id = 100000000 start_id = global_multimodal_embedding_id global_multimodal_embedding_id += range_size @@ -175,9 +177,6 @@ async def http_start_tcp_store_server( Returns: dict: A dictionary containing the status of the server. """ - global global_store_port_to_process - global global_store_port_to_client_states - global global_store_port_lock args = get_env_start_args() diff --git a/lightllm/server/core/objs/shm_req_manager.py b/lightllm/server/core/objs/shm_req_manager.py index 2aefdec726..d8aa586881 100644 --- a/lightllm/server/core/objs/shm_req_manager.py +++ b/lightllm/server/core/objs/shm_req_manager.py @@ -113,8 +113,6 @@ def release_req_index(self, req_index_in_mem): self.alloc_state_shm.arr[req_index_in_mem] = 0 self.linked_req_manager.free(req_index_in_mem) - if np.sum(self.alloc_state_shm.arr) == 0 and self.linked_req_manager.test_is_full(): - logger.info("all shm req has been release ok") return async def async_release_req_index(self, req_index_in_mem): diff --git a/lightllm/server/core/objs/start_args_type.py b/lightllm/server/core/objs/start_args_type.py index 55f9ce24c0..1bd5c6a32e 100644 --- a/lightllm/server/core/objs/start_args_type.py +++ b/lightllm/server/core/objs/start_args_type.py @@ -16,6 +16,13 @@ class StartArgs: httpserver_workers: int = field(default=1) disable_delay_response_start: bool = field(default=False) hypercorn_config: Optional[str] = field(default=None) + log_level: Optional[str] = field(default=None) + log_format: Optional[str] = field(default=None) + log_module_levels: Optional[str] = field(default=None) + log_request_sample_rate: Optional[float] = field(default=None) + log_stats_interval: Optional[float] = field(default=None) + log_http_level: Optional[str] = field(default=None) + enable_http_access_log: Optional[bool] = field(default=None) zmq_mode: str = field( default="ipc:///tmp/", metadata={"help": "use socket mode or ipc mode, only can be set in ['tcp://', 'ipc:///tmp/']"}, diff --git a/lightllm/server/detokenization/manager.py b/lightllm/server/detokenization/manager.py index 3a57aee9bc..3ce42d9b5b 100644 --- a/lightllm/server/detokenization/manager.py +++ b/lightllm/server/detokenization/manager.py @@ -14,7 +14,7 @@ from ..tokenizer import get_tokenizer import pickle import time -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import init_logger, log_debug_event, DebugStatsLogger from lightllm.utils.envs_utils import get_unique_server_name from lightllm.utils.shm_port_args import get_shm_port_args @@ -34,7 +34,7 @@ def __init__( self.pub_to_httpserver = context.socket(zmq.PUB) self.pub_to_httpserver.bind(f"{args.zmq_mode}127.0.0.1:{ports.http_server_port}") - logger.info(f"pub_to_httpserver sendhwm {self.pub_to_httpserver.getsockopt(zmq.SNDHWM)}") + logger.debug(f"pub_to_httpserver sendhwm {self.pub_to_httpserver.getsockopt(zmq.SNDHWM)}") self.tokenizer = get_tokenizer(args.model_dir, args.tokenizer_mode, trust_remote_code=args.trust_remote_code) self.all_special_ids = set(self.tokenizer.all_special_ids) self.req_id_to_out: Dict[int, DecodeReq] = {} @@ -56,6 +56,7 @@ def _add_new_group_req_index(self, recv_obj: GroupReqIndexes): return def handle_loop(self): + slow_stats = DebugStatsLogger(logger, "detoken.slow_batches") try: recv_max_count = 128 while True: @@ -76,9 +77,10 @@ def handle_loop(self): exist_need_detoken = self.gen_token_out() cost_time = (time.time() - start_time) * 1000 if cost_time > 50: - logger.info(f"detokenize batch cost time {cost_time} ms") + slow_stats.add(duration_ms_total=cost_time) if not exist_need_detoken: + slow_stats.flush() time.sleep(0.002) except Exception as e: @@ -124,7 +126,12 @@ def remove_finished_reqs(self): for decode_req in finished_reqs: decode_req.req.can_released_mark = True - logger.debug(f"detoken release req id {decode_req.req.request_id}") + log_debug_event( + logger, + "detoken.completed", + request_id=decode_req.req.request_id, + output_tokens=len(decode_req.output_ids), + ) self.shm_req_manager.put_back_req_obj(decode_req.req) self.req_id_to_out.pop(decode_req.request_id, None) return diff --git a/lightllm/server/embed_cache/utils.py b/lightllm/server/embed_cache/utils.py index 1cc655aa1b..e0aceb83e0 100644 --- a/lightllm/server/embed_cache/utils.py +++ b/lightllm/server/embed_cache/utils.py @@ -1,3 +1,7 @@ +from lightllm.utils.log_utils import init_logger + +logger = init_logger(__name__) + from lightllm.utils.shm_utils import ServiceSharedMemory, get_service_shm_name @@ -9,7 +13,7 @@ def create_shm(name, data): mem_view = shared_memory.buf mem_view[:data_size] = data except FileExistsError: - print("Warning create shm {} failed because of FileExistsError!".format(name)) + logger.warning("Shared memory already exists: name=%s", name) def read_shm(name): diff --git a/lightllm/server/health_monitor/manager.py b/lightllm/server/health_monitor/manager.py index f6ece99ce6..3e208fcfbe 100644 --- a/lightllm/server/health_monitor/manager.py +++ b/lightllm/server/health_monitor/manager.py @@ -26,21 +26,22 @@ def health_monitor(url, all_process_ids): if all_processes_is_alive: response = requests.get(url, timeout=60) if response.status_code == 200: - logger.info("Health check passed") + if consecutive_failures: + logger.info("event=health.recovered previous_failures=%s", consecutive_failures) consecutive_failures = 0 else: raise Exception(f"Health check failed with status code: {response.status_code}") else: raise Exception("not all processes is alive") except asyncio.TimeoutError: - logger.error("Health check request timed out") + logger.warning("event=health.check_failed reason=timeout") consecutive_failures += 1 except Exception as e: - logger.error(f"Health check failed with exception: {e}") + logger.warning("event=health.check_failed reason=%s", e) consecutive_failures += 1 if consecutive_failures >= 3: - logger.error(f"kill all process consecutive_failures: {consecutive_failures}") + logger.critical("event=health.termination consecutive_failures=%s", consecutive_failures) import signal # 先kill父进程的子进程 @@ -95,16 +96,15 @@ def start_health_check_process(args, pipe_writer): pipe_writer.send("init ok") all_process_ids = get_all_cared_pids() - logger.info(f"health monitor care process ids {all_process_ids}") + logger.debug(f"health monitor care process ids {all_process_ids}") - global consecutive_failures host, port = args.host, get_shm_port_args().port url = f"http://{host}:{port}/health".format(host=host, port=port) interval_seconds = int(os.environ.get("HEALTH_CHECK_INTERVAL_SECONDS", 88)) logger.info("Waiting for the server to start up.") logger.info(f"check interval seconds {interval_seconds}") time.sleep(6) - logger.info("Server has started, starting the health check process.") + logger.info("event=health.monitor_started interval_s=%s", interval_seconds) while True: health_monitor(url, all_process_ids) diff --git a/lightllm/server/httpserver/manager.py b/lightllm/server/httpserver/manager.py index 1b90bcfad6..3eee51e6ed 100644 --- a/lightllm/server/httpserver/manager.py +++ b/lightllm/server/httpserver/manager.py @@ -2,6 +2,7 @@ import zmq import zmq.asyncio import asyncio +import itertools import uvloop import rpyc import socket @@ -32,7 +33,13 @@ from lightllm.server.core.objs.shm_req_manager import ShmReqManager from lightllm.server.core.objs.atomic_array_lock import AtomicShmArrayLock, AsyncLock, AtomicLockItem from lightllm.server.router.dynamic_prompt.shared_arr import SharedInt -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import ( + request_finish_fields, + init_logger, + log_request_summary, + log_debug_event, + get_http_request_id, +) from lightllm.server.metrics.manager import MetricClient from .rl_controller import HttpRlController from .manager_ext import HttpRlManagerHelper @@ -344,7 +351,6 @@ async def _generate( # 用于等待 pd_master 下发的交换信息 pd_event: asyncio.Event = None, ) -> AsyncGenerator[Tuple[int, str, dict, FinishStatus], None]: - start_time = time.time() # 在请求登记、编码和 PD 握手前获取语法；缓存未命中时先完成编译，避免后续流程等待或校验失败。 compiled_grammar = await sampling_params.get_compiled_grammar(self.output_grammar_compiler) @@ -476,10 +482,14 @@ async def _generate( "shm_req_init_done", ) - logger.debug( - f"alloc shm_req for req_id {group_request_id}, " - f"shm_req num: {sampling_params.n} details (req_id, index_in_shm_mem): " - f"{[(req_obj.request_id, req_obj.index_in_shm_mem) for req_obj in req_objs]}" + log_debug_event( + logger, + "request.admitted", + request_id=group_request_id, + choices=sampling_params.n, + shm_index_sample=[req.index_in_shm_mem for req in req_objs[:8]], + prompt_tokens=len(prompt_ids), + elapsed_ms=round((time.time() - start_time) * 1000, 3), ) req_status = ReqStatus(group_request_id, multimodal_params, req_objs, start_time) @@ -526,12 +536,13 @@ async def _generate( yield sub_req_id, request_output, metadata, finish_status except (asyncio.CancelledError, BaseException) as e: - if isinstance(e, ClientDisconnected): - logger.warning(f"group_request_id: {group_request_id} {e.reason}") - elif isinstance(e, asyncio.CancelledError): - logger.warning(f"group_request_id: {group_request_id} has been cancelled") - else: - logger.warning(f"group_request_id: {group_request_id} has exception {str(e)}") + cancelled = isinstance(e, (ClientDisconnected, asyncio.CancelledError, GeneratorExit)) + log_request_summary( + group_request_id, + status="cancelled" if cancelled else "timeout" if isinstance(e, asyncio.TimeoutError) else "error", + timing_scope=self.args.run_mode if self.args.run_mode in ("prefill", "decode") else "end_to_end", + error_type=type(e).__name__, + ) # error need to release multimodel resources. # 对于还没有形成正式请求对象管理的多模态资源，需要单独自己释放 @@ -607,15 +618,9 @@ async def _alloc_shm_req_indexes( return alloced_req_indexes async def _log_req_header(self, request_headers, group_request_id: int): - x_request_id = request_headers.get("X-Request-Id", "") - x_session_id = request_headers.get("X-Session-Id", "") - - format_in_time = datetime.datetime.fromtimestamp(time.time()).strftime("%Y-%m-%d %H:%M:%S") - logger.info( - f"received req X-Request-Id:{x_request_id} " - f"X-Session-Id:{x_session_id} start_time:{format_in_time} " - f"lightllm_req_id:{group_request_id} " - ) + x_request_id = request_headers.get("X-Request-Id") or get_http_request_id() + + log_debug_event(logger, "request.received", request_id=group_request_id, http_request_id=x_request_id) return async def _encode( @@ -649,10 +654,13 @@ async def _encode( ) if self.args.detail_log: - logger.debug( - f"req_id: {sampling_params.group_request_id} prompt: {prompt}\n" - f"samplingparmas: {sampling_params.to_dict()}\n" - f"token_ids: {prompt_ids}" + log_debug_event( + logger, + "request.encoded", + request_id=sampling_params.group_request_id, + prompt_chars=len(prompt), + prompt_tokens=len(prompt_ids), + max_new_tokens=sampling_params.max_new_tokens, ) return prompt_ids @@ -756,7 +764,6 @@ async def transfer_to_next_module( self, group_req_objs: Optional[GroupReqObjs] = None, ): - if self.pd_mode.is_P_or_NORMAL(): if not self.args.disable_vision: self.send_to_visual.send_pyobj(group_req_objs.to_group_req_index(), protocol=pickle.HIGHEST_PROTOCOL) @@ -799,10 +806,10 @@ async def _wait_to_token_package( req_status: "ReqStatus", request: Request, ): - event = req_status.event unfinished_count = sampling_params.best_of out_token_counter = 0 + finish_reasons = set() sub_req_id_to_mtp_accepted_token_num: Dict[int, int] = {} sub_req_id_to_mtp_verify_token_num: Dict[int, int] = {} sub_req_id_to_mtp_verify_step_num: Dict[int, int] = {} @@ -862,6 +869,7 @@ async def _wait_to_token_package( yield sub_req_id, out_str, metadata, finish_status # 如果有子请求完成，就更新计数 if finish_status.is_finished(): + finish_reasons.add(finish_status.get_finish_reason()) unfinished_count -= 1 if unfinished_count == 0: @@ -869,10 +877,6 @@ async def _wait_to_token_package( mean_per_token_cost_time_ms = (total_cost_time_ms - first_token_cost_ms) / out_token_counter self.per_token_costs.add(mean_per_token_cost_time_ms) x_request_id = request.headers.get("X-Request-Id", "") if request is not None else "" - x_session_id = request.headers.get("X-Session-Id", "") if request is not None else "" - gpu_prompt_cache_ratio = gpu_prompt_cache_len / prompt_tokens - cpu_prompt_cache_ratio = cpu_prompt_cache_len / prompt_tokens - disk_prompt_cache_ratio = disk_prompt_cache_len / prompt_tokens prompt_cache_len = gpu_prompt_cache_len + cpu_prompt_cache_len + disk_prompt_cache_len prompt_cache_ratio = prompt_cache_len / prompt_tokens generation_throughput = out_token_counter / max(total_cost_time_ms / 1000.0, 1e-6) @@ -883,32 +887,26 @@ async def _wait_to_token_package( if mtp_total_verify_steps <= 0: mtp_total_verify_steps = out_token_counter - mtp_accepted_token_num mtp_avg_token_per_step = out_token_counter / max(mtp_total_verify_steps, 1) - mtp_avg_verify_tokens_per_step = mtp_verify_token_num / max(mtp_total_verify_steps, 1) - mtp_avg_accepted_tokens_per_step = mtp_accepted_token_num / max(mtp_total_verify_steps, 1) - format_start_time = datetime.datetime.fromtimestamp(start_time).strftime("%Y-%m-%d %H:%M:%S") - logger.debug( - f"X-Request-Id:{x_request_id} " - f"X-Session-Id:{x_session_id} start_time:{format_start_time} " - f"lightllm_req_id:{group_request_id} first_token_cost:{first_token_cost_ms}ms " - f"total_cost_time:{total_cost_time_ms}ms,out_token_counter:{out_token_counter} " - f"mean_per_token_cost_time: {mean_per_token_cost_time_ms}ms " - f"prompt_token_num:{prompt_tokens} " - f"gpu cache hit: {gpu_prompt_cache_ratio > 0} " - f"gpu_prompt_cache_len:{gpu_prompt_cache_len} " - f"gpu_prompt_cache_ratio:{gpu_prompt_cache_ratio} " - f"cpu cache hit: {cpu_prompt_cache_len > 0} " - f"cpu_prompt_cache_len:{cpu_prompt_cache_len} " - f"cpu_prompt_cache_ratio:{cpu_prompt_cache_ratio} " - f"disk cache hit: {disk_prompt_cache_len > 0} " - f"disk_prompt_cache_len:{disk_prompt_cache_len} " - f"disk_prompt_cache_ratio:{disk_prompt_cache_ratio} " - f"mtp_accepted_token_num:{mtp_accepted_token_num} " - f"mtp_total_verify_steps:{mtp_total_verify_steps} " - f"mtp_total_verify_tokens:{mtp_verify_token_num} " - f"mtp_avg_token_per_step:{mtp_avg_token_per_step} " - f"mtp_avg_accepted_tokens_per_step:{mtp_avg_accepted_tokens_per_step} " - f"mtp_avg_verify_tokens_per_step:{mtp_avg_verify_tokens_per_step} " - ) + if not self.is_multinode_tp_slave: + log_request_summary( + group_request_id, + **request_finish_fields(finish_reasons), + x_request_id=x_request_id, + prompt_tokens=prompt_tokens, + output_tokens=out_token_counter, + timing_scope=( + self.args.run_mode if self.args.run_mode in ("prefill", "decode") else "end_to_end" + ), + ttft_ms=round(first_token_cost_ms, 3), + duration_ms=round(total_cost_time_ms, 3), + gpu_cache_tokens=gpu_prompt_cache_len, + cpu_cache_tokens=cpu_prompt_cache_len, + disk_cache_tokens=disk_prompt_cache_len, + mtp_enabled=bool(self.args.mtp_step), + mtp_accepted_tokens=mtp_accepted_token_num, + mtp_verify_steps=mtp_total_verify_steps, + mtp_verify_tokens=mtp_verify_token_num, + ) self.metric_client.histogram_observe("lightllm_cache_length", prompt_cache_len) self.metric_client.histogram_observe("lightllm_cache_ratio", prompt_cache_ratio) @@ -948,13 +946,13 @@ async def stop_request(self, request_id: int): async def abort(self, group_req_id: int) -> bool: req_status: ReqStatus = self.req_id_to_out_inf.get(group_req_id, None) if req_status is None: - logger.warning(f"aborted group_request_id {group_req_id} not exist") + logger.debug("Abort skipped for finished request: request_id=%s", group_req_id) return False group_req_objs: GroupReqObjs = req_status.group_req_objs for req in group_req_objs.shm_req_objs: req.is_aborted = True - logger.warning(f"aborted group_request_id {group_req_objs.group_req_id}") + logger.debug("Request aborted: request_id=%s", group_req_objs.group_req_id) return True def _get_router_profiler_client(self): @@ -994,24 +992,32 @@ async def recycle_resource_loop(self): for req_status in release_req_status: self.req_id_to_out_inf.pop(req_status.group_req_objs.group_req_id, None) for req in req_status.group_req_objs.shm_req_objs: - logger.debug(f"httpserver release req_id {req.request_id}, index {req.index_in_shm_mem}") + # Resource release is summarized once per request group below. await self.shm_req_manager.async_put_back_req_obj(req) await self.shm_req_manager.async_release_req_index(req.index_in_shm_mem) await self._release_multimodal_resources(req_status.group_req_objs.multimodal_params) + log_debug_event( + logger, + "request.released", + request_id=req_status.group_req_objs.group_req_id, + choices=len(req_status.group_req_objs.shm_req_objs), + ) - # 先保留这个关键得日志，用于方便定位重构中的问题。 - if time.time() - pre_time_mark > 120: + # Bound pending-resource diagnostics and keep idle services silent. + if self.req_id_to_out_inf and time.time() - pre_time_mark > 120: pre_time_mark = time.time() - for group_req_id_ in list(self.req_id_to_out_inf.keys()): - req_status: ReqStatus = self.req_id_to_out_inf.get(group_req_id_, None) - if req_status is None: - continue + pending_sample = list(itertools.islice(self.req_id_to_out_inf.values(), 8)) + log_debug_event( + logger, + "request.pending", + count=len(self.req_id_to_out_inf), + request_id_sample=[status.group_req_objs.group_req_id for status in pending_sample], + can_release_sample=[ + status.group_req_objs.shm_req_objs[0].can_released_mark for status in pending_sample + ], + ref_count_sample=[status.group_req_objs.shm_req_objs[0].ref_count for status in pending_sample], + ) - logger.info( - f"left req id {req_status.group_req_objs.group_req_id}" - f"can release {req_status.group_req_objs.shm_req_objs[0].can_released_mark} " - f"refcount {req_status.group_req_objs.shm_req_objs[0].ref_count}" - ) return async def handle_loop(self): diff --git a/lightllm/server/httpserver/pd_loop.py b/lightllm/server/httpserver/pd_loop.py index 7631e42e3e..1782a440b8 100644 --- a/lightllm/server/httpserver/pd_loop.py +++ b/lightllm/server/httpserver/pd_loop.py @@ -15,7 +15,13 @@ from lightllm.server.pd_io_struct import NodeRole, ObjType from lightllm.server.httpserver.async_queue import AsyncQueue from lightllm.utils.net_utils import get_hostname_ip -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import ( + init_logger, + set_http_request_id, + reset_http_request_id, + log_debug_event, + log_repeated_exception, +) from lightllm.utils.envs_utils import get_lightllm_websocket_max_message_size from lightllm.server.httpserver.manager import HttpServerManager from ..pd_io_struct import PD_Client_Obj, PD_Master_Obj @@ -48,11 +54,15 @@ async def pd_handle_loop(manager: HttpServerManager, dp_index: Optional[int] = N manager.host_ip = manager.args.host id_to_handle_task: Dict[int, asyncio.Task] = {} + previous_master_ids = None while True: try: id_to_pd_master_obj = await _get_pd_master_objs(manager.args) - logger.info(f"get pd_master_objs {id_to_pd_master_obj}") + master_ids = tuple(sorted(id_to_pd_master_obj)) if id_to_pd_master_obj is not None else None + if master_ids != previous_master_ids: + log_debug_event(logger, "pd.masters_changed", master_ids=master_ids) + previous_master_ids = master_ids if id_to_pd_master_obj is not None: for node_id, pd_master_obj in list(id_to_handle_task.items()): @@ -95,7 +105,6 @@ async def _pd_handle_task(manager: HttpServerManager, pd_master_obj: PD_Master_O # 下方应用层心跳已负责存活检测，禁用协议层 keepalive，避免繁忙连接被误断。 ping_interval=None, ) as websocket: - sock = websocket.transport.get_extra_info("socket") sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) @@ -112,7 +121,13 @@ async def _pd_handle_task(manager: HttpServerManager, pd_master_obj: PD_Master_O } await websocket.send(json.dumps(regist_json)) - logger.info(f"Sent registration JSON: {regist_json}") + logger.info( + "event=pd.registration_sent role=%s endpoint=%s dp_index=%s", + regist_json["mode"], + regist_json["client_ip_port"], + dp_index, + extra={"event": "pd.registration_sent"}, + ) # 转发任务 forwarding_tokens_task = asyncio.create_task( @@ -127,6 +142,7 @@ async def _pd_handle_task(manager: HttpServerManager, pd_master_obj: PD_Master_O obj = pickle.loads(recv_bytes) if obj[0] == ObjType.REQ: prompt, sampling_params, multimodal_params = obj[1] + log_metadata = obj[2] if len(obj) > 2 else {} if dp_index is not None: if sampling_params.suggested_dp_index != -1: logger.warning( @@ -148,6 +164,7 @@ async def _pd_handle_task(manager: HttpServerManager, pd_master_obj: PD_Master_O forwarding_queue=forwarding_queue, pd_upload_websocket=websocket, pd_event=pd_event, + http_request_id=log_metadata.get("http_request_id"), ) ) generation_tasks[group_req_id] = generation_task @@ -189,9 +206,8 @@ async def delayed_abort_task(group_req_id, retry_count): logger.warning(f"pd_handle_task {pd_master_obj} cancelled") return - except Exception as e: - logger.error("connetion to pd_master has error") - logger.exception(str(e)) + except Exception: + log_repeated_exception(logger, "pd.connection_failed", master=pd_master_obj.host_ip_port) finally: child_tasks = [task for task in (forwarding_tokens_task, heartbeat_task) if task is not None] child_tasks.extend(generation_tasks.values()) @@ -247,7 +263,9 @@ async def _pd_process_generate( forwarding_queue: AsyncQueue, pd_upload_websocket: ClientConnection, pd_event: asyncio.Event, + http_request_id: Optional[str] = None, ): + context_token = set_http_request_id(http_request_id or "-") try: async for sub_req_id, request_output, metadata, finish_status in manager.generate( prompt=prompt, @@ -282,6 +300,8 @@ async def _pd_process_generate( ) except Exception: logger.exception(f"report pd node generate error failed, group_request_id: {group_request_id}") + finally: + reset_http_request_id(context_token) # 转发token的task @@ -305,7 +325,6 @@ async def _send_heartbeat_to_pd_master(websocket: ClientConnection): # 获取节点负载信息 def _get_load_info(dp_index: Optional[int] = None) -> dict: - from lightllm.server.api_http import g_objs assert g_objs.shared_token_load is not None, "shared_token_load is not initialized" diff --git a/lightllm/server/httpserver_for_pd_master/manager.py b/lightllm/server/httpserver_for_pd_master/manager.py index 55c9c9f0eb..cdfc790a54 100644 --- a/lightllm/server/httpserver_for_pd_master/manager.py +++ b/lightllm/server/httpserver_for_pd_master/manager.py @@ -18,7 +18,13 @@ from ..tokenizer import get_tokenizer from ..req_id_generator import ReqIDGenerator, convert_sub_id_to_group_id from fastapi import Request -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import ( + request_finish_fields, + init_logger, + log_request_summary, + get_http_request_id, + log_debug_event, +) from lightllm.server.metrics.manager import MetricClient from lightllm.utils.statics_utils import MovingAverage from lightllm.server.httpserver.manager import AsyncQueue @@ -272,22 +278,24 @@ async def _generate_one( except ServerBusyError: # 关闭节点自限流时，不启用与该策略配套的 busy 重试，直接透传异常。 if not self.enable_pd_node_self_request_limit: + log_request_summary(origin_request_id, status="busy", timing_scope="end_to_end") raise # 已向客户端输出 token 后不能从头生成，否则会产生重复内容。 elapsed_seconds = time.monotonic() - retry_start_time if has_yielded_result or elapsed_seconds >= self.pd_node_busy_retry_timeout_seconds: + log_request_summary(origin_request_id, status="busy", timing_scope="end_to_end") raise # 发起下一次尝试前检查客户端连接，避免为已断开的请求继续占用 P/D 节点资源。 if await request.is_disconnected(): disconnect_reason = "_generate_one busy retry check network disconnected" - logger.warning(f"group_request_id: {origin_request_id} {disconnect_reason}") + logger.debug(f"group_request_id: {origin_request_id} {disconnect_reason}") raise ClientDisconnected( group_request_id=origin_request_id, reason=disconnect_reason, ) - logger.warning( + logger.debug( f"group_request_id: {origin_request_id} PD node is busy, retrying with another node; " f"elapsed: {elapsed_seconds:.3f}s, retry timeout: " f"{self.pd_node_busy_retry_timeout_seconds}s" @@ -356,7 +364,13 @@ async def _generate_one_attempt( sampling_params = SamplingParams.from_buffer_copy(origin_sampling_params) block_group_request_id = self.id_gen.generate_id() sampling_params.group_request_id = block_group_request_id - logger.debug(f"pd log gen sub req id {block_group_request_id} for main req id {origin_request_id}") + log_debug_event( + logger, + "request.segment_started", + request_id=block_group_request_id, + parent_request_id=origin_request_id, + segment_index=segment_index, + ) sampling_params.max_new_tokens = remaining_max_new_tokens # high_priority_request 只影响 HTTP 资源申请和 Router 等待队列：首段仅在 # 输入达到长度门槛、预计 cache 命中率高于 0.8 且命中记录仍有效时提升； @@ -457,10 +471,15 @@ async def _generate_one_attempt( break except (ClientDisconnected, BaseException) as e: - logger.error(f"has exception {str(e)}") - - if isinstance(e, ClientDisconnected): - logger.warning(f"group_request_id: {origin_request_id} {e.reason}") + cancelled = isinstance(e, (ClientDisconnected, asyncio.CancelledError, GeneratorExit)) + # Busy attempts can be retried by the outer generator; they are decisions, not final failures. + if not isinstance(e, ServerBusyError): + log_request_summary( + origin_request_id, + status="cancelled" if cancelled else "timeout" if isinstance(e, asyncio.TimeoutError) else "error", + timing_scope="end_to_end", + error_type=type(e).__name__, + ) try: await self.abort(block_group_request_id, p_node=p_node, d_node=d_node) @@ -522,7 +541,7 @@ async def _wait_for_event_or_disconnect( async def raise_if_disconnected() -> None: if await request.is_disconnected(): - logger.warning(f"group_request_id: {group_request_id} {disconnect_reason}") + logger.debug(f"group_request_id: {group_request_id} {disconnect_reason}") raise ClientDisconnected( group_request_id=group_request_id, reason=disconnect_reason, @@ -541,14 +560,8 @@ async def raise_if_disconnected() -> None: continue async def _log_req_header(self, request: Request, group_request_id: int): - x_request_id = request.headers.get("X-Request-Id", "") - x_session_id = request.headers.get("X-Session-Id", "") - format_in_time = datetime.datetime.fromtimestamp(time.time()).strftime("%Y-%m-%d %H:%M:%S") - logger.info( - f"received req X-Request-Id:{x_request_id} " - f"X-Session-Id:{x_session_id} start_time:{format_in_time} " - f"lightllm_req_id:{group_request_id} " - ) + x_request_id = request.headers.get("X-Request-Id") or get_http_request_id() + log_debug_event(logger, "request.received", request_id=group_request_id, http_request_id=x_request_id) return async def fetch_pd_stream( @@ -569,9 +582,12 @@ async def fetch_pd_stream( up_status_event = req_status.up_status_event prefill_prompt_ids_event = req_status.prefill_prompt_ids_event + log_metadata = {"http_request_id": request.headers.get("X-Request-Id") or get_http_request_id()} old_max_new_tokens = sampling_params.max_new_tokens sampling_params.max_new_tokens = 1 - await p_node.websocket.send_bytes(pickle.dumps((ObjType.REQ, (prompt, sampling_params, multimodal_params)))) + await p_node.websocket.send_bytes( + pickle.dumps((ObjType.REQ, (prompt, sampling_params, multimodal_params), log_metadata)) + ) try: await self._wait_for_event_or_disconnect( @@ -591,7 +607,7 @@ async def fetch_pd_stream( sampling_params.max_new_tokens = old_max_new_tokens await d_node.websocket.send_bytes( - pickle.dumps((ObjType.REQ, (prompt_ids, sampling_params, MultimodalParams()))) + pickle.dumps((ObjType.REQ, (prompt_ids, sampling_params, MultimodalParams()), log_metadata)) ) try: @@ -707,6 +723,7 @@ async def _wait_to_token_package( group_request_id = sampling_params.group_request_id unfinished_count = sampling_params.best_of is_first_token = True + finish_reasons = set() sub_req_id_to_mtp_accepted_token_num: Dict[int, int] = {} sub_req_id_to_mtp_verify_step_num: Dict[int, int] = {} @@ -730,12 +747,24 @@ async def _wait_to_token_package( self.first_time_costs.add(first_token_cost_ms) self.latest_success_infer_time = time.time() + if finish_status.is_finished(): + reason = ( + "pd_capacity" + if finish_status.is_finished_pd_decode_capacity() + else finish_status.get_finish_reason() + ) + finish_reasons.add(reason) yield sub_req_id, out_str, metadata, finish_status if finish_status.is_finished(): unfinished_count -= 1 if unfinished_count == 0: break + except BaseException as error: + if not isinstance(error, GeneratorExit) or not finish_reasons: + interrupted = isinstance(error, (ClientDisconnected, asyncio.CancelledError, GeneratorExit)) + finish_reasons.add("abort" if interrupted else "error") + raise finally: # 分段可能因 stop、容量切换或取消而提前关闭，已有输出也需要记录总结。 if out_token_counter: @@ -743,24 +772,26 @@ async def _wait_to_token_package( mean_per_token_cost_time_ms = (total_cost_time_ms - first_token_cost_ms) / out_token_counter self.per_token_costs.add(mean_per_token_cost_time_ms) headers = request.headers if request is not None else {} - x_request_id = headers.get("X-Request-Id", "") - x_session_id = headers.get("X-Session-Id", "") + x_request_id = headers.get("X-Request-Id") or get_http_request_id() prompt_cache_ratio = prompt_cache_len / max(prompt_tokens, 1) mtp_total_verify_steps = sum(sub_req_id_to_mtp_verify_step_num.values()) if mtp_total_verify_steps <= 0: mtp_total_verify_steps = out_token_counter - sum(sub_req_id_to_mtp_accepted_token_num.values()) mtp_avg_token_per_step = out_token_counter / max(mtp_total_verify_steps, 1) - format_start_time = datetime.datetime.fromtimestamp(start_time).strftime("%Y-%m-%d %H:%M:%S") - logger.info( - f"X-Request-Id:{x_request_id} " - f"X-Session-Id:{x_session_id} start_time:{format_start_time} " - f"lightllm_req_id:{group_request_id} first_token_cost:{first_token_cost_ms}ms " - f"total_cost_time:{total_cost_time_ms}ms,out_token_counter:{out_token_counter} " - f"mean_per_token_cost_time: {mean_per_token_cost_time_ms}ms " - f"prompt_token_num:{prompt_tokens} " - f"prompt_cache_len:{prompt_cache_len} " - f"prompt_cache_ratio:{prompt_cache_ratio} " - f"mtp_avg_token_per_step:{mtp_avg_token_per_step} " + terminal = request_finish_fields(finish_reasons) if finish_reasons else {"status": "cancelled"} + log_request_summary( + group_request_id, + **terminal, + x_request_id=x_request_id, + prompt_tokens=prompt_tokens, + output_tokens=out_token_counter, + timing_scope="end_to_end", + ttft_ms=round(first_token_cost_ms, 3), + duration_ms=round(total_cost_time_ms, 3), + cache_tokens=prompt_cache_len, + cache_ratio=prompt_cache_ratio, + mtp_enabled=bool(self.args.mtp_step), + mtp_avg_token_per_step=mtp_avg_token_per_step, ) self.metric_client.histogram_observe("lightllm_request_inference_duration", total_cost_time_ms / 1000.0) self.metric_client.histogram_observe( @@ -780,7 +811,7 @@ async def stop_request(self, request_id: int): async def abort( self, group_request_id, p_node: Optional[PD_Client_Obj] = None, d_node: Optional[PD_Client_Obj] = None ): - logger.warning(f"aborted group_request_id {group_request_id}") + logger.debug(f"aborted group_request_id {group_request_id}") try: req_status = self.req_id_to_out_inf[group_request_id] @@ -980,12 +1011,6 @@ def is_pd_nodes_ready(self): is_ready = ( prefill_node_count == expected_prefill_node_count and decode_node_count == expected_decode_node_count ) - if not is_ready: - logger.warning( - f"PD nodes are not ready: current_prefill={prefill_node_count}, " - f"expected_prefill={expected_prefill_node_count}, current_decode={decode_node_count}, " - f"expected_decode={expected_decode_node_count}" - ) return is_ready except ValueError: logger.warning( diff --git a/lightllm/server/httpserver_for_pd_master/pd_selector/cache_aware.py b/lightllm/server/httpserver_for_pd_master/pd_selector/cache_aware.py index df4b434b0a..81dd93af8a 100644 --- a/lightllm/server/httpserver_for_pd_master/pd_selector/cache_aware.py +++ b/lightllm/server/httpserver_for_pd_master/pd_selector/cache_aware.py @@ -24,7 +24,7 @@ from lightllm.server.pd_io_struct import PD_Client_Obj from lightllm.utils.envs_utils import get_pd_master_recursion_limit -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import init_logger, log_debug_event from .pd_selector import PDSelectionExtraInfo from .prompt_cache_tree import PromptCacheTree @@ -173,12 +173,7 @@ def _get_cache_worker(self, workers: List[PD_Client_Obj], request_text: str) -> result = self.prompt_cache_tree.prefix_match(request_text) match_rate = 0.0 if result.input_char_count == 0 else result.matched_char_count / result.input_char_count - logger.debug( - f"CacheAwarePolicy: matched_char_count={result.matched_char_count}, " - f"input_char_count={result.input_char_count}, match_rate={match_rate:.4f}, " - f"cache_threshold={self.config.cache_threshold:.4f}, " - f"prefill_node={result.prefill_node}" - ) + # Cache match inputs are summarized in the final selection decision. if match_rate <= self.config.cache_threshold or result.prefill_node is None: return None @@ -199,11 +194,13 @@ def _select_idle_worker(self, workers: List[PD_Client_Obj], request_text: str) - idle_workers, key=lambda worker: (worker.dispatched_prompt_chars, worker.client_ip_port), ) - logger.debug( - f"CacheAwarePolicy: select idle worker, idle_worker_num={len(idle_workers)}, " - f"cache_worker={cache_worker.client_ip_port if cache_worker else None}, " - f"balance_rel_threshold={self.config.balance_rel_threshold:.4f}, " - f"selected_worker={selected_worker.client_ip_port}" + log_debug_event( + logger, + "pd.prefill_selected", + reason="idle_cache" if cache_worker else "idle_load", + candidates=len(idle_workers), + selected=selected_worker.client_ip_port, + load=selected_worker.dispatched_prompt_chars, ) return selected_worker @@ -227,15 +224,14 @@ def _select_worker_by_cache_and_load( else: selected_worker = cache_worker - logger.debug( - f"CacheAwarePolicy: cache_worker={cache_worker.client_ip_port if cache_worker else None}, " - f"cache_worker_load={cache_worker.dispatched_prompt_chars if cache_worker else None}, " - f"cache_projected_load={cache_projected_load}, " - f"least_loaded_worker={least_loaded_worker.client_ip_port}, " - f"least_loaded_worker_load={least_loaded_worker.dispatched_prompt_chars}, " - f"least_projected_load={least_projected_load}, " - f"balance_rel_threshold={self.config.balance_rel_threshold:.4f}, " - f"cache_worker_is_overloaded={cache_worker_is_overloaded}, " - f"selected_worker={selected_worker.client_ip_port}" + log_debug_event( + logger, + "pd.prefill_selected", + reason="cache_overloaded" if cache_worker_is_overloaded else ("cache" if cache_worker else "least_load"), + selected=selected_worker.client_ip_port, + load=selected_worker.dispatched_prompt_chars, + least_load=least_loaded_worker.dispatched_prompt_chars, + projected_load=selected_worker.dispatched_prompt_chars + request_load, + balance_threshold=self.config.balance_rel_threshold, ) return selected_worker diff --git a/lightllm/server/httpserver_for_pd_master/pd_selector/pd_selector.py b/lightllm/server/httpserver_for_pd_master/pd_selector/pd_selector.py index 53b380110b..94076e7dc0 100644 --- a/lightllm/server/httpserver_for_pd_master/pd_selector/pd_selector.py +++ b/lightllm/server/httpserver_for_pd_master/pd_selector/pd_selector.py @@ -5,7 +5,7 @@ from lightllm.server.pd_io_struct import PD_Client_Obj from lightllm.server.core.objs import SamplingParams from lightllm.server.multimodal_params import MultimodalParams -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import init_logger, log_debug_event logger = init_logger(__name__) @@ -108,10 +108,12 @@ def select_p_d_node( # 时才返回命中率，避免负载均衡改派节点后误判为高命中。 selection_extra_info = self.policy.get_estimated_cache_info(p_node, prompt) - logger.debug( - f"LoadBalancedCacheAwareSelector: selected p_node={p_node.client_ip_port}, " - f"d_node={d_node.client_ip_port}, " - f"cache_last_insert_time={selection_extra_info.cache_last_insert_time}" + log_debug_event( + logger, + "pd.pair_selected", + prefill=p_node.client_ip_port, + decode=d_node.client_ip_port, + cache_last_insert_time=selection_extra_info.cache_last_insert_time, ) return p_node, d_node, selection_extra_info diff --git a/lightllm/server/httpserver_for_pd_master/register_loop.py b/lightllm/server/httpserver_for_pd_master/register_loop.py index 1e1e911b9a..c3cd2ec255 100644 --- a/lightllm/server/httpserver_for_pd_master/register_loop.py +++ b/lightllm/server/httpserver_for_pd_master/register_loop.py @@ -21,11 +21,9 @@ async def register_loop(manager: HttpServerManagerForPDMaster): ports = get_shm_port_args() while True: - try: uri = f"ws://{manager.args.config_server_host}:{ports.config_server_port}/pd_master_register" async with websockets.connect(uri, max_queue=(2048 * 1024, 2048 * 1023)) as websocket: - sock = websocket.transport.get_extra_info("socket") sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) @@ -40,8 +38,7 @@ async def register_loop(manager: HttpServerManagerForPDMaster): await websocket.send("heartbeat") await asyncio.sleep(60) - except Exception as e: - logger.error("connetion to config_server has error") - logger.exception(str(e)) + except Exception: + logger.exception("connetion to config_server has error") await asyncio.sleep(10) logger.info("reconnection to config_server") diff --git a/lightllm/server/metrics/manager.py b/lightllm/server/metrics/manager.py index 22f6426a77..77886dd6c3 100644 --- a/lightllm/server/metrics/manager.py +++ b/lightllm/server/metrics/manager.py @@ -68,7 +68,7 @@ def push_metrics(self): try: self.monitor.push_metrices() if time_counter >= 60: - logger.info("push metrices success") + logger.debug("push metrices success") time_counter = 0 except: pass diff --git a/lightllm/server/router/batch.py b/lightllm/server/router/batch.py index e8d1dca23b..8a73b3887b 100644 --- a/lightllm/server/router/batch.py +++ b/lightllm/server/router/batch.py @@ -41,10 +41,12 @@ def get_all_dp_req_num(self) -> List[int]: all_dp_req_num[req.sample_params.suggested_dp_index] += 1 return all_dp_req_num - def filter_out_finished_req(self, shm_req_manager: ShmReqManager, router_statics: RouterStatics): + def filter_out_finished_req(self, shm_req_manager: ShmReqManager, router_statics: RouterStatics, throughput=None): unfinished_req_ids = [] for req in self.reqs: if req.shm_infer_released: + if throughput is not None: + throughput.finish(req) logger.debug(f"router release req id {req.request_id}") if not req.is_aborted: router_statics.update(req.candetoken_out_len) @@ -91,5 +93,8 @@ def __repr__(self): return f"batch_id={self.batch_id}, " f"reqs={self.reqs}, " def simple_log(self): - details_tuples = [(req.request_id, req.index_in_shm_mem) for req in self.reqs] - return f"batch_id={self.batch_id}, time:{time.time()}s (req_id, shm_index)s:{details_tuples}" + sample = [req.request_id for req in self.reqs[:8]] + return ( + f"batch_id={self.batch_id} requests={len(self.reqs)} " + f"input_tokens={self.input_tokens()} request_id_sample={sample}" + ) diff --git a/lightllm/server/router/engine_stats.py b/lightllm/server/router/engine_stats.py new file mode 100644 index 0000000000..6f28486612 --- /dev/null +++ b/lightllm/server/router/engine_stats.py @@ -0,0 +1,70 @@ +"""CPU-only cumulative engine counters and monotonic reporting windows.""" + +import time +import numpy as np + +from lightllm.server.core.objs.shm_array import ShmArray + + +class EngineCounters: + """One writer per local DP row (TP leader); the router is the sole reader. + + Workers attach after the router initializes the array. Never reset a live row: + sampling cumulative totals keeps short requests and overlapping windows exact. + """ + + def __init__(self, dp_size, dp_rank=None, *, owner=False): + self.storage = ShmArray("engine_token_counters", (dp_size, 2), np.int64) + if owner: + self.storage.create_shm() + self.storage.arr.fill(0) + else: + self.storage.link_shm() + self.dp_rank = dp_rank + + def add(self, prefill_tokens=0, output_tokens=0): + row = self.storage.arr[self.dp_rank] + row[0] += prefill_tokens + row[1] += output_tokens + + def totals(self): + return tuple(int(value) for value in self.storage.arr.sum(axis=0)) + + +class EngineThroughput: + """Window rates include running requests; request success is counted at release.""" + + def __init__(self, counters): + self.counters = counters + self.completed = 0 + self.last_prefill, self.last_output = counters.totals() + self.last_completed = 0 + self.last_time = time.monotonic() + + def finish(self, req): + if not req.is_aborted and (req.finish_status.is_stopped() or req.finish_status.is_finished_length()): + self.completed += 1 + + def has_activity(self): + prefill, output = self.counters.totals() + return (prefill, output, self.completed) != (self.last_prefill, self.last_output, self.last_completed) + + def resume(self, since=None): + """Start a new busy window without losing tokens from a short burst.""" + self.last_time = time.monotonic() if since is None else since + + def snapshot(self): + now = time.monotonic() + elapsed = now - self.last_time + prefill, output = self.counters.totals() + counts = (prefill - self.last_prefill, output - self.last_output, self.completed - self.last_completed) + result = dict(zip(("prefill_tokens", "output_tokens", "completed_reqs"), counts)) + result.update( + window_s=round(elapsed, 6), + prefill_tok_s=round(counts[0] / elapsed, 2) if elapsed > 0 else 0.0, + output_tok_s=round(counts[1] / elapsed, 2) if elapsed > 0 else 0.0, + completed_req_s=round(counts[2] / elapsed, 2) if elapsed > 0 else 0.0, + ) + self.last_prefill, self.last_output, self.last_completed = prefill, output, self.completed + self.last_time = now + return result diff --git a/lightllm/server/router/manager.py b/lightllm/server/router/manager.py index 189d3b6b78..8caf426de9 100644 --- a/lightllm/server/router/manager.py +++ b/lightllm/server/router/manager.py @@ -1,4 +1,5 @@ import time +import logging import uvloop import asyncio import pickle @@ -23,7 +24,7 @@ from .dynamic_prompt.radix_cache import RadixCacheReadOnlyClient from lightllm.server.multi_level_kv_cache.cpu_cache_client import CpuKvCacheClient from lightllm.server.core.objs.shm_objs_io_buffer import ShmObjsIOBuffer -from lightllm.utils.log_utils import init_logger, log_time_ready +from lightllm.utils.log_utils import init_logger, log_time_ready, PeriodicStatsLogger, log_debug_event from lightllm.utils.profiler import ProfilerCmd from lightllm.server.router.token_load import TokenLoad from lightllm.server.metrics.manager import MetricClient @@ -35,6 +36,7 @@ from lightllm.utils.shm_port_args import get_shm_port_args from lightllm.server.router.dynamic_prompt.shared_arr import SharedInt from .stats import RouterStatics +from .engine_stats import EngineCounters, EngineThroughput from .profiler_service import RouterProfilerCmdQueue, start_router_profiler_server from .rl_rpyc import RouterRlOpHelper, start_router_rl_rpyc_server from .multinode_tp_helper import RouterMultiNodeTpHelper @@ -66,6 +68,9 @@ def __init__(self, args: StartArgs): # 存储在共享内存中的真实token容量数据 self.shm_max_total_token_num = SharedInt("shm_max_total_token_num") self.shm_req_manager = ShmReqManager() + self.service_ready = SharedInt("service_ready") + self.service_ready.set_value(0) + self.engine_counters = EngineCounters(self.dp_size_in_node, owner=True) # 用共享内存进行共享，router 模块读取进行精确的调度估计 self.read_only_statics_mem_manager = ReadOnlyStaticsMemoryManager() # 初始化 radix_cache_client 用于读取 prompt cache 的管理信息 @@ -192,7 +197,17 @@ async def wait_to_model_ready(self): self.args.max_total_token_num = self.max_total_token_num self.shm_max_total_token_num.set_value(self.max_total_token_num) - logger.info(f"set shm_max_total_token_num value to {self.shm_max_total_token_num.get_value()}") + logger.info( + "event=kv_cache.ready capacity_tokens_per_dp=%s local_dp_count=%s", + self.max_total_token_num, + self.dp_size_in_node, + extra={ + "event": "kv_cache.ready", + "capacity_tokens_per_dp": self.max_total_token_num, + "local_dp_count": self.dp_size_in_node, + }, + ) + logger.debug(f"set shm_max_total_token_num value to {self.shm_max_total_token_num.get_value()}") if not self.args.disable_dynamic_prompt_cache: self.radix_cache_client = RadixCacheReadOnlyClient( @@ -227,28 +242,70 @@ async def loop_for_fwd( self, ): counter_count = 0 + self.engine_throughput = EngineThroughput(self.engine_counters) + stats_logger = PeriodicStatsLogger(init_logger("lightllm.stats")) + last_poll = time.monotonic() + resumed = False + seen_ready = False while True: await self._step() counter_count += 1 + # One node-local snapshot, never one copy per TP worker. Multi-node TP + # slave routers mirror requests and must not duplicate the master summary. + running = len(self.running_batch.reqs) if self.running_batch is not None else 0 + waiting = self.req_queue.get_wait_req_num() + now = time.monotonic() + active = bool(running or waiting) or self.engine_throughput.has_activity() + if self.service_ready.get_value() and not seen_ready: + self.engine_throughput.resume(last_poll) + stats_logger.last_time = last_poll + seen_ready = True + if self.service_ready.get_value() and stats_logger.active is False and active and not resumed: + self.engine_throughput.resume(last_poll) + stats_logger.last_time = last_poll + resumed = True + last_poll = now + if ( + (self.dp_size > 1 or self.node_rank == 0) + and self.service_ready.get_value() + and stats_logger.should_log(active) + ): + resumed = False + stats_logger.log( + scope="node", + **self.engine_throughput.snapshot(), + running=running, + waiting=waiting, + paused=self._get_paused_req_num(), + kv_active_pct_by_local_dp=[ + round(float(100 * self.get_used_tokens(i) / max(self.max_total_token_num, 1)), 1) + for i in range(self.dp_size_in_node) + ], + kv_cached_pct_by_local_dp=[ + round( + float( + 100 + * self.radix_cache_client.get_unrefed_tokens_num(i) + / max(self.max_total_token_num, 1) + ), + 1, + ) + if self.radix_cache_client is not None + else 0.0 + for i in range(self.dp_size_in_node) + ], + kv_capacity_tokens_per_dp=self.max_total_token_num, + ) if self.running_batch is not None: if counter_count % 100 == 0: - for dp_index in range(self.dp_size_in_node): - token_ratio1 = self.get_used_tokens(dp_index) / self.max_total_token_num - token_ratio2 = ( - self.max_total_token_num - - self.read_only_statics_mem_manager.get_unrefed_token_num(dp_index) - ) / self.max_total_token_num - d_i = dp_index - estimated_peak_token_count = self.shared_token_load.get_estimated_peak_token_count(d_i) - paused_req_num = self._get_paused_req_num_in_dp_index(dp_index=d_i) + if log_time_ready("router_debug_snapshot", 10) and logger.isEnabledFor(logging.DEBUG): logger.debug( - f"dp_i {d_i} current batch size: {len(self.running_batch.reqs)} \n" - f"dp_i {d_i} paused req num: {paused_req_num} \n" - f"dp_i {d_i} estimated_peak_token_count: {estimated_peak_token_count} \n" - f"dp_i {d_i} token used ratio: {token_ratio1} not contain prompt cache tree unrefed token\n" - f"dp_i {d_i} token used ratio: {token_ratio2} contain prompt cache tree unrefed token" + "event=router.policy running=%s waiting=%s paused=%s estimated_output_tokens=%s", + running, + waiting, + self._get_paused_req_num(), + self.router_statics.ema_req_out_len, ) - logger.debug(self.router_statics.log_str()) self.metric_client.gauge_set("lightllm_batch_pause_size", self._get_paused_req_num()) # pd decode mode need to update token_load more frequently self.req_queue.update_token_load(self.running_batch, force_update=self.is_pd_decode_mode) @@ -270,11 +327,6 @@ async def loop_for_fwd( self.metric_client.gauge_set("lightllm_batch_pause_size", 0.0) self.metric_client.gauge_set("lightllm_queue_size", 0.0) self.metric_client.gauge_set("lightllm_batch_current_max_tokens", 0.0) - # 60s print once - if log_time_ready("token_load_info", 60): - for dp_i in range(self.dp_size_in_node): - estimated_peak_token_count = self.shared_token_load.get_estimated_peak_token_count(dp_i) - logger.debug(f"dp_i {dp_i} estimated_peak_token_count: {estimated_peak_token_count} \n") await asyncio.sleep(self._get_schedule_time_interval()) @@ -316,7 +368,15 @@ async def _add_batch(self, batch: Batch): await asyncio.sleep(0.001) self.shm_reqs_io_buffer.write_obj(reqs) self.shm_reqs_io_buffer.set_ready() - logger.debug(f"Prefill Batch: {batch.simple_log()} \n") + if logger.isEnabledFor(logging.DEBUG): + self._batch_log_seq = getattr(self, "_batch_log_seq", 0) + 1 + log_debug_event( + logger, + "batch.submitted", + batch_id=self._batch_log_seq, + request_count=len(batch.reqs), + request_id_sample=[r.request_id for r in batch.reqs[:8]], + ) return async def _write_profiler_cmds(self): @@ -347,7 +407,9 @@ def _add_new_batch_to_running_batch(self, new_batch: Batch): def _filter_reqs_from_running_batch(self): if self.running_batch is not None: - self.running_batch.filter_out_finished_req(self.shm_req_manager, self.router_statics) + self.running_batch.filter_out_finished_req( + self.shm_req_manager, self.router_statics, throughput=getattr(self, "engine_throughput", None) + ) if self.running_batch.is_clear(): self.running_batch = None return @@ -418,9 +480,6 @@ def _generate_new_batch(self): Batch.merge_two_batch(self.running_batch, self.schedule_new_batch) ) - if new_batch is not None and len(new_batch.reqs) > 0: - logger.debug(f"generate new batch, {new_batch.simple_log()}") - self.schedule_new_batch = Batch.merge_two_batch(self.schedule_new_batch, new_batch) return diff --git a/lightllm/server/router/model_infer/infer_batch.py b/lightllm/server/router/model_infer/infer_batch.py index c89642cd85..0fe225d631 100644 --- a/lightllm/server/router/model_infer/infer_batch.py +++ b/lightllm/server/router/model_infer/infer_batch.py @@ -16,7 +16,7 @@ HybridAttPagedRadixCache, HybridAttPagedTreeNode, ) -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import init_logger, log_debug_event from lightllm.server.req_id_generator import convert_sub_id_to_group_id from lightllm.server.multimodal_params import MultimodalParams from lightllm.utils.custom_kernel_utis import custom_cat @@ -313,7 +313,7 @@ def _save_promptcache_kvbuffer(self): https://arxiv.org/abs/2403.01241 """ prompt_cache_token_id = list(self.radix_cache.root_node.children.values())[0].token_id_key - print(f"prompt_cache_token_id : {prompt_cache_token_id}") + log_debug_event(logger, "cache.prompt_loaded", tokens=len(prompt_cache_token_id), leader_only=True) index = range(len(prompt_cache_token_id)) prompt_cache_kv_buffer = self.radix_cache.mem_manager.get_index_kv_buffer(index) torch.save(prompt_cache_kv_buffer, f"prompt_cache_rank_{dist.get_rank()}.pt") @@ -353,12 +353,13 @@ def _filter(self, finished_request_ids: List[int], modify_shm_finish_state: bool self.infer_req_ids = [_id for _id in self.infer_req_ids if _id not in finished_req_ids_set] if self.radix_cache is not None and len(self.infer_req_ids) == 0: - logger.debug( - f"free a batch state:\n" - f"radix refed token num {self.radix_cache.get_refed_tokens_num()}\n" - f"radix hold token num {self.radix_cache.get_tree_total_tokens_num()}\n" - f"mem manager can alloc token num {self.req_manager.mem_manager.allocator.can_use_mem_size}\n" - f"mem manager total size {self.req_manager.mem_manager.allocator.size}\n" + log_debug_event( + logger, + "cache.idle", + referenced_tokens=self.radix_cache.get_refed_tokens_num(), + cached_tokens=self.radix_cache.get_tree_total_tokens_num(), + free_tokens=self.req_manager.mem_manager.allocator.can_use_mem_size, + leader_only=True, ) return @@ -371,7 +372,6 @@ def filter_reqs(self, finished_reqs: List["InferReq"]): @torch.no_grad() def pause_reqs(self, pause_reqs: List["InferReq"], is_master_in_dp: bool): if pause_reqs: - free_token_index = [] for req in pause_reqs: if self.args.diverse_mode: @@ -383,7 +383,7 @@ def pause_reqs(self, pause_reqs: List["InferReq"], is_master_in_dp: bool): req.paused = True if is_master_in_dp: req.shm_req.is_paused = True - logger.debug(f"infer paused req id {req.req_id}") + log_debug_event(logger, "request.paused", request_id=req.req_id, leader_only=True) if len(free_token_index) != 0: free_token_index = custom_cat(free_token_index) @@ -411,7 +411,7 @@ def recover_paused_reqs(self, paused_reqs: List["InferReq"], is_master_in_dp: bo req.paused = False if is_master_in_dp: req.shm_req.is_paused = False - logger.debug(f"infer recover paused req id {req.req_id}") + log_debug_event(logger, "request.resumed", request_id=req.req_id, leader_only=True) can_alloc_token_num -= alloc_token_num return @@ -981,9 +981,10 @@ class InferReqUpdatePack: 类绑定相关处理参数，实现解耦和延迟处理。 """ - def __init__(self, req_obj: InferReq, output_len: int): + def __init__(self, req_obj: InferReq, output_len: int, prefill_tokens: int = 0): self.req_obj = req_obj self.output_len = output_len + self.prefill_tokens = prefill_tokens def handle( self, diff --git a/lightllm/server/router/model_infer/mode_backend/base_backend.py b/lightllm/server/router/model_infer/mode_backend/base_backend.py index 6c621b6adb..1b675aaf3c 100644 --- a/lightllm/server/router/model_infer/mode_backend/base_backend.py +++ b/lightllm/server/router/model_infer/mode_backend/base_backend.py @@ -1,3 +1,4 @@ +import logging import os import numpy as np @@ -8,7 +9,8 @@ from typing import List, Tuple, Callable, Optional, Union from transformers.configuration_utils import PretrainedConfig from lightllm.utils.infer_utils import set_random_seed -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import init_logger, log_debug_event, DebugStatsLogger +from lightllm.server.router.engine_stats import EngineCounters from lightllm.models import get_draft_model_class, get_model from lightllm.server.router.model_infer.infer_batch import InferReq, InferReqUpdatePack from lightllm.server.router.token_load import TokenLoad @@ -114,6 +116,9 @@ def init_model(self, kvargs): init_distributed_env(kvargs) self.init_rank_infos() + self.engine_counters = ( + EngineCounters(self.dp_size_in_node, self.dp_rank_in_node) if self.is_master_in_dp else None + ) group_size = ( 2 if (self.args.enable_decode_microbatch_overlap or self.args.enable_prefill_microbatch_overlap) else 1 ) @@ -181,7 +186,7 @@ def init_model(self, kvargs): assert self.use_dynamic_prompt_cache self.preload_prompt_cache_kv_buffer(model_cfg) - self.logger.info(f"loaded model class {self.model.__class__}") + self.logger.info("event=model.initialized model_class=%s", self.model.__class__.__name__) cache_placement_controller = create_cache_placement_controller( args=self.args, @@ -222,7 +227,7 @@ def init_model(self, kvargs): # 如果存在需要跨进程使用mem manger的特性，则将mem manager写入到 shm中，方便 # 读取 self.model.mem_manager.write_to_shm(req_manager=self.model.req_manager) - dist.barrier(group=self.node_nccl_group) + dist.barrier(group=self.node_nccl_group, device_ids=[torch.cuda.current_device()]) # 同一 DP 组内只需主 rank 初始化真实的 capture buffer 并执行后续相关操作； # 非主 rank 不需要分配 buffer，避免重复占用内存。 @@ -611,9 +616,10 @@ def _timer_merge_radix_tree(self): ): start = time.time() self.radix_cache.merge_unreferenced_nodes() - self.logger.info( - f"radix tree merge_unreferenced_nodes cost time {time.time() - start} s in rank {self.global_rank}" - ) + if self.rank_in_dp == 0 and self.logger.isEnabledFor(logging.DEBUG): + if not hasattr(self, "_debug_compactions"): + self._debug_compactions = DebugStatsLogger(self.logger, "cache.compactions") + self._debug_compactions.add(duration_ms_total=round((time.time() - start) * 1000, 3)) return # 一些可以复用的通用功能函数 @@ -722,7 +728,6 @@ def _get_classed_reqs( can_alloc_token_num = g_infer_context.get_can_alloc_token_num() for req_obj in ready_reqs: - if req_obj.filter_mark: finished_reqs.append(req_obj) continue @@ -768,7 +773,7 @@ def _get_classed_reqs( else: # 非 overlap 模式没有在途的异步计算，可以在本轮直接清理。 finished_reqs.append(req_obj) - self.logger.info( + self.logger.warning( f"force early finish for PD decode req_id={req_obj.req_id} " f"because token capacity is insufficient" ) @@ -855,6 +860,9 @@ def _pre_post_handle(self, run_reqs: List[InferReq], is_chuncked_mode: bool) -> new_kv_len = req_obj.get_chuncked_input_token_len() else: new_kv_len = req_obj.get_cur_total_len() + prefill_tokens = max( + 0, min(new_kv_len, req_obj.shm_req.input_len) - min(req_obj.cur_kv_len, req_obj.shm_req.input_len) + ) req_obj.cur_kv_len = new_kv_len if is_master_in_dp: req_obj.shm_req.shm_cur_kv_len = req_obj.cur_kv_len @@ -862,13 +870,13 @@ def _pre_post_handle(self, run_reqs: List[InferReq], is_chuncked_mode: bool) -> # 对于没有到达需要输出 token 阶段的请求，直接略过, 说明还 # 处于chuncked prefill kv 填充的阶段。 if req_obj.cur_kv_len < req_obj.get_cur_total_len(): - pack = InferReqUpdatePack(req_obj=req_obj, output_len=0) + pack = InferReqUpdatePack(req_obj=req_obj, output_len=0, prefill_tokens=prefill_tokens) update_func_objs.append(pack) continue # 将生成的下一个token的信息写入到管理对象中。 req_obj.cur_output_len += 1 - pack = InferReqUpdatePack(req_obj=req_obj, output_len=req_obj.cur_output_len) + pack = InferReqUpdatePack(req_obj=req_obj, output_len=req_obj.cur_output_len, prefill_tokens=prefill_tokens) update_func_objs.append(pack) return update_func_objs @@ -891,11 +899,14 @@ def _post_handle( next_token_logprobs = next_token_logprobs.tolist() next_token_ranks = next_token_ranks.tolist() + counters = getattr(self, "engine_counters", None) + prefill_tokens = output_tokens = 0 for req_obj, next_token_id, next_token_logprob, next_token_rank, pack in zip( run_reqs, next_token_ids, next_token_logprobs, next_token_ranks, run_reqs_update_packs ): req_obj: InferReq = req_obj pack: InferReqUpdatePack = pack + old_output = req_obj.shm_req.shm_cur_output_len if counters is not None else 0 pack.handle( next_token_id=next_token_id, next_token_logprob=next_token_logprob, @@ -905,7 +916,12 @@ def _post_handle( extra_post_req_handle_func=extra_post_req_handle_func, pd_prefill_chunked_handle_func=pd_prefill_chunked_handle_func, ) + if counters is not None: + prefill_tokens += pack.prefill_tokens + output_tokens += max(0, req_obj.shm_req.shm_cur_output_len - old_output) + if counters is not None: + counters.add(prefill_tokens, output_tokens) g_infer_context.req_manager.req_sampling_params_manager.update_reqs_token_counter( req_objs=run_reqs, next_token_ids=next_token_ids ) @@ -1040,6 +1056,9 @@ def init_rank_infos(self): self.dp_world_size = get_dp_world_size() self.global_rank = get_global_rank() self.global_world_size = get_global_world_size() + from lightllm.utils.log_utils import set_log_context + + set_log_context(tp_rank=self.rank_in_dp, dp_rank=self.global_dp_rank, global_rank=self.global_rank) self.dp_size = get_dp_size() if self.nnodes > 1 and self.dp_size == 1: diff --git a/lightllm/server/router/model_infer/mode_backend/chunked_prefill/impl_for_reward_model.py b/lightllm/server/router/model_infer/mode_backend/chunked_prefill/impl_for_reward_model.py index dfb1020820..0da5ee1381 100644 --- a/lightllm/server/router/model_infer/mode_backend/chunked_prefill/impl_for_reward_model.py +++ b/lightllm/server/router/model_infer/mode_backend/chunked_prefill/impl_for_reward_model.py @@ -14,7 +14,6 @@ def __init__(self) -> None: return def reward_prefill(self, event_pack: OverlapEventPack, prefill_reqs: List[InferReq]): - assert self.disable_chunked_prefill is True model_input, run_reqs = prepare_prefill_inputs(prefill_reqs, is_chuncked_mode=not self.disable_chunked_prefill) @@ -28,6 +27,7 @@ def reward_prefill(self, event_pack: OverlapEventPack, prefill_reqs: List[InferR for req_obj, score in zip(run_reqs, scores): # prefill and decode is same req_obj: InferReq = req_obj + prefill_tokens = max(0, req_obj.shm_req.input_len - req_obj.cur_kv_len) req_obj.cur_kv_len = req_obj.get_cur_total_len() req_obj.cur_output_len += 1 @@ -35,6 +35,7 @@ def reward_prefill(self, event_pack: OverlapEventPack, prefill_reqs: List[InferR req_obj.update_finish_status(self.eos_id, output_len=req_obj.cur_output_len) if self.is_master_in_dp: + self.engine_counters.add(prefill_tokens, 1) # 写入 reward_score req_obj.shm_req.reward_score = score diff --git a/lightllm/server/router/model_infer/mode_backend/diverse_backend/impl.py b/lightllm/server/router/model_infer/mode_backend/diverse_backend/impl.py index 46586959f1..c14958446f 100644 --- a/lightllm/server/router/model_infer/mode_backend/diverse_backend/impl.py +++ b/lightllm/server/router/model_infer/mode_backend/diverse_backend/impl.py @@ -40,7 +40,6 @@ def beam_prefill(self, event_pack: OverlapEventPack, prefill_reqs: List[InferReq ) with torch.cuda.stream(g_infer_context.get_overlap_stream()): - model_output = self.model.forward(model_input) logits = model_output.logits @@ -139,6 +138,9 @@ def _diverse_pre_post_handle(self, run_reqs: List[InferReq], is_chuncked_mode: b new_kv_len = req_obj.get_chuncked_input_token_len() else: new_kv_len = req_obj.get_cur_total_len() + prefill_tokens = max( + 0, min(new_kv_len, req_obj.shm_req.input_len) - min(req_obj.cur_kv_len, req_obj.shm_req.input_len) + ) req_obj.cur_kv_len = new_kv_len if is_master_in_dp: req_obj.shm_req.shm_cur_kv_len = req_obj.cur_kv_len @@ -146,7 +148,7 @@ def _diverse_pre_post_handle(self, run_reqs: List[InferReq], is_chuncked_mode: b # 对于没有到达需要输出 token 阶段的请求，直接略过, 说明还 # 处于chuncked prefill kv 填充的阶段。 if req_obj.cur_kv_len < req_obj.get_cur_total_len(): - pack = InferReqUpdatePack(req_obj=req_obj, output_len=0) + pack = InferReqUpdatePack(req_obj=req_obj, output_len=0, prefill_tokens=prefill_tokens) update_func_objs.append(pack) pre_master_req_pack = pack # TODO 如果 diverse mode 需要支持 pd 分离，则应该每个分块prefill后都进行相关的复制， @@ -155,7 +157,9 @@ def _diverse_pre_post_handle(self, run_reqs: List[InferReq], is_chuncked_mode: b # 将生成的下一个token的信息写入到管理对象中。 req_obj.cur_output_len += 1 - pack = InferReqUpdatePack(req_obj=req_obj, output_len=req_obj.cur_output_len) + pack = InferReqUpdatePack( + req_obj=req_obj, output_len=req_obj.cur_output_len, prefill_tokens=prefill_tokens + ) update_func_objs.append(pack) pre_master_req_pack = pack if req_obj.slave_reqs: diff --git a/lightllm/server/router/model_infer/mode_backend/dp_backend/dp_shared_kv_trans.py b/lightllm/server/router/model_infer/mode_backend/dp_backend/dp_shared_kv_trans.py index 413a86b94e..643314f452 100644 --- a/lightllm/server/router/model_infer/mode_backend/dp_backend/dp_shared_kv_trans.py +++ b/lightllm/server/router/model_infer/mode_backend/dp_backend/dp_shared_kv_trans.py @@ -4,6 +4,7 @@ import dataclasses import torch from typing import List +from lightllm.utils.log_utils import DebugStatsLogger from lightllm.common.kv_cache_mem_manager import MemoryManager from lightllm.utils.envs_utils import get_env_start_args from lightllm.utils.dist_utils import get_dp_rank_in_node @@ -22,6 +23,7 @@ def __init__(self, max_req_num: int, dp_size_in_node: int, backend): from .impl import DPChunkedPrefillBackend self.backend: DPChunkedPrefillBackend = backend + self._debug_transfers = DebugStatsLogger(backend.logger, "kv.shared_transfer") self.max_req_num = max_req_num # 0 代表 kv_len, 1 代表 radix_cache_len @@ -100,6 +102,8 @@ def build_shared_kv_trans_tasks( return trans_tasks def kv_trans(self, trans_tasks: List["TransTask"]): + if self.backend.is_master_in_dp: + self._debug_transfers.flush() # kv 传输 if len(trans_tasks) > 0: max_kv_len_mem_indexes = [] @@ -122,8 +126,8 @@ def kv_trans(self, trans_tasks: List["TransTask"]): dp_size_in_node=self.backend.dp_size_in_node, rank_in_dp=self.backend.rank_in_dp, ) - # This copy runs during request initialization, not once per PD transfer page. - self.backend.logger.debug(f"dp_i {self.dp_rank_in_node} transfer kv tokens num: {len(mem_indexes_tensor)}") + if self.backend.is_master_in_dp: + self._debug_transfers.add(tokens_total=len(mem_indexes_tensor), tasks_total=len(trans_tasks)) for trans_task in trans_tasks: trans_task.req.cur_kv_len += len(trans_task.mem_indexes) diff --git a/lightllm/server/router/model_infer/mode_backend/pd/decode_node_impl/decode_kv_move_manager.py b/lightllm/server/router/model_infer/mode_backend/pd/decode_node_impl/decode_kv_move_manager.py index 263f7c0686..8fca36a793 100644 --- a/lightllm/server/router/model_infer/mode_backend/pd/decode_node_impl/decode_kv_move_manager.py +++ b/lightllm/server/router/model_infer/mode_backend/pd/decode_node_impl/decode_kv_move_manager.py @@ -5,7 +5,7 @@ import torch.multiprocessing as mp import time from typing import List, Dict, Optional, Tuple, Union, Callable -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import init_logger, set_log_context from lightllm.server.pd_io_struct import PDChunckedTransTaskGroup, PDAbortReq from lightllm.server.core.objs import StartArgs from lightllm.utils.graceful_utils import graceful_registry @@ -34,6 +34,7 @@ def _init_env(args, info_queue: mp.Queue, event: mp.Event): # 注册graceful 退出的处理 graceful_registry(inspect.currentframe().f_code.co_name) + set_log_context(component="kv_move") setproctitle.setproctitle(f"lightllm::{get_unique_server_name()}::decode_kv_move_manager") from .up_status import start_up_kv_status_process @@ -91,11 +92,6 @@ def task_dispatcher_loop(self): try: trans_process: KVTransProcess = self.kv_trans_processes[device_id] trans_process.task_in_queue.put(task_group) - # A task group covers the whole request, so this log does not require pd_detail_log. - if isinstance(task_group, PDChunckedTransTaskGroup) and logger.isEnabledFor(logging.DEBUG): - logger.debug( - f"kv move manager dispatch task group {task_group.task_list[0].to_str()} to device {device_id}" - ) except BaseException as e: logger.exception(str(e)) diff --git a/lightllm/server/router/model_infer/mode_backend/pd/decode_node_impl/decode_trans_process.py b/lightllm/server/router/model_infer/mode_backend/pd/decode_node_impl/decode_trans_process.py index d832258f8e..dc87fcfb21 100644 --- a/lightllm/server/router/model_infer/mode_backend/pd/decode_node_impl/decode_trans_process.py +++ b/lightllm/server/router/model_infer/mode_backend/pd/decode_node_impl/decode_trans_process.py @@ -1,4 +1,5 @@ import torch +import logging import time import inspect import threading @@ -7,7 +8,7 @@ import queue import pickle from typing import List, Dict, Union, Optional -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import init_logger, set_log_context, DebugStatsLogger from lightllm.common.kv_cache_mem_manager import MemoryManager from lightllm.server.pd_io_struct import ( PDChunckedTransTask, @@ -36,7 +37,7 @@ def start_decode_trans_process( proc = mp.Process(target=_init_env, args=(args, device_id, task_in_queue, task_out_queue, up_status_in_queue)) proc.start() assert proc.is_alive() - logger.info(f"prefill trans kv process for device: {device_id} started!") + logger.info("Decode KV transfer worker started: device=%s", device_id) return proc @@ -80,6 +81,7 @@ def _init_env( try: torch.cuda.set_device(device_id) graceful_registry(inspect.currentframe().f_code.co_name) + set_log_context(component="kv_transfer", device_id=device_id) task_out_queue.put("proc_start") @@ -104,9 +106,8 @@ def _init_env( while True: time.sleep(100) - except Exception as e: - logger.exception(str(e)) - logger.error(f"Fatal error happened in kv trans process: {e}") + except Exception: + logger.exception("KV transfer worker initialization failed") pass @@ -249,8 +250,7 @@ def accept_peer_task_loop( try: notifies_dict = self.transporter.get_new_notifs() except BaseException as e: - logger.error(f"get new notifies failed: {str(e)}") - logger.exception(str(e)) + logger.exception(f"get new notifies failed: {str(e)}") notifies_dict = {} if notifies_dict: @@ -295,8 +295,6 @@ def accept_peer_task_loop( local_trans_task.prefill_num_pages = remote_trans_task.prefill_num_pages local_trans_task.prefill_page_reg_desc = remote_trans_task.prefill_page_reg_desc self.request_page_task_queue.put(local_trans_task) - if self.args.pd_detail_log: - logger.debug(f"recv WRITE request from prefill: {remote_trans_task.to_str()}") else: # This does not necessarily mean the WRITE protocol state is corrupted. # A common benign case is: decode has already received an abort for this @@ -323,8 +321,6 @@ def accept_peer_task_loop( local_trans_task.first_gen_token_id = remote_trans_task.first_gen_token_id local_trans_task.first_gen_token_logprob = remote_trans_task.first_gen_token_logprob self.ready_page_task_queue.put(local_trans_task) - if self.args.pd_detail_log: - logger.debug(f"recv WRITE done from prefill: {remote_trans_task.to_str()}") else: # Same race as the WRITE request stage: decode may have cleaned the # waiting task because the request was aborted, then a late done notify @@ -372,8 +368,7 @@ def request_page_loop(self): except BaseException as e: with self.waiting_dict_lock: self.waiting_dict.pop(key, None) - logger.error(f"send write ready task to prefill node failed: {trans_task.to_str()}") - logger.exception(str(e)) + logger.exception(f"send write ready task to prefill node failed: {trans_task.to_str()}") self.transporter.remove_remote_agent(peer_name=trans_task.prefill_agent_name) trans_task.error_info = f"send write ready task to prefill node failed: {str(e)}" self.failed_queue.put(trans_task) @@ -406,15 +401,19 @@ def read_page_to_mems_loop(self): @log_exception def success_loop(self): torch.cuda.set_device(self.device_id) + debug_stats = DebugStatsLogger(logger, "kv.transfer.decode") while True: - copy_end_event, copy_start_event, trans_task = self.success_queue.get() + try: + item = self.success_queue.get(timeout=debug_stats.interval) + except queue.Empty: + debug_stats.flush() + continue + copy_end_event, copy_start_event, trans_task = item trans_task: PDChunckedTransTask = trans_task copy_end_event: Optional[torch.cuda.Event] = copy_end_event copy_start_event: Optional[torch.cuda.Event] = copy_start_event - read_page_gpu_time_ms = -1.0 if copy_end_event is not None: copy_end_event.synchronize() - read_page_gpu_time_ms = copy_start_event.elapsed_time(copy_end_event) if trans_task.dst_page_index is not None: self.page_index_queue.put(trans_task.dst_page_index) @@ -425,14 +424,19 @@ def success_loop(self): ret = trans_task.createRetObj() self.task_out_queue.put(ret) - if self.args.pd_detail_log: - if trans_task.start_trans_time is not None: - logger.debug( - f"trans task ret success:{ret} cost time: {trans_task.transfer_time()} s " - f"read_page_gpu_time: {read_page_gpu_time_ms:.3f} ms" - ) - else: - logger.debug(f"trans task ret success:{ret}") + if logger.isEnabledFor(logging.DEBUG): + debug_stats.add( + request_id=trans_task.request_id, + pages=1, + kv_tokens_total=trans_task.transfer_kv_num(), + state_pages=int(trans_task.page_kind != "kv"), + read_page_gpu_ms_total=( + copy_start_event.elapsed_time(copy_end_event) if copy_end_event is not None else 0 + ), + bytes_total=self.transporter.kv_move_buffer[0].numel() + * self.transporter.kv_move_buffer.element_size(), + task_duration_ms_total=round((time.time() - trans_task.create_time) * 1000, 3), + ) @log_exception def fail_loop(self): @@ -449,7 +453,7 @@ def fail_loop(self): ret = trans_task.createRetObj() self.task_out_queue.put(ret) - logger.warning(f"trans task ret fail:{ret}") + logger.error(f"KV transfer failed: {ret}") if trans_task.error_info is not None: # 提前终结所有有问题的属于同一个请求的任务。 diff --git a/lightllm/server/router/model_infer/mode_backend/pd/decode_node_impl/up_status.py b/lightllm/server/router/model_infer/mode_backend/pd/decode_node_impl/up_status.py index 1545970336..6b12cb9f91 100644 --- a/lightllm/server/router/model_infer/mode_backend/pd/decode_node_impl/up_status.py +++ b/lightllm/server/router/model_infer/mode_backend/pd/decode_node_impl/up_status.py @@ -10,7 +10,7 @@ from typing import Dict from dataclasses import asdict from lightllm.server.pd_io_struct import PDUpKVStatus -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import init_logger, set_log_context, log_debug_event, log_repeated_exception from lightllm.utils.graceful_utils import graceful_registry from lightllm.server.pd_io_struct import PD_Master_Obj import torch.multiprocessing as mp @@ -31,7 +31,6 @@ def thread_loop(self): asyncio.run(self.task_loop()) async def task_loop(self): - self.id_to_handle_task: Dict[int, asyncio.Task] = {} self.id_to_handle_queue: Dict[int, asyncio.Queue] = {} @@ -42,7 +41,7 @@ async def task_loop(self): from lightllm.server.httpserver.pd_loop import _get_pd_master_objs id_to_pd_master_obj = await _get_pd_master_objs(self.args) - logger.info(f"get pd_master_objs {id_to_pd_master_obj}") + # Connection changes are already logged by the HTTP registration loop. if id_to_pd_master_obj is not None: for node_id, pd_master_obj in self.id_to_handle_task.items(): @@ -50,7 +49,7 @@ async def task_loop(self): self.id_to_handle_task[node_id].cancel() self.id_to_handle_task.pop(node_id, None) self.id_to_handle_queue.pop(node_id, None) - logger.info(f"up_kv_status_task {pd_master_obj} cancelled") + logger.debug(f"up_kv_status_task {pd_master_obj} cancelled") for node_id, pd_master_obj in id_to_pd_master_obj.items(): if node_id not in self.id_to_handle_task: @@ -92,20 +91,18 @@ async def up_kv_status_task(self, pd_master_obj: PD_Master_Obj): task_queue = self.id_to_handle_queue[pd_master_obj.node_id] upkv_status: PDUpKVStatus = await task_queue.get() await websocket.send(pickle.dumps(upkv_status)) - # KV readiness is reported once per request, not once per transfer page. - logger.debug(f"up kv status: {upkv_status}") + # Successful KV status forwarding is represented by request/transfer summaries. else: await asyncio.sleep(3) except BaseException as e: logger.error(str(e)) raise e except asyncio.CancelledError: - logger.info(f"up_kv_status_task {pd_master_obj} cancelled") + logger.debug(f"up_kv_status_task {pd_master_obj} cancelled") return - except Exception as e: - logger.error(f"connetion to pd_master {pd_master_obj} has error: {str(e)}") - logger.exception(str(e)) + except Exception: + log_repeated_exception(logger, "pd.status_connection_failed", master=pd_master_obj.host_ip_port) await asyncio.sleep(10) logger.info("reconnection to pd_master") @@ -113,9 +110,10 @@ async def up_kv_status_task(self, pd_master_obj: PD_Master_Obj): def _init_env(args, task_in_queue: mp.SimpleQueue): start_parent_check_thread() graceful_registry(inspect.currentframe().f_code.co_name) + set_log_context(component="kv_status") setproctitle.setproctitle(f"lightllm::{get_unique_server_name()}::pd_up_kv_status") up_kv_manager = UpStatusManager(args, task_in_queue) - logger.info(f"up kv manager {str(up_kv_manager)} start ok") + logger.info("event=kv_status.ready manager=%s", type(up_kv_manager).__name__) while True: time.sleep(666) return diff --git a/lightllm/server/router/model_infer/mode_backend/pd/nccl_kv_transporter.py b/lightllm/server/router/model_infer/mode_backend/pd/nccl_kv_transporter.py index 2a0d7f29ea..70f6719866 100644 --- a/lightllm/server/router/model_infer/mode_backend/pd/nccl_kv_transporter.py +++ b/lightllm/server/router/model_infer/mode_backend/pd/nccl_kv_transporter.py @@ -307,11 +307,7 @@ def send_page(self, trans_task: PDChunckedTransTask) -> _NcclXferHandle: event = torch.cuda.Event() event.record(stream) - if self.transporter.args.pd_detail_log: - logger.debug( - f"NCCL send page posted request_id={trans_task.request_id} " - f"src_page={trans_task.src_page_index} dst_agent={self.peer_name}" - ) + # Page progress is summarized by the transfer worker. return _NcclXferHandle(peer_name=self.peer_name, event=event) def start_recv(self, trans_task: PDChunckedTransTask): @@ -362,10 +358,7 @@ def _recv_page(self, trans_task: PDChunckedTransTask): comm = self._ensure_comm(is_server=False) stream = self._get_stream() comm.recv(page_tensor, src=0, stream=stream) - if self.transporter.args.pd_detail_log: - logger.debug( - f"NCCL recv page done request_id={trans_task.request_id} " f"dst_page={trans_task.dst_page_index}" - ) + # Page progress is summarized by the transfer worker. except BaseException as e: trans_task.error_info = str(e) logger.exception(str(e)) @@ -458,7 +451,7 @@ def _start_server(self, host_ip: str, port_min: int, port_max: int) -> tuple[Thr except OSError as e: last_error = e if e.errno == errno.EADDRINUSE: - logger.info(f"NCCL RPyC control port {host_ip}:{cur_port} is in use, try next port") + logger.debug(f"NCCL RPyC control port {host_ip}:{cur_port} is in use, try next port") else: logger.warning(f"Create NCCL RPyC control channel on {host_ip}:{cur_port} failed: {e}") raise RuntimeError(f"can not allocate NCCL control port in [{port_min}, {port_max}]") from last_error diff --git a/lightllm/server/router/model_infer/mode_backend/pd/nixl_kv_transporter.py b/lightllm/server/router/model_infer/mode_backend/pd/nixl_kv_transporter.py index fe3938dcda..ff96500f5d 100644 --- a/lightllm/server/router/model_infer/mode_backend/pd/nixl_kv_transporter.py +++ b/lightllm/server/router/model_infer/mode_backend/pd/nixl_kv_transporter.py @@ -7,7 +7,7 @@ from typing import Dict from torch import Tensor from lightllm.server.pd_io_struct import PDChunckedTransTask, PDAgentMetadata -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import init_logger, log_debug_event logger = init_logger(__name__) @@ -17,14 +17,15 @@ from nixl._api import nixlBind from nixl._api import nixl_agent_config - logger.info("Nixl is available") + logger.debug("NIXL is available") except ImportError: - logger.warning("nixl is not installed, which is required for pd disagreggation!!!") NixlWrapper = None class NixlKVTransporter: def __init__(self, node_id: int, tp_idx: int, kv_move_buffer: Tensor): + if NixlWrapper is None: + raise ImportError("pd_trans_mode=nixl requires NIXL; install NIXL or select pd_trans_mode=nccl") self.node_id = node_id self.tp_idx = tp_idx self.capture_telemetry = os.getenv("LIGHTLLM_NIXL_CAPTURE_TELEMETRY", "0").lower() in ( @@ -97,8 +98,13 @@ def connect_add_remote_agent(self, remote_agent: PDAgentMetadata): ) remote_agent.page_xfer_handles = kv_page_xfer_handles - logger.info( - f"Added remote agent {peer_name} with mem desc {page_mem_desc} cost time: {time.time() - start_time} s" + logger.info("KV peer connected: peer=%s duration_s=%.3f", peer_name, time.time() - start_time) + log_debug_event( + logger, + "kv.peer.descriptor", + peer=peer_name, + descriptor_type=type(page_mem_desc).__name__, + pages=remote_agent.num_pages, ) self.remote_agents[remote_agent.agent_name] = remote_agent @@ -115,13 +121,12 @@ def remove_remote_agent(self, peer_name: str): if remote_agent.page_xfer_handles is not None: self.nixl_agent.release_dlist_handle(remote_agent.page_xfer_handles) logger.info(f"Removed remote agent {peer_name}, cost time: {time.monotonic() - start_time:.6f} s") - except BaseException as e: - logger.error( + except BaseException: + logger.exception( f"remove remote agent {peer_name} failed, cost time: {time.monotonic() - start_time:.6f} s" ) - logger.exception(str(e)) else: - logger.warning(f"try to remove remote agent, but peer name {peer_name} agent did not exist") + logger.debug("KV peer already removed: peer=%s", peer_name) def send_write_done_task_to_decode_node(self, trans_task: PDChunckedTransTask): decode_agent_name = trans_task.decode_agent_name @@ -213,9 +218,8 @@ def send_error_info_to_prefill_node(self, trans_task: PDChunckedTransTask): remote_agent_name=prefill_agent_name, notif_msg=pickle.dumps(new_trans_task), ) - except BaseException as e: - logger.error(f"send error info to prefill node failed: {trans_task.to_str()}") - logger.exception(str(e)) + except BaseException: + logger.exception(f"send error info to prefill node failed: {trans_task.to_str()}") self.remove_remote_agent(peer_name=prefill_agent_name) return @@ -239,9 +243,8 @@ def send_error_info_to_decode_node(self, trans_task: PDChunckedTransTask): remote_agent_name=decode_agent_name, notif_msg=pickle.dumps(new_trans_task), ) - except BaseException as e: - logger.error(f"send error info to decode node failed: {trans_task.to_str()}") - logger.exception(str(e)) + except BaseException: + logger.exception(f"send error info to decode node failed: {trans_task.to_str()}") self.remove_remote_agent(peer_name=decode_agent_name) return diff --git a/lightllm/server/router/model_infer/mode_backend/pd/prefill_node_impl/prefill_kv_move_manager.py b/lightllm/server/router/model_infer/mode_backend/pd/prefill_node_impl/prefill_kv_move_manager.py index c9ee171c54..cdae162de7 100644 --- a/lightllm/server/router/model_infer/mode_backend/pd/prefill_node_impl/prefill_kv_move_manager.py +++ b/lightllm/server/router/model_infer/mode_backend/pd/prefill_node_impl/prefill_kv_move_manager.py @@ -4,7 +4,7 @@ import torch.multiprocessing as mp import time from typing import List, Dict, Union, Callable -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import init_logger, set_log_context from lightllm.server.pd_io_struct import PDAbortReq, PDChunckedTransTask from lightllm.utils.graceful_utils import graceful_registry from lightllm.server.core.objs import StartArgs @@ -33,6 +33,7 @@ def _init_env(args, info_queue: mp.Queue, event: mp.Event): # 注册graceful 退出的处理 graceful_registry(inspect.currentframe().f_code.co_name) + set_log_context(component="kv_move") setproctitle.setproctitle(f"lightllm::{get_unique_server_name()}::prefill_kv_move_manager") from .prefill_trans_process import start_prefill_trans_process @@ -73,11 +74,5 @@ def task_dispatcher_loop(self): try: trans_process: KVTransProcess = self.kv_trans_processes[device_id] trans_process.task_in_queue.put(task) - if ( - isinstance(task, PDChunckedTransTask) - and self.args.pd_detail_log - and logger.isEnabledFor(logging.DEBUG) - ): - logger.debug(f"kv move manager dispatch task {task.to_str()} to device {device_id}") except BaseException as e: logger.exception(str(e)) diff --git a/lightllm/server/router/model_infer/mode_backend/pd/prefill_node_impl/prefill_trans_process.py b/lightllm/server/router/model_infer/mode_backend/pd/prefill_node_impl/prefill_trans_process.py index 68008a9604..e5753ef5a6 100644 --- a/lightllm/server/router/model_infer/mode_backend/pd/prefill_node_impl/prefill_trans_process.py +++ b/lightllm/server/router/model_infer/mode_backend/pd/prefill_node_impl/prefill_trans_process.py @@ -1,4 +1,5 @@ import torch +import logging import time import inspect import threading @@ -7,7 +8,7 @@ import queue import pickle from typing import List, Dict, Optional, Union -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import init_logger, set_log_context, DebugStatsLogger from lightllm.common.kv_cache_mem_manager import MemoryManager from lightllm.server.pd_io_struct import PDAbortReq, PDChunckedTransTask from lightllm.utils.graceful_utils import graceful_registry @@ -56,6 +57,7 @@ def _init_env( try: torch.cuda.set_device(device_id) graceful_registry(inspect.currentframe().f_code.co_name) + set_log_context(component="kv_transfer", device_id=device_id) task_out_queue.put("proc_start") # 从共享内存读取所有rank的mem_manager @@ -78,9 +80,8 @@ def _init_env( while True: time.sleep(100) - except Exception as e: - logger.exception(str(e)) - logger.error(f"Fatal error happened in kv trans process: {e}") + except Exception: + logger.exception("KV transfer worker initialization failed") pass @@ -230,13 +231,10 @@ def ready_transfer_loop(self): with self.waiting_dict_lock: self.waiting_dict[key] = trans_task self.transporter.send_write_request_task_to_decode_node(trans_task) - if self.args.pd_detail_log: - logger.debug(f"send WRITE request to decode: {key}") except BaseException as e: with self.waiting_dict_lock: self.waiting_dict.pop(key, None) - logger.error(f"send WRITE request to decode failed: {trans_task.to_str()}") - logger.exception(str(e)) + logger.exception(f"send WRITE request to decode failed: {trans_task.to_str()}") trans_task.error_info = f"send WRITE request to decode failed: {str(e)}" self.transporter.remove_remote_agent(peer_name=trans_task.decode_agent_name) self.failed_queue.put(trans_task) @@ -249,8 +247,7 @@ def accept_decode_write_task_loop(self): try: notifies_dict = self.transporter.get_new_notifs() except BaseException as e: - logger.error(f"get new notifies failed: {str(e)}") - logger.exception(str(e)) + logger.exception(f"get new notifies failed: {str(e)}") notifies_dict = {} if notifies_dict: @@ -276,12 +273,6 @@ def accept_decode_write_task_loop(self): if trans_task is not None: trans_task.dst_page_index = notify_obj.dst_page_index self.write_peer_kv_queue.put(trans_task) - if self.args.pd_detail_log: - logger.debug( - f"recv WRITE ready from decode request_id={trans_task.request_id} " - f"kv=[{trans_task.start_kv_index},{trans_task.end_kv_index}) " - f"srcpage={trans_task.src_page_index} dstpage={trans_task.dst_page_index}" - ) else: logger.warning( f"can not find pending WRITE request for ready notify: {notify_obj.to_str()}" @@ -325,12 +316,9 @@ def write_peer_kv_loop(self): trans_task.start_trans_time = time.time() with self.waiting_dict_lock: self.waiting_dict[trans_task.get_key()] = trans_task - if self.args.pd_detail_log: - logger.debug(f"start WRITE to decode node: {trans_task.to_str()}") continue except BaseException as e: - logger.error(f"write_blocks_paged failed: {trans_task.to_str()}") - logger.exception(str(e)) + logger.exception(f"write_blocks_paged failed: {trans_task.to_str()}") self.transporter.remove_remote_agent(peer_name=trans_task.decode_agent_name) trans_task.error_info = f"write_blocks_paged failed: {str(e)}" self.failed_queue.put(trans_task) @@ -372,23 +360,16 @@ def _update_task_status_once(self): completed_task = self.waiting_dict.pop(trans_task.get_key(), None) if completed_task is None: continue - if self.transporter.capture_telemetry: + if self.transporter.capture_telemetry and logger.isEnabledFor(logging.DEBUG): try: telem = self.transporter.nixl_agent.get_xfer_telemetry(completed_task.xfer_handle) - total_us = telem.xferDuration - post_us = telem.postDuration - backend_us = telem.xferDuration - telem.postDuration - nixl_backend = self.transporter.nixl_agent.query_xfer_backend(completed_task.xfer_handle) - if self.args.pd_detail_log: - logger.debug( - f"write trans task request_id={completed_task.request_id} " - f"kv=[{completed_task.start_kv_index},{completed_task.end_kv_index}) " - f"src_page={completed_task.src_page_index} " - f"dst_page={completed_task.dst_page_index} " - f"xfer time: {total_us:.3f} us, " - f"post time: {post_us:.3f} us, backend time: {backend_us:.3f} us, " - f"nixl_backend: {nixl_backend}, total_bytes: {telem.totalBytes}" - ) + if not hasattr(self, "_debug_telemetry"): + self._debug_telemetry = DebugStatsLogger(logger, "kv.transfer.telemetry") + self._debug_telemetry.add( + bytes_total=telem.totalBytes, + transfer_us_total=telem.xferDuration, + post_us_total=telem.postDuration, + ) except BaseException: logger.exception(f"get transfer telemetry failed: {completed_task.to_str()}") @@ -400,13 +381,6 @@ def _update_task_status_once(self): self.failed_queue.put(completed_task) continue - if self.args.pd_detail_log: - logger.debug( - f"send WRITE done nixl notify " - f"request_id={completed_task.request_id} " - f"kv=[{completed_task.start_kv_index},{completed_task.end_kv_index}) " - f"src_page={completed_task.src_page_index} dst_page={completed_task.dst_page_index}" - ) self.success_queue.put(completed_task) elif ret == "ERR": failed_task = self.waiting_dict.pop(trans_task.get_key(), None) @@ -422,8 +396,16 @@ def _update_task_status_once(self): @log_exception def success_loop(self): torch.cuda.set_device(self.device_id) + debug_stats = DebugStatsLogger(logger, "kv.transfer.prefill") while True: - trans_task: PDChunckedTransTask = self.success_queue.get() + try: + item = self.success_queue.get(timeout=debug_stats.interval) + except queue.Empty: + debug_stats.flush() + if hasattr(self, "_debug_telemetry"): + self._debug_telemetry.flush() + continue + trans_task: PDChunckedTransTask = item # 写回后，回收页面 if trans_task.src_page_index is not None: self.page_index_queue.put(trans_task.src_page_index) @@ -435,11 +417,16 @@ def success_loop(self): ret.first_gen_token_logprob = None self.task_out_queue.put(ret) - if self.args.pd_detail_log: - if trans_task.start_trans_time is not None: - logger.debug(f"trans task ret success:{ret} cost time: {trans_task.transfer_time()}s") - else: - logger.debug(f"trans task ret success:{ret}") + if logger.isEnabledFor(logging.DEBUG): + debug_stats.add( + request_id=trans_task.request_id, + pages=1, + kv_tokens_total=trans_task.transfer_kv_num(), + state_pages=int(trans_task.page_kind != "kv"), + bytes_total=self.transporter.kv_move_buffer[0].numel() + * self.transporter.kv_move_buffer.element_size(), + task_duration_ms_total=round((time.time() - trans_task.create_time) * 1000, 3), + ) @log_exception def fail_loop(self): @@ -455,7 +442,7 @@ def fail_loop(self): ret = trans_task.createRetObj() self.task_out_queue.put(ret) - logger.warning(f"trans task ret fail:{ret}") + logger.error(f"KV transfer failed: {ret}") if trans_task.error_info is not None: self._abort(request_id=trans_task.request_id, error_info=trans_task.error_info) diff --git a/lightllm/server/router/model_infer/model_rpc.py b/lightllm/server/router/model_infer/model_rpc.py index c66ea0ec5b..e8ef21fd4f 100644 --- a/lightllm/server/router/model_infer/model_rpc.py +++ b/lightllm/server/router/model_infer/model_rpc.py @@ -26,7 +26,7 @@ from lightllm.server.router.model_infer.mode_backend.redundancy_expert_manager import RedundancyExpertManager from lightllm.server.router.model_infer.mode_backend.rl_backend_ops import RlBackendOps from lightllm.server.core.objs.start_args_type import StartArgs -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import init_logger, set_log_context, log_info_on_leader from lightllm.utils.graceful_utils import graceful_registry from lightllm.utils.process_check import start_parent_check_thread from lightllm.utils.envs_utils import get_unique_server_name, get_model_infer_recursion_limit @@ -47,7 +47,7 @@ def __init__(self, args, rank: int, rank_in_node: int, node_world_size: int, inf self.rank_in_node = rank_in_node self.backend = None self.rl_backend_ops = None - logger.info(f"Initialized RPC server for rank {self.rank}.") + logger.debug(f"Initialized RPC server for rank {self.rank}.") return def exposed_init_model(self, kvargs): @@ -82,7 +82,7 @@ def exposed_init_model(self, kvargs): else: self.backend = ChunkedPrefillBackend() - logger.info(f"use {self.backend.__class__.__name__}") + log_info_on_leader(logger, "event=engine.backend_selected backend=%s", self.backend.__class__.__name__) self.backend.init_model(kvargs) self.rl_backend_ops = RlBackendOps(self.backend) if self.args.enable_rl else None @@ -163,10 +163,18 @@ def _init_env( # spawn 不继承父进程的递归上限；在启动推理线程前为每个 rank 显式设置。 sys.setrecursionlimit(get_model_infer_recursion_limit()) - logger.info(f"Model inference rank {rank} Python recursion limit: {sys.getrecursionlimit()}") + logger.debug(f"Model inference rank {rank} Python recursion limit: {sys.getrecursionlimit()}") # 注册graceful 退出的处理 graceful_registry(inspect.currentframe().f_code.co_name) + tp_size = max(1, args.tp // args.dp) + set_log_context( + component="model_infer", + node_rank=args.node_rank, + global_rank=rank, + tp_rank=rank % tp_size, + dp_rank=rank // tp_size, + ) setproctitle.setproctitle(f"lightllm::{get_unique_server_name()}::model_infer:RANK{rank}") start_parent_check_thread() diff --git a/lightllm/server/router/req_queue/chunked_prefill/impl_for_pd_prefill.py b/lightllm/server/router/req_queue/chunked_prefill/impl_for_pd_prefill.py index a2fa2512a1..7f9edea05e 100644 --- a/lightllm/server/router/req_queue/chunked_prefill/impl_for_pd_prefill.py +++ b/lightllm/server/router/req_queue/chunked_prefill/impl_for_pd_prefill.py @@ -11,7 +11,7 @@ class PDPrefillQueue(BaseQueue): def __init__(self, args, router, dp_index, dp_size_in_node) -> None: super().__init__(args, router, dp_index, dp_size_in_node) - logger.info( + logger.debug( "PD prefill requests normally generate only one output token; " "estimate peak KV usage by summing each request's a_len + b_len, " "including page alignment, MTP and asynchronous exit reserves" @@ -87,7 +87,6 @@ def generate_new_batch(self, current_batch: Batch): return new_batch def _calcu_batch_token_load_batch_not_none(self, current_batch: Batch): - estimated_peak_token_num = self._caclu_batch_estimated_peak_token_num(current_batch) return (estimated_peak_token_num, estimated_peak_token_num / self.max_total_tokens) diff --git a/lightllm/server/visualserver/model_infer/model_rpc.py b/lightllm/server/visualserver/model_infer/model_rpc.py index 22d5a43e73..2ac5e93720 100644 --- a/lightllm/server/visualserver/model_infer/model_rpc.py +++ b/lightllm/server/visualserver/model_infer/model_rpc.py @@ -141,11 +141,7 @@ def exposed_init_model(self, kvargs): self._init_taskes() except Exception as e: - print("#" * 16) - print("load model error:", str(e), e, type(e)) - import traceback - - traceback.print_exc() + logger.exception("Model loading failed") raise e set_random_seed(2147483647) @@ -195,7 +191,7 @@ def exposed_run_task(self, images: List["ImageItem"], ref_event_list: List[threa def _log_latency(self, image: ImageItem, stage: str): latency = time.time() - image.start_time if latency > 0.02: - logger.info(f"{stage} latency {latency:.4f} seconds for image with md5 {image.md5}") + logger.debug(f"{stage} latency {latency:.4f} seconds for image with md5 {image.md5}") image.start_time = time.time() def _init_taskes(self): diff --git a/lightllm/server/visualserver/visual_only_manager.py b/lightllm/server/visualserver/visual_only_manager.py index 835a25fe3e..d600c5d6cc 100644 --- a/lightllm/server/visualserver/visual_only_manager.py +++ b/lightllm/server/visualserver/visual_only_manager.py @@ -74,7 +74,6 @@ async def register_to_config_server_loop(self, args: StartArgs): try: uri = f"ws://{args.config_server_host}:{ports.config_server_port}/visual_register" async with websockets.connect(uri, max_queue=(2048 * 1024, 2048 * 1023)) as websocket: - sock = websocket.transport.get_extra_info("socket") sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) @@ -87,19 +86,16 @@ async def register_to_config_server_loop(self, args: StartArgs): await websocket.send("heartbeat") await asyncio.sleep(40) - except Exception as e: - logger.error("connetion to config_server has error") - logger.exception(str(e)) + except Exception: + logger.exception("connetion to config_server has error") await asyncio.sleep(10) logger.info("reconnection to config_server") async def wait_to_model_ready(self): - self.model_rpcs: List[List[VisualModelRpcClient]] = [[] for _ in range(self.vit_dp)] self.vit_attn_backend = init_vit_att_backend(index=0) for dp_rank_id in range(self.vit_dp): for tp_rank_id in range(self.vit_tp): - rpc_model = await start_model_process() self.model_rpcs[dp_rank_id].append(rpc_model) diff --git a/lightllm/utils/config_utils.py b/lightllm/utils/config_utils.py index 8f76b4a53c..a75f4122eb 100644 --- a/lightllm/utils/config_utils.py +++ b/lightllm/utils/config_utils.py @@ -166,16 +166,16 @@ def auto_set_fused_shared_experts(args) -> None: """ if args.enable_fused_shared_experts: - logger.info("skip auto setting fused shared experts: already enabled") + logger.debug("skip auto setting fused shared experts: already enabled") return if args.enable_ep_moe: - logger.info("do not enable fused shared experts: EP MoE uses a separate implementation") + logger.debug("do not enable fused shared experts: EP MoE uses a separate implementation") return model_dir = args.model_dir if not model_dir: - logger.info("do not enable fused shared experts: model_dir is empty") + logger.debug("do not enable fused shared experts: model_dir is empty") return model_type = get_model_type(model_dir) @@ -190,11 +190,11 @@ def auto_set_fused_shared_experts(args) -> None: "qwen3_5_moe_text", } if model_type not in supported_model_types: - logger.info(f"do not enable fused shared experts: unsupported model_type={model_type}") + logger.debug(f"do not enable fused shared experts: unsupported model_type={model_type}") return args.enable_fused_shared_experts = True - logger.info(f"auto enable fused shared experts for model_type={model_type}") + logger.debug(f"auto enable fused shared experts for model_type={model_type}") def _get_config_llm_keyvalue(model_path: str, key_name: list[str]): @@ -443,7 +443,7 @@ def has_vision_module(model_path: str) -> bool: else: raise Exception("unknown vision model type") except: - logger.info(f"model path: {model_path} does not has vision module") + logger.debug(f"model path: {model_path} does not has vision module") return False @@ -466,7 +466,7 @@ def has_audio_module(model_path: str) -> bool: else: raise Exception("unknown audio model type") except: - logger.info(f"model path: {model_path} does not has audio module") + logger.debug(f"model path: {model_path} does not has audio module") return False @@ -482,7 +482,7 @@ def is_linear_att_mixed_model(model_path: str) -> bool: else: return False except: - logger.info(f"model path: {model_path} does not has linear hybrid attention") + logger.debug(f"model path: {model_path} does not has linear hybrid attention") return False @@ -587,12 +587,12 @@ def auto_set_response_parsers(args) -> None: if args.tool_call_parser is None: args.tool_call_parser = get_tool_call_parser_for_model(args.model_dir) if args.tool_call_parser: - logger.info(f"Auto set tool_call_parser to {args.tool_call_parser} based on model type") + logger.debug(f"Auto set tool_call_parser to {args.tool_call_parser} based on model type") if args.reasoning_parser is None: args.reasoning_parser = get_reasoning_parser_for_model(args.model_dir) if args.reasoning_parser: - logger.info(f"Auto set reasoning_parser to {args.reasoning_parser} based on model type") + logger.debug(f"Auto set reasoning_parser to {args.reasoning_parser} based on model type") @lru_cache(maxsize=None) diff --git a/lightllm/utils/device_utils.py b/lightllm/utils/device_utils.py index 58bff90560..ea7124ac16 100644 --- a/lightllm/utils/device_utils.py +++ b/lightllm/utils/device_utils.py @@ -101,7 +101,7 @@ def is_nvidia(): and getattr(torch.version, "hip", None) is None and not is_musa() ) - logger.info(f"device is_nvidia : {ans}") + logger.debug(f"device is_nvidia : {ans}") return ans @@ -297,12 +297,12 @@ def triton_support_tensor_descriptor() -> bool: _ = TensorDescriptor - logger.info("triton support tensor_descriptor") + logger.debug("triton support tensor_descriptor") return True except Exception: pass - logger.info("triton not support tensor_descriptor") + logger.debug("triton not support tensor_descriptor") return False diff --git a/lightllm/utils/dist_check_utils.py b/lightllm/utils/dist_check_utils.py index 12b0b81993..f6f016dd8c 100644 --- a/lightllm/utils/dist_check_utils.py +++ b/lightllm/utils/dist_check_utils.py @@ -9,7 +9,7 @@ import threading from typing import TYPE_CHECKING, Callable -from lightllm.utils.log_utils import init_logger +from lightllm.utils.log_utils import init_logger, summarize_exception if TYPE_CHECKING: from lightllm.server.core.objs.start_args_type import StartArgs @@ -143,9 +143,12 @@ def _check_ok_two_gpu_all_reduce(worker_entry: Callable[[int, int], None], init_ torch_mp.spawn(worker_entry, args=(init_tcp_port,), nprocs=2, join=True) return True except Exception as error: - error_str = str(error) - error_str = error_str[-66:].replace("\n", "") - logger.warning("2-GPU all-reduce capability check failed for %s: %s", worker_entry.__name__, error_str) + logger.warning( + "event=allreduce.probe_failed probe=%s reason=%r action=disable_backend", + worker_entry.__name__, + summarize_exception(error), + ) + logger.debug("All-reduce capability probe diagnostics", exc_info=True) return False diff --git a/lightllm/utils/dist_utils.py b/lightllm/utils/dist_utils.py index c3f4684c94..ed955a0846 100644 --- a/lightllm/utils/dist_utils.py +++ b/lightllm/utils/dist_utils.py @@ -1,3 +1,4 @@ +from lightllm.utils.log_utils import set_log_context import torch.distributed as dist import os import torch @@ -77,6 +78,7 @@ def init_vision_distributed_env(kvargs): set_dp_size(dp_size) set_dp_world_size(tp_world_size) set_current_rank_in_dp(tp_rank_id) + set_log_context(tp_rank=tp_rank_id, dp_rank=kvargs.get("dp_rank_id", "-")) device_id = kvargs["device_id"] set_current_device_id(device_id) torch.cuda.set_device(device_id) @@ -112,6 +114,7 @@ def init_audio_distributed_env(kvargs): set_dp_size(1) set_dp_world_size(tp_world_size) set_current_rank_in_dp(tp_rank_id) + set_log_context(tp_rank=tp_rank_id, dp_rank=kvargs.get("dp_rank_id", "-")) device_id = kvargs["device_id"] set_current_device_id(device_id) torch.cuda.set_device(device_id) @@ -137,6 +140,7 @@ def init_distributed_env(kvargs): set_dp_world_size(get_global_world_size() // get_dp_size()) set_global_dp_rank(get_global_rank() // get_dp_world_size()) set_current_rank_in_dp(get_global_rank() % get_dp_world_size()) + set_log_context(global_rank=get_global_rank(), tp_rank=get_current_rank_in_dp(), dp_rank=get_global_dp_rank()) set_current_rank_in_node(get_global_rank() % node_world_size) set_node_world_size(node_world_size) diff --git a/lightllm/utils/embed_utils.py b/lightllm/utils/embed_utils.py index dec1537d66..81b159deef 100644 --- a/lightllm/utils/embed_utils.py +++ b/lightllm/utils/embed_utils.py @@ -52,6 +52,6 @@ def calcu_embed_cache_meta() -> "EmbedCacheMeta": ) embed_cache_meta_data.token_num = token_num - logger.info(f"embed cache token num: {embed_cache_meta_data.token_num}") + logger.debug(f"embed cache token num: {embed_cache_meta_data.token_num}") return embed_cache_meta_data diff --git a/lightllm/utils/graceful_utils.py b/lightllm/utils/graceful_utils.py index 79ce198d14..628c7209fe 100644 --- a/lightllm/utils/graceful_utils.py +++ b/lightllm/utils/graceful_utils.py @@ -5,6 +5,9 @@ def graceful_registry(sub_module_name): + from lightllm.utils.log_utils import set_log_context + + set_log_context(component=sub_module_name) import signal # 子进程在受到 SIGTERM的时候，不能自己就提前退出。 diff --git a/lightllm/utils/log_utils.py b/lightllm/utils/log_utils.py index aaca288010..a12b9685b8 100644 --- a/lightllm/utils/log_utils.py +++ b/lightllm/utils/log_utils.py @@ -1,28 +1,39 @@ -# Adapted from -# https://github.com/skypilot-org/skypilot/blob/86dc0f6283a335e4aa37b3c10716f90999f48ab6/sky/sky_logging.py -"""Logging configuration for LightLLM.""" +"""Central logging configuration; see docs/CN/source/tutorial/logging.rst.""" + +import hashlib +import json import logging -import sys +import logging.handlers +import math +import numbers +import multiprocessing import os +import socket +import sys import time -from typing import Optional - -_FORMAT = "%(levelname)s %(asctime)s %(node_role)s[%(filename)s:%(lineno)d] %(message)s" -_DATE_FORMAT = "%m-%d %H:%M:%S" +import threading +from collections import OrderedDict +from datetime import datetime +from contextvars import ContextVar +import uuid -_LOG_LEVEL = os.environ.get("LIGHTLLM_LOG_LEVEL", "info") -_LOG_LEVEL = getattr(logging, _LOG_LEVEL.upper(), 0) -_LOG_DIR = os.environ.get("LIGHTLLM_LOG_DIR", None) +_FORMAT = "%(asctime)s %(levelname)s [%(compact_context)s] %(source)s%(message)s" +_DATE_FORMAT = "%H:%M:%S" _LOG_NODE_ROLE = os.environ.get("LIGHTLLM_LOG_NODE_ROLE", "") - -_PD_NODE_ROLE_MARKERS = { - "prefill": "P", - "decode": "D", -} +_PD_NODE_ROLE_MARKERS = {"prefill": "P", "decode": "D"} +_root_logger = logging.getLogger("lightllm") +_context = {} +_context_pid = os.getpid() +_external_loggers = set() +_module_levels = {} +_log_time_mark_dict = {} +_hostname = socket.gethostname() +_http_request_context = ContextVar("lightllm_http_request", default=None) +_STANDARD_RECORD_FIELDS = set(logging.makeLogRecord({}).__dict__) | {"message", "asctime"} def set_log_node_role(run_mode: str): - """Add a P/D marker to logs and propagate it to child processes.""" + """Propagate the P/D role to spawned and exec'ed workers.""" global _LOG_NODE_ROLE _LOG_NODE_ROLE = _PD_NODE_ROLE_MARKERS.get(run_mode, "") if _LOG_NODE_ROLE: @@ -31,104 +42,649 @@ def set_log_node_role(run_mode: str): os.environ.pop("LIGHTLLM_LOG_NODE_ROLE", None) +def set_log_context(**fields): + """Set process-local component/rank metadata after process or rank initialization.""" + global _context_pid + if _context_pid != os.getpid(): + _context.clear() + _context_pid = os.getpid() + _context.update(fields) + + +def _add_context(record): + context = _context if _context_pid == os.getpid() else {} + defaults = { + "node_role": f"[{_LOG_NODE_ROLE}] " if _LOG_NODE_ROLE else "", + "hostname": _hostname, + "component": multiprocessing.current_process().name, + "node_rank": os.getenv("LIGHTLLM_LOG_NODE_RANK", "-"), + "tp_rank": "-", + "dp_rank": "-", + "global_rank": "-", + "role": _LOG_NODE_ROLE, + "tp_size": int(os.getenv("LIGHTLLM_LOG_TP_SIZE", "1")), + "dp_size": int(os.getenv("LIGHTLLM_LOG_DP_SIZE", "1")), + "nnodes": int(os.getenv("LIGHTLLM_LOG_NNODES", "1")), + "http_request_id": get_http_request_id(), + **context, + } + for key, value in defaults.items(): + if not hasattr(record, key): + setattr(record, key, value) + + +_COMPONENT_ALIASES = { + "MainProcess": "main", + "model_infer": "infer", + "http": "http", + "start_router_process": "router", + "start_detokenization_process": "detoken", + "start_metric_manager": "metrics", + "start_health_check_process": "health", + "start_cache_manager": "cache", + "kv_transfer": "kv-transfer", + "kv_move": "kv-move", + "kv_status": "kv-status", +} + + +def _compact_context(record): + component = _COMPONENT_ALIASES.get(record.component, record.component) + if os.getenv("LIGHTLLM_LOG_RUN_MODE") == "pd_master" and component == "http": + component = "master" + if component.startswith("start_"): + component = component[6:] + for suffix in ("_process", "_manager"): + if component.endswith(suffix): + component = component[: -len(suffix)] + if component == "_init_env" or component.startswith(("Process-", "SpawnProcess-", "ForkProcess-")): + component = "worker" + label = f"{record.role}/{component}" if record.role else component + fields = [label] + for key, size in (("node", record.nnodes), ("tp", record.tp_size), ("dp", record.dp_size)): + rank = getattr(record, f"{key}_rank") + if size > 1 and rank not in (None, "-"): + fields.append(f"{key}={rank}") + return " ".join(fields) + + class NewLineFormatter(logging.Formatter): - """Adds logging prefix to newlines to align multi-line messages.""" + """Prefix every physical line, including traceback lines, with process context.""" - def __init__(self, fmt, datefmt=None): - logging.Formatter.__init__(self, fmt, datefmt) + def formatTime(self, record, datefmt=None): + return datetime.fromtimestamp(record.created).strftime("%H:%M:%S.%f")[:-3] def format(self, record): - record.node_role = f"[{_LOG_NODE_ROLE}] " if _LOG_NODE_ROLE else "" - msg = logging.Formatter.format(self, record) - if record.message != "": - parts = msg.split(record.message) - msg = msg.replace("\n", "\r\n" + parts[0]) - return msg + _add_context(record) + record = logging.makeLogRecord(record.__dict__.copy()) + if record.name == "py.warnings": + # Python warning source snippets and trailing blank lines add no value + # to the console. JSON retains the original warning and source. + record.msg, record.args = record.getMessage().splitlines()[0], () + record.compact_context = _compact_context(record) + record.source = ( + f"{record.filename}:{record.lineno} " + if record.levelno <= logging.DEBUG or record.levelno >= logging.ERROR or record.exc_info + else "" + ) + # Format a separate empty record to obtain the prefix without splitting on + # user-controlled message content. Do not mutate records shared by handlers. + prefix_record = logging.makeLogRecord(record.__dict__.copy()) + prefix_record.msg, prefix_record.args = "", () + prefix_record.exc_info = prefix_record.exc_text = prefix_record.stack_info = None + prefix = super().format(prefix_record) + return super().format(record).replace("\n", "\r\n" + prefix) -_root_logger = logging.getLogger("lightllm") -_default_handler = None -_default_file_handler = None -_inference_log_file_handler = {} +class JsonFormatter(logging.Formatter): + """One JSON object per event; newlines and tracebacks remain inside the object.""" + + def format(self, record): + _add_context(record) + result = { + "timestamp": datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds"), + "level": record.levelname, + "logger": record.name, + "message": record.getMessage(), + "file": record.pathname, + "line": record.lineno, + "pid": record.process, + } + result.update({key: value for key, value in record.__dict__.items() if key not in _STANDARD_RECORD_FIELDS}) + if record.exc_info: + result["exception"] = self.formatException(record.exc_info) + if record.stack_info: + result["stack"] = self.formatStack(record.stack_info) + return json.dumps(result, ensure_ascii=False, default=str) -def _setup_logger(): - _root_logger.setLevel(_LOG_LEVEL) - global _default_handler - global _default_file_handler - fmt = NewLineFormatter(_FORMAT, datefmt=_DATE_FORMAT) - - if _default_handler is None: - _default_handler = logging.StreamHandler(sys.stdout) - _default_handler.flush = sys.stdout.flush # type: ignore - _default_handler.setLevel(_LOG_LEVEL) - _root_logger.addHandler(_default_handler) - - if _default_file_handler is None and _LOG_DIR is not None: - if not os.path.exists(_LOG_DIR): - try: - os.makedirs(_LOG_DIR) - except OSError as e: - _root_logger.warn(f"Error creating directory {_LOG_DIR} : {e}") - _default_file_handler = logging.FileHandler(_LOG_DIR + "/default.log") - _default_file_handler.setLevel(_LOG_LEVEL) - _default_file_handler.setFormatter(fmt) - _root_logger.addHandler(_default_file_handler) - - _default_handler.setFormatter(fmt) - # Setting this will avoid the message - # being propagated to the parent logger. - _root_logger.propagate = False +class ProcessFileHandler(logging.handlers.RotatingFileHandler): + """Use one rotating file per PID, including when a handler is inherited by fork.""" + def __init__(self, directory, max_bytes, backup_count): + self.directory = directory + self.pid = os.getpid() + super().__init__(self._filename(), maxBytes=max_bytes, backupCount=backup_count, encoding="utf-8", delay=True) + + def _filename(self): + return os.path.abspath(os.path.join(self.directory, f"process.{self.pid}.log")) + + def emit(self, record): + if self.pid != os.getpid(): + if self.stream is not None: + self.stream.close() + self.stream = None + self.pid = os.getpid() + self.baseFilename = self._filename() + super().emit(record) -# The logger is initialized when the module is imported. -# This is thread-safe as the module is only imported once, -# guaranteed by the Python GIL. -_setup_logger() + +def _level(value): + value = str(value).upper() + if value not in {"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"}: + raise ValueError(f"Invalid LightLLM log level: {value}") + return getattr(logging, value) + + +def _positive_number(name, default, convert=float): + value = convert(os.getenv(name, str(default))) + if not math.isfinite(value) or value <= 0: + raise ValueError(f"{name} must be positive and finite") + return value + + +def _setup_logger(): + """Configure owned handlers once per call, leaving other libraries untouched.""" + global _module_levels, _request_sample_rate, _stats_interval + level = _level(os.getenv("LIGHTLLM_LOG_LEVEL", "INFO")) + modules = json.loads(os.getenv("LIGHTLLM_LOG_MODULE_LEVELS", "{}")) + if not isinstance(modules, dict) or any(not name.startswith("lightllm.") for name in modules): + raise ValueError("LIGHTLLM_LOG_MODULE_LEVELS must map lightllm.* logger names to levels") + modules = {name: _level(value) for name, value in modules.items()} + log_format = os.getenv("LIGHTLLM_LOG_FORMAT", "text").lower() + if log_format not in {"text", "json"}: + raise ValueError("LIGHTLLM_LOG_FORMAT must be text or json") + request_rate = float(os.getenv("LIGHTLLM_LOG_REQUEST_SAMPLE_RATE", "0.01")) + if not math.isfinite(request_rate) or not 0 <= request_rate <= 1: + raise ValueError("LIGHTLLM_LOG_REQUEST_SAMPLE_RATE must be between 0 and 1") + stats_interval = _positive_number("LIGHTLLM_LOG_STATS_INTERVAL", 10) + formatter = JsonFormatter() if log_format == "json" else NewLineFormatter(_FORMAT, _DATE_FORMAT) + handlers = [logging.StreamHandler(sys.stdout)] + directory = os.getenv("LIGHTLLM_LOG_DIR") + if directory: + os.makedirs(directory, exist_ok=True) + handlers.append( + ProcessFileHandler( + directory, + _positive_number("LIGHTLLM_LOG_MAX_BYTES", 100 * 1024 * 1024, int), + _positive_number("LIGHTLLM_LOG_BACKUP_COUNT", 3, int), + ) + ) + for handler in handlers: + handler.setFormatter(formatter) + # Child loggers decide eligibility; an INFO handler would drop module DEBUG. + handler.setLevel(logging.NOTSET) + handler._lightllm_owned = True + old_handlers = [handler for handler in _root_logger.handlers if getattr(handler, "_lightllm_owned", False)] + for handler in old_handlers: + _root_logger.removeHandler(handler) + _root_logger.setLevel(level) + _root_logger.propagate = False + for handler in handlers: + _root_logger.addHandler(handler) + for name in _module_levels: + logging.getLogger(name).setLevel(logging.NOTSET) + # Request/access categories have switches independent of the service level. + logging.getLogger("lightllm.requests").setLevel(min(level, logging.INFO) if request_rate else logging.NOTSET) + logging.getLogger("lightllm.access").setLevel(logging.INFO if http_access_enabled() else logging.NOTSET) + for name, module_level in modules.items(): + logging.getLogger(name).setLevel(module_level) + _module_levels = modules + for name in _external_loggers: + logger = logging.getLogger(name) + for handler in old_handlers: + logger.removeHandler(handler) + for handler in handlers: + logger.addHandler(handler) + logger.setLevel(level) + for handler in old_handlers: + handler.close() + _request_sample_rate = request_rate + _stats_interval = stats_interval def init_logger(name: str): - pid = os.getpid() - # Use the same settings as above for root logger + """Retrieve a logger without adding handlers to LightLLM module loggers.""" logger = logging.getLogger(name) - logger.setLevel(_LOG_LEVEL) - logger.addHandler(_default_handler) - if _LOG_DIR is not None and pid is None: - logger.addHandler(_default_file_handler) - elif _LOG_DIR is not None: - if _inference_log_file_handler.get(pid, None) is not None: - logger.addHandler(_inference_log_file_handler[pid]) - else: - if not os.path.exists(_LOG_DIR): - try: - os.makedirs(_LOG_DIR) - except OSError as e: - _root_logger.warn(f"Error creating directory {_LOG_DIR} : {e}") - _inference_log_file_handler[pid] = logging.FileHandler(_LOG_DIR + f"/process.{pid}.log") - _inference_log_file_handler[pid].setLevel(_LOG_LEVEL) - _inference_log_file_handler[pid].setFormatter(NewLineFormatter(_FORMAT, datefmt=_DATE_FORMAT)) - _root_logger.addHandler(_inference_log_file_handler[pid]) - logger.addHandler(_inference_log_file_handler[pid]) - logger.propagate = False + if name != "lightllm" and not name.startswith("lightllm."): + # Scripts also use this API with __name__ == '__main__'. + _external_loggers.add(name) + logger.setLevel(_root_logger.level) + for handler in _root_logger.handlers: + if getattr(handler, "_lightllm_owned", False): + logger.addHandler(handler) + logger.propagate = False return logger -_log_time_mark_dict = {} +def configure_logging_from_args(args): + """Explicit CLI values override environment defaults and propagate to workers.""" + for attr, env in ( + ("log_level", "LIGHTLLM_LOG_LEVEL"), + ("log_format", "LIGHTLLM_LOG_FORMAT"), + ("log_module_levels", "LIGHTLLM_LOG_MODULE_LEVELS"), + ("log_request_sample_rate", "LIGHTLLM_LOG_REQUEST_SAMPLE_RATE"), + ("log_stats_interval", "LIGHTLLM_LOG_STATS_INTERVAL"), + ("log_http_level", "LIGHTLLM_LOG_HTTP_LEVEL"), + ): + value = getattr(args, attr, None) + if value is not None: + os.environ[env] = str(value) + access = getattr(args, "enable_http_access_log", None) + if access is not None: + os.environ["LIGHTLLM_LOG_HTTP_ACCESS"] = "1" if access else "0" + _level(os.getenv("LIGHTLLM_LOG_HTTP_LEVEL", "WARNING")) + os.environ["LIGHTLLM_LOG_NODE_RANK"] = str(args.node_rank) + os.environ["LIGHTLLM_LOG_TP_SIZE"] = str(max(1, getattr(args, "tp", 1) // getattr(args, "dp", 1))) + os.environ["LIGHTLLM_LOG_DP_SIZE"] = str(getattr(args, "dp", 1)) + os.environ["LIGHTLLM_LOG_NNODES"] = str(getattr(args, "nnodes", 1)) + os.environ["LIGHTLLM_LOG_RUN_MODE"] = args.run_mode + set_log_node_role(args.run_mode) + _setup_logger() + set_log_context(component="launcher", node_rank=args.node_rank) + + +def should_log_request(request_id): + """Deterministic sampling by ID; does not consume application RNG state.""" + if _request_sample_rate == 0: + return False + if _request_sample_rate == 1: + return True + digest = hashlib.blake2b(str(request_id).encode(), digest_size=8).digest() + return int.from_bytes(digest, "big") / 2**64 < _request_sample_rate + + +def http_access_enabled(): + return os.getenv("LIGHTLLM_LOG_HTTP_ACCESS", "0").lower() in {"1", "true", "yes", "on"} + + +def new_http_request_id(headers): + """Reuse a bounded printable client ID, otherwise generate a transport ID.""" + value = next((v.decode("latin-1") for k, v in headers if k.lower() == b"x-request-id"), "") + if not value or len(value) > 128 or any(ord(char) < 33 or ord(char) > 126 for char in value): + value = uuid.uuid4().hex + return value + + +def set_http_request_id(request_id): + return _http_request_context.set((os.getpid(), request_id)) + + +def reset_http_request_id(token): + _http_request_context.reset(token) + + +def get_http_request_id(): + value = _http_request_context.get() + return value[1] if value is not None and value[0] == os.getpid() else "-" + +def summarize_exception(error, limit=240): + """Keep a useful compiler/worker failure reason without dumping its transcript.""" + lines = [line.strip() for line in str(error).splitlines() if line.strip()] + useful = [ + line + for line in lines + if any(marker in line.lower() for marker in ("no such file", "not found", "error:", "runtimeerror:")) + ] + reason = (useful or lines or [type(error).__name__])[-1] + return reason if len(reason) <= limit else reason[: limit - 3] + "..." -def log_time_ready(mark_name, time_count: int): + +def progress_enabled(): + """Only display interactive bars on a terminal with text INFO logging.""" + return ( + sys.stderr.isatty() + and os.getenv("LIGHTLLM_LOG_FORMAT", "text").lower() == "text" + and _root_logger.isEnabledFor(logging.INFO) + ) + + +def log_http_access(method, path, status, duration_ms, http_request_id=None): + excluded = os.getenv("LIGHTLLM_LOG_HTTP_EXCLUDE_PATHS", "/health,/healthz,/readiness,/liveness,/metrics").split(",") + if not http_access_enabled() or any( + prefix and (path == prefix or path.startswith(prefix.rstrip("/") + "/")) for prefix in excluded + ): + return + fields = { + "event": "http.access", + "http_request_id": http_request_id or get_http_request_id(), + "method": method, + "path": path, + "status": status, + "duration_ms": round(duration_ms, 3), + } + logging.getLogger("lightllm.access").info( + "event=http.access http_request_id=%s method=%s path=%r status=%s duration_ms=%.3f", + fields["http_request_id"], + method, + path, + status, + duration_ms, + extra=fields, + stacklevel=2, + ) + + +def log_request_summary(request_id, **fields): + status = fields.get("status", "completed") + failed = status in {"error", "timeout", "failed", "busy"} + cancelled = status in {"cancelled", "abort"} + logger = logging.getLogger("lightllm.requests") + stage = fields.get("timing_scope") in {"prefill", "decode"} + if not failed and not cancelled and not should_log_request(request_id): + return + event = fields.pop( + "event", "request.failed" if failed else "request.cancelled" if cancelled else "request.completed" + ) + client_id = fields.pop("x_request_id", None) or get_http_request_id() + if client_id != "-": + fields.setdefault("http_request_id", client_id) + mtp_enabled = fields.pop("mtp_enabled", False) + fields = { + key: _debug_value(value) + for key, value in fields.items() + if not (("cache" in key and not value) or (key.startswith("mtp_") and not mtp_enabled)) + } + method = logger.error if failed else logger.debug if stage or cancelled else logger.info + method( + "event=%s request_id=%s %s", + event, + request_id, + " ".join(f"{key}={value!r}" for key, value in fields.items()), + extra={"event": event, "request_id": request_id, **fields}, + stacklevel=2, + ) + + +def request_finish_fields(reasons): + """Preserve failures across best-of children instead of trusting the last token.""" + reasons = sorted(set(reasons)) + status = ( + "error" + if "error" in reasons + else "abort" + if "abort" in reasons + else "continued" + if "pd_capacity" in reasons + else "completed" + ) + return {"status": status, "finish_reason": reasons[0] if len(reasons) == 1 else reasons} + + +def get_http_logging_args(application_access=False): + """Hypercorn access and error logging are independent of service verbosity.""" + level = os.getenv("LIGHTLLM_LOG_HTTP_LEVEL", "warning") + _level(level) + args = ["--log-level", level.lower(), "--error-logfile", "-"] + if http_access_enabled() and not application_access: + args.extend(["--access-logfile", "-"]) + return args + + +def get_log_stats_interval(): + return _stats_interval + + +def log_time_ready(mark_name, time_count: float): + """Return true once each interval, using a monotonic clock and process-local key.""" + key = (os.getpid(), mark_name) + now = time.monotonic() + previous = _log_time_mark_dict.setdefault(key, now) + if now - previous >= time_count: + _log_time_mark_dict[key] = now + return True + return False + + +def _debug_value(value, depth=0): + """Bound diagnostic values without expanding tensors, descriptors or objects.""" + if isinstance(value, str): + return value if len(value) <= 256 else value[:253] + "..." + if value is None or isinstance(value, (bool, int, float)): + return value + if isinstance(value, numbers.Integral): + return int(value) + if isinstance(value, numbers.Real): + return float(value) + if isinstance(value, (list, tuple)): + return ( + {"count": len(value)} + if depth + else {"count": len(value), "sample": [_debug_value(v, depth=1) for v in value[:8]]} + ) + if isinstance(value, (bytes, bytearray)): + return {"type": type(value).__name__, "bytes": len(value)} + return {"type": type(value).__name__} + + +def log_debug_event(logger, event, *, leader_only=False, stacklevel=2, **fields): + """A bounded phase/decision event, never a dump of request content.""" + if not logger.isEnabledFor(logging.DEBUG): + return + context = _context if _context_pid == os.getpid() else {} + if leader_only and context.get("tp_rank", 0) != 0: + return + fields = {key: _debug_value(value) for key, value in list(fields.items())[:16]} + logger.debug( + "event=%s %s", + event, + " ".join(f"{key}={value!r}" for key, value in fields.items()), + extra={"event": event, **fields}, + stacklevel=stacklevel, + ) + + +class DebugStatsLogger: + """Accumulate hot-loop counts; emit at most once per interval, with idle flush. + + Call ``flush()`` when the owning loop becomes idle so the final partial window + is not lost. State has fixed cardinality and is protected for worker threads. + """ + + def __init__(self, logger, event, interval=None): + self.logger, self.event = logger, event + self.interval = interval if interval is not None else get_log_stats_interval() + self.pid = os.getpid() + self.lock = threading.Lock() + self.count = 0 + self.totals = {} + self.request_ids = [] + self.request_ids_truncated = False + self.started = time.monotonic() + self.last_emit = None + + def _reset_after_fork(self): + if self.pid != os.getpid(): + self.__init__(self.logger, self.event, self.interval) + + def add(self, request_id=None, **totals): + if not self.logger.isEnabledFor(logging.DEBUG): + return + self._reset_after_fork() + with self.lock: + self.count += 1 + if request_id is not None and request_id not in self.request_ids: + if len(self.request_ids) < 8: + self.request_ids.append(request_id) + else: + self.request_ids_truncated = True + for key, value in totals.items(): + self.totals[key] = self.totals.get(key, 0) + value + self._flush_locked() + + def flush(self): + if not self.logger.isEnabledFor(logging.DEBUG): + return + self._reset_after_fork() + with self.lock: + self._flush_locked() + + def _flush_locked(self): + now = time.monotonic() + if not self.count or (self.last_emit is not None and now - self.last_emit < self.interval): + return + fields = {} + if self.request_ids: + fields = {"request_id_sample": self.request_ids, "request_ids_truncated": self.request_ids_truncated} + log_debug_event( + self.logger, + self.event, + stacklevel=4, + count=self.count, + window_s=round(now - self.started, 3), + **fields, + **self.totals, + ) + self.count, self.totals = 0, {} + self.request_ids = [] + self.request_ids_truncated = False + self.started = self.last_emit = now + + +_exception_windows = OrderedDict() +_exception_window_pid = os.getpid() +_exception_window_lock = threading.Lock() + + +def log_repeated_exception(logger, event, **fields): + """Keep the first traceback; count identical recurring failures by callsite. + + Distinct reasons keep their own traceback. The cache is bounded per process; + summaries never replace the first error, and are emitted at most every 10 s. """ - time_count 间隔时间超过多少s调用该函数会返回True，否则返回False - 用于控制一些日志输出的频率 + if not logger.isEnabledFor(logging.ERROR): + return + global _exception_window_pid, _exception_window_lock + if _exception_window_pid != os.getpid(): + _exception_windows.clear() + _exception_window_lock = threading.Lock() + _exception_window_pid = os.getpid() + exc_type, error, traceback = sys.exc_info() + caller = sys._getframe(1) + reason = summarize_exception(error) if error is not None else "unknown failure" + key = (logger.name, event, caller.f_code.co_filename, caller.f_lineno, exc_type, reason) + now = time.monotonic() + with _exception_window_lock: + state = _exception_windows.get(key) + first = state is None + if first: + if len(_exception_windows) >= 128: + _exception_windows.popitem(last=False) + state = {"last": now, "repeated": 0} + _exception_windows[key] = state + else: + _exception_windows.move_to_end(key) + state["repeated"] += 1 + if now - state["last"] < 10: + return + repeated = state["repeated"] + state["last"], state["repeated"] = now, 0 + fields = {key: _debug_value(value) for key, value in list(fields.items())[:12]} + logger.error( + "event=%s reason=%r repeated=%s %s", + event, + reason, + repeated, + " ".join(f"{key}={value!r}" for key, value in fields.items()), + extra={"event": event, "reason": reason, "repeated": repeated, **fields}, + exc_info=(exc_type, error, traceback) if first and error is not None else None, + stacklevel=2, + ) + + +def log_info_on_leader(logger, message, *args, **kwargs): + """Shared TP configuration is INFO on the group's leader, DEBUG on peers. + + This helper must never be used to suppress worker-specific warnings/errors. + Unknown ranks (launcher and non-inference processes) retain INFO output. """ - global _log_time_mark_dict + context = _context if _context_pid == os.getpid() else {} + method = logger.info if context.get("tp_rank", 0) == 0 else logger.debug + kwargs.setdefault("stacklevel", 2) + method(message, *args, **kwargs) - if mark_name not in _log_time_mark_dict: - _log_time_mark_dict[mark_name] = time.time() - return False - cur_time_mark = time.time() - if cur_time_mark - _log_time_mark_dict[mark_name] >= time_count: - _log_time_mark_dict[mark_name] = cur_time_mark + +class PeriodicStatsLogger: + """Rate-limit all snapshots and confirm idle for a full reporting interval.""" + + def __init__(self, logger): + self.logger = logger + self.last_time = time.monotonic() + self.active = None + self.idle_since = None + + def should_log(self, active): + now = time.monotonic() + interval = get_log_stats_interval() + if active: + self.idle_since = None + elif self.idle_since is None: + self.idle_since = now + if now - self.last_time < interval: + return False + if not active and (self.active is False or now - self.idle_since < interval): + return False + self.active = active + self.last_time = now return True - else: - return False + + def log(self, **snapshot): + event = "engine.stats" if self.active else "engine.idle" + self.logger.info( + "event=%s %s", + event, + " ".join( + f"{k}={v}" + for k, v in snapshot.items() + if k not in {"prefill_tokens", "output_tokens", "completed_reqs", "window_s", "scope"} + ), + extra={"event": event, **snapshot}, + stacklevel=2, + ) + + +def log_startup_config(logger, args): + """Only allowlisted operational settings are included, even at DEBUG.""" + fields = ( + "run_mode", + "model_name", + "tp", + "dp", + "nnodes", + "node_rank", + "max_req_total_len", + "max_total_token_num", + "batch_max_tokens", + "mem_fraction", + "pd_trans_mode", + "data_type", + "use_tgi_api", + "tool_call_parser", + "reasoning_parser", + "disable_vision", + "disable_audio", + "enable_fused_shared_experts", + ) + config = {key: getattr(args, key, None) for key in fields} + logger.info( + "event=service.config %s", + " ".join(f"{key}={value}" for key, value in config.items()), + extra={"event": "service.config", "config": config}, + stacklevel=2, + ) + + +_setup_logger() +logging.captureWarnings(True) +init_logger("py.warnings") diff --git a/lightllm/utils/multimodal_utils.py b/lightllm/utils/multimodal_utils.py index 1dc69e44a3..4f3e5f307d 100644 --- a/lightllm/utils/multimodal_utils.py +++ b/lightllm/utils/multimodal_utils.py @@ -30,7 +30,7 @@ def get(self, url: str, proxy: Optional[str]) -> Optional[bytes]: cached = self._cache.get(key) if cached is not None: self._cache.move_to_end(key) - logger.info("url_pool hit") + logger.debug("url_pool hit") return cached def put(self, url: str, proxy: Optional[str], content: bytes) -> None: @@ -80,7 +80,7 @@ def _get_xhttp_client(proxy=None): async def fetch_resource(url, request: Request, timeout, proxy=None): - logger.info(f"Begin to download resource from url: {url}") + logger.debug(f"Begin to download resource from url: {url}") cached = URL_RESOURCE_POOL.get(url, proxy) if cached is not None: return cached @@ -102,6 +102,6 @@ async def fetch_resource(url, request: Request, timeout, proxy=None): content = b"".join(ans_bytes) end_time = time.time() cost_time = end_time - start_time - logger.info(f"Download url {url} resource cost time: {cost_time} seconds") + logger.debug(f"Download url {url} resource cost time: {cost_time} seconds") URL_RESOURCE_POOL.put(url, proxy, content) return content diff --git a/lightllm/utils/net_utils.py b/lightllm/utils/net_utils.py index 16ae239f62..f8c3e6ef7c 100644 --- a/lightllm/utils/net_utils.py +++ b/lightllm/utils/net_utils.py @@ -11,7 +11,7 @@ def get_hostname_ip(): result = subprocess.run(["hostname", "-i"], capture_output=True, text=True, check=True) # 兼容 hostname -i 命令输出多个 ip 的情况 result = result.stdout.strip().split(" ")[0] - logger.info(f"get hostname ip {result}") + logger.debug(f"get hostname ip {result}") return result except subprocess.CalledProcessError as e: logger.exception(f"Error executing command: {e}") @@ -37,4 +37,4 @@ def validate_ports(ports: list): sock.bind(("", port)) except OSError as e: raise RuntimeError(f"port {port} has been used") from e - logger.info(f"validated ports: {ports}") + logger.debug(f"validated ports: {ports}") diff --git a/lightllm/utils/rpyc_fix_utils.py b/lightllm/utils/rpyc_fix_utils.py index 59d73f7812..593f5e5cf9 100644 --- a/lightllm/utils/rpyc_fix_utils.py +++ b/lightllm/utils/rpyc_fix_utils.py @@ -129,7 +129,7 @@ def fix_accept(self): return sock.setblocking(True) - self.logger.info(f"accepted {addrinfo} with fd {sock.fileno()}") + self.logger.debug(f"accepted {addrinfo} with fd {sock.fileno()}") self.clients.add(sock) self._accept_method(sock) diff --git a/lightllm/utils/sgl_utils.py b/lightllm/utils/sgl_utils.py index 475f3fd0c2..fc28296025 100644 --- a/lightllm/utils/sgl_utils.py +++ b/lightllm/utils/sgl_utils.py @@ -16,7 +16,7 @@ sgl_ops = None sgl_allreduce_ops = None HAS_SGL_KERNEL = False - logger.warning( + logger.debug( "sgl_kernel is not installed, you can't use the api of it. \ You can solve it by running `pip install sgl_kernel`." ) @@ -31,7 +31,7 @@ flash_attn_varlen_func = None flash_attn_with_kvcache = None merge_state_v2 = None - logger.warning( + logger.debug( "sgl_kernel is not installed, or the installed version did not support fa3. \ Try to upgrade it." ) diff --git a/lightllm/utils/shm_port_args.py b/lightllm/utils/shm_port_args.py index b0fe9c5523..cf6a488a07 100644 --- a/lightllm/utils/shm_port_args.py +++ b/lightllm/utils/shm_port_args.py @@ -239,7 +239,7 @@ def _get_or_alloc(self, name: str, count: int = 1) -> PortValue: port = self._alloc_free_port(reserved) ports[name] = port self._save(ports) - logger.info(f"ShmPortArgs alloc {name}={port}") + logger.debug(f"ShmPortArgs alloc {name}={port}") return port allocated: List[int] = [] @@ -249,7 +249,7 @@ def _get_or_alloc(self, name: str, count: int = 1) -> PortValue: allocated.append(port) ports[name] = allocated self._save(ports) - logger.info(f"ShmPortArgs alloc {name}={allocated}") + logger.debug(f"ShmPortArgs alloc {name}={allocated}") return allocated @staticmethod diff --git a/lightllm/utils/start_utils.py b/lightllm/utils/start_utils.py index 3d764d7789..5b076c7c64 100644 --- a/lightllm/utils/start_utils.py +++ b/lightllm/utils/start_utils.py @@ -4,6 +4,7 @@ import subprocess import sys import time +import threading import multiprocessing as mp import psutil from lightllm.utils.log_utils import init_logger @@ -14,10 +15,90 @@ logger = init_logger(__name__) +class ServiceReadiness: + """Log transitions only; steady probes never produce log traffic.""" + + def __init__(self, flag, args, address, started): + self.flag, self.args, self.address, self.started = flag, args, address, started + self.ready = False + self.ever_ready = False + self.waiting_logged = False + + def observe(self, status): + if status == 200: + if not self.ready: + event = "service.recovered" if self.ever_ready else "service.ready" + elapsed = round(time.monotonic() - self.started, 2) + duration_key = "uptime_s" if self.ever_ready else "startup_s" + fields = { + "event": event, + "run_mode": self.args.run_mode, + "model": self.args.model_name, + "address": self.address, + duration_key: elapsed, + } + logger.info( + "event=%s role=%s model=%s address=%s %s=%.2f", + event, + self.args.run_mode, + self.args.model_name, + self.address, + duration_key, + elapsed, + extra=fields, + ) + self.flag.set_value(1) + self.ready = self.ever_ready = True + else: + if self.ready: + self.flag.set_value(0) + self.ready = False + logger.warning( + "event=service.unavailable reason=readiness_probe_failed status=%s", + status, + extra={"event": "service.unavailable", "reason": "readiness_probe_failed", "http_status": status}, + ) + elif status == 503 and not self.waiting_logged and not self.ever_ready: + logger.info( + "event=service.waiting reason=pd_nodes_not_ready", + extra={"event": "service.waiting", "reason": "pd_nodes_not_ready"}, + ) + self.waiting_logged = True + + class SubmoduleManager: def __init__(self): self.processes = [] self.process_names = {} + self.readiness_stop = threading.Event() + + def start_readiness_monitor(self, http_server_process, args): + """Announce readiness after the bound HTTP endpoint and PD topology agree.""" + import requests + from lightllm.server.router.dynamic_prompt.shared_arr import SharedInt + from lightllm.utils.shm_port_args import get_shm_port_args + + ready_flag = SharedInt("service_ready") + ready_flag.set_value(0) + host = {"0.0.0.0": "127.0.0.1", "::": "::1"}.get(args.host, args.host) + host = f"[{host}]" if ":" in host else host + address = f"http://{host}:{get_shm_port_args().port}" + started = float(os.getenv("LIGHTLLM_SERVICE_START_TIME", str(time.monotonic()))) + self.readiness_stop.clear() + + def monitor(): + state = ServiceReadiness(ready_flag, args, address, started) + with requests.Session() as session: + session.trust_env = False + while not self.readiness_stop.is_set() and http_server_process.poll() is None: + try: + response = session.get(address + "/readiness", timeout=1) + state.observe(response.status_code) + except requests.RequestException: + state.observe(None) + self.readiness_stop.wait(0.5) + + threading.Thread(target=monitor, name="service-readiness", daemon=True).start() def start_submodule_processes(self, start_funcs=[], start_args=[]): assert len(start_funcs) == len(start_args) @@ -49,7 +130,7 @@ def start_submodule_processes(self, start_funcs=[], start_args=[]): proc.kill() sys.exit(1) else: - logger.info(f"init func {start_funcs[index].__name__} : {str(init_state)}") + logger.debug("Worker initialized: component=%s", start_funcs[index].__name__) assert all([proc.is_alive() for proc in processes]) return managed_processes @@ -80,6 +161,7 @@ def register_process_tree(self, root_process): def terminate_all_processes(self): from lightllm.utils.envs_utils import get_env_start_args + self.readiness_stop.set() for proc in self.processes: if proc.is_running(): kill_recursive(proc) @@ -98,7 +180,7 @@ def terminate_all_processes(self): stop_mps() if not alive_pids: - logger.info("All processes terminated gracefully.") + logger.info("event=service.stopped cleanup=complete") def setup_exit_controller(self): """启动 launcher 的独立资源清理进程。 @@ -120,13 +202,13 @@ def setup_signal_handlers(self, http_server_process=None): """在子进程启动完成后安装退出信号处理函数，覆盖启动阶段的处理函数。""" def signal_handler(sig, _frame): + self.readiness_stop.set() if sig == signal.SIGINT: logger.info("Received SIGINT (Ctrl+C), forcing immediate exit...") if http_server_process is not None: kill_recursive(http_server_process) self.terminate_all_processes() - logger.info("All processes have been forcefully terminated.") sys.exit(0) if sig == signal.SIGTERM: @@ -144,16 +226,15 @@ def signal_handler(sig, _frame): kill_recursive(http_server_process) self.terminate_all_processes() - logger.info("All processes have been terminated gracefully.") sys.exit(0) signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGHUP, signal_handler) - logger.info(f"start process pid {os.getpid()}") + logger.debug("Launcher pid=%s", os.getpid()) if http_server_process is not None: - logger.info(f"http server pid {http_server_process.pid}") + logger.debug("HTTP server pid=%s", http_server_process.pid) def supervise_processes(self, http_server_process=None): """Watch the HTTP server, when present, and all registered submodules. @@ -196,6 +277,7 @@ def supervise_processes(self, http_server_process=None): def _cleanup_after_process_failure(self, http_server_process): """Best-effort cleanup before the launcher exits with a failure.""" + self.readiness_stop.set() if http_server_process is not None and http_server_process.poll() is None: try: kill_recursive(http_server_process) @@ -231,7 +313,7 @@ def start_submodule_processes(start_funcs=[], start_args=[]): proc.kill() sys.exit(1) else: - logger.info(f"init func {start_funcs[index].__name__} : {str(init_state)}") + logger.debug("Worker initialized: component=%s", start_funcs[index].__name__) assert all([proc.is_alive() for proc in processes]) return @@ -242,16 +324,16 @@ def kill_recursive(proc): parent = psutil.Process(proc.pid) children = parent.children(recursive=True) except psutil.NoSuchProcess: - logger.warning(f"Process {proc.pid} does not exist.") + logger.debug(f"Process {proc.pid} does not exist.") return for process in [*reversed(children), parent]: try: - logger.info(f"Killing process {process.pid}") + logger.debug(f"Killing process {process.pid}") process.kill() except psutil.NoSuchProcess: # A concurrent exit must not skip the remaining children or parent. - logger.warning(f"Process {process.pid} does not exist.") + logger.debug(f"Process {process.pid} does not exist.") process_manager = SubmoduleManager() diff --git a/lightllm/utils/statics_utils.py b/lightllm/utils/statics_utils.py index ba61e9e8f1..02fd574698 100644 --- a/lightllm/utils/statics_utils.py +++ b/lightllm/utils/statics_utils.py @@ -8,6 +8,7 @@ class MovingAverage: def __init__(self): self.total = 0.0 self.count = 0 + self._last_logged_count = 0 self.last_time = time.time() def add(self, value): @@ -20,6 +21,7 @@ def average(self): return self.total / self.count def print_log(self, log_str): - if time.time() - self.last_time >= 30: - logger.info(f"{log_str}: {self.average()} ms") + if self.count > self._last_logged_count and time.time() - self.last_time >= 30: + logger.debug("event=request.latency kind=%r samples=%s mean_ms=%.3f", log_str, self.count, self.average()) + self._last_logged_count = self.count self.last_time = time.time() diff --git a/lightllm/utils/vllm_utils.py b/lightllm/utils/vllm_utils.py index 34ac2a4ab3..806b3d8c18 100644 --- a/lightllm/utils/vllm_utils.py +++ b/lightllm/utils/vllm_utils.py @@ -15,7 +15,7 @@ HAS_VLLM = False cutlass_scaled_mm = None vllm_ops = None - logger.warning( + logger.debug( "vllm is not installed, you can't use the api of it. \ You can solve it by running `pip install vllm`." ) diff --git a/unit_tests/common/basemodel/attention/fa3/test_neo.py b/unit_tests/common/basemodel/attention/fa3/test_neo.py index 6a06a1565c..0ec1cb6c6a 100644 --- a/unit_tests/common/basemodel/attention/fa3/test_neo.py +++ b/unit_tests/common/basemodel/attention/fa3/test_neo.py @@ -1,5 +1,6 @@ import importlib.util import sys +from unittest.mock import Mock from types import ModuleType, SimpleNamespace import pytest @@ -35,7 +36,7 @@ def neo_attention(q, image_token_end=None): @pytest.mark.parametrize("interface_kind", ["missing", "regular", "legacy_tag"]) -def test_unavailable_neo_interface_provides_source_install_instructions(monkeypatch, interface_kind): +def test_unavailable_neo_import_is_silent_and_explicit_use_explains_installation(monkeypatch, interface_kind): def regular_attention(q): return q @@ -43,12 +44,17 @@ def legacy_attention(q, image_token_tag=None): return q attention_func = {"missing": None, "regular": regular_attention, "legacy_tag": legacy_attention}[interface_kind] - with pytest.warns(UserWarning, match="Neo FA3 is unavailable") as warnings: - module = _load_neo(monkeypatch, attention_func) + logger = Mock() + monkeypatch.setattr("lightllm.utils.log_utils.init_logger", lambda name: logger) + module = _load_neo(monkeypatch, attention_func) assert not module.HAS_FLASH_ATTN_INTERFACE assert module.flash_attn_with_kvcache_neo is None - message = str(warnings[0].message) + assert not logger.mock_calls + with pytest.raises(ImportError) as error: + module.require_neo_fa3() + message = str(error.value) + assert "image_token_end support" in message assert "https://github.com/WANDY666/flash-attention.git" in message assert "cd flash-attention/hopper" in message assert "FLASH_ATTENTION_FORCE_BUILD=TRUE" in message @@ -117,3 +123,43 @@ def test_prefill_missing_extension_reports_install_instructions(monkeypatch): state = neo.NeoFa3PrefillAttState() with pytest.raises(ImportError, match="WANDY666/flash-attention"): state.prefill_att(None, None, None) + + +@pytest.mark.parametrize("selection", ["triton", "auto", "fa3"]) +def test_neo_selection_only_explains_missing_extension_when_requested(monkeypatch, selection): + from lightllm.common.basemodel.attention import create_utils + + monkeypatch.setattr(neo, "HAS_FLASH_ATTN_INTERFACE", False) + monkeypatch.setattr(create_utils, "HAS_FLASH_ATTN_INTERFACE", False) + monkeypatch.setattr( + create_utils, + "get_env_start_args", + lambda: SimpleNamespace( + llm_kv_type="None", + llm_prefill_att_backend=[selection], + ), + ) + monkeypatch.setattr( + create_utils, + "neo_data_type_to_backend", + { + "None": {"triton": create_utils.NeoTritonAttBackend}, + }, + ) + automatic = Mock(return_value=create_utils.NeoTritonAttBackend) + logger = Mock() + monkeypatch.setattr(create_utils, "_auto_select_backend", automatic) + monkeypatch.setattr(create_utils, "logger", logger) + if selection == "fa3": + with pytest.raises(ImportError, match="WANDY666/flash-attention"): + create_utils.get_neo_prefill_att_backend_class() + assert not logger.mock_calls + else: + assert create_utils.get_neo_prefill_att_backend_class() is create_utils.NeoTritonAttBackend + if selection == "auto": + automatic.assert_called_once() + assert logger.info.call_count + logger.debug.call_count == 1 + assert "backend.fallback" in (logger.info.call_args or logger.debug.call_args).args[0] + else: + assert not logger.mock_calls + automatic.assert_not_called() diff --git a/unit_tests/common/test_autotuner.py b/unit_tests/common/test_autotuner.py index 11f96245ad..d44712a9dd 100644 --- a/unit_tests/common/test_autotuner.py +++ b/unit_tests/common/test_autotuner.py @@ -425,3 +425,21 @@ def test_default_api_and_nested_phase_restore_after_exception(): assert not Autotuner.is_autotune_warmup() assert not Autotuner.is_kernel_autotune_warmup(AutotuneKernelType.GENERAL) assert not Autotuner.is_kernel_autotune_warmup(AutotuneKernelType.DECODE_ATTENTION) + + +def test_missing_configs_warn_once_and_keep_per_kernel_debug_details(tmp_path, monkeypatch): + from unittest.mock import Mock + + logger = Mock() + monkeypatch.setattr(autotuner_module, "logger", logger) + monkeypatch.setattr(Autotuner, "_defaults_warning_emitted", False) + monkeypatch.setattr(autotuner_module, "get_triton_autotune_level", lambda: AutotuneLevel.USE_AUTOTUNE_HIS_CONFIG) + first, _, _, _ = make_kernel(tmp_path, monkeypatch, "missing_first") + second, _, _, _ = make_kernel(tmp_path, monkeypatch, "missing_second") + first(8) + first(16) + second(8) + logger.warning.assert_called_once() + assert "event=kernel.defaults_used" in logger.warning.call_args.args[0] + assert logger.debug.call_count == 2 + assert {call.args[1] for call in logger.debug.call_args_list} == {"missing_first", "missing_second"} diff --git a/unit_tests/server/httpserver/test_pd_generate_error.py b/unit_tests/server/httpserver/test_pd_generate_error.py index cf0f4215ae..1379c892d8 100644 --- a/unit_tests/server/httpserver/test_pd_generate_error.py +++ b/unit_tests/server/httpserver/test_pd_generate_error.py @@ -431,3 +431,40 @@ async def stream(): response = TestClient(app).get("/") assert response.status_code == 400 assert "invalid image tag num" in response.text + + +def test_pd_request_context_isolated_and_reset_on_failure(): + from lightllm.utils.log_utils import get_http_request_id + + async def run(): + class ContextManager: + args = SimpleNamespace(run_mode="prefill") + + async def generate(self, **kwargs): + assert get_http_request_id() in ("client-a", "client-b") + expected = get_http_request_id() + await asyncio.sleep(0) + assert get_http_request_id() == expected + if expected == "client-b": + raise RuntimeError("test failure") + yield 123, "token", {}, FinishStatus(FinishStatus.FINISHED_STOP) + + async def generate(client_id): + params = SamplingParams() + params.group_request_id = 123 + await _pd_process_generate( + ContextManager(), + "prompt", + params, + None, + AsyncMock(), + AsyncMock(), + asyncio.Event(), + http_request_id=client_id, + ) + assert get_http_request_id() == "-" + + await asyncio.gather(generate("client-a"), generate("client-b")) + assert get_http_request_id() == "-" + + asyncio.run(run()) diff --git a/unit_tests/server/httpserver/test_pd_master_cached_tokens.py b/unit_tests/server/httpserver/test_pd_master_cached_tokens.py index 858ebe0794..e45d01e77a 100644 --- a/unit_tests/server/httpserver/test_pd_master_cached_tokens.py +++ b/unit_tests/server/httpserver/test_pd_master_cached_tokens.py @@ -8,6 +8,7 @@ from lightllm.server.core.objs import StopStrMatchMode, FinishStatus, SamplingParams from lightllm.server.router.model_infer.infer_batch import InferReq +from lightllm.server.httpserver_for_pd_master import manager as pd_master_module from lightllm.server.httpserver_for_pd_master.manager import HttpServerManagerForPDMaster from lightllm.server.httpserver_for_pd_master.pd_selector import PDSelectionExtraInfo @@ -23,6 +24,7 @@ def _make_manager(monkeypatch): mgr.req_id_to_out_inf = {} mgr.args = SimpleNamespace( disable_pd_master_decode_capacity_limit=True, + mtp_step=0, reasoning_parser=None, enable_rl=False, enable_return_routed_experts=False, @@ -333,7 +335,6 @@ async def run(): @pytest.mark.parametrize("in_reasoning", [False, True]) def test_pd_continuation_uses_local_reasoning_state(monkeypatch, in_reasoning): - manager = _make_manager(monkeypatch) manager.args.reasoning_parser = "qwen3" manager.tokenizer = SimpleNamespace( @@ -587,7 +588,7 @@ async def raw_segment(p_node, d_node, start_time, prompt, sp, multimodal_params, @pytest.mark.parametrize( "exit_mode, token_count", [("finish", 2), ("close", 1), ("error", 1), ("empty", 0), ("empty_prompt", 2)] ) -def test_pd_segment_summary_is_recorded_on_exit(monkeypatch, caplog, exit_mode, token_count): +def test_pd_segment_summary_is_recorded_on_exit(monkeypatch, exit_mode, token_count): import math import time from unittest.mock import MagicMock @@ -599,7 +600,8 @@ def test_pd_segment_summary_is_recorded_on_exit(monkeypatch, caplog, exit_mode, params = SamplingParams() params.init(SimpleNamespace(encode=lambda *_args, **_kwargs: []), max_new_tokens=4, group_request_id=80) request = SimpleNamespace(headers={"X-Request-Id": "summary-test"}, is_disconnected=lambda: asyncio.sleep(0, False)) - caplog.set_level("INFO", logger="lightllm.server.httpserver_for_pd_master.manager") + summary = MagicMock() + monkeypatch.setattr(pd_master_module, "log_request_summary", summary) async def raw_stream(*_args): if exit_mode == "empty": @@ -628,11 +630,14 @@ async def run(): assert len([result async for result in generator]) == token_count asyncio.run(run()) - summaries = [record.getMessage() for record in caplog.records if "out_token_counter:" in record.getMessage()] + summaries = summary.call_args_list if token_count: assert len(summaries) == 1 - assert "X-Request-Id:summary-test" in summaries[0] - assert f"out_token_counter:{token_count}" in summaries[0] + assert summaries[0].kwargs["x_request_id"] == "summary-test" + assert summaries[0].kwargs["output_tokens"] == token_count + assert summaries[0].kwargs["status"] == ( + "error" if exit_mode == "error" else "abort" if exit_mode == "close" else "completed" + ) manager.first_time_costs.add.assert_called_once() manager.per_token_costs.add.assert_called_once() assert math.isfinite(manager.per_token_costs.add.call_args.args[0]) @@ -649,7 +654,7 @@ async def run(): @pytest.mark.parametrize("late_status", [FinishStatus.NO_FINISH, FinishStatus.FINISHED_PD_DECODE_CAPACITY]) -def test_pd_stopped_choice_records_segment_summary(monkeypatch, caplog, late_status): +def test_pd_stopped_choice_records_segment_summary(monkeypatch, late_status): import time from unittest.mock import AsyncMock, MagicMock @@ -661,7 +666,8 @@ def test_pd_stopped_choice_records_segment_summary(monkeypatch, caplog, late_sta params = SamplingParams() params.init(SimpleNamespace(encode=lambda *_args, **_kwargs: []), max_new_tokens=4) request = SimpleNamespace(headers={}, is_disconnected=lambda: asyncio.sleep(0, False)) - caplog.set_level("INFO", logger="lightllm.server.httpserver_for_pd_master.manager") + summary = MagicMock() + monkeypatch.setattr(pd_master_module, "log_request_summary", summary) async def raw_stream(_p_node, _d_node, _prompt, sampling_params, *_args): metadata = {"id": 1, "prompt_tokens": 10, "prompt_cache_len": 3} @@ -680,9 +686,12 @@ async def run(): await generator.__anext__() assert not manager.active_req_ids manager.abort.assert_awaited_once() - summaries = [record.getMessage() for record in caplog.records if "out_token_counter:" in record.getMessage()] + summaries = summary.call_args_list assert len(summaries) == 1 - assert "lightllm_req_id:1" in summaries[0] + assert summaries[0].args[0] == 1 + assert summaries[0].kwargs["status"] == ( + "continued" if late_status == FinishStatus.FINISHED_PD_DECODE_CAPACITY else "abort" + ) manager.metric_client.histogram_observe.assert_any_call("lightllm_request_generated_tokens", 2) asyncio.run(run()) diff --git a/unit_tests/server/router/model_infer/mode_backend/test_pd_prefill_abort.py b/unit_tests/server/router/model_infer/mode_backend/test_pd_prefill_abort.py index bb732a0eba..cb250b0dc7 100644 --- a/unit_tests/server/router/model_infer/mode_backend/test_pd_prefill_abort.py +++ b/unit_tests/server/router/model_infer/mode_backend/test_pd_prefill_abort.py @@ -166,3 +166,74 @@ def test_prefill_abort_only_fails_tasks_not_submitted_to_transporter(): "active": active_task, "unrelated": unrelated_task, } + + +@pytest.mark.parametrize("telemetry", [False, True]) +@pytest.mark.parametrize("notify_fails", [False, True]) +def test_completed_transfer_reaches_recycling_queue_with_compact_logging(monkeypatch, telemetry, notify_fails): + task = SimpleNamespace( + xfer_handle=object(), + request_id=123, + error_info=None, + get_key=lambda: "transfer-123", + to_str=lambda: "request_id=123", + ) + module = _PrefillTransModule.__new__(_PrefillTransModule) + module.waiting_dict_lock = threading.Lock() + module.waiting_dict = {task.get_key(): task} + module.success_queue, module.failed_queue = queue.Queue(), queue.Queue() + module.transporter = MagicMock() + module.transporter.check_task_status.return_value = "DONE" + module.transporter.capture_telemetry = telemetry + module.transporter.nixl_agent.get_xfer_telemetry.return_value = SimpleNamespace( + totalBytes=4096, + xferDuration=20, + postDuration=2, + ) + logger = MagicMock() + logger.isEnabledFor.return_value = True + monkeypatch.setattr(prefill_trans_process, "logger", logger) + if notify_fails: + module.transporter.send_write_done_task_to_decode_node.side_effect = ConnectionError("peer gone") + module._update_task_status_once() + assert not module.waiting_dict + if notify_fails: + assert module.failed_queue.get_nowait() is task + assert module.success_queue.empty() + logger.exception.assert_called_once() + else: + assert module.success_queue.get_nowait() is task + assert module.failed_queue.empty() + if telemetry: + module.transporter.nixl_agent.get_xfer_telemetry.assert_called_once_with(task.xfer_handle) + assert logger.debug.call_args.kwargs["extra"]["bytes_total"] == 4096 + else: + module.transporter.nixl_agent.get_xfer_telemetry.assert_not_called() + + +@pytest.mark.parametrize("rank", [0, 1]) +def test_optional_cache_compaction_debug_is_rate_limited_and_tp_leader_only(monkeypatch, rank): + from lightllm.server.router.model_infer.mode_backend.base_backend import ModeBackend + from lightllm.utils import log_utils + + now = [0.0] + monkeypatch.setattr(log_utils.time, "monotonic", lambda: now[0]) + backend = ModeBackend.__new__(ModeBackend) + backend.rank_in_dp = rank + backend.logger = MagicMock() + backend.logger.isEnabledFor.return_value = True + backend._radix_tree_merge_counter = 0 + backend._enable_radix_tree_timer_merge = True + backend._radix_tree_merge_update_delta = 1 + backend.radix_cache = MagicMock() + for _ in range(2000): + backend._timer_merge_radix_tree() + assert backend.radix_cache.merge_unreferenced_nodes.call_count == 2000 + if rank == 0: + assert backend.logger.debug.call_count == 1 + now[0] = 10 + backend._debug_compactions.flush() + assert backend.logger.debug.call_count == 2 + assert sum(call.kwargs["extra"]["count"] for call in backend.logger.debug.call_args_list) == 2000 + else: + backend.logger.debug.assert_not_called() diff --git a/unit_tests/server/test_engine_throughput.py b/unit_tests/server/test_engine_throughput.py new file mode 100644 index 0000000000..dc0fe491e8 --- /dev/null +++ b/unit_tests/server/test_engine_throughput.py @@ -0,0 +1,195 @@ +"""Window accounting without GPU synchronization or request-completion spikes.""" + +from types import SimpleNamespace +from unittest.mock import Mock +import numpy as np +import pytest +import torch + +from lightllm.server import api_http # Initialize service imports before backend modules. +from lightllm.server.core.objs import FinishStatus +from lightllm.server.router import engine_stats +from lightllm.server.router.batch import Batch +from lightllm.server.router.model_infer import infer_batch +from lightllm.server.router.model_infer.mode_backend import base_backend +from lightllm.utils import log_utils + + +@pytest.fixture +def counters(monkeypatch): + arrays = {} + + class Array: + def __init__(self, name, shape, dtype): + self.name, self.shape, self.dtype = name, shape, dtype + + def create_shm(self): + self.arr = arrays.setdefault(self.name, np.zeros(self.shape, dtype=self.dtype)) + + def link_shm(self): + self.arr = arrays[self.name] + + monkeypatch.setattr(engine_stats, "ShmArray", Array) + return engine_stats.EngineCounters(2, owner=True) + + +def test_shared_dp_rows_are_aggregated_without_tp_multiplier(counters): + first = engine_stats.EngineCounters(2, 0) + second = engine_stats.EngineCounters(2, 1) + first.add(512, 4) + second.add(256, 2) + assert counters.totals() == (768, 6) + assert counters.storage.arr.tolist() == [[512, 4], [256, 2]] + + +def test_running_request_rates_and_completed_count_use_window_deltas(monkeypatch, counters): + now = [0.0] + monkeypatch.setattr(engine_stats.time, "monotonic", lambda: now[0]) + throughput = engine_stats.EngineThroughput(counters) + writer = engine_stats.EngineCounters(2, 0) + writer.add(512, 64) + now[0] = 10 + first = throughput.snapshot() + assert first["prefill_tok_s"] == 51.2 + assert first["output_tok_s"] == 6.4 + assert first["completed_req_s"] == 0 # Still running. + writer.add(128, 32) + throughput.finish(SimpleNamespace(is_aborted=False, finish_status=FinishStatus(FinishStatus.FINISHED_LENGTH))) + throughput.finish(SimpleNamespace(is_aborted=True, finish_status=FinishStatus(FinishStatus.FINISHED_ABORTED))) + now[0] = 20 + second = throughput.snapshot() + assert second["prefill_tokens"] == 128 and second["output_tokens"] == 32 + assert second["completed_reqs"] == 1 and second["completed_req_s"] == 0.1 + assert not throughput.has_activity() + + +def test_short_burst_after_idle_is_reported_once(monkeypatch, counters): + now = [0.0] + monkeypatch.setattr(engine_stats.time, "monotonic", lambda: now[0]) + monkeypatch.setattr(log_utils, "get_log_stats_interval", lambda: 10) + throughput = engine_stats.EngineThroughput(counters) + periodic = log_utils.PeriodicStatsLogger(Mock()) + assert not periodic.should_log(False) + now[0] = 10 + assert periodic.should_log(False) + throughput.snapshot() + now[0] = 11 + engine_stats.EngineCounters(2, 0).add(100, 10) + assert throughput.has_activity() + assert not periodic.should_log(throughput.has_activity()) + now[0] = 20 + assert periodic.should_log(throughput.has_activity()) + assert throughput.snapshot()["output_tokens"] == 10 + assert not periodic.should_log(False) + now[0] = 30 + assert periodic.should_log(False) + assert throughput.snapshot()["output_tokens"] == 0 + now[0] = 60 + assert not periodic.should_log(False) + + +def request(input_len, cached=0, output_len=0, max_output=100): + shm = SimpleNamespace(input_len=input_len, shm_cur_kv_len=cached, shm_cur_output_len=output_len) + status = FinishStatus() + req = SimpleNamespace( + cur_kv_len=cached, cur_output_len=output_len, shm_req=shm, finish_status=status, output_constraint=None + ) + req.get_cur_total_len = lambda: input_len + req.cur_output_len + req.get_chuncked_input_token_len = lambda: min(req.cur_kv_len + 512, req.get_cur_total_len()) + req.set_next_gen_token_id = Mock() + + def update(eos_ids, output_len): + if output_len >= max_output: + status.set_status(FinishStatus.FINISHED_LENGTH) + + req.update_finish_status = update + return req + + +def backend(monkeypatch, counters, leader=True): + result = base_backend.ModeBackend.__new__(base_backend.ModeBackend) + result.is_master_in_dp = leader + result.engine_counters = engine_stats.EngineCounters(2, 0) if leader else None + result.eos_id = [2] + monkeypatch.setattr( + base_backend, + "g_infer_context", + SimpleNamespace( + req_manager=SimpleNamespace(req_sampling_params_manager=Mock()), + ), + ) + return result + + +def post(result, reqs, packs): + count = len(reqs) + result._post_handle( + reqs, torch.zeros(count, dtype=torch.int64), torch.zeros(count), torch.zeros(count, dtype=torch.int64), packs + ) + + +def test_chunked_prefill_counts_computed_input_only_after_postprocessing(monkeypatch, counters): + result = backend(monkeypatch, counters) + req = request(1000, cached=200) + first = result._pre_post_handle([req], is_chuncked_mode=True) + assert first[0].prefill_tokens == 512 + assert counters.totals() == (0, 0) # No work reported before forward synchronization. + post(result, [req], first) + assert counters.totals() == (512, 0) + second = result._pre_post_handle([req], is_chuncked_mode=True) + post(result, [req], second) + assert counters.totals() == (800, 1) + + +def test_pd_received_kv_and_prefill_first_token_are_not_recounted(monkeypatch, counters): + result = backend(monkeypatch, counters) + req = request(1000, cached=1000, output_len=1) # KV and first token received from P. + packs = result._pre_post_handle([req], is_chuncked_mode=False) + post(result, [req], packs) + assert counters.totals() == (0, 1) + + +def test_mtp_counts_accepted_tokens_and_ignores_tokens_after_finish(monkeypatch, counters): + result = backend(monkeypatch, counters) + req = request(10, cached=10, output_len=1, max_output=3) + packs = [infer_batch.InferReqUpdatePack(req, output_len=length) for length in [2, 3, 4]] + post(result, [req] * 3, packs) + assert counters.totals() == (0, 2) + + +def test_nonleader_does_not_write_engine_counters(monkeypatch, counters): + result = backend(monkeypatch, counters, leader=False) + req = request(10) + post(result, [req], result._pre_post_handle([req], is_chuncked_mode=False)) + assert counters.totals() == (0, 0) + + +def test_batch_records_success_once_before_releasing_request(monkeypatch, counters): + throughput = engine_stats.EngineThroughput(counters) + req = SimpleNamespace( + request_id=42, + shm_infer_released=True, + is_aborted=False, + candetoken_out_len=10, + finish_status=FinishStatus(FinishStatus.FINISHED_STOP), + ) + batch = Batch(1, [req], 1) + shm_manager, statics = Mock(), Mock() + batch.filter_out_finished_req(shm_manager, statics, throughput) + batch.filter_out_finished_req(shm_manager, statics, throughput) + assert throughput.completed == 1 + shm_manager.put_back_req_obj.assert_called_once_with(req) + + +def test_resume_excludes_idle_time_without_discarding_short_burst(monkeypatch, counters): + now = [0.0] + monkeypatch.setattr(engine_stats.time, "monotonic", lambda: now[0]) + throughput = engine_stats.EngineThroughput(counters) + now[0] = 1000 + engine_stats.EngineCounters(2, 0).add(40, 20) + throughput.resume(since=999.99) + now[0] = 1009.99 + snapshot = throughput.snapshot() + assert snapshot["window_s"] == 10 + assert snapshot["output_tokens"] == 20 + assert snapshot["output_tok_s"] == 2 diff --git a/unit_tests/server/test_logging.py b/unit_tests/server/test_logging.py new file mode 100644 index 0000000000..0a9c284139 --- /dev/null +++ b/unit_tests/server/test_logging.py @@ -0,0 +1,146 @@ +"""Exercise actual HTTP logging and health-monitor boundaries without starting workers.""" + +import asyncio +from unittest.mock import Mock + +import pytest + +from lightllm.server import api_http +from lightllm.server.health_monitor import manager as health + + +@pytest.mark.parametrize("enabled", [False, True]) +def test_access_middleware_preserves_response_and_respects_switch(monkeypatch, enabled): + monkeypatch.setenv("LIGHTLLM_LOG_HTTP_ACCESS", "1" if enabled else "0") + access = Mock() + monkeypatch.setattr(api_http, "log_http_access", access) + messages = [] + + async def app(scope, receive, send): + await send({"type": "http.response.start", "status": 201, "headers": []}) + await send({"type": "http.response.body", "body": b"ok"}) + + async def send(message): + messages.append(message) + + asyncio.run(api_http._AccessLogMiddleware(app)({"type": "http", "method": "POST", "path": "/generate"}, None, send)) + assert messages[0]["status"] == 201 + assert messages[1]["body"] == b"ok" + assert access.call_args.args[:3] == ("POST", "/generate", 201) + assert access.call_args.args[3] >= 0 + request_id = access.call_args.kwargs["http_request_id"] + assert request_id + assert dict(messages[0]["headers"])[b"x-request-id"] == request_id.encode("ascii") + + +def test_access_middleware_does_not_swallow_failure(monkeypatch): + monkeypatch.setenv("LIGHTLLM_LOG_HTTP_ACCESS", "1") + access = Mock() + monkeypatch.setattr(api_http, "log_http_access", access) + + async def app(scope, receive, send): + raise RuntimeError("request failed") + + with pytest.raises(RuntimeError, match="request failed"): + asyncio.run(api_http._AccessLogMiddleware(app)({"type": "http", "method": "GET", "path": "/fail"}, None, None)) + assert access.call_args.args[:3] == ("GET", "/fail", 500) + + +def test_healthy_checks_are_silent_and_recovery_is_logged_once(monkeypatch): + logger = Mock() + monkeypatch.setattr(health, "logger", logger) + monkeypatch.setattr(health, "consecutive_failures", 0) + monkeypatch.setattr(health, "all_is_alive", lambda ids: True) + monkeypatch.setattr(health.requests, "get", lambda *a, **k: Mock(status_code=200)) + health.health_monitor("http://unused/health", []) + logger.info.assert_not_called() + health.consecutive_failures = 2 + health.health_monitor("http://unused/health", []) + assert logger.info.call_count == 1 + assert "health.recovered" in logger.info.call_args.args[0] + assert health.consecutive_failures == 0 + health.health_monitor("http://unused/health", []) + assert logger.info.call_count == 1 + + +def test_health_failures_warn_until_termination(monkeypatch): + logger = Mock() + kill = Mock() + monkeypatch.setattr(health, "logger", logger) + monkeypatch.setattr(health, "consecutive_failures", 0) + monkeypatch.setattr(health, "all_is_alive", lambda ids: True) + monkeypatch.setattr(health.requests, "get", lambda *a, **k: Mock(status_code=500)) + monkeypatch.setattr(health.os, "kill", kill) + health.health_monitor("http://unused/health", [100, 101]) + health.health_monitor("http://unused/health", [100, 101]) + logger.critical.assert_not_called() + kill.assert_not_called() + with pytest.raises(SystemExit): + health.health_monitor("http://unused/health", [100, 101]) + assert logger.warning.call_count == 3 + assert logger.critical.call_count == 1 + assert kill.call_count == 2 + + +@pytest.mark.parametrize("enabled", [False, True]) +def test_concurrent_request_ids_propagate_and_context_is_reset(monkeypatch, enabled): + from lightllm.utils.log_utils import get_http_request_id + + monkeypatch.setenv("LIGHTLLM_LOG_HTTP_ACCESS", "1" if enabled else "0") + access = Mock() + monkeypatch.setattr(api_http, "log_http_access", access) + observed = {} + + async def app(scope, receive, send): + request_id = dict(scope["headers"])[b"x-request-id"].decode() + await asyncio.sleep(0.001) + assert get_http_request_id() == request_id + observed[scope["path"]] = request_id + await send({"type": "http.response.start", "status": 200, "headers": []}) + await send({"type": "http.response.body", "body": b"ok"}) + + async def send(message): + pass + + async def run(): + middleware = api_http._AccessLogMiddleware(app) + await asyncio.gather( + middleware( + {"type": "http", "method": "POST", "path": "/one", "headers": [(b"x-request-id", b"client-123")]}, + None, + send, + ), + middleware({"type": "http", "method": "POST", "path": "/two", "headers": []}, None, send), + ) + assert get_http_request_id() == "-" + + asyncio.run(run()) + assert observed["/one"] == "client-123" + assert observed["/two"] != "client-123" + assert {call.kwargs["http_request_id"] for call in access.call_args_list} == set(observed.values()) + + +def test_request_id_context_reset_on_error(monkeypatch): + from lightllm.utils.log_utils import get_http_request_id + + monkeypatch.setattr(api_http, "log_http_access", Mock()) + + async def app(scope, receive, send): + assert get_http_request_id() != "-" + raise RuntimeError("failure") + + async def run(): + with pytest.raises(RuntimeError): + await api_http._AccessLogMiddleware(app)({"type": "http", "method": "POST", "path": "/fail"}, None, None) + assert get_http_request_id() == "-" + + asyncio.run(run()) + + +@pytest.mark.parametrize("value", [b"", b"bad\r\nid", b"x" * 129, b"\xff"]) +def test_unsafe_client_request_ids_are_replaced(value): + from lightllm.utils.log_utils import new_http_request_id + + request_id = new_http_request_id([(b"x-request-id", value)]) + assert len(request_id) == 32 + int(request_id, 16) diff --git a/unit_tests/utils/test_debug_logging.py b/unit_tests/utils/test_debug_logging.py new file mode 100644 index 0000000000..472ea6eee1 --- /dev/null +++ b/unit_tests/utils/test_debug_logging.py @@ -0,0 +1,173 @@ +"""Verify debug volume bounds, aggregate accounting and exception evidence.""" + +from collections import OrderedDict +from concurrent.futures import ThreadPoolExecutor +import logging + +from lightllm.utils import log_utils + + +def recording_logger(): + records = [] + + class Handler(logging.Handler): + def emit(self, record): + records.append(record) + + logger = logging.Logger("debug-volume-test", logging.DEBUG) + logger.addHandler(Handler()) + return logger, records + + +def test_debug_fields_do_not_expand_payloads_or_objects(): + logger, records = recording_logger() + + class Descriptor: + def __repr__(self): + raise AssertionError("Must not expand descriptors") + + recursive = [] + recursive.append(recursive) + log_utils.log_debug_event( + logger, + "bounded", + text="x" * 100000, + token_ids=list(range(100000)), + descriptor=Descriptor(), + metadata=b"x" * 100000, + nested=recursive, + ) + record = records[0] + assert len(record.getMessage()) < 1000 + assert record.token_ids["count"] == 100000 + assert len(record.token_ids["sample"]) == 8 + assert record.descriptor == {"type": "Descriptor"} + assert record.metadata == {"type": "bytes", "bytes": 100000} + assert record.nested == {"count": 1, "sample": [{"count": 1}]} + + +def test_debug_event_shared_tp_state_is_only_reported_by_leader(monkeypatch): + logger, records = recording_logger() + for rank in range(8): + monkeypatch.setattr(log_utils, "_context", {"tp_rank": rank}) + log_utils.log_debug_event(logger, "cache.idle", leader_only=True, cached_tokens=32) + assert len(records) == 1 + + +def test_debug_stats_count_pages_without_per_page_lines(monkeypatch): + now = [0.0] + monkeypatch.setattr(log_utils.time, "monotonic", lambda: now[0]) + logger, records = recording_logger() + stats = log_utils.DebugStatsLogger(logger, "kv.transfer", interval=10) + for index in range(10000): + stats.add(request_id=index, pages=1, bytes_total=4096, kv_tokens_total=16) + assert len(records) == 1 + now[0] = 10 + stats.flush() + assert len(records) == 2 + assert records[1].request_ids_truncated + assert len(records[1].request_id_sample["sample"]) == 8 + assert sum(record.pages for record in records) == 10000 + assert sum(record.bytes_total for record in records) == 10000 * 4096 + assert sum(record.kv_tokens_total for record in records) == 160000 + now[0] = 100 + stats.flush() + assert len(records) == 2 # No idle heartbeat. + + +def test_debug_stats_are_thread_safe_and_do_nothing_when_disabled(monkeypatch): + now = [0.0] + monkeypatch.setattr(log_utils.time, "monotonic", lambda: now[0]) + logger, records = recording_logger() + stats = log_utils.DebugStatsLogger(logger, "kv.transfer", interval=10) + + def add(_): + for _ in range(1000): + stats.add(pages=1) + + with ThreadPoolExecutor(max_workers=4) as pool: + list(pool.map(add, range(4))) + now[0] = 10 + stats.flush() + assert sum(record.pages for record in records) == 4000 + logger.disabled = True + stats.add(unused=1) + assert stats.count == 0 and stats.totals == {} + + +def test_debug_stats_reset_inherited_lock_after_fork(monkeypatch): + logger, records = recording_logger() + stats = log_utils.DebugStatsLogger(logger, "kv.transfer", interval=10) + stats.add(pages=1) + stats.add(pages=5) + stats.lock.acquire() + monkeypatch.setattr(stats, "pid", -1) + stats.add(pages=2) + assert records[-1].pages == 2 # No parent counts or inherited locked mutex. + + +def test_repeated_connection_failures_keep_first_traceback_and_distinct_reasons(monkeypatch): + now = [0.0] + monkeypatch.setattr(log_utils.time, "monotonic", lambda: now[0]) + monkeypatch.setattr(log_utils, "_exception_windows", OrderedDict()) + logger, records = recording_logger() + + def fail(reason): + try: + raise ConnectionError(reason) + except ConnectionError: + log_utils.log_repeated_exception(logger, "pd.connection_failed", master="localhost:1") + + for _ in range(100): + fail("connection refused") + assert len(records) == 1 and records[0].exc_info + now[0] = 10 + fail("connection refused") + assert len(records) == 2 and not records[1].exc_info + assert records[1].repeated == 100 + fail("connection reset") + assert len(records) == 3 and records[2].exc_info + for index in range(200): + fail(f"different reason {index}") + assert len(log_utils._exception_windows) == 128 + + +def test_latency_summaries_do_not_repeat_while_idle(monkeypatch): + from lightllm.utils import statics_utils + + now = [0.0] + monkeypatch.setattr(statics_utils.time, "time", lambda: now[0]) + logger, records = recording_logger() + monkeypatch.setattr(statics_utils, "logger", logger) + average = statics_utils.MovingAverage() + now[0] = 30 + average.print_log("ttft") + assert not records + average.add(10) + average.print_log("ttft") + assert len(records) == 1 + now[0] = 60 + average.print_log("ttft") + assert len(records) == 1 + average.add(30) + average.print_log("ttft") + assert len(records) == 2 + assert average.average() == 20 + + +def test_debug_numpy_scalars_retain_ids_and_counters(): + import numpy as np + + logger, records = recording_logger() + log_utils.log_debug_event(logger, "request.admitted", request_id=np.int64(42), ratio=np.float32(0.25)) + assert records[0].request_id == 42 + assert records[0].ratio == 0.25 + assert "request_id=42" in records[0].getMessage() + + +def test_debug_aggregate_points_to_business_caller(): + logger, records = recording_logger() + stats = log_utils.DebugStatsLogger(logger, "test.aggregate") + stats.add(request_id=42, pages=1) + assert records[0].pathname == __file__ + assert records[0].funcName == "test_debug_aggregate_points_to_business_caller" diff --git a/unit_tests/utils/test_log_utils.py b/unit_tests/utils/test_log_utils.py index 3534656835..fd7133d5c5 100644 --- a/unit_tests/utils/test_log_utils.py +++ b/unit_tests/utils/test_log_utils.py @@ -40,3 +40,533 @@ def test_log_node_role_marker_is_applied_to_every_line(monkeypatch): log_utils.set_log_node_role("prefill") assert _format_message("first\nsecond") == "[P] first\r\n[P] second" + + +# Configuration tests run in fresh interpreters: changing handlers in-process +# would interfere with pytest's capture and the rest of the test suite. +import json +import os +from pathlib import Path +import subprocess +import sys +import textwrap + + +def _run_logging(code, **settings): + env = {key: value for key, value in os.environ.items() if not key.startswith("LIGHTLLM_LOG_")} + env["LIGHTLLM_LOG_REQUEST_SAMPLE_RATE"] = "1" + env.update({key: str(value) for key, value in settings.items()}) + result = subprocess.run( + [sys.executable, "-c", textwrap.dedent(code)], + env=env, + text=True, + capture_output=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + return result.stdout + + +def test_default_info_and_single_handler_for_module_loggers(): + output = _run_logging( + """ + from lightllm.utils.log_utils import init_logger + logger = init_logger("lightllm.server.router.test") + assert not logger.handlers + assert logger is init_logger(logger.name) + logger.debug("hidden-debug") + logger.info("visible-info") + """ + ) + assert "hidden-debug" not in output + assert output.count("visible-info") == 1 + assert "pid=" not in output and "tp=" not in output + assert "[main]" in output + assert "test_log_utils.py" not in output + + +def test_module_override_survives_global_error_and_reconfiguration(): + output = _run_logging( + """ + from lightllm.utils.log_utils import init_logger, _setup_logger + selected = init_logger("lightllm.server.router.test") + other = init_logger("lightllm.server.httpserver.test") + _setup_logger() + _setup_logger() + selected.debug("router-debug") + other.info("hidden-info") + other.error("visible-error") + """, + LIGHTLLM_LOG_LEVEL="ERROR", + LIGHTLLM_LOG_MODULE_LEVELS='{"lightllm.server.router":"DEBUG"}', + ) + assert output.count("router-debug") == 1 + assert "hidden-info" not in output + assert output.count("visible-error") == 1 + + +def test_json_preserves_context_multiline_and_traceback(): + output = _run_logging( + """ + from lightllm.utils.log_utils import init_logger, set_log_node_role, set_log_context + set_log_node_role("decode") + set_log_context(component="model_infer", node_rank=2, tp_rank=3, dp_rank=1) + try: + raise ValueError("failure") + except ValueError: + init_logger("lightllm.test").exception("first\\nsecond", extra={"event":"test.failure", "request_id":42}) + """, + LIGHTLLM_LOG_FORMAT="json", + ) + lines = output.splitlines() + assert len(lines) == 1 + event = json.loads(lines[0]) + assert event["level"] == "ERROR" + assert event["message"] == "first\nsecond" + assert event["role"] == "D" + assert event["tp_rank"] == 3 and event["dp_rank"] == 1 + assert event["request_id"] == 42 + assert "ValueError: failure" in event["exception"] + assert event["timestamp"].endswith(("+00:00", "+08:00")) or "T" in event["timestamp"] + + +@pytest.mark.parametrize( + "settings", + [ + {"LIGHTLLM_LOG_LEVEL": "typo"}, + {"LIGHTLLM_LOG_FORMAT": "xml"}, + {"LIGHTLLM_LOG_MODULE_LEVELS": "[]"}, + {"LIGHTLLM_LOG_MODULE_LEVELS": '{"root":"DEBUG"}'}, + {"LIGHTLLM_LOG_MODULE_LEVELS": '{"lightllm.test":"bad"}'}, + {"LIGHTLLM_LOG_REQUEST_SAMPLE_RATE": "-0.1"}, + {"LIGHTLLM_LOG_REQUEST_SAMPLE_RATE": "nan"}, + {"LIGHTLLM_LOG_REQUEST_SAMPLE_RATE": "1.1"}, + {"LIGHTLLM_LOG_STATS_INTERVAL": "0"}, + {"LIGHTLLM_LOG_STATS_INTERVAL": "inf"}, + ], +) +def test_invalid_configuration_fails_explicitly(settings): + env = {key: value for key, value in os.environ.items() if not key.startswith("LIGHTLLM_LOG_")} + env.update(settings) + result = subprocess.run( + [sys.executable, "-c", "from lightllm.utils import log_utils"], + env=env, + text=True, + capture_output=True, + timeout=30, + ) + assert result.returncode != 0 + assert "ValueError" in result.stderr + + +def test_cli_overrides_environment_and_inherits_unspecified_values(): + output = _run_logging( + """ + import argparse + import os + from lightllm.server.api_cli import add_cli_args + from lightllm.utils.log_utils import configure_logging_from_args, init_logger + args = add_cli_args(argparse.ArgumentParser()).parse_args(["--log-level", "debug", "--run_mode", "prefill"]) + configure_logging_from_args(args) + assert os.environ["LIGHTLLM_LOG_LEVEL"] == "debug" + assert os.environ["LIGHTLLM_LOG_FORMAT"] == "json" + assert os.environ["LIGHTLLM_LOG_NODE_ROLE"] == "P" + init_logger("lightllm.test").debug("cli-debug") + """, + LIGHTLLM_LOG_LEVEL="ERROR", + LIGHTLLM_LOG_FORMAT="json", + ) + assert json.loads(output)["message"] == "cli-debug" + + +def test_request_and_access_categories_are_independent_and_exclude_probes(): + output = _run_logging( + """ + from lightllm.utils.log_utils import init_logger, log_request_summary, log_http_access, get_http_logging_args + init_logger("lightllm.server.router").info("hidden-service-info") + log_request_summary(12, status="completed", output_tokens=4) + log_http_access("GET", "/health", 200, 1) + log_http_access("GET", "/metrics", 200, 1) + log_http_access("POST", "/v1/chat/completions", 200, 3) + assert "--access-logfile" not in get_http_logging_args(application_access=True) + assert "--access-logfile" in get_http_logging_args() + """, + LIGHTLLM_LOG_LEVEL="ERROR", + LIGHTLLM_LOG_FORMAT="json", + LIGHTLLM_LOG_REQUEST_SAMPLE_RATE=1, + LIGHTLLM_LOG_HTTP_ACCESS=1, + ) + events = [json.loads(line) for line in output.splitlines()] + assert [event["event"] for event in events] == ["request.completed", "http.access"] + assert events[0]["request_id"] == 12 + assert events[1]["path"] == "/v1/chat/completions" + + +def test_request_logging_can_be_disabled_and_access_is_disabled_by_default(): + assert ( + _run_logging( + """ + from lightllm.utils.log_utils import log_request_summary, log_http_access, get_http_logging_args + log_request_summary(12, status="completed") + log_http_access("POST", "/generate", 200, 1) + assert "--access-logfile" not in get_http_logging_args() + """, + LIGHTLLM_LOG_REQUEST_SAMPLE_RATE=0, + ) + == "" + ) + + +def test_sampling_is_stable_and_does_not_change_random_state(monkeypatch): + import random + + monkeypatch.setattr(log_utils, "_request_sample_rate", 0.5) + state = random.getstate() + selected = [log_utils.should_log_request(i) for i in range(100)] + assert selected == [log_utils.should_log_request(i) for i in range(100)] + assert 20 < sum(selected) < 80 + assert random.getstate() == state + + +def test_periodic_stats_idle_resume_and_interval(monkeypatch): + from unittest.mock import Mock + + clock = [0.0] + monkeypatch.setattr(log_utils.time, "monotonic", lambda: clock[0]) + monkeypatch.setattr(log_utils, "_stats_interval", 10) + logger = Mock() + stats = log_utils.PeriodicStatsLogger(logger) + assert not stats.should_log(False) + clock[0] = 10 + assert stats.should_log(False) + stats.log(running=0) + assert logger.info.call_args.kwargs["extra"]["event"] == "engine.idle" + clock[0] = 11 + assert not stats.should_log(True) + clock[0] = 20 + assert stats.should_log(True) + stats.log(running=2) + assert logger.info.call_args.kwargs["extra"]["event"] == "engine.stats" + clock[0] = 21 + assert not stats.should_log(False) + clock[0] = 30 + assert not stats.should_log(False) + clock[0] = 31 + assert stats.should_log(False) + clock[0] = 100 + assert not stats.should_log(False) + + +def test_periodic_stats_flapping_never_bypasses_interval(monkeypatch): + from unittest.mock import Mock + + clock = [0.0] + monkeypatch.setattr(log_utils.time, "monotonic", lambda: clock[0]) + monkeypatch.setattr(log_utils, "_stats_interval", 10) + stats = log_utils.PeriodicStatsLogger(Mock()) + emitted = [] + for tick in range(1000): + clock[0] = tick / 10 + if stats.should_log(tick % 2 == 0): + emitted.append(clock[0]) + assert emitted + assert len(emitted) <= 10 + assert all(right - left >= 10 for left, right in zip(emitted, emitted[1:])) + + +def test_peer_configuration_is_debug_but_peer_errors_are_preserved(): + output = _run_logging( + """ + from lightllm.utils.log_utils import init_logger, set_log_context, log_info_on_leader + logger = init_logger("lightllm.test") + set_log_context(tp_rank=0) + log_info_on_leader(logger, "shared-config") + set_log_context(tp_rank=1) + log_info_on_leader(logger, "shared-config") + logger.error("peer-failure") + """ + ) + assert output.count("shared-config") == 1 + assert output.count("peer-failure") == 1 + + +@pytest.mark.parametrize("method", ["fork", "spawn"]) +def test_process_files_and_context_are_isolated(tmp_path, method): + script = tmp_path / "logging_worker.py" + script.write_text( + """ +import multiprocessing +from lightllm.utils.log_utils import init_logger, set_log_node_role, set_log_context +logger = init_logger("lightllm.worker") +def worker(): + logger.info("child-event") +if __name__ == "__main__": + set_log_node_role("prefill") + set_log_context(component="parent", tp_rank=7) + logger.info("parent-event") + child = multiprocessing.get_context(METHOD).Process(target=worker) + child.start() + child.join(20) + assert child.exitcode == 0 +""".replace( + "METHOD", repr(method) + ) + ) + env = {key: value for key, value in os.environ.items() if not key.startswith("LIGHTLLM_LOG_")} + env.update(LIGHTLLM_LOG_FORMAT="json", LIGHTLLM_LOG_DIR=str(tmp_path / "logs")) + env["PYTHONPATH"] = str(Path(__file__).resolve().parents[2]) + os.pathsep + env.get("PYTHONPATH", "") + result = subprocess.run([sys.executable, str(script)], env=env, text=True, capture_output=True, timeout=40) + assert result.returncode == 0, result.stderr + files = sorted((tmp_path / "logs").glob("*.log")) + assert len(files) == 2 + events = [json.loads(path.read_text()) for path in files] + by_message = {event["message"]: event for event in events} + assert by_message["parent-event"]["tp_rank"] == 7 + assert by_message["child-event"]["tp_rank"] == "-" + assert by_message["child-event"]["component"] != "parent" + assert all(event["role"] == "P" for event in events) + assert all(path.name == f"process.{event['pid']}.log" for path, event in zip(files, events)) + + +def test_file_rotation_is_bounded(tmp_path): + _run_logging( + """ + from lightllm.utils.log_utils import init_logger + logger = init_logger("lightllm.test") + for i in range(30): + logger.info("rotation-entry-%d", i) + """, + LIGHTLLM_LOG_DIR=tmp_path, + LIGHTLLM_LOG_MAX_BYTES=300, + LIGHTLLM_LOG_BACKUP_COUNT=2, + ) + files = list(tmp_path.iterdir()) + assert len(files) == 3 + current = next(path for path in files if path.name.endswith(".log")) + assert "rotation-entry-29" in current.read_text() + + +def test_startup_summary_does_not_dump_secrets(): + output = _run_logging( + """ + from types import SimpleNamespace + from lightllm.utils.log_utils import init_logger, log_startup_config + log_startup_config(init_logger("lightllm.test"), SimpleNamespace( + run_mode="normal", tp=2, dp=1, api_key="never-log-this", model_dir="s3://secret@bucket")) + """, + LIGHTLLM_LOG_LEVEL="DEBUG", + ) + assert "tp=2" in output + assert "never-log-this" not in output and "secret@bucket" not in output + + +@pytest.mark.parametrize( + "tty, log_format, level, expected", + [ + (False, "text", logging.INFO, False), + (True, "json", logging.INFO, False), + (True, "text", logging.ERROR, False), + (True, "text", logging.INFO, True), + ], +) +def test_progress_only_enabled_for_interactive_text(monkeypatch, tty, log_format, level, expected): + monkeypatch.setattr(log_utils.sys.stderr, "isatty", lambda: tty) + monkeypatch.setenv("LIGHTLLM_LOG_FORMAT", log_format) + monkeypatch.setattr(log_utils._root_logger, "isEnabledFor", lambda severity: level <= severity) + assert log_utils.progress_enabled() is expected + + +def test_log_helpers_preserve_business_callsite(): + output = _run_logging( + """ + import inspect + import logging + from lightllm.utils.log_utils import log_request_summary, log_http_access, PeriodicStatsLogger + def business_call(): + line = inspect.currentframe().f_lineno + 1 + log_request_summary(11, status="completed", expected_line=line) + return line + expected = business_call() + log_http_access("POST", "/generate", 200, 1) + stats = PeriodicStatsLogger(logging.getLogger("lightllm.stats")) + stats.active = True + stats.log(running=1) + """, + LIGHTLLM_LOG_FORMAT="json", + LIGHTLLM_LOG_REQUEST_SAMPLE_RATE=1, + LIGHTLLM_LOG_HTTP_ACCESS=1, + ) + events = [json.loads(line) for line in output.splitlines()] + assert all(event["file"] == "" for event in events) + assert events[0]["line"] == events[0]["expected_line"] + assert events[0]["line"] < events[1]["line"] < events[2]["line"] + + +def test_python_warnings_use_central_output(): + output = _run_logging( + """ + import warnings + from lightllm.utils import log_utils + warnings.warn("dependency-warning", UserWarning) + """, + LIGHTLLM_LOG_FORMAT="json", + ) + event = json.loads(output) + assert event["level"] == "WARNING" + assert event["logger"] == "py.warnings" + assert "dependency-warning" in event["message"] + + +@pytest.mark.parametrize( + "mode, tp_size, dp_size, nnodes, ranks, expected", + [ + ("normal", 1, 1, 1, (0, 0, 0), "[infer]"), + ("prefill", 2, 1, 1, (0, 1, 0), "[P/infer tp=1]"), + ("decode", 1, 2, 1, (0, 0, 1), "[D/infer dp=1]"), + ("normal", 2, 2, 2, (1, 0, 1), "[infer node=1 tp=0 dp=1]"), + ("normal", 2, 2, 2, ("-", "-", "-"), "[infer]"), + ], +) +def test_compact_text_context_only_shows_applicable_ranks(monkeypatch, mode, tp_size, dp_size, nnodes, ranks, expected): + import re + + monkeypatch.setenv("LIGHTLLM_LOG_TP_SIZE", str(tp_size)) + monkeypatch.setenv("LIGHTLLM_LOG_DP_SIZE", str(dp_size)) + monkeypatch.setenv("LIGHTLLM_LOG_NNODES", str(nnodes)) + monkeypatch.setattr(log_utils, "_LOG_NODE_ROLE", {"prefill": "P", "decode": "D"}.get(mode, "")) + monkeypatch.setattr( + log_utils, + "_context", + {"component": "model_infer", "node_rank": ranks[0], "tp_rank": ranks[1], "dp_rank": ranks[2]}, + ) + monkeypatch.setattr(log_utils, "_context_pid", os.getpid()) + record = logging.LogRecord("lightllm.test", logging.INFO, __file__, 12, "ready", (), None) + output = log_utils.NewLineFormatter(log_utils._FORMAT).format(record) + assert re.fullmatch(r"\d{2}:\d{2}:\d{2}\.\d{3} INFO " + re.escape(expected) + " ready", output) + assert "pid=" not in output and "global=" not in output + + +@pytest.mark.parametrize( + "level, source", [(logging.DEBUG, True), (logging.INFO, False), (logging.WARNING, False), (logging.ERROR, True)] +) +def test_compact_text_retains_source_for_debug_and_error(level, source): + record = logging.LogRecord("lightllm.test", level, "/example/business.py", 42, "event", (), None) + output = log_utils.NewLineFormatter(log_utils._FORMAT).format(record) + assert ("business.py:42" in output) is source + + +def test_compact_exception_prefixes_every_traceback_line(): + try: + raise ValueError("failed") + except ValueError: + record = logging.LogRecord( + "lightllm.test", logging.ERROR, "/example/business.py", 42, "operation failed", (), sys.exc_info() + ) + output = log_utils.NewLineFormatter(log_utils._FORMAT).format(record) + assert "ValueError: failed" in output + assert all("business.py:42" in line for line in output.splitlines()) + + +def test_pd_master_component_is_short(monkeypatch): + monkeypatch.setenv("LIGHTLLM_LOG_RUN_MODE", "pd_master") + monkeypatch.setattr(log_utils, "_context", {"component": "http"}) + monkeypatch.setattr(log_utils, "_context_pid", os.getpid()) + monkeypatch.setattr(log_utils, "_LOG_NODE_ROLE", "") + record = logging.LogRecord("lightllm.test", logging.INFO, __file__, 12, "ready", (), None) + assert "[master] ready" in log_utils.NewLineFormatter(log_utils._FORMAT).format(record) + + +def test_json_remains_complete_after_compact_text_formatting(monkeypatch): + monkeypatch.setattr(log_utils, "_context", {"component": "model_infer", "tp_rank": 1, "dp_rank": 0}) + monkeypatch.setattr(log_utils, "_context_pid", os.getpid()) + record = logging.LogRecord("lightllm.test", logging.INFO, "/example/business.py", 42, "ready", (), None) + log_utils.NewLineFormatter(log_utils._FORMAT).format(record) + result = json.loads(log_utils.JsonFormatter().format(record)) + assert "T" in result["timestamp"] + assert result["pid"] == os.getpid() + assert result["hostname"] + assert result["component"] == "model_infer" + assert result["tp_rank"] == 1 and result["dp_rank"] == 0 + assert result["file"] == "/example/business.py" and result["line"] == 42 + assert "compact_context" not in result and "source" not in result + + +def test_request_summary_defaults_and_optional_fields(): + output = _run_logging( + """ + from lightllm.utils.log_utils import log_request_summary + log_request_summary(1, status="completed", timing_scope="decode", gpu_cache_tokens=0, + cpu_cache_tokens=12, mtp_verify_steps=2, mtp_enabled=False) + log_request_summary(2, status="completed", mtp_verify_steps=2, mtp_enabled=True) + """, + LIGHTLLM_LOG_FORMAT="json", + LIGHTLLM_LOG_LEVEL="DEBUG", + ) + first, second = [json.loads(line) for line in output.splitlines()] + assert first["timing_scope"] == "decode" + assert "gpu_cache_tokens" not in first and "mtp_verify_steps" not in first + assert first["cpu_cache_tokens"] == 12 + assert second["mtp_verify_steps"] == 2 + + +def test_compiler_error_summary_preserves_root_cause(): + reason = log_utils.summarize_exception( + RuntimeError( + "Ninja build failed\n" + + "nvcc " + + "flags " * 1000 + + "\n/bin/sh: /usr/local/cuda-12.6/bin/nvcc: No such file or directory\nninja: build stopped." + ) + ) + assert "nvcc: No such file or directory" in reason + assert len(reason) <= 240 + assert len(log_utils.summarize_exception(RuntimeError("x" * 1000))) <= 240 + + +def test_errors_bypass_sampling_and_pd_stages_are_debug(): + output = _run_logging( + """ + from lightllm.utils.log_utils import log_request_summary + log_request_summary(1, status="completed", timing_scope="end_to_end") + log_request_summary(2, status="error", timing_scope="decode") + log_request_summary(3, status="cancelled", timing_scope="end_to_end") + """, + LIGHTLLM_LOG_FORMAT="json", + LIGHTLLM_LOG_REQUEST_SAMPLE_RATE=0, + ) + records = [json.loads(line) for line in output.splitlines()] + assert len(records) == 1 + assert records[0]["event"] == "request.failed" + assert records[0]["level"] == "ERROR" + output = _run_logging( + """ + from lightllm.utils.log_utils import log_request_summary + log_request_summary(4, status="completed", timing_scope="prefill") + log_request_summary(5, status="completed", timing_scope="end_to_end") + """, + LIGHTLLM_LOG_FORMAT="json", + ) + records = [json.loads(line) for line in output.splitlines()] + assert len(records) == 1 and records[0]["request_id"] == 5 + + +def test_terminal_error_is_not_hidden_by_successful_sibling(): + assert log_utils.request_finish_fields(["stop", "error"])["status"] == "error" + assert log_utils.request_finish_fields(["length", "abort"])["status"] == "abort" + assert log_utils.request_finish_fields(["length"])["finish_reason"] == "length" + + +def test_compact_stats_text_retains_exact_json_window(): + code = """ + import logging + from lightllm.utils.log_utils import PeriodicStatsLogger + stats = PeriodicStatsLogger(logging.getLogger("lightllm.stats")) + stats.active = True + stats.log(prefill_tokens=20, output_tokens=40, completed_reqs=1, window_s=2, + output_tok_s=20, running=1, kv_active_pct_by_local_dp=[3.0]) + """ + text = _run_logging(code) + assert "output_tok_s=20" in text and "running=1" in text + assert "output_tokens=" not in text and "window_s=" not in text + record = json.loads(_run_logging(code, LIGHTLLM_LOG_FORMAT="json")) + assert record["output_tokens"] == 40 and record["window_s"] == 2 diff --git a/unit_tests/utils/test_start_utils.py b/unit_tests/utils/test_start_utils.py index 5401c0c358..2c372c2ae7 100644 --- a/unit_tests/utils/test_start_utils.py +++ b/unit_tests/utils/test_start_utils.py @@ -293,3 +293,30 @@ def test_supervisor_supports_submodule_only_processes(monkeypatch): assert kill_calls == [] assert terminate_calls == [True] + + +def test_readiness_waits_for_bound_endpoint_and_pd_topology(monkeypatch): + from unittest.mock import Mock + + logger = Mock() + monkeypatch.setattr(start_utils, "logger", logger) + monkeypatch.setattr(start_utils.time, "monotonic", lambda: 25) + flag = Mock() + args = SimpleNamespace(run_mode="pd_master", model_name="test-model") + state = start_utils.ServiceReadiness(flag, args, "http://127.0.0.1:8000", 10) + state.observe(None) + state.observe(503) + state.observe(503) + assert logger.info.call_count == 1 + flag.set_value.assert_not_called() + state.observe(200) + state.observe(200) + assert logger.info.call_count == 2 + flag.set_value.assert_called_once_with(1) + assert logger.info.call_args.kwargs["extra"]["event"] == "service.ready" + state.observe(None) + state.observe(None) + logger.warning.assert_called_once() + assert flag.set_value.call_args.args == (0,) + state.observe(200) + assert logger.info.call_args.kwargs["extra"]["event"] == "service.recovered" From 0b78963c8db1f0232250570e3708530053fd77af Mon Sep 17 00:00:00 2001 From: sufubao Date: Sat, 17 Oct 2026 20:09:01 +0800 Subject: [PATCH 02/13] style(logging): match the pinned Black formatter --- lightllm/common/kv_cache_mem_manager/mem_manager.py | 2 +- lightllm/server/config_server/api_http.py | 4 ++-- lightllm/utils/log_utils.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lightllm/common/kv_cache_mem_manager/mem_manager.py b/lightllm/common/kv_cache_mem_manager/mem_manager.py index 5384de4097..311fe3b583 100755 --- a/lightllm/common/kv_cache_mem_manager/mem_manager.py +++ b/lightllm/common/kv_cache_mem_manager/mem_manager.py @@ -97,7 +97,7 @@ def profile_size(self, mem_fraction): available_memory = get_available_gpu_memory(world_size) - get_total_gpu_memory() * (1 - mem_fraction) cell_size = self.get_cell_size() pd_kv_move_buffer_size = self.get_pd_kv_move_buffer_size() - available_memory_bytes = available_memory * 1024**3 - pd_kv_move_buffer_size + available_memory_bytes = available_memory * 1024 ** 3 - pd_kv_move_buffer_size self.size = int(available_memory_bytes / cell_size) if world_size > 1: tensor = torch.tensor(self.size, dtype=torch.int64, device=f"cuda:{get_current_device_id()}") diff --git a/lightllm/server/config_server/api_http.py b/lightllm/server/config_server/api_http.py index 61a5f39700..e1a30de315 100644 --- a/lightllm/server/config_server/api_http.py +++ b/lightllm/server/config_server/api_http.py @@ -134,7 +134,7 @@ async def allocate_global_id_range(): global global_req_id range_size = 800000 with global_req_id_lock: - if global_req_id + range_size > 2**63 - 1: + if global_req_id + range_size > 2 ** 63 - 1: global_req_id = 0 start_id = global_req_id global_req_id += range_size @@ -148,7 +148,7 @@ async def allocate_global_unique_multimodal_id_range(): global global_multimodal_embedding_id range_size = 8000000 with global_multimodal_embedding_id_lock: - if global_multimodal_embedding_id + range_size > 2**63 - 1: + if global_multimodal_embedding_id + range_size > 2 ** 63 - 1: global_multimodal_embedding_id = 100000000 start_id = global_multimodal_embedding_id global_multimodal_embedding_id += range_size diff --git a/lightllm/utils/log_utils.py b/lightllm/utils/log_utils.py index a12b9685b8..84d8ba6167 100644 --- a/lightllm/utils/log_utils.py +++ b/lightllm/utils/log_utils.py @@ -301,7 +301,7 @@ def should_log_request(request_id): if _request_sample_rate == 1: return True digest = hashlib.blake2b(str(request_id).encode(), digest_size=8).digest() - return int.from_bytes(digest, "big") / 2**64 < _request_sample_rate + return int.from_bytes(digest, "big") / 2 ** 64 < _request_sample_rate def http_access_enabled(): From 73a00daca1546286d520a74ec6c1d1c09df3f684 Mon Sep 17 00:00:00 2001 From: sufubao Date: Thu, 22 Oct 2026 00:02:23 +0800 Subject: [PATCH 03/13] fix: address review findings for #1638 --- docs/CN/source/tutorial/logging.rst | 2 +- docs/EN/source/tutorial/logging.rst | 2 +- lightllm/distributed/flashinfer_all_reduce.py | 9 ++- lightllm/server/api_cli.py | 2 +- lightllm/server/httpserver/manager.py | 4 +- .../httpserver_for_pd_master/manager.py | 5 +- .../mode_backend/pd/nixl_kv_transporter.py | 1 + lightllm/utils/log_utils.py | 9 ++- lightllm/utils/start_utils.py | 7 ++- .../test_pd_master_cached_tokens.py | 20 +++--- .../test_running_request_lifecycle.py | 6 +- .../mode_backend/test_pd_transfer_logging.py | 63 +++++++++++++++++++ unit_tests/utils/test_flashinfer_logging.py | 27 ++++++++ unit_tests/utils/test_log_utils.py | 29 +++++++++ unit_tests/utils/test_start_utils.py | 37 +++++++++++ 15 files changed, 197 insertions(+), 26 deletions(-) create mode 100644 unit_tests/server/router/model_infer/mode_backend/test_pd_transfer_logging.py create mode 100644 unit_tests/utils/test_flashinfer_logging.py diff --git a/docs/CN/source/tutorial/logging.rst b/docs/CN/source/tutorial/logging.rst index 29526f118a..8240ec5c3e 100644 --- a/docs/CN/source/tutorial/logging.rst +++ b/docs/CN/source/tutorial/logging.rst @@ -137,7 +137,7 @@ JSON 每行一个对象，额外字段可用于检索；异常 traceback 位于 升级说明 -------- -原来的默认 DEBUG 改为 INFO。需要原有详细输出时显式设置 DEBUG，推荐只打开相关模块。 +默认级别仍为 INFO，详细运行诊断现在使用 DEBUG。需要原有详细输出时显式设置 DEBUG，推荐只打开相关模块。 成功请求完成摘要默认采样 1%；HTTP access 按需显式开启。 设置日志目录后统一写入 ``process..log``，不再额外创建 ``default.log``； 轮转备份为 ``process..log.1`` 等。fork 子进程会切换到自己的 PID 文件。 diff --git a/docs/EN/source/tutorial/logging.rst b/docs/EN/source/tutorial/logging.rst index 228f6027a3..5f9a49604c 100644 --- a/docs/EN/source/tutorial/logging.rst +++ b/docs/EN/source/tutorial/logging.rst @@ -120,7 +120,7 @@ for tracebacks. Text mode prefixes every physical message/traceback line. Migration --------- -The former DEBUG default is now INFO. Completion summaries are enabled by default; HTTP access is opt-in. +The default level remains INFO; detailed runtime diagnostics now use DEBUG. Completion summaries are enabled by default; HTTP access is opt-in. File logging writes only ``process..log`` and numbered rotation backups, without creating ``default.log``. Forked workers switch to their own PID files. Backup limits apply per PID; deployment tooling must expire historical PID files. Container deployments diff --git a/lightllm/distributed/flashinfer_all_reduce.py b/lightllm/distributed/flashinfer_all_reduce.py index 02a1c2fb65..0cb61c5040 100644 --- a/lightllm/distributed/flashinfer_all_reduce.py +++ b/lightllm/distributed/flashinfer_all_reduce.py @@ -104,11 +104,10 @@ def _ensure_workspace(self, hidden_dim: int, dtype: torch.dtype) -> bool: comm_backend=TorchDistBackend(group=self.group), ) except Exception as e: - if self.rank == 0: - logger.warning( - "event=allreduce.unavailable backend=flashinfer reason=%r action=disable_backend", - summarize_exception(e), - ) + logger.warning( + "event=allreduce.unavailable backend=flashinfer reason=%r action=disable_backend", + summarize_exception(e), + ) logger.debug("FlashInfer workspace initialization diagnostics", exc_info=True) self.disabled = True self._workspace = None diff --git a/lightllm/server/api_cli.py b/lightllm/server/api_cli.py index 444e155e65..db695d60a8 100644 --- a/lightllm/server/api_cli.py +++ b/lightllm/server/api_cli.py @@ -382,7 +382,7 @@ def add_cli_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: "--pd_detail_log", action="store_true", help=( - "enable verbose per-block PD KV transfer debug logs. " + "Deprecated: use module DEBUG logging for bounded PD KV transfer summaries. " "Request-level debug logs and warning/error logs do not require this flag." ), ) diff --git a/lightllm/server/httpserver/manager.py b/lightllm/server/httpserver/manager.py index 3eee51e6ed..7def2678fe 100644 --- a/lightllm/server/httpserver/manager.py +++ b/lightllm/server/httpserver/manager.py @@ -536,7 +536,9 @@ async def _generate( yield sub_req_id, request_output, metadata, finish_status except (asyncio.CancelledError, BaseException) as e: - cancelled = isinstance(e, (ClientDisconnected, asyncio.CancelledError, GeneratorExit)) + cancelled = isinstance( + e, (ClientDisconnected, asyncio.CancelledError, GeneratorExit, PDPrefillNodeStopGenToken) + ) log_request_summary( group_request_id, status="cancelled" if cancelled else "timeout" if isinstance(e, asyncio.TimeoutError) else "error", diff --git a/lightllm/server/httpserver_for_pd_master/manager.py b/lightllm/server/httpserver_for_pd_master/manager.py index cdfc790a54..7a6eff5dc6 100644 --- a/lightllm/server/httpserver_for_pd_master/manager.py +++ b/lightllm/server/httpserver_for_pd_master/manager.py @@ -418,6 +418,7 @@ async def _generate_one_attempt( sampling_params, multimodal_params, request, + origin_request_id=origin_request_id, ) raw_finish_status = FinishStatus() async for sub_req_id, request_output, metadata, raw_finish_status in results_generator: @@ -713,6 +714,7 @@ async def _wait_to_token_package( sampling_params: SamplingParams, multimodal_params: MultimodalParams, request: Request, + origin_request_id: Optional[int] = None, ): if sampling_params.disable_prompt_cache: assert False, "pd mode dont support set disable_prompt_cache to True" @@ -761,7 +763,8 @@ async def _wait_to_token_package( break except BaseException as error: - if not isinstance(error, GeneratorExit) or not finish_reasons: + stopped = origin_request_id is not None and origin_request_id not in self.active_req_ids + if not isinstance(error, GeneratorExit) or not finish_reasons or stopped: interrupted = isinstance(error, (ClientDisconnected, asyncio.CancelledError, GeneratorExit)) finish_reasons.add("abort" if interrupted else "error") raise diff --git a/lightllm/server/router/model_infer/mode_backend/pd/nixl_kv_transporter.py b/lightllm/server/router/model_infer/mode_backend/pd/nixl_kv_transporter.py index ff96500f5d..e9bcfec4c4 100644 --- a/lightllm/server/router/model_infer/mode_backend/pd/nixl_kv_transporter.py +++ b/lightllm/server/router/model_infer/mode_backend/pd/nixl_kv_transporter.py @@ -63,6 +63,7 @@ def get_new_notifs(self) -> Dict[str, list[bytes]]: return self.nixl_agent.get_new_notifs() def _register_kv_move_buffer(self, kv_move_buffer: Tensor): + self.kv_move_buffer = kv_move_buffer self.num_pages, self.page_size, self.num_layers, self.kv_head_num, self.head_dims = kv_move_buffer.shape self.dtype_byte_size = kv_move_buffer.element_size() self.page_len = self.page_size * self.num_layers * self.kv_head_num * self.head_dims * self.dtype_byte_size diff --git a/lightllm/utils/log_utils.py b/lightllm/utils/log_utils.py index 84d8ba6167..b4d4e1a5db 100644 --- a/lightllm/utils/log_utils.py +++ b/lightllm/utils/log_utils.py @@ -246,7 +246,7 @@ def _setup_logger(): logger.removeHandler(handler) for handler in handlers: logger.addHandler(handler) - logger.setLevel(level) + logger.setLevel(min(level, logging.WARNING) if name == "py.warnings" else level) for handler in old_handlers: handler.close() _request_sample_rate = request_rate @@ -259,7 +259,7 @@ def init_logger(name: str): if name != "lightllm" and not name.startswith("lightllm."): # Scripts also use this API with __name__ == '__main__'. _external_loggers.add(name) - logger.setLevel(_root_logger.level) + logger.setLevel(min(_root_logger.level, logging.WARNING) if name == "py.warnings" else _root_logger.level) for handler in _root_logger.handlers: if getattr(handler, "_lightllm_owned", False): logger.addHandler(handler) @@ -283,6 +283,11 @@ def configure_logging_from_args(args): access = getattr(args, "enable_http_access_log", None) if access is not None: os.environ["LIGHTLLM_LOG_HTTP_ACCESS"] = "1" if access else "0" + if getattr(args, "pd_detail_log", False): + logging.getLogger(__name__).warning( + "--pd_detail_log is deprecated; use LIGHTLLM_LOG_MODULE_LEVELS to enable DEBUG " + "for PD transfer modules. Transfer diagnostics now use bounded summaries." + ) _level(os.getenv("LIGHTLLM_LOG_HTTP_LEVEL", "WARNING")) os.environ["LIGHTLLM_LOG_NODE_RANK"] = str(args.node_rank) os.environ["LIGHTLLM_LOG_TP_SIZE"] = str(max(1, getattr(args, "tp", 1) // getattr(args, "dp", 1))) diff --git a/lightllm/utils/start_utils.py b/lightllm/utils/start_utils.py index 5b076c7c64..b408b8482a 100644 --- a/lightllm/utils/start_utils.py +++ b/lightllm/utils/start_utils.py @@ -80,9 +80,12 @@ def start_readiness_monitor(self, http_server_process, args): ready_flag = SharedInt("service_ready") ready_flag.set_value(0) - host = {"0.0.0.0": "127.0.0.1", "::": "::1"}.get(args.host, args.host) + bind_host = args.config_server_host if args.run_mode == "config_server" else args.host + ports = get_shm_port_args() + port = ports.config_server_port if args.run_mode == "config_server" else ports.port + host = {"0.0.0.0": "127.0.0.1", "::": "::1"}.get(bind_host, bind_host) host = f"[{host}]" if ":" in host else host - address = f"http://{host}:{get_shm_port_args().port}" + address = f"http://{host}:{port}" started = float(os.getenv("LIGHTLLM_SERVICE_START_TIME", str(time.monotonic()))) self.readiness_stop.clear() diff --git a/unit_tests/server/httpserver/test_pd_master_cached_tokens.py b/unit_tests/server/httpserver/test_pd_master_cached_tokens.py index e45d01e77a..3ce0e554b0 100644 --- a/unit_tests/server/httpserver/test_pd_master_cached_tokens.py +++ b/unit_tests/server/httpserver/test_pd_master_cached_tokens.py @@ -80,7 +80,7 @@ def _collect( ): segment_iter = iter(segments) - async def fake_wait(p_node, d_node, start_time, prompt_ids, sp, multimodal_params, request): + async def fake_wait(p_node, d_node, start_time, prompt_ids, sp, multimodal_params, request, **_kwargs): mgr.segment_inputs.append( pickle.loads(pickle.dumps((prompt_ids, sp.constraint_out_params.pd_previous_output_len))) ) @@ -145,7 +145,7 @@ async def run(): params.init(None, stop_sequences=[stop_ids], stop_str_match_mode=mode, max_new_tokens=6) segments = [] - async def raw_segment(p_node, d_node, start_time, prompt, sp, multimodal, request): + async def raw_segment(p_node, d_node, start_time, prompt, sp, multimodal, request, **_kwargs): index = len(segments) segments.append(sp.group_request_id) req = InferReq.__new__(InferReq) @@ -262,7 +262,7 @@ def test_error_result_records_hit_rate_without_inserting_prompt_cache(monkeypatc sampling_params.best_of = 1 sampling_params.max_new_tokens = 1 - async def failed_wait(_p_node, _d_node, _start_time, _prompt_ids, sp, *_args): + async def failed_wait(_p_node, _d_node, _start_time, _prompt_ids, sp, *_args, **_kwargs): yield ( sp.group_request_id, "", @@ -310,7 +310,7 @@ def test_pd_raw_stop_does_not_use_configured_reasoning_parser(monkeypatch): _initial_reasoning_state=1, ) - async def fake_wait(p_node, d_node, start_time, prompt, sp, multimodal_params, request): + async def fake_wait(p_node, d_node, start_time, prompt, sp, multimodal_params, request, **_kwargs): assert sp.stop_str_match_mode.get_mode() == (StopStrMatchMode.ALL_TEXT) metadata = {"prompt_tokens": 10, "prompt_cache_len": 0, "count_output_tokens": 1, "id": 10} yield sp.group_request_id, "reasonENDextra", metadata, FinishStatus(FinishStatus.FINISHED_LENGTH) @@ -352,7 +352,7 @@ def test_pd_continuation_uses_local_reasoning_state(monkeypatch, in_reasoning): ) segment_modes = [] - async def fake_wait(p_node, d_node, start_time, prompt, sp, multimodal_params, request): + async def fake_wait(p_node, d_node, start_time, prompt, sp, multimodal_params, request, **_kwargs): segment_modes.append(sp._initial_reasoning_state) assert sp.stop_str_match_mode.get_mode() == StopStrMatchMode.CONTENT_ONLY metadata = {"prompt_tokens": 10, "prompt_cache_len": 0, "count_output_tokens": 1} @@ -463,7 +463,7 @@ def test_pd_stop_prefix_survives_capacity_split(monkeypatch, include, parser, co ) segments = [] - async def fake_wait(p_node, d_node, start_time, prompt, sp, multimodal_params, request): + async def fake_wait(p_node, d_node, start_time, prompt, sp, multimodal_params, request, **_kwargs): segments.append(prompt) metadata = {"prompt_tokens": 10, "prompt_cache_len": 0, "count_output_tokens": 1} if len(segments) == 1: @@ -554,7 +554,7 @@ async def stop_and_notify(request_id): manager.stop_request = stop_and_notify - async def raw_segment(p_node, d_node, start_time, prompt, sp, multimodal_params, request): + async def raw_segment(p_node, d_node, start_time, prompt, sp, multimodal_params, request, **_kwargs): segments.append(prompt) metadata = {"prompt_tokens": 10, "prompt_cache_len": 0, "count_output_tokens": 1, "id": 10} try: @@ -689,9 +689,7 @@ async def run(): summaries = summary.call_args_list assert len(summaries) == 1 assert summaries[0].args[0] == 1 - assert summaries[0].kwargs["status"] == ( - "continued" if late_status == FinishStatus.FINISHED_PD_DECODE_CAPACITY else "abort" - ) + assert summaries[0].kwargs["status"] == "abort" manager.metric_client.histogram_observe.assert_any_call("lightllm_request_generated_tokens", 2) asyncio.run(run()) @@ -719,7 +717,7 @@ async def remove_req(*args, **kwargs): manager.remove_req = remove_req - async def raw_segment(p_node, d_node, start_time, prompt, sp, multimodal_params, request): + async def raw_segment(p_node, d_node, start_time, prompt, sp, multimodal_params, request, **_kwargs): index = len(segments) segments.append(sp.group_request_id) metadata = {"prompt_tokens": 10, "prompt_cache_len": 0, "count_output_tokens": 1, "id": 10} diff --git a/unit_tests/server/httpserver/test_running_request_lifecycle.py b/unit_tests/server/httpserver/test_running_request_lifecycle.py index 0487e45281..c36db3db24 100644 --- a/unit_tests/server/httpserver/test_running_request_lifecycle.py +++ b/unit_tests/server/httpserver/test_running_request_lifecycle.py @@ -253,7 +253,10 @@ async def run(): asyncio.run(run()) -def test_prefill_full_cache_hit_never_enters_running_request_count(): +def test_prefill_full_cache_hit_never_enters_running_request_count(monkeypatch): + summary = MagicMock() + monkeypatch.setattr("lightllm.server.httpserver.manager.log_request_summary", summary) + async def run(): manager = _make_manager(NodeRole.P) websocket = AsyncMock() @@ -267,6 +270,7 @@ async def run(): manager._register_running_request.assert_not_awaited() manager._unregister_running_request.assert_not_awaited() websocket.send.assert_awaited_once() + assert summary.call_args.kwargs["status"] == "cancelled" asyncio.run(run()) diff --git a/unit_tests/server/router/model_infer/mode_backend/test_pd_transfer_logging.py b/unit_tests/server/router/model_infer/mode_backend/test_pd_transfer_logging.py new file mode 100644 index 0000000000..0aa547768a --- /dev/null +++ b/unit_tests/server/router/model_infer/mode_backend/test_pd_transfer_logging.py @@ -0,0 +1,63 @@ +import queue +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +import torch + +from lightllm.server.router.model_infer.mode_backend.pd.nixl_kv_transporter import NixlKVTransporter +from lightllm.server.router.model_infer.mode_backend.pd.decode_node_impl import decode_trans_process +from lightllm.server.router.model_infer.mode_backend.pd.prefill_node_impl import prefill_trans_process + + +@pytest.mark.parametrize("side", ["prefill", "decode"]) +def test_nixl_success_logging_keeps_processing_completed_pages(monkeypatch, side): + transporter = NixlKVTransporter.__new__(NixlKVTransporter) + transporter.nixl_agent = Mock() + transporter._create_paged_xfer_handles = Mock() + buffer = torch.empty((2, 3, 4, 5, 6), dtype=torch.float16) + transporter._register_kv_move_buffer(buffer) + assert transporter.page_len == buffer[0].numel() * buffer.element_size() + transporter.release_xfer_handle = Mock() + module = prefill_trans_process if side == "prefill" else decode_trans_process + cls = module._PrefillTransModule if side == "prefill" else module._DecodeTransModule + worker = cls.__new__(cls) + worker.device_id = 0 + worker.transporter = transporter + worker.page_index_queue = queue.Queue() + worker.task_out_queue = queue.Queue() + stats = Mock(interval=1) + monkeypatch.setattr(module, "DebugStatsLogger", lambda *args: stats) + monkeypatch.setattr(module, "logger", Mock()) + monkeypatch.setattr(module.torch.cuda, "set_device", lambda device: None) + task = SimpleNamespace( + src_page_index=0, + dst_page_index=1, + xfer_handle="handle", + request_id=123, + page_kind="kv", + create_time=0, + transfer_kv_num=lambda: 3, + createRetObj=lambda: SimpleNamespace(), + ) + + class StopLoop(Exception): + pass + + items = iter([task, task, queue.Empty, StopLoop]) + + def get(timeout): + item = next(items) + if isinstance(item, type): + raise item() + return item if side == "prefill" else (None, None, item) + + worker.success_queue = SimpleNamespace(get=get) + with pytest.raises(StopLoop): + worker.success_loop() + assert worker.task_out_queue.qsize() == 2 + assert worker.page_index_queue.qsize() == 2 + assert transporter.release_xfer_handle.call_count == 2 + assert stats.add.call_count == 2 + assert stats.add.call_args.kwargs["bytes_total"] == transporter.page_len + stats.flush.assert_called_once() diff --git a/unit_tests/utils/test_flashinfer_logging.py b/unit_tests/utils/test_flashinfer_logging.py new file mode 100644 index 0000000000..792f9b4771 --- /dev/null +++ b/unit_tests/utils/test_flashinfer_logging.py @@ -0,0 +1,27 @@ +from types import SimpleNamespace +from unittest.mock import Mock + +import torch + +from lightllm.distributed import flashinfer_all_reduce as module + + +def test_nonleader_workspace_failure_remains_visible(monkeypatch): + workspace = module.FlashInferAllReduce.__new__(module.FlashInferAllReduce) + workspace._workspace = None + workspace.max_workspace_size = 4096 + workspace.rank = 1 + workspace.world_size = 2 + workspace.group = None + logger = Mock() + monkeypatch.setattr(module, "logger", logger) + monkeypatch.setattr(module, "TorchDistBackend", lambda **kwargs: None) + monkeypatch.setattr( + module, + "flashinfer_comm", + SimpleNamespace(create_allreduce_fusion_workspace=Mock(side_effect=RuntimeError("local allocation failed"))), + ) + assert workspace._ensure_workspace(128, torch.float16) is False + assert workspace.disabled is True + logger.warning.assert_called_once() + assert "local allocation failed" in logger.warning.call_args.args[1] diff --git a/unit_tests/utils/test_log_utils.py b/unit_tests/utils/test_log_utils.py index fd7133d5c5..d0399e7e6f 100644 --- a/unit_tests/utils/test_log_utils.py +++ b/unit_tests/utils/test_log_utils.py @@ -570,3 +570,32 @@ def test_compact_stats_text_retains_exact_json_window(): assert "output_tokens=" not in text and "window_s=" not in text record = json.loads(_run_logging(code, LIGHTLLM_LOG_FORMAT="json")) assert record["output_tokens"] == 40 and record["window_s"] == 2 + + +def test_python_warnings_survive_error_service_level_and_reconfiguration(): + output = _run_logging( + """ + import warnings + from lightllm.utils.log_utils import _setup_logger + warnings.warn("before-reconfigure", RuntimeWarning) + _setup_logger() + warnings.warn("after-reconfigure", RuntimeWarning) + """, + LIGHTLLM_LOG_LEVEL="ERROR", + ) + assert output.count("before-reconfigure") == 1 + assert output.count("after-reconfigure") == 1 + + +def test_legacy_pd_detail_flag_explains_bounded_debug_replacement(): + output = _run_logging( + """ + import argparse + from lightllm.server.api_cli import add_cli_args + from lightllm.utils.log_utils import configure_logging_from_args + args = add_cli_args(argparse.ArgumentParser()).parse_args(["--pd_detail_log"]) + configure_logging_from_args(args) + """ + ) + assert output.count("--pd_detail_log is deprecated") == 1 + assert "bounded summaries" in output diff --git a/unit_tests/utils/test_start_utils.py b/unit_tests/utils/test_start_utils.py index 2c372c2ae7..6344d2468e 100644 --- a/unit_tests/utils/test_start_utils.py +++ b/unit_tests/utils/test_start_utils.py @@ -320,3 +320,40 @@ def test_readiness_waits_for_bound_endpoint_and_pd_topology(monkeypatch): assert flag.set_value.call_args.args == (0,) state.observe(200) assert logger.info.call_args.kwargs["extra"]["event"] == "service.recovered" + + +@pytest.mark.parametrize( + "mode,host,expected", + [ + ("normal", "0.0.0.0", "http://127.0.0.1:8000/readiness"), + ("pd_master", "::", "http://[::1]:8000/readiness"), + ("config_server", "0.0.0.0", "http://127.0.0.2:9000/readiness"), + ], +) +def test_readiness_monitor_probes_bound_address(monkeypatch, mode, host, expected): + import requests + from unittest.mock import Mock, MagicMock + from lightllm.server.router.dynamic_prompt import shared_arr + from lightllm.utils import shm_port_args + + manager = start_utils.SubmoduleManager() + flag = Mock() + monkeypatch.setattr(shared_arr, "SharedInt", lambda name: flag) + monkeypatch.setattr(shm_port_args, "get_shm_port_args", lambda: SimpleNamespace(port=8000, config_server_port=9000)) + monkeypatch.setattr(start_utils.threading, "Thread", lambda target, **kw: SimpleNamespace(start=target)) + session = MagicMock() + session.__enter__.return_value = session + + def probe(url, timeout): + assert url == expected + assert timeout == 1 + assert session.trust_env is False + manager.readiness_stop.set() + return SimpleNamespace(status_code=200) + + session.get.side_effect = probe + monkeypatch.setattr(requests, "Session", lambda: session) + args = SimpleNamespace(run_mode=mode, host=host, config_server_host="127.0.0.2", model_name="test") + manager.start_readiness_monitor(FakeHttpServerProcess(), args) + session.get.assert_called_once() + assert flag.set_value.call_args.args == (1,) From 4516028ae6d1e2013c5bd0067dfe2d39edc10bd2 Mon Sep 17 00:00:00 2001 From: sufubao Date: Thu, 22 Oct 2026 06:57:59 +0800 Subject: [PATCH 04/13] fix(logging): preserve request terminal states and log configuration --- lightllm/server/httpserver/manager.py | 20 ++++-- .../httpserver_for_pd_master/manager.py | 27 ++++++- .../decode_node_impl/decode_trans_process.py | 9 ++- .../server/router/model_infer/model_rpc.py | 7 +- lightllm/utils/dist_utils.py | 4 +- lightllm/utils/log_utils.py | 8 +-- lightllm/utils/start_utils.py | 16 ++++- .../basemodel/attention/fa3/test_neo.py | 9 ++- .../test_pd_master_cached_tokens.py | 70 ++++++++++++++++--- .../test_running_request_lifecycle.py | 2 +- .../mode_backend/test_pd_transfer_logging.py | 46 ++++++++++-- unit_tests/utils/test_log_utils.py | 64 +++++++++++++++++ unit_tests/utils/test_start_utils.py | 4 +- 13 files changed, 249 insertions(+), 37 deletions(-) diff --git a/lightllm/server/httpserver/manager.py b/lightllm/server/httpserver/manager.py index 7def2678fe..57e01fc705 100644 --- a/lightllm/server/httpserver/manager.py +++ b/lightllm/server/httpserver/manager.py @@ -536,12 +536,18 @@ async def _generate( yield sub_req_id, request_output, metadata, finish_status except (asyncio.CancelledError, BaseException) as e: - cancelled = isinstance( - e, (ClientDisconnected, asyncio.CancelledError, GeneratorExit, PDPrefillNodeStopGenToken) - ) + cancelled = isinstance(e, (ClientDisconnected, asyncio.CancelledError, GeneratorExit)) log_request_summary( group_request_id, - status="cancelled" if cancelled else "timeout" if isinstance(e, asyncio.TimeoutError) else "error", + status=( + "completed" + if isinstance(e, PDPrefillNodeStopGenToken) + else "cancelled" + if cancelled + else "timeout" + if isinstance(e, asyncio.TimeoutError) + else "error" + ), timing_scope=self.args.run_mode if self.args.run_mode in ("prefill", "decode") else "end_to_end", error_type=type(e).__name__, ) @@ -871,7 +877,11 @@ async def _wait_to_token_package( yield sub_req_id, out_str, metadata, finish_status # 如果有子请求完成，就更新计数 if finish_status.is_finished(): - finish_reasons.add(finish_status.get_finish_reason()) + finish_reasons.add( + "pd_capacity" + if finish_status.is_finished_pd_decode_capacity() + else finish_status.get_finish_reason() + ) unfinished_count -= 1 if unfinished_count == 0: diff --git a/lightllm/server/httpserver_for_pd_master/manager.py b/lightllm/server/httpserver_for_pd_master/manager.py index 7a6eff5dc6..3a821078a0 100644 --- a/lightllm/server/httpserver_for_pd_master/manager.py +++ b/lightllm/server/httpserver_for_pd_master/manager.py @@ -321,6 +321,10 @@ async def _generate_one_attempt( p_node = None d_node = None pending_prefill_load_chars = None + results_generator = None + summary_state = {} + if not hasattr(self, "_summary_states"): + self._summary_states = {} self.active_req_ids.add(origin_request_id) try: @@ -410,6 +414,8 @@ async def _generate_one_attempt( ) p_node.dispatched_prompt_chars += pending_prefill_load_chars p_node.dispatched_req_num += 1 + summary_state = {} + self._summary_states[origin_request_id] = summary_state results_generator = self._wait_to_token_package( p_node, d_node, @@ -419,6 +425,7 @@ async def _generate_one_attempt( multimodal_params, request, origin_request_id=origin_request_id, + summary_state=summary_state, ) raw_finish_status = FinishStatus() async for sub_req_id, request_output, metadata, raw_finish_status in results_generator: @@ -428,6 +435,7 @@ async def _generate_one_attempt( if origin_request_id not in self.active_req_ids: await self.abort(block_group_request_id, p_node=p_node, d_node=d_node) # 关闭分段生成器，执行其 finally 中的总结日志和指标记录。 + summary_state["stop"] = True await results_generator.aclose() return # PD 分离模式下 metadata 中的 token 序号可能不准确，按实际产出计数。 @@ -465,6 +473,7 @@ async def _generate_one_attempt( metadata["prompt_cache_len"] = origin_prompt_cache_len or 0 yield origin_request_id, request_output, metadata, raw_finish_status + await results_generator.aclose() await self.remove_req(group_request_id=block_group_request_id) segment_index += 1 # 只有 PD Decode 容量不足产生的内部分段需要续跑；其他状态都结束整个请求。 @@ -472,9 +481,12 @@ async def _generate_one_attempt( break except (ClientDisconnected, BaseException) as e: + if results_generator is not None: + summary_state["error"] = e + await results_generator.aclose() cancelled = isinstance(e, (ClientDisconnected, asyncio.CancelledError, GeneratorExit)) # Busy attempts can be retried by the outer generator; they are decisions, not final failures. - if not isinstance(e, ServerBusyError): + if not isinstance(e, ServerBusyError) and not summary_state.get("logged"): log_request_summary( origin_request_id, status="cancelled" if cancelled else "timeout" if isinstance(e, asyncio.TimeoutError) else "error", @@ -489,6 +501,7 @@ async def _generate_one_attempt( raise e finally: + self._summary_states.pop(origin_request_id, None) self.active_req_ids.discard(origin_request_id) if p_node is not None and pending_prefill_load_chars is not None: p_node.dispatched_prompt_chars = max(0, p_node.dispatched_prompt_chars - pending_prefill_load_chars) @@ -715,10 +728,12 @@ async def _wait_to_token_package( multimodal_params: MultimodalParams, request: Request, origin_request_id: Optional[int] = None, + summary_state: Optional[dict] = None, ): if sampling_params.disable_prompt_cache: assert False, "pd mode dont support set disable_prompt_cache to True" + summary_state = summary_state if summary_state is not None else {} out_token_counter = 0 first_token_cost_ms = float("inf") prompt_cache_len = 0 @@ -763,8 +778,10 @@ async def _wait_to_token_package( break except BaseException as error: - stopped = origin_request_id is not None and origin_request_id not in self.active_req_ids - if not isinstance(error, GeneratorExit) or not finish_reasons or stopped: + error = summary_state.get("error", error) + if summary_state.get("stop"): + finish_reasons = {"stop"} + elif not isinstance(error, GeneratorExit) or not finish_reasons: interrupted = isinstance(error, (ClientDisconnected, asyncio.CancelledError, GeneratorExit)) finish_reasons.add("abort" if interrupted else "error") raise @@ -796,6 +813,7 @@ async def _wait_to_token_package( mtp_enabled=bool(self.args.mtp_step), mtp_avg_token_per_step=mtp_avg_token_per_step, ) + summary_state["logged"] = True self.metric_client.histogram_observe("lightllm_request_inference_duration", total_cost_time_ms / 1000.0) self.metric_client.histogram_observe( "lightllm_request_mean_time_per_token_duration", mean_per_token_cost_time_ms / 1000.0 @@ -809,6 +827,9 @@ async def _wait_to_token_package( async def stop_request(self, request_id: int): """标记指定子请求停止，由生成循环恢复后 abort 当前分段并清理资源。""" + state = getattr(self, "_summary_states", {}).get(request_id) + if state is not None: + state["stop"] = True self.active_req_ids.discard(request_id) async def abort( diff --git a/lightllm/server/router/model_infer/mode_backend/pd/decode_node_impl/decode_trans_process.py b/lightllm/server/router/model_infer/mode_backend/pd/decode_node_impl/decode_trans_process.py index dc87fcfb21..bfd62d64c5 100644 --- a/lightllm/server/router/model_infer/mode_backend/pd/decode_node_impl/decode_trans_process.py +++ b/lightllm/server/router/model_infer/mode_backend/pd/decode_node_impl/decode_trans_process.py @@ -424,7 +424,9 @@ def success_loop(self): ret = trans_task.createRetObj() self.task_out_queue.put(ret) - if logger.isEnabledFor(logging.DEBUG): + if logger.isEnabledFor(logging.DEBUG) and ( + trans_task.page_kind != "kv" or trans_task.transfer_kv_num() != 0 + ): debug_stats.add( request_id=trans_task.request_id, pages=1, @@ -453,7 +455,10 @@ def fail_loop(self): ret = trans_task.createRetObj() self.task_out_queue.put(ret) - logger.error(f"KV transfer failed: {ret}") + if trans_task.error_info == "aborted req": + logger.debug("KV transfer cancelled: %s", ret) + else: + logger.error("KV transfer failed: %s", ret) if trans_task.error_info is not None: # 提前终结所有有问题的属于同一个请求的任务。 diff --git a/lightllm/server/router/model_infer/model_rpc.py b/lightllm/server/router/model_infer/model_rpc.py index e8ef21fd4f..9353b28b3d 100644 --- a/lightllm/server/router/model_infer/model_rpc.py +++ b/lightllm/server/router/model_infer/model_rpc.py @@ -82,7 +82,12 @@ def exposed_init_model(self, kvargs): else: self.backend = ChunkedPrefillBackend() - log_info_on_leader(logger, "event=engine.backend_selected backend=%s", self.backend.__class__.__name__) + log_info_on_leader( + logger, + "event=engine.backend_selected backend=%s", + self.backend.__class__.__name__, + extra={"event": "engine.backend_selected", "backend": self.backend.__class__.__name__}, + ) self.backend.init_model(kvargs) self.rl_backend_ops = RlBackendOps(self.backend) if self.args.enable_rl else None diff --git a/lightllm/utils/dist_utils.py b/lightllm/utils/dist_utils.py index ed955a0846..1e09848823 100644 --- a/lightllm/utils/dist_utils.py +++ b/lightllm/utils/dist_utils.py @@ -78,7 +78,7 @@ def init_vision_distributed_env(kvargs): set_dp_size(dp_size) set_dp_world_size(tp_world_size) set_current_rank_in_dp(tp_rank_id) - set_log_context(tp_rank=tp_rank_id, dp_rank=kvargs.get("dp_rank_id", "-")) + set_log_context(tp_rank=tp_rank_id, tp_size=tp_world_size, dp_rank=kvargs.get("dp_rank_id", "-")) device_id = kvargs["device_id"] set_current_device_id(device_id) torch.cuda.set_device(device_id) @@ -114,7 +114,7 @@ def init_audio_distributed_env(kvargs): set_dp_size(1) set_dp_world_size(tp_world_size) set_current_rank_in_dp(tp_rank_id) - set_log_context(tp_rank=tp_rank_id, dp_rank=kvargs.get("dp_rank_id", "-")) + set_log_context(tp_rank=tp_rank_id, tp_size=tp_world_size, dp_rank=kvargs.get("dp_rank_id", "-")) device_id = kvargs["device_id"] set_current_device_id(device_id) torch.cuda.set_device(device_id) diff --git a/lightllm/utils/log_utils.py b/lightllm/utils/log_utils.py index b4d4e1a5db..48758132c7 100644 --- a/lightllm/utils/log_utils.py +++ b/lightllm/utils/log_utils.py @@ -117,10 +117,6 @@ def formatTime(self, record, datefmt=None): def format(self, record): _add_context(record) record = logging.makeLogRecord(record.__dict__.copy()) - if record.name == "py.warnings": - # Python warning source snippets and trailing blank lines add no value - # to the console. JSON retains the original warning and source. - record.msg, record.args = record.getMessage().splitlines()[0], () record.compact_context = _compact_context(record) record.source = ( f"{record.filename}:{record.lineno} " @@ -181,7 +177,7 @@ def emit(self, record): def _level(value): value = str(value).upper() - if value not in {"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"}: + if value not in {"DEBUG", "INFO", "WARNING", "WARN", "ERROR", "CRITICAL", "FATAL", "NOTSET"}: raise ValueError(f"Invalid LightLLM log level: {value}") return getattr(logging, value) @@ -431,6 +427,8 @@ def get_http_logging_args(application_access=False): """Hypercorn access and error logging are independent of service verbosity.""" level = os.getenv("LIGHTLLM_LOG_HTTP_LEVEL", "warning") _level(level) + if http_access_enabled() and not application_access and _level(level) > logging.INFO: + level = "info" args = ["--log-level", level.lower(), "--error-logfile", "-"] if http_access_enabled() and not application_access: args.extend(["--access-logfile", "-"]) diff --git a/lightllm/utils/start_utils.py b/lightllm/utils/start_utils.py index b408b8482a..388f60ce17 100644 --- a/lightllm/utils/start_utils.py +++ b/lightllm/utils/start_utils.py @@ -23,6 +23,7 @@ def __init__(self, flag, args, address, started): self.ready = False self.ever_ready = False self.waiting_logged = False + self.probe_failure_logged = False def observe(self, status): if status == 200: @@ -50,6 +51,14 @@ def observe(self, status): self.flag.set_value(1) self.ready = self.ever_ready = True else: + if status != 503 and not self.ever_ready and not self.probe_failure_logged: + logger.warning( + "event=service.probe_failed address=%s status=%s", + self.address, + status, + extra={"event": "service.probe_failed", "address": self.address, "http_status": status}, + ) + self.probe_failure_logged = True if self.ready: self.flag.set_value(0) self.ready = False @@ -96,9 +105,12 @@ def monitor(): while not self.readiness_stop.is_set() and http_server_process.poll() is None: try: response = session.get(address + "/readiness", timeout=1) - state.observe(response.status_code) + status = response.status_code except requests.RequestException: - state.observe(None) + status = None + if self.readiness_stop.is_set() or http_server_process.poll() is not None: + break + state.observe(status) self.readiness_stop.wait(0.5) threading.Thread(target=monitor, name="service-readiness", daemon=True).start() diff --git a/unit_tests/common/basemodel/attention/fa3/test_neo.py b/unit_tests/common/basemodel/attention/fa3/test_neo.py index 0ec1cb6c6a..74a5fd5d65 100644 --- a/unit_tests/common/basemodel/attention/fa3/test_neo.py +++ b/unit_tests/common/basemodel/attention/fa3/test_neo.py @@ -1,5 +1,6 @@ import importlib.util import sys +import warnings from unittest.mock import Mock from types import ModuleType, SimpleNamespace @@ -36,7 +37,7 @@ def neo_attention(q, image_token_end=None): @pytest.mark.parametrize("interface_kind", ["missing", "regular", "legacy_tag"]) -def test_unavailable_neo_import_is_silent_and_explicit_use_explains_installation(monkeypatch, interface_kind): +def test_unavailable_neo_import_is_silent_and_explicit_use_explains_installation(monkeypatch, capsys, interface_kind): def regular_attention(q): return q @@ -46,7 +47,11 @@ def legacy_attention(q, image_token_tag=None): attention_func = {"missing": None, "regular": regular_attention, "legacy_tag": legacy_attention}[interface_kind] logger = Mock() monkeypatch.setattr("lightllm.utils.log_utils.init_logger", lambda name: logger) - module = _load_neo(monkeypatch, attention_func) + with warnings.catch_warnings(record=True) as captured: + warnings.simplefilter("always") + module = _load_neo(monkeypatch, attention_func) + assert not captured + assert capsys.readouterr() == ("", "") assert not module.HAS_FLASH_ATTN_INTERFACE assert module.flash_attn_with_kvcache_neo is None diff --git a/unit_tests/server/httpserver/test_pd_master_cached_tokens.py b/unit_tests/server/httpserver/test_pd_master_cached_tokens.py index 3ce0e554b0..4f1f3ec640 100644 --- a/unit_tests/server/httpserver/test_pd_master_cached_tokens.py +++ b/unit_tests/server/httpserver/test_pd_master_cached_tokens.py @@ -586,7 +586,8 @@ async def raw_segment(p_node, d_node, start_time, prompt, sp, multimodal_params, @pytest.mark.parametrize( - "exit_mode, token_count", [("finish", 2), ("close", 1), ("error", 1), ("empty", 0), ("empty_prompt", 2)] + "exit_mode, token_count", + [("finish", 2), ("capacity", 2), ("close", 1), ("error", 1), ("empty", 0), ("empty_prompt", 2)], ) def test_pd_segment_summary_is_recorded_on_exit(monkeypatch, exit_mode, token_count): import math @@ -613,7 +614,9 @@ async def raw_stream(*_args): yield 80, "first", dict(metadata), FinishStatus() if exit_mode == "error": raise RuntimeError("decode failed") - yield 80, "last", dict(metadata), FinishStatus(FinishStatus.FINISHED_LENGTH) + yield 80, "last", dict(metadata), FinishStatus( + FinishStatus.FINISHED_PD_DECODE_CAPACITY if exit_mode == "capacity" else FinishStatus.FINISHED_LENGTH + ) manager.fetch_pd_stream = raw_stream @@ -636,7 +639,13 @@ async def run(): assert summaries[0].kwargs["x_request_id"] == "summary-test" assert summaries[0].kwargs["output_tokens"] == token_count assert summaries[0].kwargs["status"] == ( - "error" if exit_mode == "error" else "abort" if exit_mode == "close" else "completed" + "error" + if exit_mode == "error" + else "abort" + if exit_mode == "close" + else "continued" + if exit_mode == "capacity" + else "completed" ) manager.first_time_costs.add.assert_called_once() manager.per_token_costs.add.assert_called_once() @@ -654,7 +663,8 @@ async def run(): @pytest.mark.parametrize("late_status", [FinishStatus.NO_FINISH, FinishStatus.FINISHED_PD_DECODE_CAPACITY]) -def test_pd_stopped_choice_records_segment_summary(monkeypatch, late_status): +@pytest.mark.parametrize("close_directly", [False, True]) +def test_pd_stopped_choice_records_segment_summary(monkeypatch, late_status, close_directly): import time from unittest.mock import AsyncMock, MagicMock @@ -682,15 +692,21 @@ async def run(): await manager.stop_request(80) # stop_request 只做标记，恢复生成时才停止当前 P/D 分段。 manager.abort.assert_not_awaited() - with pytest.raises(StopAsyncIteration): - await generator.__anext__() + if close_directly: + await generator.aclose() + else: + with pytest.raises(StopAsyncIteration): + await generator.__anext__() assert not manager.active_req_ids manager.abort.assert_awaited_once() summaries = summary.call_args_list assert len(summaries) == 1 assert summaries[0].args[0] == 1 - assert summaries[0].kwargs["status"] == "abort" - manager.metric_client.histogram_observe.assert_any_call("lightllm_request_generated_tokens", 2) + assert summaries[0].kwargs["status"] == "completed" + assert summaries[0].kwargs["finish_reason"] == "stop" + manager.metric_client.histogram_observe.assert_any_call( + "lightllm_request_generated_tokens", 1 if close_directly else 2 + ) asyncio.run(run()) @@ -754,3 +770,41 @@ async def raw_segment(p_node, d_node, start_time, prompt, sp, multimodal_params, assert not manager.active_req_ids asyncio.run(asyncio.wait_for(run(), timeout=2)) + + +@pytest.mark.parametrize("bad_metadata", [False, True]) +def test_pd_failure_emits_one_summary_with_output_counts(monkeypatch, bad_metadata): + import time + from unittest.mock import AsyncMock, MagicMock + + manager = _make_manager(monkeypatch) + manager.first_time_costs = MagicMock() + manager.per_token_costs = MagicMock() + manager.metric_client = MagicMock() + manager.abort = AsyncMock() + summary = MagicMock() + monkeypatch.setattr(pd_master_module, "log_request_summary", summary) + params = SamplingParams() + params.init(SimpleNamespace(encode=lambda *_args, **_kwargs: []), max_new_tokens=4) + request = SimpleNamespace(headers={}, is_disconnected=lambda: asyncio.sleep(0, False)) + + async def raw_stream(_p, _d, _prompt, sp, *_args): + metadata = {"id": 1, "prompt_tokens": 10, "prompt_cache_len": 3} + yield sp.group_request_id, "first", dict(metadata), FinishStatus() + if bad_metadata: + yield sp.group_request_id + 1, "bad", dict(metadata), FinishStatus() + else: + raise RuntimeError("decode failed") + + manager.fetch_pd_stream = raw_stream + + async def run(): + with pytest.raises(AssertionError if bad_metadata else RuntimeError): + async for _ in manager._generate_one("prompt", params, None, request, time.time(), 80, 10): + pass + summary.assert_called_once() + assert summary.call_args.kwargs["status"] == "error" + assert summary.call_args.kwargs["output_tokens"] == (2 if bad_metadata else 1) + assert not manager.active_req_ids + + asyncio.run(run()) diff --git a/unit_tests/server/httpserver/test_running_request_lifecycle.py b/unit_tests/server/httpserver/test_running_request_lifecycle.py index c36db3db24..4b7811b06e 100644 --- a/unit_tests/server/httpserver/test_running_request_lifecycle.py +++ b/unit_tests/server/httpserver/test_running_request_lifecycle.py @@ -270,7 +270,7 @@ async def run(): manager._register_running_request.assert_not_awaited() manager._unregister_running_request.assert_not_awaited() websocket.send.assert_awaited_once() - assert summary.call_args.kwargs["status"] == "cancelled" + assert summary.call_args.kwargs["status"] == "completed" asyncio.run(run()) diff --git a/unit_tests/server/router/model_infer/mode_backend/test_pd_transfer_logging.py b/unit_tests/server/router/model_infer/mode_backend/test_pd_transfer_logging.py index 0aa547768a..36ce21737a 100644 --- a/unit_tests/server/router/model_infer/mode_backend/test_pd_transfer_logging.py +++ b/unit_tests/server/router/model_infer/mode_backend/test_pd_transfer_logging.py @@ -10,8 +10,8 @@ from lightllm.server.router.model_infer.mode_backend.pd.prefill_node_impl import prefill_trans_process -@pytest.mark.parametrize("side", ["prefill", "decode"]) -def test_nixl_success_logging_keeps_processing_completed_pages(monkeypatch, side): +@pytest.mark.parametrize("side, tokens", [("prefill", 3), ("decode", 3), ("decode", 0)]) +def test_nixl_success_logging_keeps_processing_completed_pages(monkeypatch, side, tokens): transporter = NixlKVTransporter.__new__(NixlKVTransporter) transporter.nixl_agent = Mock() transporter._create_paged_xfer_handles = Mock() @@ -37,7 +37,7 @@ def test_nixl_success_logging_keeps_processing_completed_pages(monkeypatch, side request_id=123, page_kind="kv", create_time=0, - transfer_kv_num=lambda: 3, + transfer_kv_num=lambda: tokens, createRetObj=lambda: SimpleNamespace(), ) @@ -58,6 +58,42 @@ def get(timeout): assert worker.task_out_queue.qsize() == 2 assert worker.page_index_queue.qsize() == 2 assert transporter.release_xfer_handle.call_count == 2 - assert stats.add.call_count == 2 - assert stats.add.call_args.kwargs["bytes_total"] == transporter.page_len + assert stats.add.call_count == (2 if tokens else 0) + if tokens: + assert stats.add.call_args.kwargs["bytes_total"] == transporter.page_len stats.flush.assert_called_once() + + +@pytest.mark.parametrize("error_info", ["aborted req", "transfer timed out"]) +def test_decode_transfer_cancellation_is_debug_and_failures_are_error(monkeypatch, error_info): + worker = decode_trans_process._DecodeTransModule.__new__(decode_trans_process._DecodeTransModule) + worker.device_id = 0 + worker.page_index_queue = queue.Queue() + worker.task_out_queue = queue.Queue() + worker.transporter = Mock() + worker._abort = Mock() + logger = Mock() + monkeypatch.setattr(decode_trans_process, "logger", logger) + monkeypatch.setattr(decode_trans_process.torch.cuda, "set_device", lambda _: None) + task = SimpleNamespace( + dst_page_index=None, + xfer_handle=None, + request_id=123, + error_info=error_info, + createRetObj=lambda: "result", + ) + + class StopLoop(Exception): + pass + + worker.failed_queue = Mock() + worker.failed_queue.get.side_effect = [task, StopLoop()] + with pytest.raises(StopLoop): + worker.fail_loop() + worker._abort.assert_called_once_with(request_id=123, error_info=error_info) + worker.transporter.send_error_info_to_prefill_node.assert_called_once_with(trans_task=task) + if error_info == "aborted req": + logger.error.assert_not_called() + logger.debug.assert_called_once_with("KV transfer cancelled: %s", "result") + else: + logger.error.assert_called_once_with("KV transfer failed: %s", "result") diff --git a/unit_tests/utils/test_log_utils.py b/unit_tests/utils/test_log_utils.py index d0399e7e6f..e8e35bd8ac 100644 --- a/unit_tests/utils/test_log_utils.py +++ b/unit_tests/utils/test_log_utils.py @@ -599,3 +599,67 @@ def test_legacy_pd_detail_flag_explains_bounded_debug_replacement(): ) assert output.count("--pd_detail_log is deprecated") == 1 assert "bounded summaries" in output + + +@pytest.mark.parametrize("level, expected", [("warn", 30), ("fatal", 50), ("notset", 0)]) +def test_standard_logging_level_aliases(level, expected): + _run_logging( + f"from lightllm.utils import log_utils; assert log_utils._root_logger.level == {expected}", + LIGHTLLM_LOG_LEVEL=level, + ) + + +def test_config_server_access_switch_enables_hypercorn_info(monkeypatch): + from hypercorn.config import Config + from hypercorn.logging import Logger + + monkeypatch.setenv("LIGHTLLM_LOG_HTTP_ACCESS", "1") + monkeypatch.delenv("LIGHTLLM_LOG_HTTP_LEVEL", raising=False) + args = log_utils.get_http_logging_args() + config = Config() + config.loglevel = args[args.index("--log-level") + 1] + config.accesslog = "-" + assert Logger(config).access_logger.isEnabledFor(logging.INFO) + + +def test_multiline_warning_text_is_preserved(): + record = logging.LogRecord("py.warnings", logging.WARNING, __file__, 1, "first\nsecond", (), None) + formatted = log_utils.NewLineFormatter("%(message)s").format(record) + assert "first" in formatted and "second" in formatted + + +@pytest.mark.parametrize("kind", ["vision", "audio"]) +def test_multimodal_rank_context_uses_its_own_tp_size(monkeypatch, kind): + from lightllm.utils import dist_utils + + context = {} + for setter in ("set_dp_size", "set_dp_world_size", "set_current_rank_in_dp", "set_current_device_id"): + monkeypatch.setattr(dist_utils, setter, lambda _value: None) + monkeypatch.setattr(dist_utils, "set_log_context", lambda **fields: context.update(fields)) + + class StopBeforeCuda(Exception): + pass + + def stop(_device): + raise StopBeforeCuda() + + monkeypatch.setattr(dist_utils.torch.cuda, "set_device", stop) + init = dist_utils.init_vision_distributed_env if kind == "vision" else dist_utils.init_audio_distributed_env + params = {"vit_tp": 4, "audio_tp": 4, "tp_rank_id": 3, "device_id": 0} + with pytest.raises(StopBeforeCuda): + init(params) + assert context == {"tp_rank": 3, "tp_size": 4, "dp_rank": "-"} + + +def test_default_request_sampling_is_stable_and_bounded(): + _run_logging( + """ + import os + os.environ.pop("LIGHTLLM_LOG_REQUEST_SAMPLE_RATE", None) + from lightllm.utils import log_utils + assert log_utils._request_sample_rate == 0.01 + first = [log_utils.should_log_request(i) for i in range(2000)] + assert first == [log_utils.should_log_request(i) for i in range(2000)] + assert 5 <= sum(first) <= 40 + """ + ) diff --git a/unit_tests/utils/test_start_utils.py b/unit_tests/utils/test_start_utils.py index 6344d2468e..fa63a20494 100644 --- a/unit_tests/utils/test_start_utils.py +++ b/unit_tests/utils/test_start_utils.py @@ -305,6 +305,8 @@ def test_readiness_waits_for_bound_endpoint_and_pd_topology(monkeypatch): args = SimpleNamespace(run_mode="pd_master", model_name="test-model") state = start_utils.ServiceReadiness(flag, args, "http://127.0.0.1:8000", 10) state.observe(None) + logger.warning.assert_called_once() + logger.warning.reset_mock() state.observe(503) state.observe(503) assert logger.info.call_count == 1 @@ -356,4 +358,4 @@ def probe(url, timeout): args = SimpleNamespace(run_mode=mode, host=host, config_server_host="127.0.0.2", model_name="test") manager.start_readiness_monitor(FakeHttpServerProcess(), args) session.get.assert_called_once() - assert flag.set_value.call_args.args == (1,) + flag.set_value.assert_called_once_with(0) From 2b8eb9f8965ea51b8b98d465a0332875dc49b8be Mon Sep 17 00:00:00 2001 From: sufubao Date: Thu, 22 Oct 2026 07:41:21 +0800 Subject: [PATCH 05/13] fix: address review findings for #1638 --- lightllm/server/api_http_pd.py | 1 + lightllm/server/httpserver/manager.py | 6 +++++ lightllm/server/httpserver/pd_loop.py | 9 ++++--- .../httpserver_for_pd_master/manager.py | 10 ++++--- lightllm/utils/log_utils.py | 24 ++++++++++++++--- lightllm/utils/start_utils.py | 4 ++- .../test_pd_master_cached_tokens.py | 16 ++++++++--- .../httpserver/test_pd_split_connections.py | 26 ++++++++++++++++++ .../httpserver/test_pd_websocket_lifecycle.py | 20 ++++++++++++++ .../test_running_request_lifecycle.py | 27 +++++++++++++++++++ unit_tests/server/test_pd_master_mode.py | 4 ++- unit_tests/utils/test_log_utils.py | 21 +++++++++++++++ unit_tests/utils/test_start_utils.py | 24 ++++++++++++++--- 13 files changed, 174 insertions(+), 18 deletions(-) diff --git a/lightllm/server/api_http_pd.py b/lightllm/server/api_http_pd.py index 7696086b4d..0cecc02132 100644 --- a/lightllm/server/api_http_pd.py +++ b/lightllm/server/api_http_pd.py @@ -33,6 +33,7 @@ async def register_and_keep_alive(websocket: WebSocket): logger.info(f"Client connected from IP: {client_ip}, Port: {client_port}") regist_json = json.loads(await websocket.receive_text()) peer = f"{regist_json.get('mode')}@{regist_json.get('client_ip_port')}" + peer += f" dp_index={regist_json.get('dp_index')} connection_id={regist_json.get('connection_id')}" logger.debug("PD registration received: peer=%s", peer) await g_objs.httpserver_manager.register_pd(regist_json, websocket) diff --git a/lightllm/server/httpserver/manager.py b/lightllm/server/httpserver/manager.py index 00b71c1272..f3094fe7b0 100644 --- a/lightllm/server/httpserver/manager.py +++ b/lightllm/server/httpserver/manager.py @@ -35,6 +35,7 @@ from lightllm.server.router.dynamic_prompt.shared_arr import SharedInt from lightllm.utils.log_utils import ( request_finish_fields, + summarize_exception, init_logger, log_request_summary, log_debug_event, @@ -546,10 +547,15 @@ async def _generate( if cancelled else "timeout" if isinstance(e, asyncio.TimeoutError) + else "busy" + if isinstance(e, ServerBusyError) + else "rejected" + if isinstance(e, InvalidRequestError) else "error" ), timing_scope=self.args.run_mode if self.args.run_mode in ("prefill", "decode") else "end_to_end", error_type=type(e).__name__, + reason=summarize_exception(e), ) # error need to release multimodel resources. diff --git a/lightllm/server/httpserver/pd_loop.py b/lightllm/server/httpserver/pd_loop.py index 0f705987a7..cb223be316 100644 --- a/lightllm/server/httpserver/pd_loop.py +++ b/lightllm/server/httpserver/pd_loop.py @@ -54,14 +54,17 @@ async def pd_handle_loop(manager: HttpServerManager, dp_index: Optional[int] = N manager.host_ip = manager.args.host id_to_handle_task: Dict[int, asyncio.Task] = {} - previous_master_ids = None + previous_master_ids = object() while True: try: id_to_pd_master_obj = await _get_pd_master_objs(manager.args) - master_ids = tuple(sorted(id_to_pd_master_obj)) if id_to_pd_master_obj is not None else None + master_ids = tuple(sorted(id_to_pd_master_obj or {})) if master_ids != previous_master_ids: - log_debug_event(logger, "pd.masters_changed", master_ids=master_ids) + event = "pd.masters_changed" if master_ids else "pd.waiting_for_master" + logger.info( + "event=%s master_ids=%s", event, master_ids, extra={"event": event, "master_ids": master_ids} + ) previous_master_ids = master_ids if id_to_pd_master_obj is not None: diff --git a/lightllm/server/httpserver_for_pd_master/manager.py b/lightllm/server/httpserver_for_pd_master/manager.py index 1f007c0da8..10f301a305 100644 --- a/lightllm/server/httpserver_for_pd_master/manager.py +++ b/lightllm/server/httpserver_for_pd_master/manager.py @@ -280,13 +280,15 @@ async def _generate_one( except ServerBusyError: # 关闭节点自限流时，不启用与该策略配套的 busy 重试，直接透传异常。 if not self.enable_pd_node_self_request_limit: - log_request_summary(origin_request_id, status="busy", timing_scope="end_to_end") + if not has_yielded_result: + log_request_summary(origin_request_id, status="busy", timing_scope="end_to_end") raise # 已向客户端输出 token 后不能从头生成，否则会产生重复内容。 elapsed_seconds = time.monotonic() - retry_start_time if has_yielded_result or elapsed_seconds >= self.pd_node_busy_retry_timeout_seconds: - log_request_summary(origin_request_id, status="busy", timing_scope="end_to_end") + if not has_yielded_result: + log_request_summary(origin_request_id, status="busy", timing_scope="end_to_end") raise # 发起下一次尝试前检查客户端连接，避免为已断开的请求继续占用 P/D 节点资源。 @@ -798,7 +800,9 @@ async def _wait_to_token_package( finish_reasons = {"stop"} elif not isinstance(error, GeneratorExit) or not finish_reasons: interrupted = isinstance(error, (ClientDisconnected, asyncio.CancelledError, GeneratorExit)) - finish_reasons.add("abort" if interrupted else "error") + finish_reasons.add( + "busy" if isinstance(error, ServerBusyError) else "abort" if interrupted else "error" + ) raise finally: # 分段可能因 stop、容量切换或取消而提前关闭，已有输出也需要记录总结。 diff --git a/lightllm/utils/log_utils.py b/lightllm/utils/log_utils.py index 48758132c7..9df0c840b3 100644 --- a/lightllm/utils/log_utils.py +++ b/lightllm/utils/log_utils.py @@ -380,13 +380,21 @@ def log_http_access(method, path, status, duration_ms, http_request_id=None): def log_request_summary(request_id, **fields): status = fields.get("status", "completed") failed = status in {"error", "timeout", "failed", "busy"} + rejected = status == "rejected" cancelled = status in {"cancelled", "abort"} logger = logging.getLogger("lightllm.requests") stage = fields.get("timing_scope") in {"prefill", "decode"} - if not failed and not cancelled and not should_log_request(request_id): + if not failed and not rejected and not cancelled and not should_log_request(request_id): return event = fields.pop( - "event", "request.failed" if failed else "request.cancelled" if cancelled else "request.completed" + "event", + "request.failed" + if failed + else "request.rejected" + if rejected + else "request.cancelled" + if cancelled + else "request.completed", ) client_id = fields.pop("x_request_id", None) or get_http_request_id() if client_id != "-": @@ -397,7 +405,15 @@ def log_request_summary(request_id, **fields): for key, value in fields.items() if not (("cache" in key and not value) or (key.startswith("mtp_") and not mtp_enabled)) } - method = logger.error if failed else logger.debug if stage or cancelled else logger.info + method = ( + logger.warning + if status == "busy" or rejected + else logger.error + if failed + else logger.debug + if stage or cancelled + else logger.info + ) method( "event=%s request_id=%s %s", event, @@ -414,6 +430,8 @@ def request_finish_fields(reasons): status = ( "error" if "error" in reasons + else "busy" + if "busy" in reasons else "abort" if "abort" in reasons else "continued" diff --git a/lightllm/utils/start_utils.py b/lightllm/utils/start_utils.py index 24582766c4..6833354511 100644 --- a/lightllm/utils/start_utils.py +++ b/lightllm/utils/start_utils.py @@ -51,7 +51,7 @@ def observe(self, status): self.flag.set_value(1) self.ready = self.ever_ready = True else: - if status != 503 and not self.ever_ready and not self.probe_failure_logged: + if status not in (None, 503) and not self.ever_ready and not self.probe_failure_logged: logger.warning( "event=service.probe_failed address=%s status=%s", self.address, @@ -90,6 +90,8 @@ def start_readiness_monitor(self, http_server_process, args): ready_flag = SharedInt("service_ready") ready_flag.set_value(0) bind_host = args.config_server_host if args.run_mode == "config_server" else args.host + if bind_host is None: + return ports = get_shm_port_args() port = ports.config_server_port if args.run_mode == "config_server" else ports.port host = {"0.0.0.0": "127.0.0.1", "::": "::1"}.get(bind_host, bind_host) diff --git a/unit_tests/server/httpserver/test_pd_master_cached_tokens.py b/unit_tests/server/httpserver/test_pd_master_cached_tokens.py index 2364370b3a..3788326c15 100644 --- a/unit_tests/server/httpserver/test_pd_master_cached_tokens.py +++ b/unit_tests/server/httpserver/test_pd_master_cached_tokens.py @@ -792,7 +792,7 @@ async def raw_segment(p_node, d_node, start_time, prompt, sp, multimodal_params, asyncio.run(asyncio.wait_for(run(), timeout=2)) -@pytest.mark.parametrize("bad_metadata", [False, True]) +@pytest.mark.parametrize("bad_metadata", [False, True, "busy"]) def test_pd_failure_emits_one_summary_with_output_counts(monkeypatch, bad_metadata): import time from unittest.mock import AsyncMock, MagicMock @@ -811,6 +811,10 @@ def test_pd_failure_emits_one_summary_with_output_counts(monkeypatch, bad_metada async def raw_stream(_p, _d, _prompt, sp, *_args): metadata = {"id": 1, "prompt_tokens": 10, "prompt_cache_len": 3} yield sp.group_request_id, "first", dict(metadata), FinishStatus() + if bad_metadata == "busy": + from lightllm.utils.error_utils import ServerBusyError + + raise ServerBusyError("decode busy") if bad_metadata: yield sp.group_request_id + 1, "bad", dict(metadata), FinishStatus() else: @@ -819,12 +823,16 @@ async def raw_stream(_p, _d, _prompt, sp, *_args): manager.fetch_pd_stream = raw_stream async def run(): - with pytest.raises(AssertionError if bad_metadata else RuntimeError): + from lightllm.utils.error_utils import ServerBusyError + + error = ServerBusyError if bad_metadata == "busy" else AssertionError if bad_metadata else RuntimeError + with pytest.raises(error): async for _ in manager._generate_one("prompt", params, None, request, time.time(), 80, 10): pass summary.assert_called_once() - assert summary.call_args.kwargs["status"] == "error" - assert summary.call_args.kwargs["output_tokens"] == (2 if bad_metadata else 1) + assert summary.call_args.kwargs["status"] == ("busy" if bad_metadata == "busy" else "error") + assert summary.call_args.kwargs["output_tokens"] == (2 if bad_metadata is True else 1) + assert not manager._summary_states assert not manager.active_req_ids asyncio.run(run()) diff --git a/unit_tests/server/httpserver/test_pd_split_connections.py b/unit_tests/server/httpserver/test_pd_split_connections.py index e4fe76eafb..8bfa614b5e 100644 --- a/unit_tests/server/httpserver/test_pd_split_connections.py +++ b/unit_tests/server/httpserver/test_pd_split_connections.py @@ -292,3 +292,29 @@ def test_split_option_defaults_off_and_is_prefill_only(): assert StartArgs().use_dp_split_mode_connect_pd_master is False with pytest.raises(ValueError, match="only supports prefill"): launch_server(StartArgs(run_mode="decode", use_dp_split_mode_connect_pd_master=True)) + + +def test_pd_master_waiting_is_logged_only_on_transitions(monkeypatch): + from unittest.mock import Mock + + logger = Mock() + monkeypatch.setattr(pd_loop, "logger", logger) + masters = {1: PD_Master_Obj(1, "master:8000")} + lookup = AsyncMock(side_effect=[None, None, {}, masters, masters, None]) + monkeypatch.setattr(pd_loop, "_get_pd_master_objs", lookup) + monkeypatch.setattr(pd_loop, "_pd_handle_task", AsyncMock()) + + async def tick(_seconds): + if lookup.await_count == 6: + raise asyncio.CancelledError() + + monkeypatch.setattr(pd_loop.asyncio, "sleep", tick) + + async def run(): + manager = SimpleNamespace(args=StartArgs(host="p", run_mode="prefill")) + with pytest.raises(asyncio.CancelledError): + await pd_loop.pd_handle_loop(manager) + + asyncio.run(run()) + events = [call.kwargs["extra"]["event"] for call in logger.info.call_args_list if "extra" in call.kwargs] + assert events == ["pd.waiting_for_master", "pd.masters_changed", "pd.waiting_for_master"] diff --git a/unit_tests/server/httpserver/test_pd_websocket_lifecycle.py b/unit_tests/server/httpserver/test_pd_websocket_lifecycle.py index 0c261b5dda..301e92805c 100644 --- a/unit_tests/server/httpserver/test_pd_websocket_lifecycle.py +++ b/unit_tests/server/httpserver/test_pd_websocket_lifecycle.py @@ -79,3 +79,23 @@ def test_kv_status_is_forwarded_without_registration_or_reply(monkeypatch): asyncio.run(api_http_pd.kv_move_status(websocket)) manager.update_req_status.assert_awaited_once_with(status) websocket.send_bytes.assert_not_awaited() + + +def test_registration_disconnect_keeps_dp_connection_identity(monkeypatch): + from unittest.mock import Mock + from starlette.websockets import WebSocketDisconnect + + _manager(monkeypatch) + logger = Mock() + monkeypatch.setattr(api_http_pd, "logger", logger) + websocket = _websocket( + receive_text=AsyncMock( + return_value=json.dumps( + {"mode": "prefill", "client_ip_port": "p:8000", "dp_index": 2, "connection_id": "connection-a"} + ) + ), + receive_bytes=AsyncMock(side_effect=WebSocketDisconnect()), + ) + asyncio.run(api_http_pd.register_and_keep_alive(websocket)) + for call in logger.info.call_args_list[1:]: + assert "dp_index=2 connection_id=connection-a" in call.args[0] diff --git a/unit_tests/server/httpserver/test_running_request_lifecycle.py b/unit_tests/server/httpserver/test_running_request_lifecycle.py index 4b7811b06e..8de695678f 100644 --- a/unit_tests/server/httpserver/test_running_request_lifecycle.py +++ b/unit_tests/server/httpserver/test_running_request_lifecycle.py @@ -567,3 +567,30 @@ def test_real_supported_max_req_total_len_reserves_mtp_overlap_tokens(): manager.args = SimpleNamespace(mtp_step=2, page_size=4) assert manager.get_real_supported_max_req_total_len() == 951 + + +@pytest.mark.parametrize( + "kind,status", [("busy", "busy"), ("invalid", "rejected"), ("timeout", "timeout"), ("error", "error")] +) +def test_generate_failure_summary_preserves_reason(monkeypatch, kind, status): + from lightllm.utils.error_utils import ServerBusyError, InvalidRequestError + + summary = MagicMock() + monkeypatch.setattr("lightllm.server.httpserver.manager.log_request_summary", summary) + error_type = { + "busy": ServerBusyError, + "invalid": InvalidRequestError, + "timeout": asyncio.TimeoutError, + "error": RuntimeError, + }[kind] + + async def run(): + manager = _make_manager(NodeRole.NORMAL) + manager._encode.side_effect = error_type("request rejected reason") + with pytest.raises(error_type): + await _drain_generate(manager, _sampling_params(), _multimodal_params()) + summary.assert_called_once() + assert summary.call_args.kwargs["status"] == status + assert summary.call_args.kwargs["reason"] == str(manager._encode.side_effect) + + asyncio.run(run()) diff --git a/unit_tests/server/test_pd_master_mode.py b/unit_tests/server/test_pd_master_mode.py index d9163366f0..a5bac07641 100644 --- a/unit_tests/server/test_pd_master_mode.py +++ b/unit_tests/server/test_pd_master_mode.py @@ -517,7 +517,8 @@ def test_pd_master_cli_dispatches_to_the_requested_startup_mode(monkeypatch, wor single, multiple = Mock(), Mock() monkeypatch.setattr(api_start, "pd_master_start", single) monkeypatch.setattr(api_start, "pd_master_multi_worker_start", multiple) - monkeypatch.setattr(api_server, "set_log_node_role", Mock()) + configure = Mock() + monkeypatch.setattr(api_server, "configure_logging_from_args", configure) monkeypatch.setattr(api_server.torch.multiprocessing, "set_start_method", Mock()) args = StartArgs(run_mode="pd_master", httpserver_workers=workers) @@ -531,6 +532,7 @@ def test_pd_master_cli_dispatches_to_the_requested_startup_mode(monkeypatch, wor expected, unexpected = (single, multiple) if workers == 1 else (multiple, single) expected.assert_called_once_with(args) unexpected.assert_not_called() + configure.assert_called_once_with(args) def test_multiworker_preflight_failure_precedes_environment_and_process_initialization(monkeypatch): diff --git a/unit_tests/utils/test_log_utils.py b/unit_tests/utils/test_log_utils.py index e8e35bd8ac..946d072891 100644 --- a/unit_tests/utils/test_log_utils.py +++ b/unit_tests/utils/test_log_utils.py @@ -54,6 +54,7 @@ def test_log_node_role_marker_is_applied_to_every_line(monkeypatch): def _run_logging(code, **settings): env = {key: value for key, value in os.environ.items() if not key.startswith("LIGHTLLM_LOG_")} + env["PYTHONPATH"] = str(Path(__file__).resolve().parents[2]) + os.pathsep + env.get("PYTHONPATH", "") env["LIGHTLLM_LOG_REQUEST_SAMPLE_RATE"] = "1" env.update({key: str(value) for key, value in settings.items()}) result = subprocess.run( @@ -663,3 +664,23 @@ def test_default_request_sampling_is_stable_and_bounded(): assert 5 <= sum(first) <= 40 """ ) + + +@pytest.mark.parametrize( + "status,level,event", + [ + ("busy", logging.WARNING, "request.failed"), + ("rejected", logging.WARNING, "request.rejected"), + ("error", logging.ERROR, "request.failed"), + ], +) +def test_failure_summary_level_and_sampling(monkeypatch, caplog, status, level, event): + monkeypatch.setattr(log_utils, "should_log_request", lambda request_id: False) + logger = logging.getLogger("lightllm.requests") + monkeypatch.setattr(logger, "handlers", [caplog.handler]) + with caplog.at_level(logging.DEBUG, logger="lightllm.requests"): + log_utils.log_request_summary(123, status=status, reason="bounded reason") + record = caplog.records[-1] + assert record.levelno == level + assert record.event == event + assert record.reason == "bounded reason" diff --git a/unit_tests/utils/test_start_utils.py b/unit_tests/utils/test_start_utils.py index 1dfcec1afd..db41aa873d 100644 --- a/unit_tests/utils/test_start_utils.py +++ b/unit_tests/utils/test_start_utils.py @@ -329,7 +329,11 @@ def test_readiness_waits_for_bound_endpoint_and_pd_topology(monkeypatch): args = SimpleNamespace(run_mode="pd_master", model_name="test-model") state = start_utils.ServiceReadiness(flag, args, "http://127.0.0.1:8000", 10) state.observe(None) + logger.warning.assert_not_called() + assert not state.probe_failure_logged + state.observe(500) logger.warning.assert_called_once() + assert logger.warning.call_args.kwargs["extra"]["http_status"] == 500 logger.warning.reset_mock() state.observe(503) state.observe(503) @@ -364,7 +368,8 @@ def test_readiness_monitor_probes_bound_address(monkeypatch, mode, host, expecte manager = start_utils.SubmoduleManager() flag = Mock() - monkeypatch.setattr(shared_arr, "SharedInt", lambda name: flag) + shared_int = Mock(return_value=flag) + monkeypatch.setattr(shared_arr, "SharedInt", shared_int) monkeypatch.setattr(shm_port_args, "get_shm_port_args", lambda: SimpleNamespace(port=8000, config_server_port=9000)) monkeypatch.setattr(start_utils.threading, "Thread", lambda target, **kw: SimpleNamespace(start=target)) session = MagicMock() @@ -374,12 +379,25 @@ def probe(url, timeout): assert url == expected assert timeout == 1 assert session.trust_env is False - manager.readiness_stop.set() return SimpleNamespace(status_code=200) + monkeypatch.setattr(manager.readiness_stop, "wait", lambda timeout: manager.readiness_stop.set()) session.get.side_effect = probe monkeypatch.setattr(requests, "Session", lambda: session) args = SimpleNamespace(run_mode=mode, host=host, config_server_host="127.0.0.2", model_name="test") manager.start_readiness_monitor(FakeHttpServerProcess(), args) session.get.assert_called_once() - flag.set_value.assert_called_once_with(0) + shared_int.assert_called_once_with("service_ready") + assert [call.args for call in flag.set_value.call_args_list] == [(0,), (1,)] + + +def test_readiness_monitor_leaves_missing_config_host_to_supervisor(monkeypatch): + from unittest.mock import Mock + from lightllm.server.router.dynamic_prompt import shared_arr + + monkeypatch.setattr(shared_arr, "SharedInt", Mock()) + thread = Mock() + monkeypatch.setattr(start_utils.threading, "Thread", thread) + args = SimpleNamespace(run_mode="config_server", config_server_host=None) + start_utils.SubmoduleManager().start_readiness_monitor(FakeHttpServerProcess(), args) + thread.assert_not_called() From 06a9567739961f1be012e2c96d84e6d88bdc49ef Mon Sep 17 00:00:00 2001 From: sufubao Date: Thu, 22 Oct 2026 08:10:43 +0800 Subject: [PATCH 06/13] fix: address review findings for #1638 --- .../httpserver_for_pd_master/multiprocess.py | 1 + lightllm/server/router/manager.py | 2 +- .../prefill_trans_process.py | 5 +- lightllm/utils/start_utils.py | 27 +++++-- .../test_pd_master_cached_tokens.py | 52 +++++++++++++ .../httpserver/test_pd_websocket_lifecycle.py | 32 ++++++++ .../test_running_request_lifecycle.py | 63 +++++++++++++++ .../mode_backend/test_pd_transfer_logging.py | 23 ++++-- unit_tests/server/test_engine_throughput.py | 78 +++++++++++++++++++ .../test_pd_master_multiprocess_launcher.py | 3 + unit_tests/utils/test_log_utils.py | 37 ++++++++- unit_tests/utils/test_start_utils.py | 65 ++++++++++++++++ 12 files changed, 372 insertions(+), 16 deletions(-) diff --git a/lightllm/server/httpserver_for_pd_master/multiprocess.py b/lightllm/server/httpserver_for_pd_master/multiprocess.py index d0120a10ef..bf5fc505ed 100644 --- a/lightllm/server/httpserver_for_pd_master/multiprocess.py +++ b/lightllm/server/httpserver_for_pd_master/multiprocess.py @@ -89,6 +89,7 @@ def start_multiprocess_pd_master( start_funcs=[start_health_check_process], start_args=[(args,)], ) + process_manager.start_readiness_monitor(proxy_process, args) process_manager.supervise_processes(proxy_process) diff --git a/lightllm/server/router/manager.py b/lightllm/server/router/manager.py index 8caf426de9..dbd6451e3a 100644 --- a/lightllm/server/router/manager.py +++ b/lightllm/server/router/manager.py @@ -260,7 +260,7 @@ async def loop_for_fwd( self.engine_throughput.resume(last_poll) stats_logger.last_time = last_poll seen_ready = True - if self.service_ready.get_value() and stats_logger.active is False and active and not resumed: + if self.service_ready.get_value() and stats_logger.active is not True and active and not resumed: self.engine_throughput.resume(last_poll) stats_logger.last_time = last_poll resumed = True diff --git a/lightllm/server/router/model_infer/mode_backend/pd/prefill_node_impl/prefill_trans_process.py b/lightllm/server/router/model_infer/mode_backend/pd/prefill_node_impl/prefill_trans_process.py index 55debbfc26..f4d1a92adc 100644 --- a/lightllm/server/router/model_infer/mode_backend/pd/prefill_node_impl/prefill_trans_process.py +++ b/lightllm/server/router/model_infer/mode_backend/pd/prefill_node_impl/prefill_trans_process.py @@ -444,7 +444,10 @@ def fail_loop(self): ret = trans_task.createRetObj() self.task_out_queue.put(ret) - logger.error(f"KV transfer failed: {ret}") + if trans_task.error_info == "aborted req": + logger.debug("KV transfer cancelled: %s", ret) + else: + logger.error("KV transfer failed: %s", ret) if trans_task.error_info is not None: self._abort(request_id=trans_task.request_id, error_info=trans_task.error_info) diff --git a/lightllm/utils/start_utils.py b/lightllm/utils/start_utils.py index 6833354511..de54e69aa3 100644 --- a/lightllm/utils/start_utils.py +++ b/lightllm/utils/start_utils.py @@ -16,7 +16,7 @@ class ServiceReadiness: - """Log transitions only; steady probes never produce log traffic.""" + """Announce initial readiness, then require three failures or two recovery probes.""" def __init__(self, flag, args, address, started): self.flag, self.args, self.address, self.started = flag, args, address, started @@ -24,9 +24,15 @@ def __init__(self, flag, args, address, started): self.ever_ready = False self.waiting_logged = False self.probe_failure_logged = False + self.consecutive_failures = 0 + self.consecutive_successes = 0 - def observe(self, status): + def observe(self, status, reason=None): if status == 200: + self.consecutive_failures = 0 + self.consecutive_successes += 1 + if self.ever_ready and not self.ready and self.consecutive_successes < 2: + return if not self.ready: event = "service.recovered" if self.ever_ready else "service.ready" elapsed = round(time.monotonic() - self.started, 2) @@ -51,6 +57,8 @@ def observe(self, status): self.flag.set_value(1) self.ready = self.ever_ready = True else: + self.consecutive_successes = 0 + self.consecutive_failures += 1 if status not in (None, 503) and not self.ever_ready and not self.probe_failure_logged: logger.warning( "event=service.probe_failed address=%s status=%s", @@ -59,13 +67,15 @@ def observe(self, status): extra={"event": "service.probe_failed", "address": self.address, "http_status": status}, ) self.probe_failure_logged = True - if self.ready: + if self.ready and self.consecutive_failures >= 3: + reason = reason or ("probe_connection_failed" if status is None else "readiness_probe_failed") self.flag.set_value(0) self.ready = False logger.warning( - "event=service.unavailable reason=readiness_probe_failed status=%s", + "event=service.unavailable reason=%s status=%s", + reason, status, - extra={"event": "service.unavailable", "reason": "readiness_probe_failed", "http_status": status}, + extra={"event": "service.unavailable", "reason": reason, "http_status": status}, ) elif status == 503 and not self.waiting_logged and not self.ever_ready: logger.info( @@ -105,14 +115,17 @@ def monitor(): with requests.Session() as session: session.trust_env = False while not self.readiness_stop.is_set() and http_server_process.poll() is None: + reason = None try: response = session.get(address + "/readiness", timeout=1) status = response.status_code + except requests.Timeout: + status, reason = None, "probe_timeout" except requests.RequestException: - status = None + status, reason = None, "probe_connection_failed" if self.readiness_stop.is_set() or http_server_process.poll() is not None: break - state.observe(status) + state.observe(status, reason=reason) self.readiness_stop.wait(0.5) threading.Thread(target=monitor, name="service-readiness", daemon=True).start() diff --git a/unit_tests/server/httpserver/test_pd_master_cached_tokens.py b/unit_tests/server/httpserver/test_pd_master_cached_tokens.py index 3788326c15..5a701ca825 100644 --- a/unit_tests/server/httpserver/test_pd_master_cached_tokens.py +++ b/unit_tests/server/httpserver/test_pd_master_cached_tokens.py @@ -836,3 +836,55 @@ async def run(): assert not manager.active_req_ids asyncio.run(run()) + + +@pytest.mark.parametrize("mode", ["disabled", "expired", "retry_success"]) +def test_busy_before_first_token_logs_only_terminal_outcome(monkeypatch, mode): + import time + from unittest.mock import Mock, MagicMock + from lightllm.utils.error_utils import ServerBusyError + + manager = _make_manager(monkeypatch) + manager.first_time_costs = MagicMock() + manager.per_token_costs = MagicMock() + manager.metric_client = MagicMock() + manager.abort = AsyncMock() + manager.enable_pd_node_self_request_limit = mode != "disabled" + manager.pd_node_busy_retry_timeout_seconds = 0 if mode == "expired" else 120 + summary = Mock() + monkeypatch.setattr(pd_master_module, "log_request_summary", summary) + params = SamplingParams() + params.init(SimpleNamespace(encode=lambda *_a, **_kw: []), max_new_tokens=1) + request = SimpleNamespace(headers={}, is_disconnected=lambda: asyncio.sleep(0, False)) + attempts = [] + + async def raw_stream(_p, _d, _prompt, sp, *_args): + attempts.append(sp.group_request_id) + if len(attempts) == 1: + raise ServerBusyError("decode busy") + summary.assert_not_called() + yield sp.group_request_id, "done", {"id": 1, "prompt_tokens": 10, "prompt_cache_len": 0}, FinishStatus( + FinishStatus.FINISHED_LENGTH + ) + + manager.fetch_pd_stream = raw_stream + + async def run(): + stream = manager._generate_one("prompt", params, None, request, time.time(), 80, 10) + if mode == "retry_success": + assert len([item async for item in stream]) == 1 + assert len(attempts) == 2 + else: + with pytest.raises(ServerBusyError): + async for _ in stream: + pass + assert len(attempts) == 1 + summary.assert_called_once() + fields = summary.call_args.kwargs + assert fields["status"] == ("completed" if mode == "retry_success" else "busy") + if mode != "retry_success": + assert "output_tokens" not in fields + assert not manager._summary_states + assert not manager.active_req_ids + + asyncio.run(run()) diff --git a/unit_tests/server/httpserver/test_pd_websocket_lifecycle.py b/unit_tests/server/httpserver/test_pd_websocket_lifecycle.py index 301e92805c..c5c42219be 100644 --- a/unit_tests/server/httpserver/test_pd_websocket_lifecycle.py +++ b/unit_tests/server/httpserver/test_pd_websocket_lifecycle.py @@ -99,3 +99,35 @@ def test_registration_disconnect_keeps_dp_connection_identity(monkeypatch): asyncio.run(api_http_pd.register_and_keep_alive(websocket)) for call in logger.info.call_args_list[1:]: assert "dp_index=2 connection_id=connection-a" in call.args[0] + + +@pytest.mark.parametrize("endpoint", ["kv_move_status", "register_and_keep_alive"]) +@pytest.mark.parametrize("failed", [False, True]) +def test_pd_connection_logs_distinguish_disconnect_from_failure(monkeypatch, endpoint, failed): + from unittest.mock import Mock + from fastapi import WebSocketDisconnect + + manager = _manager(monkeypatch) + logger = Mock() + monkeypatch.setattr(api_http_pd, "logger", logger) + payload = {"mode": "prefill", "client_ip_port": "p:8000", "secret_detail": "not-for-info"} + websocket = _websocket( + receive_text=AsyncMock(return_value=json.dumps(payload)), + receive_bytes=AsyncMock(side_effect=RuntimeError("broken") if failed else WebSocketDisconnect()), + ) + asyncio.run(getattr(api_http_pd, endpoint)(websocket)) + logger.error.assert_not_called() + logger.warning.assert_not_called() + if failed: + logger.exception.assert_called_once() + else: + logger.exception.assert_not_called() + if endpoint == "register_and_keep_alive": + logger.debug.assert_called_once_with( + "PD registration received: peer=%s", "prefill@p:8000 dp_index=None connection_id=None" + ) + manager.remove_pd.assert_awaited_once_with(payload) + assert logger.info.call_count == (2 if failed else 3) + else: + assert logger.info.call_count == (1 if failed else 2) + assert "not-for-info" not in str(logger.info.call_args_list) diff --git a/unit_tests/server/httpserver/test_running_request_lifecycle.py b/unit_tests/server/httpserver/test_running_request_lifecycle.py index 8de695678f..7beb21ec16 100644 --- a/unit_tests/server/httpserver/test_running_request_lifecycle.py +++ b/unit_tests/server/httpserver/test_running_request_lifecycle.py @@ -594,3 +594,66 @@ async def run(): assert summary.call_args.kwargs["reason"] == str(manager._encode.side_effect) asyncio.run(run()) + + +@pytest.mark.parametrize("slave", [False, True]) +@pytest.mark.parametrize( + "last_status,expected", + [ + (FinishStatus.FINISHED_LENGTH, "completed"), + (FinishStatus.FINISHED_ABORTED, "abort"), + (FinishStatus.FINISHED_ERROR, "error"), + (FinishStatus.FINISHED_PD_DECODE_CAPACITY, "continued"), + ], +) +def test_completion_summary_aggregates_choices_and_suppresses_tp_slave(monkeypatch, slave, last_status, expected): + import time + from lightllm.server.httpserver import manager as manager_module + + manager = _make_manager(NodeRole.NORMAL) + manager.is_multinode_tp_slave = slave + manager.args.mtp_step = 1 + manager.first_time_costs = MagicMock() + manager.per_token_costs = MagicMock() + manager.latest_success_infer_time_mark = _ValueMark() + summary = MagicMock() + monkeypatch.setattr(manager_module, "log_request_summary", summary) + params = _sampling_params() + params.best_of = 2 + + async def run(): + event = asyncio.Event() + event.set() + metadata = dict( + prompt_cache_len=1, + cpu_prompt_cache_len=1, + disk_prompt_cache_len=1, + mtp_accepted_token_num=1, + mtp_verify_token_num=2, + mtp_verify_step_num=1, + ) + status = SimpleNamespace( + event=event, + lock=asyncio.Lock(), + has_timed_out_waiting_for_inference=lambda: False, + out_token_info_list=[ + (123, "a", dict(metadata), FinishStatus(FinishStatus.FINISHED_STOP)), + (124, "b", dict(metadata), FinishStatus(last_status)), + ], + ) + result = [ + item async for item in manager._wait_to_token_package(time.time(), [1, 2, 3, 4], 123, params, status, None) + ] + assert len(result) == 2 + if slave: + summary.assert_not_called() + else: + summary.assert_called_once() + fields = summary.call_args.kwargs + assert fields["status"] == expected + assert fields["output_tokens"] == 2 + assert fields["gpu_cache_tokens"] == fields["cpu_cache_tokens"] == fields["disk_cache_tokens"] == 1 + assert fields["mtp_accepted_tokens"] == fields["mtp_verify_steps"] == 2 + assert fields["mtp_verify_tokens"] == 4 + + asyncio.run(run()) diff --git a/unit_tests/server/router/model_infer/mode_backend/test_pd_transfer_logging.py b/unit_tests/server/router/model_infer/mode_backend/test_pd_transfer_logging.py index 36ce21737a..62dded10fd 100644 --- a/unit_tests/server/router/model_infer/mode_backend/test_pd_transfer_logging.py +++ b/unit_tests/server/router/model_infer/mode_backend/test_pd_transfer_logging.py @@ -64,19 +64,23 @@ def get(timeout): stats.flush.assert_called_once() +@pytest.mark.parametrize("side", ["prefill", "decode"]) @pytest.mark.parametrize("error_info", ["aborted req", "transfer timed out"]) -def test_decode_transfer_cancellation_is_debug_and_failures_are_error(monkeypatch, error_info): - worker = decode_trans_process._DecodeTransModule.__new__(decode_trans_process._DecodeTransModule) +def test_transfer_cancellation_is_debug_and_failures_are_error(monkeypatch, error_info, side): + module = prefill_trans_process if side == "prefill" else decode_trans_process + cls = module._PrefillTransModule if side == "prefill" else module._DecodeTransModule + worker = cls.__new__(cls) worker.device_id = 0 worker.page_index_queue = queue.Queue() worker.task_out_queue = queue.Queue() worker.transporter = Mock() worker._abort = Mock() logger = Mock() - monkeypatch.setattr(decode_trans_process, "logger", logger) - monkeypatch.setattr(decode_trans_process.torch.cuda, "set_device", lambda _: None) + monkeypatch.setattr(module, "logger", logger) + monkeypatch.setattr(module.torch.cuda, "set_device", lambda _: None) task = SimpleNamespace( - dst_page_index=None, + src_page_index=2, + dst_page_index=2, xfer_handle=None, request_id=123, error_info=error_info, @@ -91,7 +95,14 @@ class StopLoop(Exception): with pytest.raises(StopLoop): worker.fail_loop() worker._abort.assert_called_once_with(request_id=123, error_info=error_info) - worker.transporter.send_error_info_to_prefill_node.assert_called_once_with(trans_task=task) + notify = ( + worker.transporter.send_error_info_to_decode_node + if side == "prefill" + else worker.transporter.send_error_info_to_prefill_node + ) + notify.assert_called_once_with(trans_task=task) + assert worker.page_index_queue.get_nowait() == 2 + assert worker.task_out_queue.get_nowait() == "result" if error_info == "aborted req": logger.error.assert_not_called() logger.debug.assert_called_once_with("KV transfer cancelled: %s", "result") diff --git a/unit_tests/server/test_engine_throughput.py b/unit_tests/server/test_engine_throughput.py index dc0fe491e8..0be4eefa86 100644 --- a/unit_tests/server/test_engine_throughput.py +++ b/unit_tests/server/test_engine_throughput.py @@ -193,3 +193,81 @@ def test_resume_excludes_idle_time_without_discarding_short_burst(monkeypatch, c assert snapshot["window_s"] == 10 assert snapshot["output_tokens"] == 20 assert snapshot["output_tok_s"] == 2 + + +@pytest.mark.parametrize("node_rank,dp_size", [(0, 1), (1, 1), (1, 2)]) +def test_router_first_burst_excludes_initial_idle(monkeypatch, counters, node_rank, dp_size): + import asyncio + from lightllm.server.router import manager + + now = [0.0] + monkeypatch.setattr(manager.time, "monotonic", lambda: now[0]) + monkeypatch.setattr(log_utils, "get_log_stats_interval", lambda: 10) + logger = Mock() + monkeypatch.setattr(manager, "init_logger", lambda name: logger) + router = manager.RouterManager.__new__(manager.RouterManager) + router.engine_counters = counters + router.service_ready = SimpleNamespace(get_value=lambda: 1) + router.req_queue = Mock() + router.req_queue.get_wait_req_num.return_value = 0 + router.running_batch = None + router.dp_size, router.node_rank, router.dp_size_in_node = dp_size, node_rank, 2 + router.radix_cache_client = None + router.max_total_token_num = 100 + router.get_used_tokens = lambda index: (25, 50)[index] + router._get_paused_req_num = lambda: 0 + router._get_schedule_time_interval = lambda: 0 + writer = engine_stats.EngineCounters(2, 0) + ticks = iter([1, 5, 6, 16]) + + class StopLoop(Exception): + pass + + async def step(): + try: + now[0] = next(ticks) + except StopIteration: + raise StopLoop() + if now[0] == 6: + writer.add(100, 10) + + router._step = step + with pytest.raises(StopLoop): + asyncio.run(router.loop_for_fwd()) + if dp_size == 1 and node_rank != 0: + logger.info.assert_not_called() + else: + logger.info.assert_called_once() + fields = logger.info.call_args.kwargs["extra"] + assert fields["event"] == "engine.stats" + assert fields["window_s"] == 11 + assert fields["output_tokens"] == 10 + assert fields["output_tok_s"] == 0.91 + assert fields["kv_active_pct_by_local_dp"] == [25.0, 50.0] + assert fields["kv_cached_pct_by_local_dp"] == [0.0, 0.0] + + +@pytest.mark.parametrize("leader", [True, False]) +@pytest.mark.parametrize("cached", [0, 40, 100]) +def test_reward_prefill_counts_uncached_tokens_only_on_leader(monkeypatch, counters, leader, cached): + from lightllm.server.router.model_infer.mode_backend.chunked_prefill import impl_for_reward_model as reward + + worker = reward.RewardModelBackend.__new__(reward.RewardModelBackend) + worker.disable_chunked_prefill = True + worker.is_master_in_dp = leader + worker.eos_id = [] + worker.engine_counters = engine_stats.EngineCounters(2, 1) if leader else None + worker.model = Mock() + worker.model.forward.return_value = SimpleNamespace(logits=torch.tensor([0.75])) + req = request(100, cached=cached) + monkeypatch.setattr(reward, "prepare_prefill_inputs", lambda *a, **kw: ("input", [req])) + events = Mock() + worker.reward_prefill(events, [req]) + assert counters.storage.arr.tolist() == [[0, 0], [100 - cached, 1] if leader else [0, 0]] + assert req.cur_output_len == 1 + if leader: + assert req.shm_req.reward_score == 0.75 + assert req.shm_req.shm_cur_output_len == 1 + events.notify_post_handle_and_wait_pre_post_handle.assert_called_once_with() + events.notify_forward_and_wait_post_handle.assert_called_once_with() + events.notify_pre_post_handle.assert_called_once_with() diff --git a/unit_tests/server/test_pd_master_multiprocess_launcher.py b/unit_tests/server/test_pd_master_multiprocess_launcher.py index 9e197378b1..ca11a733ff 100644 --- a/unit_tests/server/test_pd_master_multiprocess_launcher.py +++ b/unit_tests/server/test_pd_master_multiprocess_launcher.py @@ -359,6 +359,7 @@ def terminate(): terminate_all_processes=Mock(side_effect=terminate), setup_signal_handlers=Mock(), supervise_processes=Mock(), + start_readiness_monitor=Mock(), ) ports.close.side_effect = lambda: events.append("close ports") monkeypatch.setattr(multiprocess, "process_manager", manager) @@ -379,6 +380,7 @@ def terminate(): ports=ports, haproxy_binary="/bundled/haproxy", ) + manager.start_readiness_monitor.assert_called_once_with(proxy, args) manager.supervise_processes.assert_called_once_with(proxy) manager.setup_signal_handlers.assert_called_once_with( proxy, @@ -393,6 +395,7 @@ def terminate(): ports=ports, haproxy_binary="/bundled/haproxy", ) + manager.start_readiness_monitor.assert_not_called() manager.supervise_processes.assert_not_called() manager.terminate_all_processes.assert_called_once_with() diff --git a/unit_tests/utils/test_log_utils.py b/unit_tests/utils/test_log_utils.py index 946d072891..b222cb2fa3 100644 --- a/unit_tests/utils/test_log_utils.py +++ b/unit_tests/utils/test_log_utils.py @@ -128,7 +128,9 @@ def test_json_preserves_context_multiline_and_traceback(): assert event["tp_rank"] == 3 and event["dp_rank"] == 1 assert event["request_id"] == 42 assert "ValueError: failure" in event["exception"] - assert event["timestamp"].endswith(("+00:00", "+08:00")) or "T" in event["timestamp"] + from datetime import datetime + + assert datetime.fromisoformat(event["timestamp"]).utcoffset() is not None @pytest.mark.parametrize( @@ -149,6 +151,7 @@ def test_json_preserves_context_multiline_and_traceback(): def test_invalid_configuration_fails_explicitly(settings): env = {key: value for key, value in os.environ.items() if not key.startswith("LIGHTLLM_LOG_")} env.update(settings) + env["PYTHONPATH"] = str(Path(__file__).resolve().parents[2]) + os.pathsep + env.get("PYTHONPATH", "") result = subprocess.run( [sys.executable, "-c", "from lightllm.utils import log_utils"], env=env, @@ -684,3 +687,35 @@ def test_failure_summary_level_and_sampling(monkeypatch, caplog, status, level, assert record.levelno == level assert record.event == event assert record.reason == "bounded reason" + + +@pytest.mark.parametrize("separator", ["-", "_"]) +def test_all_logging_cli_options_survive_start_args_and_override_environment(separator): + output = _run_logging( + f""" + import argparse + import os + from lightllm.server.api_cli import add_cli_args + from lightllm.server.core.objs.start_args_type import StartArgs + from lightllm.utils.log_utils import configure_logging_from_args + flags = [ + ("log_level", "debug"), ("log_format", "json"), + ("log_module_levels", '{{"lightllm.test": "ERROR"}}'), + ("log_request_sample_rate", "0.5"), ("log_stats_interval", "2.5"), + ("log_http_level", "error"), + ] + argv = [] + for name, value in flags: + argv.extend(["--" + name.replace("_", {separator!r}), value]) + argv.append("--" + "enable_http_access_log".replace("_", {separator!r})) + args = StartArgs(**vars(add_cli_args(argparse.ArgumentParser()).parse_args(argv))) + configure_logging_from_args(args) + for name, value in flags: + assert os.environ["LIGHTLLM_" + name.upper()] == value + assert os.environ["LIGHTLLM_LOG_HTTP_ACCESS"] == "1" + print("configured") + """, + LIGHTLLM_LOG_LEVEL="critical", + LIGHTLLM_LOG_HTTP_ACCESS="0", + ) + assert output.strip() == "configured" diff --git a/unit_tests/utils/test_start_utils.py b/unit_tests/utils/test_start_utils.py index db41aa873d..a75a47292d 100644 --- a/unit_tests/utils/test_start_utils.py +++ b/unit_tests/utils/test_start_utils.py @@ -346,9 +346,13 @@ def test_readiness_waits_for_bound_endpoint_and_pd_topology(monkeypatch): assert logger.info.call_args.kwargs["extra"]["event"] == "service.ready" state.observe(None) state.observe(None) + logger.warning.assert_not_called() + state.observe(None) logger.warning.assert_called_once() assert flag.set_value.call_args.args == (0,) state.observe(200) + assert not state.ready + state.observe(200) assert logger.info.call_args.kwargs["extra"]["event"] == "service.recovered" @@ -401,3 +405,64 @@ def test_readiness_monitor_leaves_missing_config_host_to_supervisor(monkeypatch) args = SimpleNamespace(run_mode="config_server", config_server_host=None) start_utils.SubmoduleManager().start_readiness_monitor(FakeHttpServerProcess(), args) thread.assert_not_called() + + +@pytest.mark.parametrize("failure", [None, 503, 500]) +def test_readiness_tolerates_isolated_failures_and_confirms_recovery(monkeypatch, failure): + from unittest.mock import Mock + + logger = Mock() + monkeypatch.setattr(start_utils, "logger", logger) + flag = Mock() + state = start_utils.ServiceReadiness(flag, SimpleNamespace(run_mode="normal", model_name="test"), "http://test", 0) + state.observe(200) + for _ in range(2): + state.observe(failure) + assert state.ready + state.observe(200) + logger.warning.assert_not_called() + flag.set_value.assert_called_once_with(1) + for _ in range(3): + state.observe(failure) + assert not state.ready + assert flag.set_value.call_args.args == (0,) + logger.warning.assert_called_once() + state.observe(200) + assert not state.ready + state.observe(failure) + state.observe(200) + assert not state.ready + state.observe(200) + assert state.ready + assert logger.info.call_args.kwargs["extra"]["event"] == "service.recovered" + + +@pytest.mark.parametrize("failure,reason", [("timeout", "probe_timeout"), ("connection", "probe_connection_failed")]) +def test_readiness_monitor_reports_transport_reason_and_stops_after_http_exit(monkeypatch, failure, reason): + import requests + from unittest.mock import Mock, MagicMock + from lightllm.server.router.dynamic_prompt import shared_arr + from lightllm.utils import shm_port_args + + manager = start_utils.SubmoduleManager() + logger = Mock() + flag = Mock() + monkeypatch.setattr(start_utils, "logger", logger) + monkeypatch.setattr(shared_arr, "SharedInt", lambda _: flag) + monkeypatch.setattr(shm_port_args, "get_shm_port_args", lambda: SimpleNamespace(port=8000)) + monkeypatch.setattr(start_utils.threading, "Thread", lambda target, **kw: SimpleNamespace(start=target)) + session = MagicMock() + session.__enter__.return_value = session + error = requests.Timeout if failure == "timeout" else requests.ConnectionError + session.get.side_effect = [SimpleNamespace(status_code=200), error(), error(), error()] + monkeypatch.setattr(requests, "Session", lambda: session) + process = Mock() + process.poll.side_effect = [None] * 8 + [1] + monkeypatch.setattr(manager.readiness_stop, "wait", lambda _: None) + args = SimpleNamespace(run_mode="normal", host="127.0.0.1", model_name="test") + manager.start_readiness_monitor(process, args) + assert session.get.call_count == 4 + logger.warning.assert_called_once() + assert logger.warning.call_args.kwargs["extra"]["reason"] == reason + assert [call.args for call in flag.set_value.call_args_list] == [(0,), (1,), (0,)] + session.__exit__.assert_called_once() From 586b6c48e345e311cc26970be95345ab3eaea4b0 Mon Sep 17 00:00:00 2001 From: sufubao Date: Thu, 22 Oct 2026 08:15:55 +0800 Subject: [PATCH 07/13] fix(logging): retain terminal busy summaries across PD continuations --- .../httpserver_for_pd_master/manager.py | 11 +- .../test_pd_master_cached_tokens.py | 44 ++++++++ unit_tests/server/test_engine_throughput.py | 101 ++++++++++++++++++ unit_tests/server/test_logging.py | 45 ++++++++ unit_tests/utils/test_start_utils.py | 28 +++++ 5 files changed, 225 insertions(+), 4 deletions(-) diff --git a/lightllm/server/httpserver_for_pd_master/manager.py b/lightllm/server/httpserver_for_pd_master/manager.py index 10f301a305..30106b19ea 100644 --- a/lightllm/server/httpserver_for_pd_master/manager.py +++ b/lightllm/server/httpserver_for_pd_master/manager.py @@ -262,6 +262,7 @@ async def _generate_one( has_yielded_result = False while True: + summary_state = {} try: generator = self._generate_one_attempt( prompt, @@ -271,6 +272,7 @@ async def _generate_one( start_time, origin_request_id, input_token_num, + summary_state=summary_state, ) async with aclosing(generator): async for result in generator: @@ -280,14 +282,14 @@ async def _generate_one( except ServerBusyError: # 关闭节点自限流时，不启用与该策略配套的 busy 重试，直接透传异常。 if not self.enable_pd_node_self_request_limit: - if not has_yielded_result: + if not summary_state.get("logged"): log_request_summary(origin_request_id, status="busy", timing_scope="end_to_end") raise # 已向客户端输出 token 后不能从头生成，否则会产生重复内容。 elapsed_seconds = time.monotonic() - retry_start_time if has_yielded_result or elapsed_seconds >= self.pd_node_busy_retry_timeout_seconds: - if not has_yielded_result: + if not summary_state.get("logged"): log_request_summary(origin_request_id, status="busy", timing_scope="end_to_end") raise @@ -314,6 +316,7 @@ async def _generate_one_attempt( start_time: float, origin_request_id: int, input_token_num: int, + summary_state: Optional[dict] = None, ): """执行一次完整的单 choice 生成尝试。 @@ -326,7 +329,7 @@ async def _generate_one_attempt( d_node = None pending_prefill_load_chars = None results_generator = None - summary_state = {} + summary_state = summary_state if summary_state is not None else {} if not hasattr(self, "_summary_states"): self._summary_states = {} self.active_req_ids.add(origin_request_id) @@ -427,7 +430,7 @@ async def _generate_one_attempt( sampling_params.constraint_out_params.pd_previous_output_len = max( 0, block_token_num - input_token_num ) - summary_state = {} + summary_state.clear() self._summary_states[origin_request_id] = summary_state results_generator = self._wait_to_token_package( p_node, diff --git a/unit_tests/server/httpserver/test_pd_master_cached_tokens.py b/unit_tests/server/httpserver/test_pd_master_cached_tokens.py index 5a701ca825..aecbdf3d87 100644 --- a/unit_tests/server/httpserver/test_pd_master_cached_tokens.py +++ b/unit_tests/server/httpserver/test_pd_master_cached_tokens.py @@ -888,3 +888,47 @@ async def run(): assert not manager.active_req_ids asyncio.run(run()) + + +@pytest.mark.parametrize("retry_enabled", [False, True]) +def test_pd_continuation_busy_before_first_token_emits_terminal_summary(monkeypatch, retry_enabled): + import time + from unittest.mock import MagicMock + from lightllm.utils.error_utils import ServerBusyError + + manager = _make_manager(monkeypatch) + manager.enable_pd_node_self_request_limit = retry_enabled + manager.first_time_costs = MagicMock() + manager.per_token_costs = MagicMock() + manager.abort = AsyncMock() + summary = MagicMock() + monkeypatch.setattr(pd_master_module, "log_request_summary", summary) + params = SamplingParams() + params.init(SimpleNamespace(encode=lambda *_args, **_kwargs: []), max_new_tokens=4) + request = SimpleNamespace(headers={}, is_disconnected=lambda: asyncio.sleep(0, False)) + attempts = 0 + + async def raw_stream(_p, _d, _prompt, sp, *_args): + nonlocal attempts + attempts += 1 + metadata = {"id": 1, "prompt_tokens": 10, "prompt_cache_len": 3} + if attempts == 1: + yield sp.group_request_id, "first", dict(metadata), FinishStatus() + yield sp.group_request_id, "", dict(metadata), FinishStatus(FinishStatus.FINISHED_PD_DECODE_CAPACITY) + else: + raise ServerBusyError("segment busy before first token") + + manager.fetch_pd_stream = raw_stream + + async def consume(): + return [item async for item in manager._generate_one("prompt", params, None, request, time.time(), 80, 10)] + + async def run(): + with pytest.raises(ServerBusyError): + await consume() + assert [call.kwargs["status"] for call in summary.call_args_list] == ["continued", "busy"] + assert attempts == 2 + assert not manager._summary_states + assert not manager.active_req_ids + + asyncio.run(run()) diff --git a/unit_tests/server/test_engine_throughput.py b/unit_tests/server/test_engine_throughput.py index 0be4eefa86..23425c35a0 100644 --- a/unit_tests/server/test_engine_throughput.py +++ b/unit_tests/server/test_engine_throughput.py @@ -271,3 +271,104 @@ def test_reward_prefill_counts_uncached_tokens_only_on_leader(monkeypatch, count events.notify_post_handle_and_wait_pre_post_handle.assert_called_once_with() events.notify_forward_and_wait_post_handle.assert_called_once_with() events.notify_pre_post_handle.assert_called_once_with() + + +@pytest.mark.parametrize("tp_rank", [0, 1]) +def test_init_model_attaches_only_the_dp_leaders_counter_row(monkeypatch, counters, tp_rank): + from lightllm.server.core.objs.start_args_type import StartArgs + + distributed = Mock() + monkeypatch.setattr(base_backend, "init_distributed_env", distributed) + ranks = { + "get_node_world_size": 4, + "get_current_rank_in_node": 2 + tp_rank, + "get_current_device_id": 2 + tp_rank, + "get_current_rank_in_dp": tp_rank, + "get_global_dp_rank": 1, + "get_dp_rank_in_node": 1, + "get_dp_world_size": 2, + "get_global_rank": 2 + tp_rank, + "get_global_world_size": 4, + "get_dp_size": 2, + } + for name, value in ranks.items(): + monkeypatch.setattr(base_backend, name, lambda value=value: value) + monkeypatch.setattr(log_utils, "set_log_context", Mock()) + + class ModelSetupBoundary(Exception): + pass + + monkeypatch.setattr(base_backend.dist_group_manager, "create_groups", Mock(side_effect=ModelSetupBoundary())) + worker = base_backend.ModeBackend.__new__(base_backend.ModeBackend) + with pytest.raises(ModelSetupBoundary): + worker.init_model( + { + "args": StartArgs(dp=2), + "world_size": 4, + "load_way": "HF", + "weight_dir": "unused", + "max_total_token_num": 100, + } + ) + distributed.assert_called_once() + assert worker.dp_rank_in_node == 1 + if tp_rank == 0: + worker.engine_counters.add(7, 3) + assert counters.storage.arr.tolist() == [[0, 0], [7, 3]] + else: + assert worker.engine_counters is None + assert counters.totals() == (0, 0) + + +@pytest.mark.parametrize("node_rank,dp_size", [(0, 1), (1, 1), (0, 2), (1, 2)]) +def test_router_statistics_wait_for_readiness_and_emit_once_per_node(monkeypatch, counters, node_rank, dp_size): + import asyncio + from lightllm.server.router import manager as router + + now = [0.0] + clock = SimpleNamespace(monotonic=lambda: now[0]) + for module in (router, engine_stats, log_utils): + monkeypatch.setattr(module, "time", clock) + monkeypatch.setattr(log_utils, "get_log_stats_interval", lambda: 10) + logger = Mock() + monkeypatch.setattr(router, "init_logger", lambda name: logger) + manager = router.RouterManager.__new__(router.RouterManager) + manager.engine_counters = counters + manager.running_batch = None + manager.req_queue = Mock() + manager.req_queue.get_wait_req_num.return_value = 2 + manager.node_rank, manager.dp_size, manager.dp_size_in_node = node_rank, dp_size, dp_size + manager.max_total_token_num = 100 + manager.get_used_tokens = lambda index: 10 * (index + 1) + manager.radix_cache_client = SimpleNamespace(get_unrefed_tokens_num=lambda index: 5 * (index + 1)) + manager._get_paused_req_num = lambda: 1 + manager._get_schedule_time_interval = lambda: 0 + manager.service_ready = SimpleNamespace(get_value=lambda: int(now[0] >= 20)) + writer = engine_stats.EngineCounters(2, 0) + + class StopLoop(Exception): + pass + + async def step(): + now[0] += 10 + if now[0] > 30: + raise StopLoop() + if now[0] <= 20: + logger.info.assert_not_called() + writer.add(100, 10) + + manager._step = step + with pytest.raises(StopLoop): + asyncio.run(manager.loop_for_fwd()) + if dp_size == 1 and node_rank != 0: + logger.info.assert_not_called() + return + assert logger.info.call_count == 2 + first, second = [call.kwargs["extra"] for call in logger.info.call_args_list] + assert first["prefill_tokens"] == 200 and second["prefill_tokens"] == 100 + assert first["window_s"] == second["window_s"] == 10 + assert first["output_tok_s"] == 2 and second["output_tok_s"] == 1 + assert first["running"] == 0 and first["waiting"] == 2 and first["paused"] == 1 + assert first["kv_active_pct_by_local_dp"] == [10.0 * (i + 1) for i in range(dp_size)] + assert first["kv_cached_pct_by_local_dp"] == [5.0 * (i + 1) for i in range(dp_size)] + assert first["kv_capacity_tokens_per_dp"] == 100 diff --git a/unit_tests/server/test_logging.py b/unit_tests/server/test_logging.py index 0a9c284139..0fa4424e30 100644 --- a/unit_tests/server/test_logging.py +++ b/unit_tests/server/test_logging.py @@ -144,3 +144,48 @@ def test_unsafe_client_request_ids_are_replaced(value): request_id = new_http_request_id([(b"x-request-id", value)]) assert len(request_id) == 32 int(request_id, 16) + + +@pytest.mark.parametrize("mode", ["normal", "prefill", "decode", "pd_master", "config_server"]) +def test_launchers_monitor_the_bound_http_process_before_supervision(monkeypatch, mode): + from types import SimpleNamespace + from unittest.mock import call + from lightllm.server import api_start + from lightllm.server.core.objs.start_args_type import StartArgs + + args = StartArgs(run_mode=mode, model_dir="unused", health_monitor=False) + manager = Mock() + monkeypatch.setattr(api_start, "process_manager", manager) + monkeypatch.setattr(api_start, "_launch_subprocesses", lambda args: manager) + for name in ( + "_set_envs_and_config", + "set_unique_server_name", + "auto_set_max_req_total_len", + "auto_set_response_parsers", + "validate_ports", + "set_env_start_args", + "log_startup_config", + ): + monkeypatch.setattr(api_start, name, Mock()) + monkeypatch.setattr( + api_start, "get_shm_port_args", lambda **kwargs: SimpleNamespace(port=8001, config_server_port=9001) + ) + monkeypatch.setattr(api_start, "_hypercorn_config_args", lambda args: []) + monkeypatch.setattr(api_start, "get_http_logging_args", lambda **kwargs: []) + process = Mock() + popen = Mock(return_value=process) + monkeypatch.setattr(api_start.subprocess, "Popen", popen) + launcher = ( + api_start.pd_master_start + if mode == "pd_master" + else api_start.config_server_start + if mode == "config_server" + else api_start.normal_or_p_d_start + ) + launcher(args) + popen.assert_called_once() + assert manager.mock_calls[-3:] == [ + call.start_readiness_monitor(process, args), + call.setup_signal_handlers(process), + call.supervise_processes(process), + ] diff --git a/unit_tests/utils/test_start_utils.py b/unit_tests/utils/test_start_utils.py index a75a47292d..e9380f47c9 100644 --- a/unit_tests/utils/test_start_utils.py +++ b/unit_tests/utils/test_start_utils.py @@ -466,3 +466,31 @@ def test_readiness_monitor_reports_transport_reason_and_stops_after_http_exit(mo assert logger.warning.call_args.kwargs["extra"]["reason"] == reason assert [call.args for call in flag.set_value.call_args_list] == [(0,), (1,), (0,)] session.__exit__.assert_called_once() + + +@pytest.mark.parametrize( + "poll_results,probe_count,ready", [([0], 0, False), ([None, 0], 1, False), ([None, None, 0], 1, True)] +) +def test_readiness_monitor_stops_when_http_process_exits(monkeypatch, poll_results, probe_count, ready): + import requests + from unittest.mock import Mock, MagicMock + from lightllm.server.router.dynamic_prompt import shared_arr + from lightllm.utils import shm_port_args + + manager = start_utils.SubmoduleManager() + flag = Mock() + monkeypatch.setattr(shared_arr, "SharedInt", lambda name: flag) + monkeypatch.setattr(shm_port_args, "get_shm_port_args", lambda: SimpleNamespace(port=8000)) + monkeypatch.setattr(start_utils.threading, "Thread", lambda target, **kw: SimpleNamespace(start=target)) + session = MagicMock() + session.__enter__.return_value = session + session.get.return_value = SimpleNamespace(status_code=200) + monkeypatch.setattr(requests, "Session", lambda: session) + monkeypatch.setattr(manager.readiness_stop, "wait", Mock()) + process = Mock() + process.poll.side_effect = poll_results + args = SimpleNamespace(run_mode="normal", host="127.0.0.1", model_name="test") + manager.start_readiness_monitor(process, args) + assert session.get.call_count == probe_count + assert [call.args for call in flag.set_value.call_args_list] == ([(0,), (1,)] if ready else [(0,)]) + session.__exit__.assert_called_once() From e852202e1a14a8ff07a663bdc71fa67d54d6f279 Mon Sep 17 00:00:00 2001 From: sufubao Date: Thu, 22 Oct 2026 08:43:40 +0800 Subject: [PATCH 08/13] fix: address review findings for #1638 --- .../httpserver/test_pd_generate_error.py | 25 ++++++++-- .../test_pd_master_cached_tokens.py | 43 +++++++++++++++++ .../httpserver/test_pd_split_connections.py | 9 +++- .../mode_backend/test_pd_dynamic_split.py | 4 +- .../mode_backend/test_pd_prefill_abort.py | 8 ++-- .../mode_backend/test_pd_transfer_logging.py | 21 ++++++++ ...test_config_server_registration_logging.py | 48 +++++++++++++++++++ unit_tests/server/test_pd_master_mode.py | 13 +++++ unit_tests/utils/test_log_utils.py | 1 + 9 files changed, 160 insertions(+), 12 deletions(-) create mode 100644 unit_tests/server/test_config_server_registration_logging.py diff --git a/unit_tests/server/httpserver/test_pd_generate_error.py b/unit_tests/server/httpserver/test_pd_generate_error.py index 29e9d2ab39..27fe10b993 100644 --- a/unit_tests/server/httpserver/test_pd_generate_error.py +++ b/unit_tests/server/httpserver/test_pd_generate_error.py @@ -510,14 +510,16 @@ def test_pd_request_context_isolated_and_reset_on_failure(): from lightllm.utils.log_utils import get_http_request_id async def run(): + observed = [] + class ContextManager: args = SimpleNamespace(run_mode="prefill") async def generate(self, **kwargs): - assert get_http_request_id() in ("client-a", "client-b") - expected = get_http_request_id() + expected = kwargs["prompt"] + observed.append((expected, "before", get_http_request_id())) await asyncio.sleep(0) - assert get_http_request_id() == expected + observed.append((expected, "after", get_http_request_id())) if expected == "client-b": raise RuntimeError("test failure") yield 123, "token", {}, FinishStatus(FinishStatus.FINISHED_STOP) @@ -525,19 +527,32 @@ async def generate(self, **kwargs): async def generate(client_id): params = SamplingParams() params.group_request_id = 123 + upload = AsyncMock() await _pd_process_generate( ContextManager(), - "prompt", + client_id, params, None, AsyncMock(), - AsyncMock(), + upload, asyncio.Event(), http_request_id=client_id, ) assert get_http_request_id() == "-" + if client_id == "client-b": + upload.send.assert_awaited_once() + assert pickle.loads(upload.send.call_args.args[0]) == ( + ObjType.PD_UPLOAD_GENERATE_ERROR, + 123, + "RuntimeError: test failure", + ) + else: + upload.send.assert_not_awaited() await asyncio.gather(generate("client-a"), generate("client-b")) + assert sorted(observed) == sorted( + (client, phase, client) for client in ("client-a", "client-b") for phase in ("before", "after") + ) assert get_http_request_id() == "-" asyncio.run(run()) diff --git a/unit_tests/server/httpserver/test_pd_master_cached_tokens.py b/unit_tests/server/httpserver/test_pd_master_cached_tokens.py index aecbdf3d87..e73257f3fc 100644 --- a/unit_tests/server/httpserver/test_pd_master_cached_tokens.py +++ b/unit_tests/server/httpserver/test_pd_master_cached_tokens.py @@ -932,3 +932,46 @@ async def run(): assert not manager.active_req_ids asyncio.run(run()) + + +@pytest.mark.parametrize("headers, expected", [({"X-Request-Id": "header-id"}, "header-id"), ({}, "context-id")]) +def test_fetch_pd_stream_sends_request_context_to_both_nodes(headers, expected): + from lightllm.server.pd_io_struct import ObjType + from lightllm.utils.log_utils import set_http_request_id, reset_http_request_id + + class StopAfterDispatch(Exception): + pass + + async def run(): + manager = HttpServerManagerForPDMaster.__new__(HttpServerManagerForPDMaster) + manager.args = SimpleNamespace(pd_node_id=1) + manager.req_id_to_out_inf = {} + params = SamplingParams() + params.group_request_id = 123 + params.max_new_tokens = 8 + p_node = SimpleNamespace(websocket=SimpleNamespace(send_bytes=AsyncMock())) + d_node = SimpleNamespace(websocket=SimpleNamespace(send_bytes=AsyncMock())) + + async def wait(event, *args, stage, **kwargs): + if stage == "prefill": + event.prompt_ids = [1, 2, 3] + else: + raise StopAfterDispatch() + + manager._wait_for_event_or_disconnect = wait + token = set_http_request_id("context-id") + try: + stream = manager.fetch_pd_stream(p_node, d_node, "prompt", params, None, SimpleNamespace(headers=headers)) + with pytest.raises(StopAfterDispatch): + await anext(stream) + finally: + reset_http_request_id(token) + for node, prompt, max_tokens in ((p_node, "prompt", 1), (d_node, [1, 2, 3], 8)): + node.websocket.send_bytes.assert_awaited_once() + kind, (sent_prompt, sent_params, _), metadata = pickle.loads(node.websocket.send_bytes.call_args.args[0]) + assert kind == ObjType.REQ + assert sent_prompt == prompt + assert sent_params.max_new_tokens == max_tokens + assert metadata == {"http_request_id": expected} + + asyncio.run(run()) diff --git a/unit_tests/server/httpserver/test_pd_split_connections.py b/unit_tests/server/httpserver/test_pd_split_connections.py index 8bfa614b5e..8f6505b0cc 100644 --- a/unit_tests/server/httpserver/test_pd_split_connections.py +++ b/unit_tests/server/httpserver/test_pd_split_connections.py @@ -148,8 +148,9 @@ async def stop_after_connections(_seconds): asyncio.run(run()) +@pytest.mark.parametrize("with_metadata", [False, True]) @pytest.mark.parametrize("dp_index", [None, 0, 3]) -def test_connection_registers_rank_binds_requests_and_reports_rank_load(monkeypatch, dp_index): +def test_connection_registers_rank_binds_requests_and_reports_rank_load(monkeypatch, dp_index, with_metadata): from lightllm.server import api_http async def run(): @@ -171,8 +172,10 @@ async def run(): params.suggested_dp_index = -1 dispatched = asyncio.Event() payloads = [] + received_ids = [] async def generate(**kwargs): + received_ids.append(kwargs["http_request_id"]) req_params = kwargs["sampling_params"] assert req_params.suggested_dp_index == (-1 if dp_index is None else dp_index) assert kwargs["pd_upload_websocket"] is websocket @@ -185,7 +188,8 @@ async def receive(): nonlocal requests requests += 1 if requests == 1: - return pickle.dumps((ObjType.REQ, ("prompt", params, None))) + payload = (ObjType.REQ, ("prompt", params, None)) + return pickle.dumps(payload + ({"http_request_id": "client-id"},) if with_metadata else payload) await dispatched.wait() while not any(isinstance(payload, tuple) and payload[0] == ObjType.TOKEN_PACKS for payload in payloads): await asyncio.sleep(0) @@ -208,6 +212,7 @@ async def __aexit__(self, *_args): monkeypatch.setattr(pd_loop.websockets, "connect", lambda *_args, **_kwargs: Connection()) monkeypatch.setattr(pd_loop, "_pd_process_generate", generate) await asyncio.wait_for(pd_loop._pd_handle_task(manager, PD_Master_Obj(1, "master:8000"), dp_index), 2) + assert received_ids == ["client-id" if with_metadata else None] assert payloads[0].get("dp_index") == dp_index assert "dp_index" in payloads[0] for payload in payloads[1:]: diff --git a/unit_tests/server/router/model_infer/mode_backend/test_pd_dynamic_split.py b/unit_tests/server/router/model_infer/mode_backend/test_pd_dynamic_split.py index c498093fe2..2c10e1bd95 100644 --- a/unit_tests/server/router/model_infer/mode_backend/test_pd_dynamic_split.py +++ b/unit_tests/server/router/model_infer/mode_backend/test_pd_dynamic_split.py @@ -77,7 +77,7 @@ def test_pd_decode_capacity_shortage_is_delayed_for_overlap(monkeypatch): assert req.sampling_param.shm_param.max_new_tokens == 65535 assert not req.wait_pause filter_reqs.assert_called_once_with(finished_reqs=[]) - assert logger.info.call_args.args[0] == ( + logger.warning.assert_called_once_with( "force early finish for PD decode req_id=1 because token capacity is insufficient" ) @@ -96,7 +96,7 @@ def test_pd_decode_capacity_shortage_is_filtered_without_overlap(monkeypatch): assert req.sampling_param.shm_param.max_new_tokens == 65535 assert not req.wait_pause filter_reqs.assert_called_once_with(finished_reqs=[req]) - assert logger.info.call_args.args[0] == ( + logger.warning.assert_called_once_with( "force early finish for PD decode req_id=1 because token capacity is insufficient" ) diff --git a/unit_tests/server/router/model_infer/mode_backend/test_pd_prefill_abort.py b/unit_tests/server/router/model_infer/mode_backend/test_pd_prefill_abort.py index cb250b0dc7..35c73e7560 100644 --- a/unit_tests/server/router/model_infer/mode_backend/test_pd_prefill_abort.py +++ b/unit_tests/server/router/model_infer/mode_backend/test_pd_prefill_abort.py @@ -168,9 +168,10 @@ def test_prefill_abort_only_fails_tasks_not_submitted_to_transporter(): } +@pytest.mark.parametrize("debug", [False, True]) @pytest.mark.parametrize("telemetry", [False, True]) @pytest.mark.parametrize("notify_fails", [False, True]) -def test_completed_transfer_reaches_recycling_queue_with_compact_logging(monkeypatch, telemetry, notify_fails): +def test_completed_transfer_reaches_recycling_queue_with_compact_logging(monkeypatch, telemetry, notify_fails, debug): task = SimpleNamespace( xfer_handle=object(), request_id=123, @@ -191,7 +192,7 @@ def test_completed_transfer_reaches_recycling_queue_with_compact_logging(monkeyp postDuration=2, ) logger = MagicMock() - logger.isEnabledFor.return_value = True + logger.isEnabledFor.return_value = debug monkeypatch.setattr(prefill_trans_process, "logger", logger) if notify_fails: module.transporter.send_write_done_task_to_decode_node.side_effect = ConnectionError("peer gone") @@ -204,11 +205,12 @@ def test_completed_transfer_reaches_recycling_queue_with_compact_logging(monkeyp else: assert module.success_queue.get_nowait() is task assert module.failed_queue.empty() - if telemetry: + if telemetry and debug: module.transporter.nixl_agent.get_xfer_telemetry.assert_called_once_with(task.xfer_handle) assert logger.debug.call_args.kwargs["extra"]["bytes_total"] == 4096 else: module.transporter.nixl_agent.get_xfer_telemetry.assert_not_called() + logger.debug.assert_not_called() @pytest.mark.parametrize("rank", [0, 1]) diff --git a/unit_tests/server/router/model_infer/mode_backend/test_pd_transfer_logging.py b/unit_tests/server/router/model_infer/mode_backend/test_pd_transfer_logging.py index 62dded10fd..57cf8cf82a 100644 --- a/unit_tests/server/router/model_infer/mode_backend/test_pd_transfer_logging.py +++ b/unit_tests/server/router/model_infer/mode_backend/test_pd_transfer_logging.py @@ -108,3 +108,24 @@ class StopLoop(Exception): logger.debug.assert_called_once_with("KV transfer cancelled: %s", "result") else: logger.error.assert_called_once_with("KV transfer failed: %s", "result") + + +def test_missing_nixl_fails_before_initializing_transport(monkeypatch): + from lightllm.server.router.model_infer.mode_backend.pd import nixl_kv_transporter, kv_transporter + + monkeypatch.setattr(nixl_kv_transporter, "NixlWrapper", None) + with pytest.raises(ImportError, match="pd_trans_mode=nixl requires NIXL; install NIXL"): + NixlKVTransporter(node_id=1, tp_idx=0, kv_move_buffer=None) + with pytest.raises(ImportError, match="select pd_trans_mode=nccl"): + kv_transporter.create_kv_transporter(SimpleNamespace(pd_trans_mode="nixl"), 1, 0, None) + + +def test_nixl_factory_forwards_transport_arguments(monkeypatch): + from lightllm.server.router.model_infer.mode_backend.pd import nixl_kv_transporter, kv_transporter + + constructor = Mock() + monkeypatch.setattr(nixl_kv_transporter, "NixlKVTransporter", constructor) + buffer = object() + result = kv_transporter.create_kv_transporter(SimpleNamespace(pd_trans_mode="nixl"), 7, 2, buffer) + constructor.assert_called_once_with(node_id=7, tp_idx=2, kv_move_buffer=buffer) + assert result is constructor.return_value diff --git a/unit_tests/server/test_config_server_registration_logging.py b/unit_tests/server/test_config_server_registration_logging.py new file mode 100644 index 0000000000..af19987181 --- /dev/null +++ b/unit_tests/server/test_config_server_registration_logging.py @@ -0,0 +1,48 @@ +import asyncio +import pickle +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import pytest +from fastapi import WebSocketDisconnect + +from lightllm.server.config_server import api_http + + +@pytest.mark.parametrize( + "endpoint, registry_name", + [ + ("websocket_endpoint", "registered_pd_master_objs"), + ("visual_websocket_endpoint", "registered_visual_server_objs"), + ], +) +@pytest.mark.parametrize("disconnect", [False, True]) +def test_registration_logs_and_unregisters_on_exit(monkeypatch, endpoint, registry_name, disconnect): + other = object() + registry = {"other": other} + monkeypatch.setattr(api_http, registry_name, registry) + logger = Mock() + monkeypatch.setattr(api_http, "logger", logger) + error = WebSocketDisconnect() if disconnect else RuntimeError("receive failed") + + async def receive_text(): + assert registry["node"].node_id == "node" + raise error + + websocket = SimpleNamespace( + client=("127.0.0.1", 1234), + accept=AsyncMock(), + receive_bytes=AsyncMock(return_value=pickle.dumps(SimpleNamespace(node_id="node"))), + receive_text=receive_text, + ) + asyncio.run(getattr(api_http, endpoint)(websocket)) + assert registry == {"other": other} + websocket.accept.assert_awaited_once() + logger.error.assert_not_called() + if disconnect: + logger.exception.assert_not_called() + assert any(call.args[0] == "Registration connection closed: peer=%s:%s" for call in logger.info.call_args_list) + else: + logger.exception.assert_called_once() + assert "receive failed" in logger.exception.call_args.args[0] + assert "removed" in logger.info.call_args.args[0] diff --git a/unit_tests/server/test_pd_master_mode.py b/unit_tests/server/test_pd_master_mode.py index a5bac07641..482c5988ad 100644 --- a/unit_tests/server/test_pd_master_mode.py +++ b/unit_tests/server/test_pd_master_mode.py @@ -514,6 +514,12 @@ def unexpected_startup(*args, **kwargs): def test_pd_master_cli_dispatches_to_the_requested_startup_mode(monkeypatch, workers): from lightllm.server import api_server, api_start + import os + + monkeypatch.setenv("LIGHTLLM_SERVICE_START_TIME", "old") + monkeypatch.setattr(api_server.time, "monotonic", lambda: 42.5) + logger = Mock() + monkeypatch.setattr(api_server, "logger", logger) single, multiple = Mock(), Mock() monkeypatch.setattr(api_start, "pd_master_start", single) monkeypatch.setattr(api_start, "pd_master_multi_worker_start", multiple) @@ -533,6 +539,13 @@ def test_pd_master_cli_dispatches_to_the_requested_startup_mode(monkeypatch, wor expected.assert_called_once_with(args) unexpected.assert_not_called() configure.assert_called_once_with(args) + assert os.environ["LIGHTLLM_SERVICE_START_TIME"] == "42.5" + logger.info.assert_called_once() + assert logger.info.call_args.kwargs["extra"] == { + "event": "service.starting", + "run_mode": "pd_master", + "model": args.model_name, + } def test_multiworker_preflight_failure_precedes_environment_and_process_initialization(monkeypatch): diff --git a/unit_tests/utils/test_log_utils.py b/unit_tests/utils/test_log_utils.py index b222cb2fa3..5bc22eccd8 100644 --- a/unit_tests/utils/test_log_utils.py +++ b/unit_tests/utils/test_log_utils.py @@ -534,6 +534,7 @@ def test_errors_bypass_sampling_and_pd_stages_are_debug(): log_request_summary(1, status="completed", timing_scope="end_to_end") log_request_summary(2, status="error", timing_scope="decode") log_request_summary(3, status="cancelled", timing_scope="end_to_end") + log_request_summary(6, status="continued", timing_scope="end_to_end") """, LIGHTLLM_LOG_FORMAT="json", LIGHTLLM_LOG_REQUEST_SAMPLE_RATE=0, From 1940b1b9dccc548a9c36248cb1a6221730b9fa4a Mon Sep 17 00:00:00 2001 From: sufubao Date: Thu, 22 Oct 2026 09:47:17 +0800 Subject: [PATCH 09/13] fix: address review findings for #1638 --- lightllm/server/api_cli.py | 6 +- .../httpserver_for_pd_master/manager.py | 2 +- lightllm/server/router/manager.py | 8 +- lightllm/utils/log_utils.py | 15 +- lightllm/utils/start_utils.py | 3 +- test/test_api/test_abort_request.py | 28 +- .../test_pd_master_multi_choice.py | 29 +- .../basemodel/attention/fa3/test_neo.py | 9 +- unit_tests/common/test_autotuner.py | 18 + .../test_pd_master_cached_tokens.py | 18 +- .../httpserver/test_pd_websocket_lifecycle.py | 1 + .../test_running_request_lifecycle.py | 26 +- .../mode_backend/test_diverse_backend.py | 35 ++ .../mode_backend/test_pd_transfer_logging.py | 110 ++++ .../server/test_abort_request_logging.py | 50 ++ unit_tests/server/test_engine_throughput.py | 9 +- unit_tests/server/test_logging_regressions.py | 530 ++++++++++++++++++ unit_tests/utils/test_flashinfer_logging.py | 1 + unit_tests/utils/test_log_utils.py | 34 +- unit_tests/utils/test_start_utils.py | 20 +- 20 files changed, 909 insertions(+), 43 deletions(-) create mode 100644 unit_tests/server/test_abort_request_logging.py create mode 100644 unit_tests/server/test_logging_regressions.py diff --git a/lightllm/server/api_cli.py b/lightllm/server/api_cli.py index 85ed584ec0..9d52c4d83b 100644 --- a/lightllm/server/api_cli.py +++ b/lightllm/server/api_cli.py @@ -377,7 +377,11 @@ def add_cli_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: action="store_true", help="Whether or not to allow for custom models defined on the Hub in their own modeling files.", ) - parser.add_argument("--detail_log", action="store_true", help="enable to print input infos in requests.") + parser.add_argument( + "--detail_log", + action="store_true", + help="enable sampled request metadata at DEBUG; HTTP prompt text and token IDs are omitted.", + ) parser.add_argument( "--pd_detail_log", action="store_true", diff --git a/lightllm/server/httpserver_for_pd_master/manager.py b/lightllm/server/httpserver_for_pd_master/manager.py index 30106b19ea..a17f6e88f4 100644 --- a/lightllm/server/httpserver_for_pd_master/manager.py +++ b/lightllm/server/httpserver_for_pd_master/manager.py @@ -407,6 +407,7 @@ async def _generate_one_attempt( # 分段请求始终复用循环外选定的 P 节点；这里只按每段实际发送的 # prompt 更新该节点的在途 prefill 负载，不会重新选点。 + summary_state.clear() block_prompt = prompt + "".join(history_gen_token_strs) if segment_index > 0: pending_prefill_load_chars = len(block_prompt) @@ -430,7 +431,6 @@ async def _generate_one_attempt( sampling_params.constraint_out_params.pd_previous_output_len = max( 0, block_token_num - input_token_num ) - summary_state.clear() self._summary_states[origin_request_id] = summary_state results_generator = self._wait_to_token_package( p_node, diff --git a/lightllm/server/router/manager.py b/lightllm/server/router/manager.py index dbd6451e3a..8edaf0665b 100644 --- a/lightllm/server/router/manager.py +++ b/lightllm/server/router/manager.py @@ -260,16 +260,12 @@ async def loop_for_fwd( self.engine_throughput.resume(last_poll) stats_logger.last_time = last_poll seen_ready = True - if self.service_ready.get_value() and stats_logger.active is not True and active and not resumed: + if seen_ready and stats_logger.active is not True and active and not resumed: self.engine_throughput.resume(last_poll) stats_logger.last_time = last_poll resumed = True last_poll = now - if ( - (self.dp_size > 1 or self.node_rank == 0) - and self.service_ready.get_value() - and stats_logger.should_log(active) - ): + if (self.dp_size > 1 or self.node_rank == 0) and seen_ready and stats_logger.should_log(active): resumed = False stats_logger.log( scope="node", diff --git a/lightllm/utils/log_utils.py b/lightllm/utils/log_utils.py index 9df0c840b3..92a1f4a886 100644 --- a/lightllm/utils/log_utils.py +++ b/lightllm/utils/log_utils.py @@ -706,6 +706,19 @@ def log_startup_config(logger, args): ) -_setup_logger() +try: + _setup_logger() +except (ValueError, TypeError) as error: + # Imports must remain usable before the launcher validates user configuration. + saved = {key: value for key, value in os.environ.items() if key.startswith("LIGHTLLM_LOG_")} + try: + for key in saved: + os.environ.pop(key) + if saved.get("LIGHTLLM_LOG_FORMAT", "").lower() in {"text", "json"}: + os.environ["LIGHTLLM_LOG_FORMAT"] = saved["LIGHTLLM_LOG_FORMAT"] + _setup_logger() + finally: + os.environ.update(saved) + _root_logger.warning("Invalid logging configuration; using defaults until startup validation: %s", error) logging.captureWarnings(True) init_logger("py.warnings") diff --git a/lightllm/utils/start_utils.py b/lightllm/utils/start_utils.py index de54e69aa3..daa927c4be 100644 --- a/lightllm/utils/start_utils.py +++ b/lightllm/utils/start_utils.py @@ -58,7 +58,8 @@ def observe(self, status, reason=None): self.ready = self.ever_ready = True else: self.consecutive_successes = 0 - self.consecutive_failures += 1 + # A transport failure cannot establish that a live HTTP worker is unready. + self.consecutive_failures = 0 if status is None else self.consecutive_failures + 1 if status not in (None, 503) and not self.ever_ready and not self.probe_failure_logged: logger.warning( "event=service.probe_failed address=%s status=%s", diff --git a/test/test_api/test_abort_request.py b/test/test_api/test_abort_request.py index ca99f6298c..17fb6af66d 100644 --- a/test/test_api/test_abort_request.py +++ b/test/test_api/test_abort_request.py @@ -17,15 +17,18 @@ Stage A: bogus request_id -> HTTP 200, server log "not exist" warning Stage B: abort_all on an idle server -> HTTP 200, no errors Stage C: abort_all on a running stream - -> HTTP 200; server log shows "aborted group_request_id N" warning + -> HTTP 200; server log shows an abort DEBUG record -> the stream terminates within reasonable time (whether via abort or natural max_new_tokens completion) Stage D: abort by SPECIFIC request_id on a running stream -> resolve the lightllm_req_id from the server log (via X-Request-Id), POST /abort_request with that exact id, verify the targeted log - warning lands and the stream terminates + record lands and the stream terminates Stage E: server remains healthy and answers a fresh /generate +For log assertions, start the server with LIGHTLLM_LOG_LEVEL=DEBUG and +LIGHTLLM_LOG_REQUEST_SAMPLE_RATE=1. Both text and JSON logging are supported. + Usage: python test/test_api/test_abort_request.py \ --url http://127.0.0.1:8000 \ @@ -245,6 +248,19 @@ def lookup_lightllm_req_id_from_log(server_log_path: str, x_request_id: str, tim m = pattern.search(tail) if m: return int(m.group(1)) + for line in tail.splitlines(): + try: + event = json.loads(line) + except ValueError: + event = None + if isinstance(event, dict): + if event.get("event") == "request.received" and event.get("http_request_id") == x_request_id: + return int(event["request_id"]) + elif "event=request.received" in line: + client = re.search(r"http_request_id=['\"]?([^'\"\s]+)", line) + internal = re.search(r"\brequest_id=(\d+)", line) + if client and internal and client.group(1) == x_request_id: + return int(internal.group(1)) time.sleep(0.1) return None @@ -304,13 +320,13 @@ def stage_c_abort_running(url: str, server_log_path: Optional[str]) -> bool: # The api itself succeeded; whether the stream got a clean 'abort' finish reason # depends on which mode-backend the server is running. We DO assert the abort - # warning landed in the server log though, scoped to log content produced after + # record landed in the server log though, scoped to log content produced after # this stage started so we don't match earlier-stage residue. if server_log_path: if target_id is not None: - pat = re.compile(rf"aborted group_request_id {target_id}\b") + pat = re.compile(rf"(?:aborted group_request_id |Request aborted: request_id=){target_id}\b") else: - pat = re.compile(r"aborted group_request_id \d+") + pat = re.compile(r"(?:aborted group_request_id |Request aborted: request_id=)\d+") hit = grep_log_after_offset(server_log_path, log_offset, pat, timeout=5.0) if not hit: fail("could not find 'aborted group_request_id' in server log (post-stage)") @@ -355,7 +371,7 @@ def stage_d_abort_by_id(url: str, server_log_path: Optional[str]) -> bool: n = len(out.get("events") or []) print(f" stream events received: {n}, finish_reason={fr!r}") - pat = re.compile(rf"aborted group_request_id {target_id}\b") + pat = re.compile(rf"(?:aborted group_request_id |Request aborted: request_id=){target_id}\b") hit = grep_log_after_offset(server_log_path, log_offset, pat, timeout=5.0) if not hit: fail(f"could not find 'aborted group_request_id {target_id}' in server log (post-stage)") diff --git a/test/test_pd_selector/test_pd_master_multi_choice.py b/test/test_pd_selector/test_pd_master_multi_choice.py index a5a77080d7..9100f94713 100644 --- a/test/test_pd_selector/test_pd_master_multi_choice.py +++ b/test/test_pd_selector/test_pd_master_multi_choice.py @@ -19,6 +19,10 @@ def _manager() -> HttpServerManagerForPDMaster: manager.active_req_ids = set() manager.req_id_to_out_inf = {} manager.pd_manager = MagicMock() + from lightllm.server.httpserver_for_pd_master.pd_selector.pd_node_stats_client import LocalPDNodeStats + + manager.pd_node_stats = LocalPDNodeStats() + manager.pd_manager.selector.insert_prompt_cache = AsyncMock() manager.id_gen = MagicMock() manager.id_gen.generate_id.return_value = 800 manager.metric_client = MagicMock() @@ -99,6 +103,7 @@ async def wait_to_token_package( choice_sampling_params, child_multimodal_params, child_request, + **_kwargs, ): captured_params.append(choice_sampling_params) started.add(choice_sampling_params.group_request_id) @@ -255,7 +260,7 @@ async def run(): segment_index = 0 resource_wait_timeouts = [] - async def wait_to_token_package(_p_node, _d_node, _start_time, prompt_ids, params, *_args): + async def wait_to_token_package(_p_node, _d_node, _start_time, prompt_ids, params, *_args, **_kwargs): nonlocal segment_index segment_index += 1 resource_wait_timeouts.append(params.pd_node_resource_wait_timeout_seconds) @@ -365,7 +370,9 @@ async def run(): sampling_params = SamplingParams() sampling_params.max_new_tokens = 4 - async def wait_to_token_package(p_node, d_node, start_time, prompt, params, multimodal_params, request): + async def wait_to_token_package( + p_node, d_node, start_time, prompt, params, multimodal_params, request, **_kwargs + ): metadata = {"id": 10, "prompt_tokens": 2} yield params.group_request_id, "first", dict(metadata), FinishStatus() await asyncio.sleep(0.01) @@ -472,7 +479,7 @@ async def run(): dispatched_max_new_tokens = [] async def wait_to_token_package( - selected_p_node, selected_d_node, _start_time, segment_prompt_ids, sampling_params, *_args + selected_p_node, selected_d_node, _start_time, segment_prompt_ids, sampling_params, *_args, **_kwargs ): dispatched_nodes.append(selected_p_node) dispatched_d_nodes.append(selected_d_node) @@ -547,7 +554,7 @@ async def run(): manager.select_p_d_node = AsyncMock(return_value=(p_node, d_node, PDSelectionExtraInfo())) dispatched_max_new_tokens = [] - async def wait_to_token_package(_p_node, _d_node, _start_time, _prompt, sampling_params, *_args): + async def wait_to_token_package(_p_node, _d_node, _start_time, _prompt, sampling_params, *_args, **_kwargs): dispatched_max_new_tokens.append(sampling_params.max_new_tokens) is_first_segment = len(dispatched_max_new_tokens) == 1 token_count = 2 if is_first_segment else 1 @@ -638,7 +645,7 @@ async def run(): ) high_priority_request_flags = [] - async def wait_to_token_package(_p_node, _d_node, _start_time, _prompt, sampling_params, *_args): + async def wait_to_token_package(_p_node, _d_node, _start_time, _prompt, sampling_params, *_args, **_kwargs): high_priority_request_flags.append(sampling_params.high_priority_request) yield ( sampling_params.group_request_id, @@ -694,7 +701,7 @@ async def run(): ) captured_request_settings = [] - async def wait_to_token_package(_p_node, _d_node, _start_time, _prompt, sampling_params, *_args): + async def wait_to_token_package(_p_node, _d_node, _start_time, _prompt, sampling_params, *_args, **_kwargs): captured_request_settings.append( ( sampling_params.high_priority_request, @@ -735,7 +742,7 @@ async def run(): manager.enable_pd_node_self_request_limit = True attempt_count = 0 - async def generate_one_attempt(*_args): + async def generate_one_attempt(*_args, **_kwargs): nonlocal attempt_count attempt_count += 1 if attempt_count == 1: @@ -770,7 +777,7 @@ async def run(): manager = _manager() attempt_count = 0 - async def generate_one_attempt(*_args): + async def generate_one_attempt(*_args, **_kwargs): nonlocal attempt_count attempt_count += 1 raise ServerBusyError("node is busy") @@ -806,7 +813,7 @@ async def run(): manager.enable_pd_node_self_request_limit = False attempt_count = 0 - async def generate_one_attempt(*_args): + async def generate_one_attempt(*_args, **_kwargs): nonlocal attempt_count attempt_count += 1 raise ServerBusyError("node is busy") @@ -837,7 +844,7 @@ async def run(): manager.pd_node_busy_retry_timeout_seconds = 0 attempt_count = 0 - async def generate_one_attempt(*_args): + async def generate_one_attempt(*_args, **_kwargs): nonlocal attempt_count attempt_count += 1 raise ServerBusyError("node is busy") @@ -868,7 +875,7 @@ async def run(): attempt_count = 0 results = [] - async def generate_one_attempt(*_args): + async def generate_one_attempt(*_args, **_kwargs): nonlocal attempt_count attempt_count += 1 yield 800, "visible", {}, FinishStatus() diff --git a/unit_tests/common/basemodel/attention/fa3/test_neo.py b/unit_tests/common/basemodel/attention/fa3/test_neo.py index 5f815e406d..e3c5ec659e 100644 --- a/unit_tests/common/basemodel/attention/fa3/test_neo.py +++ b/unit_tests/common/basemodel/attention/fa3/test_neo.py @@ -131,9 +131,13 @@ def test_prefill_missing_extension_reports_install_instructions(monkeypatch): @pytest.mark.parametrize("selection", ["triton", "auto", "fa3"]) -def test_neo_selection_only_explains_missing_extension_when_requested(monkeypatch, selection): +@pytest.mark.parametrize("rank", [0, 1]) +def test_neo_selection_only_explains_missing_extension_when_requested(monkeypatch, selection, rank): from lightllm.common.basemodel.attention import create_utils + from lightllm.utils import log_utils + monkeypatch.setattr(log_utils, "_context", {"tp_rank": rank}) + monkeypatch.setattr(log_utils, "_context_pid", log_utils.os.getpid()) monkeypatch.setattr(neo, "HAS_FLASH_ATTN_INTERFACE", False) monkeypatch.setattr(create_utils, "HAS_FLASH_ATTN_INTERFACE", False) monkeypatch.setattr( @@ -163,7 +167,8 @@ def test_neo_selection_only_explains_missing_extension_when_requested(monkeypatc assert create_utils.get_neo_prefill_att_backend_class() is create_utils.NeoTritonAttBackend if selection == "auto": automatic.assert_called_once() - assert logger.info.call_count + logger.debug.call_count == 1 + assert logger.info.call_count == int(rank == 0) + assert logger.debug.call_count == int(rank != 0) assert "backend.fallback" in (logger.info.call_args or logger.debug.call_args).args[0] else: assert not logger.mock_calls diff --git a/unit_tests/common/test_autotuner.py b/unit_tests/common/test_autotuner.py index d44712a9dd..f6048682d1 100644 --- a/unit_tests/common/test_autotuner.py +++ b/unit_tests/common/test_autotuner.py @@ -443,3 +443,21 @@ def test_missing_configs_warn_once_and_keep_per_kernel_debug_details(tmp_path, m assert "event=kernel.defaults_used" in logger.warning.call_args.args[0] assert logger.debug.call_count == 2 assert {call.args[1] for call in logger.debug.call_args_list} == {"missing_first", "missing_second"} + + +@pytest.mark.parametrize("progress", [False, True]) +def test_autotune_progress_consumer(tmp_path, monkeypatch, progress): + original = autotuner_module.tqdm + flags = [] + + def bar(*args, **kwargs): + flags.append(kwargs["disable"]) + return original(*args, **kwargs) + + monkeypatch.setattr(autotuner_module, "tqdm", bar) + monkeypatch.setattr(autotuner_module, "progress_enabled", lambda: progress) + kernel, _, benchmarks, _ = make_kernel(tmp_path, monkeypatch, "progress_consumer") + with Autotuner.autotune_warmup(): + kernel(8) + assert benchmarks + assert flags == [not progress] diff --git a/unit_tests/server/httpserver/test_pd_master_cached_tokens.py b/unit_tests/server/httpserver/test_pd_master_cached_tokens.py index e73257f3fc..8dedca5118 100644 --- a/unit_tests/server/httpserver/test_pd_master_cached_tokens.py +++ b/unit_tests/server/httpserver/test_pd_master_cached_tokens.py @@ -891,7 +891,8 @@ async def run(): @pytest.mark.parametrize("retry_enabled", [False, True]) -def test_pd_continuation_busy_before_first_token_emits_terminal_summary(monkeypatch, retry_enabled): +@pytest.mark.parametrize("failure_site", ["stream", "dispatch"]) +def test_pd_continuation_busy_before_first_token_emits_terminal_summary(monkeypatch, retry_enabled, failure_site): import time from unittest.mock import MagicMock from lightllm.utils.error_utils import ServerBusyError @@ -919,6 +920,18 @@ async def raw_stream(_p, _d, _prompt, sp, *_args): raise ServerBusyError("segment busy before first token") manager.fetch_pd_stream = raw_stream + if failure_site == "dispatch": + original_dispatch = manager.pd_node_stats.add_dispatched_info + dispatch_count = 0 + + async def dispatch(**kwargs): + nonlocal dispatch_count + dispatch_count += 1 + if dispatch_count == 2: + raise ServerBusyError("continuation dispatch failed") + await original_dispatch(**kwargs) + + manager.pd_node_stats.add_dispatched_info = dispatch async def consume(): return [item async for item in manager._generate_one("prompt", params, None, request, time.time(), 80, 10)] @@ -927,7 +940,8 @@ async def run(): with pytest.raises(ServerBusyError): await consume() assert [call.kwargs["status"] for call in summary.call_args_list] == ["continued", "busy"] - assert attempts == 2 + assert [call.args[0] for call in summary.call_args_list] == [1, 80] + assert attempts == (2 if failure_site == "stream" else 1) assert not manager._summary_states assert not manager.active_req_ids diff --git a/unit_tests/server/httpserver/test_pd_websocket_lifecycle.py b/unit_tests/server/httpserver/test_pd_websocket_lifecycle.py index c5c42219be..1857d397c8 100644 --- a/unit_tests/server/httpserver/test_pd_websocket_lifecycle.py +++ b/unit_tests/server/httpserver/test_pd_websocket_lifecycle.py @@ -97,6 +97,7 @@ def test_registration_disconnect_keeps_dp_connection_identity(monkeypatch): receive_bytes=AsyncMock(side_effect=WebSocketDisconnect()), ) asyncio.run(api_http_pd.register_and_keep_alive(websocket)) + assert len(logger.info.call_args_list) == 3 for call in logger.info.call_args_list[1:]: assert "dp_index=2 connection_id=connection-a" in call.args[0] diff --git a/unit_tests/server/httpserver/test_running_request_lifecycle.py b/unit_tests/server/httpserver/test_running_request_lifecycle.py index 7beb21ec16..3a7f8b0a69 100644 --- a/unit_tests/server/httpserver/test_running_request_lifecycle.py +++ b/unit_tests/server/httpserver/test_running_request_lifecycle.py @@ -570,10 +570,20 @@ def test_real_supported_max_req_total_len_reserves_mtp_overlap_tokens(): @pytest.mark.parametrize( - "kind,status", [("busy", "busy"), ("invalid", "rejected"), ("timeout", "timeout"), ("error", "error")] + "kind,status", + [ + ("busy", "busy"), + ("invalid", "rejected"), + ("timeout", "timeout"), + ("error", "error"), + ("cancel", "cancelled"), + ("exit", "cancelled"), + ("disconnect", "cancelled"), + ], ) -def test_generate_failure_summary_preserves_reason(monkeypatch, kind, status): - from lightllm.utils.error_utils import ServerBusyError, InvalidRequestError +@pytest.mark.parametrize("slave", [False, True]) +def test_generate_failure_summary_preserves_reason(monkeypatch, kind, status, slave): + from lightllm.utils.error_utils import ServerBusyError, InvalidRequestError, ClientDisconnected summary = MagicMock() monkeypatch.setattr("lightllm.server.httpserver.manager.log_request_summary", summary) @@ -582,11 +592,19 @@ def test_generate_failure_summary_preserves_reason(monkeypatch, kind, status): "invalid": InvalidRequestError, "timeout": asyncio.TimeoutError, "error": RuntimeError, + "cancel": asyncio.CancelledError, + "exit": GeneratorExit, + "disconnect": ClientDisconnected, }[kind] async def run(): manager = _make_manager(NodeRole.NORMAL) - manager._encode.side_effect = error_type("request rejected reason") + manager.is_multinode_tp_slave = slave + manager._encode.side_effect = ( + ClientDisconnected(8, "request rejected reason") + if kind == "disconnect" + else error_type("request rejected reason") + ) with pytest.raises(error_type): await _drain_generate(manager, _sampling_params(), _multimodal_params()) summary.assert_called_once() diff --git a/unit_tests/server/router/model_infer/mode_backend/test_diverse_backend.py b/unit_tests/server/router/model_infer/mode_backend/test_diverse_backend.py index ab02d5892c..acb24c5273 100644 --- a/unit_tests/server/router/model_infer/mode_backend/test_diverse_backend.py +++ b/unit_tests/server/router/model_infer/mode_backend/test_diverse_backend.py @@ -72,3 +72,38 @@ def test_copy_master_kv_updates_slave_hold_kv_len(): assert InferReq._kv_cache_alloc_need(slave_req, target_kv_len=5) == 1 assert backend.radix_cache.dec_nodes == [old_shared_node] assert backend.radix_cache.add_nodes == [new_shared_node] + + +def test_diverse_prefill_counts_increment_once_and_slaves_do_not_duplicate(monkeypatch): + from unittest.mock import Mock + + monkeypatch.setattr(torch.cuda, "current_stream", lambda: Mock()) + backend = DiversehBackend.__new__(DiversehBackend) + backend.is_master_in_dp = True + backend._master_req_to_radix_cache = Mock() + backend._copy_master_req_to_slave_req = Mock() + target = [512] + master = SimpleNamespace( + is_master_req=lambda: True, + get_chuncked_input_token_len=lambda: target[0], + get_cur_total_len=lambda: 800, + cur_kv_len=0, + cur_output_len=0, + slave_reqs=[], + shm_req=SimpleNamespace(input_len=800, group_req_id=8), + ) + first = backend._diverse_pre_post_handle([master], True) + assert first[0].prefill_tokens == 512 and first[0].output_len == 0 + target[0] = 800 + slave = SimpleNamespace( + is_master_req=lambda: False, remove_master_req=Mock(), shm_req=SimpleNamespace(group_req_id=8) + ) + master.slave_reqs = [slave] + second = backend._diverse_pre_post_handle([master, slave], True) + assert [pack.prefill_tokens for pack in second] == [288, 0] + assert sum(pack.prefill_tokens for pack in first + second) == 800 + slave.remove_master_req.assert_called_once() + backend._copy_master_req_to_slave_req.assert_called_once_with(slave_req=slave) + target[0] = 801 + master.slave_reqs = [] + assert backend._diverse_pre_post_handle([master], True)[0].prefill_tokens == 0 diff --git a/unit_tests/server/router/model_infer/mode_backend/test_pd_transfer_logging.py b/unit_tests/server/router/model_infer/mode_backend/test_pd_transfer_logging.py index 57cf8cf82a..d57bec2cd3 100644 --- a/unit_tests/server/router/model_infer/mode_backend/test_pd_transfer_logging.py +++ b/unit_tests/server/router/model_infer/mode_backend/test_pd_transfer_logging.py @@ -129,3 +129,113 @@ def test_nixl_factory_forwards_transport_arguments(monkeypatch): result = kv_transporter.create_kv_transporter(SimpleNamespace(pd_trans_mode="nixl"), 7, 2, buffer) constructor.assert_called_once_with(node_id=7, tp_idx=2, kv_move_buffer=buffer) assert result is constructor.return_value + + +@pytest.mark.parametrize("level", [20, 10]) +@pytest.mark.parametrize("kind,tokens", [("kv", 3), ("kv", 0), ("state", 0)]) +def test_decode_success_real_debug_window(monkeypatch, caplog, level, kind, tokens): + import logging + from lightllm.utils import log_utils + + module = decode_trans_process + worker = module._DecodeTransModule.__new__(module._DecodeTransModule) + worker.device_id = 0 + worker.page_index_queue = queue.Queue() + worker.task_out_queue = queue.Queue() + worker.transporter = Mock(kv_move_buffer=torch.empty((1, 4), dtype=torch.float16)) + logger = logging.getLogger("decode-transfer-regression") + monkeypatch.setattr(module, "logger", logger) + monkeypatch.setattr(module.torch.cuda, "set_device", Mock()) + clock = [0] + monkeypatch.setattr(log_utils.time, "monotonic", lambda: clock[0]) + monkeypatch.setattr(log_utils, "get_log_stats_interval", lambda: 10) + transfer = Mock(return_value=tokens) + task = SimpleNamespace( + request_id=123, + page_kind=kind, + transfer_kv_num=transfer, + dst_page_index=1, + xfer_handle=None, + create_time=0, + createRetObj=lambda: "done", + ) + start, end = Mock(), Mock() + start.elapsed_time.return_value = 2.5 + count = [0] + + class StopLoop(BaseException): + pass + + def get(timeout): + assert timeout == 10 + count[0] += 1 + if count[0] <= 3: + return end, start, task + if count[0] == 4: + clock[0] = 11 + raise queue.Empty() + raise StopLoop() + + worker.success_queue = SimpleNamespace(get=get) + with caplog.at_level(level, logger=logger.name), pytest.raises(StopLoop): + worker.success_loop() + assert worker.task_out_queue.qsize() == 3 and worker.page_index_queue.qsize() == 3 + assert end.synchronize.call_count == 3 + records = [r for r in caplog.records if r.name == logger.name] + assert len(records) == 2 * int(level == 10 and (kind != "kv" or tokens > 0)) + if level == 20: + transfer.assert_not_called() + start.elapsed_time.assert_not_called() + if records: + record = records[1] + assert record.event == "kv.transfer.decode" + assert record.pages == 2 and record.kv_tokens_total == tokens * 2 + assert record.state_pages == (2 if kind == "state" else 0) + assert record.read_page_gpu_ms_total == 5 + assert record.bytes_total == 16 + assert record.request_id_sample == {"count": 1, "sample": [123]} + + +def test_prefill_idle_flushes_transfer_and_telemetry_separately(monkeypatch): + module = prefill_trans_process + worker = module._PrefillTransModule.__new__(module._PrefillTransModule) + worker.device_id = 0 + worker._debug_telemetry = Mock() + transfer = Mock(interval=7) + monkeypatch.setattr(module, "DebugStatsLogger", Mock(return_value=transfer)) + monkeypatch.setattr(module.torch.cuda, "set_device", Mock()) + + class StopLoop(BaseException): + pass + + worker.success_queue = Mock() + worker.success_queue.get.side_effect = [queue.Empty(), StopLoop()] + with pytest.raises(StopLoop): + worker.success_loop() + assert worker.success_queue.get.call_args.kwargs == {"timeout": 7} + transfer.flush.assert_called_once() + worker._debug_telemetry.flush.assert_called_once() + + +def test_prefill_info_does_not_collect_nixl_telemetry(monkeypatch): + import logging + import threading + + worker = prefill_trans_process._PrefillTransModule.__new__(prefill_trans_process._PrefillTransModule) + task = Mock(xfer_handle="handle") + task.get_key.return_value = "key" + worker.waiting_dict = {"key": task} + worker.waiting_dict_lock = threading.Lock() + worker.transporter = Mock(capture_telemetry=True) + worker.transporter.check_task_status.return_value = "DONE" + worker.success_queue = queue.Queue() + logger = logging.getLogger("telemetry-info-regression") + monkeypatch.setattr(prefill_trans_process, "logger", logger) + old = logger.level + logger.setLevel(logging.INFO) + try: + worker._update_task_status_once() + finally: + logger.setLevel(old) + worker.transporter.nixl_agent.get_xfer_telemetry.assert_not_called() + assert worker.success_queue.get_nowait() is task diff --git a/unit_tests/server/test_abort_request_logging.py b/unit_tests/server/test_abort_request_logging.py new file mode 100644 index 0000000000..06e77ade78 --- /dev/null +++ b/unit_tests/server/test_abort_request_logging.py @@ -0,0 +1,50 @@ +import importlib.util +import json +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + + +@pytest.fixture +def script(): + path = Path(__file__).resolve().parents[2] / "test/test_api/test_abort_request.py" + spec = importlib.util.spec_from_file_location("abort_request_script", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize( + "line", + [ + "received req X-Request-Id:client.1 X-Session-Id:abc lightllm_req_id:80", + "DEBUG event=request.received request_id=80 http_request_id='client.1'", + json.dumps({"event": "request.received", "request_id": 80, "http_request_id": "client.1"}), + ], +) +def test_lookup_internal_id_from_legacy_and_current_logs(script, tmp_path, line): + path = tmp_path / "server.log" + path.write_text(line + "\n") + assert script.lookup_lightllm_req_id_from_log(str(path), "client.1", timeout=0.1) == 80 + assert script.lookup_lightllm_req_id_from_log(str(path), "client.2", timeout=0.01) is None + + +@pytest.mark.parametrize("message", ["aborted group_request_id 80", "Request aborted: request_id=80"]) +@pytest.mark.parametrize("stage", ["stage_c_abort_running", "stage_d_abort_by_id"]) +def test_abort_stage_accepts_current_and_legacy_cancellation(script, monkeypatch, tmp_path, message, stage): + path = tmp_path / "server.log" + path.write_text("event=request.received request_id=80 http_request_id='client.1'\n") + thread = SimpleNamespace(join=Mock(), is_alive=lambda: False) + monkeypatch.setattr(script, "start_stream", lambda *a, **k: (thread, {"events": [{"_t": 0}]}, "client.1")) + monkeypatch.setattr(script, "wait_for_first_token", lambda *a, **k: True) + monkeypatch.setattr(script, "get_finish_reason", lambda out: "abort") + + def abort(*a, **k): + with path.open("a") as stream: + stream.write(json.dumps({"message": message}) + "\n") + return 200, "ok" + + monkeypatch.setattr(script, "post_abort", abort) + assert getattr(script, stage)("http://unused", str(path)) diff --git a/unit_tests/server/test_engine_throughput.py b/unit_tests/server/test_engine_throughput.py index 23425c35a0..fc652a8c04 100644 --- a/unit_tests/server/test_engine_throughput.py +++ b/unit_tests/server/test_engine_throughput.py @@ -196,7 +196,8 @@ def test_resume_excludes_idle_time_without_discarding_short_burst(monkeypatch, c @pytest.mark.parametrize("node_rank,dp_size", [(0, 1), (1, 1), (1, 2)]) -def test_router_first_burst_excludes_initial_idle(monkeypatch, counters, node_rank, dp_size): +@pytest.mark.parametrize("readiness", ["always", "never", "drop"]) +def test_router_first_burst_excludes_initial_idle(monkeypatch, counters, node_rank, dp_size, readiness): import asyncio from lightllm.server.router import manager @@ -207,7 +208,9 @@ def test_router_first_burst_excludes_initial_idle(monkeypatch, counters, node_ra monkeypatch.setattr(manager, "init_logger", lambda name: logger) router = manager.RouterManager.__new__(manager.RouterManager) router.engine_counters = counters - router.service_ready = SimpleNamespace(get_value=lambda: 1) + router.service_ready = SimpleNamespace( + get_value=lambda: int(readiness != "never" and (readiness != "drop" or now[0] < 6)) + ) router.req_queue = Mock() router.req_queue.get_wait_req_num.return_value = 0 router.running_batch = None @@ -234,7 +237,7 @@ async def step(): router._step = step with pytest.raises(StopLoop): asyncio.run(router.loop_for_fwd()) - if dp_size == 1 and node_rank != 0: + if readiness == "never" or (dp_size == 1 and node_rank != 0): logger.info.assert_not_called() else: logger.info.assert_called_once() diff --git a/unit_tests/server/test_logging_regressions.py b/unit_tests/server/test_logging_regressions.py new file mode 100644 index 0000000000..8e5cf9643c --- /dev/null +++ b/unit_tests/server/test_logging_regressions.py @@ -0,0 +1,530 @@ +import asyncio +import logging +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import pytest +import torch + +from lightllm.utils import log_utils + + +@pytest.mark.parametrize("rank", [0, 1]) +@pytest.mark.parametrize("progress", [False, True]) +def test_weight_loading_events_and_progress(monkeypatch, caplog, rank, progress): + from lightllm.common.basemodel.layer_weights import hf_load_utils as module + import multiprocessing.pool + + pool = Mock() + pool.__enter__ = Mock(return_value=pool) + pool.__exit__ = Mock(return_value=False) + pool.imap_unordered.return_value = iter([None, None]) + monkeypatch.setattr(multiprocessing.pool, "ThreadPool", Mock(return_value=pool)) + monkeypatch.setattr(module.utils.PetrelHelper, "list", lambda *a, **k: ["a.bin", "b.bin"]) + monkeypatch.setattr(module, "progress_enabled", lambda: progress) + bar = Mock(side_effect=lambda iterator, **kw: iterator) + monkeypatch.setattr(module, "tqdm", bar) + logger = logging.getLogger("weight-regression") + monkeypatch.setattr(module, "logger", logger) + monkeypatch.setattr(log_utils, "_context", {"tp_rank": rank}) + monkeypatch.setattr(log_utils, "_context_pid", log_utils.os.getpid()) + ticks = iter([10, 12.125]) + monkeypatch.setattr(module.time, "monotonic", lambda: next(ticks)) + with caplog.at_level(logging.DEBUG, logger=logger.name): + module.load_hf_weights("fp16", "/unused") + assert bar.call_args.kwargs["disable"] is (not progress) + records = [r for r in caplog.records if r.name == logger.name] + assert [r.event for r in records] == ["model.weights_loading", "model.weights_loaded"] + assert [r.levelno for r in records] == [logging.INFO if rank == 0 else logging.DEBUG] * 2 + assert records[1].duration_s == 2.125 + assert "duration_s=2.125" in records[1].getMessage() + + +@pytest.mark.parametrize("tp,dp,node,rank", [(4, 1, 0, 3), (8, 2, 1, 4), (8, 4, 1, 7)]) +def test_model_infer_rank_context(monkeypatch, tp, dp, node, rank): + from lightllm.server.router.model_infer import model_rpc as module + + context = Mock() + monkeypatch.setattr(module, "set_log_context", context) + monkeypatch.setattr(module, "graceful_registry", Mock()) + monkeypatch.setattr(module.setproctitle, "setproctitle", Mock()) + monkeypatch.setattr(module, "get_unique_server_name", lambda: "test") + monkeypatch.setattr(module, "start_parent_check_thread", Mock()) + monkeypatch.setattr(module, "ModelRpcServer", Mock()) + server = Mock() + monkeypatch.setattr(module, "ThreadedServer", Mock(return_value=server)) + args = SimpleNamespace(tp=tp, dp=dp, node_rank=node) + module._init_env(args, rank, 0, 4, None, "/unused", Mock()) + context.assert_called_once_with( + component="model_infer", node_rank=node, global_rank=rank, tp_rank=rank % (tp // dp), dp_rank=rank // (tp // dp) + ) + server.start.assert_called_once() + + +def test_detoken_slow_batches_flush_on_idle_and_completion(monkeypatch): + from lightllm.server.detokenization import manager as module + + manager = module.DeTokenizationManager.__new__(module.DeTokenizationManager) + manager.zmq_recv_socket = Mock() + manager.zmq_recv_socket.recv_pyobj.side_effect = module.zmq.ZMQError() + manager.gen_token_out = Mock(side_effect=[True, False, RuntimeError("stop")]) + stats = Mock() + monkeypatch.setattr(module, "DebugStatsLogger", Mock(return_value=stats)) + ticks = iter([0, 0.06, 1, 1.07, 2]) + monkeypatch.setattr(module, "time", SimpleNamespace(time=lambda: next(ticks), sleep=Mock())) + manager.handle_loop() + assert stats.add.call_count == 2 + assert stats.add.call_args_list[0].kwargs["duration_ms_total"] == pytest.approx(60) + assert stats.add.call_args_list[1].kwargs["duration_ms_total"] == pytest.approx(70) + stats.flush.assert_called_once() + req = SimpleNamespace(request_id=7, can_released_mark=False) + decode = SimpleNamespace(req=req, request_id=7, output_ids=[1, 2], can_set_release_mark=lambda: True) + manager.req_id_to_out = {7: decode} + manager.shm_req_manager = Mock() + event = Mock() + monkeypatch.setattr(module, "log_debug_event", event) + manager.remove_finished_reqs() + assert req.can_released_mark and not manager.req_id_to_out + manager.shm_req_manager.put_back_req_obj.assert_called_once_with(req) + event.assert_called_once_with(module.logger, "detoken.completed", request_id=7, output_tokens=2) + + +def test_recycle_pending_is_bounded_and_releases_each_choice(monkeypatch): + from lightllm.server.httpserver import manager as module + + manager = module.HttpServerManager.__new__(module.HttpServerManager) + manager.req_id_to_out_inf = {} + manager.shm_req_manager = SimpleNamespace(async_put_back_req_obj=AsyncMock(), async_release_req_index=AsyncMock()) + manager._release_multimodal_resources = AsyncMock() + events = Mock() + monkeypatch.setattr(module, "log_debug_event", events) + clock = [0] + monkeypatch.setattr(module.time, "time", lambda: clock[0]) + reqs = [SimpleNamespace(index_in_shm_mem=i, can_released_mark=False, ref_count=2) for i in range(10)] + statuses = { + i: SimpleNamespace( + can_release=lambda: False, + group_req_objs=SimpleNamespace(group_req_id=i, shm_req_objs=[req], multimodal_params=None), + ) + for i, req in enumerate(reqs) + } + + class StopLoop(BaseException): + pass + + step = [0] + + async def wait(): + step[0] += 1 + if step[0] == 1: + clock[0] = 121 + elif step[0] == 2: + manager.req_id_to_out_inf.update(statuses) + clock[0] = 122 + elif step[0] == 3: + clock[0] = 123 + elif step[0] == 4: + manager.req_id_to_out_inf[0].can_release = lambda: True + else: + raise StopLoop() + + manager.recycle_event = SimpleNamespace(wait=wait, clear=Mock()) + with pytest.raises(StopLoop): + asyncio.run(manager.recycle_resource_loop()) + pending = [c.kwargs for c in events.call_args_list if c.args[1] == "request.pending"] + assert len(pending) == 1 + assert pending[0]["count"] == 10 + assert pending[0]["request_id_sample"] == list(range(8)) + assert pending[0]["can_release_sample"] == [False] * 8 + assert pending[0]["ref_count_sample"] == [2] * 8 + released = [c for c in events.call_args_list if c.args[1] == "request.released"] + assert len(released) == 1 and released[0].kwargs == {"request_id": 0, "choices": 1} + manager.shm_req_manager.async_put_back_req_obj.assert_awaited_once_with(reqs[0]) + manager.shm_req_manager.async_release_req_index.assert_awaited_once_with(0) + + +@pytest.mark.parametrize("master", [False, True]) +@pytest.mark.parametrize("level", [logging.INFO, logging.DEBUG]) +def test_dp_shared_kv_transfer_stats(monkeypatch, caplog, master, level): + from lightllm.server.router.model_infer.mode_backend.dp_backend import dp_shared_kv_trans as module + + logger = logging.getLogger("dp-transfer-regression") + worker = module.DPKVSharedMoudle.__new__(module.DPKVSharedMoudle) + copy = Mock() + worker.backend = SimpleNamespace( + is_master_in_dp=master, + model=SimpleNamespace(mem_manager=SimpleNamespace(operator=SimpleNamespace(copy_kv_from_other_dp_ranks=copy))), + mem_managers=[], + dp_size_in_node=2, + rank_in_dp=0, + ) + clock = [0] + monkeypatch.setattr(log_utils.time, "monotonic", lambda: clock[0]) + worker._debug_transfers = log_utils.DebugStatsLogger(logger, "kv.shared_transfer", interval=10) + # Preserve tensor math while replacing only the unavailable CUDA placement. + original_to, original_tensor = torch.Tensor.to, torch.tensor + monkeypatch.setattr(torch.Tensor, "to", lambda tensor, **kw: original_to(tensor, **dict(kw, device="cpu"))) + monkeypatch.setattr(module.torch, "tensor", lambda *a, **kw: original_tensor(*a, **dict(kw, device="cpu"))) + req = SimpleNamespace(cur_kv_len=0, hold_kv_len=6, shm_req=SimpleNamespace()) + task = SimpleNamespace( + max_kv_len_mem_indexes=original_tensor([1, 2]), + max_kv_len_dp_rank=1, + mem_indexes=original_tensor([3, 4]), + req=req, + ) + with caplog.at_level(level, logger=logger.name): + worker.kv_trans([task]) + worker.kv_trans([task]) + worker.kv_trans([task]) + clock[0] = 11 + worker.kv_trans([]) + assert copy.call_count == 3 and req.cur_kv_len == 6 + records = [r for r in caplog.records if r.name == logger.name] + assert len(records) == 2 * int(master and level == logging.DEBUG) + if records: + assert records[0].event == "kv.shared_transfer" + assert records[1].tokens_total == 4 and records[1].tasks_total == 2 + assert records[0].tokens_total == 2 + + +def test_router_opens_launcher_readiness_key(monkeypatch): + from lightllm.server.router import manager as module + from lightllm.server.core.objs import StartArgs + + flags = {} + + def shared(name): + return flags.setdefault(name, Mock()) + + class StopInit(Exception): + pass + + monkeypatch.setattr(module, "SharedInt", shared) + monkeypatch.setattr(module, "ShmReqManager", Mock()) + monkeypatch.setattr(module, "EngineCounters", Mock()) + monkeypatch.setattr(module, "ReadOnlyStaticsMemoryManager", Mock(side_effect=StopInit)) + args = StartArgs() + args.tp = args.dp = args.nnodes = 1 + args.node_rank = 0 + with pytest.raises(StopInit): + module.RouterManager(args) + assert "service_ready" in flags + flags["service_ready"].set_value.assert_called_once_with(0) + + +@pytest.mark.parametrize("progress", [False, True]) +def test_model_warmup_progress_and_rank_events(monkeypatch, caplog, progress): + from lightllm.common.basemodel import basemodel as module + + model = module.TpPartBaseModel.__new__(module.TpPartBaseModel) + model.batch_max_tokens = 4 + model.layers_num = 8 + model.config = {} + logger = logging.getLogger("warmup-regression") + monkeypatch.setattr(module, "logger", logger) + monkeypatch.setattr(module, "progress_enabled", lambda: progress) + bar = Mock(return_value=[]) + monkeypatch.setattr(module, "tqdm", bar) + barrier = Mock() + monkeypatch.setattr(module.torch.distributed, "barrier", barrier) + monkeypatch.setattr(module.torch.cuda, "current_device", lambda: 3) + monkeypatch.setattr(module.torch.cuda, "empty_cache", Mock()) + monkeypatch.setattr(module.Autotuner, "start_autotune_warmup", Mock()) + monkeypatch.setattr(module.Autotuner, "end_autotune_warmup", Mock()) + monkeypatch.setattr(log_utils, "_context", {"tp_rank": 0}) + monkeypatch.setattr(log_utils, "_context_pid", log_utils.os.getpid()) + with caplog.at_level(logging.DEBUG, logger=logger.name): + model._autotune_warmup() + assert bar.call_args.args[0] == [4, 1] + assert bar.call_args.kwargs["disable"] is (not progress) + assert [c.kwargs for c in barrier.call_args_list] == [{"device_ids": [3]}] * 2 + records = [r for r in caplog.records if r.name == logger.name] + assert [r.event for r in records] == ["model.warmup_started", "model.warmup_completed"] + assert records[-1].duration_s >= 0 + assert model.layers_num == 8 + + +def test_graceful_registry_sets_component_without_replacing_other_context(monkeypatch): + from lightllm.utils import graceful_utils + import signal + + monkeypatch.setattr(signal, "signal", Mock()) + context = Mock() + monkeypatch.setattr(log_utils, "set_log_context", context) + graceful_utils.graceful_registry("detokenization") + context.assert_called_once_with(component="detokenization") + + +def test_http_disconnect_remains_499_and_debug(monkeypatch): + from lightllm.server import api_http + from lightllm.utils.error_utils import ClientDisconnected + + monkeypatch.setattr(api_http, "get_env_start_args", lambda: SimpleNamespace(run_mode="normal")) + monkeypatch.setattr(api_http.g_objs, "g_generate_func", AsyncMock(side_effect=ClientDisconnected(8, "closed"))) + monkeypatch.setattr(api_http.g_objs, "httpserver_manager", Mock()) + logger = Mock() + monkeypatch.setattr(api_http, "logger", logger) + response = asyncio.run(api_http.generate(Mock())) + assert response.status_code == 499 + logger.debug.assert_called_once() + logger.error.assert_not_called() + logger.warning.assert_not_called() + + +@pytest.mark.parametrize("enabled", [False, True]) +def test_access_middleware_uses_real_output_switch(monkeypatch, caplog, enabled): + from lightllm.server import api_http + + logger = logging.getLogger("lightllm.access") + monkeypatch.setattr(logger, "parent", logging.getLogger()) + monkeypatch.setattr(logger, "propagate", True) + monkeypatch.setattr(logger, "handlers", []) + monkeypatch.setenv("LIGHTLLM_LOG_HTTP_ACCESS", str(int(enabled))) + + async def app(scope, receive, send): + await send({"type": "http.response.start", "status": 201, "headers": []}) + await send({"type": "http.response.body", "body": b"ok"}) + + messages = [] + + async def send(message): + messages.append(message) + + with caplog.at_level(logging.INFO, logger=logger.name): + asyncio.run( + api_http._AccessLogMiddleware(app)({"type": "http", "method": "POST", "path": "/generate"}, None, send) + ) + assert messages[-1]["body"] == b"ok" + records = [r for r in caplog.records if r.name == logger.name] + assert len(records) == int(enabled) + if records: + assert records[0].status == 201 + + +@pytest.mark.parametrize("header,expected", [({"X-Request-Id": "client-id"}, "client-id"), ({}, "context-id")]) +def test_pd_header_logging_prefers_header_then_context(monkeypatch, header, expected): + from lightllm.server.httpserver_for_pd_master import manager as module + + manager = module.HttpServerManagerForPDMaster.__new__(module.HttpServerManagerForPDMaster) + event = Mock() + monkeypatch.setattr(module, "log_debug_event", event) + monkeypatch.setattr(module, "get_http_request_id", lambda: "context-id") + asyncio.run(manager._log_req_header(SimpleNamespace(headers=header), 80)) + event.assert_called_once_with(module.logger, "request.received", request_id=80, http_request_id=expected) + + +def test_allreduce_probe_failure_has_warning_and_debug_traceback(monkeypatch): + from lightllm.utils import dist_check_utils as module + import torch.multiprocessing + + logger = Mock() + monkeypatch.setattr(module, "logger", logger) + monkeypatch.setattr(torch.multiprocessing, "spawn", Mock(side_effect=RuntimeError("probe failed"))) + assert not module._check_ok_two_gpu_all_reduce(lambda rank, port: None, 1234) + logger.warning.assert_called_once() + assert "event=allreduce.probe_failed" in logger.warning.call_args.args[0] + assert "probe failed" in logger.warning.call_args.args[2] + logger.debug.assert_called_once_with("All-reduce capability probe diagnostics", exc_info=True) + + +def test_invalid_sse_payload_is_skipped_with_bounded_metadata(monkeypatch): + from lightllm.server import api_responses as module + + event = Mock() + monkeypatch.setattr(module, "log_debug_event", event) + + async def chunks(): + yield b"data: not-json\n\n" + yield b'data: {"choices":[{"delta":{"content":"hello"},"finish_reason":"stop"}]}\n\n' + + async def collect(): + return [item async for item in module._openai_sse_to_responses_events(chunks(), {"input": "hi"})] + + output = asyncio.run(collect()) + assert any(b"hello" in item for item in output) + event.assert_called_once_with(module.logger, "sse.payload_skipped", payload_chars=8, reason="invalid_json") + + +def test_decode_status_connection_errors_are_deduplicated_and_cancel_is_debug(monkeypatch, caplog): + from lightllm.server.router.model_infer.mode_backend.pd.decode_node_impl import up_status as module + from collections import OrderedDict + + manager = module.UpStatusManager.__new__(module.UpStatusManager) + manager.id_to_handle_queue = {} + from unittest.mock import MagicMock + + connection = MagicMock() + connection.__aenter__ = AsyncMock( + side_effect=[RuntimeError("offline"), RuntimeError("offline"), asyncio.CancelledError()] + ) + connection.__aexit__ = AsyncMock(return_value=False) + monkeypatch.setattr(module.websockets, "connect", Mock(return_value=connection)) + monkeypatch.setattr(module.asyncio, "sleep", AsyncMock()) + monkeypatch.setattr(log_utils, "_exception_windows", OrderedDict()) + monkeypatch.setattr(log_utils.time, "monotonic", lambda: 0) + logger = logging.getLogger("status-connection-regression") + monkeypatch.setattr(module, "logger", logger) + with caplog.at_level(logging.DEBUG, logger=logger.name): + asyncio.run(manager.up_kv_status_task(SimpleNamespace(host_ip_port="master:8000", node_id=1))) + records = [r for r in caplog.records if r.name == logger.name] + failures = [r for r in records if r.levelno == logging.ERROR] + assert len(failures) == 1 and failures[0].event == "pd.status_connection_failed" + assert failures[0].exc_info is not None + assert records[-1].levelno == logging.DEBUG and "cancelled" in records[-1].getMessage() + + +@pytest.mark.parametrize("overlap", [False, True]) +def test_cuda_graph_replay_cost_barrier_uses_current_device(monkeypatch, overlap): + from lightllm.common.basemodel import cuda_graph as module + + graph = module.CudaGraph.__new__(module.CudaGraph) + graph.capture_infer_cost = True + graph.enable_decode_microbatch_overlap = overlap + graph.infer_cost_ms_by_batch_size = {} + barrier = Mock() + monkeypatch.setattr(module.dist, "barrier", barrier) + monkeypatch.setattr(module.dist, "all_reduce", Mock()) + monkeypatch.setattr(module.torch.cuda, "current_device", lambda: 3) + start, end = Mock(), Mock() + start.elapsed_time.return_value = 2.5 + monkeypatch.setattr(module.torch.cuda, "Event", Mock(side_effect=[start, end])) + original_tensor = torch.tensor + monkeypatch.setattr(module.torch, "tensor", lambda *a, **kw: original_tensor(*a, **dict(kw, device="cpu"))) + replay = Mock() + graph._measure_replay_cost(replay, 4) + barrier.assert_called_once_with(group=module.dist.group.WORLD, device_ids=[3]) + assert graph.infer_cost_ms_by_batch_size == {8 if overlap else 4: 2.5} + assert replay.replay.call_count == 2 + end.synchronize.assert_called_once() + + +@pytest.mark.parametrize("rank", [0, 1]) +def test_memory_manager_selection_logs_only_on_leader(monkeypatch, caplog, rank): + from lightllm.common.kv_cache_mem_manager import mem_utils as module + + logger = logging.getLogger("memory-selection-regression") + monkeypatch.setattr(module, "logger", logger) + monkeypatch.setattr(module, "get_llm_model_class", lambda: object) + monkeypatch.setattr(module, "get_env_start_args", lambda: SimpleNamespace(llm_kv_type="None")) + monkeypatch.setattr(log_utils, "_context", {"tp_rank": rank}) + monkeypatch.setattr(log_utils, "_context_pid", log_utils.os.getpid()) + module.select_mem_manager_class.cache_clear() + try: + with caplog.at_level(logging.DEBUG, logger=logger.name): + assert module.select_mem_manager_class() is module.MemoryManager + records = [r for r in caplog.records if r.name == logger.name] + assert len(records) == 2 + assert all(r.levelno == (logging.INFO if rank == 0 else logging.DEBUG) for r in records) + finally: + module.select_mem_manager_class.cache_clear() + + +def test_idle_prefill_selection_records_decision_metadata(monkeypatch): + from lightllm.server.httpserver_for_pd_master.pd_selector import cache_aware as module + + policy = module.CacheAwarePolicy.__new__(module.CacheAwarePolicy) + idle = SimpleNamespace(dispatched_req_num=0, dispatched_prompt_chars=0, client_ip_port="idle:8000") + busy = SimpleNamespace(dispatched_req_num=1, dispatched_prompt_chars=100, client_ip_port="busy:8000") + event = Mock() + monkeypatch.setattr(module, "log_debug_event", event) + assert policy._select_idle_worker([busy, idle]) is idle + event.assert_called_once_with( + module.logger, "pd.prefill_selected", reason="idle_random", candidates=1, selected="idle:8000", load=0 + ) + event.reset_mock() + assert policy._select_idle_worker([busy]) is None + event.assert_not_called() + + +def test_openai_stream_disconnect_stops_without_error_chunks(monkeypatch): + from lightllm.server import api_openai as module + from lightllm.utils.error_utils import ClientDisconnected + + logger = Mock() + monkeypatch.setattr(module, "logger", logger) + + async def source(): + yield b"first" + raise ClientDisconnected(8, "closed") + + async def collect(): + return [item async for item in module._safe_stream_wrapper(source())] + + assert asyncio.run(collect()) == [b"first"] + logger.debug.assert_called_once() + logger.error.assert_not_called() + + +def test_registration_failure_keeps_one_traceback_before_retry(monkeypatch): + from lightllm.server.httpserver_for_pd_master import register_loop as module + from unittest.mock import MagicMock + + connection = MagicMock() + connection.__aenter__ = AsyncMock(side_effect=RuntimeError("offline")) + monkeypatch.setattr(module.websockets, "connect", Mock(return_value=connection)) + monkeypatch.setattr(module, "get_shm_port_args", lambda: SimpleNamespace(config_server_port=8000)) + logger = Mock() + monkeypatch.setattr(module, "logger", logger) + monkeypatch.setattr(module.asyncio, "sleep", AsyncMock(side_effect=asyncio.CancelledError())) + manager = SimpleNamespace(args=SimpleNamespace(host="10.0.0.1", config_server_host="config")) + with pytest.raises(asyncio.CancelledError): + asyncio.run(module.register_loop(manager)) + logger.exception.assert_called_once_with("connetion to config_server has error") + logger.error.assert_not_called() + + +@pytest.mark.parametrize("debug", [False, True]) +def test_router_batch_diagnostics_bound_request_sample(monkeypatch, debug): + from lightllm.server.router import manager as module + + router = module.RouterManager.__new__(module.RouterManager) + router.shm_reqs_io_buffer = Mock() + router.shm_reqs_io_buffer.is_empty.return_value = True + reqs = [SimpleNamespace(request_id=i, to_router_rpc_obj=lambda: "req") for i in range(12)] + logger = Mock() + logger.isEnabledFor.return_value = debug + event = Mock() + monkeypatch.setattr(module, "logger", logger) + monkeypatch.setattr(module, "log_debug_event", event) + asyncio.run(router._add_batch(SimpleNamespace(reqs=reqs))) + router.shm_reqs_io_buffer.write_obj.assert_called_once_with(["req"] * 12) + router.shm_reqs_io_buffer.set_ready.assert_called_once() + assert all(req.infer_start_time > 0 for req in reqs) + if debug: + event.assert_called_once_with( + logger, "batch.submitted", batch_id=1, request_count=12, request_id_sample=list(range(8)) + ) + else: + event.assert_not_called() + + +def test_router_ready_event_uses_agreed_model_capacity(monkeypatch): + from lightllm.server.router import manager as module + from lightllm.server.core.objs import StartArgs + + router = module.RouterManager.__new__(module.RouterManager) + router.args = StartArgs() + router.args.disable_dynamic_prompt_cache = True + router.args.run_mode = "normal" + router.args.max_req_total_len = 1024 + router.args.mtp_step = 0 + router.world_size = router.dp_size = router.dp_size_in_node = 2 + router.nnodes = 1 + router.node_rank = 0 + router.model_weightdir = "/unused" + router.load_way = "HF" + router.max_total_token_num = None + router.shm_max_total_token_num = Mock() + client = SimpleNamespace(init_model=AsyncMock(), get_max_total_token_num=AsyncMock(return_value=8192)) + monkeypatch.setattr(module.mp, "Queue", Mock()) + monkeypatch.setattr(module, "start_model_process", AsyncMock(return_value=client)) + monkeypatch.setattr(module, "get_running_max_req_size_per_dp", lambda args: 8) + monkeypatch.setattr(module, "get_shm_port_args", lambda: SimpleNamespace(nccl_port=1234)) + monkeypatch.setattr(module, "build_req_queue", Mock()) + logger = Mock() + monkeypatch.setattr(module, "logger", logger) + asyncio.run(router.wait_to_model_ready()) + records = [c.kwargs["extra"] for c in logger.info.call_args_list if "extra" in c.kwargs] + assert records == [{"event": "kv_cache.ready", "capacity_tokens_per_dp": 8192, "local_dp_count": 2}] + router.shm_max_total_token_num.set_value.assert_called_once_with(8192) + assert router.args.max_total_token_num == 8192 diff --git a/unit_tests/utils/test_flashinfer_logging.py b/unit_tests/utils/test_flashinfer_logging.py index 792f9b4771..00cc12f24b 100644 --- a/unit_tests/utils/test_flashinfer_logging.py +++ b/unit_tests/utils/test_flashinfer_logging.py @@ -25,3 +25,4 @@ def test_nonleader_workspace_failure_remains_visible(monkeypatch): assert workspace.disabled is True logger.warning.assert_called_once() assert "local allocation failed" in logger.warning.call_args.args[1] + logger.debug.assert_called_once_with("FlashInfer workspace initialization diagnostics", exc_info=True) diff --git a/unit_tests/utils/test_log_utils.py b/unit_tests/utils/test_log_utils.py index 5bc22eccd8..4692375892 100644 --- a/unit_tests/utils/test_log_utils.py +++ b/unit_tests/utils/test_log_utils.py @@ -153,12 +153,13 @@ def test_invalid_configuration_fails_explicitly(settings): env.update(settings) env["PYTHONPATH"] = str(Path(__file__).resolve().parents[2]) + os.pathsep + env.get("PYTHONPATH", "") result = subprocess.run( - [sys.executable, "-c", "from lightllm.utils import log_utils"], + [sys.executable, "-c", "from lightllm.utils import log_utils; log_utils._setup_logger()"], env=env, text=True, capture_output=True, timeout=30, ) + assert "using defaults until startup validation" in result.stdout assert result.returncode != 0 assert "ValueError" in result.stderr @@ -720,3 +721,34 @@ def test_all_logging_cli_options_survive_start_args_and_override_environment(sep LIGHTLLM_LOG_HTTP_ACCESS="0", ) assert output.strip() == "configured" + + +@pytest.mark.parametrize("setting", ["20", "Info,WARNING"]) +def test_invalid_legacy_level_import_keeps_warning_capture(setting): + output = _run_logging( + """ + import warnings + from lightllm.utils import log_utils + warnings.warn("spawn-warning-regression", UserWarning) + """, + LIGHTLLM_LOG_LEVEL=setting, + LIGHTLLM_LOG_FORMAT="json", + ) + events = [json.loads(line) for line in output.splitlines()] + assert any(event["logger"] == "py.warnings" and "spawn-warning-regression" in event["message"] for event in events) + + +def test_log_time_ready_uses_process_local_monotonic_windows(monkeypatch): + clock = [0] + pid = [100] + monkeypatch.setattr(log_utils, "_log_time_mark_dict", {}) + monkeypatch.setattr(log_utils.time, "monotonic", lambda: clock[0]) + monkeypatch.setattr(log_utils.os, "getpid", lambda: pid[0]) + assert not log_utils.log_time_ready("window", 10) + clock[0] = 10 + assert log_utils.log_time_ready("window", 10) + assert not log_utils.log_time_ready("window", 10) + pid[0] = 101 + assert not log_utils.log_time_ready("window", 10) + clock[0] = 20 + assert log_utils.log_time_ready("window", 10) diff --git a/unit_tests/utils/test_start_utils.py b/unit_tests/utils/test_start_utils.py index e9380f47c9..e233ca4ace 100644 --- a/unit_tests/utils/test_start_utils.py +++ b/unit_tests/utils/test_start_utils.py @@ -344,16 +344,24 @@ def test_readiness_waits_for_bound_endpoint_and_pd_topology(monkeypatch): assert logger.info.call_count == 2 flag.set_value.assert_called_once_with(1) assert logger.info.call_args.kwargs["extra"]["event"] == "service.ready" + assert logger.info.call_args.kwargs["extra"]["startup_s"] == 15 state.observe(None) state.observe(None) logger.warning.assert_not_called() state.observe(None) + logger.warning.assert_not_called() + assert state.ready + state.observe(503) + state.observe(503) + state.observe(503) logger.warning.assert_called_once() + assert logger.warning.call_args.kwargs["extra"]["reason"] == "readiness_probe_failed" assert flag.set_value.call_args.args == (0,) state.observe(200) assert not state.ready state.observe(200) assert logger.info.call_args.kwargs["extra"]["event"] == "service.recovered" + assert logger.info.call_args.kwargs["extra"]["uptime_s"] == 15 @pytest.mark.parametrize( @@ -424,6 +432,11 @@ def test_readiness_tolerates_isolated_failures_and_confirms_recovery(monkeypatch flag.set_value.assert_called_once_with(1) for _ in range(3): state.observe(failure) + if failure is None: + assert state.ready + flag.set_value.assert_called_once_with(1) + logger.warning.assert_not_called() + return assert not state.ready assert flag.set_value.call_args.args == (0,) logger.warning.assert_called_once() @@ -438,7 +451,7 @@ def test_readiness_tolerates_isolated_failures_and_confirms_recovery(monkeypatch @pytest.mark.parametrize("failure,reason", [("timeout", "probe_timeout"), ("connection", "probe_connection_failed")]) -def test_readiness_monitor_reports_transport_reason_and_stops_after_http_exit(monkeypatch, failure, reason): +def test_readiness_monitor_transport_failures_preserve_ready_and_stop_after_http_exit(monkeypatch, failure, reason): import requests from unittest.mock import Mock, MagicMock from lightllm.server.router.dynamic_prompt import shared_arr @@ -462,9 +475,8 @@ def test_readiness_monitor_reports_transport_reason_and_stops_after_http_exit(mo args = SimpleNamespace(run_mode="normal", host="127.0.0.1", model_name="test") manager.start_readiness_monitor(process, args) assert session.get.call_count == 4 - logger.warning.assert_called_once() - assert logger.warning.call_args.kwargs["extra"]["reason"] == reason - assert [call.args for call in flag.set_value.call_args_list] == [(0,), (1,), (0,)] + logger.warning.assert_not_called() + assert [call.args for call in flag.set_value.call_args_list] == [(0,), (1,)] session.__exit__.assert_called_once() From 1bc0510cc601f3a7e507f9185fe290959a898905 Mon Sep 17 00:00:00 2001 From: sufubao Date: Thu, 22 Oct 2026 10:26:52 +0800 Subject: [PATCH 10/13] fix: address review findings for #1638 --- .../pd_selector/cache_aware.py | 2 - .../server/test_abort_request_logging.py | 29 ++++++++++++++ unit_tests/server/test_engine_throughput.py | 31 +++++++++++++++ unit_tests/server/test_logging_regressions.py | 39 +++++++++++++++++-- unit_tests/utils/test_log_utils.py | 15 +++++-- 5 files changed, 107 insertions(+), 9 deletions(-) diff --git a/lightllm/server/httpserver_for_pd_master/pd_selector/cache_aware.py b/lightllm/server/httpserver_for_pd_master/pd_selector/cache_aware.py index 14e0e80cee..60f148bb20 100644 --- a/lightllm/server/httpserver_for_pd_master/pd_selector/cache_aware.py +++ b/lightllm/server/httpserver_for_pd_master/pd_selector/cache_aware.py @@ -175,8 +175,6 @@ async def _get_cache_worker( ) match_rate = 0.0 if result.input_char_count == 0 else result.matched_char_count / result.input_char_count - # Cache match inputs are summarized in the final selection decision. - if match_rate <= self.config.cache_threshold or result.prefill_node is None: return None, result diff --git a/unit_tests/server/test_abort_request_logging.py b/unit_tests/server/test_abort_request_logging.py index 06e77ade78..e08b6adfa0 100644 --- a/unit_tests/server/test_abort_request_logging.py +++ b/unit_tests/server/test_abort_request_logging.py @@ -48,3 +48,32 @@ def abort(*a, **k): monkeypatch.setattr(script, "post_abort", abort) assert getattr(script, stage)("http://unused", str(path)) + + +@pytest.mark.parametrize("headers,expected", [({"X-Request-Id": "client.1"}, "client.1"), ({}, "context.1")]) +def test_http_request_header_output_is_consumed_by_abort_lookup(script, monkeypatch, tmp_path, headers, expected): + import asyncio + import io + import logging + from lightllm.server.httpserver import manager as module + + output = io.StringIO() + logger = logging.getLogger("request-header-contract") + monkeypatch.setattr(logger, "handlers", [logging.StreamHandler(output)]) + monkeypatch.setattr(logger, "propagate", False) + monkeypatch.setattr(logger, "level", logging.DEBUG) + logger.setLevel(logging.DEBUG) + monkeypatch.setattr(module, "logger", logger) + monkeypatch.setattr(module, "get_http_request_id", lambda: "context.1") + manager = module.HttpServerManager.__new__(module.HttpServerManager) + asyncio.run(manager._log_req_header(headers, 80)) + line = output.getvalue() + assert "event=request.received" in line + path = tmp_path / "server.log" + path.write_text(line) + assert script.lookup_lightllm_req_id_from_log(str(path), expected, timeout=0.1) == 80 + output.truncate(0) + output.seek(0) + logger.setLevel(logging.INFO) + asyncio.run(manager._log_req_header(headers, 80)) + assert output.getvalue() == "" diff --git a/unit_tests/server/test_engine_throughput.py b/unit_tests/server/test_engine_throughput.py index fc652a8c04..5e46cc2c0e 100644 --- a/unit_tests/server/test_engine_throughput.py +++ b/unit_tests/server/test_engine_throughput.py @@ -175,6 +175,13 @@ def test_batch_records_success_once_before_releasing_request(monkeypatch, counte ) batch = Batch(1, [req], 1) shm_manager, statics = Mock(), Mock() + + def release(released): + assert throughput.completed == 1 + released.is_aborted = True + released.finish_status = FinishStatus() + + shm_manager.put_back_req_obj.side_effect = release batch.filter_out_finished_req(shm_manager, statics, throughput) batch.filter_out_finished_req(shm_manager, statics, throughput) assert throughput.completed == 1 @@ -375,3 +382,27 @@ async def step(): assert first["kv_active_pct_by_local_dp"] == [10.0 * (i + 1) for i in range(dp_size)] assert first["kv_cached_pct_by_local_dp"] == [5.0 * (i + 1) for i in range(dp_size)] assert first["kv_capacity_tokens_per_dp"] == 100 + + +def test_real_shared_counter_owner_and_worker_rows(monkeypatch): + import uuid + from lightllm.server.core.objs.shm_array import ShmArray + from lightllm.utils import shm_utils + + name = "engine_counter_test_" + uuid.uuid4().hex + monkeypatch.setattr(shm_utils, "get_unique_server_name", lambda: name) + owner = engine_stats.EngineCounters(2, owner=True) + workers = [] + try: + for rank in range(2): + worker = engine_stats.EngineCounters(2, rank) + workers.append(worker) + assert isinstance(worker.storage, ShmArray) + assert worker.storage.shape == (2, 2) + worker.add(10 + rank, 2 + rank) + assert owner.storage.arr.tolist() == [[10, 2], [11, 3]] + assert owner.totals() == (21, 5) + finally: + for worker in workers: + worker.storage.detach_shm() + owner.storage.close_shm() diff --git a/unit_tests/server/test_logging_regressions.py b/unit_tests/server/test_logging_regressions.py index 8e5cf9643c..65c66337c1 100644 --- a/unit_tests/server/test_logging_regressions.py +++ b/unit_tests/server/test_logging_regressions.py @@ -187,7 +187,8 @@ def test_dp_shared_kv_transfer_stats(monkeypatch, caplog, master, level): assert records[0].tokens_total == 2 -def test_router_opens_launcher_readiness_key(monkeypatch): +@pytest.mark.parametrize("dp,nnodes,local_dp", [(1, 1, 1), (4, 2, 2), (1, 2, 1)]) +def test_router_opens_launcher_readiness_key(monkeypatch, dp, nnodes, local_dp): from lightllm.server.router import manager as module from lightllm.server.core.objs import StartArgs @@ -201,13 +202,16 @@ class StopInit(Exception): monkeypatch.setattr(module, "SharedInt", shared) monkeypatch.setattr(module, "ShmReqManager", Mock()) - monkeypatch.setattr(module, "EngineCounters", Mock()) + counters = Mock() + monkeypatch.setattr(module, "EngineCounters", counters) monkeypatch.setattr(module, "ReadOnlyStaticsMemoryManager", Mock(side_effect=StopInit)) args = StartArgs() - args.tp = args.dp = args.nnodes = 1 + args.tp = 8 + args.dp, args.nnodes = dp, nnodes args.node_rank = 0 with pytest.raises(StopInit): module.RouterManager(args) + counters.assert_called_once_with(local_dp, owner=True) assert "service_ready" in flags flags["service_ready"].set_value.assert_called_once_with(0) @@ -528,3 +532,32 @@ def test_router_ready_event_uses_agreed_model_capacity(monkeypatch): assert records == [{"event": "kv_cache.ready", "capacity_tokens_per_dp": 8192, "local_dp_count": 2}] router.shm_max_total_token_num.set_value.assert_called_once_with(8192) assert router.args.max_total_token_num == 8192 + + +@pytest.mark.parametrize( + "cache_load,reason,selected_load", + [(120, "cache", 120), (300, "cache_overloaded", 100), (None, "least_load", 100)], +) +def test_busy_prefill_selection_records_decision_metadata(monkeypatch, cache_load, reason, selected_load): + from lightllm.server.httpserver_for_pd_master.pd_selector import cache_aware as module + + policy = module.CacheAwarePolicy.__new__(module.CacheAwarePolicy) + policy.config = SimpleNamespace(balance_rel_threshold=1.5) + least = SimpleNamespace(dispatched_req_num=1, dispatched_prompt_chars=100, client_ip_port="least:8000") + cache = SimpleNamespace( + dispatched_req_num=1, dispatched_prompt_chars=cache_load or 300, client_ip_port="cache:8000" + ) + event = Mock() + monkeypatch.setattr(module, "log_debug_event", event) + selected = policy._select_worker_by_cache_and_load([cache, least], cache if cache_load is not None else None, 20) + assert selected is (cache if reason == "cache" else least) + event.assert_called_once_with( + module.logger, + "pd.prefill_selected", + reason=reason, + selected=selected.client_ip_port, + load=selected_load, + least_load=100, + projected_load=selected_load + 20, + balance_threshold=1.5, + ) diff --git a/unit_tests/utils/test_log_utils.py b/unit_tests/utils/test_log_utils.py index 4692375892..b2db7258a2 100644 --- a/unit_tests/utils/test_log_utils.py +++ b/unit_tests/utils/test_log_utils.py @@ -538,22 +538,26 @@ def test_errors_bypass_sampling_and_pd_stages_are_debug(): log_request_summary(6, status="continued", timing_scope="end_to_end") """, LIGHTLLM_LOG_FORMAT="json", + LIGHTLLM_LOG_LEVEL="DEBUG", LIGHTLLM_LOG_REQUEST_SAMPLE_RATE=0, ) records = [json.loads(line) for line in output.splitlines()] - assert len(records) == 1 - assert records[0]["event"] == "request.failed" - assert records[0]["level"] == "ERROR" + assert [(r["request_id"], r["event"], r["level"]) for r in records] == [ + (2, "request.failed", "ERROR"), + (3, "request.cancelled", "DEBUG"), + ] output = _run_logging( """ from lightllm.utils.log_utils import log_request_summary log_request_summary(4, status="completed", timing_scope="prefill") log_request_summary(5, status="completed", timing_scope="end_to_end") + log_request_summary(7, status="completed", timing_scope="decode") """, LIGHTLLM_LOG_FORMAT="json", + LIGHTLLM_LOG_LEVEL="DEBUG", ) records = [json.loads(line) for line in output.splitlines()] - assert len(records) == 1 and records[0]["request_id"] == 5 + assert [(r["request_id"], r["level"]) for r in records] == [(4, "DEBUG"), (5, "INFO"), (7, "DEBUG")] def test_terminal_error_is_not_hidden_by_successful_sibling(): @@ -677,6 +681,9 @@ def test_default_request_sampling_is_stable_and_bounded(): ("busy", logging.WARNING, "request.failed"), ("rejected", logging.WARNING, "request.rejected"), ("error", logging.ERROR, "request.failed"), + ("timeout", logging.ERROR, "request.failed"), + ("failed", logging.ERROR, "request.failed"), + ("cancelled", logging.DEBUG, "request.cancelled"), ], ) def test_failure_summary_level_and_sampling(monkeypatch, caplog, status, level, event): From 17ca462770eaeb2edb1937abd936338e442b9756 Mon Sep 17 00:00:00 2001 From: sufubao Date: Thu, 22 Oct 2026 17:54:47 +0800 Subject: [PATCH 11/13] fix: preserve stop results and readiness logging --- lightllm/server/core/objs/req.py | 2 + lightllm/server/httpserver/manager.py | 10 ++- .../httpserver_for_pd_master/multiprocess.py | 6 +- lightllm/server/router/engine_stats.py | 4 +- lightllm/utils/start_utils.py | 23 ++++++- .../core/objs/test_stop_sequence_output.py | 37 +++++++++++ unit_tests/server/test_engine_throughput.py | 13 +++- .../server/test_pd_master_haproxy_proxy.py | 24 ++++++- unit_tests/utils/test_start_utils.py | 64 +++++++++++++++++++ 9 files changed, 171 insertions(+), 12 deletions(-) diff --git a/lightllm/server/core/objs/req.py b/lightllm/server/core/objs/req.py index 71f1c2d5ae..29427aa791 100644 --- a/lightllm/server/core/objs/req.py +++ b/lightllm/server/core/objs/req.py @@ -124,6 +124,7 @@ class Req(ctypes.Structure): # True 只表示已发出停止信号，不表示推理已经退出，也不直接决定 API 的 finish_reason。 # HTTP 因 stop 发起 abort 时，输出缓冲记录的结束原因仍为 stop；后续 abort marker 只用于收尾。 ("is_aborted", ctypes.c_bool), + ("is_stop_matched", ctypes.c_bool), # 当FinishStatus 是正常结束状态时，finish_token_index 用于标识结束的 # token 的index位置 ("finish_token_index", ctypes.c_int), @@ -182,6 +183,7 @@ def init( self.is_paused = False self.finish_status = FinishStatus() self.is_aborted = False + self.is_stop_matched = False self.shm_infer_released = False self.shm_cur_kv_len = 0 self.shm_cur_output_len = 0 diff --git a/lightllm/server/httpserver/manager.py b/lightllm/server/httpserver/manager.py index f3094fe7b0..b1365dc9ef 100644 --- a/lightllm/server/httpserver/manager.py +++ b/lightllm/server/httpserver/manager.py @@ -958,8 +958,10 @@ async def stop_request(self, request_id: int): req_status = self.req_id_to_out_inf.get(convert_sub_id_to_group_id(request_id)) if req_status is not None: for req in req_status.group_req_objs.shm_req_objs: - if req.request_id == request_id and not req.finish_status.is_finished(): - req.is_aborted = True + if req.request_id == request_id: + req.is_stop_matched = True + if not req.finish_status.is_finished(): + req.is_aborted = True async def abort(self, group_req_id: int) -> bool: req_status: ReqStatus = self.req_id_to_out_inf.get(group_req_id, None) @@ -1101,7 +1103,9 @@ async def handle_loop(self): if req.finish_token_index != src_index: finish_status = FinishStatus(FinishStatus.NO_FINISH) else: - finish_status = FinishStatus(req.finish_status.status) + finish_status = FinishStatus( + FinishStatus.FINISHED_STOP if req.is_stop_matched else req.finish_status.status + ) await req.merge_final_token_metadata( metadata, diff --git a/lightllm/server/httpserver_for_pd_master/multiprocess.py b/lightllm/server/httpserver_for_pd_master/multiprocess.py index bf5fc505ed..eb73347ee6 100644 --- a/lightllm/server/httpserver_for_pd_master/multiprocess.py +++ b/lightllm/server/httpserver_for_pd_master/multiprocess.py @@ -20,7 +20,7 @@ get_env_start_args, get_unique_server_name, ) -from lightllm.utils.log_utils import init_logger, get_http_logging_args +from lightllm.utils.log_utils import init_logger, get_http_logging_args, http_access_enabled from lightllm.utils.net_utils import format_host_port from lightllm.utils.shm_port_args import ShmPortArgs, get_shm_port_args from lightllm.utils.start_utils import process_manager @@ -114,8 +114,7 @@ def _build_haproxy_config(args: StartArgs, worker_ports: list[int], server_state "", "defaults", " mode http", - " log global", - " option httplog", + *([" log global", " option httplog"] if http_access_enabled() else []), " option http-server-close", " option abortonclose", " option forwardfor", @@ -131,6 +130,7 @@ def _build_haproxy_config(args: StartArgs, worker_ports: list[int], server_state # 对外入口：拒绝内部 P/D 通道，健康接口进入聚合 backend。 "frontend pd_master_public", f" bind {listen_address}", + " http-request set-log-level silent if { path /readiness }", " acl private_pd_channel path -i /pd_register /kv_move_status", " http-request return status 426 if private_pd_channel", " http-request set-header X-Forwarded-Proto http", diff --git a/lightllm/server/router/engine_stats.py b/lightllm/server/router/engine_stats.py index 6f28486612..685c498cdb 100644 --- a/lightllm/server/router/engine_stats.py +++ b/lightllm/server/router/engine_stats.py @@ -42,7 +42,9 @@ def __init__(self, counters): self.last_time = time.monotonic() def finish(self, req): - if not req.is_aborted and (req.finish_status.is_stopped() or req.finish_status.is_finished_length()): + if req.is_stop_matched or ( + not req.is_aborted and (req.finish_status.is_stopped() or req.finish_status.is_finished_length()) + ): self.completed += 1 def has_activity(self): diff --git a/lightllm/utils/start_utils.py b/lightllm/utils/start_utils.py index daa927c4be..617c15f340 100644 --- a/lightllm/utils/start_utils.py +++ b/lightllm/utils/start_utils.py @@ -107,7 +107,17 @@ def start_readiness_monitor(self, http_server_process, args): port = ports.config_server_port if args.run_mode == "config_server" else ports.port host = {"0.0.0.0": "127.0.0.1", "::": "::1"}.get(bind_host, bind_host) host = f"[{host}]" if ":" in host else host - address = f"http://{host}:{port}" + tls_config = None + if getattr(args, "hypercorn_config", None) and not ( + args.run_mode == "pd_master" and args.httpserver_workers > 1 + ): + from hypercorn.__main__ import _load_config + + config = _load_config(args.hypercorn_config) + if config.ssl_enabled: + tls_config = config + scheme = "https" if tls_config else "http" + address = f"{scheme}://{host}:{port}" started = float(os.getenv("LIGHTLLM_SERVICE_START_TIME", str(time.monotonic()))) self.readiness_stop.clear() @@ -115,6 +125,17 @@ def monitor(): state = ServiceReadiness(ready_flag, args, address, started) with requests.Session() as session: session.trust_env = False + if tls_config: + import ssl + from requests.adapters import HTTPAdapter + + context = ssl.create_default_context(cafile=tls_config.certfile) + context.check_hostname = False + context.verify_flags |= ssl.VERIFY_X509_PARTIAL_CHAIN + adapter = HTTPAdapter() + adapter.init_poolmanager(1, 1, ssl_context=context, assert_hostname=False) + session.mount("https://", adapter) + session.verify = tls_config.certfile while not self.readiness_stop.is_set() and http_server_process.poll() is None: reason = None try: diff --git a/unit_tests/server/core/objs/test_stop_sequence_output.py b/unit_tests/server/core/objs/test_stop_sequence_output.py index 4e0d514c63..ef9a699cf9 100644 --- a/unit_tests/server/core/objs/test_stop_sequence_output.py +++ b/unit_tests/server/core/objs/test_stop_sequence_output.py @@ -77,6 +77,7 @@ def _req(tokens, finish=FinishStatus.NO_FINISH, params=None, request_id=80): finish_token_index=1 + len(tokens) if finish != FinishStatus.NO_FINISH else -1, finish_status=FinishStatus(finish), is_aborted=False, + is_stop_matched=False, candetoken_out_len=len(tokens), out_tokens_queue=CircularQueue(), can_released_mark=False, @@ -1261,3 +1262,39 @@ def test_simulated_finish_releases_prefix_and_preserves_payloads(): ready_tokens = _drain_ready_tokens(buffer) assert ready_tokens == [("helloE", first_payload), ("", finish_payload)] assert buffer.state is buffer.State.STOPPED + + +@pytest.mark.parametrize("cancel_second", [False, True]) +def test_stop_cleanup_accounting(monkeypatch, cancel_second): + from lightllm.server.core.objs import req as req_module + from lightllm.server.router.engine_stats import EngineThroughput + from lightllm.utils.log_utils import request_finish_fields + + manager = _http_manager() + reqs = [_req([10, 11], request_id=80 + i) for i in range(2)] + manager.req_id_to_out_inf[80] = SimpleNamespace(group_req_objs=SimpleNamespace(shm_req_objs=reqs, group_req_id=80)) + monkeypatch.setattr(req_module, "get_env_start_args", lambda: SimpleNamespace(eos_id=[0])) + throughput = EngineThroughput(SimpleNamespace(totals=lambda: (0, 0))) + reasons = set() + for index, req in enumerate(reqs): + req.shm_logprobs.arr["rank"] = np.zeros(4) + req.mark_simulated_finished = lambda status, output_len, req=req: Req.mark_simulated_finished( + req, status, output_len + ) + if index == 1 and cancel_second: + asyncio.run(manager.abort(80)) + else: + buffer = StopStrMatchBuffer(manager, _params(), []) + _handle_output(buffer, req.request_id, "helloEND", {"id": 10}, FinishStatus()) + assert buffer.ready_token_list[-1][3].is_stopped() + assert req.is_aborted + infer_req = SimpleNamespace(shm_req=req, finish_status=FinishStatus(), cur_output_len=1) + infer_batch.InferReq.mark_shm_aborted_finished(infer_req) + assert req.finish_status.get_finish_reason() == "abort" + throughput.finish(req) + req.out_tokens_queue.push("", req.finish_token_index, False, 2) + output = asyncio.run(_read_http_tokens(manager, req)) + reasons.add(output[-1][3].get_finish_reason()) + assert throughput.completed == (1 if cancel_second else 2) + fields = request_finish_fields(reasons) + assert fields["status"] == ("abort" if cancel_second else "completed") diff --git a/unit_tests/server/test_engine_throughput.py b/unit_tests/server/test_engine_throughput.py index 5e46cc2c0e..6e7f745272 100644 --- a/unit_tests/server/test_engine_throughput.py +++ b/unit_tests/server/test_engine_throughput.py @@ -54,8 +54,16 @@ def test_running_request_rates_and_completed_count_use_window_deltas(monkeypatch assert first["output_tok_s"] == 6.4 assert first["completed_req_s"] == 0 # Still running. writer.add(128, 32) - throughput.finish(SimpleNamespace(is_aborted=False, finish_status=FinishStatus(FinishStatus.FINISHED_LENGTH))) - throughput.finish(SimpleNamespace(is_aborted=True, finish_status=FinishStatus(FinishStatus.FINISHED_ABORTED))) + throughput.finish( + SimpleNamespace( + is_stop_matched=False, is_aborted=False, finish_status=FinishStatus(FinishStatus.FINISHED_LENGTH) + ) + ) + throughput.finish( + SimpleNamespace( + is_stop_matched=False, is_aborted=True, finish_status=FinishStatus(FinishStatus.FINISHED_ABORTED) + ) + ) now[0] = 20 second = throughput.snapshot() assert second["prefill_tokens"] == 128 and second["output_tokens"] == 32 @@ -170,6 +178,7 @@ def test_batch_records_success_once_before_releasing_request(monkeypatch, counte request_id=42, shm_infer_released=True, is_aborted=False, + is_stop_matched=False, candetoken_out_len=10, finish_status=FinishStatus(FinishStatus.FINISHED_STOP), ) diff --git a/unit_tests/server/test_pd_master_haproxy_proxy.py b/unit_tests/server/test_pd_master_haproxy_proxy.py index 3642bc69e3..fe8aa172fe 100644 --- a/unit_tests/server/test_pd_master_haproxy_proxy.py +++ b/unit_tests/server/test_pd_master_haproxy_proxy.py @@ -178,8 +178,10 @@ def do_POST(self): check = subprocess.run([executable, "-c", "-f", str(config)], capture_output=True, text=True, timeout=5) assert check.returncode == 0, check.stderr command = [executable, "-db", "-f", str(config)] - with (tmp_path / "haproxy.stderr").open("wb") as error_log: - haproxy = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=error_log, start_new_session=True) + with (tmp_path / "haproxy.stderr").open("wb") as error_log, (tmp_path / "haproxy.stdout").open( + "wb" + ) as access_log: + haproxy = subprocess.Popen(command, stdout=access_log, stderr=error_log, start_new_session=True) state["haproxy"] = haproxy opener = build_opener(ProxyHandler({})) base_url = f"http://127.0.0.1:{public_port}" @@ -432,3 +434,21 @@ def test_real_haproxy_allows_slow_health_checks_and_rejects_a_timed_out_worker(h _wait_for_probe_response(opener, base_url, "/health", 503, timeout=30) assert _probe_response(opener, base_url, "/liveness")[0] == 200 assert _probe_response(opener, base_url, "/readiness")[0] == 200 + + +@pytest.mark.parametrize("access", ["0", "1"]) +def test_proxy_access_logging(monkeypatch, request, access): + monkeypatch.setenv("LIGHTLLM_LOG_HTTP_ACCESS", access) + base_url, opener, state = request.getfixturevalue("haproxy_proxy") + for path in ("/readiness", "/worker"): + try: + with opener.open(base_url + path, timeout=3) as response: + response.read() + except HTTPError as response: + assert response.code == 503 + response.close() + state["haproxy"].send_signal(signal.SIGUSR1) + state["haproxy"].wait(timeout=5) + output = (state["runtime"] / "haproxy.stdout").read_text() + assert "GET /readiness" not in output + assert ("GET /worker" in output) == (access == "1") diff --git a/unit_tests/utils/test_start_utils.py b/unit_tests/utils/test_start_utils.py index e233ca4ace..0ef63189d8 100644 --- a/unit_tests/utils/test_start_utils.py +++ b/unit_tests/utils/test_start_utils.py @@ -506,3 +506,67 @@ def test_readiness_monitor_stops_when_http_process_exits(monkeypatch, poll_resul assert session.get.call_count == probe_count assert [call.args for call in flag.set_value.call_args_list] == ([(0,), (1,)] if ready else [(0,)]) session.__exit__.assert_called_once() + + +def test_readiness_monitor_tls(monkeypatch, tmp_path): + import ssl + import subprocess + import threading + from http.server import BaseHTTPRequestHandler, HTTPServer + from unittest.mock import Mock + from lightllm.server.router.dynamic_prompt import shared_arr + from lightllm.utils import shm_port_args + + cert, key = tmp_path / "cert.pem", tmp_path / "key.pem" + subprocess.run( + [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-days", + "1", + "-subj", + "/CN=service.example", + "-keyout", + str(key), + "-out", + str(cert), + ], + check=True, + capture_output=True, + ) + config = tmp_path / "hypercorn.toml" + config.write_text(f'certfile = "{cert}"\nkeyfile = "{key}"\n') + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + assert self.path == "/readiness" + self.send_response(200) + self.end_headers() + + def log_message(self, *args): + pass + + server = HTTPServer(("127.0.0.1", 0), Handler) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(cert, key) + server.socket = context.wrap_socket(server.socket, server_side=True) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + manager = start_utils.SubmoduleManager() + flag = Mock() + monkeypatch.setattr(shared_arr, "SharedInt", lambda name: flag) + monkeypatch.setattr(shm_port_args, "get_shm_port_args", lambda: SimpleNamespace(port=server.server_port)) + monkeypatch.setattr(start_utils.threading, "Thread", lambda target, **kw: SimpleNamespace(start=target)) + monkeypatch.setattr(manager.readiness_stop, "wait", lambda timeout: manager.readiness_stop.set()) + args = SimpleNamespace(run_mode="normal", host="0.0.0.0", model_name="test", hypercorn_config=str(config)) + try: + manager.start_readiness_monitor(FakeHttpServerProcess(), args) + assert [call.args for call in flag.set_value.call_args_list] == [(0,), (1,)] + finally: + server.shutdown() + server.server_close() + worker.join() From d995187858da6b208f9f5205ec9e8d177b46bfb5 Mon Sep 17 00:00:00 2001 From: sufubao Date: Mon, 26 Oct 2026 11:44:28 +0800 Subject: [PATCH 12/13] fix: correct engine stats windows and support mTLS readiness --- docs/CN/source/tutorial/logging.rst | 5 ++ docs/EN/source/tutorial/logging.rst | 6 ++ lightllm/server/router/manager.py | 9 ++- lightllm/utils/start_utils.py | 3 + unit_tests/server/test_engine_throughput.py | 15 ++-- unit_tests/utils/test_start_utils.py | 88 +++++++++++++++++++++ 6 files changed, 118 insertions(+), 8 deletions(-) diff --git a/docs/CN/source/tutorial/logging.rst b/docs/CN/source/tutorial/logging.rst index 8240ec5c3e..c245034164 100644 --- a/docs/CN/source/tutorial/logging.rst +++ b/docs/CN/source/tutorial/logging.rst @@ -207,3 +207,8 @@ INFO 启动日志以 HTTP /readiness 成功后的 service.ready 为准，包含 运行时每个节点仅打印一份吞吐与队列摘要；KV 活跃占用与可复用缓存分开显示。 文本省略原始窗口计数，JSON 保留全部计数与时间窗口。P/D 内部请求摘要只在 DEBUG 输出。 失败摘要不采样，取消请求只在 DEBUG 输出；终止原因包含 stop、length、abort 或 error。 + +HTTPS 启用客户端证书认证时，将 ``LIGHTLLM_READINESS_CLIENT_CERT`` 设置为 Hypercorn +信任的 PEM 客户端证书路径，将 ``LIGHTLLM_READINESS_CLIENT_KEY`` 设置为对应的未加密 +PEM 私钥路径。证书文件已包含私钥时可省略后者。探针仍验证服务端证书；请使用具有 +客户端认证权限的独立身份。 diff --git a/docs/EN/source/tutorial/logging.rst b/docs/EN/source/tutorial/logging.rst index 5f9a49604c..c74f333977 100644 --- a/docs/EN/source/tutorial/logging.rst +++ b/docs/EN/source/tutorial/logging.rst @@ -196,3 +196,9 @@ INFO reports one throughput, queue and KV usage snapshot per node. Active KV and cache occupancy are separate. JSON retains raw window counts omitted from compact text. Successful request summaries default to 1% sampling; P/D stage summaries are DEBUG only. Failures bypass sampling; cancellations are DEBUG. Terminal reasons are preserved. + +For HTTPS with required client certificates, set ``LIGHTLLM_READINESS_CLIENT_CERT`` +to a PEM client certificate trusted by Hypercorn and ``LIGHTLLM_READINESS_CLIENT_KEY`` +to its unencrypted PEM private key. The key setting can be omitted when the key is +in the certificate file. The probe continues to verify the server certificate; +use a dedicated client identity with client-authentication permission. diff --git a/lightllm/server/router/manager.py b/lightllm/server/router/manager.py index 8edaf0665b..ce31fc09e5 100644 --- a/lightllm/server/router/manager.py +++ b/lightllm/server/router/manager.py @@ -254,13 +254,16 @@ async def loop_for_fwd( # slave routers mirror requests and must not duplicate the master summary. running = len(self.running_batch.reqs) if self.running_batch is not None else 0 waiting = self.req_queue.get_wait_req_num() + if self.schedule_new_batch is not None: + waiting += len(self.schedule_new_batch.reqs) now = time.monotonic() active = bool(running or waiting) or self.engine_throughput.has_activity() if self.service_ready.get_value() and not seen_ready: - self.engine_throughput.resume(last_poll) - stats_logger.last_time = last_poll + if not resumed: + self.engine_throughput.resume(last_poll) + stats_logger.last_time = last_poll seen_ready = True - if seen_ready and stats_logger.active is not True and active and not resumed: + if stats_logger.active is not True and active and not resumed: self.engine_throughput.resume(last_poll) stats_logger.last_time = last_poll resumed = True diff --git a/lightllm/utils/start_utils.py b/lightllm/utils/start_utils.py index 617c15f340..69c23bc9db 100644 --- a/lightllm/utils/start_utils.py +++ b/lightllm/utils/start_utils.py @@ -132,6 +132,9 @@ def monitor(): context = ssl.create_default_context(cafile=tls_config.certfile) context.check_hostname = False context.verify_flags |= ssl.VERIFY_X509_PARTIAL_CHAIN + client_cert = os.getenv("LIGHTLLM_READINESS_CLIENT_CERT") + if client_cert: + context.load_cert_chain(client_cert, os.getenv("LIGHTLLM_READINESS_CLIENT_KEY")) adapter = HTTPAdapter() adapter.init_poolmanager(1, 1, ssl_context=context, assert_hostname=False) session.mount("https://", adapter) diff --git a/unit_tests/server/test_engine_throughput.py b/unit_tests/server/test_engine_throughput.py index 6e7f745272..74620e070e 100644 --- a/unit_tests/server/test_engine_throughput.py +++ b/unit_tests/server/test_engine_throughput.py @@ -230,6 +230,7 @@ def test_router_first_burst_excludes_initial_idle(monkeypatch, counters, node_ra router.req_queue = Mock() router.req_queue.get_wait_req_num.return_value = 0 router.running_batch = None + router.schedule_new_batch = None router.dp_size, router.node_rank, router.dp_size_in_node = dp_size, node_rank, 2 router.radix_cache_client = None router.max_total_token_num = 100 @@ -340,7 +341,10 @@ class ModelSetupBoundary(Exception): @pytest.mark.parametrize("node_rank,dp_size", [(0, 1), (1, 1), (0, 2), (1, 2)]) -def test_router_statistics_wait_for_readiness_and_emit_once_per_node(monkeypatch, counters, node_rank, dp_size): +@pytest.mark.parametrize("queued,staged", [(2, 0), (0, 5)]) +def test_router_statistics_wait_for_readiness_and_emit_once_per_node( + monkeypatch, counters, node_rank, dp_size, queued, staged +): import asyncio from lightllm.server.router import manager as router @@ -354,8 +358,9 @@ def test_router_statistics_wait_for_readiness_and_emit_once_per_node(monkeypatch manager = router.RouterManager.__new__(router.RouterManager) manager.engine_counters = counters manager.running_batch = None + manager.schedule_new_batch = SimpleNamespace(reqs=[Mock() for _ in range(staged)]) if staged else None manager.req_queue = Mock() - manager.req_queue.get_wait_req_num.return_value = 2 + manager.req_queue.get_wait_req_num.return_value = queued manager.node_rank, manager.dp_size, manager.dp_size_in_node = node_rank, dp_size, dp_size manager.max_total_token_num = 100 manager.get_used_tokens = lambda index: 10 * (index + 1) @@ -385,9 +390,9 @@ async def step(): assert logger.info.call_count == 2 first, second = [call.kwargs["extra"] for call in logger.info.call_args_list] assert first["prefill_tokens"] == 200 and second["prefill_tokens"] == 100 - assert first["window_s"] == second["window_s"] == 10 - assert first["output_tok_s"] == 2 and second["output_tok_s"] == 1 - assert first["running"] == 0 and first["waiting"] == 2 and first["paused"] == 1 + assert first["window_s"] == 20 and second["window_s"] == 10 + assert first["output_tok_s"] == second["output_tok_s"] == 1 + assert first["running"] == 0 and first["waiting"] == queued + staged and first["paused"] == 1 assert first["kv_active_pct_by_local_dp"] == [10.0 * (i + 1) for i in range(dp_size)] assert first["kv_cached_pct_by_local_dp"] == [5.0 * (i + 1) for i in range(dp_size)] assert first["kv_capacity_tokens_per_dp"] == 100 diff --git a/unit_tests/utils/test_start_utils.py b/unit_tests/utils/test_start_utils.py index 0ef63189d8..f606c6d9d0 100644 --- a/unit_tests/utils/test_start_utils.py +++ b/unit_tests/utils/test_start_utils.py @@ -570,3 +570,91 @@ def log_message(self, *args): server.shutdown() server.server_close() worker.join() + + +@pytest.mark.parametrize("identity", [None, "separate", "combined"]) +def test_readiness_monitor_mutual_tls(monkeypatch, tmp_path, identity): + import ssl + import subprocess + import requests + from unittest.mock import MagicMock, Mock + from lightllm.server.router.dynamic_prompt import shared_arr + from lightllm.utils import shm_port_args + + for name in ("server", "client"): + subprocess.run( + [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-days", + "1", + "-subj", + f"/CN={name}", + "-keyout", + str(tmp_path / f"{name}.key"), + "-out", + str(tmp_path / f"{name}.pem"), + ], + check=True, + capture_output=True, + ) + config = tmp_path / "hypercorn.toml" + config.write_text(f'certfile = "{tmp_path / "server.pem"}"\nkeyfile = "{tmp_path / "server.key"}"\n') + monkeypatch.delenv("LIGHTLLM_READINESS_CLIENT_CERT", raising=False) + monkeypatch.delenv("LIGHTLLM_READINESS_CLIENT_KEY", raising=False) + if identity == "separate": + monkeypatch.setenv("LIGHTLLM_READINESS_CLIENT_CERT", str(tmp_path / "client.pem")) + monkeypatch.setenv("LIGHTLLM_READINESS_CLIENT_KEY", str(tmp_path / "client.key")) + elif identity == "combined": + combined = tmp_path / "identity.pem" + combined.write_bytes((tmp_path / "client.pem").read_bytes() + (tmp_path / "client.key").read_bytes()) + monkeypatch.setenv("LIGHTLLM_READINESS_CLIENT_CERT", str(combined)) + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.load_cert_chain(tmp_path / "server.pem", tmp_path / "server.key") + server_context.load_verify_locations(tmp_path / "client.pem") + server_context.verify_mode = ssl.CERT_REQUIRED + session = MagicMock() + session.__enter__.return_value = session + + def probe(url, timeout): + assert url == "https://127.0.0.1:8000/readiness" + context = session.mount.call_args.args[1].poolmanager.connection_pool_kw["ssl_context"] + server_in, server_out, client_in, client_out = [ssl.MemoryBIO() for _ in range(4)] + server = server_context.wrap_bio(server_in, server_out, server_side=True) + client = context.wrap_bio(client_in, client_out, server_side=False, server_hostname="server") + finished = set() + try: + for _ in range(20): + for peer, outgoing, incoming in ((client, client_out, server_in), (server, server_out, client_in)): + try: + peer.do_handshake() + finished.add(peer) + except ssl.SSLWantReadError: + pass + incoming.write(outgoing.read()) + if len(finished) == 2: + assert server.getpeercert() + return SimpleNamespace(status_code=200) + except ssl.SSLError as error: + assert identity is None + assert error.reason == "PEER_DID_NOT_RETURN_A_CERTIFICATE" + raise requests.exceptions.SSLError(str(error)) from error + pytest.fail("TLS handshake did not finish") + + manager = start_utils.SubmoduleManager() + flag = Mock() + monkeypatch.setattr(shared_arr, "SharedInt", lambda name: flag) + monkeypatch.setattr(shm_port_args, "get_shm_port_args", lambda: SimpleNamespace(port=8000)) + monkeypatch.setattr(requests, "Session", lambda: session) + monkeypatch.setattr(start_utils.threading, "Thread", lambda target, **kw: SimpleNamespace(start=target)) + monkeypatch.setattr(manager.readiness_stop, "wait", lambda timeout: manager.readiness_stop.set()) + session.get.side_effect = probe + args = SimpleNamespace(run_mode="normal", host="0.0.0.0", model_name="test", hypercorn_config=str(config)) + manager.start_readiness_monitor(FakeHttpServerProcess(), args) + session.get.assert_called_once() + assert [call.args for call in flag.set_value.call_args_list] == ([(0,)] if identity is None else [(0,), (1,)]) From 8a97175635784cd132a6e8ebd7741aa67eb9b06f Mon Sep 17 00:00:00 2001 From: sufubao Date: Tue, 27 Oct 2026 12:03:07 +0800 Subject: [PATCH 13/13] fix: preserve stop metadata and count prefill recomputation --- lightllm/server/httpserver/manager.py | 5 +- .../server/router/model_infer/infer_batch.py | 1 + .../model_infer/mode_backend/base_backend.py | 5 +- .../mode_backend/diverse_backend/impl.py | 5 +- .../mode_backend/generic_pre_process.py | 2 + lightllm/utils/start_utils.py | 19 +++++- .../core/objs/test_stop_sequence_output.py | 32 ++++++++++ .../mode_backend/test_diverse_backend.py | 7 +++ unit_tests/server/test_engine_throughput.py | 58 ++++++++++++++++++- unit_tests/utils/test_start_utils.py | 20 +++++-- 10 files changed, 135 insertions(+), 19 deletions(-) diff --git a/lightllm/server/httpserver/manager.py b/lightllm/server/httpserver/manager.py index b1365dc9ef..9f027395aa 100644 --- a/lightllm/server/httpserver/manager.py +++ b/lightllm/server/httpserver/manager.py @@ -1103,8 +1103,9 @@ async def handle_loop(self): if req.finish_token_index != src_index: finish_status = FinishStatus(FinishStatus.NO_FINISH) else: + backend_finish_status = req.finish_status.status finish_status = FinishStatus( - FinishStatus.FINISHED_STOP if req.is_stop_matched else req.finish_status.status + FinishStatus.FINISHED_STOP if req.is_stop_matched else backend_finish_status ) await req.merge_final_token_metadata( @@ -1116,7 +1117,7 @@ async def handle_loop(self): # mark_simulated_finished 追加的 EOS 只负责唤醒 Detoken/HTTP wait。 # 对外将它转换成无 token 的 finish marker，VERL 会按 id=None # 跳过 token/logprob，同时仍可读取 finish status 和路由信息。 - if finish_status.status == FinishStatus.FINISHED_ABORTED: + if backend_finish_status == FinishStatus.FINISHED_ABORTED: text = "" metadata["id"] = None metadata["logprob"] = None diff --git a/lightllm/server/router/model_infer/infer_batch.py b/lightllm/server/router/model_infer/infer_batch.py index 8ecfe836d4..8da9775472 100644 --- a/lightllm/server/router/model_infer/infer_batch.py +++ b/lightllm/server/router/model_infer/infer_batch.py @@ -548,6 +548,7 @@ def __init__( # cur_kv_len is the logical length already written; hold_kv_len is the # page-aligned physical capacity owned by this request. self.hold_kv_len = 0 + self.prefill_tokens = 0 # 请求需要被暂停 self.wait_pause = False diff --git a/lightllm/server/router/model_infer/mode_backend/base_backend.py b/lightllm/server/router/model_infer/mode_backend/base_backend.py index 1b675aaf3c..4d9794019c 100644 --- a/lightllm/server/router/model_infer/mode_backend/base_backend.py +++ b/lightllm/server/router/model_infer/mode_backend/base_backend.py @@ -860,9 +860,8 @@ def _pre_post_handle(self, run_reqs: List[InferReq], is_chuncked_mode: bool) -> new_kv_len = req_obj.get_chuncked_input_token_len() else: new_kv_len = req_obj.get_cur_total_len() - prefill_tokens = max( - 0, min(new_kv_len, req_obj.shm_req.input_len) - min(req_obj.cur_kv_len, req_obj.shm_req.input_len) - ) + prefill_tokens = req_obj.prefill_tokens + req_obj.prefill_tokens = 0 req_obj.cur_kv_len = new_kv_len if is_master_in_dp: req_obj.shm_req.shm_cur_kv_len = req_obj.cur_kv_len diff --git a/lightllm/server/router/model_infer/mode_backend/diverse_backend/impl.py b/lightllm/server/router/model_infer/mode_backend/diverse_backend/impl.py index c14958446f..0577a46df2 100644 --- a/lightllm/server/router/model_infer/mode_backend/diverse_backend/impl.py +++ b/lightllm/server/router/model_infer/mode_backend/diverse_backend/impl.py @@ -138,9 +138,8 @@ def _diverse_pre_post_handle(self, run_reqs: List[InferReq], is_chuncked_mode: b new_kv_len = req_obj.get_chuncked_input_token_len() else: new_kv_len = req_obj.get_cur_total_len() - prefill_tokens = max( - 0, min(new_kv_len, req_obj.shm_req.input_len) - min(req_obj.cur_kv_len, req_obj.shm_req.input_len) - ) + prefill_tokens = req_obj.prefill_tokens + req_obj.prefill_tokens = 0 req_obj.cur_kv_len = new_kv_len if is_master_in_dp: req_obj.shm_req.shm_cur_kv_len = req_obj.cur_kv_len diff --git a/lightllm/server/router/model_infer/mode_backend/generic_pre_process.py b/lightllm/server/router/model_infer/mode_backend/generic_pre_process.py index 7d3f34c4a7..4fb0d946dc 100644 --- a/lightllm/server/router/model_infer/mode_backend/generic_pre_process.py +++ b/lightllm/server/router/model_infer/mode_backend/generic_pre_process.py @@ -45,9 +45,11 @@ def prepare_prefill_inputs(req_objs: List[InferReq], is_chuncked_mode: bool) -> b_mtp_index.append(0) if hasattr(req, "is_decode_req_mixed_in_prefill"): b_is_decode_req.append(True) + req.prefill_tokens = 0 del req.is_decode_req_mixed_in_prefill else: b_is_decode_req.append(False) + req.prefill_tokens = input_token_len # DP 模式下某个 rank 可能没有本地请求。这里保留真实的 0 shape， # 推理所需的 dummy request 统一由 BaseModel 在执行前补齐。 diff --git a/lightllm/utils/start_utils.py b/lightllm/utils/start_utils.py index 69c23bc9db..a306446070 100644 --- a/lightllm/utils/start_utils.py +++ b/lightllm/utils/start_utils.py @@ -60,12 +60,23 @@ def observe(self, status, reason=None): self.consecutive_successes = 0 # A transport failure cannot establish that a live HTTP worker is unready. self.consecutive_failures = 0 if status is None else self.consecutive_failures + 1 - if status not in (None, 503) and not self.ever_ready and not self.probe_failure_logged: + if ( + (status not in (None, 503) or reason == "probe_tls_failed") + and not self.ever_ready + and not self.probe_failure_logged + ): + reason = reason or "unexpected_readiness_status" logger.warning( - "event=service.probe_failed address=%s status=%s", + "event=service.probe_failed address=%s status=%s reason=%s", self.address, status, - extra={"event": "service.probe_failed", "address": self.address, "http_status": status}, + reason, + extra={ + "event": "service.probe_failed", + "address": self.address, + "http_status": status, + "reason": reason, + }, ) self.probe_failure_logged = True if self.ready and self.consecutive_failures >= 3: @@ -146,6 +157,8 @@ def monitor(): status = response.status_code except requests.Timeout: status, reason = None, "probe_timeout" + except requests.exceptions.SSLError: + status, reason = None, "probe_tls_failed" except requests.RequestException: status, reason = None, "probe_connection_failed" if self.readiness_stop.is_set() or http_server_process.poll() is not None: diff --git a/unit_tests/server/core/objs/test_stop_sequence_output.py b/unit_tests/server/core/objs/test_stop_sequence_output.py index ef9a699cf9..d987a59ff7 100644 --- a/unit_tests/server/core/objs/test_stop_sequence_output.py +++ b/unit_tests/server/core/objs/test_stop_sequence_output.py @@ -1298,3 +1298,35 @@ def test_stop_cleanup_accounting(monkeypatch, cancel_second): assert throughput.completed == (1 if cancel_second else 2) fields = request_finish_fields(reasons) assert fields["status"] == ("abort" if cancel_second else "completed") + + +@pytest.mark.parametrize("final_metadata", ["enable_rl", "enable_return_routed_experts", "prompt_logprobs"]) +def test_stop_final_metadata_has_no_simulated_token(monkeypatch, final_metadata): + from lightllm.server.core.objs import req as req_module + + manager = _http_manager() + params = _params() + if final_metadata == "prompt_logprobs": + params.prompt_logprobs = 0 + else: + setattr(manager.args, final_metadata, True) + req = _req([10, 11], params=params) + req.shm_logprobs.arr["rank"] = np.zeros(4) + req.get_output_logprobs_metadata = lambda *args: {"2": {"logprob": 0.0}} + manager.req_id_to_out_inf[80] = SimpleNamespace(group_req_objs=SimpleNamespace(shm_req_objs=[req])) + buffer = StopStrMatchBuffer(manager, params, []) + _handle_output(buffer, req.request_id, "helloEND", {"id": 10}, FinishStatus()) + assert buffer.state is buffer.State.WAIT_LAST_PACK + assert req.is_stop_matched and req.is_aborted + monkeypatch.setattr(req_module, "get_env_start_args", lambda: SimpleNamespace(eos_id=[2])) + assert Req.mark_simulated_finished(req, FinishStatus.FINISHED_ABORTED, 1) + req.out_tokens_queue.push("", req.finish_token_index, False, 2) + output = asyncio.run(_read_http_tokens(manager, req)) + _handle_output(buffer, *output[-1]) + _, text, metadata, finish = buffer.ready_token_list[-1] + assert finish.get_finish_reason() == "stop" + assert text == "" + assert metadata["id"] is None + assert metadata["logprob"] is None + assert metadata["logprobs"] == {} + req.merge_final_token_metadata.assert_awaited_once() diff --git a/unit_tests/server/router/model_infer/mode_backend/test_diverse_backend.py b/unit_tests/server/router/model_infer/mode_backend/test_diverse_backend.py index acb24c5273..c7c48f4443 100644 --- a/unit_tests/server/router/model_infer/mode_backend/test_diverse_backend.py +++ b/unit_tests/server/router/model_infer/mode_backend/test_diverse_backend.py @@ -82,6 +82,8 @@ def test_diverse_prefill_counts_increment_once_and_slaves_do_not_duplicate(monke backend.is_master_in_dp = True backend._master_req_to_radix_cache = Mock() backend._copy_master_req_to_slave_req = Mock() + from lightllm.server.router.model_infer.mode_backend.generic_pre_process import prepare_prefill_inputs + target = [512] master = SimpleNamespace( is_master_req=lambda: True, @@ -91,7 +93,11 @@ def test_diverse_prefill_counts_increment_once_and_slaves_do_not_duplicate(monke cur_output_len=0, slave_reqs=[], shm_req=SimpleNamespace(input_len=800, group_req_id=8), + req_idx=0, + multimodal_params=None, + get_chuncked_input_token_ids=lambda: [0] * target[0], ) + prepare_prefill_inputs([master], True) first = backend._diverse_pre_post_handle([master], True) assert first[0].prefill_tokens == 512 and first[0].output_len == 0 target[0] = 800 @@ -99,6 +105,7 @@ def test_diverse_prefill_counts_increment_once_and_slaves_do_not_duplicate(monke is_master_req=lambda: False, remove_master_req=Mock(), shm_req=SimpleNamespace(group_req_id=8) ) master.slave_reqs = [slave] + prepare_prefill_inputs([master], True) second = backend._diverse_pre_post_handle([master, slave], True) assert [pack.prefill_tokens for pack in second] == [288, 0] assert sum(pack.prefill_tokens for pack in first + second) == 800 diff --git a/unit_tests/server/test_engine_throughput.py b/unit_tests/server/test_engine_throughput.py index 74620e070e..224eeb6f4c 100644 --- a/unit_tests/server/test_engine_throughput.py +++ b/unit_tests/server/test_engine_throughput.py @@ -100,10 +100,19 @@ def request(input_len, cached=0, output_len=0, max_output=100): shm = SimpleNamespace(input_len=input_len, shm_cur_kv_len=cached, shm_cur_output_len=output_len) status = FinishStatus() req = SimpleNamespace( - cur_kv_len=cached, cur_output_len=output_len, shm_req=shm, finish_status=status, output_constraint=None + cur_kv_len=cached, + prefill_tokens=0, + cur_output_len=output_len, + shm_req=shm, + finish_status=status, + output_constraint=None, + req_idx=0, + multimodal_params=None, ) req.get_cur_total_len = lambda: input_len + req.cur_output_len req.get_chuncked_input_token_len = lambda: min(req.cur_kv_len + 512, req.get_cur_total_len()) + req.get_input_token_ids = lambda: np.zeros(req.get_cur_total_len(), dtype=np.int64) + req.get_chuncked_input_token_ids = lambda: req.get_input_token_ids()[: req.get_chuncked_input_token_len()] req.set_next_gen_token_id = Mock() def update(eos_ids, output_len): @@ -136,15 +145,22 @@ def post(result, reqs, packs): ) +def prefill(result, reqs, chunked=True): + from lightllm.server.router.model_infer.mode_backend.generic_pre_process import prepare_prefill_inputs + + prepare_prefill_inputs(reqs, is_chuncked_mode=chunked) + return result._pre_post_handle(reqs, is_chuncked_mode=chunked) + + def test_chunked_prefill_counts_computed_input_only_after_postprocessing(monkeypatch, counters): result = backend(monkeypatch, counters) req = request(1000, cached=200) - first = result._pre_post_handle([req], is_chuncked_mode=True) + first = prefill(result, [req]) assert first[0].prefill_tokens == 512 assert counters.totals() == (0, 0) # No work reported before forward synchronization. post(result, [req], first) assert counters.totals() == (512, 0) - second = result._pre_post_handle([req], is_chuncked_mode=True) + second = prefill(result, [req]) post(result, [req], second) assert counters.totals() == (800, 1) @@ -420,3 +436,39 @@ def test_real_shared_counter_owner_and_worker_rows(monkeypatch): for worker in workers: worker.storage.detach_shm() owner.storage.close_shm() + + +@pytest.mark.parametrize("cached", [0, 128, 2175]) +@pytest.mark.parametrize("chunked", [True, False]) +@pytest.mark.parametrize("diverse", [False, True]) +def test_recompute_generated_context(monkeypatch, counters, cached, chunked, diverse): + result = backend(monkeypatch, counters) + if diverse: + from lightllm.server.router.model_infer.mode_backend.diverse_backend.impl import DiversehBackend + + monkeypatch.setattr(torch.cuda, "current_stream", lambda: Mock()) + result._pre_post_handle = DiversehBackend._diverse_pre_post_handle.__get__(result) + req = request(128, cached=cached, output_len=2048, max_output=4096) + req.is_master_req = lambda: True + req.slave_reqs = [] + while req.cur_kv_len < 2176: + packs = prefill(result, [req], chunked) + if req.cur_kv_len < 2176: + assert packs[0].prefill_tokens == 512 + assert packs[0].output_len == 0 + post(result, [req], packs) + assert counters.totals() == (2176 - cached, 1) + post(result, [req], result._pre_post_handle([req], is_chuncked_mode=False)) + assert counters.totals() == (2176 - cached, 2) + + +def test_mixed_prefill_excludes_decode_tokens(monkeypatch, counters): + result = backend(monkeypatch, counters) + prefill_req = request(128, cached=128, output_len=2048, max_output=4096) + decode_req = request(128, cached=132, output_len=5) + decode_req.is_decode_req_mixed_in_prefill = True + reqs = [prefill_req, decode_req] + packs = prefill(result, reqs) + assert [pack.prefill_tokens for pack in packs] == [512, 0] + post(result, reqs, packs) + assert counters.totals() == (512, 1) diff --git a/unit_tests/utils/test_start_utils.py b/unit_tests/utils/test_start_utils.py index f606c6d9d0..f66f2d4b9a 100644 --- a/unit_tests/utils/test_start_utils.py +++ b/unit_tests/utils/test_start_utils.py @@ -603,7 +603,10 @@ def test_readiness_monitor_mutual_tls(monkeypatch, tmp_path, identity): capture_output=True, ) config = tmp_path / "hypercorn.toml" - config.write_text(f'certfile = "{tmp_path / "server.pem"}"\nkeyfile = "{tmp_path / "server.key"}"\n') + config.write_text( + f'certfile = "{tmp_path / "server.pem"}"\nkeyfile = "{tmp_path / "server.key"}"\n' + f'ca_certs = "{tmp_path / "client.pem"}"\nverify_mode = 2\n' + ) monkeypatch.delenv("LIGHTLLM_READINESS_CLIENT_CERT", raising=False) monkeypatch.delenv("LIGHTLLM_READINESS_CLIENT_KEY", raising=False) if identity == "separate": @@ -614,10 +617,10 @@ def test_readiness_monitor_mutual_tls(monkeypatch, tmp_path, identity): combined.write_bytes((tmp_path / "client.pem").read_bytes() + (tmp_path / "client.key").read_bytes()) monkeypatch.setenv("LIGHTLLM_READINESS_CLIENT_CERT", str(combined)) - server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - server_context.load_cert_chain(tmp_path / "server.pem", tmp_path / "server.key") - server_context.load_verify_locations(tmp_path / "client.pem") - server_context.verify_mode = ssl.CERT_REQUIRED + from hypercorn.__main__ import _load_config + + server_context = _load_config(str(config)).create_ssl_context() + assert server_context.verify_mode == ssl.CERT_REQUIRED session = MagicMock() session.__enter__.return_value = session @@ -648,6 +651,8 @@ def probe(url, timeout): manager = start_utils.SubmoduleManager() flag = Mock() + logger = Mock() + monkeypatch.setattr(start_utils, "logger", logger) monkeypatch.setattr(shared_arr, "SharedInt", lambda name: flag) monkeypatch.setattr(shm_port_args, "get_shm_port_args", lambda: SimpleNamespace(port=8000)) monkeypatch.setattr(requests, "Session", lambda: session) @@ -658,3 +663,8 @@ def probe(url, timeout): manager.start_readiness_monitor(FakeHttpServerProcess(), args) session.get.assert_called_once() assert [call.args for call in flag.set_value.call_args_list] == ([(0,)] if identity is None else [(0,), (1,)]) + if identity is None: + logger.warning.assert_called_once() + assert logger.warning.call_args.kwargs["extra"]["reason"] == "probe_tls_failed" + else: + logger.warning.assert_not_called()