mirror of
https://github.com/zhenxun-org/zhenxun_bot.git
synced 2025-12-14 21:52:56 +08:00
c667fc215e
* ✨ feat(llm): 增强LLM服务，支持图片生成、响应验证与OpenRouter集成 - 【新功能】统一图片生成与编辑API `create_image`，支持文生图、图生图及多图输入 - 【新功能】引入LLM响应验证机制，通过 `validation_policy` 和 `response_validator` 确保响应内容符合预期，例如强制返回图片 - 【新功能】适配OpenRouter API，扩展LLM服务提供商支持，并添加OpenRouter特定请求头 - 【重构】将日志净化逻辑重构至 `log_sanitizer` 模块，提供统一的净化入口，并应用于NoneBot消息、LLM请求/响应日志 - 【修复】优化Gemini适配器，正确解析图片生成响应中的Base64图片数据，并更新模型能力注册表 * ✨ feat(image): 优化图片生成响应并返回完整LLMResponse * ✨ feat(llm): 为 OpenAI 兼容请求体添加日志净化 * 🐛 fix(ui): 截断UI调试HTML日志中的长base64图片数据 --------- Co-authored-by: webjoin111 <455457521@qq.com>
634 lines
22 KiB
Python
634 lines
22 KiB
Python
"""
|
||
LLM 核心基础设施模块
|
||
|
||
包含执行 LLM 请求所需的底层组件，如 HTTP 客户端、API Key 存储和智能重试逻辑。
|
||
"""
|
||
|
||
import asyncio
|
||
from dataclasses import asdict, dataclass
|
||
from enum import IntEnum
|
||
import json
|
||
import os
|
||
import time
|
||
from typing import Any
|
||
|
||
import aiofiles
|
||
import httpx
|
||
import nonebot
|
||
from pydantic import BaseModel
|
||
|
||
from zhenxun.configs.path_config import DATA_PATH
|
||
from zhenxun.services.log import logger
|
||
from zhenxun.utils.user_agent import get_user_agent
|
||
|
||
from .types import ProviderConfig
|
||
from .types.exceptions import LLMErrorCode, LLMException
|
||
|
||
driver = nonebot.get_driver()
|
||
|
||
|
||
class HttpClientConfig(BaseModel):
|
||
"""HTTP客户端配置"""
|
||
|
||
timeout: int = 180
|
||
max_connections: int = 100
|
||
max_keepalive_connections: int = 20
|
||
proxy: str | None = None
|
||
|
||
|
||
class LLMHttpClient:
|
||
"""LLM服务专用HTTP客户端"""
|
||
|
||
def __init__(self, config: HttpClientConfig | None = None):
|
||
self.config = config or HttpClientConfig()
|
||
self._client: httpx.AsyncClient | None = None
|
||
self._active_requests = 0
|
||
self._lock = asyncio.Lock()
|
||
|
||
async def _ensure_client_initialized(self) -> httpx.AsyncClient:
|
||
if self._client is None or self._client.is_closed:
|
||
async with self._lock:
|
||
if self._client is None or self._client.is_closed:
|
||
logger.debug(
|
||
f"LLMHttpClient: 正在初始化新的 httpx.AsyncClient "
|
||
f"配置: {self.config}"
|
||
)
|
||
headers = get_user_agent()
|
||
limits = httpx.Limits(
|
||
max_connections=self.config.max_connections,
|
||
max_keepalive_connections=self.config.max_keepalive_connections,
|
||
)
|
||
timeout = httpx.Timeout(self.config.timeout)
|
||
|
||
client_kwargs = {}
|
||
if self.config.proxy:
|
||
try:
|
||
version_parts = httpx.__version__.split(".")
|
||
major = int(
|
||
"".join(c for c in version_parts[0] if c.isdigit())
|
||
)
|
||
minor = (
|
||
int("".join(c for c in version_parts[1] if c.isdigit()))
|
||
if len(version_parts) > 1
|
||
else 0
|
||
)
|
||
if (major, minor) >= (0, 28):
|
||
client_kwargs["proxy"] = self.config.proxy
|
||
else:
|
||
client_kwargs["proxies"] = self.config.proxy
|
||
except (ValueError, IndexError):
|
||
client_kwargs["proxies"] = self.config.proxy
|
||
logger.warning(
|
||
f"无法解析 httpx 版本 '{httpx.__version__}'，"
|
||
"LLM模块将默认使用旧版 'proxies' 参数语法。"
|
||
)
|
||
|
||
self._client = httpx.AsyncClient(
|
||
headers=headers,
|
||
limits=limits,
|
||
timeout=timeout,
|
||
follow_redirects=True,
|
||
**client_kwargs,
|
||
)
|
||
if self._client is None:
|
||
raise LLMException(
|
||
"HTTP 客户端初始化失败。", LLMErrorCode.CONFIGURATION_ERROR
|
||
)
|
||
return self._client
|
||
|
||
async def post(self, url: str, **kwargs: Any) -> httpx.Response:
|
||
client = await self._ensure_client_initialized()
|
||
async with self._lock:
|
||
self._active_requests += 1
|
||
try:
|
||
return await client.post(url, **kwargs)
|
||
finally:
|
||
async with self._lock:
|
||
self._active_requests -= 1
|
||
|
||
async def close(self):
|
||
async with self._lock:
|
||
if self._client and not self._client.is_closed:
|
||
logger.debug(
|
||
f"LLMHttpClient: 正在关闭，配置: {self.config}. "
|
||
f"活跃请求数: {self._active_requests}"
|
||
)
|
||
if self._active_requests > 0:
|
||
logger.warning(
|
||
f"LLMHttpClient: 关闭时仍有 {self._active_requests} "
|
||
f"个请求处于活跃状态。"
|
||
)
|
||
await self._client.aclose()
|
||
self._client = None
|
||
logger.debug(f"配置为 {self.config} 的 LLMHttpClient 已完全关闭。")
|
||
|
||
@property
|
||
def is_closed(self) -> bool:
|
||
return self._client is None or self._client.is_closed
|
||
|
||
|
||
class LLMHttpClientManager:
|
||
"""管理 LLMHttpClient 实例的工厂和池"""
|
||
|
||
def __init__(self):
|
||
self._clients: dict[tuple[int, str | None], LLMHttpClient] = {}
|
||
self._lock = asyncio.Lock()
|
||
|
||
def _get_client_key(
|
||
self, provider_config: ProviderConfig
|
||
) -> tuple[int, str | None]:
|
||
return (provider_config.timeout, provider_config.proxy)
|
||
|
||
async def get_client(self, provider_config: ProviderConfig) -> LLMHttpClient:
|
||
key = self._get_client_key(provider_config)
|
||
async with self._lock:
|
||
client = self._clients.get(key)
|
||
if client and not client.is_closed:
|
||
logger.debug(
|
||
f"LLMHttpClientManager: 复用现有的 LLMHttpClient 密钥: {key}"
|
||
)
|
||
return client
|
||
|
||
if client and client.is_closed:
|
||
logger.debug(
|
||
f"LLMHttpClientManager: 发现密钥 {key} 对应的客户端已关闭。"
|
||
f"正在创建新的客户端。"
|
||
)
|
||
|
||
logger.debug(f"LLMHttpClientManager: 为密钥 {key} 创建新的 LLMHttpClient")
|
||
http_client_config = HttpClientConfig(
|
||
timeout=provider_config.timeout, proxy=provider_config.proxy
|
||
)
|
||
new_client = LLMHttpClient(config=http_client_config)
|
||
self._clients[key] = new_client
|
||
return new_client
|
||
|
||
async def shutdown(self):
|
||
async with self._lock:
|
||
logger.info(
|
||
f"LLMHttpClientManager: 正在关闭。关闭 {len(self._clients)} 个客户端。"
|
||
)
|
||
close_tasks = [
|
||
client.close()
|
||
for client in self._clients.values()
|
||
if client and not client.is_closed
|
||
]
|
||
if close_tasks:
|
||
await asyncio.gather(*close_tasks, return_exceptions=True)
|
||
self._clients.clear()
|
||
logger.info("LLMHttpClientManager: 关闭完成。")
|
||
|
||
|
||
http_client_manager = LLMHttpClientManager()
|
||
|
||
|
||
async def create_llm_http_client(
|
||
timeout: int = 180,
|
||
proxy: str | None = None,
|
||
) -> LLMHttpClient:
|
||
"""
|
||
创建LLM HTTP客户端
|
||
|
||
参数:
|
||
timeout: 超时时间（秒）。
|
||
proxy: 代理服务器地址。
|
||
|
||
返回:
|
||
LLMHttpClient: HTTP客户端实例。
|
||
"""
|
||
config = HttpClientConfig(timeout=timeout, proxy=proxy)
|
||
return LLMHttpClient(config)
|
||
|
||
|
||
class KeyStatus(IntEnum):
|
||
"""用于排序和展示的密钥状态枚举"""
|
||
|
||
DISABLED = 0
|
||
ERROR = 1
|
||
COOLDOWN = 2
|
||
WARNING = 3
|
||
HEALTHY = 4
|
||
UNUSED = 5
|
||
|
||
|
||
@dataclass
|
||
class KeyStats:
|
||
"""单个API Key的详细状态和统计信息"""
|
||
|
||
cooldown_until: float = 0.0
|
||
success_count: int = 0
|
||
failure_count: int = 0
|
||
total_latency: float = 0.0
|
||
last_error_info: str | None = None
|
||
|
||
@property
|
||
def is_available(self) -> bool:
|
||
"""检查Key当前是否可用"""
|
||
return time.time() >= self.cooldown_until
|
||
|
||
@property
|
||
def avg_latency(self) -> float:
|
||
"""计算平均延迟"""
|
||
return (
|
||
self.total_latency / self.success_count if self.success_count > 0 else 0.0
|
||
)
|
||
|
||
@property
|
||
def success_rate(self) -> float:
|
||
"""计算成功率"""
|
||
total = self.success_count + self.failure_count
|
||
return self.success_count / total * 100 if total > 0 else 100.0
|
||
|
||
@property
|
||
def status(self) -> KeyStatus:
|
||
"""根据当前统计数据动态计算状态"""
|
||
now = time.time()
|
||
cooldown_left = max(0, self.cooldown_until - now)
|
||
|
||
if cooldown_left > 31536000 - 60:
|
||
return KeyStatus.DISABLED
|
||
if cooldown_left > 0:
|
||
return KeyStatus.COOLDOWN
|
||
|
||
total_calls = self.success_count + self.failure_count
|
||
if total_calls == 0:
|
||
return KeyStatus.UNUSED
|
||
|
||
if self.success_rate < 80:
|
||
return KeyStatus.ERROR
|
||
|
||
if total_calls >= 5 and self.avg_latency > 15000:
|
||
return KeyStatus.WARNING
|
||
|
||
return KeyStatus.HEALTHY
|
||
|
||
@property
|
||
def suggested_action(self) -> str:
|
||
"""根据状态给出建议操作"""
|
||
status_actions = {
|
||
KeyStatus.DISABLED: "更换Key",
|
||
KeyStatus.ERROR: "检查网络/重置",
|
||
KeyStatus.COOLDOWN: "等待/重置",
|
||
KeyStatus.WARNING: "观察",
|
||
KeyStatus.HEALTHY: "-",
|
||
KeyStatus.UNUSED: "-",
|
||
}
|
||
return status_actions.get(self.status, "未知")
|
||
|
||
|
||
class RetryConfig:
|
||
"""重试配置"""
|
||
|
||
def __init__(
|
||
self,
|
||
max_retries: int = 3,
|
||
retry_delay: float = 1.0,
|
||
exponential_backoff: bool = True,
|
||
key_rotation: bool = True,
|
||
):
|
||
self.max_retries = max_retries
|
||
self.retry_delay = retry_delay
|
||
self.exponential_backoff = exponential_backoff
|
||
self.key_rotation = key_rotation
|
||
|
||
|
||
async def with_smart_retry(
|
||
func,
|
||
*args,
|
||
retry_config: RetryConfig | None = None,
|
||
key_store: "KeyStatusStore | None" = None,
|
||
provider_name: str | None = None,
|
||
**kwargs: Any,
|
||
) -> Any:
|
||
"""
|
||
智能重试装饰器 - 支持Key轮询和错误分类
|
||
|
||
参数:
|
||
func: 要重试的异步函数。
|
||
*args: 传递给函数的位置参数。
|
||
retry_config: 重试配置。
|
||
key_store: API密钥状态存储。
|
||
provider_name: 提供商名称。
|
||
**kwargs: 传递给函数的关键字参数。
|
||
|
||
返回:
|
||
Any: 函数执行结果。
|
||
"""
|
||
config = retry_config or RetryConfig()
|
||
last_exception: Exception | None = None
|
||
failed_keys: set[str] = set()
|
||
|
||
model_instance = next((arg for arg in args if hasattr(arg, "api_keys")), None)
|
||
all_provider_keys = model_instance.api_keys if model_instance else []
|
||
|
||
for attempt in range(config.max_retries + 1):
|
||
try:
|
||
if config.key_rotation and "failed_keys" in func.__code__.co_varnames:
|
||
kwargs["failed_keys"] = failed_keys
|
||
|
||
start_time = time.monotonic()
|
||
result = await func(*args, **kwargs)
|
||
latency = (time.monotonic() - start_time) * 1000
|
||
|
||
if key_store and isinstance(result, tuple) and len(result) == 2:
|
||
_, api_key_used = result
|
||
if api_key_used:
|
||
await key_store.record_success(api_key_used, latency)
|
||
return result
|
||
else:
|
||
return result
|
||
|
||
except LLMException as e:
|
||
last_exception = e
|
||
api_key_in_use = e.details.get("api_key")
|
||
|
||
if api_key_in_use:
|
||
failed_keys.add(api_key_in_use)
|
||
if key_store and provider_name and len(all_provider_keys) > 1:
|
||
status_code = e.details.get("status_code")
|
||
error_message = f"({e.code.name}) {e.message}"
|
||
await key_store.record_failure(
|
||
api_key_in_use, status_code, error_message
|
||
)
|
||
|
||
should_retry = _should_retry_llm_error(e, attempt, config.max_retries)
|
||
if not should_retry:
|
||
logger.error(f"不可重试的错误，停止重试: {e}")
|
||
raise
|
||
|
||
if attempt < config.max_retries:
|
||
wait_time = config.retry_delay
|
||
if config.exponential_backoff:
|
||
wait_time *= 2**attempt
|
||
logger.warning(
|
||
f"请求失败，{wait_time:.2f}秒后重试 (第{attempt + 1}次): {e}"
|
||
)
|
||
await asyncio.sleep(wait_time)
|
||
else:
|
||
logger.error(f"重试{config.max_retries}次后仍然失败: {e}")
|
||
|
||
except Exception as e:
|
||
last_exception = e
|
||
logger.error(f"非LLM异常，停止重试: {e}")
|
||
raise LLMException(
|
||
f"操作失败: {e}",
|
||
code=LLMErrorCode.GENERATION_FAILED,
|
||
cause=e,
|
||
)
|
||
|
||
if last_exception:
|
||
raise last_exception
|
||
else:
|
||
raise RuntimeError("重试函数未能正常执行且未捕获到异常")
|
||
|
||
|
||
def _should_retry_llm_error(
|
||
error: LLMException, attempt: int, max_retries: int
|
||
) -> bool:
|
||
"""判断LLM错误是否应该重试"""
|
||
non_retryable_errors = {
|
||
LLMErrorCode.MODEL_NOT_FOUND,
|
||
LLMErrorCode.CONTEXT_LENGTH_EXCEEDED,
|
||
LLMErrorCode.USER_LOCATION_NOT_SUPPORTED,
|
||
LLMErrorCode.CONFIGURATION_ERROR,
|
||
}
|
||
|
||
if error.code in non_retryable_errors:
|
||
return False
|
||
|
||
retryable_errors = {
|
||
LLMErrorCode.API_REQUEST_FAILED,
|
||
LLMErrorCode.API_TIMEOUT,
|
||
LLMErrorCode.API_RATE_LIMITED,
|
||
LLMErrorCode.API_RESPONSE_INVALID,
|
||
LLMErrorCode.RESPONSE_PARSE_ERROR,
|
||
LLMErrorCode.GENERATION_FAILED,
|
||
LLMErrorCode.CONTENT_FILTERED,
|
||
LLMErrorCode.API_KEY_INVALID,
|
||
LLMErrorCode.API_QUOTA_EXCEEDED,
|
||
}
|
||
|
||
if error.code in retryable_errors:
|
||
if error.code == LLMErrorCode.API_QUOTA_EXCEEDED:
|
||
return attempt < min(2, max_retries)
|
||
elif error.code == LLMErrorCode.CONTENT_FILTERED:
|
||
return attempt < min(1, max_retries)
|
||
return True
|
||
|
||
return False
|
||
|
||
|
||
class KeyStatusStore:
|
||
"""API Key 状态管理存储 - 支持持久化"""
|
||
|
||
def __init__(self):
|
||
self._key_stats: dict[str, KeyStats] = {}
|
||
self._provider_key_index: dict[str, int] = {}
|
||
self._lock = asyncio.Lock()
|
||
self._file_path = DATA_PATH / "llm" / "key_status.json"
|
||
|
||
async def initialize(self):
|
||
"""从文件异步加载密钥状态，在应用启动时调用"""
|
||
async with self._lock:
|
||
if not self._file_path.exists():
|
||
logger.info("未找到密钥状态文件，将使用内存状态启动。")
|
||
return
|
||
|
||
try:
|
||
logger.info(f"正在从 {self._file_path} 加载密钥状态...")
|
||
async with aiofiles.open(self._file_path, encoding="utf-8") as f:
|
||
content = await f.read()
|
||
if not content:
|
||
logger.warning("密钥状态文件为空。")
|
||
return
|
||
data = json.loads(content)
|
||
|
||
for key, stats_dict in data.items():
|
||
self._key_stats[key] = KeyStats(**stats_dict)
|
||
|
||
logger.info(f"成功加载 {len(self._key_stats)} 个密钥的状态。")
|
||
|
||
except json.JSONDecodeError:
|
||
logger.error(f"密钥状态文件 {self._file_path} 格式错误，无法解析。")
|
||
except Exception as e:
|
||
logger.error(f"加载密钥状态文件时发生错误: {e}", e=e)
|
||
|
||
async def _save_to_file_internal(self):
|
||
"""
|
||
[内部方法] 将当前密钥状态安全地写入JSON文件。
|
||
假定调用方已持有锁。
|
||
"""
|
||
data_to_save = {key: asdict(stats) for key, stats in self._key_stats.items()}
|
||
|
||
try:
|
||
self._file_path.parent.mkdir(parents=True, exist_ok=True)
|
||
temp_path = self._file_path.with_suffix(".json.tmp")
|
||
|
||
async with aiofiles.open(temp_path, "w", encoding="utf-8") as f:
|
||
await f.write(json.dumps(data_to_save, ensure_ascii=False, indent=2))
|
||
|
||
if self._file_path.exists():
|
||
self._file_path.unlink()
|
||
os.rename(temp_path, self._file_path)
|
||
logger.debug("密钥状态已成功持久化到文件。")
|
||
except Exception as e:
|
||
logger.error(f"保存密钥状态到文件失败: {e}", e=e)
|
||
|
||
async def shutdown(self):
|
||
"""在应用关闭时安全地保存状态"""
|
||
async with self._lock:
|
||
await self._save_to_file_internal()
|
||
logger.info("KeyStatusStore 已在关闭前保存状态。")
|
||
|
||
async def get_next_available_key(
|
||
self,
|
||
provider_name: str,
|
||
api_keys: list[str],
|
||
exclude_keys: set[str] | None = None,
|
||
) -> str | None:
|
||
"""
|
||
获取下一个可用的API密钥（轮询策略）
|
||
|
||
参数:
|
||
provider_name: 提供商名称。
|
||
api_keys: API密钥列表。
|
||
exclude_keys: 要排除的密钥集合。
|
||
|
||
返回:
|
||
str | None: 可用的API密钥，如果没有可用密钥则返回None。
|
||
"""
|
||
if not api_keys:
|
||
return None
|
||
|
||
exclude_keys = exclude_keys or set()
|
||
|
||
async with self._lock:
|
||
for key in api_keys:
|
||
if key not in self._key_stats:
|
||
self._key_stats[key] = KeyStats()
|
||
|
||
available_keys = [
|
||
key
|
||
for key in api_keys
|
||
if key not in exclude_keys and self._key_stats[key].is_available
|
||
]
|
||
|
||
if not available_keys:
|
||
return api_keys[0]
|
||
|
||
current_index = self._provider_key_index.get(provider_name, 0)
|
||
selected_key = available_keys[current_index % len(available_keys)]
|
||
self._provider_key_index[provider_name] = current_index + 1
|
||
|
||
total_usage = (
|
||
self._key_stats[selected_key].success_count
|
||
+ self._key_stats[selected_key].failure_count
|
||
)
|
||
logger.debug(
|
||
f"轮询选择API密钥: {self._get_key_id(selected_key)} "
|
||
f"(使用次数: {total_usage})"
|
||
)
|
||
return selected_key
|
||
|
||
async def record_success(self, api_key: str, latency: float):
|
||
"""记录成功使用，并持久化"""
|
||
async with self._lock:
|
||
stats = self._key_stats.setdefault(api_key, KeyStats())
|
||
stats.cooldown_until = 0.0
|
||
stats.success_count += 1
|
||
stats.total_latency += latency
|
||
stats.last_error_info = None
|
||
await self._save_to_file_internal()
|
||
logger.debug(
|
||
f"记录API密钥成功使用: {self._get_key_id(api_key)}, 延迟: {latency:.2f}ms"
|
||
)
|
||
|
||
async def record_failure(
|
||
self, api_key: str, status_code: int | None, error_message: str
|
||
):
|
||
"""
|
||
记录失败使用，并设置冷却时间
|
||
|
||
参数:
|
||
api_key: API密钥。
|
||
status_code: HTTP状态码。
|
||
error_message: 错误信息。
|
||
"""
|
||
key_id = self._get_key_id(api_key)
|
||
now = time.time()
|
||
cooldown_duration = 300
|
||
|
||
if status_code in [401, 403, 404]:
|
||
cooldown_duration = 31536000
|
||
log_level = "error"
|
||
log_message = f"API密钥认证/权限/路径错误，将永久禁用: {key_id}"
|
||
elif status_code == 429:
|
||
cooldown_duration = 60
|
||
log_level = "warning"
|
||
log_message = f"API密钥被限流，冷却60秒: {key_id}"
|
||
else:
|
||
log_level = "warning"
|
||
log_message = f"API密钥遇到临时性错误，冷却{cooldown_duration}秒: {key_id}"
|
||
|
||
async with self._lock:
|
||
stats = self._key_stats.setdefault(api_key, KeyStats())
|
||
stats.cooldown_until = now + cooldown_duration
|
||
stats.failure_count += 1
|
||
stats.last_error_info = error_message[:256]
|
||
await self._save_to_file_internal()
|
||
|
||
getattr(logger, log_level)(log_message)
|
||
|
||
async def reset_key_status(self, api_key: str):
|
||
"""重置密钥状态，并持久化"""
|
||
async with self._lock:
|
||
stats = self._key_stats.setdefault(api_key, KeyStats())
|
||
stats.cooldown_until = 0.0
|
||
stats.last_error_info = None
|
||
await self._save_to_file_internal()
|
||
logger.info(f"重置API密钥状态: {self._get_key_id(api_key)}")
|
||
|
||
async def get_key_stats(self, api_keys: list[str]) -> dict[str, dict]:
|
||
"""
|
||
获取密钥使用统计，并计算出用于展示的派生数据。
|
||
|
||
参数:
|
||
api_keys: API密钥列表。
|
||
|
||
返回:
|
||
dict[str, dict]: 包含丰富状态和统计信息的密钥字典。
|
||
"""
|
||
stats_dict = {}
|
||
now = time.time()
|
||
async with self._lock:
|
||
for key in api_keys:
|
||
key_id = self._get_key_id(key)
|
||
stats = self._key_stats.get(key, KeyStats())
|
||
|
||
stats_dict[key_id] = {
|
||
"status_enum": stats.status,
|
||
"cooldown_seconds_left": max(0, stats.cooldown_until - now),
|
||
"total_calls": stats.success_count + stats.failure_count,
|
||
"success_count": stats.success_count,
|
||
"failure_count": stats.failure_count,
|
||
"success_rate": stats.success_rate,
|
||
"avg_latency": stats.avg_latency,
|
||
"last_error": stats.last_error_info,
|
||
"suggested_action": stats.suggested_action,
|
||
}
|
||
return stats_dict
|
||
|
||
def _get_key_id(self, api_key: str) -> str:
|
||
"""获取API密钥的标识符（用于日志）"""
|
||
if len(api_key) <= 8:
|
||
return api_key
|
||
return f"{api_key[:4]}...{api_key[-4:]}"
|
||
|
||
|
||
key_store = KeyStatusStore()
|
||
|
||
|
||
@driver.on_shutdown
|
||
async def _shutdown_key_store():
|
||
await key_store.shutdown()
|