Diuse/zhenxun_bot
1
0
Fork 0
mirror of https://github.com/zhenxun-org/zhenxun_bot.git synced 2025-12-14 21:52:56 +08:00
Code Issues Actions 7 Packages Projects Releases Wiki Activity

Bugfix/fix aliyun (#2036)

Browse Source 
* 🐛 修复数据库超时问题

* 🔧 移除帮助图片清理功能.

* ✨ 更新插件商店功能，允许在添加插件时指定源类型为 None。优化插件 ID 查找逻辑，增强代码可读性。新增 zhenxun/ui 模块导入。

* 🔧 优化数据访问和数据库上下文逻辑，移除不必要的全局变量和日志信息，调整日志级别为调试，提升代码可读性和性能。

* ✨ 增强插件商店功能，支持在下载文件时指定稀疏检出路径和目标目录。优化二进制文件处理逻辑，提升文件下载的准确性和效率。

* ✨ 增强阿里云和GitHub的文件管理功能，新增Git不可用异常处理，优化稀疏检出逻辑，提升代码可读性和稳定性。

* ✨ 增强插件下载功能，新增对下载结果的异常处理，确保在Git不可用时抛出相应异常信息。优化错误提示，提升用户体验。

* ✨ 增强插件商店功能，优化添加插件时的提示信息，明确区分插件模块和名称。新增 Windows 下删除只读文件的处理逻辑，提升插件管理的稳定性和用户体验。

* ✨ 优化文件内容获取逻辑，新增对非二进制文件的UTF-8解码处理，提升文件读取的稳定性和准确性。
This commit is contained in:
HibiKier 2025-08-29 14:57:08 +08:00 committed by GitHub
parent 7472cabd48
commit 1e2aa99207
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 313 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
zhenxun/builtin_plugins/plugin_store/__init__.py
View File

@ -104,7 +104,9 @@ async def _(session: EventSession, plugin_id: str, source: Match[str]):
if is_number(plugin_id):
await MessageUtils.build_message(f"正在添加插件 Id: {plugin_id}").send()
else:
await MessageUtils.build_message(f"正在添加插件 Module: {plugin_id}").send()
await MessageUtils.build_message(
f"正在添加插件 Module/名称: {plugin_id}"
).send()
source_str = source.result if source.available else None
if source_str and source_str not in ["ali", "git"]:
await MessageUtils.build_message(

17
zhenxun/builtin_plugins/plugin_store/data_source.py
View File

@ -14,7 +14,7 @@ from zhenxun.utils.image_utils import BuildImage, ImageTemplate, RowStyle
from zhenxun.utils.manager.virtual_env_package_manager import VirtualEnvPackageManager
from zhenxun.utils.repo_utils import RepoFileManager
from zhenxun.utils.repo_utils.models import RepoFileInfo, RepoType
from zhenxun.utils.utils import is_number
from zhenxun.utils.utils import is_number, win_on_rm_error
from .config import (
BASE_PATH,
@ -268,6 +268,7 @@ class StoreManager:
elif source == "git":
repo_type = RepoType.GITHUB
replace_module_path = module_path.replace(".", "/")
plugin_name = module_path.split(".")[-1]
if is_dir:
files = await RepoFileManager.list_directory_files(
github_url, replace_module_path, repo_type=repo_type
@ -275,11 +276,18 @@ class StoreManager:
else:
files = [RepoFileInfo(path=f"{replace_module_path}.py", is_dir=False)]
local_path = BASE_PATH / "plugins" if is_external else BASE_PATH
target_dir = BASE_PATH / "plugins" / plugin_name
files = [file for file in files if not file.is_dir]
download_files = [(file.path, local_path / file.path) for file in files]
await RepoFileManager.download_files(
github_url, download_files, repo_type=repo_type
result = await RepoFileManager.download_files(
github_url,
download_files,
repo_type=repo_type,
sparse_path=replace_module_path,
target_dir=target_dir,
)
if not result.success:
raise PluginStoreException(result.error_message)
requirement_paths = [
file
@ -344,7 +352,8 @@ class StoreManager:
return f"插件 {plugin_info.name} 不存在..."
logger.debug(f"尝试移除插件 {plugin_info.name} 文件: {path}", LOG_COMMAND)
if plugin_info.is_dir:
shutil.rmtree(path)
# 处理 Windows 下 .git 等目录内只读文件导致的 WinError 5
shutil.rmtree(path, onerror=win_on_rm_error)
else:
path.unlink()
await PluginInitManager.remove(f"zhenxun.{plugin_info.module_path}")

27
zhenxun/utils/repo_utils/aliyun_manager.py
View File

@ -11,6 +11,7 @@ from aiocache import cached
from zhenxun.services.log import logger
from zhenxun.utils.github_utils.models import AliyunFileInfo
from zhenxun.utils.repo_utils.utils import prepare_aliyun_url
from .base_manager import BaseRepoManager
from .config import LOG_COMMAND, RepoConfig
@ -445,32 +446,6 @@ class AliyunCodeupManager(BaseRepoManager):
返回:
RepoUpdateResult: 更新结果
"""
# 定义预处理函数，构建阿里云CodeUp的URL
def prepare_aliyun_url(repo_url: str) -> str:
import base64
repo_name = repo_url.split("/tree/")[0].split("/")[-1].replace(".git", "")
# 构建仓库URL
# 阿里云CodeUp的仓库URL格式通常为：
# https://codeup.aliyun.com/{organization_id}/{organization_name}/{repo_name}.git
url = f"https://codeup.aliyun.com/{self.config.aliyun_codeup.organization_id}/{self.config.aliyun_codeup.organization_name}/{repo_name}.git"
# 添加访问令牌 - 使用base64解码后的令牌
if self.config.aliyun_codeup.rdc_access_token_encrypted:
try:
# 解码RDC访问令牌
token = base64.b64decode(
self.config.aliyun_codeup.rdc_access_token_encrypted.encode()
).decode()
# 阿里云CodeUp使用oauth2:token的格式进行身份验证
url = url.replace("https://", f"https://oauth2:{token}@")
logger.debug(f"使用RDC令牌构建阿里云URL: {url.split('@')[0]}@***")
except Exception as e:
logger.error(f"解码RDC令牌失败: {e}")
return url
# 调用基类的update_via_git方法
return await super().update_via_git(
repo_url=repo_url,

7
zhenxun/utils/repo_utils/exceptions.py
View File

@ -66,3 +66,10 @@ class ConfigError(RepoManagerError):
def __init__(self, message: str):
super().__init__(f"配置错误: {message}")
class GitUnavailableError(RepoManagerError):
"""Git不可用异常"""
def __init__(self, message: str = "Git命令不可用"):
super().__init__(message)

112
zhenxun/utils/repo_utils/file_manager.py
View File

@ -2,6 +2,7 @@
仓库文件管理器，用于从GitHub和阿里云CodeUp获取指定文件内容
"""
import contextlib
from pathlib import Path
from typing import cast, overload
@ -12,10 +13,17 @@ from zhenxun.services.log import logger
from zhenxun.utils.github_utils import GithubUtils
from zhenxun.utils.github_utils.models import AliyunTreeType, GitHubStrategy, TreeType
from zhenxun.utils.http_utils import AsyncHttpx
from zhenxun.utils.utils import is_binary_file
from .config import LOG_COMMAND, RepoConfig
from .exceptions import FileNotFoundError, NetworkError, RepoManagerError
from .exceptions import (
FileNotFoundError,
GitUnavailableError,
NetworkError,
RepoManagerError,
)
from .models import FileDownloadResult, RepoFileInfo, RepoType
from .utils import prepare_aliyun_url, sparse_checkout_clone
class RepoFileManager:
@ -74,7 +82,9 @@ class RepoFileManager:
)
if response.status_code == 200:
logger.info(f"获取github文件内容成功: {f}", LOG_COMMAND)
text_content = response.content
# 确保使用UTF-8编码解析响应内容
if not is_binary_file(f):
try:
text_content = response.content.decode("utf-8")
except UnicodeDecodeError:
@ -262,15 +272,15 @@ class RepoFileManager:
if repo_type is None:
# 尝试GitHub，失败则尝试阿里云
try:
return await self._list_github_directory_files(
repo_url, directory_path, branch, recursive
return await self._list_aliyun_directory_files(
repo_name, directory_path, branch, recursive
)
except Exception as e:
logger.warning(
"获取GitHub目录文件失败，尝试阿里云", LOG_COMMAND, e=e
"获取阿里云目录文件失败，尝试GitHub", LOG_COMMAND, e=e
)
return await self._list_aliyun_directory_files(
repo_name, directory_path, branch, recursive
return await self._list_github_directory_files(
repo_url, directory_path, branch, recursive
)
if repo_type == RepoType.GITHUB:
return await self._list_github_directory_files(
@ -466,6 +476,8 @@ class RepoFileManager:
branch: str = "main",
repo_type: RepoType | None = None,
ignore_error: bool = False,
sparse_path: str | None = None,
target_dir: Path | None = None,
) -> FileDownloadResult:
"""
下载单个文件
@ -476,10 +488,19 @@ class RepoFileManager:
branch: 分支名称
repo_type: 仓库类型，如果为None则自动判断
ignore_error: 是否忽略错误
sparse_path: 稀疏检出路径
target_dir: 稀疏目标目录
返回:
FileDownloadResult: 下载结果
"""
# 参数一致性校验：sparse_path 与 target_dir 必须同时有值或同时为 None
if (sparse_path is None) ^ (target_dir is None):
raise RepoManagerError(
"参数错误: sparse_path 与 target_dir 必须同时提供或同时为 None"
)
# 确定仓库类型和所有者
repo_name = (
repo_url.split("/tree/")[0].split("/")[-1].replace(".git", "").strip()
@ -497,12 +518,43 @@ class RepoFileManager:
file_path=file_path,
version=branch,
)
if (
any(is_binary_file(file_name) for file_name in file_path_mapping)
and repo_type != RepoType.GITHUB
and sparse_path
and target_dir
):
return await self._handle_binary_with_sparse_checkout(
repo_url=repo_url,
branch=branch,
sparse_path=sparse_path,
target_dir=target_dir,
result=result,
)
else:
# 不包含二进制时
return await self._download_and_write_files(
repo_url=repo_url,
file_paths=[f[0] for f in file_path],
file_path_mapping=file_path_mapping,
branch=branch,
repo_type=repo_type,
ignore_error=ignore_error,
result=result,
)
async def _download_and_write_files(
self,
repo_url: str,
file_paths: list[str],
file_path_mapping: dict[str, Path],
branch: str,
repo_type: RepoType | None,
ignore_error: bool,
result: FileDownloadResult,
) -> FileDownloadResult:
try:
# 由于我们传入的是列表，所以这里一定返回列表
file_paths = [f[0] for f in file_path]
if len(file_paths) == 1:
# 如果只有一个文件，可能返回单个元组
file_contents_result = await self.get_file_content(
repo_url, file_paths[0], branch, repo_type, ignore_error
)
@ -513,7 +565,6 @@ class RepoFileManager:
else:
file_contents = cast(list[tuple[str, str]], file_contents_result)
else:
# 多个文件一定返回列表
file_contents = cast(
list[tuple[str, str]],
await self.get_file_content(
@ -524,7 +575,6 @@ class RepoFileManager:
for repo_file_path, content in file_contents:
local_path = file_path_mapping[repo_file_path]
local_path.parent.mkdir(parents=True, exist_ok=True)
# 使用二进制模式写入文件，避免编码问题
if isinstance(content, str):
content_bytes = content.encode("utf-8")
else:
@ -533,7 +583,6 @@ class RepoFileManager:
async with aiofiles.open(local_path, "wb") as f:
await f.write(content_bytes)
result.success = True
# 计算文件大小
result.file_size = sum(
len(content.encode("utf-8") if isinstance(content, str) else content)
for _, content in file_contents
@ -545,3 +594,42 @@ class RepoFileManager:
result.success = False
result.error_message = str(e)
return result
async def _handle_binary_with_sparse_checkout(
self,
repo_url: str,
branch: str,
sparse_path: str,
target_dir: Path,
result: FileDownloadResult,
) -> FileDownloadResult:
try:
await sparse_checkout_clone(
repo_url=prepare_aliyun_url(repo_url),
branch=branch,
sparse_path=sparse_path,
target_dir=target_dir,
)
total_size = 0
if target_dir.exists():
for f in target_dir.rglob("*"):
if f.is_file():
with contextlib.suppress(Exception):
total_size += f.stat().st_size
result.success = True
result.file_size = total_size
logger.info(f"sparse-checkout 克隆成功: {target_dir}")
return result
except GitUnavailableError as e:
logger.error(f"Git不可用: {e}")
result.success = False
result.error_message = (
"当前插件包含二进制文件，因ali限制需要使用git，"
"当前Git不可用，请尝试添加参数 -s git 或 安装 git"
)
return result
except Exception as e:
logger.error(f"sparse-checkout 克隆失败: {e}")
result.success = False
result.error_message = str(e)
return result

121
zhenxun/utils/repo_utils/utils.py
View File

@ -3,12 +3,15 @@
"""
import asyncio
import base64
from pathlib import Path
import re
import shutil
from zhenxun.services.log import logger
from .config import LOG_COMMAND
from .config import LOG_COMMAND, RepoConfig
from .exceptions import GitUnavailableError
async def check_git() -> bool:
@ -133,3 +136,119 @@ def filter_files(
result = [file for file in result if not re.match(regex_pattern, file)]
return result
async def sparse_checkout_clone(
repo_url: str,
branch: str,
sparse_path: str,
target_dir: Path,
) -> None:
"""
使用 git 稀疏检出克隆指定路径到目标目录（完全独立于主项目 git）。
关键保障:
- 在 target_dir 下检测/初始化 .git，所有 git 操作均以 cwd=target_dir 执行
- 强制拉取与工作区覆盖: fetch --force、checkout -B、reset --hard、clean -xdf
- 反复设置 sparse-checkout 路径，确保路径更新生效
"""
target_dir.mkdir(parents=True, exist_ok=True)
if not await check_git():
raise GitUnavailableError()
git_dir = target_dir / ".git"
if not git_dir.exists():
success, out, err = await run_git_command("init", target_dir)
if not success:
raise RuntimeError(f"git init 失败: {err or out}")
success, out, err = await run_git_command(
f"remote add origin {repo_url}", target_dir
)
if not success:
raise RuntimeError(f"添加远程失败: {err or out}")
else:
success, out, err = await run_git_command(
f"remote set-url origin {repo_url}", target_dir
)
if not success:
# 兜底尝试添加
await run_git_command(f"remote add origin {repo_url}", target_dir)
# 启用稀疏检出（使用 --no-cone 模式以获得更精确的控制）
await run_git_command("config core.sparseCheckout true", target_dir)
await run_git_command("sparse-checkout init --no-cone", target_dir)
# 设置需要检出的路径（每次都覆盖配置）
if not sparse_path:
raise RuntimeError("sparse-checkout 路径不能为空")
# 使用 --no-cone 模式，直接指定要检出的具体路径
# 例如：sparse_path="plugins/mahiro" -> 只检出 plugins/mahiro/ 下的内容
success, out, err = await run_git_command(
f"sparse-checkout set {sparse_path}/", target_dir
)
if not success:
raise RuntimeError(f"配置稀疏路径失败: {err or out}")
# 强制拉取并同步到远端
success, out, err = await run_git_command(
f"fetch --force --depth 1 origin {branch}", target_dir
)
if not success:
raise RuntimeError(f"fetch 失败: {err or out}")
# 使用远端强制更新本地分支并覆盖工作区
success, out, err = await run_git_command(
f"checkout -B {branch} origin/{branch}", target_dir
)
if not success:
# 回退方案
success2, out2, err2 = await run_git_command(f"checkout {branch}", target_dir)
if not success2:
raise RuntimeError(f"checkout 失败: {(err or out) or (err2 or out2)}")
# 强制对齐工作区
await run_git_command(f"reset --hard origin/{branch}", target_dir)
await run_git_command("clean -xdf", target_dir)
dir_path = target_dir / Path(sparse_path)
for f in dir_path.iterdir():
shutil.move(f, target_dir / f.name)
dir_name = sparse_path.split("/")[0]
rm_path = target_dir / dir_name
if rm_path.exists():
shutil.rmtree(rm_path)
def prepare_aliyun_url(repo_url: str) -> str:
"""解析阿里云CodeUp的仓库URL
参数:
repo_url: 仓库URL
返回:
str: 解析后的仓库URL
"""
config = RepoConfig.get_instance()
repo_name = repo_url.split("/tree/")[0].split("/")[-1].replace(".git", "")
# 构建仓库URL
# 阿里云CodeUp的仓库URL格式通常为：
# https://codeup.aliyun.com/{organization_id}/{organization_name}/{repo_name}.git
url = f"https://codeup.aliyun.com/{config.aliyun_codeup.organization_id}/{config.aliyun_codeup.organization_name}/{repo_name}.git"
# 添加访问令牌 - 使用base64解码后的令牌
if config.aliyun_codeup.rdc_access_token_encrypted:
try:
# 解码RDC访问令牌
token = base64.b64decode(
config.aliyun_codeup.rdc_access_token_encrypted.encode()
).decode()
# 阿里云CodeUp使用oauth2:token的格式进行身份验证
url = url.replace("https://", f"https://oauth2:{token}@")
logger.debug(f"使用RDC令牌构建阿里云URL: {url.split('@')[0]}@***")
except Exception as e:
logger.error(f"解码RDC令牌失败: {e}")
return url

75
zhenxun/utils/utils.py
View File

@ -1,9 +1,12 @@
from collections.abc import Callable
from dataclasses import dataclass
from datetime import datetime
import os
from pathlib import Path
import stat
import time
from typing import ClassVar
from types import TracebackType
from typing import Any, ClassVar
import httpx
from nonebot_plugin_uninfo import Uninfo
@ -65,22 +68,39 @@ class ResourceDirManager:
def is_binary_file(file_path: str) -> bool:
"""判断是否为二进制文件"""
binary_extensions = {
".jpg",
".jpeg",
".png",
".gif",
".bmp",
".ico",
".pdf",
".zip",
".rar",
".7z",
".exe",
".dll",
}
return any(file_path.lower().endswith(ext) for ext in binary_extensions)
"""判断是否为二进制文件
参数:
file_path: 文件路径
返回:
bool: 是否为二进制文件
"""
# fmt: off
# 精简但包含图片和字体的二进制文件扩展名集合
BINARY_EXTENSIONS = frozenset({
# 图片文件
"jpg", "jpeg", "png", "gif", "bmp", "ico", "webp", "tiff", "tif", "svg",
# 字体文件
"ttf", "otf", "woff", "woff2", "eot",
# 压缩文件
"zip", "rar", "7z", "tar", "gz", "bz2", "xz",
# 可执行文件和库
"exe", "dll", "so", "dylib",
# 文档文件
"pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx",
# 多媒体文件
"mp3", "mp4", "avi", "mov", "wmv", "flv",
# 其他常见二进制文件
"bin", "dat", "db", "class", "pyc"
})
# 使用os.path.splitext高效提取扩展名
_, ext = os.path.splitext(file_path)
# 去除点号并转换为小写
ext_clean = ext.lstrip(".").lower()
return ext_clean in BINARY_EXTENSIONS
def cn2py(word: str) -> str:
@ -224,3 +244,24 @@ def is_number(text: str) -> bool:
return True
except ValueError:
return False
def win_on_rm_error(
func: Callable[[str], Any],
path: str,
_exc_info: tuple[type[BaseException], BaseException, TracebackType],
) -> None:
"""Windows下删除只读文件/目录时的回调。
去除只读属性后重试删除，避免 WinError 5。
"""
try:
os.chmod(path, stat.S_IWRITE)
except Exception:
# 即使去除权限失败也继续尝试
pass
try:
func(path)
except Exception:
# 仍失败则记录调试日志并忽略，交由上层继续处理
logger.debug(f"删除失败重试仍失败: {path}")