From 1e2aa99207ce20e299361fde05edd4649aa9ec85 Mon Sep 17 00:00:00 2001 From: HibiKier <45528451+HibiKier@users.noreply.github.com> Date: Fri, 29 Aug 2025 14:57:08 +0800 Subject: [PATCH] Bugfix/fix aliyun (#2036) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * :bug: 修复数据库超时问题 * 🔧 移除帮助图片清理功能. * ✨ 更新插件商店功能，允许在添加插件时指定源类型为 None。优化插件 ID 查找逻辑，增强代码可读性。新增 zhenxun/ui 模块导入。 * 🔧 优化数据访问和数据库上下文逻辑，移除不必要的全局变量和日志信息，调整日志级别为调试，提升代码可读性和性能。 * ✨ 增强插件商店功能，支持在下载文件时指定稀疏检出路径和目标目录。优化二进制文件处理逻辑，提升文件下载的准确性和效率。 * ✨ 增强阿里云和GitHub的文件管理功能，新增Git不可用异常处理，优化稀疏检出逻辑，提升代码可读性和稳定性。 * ✨ 增强插件下载功能，新增对下载结果的异常处理，确保在Git不可用时抛出相应异常信息。优化错误提示，提升用户体验。 * ✨ 增强插件商店功能，优化添加插件时的提示信息，明确区分插件模块和名称。新增 Windows 下删除只读文件的处理逻辑，提升插件管理的稳定性和用户体验。 * ✨ 优化文件内容获取逻辑，新增对非二进制文件的UTF-8解码处理，提升文件读取的稳定性和准确性。 --- .../builtin_plugins/plugin_store/__init__.py | 4 +- .../plugin_store/data_source.py | 17 ++- zhenxun/utils/repo_utils/aliyun_manager.py | 27 +--- zhenxun/utils/repo_utils/exceptions.py | 7 + zhenxun/utils/repo_utils/file_manager.py | 134 +++++++++++++++--- zhenxun/utils/repo_utils/utils.py | 121 +++++++++++++++- zhenxun/utils/utils.py | 75 +++++++--- 7 files changed, 313 insertions(+), 72 deletions(-) diff --git a/zhenxun/builtin_plugins/plugin_store/__init__.py b/zhenxun/builtin_plugins/plugin_store/__init__.py index f7998310..2f8d756b 100644 --- a/zhenxun/builtin_plugins/plugin_store/__init__.py +++ b/zhenxun/builtin_plugins/plugin_store/__init__.py @@ -104,7 +104,9 @@ async def _(session: EventSession, plugin_id: str, source: Match[str]): if is_number(plugin_id): await MessageUtils.build_message(f"正在添加插件 Id: {plugin_id}").send() else: - await MessageUtils.build_message(f"正在添加插件 Module: {plugin_id}").send() + await MessageUtils.build_message( + f"正在添加插件 Module/名称: {plugin_id}" + ).send() source_str = source.result if source.available else None if source_str and source_str not in ["ali", "git"]: await MessageUtils.build_message( diff --git a/zhenxun/builtin_plugins/plugin_store/data_source.py b/zhenxun/builtin_plugins/plugin_store/data_source.py index f465411d..7b8f4bc6 100644 --- a/zhenxun/builtin_plugins/plugin_store/data_source.py +++ b/zhenxun/builtin_plugins/plugin_store/data_source.py @@ -14,7 +14,7 @@ from zhenxun.utils.image_utils import BuildImage, ImageTemplate, RowStyle from zhenxun.utils.manager.virtual_env_package_manager import VirtualEnvPackageManager from zhenxun.utils.repo_utils import RepoFileManager from zhenxun.utils.repo_utils.models import RepoFileInfo, RepoType -from zhenxun.utils.utils import is_number +from zhenxun.utils.utils import is_number, win_on_rm_error from .config import ( BASE_PATH, @@ -268,6 +268,7 @@ class StoreManager: elif source == "git": repo_type = RepoType.GITHUB replace_module_path = module_path.replace(".", "/") + plugin_name = module_path.split(".")[-1] if is_dir: files = await RepoFileManager.list_directory_files( github_url, replace_module_path, repo_type=repo_type @@ -275,11 +276,18 @@ class StoreManager: else: files = [RepoFileInfo(path=f"{replace_module_path}.py", is_dir=False)] local_path = BASE_PATH / "plugins" if is_external else BASE_PATH + target_dir = BASE_PATH / "plugins" / plugin_name files = [file for file in files if not file.is_dir] download_files = [(file.path, local_path / file.path) for file in files] - await RepoFileManager.download_files( - github_url, download_files, repo_type=repo_type + result = await RepoFileManager.download_files( + github_url, + download_files, + repo_type=repo_type, + sparse_path=replace_module_path, + target_dir=target_dir, ) + if not result.success: + raise PluginStoreException(result.error_message) requirement_paths = [ file @@ -344,7 +352,8 @@ class StoreManager: return f"插件 {plugin_info.name} 不存在..." logger.debug(f"尝试移除插件 {plugin_info.name} 文件: {path}", LOG_COMMAND) if plugin_info.is_dir: - shutil.rmtree(path) + # 处理 Windows 下 .git 等目录内只读文件导致的 WinError 5 + shutil.rmtree(path, onerror=win_on_rm_error) else: path.unlink() await PluginInitManager.remove(f"zhenxun.{plugin_info.module_path}") diff --git a/zhenxun/utils/repo_utils/aliyun_manager.py b/zhenxun/utils/repo_utils/aliyun_manager.py index 863a5620..302a24de 100644 --- a/zhenxun/utils/repo_utils/aliyun_manager.py +++ b/zhenxun/utils/repo_utils/aliyun_manager.py @@ -11,6 +11,7 @@ from aiocache import cached from zhenxun.services.log import logger from zhenxun.utils.github_utils.models import AliyunFileInfo +from zhenxun.utils.repo_utils.utils import prepare_aliyun_url from .base_manager import BaseRepoManager from .config import LOG_COMMAND, RepoConfig @@ -445,32 +446,6 @@ class AliyunCodeupManager(BaseRepoManager): 返回: RepoUpdateResult: 更新结果 """ - - # 定义预处理函数，构建阿里云CodeUp的URL - def prepare_aliyun_url(repo_url: str) -> str: - import base64 - - repo_name = repo_url.split("/tree/")[0].split("/")[-1].replace(".git", "") - # 构建仓库URL - # 阿里云CodeUp的仓库URL格式通常为： - # https://codeup.aliyun.com/{organization_id}/{organization_name}/{repo_name}.git - url = f"https://codeup.aliyun.com/{self.config.aliyun_codeup.organization_id}/{self.config.aliyun_codeup.organization_name}/{repo_name}.git" - - # 添加访问令牌 - 使用base64解码后的令牌 - if self.config.aliyun_codeup.rdc_access_token_encrypted: - try: - # 解码RDC访问令牌 - token = base64.b64decode( - self.config.aliyun_codeup.rdc_access_token_encrypted.encode() - ).decode() - # 阿里云CodeUp使用oauth2:token的格式进行身份验证 - url = url.replace("https://", f"https://oauth2:{token}@") - logger.debug(f"使用RDC令牌构建阿里云URL: {url.split('@')[0]}@***") - except Exception as e: - logger.error(f"解码RDC令牌失败: {e}") - - return url - # 调用基类的update_via_git方法 return await super().update_via_git( repo_url=repo_url, diff --git a/zhenxun/utils/repo_utils/exceptions.py b/zhenxun/utils/repo_utils/exceptions.py index d508f303..a4049974 100644 --- a/zhenxun/utils/repo_utils/exceptions.py +++ b/zhenxun/utils/repo_utils/exceptions.py @@ -66,3 +66,10 @@ class ConfigError(RepoManagerError): def __init__(self, message: str): super().__init__(f"配置错误: {message}") + + +class GitUnavailableError(RepoManagerError): + """Git不可用异常""" + + def __init__(self, message: str = "Git命令不可用"): + super().__init__(message) diff --git a/zhenxun/utils/repo_utils/file_manager.py b/zhenxun/utils/repo_utils/file_manager.py index 1e9226c7..94d50db3 100644 --- a/zhenxun/utils/repo_utils/file_manager.py +++ b/zhenxun/utils/repo_utils/file_manager.py @@ -2,6 +2,7 @@ 仓库文件管理器，用于从GitHub和阿里云CodeUp获取指定文件内容 """ +import contextlib from pathlib import Path from typing import cast, overload @@ -12,10 +13,17 @@ from zhenxun.services.log import logger from zhenxun.utils.github_utils import GithubUtils from zhenxun.utils.github_utils.models import AliyunTreeType, GitHubStrategy, TreeType from zhenxun.utils.http_utils import AsyncHttpx +from zhenxun.utils.utils import is_binary_file from .config import LOG_COMMAND, RepoConfig -from .exceptions import FileNotFoundError, NetworkError, RepoManagerError +from .exceptions import ( + FileNotFoundError, + GitUnavailableError, + NetworkError, + RepoManagerError, +) from .models import FileDownloadResult, RepoFileInfo, RepoType +from .utils import prepare_aliyun_url, sparse_checkout_clone class RepoFileManager: @@ -74,18 +82,20 @@ class RepoFileManager: ) if response.status_code == 200: logger.info(f"获取github文件内容成功: {f}", LOG_COMMAND) + text_content = response.content # 确保使用UTF-8编码解析响应内容 - try: - text_content = response.content.decode("utf-8") - except UnicodeDecodeError: - # 如果UTF-8解码失败，尝试其他编码 - text_content = response.content.decode( - "utf-8", errors="ignore" - ) - logger.warning( - f"解码文件内容时出现错误，使用忽略错误模式: {f}", - LOG_COMMAND, - ) + if not is_binary_file(f): + try: + text_content = response.content.decode("utf-8") + except UnicodeDecodeError: + # 如果UTF-8解码失败，尝试其他编码 + text_content = response.content.decode( + "utf-8", errors="ignore" + ) + logger.warning( + f"解码文件内容时出现错误，使用忽略错误模式:{f}", + LOG_COMMAND, + ) results.append((f, text_content)) break else: @@ -262,15 +272,15 @@ class RepoFileManager: if repo_type is None: # 尝试GitHub，失败则尝试阿里云 try: - return await self._list_github_directory_files( - repo_url, directory_path, branch, recursive + return await self._list_aliyun_directory_files( + repo_name, directory_path, branch, recursive ) except Exception as e: logger.warning( - "获取GitHub目录文件失败，尝试阿里云", LOG_COMMAND, e=e + "获取阿里云目录文件失败，尝试GitHub", LOG_COMMAND, e=e ) - return await self._list_aliyun_directory_files( - repo_name, directory_path, branch, recursive + return await self._list_github_directory_files( + repo_url, directory_path, branch, recursive ) if repo_type == RepoType.GITHUB: return await self._list_github_directory_files( @@ -466,6 +476,8 @@ class RepoFileManager: branch: str = "main", repo_type: RepoType | None = None, ignore_error: bool = False, + sparse_path: str | None = None, + target_dir: Path | None = None, ) -> FileDownloadResult: """ 下载单个文件 @@ -476,10 +488,19 @@ class RepoFileManager: branch: 分支名称 repo_type: 仓库类型，如果为None则自动判断 ignore_error: 是否忽略错误 + sparse_path: 稀疏检出路径 + target_dir: 稀疏目标目录 返回: FileDownloadResult: 下载结果 """ + + # 参数一致性校验：sparse_path 与 target_dir 必须同时有值或同时为 None + if (sparse_path is None) ^ (target_dir is None): + raise RepoManagerError( + "参数错误: sparse_path 与 target_dir 必须同时提供或同时为 None" + ) + # 确定仓库类型和所有者 repo_name = ( repo_url.split("/tree/")[0].split("/")[-1].replace(".git", "").strip() @@ -497,12 +518,43 @@ class RepoFileManager: file_path=file_path, version=branch, ) + if ( + any(is_binary_file(file_name) for file_name in file_path_mapping) + and repo_type != RepoType.GITHUB + and sparse_path + and target_dir + ): + return await self._handle_binary_with_sparse_checkout( + repo_url=repo_url, + branch=branch, + sparse_path=sparse_path, + target_dir=target_dir, + result=result, + ) + else: + # 不包含二进制时 + return await self._download_and_write_files( + repo_url=repo_url, + file_paths=[f[0] for f in file_path], + file_path_mapping=file_path_mapping, + branch=branch, + repo_type=repo_type, + ignore_error=ignore_error, + result=result, + ) + async def _download_and_write_files( + self, + repo_url: str, + file_paths: list[str], + file_path_mapping: dict[str, Path], + branch: str, + repo_type: RepoType | None, + ignore_error: bool, + result: FileDownloadResult, + ) -> FileDownloadResult: try: - # 由于我们传入的是列表，所以这里一定返回列表 - file_paths = [f[0] for f in file_path] if len(file_paths) == 1: - # 如果只有一个文件，可能返回单个元组 file_contents_result = await self.get_file_content( repo_url, file_paths[0], branch, repo_type, ignore_error ) @@ -513,7 +565,6 @@ class RepoFileManager: else: file_contents = cast(list[tuple[str, str]], file_contents_result) else: - # 多个文件一定返回列表 file_contents = cast( list[tuple[str, str]], await self.get_file_content( @@ -524,7 +575,6 @@ class RepoFileManager: for repo_file_path, content in file_contents: local_path = file_path_mapping[repo_file_path] local_path.parent.mkdir(parents=True, exist_ok=True) - # 使用二进制模式写入文件，避免编码问题 if isinstance(content, str): content_bytes = content.encode("utf-8") else: @@ -533,7 +583,6 @@ class RepoFileManager: async with aiofiles.open(local_path, "wb") as f: await f.write(content_bytes) result.success = True - # 计算文件大小 result.file_size = sum( len(content.encode("utf-8") if isinstance(content, str) else content) for _, content in file_contents @@ -545,3 +594,42 @@ class RepoFileManager: result.success = False result.error_message = str(e) return result + + async def _handle_binary_with_sparse_checkout( + self, + repo_url: str, + branch: str, + sparse_path: str, + target_dir: Path, + result: FileDownloadResult, + ) -> FileDownloadResult: + try: + await sparse_checkout_clone( + repo_url=prepare_aliyun_url(repo_url), + branch=branch, + sparse_path=sparse_path, + target_dir=target_dir, + ) + total_size = 0 + if target_dir.exists(): + for f in target_dir.rglob("*"): + if f.is_file(): + with contextlib.suppress(Exception): + total_size += f.stat().st_size + result.success = True + result.file_size = total_size + logger.info(f"sparse-checkout 克隆成功: {target_dir}") + return result + except GitUnavailableError as e: + logger.error(f"Git不可用: {e}") + result.success = False + result.error_message = ( + "当前插件包含二进制文件，因ali限制需要使用git，" + "当前Git不可用，请尝试添加参数 -s git 或 安装 git" + ) + return result + except Exception as e: + logger.error(f"sparse-checkout 克隆失败: {e}") + result.success = False + result.error_message = str(e) + return result diff --git a/zhenxun/utils/repo_utils/utils.py b/zhenxun/utils/repo_utils/utils.py index 7aceb231..5fc03e86 100644 --- a/zhenxun/utils/repo_utils/utils.py +++ b/zhenxun/utils/repo_utils/utils.py @@ -3,12 +3,15 @@ """ import asyncio +import base64 from pathlib import Path import re +import shutil from zhenxun.services.log import logger -from .config import LOG_COMMAND +from .config import LOG_COMMAND, RepoConfig +from .exceptions import GitUnavailableError async def check_git() -> bool: @@ -133,3 +136,119 @@ def filter_files( result = [file for file in result if not re.match(regex_pattern, file)] return result + + +async def sparse_checkout_clone( + repo_url: str, + branch: str, + sparse_path: str, + target_dir: Path, +) -> None: + """ + 使用 git 稀疏检出克隆指定路径到目标目录（完全独立于主项目 git）。 + + 关键保障: + - 在 target_dir 下检测/初始化 .git，所有 git 操作均以 cwd=target_dir 执行 + - 强制拉取与工作区覆盖: fetch --force、checkout -B、reset --hard、clean -xdf + - 反复设置 sparse-checkout 路径，确保路径更新生效 + """ + target_dir.mkdir(parents=True, exist_ok=True) + + if not await check_git(): + raise GitUnavailableError() + + git_dir = target_dir / ".git" + if not git_dir.exists(): + success, out, err = await run_git_command("init", target_dir) + if not success: + raise RuntimeError(f"git init 失败: {err or out}") + success, out, err = await run_git_command( + f"remote add origin {repo_url}", target_dir + ) + if not success: + raise RuntimeError(f"添加远程失败: {err or out}") + else: + success, out, err = await run_git_command( + f"remote set-url origin {repo_url}", target_dir + ) + if not success: + # 兜底尝试添加 + await run_git_command(f"remote add origin {repo_url}", target_dir) + + # 启用稀疏检出（使用 --no-cone 模式以获得更精确的控制） + await run_git_command("config core.sparseCheckout true", target_dir) + await run_git_command("sparse-checkout init --no-cone", target_dir) + + # 设置需要检出的路径（每次都覆盖配置） + if not sparse_path: + raise RuntimeError("sparse-checkout 路径不能为空") + + # 使用 --no-cone 模式，直接指定要检出的具体路径 + # 例如：sparse_path="plugins/mahiro" -> 只检出 plugins/mahiro/ 下的内容 + success, out, err = await run_git_command( + f"sparse-checkout set {sparse_path}/", target_dir + ) + if not success: + raise RuntimeError(f"配置稀疏路径失败: {err or out}") + + # 强制拉取并同步到远端 + success, out, err = await run_git_command( + f"fetch --force --depth 1 origin {branch}", target_dir + ) + if not success: + raise RuntimeError(f"fetch 失败: {err or out}") + + # 使用远端强制更新本地分支并覆盖工作区 + success, out, err = await run_git_command( + f"checkout -B {branch} origin/{branch}", target_dir + ) + if not success: + # 回退方案 + success2, out2, err2 = await run_git_command(f"checkout {branch}", target_dir) + if not success2: + raise RuntimeError(f"checkout 失败: {(err or out) or (err2 or out2)}") + + # 强制对齐工作区 + await run_git_command(f"reset --hard origin/{branch}", target_dir) + await run_git_command("clean -xdf", target_dir) + + dir_path = target_dir / Path(sparse_path) + for f in dir_path.iterdir(): + shutil.move(f, target_dir / f.name) + dir_name = sparse_path.split("/")[0] + rm_path = target_dir / dir_name + if rm_path.exists(): + shutil.rmtree(rm_path) + + +def prepare_aliyun_url(repo_url: str) -> str: + """解析阿里云CodeUp的仓库URL + + 参数: + repo_url: 仓库URL + + 返回: + str: 解析后的仓库URL + """ + config = RepoConfig.get_instance() + + repo_name = repo_url.split("/tree/")[0].split("/")[-1].replace(".git", "") + # 构建仓库URL + # 阿里云CodeUp的仓库URL格式通常为： + # https://codeup.aliyun.com/{organization_id}/{organization_name}/{repo_name}.git + url = f"https://codeup.aliyun.com/{config.aliyun_codeup.organization_id}/{config.aliyun_codeup.organization_name}/{repo_name}.git" + + # 添加访问令牌 - 使用base64解码后的令牌 + if config.aliyun_codeup.rdc_access_token_encrypted: + try: + # 解码RDC访问令牌 + token = base64.b64decode( + config.aliyun_codeup.rdc_access_token_encrypted.encode() + ).decode() + # 阿里云CodeUp使用oauth2:token的格式进行身份验证 + url = url.replace("https://", f"https://oauth2:{token}@") + logger.debug(f"使用RDC令牌构建阿里云URL: {url.split('@')[0]}@***") + except Exception as e: + logger.error(f"解码RDC令牌失败: {e}") + + return url diff --git a/zhenxun/utils/utils.py b/zhenxun/utils/utils.py index 4fa92bcd..d5921803 100644 --- a/zhenxun/utils/utils.py +++ b/zhenxun/utils/utils.py @@ -1,9 +1,12 @@ +from collections.abc import Callable from dataclasses import dataclass from datetime import datetime import os from pathlib import Path +import stat import time -from typing import ClassVar +from types import TracebackType +from typing import Any, ClassVar import httpx from nonebot_plugin_uninfo import Uninfo @@ -65,22 +68,39 @@ class ResourceDirManager: def is_binary_file(file_path: str) -> bool: - """判断是否为二进制文件""" - binary_extensions = { - ".jpg", - ".jpeg", - ".png", - ".gif", - ".bmp", - ".ico", - ".pdf", - ".zip", - ".rar", - ".7z", - ".exe", - ".dll", - } - return any(file_path.lower().endswith(ext) for ext in binary_extensions) + """判断是否为二进制文件 + + 参数: + file_path: 文件路径 + + 返回: + bool: 是否为二进制文件 + """ + # fmt: off + # 精简但包含图片和字体的二进制文件扩展名集合 + BINARY_EXTENSIONS = frozenset({ + # 图片文件 + "jpg", "jpeg", "png", "gif", "bmp", "ico", "webp", "tiff", "tif", "svg", + # 字体文件 + "ttf", "otf", "woff", "woff2", "eot", + # 压缩文件 + "zip", "rar", "7z", "tar", "gz", "bz2", "xz", + # 可执行文件和库 + "exe", "dll", "so", "dylib", + # 文档文件 + "pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx", + # 多媒体文件 + "mp3", "mp4", "avi", "mov", "wmv", "flv", + # 其他常见二进制文件 + "bin", "dat", "db", "class", "pyc" + }) + + # 使用os.path.splitext高效提取扩展名 + _, ext = os.path.splitext(file_path) + # 去除点号并转换为小写 + ext_clean = ext.lstrip(".").lower() + + return ext_clean in BINARY_EXTENSIONS def cn2py(word: str) -> str: @@ -224,3 +244,24 @@ def is_number(text: str) -> bool: return True except ValueError: return False + + +def win_on_rm_error( + func: Callable[[str], Any], + path: str, + _exc_info: tuple[type[BaseException], BaseException, TracebackType], +) -> None: + """Windows下删除只读文件/目录时的回调。 + + 去除只读属性后重试删除，避免 WinError 5。 + """ + try: + os.chmod(path, stat.S_IWRITE) + except Exception: + # 即使去除权限失败也继续尝试 + pass + try: + func(path) + except Exception: + # 仍失败则记录调试日志并忽略，交由上层继续处理 + logger.debug(f"删除失败重试仍失败: {path}")