From 1508782f49f004c90a7c6c9c99bee83ce55e2ddf Mon Sep 17 00:00:00 2001 From: HibiKier <775757368@qq.com> Date: Sun, 21 Aug 2022 16:47:50 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BE=9B=E5=85=A8=E5=B1=80=E5=AD=97?= =?UTF-8?q?=E5=85=B8GDict=EF=BC=8C=E9=80=82=E9=85=8Domega=2013w=E5=BC=A0?= =?UTF-8?q?=E5=9B=BE=E7=9A=84=E6=95=B0=E6=8D=AE=E7=BB=93=E6=9E=84=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 + basic_plugins/scripts.py | 35 +++-------- plugins/pix_gallery/__init__.py | 3 + .../pix_gallery/_model/omega_pixiv_illusts.py | 13 ++--- plugins/pix_gallery/pix_update.py | 58 +++++++++---------- utils/manager/__init__.py | 5 ++ 6 files changed, 53 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index ec1f2890..93a13e06 100644 --- a/README.md +++ b/README.md @@ -279,6 +279,8 @@ PS: **ARM平台** 请使用全量版 同时 **如果你的机器 RAM < 1G 可能 * 修复b站转发解析av号无法解析 * B站订阅直播订阅支持短号 * 开箱提供重置开箱命令，重置今日所有开箱数据（重置次数，并不会删除今日已开箱记录） +* 提供全局字典GDict，通过from utils.manager import GDict导入 +* 适配omega 13w张图的数据结构表（建议删表重导） * 改进插件 `我有一个朋友`，避免触发过于频繁 [@pull/1001](https://github.com/HibiKier/zhenxun_bot/pull/1001) * 原神便笺新增洞天宝钱和参量质变仪提示 [@pull/1005](https://github.com/HibiKier/zhenxun_bot/pull/1005) * 新增米游社签到功能，自动领取（白嫖）米游币 [@pull/991](https://github.com/HibiKier/zhenxun_bot/pull/991) diff --git a/basic_plugins/scripts.py b/basic_plugins/scripts.py index 488f9d15..8965aac0 100755 --- a/basic_plugins/scripts.py +++ b/basic_plugins/scripts.py @@ -1,3 +1,5 @@ +import random + from asyncpg.exceptions import ( DuplicateColumnError, UndefinedColumnError, @@ -13,6 +15,7 @@ from configs.path_config import TEXT_PATH from asyncio.exceptions import TimeoutError from typing import List from utils.http_utils import AsyncHttpx +from utils.manager import GDict from utils.utils import scheduler import nonebot @@ -133,10 +136,13 @@ async def _(): "goods_info" ), # 新增纯文本 ] - for sql in sql_str: + for sql in sql_str + GDict.get('run_sql', []): try: - flag = sql[1] - sql = sql[0] + if isinstance(sql, str): + flag = f'{random.randint(1, 10000)}' + else: + flag = sql[1] + sql = sql[0] query = db.text(sql) await db.first(query) logger.info(f"完成sql操作：{sql}") @@ -148,29 +154,6 @@ async def _(): # bag_user 将文本转为字典格式 await __database_script(_flag) - # 完成后 - end_sql_str = [ - # "ALTER TABLE bag_users DROP COLUMN props;" # 删除 bag_users 的 props 字段（还不到时候） - ] - for sql in end_sql_str: - try: - query = db.text(sql) - await db.first(query) - logger.info(f"完成执行sql操作：{sql}") - except (DuplicateColumnError, UndefinedColumnError): - pass - except PostgresSyntaxError: - logger.error(f"语法错误：执行sql失败：{sql}") - - # str2json_sql = ["alter table bag_users alter COLUMN props type json USING props::json;"] # 字段类型替换 - # rename_sql = 'alter table {} rename {} to {};' # 字段更名 - # for sql in str2json_sql: - # try: - # query = db.text(sql) - # await db.first(query) - # except DuplicateColumnError: - # pass - @driver.on_bot_connect async def _(bot: Bot): diff --git a/plugins/pix_gallery/__init__.py b/plugins/pix_gallery/__init__.py index 6f7b379a..1b2bc5ac 100755 --- a/plugins/pix_gallery/__init__.py +++ b/plugins/pix_gallery/__init__.py @@ -1,4 +1,5 @@ from configs.config import Config +from utils.manager import GDict import nonebot @@ -60,6 +61,8 @@ Config.add_plugin_config( default_value=True ) +GDict['run_sql'].append("ALTER TABLE omega_pixiv_illusts ADD classified Integer;") + nonebot.load_plugins("plugins/pix_gallery") diff --git a/plugins/pix_gallery/_model/omega_pixiv_illusts.py b/plugins/pix_gallery/_model/omega_pixiv_illusts.py index 02ac051a..27721fc3 100644 --- a/plugins/pix_gallery/_model/omega_pixiv_illusts.py +++ b/plugins/pix_gallery/_model/omega_pixiv_illusts.py @@ -1,5 +1,4 @@ from typing import Optional, List, Tuple -from datetime import datetime from services.db_context import db @@ -12,13 +11,12 @@ class OmegaPixivIllusts(db.Model): uid = db.Column(db.BigInteger(), nullable=False) title = db.Column(db.String(), nullable=False) uname = db.Column(db.String(), nullable=False) + classified = db.Column(db.Integer(), nullable=False) nsfw_tag = db.Column(db.Integer(), nullable=False) width = db.Column(db.Integer(), nullable=False) height = db.Column(db.Integer(), nullable=False) tags = db.Column(db.String(), nullable=False) url = db.Column(db.String(), nullable=False) - created_at = db.Column(db.DateTime(timezone=True)) - updated_at = db.Column(db.DateTime(timezone=True)) _idx1 = db.Index("omega_pixiv_illusts_idx1", "pid", "url", unique=True) @@ -32,10 +30,9 @@ class OmegaPixivIllusts(db.Model): url: str, uid: int, uname: str, + classified: int, nsfw_tag: int, tags: str, - created_at: datetime, - updated_at: datetime, ): """ 说明: @@ -48,10 +45,9 @@ class OmegaPixivIllusts(db.Model): :param url: url链接 :param uid: 作者uid :param uname: 作者名称 - :param nsfw_tag: nsfw标签, 0=safe, 1=setu. 2=r18 + :param classified: 标记标签, 0=未标记, 1=已人工标记或从可信已标记来源获取 + :param nsfw_tag: nsfw标签,-1=未标记, 0=safe, 1=setu. 2=r18 :param tags: 相关tag - :param created_at: 创建日期 - :param updated_at: 更新日期 """ if not await cls.check_exists(pid): await cls.create( @@ -62,6 +58,7 @@ class OmegaPixivIllusts(db.Model): url=url, uid=uid, uname=uname, + classified=classified, nsfw_tag=nsfw_tag, tags=tags, ) diff --git a/plugins/pix_gallery/pix_update.py b/plugins/pix_gallery/pix_update.py index 0d8e0dc5..4db3d5fa 100755 --- a/plugins/pix_gallery/pix_update.py +++ b/plugins/pix_gallery/pix_update.py @@ -1,3 +1,5 @@ +import re + from nonebot import on_command from utils.utils import is_number from nonebot.permission import SUPERUSER @@ -11,7 +13,6 @@ import time from services.log import logger from pathlib import Path from typing import List -from datetime import datetime import asyncio import os @@ -144,38 +145,36 @@ async def _(arg: Message = CommandArg()): @check_omega.handle() async def _(): async def _tasks(line: str, all_pid: List[int], length: int, index: int): - data = line.split("VALUES", maxsplit=1)[-1].strip() - if data.startswith("("): - data = data[1:] - if data.endswith(");"): - data = data[:-2] - x = data.split(maxsplit=3) - pid = int(x[1][:-1].strip()) + data = line.split("VALUES", maxsplit=1)[-1].strip()[1:-2] + num_list = re.findall(r'(\d+)', data) + pid = int(num_list[1]) + uid = int(num_list[2]) + id_ = 3 + while num_list[id_] not in ['0', '1']: + id_ += 1 + classified = int(num_list[id_]) + nsfw_tag = int(num_list[id_ + 1]) + width = int(num_list[id_ + 2]) + height = int(num_list[id_ + 3]) + str_list = re.findall(r"'(.*?)',", data) + title = str_list[0] + uname = str_list[1] + tags = str_list[2] + url = str_list[3] if pid in all_pid: logger.info(f"添加OmegaPixivIllusts图库数据已存在 ---> pid：{pid}") return - uid = int(x[2][:-1].strip()) - x = x[3].split(", '") - title = x[0].strip()[1:-1] - tmp = x[1].split(", ") - author = tmp[0].strip()[:-1] - nsfw_tag = int(tmp[1]) - width = int(tmp[2]) - height = int(tmp[3]) - tags = x[2][:-1] - url = x[3][:-1] if await OmegaPixivIllusts.add_image_data( - pid, - title, - width, - height, - url, - uid, - author, - nsfw_tag, - tags, - datetime.min, - datetime.min, + pid=pid, + title=title, + width=width, + height=height, + url=url, + uid=uid, + nsfw_tag=nsfw_tag, + tags=tags, + uname=uname, + classified=classified ): logger.info( f"成功添加OmegaPixivIllusts图库数据 pid：{pid} 本次预计存储 {length} 张，已更新第 {index} 张" @@ -197,6 +196,7 @@ async def _(): for line in lines: if "INSERT INTO" in line.upper(): index += 1 + logger.info(f'line: {line} 加入更新计划') tasks.append( asyncio.ensure_future(_tasks(line, all_pid, length, index)) ) diff --git a/utils/manager/__init__.py b/utils/manager/__init__.py index b2f1b158..0e5e172e 100755 --- a/utils/manager/__init__.py +++ b/utils/manager/__init__.py @@ -14,6 +14,11 @@ from .requests_manager import RequestManager from configs.path_config import DATA_PATH +# 全局字典 +GDict = { + "run_sql": [] # 需要启动前运行的sql语句 +} + # 群功能开关 | 群被动技能 | 群权限 管理 group_manager: Optional[GroupManager] = GroupManager( DATA_PATH / "manager" / "group_manager.json"