提供全局字典GDict，适配omega 13w张图的数据结构表

README.md

@@ -279,6 +279,8 @@ PS: **ARM平台** 请使用全量版 同时 **如果你的机器 RAM < 1G 可能
 * 修复b站转发解析av号无法解析
 * B站订阅直播订阅支持短号
 * 开箱提供重置开箱命令，重置今日所有开箱数据（重置次数，并不会删除今日已开箱记录）
+* 提供全局字典GDict，通过from utils.manager import GDict导入
+* 适配omega 13w张图的数据结构表（建议删表重导）
 * 改进插件 `我有一个朋友`，避免触发过于频繁 [@pull/1001](https://github.com/HibiKier/zhenxun_bot/pull/1001)
 * 原神便笺新增洞天宝钱和参量质变仪提示 [@pull/1005](https://github.com/HibiKier/zhenxun_bot/pull/1005)
 * 新增米游社签到功能，自动领取（白嫖）米游币 [@pull/991](https://github.com/HibiKier/zhenxun_bot/pull/991)

basic_plugins/scripts.py

@@ -1,3 +1,5 @@
+import random
+
 from asyncpg.exceptions import (
     DuplicateColumnError,
     UndefinedColumnError,
@@ -13,6 +15,7 @@ from configs.path_config import TEXT_PATH
 from asyncio.exceptions import TimeoutError
 from typing import List
 from utils.http_utils import AsyncHttpx
+from utils.manager import GDict
 from utils.utils import scheduler
 import nonebot
 
@@ -133,10 +136,13 @@ async def _():
             "goods_info"
         ),  # 新增纯文本
     ]
-    for sql in sql_str:
+    for sql in sql_str + GDict.get('run_sql', []):
         try:
-            flag = sql[1]
+            if isinstance(sql, str):
-            sql = sql[0]
+                flag = f'{random.randint(1, 10000)}'
+            else:
+                flag = sql[1]
+                sql = sql[0]
             query = db.text(sql)
             await db.first(query)
             logger.info(f"完成sql操作：{sql}")
@@ -148,29 +154,6 @@ async def _():
     # bag_user 将文本转为字典格式
     await __database_script(_flag)
 
-    # 完成后
-    end_sql_str = [
-        # "ALTER TABLE bag_users DROP COLUMN props;"          # 删除 bag_users 的 props 字段（还不到时候）
-    ]
-    for sql in end_sql_str:
-        try:
-            query = db.text(sql)
-            await db.first(query)
-            logger.info(f"完成执行sql操作：{sql}")
-        except (DuplicateColumnError, UndefinedColumnError):
-            pass
-        except PostgresSyntaxError:
-            logger.error(f"语法错误：执行sql失败：{sql}")
-
-    # str2json_sql = ["alter table bag_users alter COLUMN props type json USING props::json;"]       # 字段类型替换
-    # rename_sql = 'alter table {} rename {} to {};'                              # 字段更名
-    # for sql in str2json_sql:
-    #     try:
-    #         query = db.text(sql)
-    #         await db.first(query)
-    #     except DuplicateColumnError:
-    #         pass
-
 
 @driver.on_bot_connect
 async def _(bot: Bot):

plugins/pix_gallery/__init__.py

@@ -1,4 +1,5 @@
 from configs.config import Config
+from utils.manager import GDict
 import nonebot
 
 
@@ -60,6 +61,8 @@ Config.add_plugin_config(
     default_value=True
 )
 
+GDict['run_sql'].append("ALTER TABLE omega_pixiv_illusts ADD classified Integer;")
+
 nonebot.load_plugins("plugins/pix_gallery")
 
 

plugins/pix_gallery/_model/omega_pixiv_illusts.py

@@ -1,5 +1,4 @@
 from typing import Optional, List, Tuple
-from datetime import datetime
 from services.db_context import db
 
 
@@ -12,13 +11,12 @@ class OmegaPixivIllusts(db.Model):
     uid = db.Column(db.BigInteger(), nullable=False)
     title = db.Column(db.String(), nullable=False)
     uname = db.Column(db.String(), nullable=False)
+    classified = db.Column(db.Integer(), nullable=False)
     nsfw_tag = db.Column(db.Integer(), nullable=False)
     width = db.Column(db.Integer(), nullable=False)
     height = db.Column(db.Integer(), nullable=False)
     tags = db.Column(db.String(), nullable=False)
     url = db.Column(db.String(), nullable=False)
-    created_at = db.Column(db.DateTime(timezone=True))
-    updated_at = db.Column(db.DateTime(timezone=True))
 
     _idx1 = db.Index("omega_pixiv_illusts_idx1", "pid", "url", unique=True)
 
@@ -32,10 +30,9 @@ class OmegaPixivIllusts(db.Model):
             url: str,
             uid: int,
             uname: str,
+            classified: int,
             nsfw_tag: int,
             tags: str,
-            created_at: datetime,
-            updated_at: datetime,
     ):
         """
         说明:
@@ -48,10 +45,9 @@ class OmegaPixivIllusts(db.Model):
             :param url: url链接
             :param uid: 作者uid
             :param uname: 作者名称
-            :param nsfw_tag: nsfw标签, 0=safe, 1=setu. 2=r18
+            :param classified: 标记标签, 0=未标记, 1=已人工标记或从可信已标记来源获取
+            :param nsfw_tag: nsfw标签,-1=未标记, 0=safe, 1=setu. 2=r18
             :param tags: 相关tag
-            :param created_at: 创建日期
-            :param updated_at: 更新日期
         """
         if not await cls.check_exists(pid):
             await cls.create(
@@ -62,6 +58,7 @@ class OmegaPixivIllusts(db.Model):
                 url=url,
                 uid=uid,
                 uname=uname,
+                classified=classified,
                 nsfw_tag=nsfw_tag,
                 tags=tags,
             )

plugins/pix_gallery/pix_update.py

@@ -1,3 +1,5 @@
+import re
+
 from nonebot import on_command
 from utils.utils import is_number
 from nonebot.permission import SUPERUSER
@@ -11,7 +13,6 @@ import time
 from services.log import logger
 from pathlib import Path
 from typing import List
-from datetime import datetime
 import asyncio
 import os
 
@@ -144,38 +145,36 @@ async def _(arg: Message = CommandArg()):
 @check_omega.handle()
 async def _():
     async def _tasks(line: str, all_pid: List[int], length: int, index: int):
-        data = line.split("VALUES", maxsplit=1)[-1].strip()
+        data = line.split("VALUES", maxsplit=1)[-1].strip()[1:-2]
-        if data.startswith("("):
+        num_list = re.findall(r'(\d+)', data)
-            data = data[1:]
+        pid = int(num_list[1])
-        if data.endswith(");"):
+        uid = int(num_list[2])
-            data = data[:-2]
+        id_ = 3
-        x = data.split(maxsplit=3)
+        while num_list[id_] not in ['0', '1']:
-        pid = int(x[1][:-1].strip())
+            id_ += 1
+        classified = int(num_list[id_])
+        nsfw_tag = int(num_list[id_ + 1])
+        width = int(num_list[id_ + 2])
+        height = int(num_list[id_ + 3])
+        str_list = re.findall(r"'(.*?)',", data)
+        title = str_list[0]
+        uname = str_list[1]
+        tags = str_list[2]
+        url = str_list[3]
         if pid in all_pid:
             logger.info(f"添加OmegaPixivIllusts图库数据已存在 ---> pid：{pid}")
             return
-        uid = int(x[2][:-1].strip())
-        x = x[3].split(", '")
-        title = x[0].strip()[1:-1]
-        tmp = x[1].split(", ")
-        author = tmp[0].strip()[:-1]
-        nsfw_tag = int(tmp[1])
-        width = int(tmp[2])
-        height = int(tmp[3])
-        tags = x[2][:-1]
-        url = x[3][:-1]
         if await OmegaPixivIllusts.add_image_data(
-                pid,
+                pid=pid,
-                title,
+                title=title,
-                width,
+                width=width,
-                height,
+                height=height,
-                url,
+                url=url,
-                uid,
+                uid=uid,
-                author,
+                nsfw_tag=nsfw_tag,
-                nsfw_tag,
+                tags=tags,
-                tags,
+                uname=uname,
-                datetime.min,
+                classified=classified
-                datetime.min,
         ):
             logger.info(
                 f"成功添加OmegaPixivIllusts图库数据 pid：{pid} 本次预计存储 {length} 张，已更新第 {index} 张"
@@ -197,6 +196,7 @@ async def _():
         for line in lines:
             if "INSERT INTO" in line.upper():
                 index += 1
+                logger.info(f'line: {line} 加入更新计划')
                 tasks.append(
                     asyncio.ensure_future(_tasks(line, all_pid, length, index))
                 )

utils/manager/__init__.py

@@ -14,6 +14,11 @@ from .requests_manager import RequestManager
 from configs.path_config import DATA_PATH
 
 
+# 全局字典
+GDict = {
+    "run_sql": []           # 需要启动前运行的sql语句
+}
+
 # 群功能开关 | 群被动技能 | 群权限  管理
 group_manager: Optional[GroupManager] = GroupManager(
     DATA_PATH / "manager" / "group_manager.json"