JTools v3.19.0

功能变动：

- 重构数据库初始化与迁移逻辑
- 优化上榜文章查询自动补全逻辑

依赖变动：

- 更新依赖库

README.md

@@ -7,65 +7,23 @@
 
 ## 数据库准备
 
-本服务的部分模块依赖外部数据源 `jianshu` 和 `jpep` 数据库，您需事先下载并进行导入。
+本服务的部分模块依赖外部数据源 `jianshu` 与 `jpep` 数据库，您需事先下载并进行导入。
 
-创建用户：
+进入 `sql` 目录：
 
-```sql
-CREATE ROLE jtools LOGIN PASSWORD 'jtools';
-```
-
-创建数据库：
-
-```sql
-CREATE DATABASE jtools WITH OWNER = jtools;
-CREATE DATABASE logs;
-```
-
-创建扩展：
-
-（`jianshu` 数据库）
-
-```sql
-CREATE EXTENSION IF NOT EXISTS pg_trgm;
-```
-
-为用户授权：
-
-（`logs` 数据库）
-
-```sql
-GRANT CREATE ON SCHEMA public TO jtools;
-```
-
-（`jianshu` 数据库）
-
-```sql
-GRANT SELECT ON TABLE article_earning_ranking_records TO jtools;
-GRANT SELECT ON TABLE lottery_win_records TO jtools;
-GRANT SELECT ON TABLE users TO jtools;
-```
-
-（`jpep` 数据库）
-
-```sql
-GRANT SELECT ON TABLE ftn_macket_records TO jtools;
-GRANT SELECT ON TABLE ftn_orders TO jtools;
+```shell
+cd sql
 ```
 
-创建索引：
+如果您需要修改数据库用户名和密码，请修改 `sql` 目录下的 `0.sql` 和每个子目录下的 `0.sql` 文件。
 
-（`jianshu` 数据库）
+您需要一个具有创建用户和数据库权限的用户（一般是超级用户）来完成数据库准备。
 
-```sql
-CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_article_earning_ranking_records_ranking ON article_earning_ranking_records (ranking);
+每个目录中的 SQL 脚本均应按照编号顺序执行。
 
-CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_lottery_win_records_time ON lottery_win_records (time);
-CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_lottery_win_records_user_slug ON lottery_win_records (user_slug);
-CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_lottery_win_records_award_name ON lottery_win_records (award_name);
+首先，执行 `sql` 目录下的脚本。
 
-CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_users_name ON users USING gin (name gin_trgm_ops);
-```
+依次切换到与 `sql` 下的子目录（数据库目录）名称相同的数据库中，先执行每个表目录中的 SQL 脚本，再执行数据库目录下的 SQL 脚本。
 
 ## 配置
 
@@ -82,7 +40,7 @@ cp config.example.toml config.toml
 - jpep_postgres.host 填写 `postgres`
 - uvicorn.host 填写 `0.0.0.0`
 
-同时，您需要填写正确的 `postgres.user` 和 `postgres.password`。
+同时，您需要填写正确的 `{db_name}_postgres.user` 和 `{db_name}_postgres.password`。
 
 `word_split_access_key` 为具有 [NLP 服务](https://ai.aliyun.com/nlp) 使用权限的阿里云用户 Access Key，您可在 [RAM 访问控制](https://ram.console.aliyun.com) 中创建用户，并为其赋予 `AliyunNLPReadOnlyAccess` 权限。
 

backend/main.py

@@ -3,31 +3,14 @@
 
 from uvicorn import run as uvicorn_run
 
-from models.debug_project_record import DebugProjectRecord
-from models.jianshu.lottery_win_record import LotteryWinRecord
-from models.jianshu.user import User
-from models.tech_stack import TechStack
-from models.tool import Tool
+from models import init_db
 from utils.config import CONFIG
-from utils.db import jianshu_pool, jpep_pool, jtools_pool
 from utils.log import logger
 
 logging.getLogger("httpx").setLevel(logging.CRITICAL)
 logging.getLogger("httpcore").setLevel(logging.CRITICAL)
 
 
-async def init_db() -> None:
-    await jianshu_pool.prepare()
-    await jpep_pool.prepare()
-    await jtools_pool.prepare()
-
-    await DebugProjectRecord.init()
-    await LotteryWinRecord.init()
-    await TechStack.init()
-    await User.init()
-    await Tool.init()
-
-
 if __name__ == "__main__":
     asyncio_run(init_db())
     logger.debug("初始化数据库成功")

backend/models/__init__.py

@@ -0,0 +1,7 @@
+from utils.db import jianshu_pool, jpep_pool, jtools_pool
+
+
+async def init_db() -> None:
+    await jianshu_pool.prepare()
+    await jpep_pool.prepare()
+    await jtools_pool.prepare()

backend/models/debug_project_record.py

@@ -17,24 +17,6 @@ class DebugProjectRecord(Table, frozen=True):
     user_slug: NonEmptyStr
     reward: PositiveInt
 
-    @classmethod
-    async def _create_table(cls) -> None:
-        async with jtools_pool.get_conn() as conn:
-            await conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS debug_project_records (
-                    id SMALLSERIAL CONSTRAINT pk_debug_project_records_id PRIMARY KEY,
-                    date DATE NOT NULL,
-                    type TEXT NOT NULL,
-                    module TEXT NOT NULL,
-                    description TEXT NOT NULL,
-                    user_name TEXT NOT NULL,
-                    user_slug VARCHAR(12) NOT NULL,
-                    reward SMALLINT NOT NULL
-                );
-                """
-            )
-
     @classmethod
     async def iter(cls) -> AsyncGenerator["DebugProjectRecord", None]:
         async with jtools_pool.get_conn() as conn:

backend/models/jianshu/user.py

@@ -80,7 +80,11 @@ async def get_by_name(cls, name: str) -> Optional["User"]:
     async def iter_similar_names(cls, name: str, limit: int) -> AsyncGenerator[str]:
         async with jianshu_pool.get_conn() as conn:
             cursor = await conn.execute(
-                "SELECT name FROM users WHERE name LIKE %s LIMIT %s;",
+                "SELECT users.name, COUNT(*) FROM users "
+                "JOIN article_earning_ranking_records ON users.slug = "
+                "article_earning_ranking_records.author_slug "
+                "WHERE users.name LIKE %s GROUP BY users.name "
+                "ORDER BY count DESC LIMIT %s;",
                 (f"{name}%", limit),
             )
 

backend/models/tech_stack.py

@@ -2,7 +2,7 @@
 from enum import Enum
 from typing import Optional
 
-from sshared.postgres import Table, create_enum
+from sshared.postgres import Table
 from sshared.strict_struct import NonEmptyStr
 
 from utils.db import jtools_pool
@@ -27,32 +27,6 @@ class TechStack(Table, frozen=True):
     description: NonEmptyStr
     url: NonEmptyStr
 
-    @classmethod
-    async def _create_enum(cls) -> None:
-        async with jtools_pool.get_conn() as conn:
-            await create_enum(
-                conn=conn, name="enum_tech_stacks_type", enum_class=TypeEnum
-            )
-            await create_enum(
-                conn=conn, name="enum_tech_stacks_scope", enum_class=ScopeEnum
-            )
-
-    @classmethod
-    async def _create_table(cls) -> None:
-        async with jtools_pool.get_conn() as conn:
-            await conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS tech_stacks (
-                    name TEXT NOT NULL CONSTRAINT pk_tech_stacks_name PRIMARY KEY,
-                    type enum_tech_stacks_type NOT NULL,
-                    scope enum_tech_stacks_scope NOT NULL,
-                    is_self_developed BOOLEAN NOT NULL,
-                    description TEXT NOT NULL,
-                    url TEXT NOT NULL
-                );
-                """
-            )
-
     @classmethod
     async def iter(
         cls, scope: Optional[ScopeEnum] = None

backend/models/tool.py

@@ -2,7 +2,7 @@
 from typing import Optional
 
 from psycopg.types.json import Jsonb
-from sshared.postgres import Table, create_enum
+from sshared.postgres import Table
 from sshared.strict_struct import NonEmptyStr
 
 from utils.db import jtools_pool
@@ -26,32 +26,6 @@ class Tool(Table, frozen=True):
     data_count_table: Optional[NonEmptyStr]
     data_source: Optional[dict[str, str]]
 
-    @classmethod
-    async def _create_enum(cls) -> None:
-        async with jtools_pool.get_conn() as conn:
-            await create_enum(
-                conn=conn, name="enum_tools_status", enum_class=StatusEnum
-            )
-
-    @classmethod
-    async def _create_table(cls) -> None:
-        async with jtools_pool.get_conn() as conn:
-            await conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS tools (
-                    slug TEXT CONSTRAINT pk_tools_slug PRIMARY KEY,
-                    status enum_tools_status NOT NULL,
-                    status_description TEXT,
-                    data_update_freq TEXT NOT NULL,
-                    last_update_time_table TEXT,
-                    last_update_time_order_by TEXT,
-                    last_update_time_target_field TEXT,
-                    data_count_table TEXT,
-                    data_source JSONB
-                );
-                """
-            )
-
     @classmethod
     async def init(cls) -> None:
         await super().init()

backend/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "jtools"
-version = "3.18.0"
+version = "3.19.0"
 description = "探索未知"
 license = {file = "LICENSE"}
 authors = [

backend/requirements-dev.txt

@@ -4,7 +4,7 @@ click==8.1.7
 colorama==0.4.6 ; platform_system == 'Windows'
 dnspython==2.7.0
 exceptiongroup==1.2.2 ; python_full_version < '3.11'
-faker==30.8.2
+faker==32.1.0
 h11==0.14.0
 h2==4.1.0
 hpack==4.0.0
@@ -37,7 +37,7 @@ six==1.16.0
 sniffio==1.3.1
 sshared==0.18.0
 sspeedup==0.25.1
-tomli==2.0.2
+tomli==2.1.0
 typing-extensions==4.12.2
 tzdata==2024.2 ; sys_platform == 'win32'
 uvicorn==0.32.0

backend/requirements.txt

@@ -4,7 +4,7 @@ click==8.1.7
 colorama==0.4.6 ; platform_system == 'Windows'
 dnspython==2.7.0
 exceptiongroup==1.2.2 ; python_full_version < '3.11'
-faker==30.8.2
+faker==32.1.0
 h11==0.14.0
 h2==4.1.0
 hpack==4.0.0
@@ -34,7 +34,7 @@ six==1.16.0
 sniffio==1.3.1
 sshared==0.18.0
 sspeedup==0.25.1
-tomli==2.0.2
+tomli==2.1.0
 typing-extensions==4.12.2
 tzdata==2024.2 ; sys_platform == 'win32'
 uvicorn==0.32.0

backend/sql/0.sql

@@ -0,0 +1,10 @@
+-- date: 2024-11-13
+-- description: 初始化
+
+CREATE ROLE jtools LOGIN PASSWORD 'jtools';
+
+CREATE DATABASE jtools WITH OWNER = jtools;
+
+GRANT CONNECT ON DATABASE jtools TO jtools;
+GRANT CONNECT ON DATABASE jianshu TO jtools;
+GRANT CONNECT ON DATABASE jpep TO jtools;

backend/sql/jianshu/0.sql

@@ -0,0 +1,14 @@
+-- date: 2024-11-13
+-- description: 初始化
+
+GRANT SELECT ON TABLE article_earning_ranking_records TO jtools;
+GRANT SELECT ON TABLE lottery_win_records TO jtools;
+GRANT SELECT ON TABLE users TO jtools;
+
+CREATE EXTENSION pg_trgm;
+
+CREATE INDEX CONCURRENTLY idx_article_earning_ranking_records_ranking ON article_earning_ranking_records (ranking);
+CREATE INDEX CONCURRENTLY idx_lottery_win_records_time ON lottery_win_records (time);
+CREATE INDEX CONCURRENTLY idx_lottery_win_records_user_slug ON lottery_win_records (user_slug);
+CREATE INDEX CONCURRENTLY idx_lottery_win_records_award_name ON lottery_win_records (award_name);
+CREATE INDEX CONCURRENTLY idx_users_name ON users USING gin (name gin_trgm_ops);

backend/sql/jianshu/1.sql

@@ -0,0 +1,4 @@
+-- date: 2024-11-14
+-- description: 添加 article_earning_ranking_records author_slug 索引
+
+CREATE INDEX CONCURRENTLY idx_article_earning_ranking_records_author_slug ON article_earning_ranking_records (author_slug);

backend/sql/jpep/0.sql

@@ -0,0 +1,5 @@
+-- date: 2024-11-13
+-- description: 初始化
+
+GRANT SELECT ON TABLE ftn_macket_records TO jtools;
+GRANT SELECT ON TABLE ftn_orders TO jtools;

backend/sql/jtools/0.sql

@@ -0,0 +1,6 @@
+-- date: 2024-11-13
+-- description: 初始化
+
+GRANT SELECT ON TABLE debug_project_records TO jtools;
+GRANT SELECT ON TABLE tech_stacks TO jtools;
+GRANT SELECT, INSERT ON TABLE tools TO jtools;

backend/sql/jtools/debug_project_records/0.sql

@@ -0,0 +1,13 @@
+-- date: 2024-11-13
+-- description: 初始化
+
+CREATE TABLE debug_project_records (
+    id SMALLSERIAL CONSTRAINT pk_debug_project_records_id PRIMARY KEY,
+    date DATE NOT NULL,
+    type TEXT NOT NULL,
+    module TEXT NOT NULL,
+    description TEXT NOT NULL,
+    user_name TEXT NOT NULL,
+    user_slug VARCHAR(12) NOT NULL,
+    reward SMALLINT NOT NULL
+);

backend/sql/jtools/tech_stacks/0.sql

@@ -0,0 +1,14 @@
+-- date: 2024-11-13
+-- description: 初始化
+
+CREATE TYPE enum_tech_stacks_type AS ENUM ('LIBRARY', 'EXTERNAL_SERVICE');
+CREATE TYPE enum_tech_stacks_scope AS ENUM ('FRONTEND', 'BACKEND', 'TOOLCHAIN');
+
+CREATE TABLE tech_stacks (
+    name TEXT NOT NULL CONSTRAINT pk_tech_stacks_name PRIMARY KEY,
+    type enum_tech_stacks_type NOT NULL,
+    scope enum_tech_stacks_scope NOT NULL,
+    is_self_developed BOOLEAN NOT NULL,
+    description TEXT NOT NULL,
+    url TEXT NOT NULL
+);

backend/sql/jtools/tools/0.sql

@@ -0,0 +1,16 @@
+-- date: 2024-11-13
+-- description: 初始化
+
+CREATE TYPE enum_tools_status AS ENUM ('NORMAL', 'DOWNGRADED', 'UNAVAILABLE');
+
+CREATE TABLE tools (
+    slug TEXT CONSTRAINT pk_tools_slug PRIMARY KEY,
+    status enum_tools_status NOT NULL,
+    status_description TEXT,
+    data_update_freq TEXT NOT NULL,
+    last_update_time_table TEXT,
+    last_update_time_order_by TEXT,
+    last_update_time_target_field TEXT,
+    data_count_table TEXT,
+    data_source JSONB
+);