zhaoxi/KiloStar
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 工具系统迁移 + 重型插件骨架 + 前端交互增强

Browse Source 
- 工具系统从 kilostar/plugin/tool_plugin/ 迁移到 data/toolset/（manifest.json 声明式）
- 新增 plugin_runtime 模块：BaseOrganization / GlobalPluginManager / loader / tool_bridge
- 新增 org_task + org_task_event 表及 DAO（alembic 0009）
- 新增 /api/v1/plugin 路由（submit/status/stream/install/reload）
- 新增 data/plugin/example_dept 示例重型插件
- regulatory_node 支持聊天历史上下文注入
- send_file 改为 artifact 存盘 + SSE 推送下载链接
- 前端 WorkflowFileCard 组件 + ToolSettings README 渲染
- utils 整理：合并 access/role_check、standalone_proxy→ray_compat、删除废弃模块
- 项目结构文档移至 docs/STRUCTURE.md 并详细展开

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
zhaoxi
2026-06-17 05:20:00 +00:00
parent 9b73ae4db4
commit 6d658b4f4d
74 changed files with 2591 additions and 1308 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kilostar/core/global_state_machine/global_state_machine.py
+52 -41
View File
@@ -13,7 +13,7 @@
# limitations under the License.
from typing import Any, Dict, List, Optional, Tuple
from kilostar.utils.standalone_proxy import actor_class, _STANDALONE
from kilostar.utils.ray_compat import actor_class, _STANDALONE
if not _STANDALONE:
import ray
@@ -77,47 +77,40 @@ class GlobalStateMachine:
# 启动期一次性发布 v1 快照，让等待中的读端立刻可用
self._publish_snapshot()
_SYSTEM_TOOLSETS = [
{
"toolset_id": "system_basic",
"name": "系统基础工具集",
"description": "文件读写、搜索、代码执行等基础能力",
"tools": ["file_reader", "write_file", "edit_file", "search_file", "python_executor", "shell_executor"],
"is_system": True,
"category": "system_basic",
},
{
"toolset_id": "system_chat",
"name": "系统对话工具集",
"description": "对话场景专用工具（发送文件等）",
"tools": ["send_file"],
"is_system": True,
"category": "system_chat",
},
{
"toolset_id": "system_workflow",
"name": "系统工作流工具集",
"description": "工作流场景专用工具（审批、发送文件等）",
"tools": ["approval", "send_file"],
"is_system": True,
"category": "system_workflow",
},
]
async def _seed_system_toolsets(self):
"""若 DB 中缺少系统预置工具集则自动补种。"""
for seed in self._SYSTEM_TOOLSETS:
if seed["toolset_id"] not in self._custom_toolsets:
await self.postgres_database.upsert_custom_toolset.remote(
toolset_id=seed["toolset_id"],
name=seed["name"],
tools=seed["tools"],
description=seed["description"],
owner_id=None,
is_system=True,
category=seed["category"],
)
self._custom_toolsets[seed["toolset_id"]] = seed
"""把磁盘上每个 toolset 包同步成一个 system custom_toolset 记录。
toolset 包就是插件单元——目录结构 ``data/toolset/<name>/`` 即代表一个工具集。
启动时把每个包"投影"成一条 ``is_system=True`` 的 custom_toolset，
前端工具插件界面看到的卡片就是这些包；将来安装第三方插件 = 把目录扔进去。
旧版本写死过 ``system_basic`` / ``system_chat`` / ``system_workflow`` 这种
逻辑分组，这里会一并清理掉，避免遗留脏数据。
"""
packages = self._global_tool_manager.toolset_packages
wanted_ids = {f"system::{name}" for name in packages.keys()}
# 清理 stale 系统 toolset（包括旧版硬编码的 system_basic/system_chat/...）
for tid, ts in list(self._custom_toolsets.items()):
if not ts.get("is_system"):
continue
if tid in wanted_ids:
continue
await self.postgres_database.delete_custom_toolset.remote(tid)
self._custom_toolsets.pop(tid, None)
for name, pkg in packages.items():
tid = f"system::{name}"
saved = await self.postgres_database.upsert_custom_toolset.remote(
toolset_id=tid,
name=pkg.get("display_name") or name,
tools=list(pkg.get("tools", [])),
description=pkg.get("description") or None,
owner_id=None,
is_system=True,
category=name,
)
self._custom_toolsets[tid] = saved
# ─── Snapshot 发布（Object Store 读路径） ────────────────────
@@ -254,6 +247,24 @@ class GlobalStateMachine:
"""仅返回 retrieval 工具集（system_node 专用，不包含 generation 工具）。"""
return self._global_tool_manager.get_retrieval_toolsets_for_scope(scope)
def list_toolset_packages(self) -> List[Dict[str, Any]]:
"""列出所有磁盘工具包（前端"工具插件"页面卡片即由此渲染）。"""
return [
{k: v for k, v in pkg.items() if k != "readme_path"}
for pkg in self._global_tool_manager.toolset_packages.values()
]
def get_toolset_package_readme(self, name: str) -> Optional[str]:
"""读取指定工具包的 README.md 内容；不存在返回 None。"""
pkg = self._global_tool_manager.toolset_packages.get(name)
if not pkg or not pkg.get("readme_path"):
return None
try:
with open(pkg["readme_path"], "r", encoding="utf-8") as f:
return f.read()
except Exception:
return None
# ─── MCP Server Registry ───────────────────────────────────
async def add_mcp_server(self, server_id: str, config: Dict[str, Any]) -> bool:
kilostar/core/global_state_machine/gsm_snapshot.py
+2 -2
View File
@@ -33,7 +33,7 @@ import asyncio
from dataclasses import dataclass, field
from typing import Any, Callable, Dict, List, Optional, Tuple
from kilostar.utils.standalone_proxy import _STANDALONE
from kilostar.utils.ray_compat import _STANDALONE
if not _STANDALONE:
import ray
@@ -63,7 +63,7 @@ class GSMSnapshot:
tool_metadata: Dict[str, Dict[str, Any]] = field(default_factory=dict)
tool_funcs: Dict[str, Callable[..., Any]] = field(default_factory=dict)
third_party_funcs: Dict[str, Callable[..., Any]] = field(default_factory=dict)
tool_mapper: Dict[str, Dict[str, type]] = field(default_factory=dict)
tool_mapper: Dict[str, Dict[str, Callable[..., Any]]] = field(default_factory=dict)
# ``{scope: [tool_name, ...]}``：系统工具按 scope 维护的工具名清单。
# 客户端按名字 + ``tool_funcs`` 在自己进程里重建 FunctionToolset，
# 避开把不可序列化/版本耦合的 toolset 实例塞进快照的坑。
kilostar/core/global_state_machine/skill_manager.py
+6 -17
View File
@@ -17,9 +17,11 @@ from collections import defaultdict
import pathlib
import json
from kilostar.utils.settings import get_plugin_dir
class GlobalSkillManager:
"""Skill 注册表：从 ``kilostar/plugin/skill/<name>/skill.json`` 启动期一次性扫描加载。"""
"""Skill 注册表：从 ``data/plugin/skill/<name>/skill.json`` 启动期一次性扫描加载。"""
skill_mapper = Dict[str, Tuple[str]]
"""skill的存储表"""
@@ -27,23 +29,16 @@ class GlobalSkillManager:
def __init__(self):
self.skill_mapper = defaultdict(tuple)
import os
skill_plugin_dir = pathlib.Path(
os.path.abspath(
os.path.join(os.path.dirname(__file__), "..", "..", "plugin", "skill")
)
)
skill_plugin_dir = get_plugin_dir() / "skill"
if not skill_plugin_dir.exists() or not skill_plugin_dir.is_dir():
return
for item in skill_plugin_dir.iterdir():
if item.is_dir() and not item.name.startswith((".", "__")):
json_path = item / "skill.json" # 拼接文件路径
json_path = item / "skill.json"
if json_path.exists():
try:
with open(json_path, "r", encoding="utf-8") as f:
skill = json.load(f)
# 提取并映射
name = skill.get("name")
if name:
self.skill_mapper[name] = (
@@ -55,13 +50,7 @@ class GlobalSkillManager:
def add_skill(self, skill_name: str) -> None:
"""Add a skill to the manager by reading its skill.json from the path"""
import os
skill_plugin_dir = pathlib.Path(
os.path.abspath(
os.path.join(os.path.dirname(__file__), "..", "..", "plugin", "skill")
)
)
skill_plugin_dir = get_plugin_dir() / "skill"
item = skill_plugin_dir / skill_name
if item.is_dir() and not item.name.startswith((".", "__")):
json_path = item / "skill.json"
kilostar/core/global_state_machine/tool_manager.py
+122 -66
View File
@@ -1,27 +1,39 @@
import pathlib
import importlib
import inspect
import json
import importlib.util
import sys
import types
from collections import defaultdict
from typing import Any, Callable, Dict, List, Type
from typing import Any, Callable, Dict, List
from kilostar.plugin.tool_plugin.base_tool import BaseToolData
from kilostar.utils.settings import get_toolset_dir
from kilostar.utils.logger import get_logger
logger = get_logger("tool_manager")
_SYSTEM_BUCKET = "system"
def _bootstrap_toolset_modules():
"""在 sys.modules 中注册 toolset 的虚拟包层级，使 toolset 内部的相对 import 能正常工作。"""
toolset_dir = get_toolset_dir()
for pkg_name, pkg_path in [
("data", toolset_dir.parent),
("data.toolset", toolset_dir),
]:
if pkg_name not in sys.modules:
mod = types.ModuleType(pkg_name)
mod.__path__ = [str(pkg_path)]
mod.__package__ = pkg_name
sys.modules[pkg_name] = mod
class GlobalToolManager:
"""工具注册表：扫描 ``kilostar/plugin/tool_plugin/`` 下所有 BaseToolData 子类，
按 ``action_scope`` 打包成 ``FunctionToolset``。
"""工具注册表：扫描 ``data/toolset/`` 下所有 toolset 的 manifest.json，
按 ``action_scope`` / ``is_system`` / ``category`` 分桶。
三类 toolset：
- **system**：``is_system=True`` 的工具，按 scope 分组
- **custom**：用户自定义工具组（由 ``rebuild_custom_toolsets`` 动态构建）
- **mcp**：由 ``mcp_helper`` 独立管理，不经过本类
``category="mcp"`` 的工具不会被本类管理。
"""
tool_metadata: Dict[str, Dict[str, Any]]
@@ -30,7 +42,7 @@ class GlobalToolManager:
_custom_toolsets: Dict[str, Any]
_third_party_funcs: Dict[str, Callable]
_all_funcs: Dict[str, Callable]
tool_mapper: Dict[str, Dict[str, Type[BaseToolData]]]
toolset_packages: Dict[str, Dict[str, Any]]
def __init__(self) -> None:
self.tool_metadata = {}
@@ -41,75 +53,126 @@ class GlobalToolManager:
self._custom_toolsets = {}
self._third_party_funcs = {}
self._all_funcs = {}
self.tool_mapper = defaultdict(dict)
self.toolset_packages = {}
tool_plugin_dir = (
pathlib.Path(__file__).parent.parent.parent / "plugin" / "tool_plugin"
)
if not tool_plugin_dir.exists() or not tool_plugin_dir.is_dir():
_bootstrap_toolset_modules()
toolset_root = get_toolset_dir()
if not toolset_root.exists() or not toolset_root.is_dir():
return
for item in tool_plugin_dir.iterdir():
if not (item.is_dir() and not item.name.startswith("__")):
for toolset_dir in toolset_root.iterdir():
if not toolset_dir.is_dir() or toolset_dir.name.startswith("__"):
continue
plugin_name = item.name
module_name = f"kilostar.plugin.tool_plugin.{plugin_name}"
manifest_path = toolset_dir / "manifest.json"
if not manifest_path.exists():
continue
self._load_toolset(toolset_dir, manifest_path)
try:
module = importlib.import_module(module_name)
except Exception as e:
logger.warning(f"Failed to import tool plugin {plugin_name}: {e}")
self._build_system_toolsets()
self._build_retrieval_toolsets()
def _load_toolset(self, toolset_dir, manifest_path) -> None:
"""从一个 toolset 目录加载 manifest 并注册所有工具。"""
try:
with open(manifest_path, "r", encoding="utf-8") as f:
manifest = json.load(f)
except Exception as e:
logger.warning(f"Failed to read manifest {manifest_path}: {e}")
return
toolset_name = toolset_dir.name
# 注册 toolset 的虚拟包
pkg_name = f"data.toolset.{toolset_name}"
if pkg_name not in sys.modules:
mod = types.ModuleType(pkg_name)
mod.__path__ = [str(toolset_dir)]
mod.__package__ = pkg_name
sys.modules[pkg_name] = mod
registered_tools: List[str] = []
for tool_def in manifest.get("tools", []):
tool_name = tool_def.get("name")
tool_file = tool_def.get("file", f"{tool_name}.py")
if not tool_name:
continue
tool_data_cls = self._find_tool_data_class(module)
if tool_data_cls is None:
file_path = toolset_dir / tool_file
if not file_path.exists():
logger.warning(f"Tool file not found: {file_path}")
continue
tool_func = getattr(module, plugin_name, None)
if not callable(tool_func):
logger.warning(
f"Tool plugin '{plugin_name}' has no callable named "
f"'{plugin_name}' in its module; skipped."
)
tool_func = self._load_tool_func(toolset_name, tool_name, file_path)
if tool_func is None:
continue
action_scopes = (
tool_data_cls.model_fields.get("action_scope").default or []
)
is_system = bool(tool_data_cls.model_fields.get("is_system").default)
category_field = tool_data_cls.model_fields.get("category")
category = (category_field.default if category_field else "other") or "other"
toolset_field = tool_data_cls.model_fields.get("toolset")
toolset_name = (toolset_field.default if toolset_field else "other") or "other"
is_system = tool_def.get("is_system", True)
action_scopes = tool_def.get("action_scope", [])
category = tool_def.get("category", "other")
config_args = tool_def.get("config_args", {})
toolset_field = tool_def.get("toolset", "other")
self.tool_metadata[plugin_name] = {
"name": plugin_name,
self.tool_metadata[tool_name] = {
"name": tool_name,
"is_system": is_system,
"category": category,
"toolset": toolset_name,
"toolset": toolset_field,
"action_scope": list(action_scopes),
"config_args": config_args,
"source_toolset": toolset_name,
}
registered_tools.append(tool_name)
if category == "mcp":
continue
self._all_funcs[plugin_name] = tool_func
self._all_funcs[tool_name] = tool_func
scopes = [s for s in action_scopes if s] or ["default"]
if is_system:
for scope in scopes:
self._tool_funcs[scope][plugin_name] = tool_func
self.tool_mapper[scope][plugin_name] = tool_data_cls
if toolset_name == "retrieval":
self._retrieval_tool_funcs[scope][plugin_name] = tool_func
self._tool_funcs[scope][tool_name] = tool_func
if toolset_field == "retrieval":
self._retrieval_tool_funcs[scope][tool_name] = tool_func
else:
self._third_party_funcs[plugin_name] = tool_func
for scope in scopes:
self.tool_mapper[scope][plugin_name] = tool_data_cls
self._third_party_funcs[tool_name] = tool_func
self._build_system_toolsets()
self._build_retrieval_toolsets()
readme_path = toolset_dir / "README.md"
self.toolset_packages[toolset_name] = {
"name": toolset_name,
"display_name": manifest.get("name", toolset_name),
"version": manifest.get("version", ""),
"description": manifest.get("description", ""),
"tools": registered_tools,
"has_readme": readme_path.exists(),
"readme_path": str(readme_path) if readme_path.exists() else None,
}
def _load_tool_func(self, toolset_name: str, tool_name: str, file_path) -> Callable | None:
"""从文件加载工具函数。"""
module_name = f"data.toolset.{toolset_name}.{tool_name}"
try:
spec = importlib.util.spec_from_file_location(module_name, str(file_path))
if spec is None or spec.loader is None:
logger.warning(f"Failed to create spec for {module_name}")
return None
module = importlib.util.module_from_spec(spec)
sys.modules[module_name] = module
spec.loader.exec_module(module)
func = getattr(module, tool_name, None)
if not callable(func):
logger.warning(
f"Tool '{tool_name}' has no callable named '{tool_name}' in {file_path}"
)
return None
return func
except Exception as e:
logger.warning(f"Failed to import tool {tool_name}: {e}")
return None
def _build_system_toolsets(self) -> None:
FunctionToolset = self._import_function_toolset()
@@ -142,11 +205,7 @@ class GlobalToolManager:
logger.error(f"Failed to build retrieval toolset {scope}: {e}")
def rebuild_custom_toolsets(self, custom_defs: Dict[str, Dict[str, Any]]) -> None:
"""根据 DB 中的 toolset 定义重建 FunctionToolset。
系统 toolset（is_system=True）允许包含 system 工具，用户 toolset 只取得到 callable
的工具（理论上业务层已校验只包含第三方工具）。
"""
"""根据 DB 中的 toolset 定义重建 FunctionToolset。"""
FunctionToolset = self._import_function_toolset()
if FunctionToolset is None:
self._custom_toolsets = {}
@@ -178,15 +237,13 @@ class GlobalToolManager:
logger.warning("pydantic_ai.toolsets unavailable")
return None
@staticmethod
def _find_tool_data_class(module) -> Type[BaseToolData] | None:
for _, obj in inspect.getmembers(module, inspect.isclass):
if issubclass(obj, BaseToolData) and obj is not BaseToolData:
return obj
return None
# ─── Toolset accessors ───
@property
def tool_mapper(self) -> Dict[str, Dict[str, Callable]]:
"""scope → {tool_name: callable}，兼容 GSM 快照构建。"""
return dict(self._tool_funcs)
def get_system_toolset(self, scope: str) -> Any | None:
return self._system_toolsets.get(scope)
@@ -230,7 +287,6 @@ class GlobalToolManager:
def get_all_tools(self) -> List[Dict[str, Any]]:
return list(self.tool_metadata.values())
# 兼容旧接口
def get_non_system_tools(self) -> List[Dict[str, Any]]:
return self.get_third_party_tools()