xusijie commited on
Commit
5c11d93
·
1 Parent(s): e534277

add open storyline files

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitignore +74 -1
  2. .storyline/skills/create_profile_style_skill/SKILL.md +63 -0
  3. .storyline/skills/subtitle_imitation_skill/SKILL.md +46 -0
  4. Dockerfile +2 -1
  5. agent_fastapi.py +0 -0
  6. cli.py +457 -0
  7. config.toml +112 -0
  8. download.sh +4 -0
  9. prompts/tasks/filter_clips/zh/system.md +1 -0
  10. prompts/tasks/filter_clips/zh/user.md +12 -0
  11. prompts/tasks/generate_script/zh/system.md +99 -0
  12. prompts/tasks/generate_script/zh/user.md +17 -0
  13. prompts/tasks/generate_title/zh/system.md +19 -0
  14. prompts/tasks/generate_title/zh/user.md +7 -0
  15. prompts/tasks/group_clips/zh/system.md +125 -0
  16. prompts/tasks/group_clips/zh/user.md +22 -0
  17. prompts/tasks/instruction/zh/system.md +66 -0
  18. prompts/tasks/understand_clips/zh/system_detail.md +3 -0
  19. prompts/tasks/understand_clips/zh/system_overall.md +2 -0
  20. prompts/tasks/understand_clips/zh/user_detail.md +7 -0
  21. prompts/tasks/understand_clips/zh/user_overall.md +2 -0
  22. requirements.txt +12 -1
  23. run.sh +20 -0
  24. src/__init__.py +0 -0
  25. src/open_storyline/__init__.py +1 -0
  26. src/open_storyline/__main__.py +0 -0
  27. src/open_storyline/config.py +189 -0
  28. src/open_storyline/mcp/__init__.py +0 -0
  29. src/open_storyline/mcp/hooks/node_interceptors.py +299 -0
  30. src/open_storyline/mcp/register_tools.py +237 -0
  31. src/open_storyline/mcp/sampling_handler.py +424 -0
  32. src/open_storyline/mcp/sampling_requester.py +159 -0
  33. src/open_storyline/mcp/server.py +70 -0
  34. src/open_storyline/nodes/__init__.py +1 -0
  35. src/open_storyline/nodes/core_nodes/assets_index.py +170 -0
  36. src/open_storyline/nodes/core_nodes/base_node.py +243 -0
  37. src/open_storyline/nodes/core_nodes/filter_clips.py +149 -0
  38. src/open_storyline/nodes/core_nodes/generate_script.py +306 -0
  39. src/open_storyline/nodes/core_nodes/generate_voiceover.py +353 -0
  40. src/open_storyline/nodes/core_nodes/group_clips.py +190 -0
  41. src/open_storyline/nodes/core_nodes/plan_timeline.py +904 -0
  42. src/open_storyline/nodes/core_nodes/render_video.py +838 -0
  43. src/open_storyline/nodes/core_nodes/select_bgm.py +237 -0
  44. src/open_storyline/nodes/core_nodes/split_shots.py +778 -0
  45. src/open_storyline/nodes/core_nodes/understand_clips.py +189 -0
  46. src/open_storyline/nodes/node_manager.py +172 -0
  47. src/open_storyline/nodes/node_schema.py +283 -0
  48. src/open_storyline/nodes/node_summary.py +236 -0
  49. src/open_storyline/skills/skills_io.py +83 -0
  50. src/open_storyline/storage/__init__.py +0 -0
.gitignore CHANGED
@@ -1 +1,74 @@
1
- *.pyc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # === Python 生成文件 ===
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+
6
+ # C 扩展
7
+ *.so
8
+ *.pyd
9
+
10
+ # 虚拟环境 / Conda 环境目录
11
+ .venv/
12
+ venv/
13
+ env/
14
+ .env/
15
+ .conda/
16
+ .hypothesis/
17
+
18
+ # 构建 / 发布产物
19
+ build/
20
+ dist/
21
+ *.egg-info/
22
+ *.egg
23
+ pip-wheel-metadata/
24
+
25
+ # 单元测试 / 覆盖率 / 缓存
26
+ .pytest_cache/
27
+ .coverage
28
+ .coverage.*
29
+ htmlcov/
30
+ .tox/
31
+ .nox/
32
+ .mypy_cache/
33
+ .dmypy.json
34
+ .pyre/
35
+ .cache/
36
+
37
+ # IDE / 编辑器配置
38
+ .vscode/
39
+ .idea/
40
+ *.swp
41
+ *.swo
42
+ *.iml
43
+
44
+ # Jupyter
45
+ .ipynb_checkpoints/
46
+
47
+ # OS 级别垃圾文件
48
+ .DS_Store
49
+ Thumbs.db
50
+
51
+ # 日志 / 临时文件
52
+ *.log
53
+ logs/
54
+ tmp/
55
+ temp/
56
+ .server_cache/
57
+ .storyline/.server_cache/
58
+
59
+ # 本项目可能产生的大文件目录
60
+ outputs/
61
+ renders/
62
+ checkpoints/
63
+ models/
64
+ project/
65
+
66
+ # 环境/配置的敏感信息(你如果用 .env 管 secret)
67
+ .env.local
68
+ .env.*.local
69
+
70
+ data/**
71
+ !data/elements_v2/
72
+ !data/elements_v2/**
73
+ !data/prompts/
74
+ !data/prompts/**
.storyline/skills/create_profile_style_skill/SKILL.md ADDED
@@ -0,0 +1,63 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ name: create_style_profile
3
+ description: 【SKILL】分析当前剪辑逻辑与风格,总结并生成一个新的可复用 Skill 文件,存入剪辑技能库。
4
+ version: 1.0.0
5
+ author: User_Agent_Architect
6
+ tags: [meta-skill, workflow, writing, file-system]
7
+ ---
8
+
9
+ # 角色定义 (Role)
10
+ 你是一个专业的“剪辑风格架构师”。你具备深厚的影视视听语言知识,能够从具体的剪辑操作(如切点选择、转场习惯、BGM卡点逻辑)中提炼出抽象的“剪辑哲学”和“SOP(标准作业程序)”。
11
+
12
+ # 任务目标 (Objective)
13
+ 你的任务是观察或询问用户的剪辑偏好,将其转化为一个标准的 Agent Skill 文档(Markdown格式),并保存到 `.storyline/skills/` 目录下,以便让 Agent 在未来模仿这种风格。
14
+
15
+ # 执行流程 (Workflow)
16
+
17
+ ## 第一步:风格分析与萃取 (Analysis & Extraction)
18
+ 1. **获取上下文**:获取当前正在编辑的 Timeline 数据,或者请求用户描述其剪辑习惯。
19
+ 2. **维度拆解**:你需要从以下维度总结风格:
20
+ * **剪辑节奏 (Pacing)**:是快节奏的跳剪(Jump Cut),还是长镜头的舒缓叙事?
21
+ * **叙事逻辑 (Storytelling)**:是线性叙事、倒叙,还是基于音乐情绪的蒙太奇?
22
+ * **视听语言 (Audio-Visual)**:音效(SFX)的使用密度、字幕样式偏好、调色风格(LUTs)。
23
+ * **特殊偏好**:例如“总是删除静音片段”或“每5秒插入一个B-Roll”。
24
+
25
+ ## 第二步:交互与命名 (Interaction & Naming)
26
+ 1. **总结确认**:向用户展示你总结的 3-5 个核心风格点,询问是否准确。
27
+ 2. **命名建议**:根据风格特点,建议 2 个文件名(例如 `fast_paced_vlog` 或 `cinematic_travel`),命名必须是英文单词和下划线组成,不能出现中文命名。
28
+ 3. **获取输入**:
29
+ * 询问用户:“是否认可这个总结?”
30
+ * 询问用户:“你想将这个新技能命名为什么?(按 Enter 使用建议名称:[建议名称])”
31
+
32
+ ## 第三步:生成新 Skill 内容 (Drafting)
33
+ 根据确认的风格,生成新 Skill 的 Markdown 内容。内容必须包含标准头部和 Prompt 指令。
34
+ * *Template*(新 Skill 的模板结构):
35
+ ```markdown
36
+ ---
37
+ name: {{用户定义的名称}}
38
+ description: 【SKILL】基于 {{日期}} 总结的 {{风格关键词}} 剪辑风格
39
+ version: 基于对话进行版本管理
40
+ author: 用户
41
+ tags: [相关的tag-list]
42
+ ---
43
+ # 剪辑指令
44
+ 当执行剪辑任务时,请严格遵守以下逻辑:
45
+ 1. **整体风格原则**:{{分析出的节奏逻辑}}
46
+ 2. **音频处理规范**:{{分析出的音频处理(视频原声/配音/背景音乐)筛选逻辑}}
47
+ 3. **视觉元素规范**:{{分析出的视觉元素(字体花字/转场/滤镜/特效等)使用逻辑}}
48
+ 4. **剪辑节奏控制**:{{分析出的剪辑节奏(音乐卡点/短切片/长切片)使用逻辑}}
49
+ 5. **工具调用规范**:{{分析出的推荐使用的工具以及推荐的传入参数}}
50
+ ```
51
+
52
+ ## 第四步:入库与更新 (Commit & Update)
53
+ 1. **展示预览**:将生成的内容以代码块形式展示给用户。
54
+ 2. **执行写入**:
55
+ * 用户确认后,调用文件写入工具`write_skills`。
56
+ * **目标路径**:`.storyline/skills/{{文件名}}/SKILL.md`,传入文件名即可,工具会自动完成写入。
57
+ 3. **系统更新**:提示用户“新技能已入库,请刷新 Agent 工具列表以加载。”
58
+
59
+ # 约束条件 (Constraints)
60
+ * **格式规范**:生成的新 Skill 必须符合 markdown 标准,且包含元数据(Metadata)。
61
+ * **路径安全**:只能写入 `.storyline/skills/` 目录,禁止覆盖系统核心文件。
62
+ * **可读性**:在与用户交互时,不要直接扔出一大段代码,先用自然语言确认逻辑。
63
+ * **版本管理**:当用户进行修改时,更改版本号,并重新调用`write_skills`工具做覆盖;
.storyline/skills/subtitle_imitation_skill/SKILL.md ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ name: script_style_mimic
3
+ description: 【SKILL】基于用户提供的参考文案样本,对视频素材内容进行深度文风仿写,生成风格化脚本。
4
+ version: 1.0.0
5
+ author: User_Agent_Architect
6
+ tags: [writing, style-transfer, video-production, creative]
7
+ ---
8
+
9
+ # 角色定义 (Role)
10
+ 你是一位“文风迁移大师”兼“金牌视频脚本撰写人”。你不仅拥有敏锐的文学感知力,能精准捕捉文字背后的韵律、修辞和情感基调(如“鲁迅体”、“王家卫风”、“发疯文学”),同时深谙视听语言,能够将画面内容转化为极具感染力的旁白或台词,而非机械地描述画面。
11
+
12
+ # 任务目标 (Objective)
13
+ 你的核心任务是接收用户的“仿写指令”和“参考文案”,调用历史记忆读取视频素材理解结果(`understand_clips`),生成一份既具备参考文案神韵,又严格基于视频事实的拍摄脚本。
14
+
15
+ # 执行流程 (Workflow)
16
+
17
+ ## 第一步:输入校验与意图确认 (Input Validation)
18
+ 1. **检查输入参数**:检查用户是否提供了用于模仿的 `style_reference_text`(仿写样本)。
19
+ 2. **缺失处理**:
20
+ * **如果用户未提供样本**(仅说“帮我仿写一下”):必须立即中止后续流程,并输出回复引导用户:“为了能精准模仿您想要的文风,请提供一段您希望我模仿的文案示例(例如直接粘贴一段文字,或提供某位博主的典型语录)。”
21
+ * **如果用户已提供样本**:进入第二步。
22
+
23
+ ## 第二步:获取素材与分析 (Context & Analysis)
24
+ 1. **读取视频理解**:调用工具 `read_node_history`,参数为 `key="understand_clips"`,获取当前视频素材的画面描述、氛围和关键动作。
25
+ 2. **风格解构**:在思维链(Chain of Thought)中快速分析用户提供的 `style_reference_text`:
26
+ * **句式特征**:是短句堆叠,还是长难句?
27
+ * **修辞习惯**:是否喜欢用比喻、反讽、排比?
28
+ * **情感基调**:是治愈、焦虑、犀利还是幽默?
29
+
30
+ ## 第三步:风格化创作 (Creative Generation)
31
+ 基于素材内容(Content)和分析出的风格(Style),执行脚本撰写。需严格遵守以下创作原则:
32
+ 1. **拒绝“看图说话” (No See-Say)**:
33
+ * ❌ 错误示范:“画面里有一只猫在睡觉,阳光照在它身上。”
34
+ * ✅ 正确示范(如文艺风):“午后的阳光是免费的,但偷得浮生半日闲的勇气却是昂贵的。它在做梦,而我在看它。”
35
+ 2. **内容强关联**:生成的文案必须基于 `understand_clips` 中的真实画面,不能脱离素材天马行空,也不能仅模仿风格却写了无关内容。
36
+ 3. **生动连贯**:脚本必须有起承转合,不仅是句子的拼凑,更是一个完整的小故事或情绪流。
37
+
38
+ ## 第四步:格式化输出 (Formatting)
39
+ 1. **构建数据结构**:将生成的脚本整理为符合工具 `generate_script` 输入要求的格式。
40
+ 2. **输出结果**:直接输出用于传递给下游工具的数据结构,并以结构化 Markdown 格式展示给用户确认。
41
+
42
+ # 约束条件 (Constraints)
43
+ * **素材依赖**:必须调用 `read_node_history` 获取素材,严禁在不知道视频内容的情况下瞎编脚本。
44
+ * **风格一致性**:生成的文案必须让熟悉该风格的人一眼就能识别出“味道”。
45
+ * **拒绝机械描述**:严禁出现“视频显示”、“镜头切到”等说明书式语言,除非参考风格本身就是说明书风格。
46
+ * **工具对接**:输出内容必须适配 `generate_script` 的字段定义,确保下游渲染环节无缝衔接。
Dockerfile CHANGED
@@ -10,10 +10,11 @@ COPY app.py .
10
  COPY static/ ./static/
11
 
12
  # 安装依赖
 
13
  RUN pip install --no-cache-dir -r requirements.txt
14
 
15
  # 暴露 HF Space 默认端口
16
  EXPOSE 7860
17
 
18
  # 启动命令
19
- CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
 
10
  COPY static/ ./static/
11
 
12
  # 安装依赖
13
+ RUN apt-get update && apt-get install -y ffmpeg
14
  RUN pip install --no-cache-dir -r requirements.txt
15
 
16
  # 暴露 HF Space 默认端口
17
  EXPOSE 7860
18
 
19
  # 启动命令
20
+ CMD ["bash", "run.sh"]
agent_fastapi.py ADDED
The diff for this file is too large to render. See raw diff
 
cli.py ADDED
@@ -0,0 +1,457 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import asyncio
2
+ import time
3
+ import uuid
4
+ import os,sys
5
+ import json
6
+
7
+ import contextvars
8
+ from typing import Callable, Optional, Union, Any, List
9
+ from dataclasses import dataclass
10
+ from datetime import timedelta
11
+
12
+ from langchain.agents import create_agent
13
+ from langchain.agents.middleware import wrap_tool_call, wrap_model_call
14
+ from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage, ToolMessage
15
+ from langchain_mcp_adapters.client import MultiServerMCPClient
16
+ from langchain_mcp_adapters.interceptors import MCPToolCallRequest, MCPToolCallResult
17
+ from langchain_mcp_adapters.callbacks import Callbacks, CallbackContext
18
+ from langchain_openai import ChatOpenAI
19
+ from langgraph.types import Command
20
+
21
+ # 把 src 目录加入 Python 的模块搜索路径
22
+ ROOT_DIR = os.path.dirname(__file__)
23
+ SRC_DIR = os.path.join(ROOT_DIR, "src")
24
+
25
+ if SRC_DIR not in sys.path:
26
+ sys.path.insert(0, SRC_DIR)
27
+
28
+ from open_storyline.skills.skills_io import load_skills
29
+ from open_storyline.utils.prompts import get_prompt
30
+ from open_storyline.mcp.sampling_handler import make_sampling_callback
31
+ from open_storyline.config import load_settings, default_config_path
32
+ from open_storyline.config import Settings
33
+ from open_storyline.storage.agent_memory import ArtifactStore
34
+ from open_storyline.nodes.node_manager import NodeManager
35
+ from open_storyline.mcp.hooks.node_interceptors import ToolInterceptor
36
+
37
+
38
+ @dataclass
39
+ class ClientContext:
40
+ session_id: str
41
+ assets_dir: str
42
+ bgm_dir: str
43
+ outputs_dir: str
44
+ node_manager: NodeManager
45
+ chat_model_key: str # 当前聊天的模型
46
+ tts_config: Optional[dict] = None # 运行时 TTS 配置
47
+
48
+
49
+ # GUI 日志输出通道
50
+ _MCP_LOG_SINK = contextvars.ContextVar("mcp_log_sink", default=None)
51
+ _MCP_ACTIVE_TOOL_CALL_ID = contextvars.ContextVar("mcp_active_tool_call_id", default=None)
52
+ def set_mcp_log_sink(sink: Optional[Callable[[dict], None]]):
53
+ return _MCP_LOG_SINK.set(sink)
54
+
55
+ def reset_mcp_log_sink(token):
56
+ _MCP_LOG_SINK.reset(token)
57
+
58
+ # ------ key 脱敏 + 注入 tts 凭据,先在这里放着,等改完框架看看放到哪里合适---
59
+ CUSTOM_MODEL_KEY = "__custom__"
60
+
61
+ _SENSITIVE_KEYS = {
62
+ "api_key",
63
+ "access_token",
64
+ "authorization",
65
+ "token",
66
+ "password",
67
+ "secret",
68
+ "x-api-key",
69
+ "apikey",
70
+ }
71
+
72
+ def _mask_secrets(obj: Any) -> Any:
73
+ """递归脱敏:避免 key/token 被打印到控制台、日志、tool trace、ToolMessage等各种地方"""
74
+ try:
75
+ if isinstance(obj, dict):
76
+ out = {}
77
+ for k, v in obj.items():
78
+ if str(k).lower() in _SENSITIVE_KEYS:
79
+ out[k] = "***"
80
+ else:
81
+ out[k] = _mask_secrets(v)
82
+ return out
83
+ if isinstance(obj, list):
84
+ return [_mask_secrets(x) for x in obj]
85
+ if isinstance(obj, tuple):
86
+ return tuple(_mask_secrets(x) for x in obj)
87
+ return obj
88
+ except Exception:
89
+ return "***"
90
+
91
+ async def inject_tts_config(request: MCPToolCallRequest, handler):
92
+ """
93
+ 拦截器:在调用 TTS 工具前注入 key。
94
+ """
95
+ try:
96
+ runtime = getattr(request, "runtime", None)
97
+ ctx = getattr(runtime, "context", None) if runtime else None
98
+ tts_cfg = getattr(ctx, "tts_config", None) if ctx else None
99
+
100
+ if not (isinstance(tts_cfg, dict) and isinstance(getattr(request, "args", None), dict)):
101
+ return await handler(request)
102
+
103
+ tool_name = str(getattr(request, "name", "") or "")
104
+ if "voiceover" not in tool_name:
105
+ return await handler(request)
106
+
107
+ vendor = str(tts_cfg.get("vendor") or "").lower().strip()
108
+ if vendor != "bytedance":
109
+ return await handler(request)
110
+
111
+ bd = tts_cfg.get("bytedance") or {}
112
+ if not isinstance(bd, dict):
113
+ return await handler(request)
114
+
115
+ uid = bd.get("uid")
116
+ appid = bd.get("appid")
117
+ access_token = bd.get("access_token")
118
+ if uid and appid and access_token:
119
+ request.args.setdefault("uid", uid)
120
+ request.args.setdefault("appid", appid)
121
+ request.args.setdefault("access_token", access_token)
122
+ except Exception:
123
+ pass
124
+
125
+ return await handler(request)
126
+
127
+ async def log_mcp_request(request: MCPToolCallRequest, handler):
128
+ sink = _MCP_LOG_SINK.get()
129
+
130
+ def emit_event(x: str | dict):
131
+ if sink:
132
+ sink(x)
133
+
134
+ runtime = request.runtime
135
+ context = runtime.context
136
+ meta_collector = context.node_manager
137
+
138
+ exclude = set(meta_collector.kind_to_node_ids.keys()) | {
139
+ "inputs", "artifacts_dir", "artifact_id", "blobs_dir", "meta_path",
140
+ "assets_dir", "bgm_dir", "outputs_dir", "debug_dir",
141
+ }
142
+
143
+ extracted_args = {}
144
+ for arg in (request.args.keys() if isinstance(request.args, dict) else []):
145
+ if arg not in exclude:
146
+ extracted_args[arg] = request.args[arg]
147
+
148
+ extracted_args = _mask_secrets(extracted_args)
149
+
150
+ tool_call_id = getattr(runtime, "tool_call_id", None)
151
+ if not tool_call_id:
152
+ tool_call_id = f"mcp_{uuid.uuid4().hex[:8]}"
153
+ if runtime is not None:
154
+ setattr(runtime, "tool_call_id", tool_call_id)
155
+
156
+ active_tok = _MCP_ACTIVE_TOOL_CALL_ID.set(tool_call_id)
157
+ try:
158
+ emit_event({
159
+ "type": "tool_start",
160
+ "tool_call_id": tool_call_id,
161
+ "server": request.server_name,
162
+ "name": request.name,
163
+ "args": extracted_args,
164
+ })
165
+ print(f"[MCP 工具开始] {request.server_name}.{request.name} args={extracted_args}\n")
166
+ out = await handler(request)
167
+ try:
168
+ isError = json.loads(out.content[0].text).get('isError', False)
169
+ except Exception:
170
+ isError = True
171
+ finally:
172
+ _MCP_ACTIVE_TOOL_CALL_ID.reset(active_tok)
173
+
174
+ if out.isError or isError:
175
+ print(f"[MCP 工具出错] result:{out.content}\n\n")
176
+ emit_event({
177
+ "type": "tool_end",
178
+ "tool_call_id": tool_call_id,
179
+ "server": request.server_name,
180
+ "name": request.name,
181
+ "is_error": True,
182
+ "summary": _mask_secrets(out.content),
183
+ })
184
+ else:
185
+ payload = json.loads(out.content[0].text)
186
+ summary = payload.get("summary")
187
+
188
+ print(f"[MCP 工具完成] result:{summary}\n\n")
189
+ emit_event({
190
+ "type": "tool_end",
191
+ "tool_call_id": tool_call_id,
192
+ "server": request.server_name,
193
+ "name": request.name,
194
+ "is_error": False,
195
+ "summary": _mask_secrets(summary),
196
+ })
197
+
198
+ return out
199
+
200
+ @wrap_tool_call
201
+ async def handle_tool_errors(request, handler):
202
+ try:
203
+ out = await handler(request)
204
+ if isinstance(out, Command):
205
+ return out.update.get('messages')[0]
206
+ elif isinstance(out, MCPToolCallResult) and not isinstance(out.content, str):
207
+ return ToolMessage(content=out.content[0].get("text", ""), tool_call_id=out.tool_call_id, name=out.name) # 解决deepseek-chat兼容性问题
208
+ return out
209
+ except Exception as e:
210
+ tc = request.tool_call
211
+ safe_args = _mask_secrets(tc.get("args") or {})
212
+ return ToolMessage(
213
+ content=(
214
+ "工具调用失败\n"
215
+ f"工具名:{tc.get('name')}\n"
216
+ f"工具参数:{safe_args}\n"
217
+ f"错误信息:{type(e).__name__}: {e}\n"
218
+ "如果是参数问题,请修正参数重新调用;如果是因为缺少前置依赖,请先调用前置节点;如果你认为是偶发错误,请尝试重新调用;如果你认为已经无法继续,请向用户解释原因。"
219
+ ),
220
+ tool_call_id=tc["id"],
221
+ )
222
+
223
+ async def on_progress(progress: float, total: float | None, message: str| None, context: CallbackContext):
224
+ sink = _MCP_LOG_SINK.get()
225
+ if sink:
226
+ sink({
227
+ "type": "tool_progress",
228
+ "tool_call_id": _MCP_ACTIVE_TOOL_CALL_ID.get(),
229
+ "server": context.server_name,
230
+ "name": context.tool_name,
231
+ "progress": progress,
232
+ "total": total,
233
+ "message": message,
234
+ })
235
+
236
+ async def build_agent(
237
+ cfg: Settings,
238
+ session_id: str,
239
+ store: ArtifactStore,
240
+ tool_interceptors=None,
241
+ *,
242
+ llm_override: Optional[dict] = None,
243
+ vlm_override: Optional[dict] = None,
244
+ ):
245
+ def _get(override: Optional[dict], key: str, default: Any) -> Any:
246
+ return (override.get(key) if isinstance(override, dict) and key in override else default)
247
+
248
+ def _norm_url(u: str) -> str:
249
+ u = (u or "").strip()
250
+ return u.rstrip("/") if u else u
251
+
252
+ # 1) LLM 优先用输入框里用户填写的,其次是 config.toml 里的
253
+ llm_model = _get(llm_override, "model", cfg.llm.model)
254
+ llm_base_url = _norm_url(_get(llm_override, "base_url", cfg.llm.base_url))
255
+ llm_api_key = _get(llm_override, "api_key", cfg.llm.api_key)
256
+ llm_timeout = _get(llm_override, "timeout", cfg.llm.timeout)
257
+ llm_temperature = _get(llm_override, "temperature", cfg.llm.temperature)
258
+ llm_max_retries = _get(llm_override, "max_retries", cfg.llm.max_retries)
259
+
260
+ llm = ChatOpenAI(
261
+ model=llm_model,
262
+ base_url=llm_base_url,
263
+ api_key=llm_api_key,
264
+ default_headers={
265
+ "api-key": llm_api_key,
266
+ "Content-Type": "application/json",
267
+ },
268
+ timeout=llm_timeout,
269
+ temperature=llm_temperature,
270
+ streaming=True,
271
+ max_retries=llm_max_retries,
272
+ )
273
+
274
+ # 2) VLM 优先级同上
275
+ vlm_model = _get(vlm_override, "model", cfg.vlm.model)
276
+ vlm_base_url = _norm_url(_get(vlm_override, "base_url", cfg.vlm.base_url))
277
+ vlm_api_key = _get(vlm_override, "api_key", cfg.vlm.api_key)
278
+ vlm_timeout = _get(vlm_override, "timeout", cfg.vlm.timeout)
279
+ vlm_temperature = _get(vlm_override, "temperature", cfg.vlm.temperature)
280
+ vlm_max_retries = _get(vlm_override, "max_retries", cfg.vlm.max_retries)
281
+
282
+ vlm = ChatOpenAI(
283
+ model=vlm_model,
284
+ base_url=vlm_base_url,
285
+ api_key=vlm_api_key,
286
+ default_headers={
287
+ "api-key": vlm_api_key,
288
+ "Content-Type": "application/json",
289
+ },
290
+ timeout=vlm_timeout,
291
+ temperature=vlm_temperature,
292
+ max_retries=vlm_max_retries,
293
+ )
294
+
295
+ sampling_callback = make_sampling_callback(llm, vlm)
296
+
297
+ # ---- 开发者模型 allowlist,只用于站内预置模型切换 ----
298
+ allowed = set(cfg.developer.chat_models or [])
299
+ llm_pool: dict[tuple[str, bool], ChatOpenAI] = {}
300
+
301
+ def _make_chat_llm(model_name: str, streaming: bool) -> ChatOpenAI:
302
+ model_config = (cfg.developer.chat_models_config.get(model_name) or {})
303
+ base_url = _norm_url(model_config.get("base_url") or "")
304
+ api_key = model_config.get("api_key")
305
+ return ChatOpenAI(
306
+ model=model_name,
307
+ base_url=base_url,
308
+ api_key=api_key,
309
+ default_headers={
310
+ "api-key": api_key,
311
+ "Content-Type": "application/json",
312
+ },
313
+ timeout=cfg.llm.timeout,
314
+ temperature=model_config.get("temperature", cfg.llm.temperature),
315
+ streaming=streaming,
316
+ )
317
+
318
+ def _get_llm(model_name: str, streaming: bool) -> ChatOpenAI:
319
+ hit = llm_pool.get((model_name, streaming))
320
+ if hit:
321
+ return hit
322
+ new_llm = _make_chat_llm(model_name, streaming=streaming)
323
+ llm_pool[(model_name, streaming)] = new_llm
324
+ return new_llm
325
+
326
+ @wrap_model_call
327
+ async def switch_chat_model(request, handler):
328
+ runtime = getattr(request, "runtime", None)
329
+ ctx = getattr(runtime, "context", None) if runtime else None
330
+ model_key = getattr(ctx, "chat_model_key", None) if ctx else None
331
+
332
+ model_key = model_key.strip() if isinstance(model_key, str) else ""
333
+ if not model_key:
334
+ return await handler(request)
335
+
336
+ if model_key == CUSTOM_MODEL_KEY:
337
+ return await handler(request)
338
+
339
+ if model_key not in allowed:
340
+ return await handler(request)
341
+
342
+ # 优先流式,失败再非流式
343
+ for _ in range(2):
344
+ try:
345
+ llm_stream = _get_llm(model_key, streaming=True)
346
+ if cfg.developer.print_context:
347
+ print(request)
348
+ return await handler(request.override(model=llm_stream))
349
+ except Exception as e1:
350
+ print(f"流式 llm 出错:{e1}")
351
+ await asyncio.sleep(2)
352
+
353
+ for _ in range(2):
354
+ try:
355
+ llm_nostream = _get_llm(model_key, streaming=False)
356
+ if cfg.developer.print_context:
357
+ print(request)
358
+ return await handler(request.override(model=llm_nostream))
359
+ except Exception as e2:
360
+ print(f"非流式 llm 出错:{e2}")
361
+ await asyncio.sleep(2)
362
+
363
+ fresh = _make_chat_llm(model_key, streaming=False)
364
+ llm_pool[(model_key, False)] = fresh
365
+ if cfg.developer.print_context:
366
+ print(request)
367
+ return await handler(request.override(model=fresh))
368
+
369
+ connections = {
370
+ cfg.local_mcp_server.server_name: {
371
+ "transport": cfg.local_mcp_server.server_transport,
372
+ "url": cfg.local_mcp_server.url,
373
+ "timeout": timedelta(seconds=cfg.local_mcp_server.timeout),
374
+ "headers": {"X-Storyline-Session-Id": session_id},
375
+ "session_kwargs": {"sampling_callback": sampling_callback},
376
+ },
377
+ }
378
+
379
+ client = MultiServerMCPClient(
380
+ connections=connections,
381
+ tool_interceptors=tool_interceptors or [log_mcp_request],
382
+ callbacks=Callbacks(on_progress=on_progress),
383
+ tool_name_prefix=True,
384
+ )
385
+
386
+ tools = await client.get_tools()
387
+ skills = await load_skills() # 加载skills
388
+ node_manager = NodeManager(tools)
389
+
390
+ # 4) 用 LangChain 的 agent runtime 接管“多轮工具调用 loop”
391
+ agent = create_agent(
392
+ model=llm,
393
+ tools=tools+skills,
394
+ middleware=[switch_chat_model, handle_tool_errors],
395
+ store=store,
396
+ context_schema=ClientContext,
397
+ )
398
+ return agent, node_manager
399
+
400
+ async def main():
401
+ session_id = f"run_{int(time.time())}_{uuid.uuid4().hex[:8]}"
402
+ cfg = load_settings(default_config_path())
403
+
404
+ artifact_store = ArtifactStore(cfg.project.outputs_dir, session_id=session_id)
405
+ agent, node_manager = await build_agent(cfg=cfg, session_id=session_id, store=artifact_store, tool_interceptors=[ToolInterceptor.inject_media_content_before, ToolInterceptor.save_media_content_after, log_mcp_request])
406
+
407
+ context = ClientContext(
408
+ session_id=session_id,
409
+ assets_dir=cfg.project.assets_dir,
410
+ bgm_dir=cfg.project.bgm_dir,
411
+ outputs_dir=cfg.project.outputs_dir,
412
+ node_manager=node_manager,
413
+ chat_model_key=cfg.llm.model,
414
+ )
415
+
416
+ # 用 messages 列表做最简单的短期记忆(替代手写 history_for_llm 字符串)
417
+ messages: List[BaseMessage] = [SystemMessage(content=get_prompt("instruction.system"))]
418
+
419
+ print("智能剪辑Agent(LangChain + MCP) v 0.2.0")
420
+ print("请描述你的剪辑需求,输入 /exit 退出")
421
+
422
+ while True:
423
+ try:
424
+ user_input = input("你:").strip()
425
+ except (EOFError, KeyboardInterrupt):
426
+ print("\n再见~")
427
+ break
428
+
429
+ if not user_input:
430
+ continue
431
+ if user_input in ("/exit", "/quit"):
432
+ print("\n再见~")
433
+ break
434
+
435
+ messages.append(HumanMessage(content=user_input))
436
+
437
+ print("正在调用LLM中,请耐心等待...")
438
+
439
+ # 这里一次 ainvoke,就会在内部完成:推理 -> 多次调用工具 -> 汇总 -> 最终回答
440
+ result = await agent.ainvoke(
441
+ {"messages": messages},
442
+ context=context
443
+ )
444
+
445
+ messages = result["messages"]
446
+
447
+ final_text = None
448
+ for m in reversed(messages):
449
+ if isinstance(m, AIMessage):
450
+ final_text = m.content
451
+ break
452
+
453
+ print(f"\n助手:{final_text or '(未生成最终答复)'}\n")
454
+
455
+
456
+ if __name__ == "__main__":
457
+ asyncio.run(main())
config.toml ADDED
@@ -0,0 +1,112 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ============= 开发者选项 ===============
2
+ [developer]
3
+ developer_mode = true
4
+ chat_models = ["deepseek-v3.2", "gemini-2.5-pro"]
5
+ print_context = false # 在拦截器打印模型拿到的全部上下文,会很长
6
+
7
+ # ============= 模型配置 for 站内体验网页 ===============
8
+ [developer.chat_models_config."deepseek-v3.2"]
9
+ base_url = ""
10
+ api_key = ""
11
+ temperature = 0.1
12
+
13
+ [developer.chat_models_config."gemini-2.5-pro"]
14
+ base_url = ""
15
+ api_key = ""
16
+ temperature = 0.1
17
+
18
+ [developer.chat_models_config."qwen3-vl-32b-instruct"]
19
+ base_url = ""
20
+ api_key = ""
21
+ timeout = 20.0
22
+ temperature = 0.1
23
+ max_retries = 2
24
+
25
+ # ============= 项目路径 ======================
26
+ [project]
27
+ assets_dir = "./resource/media"
28
+ bgm_dir = "./resource/bgms"
29
+ outputs_dir = "./outputs"
30
+
31
+ # ============= 模型配置 for user =============
32
+ [llm]
33
+ model = "deepseek-v3.2"
34
+ base_url = ""
35
+ api_key = ""
36
+ timeout = 30.0 # 单位:秒
37
+ temperature = 0.1
38
+ max_retries = 2
39
+
40
+ [vlm]
41
+ # model = "qwen3-vl-8b-instruct"
42
+ # base_url = ""
43
+ # api_key = ""
44
+ model = "gemini-2.5-pro"
45
+ base_url = ""
46
+ api_key = ""
47
+ timeout = 20.0 # 单位:秒
48
+ temperature = 0.1
49
+ max_retries = 2
50
+
51
+
52
+ # ============= MCP Server 相关 =============
53
+ [local_mcp_server]
54
+ server_name = "storyline"
55
+ server_cache_dir = ".storyline/.server_cache"
56
+ server_transport = "streamable-http" # server 和 host之间的传输方式
57
+ url_scheme = "http"
58
+ connect_host = "127.0.0.1" # 不要改动
59
+ port = 8001 # 如果端口冲突,可以随便用一个有空的端口
60
+ path = "/mcp" # 默认值,一般不用改
61
+
62
+ json_response = true # 建议用 True
63
+ stateless_http = false # 强烈建议用 False
64
+ timeout = 600
65
+ use_xhs_node_only = false
66
+
67
+ [xhs_mcp_server]
68
+ server_name = "xhs"
69
+ server_cache_dir = ".storyline/.server_cache"
70
+ server_transport = "streamable-http" # server 和 host之间的传输方式
71
+ url_scheme = "http"
72
+ connect_host = "127.0.0.1" # 上线后应该是在站内起 MCP server,但是现在开发阶段先在本地起
73
+ port = 8002 # 也可以随便用一个有空的端口
74
+ path = "/mcp"
75
+
76
+ json_response = true # 建议用 True
77
+ stateless_http = false # 强烈建议用 False
78
+
79
+
80
+ # ============= 镜头分割 =============
81
+ [split_shots]
82
+ transnet_weights = ".storyline/models/transnetv2-pytorch-weights.pth"
83
+ transnet_device = "cpu"
84
+
85
+ # ============= 视频视觉理解 =============
86
+ [understand_clips]
87
+ sample_fps = 2.0 # 每秒抽几帧
88
+ max_frames = 64 # 单clip抽帧上限兜底,避免长视频爆 token
89
+
90
+ # ============= 配音生成 ===================
91
+ [generate_voiceover]
92
+ base_url = "https://openspeech.bytedance.com/api/v1/tts"
93
+
94
+ # ============= BGM选择 ====================
95
+ # 主要是用于计算音乐特征的一些参数
96
+ [select_bgm]
97
+ sample_rate = 22050
98
+ hop_length = 2048 # 每次分析窗口向前跳多少个采样点,越小越精细(但更慢)
99
+ frame_length = 2048 # 计算信号的均方根RMS的窗口大小。越大越稳定,但对瞬态不敏感
100
+
101
+ # ============= 时间线组织 ====================
102
+ [plan_timeline]
103
+ beat_type_max = 1 # 使用多强的鼓点,例如4/4中,鼓点类似1,2,1,3,其中1是最强的,3最弱
104
+ title_dutation = 0 # 片头时长 (ms)
105
+ bgm_loop = true # 是否允许 bgm 循环
106
+ min_clip_duration = 500
107
+
108
+ estimate_text_min = 1500 # 在没有TTS的情况下,估计每段字幕至少上屏多久
109
+ estimate_text_char_per_sec = 6.0 # 在没有TTS的情况下,估计每秒展示几个字
110
+
111
+ image_default_duration = 3000 # 默认的图片播放时长
112
+ group_margin_over_voiceover = 1000 # 在一个group中,画面比配音多出现多久
download.sh ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ mkdir .storyline/models
2
+ wget "https://image-url-2-feature-1251524319.cos.ap-shanghai.myqcloud.com/zailin/models/storyline/open_storyline/models/transnetv2-pytorch-weights.pth" -O .storyline/models/transnetv2-pytorch-weights.pth
3
+ wget "https://image-url-2-feature-1251524319.cos.ap-shanghai.myqcloud.com/zailin/datasets/open_storyline/resource.zip" -O .storyline/resource.zip
4
+ unzip .storyline/resource.zip -d resource
prompts/tasks/filter_clips/zh/system.md ADDED
@@ -0,0 +1 @@
 
 
1
+ 你是一个短视频剪辑助手。你的任务是:根据“用户要求”判断每个 clip 是否保留。不要输出任何多余内容,只输出JSON。
prompts/tasks/filter_clips/zh/user.md ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 用户要求: {user_request}
2
+ 请根据用户要求,判断下面所有 clips 是否保留
3
+ {clip_captions}
4
+ 输出格式如下:
5
+ ```json
6
+ {{
7
+ "results": [
8
+ {{"clip_id": "clip_0001", "keep": true/false}}
9
+ {{"clip_id": "clip_0002", "keep": true/false}}
10
+ ]
11
+ }}
12
+ ```
prompts/tasks/generate_script/zh/system.md ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 角色设定
2
+ 你是一位资深的短视频及Vlog文案策划大师。你拥有敏锐的洞察力,擅长化身为视频的主角(第一人称“我”),用“轻叙事感”的口语,将碎片化的素材串联成有温度、有逻辑、情感饱满的故事。
3
+
4
+ # 任务目标
5
+ 你的任务是根据用户提供的 [user_request](视频核心主题)、[style](文案风格)和 [group_infos](分组素材详情),为每一个分组(Group)编写一段旁白文案,并为整个视频起一个标题。
6
+
7
+ # 输入数据说明
8
+ 输入数据包含三部分:
9
+ 1. **[user_request]**: 视频的核心主题或创作者的感悟。
10
+ 2. **[overall]**: 用户提供的所有素材的总体叙事概述。
11
+ 3. **[style]**: 指定文案的风格偏好(如:文艺抒情、幽默有趣、日常碎碎念等)。
12
+ 4. **[group_infos]**: 包含多个分组,每个分组代表视频的一个段落。关键字段:
13
+ - `summary`: 该段落的叙事目的。
14
+ - `script_chars_budget`: **关键约束**。文案字数必须严格落在该区间内。
15
+ - `clips`: 该组包含的具体画面描述。
16
+
17
+ # 风格效果配置 (Style Configuration)
18
+ 请根据输入中的 `[style]` 字段,严格采用对应的写作策略。如输入未指定,默认为**“日常碎碎念”**。
19
+
20
+ 1. **文艺抒情 (Lyrical & Poetic)**:
21
+ - **核心**: 治愈、浪漫、电影感、注重意象。
22
+ - **策略**: 弱化直白的动作描述,强化感官体验(光影、气味、温度、声音)。多用比喻和拟人,句式优美流畅,注重情感的流动和余韵,像在读一首散文诗。
23
+
24
+ 2. **幽默有趣 (Humorous & Witty)**:
25
+ - **核心**: 玩梗(适度)、反转、自嘲、节奏轻快。
26
+ - **策略**: 寻找画面中意想不到的槽点或亮点。用词生动活泼,可以使用夸张的修辞,像个有趣的“显眼包”朋友在吐槽或分享趣事,拒绝沉闷。
27
+
28
+ 3. **日常碎碎念 (Daily Mumbling)**:
29
+ - **核心**: 真实、极度生活化、大脑独白、亲切。
30
+ - **策略**: 还原大脑里的真实想法,甚至可以有一点点逻辑跳跃。关注细枝末节(如“今天风有点大”),不刻意升华,主打一种“陪伴感”和“流水账”的真实美学。
31
+
32
+ # 创作原则 (核心)
33
+ 请严格遵守以下创作原则,优先级从上到下:
34
+
35
+ 1. **主角视角与口吻 (Tone & Perspective)**:
36
+ - 全程使用**第一人称“我”**的视角叙事。
37
+ - 语言风格需符合上述 `[style]` 的设定,但必须保持**口语化**。
38
+ - **拒绝陈旧套路**: 开场**严禁**使用“家人们谁懂啊”、“姐妹们”等模板;结尾**严禁**使用“原来快乐如此简单”等空洞金句。
39
+
40
+ 2. **关键信息保真 (Information Fidelity)**:
41
+ - 必须敏锐识别并保留画面描述或主题中的**【专有名词】**(如品牌名、地名)、**【IP】**(如迪士尼)和**【具体事件】**。
42
+ - **不要泛化**: 结合具体的视觉元素(Visual)写作,切记胡编乱造。
43
+
44
+ 3. **字数与技术规范 (Technical Constraints)**:
45
+ - **字数严格控制**: 生成的 `raw_text` 长度必须严格落在 `script_chars_budget` 范围内。
46
+ - **标点符号限制**:
47
+ - **绝对禁止**使用任何形式的括号 `()` 或省略号 `...`。
48
+ - 标点需符合口语断句习惯。
49
+ - **Emoji使用**: 每段文案可适当使用 1 个与内容强相关的 Emoji。
50
+
51
+ 4. **画面关联与叙事 (Visual & Storytelling)**:
52
+ - **看图表达**: 文案必须是画面的注脚。
53
+ - **连贯性**: Group 之间要有逻辑衔接,使用自然的过渡词。
54
+ - **结构**:
55
+ - **开场 (Group 1)**: 迅速入题,根据风格设定基调。
56
+ - **结尾 (Last Group)**: 情感收束,根据风格进行升华或幽默收尾。
57
+
58
+ 5. **标题创作 (Title)**:
59
+ - 创作一个富有诗意、悬念感或总结性的 `title`,长度 8-15 字,需具备社交媒体(如小红书)的吸引力。
60
+
61
+ # 输出格式
62
+ 请仅输出一个标准的 JSON 对象,不要包含 Markdown 符号,格式如下:
63
+
64
+ ```json
65
+ {{
66
+ "group_scripts": [
67
+ {{
68
+ "group_id": "对应输入的group_id",
69
+ "duration": 对应输入的duration_sec,
70
+ "raw_text": "生成的文案内容"
71
+ }}
72
+ ],
73
+ "title": "生成的视频标题"
74
+ }}
75
+ ```
76
+
77
+ # 示例
78
+ **Input:**
79
+ [user_input]
80
+ 周末去公园野餐,感觉被治愈了
81
+ [style]
82
+ 文艺抒情
83
+ [group_infos]
84
+ [group_id=group_0001]
85
+ summary: 展示准备食物和到达公园的过程。
86
+ script_chars_budget: 15~25
87
+ clips: ...三明治特写,骑单车经过林荫道...
88
+
89
+ **Output:**
90
+ {{
91
+ "group_scripts": [
92
+ {{
93
+ "group_id": "group_0001",
94
+ "duration": 5.00,
95
+ "raw_text": "带着手作的三明治,一头撞进这片绿色的风里🍃"
96
+ }}
97
+ ],
98
+ "title": "想把自己寄给春天的风"
99
+ }}
prompts/tasks/generate_script/zh/user.md ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 用户的风格需求: {user_request}
2
+ 整体素材概述:{overall}
3
+
4
+ 分组输入开始:
5
+ {groups}
6
+ 分组输入结束
7
+
8
+ 输出格式如下:
9
+ ```json
10
+ {{
11
+ "group_scripts": [
12
+ {{ "group_id": "group_0001", "raw_text": "第一句,第二句,第三句" }},
13
+ {{ "group_id": "group_0002", "raw_text": "第一句,第二句" }}
14
+ ],
15
+ "title": "视频标题"
16
+ }}
17
+ ```
prompts/tasks/generate_title/zh/system.md ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 你是一个短视频剪辑助理。你需要理解用户提出的需求,执行合适的工具完成剪辑,并避免向用户回复过于专业的剪辑术语。你会拿到一个关于剪辑的工具函数的描述列表。
2
+ 如果用户是第一次提出“帮我剪辑/处理素材”等类似需求,请先用自然语言列出你计划如何使用给定的剪辑工具及理由,待用户确认。注意你只能使用你可用的剪辑工具,如果工具暂时不可用,请明确告诉用户你做不到。
3
+ 整个剪辑流程中,有些节点是固定的,你无法改动;你计划的范围仅限于可以改动的节点。除非用户明确想要跳过某个步骤,否则在列出计划时,默认运行所有节点,以达到完美的效果。
4
+ 注意,有些节点依赖前面节点的结果,具体的依赖关系你可以在工具描述中看到,请在工具调用前检查依赖。工具会自己寻找依赖的结果,你不需要将前面节点的结果输入到工具参数中。如果工具需要输入参数,会在工具描述中另加说明,请填入合适的参数。
5
+ 你的每一次回答必须仅在调用工具和用自然语言回复用户(markdown格式)之间选择一个,每次只调用一个工具。每次调用完工具后,向用户简单总结本次工具调用的结果和下一步的意图,增强互动感。尽可能使用多的工具以丰富视频内容,除非用户明确指出不要某个元素。
6
+
7
+ 常规剪辑流程如下,这里每一步都对应一个或多个工具供你使用:
8
+ 第0步:素材索引 "assets_index"(固定)。用于获取输入素材的路径、长宽等基础信息。
9
+ 第1步:镜头切分 "split_shots"(可跳过)。将素材按镜头切分成片段。
10
+ 第2步:内容理解 "understand_clips"(可跳过)。 为每个片段(clips)生成一段描述(captions)
11
+ 第3步:镜头筛选 "filter_clips"(可跳过)。根据用户要求,筛选出符合要求的片段(clips)
12
+ 第4步:片段分组 "group_clips"(可跳过,但应默认运行)。根据用户要求,对片段进行排序和分组,组织合理的叙事逻辑,并辅助后续文案生成。
13
+ 第5步:文案生成 "generate_script"(可跳过)。根据用户要求,生成视频文案。
14
+ 第6步:元素推荐 (可跳过,但应默认运行)。根据用户要求,推荐花字、标题、特效、转场、配音音色等元素。
15
+ 第7步:配音生成 "generate_voiceover"(可跳过)。根据文案生成对应的配音。
16
+ 第8步:背景音乐选取 "select_BGM"(可跳过)。选择合适的背景音乐。
17
+ 第9步:组织时间线 "plan_timeline"(固定)。根据前面的视频片段、文案、语音和BGM,组织成合理的时间线。
18
+ 第10步:渲染成片。"render_video"(固定)。根据时间线渲染成片。
19
+ 此外,虽然你在工具调用后只能看到summary,但你有一个工具可以读取任意中间节点的输出。你可以用它完成更复杂的任务。
prompts/tasks/generate_title/zh/user.md ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ 现在根据下面的视频内容,生成 {n_titles} 个适合短视频平台的中文标题,每个不超过 20 字,要有吸引力但不要标题党过头。
2
+
3
+ 【视频内容简介】
4
+ {summary}
5
+
6
+ 请用 JSON 数组返回:
7
+ [标题1, 标题2, ...]
prompts/tasks/group_clips/zh/system.md ADDED
@@ -0,0 +1,125 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 角色设置
2
+ 你是一位拥有顶级叙事逻辑且**极具共情能力**的**资深视频剪辑导演**。你不仅擅长梳理故事线,更能**深度洞察用户意图**。你能够透过简单的文字指令,精准捕捉用户想要的叙事策略(如倒叙、插叙、情绪对比等),并将其转化为“起-承-转-合”的视频结构。
3
+
4
+ # 核心任务
5
+ 你的任务分为两步:
6
+ 1. **深度思考**:首先**彻底解析用户意图**,以此为核心规划叙事结构,并输出详细的思考过程。
7
+ 2. **结构化输出**:根据思考结果,生成包含分组、合并与排序信息的标准 JSON 数据。
8
+
9
+ # 输入信息
10
+ 1. **user_input**: 视频核心主题与指令(这是**最高叙事准则**,可能隐含了特殊的排序要求、情绪基调或特定情节)。
11
+ 2. **captions**: 视频片段列表。
12
+ * 格式:`【编号】<time>时长</time><clip>组ID(可选)</clip>内容描述`
13
+ 3. **segment_num**: 原始视频片段的总数量。
14
+
15
+ # 工作流与逻辑规则 (最高优先级)
16
+
17
+ 在处理素材时,你必须**严格遵循**以下逻辑层级:
18
+
19
+ ## 第一层:硬性物理约束(源素材锁定)
20
+ **(不可违背的物理基础)**
21
+ 1. **锁定**:拥有相同 `<media>` 编号的片段(例如 `【0001】<media>A</media>` 和 `【0002】<media>A</media>`)视为**不可分割的整体**。
22
+ 2. **规则**:它们必须作为一个整体移动,**内部顺序不可变**,中间不可插入其他片段。
23
+
24
+ ## 第二层:用户意图与叙事重构(核心逻辑)
25
+ **(在此层级,必须优先响应用户的特殊指令,而非死板的时间逻辑)**
26
+ 1. **意图优先原则**:若 `user_input` 暗示了特殊的叙事手法(例如:“先展示结果再回顾过程”=倒叙;“忙碌与宁静的对比”=对比蒙太奇),必须**无条件优先**满足该逻辑,哪怕打乱时间线也在所不惜。若`user_input`要求不改变素材顺序,则严格按照 `<media>`编号组织顺序。
27
+ 2. **情绪/因果逻辑**:在符合用户意图的前提下,按情绪流(压抑→爆发)或因果流(问题→解决)排序。
28
+ 3. **常规时空逻辑**:若用户无特殊指令,则兜底使用“起-承-转-合”的经典时空顺序(早→晚,外→内)。
29
+ * *起*:引入环境/悬念。
30
+ * *承*:铺陈细节/推进。
31
+ * *转*:高潮/反转。
32
+ * *合*:升华/收束。
33
+
34
+ ## 第三层:分组与合并(叙事分段)
35
+ * **合并标准**:将服务于**同一用户意图**或**同一叙事节点**的片段合并为一个新的叙事组(Group)。
36
+ * **数量限制**:单组建议不超过 **4个** 片段(Clip组锁定导致的除外)。
37
+
38
+ # 输出规范
39
+
40
+ 你需要严格按照以下两个步骤进行输出:
41
+
42
+ ## 步骤一:思考过程
43
+ 请严格使用以下标签格式输出你的分析过程,并注意字数限制:
44
+
45
+ * **思考过程**
46
+ 【主题确定与分析】【内容开始】深度解读用户意图,明确视频是顺叙、倒叙还是情绪流?确定核心立意(**100字以内**)。【内容结束】
47
+ 【叙事与结构重构】【内容开始】阐述如何**基于用户意图**重组素材。说明Clip组锁定情况。解释为何采用当前的排序策略(例如:“响应用户‘先抑后扬’的需求,将失败镜头前置...”)(**100字以内**)。【内容结束】
48
+ 【核心主题】【内容开始】用一句话提炼视频的灵魂文案或中心思想(**50字以内**)。【内容结束】
49
+
50
+ ## 步骤二:JSON 结果
51
+ 基于思考过程,输出最终结构。
52
+ 1. 必须使用严格json格式。
53
+ 2. `clip_ids`:使用原始编号字符串列表。
54
+ 3. **`media_id` 提取规则**:这是一个数组,必须与 `clip_ids` 中的元素**位置一一对应**。
55
+ * 如果对应的 clip 有 `<clip>ID</clip>` 标签,则填入数字或字符串 `ID`。
56
+ * 如果对应的 clip 没有该标签,则填入 `null`。
57
+
58
+ **JSON 结构定义:**
59
+ ```json
60
+ {{
61
+ "groups": [
62
+ {{
63
+ "group_id": "group_序号 (例如 group_0001)",
64
+ "summary": "极具画面感的叙事或场景描述,解释该组镜头在故事中的作用(50字以内)。",
65
+ "media_id": [ ID1, ID2, null ],
66
+ "clip_ids": [
67
+ "clip_0001",
68
+ "clip_0002",
69
+ "clip_0003"
70
+ ]
71
+ }}
72
+ ]
73
+ }}
74
+ ```
75
+
76
+ ---
77
+ **示例输入:**
78
+ `user_input`: 无论最后结果如何,我们都曾努力过(要求:先展示失败的结果,再回忆过程)
79
+ `captions`:
80
+ `【clip_0001】<media>1</media>大家抱在一起痛哭`
81
+ `【clip_0002】<media>1</media>看着比分板发呆`
82
+ `【clip_0003】<media>2</media>深夜训练`
83
+ `【clip_0004】<media>3</media>受伤包扎`
84
+ `【clip_0005】互相打气` (注意:此条无clip标签)
85
+
86
+ **示例输出:**
87
+
88
+ **思考过程**
89
+ 【主题确定与分析】【内容开始】用户明确要求“倒叙”结构,强调“过程的意义”。基调是遗憾但热血。核心是展现虽然结果失败��起),但奋斗过程(承转)依然闪光。【内容结束】
90
+ 【叙事与结构重构】【内容开始】Clip 1组【0001, 0002】被物理锁定,虽是时间线终点,但根据用户“先展示结果”的意图,必须作为开篇“起”。随后进入回忆,“承”合并【0003, 0004】展示艰辛,“合”用无标签的【0005】互相打气收尾,升华主题。【内容结束】
91
+ 【核心主题】【内容开始】输赢只是瞬间,但并肩战斗的那些日夜,才是我们最宝贵的勋章。【内容结束】
92
+
93
+ 【合并及排序结果】
94
+ ```json
95
+ {{
96
+ "groups": [
97
+ {{
98
+ "group_id": "group_0001",
99
+ "summary": "响应倒叙要求,以赛后失利的痛哭与迷茫开篇,制造遗憾的情绪悬念,引出对过往的回忆。",
100
+ "media_id": [1, 1],
101
+ "clip_ids": [
102
+ "clip_0001",
103
+ "clip_0002"
104
+ ]
105
+ }},
106
+ {{
107
+ "group_id": "group_0002",
108
+ "summary": "镜头回溯至备战期间,展示深夜训练与受伤坚持的画面,体现团队付出的极致努力。",
109
+ "media_id": [2, 3],
110
+ "clip_ids": [
111
+ "clip_0003",
112
+ "clip_0004"
113
+ ]
114
+ }},
115
+ {{
116
+ "group_id": "group_0003",
117
+ "summary": "以赛前互相打气的温馨瞬间作结,强调拼搏本身比结果更重要,升华“努力过”的主题。",
118
+ "media_id": [null],
119
+ "clip_ids": [
120
+ "0005"
121
+ ]
122
+ }}
123
+ ]
124
+ }}
125
+ ```
prompts/tasks/group_clips/zh/user.md ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 用户要求: {user_request}
2
+ 注意clips编号可能有跳跃,以下是可用的clips:{selected_clips}
3
+
4
+ 以下是clips详细信息,请根据用户要求,对clips进行分组:
5
+ {clip_captions}
6
+ 你的输出格式如下:
7
+ ```json
8
+ {{
9
+ "groups": [
10
+ {{
11
+ "group_id": "group_0001",
12
+ "summary": "先用最安静、最治愈的镜头建立情绪。",
13
+ "clip_ids": ["clip_0003", "clip_0001"],
14
+ }},
15
+ {{
16
+ "group_id": "group_0002",
17
+ "summary": "用更具画面感的镜头承接金句。",
18
+ "clip_ids": ["clip_0011", "clip_0014"],
19
+ }}
20
+ ]
21
+ }}
22
+ ```
prompts/tasks/instruction/zh/system.md ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 你是一个短视频剪辑助理。你需要理解用户提出的需求,执行合适的工具完成剪辑,并避免向用户回复过于专业的剪辑术语。你会拿到一个关于剪辑的工具函数的描述列表。
2
+ 如果用户是第一次提出“帮我剪辑/处理素材”等类似需求,请先用自然语言列出你计划如何使用给定的剪辑工具及理由,待用户确认。注意你只能使用你可用的剪辑工具,如果工具暂时不可用,请明确告诉用户你做不到。
3
+ 当用户指定一个剪辑风格时,优先查看工具里面是否有满足的技能(描述以【SKILL】为开头的工具),如果有匹配的技能,优先使用。
4
+ 整个剪辑流程中,有些节点是固定的,你无法改动;你计划的范围仅限于可以改动的节点。除非用户明确想要跳过某个步骤,否则在列出计划时,默认运行所有节点,以达到完美的效果。
5
+ 注意,有些节点依赖前面节点的结果,具体的依赖关系你可以在工具描述中看到,请在工具调用前检查依赖。工具会自己寻找依赖的结果,你不需要将前面节点的结果输入到工具参数中。如果工具需要输入参数,会在工具描述中另加说明,请填入合适的参数。
6
+ 你的每一次回答必须仅在调用工具和用自然语言回复用户(markdown格式)之间选择一个,每次只调用一个工具。每次调用完工具后,向用户简单总结本次工具调用的结果和下一步的意图,增强互动感。尽可能使用多的工具以丰富视频内容,除非用户明确指出不要某个元素。
7
+
8
+ 常规剪辑流程如下,这里每一步都对应一个或多个工具供你使用:
9
+ 第0步:素材索引 "assets_index"(固定)。用于获取输入素材的路径、长宽等基础信息。
10
+ 第1步:镜头切分 "split_shots"(可跳过)。将素材按镜头切分成片段。
11
+ 第2步:内容理解 "understand_clips"(可跳过)。 为每个片段(clips)生成一段描述(captions)
12
+ 第3步:镜头筛选 "filter_clips"(可跳过)。根据用户要求,筛选出符合要求的片段(clips)
13
+ 第4步:片段分组 "group_clips"(可跳过,但应默认运行)。根据用户要求,对片段进行排序和分组,组织合理的叙事逻辑,并辅助后续文案生成。
14
+ 第5步:文案生成 "generate_script"(可跳过)。根据用户要求,生成视频文案。
15
+ 第6步:元素推荐 (可跳过,但应默认运行)。根据用户要求,推荐花字、标题、特效、转场、配音音色等元素。
16
+ 第7步:配音生成 "generate_voiceover"(可跳过)。根据文案生成对应的配音。
17
+ 第8步:背景音乐选取 "select_BGM"(可跳过)。选择合适的背景音乐。
18
+ 第9步:组织时间线 "plan_timeline"(固定)。根据前面的视频片段、文案、语音和BGM,组织成合理的时间线。
19
+ 第10步:渲染成片。"render_video"(固定)。根据时间线渲染成片。
20
+ 此外,虽然你在工具调用后只能看到summary,但你有一个工具可以读取任意中间节点的输出。你可以用它完成更复杂的任务。
21
+
22
+ 风格要求:
23
+ - 用简洁、口语化的语言
24
+
25
+ 当你需要调用剪辑工具时,请只输出调用信息。
26
+
27
+ 当你已经拿到足够的信息,不需要再调用工具时,直接用自然语言回答客户,不要再输出JSON
28
+
29
+ 下面是几个示例,模仿这种风格:
30
+
31
+ 示例 1:列出计划
32
+ [用户]:
33
+ 我想要你帮我把我的素材剪辑成旅行vlog,
34
+ [助手]:
35
+ 我将要使用如下流程进行剪辑:
36
+ 1. 素材索引:需要先对输入素材建立索引,提取元信息。
37
+ 2. 镜头切分:将素材按镜头切分成片段。
38
+ 3. 内容理解:对每个镜头生成文字描述。
39
+ 4. 镜头筛选:筛选出适合放在旅行vlog中的镜头。
40
+ 5. 片段分组:组织一个合理的素材顺序。
41
+ 6. 文案生成:配一段符合主题的文案。
42
+ 7. 花字检索:添加花字元素,增强视觉效果。
43
+ 8. 标题花字检索:添加标题元素,提升视频吸引力。
44
+ 9. 全局特效检索。添加适合日常vlog的全局特效/滤镜。
45
+ 10. 转场检索:添加合适的转场效果,让镜头切换更自然。
46
+ 11. 闭幕特效检索:添加合适的闭幕效果,让视频结束更自然。
47
+ 12. 滤镜检索:添加合适的滤镜,丰富视觉体验。
48
+ 13. 花字动画:添加合适的花字动画,丰富字幕效果。
49
+ 14. 片头检索:添加合适的片头,增强视频吸引力。
50
+ 15. 配音音色检索:选择合适的配音音色。
51
+ 16. 配音生成:根据文案生成配音。
52
+ 17. 音乐推荐:选择轻松愉快的vlog背景音乐。
53
+ 18. 组织时间线:这是剪辑的固定流程。
54
+ 19. 渲染成片:这也是剪辑的固定流程。
55
+
56
+ 示例 2:不需要工具时,直接回答
57
+ 用户:
58
+ 什么是“镜头切分”?
59
+ 助手:
60
+ 镜头切分是指把原始视频按照画面内容或语义边界切分成若干个独立的镜头片段,通常会结合画面变化、音频变化等特征来判断切分点,用于后续剪辑、检索或分析。
61
+
62
+ 示例3:二次编辑
63
+ 用户:
64
+ 之前你给我剪的视频有配音,我现在不想要视频的配音了。
65
+
66
+ 此时助手需要执行配音工具,参数mode选择skip。
prompts/tasks/understand_clips/zh/system_detail.md ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ 你是一个短视频剪辑素材理解助手。
2
+ 你的任务是:根据给定的视频或图片素材,为剪辑流程提供一段简洁、准确的中文描述,重点包含:主体内容、人物或物体、场景和整体氛围,方便后续做高光检测和文案生成。
3
+ 不要输出客套话或多余说明。你必须只输出一个合法 JSON 对象,不要输出任何额外文字
prompts/tasks/understand_clips/zh/system_overall.md ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ 你是一个短视频剪辑素材理解助手。
2
+ 你的任务是:根据给定的多个素材的文案描述,生成一段总体描述。重点包含:主体内容、人物或物体、场景和整体氛围,方便后续做高光检测和文案生成。不要输出客套话或多余说明。
prompts/tasks/understand_clips/zh/user_detail.md ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ 请为这个 clip 生成中文的描述与语义标签。
2
+ 输出必须严格符合以下 JSON 结构(key 不可缺失)
3
+ ```json
4
+ {{
5
+ "caption": "2-4句描述(尽量具体客观,不要编造看不到的细节)"
6
+ }}
7
+ ```
prompts/tasks/understand_clips/zh/user_overall.md ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ 下面是若干clips的单体描述,请你从整体角度总结:这些素材大致讲述了怎样的故事?请用 1~2 段话总结,并保持客观、中性。
2
+ {clips_captions}
requirements.txt CHANGED
@@ -1,2 +1,13 @@
1
  fastapi
2
- uvicorn
 
 
 
 
 
 
 
 
 
 
 
 
1
  fastapi
2
+ uvicorn
3
+ langchain_core
4
+ mcp
5
+ colorlog
6
+ librosa
7
+ transnetv2_pytorch
8
+ langchain
9
+ langchain_mcp_adapters
10
+ langchain_openai
11
+ ffmpeg-python
12
+ aiofiles
13
+ skillkit
run.sh ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ set -e
3
+
4
+ ROOT_DIR="$(cd "$(dirname "$0")" && pwd)"
5
+ export PYTHONPATH="$ROOT_DIR/src"
6
+
7
+ HOST="${HOST:-0.0.0.0}"
8
+ PORT="${PORT:-7860}"
9
+
10
+ python3 -m open_storyline.mcp.server &
11
+ MCP_PID=$!
12
+
13
+ uvicorn agent_fastapi:app \
14
+ --host "$HOST" \
15
+ --port "$PORT" &
16
+ WEB_PID=$!
17
+
18
+ trap 'kill $MCP_PID $WEB_PID' INT TERM
19
+
20
+ wait
src/__init__.py ADDED
File without changes
src/open_storyline/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ __version__ = "0.1.0"
src/open_storyline/__main__.py ADDED
File without changes
src/open_storyline/config.py ADDED
@@ -0,0 +1,189 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # open_storyline/configuration_utils.py
2
+ from __future__ import annotations
3
+ import os
4
+ from pathlib import Path
5
+ from typing import Any, Optional, Literal
6
+ import time
7
+
8
+ try:
9
+ import tomllib
10
+ except ImportError:
11
+ print("Fail to import tomllib, try to import tomlis")
12
+ import tomli as tomllib
13
+
14
+ from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, computed_field, field_validator
15
+
16
+
17
+ def _resolve_relative_path_to_config_dir(v: Path, info: ValidationInfo) -> Path:
18
+ """
19
+ 把相对路径按 config.toml 所在目录解析(而不是按 cwd)。
20
+
21
+ 依赖调用方在 model_validate(..., context={"config_dir": <Path|str>}) 里传入 config_dir。
22
+ """
23
+ ctx = info.context or {}
24
+ base = ctx.get("config_dir")
25
+ if not base:
26
+ return v
27
+
28
+ v2 = v.expanduser()
29
+ if v2.is_absolute():
30
+ return v2
31
+
32
+ base_dir = Path(base).expanduser()
33
+ return (base_dir / v2).resolve(strict=False)
34
+
35
+
36
+ def _resolve_paths_recursively(value: Any, info: ValidationInfo) -> Any:
37
+ """
38
+ 递归处理容器类型里的 Path(list/tuple/set/dict)。
39
+ """
40
+ if value is None:
41
+ return None
42
+
43
+ if isinstance(value, Path):
44
+ return _resolve_relative_path_to_config_dir(value, info)
45
+
46
+ if isinstance(value, list):
47
+ return [_resolve_paths_recursively(v, info) for v in value]
48
+
49
+ if isinstance(value, tuple):
50
+ return tuple(_resolve_paths_recursively(v, info) for v in value)
51
+
52
+ if isinstance(value, set):
53
+ return {_resolve_paths_recursively(v, info) for v in value}
54
+
55
+ if isinstance(value, dict):
56
+ return {k: _resolve_paths_recursively(v, info) for k, v in value.items()}
57
+
58
+ return value
59
+
60
+
61
+ class ConfigBaseModel(BaseModel):
62
+ model_config = ConfigDict(extra="forbid")
63
+
64
+ @field_validator("*", mode="after")
65
+ @classmethod
66
+ def _resolve_all_path_fields(cls, v: Any, info: ValidationInfo) -> Any:
67
+ # 允许对某些字段显式禁用路径解析:
68
+ # Field(..., json_schema_extra={"resolve_relative": False})
69
+ if info.field_name:
70
+ field = cls.model_fields.get(info.field_name)
71
+ extra = (field.json_schema_extra or {}) if field else {}
72
+ if extra.get("resolve_relative") is False:
73
+ return v
74
+
75
+ return _resolve_paths_recursively(v, info)
76
+
77
+ class DeveloperConfig(ConfigBaseModel):
78
+ developer_mode: bool = False
79
+ chat_models: list[str] = Field(default_factory=list)
80
+ chat_models_config: dict[str, dict[str, Any]] = Field(default_factory=dict)
81
+ print_context: bool = False
82
+
83
+ class ProjectConfig(ConfigBaseModel):
84
+ assets_dir: Path = Field(..., description="素材目录(输入视频、图片)")
85
+ bgm_dir: Path = Field(..., description="背景音乐目录")
86
+ outputs_dir: Path = Field(..., description="输出目录")
87
+ debug_mode: bool = Field(True, description="是否启用debug模式")
88
+
89
+ @computed_field(return_type=Path)
90
+ @property
91
+ def blobs_dir(self) -> Path:
92
+ return self.outputs_dir
93
+
94
+
95
+ class LLMConfig(ConfigBaseModel):
96
+ model: str
97
+ base_url: str
98
+ api_key: str
99
+ timeout: float = 30.0
100
+ temperature: Optional[float] = None
101
+ max_retries: int = 2
102
+
103
+
104
+ class VLMConfig(ConfigBaseModel):
105
+ model: str
106
+ base_url: str
107
+ api_key: str
108
+ timeout: float = 20.0
109
+ temperature: Optional[float] = None
110
+ max_retries: int = 2
111
+
112
+
113
+ class MCPConfig(ConfigBaseModel):
114
+ server_name: str = "storyline"
115
+ server_cache_dir: str = "./storyline/.server_cache"
116
+ server_transport: Literal["stdio", "sse", "streamable-http"] = "streamable-http"
117
+ url_scheme: str = "http"
118
+ connect_host: str = "127.0.0.1"
119
+ port: int = Field(ge=1, le=65535)
120
+ path: str = "/mcp"
121
+
122
+ json_response: bool = True
123
+ stateless_http: bool = False
124
+
125
+ timeout: int = 600
126
+
127
+ use_xhs_node_only: bool = True
128
+ @property
129
+ def url(self) -> str:
130
+ return f"{self.url_scheme}://{self.connect_host}:{self.port}{self.path}"
131
+
132
+
133
+ class SplitShotsConfig(ConfigBaseModel):
134
+ transnet_weights: Path = Field(..., description="transnet_v2 权重路径")
135
+ transnet_device: str = "cpu"
136
+
137
+ class UnderstandClipsConfig(ConfigBaseModel):
138
+ sample_fps: float = 2.0
139
+ max_frames: int = 64
140
+
141
+
142
+ class GenerateVoiceoverConfig(ConfigBaseModel):
143
+ base_url: str = ""
144
+
145
+
146
+ class SelectBGMConfig(ConfigBaseModel):
147
+ sample_rate: int = 22050
148
+ hop_length: int = 2048
149
+ frame_length: int = 2048
150
+
151
+
152
+ class PlanTimelineConfig(ConfigBaseModel):
153
+ beat_type_max: int = 1 # 使用多强的鼓点,例如4/4中,鼓点类似1,2,1,3,其中1是最强的,3最弱
154
+ title_dutation: int = 5000 # 片头时长
155
+ bgm_loop: bool = True # 是否允许 bgm 循环
156
+ min_clip_duration: int = 500
157
+
158
+ estimate_text_min: int = 1500 # 在没有TTS的情况下,每组字幕至少上屏多久
159
+ estimate_text_char_per_sec: float = 6.0 # 在没有TTS的情况下,估计每秒展示几个字
160
+
161
+ image_default_duration: int = 3000 # 默认的图片���放时长
162
+
163
+ group_margin_over_voiceover: int = 1000 # 在一个group中,画面比配音多出现多久
164
+
165
+
166
+ class Settings(ConfigBaseModel):
167
+ developer: DeveloperConfig
168
+ project: ProjectConfig
169
+
170
+ llm: LLMConfig
171
+ vlm: VLMConfig
172
+
173
+ local_mcp_server: MCPConfig
174
+ xhs_mcp_server: MCPConfig
175
+
176
+ split_shots: SplitShotsConfig
177
+ understand_clips: UnderstandClipsConfig
178
+ generate_voiceover: GenerateVoiceoverConfig
179
+ select_bgm: SelectBGMConfig
180
+ plan_timeline: PlanTimelineConfig
181
+
182
+
183
+ def load_settings(config_path: str | Path) -> Settings:
184
+ p = Path(config_path).expanduser().resolve()
185
+ data = tomllib.loads(p.read_text(encoding="utf-8"))
186
+ return Settings.model_validate(data, context={"config_dir": p.parent})
187
+
188
+ def default_config_path() -> str:
189
+ return os.getenv("OPENSTORYLINE_CONFIG", "config.toml")
src/open_storyline/mcp/__init__.py ADDED
File without changes
src/open_storyline/mcp/hooks/node_interceptors.py ADDED
@@ -0,0 +1,299 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from collections import defaultdict
2
+ from typing import List, Any, Dict
3
+ import os
4
+ from pathlib import Path
5
+ import json
6
+ import traceback
7
+
8
+
9
+ from langchain_mcp_adapters.interceptors import MCPToolCallRequest
10
+ from langgraph.types import Command
11
+ from langchain_core.messages import ToolMessage, ToolCall
12
+ from langchain_core.tools import ToolException
13
+ from mcp.types import CallToolResult
14
+
15
+ from open_storyline.nodes.node_manager import NodeManager
16
+
17
+ from open_storyline.storage.file import FileCompressor
18
+ from open_storyline.utils.logging import get_logger
19
+
20
+ logger = get_logger(__name__)
21
+
22
+ def compress_payload_to_base64(payload: Dict[str,List[Any]]):
23
+ if not isinstance(payload, dict):
24
+ return payload
25
+ for key,value in payload.items():
26
+ if isinstance(value, list) and all([isinstance(item, dict) for item in value]):
27
+ for item in value:
28
+ if 'path' in item.keys():
29
+ path = item['path']
30
+ compress_data = FileCompressor.compress_and_encode(path)
31
+
32
+ item.update({
33
+ "path": path,
34
+ "base64": compress_data.base64,
35
+ "md5": compress_data.md5
36
+ })
37
+
38
+ # 如果只是调用本地节点记得特殊处理,不再发送base64
39
+ class ToolInterceptor:
40
+
41
+ @staticmethod
42
+ async def inject_media_content_before(
43
+ request: MCPToolCallRequest,
44
+ handler,
45
+ ):
46
+ try:
47
+ tool_call_type = request.args.get('tool_call_type', 'auto')
48
+ # for default tool call
49
+ if tool_call_type!= 'auto':
50
+ request.args = request.args.get('args', {})
51
+
52
+ runtime = request.runtime
53
+ context = runtime.context
54
+ store = runtime.store
55
+ session_id = context.session_id
56
+ node_id = request.name
57
+ artifact_id = store.generate_artifact_id(node_id)
58
+ meta_collector: NodeManager = context.node_manager
59
+ tool_call_id = request.runtime.tool_call_id
60
+ input_data = defaultdict(list)
61
+
62
+ def load_collected_data(collected_node, input_data, store):
63
+ """加载收集到的节点数据"""
64
+ for collect_kind, artifact_meta in collected_node.items():
65
+ _, prior_node_output = store.load_result(artifact_meta.artifact_id)
66
+ compress_payload_to_base64(prior_node_output['payload'])
67
+ input_data[collect_kind] = prior_node_output['payload']
68
+
69
+ if node_id == 'assets_index':
70
+ input_data['inputs'] = []
71
+ assets_dir = Path(context.assets_dir)
72
+ for file_name in os.listdir(assets_dir):
73
+ path = assets_dir / file_name
74
+ if path.is_dir():
75
+ continue
76
+ compress_data = FileCompressor.compress_and_encode(path)
77
+ input_data['inputs'].append(
78
+ {
79
+ "path": str(path.relative_to(os.getcwd())),
80
+ "base64": compress_data.base64,
81
+ "md5": compress_data.md5,
82
+ }
83
+ )
84
+ elif node_id in list(meta_collector.id_to_tool.keys()):
85
+ # 1. 确定执行模式和依赖需求
86
+ is_skip_mode = request.args.get('mode', 'auto') != 'auto'
87
+ require_kind = (
88
+ meta_collector.id_to_default_require_prior_kind[node_id]
89
+ if is_skip_mode
90
+ else meta_collector.id_to_require_prior_kind[node_id]
91
+ )
92
+
93
+ # 2. 检查节点是否可执行
94
+ collect_result = meta_collector.check_excutable(session_id, store, require_kind)
95
+ load_collected_data(collect_result['collected_node'], input_data, store)
96
+
97
+ # 3. 处理依赖缺失情况
98
+ if not collect_result['excutable']:
99
+ missing_kinds = collect_result['missing_kind']
100
+ node_ids_missing = [
101
+ meta_collector.kind_to_node_ids[kind][0]
102
+ for kind in missing_kinds
103
+ ]
104
+
105
+ logger.info(
106
+ f"`{node_id}` require kind missing `{missing_kinds}`, "
107
+ f"need to execute prerequisite nodes: {node_ids_missing}"
108
+ )
109
+
110
+ # 4. 递归执行缺失的前序节点
111
+ async def execute_missing_dependencies(
112
+ missing_kinds: List[str],
113
+ for_node_id: str,
114
+ depth: int = 0
115
+ ):
116
+ """
117
+ 递归执行缺失的依赖节点
118
+
119
+ Args:
120
+ missing_kinds: 缺失的依赖类型列表
121
+ for_node_id: 当前正在解决依赖的节点ID
122
+ depth: 递归深度(用于日志缩进)
123
+ """
124
+ if not missing_kinds:
125
+ return
126
+
127
+ indent = " " * depth
128
+ logger.info(f"{indent}├─ Resolving dependencies for `{for_node_id}`: {missing_kinds}")
129
+
130
+ for kind in missing_kinds:
131
+ success = False
132
+ candidates = meta_collector.kind_to_node_ids[kind]
133
+
134
+ for miss_id in candidates:
135
+ try:
136
+ await execute_node_with_default_mode(
137
+ miss_id,
138
+ for_node_id=for_node_id,
139
+ depth=depth
140
+ )
141
+ logger.info(
142
+ f"{indent}│ ✓ `{miss_id}` executed successfully for kind `{kind}`"
143
+ )
144
+ success = True
145
+ break
146
+ except ToolException as e:
147
+ logger.warning(
148
+ f"{indent}│ ✗ `{miss_id}` failed: {str(e)}"
149
+ )
150
+ continue
151
+
152
+ if not success:
153
+ raise ToolException(
154
+ f"Cannot satisfy dependency `{kind}` required by `{for_node_id}`. "
155
+ f"All candidates failed: {candidates}"
156
+ )
157
+
158
+ async def execute_node_with_default_mode(
159
+ miss_id: str,
160
+ for_node_id: str,
161
+ depth: int = 0
162
+ ):
163
+ """
164
+ 以default模式执行指定节点
165
+
166
+ Args:
167
+ miss_id: 要执行的节点ID
168
+ for_node_id: 请求该节点的父节点ID
169
+ depth: 递归深度
170
+ """
171
+ indent = " " * depth
172
+ logger.info(
173
+ f"{indent}├─ [Default Mode] Executing `{miss_id}` "
174
+ f"(required by `{for_node_id}`)"
175
+ )
176
+
177
+ # 准备工具调用参数
178
+ tool = meta_collector.get_tool(miss_id)
179
+ tool_call_input = {
180
+ 'artifact_id': store.generate_artifact_id(miss_id),
181
+ 'mode': 'default'
182
+ }
183
+
184
+ # 检查该节点的依赖
185
+ default_require = meta_collector.id_to_default_require_prior_kind[miss_id]
186
+ default_collect_result = meta_collector.check_excutable(
187
+ session_id, store, default_require
188
+ )
189
+ default_collect_result = meta_collector.check_excutable(session_id, store, default_require)
190
+ # 递归处理依赖
191
+ if default_collect_result['excutable']:
192
+ load_collected_data(
193
+ default_collect_result['collected_node'],
194
+ tool_call_input,
195
+ store
196
+ )
197
+ logger.debug(f"{indent}│ Dependencies satisfied for `{miss_id}`")
198
+ else:
199
+ logger.info(
200
+ f"{indent}│ `{miss_id}` has missing dependencies: "
201
+ f"{default_collect_result['missing_kind']}"
202
+ )
203
+ await execute_missing_dependencies(
204
+ default_collect_result['missing_kind'],
205
+ for_node_id=miss_id, # 这里传入 miss_id
206
+ depth=depth + 1 # 增加递归深度
207
+ )
208
+
209
+ # 执行工具
210
+ try:
211
+ output = await tool.arun(
212
+ ToolCall(
213
+ args=tool_call_input,
214
+ tool_call_type='default',
215
+ runtime=runtime
216
+ )
217
+ )
218
+ logger.info(f"{indent}└─ ✓ `{miss_id}` completed successfully")
219
+ return output
220
+ except Exception as e:
221
+ logger.error(f"{indent}└─ ✗ `{miss_id}` execution failed: {str(e)}")
222
+ raise ToolException(f"Failed to execute `{miss_id}`: {str(e)}")
223
+
224
+ # 开始执行缺失的依赖
225
+ await execute_missing_dependencies(missing_kinds, for_node_id=node_id)
226
+
227
+ # 再次收集依赖
228
+ collect_result = meta_collector.check_excutable(session_id, store, require_kind)
229
+ load_collected_data(collect_result['collected_node'], input_data, store)
230
+ else:
231
+ input_data['artifacts_dir'] = store.artifacts_dir
232
+
233
+ new_req_args = {
234
+ 'artifact_id': artifact_id,
235
+ }
236
+ new_req_args.update(request.args)
237
+ new_req_args.update(input_data)
238
+
239
+ modified_request = request.override(
240
+ args=new_req_args
241
+ )
242
+ return await handler(modified_request)
243
+ except Exception as e:
244
+ logger.error("[ToolInterceptor]"+ "".join(traceback.format_exception(e)))
245
+ raise
246
+
247
+ @staticmethod
248
+ async def save_media_content_after(
249
+ request: MCPToolCallRequest,
250
+ handler,
251
+ ):
252
+ result = ""
253
+ """End agent run when task is marked complete."""
254
+ try:
255
+ tool_call_result: CallToolResult = await handler(request) # 捕获工具调用后的结果
256
+ client_ctx = request.runtime.context
257
+
258
+
259
+ result = tool_call_result.model_dump()
260
+ tool_result = json.loads(result['content'][0]['text'])
261
+ node_id = request.name
262
+
263
+ artifact_id = tool_result['artifact_id']
264
+ session_id = client_ctx.session_id
265
+
266
+ store = request.runtime.store
267
+
268
+ # store.save_result(session_id,node_id,tool_result)
269
+ store.save_result(
270
+ session_id,
271
+ node_id,
272
+ tool_result,
273
+ )
274
+ tool_call_id = request.runtime.tool_call_id
275
+
276
+ if node_id == 'read_node_history':
277
+ tool_excute_result = tool_result['tool_excute_result']
278
+ else:
279
+ tool_excute_result = {}
280
+
281
+ return Command(
282
+ update={
283
+ "messages": [
284
+ ToolMessage(content={
285
+ 'summary': {
286
+ 'node_summary': tool_result['summary'],
287
+ 'tool_excute_result': tool_excute_result
288
+ },
289
+ 'isError': tool_result['isError']
290
+ }, tool_call_id=tool_call_id)
291
+ ],
292
+ "status": "done"
293
+ },
294
+ # goto="__end__", # switch other agent
295
+ )
296
+ except Exception as e:
297
+ logger.error("[ToolInterceptor]"+ "".join(traceback.format_exception(e)))
298
+ logger.error(f"Tool Call result: {result}")
299
+ raise
src/open_storyline/mcp/register_tools.py ADDED
@@ -0,0 +1,237 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+ from dataclasses import asdict
3
+ from typing import Annotated
4
+ from pydantic import BaseModel, Field
5
+ import inspect
6
+ import traceback
7
+
8
+ from open_storyline.config import Settings
9
+ from open_storyline.skills.skills_io import dump_skills
10
+
11
+ from open_storyline.nodes.core_nodes.base_node import BaseNode
12
+ from open_storyline.nodes.core_nodes.assets_index import AssetsIndexNode
13
+ from open_storyline.nodes.core_nodes.split_shots import SplitShotsNode
14
+ from open_storyline.nodes.core_nodes.understand_clips import UnderstandClipsNode
15
+ from open_storyline.nodes.core_nodes.filter_clips import FilterClipsNode
16
+ from open_storyline.nodes.core_nodes.group_clips import GroupClipsNode
17
+ from open_storyline.nodes.core_nodes.generate_script import GenerateScriptNode
18
+ from open_storyline.nodes.core_nodes.generate_voiceover import GenerateVoiceoverNode
19
+ from open_storyline.nodes.core_nodes.select_bgm import SelectBGMNode
20
+ from open_storyline.nodes.core_nodes.plan_timeline import PlanTimelineNode
21
+ from open_storyline.nodes.core_nodes.render_video import RenderVideoNode
22
+
23
+ # from open_storyline.xhs_nodes.textgen_node import XhsUnderstandClipsNode, XhsGroupClipNode, XhsGenerateScriptNode
24
+ # from open_storyline.xhs_nodes.elementrec_node import (
25
+ # ElementrecTextsNode,
26
+ # ElementrecTitleNode,
27
+ # ElementrecGlobalEffectsNode,
28
+ # ElementrecLocalEffectsNode,
29
+ # ElementrecCloseEffectsNode,
30
+ # ElementrecFilterNode,
31
+ # ElementrecTTSSpeakerNode,
32
+ # ElementrecTransitionNode,
33
+ # ElementrecTextAnimationNode,
34
+ # ElementrecTitleClipNode,
35
+ # )
36
+ # from open_storyline.xhs_nodes.tts_node import FireTTSNode
37
+ # from open_storyline.xhs_nodes.music_select_node import MusicRecNode
38
+ # from open_storyline.xhs_nodes.timeline_node import TimelineNode
39
+ # from open_storyline.xhs_nodes.render_template_node import TemplateRenderNode
40
+
41
+ # from open_storyline.xhs_nodes.xhs_detect_highlights import XhsDetectHighlightsNode
42
+ # from open_storyline.xhs_nodes.xhs_filter_clips import XhsHighlightFilterNode
43
+
44
+ from open_storyline.mcp.sampling_requester import make_llm
45
+
46
+ from open_storyline.nodes.node_summary import NodeSummary
47
+ from src.open_storyline.storage.agent_memory import ArtifactStore
48
+
49
+ from mcp.server.fastmcp import Context, FastMCP
50
+ from mcp.server.session import ServerSession
51
+
52
+ def create_tool_wrapper(node: BaseNode, input_schema: type[BaseModel]):
53
+ """
54
+ 工厂函数:将自定义 Node 转换为 MCP Tool 函数
55
+ """
56
+ # 获取元数据,用于 @server.tool 的参数
57
+ meta = node.meta if hasattr(node, 'meta') else None
58
+
59
+ async def wrapper(mcp_ctx: Context, **kwargs) -> dict:
60
+ # 1. 统一处理上下文和 Session
61
+ request = mcp_ctx.request_context.request
62
+ headers = request.headers
63
+ session_id = headers.get('X-Storyline-Session-Id')
64
+
65
+ # 2. Session 生命周期管理 (提取自原代码)
66
+ session_manager = mcp_ctx.request_context.lifespan_context
67
+ if hasattr(session_manager, 'cleanup_expired_sessions'):
68
+ session_manager.cleanup_expired_sessions(session_id)
69
+
70
+ # 3. 构造参数
71
+ # 注意:FastMCP 会自动将参数注入到 kwargs 中,这里做合并
72
+ req_json = await request.json()
73
+ params = kwargs.copy()
74
+ # 优先使用 request body 中的 arguments,如果没有则使用 kwargs
75
+ params.update(req_json.get('params', {}).get('arguments', {}))
76
+
77
+ params['session_id'] = session_id
78
+ params['node_summary'] = NodeSummary() # 假设这是每次都需要新的
79
+ params['llm'] = make_llm(mcp_ctx)
80
+ result = await node(**params)
81
+ return result
82
+
83
+
84
+ new_params = []
85
+ # 第一个参数固定为 mcp_ctx
86
+ new_params.append(
87
+ inspect.Parameter(
88
+ 'mcp_ctx',
89
+ inspect.Parameter.POSITIONAL_OR_KEYWORD,
90
+ annotation=Context
91
+ )
92
+ )
93
+
94
+ new_annotations = {'mcp_ctx': Context}
95
+
96
+ if input_schema:
97
+ for field_name, field_info in input_schema.model_fields.items():
98
+ # 使用 Annotated 携带描述,FastMCP 会将其识别为 JSON Schema 的 description
99
+ annotation = Annotated[field_info.annotation, field_info]
100
+
101
+ new_params.append(
102
+ inspect.Parameter(
103
+ field_name,
104
+ inspect.Parameter.KEYWORD_ONLY,
105
+ default=field_info.default if field_info.default is not ... else inspect.Parameter.empty,
106
+ annotation=annotation
107
+ )
108
+ )
109
+ new_annotations[field_name] = annotation
110
+
111
+ wrapper.__name__ = meta.name
112
+ wrapper.__doc__ = meta.description
113
+ wrapper.__signature__ = inspect.Signature(new_params)
114
+ wrapper.__annotations__ = new_annotations
115
+
116
+ return wrapper, meta
117
+
118
+
119
+ def register(server: FastMCP, cfg: Settings) -> None:
120
+ if cfg.local_mcp_server.use_xhs_node_only:
121
+ all_node_classes = [
122
+ AssetsIndexNode,
123
+ # XhsDetectHighlightsNode, XhsUnderstandClipsNode, XhsHighlightFilterNode, XhsGroupClipNode, XhsGenerateScriptNode,
124
+ # ElementrecTextsNode, ElementrecTitleNode, ElementrecGlobalEffectsNode, ElementrecCloseEffectsNode, ElementrecFilterNode,
125
+ # ElementrecTTSSpeakerNode, ElementrecLocalEffectsNode, ElementrecTransitionNode, ElementrecTextAnimationNode, ElementrecTitleClipNode,
126
+ # FireTTSNode, MusicRecNode, TimelineNode, TemplateRenderNode
127
+ ]
128
+ else:
129
+ core_nodes = [
130
+ AssetsIndexNode, SplitShotsNode, UnderstandClipsNode, FilterClipsNode, GroupClipsNode, GenerateScriptNode, GenerateVoiceoverNode, SelectBGMNode, PlanTimelineNode, RenderVideoNode
131
+ ]
132
+
133
+ xhs_nodes = [
134
+ # XhsDetectHighlightsNode, XhsUnderstandClipsNode, XhsHighlightFilterNode, XhsGroupClipNode, XhsGenerateScriptNode,
135
+ # ElementrecTextsNode, ElementrecTitleNode, ElementrecGlobalEffectsNode, ElementrecCloseEffectsNode, ElementrecFilterNode,
136
+ # ElementrecTTSSpeakerNode, ElementrecLocalEffectsNode, ElementrecTransitionNode, ElementrecTextAnimationNode, ElementrecTitleClipNode,
137
+ # FireTTSNode, MusicRecNode, TimelineNode, TemplateRenderNode
138
+ ]
139
+ # 合并所有节点类
140
+ all_node_classes = core_nodes + xhs_nodes
141
+
142
+ # 2. 批量注册普通节点
143
+ for NodeClass in all_node_classes:
144
+ # 实例化
145
+ node_instance = NodeClass(cfg)
146
+ input_schema = node_instance.input_schema
147
+
148
+ tool_func, meta = create_tool_wrapper(node_instance, input_schema)
149
+
150
+ tool_name = NodeClass.meta.name
151
+ tool_desc = NodeClass.meta.description
152
+
153
+ # 使用 server.tool 装饰器注册
154
+ server.tool(
155
+ name=tool_name,
156
+ description=tool_desc,
157
+ meta=asdict(meta)
158
+ )(tool_func)
159
+
160
+
161
+ # 注册特殊工具 (如 read_node_history)
162
+ # local tool
163
+ @server.tool(
164
+ name="read_node_history",
165
+ description="读取任意节点的运行结果。需要输入该结果对应的artifact_id"
166
+ )
167
+ async def mcp_read_history(
168
+ mcp_ctx: Context[ServerSession, object],
169
+ query_artifact_id: Annotated[str, Field(description="用该artifact_id拿相应的json")]
170
+ ) -> dict:
171
+ node_summary = NodeSummary()
172
+ request = mcp_ctx.request_context.request
173
+ session_id = mcp_ctx.request_context.request.headers['X-Storyline-Session-Id']
174
+ req_json_content = await request.json()
175
+ params = req_json_content['params'].get('arguments', {'query_artifact_id': query_artifact_id})
176
+ params['session_id'] = session_id
177
+ params['node_summary'] = node_summary
178
+
179
+ try:
180
+ store = ArtifactStore(artifacts_dir=params.get('artifacts_dir', ".storyline/.server_cache"), session_id=session_id)
181
+ meta, data = store.load_result(params['query_artifact_id'])
182
+ summary = "历史信息读取成功"
183
+ isError = False
184
+ except Exception as e:
185
+ traceback_info = ''.join(traceback.format_exception(e))
186
+ summary = f"历史读取执行失败: {params['query_artifact_id']}\n {traceback_info}",
187
+ meta, data, isError = 'None', 'None', True
188
+
189
+ return {
190
+ 'artifact_id': params.get('artifact_id', store.generate_artifact_id(req_json_content['params'].get('name', 'read_node_history'))),
191
+ 'tool_excute_result': {
192
+ "history": {
193
+ "meta": meta,
194
+ "node_data": data,
195
+ }
196
+ },
197
+ 'summary': summary,
198
+ 'isError': isError,
199
+ }
200
+
201
+ @server.tool(
202
+ name="write_skills",
203
+ description="将生成的Agent Skill (Markdown格式) 写入文件系统。成功后返回文件绝对路径。"
204
+ )
205
+ async def mcp_write_skills(
206
+ mcp_ctx: Context[ServerSession, object],
207
+ skill_name: Annotated[str, Field(description="技能文件的名称,例如 'fast_paced_vlog',无需包含后缀")],
208
+ skill_dir: Annotated[str, Field(description="技能存储目录,默认为 '.storyline/skills/'")] = '.storyline/skills/',
209
+ skill_content: Annotated[str, Field(description="完整的Markdown内容")] = '',
210
+ ) -> dict:
211
+ """
212
+ 功能:接收大模型生成的技能内容,将其保存为本地 MD 文件。
213
+ """
214
+ node_summary = NodeSummary()
215
+ request = mcp_ctx.request_context.request
216
+ session_id = request.headers.get('X-Storyline-Session-Id', 'unknown_session')
217
+ req_json_content = await request.json()
218
+ params = req_json_content['params'].get('arguments', {'skill_name': skill_name, 'skill_dir': skill_dir, 'skill_content': skill_content})
219
+ params['session_id'] = session_id
220
+ params['node_summary'] = node_summary
221
+
222
+ res = await dump_skills(
223
+ skill_name=skill_name,
224
+ skill_dir=skill_dir,
225
+ skill_content=skill_content,
226
+ )
227
+ node_summary.info_for_llm("[Write Skills] Done.")
228
+ store = ArtifactStore(artifacts_dir=params.get('artifacts_dir', ".storyline/.server_cache"), session_id=session_id)
229
+
230
+ return {
231
+ 'artifact_id': params.get('artifact_id', store.generate_artifact_id(req_json_content['params'].get('name', 'mcp_write_skills'))),
232
+ 'tool_excute_result': {
233
+ },
234
+ 'summary': "",
235
+ 'isError': False,
236
+ }
237
+
src/open_storyline/mcp/sampling_handler.py ADDED
@@ -0,0 +1,424 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import asyncio
2
+ import os
3
+ import math
4
+ import base64
5
+ from io import BytesIO
6
+ from typing import Any, Dict, List, Tuple
7
+ from urllib.parse import urlparse
8
+
9
+ from PIL import Image
10
+ from moviepy.video.io.VideoFileClip import VideoFileClip
11
+ from langchain_core.messages import SystemMessage, HumanMessage, AIMessage
12
+
13
+ from mcp.types import CreateMessageRequestParams, CreateMessageResult, TextContent
14
+
15
+
16
+ # -----------------------------
17
+ # 可调参数:控制多模态输入大小
18
+ # -----------------------------
19
+ DEFAULT_RESIZE_EDGE = 768
20
+ DEFAULT_JPEG_QUALITY = 80
21
+ DEFAULT_MIN_FRAMES = 6
22
+ DEFAULT_MAX_FRAMES = 16
23
+ DEFAULT_FRAMES_PER_SEC = 3.0
24
+ GLOBAL_MAX_IMAGE_BLOCKS = 48 # 全局最多塞多少张图(含视频帧+图片),防止 payload 过大
25
+
26
+ IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".gif", ".tiff"}
27
+ VIDEO_EXTS = {".mp4", ".mov", ".mkv", ".avi", ".webm", ".m4v"}
28
+
29
+
30
+ def _is_data_url(u: str) -> bool:
31
+ return isinstance(u, str) and u.startswith("data:")
32
+
33
+
34
+ def _is_http_url(u: str) -> bool:
35
+ return isinstance(u, str) and (u.startswith("http://") or u.startswith("https://"))
36
+
37
+
38
+ def _strip_file_scheme(u: str) -> str:
39
+ if not isinstance(u, str):
40
+ return str(u)
41
+ if u.startswith("file://"):
42
+ parsed = urlparse(u)
43
+ return parsed.path
44
+ return u
45
+
46
+
47
+ def _guess_ext(path_or_url: str) -> str:
48
+ try:
49
+ p = urlparse(path_or_url).path if _is_http_url(path_or_url) else path_or_url
50
+ return os.path.splitext(p)[1].lower()
51
+ except Exception:
52
+ return ""
53
+
54
+
55
+ def _resize_long_edge(img: Image.Image, long_edge: int) -> Image.Image:
56
+ if long_edge <= 0:
57
+ return img
58
+ w, h = img.size
59
+ le = max(w, h)
60
+ if le <= long_edge:
61
+ return img
62
+ scale = long_edge / float(le)
63
+ nw = max(1, int(round(w * scale)))
64
+ nh = max(1, int(round(h * scale)))
65
+ return img.resize((nw, nh), Image.LANCZOS)
66
+
67
+
68
+ def _pil_to_data_url(img: Image.Image, resize_edge: int, jpeg_quality: int) -> str:
69
+ img = img.convert("RGB")
70
+ img = _resize_long_edge(img, resize_edge)
71
+ buf = BytesIO()
72
+ img.save(buf, format="JPEG", quality=jpeg_quality, optimize=True)
73
+ b64 = base64.b64encode(buf.getvalue()).decode("utf-8")
74
+ return f"data:image/jpeg;base64,{b64}"
75
+
76
+
77
+ def _image_path_to_data_url(path: str, resize_edge: int, jpeg_quality: int) -> str:
78
+ img = Image.open(path)
79
+ return _pil_to_data_url(img, resize_edge, jpeg_quality)
80
+
81
+
82
+ def _choose_num_frames(duration_sec: float, min_frames: int, max_frames: int, frames_per_sec: float) -> int:
83
+ duration_sec = max(0.0, float(duration_sec))
84
+ n = int(math.ceil(duration_sec * frames_per_sec))
85
+ n = max(min_frames, n)
86
+ n = min(max_frames, n)
87
+ return n
88
+
89
+
90
+ def _sample_video_segment_to_data_urls(
91
+ video_path: str,
92
+ in_sec: float,
93
+ out_sec: float,
94
+ resize_edge: int,
95
+ jpeg_quality: int,
96
+ min_frames: int,
97
+ max_frames: int,
98
+ frames_per_sec: float,
99
+ ) -> List[Tuple[float, str]]:
100
+ """
101
+ 只对 [in_sec, out_sec] 区间抽帧。返回 (rel_t_from_in, data_url)
102
+ """
103
+
104
+ in_sec = float(in_sec)
105
+ out_sec = float(out_sec)
106
+
107
+ clip = VideoFileClip(video_path, audio=False)
108
+ try:
109
+ vdur = float(clip.duration or 0.0)
110
+
111
+ # 如果拿不到 duration,保守只取一帧,避免 out_sec 很大导致越界
112
+ if vdur <= 0:
113
+ t = max(0.0, in_sec)
114
+ frame = clip.get_frame(t)
115
+ img = Image.fromarray(frame)
116
+ return [(0.0, _pil_to_data_url(img, resize_edge, jpeg_quality))]
117
+
118
+ # 裁剪到合法范围
119
+ in_sec = max(0.0, min(in_sec, vdur))
120
+ out_sec = max(0.0, min(out_sec, vdur))
121
+
122
+ # 若仍无效,兜底取 in_sec 一帧
123
+ if out_sec <= in_sec:
124
+ frame = clip.get_frame(in_sec)
125
+ img = Image.fromarray(frame)
126
+ return [(0.0, _pil_to_data_url(img, resize_edge, jpeg_quality))]
127
+
128
+ seg_dur = out_sec - in_sec
129
+ n = _choose_num_frames(seg_dur, min_frames, max_frames, frames_per_sec)
130
+
131
+ # 分桶中心点采样,避免边界帧
132
+ times = [((i + 0.5) / n) * seg_dur for i in range(n)]
133
+
134
+ out: List[Tuple[float, str]] = []
135
+ for rel_t in times:
136
+ abs_t = in_sec + rel_t
137
+ frame = clip.get_frame(abs_t)
138
+ img = Image.fromarray(frame)
139
+ out.append((rel_t, _pil_to_data_url(img, resize_edge, jpeg_quality)))
140
+ return out
141
+ finally:
142
+ clip.close()
143
+
144
+
145
+ def _extract_text_from_mcp_content(content: Any) -> str:
146
+ if content is None:
147
+ return ""
148
+ blocks = content if isinstance(content, list) else [content]
149
+ texts: List[str] = []
150
+ for b in blocks:
151
+ if getattr(b, "type", None) == "text":
152
+ texts.append(getattr(b, "text", "") or "")
153
+ return "\n".join([t for t in (x.strip() for x in texts) if t])
154
+
155
+
156
+ def _extract_text_from_lc_response(resp: Any) -> str:
157
+ content = getattr(resp, "content", None)
158
+ if isinstance(content, str):
159
+ return content.strip()
160
+ if isinstance(content, list):
161
+ texts = []
162
+ for blk in content:
163
+ if isinstance(blk, dict) and blk.get("type") == "text":
164
+ texts.append(str(blk.get("text", "")).strip())
165
+ return "\n".join([t for t in texts if t]).strip()
166
+ return str(resp).strip()
167
+
168
+
169
+ def _normalize_media_items(media_inputs: List[Any]) -> List[Dict[str, Any]]:
170
+ """
171
+ 支持三种输入项:
172
+ 1) "path/to/video.mp4"
173
+ 2) {"url"/"path": "...", "in_sec": 1.2, "out_sec": 3.4}
174
+ 3) ("path/to/video.mp4", 1.2, 3.4) # 可选
175
+ 输出统一为:{"url": "...", "in_sec": optional, "out_sec": optional}
176
+ """
177
+ out = []
178
+ for item in media_inputs or []:
179
+ if isinstance(item, str):
180
+ out.append({"url": item})
181
+ continue
182
+
183
+ if isinstance(item, (list, tuple)) and len(item) >= 1:
184
+ d = {"url": item[0]}
185
+ if len(item) >= 2:
186
+ d["in_sec"] = item[1]
187
+ if len(item) >= 3:
188
+ d["out_sec"] = item[2]
189
+ out.append(d)
190
+ continue
191
+
192
+ if isinstance(item, dict):
193
+ url = item.get("url") or item.get("path") or item.get("media")
194
+ if not url:
195
+ continue
196
+ d = {"url": url}
197
+ if "in_sec" in item:
198
+ d["in_sec"] = item.get("in_sec")
199
+ if "out_sec" in item:
200
+ d["out_sec"] = item.get("out_sec")
201
+ out.append(d)
202
+ continue
203
+
204
+ return out
205
+
206
+
207
+ def _build_media_blocks(
208
+ media_inputs: List[Any],
209
+ resize_edge: int,
210
+ jpeg_quality: int,
211
+ min_frames: int,
212
+ max_frames: int,
213
+ frames_per_sec: float,
214
+ global_max_images: int,
215
+ ) -> List[Dict[str, Any]]:
216
+ """
217
+ 将 media_inputs 转成 OpenAI-compatible 多模态 blocks。
218
+ 视频:按段抽帧(若有 in_sec/out_sec)或全视频抽帧(老格式)
219
+ """
220
+
221
+ blocks: List[Dict[str, Any]] = []
222
+ img_count = 0
223
+
224
+ items = _normalize_media_items(media_inputs)
225
+
226
+ for idx, mi in enumerate(items):
227
+ if img_count >= global_max_images:
228
+ break
229
+
230
+ raw_url = _strip_file_scheme(str(mi.get("url")))
231
+ ext = _guess_ext(raw_url)
232
+
233
+ in_sec = mi.get("in_sec")
234
+ out_sec = mi.get("out_sec")
235
+ has_segment = (in_sec is not None and out_sec is not None)
236
+
237
+ # 1) data url(图片)直接透传
238
+ if _is_data_url(raw_url):
239
+ blocks.append({"type": "text", "text": f"Media {idx+1}: (data url image)"})
240
+ blocks.append({"type": "image_url", "image_url": {"url": raw_url}})
241
+ img_count += 1
242
+ continue
243
+
244
+ # 2) 远程 URL:图片可透传;远程视频无法本地抽帧(给提示)
245
+ if _is_http_url(raw_url):
246
+ if ext in VIDEO_EXTS:
247
+ seg_info = f" segment [{in_sec},{out_sec}]s" if has_segment else ""
248
+ blocks.append({"type": "text", "text": f"Media {idx+1}: remote video url{seg_info} (cannot sample frames locally): {raw_url}"})
249
+ continue
250
+ blocks.append({"type": "text", "text": f"Media {idx+1}: {raw_url}"})
251
+ blocks.append({"type": "image_url", "image_url": {"url": raw_url}})
252
+ img_count += 1
253
+ continue
254
+
255
+ # 3) 本地路径
256
+ path = raw_url
257
+ if not os.path.exists(path):
258
+ blocks.append({"type": "text", "text": f"Media {idx+1}: (missing file) {path}"})
259
+ continue
260
+
261
+ # 图片
262
+ if ext in IMAGE_EXTS:
263
+ data_url = _image_path_to_data_url(path, resize_edge, jpeg_quality)
264
+ blocks.append({"type": "text", "text": f"Media {idx+1}: image file {os.path.basename(path)}"})
265
+ blocks.append({"type": "image_url", "image_url": {"url": data_url}})
266
+ img_count += 1
267
+ continue
268
+
269
+ # 视频(支持按段)
270
+ if ext in VIDEO_EXTS:
271
+ if has_segment:
272
+ in_s = float(in_sec)
273
+ out_s = float(out_sec)
274
+ else:
275
+ # 老格式:全视频。用 [0, +inf],内部会 clamp 到 duration
276
+ in_s = 0.0
277
+ out_s = 1e12
278
+
279
+ frames = _sample_video_segment_to_data_urls(
280
+ video_path=path,
281
+ in_sec=in_s,
282
+ out_sec=out_s,
283
+ resize_edge=resize_edge,
284
+ jpeg_quality=jpeg_quality,
285
+ min_frames=min_frames,
286
+ max_frames=max_frames,
287
+ frames_per_sec=frames_per_sec,
288
+ )
289
+
290
+ if has_segment:
291
+ blocks.append({"type": "text", "text": f"Media {idx+1}: video segment {os.path.basename(path)} [{in_s:.2f}s, {out_s:.2f}s] (sampled frames in time order)"})
292
+ else:
293
+ blocks.append({"type": "text", "text": f"Media {idx+1}: video file {os.path.basename(path)} (sampled frames in time order)"})
294
+
295
+ for fi, (rel_t, data_url) in enumerate(frames):
296
+ if img_count >= global_max_images:
297
+ break
298
+ blocks.append({"type": "text", "text": f"Frame {fi+1}/{len(frames)} (t≈{rel_t:.2f}s from segment start)"})
299
+ blocks.append({"type": "image_url", "image_url": {"url": data_url}})
300
+ img_count += 1
301
+ continue
302
+
303
+ blocks.append({"type": "text", "text": f"Media {idx+1}: unsupported file type: {path}"})
304
+
305
+ return blocks
306
+
307
+
308
+ def make_sampling_callback(
309
+ llm,
310
+ vlm,
311
+ *,
312
+ resize_edge: int = DEFAULT_RESIZE_EDGE,
313
+ jpeg_quality: int = DEFAULT_JPEG_QUALITY,
314
+ min_frames: int = DEFAULT_MIN_FRAMES,
315
+ max_frames: int = DEFAULT_MAX_FRAMES,
316
+ frames_per_sec: float = DEFAULT_FRAMES_PER_SEC,
317
+ global_max_images: int = GLOBAL_MAX_IMAGE_BLOCKS,
318
+ ):
319
+ """
320
+ MCP server 在工具内部请求 sampling 时回调这里:
321
+ - 读取 metadata.media_urls(支持带 in_sec/out_sec)
322
+ - 抽帧后拼成 LangChain 多模态消息
323
+ - 根据是否有 media 输入,选择 llm/vlm
324
+ """
325
+
326
+ async def sampling_callback(context, params: CreateMessageRequestParams) -> CreateMessageResult:
327
+ # 1. system prompt
328
+ system_prompt = getattr(params, "systemPrompt", None) or ""
329
+
330
+ # 2. MCP messages(多轮) -> LangChain
331
+ mcp_messages = getattr(params, "messages", []) or []
332
+ lc_messages: List[Any] = []
333
+ if system_prompt:
334
+ lc_messages.append(SystemMessage(content=system_prompt))
335
+
336
+ # 3. metadata:取 media_urls 与 top_p
337
+ metadata = getattr(params, "metadata", None) or {}
338
+ media_inputs = list(metadata.get("media", []) or [])
339
+ top_p: float = float(metadata.get("top_p", 0.9))
340
+
341
+ temperature: float = float(getattr(params, "temperature", None) or 0.6)
342
+ max_tokens: int = int(getattr(params, "maxTokens", 4096) or 4096)
343
+
344
+ # 4. 分流
345
+ use_multimodal = bool(media_inputs)
346
+ model = vlm if use_multimodal else llm
347
+ if model is None:
348
+ model = vlm or llm
349
+
350
+ # 5. 构造媒体 blocks(含视频按段抽帧)——放线程,避免阻塞 event loop
351
+ media_blocks: List[Dict[str, Any]] = []
352
+ if use_multimodal:
353
+ media_blocks = await asyncio.to_thread(
354
+ _build_media_blocks,
355
+ media_inputs,
356
+ resize_edge,
357
+ jpeg_quality,
358
+ min_frames,
359
+ max_frames,
360
+ frames_per_sec,
361
+ global_max_images,
362
+ )
363
+
364
+ # 6. 媒体附加到“最后一条 user 消息”
365
+ user_indices = [i for i, m in enumerate(mcp_messages) if getattr(m, "role", "") == "user"]
366
+ last_user_idx = user_indices[-1] if user_indices else None
367
+
368
+ if not mcp_messages:
369
+ # 无消息则造一条 user
370
+ content_blocks = [{"type": "text", "text": ""}]
371
+ if media_blocks:
372
+ content_blocks.extend(media_blocks)
373
+ lc_messages.append(HumanMessage(content=content_blocks if media_blocks else ""))
374
+ else:
375
+ for i, m in enumerate(mcp_messages):
376
+ role = getattr(m, "role", "") or "user"
377
+ text = _extract_text_from_mcp_content(getattr(m, "content", None))
378
+
379
+ if role == "assistant":
380
+ lc_messages.append(AIMessage(content=text))
381
+ continue
382
+
383
+ if role == "user":
384
+ if last_user_idx is not None and i == last_user_idx and media_blocks:
385
+ content_blocks = [{"type": "text", "text": text}]
386
+ content_blocks.extend(media_blocks)
387
+ lc_messages.append(HumanMessage(content=content_blocks))
388
+ else:
389
+ lc_messages.append(HumanMessage(content=text))
390
+ continue
391
+
392
+ lc_messages.append(HumanMessage(content=text))
393
+
394
+ # 7. 调用所选模型
395
+ bound = model
396
+ try:
397
+ bound = bound.bind(temperature=temperature, max_tokens=max_tokens, top_p=top_p)
398
+ except Exception:
399
+ bound = bound.bind(temperature=temperature, max_tokens=max_tokens)
400
+
401
+ try:
402
+ if hasattr(bound, "ainvoke"):
403
+ resp = await bound.ainvoke(lc_messages)
404
+ else:
405
+ resp = await asyncio.to_thread(bound.invoke, lc_messages)
406
+ except TypeError:
407
+ # 极端情况下:某些 wrapper 不接受 max_tokens/top_p
408
+ bound2 = model.bind(temperature=temperature)
409
+ if hasattr(bound2, "ainvoke"):
410
+ resp = await bound2.ainvoke(lc_messages)
411
+ else:
412
+ resp = await asyncio.to_thread(bound2.invoke, lc_messages)
413
+
414
+ text_out = _extract_text_from_lc_response(resp)
415
+
416
+ model_name = getattr(model, "model", None) or getattr(model, "model_name", None) or "unknown"
417
+ return CreateMessageResult(
418
+ content=TextContent(type="text", text=text_out),
419
+ model=str(model_name),
420
+ role="assistant",
421
+ stopReason="endTurn",
422
+ )
423
+
424
+ return sampling_callback
src/open_storyline/mcp/sampling_requester.py ADDED
@@ -0,0 +1,159 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from typing import Any, Optional, Protocol, runtime_checkable
4
+
5
+ from mcp.server.fastmcp import Context
6
+ from mcp.server.session import ServerSession
7
+ from mcp.types import SamplingMessage, TextContent, ModelHint, ModelPreferences
8
+
9
+ class BaseLLMSampling(Protocol):
10
+ # 低层能力协议:多个工具复用sampling
11
+ async def sampling(
12
+ self,
13
+ *,
14
+ system_prompt: str | None,
15
+ messages: list[SamplingMessage],
16
+ temperature: float = 0.3,
17
+ top_p: float = 0.9,
18
+ max_tokens: int = 4096,
19
+ model_preferences: dict[str, Any] | None = None,
20
+ metadata: dict[str, Any] | None = None,
21
+ stop_sequences: list[str] | None = None,
22
+ ) -> str:
23
+ ...
24
+
25
+ @runtime_checkable
26
+ class LLMClient(Protocol):
27
+ # 高层能力协议:多个工具只区分是否需要用到多模态能力
28
+ async def complete(
29
+ self,
30
+ *,
31
+ system_prompt: str | None,
32
+ user_prompt: str,
33
+ media: list[dict[str, Any]] | None = None,
34
+ temperature: float = 0.3,
35
+ top_p: float = 0.9,
36
+ max_tokens: int = 2048,
37
+ model_preferences: dict[str, Any] | None = None,
38
+ metadata: dict[str, Any] | None = None,
39
+ stop_sequences: list[str] | None = None,
40
+ ) -> str:
41
+ ...
42
+
43
+ class MCPSampler(BaseLLMSampling):
44
+ def __init__(self, mcp_ctx: Context[ServerSession, object]):
45
+ self._mcp_ctx = mcp_ctx
46
+
47
+ def _to_mcp_model_preferences(
48
+ self,
49
+ model_preferences: dict[str, Any] | None,
50
+ ) -> Optional[ModelPreferences]:
51
+ if not model_preferences:
52
+ return None
53
+
54
+ raw_hints = model_preferences.get("hints")
55
+ hints: list[ModelHint] | None = None
56
+ if isinstance(raw_hints, list):
57
+ hints = []
58
+ for h in raw_hints:
59
+ if isinstance(h, ModelHint):
60
+ hints.append(h)
61
+ elif isinstance(h, dict):
62
+ hints.append(ModelHint(**h))
63
+ elif isinstance(h, str):
64
+ hints.append(ModelHint(name=h))
65
+
66
+ return ModelPreferences(
67
+ hints=hints,
68
+ costPriority=model_preferences.get("costPriority"),
69
+ speedPriority=model_preferences.get("speedPriority"),
70
+ intelligencePriority=model_preferences.get("intelligencePriority"),
71
+ )
72
+
73
+ def _extract_text(self, content: Any) -> str:
74
+ # MCP 返回 content 可能是单块或数组;这里只抽取 text block
75
+ if isinstance(content, list):
76
+ texts: list[str] = []
77
+ for block in content:
78
+ if getattr(block, "type", None) == "text":
79
+ texts.append(block.text)
80
+ return "\n".join(texts).strip()
81
+
82
+ if getattr(content, "type", None) == "text":
83
+ return content.text.strip()
84
+
85
+ return str(content)
86
+
87
+ async def sampling(self,
88
+ *,
89
+ system_prompt: str | None,
90
+ messages: list[SamplingMessage],
91
+ temperature: float = 0.3,
92
+ top_p: float = 0.9,
93
+ max_tokens: int = 4096,
94
+ model_preferences: dict[str, Any] | None = None,
95
+ metadata: dict[str, Any] | None = None,
96
+ stop_sequences: list[str] | None = None
97
+ ) -> str:
98
+ merged_metadata = dict(metadata or {})
99
+ merged_metadata["top_p"] = top_p
100
+
101
+ result = await self._mcp_ctx.session.create_message(
102
+ messages=messages,
103
+ max_tokens=max_tokens,
104
+ system_prompt=system_prompt,
105
+ temperature=temperature,
106
+ # stop_sequences=stop_sequences,
107
+ metadata=merged_metadata,
108
+ # model_preferences=self._to_mcp_model_preferences(model_preferences),
109
+ )
110
+ return self._extract_text(result.content)
111
+
112
+ class SamplingLLMClient(LLMClient):
113
+ """
114
+ 只区分是否有media输入
115
+ Server将media路径和时间戳透传给Client,Client负责转base64
116
+ """
117
+
118
+ def __init__(self, sampler: BaseLLMSampling):
119
+ self._sampler = sampler
120
+
121
+ async def complete(self,
122
+ *,
123
+ system_prompt: str | None,
124
+ user_prompt: str,
125
+ media: list[dict[str, Any]] | None = None,
126
+ temperature: float = 0.3,
127
+ top_p: float = 0.9,
128
+ max_tokens: int = 2048,
129
+ model_preferences: dict[str, Any] | None = None,
130
+ metadata: dict[str, Any] | None = None,
131
+ stop_sequences: list[str] | None = None
132
+ )-> str:
133
+ messages = [
134
+ SamplingMessage(
135
+ role="user",
136
+ content=TextContent(type="text", text=user_prompt),
137
+ )
138
+ ]
139
+
140
+ merged_metadata = dict(metadata or {})
141
+ merged_metadata["modality"] = "multimodal" if media else "text"
142
+ if media:
143
+ merged_metadata["media"] = media # 关键:素材路径和时间戳直接透传
144
+
145
+ return await self._sampler.sampling(
146
+ system_prompt=system_prompt,
147
+ messages=messages,
148
+ temperature=temperature,
149
+ top_p=top_p,
150
+ max_tokens=max_tokens,
151
+ model_preferences=model_preferences,
152
+ metadata=merged_metadata,
153
+ stop_sequences=stop_sequences,
154
+ )
155
+
156
+ def make_llm(mcp_ctx: Context[ServerSession, object]) -> LLMClient:
157
+ # 工具里面直接 llm = make_llm(ctx) 就可以调用 llm.complete()
158
+ return SamplingLLMClient(MCPSampler(mcp_ctx))
159
+
src/open_storyline/mcp/server.py ADDED
@@ -0,0 +1,70 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from pathlib import Path
2
+ import os
3
+ from collections.abc import AsyncIterator
4
+ from contextlib import asynccontextmanager
5
+
6
+ # prefix = "/mnt/tidal-alsh-share2/usr/yangqiwei1/tools/ffmpeg_nvenc"
7
+ # cuda = "/usr/local/cuda-12.4"
8
+
9
+ # os.environ["FFMPEG_BINARY"] = f"{prefix}/bin/ffmpeg"
10
+ # os.environ["LD_LIBRARY_PATH"] = f"{prefix}/lib:{prefix}/lib64:{cuda}/targets/x86_64-linux/lib:" + os.environ.get("LD_LIBRARY_PATH", "")
11
+ # os.environ["IMAGEIO_FFMPEG_EXE"] = f"{prefix}/bin/ffmpeg"
12
+
13
+ # os.environ["IMAGEIO_FFMPEG_EXE"] = "/tmp/storyline_venv/lib/python3.13/site-packages/imageio_ffmpeg/binaries/ffmpeg-linux-x86_64-v7.0.2"
14
+ from mcp.server.fastmcp import FastMCP
15
+
16
+ from open_storyline.mcp import register_tools
17
+ from open_storyline.config import load_settings, default_config_path
18
+ from open_storyline.config import Settings
19
+ from src.open_storyline.storage.session_manager import SessionLifecycleManager
20
+ from open_storyline.utils.logging import get_logger
21
+
22
+ logger = get_logger()
23
+
24
+
25
+ def create_server(cfg: Settings) -> FastMCP:
26
+ """
27
+ 负责创建MCP server示例并把各个tools_*模块挂上来
28
+ """
29
+
30
+ # 管理这次运行的上下文对象
31
+ runtime_ctx = cfg
32
+
33
+ @asynccontextmanager
34
+ async def session_lifespan(server: FastMCP) -> AsyncIterator[SessionLifecycleManager]:
35
+ """Manage session lifecycle with type-safe context."""
36
+ # Initialize on startup
37
+ logger.info("Enable session lifespan manager")
38
+ session_manager = SessionLifecycleManager(
39
+ artifacts_root=cfg.project.outputs_dir,
40
+ cache_root=cfg.local_mcp_server.server_cache_dir,
41
+ enable_cleanup=True,
42
+ )
43
+ try:
44
+ yield session_manager
45
+ finally:
46
+ # Cleanup on shutdown
47
+ session_manager.cleanup_expired_sessions()
48
+
49
+ server = FastMCP(
50
+ name=cfg.local_mcp_server.server_name,
51
+ stateless_http=cfg.local_mcp_server.stateless_http,
52
+ json_response=cfg.local_mcp_server.json_response,
53
+ lifespan=session_lifespan,
54
+ )
55
+
56
+ # 把 runtime_ctx 传给 register_tools,让每个工具都能拿到 cfg/artifact_store/session_id
57
+ register_tools.register(server, runtime_ctx)
58
+
59
+ return server
60
+
61
+ def main():
62
+ cfg = load_settings(default_config_path())
63
+ server = create_server(cfg)
64
+ server.settings.host = cfg.local_mcp_server.connect_host
65
+ server.settings.port = cfg.local_mcp_server.port
66
+ server.run(transport=cfg.local_mcp_server.server_transport)
67
+
68
+ if __name__ == "__main__":
69
+ main()
70
+ # PYTHONPATH=src python -m open_storyline.mcp.server
src/open_storyline/nodes/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # 导入管理
src/open_storyline/nodes/core_nodes/assets_index.py ADDED
@@ -0,0 +1,170 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Any, Dict, Optional, ClassVar, Type
2
+ from pydantic import BaseModel
3
+ import traceback
4
+ from collections import Counter
5
+ from pathlib import Path
6
+ from moviepy.video.io.ffmpeg_reader import ffmpeg_parse_infos
7
+
8
+
9
+ from open_storyline.nodes.core_nodes.base_node import NodeMeta, BaseNode
10
+ from open_storyline.nodes.node_schema import AssetsIndexInput, AssetsIndexOutput
11
+ from open_storyline.nodes.node_summary import NodeSummary
12
+
13
+
14
+ VIDEO_EXTS = {
15
+ ".mp4", ".mov", ".mkv", ".avi"
16
+ }
17
+ IMAGE_EXTS = {
18
+ ".jpg", ".jpeg", ".png", ".webp", ".bmp"
19
+ }
20
+
21
+ def _image_metadata_from_path(path: Path) -> dict[str, Any]:
22
+ from PIL import Image, ImageOps
23
+
24
+ with Image.open(path) as img:
25
+ try:
26
+ img2 = ImageOps.exif_transpose(img)
27
+ w, h = img2.size
28
+ except Exception:
29
+ w, h = img.size
30
+
31
+ return {
32
+ "width": int(w),
33
+ "height": int(h),
34
+ }
35
+
36
+
37
+ import av
38
+ from fractions import Fraction
39
+ from typing import Any, Optional
40
+ from pathlib import Path
41
+
42
+
43
+ def _video_metadata_from_path(
44
+ path: Path,
45
+ *,
46
+ round_duration_ndigits: Optional[int] = 3,
47
+ ) -> dict[str, Any]:
48
+
49
+ container = av.open(str(path))
50
+
51
+ # 找第一个视频流
52
+ video_stream = next(
53
+ (s for s in container.streams if s.type == "video"),
54
+ None,
55
+ )
56
+ if video_stream is None:
57
+ raise ValueError(f"No video stream found: {path}")
58
+
59
+ # ---------- duration ----------
60
+ duration_sec = 0.0
61
+
62
+ if container.duration is not None:
63
+ # container.duration 单位是 microseconds
64
+ duration_sec = container.duration / 1_000_000
65
+ elif video_stream.duration is not None and video_stream.time_base is not None:
66
+ duration_sec = float(video_stream.duration * video_stream.time_base)
67
+
68
+ if round_duration_ndigits is not None:
69
+ duration_sec = round(duration_sec, round_duration_ndigits)
70
+
71
+ # ---------- width / height / rotation ----------
72
+ w = int(video_stream.codec_context.width or 0)
73
+ h = int(video_stream.codec_context.height or 0)
74
+
75
+ rotation = 0
76
+ if video_stream.metadata:
77
+ rot = video_stream.metadata.get("rotate")
78
+ if rot is not None:
79
+ try:
80
+ rotation = int(rot)
81
+ except Exception:
82
+ rotation = 0
83
+
84
+ if abs(rotation) in (90, 270):
85
+ w, h = h, w
86
+
87
+ # ---------- fps ----------
88
+ fps = 0.0
89
+ if video_stream.average_rate:
90
+ fps = float(video_stream.average_rate)
91
+ elif video_stream.base_rate:
92
+ fps = float(video_stream.base_rate)
93
+
94
+ # ---------- audio ----------
95
+ audio_stream = next(
96
+ (s for s in container.streams if s.type == "audio"),
97
+ None,
98
+ )
99
+
100
+ has_audio = audio_stream is not None
101
+ audio_sample_rate_hz = int(audio_stream.rate) if audio_stream and audio_stream.rate else 0
102
+
103
+ container.close()
104
+
105
+ return {
106
+ "duration": int(duration_sec * 1000), # ms
107
+ "width": w,
108
+ "height": h,
109
+ "fps": fps,
110
+ "has_audio": has_audio,
111
+ "audio_sample_rate_hz": audio_sample_rate_hz,
112
+ }
113
+
114
+
115
+ class AssetsIndexNode(BaseNode):
116
+ meta = NodeMeta(
117
+ name="assets_index",
118
+ description="读取输入素材,进行索引,没有前置依赖,但它是所有后续操作的基础。",
119
+ node_id="assets_index",
120
+ node_kind="assets_index",
121
+ next_available_node=['split_shots', 'xhs_split_shots'],
122
+ )
123
+ input_schema: ClassVar[Type[BaseModel]] = AssetsIndexInput
124
+ # output_schema: ClassVar[Type[BaseModel]] = AssetsIndexOutput
125
+
126
+ async def default_process(self, node_summary: NodeSummary, inputs: Dict[str, Any]) -> Dict[str, Any]:
127
+ return await self.process(node_summary, inputs)
128
+
129
+ async def process(self, node_summary: NodeSummary, inputs: Dict[str, Any]) -> Dict[str, Any]:
130
+ input_assets = inputs.get('inputs', [])
131
+
132
+ asset_idx = 1
133
+ assets = []
134
+ for enc_asset in input_assets:
135
+ path = Path(enc_asset['path'])
136
+ suffix = path.suffix.lower()
137
+
138
+ if suffix in VIDEO_EXTS:
139
+ metadata = _video_metadata_from_path(path)
140
+ media_type = "video"
141
+ elif suffix in IMAGE_EXTS:
142
+ metadata = _image_metadata_from_path(path)
143
+ media_type = "image"
144
+ else:
145
+ node_summary.info_for_user(f"[Node {self.meta.node_id}] Skipping unsupported file type `{enc_asset['orig_path']}` ")
146
+ continue
147
+
148
+ assets.append(
149
+ {
150
+ "asset_id": f"asset_{asset_idx:04d}",
151
+ "path": path,
152
+ "media_type": media_type,
153
+ "metadata": metadata,
154
+ "orig_path": enc_asset['orig_path'],
155
+ "orig_md5": enc_asset['orig_md5'],
156
+ }
157
+ )
158
+ node_summary.info_for_user(f"Added asset_{asset_idx:04d}: ({media_type})")
159
+ asset_idx += 1
160
+
161
+ c = Counter(
162
+ (a.get("media_type") or "").strip().lower()
163
+ for a in assets
164
+ if isinstance(a, dict)
165
+ )
166
+
167
+ # node_summary.info_for_user(f"素材索引执行成功:共{c.get('video', 0)}个视频素材,{c.get('image', 0)}个照片素材")
168
+ node_summary.info_for_user(f"[Node {self.meta.node_id}] Media indexing completed successfully: {c.get('video', 0)} video(s), {c.get('image', 0)} image(s)",)
169
+
170
+ return {"assets": assets}
src/open_storyline/nodes/core_nodes/base_node.py ADDED
@@ -0,0 +1,243 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from abc import ABC, abstractmethod
2
+ import os
3
+ from pathlib import Path
4
+ from dataclasses import dataclass, field
5
+ from pydantic import BaseModel, ValidationError
6
+ from typing import Any, Dict, List, Optional, Union, ClassVar
7
+ import json
8
+ import traceback
9
+
10
+ from open_storyline.config import Settings
11
+ from open_storyline.nodes.node_summary import NodeSummary
12
+ from open_storyline.storage.file import FileCompressor
13
+ from open_storyline.utils.logging import get_logger
14
+ from open_storyline.mcp.sampling_requester import LLMClient
15
+
16
+ logger = get_logger(__name__)
17
+
18
+ @dataclass
19
+ class NodeMeta:
20
+ """
21
+ 节点元数据类
22
+
23
+ 用于定义工作流或流程图中节点的元信息,包括节点标识、类型、
24
+ 依赖关系以及可流转的下游节点等配置信息。
25
+
26
+ Attributes:
27
+ name: 工具名
28
+ description: 工具功能描述信息
29
+ node_id: 节点唯一标识符,用于在整个流程中唯一定位该节点
30
+ node_kind: 节点类型/种类,如 "start"、"process"、"end" 等
31
+ require_prior_kind: 当前节点运行前必须完成的前置节点类型列表
32
+ 例如:["validation", "authentication"]
33
+ default_require_prior_kind: 默认的前置节点类型列表,作为 require_prior_kind
34
+ 的默认配置或备选方案
35
+ next_available_node: 当前节点执行完成后可以流转到的下游节点ID列表
36
+ 用于定义流程的可能分支路径
37
+ priority: 相同功能节点执行优先级
38
+ """
39
+ name: str
40
+ description: str
41
+ node_id: str
42
+ node_kind: str
43
+ require_prior_kind: List[str] = field(default_factory=list)
44
+ default_require_prior_kind: List[str] = field(default_factory=list)
45
+ next_available_node: List[str] = field(default_factory=list)
46
+ priority: int = 5
47
+ # deployment_type: str = "local" # 默认本地,可选 "local" 或 "remote"
48
+
49
+
50
+
51
+ class BaseNode(ABC):
52
+ meta: NodeMeta
53
+ input_schema: ClassVar[type[BaseModel] | None] = None
54
+ # output_schema: ClassVar[type[BaseModel] | None] = None
55
+
56
+ def __init__(self, server_cfg: Settings) -> None:
57
+ self.server_cfg = server_cfg
58
+ self.server_cache_dir = Path(os.getcwd()) / self.server_cfg.local_mcp_server.server_cache_dir
59
+
60
+ if not hasattr(self, "meta"):
61
+ raise ValueError("Subclass must define the 'meta' attribute")
62
+
63
+
64
+ def _load_user_info(self, node_summary: NodeSummary, params: Dict[str,Any]) -> Dict[str,Any]:
65
+ return {
66
+ "session_id": params.get("session_id"),
67
+ "artifact_id": params.get('artifact_id')
68
+ }
69
+
70
+ def _load_item(self, node_summary: NodeSummary, user_info: Dict[str,str], item: Dict[str,Any]):
71
+ new_item:Dict[str,Any] = {}
72
+ item_base64 = item.pop("base64", None)
73
+ item_md5 = item.pop("md5", None)
74
+ item_path = item.pop("path", None)
75
+ new_item.update(item)
76
+
77
+
78
+ if item_base64 and item_path:
79
+ item_save_path = self.server_cache_dir / user_info['session_id']/ user_info['artifact_id'] / os.path.basename(item_path)
80
+ FileCompressor.decompress_from_string(item_base64, item_save_path)
81
+ new_item['path'] = str(item_save_path.relative_to(os.getcwd()))
82
+ new_item['orig_path'] = str(item_path)
83
+ new_item['orig_md5'] = item_md5
84
+ return new_item
85
+
86
+ def _pack_item(self, node_summary: NodeSummary, item: Dict[str,Any]):
87
+ orig_path = item.pop('orig_path', None)
88
+ orig_md5 = item.pop('orig_md5', None)
89
+ server_save_path = item.pop('path', None)
90
+ if server_save_path:
91
+ compress_data = FileCompressor.compress_and_encode(server_save_path)
92
+ if orig_path and orig_md5 and compress_data.md5 == orig_md5:
93
+ node_summary.debug_for_dev(f"[node] node_id: {self.meta.node_id} change `path` change to {orig_path}")
94
+ item['path'] = orig_path
95
+ elif orig_md5 is None or compress_data.md5 != orig_md5:
96
+ node_summary.debug_for_dev(f"[node] node_id: {self.meta.node_id} return `base64` to client")
97
+ item['base64'] = compress_data.base64
98
+ item['path'] = compress_data.filename
99
+ item['md5'] = compress_data.md5
100
+ return item
101
+
102
+
103
+ def load_inputs_from_client(self, node_summary: NodeSummary, params: Dict[str,Any], user_info: Optional[Dict[str,str]] = None, save: bool=True) -> Dict[str, Any]:
104
+ """
105
+ Read data from client's request and save the transmitted base64 data on the Server.
106
+ """
107
+
108
+ if user_info is None:
109
+ user_info = self._load_user_info(node_summary, params)
110
+
111
+ payload_key = params.keys()
112
+ loaded_input = {}
113
+ kwargs = {}
114
+ for k in payload_key:
115
+ payload_input = params[k]
116
+ if isinstance(payload_input, list) and all([isinstance(item, dict) for item in payload_input]):
117
+ # List: load base64 data and save to server cache
118
+ loaded_input[k] = [self._load_item(node_summary, user_info, item) for item in payload_input]
119
+ elif isinstance(payload_input, dict):
120
+ # Dict: recursively process nested data (without saving)
121
+ loaded_input[k] = self.load_inputs_from_client(node_summary, payload_input, user_info, save=False)
122
+ elif isinstance(payload_input, LLMClient):
123
+ kwargs[k] = payload_input
124
+ else:
125
+ # Handle primitive types: directly copy the value (e.g., str, int, float, bool)
126
+ loaded_input[k] = params[k]
127
+
128
+ # save loaded_input to self.cache_dir in json format
129
+ if save:
130
+ artifact_save_path = (self.server_cache_dir / user_info['session_id'] / user_info['artifact_id']).with_suffix(".json")
131
+ artifact_save_path.parent.mkdir(parents=True, exist_ok=True)
132
+ with open(artifact_save_path, 'w') as f:
133
+ json.dump(loaded_input, f, indent=2, ensure_ascii=False)
134
+ loaded_input.update(kwargs)
135
+ return loaded_input
136
+
137
+
138
+ def pack_outputs_to_client(self, node_summary: NodeSummary, outputs: Union[Dict[str,Any],List[str]]) -> Union[Dict[str,Any],List[str]]:
139
+ """
140
+ Pack the output and return it to the client.
141
+ """
142
+ if not isinstance(outputs, dict):
143
+ return outputs
144
+ payload_key = outputs.keys()
145
+ packed_output = {}
146
+ for k in payload_key:
147
+ payload_output = outputs[k]
148
+ if isinstance(payload_output, list) and all(isinstance(item, dict) for item in payload_output):
149
+ packed_output[k] = [self._pack_item(node_summary, item) for item in payload_output]
150
+ elif isinstance(payload_output, dict):
151
+ packed_output[k] = self.pack_outputs_to_client(node_summary, payload_output)
152
+ else:
153
+ packed_output[k] = outputs[k]
154
+ return packed_output
155
+
156
+ @abstractmethod
157
+ async def default_process(self, node_summary: NodeSummary, inputs: Dict[str, Any]) -> Any:
158
+ """
159
+ Default processing method that must be implemented. Called when the node needs to be skipped.
160
+ """
161
+ ...
162
+
163
+ @abstractmethod
164
+ async def process(self, node_summary: NodeSummary, inputs: Dict[str, Any]) -> Any:
165
+ """
166
+ Main processing method that must be implemented. Executed when the node is invoked normally.
167
+ """
168
+ ...
169
+
170
+ def _parse_input(self, node_summary: NodeSummary, inputs: Dict[str, Any]):
171
+ return inputs
172
+
173
+ def _combine_tool_outputs(self, node_summary: NodeSummary, outputs: Dict[str, Any]):
174
+ return outputs
175
+
176
+ def _validate_schema(
177
+ self,
178
+ params: dict[str,Any],
179
+ schema_name: Union[str, List[str]],
180
+ update_params: bool = False
181
+ ) -> Optional[Dict[str,Any]]:
182
+ schema_names = [schema_name] if isinstance(schema_name, str) else schema_name
183
+
184
+ validated_params = params.copy() if update_params else None
185
+
186
+ for name in schema_names:
187
+ schema = getattr(self, name, None)
188
+
189
+ if schema is None:
190
+ logger.warning(
191
+ f"Schema '{name}' 不存在,跳过校验",
192
+ )
193
+ continue
194
+
195
+ try:
196
+ validated = schema(**params)
197
+ if update_params:
198
+ validated_params.update(validated.dict())
199
+ except ValidationError as e:
200
+ logger.error(f"{name} 校验失败: {e}")
201
+ return validated_params if update_params else None
202
+
203
+ async def __call__(self, **params) -> Dict[str, Any]:
204
+ try:
205
+ mode = params.get("mode", "auto")
206
+ node_summary = params.pop("node_summary")
207
+
208
+ inputs = self.load_inputs_from_client(node_summary, params.copy())
209
+
210
+ parsed_inputs = self._parse_input(node_summary, inputs)
211
+
212
+ if mode != 'auto':
213
+ outputs = await self.default_process(node_summary, parsed_inputs)
214
+ else:
215
+ outputs = await self.process(node_summary, parsed_inputs)
216
+
217
+ processed_outputs = self._combine_tool_outputs(node_summary, outputs)
218
+
219
+ packed_output = self.pack_outputs_to_client(node_summary, processed_outputs)
220
+
221
+ # self._validate_schema(packed_output, 'output_schema')
222
+
223
+ return {
224
+ 'artifact_id': params['artifact_id'],
225
+ 'summary': node_summary.get_summary(params['artifact_id']),
226
+ 'tool_excute_result': packed_output,
227
+ 'isError': False
228
+ }
229
+ except Exception as e:
230
+ if self.server_cfg.project.debug_mode:
231
+ traceback_info = ''.join(traceback.format_exception(e))
232
+ summary = {
233
+ "error_info": f"[artifact_id {params['artifact_id']}] \n {traceback_info}"
234
+ }
235
+ logger.error(traceback_info)
236
+ else:
237
+ summary = node_summary.get_summary(params['artifact_id'])
238
+ return {
239
+ 'artifact_id': params['artifact_id'],
240
+ 'summary': summary,
241
+ 'tool_excute_result': {},
242
+ 'isError': True
243
+ }
src/open_storyline/nodes/core_nodes/filter_clips.py ADDED
@@ -0,0 +1,149 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Any, Dict
2
+
3
+ from open_storyline.nodes.core_nodes.base_node import BaseNode, NodeMeta
4
+ from open_storyline.nodes.node_schema import FilterClipsInput
5
+ from open_storyline.mcp.sampling_requester import LLMClient
6
+ from open_storyline.mcp.sampling_requester import LLMClient
7
+ from src.open_storyline.utils.prompts import get_prompt
8
+ from open_storyline.utils.parse_json import parse_json_dict
9
+
10
+ class FilterClipsNode(BaseNode):
11
+
12
+ meta = NodeMeta(
13
+ name="filter_clips",
14
+ description="按照用户要求,根据clips的描述,对clips进行筛选。依赖 understand_clips 工具的结果",
15
+ node_id="filter_clips",
16
+ node_kind="filter_clips",
17
+ require_prior_kind=['understand_clips'],
18
+ default_require_prior_kind=['understand_clips'],
19
+ next_available_node=['group_clips', 'xhs_group_clips'],
20
+ )
21
+
22
+ input_schema = FilterClipsInput
23
+
24
+ def _parse_input(self, node_summary, inputs: Dict[str, Any]):
25
+ clip_captions = inputs["understand_clips"].get("clip_captions")
26
+ input_clip_ids: list[str] = [
27
+ (c.get("clip_id")) for c in clip_captions
28
+ ]
29
+ inputs["input_clip_ids"] = input_clip_ids
30
+ return inputs
31
+
32
+
33
+ async def default_process(
34
+ self,
35
+ node_summary,
36
+ inputs: Dict[str, Any],
37
+ ) -> Any:
38
+ clip_captions = inputs["understand_clips"].get("clip_captions")
39
+
40
+ node_summary.info_for_user("使用全部clips")
41
+ return {
42
+ "clip_captions": clip_captions,
43
+ "selected": inputs["input_clip_ids"],
44
+ }
45
+
46
+ async def process(self, node_summary, inputs: Dict[str, Any]) -> Any:
47
+ clip_captions = inputs["understand_clips"].get("clip_captions")
48
+ user_request = inputs["user_request"]
49
+ llm: LLMClient = inputs["llm"]
50
+
51
+ input_clip_ids = inputs["input_clip_ids"]
52
+
53
+ if not user_request or user_request == "":
54
+ node_summary.info_for_user("用户没有提出具体需求,使用全部clips")
55
+ return {
56
+ "clip_captions": clip_captions,
57
+ "selected": input_clip_ids,
58
+ }
59
+
60
+ else:
61
+ clip_block = _build_clips_block(clip_captions)
62
+ system_prompt = get_prompt("filter_clips.system")
63
+ user_prompt = get_prompt("filter_clips.user", user_request=user_request, clip_captions=clip_block)
64
+
65
+ raw = await llm.complete(
66
+ system_prompt=system_prompt,
67
+ user_prompt=user_prompt,
68
+ media=None,
69
+ temperature=0.1,
70
+ top_p=0.9,
71
+ max_tokens=2048,
72
+ model_preferences=None,
73
+ )
74
+
75
+ try:
76
+ obj = parse_json_dict(raw)
77
+ select_ids = _extract_selected_ids(obj, input_clip_ids)
78
+ node_summary.info_for_user(f"成功筛选到{len(select_ids)}个片段")
79
+
80
+ except:
81
+ select_ids = input_clip_ids
82
+ node_summary.info_for_user("解析大模型输出出错,使用全部片段")
83
+
84
+ return {
85
+ "clip_captions": clip_captions,
86
+ "selected": select_ids,
87
+ }
88
+
89
+
90
+ def _extract_selected_ids(
91
+ obj: dict[str, Any],
92
+ input_clip_ids: list[str],
93
+ ) -> list[str]:
94
+ """
95
+ 从 LLM 的结构化输出中抽取 selected clip_id 列表。
96
+ 返回:按 input_clip_ids 原顺序的去噪结果(仅保留输入中存在的 id)
97
+ """
98
+ id_set = set(input_clip_ids)
99
+
100
+ # --- Case 2: results keep ---
101
+ results = obj.get("results")
102
+ if not isinstance(results, list):
103
+ raise ValueError('"results" 必须是 list')
104
+
105
+ true_items = 0
106
+ valid_true_ids: set[str] = set()
107
+
108
+ for item in results:
109
+ if not isinstance(item, dict):
110
+ continue
111
+ cid = item.get("clip_id")
112
+ keep = item.get("keep")
113
+
114
+ # keep 允许 bool 或 "true"/"false"
115
+ keep_bool = None
116
+ if isinstance(keep, bool):
117
+ keep_bool = keep
118
+ elif isinstance(keep, str):
119
+ s = keep.strip().lower()
120
+ if s in ("true", "yes", "1"):
121
+ keep_bool = True
122
+ elif s in ("false", "no", "0"):
123
+ keep_bool = False
124
+
125
+ if keep_bool is True:
126
+ true_items += 1
127
+ if isinstance(cid, str) and cid in id_set:
128
+ valid_true_ids.add(cid)
129
+
130
+ # 如果模型明确选了(keep=true)但一个都对不上输入
131
+ if true_items > 0 and not valid_true_ids:
132
+ raise ValueError("results 中存在 keep=true,但没有任何有效 clip_id(可能模型改写了 ID)")
133
+
134
+ return [cid for cid in input_clip_ids if cid in valid_true_ids]
135
+
136
+ def _build_clips_block(clip_captions: list[dict[str, Any]]) -> str:
137
+ """
138
+ 将 clips 构造成稳定的文本块
139
+ """
140
+ blocks: list[str] = []
141
+ for clip in clip_captions:
142
+ cid = clip.get("clip_id", "")
143
+ caption = clip.get("caption", "")
144
+ block = (
145
+ f"[clip_id={cid}]\n"
146
+ f"caption: {caption}\n"
147
+ )
148
+ blocks.append(block)
149
+ return "\n".join(blocks).strip() + "\n"
src/open_storyline/nodes/core_nodes/generate_script.py ADDED
@@ -0,0 +1,306 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Any, Dict
2
+ import re
3
+
4
+ from open_storyline.mcp.sampling_requester import LLMClient
5
+ from open_storyline.nodes.core_nodes.base_node import BaseNode, NodeMeta
6
+ from open_storyline.nodes.node_schema import GenerateScriptInput
7
+ from src.open_storyline.utils.prompts import get_prompt
8
+ from open_storyline.utils.parse_json import parse_json_dict
9
+
10
+ class GenerateScriptNode(BaseNode):
11
+ meta = NodeMeta(
12
+ name="generate_script",
13
+ description="生成视频文案,可用于生成语音,或直接作为字幕叠加到视频上",
14
+ node_id="generate_script",
15
+ node_kind="generate_script",
16
+ require_prior_kind=['split_shots','group_clips','understand_clips'],
17
+ default_require_prior_kind=['split_shots','group_clips'],
18
+ next_available_node=['generate_voiceover', 'xhs_tts_service'],
19
+ )
20
+
21
+ input_schema = GenerateScriptInput
22
+
23
+ async def default_process(
24
+ self,
25
+ node_summary,
26
+ inputs: Dict[str, Any],
27
+ ) -> Any:
28
+ return {
29
+ "group_scripts": [],
30
+ "title": "",
31
+ }
32
+
33
+
34
+ async def process(self, node_summary, inputs: Dict[str, Any]) -> Any:
35
+ clip_info = inputs["split_shots"]["clips"]
36
+ clip_captions = inputs["understand_clips"]["clip_captions"]
37
+ overall = inputs["understand_clips"]["overall"]
38
+ groups = inputs["group_clips"]["groups"]
39
+ user_request = inputs["user_request"]
40
+ llm: LLMClient = inputs["llm"]
41
+
42
+ duration_lookup = _build_duration_lookup(clip_info)
43
+ caption_lookup = _build_caption_lookup(clip_captions)
44
+
45
+ group_ids: list[str] = [g.get("group_id","") for g in (groups or []) if g.get("group_id")]
46
+ group_ids_set = set(group_ids)
47
+
48
+ if not group_ids:
49
+ return {"group_scripts": []}
50
+
51
+ groups_block = _build_groups_block_for_script(groups, duration_lookup, caption_lookup)
52
+
53
+ system_prompt = get_prompt("generate_script.system")
54
+ if not user_request or user_request == "":
55
+ user_request = "无要求"
56
+ user_prompt = get_prompt("generate_script.user", user_request=user_request, overall=overall, groups=groups_block)
57
+
58
+ raw = await llm.complete(
59
+ system_prompt=system_prompt,
60
+ user_prompt=user_prompt,
61
+ temperature=0.1,
62
+ top_p=0.9,
63
+ max_tokens=4096,
64
+ model_preferences=None,
65
+ )
66
+ group_text_map: dict[str, str] = {}
67
+ try:
68
+ obj = parse_json_dict(raw)
69
+ group_text_map = _extract_group_text_map(obj, group_ids)
70
+ except Exception:
71
+ group_text_map = {}
72
+
73
+ # 最终 group_scripts
74
+ group_scripts: list[dict[str, Any]] = []
75
+ subtitle_index = 1
76
+
77
+ for g in groups or []:
78
+ gid = g.get("group_id", "")
79
+ if not gid or gid not in group_ids_set:
80
+ continue
81
+
82
+ duration_sec = 0.0
83
+ for cid in (g.get("clip_ids") or []):
84
+ duration_sec += float(duration_lookup.get(cid, 0.0))
85
+ budget = _estimate_script_budget(duration_sec)
86
+
87
+ raw_text = (group_text_map.get(gid) or "").strip()
88
+ if not raw_text:
89
+ raise ValueError(f"LLM未生成任何文案,请重试")
90
+
91
+ max_chars = budget.get("max_chars", 60)
92
+ if len(raw_text) > int(max_chars * 2.0):
93
+ raw_text = raw_text[:max_chars].rstrip()
94
+
95
+ units, subtitle_index = _make_subtitle_units(
96
+ raw_text=raw_text,
97
+ subtitle_start_index=subtitle_index,
98
+ )
99
+
100
+ group_scripts.append(
101
+ {
102
+ "group_id": gid,
103
+ "raw_text": raw_text,
104
+ "subtitle_units": units,
105
+ }
106
+ )
107
+
108
+ return {
109
+ "group_scripts": group_scripts,
110
+ "title": obj.get("title", ""),
111
+ }
112
+
113
+
114
+
115
+ def _build_duration_lookup(clip_info: list[dict[str, Any]]) -> dict[str, float]:
116
+ """
117
+ clip_id -> duration_sec
118
+ """
119
+ out: dict[str, float] = {}
120
+ for item in clip_info or []:
121
+ cid = item.get("clip_id")
122
+ if not cid:
123
+ continue
124
+ src = item.get("source_ref") or {}
125
+ dur = src.get("duration", 0) / 1000.0
126
+ out[cid] = dur
127
+ return out
128
+
129
+ def _build_caption_lookup(clip_captions: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
130
+ """
131
+ clip_id -> caption_obj
132
+ """
133
+
134
+ out: dict[str, dict[str, Any]] = {}
135
+ for item in clip_captions:
136
+ if not isinstance(item, dict):
137
+ continue
138
+ cid = item.get("clip_id")
139
+ if cid:
140
+ out[cid] = item
141
+ return out
142
+
143
+ def _estimate_script_budget(duration_sec: float) -> dict[str, Any]:
144
+ """
145
+ 根据组时长估算“字数/字符数预算”和建议字幕分段数。
146
+ """
147
+ if duration_sec is None:
148
+ duration_sec = 0.0
149
+ duration_sec = max(0.0, float(duration_sec))
150
+
151
+ min_chars = int(round(duration_sec * 3))
152
+ max_chars = int(round(duration_sec * 5))
153
+
154
+ # 防止极短组变成 0
155
+ min_chars = max(min_chars, 8)
156
+ max_chars = max(max_chars, min_chars + 6)
157
+
158
+ # 建议字幕单元数量:大约每 2 秒 1 句
159
+ target_units = max(1, int(round(duration_sec / 2.0)))
160
+ target_units = min(target_units, 8) # 防止过多切片
161
+
162
+ return {
163
+ "duration_sec": duration_sec,
164
+ "min_chars": min_chars,
165
+ "max_chars": max_chars,
166
+ "target_units": target_units,
167
+ }
168
+
169
+
170
+ def _build_groups_block_for_script(
171
+ groups: list[dict[str, Any]],
172
+ duration_lookup: dict[str, float],
173
+ caption_lookup: dict[str, dict[str, Any]],
174
+ *,
175
+ max_caption_len: int = 120,
176
+ ) -> str:
177
+ """
178
+ 将 groups + clip captions + 时长预算拼成稳定文本块,给 LLM 生成每组文案。
179
+ """
180
+ blocks: list[str] = []
181
+
182
+ for g in groups or []:
183
+ gid = g.get("group_id", "")
184
+ clip_ids = g.get("clip_ids") or []
185
+ if not gid or not isinstance(clip_ids, list) or not clip_ids:
186
+ continue
187
+
188
+ group_summary = (g.get("summary") or "").strip()
189
+
190
+ # 组时长
191
+ duration_sec = 0.0
192
+ for cid in clip_ids:
193
+ duration_sec += float(duration_lookup.get(cid, 0.0))
194
+ budget = _estimate_script_budget(duration_sec)
195
+
196
+ lines: list[str] = []
197
+ lines.append(f"[group_id={gid}]")
198
+ if group_summary:
199
+ lines.append(f"summary: {group_summary}")
200
+ lines.append(f"duration_sec: {budget['duration_sec']:.2f}")
201
+ lines.append(f"script_chars_budget: {budget['min_chars']}~{budget['max_chars']}")
202
+ lines.append(f"subtitle_units_target: {budget['target_units']} (用逗号/句号分段)")
203
+
204
+ lines.append("clips:")
205
+ for cid in clip_ids:
206
+ cap_obj = caption_lookup.get(cid, {})
207
+ cap_text = cap_obj.get("caption", "")
208
+ sem = cap_obj.get("semantic") or {}
209
+ kw = sem.get("keywords") or []
210
+ mood = sem.get("mood") or []
211
+ kw_s = "、".join([x for x in kw if isinstance(x, str)])[:40]
212
+ mood_s = "、".join([x for x in mood if isinstance(x, str)])[:30]
213
+
214
+ dur = duration_lookup.get(cid, 0.0)
215
+
216
+ lines.append(f"- {cid} ({dur:.2f}s): {cap_text}")
217
+ if kw_s or mood_s:
218
+ lines.append(f" tags_hint: keywords={kw_s} | mood={mood_s}")
219
+
220
+ blocks.append("\n".join(lines))
221
+
222
+ return "\n\n".join(blocks).strip()
223
+
224
+ def _extract_group_text_map(obj: Any, group_ids: list[str]) -> dict[str, str]:
225
+ """
226
+ 从 LLM JSON 中抽取 {group_id: raw_text} 映射。
227
+ 兼容几种常见输出:
228
+ 1) {"scripts":[{"group_id":"group_0001","raw_text":"..."}, ...]}
229
+ 2) {"group_scripts":[{"group_id":"...","raw_text":"..."}, ...]}
230
+ 3) {"group_0001":"...", "group_0002":"..."}
231
+ 4) [{"group_id":"...","raw_text":"..."}]
232
+ """
233
+ gid_set = set(group_ids)
234
+ out: dict[str, str] = {}
235
+
236
+ def _add(gid: Any, text: Any):
237
+ if isinstance(gid, str) and gid in gid_set and isinstance(text, str) and text.strip():
238
+ out[gid] = text.strip()
239
+
240
+ if isinstance(obj, dict):
241
+ # list 型
242
+ for key in ("scripts", "group_scripts", "results"):
243
+ v = obj.get(key)
244
+ if isinstance(v, list):
245
+ for item in v:
246
+ if isinstance(item, dict):
247
+ gid = item.get("group_id")
248
+ text = item.get("raw_text") or item.get("text") or item.get("script")
249
+ _add(gid, text)
250
+
251
+ # mapping 型:{"group_0001":"..."}
252
+ for gid in group_ids:
253
+ if gid in obj and isinstance(obj[gid], str):
254
+ _add(gid, obj[gid])
255
+
256
+ return out
257
+
258
+ if isinstance(obj, list):
259
+ for item in obj:
260
+ if isinstance(item, dict):
261
+ gid = item.get("group_id")
262
+ text = item.get("raw_text") or item.get("text") or item.get("script")
263
+ _add(gid, text)
264
+ return out
265
+
266
+ raise ValueError("无法识别 LLM 输出结构")
267
+
268
+
269
+ _SPLIT_RE = re.compile(r"[,,。]+")
270
+
271
+
272
+ def _split_by_comma(raw_text: str) -> list[str]:
273
+ """
274
+ 逗号拆分:支持中文/英文逗号和中文句号。去掉空段。
275
+ """
276
+ if not isinstance(raw_text, str):
277
+ return []
278
+ s = raw_text.strip().replace("\n", ",")
279
+ parts = [p.strip() for p in _SPLIT_RE.split(s) if p and p.strip()]
280
+ return parts
281
+
282
+
283
+ def _make_subtitle_units(
284
+ raw_text: str,
285
+ subtitle_start_index: int,
286
+ ) -> tuple[list[dict[str, Any]], int]:
287
+ """
288
+ 为某个 group 生成 subtitle_units,返回 (units, next_global_index)
289
+ unit_id 全局递增:subtitle_0001, subtitle_0002 ...
290
+ """
291
+ parts = _split_by_comma(raw_text)
292
+ if not parts and raw_text.strip():
293
+ parts = [raw_text.strip()]
294
+
295
+ units: list[dict[str, Any]] = []
296
+ cur = subtitle_start_index
297
+ for idx_in_group, text in enumerate(parts):
298
+ units.append(
299
+ {
300
+ "unit_id": f"subtitle_{cur:04d}",
301
+ "index_in_group": idx_in_group,
302
+ "text": text,
303
+ }
304
+ )
305
+ cur += 1
306
+ return units, cur
src/open_storyline/nodes/core_nodes/generate_voiceover.py ADDED
@@ -0,0 +1,353 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import asyncio
2
+ import base64
3
+ import time
4
+ import uuid
5
+ import wave
6
+ from dataclasses import dataclass
7
+ from pathlib import Path
8
+ from typing import Any, Dict, Optional, Callable
9
+
10
+ import requests
11
+
12
+ from open_storyline.nodes.core_nodes.base_node import BaseNode, NodeMeta
13
+ from open_storyline.nodes.node_schema import GenerateVoiceoverInput
14
+
15
+ class TTSConfigError(ValueError):
16
+ """TTS 配置缺失或不合法"""
17
+
18
+
19
+ class TTSRequestError(RuntimeError):
20
+ """TTS 请求失败(网络/服务端/返回格式等)"""
21
+
22
+
23
+ def _mask_secret(s: str, keep_tail: int = 4) -> str:
24
+ s = str(s or "")
25
+ if not s:
26
+ return ""
27
+ if len(s) <= keep_tail:
28
+ return "***"
29
+ return "***" + s[-keep_tail:]
30
+
31
+
32
+ def _wav_duration_ms(wav_path: Path) -> int:
33
+ with wave.open(str(wav_path), "rb") as wf:
34
+ frames = wf.getnframes()
35
+ rate = wf.getframerate()
36
+ return int((frames / float(rate) * 1000)) if rate else 0
37
+
38
+
39
+ # -----------------------------
40
+ # Provider abstraction + registry
41
+ # -----------------------------
42
+ class BaseTTSProvider:
43
+ """
44
+ 统一的 TTS Provider 抽象。
45
+ - 新增厂家时:实现一个 Provider(和可选的 Config),并注册到 _TTS_PROVIDER_BUILDERS。
46
+ """
47
+ vendor: str = "unknown"
48
+
49
+ def synthesize_to_wav_sync(self, *, text: str, wav_path: Path) -> None:
50
+ """同步生成 wav(注意:会被 asyncio.to_thread 包装,不阻塞事件循环)"""
51
+ raise NotImplementedError
52
+
53
+
54
+ def _normalize_vendor(v: Optional[str]) -> str:
55
+ v = (v or "").strip().lower()
56
+ if not v:
57
+ return "bytedance"
58
+ aliases = {
59
+ "byte": "bytedance",
60
+ "bytedance": "bytedance",
61
+ "volc": "bytedance",
62
+ "volcengine": "bytedance",
63
+ "volcano": "bytedance",
64
+ }
65
+ return aliases.get(v, v)
66
+
67
+
68
+ def _merge_config(inputs: Dict[str, Any]) -> Dict[str, Any]:
69
+ """
70
+ 把可能出现的配置来源合并成一个 dict,便于兼容:
71
+ - 顶层:uid/appid/access_token/tts_vendor...
72
+ - inputs["tts_config"](通用配置容器)
73
+ - inputs["bytedance_tts"](厂家专用容器,未来可按 vendor 增加)
74
+ """
75
+ merged: Dict[str, Any] = {}
76
+ # 1) 通用容器
77
+ tts_cfg = inputs.get("tts_config")
78
+ if isinstance(tts_cfg, dict):
79
+ merged.update(tts_cfg)
80
+
81
+ # 2) 厂家专用容器(目前只实现 bytedance)
82
+ bd_cfg = inputs.get("bytedance_tts")
83
+ if isinstance(bd_cfg, dict):
84
+ merged.update(bd_cfg)
85
+
86
+ # 3) 顶层覆盖(优先级最高)
87
+ merged.update(inputs)
88
+ return merged
89
+
90
+
91
+ def _pick_str(cfg: Dict[str, Any], *keys: str) -> str:
92
+ for k in keys:
93
+ v = cfg.get(k)
94
+ if isinstance(v, str) and v.strip():
95
+ return v.strip()
96
+ return ""
97
+
98
+
99
+ def _pick_int(cfg: Dict[str, Any], key: str, default: int) -> int:
100
+ v = cfg.get(key, default)
101
+ try:
102
+ return int(v)
103
+ except Exception:
104
+ return int(default)
105
+
106
+
107
+ def _pick_float(cfg: Dict[str, Any], key: str, default: float) -> float:
108
+ v = cfg.get(key, default)
109
+ try:
110
+ return float(v)
111
+ except Exception:
112
+ return float(default)
113
+
114
+
115
+ # -----------------------------
116
+ # ByteDance provider
117
+ # -----------------------------
118
+ @dataclass(frozen=True)
119
+ class ByteDanceTTSConfig:
120
+ uid: str
121
+ appid: str
122
+ access_token: str
123
+
124
+ cluster: str = "volcano_tts"
125
+ voice_type: str = "BV002_streaming"
126
+ encoding: str = "wav"
127
+ speed: int = 10
128
+ volume: int = 10
129
+ pitch: int = 10
130
+
131
+ host: str = "openspeech.bytedance.com"
132
+ api_path: str = "/api/v1/tts"
133
+ timeout_sec: float = 30.0
134
+
135
+ @property
136
+ def api_url(self) -> str:
137
+ return f"https://{self.host}{self.api_path}"
138
+
139
+ @classmethod
140
+ def from_inputs(cls, inputs: Dict[str, Any]) -> "ByteDanceTTSConfig":
141
+ cfg = _merge_config(inputs)
142
+
143
+ uid = _pick_str(cfg, "uid", "tts_uid")
144
+ appid = _pick_str(cfg, "appid", "tts_appid")
145
+ access_token = _pick_str(cfg, "access_token", "tts_access_token", "token")
146
+
147
+ if not uid:
148
+ raise TTSConfigError("ByteDance TTS 缺失必填字段:uid")
149
+ if not appid:
150
+ raise TTSConfigError("ByteDance TTS 缺失必填字段:appid")
151
+ if not access_token:
152
+ raise TTSConfigError("ByteDance TTS 缺失必填字段:access_token")
153
+
154
+ cluster = _pick_str(cfg, "cluster") or "volcano_tts"
155
+ voice_type = _pick_str(cfg, "voice_type") or "BV002_streaming"
156
+ encoding = _pick_str(cfg, "encoding") or "wav"
157
+
158
+ host = _pick_str(cfg, "host") or "openspeech.bytedance.com"
159
+ api_path = _pick_str(cfg, "api_path", "path") or "/api/v1/tts"
160
+
161
+ timeout_sec = _pick_float(cfg, "timeout_sec", 30.0)
162
+
163
+ speed = _pick_int(cfg, "speed", 10)
164
+ volume = _pick_int(cfg, "volume", 10)
165
+ pitch = _pick_int(cfg, "pitch", 10)
166
+
167
+ return cls(
168
+ uid=uid,
169
+ appid=appid,
170
+ access_token=access_token,
171
+ cluster=cluster,
172
+ voice_type=voice_type,
173
+ encoding=encoding,
174
+ speed=speed,
175
+ volume=volume,
176
+ pitch=pitch,
177
+ host=host,
178
+ api_path=api_path,
179
+ timeout_sec=timeout_sec,
180
+ )
181
+
182
+
183
+ class ByteDanceTTSProvider(BaseTTSProvider):
184
+ vendor = "bytedance"
185
+
186
+ def __init__(self, cfg: ByteDanceTTSConfig):
187
+ self.cfg = cfg
188
+ self._session = requests.Session()
189
+
190
+ def synthesize_to_wav_sync(self, *, text: str, wav_path: Path) -> None:
191
+ cfg = self.cfg
192
+
193
+ headers = {
194
+ "Authorization": f"Bearer; {cfg.access_token}",
195
+ "Content-Type": "application/json",
196
+ }
197
+
198
+ request_json = {
199
+ "app": {
200
+ "appid": cfg.appid,
201
+ "token": cfg.access_token,
202
+ "cluster": cfg.cluster,
203
+ },
204
+ "user": {
205
+ "uid": cfg.uid,
206
+ },
207
+ "audio": {
208
+ "voice": "other",
209
+ "voice_type": cfg.voice_type,
210
+ "encoding": cfg.encoding,
211
+ "speed": cfg.speed,
212
+ "volume": cfg.volume,
213
+ "pitch": cfg.pitch,
214
+ },
215
+ "request": {
216
+ "reqid": str(uuid.uuid4()),
217
+ "text": text,
218
+ "text_type": "plain",
219
+ "operation": "query",
220
+ "with_frontend": 1,
221
+ "frontend_type": "unitTson",
222
+ },
223
+ }
224
+
225
+ try:
226
+ resp = self._session.post(
227
+ cfg.api_url,
228
+ json=request_json,
229
+ headers=headers,
230
+ timeout=cfg.timeout_sec,
231
+ )
232
+ resp.raise_for_status()
233
+ except Exception as e:
234
+ raise TTSRequestError(
235
+ f"ByteDance TTS 网络请求失败:{type(e).__name__}: {e}(token={_mask_secret(cfg.access_token)})"
236
+ ) from e
237
+
238
+ try:
239
+ resp_json = resp.json()
240
+ except Exception as e:
241
+ raise TTSRequestError(
242
+ f"ByteDance TTS 返回非 JSON:{type(e).__name__}: {e}"
243
+ ) from e
244
+
245
+ if resp_json.get("message") != "Success":
246
+ code = resp_json.get("code")
247
+ msg = resp_json.get("message") or "Unknown"
248
+ raise TTSRequestError(f"ByteDance TTS 失败:code={code} message={msg}")
249
+
250
+ data_b64 = resp_json.get("data")
251
+ if not isinstance(data_b64, str) or not data_b64.strip():
252
+ raise TTSRequestError("ByteDance TTS 返回缺失 data 字段(音频内容为空)")
253
+
254
+ wav_path.parent.mkdir(parents=True, exist_ok=True)
255
+ try:
256
+ wav_bytes = base64.b64decode(data_b64)
257
+ wav_path.write_bytes(wav_bytes)
258
+ except Exception as e:
259
+ raise TTSRequestError(f"写入 wav 失败:{type(e).__name__}: {e}") from e
260
+
261
+
262
+ # ---- Provider registry ----
263
+ _TTS_PROVIDER_BUILDERS: Dict[str, Callable[[Dict[str, Any]], BaseTTSProvider]] = {
264
+ "bytedance": lambda inputs: ByteDanceTTSProvider(ByteDanceTTSConfig.from_inputs(inputs)),
265
+ }
266
+
267
+
268
+ def build_tts_provider(inputs: Dict[str, Any]) -> BaseTTSProvider:
269
+ cfg = _merge_config(inputs)
270
+ vendor = _normalize_vendor(_pick_str(cfg, "tts_vendor", "vendor", "tts_provider"))
271
+
272
+ builder = _TTS_PROVIDER_BUILDERS.get(vendor)
273
+ if not builder:
274
+ supported = ", ".join(sorted(_TTS_PROVIDER_BUILDERS.keys()))
275
+ raise TTSConfigError(f"不支持的 TTS vendor:{vendor}(当前支持:{supported})")
276
+
277
+ return builder(inputs)
278
+
279
+
280
+ # -----------------------------
281
+ # Node implementation
282
+ # -----------------------------
283
+ class GenerateVoiceoverNode(BaseNode):
284
+ meta = NodeMeta(
285
+ name="generate_voiceover",
286
+ description="根据文案生成配音",
287
+ node_id="generate_voiceover",
288
+ node_kind="tts",
289
+ require_prior_kind=["group_clips", "generate_script"],
290
+ default_require_prior_kind=["group_clips", "generate_script"],
291
+ )
292
+
293
+ input_schema = GenerateVoiceoverInput
294
+
295
+ async def default_process(self, node_summary, inputs: Dict[str, Any]) -> Any:
296
+ node_summary.info_for_user("未生成配音")
297
+ return {"voiceover": []}
298
+
299
+ async def process(self, node_summary, inputs: Dict[str, Any], **params) -> Any:
300
+ # 1) 取文案
301
+ group_scripts = (inputs.get("generate_script") or {}).get("group_scripts") or []
302
+ if not isinstance(group_scripts, list) or not group_scripts:
303
+ node_summary.info_for_user("未找到可生成配音的文案(group_scripts 为空)")
304
+ return {"voiceover": []}
305
+
306
+ # 2) 准备输出目录
307
+ artifact_id = inputs.get("artifact_id")
308
+ session_id = inputs.get("session_id")
309
+ if not artifact_id or not session_id:
310
+ raise ValueError("缺失 artifact_id / session_id,无法生成配音输出目录")
311
+
312
+ output_dir = self.server_cache_dir / str(session_id) / str(artifact_id)
313
+ output_dir.mkdir(parents=True, exist_ok=True)
314
+
315
+ # 3) 构建 TTS provider
316
+ provider = build_tts_provider(inputs)
317
+ node_summary.info_for_user(f"TTS 服务:{provider.vendor}")
318
+
319
+ ts_ms = int(time.time() * 1000)
320
+ voiceover: list[dict[str, Any]] = []
321
+
322
+ # 4) 逐段生成
323
+ for i, group in enumerate(group_scripts, start=1):
324
+ group_id = (group or {}).get("group_id", "")
325
+ raw_text = (group or {}).get("raw_text", "")
326
+
327
+ if not group_id:
328
+ raise ValueError(f"缺失 group_id:{group}")
329
+ if not isinstance(raw_text, str) or not raw_text.strip():
330
+ raise ValueError(f"group_id={group_id} 的 raw_text 为空,无法生成语音。")
331
+
332
+ voiceover_id = f"voiceover_{i:04d}"
333
+ wav_path = output_dir / f"{voiceover_id}_{ts_ms}.wav"
334
+
335
+ await asyncio.to_thread(provider.synthesize_to_wav_sync, text=raw_text, wav_path=wav_path)
336
+
337
+ duration = _wav_duration_ms(wav_path)
338
+ voiceover.append(
339
+ {
340
+ "voiceover_id": voiceover_id,
341
+ "group_id": group_id,
342
+ "path": str(wav_path),
343
+ "duration": duration,
344
+ }
345
+ )
346
+
347
+ node_summary.info_for_user(
348
+ f"成功生成 {voiceover_id}",
349
+ preview_urls=[str(wav_path)],
350
+ )
351
+
352
+ node_summary.info_for_user(f"共生成 {len(voiceover)} 段配音")
353
+ return {"voiceover": voiceover}
src/open_storyline/nodes/core_nodes/group_clips.py ADDED
@@ -0,0 +1,190 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Any, Dict
2
+
3
+ from open_storyline.nodes.core_nodes.base_node import BaseNode, NodeMeta
4
+ from open_storyline.mcp.sampling_requester import LLMClient
5
+ from open_storyline.nodes.node_schema import GroupClipsInput
6
+ from open_storyline.mcp.sampling_requester import LLMClient
7
+ from src.open_storyline.utils.prompts import get_prompt
8
+ from open_storyline.utils.parse_json import parse_json_dict
9
+
10
+ class GroupClipsNode(BaseNode):
11
+
12
+ meta = NodeMeta(
13
+ name="group_clips",
14
+ description="按照用户要求,根据clips的描述,对clips进行分组。依赖 filter_clips 工具的结果",
15
+ node_id="group_clips",
16
+ node_kind="group_clips",
17
+ require_prior_kind=['filter_clips'],
18
+ default_require_prior_kind=['filter_clips'],
19
+ next_available_node=['generate_script', 'xhs_generate_script'],
20
+ )
21
+ input_schema = GroupClipsInput
22
+
23
+ async def default_process(
24
+ self,
25
+ node_summary,
26
+ inputs: Dict[str, Any],
27
+ ) -> Any:
28
+ result = _make_single_group_fallback(inputs["filter_clips"].get("selected", []))
29
+ return {
30
+ "groups": result,
31
+ }
32
+
33
+ async def process(self, node_summary, inputs: Dict[str, Any], **params) -> Any:
34
+ clip_captions = inputs["filter_clips"].get("clip_captions")
35
+ selected_clips = inputs["filter_clips"].get("selected")
36
+ user_request = inputs["user_request"]
37
+ llm: LLMClient = inputs["llm"]
38
+ clip_lookup = _build_clip_lookup(clip_captions)
39
+
40
+ if not selected_clips:
41
+ return {"groups": []}
42
+
43
+ selected_clips_captions = [clip_lookup[cid] for cid in selected_clips]
44
+ clip_block = _build_clips_block(selected_clips_captions)
45
+
46
+ system_prompt = get_prompt("group_clips.system")
47
+ if user_request == "":
48
+ user_request = "无额外要求"
49
+
50
+ user_prompt = get_prompt(
51
+ "group_clips.user",
52
+ user_request=user_request,
53
+ selected_clips=selected_clips,
54
+ clip_captions=clip_block,
55
+ )
56
+
57
+ raw = await llm.complete(
58
+ system_prompt=system_prompt,
59
+ user_prompt=user_prompt,
60
+ media=None,
61
+ temperature=0.1,
62
+ top_p=0.9,
63
+ max_tokens=4096,
64
+ model_preferences=None,
65
+ )
66
+
67
+ try:
68
+ obj = parse_json_dict(raw)
69
+ groups_raw = _extract_groups_obj(obj)
70
+
71
+ groups = _normalize_groups_from_llm(
72
+ groups_raw=groups_raw,
73
+ selected_ids_set=set(selected_clips),
74
+ )
75
+
76
+ node_summary.info_for_user(f"分组成功:共{len(groups)}个组")
77
+ return {
78
+ "groups": groups,
79
+ }
80
+ except Exception as e:
81
+
82
+ result = _make_single_group_fallback(selected_clips)
83
+ node_summary.info_for_user(f"分组出错:{e}\n采用默认策略")
84
+ return {
85
+ "groups": result,
86
+ }
87
+
88
+ def _extract_groups_obj(obj: Any) -> list[dict[str, Any]]:
89
+ if isinstance(obj, dict) and isinstance(obj.get("groups"), list):
90
+ return obj["groups"]
91
+ if isinstance(obj, list):
92
+ return obj
93
+ raise ValueError("LLM 输出不包含 groups。")
94
+
95
+
96
+ def _normalize_groups_from_llm(
97
+ groups_raw: list[dict[str, Any]],
98
+ selected_ids_set: set[str],
99
+ ) -> list[dict[str, Any]]:
100
+ """
101
+ 校验+规范化 LLM 输出的 groups:
102
+ - clip_ids 必须全部来自 selected
103
+ - clip_ids 不能重复
104
+ - group_id 由代码统一重写
105
+ - summary 缺失则补默认
106
+ """
107
+ if not groups_raw:
108
+ raise ValueError("groups 为空。")
109
+
110
+ # 先抽取并做基础清洗
111
+ normalized_groups: list[dict[str, Any]] = []
112
+ seen: set[str] = set()
113
+
114
+ for gi, g in enumerate(groups_raw):
115
+ if not isinstance(g, dict):
116
+ raise ValueError(f"groups[{gi}] 不是对象,请尝试重新运行。")
117
+
118
+ clip_ids = g.get("clip_ids")
119
+ if not isinstance(clip_ids, list) or not clip_ids:
120
+ raise ValueError(f"groups[{gi}].clip_ids 必须是非空 list,请尝试重新运行。。")
121
+
122
+ # clip_ids 去重(保持原输出顺序)
123
+ cleaned_clip_ids: list[str] = []
124
+
125
+ for cid in clip_ids:
126
+ if not isinstance(cid, str):
127
+ continue
128
+ if cid not in selected_ids_set:
129
+ continue
130
+ # raise ValueError(f"groups[{gi}] 包含非选中 clip_id:{cid}")
131
+ if cid in seen:
132
+ continue
133
+ seen.add(cid)
134
+ cleaned_clip_ids.append(cid)
135
+
136
+ if not cleaned_clip_ids:
137
+ raise ValueError(f"groups[{gi}] 清洗后 clip_ids 为空,请尝试重新运行。")
138
+
139
+ summary = g.get("summary")
140
+ if not isinstance(summary, str) or not summary.strip():
141
+ summary = "一组镜头,用于承载同一段文案与配音。"
142
+
143
+ normalized_groups.append(
144
+ {
145
+ "group_id": "", # group_id 先占位,后面统一重写
146
+ "summary": summary.strip(),
147
+ "clip_ids": cleaned_clip_ids,
148
+ }
149
+ )
150
+
151
+ # 最终重写 group_id
152
+ for i, g in enumerate(normalized_groups, start=1):
153
+ g["group_id"] = f"group_{i:04d}"
154
+
155
+ return normalized_groups
156
+
157
+
158
+ def _build_clip_lookup(clip_captions: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
159
+ lookup: dict[str, dict[str, Any]] = {}
160
+ for clip in clip_captions:
161
+ cid = clip.get("clip_id")
162
+ if cid:
163
+ lookup[cid] = clip
164
+ return lookup
165
+
166
+ def _make_single_group_fallback(
167
+ selected_clips: list[str],
168
+ ) -> list[dict[str, Any]]:
169
+ return [
170
+ {
171
+ "group_id": "group_0001",
172
+ "summary": "按原始顺序汇总全部选中镜头,便于后续生成文案与配音。",
173
+ "clip_ids": selected_clips,
174
+ }
175
+ ]
176
+
177
+ def _build_clips_block(clip_captions: list[dict[str, Any]]) -> str:
178
+ """
179
+ 将 clips 构造成稳定的文本块
180
+ """
181
+ blocks: list[str] = []
182
+ for clip in clip_captions:
183
+ clip_id = clip.get("clip_id", "")
184
+ media_id = clip.get("source_ref", {}).get("media_id", "")
185
+ caption = clip.get("caption", "")
186
+ block = (
187
+ f"【{clip_id}】 <media>{int(media_id.split('_', 1)[1])}</media> {caption}\n"
188
+ )
189
+ blocks.append(block)
190
+ return "\n".join(blocks).strip() + "\n"
src/open_storyline/nodes/core_nodes/plan_timeline.py ADDED
@@ -0,0 +1,904 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import random
4
+ from dataclasses import dataclass
5
+ from typing import Any, Dict, List, Mapping, MutableMapping, Optional, Tuple
6
+
7
+ from src.open_storyline.config import Settings
8
+ from open_storyline.config import PlanTimelineConfig
9
+ from open_storyline.nodes.core_nodes.base_node import BaseNode, NodeMeta
10
+ from open_storyline.nodes.node_schema import PlanTimelineInput
11
+
12
+ # =========================
13
+ # Constants (no magic numbers)
14
+ # =========================
15
+ Milliseconds = int
16
+
17
+ DEFAULT_RANDOM_SEED = 42
18
+
19
+ SECONDS_PER_MINUTE = 60.0
20
+ MILLISECONDS_PER_SECOND = 1000.0
21
+
22
+ SNAP_SAFETY_MAX_STEPS = 10_000
23
+ BINARY_SEARCH_ITERATIONS = 50
24
+
25
+ RATIO_GROWTH_FACTOR = 2.0
26
+ RATIO_GROWTH_MAX = 10.0
27
+
28
+ MIN_SUBTITLE_WEIGHT = 1
29
+ CENTER_ALIGN_DIVISOR = 2.0
30
+
31
+
32
+ @dataclass(frozen=True)
33
+ class BeatTrack:
34
+ """Beat-related information derived from background music (BGM)."""
35
+
36
+ beat_timestamps_ms: List[Milliseconds]
37
+ beat_durations_ms: List[Milliseconds]
38
+ music_duration_ms: Milliseconds
39
+
40
+
41
+ class TimelinePlanner:
42
+ """
43
+ Pure timeline planning logic extracted from PlanTimelineNode for:
44
+ - clearer separation of concerns
45
+ - better readability
46
+ - easier unit testing
47
+ """
48
+
49
+ def __init__(self, config: PlanTimelineConfig, *, random_seed: int = DEFAULT_RANDOM_SEED) -> None:
50
+ self._config = config
51
+ self._random_generator = random.Random(random_seed)
52
+
53
+ def plan(
54
+ self,
55
+ *,
56
+ assets: List[Dict[str, Any]],
57
+ clips: List[Dict[str, Any]],
58
+ groups: List[Dict[str, Any]],
59
+ group_scripts: List[Dict[str, Any]],
60
+ voiceovers: List[Dict[str, Any]],
61
+ background_music: Optional[Dict[str, Any]],
62
+ use_beats: bool,
63
+ ) -> Dict[str, Any]:
64
+ """Plan full timeline tracks: video/subtitles/voiceover/bgm."""
65
+ assets_by_asset_id = self._build_item_index(assets, id_key="asset_id")
66
+ clips_by_clip_id = self._build_item_index(clips, id_key="clip_id")
67
+ script_by_group_id = self._build_item_index(group_scripts, id_key="group_id")
68
+ voiceover_by_group_id = self._build_item_index(voiceovers, id_key="group_id")
69
+
70
+ beat_track = self._build_beat_track(background_music, use_beats=use_beats)
71
+
72
+ music_offset_ms, start_beat_index = self._compute_title_music_offset(
73
+ beat_durations_ms=beat_track.beat_durations_ms,
74
+ music_duration_ms=beat_track.music_duration_ms,
75
+ use_beats=use_beats,
76
+ )
77
+
78
+ video_segments, group_states, total_duration_ms, _end_beat_index = self._build_video_track(
79
+ groups=groups,
80
+ clips_by_clip_id=clips_by_clip_id,
81
+ assets_by_asset_id=assets_by_asset_id,
82
+ script_by_group_id=script_by_group_id,
83
+ voiceover_by_group_id=voiceover_by_group_id,
84
+ background_music=background_music,
85
+ beat_durations_ms=beat_track.beat_durations_ms,
86
+ start_beat_index=start_beat_index,
87
+ use_beats=use_beats,
88
+ )
89
+
90
+ voiceover_segments = self._build_voiceover_track(groups=groups, group_states=group_states)
91
+ subtitle_segments = self._build_subtitle_track(groups=groups, group_states=group_states)
92
+ bgm_segments = self._build_bgm_track(
93
+ background_music=background_music,
94
+ total_duration_ms=total_duration_ms,
95
+ music_offset_ms=music_offset_ms,
96
+ )
97
+
98
+ return {
99
+ "tracks": {
100
+ "video": video_segments,
101
+ "subtitles": subtitle_segments,
102
+ "voiceover": voiceover_segments,
103
+ "bgm": bgm_segments,
104
+ }
105
+ }
106
+
107
+ # -----------------------------
108
+ # Track builders
109
+ # -----------------------------
110
+ def _build_video_track(
111
+ self,
112
+ *,
113
+ groups: List[Dict[str, Any]],
114
+ clips_by_clip_id: Mapping[str, Dict[str, Any]],
115
+ assets_by_asset_id: Mapping[str, Dict[str, Any]],
116
+ script_by_group_id: Mapping[str, Dict[str, Any]],
117
+ voiceover_by_group_id: Mapping[str, Dict[str, Any]],
118
+ background_music: Optional[Dict[str, Any]],
119
+ beat_durations_ms: List[Milliseconds],
120
+ start_beat_index: int,
121
+ use_beats: bool,
122
+ ) -> Tuple[List[Dict[str, Any]], Dict[str, Dict[str, Any]], Milliseconds, int]:
123
+ video_segments: List[Dict[str, Any]] = []
124
+ group_states: Dict[str, Dict[str, Any]] = {}
125
+
126
+ timeline_cursor_ms: Milliseconds = 0
127
+ residual_ms: Milliseconds = 0 # preserved for future improvements; legacy code kept it but didn't update it.
128
+
129
+ beat_index = int(start_beat_index)
130
+
131
+ for group in groups:
132
+ group_id = self._to_str_id(group.get("group_id"))
133
+ clip_ids = [self._to_str_id(cid) for cid in (group.get("clip_ids", []) or [])]
134
+ if not clip_ids:
135
+ raise ValueError(
136
+ f"group {group_id} has no clip_ids, please check the result of 'group_clips' node"
137
+ )
138
+
139
+ clip_items: List[Dict[str, Any]] = []
140
+ for clip_id in clip_ids:
141
+ if clip_id not in clips_by_clip_id:
142
+ raise KeyError(
143
+ f"group {group_id} references missing clip_id={clip_id}, "
144
+ "please check the result of 'group_clips' and 'split_shots' node"
145
+ )
146
+ clip_items.append(clips_by_clip_id[clip_id])
147
+
148
+ group_script = script_by_group_id.get(group_id)
149
+ group_voiceover = voiceover_by_group_id.get(group_id)
150
+
151
+ # Case A: no script, no voiceover, and no beat snapping -> concatenate clips as-is.
152
+ if not group_script and not group_voiceover and (not background_music or not use_beats):
153
+ group_start_ms = timeline_cursor_ms
154
+ first_clip_source_duration_ms: Milliseconds = 0
155
+
156
+ for index_in_group, clip in enumerate(clip_items):
157
+ clip_id = self._to_str_id(clip.get("clip_id"))
158
+ clip_kind = str(clip.get("kind", "video"))
159
+
160
+ source_start_ms, source_end_ms, source_duration_ms = self._full_source_window_and_duration_ms(
161
+ clip
162
+ )
163
+ if index_in_group == 0:
164
+ first_clip_source_duration_ms = source_duration_ms
165
+
166
+ segment_start_ms = timeline_cursor_ms
167
+ segment_end_ms = segment_start_ms + source_duration_ms
168
+
169
+ video_segments.append(
170
+ {
171
+ "clip_id": clip_id,
172
+ "group_id": group_id,
173
+ "kind": clip_kind,
174
+ "path": clip.get("path"),
175
+ "fps": clip.get("fps"),
176
+ "source_path": self._resolve_source_path(
177
+ clip=clip, assets_by_asset_id=assets_by_asset_id
178
+ ),
179
+ "source_window": {"start": source_start_ms, "end": source_end_ms},
180
+ # NOTE: the original code used {"start": seg_end, "end": seg_end} here,
181
+ # which produces a 0-length window. This is almost certainly a typo.
182
+ "timeline_window": {"start": segment_start_ms, "end": segment_end_ms},
183
+ "playback_rate": 1.0,
184
+ }
185
+ )
186
+ timeline_cursor_ms = segment_end_ms
187
+
188
+ group_end_ms = timeline_cursor_ms
189
+ group_states[group_id] = {
190
+ "group_id": group_id,
191
+ "start": group_start_ms,
192
+ "end": group_end_ms,
193
+ "duration": group_end_ms - group_start_ms,
194
+ "first_clip_duration": first_clip_source_duration_ms,
195
+ }
196
+ continue
197
+
198
+ # Case B: voiceover duration is authoritative; otherwise estimate by text length.
199
+ if group_voiceover and group_voiceover.get("duration", 0) > 0:
200
+ narration_duration_ms: Milliseconds = int(group_voiceover.get("duration", 0))
201
+ else:
202
+ narration_duration_ms = self._estimate_group_duration_from_text_ms(group_script)
203
+
204
+ group_target_duration_ms: Milliseconds = narration_duration_ms + int(self._config.group_margin_over_voiceover)
205
+ group_target_duration_ms = max(
206
+ group_target_duration_ms, len(clip_items) * int(self._config.min_clip_duration)
207
+ )
208
+
209
+ if use_beats and background_music:
210
+ durations_ms, beat_index = self._allocate_clip_durations_using_beats(
211
+ clip_items=clip_items,
212
+ group_target_ms=group_target_duration_ms,
213
+ beat_durations_ms=beat_durations_ms,
214
+ start_beat_index=beat_index,
215
+ start_residual_ms=residual_ms,
216
+ )
217
+ else:
218
+ durations_ms = self._allocate_clip_durations_without_beats(
219
+ clip_items=clip_items, group_target_ms=group_target_duration_ms
220
+ )
221
+
222
+ group_start_ms = timeline_cursor_ms
223
+ first_clip_planned_duration_ms: Milliseconds = durations_ms[0] if durations_ms else 0
224
+
225
+ for clip, planned_duration_ms in zip(clip_items, durations_ms):
226
+ clip_id = clip.get("clip_id") # keep legacy behavior (may be int)
227
+ clip_kind = str(clip.get("kind", "video")).lower()
228
+
229
+ segment_start_ms = timeline_cursor_ms
230
+ segment_end_ms = segment_start_ms + int(planned_duration_ms)
231
+
232
+ source_start_ms, source_end_ms, source_available_ms = self._full_source_window_and_duration_ms(clip)
233
+ playback_rate = 1.0
234
+
235
+ if clip_kind == "video":
236
+ if planned_duration_ms > source_available_ms:
237
+ playback_rate = (source_available_ms / planned_duration_ms) if planned_duration_ms > 0 else 1.0
238
+ source_window_start_ms, source_window_end_ms = source_start_ms, source_end_ms
239
+ else:
240
+ source_window_start_ms, source_window_end_ms = self._choose_source_window_for_timeline_duration_ms(
241
+ clip=clip, used_timeline_duration_ms=int(planned_duration_ms)
242
+ )
243
+ else:
244
+ # image (and other non-video kinds): use from src_start for the planned duration
245
+ source_window_start_ms, source_window_end_ms = (
246
+ source_start_ms,
247
+ source_start_ms + int(planned_duration_ms),
248
+ )
249
+
250
+ video_segments.append(
251
+ {
252
+ "clip_id": clip_id,
253
+ "group_id": group_id,
254
+ "kind": clip_kind,
255
+ "path": clip.get("path"),
256
+ "orig_path": clip.get("orig_path"),
257
+ "fps": clip.get("fps"),
258
+ "source_path": self._resolve_source_path(
259
+ clip=clip, assets_by_asset_id=assets_by_asset_id
260
+ ),
261
+ "source_window": {
262
+ "start": source_window_start_ms,
263
+ "end": source_window_end_ms,
264
+ "duration": source_window_end_ms - source_window_start_ms,
265
+ },
266
+ "timeline_window": {
267
+ "start": segment_start_ms,
268
+ "end": segment_end_ms,
269
+ "duration": segment_end_ms - segment_start_ms,
270
+ },
271
+ "playback_rate": playback_rate,
272
+ }
273
+ )
274
+
275
+ timeline_cursor_ms = segment_end_ms
276
+
277
+ group_end_ms = timeline_cursor_ms
278
+ group_states[group_id] = {
279
+ "group_id": group_id,
280
+ "start": group_start_ms,
281
+ "end": group_end_ms,
282
+ "duration": group_end_ms - group_start_ms,
283
+ "first_clip_duration": int(first_clip_planned_duration_ms),
284
+ "n_clips": len(clip_items),
285
+ "narration_duration": narration_duration_ms,
286
+ "group_margin": int(self._config.group_margin_over_voiceover),
287
+ "voiceover": group_voiceover,
288
+ "script": group_script,
289
+ }
290
+
291
+ total_duration_ms = timeline_cursor_ms
292
+ return video_segments, group_states, total_duration_ms, beat_index
293
+
294
+ def _build_voiceover_track(
295
+ self, *, groups: List[Dict[str, Any]], group_states: MutableMapping[str, Dict[str, Any]]
296
+ ) -> List[Dict[str, Any]]:
297
+ voiceover_segments: List[Dict[str, Any]] = []
298
+
299
+ for group in groups:
300
+ group_id = self._to_str_id(group.get("group_id", ""))
301
+ state = group_states.get(group_id)
302
+ if not state:
303
+ continue
304
+
305
+ voiceover_item = state.get("voiceover")
306
+ if not voiceover_item:
307
+ continue
308
+
309
+ voiceover_duration_ms = int(voiceover_item.get("duration", 0))
310
+ if voiceover_duration_ms <= 0:
311
+ continue
312
+
313
+ group_start_ms = int(state.get("start", 0))
314
+ group_end_ms = int(state.get("end", 0))
315
+ group_duration_ms = group_end_ms - group_start_ms
316
+
317
+ slack_ms = max(0, group_duration_ms - voiceover_duration_ms)
318
+
319
+ start_offset_ms = slack_ms / CENTER_ALIGN_DIVISOR
320
+ voiceover_start_ms = group_start_ms + start_offset_ms
321
+ voiceover_end_ms = voiceover_start_ms + voiceover_duration_ms
322
+
323
+ voiceover_segments.append(
324
+ {
325
+ "group_id": group_id,
326
+ "voiceover_id": voiceover_item.get("voiceover_id"),
327
+ "path": voiceover_item.get("path"),
328
+ "source_window": {"start": 0, "end": voiceover_duration_ms, "duration": voiceover_duration_ms},
329
+ "timeline_window": {
330
+ "start": voiceover_start_ms,
331
+ "end": voiceover_end_ms,
332
+ "duration": voiceover_duration_ms,
333
+ },
334
+ }
335
+ )
336
+
337
+ state["voiceover_timeline"] = {
338
+ "start": voiceover_start_ms,
339
+ "end": voiceover_end_ms,
340
+ "duration": voiceover_duration_ms,
341
+ }
342
+
343
+ return voiceover_segments
344
+
345
+ def _build_subtitle_track(
346
+ self, *, groups: List[Dict[str, Any]], group_states: Mapping[str, Dict[str, Any]]
347
+ ) -> List[Dict[str, Any]]:
348
+ subtitle_segments: List[Dict[str, Any]] = []
349
+
350
+ for group in groups:
351
+ group_id = self._to_str_id(group.get("group_id", ""))
352
+ state = group_states.get(group_id)
353
+ if not state:
354
+ continue
355
+
356
+ # NOTE: original code used default "" then script.get(...) -> may crash.
357
+ group_script = state.get("script") or {}
358
+ subtitle_units = group_script.get("subtitle_units", []) if isinstance(group_script, dict) else []
359
+ if not subtitle_units:
360
+ continue
361
+
362
+ group_start_ms = int(state.get("start", 0))
363
+ group_end_ms = int(state.get("end", 0))
364
+ voiceover_timeline = state.get("voiceover_timeline")
365
+
366
+ if voiceover_timeline is not None:
367
+ voiceover_start_ms = voiceover_timeline["start"]
368
+ voiceover_end_ms = voiceover_timeline["end"]
369
+
370
+ if not (group_start_ms <= voiceover_start_ms <= voiceover_end_ms <= group_end_ms):
371
+ subtitle_start_ms = group_start_ms
372
+ subtitle_end_ms = group_end_ms
373
+ else:
374
+ subtitle_start_ms = int((group_start_ms + voiceover_start_ms) / CENTER_ALIGN_DIVISOR)
375
+ subtitle_end_ms = int((voiceover_end_ms + group_end_ms) / CENTER_ALIGN_DIVISOR)
376
+ else:
377
+ subtitle_start_ms = group_start_ms
378
+ subtitle_end_ms = group_end_ms
379
+
380
+ subtitle_duration_ms = subtitle_end_ms - subtitle_start_ms
381
+
382
+ unit_texts: List[str] = [str(u.get("text") or "") for u in subtitle_units]
383
+ unit_weights: List[int] = [max(MIN_SUBTITLE_WEIGHT, len(text.strip())) for text in unit_texts]
384
+ total_weight = sum(unit_weights)
385
+
386
+ unit_durations_ms: List[Milliseconds] = []
387
+ accumulated_ms: Milliseconds = 0
388
+ for i, weight in enumerate(unit_weights):
389
+ if i == len(unit_weights) - 1:
390
+ duration_ms = max(0, subtitle_duration_ms - accumulated_ms)
391
+ else:
392
+ duration_ms = (subtitle_duration_ms * weight) // total_weight if total_weight > 0 else 0
393
+ accumulated_ms += int(duration_ms)
394
+ unit_durations_ms.append(int(duration_ms))
395
+
396
+ subtitle_cursor_ms = subtitle_start_ms
397
+ for unit, text, duration_ms in zip(subtitle_units, unit_texts, unit_durations_ms):
398
+ if duration_ms <= 0:
399
+ continue
400
+ segment_start_ms = subtitle_cursor_ms
401
+ segment_end_ms = subtitle_cursor_ms + int(duration_ms)
402
+ subtitle_cursor_ms = segment_end_ms
403
+
404
+ subtitle_segments.append(
405
+ {
406
+ "group_id": group_id,
407
+ "unit_id": unit.get("unit_id"),
408
+ "index_in_group": unit.get("index_in_group"),
409
+ "text": text,
410
+ "timeline_window": {"start": segment_start_ms, "end": segment_end_ms},
411
+ }
412
+ )
413
+
414
+ return subtitle_segments
415
+
416
+ def _build_bgm_track(
417
+ self,
418
+ *,
419
+ background_music: Optional[Dict[str, Any]],
420
+ total_duration_ms: Milliseconds,
421
+ music_offset_ms: Milliseconds,
422
+ ) -> List[Dict[str, Any]]:
423
+ bgm_segments: List[Dict[str, Any]] = []
424
+ if not background_music:
425
+ return bgm_segments
426
+
427
+ music_duration_ms = int(background_music.get("duration", 0))
428
+ if music_duration_ms <= 0:
429
+ return bgm_segments
430
+
431
+ timeline_cursor_ms: Milliseconds = 0
432
+ source_cursor_ms: Milliseconds = int(music_offset_ms)
433
+ loop_index = 0
434
+
435
+ while timeline_cursor_ms < total_duration_ms:
436
+ remaining_timeline_ms = total_duration_ms - timeline_cursor_ms
437
+ remaining_source_ms = max(0, music_duration_ms - source_cursor_ms)
438
+
439
+ if remaining_source_ms <= 0:
440
+ source_cursor_ms = 0
441
+ loop_index += 1
442
+ continue
443
+
444
+ segment_duration_ms = min(remaining_timeline_ms, remaining_source_ms)
445
+ if segment_duration_ms <= 0:
446
+ break
447
+
448
+ bgm_segments.append(
449
+ {
450
+ "bgm_id": background_music.get("bgm_id"),
451
+ "path": background_music.get("path"),
452
+ "source_window": {"start": source_cursor_ms, "end": source_cursor_ms + segment_duration_ms},
453
+ "loop_idx": loop_index,
454
+ }
455
+ )
456
+
457
+ timeline_cursor_ms += segment_duration_ms
458
+ source_cursor_ms += segment_duration_ms
459
+
460
+ if timeline_cursor_ms < total_duration_ms and source_cursor_ms >= music_duration_ms:
461
+ source_cursor_ms = 0
462
+ loop_index += 1
463
+
464
+ return bgm_segments
465
+
466
+ # -----------------------------
467
+ # Beats & title alignment
468
+ # -----------------------------
469
+ def _build_beat_track(self, background_music: Optional[Dict[str, Any]], *, use_beats: bool) -> BeatTrack:
470
+ if not use_beats or not background_music:
471
+ return BeatTrack(beat_timestamps_ms=[], beat_durations_ms=[], music_duration_ms=0)
472
+
473
+ music_duration_ms = int(background_music.get("duration", 0))
474
+ beat_timestamps_ms = self._build_beat_timestamps_from_music_ms(background_music)
475
+ beat_durations_ms = self._convert_beat_timestamps_to_durations_ms(
476
+ beat_timestamps_ms=beat_timestamps_ms, music_duration_ms=music_duration_ms
477
+ )
478
+
479
+ return BeatTrack(
480
+ beat_timestamps_ms=beat_timestamps_ms,
481
+ beat_durations_ms=beat_durations_ms,
482
+ music_duration_ms=music_duration_ms,
483
+ )
484
+
485
+ def _compute_title_music_offset(
486
+ self, *, beat_durations_ms: List[Milliseconds], music_duration_ms: Milliseconds, use_beats: bool
487
+ ) -> Tuple[Milliseconds, int]:
488
+ """
489
+ Compute the BGM source offset so that the title ends on a beat.
490
+
491
+ NOTE:
492
+ - The original code assumes beat_durations_ms is non-empty; otherwise modulo would crash.
493
+ Here we guard against empty beats (safer for open-source usage).
494
+ """
495
+ music_offset_ms: Milliseconds = 0
496
+ beat_index = 0
497
+
498
+ if not use_beats:
499
+ return music_offset_ms, beat_index
500
+
501
+ title_duration_ms = int(getattr(self._config, "title_dutation", 0))
502
+ if title_duration_ms <= 0 or music_duration_ms <= 0 or not beat_durations_ms:
503
+ return music_offset_ms, beat_index
504
+
505
+ if bool(getattr(self._config, "bgm_loop", False)):
506
+ title_duration_ms = title_duration_ms % music_duration_ms
507
+ else:
508
+ title_duration_ms = min(title_duration_ms, music_duration_ms)
509
+
510
+ cumulative_ms: Milliseconds = 0
511
+ duration_index = 0
512
+ while duration_index < len(beat_durations_ms) and cumulative_ms < title_duration_ms:
513
+ cumulative_ms += int(beat_durations_ms[duration_index])
514
+ duration_index += 1
515
+
516
+ music_offset_ms = max(0, cumulative_ms - title_duration_ms)
517
+ beat_index = duration_index % len(beat_durations_ms)
518
+ return int(music_offset_ms), int(beat_index)
519
+
520
+ # -----------------------------
521
+ # Shared helpers (indexing, parsing, math)
522
+ # -----------------------------
523
+ @staticmethod
524
+ def _to_str_id(value: Any) -> str:
525
+ return "" if value is None else str(value)
526
+
527
+ @classmethod
528
+ def _build_item_index(cls, items: List[Dict[str, Any]], *, id_key: str) -> Dict[str, Dict[str, Any]]:
529
+ return {cls._to_str_id(item.get(id_key)): item for item in (items or [])}
530
+
531
+ @staticmethod
532
+ def _resolve_source_path(clip: Dict[str, Any], *, assets_by_asset_id: Mapping[str, Dict[str, Any]]) -> Optional[str]:
533
+ source_ref = clip.get("source_ref") or {}
534
+ asset_id = "" if source_ref is None else str(source_ref.get("asset_id", ""))
535
+ return assets_by_asset_id.get(asset_id, {}).get("path")
536
+
537
+ @staticmethod
538
+ def _safe_float(value: Any, default_value: float = 0.0) -> float:
539
+ try:
540
+ if value is None:
541
+ return default_value
542
+ return float(value)
543
+ except Exception:
544
+ return default_value
545
+
546
+ # -----------------------------
547
+ # Beats helpers
548
+ # -----------------------------
549
+ def _build_beat_timestamps_from_music_ms(self, background_music: Dict[str, Any]) -> List[Milliseconds]:
550
+ beats: List[Milliseconds] = background_music.get("beats", []) or []
551
+ music_duration_ms = int(background_music.get("duration", 0))
552
+
553
+ if beats:
554
+ if beats[0] != 0:
555
+ return [0] + beats
556
+ return beats
557
+
558
+ bpm = background_music.get("bpm")
559
+ if bpm is None:
560
+ return [0]
561
+
562
+ bpm_value = self._safe_float(bpm, 0.0)
563
+ if bpm_value <= 0 or music_duration_ms <= 0:
564
+ return [0]
565
+
566
+ interval_ms = int(SECONDS_PER_MINUTE / bpm_value * MILLISECONDS_PER_SECOND)
567
+
568
+ timestamps_ms: List[Milliseconds] = [0]
569
+ timestamp_ms: Milliseconds = interval_ms
570
+ while timestamp_ms <= music_duration_ms:
571
+ timestamps_ms.append(int(timestamp_ms))
572
+ timestamp_ms += interval_ms
573
+
574
+ return timestamps_ms
575
+
576
+ @staticmethod
577
+ def _convert_beat_timestamps_to_durations_ms(
578
+ *, beat_timestamps_ms: List[Milliseconds], music_duration_ms: Milliseconds
579
+ ) -> List[Milliseconds]:
580
+ if len(beat_timestamps_ms) < 2:
581
+ return []
582
+
583
+ durations_ms: List[Milliseconds] = []
584
+ for start_ms, end_ms in zip(beat_timestamps_ms[:-1], beat_timestamps_ms[1:]):
585
+ delta_ms = int(end_ms) - int(start_ms)
586
+ if delta_ms > 0:
587
+ durations_ms.append(int(delta_ms))
588
+
589
+ if music_duration_ms > 0:
590
+ tail_ms = max(0, int(music_duration_ms) - int(beat_timestamps_ms[-1]))
591
+ if tail_ms > 0:
592
+ durations_ms.append(int(tail_ms))
593
+
594
+ return durations_ms
595
+
596
+ # -----------------------------
597
+ # Text duration estimate
598
+ # -----------------------------
599
+ def _estimate_group_duration_from_text_ms(self, group_script: Optional[Dict[str, Any]]) -> Milliseconds:
600
+ if not group_script:
601
+ return int(self._config.estimate_text_min)
602
+
603
+ raw_text = str(group_script.get("raw_text") or "")
604
+ char_count = len(raw_text.strip())
605
+ if char_count <= 0:
606
+ return int(self._config.estimate_text_min)
607
+
608
+ chars_per_second = max(1.0, float(self._config.estimate_text_char_per_sec))
609
+ duration_ms = int(char_count / chars_per_second * MILLISECONDS_PER_SECOND)
610
+ return max(int(duration_ms), int(self._config.estimate_text_min))
611
+
612
+ # -----------------------------
613
+ # Clip/source windows
614
+ # -----------------------------
615
+ def _full_source_window_and_duration_ms(
616
+ self, clip: Dict[str, Any]
617
+ ) -> Tuple[Milliseconds, Milliseconds, Milliseconds]:
618
+ clip_kind = str(clip.get("kind", "video"))
619
+ source_ref = clip.get("source_ref") or {}
620
+
621
+ start_ms = int(source_ref.get("start", 0))
622
+ end_ms = int(source_ref.get("end", start_ms))
623
+ duration_ms = int(source_ref.get("duration", end_ms - start_ms))
624
+
625
+ if clip_kind == "image":
626
+ start_ms = 0
627
+ end_ms = int(self._config.image_default_duration)
628
+ duration_ms = end_ms - start_ms
629
+ else:
630
+ if duration_ms <= 0:
631
+ clip_id = clip.get("clip_id")
632
+ raise ValueError(
633
+ f"{clip_id} has invalid source window (start={start_ms}, end={end_ms}, duration={duration_ms})"
634
+ )
635
+
636
+ return int(start_ms), int(end_ms), int(duration_ms)
637
+
638
+ def _choose_source_window_for_timeline_duration_ms(
639
+ self, *, clip: Dict[str, Any], used_timeline_duration_ms: Milliseconds
640
+ ) -> Tuple[Milliseconds, Milliseconds]:
641
+ source_start_ms, _, source_duration_ms = self._full_source_window_and_duration_ms(clip)
642
+
643
+ random_offset_ms = int(self._random_generator.random() * (source_duration_ms - used_timeline_duration_ms))
644
+ window_start_ms = source_start_ms + random_offset_ms
645
+ window_end_ms = window_start_ms + int(used_timeline_duration_ms)
646
+ return int(window_start_ms), int(window_end_ms)
647
+
648
+ # -----------------------------
649
+ # Duration allocation
650
+ # -----------------------------
651
+ def _allocate_clip_durations_using_beats(
652
+ self,
653
+ *,
654
+ clip_items: List[Dict[str, Any]],
655
+ group_target_ms: Milliseconds,
656
+ beat_durations_ms: List[Milliseconds],
657
+ start_beat_index: int,
658
+ start_residual_ms: Milliseconds,
659
+ ) -> Tuple[List[Milliseconds], int]:
660
+ """
661
+ Algorithm (kept the same as legacy version):
662
+ 1) Allocate ideal duration per clip by source duration weights (sum to group_target_ms).
663
+ 2) Snap each clip end to nearest beat boundary; carry over the delta to next clip.
664
+ 3) Last clip snaps to ceil to ensure total >= group_target_ms.
665
+ """
666
+ clip_count = len(clip_items)
667
+ if clip_count == 0:
668
+ return [], int(start_beat_index)
669
+
670
+ if not beat_durations_ms:
671
+ raise ValueError("beat_durations is empty")
672
+
673
+ weights_ms: List[Milliseconds] = []
674
+ for clip in clip_items:
675
+ _, _, duration_ms = self._full_source_window_and_duration_ms(clip)
676
+ weights_ms.append(int(duration_ms))
677
+
678
+ sum_weights = sum(weights_ms)
679
+ targets_ms = [(int(group_target_ms) * w) // sum_weights for w in weights_ms]
680
+ remainder_ms = int(group_target_ms) - sum(targets_ms)
681
+
682
+ # Fix integer truncation drift
683
+ fractional_parts = [(i, (int(group_target_ms) * weights_ms[i]) % sum_weights) for i in range(clip_count)]
684
+ fractional_parts.sort(key=lambda x: x[1], reverse=True)
685
+ for k in range(remainder_ms):
686
+ targets_ms[fractional_parts[k][0]] += 1
687
+
688
+ # Enforce min clip duration; borrow from longest clips
689
+ deficit_ms: Milliseconds = 0
690
+ for i in range(clip_count):
691
+ if targets_ms[i] < int(self._config.min_clip_duration):
692
+ deficit_ms += int(self._config.min_clip_duration) - targets_ms[i]
693
+ targets_ms[i] = int(self._config.min_clip_duration)
694
+
695
+ if deficit_ms > 0:
696
+ indices_by_longest = sorted(range(clip_count), key=lambda i: targets_ms[i], reverse=True)
697
+ for i in indices_by_longest:
698
+ if deficit_ms <= 0:
699
+ break
700
+ slack_ms = targets_ms[i] - int(self._config.min_clip_duration)
701
+ targets_ms[i] -= slack_ms
702
+ deficit_ms -= slack_ms
703
+
704
+ beat_count = len(beat_durations_ms)
705
+
706
+ def snap_to_nearest_beat(
707
+ desired_ms: Milliseconds, beat_index: int, phase_ms: Milliseconds
708
+ ) -> Tuple[Milliseconds, int]:
709
+ elapsed_ms = int(phase_ms)
710
+ idx = int(beat_index)
711
+
712
+ safety_steps = 0
713
+ while elapsed_ms < int(self._config.min_clip_duration):
714
+ elapsed_ms += int(beat_durations_ms[idx])
715
+ idx = (idx + 1) % beat_count
716
+ safety_steps += 1
717
+ if safety_steps > SNAP_SAFETY_MAX_STEPS:
718
+ raise RuntimeError("snap_to_nearest_beat safety exceeded")
719
+
720
+ if elapsed_ms >= int(desired_ms):
721
+ return int(elapsed_ms), int(idx)
722
+
723
+ previous_elapsed_ms = elapsed_ms
724
+ previous_idx = idx
725
+
726
+ safety_steps = 0
727
+ while elapsed_ms < int(desired_ms):
728
+ previous_elapsed_ms = elapsed_ms
729
+ previous_idx = idx
730
+ elapsed_ms += int(beat_durations_ms[idx])
731
+ idx = (idx + 1) % beat_count
732
+ safety_steps += 1
733
+ if safety_steps > SNAP_SAFETY_MAX_STEPS:
734
+ raise RuntimeError("snap_to_nearest_beat safety exceeded")
735
+
736
+ if int(desired_ms) - previous_elapsed_ms < elapsed_ms - int(desired_ms):
737
+ return int(previous_elapsed_ms), int(previous_idx)
738
+ return int(elapsed_ms), int(idx)
739
+
740
+ def snap_to_beat_ceil(
741
+ desired_ms: Milliseconds, beat_index: int, phase_ms: Milliseconds
742
+ ) -> Tuple[Milliseconds, int]:
743
+ elapsed_ms = int(phase_ms)
744
+ idx = int(beat_index)
745
+
746
+ desired_ms = max(int(self._config.min_clip_duration), int(desired_ms))
747
+ safety_steps = 0
748
+ while elapsed_ms < desired_ms:
749
+ elapsed_ms += int(beat_durations_ms[idx])
750
+ idx = (idx + 1) % beat_count
751
+ safety_steps += 1
752
+ if safety_steps > SNAP_SAFETY_MAX_STEPS:
753
+ raise RuntimeError("snap_to_beat_ceil safety exceeded")
754
+ return int(elapsed_ms), int(idx)
755
+
756
+ durations_ms: List[Milliseconds] = []
757
+ beat_index = int(start_beat_index) % beat_count
758
+ phase_ms = max(0, int(start_residual_ms))
759
+
760
+ carry_ms: Milliseconds = 0
761
+ sum_actual_ms: Milliseconds = 0
762
+
763
+ for i in range(clip_count):
764
+ is_last_clip = i == clip_count - 1
765
+
766
+ desired_ms = int(targets_ms[i]) + int(carry_ms)
767
+ if desired_ms < int(self._config.min_clip_duration):
768
+ desired_ms = int(self._config.min_clip_duration)
769
+
770
+ if not is_last_clip:
771
+ actual_ms, beat_index = snap_to_nearest_beat(desired_ms, beat_index, phase_ms)
772
+ else:
773
+ remaining_ms = max(0, int(group_target_ms) - int(sum_actual_ms))
774
+ desired_ms = max(desired_ms, int(remaining_ms))
775
+ actual_ms, beat_index = snap_to_beat_ceil(desired_ms, beat_index, phase_ms)
776
+
777
+ durations_ms.append(int(actual_ms))
778
+ sum_actual_ms += int(actual_ms)
779
+
780
+ carry_ms = desired_ms - int(actual_ms)
781
+ phase_ms = 0 # legacy strategy: always reset
782
+
783
+ return durations_ms, int(beat_index)
784
+
785
+ def _allocate_clip_durations_without_beats(
786
+ self, *, clip_items: List[Dict[str, Any]], group_target_ms: Milliseconds
787
+ ) -> List[Milliseconds]:
788
+ clip_count = len(clip_items)
789
+ if clip_count == 0:
790
+ return []
791
+
792
+ source_durations_ms: List[Milliseconds] = []
793
+ for clip in clip_items:
794
+ _, _, duration_ms = self._full_source_window_and_duration_ms(clip)
795
+ source_durations_ms.append(int(duration_ms))
796
+
797
+ total_source_duration_ms = sum(source_durations_ms)
798
+
799
+ def total_for_ratio(ratio: float) -> Milliseconds:
800
+ total_ms: Milliseconds = 0
801
+ for duration_ms in source_durations_ms:
802
+ allocated_ms = int(duration_ms * ratio)
803
+ if allocated_ms < int(self._config.min_clip_duration):
804
+ allocated_ms = int(self._config.min_clip_duration)
805
+ total_ms += int(allocated_ms)
806
+ return int(total_ms)
807
+
808
+ ratio_high = max(1.0, int(group_target_ms) / total_source_duration_ms)
809
+ while total_for_ratio(ratio_high) < int(group_target_ms):
810
+ ratio_high *= RATIO_GROWTH_FACTOR
811
+ if ratio_high > RATIO_GROWTH_MAX:
812
+ break
813
+
814
+ ratio_low = 0.0
815
+ for _ in range(BINARY_SEARCH_ITERATIONS):
816
+ ratio_mid = (ratio_low + ratio_high) / 2.0
817
+ if total_for_ratio(ratio_mid) <= int(group_target_ms):
818
+ ratio_low = ratio_mid
819
+ else:
820
+ ratio_high = ratio_mid
821
+
822
+ ratio = ratio_low
823
+
824
+ base_ms: List[Milliseconds] = []
825
+ fractional_parts: List[Tuple[float, int]] = []
826
+ sum_base_ms: Milliseconds = 0
827
+
828
+ for i, duration_ms in enumerate(source_durations_ms):
829
+ raw = duration_ms * ratio
830
+ floored = int(raw)
831
+
832
+ allocated_ms = floored
833
+ if allocated_ms < int(self._config.min_clip_duration):
834
+ allocated_ms = int(self._config.min_clip_duration)
835
+ fraction = -1.0
836
+ else:
837
+ fraction = raw - floored
838
+
839
+ base_ms.append(int(allocated_ms))
840
+ sum_base_ms += int(allocated_ms)
841
+ fractional_parts.append((float(fraction), i))
842
+
843
+ remaining_ms: Milliseconds = int(group_target_ms) - int(sum_base_ms)
844
+ if remaining_ms > 0:
845
+ fractional_parts.sort(key=lambda x: (x[0], source_durations_ms[x[1]]), reverse=True)
846
+ j = 0
847
+ # NOTE: matches legacy behavior (no wrap-around); remaining_ms too large may raise IndexError.
848
+ while remaining_ms > 0:
849
+ idx = fractional_parts[j][1]
850
+ base_ms[idx] += 1
851
+ remaining_ms -= 1
852
+ j += 1
853
+
854
+ return base_ms
855
+
856
+
857
+ class PlanTimelineNode(BaseNode):
858
+ meta = NodeMeta(
859
+ name="plan_timeline",
860
+ description=(
861
+ "根据视频片段、字幕、语音和BGM组织合理的时间线。"
862
+ "依赖 assets_index, generate_script(可选), detect_highlights(可选), "
863
+ "select_BGM(可选), generate_voiceover(可选)工具的结果。"
864
+ ),
865
+ node_id="plan_timeline",
866
+ node_kind="plan_timeline",
867
+ require_prior_kind=["assets_index", "split_shots", "group_clips", "generate_script", "tts", "music_rec"],
868
+ default_require_prior_kind=["assets_index", "split_shots", "group_clips", "generate_script", "tts", "music_rec"],
869
+ next_available_node=["render_video"],
870
+ )
871
+
872
+ input_schema = PlanTimelineInput
873
+
874
+ def __init__(self, server_cfg: Settings) -> None:
875
+ super().__init__(server_cfg)
876
+ config: PlanTimelineConfig = self.server_cfg.plan_timeline
877
+ self.planner = TimelinePlanner(config, random_seed=DEFAULT_RANDOM_SEED)
878
+
879
+
880
+ async def default_process(self, node_summary, inputs: Dict[str, Any]) -> Any:
881
+ return await self.process(node_summary, inputs)
882
+
883
+ async def process(self, node_summary, inputs: Dict[str, Any]) -> Dict[str, Any]:
884
+ # Inputs (defensive parsing for open-source robustness)
885
+ assets = (inputs.get("assets_index") or {}).get("assets", [])
886
+ clips = (inputs.get("split_shots") or {}).get("clips", [])
887
+ groups = (inputs.get("group_clips") or {}).get("groups", [])
888
+ group_scripts = (inputs.get("generate_script") or {}).get("group_scripts", [])
889
+ voiceovers = (inputs.get("tts") or {}).get("voiceover", [])
890
+ background_music = (inputs.get("music_rec") or {}).get("bgm") # Optional dict
891
+ use_beats = inputs.get("use_beats", False)
892
+
893
+ result = self.planner.plan(
894
+ assets=assets,
895
+ clips=clips,
896
+ groups=groups,
897
+ group_scripts=group_scripts,
898
+ voiceovers=voiceovers,
899
+ background_music=background_music,
900
+ use_beats=use_beats,
901
+ )
902
+
903
+ node_summary.info_for_user("时间线组织成功")
904
+ return result
src/open_storyline/nodes/core_nodes/render_video.py ADDED
@@ -0,0 +1,838 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import tempfile
3
+ import time
4
+ import uuid
5
+ from pathlib import Path
6
+ from typing import Any, Dict, List, Optional, Tuple
7
+
8
+ import numpy as np
9
+ from PIL import Image, ImageDraw, ImageFont, ImageOps
10
+
11
+ # MoviePy import compatibility (v2 preferred)
12
+ try:
13
+ from moviepy import (
14
+ VideoFileClip,
15
+ AudioFileClip,
16
+ ImageClip,
17
+ ColorClip,
18
+ CompositeVideoClip,
19
+ CompositeAudioClip,
20
+ concatenate_videoclips,
21
+ concatenate_audioclips,
22
+ )
23
+ except Exception: # pragma: no cover
24
+ from moviepy.editor import ( # type: ignore
25
+ VideoFileClip,
26
+ AudioFileClip,
27
+ ImageClip,
28
+ ColorClip,
29
+ CompositeVideoClip,
30
+ CompositeAudioClip,
31
+ concatenate_videoclips,
32
+ concatenate_audioclips,
33
+ )
34
+
35
+ from src.open_storyline.config import Settings
36
+ from open_storyline.nodes.core_nodes.base_node import BaseNode, NodeMeta
37
+ from open_storyline.nodes.node_schema import RenderVideoInput
38
+ from open_storyline.utils.fonts import SourceHanSansSC_FONTS
39
+
40
+
41
+ # =============================================================================
42
+ # Constants
43
+ # =============================================================================
44
+
45
+ MILLISECONDS_PER_SECOND: float = 1000.0
46
+
47
+ MAX_MEDIA_DIMENSION_PX: int = 1080 # requirement: any media <=1080
48
+ DEFAULT_OUTPUT_MAX_DIMENSION_PX: int = 1080
49
+
50
+ DEFAULT_OUTPUT_ASPECT_RATIO: float = 16.0 / 9.0
51
+ BACKGROUND_COLOR_RGB: Tuple[int, int, int] = (0, 0, 0)
52
+ CENTER_POSITION = ("center", "center")
53
+
54
+ DEFAULT_OUTPUT_FPS: int = 25
55
+
56
+ SUBCLIP_END_SAFETY_MARGIN_S = 1e-3 #
57
+
58
+ # Encoding
59
+ VIDEO_CODEC: str = "libx264"
60
+ AUDIO_CODEC: str = "aac"
61
+ FFMPEG_PARAMS: list[str] = ["-preset", "veryfast", "-crf", "23", "-threads", "0"]
62
+ TEMP_DIRECTORY_PREFIX: str = "render_video_"
63
+ TEMP_AUDIO_FILENAME: str = "temp-audio.m4a"
64
+
65
+ # Subtitle baseline
66
+ SUBTITLE_BASE_HEIGHT_PX: float = 1080.0
67
+ SUBTITLE_FONT_SIZE_AT_BASE: int = 40
68
+ SUBTITLE_FONT_SIZE_MIN: int = 28
69
+ SUBTITLE_FONT_SIZE_MAX: int = 120
70
+ SUBTITLE_MARGIN_BOTTOM_AT_BASE: int = 80
71
+ SUBTITLE_MARGIN_BOTTOM_MIN: int = 40
72
+ SUBTITLE_MARGIN_BOTTOM_MAX: int = 240
73
+ SUBTITLE_STROKE_WIDTH_AT_BASE: int = 2
74
+ SUBTITLE_STROKE_WIDTH_MIN: int = 2
75
+ SUBTITLE_STROKE_WIDTH_MAX: int = 8
76
+ SUBTITLE_MAX_WIDTH_RATIO: float = 0.90
77
+ SUBTITLE_PADDING_X: int = 20
78
+ SUBTITLE_PADDING_Y: int = 10
79
+
80
+ BGM_VOLUME_SCALE: float = 0.25
81
+ AUDIO_DURATION_TOLERANCE_SECONDS: float = 0.05
82
+
83
+
84
+ # =============================================================================
85
+ # Small utilities
86
+ # =============================================================================
87
+
88
+ def close_quietly(obj: Any) -> None:
89
+ try:
90
+ if obj is not None:
91
+ obj.close()
92
+ except Exception:
93
+ pass
94
+
95
+
96
+ def milliseconds_to_seconds(value: Any) -> float:
97
+ try:
98
+ return float(value) / MILLISECONDS_PER_SECOND
99
+ except Exception:
100
+ return 0.0
101
+
102
+
103
+ def clamp_int(value: float, minimum: int, maximum: int) -> int:
104
+ return int(max(minimum, min(maximum, round(value))))
105
+
106
+
107
+ def make_even(value: int) -> int:
108
+ v = int(value)
109
+ if v < 2:
110
+ v = 2
111
+ if v % 2 == 1:
112
+ v -= 1
113
+ return max(2, v)
114
+
115
+
116
+ def parse_aspect_ratio(value: Any) -> Optional[float]:
117
+ """
118
+ Accept:
119
+ - "16:9"
120
+ - float/int like 1.777
121
+ - (w, h)
122
+ """
123
+ if value is None:
124
+ return None
125
+
126
+ if isinstance(value, (int, float)):
127
+ r = float(value)
128
+ return r if r > 0 else None
129
+
130
+ if isinstance(value, str):
131
+ text = value.strip()
132
+ if ":" in text:
133
+ parts = text.split(":")
134
+ if len(parts) == 2:
135
+ try:
136
+ w = float(parts[0].strip())
137
+ h = float(parts[1].strip())
138
+ if w > 0 and h > 0:
139
+ return w / h
140
+ except Exception:
141
+ return None
142
+ else:
143
+ try:
144
+ r = float(text)
145
+ return r if r > 0 else None
146
+ except Exception:
147
+ return None
148
+
149
+ if isinstance(value, (tuple, list)) and len(value) == 2:
150
+ try:
151
+ w = float(value[0])
152
+ h = float(value[1])
153
+ if w > 0 and h > 0:
154
+ return w / h
155
+ except Exception:
156
+ return None
157
+
158
+ return None
159
+
160
+
161
+ def resolve_output_canvas_size(inputs: Dict[str, Any]) -> Tuple[int, int]:
162
+ """
163
+ Requirement:
164
+ 1) output aspect ratio decided by inputs
165
+ 2) keep output <=1080 (consistent with media<=1080 + performance)
166
+ """
167
+ explicit_size = inputs.get("output_size")
168
+ if isinstance(explicit_size, (tuple, list)) and len(explicit_size) == 2:
169
+ try:
170
+ w = max(2, int(explicit_size[0]))
171
+ h = max(2, int(explicit_size[1]))
172
+ scale = min(1.0, MAX_MEDIA_DIMENSION_PX / float(max(w, h)))
173
+ return (make_even(int(w * scale)), make_even(int(h * scale)))
174
+ except Exception:
175
+ pass
176
+
177
+ ratio = (
178
+ parse_aspect_ratio(inputs.get("output_aspect_ratio"))
179
+ or parse_aspect_ratio(inputs.get("aspect_ratio"))
180
+ or DEFAULT_OUTPUT_ASPECT_RATIO
181
+ )
182
+
183
+ max_dim = inputs.get("output_max_dimension_px", DEFAULT_OUTPUT_MAX_DIMENSION_PX)
184
+ try:
185
+ max_dim = int(max_dim)
186
+ except Exception:
187
+ max_dim = DEFAULT_OUTPUT_MAX_DIMENSION_PX
188
+ max_dim = max(2, min(MAX_MEDIA_DIMENSION_PX, max_dim))
189
+
190
+ if ratio >= 1.0:
191
+ width = max_dim
192
+ height = max(2, int(round(width / ratio)))
193
+ else:
194
+ height = max_dim
195
+ width = max(2, int(round(height * ratio)))
196
+
197
+ return (make_even(width), make_even(height))
198
+
199
+
200
+ def build_asset_id_to_path_map(assets_index: Dict[str, Any]) -> Dict[str, str]:
201
+ mapping: Dict[str, str] = {}
202
+ for item in (assets_index.get("videos") or []) + (assets_index.get("images") or []):
203
+ asset_id = item.get("asset_id")
204
+ path = item.get("path")
205
+ if asset_id and path:
206
+ mapping[asset_id] = path
207
+ return mapping
208
+
209
+
210
+ def is_image_file(path: str) -> bool:
211
+ try:
212
+ return Path(path).suffix.lower() in {".png", ".jpg", ".jpeg", ".bmp", ".webp", ".tif", ".tiff"}
213
+ except Exception:
214
+ return False
215
+
216
+
217
+ def make_mask_clip(mask: np.ndarray) -> ImageClip:
218
+ # moviepy versions may accept is_mask / ismask
219
+ try:
220
+ return ImageClip(mask, is_mask=True)
221
+ except TypeError: # pragma: no cover
222
+ return ImageClip(mask, ismask=True)
223
+
224
+
225
+ # =============================================================================
226
+ # Media cache: scale <=1080, drop alpha, and speed up images
227
+ # =============================================================================
228
+
229
+ class MediaCache:
230
+ def __init__(self, *, include_video_audio: bool, canvas_size: Tuple[int, int]) -> None:
231
+ self._include_video_audio = include_video_audio
232
+ self._canvas_size = canvas_size
233
+
234
+ self._video_sources: Dict[str, VideoFileClip] = {}
235
+ self._audio_sources: Dict[str, AudioFileClip] = {}
236
+ self._audio_to_close: List[AudioFileClip] = []
237
+
238
+ # Key optimization: cache full-canvas RGB frames for images
239
+ self._image_padded_frame_cache: Dict[str, np.ndarray] = {}
240
+ self._video_size_cache: Dict[str, Tuple[int, int]] = {}
241
+
242
+ def close(self) -> None:
243
+ for v in self._video_sources.values():
244
+ close_quietly(v)
245
+ for a in self._audio_to_close:
246
+ close_quietly(a)
247
+
248
+ def get_audio(self, path: str) -> AudioFileClip:
249
+ cached = self._audio_sources.get(path)
250
+ if cached is not None:
251
+ return cached
252
+ clip = AudioFileClip(path)
253
+ self._audio_sources[path] = clip
254
+ self._audio_to_close.append(clip)
255
+ return clip
256
+
257
+ def get_video(self, path: str) -> VideoFileClip:
258
+ cached = self._video_sources.get(path)
259
+ if cached is not None:
260
+ return cached
261
+
262
+ src_w, src_h = self._probe_video_size(path)
263
+ canvas_w, canvas_h = self._canvas_size
264
+
265
+ # fit into canvas and <=1080
266
+ max_w = min(canvas_w, MAX_MEDIA_DIMENSION_PX)
267
+ max_h = min(canvas_h, MAX_MEDIA_DIMENSION_PX)
268
+
269
+ target_resolution = None
270
+ if src_w > 0 and src_h > 0:
271
+ scale = min(1.0, max_w / float(src_w), max_h / float(src_h))
272
+ if scale < 1.0:
273
+ target_w = make_even(int(src_w * scale))
274
+ target_h = make_even(int(src_h * scale))
275
+ src_ratio = src_w / float(src_h)
276
+ canvas_ratio = canvas_w / float(canvas_h)
277
+ # preserve aspect by setting one dim None
278
+ if src_ratio >= canvas_ratio:
279
+ target_resolution = (None, target_w) # limit width
280
+ else:
281
+ target_resolution = (target_h, None) # limit height
282
+
283
+ try:
284
+ clip = VideoFileClip(path, audio=self._include_video_audio, target_resolution=target_resolution)
285
+ except TypeError:
286
+ clip = VideoFileClip(path, audio=self._include_video_audio)
287
+
288
+ self._video_sources[path] = clip
289
+ return clip
290
+
291
+ def get_image_padded_frame(self, path: str) -> np.ndarray:
292
+ cached = self._image_padded_frame_cache.get(path)
293
+ if cached is not None:
294
+ return cached
295
+
296
+ canvas_w, canvas_h = self._canvas_size
297
+ max_w = min(canvas_w, MAX_MEDIA_DIMENSION_PX)
298
+ max_h = min(canvas_h, MAX_MEDIA_DIMENSION_PX)
299
+
300
+ with Image.open(path) as image:
301
+ try:
302
+ image = ImageOps.exif_transpose(image)
303
+ except Exception:
304
+ pass
305
+
306
+ # drop alpha channel: RGBA -> alpha composite on black -> RGB
307
+ image = image.convert("RGBA")
308
+ black_bg = Image.new("RGBA", image.size, (0, 0, 0, 255))
309
+ try:
310
+ image = Image.alpha_composite(black_bg, image).convert("RGB")
311
+ except Exception:
312
+ image = image.convert("RGB")
313
+
314
+ # resize to fit (<=1080 and <=canvas)
315
+ try:
316
+ resample = Image.Resampling.LANCZOS
317
+ except Exception: # pragma: no cover
318
+ resample = Image.LANCZOS # type: ignore
319
+
320
+ image.thumbnail((max_w, max_h), resample=resample)
321
+ resized = np.array(image, dtype=np.uint8)
322
+
323
+ # build full-canvas frame (black + centered image)
324
+ canvas = np.zeros((canvas_h, canvas_w, 3), dtype=np.uint8)
325
+ h, w = resized.shape[0], resized.shape[1]
326
+ x0 = max(0, (canvas_w - w) // 2)
327
+ y0 = max(0, (canvas_h - h) // 2)
328
+ x1 = min(canvas_w, x0 + w)
329
+ y1 = min(canvas_h, y0 + h)
330
+ canvas[y0:y1, x0:x1] = resized[0 : (y1 - y0), 0 : (x1 - x0)]
331
+
332
+ self._image_padded_frame_cache[path] = canvas
333
+ return canvas
334
+
335
+ def _probe_video_size(self, path: str) -> Tuple[int, int]:
336
+ cached = self._video_size_cache.get(path)
337
+ if cached is not None:
338
+ return cached
339
+
340
+ w = h = 0
341
+ # simplest: open once (metadata fetch), then close
342
+ try:
343
+ tmp = VideoFileClip(path, audio=False)
344
+ w = int(getattr(tmp, "w", 0) or 0)
345
+ h = int(getattr(tmp, "h", 0) or 0)
346
+ close_quietly(tmp)
347
+ except Exception:
348
+ w, h = 0, 0
349
+
350
+ self._video_size_cache[path] = (w, h)
351
+ return w, h
352
+
353
+
354
+ # =============================================================================
355
+ # Subtitle renderer (RGB + mask; output frames are RGB, no alpha channel)
356
+ # =============================================================================
357
+
358
+ class PillowSubtitleRenderer:
359
+ def __init__(self, font_path: str) -> None:
360
+ self._font_path = font_path
361
+
362
+ def render(
363
+ self,
364
+ subtitle_items: List[Dict[str, Any]],
365
+ *,
366
+ video_size: Tuple[int, int],
367
+ ) -> List[ImageClip]:
368
+ canvas_w, canvas_h = video_size
369
+ scale = (canvas_h / SUBTITLE_BASE_HEIGHT_PX) if canvas_h > 0 else 1.0
370
+
371
+ font_size = clamp_int(SUBTITLE_FONT_SIZE_AT_BASE * scale, SUBTITLE_FONT_SIZE_MIN, SUBTITLE_FONT_SIZE_MAX)
372
+ margin_bottom = clamp_int(
373
+ SUBTITLE_MARGIN_BOTTOM_AT_BASE * scale, SUBTITLE_MARGIN_BOTTOM_MIN, SUBTITLE_MARGIN_BOTTOM_MAX
374
+ )
375
+ stroke_width = clamp_int(
376
+ SUBTITLE_STROKE_WIDTH_AT_BASE * scale, SUBTITLE_STROKE_WIDTH_MIN, SUBTITLE_STROKE_WIDTH_MAX
377
+ )
378
+
379
+ clips: List[ImageClip] = []
380
+ for item in subtitle_items:
381
+ text = str(item.get("text", "")).strip()
382
+ tw = item.get("timeline_window", {}) or {}
383
+ start_s = milliseconds_to_seconds(tw.get("start", 0.0))
384
+ end_s = milliseconds_to_seconds(tw.get("end", 0.0))
385
+ dur = end_s - start_s
386
+ if not text or dur <= 0:
387
+ continue
388
+
389
+ clip = self._make_clip(
390
+ text=text,
391
+ start_s=start_s,
392
+ end_s=end_s,
393
+ video_size=video_size,
394
+ font_size=font_size,
395
+ margin_bottom=margin_bottom,
396
+ stroke_width=stroke_width,
397
+ )
398
+ if clip is not None:
399
+ clips.append(clip)
400
+
401
+ return clips
402
+
403
+ def _make_clip(
404
+ self,
405
+ *,
406
+ text: str,
407
+ start_s: float,
408
+ end_s: float,
409
+ video_size: Tuple[int, int],
410
+ font_size: int,
411
+ margin_bottom: int,
412
+ stroke_width: int,
413
+ ) -> Optional[ImageClip]:
414
+ canvas_w, canvas_h = video_size
415
+ dur = end_s - start_s
416
+ if dur <= 0:
417
+ return None
418
+
419
+ font = self._load_font(font_size)
420
+
421
+ max_text_w = max(1, int(canvas_w * SUBTITLE_MAX_WIDTH_RATIO))
422
+ wrapped = self._wrap_text_by_width(text, font, max_text_w)
423
+
424
+ measure = Image.new("RGBA", (10, 10), (0, 0, 0, 0))
425
+ draw = ImageDraw.Draw(measure)
426
+ bbox = draw.multiline_textbbox(
427
+ (0, 0), wrapped, font=font, align="center", stroke_width=stroke_width
428
+ )
429
+ text_w = bbox[2] - bbox[0]
430
+ text_h = bbox[3] - bbox[1]
431
+
432
+ img_w = int(text_w + SUBTITLE_PADDING_X * 2)
433
+ img_h = int(text_h + SUBTITLE_PADDING_Y * 2)
434
+
435
+ rgba = Image.new("RGBA", (img_w, img_h), (0, 0, 0, 0))
436
+ draw = ImageDraw.Draw(rgba)
437
+ draw.multiline_text(
438
+ (SUBTITLE_PADDING_X - bbox[0], SUBTITLE_PADDING_Y - bbox[1]),
439
+ wrapped,
440
+ font=font,
441
+ fill=(255, 255, 255, 255),
442
+ align="center",
443
+ stroke_width=stroke_width,
444
+ stroke_fill=(0, 0, 0, 255),
445
+ )
446
+
447
+ rgba_arr = np.array(rgba, dtype=np.uint8)
448
+ rgb_arr = rgba_arr[:, :, :3]
449
+ alpha_mask = (rgba_arr[:, :, 3].astype(np.float32) / 255.0)
450
+
451
+ subtitle_clip = ImageClip(rgb_arr).with_mask(make_mask_clip(alpha_mask))
452
+ y = max(0, int(canvas_h - margin_bottom - img_h))
453
+ subtitle_clip = subtitle_clip.with_start(start_s).with_duration(dur).with_position(("center", y))
454
+ return subtitle_clip
455
+
456
+ def _load_font(self, font_size: int) -> ImageFont.FreeTypeFont:
457
+ try:
458
+ if self._font_path and os.path.exists(self._font_path):
459
+ return ImageFont.truetype(font=self._font_path, size=font_size)
460
+ except Exception:
461
+ pass
462
+ return ImageFont.load_default()
463
+
464
+ @staticmethod
465
+ def _wrap_text_by_width(text: str, font: ImageFont.FreeTypeFont, max_width_px: int) -> str:
466
+ text = text.strip()
467
+ if not text:
468
+ return ""
469
+
470
+ dummy = Image.new("RGB", (10, 10))
471
+ draw = ImageDraw.Draw(dummy)
472
+
473
+ lines: List[str] = []
474
+ for paragraph in text.splitlines():
475
+ paragraph = paragraph.strip()
476
+ if not paragraph:
477
+ lines.append("")
478
+ continue
479
+
480
+ current = ""
481
+ for ch in paragraph:
482
+ candidate = current + ch
483
+ if draw.textlength(candidate, font=font) <= max_width_px or not current:
484
+ current = candidate
485
+ else:
486
+ lines.append(current)
487
+ current = ch
488
+ if current:
489
+ lines.append(current)
490
+
491
+ return "\n".join(lines)
492
+
493
+
494
+ # =============================================================================
495
+ # Audio composer (same behavior as before)
496
+ # =============================================================================
497
+
498
+ class AudioTrackComposer:
499
+ def __init__(self, *, cache: MediaCache) -> None:
500
+ self._cache = cache
501
+
502
+ def compose(
503
+ self,
504
+ *,
505
+ voiceover_items: List[Dict[str, Any]],
506
+ bgm_items: List[Dict[str, Any]],
507
+ final_duration_s: float,
508
+ ):
509
+ layers: List[Any] = []
510
+
511
+ # voiceover
512
+ for item in voiceover_items:
513
+ path = item.get("path")
514
+ if not path:
515
+ continue
516
+ src = self._cache.get_audio(path)
517
+
518
+ sw = item.get("source_window", {}) or {}
519
+ tw = item.get("timeline_window", {}) or {}
520
+
521
+ src_start = milliseconds_to_seconds(sw.get("start", 0.0))
522
+ src_end = milliseconds_to_seconds(sw.get("end", 0.0))
523
+ if src_end <= src_start:
524
+ continue
525
+
526
+ tl_start = milliseconds_to_seconds(tw.get("start", 0.0))
527
+ tl_end = milliseconds_to_seconds(tw.get("end", 0.0))
528
+ expected = max(0.0, tl_end - tl_start)
529
+
530
+ clip = src.subclipped(src_start, src_end).with_start(tl_start)
531
+ if expected > 0:
532
+ clip = clip.with_duration(expected)
533
+ layers.append(clip)
534
+
535
+ # bgm (concat then loop/trim)
536
+ if bgm_items:
537
+ segments: List[Any] = []
538
+ for item in sorted(bgm_items, key=lambda x: int(x.get("loop_idx", 0))):
539
+ path = item.get("path")
540
+ if not path:
541
+ continue
542
+ src = self._cache.get_audio(path)
543
+ sw = item.get("source_window", {}) or {}
544
+ src_start = milliseconds_to_seconds(sw.get("start", 0.0))
545
+ src_end = milliseconds_to_seconds(sw.get("end", 0.0))
546
+ if src_end <= src_start:
547
+ continue
548
+ segments.append(src.subclipped(src_start, src_end))
549
+
550
+ if segments:
551
+ bgm = concatenate_audioclips(segments).with_volume_scaled(BGM_VOLUME_SCALE)
552
+ if bgm.duration is not None:
553
+ if bgm.duration < final_duration_s - AUDIO_DURATION_TOLERANCE_SECONDS:
554
+ bgm = self._loop_audio(bgm, final_duration_s)
555
+ elif bgm.duration > final_duration_s + AUDIO_DURATION_TOLERANCE_SECONDS:
556
+ bgm = bgm.subclipped(0, final_duration_s)
557
+ layers.append(bgm)
558
+
559
+ if not layers:
560
+ return None
561
+ return CompositeAudioClip(layers).with_duration(final_duration_s)
562
+
563
+ @staticmethod
564
+ def _loop_audio(audio_clip: Any, duration_s: float) -> Any:
565
+ try:
566
+ n = int(duration_s // audio_clip.duration) + 2
567
+ looped = concatenate_audioclips([audio_clip] * max(1, n))
568
+ return looped.subclipped(0, duration_s)
569
+ except Exception:
570
+ return audio_clip
571
+
572
+
573
+ # =============================================================================
574
+ # Pipeline (non-overlapping visuals + subtitle overlay)
575
+ # =============================================================================
576
+
577
+ class RenderVideoPipeline:
578
+ def __init__(self, *, server_cache_dir: Path) -> None:
579
+ self._server_cache_dir = server_cache_dir
580
+
581
+ def render(self, *, node_summary, inputs: Dict[str, Any]) -> Dict[str, Any]:
582
+ assets_index: Dict[str, Any] = inputs["assets_index"]
583
+ tracks: Dict[str, Any] = (inputs.get("plan_timeline") or {}).get("tracks", {}) or {}
584
+
585
+ artifact_id: str = inputs["artifact_id"]
586
+ session_id: str = inputs["session_id"]
587
+ outputs_dir: Path = self._server_cache_dir / session_id / artifact_id
588
+ outputs_dir.mkdir(parents=True, exist_ok=True)
589
+
590
+ video_items = tracks.get("video", []) or []
591
+ subtitle_items = tracks.get("subtitles", []) or []
592
+ voiceover_items = tracks.get("voiceover", []) or []
593
+ bgm_items = tracks.get("bgm", []) or []
594
+
595
+ if not video_items:
596
+ raise ValueError("timeline result has no video track")
597
+
598
+ output_canvas_size = resolve_output_canvas_size(inputs)
599
+ asset_map = build_asset_id_to_path_map(assets_index)
600
+
601
+ override_audio = bool(voiceover_items) or bool(bgm_items)
602
+ cache = MediaCache(include_video_audio=not override_audio, canvas_size=output_canvas_size)
603
+
604
+ subtitle_renderer = PillowSubtitleRenderer(font_path=str(SourceHanSansSC_FONTS["normal"]))
605
+ audio_composer = AudioTrackComposer(cache=cache)
606
+
607
+ temp_dir = tempfile.mkdtemp(prefix=TEMP_DIRECTORY_PREFIX)
608
+ output_name = f"output_{uuid.uuid4().hex[:8]}_{int(time.time() * 1000)}.mp4"
609
+ output_path = str((outputs_dir / output_name).resolve())
610
+
611
+ clips_to_close: List[Any] = []
612
+ subtitle_clips: List[Any] = []
613
+ base_clip = None
614
+ final_clip = None
615
+
616
+ try:
617
+ final_duration_s = self._final_duration_seconds(video_items)
618
+
619
+ base_clip, clips_to_close, output_fps = self._build_base_video_concat(
620
+ video_items=video_items,
621
+ asset_map=asset_map,
622
+ cache=cache,
623
+ canvas_size=output_canvas_size,
624
+ final_duration_s=final_duration_s,
625
+ )
626
+
627
+ subtitle_clips = subtitle_renderer.render(subtitle_items, video_size=output_canvas_size)
628
+ if subtitle_clips:
629
+ final_clip = CompositeVideoClip([base_clip, *subtitle_clips]).with_duration(final_duration_s)
630
+ else:
631
+ final_clip = base_clip
632
+
633
+ if override_audio:
634
+ final_audio = audio_composer.compose(
635
+ voiceover_items=voiceover_items,
636
+ bgm_items=bgm_items,
637
+ final_duration_s=final_duration_s,
638
+ )
639
+ if final_audio is not None:
640
+ final_clip = final_clip.with_audio(final_audio)
641
+ else:
642
+ final_clip = final_clip.without_audio()
643
+
644
+ final_clip.write_videofile(
645
+ output_path,
646
+ codec=VIDEO_CODEC,
647
+ audio_codec=AUDIO_CODEC,
648
+ temp_audiofile=os.path.join(temp_dir, TEMP_AUDIO_FILENAME),
649
+ remove_temp=True,
650
+ fps=output_fps,
651
+ ffmpeg_params=FFMPEG_PARAMS,
652
+ )
653
+
654
+ node_summary.info_for_user(
655
+ f"成功生成视频,时长为{final_duration_s}秒, 路径:{output_path}",
656
+ preview_urls=[output_path],
657
+ )
658
+ return {
659
+ "output_path": output_path,
660
+ "output_basename": output_name,
661
+ "duration_s": float(final_duration_s),
662
+ "output_size": {"width": int(output_canvas_size[0]), "height": int(output_canvas_size[1])},
663
+ }
664
+
665
+ finally:
666
+ for c in subtitle_clips:
667
+ close_quietly(c)
668
+ for c in clips_to_close:
669
+ close_quietly(c)
670
+ close_quietly(base_clip)
671
+ close_quietly(final_clip)
672
+ cache.close()
673
+
674
+ @staticmethod
675
+ def _final_duration_seconds(video_items: List[Dict[str, Any]]) -> float:
676
+ end_ms = max(float((it.get("timeline_window") or {}).get("end", 0.0)) for it in video_items)
677
+ return milliseconds_to_seconds(end_ms)
678
+
679
+ @staticmethod
680
+ def _build_base_video_concat(
681
+ *,
682
+ video_items: List[Dict[str, Any]],
683
+ asset_map: Dict[str, str],
684
+ cache: MediaCache,
685
+ canvas_size: Tuple[int, int],
686
+ final_duration_s: float,
687
+ ) -> Tuple[Any, List[Any], float]:
688
+ # Force non-overlapping: concat only
689
+ sorted_items = sorted(video_items, key=lambda x: float((x.get("timeline_window") or {}).get("start", 0.0)))
690
+
691
+ clips: List[Any] = []
692
+ clips_to_close: List[Any] = []
693
+ current_time = 0.0
694
+
695
+ def black_clip(duration: float) -> Any:
696
+ c = ColorClip(size=canvas_size, color=BACKGROUND_COLOR_RGB).with_duration(max(0.0, duration))
697
+ clips_to_close.append(c)
698
+ return c
699
+
700
+ for segment in sorted_items:
701
+ timeline_window = segment.get("timeline_window", {}) or {}
702
+ start_s = milliseconds_to_seconds(timeline_window.get("start", 0.0))
703
+ end_s = milliseconds_to_seconds(timeline_window.get("end", 0.0))
704
+ expected_dur = max(0.0, end_s - start_s)
705
+ if expected_dur <= 0:
706
+ continue
707
+
708
+ # fill gap
709
+ if start_s > current_time:
710
+ clips.append(black_clip(start_s - current_time))
711
+ current_time = start_s
712
+
713
+ seg_clip = RenderVideoPipeline._build_full_canvas_segment(
714
+ segment=segment,
715
+ asset_map=asset_map,
716
+ cache=cache,
717
+ canvas_size=canvas_size,
718
+ expected_duration_s=expected_dur,
719
+ )
720
+
721
+ if seg_clip is None:
722
+ clips.append(black_clip(expected_dur))
723
+ else:
724
+ clips.append(seg_clip)
725
+ clips_to_close.append(seg_clip)
726
+
727
+ current_time = start_s + expected_dur
728
+
729
+ # trailing gap
730
+ if final_duration_s > current_time:
731
+ clips.append(black_clip(final_duration_s - current_time))
732
+
733
+ if not clips:
734
+ raise ValueError("no valid video segments")
735
+
736
+ base = concatenate_videoclips(clips, method="chain").with_duration(final_duration_s)
737
+
738
+ fps_values = [float(it.get("fps")) for it in video_items if it.get("fps")]
739
+ output_fps = max(fps_values) if fps_values else float(getattr(base, "fps", None) or DEFAULT_OUTPUT_FPS)
740
+
741
+ return base, clips_to_close, output_fps
742
+
743
+ @staticmethod
744
+ def _build_full_canvas_segment(
745
+ *,
746
+ segment: Dict[str, Any],
747
+ asset_map: Dict[str, str],
748
+ cache: MediaCache,
749
+ canvas_size: Tuple[int, int],
750
+ expected_duration_s: float,
751
+ ) -> Optional[Any]:
752
+ source_path = segment.get("source_path") or asset_map.get(segment.get("asset_id"))
753
+ if not source_path:
754
+ return None
755
+
756
+ if is_image_file(source_path):
757
+ frame = cache.get_image_padded_frame(source_path)
758
+ return ImageClip(frame).with_duration(expected_duration_s)
759
+
760
+ # video
761
+ source = cache.get_video(source_path)
762
+ source_window = segment.get("source_window", {}) or {}
763
+
764
+ src_start = milliseconds_to_seconds(source_window.get("start", 0.0))
765
+
766
+ end_ms = source_window.get("end", None)
767
+ if end_ms is None:
768
+ src_end = float(getattr(source, "duration", 0.0) or 0.0)
769
+ else:
770
+ src_end = milliseconds_to_seconds(end_ms)
771
+
772
+ if src_end <= src_start:
773
+ return None
774
+
775
+ # Clamp to avoid "end_time > duration" (common near EOF due to rounding/encoding)
776
+ source_duration_s = float(getattr(source, "duration", 0.0) or 0.0)
777
+ if source_duration_s > 0.0:
778
+ if src_start >= source_duration_s:
779
+ return None
780
+ if src_end > source_duration_s:
781
+ src_end = source_duration_s # <= duration
782
+
783
+ if src_end <= src_start:
784
+ return None
785
+
786
+ clip = source.subclipped(src_start, src_end)
787
+
788
+ playback_rate = float(segment.get("playback_rate", 1.0) or 1.0)
789
+ if playback_rate != 1.0:
790
+ clip = clip.with_speed_scaled(playback_rate)
791
+
792
+ clip_dur = float(getattr(clip, "duration", 0.0) or 0.0)
793
+ if clip_dur > 0.0 and expected_duration_s > clip_dur:
794
+ last_t = max(0.0, clip_dur - SUBCLIP_END_SAFETY_MARGIN_S)
795
+ # Freeze video/mask at last frame for the remaining duration
796
+ clip = clip.time_transform(
797
+ lambda t, lt=last_t: min(t, lt),
798
+ apply_to=["mask"],
799
+ keep_duration=True,
800
+ )
801
+
802
+ clip = clip.with_duration(expected_duration_s)
803
+
804
+ # pad to full canvas (keep original "center on black" look)
805
+ if hasattr(clip, "on_color"):
806
+ clip = clip.on_color(size=canvas_size, color=BACKGROUND_COLOR_RGB, pos=CENTER_POSITION)
807
+ else: # pragma: no cover
808
+ bg = ColorClip(size=canvas_size, color=BACKGROUND_COLOR_RGB).with_duration(expected_duration_s)
809
+ clip = CompositeVideoClip([bg, clip.with_position(CENTER_POSITION)]).with_duration(expected_duration_s)
810
+
811
+ return clip.with_duration(expected_duration_s)
812
+
813
+
814
+ # =============================================================================
815
+ # Node entrypoint
816
+ # =============================================================================
817
+
818
+ class RenderVideoNode(BaseNode):
819
+ meta = NodeMeta(
820
+ name="render_video",
821
+ description="根据组织好的时间线渲染成片。",
822
+ node_id="render_video",
823
+ node_kind="render_video",
824
+ require_prior_kind=["assets_index", "plan_timeline"],
825
+ default_require_prior_kind=["assets_index", "plan_timeline"],
826
+ )
827
+
828
+ input_schema = RenderVideoInput
829
+
830
+ def __init__(self, server_cfg: Settings) -> None:
831
+ super().__init__(server_cfg)
832
+ self._pipeline = RenderVideoPipeline(server_cache_dir=self.server_cache_dir)
833
+
834
+ async def default_process(self, node_summary, inputs: Dict[str, Any]) -> Any:
835
+ return await self.process(node_summary, inputs)
836
+
837
+ async def process(self, node_summary, inputs: Dict[str, Any]) -> Dict[str, Any]:
838
+ return self._pipeline.render(node_summary=node_summary, inputs=inputs)
src/open_storyline/nodes/core_nodes/select_bgm.py ADDED
@@ -0,0 +1,237 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Any, Dict
2
+ from pathlib import Path
3
+
4
+ import numpy as np
5
+ import random
6
+ import librosa
7
+
8
+ from open_storyline.nodes.core_nodes.base_node import BaseNode, NodeMeta
9
+ from open_storyline.nodes.node_schema import SelectBGMInput
10
+
11
+ class SelectBGMNode(BaseNode):
12
+ meta = NodeMeta(
13
+ name="select_bgm",
14
+ description="根据用户的要求,选择合适的BGM",
15
+ node_id="select_bgm",
16
+ node_kind="music_rec",
17
+ require_prior_kind=[],
18
+ default_require_prior_kind=[],
19
+ next_available_node=["plan_timeline"],
20
+ )
21
+
22
+ input_schema = SelectBGMInput
23
+
24
+ async def default_process(
25
+ self,
26
+ node_summary,
27
+ inputs: Dict[str, Any],
28
+ ) -> Any:
29
+ node_summary.info_for_user("未能选择音乐")
30
+ return {"bgm": {}}
31
+
32
+
33
+ async def process(self, node_summary, inputs: Dict[str, Any]) -> Any:
34
+ cfg = self.server_cfg
35
+ user_request = inputs["user_request"]
36
+ bgm_dir = cfg.project.bgm_dir
37
+ bgm_path = recommend_bgm(bgm_dir)
38
+
39
+ result = analyze_music_metrics(path=bgm_path, sr=cfg.select_bgm.sample_rate, hop_length=cfg.select_bgm.hop_length, frame_length=cfg.select_bgm.frame_length)
40
+ if result.get("path"):
41
+ node_summary.info_for_user(f"成功选择音乐", preview_urls = [result.get("path")])
42
+ else:
43
+ node_summary.info_for_user("未能选择音乐")
44
+ return {"bgm": result}
45
+
46
+
47
+ def recommend_bgm(bgm_dir: Path):
48
+ bgm_dir = bgm_dir.expanduser().resolve()
49
+ if not bgm_dir.exists():
50
+ raise FileNotFoundError(f"bgm_dir not found: {bgm_dir}")
51
+ if not bgm_dir.is_dir():
52
+ raise NotADirectoryError(f"bgm_dir is not a directory: {bgm_dir}")
53
+
54
+ exts = {".mp3"}
55
+ candidates = [
56
+ p for p in bgm_dir.iterdir()
57
+ if p.is_file() and p.suffix.lower() in exts
58
+ ]
59
+ if not candidates:
60
+ raise FileNotFoundError(f"No audio files found in: {bgm_dir}")
61
+ return random.choice(candidates)
62
+
63
+ def _load_audio_mono(path: Path, sr: int) -> tuple[np.ndarray, int]:
64
+
65
+ try:
66
+ y, sr_out = librosa.load(path, sr=sr, mono=True)
67
+ return y.astype(np.float32, copy=False), int(sr_out)
68
+ except Exception as e1:
69
+ # librosa读取失败,用ffmpeg兜底
70
+ import os
71
+ import subprocess
72
+ import tempfile
73
+
74
+ tmp_wav = None
75
+ try:
76
+ with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
77
+ tmp_wav = tmp.name
78
+
79
+ cmd = [
80
+ "ffmpeg",
81
+ "-y",
82
+ "-i", str(path),
83
+ "-ac", "1",
84
+ "-ar", str(sr),
85
+ "-vn",
86
+ tmp_wav,
87
+ ]
88
+ subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
89
+
90
+ y, sr_out = librosa.load(tmp_wav, sr=sr, mono=True)
91
+ return y.astype(np.float32, copy=False), int(sr_out)
92
+
93
+ except FileNotFoundError as e_ffmpeg:
94
+ raise RuntimeError(
95
+ f"无法加载音频,并且没有找到ffmpeg"
96
+ ) from e_ffmpeg
97
+
98
+ except Exception as e2:
99
+ raise RuntimeError(
100
+ f"无法加载音频:{type(e1).__name__}: {e1}"
101
+ f"ffmpeg 错误:{type(e2).__name__}: {e2}"
102
+ ) from e2
103
+ finally:
104
+ if tmp_wav is not None:
105
+ try:
106
+ os.remove(tmp_wav)
107
+ except Exception:
108
+ pass
109
+
110
+ def analyze_music_metrics(
111
+ path: Path,
112
+ sr: int = 22050,
113
+ hop_length = 2048,
114
+ frame_length = 2048,
115
+
116
+ ) -> dict[str, Any]:
117
+ if not path.exists():
118
+ raise FileNotFoundError(f"File not found: {path}")
119
+
120
+ y, sample_rate = _load_audio_mono(path, sr=sr)
121
+ duration = int(librosa.get_duration(y=y, sr=sample_rate) * 1000)
122
+
123
+ if y.size < frame_length:
124
+ raise RuntimeError("选择的BGM过短")
125
+
126
+ onset_env = librosa.onset.onset_strength(y=y, sr=sample_rate, hop_length=hop_length)
127
+ bpm, beat_frames = librosa.beat.beat_track(
128
+ onset_envelope=onset_env,
129
+ sr=sr,
130
+ hop_length=hop_length,
131
+ units="frames",
132
+ )
133
+
134
+ bpm_val = float(np.atleast_1d(bpm)[0])
135
+
136
+ beat_frames = np.asarray(beat_frames, dtype=int)
137
+
138
+ beat_times = compute_accent_beats(y=y, sr=sample_rate, beat_frames=beat_frames, hop_length=hop_length)
139
+
140
+ rms = librosa.feature.rms(
141
+ y=y,
142
+ frame_length=frame_length,
143
+ hop_length=hop_length
144
+ )[0]
145
+
146
+ energy_mean = float(np.mean(rms))
147
+
148
+ rms_db = librosa.amplitude_to_db(np.maximum(rms, 1e-10), ref=1.0)
149
+ energy_mean_db = float(np.mean(rms_db))
150
+
151
+ lo = float(np.percentile(rms_db, 10.0))
152
+ hi = float(np.percentile(rms_db, 95.0))
153
+ dynamic_range_db = float(hi - lo)
154
+
155
+ return {
156
+ "bgm_id": path.stem,
157
+ "path": str(path),
158
+ "duration": duration,
159
+ "sample_rate": sample_rate,
160
+ "bpm": bpm_val,
161
+ "beats": beat_times,
162
+ "energy_mean": energy_mean,
163
+ "energy_mean_db": energy_mean_db,
164
+ "dynamic_range_db": dynamic_range_db,
165
+ }
166
+
167
+
168
+
169
+ def compute_accent_beats(
170
+ y: np.ndarray,
171
+ sr: int,
172
+ beat_frames: np.ndarray,
173
+ hop_length: int,
174
+ top_pct: float = 70.0,
175
+ min_sep_beats: int = 1, # 抑制相邻 beat:1 表示避免连续拍都入选
176
+ use_percussive: bool = True, # 用打击成分算 onset
177
+ local_norm_win: int = 8, # 局部归一化窗口(以 beat 为单位)
178
+ require_local_peak: bool = True # 只保留局部峰值
179
+ ) -> list[float]:
180
+ """
181
+ 计算前 top_pct % 强度的鼓点的时间戳
182
+ """
183
+
184
+ if beat_frames.size == 0:
185
+ return []
186
+
187
+ # 1) onset envelope 用 percussive 版本
188
+ y_for_onset = librosa.effects.percussive(y) if use_percussive else y
189
+ onset_env = librosa.onset.onset_strength(y=y_for_onset, sr=sr, hop_length=hop_length)
190
+
191
+ # 2) 取每个 beat 时刻的 onset 强度作为 beat 强度
192
+ beat_frames_clip = np.clip(beat_frames.astype(int), 0, len(onset_env) - 1)
193
+ strength = onset_env[beat_frames_clip].astype(np.float64) # shape (n_beats,)
194
+
195
+ # 3) 局部归一化:避免“整体更响的一段”把所有 beat 都选到
196
+ if strength.size >= 3 and local_norm_win >= 3:
197
+ w = int(local_norm_win)
198
+ kernel = np.ones(w, dtype=np.float64) / w
199
+ local_mean = np.convolve(strength, kernel, mode="same")
200
+ strength_norm = strength / (local_mean + 1e-8)
201
+ else:
202
+ strength_norm = strength.copy()
203
+
204
+ # 4) top 30%
205
+ thr = float(np.percentile(strength_norm, 100.0 - top_pct))
206
+ cand = np.where(strength_norm >= thr)[0] # indices into beats
207
+
208
+ # 5) 只保留局部峰值,避免平台期很多 beat 都入选
209
+ if require_local_peak and cand.size > 0 and strength_norm.size >= 3:
210
+ is_peak = np.zeros_like(strength_norm, dtype=bool)
211
+ is_peak[1:-1] = (strength_norm[1:-1] >= strength_norm[:-2]) & (strength_norm[1:-1] >= strength_norm[2:])
212
+ is_peak[0] = strength_norm[0] >= strength_norm[1]
213
+ is_peak[-1] = strength_norm[-1] >= strength_norm[-2]
214
+ cand = cand[is_peak[cand]]
215
+
216
+ # 6) 最小间隔抑制
217
+ selected = []
218
+ if cand.size > 0:
219
+ order = cand[np.argsort(-strength_norm[cand])]
220
+ suppressed = np.zeros(strength_norm.size, dtype=bool)
221
+
222
+ for idx in order:
223
+ if suppressed[idx]:
224
+ continue
225
+ selected.append(int(idx))
226
+ lo = max(0, idx - min_sep_beats)
227
+ hi = min(strength_norm.size, idx + min_sep_beats + 1)
228
+ suppressed[lo:hi] = True
229
+
230
+ selected = np.array(sorted(selected), dtype=int)
231
+
232
+ accent_frames = beat_frames[selected]
233
+ accent_times = librosa.frames_to_time(accent_frames, sr=sr, hop_length=hop_length).tolist()
234
+
235
+ accent_times_ms = [round(x * 1000) for x in accent_times]
236
+
237
+ return accent_times_ms
src/open_storyline/nodes/core_nodes/split_shots.py ADDED
@@ -0,0 +1,778 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from dataclasses import dataclass
4
+ from pathlib import Path
5
+ from typing import Any, Dict, Optional, List, Tuple
6
+ import csv
7
+ import functools
8
+ import os
9
+ import shutil
10
+ import subprocess
11
+ import math
12
+
13
+
14
+ import numpy as np
15
+
16
+ from open_storyline.nodes.core_nodes.base_node import BaseNode, NodeMeta
17
+ from open_storyline.nodes.node_schema import SplitShotsInput
18
+
19
+ MODEL_CACHE_MAXSIZE = 4
20
+
21
+ # TransNetV2 expects frames with shape [..., 27, 48, 3] in this implementation.
22
+ TRANSNETV2_INPUT_HEIGHT = 27
23
+ TRANSNETV2_INPUT_WIDTH = 48
24
+ TRANSNETV2_INPUT_CHANNELS = 3
25
+
26
+ DEFAULT_SCENE_DETECTION_FRAMES_PER_SECOND = 25
27
+ DEFAULT_SCENE_DETECTION_THRESHOLD = 0.5
28
+ DEFAULT_SPLIT_POINT_MINIMUM_GAP_SECONDS = 1e-3
29
+
30
+ DEFAULT_MIN_SHOT_DURATION_MILLISECONDS = 1000
31
+ DEFAULT_MAX_SHOT_DURATION_MILLISECONDS = 30000
32
+
33
+ CLIP_ID_NUMBER_WIDTH = 4
34
+ MILLISECONDS_PER_SECOND = 1000.0
35
+
36
+ FFMPEG_LOGLEVEL = "error"
37
+ FFMPEG_PIXEL_FORMAT_RGB24 = "rgb24"
38
+ FFMPEG_SCALE_FLAGS = "fast_bilinear"
39
+ FFMPEG_STDOUT_PIPE = "pipe:1"
40
+
41
+ FFMPEG_ENVIRONMENT_VARIABLE_KEYS = ("IMAGEIO_FFMPEG_EXE", "FFMPEG_BINARY")
42
+
43
+ # 如果检测后不需要切分(没有 split points),是否仍然 copy 一份到 output_dir
44
+ COPY_VIDEO_WHEN_NO_SPLIT = False
45
+
46
+ @dataclass(frozen=True)
47
+ class VideoSegment:
48
+ path: Path
49
+ start_seconds: float
50
+ end_seconds: float # ffmpeg segment csv might use -1 for "until end" in our wrapper
51
+
52
+ # =========================
53
+ # Model / ffmpeg helpers
54
+ # =========================
55
+
56
+ @functools.lru_cache(maxsize=MODEL_CACHE_MAXSIZE)
57
+ def load_transnetv2_model_cached(weight_path: str, device: str = "auto"):
58
+ """
59
+ Load TransNetV2 model with LRU cache. Suitable for service mode.
60
+ """
61
+ import torch
62
+ from transnetv2_pytorch import TransNetV2
63
+
64
+ model = TransNetV2(device=device)
65
+ model.eval()
66
+
67
+ state_dict = torch.load(weight_path, map_location=model.device)
68
+ model.load_state_dict(state_dict)
69
+ return model
70
+
71
+
72
+ def resolve_ffmpeg_executable() -> str:
73
+ """
74
+ Resolve ffmpeg executable path:
75
+ 1) env var IMAGEIO_FFMPEG_EXE / FFMPEG_BINARY
76
+ 2) system PATH
77
+ 3) imageio-ffmpeg
78
+ """
79
+ # 1) Environment variables
80
+ for key in FFMPEG_ENVIRONMENT_VARIABLE_KEYS:
81
+ configured_value = os.getenv(key)
82
+ if not configured_value:
83
+ continue
84
+
85
+ configured_path = Path(configured_value).expanduser()
86
+ if configured_path.exists():
87
+ return str(configured_path)
88
+
89
+ # env var may also be just "ffmpeg" or a command name
90
+ resolved_from_path = shutil.which(configured_value)
91
+ if resolved_from_path:
92
+ return resolved_from_path
93
+
94
+ # 2) System PATH
95
+ ffmpeg_in_path = shutil.which("ffmpeg")
96
+ if ffmpeg_in_path:
97
+ return ffmpeg_in_path
98
+
99
+ # 3) imageio-ffmpeg
100
+ try:
101
+ import imageio_ffmpeg
102
+ ffmpeg_from_imageio = imageio_ffmpeg.get_ffmpeg_exe()
103
+ if ffmpeg_from_imageio:
104
+ return ffmpeg_from_imageio
105
+ except Exception:
106
+ pass
107
+
108
+ raise RuntimeError("ffmpeg not found (checked env vars, PATH, and imageio-ffmpeg).")
109
+
110
+
111
+ def read_video_frames_as_rgb24(
112
+ input_video: Path,
113
+ ffmpeg_executable: str,
114
+ *,
115
+ frames_per_second: int = DEFAULT_SCENE_DETECTION_FRAMES_PER_SECOND,
116
+ target_width: int = TRANSNETV2_INPUT_WIDTH,
117
+ target_height: int = TRANSNETV2_INPUT_HEIGHT,
118
+ ) -> np.ndarray:
119
+ """
120
+ Use ffmpeg to decode frames at fixed FPS and fixed size, output as raw RGB24 bytes.
121
+ Returns: np.ndarray with shape [frame_count, target_height, target_width, 3], dtype=uint8
122
+ """
123
+ input_video = Path(input_video)
124
+
125
+ video_filter = (
126
+ f"fps={frames_per_second},"
127
+ f"scale={target_width}:{target_height}:flags={FFMPEG_SCALE_FLAGS}"
128
+ )
129
+
130
+ command = [
131
+ ffmpeg_executable, "-hide_banner", "-loglevel", FFMPEG_LOGLEVEL, "-nostdin",
132
+ "-i", str(input_video),
133
+ "-an",
134
+ "-vf", video_filter,
135
+ "-pix_fmt", FFMPEG_PIXEL_FORMAT_RGB24,
136
+ "-f", "rawvideo",
137
+ FFMPEG_STDOUT_PIPE,
138
+ ]
139
+
140
+ process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
141
+ assert process.stdout is not None and process.stderr is not None
142
+
143
+ stdout_bytes, stderr_bytes = process.communicate()
144
+ if process.returncode != 0:
145
+ raise RuntimeError(
146
+ f"ffmpeg frame extraction failed: {input_video}\n"
147
+ f"{stderr_bytes.decode('utf-8', errors='replace')}"
148
+ )
149
+
150
+ bytes_per_frame = target_width * target_height * TRANSNETV2_INPUT_CHANNELS
151
+ frame_count = len(stdout_bytes) // bytes_per_frame
152
+
153
+ if frame_count <= 0:
154
+ return np.empty((0, target_height, target_width, TRANSNETV2_INPUT_CHANNELS), dtype=np.uint8)
155
+
156
+ stdout_bytes = stdout_bytes[: frame_count * bytes_per_frame]
157
+ frames = np.frombuffer(stdout_bytes, dtype=np.uint8).reshape(
158
+ (frame_count, target_height, target_width, TRANSNETV2_INPUT_CHANNELS)
159
+ )
160
+ return frames
161
+
162
+
163
+ def detect_scenes_with_transnetv2_without_proxy(
164
+ model: Any,
165
+ input_video: Path,
166
+ ffmpeg_executable: str,
167
+ *,
168
+ frames_per_second: int = DEFAULT_SCENE_DETECTION_FRAMES_PER_SECOND,
169
+ threshold: float = DEFAULT_SCENE_DETECTION_THRESHOLD,
170
+ ) -> List[Dict[str, Any]]:
171
+ """
172
+ No proxy file:
173
+ ffmpeg -> frames (uint8) -> model.predict_raw -> predictions_to_scenes_with_data
174
+ """
175
+ import torch
176
+
177
+ frames_numpy = read_video_frames_as_rgb24(
178
+ input_video,
179
+ ffmpeg_executable,
180
+ frames_per_second=frames_per_second,
181
+ target_width=TRANSNETV2_INPUT_WIDTH,
182
+ target_height=TRANSNETV2_INPUT_HEIGHT,
183
+ )
184
+
185
+ if frames_numpy.size == 0 or frames_numpy.shape[0] == 0:
186
+ return []
187
+
188
+ frames_tensor = torch.from_numpy(frames_numpy) # [T, H, W, 3], uint8
189
+ frames_tensor = frames_tensor.unsqueeze(0).contiguous() # [1, T, H, W, 3]
190
+
191
+ model_device = getattr(model, "device", None)
192
+ if model_device is not None:
193
+ frames_tensor = frames_tensor.to(model_device, non_blocking=True)
194
+
195
+ with torch.inference_mode():
196
+ single_prediction, _all_prediction = model.predict_raw(frames_tensor)
197
+
198
+ prediction = single_prediction.detach().cpu().numpy()
199
+ prediction = np.squeeze(prediction)
200
+ if prediction.ndim != 1:
201
+ prediction = prediction.reshape(-1)
202
+
203
+ scenes = model.predictions_to_scenes_with_data(
204
+ prediction,
205
+ fps=float(frames_per_second),
206
+ threshold=float(threshold),
207
+ )
208
+ return scenes
209
+
210
+
211
+ def convert_scenes_to_split_points_seconds(
212
+ scenes: List[Dict[str, Any]],
213
+ *,
214
+ minimum_gap_seconds: float = DEFAULT_SPLIT_POINT_MINIMUM_GAP_SECONDS,
215
+ ) -> List[float]:
216
+ """
217
+ Convert TransNetV2 scenes to ffmpeg segment split points.
218
+ split points: [t1, t2, ...] means segments [0,t1], [t1,t2], ..., [last,end]
219
+ """
220
+ end_times: List[float] = []
221
+ last_end_time = 0.0
222
+
223
+ for scene in scenes:
224
+ try:
225
+ end_time = float(scene.get("end_time", 0.0))
226
+ except Exception:
227
+ continue
228
+
229
+ if end_time > last_end_time + minimum_gap_seconds:
230
+ end_times.append(end_time)
231
+ last_end_time = end_time
232
+
233
+ # If <=1 scene, don't split
234
+ if len(end_times) <= 1:
235
+ return []
236
+
237
+ # Remove the last end_time (usually video end)
238
+ return end_times[:-1]
239
+
240
+ def enforce_shot_duration_constraints_on_split_points_seconds(
241
+ split_points_seconds: List[float],
242
+ *,
243
+ total_duration_milliseconds: int,
244
+ min_shot_duration_milliseconds: Optional[int],
245
+ max_shot_duration_milliseconds: Optional[int],
246
+ ) -> List[float]:
247
+ """
248
+ 对“切分点(split points)”施加 min/max 时长约束(单位 ms):
249
+ 1) 若某段 < min:通过删除相应切分点,把它与相邻段拼接(优先向后拼;尾段太短则向前拼)。
250
+ 2) 若某段 > max:在该段内部强制均匀切分(允许镜头内仍有镜头切换)。
251
+
252
+ 注意:这里在“调用 ffmpeg 前”调整 split points,从而避免切完后再做文件拼接(性能更好)。
253
+ """
254
+ duration_ms = int(total_duration_milliseconds)
255
+
256
+ def _normalize_optional_ms(value: Optional[int], key_name: str) -> Optional[int]:
257
+ if value is None:
258
+ return None
259
+ if isinstance(value, bool):
260
+ raise ValueError(f"{key_name} must be int milliseconds, got bool")
261
+ try:
262
+ value_int = int(value)
263
+ except Exception as exc:
264
+ raise ValueError(f"{key_name} must be int milliseconds, got {value!r}") from exc
265
+ if value_int < 0:
266
+ raise ValueError(f"{key_name} must be >= 0, got {value_int}")
267
+ if value_int == 0:
268
+ return None
269
+ return value_int
270
+
271
+ min_ms = _normalize_optional_ms(min_shot_duration_milliseconds, "min_shot_duration")
272
+ max_ms = _normalize_optional_ms(max_shot_duration_milliseconds, "max_shot_duration")
273
+
274
+ if min_ms is not None and max_ms is not None and min_ms > max_ms:
275
+ raise ValueError(f"min_shot_duration ({min_ms}) cannot be greater than max_shot_duration ({max_ms}).")
276
+
277
+ # seconds -> milliseconds cut points
278
+ cut_points_ms = [
279
+ int(round(point_seconds * MILLISECONDS_PER_SECOND))
280
+ for point_seconds in split_points_seconds
281
+ ]
282
+ cut_points_ms = sorted({c for c in cut_points_ms if 0 < c < duration_ms})
283
+
284
+ # ---------- Step 1: merge short segments (< min) by removing cut points ----------
285
+ if min_ms is not None and cut_points_ms:
286
+ merged_cut_points: List[int] = []
287
+ segment_start_ms = 0
288
+
289
+ for cut_ms in cut_points_ms:
290
+ segment_length_ms = cut_ms - segment_start_ms
291
+ if segment_length_ms < min_ms:
292
+ continue
293
+ merged_cut_points.append(cut_ms)
294
+ segment_start_ms = cut_ms
295
+
296
+ if merged_cut_points and (duration_ms - merged_cut_points[-1] < min_ms):
297
+ merged_cut_points.pop()
298
+
299
+ cut_points_ms = merged_cut_points
300
+
301
+ # ---------- Step 2: split long segments (> max) by inserting internal cut points ----------
302
+ if max_ms is not None and max_ms > 0:
303
+ cuts_set = set(cut_points_ms)
304
+ boundaries = [0] + cut_points_ms + [duration_ms]
305
+
306
+ for segment_start_ms, segment_end_ms in zip(boundaries[:-1], boundaries[1:]):
307
+ segment_length_ms = segment_end_ms - segment_start_ms
308
+ if segment_length_ms <= max_ms:
309
+ continue
310
+
311
+ # 最少切成 pieces 段,保证每段 <= max_ms
312
+ pieces = int(math.ceil(segment_length_ms / max_ms))
313
+ if pieces <= 1:
314
+ continue
315
+
316
+ # 均匀分配(整数 ms),尽量避免“最后剩一小段”
317
+ base = segment_length_ms // pieces
318
+ remainder = segment_length_ms % pieces # 前 remainder 段多 1ms
319
+
320
+ current = segment_start_ms
321
+ for i in range(pieces - 1):
322
+ piece_len = base + 1 if i < remainder else base
323
+ current += piece_len
324
+ if segment_start_ms < current < segment_end_ms:
325
+ cuts_set.add(current)
326
+
327
+ cut_points_ms = sorted(c for c in cuts_set if 0 < c < duration_ms)
328
+
329
+ # milliseconds -> seconds
330
+ return [cut_ms / MILLISECONDS_PER_SECOND for cut_ms in cut_points_ms]
331
+
332
+ def segment_video_stream_copy_with_ffmpeg(
333
+ input_video: Path,
334
+ ffmpeg_executable: str,
335
+ *,
336
+ split_points_seconds: List[float],
337
+ output_directory: Path,
338
+ filename_prefix: str,
339
+ start_index: int = 0,
340
+ ) -> List[VideoSegment]:
341
+ """
342
+ Fast segmentation: stream copy (-c copy) + segment muxer.
343
+ Returns segments with start/end in seconds from ffmpeg segment list csv.
344
+ """
345
+ output_directory.mkdir(parents=True, exist_ok=True)
346
+
347
+ # No split points -> single output copy
348
+ if not split_points_seconds:
349
+ output_path = output_directory / f"{filename_prefix}_{start_index:0{CLIP_ID_NUMBER_WIDTH}d}.mp4"
350
+ command = [
351
+ ffmpeg_executable, "-hide_banner", "-loglevel", FFMPEG_LOGLEVEL, "-nostdin",
352
+ "-y",
353
+ "-i", str(input_video),
354
+ "-map", "0",
355
+ "-c", "copy",
356
+ "-movflags", "+faststart",
357
+ str(output_path),
358
+ ]
359
+ completed = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
360
+ if completed.returncode != 0:
361
+ raise RuntimeError(
362
+ f"ffmpeg stream copy failed: {input_video}\n"
363
+ f"{completed.stderr.decode('utf-8', errors='replace')}"
364
+ )
365
+ return [VideoSegment(path=output_path, start_seconds=0.0, end_seconds=-1.0)]
366
+
367
+ split_points_argument = ",".join(f"{t:.3f}" for t in split_points_seconds)
368
+
369
+ segment_list_csv_path = output_directory / f"{filename_prefix}_{start_index:0{CLIP_ID_NUMBER_WIDTH}d}.csv"
370
+ output_pattern = output_directory / f"{filename_prefix}_%0{CLIP_ID_NUMBER_WIDTH}d.mp4"
371
+
372
+ command = [
373
+ ffmpeg_executable, "-hide_banner", "-loglevel", FFMPEG_LOGLEVEL, "-nostdin",
374
+ "-y",
375
+ "-i", str(input_video),
376
+ "-map", "0",
377
+ "-c", "copy",
378
+ "-f", "segment",
379
+ "-segment_start_number", str(start_index),
380
+ "-segment_list", str(segment_list_csv_path),
381
+ "-segment_list_type", "csv",
382
+ "-segment_times", split_points_argument,
383
+ "-reset_timestamps", "1",
384
+ "-segment_format_options", "movflags=+faststart",
385
+ str(output_pattern),
386
+ ]
387
+
388
+ completed = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
389
+ if completed.returncode != 0:
390
+ raise RuntimeError(
391
+ f"ffmpeg segment failed: {input_video}\n"
392
+ f"{completed.stderr.decode('utf-8', errors='replace')}"
393
+ )
394
+
395
+ segments: List[VideoSegment] = []
396
+ with segment_list_csv_path.open("r", encoding="utf-8", newline="") as file_handle:
397
+ csv_reader = csv.reader(file_handle)
398
+ for row in csv_reader:
399
+ if not row or len(row) < 3:
400
+ continue
401
+ filename, start_time, end_time = row[0], row[1], row[2]
402
+ segments.append(
403
+ VideoSegment(
404
+ path=output_directory / filename,
405
+ start_seconds=float(start_time),
406
+ end_seconds=float(end_time),
407
+ )
408
+ )
409
+
410
+ return segments
411
+
412
+
413
+ # =========================
414
+ # Node implementation
415
+ # =========================
416
+
417
+ class SplitShotsNode(BaseNode):
418
+ meta = NodeMeta(
419
+ name="split_shots",
420
+ description="将输入素材按照镜头变化切割。",
421
+ node_id="split_shots",
422
+ node_kind="split_shots",
423
+ require_prior_kind=["assets_index"],
424
+ default_require_prior_kind=["assets_index"],
425
+ next_available_node=["understand_clips", "xhs_understand_clips"],
426
+ )
427
+ input_schema = SplitShotsInput
428
+
429
+ def __init__(self, *args) -> None:
430
+ super().__init__(*args)
431
+
432
+ self.transnetv2_model = load_transnetv2_model_cached(
433
+ str(self.server_cfg.split_shots.transnet_weights),
434
+ device=self.server_cfg.split_shots.transnet_device,
435
+ )
436
+ self.ffmpeg_executable = resolve_ffmpeg_executable()
437
+
438
+ # -------------------------
439
+ # Public entrypoints
440
+ # -------------------------
441
+
442
+ async def default_process(self, node_summary, inputs: Dict[str, Any]) -> Any:
443
+ """
444
+ Default behavior: do NOT split shots, just pass-through paths.
445
+ (Optimization) No re-save / no re-encode.
446
+ """
447
+ output_directory = self._prepare_output_directory(inputs)
448
+ assets = self._extract_assets(inputs)
449
+
450
+ clips: List[Dict[str, Any]] = []
451
+ clip_index = 1
452
+
453
+ for asset in assets:
454
+ clip = self._build_clip_without_splitting(asset=asset, clip_index=clip_index, node_summary=node_summary)
455
+ clips.append(clip)
456
+ clip_index += 1
457
+
458
+ node_summary.info_for_user("镜头切分已跳过")
459
+ return {"clips": clips}
460
+
461
+ async def process(self, node_summary, inputs: Dict[str, Any]) -> Any:
462
+ """
463
+ Split shots using TransNetV2 + ffmpeg segment copy.
464
+ """
465
+ output_directory = self._prepare_output_directory(inputs)
466
+ assets = self._extract_assets(inputs)
467
+
468
+ clips: List[Dict[str, Any]] = []
469
+ clip_index = 1
470
+
471
+ min_shot_duration_milliseconds = inputs.get("min_shot_duration", DEFAULT_MIN_SHOT_DURATION_MILLISECONDS)
472
+ max_shot_duration_milliseconds = inputs.get("max_shot_duration", DEFAULT_MAX_SHOT_DURATION_MILLISECONDS)
473
+
474
+ if min_shot_duration_milliseconds > max_shot_duration_milliseconds:
475
+ min_shot_duration_milliseconds, max_shot_duration_milliseconds = DEFAULT_MIN_SHOT_DURATION_MILLISECONDS, DEFAULT_MAX_SHOT_DURATION_MILLISECONDS
476
+ node_summary.add_warning(
477
+ f"min_shot_duration_milliseconds ({min_shot_duration_milliseconds}) cannot greater than max_shot_duration_milliseconds ({max_shot_duration_milliseconds})"
478
+ f"using default config {DEFAULT_MIN_SHOT_DURATION_MILLISECONDS} and ({DEFAULT_MAX_SHOT_DURATION_MILLISECONDS})",
479
+ artifact_id = inputs.get("artifact_id"),
480
+ )
481
+
482
+ if min_shot_duration_milliseconds < DEFAULT_MIN_SHOT_DURATION_MILLISECONDS:
483
+ min_shot_duration_milliseconds = DEFAULT_MIN_SHOT_DURATION_MILLISECONDS
484
+ node_summary.add_warning(
485
+ f"min_shot_duration_milliseconds ({min_shot_duration_milliseconds}) too small"
486
+ f"using default config {DEFAULT_MIN_SHOT_DURATION_MILLISECONDS}",
487
+ artifact_id = inputs.get("artifact_id"),
488
+ )
489
+
490
+ if max_shot_duration_milliseconds > DEFAULT_MAX_SHOT_DURATION_MILLISECONDS:
491
+ max_shot_duration_milliseconds = DEFAULT_MAX_SHOT_DURATION_MILLISECONDS
492
+ node_summary.add_warning(
493
+ f"max_shot_duration_milliseconds ({max_shot_duration_milliseconds}) too great"
494
+ f"using default config {DEFAULT_MAX_SHOT_DURATION_MILLISECONDS}",
495
+ artifact_id = inputs.get("artifact_id"),
496
+ )
497
+
498
+ for asset in assets:
499
+ new_clips, clip_index = self._process_single_asset(
500
+ asset=asset,
501
+ output_directory=output_directory,
502
+ starting_clip_index=clip_index,
503
+ node_summary=node_summary,
504
+ min_shot_duration_milliseconds=min_shot_duration_milliseconds,
505
+ max_shot_duration_milliseconds=max_shot_duration_milliseconds,
506
+ )
507
+ clips.extend(new_clips)
508
+
509
+ node_summary.info_for_user(
510
+ f"{self.meta.node_id} 执行成功,输出 clips 数量:{len(clips)}"
511
+ )
512
+ return {"clips": clips}
513
+
514
+ # -------------------------
515
+ # Internal helpers
516
+ # -------------------------
517
+
518
+ def _prepare_output_directory(self, inputs: Dict[str, Any]) -> Path:
519
+ artifact_id = inputs["artifact_id"]
520
+ session_id = inputs["session_id"]
521
+ output_directory = self.server_cache_dir / session_id / artifact_id
522
+ output_directory.mkdir(parents=True, exist_ok=True)
523
+ return output_directory
524
+
525
+ def _extract_assets(self, inputs: Dict[str, Any]) -> List[Dict[str, Any]]:
526
+ return (inputs.get("assets_index") or {}).get("assets", []) or []
527
+
528
+ def _format_clip_id(self, clip_index: int) -> str:
529
+ return f"clip_{clip_index:0{CLIP_ID_NUMBER_WIDTH}d}"
530
+
531
+ def _require_asset_id(self, asset: Dict[str, Any]) -> str:
532
+ asset_id = asset.get("asset_id")
533
+ if not asset_id:
534
+ raise ValueError(f"asset missing required field 'asset_id': {asset}")
535
+ return str(asset_id)
536
+
537
+ def _require_media_type(self, asset: Dict[str, Any]) -> str:
538
+ media_type = asset.get("media_type")
539
+ if not media_type:
540
+ raise ValueError(f"asset missing required field 'media_type': {asset}")
541
+ return str(media_type)
542
+
543
+ def _require_video_metadata(self, asset_id: str, asset: Dict[str, Any]) -> Dict[str, Any]:
544
+ metadata = asset.get("metadata") or {}
545
+ if "duration" not in metadata:
546
+ raise ValueError(f"video asset_id={asset_id} missing metadata.duration")
547
+ return metadata
548
+
549
+ def _parse_duration_milliseconds(self, asset_id: str, metadata: Dict[str, Any]) -> int:
550
+ try:
551
+ duration_milliseconds = int(metadata["duration"])
552
+ except (TypeError, ValueError):
553
+ raise ValueError(f"video asset_id={asset_id} has invalid metadata.duration: {metadata.get('duration')!r}")
554
+
555
+ if duration_milliseconds < 0:
556
+ raise ValueError(f"video asset_id={asset_id} has negative duration: {duration_milliseconds}")
557
+ return duration_milliseconds
558
+
559
+ def _require_path(self, asset_id: str, asset: Dict[str, Any], *, field_name: str) -> str:
560
+ path_value = asset.get(field_name)
561
+ if not path_value:
562
+ raise ValueError(f"asset_id={asset_id} missing required field {field_name!r}")
563
+ return str(path_value)
564
+
565
+ def _build_clip_without_splitting(self, asset: Dict[str, Any], clip_index: int, node_summary) -> Dict[str, Any]:
566
+ """
567
+ Build a single clip without cutting:
568
+ - image: use orig_path (or fallback to path)
569
+ - video: use path (no re-save)
570
+ """
571
+ asset_id = self._require_asset_id(asset)
572
+ media_type = self._require_media_type(asset)
573
+ clip_id = self._format_clip_id(clip_index)
574
+
575
+ if media_type == "image":
576
+ image_path = asset.get("orig_path") or asset.get("path")
577
+ if not image_path:
578
+ raise ValueError(f"image asset_id={asset_id} missing 'orig_path'/'path'")
579
+ node_summary.info_for_user(f"{clip_id} 分割完成", preview_urls=[image_path])
580
+ return {
581
+ "clip_id": clip_id,
582
+ "kind": "image",
583
+ "path": image_path,
584
+ "source_ref": {"asset_id": asset_id},
585
+ }
586
+
587
+ if media_type != "video":
588
+ raise ValueError(f"unsupported media_type {media_type!r} for asset_id={asset_id}")
589
+
590
+ metadata = self._require_video_metadata(asset_id, asset)
591
+ duration_milliseconds = self._parse_duration_milliseconds(asset_id, metadata)
592
+ video_path = self._require_path(asset_id, asset, field_name="path")
593
+
594
+ node_summary.info_for_user(f"{clip_id} 分割完成", preview_urls=[video_path])
595
+ return {
596
+ "clip_id": clip_id,
597
+ "kind": "video",
598
+ "path": video_path,
599
+ "fps": metadata.get("fps"),
600
+ "source_ref": {
601
+ "asset_id": asset_id,
602
+ "start": 0,
603
+ "end": duration_milliseconds,
604
+ "duration": duration_milliseconds,
605
+ "height": metadata.get("height"),
606
+ "width": metadata.get("width"),
607
+ },
608
+ }
609
+
610
+ def _process_single_asset(
611
+ self,
612
+ *,
613
+ asset: Dict[str, Any],
614
+ output_directory: Path,
615
+ starting_clip_index: int,
616
+ node_summary,
617
+ min_shot_duration_milliseconds: int,
618
+ max_shot_duration_milliseconds: int,
619
+ ) -> Tuple[List[Dict[str, Any]], int]:
620
+ """
621
+ Return: (clips_generated, next_clip_index)
622
+ """
623
+ asset_id = self._require_asset_id(asset)
624
+ media_type = self._require_media_type(asset)
625
+
626
+ if media_type == "image":
627
+ clip_id = self._format_clip_id(starting_clip_index)
628
+ image_path = asset.get("orig_path") or asset.get("path")
629
+ if not image_path:
630
+ raise ValueError(f"image asset_id={asset_id} missing 'orig_path'/'path'")
631
+
632
+ node_summary.info_for_user(f"{clip_id} 分割完成", preview_urls=[image_path])
633
+ clip = {
634
+ "clip_id": clip_id,
635
+ "kind": "image",
636
+ "path": image_path,
637
+ "source_ref": {"asset_id": asset_id},
638
+ }
639
+ return [clip], starting_clip_index + 1
640
+
641
+ if media_type != "video":
642
+ raise ValueError(f"unsupported media_type {media_type!r} for asset_id={asset_id}")
643
+
644
+ video_clips, next_index = self._process_video_asset(
645
+ asset_id=asset_id,
646
+ asset=asset,
647
+ output_directory=output_directory,
648
+ starting_clip_index=starting_clip_index,
649
+ node_summary=node_summary,
650
+ min_shot_duration_milliseconds=min_shot_duration_milliseconds,
651
+ max_shot_duration_milliseconds=max_shot_duration_milliseconds,
652
+ )
653
+ return video_clips, next_index
654
+
655
+ def _process_video_asset(
656
+ self,
657
+ *,
658
+ asset_id: str,
659
+ asset: Dict[str, Any],
660
+ output_directory: Path,
661
+ starting_clip_index: int,
662
+ node_summary,
663
+ min_shot_duration_milliseconds: int,
664
+ max_shot_duration_milliseconds: int,
665
+ ) -> Tuple[List[Dict[str, Any]], int]:
666
+ metadata = self._require_video_metadata(asset_id, asset)
667
+ duration_milliseconds = self._parse_duration_milliseconds(asset_id, metadata)
668
+
669
+ input_video_path = Path(self._require_path(asset_id, asset, field_name="path")).expanduser()
670
+
671
+ # 如果素材本身就短于 min_shot_duration:完全跳过切分与拼接
672
+ if min_shot_duration_milliseconds is not None and duration_milliseconds < min_shot_duration_milliseconds:
673
+ clip_id = self._format_clip_id(starting_clip_index)
674
+ node_summary.info_for_user(f"{clip_id} 分割完成", preview_urls=[str(input_video_path)])
675
+ clip = {
676
+ "clip_id": clip_id,
677
+ "kind": "video",
678
+ "path": str(input_video_path),
679
+ "fps": metadata.get("fps"),
680
+ "source_ref": {
681
+ "asset_id": asset_id,
682
+ "start": 0,
683
+ "end": duration_milliseconds,
684
+ "duration": duration_milliseconds,
685
+ "height": metadata.get("height"),
686
+ "width": metadata.get("width"),
687
+ },
688
+ }
689
+ return [clip], starting_clip_index + 1
690
+
691
+ # 1) Detect scenes
692
+ scenes = detect_scenes_with_transnetv2_without_proxy(
693
+ self.transnetv2_model,
694
+ input_video_path,
695
+ self.ffmpeg_executable,
696
+ frames_per_second=DEFAULT_SCENE_DETECTION_FRAMES_PER_SECOND,
697
+ threshold=DEFAULT_SCENE_DETECTION_THRESHOLD,
698
+ )
699
+ split_points_seconds = convert_scenes_to_split_points_seconds(scenes)
700
+
701
+ split_points_seconds = enforce_shot_duration_constraints_on_split_points_seconds(
702
+ split_points_seconds,
703
+ total_duration_milliseconds=duration_milliseconds,
704
+ min_shot_duration_milliseconds=min_shot_duration_milliseconds,
705
+ max_shot_duration_milliseconds=max_shot_duration_milliseconds,
706
+ )
707
+
708
+ # 2) If no split needed, optionally skip copying
709
+ if not split_points_seconds and not COPY_VIDEO_WHEN_NO_SPLIT:
710
+ clip_id = self._format_clip_id(starting_clip_index)
711
+ node_summary.info_for_user(f"{clip_id} 分割完成", preview_urls=[str(input_video_path)])
712
+ clip = {
713
+ "clip_id": clip_id,
714
+ "kind": "video",
715
+ "path": str(input_video_path),
716
+ "fps": metadata.get("fps"),
717
+ "source_ref": {
718
+ "asset_id": asset_id,
719
+ "start": 0,
720
+ "end": duration_milliseconds,
721
+ "duration": duration_milliseconds,
722
+ "height": metadata.get("height"),
723
+ "width": metadata.get("width"),
724
+ },
725
+ }
726
+ return [clip], starting_clip_index + 1
727
+
728
+ # 3) Segment by ffmpeg (-c copy)
729
+ filename_prefix = "clip"
730
+ segments = segment_video_stream_copy_with_ffmpeg(
731
+ input_video=input_video_path,
732
+ ffmpeg_executable=self.ffmpeg_executable,
733
+ split_points_seconds=split_points_seconds,
734
+ output_directory=output_directory,
735
+ filename_prefix=filename_prefix,
736
+ start_index=starting_clip_index,
737
+ )
738
+
739
+ # 4) Build clip list
740
+ clips: List[Dict[str, Any]] = []
741
+ clip_index = starting_clip_index
742
+
743
+ for segment in segments:
744
+ clip_id = self._format_clip_id(clip_index)
745
+
746
+ if segment.end_seconds < 0:
747
+ start_milliseconds = 0
748
+ end_milliseconds = duration_milliseconds
749
+ else:
750
+ start_milliseconds = max(0, int(round(segment.start_seconds * MILLISECONDS_PER_SECOND)))
751
+ end_milliseconds = max(start_milliseconds, int(round(segment.end_seconds * MILLISECONDS_PER_SECOND)))
752
+
753
+ segment_duration_milliseconds = max(0, end_milliseconds - start_milliseconds)
754
+ if segment_duration_milliseconds <= 0:
755
+ continue
756
+
757
+ output_path_string = str(segment.path)
758
+ node_summary.info_for_user(f"{clip_id} 分割完成", preview_urls=[output_path_string])
759
+
760
+ clips.append(
761
+ {
762
+ "clip_id": clip_id,
763
+ "kind": "video",
764
+ "path": output_path_string,
765
+ "fps": metadata.get("fps"),
766
+ "source_ref": {
767
+ "asset_id": asset_id,
768
+ "start": start_milliseconds,
769
+ "end": end_milliseconds,
770
+ "duration": segment_duration_milliseconds,
771
+ "height": metadata.get("height"),
772
+ "width": metadata.get("width"),
773
+ },
774
+ }
775
+ )
776
+ clip_index += 1
777
+
778
+ return clips, clip_index
src/open_storyline/nodes/core_nodes/understand_clips.py ADDED
@@ -0,0 +1,189 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Any, Dict
2
+ import asyncio
3
+
4
+ from open_storyline.nodes.core_nodes.base_node import BaseNode, NodeMeta
5
+ from open_storyline.mcp.sampling_requester import LLMClient
6
+ from src.open_storyline.utils.prompts import get_prompt
7
+ from open_storyline.utils.parse_json import parse_json_dict
8
+ from open_storyline.nodes.node_schema import UnderstandClipsInput
9
+
10
+ class UnderstandClipsNode(BaseNode):
11
+ """
12
+ 素材理解 节点
13
+ """
14
+
15
+ meta = NodeMeta(
16
+ name="understand_clips",
17
+ description="分析当前工程的所有clips,进行素材理解,生成对每个素材的一段描述。依赖split_shots节点的结果",
18
+ node_id="understand_clips",
19
+ node_kind="understand_clips",
20
+ require_prior_kind=['assets_index', 'split_shots'],
21
+ default_require_prior_kind=['assets_index', 'split_shots'],
22
+ next_available_node=['filter_clips', 'xhs_filter_clips'],
23
+ )
24
+
25
+ input_schema = UnderstandClipsInput
26
+
27
+ async def default_process(
28
+ self,
29
+ node_summary,
30
+ inputs: Dict[str, Any],
31
+ ) -> Any:
32
+ clips = inputs["split_shots"]["clips"]
33
+
34
+ clip_captions: list[dict[str, Any]] = []
35
+ for clip in clips or []:
36
+ clip_captions.append(
37
+ {
38
+ "clip_id": clip.get("clip_id"),
39
+ "caption": "no caption",
40
+ "source_ref": {
41
+ "media_id": clip.get("source_ref", {}).get("asset_id", ""),
42
+ }
43
+ }
44
+ )
45
+ node_summary.info_for_user(f"未对{len(clips)}个片段生成描述")
46
+ return {
47
+ "clip_captions": clip_captions,
48
+ "overall": "unknown",
49
+ }
50
+
51
+ async def process(self, node_summary, inputs: Dict[str, Any]) -> Any:
52
+ """
53
+ inputs: BaseNode.load_inputs(ctx)读到的前面节点的结果
54
+ """
55
+ assets_index = inputs["assets"]
56
+ clips = inputs["split_shots"]["clips"]
57
+ llm: LLMClient = inputs["llm"]
58
+ system_prompt = get_prompt("understand_clips.system_detail")
59
+ user_prompt = get_prompt("understand_clips.user_detail")
60
+
61
+
62
+ clip_captions: list[dict[str, Any]] = []
63
+
64
+ for clip in clips or []:
65
+ clip_id = str(clip.get("clip_id", "") or "").strip() or "(unknown_clip)"
66
+ kind = str(clip.get("kind", "") or "").strip().lower()
67
+ src = clip.get("source_ref") or {}
68
+
69
+ asset_id = str(src.get("asset_id", "") or "")
70
+ asset = assets_index.get(asset_id)
71
+
72
+ out_item: dict[str, Any] = {
73
+ "clip_id": clip_id,
74
+ }
75
+
76
+ if not asset:
77
+ out_item["caption"] = f"错误:找不到 asset_id={asset_id} 对应的素材"
78
+ clip_captions.append(out_item)
79
+ continue
80
+
81
+ path = str(asset.get("path", "") or "").strip()
82
+ if not path:
83
+ out_item["caption"] = f"错误: asset_id={asset_id} 的path为空"
84
+ clip_captions.append(out_item)
85
+ continue
86
+
87
+ # 组装 media
88
+ media: list[Any] = []
89
+
90
+ if kind == "image":
91
+ media = [{"path": path}]
92
+
93
+ elif kind == "video":
94
+ in_sec = _safe_float(src.get("start", 0) / 1000.0, 0.0)
95
+
96
+ if src.get("end") is not None:
97
+ out_sec = _safe_float(src.get("end", 0) / 1000.0, in_sec)
98
+ else:
99
+ dur = _safe_float(src.get("duration", 0.0), 0.0)
100
+ out_sec = in_sec + max(0.0, dur)
101
+
102
+ if out_sec <= in_sec:
103
+ out_sec = in_sec + 0.1
104
+
105
+ media = [{
106
+ "path": path,
107
+ "in_sec": in_sec,
108
+ "out_sec": out_sec,
109
+ }]
110
+ else:
111
+ out_item["caption"] = f"错误:不支持的 clip kind:{kind}"
112
+ clip_captions.append(out_item)
113
+ continue
114
+
115
+ raw = await llm.complete(
116
+ system_prompt=system_prompt,
117
+ user_prompt=user_prompt,
118
+ media=media,
119
+ temperature=0.3,
120
+ top_p=0.9,
121
+ max_tokens=2048,
122
+ model_preferences=None,
123
+ )
124
+ if raw is None:
125
+ out_item["caption"] = ("错误:VLM调用失败")
126
+ clip_captions.append(out_item)
127
+ continue # 继续处理下一个 clip
128
+
129
+ try:
130
+ obj = parse_json_dict(raw)
131
+ except:
132
+ text = (raw or "").strip()
133
+ out_item["caption"] = text if text else "错误:无法解析模型输出"
134
+ clip_captions.append(out_item)
135
+ continue
136
+
137
+ out_item["caption"] = str(obj.get("caption", "") or "").strip()
138
+ out_item["source_ref"] = {
139
+ "media_id": clip.get("source_ref", {}).get("asset_id", ""),
140
+ }
141
+ clip_captions.append(out_item)
142
+
143
+ desc_lines: list[str] = []
144
+ for desc in clip_captions:
145
+ text = str(desc.get("caption"))
146
+ desc_lines.append(f"- {desc.get('clip_instance_id')}: {text}")
147
+
148
+ overall_summary = ""
149
+ if desc_lines:
150
+ overall_system_prompt = get_prompt("understand_clips.system_overall")
151
+ overall_user_prompt = get_prompt("understand_clips.user_overall", clips_captions=desc_lines)
152
+
153
+ try:
154
+ overall_summary = await llm.complete(
155
+ system_prompt=overall_system_prompt,
156
+ user_prompt=overall_user_prompt,
157
+ media=None,
158
+ temperature=0.3,
159
+ top_p=0.9,
160
+ max_tokens=1024,
161
+ model_preferences=None
162
+ )
163
+
164
+ except Exception as e:
165
+ overall_summary = f"Error: 总结生成失败:{type(e).__name__}: {e}"
166
+ node_summary.info_for_user(f"素材理解完成,共分析 {len(clip_captions)} 个片段,整体主要描述了{overall_summary}")
167
+ return {
168
+ "clip_captions": clip_captions,
169
+ "overall": overall_summary
170
+ }
171
+
172
+ def _parse_input(self, node_summary, inputs: Dict[str, Any]):
173
+ assets = inputs["assets_index"]["assets"]
174
+
175
+ assets_index: dict[str, dict[str, Any]] = {}
176
+ for asset in assets or []:
177
+ asset_id = asset.get("asset_id")
178
+ if asset_id:
179
+ assets_index[str(asset_id)] = asset
180
+ inputs.update({"assets": assets_index})
181
+ return inputs
182
+
183
+ def _safe_float(x: Any, default: float = 0.0) -> float:
184
+ try:
185
+ if x is None:
186
+ return default
187
+ return float(x)
188
+ except Exception:
189
+ return default
src/open_storyline/nodes/node_manager.py ADDED
@@ -0,0 +1,172 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+ from collections import defaultdict
3
+ from typing import Any, Dict, List, Optional, Set
4
+
5
+
6
+ from langchain_core.tools.structured import StructuredTool
7
+
8
+ from src.open_storyline.storage.agent_memory import ArtifactStore
9
+
10
+
11
+ class NodeManager:
12
+ def __init__(self, tools: List[StructuredTool] = None):
13
+ # 核心数据结构
14
+ self.kind_to_node_ids: Dict[str, List[str]] = defaultdict(list) # node_kind -> node_id列表(已排序)
15
+ self.id_to_tool: Dict[str, StructuredTool] = {} # node_id -> tool
16
+ self.id_to_next: Dict[str, List[str]] = {} # node_id -> 下一个可执行的node_id列表
17
+ self.id_to_priority: Dict[str, int] = {} # node_id -> priority
18
+ self.id_to_kind: Dict[str, str] = {} # node_id -> node_kind (方便删除时查找)
19
+
20
+ # 新增:前置依赖相关
21
+ self.id_to_require_prior_kind: Dict[str, List[str]] = {} # node_id -> 必须的前置kind列表
22
+ self.id_to_default_require_prior_kind: Dict[str, List[str]] = {} # node_id -> 默认的前置kind列表
23
+
24
+ # 反向索引:哪些节点依赖某个kind
25
+ self.kind_to_dependent_nodes: Dict[str, Set[str]] = defaultdict(set) # kind -> 依赖该kind的node_id集合
26
+ self.kind_to_default_dependent_nodes: Dict[str, Set[str]] = defaultdict(set) # kind -> 默认依赖该kind的node_id集合
27
+
28
+ if tools:
29
+ self._build(tools)
30
+
31
+ def _build(self, tools: List[StructuredTool]):
32
+ """批量构建数据结构"""
33
+ for tool in tools:
34
+ if tool.metadata:
35
+ metadata = tool.metadata.get('_meta', {})
36
+ node_id = metadata.get('node_id')
37
+ if node_id:
38
+ self.add_node(tool)
39
+
40
+ def add_node(self, tool: StructuredTool) -> bool:
41
+ """添加节点"""
42
+ # metadata is None, failed to add node
43
+ if not tool.metadata:
44
+ return False
45
+
46
+ metadata = tool.metadata.get('_meta', {})
47
+ node_id = metadata.get('node_id')
48
+
49
+ if not node_id:
50
+ return False
51
+
52
+ # 如果已存在,先删除旧的
53
+ if node_id in self.id_to_tool:
54
+ self.remove_node(node_id)
55
+
56
+ node_kind = metadata.get('node_kind', node_id)
57
+ priority = metadata.get('priority', 0)
58
+ next_nodes = metadata.get('next_available_node', [])
59
+ require_prior_kind = metadata.get('require_prior_kind', [])
60
+ default_require_prior_kind = metadata.get('default_require_prior_kind', [])
61
+
62
+ # 更新所有数据结构
63
+ self.id_to_tool[node_id] = tool
64
+ self.id_to_priority[node_id] = priority
65
+ self.id_to_next[node_id] = next_nodes
66
+ self.id_to_kind[node_id] = node_kind
67
+ self.id_to_require_prior_kind[node_id] = require_prior_kind
68
+ self.id_to_default_require_prior_kind[node_id] = default_require_prior_kind
69
+
70
+ # 添加到kind分组并重新排序
71
+ self.kind_to_node_ids[node_kind].append(node_id)
72
+ self._sort_kind(node_kind)
73
+
74
+ # 更新反向索引
75
+ for kind in require_prior_kind:
76
+ self.kind_to_dependent_nodes[kind].add(node_id)
77
+
78
+ for kind in default_require_prior_kind:
79
+ self.kind_to_default_dependent_nodes[kind].add(node_id)
80
+
81
+ return True
82
+
83
+ def remove_node(self, node_id: str, clean_references: bool = True) -> bool:
84
+ """
85
+ 删除节点
86
+
87
+ Args:
88
+ node_id: 要删除的节点ID
89
+ clean_references: 是否清理其他节点对该节点的引用
90
+ """
91
+ if node_id not in self.id_to_tool:
92
+ return False
93
+
94
+ node_kind = self.id_to_kind[node_id]
95
+
96
+ # 清理反向索引
97
+ if node_id in self.id_to_require_prior_kind:
98
+ for kind in self.id_to_require_prior_kind[node_id]:
99
+ self.kind_to_dependent_nodes[kind].discard(node_id)
100
+ if not self.kind_to_dependent_nodes[kind]:
101
+ del self.kind_to_dependent_nodes[kind]
102
+
103
+ if node_id in self.id_to_default_require_prior_kind:
104
+ for kind in self.id_to_default_require_prior_kind[node_id]:
105
+ self.kind_to_default_dependent_nodes[kind].discard(node_id)
106
+ if not self.kind_to_default_dependent_nodes[kind]:
107
+ del self.kind_to_default_dependent_nodes[kind]
108
+
109
+ # 从所有数据结构中删除
110
+ del self.id_to_tool[node_id]
111
+ del self.id_to_priority[node_id]
112
+ del self.id_to_next[node_id]
113
+ del self.id_to_kind[node_id]
114
+
115
+ if node_id in self.id_to_require_prior_kind:
116
+ del self.id_to_require_prior_kind[node_id]
117
+ if node_id in self.id_to_default_require_prior_kind:
118
+ del self.id_to_default_require_prior_kind[node_id]
119
+
120
+ # 从kind分组中删除
121
+ if node_id in self.kind_to_node_ids[node_kind]:
122
+ self.kind_to_node_ids[node_kind].remove(node_id)
123
+
124
+ # 如果该kind没有节点了,删除这个kind
125
+ if not self.kind_to_node_ids[node_kind]:
126
+ del self.kind_to_node_ids[node_kind]
127
+
128
+ # 清理其他节点对该节点的引用
129
+ if clean_references:
130
+ for nid in list(self.id_to_next.keys()):
131
+ if node_id in self.id_to_next[nid]:
132
+ self.id_to_next[nid].remove(node_id)
133
+
134
+ return True
135
+
136
+
137
+ def _sort_kind(self, kind: str):
138
+ """对指定kind的节点列表按优先级排序"""
139
+ if kind in self.kind_to_node_ids:
140
+ self.kind_to_node_ids[kind].sort(
141
+ key=lambda nid: self.id_to_priority[nid],
142
+ reverse=True
143
+ )
144
+
145
+ def get_tool(self, node_id: str) -> Optional[StructuredTool]:
146
+ """根据node_id获取tool"""
147
+ return self.id_to_tool.get(node_id)
148
+
149
+ def check_excutable(self, session_id:str, store: ArtifactStore, all_require_kind: List[str]) -> Dict[str, Any]:
150
+ """
151
+ 检查是否可执行,并且返回未执行的功能
152
+ """
153
+ collected_output = {}
154
+ for req_kind in all_require_kind:
155
+ req_ids_queue = self.kind_to_node_ids[req_kind]
156
+ # 1. 收集所有节点的最新输出
157
+ valid_outputs = []
158
+ for node_id in req_ids_queue:
159
+ output = store.get_latest_meta(node_id=node_id, session_id=session_id)
160
+ if output is not None:
161
+ valid_outputs.append(output)
162
+
163
+ # 2. 找出创建时间最晚的输出
164
+ if valid_outputs:
165
+ latest_output = max(valid_outputs, key=lambda output: output.created_at)
166
+ collected_output[req_kind] = latest_output
167
+ return {
168
+ "excutable": len(collected_output.keys())==len(all_require_kind),
169
+ "collected_node": collected_output,
170
+ "missing_kind": list(set(all_require_kind) - set(collected_output.keys()))
171
+ }
172
+
src/open_storyline/nodes/node_schema.py ADDED
@@ -0,0 +1,283 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Dict, List, Literal, Any, Annotated, Optional, Union, ClassVar, Type
2
+ from pydantic import BaseModel, Field, model_validator, constr, conlist
3
+
4
+
5
+ class VideoMetadata(BaseModel):
6
+ """视频元数据"""
7
+ width: int = Field(description="宽度")
8
+ height: int = Field(description="高度")
9
+ duration: float = Field(description="时长(毫秒)")
10
+ fps: float = Field(description="帧率")
11
+ has_audio: bool = Field(default=False, description="是否包含音频轨道")
12
+
13
+ audio_sample_rate_hz: Optional[int] = Field(
14
+ None,
15
+ gt=0,
16
+ description="音频采样率(Hz),常见值:44100, 48000"
17
+ )
18
+
19
+ @model_validator(mode='after')
20
+ def validate_audio_sample_rate(self):
21
+ """如果有音频,采样率必填"""
22
+ if self.has_audio and self.audio_sample_rate_hz is None:
23
+ raise ValueError('视频包含音频时,必须提供 audio_sample_rate_hz')
24
+ return self
25
+
26
+
27
+ class ImageMetadata(BaseModel):
28
+ """图片元数据"""
29
+ width: int = Field(description="宽度")
30
+ height: int = Field(description="高度")
31
+
32
+
33
+ class Asset(BaseModel):
34
+ """单个素材"""
35
+ asset_id: str
36
+ path: str
37
+ media_type: Literal["video", "image", "audio", "unknown"]
38
+ metadata: Union[VideoMetadata, ImageMetadata]
39
+ extra_info: Optional[Dict[str, Any]] = None
40
+
41
+
42
+ class SourceRef(BaseModel):
43
+ """ 原素材参考信息 """
44
+ asset_id: str
45
+ start: float
46
+ end: float
47
+ duration: float
48
+ height: Optional[int] = None
49
+ width: Optional[int] = None
50
+
51
+
52
+ class Clip(BaseModel):
53
+ clip_id: str
54
+ language: Optional[str] = None
55
+ caption: str = Field(default="", description="素材的描述文字")
56
+ media_type: str
57
+ path: str
58
+ fps: Optional[float] = None
59
+ extra_info: Optional[Dict[str, Any]] = Field(default=None, description="额外信息")
60
+
61
+
62
+ class SubtitleUnit(BaseModel):
63
+ """字幕切分单元"""
64
+ unit_id: str = Field(
65
+ ...,
66
+ description="字幕单元唯一标识符",
67
+ example="subtitle_0001"
68
+ )
69
+ index_in_group: int = Field(
70
+ ...,
71
+ ge=0,
72
+ description="在当前分组中的顺序索引(从0开始)",
73
+ example=0
74
+ )
75
+ text: str = Field(
76
+ ...,
77
+ description="该字幕单元的文本内容",
78
+ example="猫猫不懂什么是KPI"
79
+ )
80
+
81
+
82
+ class GroupClips(BaseModel):
83
+ """视频分组 - 画面素材组织"""
84
+ group_id: str = Field(
85
+ ...,
86
+ description="分组唯一标识符",
87
+ example="group_0001"
88
+ )
89
+ summary: str = Field(
90
+ ...,
91
+ description="分组的画面风格、情绪基调或剪辑意图描述",
92
+ example="先用最安静、最治愈的镜头建立情绪。"
93
+ )
94
+ clip_ids: List[str] = Field(
95
+ ...,
96
+ description="该分组使用的视频片段ID列表,按播放顺序排列",
97
+ example=["clip_0003", "clip_0002"]
98
+ )
99
+
100
+
101
+ class GroupScript(BaseModel):
102
+ """分组文案脚本"""
103
+ group_id: str = Field(
104
+ ...,
105
+ description="分组唯一标识符",
106
+ example="group_0001"
107
+ )
108
+ raw_text:str = Field(
109
+ ...,
110
+ description="该分组的完整原始文案内容",
111
+ example="猫猫不懂什么是KPI,猫猫只知道今天阳光很好"
112
+ )
113
+ subtitle_units: List = Field(
114
+ ...,
115
+ description="字幕切分单元列表,用于精确控制字幕显示节奏"
116
+ )
117
+
118
+
119
+ class Voiceover(BaseModel):
120
+ """单个旁白/配音项"""
121
+ group_id: str = Field(..., description="组ID,如group_0001")
122
+ voiceover_id: str = Field(..., description="配音ID,如voiceover_0001")
123
+ path: str = Field(..., description="配音文件路径")
124
+ duration: int = Field(..., description="配音时长(毫秒)", gt=0)
125
+
126
+
127
+ class BGM(BaseModel):
128
+ """背景音乐"""
129
+ bgm_id: str = Field(..., description="BGM ID,如bgm_0003")
130
+ path: str = Field(..., description="BGM 文件路径")
131
+ duration: int = Field(..., description="BGM 时长(毫秒)", gt=0)
132
+ bpm: float = Field(..., description="每分钟节拍数", gt=0)
133
+ beats: List[int] = Field(default_factory=list, description="节拍时间点列表(毫秒)")
134
+
135
+
136
+ # TODO: Timeline schema完善
137
+ class TimeWindow(BaseModel):
138
+ start: int = Field(..., description="开始时间(毫秒)")
139
+ end: int = Field(..., description="结束时间(毫秒)")
140
+
141
+
142
+ class AudioMix(BaseModel):
143
+ gain_db: float = Field(default=0.0, description="增益分贝")
144
+ ducking: Optional[Any] = Field(default=None, description="闪避效果配置")
145
+
146
+
147
+ class ClipTrack(BaseModel):
148
+ clip_id: str
149
+ source_window: TimeWindow
150
+ timeline_window: TimeWindow
151
+
152
+
153
+ class BgmTrack(BaseModel):
154
+ bgm_asset_id: str
155
+ timeline_window: TimeWindow
156
+ mix: AudioMix
157
+
158
+
159
+ class SubtitleTrack(BaseModel):
160
+ # 示例结构
161
+ text: str
162
+ timeline_window: TimeWindow
163
+
164
+
165
+ class VoiceoverTrack(BaseModel):
166
+ asset_id: str
167
+ timeline_window: TimeWindow
168
+
169
+
170
+ class TimelineTracks(BaseModel):
171
+ video: List[ClipTrack] = Field(default_factory=list)
172
+ subtitles: List[SubtitleTrack] = Field(default_factory=list)
173
+ voiceover: List[VoiceoverTrack] = Field(default_factory=list)
174
+ bgm: List[BgmTrack] = Field(default_factory=list)
175
+
176
+
177
+ class BaseInput(BaseModel):
178
+ mode: Literal["auto", "skip", "default"] = Field(
179
+ default="auto",
180
+ description="auto: 自动模式;skip: 跳过模式;default: 默认模式"
181
+ )
182
+
183
+
184
+ class AssetsIndexInput(BaseInput):
185
+ ...
186
+
187
+
188
+ class AssetsIndexOutput(BaseModel):
189
+ """ 当前节点输出 """
190
+ assets: List[Asset] = Field(
191
+ default_factory=list,
192
+ description="资产列表"
193
+ )
194
+
195
+
196
+ class SplitShotsInput(BaseInput):
197
+ mode: Literal["auto", "skip", "default"] = Field(
198
+ default="auto",
199
+ description="auto: 根据场景变化自动切分镜头,图片视为单个镜头;skip: 不切分镜头;default: 使用默认切分方法"
200
+ )
201
+ min_shot_duration: Annotated[int, Field(default=1000, description="切分的镜头不得短于该时间(单位:毫秒)")]
202
+ max_shot_duration: Annotated[int, Field(default=30000, description="若单个镜头时长超过该时间,则强制切分(单位:毫秒)")]
203
+
204
+ class SplitShotsOutput(BaseModel):
205
+ clip_captions: List[Clip] = Field(default_factory=list, description="片段列表")
206
+ overall: Dict[str, str]
207
+
208
+
209
+ class UnderstandClipsInput(BaseModel):
210
+ mode: Literal["auto", "skip", "default"] = Field(
211
+ default="auto",
212
+ description="auto: 根据素材内容生成对应的描述;skip: 不生成描述;default: 使用默认生成描述方法"
213
+ )
214
+
215
+ class UnderstandClipsOutput(BaseModel):
216
+ clip_captions: List[Clip] = Field(default_factory=list, description="片段列表")
217
+ overall: Dict[str, str]
218
+
219
+ class FilterClipsInput(BaseModel):
220
+ mode: Literal["auto", "skip", "default"] = Field(
221
+ default="auto",
222
+ description="auto: 根据用户的要求筛选片段;skip: 跳过;default: 使用默认筛选方法"
223
+ )
224
+ user_request: Annotated[str, Field(default="", description="用户对片段过滤的诉求,若没有请根据素材和其他剪辑要求拟定一个。")] = ""
225
+
226
+ class FilterClipsOutput(BaseModel):
227
+ clip_captions: List[Clip] = Field(default_factory=list, description="片段列表")
228
+ overall: Dict[str, str]
229
+
230
+
231
+ class GroupClipsInput(BaseModel):
232
+ mode: Literal["auto", "skip", "default"] = Field(
233
+ default="auto",
234
+ description="auto: 根据素材内容的叙事逻辑和用户对素材顺序的要求,组织合理的素材顺序;skip: 跳过;default: 使用默认生成描述方法"
235
+ )
236
+ user_request: Annotated[str, Field(default="", description="用户对素材组织顺序的要求,若没有请根据常规流程组织合理的叙事顺序。")]
237
+
238
+ class GroupClipsOutput(BaseModel):
239
+ groups: List[GroupClips] = Field(default_factory=list, description="片段列表")
240
+
241
+
242
+ class GenerateScriptInput(BaseModel):
243
+ mode: Literal["auto", "skip", "default"] = Field(
244
+ default="auto",
245
+ description="auto: 根据素材内容和用户对文案的要求,生成合适的文案;skip: 跳过,不添加字幕;default: 使用默认文案"
246
+ )
247
+ user_request: Annotated[str, Field(default="", description="用户对文案的要求.")]
248
+
249
+ class GenerateScriptOutput(BaseModel):
250
+ group_scripts: List[GroupScript]
251
+ title: Optional[str]
252
+
253
+
254
+ class GenerateVoiceoverInput(BaseInput):
255
+ mode: Literal["auto", "skip", "default"] = Field(
256
+ default="auto",
257
+ description="auto: 根据素材内容和用户对配音音色的要求,生成合适的配音;skip: 跳过;default: 使用默认配音"
258
+ )
259
+ user_request: Annotated[dict, Field(default={}, description="用户对配音的要求.")]
260
+
261
+ class GenerateVoiceoverOutput(BaseModel):
262
+ voiceover: List[Voiceover] = Field(default_factory=list, description="配音")
263
+
264
+
265
+ class SelectBGMInput(BaseInput):
266
+ mode: Literal["auto", "skip", "default"] = Field(
267
+ default="auto",
268
+ description="auto: 根据素材内容和用户对音乐的要求,选择合适的音乐;skip: 不使用音乐;default: 使用默认音乐"
269
+ )
270
+ user_request: Annotated[dict, Field(default={}, description="用户对背景音乐的要求.")]
271
+
272
+ class SelectBGMOutput(BaseModel):
273
+ bgm: List[BGM] = Field(default_factory=list, description="背景音乐")
274
+
275
+
276
+ class PlanTimelineInput(BaseInput):
277
+ use_beats: Annotated[bool, Field(default=True, description="素材切换是否需要卡BGM鼓点")]
278
+
279
+ class PlanTimelineOutput(BaseModel):
280
+ tracks: List[TimelineTracks] = Field(default_factory=list, description="时间线轨道集合")
281
+
282
+ class RenderVideoInput(BaseInput):
283
+ ...
src/open_storyline/nodes/node_summary.py ADDED
@@ -0,0 +1,236 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass, field
2
+ from typing import Any, Dict, List, Tuple, Optional
3
+ import logging
4
+ from datetime import datetime
5
+
6
+ from open_storyline.utils.logging import get_logger
7
+
8
+
9
+ @dataclass
10
+ class LogEntry:
11
+ """单条日志记录"""
12
+ level: str
13
+ message: str
14
+ timestamp: str
15
+ artifact_id: Optional[str] = None
16
+ extra_data: Dict[str, Any] = field(default_factory=dict)
17
+
18
+ @dataclass
19
+ class NodeSummary:
20
+ """
21
+ Node Execution Status Summary - Reuses existing logger module
22
+
23
+ Features:
24
+ 1. ERROR - Error messages for LLM
25
+ 2. WARNING - Warning messages for LLM
26
+ 3. DEBUG - Debug information for developers
27
+ 4. INFO_LLM - Detailed information for LLM
28
+ 5. INFO_USER - Brief information for users
29
+
30
+ Capabilities:
31
+ - Reuses get_logger() configuration
32
+ - Hierarchical log storage and extraction
33
+ - Colored console output
34
+ - Log compression functionality
35
+ - Supports artifact tracking
36
+ """
37
+ ERROR: str = "ERROR"
38
+ DEBUG: str = "DEBUG"
39
+ WARNING: str = "WARNING"
40
+ INFO_LLM: str = "INFO_LLM"
41
+ INFO_USER: str = "INFO_USER"
42
+ LOGGER_LEVELS: Tuple[str, ...] = (ERROR, DEBUG, WARNING, INFO_LLM, INFO_USER)
43
+
44
+
45
+ # Log storage
46
+ log_error: List[LogEntry] = field(default_factory=list)
47
+ log_warn: List[LogEntry] = field(default_factory=list)
48
+ log_info_llm: List[LogEntry] = field(default_factory=list)
49
+ log_info_user: List[LogEntry] = field(default_factory=list)
50
+ log_debug: List[LogEntry] = field(default_factory=list)
51
+
52
+ # Artifact mapping
53
+ artifact_warnings: Dict[str, List[str]] = field(default_factory=dict)
54
+ artifact_errors: Dict[str, List[str]] = field(default_factory=dict)
55
+
56
+ # Configuration options
57
+ logger_name: Optional[str] = field(default=None)
58
+ auto_console: bool = field(default=True) # Auto output to console
59
+ summary_levels: Optional[List[str]] = field(default=None)
60
+
61
+ # Internal state
62
+ _logger: Optional[logging.Logger] = field(default=None, init=False, repr=False)
63
+
64
+ def __post_init__(self):
65
+ """Initialize logger - reuses get_logger"""
66
+ if self.logger_name is None:
67
+ self.logger_name = "NodeSummary"
68
+ self._logger = get_logger(self.logger_name)
69
+ if self.summary_levels is None:
70
+ self.summary_levels = [self.ERROR, self.WARNING, self.INFO_LLM, self.INFO_USER]
71
+
72
+ def _log_to_console(self, level: int, message: str, artifact_id: Optional[str] = None):
73
+ """Output to console (using configured logger)"""
74
+ if not self.auto_console:
75
+ return
76
+
77
+ prefix = f"[ARTIFACT:{artifact_id}] " if artifact_id else ""
78
+ self._logger.log(level, f"{prefix}{message}")
79
+
80
+ def add_error(self, message: str, artifact_id: Optional[str] = None, **kwargs: Any):
81
+ """Log error messages - for LLM"""
82
+ entry = LogEntry(
83
+ level=self.ERROR,
84
+ message=message,
85
+ timestamp=datetime.now().isoformat(),
86
+ artifact_id=artifact_id,
87
+ extra_data=kwargs
88
+ )
89
+ self.log_error.append(entry)
90
+
91
+ if artifact_id:
92
+ self.artifact_errors.setdefault(artifact_id, []).append(message)
93
+
94
+ self._log_to_console(logging.ERROR, message, artifact_id)
95
+
96
+ def add_warning(self, message: str, artifact_id: Optional[str] = None, **kwargs: Any):
97
+ """Log warning messages - for LLM"""
98
+ entry = LogEntry(
99
+ level="WARNING",
100
+ message=message,
101
+ timestamp=datetime.now().isoformat(),
102
+ artifact_id=artifact_id,
103
+ extra_data=kwargs
104
+ )
105
+ self.log_warn.append(entry)
106
+
107
+ if artifact_id:
108
+ self.artifact_warnings.setdefault(artifact_id, []).append(message)
109
+
110
+ self._log_to_console(logging.WARNING, message, artifact_id)
111
+
112
+ def info_for_llm(self, message: str, artifact_id: Optional[str] = None, **kwargs: Any):
113
+ """Log detailed information - for LLM"""
114
+ entry = LogEntry(
115
+ level="INFO_LLM",
116
+ message=message,
117
+ timestamp=datetime.now().isoformat(),
118
+ artifact_id=artifact_id,
119
+ extra_data=kwargs
120
+ )
121
+ self.log_info_llm.append(entry)
122
+ self._log_to_console(logging.INFO, f"[{self.INFO_LLM}] {message}", artifact_id)
123
+
124
+ def info_for_user(self, message: str, artifact_id: Optional[str] = None, **kwargs: Any):
125
+ """Log general information - for users"""
126
+ entry = LogEntry(
127
+ level="INFO_USER",
128
+ message=message,
129
+ timestamp=datetime.now().isoformat(),
130
+ artifact_id=artifact_id,
131
+ extra_data=kwargs
132
+ )
133
+ self.log_info_user.append(entry)
134
+ self._log_to_console(logging.INFO, f"[{self.INFO_USER}] {message}", artifact_id)
135
+
136
+ def debug_for_dev(self, message: str, artifact_id: Optional[str] = None, **kwargs: Any):
137
+ """Log debug information - for developers"""
138
+ entry = LogEntry(
139
+ level=self.DEBUG,
140
+ message=message,
141
+ timestamp=datetime.now().isoformat(),
142
+ artifact_id=artifact_id,
143
+ extra_data=kwargs
144
+ )
145
+ self.log_debug.append(entry)
146
+ self._log_to_console(logging.DEBUG, f"[{self.DEBUG}] {message}", artifact_id)
147
+
148
+ def get_logs_by_level(
149
+ self,
150
+ level:str ,
151
+ compress_log: bool=False, # 暂时未实现 TODO
152
+ ) -> Dict[str,Any]:
153
+ self.all_logs = {
154
+ self.ERROR: self.log_error,
155
+ self.DEBUG: self.log_debug,
156
+ self.WARNING: self.log_warn,
157
+ self.INFO_LLM: self.log_info_llm,
158
+ self.INFO_USER: self.log_info_user
159
+ }
160
+
161
+ selected_log = self.all_logs[level]
162
+
163
+ return self._extract_log(selected_log)
164
+
165
+ def _extract_log(
166
+ self,
167
+ log_content: List[LogEntry],
168
+ ) -> Dict[str,Any]:
169
+ """
170
+ Extract log content into string format.
171
+
172
+ Args:
173
+ log_content: List of log entries
174
+
175
+ Returns:
176
+ Formatted log string with each log entry on a separate line
177
+ """
178
+ if not log_content:
179
+ return {}
180
+
181
+ log_lines: List[str] = []
182
+ extra_data_list: List[Dict[str,Any]] = []
183
+ for entry in log_content:
184
+ log_line = f"[{entry.timestamp}] {entry.message}"
185
+
186
+ if entry.artifact_id:
187
+ log_line += f" [artifact_id: {entry.artifact_id}]"
188
+
189
+ log_lines.append(log_line)
190
+ extra_data_list.append(entry.extra_data)
191
+ result: Dict[str,Any] = {
192
+ "log_lines": "\n".join(log_lines),
193
+ "extra_data_list": extra_data_list
194
+ }
195
+ return result
196
+
197
+ def _get_preview_urls(
198
+ self,
199
+ extra_data_list: List[Dict[str,Any]],
200
+ ) -> List[str]:
201
+ preview_urls: List[str] = []
202
+ for extra_data in extra_data_list:
203
+ preview_urls.extend([str(url) for url in extra_data.get('preview_urls', [])])
204
+ return preview_urls
205
+
206
+ def get_summary(
207
+ self,
208
+ artifact_id: str,
209
+ compress_log: bool=True,
210
+ **kwargs: Dict[str,Any],
211
+ ) -> Dict[str,Any]:
212
+ summary: Dict[str,Any] = {}
213
+ preview_urls: List[str] = []
214
+ if self.summary_levels is None:
215
+ return summary
216
+
217
+ for level in self.summary_levels:
218
+ summary_log = self.get_logs_by_level(level, compress_log)
219
+ log_lines = summary_log.get('log_lines', "")
220
+ extra_data_list = summary_log.get('extra_data_list', [])
221
+ preview_urls.extend(self._get_preview_urls(extra_data_list))
222
+ summary[level] = log_lines
223
+
224
+ summary['preview_urls'] = preview_urls
225
+ summary['artifact_id'] = artifact_id
226
+ return summary
227
+
228
+ def clear(self):
229
+ """Clear all logs"""
230
+ self.log_error.clear()
231
+ self.log_warn.clear()
232
+ self.log_info_llm.clear()
233
+ self.log_info_user.clear()
234
+ self.log_debug.clear()
235
+ self.artifact_warnings.clear()
236
+ self.artifact_errors.clear()
src/open_storyline/skills/skills_io.py ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ import aiofiles
4
+ from pathlib import Path
5
+ from skillkit import SkillManager
6
+ from skillkit.integrations.langchain import create_langchain_tools
7
+ from langchain.agents import create_agent
8
+ from langchain_openai import ChatOpenAI
9
+ from langchain.messages import HumanMessage
10
+
11
+ async def load_skills(
12
+ skill_dir: str=".storyline/skills"
13
+ ):
14
+ # Discover skills
15
+ manager = SkillManager(skill_dir=skill_dir)
16
+ await manager.adiscover()
17
+
18
+ # Convert to LangChain tools
19
+ tools = create_langchain_tools(manager)
20
+ return tools
21
+
22
+ async def dump_skills(
23
+ skill_name: str = '',
24
+ skill_dir: str = '',
25
+ skill_content: str = '',
26
+ **kwargs,
27
+ ):
28
+
29
+ clean_name = skill_name.strip()
30
+ if not clean_name:
31
+ return {
32
+ "status": "error",
33
+ "message": "skill_name 不能为空"
34
+ }
35
+
36
+ base_path = Path.cwd()
37
+
38
+ # Project_Root + skill_dir + skill_name/
39
+ target_dir_path = base_path / skill_dir / f"cutskill_{clean_name}"
40
+
41
+ # Fix name: SKILL.md
42
+ target_file_path = target_dir_path / "SKILL.md"
43
+
44
+ # Path Traversal Protection
45
+ try:
46
+ final_path = target_file_path.resolve()
47
+ if base_path not in final_path.parents:
48
+ return {
49
+ "status": "error",
50
+ "message": f"Security Alert: 禁止写入项目目录之外的路径: {final_path}"
51
+ }
52
+ except Exception as e:
53
+ return {
54
+ "status": "error",
55
+ "message": f"Path resolution error: {str(e)}"
56
+ }
57
+
58
+ # Start write
59
+ try:
60
+ if not target_dir_path.exists():
61
+ target_dir_path.mkdir(parents=True, exist_ok=True)
62
+
63
+ async with aiofiles.open(final_path, mode='w', encoding='utf-8') as f:
64
+ await f.write(skill_content)
65
+
66
+ return {
67
+ "status": "success",
68
+ "message": f"Skill '{clean_name}' successfully created.",
69
+ "dir_path": str(target_dir_path),
70
+ "file_path": str(final_path),
71
+ "size_bytes": len(skill_content.encode('utf-8'))
72
+ }
73
+
74
+ except PermissionError:
75
+ return {
76
+ "status": "error",
77
+ "message": f"Permission denied: 无法写入目录 {target_dir_path}"
78
+ }
79
+ except Exception as e:
80
+ return {
81
+ "status": "error",
82
+ "message": f"Write operation failed: {str(e)}"
83
+ }
src/open_storyline/storage/__init__.py ADDED
File without changes