Spaces:
Sleeping
Sleeping
xusijie commited on
Commit ·
5c11d93
1
Parent(s): e534277
add open storyline files
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitignore +74 -1
- .storyline/skills/create_profile_style_skill/SKILL.md +63 -0
- .storyline/skills/subtitle_imitation_skill/SKILL.md +46 -0
- Dockerfile +2 -1
- agent_fastapi.py +0 -0
- cli.py +457 -0
- config.toml +112 -0
- download.sh +4 -0
- prompts/tasks/filter_clips/zh/system.md +1 -0
- prompts/tasks/filter_clips/zh/user.md +12 -0
- prompts/tasks/generate_script/zh/system.md +99 -0
- prompts/tasks/generate_script/zh/user.md +17 -0
- prompts/tasks/generate_title/zh/system.md +19 -0
- prompts/tasks/generate_title/zh/user.md +7 -0
- prompts/tasks/group_clips/zh/system.md +125 -0
- prompts/tasks/group_clips/zh/user.md +22 -0
- prompts/tasks/instruction/zh/system.md +66 -0
- prompts/tasks/understand_clips/zh/system_detail.md +3 -0
- prompts/tasks/understand_clips/zh/system_overall.md +2 -0
- prompts/tasks/understand_clips/zh/user_detail.md +7 -0
- prompts/tasks/understand_clips/zh/user_overall.md +2 -0
- requirements.txt +12 -1
- run.sh +20 -0
- src/__init__.py +0 -0
- src/open_storyline/__init__.py +1 -0
- src/open_storyline/__main__.py +0 -0
- src/open_storyline/config.py +189 -0
- src/open_storyline/mcp/__init__.py +0 -0
- src/open_storyline/mcp/hooks/node_interceptors.py +299 -0
- src/open_storyline/mcp/register_tools.py +237 -0
- src/open_storyline/mcp/sampling_handler.py +424 -0
- src/open_storyline/mcp/sampling_requester.py +159 -0
- src/open_storyline/mcp/server.py +70 -0
- src/open_storyline/nodes/__init__.py +1 -0
- src/open_storyline/nodes/core_nodes/assets_index.py +170 -0
- src/open_storyline/nodes/core_nodes/base_node.py +243 -0
- src/open_storyline/nodes/core_nodes/filter_clips.py +149 -0
- src/open_storyline/nodes/core_nodes/generate_script.py +306 -0
- src/open_storyline/nodes/core_nodes/generate_voiceover.py +353 -0
- src/open_storyline/nodes/core_nodes/group_clips.py +190 -0
- src/open_storyline/nodes/core_nodes/plan_timeline.py +904 -0
- src/open_storyline/nodes/core_nodes/render_video.py +838 -0
- src/open_storyline/nodes/core_nodes/select_bgm.py +237 -0
- src/open_storyline/nodes/core_nodes/split_shots.py +778 -0
- src/open_storyline/nodes/core_nodes/understand_clips.py +189 -0
- src/open_storyline/nodes/node_manager.py +172 -0
- src/open_storyline/nodes/node_schema.py +283 -0
- src/open_storyline/nodes/node_summary.py +236 -0
- src/open_storyline/skills/skills_io.py +83 -0
- src/open_storyline/storage/__init__.py +0 -0
.gitignore
CHANGED
|
@@ -1 +1,74 @@
|
|
| 1 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# === Python 生成文件 ===
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.py[cod]
|
| 4 |
+
*$py.class
|
| 5 |
+
|
| 6 |
+
# C 扩展
|
| 7 |
+
*.so
|
| 8 |
+
*.pyd
|
| 9 |
+
|
| 10 |
+
# 虚拟环境 / Conda 环境目录
|
| 11 |
+
.venv/
|
| 12 |
+
venv/
|
| 13 |
+
env/
|
| 14 |
+
.env/
|
| 15 |
+
.conda/
|
| 16 |
+
.hypothesis/
|
| 17 |
+
|
| 18 |
+
# 构建 / 发布产物
|
| 19 |
+
build/
|
| 20 |
+
dist/
|
| 21 |
+
*.egg-info/
|
| 22 |
+
*.egg
|
| 23 |
+
pip-wheel-metadata/
|
| 24 |
+
|
| 25 |
+
# 单元测试 / 覆盖率 / 缓存
|
| 26 |
+
.pytest_cache/
|
| 27 |
+
.coverage
|
| 28 |
+
.coverage.*
|
| 29 |
+
htmlcov/
|
| 30 |
+
.tox/
|
| 31 |
+
.nox/
|
| 32 |
+
.mypy_cache/
|
| 33 |
+
.dmypy.json
|
| 34 |
+
.pyre/
|
| 35 |
+
.cache/
|
| 36 |
+
|
| 37 |
+
# IDE / 编辑器配置
|
| 38 |
+
.vscode/
|
| 39 |
+
.idea/
|
| 40 |
+
*.swp
|
| 41 |
+
*.swo
|
| 42 |
+
*.iml
|
| 43 |
+
|
| 44 |
+
# Jupyter
|
| 45 |
+
.ipynb_checkpoints/
|
| 46 |
+
|
| 47 |
+
# OS 级别垃圾文件
|
| 48 |
+
.DS_Store
|
| 49 |
+
Thumbs.db
|
| 50 |
+
|
| 51 |
+
# 日志 / 临时文件
|
| 52 |
+
*.log
|
| 53 |
+
logs/
|
| 54 |
+
tmp/
|
| 55 |
+
temp/
|
| 56 |
+
.server_cache/
|
| 57 |
+
.storyline/.server_cache/
|
| 58 |
+
|
| 59 |
+
# 本项目可能产生的大文件目录
|
| 60 |
+
outputs/
|
| 61 |
+
renders/
|
| 62 |
+
checkpoints/
|
| 63 |
+
models/
|
| 64 |
+
project/
|
| 65 |
+
|
| 66 |
+
# 环境/配置的敏感信息(你如果用 .env 管 secret)
|
| 67 |
+
.env.local
|
| 68 |
+
.env.*.local
|
| 69 |
+
|
| 70 |
+
data/**
|
| 71 |
+
!data/elements_v2/
|
| 72 |
+
!data/elements_v2/**
|
| 73 |
+
!data/prompts/
|
| 74 |
+
!data/prompts/**
|
.storyline/skills/create_profile_style_skill/SKILL.md
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
name: create_style_profile
|
| 3 |
+
description: 【SKILL】分析当前剪辑逻辑与风格,总结并生成一个新的可复用 Skill 文件,存入剪辑技能库。
|
| 4 |
+
version: 1.0.0
|
| 5 |
+
author: User_Agent_Architect
|
| 6 |
+
tags: [meta-skill, workflow, writing, file-system]
|
| 7 |
+
---
|
| 8 |
+
|
| 9 |
+
# 角色定义 (Role)
|
| 10 |
+
你是一个专业的“剪辑风格架构师”。你具备深厚的影视视听语言知识,能够从具体的剪辑操作(如切点选择、转场习惯、BGM卡点逻辑)中提炼出抽象的“剪辑哲学”和“SOP(标准作业程序)”。
|
| 11 |
+
|
| 12 |
+
# 任务目标 (Objective)
|
| 13 |
+
你的任务是观察或询问用户的剪辑偏好,将其转化为一个标准的 Agent Skill 文档(Markdown格式),并保存到 `.storyline/skills/` 目录下,以便让 Agent 在未来模仿这种风格。
|
| 14 |
+
|
| 15 |
+
# 执行流程 (Workflow)
|
| 16 |
+
|
| 17 |
+
## 第一步:风格分析与萃取 (Analysis & Extraction)
|
| 18 |
+
1. **获取上下文**:获取当前正在编辑的 Timeline 数据,或者请求用户描述其剪辑习惯。
|
| 19 |
+
2. **维度拆解**:你需要从以下维度总结风格:
|
| 20 |
+
* **剪辑节奏 (Pacing)**:是快节奏的跳剪(Jump Cut),还是长镜头的舒缓叙事?
|
| 21 |
+
* **叙事逻辑 (Storytelling)**:是线性叙事、倒叙,还是基于音乐情绪的蒙太奇?
|
| 22 |
+
* **视听语言 (Audio-Visual)**:音效(SFX)的使用密度、字幕样式偏好、调色风格(LUTs)。
|
| 23 |
+
* **特殊偏好**:例如“总是删除静音片段”或“每5秒插入一个B-Roll”。
|
| 24 |
+
|
| 25 |
+
## 第二步:交互与命名 (Interaction & Naming)
|
| 26 |
+
1. **总结确认**:向用户展示你总结的 3-5 个核心风格点,询问是否准确。
|
| 27 |
+
2. **命名建议**:根据风格特点,建议 2 个文件名(例如 `fast_paced_vlog` 或 `cinematic_travel`),命名必须是英文单词和下划线组成,不能出现中文命名。
|
| 28 |
+
3. **获取输入**:
|
| 29 |
+
* 询问用户:“是否认可这个总结?”
|
| 30 |
+
* 询问用户:“你想将这个新技能命名为什么?(按 Enter 使用建议名称:[建议名称])”
|
| 31 |
+
|
| 32 |
+
## 第三步:生成新 Skill 内容 (Drafting)
|
| 33 |
+
根据确认的风格,生成新 Skill 的 Markdown 内容。内容必须包含标准头部和 Prompt 指令。
|
| 34 |
+
* *Template*(新 Skill 的模板结构):
|
| 35 |
+
```markdown
|
| 36 |
+
---
|
| 37 |
+
name: {{用户定义的名称}}
|
| 38 |
+
description: 【SKILL】基于 {{日期}} 总结的 {{风格关键词}} 剪辑风格
|
| 39 |
+
version: 基于对话进行版本管理
|
| 40 |
+
author: 用户
|
| 41 |
+
tags: [相关的tag-list]
|
| 42 |
+
---
|
| 43 |
+
# 剪辑指令
|
| 44 |
+
当执行剪辑任务时,请严格遵守以下逻辑:
|
| 45 |
+
1. **整体风格原则**:{{分析出的节奏逻辑}}
|
| 46 |
+
2. **音频处理规范**:{{分析出的音频处理(视频原声/配音/背景音乐)筛选逻辑}}
|
| 47 |
+
3. **视觉元素规范**:{{分析出的视觉元素(字体花字/转场/滤镜/特效等)使用逻辑}}
|
| 48 |
+
4. **剪辑节奏控制**:{{分析出的剪辑节奏(音乐卡点/短切片/长切片)使用逻辑}}
|
| 49 |
+
5. **工具调用规范**:{{分析出的推荐使用的工具以及推荐的传入参数}}
|
| 50 |
+
```
|
| 51 |
+
|
| 52 |
+
## 第四步:入库与更新 (Commit & Update)
|
| 53 |
+
1. **展示预览**:将生成的内容以代码块形式展示给用户。
|
| 54 |
+
2. **执行写入**:
|
| 55 |
+
* 用户确认后,调用文件写入工具`write_skills`。
|
| 56 |
+
* **目标路径**:`.storyline/skills/{{文件名}}/SKILL.md`,传入文件名即可,工具会自动完成写入。
|
| 57 |
+
3. **系统更新**:提示用户“新技能已入库,请刷新 Agent 工具列表以加载。”
|
| 58 |
+
|
| 59 |
+
# 约束条件 (Constraints)
|
| 60 |
+
* **格式规范**:生成的新 Skill 必须符合 markdown 标准,且包含元数据(Metadata)。
|
| 61 |
+
* **路径安全**:只能写入 `.storyline/skills/` 目录,禁止覆盖系统核心文件。
|
| 62 |
+
* **可读性**:在与用户交互时,不要直接扔出一大段代码,先用自然语言确认逻辑。
|
| 63 |
+
* **版本管理**:当用户进行修改时,更改版本号,并重新调用`write_skills`工具做覆盖;
|
.storyline/skills/subtitle_imitation_skill/SKILL.md
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
name: script_style_mimic
|
| 3 |
+
description: 【SKILL】基于用户提供的参考文案样本,对视频素材内容进行深度文风仿写,生成风格化脚本。
|
| 4 |
+
version: 1.0.0
|
| 5 |
+
author: User_Agent_Architect
|
| 6 |
+
tags: [writing, style-transfer, video-production, creative]
|
| 7 |
+
---
|
| 8 |
+
|
| 9 |
+
# 角色定义 (Role)
|
| 10 |
+
你是一位“文风迁移大师”兼“金牌视频脚本撰写人”。你不仅拥有敏锐的文学感知力,能精准捕捉文字背后的韵律、修辞和情感基调(如“鲁迅体”、“王家卫风”、“发疯文学”),同时深谙视听语言,能够将画面内容转化为极具感染力的旁白或台词,而非机械地描述画面。
|
| 11 |
+
|
| 12 |
+
# 任务目标 (Objective)
|
| 13 |
+
你的核心任务是接收用户的“仿写指令”和“参考文案”,调用历史记忆读取视频素材理解结果(`understand_clips`),生成一份既具备参考文案神韵,又严格基于视频事实的拍摄脚本。
|
| 14 |
+
|
| 15 |
+
# 执行流程 (Workflow)
|
| 16 |
+
|
| 17 |
+
## 第一步:输入校验与意图确认 (Input Validation)
|
| 18 |
+
1. **检查输入参数**:检查用户是否提供了用于模仿的 `style_reference_text`(仿写样本)。
|
| 19 |
+
2. **缺失处理**:
|
| 20 |
+
* **如果用户未提供样本**(仅说“帮我仿写一下”):必须立即中止后续流程,并输出回复引导用户:“为了能精准模仿您想要的文风,请提供一段您希望我模仿的文案示例(例如直接粘贴一段文字,或提供某位博主的典型语录)。”
|
| 21 |
+
* **如果用户已提供样本**:进入第二步。
|
| 22 |
+
|
| 23 |
+
## 第二步:获取素材与分析 (Context & Analysis)
|
| 24 |
+
1. **读取视频理解**:调用工具 `read_node_history`,参数为 `key="understand_clips"`,获取当前视频素材的画面描述、氛围和关键动作。
|
| 25 |
+
2. **风格解构**:在思维链(Chain of Thought)中快速分析用户提供的 `style_reference_text`:
|
| 26 |
+
* **句式特征**:是短句堆叠,还是长难句?
|
| 27 |
+
* **修辞习惯**:是否喜欢用比喻、反讽、排比?
|
| 28 |
+
* **情感基调**:是治愈、焦虑、犀利还是幽默?
|
| 29 |
+
|
| 30 |
+
## 第三步:风格化创作 (Creative Generation)
|
| 31 |
+
基于素材内容(Content)和分析出的风格(Style),执行脚本撰写。需严格遵守以下创作原则:
|
| 32 |
+
1. **拒绝“看图说话” (No See-Say)**:
|
| 33 |
+
* ❌ 错误示范:“画面里有一只猫在睡觉,阳光照在它身上。”
|
| 34 |
+
* ✅ 正确示范(如文艺风):“午后的阳光是免费的,但偷得浮生半日闲的勇气却是昂贵的。它在做梦,而我在看它。”
|
| 35 |
+
2. **内容强关联**:生成的文案必须基于 `understand_clips` 中的真实画面,不能脱离素材天马行空,也不能仅模仿风格却写了无关内容。
|
| 36 |
+
3. **生动连贯**:脚本必须有起承转合,不仅是句子的拼凑,更是一个完整的小故事或情绪流。
|
| 37 |
+
|
| 38 |
+
## 第四步:格式化输出 (Formatting)
|
| 39 |
+
1. **构建数据结构**:将生成的脚本整理为符合工具 `generate_script` 输入要求的格式。
|
| 40 |
+
2. **输出结果**:直接输出用于传递给下游工具的数据结构,并以结构化 Markdown 格式展示给用户确认。
|
| 41 |
+
|
| 42 |
+
# 约束条件 (Constraints)
|
| 43 |
+
* **素材依赖**:必须调用 `read_node_history` 获取素材,严禁在不知道视频内容的情况下瞎编脚本。
|
| 44 |
+
* **风格一致性**:生成的文案必须让熟悉该风格的人一眼就能识别出“味道”。
|
| 45 |
+
* **拒绝机械描述**:严禁出现“视频显示”、“镜头切到”等说明书式语言,除非参考风格本身就是说明书风格。
|
| 46 |
+
* **工具对接**:输出内容必须适配 `generate_script` 的字段定义,确保下游渲染环节无缝衔接。
|
Dockerfile
CHANGED
|
@@ -10,10 +10,11 @@ COPY app.py .
|
|
| 10 |
COPY static/ ./static/
|
| 11 |
|
| 12 |
# 安装依赖
|
|
|
|
| 13 |
RUN pip install --no-cache-dir -r requirements.txt
|
| 14 |
|
| 15 |
# 暴露 HF Space 默认端口
|
| 16 |
EXPOSE 7860
|
| 17 |
|
| 18 |
# 启动命令
|
| 19 |
-
CMD ["
|
|
|
|
| 10 |
COPY static/ ./static/
|
| 11 |
|
| 12 |
# 安装依赖
|
| 13 |
+
RUN apt-get update && apt-get install -y ffmpeg
|
| 14 |
RUN pip install --no-cache-dir -r requirements.txt
|
| 15 |
|
| 16 |
# 暴露 HF Space 默认端口
|
| 17 |
EXPOSE 7860
|
| 18 |
|
| 19 |
# 启动命令
|
| 20 |
+
CMD ["bash", "run.sh"]
|
agent_fastapi.py
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
cli.py
ADDED
|
@@ -0,0 +1,457 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
+
import time
|
| 3 |
+
import uuid
|
| 4 |
+
import os,sys
|
| 5 |
+
import json
|
| 6 |
+
|
| 7 |
+
import contextvars
|
| 8 |
+
from typing import Callable, Optional, Union, Any, List
|
| 9 |
+
from dataclasses import dataclass
|
| 10 |
+
from datetime import timedelta
|
| 11 |
+
|
| 12 |
+
from langchain.agents import create_agent
|
| 13 |
+
from langchain.agents.middleware import wrap_tool_call, wrap_model_call
|
| 14 |
+
from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage, ToolMessage
|
| 15 |
+
from langchain_mcp_adapters.client import MultiServerMCPClient
|
| 16 |
+
from langchain_mcp_adapters.interceptors import MCPToolCallRequest, MCPToolCallResult
|
| 17 |
+
from langchain_mcp_adapters.callbacks import Callbacks, CallbackContext
|
| 18 |
+
from langchain_openai import ChatOpenAI
|
| 19 |
+
from langgraph.types import Command
|
| 20 |
+
|
| 21 |
+
# 把 src 目录加入 Python 的模块搜索路径
|
| 22 |
+
ROOT_DIR = os.path.dirname(__file__)
|
| 23 |
+
SRC_DIR = os.path.join(ROOT_DIR, "src")
|
| 24 |
+
|
| 25 |
+
if SRC_DIR not in sys.path:
|
| 26 |
+
sys.path.insert(0, SRC_DIR)
|
| 27 |
+
|
| 28 |
+
from open_storyline.skills.skills_io import load_skills
|
| 29 |
+
from open_storyline.utils.prompts import get_prompt
|
| 30 |
+
from open_storyline.mcp.sampling_handler import make_sampling_callback
|
| 31 |
+
from open_storyline.config import load_settings, default_config_path
|
| 32 |
+
from open_storyline.config import Settings
|
| 33 |
+
from open_storyline.storage.agent_memory import ArtifactStore
|
| 34 |
+
from open_storyline.nodes.node_manager import NodeManager
|
| 35 |
+
from open_storyline.mcp.hooks.node_interceptors import ToolInterceptor
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
@dataclass
|
| 39 |
+
class ClientContext:
|
| 40 |
+
session_id: str
|
| 41 |
+
assets_dir: str
|
| 42 |
+
bgm_dir: str
|
| 43 |
+
outputs_dir: str
|
| 44 |
+
node_manager: NodeManager
|
| 45 |
+
chat_model_key: str # 当前聊天的模型
|
| 46 |
+
tts_config: Optional[dict] = None # 运行时 TTS 配置
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
# GUI 日志输出通道
|
| 50 |
+
_MCP_LOG_SINK = contextvars.ContextVar("mcp_log_sink", default=None)
|
| 51 |
+
_MCP_ACTIVE_TOOL_CALL_ID = contextvars.ContextVar("mcp_active_tool_call_id", default=None)
|
| 52 |
+
def set_mcp_log_sink(sink: Optional[Callable[[dict], None]]):
|
| 53 |
+
return _MCP_LOG_SINK.set(sink)
|
| 54 |
+
|
| 55 |
+
def reset_mcp_log_sink(token):
|
| 56 |
+
_MCP_LOG_SINK.reset(token)
|
| 57 |
+
|
| 58 |
+
# ------ key 脱敏 + 注入 tts 凭据,先在这里放着,等改完框架看看放到哪里合适---
|
| 59 |
+
CUSTOM_MODEL_KEY = "__custom__"
|
| 60 |
+
|
| 61 |
+
_SENSITIVE_KEYS = {
|
| 62 |
+
"api_key",
|
| 63 |
+
"access_token",
|
| 64 |
+
"authorization",
|
| 65 |
+
"token",
|
| 66 |
+
"password",
|
| 67 |
+
"secret",
|
| 68 |
+
"x-api-key",
|
| 69 |
+
"apikey",
|
| 70 |
+
}
|
| 71 |
+
|
| 72 |
+
def _mask_secrets(obj: Any) -> Any:
|
| 73 |
+
"""递归脱敏:避免 key/token 被打印到控制台、日志、tool trace、ToolMessage等各种地方"""
|
| 74 |
+
try:
|
| 75 |
+
if isinstance(obj, dict):
|
| 76 |
+
out = {}
|
| 77 |
+
for k, v in obj.items():
|
| 78 |
+
if str(k).lower() in _SENSITIVE_KEYS:
|
| 79 |
+
out[k] = "***"
|
| 80 |
+
else:
|
| 81 |
+
out[k] = _mask_secrets(v)
|
| 82 |
+
return out
|
| 83 |
+
if isinstance(obj, list):
|
| 84 |
+
return [_mask_secrets(x) for x in obj]
|
| 85 |
+
if isinstance(obj, tuple):
|
| 86 |
+
return tuple(_mask_secrets(x) for x in obj)
|
| 87 |
+
return obj
|
| 88 |
+
except Exception:
|
| 89 |
+
return "***"
|
| 90 |
+
|
| 91 |
+
async def inject_tts_config(request: MCPToolCallRequest, handler):
|
| 92 |
+
"""
|
| 93 |
+
拦截器:在调用 TTS 工具前注入 key。
|
| 94 |
+
"""
|
| 95 |
+
try:
|
| 96 |
+
runtime = getattr(request, "runtime", None)
|
| 97 |
+
ctx = getattr(runtime, "context", None) if runtime else None
|
| 98 |
+
tts_cfg = getattr(ctx, "tts_config", None) if ctx else None
|
| 99 |
+
|
| 100 |
+
if not (isinstance(tts_cfg, dict) and isinstance(getattr(request, "args", None), dict)):
|
| 101 |
+
return await handler(request)
|
| 102 |
+
|
| 103 |
+
tool_name = str(getattr(request, "name", "") or "")
|
| 104 |
+
if "voiceover" not in tool_name:
|
| 105 |
+
return await handler(request)
|
| 106 |
+
|
| 107 |
+
vendor = str(tts_cfg.get("vendor") or "").lower().strip()
|
| 108 |
+
if vendor != "bytedance":
|
| 109 |
+
return await handler(request)
|
| 110 |
+
|
| 111 |
+
bd = tts_cfg.get("bytedance") or {}
|
| 112 |
+
if not isinstance(bd, dict):
|
| 113 |
+
return await handler(request)
|
| 114 |
+
|
| 115 |
+
uid = bd.get("uid")
|
| 116 |
+
appid = bd.get("appid")
|
| 117 |
+
access_token = bd.get("access_token")
|
| 118 |
+
if uid and appid and access_token:
|
| 119 |
+
request.args.setdefault("uid", uid)
|
| 120 |
+
request.args.setdefault("appid", appid)
|
| 121 |
+
request.args.setdefault("access_token", access_token)
|
| 122 |
+
except Exception:
|
| 123 |
+
pass
|
| 124 |
+
|
| 125 |
+
return await handler(request)
|
| 126 |
+
|
| 127 |
+
async def log_mcp_request(request: MCPToolCallRequest, handler):
|
| 128 |
+
sink = _MCP_LOG_SINK.get()
|
| 129 |
+
|
| 130 |
+
def emit_event(x: str | dict):
|
| 131 |
+
if sink:
|
| 132 |
+
sink(x)
|
| 133 |
+
|
| 134 |
+
runtime = request.runtime
|
| 135 |
+
context = runtime.context
|
| 136 |
+
meta_collector = context.node_manager
|
| 137 |
+
|
| 138 |
+
exclude = set(meta_collector.kind_to_node_ids.keys()) | {
|
| 139 |
+
"inputs", "artifacts_dir", "artifact_id", "blobs_dir", "meta_path",
|
| 140 |
+
"assets_dir", "bgm_dir", "outputs_dir", "debug_dir",
|
| 141 |
+
}
|
| 142 |
+
|
| 143 |
+
extracted_args = {}
|
| 144 |
+
for arg in (request.args.keys() if isinstance(request.args, dict) else []):
|
| 145 |
+
if arg not in exclude:
|
| 146 |
+
extracted_args[arg] = request.args[arg]
|
| 147 |
+
|
| 148 |
+
extracted_args = _mask_secrets(extracted_args)
|
| 149 |
+
|
| 150 |
+
tool_call_id = getattr(runtime, "tool_call_id", None)
|
| 151 |
+
if not tool_call_id:
|
| 152 |
+
tool_call_id = f"mcp_{uuid.uuid4().hex[:8]}"
|
| 153 |
+
if runtime is not None:
|
| 154 |
+
setattr(runtime, "tool_call_id", tool_call_id)
|
| 155 |
+
|
| 156 |
+
active_tok = _MCP_ACTIVE_TOOL_CALL_ID.set(tool_call_id)
|
| 157 |
+
try:
|
| 158 |
+
emit_event({
|
| 159 |
+
"type": "tool_start",
|
| 160 |
+
"tool_call_id": tool_call_id,
|
| 161 |
+
"server": request.server_name,
|
| 162 |
+
"name": request.name,
|
| 163 |
+
"args": extracted_args,
|
| 164 |
+
})
|
| 165 |
+
print(f"[MCP 工具开始] {request.server_name}.{request.name} args={extracted_args}\n")
|
| 166 |
+
out = await handler(request)
|
| 167 |
+
try:
|
| 168 |
+
isError = json.loads(out.content[0].text).get('isError', False)
|
| 169 |
+
except Exception:
|
| 170 |
+
isError = True
|
| 171 |
+
finally:
|
| 172 |
+
_MCP_ACTIVE_TOOL_CALL_ID.reset(active_tok)
|
| 173 |
+
|
| 174 |
+
if out.isError or isError:
|
| 175 |
+
print(f"[MCP 工具出错] result:{out.content}\n\n")
|
| 176 |
+
emit_event({
|
| 177 |
+
"type": "tool_end",
|
| 178 |
+
"tool_call_id": tool_call_id,
|
| 179 |
+
"server": request.server_name,
|
| 180 |
+
"name": request.name,
|
| 181 |
+
"is_error": True,
|
| 182 |
+
"summary": _mask_secrets(out.content),
|
| 183 |
+
})
|
| 184 |
+
else:
|
| 185 |
+
payload = json.loads(out.content[0].text)
|
| 186 |
+
summary = payload.get("summary")
|
| 187 |
+
|
| 188 |
+
print(f"[MCP 工具完成] result:{summary}\n\n")
|
| 189 |
+
emit_event({
|
| 190 |
+
"type": "tool_end",
|
| 191 |
+
"tool_call_id": tool_call_id,
|
| 192 |
+
"server": request.server_name,
|
| 193 |
+
"name": request.name,
|
| 194 |
+
"is_error": False,
|
| 195 |
+
"summary": _mask_secrets(summary),
|
| 196 |
+
})
|
| 197 |
+
|
| 198 |
+
return out
|
| 199 |
+
|
| 200 |
+
@wrap_tool_call
|
| 201 |
+
async def handle_tool_errors(request, handler):
|
| 202 |
+
try:
|
| 203 |
+
out = await handler(request)
|
| 204 |
+
if isinstance(out, Command):
|
| 205 |
+
return out.update.get('messages')[0]
|
| 206 |
+
elif isinstance(out, MCPToolCallResult) and not isinstance(out.content, str):
|
| 207 |
+
return ToolMessage(content=out.content[0].get("text", ""), tool_call_id=out.tool_call_id, name=out.name) # 解决deepseek-chat兼容性问题
|
| 208 |
+
return out
|
| 209 |
+
except Exception as e:
|
| 210 |
+
tc = request.tool_call
|
| 211 |
+
safe_args = _mask_secrets(tc.get("args") or {})
|
| 212 |
+
return ToolMessage(
|
| 213 |
+
content=(
|
| 214 |
+
"工具调用失败\n"
|
| 215 |
+
f"工具名:{tc.get('name')}\n"
|
| 216 |
+
f"工具参数:{safe_args}\n"
|
| 217 |
+
f"错误信息:{type(e).__name__}: {e}\n"
|
| 218 |
+
"如果是参数问题,请修正参数重新调用;如果是因为缺少前置依赖,请先调用前置节点;如果你认为是偶发错误,请尝试重新调用;如果你认为已经无法继续,请向用户解释原因。"
|
| 219 |
+
),
|
| 220 |
+
tool_call_id=tc["id"],
|
| 221 |
+
)
|
| 222 |
+
|
| 223 |
+
async def on_progress(progress: float, total: float | None, message: str| None, context: CallbackContext):
|
| 224 |
+
sink = _MCP_LOG_SINK.get()
|
| 225 |
+
if sink:
|
| 226 |
+
sink({
|
| 227 |
+
"type": "tool_progress",
|
| 228 |
+
"tool_call_id": _MCP_ACTIVE_TOOL_CALL_ID.get(),
|
| 229 |
+
"server": context.server_name,
|
| 230 |
+
"name": context.tool_name,
|
| 231 |
+
"progress": progress,
|
| 232 |
+
"total": total,
|
| 233 |
+
"message": message,
|
| 234 |
+
})
|
| 235 |
+
|
| 236 |
+
async def build_agent(
|
| 237 |
+
cfg: Settings,
|
| 238 |
+
session_id: str,
|
| 239 |
+
store: ArtifactStore,
|
| 240 |
+
tool_interceptors=None,
|
| 241 |
+
*,
|
| 242 |
+
llm_override: Optional[dict] = None,
|
| 243 |
+
vlm_override: Optional[dict] = None,
|
| 244 |
+
):
|
| 245 |
+
def _get(override: Optional[dict], key: str, default: Any) -> Any:
|
| 246 |
+
return (override.get(key) if isinstance(override, dict) and key in override else default)
|
| 247 |
+
|
| 248 |
+
def _norm_url(u: str) -> str:
|
| 249 |
+
u = (u or "").strip()
|
| 250 |
+
return u.rstrip("/") if u else u
|
| 251 |
+
|
| 252 |
+
# 1) LLM 优先用输入框里用户填写的,其次是 config.toml 里的
|
| 253 |
+
llm_model = _get(llm_override, "model", cfg.llm.model)
|
| 254 |
+
llm_base_url = _norm_url(_get(llm_override, "base_url", cfg.llm.base_url))
|
| 255 |
+
llm_api_key = _get(llm_override, "api_key", cfg.llm.api_key)
|
| 256 |
+
llm_timeout = _get(llm_override, "timeout", cfg.llm.timeout)
|
| 257 |
+
llm_temperature = _get(llm_override, "temperature", cfg.llm.temperature)
|
| 258 |
+
llm_max_retries = _get(llm_override, "max_retries", cfg.llm.max_retries)
|
| 259 |
+
|
| 260 |
+
llm = ChatOpenAI(
|
| 261 |
+
model=llm_model,
|
| 262 |
+
base_url=llm_base_url,
|
| 263 |
+
api_key=llm_api_key,
|
| 264 |
+
default_headers={
|
| 265 |
+
"api-key": llm_api_key,
|
| 266 |
+
"Content-Type": "application/json",
|
| 267 |
+
},
|
| 268 |
+
timeout=llm_timeout,
|
| 269 |
+
temperature=llm_temperature,
|
| 270 |
+
streaming=True,
|
| 271 |
+
max_retries=llm_max_retries,
|
| 272 |
+
)
|
| 273 |
+
|
| 274 |
+
# 2) VLM 优先级同上
|
| 275 |
+
vlm_model = _get(vlm_override, "model", cfg.vlm.model)
|
| 276 |
+
vlm_base_url = _norm_url(_get(vlm_override, "base_url", cfg.vlm.base_url))
|
| 277 |
+
vlm_api_key = _get(vlm_override, "api_key", cfg.vlm.api_key)
|
| 278 |
+
vlm_timeout = _get(vlm_override, "timeout", cfg.vlm.timeout)
|
| 279 |
+
vlm_temperature = _get(vlm_override, "temperature", cfg.vlm.temperature)
|
| 280 |
+
vlm_max_retries = _get(vlm_override, "max_retries", cfg.vlm.max_retries)
|
| 281 |
+
|
| 282 |
+
vlm = ChatOpenAI(
|
| 283 |
+
model=vlm_model,
|
| 284 |
+
base_url=vlm_base_url,
|
| 285 |
+
api_key=vlm_api_key,
|
| 286 |
+
default_headers={
|
| 287 |
+
"api-key": vlm_api_key,
|
| 288 |
+
"Content-Type": "application/json",
|
| 289 |
+
},
|
| 290 |
+
timeout=vlm_timeout,
|
| 291 |
+
temperature=vlm_temperature,
|
| 292 |
+
max_retries=vlm_max_retries,
|
| 293 |
+
)
|
| 294 |
+
|
| 295 |
+
sampling_callback = make_sampling_callback(llm, vlm)
|
| 296 |
+
|
| 297 |
+
# ---- 开发者模型 allowlist,只用于站内预置模型切换 ----
|
| 298 |
+
allowed = set(cfg.developer.chat_models or [])
|
| 299 |
+
llm_pool: dict[tuple[str, bool], ChatOpenAI] = {}
|
| 300 |
+
|
| 301 |
+
def _make_chat_llm(model_name: str, streaming: bool) -> ChatOpenAI:
|
| 302 |
+
model_config = (cfg.developer.chat_models_config.get(model_name) or {})
|
| 303 |
+
base_url = _norm_url(model_config.get("base_url") or "")
|
| 304 |
+
api_key = model_config.get("api_key")
|
| 305 |
+
return ChatOpenAI(
|
| 306 |
+
model=model_name,
|
| 307 |
+
base_url=base_url,
|
| 308 |
+
api_key=api_key,
|
| 309 |
+
default_headers={
|
| 310 |
+
"api-key": api_key,
|
| 311 |
+
"Content-Type": "application/json",
|
| 312 |
+
},
|
| 313 |
+
timeout=cfg.llm.timeout,
|
| 314 |
+
temperature=model_config.get("temperature", cfg.llm.temperature),
|
| 315 |
+
streaming=streaming,
|
| 316 |
+
)
|
| 317 |
+
|
| 318 |
+
def _get_llm(model_name: str, streaming: bool) -> ChatOpenAI:
|
| 319 |
+
hit = llm_pool.get((model_name, streaming))
|
| 320 |
+
if hit:
|
| 321 |
+
return hit
|
| 322 |
+
new_llm = _make_chat_llm(model_name, streaming=streaming)
|
| 323 |
+
llm_pool[(model_name, streaming)] = new_llm
|
| 324 |
+
return new_llm
|
| 325 |
+
|
| 326 |
+
@wrap_model_call
|
| 327 |
+
async def switch_chat_model(request, handler):
|
| 328 |
+
runtime = getattr(request, "runtime", None)
|
| 329 |
+
ctx = getattr(runtime, "context", None) if runtime else None
|
| 330 |
+
model_key = getattr(ctx, "chat_model_key", None) if ctx else None
|
| 331 |
+
|
| 332 |
+
model_key = model_key.strip() if isinstance(model_key, str) else ""
|
| 333 |
+
if not model_key:
|
| 334 |
+
return await handler(request)
|
| 335 |
+
|
| 336 |
+
if model_key == CUSTOM_MODEL_KEY:
|
| 337 |
+
return await handler(request)
|
| 338 |
+
|
| 339 |
+
if model_key not in allowed:
|
| 340 |
+
return await handler(request)
|
| 341 |
+
|
| 342 |
+
# 优先流式,失败再非流式
|
| 343 |
+
for _ in range(2):
|
| 344 |
+
try:
|
| 345 |
+
llm_stream = _get_llm(model_key, streaming=True)
|
| 346 |
+
if cfg.developer.print_context:
|
| 347 |
+
print(request)
|
| 348 |
+
return await handler(request.override(model=llm_stream))
|
| 349 |
+
except Exception as e1:
|
| 350 |
+
print(f"流式 llm 出错:{e1}")
|
| 351 |
+
await asyncio.sleep(2)
|
| 352 |
+
|
| 353 |
+
for _ in range(2):
|
| 354 |
+
try:
|
| 355 |
+
llm_nostream = _get_llm(model_key, streaming=False)
|
| 356 |
+
if cfg.developer.print_context:
|
| 357 |
+
print(request)
|
| 358 |
+
return await handler(request.override(model=llm_nostream))
|
| 359 |
+
except Exception as e2:
|
| 360 |
+
print(f"非流式 llm 出错:{e2}")
|
| 361 |
+
await asyncio.sleep(2)
|
| 362 |
+
|
| 363 |
+
fresh = _make_chat_llm(model_key, streaming=False)
|
| 364 |
+
llm_pool[(model_key, False)] = fresh
|
| 365 |
+
if cfg.developer.print_context:
|
| 366 |
+
print(request)
|
| 367 |
+
return await handler(request.override(model=fresh))
|
| 368 |
+
|
| 369 |
+
connections = {
|
| 370 |
+
cfg.local_mcp_server.server_name: {
|
| 371 |
+
"transport": cfg.local_mcp_server.server_transport,
|
| 372 |
+
"url": cfg.local_mcp_server.url,
|
| 373 |
+
"timeout": timedelta(seconds=cfg.local_mcp_server.timeout),
|
| 374 |
+
"headers": {"X-Storyline-Session-Id": session_id},
|
| 375 |
+
"session_kwargs": {"sampling_callback": sampling_callback},
|
| 376 |
+
},
|
| 377 |
+
}
|
| 378 |
+
|
| 379 |
+
client = MultiServerMCPClient(
|
| 380 |
+
connections=connections,
|
| 381 |
+
tool_interceptors=tool_interceptors or [log_mcp_request],
|
| 382 |
+
callbacks=Callbacks(on_progress=on_progress),
|
| 383 |
+
tool_name_prefix=True,
|
| 384 |
+
)
|
| 385 |
+
|
| 386 |
+
tools = await client.get_tools()
|
| 387 |
+
skills = await load_skills() # 加载skills
|
| 388 |
+
node_manager = NodeManager(tools)
|
| 389 |
+
|
| 390 |
+
# 4) 用 LangChain 的 agent runtime 接管“多轮工具调用 loop”
|
| 391 |
+
agent = create_agent(
|
| 392 |
+
model=llm,
|
| 393 |
+
tools=tools+skills,
|
| 394 |
+
middleware=[switch_chat_model, handle_tool_errors],
|
| 395 |
+
store=store,
|
| 396 |
+
context_schema=ClientContext,
|
| 397 |
+
)
|
| 398 |
+
return agent, node_manager
|
| 399 |
+
|
| 400 |
+
async def main():
|
| 401 |
+
session_id = f"run_{int(time.time())}_{uuid.uuid4().hex[:8]}"
|
| 402 |
+
cfg = load_settings(default_config_path())
|
| 403 |
+
|
| 404 |
+
artifact_store = ArtifactStore(cfg.project.outputs_dir, session_id=session_id)
|
| 405 |
+
agent, node_manager = await build_agent(cfg=cfg, session_id=session_id, store=artifact_store, tool_interceptors=[ToolInterceptor.inject_media_content_before, ToolInterceptor.save_media_content_after, log_mcp_request])
|
| 406 |
+
|
| 407 |
+
context = ClientContext(
|
| 408 |
+
session_id=session_id,
|
| 409 |
+
assets_dir=cfg.project.assets_dir,
|
| 410 |
+
bgm_dir=cfg.project.bgm_dir,
|
| 411 |
+
outputs_dir=cfg.project.outputs_dir,
|
| 412 |
+
node_manager=node_manager,
|
| 413 |
+
chat_model_key=cfg.llm.model,
|
| 414 |
+
)
|
| 415 |
+
|
| 416 |
+
# 用 messages 列表做最简单的短期记忆(替代手写 history_for_llm 字符串)
|
| 417 |
+
messages: List[BaseMessage] = [SystemMessage(content=get_prompt("instruction.system"))]
|
| 418 |
+
|
| 419 |
+
print("智能剪辑Agent(LangChain + MCP) v 0.2.0")
|
| 420 |
+
print("请描述你的剪辑需求,输入 /exit 退出")
|
| 421 |
+
|
| 422 |
+
while True:
|
| 423 |
+
try:
|
| 424 |
+
user_input = input("你:").strip()
|
| 425 |
+
except (EOFError, KeyboardInterrupt):
|
| 426 |
+
print("\n再见~")
|
| 427 |
+
break
|
| 428 |
+
|
| 429 |
+
if not user_input:
|
| 430 |
+
continue
|
| 431 |
+
if user_input in ("/exit", "/quit"):
|
| 432 |
+
print("\n再见~")
|
| 433 |
+
break
|
| 434 |
+
|
| 435 |
+
messages.append(HumanMessage(content=user_input))
|
| 436 |
+
|
| 437 |
+
print("正在调用LLM中,请耐心等待...")
|
| 438 |
+
|
| 439 |
+
# 这里一次 ainvoke,就会在内部完成:推理 -> 多次调用工具 -> 汇总 -> 最终回答
|
| 440 |
+
result = await agent.ainvoke(
|
| 441 |
+
{"messages": messages},
|
| 442 |
+
context=context
|
| 443 |
+
)
|
| 444 |
+
|
| 445 |
+
messages = result["messages"]
|
| 446 |
+
|
| 447 |
+
final_text = None
|
| 448 |
+
for m in reversed(messages):
|
| 449 |
+
if isinstance(m, AIMessage):
|
| 450 |
+
final_text = m.content
|
| 451 |
+
break
|
| 452 |
+
|
| 453 |
+
print(f"\n助手:{final_text or '(未生成最终答复)'}\n")
|
| 454 |
+
|
| 455 |
+
|
| 456 |
+
if __name__ == "__main__":
|
| 457 |
+
asyncio.run(main())
|
config.toml
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ============= 开发者选项 ===============
|
| 2 |
+
[developer]
|
| 3 |
+
developer_mode = true
|
| 4 |
+
chat_models = ["deepseek-v3.2", "gemini-2.5-pro"]
|
| 5 |
+
print_context = false # 在拦截器打印模型拿到的全部上下文,会很长
|
| 6 |
+
|
| 7 |
+
# ============= 模型配置 for 站内体验网页 ===============
|
| 8 |
+
[developer.chat_models_config."deepseek-v3.2"]
|
| 9 |
+
base_url = ""
|
| 10 |
+
api_key = ""
|
| 11 |
+
temperature = 0.1
|
| 12 |
+
|
| 13 |
+
[developer.chat_models_config."gemini-2.5-pro"]
|
| 14 |
+
base_url = ""
|
| 15 |
+
api_key = ""
|
| 16 |
+
temperature = 0.1
|
| 17 |
+
|
| 18 |
+
[developer.chat_models_config."qwen3-vl-32b-instruct"]
|
| 19 |
+
base_url = ""
|
| 20 |
+
api_key = ""
|
| 21 |
+
timeout = 20.0
|
| 22 |
+
temperature = 0.1
|
| 23 |
+
max_retries = 2
|
| 24 |
+
|
| 25 |
+
# ============= 项目路径 ======================
|
| 26 |
+
[project]
|
| 27 |
+
assets_dir = "./resource/media"
|
| 28 |
+
bgm_dir = "./resource/bgms"
|
| 29 |
+
outputs_dir = "./outputs"
|
| 30 |
+
|
| 31 |
+
# ============= 模型配置 for user =============
|
| 32 |
+
[llm]
|
| 33 |
+
model = "deepseek-v3.2"
|
| 34 |
+
base_url = ""
|
| 35 |
+
api_key = ""
|
| 36 |
+
timeout = 30.0 # 单位:秒
|
| 37 |
+
temperature = 0.1
|
| 38 |
+
max_retries = 2
|
| 39 |
+
|
| 40 |
+
[vlm]
|
| 41 |
+
# model = "qwen3-vl-8b-instruct"
|
| 42 |
+
# base_url = ""
|
| 43 |
+
# api_key = ""
|
| 44 |
+
model = "gemini-2.5-pro"
|
| 45 |
+
base_url = ""
|
| 46 |
+
api_key = ""
|
| 47 |
+
timeout = 20.0 # 单位:秒
|
| 48 |
+
temperature = 0.1
|
| 49 |
+
max_retries = 2
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
# ============= MCP Server 相关 =============
|
| 53 |
+
[local_mcp_server]
|
| 54 |
+
server_name = "storyline"
|
| 55 |
+
server_cache_dir = ".storyline/.server_cache"
|
| 56 |
+
server_transport = "streamable-http" # server 和 host之间的传输方式
|
| 57 |
+
url_scheme = "http"
|
| 58 |
+
connect_host = "127.0.0.1" # 不要改动
|
| 59 |
+
port = 8001 # 如果端口冲突,可以随便用一个有空的端口
|
| 60 |
+
path = "/mcp" # 默认值,一般不用改
|
| 61 |
+
|
| 62 |
+
json_response = true # 建议用 True
|
| 63 |
+
stateless_http = false # 强烈建议用 False
|
| 64 |
+
timeout = 600
|
| 65 |
+
use_xhs_node_only = false
|
| 66 |
+
|
| 67 |
+
[xhs_mcp_server]
|
| 68 |
+
server_name = "xhs"
|
| 69 |
+
server_cache_dir = ".storyline/.server_cache"
|
| 70 |
+
server_transport = "streamable-http" # server 和 host之间的传输方式
|
| 71 |
+
url_scheme = "http"
|
| 72 |
+
connect_host = "127.0.0.1" # 上线后应该是在站内起 MCP server,但是现在开发阶段先在本地起
|
| 73 |
+
port = 8002 # 也可以随便用一个有空的端口
|
| 74 |
+
path = "/mcp"
|
| 75 |
+
|
| 76 |
+
json_response = true # 建议用 True
|
| 77 |
+
stateless_http = false # 强烈建议用 False
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
# ============= 镜头分割 =============
|
| 81 |
+
[split_shots]
|
| 82 |
+
transnet_weights = ".storyline/models/transnetv2-pytorch-weights.pth"
|
| 83 |
+
transnet_device = "cpu"
|
| 84 |
+
|
| 85 |
+
# ============= 视频视觉理解 =============
|
| 86 |
+
[understand_clips]
|
| 87 |
+
sample_fps = 2.0 # 每秒抽几帧
|
| 88 |
+
max_frames = 64 # 单clip抽帧上限兜底,避免长视频爆 token
|
| 89 |
+
|
| 90 |
+
# ============= 配音生成 ===================
|
| 91 |
+
[generate_voiceover]
|
| 92 |
+
base_url = "https://openspeech.bytedance.com/api/v1/tts"
|
| 93 |
+
|
| 94 |
+
# ============= BGM选择 ====================
|
| 95 |
+
# 主要是用于计算音乐特征的一些参数
|
| 96 |
+
[select_bgm]
|
| 97 |
+
sample_rate = 22050
|
| 98 |
+
hop_length = 2048 # 每次分析窗口向前跳多少个采样点,越小越精细(但更慢)
|
| 99 |
+
frame_length = 2048 # 计算信号的均方根RMS的窗口大小。越大越稳定,但对瞬态不敏感
|
| 100 |
+
|
| 101 |
+
# ============= 时间线组织 ====================
|
| 102 |
+
[plan_timeline]
|
| 103 |
+
beat_type_max = 1 # 使用多强的鼓点,例如4/4中,鼓点类似1,2,1,3,其中1是最强的,3最弱
|
| 104 |
+
title_dutation = 0 # 片头时长 (ms)
|
| 105 |
+
bgm_loop = true # 是否允许 bgm 循环
|
| 106 |
+
min_clip_duration = 500
|
| 107 |
+
|
| 108 |
+
estimate_text_min = 1500 # 在没有TTS的情况下,估计每段字幕至少上屏多久
|
| 109 |
+
estimate_text_char_per_sec = 6.0 # 在没有TTS的情况下,估计每秒展示几个字
|
| 110 |
+
|
| 111 |
+
image_default_duration = 3000 # 默认的图片播放时长
|
| 112 |
+
group_margin_over_voiceover = 1000 # 在一个group中,画面比配音多出现多久
|
download.sh
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
mkdir .storyline/models
|
| 2 |
+
wget "https://image-url-2-feature-1251524319.cos.ap-shanghai.myqcloud.com/zailin/models/storyline/open_storyline/models/transnetv2-pytorch-weights.pth" -O .storyline/models/transnetv2-pytorch-weights.pth
|
| 3 |
+
wget "https://image-url-2-feature-1251524319.cos.ap-shanghai.myqcloud.com/zailin/datasets/open_storyline/resource.zip" -O .storyline/resource.zip
|
| 4 |
+
unzip .storyline/resource.zip -d resource
|
prompts/tasks/filter_clips/zh/system.md
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
你是一个短视频剪辑助手。你的任务是:根据“用户要求”判断每个 clip 是否保留。不要输出任何多余内容,只输出JSON。
|
prompts/tasks/filter_clips/zh/user.md
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
用户要求: {user_request}
|
| 2 |
+
请根据用户要求,判断下面所有 clips 是否保留
|
| 3 |
+
{clip_captions}
|
| 4 |
+
输出格式如下:
|
| 5 |
+
```json
|
| 6 |
+
{{
|
| 7 |
+
"results": [
|
| 8 |
+
{{"clip_id": "clip_0001", "keep": true/false}}
|
| 9 |
+
{{"clip_id": "clip_0002", "keep": true/false}}
|
| 10 |
+
]
|
| 11 |
+
}}
|
| 12 |
+
```
|
prompts/tasks/generate_script/zh/system.md
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 角色设定
|
| 2 |
+
你是一位资深的短视频及Vlog文案策划大师。你拥有敏锐的洞察力,擅长化身为视频的主角(第一人称“我”),用“轻叙事感”的口语,将碎片化的素材串联成有温度、有逻辑、情感饱满的故事。
|
| 3 |
+
|
| 4 |
+
# 任务目标
|
| 5 |
+
你的任务是根据用户提供的 [user_request](视频核心主题)、[style](文案风格)和 [group_infos](分组素材详情),为每一个分组(Group)编写一段旁白文案,并为整个视频起一个标题。
|
| 6 |
+
|
| 7 |
+
# 输入数据说明
|
| 8 |
+
输入数据包含三部分:
|
| 9 |
+
1. **[user_request]**: 视频的核心主题或创作者的感悟。
|
| 10 |
+
2. **[overall]**: 用户提供的所有素材的总体叙事概述。
|
| 11 |
+
3. **[style]**: 指定文案的风格偏好(如:文艺抒情、幽默有趣、日常碎碎念等)。
|
| 12 |
+
4. **[group_infos]**: 包含多个分组,每个分组代表视频的一个段落。关键字段:
|
| 13 |
+
- `summary`: 该段落的叙事目的。
|
| 14 |
+
- `script_chars_budget`: **关键约束**。文案字数必须严格落在该区间内。
|
| 15 |
+
- `clips`: 该组包含的具体画面描述。
|
| 16 |
+
|
| 17 |
+
# 风格效果配置 (Style Configuration)
|
| 18 |
+
请根据输入中的 `[style]` 字段,严格采用对应的写作策略。如输入未指定,默认为**“日常碎碎念”**。
|
| 19 |
+
|
| 20 |
+
1. **文艺抒情 (Lyrical & Poetic)**:
|
| 21 |
+
- **核心**: 治愈、浪漫、电影感、注重意象。
|
| 22 |
+
- **策略**: 弱化直白的动作描述,强化感官体验(光影、气味、温度、声音)。多用比喻和拟人,句式优美流畅,注重情感的流动和余韵,像在读一首散文诗。
|
| 23 |
+
|
| 24 |
+
2. **幽默有趣 (Humorous & Witty)**:
|
| 25 |
+
- **核心**: 玩梗(适度)、反转、自嘲、节奏轻快。
|
| 26 |
+
- **策略**: 寻找画面中意想不到的槽点或亮点。用词生动活泼,可以使用夸张的修辞,像个有趣的“显眼包”朋友在吐槽或分享趣事,拒绝沉闷。
|
| 27 |
+
|
| 28 |
+
3. **日常碎碎念 (Daily Mumbling)**:
|
| 29 |
+
- **核心**: 真实、极度生活化、大脑独白、亲切。
|
| 30 |
+
- **策略**: 还原大脑里的真实想法,甚至可以有一点点逻辑跳跃。关注细枝末节(如“今天风有点大”),不刻意升华,主打一种“陪伴感”和“流水账”的真实美学。
|
| 31 |
+
|
| 32 |
+
# 创作原则 (核心)
|
| 33 |
+
请严格遵守以下创作原则,优先级从上到下:
|
| 34 |
+
|
| 35 |
+
1. **主角视角与口吻 (Tone & Perspective)**:
|
| 36 |
+
- 全程使用**第一人称“我”**的视角叙事。
|
| 37 |
+
- 语言风格需符合上述 `[style]` 的设定,但必须保持**口语化**。
|
| 38 |
+
- **拒绝陈旧套路**: 开场**严禁**使用“家人们谁懂啊”、“姐妹们”等模板;结尾**严禁**使用“原来快乐如此简单”等空洞金句。
|
| 39 |
+
|
| 40 |
+
2. **关键信息保真 (Information Fidelity)**:
|
| 41 |
+
- 必须敏锐识别并保留画面描述或主题中的**【专有名词】**(如品牌名、地名)、**【IP】**(如迪士尼)和**【具体事件】**。
|
| 42 |
+
- **不要泛化**: 结合具体的视觉元素(Visual)写作,切记胡编乱造。
|
| 43 |
+
|
| 44 |
+
3. **字数与技术规范 (Technical Constraints)**:
|
| 45 |
+
- **字数严格控制**: 生成的 `raw_text` 长度必须严格落在 `script_chars_budget` 范围内。
|
| 46 |
+
- **标点符号限制**:
|
| 47 |
+
- **绝对禁止**使用任何形式的括号 `()` 或省略号 `...`。
|
| 48 |
+
- 标点需符合口语断句习惯。
|
| 49 |
+
- **Emoji使用**: 每段文案可适当使用 1 个与内容强相关的 Emoji。
|
| 50 |
+
|
| 51 |
+
4. **画面关联与叙事 (Visual & Storytelling)**:
|
| 52 |
+
- **看图表达**: 文案必须是画面的注脚。
|
| 53 |
+
- **连贯性**: Group 之间要有逻辑衔接,使用自然的过渡词。
|
| 54 |
+
- **结构**:
|
| 55 |
+
- **开场 (Group 1)**: 迅速入题,根据风格设定基调。
|
| 56 |
+
- **结尾 (Last Group)**: 情感收束,根据风格进行升华或幽默收尾。
|
| 57 |
+
|
| 58 |
+
5. **标题创作 (Title)**:
|
| 59 |
+
- 创作一个富有诗意、悬念感或总结性的 `title`,长度 8-15 字,需具备社交媒体(如小红书)的吸引力。
|
| 60 |
+
|
| 61 |
+
# 输出格式
|
| 62 |
+
请仅输出一个标准的 JSON 对象,不要包含 Markdown 符号,格式如下:
|
| 63 |
+
|
| 64 |
+
```json
|
| 65 |
+
{{
|
| 66 |
+
"group_scripts": [
|
| 67 |
+
{{
|
| 68 |
+
"group_id": "对应输入的group_id",
|
| 69 |
+
"duration": 对应输入的duration_sec,
|
| 70 |
+
"raw_text": "生成的文案内容"
|
| 71 |
+
}}
|
| 72 |
+
],
|
| 73 |
+
"title": "生成的视频标题"
|
| 74 |
+
}}
|
| 75 |
+
```
|
| 76 |
+
|
| 77 |
+
# 示例
|
| 78 |
+
**Input:**
|
| 79 |
+
[user_input]
|
| 80 |
+
周末去公园野餐,感觉被治愈了
|
| 81 |
+
[style]
|
| 82 |
+
文艺抒情
|
| 83 |
+
[group_infos]
|
| 84 |
+
[group_id=group_0001]
|
| 85 |
+
summary: 展示准备食物和到达公园的过程。
|
| 86 |
+
script_chars_budget: 15~25
|
| 87 |
+
clips: ...三明治特写,骑单车经过林荫道...
|
| 88 |
+
|
| 89 |
+
**Output:**
|
| 90 |
+
{{
|
| 91 |
+
"group_scripts": [
|
| 92 |
+
{{
|
| 93 |
+
"group_id": "group_0001",
|
| 94 |
+
"duration": 5.00,
|
| 95 |
+
"raw_text": "带着手作的三明治,一头撞进这片绿色的风里🍃"
|
| 96 |
+
}}
|
| 97 |
+
],
|
| 98 |
+
"title": "想把自己寄给春天的风"
|
| 99 |
+
}}
|
prompts/tasks/generate_script/zh/user.md
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
用户的风格需求: {user_request}
|
| 2 |
+
整体素材概述:{overall}
|
| 3 |
+
|
| 4 |
+
分组输入开始:
|
| 5 |
+
{groups}
|
| 6 |
+
分组输入结束
|
| 7 |
+
|
| 8 |
+
输出格式如下:
|
| 9 |
+
```json
|
| 10 |
+
{{
|
| 11 |
+
"group_scripts": [
|
| 12 |
+
{{ "group_id": "group_0001", "raw_text": "第一句,第二句,第三句" }},
|
| 13 |
+
{{ "group_id": "group_0002", "raw_text": "第一句,第二句" }}
|
| 14 |
+
],
|
| 15 |
+
"title": "视频标题"
|
| 16 |
+
}}
|
| 17 |
+
```
|
prompts/tasks/generate_title/zh/system.md
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
你是一个短视频剪辑助理。你需要理解用户提出的需求,执行合适的工具完成剪辑,并避免向用户回复过于专业的剪辑术语。你会拿到一个关于剪辑的工具函数的描述列表。
|
| 2 |
+
如果用户是第一次提出“帮我剪辑/处理素材”等类似需求,请先用自然语言列出你计划如何使用给定的剪辑工具及理由,待用户确认。注意你只能使用你可用的剪辑工具,如果工具暂时不可用,请明确告诉用户你做不到。
|
| 3 |
+
整个剪辑流程中,有些节点是固定的,你无法改动;你计划的范围仅限于可以改动的节点。除非用户明确想要跳过某个步骤,否则在列出计划时,默认运行所有节点,以达到完美的效果。
|
| 4 |
+
注意,有些节点依赖前面节点的结果,具体的依赖关系你可以在工具描述中看到,请在工具调用前检查依赖。工具会自己寻找依赖的结果,你不需要将前面节点的结果输入到工具参数中。如果工具需要输入参数,会在工具描述中另加说明,请填入合适的参数。
|
| 5 |
+
你的每一次回答必须仅在调用工具和用自然语言回复用户(markdown格式)之间选择一个,每次只调用一个工具。每次调用完工具后,向用户简单总结本次工具调用的结果和下一步的意图,增强互动感。尽可能使用多的工具以丰富视频内容,除非用户明确指出不要某个元素。
|
| 6 |
+
|
| 7 |
+
常规剪辑流程如下,这里每一步都对应一个或多个工具供你使用:
|
| 8 |
+
第0步:素材索引 "assets_index"(固定)。用于获取输入素材的路径、长宽等基础信息。
|
| 9 |
+
第1步:镜头切分 "split_shots"(可跳过)。将素材按镜头切分成片段。
|
| 10 |
+
第2步:内容理解 "understand_clips"(可跳过)。 为每个片段(clips)生成一段描述(captions)
|
| 11 |
+
第3步:镜头筛选 "filter_clips"(可跳过)。根据用户要求,筛选出符合要求的片段(clips)
|
| 12 |
+
第4步:片段分组 "group_clips"(可跳过,但应默认运行)。根据用户要求,对片段进行排序和分组,组织合理的叙事逻辑,并辅助后续文案生成。
|
| 13 |
+
第5步:文案生成 "generate_script"(可跳过)。根据用户要求,生成视频文案。
|
| 14 |
+
第6步:元素推荐 (可跳过,但应默认运行)。根据用户要求,推荐花字、标题、特效、转场、配音音色等元素。
|
| 15 |
+
第7步:配音生成 "generate_voiceover"(可跳过)。根据文案生成对应的配音。
|
| 16 |
+
第8步:背景音乐选取 "select_BGM"(可跳过)。选择合适的背景音乐。
|
| 17 |
+
第9步:组织时间线 "plan_timeline"(固定)。根据前面的视频片段、文案、语音和BGM,组织成合理的时间线。
|
| 18 |
+
第10步:渲染成片。"render_video"(固定)。根据时间线渲染成片。
|
| 19 |
+
此外,虽然你在工具调用后只能看到summary,但你有一个工具可以读取任意中间节点的输出。你可以用它完成更复杂的任务。
|
prompts/tasks/generate_title/zh/user.md
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
现在根据下面的视频内容,生成 {n_titles} 个适合短视频平台的中文标题,每个不超过 20 字,要有吸引力但不要标题党过头。
|
| 2 |
+
|
| 3 |
+
【视频内容简介】
|
| 4 |
+
{summary}
|
| 5 |
+
|
| 6 |
+
请用 JSON 数组返回:
|
| 7 |
+
[标题1, 标题2, ...]
|
prompts/tasks/group_clips/zh/system.md
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 角色设置
|
| 2 |
+
你是一位拥有顶级叙事逻辑且**极具共情能力**的**资深视频剪辑导演**。你不仅擅长梳理故事线,更能**深度洞察用户意图**。你能够透过简单的文字指令,精准捕捉用户想要的叙事策略(如倒叙、插叙、情绪对比等),并将其转化为“起-承-转-合”的视频结构。
|
| 3 |
+
|
| 4 |
+
# 核心任务
|
| 5 |
+
你的任务分为两步:
|
| 6 |
+
1. **深度思考**:首先**彻底解析用户意图**,以此为核心规划叙事结构,并输出详细的思考过程。
|
| 7 |
+
2. **结构化输出**:根据思考结果,生成包含分组、合并与排序信息的标准 JSON 数据。
|
| 8 |
+
|
| 9 |
+
# 输入信息
|
| 10 |
+
1. **user_input**: 视频核心主题与指令(这是**最高叙事准则**,可能隐含了特殊的排序要求、情绪基调或特定情节)。
|
| 11 |
+
2. **captions**: 视频片段列表。
|
| 12 |
+
* 格式:`【编号】<time>时长</time><clip>组ID(可选)</clip>内容描述`
|
| 13 |
+
3. **segment_num**: 原始视频片段的总数量。
|
| 14 |
+
|
| 15 |
+
# 工作流与逻辑规则 (最高优先级)
|
| 16 |
+
|
| 17 |
+
在处理素材时,你必须**严格遵循**以下逻辑层级:
|
| 18 |
+
|
| 19 |
+
## 第一层:硬性物理约束(源素材锁定)
|
| 20 |
+
**(不可违背的物理基础)**
|
| 21 |
+
1. **锁定**:拥有相同 `<media>` 编号的片段(例如 `【0001】<media>A</media>` 和 `【0002】<media>A</media>`)视为**不可分割的整体**。
|
| 22 |
+
2. **规则**:它们必须作为一个整体移动,**内部顺序不可变**,中间不可插入其他片段。
|
| 23 |
+
|
| 24 |
+
## 第二层:用户意图与叙事重构(核心逻辑)
|
| 25 |
+
**(在此层级,必须优先响应用户的特殊指令,而非死板的时间逻辑)**
|
| 26 |
+
1. **意图优先原则**:若 `user_input` 暗示了特殊的叙事手法(例如:“先展示结果再回顾过程”=倒叙;“忙碌与宁静的对比”=对比蒙太奇),必须**无条件优先**满足该逻辑,哪怕打乱时间线也在所不惜。若`user_input`要求不改变素材顺序,则严格按照 `<media>`编号组织顺序。
|
| 27 |
+
2. **情绪/因果逻辑**:在符合用户意图的前提下,按情绪流(压抑→爆发)或因果流(问题→解决)排序。
|
| 28 |
+
3. **常规时空逻辑**:若用户无特殊指令,则兜底使用“起-承-转-合”的经典时空顺序(早→晚,外→内)。
|
| 29 |
+
* *起*:引入环境/悬念。
|
| 30 |
+
* *承*:铺陈细节/推进。
|
| 31 |
+
* *转*:高潮/反转。
|
| 32 |
+
* *合*:升华/收束。
|
| 33 |
+
|
| 34 |
+
## 第三层:分组与合并(叙事分段)
|
| 35 |
+
* **合并标准**:将服务于**同一用户意图**或**同一叙事节点**的片段合并为一个新的叙事组(Group)。
|
| 36 |
+
* **数量限制**:单组建议不超过 **4个** 片段(Clip组锁定导致的除外)。
|
| 37 |
+
|
| 38 |
+
# 输出规范
|
| 39 |
+
|
| 40 |
+
你需要严格按照以下两个步骤进行输出:
|
| 41 |
+
|
| 42 |
+
## 步骤一:思考过程
|
| 43 |
+
请严格使用以下标签格式输出你的分析过程,并注意字数限制:
|
| 44 |
+
|
| 45 |
+
* **思考过程**
|
| 46 |
+
【主题确定与分析】【内容开始】深度解读用户意图,明确视频是顺叙、倒叙还是情绪流?确定核心立意(**100字以内**)。【内容结束】
|
| 47 |
+
【叙事与结构重构】【内容开始】阐述如何**基于用户意图**重组素材。说明Clip组锁定情况。解释为何采用当前的排序策略(例如:“响应用户‘先抑后扬’的需求,将失败镜头前置...”)(**100字以内**)。【内容结束】
|
| 48 |
+
【核心主题】【内容开始】用一句话提炼视频的灵魂文案或中心思想(**50字以内**)。【内容结束】
|
| 49 |
+
|
| 50 |
+
## 步骤二:JSON 结果
|
| 51 |
+
基于思考过程,输出最终结构。
|
| 52 |
+
1. 必须使用严格json格式。
|
| 53 |
+
2. `clip_ids`:使用原始编号字符串列表。
|
| 54 |
+
3. **`media_id` 提取规则**:这是一个数组,必须与 `clip_ids` 中的元素**位置一一对应**。
|
| 55 |
+
* 如果对应的 clip 有 `<clip>ID</clip>` 标签,则填入数字或字符串 `ID`。
|
| 56 |
+
* 如果对应的 clip 没有该标签,则填入 `null`。
|
| 57 |
+
|
| 58 |
+
**JSON 结构定义:**
|
| 59 |
+
```json
|
| 60 |
+
{{
|
| 61 |
+
"groups": [
|
| 62 |
+
{{
|
| 63 |
+
"group_id": "group_序号 (例如 group_0001)",
|
| 64 |
+
"summary": "极具画面感的叙事或场景描述,解释该组镜头在故事中的作用(50字以内)。",
|
| 65 |
+
"media_id": [ ID1, ID2, null ],
|
| 66 |
+
"clip_ids": [
|
| 67 |
+
"clip_0001",
|
| 68 |
+
"clip_0002",
|
| 69 |
+
"clip_0003"
|
| 70 |
+
]
|
| 71 |
+
}}
|
| 72 |
+
]
|
| 73 |
+
}}
|
| 74 |
+
```
|
| 75 |
+
|
| 76 |
+
---
|
| 77 |
+
**示例输入:**
|
| 78 |
+
`user_input`: 无论最后结果如何,我们都曾努力过(要求:先展示失败的结果,再回忆过程)
|
| 79 |
+
`captions`:
|
| 80 |
+
`【clip_0001】<media>1</media>大家抱在一起痛哭`
|
| 81 |
+
`【clip_0002】<media>1</media>看着比分板发呆`
|
| 82 |
+
`【clip_0003】<media>2</media>深夜训练`
|
| 83 |
+
`【clip_0004】<media>3</media>受伤包扎`
|
| 84 |
+
`【clip_0005】互相打气` (注意:此条无clip标签)
|
| 85 |
+
|
| 86 |
+
**示例输出:**
|
| 87 |
+
|
| 88 |
+
**思考过程**
|
| 89 |
+
【主题确定与分析】【内容开始】用户明确要求“倒叙”结构,强调“过程的意义”。基调是遗憾但热血。核心是展现虽然结果失败��起),但奋斗过程(承转)依然闪光。【内容结束】
|
| 90 |
+
【叙事与结构重构】【内容开始】Clip 1组【0001, 0002】被物理锁定,虽是时间线终点,但根据用户“先展示结果”的意图,必须作为开篇“起”。随后进入回忆,“承”合并【0003, 0004】展示艰辛,“合”用无标签的【0005】互相打气收尾,升华主题。【内容结束】
|
| 91 |
+
【核心主题】【内容开始】输赢只是瞬间,但并肩战斗的那些日夜,才是我们最宝贵的勋章。【内容结束】
|
| 92 |
+
|
| 93 |
+
【合并及排序结果】
|
| 94 |
+
```json
|
| 95 |
+
{{
|
| 96 |
+
"groups": [
|
| 97 |
+
{{
|
| 98 |
+
"group_id": "group_0001",
|
| 99 |
+
"summary": "响应倒叙要求,以赛后失利的痛哭与迷茫开篇,制造遗憾的情绪悬念,引出对过往的回忆。",
|
| 100 |
+
"media_id": [1, 1],
|
| 101 |
+
"clip_ids": [
|
| 102 |
+
"clip_0001",
|
| 103 |
+
"clip_0002"
|
| 104 |
+
]
|
| 105 |
+
}},
|
| 106 |
+
{{
|
| 107 |
+
"group_id": "group_0002",
|
| 108 |
+
"summary": "镜头回溯至备战期间,展示深夜训练与受伤坚持的画面,体现团队付出的极致努力。",
|
| 109 |
+
"media_id": [2, 3],
|
| 110 |
+
"clip_ids": [
|
| 111 |
+
"clip_0003",
|
| 112 |
+
"clip_0004"
|
| 113 |
+
]
|
| 114 |
+
}},
|
| 115 |
+
{{
|
| 116 |
+
"group_id": "group_0003",
|
| 117 |
+
"summary": "以赛前互相打气的温馨瞬间作结,强调拼搏本身比结果更重要,升华“努力过”的主题。",
|
| 118 |
+
"media_id": [null],
|
| 119 |
+
"clip_ids": [
|
| 120 |
+
"0005"
|
| 121 |
+
]
|
| 122 |
+
}}
|
| 123 |
+
]
|
| 124 |
+
}}
|
| 125 |
+
```
|
prompts/tasks/group_clips/zh/user.md
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
用户要求: {user_request}
|
| 2 |
+
注意clips编号可能有跳跃,以下是可用的clips:{selected_clips}
|
| 3 |
+
|
| 4 |
+
以下是clips详细信息,请根据用户要求,对clips进行分组:
|
| 5 |
+
{clip_captions}
|
| 6 |
+
你的输出格式如下:
|
| 7 |
+
```json
|
| 8 |
+
{{
|
| 9 |
+
"groups": [
|
| 10 |
+
{{
|
| 11 |
+
"group_id": "group_0001",
|
| 12 |
+
"summary": "先用最安静、最治愈的镜头建立情绪。",
|
| 13 |
+
"clip_ids": ["clip_0003", "clip_0001"],
|
| 14 |
+
}},
|
| 15 |
+
{{
|
| 16 |
+
"group_id": "group_0002",
|
| 17 |
+
"summary": "用更具画面感的镜头承接金句。",
|
| 18 |
+
"clip_ids": ["clip_0011", "clip_0014"],
|
| 19 |
+
}}
|
| 20 |
+
]
|
| 21 |
+
}}
|
| 22 |
+
```
|
prompts/tasks/instruction/zh/system.md
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
你是一个短视频剪辑助理。你需要理解用户提出的需求,执行合适的工具完成剪辑,并避免向用户回复过于专业的剪辑术语。你会拿到一个关于剪辑的工具函数的描述列表。
|
| 2 |
+
如果用户是第一次提出“帮我剪辑/处理素材”等类似需求,请先用自然语言列出你计划如何使用给定的剪辑工具及理由,待用户确认。注意你只能使用你可用的剪辑工具,如果工具暂时不可用,请明确告诉用户你做不到。
|
| 3 |
+
当用户指定一个剪辑风格时,优先查看工具里面是否有满足的技能(描述以【SKILL】为开头的工具),如果有匹配的技能,优先使用。
|
| 4 |
+
整个剪辑流程中,有些节点是固定的,你无法改动;你计划的范围仅限于可以改动的节点。除非用户明确想要跳过某个步骤,否则在列出计划时,默认运行所有节点,以达到完美的效果。
|
| 5 |
+
注意,有些节点依赖前面节点的结果,具体的依赖关系你可以在工具描述中看到,请在工具调用前检查依赖。工具会自己寻找依赖的结果,你不需要将前面节点的结果输入到工具参数中。如果工具需要输入参数,会在工具描述中另加说明,请填入合适的参数。
|
| 6 |
+
你的每一次回答必须仅在调用工具和用自然语言回复用户(markdown格式)之间选择一个,每次只调用一个工具。每次调用完工具后,向用户简单总结本次工具调用的结果和下一步的意图,增强互动感。尽可能使用多的工具以丰富视频内容,除非用户明确指出不要某个元素。
|
| 7 |
+
|
| 8 |
+
常规剪辑流程如下,这里每一步都对应一个或多个工具供你使用:
|
| 9 |
+
第0步:素材索引 "assets_index"(固定)。用于获取输入素材的路径、长宽等基础信息。
|
| 10 |
+
第1步:镜头切分 "split_shots"(可跳过)。将素材按镜头切分成片段。
|
| 11 |
+
第2步:内容理解 "understand_clips"(可跳过)。 为每个片段(clips)生成一段描述(captions)
|
| 12 |
+
第3步:镜头筛选 "filter_clips"(可跳过)。根据用户要求,筛选出符合要求的片段(clips)
|
| 13 |
+
第4步:片段分组 "group_clips"(可跳过,但应默认运行)。根据用户要求,对片段进行排序和分组,组织合理的叙事逻辑,并辅助后续文案生成。
|
| 14 |
+
第5步:文案生成 "generate_script"(可跳过)。根据用户要求,生成视频文案。
|
| 15 |
+
第6步:元素推荐 (可跳过,但应默认运行)。根据用户要求,推荐花字、标题、特效、转场、配音音色等元素。
|
| 16 |
+
第7步:配音生成 "generate_voiceover"(可跳过)。根据文案生成对应的配音。
|
| 17 |
+
第8步:背景音乐选取 "select_BGM"(可跳过)。选择合适的背景音乐。
|
| 18 |
+
第9步:组织时间线 "plan_timeline"(固定)。根据前面的视频片段、文案、语音和BGM,组织成合理的时间线。
|
| 19 |
+
第10步:渲染成片。"render_video"(固定)。根据时间线渲染成片。
|
| 20 |
+
此外,虽然你在工具调用后只能看到summary,但你有一个工具可以读取任意中间节点的输出。你可以用它完成更复杂的任务。
|
| 21 |
+
|
| 22 |
+
风格要求:
|
| 23 |
+
- 用简洁、口语化的语言
|
| 24 |
+
|
| 25 |
+
当你需要调用剪辑工具时,请只输出调用信息。
|
| 26 |
+
|
| 27 |
+
当你已经拿到足够的信息,不需要再调用工具时,直接用自然语言回答客户,不要再输出JSON
|
| 28 |
+
|
| 29 |
+
下面是几个示例,模仿这种风格:
|
| 30 |
+
|
| 31 |
+
示例 1:列出计划
|
| 32 |
+
[用户]:
|
| 33 |
+
我想要你帮我把我的素材剪辑成旅行vlog,
|
| 34 |
+
[助手]:
|
| 35 |
+
我将要使用如下流程进行剪辑:
|
| 36 |
+
1. 素材索引:需要先对输入素材建立索引,提取元信息。
|
| 37 |
+
2. 镜头切分:将素材按镜头切分成片段。
|
| 38 |
+
3. 内容理解:对每个镜头生成文字描述。
|
| 39 |
+
4. 镜头筛选:筛选出适合放在旅行vlog中的镜头。
|
| 40 |
+
5. 片段分组:组织一个合理的素材顺序。
|
| 41 |
+
6. 文案生成:配一段符合主题的文案。
|
| 42 |
+
7. 花字检索:添加花字元素,增强视觉效果。
|
| 43 |
+
8. 标题花字检索:添加标题元素,提升视频吸引力。
|
| 44 |
+
9. 全局特效检索。添加适合日常vlog的全局特效/滤镜。
|
| 45 |
+
10. 转场检索:添加合适的转场效果,让镜头切换更自然。
|
| 46 |
+
11. 闭幕特效检索:添加合适的闭幕效果,让视频结束更自然。
|
| 47 |
+
12. 滤镜检索:添加合适的滤镜,丰富视觉体验。
|
| 48 |
+
13. 花字动画:添加合适的花字动画,丰富字幕效果。
|
| 49 |
+
14. 片头检索:添加合适的片头,增强视频吸引力。
|
| 50 |
+
15. 配音音色检索:选择合适的配音音色。
|
| 51 |
+
16. 配音生成:根据文案生成配音。
|
| 52 |
+
17. 音乐推荐:选择轻松愉快的vlog背景音乐。
|
| 53 |
+
18. 组织时间线:这是剪辑的固定流程。
|
| 54 |
+
19. 渲染成片:这也是剪辑的固定流程。
|
| 55 |
+
|
| 56 |
+
示例 2:不需要工具时,直接回答
|
| 57 |
+
用户:
|
| 58 |
+
什么是“镜头切分”?
|
| 59 |
+
助手:
|
| 60 |
+
镜头切分是指把原始视频按照画面内容或语义边界切分成若干个独立的镜头片段,通常会结合画面变化、音频变化等特征来判断切分点,用于后续剪辑、检索或分析。
|
| 61 |
+
|
| 62 |
+
示例3:二次编辑
|
| 63 |
+
用户:
|
| 64 |
+
之前你给我剪的视频有配音,我现在不想要视频的配音了。
|
| 65 |
+
|
| 66 |
+
此时助手需要执行配音工具,参数mode选择skip。
|
prompts/tasks/understand_clips/zh/system_detail.md
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
你是一个短视频剪辑素材理解助手。
|
| 2 |
+
你的任务是:根据给定的视频或图片素材,为剪辑流程提供一段简洁、准确的中文描述,重点包含:主体内容、人物或物体、场景和整体氛围,方便后续做高光检测和文案生成。
|
| 3 |
+
不要输出客套话或多余说明。你必须只输出一个合法 JSON 对象,不要输出任何额外文字
|
prompts/tasks/understand_clips/zh/system_overall.md
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
你是一个短视频剪辑素材理解助手。
|
| 2 |
+
你的任务是:根据给定的多个素材的文案描述,生成一段总体描述。重点包含:主体内容、人物或物体、场景和整体氛围,方便后续做高光检测和文案生成。不要输出客套话或多余说明。
|
prompts/tasks/understand_clips/zh/user_detail.md
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
请为这个 clip 生成中文的描述与语义标签。
|
| 2 |
+
输出必须严格符合以下 JSON 结构(key 不可缺失)
|
| 3 |
+
```json
|
| 4 |
+
{{
|
| 5 |
+
"caption": "2-4句描述(尽量具体客观,不要编造看不到的细节)"
|
| 6 |
+
}}
|
| 7 |
+
```
|
prompts/tasks/understand_clips/zh/user_overall.md
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
下面是若干clips的单体描述,请你从整体角度总结:这些素材大致讲述了怎样的故事?请用 1~2 段话总结,并保持客观、中性。
|
| 2 |
+
{clips_captions}
|
requirements.txt
CHANGED
|
@@ -1,2 +1,13 @@
|
|
| 1 |
fastapi
|
| 2 |
-
uvicorn
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
fastapi
|
| 2 |
+
uvicorn
|
| 3 |
+
langchain_core
|
| 4 |
+
mcp
|
| 5 |
+
colorlog
|
| 6 |
+
librosa
|
| 7 |
+
transnetv2_pytorch
|
| 8 |
+
langchain
|
| 9 |
+
langchain_mcp_adapters
|
| 10 |
+
langchain_openai
|
| 11 |
+
ffmpeg-python
|
| 12 |
+
aiofiles
|
| 13 |
+
skillkit
|
run.sh
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -e
|
| 3 |
+
|
| 4 |
+
ROOT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
| 5 |
+
export PYTHONPATH="$ROOT_DIR/src"
|
| 6 |
+
|
| 7 |
+
HOST="${HOST:-0.0.0.0}"
|
| 8 |
+
PORT="${PORT:-7860}"
|
| 9 |
+
|
| 10 |
+
python3 -m open_storyline.mcp.server &
|
| 11 |
+
MCP_PID=$!
|
| 12 |
+
|
| 13 |
+
uvicorn agent_fastapi:app \
|
| 14 |
+
--host "$HOST" \
|
| 15 |
+
--port "$PORT" &
|
| 16 |
+
WEB_PID=$!
|
| 17 |
+
|
| 18 |
+
trap 'kill $MCP_PID $WEB_PID' INT TERM
|
| 19 |
+
|
| 20 |
+
wait
|
src/__init__.py
ADDED
|
File without changes
|
src/open_storyline/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
__version__ = "0.1.0"
|
src/open_storyline/__main__.py
ADDED
|
File without changes
|
src/open_storyline/config.py
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# open_storyline/configuration_utils.py
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
import os
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from typing import Any, Optional, Literal
|
| 6 |
+
import time
|
| 7 |
+
|
| 8 |
+
try:
|
| 9 |
+
import tomllib
|
| 10 |
+
except ImportError:
|
| 11 |
+
print("Fail to import tomllib, try to import tomlis")
|
| 12 |
+
import tomli as tomllib
|
| 13 |
+
|
| 14 |
+
from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, computed_field, field_validator
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def _resolve_relative_path_to_config_dir(v: Path, info: ValidationInfo) -> Path:
|
| 18 |
+
"""
|
| 19 |
+
把相对路径按 config.toml 所在目录解析(而不是按 cwd)。
|
| 20 |
+
|
| 21 |
+
依赖调用方在 model_validate(..., context={"config_dir": <Path|str>}) 里传入 config_dir。
|
| 22 |
+
"""
|
| 23 |
+
ctx = info.context or {}
|
| 24 |
+
base = ctx.get("config_dir")
|
| 25 |
+
if not base:
|
| 26 |
+
return v
|
| 27 |
+
|
| 28 |
+
v2 = v.expanduser()
|
| 29 |
+
if v2.is_absolute():
|
| 30 |
+
return v2
|
| 31 |
+
|
| 32 |
+
base_dir = Path(base).expanduser()
|
| 33 |
+
return (base_dir / v2).resolve(strict=False)
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def _resolve_paths_recursively(value: Any, info: ValidationInfo) -> Any:
|
| 37 |
+
"""
|
| 38 |
+
递归处理容器类型里的 Path(list/tuple/set/dict)。
|
| 39 |
+
"""
|
| 40 |
+
if value is None:
|
| 41 |
+
return None
|
| 42 |
+
|
| 43 |
+
if isinstance(value, Path):
|
| 44 |
+
return _resolve_relative_path_to_config_dir(value, info)
|
| 45 |
+
|
| 46 |
+
if isinstance(value, list):
|
| 47 |
+
return [_resolve_paths_recursively(v, info) for v in value]
|
| 48 |
+
|
| 49 |
+
if isinstance(value, tuple):
|
| 50 |
+
return tuple(_resolve_paths_recursively(v, info) for v in value)
|
| 51 |
+
|
| 52 |
+
if isinstance(value, set):
|
| 53 |
+
return {_resolve_paths_recursively(v, info) for v in value}
|
| 54 |
+
|
| 55 |
+
if isinstance(value, dict):
|
| 56 |
+
return {k: _resolve_paths_recursively(v, info) for k, v in value.items()}
|
| 57 |
+
|
| 58 |
+
return value
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
class ConfigBaseModel(BaseModel):
|
| 62 |
+
model_config = ConfigDict(extra="forbid")
|
| 63 |
+
|
| 64 |
+
@field_validator("*", mode="after")
|
| 65 |
+
@classmethod
|
| 66 |
+
def _resolve_all_path_fields(cls, v: Any, info: ValidationInfo) -> Any:
|
| 67 |
+
# 允许对某些字段显式禁用路径解析:
|
| 68 |
+
# Field(..., json_schema_extra={"resolve_relative": False})
|
| 69 |
+
if info.field_name:
|
| 70 |
+
field = cls.model_fields.get(info.field_name)
|
| 71 |
+
extra = (field.json_schema_extra or {}) if field else {}
|
| 72 |
+
if extra.get("resolve_relative") is False:
|
| 73 |
+
return v
|
| 74 |
+
|
| 75 |
+
return _resolve_paths_recursively(v, info)
|
| 76 |
+
|
| 77 |
+
class DeveloperConfig(ConfigBaseModel):
|
| 78 |
+
developer_mode: bool = False
|
| 79 |
+
chat_models: list[str] = Field(default_factory=list)
|
| 80 |
+
chat_models_config: dict[str, dict[str, Any]] = Field(default_factory=dict)
|
| 81 |
+
print_context: bool = False
|
| 82 |
+
|
| 83 |
+
class ProjectConfig(ConfigBaseModel):
|
| 84 |
+
assets_dir: Path = Field(..., description="素材目录(输入视频、图片)")
|
| 85 |
+
bgm_dir: Path = Field(..., description="背景音乐目录")
|
| 86 |
+
outputs_dir: Path = Field(..., description="输出目录")
|
| 87 |
+
debug_mode: bool = Field(True, description="是否启用debug模式")
|
| 88 |
+
|
| 89 |
+
@computed_field(return_type=Path)
|
| 90 |
+
@property
|
| 91 |
+
def blobs_dir(self) -> Path:
|
| 92 |
+
return self.outputs_dir
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
class LLMConfig(ConfigBaseModel):
|
| 96 |
+
model: str
|
| 97 |
+
base_url: str
|
| 98 |
+
api_key: str
|
| 99 |
+
timeout: float = 30.0
|
| 100 |
+
temperature: Optional[float] = None
|
| 101 |
+
max_retries: int = 2
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
class VLMConfig(ConfigBaseModel):
|
| 105 |
+
model: str
|
| 106 |
+
base_url: str
|
| 107 |
+
api_key: str
|
| 108 |
+
timeout: float = 20.0
|
| 109 |
+
temperature: Optional[float] = None
|
| 110 |
+
max_retries: int = 2
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
class MCPConfig(ConfigBaseModel):
|
| 114 |
+
server_name: str = "storyline"
|
| 115 |
+
server_cache_dir: str = "./storyline/.server_cache"
|
| 116 |
+
server_transport: Literal["stdio", "sse", "streamable-http"] = "streamable-http"
|
| 117 |
+
url_scheme: str = "http"
|
| 118 |
+
connect_host: str = "127.0.0.1"
|
| 119 |
+
port: int = Field(ge=1, le=65535)
|
| 120 |
+
path: str = "/mcp"
|
| 121 |
+
|
| 122 |
+
json_response: bool = True
|
| 123 |
+
stateless_http: bool = False
|
| 124 |
+
|
| 125 |
+
timeout: int = 600
|
| 126 |
+
|
| 127 |
+
use_xhs_node_only: bool = True
|
| 128 |
+
@property
|
| 129 |
+
def url(self) -> str:
|
| 130 |
+
return f"{self.url_scheme}://{self.connect_host}:{self.port}{self.path}"
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
class SplitShotsConfig(ConfigBaseModel):
|
| 134 |
+
transnet_weights: Path = Field(..., description="transnet_v2 权重路径")
|
| 135 |
+
transnet_device: str = "cpu"
|
| 136 |
+
|
| 137 |
+
class UnderstandClipsConfig(ConfigBaseModel):
|
| 138 |
+
sample_fps: float = 2.0
|
| 139 |
+
max_frames: int = 64
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
class GenerateVoiceoverConfig(ConfigBaseModel):
|
| 143 |
+
base_url: str = ""
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
class SelectBGMConfig(ConfigBaseModel):
|
| 147 |
+
sample_rate: int = 22050
|
| 148 |
+
hop_length: int = 2048
|
| 149 |
+
frame_length: int = 2048
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
class PlanTimelineConfig(ConfigBaseModel):
|
| 153 |
+
beat_type_max: int = 1 # 使用多强的鼓点,例如4/4中,鼓点类似1,2,1,3,其中1是最强的,3最弱
|
| 154 |
+
title_dutation: int = 5000 # 片头时长
|
| 155 |
+
bgm_loop: bool = True # 是否允许 bgm 循环
|
| 156 |
+
min_clip_duration: int = 500
|
| 157 |
+
|
| 158 |
+
estimate_text_min: int = 1500 # 在没有TTS的情况下,每组字幕至少上屏多久
|
| 159 |
+
estimate_text_char_per_sec: float = 6.0 # 在没有TTS的情况下,估计每秒展示几个字
|
| 160 |
+
|
| 161 |
+
image_default_duration: int = 3000 # 默认的图片���放时长
|
| 162 |
+
|
| 163 |
+
group_margin_over_voiceover: int = 1000 # 在一个group中,画面比配音多出现多久
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
class Settings(ConfigBaseModel):
|
| 167 |
+
developer: DeveloperConfig
|
| 168 |
+
project: ProjectConfig
|
| 169 |
+
|
| 170 |
+
llm: LLMConfig
|
| 171 |
+
vlm: VLMConfig
|
| 172 |
+
|
| 173 |
+
local_mcp_server: MCPConfig
|
| 174 |
+
xhs_mcp_server: MCPConfig
|
| 175 |
+
|
| 176 |
+
split_shots: SplitShotsConfig
|
| 177 |
+
understand_clips: UnderstandClipsConfig
|
| 178 |
+
generate_voiceover: GenerateVoiceoverConfig
|
| 179 |
+
select_bgm: SelectBGMConfig
|
| 180 |
+
plan_timeline: PlanTimelineConfig
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def load_settings(config_path: str | Path) -> Settings:
|
| 184 |
+
p = Path(config_path).expanduser().resolve()
|
| 185 |
+
data = tomllib.loads(p.read_text(encoding="utf-8"))
|
| 186 |
+
return Settings.model_validate(data, context={"config_dir": p.parent})
|
| 187 |
+
|
| 188 |
+
def default_config_path() -> str:
|
| 189 |
+
return os.getenv("OPENSTORYLINE_CONFIG", "config.toml")
|
src/open_storyline/mcp/__init__.py
ADDED
|
File without changes
|
src/open_storyline/mcp/hooks/node_interceptors.py
ADDED
|
@@ -0,0 +1,299 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from collections import defaultdict
|
| 2 |
+
from typing import List, Any, Dict
|
| 3 |
+
import os
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
import json
|
| 6 |
+
import traceback
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
from langchain_mcp_adapters.interceptors import MCPToolCallRequest
|
| 10 |
+
from langgraph.types import Command
|
| 11 |
+
from langchain_core.messages import ToolMessage, ToolCall
|
| 12 |
+
from langchain_core.tools import ToolException
|
| 13 |
+
from mcp.types import CallToolResult
|
| 14 |
+
|
| 15 |
+
from open_storyline.nodes.node_manager import NodeManager
|
| 16 |
+
|
| 17 |
+
from open_storyline.storage.file import FileCompressor
|
| 18 |
+
from open_storyline.utils.logging import get_logger
|
| 19 |
+
|
| 20 |
+
logger = get_logger(__name__)
|
| 21 |
+
|
| 22 |
+
def compress_payload_to_base64(payload: Dict[str,List[Any]]):
|
| 23 |
+
if not isinstance(payload, dict):
|
| 24 |
+
return payload
|
| 25 |
+
for key,value in payload.items():
|
| 26 |
+
if isinstance(value, list) and all([isinstance(item, dict) for item in value]):
|
| 27 |
+
for item in value:
|
| 28 |
+
if 'path' in item.keys():
|
| 29 |
+
path = item['path']
|
| 30 |
+
compress_data = FileCompressor.compress_and_encode(path)
|
| 31 |
+
|
| 32 |
+
item.update({
|
| 33 |
+
"path": path,
|
| 34 |
+
"base64": compress_data.base64,
|
| 35 |
+
"md5": compress_data.md5
|
| 36 |
+
})
|
| 37 |
+
|
| 38 |
+
# 如果只是调用本地节点记得特殊处理,不再发送base64
|
| 39 |
+
class ToolInterceptor:
|
| 40 |
+
|
| 41 |
+
@staticmethod
|
| 42 |
+
async def inject_media_content_before(
|
| 43 |
+
request: MCPToolCallRequest,
|
| 44 |
+
handler,
|
| 45 |
+
):
|
| 46 |
+
try:
|
| 47 |
+
tool_call_type = request.args.get('tool_call_type', 'auto')
|
| 48 |
+
# for default tool call
|
| 49 |
+
if tool_call_type!= 'auto':
|
| 50 |
+
request.args = request.args.get('args', {})
|
| 51 |
+
|
| 52 |
+
runtime = request.runtime
|
| 53 |
+
context = runtime.context
|
| 54 |
+
store = runtime.store
|
| 55 |
+
session_id = context.session_id
|
| 56 |
+
node_id = request.name
|
| 57 |
+
artifact_id = store.generate_artifact_id(node_id)
|
| 58 |
+
meta_collector: NodeManager = context.node_manager
|
| 59 |
+
tool_call_id = request.runtime.tool_call_id
|
| 60 |
+
input_data = defaultdict(list)
|
| 61 |
+
|
| 62 |
+
def load_collected_data(collected_node, input_data, store):
|
| 63 |
+
"""加载收集到的节点数据"""
|
| 64 |
+
for collect_kind, artifact_meta in collected_node.items():
|
| 65 |
+
_, prior_node_output = store.load_result(artifact_meta.artifact_id)
|
| 66 |
+
compress_payload_to_base64(prior_node_output['payload'])
|
| 67 |
+
input_data[collect_kind] = prior_node_output['payload']
|
| 68 |
+
|
| 69 |
+
if node_id == 'assets_index':
|
| 70 |
+
input_data['inputs'] = []
|
| 71 |
+
assets_dir = Path(context.assets_dir)
|
| 72 |
+
for file_name in os.listdir(assets_dir):
|
| 73 |
+
path = assets_dir / file_name
|
| 74 |
+
if path.is_dir():
|
| 75 |
+
continue
|
| 76 |
+
compress_data = FileCompressor.compress_and_encode(path)
|
| 77 |
+
input_data['inputs'].append(
|
| 78 |
+
{
|
| 79 |
+
"path": str(path.relative_to(os.getcwd())),
|
| 80 |
+
"base64": compress_data.base64,
|
| 81 |
+
"md5": compress_data.md5,
|
| 82 |
+
}
|
| 83 |
+
)
|
| 84 |
+
elif node_id in list(meta_collector.id_to_tool.keys()):
|
| 85 |
+
# 1. 确定执行模式和依赖需求
|
| 86 |
+
is_skip_mode = request.args.get('mode', 'auto') != 'auto'
|
| 87 |
+
require_kind = (
|
| 88 |
+
meta_collector.id_to_default_require_prior_kind[node_id]
|
| 89 |
+
if is_skip_mode
|
| 90 |
+
else meta_collector.id_to_require_prior_kind[node_id]
|
| 91 |
+
)
|
| 92 |
+
|
| 93 |
+
# 2. 检查节点是否可执行
|
| 94 |
+
collect_result = meta_collector.check_excutable(session_id, store, require_kind)
|
| 95 |
+
load_collected_data(collect_result['collected_node'], input_data, store)
|
| 96 |
+
|
| 97 |
+
# 3. 处理依赖缺失情况
|
| 98 |
+
if not collect_result['excutable']:
|
| 99 |
+
missing_kinds = collect_result['missing_kind']
|
| 100 |
+
node_ids_missing = [
|
| 101 |
+
meta_collector.kind_to_node_ids[kind][0]
|
| 102 |
+
for kind in missing_kinds
|
| 103 |
+
]
|
| 104 |
+
|
| 105 |
+
logger.info(
|
| 106 |
+
f"`{node_id}` require kind missing `{missing_kinds}`, "
|
| 107 |
+
f"need to execute prerequisite nodes: {node_ids_missing}"
|
| 108 |
+
)
|
| 109 |
+
|
| 110 |
+
# 4. 递归执行缺失的前序节点
|
| 111 |
+
async def execute_missing_dependencies(
|
| 112 |
+
missing_kinds: List[str],
|
| 113 |
+
for_node_id: str,
|
| 114 |
+
depth: int = 0
|
| 115 |
+
):
|
| 116 |
+
"""
|
| 117 |
+
递归执行缺失的依赖节点
|
| 118 |
+
|
| 119 |
+
Args:
|
| 120 |
+
missing_kinds: 缺失的依赖类型列表
|
| 121 |
+
for_node_id: 当前正在解决依赖的节点ID
|
| 122 |
+
depth: 递归深度(用于日志缩进)
|
| 123 |
+
"""
|
| 124 |
+
if not missing_kinds:
|
| 125 |
+
return
|
| 126 |
+
|
| 127 |
+
indent = " " * depth
|
| 128 |
+
logger.info(f"{indent}├─ Resolving dependencies for `{for_node_id}`: {missing_kinds}")
|
| 129 |
+
|
| 130 |
+
for kind in missing_kinds:
|
| 131 |
+
success = False
|
| 132 |
+
candidates = meta_collector.kind_to_node_ids[kind]
|
| 133 |
+
|
| 134 |
+
for miss_id in candidates:
|
| 135 |
+
try:
|
| 136 |
+
await execute_node_with_default_mode(
|
| 137 |
+
miss_id,
|
| 138 |
+
for_node_id=for_node_id,
|
| 139 |
+
depth=depth
|
| 140 |
+
)
|
| 141 |
+
logger.info(
|
| 142 |
+
f"{indent}│ ✓ `{miss_id}` executed successfully for kind `{kind}`"
|
| 143 |
+
)
|
| 144 |
+
success = True
|
| 145 |
+
break
|
| 146 |
+
except ToolException as e:
|
| 147 |
+
logger.warning(
|
| 148 |
+
f"{indent}│ ✗ `{miss_id}` failed: {str(e)}"
|
| 149 |
+
)
|
| 150 |
+
continue
|
| 151 |
+
|
| 152 |
+
if not success:
|
| 153 |
+
raise ToolException(
|
| 154 |
+
f"Cannot satisfy dependency `{kind}` required by `{for_node_id}`. "
|
| 155 |
+
f"All candidates failed: {candidates}"
|
| 156 |
+
)
|
| 157 |
+
|
| 158 |
+
async def execute_node_with_default_mode(
|
| 159 |
+
miss_id: str,
|
| 160 |
+
for_node_id: str,
|
| 161 |
+
depth: int = 0
|
| 162 |
+
):
|
| 163 |
+
"""
|
| 164 |
+
以default模式执行指定节点
|
| 165 |
+
|
| 166 |
+
Args:
|
| 167 |
+
miss_id: 要执行的节点ID
|
| 168 |
+
for_node_id: 请求该节点的父节点ID
|
| 169 |
+
depth: 递归深度
|
| 170 |
+
"""
|
| 171 |
+
indent = " " * depth
|
| 172 |
+
logger.info(
|
| 173 |
+
f"{indent}├─ [Default Mode] Executing `{miss_id}` "
|
| 174 |
+
f"(required by `{for_node_id}`)"
|
| 175 |
+
)
|
| 176 |
+
|
| 177 |
+
# 准备工具调用参数
|
| 178 |
+
tool = meta_collector.get_tool(miss_id)
|
| 179 |
+
tool_call_input = {
|
| 180 |
+
'artifact_id': store.generate_artifact_id(miss_id),
|
| 181 |
+
'mode': 'default'
|
| 182 |
+
}
|
| 183 |
+
|
| 184 |
+
# 检查该节点的依赖
|
| 185 |
+
default_require = meta_collector.id_to_default_require_prior_kind[miss_id]
|
| 186 |
+
default_collect_result = meta_collector.check_excutable(
|
| 187 |
+
session_id, store, default_require
|
| 188 |
+
)
|
| 189 |
+
default_collect_result = meta_collector.check_excutable(session_id, store, default_require)
|
| 190 |
+
# 递归处理依赖
|
| 191 |
+
if default_collect_result['excutable']:
|
| 192 |
+
load_collected_data(
|
| 193 |
+
default_collect_result['collected_node'],
|
| 194 |
+
tool_call_input,
|
| 195 |
+
store
|
| 196 |
+
)
|
| 197 |
+
logger.debug(f"{indent}│ Dependencies satisfied for `{miss_id}`")
|
| 198 |
+
else:
|
| 199 |
+
logger.info(
|
| 200 |
+
f"{indent}│ `{miss_id}` has missing dependencies: "
|
| 201 |
+
f"{default_collect_result['missing_kind']}"
|
| 202 |
+
)
|
| 203 |
+
await execute_missing_dependencies(
|
| 204 |
+
default_collect_result['missing_kind'],
|
| 205 |
+
for_node_id=miss_id, # 这里传入 miss_id
|
| 206 |
+
depth=depth + 1 # 增加递归深度
|
| 207 |
+
)
|
| 208 |
+
|
| 209 |
+
# 执行工具
|
| 210 |
+
try:
|
| 211 |
+
output = await tool.arun(
|
| 212 |
+
ToolCall(
|
| 213 |
+
args=tool_call_input,
|
| 214 |
+
tool_call_type='default',
|
| 215 |
+
runtime=runtime
|
| 216 |
+
)
|
| 217 |
+
)
|
| 218 |
+
logger.info(f"{indent}└─ ✓ `{miss_id}` completed successfully")
|
| 219 |
+
return output
|
| 220 |
+
except Exception as e:
|
| 221 |
+
logger.error(f"{indent}└─ ✗ `{miss_id}` execution failed: {str(e)}")
|
| 222 |
+
raise ToolException(f"Failed to execute `{miss_id}`: {str(e)}")
|
| 223 |
+
|
| 224 |
+
# 开始执行缺失的依赖
|
| 225 |
+
await execute_missing_dependencies(missing_kinds, for_node_id=node_id)
|
| 226 |
+
|
| 227 |
+
# 再次收集依赖
|
| 228 |
+
collect_result = meta_collector.check_excutable(session_id, store, require_kind)
|
| 229 |
+
load_collected_data(collect_result['collected_node'], input_data, store)
|
| 230 |
+
else:
|
| 231 |
+
input_data['artifacts_dir'] = store.artifacts_dir
|
| 232 |
+
|
| 233 |
+
new_req_args = {
|
| 234 |
+
'artifact_id': artifact_id,
|
| 235 |
+
}
|
| 236 |
+
new_req_args.update(request.args)
|
| 237 |
+
new_req_args.update(input_data)
|
| 238 |
+
|
| 239 |
+
modified_request = request.override(
|
| 240 |
+
args=new_req_args
|
| 241 |
+
)
|
| 242 |
+
return await handler(modified_request)
|
| 243 |
+
except Exception as e:
|
| 244 |
+
logger.error("[ToolInterceptor]"+ "".join(traceback.format_exception(e)))
|
| 245 |
+
raise
|
| 246 |
+
|
| 247 |
+
@staticmethod
|
| 248 |
+
async def save_media_content_after(
|
| 249 |
+
request: MCPToolCallRequest,
|
| 250 |
+
handler,
|
| 251 |
+
):
|
| 252 |
+
result = ""
|
| 253 |
+
"""End agent run when task is marked complete."""
|
| 254 |
+
try:
|
| 255 |
+
tool_call_result: CallToolResult = await handler(request) # 捕获工具调用后的结果
|
| 256 |
+
client_ctx = request.runtime.context
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
result = tool_call_result.model_dump()
|
| 260 |
+
tool_result = json.loads(result['content'][0]['text'])
|
| 261 |
+
node_id = request.name
|
| 262 |
+
|
| 263 |
+
artifact_id = tool_result['artifact_id']
|
| 264 |
+
session_id = client_ctx.session_id
|
| 265 |
+
|
| 266 |
+
store = request.runtime.store
|
| 267 |
+
|
| 268 |
+
# store.save_result(session_id,node_id,tool_result)
|
| 269 |
+
store.save_result(
|
| 270 |
+
session_id,
|
| 271 |
+
node_id,
|
| 272 |
+
tool_result,
|
| 273 |
+
)
|
| 274 |
+
tool_call_id = request.runtime.tool_call_id
|
| 275 |
+
|
| 276 |
+
if node_id == 'read_node_history':
|
| 277 |
+
tool_excute_result = tool_result['tool_excute_result']
|
| 278 |
+
else:
|
| 279 |
+
tool_excute_result = {}
|
| 280 |
+
|
| 281 |
+
return Command(
|
| 282 |
+
update={
|
| 283 |
+
"messages": [
|
| 284 |
+
ToolMessage(content={
|
| 285 |
+
'summary': {
|
| 286 |
+
'node_summary': tool_result['summary'],
|
| 287 |
+
'tool_excute_result': tool_excute_result
|
| 288 |
+
},
|
| 289 |
+
'isError': tool_result['isError']
|
| 290 |
+
}, tool_call_id=tool_call_id)
|
| 291 |
+
],
|
| 292 |
+
"status": "done"
|
| 293 |
+
},
|
| 294 |
+
# goto="__end__", # switch other agent
|
| 295 |
+
)
|
| 296 |
+
except Exception as e:
|
| 297 |
+
logger.error("[ToolInterceptor]"+ "".join(traceback.format_exception(e)))
|
| 298 |
+
logger.error(f"Tool Call result: {result}")
|
| 299 |
+
raise
|
src/open_storyline/mcp/register_tools.py
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
from dataclasses import asdict
|
| 3 |
+
from typing import Annotated
|
| 4 |
+
from pydantic import BaseModel, Field
|
| 5 |
+
import inspect
|
| 6 |
+
import traceback
|
| 7 |
+
|
| 8 |
+
from open_storyline.config import Settings
|
| 9 |
+
from open_storyline.skills.skills_io import dump_skills
|
| 10 |
+
|
| 11 |
+
from open_storyline.nodes.core_nodes.base_node import BaseNode
|
| 12 |
+
from open_storyline.nodes.core_nodes.assets_index import AssetsIndexNode
|
| 13 |
+
from open_storyline.nodes.core_nodes.split_shots import SplitShotsNode
|
| 14 |
+
from open_storyline.nodes.core_nodes.understand_clips import UnderstandClipsNode
|
| 15 |
+
from open_storyline.nodes.core_nodes.filter_clips import FilterClipsNode
|
| 16 |
+
from open_storyline.nodes.core_nodes.group_clips import GroupClipsNode
|
| 17 |
+
from open_storyline.nodes.core_nodes.generate_script import GenerateScriptNode
|
| 18 |
+
from open_storyline.nodes.core_nodes.generate_voiceover import GenerateVoiceoverNode
|
| 19 |
+
from open_storyline.nodes.core_nodes.select_bgm import SelectBGMNode
|
| 20 |
+
from open_storyline.nodes.core_nodes.plan_timeline import PlanTimelineNode
|
| 21 |
+
from open_storyline.nodes.core_nodes.render_video import RenderVideoNode
|
| 22 |
+
|
| 23 |
+
# from open_storyline.xhs_nodes.textgen_node import XhsUnderstandClipsNode, XhsGroupClipNode, XhsGenerateScriptNode
|
| 24 |
+
# from open_storyline.xhs_nodes.elementrec_node import (
|
| 25 |
+
# ElementrecTextsNode,
|
| 26 |
+
# ElementrecTitleNode,
|
| 27 |
+
# ElementrecGlobalEffectsNode,
|
| 28 |
+
# ElementrecLocalEffectsNode,
|
| 29 |
+
# ElementrecCloseEffectsNode,
|
| 30 |
+
# ElementrecFilterNode,
|
| 31 |
+
# ElementrecTTSSpeakerNode,
|
| 32 |
+
# ElementrecTransitionNode,
|
| 33 |
+
# ElementrecTextAnimationNode,
|
| 34 |
+
# ElementrecTitleClipNode,
|
| 35 |
+
# )
|
| 36 |
+
# from open_storyline.xhs_nodes.tts_node import FireTTSNode
|
| 37 |
+
# from open_storyline.xhs_nodes.music_select_node import MusicRecNode
|
| 38 |
+
# from open_storyline.xhs_nodes.timeline_node import TimelineNode
|
| 39 |
+
# from open_storyline.xhs_nodes.render_template_node import TemplateRenderNode
|
| 40 |
+
|
| 41 |
+
# from open_storyline.xhs_nodes.xhs_detect_highlights import XhsDetectHighlightsNode
|
| 42 |
+
# from open_storyline.xhs_nodes.xhs_filter_clips import XhsHighlightFilterNode
|
| 43 |
+
|
| 44 |
+
from open_storyline.mcp.sampling_requester import make_llm
|
| 45 |
+
|
| 46 |
+
from open_storyline.nodes.node_summary import NodeSummary
|
| 47 |
+
from src.open_storyline.storage.agent_memory import ArtifactStore
|
| 48 |
+
|
| 49 |
+
from mcp.server.fastmcp import Context, FastMCP
|
| 50 |
+
from mcp.server.session import ServerSession
|
| 51 |
+
|
| 52 |
+
def create_tool_wrapper(node: BaseNode, input_schema: type[BaseModel]):
|
| 53 |
+
"""
|
| 54 |
+
工厂函数:将自定义 Node 转换为 MCP Tool 函数
|
| 55 |
+
"""
|
| 56 |
+
# 获取元数据,用于 @server.tool 的参数
|
| 57 |
+
meta = node.meta if hasattr(node, 'meta') else None
|
| 58 |
+
|
| 59 |
+
async def wrapper(mcp_ctx: Context, **kwargs) -> dict:
|
| 60 |
+
# 1. 统一处理上下文和 Session
|
| 61 |
+
request = mcp_ctx.request_context.request
|
| 62 |
+
headers = request.headers
|
| 63 |
+
session_id = headers.get('X-Storyline-Session-Id')
|
| 64 |
+
|
| 65 |
+
# 2. Session 生命周期管理 (提取自原代码)
|
| 66 |
+
session_manager = mcp_ctx.request_context.lifespan_context
|
| 67 |
+
if hasattr(session_manager, 'cleanup_expired_sessions'):
|
| 68 |
+
session_manager.cleanup_expired_sessions(session_id)
|
| 69 |
+
|
| 70 |
+
# 3. 构造参数
|
| 71 |
+
# 注意:FastMCP 会自动将参数注入到 kwargs 中,这里做合并
|
| 72 |
+
req_json = await request.json()
|
| 73 |
+
params = kwargs.copy()
|
| 74 |
+
# 优先使用 request body 中的 arguments,如果没有则使用 kwargs
|
| 75 |
+
params.update(req_json.get('params', {}).get('arguments', {}))
|
| 76 |
+
|
| 77 |
+
params['session_id'] = session_id
|
| 78 |
+
params['node_summary'] = NodeSummary() # 假设这是每次都需要新的
|
| 79 |
+
params['llm'] = make_llm(mcp_ctx)
|
| 80 |
+
result = await node(**params)
|
| 81 |
+
return result
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
new_params = []
|
| 85 |
+
# 第一个参数固定为 mcp_ctx
|
| 86 |
+
new_params.append(
|
| 87 |
+
inspect.Parameter(
|
| 88 |
+
'mcp_ctx',
|
| 89 |
+
inspect.Parameter.POSITIONAL_OR_KEYWORD,
|
| 90 |
+
annotation=Context
|
| 91 |
+
)
|
| 92 |
+
)
|
| 93 |
+
|
| 94 |
+
new_annotations = {'mcp_ctx': Context}
|
| 95 |
+
|
| 96 |
+
if input_schema:
|
| 97 |
+
for field_name, field_info in input_schema.model_fields.items():
|
| 98 |
+
# 使用 Annotated 携带描述,FastMCP 会将其识别为 JSON Schema 的 description
|
| 99 |
+
annotation = Annotated[field_info.annotation, field_info]
|
| 100 |
+
|
| 101 |
+
new_params.append(
|
| 102 |
+
inspect.Parameter(
|
| 103 |
+
field_name,
|
| 104 |
+
inspect.Parameter.KEYWORD_ONLY,
|
| 105 |
+
default=field_info.default if field_info.default is not ... else inspect.Parameter.empty,
|
| 106 |
+
annotation=annotation
|
| 107 |
+
)
|
| 108 |
+
)
|
| 109 |
+
new_annotations[field_name] = annotation
|
| 110 |
+
|
| 111 |
+
wrapper.__name__ = meta.name
|
| 112 |
+
wrapper.__doc__ = meta.description
|
| 113 |
+
wrapper.__signature__ = inspect.Signature(new_params)
|
| 114 |
+
wrapper.__annotations__ = new_annotations
|
| 115 |
+
|
| 116 |
+
return wrapper, meta
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def register(server: FastMCP, cfg: Settings) -> None:
|
| 120 |
+
if cfg.local_mcp_server.use_xhs_node_only:
|
| 121 |
+
all_node_classes = [
|
| 122 |
+
AssetsIndexNode,
|
| 123 |
+
# XhsDetectHighlightsNode, XhsUnderstandClipsNode, XhsHighlightFilterNode, XhsGroupClipNode, XhsGenerateScriptNode,
|
| 124 |
+
# ElementrecTextsNode, ElementrecTitleNode, ElementrecGlobalEffectsNode, ElementrecCloseEffectsNode, ElementrecFilterNode,
|
| 125 |
+
# ElementrecTTSSpeakerNode, ElementrecLocalEffectsNode, ElementrecTransitionNode, ElementrecTextAnimationNode, ElementrecTitleClipNode,
|
| 126 |
+
# FireTTSNode, MusicRecNode, TimelineNode, TemplateRenderNode
|
| 127 |
+
]
|
| 128 |
+
else:
|
| 129 |
+
core_nodes = [
|
| 130 |
+
AssetsIndexNode, SplitShotsNode, UnderstandClipsNode, FilterClipsNode, GroupClipsNode, GenerateScriptNode, GenerateVoiceoverNode, SelectBGMNode, PlanTimelineNode, RenderVideoNode
|
| 131 |
+
]
|
| 132 |
+
|
| 133 |
+
xhs_nodes = [
|
| 134 |
+
# XhsDetectHighlightsNode, XhsUnderstandClipsNode, XhsHighlightFilterNode, XhsGroupClipNode, XhsGenerateScriptNode,
|
| 135 |
+
# ElementrecTextsNode, ElementrecTitleNode, ElementrecGlobalEffectsNode, ElementrecCloseEffectsNode, ElementrecFilterNode,
|
| 136 |
+
# ElementrecTTSSpeakerNode, ElementrecLocalEffectsNode, ElementrecTransitionNode, ElementrecTextAnimationNode, ElementrecTitleClipNode,
|
| 137 |
+
# FireTTSNode, MusicRecNode, TimelineNode, TemplateRenderNode
|
| 138 |
+
]
|
| 139 |
+
# 合并所有节点类
|
| 140 |
+
all_node_classes = core_nodes + xhs_nodes
|
| 141 |
+
|
| 142 |
+
# 2. 批量注册普通节点
|
| 143 |
+
for NodeClass in all_node_classes:
|
| 144 |
+
# 实例化
|
| 145 |
+
node_instance = NodeClass(cfg)
|
| 146 |
+
input_schema = node_instance.input_schema
|
| 147 |
+
|
| 148 |
+
tool_func, meta = create_tool_wrapper(node_instance, input_schema)
|
| 149 |
+
|
| 150 |
+
tool_name = NodeClass.meta.name
|
| 151 |
+
tool_desc = NodeClass.meta.description
|
| 152 |
+
|
| 153 |
+
# 使用 server.tool 装饰器注册
|
| 154 |
+
server.tool(
|
| 155 |
+
name=tool_name,
|
| 156 |
+
description=tool_desc,
|
| 157 |
+
meta=asdict(meta)
|
| 158 |
+
)(tool_func)
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
# 注册特殊工具 (如 read_node_history)
|
| 162 |
+
# local tool
|
| 163 |
+
@server.tool(
|
| 164 |
+
name="read_node_history",
|
| 165 |
+
description="读取任意节点的运行结果。需要输入该结果对应的artifact_id"
|
| 166 |
+
)
|
| 167 |
+
async def mcp_read_history(
|
| 168 |
+
mcp_ctx: Context[ServerSession, object],
|
| 169 |
+
query_artifact_id: Annotated[str, Field(description="用该artifact_id拿相应的json")]
|
| 170 |
+
) -> dict:
|
| 171 |
+
node_summary = NodeSummary()
|
| 172 |
+
request = mcp_ctx.request_context.request
|
| 173 |
+
session_id = mcp_ctx.request_context.request.headers['X-Storyline-Session-Id']
|
| 174 |
+
req_json_content = await request.json()
|
| 175 |
+
params = req_json_content['params'].get('arguments', {'query_artifact_id': query_artifact_id})
|
| 176 |
+
params['session_id'] = session_id
|
| 177 |
+
params['node_summary'] = node_summary
|
| 178 |
+
|
| 179 |
+
try:
|
| 180 |
+
store = ArtifactStore(artifacts_dir=params.get('artifacts_dir', ".storyline/.server_cache"), session_id=session_id)
|
| 181 |
+
meta, data = store.load_result(params['query_artifact_id'])
|
| 182 |
+
summary = "历史信息读取成功"
|
| 183 |
+
isError = False
|
| 184 |
+
except Exception as e:
|
| 185 |
+
traceback_info = ''.join(traceback.format_exception(e))
|
| 186 |
+
summary = f"历史读取执行失败: {params['query_artifact_id']}\n {traceback_info}",
|
| 187 |
+
meta, data, isError = 'None', 'None', True
|
| 188 |
+
|
| 189 |
+
return {
|
| 190 |
+
'artifact_id': params.get('artifact_id', store.generate_artifact_id(req_json_content['params'].get('name', 'read_node_history'))),
|
| 191 |
+
'tool_excute_result': {
|
| 192 |
+
"history": {
|
| 193 |
+
"meta": meta,
|
| 194 |
+
"node_data": data,
|
| 195 |
+
}
|
| 196 |
+
},
|
| 197 |
+
'summary': summary,
|
| 198 |
+
'isError': isError,
|
| 199 |
+
}
|
| 200 |
+
|
| 201 |
+
@server.tool(
|
| 202 |
+
name="write_skills",
|
| 203 |
+
description="将生成的Agent Skill (Markdown格式) 写入文件系统。成功后返回文件绝对路径。"
|
| 204 |
+
)
|
| 205 |
+
async def mcp_write_skills(
|
| 206 |
+
mcp_ctx: Context[ServerSession, object],
|
| 207 |
+
skill_name: Annotated[str, Field(description="技能文件的名称,例如 'fast_paced_vlog',无需包含后缀")],
|
| 208 |
+
skill_dir: Annotated[str, Field(description="技能存储目录,默认为 '.storyline/skills/'")] = '.storyline/skills/',
|
| 209 |
+
skill_content: Annotated[str, Field(description="完整的Markdown内容")] = '',
|
| 210 |
+
) -> dict:
|
| 211 |
+
"""
|
| 212 |
+
功能:接收大模型生成的技能内容,将其保存为本地 MD 文件。
|
| 213 |
+
"""
|
| 214 |
+
node_summary = NodeSummary()
|
| 215 |
+
request = mcp_ctx.request_context.request
|
| 216 |
+
session_id = request.headers.get('X-Storyline-Session-Id', 'unknown_session')
|
| 217 |
+
req_json_content = await request.json()
|
| 218 |
+
params = req_json_content['params'].get('arguments', {'skill_name': skill_name, 'skill_dir': skill_dir, 'skill_content': skill_content})
|
| 219 |
+
params['session_id'] = session_id
|
| 220 |
+
params['node_summary'] = node_summary
|
| 221 |
+
|
| 222 |
+
res = await dump_skills(
|
| 223 |
+
skill_name=skill_name,
|
| 224 |
+
skill_dir=skill_dir,
|
| 225 |
+
skill_content=skill_content,
|
| 226 |
+
)
|
| 227 |
+
node_summary.info_for_llm("[Write Skills] Done.")
|
| 228 |
+
store = ArtifactStore(artifacts_dir=params.get('artifacts_dir', ".storyline/.server_cache"), session_id=session_id)
|
| 229 |
+
|
| 230 |
+
return {
|
| 231 |
+
'artifact_id': params.get('artifact_id', store.generate_artifact_id(req_json_content['params'].get('name', 'mcp_write_skills'))),
|
| 232 |
+
'tool_excute_result': {
|
| 233 |
+
},
|
| 234 |
+
'summary': "",
|
| 235 |
+
'isError': False,
|
| 236 |
+
}
|
| 237 |
+
|
src/open_storyline/mcp/sampling_handler.py
ADDED
|
@@ -0,0 +1,424 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
+
import os
|
| 3 |
+
import math
|
| 4 |
+
import base64
|
| 5 |
+
from io import BytesIO
|
| 6 |
+
from typing import Any, Dict, List, Tuple
|
| 7 |
+
from urllib.parse import urlparse
|
| 8 |
+
|
| 9 |
+
from PIL import Image
|
| 10 |
+
from moviepy.video.io.VideoFileClip import VideoFileClip
|
| 11 |
+
from langchain_core.messages import SystemMessage, HumanMessage, AIMessage
|
| 12 |
+
|
| 13 |
+
from mcp.types import CreateMessageRequestParams, CreateMessageResult, TextContent
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
# -----------------------------
|
| 17 |
+
# 可调参数:控制多模态输入大小
|
| 18 |
+
# -----------------------------
|
| 19 |
+
DEFAULT_RESIZE_EDGE = 768
|
| 20 |
+
DEFAULT_JPEG_QUALITY = 80
|
| 21 |
+
DEFAULT_MIN_FRAMES = 6
|
| 22 |
+
DEFAULT_MAX_FRAMES = 16
|
| 23 |
+
DEFAULT_FRAMES_PER_SEC = 3.0
|
| 24 |
+
GLOBAL_MAX_IMAGE_BLOCKS = 48 # 全局最多塞多少张图(含视频帧+图片),防止 payload 过大
|
| 25 |
+
|
| 26 |
+
IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".gif", ".tiff"}
|
| 27 |
+
VIDEO_EXTS = {".mp4", ".mov", ".mkv", ".avi", ".webm", ".m4v"}
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _is_data_url(u: str) -> bool:
|
| 31 |
+
return isinstance(u, str) and u.startswith("data:")
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _is_http_url(u: str) -> bool:
|
| 35 |
+
return isinstance(u, str) and (u.startswith("http://") or u.startswith("https://"))
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def _strip_file_scheme(u: str) -> str:
|
| 39 |
+
if not isinstance(u, str):
|
| 40 |
+
return str(u)
|
| 41 |
+
if u.startswith("file://"):
|
| 42 |
+
parsed = urlparse(u)
|
| 43 |
+
return parsed.path
|
| 44 |
+
return u
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def _guess_ext(path_or_url: str) -> str:
|
| 48 |
+
try:
|
| 49 |
+
p = urlparse(path_or_url).path if _is_http_url(path_or_url) else path_or_url
|
| 50 |
+
return os.path.splitext(p)[1].lower()
|
| 51 |
+
except Exception:
|
| 52 |
+
return ""
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _resize_long_edge(img: Image.Image, long_edge: int) -> Image.Image:
|
| 56 |
+
if long_edge <= 0:
|
| 57 |
+
return img
|
| 58 |
+
w, h = img.size
|
| 59 |
+
le = max(w, h)
|
| 60 |
+
if le <= long_edge:
|
| 61 |
+
return img
|
| 62 |
+
scale = long_edge / float(le)
|
| 63 |
+
nw = max(1, int(round(w * scale)))
|
| 64 |
+
nh = max(1, int(round(h * scale)))
|
| 65 |
+
return img.resize((nw, nh), Image.LANCZOS)
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def _pil_to_data_url(img: Image.Image, resize_edge: int, jpeg_quality: int) -> str:
|
| 69 |
+
img = img.convert("RGB")
|
| 70 |
+
img = _resize_long_edge(img, resize_edge)
|
| 71 |
+
buf = BytesIO()
|
| 72 |
+
img.save(buf, format="JPEG", quality=jpeg_quality, optimize=True)
|
| 73 |
+
b64 = base64.b64encode(buf.getvalue()).decode("utf-8")
|
| 74 |
+
return f"data:image/jpeg;base64,{b64}"
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def _image_path_to_data_url(path: str, resize_edge: int, jpeg_quality: int) -> str:
|
| 78 |
+
img = Image.open(path)
|
| 79 |
+
return _pil_to_data_url(img, resize_edge, jpeg_quality)
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def _choose_num_frames(duration_sec: float, min_frames: int, max_frames: int, frames_per_sec: float) -> int:
|
| 83 |
+
duration_sec = max(0.0, float(duration_sec))
|
| 84 |
+
n = int(math.ceil(duration_sec * frames_per_sec))
|
| 85 |
+
n = max(min_frames, n)
|
| 86 |
+
n = min(max_frames, n)
|
| 87 |
+
return n
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def _sample_video_segment_to_data_urls(
|
| 91 |
+
video_path: str,
|
| 92 |
+
in_sec: float,
|
| 93 |
+
out_sec: float,
|
| 94 |
+
resize_edge: int,
|
| 95 |
+
jpeg_quality: int,
|
| 96 |
+
min_frames: int,
|
| 97 |
+
max_frames: int,
|
| 98 |
+
frames_per_sec: float,
|
| 99 |
+
) -> List[Tuple[float, str]]:
|
| 100 |
+
"""
|
| 101 |
+
只对 [in_sec, out_sec] 区间抽帧。返回 (rel_t_from_in, data_url)
|
| 102 |
+
"""
|
| 103 |
+
|
| 104 |
+
in_sec = float(in_sec)
|
| 105 |
+
out_sec = float(out_sec)
|
| 106 |
+
|
| 107 |
+
clip = VideoFileClip(video_path, audio=False)
|
| 108 |
+
try:
|
| 109 |
+
vdur = float(clip.duration or 0.0)
|
| 110 |
+
|
| 111 |
+
# 如果拿不到 duration,保守只取一帧,避免 out_sec 很大导致越界
|
| 112 |
+
if vdur <= 0:
|
| 113 |
+
t = max(0.0, in_sec)
|
| 114 |
+
frame = clip.get_frame(t)
|
| 115 |
+
img = Image.fromarray(frame)
|
| 116 |
+
return [(0.0, _pil_to_data_url(img, resize_edge, jpeg_quality))]
|
| 117 |
+
|
| 118 |
+
# 裁剪到合法范围
|
| 119 |
+
in_sec = max(0.0, min(in_sec, vdur))
|
| 120 |
+
out_sec = max(0.0, min(out_sec, vdur))
|
| 121 |
+
|
| 122 |
+
# 若仍无效,兜底取 in_sec 一帧
|
| 123 |
+
if out_sec <= in_sec:
|
| 124 |
+
frame = clip.get_frame(in_sec)
|
| 125 |
+
img = Image.fromarray(frame)
|
| 126 |
+
return [(0.0, _pil_to_data_url(img, resize_edge, jpeg_quality))]
|
| 127 |
+
|
| 128 |
+
seg_dur = out_sec - in_sec
|
| 129 |
+
n = _choose_num_frames(seg_dur, min_frames, max_frames, frames_per_sec)
|
| 130 |
+
|
| 131 |
+
# 分桶中心点采样,避免边界帧
|
| 132 |
+
times = [((i + 0.5) / n) * seg_dur for i in range(n)]
|
| 133 |
+
|
| 134 |
+
out: List[Tuple[float, str]] = []
|
| 135 |
+
for rel_t in times:
|
| 136 |
+
abs_t = in_sec + rel_t
|
| 137 |
+
frame = clip.get_frame(abs_t)
|
| 138 |
+
img = Image.fromarray(frame)
|
| 139 |
+
out.append((rel_t, _pil_to_data_url(img, resize_edge, jpeg_quality)))
|
| 140 |
+
return out
|
| 141 |
+
finally:
|
| 142 |
+
clip.close()
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def _extract_text_from_mcp_content(content: Any) -> str:
|
| 146 |
+
if content is None:
|
| 147 |
+
return ""
|
| 148 |
+
blocks = content if isinstance(content, list) else [content]
|
| 149 |
+
texts: List[str] = []
|
| 150 |
+
for b in blocks:
|
| 151 |
+
if getattr(b, "type", None) == "text":
|
| 152 |
+
texts.append(getattr(b, "text", "") or "")
|
| 153 |
+
return "\n".join([t for t in (x.strip() for x in texts) if t])
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def _extract_text_from_lc_response(resp: Any) -> str:
|
| 157 |
+
content = getattr(resp, "content", None)
|
| 158 |
+
if isinstance(content, str):
|
| 159 |
+
return content.strip()
|
| 160 |
+
if isinstance(content, list):
|
| 161 |
+
texts = []
|
| 162 |
+
for blk in content:
|
| 163 |
+
if isinstance(blk, dict) and blk.get("type") == "text":
|
| 164 |
+
texts.append(str(blk.get("text", "")).strip())
|
| 165 |
+
return "\n".join([t for t in texts if t]).strip()
|
| 166 |
+
return str(resp).strip()
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def _normalize_media_items(media_inputs: List[Any]) -> List[Dict[str, Any]]:
|
| 170 |
+
"""
|
| 171 |
+
支持三种输入项:
|
| 172 |
+
1) "path/to/video.mp4"
|
| 173 |
+
2) {"url"/"path": "...", "in_sec": 1.2, "out_sec": 3.4}
|
| 174 |
+
3) ("path/to/video.mp4", 1.2, 3.4) # 可选
|
| 175 |
+
输出统一为:{"url": "...", "in_sec": optional, "out_sec": optional}
|
| 176 |
+
"""
|
| 177 |
+
out = []
|
| 178 |
+
for item in media_inputs or []:
|
| 179 |
+
if isinstance(item, str):
|
| 180 |
+
out.append({"url": item})
|
| 181 |
+
continue
|
| 182 |
+
|
| 183 |
+
if isinstance(item, (list, tuple)) and len(item) >= 1:
|
| 184 |
+
d = {"url": item[0]}
|
| 185 |
+
if len(item) >= 2:
|
| 186 |
+
d["in_sec"] = item[1]
|
| 187 |
+
if len(item) >= 3:
|
| 188 |
+
d["out_sec"] = item[2]
|
| 189 |
+
out.append(d)
|
| 190 |
+
continue
|
| 191 |
+
|
| 192 |
+
if isinstance(item, dict):
|
| 193 |
+
url = item.get("url") or item.get("path") or item.get("media")
|
| 194 |
+
if not url:
|
| 195 |
+
continue
|
| 196 |
+
d = {"url": url}
|
| 197 |
+
if "in_sec" in item:
|
| 198 |
+
d["in_sec"] = item.get("in_sec")
|
| 199 |
+
if "out_sec" in item:
|
| 200 |
+
d["out_sec"] = item.get("out_sec")
|
| 201 |
+
out.append(d)
|
| 202 |
+
continue
|
| 203 |
+
|
| 204 |
+
return out
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def _build_media_blocks(
|
| 208 |
+
media_inputs: List[Any],
|
| 209 |
+
resize_edge: int,
|
| 210 |
+
jpeg_quality: int,
|
| 211 |
+
min_frames: int,
|
| 212 |
+
max_frames: int,
|
| 213 |
+
frames_per_sec: float,
|
| 214 |
+
global_max_images: int,
|
| 215 |
+
) -> List[Dict[str, Any]]:
|
| 216 |
+
"""
|
| 217 |
+
将 media_inputs 转成 OpenAI-compatible 多模态 blocks。
|
| 218 |
+
视频:按段抽帧(若有 in_sec/out_sec)或全视频抽帧(老格式)
|
| 219 |
+
"""
|
| 220 |
+
|
| 221 |
+
blocks: List[Dict[str, Any]] = []
|
| 222 |
+
img_count = 0
|
| 223 |
+
|
| 224 |
+
items = _normalize_media_items(media_inputs)
|
| 225 |
+
|
| 226 |
+
for idx, mi in enumerate(items):
|
| 227 |
+
if img_count >= global_max_images:
|
| 228 |
+
break
|
| 229 |
+
|
| 230 |
+
raw_url = _strip_file_scheme(str(mi.get("url")))
|
| 231 |
+
ext = _guess_ext(raw_url)
|
| 232 |
+
|
| 233 |
+
in_sec = mi.get("in_sec")
|
| 234 |
+
out_sec = mi.get("out_sec")
|
| 235 |
+
has_segment = (in_sec is not None and out_sec is not None)
|
| 236 |
+
|
| 237 |
+
# 1) data url(图片)直接透传
|
| 238 |
+
if _is_data_url(raw_url):
|
| 239 |
+
blocks.append({"type": "text", "text": f"Media {idx+1}: (data url image)"})
|
| 240 |
+
blocks.append({"type": "image_url", "image_url": {"url": raw_url}})
|
| 241 |
+
img_count += 1
|
| 242 |
+
continue
|
| 243 |
+
|
| 244 |
+
# 2) 远程 URL:图片可透传;远程视频无法本地抽帧(给提示)
|
| 245 |
+
if _is_http_url(raw_url):
|
| 246 |
+
if ext in VIDEO_EXTS:
|
| 247 |
+
seg_info = f" segment [{in_sec},{out_sec}]s" if has_segment else ""
|
| 248 |
+
blocks.append({"type": "text", "text": f"Media {idx+1}: remote video url{seg_info} (cannot sample frames locally): {raw_url}"})
|
| 249 |
+
continue
|
| 250 |
+
blocks.append({"type": "text", "text": f"Media {idx+1}: {raw_url}"})
|
| 251 |
+
blocks.append({"type": "image_url", "image_url": {"url": raw_url}})
|
| 252 |
+
img_count += 1
|
| 253 |
+
continue
|
| 254 |
+
|
| 255 |
+
# 3) 本地路径
|
| 256 |
+
path = raw_url
|
| 257 |
+
if not os.path.exists(path):
|
| 258 |
+
blocks.append({"type": "text", "text": f"Media {idx+1}: (missing file) {path}"})
|
| 259 |
+
continue
|
| 260 |
+
|
| 261 |
+
# 图片
|
| 262 |
+
if ext in IMAGE_EXTS:
|
| 263 |
+
data_url = _image_path_to_data_url(path, resize_edge, jpeg_quality)
|
| 264 |
+
blocks.append({"type": "text", "text": f"Media {idx+1}: image file {os.path.basename(path)}"})
|
| 265 |
+
blocks.append({"type": "image_url", "image_url": {"url": data_url}})
|
| 266 |
+
img_count += 1
|
| 267 |
+
continue
|
| 268 |
+
|
| 269 |
+
# 视频(支持按段)
|
| 270 |
+
if ext in VIDEO_EXTS:
|
| 271 |
+
if has_segment:
|
| 272 |
+
in_s = float(in_sec)
|
| 273 |
+
out_s = float(out_sec)
|
| 274 |
+
else:
|
| 275 |
+
# 老格式:全视频。用 [0, +inf],内部会 clamp 到 duration
|
| 276 |
+
in_s = 0.0
|
| 277 |
+
out_s = 1e12
|
| 278 |
+
|
| 279 |
+
frames = _sample_video_segment_to_data_urls(
|
| 280 |
+
video_path=path,
|
| 281 |
+
in_sec=in_s,
|
| 282 |
+
out_sec=out_s,
|
| 283 |
+
resize_edge=resize_edge,
|
| 284 |
+
jpeg_quality=jpeg_quality,
|
| 285 |
+
min_frames=min_frames,
|
| 286 |
+
max_frames=max_frames,
|
| 287 |
+
frames_per_sec=frames_per_sec,
|
| 288 |
+
)
|
| 289 |
+
|
| 290 |
+
if has_segment:
|
| 291 |
+
blocks.append({"type": "text", "text": f"Media {idx+1}: video segment {os.path.basename(path)} [{in_s:.2f}s, {out_s:.2f}s] (sampled frames in time order)"})
|
| 292 |
+
else:
|
| 293 |
+
blocks.append({"type": "text", "text": f"Media {idx+1}: video file {os.path.basename(path)} (sampled frames in time order)"})
|
| 294 |
+
|
| 295 |
+
for fi, (rel_t, data_url) in enumerate(frames):
|
| 296 |
+
if img_count >= global_max_images:
|
| 297 |
+
break
|
| 298 |
+
blocks.append({"type": "text", "text": f"Frame {fi+1}/{len(frames)} (t≈{rel_t:.2f}s from segment start)"})
|
| 299 |
+
blocks.append({"type": "image_url", "image_url": {"url": data_url}})
|
| 300 |
+
img_count += 1
|
| 301 |
+
continue
|
| 302 |
+
|
| 303 |
+
blocks.append({"type": "text", "text": f"Media {idx+1}: unsupported file type: {path}"})
|
| 304 |
+
|
| 305 |
+
return blocks
|
| 306 |
+
|
| 307 |
+
|
| 308 |
+
def make_sampling_callback(
|
| 309 |
+
llm,
|
| 310 |
+
vlm,
|
| 311 |
+
*,
|
| 312 |
+
resize_edge: int = DEFAULT_RESIZE_EDGE,
|
| 313 |
+
jpeg_quality: int = DEFAULT_JPEG_QUALITY,
|
| 314 |
+
min_frames: int = DEFAULT_MIN_FRAMES,
|
| 315 |
+
max_frames: int = DEFAULT_MAX_FRAMES,
|
| 316 |
+
frames_per_sec: float = DEFAULT_FRAMES_PER_SEC,
|
| 317 |
+
global_max_images: int = GLOBAL_MAX_IMAGE_BLOCKS,
|
| 318 |
+
):
|
| 319 |
+
"""
|
| 320 |
+
MCP server 在工具内部请求 sampling 时回调这里:
|
| 321 |
+
- 读取 metadata.media_urls(支持带 in_sec/out_sec)
|
| 322 |
+
- 抽帧后拼成 LangChain 多模态消息
|
| 323 |
+
- 根据是否有 media 输入,选择 llm/vlm
|
| 324 |
+
"""
|
| 325 |
+
|
| 326 |
+
async def sampling_callback(context, params: CreateMessageRequestParams) -> CreateMessageResult:
|
| 327 |
+
# 1. system prompt
|
| 328 |
+
system_prompt = getattr(params, "systemPrompt", None) or ""
|
| 329 |
+
|
| 330 |
+
# 2. MCP messages(多轮) -> LangChain
|
| 331 |
+
mcp_messages = getattr(params, "messages", []) or []
|
| 332 |
+
lc_messages: List[Any] = []
|
| 333 |
+
if system_prompt:
|
| 334 |
+
lc_messages.append(SystemMessage(content=system_prompt))
|
| 335 |
+
|
| 336 |
+
# 3. metadata:取 media_urls 与 top_p
|
| 337 |
+
metadata = getattr(params, "metadata", None) or {}
|
| 338 |
+
media_inputs = list(metadata.get("media", []) or [])
|
| 339 |
+
top_p: float = float(metadata.get("top_p", 0.9))
|
| 340 |
+
|
| 341 |
+
temperature: float = float(getattr(params, "temperature", None) or 0.6)
|
| 342 |
+
max_tokens: int = int(getattr(params, "maxTokens", 4096) or 4096)
|
| 343 |
+
|
| 344 |
+
# 4. 分流
|
| 345 |
+
use_multimodal = bool(media_inputs)
|
| 346 |
+
model = vlm if use_multimodal else llm
|
| 347 |
+
if model is None:
|
| 348 |
+
model = vlm or llm
|
| 349 |
+
|
| 350 |
+
# 5. 构造媒体 blocks(含视频按段抽帧)——放线程,避免阻塞 event loop
|
| 351 |
+
media_blocks: List[Dict[str, Any]] = []
|
| 352 |
+
if use_multimodal:
|
| 353 |
+
media_blocks = await asyncio.to_thread(
|
| 354 |
+
_build_media_blocks,
|
| 355 |
+
media_inputs,
|
| 356 |
+
resize_edge,
|
| 357 |
+
jpeg_quality,
|
| 358 |
+
min_frames,
|
| 359 |
+
max_frames,
|
| 360 |
+
frames_per_sec,
|
| 361 |
+
global_max_images,
|
| 362 |
+
)
|
| 363 |
+
|
| 364 |
+
# 6. 媒体附加到“最后一条 user 消息”
|
| 365 |
+
user_indices = [i for i, m in enumerate(mcp_messages) if getattr(m, "role", "") == "user"]
|
| 366 |
+
last_user_idx = user_indices[-1] if user_indices else None
|
| 367 |
+
|
| 368 |
+
if not mcp_messages:
|
| 369 |
+
# 无消息则造一条 user
|
| 370 |
+
content_blocks = [{"type": "text", "text": ""}]
|
| 371 |
+
if media_blocks:
|
| 372 |
+
content_blocks.extend(media_blocks)
|
| 373 |
+
lc_messages.append(HumanMessage(content=content_blocks if media_blocks else ""))
|
| 374 |
+
else:
|
| 375 |
+
for i, m in enumerate(mcp_messages):
|
| 376 |
+
role = getattr(m, "role", "") or "user"
|
| 377 |
+
text = _extract_text_from_mcp_content(getattr(m, "content", None))
|
| 378 |
+
|
| 379 |
+
if role == "assistant":
|
| 380 |
+
lc_messages.append(AIMessage(content=text))
|
| 381 |
+
continue
|
| 382 |
+
|
| 383 |
+
if role == "user":
|
| 384 |
+
if last_user_idx is not None and i == last_user_idx and media_blocks:
|
| 385 |
+
content_blocks = [{"type": "text", "text": text}]
|
| 386 |
+
content_blocks.extend(media_blocks)
|
| 387 |
+
lc_messages.append(HumanMessage(content=content_blocks))
|
| 388 |
+
else:
|
| 389 |
+
lc_messages.append(HumanMessage(content=text))
|
| 390 |
+
continue
|
| 391 |
+
|
| 392 |
+
lc_messages.append(HumanMessage(content=text))
|
| 393 |
+
|
| 394 |
+
# 7. 调用所选模型
|
| 395 |
+
bound = model
|
| 396 |
+
try:
|
| 397 |
+
bound = bound.bind(temperature=temperature, max_tokens=max_tokens, top_p=top_p)
|
| 398 |
+
except Exception:
|
| 399 |
+
bound = bound.bind(temperature=temperature, max_tokens=max_tokens)
|
| 400 |
+
|
| 401 |
+
try:
|
| 402 |
+
if hasattr(bound, "ainvoke"):
|
| 403 |
+
resp = await bound.ainvoke(lc_messages)
|
| 404 |
+
else:
|
| 405 |
+
resp = await asyncio.to_thread(bound.invoke, lc_messages)
|
| 406 |
+
except TypeError:
|
| 407 |
+
# 极端情况下:某些 wrapper 不接受 max_tokens/top_p
|
| 408 |
+
bound2 = model.bind(temperature=temperature)
|
| 409 |
+
if hasattr(bound2, "ainvoke"):
|
| 410 |
+
resp = await bound2.ainvoke(lc_messages)
|
| 411 |
+
else:
|
| 412 |
+
resp = await asyncio.to_thread(bound2.invoke, lc_messages)
|
| 413 |
+
|
| 414 |
+
text_out = _extract_text_from_lc_response(resp)
|
| 415 |
+
|
| 416 |
+
model_name = getattr(model, "model", None) or getattr(model, "model_name", None) or "unknown"
|
| 417 |
+
return CreateMessageResult(
|
| 418 |
+
content=TextContent(type="text", text=text_out),
|
| 419 |
+
model=str(model_name),
|
| 420 |
+
role="assistant",
|
| 421 |
+
stopReason="endTurn",
|
| 422 |
+
)
|
| 423 |
+
|
| 424 |
+
return sampling_callback
|
src/open_storyline/mcp/sampling_requester.py
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from typing import Any, Optional, Protocol, runtime_checkable
|
| 4 |
+
|
| 5 |
+
from mcp.server.fastmcp import Context
|
| 6 |
+
from mcp.server.session import ServerSession
|
| 7 |
+
from mcp.types import SamplingMessage, TextContent, ModelHint, ModelPreferences
|
| 8 |
+
|
| 9 |
+
class BaseLLMSampling(Protocol):
|
| 10 |
+
# 低层能力协议:多个工具复用sampling
|
| 11 |
+
async def sampling(
|
| 12 |
+
self,
|
| 13 |
+
*,
|
| 14 |
+
system_prompt: str | None,
|
| 15 |
+
messages: list[SamplingMessage],
|
| 16 |
+
temperature: float = 0.3,
|
| 17 |
+
top_p: float = 0.9,
|
| 18 |
+
max_tokens: int = 4096,
|
| 19 |
+
model_preferences: dict[str, Any] | None = None,
|
| 20 |
+
metadata: dict[str, Any] | None = None,
|
| 21 |
+
stop_sequences: list[str] | None = None,
|
| 22 |
+
) -> str:
|
| 23 |
+
...
|
| 24 |
+
|
| 25 |
+
@runtime_checkable
|
| 26 |
+
class LLMClient(Protocol):
|
| 27 |
+
# 高层能力协议:多个工具只区分是否需要用到多模态能力
|
| 28 |
+
async def complete(
|
| 29 |
+
self,
|
| 30 |
+
*,
|
| 31 |
+
system_prompt: str | None,
|
| 32 |
+
user_prompt: str,
|
| 33 |
+
media: list[dict[str, Any]] | None = None,
|
| 34 |
+
temperature: float = 0.3,
|
| 35 |
+
top_p: float = 0.9,
|
| 36 |
+
max_tokens: int = 2048,
|
| 37 |
+
model_preferences: dict[str, Any] | None = None,
|
| 38 |
+
metadata: dict[str, Any] | None = None,
|
| 39 |
+
stop_sequences: list[str] | None = None,
|
| 40 |
+
) -> str:
|
| 41 |
+
...
|
| 42 |
+
|
| 43 |
+
class MCPSampler(BaseLLMSampling):
|
| 44 |
+
def __init__(self, mcp_ctx: Context[ServerSession, object]):
|
| 45 |
+
self._mcp_ctx = mcp_ctx
|
| 46 |
+
|
| 47 |
+
def _to_mcp_model_preferences(
|
| 48 |
+
self,
|
| 49 |
+
model_preferences: dict[str, Any] | None,
|
| 50 |
+
) -> Optional[ModelPreferences]:
|
| 51 |
+
if not model_preferences:
|
| 52 |
+
return None
|
| 53 |
+
|
| 54 |
+
raw_hints = model_preferences.get("hints")
|
| 55 |
+
hints: list[ModelHint] | None = None
|
| 56 |
+
if isinstance(raw_hints, list):
|
| 57 |
+
hints = []
|
| 58 |
+
for h in raw_hints:
|
| 59 |
+
if isinstance(h, ModelHint):
|
| 60 |
+
hints.append(h)
|
| 61 |
+
elif isinstance(h, dict):
|
| 62 |
+
hints.append(ModelHint(**h))
|
| 63 |
+
elif isinstance(h, str):
|
| 64 |
+
hints.append(ModelHint(name=h))
|
| 65 |
+
|
| 66 |
+
return ModelPreferences(
|
| 67 |
+
hints=hints,
|
| 68 |
+
costPriority=model_preferences.get("costPriority"),
|
| 69 |
+
speedPriority=model_preferences.get("speedPriority"),
|
| 70 |
+
intelligencePriority=model_preferences.get("intelligencePriority"),
|
| 71 |
+
)
|
| 72 |
+
|
| 73 |
+
def _extract_text(self, content: Any) -> str:
|
| 74 |
+
# MCP 返回 content 可能是单块或数组;这里只抽取 text block
|
| 75 |
+
if isinstance(content, list):
|
| 76 |
+
texts: list[str] = []
|
| 77 |
+
for block in content:
|
| 78 |
+
if getattr(block, "type", None) == "text":
|
| 79 |
+
texts.append(block.text)
|
| 80 |
+
return "\n".join(texts).strip()
|
| 81 |
+
|
| 82 |
+
if getattr(content, "type", None) == "text":
|
| 83 |
+
return content.text.strip()
|
| 84 |
+
|
| 85 |
+
return str(content)
|
| 86 |
+
|
| 87 |
+
async def sampling(self,
|
| 88 |
+
*,
|
| 89 |
+
system_prompt: str | None,
|
| 90 |
+
messages: list[SamplingMessage],
|
| 91 |
+
temperature: float = 0.3,
|
| 92 |
+
top_p: float = 0.9,
|
| 93 |
+
max_tokens: int = 4096,
|
| 94 |
+
model_preferences: dict[str, Any] | None = None,
|
| 95 |
+
metadata: dict[str, Any] | None = None,
|
| 96 |
+
stop_sequences: list[str] | None = None
|
| 97 |
+
) -> str:
|
| 98 |
+
merged_metadata = dict(metadata or {})
|
| 99 |
+
merged_metadata["top_p"] = top_p
|
| 100 |
+
|
| 101 |
+
result = await self._mcp_ctx.session.create_message(
|
| 102 |
+
messages=messages,
|
| 103 |
+
max_tokens=max_tokens,
|
| 104 |
+
system_prompt=system_prompt,
|
| 105 |
+
temperature=temperature,
|
| 106 |
+
# stop_sequences=stop_sequences,
|
| 107 |
+
metadata=merged_metadata,
|
| 108 |
+
# model_preferences=self._to_mcp_model_preferences(model_preferences),
|
| 109 |
+
)
|
| 110 |
+
return self._extract_text(result.content)
|
| 111 |
+
|
| 112 |
+
class SamplingLLMClient(LLMClient):
|
| 113 |
+
"""
|
| 114 |
+
只区分是否有media输入
|
| 115 |
+
Server将media路径和时间戳透传给Client,Client负责转base64
|
| 116 |
+
"""
|
| 117 |
+
|
| 118 |
+
def __init__(self, sampler: BaseLLMSampling):
|
| 119 |
+
self._sampler = sampler
|
| 120 |
+
|
| 121 |
+
async def complete(self,
|
| 122 |
+
*,
|
| 123 |
+
system_prompt: str | None,
|
| 124 |
+
user_prompt: str,
|
| 125 |
+
media: list[dict[str, Any]] | None = None,
|
| 126 |
+
temperature: float = 0.3,
|
| 127 |
+
top_p: float = 0.9,
|
| 128 |
+
max_tokens: int = 2048,
|
| 129 |
+
model_preferences: dict[str, Any] | None = None,
|
| 130 |
+
metadata: dict[str, Any] | None = None,
|
| 131 |
+
stop_sequences: list[str] | None = None
|
| 132 |
+
)-> str:
|
| 133 |
+
messages = [
|
| 134 |
+
SamplingMessage(
|
| 135 |
+
role="user",
|
| 136 |
+
content=TextContent(type="text", text=user_prompt),
|
| 137 |
+
)
|
| 138 |
+
]
|
| 139 |
+
|
| 140 |
+
merged_metadata = dict(metadata or {})
|
| 141 |
+
merged_metadata["modality"] = "multimodal" if media else "text"
|
| 142 |
+
if media:
|
| 143 |
+
merged_metadata["media"] = media # 关键:素材路径和时间戳直接透传
|
| 144 |
+
|
| 145 |
+
return await self._sampler.sampling(
|
| 146 |
+
system_prompt=system_prompt,
|
| 147 |
+
messages=messages,
|
| 148 |
+
temperature=temperature,
|
| 149 |
+
top_p=top_p,
|
| 150 |
+
max_tokens=max_tokens,
|
| 151 |
+
model_preferences=model_preferences,
|
| 152 |
+
metadata=merged_metadata,
|
| 153 |
+
stop_sequences=stop_sequences,
|
| 154 |
+
)
|
| 155 |
+
|
| 156 |
+
def make_llm(mcp_ctx: Context[ServerSession, object]) -> LLMClient:
|
| 157 |
+
# 工具里面直接 llm = make_llm(ctx) 就可以调用 llm.complete()
|
| 158 |
+
return SamplingLLMClient(MCPSampler(mcp_ctx))
|
| 159 |
+
|
src/open_storyline/mcp/server.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pathlib import Path
|
| 2 |
+
import os
|
| 3 |
+
from collections.abc import AsyncIterator
|
| 4 |
+
from contextlib import asynccontextmanager
|
| 5 |
+
|
| 6 |
+
# prefix = "/mnt/tidal-alsh-share2/usr/yangqiwei1/tools/ffmpeg_nvenc"
|
| 7 |
+
# cuda = "/usr/local/cuda-12.4"
|
| 8 |
+
|
| 9 |
+
# os.environ["FFMPEG_BINARY"] = f"{prefix}/bin/ffmpeg"
|
| 10 |
+
# os.environ["LD_LIBRARY_PATH"] = f"{prefix}/lib:{prefix}/lib64:{cuda}/targets/x86_64-linux/lib:" + os.environ.get("LD_LIBRARY_PATH", "")
|
| 11 |
+
# os.environ["IMAGEIO_FFMPEG_EXE"] = f"{prefix}/bin/ffmpeg"
|
| 12 |
+
|
| 13 |
+
# os.environ["IMAGEIO_FFMPEG_EXE"] = "/tmp/storyline_venv/lib/python3.13/site-packages/imageio_ffmpeg/binaries/ffmpeg-linux-x86_64-v7.0.2"
|
| 14 |
+
from mcp.server.fastmcp import FastMCP
|
| 15 |
+
|
| 16 |
+
from open_storyline.mcp import register_tools
|
| 17 |
+
from open_storyline.config import load_settings, default_config_path
|
| 18 |
+
from open_storyline.config import Settings
|
| 19 |
+
from src.open_storyline.storage.session_manager import SessionLifecycleManager
|
| 20 |
+
from open_storyline.utils.logging import get_logger
|
| 21 |
+
|
| 22 |
+
logger = get_logger()
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def create_server(cfg: Settings) -> FastMCP:
|
| 26 |
+
"""
|
| 27 |
+
负责创建MCP server示例并把各个tools_*模块挂上来
|
| 28 |
+
"""
|
| 29 |
+
|
| 30 |
+
# 管理这次运行的上下文对象
|
| 31 |
+
runtime_ctx = cfg
|
| 32 |
+
|
| 33 |
+
@asynccontextmanager
|
| 34 |
+
async def session_lifespan(server: FastMCP) -> AsyncIterator[SessionLifecycleManager]:
|
| 35 |
+
"""Manage session lifecycle with type-safe context."""
|
| 36 |
+
# Initialize on startup
|
| 37 |
+
logger.info("Enable session lifespan manager")
|
| 38 |
+
session_manager = SessionLifecycleManager(
|
| 39 |
+
artifacts_root=cfg.project.outputs_dir,
|
| 40 |
+
cache_root=cfg.local_mcp_server.server_cache_dir,
|
| 41 |
+
enable_cleanup=True,
|
| 42 |
+
)
|
| 43 |
+
try:
|
| 44 |
+
yield session_manager
|
| 45 |
+
finally:
|
| 46 |
+
# Cleanup on shutdown
|
| 47 |
+
session_manager.cleanup_expired_sessions()
|
| 48 |
+
|
| 49 |
+
server = FastMCP(
|
| 50 |
+
name=cfg.local_mcp_server.server_name,
|
| 51 |
+
stateless_http=cfg.local_mcp_server.stateless_http,
|
| 52 |
+
json_response=cfg.local_mcp_server.json_response,
|
| 53 |
+
lifespan=session_lifespan,
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
# 把 runtime_ctx 传给 register_tools,让每个工具都能拿到 cfg/artifact_store/session_id
|
| 57 |
+
register_tools.register(server, runtime_ctx)
|
| 58 |
+
|
| 59 |
+
return server
|
| 60 |
+
|
| 61 |
+
def main():
|
| 62 |
+
cfg = load_settings(default_config_path())
|
| 63 |
+
server = create_server(cfg)
|
| 64 |
+
server.settings.host = cfg.local_mcp_server.connect_host
|
| 65 |
+
server.settings.port = cfg.local_mcp_server.port
|
| 66 |
+
server.run(transport=cfg.local_mcp_server.server_transport)
|
| 67 |
+
|
| 68 |
+
if __name__ == "__main__":
|
| 69 |
+
main()
|
| 70 |
+
# PYTHONPATH=src python -m open_storyline.mcp.server
|
src/open_storyline/nodes/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# 导入管理
|
src/open_storyline/nodes/core_nodes/assets_index.py
ADDED
|
@@ -0,0 +1,170 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Any, Dict, Optional, ClassVar, Type
|
| 2 |
+
from pydantic import BaseModel
|
| 3 |
+
import traceback
|
| 4 |
+
from collections import Counter
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
from moviepy.video.io.ffmpeg_reader import ffmpeg_parse_infos
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
from open_storyline.nodes.core_nodes.base_node import NodeMeta, BaseNode
|
| 10 |
+
from open_storyline.nodes.node_schema import AssetsIndexInput, AssetsIndexOutput
|
| 11 |
+
from open_storyline.nodes.node_summary import NodeSummary
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
VIDEO_EXTS = {
|
| 15 |
+
".mp4", ".mov", ".mkv", ".avi"
|
| 16 |
+
}
|
| 17 |
+
IMAGE_EXTS = {
|
| 18 |
+
".jpg", ".jpeg", ".png", ".webp", ".bmp"
|
| 19 |
+
}
|
| 20 |
+
|
| 21 |
+
def _image_metadata_from_path(path: Path) -> dict[str, Any]:
|
| 22 |
+
from PIL import Image, ImageOps
|
| 23 |
+
|
| 24 |
+
with Image.open(path) as img:
|
| 25 |
+
try:
|
| 26 |
+
img2 = ImageOps.exif_transpose(img)
|
| 27 |
+
w, h = img2.size
|
| 28 |
+
except Exception:
|
| 29 |
+
w, h = img.size
|
| 30 |
+
|
| 31 |
+
return {
|
| 32 |
+
"width": int(w),
|
| 33 |
+
"height": int(h),
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
import av
|
| 38 |
+
from fractions import Fraction
|
| 39 |
+
from typing import Any, Optional
|
| 40 |
+
from pathlib import Path
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def _video_metadata_from_path(
|
| 44 |
+
path: Path,
|
| 45 |
+
*,
|
| 46 |
+
round_duration_ndigits: Optional[int] = 3,
|
| 47 |
+
) -> dict[str, Any]:
|
| 48 |
+
|
| 49 |
+
container = av.open(str(path))
|
| 50 |
+
|
| 51 |
+
# 找第一个视频流
|
| 52 |
+
video_stream = next(
|
| 53 |
+
(s for s in container.streams if s.type == "video"),
|
| 54 |
+
None,
|
| 55 |
+
)
|
| 56 |
+
if video_stream is None:
|
| 57 |
+
raise ValueError(f"No video stream found: {path}")
|
| 58 |
+
|
| 59 |
+
# ---------- duration ----------
|
| 60 |
+
duration_sec = 0.0
|
| 61 |
+
|
| 62 |
+
if container.duration is not None:
|
| 63 |
+
# container.duration 单位是 microseconds
|
| 64 |
+
duration_sec = container.duration / 1_000_000
|
| 65 |
+
elif video_stream.duration is not None and video_stream.time_base is not None:
|
| 66 |
+
duration_sec = float(video_stream.duration * video_stream.time_base)
|
| 67 |
+
|
| 68 |
+
if round_duration_ndigits is not None:
|
| 69 |
+
duration_sec = round(duration_sec, round_duration_ndigits)
|
| 70 |
+
|
| 71 |
+
# ---------- width / height / rotation ----------
|
| 72 |
+
w = int(video_stream.codec_context.width or 0)
|
| 73 |
+
h = int(video_stream.codec_context.height or 0)
|
| 74 |
+
|
| 75 |
+
rotation = 0
|
| 76 |
+
if video_stream.metadata:
|
| 77 |
+
rot = video_stream.metadata.get("rotate")
|
| 78 |
+
if rot is not None:
|
| 79 |
+
try:
|
| 80 |
+
rotation = int(rot)
|
| 81 |
+
except Exception:
|
| 82 |
+
rotation = 0
|
| 83 |
+
|
| 84 |
+
if abs(rotation) in (90, 270):
|
| 85 |
+
w, h = h, w
|
| 86 |
+
|
| 87 |
+
# ---------- fps ----------
|
| 88 |
+
fps = 0.0
|
| 89 |
+
if video_stream.average_rate:
|
| 90 |
+
fps = float(video_stream.average_rate)
|
| 91 |
+
elif video_stream.base_rate:
|
| 92 |
+
fps = float(video_stream.base_rate)
|
| 93 |
+
|
| 94 |
+
# ---------- audio ----------
|
| 95 |
+
audio_stream = next(
|
| 96 |
+
(s for s in container.streams if s.type == "audio"),
|
| 97 |
+
None,
|
| 98 |
+
)
|
| 99 |
+
|
| 100 |
+
has_audio = audio_stream is not None
|
| 101 |
+
audio_sample_rate_hz = int(audio_stream.rate) if audio_stream and audio_stream.rate else 0
|
| 102 |
+
|
| 103 |
+
container.close()
|
| 104 |
+
|
| 105 |
+
return {
|
| 106 |
+
"duration": int(duration_sec * 1000), # ms
|
| 107 |
+
"width": w,
|
| 108 |
+
"height": h,
|
| 109 |
+
"fps": fps,
|
| 110 |
+
"has_audio": has_audio,
|
| 111 |
+
"audio_sample_rate_hz": audio_sample_rate_hz,
|
| 112 |
+
}
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
class AssetsIndexNode(BaseNode):
|
| 116 |
+
meta = NodeMeta(
|
| 117 |
+
name="assets_index",
|
| 118 |
+
description="读取输入素材,进行索引,没有前置依赖,但它是所有后续操作的基础。",
|
| 119 |
+
node_id="assets_index",
|
| 120 |
+
node_kind="assets_index",
|
| 121 |
+
next_available_node=['split_shots', 'xhs_split_shots'],
|
| 122 |
+
)
|
| 123 |
+
input_schema: ClassVar[Type[BaseModel]] = AssetsIndexInput
|
| 124 |
+
# output_schema: ClassVar[Type[BaseModel]] = AssetsIndexOutput
|
| 125 |
+
|
| 126 |
+
async def default_process(self, node_summary: NodeSummary, inputs: Dict[str, Any]) -> Dict[str, Any]:
|
| 127 |
+
return await self.process(node_summary, inputs)
|
| 128 |
+
|
| 129 |
+
async def process(self, node_summary: NodeSummary, inputs: Dict[str, Any]) -> Dict[str, Any]:
|
| 130 |
+
input_assets = inputs.get('inputs', [])
|
| 131 |
+
|
| 132 |
+
asset_idx = 1
|
| 133 |
+
assets = []
|
| 134 |
+
for enc_asset in input_assets:
|
| 135 |
+
path = Path(enc_asset['path'])
|
| 136 |
+
suffix = path.suffix.lower()
|
| 137 |
+
|
| 138 |
+
if suffix in VIDEO_EXTS:
|
| 139 |
+
metadata = _video_metadata_from_path(path)
|
| 140 |
+
media_type = "video"
|
| 141 |
+
elif suffix in IMAGE_EXTS:
|
| 142 |
+
metadata = _image_metadata_from_path(path)
|
| 143 |
+
media_type = "image"
|
| 144 |
+
else:
|
| 145 |
+
node_summary.info_for_user(f"[Node {self.meta.node_id}] Skipping unsupported file type `{enc_asset['orig_path']}` ")
|
| 146 |
+
continue
|
| 147 |
+
|
| 148 |
+
assets.append(
|
| 149 |
+
{
|
| 150 |
+
"asset_id": f"asset_{asset_idx:04d}",
|
| 151 |
+
"path": path,
|
| 152 |
+
"media_type": media_type,
|
| 153 |
+
"metadata": metadata,
|
| 154 |
+
"orig_path": enc_asset['orig_path'],
|
| 155 |
+
"orig_md5": enc_asset['orig_md5'],
|
| 156 |
+
}
|
| 157 |
+
)
|
| 158 |
+
node_summary.info_for_user(f"Added asset_{asset_idx:04d}: ({media_type})")
|
| 159 |
+
asset_idx += 1
|
| 160 |
+
|
| 161 |
+
c = Counter(
|
| 162 |
+
(a.get("media_type") or "").strip().lower()
|
| 163 |
+
for a in assets
|
| 164 |
+
if isinstance(a, dict)
|
| 165 |
+
)
|
| 166 |
+
|
| 167 |
+
# node_summary.info_for_user(f"素材索引执行成功:共{c.get('video', 0)}个视频素材,{c.get('image', 0)}个照片素材")
|
| 168 |
+
node_summary.info_for_user(f"[Node {self.meta.node_id}] Media indexing completed successfully: {c.get('video', 0)} video(s), {c.get('image', 0)} image(s)",)
|
| 169 |
+
|
| 170 |
+
return {"assets": assets}
|
src/open_storyline/nodes/core_nodes/base_node.py
ADDED
|
@@ -0,0 +1,243 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from abc import ABC, abstractmethod
|
| 2 |
+
import os
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
from dataclasses import dataclass, field
|
| 5 |
+
from pydantic import BaseModel, ValidationError
|
| 6 |
+
from typing import Any, Dict, List, Optional, Union, ClassVar
|
| 7 |
+
import json
|
| 8 |
+
import traceback
|
| 9 |
+
|
| 10 |
+
from open_storyline.config import Settings
|
| 11 |
+
from open_storyline.nodes.node_summary import NodeSummary
|
| 12 |
+
from open_storyline.storage.file import FileCompressor
|
| 13 |
+
from open_storyline.utils.logging import get_logger
|
| 14 |
+
from open_storyline.mcp.sampling_requester import LLMClient
|
| 15 |
+
|
| 16 |
+
logger = get_logger(__name__)
|
| 17 |
+
|
| 18 |
+
@dataclass
|
| 19 |
+
class NodeMeta:
|
| 20 |
+
"""
|
| 21 |
+
节点元数据类
|
| 22 |
+
|
| 23 |
+
用于定义工作流或流程图中节点的元信息,包括节点标识、类型、
|
| 24 |
+
依赖关系以及可流转的下游节点等配置信息。
|
| 25 |
+
|
| 26 |
+
Attributes:
|
| 27 |
+
name: 工具名
|
| 28 |
+
description: 工具功能描述信息
|
| 29 |
+
node_id: 节点唯一标识符,用于在整个流程中唯一定位该节点
|
| 30 |
+
node_kind: 节点类型/种类,如 "start"、"process"、"end" 等
|
| 31 |
+
require_prior_kind: 当前节点运行前必须完成的前置节点类型列表
|
| 32 |
+
例如:["validation", "authentication"]
|
| 33 |
+
default_require_prior_kind: 默认的前置节点类型列表,作为 require_prior_kind
|
| 34 |
+
的默认配置或备选方案
|
| 35 |
+
next_available_node: 当前节点执行完成后可以流转到的下游节点ID列表
|
| 36 |
+
用于定义流程的可能分支路径
|
| 37 |
+
priority: 相同功能节点执行优先级
|
| 38 |
+
"""
|
| 39 |
+
name: str
|
| 40 |
+
description: str
|
| 41 |
+
node_id: str
|
| 42 |
+
node_kind: str
|
| 43 |
+
require_prior_kind: List[str] = field(default_factory=list)
|
| 44 |
+
default_require_prior_kind: List[str] = field(default_factory=list)
|
| 45 |
+
next_available_node: List[str] = field(default_factory=list)
|
| 46 |
+
priority: int = 5
|
| 47 |
+
# deployment_type: str = "local" # 默认本地,可选 "local" 或 "remote"
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
class BaseNode(ABC):
|
| 52 |
+
meta: NodeMeta
|
| 53 |
+
input_schema: ClassVar[type[BaseModel] | None] = None
|
| 54 |
+
# output_schema: ClassVar[type[BaseModel] | None] = None
|
| 55 |
+
|
| 56 |
+
def __init__(self, server_cfg: Settings) -> None:
|
| 57 |
+
self.server_cfg = server_cfg
|
| 58 |
+
self.server_cache_dir = Path(os.getcwd()) / self.server_cfg.local_mcp_server.server_cache_dir
|
| 59 |
+
|
| 60 |
+
if not hasattr(self, "meta"):
|
| 61 |
+
raise ValueError("Subclass must define the 'meta' attribute")
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def _load_user_info(self, node_summary: NodeSummary, params: Dict[str,Any]) -> Dict[str,Any]:
|
| 65 |
+
return {
|
| 66 |
+
"session_id": params.get("session_id"),
|
| 67 |
+
"artifact_id": params.get('artifact_id')
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
def _load_item(self, node_summary: NodeSummary, user_info: Dict[str,str], item: Dict[str,Any]):
|
| 71 |
+
new_item:Dict[str,Any] = {}
|
| 72 |
+
item_base64 = item.pop("base64", None)
|
| 73 |
+
item_md5 = item.pop("md5", None)
|
| 74 |
+
item_path = item.pop("path", None)
|
| 75 |
+
new_item.update(item)
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
if item_base64 and item_path:
|
| 79 |
+
item_save_path = self.server_cache_dir / user_info['session_id']/ user_info['artifact_id'] / os.path.basename(item_path)
|
| 80 |
+
FileCompressor.decompress_from_string(item_base64, item_save_path)
|
| 81 |
+
new_item['path'] = str(item_save_path.relative_to(os.getcwd()))
|
| 82 |
+
new_item['orig_path'] = str(item_path)
|
| 83 |
+
new_item['orig_md5'] = item_md5
|
| 84 |
+
return new_item
|
| 85 |
+
|
| 86 |
+
def _pack_item(self, node_summary: NodeSummary, item: Dict[str,Any]):
|
| 87 |
+
orig_path = item.pop('orig_path', None)
|
| 88 |
+
orig_md5 = item.pop('orig_md5', None)
|
| 89 |
+
server_save_path = item.pop('path', None)
|
| 90 |
+
if server_save_path:
|
| 91 |
+
compress_data = FileCompressor.compress_and_encode(server_save_path)
|
| 92 |
+
if orig_path and orig_md5 and compress_data.md5 == orig_md5:
|
| 93 |
+
node_summary.debug_for_dev(f"[node] node_id: {self.meta.node_id} change `path` change to {orig_path}")
|
| 94 |
+
item['path'] = orig_path
|
| 95 |
+
elif orig_md5 is None or compress_data.md5 != orig_md5:
|
| 96 |
+
node_summary.debug_for_dev(f"[node] node_id: {self.meta.node_id} return `base64` to client")
|
| 97 |
+
item['base64'] = compress_data.base64
|
| 98 |
+
item['path'] = compress_data.filename
|
| 99 |
+
item['md5'] = compress_data.md5
|
| 100 |
+
return item
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def load_inputs_from_client(self, node_summary: NodeSummary, params: Dict[str,Any], user_info: Optional[Dict[str,str]] = None, save: bool=True) -> Dict[str, Any]:
|
| 104 |
+
"""
|
| 105 |
+
Read data from client's request and save the transmitted base64 data on the Server.
|
| 106 |
+
"""
|
| 107 |
+
|
| 108 |
+
if user_info is None:
|
| 109 |
+
user_info = self._load_user_info(node_summary, params)
|
| 110 |
+
|
| 111 |
+
payload_key = params.keys()
|
| 112 |
+
loaded_input = {}
|
| 113 |
+
kwargs = {}
|
| 114 |
+
for k in payload_key:
|
| 115 |
+
payload_input = params[k]
|
| 116 |
+
if isinstance(payload_input, list) and all([isinstance(item, dict) for item in payload_input]):
|
| 117 |
+
# List: load base64 data and save to server cache
|
| 118 |
+
loaded_input[k] = [self._load_item(node_summary, user_info, item) for item in payload_input]
|
| 119 |
+
elif isinstance(payload_input, dict):
|
| 120 |
+
# Dict: recursively process nested data (without saving)
|
| 121 |
+
loaded_input[k] = self.load_inputs_from_client(node_summary, payload_input, user_info, save=False)
|
| 122 |
+
elif isinstance(payload_input, LLMClient):
|
| 123 |
+
kwargs[k] = payload_input
|
| 124 |
+
else:
|
| 125 |
+
# Handle primitive types: directly copy the value (e.g., str, int, float, bool)
|
| 126 |
+
loaded_input[k] = params[k]
|
| 127 |
+
|
| 128 |
+
# save loaded_input to self.cache_dir in json format
|
| 129 |
+
if save:
|
| 130 |
+
artifact_save_path = (self.server_cache_dir / user_info['session_id'] / user_info['artifact_id']).with_suffix(".json")
|
| 131 |
+
artifact_save_path.parent.mkdir(parents=True, exist_ok=True)
|
| 132 |
+
with open(artifact_save_path, 'w') as f:
|
| 133 |
+
json.dump(loaded_input, f, indent=2, ensure_ascii=False)
|
| 134 |
+
loaded_input.update(kwargs)
|
| 135 |
+
return loaded_input
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def pack_outputs_to_client(self, node_summary: NodeSummary, outputs: Union[Dict[str,Any],List[str]]) -> Union[Dict[str,Any],List[str]]:
|
| 139 |
+
"""
|
| 140 |
+
Pack the output and return it to the client.
|
| 141 |
+
"""
|
| 142 |
+
if not isinstance(outputs, dict):
|
| 143 |
+
return outputs
|
| 144 |
+
payload_key = outputs.keys()
|
| 145 |
+
packed_output = {}
|
| 146 |
+
for k in payload_key:
|
| 147 |
+
payload_output = outputs[k]
|
| 148 |
+
if isinstance(payload_output, list) and all(isinstance(item, dict) for item in payload_output):
|
| 149 |
+
packed_output[k] = [self._pack_item(node_summary, item) for item in payload_output]
|
| 150 |
+
elif isinstance(payload_output, dict):
|
| 151 |
+
packed_output[k] = self.pack_outputs_to_client(node_summary, payload_output)
|
| 152 |
+
else:
|
| 153 |
+
packed_output[k] = outputs[k]
|
| 154 |
+
return packed_output
|
| 155 |
+
|
| 156 |
+
@abstractmethod
|
| 157 |
+
async def default_process(self, node_summary: NodeSummary, inputs: Dict[str, Any]) -> Any:
|
| 158 |
+
"""
|
| 159 |
+
Default processing method that must be implemented. Called when the node needs to be skipped.
|
| 160 |
+
"""
|
| 161 |
+
...
|
| 162 |
+
|
| 163 |
+
@abstractmethod
|
| 164 |
+
async def process(self, node_summary: NodeSummary, inputs: Dict[str, Any]) -> Any:
|
| 165 |
+
"""
|
| 166 |
+
Main processing method that must be implemented. Executed when the node is invoked normally.
|
| 167 |
+
"""
|
| 168 |
+
...
|
| 169 |
+
|
| 170 |
+
def _parse_input(self, node_summary: NodeSummary, inputs: Dict[str, Any]):
|
| 171 |
+
return inputs
|
| 172 |
+
|
| 173 |
+
def _combine_tool_outputs(self, node_summary: NodeSummary, outputs: Dict[str, Any]):
|
| 174 |
+
return outputs
|
| 175 |
+
|
| 176 |
+
def _validate_schema(
|
| 177 |
+
self,
|
| 178 |
+
params: dict[str,Any],
|
| 179 |
+
schema_name: Union[str, List[str]],
|
| 180 |
+
update_params: bool = False
|
| 181 |
+
) -> Optional[Dict[str,Any]]:
|
| 182 |
+
schema_names = [schema_name] if isinstance(schema_name, str) else schema_name
|
| 183 |
+
|
| 184 |
+
validated_params = params.copy() if update_params else None
|
| 185 |
+
|
| 186 |
+
for name in schema_names:
|
| 187 |
+
schema = getattr(self, name, None)
|
| 188 |
+
|
| 189 |
+
if schema is None:
|
| 190 |
+
logger.warning(
|
| 191 |
+
f"Schema '{name}' 不存在,跳过校验",
|
| 192 |
+
)
|
| 193 |
+
continue
|
| 194 |
+
|
| 195 |
+
try:
|
| 196 |
+
validated = schema(**params)
|
| 197 |
+
if update_params:
|
| 198 |
+
validated_params.update(validated.dict())
|
| 199 |
+
except ValidationError as e:
|
| 200 |
+
logger.error(f"{name} 校验失败: {e}")
|
| 201 |
+
return validated_params if update_params else None
|
| 202 |
+
|
| 203 |
+
async def __call__(self, **params) -> Dict[str, Any]:
|
| 204 |
+
try:
|
| 205 |
+
mode = params.get("mode", "auto")
|
| 206 |
+
node_summary = params.pop("node_summary")
|
| 207 |
+
|
| 208 |
+
inputs = self.load_inputs_from_client(node_summary, params.copy())
|
| 209 |
+
|
| 210 |
+
parsed_inputs = self._parse_input(node_summary, inputs)
|
| 211 |
+
|
| 212 |
+
if mode != 'auto':
|
| 213 |
+
outputs = await self.default_process(node_summary, parsed_inputs)
|
| 214 |
+
else:
|
| 215 |
+
outputs = await self.process(node_summary, parsed_inputs)
|
| 216 |
+
|
| 217 |
+
processed_outputs = self._combine_tool_outputs(node_summary, outputs)
|
| 218 |
+
|
| 219 |
+
packed_output = self.pack_outputs_to_client(node_summary, processed_outputs)
|
| 220 |
+
|
| 221 |
+
# self._validate_schema(packed_output, 'output_schema')
|
| 222 |
+
|
| 223 |
+
return {
|
| 224 |
+
'artifact_id': params['artifact_id'],
|
| 225 |
+
'summary': node_summary.get_summary(params['artifact_id']),
|
| 226 |
+
'tool_excute_result': packed_output,
|
| 227 |
+
'isError': False
|
| 228 |
+
}
|
| 229 |
+
except Exception as e:
|
| 230 |
+
if self.server_cfg.project.debug_mode:
|
| 231 |
+
traceback_info = ''.join(traceback.format_exception(e))
|
| 232 |
+
summary = {
|
| 233 |
+
"error_info": f"[artifact_id {params['artifact_id']}] \n {traceback_info}"
|
| 234 |
+
}
|
| 235 |
+
logger.error(traceback_info)
|
| 236 |
+
else:
|
| 237 |
+
summary = node_summary.get_summary(params['artifact_id'])
|
| 238 |
+
return {
|
| 239 |
+
'artifact_id': params['artifact_id'],
|
| 240 |
+
'summary': summary,
|
| 241 |
+
'tool_excute_result': {},
|
| 242 |
+
'isError': True
|
| 243 |
+
}
|
src/open_storyline/nodes/core_nodes/filter_clips.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Any, Dict
|
| 2 |
+
|
| 3 |
+
from open_storyline.nodes.core_nodes.base_node import BaseNode, NodeMeta
|
| 4 |
+
from open_storyline.nodes.node_schema import FilterClipsInput
|
| 5 |
+
from open_storyline.mcp.sampling_requester import LLMClient
|
| 6 |
+
from open_storyline.mcp.sampling_requester import LLMClient
|
| 7 |
+
from src.open_storyline.utils.prompts import get_prompt
|
| 8 |
+
from open_storyline.utils.parse_json import parse_json_dict
|
| 9 |
+
|
| 10 |
+
class FilterClipsNode(BaseNode):
|
| 11 |
+
|
| 12 |
+
meta = NodeMeta(
|
| 13 |
+
name="filter_clips",
|
| 14 |
+
description="按照用户要求,根据clips的描述,对clips进行筛选。依赖 understand_clips 工具的结果",
|
| 15 |
+
node_id="filter_clips",
|
| 16 |
+
node_kind="filter_clips",
|
| 17 |
+
require_prior_kind=['understand_clips'],
|
| 18 |
+
default_require_prior_kind=['understand_clips'],
|
| 19 |
+
next_available_node=['group_clips', 'xhs_group_clips'],
|
| 20 |
+
)
|
| 21 |
+
|
| 22 |
+
input_schema = FilterClipsInput
|
| 23 |
+
|
| 24 |
+
def _parse_input(self, node_summary, inputs: Dict[str, Any]):
|
| 25 |
+
clip_captions = inputs["understand_clips"].get("clip_captions")
|
| 26 |
+
input_clip_ids: list[str] = [
|
| 27 |
+
(c.get("clip_id")) for c in clip_captions
|
| 28 |
+
]
|
| 29 |
+
inputs["input_clip_ids"] = input_clip_ids
|
| 30 |
+
return inputs
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
async def default_process(
|
| 34 |
+
self,
|
| 35 |
+
node_summary,
|
| 36 |
+
inputs: Dict[str, Any],
|
| 37 |
+
) -> Any:
|
| 38 |
+
clip_captions = inputs["understand_clips"].get("clip_captions")
|
| 39 |
+
|
| 40 |
+
node_summary.info_for_user("使用全部clips")
|
| 41 |
+
return {
|
| 42 |
+
"clip_captions": clip_captions,
|
| 43 |
+
"selected": inputs["input_clip_ids"],
|
| 44 |
+
}
|
| 45 |
+
|
| 46 |
+
async def process(self, node_summary, inputs: Dict[str, Any]) -> Any:
|
| 47 |
+
clip_captions = inputs["understand_clips"].get("clip_captions")
|
| 48 |
+
user_request = inputs["user_request"]
|
| 49 |
+
llm: LLMClient = inputs["llm"]
|
| 50 |
+
|
| 51 |
+
input_clip_ids = inputs["input_clip_ids"]
|
| 52 |
+
|
| 53 |
+
if not user_request or user_request == "":
|
| 54 |
+
node_summary.info_for_user("用户没有提出具体需求,使用全部clips")
|
| 55 |
+
return {
|
| 56 |
+
"clip_captions": clip_captions,
|
| 57 |
+
"selected": input_clip_ids,
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
else:
|
| 61 |
+
clip_block = _build_clips_block(clip_captions)
|
| 62 |
+
system_prompt = get_prompt("filter_clips.system")
|
| 63 |
+
user_prompt = get_prompt("filter_clips.user", user_request=user_request, clip_captions=clip_block)
|
| 64 |
+
|
| 65 |
+
raw = await llm.complete(
|
| 66 |
+
system_prompt=system_prompt,
|
| 67 |
+
user_prompt=user_prompt,
|
| 68 |
+
media=None,
|
| 69 |
+
temperature=0.1,
|
| 70 |
+
top_p=0.9,
|
| 71 |
+
max_tokens=2048,
|
| 72 |
+
model_preferences=None,
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
try:
|
| 76 |
+
obj = parse_json_dict(raw)
|
| 77 |
+
select_ids = _extract_selected_ids(obj, input_clip_ids)
|
| 78 |
+
node_summary.info_for_user(f"成功筛选到{len(select_ids)}个片段")
|
| 79 |
+
|
| 80 |
+
except:
|
| 81 |
+
select_ids = input_clip_ids
|
| 82 |
+
node_summary.info_for_user("解析大模型输出出错,使用全部片段")
|
| 83 |
+
|
| 84 |
+
return {
|
| 85 |
+
"clip_captions": clip_captions,
|
| 86 |
+
"selected": select_ids,
|
| 87 |
+
}
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def _extract_selected_ids(
|
| 91 |
+
obj: dict[str, Any],
|
| 92 |
+
input_clip_ids: list[str],
|
| 93 |
+
) -> list[str]:
|
| 94 |
+
"""
|
| 95 |
+
从 LLM 的结构化输出中抽取 selected clip_id 列表。
|
| 96 |
+
返回:按 input_clip_ids 原顺序的去噪结果(仅保留输入中存在的 id)
|
| 97 |
+
"""
|
| 98 |
+
id_set = set(input_clip_ids)
|
| 99 |
+
|
| 100 |
+
# --- Case 2: results keep ---
|
| 101 |
+
results = obj.get("results")
|
| 102 |
+
if not isinstance(results, list):
|
| 103 |
+
raise ValueError('"results" 必须是 list')
|
| 104 |
+
|
| 105 |
+
true_items = 0
|
| 106 |
+
valid_true_ids: set[str] = set()
|
| 107 |
+
|
| 108 |
+
for item in results:
|
| 109 |
+
if not isinstance(item, dict):
|
| 110 |
+
continue
|
| 111 |
+
cid = item.get("clip_id")
|
| 112 |
+
keep = item.get("keep")
|
| 113 |
+
|
| 114 |
+
# keep 允许 bool 或 "true"/"false"
|
| 115 |
+
keep_bool = None
|
| 116 |
+
if isinstance(keep, bool):
|
| 117 |
+
keep_bool = keep
|
| 118 |
+
elif isinstance(keep, str):
|
| 119 |
+
s = keep.strip().lower()
|
| 120 |
+
if s in ("true", "yes", "1"):
|
| 121 |
+
keep_bool = True
|
| 122 |
+
elif s in ("false", "no", "0"):
|
| 123 |
+
keep_bool = False
|
| 124 |
+
|
| 125 |
+
if keep_bool is True:
|
| 126 |
+
true_items += 1
|
| 127 |
+
if isinstance(cid, str) and cid in id_set:
|
| 128 |
+
valid_true_ids.add(cid)
|
| 129 |
+
|
| 130 |
+
# 如果模型明确选了(keep=true)但一个都对不上输入
|
| 131 |
+
if true_items > 0 and not valid_true_ids:
|
| 132 |
+
raise ValueError("results 中存在 keep=true,但没有任何有效 clip_id(可能模型改写了 ID)")
|
| 133 |
+
|
| 134 |
+
return [cid for cid in input_clip_ids if cid in valid_true_ids]
|
| 135 |
+
|
| 136 |
+
def _build_clips_block(clip_captions: list[dict[str, Any]]) -> str:
|
| 137 |
+
"""
|
| 138 |
+
将 clips 构造成稳定的文本块
|
| 139 |
+
"""
|
| 140 |
+
blocks: list[str] = []
|
| 141 |
+
for clip in clip_captions:
|
| 142 |
+
cid = clip.get("clip_id", "")
|
| 143 |
+
caption = clip.get("caption", "")
|
| 144 |
+
block = (
|
| 145 |
+
f"[clip_id={cid}]\n"
|
| 146 |
+
f"caption: {caption}\n"
|
| 147 |
+
)
|
| 148 |
+
blocks.append(block)
|
| 149 |
+
return "\n".join(blocks).strip() + "\n"
|
src/open_storyline/nodes/core_nodes/generate_script.py
ADDED
|
@@ -0,0 +1,306 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Any, Dict
|
| 2 |
+
import re
|
| 3 |
+
|
| 4 |
+
from open_storyline.mcp.sampling_requester import LLMClient
|
| 5 |
+
from open_storyline.nodes.core_nodes.base_node import BaseNode, NodeMeta
|
| 6 |
+
from open_storyline.nodes.node_schema import GenerateScriptInput
|
| 7 |
+
from src.open_storyline.utils.prompts import get_prompt
|
| 8 |
+
from open_storyline.utils.parse_json import parse_json_dict
|
| 9 |
+
|
| 10 |
+
class GenerateScriptNode(BaseNode):
|
| 11 |
+
meta = NodeMeta(
|
| 12 |
+
name="generate_script",
|
| 13 |
+
description="生成视频文案,可用于生成语音,或直接作为字幕叠加到视频上",
|
| 14 |
+
node_id="generate_script",
|
| 15 |
+
node_kind="generate_script",
|
| 16 |
+
require_prior_kind=['split_shots','group_clips','understand_clips'],
|
| 17 |
+
default_require_prior_kind=['split_shots','group_clips'],
|
| 18 |
+
next_available_node=['generate_voiceover', 'xhs_tts_service'],
|
| 19 |
+
)
|
| 20 |
+
|
| 21 |
+
input_schema = GenerateScriptInput
|
| 22 |
+
|
| 23 |
+
async def default_process(
|
| 24 |
+
self,
|
| 25 |
+
node_summary,
|
| 26 |
+
inputs: Dict[str, Any],
|
| 27 |
+
) -> Any:
|
| 28 |
+
return {
|
| 29 |
+
"group_scripts": [],
|
| 30 |
+
"title": "",
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
async def process(self, node_summary, inputs: Dict[str, Any]) -> Any:
|
| 35 |
+
clip_info = inputs["split_shots"]["clips"]
|
| 36 |
+
clip_captions = inputs["understand_clips"]["clip_captions"]
|
| 37 |
+
overall = inputs["understand_clips"]["overall"]
|
| 38 |
+
groups = inputs["group_clips"]["groups"]
|
| 39 |
+
user_request = inputs["user_request"]
|
| 40 |
+
llm: LLMClient = inputs["llm"]
|
| 41 |
+
|
| 42 |
+
duration_lookup = _build_duration_lookup(clip_info)
|
| 43 |
+
caption_lookup = _build_caption_lookup(clip_captions)
|
| 44 |
+
|
| 45 |
+
group_ids: list[str] = [g.get("group_id","") for g in (groups or []) if g.get("group_id")]
|
| 46 |
+
group_ids_set = set(group_ids)
|
| 47 |
+
|
| 48 |
+
if not group_ids:
|
| 49 |
+
return {"group_scripts": []}
|
| 50 |
+
|
| 51 |
+
groups_block = _build_groups_block_for_script(groups, duration_lookup, caption_lookup)
|
| 52 |
+
|
| 53 |
+
system_prompt = get_prompt("generate_script.system")
|
| 54 |
+
if not user_request or user_request == "":
|
| 55 |
+
user_request = "无要求"
|
| 56 |
+
user_prompt = get_prompt("generate_script.user", user_request=user_request, overall=overall, groups=groups_block)
|
| 57 |
+
|
| 58 |
+
raw = await llm.complete(
|
| 59 |
+
system_prompt=system_prompt,
|
| 60 |
+
user_prompt=user_prompt,
|
| 61 |
+
temperature=0.1,
|
| 62 |
+
top_p=0.9,
|
| 63 |
+
max_tokens=4096,
|
| 64 |
+
model_preferences=None,
|
| 65 |
+
)
|
| 66 |
+
group_text_map: dict[str, str] = {}
|
| 67 |
+
try:
|
| 68 |
+
obj = parse_json_dict(raw)
|
| 69 |
+
group_text_map = _extract_group_text_map(obj, group_ids)
|
| 70 |
+
except Exception:
|
| 71 |
+
group_text_map = {}
|
| 72 |
+
|
| 73 |
+
# 最终 group_scripts
|
| 74 |
+
group_scripts: list[dict[str, Any]] = []
|
| 75 |
+
subtitle_index = 1
|
| 76 |
+
|
| 77 |
+
for g in groups or []:
|
| 78 |
+
gid = g.get("group_id", "")
|
| 79 |
+
if not gid or gid not in group_ids_set:
|
| 80 |
+
continue
|
| 81 |
+
|
| 82 |
+
duration_sec = 0.0
|
| 83 |
+
for cid in (g.get("clip_ids") or []):
|
| 84 |
+
duration_sec += float(duration_lookup.get(cid, 0.0))
|
| 85 |
+
budget = _estimate_script_budget(duration_sec)
|
| 86 |
+
|
| 87 |
+
raw_text = (group_text_map.get(gid) or "").strip()
|
| 88 |
+
if not raw_text:
|
| 89 |
+
raise ValueError(f"LLM未生成任何文案,请重试")
|
| 90 |
+
|
| 91 |
+
max_chars = budget.get("max_chars", 60)
|
| 92 |
+
if len(raw_text) > int(max_chars * 2.0):
|
| 93 |
+
raw_text = raw_text[:max_chars].rstrip()
|
| 94 |
+
|
| 95 |
+
units, subtitle_index = _make_subtitle_units(
|
| 96 |
+
raw_text=raw_text,
|
| 97 |
+
subtitle_start_index=subtitle_index,
|
| 98 |
+
)
|
| 99 |
+
|
| 100 |
+
group_scripts.append(
|
| 101 |
+
{
|
| 102 |
+
"group_id": gid,
|
| 103 |
+
"raw_text": raw_text,
|
| 104 |
+
"subtitle_units": units,
|
| 105 |
+
}
|
| 106 |
+
)
|
| 107 |
+
|
| 108 |
+
return {
|
| 109 |
+
"group_scripts": group_scripts,
|
| 110 |
+
"title": obj.get("title", ""),
|
| 111 |
+
}
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def _build_duration_lookup(clip_info: list[dict[str, Any]]) -> dict[str, float]:
|
| 116 |
+
"""
|
| 117 |
+
clip_id -> duration_sec
|
| 118 |
+
"""
|
| 119 |
+
out: dict[str, float] = {}
|
| 120 |
+
for item in clip_info or []:
|
| 121 |
+
cid = item.get("clip_id")
|
| 122 |
+
if not cid:
|
| 123 |
+
continue
|
| 124 |
+
src = item.get("source_ref") or {}
|
| 125 |
+
dur = src.get("duration", 0) / 1000.0
|
| 126 |
+
out[cid] = dur
|
| 127 |
+
return out
|
| 128 |
+
|
| 129 |
+
def _build_caption_lookup(clip_captions: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
| 130 |
+
"""
|
| 131 |
+
clip_id -> caption_obj
|
| 132 |
+
"""
|
| 133 |
+
|
| 134 |
+
out: dict[str, dict[str, Any]] = {}
|
| 135 |
+
for item in clip_captions:
|
| 136 |
+
if not isinstance(item, dict):
|
| 137 |
+
continue
|
| 138 |
+
cid = item.get("clip_id")
|
| 139 |
+
if cid:
|
| 140 |
+
out[cid] = item
|
| 141 |
+
return out
|
| 142 |
+
|
| 143 |
+
def _estimate_script_budget(duration_sec: float) -> dict[str, Any]:
|
| 144 |
+
"""
|
| 145 |
+
根据组时长估算“字数/字符数预算”和建议字幕分段数。
|
| 146 |
+
"""
|
| 147 |
+
if duration_sec is None:
|
| 148 |
+
duration_sec = 0.0
|
| 149 |
+
duration_sec = max(0.0, float(duration_sec))
|
| 150 |
+
|
| 151 |
+
min_chars = int(round(duration_sec * 3))
|
| 152 |
+
max_chars = int(round(duration_sec * 5))
|
| 153 |
+
|
| 154 |
+
# 防止极短组变成 0
|
| 155 |
+
min_chars = max(min_chars, 8)
|
| 156 |
+
max_chars = max(max_chars, min_chars + 6)
|
| 157 |
+
|
| 158 |
+
# 建议字幕单元数量:大约每 2 秒 1 句
|
| 159 |
+
target_units = max(1, int(round(duration_sec / 2.0)))
|
| 160 |
+
target_units = min(target_units, 8) # 防止过多切片
|
| 161 |
+
|
| 162 |
+
return {
|
| 163 |
+
"duration_sec": duration_sec,
|
| 164 |
+
"min_chars": min_chars,
|
| 165 |
+
"max_chars": max_chars,
|
| 166 |
+
"target_units": target_units,
|
| 167 |
+
}
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def _build_groups_block_for_script(
|
| 171 |
+
groups: list[dict[str, Any]],
|
| 172 |
+
duration_lookup: dict[str, float],
|
| 173 |
+
caption_lookup: dict[str, dict[str, Any]],
|
| 174 |
+
*,
|
| 175 |
+
max_caption_len: int = 120,
|
| 176 |
+
) -> str:
|
| 177 |
+
"""
|
| 178 |
+
将 groups + clip captions + 时长预算拼成稳定文本块,给 LLM 生成每组文案。
|
| 179 |
+
"""
|
| 180 |
+
blocks: list[str] = []
|
| 181 |
+
|
| 182 |
+
for g in groups or []:
|
| 183 |
+
gid = g.get("group_id", "")
|
| 184 |
+
clip_ids = g.get("clip_ids") or []
|
| 185 |
+
if not gid or not isinstance(clip_ids, list) or not clip_ids:
|
| 186 |
+
continue
|
| 187 |
+
|
| 188 |
+
group_summary = (g.get("summary") or "").strip()
|
| 189 |
+
|
| 190 |
+
# 组时长
|
| 191 |
+
duration_sec = 0.0
|
| 192 |
+
for cid in clip_ids:
|
| 193 |
+
duration_sec += float(duration_lookup.get(cid, 0.0))
|
| 194 |
+
budget = _estimate_script_budget(duration_sec)
|
| 195 |
+
|
| 196 |
+
lines: list[str] = []
|
| 197 |
+
lines.append(f"[group_id={gid}]")
|
| 198 |
+
if group_summary:
|
| 199 |
+
lines.append(f"summary: {group_summary}")
|
| 200 |
+
lines.append(f"duration_sec: {budget['duration_sec']:.2f}")
|
| 201 |
+
lines.append(f"script_chars_budget: {budget['min_chars']}~{budget['max_chars']}")
|
| 202 |
+
lines.append(f"subtitle_units_target: {budget['target_units']} (用逗号/句号分段)")
|
| 203 |
+
|
| 204 |
+
lines.append("clips:")
|
| 205 |
+
for cid in clip_ids:
|
| 206 |
+
cap_obj = caption_lookup.get(cid, {})
|
| 207 |
+
cap_text = cap_obj.get("caption", "")
|
| 208 |
+
sem = cap_obj.get("semantic") or {}
|
| 209 |
+
kw = sem.get("keywords") or []
|
| 210 |
+
mood = sem.get("mood") or []
|
| 211 |
+
kw_s = "、".join([x for x in kw if isinstance(x, str)])[:40]
|
| 212 |
+
mood_s = "、".join([x for x in mood if isinstance(x, str)])[:30]
|
| 213 |
+
|
| 214 |
+
dur = duration_lookup.get(cid, 0.0)
|
| 215 |
+
|
| 216 |
+
lines.append(f"- {cid} ({dur:.2f}s): {cap_text}")
|
| 217 |
+
if kw_s or mood_s:
|
| 218 |
+
lines.append(f" tags_hint: keywords={kw_s} | mood={mood_s}")
|
| 219 |
+
|
| 220 |
+
blocks.append("\n".join(lines))
|
| 221 |
+
|
| 222 |
+
return "\n\n".join(blocks).strip()
|
| 223 |
+
|
| 224 |
+
def _extract_group_text_map(obj: Any, group_ids: list[str]) -> dict[str, str]:
|
| 225 |
+
"""
|
| 226 |
+
从 LLM JSON 中抽取 {group_id: raw_text} 映射。
|
| 227 |
+
兼容几种常见输出:
|
| 228 |
+
1) {"scripts":[{"group_id":"group_0001","raw_text":"..."}, ...]}
|
| 229 |
+
2) {"group_scripts":[{"group_id":"...","raw_text":"..."}, ...]}
|
| 230 |
+
3) {"group_0001":"...", "group_0002":"..."}
|
| 231 |
+
4) [{"group_id":"...","raw_text":"..."}]
|
| 232 |
+
"""
|
| 233 |
+
gid_set = set(group_ids)
|
| 234 |
+
out: dict[str, str] = {}
|
| 235 |
+
|
| 236 |
+
def _add(gid: Any, text: Any):
|
| 237 |
+
if isinstance(gid, str) and gid in gid_set and isinstance(text, str) and text.strip():
|
| 238 |
+
out[gid] = text.strip()
|
| 239 |
+
|
| 240 |
+
if isinstance(obj, dict):
|
| 241 |
+
# list 型
|
| 242 |
+
for key in ("scripts", "group_scripts", "results"):
|
| 243 |
+
v = obj.get(key)
|
| 244 |
+
if isinstance(v, list):
|
| 245 |
+
for item in v:
|
| 246 |
+
if isinstance(item, dict):
|
| 247 |
+
gid = item.get("group_id")
|
| 248 |
+
text = item.get("raw_text") or item.get("text") or item.get("script")
|
| 249 |
+
_add(gid, text)
|
| 250 |
+
|
| 251 |
+
# mapping 型:{"group_0001":"..."}
|
| 252 |
+
for gid in group_ids:
|
| 253 |
+
if gid in obj and isinstance(obj[gid], str):
|
| 254 |
+
_add(gid, obj[gid])
|
| 255 |
+
|
| 256 |
+
return out
|
| 257 |
+
|
| 258 |
+
if isinstance(obj, list):
|
| 259 |
+
for item in obj:
|
| 260 |
+
if isinstance(item, dict):
|
| 261 |
+
gid = item.get("group_id")
|
| 262 |
+
text = item.get("raw_text") or item.get("text") or item.get("script")
|
| 263 |
+
_add(gid, text)
|
| 264 |
+
return out
|
| 265 |
+
|
| 266 |
+
raise ValueError("无法识别 LLM 输出结构")
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
_SPLIT_RE = re.compile(r"[,,。]+")
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
def _split_by_comma(raw_text: str) -> list[str]:
|
| 273 |
+
"""
|
| 274 |
+
逗号拆分:支持中文/英文逗号和中文句号。去掉空段。
|
| 275 |
+
"""
|
| 276 |
+
if not isinstance(raw_text, str):
|
| 277 |
+
return []
|
| 278 |
+
s = raw_text.strip().replace("\n", ",")
|
| 279 |
+
parts = [p.strip() for p in _SPLIT_RE.split(s) if p and p.strip()]
|
| 280 |
+
return parts
|
| 281 |
+
|
| 282 |
+
|
| 283 |
+
def _make_subtitle_units(
|
| 284 |
+
raw_text: str,
|
| 285 |
+
subtitle_start_index: int,
|
| 286 |
+
) -> tuple[list[dict[str, Any]], int]:
|
| 287 |
+
"""
|
| 288 |
+
为某个 group 生成 subtitle_units,返回 (units, next_global_index)
|
| 289 |
+
unit_id 全局递增:subtitle_0001, subtitle_0002 ...
|
| 290 |
+
"""
|
| 291 |
+
parts = _split_by_comma(raw_text)
|
| 292 |
+
if not parts and raw_text.strip():
|
| 293 |
+
parts = [raw_text.strip()]
|
| 294 |
+
|
| 295 |
+
units: list[dict[str, Any]] = []
|
| 296 |
+
cur = subtitle_start_index
|
| 297 |
+
for idx_in_group, text in enumerate(parts):
|
| 298 |
+
units.append(
|
| 299 |
+
{
|
| 300 |
+
"unit_id": f"subtitle_{cur:04d}",
|
| 301 |
+
"index_in_group": idx_in_group,
|
| 302 |
+
"text": text,
|
| 303 |
+
}
|
| 304 |
+
)
|
| 305 |
+
cur += 1
|
| 306 |
+
return units, cur
|
src/open_storyline/nodes/core_nodes/generate_voiceover.py
ADDED
|
@@ -0,0 +1,353 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
+
import base64
|
| 3 |
+
import time
|
| 4 |
+
import uuid
|
| 5 |
+
import wave
|
| 6 |
+
from dataclasses import dataclass
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
from typing import Any, Dict, Optional, Callable
|
| 9 |
+
|
| 10 |
+
import requests
|
| 11 |
+
|
| 12 |
+
from open_storyline.nodes.core_nodes.base_node import BaseNode, NodeMeta
|
| 13 |
+
from open_storyline.nodes.node_schema import GenerateVoiceoverInput
|
| 14 |
+
|
| 15 |
+
class TTSConfigError(ValueError):
|
| 16 |
+
"""TTS 配置缺失或不合法"""
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class TTSRequestError(RuntimeError):
|
| 20 |
+
"""TTS 请求失败(网络/服务端/返回格式等)"""
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def _mask_secret(s: str, keep_tail: int = 4) -> str:
|
| 24 |
+
s = str(s or "")
|
| 25 |
+
if not s:
|
| 26 |
+
return ""
|
| 27 |
+
if len(s) <= keep_tail:
|
| 28 |
+
return "***"
|
| 29 |
+
return "***" + s[-keep_tail:]
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def _wav_duration_ms(wav_path: Path) -> int:
|
| 33 |
+
with wave.open(str(wav_path), "rb") as wf:
|
| 34 |
+
frames = wf.getnframes()
|
| 35 |
+
rate = wf.getframerate()
|
| 36 |
+
return int((frames / float(rate) * 1000)) if rate else 0
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
# -----------------------------
|
| 40 |
+
# Provider abstraction + registry
|
| 41 |
+
# -----------------------------
|
| 42 |
+
class BaseTTSProvider:
|
| 43 |
+
"""
|
| 44 |
+
统一的 TTS Provider 抽象。
|
| 45 |
+
- 新增厂家时:实现一个 Provider(和可选的 Config),并注册到 _TTS_PROVIDER_BUILDERS。
|
| 46 |
+
"""
|
| 47 |
+
vendor: str = "unknown"
|
| 48 |
+
|
| 49 |
+
def synthesize_to_wav_sync(self, *, text: str, wav_path: Path) -> None:
|
| 50 |
+
"""同步生成 wav(注意:会被 asyncio.to_thread 包装,不阻塞事件循环)"""
|
| 51 |
+
raise NotImplementedError
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _normalize_vendor(v: Optional[str]) -> str:
|
| 55 |
+
v = (v or "").strip().lower()
|
| 56 |
+
if not v:
|
| 57 |
+
return "bytedance"
|
| 58 |
+
aliases = {
|
| 59 |
+
"byte": "bytedance",
|
| 60 |
+
"bytedance": "bytedance",
|
| 61 |
+
"volc": "bytedance",
|
| 62 |
+
"volcengine": "bytedance",
|
| 63 |
+
"volcano": "bytedance",
|
| 64 |
+
}
|
| 65 |
+
return aliases.get(v, v)
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def _merge_config(inputs: Dict[str, Any]) -> Dict[str, Any]:
|
| 69 |
+
"""
|
| 70 |
+
把可能出现的配置来源合并成一个 dict,便于兼容:
|
| 71 |
+
- 顶层:uid/appid/access_token/tts_vendor...
|
| 72 |
+
- inputs["tts_config"](通用配置容器)
|
| 73 |
+
- inputs["bytedance_tts"](厂家专用容器,未来可按 vendor 增加)
|
| 74 |
+
"""
|
| 75 |
+
merged: Dict[str, Any] = {}
|
| 76 |
+
# 1) 通用容器
|
| 77 |
+
tts_cfg = inputs.get("tts_config")
|
| 78 |
+
if isinstance(tts_cfg, dict):
|
| 79 |
+
merged.update(tts_cfg)
|
| 80 |
+
|
| 81 |
+
# 2) 厂家专用容器(目前只实现 bytedance)
|
| 82 |
+
bd_cfg = inputs.get("bytedance_tts")
|
| 83 |
+
if isinstance(bd_cfg, dict):
|
| 84 |
+
merged.update(bd_cfg)
|
| 85 |
+
|
| 86 |
+
# 3) 顶层覆盖(优先级最高)
|
| 87 |
+
merged.update(inputs)
|
| 88 |
+
return merged
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def _pick_str(cfg: Dict[str, Any], *keys: str) -> str:
|
| 92 |
+
for k in keys:
|
| 93 |
+
v = cfg.get(k)
|
| 94 |
+
if isinstance(v, str) and v.strip():
|
| 95 |
+
return v.strip()
|
| 96 |
+
return ""
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def _pick_int(cfg: Dict[str, Any], key: str, default: int) -> int:
|
| 100 |
+
v = cfg.get(key, default)
|
| 101 |
+
try:
|
| 102 |
+
return int(v)
|
| 103 |
+
except Exception:
|
| 104 |
+
return int(default)
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def _pick_float(cfg: Dict[str, Any], key: str, default: float) -> float:
|
| 108 |
+
v = cfg.get(key, default)
|
| 109 |
+
try:
|
| 110 |
+
return float(v)
|
| 111 |
+
except Exception:
|
| 112 |
+
return float(default)
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
# -----------------------------
|
| 116 |
+
# ByteDance provider
|
| 117 |
+
# -----------------------------
|
| 118 |
+
@dataclass(frozen=True)
|
| 119 |
+
class ByteDanceTTSConfig:
|
| 120 |
+
uid: str
|
| 121 |
+
appid: str
|
| 122 |
+
access_token: str
|
| 123 |
+
|
| 124 |
+
cluster: str = "volcano_tts"
|
| 125 |
+
voice_type: str = "BV002_streaming"
|
| 126 |
+
encoding: str = "wav"
|
| 127 |
+
speed: int = 10
|
| 128 |
+
volume: int = 10
|
| 129 |
+
pitch: int = 10
|
| 130 |
+
|
| 131 |
+
host: str = "openspeech.bytedance.com"
|
| 132 |
+
api_path: str = "/api/v1/tts"
|
| 133 |
+
timeout_sec: float = 30.0
|
| 134 |
+
|
| 135 |
+
@property
|
| 136 |
+
def api_url(self) -> str:
|
| 137 |
+
return f"https://{self.host}{self.api_path}"
|
| 138 |
+
|
| 139 |
+
@classmethod
|
| 140 |
+
def from_inputs(cls, inputs: Dict[str, Any]) -> "ByteDanceTTSConfig":
|
| 141 |
+
cfg = _merge_config(inputs)
|
| 142 |
+
|
| 143 |
+
uid = _pick_str(cfg, "uid", "tts_uid")
|
| 144 |
+
appid = _pick_str(cfg, "appid", "tts_appid")
|
| 145 |
+
access_token = _pick_str(cfg, "access_token", "tts_access_token", "token")
|
| 146 |
+
|
| 147 |
+
if not uid:
|
| 148 |
+
raise TTSConfigError("ByteDance TTS 缺失必填字段:uid")
|
| 149 |
+
if not appid:
|
| 150 |
+
raise TTSConfigError("ByteDance TTS 缺失必填字段:appid")
|
| 151 |
+
if not access_token:
|
| 152 |
+
raise TTSConfigError("ByteDance TTS 缺失必填字段:access_token")
|
| 153 |
+
|
| 154 |
+
cluster = _pick_str(cfg, "cluster") or "volcano_tts"
|
| 155 |
+
voice_type = _pick_str(cfg, "voice_type") or "BV002_streaming"
|
| 156 |
+
encoding = _pick_str(cfg, "encoding") or "wav"
|
| 157 |
+
|
| 158 |
+
host = _pick_str(cfg, "host") or "openspeech.bytedance.com"
|
| 159 |
+
api_path = _pick_str(cfg, "api_path", "path") or "/api/v1/tts"
|
| 160 |
+
|
| 161 |
+
timeout_sec = _pick_float(cfg, "timeout_sec", 30.0)
|
| 162 |
+
|
| 163 |
+
speed = _pick_int(cfg, "speed", 10)
|
| 164 |
+
volume = _pick_int(cfg, "volume", 10)
|
| 165 |
+
pitch = _pick_int(cfg, "pitch", 10)
|
| 166 |
+
|
| 167 |
+
return cls(
|
| 168 |
+
uid=uid,
|
| 169 |
+
appid=appid,
|
| 170 |
+
access_token=access_token,
|
| 171 |
+
cluster=cluster,
|
| 172 |
+
voice_type=voice_type,
|
| 173 |
+
encoding=encoding,
|
| 174 |
+
speed=speed,
|
| 175 |
+
volume=volume,
|
| 176 |
+
pitch=pitch,
|
| 177 |
+
host=host,
|
| 178 |
+
api_path=api_path,
|
| 179 |
+
timeout_sec=timeout_sec,
|
| 180 |
+
)
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
class ByteDanceTTSProvider(BaseTTSProvider):
|
| 184 |
+
vendor = "bytedance"
|
| 185 |
+
|
| 186 |
+
def __init__(self, cfg: ByteDanceTTSConfig):
|
| 187 |
+
self.cfg = cfg
|
| 188 |
+
self._session = requests.Session()
|
| 189 |
+
|
| 190 |
+
def synthesize_to_wav_sync(self, *, text: str, wav_path: Path) -> None:
|
| 191 |
+
cfg = self.cfg
|
| 192 |
+
|
| 193 |
+
headers = {
|
| 194 |
+
"Authorization": f"Bearer; {cfg.access_token}",
|
| 195 |
+
"Content-Type": "application/json",
|
| 196 |
+
}
|
| 197 |
+
|
| 198 |
+
request_json = {
|
| 199 |
+
"app": {
|
| 200 |
+
"appid": cfg.appid,
|
| 201 |
+
"token": cfg.access_token,
|
| 202 |
+
"cluster": cfg.cluster,
|
| 203 |
+
},
|
| 204 |
+
"user": {
|
| 205 |
+
"uid": cfg.uid,
|
| 206 |
+
},
|
| 207 |
+
"audio": {
|
| 208 |
+
"voice": "other",
|
| 209 |
+
"voice_type": cfg.voice_type,
|
| 210 |
+
"encoding": cfg.encoding,
|
| 211 |
+
"speed": cfg.speed,
|
| 212 |
+
"volume": cfg.volume,
|
| 213 |
+
"pitch": cfg.pitch,
|
| 214 |
+
},
|
| 215 |
+
"request": {
|
| 216 |
+
"reqid": str(uuid.uuid4()),
|
| 217 |
+
"text": text,
|
| 218 |
+
"text_type": "plain",
|
| 219 |
+
"operation": "query",
|
| 220 |
+
"with_frontend": 1,
|
| 221 |
+
"frontend_type": "unitTson",
|
| 222 |
+
},
|
| 223 |
+
}
|
| 224 |
+
|
| 225 |
+
try:
|
| 226 |
+
resp = self._session.post(
|
| 227 |
+
cfg.api_url,
|
| 228 |
+
json=request_json,
|
| 229 |
+
headers=headers,
|
| 230 |
+
timeout=cfg.timeout_sec,
|
| 231 |
+
)
|
| 232 |
+
resp.raise_for_status()
|
| 233 |
+
except Exception as e:
|
| 234 |
+
raise TTSRequestError(
|
| 235 |
+
f"ByteDance TTS 网络请求失败:{type(e).__name__}: {e}(token={_mask_secret(cfg.access_token)})"
|
| 236 |
+
) from e
|
| 237 |
+
|
| 238 |
+
try:
|
| 239 |
+
resp_json = resp.json()
|
| 240 |
+
except Exception as e:
|
| 241 |
+
raise TTSRequestError(
|
| 242 |
+
f"ByteDance TTS 返回非 JSON:{type(e).__name__}: {e}"
|
| 243 |
+
) from e
|
| 244 |
+
|
| 245 |
+
if resp_json.get("message") != "Success":
|
| 246 |
+
code = resp_json.get("code")
|
| 247 |
+
msg = resp_json.get("message") or "Unknown"
|
| 248 |
+
raise TTSRequestError(f"ByteDance TTS 失败:code={code} message={msg}")
|
| 249 |
+
|
| 250 |
+
data_b64 = resp_json.get("data")
|
| 251 |
+
if not isinstance(data_b64, str) or not data_b64.strip():
|
| 252 |
+
raise TTSRequestError("ByteDance TTS 返回缺失 data 字段(音频内容为空)")
|
| 253 |
+
|
| 254 |
+
wav_path.parent.mkdir(parents=True, exist_ok=True)
|
| 255 |
+
try:
|
| 256 |
+
wav_bytes = base64.b64decode(data_b64)
|
| 257 |
+
wav_path.write_bytes(wav_bytes)
|
| 258 |
+
except Exception as e:
|
| 259 |
+
raise TTSRequestError(f"写入 wav 失败:{type(e).__name__}: {e}") from e
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
# ---- Provider registry ----
|
| 263 |
+
_TTS_PROVIDER_BUILDERS: Dict[str, Callable[[Dict[str, Any]], BaseTTSProvider]] = {
|
| 264 |
+
"bytedance": lambda inputs: ByteDanceTTSProvider(ByteDanceTTSConfig.from_inputs(inputs)),
|
| 265 |
+
}
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
def build_tts_provider(inputs: Dict[str, Any]) -> BaseTTSProvider:
|
| 269 |
+
cfg = _merge_config(inputs)
|
| 270 |
+
vendor = _normalize_vendor(_pick_str(cfg, "tts_vendor", "vendor", "tts_provider"))
|
| 271 |
+
|
| 272 |
+
builder = _TTS_PROVIDER_BUILDERS.get(vendor)
|
| 273 |
+
if not builder:
|
| 274 |
+
supported = ", ".join(sorted(_TTS_PROVIDER_BUILDERS.keys()))
|
| 275 |
+
raise TTSConfigError(f"不支持的 TTS vendor:{vendor}(当前支持:{supported})")
|
| 276 |
+
|
| 277 |
+
return builder(inputs)
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
# -----------------------------
|
| 281 |
+
# Node implementation
|
| 282 |
+
# -----------------------------
|
| 283 |
+
class GenerateVoiceoverNode(BaseNode):
|
| 284 |
+
meta = NodeMeta(
|
| 285 |
+
name="generate_voiceover",
|
| 286 |
+
description="根据文案生成配音",
|
| 287 |
+
node_id="generate_voiceover",
|
| 288 |
+
node_kind="tts",
|
| 289 |
+
require_prior_kind=["group_clips", "generate_script"],
|
| 290 |
+
default_require_prior_kind=["group_clips", "generate_script"],
|
| 291 |
+
)
|
| 292 |
+
|
| 293 |
+
input_schema = GenerateVoiceoverInput
|
| 294 |
+
|
| 295 |
+
async def default_process(self, node_summary, inputs: Dict[str, Any]) -> Any:
|
| 296 |
+
node_summary.info_for_user("未生成配音")
|
| 297 |
+
return {"voiceover": []}
|
| 298 |
+
|
| 299 |
+
async def process(self, node_summary, inputs: Dict[str, Any], **params) -> Any:
|
| 300 |
+
# 1) 取文案
|
| 301 |
+
group_scripts = (inputs.get("generate_script") or {}).get("group_scripts") or []
|
| 302 |
+
if not isinstance(group_scripts, list) or not group_scripts:
|
| 303 |
+
node_summary.info_for_user("未找到可生成配音的文案(group_scripts 为空)")
|
| 304 |
+
return {"voiceover": []}
|
| 305 |
+
|
| 306 |
+
# 2) 准备输出目录
|
| 307 |
+
artifact_id = inputs.get("artifact_id")
|
| 308 |
+
session_id = inputs.get("session_id")
|
| 309 |
+
if not artifact_id or not session_id:
|
| 310 |
+
raise ValueError("缺失 artifact_id / session_id,无法生成配音输出目录")
|
| 311 |
+
|
| 312 |
+
output_dir = self.server_cache_dir / str(session_id) / str(artifact_id)
|
| 313 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 314 |
+
|
| 315 |
+
# 3) 构建 TTS provider
|
| 316 |
+
provider = build_tts_provider(inputs)
|
| 317 |
+
node_summary.info_for_user(f"TTS 服务:{provider.vendor}")
|
| 318 |
+
|
| 319 |
+
ts_ms = int(time.time() * 1000)
|
| 320 |
+
voiceover: list[dict[str, Any]] = []
|
| 321 |
+
|
| 322 |
+
# 4) 逐段生成
|
| 323 |
+
for i, group in enumerate(group_scripts, start=1):
|
| 324 |
+
group_id = (group or {}).get("group_id", "")
|
| 325 |
+
raw_text = (group or {}).get("raw_text", "")
|
| 326 |
+
|
| 327 |
+
if not group_id:
|
| 328 |
+
raise ValueError(f"缺失 group_id:{group}")
|
| 329 |
+
if not isinstance(raw_text, str) or not raw_text.strip():
|
| 330 |
+
raise ValueError(f"group_id={group_id} 的 raw_text 为空,无法生成语音。")
|
| 331 |
+
|
| 332 |
+
voiceover_id = f"voiceover_{i:04d}"
|
| 333 |
+
wav_path = output_dir / f"{voiceover_id}_{ts_ms}.wav"
|
| 334 |
+
|
| 335 |
+
await asyncio.to_thread(provider.synthesize_to_wav_sync, text=raw_text, wav_path=wav_path)
|
| 336 |
+
|
| 337 |
+
duration = _wav_duration_ms(wav_path)
|
| 338 |
+
voiceover.append(
|
| 339 |
+
{
|
| 340 |
+
"voiceover_id": voiceover_id,
|
| 341 |
+
"group_id": group_id,
|
| 342 |
+
"path": str(wav_path),
|
| 343 |
+
"duration": duration,
|
| 344 |
+
}
|
| 345 |
+
)
|
| 346 |
+
|
| 347 |
+
node_summary.info_for_user(
|
| 348 |
+
f"成功生成 {voiceover_id}",
|
| 349 |
+
preview_urls=[str(wav_path)],
|
| 350 |
+
)
|
| 351 |
+
|
| 352 |
+
node_summary.info_for_user(f"共生成 {len(voiceover)} 段配音")
|
| 353 |
+
return {"voiceover": voiceover}
|
src/open_storyline/nodes/core_nodes/group_clips.py
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Any, Dict
|
| 2 |
+
|
| 3 |
+
from open_storyline.nodes.core_nodes.base_node import BaseNode, NodeMeta
|
| 4 |
+
from open_storyline.mcp.sampling_requester import LLMClient
|
| 5 |
+
from open_storyline.nodes.node_schema import GroupClipsInput
|
| 6 |
+
from open_storyline.mcp.sampling_requester import LLMClient
|
| 7 |
+
from src.open_storyline.utils.prompts import get_prompt
|
| 8 |
+
from open_storyline.utils.parse_json import parse_json_dict
|
| 9 |
+
|
| 10 |
+
class GroupClipsNode(BaseNode):
|
| 11 |
+
|
| 12 |
+
meta = NodeMeta(
|
| 13 |
+
name="group_clips",
|
| 14 |
+
description="按照用户要求,根据clips的描述,对clips进行分组。依赖 filter_clips 工具的结果",
|
| 15 |
+
node_id="group_clips",
|
| 16 |
+
node_kind="group_clips",
|
| 17 |
+
require_prior_kind=['filter_clips'],
|
| 18 |
+
default_require_prior_kind=['filter_clips'],
|
| 19 |
+
next_available_node=['generate_script', 'xhs_generate_script'],
|
| 20 |
+
)
|
| 21 |
+
input_schema = GroupClipsInput
|
| 22 |
+
|
| 23 |
+
async def default_process(
|
| 24 |
+
self,
|
| 25 |
+
node_summary,
|
| 26 |
+
inputs: Dict[str, Any],
|
| 27 |
+
) -> Any:
|
| 28 |
+
result = _make_single_group_fallback(inputs["filter_clips"].get("selected", []))
|
| 29 |
+
return {
|
| 30 |
+
"groups": result,
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
async def process(self, node_summary, inputs: Dict[str, Any], **params) -> Any:
|
| 34 |
+
clip_captions = inputs["filter_clips"].get("clip_captions")
|
| 35 |
+
selected_clips = inputs["filter_clips"].get("selected")
|
| 36 |
+
user_request = inputs["user_request"]
|
| 37 |
+
llm: LLMClient = inputs["llm"]
|
| 38 |
+
clip_lookup = _build_clip_lookup(clip_captions)
|
| 39 |
+
|
| 40 |
+
if not selected_clips:
|
| 41 |
+
return {"groups": []}
|
| 42 |
+
|
| 43 |
+
selected_clips_captions = [clip_lookup[cid] for cid in selected_clips]
|
| 44 |
+
clip_block = _build_clips_block(selected_clips_captions)
|
| 45 |
+
|
| 46 |
+
system_prompt = get_prompt("group_clips.system")
|
| 47 |
+
if user_request == "":
|
| 48 |
+
user_request = "无额外要求"
|
| 49 |
+
|
| 50 |
+
user_prompt = get_prompt(
|
| 51 |
+
"group_clips.user",
|
| 52 |
+
user_request=user_request,
|
| 53 |
+
selected_clips=selected_clips,
|
| 54 |
+
clip_captions=clip_block,
|
| 55 |
+
)
|
| 56 |
+
|
| 57 |
+
raw = await llm.complete(
|
| 58 |
+
system_prompt=system_prompt,
|
| 59 |
+
user_prompt=user_prompt,
|
| 60 |
+
media=None,
|
| 61 |
+
temperature=0.1,
|
| 62 |
+
top_p=0.9,
|
| 63 |
+
max_tokens=4096,
|
| 64 |
+
model_preferences=None,
|
| 65 |
+
)
|
| 66 |
+
|
| 67 |
+
try:
|
| 68 |
+
obj = parse_json_dict(raw)
|
| 69 |
+
groups_raw = _extract_groups_obj(obj)
|
| 70 |
+
|
| 71 |
+
groups = _normalize_groups_from_llm(
|
| 72 |
+
groups_raw=groups_raw,
|
| 73 |
+
selected_ids_set=set(selected_clips),
|
| 74 |
+
)
|
| 75 |
+
|
| 76 |
+
node_summary.info_for_user(f"分组成功:共{len(groups)}个组")
|
| 77 |
+
return {
|
| 78 |
+
"groups": groups,
|
| 79 |
+
}
|
| 80 |
+
except Exception as e:
|
| 81 |
+
|
| 82 |
+
result = _make_single_group_fallback(selected_clips)
|
| 83 |
+
node_summary.info_for_user(f"分组出错:{e}\n采用默认策略")
|
| 84 |
+
return {
|
| 85 |
+
"groups": result,
|
| 86 |
+
}
|
| 87 |
+
|
| 88 |
+
def _extract_groups_obj(obj: Any) -> list[dict[str, Any]]:
|
| 89 |
+
if isinstance(obj, dict) and isinstance(obj.get("groups"), list):
|
| 90 |
+
return obj["groups"]
|
| 91 |
+
if isinstance(obj, list):
|
| 92 |
+
return obj
|
| 93 |
+
raise ValueError("LLM 输出不包含 groups。")
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def _normalize_groups_from_llm(
|
| 97 |
+
groups_raw: list[dict[str, Any]],
|
| 98 |
+
selected_ids_set: set[str],
|
| 99 |
+
) -> list[dict[str, Any]]:
|
| 100 |
+
"""
|
| 101 |
+
校验+规范化 LLM 输出的 groups:
|
| 102 |
+
- clip_ids 必须全部来自 selected
|
| 103 |
+
- clip_ids 不能重复
|
| 104 |
+
- group_id 由代码统一重写
|
| 105 |
+
- summary 缺失则补默认
|
| 106 |
+
"""
|
| 107 |
+
if not groups_raw:
|
| 108 |
+
raise ValueError("groups 为空。")
|
| 109 |
+
|
| 110 |
+
# 先抽取并做基础清洗
|
| 111 |
+
normalized_groups: list[dict[str, Any]] = []
|
| 112 |
+
seen: set[str] = set()
|
| 113 |
+
|
| 114 |
+
for gi, g in enumerate(groups_raw):
|
| 115 |
+
if not isinstance(g, dict):
|
| 116 |
+
raise ValueError(f"groups[{gi}] 不是对象,请尝试重新运行。")
|
| 117 |
+
|
| 118 |
+
clip_ids = g.get("clip_ids")
|
| 119 |
+
if not isinstance(clip_ids, list) or not clip_ids:
|
| 120 |
+
raise ValueError(f"groups[{gi}].clip_ids 必须是非空 list,请尝试重新运行。。")
|
| 121 |
+
|
| 122 |
+
# clip_ids 去重(保持原输出顺序)
|
| 123 |
+
cleaned_clip_ids: list[str] = []
|
| 124 |
+
|
| 125 |
+
for cid in clip_ids:
|
| 126 |
+
if not isinstance(cid, str):
|
| 127 |
+
continue
|
| 128 |
+
if cid not in selected_ids_set:
|
| 129 |
+
continue
|
| 130 |
+
# raise ValueError(f"groups[{gi}] 包含非选中 clip_id:{cid}")
|
| 131 |
+
if cid in seen:
|
| 132 |
+
continue
|
| 133 |
+
seen.add(cid)
|
| 134 |
+
cleaned_clip_ids.append(cid)
|
| 135 |
+
|
| 136 |
+
if not cleaned_clip_ids:
|
| 137 |
+
raise ValueError(f"groups[{gi}] 清洗后 clip_ids 为空,请尝试重新运行。")
|
| 138 |
+
|
| 139 |
+
summary = g.get("summary")
|
| 140 |
+
if not isinstance(summary, str) or not summary.strip():
|
| 141 |
+
summary = "一组镜头,用于承载同一段文案与配音。"
|
| 142 |
+
|
| 143 |
+
normalized_groups.append(
|
| 144 |
+
{
|
| 145 |
+
"group_id": "", # group_id 先占位,后面统一重写
|
| 146 |
+
"summary": summary.strip(),
|
| 147 |
+
"clip_ids": cleaned_clip_ids,
|
| 148 |
+
}
|
| 149 |
+
)
|
| 150 |
+
|
| 151 |
+
# 最终重写 group_id
|
| 152 |
+
for i, g in enumerate(normalized_groups, start=1):
|
| 153 |
+
g["group_id"] = f"group_{i:04d}"
|
| 154 |
+
|
| 155 |
+
return normalized_groups
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def _build_clip_lookup(clip_captions: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
| 159 |
+
lookup: dict[str, dict[str, Any]] = {}
|
| 160 |
+
for clip in clip_captions:
|
| 161 |
+
cid = clip.get("clip_id")
|
| 162 |
+
if cid:
|
| 163 |
+
lookup[cid] = clip
|
| 164 |
+
return lookup
|
| 165 |
+
|
| 166 |
+
def _make_single_group_fallback(
|
| 167 |
+
selected_clips: list[str],
|
| 168 |
+
) -> list[dict[str, Any]]:
|
| 169 |
+
return [
|
| 170 |
+
{
|
| 171 |
+
"group_id": "group_0001",
|
| 172 |
+
"summary": "按原始顺序汇总全部选中镜头,便于后续生成文案与配音。",
|
| 173 |
+
"clip_ids": selected_clips,
|
| 174 |
+
}
|
| 175 |
+
]
|
| 176 |
+
|
| 177 |
+
def _build_clips_block(clip_captions: list[dict[str, Any]]) -> str:
|
| 178 |
+
"""
|
| 179 |
+
将 clips 构造成稳定的文本块
|
| 180 |
+
"""
|
| 181 |
+
blocks: list[str] = []
|
| 182 |
+
for clip in clip_captions:
|
| 183 |
+
clip_id = clip.get("clip_id", "")
|
| 184 |
+
media_id = clip.get("source_ref", {}).get("media_id", "")
|
| 185 |
+
caption = clip.get("caption", "")
|
| 186 |
+
block = (
|
| 187 |
+
f"【{clip_id}】 <media>{int(media_id.split('_', 1)[1])}</media> {caption}\n"
|
| 188 |
+
)
|
| 189 |
+
blocks.append(block)
|
| 190 |
+
return "\n".join(blocks).strip() + "\n"
|
src/open_storyline/nodes/core_nodes/plan_timeline.py
ADDED
|
@@ -0,0 +1,904 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import random
|
| 4 |
+
from dataclasses import dataclass
|
| 5 |
+
from typing import Any, Dict, List, Mapping, MutableMapping, Optional, Tuple
|
| 6 |
+
|
| 7 |
+
from src.open_storyline.config import Settings
|
| 8 |
+
from open_storyline.config import PlanTimelineConfig
|
| 9 |
+
from open_storyline.nodes.core_nodes.base_node import BaseNode, NodeMeta
|
| 10 |
+
from open_storyline.nodes.node_schema import PlanTimelineInput
|
| 11 |
+
|
| 12 |
+
# =========================
|
| 13 |
+
# Constants (no magic numbers)
|
| 14 |
+
# =========================
|
| 15 |
+
Milliseconds = int
|
| 16 |
+
|
| 17 |
+
DEFAULT_RANDOM_SEED = 42
|
| 18 |
+
|
| 19 |
+
SECONDS_PER_MINUTE = 60.0
|
| 20 |
+
MILLISECONDS_PER_SECOND = 1000.0
|
| 21 |
+
|
| 22 |
+
SNAP_SAFETY_MAX_STEPS = 10_000
|
| 23 |
+
BINARY_SEARCH_ITERATIONS = 50
|
| 24 |
+
|
| 25 |
+
RATIO_GROWTH_FACTOR = 2.0
|
| 26 |
+
RATIO_GROWTH_MAX = 10.0
|
| 27 |
+
|
| 28 |
+
MIN_SUBTITLE_WEIGHT = 1
|
| 29 |
+
CENTER_ALIGN_DIVISOR = 2.0
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
@dataclass(frozen=True)
|
| 33 |
+
class BeatTrack:
|
| 34 |
+
"""Beat-related information derived from background music (BGM)."""
|
| 35 |
+
|
| 36 |
+
beat_timestamps_ms: List[Milliseconds]
|
| 37 |
+
beat_durations_ms: List[Milliseconds]
|
| 38 |
+
music_duration_ms: Milliseconds
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
class TimelinePlanner:
|
| 42 |
+
"""
|
| 43 |
+
Pure timeline planning logic extracted from PlanTimelineNode for:
|
| 44 |
+
- clearer separation of concerns
|
| 45 |
+
- better readability
|
| 46 |
+
- easier unit testing
|
| 47 |
+
"""
|
| 48 |
+
|
| 49 |
+
def __init__(self, config: PlanTimelineConfig, *, random_seed: int = DEFAULT_RANDOM_SEED) -> None:
|
| 50 |
+
self._config = config
|
| 51 |
+
self._random_generator = random.Random(random_seed)
|
| 52 |
+
|
| 53 |
+
def plan(
|
| 54 |
+
self,
|
| 55 |
+
*,
|
| 56 |
+
assets: List[Dict[str, Any]],
|
| 57 |
+
clips: List[Dict[str, Any]],
|
| 58 |
+
groups: List[Dict[str, Any]],
|
| 59 |
+
group_scripts: List[Dict[str, Any]],
|
| 60 |
+
voiceovers: List[Dict[str, Any]],
|
| 61 |
+
background_music: Optional[Dict[str, Any]],
|
| 62 |
+
use_beats: bool,
|
| 63 |
+
) -> Dict[str, Any]:
|
| 64 |
+
"""Plan full timeline tracks: video/subtitles/voiceover/bgm."""
|
| 65 |
+
assets_by_asset_id = self._build_item_index(assets, id_key="asset_id")
|
| 66 |
+
clips_by_clip_id = self._build_item_index(clips, id_key="clip_id")
|
| 67 |
+
script_by_group_id = self._build_item_index(group_scripts, id_key="group_id")
|
| 68 |
+
voiceover_by_group_id = self._build_item_index(voiceovers, id_key="group_id")
|
| 69 |
+
|
| 70 |
+
beat_track = self._build_beat_track(background_music, use_beats=use_beats)
|
| 71 |
+
|
| 72 |
+
music_offset_ms, start_beat_index = self._compute_title_music_offset(
|
| 73 |
+
beat_durations_ms=beat_track.beat_durations_ms,
|
| 74 |
+
music_duration_ms=beat_track.music_duration_ms,
|
| 75 |
+
use_beats=use_beats,
|
| 76 |
+
)
|
| 77 |
+
|
| 78 |
+
video_segments, group_states, total_duration_ms, _end_beat_index = self._build_video_track(
|
| 79 |
+
groups=groups,
|
| 80 |
+
clips_by_clip_id=clips_by_clip_id,
|
| 81 |
+
assets_by_asset_id=assets_by_asset_id,
|
| 82 |
+
script_by_group_id=script_by_group_id,
|
| 83 |
+
voiceover_by_group_id=voiceover_by_group_id,
|
| 84 |
+
background_music=background_music,
|
| 85 |
+
beat_durations_ms=beat_track.beat_durations_ms,
|
| 86 |
+
start_beat_index=start_beat_index,
|
| 87 |
+
use_beats=use_beats,
|
| 88 |
+
)
|
| 89 |
+
|
| 90 |
+
voiceover_segments = self._build_voiceover_track(groups=groups, group_states=group_states)
|
| 91 |
+
subtitle_segments = self._build_subtitle_track(groups=groups, group_states=group_states)
|
| 92 |
+
bgm_segments = self._build_bgm_track(
|
| 93 |
+
background_music=background_music,
|
| 94 |
+
total_duration_ms=total_duration_ms,
|
| 95 |
+
music_offset_ms=music_offset_ms,
|
| 96 |
+
)
|
| 97 |
+
|
| 98 |
+
return {
|
| 99 |
+
"tracks": {
|
| 100 |
+
"video": video_segments,
|
| 101 |
+
"subtitles": subtitle_segments,
|
| 102 |
+
"voiceover": voiceover_segments,
|
| 103 |
+
"bgm": bgm_segments,
|
| 104 |
+
}
|
| 105 |
+
}
|
| 106 |
+
|
| 107 |
+
# -----------------------------
|
| 108 |
+
# Track builders
|
| 109 |
+
# -----------------------------
|
| 110 |
+
def _build_video_track(
|
| 111 |
+
self,
|
| 112 |
+
*,
|
| 113 |
+
groups: List[Dict[str, Any]],
|
| 114 |
+
clips_by_clip_id: Mapping[str, Dict[str, Any]],
|
| 115 |
+
assets_by_asset_id: Mapping[str, Dict[str, Any]],
|
| 116 |
+
script_by_group_id: Mapping[str, Dict[str, Any]],
|
| 117 |
+
voiceover_by_group_id: Mapping[str, Dict[str, Any]],
|
| 118 |
+
background_music: Optional[Dict[str, Any]],
|
| 119 |
+
beat_durations_ms: List[Milliseconds],
|
| 120 |
+
start_beat_index: int,
|
| 121 |
+
use_beats: bool,
|
| 122 |
+
) -> Tuple[List[Dict[str, Any]], Dict[str, Dict[str, Any]], Milliseconds, int]:
|
| 123 |
+
video_segments: List[Dict[str, Any]] = []
|
| 124 |
+
group_states: Dict[str, Dict[str, Any]] = {}
|
| 125 |
+
|
| 126 |
+
timeline_cursor_ms: Milliseconds = 0
|
| 127 |
+
residual_ms: Milliseconds = 0 # preserved for future improvements; legacy code kept it but didn't update it.
|
| 128 |
+
|
| 129 |
+
beat_index = int(start_beat_index)
|
| 130 |
+
|
| 131 |
+
for group in groups:
|
| 132 |
+
group_id = self._to_str_id(group.get("group_id"))
|
| 133 |
+
clip_ids = [self._to_str_id(cid) for cid in (group.get("clip_ids", []) or [])]
|
| 134 |
+
if not clip_ids:
|
| 135 |
+
raise ValueError(
|
| 136 |
+
f"group {group_id} has no clip_ids, please check the result of 'group_clips' node"
|
| 137 |
+
)
|
| 138 |
+
|
| 139 |
+
clip_items: List[Dict[str, Any]] = []
|
| 140 |
+
for clip_id in clip_ids:
|
| 141 |
+
if clip_id not in clips_by_clip_id:
|
| 142 |
+
raise KeyError(
|
| 143 |
+
f"group {group_id} references missing clip_id={clip_id}, "
|
| 144 |
+
"please check the result of 'group_clips' and 'split_shots' node"
|
| 145 |
+
)
|
| 146 |
+
clip_items.append(clips_by_clip_id[clip_id])
|
| 147 |
+
|
| 148 |
+
group_script = script_by_group_id.get(group_id)
|
| 149 |
+
group_voiceover = voiceover_by_group_id.get(group_id)
|
| 150 |
+
|
| 151 |
+
# Case A: no script, no voiceover, and no beat snapping -> concatenate clips as-is.
|
| 152 |
+
if not group_script and not group_voiceover and (not background_music or not use_beats):
|
| 153 |
+
group_start_ms = timeline_cursor_ms
|
| 154 |
+
first_clip_source_duration_ms: Milliseconds = 0
|
| 155 |
+
|
| 156 |
+
for index_in_group, clip in enumerate(clip_items):
|
| 157 |
+
clip_id = self._to_str_id(clip.get("clip_id"))
|
| 158 |
+
clip_kind = str(clip.get("kind", "video"))
|
| 159 |
+
|
| 160 |
+
source_start_ms, source_end_ms, source_duration_ms = self._full_source_window_and_duration_ms(
|
| 161 |
+
clip
|
| 162 |
+
)
|
| 163 |
+
if index_in_group == 0:
|
| 164 |
+
first_clip_source_duration_ms = source_duration_ms
|
| 165 |
+
|
| 166 |
+
segment_start_ms = timeline_cursor_ms
|
| 167 |
+
segment_end_ms = segment_start_ms + source_duration_ms
|
| 168 |
+
|
| 169 |
+
video_segments.append(
|
| 170 |
+
{
|
| 171 |
+
"clip_id": clip_id,
|
| 172 |
+
"group_id": group_id,
|
| 173 |
+
"kind": clip_kind,
|
| 174 |
+
"path": clip.get("path"),
|
| 175 |
+
"fps": clip.get("fps"),
|
| 176 |
+
"source_path": self._resolve_source_path(
|
| 177 |
+
clip=clip, assets_by_asset_id=assets_by_asset_id
|
| 178 |
+
),
|
| 179 |
+
"source_window": {"start": source_start_ms, "end": source_end_ms},
|
| 180 |
+
# NOTE: the original code used {"start": seg_end, "end": seg_end} here,
|
| 181 |
+
# which produces a 0-length window. This is almost certainly a typo.
|
| 182 |
+
"timeline_window": {"start": segment_start_ms, "end": segment_end_ms},
|
| 183 |
+
"playback_rate": 1.0,
|
| 184 |
+
}
|
| 185 |
+
)
|
| 186 |
+
timeline_cursor_ms = segment_end_ms
|
| 187 |
+
|
| 188 |
+
group_end_ms = timeline_cursor_ms
|
| 189 |
+
group_states[group_id] = {
|
| 190 |
+
"group_id": group_id,
|
| 191 |
+
"start": group_start_ms,
|
| 192 |
+
"end": group_end_ms,
|
| 193 |
+
"duration": group_end_ms - group_start_ms,
|
| 194 |
+
"first_clip_duration": first_clip_source_duration_ms,
|
| 195 |
+
}
|
| 196 |
+
continue
|
| 197 |
+
|
| 198 |
+
# Case B: voiceover duration is authoritative; otherwise estimate by text length.
|
| 199 |
+
if group_voiceover and group_voiceover.get("duration", 0) > 0:
|
| 200 |
+
narration_duration_ms: Milliseconds = int(group_voiceover.get("duration", 0))
|
| 201 |
+
else:
|
| 202 |
+
narration_duration_ms = self._estimate_group_duration_from_text_ms(group_script)
|
| 203 |
+
|
| 204 |
+
group_target_duration_ms: Milliseconds = narration_duration_ms + int(self._config.group_margin_over_voiceover)
|
| 205 |
+
group_target_duration_ms = max(
|
| 206 |
+
group_target_duration_ms, len(clip_items) * int(self._config.min_clip_duration)
|
| 207 |
+
)
|
| 208 |
+
|
| 209 |
+
if use_beats and background_music:
|
| 210 |
+
durations_ms, beat_index = self._allocate_clip_durations_using_beats(
|
| 211 |
+
clip_items=clip_items,
|
| 212 |
+
group_target_ms=group_target_duration_ms,
|
| 213 |
+
beat_durations_ms=beat_durations_ms,
|
| 214 |
+
start_beat_index=beat_index,
|
| 215 |
+
start_residual_ms=residual_ms,
|
| 216 |
+
)
|
| 217 |
+
else:
|
| 218 |
+
durations_ms = self._allocate_clip_durations_without_beats(
|
| 219 |
+
clip_items=clip_items, group_target_ms=group_target_duration_ms
|
| 220 |
+
)
|
| 221 |
+
|
| 222 |
+
group_start_ms = timeline_cursor_ms
|
| 223 |
+
first_clip_planned_duration_ms: Milliseconds = durations_ms[0] if durations_ms else 0
|
| 224 |
+
|
| 225 |
+
for clip, planned_duration_ms in zip(clip_items, durations_ms):
|
| 226 |
+
clip_id = clip.get("clip_id") # keep legacy behavior (may be int)
|
| 227 |
+
clip_kind = str(clip.get("kind", "video")).lower()
|
| 228 |
+
|
| 229 |
+
segment_start_ms = timeline_cursor_ms
|
| 230 |
+
segment_end_ms = segment_start_ms + int(planned_duration_ms)
|
| 231 |
+
|
| 232 |
+
source_start_ms, source_end_ms, source_available_ms = self._full_source_window_and_duration_ms(clip)
|
| 233 |
+
playback_rate = 1.0
|
| 234 |
+
|
| 235 |
+
if clip_kind == "video":
|
| 236 |
+
if planned_duration_ms > source_available_ms:
|
| 237 |
+
playback_rate = (source_available_ms / planned_duration_ms) if planned_duration_ms > 0 else 1.0
|
| 238 |
+
source_window_start_ms, source_window_end_ms = source_start_ms, source_end_ms
|
| 239 |
+
else:
|
| 240 |
+
source_window_start_ms, source_window_end_ms = self._choose_source_window_for_timeline_duration_ms(
|
| 241 |
+
clip=clip, used_timeline_duration_ms=int(planned_duration_ms)
|
| 242 |
+
)
|
| 243 |
+
else:
|
| 244 |
+
# image (and other non-video kinds): use from src_start for the planned duration
|
| 245 |
+
source_window_start_ms, source_window_end_ms = (
|
| 246 |
+
source_start_ms,
|
| 247 |
+
source_start_ms + int(planned_duration_ms),
|
| 248 |
+
)
|
| 249 |
+
|
| 250 |
+
video_segments.append(
|
| 251 |
+
{
|
| 252 |
+
"clip_id": clip_id,
|
| 253 |
+
"group_id": group_id,
|
| 254 |
+
"kind": clip_kind,
|
| 255 |
+
"path": clip.get("path"),
|
| 256 |
+
"orig_path": clip.get("orig_path"),
|
| 257 |
+
"fps": clip.get("fps"),
|
| 258 |
+
"source_path": self._resolve_source_path(
|
| 259 |
+
clip=clip, assets_by_asset_id=assets_by_asset_id
|
| 260 |
+
),
|
| 261 |
+
"source_window": {
|
| 262 |
+
"start": source_window_start_ms,
|
| 263 |
+
"end": source_window_end_ms,
|
| 264 |
+
"duration": source_window_end_ms - source_window_start_ms,
|
| 265 |
+
},
|
| 266 |
+
"timeline_window": {
|
| 267 |
+
"start": segment_start_ms,
|
| 268 |
+
"end": segment_end_ms,
|
| 269 |
+
"duration": segment_end_ms - segment_start_ms,
|
| 270 |
+
},
|
| 271 |
+
"playback_rate": playback_rate,
|
| 272 |
+
}
|
| 273 |
+
)
|
| 274 |
+
|
| 275 |
+
timeline_cursor_ms = segment_end_ms
|
| 276 |
+
|
| 277 |
+
group_end_ms = timeline_cursor_ms
|
| 278 |
+
group_states[group_id] = {
|
| 279 |
+
"group_id": group_id,
|
| 280 |
+
"start": group_start_ms,
|
| 281 |
+
"end": group_end_ms,
|
| 282 |
+
"duration": group_end_ms - group_start_ms,
|
| 283 |
+
"first_clip_duration": int(first_clip_planned_duration_ms),
|
| 284 |
+
"n_clips": len(clip_items),
|
| 285 |
+
"narration_duration": narration_duration_ms,
|
| 286 |
+
"group_margin": int(self._config.group_margin_over_voiceover),
|
| 287 |
+
"voiceover": group_voiceover,
|
| 288 |
+
"script": group_script,
|
| 289 |
+
}
|
| 290 |
+
|
| 291 |
+
total_duration_ms = timeline_cursor_ms
|
| 292 |
+
return video_segments, group_states, total_duration_ms, beat_index
|
| 293 |
+
|
| 294 |
+
def _build_voiceover_track(
|
| 295 |
+
self, *, groups: List[Dict[str, Any]], group_states: MutableMapping[str, Dict[str, Any]]
|
| 296 |
+
) -> List[Dict[str, Any]]:
|
| 297 |
+
voiceover_segments: List[Dict[str, Any]] = []
|
| 298 |
+
|
| 299 |
+
for group in groups:
|
| 300 |
+
group_id = self._to_str_id(group.get("group_id", ""))
|
| 301 |
+
state = group_states.get(group_id)
|
| 302 |
+
if not state:
|
| 303 |
+
continue
|
| 304 |
+
|
| 305 |
+
voiceover_item = state.get("voiceover")
|
| 306 |
+
if not voiceover_item:
|
| 307 |
+
continue
|
| 308 |
+
|
| 309 |
+
voiceover_duration_ms = int(voiceover_item.get("duration", 0))
|
| 310 |
+
if voiceover_duration_ms <= 0:
|
| 311 |
+
continue
|
| 312 |
+
|
| 313 |
+
group_start_ms = int(state.get("start", 0))
|
| 314 |
+
group_end_ms = int(state.get("end", 0))
|
| 315 |
+
group_duration_ms = group_end_ms - group_start_ms
|
| 316 |
+
|
| 317 |
+
slack_ms = max(0, group_duration_ms - voiceover_duration_ms)
|
| 318 |
+
|
| 319 |
+
start_offset_ms = slack_ms / CENTER_ALIGN_DIVISOR
|
| 320 |
+
voiceover_start_ms = group_start_ms + start_offset_ms
|
| 321 |
+
voiceover_end_ms = voiceover_start_ms + voiceover_duration_ms
|
| 322 |
+
|
| 323 |
+
voiceover_segments.append(
|
| 324 |
+
{
|
| 325 |
+
"group_id": group_id,
|
| 326 |
+
"voiceover_id": voiceover_item.get("voiceover_id"),
|
| 327 |
+
"path": voiceover_item.get("path"),
|
| 328 |
+
"source_window": {"start": 0, "end": voiceover_duration_ms, "duration": voiceover_duration_ms},
|
| 329 |
+
"timeline_window": {
|
| 330 |
+
"start": voiceover_start_ms,
|
| 331 |
+
"end": voiceover_end_ms,
|
| 332 |
+
"duration": voiceover_duration_ms,
|
| 333 |
+
},
|
| 334 |
+
}
|
| 335 |
+
)
|
| 336 |
+
|
| 337 |
+
state["voiceover_timeline"] = {
|
| 338 |
+
"start": voiceover_start_ms,
|
| 339 |
+
"end": voiceover_end_ms,
|
| 340 |
+
"duration": voiceover_duration_ms,
|
| 341 |
+
}
|
| 342 |
+
|
| 343 |
+
return voiceover_segments
|
| 344 |
+
|
| 345 |
+
def _build_subtitle_track(
|
| 346 |
+
self, *, groups: List[Dict[str, Any]], group_states: Mapping[str, Dict[str, Any]]
|
| 347 |
+
) -> List[Dict[str, Any]]:
|
| 348 |
+
subtitle_segments: List[Dict[str, Any]] = []
|
| 349 |
+
|
| 350 |
+
for group in groups:
|
| 351 |
+
group_id = self._to_str_id(group.get("group_id", ""))
|
| 352 |
+
state = group_states.get(group_id)
|
| 353 |
+
if not state:
|
| 354 |
+
continue
|
| 355 |
+
|
| 356 |
+
# NOTE: original code used default "" then script.get(...) -> may crash.
|
| 357 |
+
group_script = state.get("script") or {}
|
| 358 |
+
subtitle_units = group_script.get("subtitle_units", []) if isinstance(group_script, dict) else []
|
| 359 |
+
if not subtitle_units:
|
| 360 |
+
continue
|
| 361 |
+
|
| 362 |
+
group_start_ms = int(state.get("start", 0))
|
| 363 |
+
group_end_ms = int(state.get("end", 0))
|
| 364 |
+
voiceover_timeline = state.get("voiceover_timeline")
|
| 365 |
+
|
| 366 |
+
if voiceover_timeline is not None:
|
| 367 |
+
voiceover_start_ms = voiceover_timeline["start"]
|
| 368 |
+
voiceover_end_ms = voiceover_timeline["end"]
|
| 369 |
+
|
| 370 |
+
if not (group_start_ms <= voiceover_start_ms <= voiceover_end_ms <= group_end_ms):
|
| 371 |
+
subtitle_start_ms = group_start_ms
|
| 372 |
+
subtitle_end_ms = group_end_ms
|
| 373 |
+
else:
|
| 374 |
+
subtitle_start_ms = int((group_start_ms + voiceover_start_ms) / CENTER_ALIGN_DIVISOR)
|
| 375 |
+
subtitle_end_ms = int((voiceover_end_ms + group_end_ms) / CENTER_ALIGN_DIVISOR)
|
| 376 |
+
else:
|
| 377 |
+
subtitle_start_ms = group_start_ms
|
| 378 |
+
subtitle_end_ms = group_end_ms
|
| 379 |
+
|
| 380 |
+
subtitle_duration_ms = subtitle_end_ms - subtitle_start_ms
|
| 381 |
+
|
| 382 |
+
unit_texts: List[str] = [str(u.get("text") or "") for u in subtitle_units]
|
| 383 |
+
unit_weights: List[int] = [max(MIN_SUBTITLE_WEIGHT, len(text.strip())) for text in unit_texts]
|
| 384 |
+
total_weight = sum(unit_weights)
|
| 385 |
+
|
| 386 |
+
unit_durations_ms: List[Milliseconds] = []
|
| 387 |
+
accumulated_ms: Milliseconds = 0
|
| 388 |
+
for i, weight in enumerate(unit_weights):
|
| 389 |
+
if i == len(unit_weights) - 1:
|
| 390 |
+
duration_ms = max(0, subtitle_duration_ms - accumulated_ms)
|
| 391 |
+
else:
|
| 392 |
+
duration_ms = (subtitle_duration_ms * weight) // total_weight if total_weight > 0 else 0
|
| 393 |
+
accumulated_ms += int(duration_ms)
|
| 394 |
+
unit_durations_ms.append(int(duration_ms))
|
| 395 |
+
|
| 396 |
+
subtitle_cursor_ms = subtitle_start_ms
|
| 397 |
+
for unit, text, duration_ms in zip(subtitle_units, unit_texts, unit_durations_ms):
|
| 398 |
+
if duration_ms <= 0:
|
| 399 |
+
continue
|
| 400 |
+
segment_start_ms = subtitle_cursor_ms
|
| 401 |
+
segment_end_ms = subtitle_cursor_ms + int(duration_ms)
|
| 402 |
+
subtitle_cursor_ms = segment_end_ms
|
| 403 |
+
|
| 404 |
+
subtitle_segments.append(
|
| 405 |
+
{
|
| 406 |
+
"group_id": group_id,
|
| 407 |
+
"unit_id": unit.get("unit_id"),
|
| 408 |
+
"index_in_group": unit.get("index_in_group"),
|
| 409 |
+
"text": text,
|
| 410 |
+
"timeline_window": {"start": segment_start_ms, "end": segment_end_ms},
|
| 411 |
+
}
|
| 412 |
+
)
|
| 413 |
+
|
| 414 |
+
return subtitle_segments
|
| 415 |
+
|
| 416 |
+
def _build_bgm_track(
|
| 417 |
+
self,
|
| 418 |
+
*,
|
| 419 |
+
background_music: Optional[Dict[str, Any]],
|
| 420 |
+
total_duration_ms: Milliseconds,
|
| 421 |
+
music_offset_ms: Milliseconds,
|
| 422 |
+
) -> List[Dict[str, Any]]:
|
| 423 |
+
bgm_segments: List[Dict[str, Any]] = []
|
| 424 |
+
if not background_music:
|
| 425 |
+
return bgm_segments
|
| 426 |
+
|
| 427 |
+
music_duration_ms = int(background_music.get("duration", 0))
|
| 428 |
+
if music_duration_ms <= 0:
|
| 429 |
+
return bgm_segments
|
| 430 |
+
|
| 431 |
+
timeline_cursor_ms: Milliseconds = 0
|
| 432 |
+
source_cursor_ms: Milliseconds = int(music_offset_ms)
|
| 433 |
+
loop_index = 0
|
| 434 |
+
|
| 435 |
+
while timeline_cursor_ms < total_duration_ms:
|
| 436 |
+
remaining_timeline_ms = total_duration_ms - timeline_cursor_ms
|
| 437 |
+
remaining_source_ms = max(0, music_duration_ms - source_cursor_ms)
|
| 438 |
+
|
| 439 |
+
if remaining_source_ms <= 0:
|
| 440 |
+
source_cursor_ms = 0
|
| 441 |
+
loop_index += 1
|
| 442 |
+
continue
|
| 443 |
+
|
| 444 |
+
segment_duration_ms = min(remaining_timeline_ms, remaining_source_ms)
|
| 445 |
+
if segment_duration_ms <= 0:
|
| 446 |
+
break
|
| 447 |
+
|
| 448 |
+
bgm_segments.append(
|
| 449 |
+
{
|
| 450 |
+
"bgm_id": background_music.get("bgm_id"),
|
| 451 |
+
"path": background_music.get("path"),
|
| 452 |
+
"source_window": {"start": source_cursor_ms, "end": source_cursor_ms + segment_duration_ms},
|
| 453 |
+
"loop_idx": loop_index,
|
| 454 |
+
}
|
| 455 |
+
)
|
| 456 |
+
|
| 457 |
+
timeline_cursor_ms += segment_duration_ms
|
| 458 |
+
source_cursor_ms += segment_duration_ms
|
| 459 |
+
|
| 460 |
+
if timeline_cursor_ms < total_duration_ms and source_cursor_ms >= music_duration_ms:
|
| 461 |
+
source_cursor_ms = 0
|
| 462 |
+
loop_index += 1
|
| 463 |
+
|
| 464 |
+
return bgm_segments
|
| 465 |
+
|
| 466 |
+
# -----------------------------
|
| 467 |
+
# Beats & title alignment
|
| 468 |
+
# -----------------------------
|
| 469 |
+
def _build_beat_track(self, background_music: Optional[Dict[str, Any]], *, use_beats: bool) -> BeatTrack:
|
| 470 |
+
if not use_beats or not background_music:
|
| 471 |
+
return BeatTrack(beat_timestamps_ms=[], beat_durations_ms=[], music_duration_ms=0)
|
| 472 |
+
|
| 473 |
+
music_duration_ms = int(background_music.get("duration", 0))
|
| 474 |
+
beat_timestamps_ms = self._build_beat_timestamps_from_music_ms(background_music)
|
| 475 |
+
beat_durations_ms = self._convert_beat_timestamps_to_durations_ms(
|
| 476 |
+
beat_timestamps_ms=beat_timestamps_ms, music_duration_ms=music_duration_ms
|
| 477 |
+
)
|
| 478 |
+
|
| 479 |
+
return BeatTrack(
|
| 480 |
+
beat_timestamps_ms=beat_timestamps_ms,
|
| 481 |
+
beat_durations_ms=beat_durations_ms,
|
| 482 |
+
music_duration_ms=music_duration_ms,
|
| 483 |
+
)
|
| 484 |
+
|
| 485 |
+
def _compute_title_music_offset(
|
| 486 |
+
self, *, beat_durations_ms: List[Milliseconds], music_duration_ms: Milliseconds, use_beats: bool
|
| 487 |
+
) -> Tuple[Milliseconds, int]:
|
| 488 |
+
"""
|
| 489 |
+
Compute the BGM source offset so that the title ends on a beat.
|
| 490 |
+
|
| 491 |
+
NOTE:
|
| 492 |
+
- The original code assumes beat_durations_ms is non-empty; otherwise modulo would crash.
|
| 493 |
+
Here we guard against empty beats (safer for open-source usage).
|
| 494 |
+
"""
|
| 495 |
+
music_offset_ms: Milliseconds = 0
|
| 496 |
+
beat_index = 0
|
| 497 |
+
|
| 498 |
+
if not use_beats:
|
| 499 |
+
return music_offset_ms, beat_index
|
| 500 |
+
|
| 501 |
+
title_duration_ms = int(getattr(self._config, "title_dutation", 0))
|
| 502 |
+
if title_duration_ms <= 0 or music_duration_ms <= 0 or not beat_durations_ms:
|
| 503 |
+
return music_offset_ms, beat_index
|
| 504 |
+
|
| 505 |
+
if bool(getattr(self._config, "bgm_loop", False)):
|
| 506 |
+
title_duration_ms = title_duration_ms % music_duration_ms
|
| 507 |
+
else:
|
| 508 |
+
title_duration_ms = min(title_duration_ms, music_duration_ms)
|
| 509 |
+
|
| 510 |
+
cumulative_ms: Milliseconds = 0
|
| 511 |
+
duration_index = 0
|
| 512 |
+
while duration_index < len(beat_durations_ms) and cumulative_ms < title_duration_ms:
|
| 513 |
+
cumulative_ms += int(beat_durations_ms[duration_index])
|
| 514 |
+
duration_index += 1
|
| 515 |
+
|
| 516 |
+
music_offset_ms = max(0, cumulative_ms - title_duration_ms)
|
| 517 |
+
beat_index = duration_index % len(beat_durations_ms)
|
| 518 |
+
return int(music_offset_ms), int(beat_index)
|
| 519 |
+
|
| 520 |
+
# -----------------------------
|
| 521 |
+
# Shared helpers (indexing, parsing, math)
|
| 522 |
+
# -----------------------------
|
| 523 |
+
@staticmethod
|
| 524 |
+
def _to_str_id(value: Any) -> str:
|
| 525 |
+
return "" if value is None else str(value)
|
| 526 |
+
|
| 527 |
+
@classmethod
|
| 528 |
+
def _build_item_index(cls, items: List[Dict[str, Any]], *, id_key: str) -> Dict[str, Dict[str, Any]]:
|
| 529 |
+
return {cls._to_str_id(item.get(id_key)): item for item in (items or [])}
|
| 530 |
+
|
| 531 |
+
@staticmethod
|
| 532 |
+
def _resolve_source_path(clip: Dict[str, Any], *, assets_by_asset_id: Mapping[str, Dict[str, Any]]) -> Optional[str]:
|
| 533 |
+
source_ref = clip.get("source_ref") or {}
|
| 534 |
+
asset_id = "" if source_ref is None else str(source_ref.get("asset_id", ""))
|
| 535 |
+
return assets_by_asset_id.get(asset_id, {}).get("path")
|
| 536 |
+
|
| 537 |
+
@staticmethod
|
| 538 |
+
def _safe_float(value: Any, default_value: float = 0.0) -> float:
|
| 539 |
+
try:
|
| 540 |
+
if value is None:
|
| 541 |
+
return default_value
|
| 542 |
+
return float(value)
|
| 543 |
+
except Exception:
|
| 544 |
+
return default_value
|
| 545 |
+
|
| 546 |
+
# -----------------------------
|
| 547 |
+
# Beats helpers
|
| 548 |
+
# -----------------------------
|
| 549 |
+
def _build_beat_timestamps_from_music_ms(self, background_music: Dict[str, Any]) -> List[Milliseconds]:
|
| 550 |
+
beats: List[Milliseconds] = background_music.get("beats", []) or []
|
| 551 |
+
music_duration_ms = int(background_music.get("duration", 0))
|
| 552 |
+
|
| 553 |
+
if beats:
|
| 554 |
+
if beats[0] != 0:
|
| 555 |
+
return [0] + beats
|
| 556 |
+
return beats
|
| 557 |
+
|
| 558 |
+
bpm = background_music.get("bpm")
|
| 559 |
+
if bpm is None:
|
| 560 |
+
return [0]
|
| 561 |
+
|
| 562 |
+
bpm_value = self._safe_float(bpm, 0.0)
|
| 563 |
+
if bpm_value <= 0 or music_duration_ms <= 0:
|
| 564 |
+
return [0]
|
| 565 |
+
|
| 566 |
+
interval_ms = int(SECONDS_PER_MINUTE / bpm_value * MILLISECONDS_PER_SECOND)
|
| 567 |
+
|
| 568 |
+
timestamps_ms: List[Milliseconds] = [0]
|
| 569 |
+
timestamp_ms: Milliseconds = interval_ms
|
| 570 |
+
while timestamp_ms <= music_duration_ms:
|
| 571 |
+
timestamps_ms.append(int(timestamp_ms))
|
| 572 |
+
timestamp_ms += interval_ms
|
| 573 |
+
|
| 574 |
+
return timestamps_ms
|
| 575 |
+
|
| 576 |
+
@staticmethod
|
| 577 |
+
def _convert_beat_timestamps_to_durations_ms(
|
| 578 |
+
*, beat_timestamps_ms: List[Milliseconds], music_duration_ms: Milliseconds
|
| 579 |
+
) -> List[Milliseconds]:
|
| 580 |
+
if len(beat_timestamps_ms) < 2:
|
| 581 |
+
return []
|
| 582 |
+
|
| 583 |
+
durations_ms: List[Milliseconds] = []
|
| 584 |
+
for start_ms, end_ms in zip(beat_timestamps_ms[:-1], beat_timestamps_ms[1:]):
|
| 585 |
+
delta_ms = int(end_ms) - int(start_ms)
|
| 586 |
+
if delta_ms > 0:
|
| 587 |
+
durations_ms.append(int(delta_ms))
|
| 588 |
+
|
| 589 |
+
if music_duration_ms > 0:
|
| 590 |
+
tail_ms = max(0, int(music_duration_ms) - int(beat_timestamps_ms[-1]))
|
| 591 |
+
if tail_ms > 0:
|
| 592 |
+
durations_ms.append(int(tail_ms))
|
| 593 |
+
|
| 594 |
+
return durations_ms
|
| 595 |
+
|
| 596 |
+
# -----------------------------
|
| 597 |
+
# Text duration estimate
|
| 598 |
+
# -----------------------------
|
| 599 |
+
def _estimate_group_duration_from_text_ms(self, group_script: Optional[Dict[str, Any]]) -> Milliseconds:
|
| 600 |
+
if not group_script:
|
| 601 |
+
return int(self._config.estimate_text_min)
|
| 602 |
+
|
| 603 |
+
raw_text = str(group_script.get("raw_text") or "")
|
| 604 |
+
char_count = len(raw_text.strip())
|
| 605 |
+
if char_count <= 0:
|
| 606 |
+
return int(self._config.estimate_text_min)
|
| 607 |
+
|
| 608 |
+
chars_per_second = max(1.0, float(self._config.estimate_text_char_per_sec))
|
| 609 |
+
duration_ms = int(char_count / chars_per_second * MILLISECONDS_PER_SECOND)
|
| 610 |
+
return max(int(duration_ms), int(self._config.estimate_text_min))
|
| 611 |
+
|
| 612 |
+
# -----------------------------
|
| 613 |
+
# Clip/source windows
|
| 614 |
+
# -----------------------------
|
| 615 |
+
def _full_source_window_and_duration_ms(
|
| 616 |
+
self, clip: Dict[str, Any]
|
| 617 |
+
) -> Tuple[Milliseconds, Milliseconds, Milliseconds]:
|
| 618 |
+
clip_kind = str(clip.get("kind", "video"))
|
| 619 |
+
source_ref = clip.get("source_ref") or {}
|
| 620 |
+
|
| 621 |
+
start_ms = int(source_ref.get("start", 0))
|
| 622 |
+
end_ms = int(source_ref.get("end", start_ms))
|
| 623 |
+
duration_ms = int(source_ref.get("duration", end_ms - start_ms))
|
| 624 |
+
|
| 625 |
+
if clip_kind == "image":
|
| 626 |
+
start_ms = 0
|
| 627 |
+
end_ms = int(self._config.image_default_duration)
|
| 628 |
+
duration_ms = end_ms - start_ms
|
| 629 |
+
else:
|
| 630 |
+
if duration_ms <= 0:
|
| 631 |
+
clip_id = clip.get("clip_id")
|
| 632 |
+
raise ValueError(
|
| 633 |
+
f"{clip_id} has invalid source window (start={start_ms}, end={end_ms}, duration={duration_ms})"
|
| 634 |
+
)
|
| 635 |
+
|
| 636 |
+
return int(start_ms), int(end_ms), int(duration_ms)
|
| 637 |
+
|
| 638 |
+
def _choose_source_window_for_timeline_duration_ms(
|
| 639 |
+
self, *, clip: Dict[str, Any], used_timeline_duration_ms: Milliseconds
|
| 640 |
+
) -> Tuple[Milliseconds, Milliseconds]:
|
| 641 |
+
source_start_ms, _, source_duration_ms = self._full_source_window_and_duration_ms(clip)
|
| 642 |
+
|
| 643 |
+
random_offset_ms = int(self._random_generator.random() * (source_duration_ms - used_timeline_duration_ms))
|
| 644 |
+
window_start_ms = source_start_ms + random_offset_ms
|
| 645 |
+
window_end_ms = window_start_ms + int(used_timeline_duration_ms)
|
| 646 |
+
return int(window_start_ms), int(window_end_ms)
|
| 647 |
+
|
| 648 |
+
# -----------------------------
|
| 649 |
+
# Duration allocation
|
| 650 |
+
# -----------------------------
|
| 651 |
+
def _allocate_clip_durations_using_beats(
|
| 652 |
+
self,
|
| 653 |
+
*,
|
| 654 |
+
clip_items: List[Dict[str, Any]],
|
| 655 |
+
group_target_ms: Milliseconds,
|
| 656 |
+
beat_durations_ms: List[Milliseconds],
|
| 657 |
+
start_beat_index: int,
|
| 658 |
+
start_residual_ms: Milliseconds,
|
| 659 |
+
) -> Tuple[List[Milliseconds], int]:
|
| 660 |
+
"""
|
| 661 |
+
Algorithm (kept the same as legacy version):
|
| 662 |
+
1) Allocate ideal duration per clip by source duration weights (sum to group_target_ms).
|
| 663 |
+
2) Snap each clip end to nearest beat boundary; carry over the delta to next clip.
|
| 664 |
+
3) Last clip snaps to ceil to ensure total >= group_target_ms.
|
| 665 |
+
"""
|
| 666 |
+
clip_count = len(clip_items)
|
| 667 |
+
if clip_count == 0:
|
| 668 |
+
return [], int(start_beat_index)
|
| 669 |
+
|
| 670 |
+
if not beat_durations_ms:
|
| 671 |
+
raise ValueError("beat_durations is empty")
|
| 672 |
+
|
| 673 |
+
weights_ms: List[Milliseconds] = []
|
| 674 |
+
for clip in clip_items:
|
| 675 |
+
_, _, duration_ms = self._full_source_window_and_duration_ms(clip)
|
| 676 |
+
weights_ms.append(int(duration_ms))
|
| 677 |
+
|
| 678 |
+
sum_weights = sum(weights_ms)
|
| 679 |
+
targets_ms = [(int(group_target_ms) * w) // sum_weights for w in weights_ms]
|
| 680 |
+
remainder_ms = int(group_target_ms) - sum(targets_ms)
|
| 681 |
+
|
| 682 |
+
# Fix integer truncation drift
|
| 683 |
+
fractional_parts = [(i, (int(group_target_ms) * weights_ms[i]) % sum_weights) for i in range(clip_count)]
|
| 684 |
+
fractional_parts.sort(key=lambda x: x[1], reverse=True)
|
| 685 |
+
for k in range(remainder_ms):
|
| 686 |
+
targets_ms[fractional_parts[k][0]] += 1
|
| 687 |
+
|
| 688 |
+
# Enforce min clip duration; borrow from longest clips
|
| 689 |
+
deficit_ms: Milliseconds = 0
|
| 690 |
+
for i in range(clip_count):
|
| 691 |
+
if targets_ms[i] < int(self._config.min_clip_duration):
|
| 692 |
+
deficit_ms += int(self._config.min_clip_duration) - targets_ms[i]
|
| 693 |
+
targets_ms[i] = int(self._config.min_clip_duration)
|
| 694 |
+
|
| 695 |
+
if deficit_ms > 0:
|
| 696 |
+
indices_by_longest = sorted(range(clip_count), key=lambda i: targets_ms[i], reverse=True)
|
| 697 |
+
for i in indices_by_longest:
|
| 698 |
+
if deficit_ms <= 0:
|
| 699 |
+
break
|
| 700 |
+
slack_ms = targets_ms[i] - int(self._config.min_clip_duration)
|
| 701 |
+
targets_ms[i] -= slack_ms
|
| 702 |
+
deficit_ms -= slack_ms
|
| 703 |
+
|
| 704 |
+
beat_count = len(beat_durations_ms)
|
| 705 |
+
|
| 706 |
+
def snap_to_nearest_beat(
|
| 707 |
+
desired_ms: Milliseconds, beat_index: int, phase_ms: Milliseconds
|
| 708 |
+
) -> Tuple[Milliseconds, int]:
|
| 709 |
+
elapsed_ms = int(phase_ms)
|
| 710 |
+
idx = int(beat_index)
|
| 711 |
+
|
| 712 |
+
safety_steps = 0
|
| 713 |
+
while elapsed_ms < int(self._config.min_clip_duration):
|
| 714 |
+
elapsed_ms += int(beat_durations_ms[idx])
|
| 715 |
+
idx = (idx + 1) % beat_count
|
| 716 |
+
safety_steps += 1
|
| 717 |
+
if safety_steps > SNAP_SAFETY_MAX_STEPS:
|
| 718 |
+
raise RuntimeError("snap_to_nearest_beat safety exceeded")
|
| 719 |
+
|
| 720 |
+
if elapsed_ms >= int(desired_ms):
|
| 721 |
+
return int(elapsed_ms), int(idx)
|
| 722 |
+
|
| 723 |
+
previous_elapsed_ms = elapsed_ms
|
| 724 |
+
previous_idx = idx
|
| 725 |
+
|
| 726 |
+
safety_steps = 0
|
| 727 |
+
while elapsed_ms < int(desired_ms):
|
| 728 |
+
previous_elapsed_ms = elapsed_ms
|
| 729 |
+
previous_idx = idx
|
| 730 |
+
elapsed_ms += int(beat_durations_ms[idx])
|
| 731 |
+
idx = (idx + 1) % beat_count
|
| 732 |
+
safety_steps += 1
|
| 733 |
+
if safety_steps > SNAP_SAFETY_MAX_STEPS:
|
| 734 |
+
raise RuntimeError("snap_to_nearest_beat safety exceeded")
|
| 735 |
+
|
| 736 |
+
if int(desired_ms) - previous_elapsed_ms < elapsed_ms - int(desired_ms):
|
| 737 |
+
return int(previous_elapsed_ms), int(previous_idx)
|
| 738 |
+
return int(elapsed_ms), int(idx)
|
| 739 |
+
|
| 740 |
+
def snap_to_beat_ceil(
|
| 741 |
+
desired_ms: Milliseconds, beat_index: int, phase_ms: Milliseconds
|
| 742 |
+
) -> Tuple[Milliseconds, int]:
|
| 743 |
+
elapsed_ms = int(phase_ms)
|
| 744 |
+
idx = int(beat_index)
|
| 745 |
+
|
| 746 |
+
desired_ms = max(int(self._config.min_clip_duration), int(desired_ms))
|
| 747 |
+
safety_steps = 0
|
| 748 |
+
while elapsed_ms < desired_ms:
|
| 749 |
+
elapsed_ms += int(beat_durations_ms[idx])
|
| 750 |
+
idx = (idx + 1) % beat_count
|
| 751 |
+
safety_steps += 1
|
| 752 |
+
if safety_steps > SNAP_SAFETY_MAX_STEPS:
|
| 753 |
+
raise RuntimeError("snap_to_beat_ceil safety exceeded")
|
| 754 |
+
return int(elapsed_ms), int(idx)
|
| 755 |
+
|
| 756 |
+
durations_ms: List[Milliseconds] = []
|
| 757 |
+
beat_index = int(start_beat_index) % beat_count
|
| 758 |
+
phase_ms = max(0, int(start_residual_ms))
|
| 759 |
+
|
| 760 |
+
carry_ms: Milliseconds = 0
|
| 761 |
+
sum_actual_ms: Milliseconds = 0
|
| 762 |
+
|
| 763 |
+
for i in range(clip_count):
|
| 764 |
+
is_last_clip = i == clip_count - 1
|
| 765 |
+
|
| 766 |
+
desired_ms = int(targets_ms[i]) + int(carry_ms)
|
| 767 |
+
if desired_ms < int(self._config.min_clip_duration):
|
| 768 |
+
desired_ms = int(self._config.min_clip_duration)
|
| 769 |
+
|
| 770 |
+
if not is_last_clip:
|
| 771 |
+
actual_ms, beat_index = snap_to_nearest_beat(desired_ms, beat_index, phase_ms)
|
| 772 |
+
else:
|
| 773 |
+
remaining_ms = max(0, int(group_target_ms) - int(sum_actual_ms))
|
| 774 |
+
desired_ms = max(desired_ms, int(remaining_ms))
|
| 775 |
+
actual_ms, beat_index = snap_to_beat_ceil(desired_ms, beat_index, phase_ms)
|
| 776 |
+
|
| 777 |
+
durations_ms.append(int(actual_ms))
|
| 778 |
+
sum_actual_ms += int(actual_ms)
|
| 779 |
+
|
| 780 |
+
carry_ms = desired_ms - int(actual_ms)
|
| 781 |
+
phase_ms = 0 # legacy strategy: always reset
|
| 782 |
+
|
| 783 |
+
return durations_ms, int(beat_index)
|
| 784 |
+
|
| 785 |
+
def _allocate_clip_durations_without_beats(
|
| 786 |
+
self, *, clip_items: List[Dict[str, Any]], group_target_ms: Milliseconds
|
| 787 |
+
) -> List[Milliseconds]:
|
| 788 |
+
clip_count = len(clip_items)
|
| 789 |
+
if clip_count == 0:
|
| 790 |
+
return []
|
| 791 |
+
|
| 792 |
+
source_durations_ms: List[Milliseconds] = []
|
| 793 |
+
for clip in clip_items:
|
| 794 |
+
_, _, duration_ms = self._full_source_window_and_duration_ms(clip)
|
| 795 |
+
source_durations_ms.append(int(duration_ms))
|
| 796 |
+
|
| 797 |
+
total_source_duration_ms = sum(source_durations_ms)
|
| 798 |
+
|
| 799 |
+
def total_for_ratio(ratio: float) -> Milliseconds:
|
| 800 |
+
total_ms: Milliseconds = 0
|
| 801 |
+
for duration_ms in source_durations_ms:
|
| 802 |
+
allocated_ms = int(duration_ms * ratio)
|
| 803 |
+
if allocated_ms < int(self._config.min_clip_duration):
|
| 804 |
+
allocated_ms = int(self._config.min_clip_duration)
|
| 805 |
+
total_ms += int(allocated_ms)
|
| 806 |
+
return int(total_ms)
|
| 807 |
+
|
| 808 |
+
ratio_high = max(1.0, int(group_target_ms) / total_source_duration_ms)
|
| 809 |
+
while total_for_ratio(ratio_high) < int(group_target_ms):
|
| 810 |
+
ratio_high *= RATIO_GROWTH_FACTOR
|
| 811 |
+
if ratio_high > RATIO_GROWTH_MAX:
|
| 812 |
+
break
|
| 813 |
+
|
| 814 |
+
ratio_low = 0.0
|
| 815 |
+
for _ in range(BINARY_SEARCH_ITERATIONS):
|
| 816 |
+
ratio_mid = (ratio_low + ratio_high) / 2.0
|
| 817 |
+
if total_for_ratio(ratio_mid) <= int(group_target_ms):
|
| 818 |
+
ratio_low = ratio_mid
|
| 819 |
+
else:
|
| 820 |
+
ratio_high = ratio_mid
|
| 821 |
+
|
| 822 |
+
ratio = ratio_low
|
| 823 |
+
|
| 824 |
+
base_ms: List[Milliseconds] = []
|
| 825 |
+
fractional_parts: List[Tuple[float, int]] = []
|
| 826 |
+
sum_base_ms: Milliseconds = 0
|
| 827 |
+
|
| 828 |
+
for i, duration_ms in enumerate(source_durations_ms):
|
| 829 |
+
raw = duration_ms * ratio
|
| 830 |
+
floored = int(raw)
|
| 831 |
+
|
| 832 |
+
allocated_ms = floored
|
| 833 |
+
if allocated_ms < int(self._config.min_clip_duration):
|
| 834 |
+
allocated_ms = int(self._config.min_clip_duration)
|
| 835 |
+
fraction = -1.0
|
| 836 |
+
else:
|
| 837 |
+
fraction = raw - floored
|
| 838 |
+
|
| 839 |
+
base_ms.append(int(allocated_ms))
|
| 840 |
+
sum_base_ms += int(allocated_ms)
|
| 841 |
+
fractional_parts.append((float(fraction), i))
|
| 842 |
+
|
| 843 |
+
remaining_ms: Milliseconds = int(group_target_ms) - int(sum_base_ms)
|
| 844 |
+
if remaining_ms > 0:
|
| 845 |
+
fractional_parts.sort(key=lambda x: (x[0], source_durations_ms[x[1]]), reverse=True)
|
| 846 |
+
j = 0
|
| 847 |
+
# NOTE: matches legacy behavior (no wrap-around); remaining_ms too large may raise IndexError.
|
| 848 |
+
while remaining_ms > 0:
|
| 849 |
+
idx = fractional_parts[j][1]
|
| 850 |
+
base_ms[idx] += 1
|
| 851 |
+
remaining_ms -= 1
|
| 852 |
+
j += 1
|
| 853 |
+
|
| 854 |
+
return base_ms
|
| 855 |
+
|
| 856 |
+
|
| 857 |
+
class PlanTimelineNode(BaseNode):
|
| 858 |
+
meta = NodeMeta(
|
| 859 |
+
name="plan_timeline",
|
| 860 |
+
description=(
|
| 861 |
+
"根据视频片段、字幕、语音和BGM组织合理的时间线。"
|
| 862 |
+
"依赖 assets_index, generate_script(可选), detect_highlights(可选), "
|
| 863 |
+
"select_BGM(可选), generate_voiceover(可选)工具的结果。"
|
| 864 |
+
),
|
| 865 |
+
node_id="plan_timeline",
|
| 866 |
+
node_kind="plan_timeline",
|
| 867 |
+
require_prior_kind=["assets_index", "split_shots", "group_clips", "generate_script", "tts", "music_rec"],
|
| 868 |
+
default_require_prior_kind=["assets_index", "split_shots", "group_clips", "generate_script", "tts", "music_rec"],
|
| 869 |
+
next_available_node=["render_video"],
|
| 870 |
+
)
|
| 871 |
+
|
| 872 |
+
input_schema = PlanTimelineInput
|
| 873 |
+
|
| 874 |
+
def __init__(self, server_cfg: Settings) -> None:
|
| 875 |
+
super().__init__(server_cfg)
|
| 876 |
+
config: PlanTimelineConfig = self.server_cfg.plan_timeline
|
| 877 |
+
self.planner = TimelinePlanner(config, random_seed=DEFAULT_RANDOM_SEED)
|
| 878 |
+
|
| 879 |
+
|
| 880 |
+
async def default_process(self, node_summary, inputs: Dict[str, Any]) -> Any:
|
| 881 |
+
return await self.process(node_summary, inputs)
|
| 882 |
+
|
| 883 |
+
async def process(self, node_summary, inputs: Dict[str, Any]) -> Dict[str, Any]:
|
| 884 |
+
# Inputs (defensive parsing for open-source robustness)
|
| 885 |
+
assets = (inputs.get("assets_index") or {}).get("assets", [])
|
| 886 |
+
clips = (inputs.get("split_shots") or {}).get("clips", [])
|
| 887 |
+
groups = (inputs.get("group_clips") or {}).get("groups", [])
|
| 888 |
+
group_scripts = (inputs.get("generate_script") or {}).get("group_scripts", [])
|
| 889 |
+
voiceovers = (inputs.get("tts") or {}).get("voiceover", [])
|
| 890 |
+
background_music = (inputs.get("music_rec") or {}).get("bgm") # Optional dict
|
| 891 |
+
use_beats = inputs.get("use_beats", False)
|
| 892 |
+
|
| 893 |
+
result = self.planner.plan(
|
| 894 |
+
assets=assets,
|
| 895 |
+
clips=clips,
|
| 896 |
+
groups=groups,
|
| 897 |
+
group_scripts=group_scripts,
|
| 898 |
+
voiceovers=voiceovers,
|
| 899 |
+
background_music=background_music,
|
| 900 |
+
use_beats=use_beats,
|
| 901 |
+
)
|
| 902 |
+
|
| 903 |
+
node_summary.info_for_user("时间线组织成功")
|
| 904 |
+
return result
|
src/open_storyline/nodes/core_nodes/render_video.py
ADDED
|
@@ -0,0 +1,838 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import tempfile
|
| 3 |
+
import time
|
| 4 |
+
import uuid
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
from typing import Any, Dict, List, Optional, Tuple
|
| 7 |
+
|
| 8 |
+
import numpy as np
|
| 9 |
+
from PIL import Image, ImageDraw, ImageFont, ImageOps
|
| 10 |
+
|
| 11 |
+
# MoviePy import compatibility (v2 preferred)
|
| 12 |
+
try:
|
| 13 |
+
from moviepy import (
|
| 14 |
+
VideoFileClip,
|
| 15 |
+
AudioFileClip,
|
| 16 |
+
ImageClip,
|
| 17 |
+
ColorClip,
|
| 18 |
+
CompositeVideoClip,
|
| 19 |
+
CompositeAudioClip,
|
| 20 |
+
concatenate_videoclips,
|
| 21 |
+
concatenate_audioclips,
|
| 22 |
+
)
|
| 23 |
+
except Exception: # pragma: no cover
|
| 24 |
+
from moviepy.editor import ( # type: ignore
|
| 25 |
+
VideoFileClip,
|
| 26 |
+
AudioFileClip,
|
| 27 |
+
ImageClip,
|
| 28 |
+
ColorClip,
|
| 29 |
+
CompositeVideoClip,
|
| 30 |
+
CompositeAudioClip,
|
| 31 |
+
concatenate_videoclips,
|
| 32 |
+
concatenate_audioclips,
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
from src.open_storyline.config import Settings
|
| 36 |
+
from open_storyline.nodes.core_nodes.base_node import BaseNode, NodeMeta
|
| 37 |
+
from open_storyline.nodes.node_schema import RenderVideoInput
|
| 38 |
+
from open_storyline.utils.fonts import SourceHanSansSC_FONTS
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
# =============================================================================
|
| 42 |
+
# Constants
|
| 43 |
+
# =============================================================================
|
| 44 |
+
|
| 45 |
+
MILLISECONDS_PER_SECOND: float = 1000.0
|
| 46 |
+
|
| 47 |
+
MAX_MEDIA_DIMENSION_PX: int = 1080 # requirement: any media <=1080
|
| 48 |
+
DEFAULT_OUTPUT_MAX_DIMENSION_PX: int = 1080
|
| 49 |
+
|
| 50 |
+
DEFAULT_OUTPUT_ASPECT_RATIO: float = 16.0 / 9.0
|
| 51 |
+
BACKGROUND_COLOR_RGB: Tuple[int, int, int] = (0, 0, 0)
|
| 52 |
+
CENTER_POSITION = ("center", "center")
|
| 53 |
+
|
| 54 |
+
DEFAULT_OUTPUT_FPS: int = 25
|
| 55 |
+
|
| 56 |
+
SUBCLIP_END_SAFETY_MARGIN_S = 1e-3 #
|
| 57 |
+
|
| 58 |
+
# Encoding
|
| 59 |
+
VIDEO_CODEC: str = "libx264"
|
| 60 |
+
AUDIO_CODEC: str = "aac"
|
| 61 |
+
FFMPEG_PARAMS: list[str] = ["-preset", "veryfast", "-crf", "23", "-threads", "0"]
|
| 62 |
+
TEMP_DIRECTORY_PREFIX: str = "render_video_"
|
| 63 |
+
TEMP_AUDIO_FILENAME: str = "temp-audio.m4a"
|
| 64 |
+
|
| 65 |
+
# Subtitle baseline
|
| 66 |
+
SUBTITLE_BASE_HEIGHT_PX: float = 1080.0
|
| 67 |
+
SUBTITLE_FONT_SIZE_AT_BASE: int = 40
|
| 68 |
+
SUBTITLE_FONT_SIZE_MIN: int = 28
|
| 69 |
+
SUBTITLE_FONT_SIZE_MAX: int = 120
|
| 70 |
+
SUBTITLE_MARGIN_BOTTOM_AT_BASE: int = 80
|
| 71 |
+
SUBTITLE_MARGIN_BOTTOM_MIN: int = 40
|
| 72 |
+
SUBTITLE_MARGIN_BOTTOM_MAX: int = 240
|
| 73 |
+
SUBTITLE_STROKE_WIDTH_AT_BASE: int = 2
|
| 74 |
+
SUBTITLE_STROKE_WIDTH_MIN: int = 2
|
| 75 |
+
SUBTITLE_STROKE_WIDTH_MAX: int = 8
|
| 76 |
+
SUBTITLE_MAX_WIDTH_RATIO: float = 0.90
|
| 77 |
+
SUBTITLE_PADDING_X: int = 20
|
| 78 |
+
SUBTITLE_PADDING_Y: int = 10
|
| 79 |
+
|
| 80 |
+
BGM_VOLUME_SCALE: float = 0.25
|
| 81 |
+
AUDIO_DURATION_TOLERANCE_SECONDS: float = 0.05
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
# =============================================================================
|
| 85 |
+
# Small utilities
|
| 86 |
+
# =============================================================================
|
| 87 |
+
|
| 88 |
+
def close_quietly(obj: Any) -> None:
|
| 89 |
+
try:
|
| 90 |
+
if obj is not None:
|
| 91 |
+
obj.close()
|
| 92 |
+
except Exception:
|
| 93 |
+
pass
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def milliseconds_to_seconds(value: Any) -> float:
|
| 97 |
+
try:
|
| 98 |
+
return float(value) / MILLISECONDS_PER_SECOND
|
| 99 |
+
except Exception:
|
| 100 |
+
return 0.0
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def clamp_int(value: float, minimum: int, maximum: int) -> int:
|
| 104 |
+
return int(max(minimum, min(maximum, round(value))))
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def make_even(value: int) -> int:
|
| 108 |
+
v = int(value)
|
| 109 |
+
if v < 2:
|
| 110 |
+
v = 2
|
| 111 |
+
if v % 2 == 1:
|
| 112 |
+
v -= 1
|
| 113 |
+
return max(2, v)
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def parse_aspect_ratio(value: Any) -> Optional[float]:
|
| 117 |
+
"""
|
| 118 |
+
Accept:
|
| 119 |
+
- "16:9"
|
| 120 |
+
- float/int like 1.777
|
| 121 |
+
- (w, h)
|
| 122 |
+
"""
|
| 123 |
+
if value is None:
|
| 124 |
+
return None
|
| 125 |
+
|
| 126 |
+
if isinstance(value, (int, float)):
|
| 127 |
+
r = float(value)
|
| 128 |
+
return r if r > 0 else None
|
| 129 |
+
|
| 130 |
+
if isinstance(value, str):
|
| 131 |
+
text = value.strip()
|
| 132 |
+
if ":" in text:
|
| 133 |
+
parts = text.split(":")
|
| 134 |
+
if len(parts) == 2:
|
| 135 |
+
try:
|
| 136 |
+
w = float(parts[0].strip())
|
| 137 |
+
h = float(parts[1].strip())
|
| 138 |
+
if w > 0 and h > 0:
|
| 139 |
+
return w / h
|
| 140 |
+
except Exception:
|
| 141 |
+
return None
|
| 142 |
+
else:
|
| 143 |
+
try:
|
| 144 |
+
r = float(text)
|
| 145 |
+
return r if r > 0 else None
|
| 146 |
+
except Exception:
|
| 147 |
+
return None
|
| 148 |
+
|
| 149 |
+
if isinstance(value, (tuple, list)) and len(value) == 2:
|
| 150 |
+
try:
|
| 151 |
+
w = float(value[0])
|
| 152 |
+
h = float(value[1])
|
| 153 |
+
if w > 0 and h > 0:
|
| 154 |
+
return w / h
|
| 155 |
+
except Exception:
|
| 156 |
+
return None
|
| 157 |
+
|
| 158 |
+
return None
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
def resolve_output_canvas_size(inputs: Dict[str, Any]) -> Tuple[int, int]:
|
| 162 |
+
"""
|
| 163 |
+
Requirement:
|
| 164 |
+
1) output aspect ratio decided by inputs
|
| 165 |
+
2) keep output <=1080 (consistent with media<=1080 + performance)
|
| 166 |
+
"""
|
| 167 |
+
explicit_size = inputs.get("output_size")
|
| 168 |
+
if isinstance(explicit_size, (tuple, list)) and len(explicit_size) == 2:
|
| 169 |
+
try:
|
| 170 |
+
w = max(2, int(explicit_size[0]))
|
| 171 |
+
h = max(2, int(explicit_size[1]))
|
| 172 |
+
scale = min(1.0, MAX_MEDIA_DIMENSION_PX / float(max(w, h)))
|
| 173 |
+
return (make_even(int(w * scale)), make_even(int(h * scale)))
|
| 174 |
+
except Exception:
|
| 175 |
+
pass
|
| 176 |
+
|
| 177 |
+
ratio = (
|
| 178 |
+
parse_aspect_ratio(inputs.get("output_aspect_ratio"))
|
| 179 |
+
or parse_aspect_ratio(inputs.get("aspect_ratio"))
|
| 180 |
+
or DEFAULT_OUTPUT_ASPECT_RATIO
|
| 181 |
+
)
|
| 182 |
+
|
| 183 |
+
max_dim = inputs.get("output_max_dimension_px", DEFAULT_OUTPUT_MAX_DIMENSION_PX)
|
| 184 |
+
try:
|
| 185 |
+
max_dim = int(max_dim)
|
| 186 |
+
except Exception:
|
| 187 |
+
max_dim = DEFAULT_OUTPUT_MAX_DIMENSION_PX
|
| 188 |
+
max_dim = max(2, min(MAX_MEDIA_DIMENSION_PX, max_dim))
|
| 189 |
+
|
| 190 |
+
if ratio >= 1.0:
|
| 191 |
+
width = max_dim
|
| 192 |
+
height = max(2, int(round(width / ratio)))
|
| 193 |
+
else:
|
| 194 |
+
height = max_dim
|
| 195 |
+
width = max(2, int(round(height * ratio)))
|
| 196 |
+
|
| 197 |
+
return (make_even(width), make_even(height))
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def build_asset_id_to_path_map(assets_index: Dict[str, Any]) -> Dict[str, str]:
|
| 201 |
+
mapping: Dict[str, str] = {}
|
| 202 |
+
for item in (assets_index.get("videos") or []) + (assets_index.get("images") or []):
|
| 203 |
+
asset_id = item.get("asset_id")
|
| 204 |
+
path = item.get("path")
|
| 205 |
+
if asset_id and path:
|
| 206 |
+
mapping[asset_id] = path
|
| 207 |
+
return mapping
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
def is_image_file(path: str) -> bool:
|
| 211 |
+
try:
|
| 212 |
+
return Path(path).suffix.lower() in {".png", ".jpg", ".jpeg", ".bmp", ".webp", ".tif", ".tiff"}
|
| 213 |
+
except Exception:
|
| 214 |
+
return False
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
def make_mask_clip(mask: np.ndarray) -> ImageClip:
|
| 218 |
+
# moviepy versions may accept is_mask / ismask
|
| 219 |
+
try:
|
| 220 |
+
return ImageClip(mask, is_mask=True)
|
| 221 |
+
except TypeError: # pragma: no cover
|
| 222 |
+
return ImageClip(mask, ismask=True)
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
# =============================================================================
|
| 226 |
+
# Media cache: scale <=1080, drop alpha, and speed up images
|
| 227 |
+
# =============================================================================
|
| 228 |
+
|
| 229 |
+
class MediaCache:
|
| 230 |
+
def __init__(self, *, include_video_audio: bool, canvas_size: Tuple[int, int]) -> None:
|
| 231 |
+
self._include_video_audio = include_video_audio
|
| 232 |
+
self._canvas_size = canvas_size
|
| 233 |
+
|
| 234 |
+
self._video_sources: Dict[str, VideoFileClip] = {}
|
| 235 |
+
self._audio_sources: Dict[str, AudioFileClip] = {}
|
| 236 |
+
self._audio_to_close: List[AudioFileClip] = []
|
| 237 |
+
|
| 238 |
+
# Key optimization: cache full-canvas RGB frames for images
|
| 239 |
+
self._image_padded_frame_cache: Dict[str, np.ndarray] = {}
|
| 240 |
+
self._video_size_cache: Dict[str, Tuple[int, int]] = {}
|
| 241 |
+
|
| 242 |
+
def close(self) -> None:
|
| 243 |
+
for v in self._video_sources.values():
|
| 244 |
+
close_quietly(v)
|
| 245 |
+
for a in self._audio_to_close:
|
| 246 |
+
close_quietly(a)
|
| 247 |
+
|
| 248 |
+
def get_audio(self, path: str) -> AudioFileClip:
|
| 249 |
+
cached = self._audio_sources.get(path)
|
| 250 |
+
if cached is not None:
|
| 251 |
+
return cached
|
| 252 |
+
clip = AudioFileClip(path)
|
| 253 |
+
self._audio_sources[path] = clip
|
| 254 |
+
self._audio_to_close.append(clip)
|
| 255 |
+
return clip
|
| 256 |
+
|
| 257 |
+
def get_video(self, path: str) -> VideoFileClip:
|
| 258 |
+
cached = self._video_sources.get(path)
|
| 259 |
+
if cached is not None:
|
| 260 |
+
return cached
|
| 261 |
+
|
| 262 |
+
src_w, src_h = self._probe_video_size(path)
|
| 263 |
+
canvas_w, canvas_h = self._canvas_size
|
| 264 |
+
|
| 265 |
+
# fit into canvas and <=1080
|
| 266 |
+
max_w = min(canvas_w, MAX_MEDIA_DIMENSION_PX)
|
| 267 |
+
max_h = min(canvas_h, MAX_MEDIA_DIMENSION_PX)
|
| 268 |
+
|
| 269 |
+
target_resolution = None
|
| 270 |
+
if src_w > 0 and src_h > 0:
|
| 271 |
+
scale = min(1.0, max_w / float(src_w), max_h / float(src_h))
|
| 272 |
+
if scale < 1.0:
|
| 273 |
+
target_w = make_even(int(src_w * scale))
|
| 274 |
+
target_h = make_even(int(src_h * scale))
|
| 275 |
+
src_ratio = src_w / float(src_h)
|
| 276 |
+
canvas_ratio = canvas_w / float(canvas_h)
|
| 277 |
+
# preserve aspect by setting one dim None
|
| 278 |
+
if src_ratio >= canvas_ratio:
|
| 279 |
+
target_resolution = (None, target_w) # limit width
|
| 280 |
+
else:
|
| 281 |
+
target_resolution = (target_h, None) # limit height
|
| 282 |
+
|
| 283 |
+
try:
|
| 284 |
+
clip = VideoFileClip(path, audio=self._include_video_audio, target_resolution=target_resolution)
|
| 285 |
+
except TypeError:
|
| 286 |
+
clip = VideoFileClip(path, audio=self._include_video_audio)
|
| 287 |
+
|
| 288 |
+
self._video_sources[path] = clip
|
| 289 |
+
return clip
|
| 290 |
+
|
| 291 |
+
def get_image_padded_frame(self, path: str) -> np.ndarray:
|
| 292 |
+
cached = self._image_padded_frame_cache.get(path)
|
| 293 |
+
if cached is not None:
|
| 294 |
+
return cached
|
| 295 |
+
|
| 296 |
+
canvas_w, canvas_h = self._canvas_size
|
| 297 |
+
max_w = min(canvas_w, MAX_MEDIA_DIMENSION_PX)
|
| 298 |
+
max_h = min(canvas_h, MAX_MEDIA_DIMENSION_PX)
|
| 299 |
+
|
| 300 |
+
with Image.open(path) as image:
|
| 301 |
+
try:
|
| 302 |
+
image = ImageOps.exif_transpose(image)
|
| 303 |
+
except Exception:
|
| 304 |
+
pass
|
| 305 |
+
|
| 306 |
+
# drop alpha channel: RGBA -> alpha composite on black -> RGB
|
| 307 |
+
image = image.convert("RGBA")
|
| 308 |
+
black_bg = Image.new("RGBA", image.size, (0, 0, 0, 255))
|
| 309 |
+
try:
|
| 310 |
+
image = Image.alpha_composite(black_bg, image).convert("RGB")
|
| 311 |
+
except Exception:
|
| 312 |
+
image = image.convert("RGB")
|
| 313 |
+
|
| 314 |
+
# resize to fit (<=1080 and <=canvas)
|
| 315 |
+
try:
|
| 316 |
+
resample = Image.Resampling.LANCZOS
|
| 317 |
+
except Exception: # pragma: no cover
|
| 318 |
+
resample = Image.LANCZOS # type: ignore
|
| 319 |
+
|
| 320 |
+
image.thumbnail((max_w, max_h), resample=resample)
|
| 321 |
+
resized = np.array(image, dtype=np.uint8)
|
| 322 |
+
|
| 323 |
+
# build full-canvas frame (black + centered image)
|
| 324 |
+
canvas = np.zeros((canvas_h, canvas_w, 3), dtype=np.uint8)
|
| 325 |
+
h, w = resized.shape[0], resized.shape[1]
|
| 326 |
+
x0 = max(0, (canvas_w - w) // 2)
|
| 327 |
+
y0 = max(0, (canvas_h - h) // 2)
|
| 328 |
+
x1 = min(canvas_w, x0 + w)
|
| 329 |
+
y1 = min(canvas_h, y0 + h)
|
| 330 |
+
canvas[y0:y1, x0:x1] = resized[0 : (y1 - y0), 0 : (x1 - x0)]
|
| 331 |
+
|
| 332 |
+
self._image_padded_frame_cache[path] = canvas
|
| 333 |
+
return canvas
|
| 334 |
+
|
| 335 |
+
def _probe_video_size(self, path: str) -> Tuple[int, int]:
|
| 336 |
+
cached = self._video_size_cache.get(path)
|
| 337 |
+
if cached is not None:
|
| 338 |
+
return cached
|
| 339 |
+
|
| 340 |
+
w = h = 0
|
| 341 |
+
# simplest: open once (metadata fetch), then close
|
| 342 |
+
try:
|
| 343 |
+
tmp = VideoFileClip(path, audio=False)
|
| 344 |
+
w = int(getattr(tmp, "w", 0) or 0)
|
| 345 |
+
h = int(getattr(tmp, "h", 0) or 0)
|
| 346 |
+
close_quietly(tmp)
|
| 347 |
+
except Exception:
|
| 348 |
+
w, h = 0, 0
|
| 349 |
+
|
| 350 |
+
self._video_size_cache[path] = (w, h)
|
| 351 |
+
return w, h
|
| 352 |
+
|
| 353 |
+
|
| 354 |
+
# =============================================================================
|
| 355 |
+
# Subtitle renderer (RGB + mask; output frames are RGB, no alpha channel)
|
| 356 |
+
# =============================================================================
|
| 357 |
+
|
| 358 |
+
class PillowSubtitleRenderer:
|
| 359 |
+
def __init__(self, font_path: str) -> None:
|
| 360 |
+
self._font_path = font_path
|
| 361 |
+
|
| 362 |
+
def render(
|
| 363 |
+
self,
|
| 364 |
+
subtitle_items: List[Dict[str, Any]],
|
| 365 |
+
*,
|
| 366 |
+
video_size: Tuple[int, int],
|
| 367 |
+
) -> List[ImageClip]:
|
| 368 |
+
canvas_w, canvas_h = video_size
|
| 369 |
+
scale = (canvas_h / SUBTITLE_BASE_HEIGHT_PX) if canvas_h > 0 else 1.0
|
| 370 |
+
|
| 371 |
+
font_size = clamp_int(SUBTITLE_FONT_SIZE_AT_BASE * scale, SUBTITLE_FONT_SIZE_MIN, SUBTITLE_FONT_SIZE_MAX)
|
| 372 |
+
margin_bottom = clamp_int(
|
| 373 |
+
SUBTITLE_MARGIN_BOTTOM_AT_BASE * scale, SUBTITLE_MARGIN_BOTTOM_MIN, SUBTITLE_MARGIN_BOTTOM_MAX
|
| 374 |
+
)
|
| 375 |
+
stroke_width = clamp_int(
|
| 376 |
+
SUBTITLE_STROKE_WIDTH_AT_BASE * scale, SUBTITLE_STROKE_WIDTH_MIN, SUBTITLE_STROKE_WIDTH_MAX
|
| 377 |
+
)
|
| 378 |
+
|
| 379 |
+
clips: List[ImageClip] = []
|
| 380 |
+
for item in subtitle_items:
|
| 381 |
+
text = str(item.get("text", "")).strip()
|
| 382 |
+
tw = item.get("timeline_window", {}) or {}
|
| 383 |
+
start_s = milliseconds_to_seconds(tw.get("start", 0.0))
|
| 384 |
+
end_s = milliseconds_to_seconds(tw.get("end", 0.0))
|
| 385 |
+
dur = end_s - start_s
|
| 386 |
+
if not text or dur <= 0:
|
| 387 |
+
continue
|
| 388 |
+
|
| 389 |
+
clip = self._make_clip(
|
| 390 |
+
text=text,
|
| 391 |
+
start_s=start_s,
|
| 392 |
+
end_s=end_s,
|
| 393 |
+
video_size=video_size,
|
| 394 |
+
font_size=font_size,
|
| 395 |
+
margin_bottom=margin_bottom,
|
| 396 |
+
stroke_width=stroke_width,
|
| 397 |
+
)
|
| 398 |
+
if clip is not None:
|
| 399 |
+
clips.append(clip)
|
| 400 |
+
|
| 401 |
+
return clips
|
| 402 |
+
|
| 403 |
+
def _make_clip(
|
| 404 |
+
self,
|
| 405 |
+
*,
|
| 406 |
+
text: str,
|
| 407 |
+
start_s: float,
|
| 408 |
+
end_s: float,
|
| 409 |
+
video_size: Tuple[int, int],
|
| 410 |
+
font_size: int,
|
| 411 |
+
margin_bottom: int,
|
| 412 |
+
stroke_width: int,
|
| 413 |
+
) -> Optional[ImageClip]:
|
| 414 |
+
canvas_w, canvas_h = video_size
|
| 415 |
+
dur = end_s - start_s
|
| 416 |
+
if dur <= 0:
|
| 417 |
+
return None
|
| 418 |
+
|
| 419 |
+
font = self._load_font(font_size)
|
| 420 |
+
|
| 421 |
+
max_text_w = max(1, int(canvas_w * SUBTITLE_MAX_WIDTH_RATIO))
|
| 422 |
+
wrapped = self._wrap_text_by_width(text, font, max_text_w)
|
| 423 |
+
|
| 424 |
+
measure = Image.new("RGBA", (10, 10), (0, 0, 0, 0))
|
| 425 |
+
draw = ImageDraw.Draw(measure)
|
| 426 |
+
bbox = draw.multiline_textbbox(
|
| 427 |
+
(0, 0), wrapped, font=font, align="center", stroke_width=stroke_width
|
| 428 |
+
)
|
| 429 |
+
text_w = bbox[2] - bbox[0]
|
| 430 |
+
text_h = bbox[3] - bbox[1]
|
| 431 |
+
|
| 432 |
+
img_w = int(text_w + SUBTITLE_PADDING_X * 2)
|
| 433 |
+
img_h = int(text_h + SUBTITLE_PADDING_Y * 2)
|
| 434 |
+
|
| 435 |
+
rgba = Image.new("RGBA", (img_w, img_h), (0, 0, 0, 0))
|
| 436 |
+
draw = ImageDraw.Draw(rgba)
|
| 437 |
+
draw.multiline_text(
|
| 438 |
+
(SUBTITLE_PADDING_X - bbox[0], SUBTITLE_PADDING_Y - bbox[1]),
|
| 439 |
+
wrapped,
|
| 440 |
+
font=font,
|
| 441 |
+
fill=(255, 255, 255, 255),
|
| 442 |
+
align="center",
|
| 443 |
+
stroke_width=stroke_width,
|
| 444 |
+
stroke_fill=(0, 0, 0, 255),
|
| 445 |
+
)
|
| 446 |
+
|
| 447 |
+
rgba_arr = np.array(rgba, dtype=np.uint8)
|
| 448 |
+
rgb_arr = rgba_arr[:, :, :3]
|
| 449 |
+
alpha_mask = (rgba_arr[:, :, 3].astype(np.float32) / 255.0)
|
| 450 |
+
|
| 451 |
+
subtitle_clip = ImageClip(rgb_arr).with_mask(make_mask_clip(alpha_mask))
|
| 452 |
+
y = max(0, int(canvas_h - margin_bottom - img_h))
|
| 453 |
+
subtitle_clip = subtitle_clip.with_start(start_s).with_duration(dur).with_position(("center", y))
|
| 454 |
+
return subtitle_clip
|
| 455 |
+
|
| 456 |
+
def _load_font(self, font_size: int) -> ImageFont.FreeTypeFont:
|
| 457 |
+
try:
|
| 458 |
+
if self._font_path and os.path.exists(self._font_path):
|
| 459 |
+
return ImageFont.truetype(font=self._font_path, size=font_size)
|
| 460 |
+
except Exception:
|
| 461 |
+
pass
|
| 462 |
+
return ImageFont.load_default()
|
| 463 |
+
|
| 464 |
+
@staticmethod
|
| 465 |
+
def _wrap_text_by_width(text: str, font: ImageFont.FreeTypeFont, max_width_px: int) -> str:
|
| 466 |
+
text = text.strip()
|
| 467 |
+
if not text:
|
| 468 |
+
return ""
|
| 469 |
+
|
| 470 |
+
dummy = Image.new("RGB", (10, 10))
|
| 471 |
+
draw = ImageDraw.Draw(dummy)
|
| 472 |
+
|
| 473 |
+
lines: List[str] = []
|
| 474 |
+
for paragraph in text.splitlines():
|
| 475 |
+
paragraph = paragraph.strip()
|
| 476 |
+
if not paragraph:
|
| 477 |
+
lines.append("")
|
| 478 |
+
continue
|
| 479 |
+
|
| 480 |
+
current = ""
|
| 481 |
+
for ch in paragraph:
|
| 482 |
+
candidate = current + ch
|
| 483 |
+
if draw.textlength(candidate, font=font) <= max_width_px or not current:
|
| 484 |
+
current = candidate
|
| 485 |
+
else:
|
| 486 |
+
lines.append(current)
|
| 487 |
+
current = ch
|
| 488 |
+
if current:
|
| 489 |
+
lines.append(current)
|
| 490 |
+
|
| 491 |
+
return "\n".join(lines)
|
| 492 |
+
|
| 493 |
+
|
| 494 |
+
# =============================================================================
|
| 495 |
+
# Audio composer (same behavior as before)
|
| 496 |
+
# =============================================================================
|
| 497 |
+
|
| 498 |
+
class AudioTrackComposer:
|
| 499 |
+
def __init__(self, *, cache: MediaCache) -> None:
|
| 500 |
+
self._cache = cache
|
| 501 |
+
|
| 502 |
+
def compose(
|
| 503 |
+
self,
|
| 504 |
+
*,
|
| 505 |
+
voiceover_items: List[Dict[str, Any]],
|
| 506 |
+
bgm_items: List[Dict[str, Any]],
|
| 507 |
+
final_duration_s: float,
|
| 508 |
+
):
|
| 509 |
+
layers: List[Any] = []
|
| 510 |
+
|
| 511 |
+
# voiceover
|
| 512 |
+
for item in voiceover_items:
|
| 513 |
+
path = item.get("path")
|
| 514 |
+
if not path:
|
| 515 |
+
continue
|
| 516 |
+
src = self._cache.get_audio(path)
|
| 517 |
+
|
| 518 |
+
sw = item.get("source_window", {}) or {}
|
| 519 |
+
tw = item.get("timeline_window", {}) or {}
|
| 520 |
+
|
| 521 |
+
src_start = milliseconds_to_seconds(sw.get("start", 0.0))
|
| 522 |
+
src_end = milliseconds_to_seconds(sw.get("end", 0.0))
|
| 523 |
+
if src_end <= src_start:
|
| 524 |
+
continue
|
| 525 |
+
|
| 526 |
+
tl_start = milliseconds_to_seconds(tw.get("start", 0.0))
|
| 527 |
+
tl_end = milliseconds_to_seconds(tw.get("end", 0.0))
|
| 528 |
+
expected = max(0.0, tl_end - tl_start)
|
| 529 |
+
|
| 530 |
+
clip = src.subclipped(src_start, src_end).with_start(tl_start)
|
| 531 |
+
if expected > 0:
|
| 532 |
+
clip = clip.with_duration(expected)
|
| 533 |
+
layers.append(clip)
|
| 534 |
+
|
| 535 |
+
# bgm (concat then loop/trim)
|
| 536 |
+
if bgm_items:
|
| 537 |
+
segments: List[Any] = []
|
| 538 |
+
for item in sorted(bgm_items, key=lambda x: int(x.get("loop_idx", 0))):
|
| 539 |
+
path = item.get("path")
|
| 540 |
+
if not path:
|
| 541 |
+
continue
|
| 542 |
+
src = self._cache.get_audio(path)
|
| 543 |
+
sw = item.get("source_window", {}) or {}
|
| 544 |
+
src_start = milliseconds_to_seconds(sw.get("start", 0.0))
|
| 545 |
+
src_end = milliseconds_to_seconds(sw.get("end", 0.0))
|
| 546 |
+
if src_end <= src_start:
|
| 547 |
+
continue
|
| 548 |
+
segments.append(src.subclipped(src_start, src_end))
|
| 549 |
+
|
| 550 |
+
if segments:
|
| 551 |
+
bgm = concatenate_audioclips(segments).with_volume_scaled(BGM_VOLUME_SCALE)
|
| 552 |
+
if bgm.duration is not None:
|
| 553 |
+
if bgm.duration < final_duration_s - AUDIO_DURATION_TOLERANCE_SECONDS:
|
| 554 |
+
bgm = self._loop_audio(bgm, final_duration_s)
|
| 555 |
+
elif bgm.duration > final_duration_s + AUDIO_DURATION_TOLERANCE_SECONDS:
|
| 556 |
+
bgm = bgm.subclipped(0, final_duration_s)
|
| 557 |
+
layers.append(bgm)
|
| 558 |
+
|
| 559 |
+
if not layers:
|
| 560 |
+
return None
|
| 561 |
+
return CompositeAudioClip(layers).with_duration(final_duration_s)
|
| 562 |
+
|
| 563 |
+
@staticmethod
|
| 564 |
+
def _loop_audio(audio_clip: Any, duration_s: float) -> Any:
|
| 565 |
+
try:
|
| 566 |
+
n = int(duration_s // audio_clip.duration) + 2
|
| 567 |
+
looped = concatenate_audioclips([audio_clip] * max(1, n))
|
| 568 |
+
return looped.subclipped(0, duration_s)
|
| 569 |
+
except Exception:
|
| 570 |
+
return audio_clip
|
| 571 |
+
|
| 572 |
+
|
| 573 |
+
# =============================================================================
|
| 574 |
+
# Pipeline (non-overlapping visuals + subtitle overlay)
|
| 575 |
+
# =============================================================================
|
| 576 |
+
|
| 577 |
+
class RenderVideoPipeline:
|
| 578 |
+
def __init__(self, *, server_cache_dir: Path) -> None:
|
| 579 |
+
self._server_cache_dir = server_cache_dir
|
| 580 |
+
|
| 581 |
+
def render(self, *, node_summary, inputs: Dict[str, Any]) -> Dict[str, Any]:
|
| 582 |
+
assets_index: Dict[str, Any] = inputs["assets_index"]
|
| 583 |
+
tracks: Dict[str, Any] = (inputs.get("plan_timeline") or {}).get("tracks", {}) or {}
|
| 584 |
+
|
| 585 |
+
artifact_id: str = inputs["artifact_id"]
|
| 586 |
+
session_id: str = inputs["session_id"]
|
| 587 |
+
outputs_dir: Path = self._server_cache_dir / session_id / artifact_id
|
| 588 |
+
outputs_dir.mkdir(parents=True, exist_ok=True)
|
| 589 |
+
|
| 590 |
+
video_items = tracks.get("video", []) or []
|
| 591 |
+
subtitle_items = tracks.get("subtitles", []) or []
|
| 592 |
+
voiceover_items = tracks.get("voiceover", []) or []
|
| 593 |
+
bgm_items = tracks.get("bgm", []) or []
|
| 594 |
+
|
| 595 |
+
if not video_items:
|
| 596 |
+
raise ValueError("timeline result has no video track")
|
| 597 |
+
|
| 598 |
+
output_canvas_size = resolve_output_canvas_size(inputs)
|
| 599 |
+
asset_map = build_asset_id_to_path_map(assets_index)
|
| 600 |
+
|
| 601 |
+
override_audio = bool(voiceover_items) or bool(bgm_items)
|
| 602 |
+
cache = MediaCache(include_video_audio=not override_audio, canvas_size=output_canvas_size)
|
| 603 |
+
|
| 604 |
+
subtitle_renderer = PillowSubtitleRenderer(font_path=str(SourceHanSansSC_FONTS["normal"]))
|
| 605 |
+
audio_composer = AudioTrackComposer(cache=cache)
|
| 606 |
+
|
| 607 |
+
temp_dir = tempfile.mkdtemp(prefix=TEMP_DIRECTORY_PREFIX)
|
| 608 |
+
output_name = f"output_{uuid.uuid4().hex[:8]}_{int(time.time() * 1000)}.mp4"
|
| 609 |
+
output_path = str((outputs_dir / output_name).resolve())
|
| 610 |
+
|
| 611 |
+
clips_to_close: List[Any] = []
|
| 612 |
+
subtitle_clips: List[Any] = []
|
| 613 |
+
base_clip = None
|
| 614 |
+
final_clip = None
|
| 615 |
+
|
| 616 |
+
try:
|
| 617 |
+
final_duration_s = self._final_duration_seconds(video_items)
|
| 618 |
+
|
| 619 |
+
base_clip, clips_to_close, output_fps = self._build_base_video_concat(
|
| 620 |
+
video_items=video_items,
|
| 621 |
+
asset_map=asset_map,
|
| 622 |
+
cache=cache,
|
| 623 |
+
canvas_size=output_canvas_size,
|
| 624 |
+
final_duration_s=final_duration_s,
|
| 625 |
+
)
|
| 626 |
+
|
| 627 |
+
subtitle_clips = subtitle_renderer.render(subtitle_items, video_size=output_canvas_size)
|
| 628 |
+
if subtitle_clips:
|
| 629 |
+
final_clip = CompositeVideoClip([base_clip, *subtitle_clips]).with_duration(final_duration_s)
|
| 630 |
+
else:
|
| 631 |
+
final_clip = base_clip
|
| 632 |
+
|
| 633 |
+
if override_audio:
|
| 634 |
+
final_audio = audio_composer.compose(
|
| 635 |
+
voiceover_items=voiceover_items,
|
| 636 |
+
bgm_items=bgm_items,
|
| 637 |
+
final_duration_s=final_duration_s,
|
| 638 |
+
)
|
| 639 |
+
if final_audio is not None:
|
| 640 |
+
final_clip = final_clip.with_audio(final_audio)
|
| 641 |
+
else:
|
| 642 |
+
final_clip = final_clip.without_audio()
|
| 643 |
+
|
| 644 |
+
final_clip.write_videofile(
|
| 645 |
+
output_path,
|
| 646 |
+
codec=VIDEO_CODEC,
|
| 647 |
+
audio_codec=AUDIO_CODEC,
|
| 648 |
+
temp_audiofile=os.path.join(temp_dir, TEMP_AUDIO_FILENAME),
|
| 649 |
+
remove_temp=True,
|
| 650 |
+
fps=output_fps,
|
| 651 |
+
ffmpeg_params=FFMPEG_PARAMS,
|
| 652 |
+
)
|
| 653 |
+
|
| 654 |
+
node_summary.info_for_user(
|
| 655 |
+
f"成功生成视频,时长为{final_duration_s}秒, 路径:{output_path}",
|
| 656 |
+
preview_urls=[output_path],
|
| 657 |
+
)
|
| 658 |
+
return {
|
| 659 |
+
"output_path": output_path,
|
| 660 |
+
"output_basename": output_name,
|
| 661 |
+
"duration_s": float(final_duration_s),
|
| 662 |
+
"output_size": {"width": int(output_canvas_size[0]), "height": int(output_canvas_size[1])},
|
| 663 |
+
}
|
| 664 |
+
|
| 665 |
+
finally:
|
| 666 |
+
for c in subtitle_clips:
|
| 667 |
+
close_quietly(c)
|
| 668 |
+
for c in clips_to_close:
|
| 669 |
+
close_quietly(c)
|
| 670 |
+
close_quietly(base_clip)
|
| 671 |
+
close_quietly(final_clip)
|
| 672 |
+
cache.close()
|
| 673 |
+
|
| 674 |
+
@staticmethod
|
| 675 |
+
def _final_duration_seconds(video_items: List[Dict[str, Any]]) -> float:
|
| 676 |
+
end_ms = max(float((it.get("timeline_window") or {}).get("end", 0.0)) for it in video_items)
|
| 677 |
+
return milliseconds_to_seconds(end_ms)
|
| 678 |
+
|
| 679 |
+
@staticmethod
|
| 680 |
+
def _build_base_video_concat(
|
| 681 |
+
*,
|
| 682 |
+
video_items: List[Dict[str, Any]],
|
| 683 |
+
asset_map: Dict[str, str],
|
| 684 |
+
cache: MediaCache,
|
| 685 |
+
canvas_size: Tuple[int, int],
|
| 686 |
+
final_duration_s: float,
|
| 687 |
+
) -> Tuple[Any, List[Any], float]:
|
| 688 |
+
# Force non-overlapping: concat only
|
| 689 |
+
sorted_items = sorted(video_items, key=lambda x: float((x.get("timeline_window") or {}).get("start", 0.0)))
|
| 690 |
+
|
| 691 |
+
clips: List[Any] = []
|
| 692 |
+
clips_to_close: List[Any] = []
|
| 693 |
+
current_time = 0.0
|
| 694 |
+
|
| 695 |
+
def black_clip(duration: float) -> Any:
|
| 696 |
+
c = ColorClip(size=canvas_size, color=BACKGROUND_COLOR_RGB).with_duration(max(0.0, duration))
|
| 697 |
+
clips_to_close.append(c)
|
| 698 |
+
return c
|
| 699 |
+
|
| 700 |
+
for segment in sorted_items:
|
| 701 |
+
timeline_window = segment.get("timeline_window", {}) or {}
|
| 702 |
+
start_s = milliseconds_to_seconds(timeline_window.get("start", 0.0))
|
| 703 |
+
end_s = milliseconds_to_seconds(timeline_window.get("end", 0.0))
|
| 704 |
+
expected_dur = max(0.0, end_s - start_s)
|
| 705 |
+
if expected_dur <= 0:
|
| 706 |
+
continue
|
| 707 |
+
|
| 708 |
+
# fill gap
|
| 709 |
+
if start_s > current_time:
|
| 710 |
+
clips.append(black_clip(start_s - current_time))
|
| 711 |
+
current_time = start_s
|
| 712 |
+
|
| 713 |
+
seg_clip = RenderVideoPipeline._build_full_canvas_segment(
|
| 714 |
+
segment=segment,
|
| 715 |
+
asset_map=asset_map,
|
| 716 |
+
cache=cache,
|
| 717 |
+
canvas_size=canvas_size,
|
| 718 |
+
expected_duration_s=expected_dur,
|
| 719 |
+
)
|
| 720 |
+
|
| 721 |
+
if seg_clip is None:
|
| 722 |
+
clips.append(black_clip(expected_dur))
|
| 723 |
+
else:
|
| 724 |
+
clips.append(seg_clip)
|
| 725 |
+
clips_to_close.append(seg_clip)
|
| 726 |
+
|
| 727 |
+
current_time = start_s + expected_dur
|
| 728 |
+
|
| 729 |
+
# trailing gap
|
| 730 |
+
if final_duration_s > current_time:
|
| 731 |
+
clips.append(black_clip(final_duration_s - current_time))
|
| 732 |
+
|
| 733 |
+
if not clips:
|
| 734 |
+
raise ValueError("no valid video segments")
|
| 735 |
+
|
| 736 |
+
base = concatenate_videoclips(clips, method="chain").with_duration(final_duration_s)
|
| 737 |
+
|
| 738 |
+
fps_values = [float(it.get("fps")) for it in video_items if it.get("fps")]
|
| 739 |
+
output_fps = max(fps_values) if fps_values else float(getattr(base, "fps", None) or DEFAULT_OUTPUT_FPS)
|
| 740 |
+
|
| 741 |
+
return base, clips_to_close, output_fps
|
| 742 |
+
|
| 743 |
+
@staticmethod
|
| 744 |
+
def _build_full_canvas_segment(
|
| 745 |
+
*,
|
| 746 |
+
segment: Dict[str, Any],
|
| 747 |
+
asset_map: Dict[str, str],
|
| 748 |
+
cache: MediaCache,
|
| 749 |
+
canvas_size: Tuple[int, int],
|
| 750 |
+
expected_duration_s: float,
|
| 751 |
+
) -> Optional[Any]:
|
| 752 |
+
source_path = segment.get("source_path") or asset_map.get(segment.get("asset_id"))
|
| 753 |
+
if not source_path:
|
| 754 |
+
return None
|
| 755 |
+
|
| 756 |
+
if is_image_file(source_path):
|
| 757 |
+
frame = cache.get_image_padded_frame(source_path)
|
| 758 |
+
return ImageClip(frame).with_duration(expected_duration_s)
|
| 759 |
+
|
| 760 |
+
# video
|
| 761 |
+
source = cache.get_video(source_path)
|
| 762 |
+
source_window = segment.get("source_window", {}) or {}
|
| 763 |
+
|
| 764 |
+
src_start = milliseconds_to_seconds(source_window.get("start", 0.0))
|
| 765 |
+
|
| 766 |
+
end_ms = source_window.get("end", None)
|
| 767 |
+
if end_ms is None:
|
| 768 |
+
src_end = float(getattr(source, "duration", 0.0) or 0.0)
|
| 769 |
+
else:
|
| 770 |
+
src_end = milliseconds_to_seconds(end_ms)
|
| 771 |
+
|
| 772 |
+
if src_end <= src_start:
|
| 773 |
+
return None
|
| 774 |
+
|
| 775 |
+
# Clamp to avoid "end_time > duration" (common near EOF due to rounding/encoding)
|
| 776 |
+
source_duration_s = float(getattr(source, "duration", 0.0) or 0.0)
|
| 777 |
+
if source_duration_s > 0.0:
|
| 778 |
+
if src_start >= source_duration_s:
|
| 779 |
+
return None
|
| 780 |
+
if src_end > source_duration_s:
|
| 781 |
+
src_end = source_duration_s # <= duration
|
| 782 |
+
|
| 783 |
+
if src_end <= src_start:
|
| 784 |
+
return None
|
| 785 |
+
|
| 786 |
+
clip = source.subclipped(src_start, src_end)
|
| 787 |
+
|
| 788 |
+
playback_rate = float(segment.get("playback_rate", 1.0) or 1.0)
|
| 789 |
+
if playback_rate != 1.0:
|
| 790 |
+
clip = clip.with_speed_scaled(playback_rate)
|
| 791 |
+
|
| 792 |
+
clip_dur = float(getattr(clip, "duration", 0.0) or 0.0)
|
| 793 |
+
if clip_dur > 0.0 and expected_duration_s > clip_dur:
|
| 794 |
+
last_t = max(0.0, clip_dur - SUBCLIP_END_SAFETY_MARGIN_S)
|
| 795 |
+
# Freeze video/mask at last frame for the remaining duration
|
| 796 |
+
clip = clip.time_transform(
|
| 797 |
+
lambda t, lt=last_t: min(t, lt),
|
| 798 |
+
apply_to=["mask"],
|
| 799 |
+
keep_duration=True,
|
| 800 |
+
)
|
| 801 |
+
|
| 802 |
+
clip = clip.with_duration(expected_duration_s)
|
| 803 |
+
|
| 804 |
+
# pad to full canvas (keep original "center on black" look)
|
| 805 |
+
if hasattr(clip, "on_color"):
|
| 806 |
+
clip = clip.on_color(size=canvas_size, color=BACKGROUND_COLOR_RGB, pos=CENTER_POSITION)
|
| 807 |
+
else: # pragma: no cover
|
| 808 |
+
bg = ColorClip(size=canvas_size, color=BACKGROUND_COLOR_RGB).with_duration(expected_duration_s)
|
| 809 |
+
clip = CompositeVideoClip([bg, clip.with_position(CENTER_POSITION)]).with_duration(expected_duration_s)
|
| 810 |
+
|
| 811 |
+
return clip.with_duration(expected_duration_s)
|
| 812 |
+
|
| 813 |
+
|
| 814 |
+
# =============================================================================
|
| 815 |
+
# Node entrypoint
|
| 816 |
+
# =============================================================================
|
| 817 |
+
|
| 818 |
+
class RenderVideoNode(BaseNode):
|
| 819 |
+
meta = NodeMeta(
|
| 820 |
+
name="render_video",
|
| 821 |
+
description="根据组织好的时间线渲染成片。",
|
| 822 |
+
node_id="render_video",
|
| 823 |
+
node_kind="render_video",
|
| 824 |
+
require_prior_kind=["assets_index", "plan_timeline"],
|
| 825 |
+
default_require_prior_kind=["assets_index", "plan_timeline"],
|
| 826 |
+
)
|
| 827 |
+
|
| 828 |
+
input_schema = RenderVideoInput
|
| 829 |
+
|
| 830 |
+
def __init__(self, server_cfg: Settings) -> None:
|
| 831 |
+
super().__init__(server_cfg)
|
| 832 |
+
self._pipeline = RenderVideoPipeline(server_cache_dir=self.server_cache_dir)
|
| 833 |
+
|
| 834 |
+
async def default_process(self, node_summary, inputs: Dict[str, Any]) -> Any:
|
| 835 |
+
return await self.process(node_summary, inputs)
|
| 836 |
+
|
| 837 |
+
async def process(self, node_summary, inputs: Dict[str, Any]) -> Dict[str, Any]:
|
| 838 |
+
return self._pipeline.render(node_summary=node_summary, inputs=inputs)
|
src/open_storyline/nodes/core_nodes/select_bgm.py
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Any, Dict
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
|
| 4 |
+
import numpy as np
|
| 5 |
+
import random
|
| 6 |
+
import librosa
|
| 7 |
+
|
| 8 |
+
from open_storyline.nodes.core_nodes.base_node import BaseNode, NodeMeta
|
| 9 |
+
from open_storyline.nodes.node_schema import SelectBGMInput
|
| 10 |
+
|
| 11 |
+
class SelectBGMNode(BaseNode):
|
| 12 |
+
meta = NodeMeta(
|
| 13 |
+
name="select_bgm",
|
| 14 |
+
description="根据用户的要求,选择合适的BGM",
|
| 15 |
+
node_id="select_bgm",
|
| 16 |
+
node_kind="music_rec",
|
| 17 |
+
require_prior_kind=[],
|
| 18 |
+
default_require_prior_kind=[],
|
| 19 |
+
next_available_node=["plan_timeline"],
|
| 20 |
+
)
|
| 21 |
+
|
| 22 |
+
input_schema = SelectBGMInput
|
| 23 |
+
|
| 24 |
+
async def default_process(
|
| 25 |
+
self,
|
| 26 |
+
node_summary,
|
| 27 |
+
inputs: Dict[str, Any],
|
| 28 |
+
) -> Any:
|
| 29 |
+
node_summary.info_for_user("未能选择音乐")
|
| 30 |
+
return {"bgm": {}}
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
async def process(self, node_summary, inputs: Dict[str, Any]) -> Any:
|
| 34 |
+
cfg = self.server_cfg
|
| 35 |
+
user_request = inputs["user_request"]
|
| 36 |
+
bgm_dir = cfg.project.bgm_dir
|
| 37 |
+
bgm_path = recommend_bgm(bgm_dir)
|
| 38 |
+
|
| 39 |
+
result = analyze_music_metrics(path=bgm_path, sr=cfg.select_bgm.sample_rate, hop_length=cfg.select_bgm.hop_length, frame_length=cfg.select_bgm.frame_length)
|
| 40 |
+
if result.get("path"):
|
| 41 |
+
node_summary.info_for_user(f"成功选择音乐", preview_urls = [result.get("path")])
|
| 42 |
+
else:
|
| 43 |
+
node_summary.info_for_user("未能选择音乐")
|
| 44 |
+
return {"bgm": result}
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def recommend_bgm(bgm_dir: Path):
|
| 48 |
+
bgm_dir = bgm_dir.expanduser().resolve()
|
| 49 |
+
if not bgm_dir.exists():
|
| 50 |
+
raise FileNotFoundError(f"bgm_dir not found: {bgm_dir}")
|
| 51 |
+
if not bgm_dir.is_dir():
|
| 52 |
+
raise NotADirectoryError(f"bgm_dir is not a directory: {bgm_dir}")
|
| 53 |
+
|
| 54 |
+
exts = {".mp3"}
|
| 55 |
+
candidates = [
|
| 56 |
+
p for p in bgm_dir.iterdir()
|
| 57 |
+
if p.is_file() and p.suffix.lower() in exts
|
| 58 |
+
]
|
| 59 |
+
if not candidates:
|
| 60 |
+
raise FileNotFoundError(f"No audio files found in: {bgm_dir}")
|
| 61 |
+
return random.choice(candidates)
|
| 62 |
+
|
| 63 |
+
def _load_audio_mono(path: Path, sr: int) -> tuple[np.ndarray, int]:
|
| 64 |
+
|
| 65 |
+
try:
|
| 66 |
+
y, sr_out = librosa.load(path, sr=sr, mono=True)
|
| 67 |
+
return y.astype(np.float32, copy=False), int(sr_out)
|
| 68 |
+
except Exception as e1:
|
| 69 |
+
# librosa读取失败,用ffmpeg兜底
|
| 70 |
+
import os
|
| 71 |
+
import subprocess
|
| 72 |
+
import tempfile
|
| 73 |
+
|
| 74 |
+
tmp_wav = None
|
| 75 |
+
try:
|
| 76 |
+
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
|
| 77 |
+
tmp_wav = tmp.name
|
| 78 |
+
|
| 79 |
+
cmd = [
|
| 80 |
+
"ffmpeg",
|
| 81 |
+
"-y",
|
| 82 |
+
"-i", str(path),
|
| 83 |
+
"-ac", "1",
|
| 84 |
+
"-ar", str(sr),
|
| 85 |
+
"-vn",
|
| 86 |
+
tmp_wav,
|
| 87 |
+
]
|
| 88 |
+
subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
| 89 |
+
|
| 90 |
+
y, sr_out = librosa.load(tmp_wav, sr=sr, mono=True)
|
| 91 |
+
return y.astype(np.float32, copy=False), int(sr_out)
|
| 92 |
+
|
| 93 |
+
except FileNotFoundError as e_ffmpeg:
|
| 94 |
+
raise RuntimeError(
|
| 95 |
+
f"无法加载音频,并且没有找到ffmpeg"
|
| 96 |
+
) from e_ffmpeg
|
| 97 |
+
|
| 98 |
+
except Exception as e2:
|
| 99 |
+
raise RuntimeError(
|
| 100 |
+
f"无法加载音频:{type(e1).__name__}: {e1}"
|
| 101 |
+
f"ffmpeg 错误:{type(e2).__name__}: {e2}"
|
| 102 |
+
) from e2
|
| 103 |
+
finally:
|
| 104 |
+
if tmp_wav is not None:
|
| 105 |
+
try:
|
| 106 |
+
os.remove(tmp_wav)
|
| 107 |
+
except Exception:
|
| 108 |
+
pass
|
| 109 |
+
|
| 110 |
+
def analyze_music_metrics(
|
| 111 |
+
path: Path,
|
| 112 |
+
sr: int = 22050,
|
| 113 |
+
hop_length = 2048,
|
| 114 |
+
frame_length = 2048,
|
| 115 |
+
|
| 116 |
+
) -> dict[str, Any]:
|
| 117 |
+
if not path.exists():
|
| 118 |
+
raise FileNotFoundError(f"File not found: {path}")
|
| 119 |
+
|
| 120 |
+
y, sample_rate = _load_audio_mono(path, sr=sr)
|
| 121 |
+
duration = int(librosa.get_duration(y=y, sr=sample_rate) * 1000)
|
| 122 |
+
|
| 123 |
+
if y.size < frame_length:
|
| 124 |
+
raise RuntimeError("选择的BGM过短")
|
| 125 |
+
|
| 126 |
+
onset_env = librosa.onset.onset_strength(y=y, sr=sample_rate, hop_length=hop_length)
|
| 127 |
+
bpm, beat_frames = librosa.beat.beat_track(
|
| 128 |
+
onset_envelope=onset_env,
|
| 129 |
+
sr=sr,
|
| 130 |
+
hop_length=hop_length,
|
| 131 |
+
units="frames",
|
| 132 |
+
)
|
| 133 |
+
|
| 134 |
+
bpm_val = float(np.atleast_1d(bpm)[0])
|
| 135 |
+
|
| 136 |
+
beat_frames = np.asarray(beat_frames, dtype=int)
|
| 137 |
+
|
| 138 |
+
beat_times = compute_accent_beats(y=y, sr=sample_rate, beat_frames=beat_frames, hop_length=hop_length)
|
| 139 |
+
|
| 140 |
+
rms = librosa.feature.rms(
|
| 141 |
+
y=y,
|
| 142 |
+
frame_length=frame_length,
|
| 143 |
+
hop_length=hop_length
|
| 144 |
+
)[0]
|
| 145 |
+
|
| 146 |
+
energy_mean = float(np.mean(rms))
|
| 147 |
+
|
| 148 |
+
rms_db = librosa.amplitude_to_db(np.maximum(rms, 1e-10), ref=1.0)
|
| 149 |
+
energy_mean_db = float(np.mean(rms_db))
|
| 150 |
+
|
| 151 |
+
lo = float(np.percentile(rms_db, 10.0))
|
| 152 |
+
hi = float(np.percentile(rms_db, 95.0))
|
| 153 |
+
dynamic_range_db = float(hi - lo)
|
| 154 |
+
|
| 155 |
+
return {
|
| 156 |
+
"bgm_id": path.stem,
|
| 157 |
+
"path": str(path),
|
| 158 |
+
"duration": duration,
|
| 159 |
+
"sample_rate": sample_rate,
|
| 160 |
+
"bpm": bpm_val,
|
| 161 |
+
"beats": beat_times,
|
| 162 |
+
"energy_mean": energy_mean,
|
| 163 |
+
"energy_mean_db": energy_mean_db,
|
| 164 |
+
"dynamic_range_db": dynamic_range_db,
|
| 165 |
+
}
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def compute_accent_beats(
|
| 170 |
+
y: np.ndarray,
|
| 171 |
+
sr: int,
|
| 172 |
+
beat_frames: np.ndarray,
|
| 173 |
+
hop_length: int,
|
| 174 |
+
top_pct: float = 70.0,
|
| 175 |
+
min_sep_beats: int = 1, # 抑制相邻 beat:1 表示避免连续拍都入选
|
| 176 |
+
use_percussive: bool = True, # 用打击成分算 onset
|
| 177 |
+
local_norm_win: int = 8, # 局部归一化窗口(以 beat 为单位)
|
| 178 |
+
require_local_peak: bool = True # 只保留局部峰值
|
| 179 |
+
) -> list[float]:
|
| 180 |
+
"""
|
| 181 |
+
计算前 top_pct % 强度的鼓点的时间戳
|
| 182 |
+
"""
|
| 183 |
+
|
| 184 |
+
if beat_frames.size == 0:
|
| 185 |
+
return []
|
| 186 |
+
|
| 187 |
+
# 1) onset envelope 用 percussive 版本
|
| 188 |
+
y_for_onset = librosa.effects.percussive(y) if use_percussive else y
|
| 189 |
+
onset_env = librosa.onset.onset_strength(y=y_for_onset, sr=sr, hop_length=hop_length)
|
| 190 |
+
|
| 191 |
+
# 2) 取每个 beat 时刻的 onset 强度作为 beat 强度
|
| 192 |
+
beat_frames_clip = np.clip(beat_frames.astype(int), 0, len(onset_env) - 1)
|
| 193 |
+
strength = onset_env[beat_frames_clip].astype(np.float64) # shape (n_beats,)
|
| 194 |
+
|
| 195 |
+
# 3) 局部归一化:避免“整体更响的一段”把所有 beat 都选到
|
| 196 |
+
if strength.size >= 3 and local_norm_win >= 3:
|
| 197 |
+
w = int(local_norm_win)
|
| 198 |
+
kernel = np.ones(w, dtype=np.float64) / w
|
| 199 |
+
local_mean = np.convolve(strength, kernel, mode="same")
|
| 200 |
+
strength_norm = strength / (local_mean + 1e-8)
|
| 201 |
+
else:
|
| 202 |
+
strength_norm = strength.copy()
|
| 203 |
+
|
| 204 |
+
# 4) top 30%
|
| 205 |
+
thr = float(np.percentile(strength_norm, 100.0 - top_pct))
|
| 206 |
+
cand = np.where(strength_norm >= thr)[0] # indices into beats
|
| 207 |
+
|
| 208 |
+
# 5) 只保留局部峰值,避免平台期很多 beat 都入选
|
| 209 |
+
if require_local_peak and cand.size > 0 and strength_norm.size >= 3:
|
| 210 |
+
is_peak = np.zeros_like(strength_norm, dtype=bool)
|
| 211 |
+
is_peak[1:-1] = (strength_norm[1:-1] >= strength_norm[:-2]) & (strength_norm[1:-1] >= strength_norm[2:])
|
| 212 |
+
is_peak[0] = strength_norm[0] >= strength_norm[1]
|
| 213 |
+
is_peak[-1] = strength_norm[-1] >= strength_norm[-2]
|
| 214 |
+
cand = cand[is_peak[cand]]
|
| 215 |
+
|
| 216 |
+
# 6) 最小间隔抑制
|
| 217 |
+
selected = []
|
| 218 |
+
if cand.size > 0:
|
| 219 |
+
order = cand[np.argsort(-strength_norm[cand])]
|
| 220 |
+
suppressed = np.zeros(strength_norm.size, dtype=bool)
|
| 221 |
+
|
| 222 |
+
for idx in order:
|
| 223 |
+
if suppressed[idx]:
|
| 224 |
+
continue
|
| 225 |
+
selected.append(int(idx))
|
| 226 |
+
lo = max(0, idx - min_sep_beats)
|
| 227 |
+
hi = min(strength_norm.size, idx + min_sep_beats + 1)
|
| 228 |
+
suppressed[lo:hi] = True
|
| 229 |
+
|
| 230 |
+
selected = np.array(sorted(selected), dtype=int)
|
| 231 |
+
|
| 232 |
+
accent_frames = beat_frames[selected]
|
| 233 |
+
accent_times = librosa.frames_to_time(accent_frames, sr=sr, hop_length=hop_length).tolist()
|
| 234 |
+
|
| 235 |
+
accent_times_ms = [round(x * 1000) for x in accent_times]
|
| 236 |
+
|
| 237 |
+
return accent_times_ms
|
src/open_storyline/nodes/core_nodes/split_shots.py
ADDED
|
@@ -0,0 +1,778 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from typing import Any, Dict, Optional, List, Tuple
|
| 6 |
+
import csv
|
| 7 |
+
import functools
|
| 8 |
+
import os
|
| 9 |
+
import shutil
|
| 10 |
+
import subprocess
|
| 11 |
+
import math
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
import numpy as np
|
| 15 |
+
|
| 16 |
+
from open_storyline.nodes.core_nodes.base_node import BaseNode, NodeMeta
|
| 17 |
+
from open_storyline.nodes.node_schema import SplitShotsInput
|
| 18 |
+
|
| 19 |
+
MODEL_CACHE_MAXSIZE = 4
|
| 20 |
+
|
| 21 |
+
# TransNetV2 expects frames with shape [..., 27, 48, 3] in this implementation.
|
| 22 |
+
TRANSNETV2_INPUT_HEIGHT = 27
|
| 23 |
+
TRANSNETV2_INPUT_WIDTH = 48
|
| 24 |
+
TRANSNETV2_INPUT_CHANNELS = 3
|
| 25 |
+
|
| 26 |
+
DEFAULT_SCENE_DETECTION_FRAMES_PER_SECOND = 25
|
| 27 |
+
DEFAULT_SCENE_DETECTION_THRESHOLD = 0.5
|
| 28 |
+
DEFAULT_SPLIT_POINT_MINIMUM_GAP_SECONDS = 1e-3
|
| 29 |
+
|
| 30 |
+
DEFAULT_MIN_SHOT_DURATION_MILLISECONDS = 1000
|
| 31 |
+
DEFAULT_MAX_SHOT_DURATION_MILLISECONDS = 30000
|
| 32 |
+
|
| 33 |
+
CLIP_ID_NUMBER_WIDTH = 4
|
| 34 |
+
MILLISECONDS_PER_SECOND = 1000.0
|
| 35 |
+
|
| 36 |
+
FFMPEG_LOGLEVEL = "error"
|
| 37 |
+
FFMPEG_PIXEL_FORMAT_RGB24 = "rgb24"
|
| 38 |
+
FFMPEG_SCALE_FLAGS = "fast_bilinear"
|
| 39 |
+
FFMPEG_STDOUT_PIPE = "pipe:1"
|
| 40 |
+
|
| 41 |
+
FFMPEG_ENVIRONMENT_VARIABLE_KEYS = ("IMAGEIO_FFMPEG_EXE", "FFMPEG_BINARY")
|
| 42 |
+
|
| 43 |
+
# 如果检测后不需要切分(没有 split points),是否仍然 copy 一份到 output_dir
|
| 44 |
+
COPY_VIDEO_WHEN_NO_SPLIT = False
|
| 45 |
+
|
| 46 |
+
@dataclass(frozen=True)
|
| 47 |
+
class VideoSegment:
|
| 48 |
+
path: Path
|
| 49 |
+
start_seconds: float
|
| 50 |
+
end_seconds: float # ffmpeg segment csv might use -1 for "until end" in our wrapper
|
| 51 |
+
|
| 52 |
+
# =========================
|
| 53 |
+
# Model / ffmpeg helpers
|
| 54 |
+
# =========================
|
| 55 |
+
|
| 56 |
+
@functools.lru_cache(maxsize=MODEL_CACHE_MAXSIZE)
|
| 57 |
+
def load_transnetv2_model_cached(weight_path: str, device: str = "auto"):
|
| 58 |
+
"""
|
| 59 |
+
Load TransNetV2 model with LRU cache. Suitable for service mode.
|
| 60 |
+
"""
|
| 61 |
+
import torch
|
| 62 |
+
from transnetv2_pytorch import TransNetV2
|
| 63 |
+
|
| 64 |
+
model = TransNetV2(device=device)
|
| 65 |
+
model.eval()
|
| 66 |
+
|
| 67 |
+
state_dict = torch.load(weight_path, map_location=model.device)
|
| 68 |
+
model.load_state_dict(state_dict)
|
| 69 |
+
return model
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def resolve_ffmpeg_executable() -> str:
|
| 73 |
+
"""
|
| 74 |
+
Resolve ffmpeg executable path:
|
| 75 |
+
1) env var IMAGEIO_FFMPEG_EXE / FFMPEG_BINARY
|
| 76 |
+
2) system PATH
|
| 77 |
+
3) imageio-ffmpeg
|
| 78 |
+
"""
|
| 79 |
+
# 1) Environment variables
|
| 80 |
+
for key in FFMPEG_ENVIRONMENT_VARIABLE_KEYS:
|
| 81 |
+
configured_value = os.getenv(key)
|
| 82 |
+
if not configured_value:
|
| 83 |
+
continue
|
| 84 |
+
|
| 85 |
+
configured_path = Path(configured_value).expanduser()
|
| 86 |
+
if configured_path.exists():
|
| 87 |
+
return str(configured_path)
|
| 88 |
+
|
| 89 |
+
# env var may also be just "ffmpeg" or a command name
|
| 90 |
+
resolved_from_path = shutil.which(configured_value)
|
| 91 |
+
if resolved_from_path:
|
| 92 |
+
return resolved_from_path
|
| 93 |
+
|
| 94 |
+
# 2) System PATH
|
| 95 |
+
ffmpeg_in_path = shutil.which("ffmpeg")
|
| 96 |
+
if ffmpeg_in_path:
|
| 97 |
+
return ffmpeg_in_path
|
| 98 |
+
|
| 99 |
+
# 3) imageio-ffmpeg
|
| 100 |
+
try:
|
| 101 |
+
import imageio_ffmpeg
|
| 102 |
+
ffmpeg_from_imageio = imageio_ffmpeg.get_ffmpeg_exe()
|
| 103 |
+
if ffmpeg_from_imageio:
|
| 104 |
+
return ffmpeg_from_imageio
|
| 105 |
+
except Exception:
|
| 106 |
+
pass
|
| 107 |
+
|
| 108 |
+
raise RuntimeError("ffmpeg not found (checked env vars, PATH, and imageio-ffmpeg).")
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def read_video_frames_as_rgb24(
|
| 112 |
+
input_video: Path,
|
| 113 |
+
ffmpeg_executable: str,
|
| 114 |
+
*,
|
| 115 |
+
frames_per_second: int = DEFAULT_SCENE_DETECTION_FRAMES_PER_SECOND,
|
| 116 |
+
target_width: int = TRANSNETV2_INPUT_WIDTH,
|
| 117 |
+
target_height: int = TRANSNETV2_INPUT_HEIGHT,
|
| 118 |
+
) -> np.ndarray:
|
| 119 |
+
"""
|
| 120 |
+
Use ffmpeg to decode frames at fixed FPS and fixed size, output as raw RGB24 bytes.
|
| 121 |
+
Returns: np.ndarray with shape [frame_count, target_height, target_width, 3], dtype=uint8
|
| 122 |
+
"""
|
| 123 |
+
input_video = Path(input_video)
|
| 124 |
+
|
| 125 |
+
video_filter = (
|
| 126 |
+
f"fps={frames_per_second},"
|
| 127 |
+
f"scale={target_width}:{target_height}:flags={FFMPEG_SCALE_FLAGS}"
|
| 128 |
+
)
|
| 129 |
+
|
| 130 |
+
command = [
|
| 131 |
+
ffmpeg_executable, "-hide_banner", "-loglevel", FFMPEG_LOGLEVEL, "-nostdin",
|
| 132 |
+
"-i", str(input_video),
|
| 133 |
+
"-an",
|
| 134 |
+
"-vf", video_filter,
|
| 135 |
+
"-pix_fmt", FFMPEG_PIXEL_FORMAT_RGB24,
|
| 136 |
+
"-f", "rawvideo",
|
| 137 |
+
FFMPEG_STDOUT_PIPE,
|
| 138 |
+
]
|
| 139 |
+
|
| 140 |
+
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
| 141 |
+
assert process.stdout is not None and process.stderr is not None
|
| 142 |
+
|
| 143 |
+
stdout_bytes, stderr_bytes = process.communicate()
|
| 144 |
+
if process.returncode != 0:
|
| 145 |
+
raise RuntimeError(
|
| 146 |
+
f"ffmpeg frame extraction failed: {input_video}\n"
|
| 147 |
+
f"{stderr_bytes.decode('utf-8', errors='replace')}"
|
| 148 |
+
)
|
| 149 |
+
|
| 150 |
+
bytes_per_frame = target_width * target_height * TRANSNETV2_INPUT_CHANNELS
|
| 151 |
+
frame_count = len(stdout_bytes) // bytes_per_frame
|
| 152 |
+
|
| 153 |
+
if frame_count <= 0:
|
| 154 |
+
return np.empty((0, target_height, target_width, TRANSNETV2_INPUT_CHANNELS), dtype=np.uint8)
|
| 155 |
+
|
| 156 |
+
stdout_bytes = stdout_bytes[: frame_count * bytes_per_frame]
|
| 157 |
+
frames = np.frombuffer(stdout_bytes, dtype=np.uint8).reshape(
|
| 158 |
+
(frame_count, target_height, target_width, TRANSNETV2_INPUT_CHANNELS)
|
| 159 |
+
)
|
| 160 |
+
return frames
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def detect_scenes_with_transnetv2_without_proxy(
|
| 164 |
+
model: Any,
|
| 165 |
+
input_video: Path,
|
| 166 |
+
ffmpeg_executable: str,
|
| 167 |
+
*,
|
| 168 |
+
frames_per_second: int = DEFAULT_SCENE_DETECTION_FRAMES_PER_SECOND,
|
| 169 |
+
threshold: float = DEFAULT_SCENE_DETECTION_THRESHOLD,
|
| 170 |
+
) -> List[Dict[str, Any]]:
|
| 171 |
+
"""
|
| 172 |
+
No proxy file:
|
| 173 |
+
ffmpeg -> frames (uint8) -> model.predict_raw -> predictions_to_scenes_with_data
|
| 174 |
+
"""
|
| 175 |
+
import torch
|
| 176 |
+
|
| 177 |
+
frames_numpy = read_video_frames_as_rgb24(
|
| 178 |
+
input_video,
|
| 179 |
+
ffmpeg_executable,
|
| 180 |
+
frames_per_second=frames_per_second,
|
| 181 |
+
target_width=TRANSNETV2_INPUT_WIDTH,
|
| 182 |
+
target_height=TRANSNETV2_INPUT_HEIGHT,
|
| 183 |
+
)
|
| 184 |
+
|
| 185 |
+
if frames_numpy.size == 0 or frames_numpy.shape[0] == 0:
|
| 186 |
+
return []
|
| 187 |
+
|
| 188 |
+
frames_tensor = torch.from_numpy(frames_numpy) # [T, H, W, 3], uint8
|
| 189 |
+
frames_tensor = frames_tensor.unsqueeze(0).contiguous() # [1, T, H, W, 3]
|
| 190 |
+
|
| 191 |
+
model_device = getattr(model, "device", None)
|
| 192 |
+
if model_device is not None:
|
| 193 |
+
frames_tensor = frames_tensor.to(model_device, non_blocking=True)
|
| 194 |
+
|
| 195 |
+
with torch.inference_mode():
|
| 196 |
+
single_prediction, _all_prediction = model.predict_raw(frames_tensor)
|
| 197 |
+
|
| 198 |
+
prediction = single_prediction.detach().cpu().numpy()
|
| 199 |
+
prediction = np.squeeze(prediction)
|
| 200 |
+
if prediction.ndim != 1:
|
| 201 |
+
prediction = prediction.reshape(-1)
|
| 202 |
+
|
| 203 |
+
scenes = model.predictions_to_scenes_with_data(
|
| 204 |
+
prediction,
|
| 205 |
+
fps=float(frames_per_second),
|
| 206 |
+
threshold=float(threshold),
|
| 207 |
+
)
|
| 208 |
+
return scenes
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
def convert_scenes_to_split_points_seconds(
|
| 212 |
+
scenes: List[Dict[str, Any]],
|
| 213 |
+
*,
|
| 214 |
+
minimum_gap_seconds: float = DEFAULT_SPLIT_POINT_MINIMUM_GAP_SECONDS,
|
| 215 |
+
) -> List[float]:
|
| 216 |
+
"""
|
| 217 |
+
Convert TransNetV2 scenes to ffmpeg segment split points.
|
| 218 |
+
split points: [t1, t2, ...] means segments [0,t1], [t1,t2], ..., [last,end]
|
| 219 |
+
"""
|
| 220 |
+
end_times: List[float] = []
|
| 221 |
+
last_end_time = 0.0
|
| 222 |
+
|
| 223 |
+
for scene in scenes:
|
| 224 |
+
try:
|
| 225 |
+
end_time = float(scene.get("end_time", 0.0))
|
| 226 |
+
except Exception:
|
| 227 |
+
continue
|
| 228 |
+
|
| 229 |
+
if end_time > last_end_time + minimum_gap_seconds:
|
| 230 |
+
end_times.append(end_time)
|
| 231 |
+
last_end_time = end_time
|
| 232 |
+
|
| 233 |
+
# If <=1 scene, don't split
|
| 234 |
+
if len(end_times) <= 1:
|
| 235 |
+
return []
|
| 236 |
+
|
| 237 |
+
# Remove the last end_time (usually video end)
|
| 238 |
+
return end_times[:-1]
|
| 239 |
+
|
| 240 |
+
def enforce_shot_duration_constraints_on_split_points_seconds(
|
| 241 |
+
split_points_seconds: List[float],
|
| 242 |
+
*,
|
| 243 |
+
total_duration_milliseconds: int,
|
| 244 |
+
min_shot_duration_milliseconds: Optional[int],
|
| 245 |
+
max_shot_duration_milliseconds: Optional[int],
|
| 246 |
+
) -> List[float]:
|
| 247 |
+
"""
|
| 248 |
+
对“切分点(split points)”施加 min/max 时长约束(单位 ms):
|
| 249 |
+
1) 若某段 < min:通过删除相应切分点,把它与相邻段拼接(优先向后拼;尾段太短则向前拼)。
|
| 250 |
+
2) 若某段 > max:在该段内部强制均匀切分(允许镜头内仍有镜头切换)。
|
| 251 |
+
|
| 252 |
+
注意:这里在“调用 ffmpeg 前”调整 split points,从而避免切完后再做文件拼接(性能更好)。
|
| 253 |
+
"""
|
| 254 |
+
duration_ms = int(total_duration_milliseconds)
|
| 255 |
+
|
| 256 |
+
def _normalize_optional_ms(value: Optional[int], key_name: str) -> Optional[int]:
|
| 257 |
+
if value is None:
|
| 258 |
+
return None
|
| 259 |
+
if isinstance(value, bool):
|
| 260 |
+
raise ValueError(f"{key_name} must be int milliseconds, got bool")
|
| 261 |
+
try:
|
| 262 |
+
value_int = int(value)
|
| 263 |
+
except Exception as exc:
|
| 264 |
+
raise ValueError(f"{key_name} must be int milliseconds, got {value!r}") from exc
|
| 265 |
+
if value_int < 0:
|
| 266 |
+
raise ValueError(f"{key_name} must be >= 0, got {value_int}")
|
| 267 |
+
if value_int == 0:
|
| 268 |
+
return None
|
| 269 |
+
return value_int
|
| 270 |
+
|
| 271 |
+
min_ms = _normalize_optional_ms(min_shot_duration_milliseconds, "min_shot_duration")
|
| 272 |
+
max_ms = _normalize_optional_ms(max_shot_duration_milliseconds, "max_shot_duration")
|
| 273 |
+
|
| 274 |
+
if min_ms is not None and max_ms is not None and min_ms > max_ms:
|
| 275 |
+
raise ValueError(f"min_shot_duration ({min_ms}) cannot be greater than max_shot_duration ({max_ms}).")
|
| 276 |
+
|
| 277 |
+
# seconds -> milliseconds cut points
|
| 278 |
+
cut_points_ms = [
|
| 279 |
+
int(round(point_seconds * MILLISECONDS_PER_SECOND))
|
| 280 |
+
for point_seconds in split_points_seconds
|
| 281 |
+
]
|
| 282 |
+
cut_points_ms = sorted({c for c in cut_points_ms if 0 < c < duration_ms})
|
| 283 |
+
|
| 284 |
+
# ---------- Step 1: merge short segments (< min) by removing cut points ----------
|
| 285 |
+
if min_ms is not None and cut_points_ms:
|
| 286 |
+
merged_cut_points: List[int] = []
|
| 287 |
+
segment_start_ms = 0
|
| 288 |
+
|
| 289 |
+
for cut_ms in cut_points_ms:
|
| 290 |
+
segment_length_ms = cut_ms - segment_start_ms
|
| 291 |
+
if segment_length_ms < min_ms:
|
| 292 |
+
continue
|
| 293 |
+
merged_cut_points.append(cut_ms)
|
| 294 |
+
segment_start_ms = cut_ms
|
| 295 |
+
|
| 296 |
+
if merged_cut_points and (duration_ms - merged_cut_points[-1] < min_ms):
|
| 297 |
+
merged_cut_points.pop()
|
| 298 |
+
|
| 299 |
+
cut_points_ms = merged_cut_points
|
| 300 |
+
|
| 301 |
+
# ---------- Step 2: split long segments (> max) by inserting internal cut points ----------
|
| 302 |
+
if max_ms is not None and max_ms > 0:
|
| 303 |
+
cuts_set = set(cut_points_ms)
|
| 304 |
+
boundaries = [0] + cut_points_ms + [duration_ms]
|
| 305 |
+
|
| 306 |
+
for segment_start_ms, segment_end_ms in zip(boundaries[:-1], boundaries[1:]):
|
| 307 |
+
segment_length_ms = segment_end_ms - segment_start_ms
|
| 308 |
+
if segment_length_ms <= max_ms:
|
| 309 |
+
continue
|
| 310 |
+
|
| 311 |
+
# 最少切成 pieces 段,保证每段 <= max_ms
|
| 312 |
+
pieces = int(math.ceil(segment_length_ms / max_ms))
|
| 313 |
+
if pieces <= 1:
|
| 314 |
+
continue
|
| 315 |
+
|
| 316 |
+
# 均匀分配(整数 ms),尽量避免“最后剩一小段”
|
| 317 |
+
base = segment_length_ms // pieces
|
| 318 |
+
remainder = segment_length_ms % pieces # 前 remainder 段多 1ms
|
| 319 |
+
|
| 320 |
+
current = segment_start_ms
|
| 321 |
+
for i in range(pieces - 1):
|
| 322 |
+
piece_len = base + 1 if i < remainder else base
|
| 323 |
+
current += piece_len
|
| 324 |
+
if segment_start_ms < current < segment_end_ms:
|
| 325 |
+
cuts_set.add(current)
|
| 326 |
+
|
| 327 |
+
cut_points_ms = sorted(c for c in cuts_set if 0 < c < duration_ms)
|
| 328 |
+
|
| 329 |
+
# milliseconds -> seconds
|
| 330 |
+
return [cut_ms / MILLISECONDS_PER_SECOND for cut_ms in cut_points_ms]
|
| 331 |
+
|
| 332 |
+
def segment_video_stream_copy_with_ffmpeg(
|
| 333 |
+
input_video: Path,
|
| 334 |
+
ffmpeg_executable: str,
|
| 335 |
+
*,
|
| 336 |
+
split_points_seconds: List[float],
|
| 337 |
+
output_directory: Path,
|
| 338 |
+
filename_prefix: str,
|
| 339 |
+
start_index: int = 0,
|
| 340 |
+
) -> List[VideoSegment]:
|
| 341 |
+
"""
|
| 342 |
+
Fast segmentation: stream copy (-c copy) + segment muxer.
|
| 343 |
+
Returns segments with start/end in seconds from ffmpeg segment list csv.
|
| 344 |
+
"""
|
| 345 |
+
output_directory.mkdir(parents=True, exist_ok=True)
|
| 346 |
+
|
| 347 |
+
# No split points -> single output copy
|
| 348 |
+
if not split_points_seconds:
|
| 349 |
+
output_path = output_directory / f"{filename_prefix}_{start_index:0{CLIP_ID_NUMBER_WIDTH}d}.mp4"
|
| 350 |
+
command = [
|
| 351 |
+
ffmpeg_executable, "-hide_banner", "-loglevel", FFMPEG_LOGLEVEL, "-nostdin",
|
| 352 |
+
"-y",
|
| 353 |
+
"-i", str(input_video),
|
| 354 |
+
"-map", "0",
|
| 355 |
+
"-c", "copy",
|
| 356 |
+
"-movflags", "+faststart",
|
| 357 |
+
str(output_path),
|
| 358 |
+
]
|
| 359 |
+
completed = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
| 360 |
+
if completed.returncode != 0:
|
| 361 |
+
raise RuntimeError(
|
| 362 |
+
f"ffmpeg stream copy failed: {input_video}\n"
|
| 363 |
+
f"{completed.stderr.decode('utf-8', errors='replace')}"
|
| 364 |
+
)
|
| 365 |
+
return [VideoSegment(path=output_path, start_seconds=0.0, end_seconds=-1.0)]
|
| 366 |
+
|
| 367 |
+
split_points_argument = ",".join(f"{t:.3f}" for t in split_points_seconds)
|
| 368 |
+
|
| 369 |
+
segment_list_csv_path = output_directory / f"{filename_prefix}_{start_index:0{CLIP_ID_NUMBER_WIDTH}d}.csv"
|
| 370 |
+
output_pattern = output_directory / f"{filename_prefix}_%0{CLIP_ID_NUMBER_WIDTH}d.mp4"
|
| 371 |
+
|
| 372 |
+
command = [
|
| 373 |
+
ffmpeg_executable, "-hide_banner", "-loglevel", FFMPEG_LOGLEVEL, "-nostdin",
|
| 374 |
+
"-y",
|
| 375 |
+
"-i", str(input_video),
|
| 376 |
+
"-map", "0",
|
| 377 |
+
"-c", "copy",
|
| 378 |
+
"-f", "segment",
|
| 379 |
+
"-segment_start_number", str(start_index),
|
| 380 |
+
"-segment_list", str(segment_list_csv_path),
|
| 381 |
+
"-segment_list_type", "csv",
|
| 382 |
+
"-segment_times", split_points_argument,
|
| 383 |
+
"-reset_timestamps", "1",
|
| 384 |
+
"-segment_format_options", "movflags=+faststart",
|
| 385 |
+
str(output_pattern),
|
| 386 |
+
]
|
| 387 |
+
|
| 388 |
+
completed = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
| 389 |
+
if completed.returncode != 0:
|
| 390 |
+
raise RuntimeError(
|
| 391 |
+
f"ffmpeg segment failed: {input_video}\n"
|
| 392 |
+
f"{completed.stderr.decode('utf-8', errors='replace')}"
|
| 393 |
+
)
|
| 394 |
+
|
| 395 |
+
segments: List[VideoSegment] = []
|
| 396 |
+
with segment_list_csv_path.open("r", encoding="utf-8", newline="") as file_handle:
|
| 397 |
+
csv_reader = csv.reader(file_handle)
|
| 398 |
+
for row in csv_reader:
|
| 399 |
+
if not row or len(row) < 3:
|
| 400 |
+
continue
|
| 401 |
+
filename, start_time, end_time = row[0], row[1], row[2]
|
| 402 |
+
segments.append(
|
| 403 |
+
VideoSegment(
|
| 404 |
+
path=output_directory / filename,
|
| 405 |
+
start_seconds=float(start_time),
|
| 406 |
+
end_seconds=float(end_time),
|
| 407 |
+
)
|
| 408 |
+
)
|
| 409 |
+
|
| 410 |
+
return segments
|
| 411 |
+
|
| 412 |
+
|
| 413 |
+
# =========================
|
| 414 |
+
# Node implementation
|
| 415 |
+
# =========================
|
| 416 |
+
|
| 417 |
+
class SplitShotsNode(BaseNode):
|
| 418 |
+
meta = NodeMeta(
|
| 419 |
+
name="split_shots",
|
| 420 |
+
description="将输入素材按照镜头变化切割。",
|
| 421 |
+
node_id="split_shots",
|
| 422 |
+
node_kind="split_shots",
|
| 423 |
+
require_prior_kind=["assets_index"],
|
| 424 |
+
default_require_prior_kind=["assets_index"],
|
| 425 |
+
next_available_node=["understand_clips", "xhs_understand_clips"],
|
| 426 |
+
)
|
| 427 |
+
input_schema = SplitShotsInput
|
| 428 |
+
|
| 429 |
+
def __init__(self, *args) -> None:
|
| 430 |
+
super().__init__(*args)
|
| 431 |
+
|
| 432 |
+
self.transnetv2_model = load_transnetv2_model_cached(
|
| 433 |
+
str(self.server_cfg.split_shots.transnet_weights),
|
| 434 |
+
device=self.server_cfg.split_shots.transnet_device,
|
| 435 |
+
)
|
| 436 |
+
self.ffmpeg_executable = resolve_ffmpeg_executable()
|
| 437 |
+
|
| 438 |
+
# -------------------------
|
| 439 |
+
# Public entrypoints
|
| 440 |
+
# -------------------------
|
| 441 |
+
|
| 442 |
+
async def default_process(self, node_summary, inputs: Dict[str, Any]) -> Any:
|
| 443 |
+
"""
|
| 444 |
+
Default behavior: do NOT split shots, just pass-through paths.
|
| 445 |
+
(Optimization) No re-save / no re-encode.
|
| 446 |
+
"""
|
| 447 |
+
output_directory = self._prepare_output_directory(inputs)
|
| 448 |
+
assets = self._extract_assets(inputs)
|
| 449 |
+
|
| 450 |
+
clips: List[Dict[str, Any]] = []
|
| 451 |
+
clip_index = 1
|
| 452 |
+
|
| 453 |
+
for asset in assets:
|
| 454 |
+
clip = self._build_clip_without_splitting(asset=asset, clip_index=clip_index, node_summary=node_summary)
|
| 455 |
+
clips.append(clip)
|
| 456 |
+
clip_index += 1
|
| 457 |
+
|
| 458 |
+
node_summary.info_for_user("镜头切分已跳过")
|
| 459 |
+
return {"clips": clips}
|
| 460 |
+
|
| 461 |
+
async def process(self, node_summary, inputs: Dict[str, Any]) -> Any:
|
| 462 |
+
"""
|
| 463 |
+
Split shots using TransNetV2 + ffmpeg segment copy.
|
| 464 |
+
"""
|
| 465 |
+
output_directory = self._prepare_output_directory(inputs)
|
| 466 |
+
assets = self._extract_assets(inputs)
|
| 467 |
+
|
| 468 |
+
clips: List[Dict[str, Any]] = []
|
| 469 |
+
clip_index = 1
|
| 470 |
+
|
| 471 |
+
min_shot_duration_milliseconds = inputs.get("min_shot_duration", DEFAULT_MIN_SHOT_DURATION_MILLISECONDS)
|
| 472 |
+
max_shot_duration_milliseconds = inputs.get("max_shot_duration", DEFAULT_MAX_SHOT_DURATION_MILLISECONDS)
|
| 473 |
+
|
| 474 |
+
if min_shot_duration_milliseconds > max_shot_duration_milliseconds:
|
| 475 |
+
min_shot_duration_milliseconds, max_shot_duration_milliseconds = DEFAULT_MIN_SHOT_DURATION_MILLISECONDS, DEFAULT_MAX_SHOT_DURATION_MILLISECONDS
|
| 476 |
+
node_summary.add_warning(
|
| 477 |
+
f"min_shot_duration_milliseconds ({min_shot_duration_milliseconds}) cannot greater than max_shot_duration_milliseconds ({max_shot_duration_milliseconds})"
|
| 478 |
+
f"using default config {DEFAULT_MIN_SHOT_DURATION_MILLISECONDS} and ({DEFAULT_MAX_SHOT_DURATION_MILLISECONDS})",
|
| 479 |
+
artifact_id = inputs.get("artifact_id"),
|
| 480 |
+
)
|
| 481 |
+
|
| 482 |
+
if min_shot_duration_milliseconds < DEFAULT_MIN_SHOT_DURATION_MILLISECONDS:
|
| 483 |
+
min_shot_duration_milliseconds = DEFAULT_MIN_SHOT_DURATION_MILLISECONDS
|
| 484 |
+
node_summary.add_warning(
|
| 485 |
+
f"min_shot_duration_milliseconds ({min_shot_duration_milliseconds}) too small"
|
| 486 |
+
f"using default config {DEFAULT_MIN_SHOT_DURATION_MILLISECONDS}",
|
| 487 |
+
artifact_id = inputs.get("artifact_id"),
|
| 488 |
+
)
|
| 489 |
+
|
| 490 |
+
if max_shot_duration_milliseconds > DEFAULT_MAX_SHOT_DURATION_MILLISECONDS:
|
| 491 |
+
max_shot_duration_milliseconds = DEFAULT_MAX_SHOT_DURATION_MILLISECONDS
|
| 492 |
+
node_summary.add_warning(
|
| 493 |
+
f"max_shot_duration_milliseconds ({max_shot_duration_milliseconds}) too great"
|
| 494 |
+
f"using default config {DEFAULT_MAX_SHOT_DURATION_MILLISECONDS}",
|
| 495 |
+
artifact_id = inputs.get("artifact_id"),
|
| 496 |
+
)
|
| 497 |
+
|
| 498 |
+
for asset in assets:
|
| 499 |
+
new_clips, clip_index = self._process_single_asset(
|
| 500 |
+
asset=asset,
|
| 501 |
+
output_directory=output_directory,
|
| 502 |
+
starting_clip_index=clip_index,
|
| 503 |
+
node_summary=node_summary,
|
| 504 |
+
min_shot_duration_milliseconds=min_shot_duration_milliseconds,
|
| 505 |
+
max_shot_duration_milliseconds=max_shot_duration_milliseconds,
|
| 506 |
+
)
|
| 507 |
+
clips.extend(new_clips)
|
| 508 |
+
|
| 509 |
+
node_summary.info_for_user(
|
| 510 |
+
f"{self.meta.node_id} 执行成功,输出 clips 数量:{len(clips)}"
|
| 511 |
+
)
|
| 512 |
+
return {"clips": clips}
|
| 513 |
+
|
| 514 |
+
# -------------------------
|
| 515 |
+
# Internal helpers
|
| 516 |
+
# -------------------------
|
| 517 |
+
|
| 518 |
+
def _prepare_output_directory(self, inputs: Dict[str, Any]) -> Path:
|
| 519 |
+
artifact_id = inputs["artifact_id"]
|
| 520 |
+
session_id = inputs["session_id"]
|
| 521 |
+
output_directory = self.server_cache_dir / session_id / artifact_id
|
| 522 |
+
output_directory.mkdir(parents=True, exist_ok=True)
|
| 523 |
+
return output_directory
|
| 524 |
+
|
| 525 |
+
def _extract_assets(self, inputs: Dict[str, Any]) -> List[Dict[str, Any]]:
|
| 526 |
+
return (inputs.get("assets_index") or {}).get("assets", []) or []
|
| 527 |
+
|
| 528 |
+
def _format_clip_id(self, clip_index: int) -> str:
|
| 529 |
+
return f"clip_{clip_index:0{CLIP_ID_NUMBER_WIDTH}d}"
|
| 530 |
+
|
| 531 |
+
def _require_asset_id(self, asset: Dict[str, Any]) -> str:
|
| 532 |
+
asset_id = asset.get("asset_id")
|
| 533 |
+
if not asset_id:
|
| 534 |
+
raise ValueError(f"asset missing required field 'asset_id': {asset}")
|
| 535 |
+
return str(asset_id)
|
| 536 |
+
|
| 537 |
+
def _require_media_type(self, asset: Dict[str, Any]) -> str:
|
| 538 |
+
media_type = asset.get("media_type")
|
| 539 |
+
if not media_type:
|
| 540 |
+
raise ValueError(f"asset missing required field 'media_type': {asset}")
|
| 541 |
+
return str(media_type)
|
| 542 |
+
|
| 543 |
+
def _require_video_metadata(self, asset_id: str, asset: Dict[str, Any]) -> Dict[str, Any]:
|
| 544 |
+
metadata = asset.get("metadata") or {}
|
| 545 |
+
if "duration" not in metadata:
|
| 546 |
+
raise ValueError(f"video asset_id={asset_id} missing metadata.duration")
|
| 547 |
+
return metadata
|
| 548 |
+
|
| 549 |
+
def _parse_duration_milliseconds(self, asset_id: str, metadata: Dict[str, Any]) -> int:
|
| 550 |
+
try:
|
| 551 |
+
duration_milliseconds = int(metadata["duration"])
|
| 552 |
+
except (TypeError, ValueError):
|
| 553 |
+
raise ValueError(f"video asset_id={asset_id} has invalid metadata.duration: {metadata.get('duration')!r}")
|
| 554 |
+
|
| 555 |
+
if duration_milliseconds < 0:
|
| 556 |
+
raise ValueError(f"video asset_id={asset_id} has negative duration: {duration_milliseconds}")
|
| 557 |
+
return duration_milliseconds
|
| 558 |
+
|
| 559 |
+
def _require_path(self, asset_id: str, asset: Dict[str, Any], *, field_name: str) -> str:
|
| 560 |
+
path_value = asset.get(field_name)
|
| 561 |
+
if not path_value:
|
| 562 |
+
raise ValueError(f"asset_id={asset_id} missing required field {field_name!r}")
|
| 563 |
+
return str(path_value)
|
| 564 |
+
|
| 565 |
+
def _build_clip_without_splitting(self, asset: Dict[str, Any], clip_index: int, node_summary) -> Dict[str, Any]:
|
| 566 |
+
"""
|
| 567 |
+
Build a single clip without cutting:
|
| 568 |
+
- image: use orig_path (or fallback to path)
|
| 569 |
+
- video: use path (no re-save)
|
| 570 |
+
"""
|
| 571 |
+
asset_id = self._require_asset_id(asset)
|
| 572 |
+
media_type = self._require_media_type(asset)
|
| 573 |
+
clip_id = self._format_clip_id(clip_index)
|
| 574 |
+
|
| 575 |
+
if media_type == "image":
|
| 576 |
+
image_path = asset.get("orig_path") or asset.get("path")
|
| 577 |
+
if not image_path:
|
| 578 |
+
raise ValueError(f"image asset_id={asset_id} missing 'orig_path'/'path'")
|
| 579 |
+
node_summary.info_for_user(f"{clip_id} 分割完成", preview_urls=[image_path])
|
| 580 |
+
return {
|
| 581 |
+
"clip_id": clip_id,
|
| 582 |
+
"kind": "image",
|
| 583 |
+
"path": image_path,
|
| 584 |
+
"source_ref": {"asset_id": asset_id},
|
| 585 |
+
}
|
| 586 |
+
|
| 587 |
+
if media_type != "video":
|
| 588 |
+
raise ValueError(f"unsupported media_type {media_type!r} for asset_id={asset_id}")
|
| 589 |
+
|
| 590 |
+
metadata = self._require_video_metadata(asset_id, asset)
|
| 591 |
+
duration_milliseconds = self._parse_duration_milliseconds(asset_id, metadata)
|
| 592 |
+
video_path = self._require_path(asset_id, asset, field_name="path")
|
| 593 |
+
|
| 594 |
+
node_summary.info_for_user(f"{clip_id} 分割完成", preview_urls=[video_path])
|
| 595 |
+
return {
|
| 596 |
+
"clip_id": clip_id,
|
| 597 |
+
"kind": "video",
|
| 598 |
+
"path": video_path,
|
| 599 |
+
"fps": metadata.get("fps"),
|
| 600 |
+
"source_ref": {
|
| 601 |
+
"asset_id": asset_id,
|
| 602 |
+
"start": 0,
|
| 603 |
+
"end": duration_milliseconds,
|
| 604 |
+
"duration": duration_milliseconds,
|
| 605 |
+
"height": metadata.get("height"),
|
| 606 |
+
"width": metadata.get("width"),
|
| 607 |
+
},
|
| 608 |
+
}
|
| 609 |
+
|
| 610 |
+
def _process_single_asset(
|
| 611 |
+
self,
|
| 612 |
+
*,
|
| 613 |
+
asset: Dict[str, Any],
|
| 614 |
+
output_directory: Path,
|
| 615 |
+
starting_clip_index: int,
|
| 616 |
+
node_summary,
|
| 617 |
+
min_shot_duration_milliseconds: int,
|
| 618 |
+
max_shot_duration_milliseconds: int,
|
| 619 |
+
) -> Tuple[List[Dict[str, Any]], int]:
|
| 620 |
+
"""
|
| 621 |
+
Return: (clips_generated, next_clip_index)
|
| 622 |
+
"""
|
| 623 |
+
asset_id = self._require_asset_id(asset)
|
| 624 |
+
media_type = self._require_media_type(asset)
|
| 625 |
+
|
| 626 |
+
if media_type == "image":
|
| 627 |
+
clip_id = self._format_clip_id(starting_clip_index)
|
| 628 |
+
image_path = asset.get("orig_path") or asset.get("path")
|
| 629 |
+
if not image_path:
|
| 630 |
+
raise ValueError(f"image asset_id={asset_id} missing 'orig_path'/'path'")
|
| 631 |
+
|
| 632 |
+
node_summary.info_for_user(f"{clip_id} 分割完成", preview_urls=[image_path])
|
| 633 |
+
clip = {
|
| 634 |
+
"clip_id": clip_id,
|
| 635 |
+
"kind": "image",
|
| 636 |
+
"path": image_path,
|
| 637 |
+
"source_ref": {"asset_id": asset_id},
|
| 638 |
+
}
|
| 639 |
+
return [clip], starting_clip_index + 1
|
| 640 |
+
|
| 641 |
+
if media_type != "video":
|
| 642 |
+
raise ValueError(f"unsupported media_type {media_type!r} for asset_id={asset_id}")
|
| 643 |
+
|
| 644 |
+
video_clips, next_index = self._process_video_asset(
|
| 645 |
+
asset_id=asset_id,
|
| 646 |
+
asset=asset,
|
| 647 |
+
output_directory=output_directory,
|
| 648 |
+
starting_clip_index=starting_clip_index,
|
| 649 |
+
node_summary=node_summary,
|
| 650 |
+
min_shot_duration_milliseconds=min_shot_duration_milliseconds,
|
| 651 |
+
max_shot_duration_milliseconds=max_shot_duration_milliseconds,
|
| 652 |
+
)
|
| 653 |
+
return video_clips, next_index
|
| 654 |
+
|
| 655 |
+
def _process_video_asset(
|
| 656 |
+
self,
|
| 657 |
+
*,
|
| 658 |
+
asset_id: str,
|
| 659 |
+
asset: Dict[str, Any],
|
| 660 |
+
output_directory: Path,
|
| 661 |
+
starting_clip_index: int,
|
| 662 |
+
node_summary,
|
| 663 |
+
min_shot_duration_milliseconds: int,
|
| 664 |
+
max_shot_duration_milliseconds: int,
|
| 665 |
+
) -> Tuple[List[Dict[str, Any]], int]:
|
| 666 |
+
metadata = self._require_video_metadata(asset_id, asset)
|
| 667 |
+
duration_milliseconds = self._parse_duration_milliseconds(asset_id, metadata)
|
| 668 |
+
|
| 669 |
+
input_video_path = Path(self._require_path(asset_id, asset, field_name="path")).expanduser()
|
| 670 |
+
|
| 671 |
+
# 如果素材本身就短于 min_shot_duration:完全跳过切分与拼接
|
| 672 |
+
if min_shot_duration_milliseconds is not None and duration_milliseconds < min_shot_duration_milliseconds:
|
| 673 |
+
clip_id = self._format_clip_id(starting_clip_index)
|
| 674 |
+
node_summary.info_for_user(f"{clip_id} 分割完成", preview_urls=[str(input_video_path)])
|
| 675 |
+
clip = {
|
| 676 |
+
"clip_id": clip_id,
|
| 677 |
+
"kind": "video",
|
| 678 |
+
"path": str(input_video_path),
|
| 679 |
+
"fps": metadata.get("fps"),
|
| 680 |
+
"source_ref": {
|
| 681 |
+
"asset_id": asset_id,
|
| 682 |
+
"start": 0,
|
| 683 |
+
"end": duration_milliseconds,
|
| 684 |
+
"duration": duration_milliseconds,
|
| 685 |
+
"height": metadata.get("height"),
|
| 686 |
+
"width": metadata.get("width"),
|
| 687 |
+
},
|
| 688 |
+
}
|
| 689 |
+
return [clip], starting_clip_index + 1
|
| 690 |
+
|
| 691 |
+
# 1) Detect scenes
|
| 692 |
+
scenes = detect_scenes_with_transnetv2_without_proxy(
|
| 693 |
+
self.transnetv2_model,
|
| 694 |
+
input_video_path,
|
| 695 |
+
self.ffmpeg_executable,
|
| 696 |
+
frames_per_second=DEFAULT_SCENE_DETECTION_FRAMES_PER_SECOND,
|
| 697 |
+
threshold=DEFAULT_SCENE_DETECTION_THRESHOLD,
|
| 698 |
+
)
|
| 699 |
+
split_points_seconds = convert_scenes_to_split_points_seconds(scenes)
|
| 700 |
+
|
| 701 |
+
split_points_seconds = enforce_shot_duration_constraints_on_split_points_seconds(
|
| 702 |
+
split_points_seconds,
|
| 703 |
+
total_duration_milliseconds=duration_milliseconds,
|
| 704 |
+
min_shot_duration_milliseconds=min_shot_duration_milliseconds,
|
| 705 |
+
max_shot_duration_milliseconds=max_shot_duration_milliseconds,
|
| 706 |
+
)
|
| 707 |
+
|
| 708 |
+
# 2) If no split needed, optionally skip copying
|
| 709 |
+
if not split_points_seconds and not COPY_VIDEO_WHEN_NO_SPLIT:
|
| 710 |
+
clip_id = self._format_clip_id(starting_clip_index)
|
| 711 |
+
node_summary.info_for_user(f"{clip_id} 分割完成", preview_urls=[str(input_video_path)])
|
| 712 |
+
clip = {
|
| 713 |
+
"clip_id": clip_id,
|
| 714 |
+
"kind": "video",
|
| 715 |
+
"path": str(input_video_path),
|
| 716 |
+
"fps": metadata.get("fps"),
|
| 717 |
+
"source_ref": {
|
| 718 |
+
"asset_id": asset_id,
|
| 719 |
+
"start": 0,
|
| 720 |
+
"end": duration_milliseconds,
|
| 721 |
+
"duration": duration_milliseconds,
|
| 722 |
+
"height": metadata.get("height"),
|
| 723 |
+
"width": metadata.get("width"),
|
| 724 |
+
},
|
| 725 |
+
}
|
| 726 |
+
return [clip], starting_clip_index + 1
|
| 727 |
+
|
| 728 |
+
# 3) Segment by ffmpeg (-c copy)
|
| 729 |
+
filename_prefix = "clip"
|
| 730 |
+
segments = segment_video_stream_copy_with_ffmpeg(
|
| 731 |
+
input_video=input_video_path,
|
| 732 |
+
ffmpeg_executable=self.ffmpeg_executable,
|
| 733 |
+
split_points_seconds=split_points_seconds,
|
| 734 |
+
output_directory=output_directory,
|
| 735 |
+
filename_prefix=filename_prefix,
|
| 736 |
+
start_index=starting_clip_index,
|
| 737 |
+
)
|
| 738 |
+
|
| 739 |
+
# 4) Build clip list
|
| 740 |
+
clips: List[Dict[str, Any]] = []
|
| 741 |
+
clip_index = starting_clip_index
|
| 742 |
+
|
| 743 |
+
for segment in segments:
|
| 744 |
+
clip_id = self._format_clip_id(clip_index)
|
| 745 |
+
|
| 746 |
+
if segment.end_seconds < 0:
|
| 747 |
+
start_milliseconds = 0
|
| 748 |
+
end_milliseconds = duration_milliseconds
|
| 749 |
+
else:
|
| 750 |
+
start_milliseconds = max(0, int(round(segment.start_seconds * MILLISECONDS_PER_SECOND)))
|
| 751 |
+
end_milliseconds = max(start_milliseconds, int(round(segment.end_seconds * MILLISECONDS_PER_SECOND)))
|
| 752 |
+
|
| 753 |
+
segment_duration_milliseconds = max(0, end_milliseconds - start_milliseconds)
|
| 754 |
+
if segment_duration_milliseconds <= 0:
|
| 755 |
+
continue
|
| 756 |
+
|
| 757 |
+
output_path_string = str(segment.path)
|
| 758 |
+
node_summary.info_for_user(f"{clip_id} 分割完成", preview_urls=[output_path_string])
|
| 759 |
+
|
| 760 |
+
clips.append(
|
| 761 |
+
{
|
| 762 |
+
"clip_id": clip_id,
|
| 763 |
+
"kind": "video",
|
| 764 |
+
"path": output_path_string,
|
| 765 |
+
"fps": metadata.get("fps"),
|
| 766 |
+
"source_ref": {
|
| 767 |
+
"asset_id": asset_id,
|
| 768 |
+
"start": start_milliseconds,
|
| 769 |
+
"end": end_milliseconds,
|
| 770 |
+
"duration": segment_duration_milliseconds,
|
| 771 |
+
"height": metadata.get("height"),
|
| 772 |
+
"width": metadata.get("width"),
|
| 773 |
+
},
|
| 774 |
+
}
|
| 775 |
+
)
|
| 776 |
+
clip_index += 1
|
| 777 |
+
|
| 778 |
+
return clips, clip_index
|
src/open_storyline/nodes/core_nodes/understand_clips.py
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Any, Dict
|
| 2 |
+
import asyncio
|
| 3 |
+
|
| 4 |
+
from open_storyline.nodes.core_nodes.base_node import BaseNode, NodeMeta
|
| 5 |
+
from open_storyline.mcp.sampling_requester import LLMClient
|
| 6 |
+
from src.open_storyline.utils.prompts import get_prompt
|
| 7 |
+
from open_storyline.utils.parse_json import parse_json_dict
|
| 8 |
+
from open_storyline.nodes.node_schema import UnderstandClipsInput
|
| 9 |
+
|
| 10 |
+
class UnderstandClipsNode(BaseNode):
|
| 11 |
+
"""
|
| 12 |
+
素材理解 节点
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
meta = NodeMeta(
|
| 16 |
+
name="understand_clips",
|
| 17 |
+
description="分析当前工程的所有clips,进行素材理解,生成对每个素材的一段描述。依赖split_shots节点的结果",
|
| 18 |
+
node_id="understand_clips",
|
| 19 |
+
node_kind="understand_clips",
|
| 20 |
+
require_prior_kind=['assets_index', 'split_shots'],
|
| 21 |
+
default_require_prior_kind=['assets_index', 'split_shots'],
|
| 22 |
+
next_available_node=['filter_clips', 'xhs_filter_clips'],
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
input_schema = UnderstandClipsInput
|
| 26 |
+
|
| 27 |
+
async def default_process(
|
| 28 |
+
self,
|
| 29 |
+
node_summary,
|
| 30 |
+
inputs: Dict[str, Any],
|
| 31 |
+
) -> Any:
|
| 32 |
+
clips = inputs["split_shots"]["clips"]
|
| 33 |
+
|
| 34 |
+
clip_captions: list[dict[str, Any]] = []
|
| 35 |
+
for clip in clips or []:
|
| 36 |
+
clip_captions.append(
|
| 37 |
+
{
|
| 38 |
+
"clip_id": clip.get("clip_id"),
|
| 39 |
+
"caption": "no caption",
|
| 40 |
+
"source_ref": {
|
| 41 |
+
"media_id": clip.get("source_ref", {}).get("asset_id", ""),
|
| 42 |
+
}
|
| 43 |
+
}
|
| 44 |
+
)
|
| 45 |
+
node_summary.info_for_user(f"未对{len(clips)}个片段生成描述")
|
| 46 |
+
return {
|
| 47 |
+
"clip_captions": clip_captions,
|
| 48 |
+
"overall": "unknown",
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
async def process(self, node_summary, inputs: Dict[str, Any]) -> Any:
|
| 52 |
+
"""
|
| 53 |
+
inputs: BaseNode.load_inputs(ctx)读到的前面节点的结果
|
| 54 |
+
"""
|
| 55 |
+
assets_index = inputs["assets"]
|
| 56 |
+
clips = inputs["split_shots"]["clips"]
|
| 57 |
+
llm: LLMClient = inputs["llm"]
|
| 58 |
+
system_prompt = get_prompt("understand_clips.system_detail")
|
| 59 |
+
user_prompt = get_prompt("understand_clips.user_detail")
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
clip_captions: list[dict[str, Any]] = []
|
| 63 |
+
|
| 64 |
+
for clip in clips or []:
|
| 65 |
+
clip_id = str(clip.get("clip_id", "") or "").strip() or "(unknown_clip)"
|
| 66 |
+
kind = str(clip.get("kind", "") or "").strip().lower()
|
| 67 |
+
src = clip.get("source_ref") or {}
|
| 68 |
+
|
| 69 |
+
asset_id = str(src.get("asset_id", "") or "")
|
| 70 |
+
asset = assets_index.get(asset_id)
|
| 71 |
+
|
| 72 |
+
out_item: dict[str, Any] = {
|
| 73 |
+
"clip_id": clip_id,
|
| 74 |
+
}
|
| 75 |
+
|
| 76 |
+
if not asset:
|
| 77 |
+
out_item["caption"] = f"错误:找不到 asset_id={asset_id} 对应的素材"
|
| 78 |
+
clip_captions.append(out_item)
|
| 79 |
+
continue
|
| 80 |
+
|
| 81 |
+
path = str(asset.get("path", "") or "").strip()
|
| 82 |
+
if not path:
|
| 83 |
+
out_item["caption"] = f"错误: asset_id={asset_id} 的path为空"
|
| 84 |
+
clip_captions.append(out_item)
|
| 85 |
+
continue
|
| 86 |
+
|
| 87 |
+
# 组装 media
|
| 88 |
+
media: list[Any] = []
|
| 89 |
+
|
| 90 |
+
if kind == "image":
|
| 91 |
+
media = [{"path": path}]
|
| 92 |
+
|
| 93 |
+
elif kind == "video":
|
| 94 |
+
in_sec = _safe_float(src.get("start", 0) / 1000.0, 0.0)
|
| 95 |
+
|
| 96 |
+
if src.get("end") is not None:
|
| 97 |
+
out_sec = _safe_float(src.get("end", 0) / 1000.0, in_sec)
|
| 98 |
+
else:
|
| 99 |
+
dur = _safe_float(src.get("duration", 0.0), 0.0)
|
| 100 |
+
out_sec = in_sec + max(0.0, dur)
|
| 101 |
+
|
| 102 |
+
if out_sec <= in_sec:
|
| 103 |
+
out_sec = in_sec + 0.1
|
| 104 |
+
|
| 105 |
+
media = [{
|
| 106 |
+
"path": path,
|
| 107 |
+
"in_sec": in_sec,
|
| 108 |
+
"out_sec": out_sec,
|
| 109 |
+
}]
|
| 110 |
+
else:
|
| 111 |
+
out_item["caption"] = f"错误:不支持的 clip kind:{kind}"
|
| 112 |
+
clip_captions.append(out_item)
|
| 113 |
+
continue
|
| 114 |
+
|
| 115 |
+
raw = await llm.complete(
|
| 116 |
+
system_prompt=system_prompt,
|
| 117 |
+
user_prompt=user_prompt,
|
| 118 |
+
media=media,
|
| 119 |
+
temperature=0.3,
|
| 120 |
+
top_p=0.9,
|
| 121 |
+
max_tokens=2048,
|
| 122 |
+
model_preferences=None,
|
| 123 |
+
)
|
| 124 |
+
if raw is None:
|
| 125 |
+
out_item["caption"] = ("错误:VLM调用失败")
|
| 126 |
+
clip_captions.append(out_item)
|
| 127 |
+
continue # 继续处理下一个 clip
|
| 128 |
+
|
| 129 |
+
try:
|
| 130 |
+
obj = parse_json_dict(raw)
|
| 131 |
+
except:
|
| 132 |
+
text = (raw or "").strip()
|
| 133 |
+
out_item["caption"] = text if text else "错误:无法解析模型输出"
|
| 134 |
+
clip_captions.append(out_item)
|
| 135 |
+
continue
|
| 136 |
+
|
| 137 |
+
out_item["caption"] = str(obj.get("caption", "") or "").strip()
|
| 138 |
+
out_item["source_ref"] = {
|
| 139 |
+
"media_id": clip.get("source_ref", {}).get("asset_id", ""),
|
| 140 |
+
}
|
| 141 |
+
clip_captions.append(out_item)
|
| 142 |
+
|
| 143 |
+
desc_lines: list[str] = []
|
| 144 |
+
for desc in clip_captions:
|
| 145 |
+
text = str(desc.get("caption"))
|
| 146 |
+
desc_lines.append(f"- {desc.get('clip_instance_id')}: {text}")
|
| 147 |
+
|
| 148 |
+
overall_summary = ""
|
| 149 |
+
if desc_lines:
|
| 150 |
+
overall_system_prompt = get_prompt("understand_clips.system_overall")
|
| 151 |
+
overall_user_prompt = get_prompt("understand_clips.user_overall", clips_captions=desc_lines)
|
| 152 |
+
|
| 153 |
+
try:
|
| 154 |
+
overall_summary = await llm.complete(
|
| 155 |
+
system_prompt=overall_system_prompt,
|
| 156 |
+
user_prompt=overall_user_prompt,
|
| 157 |
+
media=None,
|
| 158 |
+
temperature=0.3,
|
| 159 |
+
top_p=0.9,
|
| 160 |
+
max_tokens=1024,
|
| 161 |
+
model_preferences=None
|
| 162 |
+
)
|
| 163 |
+
|
| 164 |
+
except Exception as e:
|
| 165 |
+
overall_summary = f"Error: 总结生成失败:{type(e).__name__}: {e}"
|
| 166 |
+
node_summary.info_for_user(f"素材理解完成,共分析 {len(clip_captions)} 个片段,整体主要描述了{overall_summary}")
|
| 167 |
+
return {
|
| 168 |
+
"clip_captions": clip_captions,
|
| 169 |
+
"overall": overall_summary
|
| 170 |
+
}
|
| 171 |
+
|
| 172 |
+
def _parse_input(self, node_summary, inputs: Dict[str, Any]):
|
| 173 |
+
assets = inputs["assets_index"]["assets"]
|
| 174 |
+
|
| 175 |
+
assets_index: dict[str, dict[str, Any]] = {}
|
| 176 |
+
for asset in assets or []:
|
| 177 |
+
asset_id = asset.get("asset_id")
|
| 178 |
+
if asset_id:
|
| 179 |
+
assets_index[str(asset_id)] = asset
|
| 180 |
+
inputs.update({"assets": assets_index})
|
| 181 |
+
return inputs
|
| 182 |
+
|
| 183 |
+
def _safe_float(x: Any, default: float = 0.0) -> float:
|
| 184 |
+
try:
|
| 185 |
+
if x is None:
|
| 186 |
+
return default
|
| 187 |
+
return float(x)
|
| 188 |
+
except Exception:
|
| 189 |
+
return default
|
src/open_storyline/nodes/node_manager.py
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
from collections import defaultdict
|
| 3 |
+
from typing import Any, Dict, List, Optional, Set
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
from langchain_core.tools.structured import StructuredTool
|
| 7 |
+
|
| 8 |
+
from src.open_storyline.storage.agent_memory import ArtifactStore
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class NodeManager:
|
| 12 |
+
def __init__(self, tools: List[StructuredTool] = None):
|
| 13 |
+
# 核心数据结构
|
| 14 |
+
self.kind_to_node_ids: Dict[str, List[str]] = defaultdict(list) # node_kind -> node_id列表(已排序)
|
| 15 |
+
self.id_to_tool: Dict[str, StructuredTool] = {} # node_id -> tool
|
| 16 |
+
self.id_to_next: Dict[str, List[str]] = {} # node_id -> 下一个可执行的node_id列表
|
| 17 |
+
self.id_to_priority: Dict[str, int] = {} # node_id -> priority
|
| 18 |
+
self.id_to_kind: Dict[str, str] = {} # node_id -> node_kind (方便删除时查找)
|
| 19 |
+
|
| 20 |
+
# 新增:前置依赖相关
|
| 21 |
+
self.id_to_require_prior_kind: Dict[str, List[str]] = {} # node_id -> 必须的前置kind列表
|
| 22 |
+
self.id_to_default_require_prior_kind: Dict[str, List[str]] = {} # node_id -> 默认的前置kind列表
|
| 23 |
+
|
| 24 |
+
# 反向索引:哪些节点依赖某个kind
|
| 25 |
+
self.kind_to_dependent_nodes: Dict[str, Set[str]] = defaultdict(set) # kind -> 依赖该kind的node_id集合
|
| 26 |
+
self.kind_to_default_dependent_nodes: Dict[str, Set[str]] = defaultdict(set) # kind -> 默认依赖该kind的node_id集合
|
| 27 |
+
|
| 28 |
+
if tools:
|
| 29 |
+
self._build(tools)
|
| 30 |
+
|
| 31 |
+
def _build(self, tools: List[StructuredTool]):
|
| 32 |
+
"""批量构建数据结构"""
|
| 33 |
+
for tool in tools:
|
| 34 |
+
if tool.metadata:
|
| 35 |
+
metadata = tool.metadata.get('_meta', {})
|
| 36 |
+
node_id = metadata.get('node_id')
|
| 37 |
+
if node_id:
|
| 38 |
+
self.add_node(tool)
|
| 39 |
+
|
| 40 |
+
def add_node(self, tool: StructuredTool) -> bool:
|
| 41 |
+
"""添加节点"""
|
| 42 |
+
# metadata is None, failed to add node
|
| 43 |
+
if not tool.metadata:
|
| 44 |
+
return False
|
| 45 |
+
|
| 46 |
+
metadata = tool.metadata.get('_meta', {})
|
| 47 |
+
node_id = metadata.get('node_id')
|
| 48 |
+
|
| 49 |
+
if not node_id:
|
| 50 |
+
return False
|
| 51 |
+
|
| 52 |
+
# 如果已存在,先删除旧的
|
| 53 |
+
if node_id in self.id_to_tool:
|
| 54 |
+
self.remove_node(node_id)
|
| 55 |
+
|
| 56 |
+
node_kind = metadata.get('node_kind', node_id)
|
| 57 |
+
priority = metadata.get('priority', 0)
|
| 58 |
+
next_nodes = metadata.get('next_available_node', [])
|
| 59 |
+
require_prior_kind = metadata.get('require_prior_kind', [])
|
| 60 |
+
default_require_prior_kind = metadata.get('default_require_prior_kind', [])
|
| 61 |
+
|
| 62 |
+
# 更新所有数据结构
|
| 63 |
+
self.id_to_tool[node_id] = tool
|
| 64 |
+
self.id_to_priority[node_id] = priority
|
| 65 |
+
self.id_to_next[node_id] = next_nodes
|
| 66 |
+
self.id_to_kind[node_id] = node_kind
|
| 67 |
+
self.id_to_require_prior_kind[node_id] = require_prior_kind
|
| 68 |
+
self.id_to_default_require_prior_kind[node_id] = default_require_prior_kind
|
| 69 |
+
|
| 70 |
+
# 添加到kind分组并重新排序
|
| 71 |
+
self.kind_to_node_ids[node_kind].append(node_id)
|
| 72 |
+
self._sort_kind(node_kind)
|
| 73 |
+
|
| 74 |
+
# 更新反向索引
|
| 75 |
+
for kind in require_prior_kind:
|
| 76 |
+
self.kind_to_dependent_nodes[kind].add(node_id)
|
| 77 |
+
|
| 78 |
+
for kind in default_require_prior_kind:
|
| 79 |
+
self.kind_to_default_dependent_nodes[kind].add(node_id)
|
| 80 |
+
|
| 81 |
+
return True
|
| 82 |
+
|
| 83 |
+
def remove_node(self, node_id: str, clean_references: bool = True) -> bool:
|
| 84 |
+
"""
|
| 85 |
+
删除节点
|
| 86 |
+
|
| 87 |
+
Args:
|
| 88 |
+
node_id: 要删除的节点ID
|
| 89 |
+
clean_references: 是否清理其他节点对该节点的引用
|
| 90 |
+
"""
|
| 91 |
+
if node_id not in self.id_to_tool:
|
| 92 |
+
return False
|
| 93 |
+
|
| 94 |
+
node_kind = self.id_to_kind[node_id]
|
| 95 |
+
|
| 96 |
+
# 清理反向索引
|
| 97 |
+
if node_id in self.id_to_require_prior_kind:
|
| 98 |
+
for kind in self.id_to_require_prior_kind[node_id]:
|
| 99 |
+
self.kind_to_dependent_nodes[kind].discard(node_id)
|
| 100 |
+
if not self.kind_to_dependent_nodes[kind]:
|
| 101 |
+
del self.kind_to_dependent_nodes[kind]
|
| 102 |
+
|
| 103 |
+
if node_id in self.id_to_default_require_prior_kind:
|
| 104 |
+
for kind in self.id_to_default_require_prior_kind[node_id]:
|
| 105 |
+
self.kind_to_default_dependent_nodes[kind].discard(node_id)
|
| 106 |
+
if not self.kind_to_default_dependent_nodes[kind]:
|
| 107 |
+
del self.kind_to_default_dependent_nodes[kind]
|
| 108 |
+
|
| 109 |
+
# 从所有数据结构中删除
|
| 110 |
+
del self.id_to_tool[node_id]
|
| 111 |
+
del self.id_to_priority[node_id]
|
| 112 |
+
del self.id_to_next[node_id]
|
| 113 |
+
del self.id_to_kind[node_id]
|
| 114 |
+
|
| 115 |
+
if node_id in self.id_to_require_prior_kind:
|
| 116 |
+
del self.id_to_require_prior_kind[node_id]
|
| 117 |
+
if node_id in self.id_to_default_require_prior_kind:
|
| 118 |
+
del self.id_to_default_require_prior_kind[node_id]
|
| 119 |
+
|
| 120 |
+
# 从kind分组中删除
|
| 121 |
+
if node_id in self.kind_to_node_ids[node_kind]:
|
| 122 |
+
self.kind_to_node_ids[node_kind].remove(node_id)
|
| 123 |
+
|
| 124 |
+
# 如果该kind没有节点了,删除这个kind
|
| 125 |
+
if not self.kind_to_node_ids[node_kind]:
|
| 126 |
+
del self.kind_to_node_ids[node_kind]
|
| 127 |
+
|
| 128 |
+
# 清理其他节点对该节点的引用
|
| 129 |
+
if clean_references:
|
| 130 |
+
for nid in list(self.id_to_next.keys()):
|
| 131 |
+
if node_id in self.id_to_next[nid]:
|
| 132 |
+
self.id_to_next[nid].remove(node_id)
|
| 133 |
+
|
| 134 |
+
return True
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def _sort_kind(self, kind: str):
|
| 138 |
+
"""对指定kind的节点列表按优先级排序"""
|
| 139 |
+
if kind in self.kind_to_node_ids:
|
| 140 |
+
self.kind_to_node_ids[kind].sort(
|
| 141 |
+
key=lambda nid: self.id_to_priority[nid],
|
| 142 |
+
reverse=True
|
| 143 |
+
)
|
| 144 |
+
|
| 145 |
+
def get_tool(self, node_id: str) -> Optional[StructuredTool]:
|
| 146 |
+
"""根据node_id获取tool"""
|
| 147 |
+
return self.id_to_tool.get(node_id)
|
| 148 |
+
|
| 149 |
+
def check_excutable(self, session_id:str, store: ArtifactStore, all_require_kind: List[str]) -> Dict[str, Any]:
|
| 150 |
+
"""
|
| 151 |
+
检查是否可执行,并且返回未执行的功能
|
| 152 |
+
"""
|
| 153 |
+
collected_output = {}
|
| 154 |
+
for req_kind in all_require_kind:
|
| 155 |
+
req_ids_queue = self.kind_to_node_ids[req_kind]
|
| 156 |
+
# 1. 收集所有节点的最新输出
|
| 157 |
+
valid_outputs = []
|
| 158 |
+
for node_id in req_ids_queue:
|
| 159 |
+
output = store.get_latest_meta(node_id=node_id, session_id=session_id)
|
| 160 |
+
if output is not None:
|
| 161 |
+
valid_outputs.append(output)
|
| 162 |
+
|
| 163 |
+
# 2. 找出创建时间最晚的输出
|
| 164 |
+
if valid_outputs:
|
| 165 |
+
latest_output = max(valid_outputs, key=lambda output: output.created_at)
|
| 166 |
+
collected_output[req_kind] = latest_output
|
| 167 |
+
return {
|
| 168 |
+
"excutable": len(collected_output.keys())==len(all_require_kind),
|
| 169 |
+
"collected_node": collected_output,
|
| 170 |
+
"missing_kind": list(set(all_require_kind) - set(collected_output.keys()))
|
| 171 |
+
}
|
| 172 |
+
|
src/open_storyline/nodes/node_schema.py
ADDED
|
@@ -0,0 +1,283 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Dict, List, Literal, Any, Annotated, Optional, Union, ClassVar, Type
|
| 2 |
+
from pydantic import BaseModel, Field, model_validator, constr, conlist
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
class VideoMetadata(BaseModel):
|
| 6 |
+
"""视频元数据"""
|
| 7 |
+
width: int = Field(description="宽度")
|
| 8 |
+
height: int = Field(description="高度")
|
| 9 |
+
duration: float = Field(description="时长(毫秒)")
|
| 10 |
+
fps: float = Field(description="帧率")
|
| 11 |
+
has_audio: bool = Field(default=False, description="是否包含音频轨道")
|
| 12 |
+
|
| 13 |
+
audio_sample_rate_hz: Optional[int] = Field(
|
| 14 |
+
None,
|
| 15 |
+
gt=0,
|
| 16 |
+
description="音频采样率(Hz),常见值:44100, 48000"
|
| 17 |
+
)
|
| 18 |
+
|
| 19 |
+
@model_validator(mode='after')
|
| 20 |
+
def validate_audio_sample_rate(self):
|
| 21 |
+
"""如果有音频,采样率必填"""
|
| 22 |
+
if self.has_audio and self.audio_sample_rate_hz is None:
|
| 23 |
+
raise ValueError('视频包含音频时,必须提供 audio_sample_rate_hz')
|
| 24 |
+
return self
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class ImageMetadata(BaseModel):
|
| 28 |
+
"""图片元数据"""
|
| 29 |
+
width: int = Field(description="宽度")
|
| 30 |
+
height: int = Field(description="高度")
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class Asset(BaseModel):
|
| 34 |
+
"""单个素材"""
|
| 35 |
+
asset_id: str
|
| 36 |
+
path: str
|
| 37 |
+
media_type: Literal["video", "image", "audio", "unknown"]
|
| 38 |
+
metadata: Union[VideoMetadata, ImageMetadata]
|
| 39 |
+
extra_info: Optional[Dict[str, Any]] = None
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
class SourceRef(BaseModel):
|
| 43 |
+
""" 原素材参考信息 """
|
| 44 |
+
asset_id: str
|
| 45 |
+
start: float
|
| 46 |
+
end: float
|
| 47 |
+
duration: float
|
| 48 |
+
height: Optional[int] = None
|
| 49 |
+
width: Optional[int] = None
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
class Clip(BaseModel):
|
| 53 |
+
clip_id: str
|
| 54 |
+
language: Optional[str] = None
|
| 55 |
+
caption: str = Field(default="", description="素材的描述文字")
|
| 56 |
+
media_type: str
|
| 57 |
+
path: str
|
| 58 |
+
fps: Optional[float] = None
|
| 59 |
+
extra_info: Optional[Dict[str, Any]] = Field(default=None, description="额外信息")
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
class SubtitleUnit(BaseModel):
|
| 63 |
+
"""字幕切分单元"""
|
| 64 |
+
unit_id: str = Field(
|
| 65 |
+
...,
|
| 66 |
+
description="字幕单元唯一标识符",
|
| 67 |
+
example="subtitle_0001"
|
| 68 |
+
)
|
| 69 |
+
index_in_group: int = Field(
|
| 70 |
+
...,
|
| 71 |
+
ge=0,
|
| 72 |
+
description="在当前分组中的顺序索引(从0开始)",
|
| 73 |
+
example=0
|
| 74 |
+
)
|
| 75 |
+
text: str = Field(
|
| 76 |
+
...,
|
| 77 |
+
description="该字幕单元的文本内容",
|
| 78 |
+
example="猫猫不懂什么是KPI"
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
class GroupClips(BaseModel):
|
| 83 |
+
"""视频分组 - 画面素材组织"""
|
| 84 |
+
group_id: str = Field(
|
| 85 |
+
...,
|
| 86 |
+
description="分组唯一标识符",
|
| 87 |
+
example="group_0001"
|
| 88 |
+
)
|
| 89 |
+
summary: str = Field(
|
| 90 |
+
...,
|
| 91 |
+
description="分组的画面风格、情绪基调或剪辑意图描述",
|
| 92 |
+
example="先用最安静、最治愈的镜头建立情绪。"
|
| 93 |
+
)
|
| 94 |
+
clip_ids: List[str] = Field(
|
| 95 |
+
...,
|
| 96 |
+
description="该分组使用的视频片段ID列表,按播放顺序排列",
|
| 97 |
+
example=["clip_0003", "clip_0002"]
|
| 98 |
+
)
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
class GroupScript(BaseModel):
|
| 102 |
+
"""分组文案脚本"""
|
| 103 |
+
group_id: str = Field(
|
| 104 |
+
...,
|
| 105 |
+
description="分组唯一标识符",
|
| 106 |
+
example="group_0001"
|
| 107 |
+
)
|
| 108 |
+
raw_text:str = Field(
|
| 109 |
+
...,
|
| 110 |
+
description="该分组的完整原始文案内容",
|
| 111 |
+
example="猫猫不懂什么是KPI,猫猫只知道今天阳光很好"
|
| 112 |
+
)
|
| 113 |
+
subtitle_units: List = Field(
|
| 114 |
+
...,
|
| 115 |
+
description="字幕切分单元列表,用于精确控制字幕显示节奏"
|
| 116 |
+
)
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
class Voiceover(BaseModel):
|
| 120 |
+
"""单个旁白/配音项"""
|
| 121 |
+
group_id: str = Field(..., description="组ID,如group_0001")
|
| 122 |
+
voiceover_id: str = Field(..., description="配音ID,如voiceover_0001")
|
| 123 |
+
path: str = Field(..., description="配音文件路径")
|
| 124 |
+
duration: int = Field(..., description="配音时长(毫秒)", gt=0)
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
class BGM(BaseModel):
|
| 128 |
+
"""背景音乐"""
|
| 129 |
+
bgm_id: str = Field(..., description="BGM ID,如bgm_0003")
|
| 130 |
+
path: str = Field(..., description="BGM 文件路径")
|
| 131 |
+
duration: int = Field(..., description="BGM 时长(毫秒)", gt=0)
|
| 132 |
+
bpm: float = Field(..., description="每分钟节拍数", gt=0)
|
| 133 |
+
beats: List[int] = Field(default_factory=list, description="节拍时间点列表(毫秒)")
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
# TODO: Timeline schema完善
|
| 137 |
+
class TimeWindow(BaseModel):
|
| 138 |
+
start: int = Field(..., description="开始时间(毫秒)")
|
| 139 |
+
end: int = Field(..., description="结束时间(毫秒)")
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
class AudioMix(BaseModel):
|
| 143 |
+
gain_db: float = Field(default=0.0, description="增益分贝")
|
| 144 |
+
ducking: Optional[Any] = Field(default=None, description="闪避效果配置")
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
class ClipTrack(BaseModel):
|
| 148 |
+
clip_id: str
|
| 149 |
+
source_window: TimeWindow
|
| 150 |
+
timeline_window: TimeWindow
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
class BgmTrack(BaseModel):
|
| 154 |
+
bgm_asset_id: str
|
| 155 |
+
timeline_window: TimeWindow
|
| 156 |
+
mix: AudioMix
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
class SubtitleTrack(BaseModel):
|
| 160 |
+
# 示例结构
|
| 161 |
+
text: str
|
| 162 |
+
timeline_window: TimeWindow
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
class VoiceoverTrack(BaseModel):
|
| 166 |
+
asset_id: str
|
| 167 |
+
timeline_window: TimeWindow
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
class TimelineTracks(BaseModel):
|
| 171 |
+
video: List[ClipTrack] = Field(default_factory=list)
|
| 172 |
+
subtitles: List[SubtitleTrack] = Field(default_factory=list)
|
| 173 |
+
voiceover: List[VoiceoverTrack] = Field(default_factory=list)
|
| 174 |
+
bgm: List[BgmTrack] = Field(default_factory=list)
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
class BaseInput(BaseModel):
|
| 178 |
+
mode: Literal["auto", "skip", "default"] = Field(
|
| 179 |
+
default="auto",
|
| 180 |
+
description="auto: 自动模式;skip: 跳过模式;default: 默认模式"
|
| 181 |
+
)
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
class AssetsIndexInput(BaseInput):
|
| 185 |
+
...
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
class AssetsIndexOutput(BaseModel):
|
| 189 |
+
""" 当前节点输出 """
|
| 190 |
+
assets: List[Asset] = Field(
|
| 191 |
+
default_factory=list,
|
| 192 |
+
description="资产列表"
|
| 193 |
+
)
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
class SplitShotsInput(BaseInput):
|
| 197 |
+
mode: Literal["auto", "skip", "default"] = Field(
|
| 198 |
+
default="auto",
|
| 199 |
+
description="auto: 根据场景变化自动切分镜头,图片视为单个镜头;skip: 不切分镜头;default: 使用默认切分方法"
|
| 200 |
+
)
|
| 201 |
+
min_shot_duration: Annotated[int, Field(default=1000, description="切分的镜头不得短于该时间(单位:毫秒)")]
|
| 202 |
+
max_shot_duration: Annotated[int, Field(default=30000, description="若单个镜头时长超过该时间,则强制切分(单位:毫秒)")]
|
| 203 |
+
|
| 204 |
+
class SplitShotsOutput(BaseModel):
|
| 205 |
+
clip_captions: List[Clip] = Field(default_factory=list, description="片段列表")
|
| 206 |
+
overall: Dict[str, str]
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
class UnderstandClipsInput(BaseModel):
|
| 210 |
+
mode: Literal["auto", "skip", "default"] = Field(
|
| 211 |
+
default="auto",
|
| 212 |
+
description="auto: 根据素材内容生成对应的描述;skip: 不生成描述;default: 使用默认生成描述方法"
|
| 213 |
+
)
|
| 214 |
+
|
| 215 |
+
class UnderstandClipsOutput(BaseModel):
|
| 216 |
+
clip_captions: List[Clip] = Field(default_factory=list, description="片段列表")
|
| 217 |
+
overall: Dict[str, str]
|
| 218 |
+
|
| 219 |
+
class FilterClipsInput(BaseModel):
|
| 220 |
+
mode: Literal["auto", "skip", "default"] = Field(
|
| 221 |
+
default="auto",
|
| 222 |
+
description="auto: 根据用户的要求筛选片段;skip: 跳过;default: 使用默认筛选方法"
|
| 223 |
+
)
|
| 224 |
+
user_request: Annotated[str, Field(default="", description="用户对片段过滤的诉求,若没有请根据素材和其他剪辑要求拟定一个。")] = ""
|
| 225 |
+
|
| 226 |
+
class FilterClipsOutput(BaseModel):
|
| 227 |
+
clip_captions: List[Clip] = Field(default_factory=list, description="片段列表")
|
| 228 |
+
overall: Dict[str, str]
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
class GroupClipsInput(BaseModel):
|
| 232 |
+
mode: Literal["auto", "skip", "default"] = Field(
|
| 233 |
+
default="auto",
|
| 234 |
+
description="auto: 根据素材内容的叙事逻辑和用户对素材顺序的要求,组织合理的素材顺序;skip: 跳过;default: 使用默认生成描述方法"
|
| 235 |
+
)
|
| 236 |
+
user_request: Annotated[str, Field(default="", description="用户对素材组织顺序的要求,若没有请根据常规流程组织合理的叙事顺序。")]
|
| 237 |
+
|
| 238 |
+
class GroupClipsOutput(BaseModel):
|
| 239 |
+
groups: List[GroupClips] = Field(default_factory=list, description="片段列表")
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
class GenerateScriptInput(BaseModel):
|
| 243 |
+
mode: Literal["auto", "skip", "default"] = Field(
|
| 244 |
+
default="auto",
|
| 245 |
+
description="auto: 根据素材内容和用户对文案的要求,生成合适的文案;skip: 跳过,不添加字幕;default: 使用默认文案"
|
| 246 |
+
)
|
| 247 |
+
user_request: Annotated[str, Field(default="", description="用户对文案的要求.")]
|
| 248 |
+
|
| 249 |
+
class GenerateScriptOutput(BaseModel):
|
| 250 |
+
group_scripts: List[GroupScript]
|
| 251 |
+
title: Optional[str]
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
class GenerateVoiceoverInput(BaseInput):
|
| 255 |
+
mode: Literal["auto", "skip", "default"] = Field(
|
| 256 |
+
default="auto",
|
| 257 |
+
description="auto: 根据素材内容和用户对配音音色的要求,生成合适的配音;skip: 跳过;default: 使用默认配音"
|
| 258 |
+
)
|
| 259 |
+
user_request: Annotated[dict, Field(default={}, description="用户对配音的要求.")]
|
| 260 |
+
|
| 261 |
+
class GenerateVoiceoverOutput(BaseModel):
|
| 262 |
+
voiceover: List[Voiceover] = Field(default_factory=list, description="配音")
|
| 263 |
+
|
| 264 |
+
|
| 265 |
+
class SelectBGMInput(BaseInput):
|
| 266 |
+
mode: Literal["auto", "skip", "default"] = Field(
|
| 267 |
+
default="auto",
|
| 268 |
+
description="auto: 根据素材内容和用户对音乐的要求,选择合适的音乐;skip: 不使用音乐;default: 使用默认音乐"
|
| 269 |
+
)
|
| 270 |
+
user_request: Annotated[dict, Field(default={}, description="用户对背景音乐的要求.")]
|
| 271 |
+
|
| 272 |
+
class SelectBGMOutput(BaseModel):
|
| 273 |
+
bgm: List[BGM] = Field(default_factory=list, description="背景音乐")
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
class PlanTimelineInput(BaseInput):
|
| 277 |
+
use_beats: Annotated[bool, Field(default=True, description="素材切换是否需要卡BGM鼓点")]
|
| 278 |
+
|
| 279 |
+
class PlanTimelineOutput(BaseModel):
|
| 280 |
+
tracks: List[TimelineTracks] = Field(default_factory=list, description="时间线轨道集合")
|
| 281 |
+
|
| 282 |
+
class RenderVideoInput(BaseInput):
|
| 283 |
+
...
|
src/open_storyline/nodes/node_summary.py
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass, field
|
| 2 |
+
from typing import Any, Dict, List, Tuple, Optional
|
| 3 |
+
import logging
|
| 4 |
+
from datetime import datetime
|
| 5 |
+
|
| 6 |
+
from open_storyline.utils.logging import get_logger
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
@dataclass
|
| 10 |
+
class LogEntry:
|
| 11 |
+
"""单条日志记录"""
|
| 12 |
+
level: str
|
| 13 |
+
message: str
|
| 14 |
+
timestamp: str
|
| 15 |
+
artifact_id: Optional[str] = None
|
| 16 |
+
extra_data: Dict[str, Any] = field(default_factory=dict)
|
| 17 |
+
|
| 18 |
+
@dataclass
|
| 19 |
+
class NodeSummary:
|
| 20 |
+
"""
|
| 21 |
+
Node Execution Status Summary - Reuses existing logger module
|
| 22 |
+
|
| 23 |
+
Features:
|
| 24 |
+
1. ERROR - Error messages for LLM
|
| 25 |
+
2. WARNING - Warning messages for LLM
|
| 26 |
+
3. DEBUG - Debug information for developers
|
| 27 |
+
4. INFO_LLM - Detailed information for LLM
|
| 28 |
+
5. INFO_USER - Brief information for users
|
| 29 |
+
|
| 30 |
+
Capabilities:
|
| 31 |
+
- Reuses get_logger() configuration
|
| 32 |
+
- Hierarchical log storage and extraction
|
| 33 |
+
- Colored console output
|
| 34 |
+
- Log compression functionality
|
| 35 |
+
- Supports artifact tracking
|
| 36 |
+
"""
|
| 37 |
+
ERROR: str = "ERROR"
|
| 38 |
+
DEBUG: str = "DEBUG"
|
| 39 |
+
WARNING: str = "WARNING"
|
| 40 |
+
INFO_LLM: str = "INFO_LLM"
|
| 41 |
+
INFO_USER: str = "INFO_USER"
|
| 42 |
+
LOGGER_LEVELS: Tuple[str, ...] = (ERROR, DEBUG, WARNING, INFO_LLM, INFO_USER)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
# Log storage
|
| 46 |
+
log_error: List[LogEntry] = field(default_factory=list)
|
| 47 |
+
log_warn: List[LogEntry] = field(default_factory=list)
|
| 48 |
+
log_info_llm: List[LogEntry] = field(default_factory=list)
|
| 49 |
+
log_info_user: List[LogEntry] = field(default_factory=list)
|
| 50 |
+
log_debug: List[LogEntry] = field(default_factory=list)
|
| 51 |
+
|
| 52 |
+
# Artifact mapping
|
| 53 |
+
artifact_warnings: Dict[str, List[str]] = field(default_factory=dict)
|
| 54 |
+
artifact_errors: Dict[str, List[str]] = field(default_factory=dict)
|
| 55 |
+
|
| 56 |
+
# Configuration options
|
| 57 |
+
logger_name: Optional[str] = field(default=None)
|
| 58 |
+
auto_console: bool = field(default=True) # Auto output to console
|
| 59 |
+
summary_levels: Optional[List[str]] = field(default=None)
|
| 60 |
+
|
| 61 |
+
# Internal state
|
| 62 |
+
_logger: Optional[logging.Logger] = field(default=None, init=False, repr=False)
|
| 63 |
+
|
| 64 |
+
def __post_init__(self):
|
| 65 |
+
"""Initialize logger - reuses get_logger"""
|
| 66 |
+
if self.logger_name is None:
|
| 67 |
+
self.logger_name = "NodeSummary"
|
| 68 |
+
self._logger = get_logger(self.logger_name)
|
| 69 |
+
if self.summary_levels is None:
|
| 70 |
+
self.summary_levels = [self.ERROR, self.WARNING, self.INFO_LLM, self.INFO_USER]
|
| 71 |
+
|
| 72 |
+
def _log_to_console(self, level: int, message: str, artifact_id: Optional[str] = None):
|
| 73 |
+
"""Output to console (using configured logger)"""
|
| 74 |
+
if not self.auto_console:
|
| 75 |
+
return
|
| 76 |
+
|
| 77 |
+
prefix = f"[ARTIFACT:{artifact_id}] " if artifact_id else ""
|
| 78 |
+
self._logger.log(level, f"{prefix}{message}")
|
| 79 |
+
|
| 80 |
+
def add_error(self, message: str, artifact_id: Optional[str] = None, **kwargs: Any):
|
| 81 |
+
"""Log error messages - for LLM"""
|
| 82 |
+
entry = LogEntry(
|
| 83 |
+
level=self.ERROR,
|
| 84 |
+
message=message,
|
| 85 |
+
timestamp=datetime.now().isoformat(),
|
| 86 |
+
artifact_id=artifact_id,
|
| 87 |
+
extra_data=kwargs
|
| 88 |
+
)
|
| 89 |
+
self.log_error.append(entry)
|
| 90 |
+
|
| 91 |
+
if artifact_id:
|
| 92 |
+
self.artifact_errors.setdefault(artifact_id, []).append(message)
|
| 93 |
+
|
| 94 |
+
self._log_to_console(logging.ERROR, message, artifact_id)
|
| 95 |
+
|
| 96 |
+
def add_warning(self, message: str, artifact_id: Optional[str] = None, **kwargs: Any):
|
| 97 |
+
"""Log warning messages - for LLM"""
|
| 98 |
+
entry = LogEntry(
|
| 99 |
+
level="WARNING",
|
| 100 |
+
message=message,
|
| 101 |
+
timestamp=datetime.now().isoformat(),
|
| 102 |
+
artifact_id=artifact_id,
|
| 103 |
+
extra_data=kwargs
|
| 104 |
+
)
|
| 105 |
+
self.log_warn.append(entry)
|
| 106 |
+
|
| 107 |
+
if artifact_id:
|
| 108 |
+
self.artifact_warnings.setdefault(artifact_id, []).append(message)
|
| 109 |
+
|
| 110 |
+
self._log_to_console(logging.WARNING, message, artifact_id)
|
| 111 |
+
|
| 112 |
+
def info_for_llm(self, message: str, artifact_id: Optional[str] = None, **kwargs: Any):
|
| 113 |
+
"""Log detailed information - for LLM"""
|
| 114 |
+
entry = LogEntry(
|
| 115 |
+
level="INFO_LLM",
|
| 116 |
+
message=message,
|
| 117 |
+
timestamp=datetime.now().isoformat(),
|
| 118 |
+
artifact_id=artifact_id,
|
| 119 |
+
extra_data=kwargs
|
| 120 |
+
)
|
| 121 |
+
self.log_info_llm.append(entry)
|
| 122 |
+
self._log_to_console(logging.INFO, f"[{self.INFO_LLM}] {message}", artifact_id)
|
| 123 |
+
|
| 124 |
+
def info_for_user(self, message: str, artifact_id: Optional[str] = None, **kwargs: Any):
|
| 125 |
+
"""Log general information - for users"""
|
| 126 |
+
entry = LogEntry(
|
| 127 |
+
level="INFO_USER",
|
| 128 |
+
message=message,
|
| 129 |
+
timestamp=datetime.now().isoformat(),
|
| 130 |
+
artifact_id=artifact_id,
|
| 131 |
+
extra_data=kwargs
|
| 132 |
+
)
|
| 133 |
+
self.log_info_user.append(entry)
|
| 134 |
+
self._log_to_console(logging.INFO, f"[{self.INFO_USER}] {message}", artifact_id)
|
| 135 |
+
|
| 136 |
+
def debug_for_dev(self, message: str, artifact_id: Optional[str] = None, **kwargs: Any):
|
| 137 |
+
"""Log debug information - for developers"""
|
| 138 |
+
entry = LogEntry(
|
| 139 |
+
level=self.DEBUG,
|
| 140 |
+
message=message,
|
| 141 |
+
timestamp=datetime.now().isoformat(),
|
| 142 |
+
artifact_id=artifact_id,
|
| 143 |
+
extra_data=kwargs
|
| 144 |
+
)
|
| 145 |
+
self.log_debug.append(entry)
|
| 146 |
+
self._log_to_console(logging.DEBUG, f"[{self.DEBUG}] {message}", artifact_id)
|
| 147 |
+
|
| 148 |
+
def get_logs_by_level(
|
| 149 |
+
self,
|
| 150 |
+
level:str ,
|
| 151 |
+
compress_log: bool=False, # 暂时未实现 TODO
|
| 152 |
+
) -> Dict[str,Any]:
|
| 153 |
+
self.all_logs = {
|
| 154 |
+
self.ERROR: self.log_error,
|
| 155 |
+
self.DEBUG: self.log_debug,
|
| 156 |
+
self.WARNING: self.log_warn,
|
| 157 |
+
self.INFO_LLM: self.log_info_llm,
|
| 158 |
+
self.INFO_USER: self.log_info_user
|
| 159 |
+
}
|
| 160 |
+
|
| 161 |
+
selected_log = self.all_logs[level]
|
| 162 |
+
|
| 163 |
+
return self._extract_log(selected_log)
|
| 164 |
+
|
| 165 |
+
def _extract_log(
|
| 166 |
+
self,
|
| 167 |
+
log_content: List[LogEntry],
|
| 168 |
+
) -> Dict[str,Any]:
|
| 169 |
+
"""
|
| 170 |
+
Extract log content into string format.
|
| 171 |
+
|
| 172 |
+
Args:
|
| 173 |
+
log_content: List of log entries
|
| 174 |
+
|
| 175 |
+
Returns:
|
| 176 |
+
Formatted log string with each log entry on a separate line
|
| 177 |
+
"""
|
| 178 |
+
if not log_content:
|
| 179 |
+
return {}
|
| 180 |
+
|
| 181 |
+
log_lines: List[str] = []
|
| 182 |
+
extra_data_list: List[Dict[str,Any]] = []
|
| 183 |
+
for entry in log_content:
|
| 184 |
+
log_line = f"[{entry.timestamp}] {entry.message}"
|
| 185 |
+
|
| 186 |
+
if entry.artifact_id:
|
| 187 |
+
log_line += f" [artifact_id: {entry.artifact_id}]"
|
| 188 |
+
|
| 189 |
+
log_lines.append(log_line)
|
| 190 |
+
extra_data_list.append(entry.extra_data)
|
| 191 |
+
result: Dict[str,Any] = {
|
| 192 |
+
"log_lines": "\n".join(log_lines),
|
| 193 |
+
"extra_data_list": extra_data_list
|
| 194 |
+
}
|
| 195 |
+
return result
|
| 196 |
+
|
| 197 |
+
def _get_preview_urls(
|
| 198 |
+
self,
|
| 199 |
+
extra_data_list: List[Dict[str,Any]],
|
| 200 |
+
) -> List[str]:
|
| 201 |
+
preview_urls: List[str] = []
|
| 202 |
+
for extra_data in extra_data_list:
|
| 203 |
+
preview_urls.extend([str(url) for url in extra_data.get('preview_urls', [])])
|
| 204 |
+
return preview_urls
|
| 205 |
+
|
| 206 |
+
def get_summary(
|
| 207 |
+
self,
|
| 208 |
+
artifact_id: str,
|
| 209 |
+
compress_log: bool=True,
|
| 210 |
+
**kwargs: Dict[str,Any],
|
| 211 |
+
) -> Dict[str,Any]:
|
| 212 |
+
summary: Dict[str,Any] = {}
|
| 213 |
+
preview_urls: List[str] = []
|
| 214 |
+
if self.summary_levels is None:
|
| 215 |
+
return summary
|
| 216 |
+
|
| 217 |
+
for level in self.summary_levels:
|
| 218 |
+
summary_log = self.get_logs_by_level(level, compress_log)
|
| 219 |
+
log_lines = summary_log.get('log_lines', "")
|
| 220 |
+
extra_data_list = summary_log.get('extra_data_list', [])
|
| 221 |
+
preview_urls.extend(self._get_preview_urls(extra_data_list))
|
| 222 |
+
summary[level] = log_lines
|
| 223 |
+
|
| 224 |
+
summary['preview_urls'] = preview_urls
|
| 225 |
+
summary['artifact_id'] = artifact_id
|
| 226 |
+
return summary
|
| 227 |
+
|
| 228 |
+
def clear(self):
|
| 229 |
+
"""Clear all logs"""
|
| 230 |
+
self.log_error.clear()
|
| 231 |
+
self.log_warn.clear()
|
| 232 |
+
self.log_info_llm.clear()
|
| 233 |
+
self.log_info_user.clear()
|
| 234 |
+
self.log_debug.clear()
|
| 235 |
+
self.artifact_warnings.clear()
|
| 236 |
+
self.artifact_errors.clear()
|
src/open_storyline/skills/skills_io.py
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
|
| 3 |
+
import aiofiles
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from skillkit import SkillManager
|
| 6 |
+
from skillkit.integrations.langchain import create_langchain_tools
|
| 7 |
+
from langchain.agents import create_agent
|
| 8 |
+
from langchain_openai import ChatOpenAI
|
| 9 |
+
from langchain.messages import HumanMessage
|
| 10 |
+
|
| 11 |
+
async def load_skills(
|
| 12 |
+
skill_dir: str=".storyline/skills"
|
| 13 |
+
):
|
| 14 |
+
# Discover skills
|
| 15 |
+
manager = SkillManager(skill_dir=skill_dir)
|
| 16 |
+
await manager.adiscover()
|
| 17 |
+
|
| 18 |
+
# Convert to LangChain tools
|
| 19 |
+
tools = create_langchain_tools(manager)
|
| 20 |
+
return tools
|
| 21 |
+
|
| 22 |
+
async def dump_skills(
|
| 23 |
+
skill_name: str = '',
|
| 24 |
+
skill_dir: str = '',
|
| 25 |
+
skill_content: str = '',
|
| 26 |
+
**kwargs,
|
| 27 |
+
):
|
| 28 |
+
|
| 29 |
+
clean_name = skill_name.strip()
|
| 30 |
+
if not clean_name:
|
| 31 |
+
return {
|
| 32 |
+
"status": "error",
|
| 33 |
+
"message": "skill_name 不能为空"
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
base_path = Path.cwd()
|
| 37 |
+
|
| 38 |
+
# Project_Root + skill_dir + skill_name/
|
| 39 |
+
target_dir_path = base_path / skill_dir / f"cutskill_{clean_name}"
|
| 40 |
+
|
| 41 |
+
# Fix name: SKILL.md
|
| 42 |
+
target_file_path = target_dir_path / "SKILL.md"
|
| 43 |
+
|
| 44 |
+
# Path Traversal Protection
|
| 45 |
+
try:
|
| 46 |
+
final_path = target_file_path.resolve()
|
| 47 |
+
if base_path not in final_path.parents:
|
| 48 |
+
return {
|
| 49 |
+
"status": "error",
|
| 50 |
+
"message": f"Security Alert: 禁止写入项目目录之外的路径: {final_path}"
|
| 51 |
+
}
|
| 52 |
+
except Exception as e:
|
| 53 |
+
return {
|
| 54 |
+
"status": "error",
|
| 55 |
+
"message": f"Path resolution error: {str(e)}"
|
| 56 |
+
}
|
| 57 |
+
|
| 58 |
+
# Start write
|
| 59 |
+
try:
|
| 60 |
+
if not target_dir_path.exists():
|
| 61 |
+
target_dir_path.mkdir(parents=True, exist_ok=True)
|
| 62 |
+
|
| 63 |
+
async with aiofiles.open(final_path, mode='w', encoding='utf-8') as f:
|
| 64 |
+
await f.write(skill_content)
|
| 65 |
+
|
| 66 |
+
return {
|
| 67 |
+
"status": "success",
|
| 68 |
+
"message": f"Skill '{clean_name}' successfully created.",
|
| 69 |
+
"dir_path": str(target_dir_path),
|
| 70 |
+
"file_path": str(final_path),
|
| 71 |
+
"size_bytes": len(skill_content.encode('utf-8'))
|
| 72 |
+
}
|
| 73 |
+
|
| 74 |
+
except PermissionError:
|
| 75 |
+
return {
|
| 76 |
+
"status": "error",
|
| 77 |
+
"message": f"Permission denied: 无法写入目录 {target_dir_path}"
|
| 78 |
+
}
|
| 79 |
+
except Exception as e:
|
| 80 |
+
return {
|
| 81 |
+
"status": "error",
|
| 82 |
+
"message": f"Write operation failed: {str(e)}"
|
| 83 |
+
}
|
src/open_storyline/storage/__init__.py
ADDED
|
File without changes
|