import os
import json
import subprocess
import sys
import time
from collections import Counter
from datetime import datetime, timezone, timedelta
from hashlib import sha256
from html import escape
from ipaddress import ip_address
from pathlib import Path
from threading import Lock
from urllib.parse import urlparse
from urllib.request import urlopen
import gradio as gr
import torch
from huggingface_hub import hf_hub_download
PROJECT_ROOT = Path(__file__).resolve().parent
SPACE_ROOT = PROJECT_ROOT
while PROJECT_ROOT != PROJECT_ROOT.parent:
if (PROJECT_ROOT / "joycent").is_dir():
break
PROJECT_ROOT = PROJECT_ROOT.parent
else:
raise RuntimeError("Could not find the Joycent source tree.")
sys.path.insert(0, str(PROJECT_ROOT))
def build_monotonic_align():
module_dir = PROJECT_ROOT / "joycent" / "model" / "monotonic_align"
if list(module_dir.glob("core*.so")):
return
subprocess.run(
[sys.executable, "setup.py", "build_ext", "--inplace"],
cwd=module_dir,
check=True,
)
build_monotonic_align()
from joycent.inference_cosyvoice import ( # noqa: E402
DEFAULT_BASE_REPO_ID,
DEFAULT_FINETUNED_FILENAME,
DEFAULT_INSTRUCT,
load_cosyvoice_model,
synthesize_cosyvoice,
)
from joycent.inference_joycent import ( # noqa: E402
extract_speaker_embedding,
load_acoustic_model,
load_facodec,
load_vocoder,
synthesize_audio,
)
from transformers import AutoModel # noqa: E402
from whisAID import WhisAIDConfig # noqa: E402
from whisper import load_audio, log_mel_spectrogram, pad_or_trim # noqa: E402
DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
JOYCENT_MODEL_ID = os.getenv("JOYCENT_MODEL_ID", "walston/joycent")
JOYCENT_MODEL_FILENAME = os.getenv("JOYCENT_MODEL_FILENAME", "grad_210.pt")
VOCODER_REPO_ID = os.getenv("JOYCENT_VOCODER_REPO_ID", "").strip()
if not VOCODER_REPO_ID or VOCODER_REPO_ID == "walston/joycent":
VOCODER_REPO_ID = "walston/joycent-vocoder"
VOCODER_FILENAME = os.getenv(
"JOYCENT_VOCODER_FILENAME",
"checkpoint-50000steps.pkl",
)
VOCODER_CONFIG_FILENAME = os.getenv(
"JOYCENT_VOCODER_CONFIG_FILENAME",
"config.yml",
)
WHISAID_MODEL_ID = os.getenv("WHISAID_MODEL_ID", "walston/whisaid-zh-grl")
COSYVOICE_BASE_REPO_ID = os.getenv(
"COSYVOICE_BASE_REPO_ID",
DEFAULT_BASE_REPO_ID,
)
COSYVOICE_MODEL_ID = os.getenv(
"COSYVOICE_MODEL_ID",
"walston/cosyvoice3-sg",
)
COSYVOICE_MODEL_FILENAME = os.getenv(
"COSYVOICE_MODEL_FILENAME",
DEFAULT_FINETUNED_FILENAME,
)
DEFAULT_PHONEMES = "sil sh ix4 zh en1 d e5 m ei2 ii iu3 sil"
DEFAULT_COSYVOICE_TEXT = "但是争取好成绩的前提是身体好"
DEFAULT_SPEAKER_REFERENCE = (
SPACE_ROOT / "assets" / "speaker_reference.wav"
)
DEFAULT_ACCENT_REFERENCE = (
SPACE_ROOT / "assets" / "accent_reference.wav"
)
JOYCENT_RUNTIME = None
COSYVOICE_RUNTIME = None
ANALYTICS = None
DISPLAY_TIMEZONE = timezone(timedelta(hours=8), "UTC+8")
DISPLAY_TIMEZONE_LABEL = "Beijing / Singapore Time (UTC+8)"
class DemoAnalytics:
def __init__(self):
self.lock = Lock()
self.users = set()
self.generations = 0
self.audio_seconds = 0.0
self.generation_models = Counter()
self.countries = Counter()
self.devices = Counter()
self.referrers = Counter()
self.country_cache = {}
self.sessions = {}
@staticmethod
def today():
return datetime.now(DISPLAY_TIMEZONE).date().isoformat()
@staticmethod
def get_header(request, name, default=""):
headers = getattr(request, "headers", {}) or {}
getter = getattr(headers, "get", None)
if getter is None:
return default
return getter(name, default) or default
def session_id(self, request):
session_hash = getattr(request, "session_hash", None)
if session_hash:
return f"session:{session_hash}"
user_agent = self.get_header(request, "user-agent")
forwarded_for = self.get_header(request, "x-forwarded-for")
client = getattr(request, "client", "") or ""
raw = f"{forwarded_for}|{client}|{user_agent}"
return "anon:" + sha256(raw.encode("utf-8")).hexdigest()[:24]
@staticmethod
def normalize_ip(candidate):
candidate = (candidate or "").strip()
if not candidate:
return ""
candidate = candidate.split(",")[0].strip()
if candidate.startswith("[") and "]" in candidate:
candidate = candidate[1:candidate.index("]")]
elif candidate.count(":") == 1 and "." in candidate:
candidate = candidate.rsplit(":", 1)[0]
try:
parsed = ip_address(candidate)
except ValueError:
return ""
if parsed.is_private or parsed.is_loopback or parsed.is_link_local:
return ""
return str(parsed)
def client_ip(self, request):
for header in (
"x-forwarded-for",
"cf-connecting-ip",
"x-real-ip",
"fastly-client-ip",
"x-client-ip",
):
ip_value = self.normalize_ip(self.get_header(request, header))
if ip_value:
return ip_value
client = getattr(request, "client", None)
host = getattr(client, "host", "") or (client[0] if isinstance(client, tuple) else "")
return self.normalize_ip(str(host))
def country_from_ip(self, ip_value):
if not ip_value:
return "Unknown"
if ip_value in self.country_cache:
return self.country_cache[ip_value]
try:
with urlopen(f"https://ipapi.co/{ip_value}/json/", timeout=1.5) as response:
payload = json.loads(response.read().decode("utf-8"))
country = payload.get("country_name") or payload.get("country") or "Unknown"
except Exception:
country = "Unknown"
self.country_cache[ip_value] = country
return country
def country(self, request):
return self.country_from_ip(self.client_ip(request))
def device(self, request):
user_agent = self.get_header(request, "user-agent").lower()
if "ipad" in user_agent or "tablet" in user_agent:
return "Tablet"
if "mobi" in user_agent or "android" in user_agent or "iphone" in user_agent:
return "Mobile"
if user_agent:
return "Desktop"
return "Unknown"
def referrer(self, request):
referer = self.get_header(request, "referer").strip()
if not referer:
return "Direct / Unknown"
parsed = urlparse(referer)
return parsed.netloc or referer[:48]
def touch(self, request):
session_id = self.session_id(request)
now = time.time()
country = self.country(request)
device = self.device(request)
referrer = self.referrer(request)
with self.lock:
is_new_session = session_id not in self.sessions
self.sessions.setdefault(
session_id,
{
"first_seen": now,
"last_seen": now,
"country": country,
"device": device,
"referrer": referrer,
},
)
self.sessions[session_id]["last_seen"] = now
self.users.add(session_id)
if is_new_session:
self.countries[country] += 1
self.devices[device] += 1
self.referrers[referrer] += 1
def record_generation(self, request, model_name, audio_seconds):
self.touch(request)
with self.lock:
self.generations += 1
self.audio_seconds += max(float(audio_seconds), 0.0)
self.generation_models[model_name] += 1
@staticmethod
def format_seconds(seconds):
seconds = max(float(seconds), 0.0)
minutes, remainder = divmod(int(seconds), 60)
if minutes:
return f"{minutes}m {remainder:02d}s"
return f"{seconds:.0f}s"
def top_items(self, counter, empty_label="No data yet"):
if not counter:
return f"
{empty_label}"
rows = []
for label, count in counter.most_common(4):
rows.append(f"{escape(str(label))}{count}")
return "".join(rows)
def render(self):
with self.lock:
users = len(self.users)
generations = self.generations
audio_seconds = self.audio_seconds
avg_audio = audio_seconds / generations if generations else 0.0
model_counts = self.generation_models.copy()
countries = self.countries.copy()
devices = self.devices.copy()
referrers = self.referrers.copy()
active_sessions = [
session
for session in self.sessions.values()
if time.time() - session["last_seen"] <= 120
]
avg_session_seconds = (
sum(session["last_seen"] - session["first_seen"] for session in self.sessions.values())
/ len(self.sessions)
if self.sessions
else 0.0
)
return f"""
Live Demo Analytics
All-time totals from this Space runtime. Clock: {DISPLAY_TIMEZONE_LABEL}.
{datetime.now(DISPLAY_TIMEZONE).strftime("%H:%M:%S UTC+8")}
All-Time Active Users{users}
All-Time Generations{generations}
Average Audio Length{avg_audio:.1f}s
Active Now{len(active_sessions)}
Average Session Time{self.format_seconds(avg_session_seconds)}
Countries / Regions
{self.top_items(countries)}
Devices
{self.top_items(devices)}
Referring Sites
{self.top_items(referrers)}
Generation Use
{self.top_items(model_counts)}
"""
ANALYTICS = DemoAnalytics()
def refresh_analytics(request: gr.Request):
ANALYTICS.touch(request)
return ANALYTICS.render()
def load_joycent_runtime():
acoustic_path = hf_hub_download(
repo_id=JOYCENT_MODEL_ID,
filename=JOYCENT_MODEL_FILENAME,
)
vocoder_path = hf_hub_download(
repo_id=VOCODER_REPO_ID,
filename=VOCODER_FILENAME,
)
vocoder_config = hf_hub_download(
repo_id=VOCODER_REPO_ID,
filename=VOCODER_CONFIG_FILENAME,
)
model, zh_dict = load_acoustic_model(acoustic_path, DEVICE)
vocoder, config = load_vocoder(
vocoder_path,
"outputs",
DEVICE,
config_path=vocoder_config,
)
fa_encoder, fa_decoder = load_facodec(DEVICE)
whisaid = AutoModel.from_config(
WhisAIDConfig(checkpoint_repo_id=WHISAID_MODEL_ID)
)
whisaid = whisaid.to(DEVICE).eval()
return model, zh_dict, vocoder, config, fa_encoder, fa_decoder, whisaid
def get_joycent_runtime():
global JOYCENT_RUNTIME
if JOYCENT_RUNTIME is not None:
return JOYCENT_RUNTIME, 0.0, True
load_start = time.perf_counter()
JOYCENT_RUNTIME = load_joycent_runtime()
return JOYCENT_RUNTIME, time.perf_counter() - load_start, False
def get_cosyvoice_runtime():
global COSYVOICE_RUNTIME
if COSYVOICE_RUNTIME is not None:
return COSYVOICE_RUNTIME, 0.0, True
load_start = time.perf_counter()
COSYVOICE_RUNTIME = load_cosyvoice_model(
base_repo_id=COSYVOICE_BASE_REPO_ID,
finetuned_repo_id=COSYVOICE_MODEL_ID,
finetuned_filename=COSYVOICE_MODEL_FILENAME,
cosyvoice_root=str(PROJECT_ROOT),
fp16=True,
)
return COSYVOICE_RUNTIME, time.perf_counter() - load_start, False
def extract_accent_embedding(audio_path, model):
audio = torch.from_numpy(load_audio(audio_path))
mel = log_mel_spectrogram(
pad_or_trim(audio),
n_mels=model.config.n_mels,
).unsqueeze(0).to(DEVICE)
with torch.no_grad():
return model(input_ids=mel).features
def synthesize_joycent(
speaker_audio,
accent_audio,
phonemes,
n_timesteps,
temperature,
length_scale,
request: gr.Request,
):
if not speaker_audio:
raise gr.Error("Please upload or record a speaker reference.")
if not accent_audio:
raise gr.Error("Please upload or record an accent reference.")
if not phonemes or not phonemes.strip():
raise gr.Error("Please enter a Mandarin phoneme sequence.")
runtime, load_time, model_cached = get_joycent_runtime()
model, zh_dict, vocoder, config, fa_encoder, fa_decoder, whisaid = runtime
if DEVICE.type == "cuda":
torch.cuda.synchronize()
feature_start = time.perf_counter()
accent_embedding = extract_accent_embedding(accent_audio, whisaid)
speaker_embedding = extract_speaker_embedding(
speaker_audio,
fa_encoder,
fa_decoder,
DEVICE,
)
if DEVICE.type == "cuda":
torch.cuda.synchronize()
feature_time = time.perf_counter() - feature_start
if DEVICE.type == "cuda":
torch.cuda.synchronize()
inference_start = time.perf_counter()
sample_rate, waveform = synthesize_audio(
phonemes.strip(),
speaker_audio,
accent_embedding,
model,
zh_dict,
fa_encoder,
fa_decoder,
vocoder,
config,
DEVICE,
n_timesteps=int(n_timesteps),
temperature=float(temperature),
length_scale=float(length_scale),
speaker_embedding=speaker_embedding,
)
if DEVICE.type == "cuda":
torch.cuda.synchronize()
inference_time = time.perf_counter() - inference_start
audio_duration = len(waveform) / sample_rate
rtf = (
(feature_time + inference_time) / audio_duration
if audio_duration > 0
else float("inf")
)
load_status = "cached" if model_cached else "loaded"
info = (
f"Device: {DEVICE}\n"
f"Acoustic model: {JOYCENT_MODEL_ID}/{JOYCENT_MODEL_FILENAME}\n"
f"Vocoder: {VOCODER_REPO_ID}/{VOCODER_FILENAME}\n"
f"Model load time: {load_time:.2f}s ({load_status})\n"
f"Feature extraction time: {feature_time:.2f}s\n"
f"Inference time: {inference_time:.2f}s\n"
f"Audio duration: {audio_duration:.2f}s\n"
f"RTF excluding model load: {rtf:.4f}"
)
ANALYTICS.record_generation(request, "Joycent", audio_duration)
return (sample_rate, waveform), info, ANALYTICS.render()
def synthesize_cosyvoice_ui(
cosyvoice_prompt_audio,
cosyvoice_text,
prompt_text,
instruct,
request: gr.Request,
):
try:
if not cosyvoice_prompt_audio:
raise gr.Error("Please upload or record a CosyVoice prompt.")
(model, model_dir), load_time, model_cached = get_cosyvoice_runtime()
if DEVICE.type == "cuda":
torch.cuda.synchronize()
start = time.perf_counter()
sample_rate, waveform = synthesize_cosyvoice(
model,
cosyvoice_text,
cosyvoice_prompt_audio,
prompt_text=prompt_text,
instruct=instruct,
)
if DEVICE.type == "cuda":
torch.cuda.synchronize()
inference_time = time.perf_counter() - start
audio_duration = waveform.shape[-1] / sample_rate
rtf = (
inference_time / audio_duration
if audio_duration > 0
else float("inf")
)
except gr.Error:
raise
except Exception as error:
raise gr.Error(f"Model loading or inference failed: {error}") from error
info = (
f"Device: {DEVICE}\n"
f"Base model: {COSYVOICE_BASE_REPO_ID}\n"
f"SG-only model: {COSYVOICE_MODEL_ID}/{COSYVOICE_MODEL_FILENAME}\n"
f"Prepared model: {model_dir}\n"
f"Model load time: {load_time:.2f}s "
f"({'cached' if model_cached else 'loaded'})\n"
"Feature extraction time: included in CosyVoice inference\n"
f"Inference time: {inference_time:.2f}s\n"
f"Audio duration: {audio_duration:.2f}s\n"
f"RTF excluding model load: {rtf:.4f}"
)
ANALYTICS.record_generation(request, "CosyVoice3 SG-only", audio_duration)
return (sample_rate, waveform.squeeze(0).numpy()), info, ANALYTICS.render()
CSS = """
.joycent-panel {
background: linear-gradient(180deg, #eef6ff 0%, #dcecff 100%);
border: 1px solid #8bbcff;
border-radius: 16px;
padding: 18px;
}
.cosyvoice-panel {
background: linear-gradient(180deg, #fff5ea 0%, #ffe3c2 100%);
border: 1px solid #f3aa62;
border-radius: 16px;
padding: 18px;
}
.joycent-panel h2 {
color: #195ca8;
}
.cosyvoice-panel h2 {
color: #a84d16;
}
.startup-notice {
background: #fff8d8;
border: 1px solid #e1b94f;
border-radius: 10px;
color: #6f5100;
padding: 10px 14px;
}
.analytics-panel {
background: #f8fafc;
border: 1px solid #cfd8e3;
border-radius: 8px;
margin-top: 18px;
padding: 16px;
}
.analytics-title {
align-items: center;
display: flex;
gap: 16px;
justify-content: space-between;
margin-bottom: 14px;
}
.analytics-title h2 {
font-size: 1.25rem;
margin: 0;
}
.analytics-title p {
color: #536070;
margin: 4px 0 0;
}
.analytics-title span {
color: #536070;
font-variant-numeric: tabular-nums;
white-space: nowrap;
}
.analytics-grid {
display: grid;
gap: 10px;
grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));
}
.analytics-grid div {
background: #ffffff;
border: 1px solid #d9e2ec;
border-radius: 8px;
min-height: 86px;
padding: 12px;
}
.analytics-grid span {
color: #536070;
display: block;
font-size: 0.86rem;
}
.analytics-grid strong {
color: #172033;
display: block;
font-size: 1.55rem;
line-height: 1.2;
margin-top: 8px;
}
.analytics-lists {
display: grid;
gap: 10px;
grid-template-columns: repeat(auto-fit, minmax(190px, 1fr));
margin-top: 10px;
}
.analytics-lists div {
background: #ffffff;
border: 1px solid #d9e2ec;
border-radius: 8px;
padding: 12px;
}
.analytics-lists h3 {
font-size: 0.95rem;
margin: 0 0 8px;
}
.analytics-lists ul {
list-style: none;
margin: 0;
padding: 0;
}
.analytics-lists li {
align-items: center;
color: #536070;
display: flex;
font-size: 0.9rem;
gap: 10px;
justify-content: space-between;
line-height: 1.6;
}
.analytics-lists b {
color: #172033;
font-variant-numeric: tabular-nums;
}
"""
with gr.Blocks(
title="Singapore Mandarin Accent TTS",
theme=gr.themes.Soft(),
css=CSS,
) as demo:
gr.Markdown(
"# Singapore Mandarin Accent TTS\n"
"Joycent and the SG-only fine-tuned CosyVoice3 model are available side by side."
)
gr.Markdown(
"**First run notice:** The first request for each model downloads and "
"loads its checkpoints, so generation may take several minutes. "
"Later requests reuse the cached model and are much faster.",
elem_classes=["startup-notice"],
)
with gr.Row(equal_height=False):
with gr.Column(elem_classes=["joycent-panel"]):
gr.Markdown("## Joycent")
joycent_speaker_input = gr.Audio(
sources=["upload", "microphone"],
type="filepath",
label="Speaker reference",
value=str(DEFAULT_SPEAKER_REFERENCE),
)
accent_input = gr.Audio(
sources=["upload", "microphone"],
type="filepath",
label="Accent reference",
value=str(DEFAULT_ACCENT_REFERENCE),
)
phoneme_input = gr.Textbox(
label="Mandarin phonemes",
value=DEFAULT_PHONEMES,
lines=3,
)
with gr.Accordion("Generation settings", open=False):
steps_input = gr.Slider(
1,
50,
value=10,
step=1,
label="Diffusion steps",
)
temperature_input = gr.Slider(
0.1,
2.0,
value=1.5,
step=0.05,
label="Temperature",
)
length_input = gr.Slider(
0.5,
1.5,
value=0.91,
step=0.01,
label="Length scale",
)
joycent_button = gr.Button(
"Generate with Joycent",
variant="primary",
)
joycent_audio_output = gr.Audio(label="Joycent output")
joycent_info_output = gr.Textbox(
label="Joycent runtime info",
lines=8,
)
with gr.Column(elem_classes=["cosyvoice-panel"]):
gr.Markdown("## CosyVoice3 SG-only")
cosyvoice_prompt_input = gr.Audio(
sources=["upload", "microphone"],
type="filepath",
label="Prompt audio",
value=str(DEFAULT_SPEAKER_REFERENCE),
)
cosyvoice_text_input = gr.Textbox(
label="Synthesis text",
value=DEFAULT_COSYVOICE_TEXT,
lines=3,
)
prompt_text_input = gr.Textbox(
label="Prompt transcript (optional)",
value="",
lines=2,
)
instruct_input = gr.Textbox(
label="Instruction",
value=DEFAULT_INSTRUCT,
lines=2,
)
cosyvoice_button = gr.Button(
"Generate with CosyVoice3",
variant="primary",
)
cosyvoice_audio_output = gr.Audio(label="CosyVoice3 output")
cosyvoice_info_output = gr.Textbox(
label="CosyVoice3 runtime info",
lines=8,
)
analytics_dashboard = gr.HTML(value=ANALYTICS.render())
analytics_timer = gr.Timer(value=10)
demo.load(fn=refresh_analytics, outputs=analytics_dashboard)
analytics_timer.tick(fn=refresh_analytics, outputs=analytics_dashboard)
joycent_button.click(
fn=synthesize_joycent,
inputs=[
joycent_speaker_input,
accent_input,
phoneme_input,
steps_input,
temperature_input,
length_input,
],
outputs=[joycent_audio_output, joycent_info_output, analytics_dashboard],
)
cosyvoice_button.click(
fn=synthesize_cosyvoice_ui,
inputs=[
cosyvoice_prompt_input,
cosyvoice_text_input,
prompt_text_input,
instruct_input,
],
outputs=[cosyvoice_audio_output, cosyvoice_info_output, analytics_dashboard],
)
if __name__ == "__main__":
demo.queue(default_concurrency_limit=1).launch(
show_api=False,
ssr_mode=False,
)