feat: voice clone audio generation + podcast workspace architecture

- Voice clone integration: When user selects voice clone in Write phase,
  backend uses their uploaded voice sample + scene script text to generate
  audio via qwen3/minimax/cosyvoice voice clone APIs
- Multi-tenant workspace storage: All podcast assets (audio, video, images,
  charts) now use workspace-specific directories per user
- Chart preview improvements: Card-based B-Roll charts UI with thumbnails,
  takeaway text, and action buttons; public endpoint for image serving
- Voice clone caching: In-memory LRU cache for voice samples (avoids
  re-downloading per scene); frontend caches voice clone metadata
- Thread pool for voice clone: Audio generation uses ThreadPoolExecutor to
  avoid blocking the FastAPI event loop
- Auto-detect voice clone IDs (vc_*, MY_VOICE_CLONE) to route correctly
- DB fallback for voice sample URL: Fetches from ContentAsset if not passed
- Fixed API URL resolution for chart previews
- Fixed GlassyCard DOM warnings for motion props
- Fixed ScriptGenerationProgressView syntax error
- Fixed usePodcastWorkflow scriptData reference
This commit is contained in:
ajaysi
2026-04-21 19:38:50 +05:30
parent 7637babd7d
commit 91b2f996fd
33 changed files with 1642 additions and 457 deletions

View File

@@ -26,6 +26,9 @@ import { VoiceSelector } from "../../shared/VoiceSelector";
export type AudioGenerationSettings = {
voiceId: string;
customVoiceId?: string;
useVoiceClone?: boolean;
voiceSampleUrl?: string;
voiceCloneEngine?: string;
speed: number;
volume: number;
pitch: number;

View File

@@ -16,7 +16,7 @@ import { GlassyCard, glassyCardSx, PrimaryButton } from "../ui";
import { LineEditor } from "./LineEditor";
import { ImageRegenerateModal, ImageGenerationSettings } from "./ImageRegenerateModal";
import { AudioRegenerateModal, AudioGenerationSettings } from "./AudioRegenerateModal";
import { podcastApi } from "../../../services/podcastApi";
import { podcastApi, getCachedVoiceCloneInfo } from "../../../services/podcastApi";
import { aiApiClient } from "../../../api/client";
import { getCachedMedia, setCachedMedia } from "../../../utils/mediaCache";
@@ -68,6 +68,9 @@ export const SceneEditor: React.FC<SceneEditorProps> = ({
const [audioSettings, setAudioSettings] = useState<AudioGenerationSettings>({
voiceId: knobs.voice_id || "Wise_Woman",
customVoiceId: knobs.custom_voice_id || undefined,
useVoiceClone: knobs.is_voice_clone || false,
voiceSampleUrl: knobs.voice_sample_url || undefined,
voiceCloneEngine: knobs.voice_clone_engine || undefined,
speed: knobs.voice_speed ?? 1.0,
volume: 1.0,
pitch: 0.0,
@@ -308,10 +311,14 @@ export const SceneEditor: React.FC<SceneEditorProps> = ({
// Generate audio
const effectiveSettings = settings || audioSettings;
const cachedClone = getCachedVoiceCloneInfo();
const result = await podcastApi.renderSceneAudio({
scene: currentScene,
voiceId: effectiveSettings.voiceId || knobs.voice_id || "Wise_Woman",
customVoiceId: effectiveSettings.customVoiceId || knobs.custom_voice_id,
customVoiceId: effectiveSettings.customVoiceId || knobs.custom_voice_id || cachedClone?.customVoiceId,
useVoiceClone: effectiveSettings.useVoiceClone || knobs.is_voice_clone || cachedClone?.isVoiceClone || false,
voiceSampleUrl: effectiveSettings.voiceSampleUrl || knobs.voice_sample_url || cachedClone?.voiceSampleUrl || undefined,
voiceCloneEngine: effectiveSettings.voiceCloneEngine || knobs.voice_clone_engine || cachedClone?.engine || undefined,
emotion: effectiveSettings.emotion || scene.emotion || knobs.voice_emotion || "neutral",
speed: effectiveSettings.speed ?? knobs.voice_speed ?? 1.0,
volume: effectiveSettings.volume ?? 1.0,

View File

@@ -7,8 +7,9 @@ import { podcastApi } from "../../../services/podcastApi";
import { GlassyCard, PrimaryButton, SecondaryButton } from "../ui";
import { SceneEditor } from "./SceneEditor";
import { InlineAudioPlayer } from "../InlineAudioPlayer";
import { aiApiClient } from "../../../api/client";
import { aiApiClient, getApiUrl } from "../../../api/client";
import { BrollInfoPanel } from "./parts/BrollInfoPanel";
import { ScriptEditorProvider } from "./ScriptEditorContext";
interface ScriptEditorProps {
projectId: string;
@@ -75,49 +76,8 @@ export const ScriptEditor: React.FC<ScriptEditorProps> = ({
}
}, [initialScript]);
useEffect(() => {
// If script already exists, don't regenerate
if (script) {
return;
}
// Only generate if we have research data
if (!rawResearch) {
return;
}
let mounted = true;
setLoading(true);
setError(null);
podcastApi
.generateScript({
projectId,
idea,
research: rawResearch,
knobs,
speakers,
durationMinutes,
podcastMode,
analysis,
outline,
})
.then((res) => {
if (mounted) {
setScript(res);
emitScriptChange(res);
setError(null);
}
})
.catch((err) => {
const message = err instanceof Error ? err.message : "Failed to generate script";
setError(message);
onError(message);
})
.finally(() => mounted && setLoading(false));
return () => {
mounted = false;
};
}, [projectId, rawResearch, idea, knobs, speakers, durationMinutes, podcastMode, analysis, outline, emitScriptChange, onError, script]);
// Note: Script generation is now handled by ScriptEditorProvider
// to ensure BrollInfoPanel and other child components have access to context
const updateScene = (updated: Scene) => {
// Use functional update to ensure we're working with latest state
@@ -309,14 +269,20 @@ export const ScriptEditor: React.FC<ScriptEditorProps> = ({
chart_type: scene.chart_data.type || "bar_comparison",
title: scene.title,
});
console.log(`[ChartPreview] Scene ${scene.id}: type=${scene.chart_data.type || 'bar_comparison'}, data=`, scene.chart_data);
const toFullUrl = (url: string) => {
if (/^https?:\/\//i.test(url)) return url;
return `${getApiUrl()}${url.startsWith("/") ? url : `/${url}`}`;
};
return {
...scene,
broll_preview_url: result.preview_url,
broll_preview_url: toFullUrl(result.preview_url),
chart_id: result.chart_id,
};
} catch (error) {
console.error(`Failed to generate chart preview for scene ${scene.id}:`, error);
console.error(`[ChartPreview] Failed for scene ${scene.id}:`, error);
return scene;
}
})
@@ -379,11 +345,28 @@ export const ScriptEditor: React.FC<ScriptEditorProps> = ({
}, [script, emitScriptChange]);
return (
<Box sx={{ mt: 4 }}>
<Stack direction="row" spacing={2} alignItems="center" sx={{ mb: 4 }}>
<SecondaryButton onClick={onBackToResearch} startIcon={<ArrowBackIcon />}>
Back to Research
</SecondaryButton>
<ScriptEditorProvider
projectId={projectId}
idea={idea}
rawResearch={rawResearch}
knobs={knobs}
speakers={speakers}
durationMinutes={durationMinutes}
initialScript={script}
podcastMode={podcastMode}
analysis={analysis}
outline={outline}
onScriptChange={(s) => {
setScript(s);
onScriptChange(s);
}}
onError={onError}
>
<Box sx={{ mt: 4 }}>
<Stack direction="row" spacing={2} alignItems="center" sx={{ mb: 4 }}>
<SecondaryButton onClick={onBackToResearch} startIcon={<ArrowBackIcon />}>
Back to Research
</SecondaryButton>
<Box sx={{ flex: 1 }}>
<Typography
variant="h4"
@@ -945,5 +928,6 @@ export const ScriptEditor: React.FC<ScriptEditorProps> = ({
</Stack>
)}
</Box>
</ScriptEditorProvider>
);
};

View File

@@ -320,6 +320,9 @@ export const ScriptEditorProvider: React.FC<ScriptEditorProviderProps> = ({
scenes: sceneData,
voiceId: knobs.voice_id,
customVoiceId: knobs.custom_voice_id,
useVoiceClone: knobs.is_voice_clone,
voiceSampleUrl: knobs.voice_sample_url,
voiceCloneEngine: knobs.voice_clone_engine,
speed: knobs.voice_speed,
emotion: knobs.voice_emotion,
englishNormalization: true,

View File

@@ -1,6 +1,6 @@
import React from "react";
import { Stack, Box, Typography, Alert, Paper, Button, CircularProgress, Chip } from "@mui/material";
import { BarChart as BarChartIcon, AutoAwesome as AutoAwesomeIcon, Refresh as RefreshIcon, DeleteOutline as DeleteIcon } from "@mui/icons-material";
import { Stack, Box, Typography, Paper, Button, CircularProgress, Chip, IconButton, Tooltip } from "@mui/material";
import { BarChart as BarChartIcon, AutoAwesome as AutoAwesomeIcon, Refresh as RefreshIcon, DeleteOutline as DeleteIcon, Fullscreen as FullscreenIcon, Visibility as VisibilityIcon } from "@mui/icons-material";
import { useScriptEditor } from "../ScriptEditorContext";
import { Script } from "../../types";
@@ -42,119 +42,246 @@ export const BrollInfoPanel: React.FC<BrollInfoPanelProps> = (props) => {
return (
<Paper
sx={{
p: 3,
background: "linear-gradient(135deg, rgba(34, 197, 94, 0.05) 0%, rgba(16, 185, 129, 0.05) 100%)",
p: 2.5,
background: "linear-gradient(135deg, rgba(34, 197, 94, 0.03) 0%, rgba(16, 185, 129, 0.03) 100%)",
border: "1px solid rgba(34, 197, 94, 0.15)",
borderRadius: 2,
}}
>
<Stack direction="row" alignItems="center" spacing={1.5} sx={{ mb: 2 }}>
<Box sx={{ width: 40, height: 40, borderRadius: "50%", background: "linear-gradient(135deg, #22c55e 0%, #10b981 100%)", display: "flex", alignItems: "center", justifyContent: "center" }}>
<BarChartIcon sx={{ color: "#ffffff", fontSize: "1.5rem" }} />
</Box>
<Box sx={{ flex: 1 }}>
<Typography variant="h6" sx={{ color: "#0f172a", fontWeight: 600, fontSize: "1.1rem" }}>
B-Roll Charts
</Typography>
<Typography variant="body2" sx={{ color: "#64748b", mt: 0.25 }}>
Programmatic charts extracted from research data
</Typography>
</Box>
<Stack direction="row" justifyContent="space-between" alignItems="center" sx={{ mb: 2 }}>
<Stack direction="row" alignItems="center" spacing={1.5}>
<Box sx={{
p: 0.75,
borderRadius: 1.5,
background: "linear-gradient(135deg, #22c55e 0%, #16a34a 100%)",
display: "flex",
alignItems: "center",
justifyContent: "center"
}}>
<BarChartIcon sx={{ fontSize: 18, color: "#fff" }} />
</Box>
<Box>
<Typography variant="subtitle2" sx={{ fontWeight: 700, color: "#0f172a", lineHeight: 1.2 }}>
B-Roll Charts
</Typography>
<Typography variant="caption" sx={{ color: "#64748b", fontSize: "0.7rem" }}>
{resolvedScenesWithCharts} chart{resolvedScenesWithCharts !== 1 ? 's' : ''} for visual storytelling
</Typography>
</Box>
</Stack>
{hasChartData && (
<Chip
label={`${resolvedScenesWithCharts} scene${resolvedScenesWithCharts > 1 ? 's' : ''} with charts`}
<Button
variant="contained"
size="small"
sx={{ background: "rgba(34, 197, 94, 0.1)", color: "#16a34a", fontWeight: 600 }}
/>
startIcon={resolvedGeneratingChartId ? <CircularProgress size={14} color="inherit" /> : <AutoAwesomeIcon sx={{ fontSize: 16 }} />}
onClick={resolvedGenerateChartPreviews}
disabled={!!resolvedGeneratingChartId}
sx={{
background: "linear-gradient(135deg, #22c55e 0%, #16a34a 100%)",
fontSize: "0.75rem",
py: 0.5,
px: 1.5,
textTransform: "none",
fontWeight: 600,
boxShadow: "0 2px 8px rgba(34, 197, 94, 0.3)",
"&:hover": {
background: "linear-gradient(135deg, #16a34a 0%, #15803d 100%)",
},
"&:disabled": {
background: "rgba(34, 197, 94, 0.5)",
}
}}
>
{resolvedGeneratingChartId ? "Generating..." : "Generate Charts"}
</Button>
)}
</Stack>
{!hasChartData ? (
<Alert severity="info" sx={{ background: "rgba(34, 197, 94, 0.06)", border: "1px solid rgba(34, 197, 94, 0.15)", "& .MuiAlert-icon": { color: "#22c55e" } }}>
<Typography variant="body2" sx={{ color: "#0f172a", lineHeight: 1.7 }}>
<strong style={{ fontWeight: 600 }}>No charts detected.</strong> If your research contains statistics or metrics, the script generation will automatically extract chart data for B-roll visualization.
</Typography>
</Alert>
) : (
<Stack spacing={2}>
<Typography variant="body2" sx={{ color: "#475569", lineHeight: 1.7 }}>
Your script contains <strong style={{ fontWeight: 600 }}>{scenesWithData.length}</strong> scene(s) with chart data.
Click below to generate chart previews for the Write phase.
</Typography>
<Stack direction="row" spacing={2}>
<Button
variant="contained"
startIcon={resolvedGeneratingChartId ? <CircularProgress size={16} color="inherit" /> : <AutoAwesomeIcon />}
onClick={resolvedGenerateChartPreviews}
disabled={!!resolvedGeneratingChartId || !resolvedGenerateChartPreviews}
sx={{
background: "linear-gradient(135deg, #22c55e 0%, #10b981 100%)",
"&:hover": { background: "linear-gradient(135deg, #16a34a 0%, #059669 100%)" },
textTransform: "none",
fontWeight: 600,
}}
>
{resolvedGeneratingChartId ? "Generating..." : "Generate Chart Previews"}
</Button>
</Stack>
{scenesWithData.map((scene) => (
<Box
key={scene.id}
sx={{
p: 2,
background: "rgba(0,0,0,0.02)",
borderRadius: 1,
display: "flex",
alignItems: "center",
justifyContent: "space-between"
}}
>
<Box>
<Typography variant="subtitle2" sx={{ fontWeight: 600 }}>
{scene.title}
</Typography>
<Typography variant="caption" sx={{ color: "#64748b" }}>
{scene.chart_data?.type || "chart"} {scene.chart_data?.labels?.length || 0} data points
</Typography>
</Box>
<Stack direction="row" spacing={1}>
{resolvedGeneratingChartId === scene.id ? (
<CircularProgress size={20} />
) : scene.broll_preview_url ? (
<>
<Chip
label="Preview Ready"
size="small"
sx={{ background: "rgba(34, 197, 94, 0.1)", color: "#16a34a" }}
{hasChartData ? (
<Stack spacing={1.5}>
{scenesWithData.map((scene) => {
const chartData = scene.chart_data;
const hasPreview = !!scene.broll_preview_url;
return (
<Box
key={scene.id}
sx={{
p: 1.5,
background: "#fff",
borderRadius: 1.5,
border: "1px solid rgba(0,0,0,0.06)",
display: "flex",
alignItems: "center",
gap: 2,
transition: "all 0.2s ease",
"&:hover": {
borderColor: "rgba(34, 197, 94, 0.3)",
boxShadow: "0 2px 8px rgba(0,0,0,0.04)",
}
}}
>
{/* Thumbnail */}
<Box
sx={{
width: 72,
height: 48,
flexShrink: 0,
borderRadius: 1,
overflow: "hidden",
background: hasPreview ? "rgba(0,0,0,0.04)" : "linear-gradient(135deg, #f1f5f9 0%, #e2e8f0 100%)",
display: "flex",
alignItems: "center",
justifyContent: "center",
cursor: hasPreview ? "pointer" : "default",
transition: "all 0.2s ease",
"&:hover": hasPreview ? {
transform: "scale(1.05)",
boxShadow: "0 4px 12px rgba(0,0,0,0.15)",
} : {}
}}
>
{resolvedGeneratingChartId === scene.id ? (
<CircularProgress size={24} sx={{ color: "#22c55e" }} />
) : hasPreview && scene.broll_preview_url ? (
<Box
component="img"
src={scene.broll_preview_url}
alt={`Chart for ${scene.title}`}
sx={{
width: "100%",
height: "100%",
objectFit: "cover",
}}
onClick={() => window.open(scene.broll_preview_url, '_blank')}
/>
<Button
size="small"
startIcon={<RefreshIcon />}
) : (
<BarChartIcon sx={{ fontSize: 20, color: "#94a3b8" }} />
)}
</Box>
{/* Chart Info */}
<Box sx={{ flex: 1, minWidth: 0 }}>
<Typography variant="subtitle2" sx={{
fontWeight: 600,
color: "#1e293b",
fontSize: "0.8rem",
overflow: "hidden",
textOverflow: "ellipsis",
whiteSpace: "nowrap",
}}>
{scene.title}
</Typography>
<Stack direction="row" spacing={1} alignItems="center" sx={{ mt: 0.25 }}>
<Chip
label={chartData?.type || "chart"}
size="small"
sx={{
height: 18,
fontSize: "0.65rem",
background: "rgba(34, 197, 94, 0.1)",
color: "#16a34a",
fontWeight: 600,
}}
/>
<Typography variant="caption" sx={{ color: "#64748b", fontSize: "0.7rem" }}>
{chartData?.labels?.length || 0} labels
</Typography>
{hasPreview && (
<Chip
label="Ready"
size="small"
sx={{
height: 18,
fontSize: "0.65rem",
background: "rgba(34, 197, 94, 0.15)",
color: "#16a34a",
fontWeight: 600,
}}
/>
)}
</Stack>
</Box>
{/* Takeaway */}
{chartData?.takeaway && (
<Box sx={{
flex: 1.5,
display: { xs: "none", md: "block" },
px: 1,
py: 0.5,
background: "rgba(34, 197, 94, 0.04)",
borderRadius: 1,
}}>
<Typography variant="caption" sx={{
color: "#475569",
fontSize: "0.7rem",
fontStyle: "italic",
display: "-webkit-box",
WebkitLineClamp: 2,
WebkitBoxOrient: "vertical",
overflow: "hidden",
}}>
"{chartData.takeaway}"
</Typography>
</Box>
)}
{/* Actions */}
<Stack direction="row" spacing={0.5}>
{hasPreview && (
<Tooltip title="View fullsize">
<IconButton
size="small"
onClick={() => scene.broll_preview_url && window.open(scene.broll_preview_url, '_blank')}
sx={{
color: "#64748b",
"&:hover": { color: "#22c55e", background: "rgba(34, 197, 94, 0.1)" }
}}
>
<FullscreenIcon sx={{ fontSize: 18 }} />
</IconButton>
</Tooltip>
)}
<Tooltip title="Regenerate">
<IconButton
size="small"
onClick={() => resolvedRegenerateChart?.(scene.id)}
disabled={!resolvedRegenerateChart}
disabled={!resolvedRegenerateChart || !!resolvedGeneratingChartId}
sx={{
color: "#64748b",
"&:hover": { color: "#f59e0b", background: "rgba(245, 158, 11, 0.1)" }
}}
>
Regenerate
</Button>
<Button
size="small"
startIcon={<DeleteIcon />}
<RefreshIcon sx={{ fontSize: 18 }} />
</IconButton>
</Tooltip>
<Tooltip title="Remove chart">
<IconButton
size="small"
onClick={() => resolvedRemoveChart?.(scene.id)}
disabled={!resolvedRemoveChart}
sx={{ color: "#ef4444" }}
sx={{
color: "#64748b",
"&:hover": { color: "#ef4444", background: "rgba(239, 68, 68, 0.1)" }
}}
>
Remove
</Button>
</>
) : null}
</Stack>
</Box>
))}
<DeleteIcon sx={{ fontSize: 18 }} />
</IconButton>
</Tooltip>
</Stack>
</Box>
);
})}
</Stack>
) : (
<Box sx={{ py: 3, textAlign: "center" }}>
<BarChartIcon sx={{ fontSize: 36, color: "#cbd5e1", mb: 1 }} />
<Typography variant="body2" sx={{ color: "#64748b", fontSize: "0.8rem" }}>
No chart data yet. Add chart data to scenes to generate B-roll visuals.
</Typography>
</Box>
)}
</Paper>
);
};
};