AI podcast project

This commit is contained in:
ajaysi
2025-12-16 16:25:52 +05:30
parent eba5210577
commit 1d745c9bc8
50 changed files with 7637 additions and 2813 deletions

View File

@@ -92,6 +92,9 @@ export const SceneActionButtons: React.FC<SceneActionButtonsProps> = ({
}
// Has audio - show all action buttons
const videoInProgress = rendering !== null;
const isCurrentVideo = rendering === scene.id;
return (
<Stack direction="row" spacing={1.5} justifyContent="flex-end" flexWrap="wrap" useFlexGap>
{/* Generate Image */}
@@ -114,21 +117,29 @@ export const SceneActionButtons: React.FC<SceneActionButtonsProps> = ({
{/* Generate Video */}
<PrimaryButton
onClick={() => onVideoRender(scene.id)}
disabled={isBusy || !hasImage || hasVideo}
onClick={() => {
onVideoRender(scene.id);
}}
disabled={isBusy || videoInProgress || !hasImage || hasVideo}
startIcon={<VideocamIcon />}
tooltip={
hasVideo
? "Video already generated"
: !hasImage
? "Generate an image first to create video"
: videoInProgress
? "A video generation is already running. Please wait..."
: isBusy
? "Another operation in progress"
: "Generate video for this scene"
}
sx={{ minWidth: 160 }}
sx={{ minWidth: 180 }}
>
{hasVideo ? "Video Ready" : "Generate Video"}
{videoInProgress && isCurrentVideo
? "Generating Video..."
: hasVideo
? "Video Ready"
: "Generate Video"}
</PrimaryButton>
{/* Download Video */}

View File

@@ -7,11 +7,13 @@ import {
OpenInNew as OpenInNewIcon,
Videocam as VideocamIcon,
} from "@mui/icons-material";
import { Scene, Job } from "../types";
import { Scene, Job, VideoGenerationSettings } from "../types";
import { GlassyCard, glassyCardSx } from "../ui";
import { InlineAudioPlayer } from "../InlineAudioPlayer";
import { SceneActionButtons } from "./SceneActionButtons";
import { aiApiClient } from "../../../api/client";
import { fetchMediaBlobUrl } from "../../../utils/fetchMediaBlobUrl";
import { VideoRegenerateModal } from "./VideoRegenerateModal";
interface SceneCardProps {
scene: Scene;
@@ -22,7 +24,7 @@ interface SceneCardProps {
avatarImageUrl?: string | null;
onRender: (sceneId: string, mode: "preview" | "full") => void;
onImageGenerate: (sceneId: string) => void;
onVideoRender: (sceneId: string) => void;
onVideoGenerate: (sceneId: string, settings: VideoGenerationSettings) => void;
onDownloadAudio: (audioUrl: string, title: string) => void;
onDownloadVideo: (videoUrl: string, title: string) => void;
onShare: (audioUrl: string, title: string) => void;
@@ -75,7 +77,7 @@ export const SceneCard: React.FC<SceneCardProps> = ({
avatarImageUrl,
onRender,
onImageGenerate,
onVideoRender,
onVideoGenerate,
onDownloadAudio,
onDownloadVideo,
onShare,
@@ -89,8 +91,27 @@ export const SceneCard: React.FC<SceneCardProps> = ({
const status = job?.status || (hasAudio ? "completed" : "idle");
const initials = getInitials(scene.title);
// Load image as blob if it's an authenticated endpoint
const [imageBlobUrl, setImageBlobUrl] = useState<string | null>(null);
const [videoBlobUrl, setVideoBlobUrl] = useState<string | null>(null);
const [showVideoModal, setShowVideoModal] = useState(false);
const [initialVideoPrompt, setInitialVideoPrompt] = useState<string>("");
// Prepare a simple default prompt based on the scene title/description
useEffect(() => {
const baseTitle = (scene.title || "").trim();
const description = (scene as any).description as string | undefined;
const descSnippet = (description || "").split(".")[0]?.trim();
let prompt = baseTitle;
if (!prompt && descSnippet) {
prompt = descSnippet;
}
if (!prompt) {
prompt = "Professional podcast scene with subtle movement";
}
setInitialVideoPrompt(prompt);
}, [scene]);
useEffect(() => {
if (!imageUrl) {
@@ -98,14 +119,11 @@ export const SceneCard: React.FC<SceneCardProps> = ({
return;
}
console.log('[SceneCard] Loading image:', { imageUrl, hasImage, sceneId: scene.id });
// Check if this is a podcast image endpoint that requires authentication
const isPodcastImage = imageUrl.includes('/api/podcast/images/') || imageUrl.includes('/api/story/images/');
if (!isPodcastImage) {
// Regular URL (external), use directly
console.log('[SceneCard] Using external image URL directly');
setImageBlobUrl(imageUrl);
return;
}
@@ -134,22 +152,17 @@ export const SceneCard: React.FC<SceneCardProps> = ({
// Remove query parameters if present
imagePath = imagePath.split('?')[0];
console.log('[SceneCard] Fetching image blob from:', imagePath);
const response = await aiApiClient.get(imagePath, {
responseType: 'blob',
});
if (!isMounted || imageUrl !== currentImageUrl) {
console.log('[SceneCard] Component unmounted or URL changed, skipping blob URL set');
return;
}
const blob = response.data;
const newBlobUrl = URL.createObjectURL(blob);
console.log('[SceneCard] Image blob loaded successfully, created blob URL');
setImageBlobUrl((prevBlobUrl) => {
// Clean up previous blob URL if exists
if (prevBlobUrl && prevBlobUrl !== newBlobUrl && prevBlobUrl.startsWith('blob:')) {
@@ -184,11 +197,9 @@ export const SceneCard: React.FC<SceneCardProps> = ({
const token = localStorage.getItem('clerk_dashboard_token') || '';
if (token) {
const urlWithToken = `${fallbackPath}?token=${encodeURIComponent(token)}`;
console.log('[SceneCard] Trying URL with query token');
setImageBlobUrl(urlWithToken);
} else {
// Fallback to original URL
console.log('[SceneCard] No token available, using original URL');
setImageBlobUrl(imageUrl);
}
} catch (fallbackErr) {
@@ -213,6 +224,39 @@ export const SceneCard: React.FC<SceneCardProps> = ({
};
}, [imageUrl, hasImage, scene.id]);
// Load video as blob when videoUrl changes (using Story Writer's utility)
useEffect(() => {
if (!job?.videoUrl) {
setVideoBlobUrl(null);
return;
}
let currentBlobUrl: string | null = null;
fetchMediaBlobUrl(job.videoUrl)
.then((blobUrl) => {
if (blobUrl) {
currentBlobUrl = blobUrl;
setVideoBlobUrl(blobUrl);
} else {
// File not found (404) - clear the blob URL
console.warn('[SceneCard] Video file not found (404):', job.videoUrl);
setVideoBlobUrl(null);
}
})
.catch((err) => {
console.error('[SceneCard] Failed to load video blob:', err);
setVideoBlobUrl(null);
});
return () => {
// Cleanup blob URL when component unmounts or URL changes
if (currentBlobUrl) {
URL.revokeObjectURL(currentBlobUrl);
}
};
}, [job?.videoUrl]);
return (
<GlassyCard sx={glassyCardSx}>
<Stack spacing={2}>
@@ -279,13 +323,12 @@ export const SceneCard: React.FC<SceneCardProps> = ({
</Box>
</Box>
)}
{hasVideo && job?.videoUrl && (
{hasVideo && videoBlobUrl && (
<Box sx={{ mt: 1 }}>
<Box
component="a"
href={job.videoUrl}
target="_blank"
rel="noopener noreferrer"
href={videoBlobUrl}
download={`${scene.title.replace(/[^a-z0-9]/gi, '_')}_video.mp4`}
sx={{ color: "#a78bfa", textDecoration: "none", display: "inline-flex", alignItems: "center", gap: 0.5 }}
>
<VideocamIcon sx={{ fontSize: 16 }} />
@@ -350,8 +393,57 @@ export const SceneCard: React.FC<SceneCardProps> = ({
<InlineAudioPlayer audioUrl={audioUrl} title={scene.title} />
)}
{/* Image Preview */}
{hasImage && (imageBlobUrl || imageUrl) && (
{/* Video Preview - Show video if available, otherwise show image */}
{hasVideo && videoBlobUrl ? (
<Box
sx={{
width: "100%",
borderRadius: 2,
overflow: "hidden",
border: "2px solid rgba(56,189,248,0.5)",
background: alpha("#0f172a", 0.85),
position: "relative",
}}
>
<Box
component="video"
src={videoBlobUrl}
controls
preload="metadata"
sx={{
width: "100%",
height: "auto",
display: "block",
maxHeight: 420,
objectFit: "cover",
backgroundColor: "black",
}}
onError={(e) => {
const videoElement = e.currentTarget as HTMLVideoElement;
console.error("[SceneCard] Video failed to load:", {
originalUrl: job?.videoUrl,
networkState: videoElement.networkState,
});
}}
/>
<Box
sx={{
position: "absolute",
top: 8,
right: 8,
bgcolor: "rgba(56,189,248,0.9)",
color: "white",
px: 1,
py: 0.5,
borderRadius: 1,
fontSize: "0.75rem",
fontWeight: 600,
}}
>
VIDEO
</Box>
</Box>
) : hasImage && (imageBlobUrl || imageUrl) ? (
<Box
sx={{
width: "100%",
@@ -373,21 +465,14 @@ export const SceneCard: React.FC<SceneCardProps> = ({
objectFit: "cover",
}}
onError={(e) => {
console.error('[SceneCard] Image failed to load:', {
console.error("[SceneCard] Image failed to load:", {
src: e.currentTarget.src,
imageUrl,
imageBlobUrl,
hasImage,
});
}}
onLoad={() => {
console.log('[SceneCard] Image loaded successfully:', {
src: imageBlobUrl || imageUrl,
});
}}
/>
</Box>
)}
) : null}
{/* Action Buttons */}
<SceneActionButtons
@@ -402,12 +487,25 @@ export const SceneCard: React.FC<SceneCardProps> = ({
isBusy={isBusy}
onRender={onRender}
onImageGenerate={onImageGenerate}
onVideoRender={onVideoRender}
onVideoRender={() => setShowVideoModal(true)}
onDownloadAudio={onDownloadAudio}
onDownloadVideo={onDownloadVideo}
onShare={onShare}
onError={onError}
/>
{/* Video Generation Settings Modal */}
<VideoRegenerateModal
open={showVideoModal}
onClose={() => setShowVideoModal(false)}
onGenerate={(settings: VideoGenerationSettings) => {
setShowVideoModal(false);
onVideoGenerate(scene.id, settings);
}}
initialPrompt={initialVideoPrompt}
initialResolution="480p"
initialSeed={-1}
/>
</Stack>
</GlassyCard>
);

View File

@@ -0,0 +1,228 @@
import React, { useEffect, useState } from "react";
import {
Dialog,
DialogTitle,
DialogContent,
DialogActions,
Stack,
Box,
Typography,
TextField,
FormControl,
FormLabel,
RadioGroup,
FormControlLabel,
Radio,
Tooltip,
} from "@mui/material";
import { Info as InfoIcon } from "@mui/icons-material";
import { PrimaryButton, SecondaryButton } from "../ui";
import type { VideoGenerationSettings } from "../types";
interface VideoRegenerateModalProps {
open: boolean;
onClose: () => void;
onGenerate: (settings: VideoGenerationSettings) => void;
initialPrompt: string;
initialResolution?: "480p" | "720p";
initialSeed?: number | null;
}
export const VideoRegenerateModal: React.FC<VideoRegenerateModalProps> = ({
open,
onClose,
onGenerate,
initialPrompt,
initialResolution = "480p",
initialSeed = -1,
}) => {
const [prompt, setPrompt] = useState(initialPrompt);
const [resolution, setResolution] = useState<"480p" | "720p">(initialResolution);
const [seed, setSeed] = useState<string>(initialSeed != null && initialSeed !== -1 ? String(initialSeed) : "");
const [maskImageUrl, setMaskImageUrl] = useState<string>("");
useEffect(() => {
setPrompt(initialPrompt);
setResolution(initialResolution);
}, [initialResolution, initialPrompt]);
const handleGenerate = () => {
const parsedSeed = seed.trim() === "" ? undefined : Number.isNaN(Number(seed)) ? undefined : Number(seed);
const settings: VideoGenerationSettings = {
prompt: prompt.trim(),
resolution,
seed: parsedSeed,
maskImageUrl: maskImageUrl.trim() || undefined,
};
onGenerate(settings);
};
return (
<Dialog
open={open}
onClose={onClose}
maxWidth="md"
fullWidth
PaperProps={{
sx: {
background: "rgba(15, 23, 42, 0.96)",
backdropFilter: "blur(18px)",
borderRadius: 4,
border: "1px solid rgba(148, 163, 184, 0.4)",
},
}}
>
<DialogTitle>
<Stack direction="row" justifyContent="space-between" alignItems="center">
<Typography variant="h6" sx={{ color: "white", fontWeight: 600 }}>
Configure Video Generation
</Typography>
<Tooltip title="Adjust how your talking-head video is rendered. These settings control resolution, prompt, and animation seed.">
<InfoIcon sx={{ color: "rgba(148,163,184,0.9)" }} />
</Tooltip>
</Stack>
<Typography variant="body2" sx={{ color: "rgba(148,163,184,0.9)", mt: 1 }}>
Fine-tune how this scene is animated. InfiniteTalk is audio-driven, so use the prompt to describe the visual
look and feel you want while keeping it concise.
</Typography>
</DialogTitle>
<DialogContent>
<Stack spacing={3} sx={{ mt: 1 }}>
{/* Prompt */}
<Box>
<FormLabel sx={{ color: "rgba(248,250,252,0.9)", mb: 0.5 }}>Visual prompt</FormLabel>
<TextField
multiline
minRows={3}
maxRows={6}
fullWidth
value={prompt}
onChange={(e) => setPrompt(e.target.value)}
placeholder="Short description of how the scene should look (lighting, mood, camera feel, etc.)"
variant="outlined"
InputProps={{
sx: {
bgcolor: "rgba(15,23,42,0.9)",
color: "white",
"& .MuiOutlinedInput-notchedOutline": {
borderColor: "rgba(148,163,184,0.4)",
},
"&:hover .MuiOutlinedInput-notchedOutline": {
borderColor: "rgba(125,211,252,0.8)",
},
},
}}
InputLabelProps={{
sx: { color: "rgba(148,163,184,0.9)" },
}}
/>
<Typography variant="caption" sx={{ color: "rgba(148,163,184,0.9)", mt: 0.5, display: "block" }}>
Example: &quot;Modern podcast studio with soft lighting, the host framed center, gentle camera movement.&quot;
</Typography>
</Box>
{/* Resolution */}
<Box>
<FormLabel sx={{ color: "rgba(248,250,252,0.9)", mb: 1 }}>Resolution &amp; quality</FormLabel>
<RadioGroup
row
value={resolution}
onChange={(e) => setResolution(e.target.value as "480p" | "720p")}
>
<FormControlLabel
value="480p"
control={<Radio color="primary" />}
label={
<Box>
<Typography variant="body2">480p (Recommended)</Typography>
<Typography variant="caption" color="text.secondary">
Faster render, lower cost, great for previews &amp; social
</Typography>
</Box>
}
/>
<FormControlLabel
value="720p"
control={<Radio color="primary" />}
label={
<Box>
<Typography variant="body2">720p (Higher quality)</Typography>
<Typography variant="caption" color="text.secondary">
Sharper video, slightly higher cost and render time
</Typography>
</Box>
}
/>
</RadioGroup>
</Box>
{/* Seed & advanced options */}
<Stack direction={{ xs: "column", sm: "row" }} spacing={2}>
<FormControl fullWidth>
<FormLabel sx={{ color: "rgba(248,250,252,0.9)", mb: 0.5 }}>Seed (optional)</FormLabel>
<TextField
type="number"
value={seed}
onChange={(e) => setSeed(e.target.value)}
placeholder="Random each time if left empty"
InputProps={{
sx: {
bgcolor: "rgba(15,23,42,0.9)",
color: "white",
"& .MuiOutlinedInput-notchedOutline": {
borderColor: "rgba(148,163,184,0.4)",
},
"&:hover .MuiOutlinedInput-notchedOutline": {
borderColor: "rgba(125,211,252,0.8)",
},
},
}}
/>
<Typography variant="caption" sx={{ color: "rgba(148,163,184,0.9)", mt: 0.5 }}>
Use the same seed to get a similar animation style across multiple scenes.
</Typography>
</FormControl>
<FormControl full-width="true">
<FormLabel sx={{ color: "rgba(248,250,252,0.9)", mb: 0.5 }}>Mask image URL (optional)</FormLabel>
<TextField
value={maskImageUrl}
onChange={(e) => setMaskImageUrl(e.target.value)}
placeholder="e.g. /api/podcast/images/your_avatar_mask.png"
InputProps={{
sx: {
bgcolor: "rgba(15,23,42,0.9)",
color: "white",
"& .MuiOutlinedInput-notchedOutline": {
borderColor: "rgba(148,163,184,0.4)",
},
"&:hover .MuiOutlinedInput-notchedOutline": {
borderColor: "rgba(125,211,252,0.8)",
},
},
}}
/>
<Typography variant="caption" sx={{ color: "rgba(148,163,184,0.9)", mt: 0.5 }}>
Optional: limit animation to a specific region (e.g. face) by providing a mask image URL. Leave empty to
animate the whole frame.
</Typography>
</FormControl>
</Stack>
</Stack>
</DialogContent>
<DialogActions sx={{ p: 2.5, pt: 0, justifyContent: "space-between" }}>
<Typography variant="caption" sx={{ color: "rgba(148,163,184,0.9)" }}>
Estimated cost at 480p is lower than 720p. You&apos;ll only be billed for successful renders.
</Typography>
<Stack direction="row" spacing={1}>
<SecondaryButton onClick={onClose}>Cancel</SecondaryButton>
<PrimaryButton onClick={handleGenerate}>Generate Video</PrimaryButton>
</Stack>
</DialogActions>
</Dialog>
);
}

View File

@@ -1,5 +1,5 @@
import { useState, useEffect, useRef, useCallback } from "react";
import { Script, Knobs, Job, RenderJobResult, TaskStatus } from "../types";
import { Script, Knobs, Job, RenderJobResult, TaskStatus, VideoGenerationSettings } from "../types";
import { podcastApi } from "../../../services/podcastApi";
interface UseRenderQueueProps {
@@ -36,7 +36,11 @@ export const useRenderQueue = ({
duration: number;
sceneCount: number;
} | null>(null);
const [combiningVideos, setCombiningVideos] = useState(false);
const [finalVideoUrl, setFinalVideoUrl] = useState<string | null>(null);
const [combiningProgress, setCombiningProgress] = useState<{ progress: number; message: string } | null>(null);
const pollingIntervals = useRef<Map<string, NodeJS.Timeout>>(new Map());
const pollingErrorCounts = useRef<Map<string, number>>(new Map());
// Cleanup polling intervals on unmount
useEffect(() => {
@@ -44,10 +48,11 @@ export const useRenderQueue = ({
return () => {
intervals.forEach((interval) => clearInterval(interval));
intervals.clear();
pollingErrorCounts.current.clear();
};
}, []);
// Initialize jobs if empty
// Initialize jobs if empty (audio/image only)
useEffect(() => {
if (jobs.length === 0 && script.scenes.length > 0) {
const initialJobs: Job[] = script.scenes.map((s) => {
@@ -59,7 +64,7 @@ export const useRenderQueue = ({
progress: hasExistingAudio ? 100 : 0,
previewUrl: null,
finalUrl: hasExistingAudio ? s.audioUrl || null : null,
imageUrl: s.imageUrl || null, // Include existing imageUrl from scene
imageUrl: s.imageUrl || null,
jobId: null,
};
});
@@ -67,25 +72,201 @@ export const useRenderQueue = ({
onUpdateJob(job.sceneId, job);
});
}
}, [script.scenes.length, jobs.length, onUpdateJob, script.scenes]);
}, [jobs.length, script.scenes.length, onUpdateJob, script.scenes]);
// Load final video URL from project on mount (for persistence across reloads)
useEffect(() => {
if (!projectId) return;
podcastApi
.loadProject(projectId)
.then((project) => {
if (project.final_video_url) {
console.log("[useRenderQueue] Loaded final video URL from project:", project.final_video_url);
setFinalVideoUrl(project.final_video_url);
}
})
.catch((error) => {
console.error("[useRenderQueue] Failed to load project for final video URL:", error);
// Don't show error to user - this is just for restoring state
});
}, [projectId]);
// Always try to attach existing videos to scenes (even after reloads)
useEffect(() => {
if (script.scenes.length === 0) return;
podcastApi
.listVideos(projectId)
.then((result) => {
const videoMap = new Map<number, string>();
result.videos.forEach((video) => {
// Use the most recent video for each scene number
if (!videoMap.has(video.scene_number)) {
// Store the raw video URL - SceneCard will handle authentication via blob loading
videoMap.set(video.scene_number, video.video_url);
}
});
script.scenes.forEach((scene) => {
const sceneNumberMatch = scene.id.match(/\d+/);
const sceneNumber = sceneNumberMatch ? parseInt(sceneNumberMatch[0], 10) : null;
if (sceneNumber === null) return;
const videoUrl = videoMap.get(sceneNumber);
if (!videoUrl) return;
const job = jobs.find((j) => j.sceneId === scene.id);
// Avoid redundant updates
if (job?.videoUrl === videoUrl) return;
onUpdateJob(scene.id, {
sceneId: scene.id,
title: scene.title,
videoUrl,
status: "completed" as const,
progress: 100,
});
});
})
.catch((error) => {
console.error("[useRenderQueue] Failed to list existing videos:", error);
});
}, [projectId, script.scenes, jobs, onUpdateJob]);
// Periodic check to rescue videos that were generated but not detected by polling
useEffect(() => {
if (rendering && script.scenes.length > 0) {
const rescueInterval = setInterval(async () => {
// Check for videos every 2 minutes while rendering is active
try {
const videoList = await podcastApi.listVideos(projectId);
const videoMap = new Map<number, string>();
videoList.videos.forEach((video) => {
if (!videoMap.has(video.scene_number)) {
// Store the raw video URL - SceneCard will handle authentication via blob loading
videoMap.set(video.scene_number, video.video_url);
}
});
// Update jobs for scenes that have videos but no videoUrl set
script.scenes.forEach((scene) => {
const sceneNumberMatch = scene.id.match(/\d+/);
const sceneNumber = sceneNumberMatch ? parseInt(sceneNumberMatch[0], 10) : null;
if (sceneNumber !== null) {
const videoUrl = videoMap.get(sceneNumber);
const job = jobs.find((j) => j.sceneId === scene.id);
if (videoUrl) {
if (!job) {
onUpdateJob(scene.id, {
sceneId: scene.id,
title: scene.title,
status: "completed" as const,
progress: 100,
videoUrl,
});
} else if (!job.videoUrl) {
onUpdateJob(scene.id, { videoUrl, status: "completed" as const, progress: 100 });
// If this was the rendering scene, stop rendering
if (rendering === scene.id) {
setRendering(null);
}
}
}
}
});
} catch (error) {
console.error("[useRenderQueue] Failed to rescue videos:", error);
}
}, 120000); // Check every 2 minutes
return () => clearInterval(rescueInterval);
}
}, [rendering, script.scenes, jobs, projectId, onUpdateJob]);
const getScene = useCallback((sceneId: string) => script.scenes.find((s) => s.id === sceneId), [script.scenes]);
const pollTaskStatus = useCallback(async (taskId: string, sceneId: string) => {
try {
const status: TaskStatus = await podcastApi.pollTaskStatus(taskId);
const status: TaskStatus | null = await podcastApi.pollTaskStatus(taskId);
// Handle null response (task not found)
if (!status) {
const errorCount = (pollingErrorCounts.current.get(sceneId) || 0) + 1;
pollingErrorCounts.current.set(sceneId, errorCount);
// Stop polling after 3 consecutive "task not found" errors
if (errorCount >= 3) {
onUpdateJob(sceneId, { status: "failed", progress: 0 });
const interval = pollingIntervals.current.get(sceneId);
if (interval) {
clearInterval(interval);
pollingIntervals.current.delete(sceneId);
}
pollingErrorCounts.current.delete(sceneId);
setRendering(null);
onError("Video generation task not found. The task may have expired or been cancelled.");
return true; // Stop polling
}
return false; // Continue polling (might be transient)
}
// Reset error count on successful poll
pollingErrorCounts.current.delete(sceneId);
onUpdateJob(sceneId, {
progress: status.progress ?? 0,
status: status.status === "completed" ? "completed" : status.status === "failed" ? "failed" : "running",
});
if (status.status === "completed" && status.result) {
// Check for completion - handle both "completed" and "processing" with 100% progress
const isCompleted = status.status === "completed" || (status.status === "processing" && status.progress === 100);
if (isCompleted && status.result) {
const result = status.result;
console.log("[useRenderQueue] Task completed, extracting video URL", {
result,
video_url: result.video_url,
status: status.status,
progress: status.progress,
});
let videoUrl = result.video_url;
if (!videoUrl) {
console.error("[useRenderQueue] No video_url in result! Attempting to rescue from file system...", { result });
// Try to rescue: check if video exists for this scene
const sceneNumberMatch = getScene(sceneId)?.id.match(/\d+/);
const sceneNumber = sceneNumberMatch ? parseInt(sceneNumberMatch[0], 10) : null;
if (sceneNumber !== null) {
podcastApi
.listVideos(projectId)
.then((videoList) => {
const sceneVideo = videoList.videos.find((v) => v.scene_number === sceneNumber);
if (sceneVideo) {
// Store the raw video URL - SceneCard will handle authentication via blob loading
onUpdateJob(sceneId, {
status: "completed",
progress: 100,
videoUrl: sceneVideo.video_url,
cost: result.cost || 0,
});
}
})
.catch((err) => console.error("[useRenderQueue] Failed to rescue video:", err));
}
return true; // Stop polling
}
// Store the raw video URL - SceneCard will handle authentication via blob loading
onUpdateJob(sceneId, {
status: "completed",
progress: 100,
videoUrl: result.video_url,
videoUrl,
cost: result.cost,
});
@@ -94,20 +275,62 @@ export const useRenderQueue = ({
clearInterval(interval);
pollingIntervals.current.delete(sceneId);
}
setRendering(null);
return true; // Stop polling
} else if (status.status === "failed") {
// Extract user-friendly error message
let errorMessage = "Video generation failed";
if (status.error) {
// Try to extract meaningful error from various formats
const errorStr = status.error;
if (errorStr.includes("Insufficient credits")) {
errorMessage = "Video generation failed: Insufficient WaveSpeed credits. Please top up your account.";
} else if (errorStr.includes("HTTPException") || errorStr.includes("502")) {
// Extract the actual error message from HTTPException details
const match = errorStr.match(/message[":\s]+"([^"]+)"/i) || errorStr.match(/detail[":\s]+"([^"]+)"/i);
if (match && match[1]) {
errorMessage = `Video generation failed: ${match[1]}`;
} else {
errorMessage = `Video generation failed: ${errorStr}`;
}
} else {
errorMessage = `Video generation failed: ${errorStr}`;
}
}
onUpdateJob(sceneId, { status: "failed", progress: 0 });
const interval = pollingIntervals.current.get(sceneId);
if (interval) {
clearInterval(interval);
pollingIntervals.current.delete(sceneId);
}
onError(status.error || "Video generation failed");
pollingErrorCounts.current.delete(sceneId);
setRendering(null);
onError(errorMessage);
return true; // Stop polling
}
return status.status === "completed" || status.status === "failed";
return false; // Continue polling
} catch (error) {
console.error("Error polling task status:", error);
return false;
const errorCount = (pollingErrorCounts.current.get(sceneId) || 0) + 1;
pollingErrorCounts.current.set(sceneId, errorCount);
// Stop polling after 5 consecutive network errors
if (errorCount >= 5) {
onUpdateJob(sceneId, { status: "failed", progress: 0 });
const interval = pollingIntervals.current.get(sceneId);
if (interval) {
clearInterval(interval);
pollingIntervals.current.delete(sceneId);
}
pollingErrorCounts.current.delete(sceneId);
setRendering(null);
const errorMsg = error instanceof Error ? error.message : String(error);
onError(`Video generation failed: Unable to check status. ${errorMsg}`);
return true; // Stop polling
}
return false; // Continue polling (might be transient network error)
}
}, [onUpdateJob, onError]);
@@ -217,6 +440,7 @@ export const useRenderQueue = ({
sceneId: scene.id,
sceneTitle: scene.title,
sceneContent: sceneContent,
baseAvatarUrl: avatarImageUrl || undefined, // Use base avatar if available
width: 1024,
height: 1024,
});
@@ -239,68 +463,112 @@ export const useRenderQueue = ({
} finally {
setGeneratingImage(null);
}
}, [generatingImage, getScene, onUpdateJob, onError]);
}, [generatingImage, getScene, avatarImageUrl, onUpdateJob, onError, script]);
const runVideoRender = useCallback(async (sceneId: string) => {
if (rendering && rendering !== sceneId) return;
const scene = getScene(sceneId);
if (!scene) return;
const sceneImageUrl = scene.imageUrl || avatarImageUrl;
if (!sceneImageUrl) {
onError("Scene image is required for video generation. Please generate images for scenes first.");
return;
}
const job = jobs.find((j) => j.sceneId === sceneId);
if (!job?.finalUrl) {
onError("Please generate audio first before creating video.");
return;
}
const estimatedCost = 0.30;
if (budgetCap && budgetCap > 0) {
const currentSpent = jobs
.filter((j) => j.status === "completed" && j.cost)
.reduce((sum, j) => sum + (j.cost || 0), 0);
if (currentSpent + estimatedCost > budgetCap) {
onError(`Budget cap exceeded. Estimated cost: $${estimatedCost.toFixed(2)}, Budget remaining: $${(budgetCap - currentSpent).toFixed(2)}`);
const runVideoRender = useCallback(
async (sceneId: string, settings?: VideoGenerationSettings) => {
if (rendering && rendering !== sceneId) {
return;
}
}
setRendering(sceneId);
onUpdateJob(sceneId, {
status: "running",
progress: 5,
});
const scene = getScene(sceneId);
if (!scene) {
return;
}
try {
const result = await podcastApi.generateVideo({
projectId,
sceneId,
sceneTitle: scene.title,
audioUrl: job.finalUrl,
avatarImageUrl: sceneImageUrl,
resolution: knobs.resolution || "720p",
});
// Guard: require image and audio before calling expensive video gen
const sceneImageUrl = scene.imageUrl || avatarImageUrl;
if (!sceneImageUrl) {
onError("Scene image is required for video generation. Please generate images for scenes first.");
return;
}
const job = jobs.find((j) => j.sceneId === sceneId);
// Use job.finalUrl if available, otherwise fall back to scene.audioUrl (from Script Editor)
const audioUrl = job?.finalUrl || scene.audioUrl;
if (!audioUrl || audioUrl.startsWith("blob:")) {
onError("Please generate audio first before creating video.");
return;
}
// Guard: ensure every scene has audio and image before enabling render queue video
const allScenesHaveAudio = script.scenes.every((s) => s.audioUrl && !s.audioUrl.startsWith("blob:"));
const allScenesHaveImage = script.scenes.every((s) => s.imageUrl);
if (!allScenesHaveAudio || !allScenesHaveImage) {
onError("Please ensure all scenes have both audio and image before generating video.");
return;
}
// Resolution & simple cost heuristic (default 480p for lower cost)
const targetResolution: "480p" | "720p" =
settings?.resolution || (knobs.resolution as "480p" | "720p") || "480p";
const baseCost = 0.3; // 5s at 720p
const estimatedCost = targetResolution === "480p" ? baseCost / 2 : baseCost;
if (budgetCap && budgetCap > 0) {
const currentSpent = jobs
.filter((j) => j.status === "completed" && j.cost)
.reduce((sum, j) => sum + (j.cost || 0), 0);
if (currentSpent + estimatedCost > budgetCap) {
onError(
`Budget cap exceeded. Estimated cost: $${estimatedCost.toFixed(
2
)}, Budget remaining: $${(budgetCap - currentSpent).toFixed(2)}`
);
return;
}
}
setRendering(sceneId);
onUpdateJob(sceneId, {
taskId: result.taskId,
status: "running",
progress: 5,
});
startPolling(result.taskId, sceneId);
} catch (error) {
onUpdateJob(sceneId, { status: "failed", progress: 0 });
const message = error instanceof Error ? error.message : "Video generation failed";
onError(message);
} finally {
setRendering(null);
}
}, [rendering, getScene, avatarImageUrl, jobs, budgetCap, projectId, knobs, onUpdateJob, onError, startPolling]);
try {
console.log("[useRenderQueue] Starting video generation", {
sceneId,
sceneTitle: scene.title,
audioUrl,
avatarImageUrl: sceneImageUrl,
resolution: targetResolution,
prompt: settings?.prompt,
seed: settings?.seed,
maskImageUrl: settings?.maskImageUrl,
});
const result = await podcastApi.generateVideo({
projectId,
sceneId,
sceneTitle: scene.title,
audioUrl,
avatarImageUrl: sceneImageUrl,
resolution: targetResolution,
prompt: settings?.prompt || undefined,
seed: settings?.seed ?? -1,
maskImageUrl: settings?.maskImageUrl || undefined,
});
if (!result.taskId) {
throw new Error("Backend did not return a task ID. Response: " + JSON.stringify(result));
}
onUpdateJob(sceneId, {
taskId: result.taskId,
status: "running",
progress: 5,
});
startPolling(result.taskId, sceneId);
} catch (error) {
onUpdateJob(sceneId, { status: "failed", progress: 0 });
const message = error instanceof Error ? error.message : "Video generation failed";
onError(message);
}
},
[rendering, getScene, avatarImageUrl, jobs, budgetCap, projectId, knobs, onUpdateJob, onError, script.scenes, startPolling]
);
const combineAudio = useCallback(async () => {
try {
@@ -361,16 +629,151 @@ export const useRenderQueue = ({
}
}, [script.scenes, jobs, projectId, onError]);
const combineFinalVideo = useCallback(async () => {
try {
setCombiningVideos(true);
onError("");
// Collect all scene video URLs
const sceneVideoUrls: string[] = [];
for (const scene of script.scenes) {
const job = jobs.find((j) => j.sceneId === scene.id);
if (!job?.videoUrl) {
throw new Error(`Scene "${scene.title}" is missing a video. Please generate videos for all scenes first.`);
}
// Remove blob URLs and query params - use the API path only
let videoUrl = job.videoUrl;
if (videoUrl.startsWith("blob:")) {
throw new Error(`Scene "${scene.title}" has a blob URL. Cannot combine blob URLs. Please use API URLs.`);
}
videoUrl = videoUrl.split("?")[0]; // Remove query params
sceneVideoUrls.push(videoUrl);
}
console.log("[combineFinalVideo] Starting combination with", sceneVideoUrls.length, "videos");
// Start combination task
const result = await podcastApi.combineVideos({
projectId,
sceneVideoUrls,
podcastTitle: script.scenes[0]?.title || "Podcast",
});
console.log("[combineFinalVideo] Task created:", result.taskId);
// Poll for completion
const taskId = result.taskId;
let done = false;
let pollCount = 0;
const maxPolls = 300; // 10 minutes max (300 * 2 seconds) - encoding can take time
let lastProgress = 0;
let lastMessage = "Starting video combination...";
while (!done && pollCount < maxPolls) {
await new Promise((r) => setTimeout(r, 2000)); // Poll every 2 seconds
pollCount++;
const status = await podcastApi.pollTaskStatus(taskId);
// Update progress and message for user feedback
if (status) {
const currentProgress = status.progress ?? 0;
const currentMessage = status.message || "Processing...";
// Update UI with progress
setCombiningProgress({
progress: currentProgress,
message: currentMessage,
});
// Only log if progress or message changed to reduce noise
if (currentProgress !== lastProgress || currentMessage !== lastMessage) {
console.log(
`[combineFinalVideo] Poll ${pollCount}: ${status.status} | ` +
`Progress: ${currentProgress.toFixed(1)}% | Message: ${currentMessage}`
);
lastProgress = currentProgress;
lastMessage = currentMessage;
}
} else {
console.log(`[combineFinalVideo] Poll ${pollCount}: No status yet...`);
}
if (!status) {
// Don't fail immediately - task might still be initializing
if (pollCount < 10) {
continue; // Wait up to 20 seconds for task to appear
}
console.error("[combineFinalVideo] Task not found after 10 polls");
throw new Error("Task not found. Video combination may have failed on the server. Please try again.");
}
if (status.status === "completed") {
done = true;
const videoUrl = status.result?.video_url;
if (!videoUrl) {
console.error("[combineFinalVideo] No video URL in result:", status.result);
throw new Error("Final video URL not found in result. Please contact support.");
}
console.log("[combineFinalVideo] Success! Video URL:", videoUrl);
setFinalVideoUrl(videoUrl);
// Save final video URL to project for persistence across reloads
try {
await podcastApi.saveProject(projectId, { final_video_url: videoUrl });
console.log("[combineFinalVideo] Saved final video URL to project");
} catch (error) {
console.warn("[combineFinalVideo] Failed to save final video URL to project:", error);
// Don't fail the operation if project save fails - video is still available
}
} else if (status.status === "failed") {
const errorMsg = status.error || status.message || "Video combination failed";
console.error("[combineFinalVideo] Task failed:", errorMsg);
throw new Error(`Video combination failed: ${errorMsg}`);
}
}
if (pollCount >= maxPolls) {
throw new Error("Video combination timed out after 10 minutes. The video may still be processing. Please check back in a few minutes or try again.");
}
} catch (error: any) {
console.error("[combineFinalVideo] Error:", error);
// Extract detailed error message
let message = "Failed to combine videos";
if (error?.response?.data?.detail) {
// Backend error with detail
message = error.response.data.detail;
} else if (error?.message) {
// Standard error message
message = error.message;
} else if (typeof error === "string") {
message = error;
}
console.error("[combineFinalVideo] Displaying error to user:", message);
onError(message);
} finally {
setCombiningVideos(false);
setCombiningProgress(null);
}
}, [script.scenes, jobs, projectId, onError]);
return {
rendering,
generatingImage,
combiningAudio,
combinedAudioResult,
combiningVideos,
combiningProgress,
finalVideoUrl,
isBusy: Boolean(rendering),
runRender,
runImageGeneration,
runVideoRender,
combineAudio,
combineFinalVideo,
};
};