Skip to content

Handle rapid/bulk YouTube uploads by enqueueing all unseen uploads #222

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Copilot wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

Handle rapid/bulk YouTube uploads by enqueueing all unseen uploads #222

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a1f1da0
Initial plan
Copilot Jul 22, 2026
ede7b91
fix: queue all unseen rapid YouTube uploads
Copilot Jul 22, 2026
5354ef4
Apply remaining changes
Copilot Jul 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
304 changes: 138 additions & 166 deletions src/utils/youtube/fetchLatestUploads.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { YouTubeVideoContentDetailsResponse } from "../../types/youtube";
import type { YouTubePlaylistResponse } from "../../types/youtube";

import { Platform } from "../../types/types.d";
import {
Expand All @@ -13,79 +13,89 @@ import {
import { discordGetAllGuildsTrackingChannel } from "../../db/discord";

import getChannelDetails from "./getChannelDetails";
import getSinglePlaylistAndReturnVideoData, {
PlaylistType,
} from "./getSinglePlaylistAndReturnVideoData";

/**
* Parse a full ISO 8601 duration string into total seconds.
* Supports formats like "PT1H2M3S", "P1DT2H3M4S", "PT30S", etc.
* Note: YouTube typically returns PT-prefixed durations, but very long
* streams/videos could include a day component (P1DT...).
*/
function parseISO8601Duration(duration: string): number {
const match = duration.match(
/^P(?:(\d+)D)?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?)?$/,
);

if (!match) return 0;
const days = parseInt(match[1] || "0", 10);
const hours = parseInt(match[2] || "0", 10);
const minutes = parseInt(match[3] || "0", 10);
const seconds = parseInt(match[4] || "0", 10);
import { PlaylistType } from "./getSinglePlaylistAndReturnVideoData";

return days * 86400 + hours * 3600 + minutes * 60 + seconds;
}
export const updates = new Map<
string,
{
channelInfo: Awaited<ReturnType<typeof getChannelDetails>>;
discordGuildsToUpdate: (typeof dbGuildYouTubeSubscriptionsTable.$inferSelect)[];
}
>();

/**
* Fetch the duration (in seconds) of a video using the YouTube Videos API.
* Returns 0 if the duration cannot be determined.
*/
async function fetchVideoDuration(videoId: string): Promise<number> {
async function fetchUnseenUploadVideoIds(
channelId: string,
latestKnownVideoId: string,
): Promise<string[]> {
const uploadPlaylistId = `UU${channelId.slice(2)}`;
const unseenVideoIds: string[] = [];
const res = await fetch(
`https://youtube.googleapis.com/youtube/v3/videos?part=contentDetails&id=${videoId}&key=${env.youtubeApiKey}`,
`https://youtube.googleapis.com/youtube/v3/playlistItems?part=snippet&maxResults=50&playlistId=${uploadPlaylistId}&key=${env.youtubeApiKey}`,
);

if (!res.ok) {
console.error("Error fetching video duration:", res.statusText);
console.error(
"Error fetching upload playlist items in fetchLatestUploads:",
res.statusText,
);
return unseenVideoIds;
}

const data = (await res.json()) as YouTubePlaylistResponse;

return 0;
if (!data.items || data.items.length === 0) {
return unseenVideoIds;
}

const data =
(await res.json()) as unknown as YouTubeVideoContentDetailsResponse;
for (const item of data.items) {
const unseenVideoId = item?.snippet?.resourceId?.videoId;

if (!data.items || data.items.length === 0) return 0;
if (!unseenVideoId) {
continue;
}

if (unseenVideoId === latestKnownVideoId) {
break;
}

const durationFirstItem = data.items[0];
unseenVideoIds.push(unseenVideoId);
}

return unseenVideoIds;
}

async function fetchPlaylistVideoIds(
channelId: string,
playlistType: PlaylistType.Short | PlaylistType.Stream,
): Promise<Set<string>> {
const playlistPrefix =
playlistType === PlaylistType.Short ? "UUSH" : "UULV";
const playlistId = `${playlistPrefix}${channelId.slice(2)}`;
const res = await fetch(
`https://youtube.googleapis.com/youtube/v3/playlistItems?part=snippet&maxResults=50&playlistId=${playlistId}&key=${env.youtubeApiKey}`,
);

if (
!durationFirstItem.contentDetails ||
!durationFirstItem.contentDetails.duration
) {
if (!res.ok) {
console.error(
"Duration not found in video details for video ID:",
videoId,
`Error fetching ${playlistType} playlist items in fetchLatestUploads:`,
res.statusText,
);

return 0;
return new Set();
}

const duration = durationFirstItem?.contentDetails?.duration;
const data = (await res.json()) as YouTubePlaylistResponse;

if (typeof duration !== "string") return 0;
if (!data.items || data.items.length === 0) {
return new Set();
}

return parseISO8601Duration(duration);
return new Set(
data.items
.map((item) => item?.snippet?.resourceId?.videoId)
.filter((videoId): videoId is string => typeof videoId === "string"),
);
}

export const updates = new Map<
string,
{
channelInfo: Awaited<ReturnType<typeof getChannelDetails>>;
discordGuildsToUpdate: (typeof dbGuildYouTubeSubscriptionsTable.$inferSelect)[];
}
>();

export default async function fetchLatestUploads() {
console.log("Fetching latest uploads...");

Expand Down Expand Up @@ -139,7 +149,7 @@ export default async function fetchLatestUploads() {
// TODO: Upload time (https://github.com/GalvinPython/feedr/issues/136)
for (const playlist of data.items) {
const channelId = playlist.snippet.channelId;
const videoId =
const latestVideoId =
playlist.snippet.thumbnails.default.url.split("/")[4];

if (!channelDict[channelId]) {
Expand All @@ -150,103 +160,30 @@ export default async function fetchLatestUploads() {
continue;
}

const requiresUpdate =
channelDict[channelId].latestAllId !== videoId;
const latestKnownVideoId = channelDict[channelId].latestAllId;
const requiresUpdate = latestKnownVideoId !== latestVideoId;

if (requiresUpdate) {
console.log(
"Channel ID:",
channelId,
"Video ID:",
videoId,
latestVideoId,
"Requires update?",
requiresUpdate,
);
// Use duration-based detection to reduce API quota usage
// and avoid UULF which is currently lagging.
// YouTube Shorts are currently limited to 3 minutes (180s).
// Videos at exactly 180s could still be shorts, so we use
// a strict greater-than check.
const durationSeconds = await fetchVideoDuration(videoId);
const SHORTS_DURATION = 180;

let contentType: PlaylistType | null = null;

if (durationSeconds > SHORTS_DURATION) {
// Over the shorts limit: cannot be a short, check only if it's a stream
const streamVideoId =
await getSinglePlaylistAndReturnVideoData(
channelId,
PlaylistType.Stream,
);
let videosToProcess = [latestVideoId];

if (videoId === streamVideoId.videoId) {
contentType = PlaylistType.Stream;
} else {
// Not a stream and over 3 min; must be a regular video
contentType = PlaylistType.Video;
}
} else {
// Under 3 minutes: could be a short or a video, check UUSH and UULV
const [shortVideoId, streamVideoId] = await Promise.all([
getSinglePlaylistAndReturnVideoData(
channelId,
PlaylistType.Short,
),
getSinglePlaylistAndReturnVideoData(
channelId,
PlaylistType.Stream,
),
]);

if (videoId === shortVideoId.videoId) {
contentType = PlaylistType.Short;
} else if (videoId === streamVideoId.videoId) {
contentType = PlaylistType.Stream;
} else {
// Not in shorts or streams playlist → regular video
contentType = PlaylistType.Video;
}
}

console.log(
"Determined content type:",
contentType,
`(duration: ${durationSeconds}s)`,
);

if (contentType) {
console.log(
`Updating ${contentType} video ID for channel`,
channelId,
"to",
videoId,
);
} else {
console.error(
"No valid video ID found for channel",
if (latestKnownVideoId) {
const unseenVideoIds = await fetchUnseenUploadVideoIds(
channelId,
"with video ID",
videoId,
);
continue;
}

const updateSuccess = await youtubeUpdateVideoId(
channelId,
videoId,
contentType,

// Temporarily using current date for update time
new Date(),
);

if (!updateSuccess.success) {
console.error(
"Error updating video ID in fetchLatestUploads",
latestKnownVideoId,
);

return;
if (unseenVideoIds.length > 0) {
// Process oldest to newest so notifications preserve upload order.
videosToProcess = unseenVideoIds.reverse();
}
}

const discordGuildsToUpdate =
Expand All @@ -264,34 +201,69 @@ export default async function fetchLatestUploads() {
}

const channelInfo = await getChannelDetails(channelId);
const [shortVideoIds, streamVideoIds] = await Promise.all([
fetchPlaylistVideoIds(channelId, PlaylistType.Short),
fetchPlaylistVideoIds(channelId, PlaylistType.Stream),
]);

for (const videoId of videosToProcess) {
let contentType = PlaylistType.Video;

console.info(`Filtered guilds for channel ID ${channelId}:`, {
count: discordGuildsToUpdate.data.filter(
(
guild,
): guild is typeof dbGuildYouTubeSubscriptionsTable.$inferSelect =>
"youtubeChannelId" in guild &&
"trackVideos" in guild &&
"trackShorts" in guild &&
"trackStreams" in guild,
).length,
});

updates.set(videoId, {
channelInfo,
discordGuildsToUpdate: discordGuildsToUpdate.data.filter(
(
guild,
): guild is typeof dbGuildYouTubeSubscriptionsTable.$inferSelect =>
"youtubeChannelId" in guild &&
((contentType === PlaylistType.Video &&
guild.trackVideos) ||
(contentType === PlaylistType.Short &&
guild.trackShorts) ||
(contentType === PlaylistType.Stream &&
guild.trackStreams)),
),
});
if (shortVideoIds.has(videoId)) {
contentType = PlaylistType.Short;
} else if (streamVideoIds.has(videoId)) {
contentType = PlaylistType.Stream;
} else {
contentType = PlaylistType.Video;
}

console.log("Determined content type:", contentType);

const updateSuccess = await youtubeUpdateVideoId(
channelId,
videoId,
contentType,

// Temporarily using current date for update time
new Date(),
);

if (!updateSuccess.success) {
console.error(
"Error updating video ID in fetchLatestUploads",
);

return;
}

console.info(`Filtered guilds for channel ID ${channelId}:`, {
count: discordGuildsToUpdate.data.filter(
(
guild,
): guild is typeof dbGuildYouTubeSubscriptionsTable.$inferSelect =>
"youtubeChannelId" in guild &&
"trackVideos" in guild &&
"trackShorts" in guild &&
"trackStreams" in guild,
).length,
});

updates.set(videoId, {
channelInfo,
discordGuildsToUpdate: discordGuildsToUpdate.data.filter(
(
guild,
): guild is typeof dbGuildYouTubeSubscriptionsTable.$inferSelect =>
"youtubeChannelId" in guild &&
((contentType === PlaylistType.Video &&
guild.trackVideos) ||
(contentType === PlaylistType.Short &&
guild.trackShorts) ||
(contentType === PlaylistType.Stream &&
guild.trackStreams)),
),
});
}
Comment on lines +209 to +266

@GalvinPython GalvinPython Jul 22, 2026

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this algorithm is checking the playlist for each video, however the playlist function will only return the latest video in that playlist. so if there's two shorts, it will be categorised as one video (default) and one short. we should check each playlist until each video id has been associated with a category to further reduce api calls

}
}
}
Expand Down