Hydrate feed timestamps before sorting

This commit is contained in:
thePR0M3TH3AN committed 2025-10-26 12:05:03 -04:00
1 parent 89de704336
commit b744524521
4 files changed
+251

No files matched your search

+5
View File
@@ -37,6 +37,7 @@ import {
createBlacklistFilterStage,
createDedupeByRootStage,
createModerationStage,
createResolvePostedAtStage,
createChronologicalSorter,
createSubscriptionAuthorsSource,
registerWatchHistoryFeed,
@@ -6730,11 +6731,13 @@ class Application {
trustedMuteHideThreshold: resolveThresholdFromApp("trustedMuteHideThreshold"),
trustedReportHideThreshold: resolveThresholdFromApp("trustedSpamHideThreshold"),
}),
createResolvePostedAtStage(),
],
sorter: createChronologicalSorter(),
hooks: {
timestamps: {
getKnownVideoPostedAt: (video) => this.getKnownVideoPostedAt(video),
resolveVideoPostedAt: (video) => this.resolveVideoPostedAt(video),
},
},
});
@@ -6799,6 +6802,7 @@ class Application {
trustedMuteHideThreshold: resolveThresholdFromApp("trustedMuteHideThreshold"),
trustedReportHideThreshold: resolveThresholdFromApp("trustedSpamHideThreshold"),
}),
createResolvePostedAtStage(),
],
sorter: createChronologicalSorter(),
hooks: {
@@ -6807,6 +6811,7 @@ class Application {
},
timestamps: {
getKnownVideoPostedAt: (video) => this.getKnownVideoPostedAt(video),
resolveVideoPostedAt: (video) => this.resolveVideoPostedAt(video),
},
},
});
+1
View File
@@ -10,6 +10,7 @@ export {
createBlacklistFilterStage,
createWatchHistorySuppressionStage,
createModerationStage,
createResolvePostedAtStage,
} from "./stages.js";
export { createChronologicalSorter } from "./sorters.js";
export {
+179
View File
@@ -79,6 +79,185 @@ export function createDedupeByRootStage({
};
}
function gatherTimestampFunctions({
context,
knownOverride,
resolveOverride,
}) {
const knownFns = [];
const resolveFns = [];
const addKnown = (fn) => {
if (typeof fn === "function") {
knownFns.push(fn);
}
};
const addResolver = (fn) => {
if (typeof fn === "function") {
resolveFns.push(fn);
}
};
addKnown(knownOverride);
addResolver(resolveOverride);
const hookCandidates = [context?.hooks, context?.runtime];
for (const candidate of hookCandidates) {
if (!candidate || typeof candidate !== "object") {
continue;
}
const timestamps = candidate.timestamps;
if (!timestamps || typeof timestamps !== "object") {
continue;
}
addKnown(timestamps.getKnownVideoPostedAt);
addKnown(timestamps.getKnownPostedAt);
addResolver(timestamps.resolveVideoPostedAt);
addResolver(timestamps.getVideoPostedAt);
}
return { knownFns, resolveFns };
}
function applyResolvedTimestamp(video, timestamp) {
if (!video || typeof video !== "object") {
return false;
}
if (!Number.isFinite(timestamp)) {
return false;
}
const normalized = Math.floor(timestamp);
if (!Number.isFinite(normalized)) {
return false;
}
if (Number.isFinite(video.rootCreatedAt)) {
const existing = Math.floor(video.rootCreatedAt);
if (Number.isFinite(existing) && existing <= normalized) {
video.rootCreatedAt = existing;
return true;
}
}
video.rootCreatedAt = normalized;
return true;
}
async function resolveTimestampWithResolvers({
video,
detail,
resolvers,
stageName,
context,
}) {
if (!Array.isArray(resolvers) || resolvers.length === 0) {
return null;
}
for (const resolver of resolvers) {
if (typeof resolver !== "function") {
continue;
}
try {
const value = await resolver(video, detail);
if (Number.isFinite(value)) {
return Math.floor(value);
}
} catch (error) {
context?.log?.(`[${stageName}] resolve timestamp hook threw`, error);
}
}
return null;
}
export function createResolvePostedAtStage({
stageName = "resolve-posted-at",
getKnownPostedAt,
resolvePostedAt,
} = {}) {
return async function resolvePostedAtStage(items = [], context = {}) {
if (!Array.isArray(items) || items.length === 0) {
return items;
}
const { knownFns, resolveFns } = gatherTimestampFunctions({
context,
knownOverride: getKnownPostedAt,
resolveOverride: resolvePostedAt,
});
if (knownFns.length === 0 && resolveFns.length === 0) {
return items;
}
const tasks = [];
for (const item of items) {
const video = item?.video;
if (!video || typeof video !== "object") {
continue;
}
if (Number.isFinite(video.rootCreatedAt)) {
continue;
}
const detail = { entry: item, context };
let knownValue = null;
for (const candidate of knownFns) {
try {
const value = candidate(video, detail);
if (Number.isFinite(value)) {
knownValue = Math.floor(value);
break;
}
} catch (error) {
context?.log?.(`[${stageName}] known timestamp hook threw`, error);
}
}
if (knownValue !== null && Number.isFinite(knownValue)) {
applyResolvedTimestamp(video, knownValue);
continue;
}
if (!resolveFns.length) {
continue;
}
tasks.push(async () => {
const resolved = await resolveTimestampWithResolvers({
video,
detail,
resolvers: resolveFns,
stageName,
context,
});
if (resolved !== null && Number.isFinite(resolved)) {
applyResolvedTimestamp(video, resolved);
}
});
}
for (const task of tasks) {
try {
await task();
} catch (error) {
context?.log?.(`[${stageName}] resolution task failed`, error);
}
}
return items;
};
}
export function createBlacklistFilterStage({
stageName = "blacklist-filter",
shouldIncludeVideo,
+66
View File
@@ -33,6 +33,7 @@ const {
createDedupeByRootStage,
createBlacklistFilterStage,
createWatchHistorySuppressionStage,
createResolvePostedAtStage,
createChronologicalSorter,
} = await import("../js/feedEngine/index.js");
@@ -168,6 +169,70 @@ async function testTimestampHookSorting() {
);
}
async function testResolvePostedAtStageHydration() {
const engine = createFeedEngine();
const feedName = "resolve-stage";
const editedVideo = {
id: "edited-stage",
videoRootId: "root-stage-a",
created_at: 500,
};
const newerRoot = {
id: "new-root-stage",
videoRootId: "root-stage-b",
created_at: 450,
};
engine.registerFeed(feedName, {
source: async () => [
{ video: editedVideo },
{ video: newerRoot },
],
stages: [createResolvePostedAtStage()],
sorter: createChronologicalSorter(),
});
const resolverCalls = [];
const result = await engine.runFeed(feedName, {
hooks: {
timestamps: {
async resolveVideoPostedAt(video) {
resolverCalls.push(video.id);
if (video.videoRootId === "root-stage-a") {
await new Promise((resolve) => setTimeout(resolve, 0));
return 100;
}
if (video.videoRootId === "root-stage-b") {
return 300;
}
return null;
},
},
},
});
assert.deepEqual(
result.videos.map((video) => video.id),
["new-root-stage", "edited-stage"],
"resolve stage should hydrate timestamps before sorting",
);
assert.equal(
editedVideo.rootCreatedAt,
100,
"resolve stage should stamp the original posted time on edited videos",
);
assert.equal(newerRoot.rootCreatedAt, 300);
assert.deepEqual(
new Set(resolverCalls),
new Set(["edited-stage", "new-root-stage"]),
"resolver should run for each video lacking a cached timestamp",
);
}
async function testBlacklistFiltering() {
const engine = createFeedEngine();
const feedName = "blacklist";
@@ -340,6 +405,7 @@ async function testTrustedMuteDownrank() {
await testDedupeOrdering();
await testRootCreatedAtSorting();
await testTimestampHookSorting();
await testResolvePostedAtStageHydration();
await testBlacklistFiltering();
await testWatchHistoryHookIsolation();
await testBlacklistOrderingWithRuntimeChanges();