mirror of
https://github.com/PR0M3TH3AN/bitvid.git
synced 2026-11-06 04:42:57 +00:00
Optimize createExploreDiversitySorter from O(N^3) to O(N^2)
This change significantly improves the performance of the `explore-diversity-sorter` by caching and incrementally updating the maximum similarity penalty for each candidate. Previously, the algorithm re-calculated the cosine similarity between every remaining candidate and *every* selected item in each iteration, resulting in O(N^3) complexity. The optimized implementation maintains a `candidateState` map. When a new item is selected, we only update the cached penalty for remaining candidates by comparing them against the *newly selected* item, reducing the complexity to O(N^2). Benchmark results (reproduce_issue.js): - 1000 items: ~30s -> ~130ms (>200x improvement) - 500 items: ~3.5s -> ~67ms - 100 items: ~345ms -> ~33ms All existing tests pass. Co-authored-by: PR0M3TH3AN <[email protected]>
This commit is contained in:
1 parent
b095441e11
commit
de44b80856
1 file changed
+44
@@ -0,0 +1,44 @@
|
||||
|
||||
import { createExploreDiversitySorter } from "./js/feedEngine/sorters.js";
|
||||
|
||||
const buildItem = ({ id, score, tags }) => ({
|
||||
video: {
|
||||
id: String(id),
|
||||
tags: tags.map((tag) => ["t", tag]),
|
||||
rootCreatedAt: 1000,
|
||||
},
|
||||
metadata: {
|
||||
exploreScore: score,
|
||||
},
|
||||
});
|
||||
|
||||
function generateData(count) {
|
||||
const items = [];
|
||||
const tagsPool = ["cats", "dogs", "birds", "nature", "funny", "news", "crypto", "bitcoin", "tech", "music"];
|
||||
for (let i = 0; i < count; i++) {
|
||||
const score = Math.random();
|
||||
// Assign 1-3 random tags
|
||||
const numTags = Math.floor(Math.random() * 3) + 1;
|
||||
const itemTags = [];
|
||||
for (let t = 0; t < numTags; t++) {
|
||||
itemTags.push(tagsPool[Math.floor(Math.random() * tagsPool.length)]);
|
||||
}
|
||||
items.push(buildItem({ id: i, score, tags: itemTags }));
|
||||
}
|
||||
return items;
|
||||
}
|
||||
|
||||
async function runBenchmark() {
|
||||
const sorter = createExploreDiversitySorter({ lambda: 0.7 });
|
||||
const counts = [100, 500, 1000];
|
||||
|
||||
for (const count of counts) {
|
||||
const items = generateData(count);
|
||||
const start = performance.now();
|
||||
sorter(items, { addWhy() {} });
|
||||
const end = performance.now();
|
||||
console.log(`Count: ${count}, Time: ${(end - start).toFixed(2)}ms`);
|
||||
}
|
||||
}
|
||||
|
||||
runBenchmark();
|
||||
Reference in new issue
Block a user