mirror of
https://github.com/PR0M3TH3AN/bitvid.git
synced 2026-10-27 07:52:24 +00:00
perf: throttle trusted mute pruning
Co-authored-by: PR0M3TH3AN <[email protected]>
This commit is contained in:
1 parent
d569e1474e
commit
f7ebad4b2f
6 files changed
+9451
No files matched your search
@@ -0,0 +1,74 @@
|
||||
import { ModerationService } from '../js/services/moderationService.js';
|
||||
import { performance } from 'perf_hooks';
|
||||
|
||||
// Mock dependencies
|
||||
const mockClient = {
|
||||
pool: {},
|
||||
};
|
||||
const mockLogger = {
|
||||
log: () => {},
|
||||
dev: { log: () => {} },
|
||||
user: { warn: () => {}, info: () => {} }
|
||||
};
|
||||
|
||||
const service = new ModerationService({
|
||||
nostrClient: mockClient,
|
||||
logger: mockLogger,
|
||||
});
|
||||
|
||||
async function runBenchmark() {
|
||||
console.log('Starting Moderation Service Stress Benchmark...');
|
||||
|
||||
// Setup Data
|
||||
// 100 authors, each having 500 muters (spam scenario).
|
||||
// 50 muters are expired.
|
||||
|
||||
const AUTHOR_COUNT = 100;
|
||||
const MUTERS_PER_AUTHOR = 500;
|
||||
const now = Date.now() / 1000;
|
||||
const expiredTime = now - (61 * 24 * 60 * 60);
|
||||
|
||||
console.log(`Populating ${AUTHOR_COUNT} authors with ${MUTERS_PER_AUTHOR} muters each...`);
|
||||
|
||||
for (let i = 0; i < AUTHOR_COUNT; i++) {
|
||||
const author = `author-${i}`.padEnd(64, '0');
|
||||
const muters = new Map();
|
||||
|
||||
for (let j = 0; j < MUTERS_PER_AUTHOR; j++) {
|
||||
const muter = `muter-${j}`.padEnd(64, '0');
|
||||
// 10% expired
|
||||
const timestamp = (j % 10 === 0) ? expiredTime : now;
|
||||
muters.set(muter, timestamp);
|
||||
}
|
||||
|
||||
service.trustedMutedAuthors.set(author, {
|
||||
muters,
|
||||
categories: new Map(),
|
||||
count: muters.size
|
||||
});
|
||||
}
|
||||
|
||||
// Benchmark Read
|
||||
console.log('Benchmarking getActiveTrustedMutersForAuthor (Read + Prune)...');
|
||||
const READ_ITERATIONS = 10000; // 10k reads
|
||||
|
||||
const start = performance.now();
|
||||
|
||||
for (let i = 0; i < READ_ITERATIONS; i++) {
|
||||
const authorIndex = i % AUTHOR_COUNT;
|
||||
const author = `author-${authorIndex}`.padEnd(64, '0');
|
||||
// This call triggers pruneTrustedMuteAggregates EVERY time
|
||||
service.getActiveTrustedMutersForAuthor(author);
|
||||
}
|
||||
|
||||
const end = performance.now();
|
||||
const totalTime = end - start;
|
||||
const avgTime = totalTime / READ_ITERATIONS;
|
||||
|
||||
console.log('---------------------------------------------------');
|
||||
console.log(`Total time: ${totalTime.toFixed(2)}ms`);
|
||||
console.log(`Average time per call: ${(avgTime * 1000).toFixed(4)}µs`);
|
||||
console.log('---------------------------------------------------');
|
||||
}
|
||||
|
||||
runBenchmark().catch(console.error);
|
||||
@@ -0,0 +1,150 @@
|
||||
import { createModerationStage } from '../js/feedEngine/stages.js';
|
||||
import { performance } from 'perf_hooks';
|
||||
|
||||
// Mock Logger
|
||||
const logger = {
|
||||
log: () => {},
|
||||
dev: { log: () => {}, debug: () => {} },
|
||||
user: { warn: () => {}, info: () => {} }
|
||||
};
|
||||
|
||||
// Realistic Mock Service
|
||||
const mutedAuthors = new Set();
|
||||
const reportedVideos = new Map();
|
||||
|
||||
// Seed data
|
||||
for (let i = 0; i < 100; i++) {
|
||||
if (Math.random() < 0.1) {
|
||||
mutedAuthors.add(`pubkey-${i}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Simulate pruning cost
|
||||
function pruneSimulation() {
|
||||
const map = new Map();
|
||||
for(let i=0; i<5; i++) map.set(i, Date.now()); // Simulate 5 muters
|
||||
for(const [k,v] of map) {
|
||||
if (v < 0) map.delete(k);
|
||||
}
|
||||
}
|
||||
|
||||
const mockService = {
|
||||
refreshViewerFromClient: async () => {},
|
||||
setActiveEventIds: async () => {},
|
||||
getAdminListSnapshot: () => ({
|
||||
whitelist: new Set(),
|
||||
whitelistHex: new Set(),
|
||||
blacklist: new Set(),
|
||||
blacklistHex: new Set()
|
||||
}),
|
||||
getAccessControlStatus: (pubkey) => ({
|
||||
hex: pubkey,
|
||||
whitelisted: false,
|
||||
blacklisted: false
|
||||
}),
|
||||
getTrustedReportSummary: (id) => {
|
||||
if (reportedVideos.has(id)) {
|
||||
return reportedVideos.get(id);
|
||||
}
|
||||
return null;
|
||||
},
|
||||
trustedReportCount: (id) => {
|
||||
const summary = reportedVideos.get(id);
|
||||
return summary ? summary.totalTrusted : 0;
|
||||
},
|
||||
getTrustedReporters: (id) => [],
|
||||
isAuthorMutedByTrusted: (pubkey) => {
|
||||
// Simulate the real service logic which calls prune
|
||||
pruneSimulation();
|
||||
return mutedAuthors.has(pubkey);
|
||||
},
|
||||
getTrustedMutersForAuthor: (pubkey) => {
|
||||
// Simulate prune here too if called
|
||||
pruneSimulation();
|
||||
if (mutedAuthors.has(pubkey)) {
|
||||
return ['trusted-muter-1', 'trusted-muter-2'];
|
||||
}
|
||||
return [];
|
||||
},
|
||||
getTrustedMuteCountsForAuthor: (pubkey) => {
|
||||
pruneSimulation();
|
||||
if (mutedAuthors.has(pubkey)) {
|
||||
return { total: 2, categories: { nudity: 1, spam: 1 } };
|
||||
}
|
||||
return { total: 0, categories: {} };
|
||||
},
|
||||
isAuthorMutedByViewer: (pubkey) => false,
|
||||
};
|
||||
|
||||
async function runBenchmark() {
|
||||
console.log('Starting Moderation Stage Benchmark (with prune cost)...');
|
||||
|
||||
const ITEM_COUNT = 10000;
|
||||
const items = [];
|
||||
|
||||
for (let i = 0; i < ITEM_COUNT; i++) {
|
||||
const videoId = `video-${i}`;
|
||||
const pubkey = `pubkey-${i % 100}`;
|
||||
|
||||
// Simulate some reported videos
|
||||
if (Math.random() < 0.05) {
|
||||
reportedVideos.set(videoId, {
|
||||
eventId: videoId,
|
||||
totalTrusted: 2,
|
||||
types: { nudity: { trusted: 2, total: 5, latest: Date.now() } },
|
||||
updatedAt: Date.now()
|
||||
});
|
||||
}
|
||||
|
||||
items.push({
|
||||
video: {
|
||||
id: videoId,
|
||||
pubkey: pubkey,
|
||||
kind: 30078,
|
||||
tags: []
|
||||
},
|
||||
metadata: {}
|
||||
});
|
||||
}
|
||||
|
||||
const context = {
|
||||
runtime: {
|
||||
moderationThresholds: {
|
||||
autoplayBlockThreshold: 1,
|
||||
blurThreshold: 1,
|
||||
trustedSpamHideThreshold: 1,
|
||||
trustedMuteHideThresholds: {}
|
||||
},
|
||||
isAuthorBlocked: () => false
|
||||
},
|
||||
log: logger.dev.log,
|
||||
addWhy: () => {}
|
||||
};
|
||||
|
||||
const moderationStage = createModerationStage({
|
||||
service: mockService,
|
||||
stageName: 'bench-moderation'
|
||||
});
|
||||
|
||||
// Warmup
|
||||
console.log('Warming up...');
|
||||
await moderationStage(items.slice(0, 100), context);
|
||||
|
||||
// Run
|
||||
console.log(`Processing ${ITEM_COUNT} items with ~10% muted authors and prune simulation...`);
|
||||
const start = performance.now();
|
||||
|
||||
const result = await moderationStage(items, context);
|
||||
|
||||
const end = performance.now();
|
||||
const totalTime = end - start;
|
||||
const avgTime = totalTime / ITEM_COUNT;
|
||||
|
||||
console.log('---------------------------------------------------');
|
||||
console.log(`Total time: ${totalTime.toFixed(2)}ms`);
|
||||
console.log(`Average time per item: ${(avgTime * 1000).toFixed(4)}µs`);
|
||||
console.log(`Items processed: ${result.length}`);
|
||||
console.log('---------------------------------------------------');
|
||||
}
|
||||
|
||||
runBenchmark().catch(console.error);
|
||||
@@ -0,0 +1,28 @@
|
||||
# Weekly Perf Deep Dive Agent - Completed
|
||||
|
||||
**Agent:** perf-deepdive-agent
|
||||
**Date:** 2026-02-21
|
||||
**Cadence:** Weekly
|
||||
|
||||
## Summary
|
||||
Executed a performance deep dive focusing on the Moderation Service.
|
||||
|
||||
### Tasks
|
||||
1. **Exploration:** Identified `moderationStage` and `moderationService` as potential hotspots due to repeated lookups in the feed loop.
|
||||
2. **Benchmarking:** Created `benchmarks/moderation_stage_bench.js` and `benchmarks/moderation_service_bench.js`.
|
||||
- Baseline for service lookups: ~39ms for 10k reads.
|
||||
3. **Optimization:** Identified that `getActiveTrustedMutersForAuthor` was performing an O(N) prune operation on every read.
|
||||
- Implemented "Lazy Pruning" in `js/services/moderationService.js`, throttling the prune to run at most once per minute per author.
|
||||
4. **Verification:**
|
||||
- Re-ran benchmarks: Latency reduced to ~22ms (~43% improvement).
|
||||
- Verified logic with `tests/moderation/*.test.mjs`.
|
||||
- Confirmed file size compliance (after minor adjustment).
|
||||
|
||||
## Artifacts
|
||||
- `weekly-perf-report-2026-02-21.md`
|
||||
- `benchmarks/moderation_stage_bench.js`
|
||||
- `benchmarks/moderation_service_bench.js`
|
||||
- `perf/hits-2026-02-21.json`
|
||||
|
||||
## Next Steps
|
||||
- Consider further optimizing `moderationStage` by batching author lookups.
|
||||
@@ -893,6 +893,11 @@ export class ModerationService {
|
||||
if (!aggregate || !(aggregate.muters instanceof Map)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Throttle pruning: only run once per minute per author
|
||||
if (aggregate.lastPrunedAt && nowSeconds - aggregate.lastPrunedAt < 60) continue;
|
||||
aggregate.lastPrunedAt = nowSeconds;
|
||||
|
||||
const expiredOwners = new Set();
|
||||
for (const [owner, updatedAt] of aggregate.muters.entries()) {
|
||||
if (ensureNumber(updatedAt) < cutoff) {
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,36 @@
|
||||
# Weekly Performance Report — 2026-02-21
|
||||
|
||||
## Summary
|
||||
- **Scenario:** Moderation Service lookups (Trusted Mutes).
|
||||
- **Optimization:** Throttled `pruneTrustedMuteAggregates` to run at most once per minute per author.
|
||||
- **Impact:** ~43% reduction in lookup latency for authors with many muters.
|
||||
- **Risk:** Low. Correctness is maintained; expired mutes might persist in memory for up to 1 extra minute, which is acceptable for a 60-day window.
|
||||
|
||||
## Methodology
|
||||
1. **Scenario Selection:** `moderationStage` in the feed engine calls `moderationService.isAuthorMutedByTrusted` for every video. Profiling revealed that `getActiveTrustedMutersForAuthor` was eagerly pruning expired mutes on *every read*.
|
||||
2. **Baseline Measurement:**
|
||||
- Script: `benchmarks/moderation_service_bench.js`
|
||||
- Setup: 100 authors, 500 muters each (10% expired).
|
||||
- 10,000 read operations.
|
||||
- **Baseline:** ~39.13ms total (3.91µs/op).
|
||||
3. **Optimization:**
|
||||
- Modified `js/services/moderationService.js` to skip pruning if `lastPrunedAt` was less than 60 seconds ago.
|
||||
4. **Verification:**
|
||||
- Re-ran `benchmarks/moderation_service_bench.js`.
|
||||
- **After:** ~22.16ms total (2.21µs/op).
|
||||
- **Improvement:** 43% faster.
|
||||
- Ran `tests/moderation/` to ensure no regressions.
|
||||
|
||||
## PRs & Issues
|
||||
- **PR:** (This commit) `perf: throttle trusted mute pruning`
|
||||
- **Follow-ups:**
|
||||
- Consider moving pruning entirely to a background interval if the mute list grows significantly.
|
||||
- Explore caching `isAuthorMutedByTrusted` results for a few seconds in `moderationStage`.
|
||||
|
||||
## Commands Run
|
||||
```bash
|
||||
node benchmarks/moderation_stage_bench.js # Feed stage macro-benchmark
|
||||
node benchmarks/moderation_service_bench.js # Service micro-benchmark
|
||||
npm run test:unit:shard1 # General unit tests
|
||||
node scripts/run-targeted-tests.mjs tests/moderation/*.test.mjs # Moderation tests
|
||||
```
|
||||
Reference in new issue
Block a user