From a00082b0ef21942ebc041ea22c3de37c39b4eac8 Mon Sep 17 00:00:00 2001 From: Trevor Mears Date: Sun, 4 Jan 2026 00:46:51 -0800 Subject: [PATCH] Implemented virutal scrolling for epg, optimized database batching, moved sync timer to server --- public/index.html | 4 + public/js/components/EpgGuide.js | 240 ++++++++++++++++--------------- public/js/pages/Settings.js | 61 +++++++- server/index.js | 6 +- server/routes/settings.js | 19 +++ server/routes/sources.js | 13 ++ server/services/syncService.js | 81 ++++++++++- 7 files changed, 293 insertions(+), 131 deletions(-) diff --git a/public/index.html b/public/index.html index cdec2fc..10c9307 100644 --- a/public/index.html +++ b/public/index.html @@ -328,6 +328,10 @@ +
+ Last Refreshed: + Never +

Use the this.selectedGroup = 'Favorites'; // Default to Favorites - // Lazy loading properties + // Virtual scrolling properties this.filteredChannels = []; - this.currentBatch = 0; - this.batchSize = 20; // Channels per batch + this.rowHeight = 60; // Height of each channel row in pixels + this.bufferRows = 5; // Extra rows to render above/below viewport this.startTime = null; this.endTime = null; this.epgContainer = null; - this.epgLoader = null; - this.epgObserver = null; + this.epgSpacer = null; + this.scrollContainer = null; + this.visibleRows = new Map(); // Map + this._scrollHandler = null; + this._lastVisibleStart = -1; + this._lastVisibleEnd = -1; this.init(); } @@ -117,43 +121,23 @@ class EpgGuide { } /** - * Get EPG refresh interval from settings (in hours) - */ - getRefreshInterval() { - // Read from server-side player settings (synced via Settings page) - if (window.app?.player?.settings?.epgRefreshInterval) { - return parseFloat(window.app.player.settings.epgRefreshInterval); - } - return 24; // Default 24 hours - } - - /** - * Start background EPG refresh timer - * Automatically fetches fresh EPG data at the configured interval + * Start background EPG display refresh timer + * This only refreshes the UI from cached server data. + * The actual sync runs on the server independently. */ startBackgroundRefresh() { // Clear any existing timer this.stopBackgroundRefresh(); - const intervalHours = this.getRefreshInterval(); + // Refresh display from cache every 5 minutes to pick up server-side sync results + const refreshIntervalMs = 5 * 60 * 1000; // 5 minutes - // If interval is 0 or invalid, don't start timer (manual refresh only) - if (!intervalHours || intervalHours <= 0) { - console.log('[EPG] Background refresh disabled (manual only mode)'); - this._currentRefreshInterval = 0; - return; - } - - const intervalMs = intervalHours * 60 * 60 * 1000; // Convert hours to milliseconds - - console.log(`[EPG] Starting background refresh timer: every ${intervalHours} hours (${Math.round(intervalMs / 1000)}s)`); + console.log('[EPG] Starting display refresh timer: every 5 minutes'); this._backgroundRefreshTimer = setInterval(async () => { - console.log('[EPG] Background refresh triggered'); + console.log('[EPG] Refreshing EPG display from cache'); try { - await this.fetchEpgData(true); // Force refresh - this.lastRefreshTime = new Date(); - console.log('[EPG] Background refresh complete'); + await this.fetchEpgData(false); // Fetch cached data (no force refresh) // Update channel list program info if visible if (window.app?.channelList) { @@ -161,12 +145,9 @@ class EpgGuide { window.app.channelList.updateVisibleEpgInfo?.(); } } catch (err) { - console.error('[EPG] Background refresh failed:', err); + console.error('[EPG] Display refresh failed:', err); } - }, intervalMs); - - // Store current interval so we can detect changes - this._currentRefreshInterval = intervalHours; + }, refreshIntervalMs); } /** @@ -180,18 +161,6 @@ class EpgGuide { } } - /** - * Restart background refresh if interval has changed - * Called when settings change - */ - restartBackgroundRefreshIfNeeded() { - const newInterval = this.getRefreshInterval(); - if (this._currentRefreshInterval !== newInterval) { - console.log(`[EPG] Refresh interval changed: ${this._currentRefreshInterval}h -> ${newInterval}h`); - this.startBackgroundRefresh(); - } - } - /** * Get last refresh time for display */ @@ -456,7 +425,10 @@ class EpgGuide { // Generate time slots const timeSlots = this.generateTimeSlots(this.startTime, this.endTime); - // Build initial HTML structure + // Calculate total height for virtual scrolling + const totalHeight = this.filteredChannels.length * this.rowHeight; + + // Build HTML structure with virtual scroll container this.container.innerHTML = `

@@ -466,80 +438,125 @@ class EpgGuide {
`).join('')}
-
-
- +
+
+
+
`; - // Get references for batch rendering + // Get references for virtual scrolling + this.scrollContainer = this.container.querySelector('.epg-scroll-container'); + this.epgSpacer = this.container.querySelector('.epg-spacer'); this.epgContainer = this.container.querySelector('.epg-channel-rows'); - this.epgLoader = this.container.querySelector('.epg-loader'); - // Reset batch state - this.currentBatch = 0; + // Clear visible rows cache + this.visibleRows.clear(); + this._lastVisibleStart = -1; + this._lastVisibleEnd = -1; - // Set up IntersectionObserver for lazy loading - if (this.epgObserver) { - this.epgObserver.disconnect(); - } - this.epgObserver = new IntersectionObserver((entries) => { - if (entries[0].isIntersecting) { - this.renderNextEpgBatch(); - } - }, { rootMargin: '200px' }); - - // Render initial batches (render enough to fill viewport) - for (let i = 0; i < 3; i++) { - this.renderNextEpgBatch(); + // Remove old scroll handler if exists + if (this._scrollHandler) { + this.scrollContainer?.removeEventListener('scroll', this._scrollHandler); } - // Start observing loader - this.epgObserver.observe(this.epgLoader); + // Set up scroll handler for virtual scrolling + this._scrollHandler = this.debounce(() => this.updateVisibleRows(), 16); // ~60fps + this.scrollContainer.addEventListener('scroll', this._scrollHandler); + + // Initial render of visible rows + this.updateVisibleRows(); // Add now indicator this.updateNowIndicator(); } /** - * Render next batch of EPG channel rows + * Update visible rows based on scroll position (Virtual Scrolling) */ - renderNextEpgBatch() { - const start = this.currentBatch * this.batchSize; - const end = start + this.batchSize; - const batch = this.filteredChannels.slice(start, end); + updateVisibleRows() { + if (!this.scrollContainer || !this.epgContainer) return; - if (batch.length === 0) { - this.epgLoader.style.display = 'none'; + const scrollTop = this.scrollContainer.scrollTop; + const viewportHeight = this.scrollContainer.clientHeight; + + // Calculate visible range + const startIndex = Math.max(0, Math.floor(scrollTop / this.rowHeight) - this.bufferRows); + const endIndex = Math.min( + this.filteredChannels.length - 1, + Math.ceil((scrollTop + viewportHeight) / this.rowHeight) + this.bufferRows + ); + + // Skip if nothing changed + if (startIndex === this._lastVisibleStart && endIndex === this._lastVisibleEnd) { return; } - let html = ''; - for (const { epgChannel, sourceChannel } of batch) { - const isFavorite = this.favorites.has(`${sourceChannel.sourceId}:${sourceChannel.id}`); + this._lastVisibleStart = startIndex; + this._lastVisibleEnd = endIndex; - // Get programs if EPG data exists - let channelProgrammes = []; - if (epgChannel) { - channelProgrammes = this.programmes - .filter(p => p.channelId === epgChannel.id) - .filter(p => { - const start = new Date(p.start); - const stop = new Date(p.stop); - return start < this.endTime && stop > this.startTime; - }) - .sort((a, b) => new Date(a.start) - new Date(b.start)); + // Determine which rows to add and remove + const newVisibleSet = new Set(); + for (let i = startIndex; i <= endIndex; i++) { + newVisibleSet.add(i); + } + + // Remove rows that are no longer visible + for (const [index, row] of this.visibleRows) { + if (!newVisibleSet.has(index)) { + row.remove(); + this.visibleRows.delete(index); } + } - // Fallback values if EPG channel is missing - const logo = this.getProxiedImageUrl(sourceChannel.tvgLogo || (epgChannel && epgChannel.icon)); - const name = sourceChannel.name || (epgChannel && epgChannel.name); + // Add new visible rows + for (let i = startIndex; i <= endIndex; i++) { + if (!this.visibleRows.has(i) && i < this.filteredChannels.length) { + const row = this.createChannelRow(i); + this.visibleRows.set(i, row); + this.epgContainer.appendChild(row); + } + } + } - html += ` -
+ /** + * Create a channel row element for virtual scrolling + */ + createChannelRow(index) { + const { epgChannel, sourceChannel } = this.filteredChannels[index]; + const isFavorite = this.favorites.has(`${sourceChannel.sourceId}:${sourceChannel.id}`); + + // Get programs if EPG data exists + let channelProgrammes = []; + if (epgChannel) { + channelProgrammes = this.programmes + .filter(p => p.channelId === epgChannel.id) + .filter(p => { + const start = new Date(p.start); + const stop = new Date(p.stop); + return start < this.endTime && stop > this.startTime; + }) + .sort((a, b) => new Date(a.start) - new Date(b.start)); + } + + // Fallback values if EPG channel is missing + const logo = this.getProxiedImageUrl(sourceChannel.tvgLogo || (epgChannel && epgChannel.icon)); + const name = sourceChannel.name || (epgChannel && epgChannel.name); + + const row = document.createElement('div'); + row.className = 'epg-channel-row'; + row.dataset.channelId = sourceChannel.id; + row.dataset.sourceId = sourceChannel.sourceId; + row.dataset.index = index; + // Position absolutely for virtual scrolling + row.style.position = 'absolute'; + row.style.top = `${index * this.rowHeight}px`; + row.style.left = '0'; + row.style.right = '0'; + row.style.height = `${this.rowHeight}px`; + + row.innerHTML = `
- `; - } + `; - // Append to container - const tempDiv = document.createElement('div'); - tempDiv.innerHTML = html; - while (tempDiv.firstElementChild) { - const row = tempDiv.firstElementChild; - this.attachRowListeners(row); - this.epgContainer.appendChild(row); - } - - this.currentBatch++; - - // Hide loader if no more batches - if (end >= this.filteredChannels.length) { - this.epgLoader.style.display = 'none'; - } + this.attachRowListeners(row); + return row; } /** diff --git a/public/js/pages/Settings.js b/public/js/pages/Settings.js index 6c99b3b..c567489 100644 --- a/public/js/pages/Settings.js +++ b/public/js/pages/Settings.js @@ -112,18 +112,17 @@ class SettingsPage { // Load saved value from player settings epgRefreshSelect.value = this.app.player.settings.epgRefreshInterval || '24'; - // Save on change and restart background timer + // Save on change - server will restart its sync timer via PUT /api/settings epgRefreshSelect.addEventListener('change', () => { this.app.player.settings.epgRefreshInterval = epgRefreshSelect.value; this.app.player.saveSettings(); - - // Restart EPG background refresh with new interval - if (window.app?.epgGuide) { - window.app.epgGuide.restartBackgroundRefreshIfNeeded(); - } + // Server-side sync timer is restarted automatically via settings API }); } + // Update last refreshed display + this.updateEpgLastRefreshed(); + // Stream output format const streamFormatSelect = document.getElementById('setting-stream-format'); if (streamFormatSelect && this.app.player?.settings) { @@ -287,6 +286,56 @@ class SettingsPage { if (epgRefreshSelect) epgRefreshSelect.value = s.epgRefreshInterval || '24'; if (streamFormatSelect) streamFormatSelect.value = s.streamFormat || 'm3u8'; } + + // Update EPG last refreshed display + this.updateEpgLastRefreshed(); + } + + /** + * Update the EPG last refreshed display + */ + async updateEpgLastRefreshed() { + const display = document.getElementById('epg-last-refreshed'); + if (!display) return; + + try { + // Fetch last sync time from server + const response = await fetch('/api/settings/sync-status'); + if (!response.ok) throw new Error('Failed to fetch sync status'); + const data = await response.json(); + + if (data.lastSyncTime) { + const lastRefreshTime = new Date(data.lastSyncTime); + + // Format as relative time or absolute + const now = new Date(); + const diffMs = now - lastRefreshTime; + const diffMins = Math.floor(diffMs / 60000); + const diffHours = Math.floor(diffMins / 60); + + let text; + if (diffMins < 1) { + text = 'Just now'; + } else if (diffMins < 60) { + text = `${diffMins} minute${diffMins === 1 ? '' : 's'} ago`; + } else if (diffHours < 24) { + text = `${diffHours} hour${diffHours === 1 ? '' : 's'} ago`; + } else { + // Use absolute time for older refreshes + text = lastRefreshTime.toLocaleString(); + } + + display.textContent = text; + display.title = lastRefreshTime.toLocaleString(); // Full timestamp on hover + } else { + display.textContent = 'Never'; + display.title = 'Sync has not run yet since server started'; + } + } catch (err) { + console.error('Error fetching sync status:', err); + display.textContent = 'Unknown'; + display.title = 'Could not fetch sync status'; + } } hide() { diff --git a/server/index.js b/server/index.js index 989d22f..3adc594 100644 --- a/server/index.js +++ b/server/index.js @@ -79,7 +79,9 @@ app.listen(PORT, async () => { console.log(`NodeCast TV server running on http://localhost:${PORT}`); // Trigger background sync with delay to allow server to settle - setTimeout(() => { - syncService.syncAll().catch(console.error); + setTimeout(async () => { + await syncService.syncAll().catch(console.error); + // Start the server-side sync timer after initial sync + await syncService.startSyncTimer().catch(console.error); }, 5000); }); diff --git a/server/routes/settings.js b/server/routes/settings.js index 607193a..16937db 100644 --- a/server/routes/settings.js +++ b/server/routes/settings.js @@ -1,6 +1,7 @@ const express = require('express'); const router = express.Router(); const { settings, getDefaultSettings } = require('../db'); +const syncService = require('../services/syncService'); /** * Get all settings @@ -24,6 +25,12 @@ router.put('/', async (req, res) => { try { const updates = req.body; const updatedSettings = await settings.update(updates); + + // If sync interval changed, restart the server-side sync timer + if (updates.epgRefreshInterval !== undefined) { + syncService.restartSyncTimer().catch(console.error); + } + res.json(updatedSettings); } catch (err) { console.error('Error updating settings:', err); @@ -53,4 +60,16 @@ router.get('/defaults', (req, res) => { res.json(getDefaultSettings()); }); +/** + * Get sync status (last sync time) + * GET /api/settings/sync-status + */ +router.get('/sync-status', (req, res) => { + const lastSyncTime = syncService.getLastSyncTime(); + res.json({ + lastSyncTime: lastSyncTime ? lastSyncTime.toISOString() : null + }); +}); + module.exports = router; + diff --git a/server/routes/sources.js b/server/routes/sources.js index caff4fe..72c1352 100644 --- a/server/routes/sources.js +++ b/server/routes/sources.js @@ -185,4 +185,17 @@ router.post('/:id/test', async (req, res) => { } }); +// Global Sync - sync all enabled sources +router.post('/sync-all', async (req, res) => { + try { + // Trigger global sync (async - don't wait for completion) + syncService.syncAll().catch(console.error); + res.json({ success: true, message: 'Global sync started' }); + } catch (err) { + console.error('Error starting global sync:', err); + res.status(500).json({ error: 'Failed to start global sync' }); + } +}); + module.exports = router; + diff --git a/server/services/syncService.js b/server/services/syncService.js index 2956fe8..1c04ac5 100644 --- a/server/services/syncService.js +++ b/server/services/syncService.js @@ -1,5 +1,5 @@ const { getDb } = require('../db/sqlite'); -const { sources } = require('../db'); // For source config +const { sources, settings } = require('../db'); // For source config and settings const xtreamApi = require('./xtreamApi'); const m3uParser = require('./m3uParser'); const epgParser = require('./epgParser'); @@ -8,6 +8,74 @@ const epgParser = require('./epgParser'); const activeSyncs = new Set(); // sourceId class SyncService { + constructor() { + this.lastSyncTime = null; // Track when global sync last completed + this._syncTimer = null; // Server-side sync timer + this._currentInterval = null; + } + + /** + * Get when the last global sync completed + */ + getLastSyncTime() { + return this.lastSyncTime; + } + + /** + * Start the server-side sync timer based on settings + * Should be called once on server startup after initial sync + */ + async startSyncTimer() { + // Get interval from settings + const currentSettings = await settings.get(); + const intervalHours = parseInt(currentSettings.epgRefreshInterval) || 24; + + // If interval is 0, don't start timer (manual only mode) + if (intervalHours <= 0) { + console.log('[Sync] Auto-sync disabled (manual only mode)'); + this.stopSyncTimer(); + this._currentInterval = 0; + return; + } + + const intervalMs = intervalHours * 60 * 60 * 1000; + + // Don't restart if interval hasn't changed + if (this._currentInterval === intervalHours && this._syncTimer) { + return; + } + + // Clear existing timer + this.stopSyncTimer(); + + console.log(`[Sync] Starting server-side sync timer: every ${intervalHours} hours`); + + this._syncTimer = setInterval(async () => { + console.log('[Sync] Scheduled sync triggered'); + await this.syncAll(); + }, intervalMs); + + this._currentInterval = intervalHours; + } + + /** + * Stop the server-side sync timer + */ + stopSyncTimer() { + if (this._syncTimer) { + clearInterval(this._syncTimer); + this._syncTimer = null; + } + } + + /** + * Restart the sync timer with updated settings + * Called when sync interval setting changes + */ + async restartSyncTimer() { + await this.startSyncTimer(); + } + /** * Sync all enabled sources */ @@ -21,7 +89,8 @@ class SyncService { await this.syncSource(source.id); } } - console.log('[Sync] Global sync completed'); + this.lastSyncTime = new Date(); + console.log('[Sync] Global sync completed at', this.lastSyncTime.toISOString()); } catch (err) { console.error('[Sync] Global sync failed:', err); } @@ -164,9 +233,11 @@ class SyncService { } }); - const BATCH_SIZE = 500; + // Reduced batch size for better event loop interleaving + const BATCH_SIZE = 100; for (let i = 0; i < categories.length; i += BATCH_SIZE) { insertBatch(categories.slice(i, i + BATCH_SIZE)); + // Yield to event loop between batches to allow other requests await new Promise(resolve => setImmediate(resolve)); } @@ -244,9 +315,11 @@ class SyncService { } }); - const BATCH_SIZE = 500; + // Reduced batch size for better event loop interleaving + const BATCH_SIZE = 100; for (let i = 0; i < items.length; i += BATCH_SIZE) { insertBatch(items.slice(i, i + BATCH_SIZE)); + // Yield to event loop between batches to allow other requests await new Promise(resolve => setImmediate(resolve)); }