Implemented virutal scrolling for epg, optimized database batching, moved sync timer to server

public/index.html

@@ -328,6 +328,10 @@
                   <option value="0">Manual only</option>
                 </select>
               </div>
+              <div class="setting-item">
+                <span class="setting-label">Last Refreshed:</span>
+                <span id="epg-last-refreshed" class="hint">Never</span>
+              </div>
               <p class="hint">Use the <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="currentColor"
                   class="icon" style="width: 1em; height: 1em; vertical-align: middle;">
                   <path

public/js/components/EpgGuide.js

@@ -21,15 +21,19 @@ class EpgGuide {
         this.favorites = new Set(); // Set<"sourceId:channelId">
         this.selectedGroup = 'Favorites'; // Default to Favorites
 
-        // Lazy loading properties
+        // Virtual scrolling properties
         this.filteredChannels = [];
-        this.currentBatch = 0;
+        this.rowHeight = 60; // Height of each channel row in pixels
-        this.batchSize = 20; // Channels per batch
+        this.bufferRows = 5; // Extra rows to render above/below viewport
         this.startTime = null;
         this.endTime = null;
         this.epgContainer = null;
-        this.epgLoader = null;
+        this.epgSpacer = null;
-        this.epgObserver = null;
+        this.scrollContainer = null;
+        this.visibleRows = new Map(); // Map<index, rowElement>
+        this._scrollHandler = null;
+        this._lastVisibleStart = -1;
+        this._lastVisibleEnd = -1;
 
         this.init();
     }
@@ -117,43 +121,23 @@ class EpgGuide {
     }
 
     /**
-     * Get EPG refresh interval from settings (in hours)
+     * Start background EPG display refresh timer
-     */
+     * This only refreshes the UI from cached server data.
-    getRefreshInterval() {
+     * The actual sync runs on the server independently.
-        // Read from server-side player settings (synced via Settings page)
-        if (window.app?.player?.settings?.epgRefreshInterval) {
-            return parseFloat(window.app.player.settings.epgRefreshInterval);
-        }
-        return 24; // Default 24 hours
-    }
-
-    /**
-     * Start background EPG refresh timer
-     * Automatically fetches fresh EPG data at the configured interval
      */
     startBackgroundRefresh() {
         // Clear any existing timer
         this.stopBackgroundRefresh();
 
-        const intervalHours = this.getRefreshInterval();
+        // Refresh display from cache every 5 minutes to pick up server-side sync results
+        const refreshIntervalMs = 5 * 60 * 1000; // 5 minutes
 
-        // If interval is 0 or invalid, don't start timer (manual refresh only)
+        console.log('[EPG] Starting display refresh timer: every 5 minutes');
-        if (!intervalHours || intervalHours <= 0) {
-            console.log('[EPG] Background refresh disabled (manual only mode)');
-            this._currentRefreshInterval = 0;
-            return;
-        }
-
-        const intervalMs = intervalHours * 60 * 60 * 1000; // Convert hours to milliseconds
-
-        console.log(`[EPG] Starting background refresh timer: every ${intervalHours} hours (${Math.round(intervalMs / 1000)}s)`);
 
         this._backgroundRefreshTimer = setInterval(async () => {
-            console.log('[EPG] Background refresh triggered');
+            console.log('[EPG] Refreshing EPG display from cache');
             try {
-                await this.fetchEpgData(true); // Force refresh
+                await this.fetchEpgData(false); // Fetch cached data (no force refresh)
-                this.lastRefreshTime = new Date();
-                console.log('[EPG] Background refresh complete');
 
                 // Update channel list program info if visible
                 if (window.app?.channelList) {
@@ -161,12 +145,9 @@ class EpgGuide {
                     window.app.channelList.updateVisibleEpgInfo?.();
                 }
             } catch (err) {
-                console.error('[EPG] Background refresh failed:', err);
+                console.error('[EPG] Display refresh failed:', err);
             }
-        }, intervalMs);
+        }, refreshIntervalMs);
-
-        // Store current interval so we can detect changes
-        this._currentRefreshInterval = intervalHours;
     }
 
     /**
@@ -180,18 +161,6 @@ class EpgGuide {
         }
     }
 
-    /**
-     * Restart background refresh if interval has changed
-     * Called when settings change
-     */
-    restartBackgroundRefreshIfNeeded() {
-        const newInterval = this.getRefreshInterval();
-        if (this._currentRefreshInterval !== newInterval) {
-            console.log(`[EPG] Refresh interval changed: ${this._currentRefreshInterval}h -> ${newInterval}h`);
-            this.startBackgroundRefresh();
-        }
-    }
-
     /**
      * Get last refresh time for display
      */
@@ -456,7 +425,10 @@ class EpgGuide {
         // Generate time slots
         const timeSlots = this.generateTimeSlots(this.startTime, this.endTime);
 
-        // Build initial HTML structure
+        // Calculate total height for virtual scrolling
+        const totalHeight = this.filteredChannels.length * this.rowHeight;
+
+        // Build HTML structure with virtual scroll container
         this.container.innerHTML = `
       <div class="epg-container" style="position: relative;">
         <div class="epg-time-header">
@@ -466,57 +438,93 @@ class EpgGuide {
             </div>
           `).join('')}
         </div>
-        <div class="epg-channel-rows"></div>
+        <div class="epg-scroll-container" style="overflow-y: auto; max-height: calc(100vh - 200px);">
-        <div class="epg-loader" style="height: 50px; display: flex; align-items: center; justify-content: center;">
+          <div class="epg-spacer" style="height: ${totalHeight}px; position: relative;">
-          <div class="loading-spinner"></div>
+            <div class="epg-channel-rows" style="position: absolute; top: 0; left: 0; right: 0;"></div>
+          </div>
         </div>
       </div>
     `;
 
-        // Get references for batch rendering
+        // Get references for virtual scrolling
+        this.scrollContainer = this.container.querySelector('.epg-scroll-container');
+        this.epgSpacer = this.container.querySelector('.epg-spacer');
         this.epgContainer = this.container.querySelector('.epg-channel-rows');
-        this.epgLoader = this.container.querySelector('.epg-loader');
 
-        // Reset batch state
+        // Clear visible rows cache
-        this.currentBatch = 0;
+        this.visibleRows.clear();
+        this._lastVisibleStart = -1;
+        this._lastVisibleEnd = -1;
 
-        // Set up IntersectionObserver for lazy loading
+        // Remove old scroll handler if exists
-        if (this.epgObserver) {
+        if (this._scrollHandler) {
-            this.epgObserver.disconnect();
+            this.scrollContainer?.removeEventListener('scroll', this._scrollHandler);
-        }
-        this.epgObserver = new IntersectionObserver((entries) => {
-            if (entries[0].isIntersecting) {
-                this.renderNextEpgBatch();
-            }
-        }, { rootMargin: '200px' });
-
-        // Render initial batches (render enough to fill viewport)
-        for (let i = 0; i < 3; i++) {
-            this.renderNextEpgBatch();
         }
 
-        // Start observing loader
+        // Set up scroll handler for virtual scrolling
-        this.epgObserver.observe(this.epgLoader);
+        this._scrollHandler = this.debounce(() => this.updateVisibleRows(), 16); // ~60fps
+        this.scrollContainer.addEventListener('scroll', this._scrollHandler);
+
+        // Initial render of visible rows
+        this.updateVisibleRows();
 
         // Add now indicator
         this.updateNowIndicator();
     }
 
     /**
-     * Render next batch of EPG channel rows
+     * Update visible rows based on scroll position (Virtual Scrolling)
      */
-    renderNextEpgBatch() {
+    updateVisibleRows() {
-        const start = this.currentBatch * this.batchSize;
+        if (!this.scrollContainer || !this.epgContainer) return;
-        const end = start + this.batchSize;
-        const batch = this.filteredChannels.slice(start, end);
 
-        if (batch.length === 0) {
+        const scrollTop = this.scrollContainer.scrollTop;
-            this.epgLoader.style.display = 'none';
+        const viewportHeight = this.scrollContainer.clientHeight;
+
+        // Calculate visible range
+        const startIndex = Math.max(0, Math.floor(scrollTop / this.rowHeight) - this.bufferRows);
+        const endIndex = Math.min(
+            this.filteredChannels.length - 1,
+            Math.ceil((scrollTop + viewportHeight) / this.rowHeight) + this.bufferRows
+        );
+
+        // Skip if nothing changed
+        if (startIndex === this._lastVisibleStart && endIndex === this._lastVisibleEnd) {
             return;
         }
 
-        let html = '';
+        this._lastVisibleStart = startIndex;
-        for (const { epgChannel, sourceChannel } of batch) {
+        this._lastVisibleEnd = endIndex;
+
+        // Determine which rows to add and remove
+        const newVisibleSet = new Set();
+        for (let i = startIndex; i <= endIndex; i++) {
+            newVisibleSet.add(i);
+        }
+
+        // Remove rows that are no longer visible
+        for (const [index, row] of this.visibleRows) {
+            if (!newVisibleSet.has(index)) {
+                row.remove();
+                this.visibleRows.delete(index);
+            }
+        }
+
+        // Add new visible rows
+        for (let i = startIndex; i <= endIndex; i++) {
+            if (!this.visibleRows.has(i) && i < this.filteredChannels.length) {
+                const row = this.createChannelRow(i);
+                this.visibleRows.set(i, row);
+                this.epgContainer.appendChild(row);
+            }
+        }
+    }
+
+    /**
+     * Create a channel row element for virtual scrolling
+     */
+    createChannelRow(index) {
+        const { epgChannel, sourceChannel } = this.filteredChannels[index];
         const isFavorite = this.favorites.has(`${sourceChannel.sourceId}:${sourceChannel.id}`);
 
         // Get programs if EPG data exists
@@ -536,10 +544,19 @@ class EpgGuide {
         const logo = this.getProxiedImageUrl(sourceChannel.tvgLogo || (epgChannel && epgChannel.icon));
         const name = sourceChannel.name || (epgChannel && epgChannel.name);
 
-            html += `
+        const row = document.createElement('div');
-        <div class="epg-channel-row" 
+        row.className = 'epg-channel-row';
-             data-channel-id="${sourceChannel.id}" 
+        row.dataset.channelId = sourceChannel.id;
-             data-source-id="${sourceChannel.sourceId}">
+        row.dataset.sourceId = sourceChannel.sourceId;
+        row.dataset.index = index;
+        // Position absolutely for virtual scrolling
+        row.style.position = 'absolute';
+        row.style.top = `${index * this.rowHeight}px`;
+        row.style.left = '0';
+        row.style.right = '0';
+        row.style.height = `${this.rowHeight}px`;
+
+        row.innerHTML = `
           <div class="epg-channel-info">
             <button class="favorite-btn ${isFavorite ? 'active' : ''}" title="${isFavorite ? 'Remove from Favorites' : 'Add to Favorites'}">
               ${isFavorite ? Icons.favorite : Icons.favoriteOutline}
@@ -552,25 +569,10 @@ class EpgGuide {
           <div class="epg-programs">
             ${this.renderProgrammes(channelProgrammes, this.startTime, this.endTime)}
           </div>
-        </div>
         `;
-        }
 
-        // Append to container
-        const tempDiv = document.createElement('div');
-        tempDiv.innerHTML = html;
-        while (tempDiv.firstElementChild) {
-            const row = tempDiv.firstElementChild;
         this.attachRowListeners(row);
-            this.epgContainer.appendChild(row);
+        return row;
-        }
-
-        this.currentBatch++;
-
-        // Hide loader if no more batches
-        if (end >= this.filteredChannels.length) {
-            this.epgLoader.style.display = 'none';
-        }
     }
 
     /**

public/js/pages/Settings.js

@@ -112,18 +112,17 @@ class SettingsPage {
             // Load saved value from player settings
             epgRefreshSelect.value = this.app.player.settings.epgRefreshInterval || '24';
 
-            // Save on change and restart background timer
+            // Save on change - server will restart its sync timer via PUT /api/settings
             epgRefreshSelect.addEventListener('change', () => {
                 this.app.player.settings.epgRefreshInterval = epgRefreshSelect.value;
                 this.app.player.saveSettings();
-
+                // Server-side sync timer is restarted automatically via settings API
-                // Restart EPG background refresh with new interval
-                if (window.app?.epgGuide) {
-                    window.app.epgGuide.restartBackgroundRefreshIfNeeded();
-                }
             });
         }
 
+        // Update last refreshed display
+        this.updateEpgLastRefreshed();
+
         // Stream output format
         const streamFormatSelect = document.getElementById('setting-stream-format');
         if (streamFormatSelect && this.app.player?.settings) {
@@ -287,6 +286,56 @@ class SettingsPage {
             if (epgRefreshSelect) epgRefreshSelect.value = s.epgRefreshInterval || '24';
             if (streamFormatSelect) streamFormatSelect.value = s.streamFormat || 'm3u8';
         }
+
+        // Update EPG last refreshed display
+        this.updateEpgLastRefreshed();
+    }
+
+    /**
+     * Update the EPG last refreshed display
+     */
+    async updateEpgLastRefreshed() {
+        const display = document.getElementById('epg-last-refreshed');
+        if (!display) return;
+
+        try {
+            // Fetch last sync time from server
+            const response = await fetch('/api/settings/sync-status');
+            if (!response.ok) throw new Error('Failed to fetch sync status');
+            const data = await response.json();
+
+            if (data.lastSyncTime) {
+                const lastRefreshTime = new Date(data.lastSyncTime);
+
+                // Format as relative time or absolute
+                const now = new Date();
+                const diffMs = now - lastRefreshTime;
+                const diffMins = Math.floor(diffMs / 60000);
+                const diffHours = Math.floor(diffMins / 60);
+
+                let text;
+                if (diffMins < 1) {
+                    text = 'Just now';
+                } else if (diffMins < 60) {
+                    text = `${diffMins} minute${diffMins === 1 ? '' : 's'} ago`;
+                } else if (diffHours < 24) {
+                    text = `${diffHours} hour${diffHours === 1 ? '' : 's'} ago`;
+                } else {
+                    // Use absolute time for older refreshes
+                    text = lastRefreshTime.toLocaleString();
+                }
+
+                display.textContent = text;
+                display.title = lastRefreshTime.toLocaleString(); // Full timestamp on hover
+            } else {
+                display.textContent = 'Never';
+                display.title = 'Sync has not run yet since server started';
+            }
+        } catch (err) {
+            console.error('Error fetching sync status:', err);
+            display.textContent = 'Unknown';
+            display.title = 'Could not fetch sync status';
+        }
     }
 
     hide() {

server/index.js

@@ -79,7 +79,9 @@ app.listen(PORT, async () => {
     console.log(`NodeCast TV server running on http://localhost:${PORT}`);
 
     // Trigger background sync with delay to allow server to settle
-    setTimeout(() => {
+    setTimeout(async () => {
-        syncService.syncAll().catch(console.error);
+        await syncService.syncAll().catch(console.error);
+        // Start the server-side sync timer after initial sync
+        await syncService.startSyncTimer().catch(console.error);
     }, 5000);
 });

server/routes/settings.js

@@ -1,6 +1,7 @@
 const express = require('express');
 const router = express.Router();
 const { settings, getDefaultSettings } = require('../db');
+const syncService = require('../services/syncService');
 
 /**
  * Get all settings
@@ -24,6 +25,12 @@ router.put('/', async (req, res) => {
     try {
         const updates = req.body;
         const updatedSettings = await settings.update(updates);
+
+        // If sync interval changed, restart the server-side sync timer
+        if (updates.epgRefreshInterval !== undefined) {
+            syncService.restartSyncTimer().catch(console.error);
+        }
+
         res.json(updatedSettings);
     } catch (err) {
         console.error('Error updating settings:', err);
@@ -53,4 +60,16 @@ router.get('/defaults', (req, res) => {
     res.json(getDefaultSettings());
 });
 
+/**
+ * Get sync status (last sync time)
+ * GET /api/settings/sync-status
+ */
+router.get('/sync-status', (req, res) => {
+    const lastSyncTime = syncService.getLastSyncTime();
+    res.json({
+        lastSyncTime: lastSyncTime ? lastSyncTime.toISOString() : null
+    });
+});
+
 module.exports = router;
+

server/routes/sources.js

@@ -185,4 +185,17 @@ router.post('/:id/test', async (req, res) => {
     }
 });
 
+// Global Sync - sync all enabled sources
+router.post('/sync-all', async (req, res) => {
+    try {
+        // Trigger global sync (async - don't wait for completion)
+        syncService.syncAll().catch(console.error);
+        res.json({ success: true, message: 'Global sync started' });
+    } catch (err) {
+        console.error('Error starting global sync:', err);
+        res.status(500).json({ error: 'Failed to start global sync' });
+    }
+});
+
 module.exports = router;
+

server/services/syncService.js

@@ -1,5 +1,5 @@
 const { getDb } = require('../db/sqlite');
-const { sources } = require('../db'); // For source config
+const { sources, settings } = require('../db'); // For source config and settings
 const xtreamApi = require('./xtreamApi');
 const m3uParser = require('./m3uParser');
 const epgParser = require('./epgParser');
@@ -8,6 +8,74 @@ const epgParser = require('./epgParser');
 const activeSyncs = new Set(); // sourceId
 
 class SyncService {
+    constructor() {
+        this.lastSyncTime = null; // Track when global sync last completed
+        this._syncTimer = null;   // Server-side sync timer
+        this._currentInterval = null;
+    }
+
+    /**
+     * Get when the last global sync completed
+     */
+    getLastSyncTime() {
+        return this.lastSyncTime;
+    }
+
+    /**
+     * Start the server-side sync timer based on settings
+     * Should be called once on server startup after initial sync
+     */
+    async startSyncTimer() {
+        // Get interval from settings
+        const currentSettings = await settings.get();
+        const intervalHours = parseInt(currentSettings.epgRefreshInterval) || 24;
+
+        // If interval is 0, don't start timer (manual only mode)
+        if (intervalHours <= 0) {
+            console.log('[Sync] Auto-sync disabled (manual only mode)');
+            this.stopSyncTimer();
+            this._currentInterval = 0;
+            return;
+        }
+
+        const intervalMs = intervalHours * 60 * 60 * 1000;
+
+        // Don't restart if interval hasn't changed
+        if (this._currentInterval === intervalHours && this._syncTimer) {
+            return;
+        }
+
+        // Clear existing timer
+        this.stopSyncTimer();
+
+        console.log(`[Sync] Starting server-side sync timer: every ${intervalHours} hours`);
+
+        this._syncTimer = setInterval(async () => {
+            console.log('[Sync] Scheduled sync triggered');
+            await this.syncAll();
+        }, intervalMs);
+
+        this._currentInterval = intervalHours;
+    }
+
+    /**
+     * Stop the server-side sync timer
+     */
+    stopSyncTimer() {
+        if (this._syncTimer) {
+            clearInterval(this._syncTimer);
+            this._syncTimer = null;
+        }
+    }
+
+    /**
+     * Restart the sync timer with updated settings
+     * Called when sync interval setting changes
+     */
+    async restartSyncTimer() {
+        await this.startSyncTimer();
+    }
+
     /**
      * Sync all enabled sources
      */
@@ -21,7 +89,8 @@ class SyncService {
                     await this.syncSource(source.id);
                 }
             }
-            console.log('[Sync] Global sync completed');
+            this.lastSyncTime = new Date();
+            console.log('[Sync] Global sync completed at', this.lastSyncTime.toISOString());
         } catch (err) {
             console.error('[Sync] Global sync failed:', err);
         }
@@ -164,9 +233,11 @@ class SyncService {
             }
         });
 
-        const BATCH_SIZE = 500;
+        // Reduced batch size for better event loop interleaving
+        const BATCH_SIZE = 100;
         for (let i = 0; i < categories.length; i += BATCH_SIZE) {
             insertBatch(categories.slice(i, i + BATCH_SIZE));
+            // Yield to event loop between batches to allow other requests
             await new Promise(resolve => setImmediate(resolve));
         }
 
@@ -244,9 +315,11 @@ class SyncService {
             }
         });
 
-        const BATCH_SIZE = 500;
+        // Reduced batch size for better event loop interleaving
+        const BATCH_SIZE = 100;
         for (let i = 0; i < items.length; i += BATCH_SIZE) {
             insertBatch(items.slice(i, i + BATCH_SIZE));
+            // Yield to event loop between batches to allow other requests
             await new Promise(resolve => setImmediate(resolve));
         }