Refactored code to use sqlite for channel, epg and vod data
This commit is contained in:
Trevor Mears
@@ -89,19 +89,21 @@ async function parse(input) {
|
||||
let currentInfo = null;
|
||||
let currentGroup = null;
|
||||
|
||||
let inputStream;
|
||||
let lines;
|
||||
|
||||
if (typeof input === 'string') {
|
||||
inputStream = Readable.from([input]);
|
||||
// Handle string input directly
|
||||
lines = input.split(/\r?\n/);
|
||||
} else {
|
||||
inputStream = input;
|
||||
// Handle stream input
|
||||
const rl = readline.createInterface({
|
||||
input: input,
|
||||
crlfDelay: Infinity
|
||||
});
|
||||
lines = rl;
|
||||
}
|
||||
|
||||
const rl = readline.createInterface({
|
||||
input: inputStream,
|
||||
crlfDelay: Infinity
|
||||
});
|
||||
|
||||
for await (const line of rl) {
|
||||
for await (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) continue;
|
||||
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
const { getDb } = require('../db/sqlite');
|
||||
const { hiddenItems, favorites } = require('../db');
|
||||
|
||||
async function migrateUserData() {
|
||||
console.log('[Migration] Starting user data migration (Hidden Items & Favorites)...');
|
||||
const db = getDb();
|
||||
|
||||
try {
|
||||
// 1. Migrate Hidden Items
|
||||
const allHidden = await hiddenItems.getAll(); // returns array of { source_id, item_type, item_id }
|
||||
|
||||
const hidStmt = db.prepare(`
|
||||
UPDATE playlist_items
|
||||
SET is_hidden = 1
|
||||
WHERE source_id = ? AND type = ? AND item_id = ?
|
||||
`);
|
||||
|
||||
const catHidStmt = db.prepare(`
|
||||
UPDATE categories
|
||||
SET is_hidden = 1
|
||||
WHERE source_id = ? AND type = ? AND category_id = ?
|
||||
`);
|
||||
|
||||
let hidCount = 0;
|
||||
const migrateHidden = db.transaction((items) => {
|
||||
for (const item of items) {
|
||||
// Map db.json types to SQLite types
|
||||
// db.json types: 'channel', 'group', 'vod_category', 'series_category'
|
||||
// SQLite types: 'live', 'movie', 'series' (for categories/items)
|
||||
|
||||
if (item.item_type === 'channel') {
|
||||
// Hidden channel -> playlist_items (type='live')
|
||||
const res = hidStmt.run(item.source_id, 'live', item.item_id);
|
||||
if (res.changes > 0) hidCount++;
|
||||
} else if (item.item_type === 'group') {
|
||||
// Hidden group -> categories (type='live')
|
||||
const res = catHidStmt.run(item.source_id, 'live', item.item_id); // item_id is group name/id
|
||||
if (res.changes > 0) hidCount++;
|
||||
} else if (item.item_type === 'vod_category') {
|
||||
const res = catHidStmt.run(item.source_id, 'movie', item.item_id);
|
||||
if (res.changes > 0) hidCount++;
|
||||
} else if (item.item_type === 'series_category') {
|
||||
const res = catHidStmt.run(item.source_id, 'series', item.item_id);
|
||||
if (res.changes > 0) hidCount++;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
migrateHidden(allHidden);
|
||||
console.log(`[Migration] Migrated ${hidCount} hidden items/categories.`);
|
||||
|
||||
|
||||
// 2. Migrate Favorites
|
||||
const allFavorites = await favorites.getAll();
|
||||
|
||||
const favStmt = db.prepare(`
|
||||
UPDATE playlist_items
|
||||
SET is_favorite = 1
|
||||
WHERE source_id = ? AND type = ? AND item_id = ?
|
||||
`);
|
||||
|
||||
let favCount = 0;
|
||||
const migrateFavorites = db.transaction((favs) => {
|
||||
for (const fav of favs) {
|
||||
// Map types
|
||||
// db.json: 'channel', 'movie', 'series'
|
||||
// SQLite: 'live', 'movie', 'series'
|
||||
|
||||
let type = fav.item_type;
|
||||
if (type === 'channel') type = 'live';
|
||||
|
||||
const res = favStmt.run(fav.source_id, type, fav.item_id);
|
||||
if (res.changes > 0) favCount++;
|
||||
}
|
||||
});
|
||||
|
||||
migrateFavorites(allFavorites);
|
||||
console.log(`[Migration] Migrated ${favCount} favorites.`);
|
||||
|
||||
} catch (e) {
|
||||
console.error('[Migration] Failed:', e);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { migrateUserData };
|
||||
@@ -0,0 +1,385 @@
|
||||
const { getDb } = require('../db/sqlite');
|
||||
const { sources } = require('../db'); // For source config
|
||||
const xtreamApi = require('./xtreamApi');
|
||||
const m3uParser = require('./m3uParser');
|
||||
const epgParser = require('./epgParser');
|
||||
|
||||
// Sync tracking
|
||||
const activeSyncs = new Set(); // sourceId
|
||||
|
||||
class SyncService {
|
||||
/**
|
||||
* Sync all enabled sources
|
||||
*/
|
||||
async syncAll() {
|
||||
console.log('[Sync] Starting global sync...');
|
||||
try {
|
||||
const allSources = await sources.getAll();
|
||||
for (const source of allSources) {
|
||||
if (source.enabled) {
|
||||
// Run sequentially to not overload
|
||||
await this.syncSource(source.id);
|
||||
}
|
||||
}
|
||||
console.log('[Sync] Global sync completed');
|
||||
} catch (err) {
|
||||
console.error('[Sync] Global sync failed:', err);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start sync for a source
|
||||
*/
|
||||
async syncSource(sourceId) {
|
||||
if (activeSyncs.has(sourceId)) {
|
||||
console.log(`[Sync] Source ${sourceId} is already syncing`);
|
||||
return;
|
||||
}
|
||||
|
||||
activeSyncs.add(sourceId);
|
||||
|
||||
try {
|
||||
const db = getDb();
|
||||
const source = await sources.getById(sourceId);
|
||||
|
||||
if (!source) {
|
||||
throw new Error(`Source ${sourceId} not found`);
|
||||
}
|
||||
|
||||
console.log(`[Sync] Starting sync for source ${source.name} (ID: ${sourceId})`);
|
||||
|
||||
if (!source.enabled) {
|
||||
console.log(`[Sync] Skipping disabled source ${source.name}`);
|
||||
activeSyncs.delete(sourceId);
|
||||
return;
|
||||
}
|
||||
|
||||
// Update status
|
||||
this.updateSyncStatus(sourceId, 'all', 'syncing');
|
||||
|
||||
if (source.type === 'xtream') {
|
||||
await this.syncXtream(source);
|
||||
} else if (source.type === 'm3u') {
|
||||
await this.syncM3u(source);
|
||||
} else if (source.type === 'epg') {
|
||||
await this.syncEpg(source);
|
||||
}
|
||||
|
||||
this.updateSyncStatus(sourceId, 'all', 'success');
|
||||
console.log(`[Sync] Completed sync for source ${source.name}`);
|
||||
|
||||
} catch (err) {
|
||||
console.error(`[Sync] Failed sync for source ${sourceId}:`, err);
|
||||
this.updateSyncStatus(sourceId, 'all', 'error', err.message);
|
||||
} finally {
|
||||
activeSyncs.delete(sourceId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update sync status in DB
|
||||
*/
|
||||
updateSyncStatus(sourceId, type, status, error = null) {
|
||||
const db = getDb();
|
||||
const stmt = db.prepare(`
|
||||
INSERT INTO sync_status (source_id, type, last_sync, status, error)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(source_id, type) DO UPDATE SET
|
||||
last_sync = excluded.last_sync,
|
||||
status = excluded.status,
|
||||
error = excluded.error
|
||||
`);
|
||||
stmt.run(sourceId, type, Date.now(), status, error);
|
||||
}
|
||||
|
||||
/**
|
||||
* Xtream Sync Logic
|
||||
*/
|
||||
async syncXtream(source) {
|
||||
const api = xtreamApi.createFromSource(source);
|
||||
const db = getDb();
|
||||
|
||||
// 1. Live Categories
|
||||
console.log(`[Sync] Fetching Live Categories for ${source.name}`);
|
||||
const liveCats = await api.getLiveCategories();
|
||||
await this.saveCategories(source.id, 'live', liveCats);
|
||||
|
||||
// 2. Live Streams
|
||||
console.log(`[Sync] Fetching Live Streams for ${source.name}`);
|
||||
const liveStreams = await api.getLiveStreams();
|
||||
await this.saveStreams(source.id, 'live', liveStreams);
|
||||
|
||||
// 3. VOD Categories
|
||||
console.log(`[Sync] Fetching VOD Categories for ${source.name}`);
|
||||
const vodCats = await api.getVodCategories();
|
||||
await this.saveCategories(source.id, 'movie', vodCats);
|
||||
|
||||
// 4. VOD Streams
|
||||
console.log(`[Sync] Fetching VOD Streams for ${source.name}`);
|
||||
const vodStreams = await api.getVodStreams();
|
||||
await this.saveStreams(source.id, 'movie', vodStreams);
|
||||
|
||||
// 5. Series Categories
|
||||
console.log(`[Sync] Fetching Series Categories for ${source.name}`);
|
||||
const seriesCats = await api.getSeriesCategories();
|
||||
await this.saveCategories(source.id, 'series', seriesCats);
|
||||
|
||||
// 6. Series
|
||||
console.log(`[Sync] Fetching Series for ${source.name}`);
|
||||
const series = await api.getSeries();
|
||||
await this.saveStreams(source.id, 'series', series);
|
||||
|
||||
// 7. EPG (Xmltv)
|
||||
// Try to fetch XMLTV if available
|
||||
console.log(`[Sync] Fetching EPG for ${source.name}`);
|
||||
try {
|
||||
const xmltvUrl = api.getXmltvUrl();
|
||||
await this.syncEpgFromUrl(source.id, xmltvUrl);
|
||||
} catch (e) {
|
||||
console.warn('[Sync] XMLTV fetch failed, skipping EPG sync for now:', e.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch save categories
|
||||
*/
|
||||
async saveCategories(sourceId, type, categories) {
|
||||
if (!categories || categories.length === 0) return;
|
||||
console.log(`[Sync] Saving ${categories.length} ${type} categories for source ${sourceId}...`);
|
||||
const db = getDb();
|
||||
const stmt = db.prepare(`
|
||||
INSERT INTO categories (id, source_id, category_id, type, name, parent_id, data)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
name = excluded.name,
|
||||
data = excluded.data
|
||||
`);
|
||||
|
||||
const insertBatch = db.transaction((batch) => {
|
||||
for (const cat of batch) {
|
||||
const catId = cat.category_id; // standard xtream field
|
||||
const name = cat.category_name;
|
||||
const id = `${sourceId}:${catId}`;
|
||||
stmt.run(id, sourceId, String(catId), type, name, cat.parent_id || null, JSON.stringify(cat));
|
||||
}
|
||||
});
|
||||
|
||||
const BATCH_SIZE = 500;
|
||||
for (let i = 0; i < categories.length; i += BATCH_SIZE) {
|
||||
insertBatch(categories.slice(i, i + BATCH_SIZE));
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
}
|
||||
|
||||
console.log(`[Sync] Saved ${categories.length} ${type} categories`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch save streams (channels, vod, series)
|
||||
*/
|
||||
async saveStreams(sourceId, type, items) {
|
||||
if (!items || items.length === 0) return;
|
||||
const db = getDb();
|
||||
const stmt = db.prepare(`
|
||||
INSERT INTO playlist_items (
|
||||
id, source_id, item_id, type, name, category_id,
|
||||
stream_icon, stream_url, container_extension,
|
||||
rating, year, added_at, data
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
name = excluded.name,
|
||||
category_id = excluded.category_id,
|
||||
stream_icon = excluded.stream_icon,
|
||||
container_extension = excluded.container_extension,
|
||||
data = excluded.data
|
||||
`);
|
||||
|
||||
const insertBatch = db.transaction((batch) => {
|
||||
for (const item of batch) {
|
||||
// Map fields based on type
|
||||
let itemId, name, catId, icon, container;
|
||||
let rating = null, year = null, added = null;
|
||||
|
||||
if (type === 'live') {
|
||||
itemId = item.stream_id;
|
||||
name = item.name;
|
||||
catId = item.category_id;
|
||||
icon = item.stream_icon;
|
||||
added = item.added;
|
||||
} else if (type === 'movie') {
|
||||
itemId = item.stream_id;
|
||||
name = item.name;
|
||||
catId = item.category_id;
|
||||
icon = item.stream_icon; // or cover
|
||||
container = item.container_extension;
|
||||
rating = item.rating;
|
||||
added = item.added;
|
||||
} else if (type === 'series') {
|
||||
itemId = item.series_id;
|
||||
name = item.name;
|
||||
catId = item.category_id;
|
||||
icon = item.cover;
|
||||
rating = item.rating;
|
||||
year = item.releaseDate;
|
||||
added = item.last_modified;
|
||||
}
|
||||
|
||||
const id = `${sourceId}:${itemId}`;
|
||||
|
||||
stmt.run(
|
||||
id,
|
||||
sourceId,
|
||||
String(itemId),
|
||||
type,
|
||||
name,
|
||||
String(catId),
|
||||
icon,
|
||||
null, // Direct URL not stored for Xtream usually, built on fly
|
||||
container,
|
||||
rating,
|
||||
year,
|
||||
added,
|
||||
JSON.stringify(item)
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
const BATCH_SIZE = 500;
|
||||
for (let i = 0; i < items.length; i += BATCH_SIZE) {
|
||||
insertBatch(items.slice(i, i + BATCH_SIZE));
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
}
|
||||
|
||||
console.log(`[Sync] Saved ${items.length} ${type} items`);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Sync EPG from URL
|
||||
*/
|
||||
async syncEpgFromUrl(sourceId, url) {
|
||||
// Use our streaming parser
|
||||
const { channels, programmes } = await epgParser.fetchAndParse(url);
|
||||
|
||||
console.log(`[Sync] EPG Parsed: ${channels.length} channels, ${programmes.length} programs`);
|
||||
|
||||
const db = getDb();
|
||||
|
||||
// 1. Save EPG Channels to playlist_items (for Name/Icon matching)
|
||||
|
||||
const channelStmt = db.prepare(`
|
||||
INSERT INTO playlist_items (
|
||||
id, source_id, item_id, type, name, stream_icon,
|
||||
stream_url, category_id, data
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
name = excluded.name,
|
||||
stream_icon = excluded.stream_icon,
|
||||
data = excluded.data
|
||||
`);
|
||||
|
||||
// Use transaction for channels
|
||||
const insertChannels = db.transaction((chanList) => {
|
||||
for (const ch of chanList) {
|
||||
const id = `${sourceId}:${ch.id}`;
|
||||
channelStmt.run(
|
||||
id,
|
||||
sourceId,
|
||||
ch.id, // XMLTV ID
|
||||
'epg_channel',
|
||||
ch.name,
|
||||
ch.icon || null,
|
||||
null, // No URL
|
||||
null, // No Category
|
||||
JSON.stringify(ch)
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
insertChannels(channels);
|
||||
console.log(`[Sync] Saved ${channels.length} EPG channels`);
|
||||
|
||||
// 2. Save Programs
|
||||
// First delete old programs for this source
|
||||
db.prepare('DELETE FROM epg_programs WHERE source_id = ?').run(sourceId);
|
||||
|
||||
const stmt = db.prepare(`
|
||||
INSERT INTO epg_programs (channel_id, source_id, start_time, end_time, title, description, data)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
|
||||
const insertMany = db.transaction((progs) => {
|
||||
for (const p of progs) {
|
||||
stmt.run(
|
||||
p.channelId,
|
||||
sourceId,
|
||||
p.start ? p.start.getTime() : 0,
|
||||
p.stop ? p.stop.getTime() : 0,
|
||||
p.title,
|
||||
p.description || p.desc,
|
||||
JSON.stringify(p)
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
insertMany(programmes);
|
||||
console.log(`[Sync] Saved ${programmes.length} programs`);
|
||||
}
|
||||
|
||||
/**
|
||||
* M3U Sync Logic
|
||||
*/
|
||||
/**
|
||||
* M3U Sync Logic
|
||||
*/
|
||||
async syncM3u(source) {
|
||||
console.log(`[Sync] Fetching M3U playlist for ${source.name}`);
|
||||
|
||||
// Fetch and parse using existing parser
|
||||
// We use fetch directly here to get the stream/text
|
||||
const response = await fetch(source.url);
|
||||
if (!response.ok) throw new Error(`Failed to fetch M3U: ${response.status}`);
|
||||
|
||||
const text = await response.text();
|
||||
const { channels, groups } = await m3uParser.parse(text);
|
||||
|
||||
console.log(`[Sync] M3U Parsed: ${channels.length} channels, ${groups.length} groups`);
|
||||
|
||||
// Save Categories (Groups)
|
||||
// M3U groups are just strings usually, we need to normalize them
|
||||
const categories = groups.map(g => ({
|
||||
category_id: g.name, // use name as ID for M3U groups
|
||||
category_name: g.name,
|
||||
parent_id: null
|
||||
}));
|
||||
|
||||
await this.saveCategories(source.id, 'live', categories);
|
||||
|
||||
// Save Channels
|
||||
// Map M3U channel format to our schema
|
||||
const playlistItems = channels.map(ch => ({
|
||||
stream_id: ch.id, // parser generates a stable-ish ID
|
||||
name: ch.name,
|
||||
category_id: ch.groupTitle || 'Uncategorized',
|
||||
stream_icon: ch.tvgLogo,
|
||||
stream_url: ch.url,
|
||||
// M3U doesn't usually have VOD metadata like rating/year easily accessible unless extended tags used
|
||||
// We assume 'live' for now, but could detect VOD from URL extension?
|
||||
// For now, treat all as type='live' for M3U or maybe check info?
|
||||
// The parser doesn't differentiate types well yet.
|
||||
}));
|
||||
|
||||
await this.saveStreams(source.id, 'live', playlistItems);
|
||||
}
|
||||
|
||||
/**
|
||||
* EPG Source Sync Logic
|
||||
*/
|
||||
async syncEpg(source) {
|
||||
console.log(`[Sync] Fetching standalone EPG for ${source.name}`);
|
||||
await this.syncEpgFromUrl(source.id, source.url);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = new SyncService();
|
||||
Reference in New Issue
Block a user