# Generated by Django - Add stream_id and channel_number fields with data migration from django.db import migrations, models import hashlib import json import logging logger = logging.getLogger(__name__) def populate_fields_and_rehash(apps, schema_editor): """ Populate stream_id and stream_chno from custom_properties for XC account streams, populate stream_chno from tvg-chno for standard M3U accounts, then rehash XC streams using stable hash keys. """ Stream = apps.get_model('dispatcharr_channels', 'Stream') M3UAccount = apps.get_model('m3u', 'M3UAccount') CoreSettings = apps.get_model('core', 'CoreSettings') # Get hash keys from settings try: stream_settings = CoreSettings.objects.get(key='stream_settings') hash_key_str = stream_settings.value.get('m3u_hash_key', '') if stream_settings.value else '' keys = [k.strip() for k in hash_key_str.split(',') if k.strip()] if hash_key_str else [] except CoreSettings.DoesNotExist: keys = [] logger.info(f"Using hash keys: {keys}") # Get XC account IDs xc_account_ids = set( M3UAccount.objects.filter(account_type='XC').values_list('id', flat=True) ) logger.info(f"Found {len(xc_account_ids)} XC accounts") # Track hash collisions for XC streams hash_map = {} # new_hash -> stream_id duplicates_to_delete = [] # Process all streams in batches batch_size = 1000 processed = 0 updated = 0 total_count = Stream.objects.count() logger.info(f"Processing {total_count} total streams") streams_to_update = [] for stream in Stream.objects.select_related('channel_group', 'm3u_account').iterator(chunk_size=batch_size): processed += 1 needs_update = False custom_props = stream.custom_properties or {} is_xc = stream.m3u_account_id in xc_account_ids if stream.m3u_account_id else False # Extract stream_id (XC accounts only) if is_xc and isinstance(custom_props, dict): provider_stream_id = custom_props.get('stream_id') if provider_stream_id: try: stream.stream_id = int(provider_stream_id) needs_update = True except (ValueError, TypeError): pass # Extract stream_chno channel_num = None if isinstance(custom_props, dict): if is_xc: # XC accounts use 'num' channel_num = custom_props.get('num') else: # Standard M3U accounts use 'tvg-chno' or 'channel-number' (case insensitive check) for key in ['tvg-chno', 'TVG-CHNO', 'tvg-Chno', 'Tvg-Chno', 'channel-number', 'Channel-Number', 'CHANNEL-NUMBER']: if key in custom_props: channel_num = custom_props.get(key) break if channel_num is not None: try: stream.stream_chno = float(channel_num) needs_update = True except (ValueError, TypeError): pass # Rehash XC streams only when 'url' is in hash keys (otherwise hash wouldn't change) if is_xc and stream.stream_id and keys and 'url' in keys: # For XC accounts, use stream_id instead of url when 'url' is in the hash keys # This ensures credential/URL changes don't break stream identity effective_url = stream.stream_id # Get group name group_name = stream.channel_group.name if stream.channel_group else None # Build hash parts stream_parts = { "name": stream.name, "url": effective_url, "tvg_id": stream.tvg_id, "m3u_id": stream.m3u_account_id, "group": group_name } hash_parts = {key: stream_parts[key] for key in keys if key in stream_parts} # When using stream_id instead of URL, we MUST include m3u_id to prevent # collisions across different XC accounts (stream_id is only unique per account) if 'm3u_id' not in hash_parts: hash_parts['m3u_id'] = stream.m3u_account_id # Generate hash serialized_obj = json.dumps(hash_parts, sort_keys=True) new_hash = hashlib.sha256(serialized_obj.encode()).hexdigest() # Check for collisions if new_hash in hash_map: # Duplicate - mark for deletion (keep the first one) duplicates_to_delete.append(stream.id) continue hash_map[new_hash] = stream.id stream.stream_hash = new_hash needs_update = True if needs_update: streams_to_update.append(stream) updated += 1 # Bulk update in batches if len(streams_to_update) >= batch_size: Stream.objects.bulk_update( streams_to_update, ['stream_id', 'stream_chno', 'stream_hash'], batch_size=500 ) logger.info(f"Updated batch: {processed}/{total_count} streams processed") streams_to_update = [] # Final batch if streams_to_update: Stream.objects.bulk_update( streams_to_update, ['stream_id', 'stream_chno', 'stream_hash'], batch_size=500 ) # Delete duplicates if any if duplicates_to_delete: logger.warning(f"Deleting {len(duplicates_to_delete)} duplicate streams due to hash collisions") Stream.objects.filter(id__in=duplicates_to_delete).delete() logger.info(f"Migration complete: {updated} streams updated, {len(duplicates_to_delete)} duplicates removed") def reverse_migration(apps, schema_editor): """ Reverse migration - clear fields but don't attempt to reverse hash changes. """ Stream = apps.get_model('dispatcharr_channels', 'Stream') Stream.objects.all().update(stream_id=None, stream_chno=None) logger.info("Cleared stream_id and stream_chno fields. Note: stream hashes were not reverted.") class Migration(migrations.Migration): dependencies = [ ('dispatcharr_channels', '0032_channel_is_adult_stream_is_adult'), ('m3u', '0018_add_profile_custom_properties'), ('core', '0020_change_coresettings_value_to_jsonfield'), ] operations = [ # Schema changes - add fields WITHOUT indexes first migrations.AddField( model_name='stream', name='stream_id', field=models.IntegerField( blank=True, help_text='Provider stream ID (e.g., XC stream_id) for stable identity across credential changes', null=True, ), ), migrations.AddField( model_name='stream', name='stream_chno', field=models.FloatField( blank=True, help_text='Provider channel number (XC num or M3U tvg-chno) for ordering - supports decimals like 2.1', null=True, ), ), # Data migration (may delete duplicates, which would conflict with pending index creation) migrations.RunPython(populate_fields_and_rehash, reverse_migration), # Add indexes AFTER data migration completes migrations.AddIndex( model_name='stream', index=models.Index(fields=['stream_id'], name='dispatcharr_stream_id_idx'), ), migrations.AddIndex( model_name='stream', index=models.Index(fields=['stream_chno'], name='dispatcharr_stream_chno_idx'), ), ]