const details = () => ({ id: 'Tdarr_Plugin_stream_organizer', Stage: 'Pre-processing', Name: 'Stream Organizer', Type: 'Video', Operation: 'Transcode', Description: ` Organizes streams by language priority (English/custom codes first). Converts text-based subtitles to SRT format and/or extracts them to external files. Handles closed captions (eia_608/cc_dec) via CCExtractor. All other streams are preserved in their original relative order. WebVTT subtitles are always converted to SRT for compatibility. `, Version: '4.8', Tags: 'action,subtitles,srt,extract,organize,language', Inputs: [ { name: 'includeAudio', type: 'string', defaultValue: 'true*', inputUI: { type: 'dropdown', options: [ 'true*', 'false' ], }, tooltip: 'Enable to reorder audio streams, putting English audio first', }, { name: 'includeSubtitles', type: 'string', defaultValue: 'true*', inputUI: { type: 'dropdown', options: [ 'true*', 'false' ], }, tooltip: 'Enable to reorder subtitle streams, putting English subtitles first', }, { name: 'standardizeToSRT', type: 'string', defaultValue: 'true*', inputUI: { type: 'dropdown', options: [ 'true*', 'false' ], }, tooltip: 'Convert text-based subtitles (ASS/SSA/WebVTT) to SRT format. Image subtitles (PGS/VobSub) will be copied.', }, { name: 'extractSubtitles', type: 'string', defaultValue: 'false', inputUI: { type: 'dropdown', options: [ 'false', 'true' ], }, tooltip: 'Extract subtitle streams to external .srt files alongside the video', }, { name: 'removeAfterExtract', type: 'string', defaultValue: 'false', inputUI: { type: 'dropdown', options: [ 'false', 'true' ], }, tooltip: 'Remove embedded subtitles after extracting them (only applies if Extract is enabled)', }, { name: 'skipCommentary', type: 'string', defaultValue: 'true*', inputUI: { type: 'dropdown', options: [ 'true*', 'false' ], }, tooltip: 'Skip extracting subtitles with "commentary" or "description" in the title', }, { name: 'setDefaultFlags', type: 'string', defaultValue: 'false', inputUI: { type: 'dropdown', options: [ 'false', 'true' ], }, tooltip: 'Set default disposition flag on first English audio and subtitle streams', }, { name: 'customLanguageCodes', type: 'string', defaultValue: 'eng,en,english,en-us,en-gb,en-ca,en-au', inputUI: { type: 'text', }, tooltip: 'Comma-separated list of language codes to consider as priority (max 20 codes). Default includes common English codes.', }, { name: 'useCCExtractor', type: 'string', defaultValue: 'false', inputUI: { type: 'dropdown', options: [ 'false', 'true' ], }, tooltip: 'If enabled, attempts to extract closed captions (eia_608/cc_dec) to external SRT via ccextractor when present.', }, { name: 'embedExtractedCC', type: 'string', defaultValue: 'false', inputUI: { type: 'dropdown', options: [ 'false', 'true' ], }, tooltip: 'If enabled, will map the newly extracted CC SRT back into the output container.', }, ], }); const TEXT_SUBTITLE_CODECS = new Set(['ass', 'ssa', 'webvtt', 'mov_text', 'text', 'subrip']); const IMAGE_SUBTITLE_CODECS = new Set(['hdmv_pgs_subtitle', 'dvd_subtitle', 'dvdsub']); const PROBLEMATIC_CODECS = new Set(['webvtt']); const UNSUPPORTED_SUBTITLE_CODECS = new Set(['eia_608', 'cc_dec', 'tx3g']); const VALID_BOOLEAN_VALUES = ['true', 'false']; const MAX_LANGUAGE_CODES = 20; const MIN_SUBTITLE_FILE_SIZE = 100; // bytes - minimum size for valid subtitle file const MAX_EXTRACTION_ATTEMPTS = 3; // maximum attempts to extract a subtitle before giving up const MAX_FILENAME_ATTEMPTS = 100; // maximum attempts to find unique filename const isUnsupportedSubtitle = (stream) => { const name = (stream.codec_name || '').toLowerCase(); const tag = (stream.codec_tag_string || '').toLowerCase(); return UNSUPPORTED_SUBTITLE_CODECS.has(name) || UNSUPPORTED_SUBTITLE_CODECS.has(tag); }; const isClosedCaption = (stream) => { const name = (stream.codec_name || '').toLowerCase(); const tag = (stream.codec_tag_string || '').toLowerCase(); return name === 'eia_608' || name === 'cc_dec' || tag === 'cc_dec'; }; const isEnglishStream = (stream, englishCodes) => { const language = stream.tags?.language?.toLowerCase(); return language && englishCodes.includes(language); }; const isTextSubtitle = (stream) => TEXT_SUBTITLE_CODECS.has(stream.codec_name); const needsSRTConversion = (stream) => isTextSubtitle(stream) && stream.codec_name !== 'subrip'; const isProblematicSubtitle = (stream) => PROBLEMATIC_CODECS.has(stream.codec_name); const shouldSkipSubtitle = (stream, skipCommentary) => { if (skipCommentary !== 'true') return false; const title = stream.tags?.title?.toLowerCase() || ''; return title.includes('commentary') || title.includes('description'); }; // Helper to check if any processing is needed const needsProcessing = (needsReorder, needsConversion, extractCount, ccActuallyExtracted, ccExtractedFile, embedExtractedCC) => { return needsReorder || needsConversion || extractCount > 0 || ccActuallyExtracted || (ccExtractedFile && embedExtractedCC === 'true'); }; const partitionStreams = (streams, predicate) => { const matched = []; const unmatched = []; streams.forEach(s => (predicate(s) ? matched : unmatched).push(s)); return [matched, unmatched]; }; const buildSafeBasePath = (filePath) => { const parsed = require('path').parse(filePath); return require('path').join(parsed.dir, parsed.name); }; /** * Robust file existence check * Uses fs.statSync to avoid caching issues with fs.existsSync */ const fileExistsRobust = (filePath, fs) => { try { const stats = fs.statSync(filePath); // Verify file is not empty (sometimes extraction fails silently) return stats.size > 0; } catch (e) { if (e.code === 'ENOENT') { return false; } // Re-throw other errors (permission issues, etc) throw new Error(`Error checking file existence for ${filePath}: ${e.message}`); } }; /** * Check if subtitle file needs extraction * Handles cases where file exists but is incomplete or outdated */ const needsSubtitleExtraction = (subsFile, sourceFile, fs) => { // Check if file exists using robust method if (!fileExistsRobust(subsFile, fs)) { return true; // File doesn't exist, needs extraction } try { const subsStats = fs.statSync(subsFile); // If subtitle file is very small, it might be incomplete if (subsStats.size < MIN_SUBTITLE_FILE_SIZE) { return true; // Re-extract } // NOTE: We removed mtime comparison because: // 1. During requeue, the "source" is a cache file with current timestamp // 2. This always triggers re-extraction even when subs already exist // 3. Size check is sufficient to detect incomplete extractions return false; // Subtitle exists and has valid size } catch (e) { // If any error checking stats, assume needs extraction return true; } }; const plugin = (file, librarySettings, inputs, otherArguments) => { const lib = require('../methods/lib')(); const fs = require('fs'); const path = require('path'); const crypto = require('crypto'); // Sanitization utilities (self-contained, no external libs) // Strip UI star markers from input values (Tdarr uses '*' to indicate default values in UI) const stripStar = (value) => { if (typeof value === 'string') { return value.replace(/\*/g, ''); } return value; }; // Sanitize string for safe shell usage (for FFmpeg output files) // Use double quotes which work better with FFmpeg and Tdarr's command construction const sanitizeForShell = (str) => { if (typeof str !== 'string') { throw new TypeError('Input must be a string'); } // Remove null bytes str = str.replace(/\0/g, ''); // Use double quotes and escape any double quotes, backslashes, and dollar signs // This works better with FFmpeg and Tdarr's command parsing // Example: file"name becomes "file\"name" return `"${str.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\$/g, '\\$')}"`; }; // Sanitize filename to remove dangerous characters const sanitizeFilename = (name, maxLength = 100) => { if (typeof name !== 'string') { return 'file'; } // Force extraction of basename (prevents directory traversal) name = path.basename(name); // Remove dangerous characters name = name.replace(/[<>:"|?*\/\\\x00-\x1f\x7f]/g, '_'); // Remove leading/trailing dots and spaces name = name.replace(/^[.\s]+|[.\s]+$/g, ''); // Ensure not empty if (name.length === 0) { name = 'file'; } // Limit length if (name.length > maxLength) { const ext = path.extname(name); const base = path.basename(name, ext); name = base.substring(0, maxLength - ext.length) + ext; } return name; }; // Validate and sanitize language codes const validateLanguageCodes = (codesString, maxCodes = 20) => { if (typeof codesString !== 'string') { return []; } return codesString .split(',') .map(code => code.trim().toLowerCase()) .filter(code => { // Validate format if (code.length === 0 || code.length > 10) return false; if (!/^[a-z0-9-]+$/.test(code)) return false; // Prevent path traversal if (code.includes('..') || code.includes('/')) return false; return true; }) .slice(0, maxCodes); }; // Initialize response first for error handling const response = { processFile: false, preset: '', container: `.${file.container}`, handbrakeMode: false, ffmpegMode: true, reQueueAfter: false, infoLog: '', }; try { inputs = lib.loadDefaultValues(inputs, details); // Sanitize starred defaults Object.keys(inputs).forEach(key => { inputs[key] = stripStar(inputs[key]); }); // Input validation const validateInputs = (inputs) => { const errors = []; const booleanInputs = [ 'includeAudio', 'includeSubtitles', 'standardizeToSRT', 'extractSubtitles', 'removeAfterExtract', 'skipCommentary', 'setDefaultFlags', 'useCCExtractor', 'embedExtractedCC' ]; for (const input of booleanInputs) { if (!VALID_BOOLEAN_VALUES.includes(inputs[input])) { errors.push(`Invalid ${input} value - must be "true" or "false"`); } } return errors; }; const validationErrors = validateInputs(inputs); if (validationErrors.length > 0) { response.infoLog += '❌ Input validation errors:\n'; validationErrors.forEach(error => { response.infoLog += ` - ${error}\n`; }); response.processFile = false; return response; } // Validate language codes const customEnglishCodes = validateLanguageCodes( inputs.customLanguageCodes, MAX_LANGUAGE_CODES ); if (customEnglishCodes.length === 0) { customEnglishCodes.push('eng', 'en', 'english', 'en-us', 'en-gb', 'en-ca', 'en-au'); } if (!Array.isArray(file.ffProbeData?.streams)) { throw new Error('FFprobe was unable to extract any streams info on this file.'); } // Optimize: Only copy what we need instead of deep cloning entire ffProbeData const streams = file.ffProbeData.streams.map((stream, index) => ({ ...stream, typeIndex: index })); const originalOrder = streams.map(s => s.typeIndex); const videoStreams = streams.filter(s => s.codec_type === 'video'); const audioStreams = streams.filter(s => s.codec_type === 'audio'); const subtitleStreams = streams.filter(s => s.codec_type === 'subtitle'); // Filter out BMP attached pictures early (incompatible with MKV) const otherStreams = streams .filter(s => !['video', 'audio', 'subtitle'].includes(s.codec_type)) .filter(stream => { if (stream.disposition?.attached_pic === 1 && stream.codec_name === 'bmp') { response.infoLog += 'ℹ️ Excluding BMP attached picture (unsupported in MKV). '; return false; } return true; }); let reorderedAudio, reorderedSubtitles; if (inputs.includeAudio === 'true') { const [englishAudio, otherAudio] = partitionStreams(audioStreams, s => isEnglishStream(s, customEnglishCodes)); reorderedAudio = [...englishAudio, ...otherAudio]; if (englishAudio.length > 0) { response.infoLog += `✅ ${englishAudio.length} English audio first. `; } } else { reorderedAudio = audioStreams; } if (inputs.includeSubtitles === 'true') { const [englishSubtitles, otherSubtitles] = partitionStreams(subtitleStreams, s => isEnglishStream(s, customEnglishCodes)); reorderedSubtitles = [...englishSubtitles, ...otherSubtitles]; if (englishSubtitles.length > 0) { response.infoLog += `✅ ${englishSubtitles.length} English subs first. `; } } else { reorderedSubtitles = subtitleStreams; } const reorderedStreams = [ ...videoStreams, ...reorderedAudio, ...reorderedSubtitles, ...otherStreams ]; const newOrder = reorderedStreams.map(s => s.typeIndex); const needsReorder = JSON.stringify(originalOrder) !== JSON.stringify(newOrder); let needsConversion = false; let conversionCount = 0; const hasProblematicSubs = subtitleStreams.some(isProblematicSubtitle); if (inputs.standardizeToSRT === 'true' || hasProblematicSubs) { subtitleStreams.forEach(stream => { if (!stream.codec_name) return; if (isUnsupportedSubtitle(stream)) return; if (needsSRTConversion(stream)) { needsConversion = true; conversionCount++; } }); } let extractCommand = ''; let extractCount = 0; let ccExtractedFile = null; let ccActuallyExtracted = false; const extractedFiles = new Set(); const extractionAttempts = new Map(); // Track extraction attempts to prevent infinite loops if (inputs.extractSubtitles === 'true' && subtitleStreams.length > 0) { const { originalLibraryFile } = otherArguments; // CRITICAL: Always use originalLibraryFile.file for extraction paths to avoid infinite loop // On re-queue, file.file points to cache dir, but we need the original library path if (!originalLibraryFile?.file) { response.infoLog += '⚠️ Cannot extract subs: originalLibraryFile not available. '; } else { const baseFile = originalLibraryFile.file; const baseName = buildSafeBasePath(baseFile); for (const stream of subtitleStreams) { if (!stream.codec_name) { response.infoLog += `ℹ️ Skipping subtitle ${stream.typeIndex} (no codec). `; continue; } if (isUnsupportedSubtitle(stream)) { response.infoLog += `ℹ️ Skipping subtitle ${stream.typeIndex} (${stream.codec_name}) incompatible. `; continue; } // Skip bitmap subtitles when extracting to SRT (can't convert bitmap to text) if (IMAGE_SUBTITLE_CODECS.has(stream.codec_name)) { response.infoLog += `ℹ️ Skipping bitmap subtitle ${stream.typeIndex} (${stream.codec_name}) - cannot extract to SRT. `; continue; } if (shouldSkipSubtitle(stream, inputs.skipCommentary)) { const title = stream.tags?.title || 'unknown'; response.infoLog += `ℹ️ Skipping ${title}. `; continue; } const lang = stream.tags?.language || 'unknown'; const safeLang = sanitizeFilename(lang).substring(0, 20); let subsFile = `${baseName}.${safeLang}.srt`; let counter = 1; // Find first available filename that hasn't been queued in this run while (extractedFiles.has(subsFile) && counter < MAX_FILENAME_ATTEMPTS) { subsFile = `${baseName}.${safeLang}.${counter}.srt`; counter++; } // Check if we actually need to extract using improved detection if (needsSubtitleExtraction(subsFile, baseFile, fs)) { // Check extraction attempt count to prevent infinite loops const attemptKey = `${baseFile}:${stream.typeIndex}`; const attempts = extractionAttempts.get(attemptKey) || 0; if (attempts >= MAX_EXTRACTION_ATTEMPTS) { response.infoLog += `⚠️ Skipping ${path.basename(subsFile)} - extraction failed ${MAX_EXTRACTION_ATTEMPTS} times. `; continue; } // File doesn't exist, is incomplete, or is outdated - extract it extractionAttempts.set(attemptKey, attempts + 1); const safeSubsFile = sanitizeForShell(subsFile); extractCommand += ` -map 0:${stream.typeIndex} ${safeSubsFile}`; extractedFiles.add(subsFile); extractCount++; } else { // File exists and is valid, skip extraction response.infoLog += `ℹ️ ${path.basename(subsFile)} already exists, skipping. `; } } if (extractCount > 0) { response.infoLog += `✅ Extracting ${extractCount} subtitle(s). `; } } } if (inputs.useCCExtractor === 'true' && subtitleStreams.some(isClosedCaption)) { const { originalLibraryFile } = otherArguments; // CRITICAL: Use originalLibraryFile.file for CC paths to avoid infinite loop if (!originalLibraryFile?.file) { response.infoLog += '⚠️ Cannot extract CC: originalLibraryFile not available. '; } else { const baseFile = originalLibraryFile.file; const baseName = buildSafeBasePath(baseFile); const ccOut = `${baseName}.cc.srt`; const ccLock = `${ccOut}.lock`; // Cache file existence check const ccFileExists = fileExistsRobust(ccOut, fs); try { // Try to create lock file atomically to prevent race conditions fs.writeFileSync(ccLock, process.pid.toString(), { flag: 'wx' }); try { // We have the lock, check if CC file actually exists if (ccFileExists) { response.infoLog += 'ℹ️ CC file exists. '; if (inputs.embedExtractedCC === 'true') { ccExtractedFile = ccOut; ccActuallyExtracted = false; } } else { // Need to extract, keep the lock (will be cleaned up after extraction) ccExtractedFile = ccOut; ccActuallyExtracted = true; response.infoLog += '✅ Will extract CC via ccextractor. '; } } finally { // Only release lock if we're not extracting (extraction command will clean it up) if (!ccActuallyExtracted && fs.existsSync(ccLock)) { fs.unlinkSync(ccLock); } } } catch (e) { if (e.code === 'EEXIST') { // Another worker has the lock response.infoLog += '⏭️ CC extraction in progress by another worker. '; // Check if file exists (other worker may have just finished) if (ccFileExists && inputs.embedExtractedCC === 'true') { ccExtractedFile = ccOut; ccActuallyExtracted = false; } } else if (e.code === 'EACCES' || e.code === 'EPERM') { // Fatal: permission issue throw new Error(`CC extraction failed: Permission denied - ${e.message}`); } else { // Other error - log and continue response.infoLog += `⚠️ CC lock error: ${e.message}. `; } } } } // Use helper function for complex conditional check if (!needsProcessing(needsReorder, needsConversion, extractCount, ccActuallyExtracted, ccExtractedFile, inputs.embedExtractedCC)) { response.infoLog += '✅ No changes needed.'; return response; } response.processFile = true; response.reQueueAfter = true; if (needsReorder) { response.infoLog += '✅ Reordering streams. '; } if (needsConversion) { if (hasProblematicSubs && inputs.standardizeToSRT !== 'true') { response.infoLog += `✅ Converting ${conversionCount} WebVTT to SRT (compatibility). `; } else { response.infoLog += `✅ Converting ${conversionCount} to SRT. `; } } let command = (inputs.extractSubtitles === 'true' && extractCount > 0) ? '-y ' : ''; command += extractCommand; if (inputs.removeAfterExtract === 'true' && inputs.extractSubtitles === 'true' && extractCount > 0) { response.infoLog += '✅ Removing embedded subs. '; // We proceed to build the map, but we'll filter out subs in the loop. } // Construct the main mapping command based on reordered streams command += ' -c:v copy -c:a copy'; const includedSubtitleStreams = []; let firstEnglishAudioIdx = null; let firstEnglishSubIdx = null; let audioOutputIdx = 0; let subOutputIdx = 0; reorderedStreams.forEach(stream => { // If removing subtitles after extract, skip mapping subtitles from source if (inputs.extractSubtitles === 'true' && inputs.removeAfterExtract === 'true' && stream.codec_type === 'subtitle') { return; } if (stream.codec_type !== 'subtitle') { command += ` -map 0:${stream.typeIndex}`; // Track first English audio for default flag if (stream.codec_type === 'audio' && firstEnglishAudioIdx === null && isEnglishStream(stream, customEnglishCodes)) { firstEnglishAudioIdx = audioOutputIdx; } if (stream.codec_type === 'audio') { audioOutputIdx++; } return; } if (!stream.codec_name) { response.infoLog += `ℹ️ Skipping map for subtitle ${stream.typeIndex} (no codec). `; return; } if (isUnsupportedSubtitle(stream)) { response.infoLog += `ℹ️ Excluding subtitle ${stream.typeIndex} (${stream.codec_name}) for compatibility. `; return; } includedSubtitleStreams.push(stream); command += ` -map 0:${stream.typeIndex}`; // Track first English subtitle for default flag if (firstEnglishSubIdx === null && isEnglishStream(stream, customEnglishCodes)) { firstEnglishSubIdx = subOutputIdx; } subOutputIdx++; }); const allIncludedAreText = includedSubtitleStreams.length > 0 && includedSubtitleStreams.every(s => TEXT_SUBTITLE_CODECS.has(s.codec_name)); const shouldConvertToSRT = (inputs.standardizeToSRT === 'true' || hasProblematicSubs) && allIncludedAreText; if (includedSubtitleStreams.length > 0) { if (shouldConvertToSRT) { command += ' -c:s srt'; } else if (inputs.standardizeToSRT === 'true' && !allIncludedAreText) { response.infoLog += '✅ Mixed subtitle types; using per-stream codec. '; includedSubtitleStreams.forEach((stream, idx) => { if (isTextSubtitle(stream) && stream.codec_name !== 'subrip') { command += ` -c:s:${idx} srt`; } else { command += ` -c:s:${idx} copy`; } }); } else if (hasProblematicSubs && !allIncludedAreText) { includedSubtitleStreams.forEach((stream, idx) => { if (isProblematicSubtitle(stream)) { command += ` -c:s:${idx} srt`; } else { command += ` -c:s:${idx} copy`; } }); } else { command += ' -c:s copy'; } } // Set default flags on first English streams if enabled if (inputs.setDefaultFlags === 'true') { if (firstEnglishAudioIdx !== null) { command += ` -disposition:a:${firstEnglishAudioIdx} default`; response.infoLog += `✅ Set default flag on English audio. `; } if (firstEnglishSubIdx !== null) { command += ` -disposition:s:${firstEnglishSubIdx} default`; response.infoLog += `✅ Set default flag on English subtitle. `; } } if (ccExtractedFile && inputs.embedExtractedCC === 'true') { // Validate CC file exists before attempting to embed (unless we're extracting it in this run) if (ccActuallyExtracted || fs.existsSync(ccExtractedFile)) { const safeCCFile = sanitizeForShell(ccExtractedFile); // calculate index for the new subtitle stream (it will be after all mapped subs) const newSubIdx = includedSubtitleStreams.length; command += ` -i "${safeCCFile}" -map 1:0 -c:s:${newSubIdx} srt`; command += ` -metadata:s:s:${newSubIdx} language=eng`; command += ` -metadata:s:s:${newSubIdx} title="Closed Captions"`; response.infoLog += '✅ Embedding extracted CC. '; } else { response.infoLog += '⚠️ CC file not found, skipping embed. '; } } if (ccActuallyExtracted) { const { originalLibraryFile } = otherArguments; const sourceFile = (originalLibraryFile?.file) || file.file; const baseName = buildSafeBasePath(sourceFile); const ccLock = `${baseName}.cc.srt.lock`; const safeInput = sanitizeForShell(sourceFile); const safeCCFile = sanitizeForShell(ccExtractedFile); const safeLock = sanitizeForShell(ccLock); // Add lock cleanup to command const cleanupCmd = `rm -f ${safeLock}`; const ccCmd = `ccextractor ${safeInput} -o ${safeCCFile}`; response.preset = `${ccCmd}; ${cleanupCmd}; ${command}`; response.infoLog += 'ℹ️ CC extraction will run before main command. '; } else { response.preset = command; } return response; } catch (error) { // Comprehensive error handling response.processFile = false; response.preset = ''; response.reQueueAfter = false; // Provide detailed error information response.infoLog = `💥 Plugin error: ${error.message}\n`; // Add stack trace for debugging (first 5 lines) if (error.stack) { const stackLines = error.stack.split('\n').slice(0, 5).join('\n'); response.infoLog += `Stack trace:\n${stackLines}\n`; } // Log additional context response.infoLog += `File: ${file.file}\n`; response.infoLog += `Container: ${file.container}\n`; return response; } }; module.exports.details = details; module.exports.plugin = plugin;