Public/removeads
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improve pylint score and fix most errors.

Browse Source
This commit is contained in:
Frédéric Tronel
2025-10-25 16:05:25 +02:00
parent efceec0e48
commit 489435a87f
1 changed files with 102 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

191
removeads.py
View File

@@ -1,9 +1,10 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
'''A module to remove parts of video (.e.g advertisements) with single frame precision.'''
import argparse import argparse
import re import re
from sys import exit from sys import exit
from datetime import datetime,timedelta,time from datetime import datetime,timedelta
import coloredlogs, logging import coloredlogs, logging
from functools import cmp_to_key from functools import cmp_to_key
from subprocess import Popen, PIPE from subprocess import Popen, PIPE
@@ -35,7 +36,7 @@ from dataclasses import dataclass, field
# Concatenate all raw H264 in a giant one (like cat), and the same for timestamps of video frames (to keep # Concatenate all raw H264 in a giant one (like cat), and the same for timestamps of video frames (to keep
# sound and video synchronized). # sound and video synchronized).
# Then use mkvmerge to remux the H264 track and the rest of tracks. # Then use mkvmerge to remux the H264 track and the rest of tracks.
# MKVmerge concatenate is able to concatenate different SPS/PPS data into a bigger Private Codec Data. # MKVmerge "concatenate" subcommand is able to concatenate different SPS/PPS data into a bigger Private Codec Data.
# However, this is proved to be not reliable. Sometimes it results in a AVC context containing a single SPS/PPS. # However, this is proved to be not reliable. Sometimes it results in a AVC context containing a single SPS/PPS.
# So we have to rely on a manual parsing of the H264 AVC context of original movie # So we have to rely on a manual parsing of the H264 AVC context of original movie
# and the ones produced for headers and trailers, and then merging them into a bigger AVC context. # and the ones produced for headers and trailers, and then merging them into a bigger AVC context.
@@ -50,15 +51,15 @@ def checkRequiredTools():
optional = ['mkvextract', 'vobsubocr','tesseract'] optional = ['mkvextract', 'vobsubocr','tesseract']
for tool in required: for tool in required:
path = which(tool) path = which(tool)
if path == None: if path is None:
logger.error('Required tool: %s is missing.' % tool) logger.error('Required tool: %s is missing.',tool)
exit(-1) exit(-1)
else: else:
paths[tool] = path paths[tool] = path
for tool in optional: for tool in optional:
path = which(tool) path = which(tool)
if path == None: if path is None:
logger.info('Optional tool: %s is missing.' % tool) logger.info('Optional tool: %s is missing.',tool)
allOptionalTools = False allOptionalTools = False
else: else:
paths[tool] = path paths[tool] = path
@@ -83,9 +84,9 @@ def getTesseractSupportedLang(tesseract):
pass pass
tesseract.wait() tesseract.wait()
if tesseract.returncode != 0: if tesseract.returncode != 0:
logger.error("Tesseract returns an error code: %d" % tesseract.returncode) logger.error("Tesseract returns an error code: %d",tesseract.returncode)
return None return None
return res return res
@@ -105,7 +106,8 @@ def getFrameRate(ffprobe, inputFile):
maxTs = None maxTs = None
interlaced = False interlaced = False
params = [ffprobe, '-loglevel', 'quiet', '-select_streams', 'v', '-show_frames', '-read_intervals', '00%+30', '-of', 'json', '/proc/self/fd/%d' % infd] params = [ffprobe, '-loglevel', 'quiet', '-select_streams', 'v', '-show_frames',
'-read_intervals', '00%+30', '-of', 'json', '/proc/self/fd/%d' % infd]
env = {**os.environ, 'LANG': 'C'} env = {**os.environ, 'LANG': 'C'}
with Popen(params, stdout=PIPE, close_fds=False, env=env) as ffprobe: with Popen(params, stdout=PIPE, close_fds=False, env=env) as ffprobe:
out, _ = ffprobe.communicate() out, _ = ffprobe.communicate()
@@ -117,9 +119,9 @@ def getFrameRate(ffprobe, inputFile):
interlaced = True interlaced = True
if 'pts_time' in frame: if 'pts_time' in frame:
ts = float(frame['pts_time']) ts = float(frame['pts_time'])
if minTs == None: if minTs is None:
minTs = ts minTs = ts
if maxTs == None: if maxTs is None:
maxTs = ts maxTs = ts
if ts < minTs: if ts < minTs:
minTs = ts minTs = ts
@@ -135,7 +137,7 @@ def getFrameRate(ffprobe, inputFile):
ffprobe.wait() ffprobe.wait()
if ffprobe.returncode != 0: if ffprobe.returncode != 0:
logger.error("ffprobe returns an error code: %d" % ffprobe.returncode) logger.error("ffprobe returns an error code: %d", ffprobe.returncode)
return None return None
frameRate1 = nbFrames1/(maxTs-minTs) frameRate1 = nbFrames1/(maxTs-minTs)
@@ -143,12 +145,12 @@ def getFrameRate(ffprobe, inputFile):
if abs(frameRate1 - frameRate2) > 0.2: if abs(frameRate1 - frameRate2) > 0.2:
if not interlaced: if not interlaced:
logger.error('Video is not interlaced and the disperancy between frame rates is too big: %f / %f' % (frameRate1, frameRate2)) logger.error('Video is not interlaced and the disperancy between frame rates is too big: %f / %f', frameRate1, frameRate2)
return None return None
if abs(frameRate1*2 - frameRate2) < 0.2: if abs(frameRate1*2 - frameRate2) < 0.2:
return frameRate2/2 return frameRate2/2
else: else:
logger.error('Video is interlaced and the disperancy between frame rates is too big: %f / %f' % (frameRate1, frameRate2)) logger.error('Video is interlaced and the disperancy between frame rates is too big: %f / %f', frameRate1, frameRate2)
return None return None
else: else:
return frameRate2 return frameRate2
@@ -160,7 +162,8 @@ def getSubTitlesTracks(ffprobe, mkvPath):
tracks={} tracks={}
nbSubTitles = 0 nbSubTitles = 0
with Popen([ffprobe, '-loglevel', 'quiet', '-select_streams', 's', '-show_entries', 'stream=index,codec_name:stream_tags=language', '-of', 'json', mkvPath], stdout=PIPE) as ffprobe: with Popen([ffprobe, '-loglevel', 'quiet', '-select_streams', 's', '-show_entries',
'stream=index,codec_name:stream_tags=language', '-of', 'json', mkvPath], stdout=PIPE) as ffprobe:
out, _ = ffprobe.communicate() out, _ = ffprobe.communicate()
out = json.load(BytesIO(out)) out = json.load(BytesIO(out))
if 'streams' in out: if 'streams' in out:
@@ -170,10 +173,10 @@ def getSubTitlesTracks(ffprobe, mkvPath):
lang = stream['tags']['language'] lang = stream['tags']['language']
if codec == 'dvd_subtitle': if codec == 'dvd_subtitle':
if lang not in tracks: if lang not in tracks:
tracks[lang] = [track] tracks[lang] = [index]
else: else:
l = tracks[lang] l = tracks[lang]
l.append(track) l.append(index)
tracks[lang] = l tracks[lang] = l
else: else:
return None return None
@@ -181,7 +184,7 @@ def getSubTitlesTracks(ffprobe, mkvPath):
ffprobe.wait() ffprobe.wait()
if ffprobe.returncode != 0: if ffprobe.returncode != 0:
logger.error("ffprobe returns an error code: %d" % ffprobe.returncode) logger.error("ffprobe returns an error code: %d", ffprobe.returncode)
return None return None
return tracks return tracks
@@ -199,7 +202,7 @@ def extractSRT(mkvextract, fileName, subtitles, langs):
if iso in langs: if iso in langs:
ocrlang = langs[iso] ocrlang = langs[iso]
else: else:
logger.warning("Language not supported by Tesseract: %s" % iso.name) logger.warning("Language not supported by Tesseract: %s", iso.name)
ocrlang ='osd' ocrlang ='osd'
if len(subtitles[lang]) == 1: if len(subtitles[lang]) == 1:
@@ -221,7 +224,7 @@ def extractSRT(mkvextract, fileName, subtitles, langs):
if line.startswith('Progress :'): if line.startswith('Progress :'):
p = re.compile('^Progress : (?P<progress>[0-9]{1,3})%$') p = re.compile('^Progress : (?P<progress>[0-9]{1,3})%$')
m = p.match(line) m = p.match(line)
if m == None: if m is None:
logger.error('Impossible to parse progress') logger.error('Impossible to parse progress')
pb.update(int(m['progress'])-pb.n) pb.update(int(m['progress'])-pb.n)
pb.update(100-pb.n) pb.update(100-pb.n)
@@ -238,7 +241,7 @@ def extractSRT(mkvextract, fileName, subtitles, langs):
logger.warning('Mkvextract returns warning') logger.warning('Mkvextract returns warning')
return res return res
else: else:
logger.error('Mkvextract returns an error code: %d' % extract.returncode) logger.error('Mkvextract returns an error code: %d', extract.returncode)
return None return None
def doOCR(vobsubocr, idxs, duration, temporaries, dumpMemFD=False): def doOCR(vobsubocr, idxs, duration, temporaries, dumpMemFD=False):
@@ -273,13 +276,13 @@ def doOCR(vobsubocr, idxs, duration, temporaries, dumpMemFD=False):
status = ocr.wait() status = ocr.wait()
if status != 0: if status != 0:
logger.error('OCR failed with status code: %d' % status) logger.error('OCR failed with status code: %d', status)
if dumpMemFD: if dumpMemFD:
try: try:
dumpSrt = open(srtname,'w') dumpSrt = open(srtname,'w')
except IOError: except IOError:
logger.error('Impossible to create file: %s' % srtname) logger.error('Impossible to create file: %s', srtname)
return None return None
lseek(srtfd, 0, SEEK_SET) lseek(srtfd, 0, SEEK_SET)
@@ -342,7 +345,7 @@ def getCodecPrivateDataFromMKV(mkvinfo, inputFile):
if m != None: if m != None:
size = int(m.group('size')) size = int(m.group('size'))
position = int(m.group('position')) position = int(m.group('position'))
logger.debug("Found codec private data at position: %s, size: %d" % (position, size)) logger.debug("Found codec private data at position: %s, size: %d", position, size)
found = True found = True
mkvinfo.wait() mkvinfo.wait()
break break
@@ -569,7 +572,7 @@ def parseScalingList(buf, bitPosition, size):
# The ISO/IEC H.264-201602 seems to take into account the case where the end of the deltas list is full of zeroes. # The ISO/IEC H.264-201602 seems to take into account the case where the end of the deltas list is full of zeroes.
def writeScalingList(buf, bitPosition, size, matrix, optimized=False): def writeScalingList(buf, bitPosition, size, matrix, optimized=False):
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
logger.debug('Dumping matrix: %s of size: %d, size parameter: %d.' % (matrix, len(matrix), size)) logger.debug('Dumping matrix: %s of size: %d, size parameter: %d.', matrix, len(matrix), size)
prev = 8 prev = 8
deltas = [] deltas = []
@@ -846,9 +849,13 @@ class SPS:
self.scaling_list={} self.scaling_list={}
self.offset_for_ref_frame={} self.offset_for_ref_frame={}
# Compute options to pass to ffmpeg so as to reproduce the same SPS.
# TODO: ... # TODO: ...
# Compute options to pass to ffmpeg so as to reproduce the same SPS.
# Very complex since some codec configuration are not provided by ffmpeg and/or libx264.
# This is only an attempt.
def ffmpegOptions(self, videoID=0): def ffmpegOptions(self, videoID=0):
logger = logging.getLogger(__name__)
x264opts = [] x264opts = []
if self.profile_idc in [ 0x42, 0x4D, 0x64, 0x6E, 0x7A, 0xF4, 0x2C]: if self.profile_idc in [ 0x42, 0x4D, 0x64, 0x6E, 0x7A, 0xF4, 0x2C]:
@@ -865,14 +872,14 @@ class SPS:
elif self.profile_idc == 0xF4: elif self.profile_idc == 0xF4:
profile = 'high444' profile = 'high444'
else: else:
logger.error('Unknow profile: %x' % self.profile) logger.error('Unknow profile: %x', self.profile_idc)
return [] return []
level = '%d.%d' % (floor(self.level/10), self.level % 10) level = '%d.%d' % (floor(self.level_idc/10), self.level_idc % 10)
x264opts.extend(['sps-id=%d' % self.seq_parameter_set_id] ) x264opts.extend(['sps-id=%d' % self.seq_parameter_set_id] )
if self.bit_depth_chroma_minus8 not in [0,1,2,4,6,8]: if self.bit_depth_chroma_minus8 not in [0,1,2,4,6,8]:
logger.error('Bit depth of chrominance is not supported: %d' % (self.bit_depth_chroma_minus8+8)) logger.error('Bit depth of chrominance is not supported: %d', self.bit_depth_chroma_minus8+8)
return [] return []
if self.chroma_format_idc in range(0,4): if self.chroma_format_idc in range(0,4):
@@ -889,7 +896,7 @@ class SPS:
# YUV:4:4:4 # YUV:4:4:4
pass pass
else: else:
logger.error('Unknow chrominance format: %x' % self.profile) logger.error('Unknow chrominance format: %x', self.chroma_format_idc)
return [] return []
res = ['-profile:v:%d' % videoID, self.profile_idc, '-level:v:%d' % videoID, level] res = ['-profile:v:%d' % videoID, self.profile_idc, '-level:v:%d' % videoID, level]
@@ -978,7 +985,7 @@ class SPS:
if self.vui_parameters_present_flag: if self.vui_parameters_present_flag:
self.vui = VUI() self.vui = VUI()
bitPosition = self.vui.fromBytes(buf,bitPosition) bitPosition = self.vui.fromBytes(buf,bitPosition)
logger.debug('VUI present: %s' % self.vui) logger.debug('VUI present: %s', self.vui)
logger.debug('Parse end of SPS. Bit position: %d. Remaining bytes: %s.' % (bitPosition, hexdump.dump(buf[floor(bitPosition/8):], sep=':'))) logger.debug('Parse end of SPS. Bit position: %d. Remaining bytes: %s.' % (bitPosition, hexdump.dump(buf[floor(bitPosition/8):], sep=':')))
@@ -1035,7 +1042,7 @@ class SPS:
bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.num_ref_frames_in_pic_order_cnt_cycle) bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.num_ref_frames_in_pic_order_cnt_cycle)
for i in range(0, self.num_ref_frames_in_pic_order_cnt_cycle): for i in range(0, self.num_ref_frames_in_pic_order_cnt_cycle):
v = self.offset_for_ref_frame[i] v = self.offset_for_ref_frame[i]
bitPosition, v = writeUnsignedExpGolomb(buf, bitPosition) bitPosition = writeUnsignedExpGolomb(buf, bitPosition, v)
bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.max_num_ref_frames) bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.max_num_ref_frames)
bitPosition = writeBoolean(buf, bitPosition, self.gaps_in_frame_num_value_allowed_flag) bitPosition = writeBoolean(buf, bitPosition, self.gaps_in_frame_num_value_allowed_flag)
bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.pic_width_in_mbs_minus1) bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.pic_width_in_mbs_minus1)
@@ -1052,9 +1059,9 @@ class SPS:
bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.frame_crop_bottom_offset) bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.frame_crop_bottom_offset)
bitPosition = writeBoolean(buf, bitPosition, self.vui_parameters_present_flag) bitPosition = writeBoolean(buf, bitPosition, self.vui_parameters_present_flag)
if self.vui_parameters_present_flag: if self.vui_parameters_present_flag:
logger.debug('SPS has VUI. Writing VUI at position: %d' % bitPosition) logger.debug('SPS has VUI. Writing VUI at position: %d', bitPosition)
bitPosition = self.vui.toBytes(buf, bitPosition) bitPosition = self.vui.toBytes(buf, bitPosition)
logger.debug('VUI written. New bit position: %d' % bitPosition) logger.debug('VUI written. New bit position: %d', bitPosition)
bitPosition = writeRBSPTrailingBits(buf, bitPosition) bitPosition = writeRBSPTrailingBits(buf, bitPosition)
@@ -1129,7 +1136,7 @@ class PPS:
elif self.slice_group_map_type == 2: elif self.slice_group_map_type == 2:
for i in range(0, self.num_slice_groups_minus1): for i in range(0, self.num_slice_groups_minus1):
bitPosition, v = readUnsignedExpGolomb(buf, bitPosition) bitPosition, v = readUnsignedExpGolomb(buf, bitPosition)
self.top_left.append[i] = v self.top_left[i] = v
bitPosition, v = readUnsignedExpGolomb(buf, bitPosition) bitPosition, v = readUnsignedExpGolomb(buf, bitPosition)
self.bottom_right[i] = v self.bottom_right[i] = v
elif self.slice_group_map_type in [3,4,5]: elif self.slice_group_map_type in [3,4,5]:
@@ -1204,7 +1211,7 @@ class PPS:
bitPosition = writeUnsignedExpGolomb(buf, bitPosition, v) bitPosition = writeUnsignedExpGolomb(buf, bitPosition, v)
elif self.slice_group_map_type == 2: elif self.slice_group_map_type == 2:
for i in range(0, self.num_slice_groups_minus1): for i in range(0, self.num_slice_groups_minus1):
v = self.top_left.append[i] v = self.top_left[i]
bitPosition = writeUnsignedExpGolomb(buf, bitPosition, v) bitPosition = writeUnsignedExpGolomb(buf, bitPosition, v)
v = self.bottom_right[i] v = self.bottom_right[i]
bitPosition = writeUnsignedExpGolomb(buf, bitPosition, v) bitPosition = writeUnsignedExpGolomb(buf, bitPosition, v)
@@ -1545,7 +1552,7 @@ def parseMKVTree(mkvinfo, inputFile):
prevDepth = -1 prevDepth = -1
for line in out.splitlines(): for line in out.splitlines():
m = p.match(line) m = p.match(line)
if m == None: if m is None:
logger.error("Impossible to match line: %s" % line) logger.error("Impossible to match line: %s" % line)
else: else:
position = int(m.group('position')) position = int(m.group('position'))
@@ -1668,7 +1675,7 @@ def changeEBMLElementSize(inputFile, position, addendum):
mask = mask>>1 mask = mask>>1
if not found: if not found:
logger.error('Size of element type cannot be determined: %b' % elementType) logger.error('Size of element type cannot be determined: %d', elementType)
exit(-1) exit(-1)
# We seek to size # We seek to size
@@ -1689,10 +1696,10 @@ def changeEBMLElementSize(inputFile, position, addendum):
mask = mask>>1 mask = mask>>1
if not found: if not found:
logger.error('Size of data size cannot be determined: %b' % sizeHead) logger.error('Size of data size cannot be determined: %d', sizeHead)
exit(-1) exit(-1)
else: else:
logger.info('Size of data size: %d.' % sizeOfDataSize) logger.info('Size of data size: %d.', sizeOfDataSize)
lseek(infd, position, SEEK_SET) lseek(infd, position, SEEK_SET)
oldSizeBuf = read(infd, sizeOfDataSize) oldSizeBuf = read(infd, sizeOfDataSize)
@@ -1842,7 +1849,7 @@ def getMovieDuration(ffprobe, inputFile):
return None return None
# ffprobe -loglevel quiet -select_streams v:0 -show_entries stream=width,height -of json ./talons.ts # ffprobe -loglevel quiet -select_streams v:0 -show_entries stream=width,height -of json ./example.ts
def getVideoDimensions(ffprobe, inputFile): def getVideoDimensions(ffprobe, inputFile):
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -1902,7 +1909,7 @@ def parseTimestamp(ts):
tsRegExp = r'^(?P<hour>[0-9]{1,2}):(?P<minute>[0-9]{1,2}):(?P<second>[0-9]{1,2})(\.(?P<us>[0-9]{1,6}))?$' tsRegExp = r'^(?P<hour>[0-9]{1,2}):(?P<minute>[0-9]{1,2}):(?P<second>[0-9]{1,2})(\.(?P<us>[0-9]{1,6}))?$'
p = re.compile(tsRegExp) p = re.compile(tsRegExp)
m = p.match(ts) m = p.match(ts)
if m == None: if m is None:
logger.warning("Impossible to parse timestamp: %s" % ts) logger.warning("Impossible to parse timestamp: %s" % ts)
return None return None
@@ -1943,7 +1950,7 @@ def parseTimeInterval(interval):
intervalRegExp = r'^(?P<hour1>[0-9]{1,2}):(?P<minute1>[0-9]{1,2}):(?P<second1>[0-9]{1,2})(\.(?P<ms1>[0-9]{1,3}))?-(?P<hour2>[0-9]{1,2}):(?P<minute2>[0-9]{1,2}):(?P<second2>[0-9]{1,2})(\.(?P<ms2>[0-9]{1,3}))?$' intervalRegExp = r'^(?P<hour1>[0-9]{1,2}):(?P<minute1>[0-9]{1,2}):(?P<second1>[0-9]{1,2})(\.(?P<ms1>[0-9]{1,3}))?-(?P<hour2>[0-9]{1,2}):(?P<minute2>[0-9]{1,2}):(?P<second2>[0-9]{1,2})(\.(?P<ms2>[0-9]{1,3}))?$'
p = re.compile(intervalRegExp) p = re.compile(intervalRegExp)
m = p.match(interval) m = p.match(interval)
if m == None: if m is None:
logger.error("Impossible to parse time interval") logger.error("Impossible to parse time interval")
return None return None
@@ -2046,7 +2053,7 @@ def ffmpegConvert(ffmpeg, ffprobe, inputFile, inputFormat, outputFile, outputFor
params.extend(['-r:0', '25', '-f', outputFormat, '/proc/self/fd/%d' % outfd]) params.extend(['-r:0', '25', '-f', outputFormat, '/proc/self/fd/%d' % outfd])
logger.debug('Executing %s' % params) logger.debug('Executing %s' % params)
with Popen(params, stdout=PIPE, close_fds=False) as ffmpeg: with Popen(params, stdout=PIPE, close_fds=False) as ffmpeg:
pb = tqdm(TextIOWrapper(ffmpeg.stdout, encoding="utf-8"), total=int(duration/timedelta(seconds=1)), unit='s', desc='Conversion') pb = tqdm(TextIOWrapper(ffmpeg.stdout, encoding="utf-8"), total=int(duration/timedelta(seconds=1)), unit='s', desc='Conversion')
for line in pb: for line in pb:
@@ -2058,27 +2065,31 @@ def ffmpegConvert(ffmpeg, ffprobe, inputFile, inputFormat, outputFile, outputFor
pb.update() pb.update()
status = ffmpeg.wait() status = ffmpeg.wait()
if status != 0: if status != 0:
logger.error('Conversion failed with status code: %d' % status) logger.error('Conversion failed with status code: %d', status)
def getTSFrame(frame): def getTSFrame(frame):
logger = logging.getLogger(__name__)
if 'pts_time' in frame: if 'pts_time' in frame:
pts_time = float(frame['pts_time']) pts_time = float(frame['pts_time'])
elif 'pkt_pts_time' in frame: elif 'pkt_pts_time' in frame:
pts_time = float(frame['pkt_pts_time']) pts_time = float(frame['pkt_pts_time'])
else: else:
logger.error('Impossible to find timestamp of frame %s' % frame) logger.error('Impossible to find timestamp of frame %s', frame)
return None return None
ts = timedelta(seconds=pts_time) ts = timedelta(seconds=pts_time)
return ts return ts
def getPacketDuration(packet): def getPacketDuration(packet):
logger = logging.getLogger(__name__)
if 'duration' in packet: if 'duration' in packet:
duration = int(packet['duration']) duration = int(packet['duration'])
elif 'pkt_duration' in packet: elif 'pkt_duration' in packet:
duration = int(packet['pkt_duration']) duration = int(packet['pkt_duration'])
else: else:
logger.error('Impossible to find duration of packet %s' % packet) logger.error('Impossible to find duration of packet %s', packet)
return None return None
return duration return duration
@@ -2107,7 +2118,7 @@ def getFramesInStream(ffprobe, inputFile, begin, end, streamKind, subStreamId=0)
frames = frames['frames'] frames = frames['frames']
for frame in frames: for frame in frames:
ts = getTSFrame(frame) ts = getTSFrame(frame)
if ts == None: if ts is None:
return None return None
if begin <= ts and ts <= end: if begin <= ts and ts <= end:
tmp[ts]=frame tmp[ts]=frame
@@ -2134,12 +2145,12 @@ def getNearestIDRFrame(ffprobe, inputFile, timestamp, before=True, delta=timedel
infd = inputFile.fileno() infd = inputFile.fileno()
set_inheritable(infd, True) set_inheritable(infd, True)
logger.debug('Looking for IDR frame in [%s, %s]' % (tbegin, tend)) logger.debug('Looking for IDR frame in [%s, %s]', tbegin, tend)
idrs = [] idrs = []
# Retains only IDR frame # Retains only IDR frame
with Popen([ffprobe, '-loglevel', 'quiet', '-read_intervals', ('%s%%%s' %(begin, end)), '-skip_frame', 'nokey', '-show_entries', 'frame', '-select_streams', 'v:0', '-of', 'json', '/proc/self/fd/%d' % infd], stdout=PIPE, close_fds=False) as ffprobe: with Popen([ffprobe, '-loglevel', 'quiet', '-read_intervals', ('%s%%%s' %(tbegin, tend)), '-skip_frame', 'nokey', '-show_entries', 'frame', '-select_streams', 'v:0', '-of', 'json', '/proc/self/fd/%d' % infd], stdout=PIPE, close_fds=False) as ffprobe:
out, _ = ffprobe.communicate() out, _ = ffprobe.communicate()
frames = json.load(BytesIO(out)) frames = json.load(BytesIO(out))
status = ffprobe.wait() status = ffprobe.wait()
@@ -2151,12 +2162,12 @@ def getNearestIDRFrame(ffprobe, inputFile, timestamp, before=True, delta=timedel
frames = frames['frames'] frames = frames['frames']
for frame in frames: for frame in frames:
ts = getTSFrame(frame) ts = getTSFrame(frame)
if ts == None: if ts is None:
return None return None
if begin <= ts and ts <= end: if tbegin <= ts and ts <= tend:
idrs.append(frame) idrs.append(frame)
else: else:
logger.error('Impossible to retrieve IDR frames inside file around [%s,%s]' % (begin, end)) logger.error('Impossible to retrieve IDR frames inside file around [%s,%s]', tbegin, tend)
return None return None
@@ -2186,7 +2197,7 @@ def getNearestIFrame(ffprobe, inputFile, timestamp, before=True, deltaMax=timede
logger.debug('Looking for an iframe in [%s, %s]' % (tbegin, tend)) logger.debug('Looking for an iframe in [%s, %s]' % (tbegin, tend))
frames = getFramesInStream(ffprobe, inputFile=inputFile, begin=tbegin, end=tend, streamKind='v') frames = getFramesInStream(ffprobe, inputFile=inputFile, begin=tbegin, end=tend, streamKind='v')
if frames == None: if frames is None:
logger.debug('Found no frame in [%s, %s]' % (tbegin, tend)) logger.debug('Found no frame in [%s, %s]' % (tbegin, tend))
delta+=timedelta(seconds=1) delta+=timedelta(seconds=1)
continue continue
@@ -2199,7 +2210,7 @@ def getNearestIFrame(ffprobe, inputFile, timestamp, before=True, deltaMax=timede
found = False found = False
for frame in iframes: for frame in iframes:
ts = getTSFrame(frame) ts = getTSFrame(frame)
if ts == None: if ts is None:
logger.warning('I-frame with no timestamp: %s' % frame) logger.warning('I-frame with no timestamp: %s' % frame)
continue continue
@@ -2223,7 +2234,7 @@ def getNearestIFrame(ffprobe, inputFile, timestamp, before=True, deltaMax=timede
nbFrames = 0 nbFrames = 0
for frame in frames: for frame in frames:
ts = getTSFrame(frame) ts = getTSFrame(frame)
if ts == None: if ts is None:
logger.warning('Frame without timestamp: %s' % frame) logger.warning('Frame without timestamp: %s' % frame)
continue continue
@@ -2263,7 +2274,7 @@ def extractMKVPart(mkvmerge, inputFile, outputFile, begin, end):
if line.startswith('Progress :'): if line.startswith('Progress :'):
p = re.compile('^Progress : (?P<progress>[0-9]{1,3})%$') p = re.compile('^Progress : (?P<progress>[0-9]{1,3})%$')
m = p.match(line) m = p.match(line)
if m == None: if m is None:
logger.error('Impossible to parse progress') logger.error('Impossible to parse progress')
pb.update(int(m['progress'])-pb.n) pb.update(int(m['progress'])-pb.n)
elif line.startswith('Warning'): elif line.startswith('Warning'):
@@ -2424,7 +2435,8 @@ def extractAllStreams(ffmpeg, ffprobe, inputFile, begin, end, streams, filesPref
level = int(stream['level']) level = int(stream['level'])
level = '%d.%d' % (floor(level/10), level%10) level = '%d.%d' % (floor(level/10), level%10)
chromaLocation = stream['chroma_location'] chromaLocation = stream['chroma_location']
fieldOrder = stream['field_order'] fieldOrder = stream
interlacedOptions = []
if fieldOrder == 'progressive': if fieldOrder == 'progressive':
interlacedOptions = ['-field_order', '0'] interlacedOptions = ['-field_order', '0']
elif fieldOrder == 'tt': elif fieldOrder == 'tt':
@@ -2441,7 +2453,7 @@ def extractAllStreams(ffmpeg, ffprobe, inputFile, begin, end, streams, filesPref
# https://superuser.com/questions/907933/correct-aspect-ratio-without-re-encoding-video-file # https://superuser.com/questions/907933/correct-aspect-ratio-without-re-encoding-video-file
codec = stream['codec_name'] codec = stream['codec_name']
imagesBytes, memfd = extractPictures(ffmpeg, inputFile=inputFile, begin=begin, nbFrames=nbFrames, width=width, height=height) imagesBytes, memfd = extractPictures(ffmpeg, inputFile=inputFile, begin=begin, nbFrames=nbFrames, width=width, height=height)
if imagesBytes == None: if imagesBytes is None:
exit(-1) exit(-1)
memfds.append(memfd) memfds.append(memfd)
@@ -2475,7 +2487,7 @@ def extractAllStreams(ffmpeg, ffprobe, inputFile, begin, end, streams, filesPref
logger.debug("Found %d packets to be extracted from audio track." % nbPackets) logger.debug("Found %d packets to be extracted from audio track." % nbPackets)
if(nbPackets > 0): if(nbPackets > 0):
packetDuration = getPacketDuration(packets[0]) packetDuration = getPacketDuration(packets[0])
if packetDuration == None: if packetDuration is None:
return None return None
else: else:
packetDuration = 0 packetDuration = 0
@@ -2486,7 +2498,7 @@ def extractAllStreams(ffmpeg, ffprobe, inputFile, begin, end, streams, filesPref
soundBytes, memfd = extractSound(ffmpeg=ffmpeg, inputFile=inputFile, begin=begin, nbPackets=nbPackets, packetDuration=packetDuration, outputFileName=tmpname, sampleRate=sampleRate, nbChannels=nbChannels) soundBytes, memfd = extractSound(ffmpeg=ffmpeg, inputFile=inputFile, begin=begin, nbPackets=nbPackets, packetDuration=packetDuration, outputFileName=tmpname, sampleRate=sampleRate, nbChannels=nbChannels)
if soundBytes == None: if soundBytes is None:
exit(-1) exit(-1)
memfds.append(memfd) memfds.append(memfd)
@@ -2661,7 +2673,7 @@ def mergeMKVs(mkvmerge, inputs, outputName, concatenate=True, timestamps=None):
if line.startswith('Progress :'): if line.startswith('Progress :'):
p = re.compile('^Progress : (?P<progress>[0-9]{1,3})%$') p = re.compile('^Progress : (?P<progress>[0-9]{1,3})%$')
m = p.match(line) m = p.match(line)
if m == None: if m is None:
logger.error('Impossible to parse progress') logger.error('Impossible to parse progress')
pb.n = int(m['progress']) pb.n = int(m['progress'])
pb.update() pb.update()
@@ -2728,7 +2740,7 @@ def extractTrackFromMKV(mkvextract, inputFile, index, outputFile, timestamps):
if line.startswith('Progress :'): if line.startswith('Progress :'):
p = re.compile('^Progress : (?P<progress>[0-9]{1,3})%$') p = re.compile('^Progress : (?P<progress>[0-9]{1,3})%$')
m = p.match(line) m = p.match(line)
if m == None: if m is None:
logger.error('Impossible to parse progress') logger.error('Impossible to parse progress')
pb.update(int(m['progress'])-pb.n) pb.update(int(m['progress'])-pb.n)
pb.update(100-pb.n) pb.update(100-pb.n)
@@ -2763,7 +2775,7 @@ def removeVideoTracksFromMKV(mkvmerge, inputFile, outputFile):
if line.startswith('Progress :'): if line.startswith('Progress :'):
p = re.compile('^Progress : (?P<progress>[0-9]{1,3})%$') p = re.compile('^Progress : (?P<progress>[0-9]{1,3})%$')
m = p.match(line) m = p.match(line)
if m == None: if m is None:
logger.error('Impossible to parse progress') logger.error('Impossible to parse progress')
pb.update(int(m['progress'])-pb.n) pb.update(int(m['progress'])-pb.n)
pb.update(100-pb.n) pb.update(100-pb.n)
@@ -2810,7 +2822,7 @@ def remuxSRTSubtitles(mkvmerge, inputFile, outputFileName, subtitles):
if line.startswith('Progress :'): if line.startswith('Progress :'):
p = re.compile('^Progress : (?P<progress>[0-9]{1,3})%$') p = re.compile('^Progress : (?P<progress>[0-9]{1,3})%$')
m = p.match(line) m = p.match(line)
if m == None: if m is None:
logger.error('Impossible to parse progress') logger.error('Impossible to parse progress')
pb.n = int(m['progress']) pb.n = int(m['progress'])
pb.update() pb.update()
@@ -2844,7 +2856,7 @@ def concatenateH264Parts(h264parts, output):
lseek(fd, 0, SEEK_SET) lseek(fd, 0, SEEK_SET)
while True: while True:
buf = read(fd, 1000000) buf = read(fd, 1000000)
if buf == None or len(buf) == 0: if buf is None or len(buf) == 0:
break break
pos = 0 pos = 0
while pos < len(buf): while pos < len(buf):
@@ -2882,6 +2894,7 @@ def concatenateH264TSParts(h264TSParts, output):
first = False first = False
def doCoarseProcessing(ffmpeg, ffprobe, mkvmerge, inputFile, begin, end, nbFrames, frameRate, filesPrefix, streams, width, height, temporaries, dumpMemFD): def doCoarseProcessing(ffmpeg, ffprobe, mkvmerge, inputFile, begin, end, nbFrames, frameRate, filesPrefix, streams, width, height, temporaries, dumpMemFD):
logger = logging.getLogger(__name__)
# Internal video with all streams (video, audio and subtitles) # Internal video with all streams (video, audio and subtitles)
internalMKVName = '%s.mkv' % filesPrefix internalMKVName = '%s.mkv' % filesPrefix
@@ -2889,7 +2902,7 @@ def doCoarseProcessing(ffmpeg, ffprobe, mkvmerge, inputFile, begin, end, nbFrame
try: try:
internalMKV = open(internalMKVName, 'w+') internalMKV = open(internalMKVName, 'w+')
except IOError: except IOError:
logger.error('Impossible to create file: %s' % internalMKVName) logger.error('Impossible to create file: %s', internalMKVName)
exit(-1) exit(-1)
# Extract internal part of MKV # Extract internal part of MKV
@@ -2927,7 +2940,7 @@ def main():
logger.error('--coarse and threshold arguments are exclusive.') logger.error('--coarse and threshold arguments are exclusive.')
exit(-1) exit(-1)
if (not args.coarse) and args.threshold == None: if (not args.coarse) and args.threshold is None:
args.threshold = 0 args.threshold = 0
allOptionalTools, paths = checkRequiredTools() allOptionalTools, paths = checkRequiredTools()
@@ -2943,7 +2956,7 @@ def main():
# Parse each interval # Parse each interval
for interval in intervals: for interval in intervals:
ts1, ts2 = parseTimeInterval(interval) ts1, ts2 = parseTimeInterval(interval)
if ts1 == None or ts2 == None: if ts1 is None or ts2 is None:
logger.error("Illegal time interval: %s" % interval) logger.error("Illegal time interval: %s" % interval)
exit(-1) exit(-1)
parts.append((ts1,ts2)) parts.append((ts1,ts2))
@@ -2975,15 +2988,15 @@ def main():
formatOfFile = getFormat(paths['ffprobe'], inputFile) formatOfFile = getFormat(paths['ffprobe'], inputFile)
if formatOfFile == None: if formatOfFile is None:
exit(-1) exit(-1)
duration = timedelta(seconds=float(formatOfFile['duration'])) duration = timedelta(seconds=float(formatOfFile['duration']))
logger.info("Durée de l'enregistrement: %s" % duration) logger.info("Durée de l'enregistrement: %s" % duration)
if args.framerate == None: if args.framerate is None:
frameRate = getFrameRate(paths['ffprobe'], inputFile) frameRate = getFrameRate(paths['ffprobe'], inputFile)
if frameRate == None: if frameRate is None:
logger.error('Impossible to estimate frame rate !') logger.error('Impossible to estimate frame rate !')
exit(-1) exit(-1)
else: else:
@@ -3053,7 +3066,7 @@ def main():
else: else:
mainVideo = None mainVideo = None
if mainVideo == None: if mainVideo is None:
logger.error('Impossible to find main video stream.') logger.error('Impossible to find main video stream.')
exit(-1) exit(-1)
@@ -3106,26 +3119,26 @@ def main():
# Get the nearest I-frame whose timestamp is greater or equal to the beginning. # Get the nearest I-frame whose timestamp is greater or equal to the beginning.
headFrames = getNearestIFrame(paths['ffprobe'], mkv, ts1, before=False) headFrames = getNearestIFrame(paths['ffprobe'], mkv, ts1, before=False)
if headFrames == None: if headFrames is None:
exit(-1) exit(-1)
# Get the nearest I-frame whose timestamp ... # Get the nearest I-frame whose timestamp ...
# TODO: wrong here ... # TODO: wrong here ...
tailFrames = getNearestIFrame(paths['ffprobe'], mkv, ts2, before=True) tailFrames = getNearestIFrame(paths['ffprobe'], mkv, ts2, before=True)
if tailFrames == None: if tailFrames is None:
exit(-1) exit(-1)
nbHeadFrames, headIFrame = headFrames nbHeadFrames, headIFrame = headFrames
nbTailFrames, tailIFrame = tailFrames nbTailFrames, tailIFrame = tailFrames
logger.info("Found %d frames between beginning of current part and first I-frame" % nbHeadFrames) logger.info("Found %d frames between beginning of current part and first I-frame", nbHeadFrames)
logger.info("Found %d frames between last I-frame and end of current part" % nbTailFrames) logger.info("Found %d frames between last I-frame and end of current part", nbTailFrames)
headIFrameTS = getTSFrame(headIFrame) headIFrameTS = getTSFrame(headIFrame)
if headIFrameTS == None: if headIFrameTS is None:
exit(-1) exit(-1)
tailIFrameTS = getTSFrame(tailIFrame) tailIFrameTS = getTSFrame(tailIFrame)
if tailIFrameTS == None: if tailIFrameTS is None:
exit(-1) exit(-1)
checks.append(pos+headIFrameTS-ts1) checks.append(pos+headIFrameTS-ts1)
@@ -3168,25 +3181,25 @@ def main():
try: try:
internalMKV = open(internalMKVName, 'w+') internalMKV = open(internalMKVName, 'w+')
except IOError: except IOError:
logger.error('Impossible to create file: %s' % internalMKVName) logger.error('Impossible to create file: %s', internalMKVName)
exit(-1) exit(-1)
try: try:
internalNoVideoMKV = open(internalNoVideoMKVName, 'w+') internalNoVideoMKV = open(internalNoVideoMKVName, 'w+')
except IOError: except IOError:
logger.error('Impossible to create file: %s' % internalNoVideoMKVName) logger.error('Impossible to create file: %s', internalNoVideoMKVName)
exit(-1) exit(-1)
try: try:
internalH264 = open(internalH264Name, 'w+') internalH264 = open(internalH264Name, 'w+')
except IOError: except IOError:
logger.error('Impossible to create file: %s' % internalH264Name) logger.error('Impossible to create file: %s', internalH264Name)
exit(-1) exit(-1)
try: try:
internalH264TS = open(internalH264TSName, 'w+') internalH264TS = open(internalH264TSName, 'w+')
except IOError: except IOError:
logger.error('Impossible to create file: %s' % internalH264TSName) logger.error('Impossible to create file: %s', internalH264TSName)
exit(-1) exit(-1)
# logger.info('Merge header, middle and trailer subpart into: %s' % internalMKVName) # logger.info('Merge header, middle and trailer subpart into: %s' % internalMKVName)
@@ -3199,7 +3212,7 @@ def main():
extractTrackFromMKV(mkvextract=paths['mkvextract'], inputFile=internalMKV, index=0, outputFile=internalH264, timestamps=internalH264TS) extractTrackFromMKV(mkvextract=paths['mkvextract'], inputFile=internalMKV, index=0, outputFile=internalH264, timestamps=internalH264TS)
# Remove video track from internal part of MKV # Remove video track from internal part of MKV
logger.info('Remove video track from %s' % internalMKVName) logger.info('Remove video track from %s', internalMKVName)
removeVideoTracksFromMKV(mkvmerge=paths['mkvmerge'], inputFile=internalMKV, outputFile=internalNoVideoMKV) removeVideoTracksFromMKV(mkvmerge=paths['mkvmerge'], inputFile=internalMKV, outputFile=internalNoVideoMKV)
temporaries.append(internalMKV) temporaries.append(internalMKV)
@@ -3224,7 +3237,7 @@ def main():
if h264TailTS != None: if h264TailTS != None:
h264TS.append(h264TailTS) h264TS.append(h264TailTS)
logger.info('Merging MKV: %s' % subparts) logger.info('Merging MKV: %s', subparts)
part = mergeMKVs(mkvmerge=paths['mkvmerge'], inputs=subparts, outputName="part-%d.mkv" % partnum, concatenate=True) part = mergeMKVs(mkvmerge=paths['mkvmerge'], inputs=subparts, outputName="part-%d.mkv" % partnum, concatenate=True)
mkvparts.append(part) mkvparts.append(part)
temporaries.append(part) temporaries.append(part)
@@ -3237,7 +3250,7 @@ def main():
# When using coarse option there is a single AVC configuration. # When using coarse option there is a single AVC configuration.
for avcConfig in otherAvcConfigs: for avcConfig in otherAvcConfigs:
mainAvcConfig.merge(avcConfig) mainAvcConfig.merge(avcConfig)
logger.debug('Merged AVC configuration: %s' % mainAvcConfig) logger.debug('Merged AVC configuration: %s', mainAvcConfig)
nbMKVParts = len(mkvparts) nbMKVParts = len(mkvparts)
if nbMKVParts > 0: if nbMKVParts > 0:
@@ -3328,12 +3341,12 @@ def main():
try: try:
idx = open(idxName,'r') idx = open(idxName,'r')
except IOError: except IOError:
logger.error("Impossible to open %s." % idxName) logger.error("Impossible to open %s.", idxName)
exit(-1) exit(-1)
try: try:
sub = open(subName,'r') sub = open(subName,'r')
except IOError: except IOError:
logger.error("Impossible to open %s." % subName) logger.error("Impossible to open %s.", subName)
exit(-1) exit(-1)
temporaries.append(idx) temporaries.append(idx)