From 23f1db5ffab9165b56d877a79931fa7664d235c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Tronel?= Date: Sun, 26 Oct 2025 17:14:29 +0100 Subject: [PATCH] Even more linting (long lines, trailing spaces, module importation order, variable names). --- removeads.py | 1618 ++++++++++++++++++++++++++++---------------------- 1 file changed, 914 insertions(+), 704 deletions(-) diff --git a/removeads.py b/removeads.py index 4af6baa..2fcfe7f 100755 --- a/removeads.py +++ b/removeads.py @@ -1,43 +1,50 @@ #!/usr/bin/env python3 '''A module to remove parts of video (.e.g advertisements) with single frame precision.''' +# Standard modules import argparse import re from sys import exit from datetime import datetime,timedelta -import coloredlogs, logging +import logging from functools import cmp_to_key from subprocess import Popen, PIPE -from os import read, write, lseek, pipe, set_inheritable, memfd_create, SEEK_SET, close, unlink, fstat, ftruncate +from os import read, write, lseek, set_inheritable, memfd_create, SEEK_SET, close, unlink,\ + fstat, ftruncate import os.path +from enum import IntEnum, unique +from shutil import copyfile, which, move +from dataclasses import dataclass, field +from math import floor, ceil, log from io import BytesIO, TextIOWrapper import json -from enum import Enum, IntEnum, unique, auto -import shutil -from tqdm import tqdm, trange -from select import select -from math import floor, ceil, log -from shutil import copyfile, which, move + +# Third party libraries +import coloredlogs +from tqdm import tqdm import hexdump from iso639 import Lang from iso639.exceptions import InvalidLanguageValue -from dataclasses import dataclass, field -# Useful SPS/PPS discussion -# TODO: improve situation of SPS and PPS header mismatch when merging MVK with mkvmerge to remove warnings. +# Useful SPS/PPS discussion. # https://copyprogramming.com/howto/including-sps-and-pps-in-a-raw-h264-track # https://gitlab.com/mbunkus/mkvtoolnix/-/issues/2390 # New strategy: a possible way of handling multiple SPS/PPS gracefully. -# Encode each head and trailer with FFMPEG using only I-frame (to be sure the NAL unit will never refer to another image). -# Encode using an different SPS-ID all of them (using sps-id parameter of libx264 library, e.g 1 instead of 0). -# For the video track produce only a raw H264 file and a file containing timestamps of the different frames. +# Encode each head and trailer with FFMPEG using only I-frame (to be sure the NAL unit will never +# refer to another image). +# Encode using an different SPS-ID all of them (using sps-id parameter of libx264 library, e.g +# 1 instead of 0). +# For the video track produce only a raw H264 file and a file containing timestamps of the +# different frames. # For the rest of the tracks (audio, subtitles) produce directly a MKV (this is already done). -# Concatenate all raw H264 in a giant one (like cat), and the same for timestamps of video frames (to keep -# sound and video synchronized). +# Concatenate all raw H264 in a giant one (like cat), and the same for timestamps of video frames +# (to keep sound and video synchronized). # Then use mkvmerge to remux the H264 track and the rest of tracks. -# MKVmerge "concatenate" subcommand is able to concatenate different SPS/PPS data into a bigger Private Codec Data. -# However, this is proved to be not reliable. Sometimes it results in a AVC context containing a single SPS/PPS. +# MKVmerge "concatenate" subcommand is able to concatenate different SPS/PPS data into a bigger +# Private Codec Data. +# However, this is proved to be not reliable. Sometimes it results in a AVC context containing +# a single SPS/PPS. # So we have to rely on a manual parsing of the H264 AVC context of original movie # and the ones produced for headers and trailers, and then merging them into a bigger AVC context. # Then finally, change the Private Codec Data in the final MKV. @@ -45,7 +52,7 @@ from dataclasses import dataclass, field def checkRequiredTools(): logger = logging.getLogger(__name__) - allOptionalTools = True + all_optional_tools = True paths = {} required = ['ffmpeg', 'ffprobe', 'mkvmerge', 'mkvinfo'] optional = ['mkvextract', 'vobsubocr','tesseract'] @@ -60,11 +67,11 @@ def checkRequiredTools(): path = which(tool) if path is None: logger.info('Optional tool: %s is missing.',tool) - allOptionalTools = False + all_optional_tools = False else: paths[tool] = path - return allOptionalTools, paths + return all_optional_tools, paths def getTesseractSupportedLang(tesseract): logger = logging.getLogger(__name__) @@ -81,6 +88,7 @@ def getTesseractSupportedLang(tesseract): key = Lang(lang) res[key] = lang except InvalidLanguageValue as e: + logger.warning('Invalid language: %s', e) pass tesseract.wait() @@ -98,12 +106,11 @@ def getFrameRate(ffprobe, inputFile): lseek(infd, 0, SEEK_SET) set_inheritable(infd, True) - meanDuration = 0. - nbFrames1 = 0 - nbFrames2 = 0 - meanInterframes = 0. - minTs = None - maxTs = None + mean_duration = 0. + nb_frames1 = 0 + nb_frames2 = 0 + min_ts = None + max_ts = None interlaced = False params = [ffprobe, '-loglevel', 'quiet', '-select_streams', 'v', '-show_frames', @@ -119,18 +126,18 @@ def getFrameRate(ffprobe, inputFile): interlaced = True if 'pts_time' in frame: ts = float(frame['pts_time']) - if minTs is None: - minTs = ts - if maxTs is None: - maxTs = ts - if ts < minTs: - minTs = ts - if ts > maxTs: - maxTs = ts - nbFrames1+=1 + if min_ts is None: + min_ts = ts + if max_ts is None: + max_ts = ts + if ts < min_ts: + min_ts = ts + if ts > max_ts: + max_ts = ts + nb_frames1+=1 if 'duration_time' in frame: - meanDuration+=float(frame['duration_time']) - nbFrames2+=1 + mean_duration+=float(frame['duration_time']) + nb_frames2+=1 else: return None @@ -140,20 +147,22 @@ def getFrameRate(ffprobe, inputFile): logger.error("ffprobe returns an error code: %d", ffprobe.returncode) return None - frameRate1 = nbFrames1/(maxTs-minTs) - frameRate2 = nbFrames2 / meanDuration + frame_rate1 = nb_frames1/(max_ts-min_ts) + frame_rate2 = nb_frames2 / mean_duration - if abs(frameRate1 - frameRate2) > 0.2: + if abs(frame_rate1 - frame_rate2) > 0.2: if not interlaced: - logger.error('Video is not interlaced and the disperancy between frame rates is too big: %f / %f', frameRate1, frameRate2) + logger.error('Video is not interlaced and the disperancy between frame rates is too \ + big: %f / %f', frame_rate1, frame_rate2) return None - if abs(frameRate1*2 - frameRate2) < 0.2: - return frameRate2/2 + if abs(frame_rate1*2 - frame_rate2) < 0.2: + return frame_rate2/2 else: - logger.error('Video is interlaced and the disperancy between frame rates is too big: %f / %f', frameRate1, frameRate2) + logger.error('Video is interlaced and the disperancy between frame rates is too big:\ + %f / %f', frame_rate1, frame_rate2) return None else: - return frameRate2 + return frame_rate2 return None @@ -161,9 +170,9 @@ def getSubTitlesTracks(ffprobe, mkvPath): logger = logging.getLogger(__name__) tracks={} - nbSubTitles = 0 with Popen([ffprobe, '-loglevel', 'quiet', '-select_streams', 's', '-show_entries', - 'stream=index,codec_name:stream_tags=language', '-of', 'json', mkvPath], stdout=PIPE) as ffprobe: + 'stream=index,codec_name:stream_tags=language', '-of', 'json', mkvPath], + stdout=PIPE) as ffprobe: out, _ = ffprobe.communicate() out = json.load(BytesIO(out)) if 'streams' in out: @@ -215,11 +224,12 @@ def extractSRT(mkvextract, fileName, subtitles, langs): res.append(('%s-%d.idx' % (lang,count), '%s-%d.sub' % (lang,count), lang, ocrlang)) count = count+1 - logger.debug('Executing %s' % params) + logger.debug('Executing %s', params) env = {**os.environ, 'LANG': 'C'} with Popen(params, stdout=PIPE, close_fds=False, env=env) as extract: - pb = tqdm(TextIOWrapper(extract.stdout, encoding="utf-8"), total=100, unit='%', desc='Extraction:') + pb = tqdm(TextIOWrapper(extract.stdout, encoding="utf-8"), total=100, unit='%', + desc='Extraction:') for line in pb: if line.startswith('Progress :'): p = re.compile('^Progress : (?P[0-9]{1,3})%$') @@ -248,15 +258,18 @@ def doOCR(vobsubocr, idxs, duration, temporaries, dumpMemFD=False): logger = logging.getLogger(__name__) res = [] - for idxName, subName, lang, iso in idxs: - srtname = '%s.srt' % os.path.splitext(idxName)[0] + for idx_name, _, lang, iso in idxs: + srtname = '%s.srt' % os.path.splitext(idx_name)[0] # Tesseract seems to recognize the three dots ... as "su" ldots = re.compile('^su\n$') - timestamps = re.compile(r'^[0-9]{2}:[0-9]{2}:[0-9]{2},[0-9]{3} \-\-> (?P[0-9]{2}):(?P[0-9]{2}):(?P[0-9]{2}),[0-9]{3}$') + # Timestamps produced by vobsubocr: 01:52:19,861 --> 01:52:21,641 + timestamps = re.compile((r'^[0-9]{2}:[0-9]{2}:[0-9]{2},[0-9]{3} \-\-> (?P[0-9]{2}):' + r'(?P[0-9]{2}):(?P[0-9]{2}),[0-9]{3}$')) srtfd = memfd_create(srtname, flags=0) - with Popen([vobsubocr, '--lang', iso, idxName], stdout=PIPE) as ocr: - pb = tqdm(TextIOWrapper(ocr.stdout, encoding="utf-8"), total=int(duration/timedelta(seconds=1)), unit='s', desc='OCR') + with Popen([vobsubocr, '--lang', iso, idx_name], stdout=PIPE) as ocr: + pb = tqdm(TextIOWrapper(ocr.stdout, encoding="utf-8"), total= + int(duration/timedelta(seconds=1)), unit='s', desc='OCR') for line in pb: m = re.match(ldots,line) if m is not None: @@ -280,23 +293,23 @@ def doOCR(vobsubocr, idxs, duration, temporaries, dumpMemFD=False): if dumpMemFD: try: - dumpSrt = open(srtname,'w') + dump_srt = open(srtname,'w') except IOError: logger.error('Impossible to create file: %s', srtname) return None lseek(srtfd, 0, SEEK_SET) - srtLength = fstat(srtfd).st_size - buf = read(srtfd, srtLength) - outfd = dumpSrt.fileno() + srt_length = fstat(srtfd).st_size + buf = read(srtfd, srt_length) + outfd = dump_srt.fileno() pos = 0 - while pos < srtLength: + while pos < srt_length: pos+=write(outfd, buf[pos:]) - temporaries.append(dumpSrt) + temporaries.append(dump_srt) - srtLength = fstat(srtfd).st_size - if srtLength > 0: + srt_length = fstat(srtfd).st_size + if srt_length > 0: res.append((srtfd, lang)) return res @@ -321,7 +334,8 @@ class SupportedFormat(IntEnum): # Extract SPS/PPS # https://gitlab.com/mbunkus/mkvtoolnix/-/issues/2390 -# ffmpeg -i -c:v copy -an -sn -bsf:v trace_headers -t 0.01 -report -loglevel 0 -f null - +# ffmpeg -i -c:v copy -an -sn -bsf:v trace_headers -t 0.01\ +# -report -loglevel 0 -f null - # Found codec private data using mkvinfo def getCodecPrivateDataFromMKV(mkvinfo, inputFile): @@ -333,13 +347,17 @@ def getCodecPrivateDataFromMKV(mkvinfo, inputFile): found = False env = {**os.environ, 'LANG': 'C'} # Output example - # Codec's private data: size 48 (H.264 profile: High @L4.0) hexdump 01 64 00 28 ff e1 00 1b 67 64 00 28 ac d9 40 78 04 4f dc d4 04 04 05 00 00 92 ef 00 1d ad a6 1f 16 2d 96 01 00 06 68 fb a3 cb 22 c0 fd f8 f8 00 at 406 size 51 data size 48 + # Codec's private data: size 48 (H.264 profile: High @L4.0) hexdump 01 64 00 28 ff e1 00 1b 67\ + # 64 00 28 ac d9 40 78 04 4f dc d4 04 04 05 00 00 92 ef 00 1d ad a6 1f 16 2d 96 01 00 06 68 fb\ + # a3 cb 22 c0 fd f8 f8 00 at 406 size 51 data size 48 - with Popen([mkvinfo, '-z', '-X', '-P', '/proc/self/fd/%d' % infd ], stdout=PIPE, close_fds=False, env=env) as mkvinfo: + with Popen([mkvinfo, '-z', '-X', '-P', '/proc/self/fd/%d' % infd ], stdout=PIPE, + close_fds=False, env=env) as mkvinfo: out, _ = mkvinfo.communicate() out = out.decode('utf8') - regExp = r"^.*Codec's private data: size ([0-9]+) \(H.264.*\) hexdump (?P([0-9a-f]{2} )+)at (?P[0-9]+) size (?P[0-9]+).*$" - p = re.compile(regExp) + reg_exp = (r"^.*Codec's private data: size ([0-9]+) \(H.264.*\) hexdump " + r"(?P([0-9a-f]{2} )+)at (?P[0-9]+) size (?P[0-9]+).*$") + p = re.compile(reg_exp) for line in out.splitlines(): m = p.match(line) if m is not None: @@ -362,152 +380,154 @@ def getCodecPrivateDataFromMKV(mkvinfo, inputFile): # ISO/IEC H.264-201602 # ISO/IEC 14496-15 -def readBit(buf, bitPosition): +def readBit(buf, bit_position): + # pylint: disable=W0612 logger = logging.getLogger(__name__) - bytePosition = floor(floor(bitPosition/8)) - byte = buf[bytePosition] - bit = (byte >> (7-(bitPosition % 8))) & 1 - return bitPosition+1, bit + byte_position = floor(floor(bit_position/8)) + byte = buf[byte_position] + bit = (byte >> (7-(bit_position % 8))) & 1 + return bit_position+1, bit -def readBoolean(buf, bitPosition): - bitPosition, b = readBit(buf, bitPosition) - return bitPosition, b==1 +def readBoolean(buf, bit_position): + bit_position, b = readBit(buf, bit_position) + return bit_position, b==1 -def readBits(buf, bitPosition, nbBits): +def readBits(buf, bit_position, nbBits): logger = logging.getLogger(__name__) v = 0 - for i in range(0, nbBits): - bitPosition, bit = readBit(buf, bitPosition) + for _ in range(0, nbBits): + bit_position, bit = readBit(buf, bit_position) v = v*2+bit - return bitPosition, v + return bit_position, v -def readByte(buf, bitPosition): - bitPosition, b = readBits(buf, bitPosition, 8) - return bitPosition, b +def readByte(buf, bit_position): + bit_position, b = readBits(buf, bit_position, 8) + return bit_position, b -def readWord(buf, bitPosition): - bitPosition, w = readBits(buf, bitPosition, 16) - return bitPosition, w +def readWord(buf, bit_position): + bit_position, w = readBits(buf, bit_position, 16) + return bit_position, w -def readLong(buf, bitPosition): - bitPosition, l = readBits(buf, bitPosition, 32) - return bitPosition, l +def readLong(buf, bit_position): + bit_position, l = readBits(buf, bit_position, 32) + return bit_position, l -def readUnsignedExpGolomb(buf, bitPosition): - nbZeroes=0 +def readUnsignedExpGolomb(buf, bit_position): + nb_zeroes=0 while True: - bitPosition, b = readBit(buf, bitPosition) + bit_position, b = readBit(buf, bit_position) if b!=0: break - nbZeroes+=1 + nb_zeroes+=1 v1 = 1 - bitPosition, v2 = readBits(buf, bitPosition, nbZeroes) - v = (v1<>1) + return bit_position, -(v>>1) else: - return bitPosition, (v+1)>>1 + return bit_position, (v+1)>>1 -def writeBit(buf, bitPosition, b): +def writeBit(buf, bit_position, b): logger = logging.getLogger(__name__) - bufLength = len(buf) - bytePosition = floor(bitPosition/8) + buf_length = len(buf) + byte_position = floor(bit_position/8) - if bytePosition >= bufLength: - extension = bytearray(bytePosition+1-bufLength) + if byte_position >= buf_length: + extension = bytearray(byte_position+1-buf_length) buf.extend(extension) - buf[bytePosition] |= (b<<(7-(bitPosition % 8))) - bitPosition+=1 + buf[byte_position] |= (b<<(7-(bit_position % 8))) + bit_position+=1 - return bitPosition + return bit_position -def writeBoolean(buf, bitPosition, b): +def writeBoolean(buf, bit_position, b): if b: - bitPosition = writeBit(buf, bitPosition, 1) + bit_position = writeBit(buf, bit_position, 1) else: - bitPosition = writeBit(buf, bitPosition, 0) - return bitPosition + bit_position = writeBit(buf, bit_position, 0) + return bit_position -def writeBits(buf, bitPosition, v, size): +def writeBits(buf, bit_position, v, size): for i in range(size-1,-1,-1): b = (v>>i)&1 - bitPosition = writeBit(buf, bitPosition, b) + bit_position = writeBit(buf, bit_position, b) - return bitPosition + return bit_position -def writeByte(buf, bitPosition, v): - bitPosition = writeBits(buf, bitPosition, v, 8) - return bitPosition +def writeByte(buf, bit_position, v): + bit_position = writeBits(buf, bit_position, v, 8) + return bit_position -def writeWord(buf, bitPosition, v): - bitPosition = writeBits(buf, bitPosition, v, 16) - return bitPosition +def writeWord(buf, bit_position, v): + bit_position = writeBits(buf, bit_position, v, 16) + return bit_position -def writeLong(buf, bitPosition, v): - bitPosition = writeBits(buf, bitPosition, v, 32) - return bitPosition +def writeLong(buf, bit_position, v): + bit_position = writeBits(buf, bit_position, v, 32) + return bit_position -def writeUnsignedExpGolomb(buf, bitPosition, v): +def writeUnsignedExpGolomb(buf, bit_position, v): logger = logging.getLogger(__name__) n = floor(log(v+1)/log(2))+1 # Write zeroes - bitPosition = writeBits(buf, bitPosition, 0, n-1) - bitPosition = writeBit(buf, bitPosition, 1) - bitPosition = writeBits(buf, bitPosition, v+1, n-1) + bit_position = writeBits(buf, bit_position, 0, n-1) + bit_position = writeBit(buf, bit_position, 1) + bit_position = writeBits(buf, bit_position, v+1, n-1) - return bitPosition + return bit_position -def writeSignedExpGolomb(buf, bitPosition, v): +def writeSignedExpGolomb(buf, bit_position, v): if v <= 0: - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, -v*2) + bit_position = writeUnsignedExpGolomb(buf, bit_position, -v*2) else: - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, v*2-1) + bit_position = writeUnsignedExpGolomb(buf, bit_position, v*2-1) - return bitPosition + return bit_position -def parseRBSPTrailingBits(buf, bitPosition): +def parseRBSPTrailingBits(buf, bit_position): logger = logging.getLogger(__name__) - bitPosition, one = readBit(buf, bitPosition) + bit_position, one = readBit(buf, bit_position) if one==0: raise Exception('Stop bit should be equal to one. Read: %d' % one) - while bitPosition%8 != 0: - bitPosition, zero = readBit(buf, bitPosition) + while bit_position%8 != 0: + bit_position, zero = readBit(buf, bit_position) if zero==1: raise Exception('Trailing bit should be equal to zero') - return bitPosition + return bit_position -def writeRBSPTrailingBits(buf, bitPosition): - bitPosition = writeBit(buf, bitPosition, 1) - while bitPosition%8 != 0: - bitPosition = writeBit(buf, bitPosition, 0) +def writeRBSPTrailingBits(buf, bit_position): + bit_position = writeBit(buf, bit_position, 1) + while bit_position%8 != 0: + bit_position = writeBit(buf, bit_position, 0) - return bitPosition + return bit_position -def moreRBSPData(buf, bitPosition): +def moreRBSPData(buf, bit_position): logger = logging.getLogger(__name__) - logger.debug('Is there more data in buffer of length: %d at bitPosition: %d' % (len(buf), bitPosition)) + logger.debug('Is there more data in buffer of length: %d at bit position: %d', + len(buf), bit_position) - byteLength = len(buf) - bitLength = byteLength*8 + byte_length = len(buf) + bit_length = byte_length*8 # We are at the end of buffer - if bitPosition == bitLength: + if bit_position == bit_length: return False else: found = False - for i in range(bitLength-1,-1,-1): + for i in range(bit_length-1,-1,-1): pos, b = readBit(buf, i) if b == 1: found = True @@ -517,7 +537,7 @@ def moreRBSPData(buf, bitPosition): raise Exception('Impossible to find trailing stop bit !') # No more data - if bitPosition == pos: + if bit_position == pos: return False return True @@ -526,7 +546,7 @@ def moreRBSPData(buf, bitPosition): def RBSP2SODB(buf): logger = logging.getLogger(__name__) - logger.debug('RBSP: %s' % hexdump.dump(buf, sep=':')) + logger.debug('RBSP: %s', hexdump.dump(buf, sep=':')) res = buf for b in [ b'\x00', b'\x01', b'\x02', b'\x03']: @@ -534,7 +554,7 @@ def RBSP2SODB(buf): replacement = b'\x00\x00' + b res = res.replace(pattern, replacement) - logger.debug('SODB: %s' % hexdump.dump(res, sep=':')) + logger.debug('SODB: %s', hexdump.dump(res, sep=':')) return res # Reverse operation SODB to RBSP. @@ -552,25 +572,26 @@ def SODB2RBSP(buf): return res # Useful for SPS and PPS -def parseScalingList(buf, bitPosition, size): +def parseScalingList(buf, bit_position, size): logger = logging.getLogger(__name__) res = [] - lastScale = 8 - nextScale = 8 - for i in range(0, size): - if nextScale != 0: - bitPosition, delta_scale = readSignedExpGolomb(buf, bitPosition) - nextScale = (lastScale+delta_scale+256) % 256 - v = lastScale if nextScale==0 else nextScale + last_scale = 8 + next_scale = 8 + for _ in range(0, size): + if next_scale != 0: + bit_position, delta_scale = readSignedExpGolomb(buf, bit_position) + next_scale = (last_scale+delta_scale+256) % 256 + v = last_scale if next_scale==0 else next_scale res.append(v) - lastScale = v + last_scale = v - return bitPosition,res + return bit_position,res # TODO: test optimized version. -# The ISO/IEC H.264-201602 seems to take into account the case where the end of the deltas list is full of zeroes. -def writeScalingList(buf, bitPosition, size, matrix, optimized=False): +# The ISO/IEC H.264-201602 seems to take into account the case where the end of the deltas list +# is full of zeroes. +def writeScalingList(buf, bit_position, size, matrix, optimized=False): logger = logging.getLogger(__name__) logger.debug('Dumping matrix: %s of size: %d, size parameter: %d.', matrix, len(matrix), size) @@ -584,7 +605,7 @@ def writeScalingList(buf, bitPosition, size, matrix, optimized=False): if not optimized: for delta in deltas: - bitPosition = writeSignedExpGolomb(buf, bitPosition, delta) + bit_position = writeSignedExpGolomb(buf, bit_position, delta) else: logger.error('Not yet implemented') exit(-1) @@ -600,9 +621,9 @@ def writeScalingList(buf, bitPosition, size, matrix, optimized=False): # if compressed: # deltas.append(0) # for delta in deltas: - # bitPosition = writeSignedExpGolomb(buf, bitPosition, delta) + # bit_position = writeSignedExpGolomb(buf, bit_position, delta) - return bitPosition + return bit_position @dataclass class HRD: @@ -622,41 +643,41 @@ class HRD: self.cpb_size_value_minus1 = {} self.cbr_flag = {} - def fromBytes(self, buf, bitPosition): - bitPosition, self.cpb_cnt_minus1 = readUnsignedExpGolomb(buf, bitPosition) - bitPosition, self.bit_rate_scale = readBits(buf, bitPosition, 4) - bitPosition, self.cpb_size_scale = readBits(buf, bitPosition, 4) + def fromBytes(self, buf, bit_position): + bit_position, self.cpb_cnt_minus1 = readUnsignedExpGolomb(buf, bit_position) + bit_position, self.bit_rate_scale = readBits(buf, bit_position, 4) + bit_position, self.cpb_size_scale = readBits(buf, bit_position, 4) for i in range(0, self.cpb_cnt_minus1+1): - bitPosition, v = readUnsignedExpGolomb(buf, bitPosition) + bit_position, v = readUnsignedExpGolomb(buf, bit_position) self.bit_rate_value_minus1[i] = v - bitPosition, v = readUnsignedExpGolomb(buf, bitPosition) + bit_position, v = readUnsignedExpGolomb(buf, bit_position) self.cpb_size_value_minus1[i] = v - bitPosition, b = readBoolean(buf, bitPosition) + bit_position, b = readBoolean(buf, bit_position) self.cbr_flag[i] = b - bitPosition, self.initial_cpb_removal_delay_length_minus1 = readBits(buf, bitPosition, 5) - bitPosition, self.cpb_removal_delay_length_minus1 = readBits(buf, bitPosition, 5) - bitPosition, self.dpb_output_delay_length_minus1 = readBits(buf, bitPosition, 5) - bitPosition, self.time_offset_length = readBits(buf, bitPosition, 5) + bit_position, self.initial_cpb_removal_delay_length_minus1 = readBits(buf, bit_position, 5) + bit_position, self.cpb_removal_delay_length_minus1 = readBits(buf, bit_position, 5) + bit_position, self.dpb_output_delay_length_minus1 = readBits(buf, bit_position, 5) + bit_position, self.time_offset_length = readBits(buf, bit_position, 5) - return bitPosition + return bit_position - def toBytes(self, buf, bitPosition): - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.cpb_cnt_minus1) - bitPosition = writeBits(buf, bitPosition, self.bit_rate_scale, 4) - bitPosition = writeBits(buf, bitPosition, self.cpb_size_scale, 4) + def toBytes(self, buf, bit_position): + bit_position = writeUnsignedExpGolomb(buf, bit_position, self.cpb_cnt_minus1) + bit_position = writeBits(buf, bit_position, self.bit_rate_scale, 4) + bit_position = writeBits(buf, bit_position, self.cpb_size_scale, 4) for i in range(0, self.cpb_cnt_minus1+1): v = self.bit_rate_value_minus1[i] - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, v) + bit_position = writeUnsignedExpGolomb(buf, bit_position, v) v = self.cpb_size_value_minus1[i] - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, v) + bit_position = writeUnsignedExpGolomb(buf, bit_position, v) b = self.cbr_flag[i] - bitPosition = writeBoolean(buf, bitPosition, b) - bitPosition = writeBits(buf, bitPosition, self.initial_cpb_removal_delay_length_minus1, 5) - bitPosition = writeBits(buf, bitPosition, self.cpb_removal_delay_length_minus1, 5) - bitPosition = writeBits(buf, bitPosition, self.dpb_output_delay_length_minus1, 5) - bitPosition = writeBits(buf, bitPosition, self.time_offset_length, 5) + bit_position = writeBoolean(buf, bit_position, b) + bit_position = writeBits(buf, bit_position, self.initial_cpb_removal_delay_length_minus1, 5) + bit_position = writeBits(buf, bit_position, self.cpb_removal_delay_length_minus1, 5) + bit_position = writeBits(buf, bit_position, self.dpb_output_delay_length_minus1, 5) + bit_position = writeBits(buf, bit_position, self.time_offset_length, 5) - return bitPosition + return bit_position @dataclass class VUI: @@ -697,111 +718,121 @@ class VUI: # This structure is not guaranteed to be located at a byte boundary. # We must explicitely indicate bit offset. - def fromBytes(self, buf, bitPosition): + def fromBytes(self, buf, bit_position): logger = logging.getLogger(__name__) - bitPosition, self.aspect_ratio_info_present_flag = readBoolean(buf, bitPosition) + bit_position, self.aspect_ratio_info_present_flag = readBoolean(buf, bit_position) if self.aspect_ratio_info_present_flag: - bitPosition, self.aspect_ratio_idc = readByte(buf, bitPosition) + bit_position, self.aspect_ratio_idc = readByte(buf, bit_position) if self.aspect_ratio_idc == 255: # Extended_SAR - bitPosition, self.sar_width = readWord(buf, bitPosition) - bitPosition, self.sar_height = readWord(buf, bitPosition) - bitPosition, self.overscan_info_present_flag = readBoolean(buf, bitPosition) + bit_position, self.sar_width = readWord(buf, bit_position) + bit_position, self.sar_height = readWord(buf, bit_position) + bit_position, self.overscan_info_present_flag = readBoolean(buf, bit_position) if self.overscan_info_present_flag: - bitPosition, self.overscan_appropriate_flag = readBoolean(buf, bitPosition) - bitPosition, self.video_signal_type_present_flag = readBoolean(buf, bitPosition) + bit_position, self.overscan_appropriate_flag = readBoolean(buf, bit_position) + bit_position, self.video_signal_type_present_flag = readBoolean(buf, bit_position) if self.video_signal_type_present_flag: - bitPosition, self.video_format = readBits(buf, bitPosition, 3) - bitPosition, self.video_full_range_flag = readBoolean(buf, bitPosition) - bitPosition, self.colour_description_present_flag = readBoolean(buf, bitPosition) + bit_position, self.video_format = readBits(buf, bit_position, 3) + bit_position, self.video_full_range_flag = readBoolean(buf, bit_position) + bit_position, self.colour_description_present_flag = readBoolean(buf, bit_position) if self.colour_description_present_flag: - bitPosition, self.colour_primaries = readByte(buf, bitPosition) - bitPosition, self.transfer_characteristics = readByte(buf, bitPosition) - bitPosition, self.matrix_coefficients = readByte(buf, bitPosition) - bitPosition, self.chroma_loc_info_present_flag = readBoolean(buf, bitPosition) + bit_position, self.colour_primaries = readByte(buf, bit_position) + bit_position, self.transfer_characteristics = readByte(buf, bit_position) + bit_position, self.matrix_coefficients = readByte(buf, bit_position) + bit_position, self.chroma_loc_info_present_flag = readBoolean(buf, bit_position) if self.chroma_loc_info_present_flag: - bitPosition, self.chroma_sample_loc_type_top_field = readUnsignedExpGolomb(buf, bitPosition) - bitPosition, self.chroma_sample_loc_type_bottom_field = readUnsignedExpGolomb(buf, bitPosition) - bitPosition, self.timing_info_present_flag = readBoolean(buf, bitPosition) + bit_position, self.chroma_sample_loc_type_top_field =\ + readUnsignedExpGolomb(buf, bit_position) + bit_position, self.chroma_sample_loc_type_bottom_field =\ + readUnsignedExpGolomb(buf,bit_position) + bit_position, self.timing_info_present_flag = readBoolean(buf, bit_position) if self.timing_info_present_flag: - bitPosition, self.num_units_in_tick = readLong(buf, bitPosition) - bitPosition, self.time_scale = readLong(buf, bitPosition) - bitPosition, self.fixed_frame_rate_flag = readBoolean(buf, bitPosition) - bitPosition, self.nal_hrd_parameters_present_flag = readBoolean(buf, bitPosition) + bit_position, self.num_units_in_tick = readLong(buf, bit_position) + bit_position, self.time_scale = readLong(buf, bit_position) + bit_position, self.fixed_frame_rate_flag = readBoolean(buf, bit_position) + bit_position, self.nal_hrd_parameters_present_flag = readBoolean(buf, bit_position) if self.nal_hrd_parameters_present_flag: hrd = HRD() - bitPosition = hrd.fromBytes(buf, bitPosition) + bit_position = hrd.fromBytes(buf, bit_position) self.hrd_parameters = hrd - bitPosition, self.vcl_hrd_parameters_present_flag = readBoolean(buf, bitPosition) + bit_position, self.vcl_hrd_parameters_present_flag = readBoolean(buf, bit_position) if self.vcl_hrd_parameters_present_flag: hrd = HRD() - bitPosition = hrd.fromBytes(buf, bitPosition) + bit_position = hrd.fromBytes(buf, bit_position) self.vcl_hrd_parameters = hrd if self.nal_hrd_parameters_present_flag or self.vcl_hrd_parameters_present_flag: - bitPosition, self.low_delay_hrd_flag = readBoolean(buf, bitPosition) - bitPosition, self.pic_struct_present_flag = readBoolean(buf, bitPosition) - bitPosition, self.bitstream_restriction_flag = readBoolean(buf, bitPosition) + bit_position, self.low_delay_hrd_flag = readBoolean(buf, bit_position) + bit_position, self.pic_struct_present_flag = readBoolean(buf, bit_position) + bit_position, self.bitstream_restriction_flag = readBoolean(buf, bit_position) if self.bitstream_restriction_flag: - bitPosition, self.motion_vectors_over_pic_boundaries_flag = readBoolean(buf, bitPosition) - bitPosition, self.max_bytes_per_pic_denom = readUnsignedExpGolomb(buf, bitPosition) - bitPosition, self.max_bits_per_mb_denom = readUnsignedExpGolomb(buf, bitPosition) - bitPosition, self.log2_max_mv_length_horizontal = readUnsignedExpGolomb(buf, bitPosition) - bitPosition, self.log2_max_mv_length_vertical = readUnsignedExpGolomb(buf, bitPosition) - bitPosition, self.max_num_reorder_frames = readUnsignedExpGolomb(buf, bitPosition) - bitPosition, self.max_dec_frame_buffering = readUnsignedExpGolomb(buf, bitPosition) + bit_position, self.motion_vectors_over_pic_boundaries_flag =\ + readBoolean(buf, bit_position) + bit_position, self.max_bytes_per_pic_denom = readUnsignedExpGolomb(buf, bit_position) + bit_position, self.max_bits_per_mb_denom = readUnsignedExpGolomb(buf, bit_position) + bit_position, self.log2_max_mv_length_horizontal = readUnsignedExpGolomb(buf, + bit_position) + bit_position, self.log2_max_mv_length_vertical = readUnsignedExpGolomb(buf, + bit_position) + bit_position, self.max_num_reorder_frames = readUnsignedExpGolomb(buf, bit_position) + bit_position, self.max_dec_frame_buffering = readUnsignedExpGolomb(buf, bit_position) - return bitPosition + return bit_position - def toBytes(self, buf, bitPosition): + def toBytes(self, buf, bit_position): logger = logging.getLogger(__name__) - bitPosition = writeBoolean(buf, bitPosition, self.aspect_ratio_info_present_flag) + bit_position = writeBoolean(buf, bit_position, self.aspect_ratio_info_present_flag) if self.aspect_ratio_info_present_flag: - bitPosition = writeByte(buf, bitPosition, self.aspect_ratio_idc) + bit_position = writeByte(buf, bit_position, self.aspect_ratio_idc) if self.aspect_ratio_idc == 255: # Extended_SAR - bitPosition = writeWord(buf, bitPosition, self.sar_width) - bitPosition = writeWord(buf, bitPosition, self.sar_height) - bitPosition = writeBoolean(buf, bitPosition, self.overscan_info_present_flag) + bit_position = writeWord(buf, bit_position, self.sar_width) + bit_position = writeWord(buf, bit_position, self.sar_height) + bit_position = writeBoolean(buf, bit_position, self.overscan_info_present_flag) if self.overscan_info_present_flag: - bitPosition = writeBoolean(buf, bitPosition, self.overscan_appropriate_flag) - bitPosition = writeBoolean(buf, bitPosition, self.video_signal_type_present_flag) + bit_position = writeBoolean(buf, bit_position, self.overscan_appropriate_flag) + bit_position = writeBoolean(buf, bit_position, self.video_signal_type_present_flag) if self.video_signal_type_present_flag: - bitPosition = writeBits(buf, bitPosition, self.video_format, 3) - bitPosition = writeBoolean(buf, bitPosition, self.video_full_range_flag) - bitPosition = writeBoolean(buf, bitPosition, self.colour_description_present_flag) + bit_position = writeBits(buf, bit_position, self.video_format, 3) + bit_position = writeBoolean(buf, bit_position, self.video_full_range_flag) + bit_position = writeBoolean(buf, bit_position, self.colour_description_present_flag) if self.colour_description_present_flag: - bitPosition = writeByte(buf, bitPosition, self.colour_primaries) - bitPosition = writeByte(buf, bitPosition, self.transfer_characteristics) - bitPosition = writeByte(buf, bitPosition, self.matrix_coefficients) - bitPosition = writeBoolean(buf, bitPosition, self.chroma_loc_info_present_flag) + bit_position = writeByte(buf, bit_position, self.colour_primaries) + bit_position = writeByte(buf, bit_position, self.transfer_characteristics) + bit_position = writeByte(buf, bit_position, self.matrix_coefficients) + bit_position = writeBoolean(buf, bit_position, self.chroma_loc_info_present_flag) if self.chroma_loc_info_present_flag: - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.chroma_sample_loc_type_top_field) - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.chroma_sample_loc_type_bottom_field) - bitPosition = writeBoolean(buf, bitPosition, self.timing_info_present_flag ) + bit_position = writeUnsignedExpGolomb(buf, bit_position, + self.chroma_sample_loc_type_top_field) + bit_position = writeUnsignedExpGolomb(buf, bit_position, + self.chroma_sample_loc_type_bottom_field) + bit_position = writeBoolean(buf, bit_position, self.timing_info_present_flag ) if self.timing_info_present_flag: - bitPosition = writeLong(buf, bitPosition, self.num_units_in_tick ) - bitPosition = writeLong(buf, bitPosition, self.time_scale) - bitPosition = writeBoolean(buf, bitPosition, self.fixed_frame_rate_flag) - bitPosition = writeBoolean(buf, bitPosition, self.nal_hrd_parameters_present_flag) + bit_position = writeLong(buf, bit_position, self.num_units_in_tick ) + bit_position = writeLong(buf, bit_position, self.time_scale) + bit_position = writeBoolean(buf, bit_position, self.fixed_frame_rate_flag) + bit_position = writeBoolean(buf, bit_position, self.nal_hrd_parameters_present_flag) if self.nal_hrd_parameters_present_flag: - bitPosition = self.hrd_parameters.toBytes(buf, bitPosition) - bitPosition = writeBoolean(buf, bitPosition, self.vcl_hrd_parameters_present_flag) + bit_position = self.hrd_parameters.toBytes(buf, bit_position) + bit_position = writeBoolean(buf, bit_position, self.vcl_hrd_parameters_present_flag) if self.vcl_hrd_parameters_present_flag: - bitPosition = self.vcl_hrd_parameters.toBytes(buf, bitPosition) + bit_position = self.vcl_hrd_parameters.toBytes(buf, bit_position) if self.nal_hrd_parameters_present_flag or self.vcl_hrd_parameters_present_flag: - bitPosition = writeBoolean(buf, bitPosition, self.low_delay_hrd_flag) - bitPosition = writeBoolean(buf, bitPosition, self.pic_struct_present_flag) - bitPosition = writeBoolean(buf, bitPosition, self.bitstream_restriction_flag) + bit_position = writeBoolean(buf, bit_position, self.low_delay_hrd_flag) + bit_position = writeBoolean(buf, bit_position, self.pic_struct_present_flag) + bit_position = writeBoolean(buf, bit_position, self.bitstream_restriction_flag) if self.bitstream_restriction_flag: - bitPosition = writeBoolean(buf, bitPosition, self.motion_vectors_over_pic_boundaries_flag) - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.max_bytes_per_pic_denom) - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.max_bits_per_mb_denom) - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.log2_max_mv_length_horizontal) - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.log2_max_mv_length_vertical) - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.max_num_reorder_frames) - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.max_dec_frame_buffering) + bit_position = writeBoolean(buf, bit_position, + self.motion_vectors_over_pic_boundaries_flag) + bit_position = writeUnsignedExpGolomb(buf, bit_position, self.max_bytes_per_pic_denom) + bit_position = writeUnsignedExpGolomb(buf, bit_position, self.max_bits_per_mb_denom) + bit_position = writeUnsignedExpGolomb(buf, bit_position, + self.log2_max_mv_length_horizontal) + bit_position = writeUnsignedExpGolomb(buf, bit_position, + self.log2_max_mv_length_vertical) + bit_position = writeUnsignedExpGolomb(buf, bit_position, self.max_num_reorder_frames) + bit_position = writeUnsignedExpGolomb(buf, bit_position, self.max_dec_frame_buffering) - return bitPosition + return bit_position @dataclass class SPS: @@ -851,8 +882,8 @@ class SPS: # TODO: ... # Compute options to pass to ffmpeg so as to reproduce the same SPS. # Very complex since some codec configuration are not provided by ffmpeg and/or libx264. - # This is only an attempt. - + # This is only an attempt for now and it is almost impossible to mimic any profile without + # patching ffmpeg and/or libx264 to add the support for corner cases. def ffmpegOptions(self, videoID=0): logger = logging.getLogger(__name__) x264opts = [] @@ -878,7 +909,8 @@ class SPS: x264opts.extend(['sps-id=%d' % self.seq_parameter_set_id] ) if self.bit_depth_chroma_minus8 not in [0,1,2,4,6,8]: - logger.error('Bit depth of chrominance is not supported: %d', self.bit_depth_chroma_minus8+8) + logger.error('Bit depth of chrominance is not supported: %d', + self.bit_depth_chroma_minus8+8) return [] if self.chroma_format_idc in range(0,4): @@ -903,165 +935,172 @@ class SPS: def fromBytes(self, buf): logger = logging.getLogger(__name__) - logger.debug('Parsing: %s' % (hexdump.dump(buf,sep=':'))) + logger.debug('Parsing: %s', hexdump.dump(buf,sep=':')) - bitPosition=0 + bit_position=0 # NAL Unit SPS - bitPosition, zero = readBit(buf, bitPosition) + bit_position, zero = readBit(buf, bit_position) if zero != 0: raise Exception('Reserved bit is not equal to 0: %d' % zero ) - bitPosition, nal_ref_idc = readBits(buf, bitPosition,2) + bit_position, nal_ref_idc = readBits(buf, bit_position,2) if nal_ref_idc != 3: raise Exception('NAL ref idc is not equal to 3: %d' % nal_ref_idc ) - bitPosition, nal_unit_type = readBits(buf, bitPosition,5) + bit_position, nal_unit_type = readBits(buf, bit_position,5) if nal_unit_type != 7: raise Exception('NAL unit type is not a SPS: %d' % nal_unit_type ) - bitPosition, self.profile_idc = readByte(buf, bitPosition) - bitPosition, self.constraint_set0_flag = readBit(buf,bitPosition) - bitPosition, self.constraint_set1_flag = readBit(buf,bitPosition) - bitPosition, self.constraint_set2_flag = readBit(buf,bitPosition) - bitPosition, self.constraint_set3_flag = readBit(buf,bitPosition) - bitPosition, self.constraint_set4_flag = readBit(buf,bitPosition) - bitPosition, self.constraint_set5_flag = readBit(buf,bitPosition) - bitPosition, v = readBits(buf, bitPosition, 2) + bit_position, self.profile_idc = readByte(buf, bit_position) + bit_position, self.constraint_set0_flag = readBit(buf,bit_position) + bit_position, self.constraint_set1_flag = readBit(buf,bit_position) + bit_position, self.constraint_set2_flag = readBit(buf,bit_position) + bit_position, self.constraint_set3_flag = readBit(buf,bit_position) + bit_position, self.constraint_set4_flag = readBit(buf,bit_position) + bit_position, self.constraint_set5_flag = readBit(buf,bit_position) + bit_position, v = readBits(buf, bit_position, 2) if v!=0: raise Exception('Reserved bits different from 0b00: %x' % v) - bitPosition, self.level_idc = readByte(buf, bitPosition) - bitPosition, self.seq_parameter_set_id = readUnsignedExpGolomb(buf, bitPosition) + bit_position, self.level_idc = readByte(buf, bit_position) + bit_position, self.seq_parameter_set_id = readUnsignedExpGolomb(buf, bit_position) if self.profile_idc in [44, 83, 86, 100, 110, 118, 122, 128, 134, 135, 138, 139, 244]: - bitPosition, self.chroma_format_idc = readUnsignedExpGolomb(buf, bitPosition) + bit_position, self.chroma_format_idc = readUnsignedExpGolomb(buf, bit_position) if self.chroma_format_idc==3: - bitPositionn, self.separate_colour_plane_flag=readBit(buf, bitPosition) - bitPosition, self.bit_depth_luma_minus8 = readUnsignedExpGolomb(buf, bitPosition) - bitPosition, self.bit_depth_chroma_minus8 = readUnsignedExpGolomb(buf, bitPosition) - bitPosition, self.qpprime_y_zero_transform_bypass_flag = readBoolean(buf, bitPosition) - bitPosition, self.seq_scaling_matrix_present_flag = readBoolean(buf, bitPosition) + bit_position, self.separate_colour_plane_flag=readBit(buf, bit_position) + bit_position, self.bit_depth_luma_minus8 = readUnsignedExpGolomb(buf, bit_position) + bit_position, self.bit_depth_chroma_minus8 = readUnsignedExpGolomb(buf, bit_position) + bit_position, self.qpprime_y_zero_transform_bypass_flag = readBoolean(buf, bit_position) + bit_position, self.seq_scaling_matrix_present_flag = readBoolean(buf, bit_position) if self.seq_scaling_matrix_present_flag: - nbMatrices = 12 if self.chroma_format_idc == 3 else 8 - for i in range(0, nbMatrices): - bitPosition, present = readBoolean(buf, bitPosition) + nb_matrices = 12 if self.chroma_format_idc == 3 else 8 + for i in range(0, nb_matrices): + bit_position, present = readBoolean(buf, bit_position) if present: if i<6: - bitPosition, matrix = parseScalingList(buf, bitPosition, 16) + bit_position, matrix = parseScalingList(buf, bit_position, 16) self.scaling_list[i] = matrix else: - bitPosition, matrix = parseScalingList(buf, bitPosition, 64) + bit_position, matrix = parseScalingList(buf, bit_position, 64) self.scaling_list[i] = matrix else: self.scaling_list[i] = [] - bitPosition, self.log2_max_frame_num_minus4 = readUnsignedExpGolomb(buf, bitPosition) - bitPosition , self.pic_order_cnt_type = readUnsignedExpGolomb(buf, bitPosition) + bit_position, self.log2_max_frame_num_minus4 = readUnsignedExpGolomb(buf, bit_position) + bit_position , self.pic_order_cnt_type = readUnsignedExpGolomb(buf, bit_position) if self.pic_order_cnt_type == 0: - bitPosition, self.log2_max_pic_order_cnt_lsb_minus4 = readUnsignedExpGolomb(buf, bitPosition) + bit_position, self.log2_max_pic_order_cnt_lsb_minus4 =\ + readUnsignedExpGolomb(buf, bit_position) elif self.pic_order_cnt_type == 1: - bitPosition, self.delta_pic_order_always_zero_flag = readBoolean(buf, bitPosition) - bitPosition, self.offset_for_non_ref_pic = readSignedExpGolomb(buf, bitPosition) - bitPosition, self.offset_for_top_to_bottom_field = readSignedExpGolomb(buf, bitPosition) - bitPosition, self.num_ref_frames_in_pic_order_cnt_cycle = readUnsignedExpGolomb(buf, bitPosition) + bit_position, self.delta_pic_order_always_zero_flag = readBoolean(buf, bit_position) + bit_position, self.offset_for_non_ref_pic = readSignedExpGolomb(buf, bit_position) + bit_position, self.offset_for_top_to_bottom_field = readSignedExpGolomb(buf, bit_position) + bit_position, self.num_ref_frames_in_pic_order_cnt_cycle =\ + readUnsignedExpGolomb(buf, bit_position) for i in range(0, self.num_ref_frames_in_pic_order_cnt_cycle): - bitPosition, v = readUnsignedExpGolomb(buf, bitPosition) + bit_position, v = readUnsignedExpGolomb(buf, bit_position) self.offset_for_ref_frame[i]=v - bitPosition, self.max_num_ref_frames = readUnsignedExpGolomb(buf, bitPosition) - bitPosition, self.gaps_in_frame_num_value_allowed_flag = readBoolean(buf, bitPosition) - bitPosition, self.pic_width_in_mbs_minus1 = readUnsignedExpGolomb(buf, bitPosition) - bitPosition, self.pic_height_in_map_units_minus1 = readUnsignedExpGolomb(buf, bitPosition) - bitPosition, self.frame_mbs_only_flag = readBoolean(buf, bitPosition) + bit_position, self.max_num_ref_frames = readUnsignedExpGolomb(buf, bit_position) + bit_position, self.gaps_in_frame_num_value_allowed_flag = readBoolean(buf, bit_position) + bit_position, self.pic_width_in_mbs_minus1 = readUnsignedExpGolomb(buf, bit_position) + bit_position, self.pic_height_in_map_units_minus1 = readUnsignedExpGolomb(buf, bit_position) + bit_position, self.frame_mbs_only_flag = readBoolean(buf, bit_position) if not self.frame_mbs_only_flag: - bitPosition, self.mb_adaptive_frame_field_flag = readBoolean(buf, bitPosition) - bitPosition, self.direct_8x8_inference_flag = readBoolean(buf, bitPosition) - bitPosition, self.frame_cropping_flag = readBoolean(buf, bitPosition) + bit_position, self.mb_adaptive_frame_field_flag = readBoolean(buf, bit_position) + bit_position, self.direct_8x8_inference_flag = readBoolean(buf, bit_position) + bit_position, self.frame_cropping_flag = readBoolean(buf, bit_position) if self.frame_cropping_flag: - bitPosition, self.frame_crop_left_offset = readUnsignedExpGolomb(buf, bitPosition) - bitPosition, self.frame_crop_right_offset = readUnsignedExpGolomb(buf, bitPosition) - bitPosition, self.frame_crop_top_offset = readUnsignedExpGolomb(buf, bitPosition) - bitPosition, self.frame_crop_bottom_offset = readUnsignedExpGolomb(buf, bitPosition) - bitPosition, self.vui_parameters_present_flag = readBoolean(buf, bitPosition) + bit_position, self.frame_crop_left_offset = readUnsignedExpGolomb(buf, bit_position) + bit_position, self.frame_crop_right_offset = readUnsignedExpGolomb(buf, bit_position) + bit_position, self.frame_crop_top_offset = readUnsignedExpGolomb(buf, bit_position) + bit_position, self.frame_crop_bottom_offset = readUnsignedExpGolomb(buf, bit_position) + bit_position, self.vui_parameters_present_flag = readBoolean(buf, bit_position) if self.vui_parameters_present_flag: self.vui = VUI() - bitPosition = self.vui.fromBytes(buf,bitPosition) + bit_position = self.vui.fromBytes(buf,bit_position) logger.debug('VUI present: %s', self.vui) - logger.debug('Parse end of SPS. Bit position: %d. Remaining bytes: %s.' % (bitPosition, hexdump.dump(buf[floor(bitPosition/8):], sep=':'))) - bitPosition = parseRBSPTrailingBits(buf, bitPosition) - logger.debug('End of SPS: %d. Remaining bytes: %s' % (bitPosition, hexdump.dump(buf[floor(bitPosition/8):], sep=':'))) - return bitPosition + logger.debug('Parse end of SPS. Bit position: %d. Remaining bytes: %s.', bit_position, + hexdump.dump(buf[floor(bit_position/8):], sep=':')) + bit_position = parseRBSPTrailingBits(buf, bit_position) + logger.debug('End of SPS: %d. Remaining bytes: %s', bit_position, + hexdump.dump(buf[floor(bit_position/8):], sep=':')) + return bit_position def toBytes(self): logger = logging.getLogger(__name__) buf = bytearray() - bitPosition = 0 - bitPosition = writeBit(buf, bitPosition,0) - bitPosition = writeBits(buf, bitPosition, 3, 2) - bitPosition = writeBits(buf, bitPosition, 7, 5) - bitPosition = writeByte(buf, bitPosition, self.profile_idc) - bitPosition = writeBit(buf, bitPosition, self.constraint_set0_flag) - bitPosition = writeBit(buf, bitPosition, self.constraint_set1_flag) - bitPosition = writeBit(buf, bitPosition, self.constraint_set2_flag) - bitPosition = writeBit(buf, bitPosition, self.constraint_set3_flag) - bitPosition = writeBit(buf, bitPosition, self.constraint_set4_flag) - bitPosition = writeBit(buf, bitPosition, self.constraint_set5_flag) - bitPosition = writeBits(buf, bitPosition, 0, 2) - bitPosition = writeByte(buf, bitPosition, self.level_idc) - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.seq_parameter_set_id) + bit_position = 0 + bit_position = writeBit(buf, bit_position,0) + bit_position = writeBits(buf, bit_position, 3, 2) + bit_position = writeBits(buf, bit_position, 7, 5) + bit_position = writeByte(buf, bit_position, self.profile_idc) + bit_position = writeBit(buf, bit_position, self.constraint_set0_flag) + bit_position = writeBit(buf, bit_position, self.constraint_set1_flag) + bit_position = writeBit(buf, bit_position, self.constraint_set2_flag) + bit_position = writeBit(buf, bit_position, self.constraint_set3_flag) + bit_position = writeBit(buf, bit_position, self.constraint_set4_flag) + bit_position = writeBit(buf, bit_position, self.constraint_set5_flag) + bit_position = writeBits(buf, bit_position, 0, 2) + bit_position = writeByte(buf, bit_position, self.level_idc) + bit_position = writeUnsignedExpGolomb(buf, bit_position, self.seq_parameter_set_id) if self.profile_idc in [44, 83, 86, 100, 110, 118, 122, 128, 134, 135, 138, 139, 244]: - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.chroma_format_idc) + bit_position = writeUnsignedExpGolomb(buf, bit_position, self.chroma_format_idc) if self.chroma_format_idc==3: - bitPosition = writeBit(buf, bitPosition, self.separate_colour_plane_flag) - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.bit_depth_luma_minus8) - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.bit_depth_chroma_minus8) - bitPosition = writeBoolean(buf, bitPosition, self.qpprime_y_zero_transform_bypass_flag ) - bitPosition = writeBoolean(buf, bitPosition, self.seq_scaling_matrix_present_flag) + bit_position = writeBit(buf, bit_position, self.separate_colour_plane_flag) + bit_position = writeUnsignedExpGolomb(buf, bit_position, self.bit_depth_luma_minus8) + bit_position = writeUnsignedExpGolomb(buf, bit_position, self.bit_depth_chroma_minus8) + bit_position = writeBoolean(buf, bit_position, self.qpprime_y_zero_transform_bypass_flag ) + bit_position = writeBoolean(buf, bit_position, self.seq_scaling_matrix_present_flag) if self.seq_scaling_matrix_present_flag: - nbMatrices = 12 if self.chroma_format_idc == 3 else 8 - for i in range(0, nbMatrices): + nb_matrices = 12 if self.chroma_format_idc == 3 else 8 + for i in range(0, nb_matrices): matrix = self.scaling_list[i] present = (len(matrix))!=0 - bitPosition = writeBoolean(buf, bitPosition, present) + bit_position = writeBoolean(buf, bit_position, present) if present: if i<6: - bitPosition = writeScalingList(buf, bitPosition, 16, matrix) + bit_position = writeScalingList(buf, bit_position, 16, matrix) else: - bitPosition = writeScalingList(buf, bitPosition, 64, matrix) + bit_position = writeScalingList(buf, bit_position, 64, matrix) - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.log2_max_frame_num_minus4) - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.pic_order_cnt_type) + bit_position = writeUnsignedExpGolomb(buf, bit_position, self.log2_max_frame_num_minus4) + bit_position = writeUnsignedExpGolomb(buf, bit_position, self.pic_order_cnt_type) if self.pic_order_cnt_type == 0: - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.log2_max_pic_order_cnt_lsb_minus4) + bit_position = writeUnsignedExpGolomb(buf, bit_position, + self.log2_max_pic_order_cnt_lsb_minus4) elif self.pic_order_cnt_type == 1: - bitPosition = writeBoolean(buf, bitPosition, self.delta_pic_order_always_zero_flag) - bitPosition = writeSignedExpGolomb(buf, bitPosition, self.offset_for_non_ref_pic) - bitPosition = writeSignedExpGolomb(buf, bitPosition, self.offset_for_top_to_bottom_field) - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.num_ref_frames_in_pic_order_cnt_cycle) + bit_position = writeBoolean(buf, bit_position, self.delta_pic_order_always_zero_flag) + bit_position = writeSignedExpGolomb(buf, bit_position, self.offset_for_non_ref_pic) + bit_position = writeSignedExpGolomb(buf, bit_position, + self.offset_for_top_to_bottom_field) + bit_position = writeUnsignedExpGolomb(buf, bit_position, + self.num_ref_frames_in_pic_order_cnt_cycle) for i in range(0, self.num_ref_frames_in_pic_order_cnt_cycle): v = self.offset_for_ref_frame[i] - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, v) - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.max_num_ref_frames) - bitPosition = writeBoolean(buf, bitPosition, self.gaps_in_frame_num_value_allowed_flag) - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.pic_width_in_mbs_minus1) - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.pic_height_in_map_units_minus1) - bitPosition = writeBoolean(buf, bitPosition, self.frame_mbs_only_flag) + bit_position = writeUnsignedExpGolomb(buf, bit_position, v) + bit_position = writeUnsignedExpGolomb(buf, bit_position, self.max_num_ref_frames) + bit_position = writeBoolean(buf, bit_position, self.gaps_in_frame_num_value_allowed_flag) + bit_position = writeUnsignedExpGolomb(buf, bit_position, self.pic_width_in_mbs_minus1) + bit_position = writeUnsignedExpGolomb(buf, bit_position, self.pic_height_in_map_units_minus1) + bit_position = writeBoolean(buf, bit_position, self.frame_mbs_only_flag) if not self.frame_mbs_only_flag: - bitPosition = writeBoolean(buf, bitPosition, self.mb_adaptive_frame_field_flag) - bitPosition = writeBoolean(buf, bitPosition, self.direct_8x8_inference_flag) - bitPosition = writeBoolean(buf, bitPosition, self.frame_cropping_flag) + bit_position = writeBoolean(buf, bit_position, self.mb_adaptive_frame_field_flag) + bit_position = writeBoolean(buf, bit_position, self.direct_8x8_inference_flag) + bit_position = writeBoolean(buf, bit_position, self.frame_cropping_flag) if self.frame_cropping_flag: - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.frame_crop_left_offset) - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.frame_crop_right_offset) - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.frame_crop_top_offset) - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.frame_crop_bottom_offset) - bitPosition = writeBoolean(buf, bitPosition, self.vui_parameters_present_flag) + bit_position = writeUnsignedExpGolomb(buf, bit_position, self.frame_crop_left_offset) + bit_position = writeUnsignedExpGolomb(buf, bit_position, self.frame_crop_right_offset) + bit_position = writeUnsignedExpGolomb(buf, bit_position, self.frame_crop_top_offset) + bit_position = writeUnsignedExpGolomb(buf, bit_position, self.frame_crop_bottom_offset) + bit_position = writeBoolean(buf, bit_position, self.vui_parameters_present_flag) if self.vui_parameters_present_flag: - logger.debug('SPS has VUI. Writing VUI at position: %d', bitPosition) - bitPosition = self.vui.toBytes(buf, bitPosition) - logger.debug('VUI written. New bit position: %d', bitPosition) + logger.debug('SPS has VUI. Writing VUI at position: %d', bit_position) + bit_position = self.vui.toBytes(buf, bit_position) + logger.debug('VUI written. New bit position: %d', bit_position) - bitPosition = writeRBSPTrailingBits(buf, bitPosition) + bit_position = writeRBSPTrailingBits(buf, bit_position) return buf @@ -1107,157 +1146,167 @@ class PPS: logger = logging.getLogger(__name__) logger.debug('Parsing: %s' % (hexdump.dump(buf,sep=':'))) - bitPosition=0 + bit_position=0 # NAL Unit PPS - bitPosition, zero = readBit(buf, bitPosition) + bit_position, zero = readBit(buf, bit_position) if zero != 0: raise Exception('Reserved bit is not equal to 0: %d' % zero ) - bitPosition, nal_ref_idc = readBits(buf, bitPosition,2) + bit_position, nal_ref_idc = readBits(buf, bit_position,2) if nal_ref_idc != 3: raise Exception('NAL ref idc is not equal to 3: %d' % nal_ref_idc ) - bitPosition, nal_unit_type = readBits(buf, bitPosition,5) + bit_position, nal_unit_type = readBits(buf, bit_position,5) if nal_unit_type != 8: raise Exception('NAL unit type is not a PPS: %d' % nal_unit_type ) - bitPosition, self.pic_parameter_set_id = readUnsignedExpGolomb(buf, bitPosition) - bitPosition, self.seq_parameter_set_id = readUnsignedExpGolomb(buf, bitPosition) - bitPosition, self.entropy_coding_mode_flag = readBoolean(buf, bitPosition) - bitPosition, self.bottom_field_pic_order_in_frame_present_flag = readBoolean(buf, bitPosition) - bitPosition, self.num_slice_groups_minus1 = readUnsignedExpGolomb(buf, bitPosition) + bit_position, self.pic_parameter_set_id = readUnsignedExpGolomb(buf, bit_position) + bit_position, self.seq_parameter_set_id = readUnsignedExpGolomb(buf, bit_position) + bit_position, self.entropy_coding_mode_flag = readBoolean(buf, bit_position) + bit_position, self.bottom_field_pic_order_in_frame_present_flag =\ + readBoolean(buf, bit_position) + bit_position, self.num_slice_groups_minus1 = readUnsignedExpGolomb(buf, bit_position) if self.num_slice_groups_minus1>0: - bitPosition, self.slice_group_map_type = readUnsignedExpGolomb(buf, bitPosition) + bit_position, self.slice_group_map_type = readUnsignedExpGolomb(buf, bit_position) if self.slice_group_map_type == 0: for i in range(0, self.num_slice_groups_minus1): - bitPosition, v = readUnsignedExpGolomb(buf, bitPosition) + bit_position, v = readUnsignedExpGolomb(buf, bit_position) self.run_length_minus1[i]=v elif self.slice_group_map_type == 2: for i in range(0, self.num_slice_groups_minus1): - bitPosition, v = readUnsignedExpGolomb(buf, bitPosition) + bit_position, v = readUnsignedExpGolomb(buf, bit_position) self.top_left[i] = v - bitPosition, v = readUnsignedExpGolomb(buf, bitPosition) + bit_position, v = readUnsignedExpGolomb(buf, bit_position) self.bottom_right[i] = v elif self.slice_group_map_type in [3,4,5]: - bitPosition, self.slice_group_change_direction_flag = readBoolean(buf, bitPosition) - bitPosition, self.slice_group_change_rate_minus1 = readUnsignedExpGolomb(buf, bitPosition) + bit_position, self.slice_group_change_direction_flag = readBoolean(buf, bit_position) + bit_position, self.slice_group_change_rate_minus1 =\ + readUnsignedExpGolomb(buf, bit_position) elif self.slice_group_map_type == 6: - bitPosition, self.pic_size_in_map_units_minus1 = readUnsignedExpGolomb(buf, bitPosition) + bit_position, self.pic_size_in_map_units_minus1 =\ + readUnsignedExpGolomb(buf, bit_position) l = ceil(log(self.num_slice_groups_minus1+1)) for i in range(0, self.pic_size_in_map_units_minus1): - bitPosition, v = readBits(buf, bitPosition, l) + bit_position, v = readBits(buf, bit_position, l) self.slice_group_id[i]=v - bitPosition, self.num_ref_idx_l0_default_active_minus1 = readUnsignedExpGolomb(buf, bitPosition) - bitPosition, self.num_ref_idx_l2_default_active_minus1 = readUnsignedExpGolomb(buf, bitPosition) - bitPosition, self.weighted_pred_flag = readBoolean(buf, bitPosition) - bitPosition, self.weighted_bipred_idc = readBits(buf, bitPosition, 2) - bitPosition, self.pic_init_qp_minus26 = readSignedExpGolomb(buf, bitPosition) - bitPosition, self.pic_init_qs_minus26 = readSignedExpGolomb(buf, bitPosition) - bitPosition, self.chroma_qp_index_offset = readSignedExpGolomb(buf, bitPosition) - bitPosition, self.deblocking_filter_control_present_flag = readBoolean(buf, bitPosition) - bitPosition, self.constrained_intra_pred_flag = readBoolean(buf, bitPosition) - bitPosition, self.redundant_pic_cnt_present_flag = readBoolean(buf, bitPosition) + bit_position, self.num_ref_idx_l0_default_active_minus1 =\ + readUnsignedExpGolomb(buf, bit_position) + bit_position, self.num_ref_idx_l2_default_active_minus1 =\ + readUnsignedExpGolomb(buf, bit_position) + bit_position, self.weighted_pred_flag = readBoolean(buf, bit_position) + bit_position, self.weighted_bipred_idc = readBits(buf, bit_position, 2) + bit_position, self.pic_init_qp_minus26 = readSignedExpGolomb(buf, bit_position) + bit_position, self.pic_init_qs_minus26 = readSignedExpGolomb(buf, bit_position) + bit_position, self.chroma_qp_index_offset = readSignedExpGolomb(buf, bit_position) + bit_position, self.deblocking_filter_control_present_flag = readBoolean(buf, bit_position) + bit_position, self.constrained_intra_pred_flag = readBoolean(buf, bit_position) + bit_position, self.redundant_pic_cnt_present_flag = readBoolean(buf, bit_position) - if moreRBSPData(buf, bitPosition): - bitPosition, self.transform_8x8_mode_flag = readBoolean(buf, bitPosition) - bitPosition, self.pic_scaling_matrix_present_flag = readBoolean(buf, bitPosition) + if moreRBSPData(buf, bit_position): + bit_position, self.transform_8x8_mode_flag = readBoolean(buf, bit_position) + bit_position, self.pic_scaling_matrix_present_flag = readBoolean(buf, bit_position) if self.pic_scaling_matrix_present_flag: - nbMatrices = 6 if chroma_format_idc == 3 else 2 + nb_matrices = 6 if chroma_format_idc == 3 else 2 if self.transform_8x8_mode_flag: - nbMatrices+=6 + nb_matrices+=6 else: - nbMatrices = 6 - for i in range(0, nbMatrices): - bitPosition, present = readBoolean(buf, bitPosition) + nb_matrices = 6 + for i in range(0, nb_matrices): + bit_position, present = readBoolean(buf, bit_position) if present: if i<6: - bitPosition, matrix = parseScalingList(buf, bitPosition, 16) + bit_position, matrix = parseScalingList(buf, bit_position, 16) self.pic_scaling_list.append(matrix) else: - bitPosition, matrix = parseScalingList(buf, bitPosition, 64) + bit_position, matrix = parseScalingList(buf, bit_position, 64) self.pic_scaling_list.append(matrix) else: self.pic_scaling_list.append([]) - bitPosition, self.second_chroma_qp_index_offset = readSignedExpGolomb(buf, bitPosition) + bit_position, self.second_chroma_qp_index_offset = readSignedExpGolomb(buf, bit_position) logger.info("parse RBSP") - bitPosition = parseRBSPTrailingBits(buf, bitPosition) + bit_position = parseRBSPTrailingBits(buf, bit_position) - return bitPosition + return bit_position def toBytes(self, chroma_format_idc): logger = logging.getLogger(__name__) buf = bytearray() - bitPosition = 0 + bit_position = 0 # NAL Unit PPS - bitPosition = writeBit(buf, bitPosition, 0) - bitPosition = writeBits(buf, bitPosition, 3, 2) - bitPosition = writeBits(buf, bitPosition, 8, 5) + bit_position = writeBit(buf, bit_position, 0) + bit_position = writeBits(buf, bit_position, 3, 2) + bit_position = writeBits(buf, bit_position, 8, 5) - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.pic_parameter_set_id) - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.seq_parameter_set_id) - bitPosition = writeBoolean(buf, bitPosition, self.entropy_coding_mode_flag) - bitPosition = writeBoolean(buf, bitPosition, self.bottom_field_pic_order_in_frame_present_flag) - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.num_slice_groups_minus1) + bit_position = writeUnsignedExpGolomb(buf, bit_position, self.pic_parameter_set_id) + bit_position = writeUnsignedExpGolomb(buf, bit_position, self.seq_parameter_set_id) + bit_position = writeBoolean(buf, bit_position, self.entropy_coding_mode_flag) + bit_position = writeBoolean(buf, bit_position,\ + self.bottom_field_pic_order_in_frame_present_flag) + bit_position = writeUnsignedExpGolomb(buf, bit_position, self.num_slice_groups_minus1) if self.num_slice_groups_minus1>0: - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.slice_group_map_type) + bit_position = writeUnsignedExpGolomb(buf, bit_position, self.slice_group_map_type) if self.slice_group_map_type == 0: for i in range(0, self.num_slice_groups_minus1): v = self.run_length_minus1[i] - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, v) + bit_position = writeUnsignedExpGolomb(buf, bit_position, v) elif self.slice_group_map_type == 2: for i in range(0, self.num_slice_groups_minus1): v = self.top_left[i] - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, v) + bit_position = writeUnsignedExpGolomb(buf, bit_position, v) v = self.bottom_right[i] - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, v) + bit_position = writeUnsignedExpGolomb(buf, bit_position, v) elif self.slice_group_map_type in [3,4,5]: - bitPosition = writeBoolean(buf, bitPosition, self.slice_group_change_direction_flag) - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.slice_group_change_rate_minus1) + bit_position = writeBoolean(buf, bit_position, self.slice_group_change_direction_flag) + bit_position = writeUnsignedExpGolomb(buf, bit_position, + self.slice_group_change_rate_minus1) elif self.slice_group_map_type == 6: - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.pic_size_in_map_units_minus1) + bit_position = writeUnsignedExpGolomb(buf, bit_position, + self.pic_size_in_map_units_minus1) l = ceil(log(self.num_slice_groups_minus1+1)) for i in range(0, self.pic_size_in_map_units_minus1): v = self.slice_group_id[i] - bitPosition, v = writeBits(buf, bitPosition, v, l) + bit_position, v = writeBits(buf, bit_position, v, l) - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.num_ref_idx_l0_default_active_minus1) - bitPosition = writeUnsignedExpGolomb(buf, bitPosition, self.num_ref_idx_l2_default_active_minus1) - bitPosition = writeBoolean(buf, bitPosition, self.weighted_pred_flag) - bitPosition = writeBits(buf, bitPosition, self.weighted_bipred_idc, 2) - bitPosition = writeSignedExpGolomb(buf, bitPosition, self.pic_init_qp_minus26) - bitPosition = writeSignedExpGolomb(buf, bitPosition, self.pic_init_qs_minus26) - bitPosition = writeSignedExpGolomb(buf, bitPosition, self.chroma_qp_index_offset) - bitPosition = writeBoolean(buf, bitPosition, self.deblocking_filter_control_present_flag) - bitPosition = writeBoolean(buf, bitPosition, self.constrained_intra_pred_flag) - bitPosition = writeBoolean(buf, bitPosition, self.redundant_pic_cnt_present_flag) + bit_position = writeUnsignedExpGolomb(buf, bit_position, + self.num_ref_idx_l0_default_active_minus1) + bit_position = writeUnsignedExpGolomb(buf, bit_position, + self.num_ref_idx_l2_default_active_minus1) + bit_position = writeBoolean(buf, bit_position, self.weighted_pred_flag) + bit_position = writeBits(buf, bit_position, self.weighted_bipred_idc, 2) + bit_position = writeSignedExpGolomb(buf, bit_position, self.pic_init_qp_minus26) + bit_position = writeSignedExpGolomb(buf, bit_position, self.pic_init_qs_minus26) + bit_position = writeSignedExpGolomb(buf, bit_position, self.chroma_qp_index_offset) + bit_position = writeBoolean(buf, bit_position, self.deblocking_filter_control_present_flag) + bit_position = writeBoolean(buf, bit_position, self.constrained_intra_pred_flag) + bit_position = writeBoolean(buf, bit_position, self.redundant_pic_cnt_present_flag) - bitPosition = writeBoolean(buf, bitPosition, self.transform_8x8_mode_flag) - bitPosition = writeBoolean(buf, bitPosition, self.pic_scaling_matrix_present_flag) + bit_position = writeBoolean(buf, bit_position, self.transform_8x8_mode_flag) + bit_position = writeBoolean(buf, bit_position, self.pic_scaling_matrix_present_flag) if self.pic_scaling_matrix_present_flag: - nbMatrices = 6 if chroma_format_idc == 3 else 2 + nb_matrices = 6 if chroma_format_idc == 3 else 2 if self.transform_8x8_mode_flag: - nbMatrices+=6 + nb_matrices+=6 else: - nbMatrices = 6 - for i in range(0, nbMatrices): + nb_matrices = 6 + for i in range(0, nb_matrices): matrix = self.pic_scaling_list[i] logger.info("Retrieved pic scaling matrix: %s %d" % (matrix, len(matrix))) present = len(matrix)!=0 logger.info("Matrix is present: %s" % present) - bitPosition = writeBoolean(buf, bitPosition, present) + bit_position = writeBoolean(buf, bit_position, present) if present: if i<6: logger.info("Writing matrix: %s" % matrix) - bitPosition = writeScalingList(buf, bitPosition, 16, matrix) + bit_position = writeScalingList(buf, bit_position, 16, matrix) else: logger.info("Writing matrix: %s" % matrix) - bitPosition = writeScalingList(buf, bitPosition, 64, matrix) - bitPosition = writeSignedExpGolomb(buf, bitPosition, self.second_chroma_qp_index_offset) + bit_position = writeScalingList(buf, bit_position, 64, matrix) + bit_position = writeSignedExpGolomb(buf, bit_position, self.second_chroma_qp_index_offset) - bitPosition = writeRBSPTrailingBits(buf, bitPosition) + bit_position = writeRBSPTrailingBits(buf, bit_position) return buf @@ -1286,74 +1335,79 @@ class AVCDecoderConfiguration: def fromBytes(self, buf): logger = logging.getLogger(__name__) logger.debug('Parsing: %s' % (hexdump.dump(buf,sep=':'))) - bitPosition = 0 - bitPosition, self.configurationVersion = readByte(buf, bitPosition) - bitPosition, self.AVCProfileIndication = readByte(buf, bitPosition) - bitPosition, self.profile_compatibility = readByte(buf, bitPosition) - bitPosition, self.AVCLevelIndication = readByte(buf, bitPosition) - bitPosition, v = readBits(buf, bitPosition, 6) + bit_position = 0 + bit_position, self.configurationVersion = readByte(buf, bit_position) + bit_position, self.AVCProfileIndication = readByte(buf, bit_position) + bit_position, self.profile_compatibility = readByte(buf, bit_position) + bit_position, self.AVCLevelIndication = readByte(buf, bit_position) + bit_position, v = readBits(buf, bit_position, 6) if v != 0b111111: raise Exception('Reserved bits are not equal to 0b111111: %x' % v ) - bitPosition, self.lengthSizeMinusOne = readBits(buf, bitPosition, 2) - bitPosition, v = readBits(buf, bitPosition, 3) + bit_position, self.lengthSizeMinusOne = readBits(buf, bit_position, 2) + bit_position, v = readBits(buf, bit_position, 3) if v != 0b111: raise Exception('Reserved bits are not equal to 0b111: %x' % v) - bitPosition, self.numOfSequenceParameterSets= readBits(buf, bitPosition, 5) + bit_position, self.numOfSequenceParameterSets= readBits(buf, bit_position, 5) logger.debug('Number of SPS: %d' % self.numOfSequenceParameterSets) - for i in range(0,self.numOfSequenceParameterSets): - bitPosition, length = readWord(buf, bitPosition) - if bitPosition % 8 != 0: - raise Exception('SPS is not located at a byte boundary: %d' % bitPosition ) + for _ in range(0,self.numOfSequenceParameterSets): + bit_position, length = readWord(buf, bit_position) + if bit_position % 8 != 0: + raise Exception('SPS is not located at a byte boundary: %d' % bit_position ) sps = SPS() - sodb = RBSP2SODB(buf[floor(bitPosition/8):]) - bitLength = sps.fromBytes(sodb) + sodb = RBSP2SODB(buf[floor(bit_position/8):]) + bit_length = sps.fromBytes(sodb) spsid = sps.seq_parameter_set_id self.sps[spsid] = sps - parsedLength = floor(bitLength/8) - logger.debug('Expected length of SPS: %d bytes. Parsed: %d bytes' % (length, parsedLength)) - # Parse length can be shorter than length because of rewriting from RBSP to SODB (that is shorter). + parsed_length = floor(bit_length/8) + logger.debug('Expected length of SPS: %d bytes. Parsed: %d bytes', length, + parsed_length) + # Parse length can be shorter than length because of rewriting from RBSP to SODB + # (that is shorter). # So we advance of indicated length. - bitPosition+=length*8 + bit_position+=length*8 - logger.debug('Bit position:%d. Reading one byte of: %s' % (bitPosition, hexdump.dump(buf[floor(bitPosition/8):], sep=':'))) - bitPosition, self.numOfPictureParameterSets = readByte(buf, bitPosition) + logger.debug('Bit position:%d. Reading one byte of: %s', bit_position, + hexdump.dump(buf[floor(bit_position/8):], sep=':')) + bit_position, self.numOfPictureParameterSets = readByte(buf, bit_position) logger.debug('Number of PPS: %d' % self.numOfPictureParameterSets) - for i in range(0,self.numOfPictureParameterSets): - bitPosition, length = readWord(buf, bitPosition) - if bitPosition % 8 != 0: - raise Exception('PPS is not located at a byte boundary: %d' % bitPosition ) + for _ in range(0,self.numOfPictureParameterSets): + bit_position, length = readWord(buf, bit_position) + if bit_position % 8 != 0: + raise Exception('PPS is not located at a byte boundary: %d' % bit_position ) pps = PPS() - sodb = RBSP2SODB(buf[floor(bitPosition/8):]) - bitLength = pps.fromBytes(sodb, self.chroma_format) + sodb = RBSP2SODB(buf[floor(bit_position/8):]) + bit_length = pps.fromBytes(sodb, self.chroma_format) ppsid = pps.pic_parameter_set_id self.pps[ppsid] = pps - parsedLength = floor(bitLength/8) - logger.debug('Expected length of PPS: %d bytes. Parsed: %d bytes' % (length, parsedLength)) - # Parse length can be shorter than length because of rewriting from RBSP to SODB (that is shorter). + parsed_length = floor(bit_length/8) + logger.debug('Expected length of PPS: %d bytes. Parsed: %d bytes', length, + parsed_length) + # Parse length can be shorter than length because of rewriting from RBSP to SODB + # (that is shorter). # So we advance of indicated length. - bitPosition+=length*8 + bit_position+=length*8 - logger.debug('Remaining bits: %s' % hexdump.dump(buf[floor(bitPosition/8):])) + logger.debug('Remaining bits: %s', hexdump.dump(buf[floor(bit_position/8):])) if self.AVCProfileIndication in [100, 110, 122, 144]: - bitPosition, reserved = readBits(buf, bitPosition, 6) + bit_position, reserved = readBits(buf, bit_position, 6) if reserved != 0b111111: raise Exception('Reserved bits are different from 111111: %x' % reserved) - bitPosition, self.chroma_format = readBits(buf, bitPosition, 2) - bitPosition, reserved = readBits(buf, bitPosition, 5) + bit_position, self.chroma_format = readBits(buf, bit_position, 2) + bit_position, reserved = readBits(buf, bit_position, 5) if reserved != 0b11111: raise Exception('Reserved bits are different from 11111: %x' % reserved) - bitPosition, self.bit_depth_luma_minus8 = readBits(buf, bitPosition, 3) - bitPosition, reserved = readBits(buf, bitPosition, 5) + bit_position, self.bit_depth_luma_minus8 = readBits(buf, bit_position, 3) + bit_position, reserved = readBits(buf, bit_position, 5) if reserved != 0b11111: raise Exception('Reserved bits are different from 11111: %x' % reserved) - bitPosition, self.bit_depth_chroma_minus8 = readBits(buf, bitPosition, 3) - bitPosition, self.numOfSequenceParameterSetExt = readByte(buf, bitPosition) - for i in range(0, self.numOfSequenceParameterSetExt): + bit_position, self.bit_depth_chroma_minus8 = readBits(buf, bit_position, 3) + bit_position, self.numOfSequenceParameterSetExt = readByte(buf, bit_position) + for _ in range(0, self.numOfSequenceParameterSetExt): # TODO: parse SPSextended logger.error('Parsing of SPS extended not yet implemented !') pass @@ -1364,55 +1418,55 @@ class AVCDecoderConfiguration: logger = logging.getLogger(__name__) buf = bytearray() - bitPosition = 0 - bitPosition = writeByte(buf, bitPosition, self.configurationVersion) - bitPosition = writeByte(buf, bitPosition, self.AVCProfileIndication) - bitPosition = writeByte(buf, bitPosition, self.profile_compatibility) - bitPosition = writeByte(buf, bitPosition, self.AVCLevelIndication) - bitPosition = writeBits(buf, bitPosition, 0b111111, 6) - bitPosition = writeBits(buf, bitPosition, self.lengthSizeMinusOne, 2) - bitPosition = writeBits(buf, bitPosition, 0b111, 3) - bitPosition = writeBits(buf, bitPosition, self.numOfSequenceParameterSets, 5) + bit_position = 0 + bit_position = writeByte(buf, bit_position, self.configurationVersion) + bit_position = writeByte(buf, bit_position, self.AVCProfileIndication) + bit_position = writeByte(buf, bit_position, self.profile_compatibility) + bit_position = writeByte(buf, bit_position, self.AVCLevelIndication) + bit_position = writeBits(buf, bit_position, 0b111111, 6) + bit_position = writeBits(buf, bit_position, self.lengthSizeMinusOne, 2) + bit_position = writeBits(buf, bit_position, 0b111, 3) + bit_position = writeBits(buf, bit_position, self.numOfSequenceParameterSets, 5) for spsid in self.sps: sps = self.sps[spsid] sodb = sps.toBytes() - sodbLength = len(sodb) + sodb_length = len(sodb) rbsp = SODB2RBSP(sodb) - rbspLength = len(rbsp) + rbsp_length = len(rbsp) - logger.debug('SODB length: %d RBSP length:%d' % (sodbLength, rbspLength)) + logger.debug('SODB length: %d RBSP length:%d' % (sodb_length, rbsp_length)) - bitPosition = writeWord(buf, bitPosition, rbspLength) + bit_position = writeWord(buf, bit_position, rbsp_length) buf.extend(rbsp) - bitPosition+=rbspLength*8 + bit_position+=rbsp_length*8 logger.debug('2. Buffer: %s' % hexdump.dump(buf, sep=':')) - bitPosition = writeByte(buf, bitPosition, self.numOfPictureParameterSets) + bit_position = writeByte(buf, bit_position, self.numOfPictureParameterSets) for ppsid in self.pps: logger.debug('Writing PPS: %d' % ppsid) pps = self.pps[ppsid] # TODO: does chroma_format should come from self ? sodb = pps.toBytes(self.chroma_format) - sodbLength = len(sodb) + sodb_length = len(sodb) rbsp = SODB2RBSP(sodb) - rbspLength = len(rbsp) + rbsp_length = len(rbsp) - logger.debug('SODB length: %d RBSP length:%d' % (sodbLength, rbspLength)) + logger.debug('SODB length: %d RBSP length:%d' % (sodb_length, rbsp_length)) - bitPosition = writeWord(buf, bitPosition, rbspLength) + bit_position = writeWord(buf, bit_position, rbsp_length) buf.extend(rbsp) - bitPosition+=rbspLength*8 + bit_position+=rbsp_length*8 if self.AVCProfileIndication in [ 100, 110, 122, 144]: - bitPosition = writeBits(buf, bitPosition, 0b111111, 6) - bitPosition = writeBits(buf, bitPosition, self.chroma_format, 2) - bitPosition = writeBits(buf, bitPosition, 0b11111, 5) - bitPosition = writeBits(buf, bitPosition, self.bit_depth_luma_minus8, 3) - bitPosition = writeBits(buf, bitPosition, 0b11111, 5) - bitPosition = writeBits(buf, bitPosition, self.bit_depth_chroma_minus8, 3) - bitPosition = writeByte(buf, bitPosition, self.numOfSequenceParameterSetExt) - for i in range(0, self.numOfSequenceParameterSetExt): + bit_position = writeBits(buf, bit_position, 0b111111, 6) + bit_position = writeBits(buf, bit_position, self.chroma_format, 2) + bit_position = writeBits(buf, bit_position, 0b11111, 5) + bit_position = writeBits(buf, bit_position, self.bit_depth_luma_minus8, 3) + bit_position = writeBits(buf, bit_position, 0b11111, 5) + bit_position = writeBits(buf, bit_position, self.bit_depth_chroma_minus8, 3) + bit_position = writeByte(buf, bit_position, self.numOfSequenceParameterSetExt) + for _ in range(0, self.numOfSequenceParameterSetExt): # TODO: dump SPSextended logger.error('Dumping SPS extended not yet implemented') pass @@ -1422,28 +1476,37 @@ class AVCDecoderConfiguration: def merge(self, config): # Check config compatibility if self.configurationVersion != config.configurationVersion: - raise Exception('Configuration versions are different: %d vs %s' % (self.configurationVersion, config.configurationVersion)) + raise Exception('Configuration versions are different: %d vs %s' %\ + (self.configurationVersion, config.configurationVersion)) if self.AVCProfileIndication != config.AVCProfileIndication: - raise Exception('AVC profiles are different: %d vs %s' % (self.AVCProfileIndication, config.AVCProfileIndication)) + raise Exception('AVC profiles are different: %d vs %s' %\ + (self.AVCProfileIndication, config.AVCProfileIndication)) if self.profile_compatibility != config.profile_compatibility: - raise Exception('Profile compatilities are different: %d vs %s' % (self.profile_compatibility, config.profile_compatibility)) + raise Exception('Profile compatilities are different: %d vs %s' %\ + (self.profile_compatibility, config.profile_compatibility)) if self.AVCLevelIndication != config.AVCLevelIndication: - raise Exception('Level indications are different: %d vs %s' % (self.AVCLevelIndication, config.AVCLevelIndication)) + raise Exception('Level indications are different: %d vs %s' %\ + (self.AVCLevelIndication, config.AVCLevelIndication)) if self.lengthSizeMinusOne != config.lengthSizeMinusOne: - raise Exception('Length units are different: %d vs %s' % (self.lengthSizeMinusOne, config.lengthSizeMinusOne)) + raise Exception('Length units are different: %d vs %s' %\ + (self.lengthSizeMinusOne, config.lengthSizeMinusOne)) if self.chroma_format != config.chroma_format: - raise Exception('Colour format are different: %d vs %s' % (self.chroma_format, config.chroma_format)) + raise Exception('Colour format are different: %d vs %s' %\ + (self.chroma_format, config.chroma_format)) if self.bit_depth_luma_minus8 != config.bit_depth_luma_minus8: - raise Exception('Depth of luminance are different: %d vs %s' % (self.bit_depth_luma_minus8, config.bit_depth_luma_minus8)) + raise Exception('Depth of luminance are different: %d vs %s' %\ + (self.bit_depth_luma_minus8, config.bit_depth_luma_minus8)) if self.bit_depth_chroma_minus8 != config.bit_depth_chroma_minus8: - raise Exception('Depth of chromaticity are different: %d vs %s' % (self.bit_depth_chroma_minus8, config.bit_depth_luma_minus8)) + raise Exception('Depth of chromaticity are different: %d vs %s' %\ + (self.bit_depth_chroma_minus8, config.bit_depth_luma_minus8)) for spsid in config.sps: sps = config.sps[spsid] if spsid in self.sps: localsps = self.sps[spsid] if sps!=localsps: - raise Exception('Profile are not compatible. They contain two different SPS with the same identifier (%d): %s\n%s\n' % (spsid, localsps, sps)) + raise Exception('Profile are not compatible. They contain two different SPS\ + with the same identifier (%d): %s\n%s\n' % (spsid, localsps, sps)) self.sps[spsid] = sps self.numOfSequenceParameterSets = len(self.sps) @@ -1453,7 +1516,8 @@ class AVCDecoderConfiguration: if ppsid in self.pps: localpps = self.pps[ppsid] if pps!=localpps: - raise Exception('Profile are not compatible. They contain two different PPS with the same identifier (%d): %s\n%s\n' % (ppsid, localpps, pps)) + raise Exception('Profile are not compatible. They contain two different PPS\ + with the same identifier (%d): %s\n%s\n' % (ppsid, localpps, pps)) self.pps[ppsid] = pps self.numOfPictureParameterSets = len(self.pps) @@ -1468,18 +1532,18 @@ def parseCodecPrivate(codecPrivateData): length = codecPrivateData[2] if length == 0: raise Exception('Matroska length cannot start with zero byte.') - for nbZeroes in range(0,8): - b = readBit(codecPrivateData[2:], nbZeroes) + for nb_zeroes in range(0,8): + b = readBit(codecPrivateData[2:], nb_zeroes) if b != 0: break - mask = 2^(7-nbZeroes)-1 + mask = 2^(7-nb_zeroes)-1 length = codecPrivateData[2] and mask - for i in range(0, nbZeroes): + for i in range(0, nb_zeroes): length*=256 length+=(codecPrivateData[3+i]) - bytePosition = 3+nbZeroes + byte_position = 3+nb_zeroes avcconfig = AVCDecoderConfiguration() - avcconfig.fromBytes(codecPrivateData[bytePosition:]) + avcconfig.fromBytes(codecPrivateData[byte_position:]) return avcconfig @@ -1490,19 +1554,19 @@ def getAvcConfigFromH264(inputFile): rbsp = inputFile.read(1000) sodb = RBSP2SODB(rbsp) - bitPosition = 0 - bitPosition, startCode = readLong(sodb, bitPosition) - if startCode != 1: - raise Exception('Starting code not detected: %x' % startCode) + bit_position = 0 + bit_position, start_code = readLong(sodb, bit_position) + if start_code != 1: + raise Exception('Starting code not detected: %x' % start_code) sps = SPS() - bitLength = sps.fromBytes(sodb[4:]) - bitPosition+=bitLength + bit_length = sps.fromBytes(sodb[4:]) + bit_position+=bit_length - bitPosition, startCode = readLong(sodb, bitPosition) - if startCode != 1: - raise Exception('Starting code not detected: %x' % startCode) + bit_position, start_code = readLong(sodb, bit_position) + if start_code != 1: + raise Exception('Starting code not detected: %x' % start_code) pps = PPS() - bitLength = pps.fromBytes(sodb[floor(bitPosition/8):], sps.chroma_format_idc) + bit_length = pps.fromBytes(sodb[floor(bit_position/8):], sps.chroma_format_idc) logger.debug(pps) avcconfig = AVCDecoderConfiguration() @@ -1539,17 +1603,19 @@ def parseMKVTree(mkvinfo, inputFile): env = {**os.environ, 'LANG': 'C'} elements = {} - with Popen([mkvinfo, '-z', '-X', '-P', '/proc/self/fd/%d' % infd ], stdout=PIPE, close_fds=False, env=env) as mkvinfo: + with Popen([mkvinfo, '-z', '-X', '-P', '/proc/self/fd/%d' % infd ], stdout=PIPE, + close_fds=False, env=env) as mkvinfo: out, _ = mkvinfo.communicate() out = out.decode('utf8') prefix = [] - regExp = r"(^(?P\+)|(\|(?P[ ]*\+))).*at (?P[0-9]+) size (?P[0-9]+).*$" - p = re.compile(regExp) - prevDepth = -1 + reg_exp = (r"(^(?P\+)|(\|(?P[ ]*\+))).*at (?P[0-9]+)" + r" size (?P[0-9]+).*$") + p = re.compile(reg_exp) + prev_depth = -1 for line in out.splitlines(): m = p.match(line) if m is None: - logger.error("Impossible to match line: %s" % line) + logger.error("Impossible to match line: %s", line) else: position = int(m.group('position')) size = int(m.group('size')) @@ -1559,23 +1625,23 @@ def parseMKVTree(mkvinfo, inputFile): else: depth = len(m.group('depth')) - if depth > prevDepth: - for i in range(depth-prevDepth): + if depth > prev_depth: + for _ in range(depth-prev_depth): prefix.append(1) - elif depth == prevDepth: + elif depth == prev_depth: subid = prefix[-1] subid+=1 prefix.pop() prefix.append(subid) else: - for i in range(prevDepth-depth): + for _ in range(prev_depth-depth): prefix.pop() subid = prefix[-1] subid+=1 prefix.pop() prefix.append(subid) - prevDepth = depth + prev_depth = depth key=".".join(map(str, prefix)) elements[key] = (position, size) @@ -1591,14 +1657,22 @@ def parseMKVTree(mkvinfo, inputFile): # 001x xxxx xxxx xxxx xxxx xxxx - Class C IDs (2^21-1 possible values) # 0001 xxxx xxxx xxxx xxxx xxxx xxxx xxxx - Class D IDs (2^28-1 possible values) # Lengths are encoded as follows: -# 1xxx xxxx - value 0 to 2^7-2 -# 01xx xxxx xxxx xxxx - value 0 to 2^14-2 -# 001x xxxx xxxx xxxx xxxx xxxx - value 0 to 2^21-2 -# 0001 xxxx xxxx xxxx xxxx xxxx xxxx xxxx - value 0 to 2^28-2 -# 0000 1xxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx - value 0 to 2^35-2 -# 0000 01xx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx - value 0 to 2^42-2 -# 0000 001x xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx - value 0 to 2^49-2 -# 0000 0001 xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx - value 0 to 2^56-2 +# 1xxx xxxx +# value 0 to 2^7-2 +# 01xx xxxx xxxx xxxx +# value 0 to 2^14-2 +# 001x xxxx xxxx xxxx xxxx xxxx +# value 0 to 2^21-2 +# 0001 xxxx xxxx xxxx xxxx xxxx xxxx xxxx +# value 0 to 2^28-2 +# 0000 1xxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx +# value 0 to 2^35-2 +# 0000 01xx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx +# value 0 to 2^42-2 +# 0000 001x xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx +# value 0 to 2^49-2 +# 0000 0001 xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx +# value 0 to 2^56-2 def getEBMLLength(length): @@ -1627,8 +1701,8 @@ def getEBMLLength(length): logger.error('Impossible to encode a length larger than 2^56-2 with EBML.') return None - encodedLength = length + ((128>>(size-1))<<((size-1)*8)) - res = (encodedLength).to_bytes(size, byteorder='big') + encoded_length = length + ((128>>(size-1))<<((size-1)*8)) + res = (encoded_length).to_bytes(size, byteorder='big') return res @@ -1639,11 +1713,12 @@ def dumpCodecPrivateData(AVCDecoderConfiguration): # Code private element res.extend(b'\x63\xA2') buf = AVCDecoderConfiguration.toBytes() - logger.debug('AVC configuration bitstream: %s (length: %d))' % (hexdump.dump(buf, sep=':'), len(buf))) + logger.debug('AVC configuration bitstream: %s (length: %d))', hexdump.dump(buf, sep=':'), + len(buf)) - EMBLlength = getEBMLLength(len(buf)) - logger.debug('EMBL encoded length: %s' % (hexdump.dump(EMBLlength, sep=':'))) - res.extend(EMBLlength) + embl_length = getEBMLLength(len(buf)) + logger.debug('EMBL encoded length: %s', hexdump.dump(embl_length, sep=':')) + res.extend(embl_length) res.extend(buf) return res @@ -1652,86 +1727,89 @@ def dumpCodecPrivateData(AVCDecoderConfiguration): def changeEBMLElementSize(inputFile, position, addendum): logger = logging.getLogger(__name__) - initialPosition = position + initial_position = position infd = inputFile.fileno() lseek(infd, position, SEEK_SET) buf = read(infd, 1) - elementType = int.from_bytes(buf, byteorder='big') + element_type = int.from_bytes(buf, byteorder='big') mask=128 found = False for i in range(1,5): - if elementType&mask: - typeSize = i + if element_type&mask: + type_size = i found = True break else: mask = mask>>1 if not found: - logger.error('Size of element type cannot be determined: %d', elementType) + logger.error('Size of element type cannot be determined: %d', element_type) exit(-1) # We seek to size - position+=typeSize + position+=type_size lseek(infd, position, SEEK_SET) buf = read(infd, 1) - sizeHead = int.from_bytes(buf, byteorder='big') - logger.info('First byte of size: %x' % sizeHead) + size_head = int.from_bytes(buf, byteorder='big') + logger.info('First byte of size: %x' % size_head) mask=128 found = False for i in range(1,9): - if sizeHead&mask: - sizeOfDataSize = i + if size_head&mask: + size_of_data_size = i found = True break else: mask = mask>>1 if not found: - logger.error('Size of data size cannot be determined: %d', sizeHead) + logger.error('Size of data size cannot be determined: %d', size_head) exit(-1) else: - logger.info('Size of data size: %d.', sizeOfDataSize) + logger.info('Size of data size: %d.', size_of_data_size) lseek(infd, position, SEEK_SET) - oldSizeBuf = read(infd, sizeOfDataSize) - maxSize = 2**(sizeOfDataSize*7)-2 - sizeOfData = int.from_bytes(oldSizeBuf, byteorder='big') - logger.info('Size of data with mask: %x mask: %d.' % (sizeOfData, mask)) - sizeOfData-= (mask<<((sizeOfDataSize-1)*8)) - logger.info('Found element at position: %d, size of type: %d size of data: %d maximal size: %d.', initialPosition, typeSize, sizeOfData, maxSize) + oldSizeBuf = read(infd, size_of_data_size) + max_size = 2**(size_of_data_size*7)-2 + size_of_data = int.from_bytes(oldSizeBuf, byteorder='big') + logger.info('Size of data with mask: %x mask: %d.' % (size_of_data, mask)) + size_of_data-= (mask<<((size_of_data_size-1)*8)) + logger.info('Found element at position: %d, size of type: %d size of data: %d \ + maximal size: %d.', initial_position, type_size, size_of_data, max_size) - newSize = sizeOfData+addendum + newSize = size_of_data+addendum delta = 0 - if newSize > maxSize: + if newSize > max_size: # TODO: Test this code ... - newEncodedSize = getEBMLLength(newSize) - sizeOfNewEncodedSize = len(newEncodedSize) - if sizeOfNewEncodedSize <= sizeOfDataSize: - logger.error('New encoded size is smaller (%d) or equal than previous size (%d). This should not happen.' , sizeOfNewEncodedSize, sizeOfDataSize) + new_encoded_size = getEBMLLength(newSize) + sizeOfNewEncodedSize = len(new_encoded_size) + if sizeOfNewEncodedSize <= size_of_data_size: + logger.error('New encoded size is smaller (%d) or equal than previous size (%d).\ + This should not happen.', sizeOfNewEncodedSize, size_of_data_size) exit(-1) # The difference of length between old size field and new one. - delta = sizeOfNewEncodedSize - sizeOfDataSize + delta = sizeOfNewEncodedSize - size_of_data_size fileLength = fstat(infd).st_size # We seek after actual length field - lseek(infd, position+sizeOfDataSize, SEEK_SET) + lseek(infd, position+size_of_data_size, SEEK_SET) # We read the rest of file - tail = read(infd, fileLength-(position+sizeOfDataSize)) + tail = read(infd, fileLength-(position+size_of_data_size)) # We increase file length ftruncate(infd, fileLength+delta) # We go to the beginning of length field lseek(infd, position, SEEK_SET) # We write the new length field - write(infd, newEncodedSize) + write(infd, new_encoded_size) # We overwrite the rest of file with its previous content that has been offset. write(infd, tail) else: - size = newSize + ((128>>(sizeOfDataSize-1))<<((sizeOfDataSize-1)*8)) - newSizeBuf = (size).to_bytes(sizeOfDataSize, byteorder='big') + size = newSize + ((128>>(size_of_data_size-1))<<((size_of_data_size-1)*8)) + newSizeBuf = (size).to_bytes(size_of_data_size, byteorder='big') - logger.info('Old encoded size: %s New encoded size: %s' % (hexdump.dump(oldSizeBuf,sep=':'), hexdump.dump(newSizeBuf, sep=':'))) + logger.info('Old encoded size: %s New encoded size: %s', hexdump.dump(oldSizeBuf,sep=':'), + hexdump.dump(newSizeBuf, sep=':')) lseek(infd, position, SEEK_SET) write(infd, newSizeBuf) @@ -1798,13 +1876,15 @@ def changeCodecPrivateData(mkvinfo, inputFile, codecData): delta = futureLength-currentLength # if there is no modification of the private codec data, no need to change anything. if delta != 0: - for i in range(0, len(keys)-1): + for _ in range(0, len(keys)-1): keys.pop() key=".".join(map(str, keys)) pos, size = elements[key] - logger.info('Trying to fix element with key: %s at position: %d with actual size: %d.' % (key, pos, size)) + logger.info('Trying to fix element with key: %s at position: %d with actual size: %d.', + key, pos, size) # Changing an element can increase its size (in very rare case). - # In that case, we update the new delta that will be larger (because the element has been resized). + # In that case, we update the new delta that will be larger (because the element has + # been resized). delta+=changeEBMLElementSize(inputFile, pos, delta) def getFormat(ffprobe, inputFile): @@ -1813,7 +1893,8 @@ def getFormat(ffprobe, inputFile): infd = inputFile.fileno() lseek(infd, 0, SEEK_SET) set_inheritable(infd, True) - with Popen([ffprobe, '-loglevel', 'quiet', '-show_format', '-of', 'json', '-i', '/proc/self/fd/%d' % infd], stdout=PIPE, close_fds=False) as ffprobe: + with Popen([ffprobe, '-loglevel', 'quiet', '-show_format', '-of', 'json', '-i', + '/proc/self/fd/%d' % infd], stdout=PIPE, close_fds=False) as ffprobe: out, _ = ffprobe.communicate() out = json.load(BytesIO(out)) if 'format' in out: @@ -1830,7 +1911,8 @@ def getMovieDuration(ffprobe, inputFile): infd = inputFile.fileno() lseek(infd, 0, SEEK_SET) set_inheritable(infd, True) - with Popen([ffprobe, '-loglevel', 'quiet', '-show_format', '-of', 'json', '-i', '/proc/self/fd/%d' % infd], stdout=PIPE, close_fds=False) as ffprobe: + with Popen([ffprobe, '-loglevel', 'quiet', '-show_format', '-of', 'json', '-i', + '/proc/self/fd/%d' % infd], stdout=PIPE, close_fds=False) as ffprobe: out, _ = ffprobe.communicate() out = json.load(BytesIO(out)) if 'format' in out and 'duration' in out['format']: @@ -1842,14 +1924,16 @@ def getMovieDuration(ffprobe, inputFile): return None -# ffprobe -loglevel quiet -select_streams v:0 -show_entries stream=width,height -of json ./example.ts +# ffprobe -loglevel quiet -select_streams v:0 -show_entries stream=width,height -of json sample.ts def getVideoDimensions(ffprobe, inputFile): logger = logging.getLogger(__name__) infd = inputFile.fileno() lseek(infd, 0, SEEK_SET) set_inheritable(infd, True) - with Popen([ffprobe, '-loglevel', 'quiet', '-select_streams', 'v:0', '-show_entries', 'stream=width,height', '-of', 'json', '-i', '/proc/self/fd/%d' % infd], stdout=PIPE, close_fds=False) as ffprobe: + with Popen([ffprobe, '-loglevel', 'quiet', '-select_streams', 'v:0', '-show_entries',\ + 'stream=width,height', '-of', 'json', '-i', '/proc/self/fd/%d' % infd],\ + stdout=PIPE, close_fds=False) as ffprobe: out, _ = ffprobe.communicate() out = json.load(BytesIO(out)) if 'streams' in out: @@ -1867,7 +1951,8 @@ def getStreams(ffprobe, inputFile): infd = inputFile.fileno() lseek(infd, 0, SEEK_SET) set_inheritable(infd, True) - with Popen([ffprobe, '-loglevel', 'quiet', '-show_streams', '-of', 'json', '-i', '/proc/self/fd/%d' % infd], stdout=PIPE, close_fds=False) as ffprobe: + with Popen([ffprobe, '-loglevel', 'quiet', '-show_streams', '-of', 'json', '-i', + '/proc/self/fd/%d' % infd], stdout=PIPE, close_fds=False) as ffprobe: out, _ = ffprobe.communicate() out = json.load(BytesIO(out)) if 'streams' in out: @@ -1883,7 +1968,8 @@ def withSubtitles(ffprobe, inputFile): infd = inputFile.fileno() lseek(infd, 0, SEEK_SET) set_inheritable(infd, True) - with Popen([ffprobe, '-loglevel', 'quiet', '-show_streams', '-of', 'json', '-i', '/proc/self/fd/%d' % infd], stdout=PIPE, close_fds=False) as ffprobe: + with Popen([ffprobe, '-loglevel', 'quiet', '-show_streams', '-of', 'json', '-i', + '/proc/self/fd/%d' % infd], stdout=PIPE, close_fds=False) as ffprobe: out, _ = ffprobe.communicate() out = json.load(BytesIO(out)) if 'streams' in out: @@ -1899,11 +1985,12 @@ def withSubtitles(ffprobe, inputFile): def parseTimestamp(ts): logger = logging.getLogger(__name__) - tsRegExp = r'^(?P[0-9]{1,2}):(?P[0-9]{1,2}):(?P[0-9]{1,2})(\.(?P[0-9]{1,6}))?$' - p = re.compile(tsRegExp) + ts_reg_exp = (r'^(?P[0-9]{1,2}):(?P[0-9]{1,2})' + r':(?P[0-9]{1,2})(\.(?P[0-9]{1,6}))?$') + p = re.compile(ts_reg_exp) m = p.match(ts) if m is None: - logger.warning("Impossible to parse timestamp: %s" % ts) + logger.warning("Impossible to parse timestamp: %s", ts) return None values = m.groupdict() @@ -1939,8 +2026,10 @@ def parseTimestamp(ts): def parseTimeInterval(interval): logger = logging.getLogger(__name__) - intervalRegExp = r'^(?P[0-9]{1,2}):(?P[0-9]{1,2}):(?P[0-9]{1,2})(\.(?P[0-9]{1,3}))?-(?P[0-9]{1,2}):(?P[0-9]{1,2}):(?P[0-9]{1,2})(\.(?P[0-9]{1,3}))?$' - p = re.compile(intervalRegExp) + interval_reg_exp = (r'^(?P[0-9]{1,2}):(?P[0-9]{1,2}):(?P[0-9]{1,2})' + r'(\.(?P[0-9]{1,3}))?-(?P[0-9]{1,2}):(?P[0-9]{1,2})' + r':(?P[0-9]{1,2})(\.(?P[0-9]{1,3}))?$') + p = re.compile(interval_reg_exp) m = p.match(interval) if m is None: logger.error("Impossible to parse time interval") @@ -2034,11 +2123,13 @@ def ffmpegConvert(ffmpeg, ffprobe, inputFile, inputFormat, outputFile, outputFor else: log = [ '-loglevel', 'quiet' ] - params = [ffmpeg, '-y',]+log+['-progress', '/dev/stdout', '-canvas_size', '%dx%d' % (width, height), '-f', inputFormat, '-i', '/proc/self/fd/%d' % infd, - '-map', '0:v', '-map', '0:a'] + params = [ffmpeg, '-y',]+log+['-progress', '/dev/stdout', '-canvas_size', '%dx%d' % + (width, height), '-f', inputFormat, '-i', '/proc/self/fd/%d' % infd, '-map', '0:v', + '-map', '0:a'] if subtitles: params.extend(['-map', '0:s']) - params.extend(['-bsf:v', 'h264_mp4toannexb,dump_extra=freq=keyframe', '-vcodec', 'copy', '-acodec', 'copy']) + params.extend(['-bsf:v', 'h264_mp4toannexb,dump_extra=freq=keyframe', '-vcodec', 'copy', + '-acodec', 'copy']) if subtitles: params.extend(['-scodec', 'dvdsub']) params.extend(['-r:0', '25', '-f', outputFormat, '/proc/self/fd/%d' % outfd]) @@ -2046,7 +2137,8 @@ def ffmpegConvert(ffmpeg, ffprobe, inputFile, inputFormat, outputFile, outputFor logger.debug('Executing %s', params) with Popen(params, stdout=PIPE, close_fds=False) as ffmpeg: - pb = tqdm(TextIOWrapper(ffmpeg.stdout, encoding="utf-8"), total=int(duration/timedelta(seconds=1)), unit='s', desc='Conversion') + pb = tqdm(TextIOWrapper(ffmpeg.stdout, encoding="utf-8"), + total=int(duration/timedelta(seconds=1)), unit='s', desc='Conversion') for line in pb: if line.startswith('out_time='): ts = line.split('=')[1].strip() @@ -2090,8 +2182,9 @@ def getFramesInStream(ffprobe, inputFile, begin, end, streamKind, subStreamId=0) infd = inputFile.fileno() set_inheritable(infd, True) - command = [ffprobe, '-loglevel', 'quiet', '-read_intervals', ('%s%%%s' %(begin, end)), '-show_entries', 'frame', - '-select_streams', '%s:%d' % (streamKind, subStreamId), '-of', 'json', '/proc/self/fd/%d' % infd] + command = [ffprobe, '-loglevel', 'quiet', '-read_intervals', ('%s%%%s' %(begin, end)), + '-show_entries', 'frame', '-select_streams', '%s:%d' % (streamKind, subStreamId), + '-of', 'json', '/proc/self/fd/%d' % infd] logger.debug('Executing: %s', command) with Popen(command, stdout=PIPE, close_fds=False) as ffprobe: @@ -2122,8 +2215,9 @@ def getFramesInStream(ffprobe, inputFile, begin, end, streamKind, subStreamId=0) logger.error('Impossible to retrieve frames inside file around [%s,%s]' % (begin, end)) return None -# TODO: +# TODO: Finish implementation of this function and use it. def getNearestIDRFrame(ffprobe, inputFile, timestamp, before=True, delta=timedelta(seconds=2)): + # pylint: disable=W0613 logger = logging.getLogger(__name__) zero = timedelta() @@ -2140,14 +2234,16 @@ def getNearestIDRFrame(ffprobe, inputFile, timestamp, before=True, delta=timedel idrs = [] # Retains only IDR frame - with Popen([ffprobe, '-loglevel', 'quiet', '-read_intervals', ('%s%%%s' %(tbegin, tend)), '-skip_frame', 'nokey', '-show_entries', 'frame', '-select_streams', 'v:0', '-of', 'json', '/proc/self/fd/%d' % infd], stdout=PIPE, close_fds=False) as ffprobe: + with Popen([ffprobe, '-loglevel', 'quiet', '-read_intervals', ('%s%%%s' %(tbegin, tend)), + '-skip_frame', 'nokey', '-show_entries', 'frame', '-select_streams', 'v:0', '-of', + 'json', '/proc/self/fd/%d' % infd], stdout=PIPE, close_fds=False) as ffprobe: out, _ = ffprobe.communicate() frames = json.load(BytesIO(out)) status = ffprobe.wait() if status != 0: logger.error('ffprobe failed with status code: %d' % status) return None - res = [] + if 'frames' in frames: frames = frames['frames'] for frame in frames: @@ -2157,7 +2253,8 @@ def getNearestIDRFrame(ffprobe, inputFile, timestamp, before=True, delta=timedel if tbegin <= ts and ts <= tend: idrs.append(frame) else: - logger.error('Impossible to retrieve IDR frames inside file around [%s,%s]', tbegin, tend) + logger.error('Impossible to retrieve IDR frames inside file around [%s,%s]', + tbegin, tend) return None def getNearestIFrame(ffprobe, inputFile, timestamp, before=True, deltaMax=timedelta(seconds=15)): @@ -2183,7 +2280,8 @@ def getNearestIFrame(ffprobe, inputFile, timestamp, before=True, deltaMax=timede tbegin = zero logger.debug('Looking for an iframe in [%s, %s]', tbegin, tend) - frames = getFramesInStream(ffprobe, inputFile=inputFile, begin=tbegin, end=tend, streamKind='v') + frames = getFramesInStream(ffprobe, inputFile=inputFile, begin=tbegin, end=tend, + streamKind='v') if frames is None: logger.debug('Found no frame in [%s, %s]', tbegin, tend) delta+=timedelta(seconds=1) @@ -2252,11 +2350,13 @@ def extractMKVPart(mkvmerge, inputFile, outputFile, begin, end): env = {**os.environ, 'LANG': 'C'} warnings = [] - command = [mkvmerge, '-o', '/proc/self/fd/%d' % outfd, '--split', 'parts:%s-%s' % (begin, end), '/proc/self/fd/%d' % infd] - logger.debug('Executing: %s' % command) + command = [mkvmerge, '-o', '/proc/self/fd/%d' % outfd, '--split', 'parts:%s-%s' % (begin, end), + '/proc/self/fd/%d' % infd] + logger.debug('Executing: %s', command) with Popen(command, stdout=PIPE, close_fds=False, env=env) as mkvmerge: - pb = tqdm(TextIOWrapper(mkvmerge.stdout, encoding="utf-8"), total=100, unit='%', desc='Extraction') + pb = tqdm(TextIOWrapper(mkvmerge.stdout, encoding="utf-8"), total=100, unit='%', + desc='Extraction') for line in pb: if line.startswith('Progress :'): p = re.compile('^Progress : (?P[0-9]{1,3})%$') @@ -2293,8 +2393,9 @@ def extractPictures(ffmpeg, inputFile, begin, nbFrames, width=640, height=480): length = imageLength*nbFrames logger.debug("Estimated length: %d" % length) - command = [ffmpeg, '-loglevel', 'quiet' ,'-y', '-ss', '%s'%begin, '-i', '/proc/self/fd/%d' % infd, '-s', '%dx%d'%(width, height), - '-vframes', '%d'%nbFrames, '-c:v', 'ppm', '-f', 'image2pipe', '/proc/self/fd/%d' % outfd ] + command = [ffmpeg, '-loglevel', 'quiet' ,'-y', '-ss', '%s'%begin, '-i', '/proc/self/fd/%d' % + infd, '-s', '%dx%d'%(width, height), '-vframes', '%d'%nbFrames, '-c:v', 'ppm', + '-f', 'image2pipe', '/proc/self/fd/%d' % outfd ] logger.debug('Executing: %s', command) images = bytes() @@ -2313,7 +2414,8 @@ def extractPictures(ffmpeg, inputFile, begin, nbFrames, width=640, height=480): lseek(outfd, 0, SEEK_SET) return images, outfd -def extractSound(ffmpeg, inputFile, begin, outputFileName, packetDuration, subChannel=0, nbPackets=0, sampleRate=48000, nbChannels=2): +def extractSound(ffmpeg, inputFile, begin, outputFileName, packetDuration, subChannel=0, + nbPackets=0, sampleRate=48000, nbChannels=2): logger = logging.getLogger(__name__) outfd = memfd_create(outputFileName, flags=0) @@ -2324,8 +2426,10 @@ def extractSound(ffmpeg, inputFile, begin, outputFileName, packetDuration, subCh sound = bytes() length = int(nbChannels*sampleRate*4*nbPackets*packetDuration/1000) - command = [ffmpeg, '-y', '-loglevel', 'quiet', '-ss', '%s'%begin, '-i', '/proc/self/fd/%d' % infd, '-frames:a:%d' % subChannel, '%d' % (nbPackets+1), - '-c:a', 'pcm_s32le', '-sample_rate', '%d' % sampleRate, '-channels', '%d' % nbChannels, '-f', 's32le', '/proc/self/fd/%d' % outfd] + command = [ffmpeg, '-y', '-loglevel', 'quiet', '-ss', '%s'%begin, '-i', '/proc/self/fd/%d' % + infd, '-frames:a:%d' % subChannel, '%d' % (nbPackets+1), '-c:a', 'pcm_s32le', + '-sample_rate', '%d' % sampleRate, '-channels', '%d' % nbChannels, '-f', 's32le', + '/proc/self/fd/%d' % outfd] logger.debug('Executing: %s', command) with Popen(command, stdout=PIPE, close_fds=False) as ffmpeg: @@ -2338,7 +2442,9 @@ def extractSound(ffmpeg, inputFile, begin, outputFileName, packetDuration, subCh sound = read(outfd, length) if len(sound) != length: - logger.info("Received %d bytes but %d were expected (channels=%d, freq=%d, packets=%d, duration=%d ms)." % (len(sound), length, nbChannels, sampleRate, nbPackets, packetDuration)) + logger.info("Received %d bytes but %d were expected (channels=%d, freq=%d, packets=%d,\ + duration=%d ms).", len(sound), length, nbChannels, sampleRate, nbPackets, + packetDuration) return None, None return sound, outfd @@ -2357,7 +2463,7 @@ def dumpPPM(pictures, prefix, temporaries): header = BytesIO(pictures[pos:]) magic = header.readline().decode('utf8') dimensions = header.readline().decode('utf8') - maxvalue = header.readline().decode('utf8') + max_value = header.readline().decode('utf8') if magic == 'P6\n': pattern = re.compile('^(?P[0-9]+) (?P[0-9]+)\n$') m = pattern.match(dimensions) @@ -2371,6 +2477,10 @@ def dumpPPM(pictures, prefix, temporaries): logger.error('Not a PPM picture') return + if max_value != 255: + logger.error('Not a valid PPM picture. Color are not encoded on byte. Max value: %d', + max_value) + headerLen=2+1+ceil(log(width, 10))+1+ceil(log(height, 10))+1+3+1 try: out = open(filename, 'w') @@ -2387,7 +2497,8 @@ def dumpPPM(pictures, prefix, temporaries): picture+=1 -def extractAllStreams(ffmpeg, ffprobe, inputFile, begin, end, streams, filesPrefix, nbFrames, frameRate, width, height, temporaries, dumpMemFD=False): +def extractAllStreams(ffmpeg, ffprobe, inputFile, begin, end, streams, filesPrefix, nbFrames, + frameRate, width, height, temporaries, dumpMemFD=False): logger = logging.getLogger(__name__) # The command line for encoding only video track @@ -2417,26 +2528,37 @@ def extractAllStreams(ffmpeg, ffprobe, inputFile, begin, end, streams, filesPref colorPrimaries = stream['color_primaries'] level = int(stream['level']) level = '%d.%d' % (floor(level/10), level%10) - chromaLocation = stream['chroma_location'] + chroma_location = stream['chroma_location'] fieldOrder = stream interlacedOptions = [] if fieldOrder == 'progressive': interlacedOptions = ['-field_order', '0'] elif fieldOrder == 'tt': - interlacedOptions = ['-top', '1', '-flags:v:%d' % videoID, '+ilme+ildct', '-field_order', '1'] + interlacedOptions = ['-top', '1', '-flags:v:%d' % videoID, '+ilme+ildct', + '-field_order', '1'] elif fieldOrder == 'bb': - interlacedOptions = ['-top', '0', '-flags:v:%d' % videoID, '+ilme+ildct', '-field_order','2'] + interlacedOptions = ['-top', '0', '-flags:v:%d' % videoID, '+ilme+ildct', + '-field_order','2'] elif fieldOrder == 'tb': - interlacedOptions = ['-top', '1', '-flags:v:%d' % videoID, '+ilme+ildct', '-field_order', '3'] + interlacedOptions = ['-top', '1', '-flags:v:%d' % videoID, '+ilme+ildct', + '-field_order', '3'] elif fieldOrder == 'bt': - interlacedOptions = ['-top', '0', '-flags:v:%d' % videoID, '+ilme+ildct', '-field_order', '4'] + interlacedOptions = ['-top', '0', '-flags:v:%d' % videoID, '+ilme+ildct', + '-field_order', '4'] - # ======================================= # - # TODO: adjust SAR and DAR - # https://superuser.com/questions/907933/correct-aspect-ratio-without-re-encoding-video-file + # ======================================= # + # TODO: adjust SAR and DAR + # https://superuser.com/questions/907933/correct-aspect-ratio-without-re-encoding-video-file + # SAR: -aspect width:height + # DAR: -bsf:v sample_aspect_ratio=1:video_format + logger.warning('Missing SAR adjustment for: %s', sar) + logger.warning('Missing DAR adjustment for: %s', dar) + logger.warning('Missing treatment for chroma location: %s', chroma_location) codec = stream['codec_name'] - imagesBytes, memfd = extractPictures(ffmpeg, inputFile=inputFile, begin=begin, nbFrames=nbFrames, width=width, height=height) + imagesBytes, memfd = extractPictures(ffmpeg, inputFile=inputFile, begin=begin, + nbFrames=nbFrames, width=width, height=height) if imagesBytes is None: + logger.error('Impossible to extract picture from video stream.') exit(-1) memfds.append(memfd) @@ -2447,10 +2569,15 @@ def extractAllStreams(ffmpeg, ffprobe, inputFile, begin, end, streams, filesPref lseek(memfd, 0, SEEK_SET) set_inheritable(memfd, True) - videoInputParams.extend(['-framerate', '%f'%frameRate, '-f', 'image2pipe', '-i', '/proc/self/fd/%d' % memfd]) - videoCodecParams.extend(['-c:v:%d' % videoID, codec, '-level:v:%d' % videoID, level, '-pix_fmt', pixelFormat]) + videoInputParams.extend(['-framerate', '%f'%frameRate, '-f', 'image2pipe', '-i', + '/proc/self/fd/%d' % memfd]) + videoCodecParams.extend(['-c:v:%d' % videoID, codec, '-level:v:%d' % videoID, level, + '-pix_fmt', pixelFormat]) videoCodecParams.extend(interlacedOptions) - videoCodecParams.extend(['-colorspace:v:%d' % videoID, colorSpace, '-color_primaries:v:%d' % videoID, colorPrimaries, '-color_trc:v:%d' % videoID, colorTransfer, '-color_range:v:%d' % videoID, colorRange]) + videoCodecParams.extend(['-colorspace:v:%d' % videoID, colorSpace, + '-color_primaries:v:%d' % videoID, colorPrimaries, + '-color_trc:v:%d' % videoID, colorTransfer, + '-color_range:v:%d' % videoID, colorRange]) videoID=videoID+1 elif stream['codec_type'] == 'audio': logger.debug('Audio stream: %s' % stream) @@ -2463,8 +2590,10 @@ def extractAllStreams(ffmpeg, ffprobe, inputFile, begin, end, streams, filesPref codec = stream['codec_name'] if 'tags' in stream: if 'language' in stream['tags']: - genericCodecParams.extend(['-metadata:s:a:%d' % audioID, 'language=%s' % stream['tags']['language']]) - packets = getFramesInStream(ffprobe, inputFile=inputFile, begin=begin, end=end, streamKind='a', subStreamId=audioID) + genericCodecParams.extend(['-metadata:s:a:%d' % audioID, 'language=%s' % + stream['tags']['language']]) + packets = getFramesInStream(ffprobe, inputFile=inputFile, begin=begin, end=end, + streamKind='a', subStreamId=audioID) nbPackets = len(packets) logger.debug("Found %d packets to be extracted from audio track.", nbPackets) if nbPackets > 0: @@ -2477,7 +2606,10 @@ def extractAllStreams(ffmpeg, ffprobe, inputFile, begin, end, streams, filesPref logger.info("Extracting %d packets of audio stream: a:%d" , nbPackets, audioID) tmpname = '%s-%d.pcm' % (filesPrefix,audioID) - soundBytes, memfd = extractSound(ffmpeg=ffmpeg, inputFile=inputFile, begin=begin, nbPackets=nbPackets, packetDuration=packetDuration, outputFileName=tmpname, sampleRate=sampleRate, nbChannels=nbChannels) + soundBytes, memfd = extractSound(ffmpeg=ffmpeg, inputFile=inputFile, begin=begin, + nbPackets=nbPackets, packetDuration=packetDuration, + outputFileName=tmpname, sampleRate=sampleRate, + nbChannels=nbChannels) if soundBytes is None: exit(-1) @@ -2501,8 +2633,10 @@ def extractAllStreams(ffmpeg, ffprobe, inputFile, begin, end, streams, filesPref lseek(memfd, 0, SEEK_SET) set_inheritable(memfd, True) - genericInputParams.extend(['-f', 's32le', '-ar', '%d'%sampleRate, '-ac', '%d'%nbChannels, '-i', '/proc/self/fd/%d' % memfd]) - genericCodecParams.extend(['-c:a:%d' % audioID, codec, '-b:a:%d' % audioID, '%d' % bitRate]) + genericInputParams.extend(['-f', 's32le', '-ar', '%d'%sampleRate, '-ac', + '%d'%nbChannels, '-i', '/proc/self/fd/%d' % memfd]) + genericCodecParams.extend(['-c:a:%d' % audioID, codec, '-b:a:%d' % audioID, + '%d' % bitRate]) audioID=audioID+1 elif stream['codec_type'] == 'subtitle': logger.info("Extracting a subtitle stream: s:%d" % subTitleID) @@ -2510,7 +2644,8 @@ def extractAllStreams(ffmpeg, ffprobe, inputFile, begin, end, streams, filesPref genericInputParams.extend(['-i', './empty.idx']) if 'tags' in stream: if 'language' in stream['tags']: - genericCodecParams.extend(['-metadata:s:s:%d' % subTitleID, 'language=%s' % stream['tags']['language']]) + genericCodecParams.extend(['-metadata:s:s:%d' % subTitleID, 'language=%s' % + stream['tags']['language']]) genericCodecParams.extend(['-c:s:%d' % subTitleID, 'copy']) subTitleID=subTitleID+1 else: @@ -2558,8 +2693,10 @@ def extractAllStreams(ffmpeg, ffprobe, inputFile, begin, end, streams, filesPref videoEncoderParams.extend(videoCodecParams) videoEncoderParams.extend([ '-x264opts', 'keyint=1:sps-id=%d' % 1,'-bsf:v', - 'h264_mp4toannexb,dump_extra=freq=keyframe,h264_metadata=overscan_appropriate_flag=1:sample_aspect_ratio=1:video_format=0:chroma_sample_loc_type=0', - '-f', 'h264', '/proc/self/fd/%d' % h264outfd]) + 'h264_mp4toannexb,dump_extra=freq=keyframe,h264_metadata=\ + overscan_appropriate_flag=1:sample_aspect_ratio=1:video_format=\ + 0:chroma_sample_loc_type=0','-f', 'h264', + '/proc/self/fd/%d' % h264outfd]) logger.info('Encoding video into a H264 file: %s' % h264FileName) logger.debug('Executing: %s' % videoEncoderParams) @@ -2580,7 +2717,7 @@ def extractAllStreams(ffmpeg, ffprobe, inputFile, begin, end, streams, filesPref h264TSOutput.write('# timestamp format v2\n') ts = 0 - for frame in range(0,nbFrames): + for _ in range(0,nbFrames): ts = ts+ceil(1000/frameRate) h264TSOutput.write('%d\n' % ts) h264TSOutput.flush() @@ -2614,7 +2751,8 @@ def mergeMKVs(mkvmerge, inputs, outputName, concatenate=True, timestamps=None): set_inheritable(outfd, True) # Timestamps of merged tracks are modified by the length of the preceding track. - # The default mode ('file') is using the largest timestamp of the whole file which may create desynchronize video and sound. + # The default mode ('file') is using the largest timestamp of the whole file which may create + # desynchronize video and sound. mergeParams = [mkvmerge, '--append-mode', 'track'] first = True @@ -2648,7 +2786,8 @@ def mergeMKVs(mkvmerge, inputs, outputName, concatenate=True, timestamps=None): logger.debug('Executing: LANG=C %s', mergeParams) with Popen(mergeParams, stdout=PIPE, close_fds=False, env=env) as mkvmerge: - pb = tqdm(TextIOWrapper(mkvmerge.stdout, encoding="utf-8"), total=100, unit='%', desc='Merging') + pb = tqdm(TextIOWrapper(mkvmerge.stdout, encoding="utf-8"), total=100, unit='%', + desc='Merging') for line in pb: if line.startswith('Progress :'): p = re.compile('^Progress : (?P[0-9]{1,3})%$') @@ -2680,7 +2819,8 @@ def findSubtitlesTracks(ffprobe, inputFile): lseek(infd, 0, SEEK_SET) set_inheritable(infd, True) - command = [ffprobe, '-loglevel','quiet', '-i', '/proc/self/fd/%d' % infd, '-select_streams', 's', '-show_entries', 'stream=index:stream_tags=language', '-of', 'json'] + command = [ffprobe, '-loglevel','quiet', '-i', '/proc/self/fd/%d' % infd, '-select_streams', + 's', '-show_entries', 'stream=index:stream_tags=language', '-of', 'json'] logger.debug('Executing: %s', command) with Popen(command, stdout=PIPE, close_fds=False) as ffprobe: @@ -2708,13 +2848,15 @@ def extractTrackFromMKV(mkvextract, inputFile, index, outputFile, timestamps): lseek(tsfd, 0, SEEK_SET) set_inheritable(tsfd, True) - params = [ mkvextract, '/proc/self/fd/%d' % infd, 'tracks', '%d:/proc/self/fd/%d' % (index, outfd), 'timestamps_v2', '%d:/proc/self/fd/%d' % (index, tsfd)] + params = [ mkvextract, '/proc/self/fd/%d' % infd, 'tracks', '%d:/proc/self/fd/%d' % + (index, outfd), 'timestamps_v2', '%d:/proc/self/fd/%d' % (index, tsfd)] env = {**os.environ, 'LANG': 'C'} logger.debug('Executing: LANG=C %s' % params) with Popen(params, stdout=PIPE, close_fds=False, env=env) as extract: - pb = tqdm(TextIOWrapper(extract.stdout, encoding="utf-8"), total=100, unit='%', desc='Extraction of track') + pb = tqdm(TextIOWrapper(extract.stdout, encoding="utf-8"), total=100, unit='%', + desc='Extraction of track') for line in pb: if line.startswith('Progress :'): p = re.compile('^Progress : (?P[0-9]{1,3})%$') @@ -2749,7 +2891,8 @@ def removeVideoTracksFromMKV(mkvmerge, inputFile, outputFile): env = {**os.environ, 'LANG': 'C'} with Popen(params, stdout=PIPE, close_fds=False, env=env) as remove: - pb = tqdm(TextIOWrapper(remove.stdout, encoding="utf-8"), total=100, unit='%', desc='Removal of video track:') + pb = tqdm(TextIOWrapper(remove.stdout, encoding="utf-8"), total=100, unit='%', + desc='Removal of video track:') for line in pb: if line.startswith('Progress :'): p = re.compile('^Progress : (?P[0-9]{1,3})%$') @@ -2796,7 +2939,8 @@ def remuxSRTSubtitles(mkvmerge, inputFile, outputFileName, subtitles): env = {**os.environ, 'LANG': 'C'} logger.info('Remux subtitles: %s' % mkvmergeParams) with Popen(mkvmergeParams, stdout=PIPE, close_fds=False, env=env) as mkvmerge: - pb = tqdm(TextIOWrapper(mkvmerge.stdout, encoding="utf-8"), total=100, unit='%', desc='Remux subtitles:') + pb = tqdm(TextIOWrapper(mkvmerge.stdout, encoding="utf-8"), total=100, unit='%', + desc='Remux subtitles:') for line in pb: if line.startswith('Progress :'): p = re.compile('^Progress : (?P[0-9]{1,3})%$') @@ -2873,7 +3017,9 @@ def concatenateH264TSParts(h264TSParts, output): first = False # TODO: finish this procedure -def doCoarseProcessing(ffmpeg, ffprobe, mkvmerge, inputFile, begin, end, nbFrames, frameRate, filesPrefix, streams, width, height, temporaries, dumpMemFD): +def doCoarseProcessing(ffmpeg, ffprobe, mkvmerge, inputFile, begin, end, nbFrames, frameRate, + filesPrefix, streams, width, height, temporaries, dumpMemFD): + # pylint: disable=W0613 logger = logging.getLogger(__name__) # Internal video with all streams (video, audio and subtitles) @@ -2886,7 +3032,8 @@ def doCoarseProcessing(ffmpeg, ffprobe, mkvmerge, inputFile, begin, end, nbFrame exit(-1) # Extract internal part of MKV - extractMKVPart(mkvmerge=mkvmerge, inputFile=inputFile, outputFile=internalMKV, begin=begin, end=end) + extractMKVPart(mkvmerge=mkvmerge, inputFile=inputFile, outputFile=internalMKV, begin=begin, + end=end) temporaries.append(internalMKV) @@ -2896,16 +3043,28 @@ def main(): logger = logging.getLogger(__name__) coloredlogs.install() parser = argparse.ArgumentParser() - parser.add_argument("-i", "--input", dest='inputFile', type=str, required=True, help="Input file to process (can be .ts, .mp4 or .mkv).") - parser.add_argument("-o", "--output", dest='outputFile', type=str, required=True, help="Output MKV file to produce.") - parser.add_argument("-p", "--part", dest='parts', nargs='+', required=False, action='append', metavar="hh:mm:ss[.mmm]-hh:mm:ss[.mmm]", help="Extract this exact part of the original file.") - parser.add_argument("-k", "--keep", action='store_true', help="Do not cleanup temporary files after processing.") - parser.add_argument("-t", "--threshold", action='store', type=int, help="Suppress headers and trailers that are smaller than the threshold.") - parser.add_argument("-c", "--coarse", action='store_true', dest='coarse', help="Do not take trailers and headers into account.") - parser.add_argument("--dump-memory", action='store_true', dest='dump', help="For debug purpose, dump all memory mapping of headers (and trailers) before (after) each part. They are kept in memory only otherwise.") - parser.add_argument("-s","--srt", action='store_true', dest='srt', help="Dump subtitles and make OCR and finally remux them in the movie (as SRT).") + parser.add_argument("-i", "--input", dest='inputFile', type=str, required=True, + help="Input file to process (can be .ts, .mp4 or .mkv).") + parser.add_argument("-o", "--output", dest='outputFile', type=str, required=True, + help="Output MKV file to produce.") + parser.add_argument("-p", "--part", dest='parts', nargs='+', required=False, action='append', + metavar="hh:mm:ss[.mmm]-hh:mm:ss[.mmm]", + help="Extract this exact part of the original file.") + parser.add_argument("-k", "--keep", action='store_true', + help="Do not cleanup temporary files after processing.") + parser.add_argument("-t", "--threshold", action='store', type=int, + help="Suppress headers and trailers that are smaller than the threshold.") + parser.add_argument("-c", "--coarse", action='store_true', dest='coarse', + help="Do not take trailers and headers into account.") + parser.add_argument("--dump-memory", action='store_true', dest='dump', + help="For debug purpose, dump all memory mapping of headers (and trailers)\ + before (after) each part. They are kept in memory only otherwise.") + parser.add_argument("-s","--srt", action='store_true', dest='srt', + help="Dump subtitles and make OCR and finally remux them in the movie\ + (as SRT).") parser.add_argument("-v","--verbose", action='store_true', dest='verbose', help="Debug.") - parser.add_argument("-f","--framerate", action='store', type=int, help="Override frame rate estimator.") + parser.add_argument("-f","--framerate", action='store', type=int, + help="Override frame rate estimator.") args = parser.parse_args() logger.info('Arguments: %s' % args) @@ -2923,7 +3082,7 @@ def main(): if (not args.coarse) and args.threshold is None: args.threshold = 0 - allOptionalTools, paths = checkRequiredTools() + all_optional_tools, paths = checkRequiredTools() # Flatten args.parts intervals = [] @@ -2999,7 +3158,8 @@ def main(): logger.info("Converting TS to MP4 (to fix timestamps).") try: with open(mp4filename, 'w+') as mp4: - ffmpegConvert(paths['ffmpeg'], paths['ffprobe'], inputFile, 'mpegts', mp4, 'mp4', duration) + ffmpegConvert(paths['ffmpeg'], paths['ffprobe'], inputFile, 'mpegts', mp4, 'mp4', + duration) temporaries.append(mp4) logger.info("Converting MP4 to MKV.") try: @@ -3007,7 +3167,8 @@ def main(): except IOError: logger.error('') - ffmpegConvert(paths['ffmpeg'], paths['ffprobe'], mp4, 'mp4', mkv, 'matroska', duration) + ffmpegConvert(paths['ffmpeg'], paths['ffprobe'], mp4, 'mp4', mkv, 'matroska', + duration) if nbParts > 0: temporaries.append(mkv) except IOError: @@ -3019,7 +3180,8 @@ def main(): mkv = open(mkvfilename, 'w+') except IOError: logger.error('') - ffmpegConvert(paths['ffmpeg'], paths['ffprobe'], inputFile, 'mp4', mkv, 'matroska', duration) + ffmpegConvert(paths['ffmpeg'], paths['ffprobe'], inputFile, 'mp4', mkv, 'matroska', + duration) if nbParts > 0: temporaries.append(mkv) else: @@ -3052,24 +3214,28 @@ def main(): # We retrieve the main private codec data _, mainCodecPrivateData = getCodecPrivateDataFromMKV(mkvinfo=paths['mkvinfo'], inputFile=mkv) - logger.debug('Main video stream has following private data: %s' % hexdump.dump(mainCodecPrivateData, sep=':')) + logger.debug('Main video stream has following private data: %s', + hexdump.dump(mainCodecPrivateData, sep=':')) # We parse them mainAvcConfig = parseCodecPrivate(mainCodecPrivateData) - logger.debug('AVC configuration: %s' % mainAvcConfig) + logger.debug('AVC configuration: %s', mainAvcConfig) # We check if the parse and dump operations are idempotent. privateData = dumpCodecPrivateData(mainAvcConfig) logger.debug('Redump AVC configuration: %s' % hexdump.dump(privateData, sep=':')) - # In rare occasion, the PPS has trailing zeroes that do not seem to be related to useful data but they differ from the private data we generate that do not contain them. - # In that case we try to redecode our own private data to see if both AVC configurations are the same. + # In rare occasion, the PPS has trailing zeroes that do not seem to be related to useful data + # but they differ from the private data we generate that do not contain them. + # In that case we try to redecode our own private data to see if both AVC configurations are + # the same. if mainCodecPrivateData != privateData: logger.warning('Difference detected in bitstream !!') isoAvcConfig = parseCodecPrivate(privateData) logger.debug('Reread AVC configuration: %s' % isoAvcConfig) - # If there exists a difference between our own reconstructed AVC configuration and the original one, we abandon + # If there exists a difference between our own reconstructed AVC configuration and the + # original one, we abandon if isoAvcConfig != mainAvcConfig: - logger.error('AVC configurations are different: %s\n%s\n' % (mainAvcConfig, isoAvcConfig)) + logger.error('AVC configurations are different: %s\n%s\n', mainAvcConfig, isoAvcConfig) exit(-1) # Pour chaque portion @@ -3083,8 +3249,11 @@ def main(): otherAvcConfigs = [] for ts1, ts2 in parts: - # Trouver l'estampille de la trame 'I' la plus proche (mais postérieure) au début de la portion. - # Trouver l'estampille de la trame 'I' la plus proche (mais antérieure) à la fin de la portion. + # TODO: translate comment in english + # Trouver l'estampille de la trame 'I' la plus proche (mais postérieure) au début + # de la portion. + # Trouver l'estampille de la trame 'I' la plus proche (mais antérieure) à la fin + # de la portion. # On a alors # debut ----- trame --------- trame --------- fin fin+1 # 'B/P' 'B/P'* 'I' 'I' 'B/P'* 'B/P' 'I/B/P' @@ -3109,8 +3278,10 @@ def main(): nbHeadFrames, headIFrame = headFrames nbTailFrames, tailIFrame = tailFrames - logger.info("Found %d frames between beginning of current part and first I-frame", nbHeadFrames) - logger.info("Found %d frames between last I-frame and end of current part", nbTailFrames) + logger.info("Found %d frames between beginning of current part and first I-frame", + nbHeadFrames) + logger.info("Found %d frames between last I-frame and end of current part", + nbTailFrames) headIFrameTS = getTSFrame(headIFrame) if headIFrameTS is None: @@ -3126,13 +3297,33 @@ def main(): # TODO: separate pipeline processing between coarse and not fine grain options. # if args.coarse: - # doCoarseProcessing(ffmpeg=paths['ffmpeg'], ffprobe=paths['ffprobe'], inputFile=mkv, begin=ts1, end=headIFrameTS, nbFrames=nbHeadFrames-1, frameRate=frameRate, filesPrefix='part-%d-head' % (partnum), streams=streams, width=width, height=height, temporaries=temporaries, dumpMemFD=args.dump) + # doCoarseProcessing(ffmpeg=paths['ffmpeg'], ffprobe=paths['ffprobe'], inputFile=mkv, + # begin=ts1, end=headIFrameTS, nbFrames=nbHeadFrames-1, + # frameRate=frameRate, filesPrefix='part-%d-head' % (partnum), + # streams=streams, width=width, height=height, + # temporaries=temporaries, dumpMemFD=args.dump) # else: - # doFineGrainProcessing(ffmpeg=paths['ffmpeg'], ffprobe=paths['ffprobe'], inputFile=mkv, begin=ts1, end=headIFrameTS, nbFrames=nbHeadFrames-1, frameRate=frameRate, filesPrefix='part-%d-head' % (partnum), streams=streams, width=width, height=height, temporaries=temporaries, dumpMemFD=args.dump) + # doFineGrainProcessing(ffmpeg=paths['ffmpeg'], ffprobe=paths['ffprobe'], + # inputFile=mkv, begin=ts1, end=headIFrameTS, + # nbFrames=nbHeadFrames-1, frameRate=frameRate, + # filesPrefix='part-%d-head' % (partnum), streams=streams, + # width=width, height=height, temporaries=temporaries, + # dumpMemFD=args.dump) if (not args.coarse) and (nbHeadFrames > args.threshold): - # We extract all frames between the beginning upto the frame that immediately preceeds the I-frame. - h264Head, h264HeadTS, mkvHead = extractAllStreams(ffmpeg=paths['ffmpeg'], ffprobe=paths['ffprobe'], inputFile=mkv, begin=ts1, end=headIFrameTS, nbFrames=nbHeadFrames-1, frameRate=frameRate, filesPrefix='part-%d-head' % (partnum), streams=streams, width=width, height=height, temporaries=temporaries, dumpMemFD=args.dump) + # We extract all frames between the beginning upto the frame that immediately preceeds + # the I-frame. + h264Head, h264HeadTS, mkvHead = extractAllStreams(ffmpeg=paths['ffmpeg'], + ffprobe=paths['ffprobe'], + inputFile=mkv, begin=ts1, + end=headIFrameTS, + nbFrames=nbHeadFrames-1, + frameRate=frameRate, + filesPrefix='part-%d-head'%(partnum), + streams=streams, width=width, + height=height, + temporaries=temporaries, + dumpMemFD=args.dump) # If we are not at an exact boundary: if mkvHead is not None: @@ -3180,15 +3371,18 @@ def main(): # logger.info('Merge header, middle and trailer subpart into: %s' % internalMKVName) # Extract internal part of MKV - extractMKVPart(mkvmerge=paths['mkvmerge'], inputFile=mkv, outputFile=internalMKV, begin=headIFrameTS, end=tailIFrameTS) + extractMKVPart(mkvmerge=paths['mkvmerge'], inputFile=mkv, outputFile=internalMKV, + begin=headIFrameTS, end=tailIFrameTS) # Extract video stream of internal part as a raw H264 and its timestamps. logger.info('Extract video track as raw H264 file.') - extractTrackFromMKV(mkvextract=paths['mkvextract'], inputFile=internalMKV, index=0, outputFile=internalH264, timestamps=internalH264TS) + extractTrackFromMKV(mkvextract=paths['mkvextract'], inputFile=internalMKV, index=0, + outputFile=internalH264, timestamps=internalH264TS) # Remove video track from internal part of MKV logger.info('Remove video track from %s', internalMKVName) - removeVideoTracksFromMKV(mkvmerge=paths['mkvmerge'], inputFile=internalMKV, outputFile=internalNoVideoMKV) + removeVideoTracksFromMKV(mkvmerge=paths['mkvmerge'], inputFile=internalMKV, + outputFile=internalNoVideoMKV) temporaries.append(internalMKV) temporaries.append(internalH264) @@ -3201,7 +3395,16 @@ def main(): if (not args.coarse) and (nbTailFrames > args.threshold): # We extract all frames between the I-frame (including it) upto the end. - h264Tail, h264TailTS, mkvTail = extractAllStreams(ffmpeg=paths['ffmpeg'], ffprobe=paths['ffprobe'], inputFile=mkv, begin=tailIFrameTS, end=ts2, nbFrames=nbTailFrames, frameRate=frameRate, filesPrefix='part-%d-tail' % (partnum), streams=streams, width=width, height=height, temporaries=temporaries, dumpMemFD=args.dump) + h264Tail, h264TailTS, mkvTail = extractAllStreams(ffmpeg=paths['ffmpeg'], + ffprobe=paths['ffprobe'], + inputFile=mkv, begin=tailIFrameTS, + end=ts2, nbFrames=nbTailFrames, + frameRate=frameRate, + filesPrefix='part-%d-tail'%(partnum), + streams=streams, + width=width, height=height, + temporaries=temporaries, + dumpMemFD=args.dump) if mkvTail is not None: subparts.append(mkvTail) @@ -3213,7 +3416,8 @@ def main(): h264TS.append(h264TailTS) logger.info('Merging MKV: %s', subparts) - part = mergeMKVs(mkvmerge=paths['mkvmerge'], inputs=subparts, outputName="part-%d.mkv" % partnum, concatenate=True) + part = mergeMKVs(mkvmerge=paths['mkvmerge'], inputs=subparts, outputName="part-%d.mkv" % + partnum, concatenate=True) mkvparts.append(part) temporaries.append(part) @@ -3254,7 +3458,8 @@ def main(): if nbMKVParts > 1: logger.info('Merging all audio and subtitles parts: %s' % mkvparts) - mergeMKVs(mkvmerge=paths['mkvmerge'], inputs=mkvparts, outputName=finalNoVideoName, concatenate=True) + mergeMKVs(mkvmerge=paths['mkvmerge'], inputs=mkvparts, outputName=finalNoVideoName, + concatenate=True) elif nbMKVParts == 1: copyfile('part-1.mkv', finalNoVideoName) else: @@ -3273,14 +3478,16 @@ def main(): fullH264TS.seek(0) logger.info('Merging final video track and all other tracks together') - finalWithVideo = mergeMKVs(mkvmerge=paths['mkvmerge'], inputs=[fullH264, finalNoVideo], outputName=finalWithVideoName, concatenate=False, timestamps={0: fullH264TS}) + finalWithVideo = mergeMKVs(mkvmerge=paths['mkvmerge'], inputs=[fullH264, finalNoVideo], + outputName=finalWithVideoName, concatenate=False, + timestamps={0: fullH264TS}) finalCodecPrivateData = dumpCodecPrivateData(mainAvcConfig) logger.debug('Final codec private data: %s' % hexdump.dump(finalCodecPrivateData, sep=':')) logger.info('Changing codec private data with the new one.') changeCodecPrivateData(paths['mkvinfo'], finalWithVideo, finalCodecPrivateData) if args.srt: - if not allOptionalTools: + if not all_optional_tools: logger.warning("Missing tools for extracting subtitles.") move(finalWithVideoName, args.outputFile) else: @@ -3303,24 +3510,27 @@ def main(): else: sts[lang] = [index] else: - logger.error("Dropping subtitle: %s because it is missing language indication") + logger.error("Dropping subtitle: %s because it is missing language\ + indication") else: - logger.error("Dropping subtitle: %s because it is missing language indication") + logger.error("Dropping subtitle: %s because it is missing language indication", + subtitle) logger.info(sts) if len(sts) > 0: - listOfSubtitles = extractSRT(paths['mkvextract'], finalWithVideoName, sts, supportedLangs) + listOfSubtitles = extractSRT(paths['mkvextract'], finalWithVideoName, sts, + supportedLangs) logger.info(listOfSubtitles) - for idxName, subName, _, _ in listOfSubtitles: + for idx_name, sub_name, _, _ in listOfSubtitles: try: - idx = open(idxName,'r') + idx = open(idx_name,'r') except IOError: - logger.error("Impossible to open %s.", idxName) + logger.error("Impossible to open %s.", idx_name) exit(-1) try: - sub = open(subName,'r') + sub = open(sub_name,'r') except IOError: - logger.error("Impossible to open %s.", subName) + logger.error("Impossible to open %s.", sub_name) exit(-1) temporaries.append(idx)